github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm: implement and use vma_has_anon_rmap(), silence KCSAN

Browse Source 
Provide a function to abstract the common task of checking whether
a VMA has an anonymous reverse mapping associated with it.

If the VMA is attached, a VMA or mmap lock must be held when calling this
function.

For an attached, anonymous, VMA:

          Transition          |              VMA/mmap Lock state
 -----------------------------|-------------------------------------------
   No anon rmap to anon rmap  | Write lock/read lock + mm->page_table_lock
   Anon rmap to no anon rmap  | Write lock

vma_has_anon_rmap() never provides a false positive (the lock precludes
it), but if only a read lock is held, a negative result must be re-checked
with mm->page_table_lock held.

A VMA obtains an anonymous reverse mapping when first faulted or forked
and it is removed when it is freed.

Detached VMAs cannot be concurrently manipulated as they are removed from
the maple tree so require no guarantees.

Use data_race() to silence KCSAN about non-existent data races between
concurrent vma->anon_vma read/write on optimistic fault tests.

Update the core VMA merge/split, rmap, mremap, KSM, fork,
khugepaged and fault preparation callers which test vma->anon_vma directly
to use vma_has_anon_rmap() instead.

Finally, update comments that reference anon_vma to reference the anon rmap
instead.

Since the lockless read in reusable_anon_vma() is doing more than checking
whether the VMA has anon rmap - it is returning the anon_vma to be used on
fault - do not alter it.

There is one odd one out - file_backed_vma_is_retractable() - which holds
neither a VMA nor mmap lock and is stabilised by the file rmap lock only,
so simply add a comment to explain why it's necessary.

Link: https://lore.kernel.org/20260917-vma-is-faulted-v3-2-5c22314a72e7@kernel.org
Signed-off-by: Lorenzo Stoakes (ARM) <ljs@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Reported-by: Guilherme Giacomo Simoes <trintaeoitogc@gmail.com>
Closes: https://lore.kernel.org/all/20260829100034.423064-1-trintaeoitogc@gmail.com/
Closes: https://lore.kernel.org/all/20260909115723.528501-1-trintaeoitogc@gmail.com/
Reviewed-by: Pedro Falcato <pfalcato@suse.de>
Reviewed-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
Acked-by: David Hildenbrand (Arm) <david@kernel.org>
Reviewed-by: Lance Yang <lance.yang@linux.dev>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Baoquan He <baoquan.he@linux.dev>
Cc: Barry Song <baohua@kernel.org>
Cc: Byungchul Park <byungchul@sk.com>
Cc: Chengming Zhou <chengming.zhou@linux.dev>
Cc: Chris Li <chrisl@kernel.org>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Gregory Price <gourry@gourry.net>
Cc: Harry Yoo <harry@kernel.org>
Cc: "Huang, Ying" <ying.huang@linux.alibaba.com>
Cc: Jann Horn <jannh@google.com>
Cc: Joshua Hahn <joshua.hahnjy@gmail.com>
Cc: Kairui Song <kasong@tencent.com>
Cc: Kemeng Shi <shikemeng@huaweicloud.com>
Cc: Liam R. Howlett <liam@infradead.org>
Cc: Matthew Brost <matthew.brost@intel.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Peter Xu <peterx@redhat.com>
Cc: Rakie Kim <rakie.kim@sk.com>
Cc: Rik van Riel <riel@surriel.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@kernel.org>
Cc: xu xin <xu.xin16@zte.com.cn>
Cc: Zi Yan <ziy@nvidia.com>
This commit is contained in:
Lorenzo Stoakes (ARM) 2026-09-17 20:47:42 +01:00 committed by Andrew Morton
parent 6a9ef02c60
commit 38564710f4
14 changed files with 79 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mm/huge_memory.c
View File

@ -258,7 +258,7 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
* Allow page fault since anon_vma may be not initialized until
* the first page fault.
*/
if (!vma->anon_vma)
if (!vma_has_anon_rmap(vma))
return (smaps || in_pf) ? orders : 0;
return orders;
@ -2173,7 +2173,7 @@ vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf)
pmd_t orig_pmd = vmf->orig_pmd;
vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);
VM_BUG_ON_VMA(!vma->anon_vma, vma);
VM_BUG_ON_VMA(!vma_has_anon_rmap(vma), vma);
if (is_huge_zero_pmd(orig_pmd)) {
vm_fault_t ret = do_huge_zero_wp_pmd(vmf);

2
mm/internal.h
View File

@ -338,7 +338,7 @@ void unlink_anon_vmas(struct vm_area_struct *vma);
static inline int anon_vma_prepare(struct vm_area_struct *vma)
{
if (likely(vma->anon_vma))
if (likely(vma_has_anon_rmap(vma)))
return 0;
return __anon_vma_prepare(vma);

5
mm/khugepaged.c
View File

@ -1094,7 +1094,7 @@ static enum scan_result hugepage_vma_revalidate(struct mm_struct *mm, unsigned l
* thp_vma_allowable_orders() may return true for qualified file
* vmas.
*/
if (expect_anon && (!(*vmap)->anon_vma || !vma_is_anonymous(*vmap)))
if (expect_anon && (!vma_has_anon_rmap(vma) || !vma_is_anonymous(vma)))
return SCAN_PAGE_ANON;
return SCAN_SUCCEED;
}
@ -2137,6 +2137,9 @@ static bool file_backed_vma_is_retractable(struct vm_area_struct *vma)
* Check vma->anon_vma to exclude MAP_PRIVATE mappings that
* got written to. These VMAs are likely not worth removing
* page tables from, as PMD-mapping is likely to be split later.
*
* Can't use vma_has_anon_rmap() here as the VMA may be stabilised
* by the file rmap lock.
*/
if (READ_ONCE(vma->anon_vma))
return false;

10
mm/ksm.c
View File

@ -778,7 +778,7 @@ static struct vm_area_struct *find_mergeable_vma(struct mm_struct *mm,
if (ksm_test_exit(mm))
return NULL;
vma = vma_lookup(mm, addr);
if (!vma || !(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
if (!vma || !(vma->vm_flags & VM_MERGEABLE) || !vma_has_anon_rmap(vma))
return NULL;
return vma;
}
@ -1242,7 +1242,7 @@ static int unmerge_and_remove_all_rmap_items(void)
goto mm_exiting;
for_each_vma(vmi, vma) {
if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
if (!(vma->vm_flags & VM_MERGEABLE) || !vma_has_anon_rmap(vma))
continue;
err = break_ksm(vma, vma->vm_start, vma->vm_end, false);
if (err)
@ -2692,7 +2692,7 @@ static struct ksm_rmap_item *scan_get_next_rmap_item(struct page **page)
continue;
if (ksm_scan.address < vma->vm_start)
ksm_scan.address = vma->vm_start;
if (!vma->anon_vma)
if (!vma_has_anon_rmap(vma))
ksm_scan.address = vma->vm_end;
while (ksm_scan.address < vma->vm_end) {
@ -2883,7 +2883,7 @@ static int __ksm_del_vma(struct vm_area_struct *vma)
if (!(vma->vm_flags & VM_MERGEABLE))
return 0;
if (vma->anon_vma) {
if (vma_has_anon_rmap(vma)) {
err = break_ksm(vma, vma->vm_start, vma->vm_end, true);
if (err)
return err;
@ -3035,7 +3035,7 @@ int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
if (!(*vm_flags & VM_MERGEABLE))
return 0; /* just ignore the advice */
if (vma->anon_vma) {
if (vma_has_anon_rmap(vma)) {
err = break_ksm(vma, start, end, true);
if (err)
return err;

4
mm/madvise.c
View File

@ -1164,7 +1164,7 @@ static long madvise_guard_install(struct madvise_behavior *madv_behavior)
* as part of the VMA lock logic.
*/
if (vma_is_anonymous(vma)) {
VM_WARN_ON_ONCE(!vma->anon_vma &&
VM_WARN_ON_ONCE(!vma_has_anon_rmap(vma) &&
madv_behavior->lock_mode != MADVISE_MMAP_READ_LOCK);
err = anon_vma_prepare(vma);
@ -1630,7 +1630,7 @@ static bool is_vma_lock_sufficient(struct vm_area_struct *vma,
* check overly paranoid which is safe.
*/
if (vma_is_anonymous(vma) &&
prepares_anon_vma(madv_behavior->behavior) && !vma->anon_vma)
prepares_anon_vma(madv_behavior->behavior) && !vma_has_anon_rmap(vma))
return false;
return true;

4
mm/memory.c
View File

@ -1536,7 +1536,7 @@ vma_needs_copy(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
* The presence of an anon_vma indicates an anonymous VMA has page
* tables which naturally cannot be reconstituted on page fault.
*/
if (src_vma->anon_vma)
if (vma_has_anon_rmap(src_vma))
return true;
/*
@ -4015,7 +4015,7 @@ vm_fault_t __vmf_anon_prepare(struct vm_fault *vmf)
struct vm_area_struct *vma = vmf->vma;
vm_fault_t ret = 0;
if (likely(vma->anon_vma))
if (likely(vma_has_anon_rmap(vma)))
return 0;
if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
if (!mmap_read_trylock(vma->vm_mm))

2
mm/mprotect.c
View File

@ -816,7 +816,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
vma_flags_set(&new_vma_flags, VMA_ACCOUNT_BIT);
}
} else if (vma_flags_test(&old_vma_flags, VMA_ACCOUNT_BIT) &&
vma_is_anonymous(vma) && !vma->anon_vma) {
vma_is_anonymous(vma) && !vma_has_anon_rmap(vma)) {
vma_flags_clear(&new_vma_flags, VMA_ACCOUNT_BIT);
}

4
mm/mremap.c
View File

@ -144,13 +144,13 @@ static void take_rmap_locks(struct vm_area_struct *vma)
{
if (vma->vm_file)
i_mmap_lock_write(vma->vm_file->f_mapping);
if (vma->anon_vma)
if (vma_has_anon_rmap(vma))
anon_vma_lock_write(vma->anon_vma);
}
static void drop_rmap_locks(struct vm_area_struct *vma)
{
if (vma->anon_vma)
if (vma_has_anon_rmap(vma))
anon_vma_unlock_write(vma->anon_vma);
if (vma->vm_file)
i_mmap_unlock_write(vma->vm_file->f_mapping);

22
mm/rmap.c
View File

@ -208,7 +208,7 @@ int __anon_vma_prepare(struct vm_area_struct *vma)
anon_vma_lock_write(anon_vma);
/* page_table_lock to protect against threads */
spin_lock(&mm->page_table_lock);
if (likely(!vma->anon_vma)) {
if (likely(!vma_has_anon_rmap(vma))) {
/*
* Make anon_vma fields visible before anon_vma is published.
* Paired with an address dependency in reusable_anon_vma().
@ -246,21 +246,21 @@ static void check_anon_vma_clone(struct vm_area_struct *dst,
VM_WARN_ON_ONCE(operation != VMA_OP_FORK && dst->vm_mm != src->vm_mm);
/* If we have anything to do src->anon_vma must be provided. */
VM_WARN_ON_ONCE(!src->anon_vma && !list_empty(&src->anon_vma_chain));
VM_WARN_ON_ONCE(!src->anon_vma && dst->anon_vma);
VM_WARN_ON_ONCE(!vma_has_anon_rmap(src) && !list_empty(&src->anon_vma_chain));
VM_WARN_ON_ONCE(!vma_has_anon_rmap(src) && vma_has_anon_rmap(dst));
/* We are establishing a new anon_vma_chain. */
VM_WARN_ON_ONCE(!list_empty(&dst->anon_vma_chain));
/*
* On fork, dst->anon_vma is set NULL (temporarily). Otherwise, anon_vma
* must be the same across dst and src.
*/
VM_WARN_ON_ONCE(dst->anon_vma && dst->anon_vma != src->anon_vma);
VM_WARN_ON_ONCE(vma_has_anon_rmap(dst) && dst->anon_vma != src->anon_vma);
/*
* Essentially equivalent to above - if not a no-op, we should expect
* dst->anon_vma to be set for everything except a fork.
*/
VM_WARN_ON_ONCE(operation != VMA_OP_FORK && src->anon_vma &&
!dst->anon_vma);
VM_WARN_ON_ONCE(operation != VMA_OP_FORK && vma_has_anon_rmap(src) &&
!vma_has_anon_rmap(dst));
/* For the anon_vma to be compatible, it can only be singular. */
VM_WARN_ON_ONCE(operation == VMA_OP_MERGE_UNFAULTED &&
!list_is_singular(&src->anon_vma_chain));
@ -273,7 +273,7 @@ static void maybe_reuse_anon_vma(struct vm_area_struct *dst,
struct anon_vma *anon_vma)
{
/* If already populated, nothing to do.*/
if (dst->anon_vma)
if (vma_has_anon_rmap(dst))
return;
/*
@ -327,7 +327,7 @@ int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src,
check_anon_vma_clone(dst, src, operation);
if (!active_anon_vma)
if (!vma_has_anon_rmap(src))
return 0;
/*
@ -384,7 +384,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
int rc;
/* Don't bother if the parent process has no anon_vma here. */
if (!pvma->anon_vma)
if (!vma_has_anon_rmap(pvma))
return 0;
/* Drop inherited anon_vma, we'll reuse existing or allocate new. */
@ -405,7 +405,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
*/
rc = anon_vma_clone(vma, pvma, VMA_OP_FORK);
/* An error arose or an existing anon_vma was reused, all done then. */
if (rc || vma->anon_vma) {
if (rc || vma_has_anon_rmap(vma)) {
put_anon_vma(anon_vma);
anon_vma_chain_free(avc);
return rc;
@ -864,7 +864,7 @@ unsigned long page_address_in_vma(const struct folio *folio,
* Note: swapoff's unuse_vma() is more efficient with this
* check, and needs it to match anon_vma when KSM is active.
*/
if (!vma->anon_vma || !anon_vma ||
if (!vma_has_anon_rmap(vma) || !anon_vma ||
vma->anon_vma->root != anon_vma->root)
return -EFAULT;
/* KSM folios don't reach here because of the !anon_vma check */

2
mm/swapfile.c
View File

@ -2707,7 +2707,7 @@ static int unuse_mm(struct mm_struct *mm, unsigned int type)
if (check_stable_address_space(mm))
goto unlock;
for_each_vma(vmi, vma) {
if (vma->anon_vma && !vma_is_hugetlb(vma)) {
if (vma_has_anon_rmap(vma) && !vma_is_hugetlb(vma)) {
ret = unuse_vma(vma, type);
if (ret)
break;

2
mm/userfaultfd.c
View File

@ -145,7 +145,7 @@ static struct vm_area_struct *uffd_lock_vma(struct mm_struct *mm,
* We know we're going to need to use anon_vma, so check
* that early.
*/
if (!(vma->vm_flags & VM_SHARED) && unlikely(!vma->anon_vma))
if (!(vma->vm_flags & VM_SHARED) && unlikely(!vma_has_anon_rmap(vma)))
vma_end_read(vma);
else
return vma;

29
mm/vma.c
View File

@ -100,7 +100,8 @@ static bool vma_is_fork_child(struct vm_area_struct *vma)
* parents. This can improve scalability caused by the anon_vma root
* lock.
*/
return vma && vma->anon_vma && !list_is_singular(&vma->anon_vma_chain);
return vma && vma_has_anon_rmap(vma) &&
!list_is_singular(&vma->anon_vma_chain);
}
static inline bool is_mergeable_vma(struct vma_merge_struct *vmg, bool merge_next)
@ -140,7 +141,7 @@ static bool is_mergeable_anon_vma(struct vma_merge_struct *vmg, bool merge_next)
VM_WARN_ON(src && src_anon != src->anon_vma);
/* Case 1 - we will dup_anon_vma() from src into tgt. */
if (!tgt_anon && src_anon) {
if (!vma_has_anon_rmap(tgt) && src_anon) {
struct vm_area_struct *copied_from = vmg->copied_from;
if (vma_is_fork_child(src))
@ -151,7 +152,7 @@ static bool is_mergeable_anon_vma(struct vma_merge_struct *vmg, bool merge_next)
return true;
}
/* Case 2 - we will simply use tgt's anon_vma. */
if (tgt_anon && !src_anon)
if (vma_has_anon_rmap(tgt) && !src_anon)
return !vma_is_fork_child(tgt);
/* Case 3 - the anon_vma's are already shared. */
return src_anon == tgt_anon;
@ -190,10 +191,10 @@ static void init_multi_vma_prep(struct vma_prepare *vp,
adjust = NULL;
vp->adj_next = adjust;
if (!vp->anon_vma && adjust)
if (!vma_has_anon_rmap(vma) && adjust)
vp->anon_vma = adjust->anon_vma;
VM_WARN_ON(vp->anon_vma && adjust && adjust->anon_vma &&
VM_WARN_ON(vma_has_anon_rmap(vma) && adjust && vma_has_anon_rmap(adjust) &&
vp->anon_vma != adjust->anon_vma);
vp->file = vma->vm_file;
@ -430,7 +431,7 @@ static void vma_complete(struct vma_prepare *vp, struct vma_iterator *vmi,
vp->remove->vm_end);
fput(vp->file);
}
if (vp->remove->anon_vma)
if (vma_has_anon_rmap(vp->remove))
unlink_anon_vmas(vp->remove);
mm->map_count--;
mpol_put(vma_policy(vp->remove));
@ -500,7 +501,7 @@ static bool can_vma_merge_right(struct vma_merge_struct *vmg,
* We therefore check this in addition to mergeability to either side.
*/
prev = vmg->prev;
return !prev->anon_vma || !next->anon_vma ||
return !vma_has_anon_rmap(prev) || !vma_has_anon_rmap(next) ||
prev->anon_vma == next->anon_vma;
}
@ -670,7 +671,7 @@ static int dup_anon_vma(struct vm_area_struct *dst,
* that is it is unfaulted, we need to ensure that the newly merged
* range is referenced by the anon_vma's of the source.
*/
if (src->anon_vma && !dst->anon_vma) {
if (vma_has_anon_rmap(src) && !vma_has_anon_rmap(dst)) {
int ret;
vma_assert_write_locked(dst);
@ -720,7 +721,7 @@ void validate_mm(struct mm_struct *mm)
}
#ifdef CONFIG_DEBUG_VM_RB
if (anon_vma) {
if (vma_has_anon_rmap(vma)) {
anon_vma_lock_read(anon_vma);
list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
anon_rmap_tree_verify(avc);
@ -1020,7 +1021,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range(
* simply a case of, if prev has no anon_vma object, which of
* next or middle contains the anon_vma we must duplicate.
*/
err = dup_anon_vma(prev, next->anon_vma ? next : middle,
err = dup_anon_vma(prev, vma_has_anon_rmap(next) ? next : middle,
&anon_dup);
} else if (merge_left) {
/*
@ -1960,7 +1961,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
* If a vma has not yet been faulted, update its anonymous pgoff to
* match the new location to increase its chance of merging.
*/
if (!vma->anon_vma) {
if (!vma_has_anon_rmap(vma)) {
anon_pgoff = addr >> PAGE_SHIFT;
if (vma_is_anonymous(vma)) {
@ -2387,7 +2388,7 @@ int mm_take_all_locks(struct mm_struct *mm)
for_each_vma(vmi, vma) {
if (signal_pending(current))
goto out_unlock;
if (vma->anon_vma)
if (vma_has_anon_rmap(vma))
list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
vm_lock_anon_vma(mm, avc->anon_vma);
}
@ -2449,7 +2450,7 @@ void mm_drop_all_locks(struct mm_struct *mm)
BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
for_each_vma(vmi, vma) {
if (vma->anon_vma)
if (vma_has_anon_rmap(vma))
list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
vm_unlock_anon_vma(avc->anon_vma);
if (vma->vm_file && vma->vm_file->f_mapping)
@ -3610,7 +3611,7 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
* Similarly in do_mmap and in do_brk_flags.
*/
if (vma_is_anonymous(vma)) {
WARN_ON_ONCE(vma->anon_vma);
WARN_ON_ONCE(vma_has_anon_rmap(vma));
vma_set_pgoff(vma, vma->vm_start >> PAGE_SHIFT);
}
vma_set_anon_pgoff(vma, vma->vm_start >> PAGE_SHIFT);

29
mm/vma.h
View File

@ -255,6 +255,33 @@ static inline pgoff_t vmg_end_pgoff(const struct vma_merge_struct *vmg)
return vmg_start_pgoff(vmg) + vmg_pages(vmg);
}
/**
* vma_has_anon_rmap() - does @vma possess an anonymous reverse mapping?
* @vma: The VMA to be checked.
*
* If the VMA is attached, a VMA or mmap lock must be held.
*
* This state is only possible for CoW mappings, see the comment for
* vma_flags_is_cow_mapping() for details.
*
* Importantly, a VMA which possesses an anonymous rmap may map anonymous
* folios.
*
* This function will not result in a false positive.
*
* However, if only a read lock is held, it may give a false negative, in which
* case it should be re-checked with mm->page_table_lock held.
*
* Returns: true if @vma has an anonymous reverse mapping, otherwise false.
*/
static inline bool vma_has_anon_rmap(const struct vm_area_struct *vma)
{
if (vma_is_attached(vma))
vma_assert_stabilised(vma);
/* KCSAN gets confused about the optimistic check. Silence it. */
return data_race(vma->anon_vma);
}
static inline void assert_sane_pgoff(struct vm_area_struct *vma, pgoff_t pgoff)
{
/* nommu doesn't set a virtual pgoff for anon VMAs. */
@ -268,7 +295,7 @@ static inline void assert_sane_pgoff(struct vm_area_struct *vma, pgoff_t pgoff)
if (!vma_is_anonymous(vma))
return;
/* If faulted in, could have been remapped. */
if (vma->anon_vma)
if (vma_has_anon_rmap(vma))
return;
/* OK this is really an anon VMA - expect virtual page offset. */
VM_WARN_ON_ONCE(pgoff != vma->vm_start >> PAGE_SHIFT);

4
tools/testing/vma/include/stubs.h
View File

@ -302,6 +302,10 @@ static inline void vma_assert_write_locked(struct vm_area_struct *vma)
{
}
static inline void vma_assert_stabilised(const struct vm_area_struct *vma)
{
}
static inline void ksm_add_vma(struct vm_area_struct *vma)
{
}