diff --git a/mm/vma.c b/mm/vma.c index 0325b3979313..af8e65ada23b 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -783,6 +783,27 @@ static void vmg_adjust_set_range(struct vma_merge_struct *vmg) } } +static void propagate_sticky_flags(struct vma_merge_struct *vmg, + const struct vm_area_struct *vma) +{ + const vma_flags_t sticky = + vma_flags_and_mask(&vma->flags, VMA_STICKY_FLAGS); + + /* Write lock prevents concurrent atomic VMA flag updates. */ + vma_assert_write_locked(vma); + vma_flags_set_mask(&vmg->vma_flags, sticky); +} + +static void commit_sticky_flags(struct vma_merge_struct *vmg) +{ + const vma_flags_t sticky = + vma_flags_and_mask(&vmg->vma_flags, VMA_STICKY_FLAGS); + struct vm_area_struct *target = vmg->target; + + vma_assert_write_locked(target); + vma_set_flags_mask(target, sticky); +} + /* * Actually perform the VMA merge operation. * @@ -818,6 +839,8 @@ static int commit_merge(struct vma_merge_struct *vmg) return -ENOMEM; vma_prepare(&vp); + /* Must be made visible before vma_complete() for rmap walkers. */ + commit_sticky_flags(vmg); /* * THP pages may need to do additional splits if we increase * middle->vm_start. @@ -857,8 +880,8 @@ static bool can_merge_remove_vma(struct vm_area_struct *vma) * merge. * * As part of this operation, if a merge occurs, the @vmg object will have its - * vma, start, end, and pgoff fields modified to execute the merge. Subsequent - * calls to this function should reset these fields. + * vma, start, end, pgoff and vma_flags fields modified to execute the merge. + * Subsequent calls to this function should reset these fields. * * Returns: The merged VMA if merge succeeds, or NULL otherwise. * @@ -872,8 +895,6 @@ static bool can_merge_remove_vma(struct vm_area_struct *vma) static __must_check struct vm_area_struct *vma_merge_existing_range( struct vma_merge_struct *vmg) { - vma_flags_t sticky_flags = vma_flags_and_mask(&vmg->vma_flags, - VMA_STICKY_FLAGS); struct vm_area_struct *middle = vmg->middle; struct vm_area_struct *prev = vmg->prev; struct vm_area_struct *next; @@ -966,24 +987,18 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( /* No matter what happens, we will be adjusting middle. */ vma_start_write(middle); + propagate_sticky_flags(vmg, middle); if (merge_right) { - vma_flags_t next_sticky; - vma_start_write(next); vmg->target = next; - next_sticky = vma_flags_and_mask(&next->flags, VMA_STICKY_FLAGS); - vma_flags_set_mask(&sticky_flags, next_sticky); + propagate_sticky_flags(vmg, next); } if (merge_left) { - vma_flags_t prev_sticky; - vma_start_write(prev); vmg->target = prev; - - prev_sticky = vma_flags_and_mask(&prev->flags, VMA_STICKY_FLAGS); - vma_flags_set_mask(&sticky_flags, prev_sticky); + propagate_sticky_flags(vmg, prev); } if (merge_both) { @@ -1054,7 +1069,6 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( if (err || commit_merge(vmg)) goto abort; - vma_set_flags_mask(vmg->target, sticky_flags); khugepaged_enter_vma(vmg->target, vmg->vm_flags); vmg->state = VMA_MERGE_SUCCESS; return vmg->target; @@ -1241,16 +1255,11 @@ int vma_expand(struct vma_merge_struct *vmg) struct vm_area_struct *target = vmg->target; struct vm_area_struct *next = vmg->next; bool remove_next = false; - vma_flags_t sticky_flags = - vma_flags_and_mask(&vmg->vma_flags, VMA_STICKY_FLAGS); - vma_flags_t target_sticky; int ret = 0; mmap_assert_write_locked(vmg->mm); vma_start_write(target); - target_sticky = vma_flags_and_mask(&target->flags, VMA_STICKY_FLAGS); - if (next && target != next && vmg->end == next->vm_end) remove_next = true; @@ -1265,7 +1274,7 @@ int vma_expand(struct vma_merge_struct *vmg) VM_WARN_ON_VMG(target->vm_start < vmg->start || target->vm_end > vmg->end, vmg); - vma_flags_set_mask(&sticky_flags, target_sticky); + propagate_sticky_flags(vmg, target); /* * If we are removing the next VMA or copying from a VMA @@ -1282,18 +1291,14 @@ int vma_expand(struct vma_merge_struct *vmg) return ret; if (remove_next) { - vma_flags_t next_sticky; - vma_start_write(next); vmg->__remove_next = true; - next_sticky = vma_flags_and_mask(&next->flags, VMA_STICKY_FLAGS); - vma_flags_set_mask(&sticky_flags, next_sticky); + propagate_sticky_flags(vmg, next); } if (commit_merge(vmg)) goto nomem; - vma_set_flags_mask(target, sticky_flags); return 0; nomem: diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index a13977405482..dc4927d26296 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -1212,7 +1212,7 @@ static inline void vma_assert_detached(struct vm_area_struct *vma) WARN_ON_ONCE(vma_is_attached(vma)); } -static inline void vma_assert_write_locked(struct vm_area_struct *); +static inline void vma_assert_write_locked(const struct vm_area_struct *); static inline void vma_mark_attached(struct vm_area_struct *vma) { vma_assert_write_locked(vma); diff --git a/tools/testing/vma/include/stubs.h b/tools/testing/vma/include/stubs.h index f786a585caf0..8d030a3ace84 100644 --- a/tools/testing/vma/include/stubs.h +++ b/tools/testing/vma/include/stubs.h @@ -320,7 +320,7 @@ static inline void anon_vma_lock_write(struct anon_vma *anon_vma) { } -static inline void vma_assert_write_locked(struct vm_area_struct *vma) +static inline void vma_assert_write_locked(const struct vm_area_struct *vma) { }