diff --git a/include/linux/mm.h b/include/linux/mm.h index 9f167950926a..6af3b7008f1a 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -602,7 +602,7 @@ enum { #define VMA_LOCKED_MASK mk_vma_flags(VMA_LOCKED_BIT, VMA_LOCKONFAULT_BIT) /* These flags can be updated atomically via VMA/mmap read lock. */ -#define VM_ATOMIC_SET_ALLOWED VM_MAYBE_GUARD +#define VMA_ATOMIC_SET_ALLOWED mk_vma_flags(VMA_MAYBE_GUARD_BIT) /* Arch-specific flags to clear when updating VM flags on protection change */ #ifndef VM_ARCH_CLEAR @@ -987,22 +987,6 @@ static inline void vm_flags_reset(struct vm_area_struct *vma, vm_flags_init(vma, flags); } -static inline void vma_flags_reset_once(struct vm_area_struct *vma, - vma_flags_t *flags) -{ - const unsigned long word = flags->__vma_flags[0]; - - /* It is assumed only the first system word must be written once. */ - vma_flags_overwrite_word_once(&vma->flags, word); - /* The remainder can be copied normally. */ - if (NUM_VMA_FLAG_BITS > BITS_PER_LONG) { - unsigned long *dst = &vma->flags.__vma_flags[1]; - const unsigned long *src = &flags->__vma_flags[1]; - - bitmap_copy(dst, src, NUM_VMA_FLAG_BITS - BITS_PER_LONG); - } -} - static inline void vm_flags_set(struct vm_area_struct *vma, vm_flags_t flags) { @@ -1039,48 +1023,6 @@ static inline void vm_flags_mod(struct vm_area_struct *vma, __vm_flags_mod(vma, set, clear); } -static __always_inline bool __vma_atomic_valid_flag(struct vm_area_struct *vma, - vma_flag_t bit) -{ - const vm_flags_t mask = BIT((__force int)bit); - - /* Only specific flags are permitted */ - if (WARN_ON_ONCE(!(mask & VM_ATOMIC_SET_ALLOWED))) - return false; - - return true; -} - -/* - * Set VMA flag atomically. Requires only VMA/mmap read lock. Only specific - * valid flags are allowed to do this. - */ -static __always_inline void vma_set_atomic_flag(struct vm_area_struct *vma, - vma_flag_t bit) -{ - unsigned long *bitmap = vma->flags.__vma_flags; - - vma_assert_stabilised(vma); - if (__vma_atomic_valid_flag(vma, bit)) - set_bit((__force int)bit, bitmap); -} - -/* - * Test for VMA flag atomically. Requires no locks. Only specific valid flags - * are allowed to do this. - * - * This is necessarily racey, so callers must ensure that serialisation is - * achieved through some other means, or that races are permissible. - */ -static __always_inline bool vma_test_atomic_flag(struct vm_area_struct *vma, - vma_flag_t bit) -{ - if (__vma_atomic_valid_flag(vma, bit)) - return test_bit((__force int)bit, &vma->vm_flags); - - return false; -} - /* Set an individual VMA flag in flags, non-atomically. */ static __always_inline void vma_flags_set_flag(vma_flags_t *flags, vma_flag_t bit) @@ -1302,33 +1244,23 @@ static __always_inline vma_flags_t vma_flags_diff_pair(const vma_flags_t *flags, return dst; } -/* Determine if flags and flags_other have precisely the same flags set. */ -static __always_inline bool vma_flags_same_pair(const vma_flags_t *flags, - const vma_flags_t *flags_other) -{ - const unsigned long *bitmap = flags->__vma_flags; - const unsigned long *bitmap_other = flags_other->__vma_flags; - - return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS); -} - -/* Determine if flags and flags_other have precisely the same flags set. */ -static __always_inline bool vma_flags_same_mask(const vma_flags_t *flags, - vma_flags_t flags_other) -{ - const unsigned long *bitmap = flags->__vma_flags; - const unsigned long *bitmap_other = flags_other.__vma_flags; - - return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS); -} - /* - * Helper macro to determine if only the specific flags are set, e.g.: - * - * if (vma_flags_same(&flags, VMA_WRITE_BIT) { ... } + * Determine if flags and flags_other have precisely the same flags set, + * excluding atomic VMA flags. */ -#define vma_flags_same(flags, ...) \ - vma_flags_same_mask(flags, mk_vma_flags(__VA_ARGS__)) +static __always_inline bool vma_flags_same_pair(const vma_flags_t *flags, + const vma_flags_t *flags_other) +{ + vma_flags_t diff = vma_flags_diff_pair(flags, flags_other); + + /* + * Atomic VMA flags may be set concurrently under the VMA read lock, so + * are disregarded for equality testing purposes. + */ + vma_flags_clear_mask(&diff, VMA_ATOMIC_SET_ALLOWED); + + return vma_flags_empty(&diff); +} /* * Test whether a specific flag in the VMA is set, e.g.: @@ -1436,6 +1368,78 @@ static __always_inline void vma_clear_flags_mask(struct vm_area_struct *vma, #define vma_clear_flags(vma, ...) \ vma_clear_flags_mask(vma, mk_vma_flags(__VA_ARGS__)) +static __always_inline bool __vma_atomic_valid_flag(struct vm_area_struct *vma, + vma_flag_t bit) +{ + const vma_flags_t mask = VMA_ATOMIC_SET_ALLOWED; + + /* Is this flag allowed to be set atomically? */ + if (WARN_ON_ONCE(!vma_flags_test(&mask, bit))) + return false; + + return true; +} + +/* + * Set VMA flag atomically. Requires only VMA/mmap read lock. Only specific + * valid flags are allowed to do this. + */ +static __always_inline void vma_set_atomic_flag(struct vm_area_struct *vma, + vma_flag_t bit) +{ + unsigned long *bitmap = vma->flags.__vma_flags; + + vma_assert_stabilised(vma); + if (__vma_atomic_valid_flag(vma, bit)) + set_bit((__force int)bit, bitmap); +} + +/* + * Test for VMA flag atomically. Requires no locks. Only specific valid flags + * are allowed to do this. + * + * This is necessarily racey, so callers must ensure that serialisation is + * achieved through some other means, or that races are permissible. + */ +static __always_inline bool vma_test_atomic_flag(struct vm_area_struct *vma, + vma_flag_t bit) +{ + if (__vma_atomic_valid_flag(vma, bit)) + return test_bit((__force int)bit, &vma->vm_flags); + + return false; +} + +/* + * Overwrite the VMA's flags with new ones, preserving atomic VMA flags in the + * process. The write is done once and once only. + */ +static inline void vma_overwrite_flags(struct vm_area_struct *vma, + vma_flags_t new_flags) +{ + vma_flags_t atomic; + + vma_assert_write_locked(vma); + + /* + * Preserve the VMA's atomic flags, which are kept stable by the VMA + * write lock. + */ + atomic = vma_flags_and_mask(&vma->flags, VMA_ATOMIC_SET_ALLOWED); + vma_flags_clear_mask(&new_flags, VMA_ATOMIC_SET_ALLOWED); + vma_flags_set_mask(&new_flags, atomic); + + /* It is assumed only the first system word must be written once. */ + vma_flags_overwrite_word_once(&vma->flags, new_flags.__vma_flags[0]); + /* The remainder can be copied normally. */ + if (NUM_VMA_FLAG_BITS > BITS_PER_LONG) { + unsigned long *dst = &vma->flags.__vma_flags[1]; + const unsigned long *src = &new_flags.__vma_flags[1]; + + bitmap_copy(dst, src, NUM_VMA_FLAG_BITS - BITS_PER_LONG); + } +} + /* * Test whether a specific VMA flag is set in a VMA descriptor, e.g.: * diff --git a/mm/madvise.c b/mm/madvise.c index 75b409e97dbf..b2e00efcaa54 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -158,7 +158,7 @@ static int madvise_update_vma(vm_flags_t new_flags, bool set_new_anon_name = madv_behavior->behavior == __MADV_SET_ANON_VMA_NAME; VMA_ITERATOR(vmi, madv_behavior->mm, range->start); - if (vma_flags_same_mask(&vma->flags, new_vma_flags) && + if (vma_flags_same_pair(&vma->flags, &new_vma_flags) && (!set_new_anon_name || anon_vma_name_eq(anon_vma_name(vma), anon_name))) return 0; @@ -177,7 +177,7 @@ static int madvise_update_vma(vm_flags_t new_flags, /* vm_flags is protected by the mmap_lock held in write mode. */ vma_start_write(vma); - vma->flags = new_vma_flags; + vma_overwrite_flags(vma, new_vma_flags); /* * If the vma become good for khugepaged to scan, * register it here without waiting a page fault that diff --git a/mm/mlock.c b/mm/mlock.c index 67ae9b18b923..1c70a7829394 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -432,14 +432,14 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma, } vma_start_write(vma); - vma_flags_reset_once(vma, &walk_flags); + vma_overwrite_flags(vma, walk_flags); lru_add_drain(); walk_page_range_vma(vma, start, end, &mlock_walk_ops, (void *)lock); lru_add_drain(); if (lock) - vma_flags_reset_once(vma, new_vma_flags); + vma_overwrite_flags(vma, *new_vma_flags); } /* @@ -494,7 +494,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT)) { /* No work to do, and mlocking twice would be wrong */ vma_start_write(vma); - vma->flags = *new_vma_flags; + vma_overwrite_flags(vma, *new_vma_flags); } else { mlock_vma_pages_range(vma, start, end, new_vma_flags); } diff --git a/mm/mprotect.c b/mm/mprotect.c index 754333698c1e..e18d4e12fc34 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -833,7 +833,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, * held in write mode. */ vma_start_write(vma); - vma_flags_reset_once(vma, &new_vma_flags); + vma_overwrite_flags(vma, new_vma_flags); if (vma_wants_manual_pte_write_upgrade(vma)) mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE; vma_set_page_prot(vma); diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index e0eab4eca998..a13977405482 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -366,6 +366,9 @@ enum { #define VMA_LOCKED_MASK mk_vma_flags(VMA_LOCKED_BIT, VMA_LOCKONFAULT_BIT) +/* These flags can be updated atomically via VMA/mmap read lock. */ +#define VMA_ATOMIC_SET_ALLOWED mk_vma_flags(VMA_MAYBE_GUARD_BIT) + #define RLIMIT_STACK 3 /* max stack size */ #define RLIMIT_MEMLOCK 8 /* max locked-in-memory address space */ @@ -895,22 +898,6 @@ static inline void vm_flags_reset(struct vm_area_struct *vma, vm_flags_init(vma, flags); } -static inline void vma_flags_reset_once(struct vm_area_struct *vma, - vma_flags_t *flags) -{ - const unsigned long word = flags->__vma_flags[0]; - - /* It is assumed only the first system word must be written once. */ - vma_flags_overwrite_word_once(&vma->flags, word); - /* The remainder can be copied normally. */ - if (NUM_VMA_FLAG_BITS > BITS_PER_LONG) { - unsigned long *dst = &vma->flags.__vma_flags[1]; - const unsigned long *src = &flags->__vma_flags[1]; - - bitmap_copy(dst, src, NUM_VMA_FLAG_BITS - BITS_PER_LONG); - } -} - static inline void vm_flags_set(struct vm_area_struct *vma, vm_flags_t flags) { @@ -1037,27 +1024,24 @@ static __always_inline vma_flags_t vma_flags_diff_pair(const vma_flags_t *flags, return dst; } +/* + * Determine if flags and flags_other have precisely the same flags set, + * excluding atomic VMA flags. + */ static __always_inline bool vma_flags_same_pair(const vma_flags_t *flags, - const vma_flags_t *flags_other) + const vma_flags_t *flags_other) { - const unsigned long *bitmap = flags->__vma_flags; - const unsigned long *bitmap_other = flags_other->__vma_flags; + vma_flags_t diff = vma_flags_diff_pair(flags, flags_other); - return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS); + /* + * Atomic VMA flags may be set concurrently under the VMA read lock, so + * are disregarded for equality testing purposes. + */ + vma_flags_clear_mask(&diff, VMA_ATOMIC_SET_ALLOWED); + + return vma_flags_empty(&diff); } -static __always_inline bool vma_flags_same_mask(const vma_flags_t *flags, - vma_flags_t flags_other) -{ - const unsigned long *bitmap = flags->__vma_flags; - const unsigned long *bitmap_other = flags_other.__vma_flags; - - return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS); -} - -#define vma_flags_same(flags, ...) \ - vma_flags_same_mask(flags, mk_vma_flags(__VA_ARGS__)) - static __always_inline bool vma_test(const struct vm_area_struct *vma, vma_flag_t bit) { @@ -1106,6 +1090,36 @@ static __always_inline void vma_clear_flags_mask(struct vm_area_struct *vma, #define vma_clear_flags(vma, ...) \ vma_clear_flags_mask(vma, mk_vma_flags(__VA_ARGS__)) +/* + * Overwrite the VMA's flags with new ones, preserving atomic VMA flags in the + * process. The write is done once and once only. + */ +static inline void vma_overwrite_flags(struct vm_area_struct *vma, + vma_flags_t new_flags) +{ + vma_flags_t atomic; + + vma_assert_write_locked(vma); + + /* + * Preserve the VMA's atomic flags, which are kept stable by the VMA + * write lock. + */ + atomic = vma_flags_and_mask(&vma->flags, VMA_ATOMIC_SET_ALLOWED); + vma_flags_clear_mask(&new_flags, VMA_ATOMIC_SET_ALLOWED); + vma_flags_set_mask(&new_flags, atomic); + + /* It is assumed only the first system word must be written once. */ + vma_flags_overwrite_word_once(&vma->flags, new_flags.__vma_flags[0]); + /* The remainder can be copied normally. */ + if (NUM_VMA_FLAG_BITS > BITS_PER_LONG) { + unsigned long *dst = &vma->flags.__vma_flags[1]; + const unsigned long *src = &new_flags.__vma_flags[1]; + + bitmap_copy(dst, src, NUM_VMA_FLAG_BITS - BITS_PER_LONG); + } +} + static __always_inline bool vma_desc_test(const struct vm_area_desc *desc, vma_flag_t bit) { diff --git a/tools/testing/vma/shared.h b/tools/testing/vma/shared.h index 97cd7a679dc1..646edf367d7c 100644 --- a/tools/testing/vma/shared.h +++ b/tools/testing/vma/shared.h @@ -51,17 +51,21 @@ __TO_SCALAR(__val1), __TO_SCALAR(__val2)); \ } while (0) -#define ASSERT_FLAGS_SAME_MASK(_flags, _flags_other) \ - ASSERT_TRUE(vma_flags_same_mask((_flags), (_flags_other))) +#define ASSERT_FLAGS_SAME_MASK(_flags, _flags_other) do { \ + const vma_flags_t __flags_other = (_flags_other); \ + ASSERT_TRUE(vma_flags_same_pair((_flags), &__flags_other)); \ + } while (0) -#define ASSERT_FLAGS_NOT_SAME_MASK(_flags, _flags_other) \ - ASSERT_FALSE(vma_flags_same_mask((_flags), (_flags_other))) +#define ASSERT_FLAGS_NOT_SAME_MASK(_flags, _flags_other) do { \ + const vma_flags_t __flags_other = (_flags_other); \ + ASSERT_FALSE(vma_flags_same_pair((_flags), &__flags_other)); \ + } while (0) #define ASSERT_FLAGS_SAME(_flags, ...) \ - ASSERT_TRUE(vma_flags_same(_flags, __VA_ARGS__)) + ASSERT_FLAGS_SAME_MASK(_flags, mk_vma_flags(__VA_ARGS__)) #define ASSERT_FLAGS_NOT_SAME(_flags, ...) \ - ASSERT_FALSE(vma_flags_same(_flags, __VA_ARGS__)) + ASSERT_FLAGS_NOT_SAME_MASK(_flags, mk_vma_flags(__VA_ARGS__)) #define ASSERT_FLAGS_EMPTY(_flags) \ ASSERT_TRUE(vma_flags_empty(_flags))