From 5d8520b5572a79257ea18410a0eca98000f872a7 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Wed, 24 Nov 2021 07:56:04 -0800 Subject: [PATCH] ANDROID: fix ABI breakage caused by mm_struct->mmu_notifier_lock addition To prevent ABI breakage, move mm->mmu_notifier_lock into mm->notifier_subscriptions and allocate mm->notifier_subscriptions during mm creation in mmu_notifier_subscriptions_init. This results in additional 176 bytes allocated for each mm, but prevents ABI breakage. mmu_notifier_subscriptions_hdr structure is introduced at the beginning of mmu_notifier_subscriptions to keep mmu_notifier_subscriptions hidden and prevent its type CRC from changing when used in other structures. Bug: 161210518 Signed-off-by: Suren Baghdasaryan Change-Id: I6f435708d642b70b22e0243c8b33108c208ce5bb --- include/linux/mm_types.h | 4 --- include/linux/mmu_notifier.h | 62 +++++++++++++++++--------------- mm/mmu_notifier.c | 68 +++++++++++++++++++++++++++++++----- 3 files changed, 94 insertions(+), 40 deletions(-) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index a769ef105e6a..c853f612a815 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -404,7 +404,6 @@ struct core_state { }; struct kioctx_table; -struct percpu_rw_semaphore_atomic; struct mm_struct { struct { struct vm_area_struct *mmap; /* list of VMAs */ @@ -562,9 +561,6 @@ struct mm_struct { struct file __rcu *exe_file; #ifdef CONFIG_MMU_NOTIFIER struct mmu_notifier_subscriptions *notifier_subscriptions; -#ifdef CONFIG_SPECULATIVE_PAGE_FAULT - struct percpu_rw_semaphore_atomic *mmu_notifier_lock; -#endif #endif #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS pgtable_t pmd_huge_pte; /* protected by page_table_lock */ diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h index 986ed43eeafc..15c77675ca33 100644 --- a/include/linux/mmu_notifier.h +++ b/include/linux/mmu_notifier.h @@ -17,6 +17,13 @@ struct mmu_notifier; struct mmu_notifier_range; struct mmu_interval_notifier; +struct mmu_notifier_subscriptions_hdr { + bool valid; +#ifdef CONFIG_SPECULATIVE_PAGE_FAULT + struct percpu_rw_semaphore_atomic *mmu_notifier_lock; +#endif +}; + /** * enum mmu_notifier_event - reason for the mmu notifier callback * @MMU_NOTIFY_UNMAP: either munmap() that unmap the range or a mremap() that @@ -283,9 +290,30 @@ struct mmu_notifier_range { void *migrate_pgmap_owner; }; +static inline +struct mmu_notifier_subscriptions_hdr *get_notifier_subscriptions_hdr( + struct mm_struct *mm) +{ + /* + * container_of() can't be used here because mmu_notifier_subscriptions + * struct should be kept invisible to mm_struct, otherwise it + * introduces KMI CRC breakage. Therefore the callers don't know what + * members struct mmu_notifier_subscriptions contains and can't call + * container_of(), which requires a member name. + * + * WARNING: For this typecasting to work, mmu_notifier_subscriptions_hdr + * should be the first member of struct mmu_notifier_subscriptions. + */ + return (struct mmu_notifier_subscriptions_hdr *)mm->notifier_subscriptions; +} + static inline int mm_has_notifiers(struct mm_struct *mm) { +#ifdef CONFIG_SPECULATIVE_PAGE_FAULT + return unlikely(get_notifier_subscriptions_hdr(mm)->valid); +#else return unlikely(mm->notifier_subscriptions); +#endif } struct mmu_notifier *mmu_notifier_get_locked(const struct mmu_notifier_ops *ops, @@ -506,41 +534,19 @@ static inline void mmu_notifier_invalidate_range(struct mm_struct *mm, #ifdef CONFIG_SPECULATIVE_PAGE_FAULT -static inline bool mmu_notifier_subscriptions_init(struct mm_struct *mm) -{ - mm->mmu_notifier_lock = kzalloc( - sizeof(struct percpu_rw_semaphore_atomic), GFP_KERNEL); - if (!mm->mmu_notifier_lock) - return false; - - percpu_init_rwsem(&mm->mmu_notifier_lock->rw_sem); - mm->notifier_subscriptions = NULL; - - return true; -} - -static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm) -{ - if (mm_has_notifiers(mm)) - __mmu_notifier_subscriptions_destroy(mm); - - if (in_atomic()) { - percpu_rwsem_async_destroy(mm->mmu_notifier_lock); - } else { - percpu_free_rwsem(&mm->mmu_notifier_lock->rw_sem); - kfree(mm->mmu_notifier_lock); - } - mm->mmu_notifier_lock = NULL; -} +extern bool mmu_notifier_subscriptions_init(struct mm_struct *mm); +extern void mmu_notifier_subscriptions_destroy(struct mm_struct *mm); static inline bool mmu_notifier_trylock(struct mm_struct *mm) { - return percpu_down_read_trylock(&mm->mmu_notifier_lock->rw_sem); + return percpu_down_read_trylock( + &get_notifier_subscriptions_hdr(mm)->mmu_notifier_lock->rw_sem); } static inline void mmu_notifier_unlock(struct mm_struct *mm) { - percpu_up_read(&mm->mmu_notifier_lock->rw_sem); + percpu_up_read( + &get_notifier_subscriptions_hdr(mm)->mmu_notifier_lock->rw_sem); } #else /* CONFIG_SPECULATIVE_PAGE_FAULT */ diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c index 7f01fa75d351..cee61bab214b 100644 --- a/mm/mmu_notifier.c +++ b/mm/mmu_notifier.c @@ -35,6 +35,12 @@ struct lockdep_map __mmu_notifier_invalidate_range_start_map = { * in mmdrop(). */ struct mmu_notifier_subscriptions { + /* + * WARNING: hdr should be the first member of this structure + * so that it can be typecasted into mmu_notifier_subscriptions_hdr. + * This is required to avoid KMI CRC breakage. + */ + struct mmu_notifier_subscriptions_hdr hdr; /* all mmu notifiers registered in this mm are queued in this list */ struct hlist_head list; bool has_itree; @@ -625,12 +631,14 @@ void __mmu_notifier_invalidate_range(struct mm_struct *mm, static inline void mmu_notifier_write_lock(struct mm_struct *mm) { - percpu_down_write(&mm->mmu_notifier_lock->rw_sem); + percpu_down_write( + &mm->notifier_subscriptions->hdr.mmu_notifier_lock->rw_sem); } static inline void mmu_notifier_write_unlock(struct mm_struct *mm) { - percpu_up_write(&mm->mmu_notifier_lock->rw_sem); + percpu_up_write( + &mm->notifier_subscriptions->hdr.mmu_notifier_lock->rw_sem); } #else /* CONFIG_SPECULATIVE_PAGE_FAULT */ @@ -640,6 +648,16 @@ static inline void mmu_notifier_write_unlock(struct mm_struct *mm) {} #endif /* CONFIG_SPECULATIVE_PAGE_FAULT */ +static void init_subscriptions(struct mmu_notifier_subscriptions *subscriptions) +{ + INIT_HLIST_HEAD(&subscriptions->list); + spin_lock_init(&subscriptions->lock); + subscriptions->invalidate_seq = 2; + subscriptions->itree = RB_ROOT_CACHED; + init_waitqueue_head(&subscriptions->wq); + INIT_HLIST_HEAD(&subscriptions->deferred_list); +} + /* * Same as mmu_notifier_register but here the caller must hold the mmap_lock in * write mode. A NULL mn signals the notifier is being registered for itree @@ -672,12 +690,7 @@ int __mmu_notifier_register(struct mmu_notifier *subscription, if (!subscriptions) return -ENOMEM; - INIT_HLIST_HEAD(&subscriptions->list); - spin_lock_init(&subscriptions->lock); - subscriptions->invalidate_seq = 2; - subscriptions->itree = RB_ROOT_CACHED; - init_waitqueue_head(&subscriptions->wq); - INIT_HLIST_HEAD(&subscriptions->deferred_list); + init_subscriptions(subscriptions); } mmu_notifier_write_lock(mm); @@ -706,6 +719,7 @@ int __mmu_notifier_register(struct mmu_notifier *subscription, */ if (subscriptions) smp_store_release(&mm->notifier_subscriptions, subscriptions); + mm->notifier_subscriptions->hdr.valid = true; if (subscription) { /* Pairs with the mmdrop in mmu_notifier_unregister_* */ @@ -1149,3 +1163,41 @@ mmu_notifier_range_update_to_read_only(const struct mmu_notifier_range *range) return range->vma->vm_flags & VM_READ; } EXPORT_SYMBOL_GPL(mmu_notifier_range_update_to_read_only); + +#ifdef CONFIG_SPECULATIVE_PAGE_FAULT + +bool mmu_notifier_subscriptions_init(struct mm_struct *mm) +{ + struct mmu_notifier_subscriptions *subscriptions; + struct percpu_rw_semaphore_atomic *sem; + + subscriptions = kzalloc( + sizeof(struct mmu_notifier_subscriptions), GFP_KERNEL); + if (!subscriptions) + return false; + + sem = kzalloc(sizeof(struct percpu_rw_semaphore_atomic), GFP_KERNEL); + if (!sem) { + kfree(subscriptions); + return false; + } + percpu_init_rwsem(&sem->rw_sem); + + init_subscriptions(subscriptions); + subscriptions->has_itree = true; + subscriptions->hdr.valid = false; + subscriptions->hdr.mmu_notifier_lock = sem; + mm->notifier_subscriptions = subscriptions; + + return true; +} + +void mmu_notifier_subscriptions_destroy(struct mm_struct *mm) +{ + percpu_rwsem_async_destroy( + mm->notifier_subscriptions->hdr.mmu_notifier_lock); + kfree(mm->notifier_subscriptions); + mm->notifier_subscriptions = NULL; +} + +#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */