diff --git a/mm/slab.h b/mm/slab.h index ef25c89b52d0..60e0df5bc5c2 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -568,6 +568,22 @@ struct slabobj_ext { }; } __aligned(8); +#ifdef CONFIG_MEM_ALLOC_PROFILING +DECLARE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT, + slab_obj_ext_has_codetag_key); + +static inline bool slab_obj_ext_has_codetag(void) +{ + return static_branch_maybe(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT, + &slab_obj_ext_has_codetag_key); +} +#else +static inline bool slab_obj_ext_has_codetag(void) +{ + return false; +} +#endif + static inline size_t cache_obj_ext_size(struct kmem_cache *s) { size_t sz = 0; @@ -712,6 +728,8 @@ slab_obj_ext_set_objcg(struct slab *slab, struct slabobj_ext *obj_ext, static inline union codetag_ref * slab_obj_ext_codetag_ref(struct slab *slab, struct slabobj_ext *obj_ext) { + VM_WARN_ON_ONCE(!slab_obj_ext_has_codetag()); + if (IS_ENABLED(CONFIG_MEMCG)) obj_ext += 1; diff --git a/mm/slub.c b/mm/slub.c index cfa370bc8130..d2cbca1ade22 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -213,6 +213,11 @@ DEFINE_STATIC_KEY_FALSE(slub_debug_enabled); static DEFINE_STATIC_KEY_FALSE(strict_numa); #endif +#ifdef CONFIG_MEM_ALLOC_PROFILING +DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT, + slab_obj_ext_has_codetag_key); +#endif + /* Structure holding extra parameters for slab allocations */ struct slab_alloc_context { unsigned long caller_addr; @@ -2420,6 +2425,25 @@ alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, __alloc_tagging_slab_free_hook(s, slab, p, objects); } +/* + * Make sure the static key used by slab_obj_ext_has_codetag() reflects the + * value of !mem_alloc_profiling_permanently_disabled() + * + * Any later mem alloc profiling shutdown won't be reflected in the static key + * because obj_exts with codetags might already exist. + */ +static void __init slab_obj_ext_has_codetag_init(void) +{ + bool need_codetag = !mem_alloc_profiling_permanently_disabled(); + + if (need_codetag != static_key_enabled(&slab_obj_ext_has_codetag_key)) { + if (need_codetag) + static_branch_enable(&slab_obj_ext_has_codetag_key); + else + static_branch_disable(&slab_obj_ext_has_codetag_key); + } +} + #else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void @@ -2434,6 +2458,10 @@ alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, { } +static inline void slab_obj_ext_has_codetag_init(void) +{ +} + #endif /* CONFIG_MEM_ALLOC_PROFILING */ @@ -8546,6 +8574,8 @@ void __init kmem_cache_init(void) boot_kmem_cache_node; int node; + slab_obj_ext_has_codetag_init(); + if (debug_guardpage_minorder()) slub_max_order = 0;