diff --git a/mm/slub.c b/mm/slub.c index b0c50df49cf3..7b6d8df06ad9 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -883,6 +883,97 @@ static inline unsigned long get_orig_size(struct kmem_cache *s, void *object) return *(unsigned long *)p; } +#ifdef CONFIG_SLAB_OBJ_EXT + +/* + * Check if memory cgroup or memory allocation profiling is enabled. + * If enabled, SLUB tries to reduce memory overhead of accounting + * slab objects. If neither is enabled when this function is called, + * the optimization is simply skipped to avoid affecting caches that do not + * need slabobj_ext metadata. + * + * However, this may disable optimization when memory cgroup or memory + * allocation profiling is used, but slabs are created too early + * even before those subsystems are initialized. + */ +static inline bool need_slab_obj_exts(struct kmem_cache *s) +{ + if (memcg_kmem_online() && (s->flags & SLAB_ACCOUNT)) + return true; + + if (mem_alloc_profiling_enabled()) + return true; + + return false; +} + +static inline unsigned int obj_exts_size_in_slab(struct slab *slab) +{ + return sizeof(struct slabobj_ext) * slab->objects; +} + +static inline unsigned long obj_exts_offset_in_slab(struct kmem_cache *s, + struct slab *slab) +{ + unsigned long objext_offset; + + objext_offset = s->size * slab->objects; + objext_offset = ALIGN(objext_offset, sizeof(struct slabobj_ext)); + return objext_offset; +} + +static inline bool obj_exts_fit_within_slab_leftover(struct kmem_cache *s, + struct slab *slab) +{ + unsigned long objext_offset = obj_exts_offset_in_slab(s, slab); + unsigned long objext_size = obj_exts_size_in_slab(slab); + + return objext_offset + objext_size <= slab_size(slab); +} + +static inline bool obj_exts_in_slab(struct kmem_cache *s, struct slab *slab) +{ + unsigned long obj_exts; + unsigned long start; + unsigned long end; + + obj_exts = slab_obj_exts(slab); + if (!obj_exts) + return false; + + start = (unsigned long)slab_address(slab); + end = start + slab_size(slab); + return (obj_exts >= start) && (obj_exts < end); +} +#else +static inline bool need_slab_obj_exts(struct kmem_cache *s) +{ + return false; +} + +static inline unsigned int obj_exts_size_in_slab(struct slab *slab) +{ + return 0; +} + +static inline unsigned long obj_exts_offset_in_slab(struct kmem_cache *s, + struct slab *slab) +{ + return 0; +} + +static inline bool obj_exts_fit_within_slab_leftover(struct kmem_cache *s, + struct slab *slab) +{ + return false; +} + +static inline bool obj_exts_in_slab(struct kmem_cache *s, struct slab *slab) +{ + return false; +} +#endif + #ifdef CONFIG_SLUB_DEBUG /* @@ -1418,7 +1509,15 @@ slab_pad_check(struct kmem_cache *s, struct slab *slab) start = slab_address(slab); length = slab_size(slab); end = start + length; - remainder = length % s->size; + + if (obj_exts_in_slab(s, slab)) { + remainder = length; + remainder -= obj_exts_offset_in_slab(s, slab); + remainder -= obj_exts_size_in_slab(slab); + } else { + remainder = length % s->size; + } + if (!remainder) return; @@ -2238,6 +2337,11 @@ static inline void free_slab_obj_exts(struct slab *slab) return; } + if (obj_exts_in_slab(slab->slab_cache, slab)) { + slab->obj_exts = 0; + return; + } + /* * obj_exts was created with __GFP_NO_OBJ_EXT flag, therefore its * corresponding extension will be NULL. alloc_tag_sub() will throw a @@ -2253,6 +2357,36 @@ static inline void free_slab_obj_exts(struct slab *slab) slab->obj_exts = 0; } +/* + * Try to allocate slabobj_ext array from unused space. + * This function must be called on a freshly allocated slab to prevent + * concurrency problems. + */ +static void alloc_slab_obj_exts_early(struct kmem_cache *s, struct slab *slab) +{ + void *addr; + unsigned long obj_exts; + + if (!need_slab_obj_exts(s)) + return; + + if (obj_exts_fit_within_slab_leftover(s, slab)) { + addr = slab_address(slab) + obj_exts_offset_in_slab(s, slab); + addr = kasan_reset_tag(addr); + obj_exts = (unsigned long)addr; + + get_slab_obj_exts(obj_exts); + memset(addr, 0, obj_exts_size_in_slab(slab)); + put_slab_obj_exts(obj_exts); + +#ifdef CONFIG_MEMCG + obj_exts |= MEMCG_DATA_OBJEXTS; +#endif + slab->obj_exts = obj_exts; + slab_set_stride(slab, sizeof(struct slabobj_ext)); + } +} + #else /* CONFIG_SLAB_OBJ_EXT */ static inline void init_slab_obj_exts(struct slab *slab) @@ -2269,6 +2403,11 @@ static inline void free_slab_obj_exts(struct slab *slab) { } +static inline void alloc_slab_obj_exts_early(struct kmem_cache *s, + struct slab *slab) +{ +} + #endif /* CONFIG_SLAB_OBJ_EXT */ #ifdef CONFIG_MEM_ALLOC_PROFILING @@ -3265,7 +3404,9 @@ static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) static __always_inline void account_slab(struct slab *slab, int order, struct kmem_cache *s, gfp_t gfp) { - if (memcg_kmem_online() && (s->flags & SLAB_ACCOUNT)) + if (memcg_kmem_online() && + (s->flags & SLAB_ACCOUNT) && + !slab_obj_exts(slab)) alloc_slab_obj_exts(slab, s, gfp, true); mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), @@ -3329,9 +3470,6 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) slab->objects = oo_objects(oo); slab->inuse = 0; slab->frozen = 0; - init_slab_obj_exts(slab); - - account_slab(slab, oo_order(oo), s, flags); slab->slab_cache = s; @@ -3340,6 +3478,13 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) start = slab_address(slab); setup_slab_debug(s, slab, start); + init_slab_obj_exts(slab); + /* + * Poison the slab before initializing the slabobj_ext array + * to prevent the array from being overwritten. + */ + alloc_slab_obj_exts_early(s, slab); + account_slab(slab, oo_order(oo), s, flags); shuffle = shuffle_freelist(s, slab);