github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-24 06:24:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm/slab: replace slab.stride with obj_exts_in_object

Browse Source 
The stride field is used to convert object index to an slabobj_ext so
both compact arrays (kmalloc() or in-slab-leftover) and spread
in-object-padding obj_ext layouts are supported.

In practice thus the stride is always sizeof(slabobj_ext) or s->size.

This simplifies the calculations, but with the upcoming slabobj_ext
handling changes, it will be easier to stop storing the stride and
instead just have a flag whether obj_ext is in the object padding.
obj_exts_in_object() can then rely on this flag and slab_obj_ext()
can use that to determine the stride.

No functional change intended.

Reviewed-by: Suren Baghdasaryan <surenb@google.com>
Reviewed-by: Hao Li <hao.li@linux.dev>
Link: https://patch.msgid.link/20260727-b4-objext_split-v3-7-c29ef0f1f257@kernel.org
Reviewed-by: Harry Yoo (Oracle) <harry@kernel.org>
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
This commit is contained in:
Vlastimil Babka (SUSE) 2026-07-27 14:54:01 +02:00
parent e684ee3bb5
commit b5bc35ace2
2 changed files with 42 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
mm/slab.h
View File

@ -81,10 +81,10 @@ struct freelist_counters {
#ifdef CONFIG_64BIT
/*
* Some optimizations use free bits in 'counters' field
* to save memory. In case ->stride field is not available,
* to save memory. If these free bits are not available,
* such optimizations are disabled.
*/
unsigned int stride;
unsigned obj_exts_in_object:1;
#endif
};
};
@ -617,22 +617,20 @@ static inline void put_slab_obj_exts(unsigned long obj_exts)
}
#ifdef CONFIG_64BIT
static inline void slab_set_stride(struct slab *slab, unsigned int stride)
static inline bool obj_exts_in_object(struct slab *slab)
{
slab->stride = stride;
}
static inline unsigned int slab_get_stride(struct slab *slab)
{
return slab->stride;
/*
* Note we cannot rely on the SLAB_OBJ_EXT_IN_OBJ flag here and need to
* check the per-slab bit. A cache can have SLAB_OBJ_EXT_IN_OBJ set, but
* allocations within_slab_leftover are preferred. And those may be
* possible or not depending on the particular slab's size.
*/
return slab->obj_exts_in_object;
}
#else
static inline void slab_set_stride(struct slab *slab, unsigned int stride)
static inline bool obj_exts_in_object(struct slab *slab)
{
VM_WARN_ON_ONCE(stride != sizeof(struct slabobj_ext));
}
static inline unsigned int slab_get_stride(struct slab *slab)
{
return sizeof(struct slabobj_ext);
return false;
}
#endif
@ -657,8 +655,15 @@ slab_obj_ext(struct kmem_cache *s, struct slab *slab, unsigned long obj_exts,
VM_WARN_ON_ONCE(obj_exts != slab_obj_exts(slab));
index = obj_to_index(s, slab, obj);
obj_ext = (struct slabobj_ext *)(obj_exts +
slab_get_stride(slab) * index);
if (!obj_exts_in_object(slab)) {
obj_ext = ((struct slabobj_ext *)obj_exts) + index;
} else {
unsigned int stride = s->size;
obj_ext = (struct slabobj_ext *)(obj_exts + index * stride);
}
return kasan_reset_tag(obj_ext);
}
@ -702,9 +707,10 @@ slab_obj_ext(struct kmem_cache *s, struct slab *slab, unsigned long obj_exts,
return NULL;
}
static inline void slab_set_stride(struct slab *slab, unsigned int stride) { }
static inline unsigned int slab_get_stride(struct slab *slab) { return 0; }
static inline bool obj_exts_in_object(struct slab *slab)
{
return false;
}
#endif /* CONFIG_SLAB_OBJ_EXT */

44
mm/slub.c
View File

@ -870,18 +870,6 @@ static inline bool obj_exts_in_slab(struct kmem_cache *s, struct slab *slab)
#endif
#if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT)
static bool obj_exts_in_object(struct kmem_cache *s, struct slab *slab)
{
/*
* Note we cannot rely on the SLAB_OBJ_EXT_IN_OBJ flag here and need to
* check the stride. A cache can have SLAB_OBJ_EXT_IN_OBJ set, but
* allocations within_slab_leftover are preferred. And those may be
* possible or not depending on the particular slab's size.
*/
return obj_exts_in_slab(s, slab) &&
(slab_get_stride(slab) == s->size);
}
static unsigned int obj_exts_offset_in_object(struct kmem_cache *s)
{
unsigned int offset = get_info_end(s);
@ -896,16 +884,20 @@ static unsigned int obj_exts_offset_in_object(struct kmem_cache *s)
return offset;
}
#else
static inline bool obj_exts_in_object(struct kmem_cache *s, struct slab *slab)
{
return false;
}
static inline void slab_set_obj_exts_in_object(struct slab *slab)
{
slab->obj_exts_in_object = 1;
}
#else
static inline unsigned int obj_exts_offset_in_object(struct kmem_cache *s)
{
return 0;
}
static inline void slab_set_obj_exts_in_object(struct slab *slab)
{
}
#endif
#ifdef CONFIG_SLUB_DEBUG
@ -1206,7 +1198,7 @@ static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p)
off += kasan_metadata_size(s, false);
if (obj_exts_in_object(s, slab))
if (obj_exts_in_object(slab))
off += sizeof(struct slabobj_ext);
if (off != size_from_object(s))
@ -1411,7 +1403,7 @@ static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p)
off += kasan_metadata_size(s, false);
if (obj_exts_in_object(s, slab))
if (obj_exts_in_object(slab))
off += sizeof(struct slabobj_ext);
if (size_from_object(s) == off)
@ -1439,7 +1431,7 @@ slab_pad_check(struct kmem_cache *s, struct slab *slab)
length = slab_size(slab);
end = start + length;
if (obj_exts_in_slab(s, slab) && !obj_exts_in_object(s, slab)) {
if (obj_exts_in_slab(s, slab) && !obj_exts_in_object(slab)) {
remainder = length;
remainder -= obj_exts_offset_in_slab(s, slab);
remainder -= obj_exts_size_in_slab(slab);
@ -2256,9 +2248,6 @@ static void alloc_slab_obj_exts_early(struct kmem_cache *s, struct slab *slab)
void *addr;
unsigned long obj_exts;
/* Initialize stride early to avoid memory ordering issues */
slab_set_stride(slab, sizeof(struct slabobj_ext));
if (!need_slab_obj_exts(s))
return;
@ -2292,7 +2281,7 @@ static void alloc_slab_obj_exts_early(struct kmem_cache *s, struct slab *slab)
obj_exts |= MEMCG_DATA_OBJEXTS;
#endif
slab->obj_exts = obj_exts;
slab_set_stride(slab, s->size);
slab_set_obj_exts_in_object(slab);
}
}
@ -3405,9 +3394,10 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags,
stat(s, ORDER_FALLBACK);
}
/* Initializes frozen, inuse, and any extra 64bit-only flags */
slab->counters = 0;
slab->objects = oo_objects(oo);
slab->inuse = 0;
slab->frozen = 0;
slab->slab_cache = s;
@ -6540,7 +6530,7 @@ static inline size_t slab_ksize(struct slab *slab)
*/
if (s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_STORE_USER))
return s->inuse;
else if (obj_exts_in_object(s, slab))
else if (obj_exts_in_object(slab))
return s->inuse;
/*
* Else we can use all the padding etc for the allocation