github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-22 20:54:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm/zsmalloc: encode class index in obj value for lockless class lookup

Browse Source 
Patch series "mm/zsmalloc: reduce lock contention in zs_free()", v6.

This series reduces lock contention in zs_free(), which dominates the
unmap path under memory pressure on Android (LMK kills) and on x86 servers
running zswap-heavy workloads.

The current zs_free() takes pool->lock (rwlock, read side) just to look up
the size_class for a handle, then takes class->lock and holds it across
__free_zspage() which can call into the buddy allocator and acquire
zone->lock.  Two costs follow:

  * pool->lock reader-counter cacheline bouncing among concurrent
    zs_free() callers.
  * class->lock held across folio_put(), so any zone->lock wait
    fans out to every other zs_free() on the same class.

The series tackles both:

  Patch 1: encode size_class index into obj alongside PFN and obj_idx,
           so zs_free() can locate the class without pool->lock.
  Patch 2: drop pool->lock from zs_free() on 64-bit; 32-bit unchanged.
  Patch 3: move zspage page-freeing out of class->lock.
  Patch 4: document the three free_zspage helper variants that result
           from the split in patch 3.

Performance results:

Test: each process independently mmap 256MB, write data, madvise
MADV_PAGEOUT to swap out via zram (lzo-rle), then concurrent munmap.

Raspberry Pi 4B (4-core ARM64 Cortex-A72):

  mode        Base       Patched     Speedup
  single      59.0ms     56.0ms      1.05x
  multi 2p    94.6ms     66.7ms      1.42x
  multi 4p    202.9ms    110.6ms     1.83x

x86 (20-core Intel i7-12700, 16 concurrent processes):

  mode        Base       Patched     Speedup
  single      11.7ms     9.8ms       1.19x
  multi 2p    24.1ms     17.2ms      1.40x
  multi 4p    63.0ms     45.3ms      1.39x


This patch (of 4):

Encode the size_class index (class_idx) into the obj value so that
zs_free() can determine the correct size_class without dereferencing the
handle->obj->PFN->zpdesc->zspage->class chain under pool->lock.  class_idx
is invariant across page migration (only PFN is rewritten), so a lockless
read of obj always yields a valid class_idx.

Where obj has more bits below the PFN field than obj_idx alone needs,
split that space into class_idx and obj_idx subfields:

 |<-- _PFN_BITS -->|<-- ZS_OBJ_CLASS_BITS -->|<-- ZS_OBJ_IDX_BITS -->|
 +-----------------+-------------------------+-----------------------+
 |       PFN       |        class_idx        |        obj_idx        |
 +-----------------+-------------------------+-----------------------+
MSB                ^                                                LSB
                   |
                   +-- ZS_OBJ_PFN_SHIFT

The macro layout changes as follows:

    Before            After               Meaning
    ----------------  ------------------  ----------------------------
    OBJ_INDEX_BITS    ZS_OBJ_IDX_BITS     width of obj_idx subfield
    OBJ_INDEX_MASK    ZS_OBJ_IDX_MASK     mask  of obj_idx subfield
    (n/a)             ZS_OBJ_CLASS_BITS   width of class_idx subfield
    (n/a)             ZS_OBJ_CLASS_MASK   mask  of class_idx subfield
    (n/a)             ZS_OBJ_PFN_SHIFT    bit offset of PFN in obj

ZS_OBJ_CLASS_BITS folds to 0 (and the layout collapses to [PFN | obj_idx])
when obj has no spare bits, i.e.  on 32-bit or on 64-bit fallback paths
where MAX_POSSIBLE_PHYSMEM_BITS == BITS_PER_LONG (e.g.  UML); zs_free()
then falls back to pool->lock.

[akpm@linux-foundation.org: fix obj_to_class_idx() defined but not used, remove duplicated #ifdef]
Link: https://lore.kernel.org/20260626015003.2965881-1-haowenchao22@gmail.com
Link: https://lore.kernel.org/20260626015003.2965881-2-haowenchao22@gmail.com
Signed-off-by: Wenchao Hao <haowenchao@xiaomi.com>
Reviewed-by: Nhat Pham <nphamcs@gmail.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Joshua Hahn <joshua.hahnjy@gmail.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Sergey Senozhatsky <senozhatsky@chromium.org>
Cc: Xueyuan Chen <xueyuan.chen21@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Wenchao Hao 2026-06-26 09:50:00 +08:00 committed by Andrew Morton
parent f078b0a072
commit 9909b088b1
1 changed files with 90 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
mm/zsmalloc.c
View File

@ -67,8 +67,8 @@
#define MAX_POSSIBLE_PHYSMEM_BITS MAX_PHYSMEM_BITS
#else
/*
* If this definition of MAX_PHYSMEM_BITS is used, OBJ_INDEX_BITS will just
* be PAGE_SHIFT
* If this definition of MAX_PHYSMEM_BITS is used, ZS_OBJ_PFN_SHIFT will
* just be PAGE_SHIFT
*/
#define MAX_POSSIBLE_PHYSMEM_BITS BITS_PER_LONG
#endif
@ -88,8 +88,23 @@
#define OBJ_TAG_BITS 1
#define OBJ_TAG_MASK OBJ_ALLOCATED_TAG
#define OBJ_INDEX_BITS (BITS_PER_LONG - _PFN_BITS)
#define OBJ_INDEX_MASK ((_AC(1, UL) << OBJ_INDEX_BITS) - 1)
/*
* obj is encoded as [PFN | class_idx | obj_idx] within an unsigned long:
*
* |<-- _PFN_BITS -->|<-- ZS_OBJ_CLASS_BITS -->|<-- ZS_OBJ_IDX_BITS -->|
* +-----------------+-------------------------+-----------------------+
* | PFN | class_idx | obj_idx |
* +-----------------+-------------------------+-----------------------+
* MSB ^ LSB
* |
* +-- ZS_OBJ_PFN_SHIFT
*
* Encoding class_idx into obj lets zs_free() locate the size_class
* without holding pool->lock; class_idx is invariant across page
* migration (only PFN changes), so a lockless read of the obj value
* always yields a valid class_idx.
*/
#define ZS_OBJ_PFN_SHIFT (BITS_PER_LONG - _PFN_BITS)
#define HUGE_BITS 1
#define FULLNESS_BITS 4
@ -98,9 +113,61 @@
#define ZS_MAX_PAGES_PER_ZSPAGE (_AC(CONFIG_ZSMALLOC_CHAIN_SIZE, UL))
/*
* Bits to index a page within a zspage = ceil(log2(ZS_MAX_PAGES_PER_ZSPAGE)).
* Computed at preprocessor time, for use in #if below. Kconfig
* restricts ZSMALLOC_CHAIN_SIZE to [4, 16].
*/
#if ZS_MAX_PAGES_PER_ZSPAGE <= 4
#define ZS_PAGES_PER_ZSPAGE_BITS 2
#elif ZS_MAX_PAGES_PER_ZSPAGE <= 8
#define ZS_PAGES_PER_ZSPAGE_BITS 3
#elif ZS_MAX_PAGES_PER_ZSPAGE <= 16
#define ZS_PAGES_PER_ZSPAGE_BITS 4
#else
#error "ZSMALLOC_CHAIN_SIZE out of expected range [4,16]"
#endif
/*
* Bits to index an object within a single PAGE_SIZE at the smallest
* possible object size: log2(PAGE_SIZE / 32) = PAGE_SHIFT - 5.
* 32 is the hard floor of ZS_MIN_ALLOC_SIZE.
*/
#define ZS_OBJS_PER_PAGE_BITS (PAGE_SHIFT - 5)
/*
* Bits to index any object in the densest possible zspage. Below this,
* ZS_MIN_ALLOC_SIZE is auto-raised by the MAX(32, ...) formula -- still
* correct, but objects are coarser.
*/
#define ZS_OBJS_PER_ZSPAGE_BITS \
(ZS_PAGES_PER_ZSPAGE_BITS + ZS_OBJS_PER_PAGE_BITS)
/*
* Encode class_idx only when obj has spare bits; otherwise
* ZS_OBJ_CLASS_BITS folds to 0 (32-bit, or 64-bit UML/fallback).
*/
#if BITS_PER_LONG >= 64 && \
ZS_OBJ_PFN_SHIFT >= (CLASS_BITS + 1) + ZS_OBJS_PER_ZSPAGE_BITS
#define ZS_OBJ_CLASS_BITS (CLASS_BITS + 1)
#else
#define ZS_OBJ_CLASS_BITS 0
#endif
#define ZS_OBJ_CLASS_MASK ((_AC(1, UL) << ZS_OBJ_CLASS_BITS) - 1)
#define ZS_OBJ_IDX_BITS (ZS_OBJ_PFN_SHIFT - ZS_OBJ_CLASS_BITS)
#define ZS_OBJ_IDX_MASK ((_AC(1, UL) << ZS_OBJ_IDX_BITS) - 1)
/*
* Belt-and-suspenders: the #if above already guarantees this when
* class_idx is enabled. Catches future tweaks that bypass it.
*/
static_assert(ZS_OBJ_IDX_BITS >= ZS_PAGES_PER_ZSPAGE_BITS,
"zsmalloc: ZS_MIN_ALLOC_SIZE would exceed ZS_MAX_ALLOC_SIZE");
/* ZS_MIN_ALLOC_SIZE must be multiple of ZS_ALIGN */
#define ZS_MIN_ALLOC_SIZE \
MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> OBJ_INDEX_BITS))
MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> ZS_OBJ_IDX_BITS))
/* each chunk includes extra space to keep handle */
#define ZS_MAX_ALLOC_SIZE PAGE_SIZE
@ -720,26 +787,29 @@ static struct zpdesc *get_next_zpdesc(struct zpdesc *zpdesc)
static void obj_to_location(unsigned long obj, struct zpdesc **zpdesc,
unsigned int *obj_idx)
{
*zpdesc = pfn_zpdesc(obj >> OBJ_INDEX_BITS);
*obj_idx = (obj & OBJ_INDEX_MASK);
*zpdesc = pfn_zpdesc(obj >> ZS_OBJ_PFN_SHIFT);
*obj_idx = (obj & ZS_OBJ_IDX_MASK);
}
static void obj_to_zpdesc(unsigned long obj, struct zpdesc **zpdesc)
{
*zpdesc = pfn_zpdesc(obj >> OBJ_INDEX_BITS);
*zpdesc = pfn_zpdesc(obj >> ZS_OBJ_PFN_SHIFT);
}
/**
* location_to_obj - get obj value encoded from (<zpdesc>, <obj_idx>)
* location_to_obj - encode (<zpdesc>, <obj_idx>, <class_idx>) into obj value
* @zpdesc: zpdesc object resides in zspage
* @obj_idx: object index
* @class_idx: size class index; ignored when ZS_OBJ_CLASS_BITS == 0
*/
static unsigned long location_to_obj(struct zpdesc *zpdesc, unsigned int obj_idx)
static unsigned long location_to_obj(struct zpdesc *zpdesc, unsigned int obj_idx,
unsigned int class_idx)
{
unsigned long obj;
obj = zpdesc_pfn(zpdesc) << OBJ_INDEX_BITS;
obj |= obj_idx & OBJ_INDEX_MASK;
obj = zpdesc_pfn(zpdesc) << ZS_OBJ_PFN_SHIFT;
obj |= (unsigned long)(class_idx & ZS_OBJ_CLASS_MASK) << ZS_OBJ_IDX_BITS;
obj |= obj_idx & ZS_OBJ_IDX_MASK;
return obj;
}
@ -1275,7 +1345,7 @@ static unsigned long obj_malloc(struct zs_pool *pool,
kunmap_local(vaddr);
mod_zspage_inuse(zspage, 1);
obj = location_to_obj(m_zpdesc, obj);
obj = location_to_obj(m_zpdesc, obj, zspage->class);
record_obj(handle, obj);
return obj;
@ -1643,9 +1713,12 @@ static void lock_zspage(struct zspage *zspage)
}
zspage_read_unlock(zspage);
}
#endif /* CONFIG_COMPACTION */
#ifdef CONFIG_COMPACTION
/* Folds to 0 when ZS_OBJ_CLASS_BITS == 0; no ifdef needed at callers. */
static unsigned int obj_to_class_idx(unsigned long obj)
{
return (obj >> ZS_OBJ_IDX_BITS) & ZS_OBJ_CLASS_MASK;
}
static void replace_sub_page(struct size_class *class, struct zspage *zspage,
struct zpdesc *newzpdesc, struct zpdesc *oldzpdesc)
@ -1761,7 +1834,8 @@ static int zs_page_migrate(struct page *newpage, struct page *page,
old_obj = handle_to_obj(handle);
obj_to_location(old_obj, &dummy, &obj_idx);
new_obj = (unsigned long)location_to_obj(newzpdesc, obj_idx);
new_obj = location_to_obj(newzpdesc, obj_idx,
obj_to_class_idx(old_obj));
record_obj(handle, new_obj);
}
}