diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 2c7198868de0..6d3ed196f6f9 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -877,13 +877,10 @@ static int trylock_zspage(struct zspage *zspage) return 0; } -static void __free_zspage(struct zs_pool *pool, struct size_class *class, - struct zspage *zspage) +static inline void __free_zspage_lockless(struct zspage *zspage) { struct zpdesc *zpdesc, *next; - assert_spin_locked(&class->lock); - VM_BUG_ON(get_zspage_inuse(zspage)); VM_BUG_ON(zspage->fullness != ZS_INUSE_RATIO_0); @@ -899,7 +896,13 @@ static void __free_zspage(struct zs_pool *pool, struct size_class *class, } while (zpdesc != NULL); cache_free_zspage(zspage); +} +static void __free_zspage(struct zs_pool *pool, struct size_class *class, + struct zspage *zspage) +{ + assert_spin_locked(&class->lock); + __free_zspage_lockless(zspage); class_stat_sub(class, ZS_OBJS_ALLOCATED, class->objs_per_zspage); atomic_long_sub(class->pages_per_zspage, &pool->pages_allocated); } @@ -1520,6 +1523,7 @@ void zs_free(struct zs_pool *pool, unsigned long handle) unsigned long obj; struct size_class *class; int fullness; + struct zspage *zspage_to_free = NULL; if (IS_ERR_OR_NULL((void *)handle)) return; @@ -1530,10 +1534,23 @@ void zs_free(struct zs_pool *pool, unsigned long handle) obj_free(class->size, obj); fullness = fix_fullness_group(class, zspage); - if (fullness == ZS_INUSE_RATIO_0) - free_zspage(pool, class, zspage); + if (fullness == ZS_INUSE_RATIO_0) { + if (trylock_zspage(zspage)) { + remove_zspage(class, zspage); + class_stat_sub(class, ZS_OBJS_ALLOCATED, + class->objs_per_zspage); + zspage_to_free = zspage; + } else { + kick_deferred_free(pool); + } + } spin_unlock(&class->lock); + + if (zspage_to_free) { + __free_zspage_lockless(zspage_to_free); + atomic_long_sub(class->pages_per_zspage, &pool->pages_allocated); + } cache_free_handle(handle); } EXPORT_SYMBOL_GPL(zs_free);