diff --git a/mm/slub.c b/mm/slub.c index 0870388b7177..40d3f7216263 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -5779,76 +5779,80 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab, new.inuse -= cnt; /* - * Might need to be taken off (due to becoming empty) or added - * to (due to not being full anymore) the partial list. - * Unless it's frozen. + * partial->partial: if the slab was on the node partial list, + * it stays there, and if it was off, it stays off, so we need + * no list handling and no list_lock. + * + * Note that "continue;" in a do-while goes on to evaluate the + * condition below, so we do perform the freelist update. */ - if (!new.inuse || was_full) { + if (!was_full && new.inuse) + continue; - n = get_node(s, slab_nid(slab)); - /* - * Speculatively acquire the list_lock. - * If the cmpxchg does not succeed then we may - * drop the list_lock without any processing. - * - * Otherwise the list_lock will synchronize with - * other processors updating the list of slabs. - */ - spin_lock_irqsave(&n->list_lock, flags); - - on_node_partial = slab_test_node_partial(slab); - } + /* + * The slab might need to be taken off (due to becoming empty) + * or added to (due to not being full anymore) the partial + * list. + * + * Speculatively acquire list_lock prior to cmpxchg(), as + * performing cmpxchg() before lock acquisition races with + * concurrent partial list operations. + * + * If the cmpxchg does not succeed then we will retry. + */ + n = get_node(s, slab_nid(slab)); + spin_lock_irqsave(&n->list_lock, flags); } while (!slab_update_freelist(s, slab, &old, &new, "__slab_free")); - if (likely(!n)) { - /* - * We didn't take the list_lock because the slab was already on - * the partial list and will remain there. - */ + /* partial->partial: we didn't take the list_lock. */ + if (likely(!n)) return; - } - /* - * This slab was partially empty but not on the per-node partial list, - * in which case we shouldn't manipulate its list, just return. - */ + on_node_partial = slab_test_node_partial(slab); + if (!was_full && !on_node_partial) { + /* + * partial->empty, offlist: a bulk refill has taken the slab + * off the partial list and will put it back, so its list + * handling is not ours to do. + */ spin_unlock_irqrestore(&n->list_lock, flags); return; } /* - * If slab became empty, should we add/keep it on the partial list or we - * have enough? + * full/partial->empty, exceed: we have enough partial slabs already. */ - if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) - goto slab_empty; + if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) { + /* partial->empty, onlist, exceed */ + if (likely(!was_full)) { + remove_partial(n, slab); + stat(s, FREE_REMOVE_PARTIAL); + } + /* else, full->empty, exceed: it is on no list to remove from */ + + spin_unlock_irqrestore(&n->list_lock, flags); + stat(s, FREE_SLAB); + discard_slab(s, slab); + return; + } /* - * Objects left in the slab. If it was not on the partial list before - * then add it. + * At this point, only three cases remain: + * full->partial + * full->empty, not exceed + * partial->empty, onlist, not exceed */ + + /* full->partial; full->empty, not exceed */ if (unlikely(was_full)) { add_partial(n, slab, ADD_TO_TAIL); stat(s, FREE_ADD_PARTIAL); } - spin_unlock_irqrestore(&n->list_lock, flags); - return; - -slab_empty: - /* - * The slab could have a single object and thus go from full to empty in - * a single free, but more likely it was on the partial list. Remove it. - */ - if (likely(!was_full)) { - remove_partial(n, slab); - stat(s, FREE_REMOVE_PARTIAL); - } + /* else, partial->empty, onlist, not exceed: it stays on partial list */ spin_unlock_irqrestore(&n->list_lock, flags); - stat(s, FREE_SLAB); - discard_slab(s, slab); } /*