mirror of
https://github.com/torvalds/linux.git
synced 2026-10-24 16:55:02 +02:00
rbtree: Update augmented data on the way down in rb_add_augmented_cached()
rb_add_augmented_cached() links the new node and then calls
->propagate() from the parent up to the root to bring the augmented
data of the ancestors up to date. It marks that as "suboptimal": every
augmented user that open-codes its insertion (interval trees, vmalloc,
drm_mm, gpu buddy, ...) instead folds the new node's value into each
ancestor while walking down, which touches no sibling and needs no
second pass. ->propagate() has to read both children of every
ancestor until it finds one whose value did not change.
Add a ->merge() callback to struct rb_augment_callbacks that folds the
augmented data of a node being inserted into one of its future
ancestors, and use it on the descent. RB_DECLARE_CALLBACKS() derives
it from the RB_AUG() list, so every user of the template gets it for
free, and sched/eevdf, the only caller of the helper, needs no change.
The new node's augmented data must describe the node alone before the
call. That was already required, since ->propagate() read it, and
sched/eevdf sets it up that way.
Measured with rbtree_test's cached augmented test switched to
rb_add_augmented_cached() (next patch), nnodes cached insert+delete,
median of 5 runs, for reference also the open-coded descent used by
augmented test 1. Raspberry Pi 4 Model B (Cortex-A72, 1.8 GHz), arch
timer ticks (54 MHz):
nnodes open-coded before after
100 459 500 442 (-11.6%)
10000 182046 186144 180208 (-3.2%)
100000 5533679 5711647 5579444 (-2.3%)
x86-64 KVM guest (Intel i7-13800H), cycles:
nnodes open-coded before after
100 7657 7622 7059 (-7.4%)
10000 4738075 5075847 4627462 (-8.8%)
100000 75618429 79840335 71324700 (-10.7%)
The other rbtree_test numbers are unchanged and the augmented invariant
checks still pass on both.
Signed-off-by: Yiwei Lin <s921975628@gmail.com>
Assisted-by: LLM
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20261005093236.62702-4-s921975628@gmail.com
This commit is contained in:
Yiwei Lin
Peter Zijlstra
parent
c2c620f434
commit
cd047867ee
|
|
@ -257,8 +257,14 @@ When erasing a node, the user must call rb_erase_augmented() instead of
|
|||
rb_erase(). rb_erase_augmented() calls back into user provided functions
|
||||
to update the augmented information on affected subtrees.
|
||||
|
||||
In both cases, the callbacks are provided through struct rb_augment_callbacks.
|
||||
3 callbacks must be defined:
|
||||
Alternatively, rb_add_augmented_cached() performs the whole insertion into
|
||||
a cached tree: it walks down to the insertion point, merges the new node's
|
||||
augmented information into every node on that path, links the node and
|
||||
rebalances. The new node's augmented information must already describe
|
||||
the node alone when it is called.
|
||||
|
||||
In all cases, the callbacks are provided through struct rb_augment_callbacks.
|
||||
3 callbacks must be defined, plus a fourth one for rb_add_augmented_cached():
|
||||
|
||||
- A propagation callback, which updates the augmented value for a given
|
||||
node and its ancestors, up to a given stop point (or NULL to update
|
||||
|
|
@ -271,6 +277,10 @@ In both cases, the callbacks are provided through struct rb_augment_callbacks.
|
|||
subtree to a newly assigned subtree root AND recomputes the augmented
|
||||
information for the former subtree root.
|
||||
|
||||
- A merge callback, which folds the augmented value of a node being inserted
|
||||
into the augmented value of one of its future ancestors. It is only used
|
||||
by rb_add_augmented_cached().
|
||||
|
||||
The compiled code for rb_erase_augmented() may inline the propagation and
|
||||
copy callbacks, which results in a large function, so each augmented rbtree
|
||||
user should have a single rb_erase_augmented() call site in order to limit
|
||||
|
|
@ -395,8 +405,19 @@ Insertion/removal are defined using the following augmented callbacks::
|
|||
old->__subtree_last = compute_subtree_last(old);
|
||||
}
|
||||
|
||||
static void augment_merge(struct rb_node *rb, struct rb_node *rb_new)
|
||||
{
|
||||
struct interval_tree_node *node =
|
||||
rb_entry(rb, struct interval_tree_node, rb);
|
||||
struct interval_tree_node *new =
|
||||
rb_entry(rb_new, struct interval_tree_node, rb);
|
||||
|
||||
if (node->__subtree_last < new->__subtree_last)
|
||||
node->__subtree_last = new->__subtree_last;
|
||||
}
|
||||
|
||||
static const struct rb_augment_callbacks augment_callbacks = {
|
||||
augment_propagate, augment_copy, augment_rotate
|
||||
augment_propagate, augment_copy, augment_rotate, augment_merge
|
||||
};
|
||||
|
||||
void interval_tree_insert(struct interval_tree_node *node,
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ struct rb_augment_callbacks {
|
|||
void (*propagate)(struct rb_node *node, struct rb_node *stop);
|
||||
void (*copy)(struct rb_node *old, struct rb_node *new);
|
||||
void (*rotate)(struct rb_node *old, struct rb_node *new);
|
||||
void (*merge)(struct rb_node *node, struct rb_node *new);
|
||||
};
|
||||
|
||||
extern void __rb_insert_augmented(struct rb_node *node, struct rb_root *root,
|
||||
|
|
@ -62,6 +63,12 @@ rb_insert_augmented_cached(struct rb_node *node,
|
|||
rb_insert_augmented(node, &root->rb_root, augment);
|
||||
}
|
||||
|
||||
/*
|
||||
* Insert @node into the leftmost cached augmented tree @tree.
|
||||
*
|
||||
* The augmented data of @node must already describe @node alone; it is
|
||||
* merged into every ancestor on the way down through augment->merge().
|
||||
*/
|
||||
static __always_inline struct rb_node *
|
||||
rb_add_augmented_cached(struct rb_node *node, struct rb_root_cached *tree,
|
||||
bool (*less)(struct rb_node *, const struct rb_node *),
|
||||
|
|
@ -73,6 +80,7 @@ rb_add_augmented_cached(struct rb_node *node, struct rb_root_cached *tree,
|
|||
|
||||
while (*link) {
|
||||
parent = *link;
|
||||
augment->merge(parent, node);
|
||||
if (less(node, parent)) {
|
||||
link = &parent->rb_left;
|
||||
} else {
|
||||
|
|
@ -82,7 +90,6 @@ rb_add_augmented_cached(struct rb_node *node, struct rb_root_cached *tree,
|
|||
}
|
||||
|
||||
rb_link_node(node, parent, link);
|
||||
augment->propagate(parent, NULL); /* suboptimal */
|
||||
rb_insert_augmented_cached(node, tree, leftmost, augment);
|
||||
|
||||
return leftmost ? node : NULL;
|
||||
|
|
@ -177,6 +184,14 @@ RBNAME ## _compute_ ## n(RBSTRUCT *s, bool exit) \
|
|||
#define RB_COMPUTE(n, RBNAME, RBSTRUCT, RBFIELD, x) \
|
||||
_RB_COMPUTE(n, RBNAME, RBSTRUCT, RBFIELD, RB_UNPACK x)
|
||||
|
||||
/* fold @new, about to become a descendant of @node, into @node */
|
||||
#define __RB_MERGE(n, RBNAME, RBSTRUCT, RBFIELD, val, aug, fold) \
|
||||
node->aug = fold(node->aug, new->aug);
|
||||
#define _RB_MERGE(n, RBNAME, RBSTRUCT, RBFIELD, args) \
|
||||
__RB_MERGE(n, RBNAME, RBSTRUCT, RBFIELD, args)
|
||||
#define RB_MERGE(n, RBNAME, RBSTRUCT, RBFIELD, x) \
|
||||
_RB_MERGE(n, RBNAME, RBSTRUCT, RBFIELD, RB_UNPACK x)
|
||||
|
||||
/*
|
||||
* Template for declaring augmented rbtree callbacks (generic multi fields)
|
||||
*
|
||||
|
|
@ -226,10 +241,18 @@ RBNAME ## _rotate(struct rb_node *rb_old, struct rb_node *rb_new) \
|
|||
RBNAME ## __copy(old, new); \
|
||||
RBNAME ## __compute(old, false); \
|
||||
} \
|
||||
static inline void \
|
||||
RBNAME ## _merge(struct rb_node *rb, struct rb_node *rb_new) \
|
||||
{ \
|
||||
RBSTRUCT *node = rb_entry(rb, RBSTRUCT, RBFIELD); \
|
||||
RBSTRUCT *new = rb_entry(rb_new, RBSTRUCT, RBFIELD); \
|
||||
RB_FOR_EACH(RB_MERGE, RBNAME, RBSTRUCT, RBFIELD, RBAUG); \
|
||||
} \
|
||||
RBSTATIC const struct rb_augment_callbacks RBNAME = { \
|
||||
.propagate = RBNAME ## _propagate, \
|
||||
.copy = RBNAME ## _copy, \
|
||||
.rotate = RBNAME ## _rotate \
|
||||
.rotate = RBNAME ## _rotate, \
|
||||
.merge = RBNAME ## _merge \
|
||||
};
|
||||
|
||||
/*
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user