github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-27 17:47:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

bpf: Optimize word-sized keys for resizable hashtable

Browse Source 
Specialize the lookup/update/delete paths for keys whose size matches
sizeof(long) (4 bytes on 32-bit, 8 bytes on 64-bit). A static-const
rhashtable_params lets the compiler inline a custom XOR-fold hashfn and
a single-word equality cmpfn, eliminating the indirect jhash dispatch.
The same hashfn and cmpfn are installed into rhashtable's stored params
at rhashtable_init time, so the rehash worker, slow-path inserts, and
rhashtable_next_key() all agree with the inlined fast paths.

The seq_file BPF iterator uses rhashtable_walk_* and is unaffected.

Signed-off-by: Mykyta Yatsenko <yatsenko@meta.com>
Link: https://lore.kernel.org/r/20260605-rhash-v7-7-5b8e05f8630d@meta.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Mykyta Yatsenko 2026-06-05 04:41:24 -07:00 committed by Alexei Starovoitov
parent 6905f86012
commit 9dcbb5045f
1 changed files with 45 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
kernel/bpf/hashtab.c
View File

@ -2763,6 +2763,31 @@ static inline void *rhtab_elem_value(struct rhtab_elem *l, u32 key_size)
return l->data + round_up(key_size, 8);
}
/* Specialize hash function and objcmp for long sized key */
static __always_inline int rhtab_key_cmp_long(struct rhashtable_compare_arg *arg,
const void *ptr)
{
const unsigned long key1 = *(const unsigned long *)arg->key;
const struct rhtab_elem *key2 = ptr;
return key1 != *(const unsigned long *)key2->data;
}
static __always_inline u32 rhtab_hashfn_long(const void *data, u32 len, u32 seed)
{
u64 k = *(const unsigned long *)data;
return (u32)(k ^ (k >> 32)) ^ seed;
}
static const struct rhashtable_params rhtab_params_long = {
.head_offset = offsetof(struct rhtab_elem, node),
.key_offset = offsetof(struct rhtab_elem, data),
.key_len = sizeof(long),
.hashfn = rhtab_hashfn_long,
.obj_cmpfn = rhtab_key_cmp_long,
};
static struct bpf_map *rhtab_map_alloc(union bpf_attr *attr)
{
struct rhashtable_params params;
@ -2788,6 +2813,11 @@ static struct bpf_map *rhtab_map_alloc(union bpf_attr *attr)
params.nelem_hint = (u32)attr->map_extra;
params.automatic_shrinking = true;
if (rhtab->map.key_size == sizeof(long)) {
params.hashfn = rhtab_hashfn_long;
params.obj_cmpfn = rhtab_key_cmp_long;
}
err = rhashtable_init(&rhtab->ht, &params);
if (err)
goto free_rhtab;
@ -2878,6 +2908,9 @@ static void *rhtab_lookup_elem(struct bpf_map *map, void *key)
/* Hold RCU lock in case sleepable program calls via gen_lookup */
guard(rcu)();
if (map->key_size == sizeof(long))
return rhashtable_lookup_likely(&rhtab->ht, key, rhtab_params_long);
return rhashtable_lookup_likely(&rhtab->ht, key, rhtab_params);
}
@ -2912,7 +2945,12 @@ static int rhtab_delete_elem(struct bpf_rhtab *rhtab, struct rhtab_elem *elem, v
* raw tracepoints, which we don't have in rhashtable.
*/
bpf_disable_instrumentation();
err = rhashtable_remove_fast(&rhtab->ht, &elem->node, rhtab_params);
if (rhtab->map.key_size == sizeof(long))
err = rhashtable_remove_fast(&rhtab->ht, &elem->node, rhtab_params_long);
else
err = rhashtable_remove_fast(&rhtab->ht, &elem->node, rhtab_params);
bpf_enable_instrumentation();
if (err)
@ -3030,7 +3068,12 @@ static long rhtab_map_update_elem(struct bpf_map *map, void *key, void *value, u
/* Prevent deadlock for NMI programs attempting to take bucket lock */
bpf_disable_instrumentation();
tmp = rhashtable_lookup_get_insert_fast(&rhtab->ht, &elem->node, rhtab_params);
if (map->key_size == sizeof(long))
tmp = rhashtable_lookup_get_insert_fast(&rhtab->ht, &elem->node, rhtab_params_long);
else
tmp = rhashtable_lookup_get_insert_fast(&rhtab->ht, &elem->node, rhtab_params);
bpf_enable_instrumentation();
if (tmp) {