diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 6f331c80130d..f744a42bb813 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -677,11 +677,18 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) return ERR_PTR(err); } -static inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd) +static __always_inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd) { - if (likely(key_len % 4 == 0)) + const u32 *k = key; + + if (key_len == sizeof(u32)) + return jhash_1word(k[0], hashrnd); + else if (key_len == sizeof(u64)) + return jhash_2words(k[0], k[1], hashrnd); + else if (likely(key_len % 4 == 0)) return jhash2(key, key_len / 4, hashrnd); - return jhash(key, key_len, hashrnd); + else + return jhash(key, key_len, hashrnd); } static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash) @@ -712,9 +719,9 @@ static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash * the unlikely event when elements moved from one bucket into another * while link list is being walked */ -static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head, - u32 hash, void *key, - u32 key_size, u32 n_buckets) +static __always_inline struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head, + u32 hash, void *key, + u32 key_size, u32 n_buckets) { struct hlist_nulls_node *n; struct htab_elem *l; @@ -730,22 +737,15 @@ static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head, return NULL; } -/* Called from syscall or from eBPF program directly, so - * arguments have to match bpf_map_lookup_elem() exactly. - * The return value is adjusted by BPF instructions - * in htab_map_gen_lookup(). - */ -static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) +static __always_inline void *__htab_lookup(struct bpf_map *map, void *key, u32 key_size) { struct bpf_htab *htab = container_of(map, struct bpf_htab, map); struct hlist_nulls_head *head; struct htab_elem *l; - u32 hash, key_size; + u32 hash; WARN_ON_ONCE(!bpf_rcu_lock_held()); - key_size = map->key_size; - hash = htab_map_hash(key, key_size, htab->hashrnd); head = select_bucket(htab, hash); @@ -755,6 +755,28 @@ static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) return l; } +/* + * Called from syscall or from eBPF program directly, so + * arguments have to match bpf_map_lookup_elem() exactly. + * The return value is adjusted by BPF instructions + * in htab_map_gen_lookup(). u32/u64 specializations + * allow utilizing compile-time key_size for optimization. + */ +static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) +{ + return __htab_lookup(map, key, map->key_size); +} + +static void *__htab_map_lookup_elem_u32(struct bpf_map *map, void *key) +{ + return __htab_lookup(map, key, sizeof(u32)); +} + +static void *__htab_map_lookup_elem_u64(struct bpf_map *map, void *key) +{ + return __htab_lookup(map, key, sizeof(u64)); +} + static void *htab_map_lookup_elem(struct bpf_map *map, void *key) { struct htab_elem *l = __htab_map_lookup_elem(map, key); @@ -783,7 +805,16 @@ static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, (void *(*)(struct bpf_map *map, void *key))NULL)); - *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); + BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem_u32, + (void *(*)(struct bpf_map *map, void *key))NULL)); + BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem_u64, + (void *(*)(struct bpf_map *map, void *key))NULL)); + if (map->key_size == sizeof(u32)) + *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem_u32); + else if (map->key_size == sizeof(u64)) + *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem_u64); + else + *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1); *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, offsetof(struct htab_elem, key) +