diff --git a/include/linux/bpf.h b/include/linux/bpf.h index feb7de6483e3..aeb8b9b97052 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -4396,6 +4396,9 @@ static inline bool bpf_map_is_percpu_map(enum bpf_map_type map_type) } } +#define BPF_EXIST_FLAGS (BPF_EXIST | BPF_NOEXIST) +#define BPF_CPU_FLAGS (BPF_F_CPU | BPF_F_ALL_CPUS) + static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 allowed_flags) { u32 cpu; diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 0fe9afd4a591..133f189bf6d2 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -438,15 +438,15 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, u32 size; int cpu, off = 0; - if (unlikely((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS)) - /* unknown flags */ + if (unlikely(((map_flags & BPF_EXIST_FLAGS) == BPF_EXIST_FLAGS) || + ((u32)map_flags & ~(BPF_EXIST_FLAGS | BPF_CPU_FLAGS)))) return -EINVAL; if (unlikely(index >= array->map.max_entries)) /* all elements were pre-allocated, cannot insert a new one */ return -E2BIG; - if (unlikely(map_flags == BPF_NOEXIST)) + if (unlikely(map_flags & BPF_NOEXIST)) /* all elements already exist */ return -EEXIST; diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 66ebb5a32807..bf78e09f4d51 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -1367,11 +1367,11 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, static int check_flags(struct bpf_htab *htab, struct htab_elem *l_old, u64 map_flags) { - if (l_old && (map_flags & ~BPF_F_LOCK) == BPF_NOEXIST) + if (l_old && (map_flags & BPF_NOEXIST)) /* elem already exists */ return -EEXIST; - if (!l_old && (map_flags & ~BPF_F_LOCK) == BPF_EXIST) + if (!l_old && (map_flags & BPF_EXIST)) /* elem doesn't exist, cannot update it */ return -ENOENT; @@ -1560,9 +1560,11 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value static int htab_map_check_update_flags(bool onallcpus, u64 map_flags) { + if (unlikely((map_flags & BPF_EXIST_FLAGS) == BPF_EXIST_FLAGS)) + return -EINVAL; if (unlikely(!onallcpus && map_flags > BPF_EXIST)) return -EINVAL; - if (unlikely(onallcpus && ((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS))) + if (unlikely(onallcpus && ((u32)map_flags & ~(BPF_EXIST_FLAGS | BPF_CPU_FLAGS)))) return -EINVAL; return 0; } @@ -1661,7 +1663,7 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, * to remove older elem from htab and this removal * operation will need a bucket lock. */ - if (map_flags != BPF_EXIST) { + if (!(map_flags & BPF_EXIST)) { l_new = prealloc_lru_pop(htab, key, hash); if (!l_new) return -ENOMEM; diff --git a/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c b/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c index 7b4a1e24363b..67124dd4d8a7 100644 --- a/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c +++ b/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c @@ -449,6 +449,95 @@ static void test_lru_percpu_hash_cpu_flag_create(void) test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_LRU_PERCPU_HASH, 0); } +static bool verify_map_value_all_cpus(int map_fd, void *key, void *values, size_t elem_sz, + u32 expected_value, int nr_cpus) +{ + int i, err; + + err = bpf_map_lookup_elem(map_fd, key, values); + if (!ASSERT_OK(err, "bpf_map_lookup_elem all_cpus")) + return false; + + for (i = 0 ; i < nr_cpus ; i++) { + if (!ASSERT_EQ(*(u32 *)(values + elem_sz * i), expected_value, + "bpf_map_lookup_elem value all_cpus")) + return false; + } + + return true; +} + +static bool verify_map_value_first_cpu(int map_fd, void *key, void *values, size_t elem_sz, + u32 expected_value, int nr_cpus) +{ + int i, err; + + err = bpf_map_lookup_elem(map_fd, key, values); + if (!ASSERT_OK(err, "bpf_map_lookup_elem cpu")) + return false; + + if (!ASSERT_EQ(*(u32 *)values, expected_value, "bpf_map_lookup_elem value cpu")) + return false; + + for (i = 1 ; i < nr_cpus ; i++) { + if (!ASSERT_NEQ(*(u32 *)(values + elem_sz * i), expected_value, + "bpf_map_lookup_elem value cpu")) + return false; + } + + return true; +} + +static void test_percpu_cgroup_storage_flags_combination(struct bpf_map *map, int nr_cpus, + struct bpf_cgroup_storage_key *key) +{ + size_t value_sz = sizeof(u32), key_sz, elem_sz; + u32 *values, test_value = 0xAABBCCDD; + int err, map_fd; + u64 flags; + + key_sz = sizeof(*key); + elem_sz = roundup(value_sz, 8); + map_fd = bpf_map__fd(map); + values = calloc(nr_cpus, elem_sz); + if (!ASSERT_OK_PTR(values, "calloc values")) + return; + + flags = BPF_NOEXIST | BPF_EXIST; + err = bpf_map__update_elem(map, key, key_sz, values, elem_sz * nr_cpus, flags); + if (!ASSERT_EQ(err, -EINVAL, "bpf_map__update_elem noexist|exist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_NOEXIST; + err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); + if (!ASSERT_EQ(err, -EINVAL, "bpf_map__update_elem all_cpus|noexist")) + goto out; + + flags = BPF_F_CPU | BPF_NOEXIST; + err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); + if (!ASSERT_EQ(err, -EINVAL, "bpf_map__update_elem cpu|noexist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_EXIST; + values[0] = test_value; + err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); + if (!ASSERT_OK(err, "bpf_map__update_elem all_cpus|exist")) + goto out; + + if (!verify_map_value_all_cpus(map_fd, key, values, elem_sz, test_value, nr_cpus)) + goto out; + + flags = BPF_F_CPU | BPF_EXIST; + values[0] = --test_value; + err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); + if (!ASSERT_OK(err, "bpf_map__update_elem cpu|exist")) + goto out; + + verify_map_value_first_cpu(map_fd, key, values, elem_sz, test_value, nr_cpus); +out: + free(values); +} + static void test_percpu_cgroup_storage_cpu_flag(void) { struct percpu_alloc_array *skel = NULL; @@ -489,6 +578,7 @@ static void test_percpu_cgroup_storage_cpu_flag(void) goto out; test_percpu_map_op_cpu_flag(map, &key, sizeof(key), 1, nr_cpus, false); + test_percpu_cgroup_storage_flags_combination(map, nr_cpus, &key); out: bpf_prog_detach2(-1, cgroup, BPF_CGROUP_INET_EGRESS); close(cgroup); @@ -537,6 +627,164 @@ static void test_hash_cpu_flag(void) test_map_op_cpu_flag(BPF_MAP_TYPE_HASH); } +static void test_percpu_map_flags_combination(enum bpf_map_type map_type) +{ + u32 max_entries = 3, key = 0, first_value = 0xAABBCCDD, test_value = first_value; + size_t value_sz = sizeof(u32), elem_sz; + int err, map_fd, nr_cpus; + bool is_hash_map; + u32 *values; + u64 flags; + + is_hash_map = (map_type == BPF_MAP_TYPE_PERCPU_HASH || + map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH); + nr_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus")) + return; + + elem_sz = roundup(value_sz, 8); + values = calloc(nr_cpus, elem_sz); + if (!ASSERT_OK_PTR(values, "calloc values")) + return; + + map_fd = bpf_map_create(map_type, "test_flags_combination_map", + sizeof(u32), value_sz, max_entries, NULL); + if (!ASSERT_GE(map_fd, 0, "bpf_map_create")) { + free(values); + return; + } + + flags = BPF_NOEXIST | BPF_EXIST; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (!ASSERT_EQ(err, -EINVAL, "bpf_map_update_elem noexist|exist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_NOEXIST; + values[0] = test_value; + key = 0; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (is_hash_map) { + if (!ASSERT_OK(err, "bpf_map_update_elem all_cpus|noexist")) + goto out; + + if (!verify_map_value_all_cpus(map_fd, &key, values, elem_sz, test_value, nr_cpus)) + goto out; + + err = bpf_map_update_elem(map_fd, &key, values, flags); + } + if (!ASSERT_EQ(err, -EEXIST, "bpf_map_update_elem all_cpus|noexist")) + goto out; + + flags = BPF_F_CPU | BPF_NOEXIST; + values[0] = --test_value; + key = 1; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (is_hash_map) { + if (!ASSERT_OK(err, "bpf_map_update_elem cpu|noexist")) + goto out; + + if (!verify_map_value_first_cpu(map_fd, &key, values, elem_sz, test_value, nr_cpus)) + goto out; + + err = bpf_map_update_elem(map_fd, &key, values, flags); + } + if (!ASSERT_EQ(err, -EEXIST, "bpf_map_update_elem cpu|noexist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_EXIST; + values[0] = --test_value; + key = 2; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (is_hash_map) { + if (!ASSERT_EQ(err, -ENOENT, "bpf_map_update_elem all_cpus|exist")) + goto out; + } else { + if (!ASSERT_OK(err, "bpf_map_update_elem all_cpus|exist")) + goto out; + + if (!verify_map_value_all_cpus(map_fd, &key, values, elem_sz, test_value, nr_cpus)) + goto out; + } + + flags = BPF_F_CPU | BPF_EXIST; + values[0] = --test_value; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (is_hash_map) { + if (!ASSERT_EQ(err, -ENOENT, "bpf_map_update_elem cpu|exist")) + goto out; + } else { + if (!ASSERT_OK(err, "bpf_map_update_elem cpu|exist")) + goto out; + + if (!verify_map_value_first_cpu(map_fd, &key, values, elem_sz, test_value, nr_cpus)) + goto out; + } + + if (map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH) + goto out; + + /* Percpu LRU hash map should not delete old entries unnecessarily */ + flags = BPF_F_ALL_CPUS | BPF_NOEXIST; + key = 2; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (!ASSERT_OK(err, "bpf_map_update_elem lru_percpu_hash all_cpus|noexist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_EXIST; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (!ASSERT_OK(err, "bpf_map_update_elem lru_percpu_hash all_cpus|exist")) + goto out; + + key = 0; + verify_map_value_all_cpus(map_fd, &key, values, elem_sz, first_value, nr_cpus); + +out: + close(map_fd); + free(values); +} + +static bool pin_current_cpu(cpu_set_t *old_mask) +{ + cpu_set_t new_mask; + int err, cpu; + + err = sched_getaffinity(0, sizeof(*old_mask), old_mask); + if (!ASSERT_OK(err, "sched_getaffinity")) + return false; + + cpu = sched_getcpu(); + if (!ASSERT_GE(cpu, 0, "sched_getcpu")) + return false; + + CPU_ZERO(&new_mask); + CPU_SET(cpu, &new_mask); + + err = sched_setaffinity(0, sizeof(new_mask), &new_mask); + return ASSERT_OK(err, "sched_setaffinity"); +} + +static void test_percpu_hash_flags_combination(void) +{ + test_percpu_map_flags_combination(BPF_MAP_TYPE_PERCPU_HASH); +} + +static void test_lru_percpu_hash_flags_combination(void) +{ + cpu_set_t old_mask; + + if (!pin_current_cpu(&old_mask)) + return; + + test_percpu_map_flags_combination(BPF_MAP_TYPE_LRU_PERCPU_HASH); + + sched_setaffinity(0, sizeof(old_mask), &old_mask); +} + +static void test_percpu_array_flags_combination(void) +{ + test_percpu_map_flags_combination(BPF_MAP_TYPE_PERCPU_ARRAY); +} + void test_percpu_alloc(void) { if (test__start_subtest("array")) @@ -565,4 +813,10 @@ void test_percpu_alloc(void) test_array_cpu_flag(); if (test__start_subtest("cpu_flag_hash")) test_hash_cpu_flag(); + if (test__start_subtest("cpu_flag_combination_percpu_hash")) + test_percpu_hash_flags_combination(); + if (test__start_subtest("cpu_flag_combination_lru_percpu_hash")) + test_lru_percpu_hash_flags_combination(); + if (test__start_subtest("cpu_flag_combination_percpu_array")) + test_percpu_array_flags_combination(); }