diff --git a/tools/lib/bpf/features.c b/tools/lib/bpf/features.c index 9f589980beef..adb5d2e7fdfa 100644 --- a/tools/lib/bpf/features.c +++ b/tools/lib/bpf/features.c @@ -77,6 +77,82 @@ static int probe_kern_global_data(int token_fd) return probe_fd(ret); } +/* + * A continuous fd_array binds every map in it to the program, whether an + * insn refers to the map or not. Kernels without the attribute reject a + * non-zero fd_array_cnt. + */ +static int probe_kern_fd_array_cnt(int token_fd) +{ + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + LIBBPF_OPTS(bpf_map_create_opts, map_opts, + .token_fd = token_fd, + .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, + ); + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, + .token_fd = token_fd, + .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, + .fd_array_cnt = 1, + ); + int ret, map, insn_cnt = ARRAY_SIZE(insns); + + map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), sizeof(int), 1, + &map_opts); + if (map < 0) { + ret = -errno; + pr_warn("Error in %s(): %s. Couldn't create simple array map.\n", + __func__, errstr(ret)); + return ret; + } + + prog_opts.fd_array = ↦ + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); + close(map); + return probe_fd(ret); +} + +/* + * The kernel checks an exclusive map bound through fd_array_cnt against the + * digest of the final program. Earlier kernels compared it as the map was + * added, against a digest they had not computed yet, and refused the load for + * the very program the map is exclusive to; kernels without exclusive maps + * refuse the map. + */ +static int probe_kern_excl_map_fd_array_cnt(int token_fd) +{ + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + __u8 hash[SHA256_DIGEST_LENGTH]; + LIBBPF_OPTS(bpf_map_create_opts, map_opts, + .token_fd = token_fd, + .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, + .excl_prog_hash = hash, + .excl_prog_hash_size = sizeof(hash), + ); + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, + .token_fd = token_fd, + .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, + .fd_array_cnt = 1, + ); + int ret, map, insn_cnt = ARRAY_SIZE(insns); + + libbpf_sha256(insns, sizeof(insns), hash); + map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_excl", sizeof(int), sizeof(int), 1, + &map_opts); + if (map < 0) + return 0; + + prog_opts.fd_array = ↦ + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); + close(map); + return probe_fd(ret); +} + static int probe_kern_btf(int token_fd) { static const char strs[] = "\0int"; @@ -742,6 +818,12 @@ static struct kern_feature_desc { [FEAT_PERCPU_DATA] = { "kernel supports percpu data", probe_kern_percpu_data, }, + [FEAT_FD_ARRAY_CNT] = { + "continuous fd_array (fd_array_cnt)", probe_kern_fd_array_cnt, + }, + [FEAT_EXCL_MAP_FD_ARRAY_CNT] = { + "exclusive map bound through fd_array_cnt", probe_kern_excl_map_fd_array_cnt, + }, }; bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index fcedb78abe00..8fcb8cf39cc0 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -797,7 +797,8 @@ struct bpf_object { /* * Slots handed out so far, slot 0 included. A kfunc insn names a module * BTF slot through its offset, slot 0 stands for vmlinux BTF and stays - * empty, so the array goes to the kernel without a count. + * empty, so the array goes to the kernel without a count, unless + * bpf_object__bind_arena() puts the arena of the object there. */ size_t fd_array_used; @@ -8804,6 +8805,56 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz); +/* + * Bind the arena of the object to the program about to be loaded. A program + * has an arena only when an insn refers to the map, but the kernel can hand + * it arena pointers all the same, as struct_ops arguments, through tagged + * struct members or as kfunc return values, and the verifier rejects those + * when the program has no arena to place them in. A continuous fd_array + * binds every map in it, the arena takes slot 0, which is free: a kfunc insn + * with offset 0 names vmlinux BTF rather than a slot. + */ +static int bpf_object__bind_arena(struct bpf_object *obj, struct bpf_program *prog, + struct bpf_prog_load_opts *load_attr) +{ + struct bpf_map *map; + int err; + + if (obj->gen_loader || obj->arena_map_idx < 0 || !kernel_supports(obj, FEAT_FD_ARRAY_CNT)) + return 0; + map = &obj->maps[obj->arena_map_idx]; + /* + * Nothing to bind when the arena is not created, or no longer is after + * a failed load, when it is exclusive to another program, and nothing + * for an offloaded program, which the kernel does not let near an + * arena. + */ + if (!map->autocreate || map->fd < 0 || (map->excl_prog && map->excl_prog != prog) || + load_attr->prog_ifindex) + return 0; + /* + * An arena exclusive to the program itself needs a kernel that checks + * the map against the digest of the final program. Earlier kernels + * compared a map bound through the fd_array against a digest they had + * not computed yet and refused the load; there the program binds its + * arena through its insns, as before. + */ + if (map->excl_prog && !kernel_supports(obj, FEAT_EXCL_MAP_FD_ARRAY_CNT)) + return 0; + + if (!obj->fd_array_used) { + err = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int), 1); + if (err) + return err; + obj->fd_array_used = 1; + } + obj->fd_array[0] = map->fd; + load_attr->fd_array = obj->fd_array; + /* with slot 0 filled every slot handed out is valid, so the array can go with a count */ + load_attr->fd_array_cnt = obj->fd_array_used; + return 0; +} + static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, const char *license, __u32 kern_version, int *prog_fd) @@ -8867,6 +8918,9 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog if (obj->data_in_arena) load_attr.prog_flags |= BPF_F_ARENA_SCALAR; load_attr.fd_array = obj->fd_array; + err = bpf_object__bind_arena(obj, prog, &load_attr); + if (err) + return err; load_attr.token_fd = obj->token_fd; if (obj->token_fd) @@ -9966,8 +10020,10 @@ static void bpf_object_cleanup_btf(struct bpf_object *obj) static void bpf_object_post_load_cleanup(struct bpf_object *obj) { - /* clean up fd_array */ + /* clean up fd_array, and the slots handed out with it */ zfree(&obj->fd_array); + obj->fd_array_cap = 0; + obj->fd_array_used = 0; /* clean up BTF */ bpf_object_cleanup_btf(obj); @@ -10946,6 +11002,12 @@ int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts attr.kern_version = OPTS_GET(opts, kern_version, 0) ?: obj->kern_version; attr.fd_array = OPTS_GET(opts, fd_array, NULL) ?: obj->fd_array; attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0); + /* the arena of the object goes with the program, as in bpf_object__load() */ + if (!OPTS_GET(opts, fd_array, NULL) && !OPTS_GET(opts, fd_array_cnt, 0)) { + err = bpf_object__bind_arena(obj, prog, &attr); + if (err) + return libbpf_err(err); + } attr.token_fd = OPTS_GET(opts, token_fd, 0) ?: obj->token_fd; if (attr.token_fd) attr.prog_flags |= BPF_F_TOKEN_FD; diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index 4e45dfacb548..e90b5336bac0 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -404,6 +404,10 @@ enum kern_feature_id { FEAT_BPF_SYSCALL_COMMON_ATTRS, /* Kernel supports percpu data */ FEAT_PERCPU_DATA, + /* Kernel binds every map in a continuous fd_array (fd_array_cnt) to the program */ + FEAT_FD_ARRAY_CNT, + /* Kernel checks an exclusive map bound through fd_array_cnt against the final digest */ + FEAT_EXCL_MAP_FD_ARRAY_CNT, __FEAT_CNT, };