diff --git a/tools/lib/bpf/features.c b/tools/lib/bpf/features.c index adcad221c601..4f19a0d79b0c 100644 --- a/tools/lib/bpf/features.c +++ b/tools/lib/bpf/features.c @@ -589,6 +589,32 @@ static int probe_uprobe_syscall(int token_fd) } #endif +static int probe_kern_btf_layout(int token_fd) +{ + static const char strs[] = "\0int"; + __u32 types[] = { + /* int */ + BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), + }; + struct btf_layout layout[] = { + { 0, 0, 0 }, + { sizeof(__u32), 0, 0 }, + }; + struct btf_header hdr = { + .magic = BTF_MAGIC, + .version = BTF_VERSION, + .hdr_len = sizeof(struct btf_header), + .type_len = sizeof(types), + .str_off = sizeof(types) + sizeof(layout), + .str_len = sizeof(strs), + .layout_off = sizeof(types), + .layout_len = sizeof(layout), + }; + + return probe_fd(libbpf__load_raw_btf_hdr(&hdr, (char *)types, strs, + (char *)layout, token_fd)); +} + typedef int (*feature_probe_fn)(int /* token_fd */); static struct kern_feature_cache feature_cache; @@ -670,6 +696,9 @@ static struct kern_feature_desc { [FEAT_UPROBE_SYSCALL] = { "kernel supports uprobe syscall", probe_uprobe_syscall, }, + [FEAT_BTF_LAYOUT] = { + "kernel supports BTF layout", probe_kern_btf_layout, + }, }; bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 1eaa7527d4da..9ea41f40dc82 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -3138,12 +3138,14 @@ static bool btf_needs_sanitization(struct bpf_object *obj) bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64); bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC); + bool has_layout = kernel_supports(obj, FEAT_BTF_LAYOUT); return !has_func || !has_datasec || !has_func_global || !has_float || - !has_decl_tag || !has_type_tag || !has_enum64 || !has_qmark_datasec; + !has_decl_tag || !has_type_tag || !has_enum64 || !has_qmark_datasec || + !has_layout; } -static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) +static struct btf *bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *orig_btf) { bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); @@ -3153,9 +3155,64 @@ static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64); bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC); + bool has_layout = kernel_supports(obj, FEAT_BTF_LAYOUT); int enum64_placeholder_id = 0; + const struct btf_header *hdr; + struct btf *btf = NULL; + const void *raw_data; struct btf_type *t; int i, j, vlen; + __u32 sz; + int err; + + /* clone BTF to sanitize a copy and leave the original intact */ + raw_data = btf__raw_data(orig_btf, &sz); + if (!raw_data) + return ERR_PTR(-ENOMEM); + /* btf_header() gives us endian-safe header info */ + hdr = btf_header(orig_btf); + + if (!has_layout && hdr->hdr_len >= sizeof(struct btf_header) && + (hdr->layout_len != 0 || hdr->layout_off != 0)) { + const struct btf_header *old_hdr = raw_data; + struct btf_header *new_hdr; + void *new_raw_data; + __u32 new_str_off; + + /* + * Need to rewrite BTF to exclude layout information and + * move string section to immediately after types. + */ + new_raw_data = malloc(sz); + if (!new_raw_data) + return ERR_PTR(-ENOMEM); + + memcpy(new_raw_data, raw_data, sz); + new_hdr = new_raw_data; + new_hdr->layout_off = 0; + new_hdr->layout_len = 0; + new_str_off = hdr->type_off + hdr->type_len; + /* Handle swapped endian case */ + if (old_hdr->magic != hdr->magic) + new_hdr->str_off = bswap_32(new_str_off); + else + new_hdr->str_off = new_str_off; + + memmove(new_raw_data + hdr->hdr_len + new_str_off, + new_raw_data + hdr->hdr_len + hdr->str_off, + hdr->str_len); + sz = hdr->hdr_len + hdr->type_off + hdr->type_len + hdr->str_len; + btf = btf__new(new_raw_data, sz); + free(new_raw_data); + } else { + btf = btf__new(raw_data, sz); + } + err = libbpf_get_error(btf); + if (err) + return ERR_PTR(err); + + /* enforce 8-byte pointers for BPF-targeted BTFs */ + btf__set_pointer_size(btf, 8); for (i = 1; i < btf__type_cnt(btf); i++) { t = (struct btf_type *)btf__type_by_id(btf, i); @@ -3233,9 +3290,10 @@ static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) if (enum64_placeholder_id == 0) { enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0); - if (enum64_placeholder_id < 0) - return enum64_placeholder_id; - + if (enum64_placeholder_id < 0) { + btf__free(btf); + return ERR_PTR(enum64_placeholder_id); + } t = (struct btf_type *)btf__type_by_id(btf, i); } @@ -3249,7 +3307,7 @@ static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) } } - return 0; + return btf; } static bool libbpf_needs_btf(const struct bpf_object *obj) @@ -3600,21 +3658,9 @@ static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) sanitize = btf_needs_sanitization(obj); if (sanitize) { - const void *raw_data; - __u32 sz; - - /* clone BTF to sanitize a copy and leave the original intact */ - raw_data = btf__raw_data(obj->btf, &sz); - kern_btf = btf__new(raw_data, sz); - err = libbpf_get_error(kern_btf); - if (err) - return err; - - /* enforce 8-byte pointers for BPF-targeted BTFs */ - btf__set_pointer_size(obj->btf, 8); - err = bpf_object__sanitize_btf(obj, kern_btf); - if (err) - return err; + kern_btf = bpf_object__sanitize_btf(obj, obj->btf); + if (IS_ERR(kern_btf)) + return PTR_ERR(kern_btf); } if (obj->gen_loader) { diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index 4bcb6ca69bb1..cabdaef79098 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -396,6 +396,8 @@ enum kern_feature_id { FEAT_LDIMM64_FULL_RANGE_OFF, /* Kernel supports uprobe syscall */ FEAT_UPROBE_SYSCALL, + /* Kernel supports BTF layout information */ + FEAT_BTF_LAYOUT, __FEAT_CNT, }; @@ -422,6 +424,10 @@ int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz); int libbpf__load_raw_btf(const char *raw_types, size_t types_len, const char *str_sec, size_t str_len, int token_fd); +int libbpf__load_raw_btf_hdr(const struct btf_header *hdr, + const char *raw_types, const char *str_sec, + const char *layout_sec, int token_fd); + int btf_load_into_kernel(struct btf *btf, char *log_buf, size_t log_sz, __u32 log_level, int token_fd); diff --git a/tools/lib/bpf/libbpf_probes.c b/tools/lib/bpf/libbpf_probes.c index bccf4bb747e1..b70d9637ecf5 100644 --- a/tools/lib/bpf/libbpf_probes.c +++ b/tools/lib/bpf/libbpf_probes.c @@ -218,6 +218,34 @@ int libbpf_probe_bpf_prog_type(enum bpf_prog_type prog_type, const void *opts) return libbpf_err(ret); } +int libbpf__load_raw_btf_hdr(const struct btf_header *hdr, const char *raw_types, + const char *str_sec, const char *layout_sec, + int token_fd) +{ + LIBBPF_OPTS(bpf_btf_load_opts, opts, + .token_fd = token_fd, + .btf_flags = token_fd ? BPF_F_TOKEN_FD : 0, + ); + int btf_fd, btf_len; + __u8 *raw_btf; + + btf_len = hdr->hdr_len + hdr->type_off + hdr->type_len + hdr->str_len + hdr->layout_len; + raw_btf = malloc(btf_len); + if (!raw_btf) + return -ENOMEM; + + memcpy(raw_btf, hdr, sizeof(*hdr)); + memcpy(raw_btf + hdr->hdr_len + hdr->type_off, raw_types, hdr->type_len); + memcpy(raw_btf + hdr->hdr_len + hdr->str_off, str_sec, hdr->str_len); + if (layout_sec) + memcpy(raw_btf + hdr->hdr_len + hdr->layout_off, layout_sec, hdr->layout_len); + + btf_fd = bpf_btf_load(raw_btf, btf_len, &opts); + + free(raw_btf); + return btf_fd; +} + int libbpf__load_raw_btf(const char *raw_types, size_t types_len, const char *str_sec, size_t str_len, int token_fd) @@ -230,26 +258,8 @@ int libbpf__load_raw_btf(const char *raw_types, size_t types_len, .str_off = types_len, .str_len = str_len, }; - LIBBPF_OPTS(bpf_btf_load_opts, opts, - .token_fd = token_fd, - .btf_flags = token_fd ? BPF_F_TOKEN_FD : 0, - ); - int btf_fd, btf_len; - __u8 *raw_btf; - btf_len = hdr.hdr_len + hdr.type_len + hdr.str_len; - raw_btf = malloc(btf_len); - if (!raw_btf) - return -ENOMEM; - - memcpy(raw_btf, &hdr, sizeof(hdr)); - memcpy(raw_btf + hdr.hdr_len, raw_types, hdr.type_len); - memcpy(raw_btf + hdr.hdr_len + hdr.type_len, str_sec, hdr.str_len); - - btf_fd = bpf_btf_load(raw_btf, btf_len, &opts); - - free(raw_btf); - return btf_fd; + return libbpf__load_raw_btf_hdr(&hdr, raw_types, str_sec, NULL, token_fd); } static int load_local_storage_btf(void)