Merge branch 'libbpf-add-bpf_program__clone-for-individual-program-loading'

Mykyta Yatsenko says:

====================
libbpf: Add bpf_program__clone() for individual program loading

This series adds bpf_program__clone() to libbpf and converts veristat to
use it, replacing the costly per-program object re-opening pattern.

veristat needs to load each BPF program in isolation to collect
per-program verification statistics. Previously it achieved this by
opening a fresh bpf_object for every program, disabling autoload on all
but the target, and loading the whole object. For object files with many
programs this meant repeating ELF parsing and BTF processing N times.

Patch 1 introduces bpf_program__clone(), which loads a single program
from a prepared object into the kernel and returns an fd owned by the
caller. It populates load parameters from the prepared object and lets
callers override any field via bpf_prog_load_opts. Fields written by the
prog_prepare_load_fn callback (expected_attach_type, attach_btf_id,
attach_btf_obj_fd) are seeded from prog/obj defaults before the
callback, then overridden with caller opts after, so explicit values
always win.

Patch 2 converts veristat to prepare the object once and clone each
program individually, eliminating redundant work.

Patch 3 adds a selftest verifying that caller-provided attach_btf_id
overrides are respected by bpf_program__clone().

Performance
Tested on selftests: 918 objects, ~4270 programs:
 - Wall time:   36.88s -> 23.18s (37% faster)
 - User time:   20.80s -> 16.07s (23% faster)
 - Kernel time: 12.07s ->  6.06s (50% faster)

Per-program loading also improves coverage: 83 programs that previously
failed now succeed.

Known regression:
 - Program-containing maps (PROG_ARRAY, DEVMAP, CPUMAP) track owner
   program type. Programs with incompatible attributes loaded against
   a shared map will be rejected. This is expected kernel behavior.

Signed-off-by: Mykyta Yatsenko <yatsenko@meta.com>
---
Changes in v5:
- Fix overriding of the attach_btf_id, attach_btf_fd, etc: the override
provided by the caller is applied after prog_prepare_load_fn().
- Added selftest to verify attach_btf_id override works as expected.
- Link to v4: https://lore.kernel.org/all/20260316-veristat_prepare-v3-0-94e5691e0494@meta.com/

Changes in v4:
- Replace OPTS_SET() with direct struct assignment for local
  bpf_prog_load_opts in bpf_program__clone() (libbpf.c)
- Remove unnecessary pattr pointer indirection (libbpf.c)
- Separate input and output fields in bpf_program__clone(): input
  fields (prog_flags, fd_array, etc.) are merged from caller opts
  before the callback; output fields (expected_attach_type,
  attach_btf_id, attach_btf_obj_fd) are initialized from prog/obj
  defaults for the callback, then overridden with caller opts after,
  so explicit caller values always win (libbpf.c)
- Add selftest for attach_btf_id override
- Link to v3: https://lore.kernel.org/r/20260206-veristat_prepare-a4a041873c53-v3@meta.com

Changes in v3:
- Clone fd_array_cnt in bpf_object__clone()
- In veristat do not fail if bpf_object__prepare() fails,
continue per-program processing to produce per program output
- Link to v2: https://lore.kernel.org/r/20260220-veristat_prepare-v2-0-15bff49022a7@meta.com

Changes in v2:
 - Removed map cloning entirely (libbpf.c)
 - Renamed bpf_prog_clone() -> bpf_program__clone()
 - Removed unnecessary obj NULL check (libbpf.c)
 - Fixed opts handling — no longer mutates caller's opts (libbpf.c)
 - Link to v1: https://lore.kernel.org/all/20260212-veristat_prepare-v1-0-c351023fb0db@meta.com/

---
====================

Link: https://patch.msgid.link/20260317-veristat_prepare-v4-0-74193d4cc9d9@meta.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

tools/lib/bpf/libbpf.c

@@ -9802,6 +9802,84 @@ __u32 bpf_program__line_info_cnt(const struct bpf_program *prog)
 	return prog->line_info_cnt;
 }
 
+int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts *opts)
+{
+	LIBBPF_OPTS(bpf_prog_load_opts, attr);
+	struct bpf_object *obj;
+	int err, fd;
+
+	if (!prog)
+		return libbpf_err(-EINVAL);
+
+	if (!OPTS_VALID(opts, bpf_prog_load_opts))
+		return libbpf_err(-EINVAL);
+
+	obj = prog->obj;
+	if (obj->state < OBJ_PREPARED)
+		return libbpf_err(-EINVAL);
+
+	/*
+	 * Caller-provided opts take priority; fall back to
+	 * prog/object defaults when the caller leaves them zero.
+	 */
+	attr.attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0) ?: prog->attach_prog_fd;
+	attr.prog_flags = OPTS_GET(opts, prog_flags, 0) ?: prog->prog_flags;
+	attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0) ?: prog->prog_ifindex;
+	attr.kern_version = OPTS_GET(opts, kern_version, 0) ?: obj->kern_version;
+	attr.fd_array = OPTS_GET(opts, fd_array, NULL) ?: obj->fd_array;
+	attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0) ?: obj->fd_array_cnt;
+	attr.token_fd = OPTS_GET(opts, token_fd, 0) ?: obj->token_fd;
+	if (attr.token_fd)
+		attr.prog_flags |= BPF_F_TOKEN_FD;
+
+	/* BTF func/line info */
+	if (obj->btf && btf__fd(obj->btf) >= 0) {
+		attr.prog_btf_fd = OPTS_GET(opts, prog_btf_fd, 0) ?: btf__fd(obj->btf);
+		attr.func_info = OPTS_GET(opts, func_info, NULL) ?: prog->func_info;
+		attr.func_info_cnt = OPTS_GET(opts, func_info_cnt, 0) ?: prog->func_info_cnt;
+		attr.func_info_rec_size =
+			OPTS_GET(opts, func_info_rec_size, 0) ?: prog->func_info_rec_size;
+		attr.line_info = OPTS_GET(opts, line_info, NULL) ?: prog->line_info;
+		attr.line_info_cnt = OPTS_GET(opts, line_info_cnt, 0) ?: prog->line_info_cnt;
+		attr.line_info_rec_size =
+			OPTS_GET(opts, line_info_rec_size, 0) ?: prog->line_info_rec_size;
+	}
+
+	attr.log_buf = OPTS_GET(opts, log_buf, NULL);
+	attr.log_size = OPTS_GET(opts, log_size, 0);
+	attr.log_level = OPTS_GET(opts, log_level, 0);
+
+	/*
+	 * Fields below may be mutated by prog_prepare_load_fn:
+	 * Seed them from prog/obj defaults here;
+	 * Later override with caller-provided opts.
+	 */
+	attr.expected_attach_type = prog->expected_attach_type;
+	attr.attach_btf_id = prog->attach_btf_id;
+	attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
+
+	if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
+		err = prog->sec_def->prog_prepare_load_fn(prog, &attr, prog->sec_def->cookie);
+		if (err)
+			return libbpf_err(err);
+	}
+
+	/* Re-apply caller overrides for output fields */
+	if (OPTS_GET(opts, expected_attach_type, 0))
+		attr.expected_attach_type =
+			OPTS_GET(opts, expected_attach_type, 0);
+	if (OPTS_GET(opts, attach_btf_id, 0))
+		attr.attach_btf_id = OPTS_GET(opts, attach_btf_id, 0);
+	if (OPTS_GET(opts, attach_btf_obj_fd, 0))
+		attr.attach_btf_obj_fd =
+			OPTS_GET(opts, attach_btf_obj_fd, 0);
+
+	fd = bpf_prog_load(prog->type, prog->name, obj->license, prog->insns, prog->insns_cnt,
+			   &attr);
+
+	return libbpf_err(fd);
+}
+
 #define SEC_DEF(sec_pfx, ptype, atype, flags, ...) {			    \
 	.sec = (char *)sec_pfx,						    \
 	.prog_type = BPF_PROG_TYPE_##ptype,				    \

tools/lib/bpf/libbpf.h

@@ -2021,6 +2021,23 @@ LIBBPF_API int libbpf_register_prog_handler(const char *sec,
  */
 LIBBPF_API int libbpf_unregister_prog_handler(int handler_id);
 
+/**
+ * @brief **bpf_program__clone()** loads a single BPF program from a prepared
+ * BPF object into the kernel, returning its file descriptor.
+ *
+ * The BPF object must have been previously prepared with
+ * **bpf_object__prepare()**. If @opts is provided, any non-zero field
+ * overrides the defaults derived from the program/object internals.
+ * If @opts is NULL, all fields are populated automatically.
+ *
+ * The returned FD is owned by the caller and must be closed with close().
+ *
+ * @param prog BPF program from a prepared object
+ * @param opts Optional load options; non-zero fields override defaults
+ * @return program FD (>= 0) on success; negative error code on failure
+ */
+LIBBPF_API int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts *opts);
+
 #ifdef __cplusplus
 } /* extern "C" */
 #endif

tools/lib/bpf/libbpf.map

@@ -452,6 +452,7 @@ LIBBPF_1.7.0 {
 		bpf_map__set_exclusive_program;
 		bpf_map__exclusive_program;
 		bpf_prog_assoc_struct_ops;
+		bpf_program__clone;
 		bpf_program__assoc_struct_ops;
 		btf__permute;
 } LIBBPF_1.6.0;

tools/testing/selftests/bpf/prog_tests/clone_attach_btf_id.c

@@ -0,0 +1,78 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2025 Meta */
+#include <test_progs.h>
+#include "clone_attach_btf_id.skel.h"
+
+/*
+ * Test that bpf_program__clone() respects caller-provided attach_btf_id
+ * override via bpf_prog_load_opts.
+ *
+ * The BPF program has SEC("fentry/bpf_fentry_test1"). Clone it twice
+ * from the same prepared object: first with no opts (callback resolves
+ * attach_btf_id from sec_name), then with attach_btf_id overridden to
+ * bpf_fentry_test2. Verify each loaded program's attach_btf_id via
+ * bpf_prog_get_info_by_fd().
+ */
+
+static int get_prog_attach_btf_id(int prog_fd)
+{
+	struct bpf_prog_info info = {};
+	__u32 info_len = sizeof(info);
+	int err;
+
+	err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len);
+	if (err)
+		return err;
+	return info.attach_btf_id;
+}
+
+void test_clone_attach_btf_id(void)
+{
+	struct clone_attach_btf_id *skel;
+	int fd1 = -1, fd2 = -1, err;
+	int btf_id_test1, btf_id_test2;
+
+	btf_id_test1 = libbpf_find_vmlinux_btf_id("bpf_fentry_test1", BPF_TRACE_FENTRY);
+	if (!ASSERT_GT(btf_id_test1, 0, "find_btf_id_test1"))
+		return;
+
+	btf_id_test2 = libbpf_find_vmlinux_btf_id("bpf_fentry_test2", BPF_TRACE_FENTRY);
+	if (!ASSERT_GT(btf_id_test2, 0, "find_btf_id_test2"))
+		return;
+
+	skel = clone_attach_btf_id__open();
+	if (!ASSERT_OK_PTR(skel, "skel_open"))
+		return;
+
+	err = bpf_object__prepare(skel->obj);
+	if (!ASSERT_OK(err, "obj_prepare"))
+		goto out;
+
+	/* Clone with no opts — callback resolves BTF from sec_name */
+	fd1 = bpf_program__clone(skel->progs.fentry_handler, NULL);
+	if (!ASSERT_GE(fd1, 0, "clone_default"))
+		goto out;
+	ASSERT_EQ(get_prog_attach_btf_id(fd1), btf_id_test1,
+		  "attach_btf_id_default");
+
+	/*
+	 * Clone with attach_btf_id override pointing to a different
+	 * function. The BPF program never accesses arguments, so the
+	 * load succeeds regardless of signature mismatch.
+	 */
+	LIBBPF_OPTS(bpf_prog_load_opts, opts,
+		    .attach_btf_id = btf_id_test2,
+	);
+	fd2 = bpf_program__clone(skel->progs.fentry_handler, &opts);
+	if (!ASSERT_GE(fd2, 0, "clone_override"))
+		goto out;
+	ASSERT_EQ(get_prog_attach_btf_id(fd2), btf_id_test2,
+		  "attach_btf_id_override");
+
+out:
+	if (fd1 >= 0)
+		close(fd1);
+	if (fd2 >= 0)
+		close(fd2);
+	clone_attach_btf_id__destroy(skel);
+}

tools/testing/selftests/bpf/progs/clone_attach_btf_id.c

@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2025 Meta */
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+char _license[] SEC("license") = "GPL";
+
+SEC("fentry/bpf_fentry_test1")
+int BPF_PROG(fentry_handler, int a)
+{
+	return 0;
+}

tools/testing/selftests/bpf/veristat.c

@@ -1236,7 +1236,7 @@ static void mask_unrelated_struct_ops_progs(struct bpf_object *obj,
 	}
 }
 
-static void fixup_obj(struct bpf_object *obj, struct bpf_program *prog, const char *filename)
+static void fixup_obj_maps(struct bpf_object *obj)
 {
 	struct bpf_map *map;
 
@@ -1251,15 +1251,23 @@ static void fixup_obj(struct bpf_object *obj, struct bpf_program *prog, const ch
 		case BPF_MAP_TYPE_INODE_STORAGE:
 		case BPF_MAP_TYPE_CGROUP_STORAGE:
 		case BPF_MAP_TYPE_CGRP_STORAGE:
-			break;
 		case BPF_MAP_TYPE_STRUCT_OPS:
-			mask_unrelated_struct_ops_progs(obj, map, prog);
 			break;
 		default:
 			if (bpf_map__max_entries(map) == 0)
 				bpf_map__set_max_entries(map, 1);
 		}
 	}
+}
+
+static void fixup_obj(struct bpf_object *obj, struct bpf_program *prog, const char *filename)
+{
+	struct bpf_map *map;
+
+	bpf_object__for_each_map(map, obj) {
+		if (bpf_map__type(map) == BPF_MAP_TYPE_STRUCT_OPS)
+			mask_unrelated_struct_ops_progs(obj, map, prog);
+	}
 
 	/* SEC(freplace) programs can't be loaded with veristat as is,
 	 * but we can try guessing their target program's expected type by
@@ -1608,6 +1616,7 @@ static int process_prog(const char *filename, struct bpf_object *obj, struct bpf
 	const char *base_filename = basename(strdupa(filename));
 	const char *prog_name = bpf_program__name(prog);
 	long mem_peak_a, mem_peak_b, mem_peak = -1;
+	LIBBPF_OPTS(bpf_prog_load_opts, opts);
 	char *buf;
 	int buf_sz, log_level;
 	struct verif_stats *stats;
@@ -1647,9 +1656,6 @@ static int process_prog(const char *filename, struct bpf_object *obj, struct bpf
 	}
 	verif_log_buf[0] = '\0';
 
-	bpf_program__set_log_buf(prog, buf, buf_sz);
-	bpf_program__set_log_level(prog, log_level);
-
 	/* increase chances of successful BPF object loading */
 	fixup_obj(obj, prog, base_filename);
 
@@ -1658,15 +1664,21 @@ static int process_prog(const char *filename, struct bpf_object *obj, struct bpf
 	if (env.force_reg_invariants)
 		bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_REG_INVARIANTS);
 
-	err = bpf_object__prepare(obj);
-	if (!err) {
-		cgroup_err = reset_stat_cgroup();
-		mem_peak_a = cgroup_memory_peak();
-		err = bpf_object__load(obj);
-		mem_peak_b = cgroup_memory_peak();
-		if (!cgroup_err && mem_peak_a >= 0 && mem_peak_b >= 0)
-			mem_peak = mem_peak_b - mem_peak_a;
+	opts.log_buf = buf;
+	opts.log_size = buf_sz;
+	opts.log_level = log_level;
+
+	cgroup_err = reset_stat_cgroup();
+	mem_peak_a = cgroup_memory_peak();
+	fd = bpf_program__clone(prog, &opts);
+	if (fd < 0) {
+		err = fd;
+		fprintf(stderr, "Failed to load program %s %d\n", prog_name, err);
 	}
+	mem_peak_b = cgroup_memory_peak();
+	if (!cgroup_err && mem_peak_a >= 0 && mem_peak_b >= 0)
+		mem_peak = mem_peak_b - mem_peak_a;
+
 	env.progs_processed++;
 
 	stats->file_name = strdup(base_filename);
@@ -1678,7 +1690,6 @@ static int process_prog(const char *filename, struct bpf_object *obj, struct bpf
 	stats->stats[MEMORY_PEAK] = mem_peak < 0 ? -1 : mem_peak / (1024 * 1024);
 
 	memset(&info, 0, info_len);
-	fd = bpf_program__fd(prog);
 	if (fd > 0 && bpf_prog_get_info_by_fd(fd, &info, &info_len) == 0) {
 		stats->stats[JITED_SIZE] = info.jited_prog_len;
 		if (env.dump_mode & DUMP_JITED)
@@ -1699,7 +1710,8 @@ static int process_prog(const char *filename, struct bpf_object *obj, struct bpf
 
 	if (verif_log_buf != buf)
 		free(buf);
-
+	if (fd > 0)
+		close(fd);
 	return 0;
 }
 
@@ -2182,8 +2194,8 @@ static int set_global_vars(struct bpf_object *obj, struct var_preset *presets, i
 static int process_obj(const char *filename)
 {
 	const char *base_filename = basename(strdupa(filename));
-	struct bpf_object *obj = NULL, *tobj;
-	struct bpf_program *prog, *tprog, *lprog;
+	struct bpf_object *obj = NULL;
+	struct bpf_program *prog;
 	libbpf_print_fn_t old_libbpf_print_fn;
 	LIBBPF_OPTS(bpf_object_open_opts, opts);
 	int err = 0, prog_cnt = 0;
@@ -2222,51 +2234,24 @@ static int process_obj(const char *filename)
 	env.files_processed++;
 
 	bpf_object__for_each_program(prog, obj) {
+		bpf_program__set_autoload(prog, true);
 		prog_cnt++;
 	}
 
-	if (prog_cnt == 1) {
-		prog = bpf_object__next_program(obj, NULL);
-		bpf_program__set_autoload(prog, true);
-		err = set_global_vars(obj, env.presets, env.npresets);
-		if (err) {
-			fprintf(stderr, "Failed to set global variables %d\n", err);
-			goto cleanup;
-		}
-		process_prog(filename, obj, prog);
+	fixup_obj_maps(obj);
+
+	err = set_global_vars(obj, env.presets, env.npresets);
+	if (err) {
+		fprintf(stderr, "Failed to set global variables %d\n", err);
 		goto cleanup;
 	}
 
+	err = bpf_object__prepare(obj);
+	if (err) /* run process_prog() anyway to output per program failures */
+		fprintf(stderr, "Failed to prepare BPF object for loading %d\n", err);
+
 	bpf_object__for_each_program(prog, obj) {
-		const char *prog_name = bpf_program__name(prog);
-
-		tobj = bpf_object__open_file(filename, &opts);
-		if (!tobj) {
-			err = -errno;
-			fprintf(stderr, "Failed to open '%s': %d\n", filename, err);
-			goto cleanup;
-		}
-
-		err = set_global_vars(tobj, env.presets, env.npresets);
-		if (err) {
-			fprintf(stderr, "Failed to set global variables %d\n", err);
-			goto cleanup;
-		}
-
-		lprog = NULL;
-		bpf_object__for_each_program(tprog, tobj) {
-			const char *tprog_name = bpf_program__name(tprog);
-
-			if (strcmp(prog_name, tprog_name) == 0) {
-				bpf_program__set_autoload(tprog, true);
-				lprog = tprog;
-			} else {
-				bpf_program__set_autoload(tprog, false);
-			}
-		}
-
-		process_prog(filename, tobj, lprog);
-		bpf_object__close(tobj);
+		process_prog(filename, obj, prog);
 	}
 
 cleanup:
@@ -3264,17 +3249,14 @@ static int handle_verif_mode(void)
 	create_stat_cgroup();
 	for (i = 0; i < env.filename_cnt; i++) {
 		err = process_obj(env.filenames[i]);
-		if (err) {
+		if (err)
 			fprintf(stderr, "Failed to process '%s': %d\n", env.filenames[i], err);
-			goto out;
-		}
 	}
 
 	qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats);
 
 	output_prog_stats();
 
-out:
 	destroy_stat_cgroup();
 	return err;
 }