Merge branch 'bpf-fix-trampoline-handling-of-128-bit-values'

Yonghong Song says:

====================
bpf: Fix trampoline handling of 128-bit values

The BPF trampoline preserves only 8 bytes of a target function's return
value (R0), and its register save area under-allocates space for 128-bit
arguments for x86_64. These two problems lead to memory corruption or
incorrect values observed by BPF programs and the real caller.

This series fixes both issues and adds two selftests, otherwise, each of
them will fail if without the corresponding fix.

Changelogs:
  v4 -> v5:
    - v4: https://lore.kernel.org/bpf/1c4223ae-a5ba-48a4-95d3-57c8ff241055@linux.dev/
    - For function test_fexit_int128_ret(), guard with __x86_64__ and __aarch64__
      to avoid s390x failure
  v3 -> v4:
    - v3: https://lore.kernel.org/bpf/20260710225206.4013062-1-yonghong.song@linux.dev/
    - Add Ack from Leon Hwang
  v2 -> v3:
    - v2: https://lore.kernel.org/bpf/20260710182204.1085329-1-yonghong.song@linux.dev/
    - Align __int128 argument at even position enforced by arm64.
  v1 -> v2:
    - v1: https://lore.kernel.org/bpf/20260710144404.2579671-1-yonghong.song@linux.dev/
    - Also handle __int128 arguments for x86_64.
====================

Link: https://patch.msgid.link/20260729050154.2585468-1-yonghong.song@linux.dev
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>

arch/x86/net/bpf_jit_comp.c

@@ -3369,11 +3369,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
 	WARN_ON_ONCE((flags & BPF_TRAMP_F_INDIRECT) &&
 		     (flags & ~(BPF_TRAMP_F_INDIRECT | BPF_TRAMP_F_RET_FENTRY_RET)));
 
-	/* extra registers for struct arguments */
-	for (i = 0; i < m->nr_args; i++) {
-		if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG)
-			nr_regs += (m->arg_size[i] + 7) / 8 - 1;
-	}
+	for (i = 0; i < m->nr_args; i++)
+		nr_regs += (m->arg_size[i] + 7) / 8 - 1;
 
 	/* x86-64 supports up to MAX_BPF_FUNC_ARGS arguments. 1-6
 	 * are passed through regs, the remains are through stack.

kernel/bpf/bpf_struct_ops.c

@@ -445,6 +445,18 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,
 			goto errout;
 		}
 
+		/*
+		 * A >8 byte return value is passed back in a register pair,
+		 * which the struct_ops trampoline does not preserve (only
+		 * 8 bytes of the return value are saved and restored).
+		 */
+		if (st_ops->func_models[i].ret_size > 8) {
+			pr_warn("func ptr %s in struct %s has a >8 byte return value, which is not supported\n",
+				mname, st_ops->name);
+			err = -EOPNOTSUPP;
+			goto errout;
+		}
+
 		stub_func_addr = *(void **)(st_ops->cfi_stubs + moff);
 		err = prepare_arg_info(btf, st_ops->name, mname,
 				       func_proto, stub_func_addr,

kernel/bpf/verifier.c

@@ -19027,6 +19027,20 @@ btf_attach_func_proto(struct bpf_verifier_log *log, struct btf *btf, u32 func_id
 	return btf_type_by_id(btf, func->type);
 }
 
+static bool attach_uses_trampoline_retval(enum bpf_attach_type type)
+{
+	switch (type) {
+	case BPF_MODIFY_RETURN:
+	case BPF_TRACE_FEXIT:
+	case BPF_TRACE_FEXIT_MULTI:
+	case BPF_TRACE_FSESSION:
+	case BPF_TRACE_FSESSION_MULTI:
+		return true;
+	default:
+		return false;
+	}
+}
+
 int bpf_check_attach_target(struct bpf_verifier_log *log,
 			    const struct bpf_prog *prog,
 			    const struct bpf_prog *tgt_prog,
@@ -19291,6 +19305,14 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
 		if (ret < 0)
 			return ret;
 
+		if (tgt_info->fmodel.ret_size > 8 &&
+		    attach_uses_trampoline_retval(prog->expected_attach_type)) {
+			bpf_log(log,
+				"Attach to function %s with a >8 byte return value is not supported for this attach type\n",
+				tname);
+			return -EOPNOTSUPP;
+		}
+
 		/*
 		 * *.multi programs don't need an address during program
 		 * verification, we just take the module ref if needed.
@@ -19565,6 +19587,9 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt
 	err = btf_distill_func_proto(NULL, btf, t, tname, &tgt_info->fmodel);
 	if (err < 0)
 		return err;
+	if (tgt_info->fmodel.ret_size > 8 &&
+	    attach_uses_trampoline_retval(prog->expected_attach_type))
+		return -EOPNOTSUPP;
 	if (btf_is_module(btf)) {
 		/* The bpf program already holds reference to module. */
 		if (WARN_ON_ONCE(!prog->aux->mod))

tools/testing/selftests/bpf/prog_tests/tracing_failure.c

@@ -76,6 +76,24 @@ static void test_fexit_noreturns(void)
 			       "Attaching fexit/fsession/fmod_ret to __noreturn function 'do_exit' is rejected.");
 }
 
+static void test_fexit_int128_ret(void)
+{
+	/*
+	 * __int128 is returned in a register pair on x86_64 and arm64, so
+	 * bpf_testmod_test_int128_ret() is BTF-encoded and attachable and the
+	 * verifier can reject its >8 byte return value. Other architectures
+	 * return a __int128 differently (e.g. s390x returns larger values by
+	 * reference, which makes pahole skip BTF encoding of the function), so
+	 * only exercise this on x86_64 and arm64.
+	 */
+#if defined(__x86_64__) || defined(__aarch64__)
+	test_tracing_fail_prog("fexit_int128_ret",
+			       "with a >8 byte return value is not supported for this attach type");
+#else
+	test__skip();
+#endif
+}
+
 void test_tracing_failure(void)
 {
 	if (test__start_subtest("bpf_spin_lock"))
@@ -86,4 +104,6 @@ void test_tracing_failure(void)
 		test_tracing_deny();
 	if (test__start_subtest("fexit_noreturns"))
 		test_fexit_noreturns();
+	if (test__start_subtest("fexit_int128_ret"))
+		test_fexit_int128_ret();
 }

tools/testing/selftests/bpf/prog_tests/tracing_struct.c

@@ -4,6 +4,7 @@
 #include <test_progs.h>
 #include "tracing_struct.skel.h"
 #include "tracing_struct_many_args.skel.h"
+#include "tracing_struct_int128.skel.h"
 
 static void test_struct_args(void)
 {
@@ -112,6 +113,39 @@ static void test_struct_many_args(void)
 	tracing_struct_many_args__destroy(skel);
 }
 
+static void test_int128_args(void)
+{
+	/*
+	 * __int128 arguments are passed in a register pair on x86_64 and
+	 * arm64, which the trampoline packs into two context slots. Other
+	 * architectures pass a __int128 differently (e.g. s390x passes larger
+	 * arguments by reference), so only exercise this on x86_64 and arm64.
+	 */
+#if defined(__x86_64__) || defined(__aarch64__)
+	struct tracing_struct_int128 *skel;
+	int err;
+
+	skel = tracing_struct_int128__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "tracing_struct_int128__open_and_load"))
+		return;
+
+	err = tracing_struct_int128__attach(skel);
+	if (!ASSERT_OK(err, "tracing_struct_int128__attach"))
+		goto destroy_skel;
+
+	ASSERT_OK(trigger_module_test_read(256), "trigger_read");
+
+	ASSERT_EQ(skel->bss->t_b, 2, "t:b");
+	ASSERT_EQ(skel->bss->t_c, 3, "t:c");
+	ASSERT_EQ(skel->bss->t_ret, 6, "t ret");
+
+destroy_skel:
+	tracing_struct_int128__destroy(skel);
+#else
+	test__skip();
+#endif
+}
+
 static void test_union_args(void)
 {
 	struct tracing_struct *skel;
@@ -145,6 +179,8 @@ void test_tracing_struct(void)
 		test_struct_args();
 	if (test__start_subtest("struct_many_args"))
 		test_struct_many_args();
+	if (test__start_subtest("int128_args"))
+		test_int128_args();
 	if (test__start_subtest("union_args"))
 		test_union_args();
 }

tools/testing/selftests/bpf/progs/tracing_failure.c

@@ -30,3 +30,9 @@ int BPF_PROG(fexit_noreturns)
 {
 	return 0;
 }
+
+SEC("?fexit/bpf_testmod_test_int128_ret")
+int BPF_PROG(fexit_int128_ret)
+{
+	return 0;
+}

tools/testing/selftests/bpf/progs/tracing_struct_int128.c

@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
+#include <vmlinux.h>
+#include <bpf/bpf_tracing.h>
+#include <bpf/bpf_helpers.h>
+
+long t_b, t_c, t_ret;
+
+SEC("fexit/bpf_testmod_test_int128_arg")
+int test_int128_arg_fexit(unsigned long long *ctx)
+{
+	t_b = (int)ctx[2];
+	t_c = (long)ctx[3];
+	t_ret = (long)ctx[4];
+	return 0;
+}
+
+char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/test_kmods/bpf_testmod.c

@@ -161,6 +161,33 @@ bpf_testmod_test_arg_ptr_to_struct(struct bpf_testmod_struct_arg_1 *a) {
 	return bpf_testmod_test_struct_arg_result;
 }
 
+#ifdef __SIZEOF_INT128__
+noinline __int128
+bpf_testmod_test_int128_ret(int a)
+{
+	bpf_testmod_test_struct_arg_result = a;
+	return (__int128)a;
+}
+
+/*
+ * The __int128 'a' is the first argument on purpose. On arm64 a 16-byte
+ * argument must start in an even-numbered register pair, so placing it
+ * after a single-register scalar would leave a padding register (x1)
+ * unused. pahole maps parameters to registers positionally and would then
+ * see the following argument in an "unexpected" register and skip BTF
+ * encoding of the whole function, making it unattachable. Keeping the
+ * __int128 first (x0:x1) avoids the padding while still exercising the
+ * trampoline packing of a 128-bit argument together with the trailing
+ * int and long arguments.
+ */
+noinline long
+bpf_testmod_test_int128_arg(__int128 a, int b, long c)
+{
+	bpf_testmod_test_struct_arg_result = (long)a + b + c;
+	return bpf_testmod_test_struct_arg_result;
+}
+#endif
+
 __weak noinline void bpf_testmod_looooooooooooooooooooooooooooooong_name(void)
 {
 }
@@ -514,6 +541,11 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj,
 
 	(void)bpf_testmod_test_arg_ptr_to_struct(&struct_arg1_2);
 
+#ifdef __SIZEOF_INT128__
+	(void)bpf_testmod_test_int128_ret(i);
+	(void)bpf_testmod_test_int128_arg((__int128)1, 2, 3);
+#endif
+
 	(void)trace_bpf_testmod_test_raw_tp_null_tp(NULL);
 
 	bpf_testmod_test_struct_ops3();