diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index 1d5f4cf88d33..b3efe0668e86 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -11,6 +11,7 @@
 #include <linux/bitfield.h>
 #include <linux/bpf.h>
 #include <linux/cfi.h>
+#include <linux/extable.h>
 #include <linux/filter.h>
 #include <linux/memory.h>
 #include <linux/printk.h>
@@ -1044,8 +1045,9 @@ static void __maybe_unused build_bhb_mitigation(struct jit_ctx *ctx)
 	}
 }
 
-static void build_epilogue(struct jit_ctx *ctx, bool was_classic)
+static void build_epilogue(struct jit_ctx *ctx, bool was_classic, bool unwind)
 {
+	const u8 tmp = bpf2a64[TMP_REG_1];
 	const u8 r0 = bpf2a64[BPF_REG_0];
 	const u8 ptr = bpf2a64[TCCNT_PTR];
 
@@ -1073,6 +1075,10 @@ static void build_epilogue(struct jit_ctx *ctx, bool was_classic)
 	if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL))
 		emit(A64_AUTIASP, ctx);
 
+	/* tmp is the offset of the pad, see emit_unwind_exit() */
+	if (unwind)
+		emit(A64_ADD(1, A64_LR, A64_LR, tmp), ctx);
+
 	emit(A64_RET(A64_LR), ctx);
 }
 
@@ -1221,6 +1227,76 @@ static int add_exception_handler(const struct bpf_insn *insn,
 	return 0;
 }
 
+/*
+ * There is no branch for unwind_goto, only an extable entry
+ *   { .insn = ret_addr - 1, .fixup = pad - ret_addr }
+ * where ret_addr is the return address of the call in front of it. unwind_exit
+ * adds .fixup to its LR.
+ * ret_addr - 1, because what follows the call can have an entry of its own.
+ * No insn starts there, so a fault cannot hit the entry.
+ * 'bl foo; mov x8, x0' is in front of unwind_goto. The pad doesn't miss the mov:
+ * r0 is x8 and the epilogue of the callee copies it to x0 and leaves x8 as it is.
+ */
+static int add_unwind_exentry(struct jit_ctx *ctx, int pad)
+{
+	int ret_addr = ctx->idx - 1;
+	struct exception_table_entry *ex;
+	off_t ins_offset;
+
+	if (!ctx->image)
+		/* First pass */
+		return 0;
+
+	if (!ctx->prog->aux->extable ||
+	    WARN_ON_ONCE(ctx->exentry_idx >= ctx->prog->aux->num_exentries))
+		return -EINVAL;
+
+	ex = &ctx->prog->aux->extable[ctx->exentry_idx];
+	ins_offset = (long)&ctx->ro_image[ret_addr] - 1 - (long)&ex->insn;
+	if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN))
+		return -ERANGE;
+
+	/* switch ex to rw buffer for writing */
+	ex = (void *)ctx->image + ((void *)ex - (void *)ctx->ro_image);
+	ex->insn = ins_offset;
+	ex->fixup = (ctx->offset[pad] - ret_addr) * AARCH64_INSN_SIZE;
+	ex->type = EX_TYPE_BPF;
+
+	ctx->exentry_idx++;
+	return 0;
+}
+
+static long bpf_unwind_fixup(u64 lr)
+{
+	const struct exception_table_entry *e;
+
+	e = search_bpf_extables(ptrauth_strip_kernel_insn_pac(lr) - 1);
+	/*
+	 * The verifier made sure that unwind_goto follows a call in a prog. When
+	 * the kernel called an async or an exception callback, it's exit.
+	 */
+	return e ? e->fixup : 0;
+}
+
+/*
+ * unwind_exit: return to the pad. LR in the frame record is signed. Instead of
+ * signing the address of the pad, let the epilogue authenticate LR as it is and
+ * add the offset of the pad to it. r0 and r2 are returned.
+ */
+static void emit_unwind_exit(struct jit_ctx *ctx)
+{
+	const u8 tmp = bpf2a64[TMP_REG_1];
+	const u8 r0 = bpf2a64[BPF_REG_0];
+	const u8 r2 = bpf2a64[BPF_REG_2];
+
+	emit(A64_PUSH(r0, r2, A64_SP), ctx);
+	emit(A64_LDR64I(A64_R(0), A64_FP, 8), ctx);
+	emit_call((u64)bpf_unwind_fixup, ctx);
+	emit(A64_MOV(1, tmp, A64_R(0)), ctx);
+	emit(A64_POP(r0, r2, A64_SP), ctx);
+	build_epilogue(ctx, false, true);
+}
+
 static const struct bpf_jit_arg_abi arm64_arg_abi = {
 	.nr_arg_regs		= 8,
 	.even_reg_align		= true,
@@ -1806,8 +1882,18 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
 		if (emit_bpf_tail_call(ctx))
 			return -EFAULT;
 		break;
+	/* unwind_goto. The verifier expanded may_goto. */
+	case BPF_JMP | BPF_JCOND:
+		ret = add_unwind_exentry(ctx, i + 1 + off);
+		if (ret)
+			return ret;
+		break;
 	/* function return */
 	case BPF_JMP | BPF_EXIT:
+		if (bpf_is_unwind_exit(insn) && bpf_prog_can_unwind(ctx->prog)) {
+			emit_unwind_exit(ctx);
+			break;
+		}
 		/* Optimization: when last instruction is EXIT,
 		   simply fallthrough to epilogue. */
 		if (i == ctx->prog->len - 1)
@@ -2312,7 +2398,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
 		goto out_off;
 
 	ctx.epilogue_offset = ctx.idx;
-	build_epilogue(&ctx, was_classic);
+	build_epilogue(&ctx, was_classic, false);
 	build_plt(&ctx);
 
 	extable_align = __alignof__(struct exception_table_entry);
@@ -2369,7 +2455,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
 		WARN_ON_ONCE(ctx.idx != ctx.epilogue_offset))
 		goto out_free_hdr;
 
-	build_epilogue(&ctx, was_classic);
+	build_epilogue(&ctx, was_classic, false);
 	build_plt(&ctx);
 
 	/* Extra pass to validate JITed code. */
@@ -2466,6 +2552,11 @@ bool bpf_jit_supports_kfunc_ret_reg_pair(void)
 	return true;
 }
 
+bool bpf_jit_supports_unwind(void)
+{
+	return true;
+}
+
 const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void)
 {
 	return &arm64_arg_abi;
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 72a2a0a89e03..d3ab36281144 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -15,6 +15,7 @@
 #include <linux/memory.h>
 #include <linux/sort.h>
 #include <linux/execmem.h>
+#include <linux/extable.h>
 #include <asm/extable.h>
 #include <asm/ftrace.h>
 #include <asm/set_memory.h>
@@ -2511,6 +2512,68 @@ static void adjust_cmp_imm(u32 *op, s32 *imm)
 	}
 }
 
+/*
+ * There is no branch for unwind_goto, only an extable entry
+ *   { .insn = ret_addr - 1, .fixup = target - ret_addr }
+ * where ret_addr is the return address of the call in front of it. unwind_exit
+ * replaces the return address of its frame with bpf_unwind_target() of it.
+ * ret_addr - 1, because what follows the call can have an entry of its own.
+ * No insn starts there, so a fault cannot hit the entry.
+ */
+static int add_unwind_exentry(struct bpf_prog *prog, u8 *image, u8 *rw_image, int *excnt,
+			      int ret_addr, int target)
+{
+	struct exception_table_entry *ex;
+	s64 delta;
+
+	if (!prog->aux->extable)
+		return 0;
+	if (*excnt >= prog->aux->num_exentries) {
+		pr_err("unwind extable bug\n");
+		return -EFAULT;
+	}
+	ex = &prog->aux->extable[(*excnt)++];
+	delta = image + ret_addr - 1 - (u8 *)&ex->insn;
+	if (!is_simm32(delta)) {
+		pr_err("extable->insn doesn't fit into 32-bit\n");
+		return -EFAULT;
+	}
+	/* switch ex to rw buffer for writes */
+	ex = (void *)rw_image + ((void *)ex - (void *)image);
+	ex->insn = delta;
+	ex->fixup = target - ret_addr;
+	ex->data = EX_TYPE_BPF;
+	return 0;
+}
+
+static u64 bpf_unwind_target(u64 ret_addr)
+{
+	const struct exception_table_entry *e = search_bpf_extables(ret_addr - 1);
+
+	/*
+	 * The verifier made sure that unwind_goto follows a call in a prog. When
+	 * the kernel called an async or an exception callback, it's exit.
+	 */
+	return e ? ret_addr + e->fixup : ret_addr;
+}
+
+/* unwind_exit: fix the return address of this frame. r0 and r2 are returned. */
+static int emit_unwind_ret_addr(u8 **pprog, u8 *ip)
+{
+	u8 *prog = *pprog, *start = prog;
+
+	EMIT2(0x50, 0x56);			/* push rax; push rsi */
+	EMIT4(0x48, 0x8B, 0x7D, 0x08);		/* mov rdi, [rbp + 8] */
+	ip += prog - start;
+	ip += x86_call_depth_emit_accounting(&prog, bpf_unwind_target, ip);
+	if (emit_call(&prog, bpf_unwind_target, ip))
+		return -EINVAL;
+	EMIT4(0x48, 0x89, 0x45, 0x08);		/* mov [rbp + 8], rax */
+	EMIT2(0x5E, 0x58);			/* pop rsi; pop rax */
+	*pprog = prog;
+	return 0;
+}
+
 static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 		  u8 *rw_image, int oldproglen, struct jit_context *ctx, bool jmp_padding)
 {
@@ -3493,6 +3556,34 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 				pop_r9(&prog);
 			break;
 
+			/* unwind_goto. The verifier expanded may_goto. */
+		case BPF_JMP | BPF_JCOND:
+			if (!priv_frame_ptr) {
+				err = add_unwind_exentry(bpf_prog, image, rw_image, &excnt,
+							 addrs[i - 1], addrs[i + insn->off]);
+				if (err)
+					return err;
+				break;
+			}
+			/*
+			 * 'pop r9' after the call has to be done on the way to
+			 * the pad too:
+			 *   call foo
+			 *   pop r9      <- exit returns here
+			 *   jmp 1f
+			 *   pop r9      <- unwind_exit returns here
+			 *   jmp pad
+			 * 1:
+			 */
+			err = add_unwind_exentry(bpf_prog, image, rw_image, &excnt,
+						 addrs[i - 1] - 2, addrs[i - 1] + 2);
+			if (err)
+				return err;
+			EMIT2(0xEB, 7);
+			pop_r9(&prog);
+			EMIT1_off32(0xE9, addrs[i + insn->off] - (addrs[i - 1] + 9));
+			break;
+
 		case BPF_JMP | BPF_TAIL_CALL:
 			if (imm32)
 				emit_bpf_tail_call_direct(bpf_prog,
@@ -3774,13 +3865,16 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 			break;
 
 		case BPF_JMP | BPF_EXIT:
+			if (src_reg == BPF_UNWIND_EXIT && bpf_prog_can_unwind(bpf_prog) &&
+			    emit_unwind_ret_addr(&prog, ip))
+				return -EINVAL;
 			if (seen_exit) {
 				jmp_offset = ctx->cleanup_addr - addrs[i];
 				goto emit_jmp;
 			}
 			seen_exit = true;
 			/* Update cleanup_addr */
-			ctx->cleanup_addr = proglen;
+			ctx->cleanup_addr = proglen + (prog - temp);
 			if (bpf_prog_was_classic(bpf_prog) &&
 			    !ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN)) {
 				if (emit_spectre_bhb_barrier(&prog, ip, bpf_prog))
@@ -5027,6 +5121,11 @@ bool bpf_jit_supports_kfunc_ret_reg_pair(void)
 	return true;
 }
 
+bool bpf_jit_supports_unwind(void)
+{
+	return true;
+}
+
 const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void)
 {
 	return &x86_arg_abi;
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 7750df973494..580f89c450b5 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1897,6 +1897,7 @@ struct bpf_prog_aux {
 	bool changes_pkt_data;
 	bool has_prologue;
 	bool might_sleep;
+	bool can_unwind;
 	bool kprobe_write_ctx;
 	/* has insns that work with the arena base, see bpf_prog_has_arena_insns() */
 	bool uses_arena_insns;
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 54481da4ed2d..6606b341351e 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -532,6 +532,8 @@ struct bpf_verifier_state {
 
 	bool speculative;
 	bool in_sleepable;
+	/* The callee left with unwind_exit. Next insn is unwind_goto. */
+	bool unwound;
 
 	/*
 	 * First and last insn idx of this verifier state.
@@ -957,6 +959,7 @@ struct bpf_subprog_info {
 	s16 fastcall_stack_off;
 	bool has_tail_call: 1;
 	bool might_throw: 1;
+	bool can_unwind: 1; /* has unwind_exit */
 	bool tail_call_reachable: 1;
 	bool has_ld_abs: 1;
 	bool is_cb: 1;
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 6f4c9bd024a0..fccd8db33647 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -254,6 +254,25 @@ static inline bool insn_is_mov_percpu_addr(const struct bpf_insn *insn)
 		.off   = 0,					\
 		.imm   = 1 })
 
+static inline bool bpf_is_unwind_goto(const struct bpf_insn *insn)
+{
+	return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_UNWIND_GOTO;
+}
+
+static inline bool bpf_is_unwind_exit(const struct bpf_insn *insn)
+{
+	return insn->code == (BPF_JMP | BPF_EXIT) && insn->src_reg == BPF_UNWIND_EXIT;
+}
+
+/*
+ * The caller of the main prog doesn't tell unwind_exit from exit. An extension
+ * is different: its caller is the caller of the subprog that it replaced.
+ */
+static inline bool bpf_prog_can_unwind(const struct bpf_prog *prog)
+{
+	return bpf_is_subprog(prog) || prog->type == BPF_PROG_TYPE_EXT;
+}
+
 static inline bool insn_is_zext(const struct bpf_insn *insn)
 {
 	return insn->code == (BPF_ALU | BPF_MOV | BPF_X) && insn->imm == 1;
@@ -1257,6 +1276,7 @@ bool bpf_jit_supports_percpu_insn(void);
 bool bpf_jit_supports_kfunc_call(void);
 bool bpf_jit_supports_callx(void);
 bool bpf_jit_supports_kfunc_ret_reg_pair(void);
+bool bpf_jit_supports_unwind(void);
 bool bpf_jit_supports_stack_args(void);
 bool bpf_jit_supports_arena_args(void);
 bool bpf_jit_supports_far_kfunc_call(void);
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 6f70db7515dd..f29b0aad440c 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -56,6 +56,19 @@
 
 enum bpf_cond_pseudo_jmp {
 	BPF_MAY_GOTO = 0,
+	/*
+	 * Follows a call of a bpf subprog. Taken when the subprog left with
+	 * BPF_EXIT that has src_reg == BPF_UNWIND_EXIT.
+	 */
+	BPF_UNWIND_GOTO = 1,
+};
+
+enum bpf_pseudo_exit {
+	/*
+	 * When bpf_exit->src_reg == BPF_UNWIND_EXIT, it's the same as bpf_exit,
+	 * except that BPF_UNWIND_GOTO after the call is taken.
+	 */
+	BPF_UNWIND_EXIT = 1,
 };
 
 /* Register numbers */
diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c
index 8dd8c5958eb8..f9edfde965dc 100644
--- a/kernel/bpf/const_fold.c
+++ b/kernel/bpf/const_fold.c
@@ -357,7 +357,7 @@ int bpf_prune_dead_branches(struct bpf_verifier_env *env)
 
 		if (!bpf_insn_is_cond_jump(insn->code))
 			continue;
-		if (bpf_is_may_goto_insn(insn))
+		if (BPF_OP(insn->code) == BPF_JCOND)
 			continue;
 
 		if (!(aux->const_reg_mask & BIT(insn->dst_reg)))
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index fc22d66d194a..3030beb531d9 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -3399,6 +3399,12 @@ bool __weak bpf_jit_supports_kfunc_ret_reg_pair(void)
 	return false;
 }
 
+/* unwind_goto and unwind_exit */
+bool __weak bpf_jit_supports_unwind(void)
+{
+	return false;
+}
+
 /*
  * How this arch places a by-value kfunc argument, or NULL for one that has
  * not opted in and so only takes an argument of a single eightbyte, which
diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c
index 36d3228d7745..889fc074496f 100644
--- a/kernel/bpf/disasm.c
+++ b/kernel/bpf/disasm.c
@@ -375,11 +375,16 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs,
 			   insn->src_reg == BPF_MAY_GOTO) {
 			verbose(cbs->private_data, "(%02x) may_goto pc%+d",
 				insn->code, insn->off);
+		} else if (insn->code == (BPF_JMP | BPF_JCOND) &&
+			   insn->src_reg == BPF_UNWIND_GOTO) {
+			verbose(cbs->private_data, "(%02x) unwind_goto pc%+d",
+				insn->code, insn->off);
 		} else if (insn->code == (BPF_JMP32 | BPF_JA)) {
 			verbose(cbs->private_data, "(%02x) gotol pc%+d",
 				insn->code, insn->imm);
 		} else if (insn->code == (BPF_JMP | BPF_EXIT)) {
-			verbose(cbs->private_data, "(%02x) exit", insn->code);
+			verbose(cbs->private_data, "(%02x) %s", insn->code,
+				insn->src_reg == BPF_UNWIND_EXIT ? "unwind_exit" : "exit");
 		} else if (BPF_SRC(insn->code) == BPF_X) {
 			verbose(cbs->private_data,
 				"(%02x) if %c%d %s %c%d goto pc%+d",
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 409b9104c723..f062ef7dd943 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -658,6 +658,7 @@ void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)
 {
 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
 	struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
+	struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1);
 	struct bpf_insn *insn = env->prog->insnsi;
 	const int insn_cnt = env->prog->len;
 	int i;
@@ -666,6 +667,20 @@ void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)
 		if (!bpf_insn_is_cond_jump(insn->code))
 			continue;
 
+		if (bpf_is_unwind_goto(insn) && aux_data[i + 1 + insn->off].seen) {
+			/*
+			 * The callee never exits. Its unwind_exit needs
+			 * unwind_goto all the same, and the cpu predicts that it
+			 * returns to what follows. Don't let the removal of dead
+			 * code put something else there.
+			 */
+			if (!aux_data[i + 1].seen) {
+				memcpy(insn + 1, &trap, sizeof(trap));
+				aux_data[i + 1].seen = env->pass_cnt;
+			}
+			continue;
+		}
+
 		if (!aux_data[i + 1].seen)
 			ja.off = insn->off;
 		else if (!aux_data[i + 1 + insn->off].seen)
@@ -1467,12 +1482,15 @@ static int jit_subprogs(struct bpf_verifier_env *env)
 			if (BPF_CLASS(insn->code) == BPF_STX &&
 			     BPF_MODE(insn->code) == BPF_PROBE_ATOMIC)
 				num_exentries++;
+			if (bpf_is_unwind_goto(insn))
+				num_exentries++;
 		}
 		func[i]->aux->num_exentries = num_exentries;
 		func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;
 		func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;
 		func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data;
 		func[i]->aux->might_sleep = env->subprog_info[i].might_sleep;
+		func[i]->aux->can_unwind = env->subprog_info[i].can_unwind;
 		func[i]->aux->token = prog->aux->token;
 		if (!i)
 			func[i]->aux->exception_boundary = env->seen_exception;
diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c
index b3d250caaaa6..d7df6632a5c4 100644
--- a/kernel/bpf/states.c
+++ b/kernel/bpf/states.c
@@ -1046,6 +1046,9 @@ static bool states_equal(struct bpf_verifier_env *env,
 	if (old->in_sleepable != cur->in_sleepable)
 		return false;
 
+	if (old->unwound != cur->unwound)
+		return false;
+
 	if (!refsafe(old, cur, &env->idmap_scratch))
 		return false;
 
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 849bc9136299..90bc7230afa8 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -5087,6 +5087,12 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
 			insns[i].imm = BPF_FUNC_tail_call;
 			/* fall-through */
 		}
+		/*
+		 * BPF_JCOND is the same opcode as BPF_CALL_ARGS. The latter is for
+		 * the interpreter, and a prog with unwind_goto is always JITed.
+		 */
+		if (prog->jited && bpf_is_unwind_goto(&insns[i]))
+			continue;
 		if (code == (BPF_JMP | BPF_CALL) ||
 		    code == (BPF_JMP | BPF_CALL_ARGS)) {
 			/* Restore the legacy xlated dump layout. */
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 01434e1d23b5..e387320e196f 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1769,6 +1769,7 @@ int bpf_copy_verifier_state(struct bpf_verifier_state *dst_state,
 		return err;
 	dst_state->speculative = src->speculative;
 	dst_state->in_sleepable = src->in_sleepable;
+	dst_state->unwound = src->unwound;
 	dst_state->curframe = src->curframe;
 	dst_state->branches = src->branches;
 	dst_state->parent = src->parent;
@@ -3430,6 +3431,46 @@ static int next_callx_callee(struct bpf_verifier_env *env, int caller, int from)
 	return bit < end ? bit - caller * cnt : -1;
 }
 
+/*
+ * 'call foo; unwind_goto pad' goes to the pad when foo left with unwind_exit.
+ * JITs don't emit a branch for it. unwind_exit finds the pad by its return
+ * address. So unwind_goto has to follow the call and nothing may jump to it.
+ */
+static int check_unwind_insns(struct bpf_verifier_env *env)
+{
+	struct bpf_insn *insn = env->prog->insnsi;
+	bool seen = false;
+	int i;
+
+	for (i = 0; i < env->prog->len; i++, insn++) {
+		if (bpf_is_unwind_exit(insn)) {
+			bpf_find_containing_subprog(env, i)->can_unwind = true;
+			seen = true;
+		}
+		if (!bpf_is_unwind_goto(insn))
+			continue;
+		seen = true;
+		if (!i || !(bpf_pseudo_call(insn - 1) || bpf_is_callx(insn - 1))) {
+			verbose(env, "unwind_goto at insn %d doesn't follow a call of a subprog\n",
+				i);
+			return -EINVAL;
+		}
+		if (bpf_is_jump_target(env, i)) {
+			verbose(env, "jump to unwind_goto at insn %d\n", i);
+			return -EINVAL;
+		}
+	}
+	if (!seen)
+		return 0;
+	if (!bpf_jit_supports_unwind()) {
+		verbose(env, "JIT does not support unwinding\n");
+		return -ENOTSUPP;
+	}
+	env->prog->jit_required = 1;
+	env->prog->aux->can_unwind = env->subprog_info[0].can_unwind;
+	return 0;
+}
+
 /*
  * Sort subprogs in topological order so that leaf subprogs come first and
  * their callers come later. This is a DFS post-order traversal of the call
@@ -11584,6 +11625,19 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 			}
 		}
 
+		if (env->subprog_info[subprog].can_unwind) {
+			struct bpf_verifier_state *branch;
+
+			if (!bpf_is_unwind_goto(insn + 1)) {
+				verbose(env, "no unwind_goto after the call of %s()\n", sub_name);
+				return -EINVAL;
+			}
+			branch = push_stack(env, *insn_idx + 1, *insn_idx, false);
+			if (IS_ERR(branch))
+				return PTR_ERR(branch);
+			branch->unwound = true;
+		}
+
 		if (env->subprog_info[subprog].might_throw) {
 			struct bpf_verifier_state *branch;
 
@@ -12057,6 +12111,19 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
 	}
 
 	caller = state->frame[state->curframe - 1];
+	if (bpf_is_unwind_exit(&env->prog->insnsi[*insn_idx])) {
+		/* the helper or the kfunc that called it doesn't expect that */
+		if (callee->in_callback_fn) {
+			verbose(env, "callback cannot unwind\n");
+			return -EINVAL;
+		}
+		if (!bpf_is_unwind_goto(&env->prog->insnsi[callee->callsite + 1])) {
+			verbose(env, "unwind_exit, but no unwind_goto after the call at insn %d\n",
+				callee->callsite);
+			return -EINVAL;
+		}
+		state->unwound = true;
+	}
 	if (callee->in_callback_fn) {
 		if (r0->type != SCALAR_VALUE) {
 			verbose(env, "R0 not a scalar value\n");
@@ -18759,6 +18826,23 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
 		return -EINVAL;
 	}
 
+	if (bpf_is_unwind_goto(insn)) {
+		if (!env->cur_state->unwound)
+			return 0;
+		env->cur_state->unwound = false;
+		/*
+		 * unwind_exit changed its return address, so the return is
+		 * mispredicted and what follows unwind_goto runs speculatively.
+		 */
+		if (!env->bypass_spec_v1) {
+			err = sanitize_speculative_path(env, NULL, *insn_idx + 1, *insn_idx);
+			if (err)
+				return err;
+		}
+		*insn_idx += insn->off;
+		return 0;
+	}
+
 	if (opcode == BPF_JCOND) {
 		struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st;
 		int idx = *insn_idx;
@@ -21223,14 +21307,14 @@ static int check_jmp_fields(struct bpf_verifier_env *env, struct bpf_insn *insn)
 		return 0;
 	case BPF_EXIT:
 		if (BPF_SRC(insn->code) != BPF_K || insn->imm != 0 ||
-		    insn->src_reg != BPF_REG_0 || insn->dst_reg != BPF_REG_0 ||
+		    insn->src_reg > BPF_UNWIND_EXIT || insn->dst_reg != BPF_REG_0 ||
 		    class == BPF_JMP32) {
 			verbose(env, "BPF_EXIT uses reserved fields\n");
 			return -EINVAL;
 		}
 		return 0;
 	case BPF_JCOND:
-		if (insn->code != (BPF_JMP | BPF_JCOND) || insn->src_reg != BPF_MAY_GOTO ||
+		if (insn->code != (BPF_JMP | BPF_JCOND) || insn->src_reg > BPF_UNWIND_GOTO ||
 		    insn->dst_reg || insn->imm) {
 			verbose(env, "invalid may_goto imm %d\n", insn->imm);
 			return -EINVAL;
@@ -22436,6 +22520,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
 		struct bpf_prog_aux *aux = tgt_prog->aux;
 		bool tgt_changes_pkt_data;
 		bool tgt_might_sleep;
+		bool tgt_can_unwind;
 
 		if (bpf_prog_is_dev_bound(prog->aux) &&
 		    !bpf_prog_dev_bound_match(prog, tgt_prog)) {
@@ -22464,6 +22549,16 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
 				prog_extension ? "Extension" : "Tracing");
 			return -EINVAL;
 		}
+		/* unwind_exit of the main prog is exit, unless it's an extension */
+		tgt_can_unwind = (subprog || tgt_prog->type == BPF_PROG_TYPE_EXT) &&
+				 (aux->func ? aux->func[subprog]->aux->can_unwind
+					    : aux->can_unwind);
+		/* unwind_exit would return to the trampoline that called the target */
+		if (tgt_can_unwind && !prog_extension &&
+		    prog->expected_attach_type != BPF_TRACE_FENTRY) {
+			bpf_log(log, "Only fentry can attach to %s, which can unwind\n", tname);
+			return -EINVAL;
+		}
 		conservative = aux->func_info_aux[subprog].unreliable;
 		if (prog_extension) {
 			if (aux->func_info_aux[subprog].linkage != BTF_FUNC_GLOBAL ||
@@ -22511,6 +22606,12 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
 					"Extension program may sleep, while original does not\n");
 				return -EINVAL;
 			}
+			/* There is no unwind_goto after the calls of the original */
+			if (prog->aux->can_unwind && !tgt_can_unwind) {
+				bpf_log(log,
+					"Extension program can unwind, while original does not\n");
+				return -EINVAL;
+			}
 		}
 		if (!tgt_prog->jited) {
 			bpf_log(log, "Can attach to only JITed progs\n");
@@ -23845,6 +23946,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	if (ret < 0)
 		goto skip_full_check;
 
+	ret = check_unwind_insns(env);
+	if (ret < 0)
+		goto skip_full_check;
+
 	ret = bpf_compute_postorder(env);
 	if (ret < 0)
 		goto skip_full_check;
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 6f70db7515dd..f29b0aad440c 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -56,6 +56,19 @@
 
 enum bpf_cond_pseudo_jmp {
 	BPF_MAY_GOTO = 0,
+	/*
+	 * Follows a call of a bpf subprog. Taken when the subprog left with
+	 * BPF_EXIT that has src_reg == BPF_UNWIND_EXIT.
+	 */
+	BPF_UNWIND_GOTO = 1,
+};
+
+enum bpf_pseudo_exit {
+	/*
+	 * When bpf_exit->src_reg == BPF_UNWIND_EXIT, it's the same as bpf_exit,
+	 * except that BPF_UNWIND_GOTO after the call is taken.
+	 */
+	BPF_UNWIND_EXIT = 1,
 };
 
 /* Register numbers */
diff --git a/tools/testing/selftests/bpf/Makefile.skel b/tools/testing/selftests/bpf/Makefile.skel
index 06e297a17156..6570c48f92b0 100644
--- a/tools/testing/selftests/bpf/Makefile.skel
+++ b/tools/testing/selftests/bpf/Makefile.skel
@@ -153,4 +153,15 @@ $(RUST_OBJS): $(RDIR)/%.bpf.o: progs/%.rs $(RUST_CORE) | $(RDIR)
 endif
 endif
 
+# Programs in C++. No skeletons: gcc doesn't build them.
+ifeq ($(BPF_CC),$(CLANG))
+CXX_OBJS := $(patsubst progs/%.cpp,$(RDIR)/%.bpf.o,$(wildcard progs/*.cpp))
+BPF_OBJS += $(CXX_OBJS)
+
+$(CXX_OBJS): $(RDIR)/%.bpf.o: progs/%.cpp | $(RDIR)
+	$(call msg,$(BPF_CC_MSG),$(BINARY),$@)
+	$(Q)$(BPF_CC) -x c++ -g -Wall -Werror -fexceptions -funwind-tables -fno-rtti	\
+		$(BPF_CC_FLAGS) -c $< -o $@ $(call skip_on_fail,BPF)
+endif
+
 endif # BPF_CC
diff --git a/tools/testing/selftests/bpf/prog_tests/unwind_attach.c b/tools/testing/selftests/bpf/prog_tests/unwind_attach.c
new file mode 100644
index 000000000000..ada8b3273ccb
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/unwind_attach.c
@@ -0,0 +1,112 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <test_progs.h>
+#include "verifier_unwind.skel.h"
+#include "unwind_attach.skel.h"
+
+static char log_buf[16 * 1024];
+
+static int run(int fd)
+{
+	char data[64] = {};
+	LIBBPF_OPTS(bpf_test_run_opts, opts,
+		.data_in = data,
+		.data_size_in = sizeof(data),
+		.repeat = 1,
+	);
+
+	ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), "test_run");
+	return opts.retval;
+}
+
+/*
+ * msg: why it cannot be loaded. Otherwise retval is what the target returns
+ * with the prog attached.
+ */
+static void attach(int tgt_fd, const char *func, const char *name, const char *msg, int retval)
+{
+	struct bpf_link *link = NULL;
+	struct unwind_attach *skel;
+	struct bpf_program *prog;
+	int err;
+
+	skel = unwind_attach__open();
+	if (!ASSERT_OK_PTR(skel, "unwind_attach__open"))
+		return;
+
+	prog = bpf_object__find_program_by_name(skel->obj, name);
+	if (!ASSERT_OK_PTR(prog, name))
+		goto out;
+	bpf_program__set_autoload(prog, true);
+	log_buf[0] = 0;
+	bpf_program__set_log_buf(prog, log_buf, sizeof(log_buf));
+	err = bpf_program__set_attach_target(prog, tgt_fd, func);
+	if (!ASSERT_OK(err, "set_attach_target"))
+		goto out;
+
+	err = unwind_attach__load(skel);
+	if (msg) {
+		ASSERT_ERR(err, name);
+		ASSERT_HAS_SUBSTR(log_buf, msg, name);
+		goto out;
+	}
+	if (!ASSERT_OK(err, name))
+		goto out;
+
+	link = bpf_program__attach(prog);
+	if (!ASSERT_OK_PTR(link, "attach"))
+		goto out;
+	ASSERT_EQ(run(tgt_fd), retval, "retval");
+	if (prog == skel->progs.fentry_unwind)
+		ASSERT_EQ(skel->bss->fentry_cnt, 1, "fentry_cnt");
+out:
+	bpf_link__destroy(link);
+	unwind_attach__destroy(skel);
+}
+
+void test_unwind_attach(void)
+{
+	struct verifier_unwind *tgt;
+	struct bpf_program *prog;
+	int tgt_fd, plain_fd, err;
+
+	if (!is_jit_enabled()) {
+		printf("%s:SKIP:JIT is disabled\n", __func__);
+		test__skip();
+		return;
+	}
+	tgt = verifier_unwind__open();
+	if (!ASSERT_OK_PTR(tgt, "verifier_unwind__open"))
+		return;
+	bpf_object__for_each_program(prog, tgt->obj)
+		bpf_program__set_autoload(prog, prog == tgt->progs.calls_global_unwinds ||
+						prog == tgt->progs.calls_global_never_unwinds);
+	err = verifier_unwind__load(tgt);
+	if (err == -ENOTSUPP) {
+		printf("%s:SKIP:JIT does not support unwinding\n", __func__);
+		test__skip();
+		goto out;
+	}
+	if (!ASSERT_OK(err, "verifier_unwind__load"))
+		goto out;
+	/*
+	 * It calls global_unwinds(), which unwinds with r0 = 5, and returns r0 + 10
+	 * from the pad.
+	 */
+	tgt_fd = bpf_program__fd(tgt->progs.calls_global_unwinds);
+	plain_fd = bpf_program__fd(tgt->progs.calls_global_never_unwinds);
+	ASSERT_EQ(run(tgt_fd), 15, "retval");
+
+	attach(tgt_fd, "global_unwinds", "fentry_unwind", NULL, 15);
+	/* unwind_exit would return to the trampoline */
+	attach(tgt_fd, "global_unwinds", "fexit_unwind",
+	       "Only fentry can attach to global_unwinds", 0);
+	/* An extension leaves either way, like what it replaced */
+	attach(tgt_fd, "global_unwinds", "freplace_returns", NULL, 9);
+	attach(tgt_fd, "global_unwinds", "freplace_unwinds", NULL, 13);
+	/* There is no unwind_goto after the call */
+	attach(plain_fd, "global_never_unwinds", "freplace_unwinds",
+	       "Extension program can unwind, while original does not", 0);
+	ASSERT_EQ(run(tgt_fd), 15, "retval");
+out:
+	verifier_unwind__destroy(tgt);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/unwind_cxx.c b/tools/testing/selftests/bpf/prog_tests/unwind_cxx.c
new file mode 100644
index 000000000000..9faba99133ad
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/unwind_cxx.c
@@ -0,0 +1,99 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <test_progs.h>
+#include <bpf/btf.h>
+
+/* The progs are under #ifdef __BPF_FEATURE_UNWIND */
+static bool llvm_lowers_unwinding(const char *file)
+{
+	struct btf *btf = btf__parse(file, NULL);
+	bool ret;
+
+	ret = btf && btf__find_by_name_kind(btf, "catches", BTF_KIND_FUNC) > 0;
+	btf__free(btf);
+	return ret;
+}
+
+static void run(struct bpf_object *obj, const char *name, int len, int retval, int dtors)
+{
+	struct bpf_program *prog = bpf_object__find_program_by_name(obj, name);
+	struct bpf_map *map = bpf_object__find_map_by_name(obj, ".data.dtors");
+	char data[256] = {};
+	LIBBPF_OPTS(bpf_test_run_opts, opts,
+		.data_in = data,
+		.data_size_in = len,
+		.repeat = 1,
+	);
+	int key = 0, val = 0;
+
+	if (!ASSERT_OK_PTR(prog, name) || !ASSERT_OK_PTR(map, "dtors"))
+		return;
+	ASSERT_OK(bpf_map__update_elem(map, &key, sizeof(key), &val, sizeof(val), 0), "clear");
+	ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(prog), &opts), "test_run");
+	ASSERT_EQ(opts.retval, retval, "retval");
+	ASSERT_OK(bpf_map__lookup_elem(map, &key, sizeof(key), &val, sizeof(val), 0), "lookup");
+	ASSERT_EQ(val, dtors, "dtors");
+}
+
+void test_unwind_cxx(void)
+{
+	const char *file = "./unwind_cxx.bpf.o";
+	struct bpf_object *obj;
+	struct bpf_program *prog;
+	int err = -EINVAL;
+
+	if (access(file, R_OK)) {
+		printf("%s:SKIP:gcc doesn't build %s\n", __func__, file);
+		test__skip();
+		return;
+	}
+	if (!is_jit_enabled()) {
+		printf("%s:SKIP:JIT is disabled\n", __func__);
+		test__skip();
+		return;
+	}
+	if (!llvm_lowers_unwinding(file)) {
+		printf("%s:SKIP:llvm doesn't lower unwinding\n", __func__);
+		test__skip();
+		return;
+	}
+	obj = bpf_object__open_file(file, NULL);
+	if (!ASSERT_OK_PTR(obj, "open"))
+		return;
+	prog = bpf_object__find_program_by_name(obj, "catches_aggregate");
+	if (!ASSERT_OK_PTR(prog, "catches_aggregate"))
+		goto out;
+	bpf_program__set_autoload(prog, false);
+	err = bpf_object__load(obj);
+	if (err == -ENOTSUPP) {
+		printf("%s:SKIP:JIT does not support unwinding\n", __func__);
+		test__skip();
+		goto out;
+	}
+	if (!ASSERT_OK(err, "load"))
+		goto out;
+
+	/* inner's destructor, then middle's */
+	run(obj, "catches", 64, 65, 12);
+	run(obj, "catches", 200, 42, 12);
+	run(obj, "does_not_catch", 64, 65, 12);
+	run(obj, "does_not_catch", 200, 0, 12);
+out:
+	bpf_object__close(obj);
+	if (err)
+		return;
+
+	if (!test__start_subtest("aggregate_return"))
+		return;
+	obj = bpf_object__open_file(file, NULL);
+	if (!ASSERT_OK_PTR(obj, "open_aggregate"))
+		return;
+	bpf_object__for_each_program(prog, obj)
+		bpf_program__set_autoload(prog,
+					 strcmp(bpf_program__name(prog), "catches_aggregate") == 0);
+	if (!ASSERT_OK(bpf_object__load(obj), "load_aggregate"))
+		goto out_aggregate;
+	run(obj, "catches_aggregate", 64, 71, 0);
+	run(obj, "catches_aggregate", 200, 42, 0);
+out_aggregate:
+	bpf_object__close(obj);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c
index 6df440d82675..04740ba37e91 100644
--- a/tools/testing/selftests/bpf/prog_tests/verifier.c
+++ b/tools/testing/selftests/bpf/prog_tests/verifier.c
@@ -130,6 +130,7 @@
 #include "verifier_typedef.skel.h"
 #include "verifier_uninit.skel.h"
 #include "verifier_unpriv.skel.h"
+#include "verifier_unwind.skel.h"
 #include "verifier_unpriv_perf.skel.h"
 #include "verifier_value_adj_spill.skel.h"
 #include "verifier_value.skel.h"
@@ -320,6 +321,7 @@ void test_verifier_tailcall_jit(void)         { RUN(verifier_tailcall_jit); }
 void test_verifier_typedef(void)              { RUN(verifier_typedef); }
 void test_verifier_uninit(void)               { RUN(verifier_uninit); }
 void test_verifier_unpriv(void)               { RUN(verifier_unpriv); }
+void test_verifier_unwind(void)               { RUN(verifier_unwind); }
 void test_verifier_unpriv_perf(void)          { RUN(verifier_unpriv_perf); }
 void test_verifier_value_adj_spill(void)      { RUN(verifier_value_adj_spill); }
 void test_verifier_value(void)                { RUN(verifier_value); }
diff --git a/tools/testing/selftests/bpf/progs/unwind_attach.c b/tools/testing/selftests/bpf/progs/unwind_attach.c
new file mode 100644
index 000000000000..df73c26c1ff4
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/unwind_attach.c
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include "../../../include/linux/filter.h"
+#include "bpf_misc.h"
+
+/* The targets are global_unwinds() and global_never_unwinds() in verifier_unwind.c */
+
+int fentry_cnt;
+
+SEC("?fentry/global_unwinds")
+int BPF_PROG(fentry_unwind)
+{
+	fentry_cnt++;
+	return 0;
+}
+
+SEC("?fexit/global_unwinds")
+int BPF_PROG(fexit_unwind)
+{
+	return 0;
+}
+
+SEC("?freplace/global_unwinds")
+unsigned long freplace_returns(void)
+{
+	return 9;
+}
+
+SEC("?freplace/global_unwinds")
+__naked unsigned long freplace_unwinds(void)
+{
+	asm volatile (
+		"r0 = 3;"
+		".8byte %[unwind_exit];"
+		:
+		: __imm_insn(unwind_exit,
+			     BPF_RAW_INSN(BPF_JMP | BPF_EXIT, 0, BPF_UNWIND_EXIT, 0, 0))
+	);
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/unwind_cxx.cpp b/tools/testing/selftests/bpf/progs/unwind_cxx.cpp
new file mode 100644
index 000000000000..3a113ffea29c
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/unwind_cxx.cpp
@@ -0,0 +1,150 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * C++ exceptions. llvm lowers unwinding to two insns:
+ *   invoke foo unwind pad    call foo; unwind_goto pad
+ *   resume                   r0 = 0; unwind_exit
+ * so 'throw' is a jump or an exit, 'catch (...)' and the destructors that run on
+ * the way are ordinary blocks.
+ *
+ * vmlinux.h and bpf_helpers.h are not C++.
+ */
+#define SEC(name) __attribute__((section(name), used))
+#define __ksym __attribute__((section(".ksyms")))
+#define __noinline __attribute__((noinline))
+#define __always_inline inline __attribute__((always_inline))
+
+#ifdef __BPF_FEATURE_UNWIND
+
+struct __sk_buff {
+	unsigned int len;
+};
+
+extern "C" {
+void bpf_rcu_read_lock(void) noexcept __ksym;
+void bpf_rcu_read_unlock(void) noexcept __ksym;
+void bpf_preempt_disable(void) noexcept __ksym;
+void bpf_preempt_enable(void) noexcept __ksym;
+
+/* Starts an unwind. Not a kfunc: llvm lowers the call and it doesn't come back. */
+int _Unwind_RaiseException(void *exc);
+
+/* All that is left of the C++ runtime. Nobody looks at the exception object. */
+static char exc_obj[16];
+
+__always_inline void *__cxa_allocate_exception(unsigned long size) noexcept
+{
+	return exc_obj;
+}
+
+__always_inline void __cxa_throw(void *exc, void *tinfo, void (*dtor)(void *))
+{
+	_Unwind_RaiseException(exc);
+}
+
+__always_inline void *__cxa_begin_catch(void *exc) noexcept
+{
+	return exc;
+}
+
+__always_inline void __cxa_end_catch(void) noexcept
+{
+}
+}
+
+/* One decimal digit per destructor, in the order they ran */
+volatile int dtors SEC(".data.dtors");
+
+/* The verifier insists that both are undone on every path out of the prog */
+struct rcu_guard {
+	__always_inline rcu_guard()
+	{
+		bpf_rcu_read_lock();
+	}
+	__always_inline ~rcu_guard()
+	{
+		bpf_rcu_read_unlock();
+		dtors = dtors * 10 + 1;
+	}
+};
+
+struct plain_rcu_guard {
+	__always_inline plain_rcu_guard()
+	{
+		bpf_rcu_read_lock();
+	}
+	__always_inline ~plain_rcu_guard()
+	{
+		bpf_rcu_read_unlock();
+	}
+};
+
+struct preempt_guard {
+	__always_inline preempt_guard()
+	{
+		bpf_preempt_disable();
+	}
+	__always_inline ~preempt_guard()
+	{
+		bpf_preempt_enable();
+		dtors = dtors * 10 + 2;
+	}
+};
+
+static __noinline int inner(int len)
+{
+	rcu_guard guard;
+
+	if (len > 100)
+		throw 1;
+	return len;
+}
+
+static __noinline int middle(int len)
+{
+	preempt_guard guard;
+
+	return inner(len) + 1;
+}
+
+using u128 = unsigned __int128;
+
+/* Global subprogs returning 16 bytes use R0:R2, including unwind exits. */
+extern "C" __noinline u128 aggregate_return(int len)
+{
+	plain_rcu_guard guard;
+
+	if (len > 100)
+		throw 1;
+	return (static_cast<u128>(len) << 64) | 7;
+}
+
+extern "C" SEC("tc") int catches(struct __sk_buff *skb)
+{
+	try {
+		return middle(skb->len);
+	} catch (...) {
+		return 42;
+	}
+}
+
+/* The exception leaves the prog: it returns 0 */
+extern "C" SEC("tc") int does_not_catch(struct __sk_buff *skb)
+{
+	return middle(skb->len);
+}
+
+extern "C" SEC("tc") int catches_aggregate(struct __sk_buff *skb)
+{
+	try {
+		u128 value = aggregate_return(skb->len);
+
+		return static_cast<unsigned long>(value) +
+		       static_cast<unsigned long>(value >> 64);
+	} catch (...) {
+		return 42;
+	}
+}
+
+#endif
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_unwind.c b/tools/testing/selftests/bpf/progs/verifier_unwind.c
new file mode 100644
index 000000000000..0a75cb8af67f
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/verifier_unwind.c
@@ -0,0 +1,1211 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include "../../../include/linux/filter.h"
+#include "bpf_misc.h"
+
+/*
+ * llvm lowers unwinding to two insns:
+ *   invoke foo unwind pad    call foo; unwind_goto pad
+ *   resume                   unwind_exit
+ * The tests spell them out as raw insns, so they don't depend on llvm version.
+ * The argument of unwind_goto() is the number of insns to skip.
+ */
+#define unwind_goto(off) ".8byte %[insn_unwind_goto_" #off "];"
+#define unwind_exit ".8byte %[insn_unwind_exit];"
+
+#define __imm_unwind_goto(off) \
+	__imm_insn(insn_unwind_goto_##off, \
+		   BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, BPF_UNWIND_GOTO, off, 0))
+#define __imm_unwind_exit \
+	__imm_insn(insn_unwind_exit, \
+		   BPF_RAW_INSN(BPF_JMP | BPF_EXIT, 0, BPF_UNWIND_EXIT, 0, 0))
+
+extern void bpf_rcu_read_lock(void) __ksym;
+extern void bpf_rcu_read_unlock(void) __ksym;
+extern void bpf_throw(__u64 cookie) __ksym;
+
+struct {
+	__uint(type, BPF_MAP_TYPE_PROG_ARRAY);
+	__uint(max_entries, 1);
+	__uint(key_size, sizeof(int));
+	__uint(value_size, sizeof(int));
+} prog_array SEC(".maps");
+
+struct timer_elem {
+	struct bpf_timer t;
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARRAY);
+	__uint(max_entries, 1);
+	__type(key, int);
+	__type(value, struct timer_elem);
+} timer_map SEC(".maps");
+
+/* The verifier doesn't know what is in there */
+unsigned long zero;
+unsigned long one = 1;
+
+/*
+ * The verifier follows a static subprog, so it knows which way it leaves when
+ * it knows the argument. To have both ways at run time the tests pass skb->len,
+ * which is 64, or skb->len - 64.
+ *
+ * r1 == 0: returns 7, otherwise unwinds with r0 = 5
+ */
+__naked __noinline __used
+static unsigned long may_unwind(void)
+{
+	asm volatile (
+		"if r1 != 0 goto 1f;"
+		"r0 = 7;"
+		"exit;"
+	"1:"
+		"r0 = 5;"
+		unwind_exit
+		:
+		: __imm_unwind_exit
+	);
+}
+
+/* owns nothing: its pad only carries on */
+__naked __noinline __used
+static unsigned long passes_on(void)
+{
+	asm volatile (
+		"call may_unwind;"
+		unwind_goto(2)
+		"r0 += 1;"
+		"exit;"
+		unwind_exit
+		:
+		: __imm_unwind_goto(2),
+		  __imm_unwind_exit
+	);
+}
+
+__naked __noinline __used
+static unsigned long always_unwinds(void)
+{
+	asm volatile (
+		"r0 = 5;"
+		unwind_exit
+		:
+		: __imm_unwind_exit
+	);
+}
+
+__naked __noinline __used
+static unsigned long never_unwinds(void)
+{
+	asm volatile (
+		"r0 = 7;"
+		"exit;"
+	);
+}
+
+/* Both can unwind. This one returns 7. */
+__naked __noinline __used
+unsigned long global_returns(void)
+{
+	asm volatile (
+		"r1 = %[zero] ll;"
+		"r1 = *(u64 *)(r1 + 0);"
+		"if r1 != 0 goto 1f;"
+		"r0 = 7;"
+		"exit;"
+	"1:"
+		"r0 = 5;"
+		unwind_exit
+		:
+		: __imm_addr(zero),
+		  __imm_unwind_exit
+	);
+}
+
+/* This one unwinds with r0 = 5 */
+__naked __noinline __used
+unsigned long global_unwinds(void)
+{
+	asm volatile (
+		"r1 = %[one] ll;"
+		"r1 = *(u64 *)(r1 + 0);"
+		"if r1 != 0 goto 1f;"
+		"r0 = 7;"
+		"exit;"
+	"1:"
+		"r0 = 5;"
+		unwind_exit
+		:
+		: __imm_addr(one),
+		  __imm_unwind_exit
+	);
+}
+
+SEC("tc")
+__description("unwind: the callee returns")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(7)
+__naked int callee_returns(void)
+{
+	asm volatile (
+		"r1 = *(u32 *)(r1 + %[len]);"
+		"r1 -= 64;"
+		"call may_unwind;"
+		unwind_goto(1)
+		"exit;"
+		"r0 = 42;"
+		"exit;"
+		:
+		: __imm_const(len, offsetof(struct __sk_buff, len)),
+		  __imm_unwind_goto(1)
+		: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: the callee unwinds")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(42)
+__naked int callee_unwinds(void)
+{
+	asm volatile (
+		"r1 = *(u32 *)(r1 + %[len]);"
+		"call may_unwind;"
+		unwind_goto(1)
+		"exit;"
+		"r0 = 42;"
+		"exit;"
+		:
+		: __imm_const(len, offsetof(struct __sk_buff, len)),
+		  __imm_unwind_goto(1)
+		: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: return through two frames")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(8)
+__naked int two_frames_return(void)
+{
+	asm volatile (
+		"r1 = *(u32 *)(r1 + %[len]);"
+		"r1 -= 64;"
+		"call passes_on;"
+		unwind_goto(1)
+		"exit;"
+		"r0 = 42;"
+		"exit;"
+		:
+		: __imm_const(len, offsetof(struct __sk_buff, len)),
+		  __imm_unwind_goto(1)
+		: __clobber_all
+	);
+}
+
+/* unwind_exit returns r0 like exit does */
+SEC("tc")
+__description("unwind: unwind through two frames, r0 reaches the pad")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(15)
+__naked int two_frames_unwind(void)
+{
+	asm volatile (
+		"r1 = *(u32 *)(r1 + %[len]);"
+		"call passes_on;"
+		unwind_goto(1)
+		"exit;"
+		"r0 += 10;"
+		"exit;"
+		:
+		: __imm_const(len, offsetof(struct __sk_buff, len)),
+		  __imm_unwind_goto(1)
+		: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: r6-r9 and the stack are intact in the pad")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(1111)
+__naked int pad_sees_callee_saved(void)
+{
+	asm volatile (
+		"r1 = *(u32 *)(r1 + %[len]);"
+		"r6 = 1;"
+		"r7 = 10;"
+		"r8 = 100;"
+		"r9 = 400;"
+		"*(u64 *)(r10 - 8) = 600;"
+		"call passes_on;"
+		unwind_goto(2)
+		"r0 = 0;"
+		"exit;"
+		"r0 = r6;"
+		"r0 += r7;"
+		"r0 += r8;"
+		"r0 += r9;"
+		"r1 = *(u64 *)(r10 - 8);"
+		"r0 += r1;"
+		"exit;"
+		:
+		: __imm_const(len, offsetof(struct __sk_buff, len)),
+		  __imm_unwind_goto(2)
+		: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: r1-r5 are not readable in the pad")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__failure __msg("R3 !read_ok")
+__naked int pad_reads_r3(void)
+{
+	asm volatile (
+		"r3 = 0;"
+		"call always_unwinds;"
+		unwind_goto(1)
+		"exit;"
+		"r0 = r3;"
+		"exit;"
+		:
+		: __imm_unwind_goto(1)
+		: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: unwind_exit of the main prog is exit")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(3)
+__naked int main_prog_unwinds(void)
+{
+	asm volatile (
+		"r0 = 3;"
+		unwind_exit
+		:
+		: __imm_unwind_exit
+		: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: unwind_exit of the main prog needs r0")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__failure __msg("R0 !read_ok")
+__naked int main_prog_unwinds_no_r0(void)
+{
+	asm volatile (
+		unwind_exit
+		:
+		: __imm_unwind_exit
+		: __clobber_all
+	);
+}
+
+/*
+ * What follows unwind_goto is dead code. unwind_exit still has to find the pad,
+ * and the cpu predicts that the callee returns: there is a trap for that.
+ */
+SEC("tc")
+__description("unwind: the callee never returns")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(42)
+__xlated("0: call pc+4")
+__xlated("1: unwind_goto pc+1")
+__xlated("2: goto pc-1")
+__xlated("3: r0 = 42")
+__xlated("4: exit")
+__naked int callee_never_returns(void)
+{
+	asm volatile (
+		"call always_unwinds;"
+		unwind_goto(2)
+		"r0 = 1;"
+		"exit;"
+		"r0 = 42;"
+		"exit;"
+		:
+		: __imm_unwind_goto(2)
+		: __clobber_all
+	);
+}
+
+/* The pad is dead code and goes away together with unwind_goto */
+SEC("tc")
+__description("unwind: the callee never unwinds")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(7)
+__xlated("0: call pc+1")
+__xlated("1: exit")
+__xlated("2: r0 = 7")
+__xlated("3: exit")
+__naked int callee_never_unwinds(void)
+{
+	asm volatile (
+		"call never_unwinds;"
+		unwind_goto(1)
+		"exit;"
+		"r0 = *(u64 *)(r0 + 0);"
+		"exit;"
+		:
+		: __imm_unwind_goto(1)
+		: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: the pad drops what the frame owns")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(42)
+__naked int pad_unlocks(void)
+{
+	asm volatile (
+		"r6 = *(u32 *)(r1 + %[len]);"
+		"call %[bpf_rcu_read_lock];"
+		"r1 = r6;"
+		"call may_unwind;"
+		unwind_goto(3)
+		"call %[bpf_rcu_read_unlock];"
+		"r0 = 0;"
+		"exit;"
+		"call %[bpf_rcu_read_unlock];"
+		"r0 = 42;"
+		"exit;"
+		:
+		: __imm_const(len, offsetof(struct __sk_buff, len)),
+		  __imm(bpf_rcu_read_lock),
+		  __imm(bpf_rcu_read_unlock),
+		  __imm_unwind_goto(3)
+		: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: the pad is checked like any other code")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__failure __msg("in main prog cannot be used inside bpf_rcu_read_lock-ed region")
+__naked int pad_does_not_unlock(void)
+{
+	asm volatile (
+		"r6 = *(u32 *)(r1 + %[len]);"
+		"call %[bpf_rcu_read_lock];"
+		"r1 = r6;"
+		"call may_unwind;"
+		unwind_goto(3)
+		"call %[bpf_rcu_read_unlock];"
+		"r0 = 0;"
+		"exit;"
+		"r0 = 42;"
+		"exit;"
+		:
+		: __imm_const(len, offsetof(struct __sk_buff, len)),
+		  __imm(bpf_rcu_read_lock),
+		  __imm(bpf_rcu_read_unlock),
+		  __imm_unwind_goto(3)
+		: __clobber_all
+	);
+}
+
+/* Leaves either way with the same registers */
+__naked __noinline __used
+static unsigned long same_regs(void)
+{
+	asm volatile (
+		"r0 = 0;"
+		"if r1 != 0 goto 1f;"
+		"exit;"
+	"1:"
+		unwind_exit
+		:
+		: __imm_unwind_exit
+	);
+}
+
+/*
+ * After the call the two states differ only in the way the callee left.
+ * The second one must not be pruned, or nobody looks at the pad.
+ */
+SEC("tc")
+__description("unwind: the way the callee left is a part of the state")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__failure __msg("R0 invalid mem access 'scalar'")
+__flag(BPF_F_TEST_STATE_FREQ)
+__naked int no_pruning_after_the_call(void)
+{
+	asm volatile (
+		"r1 = *(u32 *)(r1 + %[len]);"
+		"call same_regs;"
+		unwind_goto(1)
+		"exit;"
+		"r0 = *(u64 *)(r0 + 0);"
+		"exit;"
+		:
+		: __imm_const(len, offsetof(struct __sk_buff, len)),
+		  __imm_unwind_goto(1)
+		: __clobber_all
+	);
+}
+
+/* Returns ctx, or unwinds with a number in r0 */
+__naked __noinline __used
+static unsigned long ctx_or_unwind(void)
+{
+	asm volatile (
+		"r2 = *(u32 *)(r1 + %[len]);"
+		"if r2 > 100 goto 1f;"
+		"r0 = r1;"
+		"exit;"
+	"1:"
+		"r0 = 0;"
+		unwind_exit
+		:
+		: __imm_const(len, offsetof(struct __sk_buff, len)),
+		  __imm_unwind_exit
+	);
+}
+
+/*
+ * unwind_exit changes its return address, so the cpu mispredicts the return and
+ * runs what follows unwind_goto with r0 of unwind_exit.
+ */
+SEC("socket")
+__description("unwind: what follows unwind_goto runs speculatively")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(50) /* 64 without the ethernet header */
+__success_unpriv __retval_unpriv(50)
+__caps_unpriv(CAP_BPF)
+#ifdef SPEC_V1
+__xlated_unpriv("unwind_goto pc+{{.*}}")
+__xlated_unpriv("nospec") /* inserted to prevent `R0 invalid mem access 'scalar'` */
+__xlated_unpriv("r0 = *(u32 *)(r0 +{{.*}})")
+#endif
+__naked int speculative_fall_through(void)
+{
+	asm volatile (
+		"call ctx_or_unwind;"
+		unwind_goto(2)
+		"r0 = *(u32 *)(r0 + %[len]);"
+		"exit;"
+		"r0 = 42;"
+		"exit;"
+		:
+		: __imm_const(len, offsetof(struct __sk_buff, len)),
+		  __imm_unwind_goto(2)
+		: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: a global subprog returns")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(7)
+__naked int calls_global_returns(void)
+{
+	asm volatile (
+		"call global_returns;"
+		unwind_goto(1)
+		"exit;"
+		"r0 = 42;"
+		"exit;"
+		:
+		: __imm_unwind_goto(1)
+		: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: a global subprog unwinds")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(15)
+__naked int calls_global_unwinds(void)
+{
+	asm volatile (
+		"call global_unwinds;"
+		unwind_goto(1)
+		"exit;"
+		"r0 += 10;"
+		"exit;"
+		:
+		: __imm_unwind_goto(1)
+		: __clobber_all
+	);
+}
+
+/* The verifier doesn't look inside */
+SEC("tc")
+__description("unwind: the pad of a call of a global subprog is always checked")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__failure __msg("R0 invalid mem access 'scalar'")
+__naked int global_pad_is_checked(void)
+{
+	asm volatile (
+		"call global_returns;"
+		unwind_goto(1)
+		"exit;"
+		"r0 = *(u64 *)(r0 + 0);"
+		"exit;"
+		:
+		: __imm_unwind_goto(1)
+		: __clobber_all
+	);
+}
+
+__naked __noinline __used
+unsigned long global_never_unwinds(void)
+{
+	asm volatile (
+		"r0 = 7;"
+		"exit;"
+	);
+}
+
+SEC("tc")
+__description("unwind: a global subprog that cannot unwind")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(7)
+__naked int calls_global_never_unwinds(void)
+{
+	asm volatile (
+		"call global_never_unwinds;"
+		"exit;"
+		::: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: no unwind_goto after a call of a global subprog")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__failure __msg("no unwind_goto after the call of global_returns()")
+__naked int global_no_unwind_goto(void)
+{
+	asm volatile (
+		"call global_returns;"
+		"exit;"
+		::: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: no unwind_goto after a call of a static subprog")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__failure __msg("unwind_exit, but no unwind_goto after the call at insn 1")
+__naked int static_no_unwind_goto(void)
+{
+	asm volatile (
+		"r1 = *(u32 *)(r1 + %[len]);"
+		"call may_unwind;"
+		"exit;"
+		:
+		: __imm_const(len, offsetof(struct __sk_buff, len))
+		: __clobber_all
+	);
+}
+
+/* The verifier sees that it doesn't get to unwind_exit */
+SEC("tc")
+__description("unwind: no unwind_goto is fine when the callee doesn't unwind")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(7)
+__naked int static_does_not_get_there(void)
+{
+	asm volatile (
+		"r1 = 0;"
+		"call may_unwind;"
+		"exit;"
+		::: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: callx, the callee returns")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(7)
+__naked int callx_returns(void)
+{
+	asm volatile (
+		"r1 = *(u32 *)(r1 + %[len]);"
+		"r1 -= 64;"
+		"r6 = %[may_unwind] ll;"
+		"callx r6;"
+		unwind_goto(1)
+		"exit;"
+		"r0 = 42;"
+		"exit;"
+		:
+		: __imm_const(len, offsetof(struct __sk_buff, len)),
+		  __imm_addr(may_unwind),
+		  __imm_unwind_goto(1)
+		: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: callx, the callee unwinds")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(42)
+__naked int callx_unwinds(void)
+{
+	asm volatile (
+		"r1 = *(u32 *)(r1 + %[len]);"
+		"r6 = %[may_unwind] ll;"
+		"callx r6;"
+		unwind_goto(1)
+		"exit;"
+		"r0 = 42;"
+		"exit;"
+		:
+		: __imm_const(len, offsetof(struct __sk_buff, len)),
+		  __imm_addr(may_unwind),
+		  __imm_unwind_goto(1)
+		: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: unwind_goto after a plain insn")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__failure __msg("unwind_goto at insn 1 doesn't follow a call of a subprog")
+__naked int unwind_goto_after_mov(void)
+{
+	asm volatile (
+		"r0 = 0;"
+		unwind_goto(0)
+		"exit;"
+		:
+		: __imm_unwind_goto(0)
+		: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: unwind_goto is the first insn")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__failure __msg("unwind_goto at insn 0 doesn't follow a call of a subprog")
+__naked int unwind_goto_first(void)
+{
+	asm volatile (
+		unwind_goto(0)
+		"r0 = 0;"
+		"exit;"
+		:
+		: __imm_unwind_goto(0)
+		: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: unwind_goto after a call of a helper")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__failure __msg("unwind_goto at insn 1 doesn't follow a call of a subprog")
+__naked int unwind_goto_after_helper(void)
+{
+	asm volatile (
+		"call %[bpf_get_prandom_u32];"
+		unwind_goto(0)
+		"exit;"
+		:
+		: __imm(bpf_get_prandom_u32),
+		  __imm_unwind_goto(0)
+		: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: unwind_goto after a call of a kfunc")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__failure __msg("unwind_goto at insn 1 doesn't follow a call of a subprog")
+__naked int unwind_goto_after_kfunc(void)
+{
+	asm volatile (
+		"call %[bpf_rcu_read_lock];"
+		unwind_goto(0)
+		"call %[bpf_rcu_read_unlock];"
+		"r0 = 0;"
+		"exit;"
+		:
+		: __imm(bpf_rcu_read_lock),
+		  __imm(bpf_rcu_read_unlock),
+		  __imm_unwind_goto(0)
+		: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: jump to unwind_goto")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__failure __msg("jump to unwind_goto at insn 3")
+__naked int jump_to_unwind_goto(void)
+{
+	asm volatile (
+		"r0 = 0;"
+		"if r1 != 0 goto 1f;"
+		"call never_unwinds;"
+	"1:"
+		unwind_goto(0)
+		"exit;"
+		:
+		: __imm_unwind_goto(0)
+		: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: reserved fields of unwind_goto")
+__failure __msg("invalid may_goto imm 1")
+__naked int unwind_goto_reserved(void)
+{
+	asm volatile (
+		"call never_unwinds;"
+		".8byte %[bad];"
+		"exit;"
+		:
+		: __imm_insn(bad, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, BPF_UNWIND_GOTO, 0, 1))
+		: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: unknown kind of jcond")
+__failure __msg("invalid may_goto imm 0")
+__naked int jcond_unknown(void)
+{
+	asm volatile (
+		"call never_unwinds;"
+		".8byte %[bad];"
+		"exit;"
+		:
+		: __imm_insn(bad, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, BPF_UNWIND_GOTO + 1, 0, 0))
+		: __clobber_all
+	);
+}
+
+SEC("tc")
+__description("unwind: unknown kind of exit")
+__failure __msg("BPF_EXIT uses reserved fields")
+__naked int exit_unknown(void)
+{
+	asm volatile (
+		"r0 = 0;"
+		".8byte %[bad];"
+		:
+		: __imm_insn(bad, BPF_RAW_INSN(BPF_JMP | BPF_EXIT, 0, BPF_UNWIND_EXIT + 1, 0, 0))
+		: __clobber_all
+	);
+}
+
+/* r1 is the index: unwinds in the second iteration */
+__naked __noinline __used
+static unsigned long cb_unwinds(void)
+{
+	asm volatile (
+		"r0 = 0;"
+		"if r1 != 0 goto 1f;"
+		"exit;"
+	"1:"
+		unwind_exit
+		:
+		: __imm_unwind_exit
+	);
+}
+
+SEC("tc")
+__description("unwind: callback unwinds")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__failure __msg("callback cannot unwind")
+__naked int callback_unwinds(void)
+{
+	asm volatile (
+		"r1 = 3;"
+		"r2 = %[cb_unwinds];"
+		"r3 = 0;"
+		"r4 = 0;"
+		"call %[bpf_loop];"
+		"exit;"
+		:
+		: __imm_ptr(cb_unwinds),
+		  __imm(bpf_loop)
+		: __clobber_common
+	);
+}
+
+__naked __noinline __used
+static unsigned long async_cb_unwinds(void)
+{
+	asm volatile (
+		"r0 = 0;"
+		unwind_exit
+		:
+		: __imm_unwind_exit
+	);
+}
+
+/* The kernel calls it. It's the top frame like the main prog: unwind_exit is exit. */
+SEC("tc")
+__description("unwind: async callback unwinds")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(0)
+int async_callback_unwinds(void *ctx)
+{
+	struct timer_elem *val;
+	int key = 0;
+
+	val = bpf_map_lookup_elem(&timer_map, &key);
+	if (!val)
+		return 0;
+
+	bpf_timer_init(&val->t, &timer_map, 0);
+	bpf_timer_set_callback(&val->t, (void *)async_cb_unwinds);
+	bpf_timer_start(&val->t, 0, 0);
+	return 0;
+}
+
+/* Not naked: the argument has to have a name in BTF */
+__noinline int exception_cb_unwinds(__u64 cookie)
+{
+	if (cookie)
+		asm volatile ("r0 = 7;" unwind_exit :: __imm_unwind_exit : "r0");
+	return 0;
+}
+
+SEC("tc")
+__description("unwind: exception callback unwinds")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__attribute__((btf_decl_tag("exception_callback:exception_cb_unwinds")))
+__success __retval(7)
+int exception_callback_unwinds(struct __sk_buff *skb)
+{
+	bpf_throw(skb->len);
+	return 0;
+}
+
+/* What it calls can unwind as long as it doesn't let it out */
+__naked __noinline __used
+static unsigned long cb_catches(void)
+{
+	asm volatile (
+		"call may_unwind;"
+		unwind_goto(0)
+		"r0 = 0;"
+		"exit;"
+		:
+		: __imm_unwind_goto(0)
+	);
+}
+
+SEC("tc")
+__description("unwind: callback catches")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(3)
+__naked int callback_catches(void)
+{
+	asm volatile (
+		"r1 = 3;"
+		"r2 = %[cb_catches];"
+		"r3 = 0;"
+		"r4 = 0;"
+		"call %[bpf_loop];"
+		"exit;"
+		:
+		: __imm_ptr(cb_catches),
+		  __imm(bpf_loop)
+		: __clobber_common
+	);
+}
+
+/* Both leave the subprog like exit does */
+__naked __noinline __used
+static unsigned long does_tail_call(void)
+{
+	asm volatile (
+		"r6 = *(u32 *)(r1 + %[len]);"
+		"r2 = %[prog_array] ll;"
+		"r3 = 0;"
+		"call %[bpf_tail_call];"
+		"r0 = 7;"
+		"if r6 == 64 goto 1f;"
+		unwind_exit
+	"1:"
+		"exit;"
+		:
+		: __imm_const(len, offsetof(struct __sk_buff, len)),
+		  __imm(bpf_tail_call),
+		  __imm_addr(prog_array),
+		  __imm_unwind_exit
+	);
+}
+
+SEC("tc")
+__description("unwind: tail call in a subprog that can unwind")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(7)
+__naked int tail_call_in_subprog(void)
+{
+	asm volatile (
+		"call does_tail_call;"
+		unwind_goto(1)
+		"exit;"
+		"r0 = 42;"
+		"exit;"
+		:
+		: __imm_unwind_goto(1)
+		: __clobber_all
+	);
+}
+
+/* The packet is shorter than that: ld_abs leaves the subprog with r0 = 0 */
+__naked __noinline __used
+static unsigned long does_ld_abs(void)
+{
+	asm volatile (
+		"r6 = r1;"
+		"r0 = *(u8 *)skb[1000];"
+		"r0 = 5;"
+		unwind_exit
+		:
+		: __imm_unwind_exit
+	);
+}
+
+SEC("tc")
+__description("unwind: ld_abs in a subprog that can unwind")
+__arch_x86_64
+__arch_arm64
+__load_if_JITed()
+__success __retval(7)
+__naked int ld_abs_in_subprog(void)
+{
+	asm volatile (
+		"call does_ld_abs;"
+		unwind_goto(2)
+		"r0 += 7;"
+		"exit;"
+		"r0 = 42;"
+		"exit;"
+		:
+		: __imm_unwind_goto(2)
+		: __clobber_all
+	);
+}
+
+/*
+ * JITs don't emit anything for unwind_goto. It's an extable entry, where
+ * unwind_exit looks its return address up.
+ * With KASAN x86-64 JIT wraps a load in pushes, a call and pops. 'r2 = 0x1234'
+ * is where the patterns start, so that they don't take that call for the one
+ * of the subprog.
+ */
+SEC("tc")
+__description("unwind: jit")
+__success __retval(42)
+__load_if_JITed()
+__arch_x86_64
+__jited("	movl	$0x1234, %esi")
+__jited("...") /* call depth accounting */
+__jited("	callq	0x{{.*}}")
+__jited("	movl	$0x7, %eax")
+__jited("...")
+__jited("	movl	$0x5, %eax")
+__jited("	pushq	%rax")
+__jited("	pushq	%rsi")
+__jited("	movq	0x8(%rbp), %rdi")
+__jited("...") /* call depth accounting */
+__jited("	callq	0x{{.*}}")
+__jited("	movq	%rax, 0x8(%rbp)")
+__jited("	popq	%rsi")
+__jited("	popq	%rax")
+__arch_arm64
+__jited("	mov	x1, #0x1234")
+__jited("	bl	{{.*}}")
+__jited("	mov	x8, x0")
+__jited("	mov	x8, #0x7")
+__jited("...")
+__jited("	mov	x8, #0x5")
+__jited("	stp	x8, x1, [sp, #-0x10]!")
+__jited("	ldr	x0, [x29, #0x8]")
+__jited("...")
+__jited("	mov	x10, x0")
+__jited("	ldp	x8, x1, [sp], #0x10")
+__jited("...")
+__jited("	add	x30, x30, x10")
+__jited("	ret")
+__naked int jit(void)
+{
+	asm volatile (
+		"r1 = *(u32 *)(r1 + %[len]);"
+		"r2 = 0x1234;"
+		"call may_unwind;"
+		unwind_goto(2)
+		"r0 = 7;"
+		"exit;"
+		"r0 = 42;"
+		"exit;"
+		:
+		: __imm_const(len, offsetof(struct __sk_buff, len)),
+		  __imm_unwind_goto(2)
+		: __clobber_all
+	);
+}
+
+/* r1 == 0: returns 7, otherwise unwinds with r0 = 5 */
+__naked __noinline __used
+static unsigned long may_unwind_big_stack(void)
+{
+	asm volatile (
+		"*(u64 *)(r10 - 200) = r1;"
+		"if r1 != 0 goto 1f;"
+		"r0 = 7;"
+		"exit;"
+	"1:"
+		"r0 = 5;"
+		unwind_exit
+		:
+		: __imm_unwind_exit
+	);
+}
+
+/*
+ * x86-64 keeps the pointer to the private stack in r9 and saves it around a call.
+ * The value that is read back in the pad is there only when r9 is restored.
+ */
+SEC("raw_tp")
+__description("unwind: private stack, the callee unwinds")
+__success __retval(47)
+__load_if_JITed()
+__arch_x86_64
+__jited("	movl	$0x1234, %esi") /* past the load, see above */
+__jited("	pushq	%r9")
+__jited("...") /* call depth accounting */
+__jited("	callq	0x{{.*}}")
+__jited("	popq	%r9")
+__jited("	jmp	L{{.*}}")
+__jited("	popq	%r9")
+__jited("	jmp	L{{.*}}")
+__arch_arm64
+__naked int private_stack_unwinds(void)
+{
+	asm volatile (
+		"*(u64 *)(r10 - 200) = 42;"
+		"r1 = %[one] ll;"
+		"r1 = *(u64 *)(r1 + 0);"
+		"r2 = 0x1234;"
+		"call may_unwind_big_stack;"
+		unwind_goto(1)
+		"exit;"
+		"r1 = *(u64 *)(r10 - 200);"
+		"r0 += r1;"
+		"exit;"
+		:
+		: __imm_addr(one),
+		  __imm_unwind_goto(1)
+		: __clobber_all
+	);
+}
+
+SEC("raw_tp")
+__description("unwind: private stack, the callee returns")
+__success __retval(49)
+__load_if_JITed()
+__arch_x86_64
+__arch_arm64
+__naked int private_stack_returns(void)
+{
+	asm volatile (
+		"*(u64 *)(r10 - 200) = 42;"
+		"r1 = %[zero] ll;"
+		"r1 = *(u64 *)(r1 + 0);"
+		"call may_unwind_big_stack;"
+		unwind_goto(3)
+		"r1 = *(u64 *)(r10 - 200);"
+		"r0 += r1;"
+		"exit;"
+		"r0 = 0;"
+		"exit;"
+		:
+		: __imm_addr(zero),
+		  __imm_unwind_goto(3)
+		: __clobber_all
+	);
+}
+
+/*
+ * BTF FUNC records are not generated for kfuncs referenced
+ * from inline assembly. These records are necessary for
+ * libbpf to link the program. The function below is a hack
+ * to ensure that BTF FUNC records are generated.
+ */
+void __kfunc_btf_root(void)
+{
+	bpf_rcu_read_lock();
+	bpf_rcu_read_unlock();
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c
index 18d47e925400..824e5912ebcb 100644
--- a/tools/testing/selftests/bpf/test_loader.c
+++ b/tools/testing/selftests/bpf/test_loader.c
@@ -1546,7 +1546,8 @@ void run_subtest(struct test_loader *tester,
 		}
 
 		err = do_prog_test_run(bpf_program__fd(tprog), &retval,
-				       bpf_program__type(tprog) == BPF_PROG_TYPE_SYSCALL ? true : false,
+				       bpf_program__type(tprog) == BPF_PROG_TYPE_SYSCALL ||
+				       bpf_program__type(tprog) == BPF_PROG_TYPE_RAW_TRACEPOINT,
 				       spec->linear_sz);
 		if (!err && retval != subspec->retval && subspec->retval != POINTER_VALUE) {
 			PRINT_FAIL("Unexpected retval: %d != %d\n", retval, subspec->retval);