diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index b0bd9ba951df..d8a579680e5b 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -393,7 +393,7 @@ subprog_jt(int t, struct bpf_verifier_env *env) if (!subprog->jt) { verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start); bpf_diag_program_structure( - env, subprog_start, "missing jump table", + env, t, "missing jump table", "Make sure subprograms containing gotox instructions are accompanied by jump tables referencing these subprograms.", "No jump table was found for the subprogram that starts at instruction %u.", subprog_start); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 1599bac1bac9..b840b3eb9b22 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3107,6 +3107,34 @@ static int add_kfuncs(struct bpf_verifier_env *env) return 0; } +static void find_subprog_properties(struct bpf_verifier_env *env) +{ + struct bpf_subprog_info *subprog = env->subprog_info; + struct bpf_insn *insn = env->prog->insnsi; + int cur_subprog; + + for (cur_subprog = 0; cur_subprog < env->subprog_cnt; cur_subprog++) { + int i; + + for (i = subprog[cur_subprog].start; + i < subprog[cur_subprog + 1].start; i++) { + u8 code = insn[i].code; + + if (code == (BPF_JMP | BPF_CALL) && + insn[i].src_reg == 0 && + insn[i].imm == BPF_FUNC_tail_call) { + subprog[cur_subprog].has_tail_call = true; + subprog[cur_subprog].tail_call_reachable = true; + } + if (BPF_CLASS(code) == BPF_LD && + (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) + subprog[cur_subprog].has_ld_abs = true; + if (bpf_is_callx(&insn[i])) + env->has_callx = true; + } + } +} + static int check_subprogs(struct bpf_verifier_env *env) { int i, subprog_start, subprog_end, off, cur_subprog = 0; @@ -3120,17 +3148,6 @@ static int check_subprogs(struct bpf_verifier_env *env) for (i = 0; i < insn_cnt; i++) { u8 code = insn[i].code; - if (code == (BPF_JMP | BPF_CALL) && - insn[i].src_reg == 0 && - insn[i].imm == BPF_FUNC_tail_call) { - subprog[cur_subprog].has_tail_call = true; - subprog[cur_subprog].tail_call_reachable = true; - } - if (BPF_CLASS(code) == BPF_LD && - (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) - subprog[cur_subprog].has_ld_abs = true; - if (bpf_is_callx(&insn[i])) - env->has_callx = true; if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) goto next; if (BPF_OP(code) == BPF_CALL) @@ -3154,9 +3171,10 @@ static int check_subprogs(struct bpf_verifier_env *env) } next: if (i == subprog_end - 1) { - /* to avoid fall-through from one subprog into another + /* + * To avoid fall-through from one subprog into another, * the last insn of the subprog should be either exit - * or unconditional jump back or bpf_throw call + * or unconditional jump back or bpf_throw call. */ if (code != (BPF_JMP | BPF_EXIT) && code != (BPF_JMP32 | BPF_JA) && @@ -22511,17 +22529,19 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; - /* Discover all subprograms before validating their layout and BTF. */ + /* Discover all subprograms and collect the properties needed by BTF validation. */ ret = add_subprogs(env); if (ret < 0) goto skip_full_check; - ret = check_subprogs(env); + find_subprog_properties(env); + + /* Validate BTF before reporting subprogram layout errors. */ + ret = bpf_check_btf_info(env, attr, uattr); if (ret < 0) goto skip_full_check; - /* Validate BTF against the complete subprogram layout. */ - ret = bpf_check_btf_info(env, attr, uattr); + ret = check_subprogs(env); if (ret < 0) goto skip_full_check; diff --git a/tools/testing/selftests/bpf/progs/preempt_lock.c b/tools/testing/selftests/bpf/progs/preempt_lock.c index 81c459435680..955391da326a 100644 --- a/tools/testing/selftests/bpf/progs/preempt_lock.c +++ b/tools/testing/selftests/bpf/progs/preempt_lock.c @@ -6,6 +6,8 @@ #include "bpf_experimental.h" extern int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void *unsafe_ptr__ign, u64 flags) __weak __ksym; +extern void bpf_rcu_read_lock(void) __ksym; +extern void bpf_rcu_read_unlock(void) __ksym; SEC("?tc") __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_preempt_disable-ed region") @@ -179,6 +181,30 @@ int preempt_sleepable_kfunc(void *ctx) return 0; } +SEC("?fentry/" SYS_PREFIX "sys_getpgid") +__failure __msg("program must be sleepable to call sleepable kfunc bpf_copy_from_user_str") +__msg("cannot be used in non-sleepable program") +int non_sleepable_kfunc(void *ctx) +{ + u32 data; + + bpf_copy_from_user_str(&data, sizeof(data), NULL, 0); + return 0; +} + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +__failure __msg("kernel func bpf_copy_from_user_str is sleepable within rcu_read_lock region") +__msg("cannot be used in RCU read lock region") +int sleepable_kfunc_in_rcu(void *ctx) +{ + u32 data; + + bpf_rcu_read_lock(); + bpf_copy_from_user_str(&data, sizeof(data), NULL, 0); + bpf_rcu_read_unlock(); + return 0; +} + int __noinline preempt_global_subprog(void) { preempt_balance_subprog(); diff --git a/tools/testing/selftests/bpf/progs/verifier_cfg.c b/tools/testing/selftests/bpf/progs/verifier_cfg.c index 6379dfc9389b..b429fa7a08f5 100644 --- a/tools/testing/selftests/bpf/progs/verifier_cfg.c +++ b/tools/testing/selftests/bpf/progs/verifier_cfg.c @@ -56,6 +56,46 @@ __naked void out_of_range_jump2(void) " ::: __clobber_all); } +static __naked __noinline __used int cross_subprog_target(void) +{ + asm volatile (" \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("jump across subprogram boundary") +__failure __msg("jump out of range from insn 1") +__msg("jump_across_subprog_boundary @ verifier_cfg.c") +__naked void jump_across_subprog_boundary(void) +{ + asm volatile (" \ + call cross_subprog_target; \ + goto +1; \ + exit; \ +" ::: __clobber_all); +} + +static __naked __noinline __used int fallthrough_subprog(void) +{ + asm volatile (" \ + r0 = 0; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("subprogram fallthrough") +__failure __msg("last insn is not an exit or jmp") +__msg("fallthrough_subprog @ verifier_cfg.c") +__naked void subprog_fallthrough(void) +{ + asm volatile (" \ + call fallthrough_subprog; \ + exit; \ +" ::: __clobber_all); +} + SEC("socket") __description("invalid DW LDSX instruction in diagnostics") __failure __msg("BUG_ldx_99") diff --git a/tools/testing/selftests/bpf/progs/verifier_gotox.c b/tools/testing/selftests/bpf/progs/verifier_gotox.c index f5a9878c7b8d..dc7baa0f2458 100644 --- a/tools/testing/selftests/bpf/progs/verifier_gotox.c +++ b/tools/testing/selftests/bpf/progs/verifier_gotox.c @@ -101,9 +101,11 @@ __naked void jump_table_terminal_gotox_subprog(void) */ SEC("socket") __failure __msg("no jump tables found for subprog starting at 0") +__msg(">>> 1 | (0d) gotox r0") __naked void jump_table_no_jump_table(void) { asm volatile (" \ + r0 = 0; \ .8byte %[gotox_r0]; \ r0 = 1; \ exit; \