diff --git a/Documentation/bpf/clang-notes.rst b/Documentation/bpf/clang-notes.rst index 2c872a1ee08e..3ccc7b09d19e 100644 --- a/Documentation/bpf/clang-notes.rst +++ b/Documentation/bpf/clang-notes.rst @@ -23,8 +23,11 @@ For CPU versions prior to 3, Clang v7.0 and later can enable ``BPF_ALU`` support Jump instructions ================= -If ``-O0`` is used, Clang will generate the ``BPF_CALL | BPF_X | BPF_JMP`` (0x8d) -instruction, which is not supported by the Linux kernel verifier. +Clang generates the ``BPF_CALL | BPF_X | BPF_JMP`` (0x8d) instruction for calls +through a function pointer. The Linux kernel verifier accepts it only when it +can prove that the register holds the address of a static BPF function, see +Documentation/bpf/linux-notes.rst. If ``-O0`` is used, Clang will generate this +instruction for helper calls as well, which is not supported. Atomic operations ================= diff --git a/Documentation/bpf/linux-notes.rst b/Documentation/bpf/linux-notes.rst index 00d2693de025..6c036b54a29f 100644 --- a/Documentation/bpf/linux-notes.rst +++ b/Documentation/bpf/linux-notes.rst @@ -15,10 +15,33 @@ Byte swap instructions Jump instructions ================= -``BPF_CALL | BPF_X | BPF_JMP`` (0x8d), where the helper function -integer would be read from a specified register, is not currently supported -by the verifier. Any programs with this instruction will fail to load -until such support is added. +``BPF_CALL | BPF_X | BPF_JMP`` (0x8d), ``callx dst``, performs an indirect +call of a BPF function whose address is held in the ``dst`` register. The +``src``, ``offset`` and ``imm`` fields are reserved and must be zero. + +The address of a BPF function gets into a register in one of two ways: + +* it is loaded by a 64-bit immediate instruction with ``src`` = + ``BPF_PSEUDO_FUNC``; +* it is read, with a 64-bit load, from a frozen read-only array map, that no + other program uses, that holds its read-only data: tables of functions, + structures of operations, vtables, where pointers to functions may be mixed + with other data. In the map a pointer to a function is the offset in bytes + of its first instruction in the program, and that is how the verifier + recognizes it. It is replaced with the address of the function when + the program is loaded. The program reads it from there, which requires + ``CAP_PERFMON``. + +In both cases only static functions can be referenced. Therefore all functions +that can be called indirectly are known to the verifier before it starts to +analyze the program, and ``callx`` is verified as a direct call of every +function that ``dst`` may point to at that instruction. The same rules apply: +the calls can not be recursive, and the depth of the call chain and its +combined stack size are limited. + +Calling helper or kernel functions through a register, indirect calls of global +functions, and tail calls in functions that are called via ``callx`` are not +supported. ``callx`` requires the BPF JIT. Maps ==== diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 6c04fee46876..9544b2f483e5 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1789,6 +1789,17 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn emit(A64_MOV(1, r0, A64_R(0)), ctx); break; } + /* indirect call of a bpf subprog, dst holds its address */ + case BPF_JMP | BPF_CALL | BPF_X: + /* + * It's the same as a direct call of a subprog that is out of + * range of BL: the subprog starts with BTI JC, the arguments + * are in place, and the registers that hold the tail call + * counter and the private stack are callee saved. + */ + emit(A64_BLR(dst), ctx); + emit(A64_MOV(1, bpf2a64[BPF_REG_0], A64_R(0)), ctx); + break; /* tail call */ case BPF_JMP | BPF_TAIL_CALL: if (emit_bpf_tail_call(ctx)) @@ -2485,6 +2496,11 @@ bool bpf_jit_supports_subprog_tailcalls(void) return true; } +bool bpf_jit_supports_callx(void) +{ + return true; +} + static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *node, int bargs_off, int retval_off, int run_ctx_off, bool save_ret) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index d4a980140b48..9fbef7504e51 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -749,6 +749,46 @@ static void emit_indirect_jump(u8 **pprog, int bpf_reg, u8 *ip) *pprog = prog; } +static void __emit_indirect_call(u8 **pprog, int reg, bool ereg) +{ + u8 *prog = *pprog; + + if (ereg) + EMIT1(0x41); + + EMIT2(0xFF, 0xD0 + reg); + + *pprog = prog; +} + +/* call *bpf_reg */ +static int emit_indirect_call(u8 **pprog, int bpf_reg, u8 *ip) +{ + u8 *prog = *pprog; + int reg = reg2hex[bpf_reg]; + bool ereg = is_ereg(bpf_reg); + int err = 0; + + if (cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) { + OPTIMIZER_HIDE_VAR(reg); + err = emit_call(&prog, its_static_thunk(reg + 8*ereg), ip); + } else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) { + EMIT_LFENCE(); + __emit_indirect_call(&prog, reg, ereg); + } else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) { + OPTIMIZER_HIDE_VAR(reg); + if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH)) + err = emit_call(&prog, &__x86_indirect_call_thunk_array[reg + 8*ereg], ip); + else + err = emit_call(&prog, &__x86_indirect_thunk_array[reg + 8*ereg], ip); + } else { + __emit_indirect_call(&prog, reg, ereg); + } + + *pprog = prog; + return err; +} + static void emit_return(u8 **pprog, u8 *ip) { u8 *prog = *pprog; @@ -2941,6 +2981,24 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * break; } + /* callx: call of a bpf subprog whose address is in dst_reg */ + case BPF_JMP | BPF_CALL | BPF_X: + /* + * The verifier makes sure that callees of callx are + * not tail call reachable, hence unlike a direct call + * of a subprog there is no need to pass + * tail_call_cnt_ptr in rax. + */ + if (priv_frame_ptr) { + push_r9(&prog); + ip += 2; + } + if (emit_indirect_call(&prog, insn->dst_reg, ip)) + return -EINVAL; + if (priv_frame_ptr) + pop_r9(&prog); + break; + case BPF_JMP | BPF_TAIL_CALL: if (imm32) emit_bpf_tail_call_direct(bpf_prog, @@ -4414,6 +4472,16 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr return prog; } +bool bpf_jit_supports_callx(void) +{ + /* + * FineIBT poisons ENDBR at the entry of a JITed function and expects + * indirect callers to go through the CFI preamble instead. + * callx doesn't do that yet. + */ + return cfi_mode != CFI_FINEIBT; +} + bool bpf_jit_supports_kfunc_call(void) { return true; diff --git a/include/linux/bpf.h b/include/linux/bpf.h index e7c5e203eddd..9525d3ecd923 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -344,8 +344,18 @@ struct bpf_map { s64 __percpu *elem_count; u64 cookie; /* write-once */ char *excl_prog_sha; + /* + * Which programs use the map, see bpf_map_claim(): 0 - none so far, + * aux of the program - only that one, the same with BPF_MAP_USER_PATCHED + * set - only that one and it stored the addresses of its functions into + * the map, BPF_MAP_USER_MANY - more than one. + */ + unsigned long user; }; +#define BPF_MAP_USER_MANY 1UL +#define BPF_MAP_USER_PATCHED 1UL + static inline const char *btf_field_type_name(enum btf_field_type type) { switch (type) { diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 92f528c45605..578bccf5cf76 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -786,6 +786,21 @@ int bpf_log_attr_finalize(struct bpf_log_attr *attr, struct bpf_verifier_log *lo #define BPF_MAX_SUBPROGS 256 +/* + * A pointer to a static subprog in the value of a frozen read-only array map: + * a 64-bit value that is the offset in bytes of the first instruction of + * the subprog in the program. + */ +struct bpf_func_ptr { + struct bpf_map *map; + u32 map_off; /* offset of the pointer in the value of the map */ + u32 orig_off; /* what the map has: the first instruction of the subprog */ + u32 xlated_off; /* the same after instructions were patched and removed */ +}; + +/* the subprog that a bpf_func_ptr pointed to was removed as dead code */ +#define BPF_FUNC_PTR_DELETED ((u32)-1) + struct bpf_subprog_arg_info { enum bpf_arg_type arg_type; union { @@ -965,6 +980,20 @@ struct bpf_verifier_env { struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 2]; /* max + 2 for the fake and exception subprogs */ /* subprog indices sorted in topological order: leaves first, callers last */ int subprog_topo_order[BPF_MAX_SUBPROGS + 2]; + /* + * Pointers to static subprogs found in frozen read-only maps of the + * program, see resolve_func_ptrs(). Sorted by map and map_off. + */ + struct bpf_func_ptr *func_ptrs; + u32 func_ptr_cnt; + bool has_callx; + /* + * Call graph edges created by callx instructions. A bitmap of + * subprog_cnt * subprog_cnt bits, where bit (caller * subprog_cnt + callee) + * is set when the main verification pass sees 'caller' calling 'callee' + * via callx. Allocated when the first such edge is recorded. + */ + unsigned long *callx_edges; union { struct bpf_idmap idmap_scratch; struct bpf_idset idset_scratch; @@ -1089,6 +1118,12 @@ static inline bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn) insn->src_reg == BPF_PSEUDO_KFUNC_CALL; } +/* callx: indirect call of a bpf subprog whose address is in insn->dst_reg */ +static inline bool bpf_is_callx(const struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_CALL | BPF_X); +} + __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, va_list args); __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, @@ -1311,6 +1346,13 @@ static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u } bool bpf_map_is_rdonly(const struct bpf_map *map); +struct bpf_func_ptr *bpf_map_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, u32 *cnt); +struct bpf_func_ptr *bpf_map_range_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, + u64 off, u64 size, u32 *cnt); +void bpf_adjust_func_ptrs(struct bpf_verifier_env *env, u32 off, u32 len); +void bpf_adjust_func_ptrs_after_remove(struct bpf_verifier_env *env, u32 off, u32 len); int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val, bool is_ldsx); diff --git a/include/linux/filter.h b/include/linux/filter.h index 422284b4fa96..4f0662e42897 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1237,6 +1237,7 @@ bool bpf_jit_inlines_helper_call(s32 imm); bool bpf_jit_supports_subprog_tailcalls(void); bool bpf_jit_supports_percpu_insn(void); bool bpf_jit_supports_kfunc_call(void); +bool bpf_jit_supports_callx(void); bool bpf_jit_supports_kfunc_ret_reg_pair(void); bool bpf_jit_supports_stack_args(void); bool bpf_jit_supports_arena_args(void); diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 507a366dffa4..4da99dec0818 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -406,15 +406,18 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, if (class == BPF_STX) bt_set_reg(bt, sreg); } else if (class == BPF_JMP || class == BPF_JMP32) { - if (bpf_pseudo_call(insn)) { - int subprog_insn_idx, subprog; + if (bpf_pseudo_call(insn) || bpf_is_callx(insn)) { + int subprog_insn_idx, subprog = -1; - subprog_insn_idx = idx + insn->imm + 1; - subprog = bpf_find_subprog(env, subprog_insn_idx); - if (subprog < 0) - return -EFAULT; + if (bpf_pseudo_call(insn)) { + subprog_insn_idx = idx + insn->imm + 1; + subprog = bpf_find_subprog(env, subprog_insn_idx); + if (subprog < 0) + return -EFAULT; + } - if (bpf_subprog_is_global(env, subprog)) { + /* callx calls static subprogs only */ + if (subprog >= 0 && bpf_subprog_is_global(env, subprog)) { /* check that jump history doesn't have any * extra instructions from subprog; the next * instruction after call to global subprog @@ -536,7 +539,8 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, * never do that. */ from_subprog_call = subseq_idx - 1 >= 0 && - bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]); + (bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) || + bpf_is_callx(&env->prog->insnsi[subseq_idx - 1])); r0_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_0); r2_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_2); diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index 842c7d1eabcc..0de2f634ef67 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -421,6 +421,76 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env) return keep_exploring ? KEEP_EXPLORING : DONE_EXPLORING; } +/* + * Return pointers to functions in the read-only map that ld_imm64 instruction + * 't' loads the address of, or of its value, if there are any. + */ +static struct bpf_func_ptr *insn_func_ptrs(struct bpf_verifier_env *env, int t, u32 *cnt) +{ + struct bpf_insn *insn = &env->prog->insnsi[t]; + + *cnt = 0; + if (!env->func_ptr_cnt || !bpf_is_ldimm64(insn)) + return NULL; + if (insn->src_reg != BPF_PSEUDO_MAP_VALUE && + insn->src_reg != BPF_PSEUDO_MAP_IDX_VALUE && + insn->src_reg != BPF_PSEUDO_MAP_FD && + insn->src_reg != BPF_PSEUDO_MAP_IDX) + return NULL; + + return bpf_map_func_ptrs(env, env->used_maps[env->insn_aux_data[t].map_index], cnt); +} + +/* + * ld_imm64 that loads the address of a map that has pointers to functions + * is similar to ld_imm64 with BPF_PSEUDO_FUNC that loads the address of one + * function: any of them may be read from the map and called via callx later. + * Treat it as a call of all of them. + */ +static int visit_func_ptrs_insn(int t, struct bpf_verifier_env *env, + struct bpf_func_ptr *ptrs, u32 cnt) +{ + int *insn_stack = env->cfg.insn_stack; + int *insn_state = env->cfg.insn_state; + int ret, w; + u32 i; + + ret = push_insn(t, t + 2, FALLTHROUGH, env); + if (ret) + return ret; + + /* + * Explore one function at a time like push_insn() does, so that + * DISCOVERED function is a caller of 't' and not the one that waits in + * insn_stack[], which merge_callee_effects() relies on. + */ + for (i = 0; i < cnt; i++) { + w = ptrs[i].xlated_off; + if (insn_state[w]) + continue; + + if (env->cfg.cur_stack >= env->prog->len) + return -E2BIG; + + insn_stack[env->cfg.cur_stack++] = w; + insn_state[w] = DISCOVERED; + return KEEP_EXPLORING; + } + + /* all functions are explored, their effects are known */ + mark_prune_point(env, t); + for (i = 0; i < cnt; i++) { + w = ptrs[i].xlated_off; + merge_callee_effects(env, t, w); + + /* the same marks as push_insn() leaves on a branch target */ + mark_prune_point(env, w); + mark_jmp_point(env, w); + mark_jump_target(env, w); + } + return DONE_EXPLORING; +} + /* * Instructions that can abnormally return from a subprog (tail_call * upon success, ld_{abs,ind} upon load failure) have a hidden exit @@ -453,11 +523,17 @@ static int visit_abnormal_return_insn(struct bpf_verifier_env *env, int t) static int visit_insn(int t, struct bpf_verifier_env *env) { struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t]; + struct bpf_func_ptr *ptrs; int ret, off, insn_sz; + u32 cnt; if (bpf_pseudo_func(insn)) return visit_func_call_insn(t, insns, env, true); + ptrs = insn_func_ptrs(env, t, &cnt); + if (ptrs) + return visit_func_ptrs_insn(t, env, ptrs, cnt); + /* All non-branch instructions have a single fall-through edge. */ if (BPF_CLASS(insn->code) != BPF_JMP && BPF_CLASS(insn->code) != BPF_JMP32) { diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c index 7f1b30059cc8..b1528adbeb79 100644 --- a/kernel/bpf/const_fold.c +++ b/kernel/bpf/const_fold.c @@ -180,9 +180,12 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info * bool is_ldsx = mode == BPF_MEMSX; int off = src->val + insn->off; u64 val = 0; + u32 cnt; if (!bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr || off < 0 || off + size > map->value_size || + /* so are the addresses of functions that the map points to */ + bpf_map_range_func_ptrs(env, map, off, size, &cnt) || bpf_map_direct_read(map, off, size, &val, is_ldsx)) { *dst = unknown; break; @@ -191,7 +194,8 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info * dst->val = val; break; case BPF_JMP: - if (opcode != BPF_CALL) + /* both 'call imm' and 'callx reg' clobber caller saved registers */ + if (BPF_OP(insn->code) != BPF_CALL) break; process_call: for (r = BPF_REG_0; r <= BPF_REG_5; r++) diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 227211166dcc..273f74068068 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1831,6 +1831,7 @@ bool bpf_opcode_in_insntable(u8 code) [BPF_LD | BPF_IND | BPF_H] = true, [BPF_LD | BPF_IND | BPF_W] = true, [BPF_JMP | BPF_JA | BPF_X] = true, + [BPF_JMP | BPF_CALL | BPF_X] = true, [BPF_JMP | BPF_JCOND] = true, }; #undef BPF_INSN_3_TBL @@ -3028,6 +3029,11 @@ void __bpf_free_used_maps(struct bpf_prog_aux *aux, map->ops->map_poke_untrack(map, aux); if (sleepable) atomic64_dec(&map->sleepable_refcnt); + /* + * The program that didn't load is not a user of the map. libbpf + * loads the program again to get the log of the verifier. + */ + cmpxchg(&map->user, (unsigned long)aux, 0); bpf_map_put(map); } } @@ -3287,6 +3293,12 @@ bool __weak bpf_jit_supports_kfunc_call(void) return false; } +/* Return TRUE if the JIT backend supports callx (indirect call) instruction. */ +bool __weak bpf_jit_supports_callx(void) +{ + return false; +} + bool __weak bpf_jit_supports_kfunc_ret_reg_pair(void) { return false; diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index 3ce8d74b0e40..36d3228d7745 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -350,7 +350,10 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, if (opcode == BPF_CALL) { char tmp[64]; - if (insn->src_reg == BPF_PSEUDO_CALL) { + if (BPF_SRC(insn->code) == BPF_X) { + verbose(cbs->private_data, "(%02x) callx r%d", + insn->code, insn->dst_reg); + } else if (insn->src_reg == BPF_PSEUDO_CALL) { verbose(cbs->private_data, "(%02x) call pc%s", insn->code, __func_get_name(cbs, insn, diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 2add8001c3ec..053b24deb471 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -361,6 +361,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, adjust_insn_aux_data(env, new_prog, off, len, &original_insn); adjust_subprog_starts(env, off, len); adjust_insn_arrays(env, off, len); + bpf_adjust_func_ptrs(env, off, len); adjust_poke_descs(new_prog, off, len); return new_prog; } @@ -559,6 +560,9 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) if (err) return err; + /* before subprogs are adjusted, since it looks at them */ + bpf_adjust_func_ptrs_after_remove(env, off, cnt); + err = adjust_subprog_starts_after_remove(env, off, cnt); if (err) return err; @@ -619,12 +623,41 @@ void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) } } +/* + * The address of a function might be taken by the code that is not dead, + * while the function itself is dead code, since nothing calls it. Keep the first + * instruction of such function, so that the address remains valid and differs + * from the addresses of other functions. Make it a trap like + * sanitize_dead_code() does. + */ +static void keep_funcs_with_addr_taken(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux_data = env->insn_aux_data; + struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); + struct bpf_insn *insn = env->prog->insnsi; + int insn_cnt = env->prog->len; + int i, t; + + for (i = 0; i < insn_cnt; i++) { + if (!aux_data[i].seen || !bpf_pseudo_func(insn + i)) + continue; + t = i + insn[i].imm + 1; + if (aux_data[t].seen) + continue; + memcpy(insn + t, &trap, sizeof(trap)); + aux_data[t].zext_dst = false; + aux_data[t].seen = env->pass_cnt; + } +} + int bpf_opt_remove_dead_code(struct bpf_verifier_env *env) { struct bpf_insn_aux_data *aux_data = env->insn_aux_data; int insn_cnt = env->prog->len; int i, err; + keep_funcs_with_addr_taken(env); + for (i = 0; i < insn_cnt; i++) { int j; @@ -1095,6 +1128,59 @@ static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_s env->subprog_info[env->subprog_cnt].start = env->prog->len; } +/* + * Replace the offsets of functions with their addresses in the maps of + * the program, see resolve_func_ptrs() in verifier.c. The program must be + * the only user of such map. From now on no other program can use it, see + * bpf_map_claim(). + */ +static int resolve_func_ptrs(struct bpf_verifier_env *env, struct bpf_prog *prog, + struct bpf_prog **func) +{ + unsigned long me = (unsigned long)prog->aux; + struct bpf_func_ptr *ptr; + int i, err, subprog; + u64 addr, new, *slot; + + for (i = 0; i < env->func_ptr_cnt; i++) { + ptr = &env->func_ptrs[i]; + + /* pointers are sorted by map */ + if ((!i || ptr->map != ptr[-1].map) && + cmpxchg(&ptr->map->user, me, me | BPF_MAP_USER_PATCHED) != me) { + verbose(env, "map '%s' is used by another program\n", ptr->map->name); + return -EBUSY; + } + + /* it's the address of the value of the map whatever the offset is */ + err = ptr->map->ops->map_direct_value_addr(ptr->map, &addr, 0); + if (verifier_bug_if(err, env, "no value of map '%s'", ptr->map->name)) + return -EFAULT; + slot = (u64 *)(unsigned long)(addr + ptr->map_off); + + /* + * The function that is dead code is removed and the pointer is + * NULL. The program may read it, but callx of it is dead code + * too, see check_func_callx(). + */ + new = 0; + if (ptr->xlated_off != BPF_FUNC_PTR_DELETED) { + subprog = bpf_find_subprog(env, ptr->xlated_off); + if (verifier_bug_if(!func || subprog <= 0, env, "no function at insn %u", + ptr->xlated_off)) + return -EFAULT; + new = (unsigned long)func[subprog]->bpf_func; + } + + /* the map is frozen and nothing else uses it */ + if (verifier_bug_if(*slot != (u64)ptr->orig_off * sizeof(struct bpf_insn), env, + "map '%s' offset %u changed", ptr->map->name, ptr->map_off)) + return -EFAULT; + WRITE_ONCE(*slot, new); + } + return 0; +} + static int jit_subprogs(struct bpf_verifier_env *env) { struct bpf_prog *prog = env->prog, **func, *tmp; @@ -1285,6 +1371,11 @@ static int jit_subprogs(struct bpf_verifier_env *env) cond_resched(); } + /* the addresses of all functions are final */ + err = resolve_func_ptrs(env, prog, func); + if (err) + goto out_free; + /* * Cleanup func[i]->aux fields which aren't required * or can become invalid in future @@ -1371,8 +1462,9 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) struct bpf_prog *prog, *orig_prog; u32 *orig_subprog_starts; + /* all functions that the maps point to are removed as dead code */ if (env->subprog_cnt <= 1) - return 0; + return resolve_func_ptrs(env, env->prog, NULL); prog = orig_prog = env->prog; if (bpf_prog_need_blind(prog)) { diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 44ecdc5b4ec2..a204b492acc8 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -356,12 +356,29 @@ int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_ return 0; } +/* + * Stack slots of outer frames that the callee may read are accounted as read + * by the @callsite insn itself, i.e. they are live before @callsite and may be + * dead after it, when: + * - @callsite calls a callback, which may be called several times; + * - @callsite is callx. Its callee is not known when stack liveness is + * computed, so there is no func instance for it at @callsite. While + * the verifier is in the callee lookup_instance() finds its standalone + * instance that knows nothing about outer frames. What the callee may read + * there is recorded by record_call_access(). + */ +static bool callee_stack_access_at_callsite(struct bpf_verifier_env *env, u32 callsite) +{ + return bpf_calls_callback(env, callsite) || + bpf_is_callx(&env->prog->insnsi[callsite]); +} + bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 half_spi) { /* * Slot is alive if it is read before q->insn_idx in current func instance, * or if for some outer func instance: - * - alive before callsite if callsite calls callback, otherwise + * - alive before callsite if callsite calls callback or is callx, otherwise * - alive after callsite */ struct live_stack_query *q = &env->liveness->live_stack_query; @@ -394,7 +411,7 @@ bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 half_sp /* Get callsite from verifier state, not from instance callchain */ callsite = q->callsites[i]; - alive = bpf_calls_callback(env, callsite) + alive = callee_stack_access_at_callsite(env, callsite) ? is_live_before(instance, callsite, rel, half_spi) : is_live_before(instance, callsite + 1, rel, half_spi); if (alive) @@ -1439,7 +1456,15 @@ static int record_call_access(struct bpf_verifier_env *env, if (bpf_pseudo_call(insn)) return 0; - if (bpf_get_call_summary(env, insn, &cs)) + if (bpf_is_callx(insn)) + /* + * The callee is not known statically. Assume that all arg + * slots are passed and let record_arg_access() conservatively + * mark the stack of all frames as read if any of them is + * derived from a frame pointer. + */ + arg_slot_cnt = MAX_BPF_FUNC_REG_ARGS + MAX_STACK_ARG_SLOTS; + else if (bpf_get_call_summary(env, insn, &cs)) arg_slot_cnt = cs.arg_slot_cnt; for (r = BPF_REG_1; r < BPF_REG_1 + min(arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); r++) { @@ -1533,7 +1558,7 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, bool has_extra = false; u8 cls = BPF_CLASS(insns[idx].code); bool is_ldx_stx_call = cls == BPF_LDX || cls == BPF_STX || - insns[idx].code == (BPF_JMP | BPF_CALL); + (cls == BPF_JMP && BPF_OP(insns[idx].code) == BPF_CALL); verbose(env, "%3d: ", idx); bpf_verbose_insn(env, &insns[idx]); @@ -1722,7 +1747,7 @@ static int compute_subprog_args(struct bpf_verifier_env *env, if (err) goto err_free; - if (insn->code == (BPF_JMP | BPF_CALL)) { + if (BPF_CLASS(insn->code) == BPF_JMP && BPF_OP(insn->code) == BPF_CALL) { err = record_call_access(env, instance, at_in[i], idx); if (err) goto err_free; @@ -2202,6 +2227,9 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, use = GENMASK(min_t(u8, cs.arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS), 1); def = mask_widen(def); use = mask_widen(use); + /* callx reads the address of the callee from dst_reg */ + if (bpf_is_callx(insn)) + use |= dst; break; default: def = 0; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a7c9e2d8965d..fec5a1ae6a4d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3023,6 +3023,22 @@ static int add_subprogs(struct bpf_verifier_env *env) return ret; } + /* + * func_info describes all functions of the program. Those that are + * referenced only from data, e.g. from a table of functions to be + * called via callx, are not seen by the loop above. They are possible + * callees that have to be known upfront as well. + */ + if (env->bpf_capable) { + struct bpf_prog_aux *aux = env->prog->aux; + + for (i = 1; i < aux->func_info_cnt; i++) { + ret = add_subprog(env, aux->func_info[i].insn_off); + if (ret < 0) + return ret; + } + } + ret = bpf_find_exception_callback_insn_off(env); if (ret < 0) return ret; @@ -3104,6 +3120,8 @@ static int check_subprogs(struct bpf_verifier_env *env) if (BPF_CLASS(code) == BPF_LD && (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) subprog[cur_subprog].has_ld_abs = true; + if (bpf_is_callx(&insn[i])) + env->has_callx = true; if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) goto next; if (BPF_OP(code) == BPF_CALL) @@ -3149,12 +3167,50 @@ static int check_subprogs(struct bpf_verifier_env *env) return 0; } +/* + * The callee of callx is known to the main verification pass only, which + * records the 'caller' -> 'callee' edge of the call graph for the checks + * that follow it: absence of recursion and the maximum stack depth. + */ +static int record_callx_edge(struct bpf_verifier_env *env, int caller, int callee) +{ + u32 cnt = env->subprog_cnt; + + if (!env->callx_edges) { + env->callx_edges = kvcalloc(BITS_TO_LONGS(cnt * cnt), sizeof(long), + GFP_KERNEL_ACCOUNT); + if (!env->callx_edges) + return -ENOMEM; + } + __set_bit(caller * cnt + callee, env->callx_edges); + return 0; +} + +/* + * Return the first subprog with the number >= 'from' that 'caller' calls + * via callx, or -1 when there is none. + */ +static int next_callx_callee(struct bpf_verifier_env *env, int caller, int from) +{ + u32 cnt = env->subprog_cnt; + unsigned long bit, end = (caller + 1) * cnt; + + if (!env->callx_edges || from >= cnt) + return -1; + bit = find_next_bit(env->callx_edges, end, caller * cnt + from); + return bit < end ? bit - caller * cnt : -1; +} + /* * Sort subprogs in topological order so that leaf subprogs come first and * their callers come later. This is a DFS post-order traversal of the call * graph. Scan only reachable instructions (those in the computed postorder) of * the current subprog to discover callees (direct subprogs and sync * callbacks). + * + * The callees of callx are not known before the main verification pass. + * When callx is used the sort is repeated after it with the recorded callx + * edges added to the call graph to reject recursion through indirect calls. */ static int sort_subprogs_topo(struct bpf_verifier_env *env) { @@ -3195,12 +3251,22 @@ static int sort_subprogs_topo(struct bpf_verifier_env *env) int idx = insn_postorder[j]; int callee; - if (!bpf_pseudo_call(&insn[idx]) && !bpf_pseudo_func(&insn[idx])) + if (bpf_is_callx(&insn[idx])) { + /* find a callee that is not explored yet */ + callee = -1; + do { + callee = next_callx_callee(env, cur, callee + 1); + } while (callee >= 0 && color[callee] == 2); + if (callee < 0) + continue; + } else if (bpf_pseudo_call(&insn[idx]) || bpf_pseudo_func(&insn[idx])) { + callee = bpf_find_subprog(env, idx + insn[idx].imm + 1); + if (callee < 0) { + ret = -EFAULT; + goto out; + } + } else { continue; - callee = bpf_find_subprog(env, idx + insn[idx].imm + 1); - if (callee < 0) { - ret = -EFAULT; - goto out; } if (color[callee] == 2) continue; @@ -5372,6 +5438,9 @@ struct bpf_subprog_call_depth_info { int ret_insn; /* caller instruction where we return to. */ int caller; /* caller subprogram idx */ int frame; /* # of consecutive static call stack frames on top of stack */ + int callx_insn; /* callx instruction whose callees are being walked */ + int callx_next; /* next callee of callx_insn to walk */ + bool via_callx; /* the subprogram is entered via callx */ }; /* starting from main bpf function walk all instructions of the function @@ -5391,11 +5460,13 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, /* no caller idx */ dinfo[idx].caller = -1; + dinfo[idx].via_callx = false; i = subprog[idx].start; if (!priv_stack_supported) subprog[idx].priv_stack_mode = NO_PRIV_STACK; process_func: + dinfo[idx].callx_insn = -1; if (subprog[idx].has_ld_abs) { for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { if (subprog[tmp].is_cb) { @@ -5472,11 +5543,10 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, for (; i < subprog_end; i++) { int next_insn, sidx; - if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) { + if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off && + bpf_is_throw_kfunc(insn + i)) { bool err = false; - if (!bpf_is_throw_kfunc(insn + i)) - continue; for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) { if (subprog[tmp].is_cb) { err = true; @@ -5491,31 +5561,70 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, return -EINVAL; } - if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i)) - continue; - /* remember insn and function to return to */ - - /* find the callee */ - next_insn = i + insn[i].imm + 1; - sidx = bpf_find_subprog(env, next_insn); - if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn)) - return -EFAULT; - if (subprog[sidx].is_async_cb) { - /* async callbacks don't increase bpf prog stack size unless called directly */ - if (!bpf_pseudo_call(insn + i)) - continue; - if (subprog[sidx].is_exception_cb) { - verbose(env, "insn %d cannot call exception cb directly", i); - return -EINVAL; + if (bpf_is_callx(insn + i) || bpf_calls_callback(env, i)) { + /* + * Walk the callees and the callbacks recorded by + * the main verification pass one by one, returning to + * this insn after each. + */ + if (dinfo[idx].callx_insn != i) { + dinfo[idx].callx_insn = i; + dinfo[idx].callx_next = 0; } + sidx = next_callx_callee(env, idx, dinfo[idx].callx_next); + if (sidx < 0) + continue; + dinfo[idx].callx_next = sidx + 1; + dinfo[idx].ret_insn = i; + next_insn = subprog[sidx].start; + } else if (bpf_pseudo_call(insn + i) || bpf_pseudo_func(insn + i)) { + /* find the callee */ + next_insn = i + insn[i].imm + 1; + sidx = bpf_find_subprog(env, next_insn); + if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn)) + return -EFAULT; + if (subprog[sidx].is_async_cb) { + /* async callbacks don't increase bpf prog stack size unless called directly */ + if (!bpf_pseudo_call(insn + i)) + continue; + if (subprog[sidx].is_exception_cb) { + verbose(env, "insn %d cannot call exception cb directly", i); + return -EINVAL; + } + } + /* remember insn to return to */ + dinfo[idx].ret_insn = i + 1; + } else { + continue; + } + + /* + * sort_subprogs_topo() tolerates cycles in the call graph that + * go through the address of a function being taken, since it + * doesn't know what it is taken for. Such cycle is a recursion + * unless it's an async callback, which are skipped above. + * The main verification pass limits the depth of the recursion, + * but it doesn't follow calls of global functions. + */ + for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { + if (tmp != sidx) + continue; + verbose(env, "recursive call from %s() to %s()\n", + bpf_subprog_name(env, idx), bpf_subprog_name(env, sidx)); + bpf_diag_program_structure( + env, i, "recursive subprogram call", + "Rewrite the recursion as an explicit bounded loop, or split the logic so subprogram calls do not form a cycle.", + "This call or callback would make the subprogram call graph recursive. " + "The verifier requires a finite, acyclic call graph so it can bound stack depth and analysis."); + return -EINVAL; } /* store caller info for after we return from callee */ dinfo[idx].frame = frame; - dinfo[idx].ret_insn = i + 1; /* push caller idx into callee's dinfo */ dinfo[sidx].caller = idx; + dinfo[sidx].via_callx = bpf_is_callx(insn + i); i = next_insn; @@ -5549,6 +5658,14 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, verbose(env, "tail_calls are not allowed in programs with stack args\n"); return -EINVAL; } + /* + * JITs pass tail call counter in a register that is + * not available when the callee is called via callx. + */ + if (dinfo[tmp].via_callx) { + verbose(env, "tail_calls are not allowed in functions called via callx\n"); + return -EINVAL; + } subprog[tmp].tail_call_reachable = true; } } else if (!idx && subprog[0].has_tail_call && subprog[0].stack_arg_cnt) { @@ -5913,6 +6030,110 @@ int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val, return 0; } +static int cmp_func_ptrs(const void *_a, const void *_b) +{ + const struct bpf_func_ptr *a = _a, *b = _b; + + if (a->map != b->map) + return a->map < b->map ? -1 : 1; + if (a->map_off != b->map_off) + return a->map_off < b->map_off ? -1 : 1; + return 0; +} + +/* Find the first pointer to a function at or after 'off' in the value of 'map' */ +static u32 func_ptr_lower_bound(struct bpf_verifier_env *env, const struct bpf_map *map, u64 off) +{ + u32 l = 0, r = env->func_ptr_cnt, m; + struct bpf_func_ptr *p; + + while (l < r) { + m = l + (r - l) / 2; + p = &env->func_ptrs[m]; + if (p->map < map || (p->map == map && p->map_off < off)) + l = m + 1; + else + r = m; + } + return l; +} + +/* + * Return pointers to functions that overlap with 'size' bytes at offset 'off' + * of the value of 'map' and their number in 'cnt'. + */ +struct bpf_func_ptr *bpf_map_range_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, + u64 off, u64 size, u32 *cnt) +{ + u32 first, last; + + *cnt = 0; + if (!env->func_ptr_cnt || !size) + return NULL; + + /* a pointer that starts up to 7 bytes before 'off' overlaps too */ + first = func_ptr_lower_bound(env, map, off >= sizeof(u64) ? off - sizeof(u64) + 1 : 0); + last = func_ptr_lower_bound(env, map, off + size); + if (first >= last) + return NULL; + + *cnt = last - first; + return &env->func_ptrs[first]; +} + +/* Return all pointers to functions in the value of 'map' */ +struct bpf_func_ptr *bpf_map_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, u32 *cnt) +{ + return bpf_map_range_func_ptrs(env, map, 0, (u64)map->value_size, cnt); +} + +/* instructions [off, off + len) replaced the instruction at 'off' */ +void bpf_adjust_func_ptrs(struct bpf_verifier_env *env, u32 off, u32 len) +{ + struct bpf_func_ptr *p; + u32 i; + + if (len <= 1) + return; + + for (i = 0; i < env->func_ptr_cnt; i++) { + p = &env->func_ptrs[i]; + if (p->xlated_off <= off || p->xlated_off == BPF_FUNC_PTR_DELETED) + continue; + p->xlated_off += len - 1; + } +} + +/* + * Instructions [off, off + len) are about to be removed. It's called before + * the starts of subprogs are adjusted. A subprog is gone when all of its + * instructions are. Otherwise, e.g. when its first instruction is a nop, + * it starts where the removed instructions did. + */ +void bpf_adjust_func_ptrs_after_remove(struct bpf_verifier_env *env, u32 off, u32 len) +{ + struct bpf_func_ptr *p; + int subprog; + u32 i; + + for (i = 0; i < env->func_ptr_cnt; i++) { + p = &env->func_ptrs[i]; + if (p->xlated_off < off || p->xlated_off == BPF_FUNC_PTR_DELETED) + continue; + if (p->xlated_off >= off + len) { + p->xlated_off -= len; + continue; + } + subprog = bpf_find_subprog(env, p->xlated_off); + if (subprog > 0 && env->subprog_info[subprog + 1].start > off + len) + p->xlated_off = off; + else + p->xlated_off = BPF_FUNC_PTR_DELETED; + } +} + #define BTF_TYPE_SAFE_RCU(__type) __PASTE(__type, __safe_rcu) #define BTF_TYPE_SAFE_RCU_OR_NULL(__type) __PASTE(__type, __safe_rcu_or_null) #define BTF_TYPE_SAFE_TRUSTED(__type) __PASTE(__type, __safe_trusted) @@ -6422,12 +6643,147 @@ static void add_scalar_to_reg(struct bpf_reg_state *dst_reg, s64 val) reg_bounds_sync(dst_reg); } +/* there is no support for callx in the interpreter */ +static int require_callx_jit(struct bpf_verifier_env *env) +{ + if (!env->prog->jit_requested) { + verbose(env, "JIT is required to use callx\n"); + return -EOPNOTSUPP; + } + if (!bpf_jit_supports_callx()) { + verbose(env, "JIT doesn't support callx\n"); + return -EOPNOTSUPP; + } + env->prog->jit_required = true; + return 0; +} + +static void mark_reg_func_ptr(struct bpf_verifier_env *env, struct bpf_reg_state *regs, + int regno, int subprog) +{ + mark_reg_known_zero(env, regs, regno); + regs[regno].type = PTR_TO_FUNC; + regs[regno].subprogno = subprog; +} + +/* a read from a table of functions branches into that many states at most */ +#define BPF_MAX_FUNC_PTR_TARGETS 64 +/* and that many pointers at most are next to what is read at variable offset */ +#define BPF_MAX_FUNC_PTR_RANGE 4096 + +/* Count up to 'limit' offsets in [min_off, max_off] that the read is possible at */ +static u32 count_read_offsets(struct tnum offs, u64 min_off, u64 max_off, u32 limit) +{ + u64 o = min_off, next; + u32 cnt = 0; + + if (!tnum_in(offs, tnum_const(o))) { + next = tnum_step(offs, o); + if (next <= o) + return 0; + o = next; + } + while (o <= max_off && cnt < limit) { + cnt++; + next = tnum_step(offs, o); + if (next <= o) + break; + o = next; + } + return cnt; +} + +/* + * A read from a frozen read-only map that has pointers to functions, see + * resolve_func_ptrs(). The bounds and var_off of the offset tell where the read + * is possible at, e.g. var_off is the stride of the elements of an array. + * A read of exactly one pointer yields PTR_TO_FUNC. When the offset is variable + * and only pointers can be read, which is how an element of a table of + * functions is loaded, the verification continues with each of them. Other + * reads that overlap with a pointer are rejected, because their result is not + * known until the program is jitted. + * + * Return -ENOENT if there are no pointers to functions in the bytes that are read. + */ +static int check_func_ptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, + int size, int value_regno) +{ + u64 min_off = reg_umin(reg) + off, max_off = reg_umax(reg) + off; + struct tnum offs = tnum_add(reg->var_off, tnum_const(off)); + struct bpf_reg_state *regs = cur_regs(env); + struct bpf_map *map = reg->map_ptr; + struct bpf_verifier_state *branch; + int subprog, targets[BPF_MAX_FUNC_PTR_TARGETS]; + struct bpf_func_ptr *ptrs; + u32 i, cnt, n = 0; + u64 o, lo, hi; + int err; + + ptrs = bpf_map_range_func_ptrs(env, map, min_off, max_off - min_off + size, &cnt); + if (!ptrs) + return -ENOENT; + if (cnt > BPF_MAX_FUNC_PTR_RANGE) + goto overlap; + + for (i = 0; i < cnt; i++) { + /* the offsets at which the read overlaps with the pointer */ + lo = ptrs[i].map_off >= size ? ptrs[i].map_off - size + 1 : 0; + lo = max(lo, min_off); + hi = min(ptrs[i].map_off + sizeof(u64) - 1, max_off); + for (o = lo; o <= hi; o++) { + if (!tnum_in(offs, tnum_const(o))) + continue; + if (o != ptrs[i].map_off || size != sizeof(u64) || value_regno < 0) + goto overlap; + if (n == BPF_MAX_FUNC_PTR_TARGETS) { + verbose(env, "read from map '%s' may yield more than %d pointers to functions\n", + map->name, BPF_MAX_FUNC_PTR_TARGETS); + return -E2BIG; + } + subprog = bpf_find_subprog(env, ptrs[i].xlated_off); + if (verifier_bug_if(subprog <= 0, env, + "no function at insn %u for map '%s' offset %u", + ptrs[i].xlated_off, map->name, ptrs[i].map_off)) + return -EFAULT; + targets[n++] = subprog; + } + } + /* pointers are next to what is read, e.g. the data of an array of vtables */ + if (!n) + return -ENOENT; + /* either every offset that the read is possible at is a pointer or none */ + if (count_read_offsets(offs, min_off, max_off, n + 1) != n) + goto overlap; + + /* the map has the offset of the function until the program is jitted */ + err = require_callx_jit(env); + if (err) + return err; + + for (i = 0; i < n - 1; i++) { + branch = push_stack(env, env->insn_idx + 1, env->insn_idx, + env->cur_state->speculative); + if (IS_ERR(branch)) + return PTR_ERR(branch); + mark_reg_func_ptr(env, branch->frame[branch->curframe]->regs, value_regno, + targets[i]); + } + mark_reg_func_ptr(env, regs, value_regno, targets[n - 1]); + return 0; + +overlap: + verbose(env, "read of %d bytes at offset [%llu,%llu] of map '%s' overlaps with a pointer to a function\n", + size, min_off, max_off, map->name); + return -EACCES; +} + static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, int bpf_size, int value_regno, bool is_ldsx) { struct bpf_reg_state *regs = cur_regs(env); int size = bpf_size_to_bytes(bpf_size); struct bpf_map *map = reg->map_ptr; + int err; switch (map->map_type) { case BPF_MAP_TYPE_INSN_ARRAY: @@ -6445,13 +6801,18 @@ static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state break; } + if (env->func_ptr_cnt) { + err = check_func_ptr_read(env, reg, off, size, value_regno); + if (err != -ENOENT) + return err; + } + /* If map is read-only, track its contents as scalars. */ if (tnum_is_const(reg->var_off) && bpf_map_is_rdonly(map) && map->ops->map_direct_value_addr) { int map_off = off + reg->var_off.value; u64 val = 0; - int err; err = bpf_map_direct_read(map, map_off, size, &val, is_ldsx); if (err) @@ -8763,6 +9124,7 @@ static int check_arg_const_str(struct bpf_verifier_env *env, int map_off; u64 map_addr; char *str_ptr; + u32 cnt; if (reg->type != PTR_TO_MAP_VALUE) return -EINVAL; @@ -8812,6 +9174,11 @@ static int check_arg_const_str(struct bpf_verifier_env *env, verbose(env, "string is not zero-terminated\n"); return -EINVAL; } + /* the bytes of a pointer to a function are not known until the program is jitted */ + if (bpf_map_range_func_ptrs(env, map, map_off, strlen(str_ptr + map_off) + 1, &cnt)) { + verbose(env, "string overlaps with a pointer to a function\n"); + return -EACCES; + } return 0; } @@ -10579,6 +10946,15 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins return 0; } + /* + * The address of the callback might be taken in another function or + * read from a map, see check_func_ptr_read(). Like with callx it's + * known here what is called. + */ + err = record_callx_edge(env, caller->subprogno, subprog); + if (err) + return err; + /* for callback functions enqueue entry to callback and * proceed with next instruction within current frame. */ @@ -10600,13 +10976,61 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins static int process_bpf_exit_full(struct bpf_verifier_env *env, bool *do_print_state, bool exception_exit); -static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, - int *insn_idx) +/* + * Call of a static subprog. The callee is verified in the context of + * the caller, hence set up a new frame and continue from the first + * instruction of the callee. + */ +static int check_static_func_call(struct bpf_verifier_env *env, int subprog, + int *insn_idx) { struct bpf_verifier_state *state = env->cur_state; struct bpf_subprog_info *caller_info; u16 callee_incoming, stack_arg_cnt; struct bpf_func_state *caller; + int err; + + caller = state->frame[state->curframe]; + + /* + * Track caller's total stack arg count (incoming + max outgoing). + * This is needed so the JIT knows how much stack arg space to allocate. + */ + caller_info = &env->subprog_info[caller->subprogno]; + callee_incoming = bpf_in_stack_arg_cnt(&env->subprog_info[subprog]); + stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + callee_incoming; + if (stack_arg_cnt > caller_info->stack_arg_cnt) + caller_info->stack_arg_cnt = stack_arg_cnt; + + /* + * For regular function entry setup new frame and continue + * from that frame. + */ + err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state); + if (err) + return err; + + bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); + clear_caller_saved_regs(env, caller->regs); + + /* and go analyze first insn of the callee */ + *insn_idx = env->subprog_info[subprog].start - 1; + + if (env->log.level & BPF_LOG_LEVEL) { + verbose(env, "caller:\n"); + print_verifier_state(env, state, caller->frameno, true); + verbose(env, "callee:\n"); + print_verifier_state(env, state, state->curframe, true); + } + + return 0; +} + +static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, + int *insn_idx) +{ + struct bpf_verifier_state *state = env->cur_state; + struct bpf_func_state *caller; int err, subprog, target_insn; u32 i, nregs; @@ -10695,37 +11119,68 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; } - /* - * Track caller's total stack arg count (incoming + max outgoing). - * This is needed so the JIT knows how much stack arg space to allocate. - */ - caller_info = &env->subprog_info[caller->subprogno]; - callee_incoming = bpf_in_stack_arg_cnt(&env->subprog_info[subprog]); - stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + callee_incoming; - if (stack_arg_cnt > caller_info->stack_arg_cnt) - caller_info->stack_arg_cnt = stack_arg_cnt; + return check_static_func_call(env, subprog, insn_idx); +} - /* for regular function entry setup new frame and continue - * from that frame. - */ - err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state); +/* + * callx dst_reg: call a bpf subprog whose address is in dst_reg. + * + * The address of a subprog is either loaded into a register by ld_imm64 with + * src_reg == BPF_PSEUDO_FUNC, or it is read from a frozen read-only map, see + * resolve_func_ptrs(). Both are possible for static subprogs only. Hence all + * possible callees of callx are discovered by add_subprogs() and are reachable + * in the control flow graph before the main verification pass begins. + * PTR_TO_FUNC register identifies the callee, so from here on callx is verified + * as a direct call of that static subprog. + */ +static int check_func_callx(struct bpf_verifier_env *env, struct bpf_insn *insn, + int *insn_idx) +{ + struct bpf_func_state *caller = cur_func(env); + struct bpf_reg_state *reg; + const char *reason; + int err, subprog; + + err = require_callx_jit(env); if (err) return err; - bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); - clear_caller_saved_regs(env, caller->regs); + err = check_reg_arg(env, insn->dst_reg, SRC_OP); + if (err) + return err; - /* and go analyze first insn of the callee */ - *insn_idx = env->subprog_info[subprog].start - 1; - - if (env->log.level & BPF_LOG_LEVEL) { - verbose(env, "caller:\n"); - print_verifier_state(env, state, caller->frameno, true); - verbose(env, "callee:\n"); - print_verifier_state(env, state, state->curframe, true); + reg = reg_state(env, insn->dst_reg); + if (reg->type != PTR_TO_FUNC) { + verbose(env, "R%d has type %s, expected func\n", insn->dst_reg, + reg_type_str(env, reg->type)); + reason = bpf_diag_fmt( + env, "R%d holds %s, but callx can only call through the address of a static BPF function.", + insn->dst_reg, bpf_diag_reg_type_plain(env, reg->type)); + bpf_diag_register_type( + env, *insn_idx, insn->dst_reg, "indirect call through a non-function pointer", reason, + "Load the address of a static BPF function into the register before callx."); + return -EACCES; } - return 0; + /* + * Arithmetic on PTR_TO_FUNC is allowed, but only unmodified address + * of a subprog can be called. + */ + err = check_ptr_off_reg(env, reg, insn->dst_reg); + if (err) + return err; + + /* PTR_TO_FUNC is a pointer to a static subprog */ + subprog = reg->subprogno; + err = btf_check_subprog_call(env, subprog, caller->regs); + if (err == -EFAULT) + return err; + + err = record_callx_edge(env, caller->subprogno, subprog); + if (err) + return err; + + return check_static_func_call(env, subprog, insn_idx); } int map_set_for_each_callback_args(struct bpf_verifier_env *env, @@ -18790,11 +19245,13 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) env->jmps_processed++; if (opcode == BPF_CALL) { if (env->cur_state->active_locks) { - if ((insn->src_reg == BPF_REG_0 && - insn->imm != BPF_FUNC_spin_unlock && - insn->imm != BPF_FUNC_kptr_xchg) || - (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && - !kfunc_spin_allowed(env, insn->imm, insn->off))) { + /* similar to static subprog calls callx is allowed under a lock */ + if (!bpf_is_callx(insn) && + ((insn->src_reg == BPF_REG_0 && + insn->imm != BPF_FUNC_spin_unlock && + insn->imm != BPF_FUNC_kptr_xchg) || + (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && + !kfunc_spin_allowed(env, insn->imm, insn->off)))) { verbose(env, "function calls are not allowed while holding a lock\n"); bpf_diag_ctx_active( @@ -18807,6 +19264,8 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) mark_reg_scratched(env, BPF_REG_0); if (bpf_in_stack_arg_cnt(&env->subprog_info[cur_func(env)->subprogno])) cur_func(env)->no_stack_arg_load = true; + if (bpf_is_callx(insn)) + return check_func_callx(env, insn, &env->insn_idx); if (insn->src_reg == BPF_PSEUDO_CALL) return check_func_call(env, insn, &env->insn_idx); if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) @@ -19382,6 +19841,30 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, return 0; } +/* + * Keep track of whether the map is used by one program only. Such program may + * store the addresses of its functions into the map when it's frozen, see + * resolve_func_ptrs(), since nothing else relies on what the map has. After + * that the map is not available to other programs. + */ +static int bpf_map_claim(struct bpf_verifier_env *env, struct bpf_map *map) +{ + unsigned long me = (unsigned long)env->prog->aux, old; + + for (;;) { + old = READ_ONCE(map->user); + if (old == me || old == BPF_MAP_USER_MANY) + return 0; + if (old & BPF_MAP_USER_PATCHED) { + verbose(env, "map '%s' has addresses of functions of another program\n", + map->name); + return -EBUSY; + } + if (cmpxchg(&map->user, old, old ? BPF_MAP_USER_MANY : me) == old) + return 0; + } +} + static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) { int i, err; @@ -19419,6 +19902,10 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) env->used_maps[env->used_map_cnt++] = map; + err = bpf_map_claim(env, map); + if (err) + return err; + if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) { err = bpf_insn_array_init(map, env->prog); if (err) { @@ -19573,6 +20060,14 @@ static int check_jmp_fields(struct bpf_verifier_env *env, struct bpf_insn *insn) switch (opcode) { case BPF_CALL: + if (bpf_is_callx(insn)) { + /* callx dst_reg */ + if (insn->src_reg != BPF_REG_0 || insn->imm != 0 || insn->off != 0) { + verbose(env, "BPF_CALL|BPF_X uses reserved fields\n"); + return -EINVAL; + } + return 0; + } if (BPF_SRC(insn->code) != BPF_K || (insn->src_reg != BPF_PSEUDO_KFUNC_CALL && insn->off != 0) || (insn->src_reg != BPF_REG_0 && insn->src_reg != BPF_PSEUDO_CALL && @@ -19827,6 +20322,104 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env) return 0; } +/* that many pointers to functions are recognized in the maps of a program at most */ +#define BPF_MAX_FUNC_PTRS (1 << 20) + +static int add_func_ptr(struct bpf_verifier_env *env, struct bpf_map *map, u32 map_off, + u32 xlated_off) +{ + struct bpf_func_ptr *ptrs; + + if (env->func_ptr_cnt == BPF_MAX_FUNC_PTRS) { + verbose(env, "too many pointers to functions in maps, the limit is %d\n", + BPF_MAX_FUNC_PTRS); + return -E2BIG; + } + + /* grow by doubling, the array is sorted and searched later */ + if (!(env->func_ptr_cnt & (env->func_ptr_cnt - 1))) { + ptrs = kvrealloc(env->func_ptrs, + array_size(max(2 * env->func_ptr_cnt, 16U), sizeof(*ptrs)), + GFP_KERNEL_ACCOUNT); + if (!ptrs) + return -ENOMEM; + env->func_ptrs = ptrs; + } + env->func_ptrs[env->func_ptr_cnt++] = (struct bpf_func_ptr){ + .map = map, + .map_off = map_off, + .orig_off = xlated_off, + .xlated_off = xlated_off, + }; + return 0; +} + +/* + * Compilers put pointers to functions into read-only data: tables of functions, + * structures of operations, vtables, where they are mixed with other data. + * The loader stores such data in a frozen read-only array map and resolves + * a pointer to a static function to the offset in bytes of its first + * instruction in the program: the address of the function in the program. + * + * Find 64-bit values that look like that in the maps of a program that uses + * callx. It's a guess. When the value is not a pointer, the program either + * fails to load, because it does with a pointer what can be done with + * a number only, or it sees the address of a function instead of the number. + * It's known before the control flow graph of the program is built and the main + * verification pass begins which functions may be called via callx. + * + * When the program is jitted the offsets are replaced with the addresses of + * the functions in the map itself, see jit_subprogs(). Hence the program has to + * be the only user of the map, see bpf_map_claim(): nothing else may rely on + * what the map had. + * + * The program reads the addresses of its functions from there like any other + * data, so it has to be allowed to leak pointers. + */ +static int resolve_func_ptrs(struct bpf_verifier_env *env) +{ + int insn_cnt = env->prog->len; + int i, err, subprog; + struct bpf_map *map; + u64 addr, val; + u32 off; + + if (!env->has_callx || !env->allow_ptr_leaks) + return 0; + + for (i = 0; i < env->used_map_cnt; i++) { + map = env->used_maps[i]; + /* coincidences in maps that are shared with other programs don't matter */ + if (READ_ONCE(map->user) != (unsigned long)env->prog->aux) + continue; + if (map->map_type != BPF_MAP_TYPE_ARRAY || map->max_entries != 1 || + !bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr || + !IS_ERR_OR_NULL(map->record)) + continue; + if (map->ops->map_direct_value_addr(map, &addr, 0)) + continue; + + for (off = 0; off + sizeof(u64) <= map->value_size; off += sizeof(u64)) { + if (!(off % SZ_64K)) + cond_resched(); + val = *(u64 *)(unsigned long)(addr + off); + if (!val || val % sizeof(struct bpf_insn) || + val / sizeof(struct bpf_insn) >= insn_cnt) + continue; + subprog = bpf_find_subprog(env, val / sizeof(struct bpf_insn)); + if (subprog <= 0 || bpf_subprog_is_global(env, subprog)) + continue; + err = add_func_ptr(env, map, off, val / sizeof(struct bpf_insn)); + if (err) + return err; + } + } + if (env->func_ptr_cnt) + sort(env->func_ptrs, env->func_ptr_cnt, sizeof(*env->func_ptrs), + cmp_func_ptrs, NULL); + return 0; +} + /* drop refcnt of maps used by the rejected program */ static void release_maps(struct bpf_verifier_env *env) { @@ -21797,6 +22390,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + /* Find pointers to functions in the read-only maps of the program. */ + ret = resolve_func_ptrs(env); + if (ret < 0) + goto skip_full_check; + /* Build kfunc prototypes after resolving program resources. */ ret = add_kfuncs(env); if (ret < 0) @@ -21856,6 +22454,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, ret = do_check_main(env); ret = ret ?: do_check_subprogs(env); + /* reject recursion through the callx edges found by the main pass */ + if (ret == 0 && env->callx_edges) + ret = sort_subprogs_topo(env); + if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux)) ret = bpf_prog_offload_finalize(env); @@ -21998,6 +22600,8 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, kvfree(env->scc_info); kvfree(env->succ); kvfree(env->gotox_tmp_buf); + kvfree(env->callx_edges); + kvfree(env->func_ptrs); bpf_diag_free(env); kvfree(env); return ret; diff --git a/tools/lib/bpf/bpf_gen_internal.h b/tools/lib/bpf/bpf_gen_internal.h index 6c5ad6c55e8a..206adf28793d 100644 --- a/tools/lib/bpf/bpf_gen_internal.h +++ b/tools/lib/bpf/bpf_gen_internal.h @@ -51,9 +51,17 @@ struct bpf_gen { __u32 nr_ksyms; int fd_array; int nr_fd_array; + /* + * Maps with pointers to functions, that programs get their own copies + * of, take slots in fd_array after nr_obj_maps maps of the object. + */ + __u32 nr_obj_maps; + __u32 nr_func_ptr_maps; + __u32 max_func_ptr_maps; }; -void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps); +void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps, + int max_func_ptr_maps); int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps); void bpf_gen__free(struct bpf_gen *gen); void bpf_gen__load_btf(struct bpf_gen *gen, const void *raw_data, __u32 raw_size); @@ -68,6 +76,9 @@ void bpf_gen__prog_load(struct bpf_gen *gen, void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *value, __u32 value_size, __u64 flags); void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx); +int bpf_gen__func_ptr_map_create(struct bpf_gen *gen, const char *map_name, int obj_map_idx, + void *value, __u32 value_size, const __u32 *ptr_offs, + const __u64 *ptr_vals, int ptr_cnt); void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *name, enum bpf_attach_type type); void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, bool is_typeless, bool is_ld64, int kind, int insn_idx); diff --git a/tools/lib/bpf/gen_loader.c b/tools/lib/bpf/gen_loader.c index af3a04f161ac..251392aa8b41 100644 --- a/tools/lib/bpf/gen_loader.c +++ b/tools/lib/bpf/gen_loader.c @@ -112,13 +112,23 @@ static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn in static int add_data(struct bpf_gen *gen, const void *data, __u32 size); static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off); -void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps) +void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps, + int max_func_ptr_maps) { size_t stack_sz = sizeof(struct loader_stack), nr_progs_sz; int i; gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int)); gen->log_level = log_level; + gen->nr_obj_maps = nr_maps; + gen->max_func_ptr_maps = max_func_ptr_maps; + if (nr_maps + max_func_ptr_maps > MAX_USED_MAPS) { + pr_warn("Total maps exceeds %d\n", MAX_USED_MAPS); + gen->error = -E2BIG; + return; + } + /* their fds are closed like the fds of the maps of the object when loading fails */ + nr_maps += max_func_ptr_maps; /* save ctx pointer into R6 */ emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1)); @@ -385,6 +395,9 @@ int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) return gen->error; } emit_sys_close_stack(gen, stack_off(btf_fd)); + /* programs hold their maps with pointers to functions, nothing else needs them */ + for (i = 0; i < gen->nr_func_ptr_maps; i++) + emit_sys_close_blob(gen, blob_fd_array_off(gen, gen->nr_obj_maps + i)); for (i = 0; i < gen->nr_progs; i++) move_stack2ctx(gen, sizeof(struct bpf_loader_ctx) + @@ -1127,52 +1140,41 @@ void bpf_gen__prog_load(struct bpf_gen *gen, gen->nr_progs++; } -void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, - __u32 value_size, __u64 flags) +/* + * if (map_desc[map_idx].initial_value) { + * if (ctx->flags & BPF_SKEL_KERNEL) + * bpf_probe_read_kernel(value, value_size, initial_value); + * else + * bpf_copy_from_user(value, value_size, initial_value); + * nr_more_insns that the caller emits + * } + */ +static void emit_copy_initial_value(struct bpf_gen *gen, int map_idx, int value, + __u32 value_size, int nr_more_insns) { - int attr_size = offsetofend(union bpf_attr, flags); - int map_update_attr, value, key; - union bpf_attr attr; - int zero = 0; + emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, + sizeof(struct bpf_loader_ctx) + + sizeof(struct bpf_map_desc) * map_idx + + offsetof(struct bpf_map_desc, initial_value))); + emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8 + nr_more_insns)); + emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, + 0, 0, 0, value)); + emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); + emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, + offsetof(struct bpf_loader_ctx, flags))); + emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2)); + emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); + emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); + emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); +} - memset(&attr, 0, attr_size); - attr.flags = tgt_endian(flags); +/* Update the element of the map whose fd is in the slot map_idx of fd_array */ +static void emit_map_update_elem(struct bpf_gen *gen, int map_idx, union bpf_attr *attr, + int attr_size, int key, int value, __u32 value_size) +{ + int map_update_attr; - value = add_data(gen, pvalue, value_size); - key = add_data(gen, &zero, sizeof(zero)); - - /* - * if (map_desc[map_idx].initial_value) { - * if (ctx->flags & BPF_SKEL_KERNEL) - * bpf_probe_read_kernel(value, value_size, initial_value); - * else - * bpf_copy_from_user(value, value_size, initial_value); - * } - * - * The runtime initial_value comes from the host-supplied loader - * ctx and would overwrite the blob value that the program signature - * covers and the kernel verifies at load time. For a signed loader - * (gen_hash) the attested blob value must be authoritative, so skip - * the override and leave the signed value in place. - */ - if (!OPTS_GET(gen->opts, gen_hash, false)) { - emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, - sizeof(struct bpf_loader_ctx) + - sizeof(struct bpf_map_desc) * map_idx + - offsetof(struct bpf_map_desc, initial_value))); - emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8)); - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, value)); - emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); - emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, - offsetof(struct bpf_loader_ctx, flags))); - emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); - emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); - } - - map_update_attr = add_data(gen, &attr, attr_size); + map_update_attr = add_data(gen, attr, attr_size); pr_debug("gen: map_update_elem: idx %d, value: off %d size %u, attr: off %d size %d\n", map_idx, value, value_size, map_update_attr, attr_size); move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, @@ -1185,6 +1187,33 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, emit_check_err(gen); } +void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, + __u32 value_size, __u64 flags) +{ + int attr_size = offsetofend(union bpf_attr, flags); + union bpf_attr attr; + int value, key; + int zero = 0; + + memset(&attr, 0, attr_size); + attr.flags = tgt_endian(flags); + + value = add_data(gen, pvalue, value_size); + key = add_data(gen, &zero, sizeof(zero)); + + /* + * The runtime initial_value comes from the host-supplied loader + * ctx and would overwrite the blob value that the program signature + * covers and the kernel verifies at load time. For a signed loader + * (gen_hash) the attested blob value must be authoritative, so skip + * the override and leave the signed value in place. + */ + if (!OPTS_GET(gen->opts, gen_hash, false)) + emit_copy_initial_value(gen, map_idx, value, value_size, 0); + + emit_map_update_elem(gen, map_idx, &attr, attr_size, key, value, value_size); +} + void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slot, int inner_map_idx) { @@ -1214,6 +1243,77 @@ void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slo emit_check_err(gen); } +/* + * A copy of the read-only data map obj_map_idx for a program, with the offsets + * of its functions in it, see create_func_ptr_map() in libbpf.c. It's not + * a map of the object: it's not in the loader ctx. Its content is the content + * of obj_map_idx, that the host may supply when the skeleton is loaded, with + * ptr_cnt 64-bit ptr_vals at ptr_offs. + * Return the index of the map in fd_array for instructions to refer to. + */ +int bpf_gen__func_ptr_map_create(struct bpf_gen *gen, const char *map_name, int obj_map_idx, + void *pvalue, __u32 value_size, const __u32 *ptr_offs, + const __u64 *ptr_vals, int ptr_cnt) +{ + int attr_size = offsetofend(union bpf_attr, map_extra); + int map_create_attr, map_idx, key, value, zero = 0, i; + union bpf_attr attr; + + if (gen->nr_func_ptr_maps == gen->max_func_ptr_maps) { + gen->error = -EDOM; /* internal bug */ + return 0; + } + map_idx = gen->nr_obj_maps + gen->nr_func_ptr_maps++; + + memset(&attr, 0, attr_size); + attr.map_type = tgt_endian(BPF_MAP_TYPE_ARRAY); + attr.key_size = tgt_endian((__u32)sizeof(int)); + attr.value_size = tgt_endian(value_size); + attr.max_entries = tgt_endian((__u32)1); + attr.map_flags = tgt_endian((__u32)BPF_F_RDONLY_PROG); + if (map_name) + libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name)); + + map_create_attr = add_data(gen, &attr, attr_size); + pr_debug("gen: func_ptr_map_create: %s idx %d value_size %u, attr: off %d size %d\n", + map_name, map_idx, value_size, map_create_attr, attr_size); + emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size); + debug_ret(gen, "func_ptr_map_create %s idx %d value_size %d", map_name, map_idx, + value_size); + emit_check_err(gen); + /* remember map_fd in fd_array */ + emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, + 0, 0, 0, blob_fd_array_off(gen, map_idx))); + emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0)); + + /* pvalue has the pointers already */ + value = add_data(gen, pvalue, value_size); + key = add_data(gen, &zero, sizeof(zero)); + + /* see bpf_gen__map_update_elem() */ + if (!OPTS_GET(gen->opts, gen_hash, false)) { + /* the jump over these instructions has 16-bit offset */ + if (ptr_cnt > 10000) { + gen->error = -E2BIG; + return 0; + } + emit_copy_initial_value(gen, obj_map_idx, value, value_size, 3 * ptr_cnt); + /* the content that the host supplied doesn't have them */ + for (i = 0; i < ptr_cnt; i++) { + emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, + 0, 0, 0, value + ptr_offs[i])); + emit(gen, BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, ptr_vals[i])); + } + } + + attr_size = offsetofend(union bpf_attr, flags); + memset(&attr, 0, attr_size); + emit_map_update_elem(gen, map_idx, &attr, attr_size, key, value, value_size); + + bpf_gen__map_freeze(gen, map_idx); + return map_idx; +} + void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx) { int attr_size = offsetofend(union bpf_attr, map_fd); diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index cd1ea1bb53cb..39fc3c5e6bd2 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -497,6 +497,8 @@ struct bpf_program { bool autoload; bool autoattach; bool sym_global; + /* the program or a function that it calls has callx */ + bool has_callx; bool mark_btf_static; enum bpf_prog_type type; enum bpf_attach_type expected_attach_type; @@ -544,6 +546,7 @@ struct bpf_struct_ops { #define PERCPU_SEC ".percpu" #define BSS_SEC ".bss" #define RODATA_SEC ".rodata" +#define DATA_REL_RO_SEC ".data.rel.ro" #define KCONFIG_SEC ".kconfig" #define KSYMS_SEC ".ksyms" #define STRUCT_OPS_SEC ".struct_ops" @@ -601,6 +604,9 @@ struct bpf_map { bool autoattach; __u64 map_extra; struct bpf_program *excl_prog; + /* pointers to functions in the data of an internal map, see obj->func_ptrs */ + struct func_ptr *func_ptrs; + size_t func_ptr_cnt; }; enum extern_type { @@ -780,6 +786,29 @@ struct bpf_object { } *jumptable_maps; size_t jumptable_map_cnt; + /* + * Pointers to functions found in read-only data sections: tables of + * functions, structures of operations, vtables. Sorted by section + * and offset. + */ + struct func_ptr { + int sec_idx; /* ELF section that contains the pointer */ + size_t sec_off; /* offset of the pointer in the section */ + size_t text_off; /* offset of the function in .text section */ + } *func_ptrs; + size_t func_ptr_cnt; + + /* + * Read-only data with pointers to functions is different for every + * program that uses it, because so are the offsets of the functions. + */ + struct { + struct bpf_program *prog; + int map_idx; + int fd; + } *func_ptr_maps; + size_t func_ptr_map_cnt; + struct kern_feature_cache *feat_cache; char *token_path; int token_fd; @@ -845,6 +874,16 @@ static bool insn_is_pseudo_func(struct bpf_insn *insn) return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC; } +static bool prog_has_callx(const struct bpf_program *prog) +{ + size_t i; + + for (i = 0; i < prog->insns_cnt; i++) + if (prog->insns[i].code == (BPF_JMP | BPF_CALL | BPF_X)) + return true; + return false; +} + static int bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog, const char *name, size_t sec_idx, const char *sec_name, @@ -4024,6 +4063,17 @@ static int bpf_object__elf_collect(struct bpf_object *obj) err = bpf_object__add_programs(obj, data, name, idx); if (err) return err; + } else if (strcmp(name, DATA_REL_RO_SEC) == 0 || + str_has_pfx(name, DATA_REL_RO_SEC ".")) { + /* + * Constants with pointers in them, e.g. vtables, + * that position independent code keeps here to + * have them relocated. There is nothing that + * writes to it after that. + */ + sec_desc->sec_type = SEC_RODATA; + sec_desc->shdr = sh; + sec_desc->data = data; } else if (strcmp(name, DATA_SEC) == 0 || str_has_pfx(name, DATA_SEC ".")) { sec_desc->sec_type = SEC_DATA; @@ -4068,8 +4118,16 @@ static int bpf_object__elf_collect(struct bpf_object *obj) targ_sec_idx >= obj->efile.sec_cnt) return -LIBBPF_ERRNO__FORMAT; - /* Only do relo for section with exec instructions */ + /* + * Only do relo for section with exec instructions, + * struct_ops, maps, and read-only data that might + * have pointers to functions. + */ if (!section_have_execinstr(obj, targ_sec_idx) && + strcmp(name, ".rel" RODATA_SEC) && + !str_has_pfx(name, ".rel" RODATA_SEC ".") && + strcmp(name, ".rel" DATA_REL_RO_SEC) && + !str_has_pfx(name, ".rel" DATA_REL_RO_SEC ".") && strcmp(name, ".rel" STRUCT_OPS_SEC) && strcmp(name, ".rel" STRUCT_OPS_LINK_SEC) && strcmp(name, ".rel?" STRUCT_OPS_SEC) && @@ -6471,6 +6529,161 @@ static int create_jt_map(struct bpf_object *obj, struct bpf_program *prog, struc return err; } +/* + * The kernel recognizes a pointer to a function in a frozen read-only map by + * its value: the offset in bytes of the function in the program. It makes + * callx work for tables of functions, structures of operations and vtables, + * where pointers are mixed with other data. Functions have different offsets + * in different programs, so create a copy of the map for the program. + * The kernel replaces the offsets with the addresses of the functions when it + * loads the program, which has to be the only user of the map. + */ +static int create_func_ptr_map(struct bpf_object *obj, struct bpf_program *prog, int map_idx) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_RDONLY_PROG); + struct bpf_map *map = &obj->maps[map_idx]; + __u32 value_size = map->def.value_size; + size_t i, j, cnt, sec_insn_off; + struct func_ptr *ptrs; + int map_fd, err, zero = 0; + __u64 val; + void *data, *tmp; + + for (i = 0; i < obj->func_ptr_map_cnt; i++) + if (obj->func_ptr_maps[i].prog == prog && + obj->func_ptr_maps[i].map_idx == map_idx) + return obj->func_ptr_maps[i].fd; + + /* the map is not created yet, what it's going to have is in mmaped */ + if (!map->mmaped) + return -EINVAL; + + data = malloc(value_size); + if (!data) + return -ENOMEM; + memcpy(data, map->mmaped, value_size); + + ptrs = map->func_ptrs; + cnt = map->func_ptr_cnt; + for (i = 0; i < cnt; i++) { + if (ptrs[i].sec_off + sizeof(val) > value_size) { + err = -LIBBPF_ERRNO__FORMAT; + goto err_free; + } + /* + * Static functions were appended by bpf_object__append_func_ptrs_code(). + * A global function is in the program only if the code refers to it. + */ + sec_insn_off = ptrs[i].text_off / BPF_INSN_SZ; + for (j = 0; j < prog->subprog_cnt; j++) + if (prog->subprogs[j].sec_insn_off == sec_insn_off) + break; + if (j == prog->subprog_cnt) { + pr_debug("prog '%s': map '%s': no function for the pointer at offset %zu, it's NULL\n", + prog->name, map->name, ptrs[i].sec_off); + val = 0; + } else { + val = (__u64)prog->subprogs[j].sub_insn_off * BPF_INSN_SZ; + } + /* light skeleton can be generated for a target of another endianness */ + if (!is_native_endianness(obj)) + val = bswap_64(val); + memcpy(data + ptrs[i].sec_off, &val, sizeof(val)); + } + + /* + * The kernel takes any aligned 64-bit value that is equal to the offset + * of a function for a pointer. Tell when it's going to get it wrong. + */ + for (i = 0, j = 0; i + sizeof(val) <= value_size; i += sizeof(val)) { + __u32 k; + + while (j < cnt && ptrs[j].sec_off < i) + j++; + if (j < cnt && ptrs[j].sec_off == i) + continue; + memcpy(&val, data + i, sizeof(val)); + if (!is_native_endianness(obj)) + val = bswap_64(val); + if (!val || val % BPF_INSN_SZ) + continue; + for (k = 0; k < prog->subprog_cnt; k++) { + if ((__u64)prog->subprogs[k].sub_insn_off * BPF_INSN_SZ != val) + continue; + pr_warn("prog '%s': map '%s': value %llu at offset %zu is the offset of a function, the kernel will treat it as a pointer to it\n", + prog->name, map->name, (unsigned long long)val, i); + break; + } + } + + if (obj->gen_loader) { + __u32 *ptr_offs = calloc(cnt, sizeof(*ptr_offs)); + __u64 *ptr_vals = calloc(cnt, sizeof(*ptr_vals)); + + if (!ptr_offs || !ptr_vals) { + free(ptr_offs); + free(ptr_vals); + err = -ENOMEM; + goto err_free; + } + for (i = 0; i < cnt; i++) { + ptr_offs[i] = ptrs[i].sec_off; + memcpy(&ptr_vals[i], data + ptrs[i].sec_off, sizeof(val)); + if (!is_native_endianness(obj)) + ptr_vals[i] = bswap_64(ptr_vals[i]); + } + /* it's an index in fd_array of the loader, not an fd */ + map_fd = bpf_gen__func_ptr_map_create(obj->gen_loader, map->name, map_idx, data, + value_size, ptr_offs, ptr_vals, cnt); + free(ptr_offs); + free(ptr_vals); + goto done; + } + + opts.token_fd = obj->token_fd; + if (obj->token_fd) + opts.map_flags |= BPF_F_TOKEN_FD; + + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, map->name, sizeof(int), value_size, 1, &opts); + if (map_fd < 0) { + err = map_fd; + goto err_free; + } + + err = bpf_map_update_elem(map_fd, &zero, data, 0); + if (!err) + err = bpf_map_freeze(map_fd); + if (err) { + err = -errno; + goto err_close; + } +done: + + tmp = libbpf_reallocarray(obj->func_ptr_maps, obj->func_ptr_map_cnt + 1, + sizeof(*obj->func_ptr_maps)); + if (!tmp) { + err = -ENOMEM; + goto err_close; + } + obj->func_ptr_maps = tmp; + obj->func_ptr_maps[obj->func_ptr_map_cnt].prog = prog; + obj->func_ptr_maps[obj->func_ptr_map_cnt].map_idx = map_idx; + obj->func_ptr_maps[obj->func_ptr_map_cnt].fd = map_fd; + obj->func_ptr_map_cnt++; + + pr_debug("prog '%s': created a copy of map '%s' with %zu pointers to functions\n", + prog->name, map->name, cnt); + free(data); + return map_fd; + +err_close: + if (!obj->gen_loader) + close(map_fd); +err_free: + free(data); + return err; +} + /* Relocate data references within program code: * - map references; * - global variable references; @@ -6508,7 +6721,20 @@ bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) if (relo->map_idx == obj->arena_map_idx) insn[1].imm += obj->arena_data_off; - if (obj->gen_loader) { + if (map->autocreate && map->func_ptr_cnt && prog->has_callx) { + int map_fd; + + /* the program gets its own map with pointers to its functions */ + map_fd = create_func_ptr_map(obj, prog, relo->map_idx); + if (map_fd < 0) { + pr_warn("prog '%s': relo #%d: can't create a copy of map '%s' with pointers to functions\n", + prog->name, i, map->name); + return map_fd; + } + insn[0].src_reg = obj->gen_loader ? BPF_PSEUDO_MAP_IDX_VALUE : + BPF_PSEUDO_MAP_VALUE; + insn[0].imm = map_fd; + } else if (obj->gen_loader) { insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; insn[0].imm = relo->map_idx; } else if (map->autocreate) { @@ -6944,6 +7170,75 @@ bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, return 0; } +/* Append to the main program all functions that the data of the map points to */ +static int +bpf_object__append_func_ptrs_code(struct bpf_object *obj, struct bpf_program *main_prog, + const struct bpf_map *map) +{ + struct bpf_program *subprog; + size_t i, cnt, sec_insn_off; + struct func_ptr *ptrs; + int err; + + ptrs = map->func_ptrs; + cnt = map->func_ptr_cnt; + for (i = 0; i < cnt; i++) { + sec_insn_off = ptrs[i].text_off / BPF_INSN_SZ; + subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sec_insn_off); + if (!subprog || subprog->sec_insn_off != sec_insn_off) { + pr_warn("prog '%s': map '%s': no function at .text+%zu for the pointer at offset %zu\n", + main_prog->name, map->name, ptrs[i].text_off, ptrs[i].sec_off); + return -LIBBPF_ERRNO__RELOC; + } + + /* + * callx can't call global functions. Don't add one to the + * program only because the data points to it. + */ + if (subprog->sym_global) + continue; + + /* see the comment in bpf_object__reloc_code() */ + if (subprog->sub_insn_off == 0) { + err = bpf_object__append_subprog_code(obj, main_prog, subprog); + if (err) + return err; + err = bpf_object__reloc_code(obj, main_prog, subprog); + if (err) + return err; + } + } + return 0; +} + +/* + * The program that has callx might call any function that the data it refers + * to points to. Append them. Programs that don't have callx can't call them + * and the kernel doesn't look for pointers to functions in their data. + */ +static int +bpf_object__append_func_ptrs(struct bpf_object *obj, struct bpf_program *prog) +{ + size_t i; + int err; + + prog->has_callx = prog_has_callx(prog); + if (!prog->has_callx) + return 0; + + /* relocations of the functions that are appended are appended too */ + for (i = 0; i < prog->nr_reloc; i++) { + struct reloc_desc *relo = &prog->reloc_desc[i]; + + if (relo->type != RELO_DATA || !obj->maps[relo->map_idx].func_ptr_cnt) + continue; + err = bpf_object__append_func_ptrs_code(obj, prog, &obj->maps[relo->map_idx]); + if (err) + return err; + } + return 0; +} + /* * Relocate sub-program calls. * @@ -7046,7 +7341,7 @@ bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog) if (err) return err; - return 0; + return bpf_object__append_func_ptrs(obj, prog); } static void @@ -7757,6 +8052,98 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj, return 0; } +/* + * Collect pointers to functions in a read-only data section. They are + * R_BPF_64_ABS64 relocations against .text section, where the offset of + * a static function in the section is stored in place. Relocations in data + * sections were ignored before pointers to functions were supported. Those + * that are something else, e.g. pointers to data, still are. + */ +static int bpf_object__collect_rodata_relos(struct bpf_object *obj, + Elf64_Shdr *shdr, Elf_Data *data) +{ + size_t sec_idx = shdr->sh_info, sym_idx; + int i, nrels = shdr->sh_size / shdr->sh_entsize; + const char *relo_sec_name; + struct func_ptr *ptrs; + Elf_Data *scn_data; + Elf64_Sym *sym; + Elf64_Rel *rel; + __u64 addend; + + relo_sec_name = elf_sec_str(obj, shdr->sh_name) ?: ""; + scn_data = obj->efile.secs[sec_idx].data; + if (!scn_data) + return -LIBBPF_ERRNO__FORMAT; + + for (i = 0; i < nrels; i++) { + rel = elf_rel_by_idx(data, i); + if (!rel) { + pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i); + return -LIBBPF_ERRNO__FORMAT; + } + + sym_idx = ELF64_R_SYM(rel->r_info); + sym = elf_sym_by_idx(obj, sym_idx); + if (!sym) { + pr_warn("sec '%s': symbol #%zu not found for relo #%d\n", + relo_sec_name, sym_idx, i); + return -LIBBPF_ERRNO__FORMAT; + } + + if (ELF64_R_TYPE(rel->r_info) != R_BPF_64_ABS64 || + !sym_is_subprog(sym, obj->efile.text_shndx)) { + pr_debug("sec '%s': relo #%d: not a pointer to a function, skipping...\n", + relo_sec_name, i); + continue; + } + + /* the kernel finds aligned pointers only */ + if (rel->r_offset % sizeof(__u64) || rel->r_offset >= scn_data->d_size || + scn_data->d_size - rel->r_offset < sizeof(__u64)) { + pr_debug("sec '%s': relo #%d: unsupported offset 0x%zx, skipping...\n", + relo_sec_name, i, (size_t)rel->r_offset); + continue; + } + + memcpy(&addend, scn_data->d_buf + rel->r_offset, sizeof(addend)); + if (!is_native_endianness(obj)) + addend = bswap_64(addend); + if ((sym->st_value + addend) % BPF_INSN_SZ) { + pr_debug("sec '%s': relo #%d: bad pointer to a function at offset %zu+%llu, skipping...\n", + relo_sec_name, i, (size_t)sym->st_value, + (unsigned long long)addend); + continue; + } + + ptrs = libbpf_reallocarray(obj->func_ptrs, obj->func_ptr_cnt + 1, sizeof(*ptrs)); + if (!ptrs) + return -ENOMEM; + obj->func_ptrs = ptrs; + + ptrs[obj->func_ptr_cnt].sec_idx = sec_idx; + ptrs[obj->func_ptr_cnt].sec_off = rel->r_offset; + ptrs[obj->func_ptr_cnt].text_off = sym->st_value + addend; + obj->func_ptr_cnt++; + + pr_debug("sec '%s': relo #%d: pointer at offset %zu to a function at .text+%zu\n", + relo_sec_name, i, (size_t)rel->r_offset, (size_t)(sym->st_value + addend)); + } + return 0; +} + +static int cmp_func_ptrs(const void *_a, const void *_b) +{ + const struct func_ptr *a = _a; + const struct func_ptr *b = _b; + + if (a->sec_idx != b->sec_idx) + return a->sec_idx < b->sec_idx ? -1 : 1; + if (a->sec_off != b->sec_off) + return a->sec_off < b->sec_off ? -1 : 1; + return 0; +} + static int bpf_object__collect_relos(struct bpf_object *obj) { int i, err; @@ -7779,7 +8166,9 @@ static int bpf_object__collect_relos(struct bpf_object *obj) return -LIBBPF_ERRNO__INTERNAL; } - if (obj->efile.secs[idx].sec_type == SEC_ST_OPS) + if (obj->efile.secs[idx].sec_type == SEC_RODATA) + err = bpf_object__collect_rodata_relos(obj, shdr, data); + else if (obj->efile.secs[idx].sec_type == SEC_ST_OPS) err = bpf_object__collect_st_ops_relos(obj, shdr, data); else if (idx == obj->efile.btf_maps_shndx) err = bpf_object__collect_map_relos(obj, shdr, data); @@ -7789,6 +8178,25 @@ static int bpf_object__collect_relos(struct bpf_object *obj) return err; } + /* sort by section, so that pointers in the data of a map are next to each other */ + if (obj->func_ptr_cnt) + qsort(obj->func_ptrs, obj->func_ptr_cnt, sizeof(*obj->func_ptrs), cmp_func_ptrs); + + for (i = 0; i < obj->nr_maps; i++) { + struct bpf_map *map = &obj->maps[i]; + size_t j; + + if (map->libbpf_type != LIBBPF_MAP_RODATA) + continue; + for (j = 0; j < obj->func_ptr_cnt; j++) { + if (obj->func_ptrs[j].sec_idx != map->sec_idx) + continue; + if (!map->func_ptr_cnt) + map->func_ptrs = &obj->func_ptrs[j]; + map->func_ptr_cnt++; + } + } + bpf_object__sort_relos(obj); return 0; } @@ -9208,8 +9616,27 @@ static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const ch * permit cross-endian creation of "light skeleton". */ if (obj->gen_loader) { + int nr_func_ptr_maps = 0, nr_progs = 0, i; + bool text_has_callx = false; + + /* + * Every program that has callx may get a copy of every map with + * pointers to functions. Which functions a program calls is not + * known yet. Any of them may call the ones that have callx. + */ + for (i = 0; i < obj->nr_maps; i++) + if (obj->maps[i].autocreate && obj->maps[i].func_ptr_cnt) + nr_func_ptr_maps++; + for (i = 0; i < obj->nr_programs; i++) + if (prog_is_subprog(obj, &obj->programs[i]) && + prog_has_callx(&obj->programs[i])) + text_has_callx = true; + for (i = 0; i < obj->nr_programs; i++) + if (obj->programs[i].autoload && !prog_is_subprog(obj, &obj->programs[i]) && + (text_has_callx || prog_has_callx(&obj->programs[i]))) + nr_progs++; bpf_gen__init(obj->gen_loader, obj->log_level | extra_log_level, - obj->nr_programs, obj->nr_maps); + obj->nr_programs, obj->nr_maps, nr_func_ptr_maps * nr_progs); } else if (!is_native_endianness(obj)) { pr_warn("object '%s': loading non-native endianness is unsupported\n", obj->name); return libbpf_err(-LIBBPF_ERRNO__ENDIAN); @@ -9749,6 +10176,12 @@ void bpf_object__close(struct bpf_object *obj) close(obj->jumptable_maps[i].fd); zfree(&obj->jumptable_maps); + for (i = 0; i < obj->func_ptr_map_cnt; i++) + if (!obj->gen_loader) + close(obj->func_ptr_maps[i].fd); + zfree(&obj->func_ptr_maps); + zfree(&obj->func_ptrs); + if (obj->btf_module_allowlist) { for (i = 0; i < obj->btf_module_allowlist_cnt; i++) zfree(&obj->btf_module_allowlist[i]); diff --git a/tools/lib/bpf/linker.c b/tools/lib/bpf/linker.c index 78f92c39290a..53f64a1a1f25 100644 --- a/tools/lib/bpf/linker.c +++ b/tools/lib/bpf/linker.c @@ -2274,6 +2274,24 @@ static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *ob insn->imm += sec->dst_off / sizeof(struct bpf_insn); else insn->imm += sec->dst_off; + } else if (sym_type == R_BPF_64_ABS64 && + (sec->shdr->sh_flags & SHF_EXECINSTR)) { + /* + * A pointer to a static function in a data section, + * which is stored in place as an offset of the + * function in its section. Data sections are kept + * in the byte order of the object. + */ + void *ptr = dst_linked_sec->raw_data + dst_rel->r_offset; + __u64 off; + + memcpy(&off, ptr, sizeof(off)); + if (linker->swapped_endian) + off = bswap_64(off); + off += sec->dst_off; + if (linker->swapped_endian) + off = bswap_64(off); + memcpy(ptr, &off, sizeof(off)); } else { pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n"); return -EINVAL; diff --git a/tools/testing/selftests/bpf/Makefile.skel b/tools/testing/selftests/bpf/Makefile.skel index 580d1d82c186..3d92cdca62ed 100644 --- a/tools/testing/selftests/bpf/Makefile.skel +++ b/tools/testing/selftests/bpf/Makefile.skel @@ -33,7 +33,7 @@ LINKED_SKELS := test_static_linked.skel.h linked_funcs.skel.h \ LSKELS := fexit_sleep.c trace_printk.c trace_vprintk.c map_ptr_kern.c \ core_kern.c core_kern_overflow.c test_ringbuf.c \ test_ringbuf_n.c test_ringbuf_map_key.c test_ringbuf_write.c \ - test_ringbuf_overwrite.c + test_ringbuf_overwrite.c callx_rodata.c LSKELS_SIGNED := fentry_test.c fexit_test.c atomics.c diff --git a/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c b/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c new file mode 100644 index 000000000000..b58ebee9b1ef --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * The kernel replaces the offsets of functions in a frozen read-only map with + * their addresses when the program is loaded. The program has to be the only + * user of the map, and no other program can use the map after that. + */ +#include +#include +#include + +#if defined(__x86_64__) || defined(__aarch64__) + +#define CALLEE_INSN 6 +#define DATA 0x1234 + +/* + * main: r2 = &value; r2 = *(u64 *)(r2 + 8); r1 = 10; callx r2; exit + * add1: r0 = r1; r0 += 1; exit + * + * where value is { DATA, offset of add1 in the program }. + */ +static const struct bpf_insn callx_insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_2, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_2, 8), + BPF_MOV64_IMM(BPF_REG_1, 10), + BPF_RAW_INSN(BPF_JMP | BPF_CALL | BPF_X, BPF_REG_2, 0, 0, 0), + BPF_EXIT_INSN(), + BPF_MOV64_REG(BPF_REG_0, BPF_REG_1), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1), + BPF_EXIT_INSN(), +}; + +/* reads the data of the map */ +static const struct bpf_insn reader_insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_2, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0), + BPF_EXIT_INSN(), +}; + +/* refers to the map and is rejected: r0 is not set */ +static const struct bpf_insn bad_insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_2, 0, 0), + BPF_EXIT_INSN(), +}; + +static char log_buf[16 * 1024]; +static struct bpf_func_info func_info[2]; +static int btf_fd; + +static int create_map(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_RDONLY_PROG); + __u64 value[2] = { DATA, CALLEE_INSN * sizeof(struct bpf_insn) }; + int fd, zero = 0; + + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "callx_rodata", sizeof(int), sizeof(value), 1, + &opts); + if (!ASSERT_OK_FD(fd, "map_create")) + return -1; + if (!ASSERT_OK(bpf_map_update_elem(fd, &zero, value, 0), "map_update") || + !ASSERT_OK(bpf_map_freeze(fd), "map_freeze")) { + close(fd); + return -1; + } + return fd; +} + +static int load(const struct bpf_insn *prog_insns, int cnt, int map_fd) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts, + .log_buf = log_buf, + .log_size = sizeof(log_buf), + .log_level = 1, + ); + struct bpf_insn insns[ARRAY_SIZE(callx_insns)]; + + if (prog_insns == callx_insns) { + /* add1() is referred to by the data only and is found through func_info */ + opts.prog_btf_fd = btf_fd; + opts.func_info = func_info; + opts.func_info_cnt = 2; + opts.func_info_rec_size = sizeof(func_info[0]); + } + memcpy(insns, prog_insns, cnt * sizeof(insns[0])); + insns[0].imm = map_fd; + log_buf[0] = 0; + return bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "callx_map", "GPL", insns, cnt, &opts); +} + +#define LOAD(insns, map_fd) load(insns, ARRAY_SIZE(insns), map_fd) + +static void run(int prog_fd, int expected, const char *name) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + char pkt[64] = {}; + + topts.data_in = pkt; + topts.data_size_in = sizeof(pkt); + if (ASSERT_OK(bpf_prog_test_run_opts(prog_fd, &topts), name)) + ASSERT_EQ(topts.retval, expected, name); +} + +void test_callx_func_ptr_map(void) +{ + int int_id, proto_id, map_fd = -1, prog_fd = -1, reader_fd = -1, fd, zero = 0; + int main_id, add1_id; + struct btf *btf; + __u64 value[2]; + + /* there is no support for callx in the interpreter */ + if (!is_jit_enabled()) { + test__skip(); + return; + } + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf_new")) + return; + int_id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + proto_id = btf__add_func_proto(btf, int_id); + main_id = btf__add_func(btf, "main_prog", BTF_FUNC_GLOBAL, proto_id); + add1_id = btf__add_func(btf, "add1", BTF_FUNC_STATIC, proto_id); + if (!ASSERT_GT(main_id, 0, "btf_add_main") || + !ASSERT_GT(add1_id, 0, "btf_add_add1") || + !ASSERT_OK(btf__load_into_kernel(btf), "btf_load")) + goto out; + func_info[0].insn_off = 0; + func_info[0].type_id = main_id; + func_info[1].insn_off = CALLEE_INSN; + func_info[1].type_id = add1_id; + btf_fd = btf__fd(btf); + + /* + * Another program relies on what the map has: it's verified with + * the data folded into constants. Pointers to functions are not looked + * for in such map. + */ + map_fd = create_map(); + if (map_fd < 0) + goto out; + reader_fd = LOAD(reader_insns, map_fd); + if (!ASSERT_OK_FD(reader_fd, "load_reader")) + goto out; + fd = LOAD(callx_insns, map_fd); + if (!ASSERT_LT(fd, 0, "load_shared")) + close(fd); + ASSERT_HAS_SUBSTR(log_buf, "unreachable insn 6", "log_shared"); + run(reader_fd, DATA, "run_reader"); + close(reader_fd); + reader_fd = -1; + close(map_fd); + + map_fd = create_map(); + if (map_fd < 0) + goto out; + + /* a program that is rejected is not a user, libbpf loads it again to get the log */ + fd = LOAD(bad_insns, map_fd); + if (!ASSERT_LT(fd, 0, "load_bad")) + close(fd); + + prog_fd = LOAD(callx_insns, map_fd); + if (!ASSERT_OK_FD(prog_fd, "load_callx")) { + printf("%s\n", log_buf); + goto out; + } + run(prog_fd, 11, "run_callx"); + + /* the offset of the function is gone from the map, the data is intact */ + if (ASSERT_OK(bpf_map_lookup_elem(map_fd, &zero, value), "map_lookup")) { + ASSERT_EQ(value[0], DATA, "data"); + ASSERT_NEQ(value[1], CALLEE_INSN * sizeof(struct bpf_insn), "pointer"); + } + + /* no other program can use the map now, another instance of the same one too */ + fd = LOAD(reader_insns, map_fd); + if (!ASSERT_EQ(fd, -EBUSY, "load_reader_after")) + close(fd); + ASSERT_HAS_SUBSTR(log_buf, "has addresses of functions of another program", "log_reader"); + fd = LOAD(callx_insns, map_fd); + if (!ASSERT_EQ(fd, -EBUSY, "load_second_instance")) + close(fd); + + run(prog_fd, 11, "run_callx_again"); +out: + if (reader_fd >= 0) + close(reader_fd); + if (prog_fd >= 0) + close(prog_fd); + if (map_fd >= 0) + close(map_fd); + btf__free(btf); +} + +#else + +void test_callx_func_ptr_map(void) +{ + test__skip(); +} + +#endif diff --git a/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c b/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c new file mode 100644 index 000000000000..96f6179f0492 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include "callx_rodata.lskel.h" + +#if defined(__x86_64__) || defined(__aarch64__) + +static void run(int prog_fd, int expected, const char *name) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + char pkt[64] = {}; + + topts.data_in = pkt; + topts.data_size_in = sizeof(pkt); + if (ASSERT_OK(bpf_prog_test_run_opts(prog_fd, &topts), name)) + ASSERT_EQ(topts.retval, expected, name); +} + +/* + * Every program that has callx gets its own copy of .rodata with the offsets + * of its functions. The copies have what user space puts into .rodata before + * the load. The program that doesn't have callx uses .rodata. + */ +void test_callx_rodata_lskel(void) +{ + struct callx_rodata_lskel *skel; + + /* there is no support for callx in the interpreter */ + if (!is_jit_enabled()) { + test__skip(); + return; + } + + skel = callx_rodata_lskel__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + + skel->rodata->bias = 7; + + if (!ASSERT_OK(callx_rodata_lskel__load(skel), "load")) + goto out; + + skel->bss->op_idx = 0; + run(skel->progs.select_op.prog_fd, 17, "add_bias"); + skel->bss->op_idx = 1; + run(skel->progs.select_op.prog_fd, 30, "mul3"); + skel->bss->op_idx = 2; + run(skel->progs.select_op.prog_fd, -1, "out_of_range"); + /* mul3(add_bias(4)) */ + run(skel->progs.both_ops.prog_fd, 33, "both_ops"); + run(skel->progs.read_bias.prog_fd, 7, "read_bias"); +out: + callx_rodata_lskel__destroy(skel); +} + +#else + +void test_callx_rodata_lskel(void) +{ + test__skip(); +} + +#endif diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 8b4be06bb181..9c3ff7d83fbe 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -28,6 +28,8 @@ #include "verifier_btf_ctx_access.skel.h" #include "verifier_btf_unreliable_prog.skel.h" #include "verifier_call_large_imm.skel.h" +#include "verifier_callx.skel.h" +#include "verifier_callx_rodata.skel.h" #include "verifier_cfg.skel.h" #include "verifier_cgroup_inv_retcode.skel.h" #include "verifier_cgroup_skb.skel.h" @@ -173,6 +175,14 @@ static void run_tests_aux(const char *skel_name, #define RUN(skel) run_tests_aux(#skel, skel##__elf_bytes, NULL) +/* for tests of what the interpreter doesn't support */ +#define RUN_JITED(skel) do { \ + if (is_jit_enabled()) \ + RUN(skel); \ + else \ + test__skip(); \ +} while (0) + void test_arena_kfunc(void) { RUN_TESTS(arena_kfunc); } void test_arena_kfunc_jit(void) { RUN_TESTS(arena_kfunc_jit); } @@ -199,6 +209,8 @@ void test_verifier_bswap(void) { RUN(verifier_bswap); } void test_verifier_btf_ctx_access(void) { RUN(verifier_btf_ctx_access); } void test_verifier_btf_unreliable_prog(void) { RUN(verifier_btf_unreliable_prog); } void test_verifier_call_large_imm(void) { RUN(verifier_call_large_imm); } +void test_verifier_callx(void) { RUN_JITED(verifier_callx); } +void test_verifier_callx_rodata(void) { RUN_JITED(verifier_callx_rodata); } void test_verifier_cfg(void) { RUN(verifier_cfg); } void test_verifier_cgroup_inv_retcode(void) { RUN(verifier_cgroup_inv_retcode); } void test_verifier_cgroup_skb(void) { RUN(verifier_cgroup_skb); } diff --git a/tools/testing/selftests/bpf/progs/callx_rodata.c b/tools/testing/selftests/bpf/progs/callx_rodata.c new file mode 100644 index 000000000000..07ae2d1b6b8e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/callx_rodata.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0 +/* callx through pointers to functions in .rodata, loaded by light skeleton */ + +#include +#include + +typedef int (*op_fn)(int); + +/* set by user space before the programs are loaded, it's in .rodata too */ +const volatile int bias = 1; + +int op_idx; + +static __noinline int add_bias(int x) +{ + return x + bias; +} + +static __noinline int mul3(int x) +{ + return x * 3; +} + +/* volatile, so that the compiler doesn't replace the table with direct calls */ +static op_fn const volatile ops[] = { add_bias, mul3 }; + +SEC("socket") +int select_op(void *ctx) +{ + unsigned int i = op_idx; + + if (i >= sizeof(ops) / sizeof(ops[0])) + return -1; + return ops[i](10); +} + +/* functions have other offsets in this program */ +SEC("socket") +int both_ops(void *ctx) +{ + return ops[1](ops[0](4)); +} + +/* no callx here */ +SEC("socket") +int read_bias(void *ctx) +{ + return bias; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_callx.c b/tools/testing/selftests/bpf/progs/verifier_callx.c new file mode 100644 index 000000000000..238fc75ad154 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_callx.c @@ -0,0 +1,1039 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Tests for callx: indirect calls of bpf subprogs */ + +#include +#include +#include "bpf_misc.h" +#include "../../../include/linux/filter.h" + +#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) + +#define CALLX_INSN(DST, SRC, OFF, IMM) \ + BPF_RAW_INSN(BPF_JMP | BPF_CALL | BPF_X, DST, SRC, OFF, IMM) + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, long long); +} map_array SEC(".maps"); + +struct val_with_lock { + struct bpf_spin_lock lock; + int cnt; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct val_with_lock); +} map_lock SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(int)); + __uint(value_size, sizeof(int)); +} map_prog SEC(".maps"); + +__naked __noinline __used +static unsigned long add1(void) +{ + asm volatile ( + "r0 = r1;" + "r0 += 1;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long add2(void) +{ + asm volatile ( + "r0 = r1;" + "r0 += 2;" + "exit;" + ); +} + +/* apply(fn, x) { return fn(x); } */ +__naked __noinline __used +static unsigned long apply(void) +{ + asm volatile ( + "r3 = r1;" + "r1 = r2;" + "callx r3;" + "exit;" + ); +} + +SEC("socket") +__success __retval(6) +__naked void callx_basic(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* callx is printed by the verifier log and xlated dump */ +SEC("socket") +__success __log_level(2) +__msg("(8d) callx r2") +__xlated("callx r2") +__naked void callx_disasm(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* different callees are called by the same callx on different paths */ +SEC("socket") +__success __retval(11) +__naked void callx_two_callees(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r2 = %[add1] ll;" + "if r6 == 0 goto +2;" + "r2 = %[add2] ll;" + "r1 = 10;" + "callx r2;" + /* add1(10) - 0 or add2(10) - 1 */ + "r0 -= r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(add1), + __imm_addr(add2) + : __clobber_all); +} + +/* pointer to a function is passed as an argument */ +SEC("socket") +__success __retval(45) +__naked void callx_fn_as_arg(void) +{ + asm volatile ( + "r1 = %[add2] ll;" + "r2 = 40;" + "call apply;" + "r6 = r0;" + "r1 = %[add1] ll;" + "r2 = 2;" + "call apply;" + "r0 += r6;" + "exit;" + : + : __imm_addr(add1), + __imm_addr(add2) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long get_add1(void) +{ + asm volatile ( + "r0 = %[add1] ll;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* pointer to a function is returned from a subprog and called via r0 */ +SEC("socket") +__success __retval(2) +__naked void callx_r0(void) +{ + asm volatile ( + "call get_add1;" + "r1 = 1;" + "callx r0;" + "exit;" + ::: __clobber_all); +} + +/* pointer to a function survives spill/fill */ +SEC("socket") +__success __retval(9) +__naked void callx_spill_fill(void) +{ + asm volatile ( + "r2 = %[add2] ll;" + "*(u64 *)(r10 - 8) = r2;" + "call %[bpf_get_prandom_u32];" + "r1 = 7;" + "r9 = *(u64 *)(r10 - 8);" + "callx r9;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(add2) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long clobber_callee_saved(void) +{ + asm volatile ( + "r6 = 100;" + "r7 = 100;" + "r8 = 100;" + "r9 = 100;" + "r0 = r1;" + "exit;" + ); +} + +/* r6-r9 are preserved across callx */ +SEC("socket") +__success __retval(11) +__naked void callx_callee_saved_regs(void) +{ + asm volatile ( + "r6 = 1;" + "r7 = 2;" + "r8 = 3;" + "r9 = 4;" + "r1 = 1;" + "r2 = %[clobber_callee_saved] ll;" + "callx r2;" + "r0 += r6;" + "r0 += r7;" + "r0 += r8;" + "r0 += r9;" + "exit;" + : + : __imm_addr(clobber_callee_saved) + : __clobber_all); +} + +/* r1-r5 are scratched by callx */ +SEC("socket") +__failure __msg("R1 !read_ok") +__naked void callx_scratches_args(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "callx r2;" + "r0 = r1;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* return value of the callee is tracked */ +SEC("socket") +__success __log_level(2) +__msg("to caller at 4:") +__msg("R0=6") +__naked void callx_retval_is_tracked(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long write42(void) +{ + asm volatile ( + "r2 = 42;" + "*(u64 *)(r1 + 0) = r2;" + "r0 = 0;" + "exit;" + ); +} + +/* callee writes into the stack of the caller */ +SEC("socket") +__success __retval(42) +__naked void callx_callee_writes_caller_stack(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "r1 = r10;" + "r1 += -8;" + "r2 = %[write42] ll;" + "callx r2;" + "r0 = *(u64 *)(r10 - 8);" + "exit;" + : + : __imm_addr(write42) + : __clobber_all); +} + +SEC("socket") +__failure __msg("R1 has type scalar, expected func") +__naked void callx_scalar(void) +{ + asm volatile ( + "r1 = 0;" + "callx r1;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("R2 !read_ok") +__naked void callx_uninit_reg(void) +{ + asm volatile ( + "callx r2;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("R10 has type fp, expected func") +__naked void callx_fp(void) +{ + asm volatile ( + "callx r10;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("R1 has type map_value, expected func") +__naked void callx_map_value(void) +{ + asm volatile ( + "r1 = %[map_array] ll;" + "r2 = r10;" + "r2 += -4;" + "r3 = 0;" + "*(u32 *)(r2 + 0) = r3;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 1f;" + "r1 = r0;" + "callx r1;" + "1:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_array) + : __clobber_all); +} + +/* the address of a subprog can't be modified before the call */ +SEC("socket") +__failure __msg("dereference of modified func ptr R2 off=8 disallowed") +__naked void callx_modified_ptr(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "r2 += 8;" + "callx r2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +SEC("socket") +__failure __msg("variable func access var_off=") +__naked void callx_variable_ptr(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r0 &= 8;" + "r2 = %[add1] ll;" + "r2 += r0;" + "r1 = 5;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(add1) + : __clobber_all); +} + +__noinline __used +int global_add3(int x) +{ + return x + 3; +} + +/* only static subprogs can be called via callx */ +SEC("socket") +__failure __msg("callback function not static") +__naked void callx_global_func(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[global_add3] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(global_add3) + : __clobber_all); +} + +#define DEFINE_CALLX_RESERVED_FIELDS_PROG(NAME, SRC_REG, OFF, IMM) \ + SEC("socket") \ + __failure __msg("BPF_CALL|BPF_X uses reserved fields") \ + __naked void callx_reserved_field_ ## NAME(void) \ + { \ + asm volatile ( \ + "r1 = 5;" \ + "r2 = %[add1] ll;" \ + ".8byte %[callx_r2];" \ + "exit;" \ + : \ + : __imm_addr(add1), \ + __imm_insn(callx_r2, CALLX_INSN(BPF_REG_2, (SRC_REG), (OFF), (IMM))) \ + : __clobber_all); \ + } + +DEFINE_CALLX_RESERVED_FIELDS_PROG(src_reg, BPF_REG_1, 0, 0) +DEFINE_CALLX_RESERVED_FIELDS_PROG(off, BPF_REG_0, 1, 0) +DEFINE_CALLX_RESERVED_FIELDS_PROG(imm, BPF_REG_0, 0, 1) + +SEC("socket") +__failure __msg("unknown opcode 8e") +__naked void callx_jmp32(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + ".8byte %[callx32_r2];" + "exit;" + : + : __imm_addr(add1), + __imm_insn(callx32_r2, + BPF_RAW_INSN(BPF_JMP32 | BPF_CALL | BPF_X, BPF_REG_2, 0, 0, 0)) + : __clobber_all); +} + +/* similar to calls of static subprogs callx is allowed under a lock */ +SEC("tc") +__success __retval(3) +__naked void callx_under_lock(void) +{ + asm volatile ( + "r1 = 0;" + "*(u32 *)(r10 - 4) = r1;" + "r2 = r10;" + "r2 += -4;" + "r1 = %[map_lock] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 != 0 goto 1f;" + "exit;" + "1:" + "r6 = r0;" + "r1 = r6;" + "call %[bpf_spin_lock];" + "r1 = 1;" + "r2 = %[add2] ll;" + "callx r2;" + "r7 = r0;" + "r1 = r6;" + "call %[bpf_spin_unlock];" + "r0 = r7;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm(bpf_spin_lock), + __imm(bpf_spin_unlock), + __imm_addr(map_lock), + __imm_addr(add2) + : __clobber_all); +} + +/* helpers are still not allowed under a lock in the callee */ +__naked __noinline __used +static unsigned long call_helper(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("tc") +__failure __msg("function calls are not allowed while holding a lock") +__naked void callx_helper_under_lock(void) +{ + asm volatile ( + "r1 = 0;" + "*(u32 *)(r10 - 4) = r1;" + "r2 = r10;" + "r2 += -4;" + "r1 = %[map_lock] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 != 0 goto 1f;" + "exit;" + "1:" + "r6 = r0;" + "r1 = r6;" + "call %[bpf_spin_lock];" + "r2 = %[call_helper] ll;" + "callx r2;" + "r1 = r6;" + "call %[bpf_spin_unlock];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm(bpf_spin_lock), + __imm(bpf_spin_unlock), + __imm_addr(map_lock), + __imm_addr(call_helper) + : __clobber_all); +} + +/* self(fn) { return fn(fn); } */ +__naked __noinline __used +static unsigned long self(void) +{ + asm volatile ( + "callx r1;" + "exit;" + ); +} + +/* unbounded recursion is caught by the main verification pass */ +SEC("socket") +__failure __msg("frames is too deep") +__naked void callx_unbounded_recursion(void) +{ + asm volatile ( + "r1 = %[self] ll;" + "call self;" + "exit;" + : + : __imm_addr(self) + : __clobber_all); +} + +/* countdown(fn, n) { return n ? fn(fn, n - 1) : 0; } */ +__naked __noinline __used +static unsigned long countdown(void) +{ + asm volatile ( + "r0 = 0;" + "if r2 == 0 goto 1f;" + "r2 += -1;" + "callx r1;" + "1:" + "exit;" + ); +} + +/* + * The depth of the recursion is known to the main verification pass, + * but recursive calls are not allowed regardless. + */ +SEC("socket") +__failure __msg("recursive call from countdown() to countdown()") +__naked void callx_bounded_recursion(void) +{ + asm volatile ( + "r1 = %[countdown] ll;" + "r2 = 2;" + "call countdown;" + "exit;" + : + : __imm_addr(countdown) + : __clobber_all); +} + +/* ping(fn1, fn2, n) { return n ? fn2(fn2, fn1, n - 1) : 0; } */ +__naked __noinline __used +static unsigned long ping(void) +{ + asm volatile ( + "r0 = 0;" + "if r3 == 0 goto 1f;" + "r3 += -1;" + "r4 = r1;" + "r1 = r2;" + "r2 = r4;" + "callx r1;" + "1:" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long pong(void) +{ + asm volatile ( + "r0 = 0;" + "if r3 == 0 goto 1f;" + "r3 += -1;" + "r4 = r1;" + "r1 = r2;" + "r2 = r4;" + "callx r1;" + "1:" + "exit;" + ); +} + +SEC("socket") +__failure __msg("recursive call from") +__naked void callx_mutual_recursion(void) +{ + asm volatile ( + "r1 = %[ping] ll;" + "r2 = %[pong] ll;" + "r3 = 3;" + "call ping;" + "exit;" + : + : __imm_addr(ping), + __imm_addr(pong) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long use_stack_304(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "exit;" + ); +} + +/* stack of the callee of callx is accounted */ +SEC("socket") +__failure __msg("combined stack size of 2 calls is") +__naked void callx_stack_depth(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "r2 = %[use_stack_304] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(use_stack_304) + : __clobber_all); +} + +/* apply_stack_304(fn) { char buf[304]; return fn(); } */ +__naked __noinline __used +static unsigned long apply_stack_304(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "callx r1;" + "exit;" + ); +} + +/* + * The address of use_stack_304() is taken by the main prog that doesn't + * use stack, but it is called from apply_stack_304(). + */ +SEC("socket") +__failure __msg("combined stack size of 3 calls is") +__naked void callx_stack_depth_nested(void) +{ + asm volatile ( + "r1 = %[use_stack_304] ll;" + "call apply_stack_304;" + "exit;" + : + : __imm_addr(use_stack_304) + : __clobber_all); +} + +SEC("socket") +__success __retval(0) +__naked void callx_stack_depth_ok(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 200) = r0;" + "r2 = %[use_stack_304] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(use_stack_304) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long do_tail_call(void) +{ + asm volatile ( + "r2 = %[map_prog] ll;" + "r3 = 0;" + "call %[bpf_tail_call];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_tail_call), + __imm_addr(map_prog) + : __clobber_all); +} + +SEC("socket") +__failure __msg("tail_calls are not allowed in functions called via callx") +__naked void callx_tail_call_in_callee(void) +{ + asm volatile ( + "r2 = %[do_tail_call] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(do_tail_call) + : __clobber_all); +} + +/* tail call in the caller of callx is fine */ +SEC("socket") +__success __retval(3) +__naked void callx_tail_call_in_caller(void) +{ + asm volatile ( + "r6 = r1;" + "r1 = 2;" + "r2 = %[add1] ll;" + "callx r2;" + "r7 = r0;" + "r1 = r6;" + "r2 = %[map_prog] ll;" + "r3 = 0;" + "call %[bpf_tail_call];" + "r0 = r7;" + "exit;" + : + : __imm(bpf_tail_call), + __imm_addr(map_prog), + __imm_addr(add1) + : __clobber_all); +} + +/* read_idx(p) { return ((char *)map_value)[*p]; } */ +__naked __noinline __used +static unsigned long read_idx(void) +{ + asm volatile ( + "r6 = *(u64 *)(r1 + 0);" + "r1 = 0;" + "*(u32 *)(r10 - 4) = r1;" + "r2 = r10;" + "r2 += -4;" + "r1 = %[map_array] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 1f;" + "r0 += r6;" + "r0 = *(u8 *)(r0 + 0);" + "1:" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_array) + : __clobber_all); +} + +/* + * Stack slots of the caller that might be read by the callee of callx + * have to be considered alive at the checkpoints before callx and + * inside of the callee. Otherwise the state with fp-8 == 1000 is pruned + * and out of bounds access in read_idx() goes unnoticed. + */ +SEC("socket") +__failure __msg("invalid access to map value, value_size=8 off=1000 size=1") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void callx_callee_reads_caller_stack(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 1000;" + "*(u64 *)(r10 - 8) = r1;" + "if r0 == 0 goto 1f;" + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "1:" + "r1 = r10;" + "r1 += -8;" + "r2 = %[read_idx] ll;" + "callx r2;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(read_idx) + : __clobber_all); +} + +/* in bounds access in read_idx() is fine */ +SEC("socket") +__success __retval(0) +__flag(BPF_F_TEST_STATE_FREQ) +__naked void callx_callee_reads_caller_stack_ok(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 7;" + "*(u64 *)(r10 - 8) = r1;" + "if r0 == 0 goto 1f;" + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "1:" + "r1 = r10;" + "r1 += -8;" + "r2 = %[read_idx] ll;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(read_idx) + : __clobber_all); +} + +/* same as above, but the pointer to the stack is passed through one more frame */ +SEC("socket") +__failure __msg("invalid access to map value, value_size=8 off=1000 size=1") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void callx_callee_reads_caller_stack_nested(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 1000;" + "*(u64 *)(r10 - 8) = r1;" + "if r0 == 0 goto 1f;" + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "1:" + "r1 = %[read_idx] ll;" + "r2 = r10;" + "r2 += -8;" + "call apply;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(read_idx) + : __clobber_all); +} + +/* registers that are constant before callx are not constant after it */ +SEC("socket") +__success __retval(1) +__naked void callx_clobbers_const_regs(void) +{ + asm volatile ( + "r0 = 0;" + "r1 = 0;" + "r2 = %[add1] ll;" + "callx r2;" + /* dead branch pruning must not assume that r0 is still 0 */ + "if r0 == 0 goto 1f;" + "r0 = 1;" + "exit;" + "1:" + "r0 = 2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* scalar argument passed through callx is tracked precisely */ +__naked __noinline __used +static unsigned long identity(void) +{ + asm volatile ( + "r0 = r1;" + "exit;" + ); +} + +long long vals[] SEC(".data.vals") = {1, 2, 3, 4}; + +SEC("socket") +__success __log_level(2) +__msg("mark_precise: frame0: regs=r0 stack= before 12: (95) exit") +__msg("mark_precise: frame1: regs=r0 stack= before 11: (bf) r0 = r1") +__msg("mark_precise: frame1: regs=r1 stack= before 4: (8d) callx r2") +__msg("mark_precise: frame0: regs=r1 stack= before 3: (bf) r1 = r6") +__msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3") +__retval(4) +__naked void callx_precision(void) +{ + asm volatile ( + "r2 = %[identity] ll;" + "r6 = 3;" + "r1 = r6;" + "callx r2;" + "r0 *= 8;" + "r1 = %[vals] ll;" + "r1 += r0;" + "r0 = *(u64 *)(r1 + 0);" + "exit;" + : + : __imm_addr(identity), + __imm_addr(vals) + : __clobber_all); +} + +/* + * The address of a function that is never called is valid. The function is + * the last one in the program. + */ +SEC("socket") +__success __retval(6) +__naked void callx_addr_of_dead_func(void) +{ + asm volatile ( + "r6 = %[add1] ll;" + "r7 = %[add2] ll;" + "r1 = 5;" + "callx r6;" + "if r7 != r6 goto +1;" + "r0 = 0;" + "exit;" + : + : __imm_addr(add1), __imm_addr(add2) + : __clobber_all); +} + +/* and when it's followed by another function */ +SEC("socket") +__success __retval(7) +__naked void callx_addr_of_dead_func_first(void) +{ + asm volatile ( + "r7 = %[add1] ll;" + "r6 = %[add2] ll;" + "r1 = 5;" + "callx r6;" + "if r7 != r6 goto +1;" + "r0 = 0;" + "exit;" + : + : __imm_addr(add1), __imm_addr(add2) + : __clobber_all); +} + +/* the same without callx */ +SEC("socket") +__success __retval(6) +__naked void call_addr_of_dead_func(void) +{ + asm volatile ( + "r1 = 5;" + "call add1;" + "r7 = %[add2] ll;" + "exit;" + : + : __imm_addr(add2) + : __clobber_all); +} + +/* function pointers in C */ + +typedef int (*op_fn)(int); + +static __noinline int mul3(int x) +{ + return x * 3; +} + +static __noinline int sub7(int x) +{ + return x - 7; +} + +static __noinline int apply_op(op_fn op, int x) +{ + return op(x); +} + +SEC("socket") +__success __retval(36) +int callx_c_fn_as_arg(void *ctx) +{ + /* (5 * 3) + (28 - 7) */ + return apply_op(mul3, 5) + apply_op(sub7, 28); +} + +SEC("socket") +__success __retval(30) +int callx_c_select(void *ctx) +{ + __u32 rnd = bpf_get_prandom_u32() & 1; + op_fn op = rnd ? mul3 : sub7; + int x = rnd ? 10 : 37; + + /* 10 * 3 or 37 - 7 */ + return op(x); +} + +struct ops { + op_fn first; + op_fn second; + int bias; +}; + +static __noinline int run_ops(const struct ops *ops, int x) +{ + return ops->second(ops->first(x)) + ops->bias; +} + +SEC("socket") +__success __retval(100) +int callx_c_ops_on_stack(void *ctx) +{ + struct ops a, b; + + /* avoid an initializer with function pointers in .rodata */ + a.first = mul3; + a.second = sub7; + a.bias = 10; + b.first = sub7; + b.second = mul3; + b.bias = 1; + + /* ((7 * 3) - 7 + 10) + ((32 - 7) * 3 + 1) */ + return run_ops(&a, 7) + run_ops(&b, 32); +} + +#else + +SEC("socket") +__success +int dummy(void *ctx) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c b/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c new file mode 100644 index 000000000000..af1f9305da37 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c @@ -0,0 +1,846 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Tests for callx through pointers to functions in read-only data */ + +#include +#include +#include "bpf_misc.h" +#include "../../../include/linux/filter.h" + +#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) + +/* + * Read-only data with pointers to functions, where the compiler puts tables + * of functions, structures of operations and vtables. libbpf resolves + * a pointer to the offset of the function in the program and the kernel + * recognizes it by that value. + */ +#define DATA(SECTION, NAME, ...) \ + ".pushsection " SECTION ",@progbits;" \ + ".balign 8;" \ + #NAME "_%=:" \ + __VA_ARGS__ \ + ".type " #NAME "_%=, @object;" \ + ".size " #NAME "_%=, .-" #NAME "_%=;" \ + ".popsection;" + +#define RODATA(NAME, ...) DATA(".rodata,\"a\"", NAME, __VA_ARGS__) + +#define FUNC_TABLE2(NAME, F0, F1) RODATA(NAME, ".quad " #F0 "; .quad " #F1 ";") + +__naked __noinline __used +static unsigned long add1(void) +{ + asm volatile ( + "r0 = r1;" + "r0 += 1;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long add2(void) +{ + asm volatile ( + "r0 = r1;" + "r0 += 2;" + "exit;" + ); +} + +/* the second element of the table is called */ +SEC("socket") +__success __retval(12) +__naked void callx_rodata_const_index(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 8);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* + * The program reads the address of a function from the data, so pointers to + * functions in data are recognized only for programs that may leak pointers. + * Otherwise nothing refers to the functions. + */ +SEC("socket") +__success __retval(12) +__failure_unpriv __msg_unpriv("unreachable insn") +__caps_unpriv(CAP_BPF) +__naked void callx_rodata_needs_perfmon(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 8);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* the address of an element is used instead of the address of the table */ +SEC("socket") +__success __retval(12) +__naked void callx_rodata_elem_addr(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= + 8 ll;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* pointers to functions are mixed with other data like in a vtable */ +SEC("socket") +__success __retval(23) +__log_level(2) +__msg("r1 = *(u64 *)(r6 +0) ; R1=7") +__msg("r2 = *(u64 *)(r6 +8) ; R2=func()") +__naked void callx_rodata_mixed_with_data(void) +{ + asm volatile ( + RODATA(vt, ".quad 7; .quad add1; .quad 13; .quad add2;") + "r6 = vt_%= ll;" + "r1 = *(u64 *)(r6 + 0);" + "r2 = *(u64 *)(r6 + 8);" + /* add1(7) */ + "callx r2;" + "r7 = r0;" + "r1 = *(u64 *)(r6 + 16);" + "r2 = *(u64 *)(r6 + 24);" + /* add2(13) */ + "callx r2;" + "r0 += r7;" + "exit;" + ::: __clobber_all); +} + +/* + * Position independent code keeps constants with pointers in .data.rel.ro. + * libbpf treats it as read-only data. + */ +SEC("socket") +__success __retval(23) +__naked void callx_data_rel_ro(void) +{ + asm volatile ( + DATA(".data.rel.ro,\"aw\"", vt, ".quad 7; .quad add1; .quad 13; .quad add2;") + "r6 = vt_%= ll;" + "r1 = *(u64 *)(r6 + 0);" + "r2 = *(u64 *)(r6 + 8);" + "callx r2;" + "r7 = r0;" + "r1 = *(u64 *)(r6 + 16);" + "r2 = *(u64 *)(r6 + 24);" + "callx r2;" + "r0 += r7;" + "exit;" + ::: __clobber_all); +} + +/* an array of structures: the index selects one of the functions, not the data */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_array_of_structs(void) +{ + asm volatile ( + RODATA(arr, ".quad add1; .quad 0x1111; .quad add2; .quad 0x2222;") + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r3 = r6;" + "r3 <<= 4;" + "r2 = arr_%= ll;" + "r2 += r3;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + /* add1(10) - 0 or add2(10) - 1 */ + "r0 -= r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* and the data of the element that the index selects can be read */ +SEC("socket") +__success __retval(16) +__naked void callx_rodata_array_of_structs_data(void) +{ + asm volatile ( + RODATA(arr, ".quad add1; .quad 5; .quad add2; .quad 5;") + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r3 = r6;" + "r3 <<= 4;" + "r7 = arr_%= ll;" + "r7 += r3;" + "r2 = *(u64 *)(r7 + 0);" + "r1 = 10;" + "callx r2;" + /* add1(10) - 0 or add2(10) - 1 */ + "r0 -= r6;" + "r1 = *(u64 *)(r7 + 8);" + "r0 += r1;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* but not the data and the pointer with the same load */ +SEC("socket") +__failure __msg("overlaps with a pointer to a function") +__naked void callx_rodata_array_of_structs_any(void) +{ + asm volatile ( + RODATA(arr, ".quad add1; .quad 5; .quad add2; .quad 5;") + "call %[bpf_get_prandom_u32];" + "r0 &= 3;" + "r0 <<= 3;" + "r2 = arr_%= ll;" + "r2 += r0;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* dynamic dispatch: which vtable is used is not known until run time */ +SEC("socket") +__success __retval(2) +__naked void callx_rodata_two_vtables(void) +{ + asm volatile ( + RODATA(vt_a, ".quad 1; .quad add1;") + RODATA(vt_b, ".quad 0; .quad add2;") + "call %[bpf_get_prandom_u32];" + "r6 = vt_a_%= ll;" + "r0 &= 1;" + "if r0 == 0 goto +2;" + "r6 = vt_b_%= ll;" + "r1 = *(u64 *)(r6 + 0);" + "r2 = *(u64 *)(r6 + 8);" + /* add1(1) or add2(0) */ + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* both functions are verified, either of them is called */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_var_index(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r3 = r6;" + "r3 <<= 3;" + "r2 = tbl_%= ll;" + "r2 += r3;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + /* add1(10) - 0 or add2(10) - 1 */ + "r0 -= r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* a table that has no symbol, the compiler generates such for a switch statement */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_no_symbol(void) +{ + asm volatile ( + ".pushsection .rodata,\"a\",@progbits;" + ".balign 8;" + ".Lanon_%=:" + ".quad add1;" + ".quad add2;" + ".popsection;" + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r3 = r6;" + "r3 <<= 3;" + "r2 = .Lanon_%= ll;" + "r2 += r3;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "r0 -= r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* the first instruction of the function is removed by the verifier */ +__naked __noinline __used +static unsigned long nop_add1(void) +{ + asm volatile ( + "goto +0;" + "r0 = r1;" + "r0 += 1;" + "exit;" + ); +} + +/* the pointer follows the function when instructions are removed */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_func_starts_with_nop(void) +{ + asm volatile ( + RODATA(tbl, ".quad nop_add1; .quad 0;") + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* can't be called with a scalar in r1 */ +__naked __noinline __used +static unsigned long deref_r1(void) +{ + asm volatile ( + "r0 = *(u64 *)(r1 + 0);" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long ret0(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +/* every function that might be called is verified */ +SEC("socket") +__failure __msg("R1 invalid mem access 'scalar'") +__naked void callx_rodata_all_callees_verified(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, ret0, deref_r1) + "call %[bpf_get_prandom_u32];" + "r0 &= 1;" + "r0 <<= 3;" + "r2 = tbl_%= ll;" + "r2 += r0;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 0;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* a function that is never called is not verified, it's dead code */ +SEC("socket") +__success __retval(0) +__naked void callx_rodata_unused_callee(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, ret0, deref_r1) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 0;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* a pointer that is read and never called doesn't keep the function, it's NULL */ +SEC("socket") +__success __retval(1) +__naked void callx_rodata_read_unused_callee(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, ret0, deref_r1) + "r6 = tbl_%= ll;" + "r2 = *(u64 *)(r6 + 0);" + "r1 = 0;" + "callx r2;" + "r2 = *(u64 *)(r6 + 8);" + "if r2 != 0 goto +1;" + "r0 = 1;" + "exit;" + ::: __clobber_all); +} + +/* the index may select what is not a pointer to a function */ +SEC("socket") +__failure __msg("overlaps with a pointer to a function") +__naked void callx_rodata_index_beyond_table(void) +{ + asm volatile ( + RODATA(tbl, ".quad add1; .quad add2; .quad 0x1234; .quad 0x5678;") + "call %[bpf_get_prandom_u32];" + "r0 &= 3;" + "r0 <<= 3;" + "r2 = tbl_%= ll;" + "r2 += r0;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* the address of a function is not known until the program is jitted */ +SEC("socket") +__failure __msg("read of 4 bytes at offset") +__msg("overlaps with a pointer to a function") +__naked void callx_rodata_partial_read(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r6 = tbl_%= ll;" + "r2 = *(u64 *)(r6 + 8);" + "r1 = 10;" + "callx r2;" + "r0 = *(u32 *)(r6 + 0);" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("read of 8 bytes at offset") +__msg("overlaps with a pointer to a function") +__naked void callx_rodata_misaligned_read(void) +{ + asm volatile ( + RODATA(tbl, ".quad add1; .quad add2; .quad 0;") + "r6 = tbl_%= ll;" + "r2 = *(u64 *)(r6 + 0);" + "r1 = 10;" + "callx r2;" + "r0 = *(u64 *)(r6 + 4);" + "exit;" + ::: __clobber_all); +} + +/* + * The kernel doesn't look for pointers to functions in the data of a program + * that doesn't have callx. It's a number there. + */ +SEC("socket") +__success +__naked void callx_rodata_no_callx(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r0 = *(u32 *)(r2 + 0);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* the data next to a pointer is still known to the verifier */ +SEC("socket") +__success __retval(0x1234) +__log_level(2) +__msg("R0=4660") +__naked void callx_rodata_data_is_const(void) +{ + asm volatile ( + RODATA(tbl, ".quad add1; .quad 0x1234;") + "r6 = tbl_%= ll;" + "r2 = *(u64 *)(r6 + 0);" + "r1 = 10;" + "callx r2;" + "r0 = *(u64 *)(r6 + 8);" + "exit;" + ::: __clobber_all); +} + +/* + * The map has the offset of the function until the program is jitted. + * bpf_compute_const_regs() must not take it for a constant, otherwise + * bpf_prune_dead_branches() turns the branch below into the wrong one. + */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_ptr_is_not_const(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "r0 = 0;" + /* the offset is less, the address is greater */ + "if r2 < 0x100000 goto 1f;" + "r1 = 10;" + "callx r2;" + "1:" + "exit;" + ::: __clobber_all); +} + +/* the pointer read from the data can't be modified */ +SEC("socket") +__failure __msg("dereference of modified func ptr R2 off=8 disallowed") +__naked void callx_rodata_modified_ptr(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "r2 += 8;" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +__noinline __used +int global_add3(int x) +{ + return x + 3; +} + +/* + * A pointer to a global function is not recognized, it's a number. The function + * is called directly as well, so that libbpf adds it to the program and stores + * its offset into the table. + */ +SEC("socket") +__failure __msg("R2 has type scalar, expected func") +__naked void callx_rodata_global_func(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, global_add3) + "r1 = 1;" + "call global_add3;" + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 8);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* selfcall(n) { return n ? tbl[0](n - 1) : 0; }, where tbl[0] == selfcall */ +__naked __noinline __used +static unsigned long selfcall(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, selfcall, ret0) + "r0 = 0;" + "if r1 == 0 goto 1f;" + "r1 += -1;" + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "callx r2;" + "1:" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("recursive call from selfcall() to selfcall()") +__naked void callx_rodata_recursion(void) +{ + asm volatile ( + "r1 = 2;" + "call selfcall;" + "exit;" + ::: __clobber_all); +} + +__naked __noinline __used +static unsigned long use_stack_304(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "exit;" + ); +} + +/* stack of all possible callees is accounted */ +SEC("socket") +__failure __msg("combined stack size of 2 calls is") +__naked void callx_rodata_stack_depth(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, ret0, use_stack_304) + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "call %[bpf_get_prandom_u32];" + "r0 &= 1;" + "r0 <<= 3;" + "r2 = tbl_%= ll;" + "r2 += r0;" + "r2 = *(u64 *)(r2 + 0);" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* stack of a callback that is read from the data is accounted too */ +SEC("socket") +__failure __msg("combined stack size of 2 calls is") +__naked void callx_rodata_callback_stack_depth(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, use_stack_304, ret0) + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "r6 = tbl_%= ll;" + "r1 = 1;" + "r2 = *(u64 *)(r6 + 0);" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_loop];" + "r2 = *(u64 *)(r6 + 8);" + "r1 = 0;" + "callx r2;" + "exit;" + : + : __imm(bpf_loop) + : __clobber_all); +} + +static __noinline __used int pull_data(struct __sk_buff *skb) +{ + return bpf_skb_pull_data(skb, 0); +} + +static __noinline __used int calls_pull_data(struct __sk_buff *skb) +{ + return pull_data(skb) + 1; +} + +__noinline __used int global_calls_pull_data(struct __sk_buff *skb) +{ + return calls_pull_data(skb) + 2; +} + +/* + * check_cfg() explores the functions that the data points to when it sees + * the address of the data. It must not lose the effects of the functions + * that call each other: global_calls_pull_data() changes packet data. + */ +SEC("tc") +__failure __msg("R7 invalid mem access 'scalar'") +__naked void callx_rodata_callee_effects(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, pull_data, calls_pull_data) + "r6 = r1;" + "r7 = *(u32 *)(r6 + %[data]);" + "r8 = *(u32 *)(r6 + %[data_end]);" + "r2 = r7;" + "r2 += 8;" + "if r2 > r8 goto 1f;" + "r1 = r6;" + "call global_calls_pull_data;" + "r0 = *(u8 *)(r7 + 0);" + "1:" + "r9 = tbl_%= ll;" + "r2 = *(u64 *)(r9 + 8);" + "r1 = r6;" + "callx r2;" + "r0 = 0;" + "exit;" + : + : __imm_const(data, offsetof(struct __sk_buff, data)), + __imm_const(data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + +/* pointers to functions in C */ + +typedef int (*op_fn)(int); + +#define DEFINE_OP(N) static __noinline int op##N(int x) { return x * (N + 2) + N; } + +DEFINE_OP(0) DEFINE_OP(1) DEFINE_OP(2) DEFINE_OP(3) +DEFINE_OP(4) DEFINE_OP(5) DEFINE_OP(6) DEFINE_OP(7) +DEFINE_OP(8) DEFINE_OP(9) DEFINE_OP(10) DEFINE_OP(11) +DEFINE_OP(12) DEFINE_OP(13) DEFINE_OP(14) DEFINE_OP(15) + +static op_fn const ops[] = { + op0, op1, op2, op3, op4, op5, op6, op7, + op8, op9, op10, op11, op12, op13, op14, op15, +}; + +int op_idx = 11; + +SEC("socket") +__success __retval(50) +int callx_c_table(void *ctx) +{ + unsigned int i = op_idx; + + if (i >= sizeof(ops) / sizeof(ops[0])) + return -1; + /* op11(3) = 3 * 13 + 11 */ + return ops[i](3); +} + +SEC("socket") +__success __retval(50) +int callx_c_table_null_check(void *ctx) +{ + unsigned int i = op_idx; + op_fn op; + + if (i >= sizeof(ops) / sizeof(ops[0])) + return -1; + /* the address of a function is not known until the program is jitted */ + op = ops[i]; + if (!op) + return -2; + return op(3); +} + +/* a structure of operations, where pointers to functions are mixed with data */ +struct shape_ops { + int id; + op_fn area; + long scale; + op_fn perimeter; +}; + +static const struct shape_ops square_ops = { 1, op1, 10, op2 }; +static const struct shape_ops circle_ops = { 2, op3, 20, op1 }; + +static __noinline int use_shape(const struct shape_ops *ops, int x) +{ + return ops->area(x) * ops->scale + ops->perimeter(ops->id); +} + +SEC("socket") +__success __retval(433) +int callx_c_ops_mixed_with_data(void *ctx) +{ + /* + * square: op1(2) * 10 + op2(1) = 7 * 10 + 6 = 76 + * circle: op3(3) * 20 + op1(2) = 18 * 20 + 7 = 367 + * minus 10, since op_idx is 11 + */ + return use_shape(&square_ops, 2) + use_shape(&circle_ops, 3) - (op_idx == 11 ? 10 : 0); +} + +/* the ops are selected at run time */ +SEC("socket") +__success __retval(367) +int callx_c_ops_selected(void *ctx) +{ + const struct shape_ops *ops = op_idx == 11 ? &circle_ops : &square_ops; + + return use_shape(ops, 3); +} + +/* the compiler might turn the switch into a table that has no symbol */ +static __noinline int call_by_switch(unsigned int idx, int x) +{ + op_fn op; + + switch (idx) { + case 0: + op = op8; + break; + case 1: + op = op9; + break; + case 2: + op = op10; + break; + case 3: + op = op11; + break; + case 4: + op = op12; + break; + case 5: + op = op13; + break; + case 6: + op = op14; + break; + case 7: + op = op15; + break; + default: + return -1; + } + return op(x); +} + +SEC("socket") +__success __retval(50) +int callx_c_switch_table(void *ctx) +{ + /* op11(3) = 3 * 13 + 11 */ + return call_by_switch(op_idx - 8, 3); +} + +/* + * Misaligned pointers to functions are ignored, the rest of the data is + * accessible as before. The program doesn't have callx, so libbpf doesn't + * append the functions that .rodata points to and doesn't make a copy of + * .rodata for it. + */ +static const struct { + char tag; + op_fn op; +} __attribute__((packed)) packed_ops[] = { + { 5, op0 }, + { 7, op1 }, +}; + +SEC("socket") +__success __retval(7) +int callx_c_packed_struct(void *ctx) +{ + return packed_ops[op_idx & 1].tag; +} + +#else + +SEC("socket") +__success +int dummy(void *ctx) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c index 966f49348787..27fbe54e8795 100644 --- a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c +++ b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c @@ -536,4 +536,35 @@ int return_from_void_global(struct __sk_buff *skb) return 0; } +int global_calls_loop(int x); + +static __noinline int static_calls_global(int x) +{ + return global_calls_loop(x); +} + +static __noinline int loop_cb_calls_static(u32 i, void *ctx) +{ + return static_calls_global(i); +} + +__noinline int global_calls_loop(int x) +{ + bpf_loop(1, loop_cb_calls_static, NULL, 0); + return 0; +} + +/* + * loop_cb_calls_static() -> static_calls_global() -> global_calls_loop() -> + * bpf_loop() -> loop_cb_calls_static() is an unbounded recursion that the + * main verification pass can't see, because it doesn't follow calls of global + * functions. None of the functions use stack. + */ +SEC("?raw_tp") +__failure __msg("recursive call from global_calls_loop() to loop_cb_calls_static()") +int recursion_via_global_func_and_callback(const void *ctx) +{ + return loop_cb_calls_static(0, NULL); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/verifier/basic_call.c b/tools/testing/selftests/bpf/verifier/basic_call.c index a8c6ab4c1622..0f93c4551f23 100644 --- a/tools/testing/selftests/bpf/verifier/basic_call.c +++ b/tools/testing/selftests/bpf/verifier/basic_call.c @@ -4,7 +4,7 @@ BPF_RAW_INSN(BPF_JMP | BPF_CALL | BPF_X, 0, 0, 0, 0), BPF_EXIT_INSN(), }, - .errstr = "unknown opcode 8d", + .errstr = "R0 !read_ok", .result = REJECT, }, {