diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index b0075ece4a6e..0f61662b900b 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1284,12 +1284,25 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn case BPF_ALU | BPF_MOV | BPF_X: case BPF_ALU64 | BPF_MOV | BPF_X: if (insn_is_cast_user(insn)) { - emit(A64_MOV(0, tmp, src), ctx); // 32-bit mov clears the upper 32 bits - emit_a64_mov_i(0, dst, ctx->user_vm_start >> 32, ctx); - emit(A64_LSL(1, dst, dst, 32), ctx); - emit(A64_CBZ(1, tmp, 2), ctx); - emit(A64_ORR(1, tmp, dst, tmp), ctx); - emit(A64_MOV(1, dst, tmp), ctx); + u32 upper = ctx->user_vm_start >> 32; + u16 upper_low = upper & 0xffff; + u16 upper_high = upper >> 16; + int nr_movk = !!upper_low + !!upper_high; + + /* + * Build the user address: the low 32 bits are the arena + * offset, the upper 32 bits come from user_vm_start. A + * zero offset must stay NULL, so branch over the MOVKs + * when it is zero. + */ + emit(A64_MOV(0, dst, src), ctx); /* 32-bit mov clears the upper 32 bits */ + if (nr_movk) { + emit(A64_CBZ(0, dst, nr_movk + 1), ctx); + if (upper_low) + emit(A64_MOVK(1, dst, upper_low, 32), ctx); + if (upper_high) + emit(A64_MOVK(1, dst, upper_high, 48), ctx); + } break; } else if (insn_is_mov_percpu_addr(insn)) { if (dst != src)