mirror of
https://github.com/torvalds/linux.git
synced 2026-11-07 06:17:02 +01:00
Merge branch 'bpf-mips-add-signed-div-mod-support'
Nicholas Dudar says: ==================== bpf, mips: Add signed div/mod support The MIPS JITs do not interpret the insn->off value that selects signed DIV/MOD semantics, so negative operands produce unsigned results. Factor div/mod emission into helpers, add the missing signed uasm operations, and decode insn->off in the div/mod emitters. On MIPS32, use signed 64-bit helpers for ALU64 operations. The DIV/MOD implementation is unchanged from v3. Before this rebase, all signed DIV/MOD test_bpf cases passed with the pending MOVSX respin applied under QEMU Malta on little-endian MIPS32 R1/R2/R6 and MIPS64 R2/R6. The full suite had 12 expected failures for unsupported operations (8 BSWAP, 3 MEMSX and 1 JMP32_JA). R1 had 18 failures in total: those 12 plus 6 additional failures reproduced on unchanged v3 in the same environment. Earlier v3 testing covered big-endian MIPS32. Uasm object builds passed for MIPS32 R1/R6, MIPS64 R2/R6 and microMIPS32 R2 before the rebase. The new SEB/SEH encodings match GNU as for every register pair on MIPS32/64 R2/R3/R5/R6 in both endian modes; all pre-existing native opcode entries are unchanged. No physical hardware was tested. Changes in v4: - Add seb/seh uasm emitters for R2-or-newer CPUs. - Rebase onto current bpf-next without changing the implementation. v3: https://lore.kernel.org/bpf/20260810194215.3754591-1-main.kalliope@gmail.com/ MOVSX review: https://lore.kernel.org/bpf/CAM1=_QTN3JJ6jtckf32F8wxBzLDbHSrzUttyMBWNqZKmRe=2rg@mail.gmail.com/ ==================== Link: https://patch.msgid.link/20260911213340.3767930-1-main.kalliope@gmail.com Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Alexei Starovoitov
commit
b3696fdef2
|
|
@ -85,15 +85,20 @@ Ip_u1u2(_ctc1);
|
|||
Ip_u2u1(_ctcmsa);
|
||||
Ip_u2u1s3(_daddiu);
|
||||
Ip_u3u1u2(_daddu);
|
||||
Ip_u1u2(_ddiv);
|
||||
Ip_u3u1u2(_ddiv_r6);
|
||||
Ip_u1u2(_ddivu);
|
||||
Ip_u3u1u2(_ddivu_r6);
|
||||
Ip_u1(_di);
|
||||
Ip_u2u1msbu3(_dins);
|
||||
Ip_u2u1msbu3(_dinsm);
|
||||
Ip_u2u1msbu3(_dinsu);
|
||||
Ip_u1u2(_div);
|
||||
Ip_u3u1u2(_div_r6);
|
||||
Ip_u1u2(_divu);
|
||||
Ip_u3u1u2(_divu_r6);
|
||||
Ip_u1u2u3(_dmfc0);
|
||||
Ip_u3u1u2(_dmod);
|
||||
Ip_u3u1u2(_dmodu);
|
||||
Ip_u1u2u3(_dmtc0);
|
||||
Ip_u1u2(_dmultu);
|
||||
|
|
@ -135,6 +140,7 @@ Ip_u1u2u3(_mfc0);
|
|||
Ip_u1u2u3(_mfhc0);
|
||||
Ip_u1(_mfhi);
|
||||
Ip_u1(_mflo);
|
||||
Ip_u3u1u2(_mod);
|
||||
Ip_u3u1u2(_modu);
|
||||
Ip_u3u1u2(_movn);
|
||||
Ip_u3u1u2(_movz);
|
||||
|
|
@ -156,6 +162,8 @@ Ip_u2s3u1(_sb);
|
|||
Ip_u2s3u1(_sc);
|
||||
Ip_u2s3u1(_scd);
|
||||
Ip_u2s3u1(_sd);
|
||||
Ip_u2u1(_seb);
|
||||
Ip_u2u1(_seh);
|
||||
Ip_u3u1u2(_seleqz);
|
||||
Ip_u3u1u2(_selnez);
|
||||
Ip_u2s3u1(_sh);
|
||||
|
|
|
|||
|
|
@ -75,6 +75,9 @@ static const struct insn insn_table[insn_invalid] = {
|
|||
[insn_ctcmsa] = {M(msa_op, 0, msa_ctc_op, 0, 0, msa_elm_op), RD | RE},
|
||||
[insn_daddiu] = {M(daddiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
|
||||
[insn_daddu] = {M(spec_op, 0, 0, 0, 0, daddu_op), RS | RT | RD},
|
||||
[insn_ddiv] = {M(spec_op, 0, 0, 0, 0, ddiv_op), RS | RT},
|
||||
[insn_ddiv_r6] = {M(spec_op, 0, 0, 0, ddiv_ddiv6_op, ddiv_op),
|
||||
RS | RT | RD},
|
||||
[insn_ddivu] = {M(spec_op, 0, 0, 0, 0, ddivu_op), RS | RT},
|
||||
[insn_ddivu_r6] = {M(spec_op, 0, 0, 0, ddivu_ddivu6_op, ddivu_op),
|
||||
RS | RT | RD},
|
||||
|
|
@ -82,10 +85,15 @@ static const struct insn insn_table[insn_invalid] = {
|
|||
[insn_dins] = {M(spec3_op, 0, 0, 0, 0, dins_op), RS | RT | RD | RE},
|
||||
[insn_dinsm] = {M(spec3_op, 0, 0, 0, 0, dinsm_op), RS | RT | RD | RE},
|
||||
[insn_dinsu] = {M(spec3_op, 0, 0, 0, 0, dinsu_op), RS | RT | RD | RE},
|
||||
[insn_div] = {M(spec_op, 0, 0, 0, 0, div_op), RS | RT},
|
||||
[insn_div_r6] = {M(spec_op, 0, 0, 0, div_div6_op, div_op),
|
||||
RS | RT | RD},
|
||||
[insn_divu] = {M(spec_op, 0, 0, 0, 0, divu_op), RS | RT},
|
||||
[insn_divu_r6] = {M(spec_op, 0, 0, 0, divu_divu6_op, divu_op),
|
||||
RS | RT | RD},
|
||||
[insn_dmfc0] = {M(cop0_op, dmfc_op, 0, 0, 0, 0), RT | RD | SET},
|
||||
[insn_dmod] = {M(spec_op, 0, 0, 0, ddiv_dmod_op, ddiv_op),
|
||||
RS | RT | RD},
|
||||
[insn_dmodu] = {M(spec_op, 0, 0, 0, ddivu_dmodu_op, ddivu_op),
|
||||
RS | RT | RD},
|
||||
[insn_dmtc0] = {M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET},
|
||||
|
|
@ -140,6 +148,8 @@ static const struct insn insn_table[insn_invalid] = {
|
|||
[insn_mfhc0] = {M(cop0_op, mfhc0_op, 0, 0, 0, 0), RT | RD | SET},
|
||||
[insn_mfhi] = {M(spec_op, 0, 0, 0, 0, mfhi_op), RD},
|
||||
[insn_mflo] = {M(spec_op, 0, 0, 0, 0, mflo_op), RD},
|
||||
[insn_mod] = {M(spec_op, 0, 0, 0, div_mod_op, div_op),
|
||||
RS | RT | RD},
|
||||
[insn_modu] = {M(spec_op, 0, 0, 0, divu_modu_op, divu_op),
|
||||
RS | RT | RD},
|
||||
[insn_movn] = {M(spec_op, 0, 0, 0, 0, movn_op), RS | RT | RD},
|
||||
|
|
@ -177,6 +187,8 @@ static const struct insn insn_table[insn_invalid] = {
|
|||
[insn_scd] = {M6(spec3_op, 0, 0, 0, scd6_op), RS | RT | SIMM9},
|
||||
#endif
|
||||
[insn_sd] = {M(sd_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
|
||||
[insn_seb] = {M(spec3_op, 0, 0, 0, seb_op, bshfl_op), RT | RD},
|
||||
[insn_seh] = {M(spec3_op, 0, 0, 0, seh_op, bshfl_op), RT | RD},
|
||||
[insn_seleqz] = {M(spec_op, 0, 0, 0, 0, seleqz_op), RS | RT | RD},
|
||||
[insn_selnez] = {M(spec_op, 0, 0, 0, 0, selnez_op), RS | RT | RD},
|
||||
[insn_sh] = {M(sh_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
|
||||
|
|
|
|||
|
|
@ -49,19 +49,21 @@ enum opcode {
|
|||
insn_addiu, insn_addu, insn_and, insn_andi, insn_bbit0, insn_bbit1,
|
||||
insn_beq, insn_beql, insn_bgez, insn_bgezl, insn_bgtz, insn_blez,
|
||||
insn_bltz, insn_bltzl, insn_bne, insn_break, insn_cache, insn_cfc1,
|
||||
insn_cfcmsa, insn_ctc1, insn_ctcmsa, insn_daddiu, insn_daddu, insn_ddivu,
|
||||
insn_ddivu_r6, insn_di, insn_dins, insn_dinsm, insn_dinsu, insn_divu,
|
||||
insn_divu_r6, insn_dmfc0, insn_dmodu, insn_dmtc0, insn_dmultu,
|
||||
insn_cfcmsa, insn_ctc1, insn_ctcmsa, insn_daddiu, insn_daddu, insn_ddiv,
|
||||
insn_ddiv_r6, insn_ddivu, insn_ddivu_r6, insn_di, insn_dins, insn_dinsm,
|
||||
insn_dinsu, insn_div, insn_div_r6, insn_divu, insn_divu_r6, insn_dmfc0,
|
||||
insn_dmod, insn_dmodu, insn_dmtc0, insn_dmultu,
|
||||
insn_dmulu, insn_drotr, insn_drotr32, insn_dsbh, insn_dshd, insn_dsll,
|
||||
insn_dsll32, insn_dsllv, insn_dsra, insn_dsra32, insn_dsrav, insn_dsrl,
|
||||
insn_dsrl32, insn_dsrlv, insn_dsubu, insn_eret, insn_ext, insn_ins,
|
||||
insn_j, insn_jal, insn_jalr, insn_jr, insn_lb, insn_lbu, insn_ld,
|
||||
insn_lddir, insn_ldpte, insn_ldx, insn_lh, insn_lhu, insn_ll, insn_lld,
|
||||
insn_lui, insn_lw, insn_lwu, insn_lwx, insn_mfc0, insn_mfhc0, insn_mfhi,
|
||||
insn_mflo, insn_modu, insn_movn, insn_movz, insn_mtc0, insn_mthc0,
|
||||
insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, insn_muhu, insn_nor,
|
||||
insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, insn_sb, insn_sc,
|
||||
insn_scd, insn_seleqz, insn_selnez, insn_sd, insn_sh, insn_sll,
|
||||
insn_mflo, insn_mod, insn_modu, insn_movn, insn_movz, insn_mtc0,
|
||||
insn_mthc0, insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu,
|
||||
insn_muhu, insn_nor, insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr,
|
||||
insn_sb, insn_sc, insn_scd, insn_seb, insn_seh,
|
||||
insn_seleqz, insn_selnez, insn_sd, insn_sh, insn_sll,
|
||||
insn_sllv, insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra,
|
||||
insn_srav, insn_srl, insn_srlv, insn_subu, insn_sw, insn_sync,
|
||||
insn_syscall, insn_tlbp, insn_tlbr, insn_tlbwi, insn_tlbwr, insn_wait,
|
||||
|
|
@ -287,9 +289,12 @@ I_u1u2(_cfc1)
|
|||
I_u2u1(_cfcmsa)
|
||||
I_u1u2(_ctc1)
|
||||
I_u2u1(_ctcmsa)
|
||||
I_u1u2(_ddiv)
|
||||
I_u3u1u2(_ddiv_r6)
|
||||
I_u1u2(_ddivu)
|
||||
I_u3u1u2(_ddivu_r6)
|
||||
I_u1u2u3(_dmfc0)
|
||||
I_u3u1u2(_dmod)
|
||||
I_u3u1u2(_dmodu)
|
||||
I_u1u2u3(_dmtc0)
|
||||
I_u1u2(_dmultu)
|
||||
|
|
@ -297,6 +302,8 @@ I_u3u1u2(_dmulu)
|
|||
I_u2u1s3(_daddiu)
|
||||
I_u3u1u2(_daddu)
|
||||
I_u1(_di);
|
||||
I_u1u2(_div)
|
||||
I_u3u1u2(_div_r6)
|
||||
I_u1u2(_divu)
|
||||
I_u3u1u2(_divu_r6)
|
||||
I_u2u1(_dsbh);
|
||||
|
|
@ -332,6 +339,7 @@ I_u2s3u1(_lw)
|
|||
I_u2s3u1(_lwu)
|
||||
I_u1u2u3(_mfc0)
|
||||
I_u1u2u3(_mfhc0)
|
||||
I_u3u1u2(_mod)
|
||||
I_u3u1u2(_modu)
|
||||
I_u3u1u2(_movn)
|
||||
I_u3u1u2(_movz)
|
||||
|
|
@ -353,6 +361,8 @@ I_u2s3u1(_sb)
|
|||
I_u2s3u1(_sc)
|
||||
I_u2s3u1(_scd)
|
||||
I_u2s3u1(_sd)
|
||||
I_u2u1(_seb)
|
||||
I_u2u1(_seh)
|
||||
I_u3u1u2(_seleqz)
|
||||
I_u3u1u2(_selnez)
|
||||
I_u2s3u1(_sh)
|
||||
|
|
|
|||
|
|
@ -208,7 +208,7 @@ void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src)
|
|||
}
|
||||
|
||||
/* Validate ALU immediate range */
|
||||
bool valid_alu_i(u8 op, s32 imm)
|
||||
bool valid_alu_i(u8 op, s32 imm, s16 off)
|
||||
{
|
||||
switch (BPF_OP(op)) {
|
||||
case BPF_NEG:
|
||||
|
|
@ -237,6 +237,9 @@ bool valid_alu_i(u8 op, s32 imm)
|
|||
return imm == 0 || (imm > 0 && is_power_of_2(imm));
|
||||
case BPF_DIV:
|
||||
case BPF_MOD:
|
||||
/* Do not use unsigned shift/mask rewrites for signed div/mod. */
|
||||
if (off == 1)
|
||||
return false;
|
||||
/* imm must be an 17-bit power of two */
|
||||
return (u32)imm <= 0x10000 && is_power_of_2((u32)imm);
|
||||
}
|
||||
|
|
@ -338,8 +341,42 @@ void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op)
|
|||
clobber_reg(ctx, dst);
|
||||
}
|
||||
|
||||
/* ALU division operation (32-bit) */
|
||||
static void emit_div(struct jit_context *ctx, u8 dst, u8 src, s16 off)
|
||||
{
|
||||
if (cpu_has_mips32r6) {
|
||||
if (off == 1)
|
||||
emit(ctx, div_r6, dst, dst, src);
|
||||
else
|
||||
emit(ctx, divu_r6, dst, dst, src);
|
||||
} else {
|
||||
if (off == 1)
|
||||
emit(ctx, div, dst, src);
|
||||
else
|
||||
emit(ctx, divu, dst, src);
|
||||
emit(ctx, mflo, dst);
|
||||
}
|
||||
}
|
||||
|
||||
/* ALU modulo operation (32-bit) */
|
||||
static void emit_mod(struct jit_context *ctx, u8 dst, u8 src, s16 off)
|
||||
{
|
||||
if (cpu_has_mips32r6) {
|
||||
if (off == 1)
|
||||
emit(ctx, mod, dst, dst, src);
|
||||
else
|
||||
emit(ctx, modu, dst, dst, src);
|
||||
} else {
|
||||
if (off == 1)
|
||||
emit(ctx, div, dst, src);
|
||||
else
|
||||
emit(ctx, divu, dst, src);
|
||||
emit(ctx, mfhi, dst);
|
||||
}
|
||||
}
|
||||
|
||||
/* ALU register operation (32-bit) */
|
||||
void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op)
|
||||
void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op, s16 off)
|
||||
{
|
||||
switch (BPF_OP(op)) {
|
||||
/* dst = dst & src */
|
||||
|
|
@ -385,21 +422,11 @@ void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op)
|
|||
break;
|
||||
/* dst = dst / src */
|
||||
case BPF_DIV:
|
||||
if (cpu_has_mips32r6) {
|
||||
emit(ctx, divu_r6, dst, dst, src);
|
||||
} else {
|
||||
emit(ctx, divu, dst, src);
|
||||
emit(ctx, mflo, dst);
|
||||
}
|
||||
emit_div(ctx, dst, src, off);
|
||||
break;
|
||||
/* dst = dst % src */
|
||||
case BPF_MOD:
|
||||
if (cpu_has_mips32r6) {
|
||||
emit(ctx, modu, dst, dst, src);
|
||||
} else {
|
||||
emit(ctx, divu, dst, src);
|
||||
emit(ctx, mfhi, dst);
|
||||
}
|
||||
emit_mod(ctx, dst, src, off);
|
||||
break;
|
||||
}
|
||||
clobber_reg(ctx, dst);
|
||||
|
|
|
|||
|
|
@ -163,7 +163,7 @@ void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm);
|
|||
void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src);
|
||||
|
||||
/* Validate ALU/ALU64 immediate range */
|
||||
bool valid_alu_i(u8 op, s32 imm);
|
||||
bool valid_alu_i(u8 op, s32 imm, s16 off);
|
||||
|
||||
/* Rewrite ALU/ALU64 immediate operation */
|
||||
bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val);
|
||||
|
|
@ -172,7 +172,7 @@ bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val);
|
|||
void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op);
|
||||
|
||||
/* ALU register operation (32-bit) */
|
||||
void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op);
|
||||
void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op, s16 off);
|
||||
|
||||
/* Atomic read-modify-write (32-bit) */
|
||||
void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code);
|
||||
|
|
|
|||
|
|
@ -512,7 +512,7 @@ static void emit_mul_r64(struct jit_context *ctx,
|
|||
clobber_reg64(ctx, dst);
|
||||
}
|
||||
|
||||
/* Helper function for 64-bit modulo */
|
||||
/* Helper function for unsigned 64-bit modulo */
|
||||
static u64 jit_mod64(u64 a, u64 b)
|
||||
{
|
||||
u64 rem;
|
||||
|
|
@ -521,13 +521,22 @@ static u64 jit_mod64(u64 a, u64 b)
|
|||
return rem;
|
||||
}
|
||||
|
||||
/* Helper function for signed 64-bit modulo */
|
||||
static s64 jit_smod64(s64 a, s64 b)
|
||||
{
|
||||
s64 quot = div64_s64(a, b);
|
||||
|
||||
return a - quot * b;
|
||||
}
|
||||
|
||||
/* ALU div/mod register (64-bit) */
|
||||
static void emit_divmod_r64(struct jit_context *ctx,
|
||||
const u8 dst[], const u8 src[], u8 op)
|
||||
static void emit_divmod_r64(struct jit_context *ctx, const u8 dst[],
|
||||
const u8 src[], u8 op, s16 off)
|
||||
{
|
||||
const u8 *r0 = bpf2mips32[BPF_REG_0]; /* Mapped to v0-v1 */
|
||||
const u8 *r1 = bpf2mips32[BPF_REG_1]; /* Mapped to a0-a1 */
|
||||
const u8 *r2 = bpf2mips32[BPF_REG_2]; /* Mapped to a2-a3 */
|
||||
bool is_signed = off == 1;
|
||||
int exclude, k;
|
||||
u32 addr = 0;
|
||||
|
||||
|
|
@ -546,11 +555,11 @@ static void emit_divmod_r64(struct jit_context *ctx,
|
|||
switch (BPF_OP(op)) {
|
||||
/* dst = dst / src */
|
||||
case BPF_DIV:
|
||||
addr = (u32)&div64_u64;
|
||||
addr = is_signed ? (u32)&div64_s64 : (u32)&div64_u64;
|
||||
break;
|
||||
/* dst = dst % src */
|
||||
case BPF_MOD:
|
||||
addr = (u32)&jit_mod64;
|
||||
addr = is_signed ? (u32)&jit_smod64 : (u32)&jit_mod64;
|
||||
break;
|
||||
}
|
||||
emit_mov_i(ctx, MIPS_R_T9, addr);
|
||||
|
|
@ -1516,9 +1525,9 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
|
|||
case BPF_ALU | BPF_MUL | BPF_K:
|
||||
case BPF_ALU | BPF_DIV | BPF_K:
|
||||
case BPF_ALU | BPF_MOD | BPF_K:
|
||||
if (!valid_alu_i(BPF_OP(code), imm)) {
|
||||
if (!valid_alu_i(BPF_OP(code), imm, off)) {
|
||||
emit_mov_i(ctx, MIPS_R_T6, imm);
|
||||
emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code));
|
||||
emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code), off);
|
||||
} else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
|
||||
emit_alu_i(ctx, lo(dst), val, alu);
|
||||
}
|
||||
|
|
@ -1546,7 +1555,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
|
|||
case BPF_ALU | BPF_MUL | BPF_X:
|
||||
case BPF_ALU | BPF_DIV | BPF_X:
|
||||
case BPF_ALU | BPF_MOD | BPF_X:
|
||||
emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code));
|
||||
emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code), off);
|
||||
emit_zext_ver(ctx, dst);
|
||||
break;
|
||||
/* dst = imm (64-bit) */
|
||||
|
|
@ -1599,7 +1608,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
|
|||
* and then do the operation on this register.
|
||||
*/
|
||||
emit_mov_se_i64(ctx, tmp, imm);
|
||||
emit_divmod_r64(ctx, dst, tmp, BPF_OP(code));
|
||||
emit_divmod_r64(ctx, dst, tmp, BPF_OP(code), off);
|
||||
break;
|
||||
/* dst = dst & src (64-bit) */
|
||||
/* dst = dst | src (64-bit) */
|
||||
|
|
@ -1629,7 +1638,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
|
|||
/* dst = dst % src (64-bit) */
|
||||
case BPF_ALU64 | BPF_DIV | BPF_X:
|
||||
case BPF_ALU64 | BPF_MOD | BPF_X:
|
||||
emit_divmod_r64(ctx, dst, src, BPF_OP(code));
|
||||
emit_divmod_r64(ctx, dst, src, BPF_OP(code), off);
|
||||
break;
|
||||
/* dst = htole(dst) */
|
||||
/* dst = htobe(dst) */
|
||||
|
|
|
|||
|
|
@ -197,8 +197,43 @@ static void emit_alu_i64(struct jit_context *ctx, u8 dst, s32 imm, u8 op)
|
|||
clobber_reg(ctx, dst);
|
||||
}
|
||||
|
||||
/* ALU division operation (64-bit) */
|
||||
static void emit_div64(struct jit_context *ctx, u8 dst, u8 src, s16 off)
|
||||
{
|
||||
if (cpu_has_mips64r6) {
|
||||
if (off == 1)
|
||||
emit(ctx, ddiv_r6, dst, dst, src);
|
||||
else
|
||||
emit(ctx, ddivu_r6, dst, dst, src);
|
||||
} else {
|
||||
if (off == 1)
|
||||
emit(ctx, ddiv, dst, src);
|
||||
else
|
||||
emit(ctx, ddivu, dst, src);
|
||||
emit(ctx, mflo, dst);
|
||||
}
|
||||
}
|
||||
|
||||
/* ALU modulo operation (64-bit) */
|
||||
static void emit_mod64(struct jit_context *ctx, u8 dst, u8 src, s16 off)
|
||||
{
|
||||
if (cpu_has_mips64r6) {
|
||||
if (off == 1)
|
||||
emit(ctx, dmod, dst, dst, src);
|
||||
else
|
||||
emit(ctx, dmodu, dst, dst, src);
|
||||
} else {
|
||||
if (off == 1)
|
||||
emit(ctx, ddiv, dst, src);
|
||||
else
|
||||
emit(ctx, ddivu, dst, src);
|
||||
emit(ctx, mfhi, dst);
|
||||
}
|
||||
}
|
||||
|
||||
/* ALU register operation (64-bit) */
|
||||
static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op)
|
||||
static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op,
|
||||
s16 off)
|
||||
{
|
||||
switch (BPF_OP(op)) {
|
||||
/* dst = dst << src */
|
||||
|
|
@ -235,25 +270,15 @@ static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op)
|
|||
break;
|
||||
/* dst = dst / src */
|
||||
case BPF_DIV:
|
||||
if (cpu_has_mips64r6) {
|
||||
emit(ctx, ddivu_r6, dst, dst, src);
|
||||
} else {
|
||||
emit(ctx, ddivu, dst, src);
|
||||
emit(ctx, mflo, dst);
|
||||
}
|
||||
emit_div64(ctx, dst, src, off);
|
||||
break;
|
||||
/* dst = dst % src */
|
||||
case BPF_MOD:
|
||||
if (cpu_has_mips64r6) {
|
||||
emit(ctx, dmodu, dst, dst, src);
|
||||
} else {
|
||||
emit(ctx, ddivu, dst, src);
|
||||
emit(ctx, mfhi, dst);
|
||||
}
|
||||
emit_mod64(ctx, dst, src, off);
|
||||
break;
|
||||
default:
|
||||
/* Width-generic operations */
|
||||
emit_alu_r(ctx, dst, src, op);
|
||||
emit_alu_r(ctx, dst, src, op, off);
|
||||
}
|
||||
clobber_reg(ctx, dst);
|
||||
}
|
||||
|
|
@ -674,9 +699,9 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
|
|||
case BPF_ALU | BPF_AND | BPF_K:
|
||||
case BPF_ALU | BPF_XOR | BPF_K:
|
||||
case BPF_ALU | BPF_LSH | BPF_K:
|
||||
if (!valid_alu_i(BPF_OP(code), imm)) {
|
||||
if (!valid_alu_i(BPF_OP(code), imm, off)) {
|
||||
emit_mov_i(ctx, MIPS_R_T4, imm);
|
||||
emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
|
||||
emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code), off);
|
||||
} else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
|
||||
emit_alu_i(ctx, dst, val, alu);
|
||||
}
|
||||
|
|
@ -696,10 +721,10 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
|
|||
case BPF_ALU | BPF_MUL | BPF_K:
|
||||
case BPF_ALU | BPF_DIV | BPF_K:
|
||||
case BPF_ALU | BPF_MOD | BPF_K:
|
||||
if (!valid_alu_i(BPF_OP(code), imm)) {
|
||||
if (!valid_alu_i(BPF_OP(code), imm, off)) {
|
||||
emit_sext(ctx, dst, dst);
|
||||
emit_mov_i(ctx, MIPS_R_T4, imm);
|
||||
emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
|
||||
emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code), off);
|
||||
} else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
|
||||
emit_sext(ctx, dst, dst);
|
||||
emit_alu_i(ctx, dst, val, alu);
|
||||
|
|
@ -714,7 +739,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
|
|||
case BPF_ALU | BPF_OR | BPF_X:
|
||||
case BPF_ALU | BPF_XOR | BPF_X:
|
||||
case BPF_ALU | BPF_LSH | BPF_X:
|
||||
emit_alu_r(ctx, dst, src, BPF_OP(code));
|
||||
emit_alu_r(ctx, dst, src, BPF_OP(code), off);
|
||||
emit_zext_ver(ctx, dst);
|
||||
break;
|
||||
/* dst = dst >> src */
|
||||
|
|
@ -733,7 +758,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
|
|||
case BPF_ALU | BPF_MOD | BPF_X:
|
||||
emit_sext(ctx, dst, dst);
|
||||
emit_sext(ctx, MIPS_R_T4, src);
|
||||
emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
|
||||
emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code), off);
|
||||
emit_zext_ver(ctx, dst);
|
||||
break;
|
||||
/* dst = imm (64-bit) */
|
||||
|
|
@ -770,9 +795,9 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
|
|||
case BPF_ALU64 | BPF_MUL | BPF_K:
|
||||
case BPF_ALU64 | BPF_DIV | BPF_K:
|
||||
case BPF_ALU64 | BPF_MOD | BPF_K:
|
||||
if (!valid_alu_i(BPF_OP(code), imm)) {
|
||||
if (!valid_alu_i(BPF_OP(code), imm, off)) {
|
||||
emit_mov_i(ctx, MIPS_R_T4, imm);
|
||||
emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code));
|
||||
emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code), off);
|
||||
} else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
|
||||
emit_alu_i64(ctx, dst, val, alu);
|
||||
}
|
||||
|
|
@ -799,7 +824,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
|
|||
case BPF_ALU64 | BPF_MUL | BPF_X:
|
||||
case BPF_ALU64 | BPF_DIV | BPF_X:
|
||||
case BPF_ALU64 | BPF_MOD | BPF_X:
|
||||
emit_alu_r64(ctx, dst, src, BPF_OP(code));
|
||||
emit_alu_r64(ctx, dst, src, BPF_OP(code), off);
|
||||
break;
|
||||
/* dst = htole(dst) */
|
||||
/* dst = htobe(dst) */
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user