github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-29 17:43:52 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

bpf: use realloc in bpf_patch_insn_data

Browse Source 
Avoid excessive vzalloc/vfree calls when patching instructions in
do_misc_fixups(). bpf_patch_insn_data() uses vzalloc to allocate new
memory for env->insn_aux_data for each patch as follows:

  struct bpf_prog *bpf_patch_insn_data(env, ...)
  {
    ...
    new_data = vzalloc(... O(program size) ...);
    ...
    adjust_insn_aux_data(env, new_data, ...);
    ...
  }

  void adjust_insn_aux_data(env, new_data, ...)
  {
    ...
    memcpy(new_data, env->insn_aux_data);
    vfree(env->insn_aux_data);
    env->insn_aux_data = new_data;
    ...
  }

The vzalloc/vfree pair is hot in perf report collected for e.g.
pyperf180 test case. It can be replaced with a call to vrealloc in
order to reduce the number of actual memory allocations.

This is a stop-gap solution, as bpf_patch_insn_data is still hot in
the profile. More comprehansive solutions had been discussed before
e.g. as in [1].

[1] https://lore.kernel.org/bpf/CAEf4BzY_E8MSL4mD0UPuuiDcbJhh9e2xQo2=5w+ppRWWiYSGvQ@mail.gmail.com/

Suggested-by: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Signed-off-by: Eduard Zingerman <eddyz87@gmail.com>
Acked-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Tested-by: Anton Protopopov <a.s.protopopov@gmail.com>
Link: https://lore.kernel.org/r/20250807010205.3210608-3-eddyz87@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Eduard Zingerman 2025-08-06 18:02:05 -07:00 committed by Alexei Starovoitov
parent cb070a8156
commit 77620d1267
1 changed files with 15 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
kernel/bpf/verifier.c
View File

@ -20699,12 +20699,11 @@ static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)
* [0, off) and [off, end) to new locations, so the patched range stays zero * [0, off) and [off, end) to new locations, so the patched range stays zero
*/ */
static void adjust_insn_aux_data(struct bpf_verifier_env *env, static void adjust_insn_aux_data(struct bpf_verifier_env *env,
struct bpf_insn_aux_data *new_data,
struct bpf_prog *new_prog, u32 off, u32 cnt) struct bpf_prog *new_prog, u32 off, u32 cnt)
{ {
struct bpf_insn_aux_data *old_data = env->insn_aux_data; struct bpf_insn_aux_data *data = env->insn_aux_data;
struct bpf_insn *insn = new_prog->insnsi; struct bpf_insn *insn = new_prog->insnsi;
u32 old_seen = old_data[off].seen; u32 old_seen = data[off].seen;
u32 prog_len; u32 prog_len;
int i; int i;
@ -20712,22 +20711,20 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env,
* (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the
* original insn at old prog. * original insn at old prog.
*/ */
old_data[off].zext_dst = insn_has_def32(insn + off + cnt - 1); data[off].zext_dst = insn_has_def32(insn + off + cnt - 1);
if (cnt == 1) if (cnt == 1)
return; return;
prog_len = new_prog->len; prog_len = new_prog->len;
memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off); memmove(data + off + cnt - 1, data + off,
memcpy(new_data + off + cnt - 1, old_data + off, sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1));
sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); memset(data + off, 0, sizeof(struct bpf_insn_aux_data) * (cnt - 1));
for (i = off; i < off + cnt - 1; i++) { for (i = off; i < off + cnt - 1; i++) {
/* Expand insni[off]'s seen count to the patched range. */ /* Expand insni[off]'s seen count to the patched range. */
new_data[i].seen = old_seen; data[i].seen = old_seen;
new_data[i].zext_dst = insn_has_def32(insn + i); data[i].zext_dst = insn_has_def32(insn + i);
} }
env->insn_aux_data = new_data;
vfree(old_data);
} }
static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
@ -20765,10 +20762,14 @@ static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 of
struct bpf_insn_aux_data *new_data = NULL; struct bpf_insn_aux_data *new_data = NULL;
if (len > 1) { if (len > 1) {
new_data = vzalloc(array_size(env->prog->len + len - 1, new_data = vrealloc(env->insn_aux_data,
sizeof(struct bpf_insn_aux_data))); array_size(env->prog->len + len - 1,
sizeof(struct bpf_insn_aux_data)),
GFP_KERNEL_ACCOUNT | __GFP_ZERO);
if (!new_data) if (!new_data)
return NULL; return NULL;
env->insn_aux_data = new_data;
} }
new_prog = bpf_patch_insn_single(env->prog, off, patch, len); new_prog = bpf_patch_insn_single(env->prog, off, patch, len);
@ -20780,7 +20781,7 @@ static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 of
vfree(new_data); vfree(new_data);
return NULL; return NULL;
} }
adjust_insn_aux_data(env, new_data, new_prog, off, len); adjust_insn_aux_data(env, new_prog, off, len);
adjust_subprog_starts(env, off, len); adjust_subprog_starts(env, off, len);
adjust_poke_descs(new_prog, off, len); adjust_poke_descs(new_prog, off, len);
return new_prog; return new_prog;