From: Kumar Kartikeya Dwivedi <memxor@gmail.com> mainline inclusion from mainline-v7.3-rc5 commit 261b61d3735b042ae25634f795c4540be0fc140c category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/20695 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=... -------------------------------- After do_check() returns, the verifier runs several instruction rewrite passes. Some of them patch or remove one instruction at a time. Each operation moves the remaining instruction and auxiliary-data arrays and adjusts all branch offsets, making the overall work quadratic in the program length. A privileged loader can submit 131072 unconditional jumps by zero followed by a valid return. Verification finishes quickly, but bpf_opt_remove_nops() then spends a long time removing each jump separately. Since this post-verification work neither checks for signals nor reschedules, a pending SIGKILL cannot terminate the task until the rewrite finishes. Make bpf_patch_insn_data() and verifier_remove_insns() common cancellation and rescheduling points. These helpers run from BPF_PROG_LOAD process context, and bpf_patch_insn_data() can already sleep while reallocating auxiliary data. Report interrupted constant blinding as -EINTR and propagate it through both JIT paths, including kernels that permit interpreter fallback. Other blinding failures retain the existing fallback behavior. This does not reduce the quadratic cost of the rewrite passes, but it makes the work preemptible and allows a killed loader to be torn down promptly. Fixes: 52875a04f4b2 ("bpf: verifier: remove dead code") Reported-by: Nicholas Carlini <npc@anthropic.com> Suggested-by: Nicholas Carlini <npc@anthropic.com> Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com> Acked-by: Eduard Zingerman <eddyz87@gmail.com> Link: https://patch.msgid.link/20260917233222.2542500-2-memxor@gmail.com Signed-off-by: Eduard Zingerman <eddyz87@gmail.com> Conflicts: arch/arm/net/bpf_jit_32.c arch/arm64/net/bpf_jit_comp.c arch/loongarch/net/bpf_jit.c arch/mips/net/bpf_jit_comp.c arch/parisc/net/bpf_jit_core.c arch/powerpc/net/bpf_jit_comp.c arch/riscv/net/bpf_jit_core.c arch/s390/net/bpf_jit_comp.c arch/sparc/net/bpf_jit_comp_64.c arch/sw_64/net/bpf_jit_comp.c arch/x86/net/bpf_jit_comp.c arch/x86/net/bpf_jit_comp32.c kernel/bpf/core.c kernel/bpf/verifier.c kernel/bpf/fixups.c [generic bpf_jit_blind_constants not merged] Signed-off-by: Pu Lehui <pulehui@huawei.com> --- arch/arm/net/bpf_jit_32.c | 14 ++++++++------ arch/arm64/net/bpf_jit_comp.c | 10 +++++++--- arch/loongarch/net/bpf_jit.c | 10 ++++++---- arch/mips/net/bpf_jit_comp.c | 12 +++++++----- arch/parisc/net/bpf_jit_core.c | 9 ++++++++- arch/powerpc/net/bpf_jit_comp.c | 9 ++++++++- arch/riscv/net/bpf_jit_core.c | 9 ++++++++- arch/s390/net/bpf_jit_comp.c | 9 ++++++--- arch/sparc/net/bpf_jit_comp_64.c | 10 +++++++--- arch/sw_64/net/bpf_jit_comp.c | 10 +++++++--- arch/x86/net/bpf_jit_comp.c | 9 ++++++--- arch/x86/net/bpf_jit_comp32.c | 9 ++++++--- kernel/bpf/core.c | 12 +++++++++++- kernel/bpf/verifier.c | 20 ++++++++++++++++++++ 14 files changed, 115 insertions(+), 37 deletions(-) diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index 8adc45ad1b2a..0ff2a70d8c06 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -1894,14 +1894,16 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) if (!prog->jit_requested) return orig_prog; - /* If constant blinding was enabled and we failed during blinding - * then we must fall back to the interpreter. Otherwise, we save - * the new JITed code. - */ tmp = bpf_jit_blind_constants(prog); - - if (IS_ERR(tmp)) + /* + * Fall back to the interpreter after blinding failures, except when + * the loader was killed. + */ + if (IS_ERR(tmp)) { + if (PTR_ERR(tmp) == -EINTR) + return tmp; return orig_prog; + } if (tmp != prog) { tmp_blinded = true; prog = tmp; diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 971b75b8059f..87ec8dbb43af 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1580,11 +1580,15 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) return orig_prog; tmp = bpf_jit_blind_constants(prog); - /* If blinding was requested and we failed during blinding, - * we must fall back to the interpreter. + /* + * Fall back to the interpreter after blinding failures, except when + * the loader was killed. */ - if (IS_ERR(tmp)) + if (IS_ERR(tmp)) { + if (PTR_ERR(tmp) == -EINTR) + return tmp; return orig_prog; + } if (tmp != prog) { tmp_blinded = true; prog = tmp; diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 123df6a771d7..f44803e66ff4 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -1125,12 +1125,14 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) tmp = bpf_jit_blind_constants(prog); /* - * If blinding was requested and we failed during blinding, - * we must fall back to the interpreter. Otherwise, we save - * the new JITed code. + * Fall back to the interpreter after blinding failures, except when + * the loader was killed. */ - if (IS_ERR(tmp)) + if (IS_ERR(tmp)) { + if (PTR_ERR(tmp) == -EINTR) + return tmp; return orig_prog; + } if (tmp != prog) { tmp_blinded = true; diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c index e355dfca4400..6e5ede70b306 100644 --- a/arch/mips/net/bpf_jit_comp.c +++ b/arch/mips/net/bpf_jit_comp.c @@ -926,14 +926,16 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) */ if (!prog->jit_requested) return orig_prog; + tmp = bpf_jit_blind_constants(prog); /* - * If constant blinding was enabled and we failed during blinding - * then we must fall back to the interpreter. Otherwise, we save - * the new JITed code. + * Fall back to the interpreter after blinding failures, except when + * the loader was killed. */ - tmp = bpf_jit_blind_constants(prog); - if (IS_ERR(tmp)) + if (IS_ERR(tmp)) { + if (PTR_ERR(tmp) == -EINTR) + return tmp; return orig_prog; + } if (tmp != prog) { tmp_blinded = true; prog = tmp; diff --git a/arch/parisc/net/bpf_jit_core.c b/arch/parisc/net/bpf_jit_core.c index 06cbcd6fe87b..80ec6a6c2d49 100644 --- a/arch/parisc/net/bpf_jit_core.c +++ b/arch/parisc/net/bpf_jit_core.c @@ -54,8 +54,15 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) return orig_prog; tmp = bpf_jit_blind_constants(prog); - if (IS_ERR(tmp)) + /* + * Fall back to the interpreter after blinding failures, except when + * the loader was killed. + */ + if (IS_ERR(tmp)) { + if (PTR_ERR(tmp) == -EINTR) + return tmp; return orig_prog; + } if (tmp != prog) { tmp_blinded = true; prog = tmp; diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 57de9d6b0089..ab316a492075 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -75,8 +75,15 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) return org_fp; tmp_fp = bpf_jit_blind_constants(org_fp); - if (IS_ERR(tmp_fp)) + /* + * Fall back to the interpreter after blinding failures, except when + * the loader was killed. + */ + if (IS_ERR(tmp_fp)) { + if (PTR_ERR(tmp_fp) == -EINTR) + return tmp_fp; return org_fp; + } if (tmp_fp != org_fp) { bpf_blinded = true; diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index 7b70ccb7fec3..bbfaaa880d4f 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -54,8 +54,15 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) return orig_prog; tmp = bpf_jit_blind_constants(prog); - if (IS_ERR(tmp)) + /* + * Fall back to the interpreter after blinding failures, except when + * the loader was killed. + */ + if (IS_ERR(tmp)) { + if (PTR_ERR(tmp) == -EINTR) + return tmp; return orig_prog; + } if (tmp != prog) { tmp_blinded = true; prog = tmp; diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 99eae9cf7972..3c171c1c56e7 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -1952,11 +1952,14 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) tmp = bpf_jit_blind_constants(fp); /* - * If blinding was requested and we failed during blinding, - * we must fall back to the interpreter. + * Fall back to the interpreter after blinding failures, except when + * the loader was killed. */ - if (IS_ERR(tmp)) + if (IS_ERR(tmp)) { + if (PTR_ERR(tmp) == -EINTR) + return tmp; return orig_fp; + } if (tmp != fp) { tmp_blinded = true; fp = tmp; diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c index 73bf0aea8baf..0fe2df433082 100644 --- a/arch/sparc/net/bpf_jit_comp_64.c +++ b/arch/sparc/net/bpf_jit_comp_64.c @@ -1493,11 +1493,15 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) return orig_prog; tmp = bpf_jit_blind_constants(prog); - /* If blinding was requested and we failed during blinding, - * we must fall back to the interpreter. + /* + * Fall back to the interpreter after blinding failures, except when + * the loader was killed. */ - if (IS_ERR(tmp)) + if (IS_ERR(tmp)) { + if (PTR_ERR(tmp) == -EINTR) + return tmp; return orig_prog; + } if (tmp != prog) { tmp_blinded = true; prog = tmp; diff --git a/arch/sw_64/net/bpf_jit_comp.c b/arch/sw_64/net/bpf_jit_comp.c index 057461689832..e154adc33073 100644 --- a/arch/sw_64/net/bpf_jit_comp.c +++ b/arch/sw_64/net/bpf_jit_comp.c @@ -1969,11 +1969,15 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) return orig_prog; tmp = bpf_jit_blind_constants(prog); - /* If blinding was requested and we failed during blinding, - * we must fall back to the interpreter. + /* + * Fall back to the interpreter after blinding failures, except when + * the loader was killed. */ - if (IS_ERR(tmp)) + if (IS_ERR(tmp)) { + if (PTR_ERR(tmp) == -EINTR) + return tmp; return orig_prog; + } if (tmp != prog) { tmp_blinded = true; prog = tmp; diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index ad50d9a270fc..46fb1c77b370 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2986,11 +2986,14 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) tmp = bpf_jit_blind_constants(prog); /* - * If blinding was requested and we failed during blinding, - * we must fall back to the interpreter. + * Fall back to the interpreter after blinding failures, except when + * the loader was killed. */ - if (IS_ERR(tmp)) + if (IS_ERR(tmp)) { + if (PTR_ERR(tmp) == -EINTR) + return tmp; return orig_prog; + } if (tmp != prog) { tmp_blinded = true; prog = tmp; diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c index f2fc8c38629b..d45644254cba 100644 --- a/arch/x86/net/bpf_jit_comp32.c +++ b/arch/x86/net/bpf_jit_comp32.c @@ -2535,11 +2535,14 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) tmp = bpf_jit_blind_constants(prog); /* - * If blinding was requested and we failed during blinding, - * we must fall back to the interpreter. + * Fall back to the interpreter after blinding failures, except when + * the loader was killed. */ - if (IS_ERR(tmp)) + if (IS_ERR(tmp)) { + if (PTR_ERR(tmp) == -EINTR) + return tmp; return orig_prog; + } if (tmp != prog) { tmp_blinded = true; prog = tmp; diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index bf493b8ae8fd..06748b956b5c 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -19,6 +19,7 @@ #include <uapi/linux/btf.h> #include <linux/filter.h> +#include <linux/sched/signal.h> #include <linux/skbuff.h> #include <linux/vmalloc.h> #include <linux/random.h> @@ -1534,6 +1535,8 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog) * fix it up here on error. */ bpf_jit_prog_release_other(prog, clone); + if (fatal_signal_pending(current)) + return ERR_PTR(-EINTR); return tmp; } @@ -2466,6 +2469,8 @@ static void bpf_prog_select_func(struct bpf_prog *fp) */ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) { + struct bpf_prog *jit_prog; + /* In case of BPF to BPF calls, verifier did all the prep * work with regards to JITing, etc. */ @@ -2491,7 +2496,12 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) if (*err) return fp; - fp = bpf_int_jit_compile(fp); + jit_prog = bpf_int_jit_compile(fp); + if (IS_ERR(jit_prog)) { + *err = PTR_ERR(jit_prog); + return fp; + } + fp = jit_prog; bpf_prog_jit_attempt_done(fp); if (!fp->jited && jit_needed) { *err = -ENOTSUPP; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7f3bb5bf8a57..66cfb1d955c1 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -19,6 +19,7 @@ #include <linux/bsearch.h> #include <linux/sort.h> #include <linux/perf_event.h> +#include <linux/sched/signal.h> #include <linux/ctype.h> #include <linux/error-injection.h> #include <linux/bpf_lsm.h> @@ -18452,12 +18453,28 @@ static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) } } +/* + * Some post-verification instruction rewriting passes require an + * O(prog->len) operation per instruction. Keep their shared primitives + * killable and preemptible. + */ +static bool bpf_rewrite_must_abort(void) +{ + if (fatal_signal_pending(current)) + return true; + cond_resched(); + return false; +} + static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, const struct bpf_insn *patch, u32 len) { struct bpf_prog *new_prog; struct bpf_insn_aux_data *new_data = NULL; + if (bpf_rewrite_must_abort()) + return NULL; + if (len > 1) { new_data = vzalloc(array_size(env->prog->len + len - 1, sizeof(struct bpf_insn_aux_data))); @@ -18606,6 +18623,9 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) unsigned int orig_prog_len = env->prog->len; int err; + if (bpf_rewrite_must_abort()) + return -EINTR; + if (bpf_prog_is_offloaded(env->prog->aux)) bpf_prog_offload_remove_insns(env, off, cnt); -- 2.34.1