From: Florent Revest revest@chromium.org
mainline inclusion from mainline-v6.5-rc1 commit 90564f1e3dd66052af29e2c29cba3dc98b6604d0 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9FGRE
Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...
--------------------------------
This extends the BPF trampoline JIT to support attachment to functions that take small structures (up to 128bit) as argument. This is trivially achieved by saving/restoring a number of "argument registers" rather than a number of arguments.
The AAPCS64 section 6.8.2 describes the parameter passing ABI. "Composite types" (like C structs) below 16 bytes (as enforced by the BPF verifier) are provided as part of the 8 argument registers as explained in the section C.12.
Signed-off-by: Florent Revest revest@chromium.org Signed-off-by: Daniel Borkmann daniel@iogearbox.net Acked-by: Yonghong Song yhs@fb.com Acked-by: Xu Kuohai xukuohai@huawei.com Link: https://lore.kernel.org/bpf/20230511140507.514888-1-revest@chromium.org Conflicts: arch/arm64/net/bpf_jit_comp.c Signed-off-by: Pu Lehui pulehui@huawei.com --- arch/arm64/net/bpf_jit_comp.c | 43 ++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 21 deletions(-)
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 7c4f78e79436..5d93efdf65b4 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1527,21 +1527,21 @@ static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_progs *tp, } }
-static void save_args(struct jit_ctx *ctx, int args_off, int nargs) +static void save_args(struct jit_ctx *ctx, int args_off, int nregs) { int i;
- for (i = 0; i < nargs; i++) { + for (i = 0; i < nregs; i++) { emit(A64_STR64I(i, A64_SP, args_off), ctx); args_off += 8; } }
-static void restore_args(struct jit_ctx *ctx, int args_off, int nargs) +static void restore_args(struct jit_ctx *ctx, int args_off, int nregs) { int i;
- for (i = 0; i < nargs; i++) { + for (i = 0; i < nregs; i++) { emit(A64_LDR64I(i, A64_SP, args_off), ctx); args_off += 8; } @@ -1560,7 +1560,7 @@ static void restore_args(struct jit_ctx *ctx, int args_off, int nargs) */ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, struct bpf_tramp_progs *tprogs, void *orig_call, - int nargs, u32 flags) + int nregs, u32 flags) { int i; int stack_size; @@ -1587,16 +1587,16 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, * SP + retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or * BPF_TRAMP_F_RET_FENTRY_RET * - * [ argN ] + * [ arg reg N ] * [ ... ] - * SP + args_off [ arg1 ] + * SP + args_off [ arg reg 1 ] */
stack_size = 0;
args_off = stack_size; /* room for args */ - stack_size += nargs * 8; + stack_size += nregs * 8;
/* room for return value */ retval_off = stack_size; @@ -1633,8 +1633,8 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, /* allocate stack space */ emit(A64_SUB_I(1, A64_SP, A64_SP, stack_size), ctx);
- /* save args */ - save_args(ctx, args_off, nargs); + /* save arg regs */ + save_args(ctx, args_off, nregs);
/* save callee saved registers */ emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx); @@ -1658,7 +1658,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, }
if (flags & BPF_TRAMP_F_CALL_ORIG) { - restore_args(ctx, args_off, nargs); + restore_args(ctx, args_off, nregs); /* call original func */ emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx); emit(A64_BLR(A64_R(10)), ctx); @@ -1685,7 +1685,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, }
if (flags & BPF_TRAMP_F_RESTORE_REGS) - restore_args(ctx, args_off, nargs); + restore_args(ctx, args_off, nregs);
/* restore callee saved register x19 */ emit(A64_LDR64I(A64_R(19), A64_SP, regs_off), ctx); @@ -1725,24 +1725,25 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *orig_call) { int i, ret; - int nargs = m->nr_args; + int nregs = m->nr_args; int max_insns = ((long)image_end - (long)image) / AARCH64_INSN_SIZE; struct jit_ctx ctx = { .image = NULL, .idx = 0, };
- /* the first 8 arguments are passed by registers */ - if (nargs > 8) - return -ENOTSUPP; - - /* don't support struct argument */ + /* extra registers needed for struct argument */ for (i = 0; i < MAX_BPF_FUNC_ARGS; i++) { + /* The arg_size is at most 16 bytes, enforced by the verifier. */ if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) - return -ENOTSUPP; + nregs += (m->arg_size[i] + 7) / 8 - 1; }
- ret = prepare_trampoline(&ctx, im, tprogs, orig_call, nargs, flags); + /* the first 8 registers are used for arguments */ + if (nregs > 8) + return -ENOTSUPP; + + ret = prepare_trampoline(&ctx, im, tprogs, orig_call, nregs, flags); if (ret < 0) return ret;
@@ -1753,7 +1754,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, ctx.idx = 0;
jit_fill_hole(image, (unsigned int)(image_end - image)); - ret = prepare_trampoline(&ctx, im, tprogs, orig_call, nargs, flags); + ret = prepare_trampoline(&ctx, im, tprogs, orig_call, nregs, flags);
if (ret > 0 && validate_code(&ctx) < 0) ret = -EINVAL;