diff options
| -rw-r--r-- | arch/arm64/net/bpf_jit_comp.c | 71 | ||||
| -rw-r--r-- | include/linux/bpf.h | 7 |
2 files changed, 68 insertions, 10 deletions
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 0c4d44bcfbf4..2dc5037694ba 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2510,6 +2510,12 @@ static bool is_struct_ops_tramp(const struct bpf_tramp_links *fentry_links) fentry_links->links[0]->link.type == BPF_LINK_TYPE_STRUCT_OPS; } +static void store_func_meta(struct jit_ctx *ctx, u64 func_meta, int func_meta_off) +{ + emit_a64_mov_i64(A64_R(10), func_meta, ctx); + emit(A64_STR64I(A64_R(10), A64_SP, func_meta_off), ctx); +} + /* Based on the x86's implementation of arch_prepare_bpf_trampoline(). * * bpf prog and function entry before bpf trampoline hooked: @@ -2533,7 +2539,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, int regs_off; int retval_off; int bargs_off; - int nfuncargs_off; + int func_meta_off; int ip_off; int run_ctx_off; int oargs_off; @@ -2544,6 +2550,9 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, bool save_ret; __le32 **branches = NULL; bool is_struct_ops = is_struct_ops_tramp(fentry); + int cookie_off, cookie_cnt, cookie_bargs_off; + int fsession_cnt = bpf_fsession_cnt(tlinks); + u64 func_meta; /* trampoline stack layout: * [ parent ip ] @@ -2562,10 +2571,14 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, * [ ... ] * SP + bargs_off [ arg reg 1 ] for bpf * - * SP + nfuncargs_off [ arg regs count ] + * SP + func_meta_off [ regs count, etc ] * * SP + ip_off [ traced function ] BPF_TRAMP_F_IP_ARG flag * + * [ stack cookie N ] + * [ ... ] + * SP + cookie_off [ stack cookie 1 ] + * * SP + run_ctx_off [ bpf_tramp_run_ctx ] * * [ stack arg N ] @@ -2582,13 +2595,18 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, /* room for bpf_tramp_run_ctx */ stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8); + cookie_off = stack_size; + /* room for session cookies */ + cookie_cnt = bpf_fsession_cookie_cnt(tlinks); + stack_size += cookie_cnt * 8; + ip_off = stack_size; /* room for IP address argument */ if (flags & BPF_TRAMP_F_IP_ARG) stack_size += 8; - nfuncargs_off = stack_size; - /* room for args count */ + func_meta_off = stack_size; + /* room for function metadata, such as regs count */ stack_size += 8; bargs_off = stack_size; @@ -2646,9 +2664,9 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, emit(A64_STR64I(A64_R(10), A64_SP, ip_off), ctx); } - /* save arg regs count*/ - emit(A64_MOVZ(1, A64_R(10), nfuncargs, 0), ctx); - emit(A64_STR64I(A64_R(10), A64_SP, nfuncargs_off), ctx); + /* save function metadata */ + func_meta = nfuncargs; + store_func_meta(ctx, func_meta, func_meta_off); /* save args for bpf */ save_args(ctx, bargs_off, oargs_off, m, a, false); @@ -2666,10 +2684,27 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, emit_call((const u64)__bpf_tramp_enter, ctx); } - for (i = 0; i < fentry->nr_links; i++) + if (fsession_cnt) { + /* clear all the session cookies' value */ + emit(A64_MOVZ(1, A64_R(10), 0, 0), ctx); + for (int i = 0; i < cookie_cnt; i++) + emit(A64_STR64I(A64_R(10), A64_SP, cookie_off + 8 * i), ctx); + /* clear the return value to make sure fentry always gets 0 */ + emit(A64_STR64I(A64_R(10), A64_SP, retval_off), ctx); + } + + cookie_bargs_off = (bargs_off - cookie_off) / 8; + for (i = 0; i < fentry->nr_links; i++) { + if (bpf_prog_calls_session_cookie(fentry->links[i])) { + u64 meta = func_meta | (cookie_bargs_off << BPF_TRAMP_COOKIE_INDEX_SHIFT); + + store_func_meta(ctx, meta, func_meta_off); + cookie_bargs_off--; + } invoke_bpf_prog(ctx, fentry->links[i], bargs_off, retval_off, run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET); + } if (fmod_ret->nr_links) { branches = kcalloc(fmod_ret->nr_links, sizeof(__le32 *), @@ -2701,9 +2736,22 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, *branches[i] = cpu_to_le32(A64_CBNZ(1, A64_R(10), offset)); } - for (i = 0; i < fexit->nr_links; i++) + /* set the "is_return" flag for fsession */ + func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT); + if (fsession_cnt) + store_func_meta(ctx, func_meta, func_meta_off); + + cookie_bargs_off = (bargs_off - cookie_off) / 8; + for (i = 0; i < fexit->nr_links; i++) { + if (bpf_prog_calls_session_cookie(fexit->links[i])) { + u64 meta = func_meta | (cookie_bargs_off << BPF_TRAMP_COOKIE_INDEX_SHIFT); + + store_func_meta(ctx, meta, func_meta_off); + cookie_bargs_off--; + } invoke_bpf_prog(ctx, fexit->links[i], bargs_off, retval_off, run_ctx_off, false); + } if (flags & BPF_TRAMP_F_CALL_ORIG) { im->ip_epilogue = ctx->ro_image + ctx->idx; @@ -2753,6 +2801,11 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, return ctx->idx; } +bool bpf_jit_supports_fsession(void) +{ + return true; +} + int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks, void *func_addr) { diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 3b0ceb759075..cd9b96434904 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2196,13 +2196,18 @@ static inline int bpf_fsession_cnt(struct bpf_tramp_links *links) return cnt; } +static inline bool bpf_prog_calls_session_cookie(struct bpf_tramp_link *link) +{ + return link->link.prog->call_session_cookie; +} + static inline int bpf_fsession_cookie_cnt(struct bpf_tramp_links *links) { struct bpf_tramp_links fentries = links[BPF_TRAMP_FENTRY]; int cnt = 0; for (int i = 0; i < links[BPF_TRAMP_FENTRY].nr_links; i++) { - if (fentries.links[i]->link.prog->call_session_cookie) + if (bpf_prog_calls_session_cookie(fentries.links[i])) cnt++; } |