From b6ec79518ef0c84d1ed0f76b8af9592a75eb29b6 Mon Sep 17 00:00:00 2001 From: Hou Tao Date: Thu, 27 Jan 2022 16:32:40 +0800 Subject: bpf, x86: Remove unnecessary handling of BPF_SUB atomic op According to the LLVM commit (https://reviews.llvm.org/D72184), sync_fetch_and_sub() is implemented as a negation followed by sync_fetch_and_add(), so there will be no BPF_SUB op, thus just remove it. BPF_SUB is also rejected by the verifier anyway. Signed-off-by: Hou Tao Signed-off-by: Daniel Borkmann Acked-by: Brendan Jackman Link: https://lore.kernel.org/bpf/20220127083240.1425481-1-houtao1@huawei.com --- arch/x86/net/bpf_jit_comp.c | 1 - 1 file changed, 1 deletion(-) (limited to 'arch') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 2b1e266ff95c..36f6fc3e6e69 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -787,7 +787,6 @@ static int emit_atomic(u8 **pprog, u8 atomic_op, /* emit opcode */ switch (atomic_op) { case BPF_ADD: - case BPF_SUB: case BPF_AND: case BPF_OR: case BPF_XOR: -- cgit v1.2.3 From e461bd6f43f4e568f7436a8b6bc21c4ce6914c36 Mon Sep 17 00:00:00 2001 From: Robert Hancock Date: Thu, 27 Jan 2022 10:37:36 -0600 Subject: arm64: dts: zynqmp: Added GEM reset definitions The Cadence GEM/MACB driver now utilizes the platform-level reset on the ZynqMP platform. Add reset definitions to the ZynqMP platform device tree to allow this to be used. Signed-off-by: Robert Hancock Signed-off-by: David S. Miller --- arch/arm64/boot/dts/xilinx/zynqmp.dtsi | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'arch') diff --git a/arch/arm64/boot/dts/xilinx/zynqmp.dtsi b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi index 74e66443e4ce..9bec3ba20c69 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp.dtsi +++ b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi @@ -512,6 +512,8 @@ #stream-id-cells = <1>; iommus = <&smmu 0x874>; power-domains = <&zynqmp_firmware PD_ETH_0>; + resets = <&zynqmp_reset ZYNQMP_RESET_GEM0>; + reset-names = "gem0_rst"; }; gem1: ethernet@ff0c0000 { @@ -526,6 +528,8 @@ #stream-id-cells = <1>; iommus = <&smmu 0x875>; power-domains = <&zynqmp_firmware PD_ETH_1>; + resets = <&zynqmp_reset ZYNQMP_RESET_GEM1>; + reset-names = "gem1_rst"; }; gem2: ethernet@ff0d0000 { @@ -540,6 +544,8 @@ #stream-id-cells = <1>; iommus = <&smmu 0x876>; power-domains = <&zynqmp_firmware PD_ETH_2>; + resets = <&zynqmp_reset ZYNQMP_RESET_GEM2>; + reset-names = "gem2_rst"; }; gem3: ethernet@ff0e0000 { @@ -554,6 +560,8 @@ #stream-id-cells = <1>; iommus = <&smmu 0x877>; power-domains = <&zynqmp_firmware PD_ETH_3>; + resets = <&zynqmp_reset ZYNQMP_RESET_GEM3>; + reset-names = "gem3_rst"; }; gpio: gpio@ff0a0000 { -- cgit v1.2.3 From 26859240e4ee701e0379f08634957adaff67e43a Mon Sep 17 00:00:00 2001 From: Akhmat Karakotov Date: Mon, 31 Jan 2022 16:31:22 +0300 Subject: txhash: Add socket option to control TX hash rethink behavior Add the SO_TXREHASH socket option to control hash rethink behavior per socket. When default mode is set, sockets disable rehash at initialization and use sysctl option when entering listen state. setsockopt() overrides default behavior. Signed-off-by: Akhmat Karakotov Reviewed-by: Eric Dumazet Signed-off-by: David S. Miller --- arch/alpha/include/uapi/asm/socket.h | 2 ++ arch/mips/include/uapi/asm/socket.h | 2 ++ arch/parisc/include/uapi/asm/socket.h | 2 ++ arch/sparc/include/uapi/asm/socket.h | 2 ++ include/net/sock.h | 12 +++--------- include/uapi/asm-generic/socket.h | 2 ++ include/uapi/linux/socket.h | 1 + net/core/sock.c | 13 +++++++++++++ net/ipv4/inet_connection_sock.c | 3 +++ 9 files changed, 30 insertions(+), 9 deletions(-) (limited to 'arch') diff --git a/arch/alpha/include/uapi/asm/socket.h b/arch/alpha/include/uapi/asm/socket.h index 284d28755b8d..7d81535893af 100644 --- a/arch/alpha/include/uapi/asm/socket.h +++ b/arch/alpha/include/uapi/asm/socket.h @@ -133,6 +133,8 @@ #define SO_RESERVE_MEM 73 +#define SO_TXREHASH 74 + #if !defined(__KERNEL__) #if __BITS_PER_LONG == 64 diff --git a/arch/mips/include/uapi/asm/socket.h b/arch/mips/include/uapi/asm/socket.h index 24e0efb360f6..1d55e57b8466 100644 --- a/arch/mips/include/uapi/asm/socket.h +++ b/arch/mips/include/uapi/asm/socket.h @@ -144,6 +144,8 @@ #define SO_RESERVE_MEM 73 +#define SO_TXREHASH 74 + #if !defined(__KERNEL__) #if __BITS_PER_LONG == 64 diff --git a/arch/parisc/include/uapi/asm/socket.h b/arch/parisc/include/uapi/asm/socket.h index 845ddc63c882..654061e0964e 100644 --- a/arch/parisc/include/uapi/asm/socket.h +++ b/arch/parisc/include/uapi/asm/socket.h @@ -125,6 +125,8 @@ #define SO_RESERVE_MEM 0x4047 +#define SO_TXREHASH 0x4048 + #if !defined(__KERNEL__) #if __BITS_PER_LONG == 64 diff --git a/arch/sparc/include/uapi/asm/socket.h b/arch/sparc/include/uapi/asm/socket.h index 2672dd03faf3..666f81e617ea 100644 --- a/arch/sparc/include/uapi/asm/socket.h +++ b/arch/sparc/include/uapi/asm/socket.h @@ -126,6 +126,8 @@ #define SO_RESERVE_MEM 0x0052 +#define SO_TXREHASH 0x0053 + #if !defined(__KERNEL__) diff --git a/include/net/sock.h b/include/net/sock.h index 0540e1b2aeb0..d6c13f0fba40 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -316,6 +316,7 @@ struct sk_filter; * @sk_rcvtimeo: %SO_RCVTIMEO setting * @sk_sndtimeo: %SO_SNDTIMEO setting * @sk_txhash: computed flow hash for use on transmit + * @sk_txrehash: enable TX hash rethink * @sk_filter: socket filtering instructions * @sk_timer: sock cleanup timer * @sk_stamp: time stamp of last packet received @@ -491,6 +492,7 @@ struct sock { u32 sk_ack_backlog; u32 sk_max_ack_backlog; kuid_t sk_uid; + u8 sk_txrehash; #ifdef CONFIG_NET_RX_BUSY_POLL u8 sk_prefer_busy_poll; u16 sk_busy_poll_budget; @@ -2066,18 +2068,10 @@ static inline void sk_set_txhash(struct sock *sk) static inline bool sk_rethink_txhash(struct sock *sk) { - u8 rehash; - - if (!sk->sk_txhash) - return false; - - rehash = READ_ONCE(sock_net(sk)->core.sysctl_txrehash); - - if (rehash) { + if (sk->sk_txhash && sk->sk_txrehash == SOCK_TXREHASH_ENABLED) { sk_set_txhash(sk); return true; } - return false; } diff --git a/include/uapi/asm-generic/socket.h b/include/uapi/asm-generic/socket.h index c77a1313b3b0..467ca2f28760 100644 --- a/include/uapi/asm-generic/socket.h +++ b/include/uapi/asm-generic/socket.h @@ -128,6 +128,8 @@ #define SO_RESERVE_MEM 73 +#define SO_TXREHASH 74 + #if !defined(__KERNEL__) #if __BITS_PER_LONG == 64 || (defined(__x86_64__) && defined(__ILP32__)) diff --git a/include/uapi/linux/socket.h b/include/uapi/linux/socket.h index 0accd6102ece..51d6bb2f6765 100644 --- a/include/uapi/linux/socket.h +++ b/include/uapi/linux/socket.h @@ -31,6 +31,7 @@ struct __kernel_sockaddr_storage { #define SOCK_BUF_LOCK_MASK (SOCK_SNDBUF_LOCK | SOCK_RCVBUF_LOCK) +#define SOCK_TXREHASH_DEFAULT ((u8)-1) #define SOCK_TXREHASH_DISABLED 0 #define SOCK_TXREHASH_ENABLED 1 diff --git a/net/core/sock.c b/net/core/sock.c index cccf21f3618d..5e711b42898f 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -1447,6 +1447,14 @@ set_sndbuf: break; } + case SO_TXREHASH: + if (val < -1 || val > 1) { + ret = -EINVAL; + break; + } + sk->sk_txrehash = (u8)val; + break; + default: ret = -ENOPROTOOPT; break; @@ -1834,6 +1842,10 @@ int sock_getsockopt(struct socket *sock, int level, int optname, v.val = sk->sk_reserved_mem; break; + case SO_TXREHASH: + v.val = sk->sk_txrehash; + break; + default: /* We implement the SO_SNDLOWAT etc to not be settable * (1003.1g 7). @@ -3279,6 +3291,7 @@ void sock_init_data(struct socket *sock, struct sock *sk) sk->sk_pacing_rate = ~0UL; WRITE_ONCE(sk->sk_pacing_shift, 10); sk->sk_incoming_cpu = -1; + sk->sk_txrehash = SOCK_TXREHASH_DEFAULT; sk_rx_queue_clear(sk); /* diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index fc2a985f6064..b81fb13fc5f4 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -1046,6 +1046,9 @@ int inet_csk_listen_start(struct sock *sk) sk->sk_ack_backlog = 0; inet_csk_delack_init(sk); + if (sk->sk_txrehash == SOCK_TXREHASH_DEFAULT) + sk->sk_txrehash = READ_ONCE(sock_net(sk)->core.sysctl_txrehash); + /* There is race window here: we announce ourselves listening, * but this transition is still not validated by get_port(). * It is OK, because this socket enters to hash table only -- cgit v1.2.3 From b5e975d256dbfebd62413eb04fbff6803f02a43c Mon Sep 17 00:00:00 2001 From: Hou Tao Date: Sun, 30 Jan 2022 17:29:15 +0800 Subject: bpf, arm64: Enable kfunc call Since commit b2eed9b58811 ("arm64/kernel: kaslr: reduce module randomization range to 2 GB"), for arm64 whether KASLR is enabled or not, the module is placed within 2GB of the kernel region, so s32 in bpf_kfunc_desc is sufficient to represente the offset of module function relative to __bpf_call_base. The only thing needed is to override bpf_jit_supports_kfunc_call(). Signed-off-by: Hou Tao Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20220130092917.14544-2-hotforest@gmail.com --- arch/arm64/net/bpf_jit_comp.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'arch') diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index e96d4d87291f..74f9a9b6a053 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1143,6 +1143,11 @@ out: return prog; } +bool bpf_jit_supports_kfunc_call(void) +{ + return true; +} + u64 bpf_jit_alloc_exec_limit(void) { return VMALLOC_END - VMALLOC_START; -- cgit v1.2.3 From fac54e2bfb5be2b0bbf115fe80d45f59fd773048 Mon Sep 17 00:00:00 2001 From: Song Liu Date: Fri, 4 Feb 2022 10:57:34 -0800 Subject: x86/Kconfig: Select HAVE_ARCH_HUGE_VMALLOC with HAVE_ARCH_HUGE_VMAP This enables module_alloc() to allocate huge page for 2MB+ requests. To check the difference of this change, we need enable config CONFIG_PTDUMP_DEBUGFS, and call module_alloc(2MB). Before the change, /sys/kernel/debug/page_tables/kernel shows pte for this map. With the change, /sys/kernel/debug/page_tables/ show pmd for thie map. Signed-off-by: Song Liu Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20220204185742.271030-2-song@kernel.org --- arch/x86/Kconfig | 1 + 1 file changed, 1 insertion(+) (limited to 'arch') diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 6fddb63271d9..e0e0d00cf103 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -159,6 +159,7 @@ config X86 select HAVE_ALIGNED_STRUCT_PAGE if SLUB select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_HUGE_VMAP if X86_64 || X86_PAE + select HAVE_ARCH_HUGE_VMALLOC if HAVE_ARCH_HUGE_VMAP select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_JUMP_LABEL_RELATIVE select HAVE_ARCH_KASAN if X86_64 -- cgit v1.2.3 From 0e06b40371682b6b70ed0302a7baec0698ada95c Mon Sep 17 00:00:00 2001 From: Song Liu Date: Fri, 4 Feb 2022 10:57:38 -0800 Subject: x86/alternative: Introduce text_poke_copy This will be used by BPF jit compiler to dump JITed binary to a RX huge page, and thus allow multiple BPF programs sharing the a huge (2MB) page. Signed-off-by: Song Liu Signed-off-by: Alexei Starovoitov Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/bpf/20220204185742.271030-6-song@kernel.org --- arch/x86/include/asm/text-patching.h | 1 + arch/x86/kernel/alternative.c | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) (limited to 'arch') diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h index b7421780e4e9..4cc18ba1b75e 100644 --- a/arch/x86/include/asm/text-patching.h +++ b/arch/x86/include/asm/text-patching.h @@ -44,6 +44,7 @@ extern void text_poke_early(void *addr, const void *opcode, size_t len); extern void *text_poke(void *addr, const void *opcode, size_t len); extern void text_poke_sync(void); extern void *text_poke_kgdb(void *addr, const void *opcode, size_t len); +extern void *text_poke_copy(void *addr, const void *opcode, size_t len); extern int poke_int3_handler(struct pt_regs *regs); extern void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate); diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 5007c3ffe96f..018b61febf0e 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -1102,6 +1102,40 @@ void *text_poke_kgdb(void *addr, const void *opcode, size_t len) return __text_poke(addr, opcode, len); } +/** + * text_poke_copy - Copy instructions into (an unused part of) RX memory + * @addr: address to modify + * @opcode: source of the copy + * @len: length to copy, could be more than 2x PAGE_SIZE + * + * Not safe against concurrent execution; useful for JITs to dump + * new code blocks into unused regions of RX memory. Can be used in + * conjunction with synchronize_rcu_tasks() to wait for existing + * execution to quiesce after having made sure no existing functions + * pointers are live. + */ +void *text_poke_copy(void *addr, const void *opcode, size_t len) +{ + unsigned long start = (unsigned long)addr; + size_t patched = 0; + + if (WARN_ON_ONCE(core_kernel_text(start))) + return NULL; + + mutex_lock(&text_mutex); + while (patched < len) { + unsigned long ptr = start + patched; + size_t s; + + s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched); + + __text_poke((void *)ptr, opcode + patched, s); + patched += s; + } + mutex_unlock(&text_mutex); + return addr; +} + static void do_sync_core(void *info) { sync_core(); -- cgit v1.2.3 From ebc1415d9b4f043cef5a1fb002ec316e32167e7a Mon Sep 17 00:00:00 2001 From: Song Liu Date: Fri, 4 Feb 2022 10:57:39 -0800 Subject: bpf: Introduce bpf_arch_text_copy This will be used to copy JITed text to RO protected module memory. On x86, bpf_arch_text_copy is implemented with text_poke_copy. bpf_arch_text_copy returns pointer to dst on success, and ERR_PTR(errno) on errors. Signed-off-by: Song Liu Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20220204185742.271030-7-song@kernel.org --- arch/x86/net/bpf_jit_comp.c | 7 +++++++ include/linux/bpf.h | 2 ++ kernel/bpf/core.c | 5 +++++ 3 files changed, 14 insertions(+) (limited to 'arch') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 36f6fc3e6e69..c13d148f7396 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2412,3 +2412,10 @@ bool bpf_jit_supports_kfunc_call(void) { return true; } + +void *bpf_arch_text_copy(void *dst, void *src, size_t len) +{ + if (text_poke_copy(dst, src, len) == NULL) + return ERR_PTR(-EINVAL); + return dst; +} diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 366f88afd56b..ea0d7fd4a410 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2362,6 +2362,8 @@ enum bpf_text_poke_type { int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, void *addr1, void *addr2); +void *bpf_arch_text_copy(void *dst, void *src, size_t len); + struct btf_id_set; bool btf_id_set_contains(const struct btf_id_set *set, u32 id); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index e3fe53df0a71..a5ec480f9862 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2440,6 +2440,11 @@ int __weak bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, return -ENOTSUPP; } +void * __weak bpf_arch_text_copy(void *dst, void *src, size_t len) +{ + return ERR_PTR(-ENOTSUPP); +} + DEFINE_STATIC_KEY_FALSE(bpf_stats_enabled_key); EXPORT_SYMBOL(bpf_stats_enabled_key); -- cgit v1.2.3 From 1022a5498f6f745c3b5fd3f050a5e11e7ca354f0 Mon Sep 17 00:00:00 2001 From: Song Liu Date: Fri, 4 Feb 2022 10:57:42 -0800 Subject: bpf, x86_64: Use bpf_jit_binary_pack_alloc Use bpf_jit_binary_pack_alloc in x86_64 jit. The jit engine first writes the program to the rw buffer. When the jit is done, the program is copied to the final location with bpf_jit_binary_pack_finalize. Note that we need to do bpf_tail_call_direct_fixup after finalize. Therefore, the text_live = false logic in __bpf_arch_text_poke is no longer needed. Signed-off-by: Song Liu Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20220204185742.271030-10-song@kernel.org --- arch/x86/net/bpf_jit_comp.c | 58 ++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 27 deletions(-) (limited to 'arch') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index c13d148f7396..643f38b91e30 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -330,8 +330,7 @@ static int emit_jump(u8 **pprog, void *func, void *ip) } static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, - void *old_addr, void *new_addr, - const bool text_live) + void *old_addr, void *new_addr) { const u8 *nop_insn = x86_nops[5]; u8 old_insn[X86_PATCH_SIZE]; @@ -365,10 +364,7 @@ static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, goto out; ret = 1; if (memcmp(ip, new_insn, X86_PATCH_SIZE)) { - if (text_live) - text_poke_bp(ip, new_insn, X86_PATCH_SIZE, NULL); - else - memcpy(ip, new_insn, X86_PATCH_SIZE); + text_poke_bp(ip, new_insn, X86_PATCH_SIZE, NULL); ret = 0; } out: @@ -384,7 +380,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, /* BPF poking in modules is not supported */ return -EINVAL; - return __bpf_arch_text_poke(ip, t, old_addr, new_addr, true); + return __bpf_arch_text_poke(ip, t, old_addr, new_addr); } #define EMIT_LFENCE() EMIT3(0x0F, 0xAE, 0xE8) @@ -558,24 +554,15 @@ static void bpf_tail_call_direct_fixup(struct bpf_prog *prog) mutex_lock(&array->aux->poke_mutex); target = array->ptrs[poke->tail_call.key]; if (target) { - /* Plain memcpy is used when image is not live yet - * and still not locked as read-only. Once poke - * location is active (poke->tailcall_target_stable), - * any parallel bpf_arch_text_poke() might occur - * still on the read-write image until we finally - * locked it as read-only. Both modifications on - * the given image are under text_mutex to avoid - * interference. - */ ret = __bpf_arch_text_poke(poke->tailcall_target, BPF_MOD_JUMP, NULL, (u8 *)target->bpf_func + - poke->adj_off, false); + poke->adj_off); BUG_ON(ret < 0); ret = __bpf_arch_text_poke(poke->tailcall_bypass, BPF_MOD_JUMP, (u8 *)poke->tailcall_target + - X86_PATCH_SIZE, NULL, false); + X86_PATCH_SIZE, NULL); BUG_ON(ret < 0); } WRITE_ONCE(poke->tailcall_target_stable, true); @@ -866,7 +853,7 @@ static void emit_nops(u8 **pprog, int len) #define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp))) -static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, +static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image, int oldproglen, struct jit_context *ctx, bool jmp_padding) { bool tail_call_reachable = bpf_prog->aux->tail_call_reachable; @@ -893,8 +880,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, push_callee_regs(&prog, callee_regs_used); ilen = prog - temp; - if (image) - memcpy(image + proglen, temp, ilen); + if (rw_image) + memcpy(rw_image + proglen, temp, ilen); proglen += ilen; addrs[0] = proglen; prog = temp; @@ -1323,6 +1310,9 @@ st: if (is_imm8(insn->off)) pr_err("extable->insn doesn't fit into 32-bit\n"); return -EFAULT; } + /* switch ex to rw buffer for writes */ + ex = (void *)rw_image + ((void *)ex - (void *)image); + ex->insn = delta; ex->data = EX_TYPE_BPF; @@ -1705,7 +1695,7 @@ emit_jmp: pr_err("bpf_jit: fatal error\n"); return -EFAULT; } - memcpy(image + proglen, temp, ilen); + memcpy(rw_image + proglen, temp, ilen); } proglen += ilen; addrs[i] = proglen; @@ -2246,6 +2236,7 @@ int arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs) } struct x64_jit_data { + struct bpf_binary_header *rw_header; struct bpf_binary_header *header; int *addrs; u8 *image; @@ -2258,6 +2249,7 @@ struct x64_jit_data { struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { + struct bpf_binary_header *rw_header = NULL; struct bpf_binary_header *header = NULL; struct bpf_prog *tmp, *orig_prog = prog; struct x64_jit_data *jit_data; @@ -2266,6 +2258,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) bool tmp_blinded = false; bool extra_pass = false; bool padding = false; + u8 *rw_image = NULL; u8 *image = NULL; int *addrs; int pass; @@ -2301,6 +2294,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) oldproglen = jit_data->proglen; image = jit_data->image; header = jit_data->header; + rw_header = jit_data->rw_header; + rw_image = (void *)rw_header + ((void *)image - (void *)header); extra_pass = true; padding = true; goto skip_init_addrs; @@ -2331,12 +2326,12 @@ skip_init_addrs: for (pass = 0; pass < MAX_PASSES || image; pass++) { if (!padding && pass >= PADDING_PASSES) padding = true; - proglen = do_jit(prog, addrs, image, oldproglen, &ctx, padding); + proglen = do_jit(prog, addrs, image, rw_image, oldproglen, &ctx, padding); if (proglen <= 0) { out_image: image = NULL; if (header) - bpf_jit_binary_free(header); + bpf_jit_binary_pack_free(header, rw_header); prog = orig_prog; goto out_addrs; } @@ -2360,8 +2355,9 @@ out_image: sizeof(struct exception_table_entry); /* allocate module memory for x86 insns and extable */ - header = bpf_jit_binary_alloc(roundup(proglen, align) + extable_size, - &image, align, jit_fill_hole); + header = bpf_jit_binary_pack_alloc(roundup(proglen, align) + extable_size, + &image, align, &rw_header, &rw_image, + jit_fill_hole); if (!header) { prog = orig_prog; goto out_addrs; @@ -2377,14 +2373,22 @@ out_image: if (image) { if (!prog->is_func || extra_pass) { + /* + * bpf_jit_binary_pack_finalize fails in two scenarios: + * 1) header is not pointing to proper module memory; + * 2) the arch doesn't support bpf_arch_text_copy(). + * + * Both cases are serious bugs that we should not continue. + */ + BUG_ON(bpf_jit_binary_pack_finalize(prog, header, rw_header)); bpf_tail_call_direct_fixup(prog); - bpf_jit_binary_lock_ro(header); } else { jit_data->addrs = addrs; jit_data->ctx = ctx; jit_data->proglen = proglen; jit_data->image = image; jit_data->header = header; + jit_data->rw_header = rw_header; } prog->bpf_func = (void *)image; prog->jited = 1; -- cgit v1.2.3 From f95f768f0af4cec806ce86cd67934a10617d96d0 Mon Sep 17 00:00:00 2001 From: Song Liu Date: Mon, 7 Feb 2022 22:25:33 -0800 Subject: bpf, x86_64: Fail gracefully on bpf_jit_binary_pack_finalize failures Instead of BUG_ON(), fail gracefully and return orig_prog. Fixes: 1022a5498f6f ("bpf, x86_64: Use bpf_jit_binary_pack_alloc") Signed-off-by: Song Liu Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20220208062533.3802081-1-song@kernel.org --- arch/x86/net/bpf_jit_comp.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 643f38b91e30..c7db0fe4de2f 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2378,9 +2378,13 @@ out_image: * 1) header is not pointing to proper module memory; * 2) the arch doesn't support bpf_arch_text_copy(). * - * Both cases are serious bugs that we should not continue. + * Both cases are serious bugs and justify WARN_ON. */ - BUG_ON(bpf_jit_binary_pack_finalize(prog, header, rw_header)); + if (WARN_ON(bpf_jit_binary_pack_finalize(prog, header, rw_header))) { + prog = orig_prog; + goto out_addrs; + } + bpf_tail_call_direct_fixup(prog); } else { jit_data->addrs = addrs; -- cgit v1.2.3 From 0f350231b5ac8867d552122b9ec88e8d7da00e8c Mon Sep 17 00:00:00 2001 From: Song Liu Date: Tue, 8 Feb 2022 14:05:08 -0800 Subject: bpf: Fix leftover header->pages in sparc and powerpc code. Replace header->pages * PAGE_SIZE with new header->size. Fixes: ed2d9e1a26cc ("bpf: Use size instead of pages in bpf_binary_header") Reported-by: kernel test robot Signed-off-by: Song Liu Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20220208220509.4180389-2-song@kernel.org --- arch/powerpc/net/bpf_jit_comp.c | 2 +- arch/sparc/net/bpf_jit_comp_64.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index d6ffdd0f2309..9003c313475d 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -247,7 +247,7 @@ skip_codegen_passes: fp->jited = 1; fp->jited_len = proglen + FUNCTION_DESCR_SIZE; - bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + (bpf_hdr->pages * PAGE_SIZE)); + bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + bpf_hdr->size); if (!fp->is_func || extra_pass) { bpf_jit_binary_lock_ro(bpf_hdr); bpf_prog_fill_jited_linfo(fp, addrs); diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c index b1e38784eb23..fa0759bfe498 100644 --- a/arch/sparc/net/bpf_jit_comp_64.c +++ b/arch/sparc/net/bpf_jit_comp_64.c @@ -1599,7 +1599,7 @@ skip_init_ctx: if (bpf_jit_enable > 1) bpf_jit_dump(prog->len, image_size, pass, ctx.image); - bpf_flush_icache(header, (u8 *)header + (header->pages * PAGE_SIZE)); + bpf_flush_icache(header, (u8 *)header + header->size); if (!prog->is_func || extra_pass) { bpf_jit_binary_lock_ro(header); -- cgit v1.2.3 From 68e4f238b0e9d3670a1612ad900a6e98b2b3f7dd Mon Sep 17 00:00:00 2001 From: Hou Tao Date: Sat, 26 Feb 2022 20:19:05 +0800 Subject: bpf, arm64: Call build_prologue() first in first JIT pass BPF line info needs ctx->offset to be the instruction offset in the whole JITed image instead of the body itself, so also call build_prologue() first in first JIT pass. Fixes: 37ab566c178d ("bpf: arm64: Enable arm64 jit to provide bpf_line_info") Signed-off-by: Hou Tao Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20220226121906.5709-2-houtao1@huawei.com --- arch/arm64/net/bpf_jit_comp.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'arch') diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 74f9a9b6a053..7152ee28facc 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1049,15 +1049,18 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) goto out_off; } - /* 1. Initial fake pass to compute ctx->idx. */ - - /* Fake pass to fill in ctx->offset. */ - if (build_body(&ctx, extra_pass)) { + /* + * 1. Initial fake pass to compute ctx->idx and ctx->offset. + * + * BPF line info needs ctx->offset[i] to be the offset of + * instruction[i] in jited image, so build prologue first. + */ + if (build_prologue(&ctx, was_classic)) { prog = orig_prog; goto out_off; } - if (build_prologue(&ctx, was_classic)) { + if (build_body(&ctx, extra_pass)) { prog = orig_prog; goto out_off; } -- cgit v1.2.3 From dda7596c109fc382876118627e29db7607cde35d Mon Sep 17 00:00:00 2001 From: Hou Tao Date: Sat, 26 Feb 2022 20:19:06 +0800 Subject: bpf, arm64: Feed byte-offset into bpf line info insn_to_jit_off passed to bpf_prog_fill_jited_linfo() is calculated in instruction granularity instead of bytes granularity, but BPF line info requires byte offset. bpf_prog_fill_jited_linfo() will be the last user of ctx.offset before it is freed, so convert the offset into byte-offset before calling into bpf_prog_fill_jited_linfo() in order to fix the line info dump on arm64. Fixes: 37ab566c178d ("bpf: arm64: Enable arm64 jit to provide bpf_line_info") Suggested-by: Daniel Borkmann Signed-off-by: Hou Tao Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20220226121906.5709-3-houtao1@huawei.com --- arch/arm64/net/bpf_jit_comp.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'arch') diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 7152ee28facc..4f137424935f 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1133,6 +1133,11 @@ skip_init_ctx: prog->jited_len = prog_size; if (!prog->is_func || extra_pass) { + int i; + + /* offset[prog->len] is the size of program */ + for (i = 0; i <= prog->len; i++) + ctx.offset[i] *= AARCH64_INSN_SIZE; bpf_prog_fill_jited_linfo(prog, ctx.offset + 1); out_off: kfree(ctx.offset); -- cgit v1.2.3 From 1902472b4fa97dba1fd10a204c6b231d6a560081 Mon Sep 17 00:00:00 2001 From: Hou Tao Date: Thu, 17 Feb 2022 15:22:31 +0800 Subject: bpf, arm64: Support more atomic operations Atomics for eBPF patch series adds support for atomic[64]_fetch_add, atomic[64]_[fetch_]{and,or,xor} and atomic[64]_{xchg|cmpxchg}, but it only adds support for x86-64, so support these atomic operations for arm64 as well. Basically the implementation procedure is almost mechanical translation of code snippets in atomic_ll_sc.h & atomic_lse.h & cmpxchg.h located under arch/arm64/include/asm. When LSE atomic is unavailable, an extra temporary register is needed for (BPF_ADD | BPF_FETCH) to save the value of src register, instead of adding TMP_REG_4 just use BPF_REG_AX instead. Also make emit_lse_atomic() as an empty inline function when CONFIG_ARM64_LSE_ATOMICS is disabled. For cpus_have_cap(ARM64_HAS_LSE_ATOMICS) case and no-LSE-ATOMICS case, the following three tests: "./test_verifier", "./test_progs -t atomic" and "insmod ./test_bpf.ko" are exercised and passed. Signed-off-by: Hou Tao Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20220217072232.1186625-4-houtao1@huawei.com --- arch/arm64/net/bpf_jit.h | 39 +++++++- arch/arm64/net/bpf_jit_comp.c | 223 ++++++++++++++++++++++++++++++++++-------- 2 files changed, 217 insertions(+), 45 deletions(-) (limited to 'arch') diff --git a/arch/arm64/net/bpf_jit.h b/arch/arm64/net/bpf_jit.h index 9d9250c7cc72..dd59b5ad8fe4 100644 --- a/arch/arm64/net/bpf_jit.h +++ b/arch/arm64/net/bpf_jit.h @@ -88,17 +88,42 @@ /* [Rn] = Rt; (atomic) Rs = [state] */ #define A64_STXR(sf, Rt, Rn, Rs) \ A64_LSX(sf, Rt, Rn, Rs, STORE_EX) +/* [Rn] = Rt (store release); (atomic) Rs = [state] */ +#define A64_STLXR(sf, Rt, Rn, Rs) \ + aarch64_insn_gen_load_store_ex(Rt, Rn, Rs, A64_SIZE(sf), \ + AARCH64_INSN_LDST_STORE_REL_EX) /* * LSE atomics * - * STADD is simply encoded as an alias for LDADD with XZR as - * the destination register. + * ST{ADD,CLR,SET,EOR} is simply encoded as an alias for + * LDD{ADD,CLR,SET,EOR} with XZR as the destination register. */ -#define A64_STADD(sf, Rn, Rs) \ +#define A64_ST_OP(sf, Rn, Rs, op) \ aarch64_insn_gen_atomic_ld_op(A64_ZR, Rn, Rs, \ - A64_SIZE(sf), AARCH64_INSN_MEM_ATOMIC_ADD, \ + A64_SIZE(sf), AARCH64_INSN_MEM_ATOMIC_##op, \ AARCH64_INSN_MEM_ORDER_NONE) +/* [Rn] = Rs */ +#define A64_STADD(sf, Rn, Rs) A64_ST_OP(sf, Rn, Rs, ADD) +#define A64_STCLR(sf, Rn, Rs) A64_ST_OP(sf, Rn, Rs, CLR) +#define A64_STEOR(sf, Rn, Rs) A64_ST_OP(sf, Rn, Rs, EOR) +#define A64_STSET(sf, Rn, Rs) A64_ST_OP(sf, Rn, Rs, SET) + +#define A64_LD_OP_AL(sf, Rt, Rn, Rs, op) \ + aarch64_insn_gen_atomic_ld_op(Rt, Rn, Rs, \ + A64_SIZE(sf), AARCH64_INSN_MEM_ATOMIC_##op, \ + AARCH64_INSN_MEM_ORDER_ACQREL) +/* Rt = [Rn] (load acquire); [Rn] = Rs (store release) */ +#define A64_LDADDAL(sf, Rt, Rn, Rs) A64_LD_OP_AL(sf, Rt, Rn, Rs, ADD) +#define A64_LDCLRAL(sf, Rt, Rn, Rs) A64_LD_OP_AL(sf, Rt, Rn, Rs, CLR) +#define A64_LDEORAL(sf, Rt, Rn, Rs) A64_LD_OP_AL(sf, Rt, Rn, Rs, EOR) +#define A64_LDSETAL(sf, Rt, Rn, Rs) A64_LD_OP_AL(sf, Rt, Rn, Rs, SET) +/* Rt = [Rn] (load acquire); [Rn] = Rs (store release) */ +#define A64_SWPAL(sf, Rt, Rn, Rs) A64_LD_OP_AL(sf, Rt, Rn, Rs, SWP) +/* Rs = CAS(Rn, Rs, Rt) (load acquire & store release) */ +#define A64_CASAL(sf, Rt, Rn, Rs) \ + aarch64_insn_gen_cas(Rt, Rn, Rs, A64_SIZE(sf), \ + AARCH64_INSN_MEM_ORDER_ACQREL) /* Add/subtract (immediate) */ #define A64_ADDSUB_IMM(sf, Rd, Rn, imm12, type) \ @@ -203,6 +228,9 @@ #define A64_ANDS(sf, Rd, Rn, Rm) A64_LOGIC_SREG(sf, Rd, Rn, Rm, AND_SETFLAGS) /* Rn & Rm; set condition flags */ #define A64_TST(sf, Rn, Rm) A64_ANDS(sf, A64_ZR, Rn, Rm) +/* Rd = ~Rm (alias of ORN with A64_ZR as Rn) */ +#define A64_MVN(sf, Rd, Rm) \ + A64_LOGIC_SREG(sf, Rd, A64_ZR, Rm, ORN) /* Logical (immediate) */ #define A64_LOGIC_IMM(sf, Rd, Rn, imm, type) ({ \ @@ -226,4 +254,7 @@ #define A64_BTI_J A64_HINT(AARCH64_INSN_HINT_BTIJ) #define A64_BTI_JC A64_HINT(AARCH64_INSN_HINT_BTIJC) +/* DMB */ +#define A64_DMB_ISH aarch64_insn_gen_dmb(AARCH64_INSN_MB_ISH) + #endif /* _BPF_JIT_H */ diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 4f137424935f..e850c69e128c 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -27,6 +27,17 @@ #define TCALL_CNT (MAX_BPF_JIT_REG + 2) #define TMP_REG_3 (MAX_BPF_JIT_REG + 3) +#define check_imm(bits, imm) do { \ + if ((((imm) > 0) && ((imm) >> (bits))) || \ + (((imm) < 0) && (~(imm) >> (bits)))) { \ + pr_info("[%2d] imm=%d(0x%x) out of range\n", \ + i, imm, imm); \ + return -EINVAL; \ + } \ +} while (0) +#define check_imm19(imm) check_imm(19, imm) +#define check_imm26(imm) check_imm(26, imm) + /* Map BPF registers to A64 registers */ static const int bpf2a64[] = { /* return value from in-kernel function, and exit value from eBPF */ @@ -329,6 +340,170 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) #undef jmp_offset } +#ifdef CONFIG_ARM64_LSE_ATOMICS +static int emit_lse_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx) +{ + const u8 code = insn->code; + const u8 dst = bpf2a64[insn->dst_reg]; + const u8 src = bpf2a64[insn->src_reg]; + const u8 tmp = bpf2a64[TMP_REG_1]; + const u8 tmp2 = bpf2a64[TMP_REG_2]; + const bool isdw = BPF_SIZE(code) == BPF_DW; + const s16 off = insn->off; + u8 reg; + + if (!off) { + reg = dst; + } else { + emit_a64_mov_i(1, tmp, off, ctx); + emit(A64_ADD(1, tmp, tmp, dst), ctx); + reg = tmp; + } + + switch (insn->imm) { + /* lock *(u32/u64 *)(dst_reg + off) = src_reg */ + case BPF_ADD: + emit(A64_STADD(isdw, reg, src), ctx); + break; + case BPF_AND: + emit(A64_MVN(isdw, tmp2, src), ctx); + emit(A64_STCLR(isdw, reg, tmp2), ctx); + break; + case BPF_OR: + emit(A64_STSET(isdw, reg, src), ctx); + break; + case BPF_XOR: + emit(A64_STEOR(isdw, reg, src), ctx); + break; + /* src_reg = atomic_fetch_(dst_reg + off, src_reg) */ + case BPF_ADD | BPF_FETCH: + emit(A64_LDADDAL(isdw, src, reg, src), ctx); + break; + case BPF_AND | BPF_FETCH: + emit(A64_MVN(isdw, tmp2, src), ctx); + emit(A64_LDCLRAL(isdw, src, reg, tmp2), ctx); + break; + case BPF_OR | BPF_FETCH: + emit(A64_LDSETAL(isdw, src, reg, src), ctx); + break; + case BPF_XOR | BPF_FETCH: + emit(A64_LDEORAL(isdw, src, reg, src), ctx); + break; + /* src_reg = atomic_xchg(dst_reg + off, src_reg); */ + case BPF_XCHG: + emit(A64_SWPAL(isdw, src, reg, src), ctx); + break; + /* r0 = atomic_cmpxchg(dst_reg + off, r0, src_reg); */ + case BPF_CMPXCHG: + emit(A64_CASAL(isdw, src, reg, bpf2a64[BPF_REG_0]), ctx); + break; + default: + pr_err_once("unknown atomic op code %02x\n", insn->imm); + return -EINVAL; + } + + return 0; +} +#else +static inline int emit_lse_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx) +{ + return -EINVAL; +} +#endif + +static int emit_ll_sc_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx) +{ + const u8 code = insn->code; + const u8 dst = bpf2a64[insn->dst_reg]; + const u8 src = bpf2a64[insn->src_reg]; + const u8 tmp = bpf2a64[TMP_REG_1]; + const u8 tmp2 = bpf2a64[TMP_REG_2]; + const u8 tmp3 = bpf2a64[TMP_REG_3]; + const int i = insn - ctx->prog->insnsi; + const s32 imm = insn->imm; + const s16 off = insn->off; + const bool isdw = BPF_SIZE(code) == BPF_DW; + u8 reg; + s32 jmp_offset; + + if (!off) { + reg = dst; + } else { + emit_a64_mov_i(1, tmp, off, ctx); + emit(A64_ADD(1, tmp, tmp, dst), ctx); + reg = tmp; + } + + if (imm == BPF_ADD || imm == BPF_AND || + imm == BPF_OR || imm == BPF_XOR) { + /* lock *(u32/u64 *)(dst_reg + off) = src_reg */ + emit(A64_LDXR(isdw, tmp2, reg), ctx); + if (imm == BPF_ADD) + emit(A64_ADD(isdw, tmp2, tmp2, src), ctx); + else if (imm == BPF_AND) + emit(A64_AND(isdw, tmp2, tmp2, src), ctx); + else if (imm == BPF_OR) + emit(A64_ORR(isdw, tmp2, tmp2, src), ctx); + else + emit(A64_EOR(isdw, tmp2, tmp2, src), ctx); + emit(A64_STXR(isdw, tmp2, reg, tmp3), ctx); + jmp_offset = -3; + check_imm19(jmp_offset); + emit(A64_CBNZ(0, tmp3, jmp_offset), ctx); + } else if (imm == (BPF_ADD | BPF_FETCH) || + imm == (BPF_AND | BPF_FETCH) || + imm == (BPF_OR | BPF_FETCH) || + imm == (BPF_XOR | BPF_FETCH)) { + /* src_reg = atomic_fetch_(dst_reg + off, src_reg) */ + const u8 ax = bpf2a64[BPF_REG_AX]; + + emit(A64_MOV(isdw, ax, src), ctx); + emit(A64_LDXR(isdw, src, reg), ctx); + if (imm == (BPF_ADD | BPF_FETCH)) + emit(A64_ADD(isdw, tmp2, src, ax), ctx); + else if (imm == (BPF_AND | BPF_FETCH)) + emit(A64_AND(isdw, tmp2, src, ax), ctx); + else if (imm == (BPF_OR | BPF_FETCH)) + emit(A64_ORR(isdw, tmp2, src, ax), ctx); + else + emit(A64_EOR(isdw, tmp2, src, ax), ctx); + emit(A64_STLXR(isdw, tmp2, reg, tmp3), ctx); + jmp_offset = -3; + check_imm19(jmp_offset); + emit(A64_CBNZ(0, tmp3, jmp_offset), ctx); + emit(A64_DMB_ISH, ctx); + } else if (imm == BPF_XCHG) { + /* src_reg = atomic_xchg(dst_reg + off, src_reg); */ + emit(A64_MOV(isdw, tmp2, src), ctx); + emit(A64_LDXR(isdw, src, reg), ctx); + emit(A64_STLXR(isdw, tmp2, reg, tmp3), ctx); + jmp_offset = -2; + check_imm19(jmp_offset); + emit(A64_CBNZ(0, tmp3, jmp_offset), ctx); + emit(A64_DMB_ISH, ctx); + } else if (imm == BPF_CMPXCHG) { + /* r0 = atomic_cmpxchg(dst_reg + off, r0, src_reg); */ + const u8 r0 = bpf2a64[BPF_REG_0]; + + emit(A64_MOV(isdw, tmp2, r0), ctx); + emit(A64_LDXR(isdw, r0, reg), ctx); + emit(A64_EOR(isdw, tmp3, r0, tmp2), ctx); + jmp_offset = 4; + check_imm19(jmp_offset); + emit(A64_CBNZ(isdw, tmp3, jmp_offset), ctx); + emit(A64_STLXR(isdw, src, reg, tmp3), ctx); + jmp_offset = -4; + check_imm19(jmp_offset); + emit(A64_CBNZ(0, tmp3, jmp_offset), ctx); + emit(A64_DMB_ISH, ctx); + } else { + pr_err_once("unknown atomic op code %02x\n", imm); + return -EINVAL; + } + + return 0; +} + static void build_epilogue(struct jit_ctx *ctx) { const u8 r0 = bpf2a64[BPF_REG_0]; @@ -434,29 +609,16 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, const u8 src = bpf2a64[insn->src_reg]; const u8 tmp = bpf2a64[TMP_REG_1]; const u8 tmp2 = bpf2a64[TMP_REG_2]; - const u8 tmp3 = bpf2a64[TMP_REG_3]; const s16 off = insn->off; const s32 imm = insn->imm; const int i = insn - ctx->prog->insnsi; const bool is64 = BPF_CLASS(code) == BPF_ALU64 || BPF_CLASS(code) == BPF_JMP; - const bool isdw = BPF_SIZE(code) == BPF_DW; - u8 jmp_cond, reg; + u8 jmp_cond; s32 jmp_offset; u32 a64_insn; int ret; -#define check_imm(bits, imm) do { \ - if ((((imm) > 0) && ((imm) >> (bits))) || \ - (((imm) < 0) && (~(imm) >> (bits)))) { \ - pr_info("[%2d] imm=%d(0x%x) out of range\n", \ - i, imm, imm); \ - return -EINVAL; \ - } \ -} while (0) -#define check_imm19(imm) check_imm(19, imm) -#define check_imm26(imm) check_imm(26, imm) - switch (code) { /* dst = src */ case BPF_ALU | BPF_MOV | BPF_X: @@ -891,33 +1053,12 @@ emit_cond_jmp: case BPF_STX | BPF_ATOMIC | BPF_W: case BPF_STX | BPF_ATOMIC | BPF_DW: - if (insn->imm != BPF_ADD) { - pr_err_once("unknown atomic op code %02x\n", insn->imm); - return -EINVAL; - } - - /* STX XADD: lock *(u32 *)(dst + off) += src - * and - * STX XADD: lock *(u64 *)(dst + off) += src - */ - - if (!off) { - reg = dst; - } else { - emit_a64_mov_i(1, tmp, off, ctx); - emit(A64_ADD(1, tmp, tmp, dst), ctx); - reg = tmp; - } - if (cpus_have_cap(ARM64_HAS_LSE_ATOMICS)) { - emit(A64_STADD(isdw, reg, src), ctx); - } else { - emit(A64_LDXR(isdw, tmp2, reg), ctx); - emit(A64_ADD(isdw, tmp2, tmp2, src), ctx); - emit(A64_STXR(isdw, tmp2, reg, tmp3), ctx); - jmp_offset = -3; - check_imm19(jmp_offset); - emit(A64_CBNZ(0, tmp3, jmp_offset), ctx); - } + if (cpus_have_cap(ARM64_HAS_LSE_ATOMICS)) + ret = emit_lse_atomic(insn, ctx); + else + ret = emit_ll_sc_atomic(insn, ctx); + if (ret) + return ret; break; default: -- cgit v1.2.3 From eed1fcee556fd8569afe94178b0c2784a5a6b717 Mon Sep 17 00:00:00 2001 From: Song Liu Date: Wed, 2 Mar 2022 09:51:25 -0800 Subject: x86: Disable HAVE_ARCH_HUGE_VMALLOC on 32-bit x86 kernel test robot reported kernel BUG like: [ 44.587744][ T1] kernel BUG at arch/x86/mm/physaddr.c:76! [ 44.590151][ T1] __vmalloc_area_node (mm/vmalloc.c:622 mm/vmalloc.c:2995) [ 44.590151][ T1] __vmalloc_node_range (mm/vmalloc.c:3108) [ 44.590151][ T1] __vmalloc_node (mm/vmalloc.c:3157) which is triggered with HAVE_ARCH_HUGE_VMALLOC on 32-bit x86. Since BPF only uses HAVE_ARCH_HUGE_VMALLOC for x86_64, turn it off for 32-bit x86. Fixes: fac54e2bfb5b ("x86/Kconfig: Select HAVE_ARCH_HUGE_VMALLOC with HAVE_ARCH_HUGE_VMAP") Reported-by: kernel test robot Signed-off-by: Song Liu Signed-off-by: Alexei Starovoitov Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20220302175126.247459-2-song@kernel.org --- arch/x86/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 995f2dc28631..9b356da6f46b 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -158,7 +158,7 @@ config X86 select HAVE_ALIGNED_STRUCT_PAGE if SLUB select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_HUGE_VMAP if X86_64 || X86_PAE - select HAVE_ARCH_HUGE_VMALLOC if HAVE_ARCH_HUGE_VMAP + select HAVE_ARCH_HUGE_VMALLOC if X86_64 select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_JUMP_LABEL_RELATIVE select HAVE_ARCH_KASAN if X86_64 -- cgit v1.2.3 From 676b2daabaf9a993db0e02a5ce79b984aaa0388b Mon Sep 17 00:00:00 2001 From: Song Liu Date: Wed, 2 Mar 2022 09:51:26 -0800 Subject: bpf, x86: Set header->size properly before freeing it On do_jit failure path, the header is freed by bpf_jit_binary_pack_free. While bpf_jit_binary_pack_free doesn't require proper ro_header->size, bpf_prog_pack_free still uses it. Set header->size in bpf_int_jit_compile before calling bpf_jit_binary_pack_free. Fixes: 1022a5498f6f ("bpf, x86_64: Use bpf_jit_binary_pack_alloc") Fixes: 33c9805860e5 ("bpf: Introduce bpf_jit_binary_pack_[alloc|finalize|free]") Reported-by: Kui-Feng Lee Signed-off-by: Song Liu Signed-off-by: Alexei Starovoitov Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20220302175126.247459-3-song@kernel.org --- arch/x86/net/bpf_jit_comp.c | 5 ++++- kernel/bpf/core.c | 9 ++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) (limited to 'arch') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index c7db0fe4de2f..e6ff8f4f9ea4 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2330,8 +2330,11 @@ skip_init_addrs: if (proglen <= 0) { out_image: image = NULL; - if (header) + if (header) { + bpf_arch_text_copy(&header->size, &rw_header->size, + sizeof(rw_header->size)); bpf_jit_binary_pack_free(header, rw_header); + } prog = orig_prog; goto out_addrs; } diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index ebb0193d07f0..ab630f773ec1 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1112,13 +1112,16 @@ int bpf_jit_binary_pack_finalize(struct bpf_prog *prog, * 1) when the program is freed after; * 2) when the JIT engine fails (before bpf_jit_binary_pack_finalize). * For case 2), we need to free both the RO memory and the RW buffer. - * Also, ro_header->size in 2) is not properly set yet, so rw_header->size - * is used for uncharge. + * + * bpf_jit_binary_pack_free requires proper ro_header->size. However, + * bpf_jit_binary_pack_alloc does not set it. Therefore, ro_header->size + * must be set with either bpf_jit_binary_pack_finalize (normal path) or + * bpf_arch_text_copy (when jit fails). */ void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header, struct bpf_binary_header *rw_header) { - u32 size = rw_header ? rw_header->size : ro_header->size; + u32 size = ro_header->size; bpf_prog_pack_free(ro_header); kvfree(rw_header); -- cgit v1.2.3 From aa4e5761bff54036a13fba539165033772799beb Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 3 Mar 2022 18:14:58 +0100 Subject: net: xtensa: Use netif_rx(). Since commit baebdf48c3600 ("net: dev: Makes sure netif_rx() can be invoked in any context.") the function netif_rx() can be used in preemptible/thread context as well as in interrupt context. Use netif_rx(). Cc: Chris Zankel Cc: Max Filippov Cc: linux-xtensa@linux-xtensa.org Signed-off-by: Sebastian Andrzej Siewior Acked-by: Max Filippov Signed-off-by: David S. Miller --- arch/xtensa/platforms/iss/network.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/xtensa/platforms/iss/network.c b/arch/xtensa/platforms/iss/network.c index 962e5e145209..9fb99d18e3c2 100644 --- a/arch/xtensa/platforms/iss/network.c +++ b/arch/xtensa/platforms/iss/network.c @@ -304,7 +304,7 @@ static int iss_net_rx(struct net_device *dev) lp->stats.rx_bytes += skb->len; lp->stats.rx_packets++; - netif_rx_ni(skb); + netif_rx(skb); return pkt_len; } kfree_skb(skb); -- cgit v1.2.3 From 6015fb905d89063231ed33bc15be19ef0fc339b8 Mon Sep 17 00:00:00 2001 From: Horatiu Vultur Date: Fri, 4 Mar 2022 12:08:54 +0100 Subject: dts: sparx5: Enable ptp interrupt Add support for ptp interrupt. This interrupt is used when using 2-step timestamping. For each timestamp that is added in a queue, an interrupt is generated. Signed-off-by: Horatiu Vultur Signed-off-by: David S. Miller --- arch/arm64/boot/dts/microchip/sparx5.dtsi | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/arm64/boot/dts/microchip/sparx5.dtsi b/arch/arm64/boot/dts/microchip/sparx5.dtsi index 787ebcec121d..2dd5e38820b1 100644 --- a/arch/arm64/boot/dts/microchip/sparx5.dtsi +++ b/arch/arm64/boot/dts/microchip/sparx5.dtsi @@ -471,9 +471,10 @@ <0x6 0x10004000 0x7fc000>, <0x6 0x11010000 0xaf0000>; reg-names = "cpu", "dev", "gcb"; - interrupt-names = "xtr", "fdma"; + interrupt-names = "xtr", "fdma", "ptp"; interrupts = , - ; + , + ; resets = <&reset 0>; reset-names = "switch"; }; -- cgit v1.2.3 From 3af722cb735d212554027ec81e2aa2e6bf1ee34d Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Tue, 8 Mar 2022 17:12:10 +0100 Subject: powerpc/net: Implement powerpc specific csum_shift() to remove branch Today's implementation of csum_shift() leads to branching based on parity of 'offset' 000002f8 : 2f8: 70 a5 00 01 andi. r5,r5,1 2fc: 41 a2 00 08 beq 304 300: 54 84 c0 3e rotlwi r4,r4,24 304: 7c 63 20 14 addc r3,r3,r4 308: 7c 63 01 94 addze r3,r3 30c: 4e 80 00 20 blr Use first bit of 'offset' directly as input of the rotation instead of branching. 000002f8 : 2f8: 54 a5 1f 38 rlwinm r5,r5,3,28,28 2fc: 20 a5 00 20 subfic r5,r5,32 300: 5c 84 28 3e rotlw r4,r4,r5 304: 7c 63 20 14 addc r3,r3,r4 308: 7c 63 01 94 addze r3,r3 30c: 4e 80 00 20 blr And change to left shift instead of right shift to skip one more instruction. This has no impact on the final sum. 000002f8 : 2f8: 54 a5 1f 38 rlwinm r5,r5,3,28,28 2fc: 5c 84 28 3e rotlw r4,r4,r5 300: 7c 63 20 14 addc r3,r3,r4 304: 7c 63 01 94 addze r3,r3 308: 4e 80 00 20 blr Seems like only powerpc benefits from a branchless implementation. Other main architectures like ARM or X86 get better code with the generic implementation and its branch. Signed-off-by: Christophe Leroy Signed-off-by: David S. Miller --- arch/powerpc/include/asm/checksum.h | 7 +++++++ include/net/checksum.h | 2 ++ 2 files changed, 9 insertions(+) (limited to 'arch') diff --git a/arch/powerpc/include/asm/checksum.h b/arch/powerpc/include/asm/checksum.h index 350de8f90250..ab3832b93f0a 100644 --- a/arch/powerpc/include/asm/checksum.h +++ b/arch/powerpc/include/asm/checksum.h @@ -112,6 +112,13 @@ static __always_inline __wsum csum_add(__wsum csum, __wsum addend) #endif } +#define HAVE_ARCH_CSUM_SHIFT +static __always_inline __wsum csum_shift(__wsum sum, int offset) +{ + /* rotate sum to align it with a 16b boundary */ + return (__force __wsum)rol32((__force u32)sum, (offset & 1) << 3); +} + /* * This is a version of ip_compute_csum() optimized for IP headers, * which always checksum on 4 octet boundaries. ihl is the number diff --git a/include/net/checksum.h b/include/net/checksum.h index 79c67f14c448..6bc783b7a06c 100644 --- a/include/net/checksum.h +++ b/include/net/checksum.h @@ -80,6 +80,7 @@ static __always_inline __sum16 csum16_sub(__sum16 csum, __be16 addend) return csum16_add(csum, ~addend); } +#ifndef HAVE_ARCH_CSUM_SHIFT static __always_inline __wsum csum_shift(__wsum sum, int offset) { /* rotate sum to align it with a 16b boundary */ @@ -87,6 +88,7 @@ static __always_inline __wsum csum_shift(__wsum sum, int offset) return (__force __wsum)ror32((__force u32)sum, 8); return sum; } +#endif static __always_inline __wsum csum_block_add(__wsum csum, __wsum csum2, int offset) -- cgit v1.2.3 From 3cbe93a1f540dbc997fc24a38796266e2b473e06 Mon Sep 17 00:00:00 2001 From: Ioana Ciornei Date: Fri, 11 Mar 2022 23:22:28 +0200 Subject: arch: arm64: dts: lx2160a: describe the SerDes block #1 Describe the SerDes block #1 using the generic phys infrastructure. This way, the ethernet nodes can each reference their serdes lanes individually using the 'phys' dts property. Signed-off-by: Ioana Ciornei Signed-off-by: David S. Miller --- arch/arm64/boot/dts/freescale/fsl-lx2160a-clearfog-itx.dtsi | 4 ++++ arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi | 6 ++++++ 2 files changed, 10 insertions(+) (limited to 'arch') diff --git a/arch/arm64/boot/dts/freescale/fsl-lx2160a-clearfog-itx.dtsi b/arch/arm64/boot/dts/freescale/fsl-lx2160a-clearfog-itx.dtsi index 17f8e733972a..41702e7386e3 100644 --- a/arch/arm64/boot/dts/freescale/fsl-lx2160a-clearfog-itx.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-lx2160a-clearfog-itx.dtsi @@ -63,21 +63,25 @@ &dpmac7 { sfp = <&sfp0>; managed = "in-band-status"; + phys = <&serdes_1 3>; }; &dpmac8 { sfp = <&sfp1>; managed = "in-band-status"; + phys = <&serdes_1 2>; }; &dpmac9 { sfp = <&sfp2>; managed = "in-band-status"; + phys = <&serdes_1 1>; }; &dpmac10 { sfp = <&sfp3>; managed = "in-band-status"; + phys = <&serdes_1 0>; }; &emdio2 { diff --git a/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi b/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi index 7032505f5ef3..92a881302708 100644 --- a/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi @@ -612,6 +612,12 @@ ranges; dma-ranges = <0x0 0x0 0x0 0x0 0x10000 0x00000000>; + serdes_1: phy@1ea0000 { + compatible = "fsl,lynx-28g"; + reg = <0x0 0x1ea0000 0x0 0x1e30>; + #phy-cells = <1>; + }; + crypto: crypto@8000000 { compatible = "fsl,sec-v5.0", "fsl,sec-v4.0"; fsl,sec-era = <10>; -- cgit v1.2.3 From 79e1177809f2a18518e2013de4abe989ff3c1aa8 Mon Sep 17 00:00:00 2001 From: Biao Huang Date: Mon, 14 Mar 2022 15:57:10 +0800 Subject: arm64: dts: mt2712: update ethernet device node Since there are some changes in ethernet driver: update ethernet device node in dts to accommodate to it. 1. stmmac_probe_config_dt() in stmmac_platform.c will initialize specified parameters according to compatible string "snps,dwmac-4.20a", then, dwmac-mediatek.c can skip the initialization if add compatible string "snps,dwmac-4.20a" in eth device node. 2. commit 882007ed7832 ("net-next: dt-binding: dwmac-mediatek: add more description for RMII") added rmii internal support, we should add corresponding clocks/clocks-names in eth device node. 3. add "snps,reset-delays-us = <0 10000 10000>;" to ensure reset delay can meet PHY requirement. Signed-off-by: Biao Huang Signed-off-by: David S. Miller --- arch/arm64/boot/dts/mediatek/mt2712-evb.dts | 1 + arch/arm64/boot/dts/mediatek/mt2712e.dtsi | 14 +++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) (limited to 'arch') diff --git a/arch/arm64/boot/dts/mediatek/mt2712-evb.dts b/arch/arm64/boot/dts/mediatek/mt2712-evb.dts index 7d369fdd3117..11aa135aa0f3 100644 --- a/arch/arm64/boot/dts/mediatek/mt2712-evb.dts +++ b/arch/arm64/boot/dts/mediatek/mt2712-evb.dts @@ -110,6 +110,7 @@ phy-handle = <ðernet_phy0>; mediatek,tx-delay-ps = <1530>; snps,reset-gpio = <&pio 87 GPIO_ACTIVE_LOW>; + snps,reset-delays-us = <0 10000 10000>; pinctrl-names = "default", "sleep"; pinctrl-0 = <ð_default>; pinctrl-1 = <ð_sleep>; diff --git a/arch/arm64/boot/dts/mediatek/mt2712e.dtsi b/arch/arm64/boot/dts/mediatek/mt2712e.dtsi index de16c0d80c30..a27b7628c5f7 100644 --- a/arch/arm64/boot/dts/mediatek/mt2712e.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt2712e.dtsi @@ -726,7 +726,7 @@ }; eth: ethernet@1101c000 { - compatible = "mediatek,mt2712-gmac"; + compatible = "mediatek,mt2712-gmac", "snps,dwmac-4.20a"; reg = <0 0x1101c000 0 0x1300>; interrupts = ; interrupt-names = "macirq"; @@ -734,15 +734,19 @@ clock-names = "axi", "apb", "mac_main", - "ptp_ref"; + "ptp_ref", + "rmii_internal"; clocks = <&pericfg CLK_PERI_GMAC>, <&pericfg CLK_PERI_GMAC_PCLK>, <&topckgen CLK_TOP_ETHER_125M_SEL>, - <&topckgen CLK_TOP_ETHER_50M_SEL>; + <&topckgen CLK_TOP_ETHER_50M_SEL>, + <&topckgen CLK_TOP_ETHER_50M_RMII_SEL>; assigned-clocks = <&topckgen CLK_TOP_ETHER_125M_SEL>, - <&topckgen CLK_TOP_ETHER_50M_SEL>; + <&topckgen CLK_TOP_ETHER_50M_SEL>, + <&topckgen CLK_TOP_ETHER_50M_RMII_SEL>; assigned-clock-parents = <&topckgen CLK_TOP_ETHERPLL_125M>, - <&topckgen CLK_TOP_APLL1_D3>; + <&topckgen CLK_TOP_APLL1_D3>, + <&topckgen CLK_TOP_ETHERPLL_50M>; power-domains = <&scpsys MT2712_POWER_DOMAIN_AUDIO>; mediatek,pericfg = <&pericfg>; snps,axi-config = <&stmmac_axi_setup>; -- cgit v1.2.3 From 73e14451f39e54f83ea3badb6d6b8a423f901845 Mon Sep 17 00:00:00 2001 From: Hou Tao Date: Wed, 9 Mar 2022 20:33:18 +0800 Subject: bpf, x86: Fall back to interpreter mode when extra pass fails Extra pass for subprog jit may fail (e.g. due to bpf_jit_harden race), but bpf_func is not cleared for the subprog and jit_subprogs will succeed. The running of the bpf program may lead to oops because the memory for the jited subprog image has already been freed. So fall back to interpreter mode by clearing bpf_func/jited/jited_len when extra pass fails. Signed-off-by: Hou Tao Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20220309123321.2400262-2-houtao1@huawei.com --- arch/x86/net/bpf_jit_comp.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index e6ff8f4f9ea4..ec3f00be2ac5 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2335,7 +2335,13 @@ out_image: sizeof(rw_header->size)); bpf_jit_binary_pack_free(header, rw_header); } + /* Fall back to interpreter mode */ prog = orig_prog; + if (extra_pass) { + prog->bpf_func = NULL; + prog->jited = 0; + prog->jited_len = 0; + } goto out_addrs; } if (image) { @@ -2384,8 +2390,9 @@ out_image: * Both cases are serious bugs and justify WARN_ON. */ if (WARN_ON(bpf_jit_binary_pack_finalize(prog, header, rw_header))) { - prog = orig_prog; - goto out_addrs; + /* header has been freed */ + header = NULL; + goto out_image; } bpf_tail_call_direct_fixup(prog); -- cgit v1.2.3 From 75caf33eda242e2f34f61e475d666359749ae5ff Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Tue, 15 Mar 2022 23:01:02 +0900 Subject: rethook: x86: Add rethook x86 implementation Add rethook for x86 implementation. Most of the code has been copied from kretprobes on x86. Signed-off-by: Masami Hiramatsu Signed-off-by: Steven Rostedt (Google) Tested-by: Steven Rostedt (Google) Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/164735286243.1084943.7477055110527046644.stgit@devnote2 --- arch/x86/Kconfig | 1 + arch/x86/include/asm/unwind.h | 8 ++- arch/x86/kernel/Makefile | 1 + arch/x86/kernel/kprobes/common.h | 1 + arch/x86/kernel/rethook.c | 119 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 arch/x86/kernel/rethook.c (limited to 'arch') diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 9b356da6f46b..1270b65a5546 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -221,6 +221,7 @@ config X86 select HAVE_KPROBES_ON_FTRACE select HAVE_FUNCTION_ERROR_INJECTION select HAVE_KRETPROBES + select HAVE_RETHOOK select HAVE_KVM select HAVE_LIVEPATCH if X86_64 select HAVE_MIXED_BREAKPOINTS_REGS diff --git a/arch/x86/include/asm/unwind.h b/arch/x86/include/asm/unwind.h index 2a1f8734416d..192df5b2094d 100644 --- a/arch/x86/include/asm/unwind.h +++ b/arch/x86/include/asm/unwind.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -16,7 +17,7 @@ struct unwind_state { unsigned long stack_mask; struct task_struct *task; int graph_idx; -#ifdef CONFIG_KRETPROBES +#if defined(CONFIG_KRETPROBES) || defined(CONFIG_RETHOOK) struct llist_node *kr_cur; #endif bool error; @@ -107,6 +108,11 @@ static inline unsigned long unwind_recover_kretprobe(struct unwind_state *state, unsigned long addr, unsigned long *addr_p) { +#ifdef CONFIG_RETHOOK + if (is_rethook_trampoline(addr)) + return rethook_find_ret_addr(state->task, (unsigned long)addr_p, + &state->kr_cur); +#endif #ifdef CONFIG_KRETPROBES return is_kretprobe_trampoline(addr) ? kretprobe_find_ret_addr(state->task, addr_p, &state->kr_cur) : diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 6aef9ee28a39..792a893a5cc5 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -106,6 +106,7 @@ obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o obj-$(CONFIG_FTRACE_SYSCALLS) += ftrace.o obj-$(CONFIG_X86_TSC) += trace_clock.o obj-$(CONFIG_TRACING) += trace.o +obj-$(CONFIG_RETHOOK) += rethook.o obj-$(CONFIG_CRASH_CORE) += crash_core_$(BITS).o obj-$(CONFIG_KEXEC_CORE) += machine_kexec_$(BITS).o obj-$(CONFIG_KEXEC_CORE) += relocate_kernel_$(BITS).o crash.o diff --git a/arch/x86/kernel/kprobes/common.h b/arch/x86/kernel/kprobes/common.h index 7d3a2e2daf01..c993521d4933 100644 --- a/arch/x86/kernel/kprobes/common.h +++ b/arch/x86/kernel/kprobes/common.h @@ -6,6 +6,7 @@ #include #include +#include #ifdef CONFIG_X86_64 diff --git a/arch/x86/kernel/rethook.c b/arch/x86/kernel/rethook.c new file mode 100644 index 000000000000..f0f2f0608282 --- /dev/null +++ b/arch/x86/kernel/rethook.c @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * x86 implementation of rethook. Mostly copied from arch/x86/kernel/kprobes/core.c. + */ +#include +#include +#include + +#include "kprobes/common.h" + +__visible void arch_rethook_trampoline_callback(struct pt_regs *regs); + +/* + * When a target function returns, this code saves registers and calls + * arch_rethook_trampoline_callback(), which calls the rethook handler. + */ +asm( + ".text\n" + ".global arch_rethook_trampoline\n" + ".type arch_rethook_trampoline, @function\n" + "arch_rethook_trampoline:\n" +#ifdef CONFIG_X86_64 + /* Push a fake return address to tell the unwinder it's a kretprobe. */ + " pushq $arch_rethook_trampoline\n" + UNWIND_HINT_FUNC + /* Save the 'sp - 8', this will be fixed later. */ + " pushq %rsp\n" + " pushfq\n" + SAVE_REGS_STRING + " movq %rsp, %rdi\n" + " call arch_rethook_trampoline_callback\n" + RESTORE_REGS_STRING + /* In the callback function, 'regs->flags' is copied to 'regs->sp'. */ + " addq $8, %rsp\n" + " popfq\n" +#else + /* Push a fake return address to tell the unwinder it's a kretprobe. */ + " pushl $arch_rethook_trampoline\n" + UNWIND_HINT_FUNC + /* Save the 'sp - 4', this will be fixed later. */ + " pushl %esp\n" + " pushfl\n" + SAVE_REGS_STRING + " movl %esp, %eax\n" + " call arch_rethook_trampoline_callback\n" + RESTORE_REGS_STRING + /* In the callback function, 'regs->flags' is copied to 'regs->sp'. */ + " addl $4, %esp\n" + " popfl\n" +#endif + " ret\n" + ".size arch_rethook_trampoline, .-arch_rethook_trampoline\n" +); +NOKPROBE_SYMBOL(arch_rethook_trampoline); + +/* + * Called from arch_rethook_trampoline + */ +__used __visible void arch_rethook_trampoline_callback(struct pt_regs *regs) +{ + unsigned long *frame_pointer; + + /* fixup registers */ + regs->cs = __KERNEL_CS; +#ifdef CONFIG_X86_32 + regs->gs = 0; +#endif + regs->ip = (unsigned long)&arch_rethook_trampoline; + regs->orig_ax = ~0UL; + regs->sp += sizeof(long); + frame_pointer = ®s->sp + 1; + + /* + * The return address at 'frame_pointer' is recovered by the + * arch_rethook_fixup_return() which called from this + * rethook_trampoline_handler(). + */ + rethook_trampoline_handler(regs, (unsigned long)frame_pointer); + + /* + * Copy FLAGS to 'pt_regs::sp' so that arch_rethook_trapmoline() + * can do RET right after POPF. + */ + regs->sp = regs->flags; +} +NOKPROBE_SYMBOL(arch_rethook_trampoline_callback); + +/* + * arch_rethook_trampoline() skips updating frame pointer. The frame pointer + * saved in arch_rethook_trampoline_callback() points to the real caller + * function's frame pointer. Thus the arch_rethook_trampoline() doesn't have + * a standard stack frame with CONFIG_FRAME_POINTER=y. + * Let's mark it non-standard function. Anyway, FP unwinder can correctly + * unwind without the hint. + */ +STACK_FRAME_NON_STANDARD_FP(arch_rethook_trampoline); + +/* This is called from rethook_trampoline_handler(). */ +void arch_rethook_fixup_return(struct pt_regs *regs, + unsigned long correct_ret_addr) +{ + unsigned long *frame_pointer = ®s->sp + 1; + + /* Replace fake return address with real one. */ + *frame_pointer = correct_ret_addr; +} +NOKPROBE_SYMBOL(arch_rethook_fixup_return); + +void arch_rethook_prepare(struct rethook_node *rh, struct pt_regs *regs, bool mcount) +{ + unsigned long *stack = (unsigned long *)regs->sp; + + rh->ret_addr = stack[0]; + rh->frame = regs->sp; + + /* Replace the return addr with trampoline addr */ + stack[0] = (unsigned long) arch_rethook_trampoline; +} +NOKPROBE_SYMBOL(arch_rethook_prepare); -- cgit v1.2.3 From 83acdce6894908337ca82973149d9709d28204d7 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Tue, 15 Mar 2022 23:01:13 +0900 Subject: arm64: rethook: Add arm64 rethook implementation Add rethook arm64 implementation. Most of the code has been copied from kretprobes on arm64. Signed-off-by: Masami Hiramatsu Signed-off-by: Steven Rostedt (Google) Tested-by: Steven Rostedt (Google) Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/164735287344.1084943.9787335632585653418.stgit@devnote2 --- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/stacktrace.h | 2 +- arch/arm64/kernel/probes/Makefile | 1 + arch/arm64/kernel/probes/rethook.c | 25 ++++++++ arch/arm64/kernel/probes/rethook_trampoline.S | 87 +++++++++++++++++++++++++++ arch/arm64/kernel/stacktrace.c | 7 ++- 6 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/kernel/probes/rethook.c create mode 100644 arch/arm64/kernel/probes/rethook_trampoline.S (limited to 'arch') diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 09b885cc4db5..62cff9d0a5bb 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -201,6 +201,7 @@ config ARM64 select HAVE_SYSCALL_TRACEPOINTS select HAVE_KPROBES select HAVE_KRETPROBES + select HAVE_RETHOOK select HAVE_GENERIC_VDSO select IOMMU_DMA if IOMMU_SUPPORT select IRQ_DOMAIN diff --git a/arch/arm64/include/asm/stacktrace.h b/arch/arm64/include/asm/stacktrace.h index e77cdef9ca29..bf04107da97c 100644 --- a/arch/arm64/include/asm/stacktrace.h +++ b/arch/arm64/include/asm/stacktrace.h @@ -58,7 +58,7 @@ struct stackframe { DECLARE_BITMAP(stacks_done, __NR_STACK_TYPES); unsigned long prev_fp; enum stack_type prev_type; -#ifdef CONFIG_KRETPROBES +#if defined(CONFIG_KRETPROBES) || defined(CONFIG_RETHOOK) struct llist_node *kr_cur; #endif }; diff --git a/arch/arm64/kernel/probes/Makefile b/arch/arm64/kernel/probes/Makefile index 8e4be92e25b1..24e689f44c32 100644 --- a/arch/arm64/kernel/probes/Makefile +++ b/arch/arm64/kernel/probes/Makefile @@ -4,3 +4,4 @@ obj-$(CONFIG_KPROBES) += kprobes.o decode-insn.o \ simulate-insn.o obj-$(CONFIG_UPROBES) += uprobes.o decode-insn.o \ simulate-insn.o +obj-$(CONFIG_RETHOOK) += rethook.o rethook_trampoline.o diff --git a/arch/arm64/kernel/probes/rethook.c b/arch/arm64/kernel/probes/rethook.c new file mode 100644 index 000000000000..edc6b804ad6a --- /dev/null +++ b/arch/arm64/kernel/probes/rethook.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Generic return hook for arm64. + * Most of the code is copied from arch/arm64/kernel/probes/kprobes.c + */ + +#include +#include + +/* This is called from arch_rethook_trampoline() */ +unsigned long __used arch_rethook_trampoline_callback(struct pt_regs *regs) +{ + return rethook_trampoline_handler(regs, regs->regs[29]); +} +NOKPROBE_SYMBOL(arch_rethook_trampoline_callback); + +void arch_rethook_prepare(struct rethook_node *rhn, struct pt_regs *regs, bool mcount) +{ + rhn->ret_addr = regs->regs[30]; + rhn->frame = regs->regs[29]; + + /* replace return addr (x30) with trampoline */ + regs->regs[30] = (u64)arch_rethook_trampoline; +} +NOKPROBE_SYMBOL(arch_rethook_prepare); diff --git a/arch/arm64/kernel/probes/rethook_trampoline.S b/arch/arm64/kernel/probes/rethook_trampoline.S new file mode 100644 index 000000000000..610f520ee72b --- /dev/null +++ b/arch/arm64/kernel/probes/rethook_trampoline.S @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * trampoline entry and return code for rethook. + * Copied from arch/arm64/kernel/probes/kprobes_trampoline.S + */ + +#include +#include +#include + + .text + + .macro save_all_base_regs + stp x0, x1, [sp, #S_X0] + stp x2, x3, [sp, #S_X2] + stp x4, x5, [sp, #S_X4] + stp x6, x7, [sp, #S_X6] + stp x8, x9, [sp, #S_X8] + stp x10, x11, [sp, #S_X10] + stp x12, x13, [sp, #S_X12] + stp x14, x15, [sp, #S_X14] + stp x16, x17, [sp, #S_X16] + stp x18, x19, [sp, #S_X18] + stp x20, x21, [sp, #S_X20] + stp x22, x23, [sp, #S_X22] + stp x24, x25, [sp, #S_X24] + stp x26, x27, [sp, #S_X26] + stp x28, x29, [sp, #S_X28] + add x0, sp, #PT_REGS_SIZE + stp lr, x0, [sp, #S_LR] + /* + * Construct a useful saved PSTATE + */ + mrs x0, nzcv + mrs x1, daif + orr x0, x0, x1 + mrs x1, CurrentEL + orr x0, x0, x1 + mrs x1, SPSel + orr x0, x0, x1 + stp xzr, x0, [sp, #S_PC] + .endm + + .macro restore_all_base_regs + ldr x0, [sp, #S_PSTATE] + and x0, x0, #(PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT) + msr nzcv, x0 + ldp x0, x1, [sp, #S_X0] + ldp x2, x3, [sp, #S_X2] + ldp x4, x5, [sp, #S_X4] + ldp x6, x7, [sp, #S_X6] + ldp x8, x9, [sp, #S_X8] + ldp x10, x11, [sp, #S_X10] + ldp x12, x13, [sp, #S_X12] + ldp x14, x15, [sp, #S_X14] + ldp x16, x17, [sp, #S_X16] + ldp x18, x19, [sp, #S_X18] + ldp x20, x21, [sp, #S_X20] + ldp x22, x23, [sp, #S_X22] + ldp x24, x25, [sp, #S_X24] + ldp x26, x27, [sp, #S_X26] + ldp x28, x29, [sp, #S_X28] + .endm + +SYM_CODE_START(arch_rethook_trampoline) + sub sp, sp, #PT_REGS_SIZE + + save_all_base_regs + + /* Setup a frame pointer. */ + add x29, sp, #S_FP + + mov x0, sp + bl arch_rethook_trampoline_callback + /* + * Replace trampoline address in lr with actual orig_ret_addr return + * address. + */ + mov lr, x0 + + /* The frame pointer (x29) is restored with other registers. */ + restore_all_base_regs + + add sp, sp, #PT_REGS_SIZE + ret + +SYM_CODE_END(arch_rethook_trampoline) diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c index e4103e085681..efb5b85b57c2 100644 --- a/arch/arm64/kernel/stacktrace.c +++ b/arch/arm64/kernel/stacktrace.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -38,7 +39,7 @@ static notrace void start_backtrace(struct stackframe *frame, unsigned long fp, { frame->fp = fp; frame->pc = pc; -#ifdef CONFIG_KRETPROBES +#if defined(CONFIG_KRETPROBES) || defined(CONFIG_RETHOOK) frame->kr_cur = NULL; #endif @@ -138,6 +139,10 @@ static int notrace unwind_frame(struct task_struct *tsk, if (is_kretprobe_trampoline(frame->pc)) frame->pc = kretprobe_find_ret_addr(tsk, (void *)frame->fp, &frame->kr_cur); #endif +#ifdef CONFIG_RETHOOK + if (is_rethook_trampoline(frame->pc)) + frame->pc = rethook_find_ret_addr(tsk, frame->fp, &frame->kr_cur); +#endif return 0; } -- cgit v1.2.3 From 02752bd99dc2daae05c12f7063bf0632e22b4c1c Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Tue, 15 Mar 2022 23:01:25 +0900 Subject: powerpc: Add rethook support Add rethook powerpc64 implementation. Most of the code has been copied from kretprobes on powerpc64. Signed-off-by: Masami Hiramatsu Signed-off-by: Steven Rostedt (Google) Tested-by: Steven Rostedt (Google) Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/164735288495.1084943.539630613772422267.stgit@devnote2 --- arch/powerpc/Kconfig | 1 + arch/powerpc/kernel/Makefile | 1 + arch/powerpc/kernel/rethook.c | 72 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) create mode 100644 arch/powerpc/kernel/rethook.c (limited to 'arch') diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index b779603978e1..5feaa241fb56 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -229,6 +229,7 @@ config PPC select HAVE_PERF_EVENTS_NMI if PPC64 select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP + select HAVE_RETHOOK if KPROBES select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE select HAVE_RSEQ diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile index 4d7829399570..feb24ea83ca6 100644 --- a/arch/powerpc/kernel/Makefile +++ b/arch/powerpc/kernel/Makefile @@ -115,6 +115,7 @@ obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_KPROBES) += kprobes.o obj-$(CONFIG_OPTPROBES) += optprobes.o optprobes_head.o obj-$(CONFIG_KPROBES_ON_FTRACE) += kprobes-ftrace.o +obj-$(CONFIG_RETHOOK) += rethook.o obj-$(CONFIG_UPROBES) += uprobes.o obj-$(CONFIG_PPC_UDBG_16550) += legacy_serial.o udbg_16550.o obj-$(CONFIG_SWIOTLB) += dma-swiotlb.o diff --git a/arch/powerpc/kernel/rethook.c b/arch/powerpc/kernel/rethook.c new file mode 100644 index 000000000000..a8a128748efa --- /dev/null +++ b/arch/powerpc/kernel/rethook.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PowerPC implementation of rethook. This depends on kprobes. + */ + +#include +#include + +/* + * Function return trampoline: + * - init_kprobes() establishes a probepoint here + * - When the probed function returns, this probe + * causes the handlers to fire + */ +asm(".global arch_rethook_trampoline\n" + ".type arch_rethook_trampoline, @function\n" + "arch_rethook_trampoline:\n" + "nop\n" + "blr\n" + ".size arch_rethook_trampoline, .-arch_rethook_trampoline\n"); + +/* + * Called when the probe at kretprobe trampoline is hit + */ +static int trampoline_rethook_handler(struct kprobe *p, struct pt_regs *regs) +{ + unsigned long orig_ret_address; + + orig_ret_address = rethook_trampoline_handler(regs, 0); + /* + * We get here through one of two paths: + * 1. by taking a trap -> kprobe_handler() -> here + * 2. by optprobe branch -> optimized_callback() -> opt_pre_handler() -> here + * + * When going back through (1), we need regs->nip to be setup properly + * as it is used to determine the return address from the trap. + * For (2), since nip is not honoured with optprobes, we instead setup + * the link register properly so that the subsequent 'blr' in + * __kretprobe_trampoline jumps back to the right instruction. + * + * For nip, we should set the address to the previous instruction since + * we end up emulating it in kprobe_handler(), which increments the nip + * again. + */ + regs_set_return_ip(regs, orig_ret_address - 4); + regs->link = orig_ret_address; + + return 0; +} +NOKPROBE_SYMBOL(trampoline_rethook_handler); + +void arch_rethook_prepare(struct rethook_node *rh, struct pt_regs *regs, bool mcount) +{ + rh->ret_addr = regs->link; + rh->frame = 0; + + /* Replace the return addr with trampoline addr */ + regs->link = (unsigned long)arch_rethook_trampoline; +} +NOKPROBE_SYMBOL(arch_prepare_kretprobe); + +static struct kprobe trampoline_p = { + .addr = (kprobe_opcode_t *) &arch_rethook_trampoline, + .pre_handler = trampoline_rethook_handler +}; + +static int init_arch_rethook(void) +{ + return register_kprobe(&trampoline_p); +} + +core_initcall(init_arch_rethook); -- cgit v1.2.3 From 515a49173b80a4aabcbad9a4fa2a247042378ea1 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Tue, 15 Mar 2022 23:01:36 +0900 Subject: ARM: rethook: Add rethook arm implementation Add rethook arm implementation. Most of the code has been copied from kretprobes on arm. Since the arm's ftrace implementation is a bit special, this needs a special care using from fprobe. Signed-off-by: Masami Hiramatsu Signed-off-by: Steven Rostedt (Google) Tested-by: Steven Rostedt (Google) Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/164735289643.1084943.15184590256680485720.stgit@devnote2 --- arch/arm/Kconfig | 1 + arch/arm/include/asm/stacktrace.h | 4 +- arch/arm/kernel/stacktrace.c | 6 +++ arch/arm/probes/Makefile | 1 + arch/arm/probes/rethook.c | 103 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 arch/arm/probes/rethook.c (limited to 'arch') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 4c97cb40eebb..440f69ee8af5 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -107,6 +107,7 @@ config ARM select HAVE_MOD_ARCH_SPECIFIC select HAVE_NMI select HAVE_OPTPROBES if !THUMB2_KERNEL + select HAVE_RETHOOK select HAVE_PERF_EVENTS select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP diff --git a/arch/arm/include/asm/stacktrace.h b/arch/arm/include/asm/stacktrace.h index 8f54f9ad8a9b..babed1707ca8 100644 --- a/arch/arm/include/asm/stacktrace.h +++ b/arch/arm/include/asm/stacktrace.h @@ -14,7 +14,7 @@ struct stackframe { unsigned long sp; unsigned long lr; unsigned long pc; -#ifdef CONFIG_KRETPROBES +#if defined(CONFIG_KRETPROBES) || defined(CONFIG_RETHOOK) struct llist_node *kr_cur; struct task_struct *tsk; #endif @@ -27,7 +27,7 @@ void arm_get_current_stackframe(struct pt_regs *regs, struct stackframe *frame) frame->sp = regs->ARM_sp; frame->lr = regs->ARM_lr; frame->pc = regs->ARM_pc; -#ifdef CONFIG_KRETPROBES +#if defined(CONFIG_KRETPROBES) || defined(CONFIG_RETHOOK) frame->kr_cur = NULL; frame->tsk = current; #endif diff --git a/arch/arm/kernel/stacktrace.c b/arch/arm/kernel/stacktrace.c index 75e905508f27..f509c6be4f57 100644 --- a/arch/arm/kernel/stacktrace.c +++ b/arch/arm/kernel/stacktrace.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only #include #include +#include #include #include #include @@ -66,6 +67,11 @@ int notrace unwind_frame(struct stackframe *frame) frame->sp = *(unsigned long *)(fp - 8); frame->pc = *(unsigned long *)(fp - 4); #endif +#ifdef CONFIG_RETHOOK + if (is_rethook_trampoline(frame->pc)) + frame->pc = rethook_find_ret_addr(frame->tsk, frame->fp, + &frame->kr_cur); +#endif #ifdef CONFIG_KRETPROBES if (is_kretprobe_trampoline(frame->pc)) frame->pc = kretprobe_find_ret_addr(frame->tsk, diff --git a/arch/arm/probes/Makefile b/arch/arm/probes/Makefile index 8b0ea5ace100..10c083a22223 100644 --- a/arch/arm/probes/Makefile +++ b/arch/arm/probes/Makefile @@ -6,3 +6,4 @@ obj-$(CONFIG_KPROBES) += decode-thumb.o else obj-$(CONFIG_KPROBES) += decode-arm.o endif +obj-$(CONFIG_RETHOOK) += rethook.o diff --git a/arch/arm/probes/rethook.c b/arch/arm/probes/rethook.c new file mode 100644 index 000000000000..1c1357a86365 --- /dev/null +++ b/arch/arm/probes/rethook.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * arm implementation of rethook. Mostly copied from arch/arm/probes/kprobes/core.c + */ + +#include +#include + +/* Called from arch_rethook_trampoline */ +static __used unsigned long arch_rethook_trampoline_callback(struct pt_regs *regs) +{ + return rethook_trampoline_handler(regs, regs->ARM_fp); +} +NOKPROBE_SYMBOL(arch_rethook_trampoline_callback); + +/* + * When a rethook'ed function returns, it returns to arch_rethook_trampoline + * which calls rethook callback. We construct a struct pt_regs to + * give a view of registers r0-r11, sp, lr, and pc to the user + * return-handler. This is not a complete pt_regs structure, but that + * should be enough for stacktrace from the return handler with or + * without pt_regs. + */ +asm( + ".text\n" + ".global arch_rethook_trampoline\n" + ".type arch_rethook_trampoline, %function\n" + "arch_rethook_trampoline:\n" +#ifdef CONFIG_FRAME_POINTER + "ldr lr, =arch_rethook_trampoline \n\t" + /* this makes a framepointer on pt_regs. */ +#ifdef CONFIG_CC_IS_CLANG + "stmdb sp, {sp, lr, pc} \n\t" + "sub sp, sp, #12 \n\t" + /* In clang case, pt_regs->ip = lr. */ + "stmdb sp!, {r0 - r11, lr} \n\t" + /* fp points regs->r11 (fp) */ + "add fp, sp, #44 \n\t" +#else /* !CONFIG_CC_IS_CLANG */ + /* In gcc case, pt_regs->ip = fp. */ + "stmdb sp, {fp, sp, lr, pc} \n\t" + "sub sp, sp, #16 \n\t" + "stmdb sp!, {r0 - r11} \n\t" + /* fp points regs->r15 (pc) */ + "add fp, sp, #60 \n\t" +#endif /* CONFIG_CC_IS_CLANG */ +#else /* !CONFIG_FRAME_POINTER */ + "sub sp, sp, #16 \n\t" + "stmdb sp!, {r0 - r11} \n\t" +#endif /* CONFIG_FRAME_POINTER */ + "mov r0, sp \n\t" + "bl arch_rethook_trampoline_callback \n\t" + "mov lr, r0 \n\t" + "ldmia sp!, {r0 - r11} \n\t" + "add sp, sp, #16 \n\t" +#ifdef CONFIG_THUMB2_KERNEL + "bx lr \n\t" +#else + "mov pc, lr \n\t" +#endif + ".size arch_rethook_trampoline, .-arch_rethook_trampoline\n" +); +NOKPROBE_SYMBOL(arch_rethook_trampoline); + +/* + * At the entry of function with mcount. The stack and registers are prepared + * for the mcount function as below. + * + * mov ip, sp + * push {fp, ip, lr, pc} + * sub fp, ip, #4 ; FP[0] = PC, FP[-4] = LR, and FP[-12] = call-site FP. + * push {lr} + * bl <__gnu_mcount_nc> ; call ftrace + * + * And when returning from the function, call-site FP, SP and PC are restored + * from stack as below; + * + * ldm sp, {fp, sp, pc} + * + * Thus, if the arch_rethook_prepare() is called from real function entry, + * it must change the LR and save FP in pt_regs. But if it is called via + * mcount context (ftrace), it must change the LR on stack, which is next + * to the PC (= FP[-4]), and save the FP value at FP[-12]. + */ +void arch_rethook_prepare(struct rethook_node *rh, struct pt_regs *regs, bool mcount) +{ + unsigned long *ret_addr, *frame; + + if (mcount) { + ret_addr = (unsigned long *)(regs->ARM_fp - 4); + frame = (unsigned long *)(regs->ARM_fp - 12); + } else { + ret_addr = ®s->ARM_lr; + frame = ®s->ARM_fp; + } + + rh->ret_addr = *ret_addr; + rh->frame = *frame; + + /* Replace the return addr with trampoline addr. */ + *ret_addr = (unsigned long)arch_rethook_trampoline; +} +NOKPROBE_SYMBOL(arch_rethook_prepare); -- cgit v1.2.3 From d8dc09a4db45e49cc1ee5170632833ec11fafa7e Mon Sep 17 00:00:00 2001 From: Julia Lawall Date: Fri, 18 Mar 2022 11:37:04 +0100 Subject: bpf, arm: Fix various typos in comments Various spelling mistakes in comments. Detected with the help of Coccinelle. Signed-off-by: Julia Lawall Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20220318103729.157574-9-Julia.Lawall@inria.fr --- arch/arm/net/bpf_jit_32.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index 10ceebb7530b..9e457156ad4d 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -1864,7 +1864,7 @@ static int build_body(struct jit_ctx *ctx) if (ctx->target == NULL) ctx->offsets[i] = ctx->idx; - /* If unsuccesfull, return with error code */ + /* If unsuccesful, return with error code */ if (ret) return ret; } @@ -1973,7 +1973,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) * for jit, although it can decrease the size of the image. * * As each arm instruction is of length 32bit, we are translating - * number of JITed intructions into the size required to store these + * number of JITed instructions into the size required to store these * JITed code. */ image_size = sizeof(u32) * ctx.idx; -- cgit v1.2.3 From ecaed3b9deea9ff3e7d98b9ee8722c7d78de5d97 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Mon, 21 Mar 2022 21:40:45 -0700 Subject: Revert "ARM: rethook: Add rethook arm implementation" This reverts commit 515a49173b80a4aabcbad9a4fa2a247042378ea1. Signed-off-by: Alexei Starovoitov --- arch/arm/Kconfig | 1 - arch/arm/include/asm/stacktrace.h | 4 +- arch/arm/kernel/stacktrace.c | 6 --- arch/arm/probes/Makefile | 1 - arch/arm/probes/rethook.c | 103 -------------------------------------- 5 files changed, 2 insertions(+), 113 deletions(-) delete mode 100644 arch/arm/probes/rethook.c (limited to 'arch') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 440f69ee8af5..4c97cb40eebb 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -107,7 +107,6 @@ config ARM select HAVE_MOD_ARCH_SPECIFIC select HAVE_NMI select HAVE_OPTPROBES if !THUMB2_KERNEL - select HAVE_RETHOOK select HAVE_PERF_EVENTS select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP diff --git a/arch/arm/include/asm/stacktrace.h b/arch/arm/include/asm/stacktrace.h index babed1707ca8..8f54f9ad8a9b 100644 --- a/arch/arm/include/asm/stacktrace.h +++ b/arch/arm/include/asm/stacktrace.h @@ -14,7 +14,7 @@ struct stackframe { unsigned long sp; unsigned long lr; unsigned long pc; -#if defined(CONFIG_KRETPROBES) || defined(CONFIG_RETHOOK) +#ifdef CONFIG_KRETPROBES struct llist_node *kr_cur; struct task_struct *tsk; #endif @@ -27,7 +27,7 @@ void arm_get_current_stackframe(struct pt_regs *regs, struct stackframe *frame) frame->sp = regs->ARM_sp; frame->lr = regs->ARM_lr; frame->pc = regs->ARM_pc; -#if defined(CONFIG_KRETPROBES) || defined(CONFIG_RETHOOK) +#ifdef CONFIG_KRETPROBES frame->kr_cur = NULL; frame->tsk = current; #endif diff --git a/arch/arm/kernel/stacktrace.c b/arch/arm/kernel/stacktrace.c index f509c6be4f57..75e905508f27 100644 --- a/arch/arm/kernel/stacktrace.c +++ b/arch/arm/kernel/stacktrace.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only #include #include -#include #include #include #include @@ -67,11 +66,6 @@ int notrace unwind_frame(struct stackframe *frame) frame->sp = *(unsigned long *)(fp - 8); frame->pc = *(unsigned long *)(fp - 4); #endif -#ifdef CONFIG_RETHOOK - if (is_rethook_trampoline(frame->pc)) - frame->pc = rethook_find_ret_addr(frame->tsk, frame->fp, - &frame->kr_cur); -#endif #ifdef CONFIG_KRETPROBES if (is_kretprobe_trampoline(frame->pc)) frame->pc = kretprobe_find_ret_addr(frame->tsk, diff --git a/arch/arm/probes/Makefile b/arch/arm/probes/Makefile index 10c083a22223..8b0ea5ace100 100644 --- a/arch/arm/probes/Makefile +++ b/arch/arm/probes/Makefile @@ -6,4 +6,3 @@ obj-$(CONFIG_KPROBES) += decode-thumb.o else obj-$(CONFIG_KPROBES) += decode-arm.o endif -obj-$(CONFIG_RETHOOK) += rethook.o diff --git a/arch/arm/probes/rethook.c b/arch/arm/probes/rethook.c deleted file mode 100644 index 1c1357a86365..000000000000 --- a/arch/arm/probes/rethook.c +++ /dev/null @@ -1,103 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * arm implementation of rethook. Mostly copied from arch/arm/probes/kprobes/core.c - */ - -#include -#include - -/* Called from arch_rethook_trampoline */ -static __used unsigned long arch_rethook_trampoline_callback(struct pt_regs *regs) -{ - return rethook_trampoline_handler(regs, regs->ARM_fp); -} -NOKPROBE_SYMBOL(arch_rethook_trampoline_callback); - -/* - * When a rethook'ed function returns, it returns to arch_rethook_trampoline - * which calls rethook callback. We construct a struct pt_regs to - * give a view of registers r0-r11, sp, lr, and pc to the user - * return-handler. This is not a complete pt_regs structure, but that - * should be enough for stacktrace from the return handler with or - * without pt_regs. - */ -asm( - ".text\n" - ".global arch_rethook_trampoline\n" - ".type arch_rethook_trampoline, %function\n" - "arch_rethook_trampoline:\n" -#ifdef CONFIG_FRAME_POINTER - "ldr lr, =arch_rethook_trampoline \n\t" - /* this makes a framepointer on pt_regs. */ -#ifdef CONFIG_CC_IS_CLANG - "stmdb sp, {sp, lr, pc} \n\t" - "sub sp, sp, #12 \n\t" - /* In clang case, pt_regs->ip = lr. */ - "stmdb sp!, {r0 - r11, lr} \n\t" - /* fp points regs->r11 (fp) */ - "add fp, sp, #44 \n\t" -#else /* !CONFIG_CC_IS_CLANG */ - /* In gcc case, pt_regs->ip = fp. */ - "stmdb sp, {fp, sp, lr, pc} \n\t" - "sub sp, sp, #16 \n\t" - "stmdb sp!, {r0 - r11} \n\t" - /* fp points regs->r15 (pc) */ - "add fp, sp, #60 \n\t" -#endif /* CONFIG_CC_IS_CLANG */ -#else /* !CONFIG_FRAME_POINTER */ - "sub sp, sp, #16 \n\t" - "stmdb sp!, {r0 - r11} \n\t" -#endif /* CONFIG_FRAME_POINTER */ - "mov r0, sp \n\t" - "bl arch_rethook_trampoline_callback \n\t" - "mov lr, r0 \n\t" - "ldmia sp!, {r0 - r11} \n\t" - "add sp, sp, #16 \n\t" -#ifdef CONFIG_THUMB2_KERNEL - "bx lr \n\t" -#else - "mov pc, lr \n\t" -#endif - ".size arch_rethook_trampoline, .-arch_rethook_trampoline\n" -); -NOKPROBE_SYMBOL(arch_rethook_trampoline); - -/* - * At the entry of function with mcount. The stack and registers are prepared - * for the mcount function as below. - * - * mov ip, sp - * push {fp, ip, lr, pc} - * sub fp, ip, #4 ; FP[0] = PC, FP[-4] = LR, and FP[-12] = call-site FP. - * push {lr} - * bl <__gnu_mcount_nc> ; call ftrace - * - * And when returning from the function, call-site FP, SP and PC are restored - * from stack as below; - * - * ldm sp, {fp, sp, pc} - * - * Thus, if the arch_rethook_prepare() is called from real function entry, - * it must change the LR and save FP in pt_regs. But if it is called via - * mcount context (ftrace), it must change the LR on stack, which is next - * to the PC (= FP[-4]), and save the FP value at FP[-12]. - */ -void arch_rethook_prepare(struct rethook_node *rh, struct pt_regs *regs, bool mcount) -{ - unsigned long *ret_addr, *frame; - - if (mcount) { - ret_addr = (unsigned long *)(regs->ARM_fp - 4); - frame = (unsigned long *)(regs->ARM_fp - 12); - } else { - ret_addr = ®s->ARM_lr; - frame = ®s->ARM_fp; - } - - rh->ret_addr = *ret_addr; - rh->frame = *frame; - - /* Replace the return addr with trampoline addr. */ - *ret_addr = (unsigned long)arch_rethook_trampoline; -} -NOKPROBE_SYMBOL(arch_rethook_prepare); -- cgit v1.2.3 From 35df0155e68a1fb20646f34247f19131170693bd Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Mon, 21 Mar 2022 21:40:49 -0700 Subject: Revert "powerpc: Add rethook support" This reverts commit 02752bd99dc2daae05c12f7063bf0632e22b4c1c. Signed-off-by: Alexei Starovoitov --- arch/powerpc/Kconfig | 1 - arch/powerpc/kernel/Makefile | 1 - arch/powerpc/kernel/rethook.c | 72 ------------------------------------------- 3 files changed, 74 deletions(-) delete mode 100644 arch/powerpc/kernel/rethook.c (limited to 'arch') diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 5feaa241fb56..b779603978e1 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -229,7 +229,6 @@ config PPC select HAVE_PERF_EVENTS_NMI if PPC64 select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP - select HAVE_RETHOOK if KPROBES select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE select HAVE_RSEQ diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile index feb24ea83ca6..4d7829399570 100644 --- a/arch/powerpc/kernel/Makefile +++ b/arch/powerpc/kernel/Makefile @@ -115,7 +115,6 @@ obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_KPROBES) += kprobes.o obj-$(CONFIG_OPTPROBES) += optprobes.o optprobes_head.o obj-$(CONFIG_KPROBES_ON_FTRACE) += kprobes-ftrace.o -obj-$(CONFIG_RETHOOK) += rethook.o obj-$(CONFIG_UPROBES) += uprobes.o obj-$(CONFIG_PPC_UDBG_16550) += legacy_serial.o udbg_16550.o obj-$(CONFIG_SWIOTLB) += dma-swiotlb.o diff --git a/arch/powerpc/kernel/rethook.c b/arch/powerpc/kernel/rethook.c deleted file mode 100644 index a8a128748efa..000000000000 --- a/arch/powerpc/kernel/rethook.c +++ /dev/null @@ -1,72 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * PowerPC implementation of rethook. This depends on kprobes. - */ - -#include -#include - -/* - * Function return trampoline: - * - init_kprobes() establishes a probepoint here - * - When the probed function returns, this probe - * causes the handlers to fire - */ -asm(".global arch_rethook_trampoline\n" - ".type arch_rethook_trampoline, @function\n" - "arch_rethook_trampoline:\n" - "nop\n" - "blr\n" - ".size arch_rethook_trampoline, .-arch_rethook_trampoline\n"); - -/* - * Called when the probe at kretprobe trampoline is hit - */ -static int trampoline_rethook_handler(struct kprobe *p, struct pt_regs *regs) -{ - unsigned long orig_ret_address; - - orig_ret_address = rethook_trampoline_handler(regs, 0); - /* - * We get here through one of two paths: - * 1. by taking a trap -> kprobe_handler() -> here - * 2. by optprobe branch -> optimized_callback() -> opt_pre_handler() -> here - * - * When going back through (1), we need regs->nip to be setup properly - * as it is used to determine the return address from the trap. - * For (2), since nip is not honoured with optprobes, we instead setup - * the link register properly so that the subsequent 'blr' in - * __kretprobe_trampoline jumps back to the right instruction. - * - * For nip, we should set the address to the previous instruction since - * we end up emulating it in kprobe_handler(), which increments the nip - * again. - */ - regs_set_return_ip(regs, orig_ret_address - 4); - regs->link = orig_ret_address; - - return 0; -} -NOKPROBE_SYMBOL(trampoline_rethook_handler); - -void arch_rethook_prepare(struct rethook_node *rh, struct pt_regs *regs, bool mcount) -{ - rh->ret_addr = regs->link; - rh->frame = 0; - - /* Replace the return addr with trampoline addr */ - regs->link = (unsigned long)arch_rethook_trampoline; -} -NOKPROBE_SYMBOL(arch_prepare_kretprobe); - -static struct kprobe trampoline_p = { - .addr = (kprobe_opcode_t *) &arch_rethook_trampoline, - .pre_handler = trampoline_rethook_handler -}; - -static int init_arch_rethook(void) -{ - return register_kprobe(&trampoline_p); -} - -core_initcall(init_arch_rethook); -- cgit v1.2.3 From 0f8f8030038ae37c32a233186caab2c381396784 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Mon, 21 Mar 2022 21:40:51 -0700 Subject: Revert "arm64: rethook: Add arm64 rethook implementation" This reverts commit 83acdce6894908337ca82973149d9709d28204d7. Signed-off-by: Alexei Starovoitov --- arch/arm64/Kconfig | 1 - arch/arm64/include/asm/stacktrace.h | 2 +- arch/arm64/kernel/probes/Makefile | 1 - arch/arm64/kernel/probes/rethook.c | 25 -------- arch/arm64/kernel/probes/rethook_trampoline.S | 87 --------------------------- arch/arm64/kernel/stacktrace.c | 7 +-- 6 files changed, 2 insertions(+), 121 deletions(-) delete mode 100644 arch/arm64/kernel/probes/rethook.c delete mode 100644 arch/arm64/kernel/probes/rethook_trampoline.S (limited to 'arch') diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 62cff9d0a5bb..09b885cc4db5 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -201,7 +201,6 @@ config ARM64 select HAVE_SYSCALL_TRACEPOINTS select HAVE_KPROBES select HAVE_KRETPROBES - select HAVE_RETHOOK select HAVE_GENERIC_VDSO select IOMMU_DMA if IOMMU_SUPPORT select IRQ_DOMAIN diff --git a/arch/arm64/include/asm/stacktrace.h b/arch/arm64/include/asm/stacktrace.h index bf04107da97c..e77cdef9ca29 100644 --- a/arch/arm64/include/asm/stacktrace.h +++ b/arch/arm64/include/asm/stacktrace.h @@ -58,7 +58,7 @@ struct stackframe { DECLARE_BITMAP(stacks_done, __NR_STACK_TYPES); unsigned long prev_fp; enum stack_type prev_type; -#if defined(CONFIG_KRETPROBES) || defined(CONFIG_RETHOOK) +#ifdef CONFIG_KRETPROBES struct llist_node *kr_cur; #endif }; diff --git a/arch/arm64/kernel/probes/Makefile b/arch/arm64/kernel/probes/Makefile index 24e689f44c32..8e4be92e25b1 100644 --- a/arch/arm64/kernel/probes/Makefile +++ b/arch/arm64/kernel/probes/Makefile @@ -4,4 +4,3 @@ obj-$(CONFIG_KPROBES) += kprobes.o decode-insn.o \ simulate-insn.o obj-$(CONFIG_UPROBES) += uprobes.o decode-insn.o \ simulate-insn.o -obj-$(CONFIG_RETHOOK) += rethook.o rethook_trampoline.o diff --git a/arch/arm64/kernel/probes/rethook.c b/arch/arm64/kernel/probes/rethook.c deleted file mode 100644 index edc6b804ad6a..000000000000 --- a/arch/arm64/kernel/probes/rethook.c +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Generic return hook for arm64. - * Most of the code is copied from arch/arm64/kernel/probes/kprobes.c - */ - -#include -#include - -/* This is called from arch_rethook_trampoline() */ -unsigned long __used arch_rethook_trampoline_callback(struct pt_regs *regs) -{ - return rethook_trampoline_handler(regs, regs->regs[29]); -} -NOKPROBE_SYMBOL(arch_rethook_trampoline_callback); - -void arch_rethook_prepare(struct rethook_node *rhn, struct pt_regs *regs, bool mcount) -{ - rhn->ret_addr = regs->regs[30]; - rhn->frame = regs->regs[29]; - - /* replace return addr (x30) with trampoline */ - regs->regs[30] = (u64)arch_rethook_trampoline; -} -NOKPROBE_SYMBOL(arch_rethook_prepare); diff --git a/arch/arm64/kernel/probes/rethook_trampoline.S b/arch/arm64/kernel/probes/rethook_trampoline.S deleted file mode 100644 index 610f520ee72b..000000000000 --- a/arch/arm64/kernel/probes/rethook_trampoline.S +++ /dev/null @@ -1,87 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * trampoline entry and return code for rethook. - * Copied from arch/arm64/kernel/probes/kprobes_trampoline.S - */ - -#include -#include -#include - - .text - - .macro save_all_base_regs - stp x0, x1, [sp, #S_X0] - stp x2, x3, [sp, #S_X2] - stp x4, x5, [sp, #S_X4] - stp x6, x7, [sp, #S_X6] - stp x8, x9, [sp, #S_X8] - stp x10, x11, [sp, #S_X10] - stp x12, x13, [sp, #S_X12] - stp x14, x15, [sp, #S_X14] - stp x16, x17, [sp, #S_X16] - stp x18, x19, [sp, #S_X18] - stp x20, x21, [sp, #S_X20] - stp x22, x23, [sp, #S_X22] - stp x24, x25, [sp, #S_X24] - stp x26, x27, [sp, #S_X26] - stp x28, x29, [sp, #S_X28] - add x0, sp, #PT_REGS_SIZE - stp lr, x0, [sp, #S_LR] - /* - * Construct a useful saved PSTATE - */ - mrs x0, nzcv - mrs x1, daif - orr x0, x0, x1 - mrs x1, CurrentEL - orr x0, x0, x1 - mrs x1, SPSel - orr x0, x0, x1 - stp xzr, x0, [sp, #S_PC] - .endm - - .macro restore_all_base_regs - ldr x0, [sp, #S_PSTATE] - and x0, x0, #(PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT) - msr nzcv, x0 - ldp x0, x1, [sp, #S_X0] - ldp x2, x3, [sp, #S_X2] - ldp x4, x5, [sp, #S_X4] - ldp x6, x7, [sp, #S_X6] - ldp x8, x9, [sp, #S_X8] - ldp x10, x11, [sp, #S_X10] - ldp x12, x13, [sp, #S_X12] - ldp x14, x15, [sp, #S_X14] - ldp x16, x17, [sp, #S_X16] - ldp x18, x19, [sp, #S_X18] - ldp x20, x21, [sp, #S_X20] - ldp x22, x23, [sp, #S_X22] - ldp x24, x25, [sp, #S_X24] - ldp x26, x27, [sp, #S_X26] - ldp x28, x29, [sp, #S_X28] - .endm - -SYM_CODE_START(arch_rethook_trampoline) - sub sp, sp, #PT_REGS_SIZE - - save_all_base_regs - - /* Setup a frame pointer. */ - add x29, sp, #S_FP - - mov x0, sp - bl arch_rethook_trampoline_callback - /* - * Replace trampoline address in lr with actual orig_ret_addr return - * address. - */ - mov lr, x0 - - /* The frame pointer (x29) is restored with other registers. */ - restore_all_base_regs - - add sp, sp, #PT_REGS_SIZE - ret - -SYM_CODE_END(arch_rethook_trampoline) diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c index efb5b85b57c2..e4103e085681 100644 --- a/arch/arm64/kernel/stacktrace.c +++ b/arch/arm64/kernel/stacktrace.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -39,7 +38,7 @@ static notrace void start_backtrace(struct stackframe *frame, unsigned long fp, { frame->fp = fp; frame->pc = pc; -#if defined(CONFIG_KRETPROBES) || defined(CONFIG_RETHOOK) +#ifdef CONFIG_KRETPROBES frame->kr_cur = NULL; #endif @@ -139,10 +138,6 @@ static int notrace unwind_frame(struct task_struct *tsk, if (is_kretprobe_trampoline(frame->pc)) frame->pc = kretprobe_find_ret_addr(tsk, (void *)frame->fp, &frame->kr_cur); #endif -#ifdef CONFIG_RETHOOK - if (is_rethook_trampoline(frame->pc)) - frame->pc = rethook_find_ret_addr(tsk, frame->fp, &frame->kr_cur); -#endif return 0; } -- cgit v1.2.3 From 4e8ca13440b4b84873da44871f8824a12381d16b Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Mon, 21 Mar 2022 21:40:53 -0700 Subject: Revert "rethook: x86: Add rethook x86 implementation" This reverts commit 75caf33eda242e2f34f61e475d666359749ae5ff. Signed-off-by: Alexei Starovoitov --- arch/x86/Kconfig | 1 - arch/x86/include/asm/unwind.h | 8 +-- arch/x86/kernel/Makefile | 1 - arch/x86/kernel/kprobes/common.h | 1 - arch/x86/kernel/rethook.c | 119 --------------------------------------- 5 files changed, 1 insertion(+), 129 deletions(-) delete mode 100644 arch/x86/kernel/rethook.c (limited to 'arch') diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 1270b65a5546..9b356da6f46b 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -221,7 +221,6 @@ config X86 select HAVE_KPROBES_ON_FTRACE select HAVE_FUNCTION_ERROR_INJECTION select HAVE_KRETPROBES - select HAVE_RETHOOK select HAVE_KVM select HAVE_LIVEPATCH if X86_64 select HAVE_MIXED_BREAKPOINTS_REGS diff --git a/arch/x86/include/asm/unwind.h b/arch/x86/include/asm/unwind.h index 192df5b2094d..2a1f8734416d 100644 --- a/arch/x86/include/asm/unwind.h +++ b/arch/x86/include/asm/unwind.h @@ -5,7 +5,6 @@ #include #include #include -#include #include #include @@ -17,7 +16,7 @@ struct unwind_state { unsigned long stack_mask; struct task_struct *task; int graph_idx; -#if defined(CONFIG_KRETPROBES) || defined(CONFIG_RETHOOK) +#ifdef CONFIG_KRETPROBES struct llist_node *kr_cur; #endif bool error; @@ -108,11 +107,6 @@ static inline unsigned long unwind_recover_kretprobe(struct unwind_state *state, unsigned long addr, unsigned long *addr_p) { -#ifdef CONFIG_RETHOOK - if (is_rethook_trampoline(addr)) - return rethook_find_ret_addr(state->task, (unsigned long)addr_p, - &state->kr_cur); -#endif #ifdef CONFIG_KRETPROBES return is_kretprobe_trampoline(addr) ? kretprobe_find_ret_addr(state->task, addr_p, &state->kr_cur) : diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 792a893a5cc5..6aef9ee28a39 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -106,7 +106,6 @@ obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o obj-$(CONFIG_FTRACE_SYSCALLS) += ftrace.o obj-$(CONFIG_X86_TSC) += trace_clock.o obj-$(CONFIG_TRACING) += trace.o -obj-$(CONFIG_RETHOOK) += rethook.o obj-$(CONFIG_CRASH_CORE) += crash_core_$(BITS).o obj-$(CONFIG_KEXEC_CORE) += machine_kexec_$(BITS).o obj-$(CONFIG_KEXEC_CORE) += relocate_kernel_$(BITS).o crash.o diff --git a/arch/x86/kernel/kprobes/common.h b/arch/x86/kernel/kprobes/common.h index c993521d4933..7d3a2e2daf01 100644 --- a/arch/x86/kernel/kprobes/common.h +++ b/arch/x86/kernel/kprobes/common.h @@ -6,7 +6,6 @@ #include #include -#include #ifdef CONFIG_X86_64 diff --git a/arch/x86/kernel/rethook.c b/arch/x86/kernel/rethook.c deleted file mode 100644 index f0f2f0608282..000000000000 --- a/arch/x86/kernel/rethook.c +++ /dev/null @@ -1,119 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * x86 implementation of rethook. Mostly copied from arch/x86/kernel/kprobes/core.c. - */ -#include -#include -#include - -#include "kprobes/common.h" - -__visible void arch_rethook_trampoline_callback(struct pt_regs *regs); - -/* - * When a target function returns, this code saves registers and calls - * arch_rethook_trampoline_callback(), which calls the rethook handler. - */ -asm( - ".text\n" - ".global arch_rethook_trampoline\n" - ".type arch_rethook_trampoline, @function\n" - "arch_rethook_trampoline:\n" -#ifdef CONFIG_X86_64 - /* Push a fake return address to tell the unwinder it's a kretprobe. */ - " pushq $arch_rethook_trampoline\n" - UNWIND_HINT_FUNC - /* Save the 'sp - 8', this will be fixed later. */ - " pushq %rsp\n" - " pushfq\n" - SAVE_REGS_STRING - " movq %rsp, %rdi\n" - " call arch_rethook_trampoline_callback\n" - RESTORE_REGS_STRING - /* In the callback function, 'regs->flags' is copied to 'regs->sp'. */ - " addq $8, %rsp\n" - " popfq\n" -#else - /* Push a fake return address to tell the unwinder it's a kretprobe. */ - " pushl $arch_rethook_trampoline\n" - UNWIND_HINT_FUNC - /* Save the 'sp - 4', this will be fixed later. */ - " pushl %esp\n" - " pushfl\n" - SAVE_REGS_STRING - " movl %esp, %eax\n" - " call arch_rethook_trampoline_callback\n" - RESTORE_REGS_STRING - /* In the callback function, 'regs->flags' is copied to 'regs->sp'. */ - " addl $4, %esp\n" - " popfl\n" -#endif - " ret\n" - ".size arch_rethook_trampoline, .-arch_rethook_trampoline\n" -); -NOKPROBE_SYMBOL(arch_rethook_trampoline); - -/* - * Called from arch_rethook_trampoline - */ -__used __visible void arch_rethook_trampoline_callback(struct pt_regs *regs) -{ - unsigned long *frame_pointer; - - /* fixup registers */ - regs->cs = __KERNEL_CS; -#ifdef CONFIG_X86_32 - regs->gs = 0; -#endif - regs->ip = (unsigned long)&arch_rethook_trampoline; - regs->orig_ax = ~0UL; - regs->sp += sizeof(long); - frame_pointer = ®s->sp + 1; - - /* - * The return address at 'frame_pointer' is recovered by the - * arch_rethook_fixup_return() which called from this - * rethook_trampoline_handler(). - */ - rethook_trampoline_handler(regs, (unsigned long)frame_pointer); - - /* - * Copy FLAGS to 'pt_regs::sp' so that arch_rethook_trapmoline() - * can do RET right after POPF. - */ - regs->sp = regs->flags; -} -NOKPROBE_SYMBOL(arch_rethook_trampoline_callback); - -/* - * arch_rethook_trampoline() skips updating frame pointer. The frame pointer - * saved in arch_rethook_trampoline_callback() points to the real caller - * function's frame pointer. Thus the arch_rethook_trampoline() doesn't have - * a standard stack frame with CONFIG_FRAME_POINTER=y. - * Let's mark it non-standard function. Anyway, FP unwinder can correctly - * unwind without the hint. - */ -STACK_FRAME_NON_STANDARD_FP(arch_rethook_trampoline); - -/* This is called from rethook_trampoline_handler(). */ -void arch_rethook_fixup_return(struct pt_regs *regs, - unsigned long correct_ret_addr) -{ - unsigned long *frame_pointer = ®s->sp + 1; - - /* Replace fake return address with real one. */ - *frame_pointer = correct_ret_addr; -} -NOKPROBE_SYMBOL(arch_rethook_fixup_return); - -void arch_rethook_prepare(struct rethook_node *rh, struct pt_regs *regs, bool mcount) -{ - unsigned long *stack = (unsigned long *)regs->sp; - - rh->ret_addr = stack[0]; - rh->frame = regs->sp; - - /* Replace the return addr with trampoline addr */ - stack[0] = (unsigned long) arch_rethook_trampoline; -} -NOKPROBE_SYMBOL(arch_rethook_prepare); -- cgit v1.2.3