summaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
Diffstat (limited to 'tools')
-rw-r--r--tools/lib/bpf/features.c24
-rw-r--r--tools/lib/bpf/libbpf_internal.h2
-rw-r--r--tools/lib/bpf/usdt.c47
-rw-r--r--tools/testing/selftests/bpf/.gitignore2
-rw-r--r--tools/testing/selftests/bpf/Makefile5
-rw-r--r--tools/testing/selftests/bpf/bench.c4
-rw-r--r--tools/testing/selftests/bpf/benchs/bench_trigger.c60
-rwxr-xr-xtools/testing/selftests/bpf/benchs/run_bench_uprobes.sh2
-rw-r--r--tools/testing/selftests/bpf/prog_tests/usdt.c92
-rw-r--r--tools/testing/selftests/bpf/progs/test_usdt.c12
-rw-r--r--tools/testing/selftests/bpf/progs/trigger_bench.c10
-rw-r--r--tools/testing/selftests/bpf/usdt.h2
-rw-r--r--tools/testing/selftests/bpf/usdt_1.c18
-rw-r--r--tools/testing/selftests/bpf/usdt_2.c16
14 files changed, 289 insertions, 7 deletions
diff --git a/tools/lib/bpf/features.c b/tools/lib/bpf/features.c
index 2fa434f09cce..adcad221c601 100644
--- a/tools/lib/bpf/features.c
+++ b/tools/lib/bpf/features.c
@@ -568,6 +568,27 @@ static int probe_ldimm64_full_range_off(int token_fd)
return 1;
}
+#ifdef __x86_64__
+
+#ifndef __NR_uprobe
+#define __NR_uprobe 336
+#endif
+
+static int probe_uprobe_syscall(int token_fd)
+{
+ /*
+ * If kernel supports uprobe() syscall, it will return -ENXIO when called
+ * from the outside of a kernel-generated uprobe trampoline.
+ */
+ return syscall(__NR_uprobe) < 0 && errno == ENXIO;
+}
+#else
+static int probe_uprobe_syscall(int token_fd)
+{
+ return 0;
+}
+#endif
+
typedef int (*feature_probe_fn)(int /* token_fd */);
static struct kern_feature_cache feature_cache;
@@ -646,6 +667,9 @@ static struct kern_feature_desc {
[FEAT_LDIMM64_FULL_RANGE_OFF] = {
"full range LDIMM64 support", probe_ldimm64_full_range_off,
},
+ [FEAT_UPROBE_SYSCALL] = {
+ "kernel supports uprobe syscall", probe_uprobe_syscall,
+ },
};
bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id)
diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h
index 974147e8a8aa..4bcb6ca69bb1 100644
--- a/tools/lib/bpf/libbpf_internal.h
+++ b/tools/lib/bpf/libbpf_internal.h
@@ -394,6 +394,8 @@ enum kern_feature_id {
FEAT_BTF_QMARK_DATASEC,
/* Kernel supports LDIMM64 imm offsets past 512 MiB. */
FEAT_LDIMM64_FULL_RANGE_OFF,
+ /* Kernel supports uprobe syscall */
+ FEAT_UPROBE_SYSCALL,
__FEAT_CNT,
};
diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c
index d1524f6f54ae..e3710933fd52 100644
--- a/tools/lib/bpf/usdt.c
+++ b/tools/lib/bpf/usdt.c
@@ -262,6 +262,7 @@ struct usdt_manager {
bool has_bpf_cookie;
bool has_sema_refcnt;
bool has_uprobe_multi;
+ bool has_uprobe_syscall;
};
struct usdt_manager *usdt_manager_new(struct bpf_object *obj)
@@ -301,6 +302,13 @@ struct usdt_manager *usdt_manager_new(struct bpf_object *obj)
* usdt probes.
*/
man->has_uprobe_multi = kernel_supports(obj, FEAT_UPROBE_MULTI_LINK);
+
+ /*
+ * Detect kernel support for uprobe() syscall, it's presence means we can
+ * take advantage of faster nop5 uprobe handling.
+ * Added in: 56101b69c919 ("uprobes/x86: Add uprobe syscall to speed up uprobe")
+ */
+ man->has_uprobe_syscall = kernel_supports(obj, FEAT_UPROBE_SYSCALL);
return man;
}
@@ -585,13 +593,34 @@ static int parse_usdt_note(GElf_Nhdr *nhdr, const char *data, size_t name_off,
static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, __u64 usdt_cookie);
-static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *path, pid_t pid,
- const char *usdt_provider, const char *usdt_name, __u64 usdt_cookie,
- struct usdt_target **out_targets, size_t *out_target_cnt)
+#if defined(__x86_64__)
+static bool has_nop_combo(int fd, long off)
+{
+ unsigned char nop_combo[6] = {
+ 0x90, 0x0f, 0x1f, 0x44, 0x00, 0x00 /* nop,nop5 */
+ };
+ unsigned char buf[6];
+
+ if (pread(fd, buf, 6, off) != 6)
+ return false;
+ return memcmp(buf, nop_combo, 6) == 0;
+}
+#else
+static bool has_nop_combo(int fd, long off)
+{
+ return false;
+}
+#endif
+
+static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, const char *path,
+ pid_t pid, const char *usdt_provider, const char *usdt_name,
+ __u64 usdt_cookie, struct usdt_target **out_targets,
+ size_t *out_target_cnt)
{
size_t off, name_off, desc_off, seg_cnt = 0, vma_seg_cnt = 0, target_cnt = 0;
struct elf_seg *segs = NULL, *vma_segs = NULL;
struct usdt_target *targets = NULL, *target;
+ Elf *elf = elf_fd->elf;
long base_addr = 0;
Elf_Scn *notes_scn, *base_scn;
GElf_Shdr base_shdr, notes_shdr;
@@ -784,6 +813,16 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *
target = &targets[target_cnt];
memset(target, 0, sizeof(*target));
+ /*
+ * We have uprobe syscall and usdt with nop,nop5 instructions combo,
+ * so we can place the uprobe directly on nop5 (+1) and get this probe
+ * optimized.
+ */
+ if (man->has_uprobe_syscall && has_nop_combo(elf_fd->fd, usdt_rel_ip)) {
+ usdt_abs_ip++;
+ usdt_rel_ip++;
+ }
+
target->abs_ip = usdt_abs_ip;
target->rel_ip = usdt_rel_ip;
target->sema_off = usdt_sema_off;
@@ -998,7 +1037,7 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct
/* discover USDT in given binary, optionally limiting
* activations to a given PID, if pid > 0
*/
- err = collect_usdt_targets(man, elf_fd.elf, path, pid, usdt_provider, usdt_name,
+ err = collect_usdt_targets(man, &elf_fd, path, pid, usdt_provider, usdt_name,
usdt_cookie, &targets, &target_cnt);
if (err <= 0) {
err = (err == 0) ? -ENOENT : err;
diff --git a/tools/testing/selftests/bpf/.gitignore b/tools/testing/selftests/bpf/.gitignore
index a3ea98211ea6..bfdc5518ecc8 100644
--- a/tools/testing/selftests/bpf/.gitignore
+++ b/tools/testing/selftests/bpf/.gitignore
@@ -47,3 +47,5 @@ verification_cert.h
*.BTF
*.BTF_ids
*.BTF.base
+usdt_1
+usdt_2
diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile
index 72a9ba41f95e..49455ad51d66 100644
--- a/tools/testing/selftests/bpf/Makefile
+++ b/tools/testing/selftests/bpf/Makefile
@@ -754,7 +754,8 @@ TRUNNER_EXTRA_SOURCES := test_progs.c \
$(VERIFY_SIG_HDR) \
flow_dissector_load.h \
ip_check_defrag_frags.h \
- bpftool_helpers.c
+ bpftool_helpers.c \
+ usdt_1.c usdt_2.c
TRUNNER_LIB_SOURCES := find_bit.c
TRUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \
$(OUTPUT)/liburandom_read.so \
@@ -878,6 +879,8 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \
$(OUTPUT)/bench_bpf_crypto.o \
$(OUTPUT)/bench_sockmap.o \
$(OUTPUT)/bench_lpm_trie_map.o \
+ $(OUTPUT)/usdt_1.o \
+ $(OUTPUT)/usdt_2.o \
#
$(call msg,BINARY,,$@)
$(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@
diff --git a/tools/testing/selftests/bpf/bench.c b/tools/testing/selftests/bpf/bench.c
index 8368bd3a0665..029b3e21f438 100644
--- a/tools/testing/selftests/bpf/bench.c
+++ b/tools/testing/selftests/bpf/bench.c
@@ -541,6 +541,8 @@ extern const struct bench bench_trig_uprobe_nop5;
extern const struct bench bench_trig_uretprobe_nop5;
extern const struct bench bench_trig_uprobe_multi_nop5;
extern const struct bench bench_trig_uretprobe_multi_nop5;
+extern const struct bench bench_trig_usdt_nop;
+extern const struct bench bench_trig_usdt_nop5;
#endif
extern const struct bench bench_rb_libbpf;
@@ -617,6 +619,8 @@ static const struct bench *benchs[] = {
&bench_trig_uretprobe_nop5,
&bench_trig_uprobe_multi_nop5,
&bench_trig_uretprobe_multi_nop5,
+ &bench_trig_usdt_nop,
+ &bench_trig_usdt_nop5,
#endif
/* ringbuf/perfbuf benchmarks */
&bench_rb_libbpf,
diff --git a/tools/testing/selftests/bpf/benchs/bench_trigger.c b/tools/testing/selftests/bpf/benchs/bench_trigger.c
index f74b313d6ae4..2f22ec61667b 100644
--- a/tools/testing/selftests/bpf/benchs/bench_trigger.c
+++ b/tools/testing/selftests/bpf/benchs/bench_trigger.c
@@ -407,6 +407,23 @@ static void *uprobe_producer_nop5(void *input)
uprobe_target_nop5();
return NULL;
}
+
+void usdt_1(void);
+void usdt_2(void);
+
+static void *uprobe_producer_usdt_nop(void *input)
+{
+ while (true)
+ usdt_1();
+ return NULL;
+}
+
+static void *uprobe_producer_usdt_nop5(void *input)
+{
+ while (true)
+ usdt_2();
+ return NULL;
+}
#endif
static void usetup(bool use_retprobe, bool use_multi, void *target_addr)
@@ -544,6 +561,47 @@ static void uretprobe_multi_nop5_setup(void)
{
usetup(true, true /* use_multi */, &uprobe_target_nop5);
}
+
+static void usdt_setup(const char *name)
+{
+ struct bpf_link *link;
+ int err;
+
+ setup_libbpf();
+
+ ctx.skel = trigger_bench__open();
+ if (!ctx.skel) {
+ fprintf(stderr, "failed to open skeleton\n");
+ exit(1);
+ }
+
+ bpf_program__set_autoload(ctx.skel->progs.bench_trigger_usdt, true);
+
+ err = trigger_bench__load(ctx.skel);
+ if (err) {
+ fprintf(stderr, "failed to load skeleton\n");
+ exit(1);
+ }
+
+ link = bpf_program__attach_usdt(ctx.skel->progs.bench_trigger_usdt,
+ 0 /*self*/, "/proc/self/exe",
+ "optimized_attach", name, NULL);
+ if (libbpf_get_error(link)) {
+ fprintf(stderr, "failed to attach optimized_attach:%s usdt probe\n", name);
+ exit(1);
+ }
+ ctx.skel->links.bench_trigger_usdt = link;
+}
+
+static void usdt_nop_setup(void)
+{
+ usdt_setup("usdt_1");
+}
+
+static void usdt_nop5_setup(void)
+{
+ usdt_setup("usdt_2");
+}
#endif
const struct bench bench_trig_syscall_count = {
@@ -611,4 +669,6 @@ BENCH_TRIG_USERMODE(uprobe_nop5, nop5, "uprobe-nop5");
BENCH_TRIG_USERMODE(uretprobe_nop5, nop5, "uretprobe-nop5");
BENCH_TRIG_USERMODE(uprobe_multi_nop5, nop5, "uprobe-multi-nop5");
BENCH_TRIG_USERMODE(uretprobe_multi_nop5, nop5, "uretprobe-multi-nop5");
+BENCH_TRIG_USERMODE(usdt_nop, usdt_nop, "usdt-nop");
+BENCH_TRIG_USERMODE(usdt_nop5, usdt_nop5, "usdt-nop5");
#endif
diff --git a/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh b/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh
index 03f55405484b..9ec59423b949 100755
--- a/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh
+++ b/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh
@@ -2,7 +2,7 @@
set -eufo pipefail
-for i in usermode-count syscall-count {uprobe,uretprobe}-{nop,push,ret,nop5}
+for i in usermode-count syscall-count {uprobe,uretprobe}-{nop,push,ret,nop5} usdt-nop usdt-nop5
do
summary=$(sudo ./bench -w2 -d5 -a trig-$i | tail -n1 | cut -d'(' -f1 | cut -d' ' -f3-)
printf "%-15s: %s\n" $i "$summary"
diff --git a/tools/testing/selftests/bpf/prog_tests/usdt.c b/tools/testing/selftests/bpf/prog_tests/usdt.c
index f4be5269fa90..69759b27794d 100644
--- a/tools/testing/selftests/bpf/prog_tests/usdt.c
+++ b/tools/testing/selftests/bpf/prog_tests/usdt.c
@@ -247,6 +247,96 @@ cleanup:
#undef TRIGGER
}
+#ifdef __x86_64__
+extern void usdt_1(void);
+extern void usdt_2(void);
+
+static unsigned char nop1[1] = { 0x90 };
+static unsigned char nop1_nop5_combo[6] = { 0x90, 0x0f, 0x1f, 0x44, 0x00, 0x00 };
+
+static void *find_instr(void *fn, unsigned char *instr, size_t cnt)
+{
+ int i;
+
+ for (i = 0; i < 10; i++) {
+ if (!memcmp(instr, fn + i, cnt))
+ return fn + i;
+ }
+ return NULL;
+}
+
+static void subtest_optimized_attach(void)
+{
+ struct test_usdt *skel;
+ __u8 *addr_1, *addr_2;
+
+ /* usdt_1 USDT probe has single nop instruction */
+ addr_1 = find_instr(usdt_1, nop1_nop5_combo, 6);
+ if (!ASSERT_NULL(addr_1, "usdt_1_find_nop1_nop5_combo"))
+ return;
+
+ addr_1 = find_instr(usdt_1, nop1, 1);
+ if (!ASSERT_OK_PTR(addr_1, "usdt_1_find_nop1"))
+ return;
+
+ /* usdt_2 USDT probe has nop,nop5 instructions combo */
+ addr_2 = find_instr(usdt_2, nop1_nop5_combo, 6);
+ if (!ASSERT_OK_PTR(addr_2, "usdt_2_find_nop1_nop5_combo"))
+ return;
+
+ skel = test_usdt__open_and_load();
+ if (!ASSERT_OK_PTR(skel, "test_usdt__open_and_load"))
+ return;
+
+ skel->bss->expected_ip = (unsigned long) addr_1;
+
+ /*
+ * Attach program on top of usdt_1 which is single nop probe,
+ * so the probe won't get optimized.
+ */
+ skel->links.usdt_executed = bpf_program__attach_usdt(skel->progs.usdt_executed,
+ 0 /*self*/, "/proc/self/exe",
+ "optimized_attach", "usdt_1", NULL);
+ if (!ASSERT_OK_PTR(skel->links.usdt_executed, "bpf_program__attach_usdt"))
+ goto cleanup;
+
+ usdt_1();
+ usdt_1();
+
+ /* int3 is on addr_1 address */
+ ASSERT_EQ(*addr_1, 0xcc, "int3");
+ ASSERT_EQ(skel->bss->executed, 2, "executed");
+
+ bpf_link__destroy(skel->links.usdt_executed);
+
+ /* we expect the nop5 ip */
+ skel->bss->expected_ip = (unsigned long) addr_2 + 1;
+
+ /*
+ * Attach program on top of usdt_2 which is probe defined on top
+ * of nop1,nop5 combo, so the probe gets optimized on top of nop5.
+ */
+ skel->links.usdt_executed = bpf_program__attach_usdt(skel->progs.usdt_executed,
+ 0 /*self*/, "/proc/self/exe",
+ "optimized_attach", "usdt_2", NULL);
+ if (!ASSERT_OK_PTR(skel->links.usdt_executed, "bpf_program__attach_usdt"))
+ goto cleanup;
+
+ usdt_2();
+ usdt_2();
+
+ /* nop stays on addr_2 address */
+ ASSERT_EQ(*addr_2, 0x90, "nop");
+
+ /* call is on addr_2 + 1 address */
+ ASSERT_EQ(*(addr_2 + 1), 0xe8, "call");
+ ASSERT_EQ(skel->bss->executed, 4, "executed");
+
+cleanup:
+ test_usdt__destroy(skel);
+}
+#endif
+
unsigned short test_usdt_100_semaphore SEC(".probes");
unsigned short test_usdt_300_semaphore SEC(".probes");
unsigned short test_usdt_400_semaphore SEC(".probes");
@@ -516,6 +606,8 @@ void test_usdt(void)
#ifdef __x86_64__
if (test__start_subtest("basic_optimized"))
subtest_basic_usdt(true);
+ if (test__start_subtest("optimized_attach"))
+ subtest_optimized_attach();
#endif
if (test__start_subtest("multispec"))
subtest_multispec_usdt();
diff --git a/tools/testing/selftests/bpf/progs/test_usdt.c b/tools/testing/selftests/bpf/progs/test_usdt.c
index a78c87537b07..f00cb52874e0 100644
--- a/tools/testing/selftests/bpf/progs/test_usdt.c
+++ b/tools/testing/selftests/bpf/progs/test_usdt.c
@@ -138,4 +138,16 @@ int usdt_sib(struct pt_regs *ctx)
return 0;
}
+#ifdef __TARGET_ARCH_x86
+int executed;
+unsigned long expected_ip;
+
+SEC("usdt")
+int usdt_executed(struct pt_regs *ctx)
+{
+ if (expected_ip == ctx->ip)
+ executed++;
+ return 0;
+}
+#endif
char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/trigger_bench.c b/tools/testing/selftests/bpf/progs/trigger_bench.c
index 4ea0422d1042..3225b4aee8ff 100644
--- a/tools/testing/selftests/bpf/progs/trigger_bench.c
+++ b/tools/testing/selftests/bpf/progs/trigger_bench.c
@@ -1,10 +1,11 @@
// SPDX-License-Identifier: GPL-2.0
// Copyright (c) 2020 Facebook
-#include <linux/bpf.h>
+#include "vmlinux.h"
#include <asm/unistd.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include "bpf_misc.h"
+#include "bpf/usdt.bpf.h"
char _license[] SEC("license") = "GPL";
@@ -180,3 +181,10 @@ int bench_trigger_rawtp(void *ctx)
handle(ctx);
return 0;
}
+
+SEC("?usdt")
+int bench_trigger_usdt(void *ctx)
+{
+ inc_counter();
+ return 0;
+}
diff --git a/tools/testing/selftests/bpf/usdt.h b/tools/testing/selftests/bpf/usdt.h
index 549d1f774810..c71e21df38b3 100644
--- a/tools/testing/selftests/bpf/usdt.h
+++ b/tools/testing/selftests/bpf/usdt.h
@@ -312,6 +312,8 @@ struct usdt_sema { volatile unsigned short active; };
#ifndef USDT_NOP
#if defined(__ia64__) || defined(__s390__) || defined(__s390x__)
#define USDT_NOP nop 0
+#elif defined(__x86_64__)
+#define USDT_NOP .byte 0x90, 0x0f, 0x1f, 0x44, 0x00, 0x0 /* nop, nop5 */
#else
#define USDT_NOP nop
#endif
diff --git a/tools/testing/selftests/bpf/usdt_1.c b/tools/testing/selftests/bpf/usdt_1.c
new file mode 100644
index 000000000000..4f06e8bcf58b
--- /dev/null
+++ b/tools/testing/selftests/bpf/usdt_1.c
@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#if defined(__x86_64__)
+
+/*
+ * Include usdt.h with defined USDT_NOP macro to use single
+ * nop instruction.
+ */
+#define USDT_NOP .byte 0x90
+#include "usdt.h"
+
+__attribute__((aligned(16)))
+void usdt_1(void)
+{
+ USDT(optimized_attach, usdt_1);
+}
+
+#endif
diff --git a/tools/testing/selftests/bpf/usdt_2.c b/tools/testing/selftests/bpf/usdt_2.c
new file mode 100644
index 000000000000..789883aaca4c
--- /dev/null
+++ b/tools/testing/selftests/bpf/usdt_2.c
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#if defined(__x86_64__)
+
+/*
+ * Include usdt.h with default nop,nop5 instructions combo.
+ */
+#include "usdt.h"
+
+__attribute__((aligned(16)))
+void usdt_2(void)
+{
+ USDT(optimized_attach, usdt_2);
+}
+
+#endif
generated by cgit v1.2.3 (git 2.39.0) at 2026-05-13 18:26:43 +0300