summaryrefslogtreecommitdiff
path: root/tools/lib/bpf
diff options
context:
space:
mode:
Diffstat (limited to 'tools/lib/bpf')
-rw-r--r--tools/lib/bpf/Build2
-rw-r--r--tools/lib/bpf/Makefile4
-rw-r--r--tools/lib/bpf/bpf.c146
-rw-r--r--tools/lib/bpf/bpf.h114
-rw-r--r--tools/lib/bpf/bpf_tracing.h2
-rw-r--r--tools/lib/bpf/elf.c440
-rw-r--r--tools/lib/bpf/hashmap.h10
-rw-r--r--tools/lib/bpf/libbpf.c756
-rw-r--r--tools/lib/bpf/libbpf.h85
-rw-r--r--tools/lib/bpf/libbpf.map5
-rw-r--r--tools/lib/bpf/libbpf_common.h16
-rw-r--r--tools/lib/bpf/libbpf_internal.h21
-rw-r--r--tools/lib/bpf/netlink.c5
-rw-r--r--tools/lib/bpf/relo_core.c2
-rw-r--r--tools/lib/bpf/usdt.bpf.h4
-rw-r--r--tools/lib/bpf/usdt.c121
16 files changed, 1397 insertions, 336 deletions
diff --git a/tools/lib/bpf/Build b/tools/lib/bpf/Build
index b8b0a6369363..2d0c282c8588 100644
--- a/tools/lib/bpf/Build
+++ b/tools/lib/bpf/Build
@@ -1,4 +1,4 @@
libbpf-y := libbpf.o bpf.o nlattr.o btf.o libbpf_errno.o str_error.o \
netlink.o bpf_prog_linfo.o libbpf_probes.o hashmap.o \
btf_dump.o ringbuf.o strset.o linker.o gen_loader.o relo_core.o \
- usdt.o zip.o
+ usdt.o zip.o elf.o
diff --git a/tools/lib/bpf/Makefile b/tools/lib/bpf/Makefile
index cf7f02c67968..4be7144e4803 100644
--- a/tools/lib/bpf/Makefile
+++ b/tools/lib/bpf/Makefile
@@ -293,11 +293,11 @@ help:
@echo ' HINT: use "V=1" to enable verbose build'
@echo ' all - build libraries and pkgconfig'
@echo ' clean - remove all generated files'
- @echo ' check - check abi and version info'
+ @echo ' check - check ABI and version info'
@echo ''
@echo 'libbpf install targets:'
@echo ' HINT: use "prefix"(defaults to "/usr/local") or "DESTDIR" (defaults to "/")'
- @echo ' to adjust target desitantion, e.g. "make prefix=/usr/local install"'
+ @echo ' to adjust target destination, e.g. "make prefix=/usr/local install"'
@echo ' install - build and install all headers, libraries and pkgconfig'
@echo ' install_headers - install only headers to include/bpf'
@echo ''
diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c
index ed86b37d8024..b0f1913763a3 100644
--- a/tools/lib/bpf/bpf.c
+++ b/tools/lib/bpf/bpf.c
@@ -629,55 +629,89 @@ int bpf_prog_attach(int prog_fd, int target_fd, enum bpf_attach_type type,
return bpf_prog_attach_opts(prog_fd, target_fd, type, &opts);
}
-int bpf_prog_attach_opts(int prog_fd, int target_fd,
- enum bpf_attach_type type,
- const struct bpf_prog_attach_opts *opts)
+int bpf_prog_attach_opts(int prog_fd, int target, enum bpf_attach_type type,
+ const struct bpf_prog_attach_opts *opts)
{
- const size_t attr_sz = offsetofend(union bpf_attr, replace_bpf_fd);
+ const size_t attr_sz = offsetofend(union bpf_attr, expected_revision);
+ __u32 relative_id, flags;
+ int ret, relative_fd;
union bpf_attr attr;
- int ret;
if (!OPTS_VALID(opts, bpf_prog_attach_opts))
return libbpf_err(-EINVAL);
+ relative_id = OPTS_GET(opts, relative_id, 0);
+ relative_fd = OPTS_GET(opts, relative_fd, 0);
+ flags = OPTS_GET(opts, flags, 0);
+
+ /* validate we don't have unexpected combinations of non-zero fields */
+ if (relative_fd && relative_id)
+ return libbpf_err(-EINVAL);
+
memset(&attr, 0, attr_sz);
- attr.target_fd = target_fd;
- attr.attach_bpf_fd = prog_fd;
- attr.attach_type = type;
- attr.attach_flags = OPTS_GET(opts, flags, 0);
- attr.replace_bpf_fd = OPTS_GET(opts, replace_prog_fd, 0);
+ attr.target_fd = target;
+ attr.attach_bpf_fd = prog_fd;
+ attr.attach_type = type;
+ attr.replace_bpf_fd = OPTS_GET(opts, replace_fd, 0);
+ attr.expected_revision = OPTS_GET(opts, expected_revision, 0);
+
+ if (relative_id) {
+ attr.attach_flags = flags | BPF_F_ID;
+ attr.relative_id = relative_id;
+ } else {
+ attr.attach_flags = flags;
+ attr.relative_fd = relative_fd;
+ }
ret = sys_bpf(BPF_PROG_ATTACH, &attr, attr_sz);
return libbpf_err_errno(ret);
}
-int bpf_prog_detach(int target_fd, enum bpf_attach_type type)
+int bpf_prog_detach_opts(int prog_fd, int target, enum bpf_attach_type type,
+ const struct bpf_prog_detach_opts *opts)
{
- const size_t attr_sz = offsetofend(union bpf_attr, replace_bpf_fd);
+ const size_t attr_sz = offsetofend(union bpf_attr, expected_revision);
+ __u32 relative_id, flags;
+ int ret, relative_fd;
union bpf_attr attr;
- int ret;
+
+ if (!OPTS_VALID(opts, bpf_prog_detach_opts))
+ return libbpf_err(-EINVAL);
+
+ relative_id = OPTS_GET(opts, relative_id, 0);
+ relative_fd = OPTS_GET(opts, relative_fd, 0);
+ flags = OPTS_GET(opts, flags, 0);
+
+ /* validate we don't have unexpected combinations of non-zero fields */
+ if (relative_fd && relative_id)
+ return libbpf_err(-EINVAL);
memset(&attr, 0, attr_sz);
- attr.target_fd = target_fd;
- attr.attach_type = type;
+ attr.target_fd = target;
+ attr.attach_bpf_fd = prog_fd;
+ attr.attach_type = type;
+ attr.expected_revision = OPTS_GET(opts, expected_revision, 0);
+
+ if (relative_id) {
+ attr.attach_flags = flags | BPF_F_ID;
+ attr.relative_id = relative_id;
+ } else {
+ attr.attach_flags = flags;
+ attr.relative_fd = relative_fd;
+ }
ret = sys_bpf(BPF_PROG_DETACH, &attr, attr_sz);
return libbpf_err_errno(ret);
}
-int bpf_prog_detach2(int prog_fd, int target_fd, enum bpf_attach_type type)
+int bpf_prog_detach(int target_fd, enum bpf_attach_type type)
{
- const size_t attr_sz = offsetofend(union bpf_attr, replace_bpf_fd);
- union bpf_attr attr;
- int ret;
-
- memset(&attr, 0, attr_sz);
- attr.target_fd = target_fd;
- attr.attach_bpf_fd = prog_fd;
- attr.attach_type = type;
+ return bpf_prog_detach_opts(0, target_fd, type, NULL);
+}
- ret = sys_bpf(BPF_PROG_DETACH, &attr, attr_sz);
- return libbpf_err_errno(ret);
+int bpf_prog_detach2(int prog_fd, int target_fd, enum bpf_attach_type type)
+{
+ return bpf_prog_detach_opts(prog_fd, target_fd, type, NULL);
}
int bpf_link_create(int prog_fd, int target_fd,
@@ -685,9 +719,9 @@ int bpf_link_create(int prog_fd, int target_fd,
const struct bpf_link_create_opts *opts)
{
const size_t attr_sz = offsetofend(union bpf_attr, link_create);
- __u32 target_btf_id, iter_info_len;
+ __u32 target_btf_id, iter_info_len, relative_id;
+ int fd, err, relative_fd;
union bpf_attr attr;
- int fd, err;
if (!OPTS_VALID(opts, bpf_link_create_opts))
return libbpf_err(-EINVAL);
@@ -733,6 +767,17 @@ int bpf_link_create(int prog_fd, int target_fd,
if (!OPTS_ZEROED(opts, kprobe_multi))
return libbpf_err(-EINVAL);
break;
+ case BPF_TRACE_UPROBE_MULTI:
+ attr.link_create.uprobe_multi.flags = OPTS_GET(opts, uprobe_multi.flags, 0);
+ attr.link_create.uprobe_multi.cnt = OPTS_GET(opts, uprobe_multi.cnt, 0);
+ attr.link_create.uprobe_multi.path = ptr_to_u64(OPTS_GET(opts, uprobe_multi.path, 0));
+ attr.link_create.uprobe_multi.offsets = ptr_to_u64(OPTS_GET(opts, uprobe_multi.offsets, 0));
+ attr.link_create.uprobe_multi.ref_ctr_offsets = ptr_to_u64(OPTS_GET(opts, uprobe_multi.ref_ctr_offsets, 0));
+ attr.link_create.uprobe_multi.cookies = ptr_to_u64(OPTS_GET(opts, uprobe_multi.cookies, 0));
+ attr.link_create.uprobe_multi.pid = OPTS_GET(opts, uprobe_multi.pid, 0);
+ if (!OPTS_ZEROED(opts, uprobe_multi))
+ return libbpf_err(-EINVAL);
+ break;
case BPF_TRACE_FENTRY:
case BPF_TRACE_FEXIT:
case BPF_MODIFY_RETURN:
@@ -741,6 +786,30 @@ int bpf_link_create(int prog_fd, int target_fd,
if (!OPTS_ZEROED(opts, tracing))
return libbpf_err(-EINVAL);
break;
+ case BPF_NETFILTER:
+ attr.link_create.netfilter.pf = OPTS_GET(opts, netfilter.pf, 0);
+ attr.link_create.netfilter.hooknum = OPTS_GET(opts, netfilter.hooknum, 0);
+ attr.link_create.netfilter.priority = OPTS_GET(opts, netfilter.priority, 0);
+ attr.link_create.netfilter.flags = OPTS_GET(opts, netfilter.flags, 0);
+ if (!OPTS_ZEROED(opts, netfilter))
+ return libbpf_err(-EINVAL);
+ break;
+ case BPF_TCX_INGRESS:
+ case BPF_TCX_EGRESS:
+ relative_fd = OPTS_GET(opts, tcx.relative_fd, 0);
+ relative_id = OPTS_GET(opts, tcx.relative_id, 0);
+ if (relative_fd && relative_id)
+ return libbpf_err(-EINVAL);
+ if (relative_id) {
+ attr.link_create.tcx.relative_id = relative_id;
+ attr.link_create.flags |= BPF_F_ID;
+ } else {
+ attr.link_create.tcx.relative_fd = relative_fd;
+ }
+ attr.link_create.tcx.expected_revision = OPTS_GET(opts, tcx.expected_revision, 0);
+ if (!OPTS_ZEROED(opts, tcx))
+ return libbpf_err(-EINVAL);
+ break;
default:
if (!OPTS_ZEROED(opts, flags))
return libbpf_err(-EINVAL);
@@ -833,8 +902,7 @@ int bpf_iter_create(int link_fd)
return libbpf_err_errno(fd);
}
-int bpf_prog_query_opts(int target_fd,
- enum bpf_attach_type type,
+int bpf_prog_query_opts(int target, enum bpf_attach_type type,
struct bpf_prog_query_opts *opts)
{
const size_t attr_sz = offsetofend(union bpf_attr, query);
@@ -845,18 +913,20 @@ int bpf_prog_query_opts(int target_fd,
return libbpf_err(-EINVAL);
memset(&attr, 0, attr_sz);
-
- attr.query.target_fd = target_fd;
- attr.query.attach_type = type;
- attr.query.query_flags = OPTS_GET(opts, query_flags, 0);
- attr.query.prog_cnt = OPTS_GET(opts, prog_cnt, 0);
- attr.query.prog_ids = ptr_to_u64(OPTS_GET(opts, prog_ids, NULL));
- attr.query.prog_attach_flags = ptr_to_u64(OPTS_GET(opts, prog_attach_flags, NULL));
+ attr.query.target_fd = target;
+ attr.query.attach_type = type;
+ attr.query.query_flags = OPTS_GET(opts, query_flags, 0);
+ attr.query.count = OPTS_GET(opts, count, 0);
+ attr.query.prog_ids = ptr_to_u64(OPTS_GET(opts, prog_ids, NULL));
+ attr.query.link_ids = ptr_to_u64(OPTS_GET(opts, link_ids, NULL));
+ attr.query.prog_attach_flags = ptr_to_u64(OPTS_GET(opts, prog_attach_flags, NULL));
+ attr.query.link_attach_flags = ptr_to_u64(OPTS_GET(opts, link_attach_flags, NULL));
ret = sys_bpf(BPF_PROG_QUERY, &attr, attr_sz);
OPTS_SET(opts, attach_flags, attr.query.attach_flags);
- OPTS_SET(opts, prog_cnt, attr.query.prog_cnt);
+ OPTS_SET(opts, revision, attr.query.revision);
+ OPTS_SET(opts, count, attr.query.count);
return libbpf_err_errno(ret);
}
diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h
index 9aa0ee473754..74c2887cfd24 100644
--- a/tools/lib/bpf/bpf.h
+++ b/tools/lib/bpf/bpf.h
@@ -312,22 +312,68 @@ LIBBPF_API int bpf_obj_get(const char *pathname);
LIBBPF_API int bpf_obj_get_opts(const char *pathname,
const struct bpf_obj_get_opts *opts);
-struct bpf_prog_attach_opts {
- size_t sz; /* size of this struct for forward/backward compatibility */
- unsigned int flags;
- int replace_prog_fd;
-};
-#define bpf_prog_attach_opts__last_field replace_prog_fd
-
LIBBPF_API int bpf_prog_attach(int prog_fd, int attachable_fd,
enum bpf_attach_type type, unsigned int flags);
-LIBBPF_API int bpf_prog_attach_opts(int prog_fd, int attachable_fd,
- enum bpf_attach_type type,
- const struct bpf_prog_attach_opts *opts);
LIBBPF_API int bpf_prog_detach(int attachable_fd, enum bpf_attach_type type);
LIBBPF_API int bpf_prog_detach2(int prog_fd, int attachable_fd,
enum bpf_attach_type type);
+struct bpf_prog_attach_opts {
+ size_t sz; /* size of this struct for forward/backward compatibility */
+ __u32 flags;
+ union {
+ int replace_prog_fd;
+ int replace_fd;
+ };
+ int relative_fd;
+ __u32 relative_id;
+ __u64 expected_revision;
+ size_t :0;
+};
+#define bpf_prog_attach_opts__last_field expected_revision
+
+struct bpf_prog_detach_opts {
+ size_t sz; /* size of this struct for forward/backward compatibility */
+ __u32 flags;
+ int relative_fd;
+ __u32 relative_id;
+ __u64 expected_revision;
+ size_t :0;
+};
+#define bpf_prog_detach_opts__last_field expected_revision
+
+/**
+ * @brief **bpf_prog_attach_opts()** attaches the BPF program corresponding to
+ * *prog_fd* to a *target* which can represent a file descriptor or netdevice
+ * ifindex.
+ *
+ * @param prog_fd BPF program file descriptor
+ * @param target attach location file descriptor or ifindex
+ * @param type attach type for the BPF program
+ * @param opts options for configuring the attachment
+ * @return 0, on success; negative error code, otherwise (errno is also set to
+ * the error code)
+ */
+LIBBPF_API int bpf_prog_attach_opts(int prog_fd, int target,
+ enum bpf_attach_type type,
+ const struct bpf_prog_attach_opts *opts);
+
+/**
+ * @brief **bpf_prog_detach_opts()** detaches the BPF program corresponding to
+ * *prog_fd* from a *target* which can represent a file descriptor or netdevice
+ * ifindex.
+ *
+ * @param prog_fd BPF program file descriptor
+ * @param target detach location file descriptor or ifindex
+ * @param type detach type for the BPF program
+ * @param opts options for configuring the detachment
+ * @return 0, on success; negative error code, otherwise (errno is also set to
+ * the error code)
+ */
+LIBBPF_API int bpf_prog_detach_opts(int prog_fd, int target,
+ enum bpf_attach_type type,
+ const struct bpf_prog_detach_opts *opts);
+
union bpf_iter_link_info; /* defined in up-to-date linux/bpf.h */
struct bpf_link_create_opts {
size_t sz; /* size of this struct for forward/backward compatibility */
@@ -347,12 +393,32 @@ struct bpf_link_create_opts {
const __u64 *cookies;
} kprobe_multi;
struct {
+ __u32 flags;
+ __u32 cnt;
+ const char *path;
+ const unsigned long *offsets;
+ const unsigned long *ref_ctr_offsets;
+ const __u64 *cookies;
+ __u32 pid;
+ } uprobe_multi;
+ struct {
__u64 cookie;
} tracing;
+ struct {
+ __u32 pf;
+ __u32 hooknum;
+ __s32 priority;
+ __u32 flags;
+ } netfilter;
+ struct {
+ __u32 relative_fd;
+ __u32 relative_id;
+ __u64 expected_revision;
+ } tcx;
};
size_t :0;
};
-#define bpf_link_create_opts__last_field kprobe_multi.cookies
+#define bpf_link_create_opts__last_field uprobe_multi.pid
LIBBPF_API int bpf_link_create(int prog_fd, int target_fd,
enum bpf_attach_type attach_type,
@@ -489,13 +555,31 @@ struct bpf_prog_query_opts {
__u32 query_flags;
__u32 attach_flags; /* output argument */
__u32 *prog_ids;
- __u32 prog_cnt; /* input+output argument */
+ union {
+ /* input+output argument */
+ __u32 prog_cnt;
+ __u32 count;
+ };
__u32 *prog_attach_flags;
+ __u32 *link_ids;
+ __u32 *link_attach_flags;
+ __u64 revision;
+ size_t :0;
};
-#define bpf_prog_query_opts__last_field prog_attach_flags
+#define bpf_prog_query_opts__last_field revision
-LIBBPF_API int bpf_prog_query_opts(int target_fd,
- enum bpf_attach_type type,
+/**
+ * @brief **bpf_prog_query_opts()** queries the BPF programs and BPF links
+ * which are attached to *target* which can represent a file descriptor or
+ * netdevice ifindex.
+ *
+ * @param target query location file descriptor or ifindex
+ * @param type attach type for the BPF program
+ * @param opts options for configuring the query
+ * @return 0, on success; negative error code, otherwise (errno is also set to
+ * the error code)
+ */
+LIBBPF_API int bpf_prog_query_opts(int target, enum bpf_attach_type type,
struct bpf_prog_query_opts *opts);
LIBBPF_API int bpf_prog_query(int target_fd, enum bpf_attach_type type,
__u32 query_flags, __u32 *attach_flags,
diff --git a/tools/lib/bpf/bpf_tracing.h b/tools/lib/bpf/bpf_tracing.h
index be076a4041ab..3803479dbe10 100644
--- a/tools/lib/bpf/bpf_tracing.h
+++ b/tools/lib/bpf/bpf_tracing.h
@@ -2,7 +2,7 @@
#ifndef __BPF_TRACING_H__
#define __BPF_TRACING_H__
-#include <bpf/bpf_helpers.h>
+#include "bpf_helpers.h"
/* Scan the ARCH passed in from ARCH env variable (see Makefile) */
#if defined(__TARGET_ARCH_x86)
diff --git a/tools/lib/bpf/elf.c b/tools/lib/bpf/elf.c
new file mode 100644
index 000000000000..9d0296c1726a
--- /dev/null
+++ b/tools/lib/bpf/elf.c
@@ -0,0 +1,440 @@
+// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
+
+#include <libelf.h>
+#include <gelf.h>
+#include <fcntl.h>
+#include <linux/kernel.h>
+
+#include "libbpf_internal.h"
+#include "str_error.h"
+
+#define STRERR_BUFSIZE 128
+
+int elf_open(const char *binary_path, struct elf_fd *elf_fd)
+{
+ char errmsg[STRERR_BUFSIZE];
+ int fd, ret;
+ Elf *elf;
+
+ if (elf_version(EV_CURRENT) == EV_NONE) {
+ pr_warn("elf: failed to init libelf for %s\n", binary_path);
+ return -LIBBPF_ERRNO__LIBELF;
+ }
+ fd = open(binary_path, O_RDONLY | O_CLOEXEC);
+ if (fd < 0) {
+ ret = -errno;
+ pr_warn("elf: failed to open %s: %s\n", binary_path,
+ libbpf_strerror_r(ret, errmsg, sizeof(errmsg)));
+ return ret;
+ }
+ elf = elf_begin(fd, ELF_C_READ_MMAP, NULL);
+ if (!elf) {
+ pr_warn("elf: could not read elf from %s: %s\n", binary_path, elf_errmsg(-1));
+ close(fd);
+ return -LIBBPF_ERRNO__FORMAT;
+ }
+ elf_fd->fd = fd;
+ elf_fd->elf = elf;
+ return 0;
+}
+
+void elf_close(struct elf_fd *elf_fd)
+{
+ if (!elf_fd)
+ return;
+ elf_end(elf_fd->elf);
+ close(elf_fd->fd);
+}
+
+/* Return next ELF section of sh_type after scn, or first of that type if scn is NULL. */
+static Elf_Scn *elf_find_next_scn_by_type(Elf *elf, int sh_type, Elf_Scn *scn)
+{
+ while ((scn = elf_nextscn(elf, scn)) != NULL) {
+ GElf_Shdr sh;
+
+ if (!gelf_getshdr(scn, &sh))
+ continue;
+ if (sh.sh_type == sh_type)
+ return scn;
+ }
+ return NULL;
+}
+
+struct elf_sym {
+ const char *name;
+ GElf_Sym sym;
+ GElf_Shdr sh;
+};
+
+struct elf_sym_iter {
+ Elf *elf;
+ Elf_Data *syms;
+ size_t nr_syms;
+ size_t strtabidx;
+ size_t next_sym_idx;
+ struct elf_sym sym;
+ int st_type;
+};
+
+static int elf_sym_iter_new(struct elf_sym_iter *iter,
+ Elf *elf, const char *binary_path,
+ int sh_type, int st_type)
+{
+ Elf_Scn *scn = NULL;
+ GElf_Ehdr ehdr;
+ GElf_Shdr sh;
+
+ memset(iter, 0, sizeof(*iter));
+
+ if (!gelf_getehdr(elf, &ehdr)) {
+ pr_warn("elf: failed to get ehdr from %s: %s\n", binary_path, elf_errmsg(-1));
+ return -EINVAL;
+ }
+
+ scn = elf_find_next_scn_by_type(elf, sh_type, NULL);
+ if (!scn) {
+ pr_debug("elf: failed to find symbol table ELF sections in '%s'\n",
+ binary_path);
+ return -ENOENT;
+ }
+
+ if (!gelf_getshdr(scn, &sh))
+ return -EINVAL;
+
+ iter->strtabidx = sh.sh_link;
+ iter->syms = elf_getdata(scn, 0);
+ if (!iter->syms) {
+ pr_warn("elf: failed to get symbols for symtab section in '%s': %s\n",
+ binary_path, elf_errmsg(-1));
+ return -EINVAL;
+ }
+ iter->nr_syms = iter->syms->d_size / sh.sh_entsize;
+ iter->elf = elf;
+ iter->st_type = st_type;
+ return 0;
+}
+
+static struct elf_sym *elf_sym_iter_next(struct elf_sym_iter *iter)
+{
+ struct elf_sym *ret = &iter->sym;
+ GElf_Sym *sym = &ret->sym;
+ const char *name = NULL;
+ Elf_Scn *sym_scn;
+ size_t idx;
+
+ for (idx = iter->next_sym_idx; idx < iter->nr_syms; idx++) {
+ if (!gelf_getsym(iter->syms, idx, sym))
+ continue;
+ if (GELF_ST_TYPE(sym->st_info) != iter->st_type)
+ continue;
+ name = elf_strptr(iter->elf, iter->strtabidx, sym->st_name);
+ if (!name)
+ continue;
+ sym_scn = elf_getscn(iter->elf, sym->st_shndx);
+ if (!sym_scn)
+ continue;
+ if (!gelf_getshdr(sym_scn, &ret->sh))
+ continue;
+
+ iter->next_sym_idx = idx + 1;
+ ret->name = name;
+ return ret;
+ }
+
+ return NULL;
+}
+
+
+/* Transform symbol's virtual address (absolute for binaries and relative
+ * for shared libs) into file offset, which is what kernel is expecting
+ * for uprobe/uretprobe attachment.
+ * See Documentation/trace/uprobetracer.rst for more details. This is done
+ * by looking up symbol's containing section's header and using iter's virtual
+ * address (sh_addr) and corresponding file offset (sh_offset) to transform
+ * sym.st_value (virtual address) into desired final file offset.
+ */
+static unsigned long elf_sym_offset(struct elf_sym *sym)
+{
+ return sym->sym.st_value - sym->sh.sh_addr + sym->sh.sh_offset;
+}
+
+/* Find offset of function name in the provided ELF object. "binary_path" is
+ * the path to the ELF binary represented by "elf", and only used for error
+ * reporting matters. "name" matches symbol name or name@@LIB for library
+ * functions.
+ */
+long elf_find_func_offset(Elf *elf, const char *binary_path, const char *name)
+{
+ int i, sh_types[2] = { SHT_DYNSYM, SHT_SYMTAB };
+ bool is_shared_lib, is_name_qualified;
+ long ret = -ENOENT;
+ size_t name_len;
+ GElf_Ehdr ehdr;
+
+ if (!gelf_getehdr(elf, &ehdr)) {
+ pr_warn("elf: failed to get ehdr from %s: %s\n", binary_path, elf_errmsg(-1));
+ ret = -LIBBPF_ERRNO__FORMAT;
+ goto out;
+ }
+ /* for shared lib case, we do not need to calculate relative offset */
+ is_shared_lib = ehdr.e_type == ET_DYN;
+
+ name_len = strlen(name);
+ /* Does name specify "@@LIB"? */
+ is_name_qualified = strstr(name, "@@") != NULL;
+
+ /* Search SHT_DYNSYM, SHT_SYMTAB for symbol. This search order is used because if
+ * a binary is stripped, it may only have SHT_DYNSYM, and a fully-statically
+ * linked binary may not have SHT_DYMSYM, so absence of a section should not be
+ * reported as a warning/error.
+ */
+ for (i = 0; i < ARRAY_SIZE(sh_types); i++) {
+ struct elf_sym_iter iter;
+ struct elf_sym *sym;
+ int last_bind = -1;
+ int cur_bind;
+
+ ret = elf_sym_iter_new(&iter, elf, binary_path, sh_types[i], STT_FUNC);
+ if (ret == -ENOENT)
+ continue;
+ if (ret)
+ goto out;
+
+ while ((sym = elf_sym_iter_next(&iter))) {
+ /* User can specify func, func@@LIB or func@@LIB_VERSION. */
+ if (strncmp(sym->name, name, name_len) != 0)
+ continue;
+ /* ...but we don't want a search for "foo" to match 'foo2" also, so any
+ * additional characters in sname should be of the form "@@LIB".
+ */
+ if (!is_name_qualified && sym->name[name_len] != '\0' && sym->name[name_len] != '@')
+ continue;
+
+ cur_bind = GELF_ST_BIND(sym->sym.st_info);
+
+ if (ret > 0) {
+ /* handle multiple matches */
+ if (last_bind != STB_WEAK && cur_bind != STB_WEAK) {
+ /* Only accept one non-weak bind. */
+ pr_warn("elf: ambiguous match for '%s', '%s' in '%s'\n",
+ sym->name, name, binary_path);
+ ret = -LIBBPF_ERRNO__FORMAT;
+ goto out;
+ } else if (cur_bind == STB_WEAK) {
+ /* already have a non-weak bind, and
+ * this is a weak bind, so ignore.
+ */
+ continue;
+ }
+ }
+
+ ret = elf_sym_offset(sym);
+ last_bind = cur_bind;
+ }
+ if (ret > 0)
+ break;
+ }
+
+ if (ret > 0) {
+ pr_debug("elf: symbol address match for '%s' in '%s': 0x%lx\n", name, binary_path,
+ ret);
+ } else {
+ if (ret == 0) {
+ pr_warn("elf: '%s' is 0 in symtab for '%s': %s\n", name, binary_path,
+ is_shared_lib ? "should not be 0 in a shared library" :
+ "try using shared library path instead");
+ ret = -ENOENT;
+ } else {
+ pr_warn("elf: failed to find symbol '%s' in '%s'\n", name, binary_path);
+ }
+ }
+out:
+ return ret;
+}
+
+/* Find offset of function name in ELF object specified by path. "name" matches
+ * symbol name or name@@LIB for library functions.
+ */
+long elf_find_func_offset_from_file(const char *binary_path, const char *name)
+{
+ struct elf_fd elf_fd;
+ long ret = -ENOENT;
+
+ ret = elf_open(binary_path, &elf_fd);
+ if (ret)
+ return ret;
+ ret = elf_find_func_offset(elf_fd.elf, binary_path, name);
+ elf_close(&elf_fd);
+ return ret;
+}
+
+struct symbol {
+ const char *name;
+ int bind;
+ int idx;
+};
+
+static int symbol_cmp(const void *a, const void *b)
+{
+ const struct symbol *sym_a = a;
+ const struct symbol *sym_b = b;
+
+ return strcmp(sym_a->name, sym_b->name);
+}
+
+/*
+ * Return offsets in @poffsets for symbols specified in @syms array argument.
+ * On success returns 0 and offsets are returned in allocated array with @cnt
+ * size, that needs to be released by the caller.
+ */
+int elf_resolve_syms_offsets(const char *binary_path, int cnt,
+ const char **syms, unsigned long **poffsets)
+{
+ int sh_types[2] = { SHT_DYNSYM, SHT_SYMTAB };
+ int err = 0, i, cnt_done = 0;
+ unsigned long *offsets;
+ struct symbol *symbols;
+ struct elf_fd elf_fd;
+
+ err = elf_open(binary_path, &elf_fd);
+ if (err)
+ return err;
+
+ offsets = calloc(cnt, sizeof(*offsets));
+ symbols = calloc(cnt, sizeof(*symbols));
+
+ if (!offsets || !symbols) {
+ err = -ENOMEM;
+ goto out;
+ }
+
+ for (i = 0; i < cnt; i++) {
+ symbols[i].name = syms[i];
+ symbols[i].idx = i;
+ }
+
+ qsort(symbols, cnt, sizeof(*symbols), symbol_cmp);
+
+ for (i = 0; i < ARRAY_SIZE(sh_types); i++) {
+ struct elf_sym_iter iter;
+ struct elf_sym *sym;
+
+ err = elf_sym_iter_new(&iter, elf_fd.elf, binary_path, sh_types[i], STT_FUNC);
+ if (err == -ENOENT)
+ continue;
+ if (err)
+ goto out;
+
+ while ((sym = elf_sym_iter_next(&iter))) {
+ unsigned long sym_offset = elf_sym_offset(sym);
+ int bind = GELF_ST_BIND(sym->sym.st_info);
+ struct symbol *found, tmp = {
+ .name = sym->name,
+ };
+ unsigned long *offset;
+
+ found = bsearch(&tmp, symbols, cnt, sizeof(*symbols), symbol_cmp);
+ if (!found)
+ continue;
+
+ offset = &offsets[found->idx];
+ if (*offset > 0) {
+ /* same offset, no problem */
+ if (*offset == sym_offset)
+ continue;
+ /* handle multiple matches */
+ if (found->bind != STB_WEAK && bind != STB_WEAK) {
+ /* Only accept one non-weak bind. */
+ pr_warn("elf: ambiguous match found '%s@%lu' in '%s' previous offset %lu\n",
+ sym->name, sym_offset, binary_path, *offset);
+ err = -ESRCH;
+ goto out;
+ } else if (bind == STB_WEAK) {
+ /* already have a non-weak bind, and
+ * this is a weak bind, so ignore.
+ */
+ continue;
+ }
+ } else {
+ cnt_done++;
+ }
+ *offset = sym_offset;
+ found->bind = bind;
+ }
+ }
+
+ if (cnt != cnt_done) {
+ err = -ENOENT;
+ goto out;
+ }
+
+ *poffsets = offsets;
+
+out:
+ free(symbols);
+ if (err)
+ free(offsets);
+ elf_close(&elf_fd);
+ return err;
+}
+
+/*
+ * Return offsets in @poffsets for symbols specified by @pattern argument.
+ * On success returns 0 and offsets are returned in allocated @poffsets
+ * array with the @pctn size, that needs to be released by the caller.
+ */
+int elf_resolve_pattern_offsets(const char *binary_path, const char *pattern,
+ unsigned long **poffsets, size_t *pcnt)
+{
+ int sh_types[2] = { SHT_SYMTAB, SHT_DYNSYM };
+ unsigned long *offsets = NULL;
+ size_t cap = 0, cnt = 0;
+ struct elf_fd elf_fd;
+ int err = 0, i;
+
+ err = elf_open(binary_path, &elf_fd);
+ if (err)
+ return err;
+
+ for (i = 0; i < ARRAY_SIZE(sh_types); i++) {
+ struct elf_sym_iter iter;
+ struct elf_sym *sym;
+
+ err = elf_sym_iter_new(&iter, elf_fd.elf, binary_path, sh_types[i], STT_FUNC);
+ if (err == -ENOENT)
+ continue;
+ if (err)
+ goto out;
+
+ while ((sym = elf_sym_iter_next(&iter))) {
+ if (!glob_match(sym->name, pattern))
+ continue;
+
+ err = libbpf_ensure_mem((void **) &offsets, &cap, sizeof(*offsets),
+ cnt + 1);
+ if (err)
+ goto out;
+
+ offsets[cnt++] = elf_sym_offset(sym);
+ }
+
+ /* If we found anything in the first symbol section,
+ * do not search others to avoid duplicates.
+ */
+ if (cnt)
+ break;
+ }
+
+ if (cnt) {
+ *poffsets = offsets;
+ *pcnt = cnt;
+ } else {
+ err = -ENOENT;
+ }
+
+out:
+ if (err)
+ free(offsets);
+ elf_close(&elf_fd);
+ return err;
+}
diff --git a/tools/lib/bpf/hashmap.h b/tools/lib/bpf/hashmap.h
index 0a5bf1937a7c..c12f8320e668 100644
--- a/tools/lib/bpf/hashmap.h
+++ b/tools/lib/bpf/hashmap.h
@@ -80,16 +80,6 @@ struct hashmap {
size_t sz;
};
-#define HASHMAP_INIT(hash_fn, equal_fn, ctx) { \
- .hash_fn = (hash_fn), \
- .equal_fn = (equal_fn), \
- .ctx = (ctx), \
- .buckets = NULL, \
- .cap = 0, \
- .cap_bits = 0, \
- .sz = 0, \
-}
-
void hashmap__init(struct hashmap *map, hashmap_hash_fn hash_fn,
hashmap_equal_fn equal_fn, void *ctx);
struct hashmap *hashmap__new(hashmap_hash_fn hash_fn,
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 214f828ece6b..96ff1aa4bf6a 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -118,6 +118,9 @@ static const char * const attach_type_name[] = {
[BPF_TRACE_KPROBE_MULTI] = "trace_kprobe_multi",
[BPF_STRUCT_OPS] = "struct_ops",
[BPF_NETFILTER] = "netfilter",
+ [BPF_TCX_INGRESS] = "tcx_ingress",
+ [BPF_TCX_EGRESS] = "tcx_egress",
+ [BPF_TRACE_UPROBE_MULTI] = "trace_uprobe_multi",
};
static const char * const link_type_name[] = {
@@ -132,6 +135,8 @@ static const char * const link_type_name[] = {
[BPF_LINK_TYPE_KPROBE_MULTI] = "kprobe_multi",
[BPF_LINK_TYPE_STRUCT_OPS] = "struct_ops",
[BPF_LINK_TYPE_NETFILTER] = "netfilter",
+ [BPF_LINK_TYPE_TCX] = "tcx",
+ [BPF_LINK_TYPE_UPROBE_MULTI] = "uprobe_multi",
};
static const char * const map_type_name[] = {
@@ -362,6 +367,8 @@ enum sec_def_flags {
SEC_SLEEPABLE = 8,
/* BPF program support non-linear XDP buffer */
SEC_XDP_FRAGS = 16,
+ /* Setup proper attach type for usdt probes. */
+ SEC_USDT = 32,
};
struct bpf_sec_def {
@@ -547,6 +554,7 @@ struct extern_desc {
int btf_id;
int sec_btf_id;
const char *name;
+ char *essent_name;
bool is_set;
bool is_weak;
union {
@@ -1975,9 +1983,9 @@ static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
return -ENAMETOOLONG;
/* gzopen also accepts uncompressed files. */
- file = gzopen(buf, "r");
+ file = gzopen(buf, "re");
if (!file)
- file = gzopen("/proc/config.gz", "r");
+ file = gzopen("/proc/config.gz", "re");
if (!file) {
pr_warn("failed to open system Kconfig\n");
@@ -3767,6 +3775,7 @@ static int bpf_object__collect_externs(struct bpf_object *obj)
struct extern_desc *ext;
int i, n, off, dummy_var_btf_id;
const char *ext_name, *sec_name;
+ size_t ext_essent_len;
Elf_Scn *scn;
Elf64_Shdr *sh;
@@ -3816,6 +3825,14 @@ static int bpf_object__collect_externs(struct bpf_object *obj)
ext->sym_idx = i;
ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK;
+ ext_essent_len = bpf_core_essential_name_len(ext->name);
+ ext->essent_name = NULL;
+ if (ext_essent_len != strlen(ext->name)) {
+ ext->essent_name = strndup(ext->name, ext_essent_len);
+ if (!ext->essent_name)
+ return -ENOMEM;
+ }
+
ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
if (ext->sec_btf_id <= 0) {
pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n",
@@ -4814,6 +4831,39 @@ static int probe_perf_link(void)
return link_fd < 0 && err == -EBADF;
}
+static int probe_uprobe_multi_link(void)
+{
+ LIBBPF_OPTS(bpf_prog_load_opts, load_opts,
+ .expected_attach_type = BPF_TRACE_UPROBE_MULTI,
+ );
+ LIBBPF_OPTS(bpf_link_create_opts, link_opts);
+ struct bpf_insn insns[] = {
+ BPF_MOV64_IMM(BPF_REG_0, 0),
+ BPF_EXIT_INSN(),
+ };
+ int prog_fd, link_fd, err;
+ unsigned long offset = 0;
+
+ prog_fd = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL",
+ insns, ARRAY_SIZE(insns), &load_opts);
+ if (prog_fd < 0)
+ return -errno;
+
+ /* Creating uprobe in '/' binary should fail with -EBADF. */
+ link_opts.uprobe_multi.path = "/";
+ link_opts.uprobe_multi.offsets = &offset;
+ link_opts.uprobe_multi.cnt = 1;
+
+ link_fd = bpf_link_create(prog_fd, -1, BPF_TRACE_UPROBE_MULTI, &link_opts);
+ err = -errno; /* close() can clobber errno */
+
+ if (link_fd >= 0)
+ close(link_fd);
+ close(prog_fd);
+
+ return link_fd < 0 && err == -EBADF;
+}
+
static int probe_kern_bpf_cookie(void)
{
struct bpf_insn insns[] = {
@@ -4910,6 +4960,9 @@ static struct kern_feature_desc {
[FEAT_SYSCALL_WRAPPER] = {
"Kernel using syscall wrapper", probe_kern_syscall_wrapper,
},
+ [FEAT_UPROBE_MULTI_LINK] = {
+ "BPF multi-uprobe link support", probe_uprobe_multi_link,
+ },
};
bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)
@@ -5471,6 +5524,10 @@ static int load_module_btfs(struct bpf_object *obj)
err = bpf_btf_get_next_id(id, &id);
if (err && errno == ENOENT)
return 0;
+ if (err && errno == EPERM) {
+ pr_debug("skipping module BTFs loading, missing privileges\n");
+ return 0;
+ }
if (err) {
err = -errno;
pr_warn("failed to iterate BTF objects: %d\n", err);
@@ -6157,7 +6214,11 @@ static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_progra
if (main_prog == subprog)
return 0;
relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos));
- if (!relos)
+ /* if new count is zero, reallocarray can return a valid NULL result;
+ * in this case the previous pointer will be freed, so we *have to*
+ * reassign old pointer to the new value (even if it's NULL)
+ */
+ if (!relos && new_cnt)
return -ENOMEM;
if (subprog->nr_reloc)
memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc,
@@ -6769,6 +6830,10 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog,
if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;
+ /* special check for usdt to use uprobe_multi link */
+ if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK))
+ prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
+
if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
int btf_obj_fd = 0, btf_type_id = 0, err;
const char *attach_name;
@@ -6837,7 +6902,6 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog
if (!insns || !insns_cnt)
return -EINVAL;
- load_attr.expected_attach_type = prog->expected_attach_type;
if (kernel_supports(obj, FEAT_PROG_NAME))
prog_name = prog->name;
load_attr.attach_prog_fd = prog->attach_prog_fd;
@@ -6873,6 +6937,9 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog
insns_cnt = prog->insns_cnt;
}
+ /* allow prog_prepare_load_fn to change expected_attach_type */
+ load_attr.expected_attach_type = prog->expected_attach_type;
+
if (obj->gen_loader) {
bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
license, insns, insns_cnt, &load_attr,
@@ -7613,7 +7680,8 @@ static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj,
local_func_proto_id = ext->ksym.type_id;
- kfunc_id = find_ksym_btf_id(obj, ext->name, BTF_KIND_FUNC, &kern_btf, &mod_btf);
+ kfunc_id = find_ksym_btf_id(obj, ext->essent_name ?: ext->name, BTF_KIND_FUNC, &kern_btf,
+ &mod_btf);
if (kfunc_id < 0) {
if (kfunc_id == -ESRCH && ext->is_weak)
return 0;
@@ -7628,6 +7696,9 @@ static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj,
ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,
kern_btf, kfunc_proto_id);
if (ret <= 0) {
+ if (ext->is_weak)
+ return 0;
+
pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with %s [%d]\n",
ext->name, local_func_proto_id,
mod_btf ? mod_btf->name : "vmlinux", kfunc_proto_id);
@@ -8305,6 +8376,21 @@ int bpf_object__pin(struct bpf_object *obj, const char *path)
return 0;
}
+int bpf_object__unpin(struct bpf_object *obj, const char *path)
+{
+ int err;
+
+ err = bpf_object__unpin_programs(obj, path);
+ if (err)
+ return libbpf_err(err);
+
+ err = bpf_object__unpin_maps(obj, path);
+ if (err)
+ return libbpf_err(err);
+
+ return 0;
+}
+
static void bpf_map__destroy(struct bpf_map *map)
{
if (map->inner_map) {
@@ -8352,6 +8438,7 @@ void bpf_object__close(struct bpf_object *obj)
bpf_object__elf_finish(obj);
bpf_object_unload(obj);
btf__free(obj->btf);
+ btf__free(obj->btf_vmlinux);
btf_ext__free(obj->btf_ext);
for (i = 0; i < obj->nr_maps; i++)
@@ -8359,6 +8446,10 @@ void bpf_object__close(struct bpf_object *obj)
zfree(&obj->btf_custom_path);
zfree(&obj->kconfig);
+
+ for (i = 0; i < obj->nr_extern; i++)
+ zfree(&obj->externs[i].essent_name);
+
zfree(&obj->externs);
obj->nr_extern = 0;
@@ -8528,7 +8619,8 @@ int bpf_program__set_insns(struct bpf_program *prog,
return -EBUSY;
insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
- if (!insns) {
+ /* NULL is a valid return from reallocarray if the new count is zero */
+ if (!insns && new_insn_cnt) {
pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
return -ENOMEM;
}
@@ -8558,13 +8650,31 @@ enum bpf_prog_type bpf_program__type(const struct bpf_program *prog)
return prog->type;
}
+static size_t custom_sec_def_cnt;
+static struct bpf_sec_def *custom_sec_defs;
+static struct bpf_sec_def custom_fallback_def;
+static bool has_custom_fallback_def;
+static int last_custom_sec_def_handler_id;
+
int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
{
if (prog->obj->loaded)
return libbpf_err(-EBUSY);
+ /* if type is not changed, do nothing */
+ if (prog->type == type)
+ return 0;
+
prog->type = type;
- prog->sec_def = NULL;
+
+ /* If a program type was changed, we need to reset associated SEC()
+ * handler, as it will be invalid now. The only exception is a generic
+ * fallback handler, which by definition is program type-agnostic and
+ * is a catch-all custom handler, optionally set by the application,
+ * so should be able to handle any type of BPF program.
+ */
+ if (prog->sec_def != &custom_fallback_def)
+ prog->sec_def = NULL;
return 0;
}
@@ -8651,6 +8761,7 @@ static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_lin
static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link);
static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
+static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link);
static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link);
@@ -8666,12 +8777,21 @@ static const struct bpf_sec_def section_defs[] = {
SEC_DEF("uretprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),
SEC_DEF("kprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
SEC_DEF("kretprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
+ SEC_DEF("uprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),
+ SEC_DEF("uretprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),
+ SEC_DEF("uprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),
+ SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),
SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
- SEC_DEF("usdt+", KPROBE, 0, SEC_NONE, attach_usdt),
- SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE),
- SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE),
- SEC_DEF("action", SCHED_ACT, 0, SEC_NONE),
+ SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt),
+ SEC_DEF("usdt.s+", KPROBE, 0, SEC_USDT | SEC_SLEEPABLE, attach_usdt),
+ SEC_DEF("tc/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), /* alias for tcx */
+ SEC_DEF("tc/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE), /* alias for tcx */
+ SEC_DEF("tcx/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE),
+ SEC_DEF("tcx/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE),
+ SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */
+ SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */
+ SEC_DEF("action", SCHED_ACT, 0, SEC_NONE), /* deprecated / legacy, use tcx */
SEC_DEF("tracepoint+", TRACEPOINT, 0, SEC_NONE, attach_tp),
SEC_DEF("tp+", TRACEPOINT, 0, SEC_NONE, attach_tp),
SEC_DEF("raw_tracepoint+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),
@@ -8740,13 +8860,6 @@ static const struct bpf_sec_def section_defs[] = {
SEC_DEF("netfilter", NETFILTER, BPF_NETFILTER, SEC_NONE),
};
-static size_t custom_sec_def_cnt;
-static struct bpf_sec_def *custom_sec_defs;
-static struct bpf_sec_def custom_fallback_def;
-static bool has_custom_fallback_def;
-
-static int last_custom_sec_def_handler_id;
-
int libbpf_register_prog_handler(const char *sec,
enum bpf_prog_type prog_type,
enum bpf_attach_type exp_attach_type,
@@ -8826,7 +8939,11 @@ int libbpf_unregister_prog_handler(int handler_id)
/* try to shrink the array, but it's ok if we couldn't */
sec_defs = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt, sizeof(*sec_defs));
- if (sec_defs)
+ /* if new count is zero, reallocarray can return a valid NULL result;
+ * in this case the previous pointer will be freed, so we *have to*
+ * reassign old pointer to the new value (even if it's NULL)
+ */
+ if (sec_defs || custom_sec_def_cnt == 0)
custom_sec_defs = sec_defs;
return 0;
@@ -10224,6 +10341,18 @@ static const char *tracefs_uprobe_events(void)
return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events";
}
+static const char *tracefs_available_filter_functions(void)
+{
+ return use_debugfs() ? DEBUGFS"/available_filter_functions"
+ : TRACEFS"/available_filter_functions";
+}
+
+static const char *tracefs_available_filter_functions_addrs(void)
+{
+ return use_debugfs() ? DEBUGFS"/available_filter_functions_addrs"
+ : TRACEFS"/available_filter_functions_addrs";
+}
+
static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz,
const char *kfunc_name, size_t offset)
{
@@ -10506,7 +10635,7 @@ struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog,
}
/* Adapted from perf/util/string.c */
-static bool glob_match(const char *str, const char *pat)
+bool glob_match(const char *str, const char *pat)
{
while (*str && *pat && *pat != '*') {
if (*pat == '?') { /* Matches any single character */
@@ -10539,25 +10668,158 @@ struct kprobe_multi_resolve {
size_t cnt;
};
-static int
-resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
- const char *sym_name, void *ctx)
+struct avail_kallsyms_data {
+ char **syms;
+ size_t cnt;
+ struct kprobe_multi_resolve *res;
+};
+
+static int avail_func_cmp(const void *a, const void *b)
{
- struct kprobe_multi_resolve *res = ctx;
+ return strcmp(*(const char **)a, *(const char **)b);
+}
+
+static int avail_kallsyms_cb(unsigned long long sym_addr, char sym_type,
+ const char *sym_name, void *ctx)
+{
+ struct avail_kallsyms_data *data = ctx;
+ struct kprobe_multi_resolve *res = data->res;
int err;
- if (!glob_match(sym_name, res->pattern))
+ if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp))
return 0;
- err = libbpf_ensure_mem((void **) &res->addrs, &res->cap, sizeof(unsigned long),
- res->cnt + 1);
+ err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1);
if (err)
return err;
- res->addrs[res->cnt++] = (unsigned long) sym_addr;
+ res->addrs[res->cnt++] = (unsigned long)sym_addr;
return 0;
}
+static int libbpf_available_kallsyms_parse(struct kprobe_multi_resolve *res)
+{
+ const char *available_functions_file = tracefs_available_filter_functions();
+ struct avail_kallsyms_data data;
+ char sym_name[500];
+ FILE *f;
+ int err = 0, ret, i;
+ char **syms = NULL;
+ size_t cap = 0, cnt = 0;
+
+ f = fopen(available_functions_file, "re");
+ if (!f) {
+ err = -errno;
+ pr_warn("failed to open %s: %d\n", available_functions_file, err);
+ return err;
+ }
+
+ while (true) {
+ char *name;
+
+ ret = fscanf(f, "%499s%*[^\n]\n", sym_name);
+ if (ret == EOF && feof(f))
+ break;
+
+ if (ret != 1) {
+ pr_warn("failed to parse available_filter_functions entry: %d\n", ret);
+ err = -EINVAL;
+ goto cleanup;
+ }
+
+ if (!glob_match(sym_name, res->pattern))
+ continue;
+
+ err = libbpf_ensure_mem((void **)&syms, &cap, sizeof(*syms), cnt + 1);
+ if (err)
+ goto cleanup;
+
+ name = strdup(sym_name);
+ if (!name) {
+ err = -errno;
+ goto cleanup;
+ }
+
+ syms[cnt++] = name;
+ }
+
+ /* no entries found, bail out */
+ if (cnt == 0) {
+ err = -ENOENT;
+ goto cleanup;
+ }
+
+ /* sort available functions */
+ qsort(syms, cnt, sizeof(*syms), avail_func_cmp);
+
+ data.syms = syms;
+ data.res = res;
+ data.cnt = cnt;
+ libbpf_kallsyms_parse(avail_kallsyms_cb, &data);
+
+ if (res->cnt == 0)
+ err = -ENOENT;
+
+cleanup:
+ for (i = 0; i < cnt; i++)
+ free((char *)syms[i]);
+ free(syms);
+
+ fclose(f);
+ return err;
+}
+
+static bool has_available_filter_functions_addrs(void)
+{
+ return access(tracefs_available_filter_functions_addrs(), R_OK) != -1;
+}
+
+static int libbpf_available_kprobes_parse(struct kprobe_multi_resolve *res)
+{
+ const char *available_path = tracefs_available_filter_functions_addrs();
+ char sym_name[500];
+ FILE *f;
+ int ret, err = 0;
+ unsigned long long sym_addr;
+
+ f = fopen(available_path, "re");
+ if (!f) {
+ err = -errno;
+ pr_warn("failed to open %s: %d\n", available_path, err);
+ return err;
+ }
+
+ while (true) {
+ ret = fscanf(f, "%llx %499s%*[^\n]\n", &sym_addr, sym_name);
+ if (ret == EOF && feof(f))
+ break;
+
+ if (ret != 2) {
+ pr_warn("failed to parse available_filter_functions_addrs entry: %d\n",
+ ret);
+ err = -EINVAL;
+ goto cleanup;
+ }
+
+ if (!glob_match(sym_name, res->pattern))
+ continue;
+
+ err = libbpf_ensure_mem((void **)&res->addrs, &res->cap,
+ sizeof(*res->addrs), res->cnt + 1);
+ if (err)
+ goto cleanup;
+
+ res->addrs[res->cnt++] = (unsigned long)sym_addr;
+ }
+
+ if (res->cnt == 0)
+ err = -ENOENT;
+
+cleanup:
+ fclose(f);
+ return err;
+}
+
struct bpf_link *
bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
const char *pattern,
@@ -10594,13 +10856,12 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
return libbpf_err_ptr(-EINVAL);
if (pattern) {
- err = libbpf_kallsyms_parse(resolve_kprobe_multi_cb, &res);
+ if (has_available_filter_functions_addrs())
+ err = libbpf_available_kprobes_parse(&res);
+ else
+ err = libbpf_available_kallsyms_parse(&res);
if (err)
goto error;
- if (!res.cnt) {
- err = -ENOENT;
- goto error;
- }
addrs = res.addrs;
cnt = res.cnt;
}
@@ -10727,6 +10988,37 @@ static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, stru
return libbpf_get_error(*link);
}
+static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link)
+{
+ char *probe_type = NULL, *binary_path = NULL, *func_name = NULL;
+ LIBBPF_OPTS(bpf_uprobe_multi_opts, opts);
+ int n, ret = -EINVAL;
+
+ *link = NULL;
+
+ n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%ms",
+ &probe_type, &binary_path, &func_name);
+ switch (n) {
+ case 1:
+ /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
+ ret = 0;
+ break;
+ case 3:
+ opts.retprobe = strcmp(probe_type, "uretprobe.multi") == 0;
+ *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts);
+ ret = libbpf_get_error(*link);
+ break;
+ default:
+ pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
+ prog->sec_name);
+ break;
+ }
+ free(probe_type);
+ free(binary_path);
+ free(func_name);
+ return ret;
+}
+
static void gen_uprobe_legacy_event_name(char *buf, size_t buf_sz,
const char *binary_path, uint64_t offset)
{
@@ -10809,191 +11101,6 @@ err_clean_legacy:
return err;
}
-/* Return next ELF section of sh_type after scn, or first of that type if scn is NULL. */
-static Elf_Scn *elf_find_next_scn_by_type(Elf *elf, int sh_type, Elf_Scn *scn)
-{
- while ((scn = elf_nextscn(elf, scn)) != NULL) {
- GElf_Shdr sh;
-
- if (!gelf_getshdr(scn, &sh))
- continue;
- if (sh.sh_type == sh_type)
- return scn;
- }
- return NULL;
-}
-
-/* Find offset of function name in the provided ELF object. "binary_path" is
- * the path to the ELF binary represented by "elf", and only used for error
- * reporting matters. "name" matches symbol name or name@@LIB for library
- * functions.
- */
-static long elf_find_func_offset(Elf *elf, const char *binary_path, const char *name)
-{
- int i, sh_types[2] = { SHT_DYNSYM, SHT_SYMTAB };
- bool is_shared_lib, is_name_qualified;
- long ret = -ENOENT;
- size_t name_len;
- GElf_Ehdr ehdr;
-
- if (!gelf_getehdr(elf, &ehdr)) {
- pr_warn("elf: failed to get ehdr from %s: %s\n", binary_path, elf_errmsg(-1));
- ret = -LIBBPF_ERRNO__FORMAT;
- goto out;
- }
- /* for shared lib case, we do not need to calculate relative offset */
- is_shared_lib = ehdr.e_type == ET_DYN;
-
- name_len = strlen(name);
- /* Does name specify "@@LIB"? */
- is_name_qualified = strstr(name, "@@") != NULL;
-
- /* Search SHT_DYNSYM, SHT_SYMTAB for symbol. This search order is used because if
- * a binary is stripped, it may only have SHT_DYNSYM, and a fully-statically
- * linked binary may not have SHT_DYMSYM, so absence of a section should not be
- * reported as a warning/error.
- */
- for (i = 0; i < ARRAY_SIZE(sh_types); i++) {
- size_t nr_syms, strtabidx, idx;
- Elf_Data *symbols = NULL;
- Elf_Scn *scn = NULL;
- int last_bind = -1;
- const char *sname;
- GElf_Shdr sh;
-
- scn = elf_find_next_scn_by_type(elf, sh_types[i], NULL);
- if (!scn) {
- pr_debug("elf: failed to find symbol table ELF sections in '%s'\n",
- binary_path);
- continue;
- }
- if (!gelf_getshdr(scn, &sh))
- continue;
- strtabidx = sh.sh_link;
- symbols = elf_getdata(scn, 0);
- if (!symbols) {
- pr_warn("elf: failed to get symbols for symtab section in '%s': %s\n",
- binary_path, elf_errmsg(-1));
- ret = -LIBBPF_ERRNO__FORMAT;
- goto out;
- }
- nr_syms = symbols->d_size / sh.sh_entsize;
-
- for (idx = 0; idx < nr_syms; idx++) {
- int curr_bind;
- GElf_Sym sym;
- Elf_Scn *sym_scn;
- GElf_Shdr sym_sh;
-
- if (!gelf_getsym(symbols, idx, &sym))
- continue;
-
- if (GELF_ST_TYPE(sym.st_info) != STT_FUNC)
- continue;
-
- sname = elf_strptr(elf, strtabidx, sym.st_name);
- if (!sname)
- continue;
-
- curr_bind = GELF_ST_BIND(sym.st_info);
-
- /* User can specify func, func@@LIB or func@@LIB_VERSION. */
- if (strncmp(sname, name, name_len) != 0)
- continue;
- /* ...but we don't want a search for "foo" to match 'foo2" also, so any
- * additional characters in sname should be of the form "@@LIB".
- */
- if (!is_name_qualified && sname[name_len] != '\0' && sname[name_len] != '@')
- continue;
-
- if (ret >= 0) {
- /* handle multiple matches */
- if (last_bind != STB_WEAK && curr_bind != STB_WEAK) {
- /* Only accept one non-weak bind. */
- pr_warn("elf: ambiguous match for '%s', '%s' in '%s'\n",
- sname, name, binary_path);
- ret = -LIBBPF_ERRNO__FORMAT;
- goto out;
- } else if (curr_bind == STB_WEAK) {
- /* already have a non-weak bind, and
- * this is a weak bind, so ignore.
- */
- continue;
- }
- }
-
- /* Transform symbol's virtual address (absolute for
- * binaries and relative for shared libs) into file
- * offset, which is what kernel is expecting for
- * uprobe/uretprobe attachment.
- * See Documentation/trace/uprobetracer.rst for more
- * details.
- * This is done by looking up symbol's containing
- * section's header and using it's virtual address
- * (sh_addr) and corresponding file offset (sh_offset)
- * to transform sym.st_value (virtual address) into
- * desired final file offset.
- */
- sym_scn = elf_getscn(elf, sym.st_shndx);
- if (!sym_scn)
- continue;
- if (!gelf_getshdr(sym_scn, &sym_sh))
- continue;
-
- ret = sym.st_value - sym_sh.sh_addr + sym_sh.sh_offset;
- last_bind = curr_bind;
- }
- if (ret > 0)
- break;
- }
-
- if (ret > 0) {
- pr_debug("elf: symbol address match for '%s' in '%s': 0x%lx\n", name, binary_path,
- ret);
- } else {
- if (ret == 0) {
- pr_warn("elf: '%s' is 0 in symtab for '%s': %s\n", name, binary_path,
- is_shared_lib ? "should not be 0 in a shared library" :
- "try using shared library path instead");
- ret = -ENOENT;
- } else {
- pr_warn("elf: failed to find symbol '%s' in '%s'\n", name, binary_path);
- }
- }
-out:
- return ret;
-}
-
-/* Find offset of function name in ELF object specified by path. "name" matches
- * symbol name or name@@LIB for library functions.
- */
-static long elf_find_func_offset_from_file(const char *binary_path, const char *name)
-{
- char errmsg[STRERR_BUFSIZE];
- long ret = -ENOENT;
- Elf *elf;
- int fd;
-
- fd = open(binary_path, O_RDONLY | O_CLOEXEC);
- if (fd < 0) {
- ret = -errno;
- pr_warn("failed to open %s: %s\n", binary_path,
- libbpf_strerror_r(ret, errmsg, sizeof(errmsg)));
- return ret;
- }
- elf = elf_begin(fd, ELF_C_READ_MMAP, NULL);
- if (!elf) {
- pr_warn("elf: could not read elf from %s: %s\n", binary_path, elf_errmsg(-1));
- close(fd);
- return -LIBBPF_ERRNO__FORMAT;
- }
-
- ret = elf_find_func_offset(elf, binary_path, name);
- elf_end(elf);
- close(fd);
- return ret;
-}
-
/* Find offset of function name in archive specified by path. Currently
* supported are .zip files that do not compress their contents, as used on
* Android in the form of APKs, for example. "file_name" is the name of the ELF
@@ -11136,6 +11243,120 @@ static int resolve_full_path(const char *file, char *result, size_t result_sz)
return -ENOENT;
}
+struct bpf_link *
+bpf_program__attach_uprobe_multi(const struct bpf_program *prog,
+ pid_t pid,
+ const char *path,
+ const char *func_pattern,
+ const struct bpf_uprobe_multi_opts *opts)
+{
+ const unsigned long *ref_ctr_offsets = NULL, *offsets = NULL;
+ LIBBPF_OPTS(bpf_link_create_opts, lopts);
+ unsigned long *resolved_offsets = NULL;
+ int err = 0, link_fd, prog_fd;
+ struct bpf_link *link = NULL;
+ char errmsg[STRERR_BUFSIZE];
+ char full_path[PATH_MAX];
+ const __u64 *cookies;
+ const char **syms;
+ size_t cnt;
+
+ if (!OPTS_VALID(opts, bpf_uprobe_multi_opts))
+ return libbpf_err_ptr(-EINVAL);
+
+ syms = OPTS_GET(opts, syms, NULL);
+ offsets = OPTS_GET(opts, offsets, NULL);
+ ref_ctr_offsets = OPTS_GET(opts, ref_ctr_offsets, NULL);
+ cookies = OPTS_GET(opts, cookies, NULL);
+ cnt = OPTS_GET(opts, cnt, 0);
+
+ /*
+ * User can specify 2 mutually exclusive set of inputs:
+ *
+ * 1) use only path/func_pattern/pid arguments
+ *
+ * 2) use path/pid with allowed combinations of:
+ * syms/offsets/ref_ctr_offsets/cookies/cnt
+ *
+ * - syms and offsets are mutually exclusive
+ * - ref_ctr_offsets and cookies are optional
+ *
+ * Any other usage results in error.
+ */
+
+ if (!path)
+ return libbpf_err_ptr(-EINVAL);
+ if (!func_pattern && cnt == 0)
+ return libbpf_err_ptr(-EINVAL);
+
+ if (func_pattern) {
+ if (syms || offsets || ref_ctr_offsets || cookies || cnt)
+ return libbpf_err_ptr(-EINVAL);
+ } else {
+ if (!!syms == !!offsets)
+ return libbpf_err_ptr(-EINVAL);
+ }
+
+ if (func_pattern) {
+ if (!strchr(path, '/')) {
+ err = resolve_full_path(path, full_path, sizeof(full_path));
+ if (err) {
+ pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
+ prog->name, path, err);
+ return libbpf_err_ptr(err);
+ }
+ path = full_path;
+ }
+
+ err = elf_resolve_pattern_offsets(path, func_pattern,
+ &resolved_offsets, &cnt);
+ if (err < 0)
+ return libbpf_err_ptr(err);
+ offsets = resolved_offsets;
+ } else if (syms) {
+ err = elf_resolve_syms_offsets(path, cnt, syms, &resolved_offsets);
+ if (err < 0)
+ return libbpf_err_ptr(err);
+ offsets = resolved_offsets;
+ }
+
+ lopts.uprobe_multi.path = path;
+ lopts.uprobe_multi.offsets = offsets;
+ lopts.uprobe_multi.ref_ctr_offsets = ref_ctr_offsets;
+ lopts.uprobe_multi.cookies = cookies;
+ lopts.uprobe_multi.cnt = cnt;
+ lopts.uprobe_multi.flags = OPTS_GET(opts, retprobe, false) ? BPF_F_UPROBE_MULTI_RETURN : 0;
+
+ if (pid == 0)
+ pid = getpid();
+ if (pid > 0)
+ lopts.uprobe_multi.pid = pid;
+
+ link = calloc(1, sizeof(*link));
+ if (!link) {
+ err = -ENOMEM;
+ goto error;
+ }
+ link->detach = &bpf_link__detach_fd;
+
+ prog_fd = bpf_program__fd(prog);
+ link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &lopts);
+ if (link_fd < 0) {
+ err = -errno;
+ pr_warn("prog '%s': failed to attach multi-uprobe: %s\n",
+ prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
+ goto error;
+ }
+ link->fd = link_fd;
+ free(resolved_offsets);
+ return link;
+
+error:
+ free(resolved_offsets);
+ free(link);
+ return libbpf_err_ptr(err);
+}
+
LIBBPF_API struct bpf_link *
bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,
const char *binary_path, size_t func_offset,
@@ -11680,11 +11901,10 @@ static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_li
}
static struct bpf_link *
-bpf_program__attach_fd(const struct bpf_program *prog, int target_fd, int btf_id,
- const char *target_name)
+bpf_program_attach_fd(const struct bpf_program *prog,
+ int target_fd, const char *target_name,
+ const struct bpf_link_create_opts *opts)
{
- DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,
- .target_btf_id = btf_id);
enum bpf_attach_type attach_type;
char errmsg[STRERR_BUFSIZE];
struct bpf_link *link;
@@ -11702,7 +11922,7 @@ bpf_program__attach_fd(const struct bpf_program *prog, int target_fd, int btf_id
link->detach = &bpf_link__detach_fd;
attach_type = bpf_program__expected_attach_type(prog);
- link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts);
+ link_fd = bpf_link_create(prog_fd, target_fd, attach_type, opts);
if (link_fd < 0) {
link_fd = -errno;
free(link);
@@ -11718,19 +11938,54 @@ bpf_program__attach_fd(const struct bpf_program *prog, int target_fd, int btf_id
struct bpf_link *
bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd)
{
- return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup");
+ return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", NULL);
}
struct bpf_link *
bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd)
{
- return bpf_program__attach_fd(prog, netns_fd, 0, "netns");
+ return bpf_program_attach_fd(prog, netns_fd, "netns", NULL);
}
struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex)
{
/* target_fd/target_ifindex use the same field in LINK_CREATE */
- return bpf_program__attach_fd(prog, ifindex, 0, "xdp");
+ return bpf_program_attach_fd(prog, ifindex, "xdp", NULL);
+}
+
+struct bpf_link *
+bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex,
+ const struct bpf_tcx_opts *opts)
+{
+ LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
+ __u32 relative_id;
+ int relative_fd;
+
+ if (!OPTS_VALID(opts, bpf_tcx_opts))
+ return libbpf_err_ptr(-EINVAL);
+
+ relative_id = OPTS_GET(opts, relative_id, 0);
+ relative_fd = OPTS_GET(opts, relative_fd, 0);
+
+ /* validate we don't have unexpected combinations of non-zero fields */
+ if (!ifindex) {
+ pr_warn("prog '%s': target netdevice ifindex cannot be zero\n",
+ prog->name);
+ return libbpf_err_ptr(-EINVAL);
+ }
+ if (relative_fd && relative_id) {
+ pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n",
+ prog->name);
+ return libbpf_err_ptr(-EINVAL);
+ }
+
+ link_create_opts.tcx.expected_revision = OPTS_GET(opts, expected_revision, 0);
+ link_create_opts.tcx.relative_fd = relative_fd;
+ link_create_opts.tcx.relative_id = relative_id;
+ link_create_opts.flags = OPTS_GET(opts, flags, 0);
+
+ /* target_fd/target_ifindex use the same field in LINK_CREATE */
+ return bpf_program_attach_fd(prog, ifindex, "tcx", &link_create_opts);
}
struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog,
@@ -11752,11 +12007,16 @@ struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog,
}
if (target_fd) {
+ LIBBPF_OPTS(bpf_link_create_opts, target_opts);
+
btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd);
if (btf_id < 0)
return libbpf_err_ptr(btf_id);
- return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace");
+ target_opts.target_btf_id = btf_id;
+
+ return bpf_program_attach_fd(prog, target_fd, "freplace",
+ &target_opts);
} else {
/* no target, so use raw_tracepoint_open for compatibility
* with old kernels
@@ -11811,6 +12071,48 @@ static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_l
return libbpf_get_error(*link);
}
+struct bpf_link *bpf_program__attach_netfilter(const struct bpf_program *prog,
+ const struct bpf_netfilter_opts *opts)
+{
+ LIBBPF_OPTS(bpf_link_create_opts, lopts);
+ struct bpf_link *link;
+ int prog_fd, link_fd;
+
+ if (!OPTS_VALID(opts, bpf_netfilter_opts))
+ return libbpf_err_ptr(-EINVAL);
+
+ prog_fd = bpf_program__fd(prog);
+ if (prog_fd < 0) {
+ pr_warn("prog '%s': can't attach before loaded\n", prog->name);
+ return libbpf_err_ptr(-EINVAL);
+ }
+
+ link = calloc(1, sizeof(*link));
+ if (!link)
+ return libbpf_err_ptr(-ENOMEM);
+
+ link->detach = &bpf_link__detach_fd;
+
+ lopts.netfilter.pf = OPTS_GET(opts, pf, 0);
+ lopts.netfilter.hooknum = OPTS_GET(opts, hooknum, 0);
+ lopts.netfilter.priority = OPTS_GET(opts, priority, 0);
+ lopts.netfilter.flags = OPTS_GET(opts, flags, 0);
+
+ link_fd = bpf_link_create(prog_fd, 0, BPF_NETFILTER, &lopts);
+ if (link_fd < 0) {
+ char errmsg[STRERR_BUFSIZE];
+
+ link_fd = -errno;
+ free(link);
+ pr_warn("prog '%s': failed to attach to netfilter: %s\n",
+ prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
+ return libbpf_err_ptr(link_fd);
+ }
+ link->fd = link_fd;
+
+ return link;
+}
+
struct bpf_link *bpf_program__attach(const struct bpf_program *prog)
{
struct bpf_link *link = NULL;
diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h
index 754da73c643b..0e52621cba43 100644
--- a/tools/lib/bpf/libbpf.h
+++ b/tools/lib/bpf/libbpf.h
@@ -266,6 +266,7 @@ LIBBPF_API int bpf_object__pin_programs(struct bpf_object *obj,
LIBBPF_API int bpf_object__unpin_programs(struct bpf_object *obj,
const char *path);
LIBBPF_API int bpf_object__pin(struct bpf_object *object, const char *path);
+LIBBPF_API int bpf_object__unpin(struct bpf_object *object, const char *path);
LIBBPF_API const char *bpf_object__name(const struct bpf_object *obj);
LIBBPF_API unsigned int bpf_object__kversion(const struct bpf_object *obj);
@@ -529,6 +530,57 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
const char *pattern,
const struct bpf_kprobe_multi_opts *opts);
+struct bpf_uprobe_multi_opts {
+ /* size of this struct, for forward/backward compatibility */
+ size_t sz;
+ /* array of function symbols to attach to */
+ const char **syms;
+ /* array of function addresses to attach to */
+ const unsigned long *offsets;
+ /* optional, array of associated ref counter offsets */
+ const unsigned long *ref_ctr_offsets;
+ /* optional, array of associated BPF cookies */
+ const __u64 *cookies;
+ /* number of elements in syms/addrs/cookies arrays */
+ size_t cnt;
+ /* create return uprobes */
+ bool retprobe;
+ size_t :0;
+};
+
+#define bpf_uprobe_multi_opts__last_field retprobe
+
+/**
+ * @brief **bpf_program__attach_uprobe_multi()** attaches a BPF program
+ * to multiple uprobes with uprobe_multi link.
+ *
+ * User can specify 2 mutually exclusive set of inputs:
+ *
+ * 1) use only path/func_pattern/pid arguments
+ *
+ * 2) use path/pid with allowed combinations of
+ * syms/offsets/ref_ctr_offsets/cookies/cnt
+ *
+ * - syms and offsets are mutually exclusive
+ * - ref_ctr_offsets and cookies are optional
+ *
+ *
+ * @param prog BPF program to attach
+ * @param pid Process ID to attach the uprobe to, 0 for self (own process),
+ * -1 for all processes
+ * @param binary_path Path to binary
+ * @param func_pattern Regular expression to specify functions to attach
+ * BPF program to
+ * @param opts Additional options (see **struct bpf_uprobe_multi_opts**)
+ * @return 0, on success; negative error code, otherwise
+ */
+LIBBPF_API struct bpf_link *
+bpf_program__attach_uprobe_multi(const struct bpf_program *prog,
+ pid_t pid,
+ const char *binary_path,
+ const char *func_pattern,
+ const struct bpf_uprobe_multi_opts *opts);
+
struct bpf_ksyscall_opts {
/* size of this struct, for forward/backward compatibility */
size_t sz;
@@ -718,6 +770,36 @@ LIBBPF_API struct bpf_link *
bpf_program__attach_freplace(const struct bpf_program *prog,
int target_fd, const char *attach_func_name);
+struct bpf_netfilter_opts {
+ /* size of this struct, for forward/backward compatibility */
+ size_t sz;
+
+ __u32 pf;
+ __u32 hooknum;
+ __s32 priority;
+ __u32 flags;
+};
+#define bpf_netfilter_opts__last_field flags
+
+LIBBPF_API struct bpf_link *
+bpf_program__attach_netfilter(const struct bpf_program *prog,
+ const struct bpf_netfilter_opts *opts);
+
+struct bpf_tcx_opts {
+ /* size of this struct, for forward/backward compatibility */
+ size_t sz;
+ __u32 flags;
+ __u32 relative_fd;
+ __u32 relative_id;
+ __u64 expected_revision;
+ size_t :0;
+};
+#define bpf_tcx_opts__last_field expected_revision
+
+LIBBPF_API struct bpf_link *
+bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex,
+ const struct bpf_tcx_opts *opts);
+
struct bpf_map;
LIBBPF_API struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map);
@@ -1090,9 +1172,10 @@ struct bpf_xdp_query_opts {
__u32 skb_prog_id; /* output */
__u8 attach_mode; /* output */
__u64 feature_flags; /* output */
+ __u32 xdp_zc_max_segs; /* output */
size_t :0;
};
-#define bpf_xdp_query_opts__last_field feature_flags
+#define bpf_xdp_query_opts__last_field xdp_zc_max_segs
LIBBPF_API int bpf_xdp_attach(int ifindex, int prog_fd, __u32 flags,
const struct bpf_xdp_attach_opts *opts);
diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map
index 7521a2fb7626..57712321490f 100644
--- a/tools/lib/bpf/libbpf.map
+++ b/tools/lib/bpf/libbpf.map
@@ -395,4 +395,9 @@ LIBBPF_1.2.0 {
LIBBPF_1.3.0 {
global:
bpf_obj_pin_opts;
+ bpf_object__unpin;
+ bpf_prog_detach_opts;
+ bpf_program__attach_netfilter;
+ bpf_program__attach_tcx;
+ bpf_program__attach_uprobe_multi;
} LIBBPF_1.2.0;
diff --git a/tools/lib/bpf/libbpf_common.h b/tools/lib/bpf/libbpf_common.h
index 9a7937f339df..b7060f254486 100644
--- a/tools/lib/bpf/libbpf_common.h
+++ b/tools/lib/bpf/libbpf_common.h
@@ -70,4 +70,20 @@
}; \
})
+/* Helper macro to clear and optionally reinitialize libbpf options struct
+ *
+ * Small helper macro to reset all fields and to reinitialize the common
+ * structure size member. Values provided by users in struct initializer-
+ * syntax as varargs can be provided as well to reinitialize options struct
+ * specific members.
+ */
+#define LIBBPF_OPTS_RESET(NAME, ...) \
+ do { \
+ memset(&NAME, 0, sizeof(NAME)); \
+ NAME = (typeof(NAME)) { \
+ .sz = sizeof(NAME), \
+ __VA_ARGS__ \
+ }; \
+ } while (0)
+
#endif /* __LIBBPF_LIBBPF_COMMON_H */
diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h
index e4d05662a96c..f0f08635adb0 100644
--- a/tools/lib/bpf/libbpf_internal.h
+++ b/tools/lib/bpf/libbpf_internal.h
@@ -15,6 +15,7 @@
#include <linux/err.h>
#include <fcntl.h>
#include <unistd.h>
+#include <libelf.h>
#include "relo_core.h"
/* make sure libbpf doesn't use kernel-only integer typedefs */
@@ -354,6 +355,8 @@ enum kern_feature_id {
FEAT_BTF_ENUM64,
/* Kernel uses syscall wrapper (CONFIG_ARCH_HAS_SYSCALL_WRAPPER) */
FEAT_SYSCALL_WRAPPER,
+ /* BPF multi-uprobe link support */
+ FEAT_UPROBE_MULTI_LINK,
__FEAT_CNT,
};
@@ -577,4 +580,22 @@ static inline bool is_pow_of_2(size_t x)
#define PROG_LOAD_ATTEMPTS 5
int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int attempts);
+bool glob_match(const char *str, const char *pat);
+
+long elf_find_func_offset(Elf *elf, const char *binary_path, const char *name);
+long elf_find_func_offset_from_file(const char *binary_path, const char *name);
+
+struct elf_fd {
+ Elf *elf;
+ int fd;
+};
+
+int elf_open(const char *binary_path, struct elf_fd *elf_fd);
+void elf_close(struct elf_fd *elf_fd);
+
+int elf_resolve_syms_offsets(const char *binary_path, int cnt,
+ const char **syms, unsigned long **poffsets);
+int elf_resolve_pattern_offsets(const char *binary_path, const char *pattern,
+ unsigned long **poffsets, size_t *pcnt);
+
#endif /* __LIBBPF_LIBBPF_INTERNAL_H */
diff --git a/tools/lib/bpf/netlink.c b/tools/lib/bpf/netlink.c
index 84dd5fa14905..090bcf6e3b3d 100644
--- a/tools/lib/bpf/netlink.c
+++ b/tools/lib/bpf/netlink.c
@@ -45,6 +45,7 @@ struct xdp_id_md {
struct xdp_features_md {
int ifindex;
+ __u32 xdp_zc_max_segs;
__u64 flags;
};
@@ -421,6 +422,9 @@ static int parse_xdp_features(struct nlmsghdr *nh, libbpf_dump_nlmsg_t fn,
return NL_CONT;
md->flags = libbpf_nla_getattr_u64(tb[NETDEV_A_DEV_XDP_FEATURES]);
+ if (tb[NETDEV_A_DEV_XDP_ZC_MAX_SEGS])
+ md->xdp_zc_max_segs =
+ libbpf_nla_getattr_u32(tb[NETDEV_A_DEV_XDP_ZC_MAX_SEGS]);
return NL_DONE;
}
@@ -493,6 +497,7 @@ int bpf_xdp_query(int ifindex, int xdp_flags, struct bpf_xdp_query_opts *opts)
return libbpf_err(err);
opts->feature_flags = md.flags;
+ opts->xdp_zc_max_segs = md.xdp_zc_max_segs;
skip_feature_flags:
return 0;
diff --git a/tools/lib/bpf/relo_core.c b/tools/lib/bpf/relo_core.c
index a26b2f5fa0fc..63a4d5ad12d1 100644
--- a/tools/lib/bpf/relo_core.c
+++ b/tools/lib/bpf/relo_core.c
@@ -776,7 +776,7 @@ static int bpf_core_calc_field_relo(const char *prog_name,
break;
case BPF_CORE_FIELD_SIGNED:
*val = (btf_is_any_enum(mt) && BTF_INFO_KFLAG(mt->info)) ||
- (btf_int_encoding(mt) & BTF_INT_SIGNED);
+ (btf_is_int(mt) && (btf_int_encoding(mt) & BTF_INT_SIGNED));
if (validate)
*validate = true; /* signedness is never ambiguous */
break;
diff --git a/tools/lib/bpf/usdt.bpf.h b/tools/lib/bpf/usdt.bpf.h
index 0bd4c135acc2..f6763300b26a 100644
--- a/tools/lib/bpf/usdt.bpf.h
+++ b/tools/lib/bpf/usdt.bpf.h
@@ -4,8 +4,8 @@
#define __USDT_BPF_H__
#include <linux/errno.h>
-#include <bpf/bpf_helpers.h>
-#include <bpf/bpf_tracing.h>
+#include "bpf_helpers.h"
+#include "bpf_tracing.h"
/* Below types and maps are internal implementation details of libbpf's USDT
* support and are subjects to change. Also, bpf_usdt_xxx() API helpers should
diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c
index f1a141555f08..93794f01bb67 100644
--- a/tools/lib/bpf/usdt.c
+++ b/tools/lib/bpf/usdt.c
@@ -250,6 +250,7 @@ struct usdt_manager {
bool has_bpf_cookie;
bool has_sema_refcnt;
+ bool has_uprobe_multi;
};
struct usdt_manager *usdt_manager_new(struct bpf_object *obj)
@@ -284,6 +285,11 @@ struct usdt_manager *usdt_manager_new(struct bpf_object *obj)
*/
man->has_sema_refcnt = faccessat(AT_FDCWD, ref_ctr_sysfs_path, F_OK, AT_EACCESS) == 0;
+ /*
+ * Detect kernel support for uprobe multi link to be used for attaching
+ * usdt probes.
+ */
+ man->has_uprobe_multi = kernel_supports(obj, FEAT_UPROBE_MULTI_LINK);
return man;
}
@@ -808,6 +814,8 @@ struct bpf_link_usdt {
long abs_ip;
struct bpf_link *link;
} *uprobes;
+
+ struct bpf_link *multi_link;
};
static int bpf_link_usdt_detach(struct bpf_link *link)
@@ -816,6 +824,9 @@ static int bpf_link_usdt_detach(struct bpf_link *link)
struct usdt_manager *man = usdt_link->usdt_man;
int i;
+ bpf_link__destroy(usdt_link->multi_link);
+
+ /* When having multi_link, uprobe_cnt is 0 */
for (i = 0; i < usdt_link->uprobe_cnt; i++) {
/* detach underlying uprobe link */
bpf_link__destroy(usdt_link->uprobes[i].link);
@@ -852,8 +863,11 @@ static int bpf_link_usdt_detach(struct bpf_link *link)
* system is so exhausted on memory, it's the least of user's
* concerns, probably.
* So just do our best here to return those IDs to usdt_manager.
+ * Another edge case when we can legitimately get NULL is when
+ * new_cnt is zero, which can happen in some edge cases, so we
+ * need to be careful about that.
*/
- if (new_free_ids) {
+ if (new_free_ids || new_cnt == 0) {
memcpy(new_free_ids + man->free_spec_cnt, usdt_link->spec_ids,
usdt_link->spec_cnt * sizeof(*usdt_link->spec_ids));
man->free_spec_ids = new_free_ids;
@@ -943,32 +957,24 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct
const char *usdt_provider, const char *usdt_name,
__u64 usdt_cookie)
{
- int i, fd, err, spec_map_fd, ip_map_fd;
+ unsigned long *offsets = NULL, *ref_ctr_offsets = NULL;
+ int i, err, spec_map_fd, ip_map_fd;
LIBBPF_OPTS(bpf_uprobe_opts, opts);
struct hashmap *specs_hash = NULL;
struct bpf_link_usdt *link = NULL;
struct usdt_target *targets = NULL;
+ __u64 *cookies = NULL;
+ struct elf_fd elf_fd;
size_t target_cnt;
- Elf *elf;
spec_map_fd = bpf_map__fd(man->specs_map);
ip_map_fd = bpf_map__fd(man->ip_to_spec_id_map);
- fd = open(path, O_RDONLY | O_CLOEXEC);
- if (fd < 0) {
- err = -errno;
- pr_warn("usdt: failed to open ELF binary '%s': %d\n", path, err);
+ err = elf_open(path, &elf_fd);
+ if (err)
return libbpf_err_ptr(err);
- }
-
- elf = elf_begin(fd, ELF_C_READ_MMAP, NULL);
- if (!elf) {
- err = -EBADF;
- pr_warn("usdt: failed to parse ELF binary '%s': %s\n", path, elf_errmsg(-1));
- goto err_out;
- }
- err = sanity_check_usdt_elf(elf, path);
+ err = sanity_check_usdt_elf(elf_fd.elf, path);
if (err)
goto err_out;
@@ -981,7 +987,7 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct
/* discover USDT in given binary, optionally limiting
* activations to a given PID, if pid > 0
*/
- err = collect_usdt_targets(man, elf, path, pid, usdt_provider, usdt_name,
+ err = collect_usdt_targets(man, elf_fd.elf, path, pid, usdt_provider, usdt_name,
usdt_cookie, &targets, &target_cnt);
if (err <= 0) {
err = (err == 0) ? -ENOENT : err;
@@ -1004,10 +1010,21 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct
link->link.detach = &bpf_link_usdt_detach;
link->link.dealloc = &bpf_link_usdt_dealloc;
- link->uprobes = calloc(target_cnt, sizeof(*link->uprobes));
- if (!link->uprobes) {
- err = -ENOMEM;
- goto err_out;
+ if (man->has_uprobe_multi) {
+ offsets = calloc(target_cnt, sizeof(*offsets));
+ cookies = calloc(target_cnt, sizeof(*cookies));
+ ref_ctr_offsets = calloc(target_cnt, sizeof(*ref_ctr_offsets));
+
+ if (!offsets || !ref_ctr_offsets || !cookies) {
+ err = -ENOMEM;
+ goto err_out;
+ }
+ } else {
+ link->uprobes = calloc(target_cnt, sizeof(*link->uprobes));
+ if (!link->uprobes) {
+ err = -ENOMEM;
+ goto err_out;
+ }
}
for (i = 0; i < target_cnt; i++) {
@@ -1048,37 +1065,65 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct
goto err_out;
}
- opts.ref_ctr_offset = target->sema_off;
- opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0;
- uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path,
- target->rel_ip, &opts);
- err = libbpf_get_error(uprobe_link);
- if (err) {
- pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n",
- i, usdt_provider, usdt_name, path, err);
+ if (man->has_uprobe_multi) {
+ offsets[i] = target->rel_ip;
+ ref_ctr_offsets[i] = target->sema_off;
+ cookies[i] = spec_id;
+ } else {
+ opts.ref_ctr_offset = target->sema_off;
+ opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0;
+ uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path,
+ target->rel_ip, &opts);
+ err = libbpf_get_error(uprobe_link);
+ if (err) {
+ pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n",
+ i, usdt_provider, usdt_name, path, err);
+ goto err_out;
+ }
+
+ link->uprobes[i].link = uprobe_link;
+ link->uprobes[i].abs_ip = target->abs_ip;
+ link->uprobe_cnt++;
+ }
+ }
+
+ if (man->has_uprobe_multi) {
+ LIBBPF_OPTS(bpf_uprobe_multi_opts, opts_multi,
+ .ref_ctr_offsets = ref_ctr_offsets,
+ .offsets = offsets,
+ .cookies = cookies,
+ .cnt = target_cnt,
+ );
+
+ link->multi_link = bpf_program__attach_uprobe_multi(prog, pid, path,
+ NULL, &opts_multi);
+ if (!link->multi_link) {
+ err = -errno;
+ pr_warn("usdt: failed to attach uprobe multi for '%s:%s' in '%s': %d\n",
+ usdt_provider, usdt_name, path, err);
goto err_out;
}
- link->uprobes[i].link = uprobe_link;
- link->uprobes[i].abs_ip = target->abs_ip;
- link->uprobe_cnt++;
+ free(offsets);
+ free(ref_ctr_offsets);
+ free(cookies);
}
free(targets);
hashmap__free(specs_hash);
- elf_end(elf);
- close(fd);
-
+ elf_close(&elf_fd);
return &link->link;
err_out:
+ free(offsets);
+ free(ref_ctr_offsets);
+ free(cookies);
+
if (link)
bpf_link__destroy(&link->link);
free(targets);
hashmap__free(specs_hash);
- if (elf)
- elf_end(elf);
- close(fd);
+ elf_close(&elf_fd);
return libbpf_err_ptr(err);
}
generated by cgit v1.2.3 (git 2.39.0) at 2024-12-29 20:35:44 +0300