diff options
Diffstat (limited to 'tools')
67 files changed, 3201 insertions, 99 deletions
diff --git a/tools/bpf/bpftool/Documentation/bpftool-gen.rst b/tools/bpf/bpftool/Documentation/bpftool-gen.rst index 68454ef28f58..5006e724d1bc 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-gen.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-gen.rst @@ -260,9 +260,9 @@ EXAMPLES This is example BPF application with two BPF programs and a mix of BPF maps and global variables. Source code is split across two source code files. -**$ clang -target bpf -g example1.bpf.c -o example1.bpf.o** +**$ clang --target=bpf -g example1.bpf.c -o example1.bpf.o** -**$ clang -target bpf -g example2.bpf.c -o example2.bpf.o** +**$ clang --target=bpf -g example2.bpf.c -o example2.bpf.o** **$ bpftool gen object example.bpf.o example1.bpf.o example2.bpf.o** diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile index 681fbcc5ed50..e9154ace80ff 100644 --- a/tools/bpf/bpftool/Makefile +++ b/tools/bpf/bpftool/Makefile @@ -216,7 +216,7 @@ $(OUTPUT)%.bpf.o: skeleton/%.bpf.c $(OUTPUT)vmlinux.h $(LIBBPF_BOOTSTRAP) -I$(srctree)/tools/include/uapi/ \ -I$(LIBBPF_BOOTSTRAP_INCLUDE) \ -g -O2 -Wall -fno-stack-protector \ - -target bpf -c $< -o $@ + --target=bpf -c $< -o $@ $(Q)$(LLVM_STRIP) -g $@ $(OUTPUT)%.skel.h: $(OUTPUT)%.bpf.o $(BPFTOOL_BOOTSTRAP) diff --git a/tools/bpf/bpftool/btf_dumper.c b/tools/bpf/bpftool/btf_dumper.c index 294de231db99..1b7f69714604 100644 --- a/tools/bpf/bpftool/btf_dumper.c +++ b/tools/bpf/bpftool/btf_dumper.c @@ -835,7 +835,7 @@ static void dotlabel_puts(const char *s) case '|': case ' ': putchar('\\'); - /* fallthrough */ + fallthrough; default: putchar(*s); } diff --git a/tools/bpf/bpftool/feature.c b/tools/bpf/bpftool/feature.c index 0675d6a46413..edda4fc2c4d0 100644 --- a/tools/bpf/bpftool/feature.c +++ b/tools/bpf/bpftool/feature.c @@ -757,7 +757,7 @@ probe_helpers_for_progtype(enum bpf_prog_type prog_type, case BPF_FUNC_probe_write_user: if (!full_mode) continue; - /* fallthrough */ + fallthrough; default: probe_res |= probe_helper_for_progtype(prog_type, supported_type, define_prefix, id, prog_type_str, diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c index 2d786072ed0d..65a168df63bc 100644 --- a/tools/bpf/bpftool/link.c +++ b/tools/bpf/bpftool/link.c @@ -5,6 +5,7 @@ #include <linux/err.h> #include <linux/netfilter.h> #include <linux/netfilter_arp.h> +#include <linux/perf_event.h> #include <net/if.h> #include <stdio.h> #include <unistd.h> @@ -14,8 +15,78 @@ #include "json_writer.h" #include "main.h" +#include "xlated_dumper.h" + +#define PERF_HW_CACHE_LEN 128 static struct hashmap *link_table; +static struct dump_data dd; + +static const char *perf_type_name[PERF_TYPE_MAX] = { + [PERF_TYPE_HARDWARE] = "hardware", + [PERF_TYPE_SOFTWARE] = "software", + [PERF_TYPE_TRACEPOINT] = "tracepoint", + [PERF_TYPE_HW_CACHE] = "hw-cache", + [PERF_TYPE_RAW] = "raw", + [PERF_TYPE_BREAKPOINT] = "breakpoint", +}; + +const char *event_symbols_hw[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = "cpu-cycles", + [PERF_COUNT_HW_INSTRUCTIONS] = "instructions", + [PERF_COUNT_HW_CACHE_REFERENCES] = "cache-references", + [PERF_COUNT_HW_CACHE_MISSES] = "cache-misses", + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = "branch-instructions", + [PERF_COUNT_HW_BRANCH_MISSES] = "branch-misses", + [PERF_COUNT_HW_BUS_CYCLES] = "bus-cycles", + [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = "stalled-cycles-frontend", + [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = "stalled-cycles-backend", + [PERF_COUNT_HW_REF_CPU_CYCLES] = "ref-cycles", +}; + +const char *event_symbols_sw[PERF_COUNT_SW_MAX] = { + [PERF_COUNT_SW_CPU_CLOCK] = "cpu-clock", + [PERF_COUNT_SW_TASK_CLOCK] = "task-clock", + [PERF_COUNT_SW_PAGE_FAULTS] = "page-faults", + [PERF_COUNT_SW_CONTEXT_SWITCHES] = "context-switches", + [PERF_COUNT_SW_CPU_MIGRATIONS] = "cpu-migrations", + [PERF_COUNT_SW_PAGE_FAULTS_MIN] = "minor-faults", + [PERF_COUNT_SW_PAGE_FAULTS_MAJ] = "major-faults", + [PERF_COUNT_SW_ALIGNMENT_FAULTS] = "alignment-faults", + [PERF_COUNT_SW_EMULATION_FAULTS] = "emulation-faults", + [PERF_COUNT_SW_DUMMY] = "dummy", + [PERF_COUNT_SW_BPF_OUTPUT] = "bpf-output", + [PERF_COUNT_SW_CGROUP_SWITCHES] = "cgroup-switches", +}; + +const char *evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX] = { + [PERF_COUNT_HW_CACHE_L1D] = "L1-dcache", + [PERF_COUNT_HW_CACHE_L1I] = "L1-icache", + [PERF_COUNT_HW_CACHE_LL] = "LLC", + [PERF_COUNT_HW_CACHE_DTLB] = "dTLB", + [PERF_COUNT_HW_CACHE_ITLB] = "iTLB", + [PERF_COUNT_HW_CACHE_BPU] = "branch", + [PERF_COUNT_HW_CACHE_NODE] = "node", +}; + +const char *evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX] = { + [PERF_COUNT_HW_CACHE_OP_READ] = "load", + [PERF_COUNT_HW_CACHE_OP_WRITE] = "store", + [PERF_COUNT_HW_CACHE_OP_PREFETCH] = "prefetch", +}; + +const char *evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX] = { + [PERF_COUNT_HW_CACHE_RESULT_ACCESS] = "refs", + [PERF_COUNT_HW_CACHE_RESULT_MISS] = "misses", +}; + +#define perf_event_name(array, id) ({ \ + const char *event_str = NULL; \ + \ + if ((id) >= 0 && (id) < ARRAY_SIZE(array)) \ + event_str = array[id]; \ + event_str; \ +}) static int link_parse_fd(int *argc, char ***argv) { @@ -166,6 +237,154 @@ static int get_prog_info(int prog_id, struct bpf_prog_info *info) return err; } +static int cmp_u64(const void *A, const void *B) +{ + const __u64 *a = A, *b = B; + + return *a - *b; +} + +static void +show_kprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr) +{ + __u32 i, j = 0; + __u64 *addrs; + + jsonw_bool_field(json_wtr, "retprobe", + info->kprobe_multi.flags & BPF_F_KPROBE_MULTI_RETURN); + jsonw_uint_field(json_wtr, "func_cnt", info->kprobe_multi.count); + jsonw_name(json_wtr, "funcs"); + jsonw_start_array(json_wtr); + addrs = u64_to_ptr(info->kprobe_multi.addrs); + qsort(addrs, info->kprobe_multi.count, sizeof(addrs[0]), cmp_u64); + + /* Load it once for all. */ + if (!dd.sym_count) + kernel_syms_load(&dd); + for (i = 0; i < dd.sym_count; i++) { + if (dd.sym_mapping[i].address != addrs[j]) + continue; + jsonw_start_object(json_wtr); + jsonw_uint_field(json_wtr, "addr", dd.sym_mapping[i].address); + jsonw_string_field(json_wtr, "func", dd.sym_mapping[i].name); + /* Print null if it is vmlinux */ + if (dd.sym_mapping[i].module[0] == '\0') { + jsonw_name(json_wtr, "module"); + jsonw_null(json_wtr); + } else { + jsonw_string_field(json_wtr, "module", dd.sym_mapping[i].module); + } + jsonw_end_object(json_wtr); + if (j++ == info->kprobe_multi.count) + break; + } + jsonw_end_array(json_wtr); +} + +static void +show_perf_event_kprobe_json(struct bpf_link_info *info, json_writer_t *wtr) +{ + jsonw_bool_field(wtr, "retprobe", info->perf_event.type == BPF_PERF_EVENT_KRETPROBE); + jsonw_uint_field(wtr, "addr", info->perf_event.kprobe.addr); + jsonw_string_field(wtr, "func", + u64_to_ptr(info->perf_event.kprobe.func_name)); + jsonw_uint_field(wtr, "offset", info->perf_event.kprobe.offset); +} + +static void +show_perf_event_uprobe_json(struct bpf_link_info *info, json_writer_t *wtr) +{ + jsonw_bool_field(wtr, "retprobe", info->perf_event.type == BPF_PERF_EVENT_URETPROBE); + jsonw_string_field(wtr, "file", + u64_to_ptr(info->perf_event.uprobe.file_name)); + jsonw_uint_field(wtr, "offset", info->perf_event.uprobe.offset); +} + +static void +show_perf_event_tracepoint_json(struct bpf_link_info *info, json_writer_t *wtr) +{ + jsonw_string_field(wtr, "tracepoint", + u64_to_ptr(info->perf_event.tracepoint.tp_name)); +} + +static char *perf_config_hw_cache_str(__u64 config) +{ + const char *hw_cache, *result, *op; + char *str = malloc(PERF_HW_CACHE_LEN); + + if (!str) { + p_err("mem alloc failed"); + return NULL; + } + + hw_cache = perf_event_name(evsel__hw_cache, config & 0xff); + if (hw_cache) + snprintf(str, PERF_HW_CACHE_LEN, "%s-", hw_cache); + else + snprintf(str, PERF_HW_CACHE_LEN, "%lld-", config & 0xff); + + op = perf_event_name(evsel__hw_cache_op, (config >> 8) & 0xff); + if (op) + snprintf(str + strlen(str), PERF_HW_CACHE_LEN - strlen(str), + "%s-", op); + else + snprintf(str + strlen(str), PERF_HW_CACHE_LEN - strlen(str), + "%lld-", (config >> 8) & 0xff); + + result = perf_event_name(evsel__hw_cache_result, config >> 16); + if (result) + snprintf(str + strlen(str), PERF_HW_CACHE_LEN - strlen(str), + "%s", result); + else + snprintf(str + strlen(str), PERF_HW_CACHE_LEN - strlen(str), + "%lld", config >> 16); + return str; +} + +static const char *perf_config_str(__u32 type, __u64 config) +{ + const char *perf_config; + + switch (type) { + case PERF_TYPE_HARDWARE: + perf_config = perf_event_name(event_symbols_hw, config); + break; + case PERF_TYPE_SOFTWARE: + perf_config = perf_event_name(event_symbols_sw, config); + break; + case PERF_TYPE_HW_CACHE: + perf_config = perf_config_hw_cache_str(config); + break; + default: + perf_config = NULL; + break; + } + return perf_config; +} + +static void +show_perf_event_event_json(struct bpf_link_info *info, json_writer_t *wtr) +{ + __u64 config = info->perf_event.event.config; + __u32 type = info->perf_event.event.type; + const char *perf_type, *perf_config; + + perf_type = perf_event_name(perf_type_name, type); + if (perf_type) + jsonw_string_field(wtr, "event_type", perf_type); + else + jsonw_uint_field(wtr, "event_type", type); + + perf_config = perf_config_str(type, config); + if (perf_config) + jsonw_string_field(wtr, "event_config", perf_config); + else + jsonw_uint_field(wtr, "event_config", config); + + if (type == PERF_TYPE_HW_CACHE && perf_config) + free((void *)perf_config); +} + static int show_link_close_json(int fd, struct bpf_link_info *info) { struct bpf_prog_info prog_info; @@ -218,6 +437,29 @@ static int show_link_close_json(int fd, struct bpf_link_info *info) jsonw_uint_field(json_wtr, "map_id", info->struct_ops.map_id); break; + case BPF_LINK_TYPE_KPROBE_MULTI: + show_kprobe_multi_json(info, json_wtr); + break; + case BPF_LINK_TYPE_PERF_EVENT: + switch (info->perf_event.type) { + case BPF_PERF_EVENT_EVENT: + show_perf_event_event_json(info, json_wtr); + break; + case BPF_PERF_EVENT_TRACEPOINT: + show_perf_event_tracepoint_json(info, json_wtr); + break; + case BPF_PERF_EVENT_KPROBE: + case BPF_PERF_EVENT_KRETPROBE: + show_perf_event_kprobe_json(info, json_wtr); + break; + case BPF_PERF_EVENT_UPROBE: + case BPF_PERF_EVENT_URETPROBE: + show_perf_event_uprobe_json(info, json_wtr); + break; + default: + break; + } + break; default: break; } @@ -351,6 +593,113 @@ void netfilter_dump_plain(const struct bpf_link_info *info) printf(" flags 0x%x", info->netfilter.flags); } +static void show_kprobe_multi_plain(struct bpf_link_info *info) +{ + __u32 i, j = 0; + __u64 *addrs; + + if (!info->kprobe_multi.count) + return; + + if (info->kprobe_multi.flags & BPF_F_KPROBE_MULTI_RETURN) + printf("\n\tkretprobe.multi "); + else + printf("\n\tkprobe.multi "); + printf("func_cnt %u ", info->kprobe_multi.count); + addrs = (__u64 *)u64_to_ptr(info->kprobe_multi.addrs); + qsort(addrs, info->kprobe_multi.count, sizeof(__u64), cmp_u64); + + /* Load it once for all. */ + if (!dd.sym_count) + kernel_syms_load(&dd); + if (!dd.sym_count) + return; + + printf("\n\t%-16s %s", "addr", "func [module]"); + for (i = 0; i < dd.sym_count; i++) { + if (dd.sym_mapping[i].address != addrs[j]) + continue; + printf("\n\t%016lx %s", + dd.sym_mapping[i].address, dd.sym_mapping[i].name); + if (dd.sym_mapping[i].module[0] != '\0') + printf(" [%s] ", dd.sym_mapping[i].module); + else + printf(" "); + + if (j++ == info->kprobe_multi.count) + break; + } +} + +static void show_perf_event_kprobe_plain(struct bpf_link_info *info) +{ + const char *buf; + + buf = u64_to_ptr(info->perf_event.kprobe.func_name); + if (buf[0] == '\0' && !info->perf_event.kprobe.addr) + return; + + if (info->perf_event.type == BPF_PERF_EVENT_KRETPROBE) + printf("\n\tkretprobe "); + else + printf("\n\tkprobe "); + if (info->perf_event.kprobe.addr) + printf("%llx ", info->perf_event.kprobe.addr); + printf("%s", buf); + if (info->perf_event.kprobe.offset) + printf("+%#x", info->perf_event.kprobe.offset); + printf(" "); +} + +static void show_perf_event_uprobe_plain(struct bpf_link_info *info) +{ + const char *buf; + + buf = u64_to_ptr(info->perf_event.uprobe.file_name); + if (buf[0] == '\0') + return; + + if (info->perf_event.type == BPF_PERF_EVENT_URETPROBE) + printf("\n\turetprobe "); + else + printf("\n\tuprobe "); + printf("%s+%#x ", buf, info->perf_event.uprobe.offset); +} + +static void show_perf_event_tracepoint_plain(struct bpf_link_info *info) +{ + const char *buf; + + buf = u64_to_ptr(info->perf_event.tracepoint.tp_name); + if (buf[0] == '\0') + return; + + printf("\n\ttracepoint %s ", buf); +} + +static void show_perf_event_event_plain(struct bpf_link_info *info) +{ + __u64 config = info->perf_event.event.config; + __u32 type = info->perf_event.event.type; + const char *perf_type, *perf_config; + + printf("\n\tevent "); + perf_type = perf_event_name(perf_type_name, type); + if (perf_type) + printf("%s:", perf_type); + else + printf("%u :", type); + + perf_config = perf_config_str(type, config); + if (perf_config) + printf("%s ", perf_config); + else + printf("%llu ", config); + + if (type == PERF_TYPE_HW_CACHE && perf_config) + free((void *)perf_config); +} + static int show_link_close_plain(int fd, struct bpf_link_info *info) { struct bpf_prog_info prog_info; @@ -396,6 +745,29 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info) case BPF_LINK_TYPE_NETFILTER: netfilter_dump_plain(info); break; + case BPF_LINK_TYPE_KPROBE_MULTI: + show_kprobe_multi_plain(info); + break; + case BPF_LINK_TYPE_PERF_EVENT: + switch (info->perf_event.type) { + case BPF_PERF_EVENT_EVENT: + show_perf_event_event_plain(info); + break; + case BPF_PERF_EVENT_TRACEPOINT: + show_perf_event_tracepoint_plain(info); + break; + case BPF_PERF_EVENT_KPROBE: + case BPF_PERF_EVENT_KRETPROBE: + show_perf_event_kprobe_plain(info); + break; + case BPF_PERF_EVENT_UPROBE: + case BPF_PERF_EVENT_URETPROBE: + show_perf_event_uprobe_plain(info); + break; + default: + break; + } + break; default: break; } @@ -417,10 +789,13 @@ static int do_show_link(int fd) { struct bpf_link_info info; __u32 len = sizeof(info); - char buf[256]; + __u64 *addrs = NULL; + char buf[PATH_MAX]; + int count; int err; memset(&info, 0, sizeof(info)); + buf[0] = '\0'; again: err = bpf_link_get_info_by_fd(fd, &info, &len); if (err) { @@ -431,22 +806,67 @@ again: } if (info.type == BPF_LINK_TYPE_RAW_TRACEPOINT && !info.raw_tracepoint.tp_name) { - info.raw_tracepoint.tp_name = (unsigned long)&buf; + info.raw_tracepoint.tp_name = ptr_to_u64(&buf); info.raw_tracepoint.tp_name_len = sizeof(buf); goto again; } if (info.type == BPF_LINK_TYPE_ITER && !info.iter.target_name) { - info.iter.target_name = (unsigned long)&buf; + info.iter.target_name = ptr_to_u64(&buf); info.iter.target_name_len = sizeof(buf); goto again; } + if (info.type == BPF_LINK_TYPE_KPROBE_MULTI && + !info.kprobe_multi.addrs) { + count = info.kprobe_multi.count; + if (count) { + addrs = calloc(count, sizeof(__u64)); + if (!addrs) { + p_err("mem alloc failed"); + close(fd); + return -ENOMEM; + } + info.kprobe_multi.addrs = ptr_to_u64(addrs); + goto again; + } + } + if (info.type == BPF_LINK_TYPE_PERF_EVENT) { + switch (info.perf_event.type) { + case BPF_PERF_EVENT_TRACEPOINT: + if (!info.perf_event.tracepoint.tp_name) { + info.perf_event.tracepoint.tp_name = ptr_to_u64(&buf); + info.perf_event.tracepoint.name_len = sizeof(buf); + goto again; + } + break; + case BPF_PERF_EVENT_KPROBE: + case BPF_PERF_EVENT_KRETPROBE: + if (!info.perf_event.kprobe.func_name) { + info.perf_event.kprobe.func_name = ptr_to_u64(&buf); + info.perf_event.kprobe.name_len = sizeof(buf); + goto again; + } + break; + case BPF_PERF_EVENT_UPROBE: + case BPF_PERF_EVENT_URETPROBE: + if (!info.perf_event.uprobe.file_name) { + info.perf_event.uprobe.file_name = ptr_to_u64(&buf); + info.perf_event.uprobe.name_len = sizeof(buf); + goto again; + } + break; + default: + break; + } + } if (json_output) show_link_close_json(fd, &info); else show_link_close_plain(fd, &info); + if (addrs) + free(addrs); close(fd); return 0; } @@ -471,7 +891,8 @@ static int do_show(int argc, char **argv) fd = link_parse_fd(&argc, &argv); if (fd < 0) return fd; - return do_show_link(fd); + do_show_link(fd); + goto out; } if (argc) @@ -510,6 +931,9 @@ static int do_show(int argc, char **argv) if (show_pinned) delete_pinned_obj_table(link_table); +out: + if (dd.sym_count) + kernel_syms_destroy(&dd); return errno == ENOENT ? 0 : -1; } diff --git a/tools/bpf/bpftool/skeleton/pid_iter.bpf.c b/tools/bpf/bpftool/skeleton/pid_iter.bpf.c index eb05ea53afb1..26004f0c5a6a 100644 --- a/tools/bpf/bpftool/skeleton/pid_iter.bpf.c +++ b/tools/bpf/bpftool/skeleton/pid_iter.bpf.c @@ -15,6 +15,19 @@ enum bpf_obj_type { BPF_OBJ_BTF, }; +struct bpf_perf_link___local { + struct bpf_link link; + struct file *perf_file; +} __attribute__((preserve_access_index)); + +struct perf_event___local { + u64 bpf_cookie; +} __attribute__((preserve_access_index)); + +enum bpf_link_type___local { + BPF_LINK_TYPE_PERF_EVENT___local = 7, +}; + extern const void bpf_link_fops __ksym; extern const void bpf_map_fops __ksym; extern const void bpf_prog_fops __ksym; @@ -41,10 +54,10 @@ static __always_inline __u32 get_obj_id(void *ent, enum bpf_obj_type type) /* could be used only with BPF_LINK_TYPE_PERF_EVENT links */ static __u64 get_bpf_cookie(struct bpf_link *link) { - struct bpf_perf_link *perf_link; - struct perf_event *event; + struct bpf_perf_link___local *perf_link; + struct perf_event___local *event; - perf_link = container_of(link, struct bpf_perf_link, link); + perf_link = container_of(link, struct bpf_perf_link___local, link); event = BPF_CORE_READ(perf_link, perf_file, private_data); return BPF_CORE_READ(event, bpf_cookie); } @@ -84,10 +97,13 @@ int iter(struct bpf_iter__task_file *ctx) e.pid = task->tgid; e.id = get_obj_id(file->private_data, obj_type); - if (obj_type == BPF_OBJ_LINK) { + if (obj_type == BPF_OBJ_LINK && + bpf_core_enum_value_exists(enum bpf_link_type___local, + BPF_LINK_TYPE_PERF_EVENT___local)) { struct bpf_link *link = (struct bpf_link *) file->private_data; - if (BPF_CORE_READ(link, type) == BPF_LINK_TYPE_PERF_EVENT) { + if (link->type == bpf_core_enum_value(enum bpf_link_type___local, + BPF_LINK_TYPE_PERF_EVENT___local)) { e.has_bpf_cookie = true; e.bpf_cookie = get_bpf_cookie(link); } diff --git a/tools/bpf/bpftool/skeleton/profiler.bpf.c b/tools/bpf/bpftool/skeleton/profiler.bpf.c index ce5b65e07ab1..2f80edc682f1 100644 --- a/tools/bpf/bpftool/skeleton/profiler.bpf.c +++ b/tools/bpf/bpftool/skeleton/profiler.bpf.c @@ -4,6 +4,12 @@ #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> +struct bpf_perf_event_value___local { + __u64 counter; + __u64 enabled; + __u64 running; +} __attribute__((preserve_access_index)); + /* map of perf event fds, num_cpu * num_metric entries */ struct { __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); @@ -15,14 +21,14 @@ struct { struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(key_size, sizeof(u32)); - __uint(value_size, sizeof(struct bpf_perf_event_value)); + __uint(value_size, sizeof(struct bpf_perf_event_value___local)); } fentry_readings SEC(".maps"); /* accumulated readings */ struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(key_size, sizeof(u32)); - __uint(value_size, sizeof(struct bpf_perf_event_value)); + __uint(value_size, sizeof(struct bpf_perf_event_value___local)); } accum_readings SEC(".maps"); /* sample counts, one per cpu */ @@ -39,7 +45,7 @@ const volatile __u32 num_metric = 1; SEC("fentry/XXX") int BPF_PROG(fentry_XXX) { - struct bpf_perf_event_value *ptrs[MAX_NUM_MATRICS]; + struct bpf_perf_event_value___local *ptrs[MAX_NUM_MATRICS]; u32 key = bpf_get_smp_processor_id(); u32 i; @@ -53,10 +59,10 @@ int BPF_PROG(fentry_XXX) } for (i = 0; i < num_metric && i < MAX_NUM_MATRICS; i++) { - struct bpf_perf_event_value reading; + struct bpf_perf_event_value___local reading; int err; - err = bpf_perf_event_read_value(&events, key, &reading, + err = bpf_perf_event_read_value(&events, key, (void *)&reading, sizeof(reading)); if (err) return 0; @@ -68,14 +74,14 @@ int BPF_PROG(fentry_XXX) } static inline void -fexit_update_maps(u32 id, struct bpf_perf_event_value *after) +fexit_update_maps(u32 id, struct bpf_perf_event_value___local *after) { - struct bpf_perf_event_value *before, diff; + struct bpf_perf_event_value___local *before, diff; before = bpf_map_lookup_elem(&fentry_readings, &id); /* only account samples with a valid fentry_reading */ if (before && before->counter) { - struct bpf_perf_event_value *accum; + struct bpf_perf_event_value___local *accum; diff.counter = after->counter - before->counter; diff.enabled = after->enabled - before->enabled; @@ -93,7 +99,7 @@ fexit_update_maps(u32 id, struct bpf_perf_event_value *after) SEC("fexit/XXX") int BPF_PROG(fexit_XXX) { - struct bpf_perf_event_value readings[MAX_NUM_MATRICS]; + struct bpf_perf_event_value___local readings[MAX_NUM_MATRICS]; u32 cpu = bpf_get_smp_processor_id(); u32 i, zero = 0; int err; @@ -102,7 +108,8 @@ int BPF_PROG(fexit_XXX) /* read all events before updating the maps, to reduce error */ for (i = 0; i < num_metric && i < MAX_NUM_MATRICS; i++) { err = bpf_perf_event_read_value(&events, cpu + i * num_cpu, - readings + i, sizeof(*readings)); + (void *)(readings + i), + sizeof(*readings)); if (err) return 0; } diff --git a/tools/bpf/bpftool/xlated_dumper.c b/tools/bpf/bpftool/xlated_dumper.c index da608e10c843..567f56dfd9f1 100644 --- a/tools/bpf/bpftool/xlated_dumper.c +++ b/tools/bpf/bpftool/xlated_dumper.c @@ -46,7 +46,11 @@ out: } dd->sym_mapping = tmp; sym = &dd->sym_mapping[dd->sym_count]; - if (sscanf(buff, "%p %*c %s", &address, sym->name) != 2) + + /* module is optional */ + sym->module[0] = '\0'; + /* trim the square brackets around the module name */ + if (sscanf(buff, "%p %*c %s [%[^]]s", &address, sym->name, sym->module) < 2) continue; sym->address = (unsigned long)address; if (!strcmp(sym->name, "__bpf_call_base")) { diff --git a/tools/bpf/bpftool/xlated_dumper.h b/tools/bpf/bpftool/xlated_dumper.h index 9a946377b0e6..db3ba0671501 100644 --- a/tools/bpf/bpftool/xlated_dumper.h +++ b/tools/bpf/bpftool/xlated_dumper.h @@ -5,12 +5,14 @@ #define __BPF_TOOL_XLATED_DUMPER_H #define SYM_MAX_NAME 256 +#define MODULE_MAX_NAME 64 struct bpf_prog_linfo; struct kernel_sym { unsigned long address; char name[SYM_MAX_NAME]; + char module[MODULE_MAX_NAME]; }; struct dump_data { diff --git a/tools/bpf/runqslower/Makefile b/tools/bpf/runqslower/Makefile index 47acf6936516..d8288936c912 100644 --- a/tools/bpf/runqslower/Makefile +++ b/tools/bpf/runqslower/Makefile @@ -62,7 +62,7 @@ $(OUTPUT)/%.skel.h: $(OUTPUT)/%.bpf.o | $(BPFTOOL) $(QUIET_GEN)$(BPFTOOL) gen skeleton $< > $@ $(OUTPUT)/%.bpf.o: %.bpf.c $(BPFOBJ) | $(OUTPUT) - $(QUIET_GEN)$(CLANG) -g -O2 -target bpf $(INCLUDES) \ + $(QUIET_GEN)$(CLANG) -g -O2 --target=bpf $(INCLUDES) \ -c $(filter %.c,$^) -o $@ && \ $(LLVM_STRIP) -g $@ diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile index 0f0aa9b7d7b5..6654b1a35ab3 100644 --- a/tools/build/feature/Makefile +++ b/tools/build/feature/Makefile @@ -372,7 +372,7 @@ $(OUTPUT)test-libzstd.bin: $(BUILD) -lzstd $(OUTPUT)test-clang-bpf-co-re.bin: - $(CLANG) -S -g -target bpf -o - $(patsubst %.bin,%.c,$(@F)) | \ + $(CLANG) -S -g --target=bpf -o - $(patsubst %.bin,%.c,$(@F)) | \ grep BTF_KIND_VAR $(OUTPUT)test-file-handle.bin: diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 60a9d59beeab..600d0caebbd8 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1057,6 +1057,16 @@ enum bpf_link_type { MAX_BPF_LINK_TYPE, }; +enum bpf_perf_event_type { + BPF_PERF_EVENT_UNSPEC = 0, + BPF_PERF_EVENT_UPROBE = 1, + BPF_PERF_EVENT_URETPROBE = 2, + BPF_PERF_EVENT_KPROBE = 3, + BPF_PERF_EVENT_KRETPROBE = 4, + BPF_PERF_EVENT_TRACEPOINT = 5, + BPF_PERF_EVENT_EVENT = 6, +}; + /* cgroup-bpf attach flags used in BPF_PROG_ATTACH command * * NONE(default): No further bpf programs allowed in the subtree. @@ -6439,6 +6449,36 @@ struct bpf_link_info { __s32 priority; __u32 flags; } netfilter; + struct { + __aligned_u64 addrs; + __u32 count; /* in/out: kprobe_multi function count */ + __u32 flags; + } kprobe_multi; + struct { + __u32 type; /* enum bpf_perf_event_type */ + __u32 :32; + union { + struct { + __aligned_u64 file_name; /* in/out */ + __u32 name_len; + __u32 offset; /* offset from file_name */ + } uprobe; /* BPF_PERF_EVENT_UPROBE, BPF_PERF_EVENT_URETPROBE */ + struct { + __aligned_u64 func_name; /* in/out */ + __u32 name_len; + __u32 offset; /* offset from func_name */ + __u64 addr; + } kprobe; /* BPF_PERF_EVENT_KPROBE, BPF_PERF_EVENT_KRETPROBE */ + struct { + __aligned_u64 tp_name; /* in/out */ + __u32 name_len; + } tracepoint; /* BPF_PERF_EVENT_TRACEPOINT */ + struct { + __u64 config; + __u32 type; + } event; /* BPF_PERF_EVENT_EVENT */ + }; + } perf_event; }; } __attribute__((aligned(8))); diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index ed86b37d8024..3b0da19715e1 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -741,6 +741,14 @@ int bpf_link_create(int prog_fd, int target_fd, if (!OPTS_ZEROED(opts, tracing)) return libbpf_err(-EINVAL); break; + case BPF_NETFILTER: + attr.link_create.netfilter.pf = OPTS_GET(opts, netfilter.pf, 0); + attr.link_create.netfilter.hooknum = OPTS_GET(opts, netfilter.hooknum, 0); + attr.link_create.netfilter.priority = OPTS_GET(opts, netfilter.priority, 0); + attr.link_create.netfilter.flags = OPTS_GET(opts, netfilter.flags, 0); + if (!OPTS_ZEROED(opts, netfilter)) + return libbpf_err(-EINVAL); + break; default: if (!OPTS_ZEROED(opts, flags)) return libbpf_err(-EINVAL); diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 9aa0ee473754..c676295ab9bf 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -349,6 +349,12 @@ struct bpf_link_create_opts { struct { __u64 cookie; } tracing; + struct { + __u32 pf; + __u32 hooknum; + __s32 priority; + __u32 flags; + } netfilter; }; size_t :0; }; diff --git a/tools/lib/bpf/hashmap.h b/tools/lib/bpf/hashmap.h index 0a5bf1937a7c..c12f8320e668 100644 --- a/tools/lib/bpf/hashmap.h +++ b/tools/lib/bpf/hashmap.h @@ -80,16 +80,6 @@ struct hashmap { size_t sz; }; -#define HASHMAP_INIT(hash_fn, equal_fn, ctx) { \ - .hash_fn = (hash_fn), \ - .equal_fn = (equal_fn), \ - .ctx = (ctx), \ - .buckets = NULL, \ - .cap = 0, \ - .cap_bits = 0, \ - .sz = 0, \ -} - void hashmap__init(struct hashmap *map, hashmap_hash_fn hash_fn, hashmap_equal_fn equal_fn, void *ctx); struct hashmap *hashmap__new(hashmap_hash_fn hash_fn, diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 214f828ece6b..63311a73c16d 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -5471,6 +5471,10 @@ static int load_module_btfs(struct bpf_object *obj) err = bpf_btf_get_next_id(id, &id); if (err && errno == ENOENT) return 0; + if (err && errno == EPERM) { + pr_debug("skipping module BTFs loading, missing privileges\n"); + return 0; + } if (err) { err = -errno; pr_warn("failed to iterate BTF objects: %d\n", err); @@ -6157,7 +6161,11 @@ static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_progra if (main_prog == subprog) return 0; relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos)); - if (!relos) + /* if new count is zero, reallocarray can return a valid NULL result; + * in this case the previous pointer will be freed, so we *have to* + * reassign old pointer to the new value (even if it's NULL) + */ + if (!relos && new_cnt) return -ENOMEM; if (subprog->nr_reloc) memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc, @@ -8528,7 +8536,8 @@ int bpf_program__set_insns(struct bpf_program *prog, return -EBUSY; insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns)); - if (!insns) { + /* NULL is a valid return from reallocarray if the new count is zero */ + if (!insns && new_insn_cnt) { pr_warn("prog '%s': failed to realloc prog code\n", prog->name); return -ENOMEM; } @@ -8558,13 +8567,31 @@ enum bpf_prog_type bpf_program__type(const struct bpf_program *prog) return prog->type; } +static size_t custom_sec_def_cnt; +static struct bpf_sec_def *custom_sec_defs; +static struct bpf_sec_def custom_fallback_def; +static bool has_custom_fallback_def; +static int last_custom_sec_def_handler_id; + int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) { if (prog->obj->loaded) return libbpf_err(-EBUSY); + /* if type is not changed, do nothing */ + if (prog->type == type) + return 0; + prog->type = type; - prog->sec_def = NULL; + + /* If a program type was changed, we need to reset associated SEC() + * handler, as it will be invalid now. The only exception is a generic + * fallback handler, which by definition is program type-agnostic and + * is a catch-all custom handler, optionally set by the application, + * so should be able to handle any type of BPF program. + */ + if (prog->sec_def != &custom_fallback_def) + prog->sec_def = NULL; return 0; } @@ -8740,13 +8767,6 @@ static const struct bpf_sec_def section_defs[] = { SEC_DEF("netfilter", NETFILTER, BPF_NETFILTER, SEC_NONE), }; -static size_t custom_sec_def_cnt; -static struct bpf_sec_def *custom_sec_defs; -static struct bpf_sec_def custom_fallback_def; -static bool has_custom_fallback_def; - -static int last_custom_sec_def_handler_id; - int libbpf_register_prog_handler(const char *sec, enum bpf_prog_type prog_type, enum bpf_attach_type exp_attach_type, @@ -8826,7 +8846,11 @@ int libbpf_unregister_prog_handler(int handler_id) /* try to shrink the array, but it's ok if we couldn't */ sec_defs = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt, sizeof(*sec_defs)); - if (sec_defs) + /* if new count is zero, reallocarray can return a valid NULL result; + * in this case the previous pointer will be freed, so we *have to* + * reassign old pointer to the new value (even if it's NULL) + */ + if (sec_defs || custom_sec_def_cnt == 0) custom_sec_defs = sec_defs; return 0; @@ -10224,6 +10248,18 @@ static const char *tracefs_uprobe_events(void) return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events"; } +static const char *tracefs_available_filter_functions(void) +{ + return use_debugfs() ? DEBUGFS"/available_filter_functions" + : TRACEFS"/available_filter_functions"; +} + +static const char *tracefs_available_filter_functions_addrs(void) +{ + return use_debugfs() ? DEBUGFS"/available_filter_functions_addrs" + : TRACEFS"/available_filter_functions_addrs"; +} + static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz, const char *kfunc_name, size_t offset) { @@ -10539,25 +10575,158 @@ struct kprobe_multi_resolve { size_t cnt; }; -static int -resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type, - const char *sym_name, void *ctx) +struct avail_kallsyms_data { + char **syms; + size_t cnt; + struct kprobe_multi_resolve *res; +}; + +static int avail_func_cmp(const void *a, const void *b) { - struct kprobe_multi_resolve *res = ctx; + return strcmp(*(const char **)a, *(const char **)b); +} + +static int avail_kallsyms_cb(unsigned long long sym_addr, char sym_type, + const char *sym_name, void *ctx) +{ + struct avail_kallsyms_data *data = ctx; + struct kprobe_multi_resolve *res = data->res; int err; - if (!glob_match(sym_name, res->pattern)) + if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp)) return 0; - err = libbpf_ensure_mem((void **) &res->addrs, &res->cap, sizeof(unsigned long), - res->cnt + 1); + err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1); if (err) return err; - res->addrs[res->cnt++] = (unsigned long) sym_addr; + res->addrs[res->cnt++] = (unsigned long)sym_addr; return 0; } +static int libbpf_available_kallsyms_parse(struct kprobe_multi_resolve *res) +{ + const char *available_functions_file = tracefs_available_filter_functions(); + struct avail_kallsyms_data data; + char sym_name[500]; + FILE *f; + int err = 0, ret, i; + char **syms = NULL; + size_t cap = 0, cnt = 0; + + f = fopen(available_functions_file, "re"); + if (!f) { + err = -errno; + pr_warn("failed to open %s: %d\n", available_functions_file, err); + return err; + } + + while (true) { + char *name; + + ret = fscanf(f, "%499s%*[^\n]\n", sym_name); + if (ret == EOF && feof(f)) + break; + + if (ret != 1) { + pr_warn("failed to parse available_filter_functions entry: %d\n", ret); + err = -EINVAL; + goto cleanup; + } + + if (!glob_match(sym_name, res->pattern)) + continue; + + err = libbpf_ensure_mem((void **)&syms, &cap, sizeof(*syms), cnt + 1); + if (err) + goto cleanup; + + name = strdup(sym_name); + if (!name) { + err = -errno; + goto cleanup; + } + + syms[cnt++] = name; + } + + /* no entries found, bail out */ + if (cnt == 0) { + err = -ENOENT; + goto cleanup; + } + + /* sort available functions */ + qsort(syms, cnt, sizeof(*syms), avail_func_cmp); + + data.syms = syms; + data.res = res; + data.cnt = cnt; + libbpf_kallsyms_parse(avail_kallsyms_cb, &data); + + if (res->cnt == 0) + err = -ENOENT; + +cleanup: + for (i = 0; i < cnt; i++) + free((char *)syms[i]); + free(syms); + + fclose(f); + return err; +} + +static bool has_available_filter_functions_addrs(void) +{ + return access(tracefs_available_filter_functions_addrs(), R_OK) != -1; +} + +static int libbpf_available_kprobes_parse(struct kprobe_multi_resolve *res) +{ + const char *available_path = tracefs_available_filter_functions_addrs(); + char sym_name[500]; + FILE *f; + int ret, err = 0; + unsigned long long sym_addr; + + f = fopen(available_path, "re"); + if (!f) { + err = -errno; + pr_warn("failed to open %s: %d\n", available_path, err); + return err; + } + + while (true) { + ret = fscanf(f, "%llx %499s%*[^\n]\n", &sym_addr, sym_name); + if (ret == EOF && feof(f)) + break; + + if (ret != 2) { + pr_warn("failed to parse available_filter_functions_addrs entry: %d\n", + ret); + err = -EINVAL; + goto cleanup; + } + + if (!glob_match(sym_name, res->pattern)) + continue; + + err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, + sizeof(*res->addrs), res->cnt + 1); + if (err) + goto cleanup; + + res->addrs[res->cnt++] = (unsigned long)sym_addr; + } + + if (res->cnt == 0) + err = -ENOENT; + +cleanup: + fclose(f); + return err; +} + struct bpf_link * bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, const char *pattern, @@ -10594,13 +10763,12 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, return libbpf_err_ptr(-EINVAL); if (pattern) { - err = libbpf_kallsyms_parse(resolve_kprobe_multi_cb, &res); + if (has_available_filter_functions_addrs()) + err = libbpf_available_kprobes_parse(&res); + else + err = libbpf_available_kallsyms_parse(&res); if (err) goto error; - if (!res.cnt) { - err = -ENOENT; - goto error; - } addrs = res.addrs; cnt = res.cnt; } @@ -11811,6 +11979,48 @@ static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_l return libbpf_get_error(*link); } +struct bpf_link *bpf_program__attach_netfilter(const struct bpf_program *prog, + const struct bpf_netfilter_opts *opts) +{ + LIBBPF_OPTS(bpf_link_create_opts, lopts); + struct bpf_link *link; + int prog_fd, link_fd; + + if (!OPTS_VALID(opts, bpf_netfilter_opts)) + return libbpf_err_ptr(-EINVAL); + + prog_fd = bpf_program__fd(prog); + if (prog_fd < 0) { + pr_warn("prog '%s': can't attach before loaded\n", prog->name); + return libbpf_err_ptr(-EINVAL); + } + + link = calloc(1, sizeof(*link)); + if (!link) + return libbpf_err_ptr(-ENOMEM); + + link->detach = &bpf_link__detach_fd; + + lopts.netfilter.pf = OPTS_GET(opts, pf, 0); + lopts.netfilter.hooknum = OPTS_GET(opts, hooknum, 0); + lopts.netfilter.priority = OPTS_GET(opts, priority, 0); + lopts.netfilter.flags = OPTS_GET(opts, flags, 0); + + link_fd = bpf_link_create(prog_fd, 0, BPF_NETFILTER, &lopts); + if (link_fd < 0) { + char errmsg[STRERR_BUFSIZE]; + + link_fd = -errno; + free(link); + pr_warn("prog '%s': failed to attach to netfilter: %s\n", + prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); + return libbpf_err_ptr(link_fd); + } + link->fd = link_fd; + + return link; +} + struct bpf_link *bpf_program__attach(const struct bpf_program *prog) { struct bpf_link *link = NULL; diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index 754da73c643b..10642ad69d76 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -718,6 +718,21 @@ LIBBPF_API struct bpf_link * bpf_program__attach_freplace(const struct bpf_program *prog, int target_fd, const char *attach_func_name); +struct bpf_netfilter_opts { + /* size of this struct, for forward/backward compatibility */ + size_t sz; + + __u32 pf; + __u32 hooknum; + __s32 priority; + __u32 flags; +}; +#define bpf_netfilter_opts__last_field flags + +LIBBPF_API struct bpf_link * +bpf_program__attach_netfilter(const struct bpf_program *prog, + const struct bpf_netfilter_opts *opts); + struct bpf_map; LIBBPF_API struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map); diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index 7521a2fb7626..d9ec4407befa 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -395,4 +395,5 @@ LIBBPF_1.2.0 { LIBBPF_1.3.0 { global: bpf_obj_pin_opts; + bpf_program__attach_netfilter; } LIBBPF_1.2.0; diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c index f1a141555f08..37455d00b239 100644 --- a/tools/lib/bpf/usdt.c +++ b/tools/lib/bpf/usdt.c @@ -852,8 +852,11 @@ static int bpf_link_usdt_detach(struct bpf_link *link) * system is so exhausted on memory, it's the least of user's * concerns, probably. * So just do our best here to return those IDs to usdt_manager. + * Another edge case when we can legitimately get NULL is when + * new_cnt is zero, which can happen in some edge cases, so we + * need to be careful about that. */ - if (new_free_ids) { + if (new_free_ids || new_cnt == 0) { memcpy(new_free_ids + man->free_spec_cnt, usdt_link->spec_ids, usdt_link->spec_cnt * sizeof(*usdt_link->spec_ids)); man->free_spec_ids = new_free_ids; diff --git a/tools/testing/selftests/bpf/DENYLIST.aarch64 b/tools/testing/selftests/bpf/DENYLIST.aarch64 index 08adc805878b..3b61e8b35d62 100644 --- a/tools/testing/selftests/bpf/DENYLIST.aarch64 +++ b/tools/testing/selftests/bpf/DENYLIST.aarch64 @@ -10,3 +10,5 @@ kprobe_multi_test/link_api_addrs # link_fd unexpected link_fd: a kprobe_multi_test/link_api_syms # link_fd unexpected link_fd: actual -95 < expected 0 kprobe_multi_test/skel_api # libbpf: failed to load BPF skeleton 'kprobe_multi': -3 module_attach # prog 'kprobe_multi': failed to auto-attach: -95 +fentry_test/fentry_many_args # fentry_many_args:FAIL:fentry_many_args_attach unexpected error: -524 +fexit_test/fexit_many_args # fexit_many_args:FAIL:fexit_many_args_attach unexpected error: -524 diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 538df8fb8c42..882be03b179f 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -12,7 +12,11 @@ BPFDIR := $(LIBDIR)/bpf TOOLSINCDIR := $(TOOLSDIR)/include BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool APIDIR := $(TOOLSINCDIR)/uapi +ifneq ($(O),) +GENDIR := $(O)/include/generated +else GENDIR := $(abspath ../../../../include/generated) +endif GENHDR := $(GENDIR)/autoconf.h HOSTPKG_CONFIG := pkg-config @@ -331,7 +335,7 @@ $(RESOLVE_BTFIDS): $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/resolve_btfids \ OUTPUT=$(HOST_BUILD_DIR)/resolve_btfids/ BPFOBJ=$(HOST_BPFOBJ) # Get Clang's default includes on this system, as opposed to those seen by -# '-target bpf'. This fixes "missing" files on some architectures/distros, +# '--target=bpf'. This fixes "missing" files on some architectures/distros, # such as asm/byteorder.h, asm/socket.h, asm/sockios.h, sys/cdefs.h etc. # # Use '-idirafter': Don't interfere with include mechanics except where the @@ -372,12 +376,12 @@ $(OUTPUT)/cgroup_getset_retval_hooks.o: cgroup_getset_retval_hooks.h # $3 - CFLAGS define CLANG_BPF_BUILD_RULE $(call msg,CLNG-BPF,$(TRUNNER_BINARY),$2) - $(Q)$(CLANG) $3 -O2 -target bpf -c $1 -mcpu=v3 -o $2 + $(Q)$(CLANG) $3 -O2 --target=bpf -c $1 -mcpu=v3 -o $2 endef # Similar to CLANG_BPF_BUILD_RULE, but with disabled alu32 define CLANG_NOALU32_BPF_BUILD_RULE $(call msg,CLNG-BPF,$(TRUNNER_BINARY),$2) - $(Q)$(CLANG) $3 -O2 -target bpf -c $1 -mcpu=v2 -o $2 + $(Q)$(CLANG) $3 -O2 --target=bpf -c $1 -mcpu=v2 -o $2 endef # Build BPF object using GCC define GCC_BPF_BUILD_RULE @@ -644,11 +648,13 @@ $(OUTPUT)/bench_local_storage.o: $(OUTPUT)/local_storage_bench.skel.h $(OUTPUT)/bench_local_storage_rcu_tasks_trace.o: $(OUTPUT)/local_storage_rcu_tasks_trace_bench.skel.h $(OUTPUT)/bench_local_storage_create.o: $(OUTPUT)/bench_local_storage_create.skel.h $(OUTPUT)/bench_bpf_hashmap_lookup.o: $(OUTPUT)/bpf_hashmap_lookup.skel.h +$(OUTPUT)/bench_htab_mem.o: $(OUTPUT)/htab_mem_bench.skel.h $(OUTPUT)/bench.o: bench.h testing_helpers.h $(BPFOBJ) $(OUTPUT)/bench: LDLIBS += -lm $(OUTPUT)/bench: $(OUTPUT)/bench.o \ $(TESTING_HELPERS) \ $(TRACE_HELPERS) \ + $(CGROUP_HELPERS) \ $(OUTPUT)/bench_count.o \ $(OUTPUT)/bench_rename.o \ $(OUTPUT)/bench_trigger.o \ @@ -661,6 +667,7 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \ $(OUTPUT)/bench_local_storage_rcu_tasks_trace.o \ $(OUTPUT)/bench_bpf_hashmap_lookup.o \ $(OUTPUT)/bench_local_storage_create.o \ + $(OUTPUT)/bench_htab_mem.o \ # $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@ diff --git a/tools/testing/selftests/bpf/bench.c b/tools/testing/selftests/bpf/bench.c index 41fe5a82b88b..73ce11b0547d 100644 --- a/tools/testing/selftests/bpf/bench.c +++ b/tools/testing/selftests/bpf/bench.c @@ -279,6 +279,7 @@ extern struct argp bench_local_storage_rcu_tasks_trace_argp; extern struct argp bench_strncmp_argp; extern struct argp bench_hashmap_lookup_argp; extern struct argp bench_local_storage_create_argp; +extern struct argp bench_htab_mem_argp; static const struct argp_child bench_parsers[] = { { &bench_ringbufs_argp, 0, "Ring buffers benchmark", 0 }, @@ -290,6 +291,7 @@ static const struct argp_child bench_parsers[] = { "local_storage RCU Tasks Trace slowdown benchmark", 0 }, { &bench_hashmap_lookup_argp, 0, "Hashmap lookup benchmark", 0 }, { &bench_local_storage_create_argp, 0, "local-storage-create benchmark", 0 }, + { &bench_htab_mem_argp, 0, "hash map memory benchmark", 0 }, {}, }; @@ -520,6 +522,7 @@ extern const struct bench bench_local_storage_cache_hashmap_control; extern const struct bench bench_local_storage_tasks_trace; extern const struct bench bench_bpf_hashmap_lookup; extern const struct bench bench_local_storage_create; +extern const struct bench bench_htab_mem; static const struct bench *benchs[] = { &bench_count_global, @@ -561,6 +564,7 @@ static const struct bench *benchs[] = { &bench_local_storage_tasks_trace, &bench_bpf_hashmap_lookup, &bench_local_storage_create, + &bench_htab_mem, }; static void find_benchmark(void) diff --git a/tools/testing/selftests/bpf/benchs/bench_htab_mem.c b/tools/testing/selftests/bpf/benchs/bench_htab_mem.c new file mode 100644 index 000000000000..9146d3f414d2 --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/bench_htab_mem.c @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2023. Huawei Technologies Co., Ltd */ +#include <argp.h> +#include <stdbool.h> +#include <pthread.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/param.h> +#include <fcntl.h> + +#include "bench.h" +#include "bpf_util.h" +#include "cgroup_helpers.h" +#include "htab_mem_bench.skel.h" + +struct htab_mem_use_case { + const char *name; + const char **progs; + /* Do synchronization between addition thread and deletion thread */ + bool need_sync; +}; + +static struct htab_mem_ctx { + const struct htab_mem_use_case *uc; + struct htab_mem_bench *skel; + pthread_barrier_t *notify; + int fd; +} ctx; + +const char *ow_progs[] = {"overwrite", NULL}; +const char *batch_progs[] = {"batch_add_batch_del", NULL}; +const char *add_del_progs[] = {"add_only", "del_only", NULL}; +const static struct htab_mem_use_case use_cases[] = { + { .name = "overwrite", .progs = ow_progs }, + { .name = "batch_add_batch_del", .progs = batch_progs }, + { .name = "add_del_on_diff_cpu", .progs = add_del_progs, .need_sync = true }, +}; + +static struct htab_mem_args { + u32 value_size; + const char *use_case; + bool preallocated; +} args = { + .value_size = 8, + .use_case = "overwrite", + .preallocated = false, +}; + +enum { + ARG_VALUE_SIZE = 10000, + ARG_USE_CASE = 10001, + ARG_PREALLOCATED = 10002, +}; + +static const struct argp_option opts[] = { + { "value-size", ARG_VALUE_SIZE, "VALUE_SIZE", 0, + "Set the value size of hash map (default 8)" }, + { "use-case", ARG_USE_CASE, "USE_CASE", 0, + "Set the use case of hash map: overwrite|batch_add_batch_del|add_del_on_diff_cpu" }, + { "preallocated", ARG_PREALLOCATED, NULL, 0, "use preallocated hash map" }, + {}, +}; + +static error_t htab_mem_parse_arg(int key, char *arg, struct argp_state *state) +{ + switch (key) { + case ARG_VALUE_SIZE: + args.value_size = strtoul(arg, NULL, 10); + if (args.value_size > 4096) { + fprintf(stderr, "too big value size %u\n", args.value_size); + argp_usage(state); + } + break; + case ARG_USE_CASE: + args.use_case = strdup(arg); + if (!args.use_case) { + fprintf(stderr, "no mem for use-case\n"); + argp_usage(state); + } + break; + case ARG_PREALLOCATED: + args.preallocated = true; + break; + default: + return ARGP_ERR_UNKNOWN; + } + + return 0; +} + +const struct argp bench_htab_mem_argp = { + .options = opts, + .parser = htab_mem_parse_arg, +}; + +static void htab_mem_validate(void) +{ + if (!strcmp(use_cases[2].name, args.use_case) && env.producer_cnt % 2) { + fprintf(stderr, "%s needs an even number of producers\n", args.use_case); + exit(1); + } +} + +static int htab_mem_bench_init_barriers(void) +{ + pthread_barrier_t *barriers; + unsigned int i, nr; + + if (!ctx.uc->need_sync) + return 0; + + nr = (env.producer_cnt + 1) / 2; + barriers = calloc(nr, sizeof(*barriers)); + if (!barriers) + return -1; + + /* Used for synchronization between two threads */ + for (i = 0; i < nr; i++) + pthread_barrier_init(&barriers[i], NULL, 2); + + ctx.notify = barriers; + return 0; +} + +static void htab_mem_bench_exit_barriers(void) +{ + unsigned int i, nr; + + if (!ctx.notify) + return; + + nr = (env.producer_cnt + 1) / 2; + for (i = 0; i < nr; i++) + pthread_barrier_destroy(&ctx.notify[i]); + free(ctx.notify); +} + +static const struct htab_mem_use_case *htab_mem_find_use_case_or_exit(const char *name) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(use_cases); i++) { + if (!strcmp(name, use_cases[i].name)) + return &use_cases[i]; + } + + fprintf(stderr, "no such use-case: %s\n", name); + fprintf(stderr, "available use case:"); + for (i = 0; i < ARRAY_SIZE(use_cases); i++) + fprintf(stderr, " %s", use_cases[i].name); + fprintf(stderr, "\n"); + exit(1); +} + +static void htab_mem_setup(void) +{ + struct bpf_map *map; + const char **names; + int err; + + setup_libbpf(); + + ctx.uc = htab_mem_find_use_case_or_exit(args.use_case); + err = htab_mem_bench_init_barriers(); + if (err) { + fprintf(stderr, "failed to init barrier\n"); + exit(1); + } + + ctx.fd = cgroup_setup_and_join("/htab_mem"); + if (ctx.fd < 0) + goto cleanup; + + ctx.skel = htab_mem_bench__open(); + if (!ctx.skel) { + fprintf(stderr, "failed to open skeleton\n"); + goto cleanup; + } + + map = ctx.skel->maps.htab; + bpf_map__set_value_size(map, args.value_size); + /* Ensure that different CPUs can operate on different subset */ + bpf_map__set_max_entries(map, MAX(8192, 64 * env.nr_cpus)); + if (args.preallocated) + bpf_map__set_map_flags(map, bpf_map__map_flags(map) & ~BPF_F_NO_PREALLOC); + + names = ctx.uc->progs; + while (*names) { + struct bpf_program *prog; + + prog = bpf_object__find_program_by_name(ctx.skel->obj, *names); + if (!prog) { + fprintf(stderr, "no such program %s\n", *names); + goto cleanup; + } + bpf_program__set_autoload(prog, true); + names++; + } + ctx.skel->bss->nr_thread = env.producer_cnt; + + err = htab_mem_bench__load(ctx.skel); + if (err) { + fprintf(stderr, "failed to load skeleton\n"); + goto cleanup; + } + err = htab_mem_bench__attach(ctx.skel); + if (err) { + fprintf(stderr, "failed to attach skeleton\n"); + goto cleanup; + } + return; + +cleanup: + htab_mem_bench__destroy(ctx.skel); + htab_mem_bench_exit_barriers(); + if (ctx.fd >= 0) { + close(ctx.fd); + cleanup_cgroup_environment(); + } + exit(1); +} + +static void htab_mem_add_fn(pthread_barrier_t *notify) +{ + while (true) { + /* Do addition */ + (void)syscall(__NR_getpgid, 0); + /* Notify deletion thread to do deletion */ + pthread_barrier_wait(notify); + /* Wait for deletion to complete */ + pthread_barrier_wait(notify); + } +} + +static void htab_mem_delete_fn(pthread_barrier_t *notify) +{ + while (true) { + /* Wait for addition to complete */ + pthread_barrier_wait(notify); + /* Do deletion */ + (void)syscall(__NR_getppid); + /* Notify addition thread to do addition */ + pthread_barrier_wait(notify); + } +} + +static void *htab_mem_producer(void *arg) +{ + pthread_barrier_t *notify; + int seq; + + if (!ctx.uc->need_sync) { + while (true) + (void)syscall(__NR_getpgid, 0); + return NULL; + } + + seq = (long)arg; + notify = &ctx.notify[seq / 2]; + if (seq & 1) + htab_mem_delete_fn(notify); + else + htab_mem_add_fn(notify); + return NULL; +} + +static void htab_mem_read_mem_cgrp_file(const char *name, unsigned long *value) +{ + char buf[32]; + ssize_t got; + int fd; + + fd = openat(ctx.fd, name, O_RDONLY); + if (fd < 0) { + /* cgroup v1 ? */ + fprintf(stderr, "no %s\n", name); + *value = 0; + return; + } + + got = read(fd, buf, sizeof(buf) - 1); + if (got <= 0) { + *value = 0; + return; + } + buf[got] = 0; + + *value = strtoull(buf, NULL, 0); + + close(fd); +} + +static void htab_mem_measure(struct bench_res *res) +{ + res->hits = atomic_swap(&ctx.skel->bss->op_cnt, 0) / env.producer_cnt; + htab_mem_read_mem_cgrp_file("memory.current", &res->gp_ct); +} + +static void htab_mem_report_progress(int iter, struct bench_res *res, long delta_ns) +{ + double loop, mem; + + loop = res->hits / 1000.0 / (delta_ns / 1000000000.0); + mem = res->gp_ct / 1048576.0; + printf("Iter %3d (%7.3lfus): ", iter, (delta_ns - 1000000000) / 1000.0); + printf("per-prod-op %7.2lfk/s, memory usage %7.2lfMiB\n", loop, mem); +} + +static void htab_mem_report_final(struct bench_res res[], int res_cnt) +{ + double mem_mean = 0.0, mem_stddev = 0.0; + double loop_mean = 0.0, loop_stddev = 0.0; + unsigned long peak_mem; + int i; + + for (i = 0; i < res_cnt; i++) { + loop_mean += res[i].hits / 1000.0 / (0.0 + res_cnt); + mem_mean += res[i].gp_ct / 1048576.0 / (0.0 + res_cnt); + } + if (res_cnt > 1) { + for (i = 0; i < res_cnt; i++) { + loop_stddev += (loop_mean - res[i].hits / 1000.0) * + (loop_mean - res[i].hits / 1000.0) / + (res_cnt - 1.0); + mem_stddev += (mem_mean - res[i].gp_ct / 1048576.0) * + (mem_mean - res[i].gp_ct / 1048576.0) / + (res_cnt - 1.0); + } + loop_stddev = sqrt(loop_stddev); + mem_stddev = sqrt(mem_stddev); + } + + htab_mem_read_mem_cgrp_file("memory.peak", &peak_mem); + printf("Summary: per-prod-op %7.2lf \u00B1 %7.2lfk/s, memory usage %7.2lf \u00B1 %7.2lfMiB," + " peak memory usage %7.2lfMiB\n", + loop_mean, loop_stddev, mem_mean, mem_stddev, peak_mem / 1048576.0); + + cleanup_cgroup_environment(); +} + +const struct bench bench_htab_mem = { + .name = "htab-mem", + .argp = &bench_htab_mem_argp, + .validate = htab_mem_validate, + .setup = htab_mem_setup, + .producer_thread = htab_mem_producer, + .measure = htab_mem_measure, + .report_progress = htab_mem_report_progress, + .report_final = htab_mem_report_final, +}; diff --git a/tools/testing/selftests/bpf/benchs/bench_ringbufs.c b/tools/testing/selftests/bpf/benchs/bench_ringbufs.c index 3ca14ad36607..e1ee979e6acc 100644 --- a/tools/testing/selftests/bpf/benchs/bench_ringbufs.c +++ b/tools/testing/selftests/bpf/benchs/bench_ringbufs.c @@ -399,7 +399,7 @@ static void perfbuf_libbpf_setup(void) ctx->skel = perfbuf_setup_skeleton(); memset(&attr, 0, sizeof(attr)); - attr.config = PERF_COUNT_SW_BPF_OUTPUT, + attr.config = PERF_COUNT_SW_BPF_OUTPUT; attr.type = PERF_TYPE_SOFTWARE; attr.sample_type = PERF_SAMPLE_RAW; /* notify only every Nth sample */ diff --git a/tools/testing/selftests/bpf/benchs/run_bench_htab_mem.sh b/tools/testing/selftests/bpf/benchs/run_bench_htab_mem.sh new file mode 100755 index 000000000000..9ff5832463a2 --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/run_bench_htab_mem.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +source ./benchs/run_common.sh + +set -eufo pipefail + +htab_mem() +{ + echo -n "per-prod-op: " + echo -n "$*" | sed -E "s/.* per-prod-op\s+([0-9]+\.[0-9]+ ± [0-9]+\.[0-9]+k\/s).*/\1/" + echo -n -e ", avg mem: " + echo -n "$*" | sed -E "s/.* memory usage\s+([0-9]+\.[0-9]+ ± [0-9]+\.[0-9]+MiB).*/\1/" + echo -n ", peak mem: " + echo "$*" | sed -E "s/.* peak memory usage\s+([0-9]+\.[0-9]+MiB).*/\1/" +} + +summarize_htab_mem() +{ + local bench="$1" + local summary=$(echo $2 | tail -n1) + + printf "%-20s %s\n" "$bench" "$(htab_mem $summary)" +} + +htab_mem_bench() +{ + local name + + for name in overwrite batch_add_batch_del add_del_on_diff_cpu + do + summarize_htab_mem "$name" "$($RUN_BENCH htab-mem --use-case $name -p8 "$@")" + done +} + +header "preallocated" +htab_mem_bench "--preallocated" + +header "normal bpf ma" +htab_mem_bench diff --git a/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c b/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c index aaf6ef1201c7..a6f991b56345 100644 --- a/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c +++ b/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c @@ -34,6 +34,11 @@ struct bpf_testmod_struct_arg_3 { int b[]; }; +struct bpf_testmod_struct_arg_4 { + u64 a; + int b; +}; + __diag_push(); __diag_ignore_all("-Wmissing-prototypes", "Global functions as their definitions will be in bpf_testmod.ko BTF"); @@ -75,6 +80,24 @@ bpf_testmod_test_struct_arg_6(struct bpf_testmod_struct_arg_3 *a) { return bpf_testmod_test_struct_arg_result; } +noinline int +bpf_testmod_test_struct_arg_7(u64 a, void *b, short c, int d, void *e, + struct bpf_testmod_struct_arg_4 f) +{ + bpf_testmod_test_struct_arg_result = a + (long)b + c + d + + (long)e + f.a + f.b; + return bpf_testmod_test_struct_arg_result; +} + +noinline int +bpf_testmod_test_struct_arg_8(u64 a, void *b, short c, int d, void *e, + struct bpf_testmod_struct_arg_4 f, int g) +{ + bpf_testmod_test_struct_arg_result = a + (long)b + c + d + + (long)e + f.a + f.b + g; + return bpf_testmod_test_struct_arg_result; +} + __bpf_kfunc void bpf_testmod_test_mod_kfunc(int i) { @@ -191,6 +214,20 @@ noinline int bpf_testmod_fentry_test3(char a, int b, u64 c) return a + b + c; } +noinline int bpf_testmod_fentry_test7(u64 a, void *b, short c, int d, + void *e, char f, int g) +{ + return a + (long)b + c + d + (long)e + f + g; +} + +noinline int bpf_testmod_fentry_test11(u64 a, void *b, short c, int d, + void *e, char f, int g, + unsigned int h, long i, __u64 j, + unsigned long k) +{ + return a + (long)b + c + d + (long)e + f + g + h + i + j + k; +} + int bpf_testmod_fentry_ok; noinline ssize_t @@ -206,6 +243,7 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, struct bpf_testmod_struct_arg_1 struct_arg1 = {10}; struct bpf_testmod_struct_arg_2 struct_arg2 = {2, 3}; struct bpf_testmod_struct_arg_3 *struct_arg3; + struct bpf_testmod_struct_arg_4 struct_arg4 = {21, 22}; int i = 1; while (bpf_testmod_return_ptr(i)) @@ -216,6 +254,11 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, (void)bpf_testmod_test_struct_arg_3(1, 4, struct_arg2); (void)bpf_testmod_test_struct_arg_4(struct_arg1, 1, 2, 3, struct_arg2); (void)bpf_testmod_test_struct_arg_5(); + (void)bpf_testmod_test_struct_arg_7(16, (void *)17, 18, 19, + (void *)20, struct_arg4); + (void)bpf_testmod_test_struct_arg_8(16, (void *)17, 18, 19, + (void *)20, struct_arg4, 23); + struct_arg3 = kmalloc((sizeof(struct bpf_testmod_struct_arg_3) + sizeof(int)), GFP_KERNEL); @@ -243,7 +286,11 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, if (bpf_testmod_fentry_test1(1) != 2 || bpf_testmod_fentry_test2(2, 3) != 5 || - bpf_testmod_fentry_test3(4, 5, 6) != 15) + bpf_testmod_fentry_test3(4, 5, 6) != 15 || + bpf_testmod_fentry_test7(16, (void *)17, 18, 19, (void *)20, + 21, 22) != 133 || + bpf_testmod_fentry_test11(16, (void *)17, 18, 19, (void *)20, + 21, 22, 23, 24, 25, 26) != 231) goto out; bpf_testmod_fentry_ok = 1; diff --git a/tools/testing/selftests/bpf/cgroup_helpers.c b/tools/testing/selftests/bpf/cgroup_helpers.c index 9e95b37a7dff..2caee8423ee0 100644 --- a/tools/testing/selftests/bpf/cgroup_helpers.c +++ b/tools/testing/selftests/bpf/cgroup_helpers.c @@ -278,6 +278,18 @@ int join_cgroup(const char *relative_path) } /** + * join_root_cgroup() - Join the root cgroup + * + * This function joins the root cgroup. + * + * On success, it returns 0, otherwise on failure it returns 1. + */ +int join_root_cgroup(void) +{ + return join_cgroup_from_top(CGROUP_MOUNT_PATH); +} + +/** * join_parent_cgroup() - Join a cgroup in the parent process workdir * @relative_path: The cgroup path, relative to parent process workdir, to join * diff --git a/tools/testing/selftests/bpf/cgroup_helpers.h b/tools/testing/selftests/bpf/cgroup_helpers.h index f099a166c94d..5c2cb9c8b546 100644 --- a/tools/testing/selftests/bpf/cgroup_helpers.h +++ b/tools/testing/selftests/bpf/cgroup_helpers.h @@ -22,6 +22,7 @@ void remove_cgroup(const char *relative_path); unsigned long long get_cgroup_id(const char *relative_path); int join_cgroup(const char *relative_path); +int join_root_cgroup(void); int join_parent_cgroup(const char *relative_path); int setup_cgroup_environment(void); diff --git a/tools/testing/selftests/bpf/cgroup_tcp_skb.h b/tools/testing/selftests/bpf/cgroup_tcp_skb.h new file mode 100644 index 000000000000..7f6b24f102fb --- /dev/null +++ b/tools/testing/selftests/bpf/cgroup_tcp_skb.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ + +/* Define states of a socket to tracking messages sending to and from the + * socket. + * + * These states are based on rfc9293 with some modifications to support + * tracking of messages sent out from a socket. For example, when a SYN is + * received, a new socket is transiting to the SYN_RECV state defined in + * rfc9293. But, we put it in SYN_RECV_SENDING_SYN_ACK state and when + * SYN-ACK is sent out, it moves to SYN_RECV state. With this modification, + * we can track the message sent out from a socket. + */ + +#ifndef __CGROUP_TCP_SKB_H__ +#define __CGROUP_TCP_SKB_H__ + +enum { + INIT, + CLOSED, + SYN_SENT, + SYN_RECV_SENDING_SYN_ACK, + SYN_RECV, + ESTABLISHED, + FIN_WAIT1, + FIN_WAIT2, + CLOSE_WAIT_SENDING_ACK, + CLOSE_WAIT, + CLOSING, + LAST_ACK, + TIME_WAIT_SENDING_ACK, + TIME_WAIT, +}; + +#endif /* __CGROUP_TCP_SKB_H__ */ diff --git a/tools/testing/selftests/bpf/gnu/stubs.h b/tools/testing/selftests/bpf/gnu/stubs.h index 719225b16626..1c638d9dce1a 100644 --- a/tools/testing/selftests/bpf/gnu/stubs.h +++ b/tools/testing/selftests/bpf/gnu/stubs.h @@ -1 +1 @@ -/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */ +/* dummy .h to trick /usr/include/features.h to work with 'clang --target=bpf' */ diff --git a/tools/testing/selftests/bpf/map_tests/map_percpu_stats.c b/tools/testing/selftests/bpf/map_tests/map_percpu_stats.c new file mode 100644 index 000000000000..1a9eeefda9a8 --- /dev/null +++ b/tools/testing/selftests/bpf/map_tests/map_percpu_stats.c @@ -0,0 +1,447 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Isovalent */ + +#include <errno.h> +#include <unistd.h> +#include <pthread.h> + +#include <bpf/bpf.h> +#include <bpf/libbpf.h> + +#include <bpf_util.h> +#include <test_maps.h> + +#include "map_percpu_stats.skel.h" + +#define MAX_ENTRIES 16384 +#define MAX_ENTRIES_HASH_OF_MAPS 64 +#define N_THREADS 8 +#define MAX_MAP_KEY_SIZE 4 + +static void map_info(int map_fd, struct bpf_map_info *info) +{ + __u32 len = sizeof(*info); + int ret; + + memset(info, 0, sizeof(*info)); + + ret = bpf_obj_get_info_by_fd(map_fd, info, &len); + CHECK(ret < 0, "bpf_obj_get_info_by_fd", "error: %s\n", strerror(errno)); +} + +static const char *map_type_to_s(__u32 type) +{ + switch (type) { + case BPF_MAP_TYPE_HASH: + return "HASH"; + case BPF_MAP_TYPE_PERCPU_HASH: + return "PERCPU_HASH"; + case BPF_MAP_TYPE_LRU_HASH: + return "LRU_HASH"; + case BPF_MAP_TYPE_LRU_PERCPU_HASH: + return "LRU_PERCPU_HASH"; + case BPF_MAP_TYPE_HASH_OF_MAPS: + return "BPF_MAP_TYPE_HASH_OF_MAPS"; + default: + return "<define-me>"; + } +} + +static __u32 map_count_elements(__u32 type, int map_fd) +{ + __u32 key = -1; + int n = 0; + + while (!bpf_map_get_next_key(map_fd, &key, &key)) + n++; + return n; +} + +#define BATCH true + +static void delete_and_lookup_batch(int map_fd, void *keys, __u32 count) +{ + static __u8 values[(8 << 10) * MAX_ENTRIES]; + void *in_batch = NULL, *out_batch; + __u32 save_count = count; + int ret; + + ret = bpf_map_lookup_and_delete_batch(map_fd, + &in_batch, &out_batch, + keys, values, &count, + NULL); + + /* + * Despite what uapi header says, lookup_and_delete_batch will return + * -ENOENT in case we successfully have deleted all elements, so check + * this separately + */ + CHECK(ret < 0 && (errno != ENOENT || !count), "bpf_map_lookup_and_delete_batch", + "error: %s\n", strerror(errno)); + + CHECK(count != save_count, + "bpf_map_lookup_and_delete_batch", + "deleted not all elements: removed=%u expected=%u\n", + count, save_count); +} + +static void delete_all_elements(__u32 type, int map_fd, bool batch) +{ + static __u8 val[8 << 10]; /* enough for 1024 CPUs */ + __u32 key = -1; + void *keys; + __u32 i, n; + int ret; + + keys = calloc(MAX_MAP_KEY_SIZE, MAX_ENTRIES); + CHECK(!keys, "calloc", "error: %s\n", strerror(errno)); + + for (n = 0; !bpf_map_get_next_key(map_fd, &key, &key); n++) + memcpy(keys + n*MAX_MAP_KEY_SIZE, &key, MAX_MAP_KEY_SIZE); + + if (batch) { + /* Can't mix delete_batch and delete_and_lookup_batch because + * they have different semantics in relation to the keys + * argument. However, delete_batch utilize map_delete_elem, + * so we actually test it in non-batch scenario */ + delete_and_lookup_batch(map_fd, keys, n); + } else { + /* Intentionally mix delete and lookup_and_delete so we can test both */ + for (i = 0; i < n; i++) { + void *keyp = keys + i*MAX_MAP_KEY_SIZE; + + if (i % 2 || type == BPF_MAP_TYPE_HASH_OF_MAPS) { + ret = bpf_map_delete_elem(map_fd, keyp); + CHECK(ret < 0, "bpf_map_delete_elem", + "error: key %u: %s\n", i, strerror(errno)); + } else { + ret = bpf_map_lookup_and_delete_elem(map_fd, keyp, val); + CHECK(ret < 0, "bpf_map_lookup_and_delete_elem", + "error: key %u: %s\n", i, strerror(errno)); + } + } + } + + free(keys); +} + +static bool is_lru(__u32 map_type) +{ + return map_type == BPF_MAP_TYPE_LRU_HASH || + map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; +} + +struct upsert_opts { + __u32 map_type; + int map_fd; + __u32 n; +}; + +static int create_small_hash(void) +{ + int map_fd; + + map_fd = bpf_map_create(BPF_MAP_TYPE_HASH, "small", 4, 4, 4, NULL); + CHECK(map_fd < 0, "bpf_map_create()", "error:%s (name=%s)\n", + strerror(errno), "small"); + + return map_fd; +} + +static void *patch_map_thread(void *arg) +{ + struct upsert_opts *opts = arg; + int val; + int ret; + int i; + + for (i = 0; i < opts->n; i++) { + if (opts->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) + val = create_small_hash(); + else + val = rand(); + ret = bpf_map_update_elem(opts->map_fd, &i, &val, 0); + CHECK(ret < 0, "bpf_map_update_elem", "key=%d error: %s\n", i, strerror(errno)); + + if (opts->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) + close(val); + } + return NULL; +} + +static void upsert_elements(struct upsert_opts *opts) +{ + pthread_t threads[N_THREADS]; + int ret; + int i; + + for (i = 0; i < ARRAY_SIZE(threads); i++) { + ret = pthread_create(&i[threads], NULL, patch_map_thread, opts); + CHECK(ret != 0, "pthread_create", "error: %s\n", strerror(ret)); + } + + for (i = 0; i < ARRAY_SIZE(threads); i++) { + ret = pthread_join(i[threads], NULL); + CHECK(ret != 0, "pthread_join", "error: %s\n", strerror(ret)); + } +} + +static __u32 read_cur_elements(int iter_fd) +{ + char buf[64]; + ssize_t n; + __u32 ret; + + n = read(iter_fd, buf, sizeof(buf)-1); + CHECK(n <= 0, "read", "error: %s\n", strerror(errno)); + buf[n] = '\0'; + + errno = 0; + ret = (__u32)strtol(buf, NULL, 10); + CHECK(errno != 0, "strtol", "error: %s\n", strerror(errno)); + + return ret; +} + +static __u32 get_cur_elements(int map_id) +{ + struct map_percpu_stats *skel; + struct bpf_link *link; + __u32 n_elements; + int iter_fd; + int ret; + + skel = map_percpu_stats__open(); + CHECK(skel == NULL, "map_percpu_stats__open", "error: %s", strerror(errno)); + + skel->bss->target_id = map_id; + + ret = map_percpu_stats__load(skel); + CHECK(ret != 0, "map_percpu_stats__load", "error: %s", strerror(errno)); + + link = bpf_program__attach_iter(skel->progs.dump_bpf_map, NULL); + CHECK(!link, "bpf_program__attach_iter", "error: %s\n", strerror(errno)); + + iter_fd = bpf_iter_create(bpf_link__fd(link)); + CHECK(iter_fd < 0, "bpf_iter_create", "error: %s\n", strerror(errno)); + + n_elements = read_cur_elements(iter_fd); + + close(iter_fd); + bpf_link__destroy(link); + map_percpu_stats__destroy(skel); + + return n_elements; +} + +static void check_expected_number_elements(__u32 n_inserted, int map_fd, + struct bpf_map_info *info) +{ + __u32 n_real; + __u32 n_iter; + + /* Count the current number of elements in the map by iterating through + * all the map keys via bpf_get_next_key + */ + n_real = map_count_elements(info->type, map_fd); + + /* The "real" number of elements should be the same as the inserted + * number of elements in all cases except LRU maps, where some elements + * may have been evicted + */ + if (n_inserted == 0 || !is_lru(info->type)) + CHECK(n_inserted != n_real, "map_count_elements", + "n_real(%u) != n_inserted(%u)\n", n_real, n_inserted); + + /* Count the current number of elements in the map using an iterator */ + n_iter = get_cur_elements(info->id); + + /* Both counts should be the same, as all updates are over */ + CHECK(n_iter != n_real, "get_cur_elements", + "n_iter=%u, expected %u (map_type=%s,map_flags=%08x)\n", + n_iter, n_real, map_type_to_s(info->type), info->map_flags); +} + +static void __test(int map_fd) +{ + struct upsert_opts opts = { + .map_fd = map_fd, + }; + struct bpf_map_info info; + + map_info(map_fd, &info); + opts.map_type = info.type; + opts.n = info.max_entries; + + /* Reduce the number of elements we are updating such that we don't + * bump into -E2BIG from non-preallocated hash maps, but still will + * have some evictions for LRU maps */ + if (opts.map_type != BPF_MAP_TYPE_HASH_OF_MAPS) + opts.n -= 512; + else + opts.n /= 2; + + /* + * Upsert keys [0, n) under some competition: with random values from + * N_THREADS threads. Check values, then delete all elements and check + * values again. + */ + upsert_elements(&opts); + check_expected_number_elements(opts.n, map_fd, &info); + delete_all_elements(info.type, map_fd, !BATCH); + check_expected_number_elements(0, map_fd, &info); + + /* Now do the same, but using batch delete operations */ + upsert_elements(&opts); + check_expected_number_elements(opts.n, map_fd, &info); + delete_all_elements(info.type, map_fd, BATCH); + check_expected_number_elements(0, map_fd, &info); + + close(map_fd); +} + +static int map_create_opts(__u32 type, const char *name, + struct bpf_map_create_opts *map_opts, + __u32 key_size, __u32 val_size) +{ + int max_entries; + int map_fd; + + if (type == BPF_MAP_TYPE_HASH_OF_MAPS) + max_entries = MAX_ENTRIES_HASH_OF_MAPS; + else + max_entries = MAX_ENTRIES; + + map_fd = bpf_map_create(type, name, key_size, val_size, max_entries, map_opts); + CHECK(map_fd < 0, "bpf_map_create()", "error:%s (name=%s)\n", + strerror(errno), name); + + return map_fd; +} + +static int map_create(__u32 type, const char *name, struct bpf_map_create_opts *map_opts) +{ + return map_create_opts(type, name, map_opts, sizeof(int), sizeof(int)); +} + +static int create_hash(void) +{ + struct bpf_map_create_opts map_opts = { + .sz = sizeof(map_opts), + .map_flags = BPF_F_NO_PREALLOC, + }; + + return map_create(BPF_MAP_TYPE_HASH, "hash", &map_opts); +} + +static int create_percpu_hash(void) +{ + struct bpf_map_create_opts map_opts = { + .sz = sizeof(map_opts), + .map_flags = BPF_F_NO_PREALLOC, + }; + + return map_create(BPF_MAP_TYPE_PERCPU_HASH, "percpu_hash", &map_opts); +} + +static int create_hash_prealloc(void) +{ + return map_create(BPF_MAP_TYPE_HASH, "hash", NULL); +} + +static int create_percpu_hash_prealloc(void) +{ + return map_create(BPF_MAP_TYPE_PERCPU_HASH, "percpu_hash_prealloc", NULL); +} + +static int create_lru_hash(__u32 type, __u32 map_flags) +{ + struct bpf_map_create_opts map_opts = { + .sz = sizeof(map_opts), + .map_flags = map_flags, + }; + + return map_create(type, "lru_hash", &map_opts); +} + +static int create_hash_of_maps(void) +{ + struct bpf_map_create_opts map_opts = { + .sz = sizeof(map_opts), + .map_flags = BPF_F_NO_PREALLOC, + .inner_map_fd = create_small_hash(), + }; + int ret; + + ret = map_create_opts(BPF_MAP_TYPE_HASH_OF_MAPS, "hash_of_maps", + &map_opts, sizeof(int), sizeof(int)); + close(map_opts.inner_map_fd); + return ret; +} + +static void map_percpu_stats_hash(void) +{ + __test(create_hash()); + printf("test_%s:PASS\n", __func__); +} + +static void map_percpu_stats_percpu_hash(void) +{ + __test(create_percpu_hash()); + printf("test_%s:PASS\n", __func__); +} + +static void map_percpu_stats_hash_prealloc(void) +{ + __test(create_hash_prealloc()); + printf("test_%s:PASS\n", __func__); +} + +static void map_percpu_stats_percpu_hash_prealloc(void) +{ + __test(create_percpu_hash_prealloc()); + printf("test_%s:PASS\n", __func__); +} + +static void map_percpu_stats_lru_hash(void) +{ + __test(create_lru_hash(BPF_MAP_TYPE_LRU_HASH, 0)); + printf("test_%s:PASS\n", __func__); +} + +static void map_percpu_stats_lru_hash_no_common(void) +{ + __test(create_lru_hash(BPF_MAP_TYPE_LRU_HASH, BPF_F_NO_COMMON_LRU)); + printf("test_%s:PASS\n", __func__); +} + +static void map_percpu_stats_percpu_lru_hash(void) +{ + __test(create_lru_hash(BPF_MAP_TYPE_LRU_PERCPU_HASH, 0)); + printf("test_%s:PASS\n", __func__); +} + +static void map_percpu_stats_percpu_lru_hash_no_common(void) +{ + __test(create_lru_hash(BPF_MAP_TYPE_LRU_PERCPU_HASH, BPF_F_NO_COMMON_LRU)); + printf("test_%s:PASS\n", __func__); +} + +static void map_percpu_stats_hash_of_maps(void) +{ + __test(create_hash_of_maps()); + printf("test_%s:PASS\n", __func__); +} + +void test_map_percpu_stats(void) +{ + map_percpu_stats_hash(); + map_percpu_stats_percpu_hash(); + map_percpu_stats_hash_prealloc(); + map_percpu_stats_percpu_hash_prealloc(); + map_percpu_stats_lru_hash(); + map_percpu_stats_lru_hash_no_common(); + map_percpu_stats_percpu_lru_hash(); + map_percpu_stats_percpu_lru_hash_no_common(); + map_percpu_stats_hash_of_maps(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c index c8ba4009e4ab..b30ff6b3b81a 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c @@ -123,12 +123,13 @@ static void test_bpf_nf_ct(int mode) ASSERT_EQ(skel->data->test_snat_addr, 0, "Test for source natting"); ASSERT_EQ(skel->data->test_dnat_addr, 0, "Test for destination natting"); end: - if (srv_client_fd != -1) - close(srv_client_fd); if (client_fd != -1) close(client_fd); + if (srv_client_fd != -1) + close(srv_client_fd); if (srv_fd != -1) close(srv_fd); + snprintf(cmd, sizeof(cmd), iptables, "-D"); system(cmd); test_bpf_nf__destroy(skel); diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_tcp_skb.c b/tools/testing/selftests/bpf/prog_tests/cgroup_tcp_skb.c new file mode 100644 index 000000000000..95bab61a1e57 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_tcp_skb.c @@ -0,0 +1,402 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Facebook */ +#include <test_progs.h> +#include <linux/in6.h> +#include <sys/socket.h> +#include <sched.h> +#include <unistd.h> +#include "cgroup_helpers.h" +#include "testing_helpers.h" +#include "cgroup_tcp_skb.skel.h" +#include "cgroup_tcp_skb.h" + +#define CGROUP_TCP_SKB_PATH "/test_cgroup_tcp_skb" + +static int install_filters(int cgroup_fd, + struct bpf_link **egress_link, + struct bpf_link **ingress_link, + struct bpf_program *egress_prog, + struct bpf_program *ingress_prog, + struct cgroup_tcp_skb *skel) +{ + /* Prepare filters */ + skel->bss->g_sock_state = 0; + skel->bss->g_unexpected = 0; + *egress_link = + bpf_program__attach_cgroup(egress_prog, + cgroup_fd); + if (!ASSERT_OK_PTR(egress_link, "egress_link")) + return -1; + *ingress_link = + bpf_program__attach_cgroup(ingress_prog, + cgroup_fd); + if (!ASSERT_OK_PTR(ingress_link, "ingress_link")) + return -1; + + return 0; +} + +static void uninstall_filters(struct bpf_link **egress_link, + struct bpf_link **ingress_link) +{ + bpf_link__destroy(*egress_link); + *egress_link = NULL; + bpf_link__destroy(*ingress_link); + *ingress_link = NULL; +} + +static int create_client_sock_v6(void) +{ + int fd; + + fd = socket(AF_INET6, SOCK_STREAM, 0); + if (fd < 0) { + perror("socket"); + return -1; + } + + return fd; +} + +static int create_server_sock_v6(void) +{ + struct sockaddr_in6 addr = { + .sin6_family = AF_INET6, + .sin6_port = htons(0), + .sin6_addr = IN6ADDR_LOOPBACK_INIT, + }; + int fd, err; + + fd = socket(AF_INET6, SOCK_STREAM, 0); + if (fd < 0) { + perror("socket"); + return -1; + } + + err = bind(fd, (struct sockaddr *)&addr, sizeof(addr)); + if (err < 0) { + perror("bind"); + return -1; + } + + err = listen(fd, 1); + if (err < 0) { + perror("listen"); + return -1; + } + + return fd; +} + +static int get_sock_port_v6(int fd) +{ + struct sockaddr_in6 addr; + socklen_t len; + int err; + + len = sizeof(addr); + err = getsockname(fd, (struct sockaddr *)&addr, &len); + if (err < 0) { + perror("getsockname"); + return -1; + } + + return ntohs(addr.sin6_port); +} + +static int connect_client_server_v6(int client_fd, int listen_fd) +{ + struct sockaddr_in6 addr = { + .sin6_family = AF_INET6, + .sin6_addr = IN6ADDR_LOOPBACK_INIT, + }; + int err; + + addr.sin6_port = htons(get_sock_port_v6(listen_fd)); + if (addr.sin6_port < 0) + return -1; + + err = connect(client_fd, (struct sockaddr *)&addr, sizeof(addr)); + if (err < 0) { + perror("connect"); + return -1; + } + + return 0; +} + +/* Connect to the server in a cgroup from the outside of the cgroup. */ +static int talk_to_cgroup(int *client_fd, int *listen_fd, int *service_fd, + struct cgroup_tcp_skb *skel) +{ + int err, cp; + char buf[5]; + + /* Create client & server socket */ + err = join_root_cgroup(); + if (!ASSERT_OK(err, "join_root_cgroup")) + return -1; + *client_fd = create_client_sock_v6(); + if (!ASSERT_GE(*client_fd, 0, "client_fd")) + return -1; + err = join_cgroup(CGROUP_TCP_SKB_PATH); + if (!ASSERT_OK(err, "join_cgroup")) + return -1; + *listen_fd = create_server_sock_v6(); + if (!ASSERT_GE(*listen_fd, 0, "listen_fd")) + return -1; + skel->bss->g_sock_port = get_sock_port_v6(*listen_fd); + + /* Connect client to server */ + err = connect_client_server_v6(*client_fd, *listen_fd); + if (!ASSERT_OK(err, "connect_client_server_v6")) + return -1; + *service_fd = accept(*listen_fd, NULL, NULL); + if (!ASSERT_GE(*service_fd, 0, "service_fd")) + return -1; + err = join_root_cgroup(); + if (!ASSERT_OK(err, "join_root_cgroup")) + return -1; + cp = write(*client_fd, "hello", 5); + if (!ASSERT_EQ(cp, 5, "write")) + return -1; + cp = read(*service_fd, buf, 5); + if (!ASSERT_EQ(cp, 5, "read")) + return -1; + + return 0; +} + +/* Connect to the server out of a cgroup from inside the cgroup. */ +static int talk_to_outside(int *client_fd, int *listen_fd, int *service_fd, + struct cgroup_tcp_skb *skel) + +{ + int err, cp; + char buf[5]; + + /* Create client & server socket */ + err = join_root_cgroup(); + if (!ASSERT_OK(err, "join_root_cgroup")) + return -1; + *listen_fd = create_server_sock_v6(); + if (!ASSERT_GE(*listen_fd, 0, "listen_fd")) + return -1; + err = join_cgroup(CGROUP_TCP_SKB_PATH); + if (!ASSERT_OK(err, "join_cgroup")) + return -1; + *client_fd = create_client_sock_v6(); + if (!ASSERT_GE(*client_fd, 0, "client_fd")) + return -1; + err = join_root_cgroup(); + if (!ASSERT_OK(err, "join_root_cgroup")) + return -1; + skel->bss->g_sock_port = get_sock_port_v6(*listen_fd); + + /* Connect client to server */ + err = connect_client_server_v6(*client_fd, *listen_fd); + if (!ASSERT_OK(err, "connect_client_server_v6")) + return -1; + *service_fd = accept(*listen_fd, NULL, NULL); + if (!ASSERT_GE(*service_fd, 0, "service_fd")) + return -1; + cp = write(*client_fd, "hello", 5); + if (!ASSERT_EQ(cp, 5, "write")) + return -1; + cp = read(*service_fd, buf, 5); + if (!ASSERT_EQ(cp, 5, "read")) + return -1; + + return 0; +} + +static int close_connection(int *closing_fd, int *peer_fd, int *listen_fd, + struct cgroup_tcp_skb *skel) +{ + __u32 saved_packet_count = 0; + int err; + int i; + + /* Wait for ACKs to be sent */ + saved_packet_count = skel->bss->g_packet_count; + usleep(100000); /* 0.1s */ + for (i = 0; + skel->bss->g_packet_count != saved_packet_count && i < 10; + i++) { + saved_packet_count = skel->bss->g_packet_count; + usleep(100000); /* 0.1s */ + } + if (!ASSERT_EQ(skel->bss->g_packet_count, saved_packet_count, + "packet_count")) + return -1; + + skel->bss->g_packet_count = 0; + saved_packet_count = 0; + + /* Half shutdown to make sure the closing socket having a chance to + * receive a FIN from the peer. + */ + err = shutdown(*closing_fd, SHUT_WR); + if (!ASSERT_OK(err, "shutdown closing_fd")) + return -1; + + /* Wait for FIN and the ACK of the FIN to be observed */ + for (i = 0; + skel->bss->g_packet_count < saved_packet_count + 2 && i < 10; + i++) + usleep(100000); /* 0.1s */ + if (!ASSERT_GE(skel->bss->g_packet_count, saved_packet_count + 2, + "packet_count")) + return -1; + + saved_packet_count = skel->bss->g_packet_count; + + /* Fully shutdown the connection */ + err = close(*peer_fd); + if (!ASSERT_OK(err, "close peer_fd")) + return -1; + *peer_fd = -1; + + /* Wait for FIN and the ACK of the FIN to be observed */ + for (i = 0; + skel->bss->g_packet_count < saved_packet_count + 2 && i < 10; + i++) + usleep(100000); /* 0.1s */ + if (!ASSERT_GE(skel->bss->g_packet_count, saved_packet_count + 2, + "packet_count")) + return -1; + + err = close(*closing_fd); + if (!ASSERT_OK(err, "close closing_fd")) + return -1; + *closing_fd = -1; + + close(*listen_fd); + *listen_fd = -1; + + return 0; +} + +/* This test case includes four scenarios: + * 1. Connect to the server from outside the cgroup and close the connection + * from outside the cgroup. + * 2. Connect to the server from outside the cgroup and close the connection + * from inside the cgroup. + * 3. Connect to the server from inside the cgroup and close the connection + * from outside the cgroup. + * 4. Connect to the server from inside the cgroup and close the connection + * from inside the cgroup. + * + * The test case is to verify that cgroup_skb/{egress,ingress} filters + * receive expected packets including SYN, SYN/ACK, ACK, FIN, and FIN/ACK. + */ +void test_cgroup_tcp_skb(void) +{ + struct bpf_link *ingress_link = NULL; + struct bpf_link *egress_link = NULL; + int client_fd = -1, listen_fd = -1; + struct cgroup_tcp_skb *skel; + int service_fd = -1; + int cgroup_fd = -1; + int err; + + skel = cgroup_tcp_skb__open_and_load(); + if (!ASSERT_OK(!skel, "skel_open_load")) + return; + + err = setup_cgroup_environment(); + if (!ASSERT_OK(err, "setup_cgroup_environment")) + goto cleanup; + + cgroup_fd = create_and_get_cgroup(CGROUP_TCP_SKB_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "cgroup_fd")) + goto cleanup; + + /* Scenario 1 */ + err = install_filters(cgroup_fd, &egress_link, &ingress_link, + skel->progs.server_egress, + skel->progs.server_ingress, + skel); + if (!ASSERT_OK(err, "install_filters")) + goto cleanup; + + err = talk_to_cgroup(&client_fd, &listen_fd, &service_fd, skel); + if (!ASSERT_OK(err, "talk_to_cgroup")) + goto cleanup; + + err = close_connection(&client_fd, &service_fd, &listen_fd, skel); + if (!ASSERT_OK(err, "close_connection")) + goto cleanup; + + ASSERT_EQ(skel->bss->g_unexpected, 0, "g_unexpected"); + ASSERT_EQ(skel->bss->g_sock_state, CLOSED, "g_sock_state"); + + uninstall_filters(&egress_link, &ingress_link); + + /* Scenario 2 */ + err = install_filters(cgroup_fd, &egress_link, &ingress_link, + skel->progs.server_egress_srv, + skel->progs.server_ingress_srv, + skel); + + err = talk_to_cgroup(&client_fd, &listen_fd, &service_fd, skel); + if (!ASSERT_OK(err, "talk_to_cgroup")) + goto cleanup; + + err = close_connection(&service_fd, &client_fd, &listen_fd, skel); + if (!ASSERT_OK(err, "close_connection")) + goto cleanup; + + ASSERT_EQ(skel->bss->g_unexpected, 0, "g_unexpected"); + ASSERT_EQ(skel->bss->g_sock_state, TIME_WAIT, "g_sock_state"); + + uninstall_filters(&egress_link, &ingress_link); + + /* Scenario 3 */ + err = install_filters(cgroup_fd, &egress_link, &ingress_link, + skel->progs.client_egress_srv, + skel->progs.client_ingress_srv, + skel); + + err = talk_to_outside(&client_fd, &listen_fd, &service_fd, skel); + if (!ASSERT_OK(err, "talk_to_outside")) + goto cleanup; + + err = close_connection(&service_fd, &client_fd, &listen_fd, skel); + if (!ASSERT_OK(err, "close_connection")) + goto cleanup; + + ASSERT_EQ(skel->bss->g_unexpected, 0, "g_unexpected"); + ASSERT_EQ(skel->bss->g_sock_state, CLOSED, "g_sock_state"); + + uninstall_filters(&egress_link, &ingress_link); + + /* Scenario 4 */ + err = install_filters(cgroup_fd, &egress_link, &ingress_link, + skel->progs.client_egress, + skel->progs.client_ingress, + skel); + + err = talk_to_outside(&client_fd, &listen_fd, &service_fd, skel); + if (!ASSERT_OK(err, "talk_to_outside")) + goto cleanup; + + err = close_connection(&client_fd, &service_fd, &listen_fd, skel); + if (!ASSERT_OK(err, "close_connection")) + goto cleanup; + + ASSERT_EQ(skel->bss->g_unexpected, 0, "g_unexpected"); + ASSERT_EQ(skel->bss->g_sock_state, TIME_WAIT, "g_sock_state"); + + uninstall_filters(&egress_link, &ingress_link); + +cleanup: + close(client_fd); + close(listen_fd); + close(service_fd); + close(cgroup_fd); + bpf_link__destroy(egress_link); + bpf_link__destroy(ingress_link); + cleanup_cgroup_environment(); + cgroup_tcp_skb__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/fentry_test.c b/tools/testing/selftests/bpf/prog_tests/fentry_test.c index c0d1d61d5f66..aee1bc77a17f 100644 --- a/tools/testing/selftests/bpf/prog_tests/fentry_test.c +++ b/tools/testing/selftests/bpf/prog_tests/fentry_test.c @@ -2,8 +2,9 @@ /* Copyright (c) 2019 Facebook */ #include <test_progs.h> #include "fentry_test.lskel.h" +#include "fentry_many_args.skel.h" -static int fentry_test(struct fentry_test_lskel *fentry_skel) +static int fentry_test_common(struct fentry_test_lskel *fentry_skel) { int err, prog_fd, i; int link_fd; @@ -37,7 +38,7 @@ static int fentry_test(struct fentry_test_lskel *fentry_skel) return 0; } -void test_fentry_test(void) +static void fentry_test(void) { struct fentry_test_lskel *fentry_skel = NULL; int err; @@ -46,13 +47,47 @@ void test_fentry_test(void) if (!ASSERT_OK_PTR(fentry_skel, "fentry_skel_load")) goto cleanup; - err = fentry_test(fentry_skel); + err = fentry_test_common(fentry_skel); if (!ASSERT_OK(err, "fentry_first_attach")) goto cleanup; - err = fentry_test(fentry_skel); + err = fentry_test_common(fentry_skel); ASSERT_OK(err, "fentry_second_attach"); cleanup: fentry_test_lskel__destroy(fentry_skel); } + +static void fentry_many_args(void) +{ + struct fentry_many_args *fentry_skel = NULL; + int err; + + fentry_skel = fentry_many_args__open_and_load(); + if (!ASSERT_OK_PTR(fentry_skel, "fentry_many_args_skel_load")) + goto cleanup; + + err = fentry_many_args__attach(fentry_skel); + if (!ASSERT_OK(err, "fentry_many_args_attach")) + goto cleanup; + + ASSERT_OK(trigger_module_test_read(1), "trigger_read"); + + ASSERT_EQ(fentry_skel->bss->test1_result, 1, + "fentry_many_args_result1"); + ASSERT_EQ(fentry_skel->bss->test2_result, 1, + "fentry_many_args_result2"); + ASSERT_EQ(fentry_skel->bss->test3_result, 1, + "fentry_many_args_result3"); + +cleanup: + fentry_many_args__destroy(fentry_skel); +} + +void test_fentry_test(void) +{ + if (test__start_subtest("fentry")) + fentry_test(); + if (test__start_subtest("fentry_many_args")) + fentry_many_args(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/fexit_test.c b/tools/testing/selftests/bpf/prog_tests/fexit_test.c index 101b7343036b..1c13007e37dd 100644 --- a/tools/testing/selftests/bpf/prog_tests/fexit_test.c +++ b/tools/testing/selftests/bpf/prog_tests/fexit_test.c @@ -2,8 +2,9 @@ /* Copyright (c) 2019 Facebook */ #include <test_progs.h> #include "fexit_test.lskel.h" +#include "fexit_many_args.skel.h" -static int fexit_test(struct fexit_test_lskel *fexit_skel) +static int fexit_test_common(struct fexit_test_lskel *fexit_skel) { int err, prog_fd, i; int link_fd; @@ -37,7 +38,7 @@ static int fexit_test(struct fexit_test_lskel *fexit_skel) return 0; } -void test_fexit_test(void) +static void fexit_test(void) { struct fexit_test_lskel *fexit_skel = NULL; int err; @@ -46,13 +47,47 @@ void test_fexit_test(void) if (!ASSERT_OK_PTR(fexit_skel, "fexit_skel_load")) goto cleanup; - err = fexit_test(fexit_skel); + err = fexit_test_common(fexit_skel); if (!ASSERT_OK(err, "fexit_first_attach")) goto cleanup; - err = fexit_test(fexit_skel); + err = fexit_test_common(fexit_skel); ASSERT_OK(err, "fexit_second_attach"); cleanup: fexit_test_lskel__destroy(fexit_skel); } + +static void fexit_many_args(void) +{ + struct fexit_many_args *fexit_skel = NULL; + int err; + + fexit_skel = fexit_many_args__open_and_load(); + if (!ASSERT_OK_PTR(fexit_skel, "fexit_many_args_skel_load")) + goto cleanup; + + err = fexit_many_args__attach(fexit_skel); + if (!ASSERT_OK(err, "fexit_many_args_attach")) + goto cleanup; + + ASSERT_OK(trigger_module_test_read(1), "trigger_read"); + + ASSERT_EQ(fexit_skel->bss->test1_result, 1, + "fexit_many_args_result1"); + ASSERT_EQ(fexit_skel->bss->test2_result, 1, + "fexit_many_args_result2"); + ASSERT_EQ(fexit_skel->bss->test3_result, 1, + "fexit_many_args_result3"); + +cleanup: + fexit_many_args__destroy(fexit_skel); +} + +void test_fexit_test(void) +{ + if (test__start_subtest("fexit")) + fexit_test(); + if (test__start_subtest("fexit_many_args")) + fexit_many_args(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/get_func_args_test.c b/tools/testing/selftests/bpf/prog_tests/get_func_args_test.c index 28cf63963cb7..64a9c95d4acf 100644 --- a/tools/testing/selftests/bpf/prog_tests/get_func_args_test.c +++ b/tools/testing/selftests/bpf/prog_tests/get_func_args_test.c @@ -30,7 +30,9 @@ void test_get_func_args_test(void) prog_fd = bpf_program__fd(skel->progs.fmod_ret_test); err = bpf_prog_test_run_opts(prog_fd, &topts); ASSERT_OK(err, "test_run"); - ASSERT_EQ(topts.retval, 1234, "test_run"); + + ASSERT_EQ(topts.retval >> 16, 1, "test_run"); + ASSERT_EQ(topts.retval & 0xffff, 1234 + 29, "test_run"); ASSERT_EQ(skel->bss->test1_result, 1, "test1_result"); ASSERT_EQ(skel->bss->test2_result, 1, "test2_result"); diff --git a/tools/testing/selftests/bpf/prog_tests/global_map_resize.c b/tools/testing/selftests/bpf/prog_tests/global_map_resize.c index fd41425d2e5c..56b5baef35c8 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_map_resize.c +++ b/tools/testing/selftests/bpf/prog_tests/global_map_resize.c @@ -22,7 +22,7 @@ static void global_map_resize_bss_subtest(void) struct test_global_map_resize *skel; struct bpf_map *map; const __u32 desired_sz = sizeof(skel->bss->sum) + sysconf(_SC_PAGE_SIZE) * 2; - size_t array_len, actual_sz; + size_t array_len, actual_sz, new_sz; skel = test_global_map_resize__open(); if (!ASSERT_OK_PTR(skel, "test_global_map_resize__open")) @@ -42,6 +42,10 @@ static void global_map_resize_bss_subtest(void) if (!ASSERT_EQ(bpf_map__value_size(map), desired_sz, "resize")) goto teardown; + new_sz = sizeof(skel->data_percpu_arr->percpu_arr[0]) * libbpf_num_possible_cpus(); + err = bpf_map__set_value_size(skel->maps.data_percpu_arr, new_sz); + ASSERT_OK(err, "percpu_arr_resize"); + /* set the expected number of elements based on the resized array */ array_len = (desired_sz - sizeof(skel->bss->sum)) / sizeof(skel->bss->array[0]); if (!ASSERT_GT(array_len, 1, "array_len")) @@ -84,11 +88,11 @@ teardown: static void global_map_resize_data_subtest(void) { - int err; struct test_global_map_resize *skel; struct bpf_map *map; const __u32 desired_sz = sysconf(_SC_PAGE_SIZE) * 2; - size_t array_len, actual_sz; + size_t array_len, actual_sz, new_sz; + int err; skel = test_global_map_resize__open(); if (!ASSERT_OK_PTR(skel, "test_global_map_resize__open")) @@ -108,6 +112,10 @@ static void global_map_resize_data_subtest(void) if (!ASSERT_EQ(bpf_map__value_size(map), desired_sz, "resize")) goto teardown; + new_sz = sizeof(skel->data_percpu_arr->percpu_arr[0]) * libbpf_num_possible_cpus(); + err = bpf_map__set_value_size(skel->maps.data_percpu_arr, new_sz); + ASSERT_OK(err, "percpu_arr_resize"); + /* set the expected number of elements based on the resized array */ array_len = (desired_sz - sizeof(skel->bss->sum)) / sizeof(skel->data_custom->my_array[0]); if (!ASSERT_GT(array_len, 1, "array_len")) diff --git a/tools/testing/selftests/bpf/prog_tests/modify_return.c b/tools/testing/selftests/bpf/prog_tests/modify_return.c index 5d9955af6247..a70c99c2f8c8 100644 --- a/tools/testing/selftests/bpf/prog_tests/modify_return.c +++ b/tools/testing/selftests/bpf/prog_tests/modify_return.c @@ -41,6 +41,10 @@ static void run_test(__u32 input_retval, __u16 want_side_effect, __s16 want_ret) ASSERT_EQ(skel->bss->fexit_result, 1, "modify_return fexit_result"); ASSERT_EQ(skel->bss->fmod_ret_result, 1, "modify_return fmod_ret_result"); + ASSERT_EQ(skel->bss->fentry_result2, 1, "modify_return fentry_result2"); + ASSERT_EQ(skel->bss->fexit_result2, 1, "modify_return fexit_result2"); + ASSERT_EQ(skel->bss->fmod_ret_result2, 1, "modify_return fmod_ret_result2"); + cleanup: modify_return__destroy(skel); } @@ -49,9 +53,9 @@ cleanup: void serial_test_modify_return(void) { run_test(0 /* input_retval */, - 1 /* want_side_effect */, - 4 /* want_ret */); + 2 /* want_side_effect */, + 33 /* want_ret */); run_test(-EINVAL /* input_retval */, 0 /* want_side_effect */, - -EINVAL /* want_ret */); + -EINVAL * 2 /* want_ret */); } diff --git a/tools/testing/selftests/bpf/prog_tests/netfilter_link_attach.c b/tools/testing/selftests/bpf/prog_tests/netfilter_link_attach.c new file mode 100644 index 000000000000..4297a2a4cb11 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/netfilter_link_attach.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include <netinet/in.h> +#include <linux/netfilter.h> + +#include "test_progs.h" +#include "test_netfilter_link_attach.skel.h" + +struct nf_link_test { + __u32 pf; + __u32 hooknum; + __s32 priority; + __u32 flags; + + bool expect_success; + const char * const name; +}; + +static const struct nf_link_test nf_hook_link_tests[] = { + { .name = "allzero", }, + { .pf = NFPROTO_NUMPROTO, .name = "invalid-pf", }, + { .pf = NFPROTO_IPV4, .hooknum = 42, .name = "invalid-hooknum", }, + { .pf = NFPROTO_IPV4, .priority = INT_MIN, .name = "invalid-priority-min", }, + { .pf = NFPROTO_IPV4, .priority = INT_MAX, .name = "invalid-priority-max", }, + { .pf = NFPROTO_IPV4, .flags = UINT_MAX, .name = "invalid-flags", }, + + { .pf = NFPROTO_INET, .priority = 1, .name = "invalid-inet-not-supported", }, + + { .pf = NFPROTO_IPV4, .priority = -10000, .expect_success = true, .name = "attach ipv4", }, + { .pf = NFPROTO_IPV6, .priority = 10001, .expect_success = true, .name = "attach ipv6", }, +}; + +void test_netfilter_link_attach(void) +{ + struct test_netfilter_link_attach *skel; + struct bpf_program *prog; + LIBBPF_OPTS(bpf_netfilter_opts, opts); + int i; + + skel = test_netfilter_link_attach__open_and_load(); + if (!ASSERT_OK_PTR(skel, "test_netfilter_link_attach__open_and_load")) + goto out; + + prog = skel->progs.nf_link_attach_test; + if (!ASSERT_OK_PTR(prog, "attach program")) + goto out; + + for (i = 0; i < ARRAY_SIZE(nf_hook_link_tests); i++) { + struct bpf_link *link; + + if (!test__start_subtest(nf_hook_link_tests[i].name)) + continue; + +#define X(opts, m, i) opts.m = nf_hook_link_tests[(i)].m + X(opts, pf, i); + X(opts, hooknum, i); + X(opts, priority, i); + X(opts, flags, i); +#undef X + link = bpf_program__attach_netfilter(prog, &opts); + if (nf_hook_link_tests[i].expect_success) { + struct bpf_link *link2; + + if (!ASSERT_OK_PTR(link, "program attach successful")) + continue; + + link2 = bpf_program__attach_netfilter(prog, &opts); + ASSERT_ERR_PTR(link2, "attach program with same pf/hook/priority"); + + if (!ASSERT_OK(bpf_link__destroy(link), "link destroy")) + break; + + link2 = bpf_program__attach_netfilter(prog, &opts); + if (!ASSERT_OK_PTR(link2, "program reattach successful")) + continue; + if (!ASSERT_OK(bpf_link__destroy(link2), "link destroy")) + break; + } else { + ASSERT_ERR_PTR(link, "program load failure"); + } + } + +out: + test_netfilter_link_attach__destroy(skel); +} + diff --git a/tools/testing/selftests/bpf/prog_tests/ptr_untrusted.c b/tools/testing/selftests/bpf/prog_tests/ptr_untrusted.c new file mode 100644 index 000000000000..8d077d150c56 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/ptr_untrusted.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2023 Yafang Shao <laoar.shao@gmail.com> */ + +#include <string.h> +#include <linux/bpf.h> +#include <test_progs.h> +#include "test_ptr_untrusted.skel.h" + +#define TP_NAME "sched_switch" + +void serial_test_ptr_untrusted(void) +{ + struct test_ptr_untrusted *skel; + int err; + + skel = test_ptr_untrusted__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + /* First, attach lsm prog */ + skel->links.lsm_run = bpf_program__attach_lsm(skel->progs.lsm_run); + if (!ASSERT_OK_PTR(skel->links.lsm_run, "lsm_attach")) + goto cleanup; + + /* Second, attach raw_tp prog. The lsm prog will be triggered. */ + skel->links.raw_tp_run = bpf_program__attach_raw_tracepoint(skel->progs.raw_tp_run, + TP_NAME); + if (!ASSERT_OK_PTR(skel->links.raw_tp_run, "raw_tp_attach")) + goto cleanup; + + err = strncmp(skel->bss->tp_name, TP_NAME, strlen(TP_NAME)); + ASSERT_EQ(err, 0, "cmp_tp_name"); + +cleanup: + test_ptr_untrusted__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c b/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c index 13bcaeb028b8..56685fc03c7e 100644 --- a/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c +++ b/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c @@ -347,7 +347,7 @@ static void syncookie_estab(void) exp_active_estab_in.max_delack_ms = 22; exp_passive_hdr_stg.syncookie = true; - exp_active_hdr_stg.resend_syn = true, + exp_active_hdr_stg.resend_syn = true; prepare_out(); diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c index 1c75a32186d6..fe0fb0c9849a 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c @@ -55,6 +55,25 @@ static void test_fentry(void) ASSERT_EQ(skel->bss->t6, 1, "t6 ret"); + ASSERT_EQ(skel->bss->t7_a, 16, "t7:a"); + ASSERT_EQ(skel->bss->t7_b, 17, "t7:b"); + ASSERT_EQ(skel->bss->t7_c, 18, "t7:c"); + ASSERT_EQ(skel->bss->t7_d, 19, "t7:d"); + ASSERT_EQ(skel->bss->t7_e, 20, "t7:e"); + ASSERT_EQ(skel->bss->t7_f_a, 21, "t7:f.a"); + ASSERT_EQ(skel->bss->t7_f_b, 22, "t7:f.b"); + ASSERT_EQ(skel->bss->t7_ret, 133, "t7 ret"); + + ASSERT_EQ(skel->bss->t8_a, 16, "t8:a"); + ASSERT_EQ(skel->bss->t8_b, 17, "t8:b"); + ASSERT_EQ(skel->bss->t8_c, 18, "t8:c"); + ASSERT_EQ(skel->bss->t8_d, 19, "t8:d"); + ASSERT_EQ(skel->bss->t8_e, 20, "t8:e"); + ASSERT_EQ(skel->bss->t8_f_a, 21, "t8:f.a"); + ASSERT_EQ(skel->bss->t8_f_b, 22, "t8:f.b"); + ASSERT_EQ(skel->bss->t8_g, 23, "t8:g"); + ASSERT_EQ(skel->bss->t8_ret, 156, "t8 ret"); + tracing_struct__detach(skel); destroy_skel: tracing_struct__destroy(skel); diff --git a/tools/testing/selftests/bpf/prog_tests/trampoline_count.c b/tools/testing/selftests/bpf/prog_tests/trampoline_count.c index e91d0d1769f1..6cd7349d4a2b 100644 --- a/tools/testing/selftests/bpf/prog_tests/trampoline_count.c +++ b/tools/testing/selftests/bpf/prog_tests/trampoline_count.c @@ -88,8 +88,8 @@ void serial_test_trampoline_count(void) if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) goto cleanup; - ASSERT_EQ(opts.retval & 0xffff, 4, "bpf_modify_return_test.result"); - ASSERT_EQ(opts.retval >> 16, 1, "bpf_modify_return_test.side_effect"); + ASSERT_EQ(opts.retval & 0xffff, 33, "bpf_modify_return_test.result"); + ASSERT_EQ(opts.retval >> 16, 2, "bpf_modify_return_test.side_effect"); cleanup: for (; i >= 0; i--) { diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 070a13833c3f..c375e59ff28d 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -58,6 +58,7 @@ #include "verifier_stack_ptr.skel.h" #include "verifier_subprog_precision.skel.h" #include "verifier_subreg.skel.h" +#include "verifier_typedef.skel.h" #include "verifier_uninit.skel.h" #include "verifier_unpriv.skel.h" #include "verifier_unpriv_perf.skel.h" @@ -159,6 +160,7 @@ void test_verifier_spin_lock(void) { RUN(verifier_spin_lock); } void test_verifier_stack_ptr(void) { RUN(verifier_stack_ptr); } void test_verifier_subprog_precision(void) { RUN(verifier_subprog_precision); } void test_verifier_subreg(void) { RUN(verifier_subreg); } +void test_verifier_typedef(void) { RUN(verifier_typedef); } void test_verifier_uninit(void) { RUN(verifier_uninit); } void test_verifier_unpriv(void) { RUN(verifier_unpriv); } void test_verifier_unpriv_perf(void) { RUN(verifier_unpriv_perf); } diff --git a/tools/testing/selftests/bpf/progs/cgroup_tcp_skb.c b/tools/testing/selftests/bpf/progs/cgroup_tcp_skb.c new file mode 100644 index 000000000000..1e2e73f3b749 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/cgroup_tcp_skb.c @@ -0,0 +1,382 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ +#include <linux/bpf.h> +#include <bpf/bpf_endian.h> +#include <bpf/bpf_helpers.h> + +#include <linux/if_ether.h> +#include <linux/in.h> +#include <linux/in6.h> +#include <linux/ipv6.h> +#include <linux/tcp.h> + +#include <sys/types.h> +#include <sys/socket.h> + +#include "cgroup_tcp_skb.h" + +char _license[] SEC("license") = "GPL"; + +__u16 g_sock_port = 0; +__u32 g_sock_state = 0; +int g_unexpected = 0; +__u32 g_packet_count = 0; + +int needed_tcp_pkt(struct __sk_buff *skb, struct tcphdr *tcph) +{ + struct ipv6hdr ip6h; + + if (skb->protocol != bpf_htons(ETH_P_IPV6)) + return 0; + if (bpf_skb_load_bytes(skb, 0, &ip6h, sizeof(ip6h))) + return 0; + + if (ip6h.nexthdr != IPPROTO_TCP) + return 0; + + if (bpf_skb_load_bytes(skb, sizeof(ip6h), tcph, sizeof(*tcph))) + return 0; + + if (tcph->source != bpf_htons(g_sock_port) && + tcph->dest != bpf_htons(g_sock_port)) + return 0; + + return 1; +} + +/* Run accept() on a socket in the cgroup to receive a new connection. */ +static int egress_accept(struct tcphdr *tcph) +{ + if (g_sock_state == SYN_RECV_SENDING_SYN_ACK) { + if (tcph->fin || !tcph->syn || !tcph->ack) + g_unexpected++; + else + g_sock_state = SYN_RECV; + return 1; + } + + return 0; +} + +static int ingress_accept(struct tcphdr *tcph) +{ + switch (g_sock_state) { + case INIT: + if (!tcph->syn || tcph->fin || tcph->ack) + g_unexpected++; + else + g_sock_state = SYN_RECV_SENDING_SYN_ACK; + break; + case SYN_RECV: + if (tcph->fin || tcph->syn || !tcph->ack) + g_unexpected++; + else + g_sock_state = ESTABLISHED; + break; + default: + return 0; + } + + return 1; +} + +/* Run connect() on a socket in the cgroup to start a new connection. */ +static int egress_connect(struct tcphdr *tcph) +{ + if (g_sock_state == INIT) { + if (!tcph->syn || tcph->fin || tcph->ack) + g_unexpected++; + else + g_sock_state = SYN_SENT; + return 1; + } + + return 0; +} + +static int ingress_connect(struct tcphdr *tcph) +{ + if (g_sock_state == SYN_SENT) { + if (tcph->fin || !tcph->syn || !tcph->ack) + g_unexpected++; + else + g_sock_state = ESTABLISHED; + return 1; + } + + return 0; +} + +/* The connection is closed by the peer outside the cgroup. */ +static int egress_close_remote(struct tcphdr *tcph) +{ + switch (g_sock_state) { + case ESTABLISHED: + break; + case CLOSE_WAIT_SENDING_ACK: + if (tcph->fin || tcph->syn || !tcph->ack) + g_unexpected++; + else + g_sock_state = CLOSE_WAIT; + break; + case CLOSE_WAIT: + if (!tcph->fin) + g_unexpected++; + else + g_sock_state = LAST_ACK; + break; + default: + return 0; + } + + return 1; +} + +static int ingress_close_remote(struct tcphdr *tcph) +{ + switch (g_sock_state) { + case ESTABLISHED: + if (tcph->fin) + g_sock_state = CLOSE_WAIT_SENDING_ACK; + break; + case LAST_ACK: + if (tcph->fin || tcph->syn || !tcph->ack) + g_unexpected++; + else + g_sock_state = CLOSED; + break; + default: + return 0; + } + + return 1; +} + +/* The connection is closed by the endpoint inside the cgroup. */ +static int egress_close_local(struct tcphdr *tcph) +{ + switch (g_sock_state) { + case ESTABLISHED: + if (tcph->fin) + g_sock_state = FIN_WAIT1; + break; + case TIME_WAIT_SENDING_ACK: + if (tcph->fin || tcph->syn || !tcph->ack) + g_unexpected++; + else + g_sock_state = TIME_WAIT; + break; + default: + return 0; + } + + return 1; +} + +static int ingress_close_local(struct tcphdr *tcph) +{ + switch (g_sock_state) { + case ESTABLISHED: + break; + case FIN_WAIT1: + if (tcph->fin || tcph->syn || !tcph->ack) + g_unexpected++; + else + g_sock_state = FIN_WAIT2; + break; + case FIN_WAIT2: + if (!tcph->fin || tcph->syn || !tcph->ack) + g_unexpected++; + else + g_sock_state = TIME_WAIT_SENDING_ACK; + break; + default: + return 0; + } + + return 1; +} + +/* Check the types of outgoing packets of a server socket to make sure they + * are consistent with the state of the server socket. + * + * The connection is closed by the client side. + */ +SEC("cgroup_skb/egress") +int server_egress(struct __sk_buff *skb) +{ + struct tcphdr tcph; + + if (!needed_tcp_pkt(skb, &tcph)) + return 1; + + g_packet_count++; + + /* Egress of the server socket. */ + if (egress_accept(&tcph) || egress_close_remote(&tcph)) + return 1; + + g_unexpected++; + return 1; +} + +/* Check the types of incoming packets of a server socket to make sure they + * are consistent with the state of the server socket. + * + * The connection is closed by the client side. + */ +SEC("cgroup_skb/ingress") +int server_ingress(struct __sk_buff *skb) +{ + struct tcphdr tcph; + + if (!needed_tcp_pkt(skb, &tcph)) + return 1; + + g_packet_count++; + + /* Ingress of the server socket. */ + if (ingress_accept(&tcph) || ingress_close_remote(&tcph)) + return 1; + + g_unexpected++; + return 1; +} + +/* Check the types of outgoing packets of a server socket to make sure they + * are consistent with the state of the server socket. + * + * The connection is closed by the server side. + */ +SEC("cgroup_skb/egress") +int server_egress_srv(struct __sk_buff *skb) +{ + struct tcphdr tcph; + + if (!needed_tcp_pkt(skb, &tcph)) + return 1; + + g_packet_count++; + + /* Egress of the server socket. */ + if (egress_accept(&tcph) || egress_close_local(&tcph)) + return 1; + + g_unexpected++; + return 1; +} + +/* Check the types of incoming packets of a server socket to make sure they + * are consistent with the state of the server socket. + * + * The connection is closed by the server side. + */ +SEC("cgroup_skb/ingress") +int server_ingress_srv(struct __sk_buff *skb) +{ + struct tcphdr tcph; + + if (!needed_tcp_pkt(skb, &tcph)) + return 1; + + g_packet_count++; + + /* Ingress of the server socket. */ + if (ingress_accept(&tcph) || ingress_close_local(&tcph)) + return 1; + + g_unexpected++; + return 1; +} + +/* Check the types of outgoing packets of a client socket to make sure they + * are consistent with the state of the client socket. + * + * The connection is closed by the server side. + */ +SEC("cgroup_skb/egress") +int client_egress_srv(struct __sk_buff *skb) +{ + struct tcphdr tcph; + + if (!needed_tcp_pkt(skb, &tcph)) + return 1; + + g_packet_count++; + + /* Egress of the server socket. */ + if (egress_connect(&tcph) || egress_close_remote(&tcph)) + return 1; + + g_unexpected++; + return 1; +} + +/* Check the types of incoming packets of a client socket to make sure they + * are consistent with the state of the client socket. + * + * The connection is closed by the server side. + */ +SEC("cgroup_skb/ingress") +int client_ingress_srv(struct __sk_buff *skb) +{ + struct tcphdr tcph; + + if (!needed_tcp_pkt(skb, &tcph)) + return 1; + + g_packet_count++; + + /* Ingress of the server socket. */ + if (ingress_connect(&tcph) || ingress_close_remote(&tcph)) + return 1; + + g_unexpected++; + return 1; +} + +/* Check the types of outgoing packets of a client socket to make sure they + * are consistent with the state of the client socket. + * + * The connection is closed by the client side. + */ +SEC("cgroup_skb/egress") +int client_egress(struct __sk_buff *skb) +{ + struct tcphdr tcph; + + if (!needed_tcp_pkt(skb, &tcph)) + return 1; + + g_packet_count++; + + /* Egress of the server socket. */ + if (egress_connect(&tcph) || egress_close_local(&tcph)) + return 1; + + g_unexpected++; + return 1; +} + +/* Check the types of incoming packets of a client socket to make sure they + * are consistent with the state of the client socket. + * + * The connection is closed by the client side. + */ +SEC("cgroup_skb/ingress") +int client_ingress(struct __sk_buff *skb) +{ + struct tcphdr tcph; + + if (!needed_tcp_pkt(skb, &tcph)) + return 1; + + g_packet_count++; + + /* Ingress of the server socket. */ + if (ingress_connect(&tcph) || ingress_close_local(&tcph)) + return 1; + + g_unexpected++; + return 1; +} diff --git a/tools/testing/selftests/bpf/progs/fentry_many_args.c b/tools/testing/selftests/bpf/progs/fentry_many_args.c new file mode 100644 index 000000000000..b61bb92fee2c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/fentry_many_args.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Tencent */ +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +__u64 test1_result = 0; +SEC("fentry/bpf_testmod_fentry_test7") +int BPF_PROG(test1, __u64 a, void *b, short c, int d, void *e, char f, + int g) +{ + test1_result = a == 16 && b == (void *)17 && c == 18 && d == 19 && + e == (void *)20 && f == 21 && g == 22; + return 0; +} + +__u64 test2_result = 0; +SEC("fentry/bpf_testmod_fentry_test11") +int BPF_PROG(test2, __u64 a, void *b, short c, int d, void *e, char f, + int g, unsigned int h, long i, __u64 j, unsigned long k) +{ + test2_result = a == 16 && b == (void *)17 && c == 18 && d == 19 && + e == (void *)20 && f == 21 && g == 22 && h == 23 && + i == 24 && j == 25 && k == 26; + return 0; +} + +__u64 test3_result = 0; +SEC("fentry/bpf_testmod_fentry_test11") +int BPF_PROG(test3, __u64 a, __u64 b, __u64 c, __u64 d, __u64 e, __u64 f, + __u64 g, __u64 h, __u64 i, __u64 j, __u64 k) +{ + test3_result = a == 16 && b == 17 && c == 18 && d == 19 && + e == 20 && f == 21 && g == 22 && h == 23 && + i == 24 && j == 25 && k == 26; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/fexit_many_args.c b/tools/testing/selftests/bpf/progs/fexit_many_args.c new file mode 100644 index 000000000000..53b335c2dafb --- /dev/null +++ b/tools/testing/selftests/bpf/progs/fexit_many_args.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Tencent */ +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +__u64 test1_result = 0; +SEC("fexit/bpf_testmod_fentry_test7") +int BPF_PROG(test1, __u64 a, void *b, short c, int d, void *e, char f, + int g, int ret) +{ + test1_result = a == 16 && b == (void *)17 && c == 18 && d == 19 && + e == (void *)20 && f == 21 && g == 22 && ret == 133; + return 0; +} + +__u64 test2_result = 0; +SEC("fexit/bpf_testmod_fentry_test11") +int BPF_PROG(test2, __u64 a, void *b, short c, int d, void *e, char f, + int g, unsigned int h, long i, __u64 j, unsigned long k, + int ret) +{ + test2_result = a == 16 && b == (void *)17 && c == 18 && d == 19 && + e == (void *)20 && f == 21 && g == 22 && h == 23 && + i == 24 && j == 25 && k == 26 && ret == 231; + return 0; +} + +__u64 test3_result = 0; +SEC("fexit/bpf_testmod_fentry_test11") +int BPF_PROG(test3, __u64 a, __u64 b, __u64 c, __u64 d, __u64 e, __u64 f, + __u64 g, __u64 h, __u64 i, __u64 j, __u64 k, __u64 ret) +{ + test3_result = a == 16 && b == 17 && c == 18 && d == 19 && + e == 20 && f == 21 && g == 22 && h == 23 && + i == 24 && j == 25 && k == 26 && ret == 231; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/htab_mem_bench.c b/tools/testing/selftests/bpf/progs/htab_mem_bench.c new file mode 100644 index 000000000000..b1b721b14d67 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/htab_mem_bench.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2023. Huawei Technologies Co., Ltd */ +#include <stdbool.h> +#include <errno.h> +#include <linux/types.h> +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +#define OP_BATCH 64 + +struct update_ctx { + unsigned int from; + unsigned int step; +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, 4); + __uint(map_flags, BPF_F_NO_PREALLOC); +} htab SEC(".maps"); + +char _license[] SEC("license") = "GPL"; + +unsigned char zeroed_value[4096]; +unsigned int nr_thread = 0; +long op_cnt = 0; + +static int write_htab(unsigned int i, struct update_ctx *ctx, unsigned int flags) +{ + bpf_map_update_elem(&htab, &ctx->from, zeroed_value, flags); + ctx->from += ctx->step; + + return 0; +} + +static int overwrite_htab(unsigned int i, struct update_ctx *ctx) +{ + return write_htab(i, ctx, 0); +} + +static int newwrite_htab(unsigned int i, struct update_ctx *ctx) +{ + return write_htab(i, ctx, BPF_NOEXIST); +} + +static int del_htab(unsigned int i, struct update_ctx *ctx) +{ + bpf_map_delete_elem(&htab, &ctx->from); + ctx->from += ctx->step; + + return 0; +} + +SEC("?tp/syscalls/sys_enter_getpgid") +int overwrite(void *ctx) +{ + struct update_ctx update; + + update.from = bpf_get_smp_processor_id(); + update.step = nr_thread; + bpf_loop(OP_BATCH, overwrite_htab, &update, 0); + __sync_fetch_and_add(&op_cnt, 1); + return 0; +} + +SEC("?tp/syscalls/sys_enter_getpgid") +int batch_add_batch_del(void *ctx) +{ + struct update_ctx update; + + update.from = bpf_get_smp_processor_id(); + update.step = nr_thread; + bpf_loop(OP_BATCH, overwrite_htab, &update, 0); + + update.from = bpf_get_smp_processor_id(); + bpf_loop(OP_BATCH, del_htab, &update, 0); + + __sync_fetch_and_add(&op_cnt, 2); + return 0; +} + +SEC("?tp/syscalls/sys_enter_getpgid") +int add_only(void *ctx) +{ + struct update_ctx update; + + update.from = bpf_get_smp_processor_id() / 2; + update.step = nr_thread / 2; + bpf_loop(OP_BATCH, newwrite_htab, &update, 0); + __sync_fetch_and_add(&op_cnt, 1); + return 0; +} + +SEC("?tp/syscalls/sys_enter_getppid") +int del_only(void *ctx) +{ + struct update_ctx update; + + update.from = bpf_get_smp_processor_id() / 2; + update.step = nr_thread / 2; + bpf_loop(OP_BATCH, del_htab, &update, 0); + __sync_fetch_and_add(&op_cnt, 1); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/linked_list.c b/tools/testing/selftests/bpf/progs/linked_list.c index 57440a554304..84d1777a9e6c 100644 --- a/tools/testing/selftests/bpf/progs/linked_list.c +++ b/tools/testing/selftests/bpf/progs/linked_list.c @@ -96,7 +96,7 @@ static __always_inline int list_push_pop_multiple(struct bpf_spin_lock *lock, struct bpf_list_head *head, bool leave_in_map) { struct bpf_list_node *n; - struct foo *f[8], *pf; + struct foo *f[200], *pf; int i; /* Loop following this check adds nodes 2-at-a-time in order to diff --git a/tools/testing/selftests/bpf/progs/map_percpu_stats.c b/tools/testing/selftests/bpf/progs/map_percpu_stats.c new file mode 100644 index 000000000000..10b2325c1720 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/map_percpu_stats.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Isovalent */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +__u32 target_id; + +__s64 bpf_map_sum_elem_count(struct bpf_map *map) __ksym; + +SEC("iter/bpf_map") +int dump_bpf_map(struct bpf_iter__bpf_map *ctx) +{ + struct seq_file *seq = ctx->meta->seq; + struct bpf_map *map = ctx->map; + + if (map && map->id == target_id) + BPF_SEQ_PRINTF(seq, "%lld", bpf_map_sum_elem_count(map)); + + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/modify_return.c b/tools/testing/selftests/bpf/progs/modify_return.c index 8b7466a15c6b..3376d4849f58 100644 --- a/tools/testing/selftests/bpf/progs/modify_return.c +++ b/tools/testing/selftests/bpf/progs/modify_return.c @@ -47,3 +47,43 @@ int BPF_PROG(fexit_test, int a, __u64 b, int ret) return 0; } + +static int sequence2; + +__u64 fentry_result2 = 0; +SEC("fentry/bpf_modify_return_test2") +int BPF_PROG(fentry_test2, int a, int *b, short c, int d, void *e, char f, + int g) +{ + sequence2++; + fentry_result2 = (sequence2 == 1); + return 0; +} + +__u64 fmod_ret_result2 = 0; +SEC("fmod_ret/bpf_modify_return_test2") +int BPF_PROG(fmod_ret_test2, int a, int *b, short c, int d, void *e, char f, + int g, int ret) +{ + sequence2++; + /* This is the first fmod_ret program, the ret passed should be 0 */ + fmod_ret_result2 = (sequence2 == 2 && ret == 0); + return input_retval; +} + +__u64 fexit_result2 = 0; +SEC("fexit/bpf_modify_return_test2") +int BPF_PROG(fexit_test2, int a, int *b, short c, int d, void *e, char f, + int g, int ret) +{ + sequence2++; + /* If the input_reval is non-zero a successful modification should have + * occurred. + */ + if (input_retval) + fexit_result2 = (sequence2 == 3 && ret == input_retval); + else + fexit_result2 = (sequence2 == 3 && ret == 29); + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/nested_trust_failure.c b/tools/testing/selftests/bpf/progs/nested_trust_failure.c index 0d1aa6bbace4..ea39497f11ed 100644 --- a/tools/testing/selftests/bpf/progs/nested_trust_failure.c +++ b/tools/testing/selftests/bpf/progs/nested_trust_failure.c @@ -10,6 +10,13 @@ char _license[] SEC("license") = "GPL"; +struct { + __uint(type, BPF_MAP_TYPE_SK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, u64); +} sk_storage_map SEC(".maps"); + /* Prototype for all of the program trace events below: * * TRACE_EVENT(task_newtask, @@ -31,3 +38,12 @@ int BPF_PROG(test_invalid_nested_offset, struct task_struct *task, u64 clone_fla bpf_cpumask_first_zero(&task->cpus_mask); return 0; } + +/* Although R2 is of type sk_buff but sock_common is expected, we will hit untrusted ptr first. */ +SEC("tp_btf/tcp_probe") +__failure __msg("R2 type=untrusted_ptr_ expected=ptr_, trusted_ptr_, rcu_ptr_") +int BPF_PROG(test_invalid_skb_field, struct sock *sk, struct sk_buff *skb) +{ + bpf_sk_storage_get(&sk_storage_map, skb->next, 0, 0); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/nested_trust_success.c b/tools/testing/selftests/bpf/progs/nested_trust_success.c index 886ade4aa99d..833840bffd3b 100644 --- a/tools/testing/selftests/bpf/progs/nested_trust_success.c +++ b/tools/testing/selftests/bpf/progs/nested_trust_success.c @@ -10,6 +10,13 @@ char _license[] SEC("license") = "GPL"; +struct { + __uint(type, BPF_MAP_TYPE_SK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, u64); +} sk_storage_map SEC(".maps"); + SEC("tp_btf/task_newtask") __success int BPF_PROG(test_read_cpumask, struct task_struct *task, u64 clone_flags) @@ -17,3 +24,11 @@ int BPF_PROG(test_read_cpumask, struct task_struct *task, u64 clone_flags) bpf_cpumask_test_cpu(0, task->cpus_ptr); return 0; } + +SEC("tp_btf/tcp_probe") +__success +int BPF_PROG(test_skb_field, struct sock *sk, struct sk_buff *skb) +{ + bpf_sk_storage_get(&sk_storage_map, skb->sk, 0, 0); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/test_global_map_resize.c b/tools/testing/selftests/bpf/progs/test_global_map_resize.c index 2588f2384246..1fbb73d3e5d5 100644 --- a/tools/testing/selftests/bpf/progs/test_global_map_resize.c +++ b/tools/testing/selftests/bpf/progs/test_global_map_resize.c @@ -29,13 +29,16 @@ int my_int SEC(".data.non_array"); int my_array_first[1] SEC(".data.array_not_last"); int my_int_last SEC(".data.array_not_last"); +int percpu_arr[1] SEC(".data.percpu_arr"); + SEC("tp/syscalls/sys_enter_getpid") int bss_array_sum(void *ctx) { if (pid != (bpf_get_current_pid_tgid() >> 32)) return 0; - sum = 0; + /* this will be zero, we just rely on verifier not rejecting this */ + sum = percpu_arr[bpf_get_smp_processor_id()]; for (size_t i = 0; i < bss_array_len; ++i) sum += array[i]; @@ -49,7 +52,8 @@ int data_array_sum(void *ctx) if (pid != (bpf_get_current_pid_tgid() >> 32)) return 0; - sum = 0; + /* this will be zero, we just rely on verifier not rejecting this */ + sum = percpu_arr[bpf_get_smp_processor_id()]; for (size_t i = 0; i < data_array_len; ++i) sum += my_array[i]; diff --git a/tools/testing/selftests/bpf/progs/test_netfilter_link_attach.c b/tools/testing/selftests/bpf/progs/test_netfilter_link_attach.c new file mode 100644 index 000000000000..03a475160abe --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_netfilter_link_attach.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> + +#define NF_ACCEPT 1 + +SEC("netfilter") +int nf_link_attach_test(struct bpf_nf_ctx *ctx) +{ + return NF_ACCEPT; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_ptr_untrusted.c b/tools/testing/selftests/bpf/progs/test_ptr_untrusted.c new file mode 100644 index 000000000000..4bdd65b5aa2d --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_ptr_untrusted.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2023 Yafang Shao <laoar.shao@gmail.com> */ + +#include "vmlinux.h" +#include <bpf/bpf_tracing.h> + +char tp_name[128]; + +SEC("lsm/bpf") +int BPF_PROG(lsm_run, int cmd, union bpf_attr *attr, unsigned int size) +{ + switch (cmd) { + case BPF_RAW_TRACEPOINT_OPEN: + bpf_probe_read_user_str(tp_name, sizeof(tp_name) - 1, + (void *)attr->raw_tracepoint.name); + break; + default: + break; + } + return 0; +} + +SEC("raw_tracepoint") +int BPF_PROG(raw_tp_run) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/tracing_struct.c b/tools/testing/selftests/bpf/progs/tracing_struct.c index c435a3a8328a..515daef3c84b 100644 --- a/tools/testing/selftests/bpf/progs/tracing_struct.c +++ b/tools/testing/selftests/bpf/progs/tracing_struct.c @@ -18,6 +18,11 @@ struct bpf_testmod_struct_arg_3 { int b[]; }; +struct bpf_testmod_struct_arg_4 { + u64 a; + int b; +}; + long t1_a_a, t1_a_b, t1_b, t1_c, t1_ret, t1_nregs; __u64 t1_reg0, t1_reg1, t1_reg2, t1_reg3; long t2_a, t2_b_a, t2_b_b, t2_c, t2_ret; @@ -25,6 +30,9 @@ long t3_a, t3_b, t3_c_a, t3_c_b, t3_ret; long t4_a_a, t4_b, t4_c, t4_d, t4_e_a, t4_e_b, t4_ret; long t5_ret; int t6; +long t7_a, t7_b, t7_c, t7_d, t7_e, t7_f_a, t7_f_b, t7_ret; +long t8_a, t8_b, t8_c, t8_d, t8_e, t8_f_a, t8_f_b, t8_g, t8_ret; + SEC("fentry/bpf_testmod_test_struct_arg_1") int BPF_PROG2(test_struct_arg_1, struct bpf_testmod_struct_arg_2, a, int, b, int, c) @@ -130,4 +138,50 @@ int BPF_PROG2(test_struct_arg_11, struct bpf_testmod_struct_arg_3 *, a) return 0; } +SEC("fentry/bpf_testmod_test_struct_arg_7") +int BPF_PROG2(test_struct_arg_12, __u64, a, void *, b, short, c, int, d, + void *, e, struct bpf_testmod_struct_arg_4, f) +{ + t7_a = a; + t7_b = (long)b; + t7_c = c; + t7_d = d; + t7_e = (long)e; + t7_f_a = f.a; + t7_f_b = f.b; + return 0; +} + +SEC("fexit/bpf_testmod_test_struct_arg_7") +int BPF_PROG2(test_struct_arg_13, __u64, a, void *, b, short, c, int, d, + void *, e, struct bpf_testmod_struct_arg_4, f, int, ret) +{ + t7_ret = ret; + return 0; +} + +SEC("fentry/bpf_testmod_test_struct_arg_8") +int BPF_PROG2(test_struct_arg_14, __u64, a, void *, b, short, c, int, d, + void *, e, struct bpf_testmod_struct_arg_4, f, int, g) +{ + t8_a = a; + t8_b = (long)b; + t8_c = c; + t8_d = d; + t8_e = (long)e; + t8_f_a = f.a; + t8_f_b = f.b; + t8_g = g; + return 0; +} + +SEC("fexit/bpf_testmod_test_struct_arg_8") +int BPF_PROG2(test_struct_arg_15, __u64, a, void *, b, short, c, int, d, + void *, e, struct bpf_testmod_struct_arg_4, f, int, g, + int, ret) +{ + t8_ret = ret; + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_typedef.c b/tools/testing/selftests/bpf/progs/verifier_typedef.c new file mode 100644 index 000000000000..08481cfaac4b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_typedef.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +SEC("fentry/bpf_fentry_test_sinfo") +__description("typedef: resolve") +__success __retval(0) +__naked void resolve_typedef(void) +{ + asm volatile (" \ + r1 = *(u64 *)(r1 +0); \ + r2 = *(u64 *)(r1 +%[frags_offs]); \ + r0 = 0; \ + exit; \ +" : + : __imm_const(frags_offs, + offsetof(struct skb_shared_info, frags)) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/trace_helpers.c b/tools/testing/selftests/bpf/trace_helpers.c index 9b070cdf44ac..f83d9f65c65b 100644 --- a/tools/testing/selftests/bpf/trace_helpers.c +++ b/tools/testing/selftests/bpf/trace_helpers.c @@ -18,7 +18,7 @@ #define TRACEFS_PIPE "/sys/kernel/tracing/trace_pipe" #define DEBUGFS_PIPE "/sys/kernel/debug/tracing/trace_pipe" -#define MAX_SYMS 300000 +#define MAX_SYMS 400000 static struct ksym syms[MAX_SYMS]; static int sym_cnt; @@ -46,6 +46,9 @@ int load_kallsyms_refresh(void) break; if (!addr) continue; + if (i >= MAX_SYMS) + return -EFBIG; + syms[i].addr = (long) addr; syms[i].name = strdup(func); i++; diff --git a/tools/testing/selftests/bpf/verifier/atomic_cmpxchg.c b/tools/testing/selftests/bpf/verifier/atomic_cmpxchg.c index b39665f33524..319337bdcfc8 100644 --- a/tools/testing/selftests/bpf/verifier/atomic_cmpxchg.c +++ b/tools/testing/selftests/bpf/verifier/atomic_cmpxchg.c @@ -242,4 +242,5 @@ .result = REJECT, .errstr = "R0 invalid mem access", .errstr_unpriv = "R10 partial copy of pointer", + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, diff --git a/tools/testing/selftests/bpf/verifier/ctx_skb.c b/tools/testing/selftests/bpf/verifier/ctx_skb.c index 83cecfbd6739..0b394a7f7a2d 100644 --- a/tools/testing/selftests/bpf/verifier/ctx_skb.c +++ b/tools/testing/selftests/bpf/verifier/ctx_skb.c @@ -1169,6 +1169,7 @@ }, .result = ACCEPT, .prog_type = BPF_PROG_TYPE_SK_SKB, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "pkt_end < pkt taken check", @@ -1190,4 +1191,5 @@ }, .result = ACCEPT, .prog_type = BPF_PROG_TYPE_SK_SKB, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, diff --git a/tools/testing/selftests/bpf/verifier/jmp32.c b/tools/testing/selftests/bpf/verifier/jmp32.c index 1a27a6210554..43776f6f92f4 100644 --- a/tools/testing/selftests/bpf/verifier/jmp32.c +++ b/tools/testing/selftests/bpf/verifier/jmp32.c @@ -290,6 +290,7 @@ .result_unpriv = REJECT, .result = ACCEPT, .retval = 2, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "jgt32: BPF_K", @@ -360,6 +361,7 @@ .result_unpriv = REJECT, .result = ACCEPT, .retval = 2, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "jle32: BPF_K", @@ -430,6 +432,7 @@ .result_unpriv = REJECT, .result = ACCEPT, .retval = 2, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "jlt32: BPF_K", @@ -500,6 +503,7 @@ .result_unpriv = REJECT, .result = ACCEPT, .retval = 2, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "jsge32: BPF_K", @@ -570,6 +574,7 @@ .result_unpriv = REJECT, .result = ACCEPT, .retval = 2, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "jsgt32: BPF_K", @@ -640,6 +645,7 @@ .result_unpriv = REJECT, .result = ACCEPT, .retval = 2, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "jsle32: BPF_K", @@ -710,6 +716,7 @@ .result_unpriv = REJECT, .result = ACCEPT, .retval = 2, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "jslt32: BPF_K", @@ -780,6 +787,7 @@ .result_unpriv = REJECT, .result = ACCEPT, .retval = 2, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "jgt32: range bound deduction, reg op imm", diff --git a/tools/testing/selftests/bpf/verifier/map_kptr.c b/tools/testing/selftests/bpf/verifier/map_kptr.c index a0cfc06d75bc..d25c3e9605f1 100644 --- a/tools/testing/selftests/bpf/verifier/map_kptr.c +++ b/tools/testing/selftests/bpf/verifier/map_kptr.c @@ -68,6 +68,7 @@ .fixup_map_kptr = { 1 }, .result = REJECT, .errstr = "kptr access cannot have variable offset", + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "map_kptr: bpf_kptr_xchg non-const var_off", @@ -121,6 +122,7 @@ .fixup_map_kptr = { 1 }, .result = REJECT, .errstr = "kptr access misaligned expected=0 off=7", + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "map_kptr: reject var_off != 0", diff --git a/tools/testing/selftests/bpf/verifier/precise.c b/tools/testing/selftests/bpf/verifier/precise.c index 99272bb890da..0d84dd1f38b6 100644 --- a/tools/testing/selftests/bpf/verifier/precise.c +++ b/tools/testing/selftests/bpf/verifier/precise.c @@ -216,7 +216,7 @@ }, .fixup_map_ringbuf = { 1 }, .prog_type = BPF_PROG_TYPE_XDP, - .flags = BPF_F_TEST_STATE_FREQ, + .flags = BPF_F_TEST_STATE_FREQ | F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, .errstr = "invalid access to memory, mem_size=1 off=42 size=8", .result = REJECT, }, diff --git a/tools/testing/selftests/hid/Makefile b/tools/testing/selftests/hid/Makefile index 01c0491d64da..2e986cbf1a46 100644 --- a/tools/testing/selftests/hid/Makefile +++ b/tools/testing/selftests/hid/Makefile @@ -167,7 +167,7 @@ $(RESOLVE_BTFIDS): $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/resolve_btfids \ OUTPUT=$(HOST_BUILD_DIR)/resolve_btfids/ BPFOBJ=$(HOST_BPFOBJ) # Get Clang's default includes on this system, as opposed to those seen by -# '-target bpf'. This fixes "missing" files on some architectures/distros, +# '--target=bpf'. This fixes "missing" files on some architectures/distros, # such as asm/byteorder.h, asm/socket.h, asm/sockios.h, sys/cdefs.h etc. # # Use '-idirafter': Don't interfere with include mechanics except where the @@ -196,12 +196,12 @@ CLANG_CFLAGS = $(CLANG_SYS_INCLUDES) \ # $3 - CFLAGS define CLANG_BPF_BUILD_RULE $(call msg,CLNG-BPF,$(TRUNNER_BINARY),$2) - $(Q)$(CLANG) $3 -O2 -target bpf -c $1 -mcpu=v3 -o $2 + $(Q)$(CLANG) $3 -O2 --target=bpf -c $1 -mcpu=v3 -o $2 endef # Similar to CLANG_BPF_BUILD_RULE, but with disabled alu32 define CLANG_NOALU32_BPF_BUILD_RULE $(call msg,CLNG-BPF,$(TRUNNER_BINARY),$2) - $(Q)$(CLANG) $3 -O2 -target bpf -c $1 -mcpu=v2 -o $2 + $(Q)$(CLANG) $3 -O2 --target=bpf -c $1 -mcpu=v2 -o $2 endef # Build BPF object using GCC define GCC_BPF_BUILD_RULE diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile index 7f3ab2a93ed6..2f69f7274e3d 100644 --- a/tools/testing/selftests/net/Makefile +++ b/tools/testing/selftests/net/Makefile @@ -113,7 +113,7 @@ $(MAKE_DIRS): mkdir -p $@ # Get Clang's default includes on this system, as opposed to those seen by -# '-target bpf'. This fixes "missing" files on some architectures/distros, +# '--target=bpf'. This fixes "missing" files on some architectures/distros, # such as asm/byteorder.h, asm/socket.h, asm/sockios.h, sys/cdefs.h etc. # # Use '-idirafter': Don't interfere with include mechanics except where the @@ -131,7 +131,7 @@ endif CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH)) $(OUTPUT)/nat6to4.o: nat6to4.c $(BPFOBJ) | $(MAKE_DIRS) - $(CLANG) -O2 -target bpf -c $< $(CCINCLUDE) $(CLANG_SYS_INCLUDES) -o $@ + $(CLANG) -O2 --target=bpf -c $< $(CCINCLUDE) $(CLANG_SYS_INCLUDES) -o $@ $(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \ $(APIDIR)/linux/bpf.h \ diff --git a/tools/testing/selftests/tc-testing/Makefile b/tools/testing/selftests/tc-testing/Makefile index cb553eac9f41..3c4b7fa05075 100644 --- a/tools/testing/selftests/tc-testing/Makefile +++ b/tools/testing/selftests/tc-testing/Makefile @@ -24,7 +24,7 @@ CLANG_FLAGS = -I. -I$(APIDIR) \ $(OUTPUT)/%.o: %.c $(CLANG) $(CLANG_FLAGS) \ - -O2 -target bpf -emit-llvm -c $< -o - | \ + -O2 --target=bpf -emit-llvm -c $< -o - | \ $(LLC) -march=bpf -mcpu=$(CPU) $(LLC_FLAGS) -filetype=obj -o $@ TEST_PROGS += ./tdc.sh |