diff options
3 files changed, 48 insertions, 53 deletions
diff --git a/tools/testing/selftests/bpf/prog_tests/task_local_data.h b/tools/testing/selftests/bpf/prog_tests/task_local_data.h index 7819f318b2fb..1e5c67c78ffb 100644 --- a/tools/testing/selftests/bpf/prog_tests/task_local_data.h +++ b/tools/testing/selftests/bpf/prog_tests/task_local_data.h @@ -22,14 +22,17 @@ /* * OPTIONS * - * Define the option before including the header + * Define the option before including the header. Using different options in + * different translation units is strongly discouraged. * * TLD_FREE_DATA_ON_THREAD_EXIT - Frees memory on thread exit automatically * * Thread-specific memory for storing TLD is allocated lazily on the first call to * tld_get_data(). The thread that calls it must also call tld_free() on thread exit * to prevent memory leak. Pthread will be included if the option is defined. A pthread - * key will be registered with a destructor that calls tld_free(). + * key will be registered with a destructor that calls tld_free(). Enabled only when + * the option is defined and TLD_DEFINE_KEY/tld_create_key() is called in the same + * translation unit. * * * TLD_DYN_DATA_SIZE - The maximum size of memory allocated for TLDs created dynamically @@ -47,19 +50,16 @@ * TLD_NAME_LEN - The maximum length of the name of a TLD (default: 62) * * Setting TLD_NAME_LEN will affect the maximum number of TLDs a process can store, - * TLD_MAX_DATA_CNT. + * TLD_MAX_DATA_CNT. Must be consistent with task_local_data.bpf.h. * * - * TLD_DATA_USE_ALIGNED_ALLOC - Always use aligned_alloc() instead of malloc() + * TLD_DONT_ROUND_UP_DATA_SIZE - Don't round up memory size allocated for data if + * the memory allocator has low overhead aligned_alloc() implementation. * - * When allocating the memory for storing TLDs, we need to make sure there is a memory - * region of the X bytes within a page. This is due to the limit posed by UPTR: memory - * pinned to the kernel cannot exceed a page nor can it cross the page boundary. The - * library normally calls malloc(2*X) given X bytes of total TLDs, and only uses - * aligned_alloc(PAGE_SIZE, X) when X >= PAGE_SIZE / 2. This is to reduce memory wastage - * as not all memory allocator can use the exact amount of memory requested to fulfill - * aligned_alloc(). For example, some may round the size up to the alignment. Enable the - * option to always use aligned_alloc() if the implementation has low memory overhead. + * For some memory allocators, when calling aligned_alloc(alignment, size), size + * does not need to be an integral multiple of alignment and it can be fulfilled + * without using round_up(size, alignment) bytes of memory. Enable this option to + * reduce memory usage. */ #define TLD_PAGE_SIZE getpagesize() @@ -68,7 +68,7 @@ #define TLD_ROUND_MASK(x, y) ((__typeof__(x))((y) - 1)) #define TLD_ROUND_UP(x, y) ((((x) - 1) | TLD_ROUND_MASK(x, y)) + 1) -#define TLD_READ_ONCE(x) (*(volatile typeof(x) *)&(x)) +#define TLD_ROUND_UP_POWER_OF_TWO(x) (1UL << (sizeof(x) * 8 - __builtin_clzl(x - 1))) #ifndef TLD_DYN_DATA_SIZE #define TLD_DYN_DATA_SIZE 64 @@ -90,7 +90,7 @@ typedef struct { struct tld_metadata { char name[TLD_NAME_LEN]; - _Atomic __u16 size; + _Atomic __u16 size; /* size of tld_data_u->data */ }; struct tld_meta_u { @@ -101,7 +101,7 @@ struct tld_meta_u { struct tld_data_u { __u64 start; /* offset of tld_data_u->data in a page */ - char data[]; + char data[] __attribute__((aligned(8))); }; struct tld_map_value { @@ -111,9 +111,9 @@ struct tld_map_value { struct tld_meta_u * _Atomic tld_meta_p __attribute__((weak)); __thread struct tld_data_u *tld_data_p __attribute__((weak)); -__thread void *tld_data_alloc_p __attribute__((weak)); #ifdef TLD_FREE_DATA_ON_THREAD_EXIT +bool _Atomic tld_pthread_key_init __attribute__((weak)); pthread_key_t tld_pthread_key __attribute__((weak)); static void tld_free(void); @@ -144,20 +144,16 @@ static int __tld_init_meta_p(void) goto out; } -#ifdef TLD_FREE_DATA_ON_THREAD_EXIT - pthread_key_create(&tld_pthread_key, __tld_thread_exit_handler); -#endif out: return err; } static int __tld_init_data_p(int map_fd) { - bool use_aligned_alloc = false; struct tld_map_value map_val; struct tld_data_u *data; - void *data_alloc = NULL; int err, tid_fd = -1; + size_t size, size_pot; tid_fd = syscall(SYS_pidfd_open, sys_gettid(), O_EXCL); if (tid_fd < 0) { @@ -165,47 +161,37 @@ static int __tld_init_data_p(int map_fd) goto out; } -#ifdef TLD_DATA_USE_ALIGNED_ALLOC - use_aligned_alloc = true; -#endif - /* * tld_meta_p->size = TLD_DYN_DATA_SIZE + * total size of TLDs defined via TLD_DEFINE_KEY() */ - data_alloc = (use_aligned_alloc || tld_meta_p->size * 2 >= TLD_PAGE_SIZE) ? - aligned_alloc(TLD_PAGE_SIZE, tld_meta_p->size) : - malloc(tld_meta_p->size * 2); - if (!data_alloc) { + size = tld_meta_p->size + sizeof(struct tld_data_u); + size_pot = TLD_ROUND_UP_POWER_OF_TWO(size); +#ifdef TLD_DONT_ROUND_UP_DATA_SIZE + data = (struct tld_data_u *)aligned_alloc(size_pot, size); +#else + data = (struct tld_data_u *)aligned_alloc(size_pot, size_pot); +#endif + if (!data) { err = -ENOMEM; goto out; } /* * Always pass a page-aligned address to UPTR since the size of tld_map_value::data - * is a page in BTF. If data_alloc spans across two pages, use the page that contains large - * enough memory. + * is a page in BTF. */ - if (TLD_PAGE_SIZE - (~TLD_PAGE_MASK & (intptr_t)data_alloc) >= tld_meta_p->size) { - map_val.data = (void *)(TLD_PAGE_MASK & (intptr_t)data_alloc); - data = data_alloc; - data->start = (~TLD_PAGE_MASK & (intptr_t)data_alloc) + - offsetof(struct tld_data_u, data); - } else { - map_val.data = (void *)(TLD_ROUND_UP((intptr_t)data_alloc, TLD_PAGE_SIZE)); - data = (void *)(TLD_ROUND_UP((intptr_t)data_alloc, TLD_PAGE_SIZE)); - data->start = offsetof(struct tld_data_u, data); - } - map_val.meta = TLD_READ_ONCE(tld_meta_p); + map_val.data = (void *)(TLD_PAGE_MASK & (intptr_t)data); + data->start = (~TLD_PAGE_MASK & (intptr_t)data) + sizeof(struct tld_data_u); + map_val.meta = tld_meta_p; err = bpf_map_update_elem(map_fd, &tid_fd, &map_val, 0); if (err) { - free(data_alloc); + free(data); goto out; } tld_data_p = data; - tld_data_alloc_p = data_alloc; #ifdef TLD_FREE_DATA_ON_THREAD_EXIT pthread_setspecific(tld_pthread_key, (void *)1); #endif @@ -218,14 +204,23 @@ out: static tld_key_t __tld_create_key(const char *name, size_t size, bool dyn_data) { int err, i, sz, off = 0; + bool uninit = false; __u16 cnt; - if (!TLD_READ_ONCE(tld_meta_p)) { + if (!tld_meta_p) { err = __tld_init_meta_p(); if (err) return (tld_key_t){(__s16)err}; } +#ifdef TLD_FREE_DATA_ON_THREAD_EXIT + if (atomic_compare_exchange_strong(&tld_pthread_key_init, &uninit, true)) { + err = pthread_key_create(&tld_pthread_key, __tld_thread_exit_handler); + if (err) + return (tld_key_t){(__s16)err}; + } +#endif + for (i = 0; i < (int)TLD_MAX_DATA_CNT; i++) { retry: cnt = atomic_load(&tld_meta_p->cnt); @@ -291,7 +286,7 @@ retry: #define TLD_DEFINE_KEY(key, name, size) \ tld_key_t key; \ \ -__attribute__((constructor)) \ +__attribute__((constructor(101))) \ void __tld_define_key_##key(void) \ { \ key = __tld_create_key(name, size, false); \ @@ -351,7 +346,7 @@ static inline int tld_key_err_or_zero(tld_key_t key) __attribute__((unused)) static void *tld_get_data(int map_fd, tld_key_t key) { - if (!TLD_READ_ONCE(tld_meta_p)) + if (!tld_meta_p) return NULL; /* tld_data_p is allocated on the first invocation of tld_get_data() */ @@ -368,14 +363,14 @@ static void *tld_get_data(int map_fd, tld_key_t key) * * Users must call tld_free() on thread exit to prevent memory leak. Alternatively, * define TLD_FREE_DATA_ON_THREAD_EXIT and a thread exit handler will be registered - * to free the memory automatically. + * to free the memory automatically. Calling tld_free() before thread exit is + * undefined behavior, which may lead to null-pointer dereference. */ __attribute__((unused)) static void tld_free(void) { - if (tld_data_alloc_p) { - free(tld_data_alloc_p); - tld_data_alloc_p = NULL; + if (tld_data_p) { + free(tld_data_p); tld_data_p = NULL; } } diff --git a/tools/testing/selftests/bpf/prog_tests/test_task_local_data.c b/tools/testing/selftests/bpf/prog_tests/test_task_local_data.c index 9556ad3d986f..e219ff506b56 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_task_local_data.c +++ b/tools/testing/selftests/bpf/prog_tests/test_task_local_data.c @@ -26,7 +26,7 @@ TLD_DEFINE_KEY(value0_key, "value0", sizeof(int)); */ static void reset_tld(void) { - if (TLD_READ_ONCE(tld_meta_p)) { + if (tld_meta_p) { /* Remove TLDs created by tld_create_key() */ tld_meta_p->cnt = 1; tld_meta_p->size = TLD_DYN_DATA_SIZE; diff --git a/tools/testing/selftests/bpf/progs/task_local_data.bpf.h b/tools/testing/selftests/bpf/progs/task_local_data.bpf.h index 8b6f7af43648..1f396711f487 100644 --- a/tools/testing/selftests/bpf/progs/task_local_data.bpf.h +++ b/tools/testing/selftests/bpf/progs/task_local_data.bpf.h @@ -87,7 +87,7 @@ struct tld_meta_u { struct tld_data_u { __u64 start; /* offset of tld_data_u->data in a page */ - char data[__PAGE_SIZE - sizeof(__u64)]; + char data[__PAGE_SIZE - sizeof(__u64)] __attribute__((aligned(8))); }; struct tld_map_value { |