diff options
Diffstat (limited to 'include')
| -rw-r--r-- | include/linux/sched.h | 8 | ||||
| -rw-r--r-- | include/linux/sched/mm.h | 35 | ||||
| -rw-r--r-- | include/linux/sync_core.h | 21 | ||||
| -rw-r--r-- | include/uapi/linux/membarrier.h | 74 |
4 files changed, 128 insertions, 10 deletions
diff --git a/include/linux/sched.h b/include/linux/sched.h index 166144c04ef6..92744e3f1556 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -555,6 +555,14 @@ struct task_struct { unsigned long wakee_flip_decay_ts; struct task_struct *last_wakee; + /* + * recent_used_cpu is initially set as the last CPU used by a task + * that wakes affine another task. Waker/wakee relationships can + * push tasks around a CPU where each wakeup moves to the next one. + * Tracking a recently used CPU allows a quick search for a recently + * used CPU that may be idle. + */ + int recent_used_cpu; int wake_cpu; #endif int on_rq; diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index bd422561a75e..1149533aa2fa 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -7,6 +7,7 @@ #include <linux/sched.h> #include <linux/mm_types.h> #include <linux/gfp.h> +#include <linux/sync_core.h> /* * Routines for handling mm_structs @@ -194,18 +195,48 @@ static inline void memalloc_noreclaim_restore(unsigned int flags) #ifdef CONFIG_MEMBARRIER enum { - MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY = (1U << 0), - MEMBARRIER_STATE_SWITCH_MM = (1U << 1), + MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY = (1U << 0), + MEMBARRIER_STATE_PRIVATE_EXPEDITED = (1U << 1), + MEMBARRIER_STATE_GLOBAL_EXPEDITED_READY = (1U << 2), + MEMBARRIER_STATE_GLOBAL_EXPEDITED = (1U << 3), + MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY = (1U << 4), + MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE = (1U << 5), }; +enum { + MEMBARRIER_FLAG_SYNC_CORE = (1U << 0), +}; + +#ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS +#include <asm/membarrier.h> +#endif + +static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm) +{ + if (likely(!(atomic_read(&mm->membarrier_state) & + MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) + return; + sync_core_before_usermode(); +} + static inline void membarrier_execve(struct task_struct *t) { atomic_set(&t->mm->membarrier_state, 0); } #else +#ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS +static inline void membarrier_arch_switch_mm(struct mm_struct *prev, + struct mm_struct *next, + struct task_struct *tsk) +{ +} +#endif static inline void membarrier_execve(struct task_struct *t) { } +static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm) +{ +} #endif #endif /* _LINUX_SCHED_MM_H */ diff --git a/include/linux/sync_core.h b/include/linux/sync_core.h new file mode 100644 index 000000000000..013da4b8b327 --- /dev/null +++ b/include/linux/sync_core.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_SYNC_CORE_H +#define _LINUX_SYNC_CORE_H + +#ifdef CONFIG_ARCH_HAS_SYNC_CORE_BEFORE_USERMODE +#include <asm/sync_core.h> +#else +/* + * This is a dummy sync_core_before_usermode() implementation that can be used + * on all architectures which return to user-space through core serializing + * instructions. + * If your architecture returns to user-space through non-core-serializing + * instructions, you need to write your own functions. + */ +static inline void sync_core_before_usermode(void) +{ +} +#endif + +#endif /* _LINUX_SYNC_CORE_H */ + diff --git a/include/uapi/linux/membarrier.h b/include/uapi/linux/membarrier.h index 4e01ad7ffe98..5891d7614c8c 100644 --- a/include/uapi/linux/membarrier.h +++ b/include/uapi/linux/membarrier.h @@ -31,7 +31,7 @@ * enum membarrier_cmd - membarrier system call command * @MEMBARRIER_CMD_QUERY: Query the set of supported commands. It returns * a bitmask of valid commands. - * @MEMBARRIER_CMD_SHARED: Execute a memory barrier on all running threads. + * @MEMBARRIER_CMD_GLOBAL: Execute a memory barrier on all running threads. * Upon return from system call, the caller thread * is ensured that all running threads have passed * through a state where all memory accesses to @@ -40,6 +40,28 @@ * (non-running threads are de facto in such a * state). This covers threads from all processes * running on the system. This command returns 0. + * @MEMBARRIER_CMD_GLOBAL_EXPEDITED: + * Execute a memory barrier on all running threads + * of all processes which previously registered + * with MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED. + * Upon return from system call, the caller thread + * is ensured that all running threads have passed + * through a state where all memory accesses to + * user-space addresses match program order between + * entry to and return from the system call + * (non-running threads are de facto in such a + * state). This only covers threads from processes + * which registered with + * MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED. + * This command returns 0. Given that + * registration is about the intent to receive + * the barriers, it is valid to invoke + * MEMBARRIER_CMD_GLOBAL_EXPEDITED from a + * non-registered process. + * @MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED: + * Register the process intent to receive + * MEMBARRIER_CMD_GLOBAL_EXPEDITED memory + * barriers. Always returns 0. * @MEMBARRIER_CMD_PRIVATE_EXPEDITED: * Execute a memory barrier on each running * thread belonging to the same process as the current @@ -51,7 +73,7 @@ * to and return from the system call * (non-running threads are de facto in such a * state). This only covers threads from the - * same processes as the caller thread. This + * same process as the caller thread. This * command returns 0 on success. The * "expedited" commands complete faster than * the non-expedited ones, they never block, @@ -64,18 +86,54 @@ * Register the process intent to use * MEMBARRIER_CMD_PRIVATE_EXPEDITED. Always * returns 0. + * @MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE: + * In addition to provide memory ordering + * guarantees described in + * MEMBARRIER_CMD_PRIVATE_EXPEDITED, ensure + * the caller thread, upon return from system + * call, that all its running threads siblings + * have executed a core serializing + * instruction. (architectures are required to + * guarantee that non-running threads issue + * core serializing instructions before they + * resume user-space execution). This only + * covers threads from the same process as the + * caller thread. This command returns 0 on + * success. The "expedited" commands complete + * faster than the non-expedited ones, they + * never block, but have the downside of + * causing extra overhead. If this command is + * not implemented by an architecture, -EINVAL + * is returned. A process needs to register its + * intent to use the private expedited sync + * core command prior to using it, otherwise + * this command returns -EPERM. + * @MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE: + * Register the process intent to use + * MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE. + * If this command is not implemented by an + * architecture, -EINVAL is returned. + * Returns 0 on success. + * @MEMBARRIER_CMD_SHARED: + * Alias to MEMBARRIER_CMD_GLOBAL. Provided for + * header backward compatibility. * * Command to be passed to the membarrier system call. The commands need to * be a single bit each, except for MEMBARRIER_CMD_QUERY which is assigned to * the value 0. */ enum membarrier_cmd { - MEMBARRIER_CMD_QUERY = 0, - MEMBARRIER_CMD_SHARED = (1 << 0), - /* reserved for MEMBARRIER_CMD_SHARED_EXPEDITED (1 << 1) */ - /* reserved for MEMBARRIER_CMD_PRIVATE (1 << 2) */ - MEMBARRIER_CMD_PRIVATE_EXPEDITED = (1 << 3), - MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED = (1 << 4), + MEMBARRIER_CMD_QUERY = 0, + MEMBARRIER_CMD_GLOBAL = (1 << 0), + MEMBARRIER_CMD_GLOBAL_EXPEDITED = (1 << 1), + MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED = (1 << 2), + MEMBARRIER_CMD_PRIVATE_EXPEDITED = (1 << 3), + MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED = (1 << 4), + MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE = (1 << 5), + MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE = (1 << 6), + + /* Alias for header backward compatibility. */ + MEMBARRIER_CMD_SHARED = MEMBARRIER_CMD_GLOBAL, }; #endif /* _UAPI_LINUX_MEMBARRIER_H */ |