From 24ef659a857c3cba40b64ea51ea4fce8d2fb7bbc Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 28 Oct 2013 09:22:24 -0700 Subject: rcu: Provide better diagnostics for blocking in RCU callback functions Currently blocking in an RCU callback function will result in "scheduling while atomic", which could be triggered for any number of reasons. To aid debugging, this patch introduces a rcu_callback_map that is used to tie the inappropriate voluntary context switch back to the fact that the function is being invoked from within a callback. Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux/rcupdate.h') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 39cbb889e20d..a94a5805d378 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -325,6 +325,7 @@ static inline void rcu_lock_release(struct lockdep_map *map) extern struct lockdep_map rcu_lock_map; extern struct lockdep_map rcu_bh_lock_map; extern struct lockdep_map rcu_sched_lock_map; +extern struct lockdep_map rcu_callback_map; extern int debug_lockdep_rcu_enabled(void); /** -- cgit v1.2.3 From ac7c8e3dd2ed6489be1f6bc722029fb4855c3a34 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 11 Oct 2013 10:17:39 -0700 Subject: rcu: Add comment on evaluate-once properties of rcu_assign_pointer(). The rcu_assign_pointer() macro, as with most cpp macros, must not evaluate its argument more than once. And it in fact does not. But this might not be obvious to the casual observer, because one of the arguments appears no less than three times. However, but one expansion is only visible to sparse (__CHECKER__), and one lives inside a typeof (where it will never be evaluated), so this is in fact safe. This commit therefore adds a comment making this explicit. Reported-by: Josh Triplett Signed-off-by: Paul E. McKenney Reviewed-by: Josh Triplett --- include/linux/rcupdate.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'include/linux/rcupdate.h') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 39cbb889e20d..00ad28168ef0 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -911,6 +911,14 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) * rcu_assign_pointer() is a very bad thing that results in * impossible-to-diagnose memory corruption. So please be careful. * See the RCU_INIT_POINTER() comment header for details. + * + * Note that rcu_assign_pointer() evaluates each of its arguments only + * once, appearances notwithstanding. One of the "extra" evaluations + * is in typeof() and the other visible only to sparse (__CHECKER__), + * neither of which actually execute the argument. As with most cpp + * macros, this execute-arguments-only-once property is important, so + * please be careful when making changes to rcu_assign_pointer() and the + * other macros that it invokes. */ #define rcu_assign_pointer(p, v) \ __rcu_assign_pointer((p), (v), __rcu) -- cgit v1.2.3 From 9d162cd06349dfee6b4f254b3abf1355cf0aee43 Mon Sep 17 00:00:00 2001 From: Josh Triplett Date: Sun, 1 Sep 2013 16:42:52 -0700 Subject: rcu: Make rcu_assign_pointer's assignment volatile and type-safe The rcu_assign_pointer() primitive needs to use ACCESS_ONCE to make the assignment to the destination pointer volatile, to protect against compilers too clever for their own good. Signed-off-by: Josh Triplett Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux/rcupdate.h') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 00ad28168ef0..97853cd2d7b4 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -551,7 +551,7 @@ static inline void rcu_preempt_sleep_check(void) #define __rcu_assign_pointer(p, v, space) \ do { \ smp_wmb(); \ - (p) = (typeof(*v) __force space *)(v); \ + ACCESS_ONCE(p) = (typeof(*(v)) __force space *)(v); \ } while (0) -- cgit v1.2.3 From 462225ae47d7175f886281d8a91708550cd5178c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 11 Nov 2013 09:59:34 -0800 Subject: rcu: Add an RCU_INITIALIZER for global RCU-protected pointers There is currently no way to initialize a global RCU-protected pointer without either putting up with sparse complaints or open-coding an obscure cast. This commit therefore creates RCU_INITIALIZER(), which is intended to be used as follows: struct foo __rcu *p = RCU_INITIALIZER(&my_rcu_structure); This commit also applies RCU_INITIALIZER() to eliminate repeated open-coded obscure casts in __rcu_assign_pointer(), RCU_INIT_POINTER(), and RCU_POINTER_INITIALIZER(). This commit also inlines __rcu_assign_pointer() into its only caller, rcu_assign_pointer(). Suggested-by: Steven Rostedt Signed-off-by: Paul E. McKenney Reviewed-by: Josh Triplett --- include/linux/rcupdate.h | 80 +++++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 38 deletions(-) (limited to 'include/linux/rcupdate.h') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 97853cd2d7b4..ea3816cf1a13 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -548,10 +548,48 @@ static inline void rcu_preempt_sleep_check(void) smp_read_barrier_depends(); \ (_________p1); \ }) -#define __rcu_assign_pointer(p, v, space) \ + +/** + * RCU_INITIALIZER() - statically initialize an RCU-protected global variable + * @v: The value to statically initialize with. + */ +#define RCU_INITIALIZER(v) (typeof(*(v)) __force __rcu *)(v) + +/** + * rcu_assign_pointer() - assign to RCU-protected pointer + * @p: pointer to assign to + * @v: value to assign (publish) + * + * Assigns the specified value to the specified RCU-protected + * pointer, ensuring that any concurrent RCU readers will see + * any prior initialization. + * + * Inserts memory barriers on architectures that require them + * (which is most of them), and also prevents the compiler from + * reordering the code that initializes the structure after the pointer + * assignment. More importantly, this call documents which pointers + * will be dereferenced by RCU read-side code. + * + * In some special cases, you may use RCU_INIT_POINTER() instead + * of rcu_assign_pointer(). RCU_INIT_POINTER() is a bit faster due + * to the fact that it does not constrain either the CPU or the compiler. + * That said, using RCU_INIT_POINTER() when you should have used + * rcu_assign_pointer() is a very bad thing that results in + * impossible-to-diagnose memory corruption. So please be careful. + * See the RCU_INIT_POINTER() comment header for details. + * + * Note that rcu_assign_pointer() evaluates each of its arguments only + * once, appearances notwithstanding. One of the "extra" evaluations + * is in typeof() and the other visible only to sparse (__CHECKER__), + * neither of which actually execute the argument. As with most cpp + * macros, this execute-arguments-only-once property is important, so + * please be careful when making changes to rcu_assign_pointer() and the + * other macros that it invokes. + */ +#define rcu_assign_pointer(p, v) \ do { \ smp_wmb(); \ - ACCESS_ONCE(p) = (typeof(*(v)) __force space *)(v); \ + ACCESS_ONCE(p) = RCU_INITIALIZER(v); \ } while (0) @@ -889,40 +927,6 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) preempt_enable_notrace(); } -/** - * rcu_assign_pointer() - assign to RCU-protected pointer - * @p: pointer to assign to - * @v: value to assign (publish) - * - * Assigns the specified value to the specified RCU-protected - * pointer, ensuring that any concurrent RCU readers will see - * any prior initialization. - * - * Inserts memory barriers on architectures that require them - * (which is most of them), and also prevents the compiler from - * reordering the code that initializes the structure after the pointer - * assignment. More importantly, this call documents which pointers - * will be dereferenced by RCU read-side code. - * - * In some special cases, you may use RCU_INIT_POINTER() instead - * of rcu_assign_pointer(). RCU_INIT_POINTER() is a bit faster due - * to the fact that it does not constrain either the CPU or the compiler. - * That said, using RCU_INIT_POINTER() when you should have used - * rcu_assign_pointer() is a very bad thing that results in - * impossible-to-diagnose memory corruption. So please be careful. - * See the RCU_INIT_POINTER() comment header for details. - * - * Note that rcu_assign_pointer() evaluates each of its arguments only - * once, appearances notwithstanding. One of the "extra" evaluations - * is in typeof() and the other visible only to sparse (__CHECKER__), - * neither of which actually execute the argument. As with most cpp - * macros, this execute-arguments-only-once property is important, so - * please be careful when making changes to rcu_assign_pointer() and the - * other macros that it invokes. - */ -#define rcu_assign_pointer(p, v) \ - __rcu_assign_pointer((p), (v), __rcu) - /** * RCU_INIT_POINTER() - initialize an RCU protected pointer * @@ -957,7 +961,7 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) */ #define RCU_INIT_POINTER(p, v) \ do { \ - p = (typeof(*v) __force __rcu *)(v); \ + p = RCU_INITIALIZER(v); \ } while (0) /** @@ -966,7 +970,7 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) * GCC-style initialization for an RCU-protected pointer in a structure field. */ #define RCU_POINTER_INITIALIZER(p, v) \ - .p = (typeof(*v) __force __rcu *)(v) + .p = RCU_INITIALIZER(v) /* * Does the specified offset indicate that the corresponding rcu_head -- cgit v1.2.3 From 584dc4ce55267765b415a8517613d1207f1741e5 Mon Sep 17 00:00:00 2001 From: Teodora Baluta Date: Mon, 11 Nov 2013 17:11:23 +0200 Subject: rcu: Remove "extern" from function declarations in include/linux/*rcu*.h Function prototypes don't need to have the "extern" keyword since this is the default behavior. Its explicit use is redundant. This commit therefore removes them. Signed-off-by: Teodora Baluta Signed-off-by: Paul E. McKenney --- include/linux/rculist.h | 4 +-- include/linux/rcupdate.h | 80 ++++++++++++++++++++++++------------------------ include/linux/rcutiny.h | 2 +- include/linux/rcutree.h | 36 +++++++++++----------- 4 files changed, 61 insertions(+), 61 deletions(-) (limited to 'include/linux/rcupdate.h') diff --git a/include/linux/rculist.h b/include/linux/rculist.h index 45a0a9e81478..dbaf99084112 100644 --- a/include/linux/rculist.h +++ b/include/linux/rculist.h @@ -55,8 +55,8 @@ static inline void __list_add_rcu(struct list_head *new, next->prev = new; } #else -extern void __list_add_rcu(struct list_head *new, - struct list_head *prev, struct list_head *next); +void __list_add_rcu(struct list_head *new, + struct list_head *prev, struct list_head *next); #endif /** diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index a94a5805d378..52c1b13c4d76 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -50,13 +50,13 @@ extern int rcutorture_runnable; /* for sysctl */ #endif /* #ifdef CONFIG_RCU_TORTURE_TEST */ #if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) -extern void rcutorture_record_test_transition(void); -extern void rcutorture_record_progress(unsigned long vernum); -extern void do_trace_rcu_torture_read(const char *rcutorturename, - struct rcu_head *rhp, - unsigned long secs, - unsigned long c_old, - unsigned long c); +void rcutorture_record_test_transition(void); +void rcutorture_record_progress(unsigned long vernum); +void do_trace_rcu_torture_read(const char *rcutorturename, + struct rcu_head *rhp, + unsigned long secs, + unsigned long c_old, + unsigned long c); #else static inline void rcutorture_record_test_transition(void) { @@ -65,11 +65,11 @@ static inline void rcutorture_record_progress(unsigned long vernum) { } #ifdef CONFIG_RCU_TRACE -extern void do_trace_rcu_torture_read(const char *rcutorturename, - struct rcu_head *rhp, - unsigned long secs, - unsigned long c_old, - unsigned long c); +void do_trace_rcu_torture_read(const char *rcutorturename, + struct rcu_head *rhp, + unsigned long secs, + unsigned long c_old, + unsigned long c); #else #define do_trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \ do { } while (0) @@ -118,8 +118,8 @@ extern void do_trace_rcu_torture_read(const char *rcutorturename, * if CPU A and CPU B are the same CPU (but again only if the system has * more than one CPU). */ -extern void call_rcu(struct rcu_head *head, - void (*func)(struct rcu_head *head)); +void call_rcu(struct rcu_head *head, + void (*func)(struct rcu_head *head)); #else /* #ifdef CONFIG_PREEMPT_RCU */ @@ -149,8 +149,8 @@ extern void call_rcu(struct rcu_head *head, * See the description of call_rcu() for more detailed information on * memory ordering guarantees. */ -extern void call_rcu_bh(struct rcu_head *head, - void (*func)(struct rcu_head *head)); +void call_rcu_bh(struct rcu_head *head, + void (*func)(struct rcu_head *head)); /** * call_rcu_sched() - Queue an RCU for invocation after sched grace period. @@ -171,16 +171,16 @@ extern void call_rcu_bh(struct rcu_head *head, * See the description of call_rcu() for more detailed information on * memory ordering guarantees. */ -extern void call_rcu_sched(struct rcu_head *head, - void (*func)(struct rcu_head *rcu)); +void call_rcu_sched(struct rcu_head *head, + void (*func)(struct rcu_head *rcu)); -extern void synchronize_sched(void); +void synchronize_sched(void); #ifdef CONFIG_PREEMPT_RCU -extern void __rcu_read_lock(void); -extern void __rcu_read_unlock(void); -extern void rcu_read_unlock_special(struct task_struct *t); +void __rcu_read_lock(void); +void __rcu_read_unlock(void); +void rcu_read_unlock_special(struct task_struct *t); void synchronize_rcu(void); /* @@ -216,19 +216,19 @@ static inline int rcu_preempt_depth(void) #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ /* Internal to kernel */ -extern void rcu_init(void); -extern void rcu_sched_qs(int cpu); -extern void rcu_bh_qs(int cpu); -extern void rcu_check_callbacks(int cpu, int user); +void rcu_init(void); +void rcu_sched_qs(int cpu); +void rcu_bh_qs(int cpu); +void rcu_check_callbacks(int cpu, int user); struct notifier_block; -extern void rcu_idle_enter(void); -extern void rcu_idle_exit(void); -extern void rcu_irq_enter(void); -extern void rcu_irq_exit(void); +void rcu_idle_enter(void); +void rcu_idle_exit(void); +void rcu_irq_enter(void); +void rcu_irq_exit(void); #ifdef CONFIG_RCU_USER_QS -extern void rcu_user_enter(void); -extern void rcu_user_exit(void); +void rcu_user_enter(void); +void rcu_user_exit(void); #else static inline void rcu_user_enter(void) { } static inline void rcu_user_exit(void) { } @@ -262,7 +262,7 @@ static inline void rcu_user_hooks_switch(struct task_struct *prev, } while (0) #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) || defined(CONFIG_SMP) -extern bool __rcu_is_watching(void); +bool __rcu_is_watching(void); #endif /* #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) || defined(CONFIG_SMP) */ /* @@ -289,8 +289,8 @@ void wait_rcu_gp(call_rcu_func_t crf); * initialization. */ #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD -extern void init_rcu_head_on_stack(struct rcu_head *head); -extern void destroy_rcu_head_on_stack(struct rcu_head *head); +void init_rcu_head_on_stack(struct rcu_head *head); +void destroy_rcu_head_on_stack(struct rcu_head *head); #else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */ static inline void init_rcu_head_on_stack(struct rcu_head *head) { @@ -363,7 +363,7 @@ static inline int rcu_read_lock_held(void) * rcu_read_lock_bh_held() is defined out of line to avoid #include-file * hell. */ -extern int rcu_read_lock_bh_held(void); +int rcu_read_lock_bh_held(void); /** * rcu_read_lock_sched_held() - might we be in RCU-sched read-side critical section? @@ -449,7 +449,7 @@ static inline int rcu_read_lock_sched_held(void) #ifdef CONFIG_PROVE_RCU -extern int rcu_my_thread_group_empty(void); +int rcu_my_thread_group_empty(void); /** * rcu_lockdep_assert - emit lockdep splat if specified condition not met @@ -1006,7 +1006,7 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) __kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head)) #ifdef CONFIG_RCU_NOCB_CPU -extern bool rcu_is_nocb_cpu(int cpu); +bool rcu_is_nocb_cpu(int cpu); #else static inline bool rcu_is_nocb_cpu(int cpu) { return false; } #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */ @@ -1014,8 +1014,8 @@ static inline bool rcu_is_nocb_cpu(int cpu) { return false; } /* Only for use by adaptive-ticks code. */ #ifdef CONFIG_NO_HZ_FULL_SYSIDLE -extern bool rcu_sys_is_idle(void); -extern void rcu_sysidle_force_exit(void); +bool rcu_sys_is_idle(void); +void rcu_sysidle_force_exit(void); #else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */ static inline bool rcu_sys_is_idle(void) diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index 09ebcbe9fd78..6f01771b571c 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -125,7 +125,7 @@ static inline void exit_rcu(void) #ifdef CONFIG_DEBUG_LOCK_ALLOC extern int rcu_scheduler_active __read_mostly; -extern void rcu_scheduler_starting(void); +void rcu_scheduler_starting(void); #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ static inline void rcu_scheduler_starting(void) { diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 4b9c81548742..72137ee8c603 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -30,9 +30,9 @@ #ifndef __LINUX_RCUTREE_H #define __LINUX_RCUTREE_H -extern void rcu_note_context_switch(int cpu); -extern int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies); -extern void rcu_cpu_stall_reset(void); +void rcu_note_context_switch(int cpu); +int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies); +void rcu_cpu_stall_reset(void); /* * Note a virtualization-based context switch. This is simply a @@ -44,9 +44,9 @@ static inline void rcu_virt_note_context_switch(int cpu) rcu_note_context_switch(cpu); } -extern void synchronize_rcu_bh(void); -extern void synchronize_sched_expedited(void); -extern void synchronize_rcu_expedited(void); +void synchronize_rcu_bh(void); +void synchronize_sched_expedited(void); +void synchronize_rcu_expedited(void); void kfree_call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu)); @@ -71,25 +71,25 @@ static inline void synchronize_rcu_bh_expedited(void) synchronize_sched_expedited(); } -extern void rcu_barrier(void); -extern void rcu_barrier_bh(void); -extern void rcu_barrier_sched(void); +void rcu_barrier(void); +void rcu_barrier_bh(void); +void rcu_barrier_sched(void); extern unsigned long rcutorture_testseq; extern unsigned long rcutorture_vernum; -extern long rcu_batches_completed(void); -extern long rcu_batches_completed_bh(void); -extern long rcu_batches_completed_sched(void); +long rcu_batches_completed(void); +long rcu_batches_completed_bh(void); +long rcu_batches_completed_sched(void); -extern void rcu_force_quiescent_state(void); -extern void rcu_bh_force_quiescent_state(void); -extern void rcu_sched_force_quiescent_state(void); +void rcu_force_quiescent_state(void); +void rcu_bh_force_quiescent_state(void); +void rcu_sched_force_quiescent_state(void); -extern void exit_rcu(void); +void exit_rcu(void); -extern void rcu_scheduler_starting(void); +void rcu_scheduler_starting(void); extern int rcu_scheduler_active __read_mostly; -extern bool rcu_is_watching(void); +bool rcu_is_watching(void); #endif /* __LINUX_RCUTREE_H */ -- cgit v1.2.3