diff options
| -rw-r--r-- | include/linux/tracepoint.h | 9 | ||||
| -rw-r--r-- | include/trace/trace_events.h | 4 | ||||
| -rw-r--r-- | kernel/tracepoint.c | 18 |
3 files changed, 21 insertions, 10 deletions
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h index 8a56f3278b1b..22ca1c8b54f3 100644 --- a/include/linux/tracepoint.h +++ b/include/linux/tracepoint.h @@ -108,14 +108,15 @@ void for_each_tracepoint_in_module(struct module *mod, * An alternative is to use the following for batch reclaim associated * with a given tracepoint: * - * - tracepoint_is_faultable() == false: call_rcu() + * - tracepoint_is_faultable() == false: call_srcu() * - tracepoint_is_faultable() == true: call_rcu_tasks_trace() */ #ifdef CONFIG_TRACEPOINTS +extern struct srcu_struct tracepoint_srcu; static inline void tracepoint_synchronize_unregister(void) { synchronize_rcu_tasks_trace(); - synchronize_rcu(); + synchronize_srcu(&tracepoint_srcu); } static inline bool tracepoint_is_faultable(struct tracepoint *tp) { @@ -275,13 +276,13 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) return static_branch_unlikely(&__tracepoint_##name.key);\ } -#define __DECLARE_TRACE(name, proto, args, cond, data_proto) \ +#define __DECLARE_TRACE(name, proto, args, cond, data_proto) \ __DECLARE_TRACE_COMMON(name, PARAMS(proto), PARAMS(args), PARAMS(data_proto)) \ static inline void __do_trace_##name(proto) \ { \ TRACEPOINT_CHECK(name) \ if (cond) { \ - guard(preempt_notrace)(); \ + guard(srcu_fast_notrace)(&tracepoint_srcu); \ __DO_TRACE_CALL(name, TP_ARGS(args)); \ } \ } \ diff --git a/include/trace/trace_events.h b/include/trace/trace_events.h index 4f22136fd465..fbc07d353be6 100644 --- a/include/trace/trace_events.h +++ b/include/trace/trace_events.h @@ -436,6 +436,7 @@ __DECLARE_EVENT_CLASS(call, PARAMS(proto), PARAMS(args), PARAMS(tstruct), \ static notrace void \ trace_event_raw_event_##call(void *__data, proto) \ { \ + guard(preempt_notrace)(); \ do_trace_event_raw_event_##call(__data, args); \ } @@ -447,9 +448,8 @@ static notrace void \ trace_event_raw_event_##call(void *__data, proto) \ { \ might_fault(); \ - preempt_disable_notrace(); \ + guard(preempt_notrace)(); \ do_trace_event_raw_event_##call(__data, args); \ - preempt_enable_notrace(); \ } /* diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c index 62719d2941c9..fd2ee879815c 100644 --- a/kernel/tracepoint.c +++ b/kernel/tracepoint.c @@ -34,9 +34,13 @@ enum tp_transition_sync { struct tp_transition_snapshot { unsigned long rcu; + unsigned long srcu_gp; bool ongoing; }; +DEFINE_SRCU_FAST(tracepoint_srcu); +EXPORT_SYMBOL_GPL(tracepoint_srcu); + /* Protected by tracepoints_mutex */ static struct tp_transition_snapshot tp_transition_snapshot[_NR_TP_TRANSITION_SYNC]; @@ -46,6 +50,7 @@ static void tp_rcu_get_state(enum tp_transition_sync sync) /* Keep the latest get_state snapshot. */ snapshot->rcu = get_state_synchronize_rcu(); + snapshot->srcu_gp = start_poll_synchronize_srcu(&tracepoint_srcu); snapshot->ongoing = true; } @@ -56,6 +61,8 @@ static void tp_rcu_cond_sync(enum tp_transition_sync sync) if (!snapshot->ongoing) return; cond_synchronize_rcu(snapshot->rcu); + if (!poll_state_synchronize_srcu(&tracepoint_srcu, snapshot->srcu_gp)) + synchronize_srcu(&tracepoint_srcu); snapshot->ongoing = false; } @@ -112,10 +119,13 @@ static inline void release_probes(struct tracepoint *tp, struct tracepoint_func struct tp_probes *tp_probes = container_of(old, struct tp_probes, probes[0]); - if (tracepoint_is_faultable(tp)) - call_rcu_tasks_trace(&tp_probes->rcu, rcu_free_old_probes); - else - call_rcu(&tp_probes->rcu, rcu_free_old_probes); + if (tracepoint_is_faultable(tp)) { + call_rcu_tasks_trace(&tp_probes->rcu, + rcu_free_old_probes); + } else { + call_srcu(&tracepoint_srcu, &tp_probes->rcu, + rcu_free_old_probes); + } } } |