From cf6444c3e1bb7dd5974441bbd74840e9821d36f9 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:39 +0200 Subject: tick/sched: Unify idle cputime accounting The non-vtime dynticks-idle cputime accounting is a big mess that accumulates within two concurrent statistics, each having their own shortcomings: * The accounting for online CPUs which is based on the delta between tick_nohz_start_idle() and tick_nohz_stop_idle(). Pros: - Works when the tick is off - Has nsecs granularity Cons: - Account idle steal time but doesn't substract it from idle cputime. - Assumes CONFIG_IRQ_TIME_ACCOUNTING by not accounting IRQs but the IRQ time is simply ignored when CONFIG_IRQ_TIME_ACCOUNTING=n - The windows between 1) idle task scheduling and the first call to tick_nohz_start_idle() and 2) idle task between the last tick_nohz_stop_idle() and the rest of the idle time are blindspots wrt. cputime accounting (though mostly insignificant amount) - Relies on private fields outside of kernel stats, with specific accessors. * The accounting for offline CPUs which is based on ticks and the jiffies delta during which the tick was stopped. Pros: - Handles steal time correctly - Handle CONFIG_IRQ_TIME_ACCOUNTING=y and CONFIG_IRQ_TIME_ACCOUNTING=n correctly. - Handles the whole idle task - Accounts directly to kernel stats, without midlayer accumulator. Cons: - Doesn't elapse when the tick is off, which doesn't make it suitable for online CPUs. - Has TICK_NSEC granularity (jiffies) - Needs to track the dyntick-idle ticks that were accounted and substract them from the total jiffies time spent while the tick was stopped. This is an ugly workaround. Having two different accounting for a single context is not the only problem: since those accountings are of different natures, it is possible to observe the global idle time going backward after a CPU goes offline. Clean up the situation with introducing a hybrid approach that stays coherent and works for both online and offline CPUs: * Tick based or native vtime accounting operate before the idle loop is entered and resume once the idle loop prepares to exit. * When the idle loop starts, switch to dynticks-idle accounting as is done currently, except that the statistics accumulate directly to the relevant kernel stat fields. * Private dyntick cputime accounting fields are removed. * Works on both online and offline case. Further improvement will include: * Only switch to dynticks-idle cputime accounting when the tick actually goes in dynticks mode. * Handle CONFIG_IRQ_TIME_ACCOUNTING=n correctly such that the dynticks-idle accounting still elapses while on IRQs. * Correctly substract idle steal cputime from idle time Reported-by: Xin Zhao Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-8-frederic@kernel.org --- include/linux/kernel_stat.h | 24 ++++++++++++++++++++---- include/linux/vtime.h | 7 ++++++- 2 files changed, 26 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index dd020ecaf67b..ba65aad308a1 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -34,6 +34,9 @@ enum cpu_usage_stat { }; struct kernel_cpustat { +#ifdef CONFIG_NO_HZ_COMMON + int idle_dyntick; +#endif u64 cpustat[NR_STATS]; }; @@ -99,6 +102,20 @@ static inline unsigned long kstat_cpu_irqs_sum(unsigned int cpu) return kstat_cpu(cpu).irqs_sum; } +#ifdef CONFIG_NO_HZ_COMMON +extern void kcpustat_dyntick_start(void); +extern void kcpustat_dyntick_stop(void); +static inline bool kcpustat_idle_dyntick(void) +{ + return __this_cpu_read(kernel_cpustat.idle_dyntick); +} +#else +static inline bool kcpustat_idle_dyntick(void) +{ + return false; +} +#endif /* CONFIG_NO_HZ_COMMON */ + #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN extern u64 kcpustat_field(enum cpu_usage_stat usage, int cpu); extern void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu); @@ -113,7 +130,7 @@ static inline void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu) *dst = kcpustat_cpu(cpu); } -#endif +#endif /* !CONFIG_VIRT_CPU_ACCOUNTING_GEN */ extern void account_user_time(struct task_struct *, u64); extern void account_guest_time(struct task_struct *, u64); @@ -127,14 +144,13 @@ extern u64 get_idle_time(struct kernel_cpustat *kcs, int cpu); #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE static inline void account_process_tick(struct task_struct *tsk, int user) { - vtime_flush(tsk); + if (!kcpustat_idle_dyntick()) + vtime_flush(tsk); } #else extern void account_process_tick(struct task_struct *, int user); #endif -extern void account_idle_ticks(unsigned long ticks); - #ifdef CONFIG_SCHED_CORE extern void __account_forceidle_time(struct task_struct *tsk, u64 delta); #endif diff --git a/include/linux/vtime.h b/include/linux/vtime.h index 5516ad549e60..9dc25b04a119 100644 --- a/include/linux/vtime.h +++ b/include/linux/vtime.h @@ -31,6 +31,11 @@ static inline bool vtime_generic_enabled_cpu(int cpu) return context_tracking_enabled_cpu(cpu); } +static inline bool vtime_generic_enabled_this_cpu(void) +{ + return context_tracking_enabled_this_cpu(); +} + #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE extern void vtime_account_idle(struct task_struct *tsk); extern void vtime_account_irq(struct task_struct *tsk, unsigned int offset); @@ -90,7 +95,7 @@ static inline bool vtime_accounting_enabled_cpu(int cpu) static inline bool vtime_accounting_enabled_this_cpu(void) { - return context_tracking_enabled_this_cpu(); + return vtime_generic_enabled_this_cpu(); } extern void vtime_task_switch_generic(struct task_struct *prev); -- cgit v1.2.3