diff options
| -rw-r--r-- | arch/ia64/include/asm/cputime.h | 2 | ||||
| -rw-r--r-- | arch/ia64/kernel/time.c | 26 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/cputime.h | 2 | ||||
| -rw-r--r-- | arch/powerpc/kernel/time.c | 24 | ||||
| -rw-r--r-- | arch/s390/include/asm/cputime.h | 1 | ||||
| -rw-r--r-- | arch/s390/kernel/vtime.c | 13 | ||||
| -rw-r--r-- | include/linux/kernel_stat.h | 8 | ||||
| -rw-r--r-- | include/linux/kvm_host.h | 4 | ||||
| -rw-r--r-- | include/linux/vtime.h | 9 | ||||
| -rw-r--r-- | kernel/sched/cputime.c | 31 |
10 files changed, 65 insertions, 55 deletions
diff --git a/arch/ia64/include/asm/cputime.h b/arch/ia64/include/asm/cputime.h index 3deac956d325..7fcf7f08ab06 100644 --- a/arch/ia64/include/asm/cputime.h +++ b/arch/ia64/include/asm/cputime.h @@ -103,5 +103,7 @@ static inline void cputime_to_timeval(const cputime_t ct, struct timeval *val) #define cputime64_to_clock_t(__ct) \ cputime_to_clock_t((__force cputime_t)__ct) +extern void arch_vtime_task_switch(struct task_struct *tsk); + #endif /* CONFIG_VIRT_CPU_ACCOUNTING */ #endif /* __IA64_CPUTIME_H */ diff --git a/arch/ia64/kernel/time.c b/arch/ia64/kernel/time.c index 5e4850305d3f..b1995efbfd21 100644 --- a/arch/ia64/kernel/time.c +++ b/arch/ia64/kernel/time.c @@ -83,7 +83,7 @@ static struct clocksource *itc_clocksource; extern cputime_t cycle_to_cputime(u64 cyc); -static void vtime_account_user(struct task_struct *tsk) +void vtime_account_user(struct task_struct *tsk) { cputime_t delta_utime; struct thread_info *ti = task_thread_info(tsk); @@ -100,18 +100,11 @@ static void vtime_account_user(struct task_struct *tsk) * accumulated times to the current process, and to prepare accounting on * the next process. */ -void vtime_task_switch(struct task_struct *prev) +void arch_vtime_task_switch(struct task_struct *prev) { struct thread_info *pi = task_thread_info(prev); struct thread_info *ni = task_thread_info(current); - if (idle_task(smp_processor_id()) != prev) - __vtime_account_system(prev); - else - __vtime_account_idle(prev); - - vtime_account_user(prev); - pi->ac_stamp = ni->ac_stamp; ni->ac_stime = ni->ac_utime = 0; } @@ -126,6 +119,8 @@ static cputime_t vtime_delta(struct task_struct *tsk) cputime_t delta_stime; __u64 now; + WARN_ON_ONCE(!irqs_disabled()); + now = ia64_get_itc(); delta_stime = cycle_to_cputime(ti->ac_stime + (now - ti->ac_stamp)); @@ -135,27 +130,18 @@ static cputime_t vtime_delta(struct task_struct *tsk) return delta_stime; } -void __vtime_account_system(struct task_struct *tsk) +void vtime_account_system(struct task_struct *tsk) { cputime_t delta = vtime_delta(tsk); account_system_time(tsk, 0, delta, delta); } -void __vtime_account_idle(struct task_struct *tsk) +void vtime_account_idle(struct task_struct *tsk) { account_idle_time(vtime_delta(tsk)); } -/* - * Called from the timer interrupt handler to charge accumulated user time - * to the current process. Must be called with interrupts disabled. - */ -void account_process_tick(struct task_struct *p, int user_tick) -{ - vtime_account_user(p); -} - #endif /* CONFIG_VIRT_CPU_ACCOUNTING */ static irqreturn_t diff --git a/arch/powerpc/include/asm/cputime.h b/arch/powerpc/include/asm/cputime.h index 487d46ff68a1..483733bd06d4 100644 --- a/arch/powerpc/include/asm/cputime.h +++ b/arch/powerpc/include/asm/cputime.h @@ -228,6 +228,8 @@ static inline cputime_t clock_t_to_cputime(const unsigned long clk) #define cputime64_to_clock_t(ct) cputime_to_clock_t((cputime_t)(ct)) +static inline void arch_vtime_task_switch(struct task_struct *tsk) { } + #endif /* __KERNEL__ */ #endif /* CONFIG_VIRT_CPU_ACCOUNTING */ #endif /* __POWERPC_CPUTIME_H */ diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 0db456f30d45..b3b14352b05e 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -297,6 +297,8 @@ static u64 vtime_delta(struct task_struct *tsk, u64 now, nowscaled, deltascaled; u64 udelta, delta, user_scaled; + WARN_ON_ONCE(!irqs_disabled()); + now = mftb(); nowscaled = read_spurr(now); get_paca()->system_time += now - get_paca()->starttime; @@ -336,7 +338,7 @@ static u64 vtime_delta(struct task_struct *tsk, return delta; } -void __vtime_account_system(struct task_struct *tsk) +void vtime_account_system(struct task_struct *tsk) { u64 delta, sys_scaled, stolen; @@ -346,7 +348,7 @@ void __vtime_account_system(struct task_struct *tsk) account_steal_time(stolen); } -void __vtime_account_idle(struct task_struct *tsk) +void vtime_account_idle(struct task_struct *tsk) { u64 delta, sys_scaled, stolen; @@ -355,15 +357,15 @@ void __vtime_account_idle(struct task_struct *tsk) } /* - * Transfer the user and system times accumulated in the paca - * by the exception entry and exit code to the generic process - * user and system time records. + * Transfer the user time accumulated in the paca + * by the exception entry and exit code to the generic + * process user time records. * Must be called with interrupts disabled. - * Assumes that vtime_account() has been called recently - * (i.e. since the last entry from usermode) so that + * Assumes that vtime_account_system/idle() has been called + * recently (i.e. since the last entry from usermode) so that * get_paca()->user_time_scaled is up to date. */ -void account_process_tick(struct task_struct *tsk, int user_tick) +void vtime_account_user(struct task_struct *tsk) { cputime_t utime, utimescaled; @@ -375,12 +377,6 @@ void account_process_tick(struct task_struct *tsk, int user_tick) account_user_time(tsk, utime, utimescaled); } -void vtime_task_switch(struct task_struct *prev) -{ - vtime_account(prev); - account_process_tick(prev, 0); -} - #else /* ! CONFIG_VIRT_CPU_ACCOUNTING */ #define calc_cputime_factors() #endif diff --git a/arch/s390/include/asm/cputime.h b/arch/s390/include/asm/cputime.h index 023d5ae24482..d2ff41370c0c 100644 --- a/arch/s390/include/asm/cputime.h +++ b/arch/s390/include/asm/cputime.h @@ -14,6 +14,7 @@ #define __ARCH_HAS_VTIME_ACCOUNT +#define __ARCH_HAS_VTIME_TASK_SWITCH /* We want to use full resolution of the CPU timer: 2**-12 micro-seconds. */ diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index 783e988c4e1e..e84b8b68444a 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -112,7 +112,12 @@ void vtime_task_switch(struct task_struct *prev) S390_lowcore.system_timer = ti->system_timer; } -void account_process_tick(struct task_struct *tsk, int user_tick) +/* + * In s390, accounting pending user time also implies + * accounting system time in order to correctly compute + * the stolen time accounting. + */ +void vtime_account_user(struct task_struct *tsk) { if (do_account_vtime(tsk, HARDIRQ_OFFSET)) virt_timer_expire(); @@ -127,6 +132,8 @@ void vtime_account(struct task_struct *tsk) struct thread_info *ti = task_thread_info(tsk); u64 timer, system; + WARN_ON_ONCE(!irqs_disabled()); + timer = S390_lowcore.last_update_timer; S390_lowcore.last_update_timer = get_vtimer(); S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; @@ -140,9 +147,9 @@ void vtime_account(struct task_struct *tsk) } EXPORT_SYMBOL_GPL(vtime_account); -void __vtime_account_system(struct task_struct *tsk) +void vtime_account_system(struct task_struct *tsk) __attribute__((alias("vtime_account"))); -EXPORT_SYMBOL_GPL(__vtime_account_system); +EXPORT_SYMBOL_GPL(vtime_account_system); void __kprobes vtime_stop_cpu(void) { diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index 1865b1f29770..66b70780e910 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -127,7 +127,15 @@ extern void account_system_time(struct task_struct *, int, cputime_t, cputime_t) extern void account_steal_time(cputime_t); extern void account_idle_time(cputime_t); +#ifdef CONFIG_VIRT_CPU_ACCOUNTING +static inline void account_process_tick(struct task_struct *tsk, int user) +{ + vtime_account_user(tsk); +} +#else extern void account_process_tick(struct task_struct *, int user); +#endif + extern void account_steal_ticks(unsigned long ticks); extern void account_idle_ticks(unsigned long ticks); diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 3738c260dde7..d5cddd8dcc5c 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -730,7 +730,7 @@ static inline void kvm_guest_enter(void) * This is running in ioctl context so we can avoid * the call to vtime_account() with its unnecessary idle check. */ - vtime_account_system(current); + vtime_account_system_irqsafe(current); current->flags |= PF_VCPU; /* KVM does not hold any references to rcu protected data when it * switches CPU into a guest mode. In fact switching to a guest mode @@ -748,7 +748,7 @@ static inline void kvm_guest_exit(void) * This is running in ioctl context so we can avoid * the call to vtime_account() with its unnecessary idle check. */ - vtime_account_system(current); + vtime_account_system_irqsafe(current); current->flags &= ~PF_VCPU; } diff --git a/include/linux/vtime.h b/include/linux/vtime.h index 0c2a2d303020..ae30ab58431a 100644 --- a/include/linux/vtime.h +++ b/include/linux/vtime.h @@ -5,14 +5,15 @@ struct task_struct; #ifdef CONFIG_VIRT_CPU_ACCOUNTING extern void vtime_task_switch(struct task_struct *prev); -extern void __vtime_account_system(struct task_struct *tsk); extern void vtime_account_system(struct task_struct *tsk); -extern void __vtime_account_idle(struct task_struct *tsk); +extern void vtime_account_system_irqsafe(struct task_struct *tsk); +extern void vtime_account_idle(struct task_struct *tsk); +extern void vtime_account_user(struct task_struct *tsk); extern void vtime_account(struct task_struct *tsk); #else static inline void vtime_task_switch(struct task_struct *prev) { } -static inline void __vtime_account_system(struct task_struct *tsk) { } static inline void vtime_account_system(struct task_struct *tsk) { } +static inline void vtime_account_system_irqsafe(struct task_struct *tsk) { } static inline void vtime_account(struct task_struct *tsk) { } #endif @@ -40,7 +41,7 @@ static inline void vtime_account_irq_enter(struct task_struct *tsk) static inline void vtime_account_irq_exit(struct task_struct *tsk) { /* On hard|softirq exit we always account to hard|softirq cputime */ - __vtime_account_system(tsk); + vtime_account_system(tsk); irqtime_account_irq(tsk); } diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index b7f731768625..293b202fcf79 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -461,20 +461,33 @@ void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime *st = cputime.stime; } -void vtime_account_system(struct task_struct *tsk) +void vtime_account_system_irqsafe(struct task_struct *tsk) { unsigned long flags; local_irq_save(flags); - __vtime_account_system(tsk); + vtime_account_system(tsk); local_irq_restore(flags); } -EXPORT_SYMBOL_GPL(vtime_account_system); +EXPORT_SYMBOL_GPL(vtime_account_system_irqsafe); + +#ifndef __ARCH_HAS_VTIME_TASK_SWITCH +void vtime_task_switch(struct task_struct *prev) +{ + if (is_idle_task(prev)) + vtime_account_idle(prev); + else + vtime_account_system(prev); + + vtime_account_user(prev); + arch_vtime_task_switch(prev); +} +#endif /* * Archs that account the whole time spent in the idle task * (outside irq) as idle time can rely on this and just implement - * __vtime_account_system() and __vtime_account_idle(). Archs that + * vtime_account_system() and vtime_account_idle(). Archs that * have other meaning of the idle time (s390 only includes the * time spent by the CPU when it's in low power mode) must override * vtime_account(). @@ -482,16 +495,10 @@ EXPORT_SYMBOL_GPL(vtime_account_system); #ifndef __ARCH_HAS_VTIME_ACCOUNT void vtime_account(struct task_struct *tsk) { - unsigned long flags; - - local_irq_save(flags); - if (in_interrupt() || !is_idle_task(tsk)) - __vtime_account_system(tsk); + vtime_account_system(tsk); else - __vtime_account_idle(tsk); - - local_irq_restore(flags); + vtime_account_idle(tsk); } EXPORT_SYMBOL_GPL(vtime_account); #endif /* __ARCH_HAS_VTIME_ACCOUNT */ |