diff options
Diffstat (limited to 'drivers/idle/intel_idle.c')
| -rw-r--r-- | drivers/idle/intel_idle.c | 89 |
1 files changed, 82 insertions, 7 deletions
diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index 56ac09d6c930..4a5c4a44ffb1 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -59,7 +59,10 @@ #include <linux/hrtimer.h> /* ktime_get_real() */ #include <trace/events/power.h> #include <linux/sched.h> +#include <linux/notifier.h> +#include <linux/cpu.h> #include <asm/mwait.h> +#include <asm/msr.h> #define INTEL_IDLE_VERSION "0.4" #define PREFIX "intel_idle: " @@ -73,6 +76,7 @@ static int max_cstate = MWAIT_MAX_NUM_CSTATES - 1; static unsigned int mwait_substates; +#define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF /* Reliable LAPIC Timer States, bit 1 for C1 etc. */ static unsigned int lapic_timer_reliable_states = (1 << 1); /* Default to only C1 */ @@ -82,6 +86,20 @@ static int intel_idle(struct cpuidle_device *dev, struct cpuidle_state *state); static struct cpuidle_state *cpuidle_state_table; /* + * Hardware C-state auto-demotion may not always be optimal. + * Indicate which enable bits to clear here. + */ +static unsigned long long auto_demotion_disable_flags; + +/* + * Set this flag for states where the HW flushes the TLB for us + * and so we don't need cross-calls to keep it consistent. + * If this flag is set, SW flushes the TLB, so even if the + * HW doesn't do the flushing, this flag is safe to use. + */ +#define CPUIDLE_FLAG_TLB_FLUSHED 0x10000 + +/* * States are indexed by the cstate number, * which is also the index into the MWAIT hint array. * Thus C0 is a dummy. @@ -122,7 +140,7 @@ static struct cpuidle_state snb_cstates[MWAIT_MAX_NUM_CSTATES] = { .driver_data = (void *) 0x00, .flags = CPUIDLE_FLAG_TIME_VALID, .exit_latency = 1, - .target_residency = 4, + .target_residency = 1, .enter = &intel_idle }, { /* MWAIT C2 */ .name = "SNB-C3", @@ -130,7 +148,7 @@ static struct cpuidle_state snb_cstates[MWAIT_MAX_NUM_CSTATES] = { .driver_data = (void *) 0x10, .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 80, - .target_residency = 160, + .target_residency = 211, .enter = &intel_idle }, { /* MWAIT C3 */ .name = "SNB-C6", @@ -138,7 +156,7 @@ static struct cpuidle_state snb_cstates[MWAIT_MAX_NUM_CSTATES] = { .driver_data = (void *) 0x20, .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 104, - .target_residency = 208, + .target_residency = 345, .enter = &intel_idle }, { /* MWAIT C4 */ .name = "SNB-C7", @@ -146,7 +164,7 @@ static struct cpuidle_state snb_cstates[MWAIT_MAX_NUM_CSTATES] = { .driver_data = (void *) 0x30, .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 109, - .target_residency = 300, + .target_residency = 345, .enter = &intel_idle }, }; @@ -220,8 +238,6 @@ static int intel_idle(struct cpuidle_device *dev, struct cpuidle_state *state) kt_before = ktime_get_real(); stop_critical_timings(); - trace_power_start(POWER_CSTATE, (eax >> 4) + 1, cpu); - trace_cpu_idle((eax >> 4) + 1, cpu); if (!need_resched()) { __monitor((void *)¤t_thread_info()->flags, 0, 0); @@ -243,6 +259,44 @@ static int intel_idle(struct cpuidle_device *dev, struct cpuidle_state *state) return usec_delta; } +static void __setup_broadcast_timer(void *arg) +{ + unsigned long reason = (unsigned long)arg; + int cpu = smp_processor_id(); + + reason = reason ? + CLOCK_EVT_NOTIFY_BROADCAST_ON : CLOCK_EVT_NOTIFY_BROADCAST_OFF; + + clockevents_notify(reason, &cpu); +} + +static int setup_broadcast_cpuhp_notify(struct notifier_block *n, + unsigned long action, void *hcpu) +{ + int hotcpu = (unsigned long)hcpu; + + switch (action & 0xf) { + case CPU_ONLINE: + smp_call_function_single(hotcpu, __setup_broadcast_timer, + (void *)true, 1); + break; + } + return NOTIFY_OK; +} + +static struct notifier_block setup_broadcast_notifier = { + .notifier_call = setup_broadcast_cpuhp_notify, +}; + +static void auto_demotion_disable(void *dummy) +{ + unsigned long long msr_bits; + + rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits); + msr_bits &= ~auto_demotion_disable_flags; + wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits); +} + /* * intel_idle_probe() */ @@ -286,11 +340,17 @@ static int intel_idle_probe(void) case 0x25: /* Westmere */ case 0x2C: /* Westmere */ cpuidle_state_table = nehalem_cstates; + auto_demotion_disable_flags = + (NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE); break; case 0x1C: /* 28 - Atom Processor */ + cpuidle_state_table = atom_cstates; + break; + case 0x26: /* 38 - Lincroft Atom Processor */ cpuidle_state_table = atom_cstates; + auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE; break; case 0x2A: /* SNB */ @@ -305,7 +365,11 @@ static int intel_idle_probe(void) } if (boot_cpu_has(X86_FEATURE_ARAT)) /* Always Reliable APIC Timer */ - lapic_timer_reliable_states = 0xFFFFFFFF; + lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE; + else { + smp_call_function(__setup_broadcast_timer, (void *)true, 1); + register_cpu_notifier(&setup_broadcast_notifier); + } pr_debug(PREFIX "v" INTEL_IDLE_VERSION " model 0x%X\n", boot_cpu_data.x86_model); @@ -394,6 +458,8 @@ static int intel_idle_cpuidle_devices_init(void) return -EIO; } } + if (auto_demotion_disable_flags) + smp_call_function(auto_demotion_disable, NULL, 1); return 0; } @@ -403,6 +469,10 @@ static int __init intel_idle_init(void) { int retval; + /* Do not load intel_idle at all for now if idle= is passed */ + if (boot_option_idle_override != IDLE_NO_OVERRIDE) + return -ENODEV; + retval = intel_idle_probe(); if (retval) return retval; @@ -428,6 +498,11 @@ static void __exit intel_idle_exit(void) intel_idle_cpuidle_devices_uninit(); cpuidle_unregister_driver(&intel_idle_driver); + if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) { + smp_call_function(__setup_broadcast_timer, (void *)false, 1); + unregister_cpu_notifier(&setup_broadcast_notifier); + } + return; } |