From aa6b43d57f995f6eec0e9e744dcb200e09ecb8e0 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 15 Sep 2020 12:31:58 +0200 Subject: ACPI: processor: Use CPUIDLE_FLAG_TIMER_STOP Make acpi_processor_idle use the common broadcast code, there's no reason not to. This also removes some RCU usage after rcu_idle_enter(). Signed-off-by: Peter Zijlstra (Intel) Reported-by: Borislav Petkov Tested-by: Borislav Petkov Signed-off-by: Rafael J. Wysocki --- drivers/acpi/processor_idle.c | 47 ++++++++++++++----------------------------- 1 file changed, 15 insertions(+), 32 deletions(-) (limited to 'drivers/acpi') diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index 71a30b0d0f05..01ea986b6197 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -161,18 +161,10 @@ static void lapic_timer_propagate_broadcast(struct acpi_processor *pr) } /* Power(C) State timer broadcast control */ -static void lapic_timer_state_broadcast(struct acpi_processor *pr, - struct acpi_processor_cx *cx, - int broadcast) +static bool lapic_timer_needs_broadcast(struct acpi_processor *pr, + struct acpi_processor_cx *cx) { - int state = cx - pr->power.states; - - if (state >= pr->power.timer_broadcast_on_state) { - if (broadcast) - tick_broadcast_enter(); - else - tick_broadcast_exit(); - } + return cx - pr->power.states >= pr->power.timer_broadcast_on_state; } #else @@ -180,9 +172,9 @@ static void lapic_timer_state_broadcast(struct acpi_processor *pr, static void lapic_timer_check_state(int state, struct acpi_processor *pr, struct acpi_processor_cx *cstate) { } static void lapic_timer_propagate_broadcast(struct acpi_processor *pr) { } -static void lapic_timer_state_broadcast(struct acpi_processor *pr, - struct acpi_processor_cx *cx, - int broadcast) + +static bool lapic_timer_needs_broadcast(struct acpi_processor *pr, + struct acpi_processor_cx *cx) { } @@ -568,20 +560,12 @@ static DEFINE_RAW_SPINLOCK(c3_lock); * acpi_idle_enter_bm - enters C3 with proper BM handling * @pr: Target processor * @cx: Target state context - * @timer_bc: Whether or not to change timer mode to broadcast */ static void acpi_idle_enter_bm(struct acpi_processor *pr, - struct acpi_processor_cx *cx, bool timer_bc) + struct acpi_processor_cx *cx) { acpi_unlazy_tlb(smp_processor_id()); - /* - * Must be done before busmaster disable as we might need to - * access HPET ! - */ - if (timer_bc) - lapic_timer_state_broadcast(pr, cx, 1); - /* * disable bus master * bm_check implies we need ARB_DIS @@ -609,9 +593,6 @@ static void acpi_idle_enter_bm(struct acpi_processor *pr, c3_cpu_count--; raw_spin_unlock(&c3_lock); } - - if (timer_bc) - lapic_timer_state_broadcast(pr, cx, 0); } static int acpi_idle_enter(struct cpuidle_device *dev, @@ -630,7 +611,7 @@ static int acpi_idle_enter(struct cpuidle_device *dev, cx = per_cpu(acpi_cstate[index], dev->cpu); } else if (cx->type == ACPI_STATE_C3 && pr->flags.bm_check) { if (cx->bm_sts_skip || !acpi_idle_bm_check()) { - acpi_idle_enter_bm(pr, cx, true); + acpi_idle_enter_bm(pr, cx); return index; } else if (drv->safe_state_index >= 0) { index = drv->safe_state_index; @@ -642,15 +623,11 @@ static int acpi_idle_enter(struct cpuidle_device *dev, } } - lapic_timer_state_broadcast(pr, cx, 1); - if (cx->type == ACPI_STATE_C3) ACPI_FLUSH_CPU_CACHE(); acpi_idle_do_entry(cx); - lapic_timer_state_broadcast(pr, cx, 0); - return index; } @@ -666,7 +643,7 @@ static int acpi_idle_enter_s2idle(struct cpuidle_device *dev, return 0; if (pr->flags.bm_check) { - acpi_idle_enter_bm(pr, cx, false); + acpi_idle_enter_bm(pr, cx); return 0; } else { ACPI_FLUSH_CPU_CACHE(); @@ -682,6 +659,7 @@ static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr, { int i, count = ACPI_IDLE_STATE_START; struct acpi_processor_cx *cx; + struct cpuidle_state *state; if (max_cstate == 0) max_cstate = 1; @@ -694,6 +672,11 @@ static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr, per_cpu(acpi_cstate[count], dev->cpu) = cx; + if (lapic_timer_needs_broadcast(pr, cx)) { + state = &acpi_idle_driver.states[count]; + state->flags |= CPUIDLE_FLAG_TIMER_STOP; + } + count++; if (count == CPUIDLE_STATE_MAX) break; -- cgit v1.2.3 From a889a23a98fee183c9b6b0b14b2fd70583429c5e Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 15 Sep 2020 12:31:59 +0200 Subject: ACPI: processor: Use CPUIDLE_FLAG_TLB_FLUSHED Make acpi_processor_idle() use the generic TLB flushing code. This again removes RCU usage after rcu_idle_enter(). (XXX make every C3 invalidate TLBs, not just C3-BM) Signed-off-by: Peter Zijlstra (Intel) Tested-by: Borislav Petkov Signed-off-by: Rafael J. Wysocki --- arch/ia64/include/asm/acpi.h | 2 -- arch/x86/include/asm/acpi.h | 2 -- drivers/acpi/processor_idle.c | 10 +++++----- 3 files changed, 5 insertions(+), 9 deletions(-) (limited to 'drivers/acpi') diff --git a/arch/ia64/include/asm/acpi.h b/arch/ia64/include/asm/acpi.h index b66ba907019c..87927eb824cc 100644 --- a/arch/ia64/include/asm/acpi.h +++ b/arch/ia64/include/asm/acpi.h @@ -74,8 +74,6 @@ static inline void arch_acpi_set_pdc_bits(u32 *buf) buf[2] |= ACPI_PDC_EST_CAPABILITY_SMP; } -#define acpi_unlazy_tlb(x) - #ifdef CONFIG_ACPI_NUMA extern cpumask_t early_cpu_possible_map; #define for_each_possible_early_cpu(cpu) \ diff --git a/arch/x86/include/asm/acpi.h b/arch/x86/include/asm/acpi.h index ca0976456a6b..6d2df1ee427b 100644 --- a/arch/x86/include/asm/acpi.h +++ b/arch/x86/include/asm/acpi.h @@ -159,8 +159,6 @@ static inline u64 x86_default_get_root_pointer(void) extern int x86_acpi_numa_init(void); #endif /* CONFIG_ACPI_NUMA */ -#define acpi_unlazy_tlb(x) leave_mm(x) - #ifdef CONFIG_ACPI_APEI static inline pgprot_t arch_apei_get_mem_attribute(phys_addr_t addr) { diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index 01ea986b6197..9ec504067f98 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -564,8 +564,6 @@ static DEFINE_RAW_SPINLOCK(c3_lock); static void acpi_idle_enter_bm(struct acpi_processor *pr, struct acpi_processor_cx *cx) { - acpi_unlazy_tlb(smp_processor_id()); - /* * disable bus master * bm_check implies we need ARB_DIS @@ -665,6 +663,7 @@ static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr, max_cstate = 1; for (i = 1; i < ACPI_PROCESSOR_MAX_POWER && i <= max_cstate; i++) { + state = &acpi_idle_driver.states[count]; cx = &pr->power.states[i]; if (!cx->valid) @@ -672,10 +671,11 @@ static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr, per_cpu(acpi_cstate[count], dev->cpu) = cx; - if (lapic_timer_needs_broadcast(pr, cx)) { - state = &acpi_idle_driver.states[count]; + if (lapic_timer_needs_broadcast(pr, cx)) state->flags |= CPUIDLE_FLAG_TIMER_STOP; - } + + if (cx->type == ACPI_STATE_C3) + state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; count++; if (count == CPUIDLE_STATE_MAX) -- cgit v1.2.3 From 1fecfdbb7acc6624655450a609221c89b5197a06 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 15 Sep 2020 12:32:01 +0200 Subject: ACPI: processor: Take over RCU-idle for C3-BM idle The C3 BusMaster idle code takes lock in a number of places, some deep inside the ACPI code. Instead of wrapping it all in RCU_NONIDLE, have the driver take over RCU-idle duty and avoid flipping RCU state back and forth a lot. ( by marking 'C3 && bm_check' as RCU_IDLE, we _must_ call enter_bm() for that combination, otherwise we'll loose RCU-idle, this requires shuffling some code around ) Signed-off-by: Peter Zijlstra (Intel) Tested-by: Borislav Petkov Signed-off-by: Rafael J. Wysocki --- drivers/acpi/processor_idle.c | 69 ++++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 20 deletions(-) (limited to 'drivers/acpi') diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index 9ec504067f98..7ecb90e90afd 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -558,22 +558,43 @@ static DEFINE_RAW_SPINLOCK(c3_lock); /** * acpi_idle_enter_bm - enters C3 with proper BM handling + * @drv: cpuidle driver * @pr: Target processor * @cx: Target state context + * @index: index of target state */ -static void acpi_idle_enter_bm(struct acpi_processor *pr, - struct acpi_processor_cx *cx) +static int acpi_idle_enter_bm(struct cpuidle_driver *drv, + struct acpi_processor *pr, + struct acpi_processor_cx *cx, + int index) { + static struct acpi_processor_cx safe_cx = { + .entry_method = ACPI_CSTATE_HALT, + }; + /* * disable bus master * bm_check implies we need ARB_DIS * bm_control implies whether we can do ARB_DIS * - * That leaves a case where bm_check is set and bm_control is - * not set. In that case we cannot do much, we enter C3 - * without doing anything. + * That leaves a case where bm_check is set and bm_control is not set. + * In that case we cannot do much, we enter C3 without doing anything. */ - if (pr->flags.bm_control) { + bool dis_bm = pr->flags.bm_control; + + /* If we can skip BM, demote to a safe state. */ + if (!cx->bm_sts_skip && acpi_idle_bm_check()) { + dis_bm = false; + index = drv->safe_state_index; + if (index >= 0) { + cx = this_cpu_read(acpi_cstate[index]); + } else { + cx = &safe_cx; + index = -EBUSY; + } + } + + if (dis_bm) { raw_spin_lock(&c3_lock); c3_cpu_count++; /* Disable bus master arbitration when all CPUs are in C3 */ @@ -582,15 +603,21 @@ static void acpi_idle_enter_bm(struct acpi_processor *pr, raw_spin_unlock(&c3_lock); } + rcu_idle_enter(); + acpi_idle_do_entry(cx); + rcu_idle_exit(); + /* Re-enable bus master arbitration */ - if (pr->flags.bm_control) { + if (dis_bm) { raw_spin_lock(&c3_lock); acpi_write_bit_register(ACPI_BITREG_ARB_DISABLE, 0); c3_cpu_count--; raw_spin_unlock(&c3_lock); } + + return index; } static int acpi_idle_enter(struct cpuidle_device *dev, @@ -604,20 +631,13 @@ static int acpi_idle_enter(struct cpuidle_device *dev, return -EINVAL; if (cx->type != ACPI_STATE_C1) { + if (cx->type == ACPI_STATE_C3 && pr->flags.bm_check) + return acpi_idle_enter_bm(drv, pr, cx, index); + + /* C2 to C1 demotion. */ if (acpi_idle_fallback_to_c1(pr) && num_online_cpus() > 1) { index = ACPI_IDLE_STATE_START; cx = per_cpu(acpi_cstate[index], dev->cpu); - } else if (cx->type == ACPI_STATE_C3 && pr->flags.bm_check) { - if (cx->bm_sts_skip || !acpi_idle_bm_check()) { - acpi_idle_enter_bm(pr, cx); - return index; - } else if (drv->safe_state_index >= 0) { - index = drv->safe_state_index; - cx = per_cpu(acpi_cstate[index], dev->cpu); - } else { - acpi_safe_halt(); - return -EBUSY; - } } } @@ -641,7 +661,13 @@ static int acpi_idle_enter_s2idle(struct cpuidle_device *dev, return 0; if (pr->flags.bm_check) { - acpi_idle_enter_bm(pr, cx); + u8 bm_sts_skip = cx->bm_sts_skip; + + /* Don't check BM_STS, do an unconditional ARB_DIS for S2IDLE */ + cx->bm_sts_skip = 1; + acpi_idle_enter_bm(drv, pr, cx, index); + cx->bm_sts_skip = bm_sts_skip; + return 0; } else { ACPI_FLUSH_CPU_CACHE(); @@ -674,8 +700,11 @@ static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr, if (lapic_timer_needs_broadcast(pr, cx)) state->flags |= CPUIDLE_FLAG_TIMER_STOP; - if (cx->type == ACPI_STATE_C3) + if (cx->type == ACPI_STATE_C3) { state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; + if (pr->flags.bm_check) + state->flags |= CPUIDLE_FLAG_RCU_IDLE; + } count++; if (count == CPUIDLE_STATE_MAX) -- cgit v1.2.3