summaryrefslogtreecommitdiff
path: root/drivers/cpufreq
diff options
context:
space:
mode:
authorViresh Kumar <viresh.kumar@linaro.org>2021-07-01 05:02:37 +0300
committerViresh Kumar <viresh.kumar@linaro.org>2021-07-01 05:02:37 +0300
commitc503c193db7d7ccc0c58b1ef694eaef331318149 (patch)
tree44d3215196f385e88fa5da6affc58649c90e0ea9 /drivers/cpufreq
parent9821a195d4e263801884b105554e801642c59f2a (diff)
parent1eb5dde674f57b1a1918dab33f09e35cdd64eb07 (diff)
downloadlinux-c503c193db7d7ccc0c58b1ef694eaef331318149.tar.xz
Merge branch 'cpufreq/cppc-fie' into cpufreq/arm/linux-next
Diffstat (limited to 'drivers/cpufreq')
-rw-r--r--drivers/cpufreq/acpi-cpufreq.c6
-rw-r--r--drivers/cpufreq/cppc_cpufreq.c205
-rw-r--r--drivers/cpufreq/cpufreq.c11
-rw-r--r--drivers/cpufreq/cpufreq_stats.c5
-rw-r--r--drivers/cpufreq/intel_pstate.c288
-rw-r--r--drivers/cpufreq/loongson2_cpufreq.c1
-rw-r--r--drivers/cpufreq/sc520_freq.c1
-rw-r--r--drivers/cpufreq/sh-cpufreq.c1
8 files changed, 382 insertions, 136 deletions
diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c
index d1bbc16fba4b..7e7450453714 100644
--- a/drivers/cpufreq/acpi-cpufreq.c
+++ b/drivers/cpufreq/acpi-cpufreq.c
@@ -646,7 +646,11 @@ static u64 get_max_boost_ratio(unsigned int cpu)
return 0;
}
- highest_perf = perf_caps.highest_perf;
+ if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
+ highest_perf = amd_get_highest_perf();
+ else
+ highest_perf = perf_caps.highest_perf;
+
nominal_perf = perf_caps.nominal_perf;
if (!highest_perf || !nominal_perf) {
diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c
index 3848b4c222e1..d4c27022b9c9 100644
--- a/drivers/cpufreq/cppc_cpufreq.c
+++ b/drivers/cpufreq/cppc_cpufreq.c
@@ -74,13 +74,12 @@ struct cppc_freq_invariance {
static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_freq_inv);
static struct kthread_worker *kworker_fie;
-static bool fie_disabled;
static struct cpufreq_driver cppc_cpufreq_driver;
static unsigned int hisi_cppc_cpufreq_get_rate(unsigned int cpu);
static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data,
- struct cppc_perf_fb_ctrs fb_ctrs_t0,
- struct cppc_perf_fb_ctrs fb_ctrs_t1);
+ struct cppc_perf_fb_ctrs *fb_ctrs_t0,
+ struct cppc_perf_fb_ctrs *fb_ctrs_t1);
/**
* cppc_scale_freq_workfn - CPPC arch_freq_scale updater for frequency invariance
@@ -115,13 +114,15 @@ static void cppc_scale_freq_workfn(struct kthread_work *work)
return;
}
+ perf = cppc_perf_from_fbctrs(cpu_data, &cppc_fi->prev_perf_fb_ctrs,
+ &fb_ctrs);
cppc_fi->prev_perf_fb_ctrs = fb_ctrs;
- perf = cppc_perf_from_fbctrs(cpu_data, cppc_fi->prev_perf_fb_ctrs,
- fb_ctrs);
perf <<= SCHED_CAPACITY_SHIFT;
local_freq_scale = div64_u64(perf, cpu_data->perf_caps.highest_perf);
- if (WARN_ON(local_freq_scale > 1024))
+
+ /* This can happen due to counter's overflow */
+ if (unlikely(local_freq_scale > 1024))
local_freq_scale = 1024;
per_cpu(arch_freq_scale, cppc_fi->cpu) = local_freq_scale;
@@ -151,35 +152,63 @@ static struct scale_freq_data cppc_sftd = {
.set_freq_scale = cppc_scale_freq_tick,
};
-static void cppc_freq_invariance_policy_init(struct cpufreq_policy *policy,
- struct cppc_cpudata *cpu_data)
+static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)
{
- struct cppc_perf_fb_ctrs fb_ctrs = {0};
struct cppc_freq_invariance *cppc_fi;
- int i, ret;
+ int cpu, ret;
if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate)
return;
- if (fie_disabled)
- return;
-
- for_each_cpu(i, policy->cpus) {
- cppc_fi = &per_cpu(cppc_freq_inv, i);
- cppc_fi->cpu = i;
- cppc_fi->cpu_data = cpu_data;
+ for_each_cpu(cpu, policy->cpus) {
+ cppc_fi = &per_cpu(cppc_freq_inv, cpu);
+ cppc_fi->cpu = cpu;
+ cppc_fi->cpu_data = policy->driver_data;
kthread_init_work(&cppc_fi->work, cppc_scale_freq_workfn);
init_irq_work(&cppc_fi->irq_work, cppc_irq_work);
- ret = cppc_get_perf_ctrs(i, &fb_ctrs);
+ ret = cppc_get_perf_ctrs(cpu, &cppc_fi->prev_perf_fb_ctrs);
if (ret) {
- pr_warn("%s: failed to read perf counters: %d\n",
- __func__, ret);
- fie_disabled = true;
- } else {
- cppc_fi->prev_perf_fb_ctrs = fb_ctrs;
+ pr_warn("%s: failed to read perf counters for cpu:%d: %d\n",
+ __func__, cpu, ret);
+
+ /*
+ * Don't abort if the CPU was offline while the driver
+ * was getting registered.
+ */
+ if (cpu_online(cpu))
+ return;
}
}
+
+ /* Register for freq-invariance */
+ topology_set_scale_freq_source(&cppc_sftd, policy->cpus);
+}
+
+/*
+ * We free all the resources on policy's removal and not on CPU removal as the
+ * irq-work are per-cpu and the hotplug core takes care of flushing the pending
+ * irq-works (hint: smpcfd_dying_cpu()) on CPU hotplug. Even if the kthread-work
+ * fires on another CPU after the concerned CPU is removed, it won't harm.
+ *
+ * We just need to make sure to remove them all on policy->exit().
+ */
+static void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy)
+{
+ struct cppc_freq_invariance *cppc_fi;
+ int cpu;
+
+ if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate)
+ return;
+
+ /* policy->cpus will be empty here, use related_cpus instead */
+ topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_CPPC, policy->related_cpus);
+
+ for_each_cpu(cpu, policy->related_cpus) {
+ cppc_fi = &per_cpu(cppc_freq_inv, cpu);
+ irq_work_sync(&cppc_fi->irq_work);
+ kthread_cancel_work_sync(&cppc_fi->work);
+ }
}
static void __init cppc_freq_invariance_init(void)
@@ -202,9 +231,6 @@ static void __init cppc_freq_invariance_init(void)
if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate)
return;
- if (fie_disabled)
- return;
-
kworker_fie = kthread_create_worker(0, "cppc_fie");
if (IS_ERR(kworker_fie))
return;
@@ -216,37 +242,23 @@ static void __init cppc_freq_invariance_init(void)
kthread_destroy_worker(kworker_fie);
return;
}
-
- /* Register for freq-invariance */
- topology_set_scale_freq_source(&cppc_sftd, cpu_present_mask);
}
static void cppc_freq_invariance_exit(void)
{
- struct cppc_freq_invariance *cppc_fi;
- int i;
-
if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate)
return;
- if (fie_disabled)
- return;
-
- topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_CPPC, cpu_present_mask);
-
- for_each_possible_cpu(i) {
- cppc_fi = &per_cpu(cppc_freq_inv, i);
- irq_work_sync(&cppc_fi->irq_work);
- }
-
kthread_destroy_worker(kworker_fie);
kworker_fie = NULL;
}
#else
-static inline void
-cppc_freq_invariance_policy_init(struct cpufreq_policy *policy,
- struct cppc_cpudata *cpu_data)
+static inline void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)
+{
+}
+
+static inline void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy)
{
}
@@ -384,27 +396,6 @@ static int cppc_verify_policy(struct cpufreq_policy_data *policy)
return 0;
}
-static void cppc_cpufreq_stop_cpu(struct cpufreq_policy *policy)
-{
- struct cppc_cpudata *cpu_data = policy->driver_data;
- struct cppc_perf_caps *caps = &cpu_data->perf_caps;
- unsigned int cpu = policy->cpu;
- int ret;
-
- cpu_data->perf_ctrls.desired_perf = caps->lowest_perf;
-
- ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
- if (ret)
- pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n",
- caps->lowest_perf, cpu, ret);
-
- /* Remove CPU node from list and free driver data for policy */
- free_cpumask_var(cpu_data->shared_cpu_map);
- list_del(&cpu_data->node);
- kfree(policy->driver_data);
- policy->driver_data = NULL;
-}
-
/*
* The PCC subspace describes the rate at which platform can accept commands
* on the shared PCC channel (including READs which do not count towards freq
@@ -479,6 +470,16 @@ out:
return NULL;
}
+static void cppc_cpufreq_put_cpu_data(struct cpufreq_policy *policy)
+{
+ struct cppc_cpudata *cpu_data = policy->driver_data;
+
+ list_del(&cpu_data->node);
+ free_cpumask_var(cpu_data->shared_cpu_map);
+ kfree(cpu_data);
+ policy->driver_data = NULL;
+}
+
static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
{
unsigned int cpu = policy->cpu;
@@ -532,7 +533,8 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
default:
pr_debug("Unsupported CPU co-ord type: %d\n",
policy->shared_type);
- return -EFAULT;
+ ret = -EFAULT;
+ goto out;
}
/*
@@ -550,13 +552,37 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
if (ret) {
pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n",
caps->highest_perf, cpu, ret);
- } else {
- cppc_freq_invariance_policy_init(policy, cpu_data);
+ goto out;
}
+ cppc_cpufreq_cpu_fie_init(policy);
+ return 0;
+
+out:
+ cppc_cpufreq_put_cpu_data(policy);
return ret;
}
+static int cppc_cpufreq_cpu_exit(struct cpufreq_policy *policy)
+{
+ struct cppc_cpudata *cpu_data = policy->driver_data;
+ struct cppc_perf_caps *caps = &cpu_data->perf_caps;
+ unsigned int cpu = policy->cpu;
+ int ret;
+
+ cppc_cpufreq_cpu_fie_exit(policy);
+
+ cpu_data->perf_ctrls.desired_perf = caps->lowest_perf;
+
+ ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
+ if (ret)
+ pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n",
+ caps->lowest_perf, cpu, ret);
+
+ cppc_cpufreq_put_cpu_data(policy);
+ return 0;
+}
+
static inline u64 get_delta(u64 t1, u64 t0)
{
if (t1 > t0 || t0 > ~(u32)0)
@@ -566,18 +592,18 @@ static inline u64 get_delta(u64 t1, u64 t0)
}
static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data,
- struct cppc_perf_fb_ctrs fb_ctrs_t0,
- struct cppc_perf_fb_ctrs fb_ctrs_t1)
+ struct cppc_perf_fb_ctrs *fb_ctrs_t0,
+ struct cppc_perf_fb_ctrs *fb_ctrs_t1)
{
u64 delta_reference, delta_delivered;
u64 reference_perf;
- reference_perf = fb_ctrs_t0.reference_perf;
+ reference_perf = fb_ctrs_t0->reference_perf;
- delta_reference = get_delta(fb_ctrs_t1.reference,
- fb_ctrs_t0.reference);
- delta_delivered = get_delta(fb_ctrs_t1.delivered,
- fb_ctrs_t0.delivered);
+ delta_reference = get_delta(fb_ctrs_t1->reference,
+ fb_ctrs_t0->reference);
+ delta_delivered = get_delta(fb_ctrs_t1->delivered,
+ fb_ctrs_t0->delivered);
/* Check to avoid divide-by zero and invalid delivered_perf */
if (!delta_reference || !delta_delivered)
@@ -586,23 +612,12 @@ static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data,
return (reference_perf * delta_delivered) / delta_reference;
}
-static int cppc_get_rate_from_fbctrs(struct cppc_cpudata *cpu_data,
- struct cppc_perf_fb_ctrs fb_ctrs_t0,
- struct cppc_perf_fb_ctrs fb_ctrs_t1)
-{
- u64 delivered_perf;
-
- delivered_perf = cppc_perf_from_fbctrs(cpu_data, fb_ctrs_t0,
- fb_ctrs_t1);
-
- return cppc_cpufreq_perf_to_khz(cpu_data, delivered_perf);
-}
-
static unsigned int cppc_cpufreq_get_rate(unsigned int cpu)
{
struct cppc_perf_fb_ctrs fb_ctrs_t0 = {0}, fb_ctrs_t1 = {0};
struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
struct cppc_cpudata *cpu_data = policy->driver_data;
+ u64 delivered_perf;
int ret;
cpufreq_cpu_put(policy);
@@ -617,7 +632,10 @@ static unsigned int cppc_cpufreq_get_rate(unsigned int cpu)
if (ret)
return ret;
- return cppc_get_rate_from_fbctrs(cpu_data, fb_ctrs_t0, fb_ctrs_t1);
+ delivered_perf = cppc_perf_from_fbctrs(cpu_data, &fb_ctrs_t0,
+ &fb_ctrs_t1);
+
+ return cppc_cpufreq_perf_to_khz(cpu_data, delivered_perf);
}
static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state)
@@ -665,7 +683,7 @@ static struct cpufreq_driver cppc_cpufreq_driver = {
.target = cppc_cpufreq_set_target,
.get = cppc_cpufreq_get_rate,
.init = cppc_cpufreq_cpu_init,
- .stop_cpu = cppc_cpufreq_stop_cpu,
+ .exit = cppc_cpufreq_cpu_exit,
.set_boost = cppc_cpufreq_set_boost,
.attr = cppc_cpufreq_attr,
.name = "cppc_cpufreq",
@@ -726,10 +744,11 @@ static int __init cppc_cpufreq_init(void)
INIT_LIST_HEAD(&cpu_data_list);
cppc_check_hisi_workaround();
+ cppc_freq_invariance_init();
ret = cpufreq_register_driver(&cppc_cpufreq_driver);
- if (!ret)
- cppc_freq_invariance_init();
+ if (ret)
+ cppc_freq_invariance_exit();
return ret;
}
@@ -748,8 +767,8 @@ static inline void free_cpu_data(void)
static void __exit cppc_cpufreq_exit(void)
{
- cppc_freq_invariance_exit();
cpufreq_unregister_driver(&cppc_cpufreq_driver);
+ cppc_freq_invariance_exit();
free_cpu_data();
}
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index 802abc925b2a..cbab834c37a0 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -1367,9 +1367,14 @@ static int cpufreq_online(unsigned int cpu)
goto out_free_policy;
}
+ /*
+ * The initialization has succeeded and the policy is online.
+ * If there is a problem with its frequency table, take it
+ * offline and drop it.
+ */
ret = cpufreq_table_validate_and_sort(policy);
if (ret)
- goto out_exit_policy;
+ goto out_offline_policy;
/* related_cpus should at least include policy->cpus. */
cpumask_copy(policy->related_cpus, policy->cpus);
@@ -1515,6 +1520,10 @@ out_destroy_policy:
up_write(&policy->rwsem);
+out_offline_policy:
+ if (cpufreq_driver->offline)
+ cpufreq_driver->offline(policy);
+
out_exit_policy:
if (cpufreq_driver->exit)
cpufreq_driver->exit(policy);
diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c
index da717f7cd9a9..1570d6f3e75d 100644
--- a/drivers/cpufreq/cpufreq_stats.c
+++ b/drivers/cpufreq/cpufreq_stats.c
@@ -211,7 +211,7 @@ void cpufreq_stats_free_table(struct cpufreq_policy *policy)
void cpufreq_stats_create_table(struct cpufreq_policy *policy)
{
- unsigned int i = 0, count = 0, ret = -ENOMEM;
+ unsigned int i = 0, count;
struct cpufreq_stats *stats;
unsigned int alloc_size;
struct cpufreq_frequency_table *pos;
@@ -253,8 +253,7 @@ void cpufreq_stats_create_table(struct cpufreq_policy *policy)
stats->last_index = freq_table_get_index(stats, policy->cur);
policy->stats = stats;
- ret = sysfs_create_group(&policy->kobj, &stats_attr_group);
- if (!ret)
+ if (!sysfs_create_group(&policy->kobj, &stats_attr_group))
return;
/* We failed, release resources */
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index f0401064d7aa..bb4549959b11 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -121,9 +121,10 @@ struct sample {
* @max_pstate_physical:This is physical Max P state for a processor
* This can be higher than the max_pstate which can
* be limited by platform thermal design power limits
- * @scaling: Scaling factor to convert frequency to cpufreq
- * frequency units
+ * @perf_ctl_scaling: PERF_CTL P-state to frequency scaling factor
+ * @scaling: Scaling factor between performance and frequency
* @turbo_pstate: Max Turbo P state possible for this platform
+ * @min_freq: @min_pstate frequency in cpufreq units
* @max_freq: @max_pstate frequency in cpufreq units
* @turbo_freq: @turbo_pstate frequency in cpufreq units
*
@@ -134,8 +135,10 @@ struct pstate_data {
int min_pstate;
int max_pstate;
int max_pstate_physical;
+ int perf_ctl_scaling;
int scaling;
int turbo_pstate;
+ unsigned int min_freq;
unsigned int max_freq;
unsigned int turbo_freq;
};
@@ -366,7 +369,7 @@ static void intel_pstate_set_itmt_prio(int cpu)
}
}
-static int intel_pstate_get_cppc_guranteed(int cpu)
+static int intel_pstate_get_cppc_guaranteed(int cpu)
{
struct cppc_perf_caps cppc_perf;
int ret;
@@ -382,7 +385,7 @@ static int intel_pstate_get_cppc_guranteed(int cpu)
}
#else /* CONFIG_ACPI_CPPC_LIB */
-static void intel_pstate_set_itmt_prio(int cpu)
+static inline void intel_pstate_set_itmt_prio(int cpu)
{
}
#endif /* CONFIG_ACPI_CPPC_LIB */
@@ -467,6 +470,20 @@ static void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
acpi_processor_unregister_performance(policy->cpu);
}
+
+static bool intel_pstate_cppc_perf_valid(u32 perf, struct cppc_perf_caps *caps)
+{
+ return perf && perf <= caps->highest_perf && perf >= caps->lowest_perf;
+}
+
+static bool intel_pstate_cppc_perf_caps(struct cpudata *cpu,
+ struct cppc_perf_caps *caps)
+{
+ if (cppc_get_perf_caps(cpu->cpu, caps))
+ return false;
+
+ return caps->highest_perf && caps->lowest_perf <= caps->highest_perf;
+}
#else /* CONFIG_ACPI */
static inline void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
{
@@ -483,12 +500,146 @@ static inline bool intel_pstate_acpi_pm_profile_server(void)
#endif /* CONFIG_ACPI */
#ifndef CONFIG_ACPI_CPPC_LIB
-static int intel_pstate_get_cppc_guranteed(int cpu)
+static inline int intel_pstate_get_cppc_guaranteed(int cpu)
{
return -ENOTSUPP;
}
#endif /* CONFIG_ACPI_CPPC_LIB */
+static void intel_pstate_hybrid_hwp_perf_ctl_parity(struct cpudata *cpu)
+{
+ pr_debug("CPU%d: Using PERF_CTL scaling for HWP\n", cpu->cpu);
+
+ cpu->pstate.scaling = cpu->pstate.perf_ctl_scaling;
+}
+
+/**
+ * intel_pstate_hybrid_hwp_calibrate - Calibrate HWP performance levels.
+ * @cpu: Target CPU.
+ *
+ * On hybrid processors, HWP may expose more performance levels than there are
+ * P-states accessible through the PERF_CTL interface. If that happens, the
+ * scaling factor between HWP performance levels and CPU frequency will be less
+ * than the scaling factor between P-state values and CPU frequency.
+ *
+ * In that case, the scaling factor between HWP performance levels and CPU
+ * frequency needs to be determined which can be done with the help of the
+ * observation that certain HWP performance levels should correspond to certain
+ * P-states, like for example the HWP highest performance should correspond
+ * to the maximum turbo P-state of the CPU.
+ */
+static void intel_pstate_hybrid_hwp_calibrate(struct cpudata *cpu)
+{
+ int perf_ctl_max_phys = cpu->pstate.max_pstate_physical;
+ int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling;
+ int perf_ctl_turbo = pstate_funcs.get_turbo();
+ int turbo_freq = perf_ctl_turbo * perf_ctl_scaling;
+ int perf_ctl_max = pstate_funcs.get_max();
+ int max_freq = perf_ctl_max * perf_ctl_scaling;
+ int scaling = INT_MAX;
+ int freq;
+
+ pr_debug("CPU%d: perf_ctl_max_phys = %d\n", cpu->cpu, perf_ctl_max_phys);
+ pr_debug("CPU%d: perf_ctl_max = %d\n", cpu->cpu, perf_ctl_max);
+ pr_debug("CPU%d: perf_ctl_turbo = %d\n", cpu->cpu, perf_ctl_turbo);
+ pr_debug("CPU%d: perf_ctl_scaling = %d\n", cpu->cpu, perf_ctl_scaling);
+
+ pr_debug("CPU%d: HWP_CAP guaranteed = %d\n", cpu->cpu, cpu->pstate.max_pstate);
+ pr_debug("CPU%d: HWP_CAP highest = %d\n", cpu->cpu, cpu->pstate.turbo_pstate);
+
+#ifdef CONFIG_ACPI
+ if (IS_ENABLED(CONFIG_ACPI_CPPC_LIB)) {
+ struct cppc_perf_caps caps;
+
+ if (intel_pstate_cppc_perf_caps(cpu, &caps)) {
+ if (intel_pstate_cppc_perf_valid(caps.nominal_perf, &caps)) {
+ pr_debug("CPU%d: Using CPPC nominal\n", cpu->cpu);
+
+ /*
+ * If the CPPC nominal performance is valid, it
+ * can be assumed to correspond to cpu_khz.
+ */
+ if (caps.nominal_perf == perf_ctl_max_phys) {
+ intel_pstate_hybrid_hwp_perf_ctl_parity(cpu);
+ return;
+ }
+ scaling = DIV_ROUND_UP(cpu_khz, caps.nominal_perf);
+ } else if (intel_pstate_cppc_perf_valid(caps.guaranteed_perf, &caps)) {
+ pr_debug("CPU%d: Using CPPC guaranteed\n", cpu->cpu);
+
+ /*
+ * If the CPPC guaranteed performance is valid,
+ * it can be assumed to correspond to max_freq.
+ */
+ if (caps.guaranteed_perf == perf_ctl_max) {
+ intel_pstate_hybrid_hwp_perf_ctl_parity(cpu);
+ return;
+ }
+ scaling = DIV_ROUND_UP(max_freq, caps.guaranteed_perf);
+ }
+ }
+ }
+#endif
+ /*
+ * If using the CPPC data to compute the HWP-to-frequency scaling factor
+ * doesn't work, use the HWP_CAP gauranteed perf for this purpose with
+ * the assumption that it corresponds to max_freq.
+ */
+ if (scaling > perf_ctl_scaling) {
+ pr_debug("CPU%d: Using HWP_CAP guaranteed\n", cpu->cpu);
+
+ if (cpu->pstate.max_pstate == perf_ctl_max) {
+ intel_pstate_hybrid_hwp_perf_ctl_parity(cpu);
+ return;
+ }
+ scaling = DIV_ROUND_UP(max_freq, cpu->pstate.max_pstate);
+ if (scaling > perf_ctl_scaling) {
+ /*
+ * This should not happen, because it would mean that
+ * the number of HWP perf levels was less than the
+ * number of P-states, so use the PERF_CTL scaling in
+ * that case.
+ */
+ pr_debug("CPU%d: scaling (%d) out of range\n", cpu->cpu,
+ scaling);
+
+ intel_pstate_hybrid_hwp_perf_ctl_parity(cpu);
+ return;
+ }
+ }
+
+ /*
+ * If the product of the HWP performance scaling factor obtained above
+ * and the HWP_CAP highest performance is greater than the maximum turbo
+ * frequency corresponding to the pstate_funcs.get_turbo() return value,
+ * the scaling factor is too high, so recompute it so that the HWP_CAP
+ * highest performance corresponds to the maximum turbo frequency.
+ */
+ if (turbo_freq < cpu->pstate.turbo_pstate * scaling) {
+ pr_debug("CPU%d: scaling too high (%d)\n", cpu->cpu, scaling);
+
+ cpu->pstate.turbo_freq = turbo_freq;
+ scaling = DIV_ROUND_UP(turbo_freq, cpu->pstate.turbo_pstate);
+ }
+
+ cpu->pstate.scaling = scaling;
+
+ pr_debug("CPU%d: HWP-to-frequency scaling factor: %d\n", cpu->cpu, scaling);
+
+ cpu->pstate.max_freq = rounddown(cpu->pstate.max_pstate * scaling,
+ perf_ctl_scaling);
+
+ freq = perf_ctl_max_phys * perf_ctl_scaling;
+ cpu->pstate.max_pstate_physical = DIV_ROUND_UP(freq, scaling);
+
+ cpu->pstate.min_freq = cpu->pstate.min_pstate * perf_ctl_scaling;
+ /*
+ * Cast the min P-state value retrieved via pstate_funcs.get_min() to
+ * the effective range of HWP performance levels.
+ */
+ cpu->pstate.min_pstate = DIV_ROUND_UP(cpu->pstate.min_freq, scaling);
+}
+
static inline void update_turbo_state(void)
{
u64 misc_en;
@@ -795,19 +946,22 @@ cpufreq_freq_attr_rw(energy_performance_preference);
static ssize_t show_base_frequency(struct cpufreq_policy *policy, char *buf)
{
- struct cpudata *cpu;
- u64 cap;
- int ratio;
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
+ int ratio, freq;
- ratio = intel_pstate_get_cppc_guranteed(policy->cpu);
+ ratio = intel_pstate_get_cppc_guaranteed(policy->cpu);
if (ratio <= 0) {
+ u64 cap;
+
rdmsrl_on_cpu(policy->cpu, MSR_HWP_CAPABILITIES, &cap);
ratio = HWP_GUARANTEED_PERF(cap);
}
- cpu = all_cpu_data[policy->cpu];
+ freq = ratio * cpu->pstate.scaling;
+ if (cpu->pstate.scaling != cpu->pstate.perf_ctl_scaling)
+ freq = rounddown(freq, cpu->pstate.perf_ctl_scaling);
- return sprintf(buf, "%d\n", ratio * cpu->pstate.scaling);
+ return sprintf(buf, "%d\n", freq);
}
cpufreq_freq_attr_ro(base_frequency);
@@ -831,9 +985,20 @@ static void __intel_pstate_get_hwp_cap(struct cpudata *cpu)
static void intel_pstate_get_hwp_cap(struct cpudata *cpu)
{
+ int scaling = cpu->pstate.scaling;
+
__intel_pstate_get_hwp_cap(cpu);
- cpu->pstate.max_freq = cpu->pstate.max_pstate * cpu->pstate.scaling;
- cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
+
+ cpu->pstate.max_freq = cpu->pstate.max_pstate * scaling;
+ cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * scaling;
+ if (scaling != cpu->pstate.perf_ctl_scaling) {
+ int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling;
+
+ cpu->pstate.max_freq = rounddown(cpu->pstate.max_freq,
+ perf_ctl_scaling);
+ cpu->pstate.turbo_freq = rounddown(cpu->pstate.turbo_freq,
+ perf_ctl_scaling);
+ }
}
static void intel_pstate_hwp_set(unsigned int cpu)
@@ -1365,8 +1530,6 @@ define_one_global_rw(energy_efficiency);
static struct attribute *intel_pstate_attributes[] = {
&status.attr,
&no_turbo.attr,
- &turbo_pct.attr,
- &num_pstates.attr,
NULL
};
@@ -1391,6 +1554,14 @@ static void __init intel_pstate_sysfs_expose_params(void)
if (WARN_ON(rc))
return;
+ if (!boot_cpu_has(X86_FEATURE_HYBRID_CPU)) {
+ rc = sysfs_create_file(intel_pstate_kobject, &turbo_pct.attr);
+ WARN_ON(rc);
+
+ rc = sysfs_create_file(intel_pstate_kobject, &num_pstates.attr);
+ WARN_ON(rc);
+ }
+
/*
* If per cpu limits are enforced there are no global limits, so
* return without creating max/min_perf_pct attributes
@@ -1417,6 +1588,11 @@ static void __init intel_pstate_sysfs_remove(void)
sysfs_remove_group(intel_pstate_kobject, &intel_pstate_attr_group);
+ if (!boot_cpu_has(X86_FEATURE_HYBRID_CPU)) {
+ sysfs_remove_file(intel_pstate_kobject, &num_pstates.attr);
+ sysfs_remove_file(intel_pstate_kobject, &turbo_pct.attr);
+ }
+
if (!per_cpu_limits) {
sysfs_remove_file(intel_pstate_kobject, &max_perf_pct.attr);
sysfs_remove_file(intel_pstate_kobject, &min_perf_pct.attr);
@@ -1713,19 +1889,33 @@ static void intel_pstate_max_within_limits(struct cpudata *cpu)
static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
{
+ bool hybrid_cpu = boot_cpu_has(X86_FEATURE_HYBRID_CPU);
+ int perf_ctl_max_phys = pstate_funcs.get_max_physical();
+ int perf_ctl_scaling = hybrid_cpu ? cpu_khz / perf_ctl_max_phys :
+ pstate_funcs.get_scaling();
+
cpu->pstate.min_pstate = pstate_funcs.get_min();
- cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical();
- cpu->pstate.scaling = pstate_funcs.get_scaling();
+ cpu->pstate.max_pstate_physical = perf_ctl_max_phys;
+ cpu->pstate.perf_ctl_scaling = perf_ctl_scaling;
if (hwp_active && !hwp_mode_bdw) {
__intel_pstate_get_hwp_cap(cpu);
+
+ if (hybrid_cpu)
+ intel_pstate_hybrid_hwp_calibrate(cpu);
+ else
+ cpu->pstate.scaling = perf_ctl_scaling;
} else {
+ cpu->pstate.scaling = perf_ctl_scaling;
cpu->pstate.max_pstate = pstate_funcs.get_max();
cpu->pstate.turbo_pstate = pstate_funcs.get_turbo();
}
- cpu->pstate.max_freq = cpu->pstate.max_pstate * cpu->pstate.scaling;
- cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
+ if (cpu->pstate.scaling == perf_ctl_scaling) {
+ cpu->pstate.min_freq = cpu->pstate.min_pstate * perf_ctl_scaling;
+ cpu->pstate.max_freq = cpu->pstate.max_pstate * perf_ctl_scaling;
+ cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * perf_ctl_scaling;
+ }
if (pstate_funcs.get_aperf_mperf_shift)
cpu->aperf_mperf_shift = pstate_funcs.get_aperf_mperf_shift();
@@ -2087,6 +2277,8 @@ static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
X86_MATCH(ATOM_GOLDMONT, core_funcs),
X86_MATCH(ATOM_GOLDMONT_PLUS, core_funcs),
X86_MATCH(SKYLAKE_X, core_funcs),
+ X86_MATCH(COMETLAKE, core_funcs),
+ X86_MATCH(ICELAKE_X, core_funcs),
{}
};
MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
@@ -2195,23 +2387,34 @@ static void intel_pstate_update_perf_limits(struct cpudata *cpu,
unsigned int policy_min,
unsigned int policy_max)
{
- int scaling = cpu->pstate.scaling;
+ int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling;
int32_t max_policy_perf, min_policy_perf;
+ max_policy_perf = policy_max / perf_ctl_scaling;
+ if (policy_max == policy_min) {
+ min_policy_perf = max_policy_perf;
+ } else {
+ min_policy_perf = policy_min / perf_ctl_scaling;
+ min_policy_perf = clamp_t(int32_t, min_policy_perf,
+ 0, max_policy_perf);
+ }
+
/*
* HWP needs some special consideration, because HWP_REQUEST uses
* abstract values to represent performance rather than pure ratios.
*/
- if (hwp_active)
+ if (hwp_active) {
intel_pstate_get_hwp_cap(cpu);
- max_policy_perf = policy_max / scaling;
- if (policy_max == policy_min) {
- min_policy_perf = max_policy_perf;
- } else {
- min_policy_perf = policy_min / scaling;
- min_policy_perf = clamp_t(int32_t, min_policy_perf,
- 0, max_policy_perf);
+ if (cpu->pstate.scaling != perf_ctl_scaling) {
+ int scaling = cpu->pstate.scaling;
+ int freq;
+
+ freq = max_policy_perf * perf_ctl_scaling;
+ max_policy_perf = DIV_ROUND_UP(freq, scaling);
+ freq = min_policy_perf * perf_ctl_scaling;
+ min_policy_perf = DIV_ROUND_UP(freq, scaling);
+ }
}
pr_debug("cpu:%d min_policy_perf:%d max_policy_perf:%d\n",
@@ -2329,7 +2532,7 @@ static int intel_pstate_verify_policy(struct cpufreq_policy_data *policy)
return 0;
}
-static int intel_pstate_cpu_offline(struct cpufreq_policy *policy)
+static int intel_cpufreq_cpu_offline(struct cpufreq_policy *policy)
{
struct cpudata *cpu = all_cpu_data[policy->cpu];
@@ -2374,11 +2577,11 @@ static int intel_pstate_cpu_online(struct cpufreq_policy *policy)
return 0;
}
-static void intel_pstate_stop_cpu(struct cpufreq_policy *policy)
+static int intel_pstate_cpu_offline(struct cpufreq_policy *policy)
{
- pr_debug("CPU %d stopping\n", policy->cpu);
-
intel_pstate_clear_update_util_hook(policy->cpu);
+
+ return intel_cpufreq_cpu_offline(policy);
}
static int intel_pstate_cpu_exit(struct cpufreq_policy *policy)
@@ -2405,7 +2608,7 @@ static int __intel_pstate_cpu_init(struct cpufreq_policy *policy)
cpu->min_perf_ratio = 0;
/* cpuinfo and default policy values */
- policy->cpuinfo.min_freq = cpu->pstate.min_pstate * cpu->pstate.scaling;
+ policy->cpuinfo.min_freq = cpu->pstate.min_freq;
update_turbo_state();
global.turbo_disabled_mf = global.turbo_disabled;
policy->cpuinfo.max_freq = global.turbo_disabled ?
@@ -2451,7 +2654,6 @@ static struct cpufreq_driver intel_pstate = {
.resume = intel_pstate_resume,
.init = intel_pstate_cpu_init,
.exit = intel_pstate_cpu_exit,
- .stop_cpu = intel_pstate_stop_cpu,
.offline = intel_pstate_cpu_offline,
.online = intel_pstate_cpu_online,
.update_limits = intel_pstate_update_limits,
@@ -2753,7 +2955,7 @@ static struct cpufreq_driver intel_cpufreq = {
.fast_switch = intel_cpufreq_fast_switch,
.init = intel_cpufreq_cpu_init,
.exit = intel_cpufreq_cpu_exit,
- .offline = intel_pstate_cpu_offline,
+ .offline = intel_cpufreq_cpu_offline,
.online = intel_pstate_cpu_online,
.suspend = intel_pstate_suspend,
.resume = intel_pstate_resume,
@@ -3033,6 +3235,14 @@ static const struct x86_cpu_id hwp_support_ids[] __initconst = {
{}
};
+static bool intel_pstate_hwp_is_enabled(void)
+{
+ u64 value;
+
+ rdmsrl(MSR_PM_ENABLE, value);
+ return !!(value & 0x1);
+}
+
static int __init intel_pstate_init(void)
{
const struct x86_cpu_id *id;
@@ -3051,8 +3261,12 @@ static int __init intel_pstate_init(void)
* Avoid enabling HWP for processors without EPP support,
* because that means incomplete HWP implementation which is a
* corner case and supporting it is generally problematic.
+ *
+ * If HWP is enabled already, though, there is no choice but to
+ * deal with it.
*/
- if (!no_hwp && boot_cpu_has(X86_FEATURE_HWP_EPP)) {
+ if ((!no_hwp && boot_cpu_has(X86_FEATURE_HWP_EPP)) ||
+ intel_pstate_hwp_is_enabled()) {
hwp_active++;
hwp_mode_bdw = id->driver_data;
intel_pstate.attr = hwp_cpufreq_attrs;
@@ -3123,6 +3337,8 @@ hwp_cpu_matched:
}
pr_info("HWP enabled\n");
+ } else if (boot_cpu_has(X86_FEATURE_HYBRID_CPU)) {
+ pr_warn("Problematic setup: Hybrid processor with disabled HWP\n");
}
return 0;
diff --git a/drivers/cpufreq/loongson2_cpufreq.c b/drivers/cpufreq/loongson2_cpufreq.c
index d05e761d9572..afc59b292153 100644
--- a/drivers/cpufreq/loongson2_cpufreq.c
+++ b/drivers/cpufreq/loongson2_cpufreq.c
@@ -16,7 +16,6 @@
#include <linux/cpufreq.h>
#include <linux/module.h>
#include <linux/err.h>
-#include <linux/sched.h> /* set_cpus_allowed() */
#include <linux/delay.h>
#include <linux/platform_device.h>
diff --git a/drivers/cpufreq/sc520_freq.c b/drivers/cpufreq/sc520_freq.c
index 73a208559fe2..330c8d6cf93c 100644
--- a/drivers/cpufreq/sc520_freq.c
+++ b/drivers/cpufreq/sc520_freq.c
@@ -42,6 +42,7 @@ static unsigned int sc520_freq_get_cpu_frequency(unsigned int cpu)
default:
pr_err("error: cpuctl register has unexpected value %02x\n",
clockspeed_reg);
+ fallthrough;
case 0x01:
return 100000;
case 0x02:
diff --git a/drivers/cpufreq/sh-cpufreq.c b/drivers/cpufreq/sh-cpufreq.c
index 0ac265d47ef0..1a251e635ebd 100644
--- a/drivers/cpufreq/sh-cpufreq.c
+++ b/drivers/cpufreq/sh-cpufreq.c
@@ -23,7 +23,6 @@
#include <linux/cpumask.h>
#include <linux/cpu.h>
#include <linux/smp.h>
-#include <linux/sched.h> /* set_cpus_allowed() */
#include <linux/clk.h>
#include <linux/percpu.h>
#include <linux/sh_clk.h>