summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.txt2
-rw-r--r--arch/arm/include/asm/topology.h5
-rw-r--r--arch/arm64/include/asm/topology.h2
-rw-r--r--arch/arm64/kernel/topology.c9
-rw-r--r--drivers/base/arch_topology.c15
-rw-r--r--drivers/cpufreq/Kconfig.arm2
-rw-r--r--drivers/cpufreq/armada-37xx-cpufreq.c6
-rw-r--r--drivers/cpufreq/cpufreq-dt-platdev.c1
-rw-r--r--drivers/cpufreq/cpufreq-dt.c296
-rw-r--r--drivers/cpufreq/cpufreq.c45
-rw-r--r--drivers/cpufreq/cpufreq_stats.c105
-rw-r--r--drivers/cpufreq/imx6q-cpufreq.c10
-rw-r--r--drivers/cpufreq/qcom-cpufreq-hw.c144
-rw-r--r--drivers/cpufreq/s5pv210-cpufreq.c31
-rw-r--r--drivers/cpufreq/scmi-cpufreq.c12
-rw-r--r--drivers/cpufreq/scpi-cpufreq.c6
-rw-r--r--drivers/cpufreq/sti-cpufreq.c6
-rw-r--r--drivers/cpufreq/tegra186-cpufreq.c30
-rw-r--r--drivers/cpufreq/vexpress-spc-cpufreq.c12
-rw-r--r--include/linux/arch_topology.h6
-rw-r--r--include/linux/cpufreq.h15
-rw-r--r--kernel/sched/cpufreq_schedutil.c18
22 files changed, 453 insertions, 325 deletions
diff --git a/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.txt b/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.txt
index 33856947c561..9299028ee712 100644
--- a/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.txt
+++ b/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.txt
@@ -8,7 +8,7 @@ Properties:
- compatible
Usage: required
Value type: <string>
- Definition: must be "qcom,cpufreq-hw".
+ Definition: must be "qcom,cpufreq-hw" or "qcom,cpufreq-epss".
- clocks
Usage: required
diff --git a/arch/arm/include/asm/topology.h b/arch/arm/include/asm/topology.h
index e0593cf095d0..470299ee2fba 100644
--- a/arch/arm/include/asm/topology.h
+++ b/arch/arm/include/asm/topology.h
@@ -7,8 +7,13 @@
#include <linux/cpumask.h>
#include <linux/arch_topology.h>
+/* big.LITTLE switcher is incompatible with frequency invariance */
+#ifndef CONFIG_BL_SWITCHER
/* Replace task scheduler's default frequency-invariant accounting */
+#define arch_set_freq_scale topology_set_freq_scale
#define arch_scale_freq_capacity topology_get_freq_scale
+#define arch_scale_freq_invariant topology_scale_freq_invariant
+#endif
/* Replace task scheduler's default cpu-invariant accounting */
#define arch_scale_cpu_capacity topology_get_cpu_scale
diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h
index e042f6527981..11a465243f66 100644
--- a/arch/arm64/include/asm/topology.h
+++ b/arch/arm64/include/asm/topology.h
@@ -26,7 +26,9 @@ void topology_scale_freq_tick(void);
#endif /* CONFIG_ARM64_AMU_EXTN */
/* Replace task scheduler's default frequency-invariant accounting */
+#define arch_set_freq_scale topology_set_freq_scale
#define arch_scale_freq_capacity topology_get_freq_scale
+#define arch_scale_freq_invariant topology_scale_freq_invariant
/* Replace task scheduler's default cpu-invariant accounting */
#define arch_scale_cpu_capacity topology_get_cpu_scale
diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c
index 0801a0f3c156..4064d39bb66d 100644
--- a/arch/arm64/kernel/topology.c
+++ b/arch/arm64/kernel/topology.c
@@ -246,6 +246,13 @@ static int __init init_amu_fie(void)
static_branch_enable(&amu_fie_key);
}
+ /*
+ * If the system is not fully invariant after AMU init, disable
+ * partial use of counters for frequency invariance.
+ */
+ if (!topology_scale_freq_invariant())
+ static_branch_disable(&amu_fie_key);
+
free_valid_mask:
free_cpumask_var(valid_cpus);
@@ -253,7 +260,7 @@ free_valid_mask:
}
late_initcall_sync(init_amu_fie);
-bool arch_freq_counters_available(struct cpumask *cpus)
+bool arch_freq_counters_available(const struct cpumask *cpus)
{
return amu_freq_invariant() &&
cpumask_subset(cpus, amu_fie_cpus);
diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c
index 75f72d684294..c1a9e2fb634e 100644
--- a/drivers/base/arch_topology.c
+++ b/drivers/base/arch_topology.c
@@ -21,18 +21,27 @@
#include <linux/sched.h>
#include <linux/smp.h>
-__weak bool arch_freq_counters_available(struct cpumask *cpus)
+bool topology_scale_freq_invariant(void)
+{
+ return cpufreq_supports_freq_invariance() ||
+ arch_freq_counters_available(cpu_online_mask);
+}
+
+__weak bool arch_freq_counters_available(const struct cpumask *cpus)
{
return false;
}
DEFINE_PER_CPU(unsigned long, freq_scale) = SCHED_CAPACITY_SCALE;
-void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq,
- unsigned long max_freq)
+void topology_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq,
+ unsigned long max_freq)
{
unsigned long scale;
int i;
+ if (WARN_ON_ONCE(!cur_freq || !max_freq))
+ return;
+
/*
* If the use of counters for FIE is enabled, just return as we don't
* want to update the scale factor with information from CPUFREQ.
diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm
index cb72fb507d57..bf5830eb664f 100644
--- a/drivers/cpufreq/Kconfig.arm
+++ b/drivers/cpufreq/Kconfig.arm
@@ -283,7 +283,7 @@ config ARM_SPEAR_CPUFREQ
config ARM_STI_CPUFREQ
tristate "STi CPUFreq support"
- depends on SOC_STIH407
+ depends on CPUFREQ_DT && SOC_STIH407
help
This driver uses the generic OPP framework to match the running
platform with a predefined set of suitable values. If not provided
diff --git a/drivers/cpufreq/armada-37xx-cpufreq.c b/drivers/cpufreq/armada-37xx-cpufreq.c
index df1c941260d1..b4af4094309b 100644
--- a/drivers/cpufreq/armada-37xx-cpufreq.c
+++ b/drivers/cpufreq/armada-37xx-cpufreq.c
@@ -484,6 +484,12 @@ remove_opp:
/* late_initcall, to guarantee the driver is loaded after A37xx clock driver */
late_initcall(armada37xx_cpufreq_driver_init);
+static const struct of_device_id __maybe_unused armada37xx_cpufreq_of_match[] = {
+ { .compatible = "marvell,armada-3700-nb-pm" },
+ { },
+};
+MODULE_DEVICE_TABLE(of, armada37xx_cpufreq_of_match);
+
MODULE_AUTHOR("Gregory CLEMENT <gregory.clement@free-electrons.com>");
MODULE_DESCRIPTION("Armada 37xx cpufreq driver");
MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c
index 7d01df7bfa6c..3776d960f405 100644
--- a/drivers/cpufreq/cpufreq-dt-platdev.c
+++ b/drivers/cpufreq/cpufreq-dt-platdev.c
@@ -137,6 +137,7 @@ static const struct of_device_id blacklist[] __initconst = {
{ .compatible = "st,stih407", },
{ .compatible = "st,stih410", },
+ { .compatible = "st,stih418", },
{ .compatible = "sigma,tango4", },
diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c
index 944d7b45afe9..e363ae04aac6 100644
--- a/drivers/cpufreq/cpufreq-dt.c
+++ b/drivers/cpufreq/cpufreq-dt.c
@@ -13,6 +13,7 @@
#include <linux/cpufreq.h>
#include <linux/cpumask.h>
#include <linux/err.h>
+#include <linux/list.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/pm_opp.h>
@@ -24,32 +25,41 @@
#include "cpufreq-dt.h"
struct private_data {
- struct opp_table *opp_table;
+ struct list_head node;
+
+ cpumask_var_t cpus;
struct device *cpu_dev;
- const char *reg_name;
+ struct opp_table *opp_table;
+ struct opp_table *reg_opp_table;
bool have_static_opps;
};
+static LIST_HEAD(priv_list);
+
static struct freq_attr *cpufreq_dt_attr[] = {
&cpufreq_freq_attr_scaling_available_freqs,
NULL, /* Extra space for boost-attr if required */
NULL,
};
+static struct private_data *cpufreq_dt_find_data(int cpu)
+{
+ struct private_data *priv;
+
+ list_for_each_entry(priv, &priv_list, node) {
+ if (cpumask_test_cpu(cpu, priv->cpus))
+ return priv;
+ }
+
+ return NULL;
+}
+
static int set_target(struct cpufreq_policy *policy, unsigned int index)
{
struct private_data *priv = policy->driver_data;
unsigned long freq = policy->freq_table[index].frequency;
- int ret;
-
- ret = dev_pm_opp_set_rate(priv->cpu_dev, freq * 1000);
-
- if (!ret) {
- arch_set_freq_scale(policy->related_cpus, freq,
- policy->cpuinfo.max_freq);
- }
- return ret;
+ return dev_pm_opp_set_rate(priv->cpu_dev, freq * 1000);
}
/*
@@ -90,83 +100,24 @@ node_put:
return name;
}
-static int resources_available(void)
-{
- struct device *cpu_dev;
- struct regulator *cpu_reg;
- struct clk *cpu_clk;
- int ret = 0;
- const char *name;
-
- cpu_dev = get_cpu_device(0);
- if (!cpu_dev) {
- pr_err("failed to get cpu0 device\n");
- return -ENODEV;
- }
-
- cpu_clk = clk_get(cpu_dev, NULL);
- ret = PTR_ERR_OR_ZERO(cpu_clk);
- if (ret) {
- /*
- * If cpu's clk node is present, but clock is not yet
- * registered, we should try defering probe.
- */
- if (ret == -EPROBE_DEFER)
- dev_dbg(cpu_dev, "clock not ready, retry\n");
- else
- dev_err(cpu_dev, "failed to get clock: %d\n", ret);
-
- return ret;
- }
-
- clk_put(cpu_clk);
-
- ret = dev_pm_opp_of_find_icc_paths(cpu_dev, NULL);
- if (ret)
- return ret;
-
- name = find_supply_name(cpu_dev);
- /* Platform doesn't require regulator */
- if (!name)
- return 0;
-
- cpu_reg = regulator_get_optional(cpu_dev, name);
- ret = PTR_ERR_OR_ZERO(cpu_reg);
- if (ret) {
- /*
- * If cpu's regulator supply node is present, but regulator is
- * not yet registered, we should try defering probe.
- */
- if (ret == -EPROBE_DEFER)
- dev_dbg(cpu_dev, "cpu0 regulator not ready, retry\n");
- else
- dev_dbg(cpu_dev, "no regulator for cpu0: %d\n", ret);
-
- return ret;
- }
-
- regulator_put(cpu_reg);
- return 0;
-}
-
static int cpufreq_init(struct cpufreq_policy *policy)
{
struct cpufreq_frequency_table *freq_table;
- struct opp_table *opp_table = NULL;
struct private_data *priv;
struct device *cpu_dev;
struct clk *cpu_clk;
unsigned int transition_latency;
- bool fallback = false;
- const char *name;
int ret;
- cpu_dev = get_cpu_device(policy->cpu);
- if (!cpu_dev) {
- pr_err("failed to get cpu%d device\n", policy->cpu);
+ priv = cpufreq_dt_find_data(policy->cpu);
+ if (!priv) {
+ pr_err("failed to find data for cpu%d\n", policy->cpu);
return -ENODEV;
}
+ cpu_dev = priv->cpu_dev;
+ cpumask_copy(policy->cpus, priv->cpus);
+
cpu_clk = clk_get(cpu_dev, NULL);
if (IS_ERR(cpu_clk)) {
ret = PTR_ERR(cpu_clk);
@@ -174,45 +125,6 @@ static int cpufreq_init(struct cpufreq_policy *policy)
return ret;
}
- /* Get OPP-sharing information from "operating-points-v2" bindings */
- ret = dev_pm_opp_of_get_sharing_cpus(cpu_dev, policy->cpus);
- if (ret) {
- if (ret != -ENOENT)
- goto out_put_clk;
-
- /*
- * operating-points-v2 not supported, fallback to old method of
- * finding shared-OPPs for backward compatibility if the
- * platform hasn't set sharing CPUs.
- */
- if (dev_pm_opp_get_sharing_cpus(cpu_dev, policy->cpus))
- fallback = true;
- }
-
- /*
- * OPP layer will be taking care of regulators now, but it needs to know
- * the name of the regulator first.
- */
- name = find_supply_name(cpu_dev);
- if (name) {
- opp_table = dev_pm_opp_set_regulators(cpu_dev, &name, 1);
- if (IS_ERR(opp_table)) {
- ret = PTR_ERR(opp_table);
- dev_err(cpu_dev, "Failed to set regulator for cpu%d: %d\n",
- policy->cpu, ret);
- goto out_put_clk;
- }
- }
-
- priv = kzalloc(sizeof(*priv), GFP_KERNEL);
- if (!priv) {
- ret = -ENOMEM;
- goto out_put_regulator;
- }
-
- priv->reg_name = name;
- priv->opp_table = opp_table;
-
/*
* Initialize OPP tables for all policy->cpus. They will be shared by
* all CPUs which have marked their CPUs shared with OPP bindings.
@@ -232,31 +144,17 @@ static int cpufreq_init(struct cpufreq_policy *policy)
*/
ret = dev_pm_opp_get_opp_count(cpu_dev);
if (ret <= 0) {
- dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n");
- ret = -EPROBE_DEFER;
+ dev_err(cpu_dev, "OPP table can't be empty\n");
+ ret = -ENODEV;
goto out_free_opp;
}
- if (fallback) {
- cpumask_setall(policy->cpus);
-
- /*
- * OPP tables are initialized only for policy->cpu, do it for
- * others as well.
- */
- ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);
- if (ret)
- dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n",
- __func__, ret);
- }
-
ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table);
if (ret) {
dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret);
goto out_free_opp;
}
- priv->cpu_dev = cpu_dev;
policy->driver_data = priv;
policy->clk = cpu_clk;
policy->freq_table = freq_table;
@@ -288,11 +186,6 @@ out_free_cpufreq_table:
out_free_opp:
if (priv->have_static_opps)
dev_pm_opp_of_cpumask_remove_table(policy->cpus);
- kfree(priv);
-out_put_regulator:
- if (name)
- dev_pm_opp_put_regulators(opp_table);
-out_put_clk:
clk_put(cpu_clk);
return ret;
@@ -320,12 +213,7 @@ static int cpufreq_exit(struct cpufreq_policy *policy)
dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table);
if (priv->have_static_opps)
dev_pm_opp_of_cpumask_remove_table(policy->related_cpus);
- if (priv->reg_name)
- dev_pm_opp_put_regulators(priv->opp_table);
-
clk_put(policy->clk);
- kfree(priv);
-
return 0;
}
@@ -344,21 +232,119 @@ static struct cpufreq_driver dt_cpufreq_driver = {
.suspend = cpufreq_generic_suspend,
};
-static int dt_cpufreq_probe(struct platform_device *pdev)
+static int dt_cpufreq_early_init(struct device *dev, int cpu)
{
- struct cpufreq_dt_platform_data *data = dev_get_platdata(&pdev->dev);
+ struct private_data *priv;
+ struct device *cpu_dev;
+ const char *reg_name;
int ret;
+ /* Check if this CPU is already covered by some other policy */
+ if (cpufreq_dt_find_data(cpu))
+ return 0;
+
+ cpu_dev = get_cpu_device(cpu);
+ if (!cpu_dev)
+ return -EPROBE_DEFER;
+
+ priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ if (!alloc_cpumask_var(&priv->cpus, GFP_KERNEL))
+ return -ENOMEM;
+
+ priv->cpu_dev = cpu_dev;
+
+ /* Try to get OPP table early to ensure resources are available */
+ priv->opp_table = dev_pm_opp_get_opp_table(cpu_dev);
+ if (IS_ERR(priv->opp_table)) {
+ ret = PTR_ERR(priv->opp_table);
+ if (ret != -EPROBE_DEFER)
+ dev_err(cpu_dev, "failed to get OPP table: %d\n", ret);
+ goto free_cpumask;
+ }
+
/*
- * All per-cluster (CPUs sharing clock/voltages) initialization is done
- * from ->init(). In probe(), we just need to make sure that clk and
- * regulators are available. Else defer probe and retry.
- *
- * FIXME: Is checking this only for CPU0 sufficient ?
+ * OPP layer will be taking care of regulators now, but it needs to know
+ * the name of the regulator first.
*/
- ret = resources_available();
- if (ret)
- return ret;
+ reg_name = find_supply_name(cpu_dev);
+ if (reg_name) {
+ priv->reg_opp_table = dev_pm_opp_set_regulators(cpu_dev,
+ &reg_name, 1);
+ if (IS_ERR(priv->reg_opp_table)) {
+ ret = PTR_ERR(priv->reg_opp_table);
+ if (ret != -EPROBE_DEFER)
+ dev_err(cpu_dev, "failed to set regulators: %d\n",
+ ret);
+ goto put_table;
+ }
+ }
+
+ /* Find OPP sharing information so we can fill pri->cpus here */
+ /* Get OPP-sharing information from "operating-points-v2" bindings */
+ ret = dev_pm_opp_of_get_sharing_cpus(cpu_dev, priv->cpus);
+ if (ret) {
+ if (ret != -ENOENT)
+ goto put_reg;
+
+ /*
+ * operating-points-v2 not supported, fallback to all CPUs share
+ * OPP for backward compatibility if the platform hasn't set
+ * sharing CPUs.
+ */
+ if (dev_pm_opp_get_sharing_cpus(cpu_dev, priv->cpus)) {
+ cpumask_setall(priv->cpus);
+
+ /*
+ * OPP tables are initialized only for cpu, do it for
+ * others as well.
+ */
+ ret = dev_pm_opp_set_sharing_cpus(cpu_dev, priv->cpus);
+ if (ret)
+ dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n",
+ __func__, ret);
+ }
+ }
+
+ list_add(&priv->node, &priv_list);
+ return 0;
+
+put_reg:
+ if (priv->reg_opp_table)
+ dev_pm_opp_put_regulators(priv->reg_opp_table);
+put_table:
+ dev_pm_opp_put_opp_table(priv->opp_table);
+free_cpumask:
+ free_cpumask_var(priv->cpus);
+ return ret;
+}
+
+static void dt_cpufreq_release(void)
+{
+ struct private_data *priv, *tmp;
+
+ list_for_each_entry_safe(priv, tmp, &priv_list, node) {
+ if (priv->reg_opp_table)
+ dev_pm_opp_put_regulators(priv->reg_opp_table);
+ dev_pm_opp_put_opp_table(priv->opp_table);
+ free_cpumask_var(priv->cpus);
+ list_del(&priv->node);
+ }
+}
+
+static int dt_cpufreq_probe(struct platform_device *pdev)
+{
+ struct cpufreq_dt_platform_data *data = dev_get_platdata(&pdev->dev);
+ int ret, cpu;
+
+ /* Request resources early so we can return in case of -EPROBE_DEFER */
+ for_each_possible_cpu(cpu) {
+ ret = dt_cpufreq_early_init(&pdev->dev, cpu);
+ if (ret)
+ goto err;
+ }
if (data) {
if (data->have_governor_per_policy)
@@ -374,15 +360,21 @@ static int dt_cpufreq_probe(struct platform_device *pdev)
}
ret = cpufreq_register_driver(&dt_cpufreq_driver);
- if (ret)
+ if (ret) {
dev_err(&pdev->dev, "failed register driver: %d\n", ret);
+ goto err;
+ }
+ return 0;
+err:
+ dt_cpufreq_release();
return ret;
}
static int dt_cpufreq_remove(struct platform_device *pdev)
{
cpufreq_unregister_driver(&dt_cpufreq_driver);
+ dt_cpufreq_release();
return 0;
}
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index 47aa90f9a7c2..1877f5e2e5b0 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -61,6 +61,12 @@ static struct cpufreq_driver *cpufreq_driver;
static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
static DEFINE_RWLOCK(cpufreq_driver_lock);
+static DEFINE_STATIC_KEY_FALSE(cpufreq_freq_invariance);
+bool cpufreq_supports_freq_invariance(void)
+{
+ return static_branch_likely(&cpufreq_freq_invariance);
+}
+
/* Flag to suspend/resume CPUFreq governors */
static bool cpufreq_suspended;
@@ -154,12 +160,6 @@ u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
}
EXPORT_SYMBOL_GPL(get_cpu_idle_time);
-__weak void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq,
- unsigned long max_freq)
-{
-}
-EXPORT_SYMBOL_GPL(arch_set_freq_scale);
-
/*
* This is a generic cpufreq init() routine which can be used by cpufreq
* drivers of SMP systems. It will do following:
@@ -446,6 +446,10 @@ void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
cpufreq_notify_post_transition(policy, freqs, transition_failed);
+ arch_set_freq_scale(policy->related_cpus,
+ policy->cur,
+ policy->cpuinfo.max_freq);
+
policy->transition_ongoing = false;
policy->transition_task = NULL;
@@ -2056,9 +2060,26 @@ EXPORT_SYMBOL(cpufreq_unregister_notifier);
unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,
unsigned int target_freq)
{
+ unsigned int freq;
+ int cpu;
+
target_freq = clamp_val(target_freq, policy->min, policy->max);
+ freq = cpufreq_driver->fast_switch(policy, target_freq);
- return cpufreq_driver->fast_switch(policy, target_freq);
+ if (!freq)
+ return 0;
+
+ policy->cur = freq;
+ arch_set_freq_scale(policy->related_cpus, freq,
+ policy->cpuinfo.max_freq);
+ cpufreq_stats_record_transition(policy, freq);
+
+ if (trace_cpu_frequency_enabled()) {
+ for_each_cpu(cpu, policy->cpus)
+ trace_cpu_frequency(freq, cpu);
+ }
+
+ return freq;
}
EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch);
@@ -2710,6 +2731,15 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data)
cpufreq_driver = driver_data;
write_unlock_irqrestore(&cpufreq_driver_lock, flags);
+ /*
+ * Mark support for the scheduler's frequency invariance engine for
+ * drivers that implement target(), target_index() or fast_switch().
+ */
+ if (!cpufreq_driver->setpolicy) {
+ static_branch_enable_cpuslocked(&cpufreq_freq_invariance);
+ pr_debug("supports frequency invariance");
+ }
+
if (driver_data->setpolicy)
driver_data->flags |= CPUFREQ_CONST_LOOPS;
@@ -2779,6 +2809,7 @@ int cpufreq_unregister_driver(struct cpufreq_driver *driver)
cpus_read_lock();
subsys_interface_unregister(&cpufreq_interface);
remove_boost_sysfs_file();
+ static_branch_disable_cpuslocked(&cpufreq_freq_invariance);
cpuhp_remove_state_nocalls_cpuslocked(hp_online);
write_lock_irqsave(&cpufreq_driver_lock, flags);
diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c
index 94d959a8e954..6cd5c8ab5d49 100644
--- a/drivers/cpufreq/cpufreq_stats.c
+++ b/drivers/cpufreq/cpufreq_stats.c
@@ -19,64 +19,104 @@ struct cpufreq_stats {
unsigned int state_num;
unsigned int last_index;
u64 *time_in_state;
- spinlock_t lock;
unsigned int *freq_table;
unsigned int *trans_table;
+
+ /* Deferred reset */
+ unsigned int reset_pending;
+ unsigned long long reset_time;
};
-static void cpufreq_stats_update(struct cpufreq_stats *stats)
+static void cpufreq_stats_update(struct cpufreq_stats *stats,
+ unsigned long long time)
{
unsigned long long cur_time = get_jiffies_64();
- stats->time_in_state[stats->last_index] += cur_time - stats->last_time;
+ stats->time_in_state[stats->last_index] += cur_time - time;
stats->last_time = cur_time;
}
-static void cpufreq_stats_clear_table(struct cpufreq_stats *stats)
+static void cpufreq_stats_reset_table(struct cpufreq_stats *stats)
{
unsigned int count = stats->max_state;
- spin_lock(&stats->lock);
memset(stats->time_in_state, 0, count * sizeof(u64));
memset(stats->trans_table, 0, count * count * sizeof(int));
stats->last_time = get_jiffies_64();
stats->total_trans = 0;
- spin_unlock(&stats->lock);
+
+ /* Adjust for the time elapsed since reset was requested */
+ WRITE_ONCE(stats->reset_pending, 0);
+ /*
+ * Prevent the reset_time read from being reordered before the
+ * reset_pending accesses in cpufreq_stats_record_transition().
+ */
+ smp_rmb();
+ cpufreq_stats_update(stats, READ_ONCE(stats->reset_time));
}
static ssize_t show_total_trans(struct cpufreq_policy *policy, char *buf)
{
- return sprintf(buf, "%d\n", policy->stats->total_trans);
+ struct cpufreq_stats *stats = policy->stats;
+
+ if (READ_ONCE(stats->reset_pending))
+ return sprintf(buf, "%d\n", 0);
+ else
+ return sprintf(buf, "%u\n", stats->total_trans);
}
cpufreq_freq_attr_ro(total_trans);
static ssize_t show_time_in_state(struct cpufreq_policy *policy, char *buf)
{
struct cpufreq_stats *stats = policy->stats;
+ bool pending = READ_ONCE(stats->reset_pending);
+ unsigned long long time;
ssize_t len = 0;
int i;
- if (policy->fast_switch_enabled)
- return 0;
-
- spin_lock(&stats->lock);
- cpufreq_stats_update(stats);
- spin_unlock(&stats->lock);
-
for (i = 0; i < stats->state_num; i++) {
+ if (pending) {
+ if (i == stats->last_index) {
+ /*
+ * Prevent the reset_time read from occurring
+ * before the reset_pending read above.
+ */
+ smp_rmb();
+ time = get_jiffies_64() - READ_ONCE(stats->reset_time);
+ } else {
+ time = 0;
+ }
+ } else {
+ time = stats->time_in_state[i];
+ if (i == stats->last_index)
+ time += get_jiffies_64() - stats->last_time;
+ }
+
len += sprintf(buf + len, "%u %llu\n", stats->freq_table[i],
- (unsigned long long)
- jiffies_64_to_clock_t(stats->time_in_state[i]));
+ jiffies_64_to_clock_t(time));
}
return len;
}
cpufreq_freq_attr_ro(time_in_state);
+/* We don't care what is written to the attribute */
static ssize_t store_reset(struct cpufreq_policy *policy, const char *buf,
size_t count)
{
- /* We don't care what is written to the attribute. */
- cpufreq_stats_clear_table(policy->stats);
+ struct cpufreq_stats *stats = policy->stats;
+
+ /*
+ * Defer resetting of stats to cpufreq_stats_record_transition() to
+ * avoid races.
+ */
+ WRITE_ONCE(stats->reset_time, get_jiffies_64());
+ /*
+ * The memory barrier below is to prevent the readers of reset_time from
+ * seeing a stale or partially updated value.
+ */
+ smp_wmb();
+ WRITE_ONCE(stats->reset_pending, 1);
+
return count;
}
cpufreq_freq_attr_wo(reset);
@@ -84,11 +124,9 @@ cpufreq_freq_attr_wo(reset);
static ssize_t show_trans_table(struct cpufreq_policy *policy, char *buf)
{
struct cpufreq_stats *stats = policy->stats;
+ bool pending = READ_ONCE(stats->reset_pending);
ssize_t len = 0;
- int i, j;
-
- if (policy->fast_switch_enabled)
- return 0;
+ int i, j, count;
len += scnprintf(buf + len, PAGE_SIZE - len, " From : To\n");
len += scnprintf(buf + len, PAGE_SIZE - len, " : ");
@@ -113,8 +151,13 @@ static ssize_t show_trans_table(struct cpufreq_policy *policy, char *buf)
for (j = 0; j < stats->state_num; j++) {
if (len >= PAGE_SIZE)
break;
- len += scnprintf(buf + len, PAGE_SIZE - len, "%9u ",
- stats->trans_table[i*stats->max_state+j]);
+
+ if (pending)
+ count = 0;
+ else
+ count = stats->trans_table[i * stats->max_state + j];
+
+ len += scnprintf(buf + len, PAGE_SIZE - len, "%9u ", count);
}
if (len >= PAGE_SIZE)
break;
@@ -208,7 +251,6 @@ void cpufreq_stats_create_table(struct cpufreq_policy *policy)
stats->state_num = i;
stats->last_time = get_jiffies_64();
stats->last_index = freq_table_get_index(stats, policy->cur);
- spin_lock_init(&stats->lock);
policy->stats = stats;
ret = sysfs_create_group(&policy->kobj, &stats_attr_group);
@@ -228,23 +270,22 @@ void cpufreq_stats_record_transition(struct cpufreq_policy *policy,
struct cpufreq_stats *stats = policy->stats;
int old_index, new_index;
- if (!stats) {
- pr_debug("%s: No stats found\n", __func__);
+ if (unlikely(!stats))
return;
- }
+
+ if (unlikely(READ_ONCE(stats->reset_pending)))
+ cpufreq_stats_reset_table(stats);
old_index = stats->last_index;
new_index = freq_table_get_index(stats, new_freq);
/* We can't do stats->time_in_state[-1]= .. */
- if (old_index == -1 || new_index == -1 || old_index == new_index)
+ if (unlikely(old_index == -1 || new_index == -1 || old_index == new_index))
return;
- spin_lock(&stats->lock);
- cpufreq_stats_update(stats);
+ cpufreq_stats_update(stats, stats->last_time);
stats->last_index = new_index;
stats->trans_table[old_index * stats->max_state + new_index]++;
stats->total_trans++;
- spin_unlock(&stats->lock);
}
diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c
index ef7b34c1fd2b..5bf5fc759881 100644
--- a/drivers/cpufreq/imx6q-cpufreq.c
+++ b/drivers/cpufreq/imx6q-cpufreq.c
@@ -48,7 +48,6 @@ static struct clk_bulk_data clks[] = {
};
static struct device *cpu_dev;
-static bool free_opp;
static struct cpufreq_frequency_table *freq_table;
static unsigned int max_freq;
static unsigned int transition_latency;
@@ -390,9 +389,6 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev)
goto put_reg;
}
- /* Because we have added the OPPs here, we must free them */
- free_opp = true;
-
if (of_machine_is_compatible("fsl,imx6ul") ||
of_machine_is_compatible("fsl,imx6ull")) {
ret = imx6ul_opp_check_speed_grading(cpu_dev);
@@ -507,8 +503,7 @@ soc_opp_out:
free_freq_table:
dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table);
out_free_opp:
- if (free_opp)
- dev_pm_opp_of_remove_table(cpu_dev);
+ dev_pm_opp_of_remove_table(cpu_dev);
put_reg:
if (!IS_ERR(arm_reg))
regulator_put(arm_reg);
@@ -528,8 +523,7 @@ static int imx6q_cpufreq_remove(struct platform_device *pdev)
{
cpufreq_unregister_driver(&imx6q_cpufreq_driver);
dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table);
- if (free_opp)
- dev_pm_opp_of_remove_table(cpu_dev);
+ dev_pm_opp_of_remove_table(cpu_dev);
regulator_put(arm_reg);
if (!IS_ERR(pu_reg))
regulator_put(pu_reg);
diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c
index 3fb044b907a8..9ed5341dc515 100644
--- a/drivers/cpufreq/qcom-cpufreq-hw.c
+++ b/drivers/cpufreq/qcom-cpufreq-hw.c
@@ -19,18 +19,23 @@
#define LUT_L_VAL GENMASK(7, 0)
#define LUT_CORE_COUNT GENMASK(18, 16)
#define LUT_VOLT GENMASK(11, 0)
-#define LUT_ROW_SIZE 32
#define CLK_HW_DIV 2
#define LUT_TURBO_IND 1
-/* Register offsets */
-#define REG_ENABLE 0x0
-#define REG_FREQ_LUT 0x110
-#define REG_VOLT_LUT 0x114
-#define REG_PERF_STATE 0x920
+struct qcom_cpufreq_soc_data {
+ u32 reg_enable;
+ u32 reg_freq_lut;
+ u32 reg_volt_lut;
+ u32 reg_perf_state;
+ u8 lut_row_size;
+};
+
+struct qcom_cpufreq_data {
+ void __iomem *base;
+ const struct qcom_cpufreq_soc_data *soc_data;
+};
static unsigned long cpu_hw_rate, xo_rate;
-static struct platform_device *global_pdev;
static bool icc_scaling_enabled;
static int qcom_cpufreq_set_bw(struct cpufreq_policy *policy,
@@ -77,22 +82,22 @@ static int qcom_cpufreq_update_opp(struct device *cpu_dev,
static int qcom_cpufreq_hw_target_index(struct cpufreq_policy *policy,
unsigned int index)
{
- void __iomem *perf_state_reg = policy->driver_data;
+ struct qcom_cpufreq_data *data = policy->driver_data;
+ const struct qcom_cpufreq_soc_data *soc_data = data->soc_data;
unsigned long freq = policy->freq_table[index].frequency;
- writel_relaxed(index, perf_state_reg);
+ writel_relaxed(index, data->base + soc_data->reg_perf_state);
if (icc_scaling_enabled)
qcom_cpufreq_set_bw(policy, freq);
- arch_set_freq_scale(policy->related_cpus, freq,
- policy->cpuinfo.max_freq);
return 0;
}
static unsigned int qcom_cpufreq_hw_get(unsigned int cpu)
{
- void __iomem *perf_state_reg;
+ struct qcom_cpufreq_data *data;
+ const struct qcom_cpufreq_soc_data *soc_data;
struct cpufreq_policy *policy;
unsigned int index;
@@ -100,9 +105,10 @@ static unsigned int qcom_cpufreq_hw_get(unsigned int cpu)
if (!policy)
return 0;
- perf_state_reg = policy->driver_data;
+ data = policy->driver_data;
+ soc_data = data->soc_data;
- index = readl_relaxed(perf_state_reg);
+ index = readl_relaxed(data->base + soc_data->reg_perf_state);
index = min(index, LUT_MAX_ENTRIES - 1);
return policy->freq_table[index].frequency;
@@ -111,23 +117,18 @@ static unsigned int qcom_cpufreq_hw_get(unsigned int cpu)
static unsigned int qcom_cpufreq_hw_fast_switch(struct cpufreq_policy *policy,
unsigned int target_freq)
{
- void __iomem *perf_state_reg = policy->driver_data;
+ struct qcom_cpufreq_data *data = policy->driver_data;
+ const struct qcom_cpufreq_soc_data *soc_data = data->soc_data;
unsigned int index;
- unsigned long freq;
index = policy->cached_resolved_idx;
- writel_relaxed(index, perf_state_reg);
+ writel_relaxed(index, data->base + soc_data->reg_perf_state);
- freq = policy->freq_table[index].frequency;
- arch_set_freq_scale(policy->related_cpus, freq,
- policy->cpuinfo.max_freq);
-
- return freq;
+ return policy->freq_table[index].frequency;
}
static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev,
- struct cpufreq_policy *policy,
- void __iomem *base)
+ struct cpufreq_policy *policy)
{
u32 data, src, lval, i, core_count, prev_freq = 0, freq;
u32 volt;
@@ -135,6 +136,8 @@ static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev,
struct dev_pm_opp *opp;
unsigned long rate;
int ret;
+ struct qcom_cpufreq_data *drv_data = policy->driver_data;
+ const struct qcom_cpufreq_soc_data *soc_data = drv_data->soc_data;
table = kcalloc(LUT_MAX_ENTRIES + 1, sizeof(*table), GFP_KERNEL);
if (!table)
@@ -161,14 +164,14 @@ static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev,
}
for (i = 0; i < LUT_MAX_ENTRIES; i++) {
- data = readl_relaxed(base + REG_FREQ_LUT +
- i * LUT_ROW_SIZE);
+ data = readl_relaxed(drv_data->base + soc_data->reg_freq_lut +
+ i * soc_data->lut_row_size);
src = FIELD_GET(LUT_SRC, data);
lval = FIELD_GET(LUT_L_VAL, data);
core_count = FIELD_GET(LUT_CORE_COUNT, data);
- data = readl_relaxed(base + REG_VOLT_LUT +
- i * LUT_ROW_SIZE);
+ data = readl_relaxed(drv_data->base + soc_data->reg_volt_lut +
+ i * soc_data->lut_row_size);
volt = FIELD_GET(LUT_VOLT, data) * 1000;
if (src)
@@ -177,10 +180,15 @@ static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev,
freq = cpu_hw_rate / 1000;
if (freq != prev_freq && core_count != LUT_TURBO_IND) {
- table[i].frequency = freq;
- qcom_cpufreq_update_opp(cpu_dev, freq, volt);
- dev_dbg(cpu_dev, "index=%d freq=%d, core_count %d\n", i,
+ if (!qcom_cpufreq_update_opp(cpu_dev, freq, volt)) {
+ table[i].frequency = freq;
+ dev_dbg(cpu_dev, "index=%d freq=%d, core_count %d\n", i,
freq, core_count);
+ } else {
+ dev_warn(cpu_dev, "failed to update OPP for freq=%d\n", freq);
+ table[i].frequency = CPUFREQ_ENTRY_INVALID;
+ }
+
} else if (core_count == LUT_TURBO_IND) {
table[i].frequency = CPUFREQ_ENTRY_INVALID;
}
@@ -197,9 +205,13 @@ static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev,
* as the boost frequency
*/
if (prev->frequency == CPUFREQ_ENTRY_INVALID) {
- prev->frequency = prev_freq;
- prev->flags = CPUFREQ_BOOST_FREQ;
- qcom_cpufreq_update_opp(cpu_dev, prev_freq, volt);
+ if (!qcom_cpufreq_update_opp(cpu_dev, prev_freq, volt)) {
+ prev->frequency = prev_freq;
+ prev->flags = CPUFREQ_BOOST_FREQ;
+ } else {
+ dev_warn(cpu_dev, "failed to update OPP for freq=%d\n",
+ freq);
+ }
}
break;
@@ -238,14 +250,38 @@ static void qcom_get_related_cpus(int index, struct cpumask *m)
}
}
+static const struct qcom_cpufreq_soc_data qcom_soc_data = {
+ .reg_enable = 0x0,
+ .reg_freq_lut = 0x110,
+ .reg_volt_lut = 0x114,
+ .reg_perf_state = 0x920,
+ .lut_row_size = 32,
+};
+
+static const struct qcom_cpufreq_soc_data epss_soc_data = {
+ .reg_enable = 0x0,
+ .reg_freq_lut = 0x100,
+ .reg_volt_lut = 0x200,
+ .reg_perf_state = 0x320,
+ .lut_row_size = 4,
+};
+
+static const struct of_device_id qcom_cpufreq_hw_match[] = {
+ { .compatible = "qcom,cpufreq-hw", .data = &qcom_soc_data },
+ { .compatible = "qcom,cpufreq-epss", .data = &epss_soc_data },
+ {}
+};
+MODULE_DEVICE_TABLE(of, qcom_cpufreq_hw_match);
+
static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy)
{
- struct device *dev = &global_pdev->dev;
+ struct platform_device *pdev = cpufreq_get_driver_data();
+ struct device *dev = &pdev->dev;
struct of_phandle_args args;
struct device_node *cpu_np;
struct device *cpu_dev;
- struct resource *res;
void __iomem *base;
+ struct qcom_cpufreq_data *data;
int ret, index;
cpu_dev = get_cpu_device(policy->cpu);
@@ -267,16 +303,21 @@ static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy)
index = args.args[0];
- res = platform_get_resource(global_pdev, IORESOURCE_MEM, index);
- if (!res)
- return -ENODEV;
+ base = devm_platform_ioremap_resource(pdev, index);
+ if (IS_ERR(base))
+ return PTR_ERR(base);
- base = devm_ioremap(dev, res->start, resource_size(res));
- if (!base)
- return -ENOMEM;
+ data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
+ if (!data) {
+ ret = -ENOMEM;
+ goto error;
+ }
+
+ data->soc_data = of_device_get_match_data(&pdev->dev);
+ data->base = base;
/* HW should be in enabled state to proceed */
- if (!(readl_relaxed(base + REG_ENABLE) & 0x1)) {
+ if (!(readl_relaxed(base + data->soc_data->reg_enable) & 0x1)) {
dev_err(dev, "Domain-%d cpufreq hardware not enabled\n", index);
ret = -ENODEV;
goto error;
@@ -289,9 +330,9 @@ static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy)
goto error;
}
- policy->driver_data = base + REG_PERF_STATE;
+ policy->driver_data = data;
- ret = qcom_cpufreq_hw_read_lut(cpu_dev, policy, base);
+ ret = qcom_cpufreq_hw_read_lut(cpu_dev, policy);
if (ret) {
dev_err(dev, "Domain-%d failed to read LUT\n", index);
goto error;
@@ -315,12 +356,13 @@ error:
static int qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy)
{
struct device *cpu_dev = get_cpu_device(policy->cpu);
- void __iomem *base = policy->driver_data - REG_PERF_STATE;
+ struct qcom_cpufreq_data *data = policy->driver_data;
+ struct platform_device *pdev = cpufreq_get_driver_data();
dev_pm_opp_remove_all_dynamic(cpu_dev);
dev_pm_opp_of_cpumask_remove_table(policy->related_cpus);
kfree(policy->freq_table);
- devm_iounmap(&global_pdev->dev, base);
+ devm_iounmap(&pdev->dev, data->base);
return 0;
}
@@ -365,7 +407,7 @@ static int qcom_cpufreq_hw_driver_probe(struct platform_device *pdev)
cpu_hw_rate = clk_get_rate(clk) / CLK_HW_DIV;
clk_put(clk);
- global_pdev = pdev;
+ cpufreq_qcom_hw_driver.driver_data = pdev;
/* Check for optional interconnect paths on CPU0 */
cpu_dev = get_cpu_device(0);
@@ -390,12 +432,6 @@ static int qcom_cpufreq_hw_driver_remove(struct platform_device *pdev)
return cpufreq_unregister_driver(&cpufreq_qcom_hw_driver);
}
-static const struct of_device_id qcom_cpufreq_hw_match[] = {
- { .compatible = "qcom,cpufreq-hw" },
- {}
-};
-MODULE_DEVICE_TABLE(of, qcom_cpufreq_hw_match);
-
static struct platform_driver qcom_cpufreq_hw_driver = {
.probe = qcom_cpufreq_hw_driver_probe,
.remove = qcom_cpufreq_hw_driver_remove,
diff --git a/drivers/cpufreq/s5pv210-cpufreq.c b/drivers/cpufreq/s5pv210-cpufreq.c
index e84281e2561d..bed496cf8d24 100644
--- a/drivers/cpufreq/s5pv210-cpufreq.c
+++ b/drivers/cpufreq/s5pv210-cpufreq.c
@@ -590,6 +590,7 @@ static struct notifier_block s5pv210_cpufreq_reboot_notifier = {
static int s5pv210_cpufreq_probe(struct platform_device *pdev)
{
+ struct device *dev = &pdev->dev;
struct device_node *np;
int id, result = 0;
@@ -602,28 +603,20 @@ static int s5pv210_cpufreq_probe(struct platform_device *pdev)
* cpufreq-dt driver.
*/
arm_regulator = regulator_get(NULL, "vddarm");
- if (IS_ERR(arm_regulator)) {
- if (PTR_ERR(arm_regulator) == -EPROBE_DEFER)
- pr_debug("vddarm regulator not ready, defer\n");
- else
- pr_err("failed to get regulator vddarm\n");
- return PTR_ERR(arm_regulator);
- }
+ if (IS_ERR(arm_regulator))
+ return dev_err_probe(dev, PTR_ERR(arm_regulator),
+ "failed to get regulator vddarm\n");
int_regulator = regulator_get(NULL, "vddint");
if (IS_ERR(int_regulator)) {
- if (PTR_ERR(int_regulator) == -EPROBE_DEFER)
- pr_debug("vddint regulator not ready, defer\n");
- else
- pr_err("failed to get regulator vddint\n");
- result = PTR_ERR(int_regulator);
+ result = dev_err_probe(dev, PTR_ERR(int_regulator),
+ "failed to get regulator vddint\n");
goto err_int_regulator;
}
np = of_find_compatible_node(NULL, NULL, "samsung,s5pv210-clock");
if (!np) {
- pr_err("%s: failed to find clock controller DT node\n",
- __func__);
+ dev_err(dev, "failed to find clock controller DT node\n");
result = -ENODEV;
goto err_clock;
}
@@ -631,7 +624,7 @@ static int s5pv210_cpufreq_probe(struct platform_device *pdev)
clk_base = of_iomap(np, 0);
of_node_put(np);
if (!clk_base) {
- pr_err("%s: failed to map clock registers\n", __func__);
+ dev_err(dev, "failed to map clock registers\n");
result = -EFAULT;
goto err_clock;
}
@@ -639,8 +632,7 @@ static int s5pv210_cpufreq_probe(struct platform_device *pdev)
for_each_compatible_node(np, NULL, "samsung,s5pv210-dmc") {
id = of_alias_get_id(np, "dmc");
if (id < 0 || id >= ARRAY_SIZE(dmc_base)) {
- pr_err("%s: failed to get alias of dmc node '%pOFn'\n",
- __func__, np);
+ dev_err(dev, "failed to get alias of dmc node '%pOFn'\n", np);
of_node_put(np);
result = id;
goto err_clk_base;
@@ -648,8 +640,7 @@ static int s5pv210_cpufreq_probe(struct platform_device *pdev)
dmc_base[id] = of_iomap(np, 0);
if (!dmc_base[id]) {
- pr_err("%s: failed to map dmc%d registers\n",
- __func__, id);
+ dev_err(dev, "failed to map dmc%d registers\n", id);
of_node_put(np);
result = -EFAULT;
goto err_dmc;
@@ -658,7 +649,7 @@ static int s5pv210_cpufreq_probe(struct platform_device *pdev)
for (id = 0; id < ARRAY_SIZE(dmc_base); ++id) {
if (!dmc_base[id]) {
- pr_err("%s: failed to find dmc%d node\n", __func__, id);
+ dev_err(dev, "failed to find dmc%d node\n", id);
result = -ENODEV;
goto err_dmc;
}
diff --git a/drivers/cpufreq/scmi-cpufreq.c b/drivers/cpufreq/scmi-cpufreq.c
index fb42e3390377..6dd1311660b5 100644
--- a/drivers/cpufreq/scmi-cpufreq.c
+++ b/drivers/cpufreq/scmi-cpufreq.c
@@ -48,16 +48,11 @@ static unsigned int scmi_cpufreq_get_rate(unsigned int cpu)
static int
scmi_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int index)
{
- int ret;
struct scmi_data *priv = policy->driver_data;
struct scmi_perf_ops *perf_ops = handle->perf_ops;
u64 freq = policy->freq_table[index].frequency;
- ret = perf_ops->freq_set(handle, priv->domain_id, freq * 1000, false);
- if (!ret)
- arch_set_freq_scale(policy->related_cpus, freq,
- policy->cpuinfo.max_freq);
- return ret;
+ return perf_ops->freq_set(handle, priv->domain_id, freq * 1000, false);
}
static unsigned int scmi_cpufreq_fast_switch(struct cpufreq_policy *policy,
@@ -67,11 +62,8 @@ static unsigned int scmi_cpufreq_fast_switch(struct cpufreq_policy *policy,
struct scmi_perf_ops *perf_ops = handle->perf_ops;
if (!perf_ops->freq_set(handle, priv->domain_id,
- target_freq * 1000, true)) {
- arch_set_freq_scale(policy->related_cpus, target_freq,
- policy->cpuinfo.max_freq);
+ target_freq * 1000, true))
return target_freq;
- }
return 0;
}
diff --git a/drivers/cpufreq/scpi-cpufreq.c b/drivers/cpufreq/scpi-cpufreq.c
index b0f5388b8854..43db05b949d9 100644
--- a/drivers/cpufreq/scpi-cpufreq.c
+++ b/drivers/cpufreq/scpi-cpufreq.c
@@ -47,9 +47,8 @@ static unsigned int scpi_cpufreq_get_rate(unsigned int cpu)
static int
scpi_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int index)
{
- unsigned long freq = policy->freq_table[index].frequency;
+ u64 rate = policy->freq_table[index].frequency * 1000;
struct scpi_data *priv = policy->driver_data;
- u64 rate = freq * 1000;
int ret;
ret = clk_set_rate(priv->clk, rate);
@@ -60,9 +59,6 @@ scpi_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int index)
if (clk_get_rate(priv->clk) != rate)
return -EIO;
- arch_set_freq_scale(policy->related_cpus, freq,
- policy->cpuinfo.max_freq);
-
return 0;
}
diff --git a/drivers/cpufreq/sti-cpufreq.c b/drivers/cpufreq/sti-cpufreq.c
index a5ad96d29adc..4ac6fb23792a 100644
--- a/drivers/cpufreq/sti-cpufreq.c
+++ b/drivers/cpufreq/sti-cpufreq.c
@@ -141,7 +141,8 @@ static const struct reg_field sti_stih407_dvfs_regfields[DVFS_MAX_REGFIELDS] = {
static const struct reg_field *sti_cpufreq_match(void)
{
if (of_machine_is_compatible("st,stih407") ||
- of_machine_is_compatible("st,stih410"))
+ of_machine_is_compatible("st,stih410") ||
+ of_machine_is_compatible("st,stih418"))
return sti_stih407_dvfs_regfields;
return NULL;
@@ -258,7 +259,8 @@ static int sti_cpufreq_init(void)
int ret;
if ((!of_machine_is_compatible("st,stih407")) &&
- (!of_machine_is_compatible("st,stih410")))
+ (!of_machine_is_compatible("st,stih410")) &&
+ (!of_machine_is_compatible("st,stih418")))
return -ENODEV;
ddata.cpu = get_cpu_device(0);
diff --git a/drivers/cpufreq/tegra186-cpufreq.c b/drivers/cpufreq/tegra186-cpufreq.c
index 01e1f58ba422..4b4079f51559 100644
--- a/drivers/cpufreq/tegra186-cpufreq.c
+++ b/drivers/cpufreq/tegra186-cpufreq.c
@@ -14,6 +14,7 @@
#define EDVD_CORE_VOLT_FREQ(core) (0x20 + (core) * 0x4)
#define EDVD_CORE_VOLT_FREQ_F_SHIFT 0
+#define EDVD_CORE_VOLT_FREQ_F_MASK 0xffff
#define EDVD_CORE_VOLT_FREQ_V_SHIFT 16
struct tegra186_cpufreq_cluster_info {
@@ -91,10 +92,39 @@ static int tegra186_cpufreq_set_target(struct cpufreq_policy *policy,
return 0;
}
+static unsigned int tegra186_cpufreq_get(unsigned int cpu)
+{
+ struct cpufreq_frequency_table *tbl;
+ struct cpufreq_policy *policy;
+ void __iomem *edvd_reg;
+ unsigned int i, freq = 0;
+ u32 ndiv;
+
+ policy = cpufreq_cpu_get(cpu);
+ if (!policy)
+ return 0;
+
+ tbl = policy->freq_table;
+ edvd_reg = policy->driver_data;
+ ndiv = readl(edvd_reg) & EDVD_CORE_VOLT_FREQ_F_MASK;
+
+ for (i = 0; tbl[i].frequency != CPUFREQ_TABLE_END; i++) {
+ if ((tbl[i].driver_data & EDVD_CORE_VOLT_FREQ_F_MASK) == ndiv) {
+ freq = tbl[i].frequency;
+ break;
+ }
+ }
+
+ cpufreq_cpu_put(policy);
+
+ return freq;
+}
+
static struct cpufreq_driver tegra186_cpufreq_driver = {
.name = "tegra186",
.flags = CPUFREQ_STICKY | CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
CPUFREQ_NEED_INITIAL_FREQ_CHECK,
+ .get = tegra186_cpufreq_get,
.verify = cpufreq_generic_frequency_table_verify,
.target_index = tegra186_cpufreq_set_target,
.init = tegra186_cpufreq_init,
diff --git a/drivers/cpufreq/vexpress-spc-cpufreq.c b/drivers/cpufreq/vexpress-spc-cpufreq.c
index 4e8b1dee7c9a..e89b905754d2 100644
--- a/drivers/cpufreq/vexpress-spc-cpufreq.c
+++ b/drivers/cpufreq/vexpress-spc-cpufreq.c
@@ -182,7 +182,6 @@ static int ve_spc_cpufreq_set_target(struct cpufreq_policy *policy,
{
u32 cpu = policy->cpu, cur_cluster, new_cluster, actual_cluster;
unsigned int freqs_new;
- int ret;
cur_cluster = cpu_to_cluster(cpu);
new_cluster = actual_cluster = per_cpu(physical_cluster, cpu);
@@ -197,15 +196,8 @@ static int ve_spc_cpufreq_set_target(struct cpufreq_policy *policy,
new_cluster = A15_CLUSTER;
}
- ret = ve_spc_cpufreq_set_rate(cpu, actual_cluster, new_cluster,
- freqs_new);
-
- if (!ret) {
- arch_set_freq_scale(policy->related_cpus, freqs_new,
- policy->cpuinfo.max_freq);
- }
-
- return ret;
+ return ve_spc_cpufreq_set_rate(cpu, actual_cluster, new_cluster,
+ freqs_new);
}
static inline u32 get_table_count(struct cpufreq_frequency_table *table)
diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h
index 69b1dabe39dc..0f6cd6b73a61 100644
--- a/include/linux/arch_topology.h
+++ b/include/linux/arch_topology.h
@@ -30,7 +30,11 @@ static inline unsigned long topology_get_freq_scale(int cpu)
return per_cpu(freq_scale, cpu);
}
-bool arch_freq_counters_available(struct cpumask *cpus);
+void topology_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq,
+ unsigned long max_freq);
+bool topology_scale_freq_invariant(void);
+
+bool arch_freq_counters_available(const struct cpumask *cpus);
DECLARE_PER_CPU(unsigned long, thermal_pressure);
diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h
index a911e5d06845..fa37b1c66443 100644
--- a/include/linux/cpufreq.h
+++ b/include/linux/cpufreq.h
@@ -217,6 +217,7 @@ void refresh_frequency_limits(struct cpufreq_policy *policy);
void cpufreq_update_policy(unsigned int cpu);
void cpufreq_update_limits(unsigned int cpu);
bool have_governor_per_policy(void);
+bool cpufreq_supports_freq_invariance(void);
struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy);
void cpufreq_enable_fast_switch(struct cpufreq_policy *policy);
void cpufreq_disable_fast_switch(struct cpufreq_policy *policy);
@@ -237,6 +238,10 @@ static inline unsigned int cpufreq_get_hw_max_freq(unsigned int cpu)
{
return 0;
}
+static inline bool cpufreq_supports_freq_invariance(void)
+{
+ return false;
+}
static inline void disable_cpufreq(void) { }
#endif
@@ -1006,8 +1011,14 @@ static inline void sched_cpufreq_governor_change(struct cpufreq_policy *policy,
extern void arch_freq_prepare_all(void);
extern unsigned int arch_freq_get_on_cpu(int cpu);
-extern void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq,
- unsigned long max_freq);
+#ifndef arch_set_freq_scale
+static __always_inline
+void arch_set_freq_scale(const struct cpumask *cpus,
+ unsigned long cur_freq,
+ unsigned long max_freq)
+{
+}
+#endif
/* the following are really really optional */
extern struct freq_attr cpufreq_freq_attr_scaling_available_freqs;
diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c
index e39008242cf4..5ae7b4e6e8d6 100644
--- a/kernel/sched/cpufreq_schedutil.c
+++ b/kernel/sched/cpufreq_schedutil.c
@@ -114,22 +114,8 @@ static bool sugov_update_next_freq(struct sugov_policy *sg_policy, u64 time,
static void sugov_fast_switch(struct sugov_policy *sg_policy, u64 time,
unsigned int next_freq)
{
- struct cpufreq_policy *policy = sg_policy->policy;
- int cpu;
-
- if (!sugov_update_next_freq(sg_policy, time, next_freq))
- return;
-
- next_freq = cpufreq_driver_fast_switch(policy, next_freq);
- if (!next_freq)
- return;
-
- policy->cur = next_freq;
-
- if (trace_cpu_frequency_enabled()) {
- for_each_cpu(cpu, policy->cpus)
- trace_cpu_frequency(next_freq, cpu);
- }
+ if (sugov_update_next_freq(sg_policy, time, next_freq))
+ cpufreq_driver_fast_switch(sg_policy->policy, next_freq);
}
static void sugov_deferred_update(struct sugov_policy *sg_policy, u64 time,
generated by cgit v1.2.3 (git 2.39.0) at 2024-11-18 13:42:04 +0300