From 801ade22c4fc078ea9b11e1f3fe3aae8b8bef886 Mon Sep 17 00:00:00 2001 From: Daniel Schwierzeck Date: Fri, 21 Aug 2026 10:45:39 +0200 Subject: [PATCH 01/34] cpufreq: airoha: fix probe failure with correct error check dev_pm_domain_attach_list() returns the number of attached PM domains on success (a positive count), or a negative error code on failure. The driver checks "if (ret)" which treats a successful attachment of one or more PM domains as an error, causing the probe to fail with return code 1. Fix by checking "if (ret < 0)" like all other users of this API. Cc: stable@vger.kernel.org Fixes: 84cf9e541ccc ("cpufreq: airoha: Add EN7581 CPUFreq SMCCC driver") Signed-off-by: Daniel Schwierzeck Signed-off-by: Christian Marangi Reviewed-by: Zhongqiu Han Signed-off-by: Viresh Kumar --- drivers/cpufreq/airoha-cpufreq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/airoha-cpufreq.c b/drivers/cpufreq/airoha-cpufreq.c index 3e7770860d13..8283c528d0c1 100644 --- a/drivers/cpufreq/airoha-cpufreq.c +++ b/drivers/cpufreq/airoha-cpufreq.c @@ -64,7 +64,7 @@ static int airoha_cpufreq_probe(struct platform_device *pdev) /* Attach PM for OPP */ ret = dev_pm_domain_attach_list(cpu_dev, &attach_data, &priv->pd_list); - if (ret) + if (ret < 0) goto clear_opp_config; cpufreq_dt = platform_device_register_simple("cpufreq-dt", -1, NULL, 0); From a23fca4ae137ec8e7620cc4f00b468b89cf5a7bd Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Wed, 9 Sep 2026 10:26:50 +0000 Subject: [PATCH 02/34] cpufreq/amd-pstate-ut: Fix amd_pstate_ut_check_freq failure with 'Requested CPU Min frequency' BIOS option Since commit 608a76b65288 ("cpufreq/amd-pstate: Add support for the "Requested CPU Min frequency" BIOS option"), amd-pstate driver sets policy->min to frequency corresponding to bios_min_perf if a valid BIOS programmed min frequency value is detected. amd_pstate_ut_check_freq expects policy->min to always match lowest_nonlinear_freq which does not hold true on platforms with user configured BIOS min freq. Update the test case to compare policy->min to bios_min_freq on platforms that set it. Final comparison is adjusted to account for insane values by clamping the result within the supported frequency range. While at it, move freq_to_perf() and perf_to_freq() helpers to internal header to allow their use from amd-pstate-ut. Reviewed-by: Mario Limonciello Fixes: 608a76b65288 ("cpufreq/amd-pstate: Add support for the "Requested CPU Min frequency" BIOS option") Signed-off-by: K Prateek Nayak Link: https://lore.kernel.org/r/20260909102650.4582-1-kprateek.nayak@amd.com Signed-off-by: Mario Limonciello --- drivers/cpufreq/amd-pstate-ut.c | 23 ++++++++++++++++++++++- drivers/cpufreq/amd-pstate.c | 13 ------------- drivers/cpufreq/amd-pstate.h | 14 ++++++++++++++ 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index e23773680e05..c2c1a166b3a9 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -226,11 +226,14 @@ static int amd_pstate_ut_check_freq(u32 index) for_each_online_cpu(cpu) { struct cpufreq_policy *policy __free(put_cpufreq_policy) = NULL; struct amd_cpudata *cpudata; + union perf_cached perf; policy = cpufreq_cpu_get(cpu); if (!policy) continue; + cpudata = policy->driver_data; + perf = READ_ONCE(cpudata->perf); if (!((policy->cpuinfo.max_freq >= cpudata->nominal_freq) && (cpudata->nominal_freq > cpudata->lowest_nonlinear_freq) && @@ -242,7 +245,25 @@ static int amd_pstate_ut_check_freq(u32 index) return -EINVAL; } - if (cpudata->lowest_nonlinear_freq != policy->min) { + if (perf.bios_min_perf) { + u32 bios_min_freq = perf_to_freq(perf, cpudata->nominal_freq, + perf.bios_min_perf); + + /* + * User set bios_min_freq cannot be trusted to + * be within the driver limits. Clamp it similar + * to cpufreq_verify_within_cpu_limits(). + */ + bios_min_freq = clamp_t(u32, bios_min_freq, + policy->cpuinfo.min_freq, + policy->cpuinfo.max_freq); + + if (bios_min_freq != policy->min) { + pr_err("%s cpu%d bios_min_freq=%d policy_min=%d, they should be equal!\n", + __func__, cpu, bios_min_freq, policy->min); + return -EINVAL; + } + } else if (cpudata->lowest_nonlinear_freq != policy->min) { pr_err("%s cpu%d cpudata_lowest_nonlinear_freq=%d policy_min=%d, they should be equal!\n", __func__, cpu, cpudata->lowest_nonlinear_freq, policy->min); return -EINVAL; diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 8bfd46d60843..ae1673b36ac4 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -145,19 +145,6 @@ static struct quirk_entry quirk_amd_7k62 = { .lowest_freq = 550, }; -static inline u8 freq_to_perf(union perf_cached perf, u32 nominal_freq, unsigned int freq_val) -{ - u32 perf_val = DIV_ROUND_UP_ULL((u64)freq_val * perf.nominal_perf, nominal_freq); - - return (u8)clamp(perf_val, perf.lowest_perf, perf.highest_perf); -} - -static inline u32 perf_to_freq(union perf_cached perf, u32 nominal_freq, u8 perf_val) -{ - return DIV_ROUND_UP_ULL((u64)nominal_freq * perf_val, - perf.nominal_perf); -} - static int __init dmi_matched_7k62_bios_bug(const struct dmi_system_id *dmi) { /** diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index 9f5a81976eae..578465187b7a 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -159,6 +159,20 @@ enum amd_pstate_mode { AMD_PSTATE_GUIDED, AMD_PSTATE_MAX, }; + +static inline u8 freq_to_perf(union perf_cached perf, u32 nominal_freq, unsigned int freq_val) +{ + u32 perf_val = DIV_ROUND_UP_ULL((u64)freq_val * perf.nominal_perf, nominal_freq); + + return (u8)clamp(perf_val, perf.lowest_perf, perf.highest_perf); +} + +static inline u32 perf_to_freq(union perf_cached perf, u32 nominal_freq, u8 perf_val) +{ + return DIV_ROUND_UP_ULL((u64)nominal_freq * perf_val, + perf.nominal_perf); +} + const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode); int amd_pstate_get_status(void); int amd_pstate_update_status(const char *buf, size_t size); From db40a7877cf14fc34fa27ee479eb6561d10648cf Mon Sep 17 00:00:00 2001 From: "Mario Limonciello (AMD)" Date: Tue, 18 Aug 2026 23:11:51 -0500 Subject: [PATCH 03/34] cpufreq/amd-pstate: Add per SoC and per core type EPP tuning values On newer SoCs the behavior of the platform has changed, and using the same EPP values for everything will yield worse results than expected. Add a x86_cpu_id match table that allows setting policies on a per-SoC/per-core type basis. If a SoC is missing, fallback to legacy values that are used. Changes also: - Change epp_values array and amd_pstate_cpu_epp_values() from unsigned int to u8 - Add static_assert to ensure epp_values has rows for all CPU types - Fix show_energy_performance_preference() loop to exclude uninitialized EPP_INDEX_CUSTOM and EPP_INDEX_DYNAMIC slots - Add debug print statements - Remove amd_pstate_get_epp_from_platform_profile and inline the logic - Fix cleanup path in amd_pstate_set_dynamic_epp - Change show_energy_performance_preference to use for loop with preference tracking - Add debug prints in amd_pstate_init_epp_values Behavioral change: show_energy_performance_preference() now returns a numeric EPP value via sysfs when the cached EPP does not match any named preference, instead of returning -EINVAL as before. This provides visibility into custom EPP values set by hardware or BIOS. No new platforms are added in this commit. Reviewed-by: K Prateek Nayak Tested-by: K Prateek Nayak Signed-off-by: Mario Limonciello (AMD) --- drivers/cpufreq/amd-pstate.c | 183 ++++++++++++++++++++++++++++------- drivers/cpufreq/amd-pstate.h | 2 + 2 files changed, 148 insertions(+), 37 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index ae1673b36ac4..8047f168b6d7 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -55,10 +55,10 @@ #define AMD_PSTATE_TRANSITION_DELAY 1000 #define AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY 600 -#define AMD_CPPC_EPP_PERFORMANCE 0x00 -#define AMD_CPPC_EPP_BALANCE_PERFORMANCE 0x80 -#define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xBF -#define AMD_CPPC_EPP_POWERSAVE 0xFF +#define AMD_CPPC_EPP_LEGACY_PERFORMANCE 0x00 +#define AMD_CPPC_EPP_LEGACY_BALANCE_PERFORMANCE 0x80 +#define AMD_CPPC_EPP_LEGACY_BALANCE_POWERSAVE 0xBF +#define AMD_CPPC_EPP_LEGACY_POWERSAVE 0xFF static const char * const amd_pstate_mode_string[] = { [AMD_PSTATE_UNDEFINED] = "undefined", @@ -129,14 +129,86 @@ static const char * const energy_perf_strings[] = { }; static_assert(ARRAY_SIZE(energy_perf_strings) == EPP_INDEX_MAX); -static unsigned int epp_values[] = { - [EPP_INDEX_DEFAULT] = 0, - [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, - [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, - [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, - [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, +/* + * The numeric EPP value programmed for each named preference. First dimension + * is CPU type (TOPO_CPU_TYPE_ANY for non-hybrid, TOPO_CPU_TYPE_PERFORMANCE/ + * EFFICIENCY/LOW_POWER for hybrid). The initializer holds the legacy values + * used as the fallback on any platform not listed in amd_pstate_epp_soc_ids[]; + * amd_pstate_init_epp_values() overwrites slots at boot when the running SoC + * has a per-SoC (and potentially per-CPU-type) override. + */ +static u8 epp_values[][EPP_INDEX_MAX] = { + /* + * Initialize all CPU types to legacy defaults. + * amd_pstate_init_epp_values() will fix these up + * based on the platform during boot. + */ + [TOPO_CPU_TYPE_ANY ... TOPO_CPU_TYPE_LOW_POWER] = { + [EPP_INDEX_DEFAULT] = 0, + [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_LEGACY_PERFORMANCE, + [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_LEGACY_BALANCE_PERFORMANCE, + [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_LEGACY_BALANCE_POWERSAVE, + [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_LEGACY_POWERSAVE, + }, +}; +static_assert(ARRAY_SIZE(epp_values) == TOPO_CPU_TYPE_LOW_POWER + 1, + "epp_values must have entries for all CPU types up to TOPO_CPU_TYPE_LOW_POWER"); + +/* + * Get the EPP value row for a given CPU, accounting for hybrid CPU types. + * Non-hybrid systems use TOPO_CPU_TYPE_ANY; hybrid systems use the CPU's + * actual type (PERFORMANCE/EFFICIENCY/LOW_POWER). + */ +static inline u8 *amd_pstate_cpu_epp_values(enum x86_topology_cpu_type cpu_type) +{ + switch (cpu_type) { + case TOPO_CPU_TYPE_PERFORMANCE: + case TOPO_CPU_TYPE_EFFICIENCY: + case TOPO_CPU_TYPE_LOW_POWER: + return epp_values[cpu_type]; + default: + return epp_values[TOPO_CPU_TYPE_ANY]; + } +} + +/** + * struct amd_pstate_epp_values - EPP values for the four named preferences + * @performance: value for the "performance" preference + * @balance_performance: value for the "balance_performance" preference + * @balance_power: value for the "balance_power" preference + * @power: value for the "power" preference + */ +struct amd_pstate_epp_values { + u8 performance; + u8 balance_performance; + u8 balance_power; + u8 power; +}; + +/** + * struct amd_pstate_epp_soc - per-CPU-type EPP overrides for hybrid systems + * @performance_core: values for TOPO_CPU_TYPE_PERFORMANCE cores + * @efficiency_core: values for TOPO_CPU_TYPE_EFFICIENCY cores + * @low_power_core: values for TOPO_CPU_TYPE_LOW_POWER cores + * + * Referenced from amd_pstate_epp_soc_ids[] to give a hybrid platform its own + * numeric EPP values for the four named preferences, with distinct values per + * CPU type. Non-hybrid systems are not listed in the table and always use the + * legacy defaults. + */ +struct amd_pstate_epp_soc { + struct amd_pstate_epp_values performance_core; + struct amd_pstate_epp_values efficiency_core; + struct amd_pstate_epp_values low_power_core; +}; + +/* + * Per-CPU-type EPP overrides for hybrid systems. Only hybrid SoCs should be + * listed here; non-hybrid systems always use the legacy defaults. + */ +static const struct x86_cpu_id amd_pstate_epp_soc_ids[] __initconst = { + {} }; -static_assert(ARRAY_SIZE(epp_values) == EPP_INDEX_MAX - 2); typedef int (*cppc_mode_transition_fn)(int); @@ -1055,6 +1127,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) return -ENOMEM; cpudata->cpu = policy->cpu; + cpudata->cpu_type = cpu_data(policy->cpu).topo.cpu_type; ret = amd_pstate_init_perf(cpudata); if (ret) @@ -1182,13 +1255,16 @@ static int amd_pstate_power_supply_notifier(struct notifier_block *nb, static int amd_pstate_get_epp_from_platform_profile(struct cpufreq_policy *policy, enum platform_profile_option profile) { + struct amd_cpudata *cpudata = policy->driver_data; + u8 *values = amd_pstate_cpu_epp_values(cpudata->cpu_type); + switch (profile) { case PLATFORM_PROFILE_PERFORMANCE: - return AMD_CPPC_EPP_PERFORMANCE; + return values[EPP_INDEX_PERFORMANCE]; case PLATFORM_PROFILE_BALANCED: return amd_pstate_get_balanced_epp(policy); case PLATFORM_PROFILE_LOW_POWER: - return AMD_CPPC_EPP_POWERSAVE; + return values[EPP_INDEX_POWERSAVE]; default: break; } @@ -1398,6 +1474,7 @@ ssize_t store_energy_performance_preference(struct cpufreq_policy *policy, const char *buf, size_t count) { struct amd_cpudata *cpudata = policy->driver_data; + u8 *values = amd_pstate_cpu_epp_values(cpudata->cpu_type); ssize_t ret; bool raw_epp = false; u8 epp; @@ -1432,12 +1509,13 @@ ssize_t store_energy_performance_preference(struct cpufreq_policy *policy, } if (ret) - epp = epp_values[ret]; + epp = values[ret]; else epp = cpudata->epp_default_dc; } - if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { + if (epp > 0 && epp != values[EPP_INDEX_PERFORMANCE] && + cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { pr_debug("EPP cannot be set under performance policy\n"); return -EBUSY; } @@ -1462,34 +1540,31 @@ EXPORT_SYMBOL_FOR_PSTATE_UT(store_energy_performance_preference); ssize_t show_energy_performance_preference(struct cpufreq_policy *policy, char *buf) { struct amd_cpudata *cpudata = policy->driver_data; - u8 preference, epp; + u8 *values = amd_pstate_cpu_epp_values(cpudata->cpu_type); + u8 epp; + int i; epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); if (!cpudata->dynamic_epp && cpudata->raw_epp) return sysfs_emit(buf, "%u\n", epp); - switch (epp) { - case AMD_CPPC_EPP_PERFORMANCE: - preference = EPP_INDEX_PERFORMANCE; - break; - case AMD_CPPC_EPP_BALANCE_PERFORMANCE: - preference = EPP_INDEX_BALANCE_PERFORMANCE; - break; - case AMD_CPPC_EPP_BALANCE_POWERSAVE: - preference = EPP_INDEX_BALANCE_POWERSAVE; - break; - case AMD_CPPC_EPP_POWERSAVE: - preference = EPP_INDEX_POWERSAVE; - break; - default: - return -EINVAL; + /* + * Map the cached EPP value back to a named preference. Skip the + * "default" slot (index 0) so an EPP of 0 reports as "performance". + * Stop at POWERSAVE; CUSTOM and DYNAMIC are not initialized in epp_values. + */ + for (i = EPP_INDEX_PERFORMANCE; i <= EPP_INDEX_POWERSAVE; i++) { + const char *name = energy_perf_strings[i]; + + if (epp == values[i]) { + if (cpudata->dynamic_epp) + return sysfs_emit(buf, "dynamic(profile:%s)\n", name); + return sysfs_emit(buf, "%s\n", name); + } } - if (cpudata->dynamic_epp) - return sysfs_emit(buf, "dynamic(profile:%s)\n", energy_perf_strings[preference]); - - return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); + return sysfs_emit(buf, "%u\n", epp); } EXPORT_SYMBOL_FOR_PSTATE_UT(show_energy_performance_preference); @@ -1881,6 +1956,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) return -ENOMEM; cpudata->cpu = policy->cpu; + cpudata->cpu_type = cpu_data(policy->cpu).topo.cpu_type; ret = amd_pstate_init_perf(cpudata); if (ret) @@ -1931,9 +2007,11 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) cpudata->epp_default_ac = cpudata->epp_default_dc = default_epp; cpudata->current_profile = PLATFORM_PROFILE_PERFORMANCE; } else { + u8 *values = amd_pstate_cpu_epp_values(cpudata->cpu_type); + policy->policy = CPUFREQ_POLICY_POWERSAVE; - cpudata->epp_default_ac = AMD_CPPC_EPP_PERFORMANCE; - cpudata->epp_default_dc = AMD_CPPC_EPP_BALANCE_PERFORMANCE; + cpudata->epp_default_ac = values[EPP_INDEX_PERFORMANCE]; + cpudata->epp_default_dc = values[EPP_INDEX_BALANCE_PERFORMANCE]; cpudata->current_profile = PLATFORM_PROFILE_BALANCED; } @@ -2232,6 +2310,34 @@ static bool amd_cppc_supported(void) return true; } +/* + * Resolve the numeric EPP values for hybrid systems. Only hybrid SoCs are listed + * in amd_pstate_epp_soc_ids[]; non-hybrid systems always use the legacy defaults. + */ +static inline void __init amd_pstate_set_epp_values(enum x86_topology_cpu_type type, + const struct amd_pstate_epp_values *core) +{ + epp_values[type][EPP_INDEX_PERFORMANCE] = core->performance; + epp_values[type][EPP_INDEX_BALANCE_PERFORMANCE] = core->balance_performance; + epp_values[type][EPP_INDEX_BALANCE_POWERSAVE] = core->balance_power; + epp_values[type][EPP_INDEX_POWERSAVE] = core->power; +} + +static void __init amd_pstate_init_epp_values(void) +{ + const struct x86_cpu_id *id = x86_match_cpu(amd_pstate_epp_soc_ids); + const struct amd_pstate_epp_soc *soc; + + if (!id || !id->driver_data) + return; + + soc = (const struct amd_pstate_epp_soc *)id->driver_data; + + amd_pstate_set_epp_values(TOPO_CPU_TYPE_PERFORMANCE, &soc->performance_core); + amd_pstate_set_epp_values(TOPO_CPU_TYPE_EFFICIENCY, &soc->efficiency_core); + amd_pstate_set_epp_values(TOPO_CPU_TYPE_LOW_POWER, &soc->low_power_core); +} + static int __init amd_pstate_init(void) { struct device *dev_root; @@ -2259,6 +2365,9 @@ static int __init amd_pstate_init(void) /* check if this machine need CPPC quirks */ dmi_check_system(amd_pstate_quirks_table); + /* resolve per-SoC EPP values for the named preferences */ + amd_pstate_init_epp_values(); + /* * determine the driver mode from the command line or kernel config. * If no command line input is provided, cppc_state will be AMD_PSTATE_UNDEFINED. diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index 578465187b7a..c7189d177b81 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -146,6 +146,8 @@ struct amd_cpudata { enum platform_profile_option current_profile; struct device *ppdev; char *profile_name; + + enum x86_topology_cpu_type cpu_type; }; /* From 114a157e8d30601d024b53564a0fe54202f6567f Mon Sep 17 00:00:00 2001 From: "Mario Limonciello (AMD)" Date: Tue, 18 Aug 2026 23:12:16 -0500 Subject: [PATCH 04/34] cpufreq/amd-pstate: Add EPP tunings for Zen6 client platforms Zen6 client platforms perform better with individual tunings for different core types. Add initial EPP tuning values for those platforms. Reviewed-by: K Prateek Nayak Tested-by: K Prateek Nayak Signed-off-by: Mario Limonciello (AMD) --- drivers/cpufreq/amd-pstate.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 8047f168b6d7..4344cc93904b 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -206,7 +206,33 @@ struct amd_pstate_epp_soc { * Per-CPU-type EPP overrides for hybrid systems. Only hybrid SoCs should be * listed here; non-hybrid systems always use the legacy defaults. */ +static const struct amd_pstate_epp_soc epp_soc_zen6_client __initconst = { + .performance_core = { + .performance = 25, + .balance_performance = 51, + .balance_power = 64, + .power = 64, + }, + .efficiency_core = { + .performance = 25, + .balance_performance = 51, + .balance_power = 64, + .power = 115, + }, + .low_power_core = { + .performance = 25, + .balance_performance = 51, + .balance_power = 64, + .power = 115, + }, +}; + static const struct x86_cpu_id amd_pstate_epp_soc_ids[] __initconst = { + X86_MATCH_VENDOR_FAM_MODEL(AMD, 0x1A, 0x80, &epp_soc_zen6_client), + X86_MATCH_VENDOR_FAM_MODEL(AMD, 0x1A, 0x81, &epp_soc_zen6_client), + X86_MATCH_VENDOR_FAM_MODEL(AMD, 0x1A, 0x84, &epp_soc_zen6_client), + X86_MATCH_VENDOR_FAM_MODEL(AMD, 0x1A, 0x85, &epp_soc_zen6_client), + X86_MATCH_VENDOR_FAM_MODEL(AMD, 0x1A, 0xe0, &epp_soc_zen6_client), {} }; From 350de8c394d4e61df6ebe31d8ddd928a69766fd3 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 31 Aug 2026 16:39:47 -0500 Subject: [PATCH 05/34] cpufreq/amd-pstate: Show a warning if missing EPP tunings EPP tunings are going to be more important going forward. Show a warning if on a heterogenous platform without them. Reviewed-by: K Prateek Nayak Tested-by: K Prateek Nayak Suggested-by: Christian Ludloff Signed-off-by: Mario Limonciello --- drivers/cpufreq/amd-pstate.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 4344cc93904b..83960a6336b9 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -2354,8 +2354,12 @@ static void __init amd_pstate_init_epp_values(void) const struct x86_cpu_id *id = x86_match_cpu(amd_pstate_epp_soc_ids); const struct amd_pstate_epp_soc *soc; - if (!id || !id->driver_data) + if (!id || !id->driver_data) { + if (cpu_feature_enabled(X86_FEATURE_ZEN6) && + cpu_feature_enabled(X86_FEATURE_AMD_HTR_CORES)) + pr_warn_once("No EPP tunings found for platform\n"); return; + } soc = (const struct amd_pstate_epp_soc *)id->driver_data; From c8663e0457e6e6c72006478f4b6da99060030ec1 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 5 Sep 2026 03:07:06 +0200 Subject: [PATCH 06/34] cpufreq/amd-pstate: Remove obsolete amd_dynamic_epp documentation The amd_dynamic_epp command-line parameter was removed. Dynamic EPP is now selected through the per-policy energy_performance_preference attribute, which is already documented. Fixes: 32692fcf61ed ("cpufreq/amd-pstate: Remove "amd_dynamic_epp" cmdline and "dynamic_epp" sysfs") Assisted-by: LLM Signed-off-by: Karl Mehltretter Reviewed-by: K Prateek Nayak Reviewed-by: Randy Dunlap Link: https://lore.kernel.org/r/20260905010706.57380-1-kmehltretter@gmail.com Signed-off-by: Mario Limonciello --- Documentation/admin-guide/kernel-parameters.txt | 7 ------- Documentation/admin-guide/pm/amd-pstate.rst | 6 ------ 2 files changed, 13 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 68647ff4bdd2..378ba224948c 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -501,13 +501,6 @@ Kernel parameters disable Disable amd-pstate preferred core. - amd_dynamic_epp= - [X86] - disable - Disable amd-pstate dynamic EPP. - enable - Enable amd-pstate dynamic EPP. - amijoy.map= [HW,JOY] Amiga joystick support Map of devices attached to JOY0DAT and JOY1DAT Format: , diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index 6fd72ba52df2..4c5d3d8acf64 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -481,12 +481,6 @@ For systems that support ``amd-pstate`` preferred core, the core rankings will always be advertised by the platform. But OS can choose to ignore that via the kernel parameter ``amd_prefcore=disable``. -``amd_dynamic_epp`` - -When AMD pstate is in auto mode, dynamic EPP will control whether the kernel -autonomously changes the EPP mode. The default is disabled. It can be enabled -with the kernel parameter ``amd_dynamic_epp=enable``. - User Space Interface in ``sysfs`` - General =========================================== From 0fa55c5539173bceaeb400c69b1974aaa0cdb8dc Mon Sep 17 00:00:00 2001 From: Shengming Hu Date: Wed, 16 Sep 2026 14:09:26 +0800 Subject: [PATCH 07/34] cpufreq: conservative: Restrict deferred downscaling to low load For a shared cpufreq policy, dbs_update() derives the load used for the frequency decision from the maximum load among its CPUs, but it may also record deferred idle periods accumulated by an idle CPU in the policy. Consequently, a single update can contain both a high decision load from one CPU and multiple deferred idle periods from another CPU. The conservative governor applies the deferred down steps before evaluating the decision load. If that load subsequently triggers an up step, the deferred down steps can outweigh the single up step and produce a net frequency reduction. This was observed on a policy shared by CPUs 2 and 3. A CPU-bound SCHED_EXT task kept CPU 2 fully utilized while CPU 3 remained idle. On this system, SCHED_EXT generated update-util callbacks less frequently than CFS, resulting in sparse DBS updates such as: load=100 idle_periods=7 interval=59 ms load=100 idle_periods=4 interval=39 ms load=100 idle_periods=2 interval=19 ms load=100 idle_periods=7 interval=59 ms With the default 5% frequency step and a 2.6 GHz policy maximum, seven deferred periods reduce the requested frequency by seven 130 MHz steps before the high decision load adds only one step. Repeated updates therefore keep the policy near 530 MHz even though CPU 2 is fully utilized. There are two load values relevant to this behavior: * sample_load is calculated from the busy and elapsed time of the current sample. * decision_load is returned by dbs_update() for the governor's frequency decision. After a long idle interval, it may reuse prev_load to improve the response to a waking task and can therefore differ from sample_load. The problem is that idle_periods is derived from accumulated idle time and is applied independently of whether the current sample load is in the conservative governor's downscaling region. A high sample load and multiple deferred idle periods can therefore coexist, particularly when updates are sparse. As suggested by Zhongqiu Han, calculate the sample load separately from the decision load. If no time has elapsed, use prev_load for the sample load as the existing load calculation does, since no load can be calculated for that interval. Record the maximum sample load for the policy and apply deferred down steps only when it is below down_threshold. This makes deferred downscaling follow the same threshold that the conservative governor normally uses to determine whether the policy load is low enough to reduce the frequency. The existing code applies deferred down steps whenever idle_periods is valid, regardless of the sample load. When the decision load is in the hold region, the locally adjusted frequency is not submitted unless another frequency-update branch is taken. Thus, when the sample and decision loads are the same and both are in the hold region, this change does not alter the requested frequency. The difference becomes visible when the decision load causes a frequency update. In particular, if it exceeds up_threshold, the governor applies a single up step after the deferred down steps and submits the resulting request. Multiple deferred down steps can then outweigh that up step and cause a frequency reduction even when the sample load is above down_threshold. With this change, deferred down steps are skipped whenever the maximum sample load is at or above down_threshold, keeping deferred downscaling consistent with the conservative governor's threshold semantics. The requested frequency may therefore be higher than with the existing behavior when deferred idle periods are present and the decision load causes a frequency update. This is intentional, because the lower request previously resulted from applying deferred downscaling while the sampled policy load was outside the governor's downscaling region. Accelerated deferred downscaling remains available when the policy sample load is below down_threshold, including after a workload has completed and the policy has become idle. Fixes: 00bfe05889e9 ("cpufreq: conservative: Decrease frequency faster for deferred updates") Cc: All applicable Reviewed-by: Luo Haiyang Reviewed-by: Run Zhang Suggested-by: Zhongqiu Han Signed-off-by: Shengming Hu Link: https://patch.msgid.link/20260916140926740-DCokXh6dAgtqDn1EWdCh@zte.com.cn Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/cpufreq_conservative.c | 8 ++++--- drivers/cpufreq/cpufreq_governor.c | 29 +++++++++++++++++++------- drivers/cpufreq/cpufreq_governor.h | 2 ++ 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c index 0b32ae28ec85..02bfd46543e9 100644 --- a/drivers/cpufreq/cpufreq_conservative.c +++ b/drivers/cpufreq/cpufreq_conservative.c @@ -85,10 +85,12 @@ static unsigned int cs_dbs_update(struct cpufreq_policy *policy) freq_step = get_freq_step(cs_tuners, policy); /* - * Decrease requested_freq one freq_step for each idle period that - * we didn't update the frequency. + * Apply deferred down steps only when the policy sample load is + * below down_threshold. Otherwise, multiple deferred down steps may + * cause a net frequency decrease outside the downscaling region. */ - if (policy_dbs->idle_periods < UINT_MAX) { + if (policy_dbs->max_sample_load < cs_tuners->down_threshold && + policy_dbs->idle_periods < UINT_MAX) { unsigned int freq_steps = policy_dbs->idle_periods * freq_step; if (requested_freq > policy->min + freq_steps) diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c index 710d93ec89b5..f3bd3dbe7c41 100644 --- a/drivers/cpufreq/cpufreq_governor.c +++ b/drivers/cpufreq/cpufreq_governor.c @@ -124,7 +124,8 @@ unsigned int dbs_update(struct cpufreq_policy *policy) struct policy_dbs_info *policy_dbs = policy->governor_data; struct dbs_data *dbs_data = policy_dbs->dbs_data; unsigned int ignore_nice = dbs_data->ignore_nice_load; - unsigned int max_load = 0, idle_periods = UINT_MAX; + unsigned int max_load = 0, max_sample_load = 0; + unsigned int idle_periods = UINT_MAX; unsigned int sampling_rate, io_busy, j; u64 cur_nice; @@ -147,7 +148,7 @@ unsigned int dbs_update(struct cpufreq_policy *policy) struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j); u64 update_time, cur_idle_time; unsigned int idle_time, time_elapsed; - unsigned int load; + unsigned int load, sample_load; cur_idle_time = get_cpu_idle_time(j, &update_time, io_busy); @@ -186,6 +187,20 @@ unsigned int dbs_update(struct cpufreq_policy *policy) j_cdbs->prev_cpu_nice = cur_nice; + /* + * Compute the sample load separately from the prev_load value + * that may be reused after a long idle interval. The conservative + * governor uses it to decide whether to apply deferred down steps. + * If no time has elapsed, retain the existing behavior and use + * prev_load. + */ + if (unlikely(!time_elapsed)) + sample_load = j_cdbs->prev_load; + else if (time_elapsed > idle_time) + sample_load = 100 * (time_elapsed - idle_time) / time_elapsed; + else + sample_load = 0; + if (unlikely(!time_elapsed)) { /* * That can only happen when this function is called @@ -220,11 +235,7 @@ unsigned int dbs_update(struct cpufreq_policy *policy) load = j_cdbs->prev_load; j_cdbs->prev_load = 0; } else { - if (time_elapsed > idle_time) - load = 100 * (time_elapsed - idle_time) / time_elapsed; - else - load = 0; - + load = sample_load; j_cdbs->prev_load = load; } @@ -237,9 +248,13 @@ unsigned int dbs_update(struct cpufreq_policy *policy) if (load > max_load) max_load = load; + + if (sample_load > max_sample_load) + max_sample_load = sample_load; } policy_dbs->idle_periods = idle_periods; + policy_dbs->max_sample_load = max_sample_load; return max_load; } diff --git a/drivers/cpufreq/cpufreq_governor.h b/drivers/cpufreq/cpufreq_governor.h index 73b8ed7cfaae..806d8fb4dff1 100644 --- a/drivers/cpufreq/cpufreq_governor.h +++ b/drivers/cpufreq/cpufreq_governor.h @@ -95,6 +95,8 @@ struct policy_dbs_info { /* Multiplier for increasing sample delay temporarily. */ unsigned int rate_mult; unsigned int idle_periods; /* For conservative */ + /* Maximum load from the current policy sample. */ + unsigned int max_sample_load; /* Status indicators */ bool is_shared; /* This object is used by multiple CPUs */ bool work_in_progress; /* Work is being queued up or in progress */ From 50fb9e04c9b1fe822baccba57801e2bc7795552e Mon Sep 17 00:00:00 2001 From: Xueqin Luo Date: Tue, 25 Aug 2026 21:14:00 +0800 Subject: [PATCH 08/34] cpufreq/amd-pstate: Restore previous EPP if profile_name allocation fails amd_pstate_set_dynamic_epp() first applies the EPP value derived from the current platform profile via amd_pstate_set_epp() and only then allocates the profile name and registers the platform profile handler and the power supply notifier. All failures after the EPP has been programmed jump to the cleanup label, which restores the previously programmed EPP value. However, if the kasprintf() of the profile name fails, the function returns -ENOMEM directly, bypassing the cleanup label, so the EPP value programmed above is never rolled back even though enabling dynamic EPP has failed. Make the allocation failure path jump to the cleanup label as well, so that the previous EPP value is restored consistently on all error paths. Fixes: b7294b627598 ("cpufreq/amd-pstate: Add dynamic EPP as an "energy_performance_preference" mode") Signed-off-by: Xueqin Luo Link: https://lore.kernel.org/r/20260825131400.1509428-1-luoxueqin@kylinos.cn Signed-off-by: Mario Limonciello --- drivers/cpufreq/amd-pstate.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 83960a6336b9..8b2eb72c3973 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1381,8 +1381,10 @@ static int amd_pstate_set_dynamic_epp(struct cpufreq_policy *policy) return ret; cpudata->profile_name = kasprintf(GFP_KERNEL, "amd-pstate-epp-cpu%d", cpudata->cpu); - if (!cpudata->profile_name) - return -ENOMEM; + if (!cpudata->profile_name) { + ret = -ENOMEM; + goto cleanup; + } cpudata->ppdev = platform_profile_register(get_cpu_device(policy->cpu), cpudata->profile_name, From b400642c5e93863ec17ea15fcbd3efc81c299531 Mon Sep 17 00:00:00 2001 From: Wentao Guan Date: Fri, 18 Sep 2026 17:09:10 +0800 Subject: [PATCH 09/34] cpufreq/amd-pstate: Skip auto_sel write when it already matches the mode Since commit 9dfd13f80c85 ("cpufreq/amd-pstate: Toggle auto_sel in active mode on shared memory systems"), shmem_init_perf() programs AUTONOMOUS_SELECTION_ENABLE unconditionally, including in active mode. Platforms that exclusively support autonomous selection report this field as an Integer of 1, as required by the ACPI specification, instead of as a register descriptor. cppc_get_reg_val() reads such a field back successfully, but cppc_set_reg_val() rejects the write with -EOPNOTSUPP because the entry is not an ACPI_TYPE_BUFFER. shmem_init_perf() propagates that error, so every amd_pstate_epp_cpu_init() fails, cpufreq_register_driver() ends up with an empty policy list and returns -ENODEV, and amd-pstate refuses to load at all: amd_pstate: failed to set auto_sel, ret: -95 amd_pstate: Failed to initialize CPU 0: -95 ... amd_pstate: failed to register with return -19 Skip the write when the register already holds the value the requested mode needs. Active and guided mode are fixed on these platforms, which report auto_sel as 1, while passive mode still fails to load on a platform that cannot disable autonomous selection, exactly as it did before the offending commit. Fixes: 9dfd13f80c85 ("cpufreq/amd-pstate: Toggle auto_sel in active mode on shared memory systems") Cc: stable@vger.kernel.org Reviewed-by: K Prateek Nayak Tested-by: K Prateek Nayak Tested-by: LFRon Signed-off-by: Wentao Guan Link: https://lore.kernel.org/r/20260918090909.86544-1-guanwentao@uniontech.com Signed-off-by: Mario Limonciello --- drivers/cpufreq/amd-pstate.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 8b2eb72c3973..c93a4b765dff 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -609,8 +609,10 @@ static int shmem_init_perf(struct amd_cpudata *cpudata) return 0; } - ret = cppc_set_auto_sel(cpudata->cpu, - (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); + if ((cppc_state != AMD_PSTATE_PASSIVE) == auto_sel) + return 0; + + ret = cppc_set_auto_sel(cpudata->cpu, !auto_sel); if (ret) pr_warn("failed to set auto_sel, ret: %d\n", ret); From b667a680917119827916bf2eb7d6f8de9c47531f Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 21 Sep 2026 21:23:47 -0500 Subject: [PATCH 10/34] cpufreq/amd-pstate: Fix TOCTOU when changing driver mode via sysfs amd_pstate_update_status() evaluates mode_state_machine[cppc_state] [mode_idx] before taking amd_pstate_driver_lock, and then re-evaluates it again once the lock is held. cppc_state is global and only stable under the lock, so concurrent sysfs writes race: CPU 0 CPU 1 ----- ----- // cppc_state == A, mode == B if (mode_state_machine[A][B]) // sees non-NULL if (mode_state_machine[A][B]) guard(&amd_pstate_driver_lock); mode_state_machine[A][B](B); cppc_state = B; guard(&amd_pstate_driver_lock); // cppc_state is now B mode_state_machine[B][B](B); // NULL -> NULL deref The re-read under the lock can resolve to a self-transition (NULL pointer, immediate NULL deref) or to an unexpected transition that redundantly runs amd_pstate_driver_cleanup(), double-freeing current_pstate_driver->attr. Take the lock first, then read cppc_state and resolve the transition function exactly once into a local before calling it, so the check and the call observe the same state. Reported-by: Sashiko Closes: https://sashiko.dev/#/bug/linux-f86ff3a6-55df-4eef-8e4d-62ac270dba25 Fixes: 6f0b13f16f7a ("cpufreq/amd-pstate: Overhaul locking") Signed-off-by: Mario Limonciello Reviewed-by: K Prateek Nayak Tested-by: K Prateek Nayak Link: https://lore.kernel.org/r/20260922022347.1270746-1-mario.limonciello@amd.com Signed-off-by: Mario Limonciello --- drivers/cpufreq/amd-pstate.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index c93a4b765dff..d054720abee3 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1886,6 +1886,7 @@ EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_get_status); int amd_pstate_update_status(const char *buf, size_t size) { + cppc_mode_transition_fn fn; int mode_idx; if (size > strlen("passive") || size < strlen("active")) @@ -1895,10 +1896,12 @@ int amd_pstate_update_status(const char *buf, size_t size) if (mode_idx < 0) return mode_idx; - if (mode_state_machine[cppc_state][mode_idx]) { - guard(mutex)(&amd_pstate_driver_lock); - return mode_state_machine[cppc_state][mode_idx](mode_idx); - } + guard(mutex)(&amd_pstate_driver_lock); + + /* Resolve the transition under the lock; cppc_state is only stable while held. */ + fn = mode_state_machine[cppc_state][mode_idx]; + if (fn) + return fn(mode_idx); return 0; } From 52763c00945ab30241d9c097547600b9e234e559 Mon Sep 17 00:00:00 2001 From: Vishal Badole Date: Tue, 29 Sep 2026 02:05:23 +0530 Subject: [PATCH 11/34] cpufreq: amd-pstate: Update Zen6 client EPP tuning values The per-CPU-type EPP values for the Zen6 client platform are further tuned based on platform characterization. Adjust the performance and low-power core entries so each core type uses its intended EPP settings. Reviewed-by: Mario Limonciello Signed-off-by: Vishal Badole Link: https://lore.kernel.org/r/20260928203523.1290623-1-Vishal.Badole@amd.com Signed-off-by: Mario Limonciello --- drivers/cpufreq/amd-pstate.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index d054720abee3..8d5886f1e905 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -211,7 +211,7 @@ static const struct amd_pstate_epp_soc epp_soc_zen6_client __initconst = { .performance = 25, .balance_performance = 51, .balance_power = 64, - .power = 64, + .power = 115, }, .efficiency_core = { .performance = 25, @@ -221,9 +221,9 @@ static const struct amd_pstate_epp_soc epp_soc_zen6_client __initconst = { }, .low_power_core = { .performance = 25, - .balance_performance = 51, + .balance_performance = 64, .balance_power = 64, - .power = 115, + .power = 64, }, }; From 4817d3e2b90dac50eb11cee32a0cf92d294261ea Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Thu, 1 Oct 2026 09:23:39 -0500 Subject: [PATCH 12/34] ACPI: CPPC: Refactor boost ratio handling Rename amd_get_boost_ratio_numerator() to amd_get_effective_highest_perf() to better reflect that it returns the effective highest performance value for a CPU, accounting for preferred cores and heterogeneous topologies. Add a new helper amd_get_boost_ratio() for doing both for boost ratio calculations and update callers to use it. This refactoring is needed because on some AMD systems (Zen6+), the boost ratio should be calculated from frequency values rather than performance values. Newer versions of the ACPI will introduce a proper interface for getting this information, but for now hardcode values so that these systems perform as intended and to unblock future scheduler work while we continue to wait. For now use some temporary hardcoded values for Zen6 platforms. Reviewed-by: K Prateek Nayak Tested-by: K Prateek Nayak Acked-by: Rafael J. Wysocki (Intel) Signed-off-by: Mario Limonciello Link: https://lore.kernel.org/r/20261001142341.2934582-2-mario.limonciello@amd.com Signed-off-by: Mario Limonciello --- arch/x86/kernel/acpi/cppc.c | 166 ++++++++++++++++++++++----------- drivers/cpufreq/acpi-cpufreq.c | 5 +- drivers/cpufreq/amd-pstate.c | 14 ++- include/acpi/cppc_acpi.h | 14 ++- 4 files changed, 134 insertions(+), 65 deletions(-) diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c index bbade0da5130..f230e9323187 100644 --- a/arch/x86/kernel/acpi/cppc.c +++ b/arch/x86/kernel/acpi/cppc.c @@ -10,6 +10,7 @@ #include #include #include +#include #define CPPC_HIGHEST_PERF_PERFORMANCE 196 #define CPPC_HIGHEST_PERF_PREFCORE 166 @@ -81,31 +82,18 @@ int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) static void amd_set_max_freq_ratio(void) { - struct cppc_perf_caps perf_caps; - u64 numerator, nominal_perf; + u64 numerator, denominator; u64 perf_ratio; int rc; - rc = cppc_get_perf_caps(0, &perf_caps); + rc = amd_get_boost_ratio(0, &numerator, &denominator); if (rc) { - pr_debug("Could not retrieve perf counters (%d)\n", rc); - return; - } - - rc = amd_get_boost_ratio_numerator(0, &numerator); - if (rc) { - pr_debug("Could not retrieve highest performance (%d)\n", rc); - return; - } - nominal_perf = perf_caps.nominal_perf; - - if (!nominal_perf) { - pr_debug("Could not retrieve nominal performance\n"); + pr_debug("Could not retrieve boost ratio (%d)\n", rc); return; } /* midpoint between max_boost and max_P */ - perf_ratio = (div_u64(numerator * SCHED_CAPACITY_SCALE, nominal_perf) + SCHED_CAPACITY_SCALE) >> 1; + perf_ratio = (div_u64(numerator * SCHED_CAPACITY_SCALE, denominator) + SCHED_CAPACITY_SCALE) >> 1; freq_invariance_set_perf_ratio(perf_ratio, false); } @@ -225,35 +213,30 @@ int amd_detect_prefcore(bool *detected) EXPORT_SYMBOL_GPL(amd_detect_prefcore); /** - * amd_get_boost_ratio_numerator: Get the numerator to use for boost ratio calculation - * @cpu: CPU to get numerator for. - * @numerator: Output variable for numerator. + * amd_get_effective_highest_perf: Get the effective highest performance value + * @cpu: CPU to get highest performance for. * - * Determine the numerator to use for calculating the boost ratio on - * a CPU. On systems that support preferred cores, this will be a hardcoded - * value. On other systems this will the highest performance register value. + * Get the effective highest performance value for a CPU, accounting for + * preferred cores and heterogeneous topologies. On systems with preferred + * cores, this may be a hardcoded value. On heterogeneous systems, this + * may be a per-CPU value. On other systems, this is the shared highest + * performance value. * - * If booting the system with amd-pstate enabled but preferred cores disabled then - * the correct boost numerator will be returned to match hardware capabilities - * even if the preferred cores scheduling hints are not enabled. - * - * Return: 0 for success, negative error code otherwise. + * Return: Effective highest performance value, or negative error code. */ -int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator) +int amd_get_effective_highest_perf(unsigned int cpu) { bool prefcore; int ret; u32 tmp; ret = amd_detect_prefcore(&prefcore); - if (ret) + if (ret < 0) return ret; /* without preferred cores, return the highest perf register value */ - if (!prefcore) { - *numerator = boost_numerator; - return 0; - } + if (!prefcore) + return boost_numerator; /* * For AMD CPUs with Family ID 19H and Model ID range 0x70 to 0x7f, @@ -263,8 +246,7 @@ int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator) if (cpu_feature_enabled(X86_FEATURE_ZEN4)) { switch (boot_cpu_data.x86_model) { case 0x70 ... 0x7f: - *numerator = CPPC_HIGHEST_PERF_PERFORMANCE; - return 0; + return CPPC_HIGHEST_PERF_PERFORMANCE; default: break; } @@ -272,28 +254,100 @@ int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator) /* detect if running on heterogeneous design */ if (cpu_feature_enabled(X86_FEATURE_AMD_HTR_CORES)) { - switch (cpu_data(cpu).topo.cpu_type) { - case TOPO_CPU_TYPE_UNKNOWN: - case TOPO_CPU_TYPE_ANY: - pr_warn("Undefined core type found for cpu %d\n", cpu); - break; - case TOPO_CPU_TYPE_PERFORMANCE: - /* use the max scale for performance cores */ - *numerator = CPPC_HIGHEST_PERF_PERFORMANCE; - return 0; - case TOPO_CPU_TYPE_LOW_POWER: - case TOPO_CPU_TYPE_EFFICIENCY: - /* use the highest perf value for efficiency and low-power cores */ - ret = amd_get_highest_perf(cpu, &tmp); - if (ret) - return ret; - *numerator = tmp; - return 0; - } + if (cpu_feature_enabled(X86_FEATURE_ZEN5) && + cpu_data(cpu).topo.cpu_type == TOPO_CPU_TYPE_PERFORMANCE) + return CPPC_HIGHEST_PERF_PERFORMANCE; + + /* Zen 5 efficiency, and Zen 6+ */ + ret = amd_get_highest_perf(cpu, &tmp); + if (ret < 0) + return ret; + + return tmp; } - *numerator = CPPC_HIGHEST_PERF_PREFCORE; + return CPPC_HIGHEST_PERF_PREFCORE; +} +EXPORT_SYMBOL_GPL(amd_get_effective_highest_perf); + +struct amd_max_freq { + u32 performance; + u32 efficiency; + u32 low_power; +}; + +static const struct amd_max_freq max_freq_zen6_client = { + .performance = 5025, + .efficiency = 3524, + .low_power = 2399, +}; + +int amd_get_max_frequency(unsigned int cpu) +{ + enum x86_topology_cpu_type type = cpu_data(cpu).topo.cpu_type; + const struct amd_max_freq *max_freq = NULL; + + if (cpu_feature_enabled(X86_FEATURE_ZEN6) && + cpu_feature_enabled(X86_FEATURE_AMD_HTR_CORES)) + max_freq = &max_freq_zen6_client; + + if (!max_freq) + return 0; + + switch (type) { + case TOPO_CPU_TYPE_PERFORMANCE: + return max_freq->performance; + case TOPO_CPU_TYPE_EFFICIENCY: + return max_freq->efficiency; + case TOPO_CPU_TYPE_LOW_POWER: + return max_freq->low_power; + default: + break; + } return 0; } -EXPORT_SYMBOL_GPL(amd_get_boost_ratio_numerator); +EXPORT_SYMBOL_GPL(amd_get_max_frequency); + +/** + * amd_get_boost_ratio: Get numerator and denominator for boost ratio + * @cpu: CPU to get the boost ratio for. + * @numerator: Output variable for numerator. + * @denominator: Output variable for denominator. + * + * Get the numerator and denominator for calculating the boost ratio. + * + * Return: 0 for success, negative error code otherwise. + */ +int amd_get_boost_ratio(unsigned int cpu, u64 *numerator, u64 *denominator) +{ + struct cppc_perf_caps perf_caps; + int max_frequency; + int ret; + + ret = cppc_get_perf_caps(cpu, &perf_caps); + if (ret) + return ret; + + max_frequency = amd_get_max_frequency(cpu); + /* Use frequency values if available */ + if (max_frequency && perf_caps.nominal_freq) { + *numerator = max_frequency; + *denominator = perf_caps.nominal_freq; + return 0; + } + + /* Fall back to performance values */ + ret = amd_get_effective_highest_perf(cpu); + if (ret < 0) + return ret; + + *numerator = ret; + + *denominator = perf_caps.nominal_perf; + if (!*denominator) + return -EINVAL; + + return 0; +} +EXPORT_SYMBOL_GPL(amd_get_boost_ratio); diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index 10ea6035f4ad..523ee1a06ef8 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -640,12 +640,13 @@ static u64 get_max_boost_ratio(unsigned int cpu, u64 *nominal_freq) } if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { - ret = amd_get_boost_ratio_numerator(cpu, &highest_perf); - if (ret) { + ret = amd_get_effective_highest_perf(cpu); + if (ret < 0) { pr_debug("CPU%d: Unable to get boost ratio numerator (%d)\n", cpu, ret); return 0; } + highest_perf = ret; } else { highest_perf = perf_caps.highest_perf; } diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 8d5886f1e905..83ab0385a3ea 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -555,11 +555,13 @@ static int msr_init_perf(struct amd_cpudata *cpudata) if (ret) return ret; - ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); - if (ret) + ret = amd_get_effective_highest_perf(cpudata->cpu); + if (ret < 0) return ret; - ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &cppc_req); + numerator = ret; + + ret = rdmsrq_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &cppc_req); if (ret) return ret; @@ -590,10 +592,12 @@ static int shmem_init_perf(struct amd_cpudata *cpudata) if (ret) return ret; - ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); - if (ret) + ret = amd_get_effective_highest_perf(cpudata->cpu); + if (ret < 0) return ret; + numerator = ret; + perf.highest_perf = numerator; perf.max_limit_perf = numerator; perf.min_limit_perf = cppc_perf.lowest_perf; diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h index 94a6277edab2..4c0b11a718e3 100644 --- a/include/acpi/cppc_acpi.h +++ b/include/acpi/cppc_acpi.h @@ -192,7 +192,9 @@ extern int cppc_set_auto_sel(int cpu, bool enable); extern int cppc_get_perf_limited(int cpu, u64 *perf_limited); extern int cppc_set_perf_limited(int cpu, u64 bits_to_clear); extern int amd_get_highest_perf(unsigned int cpu, u32 *highest_perf); -extern int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator); +extern int amd_get_effective_highest_perf(unsigned int cpu); +extern int amd_get_max_frequency(unsigned int cpu); +extern int amd_get_boost_ratio(unsigned int cpu, u64 *numerator, u64 *denominator); extern int amd_detect_prefcore(bool *detected); #else /* !CONFIG_ACPI_CPPC_LIB */ static inline int cppc_get_desired_perf(int cpunum, u64 *desired_perf) @@ -305,7 +307,15 @@ static inline int amd_get_highest_perf(unsigned int cpu, u32 *highest_perf) { return -ENODEV; } -static inline int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator) +static inline int amd_get_effective_highest_perf(unsigned int cpu) +{ + return -EOPNOTSUPP; +} +static inline int amd_get_max_frequency(unsigned int cpu) +{ + return 0; +} +static inline int amd_get_boost_ratio(unsigned int cpu, u64 *numerator, u64 *denominator) { return -EOPNOTSUPP; } From f9da951d309b24aea5052b76ab9438d7854a17ae Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Thu, 1 Oct 2026 09:23:40 -0500 Subject: [PATCH 13/34] cpufreq/acpi-cpufreq: Use amd_get_boost_ratio() Update get_max_boost_ratio() to use the new amd_get_boost_ratio() helper function instead of calling amd_get_effective_highest_perf() directly. This ensures the boost ratio is calculated correctly on systems where frequency values should be used instead of performance values. Reviewed-by: K Prateek Nayak Tested-by: K Prateek Nayak Acked-by: Rafael J. Wysocki (Intel) Signed-off-by: Mario Limonciello Link: https://lore.kernel.org/r/20261001142341.2934582-3-mario.limonciello@amd.com Signed-off-by: Mario Limonciello --- drivers/cpufreq/acpi-cpufreq.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index 523ee1a06ef8..2e74a000cdc7 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -626,7 +626,7 @@ static int acpi_cpufreq_blacklist(struct cpuinfo_x86 *c) static u64 get_max_boost_ratio(unsigned int cpu, u64 *nominal_freq) { struct cppc_perf_caps perf_caps; - u64 highest_perf, nominal_perf; + u64 numerator, denominator; int ret; if (acpi_pstate_strict) @@ -640,33 +640,31 @@ static u64 get_max_boost_ratio(unsigned int cpu, u64 *nominal_freq) } if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { - ret = amd_get_effective_highest_perf(cpu); - if (ret < 0) { - pr_debug("CPU%d: Unable to get boost ratio numerator (%d)\n", + ret = amd_get_boost_ratio(cpu, &numerator, &denominator); + if (ret) { + pr_debug("CPU%d: Unable to get boost ratio (%d)\n", cpu, ret); return 0; } - highest_perf = ret; } else { - highest_perf = perf_caps.highest_perf; + numerator = perf_caps.highest_perf; + denominator = perf_caps.nominal_perf; } - nominal_perf = perf_caps.nominal_perf; - if (nominal_freq) *nominal_freq = perf_caps.nominal_freq * 1000; - if (!highest_perf || !nominal_perf) { + if (!numerator || !denominator) { pr_debug("CPU%d: highest or nominal performance missing\n", cpu); return 0; } - if (highest_perf < nominal_perf) { + if (numerator < denominator) { pr_debug("CPU%d: nominal performance above highest\n", cpu); return 0; } - return div_u64(highest_perf << SCHED_CAPACITY_SHIFT, nominal_perf); + return div_u64(numerator << SCHED_CAPACITY_SHIFT, denominator); } #else From b3f9051ae263940af4a69212c88fd094401cf92d Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Thu, 1 Oct 2026 09:23:41 -0500 Subject: [PATCH 14/34] cpufreq/amd-pstate: Get Highest Freq for a CPU If the highest frequency is known for a CPU, use this instead of trying to calculate by linear interpolation. Reviewed-by: K Prateek Nayak Tested-by: K Prateek Nayak Signed-off-by: Mario Limonciello Link: https://lore.kernel.org/r/20261001142341.2934582-4-mario.limonciello@amd.com Signed-off-by: Mario Limonciello --- drivers/cpufreq/amd-pstate.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 83ab0385a3ea..743d90ed6af4 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -778,7 +778,10 @@ static void amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) struct amd_cpudata *cpudata = policy->driver_data; union perf_cached perf = READ_ONCE(cpudata->perf); - perf.max_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->max); + if (policy->max >= READ_ONCE(cpudata->max_freq)) + perf.max_limit_perf = perf.highest_perf; + else + perf.max_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->max); WRITE_ONCE(cpudata->max_limit_freq, policy->max); if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { @@ -1104,8 +1107,10 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) WRITE_ONCE(cpudata->nominal_freq, nominal_freq); - /* max_freq is calculated according to (nominal_freq * highest_perf)/nominal_perf */ - max_freq = perf_to_freq(perf, nominal_freq, perf.highest_perf); + /* try to look up from BIOS/quirk first, fall back to (nominal_freq * highest_perf)/nominal_perf */ + max_freq = amd_get_max_frequency(cpudata->cpu) * 1000; + if (!max_freq) + max_freq = perf_to_freq(perf, nominal_freq, perf.highest_perf); WRITE_ONCE(cpudata->max_freq, max_freq); lowest_nonlinear_freq = perf_to_freq(perf, nominal_freq, perf.lowest_nonlinear_perf); From f6d3cbc63935fa8911da87e91ed768824d495470 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 21 Sep 2026 14:02:32 -0500 Subject: [PATCH 15/34] cpufreq: amd-pstate: Restore previous mode when changing driver mode fails amd_pstate_change_driver_mode() unregisters the currently active driver before registering the requested mode. If amd_pstate_register_driver() fails for the new mode, the function returns the error without restoring anything, leaving the system with no cpufreq scaling driver at all until a valid mode is manually re-selected. Remember the mode that was active before the transition and, if registering the requested mode fails, register the previous mode again so the system keeps a working scaling driver. The original error is still returned to the caller so the sysfs write reports the failure. Reported-by: Sashiko Closes: https://sashiko.dev/#/bug/linux-5b138ddb-c88b-4f82-8a2d-f39446b625d4 Fixes: 3ca7bc818d8c ("cpufreq: amd-pstate: Add guided mode control support via sysfs") Signed-off-by: Mario Limonciello Link: https://lore.kernel.org/r/20260921190235.3651688-1-mario.limonciello@amd.com Signed-off-by: Mario Limonciello --- drivers/cpufreq/amd-pstate.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 743d90ed6af4..a1d58d1f5f3a 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1837,6 +1837,7 @@ static int amd_pstate_change_mode_without_dvr_change(int mode) static int amd_pstate_change_driver_mode(int mode) { + int old_mode = cppc_state; int ret; lockdep_assert_held(&amd_pstate_driver_lock); @@ -1846,10 +1847,16 @@ static int amd_pstate_change_driver_mode(int mode) return ret; ret = amd_pstate_register_driver(mode); - if (ret) - return ret; + if (ret) { + pr_err("Failed to register %s mode, restoring %s mode\n", + amd_pstate_get_mode_string(mode), + amd_pstate_get_mode_string(old_mode)); + if (amd_pstate_register_driver(old_mode)) + pr_err("Failed to restore %s mode\n", + amd_pstate_get_mode_string(old_mode)); + } - return 0; + return ret; } static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = { From 8de705037124dd1cd49bd8cae6536d63ec4e1807 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 21 Sep 2026 14:02:33 -0500 Subject: [PATCH 16/34] cpufreq: amd-pstate: Propagate cppc_set_auto_sel() errors on mode change amd_pstate_change_mode_without_dvr_change() iterates over the online CPUs and calls cppc_set_auto_sel() to enable or disable hardware autonomous selection, but discards its return value and unconditionally returns 0. If the firmware rejects the request, the failure is hidden: the cpufreq core records the mode transition as successful while the hardware stays in its previous autonomous-selection state. The software mode and the actual hardware behaviour then disagree, breaking the expected frequency scaling. Check the return value of cppc_set_auto_sel() and propagate the first error to the caller so the sysfs write reports the failure. Reported-by: Sashiko Closes: https://sashiko.dev/#/bug/linux-d114a94a-655b-43a6-91b1-9484889726c9 Fixes: 3ca7bc818d8c ("cpufreq: amd-pstate: Add guided mode control support via sysfs") Signed-off-by: Mario Limonciello Link: https://lore.kernel.org/r/20260921190235.3651688-2-mario.limonciello@amd.com Signed-off-by: Mario Limonciello --- drivers/cpufreq/amd-pstate.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index a1d58d1f5f3a..d8e2eec6e725 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1822,6 +1822,7 @@ static int amd_pstate_unregister_driver(int dummy) static int amd_pstate_change_mode_without_dvr_change(int mode) { int cpu = 0; + int ret; cppc_state = mode; @@ -1829,7 +1830,9 @@ static int amd_pstate_change_mode_without_dvr_change(int mode) return 0; for_each_online_cpu(cpu) { - cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); + ret = cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); + if (ret) + return ret; } return 0; From 6b64d65c83688db2e58fbd9de7c74090f15271b7 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 21 Sep 2026 14:02:35 -0500 Subject: [PATCH 17/34] cpufreq: amd-pstate-ut: Fix max_freq and EPP test failures on Zen6 Zen6 platforms expose conditions that cause two amd-pstate unit tests to fail: low-power cores can report max_freq below the shared nominal_freq, and multiple EPP preferences can alias to the same raw value. Relax the max_freq >= nominal_freq assertion to max_freq >= lowest_nonlinear_freq, and tolerate aliased EPP preferences by validating the raw programmed value instead of string comparison. Signed-off-by: K Prateek Nayak Tested-by: Mario Limonciello Signed-off-by: Mario Limonciello --- drivers/cpufreq/amd-pstate-ut.c | 54 ++++++++++++++++++++++++--------- drivers/cpufreq/amd-pstate.c | 44 ++------------------------- drivers/cpufreq/amd-pstate.h | 44 +++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 57 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index c2c1a166b3a9..506d7bb2e660 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -216,7 +216,14 @@ static int amd_pstate_ut_check_perf(u32 index) /* * Check if frequency values are reasonable. - * max_freq >= nominal_freq > lowest_nonlinear_freq > min_freq > 0 + * nominal_freq > lowest_nonlinear_freq >= min_freq > 0 + * max_freq >= lowest_nonlinear_freq + * + * On most parts the boost frequency is the highest, i.e. + * max_freq >= nominal_freq. On heterogeneous designs (e.g. Zen6) a + * low-power core can have a maximum frequency below the shared nominal + * reference frequency, so only require the boost frequency to sit above + * the lowest nonlinear frequency here. * check max freq when set support boost mode. */ static int amd_pstate_ut_check_freq(u32 index) @@ -235,11 +242,11 @@ static int amd_pstate_ut_check_freq(u32 index) cpudata = policy->driver_data; perf = READ_ONCE(cpudata->perf); - if (!((policy->cpuinfo.max_freq >= cpudata->nominal_freq) && + if (!((policy->cpuinfo.max_freq >= cpudata->lowest_nonlinear_freq) && (cpudata->nominal_freq > cpudata->lowest_nonlinear_freq) && (cpudata->lowest_nonlinear_freq >= policy->cpuinfo.min_freq) && (policy->cpuinfo.min_freq > 0))) { - pr_err("%s cpu%d max=%d >= nominal=%d > lowest_nonlinear=%d > min=%d > 0, the formula is incorrect!\n", + pr_err("%s cpu%d max=%d, nominal=%d, lowest_nonlinear=%d, min=%d, the formula is incorrect!\n", __func__, cpu, policy->cpuinfo.max_freq, cpudata->nominal_freq, cpudata->lowest_nonlinear_freq, policy->cpuinfo.min_freq); return -EINVAL; @@ -297,12 +304,12 @@ static int amd_pstate_set_mode(enum amd_pstate_mode mode) static int amd_pstate_ut_epp(u32 index) { - static const char * const epp_strings[] = { - "dynamic", - "power", - "balance_power", - "balance_performance", - "performance", + enum energy_perf_value_index epp_modes[] = { + EPP_INDEX_DYNAMIC, + EPP_INDEX_POWERSAVE, + EPP_INDEX_BALANCE_POWERSAVE, + EPP_INDEX_BALANCE_PERFORMANCE, + EPP_INDEX_PERFORMANCE, }; char *buf __free(cleanup_page) = NULL; struct cpufreq_policy *policy = NULL; @@ -371,9 +378,9 @@ static int amd_pstate_ut_epp(u32 index) } } - for (i = 0; i < ARRAY_SIZE(epp_strings); i++) { + for (i = 0; i < ARRAY_SIZE(epp_modes); i++) { memset(buf, 0, PAGE_SIZE); - snprintf(buf, PAGE_SIZE, "%s", epp_strings[i]); + snprintf(buf, PAGE_SIZE, "%s", energy_perf_strings[epp_modes[i]]); ret = store_energy_performance_preference(policy, buf, strlen(buf)); if (ret < 0) goto out; @@ -389,8 +396,24 @@ static int amd_pstate_ut_epp(u32 index) */ strreplace(buf, '(', '\0'); - if (strcmp(buf, epp_strings[i])) { - pr_err("String EPP value mismatch: %s != %s\n", buf, epp_strings[i]); + if (strcmp(buf, energy_perf_strings[epp_modes[i]])) { + u8 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); + + /* + * The preference read back may legitimately differ from the one + * written: some platforms (e.g. Zen6) program the same raw EPP + * value for more than one named preference, so show() reports the + * first name that matches that value. + * + * If strings mismatch, confirm the CPU is programmed with the correct + * EPP value for the mode. This verifies hardware programming while + * tolerating such aliasing in driver. + */ + if (epp == amd_pstate_cpu_epp_values(cpudata->cpu_type)[epp_modes[i]]) + continue; + + pr_err("String EPP value mismatch: %s != %s\n", + buf, energy_perf_strings[epp_modes[i]]); ret = -EINVAL; goto out; } @@ -406,11 +429,12 @@ static int amd_pstate_ut_epp(u32 index) * restore it here before dropping policy reference. */ if (orig_dynamic_epp) { + const char * const dynamic_string = energy_perf_strings[epp_modes[0]]; int ret2; ret2 = store_energy_performance_preference(policy, - epp_strings[0], - strlen(epp_strings[0])); + dynamic_string, + strlen(dynamic_string)); if (!ret && (ret2 < 0)) ret = ret2; } diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index d8e2eec6e725..4bf158f677f3 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -44,7 +44,6 @@ #include #include -#include #include #include @@ -89,46 +88,6 @@ static int cppc_state = AMD_PSTATE_UNDEFINED; static bool amd_pstate_prefcore = true; static struct quirk_entry *quirks; -/* - * AMD Energy Preference Performance (EPP) - * The EPP is used in the CCLK DPM controller to drive - * the frequency that a core is going to operate during - * short periods of activity. EPP values will be utilized for - * different OS profiles (balanced, performance, power savings) - * display strings corresponding to EPP index in the - * energy_perf_strings[] - * index String - *------------------------------------- - * 0 default - * 1 performance - * 2 balance_performance - * 3 balance_power - * 4 power - * 5 custom (for raw EPP values) - * 6 dynamic (platform profile driven selection) - */ -enum energy_perf_value_index { - EPP_INDEX_DEFAULT = 0, - EPP_INDEX_PERFORMANCE, - EPP_INDEX_BALANCE_PERFORMANCE, - EPP_INDEX_BALANCE_POWERSAVE, - EPP_INDEX_POWERSAVE, - EPP_INDEX_CUSTOM, - EPP_INDEX_DYNAMIC, - EPP_INDEX_MAX, -}; - -static const char * const energy_perf_strings[] = { - [EPP_INDEX_DEFAULT] = "default", - [EPP_INDEX_PERFORMANCE] = "performance", - [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", - [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", - [EPP_INDEX_POWERSAVE] = "power", - [EPP_INDEX_CUSTOM] = "custom", - [EPP_INDEX_DYNAMIC] = "dynamic", -}; -static_assert(ARRAY_SIZE(energy_perf_strings) == EPP_INDEX_MAX); - /* * The numeric EPP value programmed for each named preference. First dimension * is CPU type (TOPO_CPU_TYPE_ANY for non-hybrid, TOPO_CPU_TYPE_PERFORMANCE/ @@ -159,7 +118,7 @@ static_assert(ARRAY_SIZE(epp_values) == TOPO_CPU_TYPE_LOW_POWER + 1, * Non-hybrid systems use TOPO_CPU_TYPE_ANY; hybrid systems use the CPU's * actual type (PERFORMANCE/EFFICIENCY/LOW_POWER). */ -static inline u8 *amd_pstate_cpu_epp_values(enum x86_topology_cpu_type cpu_type) +u8 *amd_pstate_cpu_epp_values(enum x86_topology_cpu_type cpu_type) { switch (cpu_type) { case TOPO_CPU_TYPE_PERFORMANCE: @@ -170,6 +129,7 @@ static inline u8 *amd_pstate_cpu_epp_values(enum x86_topology_cpu_type cpu_type) return epp_values[TOPO_CPU_TYPE_ANY]; } } +EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_cpu_epp_values); /** * struct amd_pstate_epp_values - EPP values for the four named preferences diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index c7189d177b81..422188c639b1 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -11,6 +11,8 @@ #include #include +#include + #if IS_MODULE(CONFIG_X86_AMD_PSTATE_UT) #define EXPORT_SYMBOL_FOR_PSTATE_UT(symbol) \ EXPORT_SYMBOL_FOR_MODULES(symbol, "amd-pstate-ut") @@ -162,6 +164,48 @@ enum amd_pstate_mode { AMD_PSTATE_MAX, }; +/* + * AMD Energy Preference Performance (EPP) + * The EPP is used in the CCLK DPM controller to drive + * the frequency that a core is going to operate during + * short periods of activity. EPP values will be utilized for + * different OS profiles (balanced, performance, power savings) + * display strings corresponding to EPP index in the + * energy_perf_strings[] + * index String + *------------------------------------- + * 0 default + * 1 performance + * 2 balance_performance + * 3 balance_power + * 4 power + * 5 custom (for raw EPP values) + * 6 dynamic (platform profile driven selection) + */ +enum energy_perf_value_index { + EPP_INDEX_DEFAULT = 0, + EPP_INDEX_PERFORMANCE, + EPP_INDEX_BALANCE_PERFORMANCE, + EPP_INDEX_BALANCE_POWERSAVE, + EPP_INDEX_POWERSAVE, + EPP_INDEX_CUSTOM, + EPP_INDEX_DYNAMIC, + EPP_INDEX_MAX, +}; + +static const char * const energy_perf_strings[] = { + [EPP_INDEX_DEFAULT] = "default", + [EPP_INDEX_PERFORMANCE] = "performance", + [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", + [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", + [EPP_INDEX_POWERSAVE] = "power", + [EPP_INDEX_CUSTOM] = "custom", + [EPP_INDEX_DYNAMIC] = "dynamic", +}; +static_assert(ARRAY_SIZE(energy_perf_strings) == EPP_INDEX_MAX); + +u8 *amd_pstate_cpu_epp_values(enum x86_topology_cpu_type cpu_type); + static inline u8 freq_to_perf(union perf_cached perf, u32 nominal_freq, unsigned int freq_val) { u32 perf_val = DIV_ROUND_UP_ULL((u64)freq_val * perf.nominal_perf, nominal_freq); From 02c5d7bfa8724789f6620f7787532666edc891a1 Mon Sep 17 00:00:00 2001 From: Mehmet Koseoglu Date: Thu, 1 Oct 2026 13:19:15 +0300 Subject: [PATCH 18/34] rust: cpufreq: reject NULL from cpufreq_cpu_get() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cpufreq_cpu_get() returns either a referenced policy or NULL. PolicyCpu::from_cpu() passed its return value to from_err_ptr(), which rejects ERR_PTR values but accepts NULL. If the lookup fails, Policy::from_raw_mut() therefore constructs a mutable reference from NULL. Dropping the resulting PolicyCpu then passes the invalid pointer to cpufreq_cpu_put(), causing an oops in kobject_put(). Reject NULL with NonNull before constructing the Policy reference. Return ENODEV instead. A KUnit negative-control run reproduced the oops with the original conversion. The same test passed with this change. The reproducer is available on request. Fixes: 6ebdd7c93177 ("rust: cpufreq: Extend abstractions for policy and driver ops") Cc: stable@vger.kernel.org Assisted-by: LLM Signed-off-by: Mehmet Koseoglu Reviewed-by: Onur Özkan Reviewed-by: spidermana Signed-off-by: Viresh Kumar --- rust/kernel/cpufreq.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index affa2b9490ef..b75d692846a2 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -14,7 +14,13 @@ cpumask, device::{Bound, Device}, devres, - error::{code::*, from_err_ptr, from_result, to_result, Result, VTABLE_DEFAULT_ERROR}, + error::{ + code::*, + from_result, + to_result, + Result, + VTABLE_DEFAULT_ERROR, // + }, ffi::{c_char, c_ulong}, prelude::*, types::ForeignOwnable, @@ -29,7 +35,10 @@ marker::PhantomData, ops::{Deref, DerefMut}, pin::Pin, - ptr, + ptr::{ + self, + NonNull, // + }, }; use macros::vtable; @@ -687,12 +696,13 @@ fn clear_data(&mut self) -> Option { impl<'a> PolicyCpu<'a> { fn from_cpu(cpu: CpuId) -> Result { // SAFETY: It is safe to call `cpufreq_cpu_get` for any valid CPU. - let ptr = from_err_ptr(unsafe { bindings::cpufreq_cpu_get(u32::from(cpu)) })?; + let ptr = + NonNull::new(unsafe { bindings::cpufreq_cpu_get(u32::from(cpu)) }).ok_or(ENODEV)?; Ok(Self( - // SAFETY: The `ptr` is guaranteed to be valid and remains valid for the lifetime of - // the returned reference. - unsafe { Policy::from_raw_mut(ptr) }, + // SAFETY: `ptr` is non-NULL and `cpufreq_cpu_get()` took a reference on it, so it is + // valid for writing and remains valid for the lifetime of the returned reference. + unsafe { Policy::from_raw_mut(ptr.as_ptr()) }, )) } } From e5b33fa47792375ca7fc49ccadee87d8e4b72929 Mon Sep 17 00:00:00 2001 From: Xueqin Luo Date: Mon, 31 Aug 2026 13:44:38 +0800 Subject: [PATCH 19/34] cpufreq: sparc-us2e: fix frequency table index copy-paste error In us2e_freq_cpu_init(), the last two frequency writes both use index [2] instead of [3] and [4] respectively, and the terminator uses [3] instead of [5]. This is a copy-paste error where the index was not incremented, causing the divider-6 and divider-8 entries to overwrite the already-written divider-4 entry. As a result, only three frequency steps (div 1, 2, and 4) are actually available to the cpufreq core, while the intended dividers 6 and 8 are silently lost. The struct us2e_freq_percpu_info::table[6] has room for 5 entries plus a terminator, matching the 5 hardware dividers. Fix the indices so all five frequency steps are correctly populated: table[0]=div1, table[1]=div2, table[2]=div4, table[3]=div6, table[4]=div8, table[5]=TABLE_END. Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Signed-off-by: Xueqin Luo Signed-off-by: Viresh Kumar --- drivers/cpufreq/sparc-us2e-cpufreq.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/drivers/cpufreq/sparc-us2e-cpufreq.c b/drivers/cpufreq/sparc-us2e-cpufreq.c index a68706406b88..5cda391ad03b 100644 --- a/drivers/cpufreq/sparc-us2e-cpufreq.c +++ b/drivers/cpufreq/sparc-us2e-cpufreq.c @@ -282,12 +282,11 @@ static int us2e_freq_cpu_init(struct cpufreq_policy *policy) table[1].frequency = clock_tick / 2; table[2].driver_data = 2; table[2].frequency = clock_tick / 4; - table[2].driver_data = 3; - table[2].frequency = clock_tick / 6; - table[2].driver_data = 4; - table[2].frequency = clock_tick / 8; - table[2].driver_data = 5; - table[3].frequency = CPUFREQ_TABLE_END; + table[3].driver_data = 3; + table[3].frequency = clock_tick / 6; + table[4].driver_data = 4; + table[4].frequency = clock_tick / 8; + table[5].frequency = CPUFREQ_TABLE_END; policy->cpuinfo.transition_latency = 0; policy->cur = clock_tick; From f0a750dad7da1c653e85f0698063235453b43831 Mon Sep 17 00:00:00 2001 From: Xueqin Luo Date: Mon, 31 Aug 2026 15:30:08 +0800 Subject: [PATCH 20/34] cpufreq: tegra194: fix double-pointer error in get_cpu_ndiv ndiv is already a u64 pointer, passing &ndiv to smp_call_function_single() results in a u64** being written to instead of the caller's u64 variable, so the caller always reads back an uninitialized ndiv. Drop the spurious '&'. Fixes: 0839ed1fd7ac ("cpufreq: tegra194: add soc data to support multiple soc") Signed-off-by: Xueqin Luo Reviewed-by: Sumit Gupta Signed-off-by: Viresh Kumar --- drivers/cpufreq/tegra194-cpufreq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/tegra194-cpufreq.c b/drivers/cpufreq/tegra194-cpufreq.c index c6375e14d445..1eb96e9f3310 100644 --- a/drivers/cpufreq/tegra194-cpufreq.c +++ b/drivers/cpufreq/tegra194-cpufreq.c @@ -367,7 +367,7 @@ static void tegra194_get_cpu_ndiv_sysreg(void *ndiv) static int tegra194_get_cpu_ndiv(u32 cpu, u32 cpuid, u32 clusterid, u64 *ndiv) { - return smp_call_function_single(cpu, tegra194_get_cpu_ndiv_sysreg, &ndiv, true); + return smp_call_function_single(cpu, tegra194_get_cpu_ndiv_sysreg, ndiv, true); } static void tegra194_set_cpu_ndiv_sysreg(void *data) From fdbe4cae71c8196cd7681ad47254b5b0560d3c1e Mon Sep 17 00:00:00 2001 From: Xueqin Luo Date: Mon, 31 Aug 2026 17:11:59 +0800 Subject: [PATCH 21/34] cpufreq: sti: avoid NULL dereference in dev_err() ddata.cpu is NULL-checked, then immediately passed to dev_err(). Replace with pr_err() to avoid dereferencing NULL. Fixes: ab0ea257fc58 ("cpufreq: st: Provide runtime initialised driver for ST's platforms") Signed-off-by: Xueqin Luo Signed-off-by: Viresh Kumar --- drivers/cpufreq/sti-cpufreq.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/sti-cpufreq.c b/drivers/cpufreq/sti-cpufreq.c index b15b3142b5fe..144ac0f1d85d 100644 --- a/drivers/cpufreq/sti-cpufreq.c +++ b/drivers/cpufreq/sti-cpufreq.c @@ -263,7 +263,7 @@ static int __init sti_cpufreq_init(void) ddata.cpu = get_cpu_device(0); if (!ddata.cpu) { - dev_err(ddata.cpu, "Failed to get device for CPU0\n"); + pr_err("Failed to get device for CPU0\n"); goto skip_voltage_scaling; } @@ -281,7 +281,7 @@ static int __init sti_cpufreq_init(void) goto register_cpufreq_dt; skip_voltage_scaling: - dev_err(ddata.cpu, "Not doing voltage scaling\n"); + pr_err("Not doing voltage scaling\n"); register_cpufreq_dt: platform_device_register_simple("cpufreq-dt", -1, NULL, 0); From 53732733c7af437b0ed710a1df91a67a0d3dfc07 Mon Sep 17 00:00:00 2001 From: Frank Zhang Date: Fri, 4 Sep 2026 15:56:35 +0800 Subject: [PATCH 22/34] rust: rcpufreq_dt: add module alias Add the missing `platform:cpufreq-dt` module alias so that the driver can be automatically loaded when the platform device is registered. Fixes: 06149d8f2216 ("cpufreq: Add Rust-based cpufreq-dt driver") Cc: stable@vger.kernel.org Signed-off-by: Frank Zhang Signed-off-by: Viresh Kumar --- drivers/cpufreq/rcpufreq_dt.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/cpufreq/rcpufreq_dt.rs b/drivers/cpufreq/rcpufreq_dt.rs index d7ead60bf8c2..2b1f18b7a1e7 100644 --- a/drivers/cpufreq/rcpufreq_dt.rs +++ b/drivers/cpufreq/rcpufreq_dt.rs @@ -225,4 +225,5 @@ fn probe<'bound>( authors: ["Viresh Kumar "], description: "Generic CPUFreq DT driver", license: "GPL v2", + alias: ["platform:cpufreq-dt"], } From 3b585d7fa3a1256c8bc3b9bb3b1dcfc3b2ff5d91 Mon Sep 17 00:00:00 2001 From: Sumeet Pawnikar Date: Sun, 13 Sep 2026 00:41:53 +0530 Subject: [PATCH 23/34] cpufreq: Use %pe to print error pointers symbolically Replace PTR_ERR() and %ld with %pe and pass the original pointer directly to pr_err() and pr_warn(). The %pe format specifier prints a symbolic error name (e.g. -ENOMEM) when CONFIG_SYMBOLIC_ERRNAME is enabled, otherwise it falls back gracefully and prints the raw integer value. This makes messages more readable without any functional change. Signed-off-by: Sumeet Pawnikar Signed-off-by: Viresh Kumar --- drivers/cpufreq/bmips-cpufreq.c | 4 ++-- drivers/cpufreq/cppc_cpufreq.c | 4 ++-- drivers/cpufreq/qoriq-cpufreq.c | 3 +-- drivers/cpufreq/s3c64xx-cpufreq.c | 5 ++--- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/drivers/cpufreq/bmips-cpufreq.c b/drivers/cpufreq/bmips-cpufreq.c index a8e35bc75fb2..389fff5e6f65 100644 --- a/drivers/cpufreq/bmips-cpufreq.c +++ b/drivers/cpufreq/bmips-cpufreq.c @@ -132,8 +132,8 @@ static int bmips_cpufreq_init(struct cpufreq_policy *policy) freq_table = bmips_cpufreq_get_freq_table(policy); if (IS_ERR(freq_table)) { - pr_err("%s: couldn't determine frequency table (%ld).\n", - BMIPS_CPUFREQ_NAME, PTR_ERR(freq_table)); + pr_err("%s: couldn't determine frequency table (%pe).\n", + BMIPS_CPUFREQ_NAME, freq_table); return PTR_ERR(freq_table); } diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index 80893844353c..f767898ebfb5 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -230,8 +230,8 @@ static void cppc_fie_kworker_init(void) kworker_fie = kthread_run_worker(0, "cppc_fie"); if (IS_ERR(kworker_fie)) { - pr_warn("%s: failed to create kworker_fie: %ld\n", __func__, - PTR_ERR(kworker_fie)); + pr_warn("%s: failed to create kworker_fie: %pe\n", __func__, + kworker_fie); fie_disabled = FIE_DISABLED; kworker_fie = NULL; return; diff --git a/drivers/cpufreq/qoriq-cpufreq.c b/drivers/cpufreq/qoriq-cpufreq.c index 42edb41ad459..a21b6f01218a 100644 --- a/drivers/cpufreq/qoriq-cpufreq.c +++ b/drivers/cpufreq/qoriq-cpufreq.c @@ -57,8 +57,7 @@ static u32 get_bus_freq(void) /* get platform freq by its clock name */ pltclk = clk_get(NULL, "cg-pll0-div1"); if (IS_ERR(pltclk)) { - pr_err("%s: can't get bus frequency %ld\n", - __func__, PTR_ERR(pltclk)); + pr_err("%s: can't get bus frequency %pe\n", __func__, pltclk); return PTR_ERR(pltclk); } diff --git a/drivers/cpufreq/s3c64xx-cpufreq.c b/drivers/cpufreq/s3c64xx-cpufreq.c index 9cef71528076..9a01592425ee 100644 --- a/drivers/cpufreq/s3c64xx-cpufreq.c +++ b/drivers/cpufreq/s3c64xx-cpufreq.c @@ -152,15 +152,14 @@ static int s3c64xx_cpufreq_driver_init(struct cpufreq_policy *policy) policy->clk = clk_get(NULL, "armclk"); if (IS_ERR(policy->clk)) { - pr_err("Unable to obtain ARMCLK: %ld\n", - PTR_ERR(policy->clk)); + pr_err("Unable to obtain ARMCLK: %pe\n", policy->clk); return PTR_ERR(policy->clk); } #ifdef CONFIG_REGULATOR vddarm = regulator_get(NULL, "vddarm"); if (IS_ERR(vddarm)) { - pr_err("Failed to obtain VDDARM: %ld\n", PTR_ERR(vddarm)); + pr_err("Failed to obtain VDDARM: %pe\n", vddarm); pr_err("Only frequency scaling available\n"); vddarm = NULL; } else { From cce713d008bdcca5e46541de86eab79b80d394bc Mon Sep 17 00:00:00 2001 From: Alexander Koskovich Date: Mon, 5 Oct 2026 17:47:27 +0000 Subject: [PATCH 24/34] cpufreq: Add Milos to cpufreq-dt-platdev blocklist The Qualcomm Milos platform uses the qcom-cpufreq-hw driver so add it to the cpufreq-dt-platdev driver's blocklist. Signed-off-by: Alexander Koskovich Signed-off-by: Viresh Kumar --- drivers/cpufreq/cpufreq-dt-platdev.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c index 371ca1fe9020..574c5bec4831 100644 --- a/drivers/cpufreq/cpufreq-dt-platdev.c +++ b/drivers/cpufreq/cpufreq-dt-platdev.c @@ -155,6 +155,7 @@ static const struct of_device_id blocklist[] __initconst = { { .compatible = "nvidia,tegra234", }, { .compatible = "qcom,apq8096", }, + { .compatible = "qcom,milos", }, { .compatible = "qcom,msm8909", }, { .compatible = "qcom,msm8996", }, { .compatible = "qcom,msm8998", }, From 95cc3927ce26cadc325833a16b13c2a3c08353de Mon Sep 17 00:00:00 2001 From: Kathiravan Thirumoorthy Date: Wed, 7 Oct 2026 12:31:56 +0530 Subject: [PATCH 25/34] cpufreq: qcom-nvmem: Add IPQ9650 support Add IPQ9650 support to the Qualcomm NVMEM cpufreq driver so the supported OPP can be selected at runtime using the eFuse value and the opp-supported-hw OPP property. Also block the generic cpufreq-dt platform device for IPQ9650 so the NVMEM-based driver is used. Signed-off-by: Kathiravan Thirumoorthy Reviewed-by: Dmitry Baryshkov Signed-off-by: Viresh Kumar --- drivers/cpufreq/cpufreq-dt-platdev.c | 1 + drivers/cpufreq/qcom-cpufreq-nvmem.c | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c index 574c5bec4831..bce4c87573a1 100644 --- a/drivers/cpufreq/cpufreq-dt-platdev.c +++ b/drivers/cpufreq/cpufreq-dt-platdev.c @@ -210,6 +210,7 @@ static const struct of_device_id blocklist[] __initconst = { { .compatible = "qcom,ipq8064", }, { .compatible = "qcom,ipq8074", }, { .compatible = "qcom,ipq9574", }, + { .compatible = "qcom,ipq9650", }, { .compatible = "qcom,apq8064", }, { .compatible = "qcom,msm8974", }, { .compatible = "qcom,msm8960", }, diff --git a/drivers/cpufreq/qcom-cpufreq-nvmem.c b/drivers/cpufreq/qcom-cpufreq-nvmem.c index efa766e98d86..375f12d5d11a 100644 --- a/drivers/cpufreq/qcom-cpufreq-nvmem.c +++ b/drivers/cpufreq/qcom-cpufreq-nvmem.c @@ -211,6 +211,14 @@ static int qcom_cpufreq_kryo_name_version(struct device *cpu_dev, case QCOM_ID_IPQ5404: drv->versions = (*speedbin == 0x3b) ? BIT(1) : BIT(0); break; + case QCOM_ID_IPQ9610: + case QCOM_ID_IPQ9620: + case QCOM_ID_IPQ9630: + case QCOM_ID_IPQ9640: + case QCOM_ID_IPQ9650: + case QCOM_ID_IPQ9670: + drv->versions = (*speedbin == 0x4b) ? BIT(3) : BIT(2); + break; case QCOM_ID_MSM8996SG: case QCOM_ID_APQ8096SG: drv->versions = 1 << ((unsigned int)(*speedbin) + 4); @@ -633,6 +641,7 @@ static const struct of_device_id qcom_cpufreq_match_list[] __initconst __maybe_u { .compatible = "qcom,ipq8074", .data = &match_data_ipq8074 }, { .compatible = "qcom,apq8064", .data = &match_data_krait }, { .compatible = "qcom,ipq9574", .data = &match_data_kryo }, + { .compatible = "qcom,ipq9650", .data = &match_data_kryo }, { .compatible = "qcom,msm8974", .data = &match_data_krait }, { .compatible = "qcom,msm8960", .data = &match_data_krait }, {}, From 6fa50b1988b8dfd2e92a9cecba073fe4347a5e23 Mon Sep 17 00:00:00 2001 From: Haotian Zhang Date: Fri, 9 Oct 2026 01:24:42 +0800 Subject: [PATCH 26/34] cpufreq: qcom-cpufreq-hw: fix device node refcount leak in phandle parsing of_parse_phandle_with_args() transfers a reference to the provider node in out_args->np, which the caller is responsible for releasing with of_node_put(). Neither qcom_get_related_cpus() nor qcom_cpufreq_hw_cpu_init() ever puts args.np: they only put cpu_np, which is a different node obtained from of_cpu_device_node_get(). The reference is therefore leaked on every successful call, and qcom_get_related_cpus() leaks one per present CPU. Release args.np after the parsed index has been read in both call sites. Fixes: 2849dd8bc72b ("cpufreq: qcom-hw: Add support for QCOM cpufreq HW driver") Assisted-by: DeepSeek-V4.1-Flash Signed-off-by: Haotian Zhang Signed-off-by: Viresh Kumar --- drivers/cpufreq/qcom-cpufreq-hw.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c index 874ff3fb9d97..060a460716f7 100644 --- a/drivers/cpufreq/qcom-cpufreq-hw.c +++ b/drivers/cpufreq/qcom-cpufreq-hw.c @@ -322,6 +322,8 @@ static void qcom_get_related_cpus(int index, struct cpumask *m) if (index == args.args[0]) cpumask_set_cpu(cpu, m); + + of_node_put(args.np); } } @@ -554,6 +556,7 @@ static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) return ret; index = args.args[0]; + of_node_put(args.np); data = &qcom_cpufreq.data[index]; /* HW should be in enabled state to proceed */ From a4b90518dd01f85e1e4bf7e9361396b2eded1f7e Mon Sep 17 00:00:00 2001 From: Ryan Brue Date: Fri, 9 Oct 2026 14:44:43 -0500 Subject: [PATCH 27/34] cpufreq: mediatek: Enter system sleep at the suspend OPP cpufreq_suspend() stops the governors at the start of system suspend, and this driver has no .suspend callback, so each cluster sleeps at whatever OPP it was last left at. On an MT8173 tablet (Amazon Fire HD 10, 2017), entering suspend-to-RAM with both clusters at their highest OPP measured 63.5 mA. With this change and the lowest OPP marked, the same entry measured 52.0 mA. Use the generic suspend-frequency support, as cpufreq-dt does. Tables without an opp-suspend entry leave suspend_freq at 0, which makes cpufreq_generic_suspend() a no-op, so they are unaffected. Assisted-by: LLM Signed-off-by: Ryan Brue Signed-off-by: Viresh Kumar --- drivers/cpufreq/mediatek-cpufreq.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/cpufreq/mediatek-cpufreq.c b/drivers/cpufreq/mediatek-cpufreq.c index 052ca7cd2f4f..f9b1eca176eb 100644 --- a/drivers/cpufreq/mediatek-cpufreq.c +++ b/drivers/cpufreq/mediatek-cpufreq.c @@ -603,6 +603,7 @@ static int mtk_cpufreq_init(struct cpufreq_policy *policy) policy->freq_table = freq_table; policy->driver_data = info; policy->clk = info->cpu_clk; + policy->suspend_freq = dev_pm_opp_get_suspend_opp_freq(info->cpu_dev) / 1000; return 0; } @@ -624,6 +625,7 @@ static struct cpufreq_driver mtk_cpufreq_driver = { .init = mtk_cpufreq_init, .exit = mtk_cpufreq_exit, .register_em = cpufreq_register_em_with_opp, + .suspend = cpufreq_generic_suspend, .name = "mtk-cpufreq", }; From 71245acb8c937382fc31806572314d36667b17f6 Mon Sep 17 00:00:00 2001 From: Ryan Brue Date: Fri, 9 Oct 2026 14:44:44 -0500 Subject: [PATCH 28/34] arm64: dts: mediatek: mt8173: Mark the lowest CPU OPPs for suspend Both clusters otherwise enter system sleep at whatever OPP the governor last chose. On the Amazon Fire HD 10 (2017), entering suspend-to-RAM with both clusters at their highest OPP measured 63.5 mA, against 52.0 mA when they suspend at 507 MHz. Mark the 507 MHz OPP of each cluster, the lowest in both tables, as the one to suspend at. mediatek-cpufreq uses it for system sleep, and a kernel without that support ignores it. Assisted-by: LLM Signed-off-by: Ryan Brue Signed-off-by: Viresh Kumar --- arch/arm64/boot/dts/mediatek/mt8173.dtsi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/boot/dts/mediatek/mt8173.dtsi b/arch/arm64/boot/dts/mediatek/mt8173.dtsi index 78c2ccd5be13..7f1af15cb934 100644 --- a/arch/arm64/boot/dts/mediatek/mt8173.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8173.dtsi @@ -56,6 +56,7 @@ cluster0_opp: opp-table-0 { opp-507000000 { opp-hz = /bits/ 64 <507000000>; opp-microvolt = <859000>; + opp-suspend; }; opp-702000000 { opp-hz = /bits/ 64 <702000000>; @@ -93,6 +94,7 @@ cluster1_opp: opp-table-1 { opp-507000000 { opp-hz = /bits/ 64 <507000000>; opp-microvolt = <828000>; + opp-suspend; }; opp-702000000 { opp-hz = /bits/ 64 <702000000>; From ce390caf5541c493c9061e7a2d98a463e66b736a Mon Sep 17 00:00:00 2001 From: Hui Su Date: Tue, 13 Oct 2026 12:42:35 +0900 Subject: [PATCH 29/34] cpufreq: schedutil: Serialize sugov_start() against rate limit updates The current value of sg_policy->tunables->rate_limit_us is used in sugov_start() without locking to update sg_policy->freq_update_delay_ns, so if rate_limit_us_store() runs in parallel with it, the rate limit update may go unnoticed and the new freq_update_delay_ns value may be stale until the rate limit is updated again. To avoid that, acquire the update_lock mutex of the rate limit sysfs attribute, that is held by rate_limit_us_store() during rate limit updates, around the freq_update_delay_ns in sugov_start(). Fixes: 9bdcb44e391d ("cpufreq: schedutil: New governor based on scheduler utilization data") Reviewed-by: Zhongqiu Han Signed-off-by: Hui Su Cc: All applicable [ rjw: Subject adjustment, new changelog, edits of the new comment ] [ rjw: Add empty code line after guard()() ] Link: https://patch.msgid.link/20261013034235.2678177-1-sh_def@163.com Signed-off-by: Rafael J. Wysocki --- kernel/sched/cpufreq_schedutil.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index 49ccd6f1c185..14405a614991 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -73,6 +73,18 @@ static void sugov_update_rate_limit_us(struct sugov_policy *sg_policy) (s64)sg_policy->tunables->rate_limit_us * NSEC_PER_USEC; } +static void sugov_update_rate_limit_us_sync(struct sugov_policy *sg_policy) +{ + /* + * Serialize with rate_limit_us_store() to prevent this path from using + * a stale rate limit tunable value to set the limit for the given + * policy. + */ + guard(mutex)(&sg_policy->tunables->attr_set.update_lock); + + sugov_update_rate_limit_us(sg_policy); +} + static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time) { s64 delta_ns; @@ -861,7 +873,7 @@ static int sugov_start(struct cpufreq_policy *policy) void (*uu)(struct update_util_data *data, u64 time, unsigned int flags); unsigned int cpu; - sugov_update_rate_limit_us(sg_policy); + sugov_update_rate_limit_us_sync(sg_policy); sg_policy->last_freq_update_time = 0; sg_policy->next_freq = 0; sg_policy->work_in_progress = false; From 48a82a3f44576a1b227f964fdb150e1cae7455bf Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Fri, 28 Aug 2026 11:11:57 +0800 Subject: [PATCH 30/34] cpufreq: loongson3: Make this driver depend on MACH_LOONGSON64 32BIT Loongson machines don't have SMC and FreqCtrl registers, so make this driver depend on MACH_LOONGSON64 (only for LoongArch based 64bit machines currently). Cc: stable@vger.kernel.org Fixes: ccf51454145b ("cpufreq: Add Loongson-3 CPUFreq driver support") Signed-off-by: Huacai Chen Reviewed-by: Zhongqiu Han Signed-off-by: Viresh Kumar --- drivers/cpufreq/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index db83f3365698..edc1299098d4 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -288,6 +288,7 @@ endif if LOONGARCH config LOONGSON3_CPUFREQ tristate "Loongson3 CPUFreq Driver" + depends on MACH_LOONGSON64 help This option adds a CPUFreq driver for Loongson processors which support software configurable cpu frequency. From 290814d3c22b300eb36b74dc76ad36bd7fdc1220 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Fri, 28 Aug 2026 11:11:58 +0800 Subject: [PATCH 31/34] cpufreq: loongson3: Adjust the width of id and val in smc_message The id field of smc_message is usually stand for the CPU ID. In the past this driver was only tested for desktop and laptop productions so 4-bits are enough. But now we have servers that can have as many as 256 cores, so we need 8-bits CPU ID. On the other hand, the val field is usually stand for CPU frequency so 12-bits are enough to represent 4GHz, and for higher frequencies there is an extra bit to extend (enough for 8GHz). In theory, this is a incompatible change, but fortunately the old SMC firmwares are not widely shipped and can be updated on the air, thus we can safely adjust the widths. Cc: stable@vger.kernel.org Fixes: ccf51454145b ("cpufreq: Add Loongson-3 CPUFreq driver support") Signed-off-by: Hongliang Wang Signed-off-by: Huacai Chen Reviewed-by: Zhongqiu Han Signed-off-by: Viresh Kumar --- drivers/cpufreq/loongson3_cpufreq.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/loongson3_cpufreq.c b/drivers/cpufreq/loongson3_cpufreq.c index 1e8715ea1b77..630f679aa739 100644 --- a/drivers/cpufreq/loongson3_cpufreq.c +++ b/drivers/cpufreq/loongson3_cpufreq.c @@ -21,9 +21,9 @@ union smc_message { u32 value; struct { - u32 id : 4; + u32 id : 8; u32 info : 4; - u32 val : 16; + u32 val : 12; u32 cmd : 6; u32 extra : 1; u32 complete : 1; From 5d7802ada30b3101f408706b7399b0e170d2febd Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Fri, 28 Aug 2026 11:11:59 +0800 Subject: [PATCH 32/34] cpufreq: loongson3: Replace per-package mutex with per-node Our server productions (e.g. Loongson-3D6000/3E6000) can have multiple nodes in one package and SMC mailboxes are also per-node. So replace the per-package mutex with per-node one. Cc: stable@vger.kernel.org Fixes: ccf51454145b ("cpufreq: Add Loongson-3 CPUFreq driver support") Signed-off-by: Hongliang Wang Signed-off-by: Huacai Chen Reviewed-by: Zhongqiu Han Signed-off-by: Viresh Kumar --- drivers/cpufreq/loongson3_cpufreq.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/cpufreq/loongson3_cpufreq.c b/drivers/cpufreq/loongson3_cpufreq.c index 630f679aa739..e3cd78a5ab18 100644 --- a/drivers/cpufreq/loongson3_cpufreq.c +++ b/drivers/cpufreq/loongson3_cpufreq.c @@ -169,7 +169,7 @@ struct loongson3_freq_data { struct cpufreq_frequency_table table[]; }; -static struct mutex cpufreq_mutex[MAX_PACKAGES]; +static struct mutex cpufreq_mutex[MAX_NUMNODES]; static struct cpufreq_driver loongson3_cpufreq_driver; static DEFINE_PER_CPU(struct loongson3_freq_data *, freq_data); @@ -177,14 +177,14 @@ static inline int do_service_request(u32 id, u32 info, u32 cmd, u32 val, u32 ext { int retries; unsigned int cpu = raw_smp_processor_id(); - unsigned int package = cpu_data[cpu].package; + unsigned int nid = cpu_to_node(cpu); union smc_message msg, last; - mutex_lock(&cpufreq_mutex[package]); + mutex_lock(&cpufreq_mutex[nid]); last.value = iocsr_read32(LOONGARCH_IOCSR_SMCMBX); if (!last.complete) { - mutex_unlock(&cpufreq_mutex[package]); + mutex_unlock(&cpufreq_mutex[nid]); return -EPERM; } @@ -208,11 +208,11 @@ static inline int do_service_request(u32 id, u32 info, u32 cmd, u32 val, u32 ext } if (!msg.complete || msg.cmd != CMD_OK) { - mutex_unlock(&cpufreq_mutex[package]); + mutex_unlock(&cpufreq_mutex[nid]); return -EPERM; } - mutex_unlock(&cpufreq_mutex[package]); + mutex_unlock(&cpufreq_mutex[nid]); return msg.val; } @@ -337,7 +337,7 @@ static int loongson3_cpufreq_probe(struct platform_device *pdev) { int i, ret; - for (i = 0; i < MAX_PACKAGES; i++) { + for (i = 0; i < MAX_NUMNODES; i++) { ret = devm_mutex_init(&pdev->dev, &cpufreq_mutex[i]); if (ret) return ret; From e38360d7c62c8fa530b58905f99a4557d4d6b352 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Fri, 28 Aug 2026 11:12:00 +0800 Subject: [PATCH 33/34] cpufreq: loongson3: Use global physical CPU ID in get/target callbacks Our server productions (e.g. Loongson-3D6000/3E6000) can have discrete global physical CPU IDs while the core ID inside the packages are always continuous. In these cases we should use global physical CPU IDs to get and set frequencies. Cc: stable@vger.kernel.org Fixes: ccf51454145b ("cpufreq: Add Loongson-3 CPUFreq driver support") Signed-off-by: Hongliang Wang Signed-off-by: Huacai Chen Reviewed-by: Zhongqiu Han Signed-off-by: Viresh Kumar --- drivers/cpufreq/loongson3_cpufreq.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/drivers/cpufreq/loongson3_cpufreq.c b/drivers/cpufreq/loongson3_cpufreq.c index e3cd78a5ab18..1ae821e265fb 100644 --- a/drivers/cpufreq/loongson3_cpufreq.c +++ b/drivers/cpufreq/loongson3_cpufreq.c @@ -16,6 +16,7 @@ #include #include #include +#include /* Message */ union smc_message { @@ -219,38 +220,40 @@ static inline int do_service_request(u32 id, u32 info, u32 cmd, u32 val, u32 ext static unsigned int loongson3_cpufreq_get(unsigned int cpu) { - int ret; + int ret, core = cpu_logical_map(cpu); - ret = do_service_request(cpu, FREQ_INFO_TYPE_FREQ, CMD_GET_FREQ_INFO, 0, 0); + ret = do_service_request(core, FREQ_INFO_TYPE_FREQ, CMD_GET_FREQ_INFO, 0, 0); - return ret * KILO; + return (ret < 0) ? 0 : ret * KILO; } static int loongson3_cpufreq_target(struct cpufreq_policy *policy, unsigned int index) { - int ret; + int ret, core = cpu_logical_map(policy->cpu); - ret = do_service_request(cpu_data[policy->cpu].core, - FREQ_INFO_TYPE_LEVEL, CMD_SET_FREQ_INFO, index, 0); + ret = do_service_request(core, FREQ_INFO_TYPE_LEVEL, CMD_SET_FREQ_INFO, + index, 0); return (ret >= 0) ? 0 : ret; } static int configure_freq_table(int cpu) { - int i, ret, boost_level, max_level, freq_level; + int i, ret, core, boost_level, max_level, freq_level; struct platform_device *pdev = cpufreq_get_driver_data(); struct loongson3_freq_data *data; if (per_cpu(freq_data, cpu)) return 0; - ret = do_service_request(cpu, 0, CMD_GET_FREQ_LEVEL_NUM, 0, 0); + core = cpu_logical_map(cpu); + + ret = do_service_request(core, 0, CMD_GET_FREQ_LEVEL_NUM, 0, 0); if (ret < 0) return ret; max_level = ret; - ret = do_service_request(cpu, 0, CMD_GET_FREQ_BOOST_LEVEL, 0, 0); + ret = do_service_request(core, 0, CMD_GET_FREQ_BOOST_LEVEL, 0, 0); if (ret < 0) return ret; boost_level = ret; @@ -263,7 +266,7 @@ static int configure_freq_table(int cpu) data->def_freq_level = boost_level - 1; for (i = 0; i < freq_level; i++) { - ret = do_service_request(cpu, FREQ_INFO_TYPE_FREQ, CMD_GET_FREQ_LEVEL_INFO, i, 0); + ret = do_service_request(core, FREQ_INFO_TYPE_FREQ, CMD_GET_FREQ_LEVEL_INFO, i, 0); if (ret < 0) { devm_kfree(&pdev->dev, data); return ret; From 81792daacfb41ca07b04bd3956ea9e672080cc98 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Fri, 28 Aug 2026 11:12:01 +0800 Subject: [PATCH 34/34] cpufreq: loongson3: Replace IOCSR read/write with MMIO ones Our server productions (e.g. Loongson-3D6000/3E6000) can have multiple nodes in one package and SMC mailboxes are also per-node. However, IOCSR read/write can only perform on the current node, while sometimes we want to perform on other nodes (e.g. when switch governor, the get and target callbacks are not run on target core). So replace IOCSR read/write with MMIO ones. Cc: stable@vger.kernel.org Fixes: ccf51454145b ("cpufreq: Add Loongson-3 CPUFreq driver support") Tested-by: Xi Ruoyao Signed-off-by: Huacai Chen Reviewed-by: Zhongqiu Han Signed-off-by: Viresh Kumar --- drivers/cpufreq/loongson3_cpufreq.c | 34 +++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/drivers/cpufreq/loongson3_cpufreq.c b/drivers/cpufreq/loongson3_cpufreq.c index 1ae821e265fb..aea7c31e155a 100644 --- a/drivers/cpufreq/loongson3_cpufreq.c +++ b/drivers/cpufreq/loongson3_cpufreq.c @@ -17,6 +17,7 @@ #include #include #include +#include /* Message */ union smc_message { @@ -165,6 +166,12 @@ union smc_message { #define FREQ_MAX_LEVEL 16 +#define MMIO_SMCMBX(node) \ + ((void __iomem *)(nid_to_addrbase(node) | IO_BASE | LOONGSON_REG_BASE | LOONGARCH_IOCSR_SMCMBX)) + +#define MMIO_MISC_FUNC(node) \ + ((void __iomem *)(nid_to_addrbase(node) | IO_BASE | LOONGSON_REG_BASE | LOONGARCH_IOCSR_MISC_FUNC)) + struct loongson3_freq_data { unsigned int def_freq_level; struct cpufreq_frequency_table table[]; @@ -177,13 +184,27 @@ static DEFINE_PER_CPU(struct loongson3_freq_data *, freq_data); static inline int do_service_request(u32 id, u32 info, u32 cmd, u32 val, u32 extra) { int retries; - unsigned int cpu = raw_smp_processor_id(); - unsigned int nid = cpu_to_node(cpu); + unsigned int cpu = 0, nid; union smc_message msg, last; +#ifdef CONFIG_SMP + switch (cmd) { + case CMD_GET_FREQ_INFO: + case CMD_SET_FREQ_INFO: + case CMD_GET_FREQ_LEVEL_NUM: + case CMD_GET_FREQ_LEVEL_INFO: + case CMD_GET_FREQ_BOOST_LEVEL: + cpu = cpu_number_map(id); + break; + default: + cpu = raw_smp_processor_id(); + } +#endif + nid = cpu_to_node(cpu); + mutex_lock(&cpufreq_mutex[nid]); - last.value = iocsr_read32(LOONGARCH_IOCSR_SMCMBX); + last.value = readl(MMIO_SMCMBX(nid)); if (!last.complete) { mutex_unlock(&cpufreq_mutex[nid]); return -EPERM; @@ -196,12 +217,11 @@ static inline int do_service_request(u32 id, u32 info, u32 cmd, u32 val, u32 ext msg.extra = extra; msg.complete = 0; - iocsr_write32(msg.value, LOONGARCH_IOCSR_SMCMBX); - iocsr_write32(iocsr_read32(LOONGARCH_IOCSR_MISC_FUNC) | IOCSR_MISC_FUNC_SOFT_INT, - LOONGARCH_IOCSR_MISC_FUNC); + writel(msg.value, MMIO_SMCMBX(nid)); + writel(readl(MMIO_MISC_FUNC(nid)) | IOCSR_MISC_FUNC_SOFT_INT, MMIO_MISC_FUNC(nid)); for (retries = 0; retries < 10000; retries++) { - msg.value = iocsr_read32(LOONGARCH_IOCSR_SMCMBX); + msg.value = readl(MMIO_SMCMBX(nid)); if (msg.complete) break;