From 387b51709db546033cf9a940860b6614a5cda5b6 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 28 Mar 2025 21:40:43 +0100 Subject: [PATCH 01/32] cpufreq: Consolidate some code in cpufreq_online() Notice that the policy->cpu update in cpufreq_policy_alloc() can be moved to cpufreq_online() and then it can be carried out under the policy rwsem, along with the clearing of policy->governor (unnecessary in the "new policy" code branch, but also not harmful). If this is done, the bottom parts of the "if (policy)" branches become identical and they can be collapsed and moved below the conditional. Modify the code accordingly which makes it somewhat easier to follow. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar Reviewed-by: Mario Limonciello Acked-by: Sudeep Holla Tested-by: Sudeep Holla Link: https://patch.msgid.link/13741234.uLZWGnKmhe@rjwysocki.net --- drivers/cpufreq/cpufreq.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 3841c9da6cac..72c31a99c5c3 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1334,7 +1334,6 @@ static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) init_waitqueue_head(&policy->transition_wait); INIT_WORK(&policy->update, handle_update); - policy->cpu = cpu; return policy; err_min_qos_notifier: @@ -1422,17 +1421,18 @@ static int cpufreq_online(unsigned int cpu) /* This is the only online CPU for the policy. Start over. */ new_policy = false; - down_write(&policy->rwsem); - policy->cpu = cpu; - policy->governor = NULL; } else { new_policy = true; policy = cpufreq_policy_alloc(cpu); if (!policy) return -ENOMEM; - down_write(&policy->rwsem); } + down_write(&policy->rwsem); + + policy->cpu = cpu; + policy->governor = NULL; + if (!new_policy && cpufreq_driver->online) { /* Recover policy->cpus using related_cpus */ cpumask_copy(policy->cpus, policy->related_cpus); From 68974e3a15b92639396f39d1cd09d5288ba687bc Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 28 Mar 2025 21:41:49 +0100 Subject: [PATCH 02/32] cpufreq: Split cpufreq_online() In preparation for the introduction of cpufreq policy locking guards, move the part of cpufreq_online() that is carried out under the policy rwsem into a separate function called cpufreq_policy_online(). No intentional functional impact. Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar Reviewed-by: Mario Limonciello Acked-by: Sudeep Holla Tested-by: Sudeep Holla Link: https://patch.msgid.link/3354747.aeNJFYEL58@rjwysocki.net --- drivers/cpufreq/cpufreq.c | 95 ++++++++++++++++++++++----------------- 1 file changed, 54 insertions(+), 41 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 72c31a99c5c3..31ede4d998d2 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1402,32 +1402,13 @@ static void cpufreq_policy_free(struct cpufreq_policy *policy) kfree(policy); } -static int cpufreq_online(unsigned int cpu) +static int cpufreq_policy_online(struct cpufreq_policy *policy, + unsigned int cpu, bool new_policy) { - struct cpufreq_policy *policy; - bool new_policy; unsigned long flags; unsigned int j; int ret; - pr_debug("%s: bringing CPU%u online\n", __func__, cpu); - - /* Check if this CPU already has a policy to manage it */ - policy = per_cpu(cpufreq_cpu_data, cpu); - if (policy) { - WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus)); - if (!policy_is_inactive(policy)) - return cpufreq_add_policy_cpu(policy, cpu); - - /* This is the only online CPU for the policy. Start over. */ - new_policy = false; - } else { - new_policy = true; - policy = cpufreq_policy_alloc(cpu); - if (!policy) - return -ENOMEM; - } - down_write(&policy->rwsem); policy->cpu = cpu; @@ -1454,7 +1435,7 @@ static int cpufreq_online(unsigned int cpu) if (ret) { pr_debug("%s: %d: initialization failed\n", __func__, __LINE__); - goto out_free_policy; + goto out_clear_policy; } /* @@ -1605,8 +1586,59 @@ static int cpufreq_online(unsigned int cpu) goto out_destroy_policy; } +out_unlock: up_write(&policy->rwsem); + return ret; + +out_destroy_policy: + for_each_cpu(j, policy->real_cpus) + remove_cpu_dev_symlink(policy, j, get_cpu_device(j)); + +out_offline_policy: + if (cpufreq_driver->offline) + cpufreq_driver->offline(policy); + +out_exit_policy: + if (cpufreq_driver->exit) + cpufreq_driver->exit(policy); + +out_clear_policy: + cpumask_clear(policy->cpus); + + goto out_unlock; +} + +static int cpufreq_online(unsigned int cpu) +{ + struct cpufreq_policy *policy; + bool new_policy; + int ret; + + pr_debug("%s: bringing CPU%u online\n", __func__, cpu); + + /* Check if this CPU already has a policy to manage it */ + policy = per_cpu(cpufreq_cpu_data, cpu); + if (policy) { + WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus)); + if (!policy_is_inactive(policy)) + return cpufreq_add_policy_cpu(policy, cpu); + + /* This is the only online CPU for the policy. Start over. */ + new_policy = false; + } else { + new_policy = true; + policy = cpufreq_policy_alloc(cpu); + if (!policy) + return -ENOMEM; + } + + ret = cpufreq_policy_online(policy, cpu, new_policy); + if (ret) { + cpufreq_policy_free(policy); + return ret; + } + kobject_uevent(&policy->kobj, KOBJ_ADD); /* Callback for handling stuff after policy is ready */ @@ -1633,25 +1665,6 @@ static int cpufreq_online(unsigned int cpu) pr_debug("initialization complete\n"); return 0; - -out_destroy_policy: - for_each_cpu(j, policy->real_cpus) - remove_cpu_dev_symlink(policy, j, get_cpu_device(j)); - -out_offline_policy: - if (cpufreq_driver->offline) - cpufreq_driver->offline(policy); - -out_exit_policy: - if (cpufreq_driver->exit) - cpufreq_driver->exit(policy); - -out_free_policy: - cpumask_clear(policy->cpus); - up_write(&policy->rwsem); - - cpufreq_policy_free(policy); - return ret; } /** From 6fec833b9d70c54ceacbf7d07665215fbd0cddef Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 28 Mar 2025 21:42:48 +0100 Subject: [PATCH 03/32] cpufreq: Add and use cpufreq policy locking guards Introduce "read" and "write" locking guards for cpufreq policies and use them where applicable in the cpufreq core. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar Reviewed-by: Mario Limonciello Acked-by: Sudeep Holla Tested-by: Sudeep Holla Link: https://patch.msgid.link/8518682.T7Z3S40VBb@rjwysocki.net --- drivers/cpufreq/cpufreq.c | 127 +++++++++++++++++--------------------- include/linux/cpufreq.h | 6 ++ 2 files changed, 63 insertions(+), 70 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 31ede4d998d2..a3abd9c0440d 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1009,17 +1009,16 @@ static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf) { struct cpufreq_policy *policy = to_policy(kobj); struct freq_attr *fattr = to_attr(attr); - ssize_t ret = -EBUSY; if (!fattr->show) return -EIO; - down_read(&policy->rwsem); - if (likely(!policy_is_inactive(policy))) - ret = fattr->show(policy, buf); - up_read(&policy->rwsem); + guard(cpufreq_policy_read)(policy); - return ret; + if (likely(!policy_is_inactive(policy))) + return fattr->show(policy, buf); + + return -EBUSY; } static ssize_t store(struct kobject *kobj, struct attribute *attr, @@ -1027,17 +1026,16 @@ static ssize_t store(struct kobject *kobj, struct attribute *attr, { struct cpufreq_policy *policy = to_policy(kobj); struct freq_attr *fattr = to_attr(attr); - ssize_t ret = -EBUSY; if (!fattr->store) return -EIO; - down_write(&policy->rwsem); - if (likely(!policy_is_inactive(policy))) - ret = fattr->store(policy, buf, count); - up_write(&policy->rwsem); + guard(cpufreq_policy_write)(policy); - return ret; + if (likely(!policy_is_inactive(policy))) + return fattr->store(policy, buf, count); + + return -EBUSY; } static void cpufreq_sysfs_release(struct kobject *kobj) @@ -1195,7 +1193,8 @@ static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cp if (cpumask_test_cpu(cpu, policy->cpus)) return 0; - down_write(&policy->rwsem); + guard(cpufreq_policy_write)(policy); + if (has_target()) cpufreq_stop_governor(policy); @@ -1206,7 +1205,7 @@ static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cp if (ret) pr_err("%s: Failed to start governor\n", __func__); } - up_write(&policy->rwsem); + return ret; } @@ -1226,9 +1225,10 @@ static void handle_update(struct work_struct *work) container_of(work, struct cpufreq_policy, update); pr_debug("handle_update for cpu %u called\n", policy->cpu); - down_write(&policy->rwsem); + + guard(cpufreq_policy_write)(policy); + refresh_frequency_limits(policy); - up_write(&policy->rwsem); } static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq, @@ -1254,11 +1254,11 @@ static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy) struct kobject *kobj; struct completion *cmp; - down_write(&policy->rwsem); - cpufreq_stats_free_table(policy); - kobj = &policy->kobj; - cmp = &policy->kobj_unregister; - up_write(&policy->rwsem); + scoped_guard(cpufreq_policy_write, policy) { + cpufreq_stats_free_table(policy); + kobj = &policy->kobj; + cmp = &policy->kobj_unregister; + } kobject_put(kobj); /* @@ -1409,7 +1409,7 @@ static int cpufreq_policy_online(struct cpufreq_policy *policy, unsigned int j; int ret; - down_write(&policy->rwsem); + guard(cpufreq_policy_write)(policy); policy->cpu = cpu; policy->governor = NULL; @@ -1586,10 +1586,7 @@ static int cpufreq_policy_online(struct cpufreq_policy *policy, goto out_destroy_policy; } -out_unlock: - up_write(&policy->rwsem); - - return ret; + return 0; out_destroy_policy: for_each_cpu(j, policy->real_cpus) @@ -1606,7 +1603,7 @@ static int cpufreq_policy_online(struct cpufreq_policy *policy, out_clear_policy: cpumask_clear(policy->cpus); - goto out_unlock; + return ret; } static int cpufreq_online(unsigned int cpu) @@ -1754,11 +1751,10 @@ static int cpufreq_offline(unsigned int cpu) return 0; } - down_write(&policy->rwsem); + guard(cpufreq_policy_write)(policy); __cpufreq_offline(cpu, policy); - up_write(&policy->rwsem); return 0; } @@ -1775,33 +1771,29 @@ static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif) if (!policy) return; - down_write(&policy->rwsem); + scoped_guard(cpufreq_policy_write, policy) { + if (cpu_online(cpu)) + __cpufreq_offline(cpu, policy); - if (cpu_online(cpu)) - __cpufreq_offline(cpu, policy); + remove_cpu_dev_symlink(policy, cpu, dev); - remove_cpu_dev_symlink(policy, cpu, dev); + if (!cpumask_empty(policy->real_cpus)) + return; - if (!cpumask_empty(policy->real_cpus)) { - up_write(&policy->rwsem); - return; + /* + * Unregister cpufreq cooling once all the CPUs of the policy + * are removed. + */ + if (cpufreq_thermal_control_enabled(cpufreq_driver)) { + cpufreq_cooling_unregister(policy->cdev); + policy->cdev = NULL; + } + + /* We did light-weight exit earlier, do full tear down now */ + if (cpufreq_driver->offline && cpufreq_driver->exit) + cpufreq_driver->exit(policy); } - /* - * Unregister cpufreq cooling once all the CPUs of the policy are - * removed. - */ - if (cpufreq_thermal_control_enabled(cpufreq_driver)) { - cpufreq_cooling_unregister(policy->cdev); - policy->cdev = NULL; - } - - /* We did light-weight exit earlier, do full tear down now */ - if (cpufreq_driver->offline && cpufreq_driver->exit) - cpufreq_driver->exit(policy); - - up_write(&policy->rwsem); - cpufreq_policy_free(policy); } @@ -1954,15 +1946,16 @@ unsigned int cpufreq_get(unsigned int cpu) struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); unsigned int ret_freq = 0; - if (policy) { - down_read(&policy->rwsem); + if (!policy) + return 0; + + scoped_guard(cpufreq_policy_read, policy) { if (cpufreq_driver->get) ret_freq = __cpufreq_get(policy); - up_read(&policy->rwsem); - - cpufreq_cpu_put(policy); } + cpufreq_cpu_put(policy); + return ret_freq; } EXPORT_SYMBOL(cpufreq_get); @@ -2022,9 +2015,9 @@ void cpufreq_suspend(void) for_each_active_policy(policy) { if (has_target()) { - down_write(&policy->rwsem); - cpufreq_stop_governor(policy); - up_write(&policy->rwsem); + scoped_guard(cpufreq_policy_write, policy) { + cpufreq_stop_governor(policy); + } } if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy)) @@ -2065,9 +2058,9 @@ void cpufreq_resume(void) pr_err("%s: Failed to resume driver: %s\n", __func__, cpufreq_driver->name); } else if (has_target()) { - down_write(&policy->rwsem); - ret = cpufreq_start_governor(policy); - up_write(&policy->rwsem); + scoped_guard(cpufreq_policy_write, policy) { + ret = cpufreq_start_governor(policy); + } if (ret) pr_err("%s: Failed to start governor for CPU%u's policy\n", @@ -2434,15 +2427,9 @@ int cpufreq_driver_target(struct cpufreq_policy *policy, unsigned int target_freq, unsigned int relation) { - int ret; + guard(cpufreq_policy_write)(policy); - down_write(&policy->rwsem); - - ret = __cpufreq_driver_target(policy, target_freq, relation); - - up_write(&policy->rwsem); - - return ret; + return __cpufreq_driver_target(policy, target_freq, relation); } EXPORT_SYMBOL_GPL(cpufreq_driver_target); diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 400fee6427a5..cb972d2aa8df 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -170,6 +170,12 @@ struct cpufreq_policy { struct notifier_block nb_max; }; +DEFINE_GUARD(cpufreq_policy_write, struct cpufreq_policy *, + down_write(&_T->rwsem), up_write(&_T->rwsem)) + +DEFINE_GUARD(cpufreq_policy_read, struct cpufreq_policy *, + down_read(&_T->rwsem), up_read(&_T->rwsem)) + /* * Used for passing new cpufreq policy data to the cpufreq driver's ->verify() * callback for sanitization. That callback is only expected to modify the min From 973207ae3d7c3c92df4a382df5d7bd695deaa904 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 28 Mar 2025 21:43:46 +0100 Subject: [PATCH 04/32] cpufreq: intel_pstate: Rearrange max frequency updates handling code Rename __intel_pstate_update_max_freq() to intel_pstate_update_max_freq() and move the cpufreq policy reference counting and locking into it (and implement the locking with the recently introduced cpufreq policy "write" locking guard). No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Mario Limonciello Acked-by: Srinivas Pandruvada Link: https://patch.msgid.link/2315023.iZASKD2KPV@rjwysocki.net --- drivers/cpufreq/intel_pstate.c | 52 +++++++++++----------------------- 1 file changed, 17 insertions(+), 35 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 4aad79d26c64..108e4c6a371e 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1353,9 +1353,16 @@ static void intel_pstate_update_policies(void) cpufreq_update_policy(cpu); } -static void __intel_pstate_update_max_freq(struct cpudata *cpudata, - struct cpufreq_policy *policy) +static bool intel_pstate_update_max_freq(struct cpudata *cpudata) { + struct cpufreq_policy *policy __free(put_cpufreq_policy); + + policy = cpufreq_cpu_get(cpudata->cpu); + if (!policy) + return false; + + guard(cpufreq_policy_write)(policy); + if (hwp_active) intel_pstate_get_hwp_cap(cpudata); @@ -1363,44 +1370,24 @@ static void __intel_pstate_update_max_freq(struct cpudata *cpudata, cpudata->pstate.max_freq : cpudata->pstate.turbo_freq; refresh_frequency_limits(policy); + + return true; } static void intel_pstate_update_limits(unsigned int cpu) { - struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu); - struct cpudata *cpudata; + struct cpudata *cpudata = all_cpu_data[cpu]; - if (!policy) - return; - - cpudata = all_cpu_data[cpu]; - - __intel_pstate_update_max_freq(cpudata, policy); - - /* Prevent the driver from being unregistered now. */ - mutex_lock(&intel_pstate_driver_lock); - - cpufreq_cpu_release(policy); - - hybrid_update_capacity(cpudata); - - mutex_unlock(&intel_pstate_driver_lock); + if (intel_pstate_update_max_freq(cpudata)) + hybrid_update_capacity(cpudata); } static void intel_pstate_update_limits_for_all(void) { int cpu; - for_each_possible_cpu(cpu) { - struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu); - - if (!policy) - continue; - - __intel_pstate_update_max_freq(all_cpu_data[cpu], policy); - - cpufreq_cpu_release(policy); - } + for_each_possible_cpu(cpu) + intel_pstate_update_max_freq(all_cpu_data[cpu]); mutex_lock(&hybrid_capacity_lock); @@ -1840,13 +1827,8 @@ static void intel_pstate_notify_work(struct work_struct *work) { struct cpudata *cpudata = container_of(to_delayed_work(work), struct cpudata, hwp_notify_work); - struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpudata->cpu); - - if (policy) { - __intel_pstate_update_max_freq(cpudata, policy); - - cpufreq_cpu_release(policy); + if (intel_pstate_update_max_freq(cpudata)) { /* * The driver will not be unregistered while this function is * running, so update the capacity without acquiring the driver From 9a74bfdfd07f99b09fa4aaccacd17248b8629a07 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 28 Mar 2025 21:44:50 +0100 Subject: [PATCH 05/32] cpufreq: Use locking guard and __free() in cpufreq_update_policy() Instead of using cpufreq_cpu_acquire() and cpufreq_cpu_release() in cpufreq_update_policy(), which is the last user of these functions, make it use __free() for policy reference counting cleanup and the "write" locking guard for policy locking. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar Reviewed-by: Mario Limonciello Acked-by: Sudeep Holla Tested-by: Sudeep Holla Link: https://patch.msgid.link/22654186.EfDdHjke4D@rjwysocki.net --- drivers/cpufreq/cpufreq.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index a3abd9c0440d..40244ff620b6 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2780,23 +2780,23 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, */ void cpufreq_update_policy(unsigned int cpu) { - struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu); + struct cpufreq_policy *policy __free(put_cpufreq_policy); + policy = cpufreq_cpu_get(cpu); if (!policy) return; + guard(cpufreq_policy_write)(policy); + /* * BIOS might change freq behind our back * -> ask driver for current freq and notify governors about a change */ if (cpufreq_driver->get && has_target() && (cpufreq_suspended || WARN_ON(!cpufreq_verify_current_freq(policy, false)))) - goto unlock; + return; refresh_frequency_limits(policy); - -unlock: - cpufreq_cpu_release(policy); } EXPORT_SYMBOL(cpufreq_update_policy); From c7282dce257480f0f22ed3db69cfb400a18709f4 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 28 Mar 2025 21:45:38 +0100 Subject: [PATCH 06/32] cpufreq: Drop cpufreq_cpu_acquire() and cpufreq_cpu_release() Since cpufreq_cpu_acquire() and cpufreq_cpu_release() have no more users in the tree, remove them. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar Reviewed-by: Mario Limonciello Acked-by: Sudeep Holla Tested-by: Sudeep Holla Link: https://patch.msgid.link/3880470.kQq0lBPeGt@rjwysocki.net --- drivers/cpufreq/cpufreq.c | 45 --------------------------------------- include/linux/cpufreq.h | 2 -- 2 files changed, 47 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 40244ff620b6..2ed873777fb5 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -255,51 +255,6 @@ void cpufreq_cpu_put(struct cpufreq_policy *policy) } EXPORT_SYMBOL_GPL(cpufreq_cpu_put); -/** - * cpufreq_cpu_release - Unlock a policy and decrement its usage counter. - * @policy: cpufreq policy returned by cpufreq_cpu_acquire(). - */ -void cpufreq_cpu_release(struct cpufreq_policy *policy) -{ - if (WARN_ON(!policy)) - return; - - lockdep_assert_held(&policy->rwsem); - - up_write(&policy->rwsem); - - cpufreq_cpu_put(policy); -} - -/** - * cpufreq_cpu_acquire - Find policy for a CPU, mark it as busy and lock it. - * @cpu: CPU to find the policy for. - * - * Call cpufreq_cpu_get() to get a reference on the cpufreq policy for @cpu and - * if the policy returned by it is not NULL, acquire its rwsem for writing. - * Return the policy if it is active or release it and return NULL otherwise. - * - * The policy returned by this function has to be released with the help of - * cpufreq_cpu_release() in order to release its rwsem and balance its usage - * counter properly. - */ -struct cpufreq_policy *cpufreq_cpu_acquire(unsigned int cpu) -{ - struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); - - if (!policy) - return NULL; - - down_write(&policy->rwsem); - - if (policy_is_inactive(policy)) { - cpufreq_cpu_release(policy); - return NULL; - } - - return policy; -} - /********************************************************************* * EXTERNALLY AFFECTING FREQUENCY CHANGES * *********************************************************************/ diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index cb972d2aa8df..a33a094ef755 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -241,8 +241,6 @@ void disable_cpufreq(void); u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy); -struct cpufreq_policy *cpufreq_cpu_acquire(unsigned int cpu); -void cpufreq_cpu_release(struct cpufreq_policy *policy); int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu); void refresh_frequency_limits(struct cpufreq_policy *policy); void cpufreq_update_policy(unsigned int cpu); From ece898da386214cf9bf693fe21694b556b785428 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 28 Mar 2025 21:46:22 +0100 Subject: [PATCH 07/32] cpufreq: Use __free() for policy reference counting cleanup Use __free() for policy reference counting cleanup where applicable in the cpufreq core. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar Reviewed-by: Mario Limonciello Acked-by: Sudeep Holla Tested-by: Sudeep Holla Link: https://patch.msgid.link/9437968.CDJkKcVGEf@rjwysocki.net --- drivers/cpufreq/cpufreq.c | 57 +++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 2ed873777fb5..29130aa1b09a 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1818,27 +1818,26 @@ static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, b */ unsigned int cpufreq_quick_get(unsigned int cpu) { - struct cpufreq_policy *policy; - unsigned int ret_freq = 0; + struct cpufreq_policy *policy __free(put_cpufreq_policy) = NULL; unsigned long flags; read_lock_irqsave(&cpufreq_driver_lock, flags); if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) { - ret_freq = cpufreq_driver->get(cpu); + unsigned int ret_freq = cpufreq_driver->get(cpu); + read_unlock_irqrestore(&cpufreq_driver_lock, flags); + return ret_freq; } read_unlock_irqrestore(&cpufreq_driver_lock, flags); policy = cpufreq_cpu_get(cpu); - if (policy) { - ret_freq = policy->cur; - cpufreq_cpu_put(policy); - } + if (policy) + return policy->cur; - return ret_freq; + return 0; } EXPORT_SYMBOL(cpufreq_quick_get); @@ -1850,15 +1849,13 @@ EXPORT_SYMBOL(cpufreq_quick_get); */ unsigned int cpufreq_quick_get_max(unsigned int cpu) { - struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); - unsigned int ret_freq = 0; + struct cpufreq_policy *policy __free(put_cpufreq_policy); - if (policy) { - ret_freq = policy->max; - cpufreq_cpu_put(policy); - } + policy = cpufreq_cpu_get(cpu); + if (policy) + return policy->max; - return ret_freq; + return 0; } EXPORT_SYMBOL(cpufreq_quick_get_max); @@ -1870,15 +1867,13 @@ EXPORT_SYMBOL(cpufreq_quick_get_max); */ __weak unsigned int cpufreq_get_hw_max_freq(unsigned int cpu) { - struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); - unsigned int ret_freq = 0; + struct cpufreq_policy *policy __free(put_cpufreq_policy); - if (policy) { - ret_freq = policy->cpuinfo.max_freq; - cpufreq_cpu_put(policy); - } + policy = cpufreq_cpu_get(cpu); + if (policy) + return policy->cpuinfo.max_freq; - return ret_freq; + return 0; } EXPORT_SYMBOL(cpufreq_get_hw_max_freq); @@ -1898,20 +1893,18 @@ static unsigned int __cpufreq_get(struct cpufreq_policy *policy) */ unsigned int cpufreq_get(unsigned int cpu) { - struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); - unsigned int ret_freq = 0; + struct cpufreq_policy *policy __free(put_cpufreq_policy); + policy = cpufreq_cpu_get(cpu); if (!policy) return 0; - scoped_guard(cpufreq_policy_read, policy) { - if (cpufreq_driver->get) - ret_freq = __cpufreq_get(policy); - } + guard(cpufreq_policy_read)(policy); - cpufreq_cpu_put(policy); + if (cpufreq_driver->get) + return __cpufreq_get(policy); - return ret_freq; + return 0; } EXPORT_SYMBOL(cpufreq_get); @@ -2566,7 +2559,8 @@ EXPORT_SYMBOL_GPL(cpufreq_unregister_governor); */ int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu) { - struct cpufreq_policy *cpu_policy; + struct cpufreq_policy *cpu_policy __free(put_cpufreq_policy); + if (!policy) return -EINVAL; @@ -2576,7 +2570,6 @@ int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu) memcpy(policy, cpu_policy, sizeof(*policy)); - cpufreq_cpu_put(cpu_policy); return 0; } EXPORT_SYMBOL(cpufreq_get_policy); From 684e1855211145b71f8d9aaa49d2cc67067cd42a Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 28 Mar 2025 21:47:31 +0100 Subject: [PATCH 08/32] cpufreq: Introduce cpufreq_policy_refresh() Since cpufreq_update_limits() obtains a cpufreq policy pointer for the given CPU and reference counts the object pointed to by it, calling cpufreq_update_policy() from cpufreq_update_limits() is somewhat wasteful because that function calls cpufreq_cpu_get() on the same CPU again. To avoid that unnecessary overhead, move the part of the code running under the policy rwsem from cpufreq_update_policy() to a new function called cpufreq_policy_refresh() and invoke that new function from both cpufreq_update_policy() and cpufreq_update_limits(). Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar Reviewed-by: Mario Limonciello Acked-by: Sudeep Holla Tested-by: Sudeep Holla Link: https://patch.msgid.link/6047110.MhkbZ0Pkbq@rjwysocki.net --- drivers/cpufreq/cpufreq.c | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 29130aa1b09a..c885e0ec174f 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2717,6 +2717,21 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, return ret; } +static void cpufreq_policy_refresh(struct cpufreq_policy *policy) +{ + guard(cpufreq_policy_write)(policy); + + /* + * BIOS might change freq behind our back + * -> ask driver for current freq and notify governors about a change + */ + if (cpufreq_driver->get && has_target() && + (cpufreq_suspended || WARN_ON(!cpufreq_verify_current_freq(policy, false)))) + return; + + refresh_frequency_limits(policy); +} + /** * cpufreq_update_policy - Re-evaluate an existing cpufreq policy. * @cpu: CPU to re-evaluate the policy for. @@ -2734,17 +2749,7 @@ void cpufreq_update_policy(unsigned int cpu) if (!policy) return; - guard(cpufreq_policy_write)(policy); - - /* - * BIOS might change freq behind our back - * -> ask driver for current freq and notify governors about a change - */ - if (cpufreq_driver->get && has_target() && - (cpufreq_suspended || WARN_ON(!cpufreq_verify_current_freq(policy, false)))) - return; - - refresh_frequency_limits(policy); + cpufreq_policy_refresh(policy); } EXPORT_SYMBOL(cpufreq_update_policy); @@ -2753,7 +2758,7 @@ EXPORT_SYMBOL(cpufreq_update_policy); * @cpu: CPU to update the policy limits for. * * Invoke the driver's ->update_limits callback if present or call - * cpufreq_update_policy() for @cpu. + * cpufreq_policy_refresh() for @cpu. */ void cpufreq_update_limits(unsigned int cpu) { @@ -2766,7 +2771,7 @@ void cpufreq_update_limits(unsigned int cpu) if (cpufreq_driver->update_limits) cpufreq_driver->update_limits(cpu); else - cpufreq_update_policy(cpu); + cpufreq_policy_refresh(policy); } EXPORT_SYMBOL_GPL(cpufreq_update_limits); From eaff6b62d3439ca6ee00dba4f77673a8c37dac20 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 28 Mar 2025 21:48:54 +0100 Subject: [PATCH 09/32] cpufreq: Pass policy pointer to ->update_limits() Since cpufreq_update_limits() obtains a cpufreq policy pointer for the given CPU and reference counts the corresponding policy object, it may as well pass the policy pointer to the cpufreq driver's ->update_limits() callback which allows that callback to avoid invoking cpufreq_cpu_get() for the same CPU. Accordingly, redefine ->update_limits() to take a policy pointer instead of a CPU number and update both drivers implementing it, intel_pstate and amd-pstate, as needed. Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar Reviewed-by: Mario Limonciello Acked-by: Srinivas Pandruvada Acked-by: Sudeep Holla Tested-by: Sudeep Holla Link: https://patch.msgid.link/8560367.NyiUUSuA9g@rjwysocki.net --- drivers/cpufreq/amd-pstate.c | 7 ++----- drivers/cpufreq/cpufreq.c | 2 +- drivers/cpufreq/intel_pstate.c | 29 ++++++++++++++++++----------- include/linux/cpufreq.h | 2 +- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 6789eed1bb5b..b9d59c7425f5 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -821,19 +821,16 @@ static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) schedule_work(&sched_prefcore_work); } -static void amd_pstate_update_limits(unsigned int cpu) +static void amd_pstate_update_limits(struct cpufreq_policy *policy) { - struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); struct amd_cpudata *cpudata; u32 prev_high = 0, cur_high = 0; bool highest_perf_changed = false; + unsigned int cpu = policy->cpu; if (!amd_pstate_prefcore) return; - if (!policy) - return; - if (amd_get_highest_perf(cpu, &cur_high)) return; diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index c885e0ec174f..2b91ba503b32 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2769,7 +2769,7 @@ void cpufreq_update_limits(unsigned int cpu) return; if (cpufreq_driver->update_limits) - cpufreq_driver->update_limits(cpu); + cpufreq_driver->update_limits(policy); else cpufreq_policy_refresh(policy); } diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 108e4c6a371e..f5ca04b98b92 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1353,14 +1353,9 @@ static void intel_pstate_update_policies(void) cpufreq_update_policy(cpu); } -static bool intel_pstate_update_max_freq(struct cpudata *cpudata) +static void __intel_pstate_update_max_freq(struct cpufreq_policy *policy, + struct cpudata *cpudata) { - struct cpufreq_policy *policy __free(put_cpufreq_policy); - - policy = cpufreq_cpu_get(cpudata->cpu); - if (!policy) - return false; - guard(cpufreq_policy_write)(policy); if (hwp_active) @@ -1370,16 +1365,28 @@ static bool intel_pstate_update_max_freq(struct cpudata *cpudata) cpudata->pstate.max_freq : cpudata->pstate.turbo_freq; refresh_frequency_limits(policy); +} + +static bool intel_pstate_update_max_freq(struct cpudata *cpudata) +{ + struct cpufreq_policy *policy __free(put_cpufreq_policy); + + policy = cpufreq_cpu_get(cpudata->cpu); + if (!policy) + return false; + + __intel_pstate_update_max_freq(policy, cpudata); return true; } -static void intel_pstate_update_limits(unsigned int cpu) +static void intel_pstate_update_limits(struct cpufreq_policy *policy) { - struct cpudata *cpudata = all_cpu_data[cpu]; + struct cpudata *cpudata = all_cpu_data[policy->cpu]; - if (intel_pstate_update_max_freq(cpudata)) - hybrid_update_capacity(cpudata); + __intel_pstate_update_max_freq(policy, cpudata); + + hybrid_update_capacity(cpudata); } static void intel_pstate_update_limits_for_all(void) diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index a33a094ef755..f3cf2adea18f 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -399,7 +399,7 @@ struct cpufreq_driver { unsigned int (*get)(unsigned int cpu); /* Called to update policy limits on firmware notifications. */ - void (*update_limits)(unsigned int cpu); + void (*update_limits)(struct cpufreq_policy *policy); /* optional */ int (*bios_limit)(int cpu, unsigned int *limit); From cb271c2edfd0ab7204d5ef3c9d5ae9a0710f5bf2 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 13 Apr 2025 19:36:56 +0200 Subject: [PATCH 10/32] rust: device: implement impl_device_context_deref! The Deref hierarchy for device context generics is the same for every (bus specific) device. Implement those with a generic macro to avoid duplicated boiler plate code and ensure the correct Deref hierarchy for every device implementation. Co-developed-by: Benno Lossin Signed-off-by: Benno Lossin Reviewed-by: Christian Schrefl Link: https://lore.kernel.org/r/20250413173758.12068-2-dakr@kernel.org [ Add missing `::` prefix in macros. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/device.rs | 44 +++++++++++++++++++++++++++++++++++++++++ rust/kernel/pci.rs | 16 +++------------ rust/kernel/platform.rs | 17 +++------------- 3 files changed, 50 insertions(+), 27 deletions(-) diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 21b343a1dc4d..9520e054996d 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -235,6 +235,50 @@ impl Sealed for super::Normal {} impl DeviceContext for Core {} impl DeviceContext for Normal {} +/// # Safety +/// +/// The type given as `$device` must be a transparent wrapper of a type that doesn't depend on the +/// generic argument of `$device`. +#[doc(hidden)] +#[macro_export] +macro_rules! __impl_device_context_deref { + (unsafe { $device:ident, $src:ty => $dst:ty }) => { + impl ::core::ops::Deref for $device<$src> { + type Target = $device<$dst>; + + fn deref(&self) -> &Self::Target { + let ptr: *const Self = self; + + // CAST: `$device<$src>` and `$device<$dst>` transparently wrap the same type by the + // safety requirement of the macro. + let ptr = ptr.cast::(); + + // SAFETY: `ptr` was derived from `&self`. + unsafe { &*ptr } + } + } + }; +} + +/// Implement [`core::ops::Deref`] traits for allowed [`DeviceContext`] conversions of a (bus +/// specific) device. +/// +/// # Safety +/// +/// The type given as `$device` must be a transparent wrapper of a type that doesn't depend on the +/// generic argument of `$device`. +#[macro_export] +macro_rules! impl_device_context_deref { + (unsafe { $device:ident }) => { + // SAFETY: This macro has the exact same safety requirement as + // `__impl_device_context_deref!`. + ::kernel::__impl_device_context_deref!(unsafe { + $device, + $crate::device::Core => $crate::device::Normal + }); + }; +} + #[doc(hidden)] #[macro_export] macro_rules! dev_printk { diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index c97d6d470b28..8474608e7a90 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -422,19 +422,9 @@ pub fn set_master(&self) { } } -impl Deref for Device { - type Target = Device; - - fn deref(&self) -> &Self::Target { - let ptr: *const Self = self; - - // CAST: `Device` is a transparent wrapper of `Opaque`. - let ptr = ptr.cast::(); - - // SAFETY: `ptr` was derived from `&self`. - unsafe { &*ptr } - } -} +// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic +// argument. +kernel::impl_device_context_deref!(unsafe { Device }); impl From<&Device> for ARef { fn from(dev: &Device) -> Self { diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 4917cb34e2fe..22590bdff7bb 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -16,7 +16,6 @@ use core::{ marker::PhantomData, - ops::Deref, ptr::{addr_of_mut, NonNull}, }; @@ -190,19 +189,9 @@ fn as_raw(&self) -> *mut bindings::platform_device { } } -impl Deref for Device { - type Target = Device; - - fn deref(&self) -> &Self::Target { - let ptr: *const Self = self; - - // CAST: `Device` is a transparent wrapper of `Opaque`. - let ptr = ptr.cast::(); - - // SAFETY: `ptr` was derived from `&self`. - unsafe { &*ptr } - } -} +// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic +// argument. +kernel::impl_device_context_deref!(unsafe { Device }); impl From<&Device> for ARef { fn from(dev: &Device) -> Self { From fbb92b6a534081cabd75861ac9c7a8d29d8effda Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 13 Apr 2025 19:36:57 +0200 Subject: [PATCH 11/32] rust: device: implement impl_device_context_into_aref! Implement a macro to implement all From conversions of a certain device to ARef. This avoids unnecessary boiler plate code for every device implementation. Reviewed-by: Benno Lossin Link: https://lore.kernel.org/r/20250413173758.12068-3-dakr@kernel.org [ Add missing `::` prefix in macros. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/device.rs | 21 +++++++++++++++++++++ rust/kernel/pci.rs | 7 +------ rust/kernel/platform.rs | 9 ++------- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 9520e054996d..7b96c99879a6 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -279,6 +279,27 @@ macro_rules! impl_device_context_deref { }; } +#[doc(hidden)] +#[macro_export] +macro_rules! __impl_device_context_into_aref { + ($src:ty, $device:tt) => { + impl ::core::convert::From<&$device<$src>> for $crate::types::ARef<$device> { + fn from(dev: &$device<$src>) -> Self { + (&**dev).into() + } + } + }; +} + +/// Implement [`core::convert::From`], such that all `&Device` can be converted to an +/// `ARef`. +#[macro_export] +macro_rules! impl_device_context_into_aref { + ($device:tt) => { + ::kernel::__impl_device_context_into_aref!($crate::device::Core, $device); + }; +} + #[doc(hidden)] #[macro_export] macro_rules! dev_printk { diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index 8474608e7a90..7c6ec05383e0 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -425,12 +425,7 @@ pub fn set_master(&self) { // SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic // argument. kernel::impl_device_context_deref!(unsafe { Device }); - -impl From<&Device> for ARef { - fn from(dev: &Device) -> Self { - (&**dev).into() - } -} +kernel::impl_device_context_into_aref!(Device); // SAFETY: Instances of `Device` are always reference-counted. unsafe impl crate::types::AlwaysRefCounted for Device { diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 22590bdff7bb..9476b717c425 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -10,7 +10,7 @@ of, prelude::*, str::CStr, - types::{ARef, ForeignOwnable, Opaque}, + types::{ForeignOwnable, Opaque}, ThisModule, }; @@ -192,12 +192,7 @@ fn as_raw(&self) -> *mut bindings::platform_device { // SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic // argument. kernel::impl_device_context_deref!(unsafe { Device }); - -impl From<&Device> for ARef { - fn from(dev: &Device) -> Self { - (&**dev).into() - } -} +kernel::impl_device_context_into_aref!(Device); // SAFETY: Instances of `Device` are always reference-counted. unsafe impl crate::types::AlwaysRefCounted for Device { From d32e4c24a7fe01195ba427c67fc15864279a3c0d Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 13 Apr 2025 19:36:58 +0200 Subject: [PATCH 12/32] rust: device: implement device context for Device Analogous to bus specific device, implement the DeviceContext generic for generic devices. This is used for APIs that work with generic devices (such as Devres) to evaluate the device's context. Reviewed-by: Benno Lossin Link: https://lore.kernel.org/r/20250413173758.12068-4-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/device.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 7b96c99879a6..53a13dd4d2d4 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -9,7 +9,7 @@ str::CStr, types::{ARef, Opaque}, }; -use core::{fmt, ptr}; +use core::{fmt, marker::PhantomData, ptr}; #[cfg(CONFIG_PRINTK)] use crate::c_str; @@ -42,7 +42,7 @@ /// `bindings::device::release` is valid to be called from any thread, hence `ARef` can be /// dropped from any thread. #[repr(transparent)] -pub struct Device(Opaque); +pub struct Device(Opaque, PhantomData); impl Device { /// Creates a new reference-counted abstraction instance of an existing `struct device` pointer. @@ -59,7 +59,9 @@ pub unsafe fn get_device(ptr: *mut bindings::device) -> ARef { // SAFETY: By the safety requirements ptr is valid unsafe { Self::as_ref(ptr) }.into() } +} +impl Device { /// Obtain the raw `struct device *`. pub(crate) fn as_raw(&self) -> *mut bindings::device { self.0.get() @@ -189,6 +191,11 @@ pub fn property_present(&self, name: &CStr) -> bool { } } +// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic +// argument. +kernel::impl_device_context_deref!(unsafe { Device }); +kernel::impl_device_context_into_aref!(Device); + // SAFETY: Instances of `Device` are always reference-counted. unsafe impl crate::types::AlwaysRefCounted for Device { fn inc_ref(&self) { From da6c47c6cb45ffb392e016631c08098ad2a6d418 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 13 Apr 2025 19:36:59 +0200 Subject: [PATCH 13/32] rust: platform: preserve device context in AsRef Since device::Device has a generic over its context, preserve this device context in AsRef. For instance, when calling platform::Device the new AsRef implementation returns device::Device. Reviewed-by: Benno Lossin Link: https://lore.kernel.org/r/20250413173758.12068-5-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/platform.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 9476b717c425..b1c48cd95cd6 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -183,7 +183,7 @@ pub struct Device( PhantomData, ); -impl Device { +impl Device { fn as_raw(&self) -> *mut bindings::platform_device { self.0.get() } @@ -207,8 +207,8 @@ unsafe fn dec_ref(obj: NonNull) { } } -impl AsRef for Device { - fn as_ref(&self) -> &device::Device { +impl AsRef> for Device { + fn as_ref(&self) -> &device::Device { // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid // `struct platform_device`. let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) }; From 3edaefbf2b1beb9ae1cb2a842f455157b951e9f1 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 13 Apr 2025 19:37:00 +0200 Subject: [PATCH 14/32] rust: pci: preserve device context in AsRef Since device::Device has a generic over its context, preserve this device context in AsRef. For instance, when calling pci::Device the new AsRef implementation returns device::Device. Reviewed-by: Benno Lossin Link: https://lore.kernel.org/r/20250413173758.12068-6-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/pci.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index 7c6ec05383e0..1234b0c4a403 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -360,11 +360,13 @@ fn deref(&self) -> &Self::Target { } } -impl Device { +impl Device { fn as_raw(&self) -> *mut bindings::pci_dev { self.0.get() } +} +impl Device { /// Returns the PCI vendor ID. pub fn vendor_id(&self) -> u16 { // SAFETY: `self.as_raw` is a valid pointer to a `struct pci_dev`. @@ -440,8 +442,8 @@ unsafe fn dec_ref(obj: NonNull) { } } -impl AsRef for Device { - fn as_ref(&self) -> &device::Device { +impl AsRef> for Device { + fn as_ref(&self) -> &device::Device { // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid // `struct pci_dev`. let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) }; From f933b7489ffca25c9a33b65441679ee3d2943024 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 13 Apr 2025 19:37:01 +0200 Subject: [PATCH 15/32] rust: device: implement Bound device context The Bound device context indicates that a device is bound to a driver. It must be used for APIs that require the device to be bound, such as Devres or dma::CoherentAllocation. Implement Bound and add the corresponding Deref hierarchy, as well as the corresponding ARef conversion for this device context. Reviewed-by: Benno Lossin Link: https://lore.kernel.org/r/20250413173758.12068-7-dakr@kernel.org [ Add missing `::` prefix in macros. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/device.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 53a13dd4d2d4..0353c5552769 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -232,13 +232,19 @@ pub trait DeviceContext: private::Sealed {} /// any of the bus callbacks, such as `probe()`. pub struct Core; +/// The [`Bound`] context is the context of a bus specific device reference when it is guaranteed to +/// be bound for the duration of its lifetime. +pub struct Bound; + mod private { pub trait Sealed {} + impl Sealed for super::Bound {} impl Sealed for super::Core {} impl Sealed for super::Normal {} } +impl DeviceContext for Bound {} impl DeviceContext for Core {} impl DeviceContext for Normal {} @@ -281,7 +287,14 @@ macro_rules! impl_device_context_deref { // `__impl_device_context_deref!`. ::kernel::__impl_device_context_deref!(unsafe { $device, - $crate::device::Core => $crate::device::Normal + $crate::device::Core => $crate::device::Bound + }); + + // SAFETY: This macro has the exact same safety requirement as + // `__impl_device_context_deref!`. + ::kernel::__impl_device_context_deref!(unsafe { + $device, + $crate::device::Bound => $crate::device::Normal }); }; } @@ -304,6 +317,7 @@ fn from(dev: &$device<$src>) -> Self { macro_rules! impl_device_context_into_aref { ($device:tt) => { ::kernel::__impl_device_context_into_aref!($crate::device::Core, $device); + ::kernel::__impl_device_context_into_aref!($crate::device::Bound, $device); }; } From f2a399d7b67c4a6fc0f8e59d1a9eb484efc71b5c Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 13 Apr 2025 19:37:02 +0200 Subject: [PATCH 16/32] rust: pci: move iomap_region() to impl Device Require the Bound device context to be able to call iomap_region() and iomap_region_sized(). Creating I/O mapping requires the device to be bound. Reviewed-by: Benno Lossin Link: https://lore.kernel.org/r/20250413173758.12068-8-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/pci.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index 1234b0c4a403..3664d35b8e79 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -390,7 +390,9 @@ pub fn resource_len(&self, bar: u32) -> Result { // - by its type invariant `self.as_raw` is always a valid pointer to a `struct pci_dev`. Ok(unsafe { bindings::pci_resource_len(self.as_raw(), bar.try_into()?) }) } +} +impl Device { /// Mapps an entire PCI-BAR after performing a region-request on it. I/O operation bound checks /// can be performed on compile time for offsets (plus the requested type size) < SIZE. pub fn iomap_region_sized( From f720efda2db5e609b32100c25d9cf383f082d945 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 13 Apr 2025 19:37:03 +0200 Subject: [PATCH 17/32] rust: devres: require a bound device Require the Bound device context to be able to a new Devres container. This ensures that we can't register devres callbacks for unbound devices. Link: https://lore.kernel.org/r/20250413173758.12068-9-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/devres.rs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index ddb1ce4a78d9..1e58f5d22044 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -8,7 +8,7 @@ use crate::{ alloc::Flags, bindings, - device::Device, + device::{Bound, Device}, error::{Error, Result}, ffi::c_void, prelude::*, @@ -45,7 +45,7 @@ struct DevresInner { /// # Example /// /// ```no_run -/// # use kernel::{bindings, c_str, device::Device, devres::Devres, io::{Io, IoRaw}}; +/// # use kernel::{bindings, c_str, device::{Bound, Device}, devres::Devres, io::{Io, IoRaw}}; /// # use core::ops::Deref; /// /// // See also [`pci::Bar`] for a real example. @@ -83,13 +83,10 @@ struct DevresInner { /// unsafe { Io::from_raw(&self.0) } /// } /// } -/// # fn no_run() -> Result<(), Error> { -/// # // SAFETY: Invalid usage; just for the example to get an `ARef` instance. -/// # let dev = unsafe { Device::get_device(core::ptr::null_mut()) }; -/// +/// # fn no_run(dev: &Device) -> Result<(), Error> { /// // SAFETY: Invalid usage for example purposes. /// let iomem = unsafe { IoMem::<{ core::mem::size_of::() }>::new(0xBAAAAAAD)? }; -/// let devres = Devres::new(&dev, iomem, GFP_KERNEL)?; +/// let devres = Devres::new(dev, iomem, GFP_KERNEL)?; /// /// let res = devres.try_access().ok_or(ENXIO)?; /// res.write8(0x42, 0x0); @@ -99,7 +96,7 @@ struct DevresInner { pub struct Devres(Arc>); impl DevresInner { - fn new(dev: &Device, data: T, flags: Flags) -> Result>> { + fn new(dev: &Device, data: T, flags: Flags) -> Result>> { let inner = Arc::pin_init( pin_init!( DevresInner { dev: dev.into(), @@ -171,7 +168,7 @@ fn remove_action(this: &Arc) { impl Devres { /// Creates a new [`Devres`] instance of the given `data`. The `data` encapsulated within the /// returned `Devres` instance' `data` will be revoked once the device is detached. - pub fn new(dev: &Device, data: T, flags: Flags) -> Result { + pub fn new(dev: &Device, data: T, flags: Flags) -> Result { let inner = DevresInner::new(dev, data, flags)?; Ok(Devres(inner)) @@ -179,7 +176,7 @@ pub fn new(dev: &Device, data: T, flags: Flags) -> Result { /// Same as [`Devres::new`], but does not return a `Devres` instance. Instead the given `data` /// is owned by devres and will be revoked / dropped, once the device is detached. - pub fn new_foreign_owned(dev: &Device, data: T, flags: Flags) -> Result { + pub fn new_foreign_owned(dev: &Device, data: T, flags: Flags) -> Result { let _ = DevresInner::new(dev, data, flags)?; Ok(()) From a7e735169db72c6c991a5f8397fb094a80ea3391 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Fri, 11 Apr 2025 13:53:37 +0530 Subject: [PATCH 18/32] rust: cpumask: Add few more helpers Add few more cpumask helpers that are required by the Rust abstraction. Signed-off-by: Viresh Kumar Acked-by: Yury Norov [NVIDIA] --- rust/helpers/cpumask.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/rust/helpers/cpumask.c b/rust/helpers/cpumask.c index 2d380a86c34a..eb10598a0242 100644 --- a/rust/helpers/cpumask.c +++ b/rust/helpers/cpumask.c @@ -7,16 +7,41 @@ void rust_helper_cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) cpumask_set_cpu(cpu, dstp); } +void rust_helper___cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) +{ + __cpumask_set_cpu(cpu, dstp); +} + void rust_helper_cpumask_clear_cpu(int cpu, struct cpumask *dstp) { cpumask_clear_cpu(cpu, dstp); } +void rust_helper___cpumask_clear_cpu(int cpu, struct cpumask *dstp) +{ + __cpumask_clear_cpu(cpu, dstp); +} + +bool rust_helper_cpumask_test_cpu(int cpu, struct cpumask *srcp) +{ + return cpumask_test_cpu(cpu, srcp); +} + void rust_helper_cpumask_setall(struct cpumask *dstp) { cpumask_setall(dstp); } +bool rust_helper_cpumask_empty(struct cpumask *srcp) +{ + return cpumask_empty(srcp); +} + +bool rust_helper_cpumask_full(struct cpumask *srcp) +{ + return cpumask_full(srcp); +} + unsigned int rust_helper_cpumask_weight(struct cpumask *srcp) { return cpumask_weight(srcp); From 8961b8cb3099abd06486394aaf12dd535baaf070 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 6 Jun 2024 15:23:39 +0530 Subject: [PATCH 19/32] rust: cpumask: Add initial abstractions Add initial Rust abstractions for struct cpumask, covering a subset of its APIs. Additional APIs can be added as needed. These abstractions will be used in upcoming Rust support for cpufreq and OPP frameworks. Signed-off-by: Viresh Kumar Reviewed-by: Yury Norov [NVIDIA] --- rust/kernel/cpumask.rs | 330 +++++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 1 + 2 files changed, 331 insertions(+) create mode 100644 rust/kernel/cpumask.rs diff --git a/rust/kernel/cpumask.rs b/rust/kernel/cpumask.rs new file mode 100644 index 000000000000..c90bfac9346a --- /dev/null +++ b/rust/kernel/cpumask.rs @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! CPU Mask abstractions. +//! +//! C header: [`include/linux/cpumask.h`](srctree/include/linux/cpumask.h) + +use crate::{ + alloc::{AllocError, Flags}, + prelude::*, + types::Opaque, +}; + +#[cfg(CONFIG_CPUMASK_OFFSTACK)] +use core::ptr::{self, NonNull}; + +#[cfg(not(CONFIG_CPUMASK_OFFSTACK))] +use core::mem::MaybeUninit; + +use core::ops::{Deref, DerefMut}; + +/// A CPU Mask. +/// +/// Rust abstraction for the C `struct cpumask`. +/// +/// # Invariants +/// +/// A [`Cpumask`] instance always corresponds to a valid C `struct cpumask`. +/// +/// The callers must ensure that the `struct cpumask` is valid for access and +/// remains valid for the lifetime of the returned reference. +/// +/// ## Examples +/// +/// The following example demonstrates how to update a [`Cpumask`]. +/// +/// ``` +/// use kernel::bindings; +/// use kernel::cpumask::Cpumask; +/// +/// fn set_clear_cpu(ptr: *mut bindings::cpumask, set_cpu: u32, clear_cpu: i32) { +/// // SAFETY: The `ptr` is valid for writing and remains valid for the lifetime of the +/// // returned reference. +/// let mask = unsafe { Cpumask::as_mut_ref(ptr) }; +/// +/// mask.set(set_cpu); +/// mask.clear(clear_cpu); +/// } +/// ``` +#[repr(transparent)] +pub struct Cpumask(Opaque); + +impl Cpumask { + /// Creates a mutable reference to an existing `struct cpumask` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime + /// of the returned reference. + pub unsafe fn as_mut_ref<'a>(ptr: *mut bindings::cpumask) -> &'a mut Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the + // lifetime of the returned reference. + unsafe { &mut *ptr.cast() } + } + + /// Creates a reference to an existing `struct cpumask` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime + /// of the returned reference. + pub unsafe fn as_ref<'a>(ptr: *const bindings::cpumask) -> &'a Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the + // lifetime of the returned reference. + unsafe { &*ptr.cast() } + } + + /// Obtain the raw `struct cpumask` pointer. + pub fn as_raw(&self) -> *mut bindings::cpumask { + let this: *const Self = self; + this.cast_mut().cast() + } + + /// Set `cpu` in the cpumask. + /// + /// ATTENTION: Contrary to C, this Rust `set()` method is non-atomic. + /// This mismatches kernel naming convention and corresponds to the C + /// function `__cpumask_set_cpu()`. + #[inline] + pub fn set(&mut self, cpu: u32) { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `__cpumask_set_cpu`. + unsafe { bindings::__cpumask_set_cpu(cpu, self.as_raw()) }; + } + + /// Clear `cpu` in the cpumask. + /// + /// ATTENTION: Contrary to C, this Rust `clear()` method is non-atomic. + /// This mismatches kernel naming convention and corresponds to the C + /// function `__cpumask_clear_cpu()`. + #[inline] + pub fn clear(&mut self, cpu: i32) { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to + // `__cpumask_clear_cpu`. + unsafe { bindings::__cpumask_clear_cpu(cpu, self.as_raw()) }; + } + + /// Test `cpu` in the cpumask. + /// + /// Equivalent to the kernel's `cpumask_test_cpu` API. + #[inline] + pub fn test(&self, cpu: i32) -> bool { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_test_cpu`. + unsafe { bindings::cpumask_test_cpu(cpu, self.as_raw()) } + } + + /// Set all CPUs in the cpumask. + /// + /// Equivalent to the kernel's `cpumask_setall` API. + #[inline] + pub fn setall(&mut self) { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_setall`. + unsafe { bindings::cpumask_setall(self.as_raw()) }; + } + + /// Checks if cpumask is empty. + /// + /// Equivalent to the kernel's `cpumask_empty` API. + #[inline] + pub fn empty(&self) -> bool { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_empty`. + unsafe { bindings::cpumask_empty(self.as_raw()) } + } + + /// Checks if cpumask is full. + /// + /// Equivalent to the kernel's `cpumask_full` API. + #[inline] + pub fn full(&self) -> bool { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_full`. + unsafe { bindings::cpumask_full(self.as_raw()) } + } + + /// Get weight of the cpumask. + /// + /// Equivalent to the kernel's `cpumask_weight` API. + #[inline] + pub fn weight(&self) -> u32 { + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_weight`. + unsafe { bindings::cpumask_weight(self.as_raw()) } + } + + /// Copy cpumask. + /// + /// Equivalent to the kernel's `cpumask_copy` API. + #[inline] + pub fn copy(&self, dstp: &mut Self) { + // SAFETY: By the type invariant, `Self::as_raw` is a valid argument to `cpumask_copy`. + unsafe { bindings::cpumask_copy(dstp.as_raw(), self.as_raw()) }; + } +} + +/// A CPU Mask pointer. +/// +/// Rust abstraction for the C `struct cpumask_var_t`. +/// +/// # Invariants +/// +/// A [`CpumaskVar`] instance always corresponds to a valid C `struct cpumask_var_t`. +/// +/// The callers must ensure that the `struct cpumask_var_t` is valid for access and remains valid +/// for the lifetime of [`CpumaskVar`]. +/// +/// ## Examples +/// +/// The following example demonstrates how to create and update a [`CpumaskVar`]. +/// +/// ``` +/// use kernel::cpumask::CpumaskVar; +/// +/// let mut mask = CpumaskVar::new_zero(GFP_KERNEL).unwrap(); +/// +/// assert!(mask.empty()); +/// mask.set(2); +/// assert!(mask.test(2)); +/// mask.set(3); +/// assert!(mask.test(3)); +/// assert_eq!(mask.weight(), 2); +/// +/// let mask2 = CpumaskVar::try_clone(&mask).unwrap(); +/// assert!(mask2.test(2)); +/// assert!(mask2.test(3)); +/// assert_eq!(mask2.weight(), 2); +/// ``` +pub struct CpumaskVar { + #[cfg(CONFIG_CPUMASK_OFFSTACK)] + ptr: NonNull, + #[cfg(not(CONFIG_CPUMASK_OFFSTACK))] + mask: Cpumask, +} + +impl CpumaskVar { + /// Creates a zero-initialized instance of the [`CpumaskVar`]. + pub fn new_zero(_flags: Flags) -> Result { + Ok(Self { + #[cfg(CONFIG_CPUMASK_OFFSTACK)] + ptr: { + let mut ptr: *mut bindings::cpumask = ptr::null_mut(); + + // SAFETY: It is safe to call this method as the reference to `ptr` is valid. + // + // INVARIANT: The associated memory is freed when the `CpumaskVar` goes out of + // scope. + unsafe { bindings::zalloc_cpumask_var(&mut ptr, _flags.as_raw()) }; + NonNull::new(ptr.cast()).ok_or(AllocError)? + }, + + #[cfg(not(CONFIG_CPUMASK_OFFSTACK))] + // SAFETY: FFI type is valid to be zero-initialized. + // + // INVARIANT: The associated memory is freed when the `CpumaskVar` goes out of scope. + mask: unsafe { core::mem::zeroed() }, + }) + } + + /// Creates an instance of the [`CpumaskVar`]. + /// + /// # Safety + /// + /// The caller must ensure that the returned [`CpumaskVar`] is properly initialized before + /// getting used. + pub unsafe fn new(_flags: Flags) -> Result { + Ok(Self { + #[cfg(CONFIG_CPUMASK_OFFSTACK)] + ptr: { + let mut ptr: *mut bindings::cpumask = ptr::null_mut(); + + // SAFETY: It is safe to call this method as the reference to `ptr` is valid. + // + // INVARIANT: The associated memory is freed when the `CpumaskVar` goes out of + // scope. + unsafe { bindings::alloc_cpumask_var(&mut ptr, _flags.as_raw()) }; + NonNull::new(ptr.cast()).ok_or(AllocError)? + }, + #[cfg(not(CONFIG_CPUMASK_OFFSTACK))] + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The associated memory is freed when the `CpumaskVar` goes out of scope. + mask: unsafe { MaybeUninit::uninit().assume_init() }, + }) + } + + /// Creates a mutable reference to an existing `struct cpumask_var_t` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime + /// of the returned reference. + pub unsafe fn as_mut_ref<'a>(ptr: *mut bindings::cpumask_var_t) -> &'a mut Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the + // lifetime of the returned reference. + unsafe { &mut *ptr.cast() } + } + + /// Creates a reference to an existing `struct cpumask_var_t` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime + /// of the returned reference. + pub unsafe fn as_ref<'a>(ptr: *const bindings::cpumask_var_t) -> &'a Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the + // lifetime of the returned reference. + unsafe { &*ptr.cast() } + } + + /// Clones cpumask. + pub fn try_clone(cpumask: &Cpumask) -> Result { + // SAFETY: The returned cpumask_var is initialized right after this call. + let mut cpumask_var = unsafe { Self::new(GFP_KERNEL) }?; + + cpumask.copy(&mut cpumask_var); + Ok(cpumask_var) + } +} + +// Make [`CpumaskVar`] behave like a pointer to [`Cpumask`]. +impl Deref for CpumaskVar { + type Target = Cpumask; + + #[cfg(CONFIG_CPUMASK_OFFSTACK)] + fn deref(&self) -> &Self::Target { + // SAFETY: The caller owns CpumaskVar, so it is safe to deref the cpumask. + unsafe { &*self.ptr.as_ptr() } + } + + #[cfg(not(CONFIG_CPUMASK_OFFSTACK))] + fn deref(&self) -> &Self::Target { + &self.mask + } +} + +impl DerefMut for CpumaskVar { + #[cfg(CONFIG_CPUMASK_OFFSTACK)] + fn deref_mut(&mut self) -> &mut Cpumask { + // SAFETY: The caller owns CpumaskVar, so it is safe to deref the cpumask. + unsafe { self.ptr.as_mut() } + } + + #[cfg(not(CONFIG_CPUMASK_OFFSTACK))] + fn deref_mut(&mut self) -> &mut Cpumask { + &mut self.mask + } +} + +impl Drop for CpumaskVar { + fn drop(&mut self) { + #[cfg(CONFIG_CPUMASK_OFFSTACK)] + // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `free_cpumask_var`. + unsafe { + bindings::free_cpumask_var(self.as_raw()) + }; + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index de07aadd1ff5..75f78f6bfaa6 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -42,6 +42,7 @@ pub mod block; #[doc(hidden)] pub mod build_assert; +pub mod cpumask; pub mod cred; pub mod device; pub mod device_id; From ab49f64078da82ae5c362900b65a872f8a9bd585 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Tue, 25 Feb 2025 10:05:41 +0530 Subject: [PATCH 20/32] MAINTAINERS: Add entry for Rust cpumask API Update the MAINTAINERS file to include the Rust abstractions for cpumask API. Yury has indicated that he does not wish to maintain the Rust code but would like to be listed as a reviewer. Signed-off-by: Viresh Kumar Reviewed-by: Yury Norov --- MAINTAINERS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 96b827049501..bd7c54af4fd4 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6237,6 +6237,12 @@ L: linux-riscv@lists.infradead.org S: Maintained F: drivers/cpuidle/cpuidle-riscv-sbi.c +CPUMASK API [RUST] +M: Viresh Kumar +R: Yury Norov +S: Maintained +F: rust/kernel/cpumask.rs + CRAMFS FILESYSTEM M: Nicolas Pitre S: Maintained From b7b7b981cb037d250ad06c3a6558d69e04ed77c7 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 20 Feb 2025 16:34:00 +0530 Subject: [PATCH 21/32] rust: clk: Add helpers for Rust code Non-trivial C macros and inlined C functions cannot be used directly in the Rust code and are used via functions ("helpers") that wrap those so that they can be called from Rust. In order to prepare for adding Rust abstractions for the clock APIs, add clock helpers required by the Rust implementation. Reviewed-by: Daniel Almeida Signed-off-by: Viresh Kumar --- MAINTAINERS | 1 + rust/bindings/bindings_helper.h | 1 + rust/helpers/clk.c | 66 +++++++++++++++++++++++++++++++++ rust/helpers/helpers.c | 1 + 4 files changed, 69 insertions(+) create mode 100644 rust/helpers/clk.c diff --git a/MAINTAINERS b/MAINTAINERS index bd7c54af4fd4..608689342aaf 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5883,6 +5883,7 @@ F: include/dt-bindings/clock/ F: include/linux/clk-pr* F: include/linux/clk/ F: include/linux/of_clk.h +F: rust/helpers/clk.c X: drivers/clk/clkdev.c COMMON INTERNET FILE SYSTEM CLIENT (CIFS and SMB3) diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index ab37e1d35c70..f53d6e1a21f2 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/helpers/clk.c b/rust/helpers/clk.c new file mode 100644 index 000000000000..6d04372c9f3b --- /dev/null +++ b/rust/helpers/clk.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +/* + * The "inline" implementation of below helpers are only available when + * CONFIG_HAVE_CLK or CONFIG_HAVE_CLK_PREPARE aren't set. + */ +#ifndef CONFIG_HAVE_CLK +struct clk *rust_helper_clk_get(struct device *dev, const char *id) +{ + return clk_get(dev, id); +} + +void rust_helper_clk_put(struct clk *clk) +{ + clk_put(clk); +} + +int rust_helper_clk_enable(struct clk *clk) +{ + return clk_enable(clk); +} + +void rust_helper_clk_disable(struct clk *clk) +{ + clk_disable(clk); +} + +unsigned long rust_helper_clk_get_rate(struct clk *clk) +{ + return clk_get_rate(clk); +} + +int rust_helper_clk_set_rate(struct clk *clk, unsigned long rate) +{ + return clk_set_rate(clk, rate); +} +#endif + +#ifndef CONFIG_HAVE_CLK_PREPARE +int rust_helper_clk_prepare(struct clk *clk) +{ + return clk_prepare(clk); +} + +void rust_helper_clk_unprepare(struct clk *clk) +{ + clk_unprepare(clk); +} +#endif + +struct clk *rust_helper_clk_get_optional(struct device *dev, const char *id) +{ + return clk_get_optional(dev, id); +} + +int rust_helper_clk_prepare_enable(struct clk *clk) +{ + return clk_prepare_enable(clk); +} + +void rust_helper_clk_disable_unprepare(struct clk *clk) +{ + clk_disable_unprepare(clk); +} diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index e1c21eba9b15..ae595c9cd91b 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -11,6 +11,7 @@ #include "bug.c" #include "build_assert.c" #include "build_bug.c" +#include "clk.c" #include "cpumask.c" #include "cred.c" #include "device.c" From d01d7020560116876c6e39e056737b8f81f153e0 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 20 Feb 2025 16:45:47 +0530 Subject: [PATCH 22/32] rust: clk: Add initial abstractions Add initial abstractions for the clk APIs. These provide the minimal functionality needed for common use cases, making them straightforward to introduce in the first iteration. These will be used by Rust based cpufreq / OPP layers to begin with. Tested-by: Daniel Almeida Reviewed-by: Daniel Almeida Signed-off-by: Viresh Kumar --- MAINTAINERS | 1 + rust/kernel/clk.rs | 334 +++++++++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 1 + 3 files changed, 336 insertions(+) create mode 100644 rust/kernel/clk.rs diff --git a/MAINTAINERS b/MAINTAINERS index 608689342aaf..12cde55579a0 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5884,6 +5884,7 @@ F: include/linux/clk-pr* F: include/linux/clk/ F: include/linux/of_clk.h F: rust/helpers/clk.c +F: rust/kernel/clk.rs X: drivers/clk/clkdev.c COMMON INTERNET FILE SYSTEM CLIENT (CIFS and SMB3) diff --git a/rust/kernel/clk.rs b/rust/kernel/clk.rs new file mode 100644 index 000000000000..6041c6d07527 --- /dev/null +++ b/rust/kernel/clk.rs @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Clock abstractions. +//! +//! C header: [`include/linux/clk.h`](srctree/include/linux/clk.h) +//! +//! Reference: + +use crate::ffi::c_ulong; + +/// The frequency unit. +/// +/// Represents a frequency in hertz, wrapping a [`c_ulong`] value. +/// +/// ## Examples +/// +/// ``` +/// use kernel::clk::Hertz; +/// +/// let hz = 1_000_000_000; +/// let rate = Hertz(hz); +/// +/// assert_eq!(rate.as_hz(), hz); +/// assert_eq!(rate, Hertz(hz)); +/// assert_eq!(rate, Hertz::from_khz(hz / 1_000)); +/// assert_eq!(rate, Hertz::from_mhz(hz / 1_000_000)); +/// assert_eq!(rate, Hertz::from_ghz(hz / 1_000_000_000)); +/// ``` +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct Hertz(pub c_ulong); + +impl Hertz { + /// Create a new instance from kilohertz (kHz) + pub fn from_khz(khz: c_ulong) -> Self { + Self(khz * 1_000) + } + + /// Create a new instance from megahertz (MHz) + pub fn from_mhz(mhz: c_ulong) -> Self { + Self(mhz * 1_000_000) + } + + /// Create a new instance from gigahertz (GHz) + pub fn from_ghz(ghz: c_ulong) -> Self { + Self(ghz * 1_000_000_000) + } + + /// Get the frequency in hertz + pub fn as_hz(&self) -> c_ulong { + self.0 + } + + /// Get the frequency in kilohertz + pub fn as_khz(&self) -> c_ulong { + self.0 / 1_000 + } + + /// Get the frequency in megahertz + pub fn as_mhz(&self) -> c_ulong { + self.0 / 1_000_000 + } + + /// Get the frequency in gigahertz + pub fn as_ghz(&self) -> c_ulong { + self.0 / 1_000_000_000 + } +} + +impl From for c_ulong { + fn from(freq: Hertz) -> Self { + freq.0 + } +} + +#[cfg(CONFIG_COMMON_CLK)] +mod common_clk { + use super::Hertz; + use crate::{ + device::Device, + error::{from_err_ptr, to_result, Result}, + prelude::*, + }; + + use core::{ops::Deref, ptr}; + + /// A reference-counted clock. + /// + /// Rust abstraction for the C [`struct clk`]. + /// + /// # Invariants + /// + /// A [`Clk`] instance holds either a pointer to a valid [`struct clk`] created by the C + /// portion of the kernel or a NULL pointer. + /// + /// Instances of this type are reference-counted. Calling [`Clk::get`] ensures that the + /// allocation remains valid for the lifetime of the [`Clk`]. + /// + /// ## Examples + /// + /// The following example demonstrates how to obtain and configure a clock for a device. + /// + /// ``` + /// use kernel::c_str; + /// use kernel::clk::{Clk, Hertz}; + /// use kernel::device::Device; + /// use kernel::error::Result; + /// + /// fn configure_clk(dev: &Device) -> Result { + /// let clk = Clk::get(dev, Some(c_str!("apb_clk")))?; + /// + /// clk.prepare_enable()?; + /// + /// let expected_rate = Hertz::from_ghz(1); + /// + /// if clk.rate() != expected_rate { + /// clk.set_rate(expected_rate)?; + /// } + /// + /// clk.disable_unprepare(); + /// Ok(()) + /// } + /// ``` + /// + /// [`struct clk`]: https://docs.kernel.org/driver-api/clk.html + #[repr(transparent)] + pub struct Clk(*mut bindings::clk); + + impl Clk { + /// Gets [`Clk`] corresponding to a [`Device`] and a connection id. + /// + /// Equivalent to the kernel's [`clk_get`] API. + /// + /// [`clk_get`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_get + pub fn get(dev: &Device, name: Option<&CStr>) -> Result { + let con_id = if let Some(name) = name { + name.as_ptr() + } else { + ptr::null() + }; + + // SAFETY: It is safe to call [`clk_get`] for a valid device pointer. + // + // INVARIANT: The reference-count is decremented when [`Clk`] goes out of scope. + Ok(Self(from_err_ptr(unsafe { + bindings::clk_get(dev.as_raw(), con_id) + })?)) + } + + /// Obtain the raw [`struct clk`] pointer. + #[inline] + pub fn as_raw(&self) -> *mut bindings::clk { + self.0 + } + + /// Enable the clock. + /// + /// Equivalent to the kernel's [`clk_enable`] API. + /// + /// [`clk_enable`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_enable + #[inline] + pub fn enable(&self) -> Result { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_enable`]. + to_result(unsafe { bindings::clk_enable(self.as_raw()) }) + } + + /// Disable the clock. + /// + /// Equivalent to the kernel's [`clk_disable`] API. + /// + /// [`clk_disable`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_disable + #[inline] + pub fn disable(&self) { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_disable`]. + unsafe { bindings::clk_disable(self.as_raw()) }; + } + + /// Prepare the clock. + /// + /// Equivalent to the kernel's [`clk_prepare`] API. + /// + /// [`clk_prepare`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_prepare + #[inline] + pub fn prepare(&self) -> Result { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_prepare`]. + to_result(unsafe { bindings::clk_prepare(self.as_raw()) }) + } + + /// Unprepare the clock. + /// + /// Equivalent to the kernel's [`clk_unprepare`] API. + /// + /// [`clk_unprepare`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_unprepare + #[inline] + pub fn unprepare(&self) { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_unprepare`]. + unsafe { bindings::clk_unprepare(self.as_raw()) }; + } + + /// Prepare and enable the clock. + /// + /// Equivalent to calling [`Clk::prepare`] followed by [`Clk::enable`]. + #[inline] + pub fn prepare_enable(&self) -> Result { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_prepare_enable`]. + to_result(unsafe { bindings::clk_prepare_enable(self.as_raw()) }) + } + + /// Disable and unprepare the clock. + /// + /// Equivalent to calling [`Clk::disable`] followed by [`Clk::unprepare`]. + #[inline] + pub fn disable_unprepare(&self) { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_disable_unprepare`]. + unsafe { bindings::clk_disable_unprepare(self.as_raw()) }; + } + + /// Get clock's rate. + /// + /// Equivalent to the kernel's [`clk_get_rate`] API. + /// + /// [`clk_get_rate`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_get_rate + #[inline] + pub fn rate(&self) -> Hertz { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_get_rate`]. + Hertz(unsafe { bindings::clk_get_rate(self.as_raw()) }) + } + + /// Set clock's rate. + /// + /// Equivalent to the kernel's [`clk_set_rate`] API. + /// + /// [`clk_set_rate`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_set_rate + #[inline] + pub fn set_rate(&self, rate: Hertz) -> Result { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for + // [`clk_set_rate`]. + to_result(unsafe { bindings::clk_set_rate(self.as_raw(), rate.as_hz()) }) + } + } + + impl Drop for Clk { + fn drop(&mut self) { + // SAFETY: By the type invariants, self.as_raw() is a valid argument for [`clk_put`]. + unsafe { bindings::clk_put(self.as_raw()) }; + } + } + + /// A reference-counted optional clock. + /// + /// A lightweight wrapper around an optional [`Clk`]. An [`OptionalClk`] represents a [`Clk`] + /// that a driver can function without but may improve performance or enable additional + /// features when available. + /// + /// # Invariants + /// + /// An [`OptionalClk`] instance encapsulates a [`Clk`] with either a valid [`struct clk`] or + /// `NULL` pointer. + /// + /// Instances of this type are reference-counted. Calling [`OptionalClk::get`] ensures that the + /// allocation remains valid for the lifetime of the [`OptionalClk`]. + /// + /// ## Examples + /// + /// The following example demonstrates how to obtain and configure an optional clock for a + /// device. The code functions correctly whether or not the clock is available. + /// + /// ``` + /// use kernel::c_str; + /// use kernel::clk::{OptionalClk, Hertz}; + /// use kernel::device::Device; + /// use kernel::error::Result; + /// + /// fn configure_clk(dev: &Device) -> Result { + /// let clk = OptionalClk::get(dev, Some(c_str!("apb_clk")))?; + /// + /// clk.prepare_enable()?; + /// + /// let expected_rate = Hertz::from_ghz(1); + /// + /// if clk.rate() != expected_rate { + /// clk.set_rate(expected_rate)?; + /// } + /// + /// clk.disable_unprepare(); + /// Ok(()) + /// } + /// ``` + /// + /// [`struct clk`]: https://docs.kernel.org/driver-api/clk.html + pub struct OptionalClk(Clk); + + impl OptionalClk { + /// Gets [`OptionalClk`] corresponding to a [`Device`] and a connection id. + /// + /// Equivalent to the kernel's [`clk_get_optional`] API. + /// + /// [`clk_get_optional`]: + /// https://docs.kernel.org/core-api/kernel-api.html#c.clk_get_optional + pub fn get(dev: &Device, name: Option<&CStr>) -> Result { + let con_id = if let Some(name) = name { + name.as_ptr() + } else { + ptr::null() + }; + + // SAFETY: It is safe to call [`clk_get_optional`] for a valid device pointer. + // + // INVARIANT: The reference-count is decremented when [`OptionalClk`] goes out of + // scope. + Ok(Self(Clk(from_err_ptr(unsafe { + bindings::clk_get_optional(dev.as_raw(), con_id) + })?))) + } + } + + // Make [`OptionalClk`] behave like [`Clk`]. + impl Deref for OptionalClk { + type Target = Clk; + + fn deref(&self) -> &Clk { + &self.0 + } + } +} + +#[cfg(CONFIG_COMMON_CLK)] +pub use common_clk::*; diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 75f78f6bfaa6..3fd7c17cbc06 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -42,6 +42,7 @@ pub mod block; #[doc(hidden)] pub mod build_assert; +pub mod clk; pub mod cpumask; pub mod cred; pub mod device; From a4e3b76e4d5c265824e686661a79f75df09c4834 Mon Sep 17 00:00:00 2001 From: Anisse Astier Date: Wed, 22 Jan 2025 14:39:52 +0100 Subject: [PATCH 23/32] rust: macros: enable use of hyphens in module names Some modules might need naming that contains hyphens "-" to match the auto-probing by name in the platform devices that comes from the device tree. But Rust identifier cannot contain hyphens, so replace them with underscores. Signed-off-by: Anisse Astier Reviewed-by: Alice Ryhl [ Viresh: Replace "-" with '-', minor commit log fix, rename variable and fix line length checkpatch warnings ] Reviewed-by: Benno Lossin Signed-off-by: Viresh Kumar --- rust/macros/module.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/rust/macros/module.rs b/rust/macros/module.rs index a9418fbc9b44..6ff34096d7ee 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -185,7 +185,9 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { let info = ModuleInfo::parse(&mut it); - let mut modinfo = ModInfoBuilder::new(info.name.as_ref()); + // Rust does not allow hyphens in identifiers, use underscore instead. + let ident = info.name.replace('-', "_"); + let mut modinfo = ModInfoBuilder::new(ident.as_ref()); if let Some(author) = info.author { modinfo.emit("author", &author); } @@ -310,14 +312,15 @@ mod __module_init {{ #[doc(hidden)] #[link_section = \"{initcall_section}\"] #[used] - pub static __{name}_initcall: extern \"C\" fn() -> kernel::ffi::c_int = __{name}_init; + pub static __{ident}_initcall: extern \"C\" fn() -> + kernel::ffi::c_int = __{ident}_init; #[cfg(not(MODULE))] #[cfg(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS)] core::arch::global_asm!( r#\".section \"{initcall_section}\", \"a\" - __{name}_initcall: - .long __{name}_init - . + __{ident}_initcall: + .long __{ident}_init - . .previous \"# ); @@ -325,7 +328,7 @@ mod __module_init {{ #[cfg(not(MODULE))] #[doc(hidden)] #[no_mangle] - pub extern \"C\" fn __{name}_init() -> kernel::ffi::c_int {{ + pub extern \"C\" fn __{ident}_init() -> kernel::ffi::c_int {{ // SAFETY: This function is inaccessible to the outside due to the double // module wrapping it. It is called exactly once by the C side via its // placement above in the initcall section. @@ -335,13 +338,13 @@ mod __module_init {{ #[cfg(not(MODULE))] #[doc(hidden)] #[no_mangle] - pub extern \"C\" fn __{name}_exit() {{ + pub extern \"C\" fn __{ident}_exit() {{ // SAFETY: // - This function is inaccessible to the outside due to the double // module wrapping it. It is called exactly once by the C side via its // unique name, - // - furthermore it is only called after `__{name}_init` has returned `0` - // (which delegates to `__init`). + // - furthermore it is only called after `__{ident}_init` has + // returned `0` (which delegates to `__init`). unsafe {{ __exit() }} }} @@ -381,6 +384,7 @@ unsafe fn __exit() {{ ", type_ = info.type_, name = info.name, + ident = ident, modinfo = modinfo.buffer, initcall_section = ".initcall6.init" ) From 3accb57d56a9bcf1cab1e908e88a235e899a2e82 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 8 Jan 2025 16:18:38 +0530 Subject: [PATCH 24/32] rust: cpu: Add from_cpu() This implements cpu::from_cpu(), which returns a reference to Device for a CPU. The C struct is created at initialization time for CPUs and is never freed and so ARef isn't returned from this function. The new helper will be used by Rust based cpufreq drivers. Signed-off-by: Viresh Kumar --- MAINTAINERS | 1 + rust/bindings/bindings_helper.h | 1 + rust/kernel/cpu.rs | 30 ++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 1 + 4 files changed, 33 insertions(+) create mode 100644 rust/kernel/cpu.rs diff --git a/MAINTAINERS b/MAINTAINERS index 12cde55579a0..475abf72869c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6155,6 +6155,7 @@ F: include/linux/cpuhotplug.h F: include/linux/smpboot.h F: kernel/cpu.c F: kernel/smpboot.* +F: rust/kernel/cpu.rs CPU IDLE TIME MANAGEMENT FRAMEWORK M: "Rafael J. Wysocki" diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index f53d6e1a21f2..ac92c67d2c38 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/kernel/cpu.rs b/rust/kernel/cpu.rs new file mode 100644 index 000000000000..10c5c3b25873 --- /dev/null +++ b/rust/kernel/cpu.rs @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic CPU definitions. +//! +//! C header: [`include/linux/cpu.h`](srctree/include/linux/cpu.h) + +use crate::{bindings, device::Device, error::Result, prelude::ENODEV}; + +/// Creates a new instance of CPU's device. +/// +/// # Safety +/// +/// Reference counting is not implemented for the CPU device in the C code. When a CPU is +/// hot-unplugged, the corresponding CPU device is unregistered, but its associated memory +/// is not freed. +/// +/// Callers must ensure that the CPU device is not used after it has been unregistered. +/// This can be achieved, for example, by registering a CPU hotplug notifier and removing +/// any references to the CPU device within the notifier's callback. +pub unsafe fn from_cpu(cpu: u32) -> Result<&'static Device> { + // SAFETY: It is safe to call `get_cpu_device()` for any CPU. + let ptr = unsafe { bindings::get_cpu_device(cpu) }; + if ptr.is_null() { + return Err(ENODEV); + } + + // SAFETY: The pointer returned by `get_cpu_device()`, if not `NULL`, is a valid pointer to + // a `struct device` and is never freed by the C code. + Ok(unsafe { Device::as_ref(ptr) }) +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 3fd7c17cbc06..de0a840fcc99 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -43,6 +43,7 @@ #[doc(hidden)] pub mod build_assert; pub mod clk; +pub mod cpu; pub mod cpumask; pub mod cred; pub mod device; From 8f835497b32b59c0ba8b5e1805b7e89dd38cd9f9 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 18 Dec 2023 11:15:13 +0530 Subject: [PATCH 25/32] rust: opp: Add initial abstractions for OPP framework Introduce initial Rust abstractions for the Operating Performance Points (OPP) framework. This includes bindings for `struct dev_pm_opp` and `struct dev_pm_opp_data`, laying the groundwork for further OPP integration. Signed-off-by: Viresh Kumar --- MAINTAINERS | 1 + rust/bindings/bindings_helper.h | 1 + rust/kernel/lib.rs | 2 + rust/kernel/opp.rs | 299 ++++++++++++++++++++++++++++++++ 4 files changed, 303 insertions(+) create mode 100644 rust/kernel/opp.rs diff --git a/MAINTAINERS b/MAINTAINERS index 475abf72869c..931e418f89ed 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -18165,6 +18165,7 @@ F: Documentation/devicetree/bindings/opp/ F: Documentation/power/opp.rst F: drivers/opp/ F: include/linux/pm_opp.h +F: rust/kernel/opp.rs OPL4 DRIVER M: Clemens Ladisch diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index ac92c67d2c38..529f22891e0b 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index de0a840fcc99..ea589254b4ac 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -67,6 +67,8 @@ #[cfg(CONFIG_NET)] pub mod net; pub mod of; +#[cfg(CONFIG_PM_OPP)] +pub mod opp; pub mod page; #[cfg(CONFIG_PCI)] pub mod pci; diff --git a/rust/kernel/opp.rs b/rust/kernel/opp.rs new file mode 100644 index 000000000000..8f0493a8b6e8 --- /dev/null +++ b/rust/kernel/opp.rs @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Operating performance points. +//! +//! This module provides rust abstractions for interacting with the OPP subsystem. +//! +//! C header: [`include/linux/pm_opp.h`](srctree/include/linux/pm_opp.h) +//! +//! Reference: + +use crate::{ + clk::Hertz, + device::Device, + error::{code::*, to_result, Result}, + ffi::c_ulong, + types::{ARef, AlwaysRefCounted, Opaque}, +}; + +use core::ptr; + +/// The voltage unit. +/// +/// Represents voltage in microvolts, wrapping a [`c_ulong`] value. +/// +/// ## Examples +/// +/// ``` +/// use kernel::opp::MicroVolt; +/// +/// let raw = 90500; +/// let volt = MicroVolt(raw); +/// +/// assert_eq!(usize::from(volt), raw); +/// assert_eq!(volt, MicroVolt(raw)); +/// ``` +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct MicroVolt(pub c_ulong); + +impl From for c_ulong { + #[inline] + fn from(volt: MicroVolt) -> Self { + volt.0 + } +} + +/// The power unit. +/// +/// Represents power in microwatts, wrapping a [`c_ulong`] value. +/// +/// ## Examples +/// +/// ``` +/// use kernel::opp::MicroWatt; +/// +/// let raw = 1000000; +/// let power = MicroWatt(raw); +/// +/// assert_eq!(usize::from(power), raw); +/// assert_eq!(power, MicroWatt(raw)); +/// ``` +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct MicroWatt(pub c_ulong); + +impl From for c_ulong { + #[inline] + fn from(power: MicroWatt) -> Self { + power.0 + } +} + +/// Handle for a dynamically created [`OPP`]. +/// +/// The associated [`OPP`] is automatically removed when the [`Token`] is dropped. +/// +/// ## Examples +/// +/// The following example demonstrates how to create an [`OPP`] dynamically. +/// +/// ``` +/// use kernel::clk::Hertz; +/// use kernel::device::Device; +/// use kernel::error::Result; +/// use kernel::opp::{Data, MicroVolt, Token}; +/// use kernel::types::ARef; +/// +/// fn create_opp(dev: &ARef, freq: Hertz, volt: MicroVolt, level: u32) -> Result { +/// let data = Data::new(freq, volt, level, false); +/// +/// // OPP is removed once token goes out of scope. +/// data.add_opp(dev) +/// } +/// ``` +pub struct Token { + dev: ARef, + freq: Hertz, +} + +impl Token { + /// Dynamically adds an [`OPP`] and returns a [`Token`] that removes it on drop. + fn new(dev: &ARef, mut data: Data) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_add_dynamic(dev.as_raw(), &mut data.0) })?; + Ok(Self { + dev: dev.clone(), + freq: data.freq(), + }) + } +} + +impl Drop for Token { + fn drop(&mut self) { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + unsafe { bindings::dev_pm_opp_remove(self.dev.as_raw(), self.freq.into()) }; + } +} + +/// OPP data. +/// +/// Rust abstraction for the C `struct dev_pm_opp_data`, used to define operating performance +/// points (OPPs) dynamically. +/// +/// ## Examples +/// +/// The following example demonstrates how to create an [`OPP`] with [`Data`]. +/// +/// ``` +/// use kernel::clk::Hertz; +/// use kernel::device::Device; +/// use kernel::error::Result; +/// use kernel::opp::{Data, MicroVolt, Token}; +/// use kernel::types::ARef; +/// +/// fn create_opp(dev: &ARef, freq: Hertz, volt: MicroVolt, level: u32) -> Result { +/// let data = Data::new(freq, volt, level, false); +/// +/// // OPP is removed once token goes out of scope. +/// data.add_opp(dev) +/// } +/// ``` +#[repr(transparent)] +pub struct Data(bindings::dev_pm_opp_data); + +impl Data { + /// Creates a new instance of [`Data`]. + /// + /// This can be used to define a dynamic OPP to be added to a device. + pub fn new(freq: Hertz, volt: MicroVolt, level: u32, turbo: bool) -> Self { + Self(bindings::dev_pm_opp_data { + turbo, + freq: freq.into(), + u_volt: volt.into(), + level, + }) + } + + /// Adds an [`OPP`] dynamically. + /// + /// Returns a [`Token`] that ensures the OPP is automatically removed + /// when it goes out of scope. + #[inline] + pub fn add_opp(self, dev: &ARef) -> Result { + Token::new(dev, self) + } + + /// Returns the frequency associated with this OPP data. + #[inline] + fn freq(&self) -> Hertz { + Hertz(self.0.freq) + } +} + +/// A reference-counted Operating performance point (OPP). +/// +/// Rust abstraction for the C `struct dev_pm_opp`. +/// +/// # Invariants +/// +/// The pointer stored in `Self` is non-null and valid for the lifetime of the [`OPP`]. +/// +/// Instances of this type are reference-counted. The reference count is incremented by the +/// `dev_pm_opp_get` function and decremented by `dev_pm_opp_put`. The Rust type `ARef` +/// represents a pointer that owns a reference count on the [`OPP`]. +/// +/// A reference to the [`OPP`], &[`OPP`], isn't refcounted by the Rust code. +#[repr(transparent)] +pub struct OPP(Opaque); + +/// SAFETY: It is okay to send the ownership of [`OPP`] across thread boundaries. +unsafe impl Send for OPP {} + +/// SAFETY: It is okay to access [`OPP`] through shared references from other threads because we're +/// either accessing properties that don't change or that are properly synchronised by C code. +unsafe impl Sync for OPP {} + +/// SAFETY: The type invariants guarantee that [`OPP`] is always refcounted. +unsafe impl AlwaysRefCounted for OPP { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference means that the refcount is nonzero. + unsafe { bindings::dev_pm_opp_get(self.0.get()) }; + } + + unsafe fn dec_ref(obj: ptr::NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is nonzero. + unsafe { bindings::dev_pm_opp_put(obj.cast().as_ptr()) } + } +} + +impl OPP { + /// Creates an owned reference to a [`OPP`] from a valid pointer. + /// + /// The refcount is incremented by the C code and will be decremented by `dec_ref` when the + /// [`ARef`] object is dropped. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid and the refcount of the [`OPP`] is incremented. + /// The caller must also ensure that it doesn't explicitly drop the refcount of the [`OPP`], as + /// the returned [`ARef`] object takes over the refcount increment on the underlying object and + /// the same will be dropped along with it. + pub unsafe fn from_raw_opp_owned(ptr: *mut bindings::dev_pm_opp) -> Result> { + let ptr = ptr::NonNull::new(ptr).ok_or(ENODEV)?; + + // SAFETY: The safety requirements guarantee the validity of the pointer. + // + // INVARIANT: The reference-count is decremented when [`OPP`] goes out of scope. + Ok(unsafe { ARef::from_raw(ptr.cast()) }) + } + + /// Creates a reference to a [`OPP`] from a valid pointer. + /// + /// The refcount is not updated by the Rust API unless the returned reference is converted to + /// an [`ARef`] object. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid and remains valid for the duration of `'a`. + #[inline] + pub unsafe fn from_raw_opp<'a>(ptr: *mut bindings::dev_pm_opp) -> Result<&'a Self> { + // SAFETY: The caller guarantees that the pointer is not dangling and stays valid for the + // duration of 'a. The cast is okay because [`OPP`] is `repr(transparent)`. + Ok(unsafe { &*ptr.cast() }) + } + + #[inline] + fn as_raw(&self) -> *mut bindings::dev_pm_opp { + self.0.get() + } + + /// Returns the frequency of an [`OPP`]. + pub fn freq(&self, index: Option) -> Hertz { + let index = index.unwrap_or(0); + + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // use it. + Hertz(unsafe { bindings::dev_pm_opp_get_freq_indexed(self.as_raw(), index) }) + } + + /// Returns the voltage of an [`OPP`]. + #[inline] + pub fn voltage(&self) -> MicroVolt { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // use it. + MicroVolt(unsafe { bindings::dev_pm_opp_get_voltage(self.as_raw()) }) + } + + /// Returns the level of an [`OPP`]. + #[inline] + pub fn level(&self) -> u32 { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // use it. + unsafe { bindings::dev_pm_opp_get_level(self.as_raw()) } + } + + /// Returns the power of an [`OPP`]. + #[inline] + pub fn power(&self) -> MicroWatt { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // use it. + MicroWatt(unsafe { bindings::dev_pm_opp_get_power(self.as_raw()) }) + } + + /// Returns the required pstate of an [`OPP`]. + #[inline] + pub fn required_pstate(&self, index: u32) -> u32 { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // use it. + unsafe { bindings::dev_pm_opp_get_required_pstate(self.as_raw(), index) } + } + + /// Returns true if the [`OPP`] is turbo. + #[inline] + pub fn is_turbo(&self) -> bool { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // use it. + unsafe { bindings::dev_pm_opp_is_turbo(self.as_raw()) } + } +} From d52c7e868fbc134d88bb04d5909f2df2f9da5202 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 18 Dec 2023 11:15:13 +0530 Subject: [PATCH 26/32] rust: opp: Add abstractions for the OPP table Introduce Rust abstractions for `struct opp_table`, enabling access to OPP tables from Rust. Signed-off-by: Viresh Kumar --- rust/kernel/opp.rs | 487 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 486 insertions(+), 1 deletion(-) diff --git a/rust/kernel/opp.rs b/rust/kernel/opp.rs index 8f0493a8b6e8..26a0ecfe046a 100644 --- a/rust/kernel/opp.rs +++ b/rust/kernel/opp.rs @@ -10,8 +10,9 @@ use crate::{ clk::Hertz, + cpumask::{Cpumask, CpumaskVar}, device::Device, - error::{code::*, to_result, Result}, + error::{code::*, from_err_ptr, to_result, Error, Result}, ffi::c_ulong, types::{ARef, AlwaysRefCounted, Opaque}, }; @@ -171,6 +172,469 @@ fn freq(&self) -> Hertz { } } +/// [`OPP`] search options. +/// +/// ## Examples +/// +/// Defines how to search for an [`OPP`] in a [`Table`] relative to a frequency. +/// +/// ``` +/// use kernel::clk::Hertz; +/// use kernel::error::Result; +/// use kernel::opp::{OPP, SearchType, Table}; +/// use kernel::types::ARef; +/// +/// fn find_opp(table: &Table, freq: Hertz) -> Result> { +/// let opp = table.opp_from_freq(freq, Some(true), None, SearchType::Exact)?; +/// +/// pr_info!("OPP frequency is: {:?}\n", opp.freq(None)); +/// pr_info!("OPP voltage is: {:?}\n", opp.voltage()); +/// pr_info!("OPP level is: {}\n", opp.level()); +/// pr_info!("OPP power is: {:?}\n", opp.power()); +/// +/// Ok(opp) +/// } +/// ``` +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum SearchType { + /// Match the exact frequency. + Exact, + /// Find the highest frequency less than or equal to the given value. + Floor, + /// Find the lowest frequency greater than or equal to the given value. + Ceil, +} + +/// A reference-counted OPP table. +/// +/// Rust abstraction for the C `struct opp_table`. +/// +/// # Invariants +/// +/// The pointer stored in `Self` is non-null and valid for the lifetime of the [`Table`]. +/// +/// Instances of this type are reference-counted. +/// +/// ## Examples +/// +/// The following example demonstrates how to get OPP [`Table`] for a [`Cpumask`] and set its +/// frequency. +/// +/// ``` +/// use kernel::clk::Hertz; +/// use kernel::cpumask::Cpumask; +/// use kernel::device::Device; +/// use kernel::error::Result; +/// use kernel::opp::Table; +/// use kernel::types::ARef; +/// +/// fn get_table(dev: &ARef, mask: &mut Cpumask, freq: Hertz) -> Result { +/// let mut opp_table = Table::from_of_cpumask(dev, mask)?; +/// +/// if opp_table.opp_count()? == 0 { +/// return Err(EINVAL); +/// } +/// +/// pr_info!("Max transition latency is: {} ns\n", opp_table.max_transition_latency_ns()); +/// pr_info!("Suspend frequency is: {:?}\n", opp_table.suspend_freq()); +/// +/// opp_table.set_rate(freq)?; +/// Ok(opp_table) +/// } +/// ``` +pub struct Table { + ptr: *mut bindings::opp_table, + dev: ARef, + #[allow(dead_code)] + em: bool, + #[allow(dead_code)] + of: bool, + cpus: Option, +} + +/// SAFETY: It is okay to send ownership of [`Table`] across thread boundaries. +unsafe impl Send for Table {} + +/// SAFETY: It is okay to access [`Table`] through shared references from other threads because +/// we're either accessing properties that don't change or that are properly synchronised by C code. +unsafe impl Sync for Table {} + +impl Table { + /// Creates a new reference-counted [`Table`] from a raw pointer. + /// + /// # Safety + /// + /// Callers must ensure that `ptr` is valid and non-null. + unsafe fn from_raw_table(ptr: *mut bindings::opp_table, dev: &ARef) -> Self { + // SAFETY: By the safety requirements, ptr is valid and its refcount will be incremented. + // + // INVARIANT: The reference-count is decremented when [`Table`] goes out of scope. + unsafe { bindings::dev_pm_opp_get_opp_table_ref(ptr) }; + + Self { + ptr, + dev: dev.clone(), + em: false, + of: false, + cpus: None, + } + } + + /// Creates a new reference-counted [`Table`] instance for a [`Device`]. + pub fn from_dev(dev: &Device) -> Result { + // SAFETY: The requirements are satisfied by the existence of the [`Device`] and its safety + // requirements. + // + // INVARIANT: The reference-count is incremented by the C code and is decremented when + // [`Table`] goes out of scope. + let ptr = from_err_ptr(unsafe { bindings::dev_pm_opp_get_opp_table(dev.as_raw()) })?; + + Ok(Self { + ptr, + dev: dev.into(), + em: false, + of: false, + cpus: None, + }) + } + + /// Creates a new reference-counted [`Table`] instance for a [`Device`] based on device tree + /// entries. + #[cfg(CONFIG_OF)] + pub fn from_of(dev: &ARef, index: i32) -> Result { + // SAFETY: The requirements are satisfied by the existence of the [`Device`] and its safety + // requirements. + // + // INVARIANT: The reference-count is incremented by the C code and is decremented when + // [`Table`] goes out of scope. + to_result(unsafe { bindings::dev_pm_opp_of_add_table_indexed(dev.as_raw(), index) })?; + + // Get the newly created [`Table`]. + let mut table = Self::from_dev(dev)?; + table.of = true; + + Ok(table) + } + + /// Remove device tree based [`Table`]. + #[cfg(CONFIG_OF)] + #[inline] + fn remove_of(&self) { + // SAFETY: The requirements are satisfied by the existence of the [`Device`] and its safety + // requirements. We took the reference from [`from_of`] earlier, it is safe to drop the + // same now. + unsafe { bindings::dev_pm_opp_of_remove_table(self.dev.as_raw()) }; + } + + /// Creates a new reference-counted [`Table`] instance for a [`Cpumask`] based on device tree + /// entries. + #[cfg(CONFIG_OF)] + pub fn from_of_cpumask(dev: &Device, cpumask: &mut Cpumask) -> Result { + // SAFETY: The cpumask is valid and the returned pointer will be owned by the [`Table`] + // instance. + // + // INVARIANT: The reference-count is incremented by the C code and is decremented when + // [`Table`] goes out of scope. + to_result(unsafe { bindings::dev_pm_opp_of_cpumask_add_table(cpumask.as_raw()) })?; + + // Fetch the newly created table. + let mut table = Self::from_dev(dev)?; + table.cpus = Some(CpumaskVar::try_clone(cpumask)?); + + Ok(table) + } + + /// Remove device tree based [`Table`] for a [`Cpumask`]. + #[cfg(CONFIG_OF)] + #[inline] + fn remove_of_cpumask(&self, cpumask: &Cpumask) { + // SAFETY: The cpumask is valid and we took the reference from [`from_of_cpumask`] earlier, + // it is safe to drop the same now. + unsafe { bindings::dev_pm_opp_of_cpumask_remove_table(cpumask.as_raw()) }; + } + + /// Returns the number of [`OPP`]s in the [`Table`]. + pub fn opp_count(&self) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + let ret = unsafe { bindings::dev_pm_opp_get_opp_count(self.dev.as_raw()) }; + if ret < 0 { + Err(Error::from_errno(ret)) + } else { + Ok(ret as u32) + } + } + + /// Returns max clock latency (in nanoseconds) of the [`OPP`]s in the [`Table`]. + #[inline] + pub fn max_clock_latency_ns(&self) -> usize { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + unsafe { bindings::dev_pm_opp_get_max_clock_latency(self.dev.as_raw()) } + } + + /// Returns max volt latency (in nanoseconds) of the [`OPP`]s in the [`Table`]. + #[inline] + pub fn max_volt_latency_ns(&self) -> usize { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + unsafe { bindings::dev_pm_opp_get_max_volt_latency(self.dev.as_raw()) } + } + + /// Returns max transition latency (in nanoseconds) of the [`OPP`]s in the [`Table`]. + #[inline] + pub fn max_transition_latency_ns(&self) -> usize { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + unsafe { bindings::dev_pm_opp_get_max_transition_latency(self.dev.as_raw()) } + } + + /// Returns the suspend [`OPP`]'s frequency. + #[inline] + pub fn suspend_freq(&self) -> Hertz { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + Hertz(unsafe { bindings::dev_pm_opp_get_suspend_opp_freq(self.dev.as_raw()) }) + } + + /// Synchronizes regulators used by the [`Table`]. + #[inline] + pub fn sync_regulators(&self) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_sync_regulators(self.dev.as_raw()) }) + } + + /// Gets sharing CPUs. + #[inline] + pub fn sharing_cpus(dev: &Device, cpumask: &mut Cpumask) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_get_sharing_cpus(dev.as_raw(), cpumask.as_raw()) }) + } + + /// Sets sharing CPUs. + pub fn set_sharing_cpus(&mut self, cpumask: &mut Cpumask) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { + bindings::dev_pm_opp_set_sharing_cpus(self.dev.as_raw(), cpumask.as_raw()) + })?; + + if let Some(mask) = self.cpus.as_mut() { + // Update the cpumask as this will be used while removing the table. + cpumask.copy(mask); + } + + Ok(()) + } + + /// Gets sharing CPUs from device tree. + #[cfg(CONFIG_OF)] + #[inline] + pub fn of_sharing_cpus(dev: &Device, cpumask: &mut Cpumask) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { + bindings::dev_pm_opp_of_get_sharing_cpus(dev.as_raw(), cpumask.as_raw()) + }) + } + + /// Updates the voltage value for an [`OPP`]. + #[inline] + pub fn adjust_voltage( + &self, + freq: Hertz, + volt: MicroVolt, + volt_min: MicroVolt, + volt_max: MicroVolt, + ) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { + bindings::dev_pm_opp_adjust_voltage( + self.dev.as_raw(), + freq.into(), + volt.into(), + volt_min.into(), + volt_max.into(), + ) + }) + } + + /// Configures device with [`OPP`] matching the frequency value. + #[inline] + pub fn set_rate(&self, freq: Hertz) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_set_rate(self.dev.as_raw(), freq.into()) }) + } + + /// Configures device with [`OPP`]. + #[inline] + pub fn set_opp(&self, opp: &OPP) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_set_opp(self.dev.as_raw(), opp.as_raw()) }) + } + + /// Finds [`OPP`] based on frequency. + pub fn opp_from_freq( + &self, + freq: Hertz, + available: Option, + index: Option, + stype: SearchType, + ) -> Result> { + let raw_dev = self.dev.as_raw(); + let index = index.unwrap_or(0); + let mut rate = freq.into(); + + let ptr = from_err_ptr(match stype { + SearchType::Exact => { + if let Some(available) = available { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and + // its safety requirements. The returned pointer will be owned by the new + // [`OPP`] instance. + unsafe { + bindings::dev_pm_opp_find_freq_exact_indexed( + raw_dev, rate, index, available, + ) + } + } else { + return Err(EINVAL); + } + } + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Ceil => unsafe { + bindings::dev_pm_opp_find_freq_ceil_indexed(raw_dev, &mut rate, index) + }, + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Floor => unsafe { + bindings::dev_pm_opp_find_freq_floor_indexed(raw_dev, &mut rate, index) + }, + })?; + + // SAFETY: The `ptr` is guaranteed by the C code to be valid. + unsafe { OPP::from_raw_opp_owned(ptr) } + } + + /// Finds [`OPP`] based on level. + pub fn opp_from_level(&self, mut level: u32, stype: SearchType) -> Result> { + let raw_dev = self.dev.as_raw(); + + let ptr = from_err_ptr(match stype { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Exact => unsafe { bindings::dev_pm_opp_find_level_exact(raw_dev, level) }, + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Ceil => unsafe { + bindings::dev_pm_opp_find_level_ceil(raw_dev, &mut level) + }, + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Floor => unsafe { + bindings::dev_pm_opp_find_level_floor(raw_dev, &mut level) + }, + })?; + + // SAFETY: The `ptr` is guaranteed by the C code to be valid. + unsafe { OPP::from_raw_opp_owned(ptr) } + } + + /// Finds [`OPP`] based on bandwidth. + pub fn opp_from_bw(&self, mut bw: u32, index: i32, stype: SearchType) -> Result> { + let raw_dev = self.dev.as_raw(); + + let ptr = from_err_ptr(match stype { + // The OPP core doesn't support this yet. + SearchType::Exact => return Err(EINVAL), + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Ceil => unsafe { + bindings::dev_pm_opp_find_bw_ceil(raw_dev, &mut bw, index) + }, + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The returned pointer will be owned by the new [`OPP`] instance. + SearchType::Floor => unsafe { + bindings::dev_pm_opp_find_bw_floor(raw_dev, &mut bw, index) + }, + })?; + + // SAFETY: The `ptr` is guaranteed by the C code to be valid. + unsafe { OPP::from_raw_opp_owned(ptr) } + } + + /// Enables the [`OPP`]. + #[inline] + pub fn enable_opp(&self, freq: Hertz) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_enable(self.dev.as_raw(), freq.into()) }) + } + + /// Disables the [`OPP`]. + #[inline] + pub fn disable_opp(&self, freq: Hertz) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { bindings::dev_pm_opp_disable(self.dev.as_raw(), freq.into()) }) + } + + /// Registers with the Energy model. + #[cfg(CONFIG_OF)] + pub fn of_register_em(&mut self, cpumask: &mut Cpumask) -> Result { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { + bindings::dev_pm_opp_of_register_em(self.dev.as_raw(), cpumask.as_raw()) + })?; + + self.em = true; + Ok(()) + } + + /// Unregisters with the Energy model. + #[cfg(all(CONFIG_OF, CONFIG_ENERGY_MODEL))] + #[inline] + fn of_unregister_em(&self) { + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. We registered with the EM framework earlier, it is safe to unregister now. + unsafe { bindings::em_dev_unregister_perf_domain(self.dev.as_raw()) }; + } +} + +impl Drop for Table { + fn drop(&mut self) { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe + // to relinquish it now. + unsafe { bindings::dev_pm_opp_put_opp_table(self.ptr) }; + + #[cfg(CONFIG_OF)] + { + #[cfg(CONFIG_ENERGY_MODEL)] + if self.em { + self.of_unregister_em(); + } + + if self.of { + self.remove_of(); + } else if let Some(cpumask) = self.cpus.take() { + self.remove_of_cpumask(&cpumask); + } + } + } +} + /// A reference-counted Operating performance point (OPP). /// /// Rust abstraction for the C `struct dev_pm_opp`. @@ -184,6 +648,27 @@ fn freq(&self) -> Hertz { /// represents a pointer that owns a reference count on the [`OPP`]. /// /// A reference to the [`OPP`], &[`OPP`], isn't refcounted by the Rust code. +/// +/// ## Examples +/// +/// The following example demonstrates how to get [`OPP`] corresponding to a frequency value and +/// configure the device with it. +/// +/// ``` +/// use kernel::clk::Hertz; +/// use kernel::error::Result; +/// use kernel::opp::{SearchType, Table}; +/// +/// fn configure_opp(table: &Table, freq: Hertz) -> Result { +/// let opp = table.opp_from_freq(freq, Some(true), None, SearchType::Exact)?; +/// +/// if opp.freq(None) != freq { +/// return Err(EINVAL); +/// } +/// +/// table.set_opp(&opp) +/// } +/// ``` #[repr(transparent)] pub struct OPP(Opaque); From ce32e2d47ce6c472a931eabe53f841c62b6abfe5 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 18 Dec 2023 11:15:13 +0530 Subject: [PATCH 27/32] rust: opp: Add abstractions for the configuration options Introduce Rust abstractions for the OPP core configuration options, enabling safe access to various configurable aspects of the OPP framework. Signed-off-by: Viresh Kumar --- rust/kernel/opp.rs | 295 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 293 insertions(+), 2 deletions(-) diff --git a/rust/kernel/opp.rs b/rust/kernel/opp.rs index 26a0ecfe046a..5293d878cb55 100644 --- a/rust/kernel/opp.rs +++ b/rust/kernel/opp.rs @@ -12,12 +12,29 @@ clk::Hertz, cpumask::{Cpumask, CpumaskVar}, device::Device, - error::{code::*, from_err_ptr, to_result, Error, Result}, + error::{code::*, from_err_ptr, from_result, to_result, Error, Result, VTABLE_DEFAULT_ERROR}, ffi::c_ulong, + prelude::*, + str::CString, types::{ARef, AlwaysRefCounted, Opaque}, }; -use core::ptr; +use core::{marker::PhantomData, ptr}; + +use macros::vtable; + +/// Creates a null-terminated slice of pointers to [`Cstring`]s. +fn to_c_str_array(names: &[CString]) -> Result> { + // Allocated a null-terminated vector of pointers. + let mut list = KVec::with_capacity(names.len() + 1, GFP_KERNEL)?; + + for name in names.iter() { + list.push(name.as_ptr() as _, GFP_KERNEL)?; + } + + list.push(ptr::null(), GFP_KERNEL)?; + Ok(list) +} /// The voltage unit. /// @@ -205,6 +222,280 @@ pub enum SearchType { Ceil, } +/// OPP configuration callbacks. +/// +/// Implement this trait to customize OPP clock and regulator setup for your device. +#[vtable] +pub trait ConfigOps { + /// This is typically used to scale clocks when transitioning between OPPs. + #[inline] + fn config_clks(_dev: &Device, _table: &Table, _opp: &OPP, _scaling_down: bool) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// This provides access to the old and new OPPs, allowing for safe regulator adjustments. + #[inline] + fn config_regulators( + _dev: &Device, + _opp_old: &OPP, + _opp_new: &OPP, + _data: *mut *mut bindings::regulator, + _count: u32, + ) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } +} + +/// OPP configuration token. +/// +/// Returned by the OPP core when configuration is applied to a [`Device`]. The associated +/// configuration is automatically cleared when the token is dropped. +pub struct ConfigToken(i32); + +impl Drop for ConfigToken { + fn drop(&mut self) { + // SAFETY: This is the same token value returned by the C code via `dev_pm_opp_set_config`. + unsafe { bindings::dev_pm_opp_clear_config(self.0) }; + } +} + +/// OPP configurations. +/// +/// Rust abstraction for the C `struct dev_pm_opp_config`. +/// +/// ## Examples +/// +/// The following example demonstrates how to set OPP property-name configuration for a [`Device`]. +/// +/// ``` +/// use kernel::device::Device; +/// use kernel::error::Result; +/// use kernel::opp::{Config, ConfigOps, ConfigToken}; +/// use kernel::str::CString; +/// use kernel::types::ARef; +/// use kernel::macros::vtable; +/// +/// #[derive(Default)] +/// struct Driver; +/// +/// #[vtable] +/// impl ConfigOps for Driver {} +/// +/// fn configure(dev: &ARef) -> Result { +/// let name = CString::try_from_fmt(fmt!("{}", "slow"))?; +/// +/// // The OPP configuration is cleared once the [`ConfigToken`] goes out of scope. +/// Config::::new() +/// .set_prop_name(name)? +/// .set(dev) +/// } +/// ``` +#[derive(Default)] +pub struct Config +where + T: Default, +{ + clk_names: Option>, + prop_name: Option, + regulator_names: Option>, + supported_hw: Option>, + + // Tuple containing (required device, index) + required_dev: Option<(ARef, u32)>, + _data: PhantomData, +} + +impl Config { + /// Creates a new instance of [`Config`]. + #[inline] + pub fn new() -> Self { + Self::default() + } + + /// Initializes clock names. + pub fn set_clk_names(mut self, names: KVec) -> Result { + if self.clk_names.is_some() { + return Err(EBUSY); + } + + if names.is_empty() { + return Err(EINVAL); + } + + self.clk_names = Some(names); + Ok(self) + } + + /// Initializes property name. + pub fn set_prop_name(mut self, name: CString) -> Result { + if self.prop_name.is_some() { + return Err(EBUSY); + } + + self.prop_name = Some(name); + Ok(self) + } + + /// Initializes regulator names. + pub fn set_regulator_names(mut self, names: KVec) -> Result { + if self.regulator_names.is_some() { + return Err(EBUSY); + } + + if names.is_empty() { + return Err(EINVAL); + } + + self.regulator_names = Some(names); + + Ok(self) + } + + /// Initializes required devices. + pub fn set_required_dev(mut self, dev: ARef, index: u32) -> Result { + if self.required_dev.is_some() { + return Err(EBUSY); + } + + self.required_dev = Some((dev, index)); + Ok(self) + } + + /// Initializes supported hardware. + pub fn set_supported_hw(mut self, hw: KVec) -> Result { + if self.supported_hw.is_some() { + return Err(EBUSY); + } + + if hw.is_empty() { + return Err(EINVAL); + } + + self.supported_hw = Some(hw); + Ok(self) + } + + /// Sets the configuration with the OPP core. + /// + /// The returned [`ConfigToken`] will remove the configuration when dropped. + pub fn set(self, dev: &Device) -> Result { + let (_clk_list, clk_names) = match &self.clk_names { + Some(x) => { + let list = to_c_str_array(x)?; + let ptr = list.as_ptr(); + (Some(list), ptr) + } + None => (None, ptr::null()), + }; + + let (_regulator_list, regulator_names) = match &self.regulator_names { + Some(x) => { + let list = to_c_str_array(x)?; + let ptr = list.as_ptr(); + (Some(list), ptr) + } + None => (None, ptr::null()), + }; + + let prop_name = self + .prop_name + .as_ref() + .map_or(ptr::null(), |p| p.as_char_ptr()); + + let (supported_hw, supported_hw_count) = self + .supported_hw + .as_ref() + .map_or((ptr::null(), 0), |hw| (hw.as_ptr(), hw.len() as u32)); + + let (required_dev, required_dev_index) = self + .required_dev + .as_ref() + .map_or((ptr::null_mut(), 0), |(dev, idx)| (dev.as_raw(), *idx)); + + let mut config = bindings::dev_pm_opp_config { + clk_names, + config_clks: if T::HAS_CONFIG_CLKS { + Some(Self::config_clks) + } else { + None + }, + prop_name, + regulator_names, + config_regulators: if T::HAS_CONFIG_REGULATORS { + Some(Self::config_regulators) + } else { + None + }, + supported_hw, + supported_hw_count, + + required_dev, + required_dev_index, + }; + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. The OPP core guarantees not to access fields of [`Config`] after this call + // and so we don't need to save a copy of them for future use. + let ret = unsafe { bindings::dev_pm_opp_set_config(dev.as_raw(), &mut config) }; + if ret < 0 { + Err(Error::from_errno(ret)) + } else { + Ok(ConfigToken(ret)) + } + } + + /// Config's clk callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn config_clks( + dev: *mut bindings::device, + opp_table: *mut bindings::opp_table, + opp: *mut bindings::dev_pm_opp, + _data: *mut kernel::ffi::c_void, + scaling_down: bool, + ) -> kernel::ffi::c_int { + from_result(|| { + // SAFETY: 'dev' is guaranteed by the C code to be valid. + let dev = unsafe { Device::get_device(dev) }; + T::config_clks( + &dev, + // SAFETY: 'opp_table' is guaranteed by the C code to be valid. + &unsafe { Table::from_raw_table(opp_table, &dev) }, + // SAFETY: 'opp' is guaranteed by the C code to be valid. + unsafe { OPP::from_raw_opp(opp)? }, + scaling_down, + ) + .map(|()| 0) + }) + } + + /// Config's regulator callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn config_regulators( + dev: *mut bindings::device, + old_opp: *mut bindings::dev_pm_opp, + new_opp: *mut bindings::dev_pm_opp, + regulators: *mut *mut bindings::regulator, + count: kernel::ffi::c_uint, + ) -> kernel::ffi::c_int { + from_result(|| { + // SAFETY: 'dev' is guaranteed by the C code to be valid. + let dev = unsafe { Device::get_device(dev) }; + T::config_regulators( + &dev, + // SAFETY: 'old_opp' is guaranteed by the C code to be valid. + unsafe { OPP::from_raw_opp(old_opp)? }, + // SAFETY: 'new_opp' is guaranteed by the C code to be valid. + unsafe { OPP::from_raw_opp(new_opp)? }, + regulators, + count, + ) + .map(|()| 0) + }) + } +} + /// A reference-counted OPP table. /// /// Rust abstraction for the C `struct opp_table`. From 2207856ff0bc8d953d6e89bda70b8978c2de8bab Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Tue, 7 Jan 2025 12:51:08 +0530 Subject: [PATCH 28/32] rust: cpufreq: Add initial abstractions for cpufreq framework Introduce initial Rust abstractions for the cpufreq core. This includes basic representations for cpufreq flags, relation types, and the cpufreq table. Signed-off-by: Viresh Kumar --- MAINTAINERS | 1 + rust/bindings/bindings_helper.h | 1 + rust/helpers/cpufreq.c | 10 + rust/helpers/helpers.c | 1 + rust/kernel/cpufreq.rs | 365 ++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 2 + 6 files changed, 380 insertions(+) create mode 100644 rust/helpers/cpufreq.c create mode 100644 rust/kernel/cpufreq.rs diff --git a/MAINTAINERS b/MAINTAINERS index 931e418f89ed..aa56eacbda71 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6142,6 +6142,7 @@ F: drivers/cpufreq/ F: include/linux/cpufreq.h F: include/linux/sched/cpufreq.h F: kernel/sched/cpufreq*.c +F: rust/kernel/cpufreq.rs F: tools/testing/selftests/cpufreq/ CPU HOTPLUG diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 529f22891e0b..7c1d78f68076 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/helpers/cpufreq.c b/rust/helpers/cpufreq.c new file mode 100644 index 000000000000..7c1343c4d65e --- /dev/null +++ b/rust/helpers/cpufreq.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#ifdef CONFIG_CPU_FREQ +void rust_helper_cpufreq_register_em_with_opp(struct cpufreq_policy *policy) +{ + cpufreq_register_em_with_opp(policy); +} +#endif diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index ae595c9cd91b..df1fcfb3adf3 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -12,6 +12,7 @@ #include "build_assert.c" #include "build_bug.c" #include "clk.c" +#include "cpufreq.c" #include "cpumask.c" #include "cred.c" #include "device.c" diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs new file mode 100644 index 000000000000..94fa63bda75a --- /dev/null +++ b/rust/kernel/cpufreq.rs @@ -0,0 +1,365 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! CPU frequency scaling. +//! +//! This module provides rust abstractions for interacting with the cpufreq subsystem. +//! +//! C header: [`include/linux/cpufreq.h`](srctree/include/linux/cpufreq.h) +//! +//! Reference: + +use crate::{ + clk::Hertz, + error::{code::*, to_result, Result}, + ffi::c_ulong, + prelude::*, + types::Opaque, +}; + +use core::{ops::Deref, pin::Pin}; + +/// Default transition latency value in nanoseconds. +pub const ETERNAL_LATENCY_NS: u32 = bindings::CPUFREQ_ETERNAL as u32; + +/// CPU frequency driver flags. +pub mod flags { + /// Driver needs to update internal limits even if frequency remains unchanged. + pub const NEED_UPDATE_LIMITS: u16 = 1 << 0; + + /// Platform where constants like `loops_per_jiffy` are unaffected by frequency changes. + pub const CONST_LOOPS: u16 = 1 << 1; + + /// Register driver as a thermal cooling device automatically. + pub const IS_COOLING_DEV: u16 = 1 << 2; + + /// Supports multiple clock domains with per-policy governors in `cpu/cpuN/cpufreq/`. + pub const HAVE_GOVERNOR_PER_POLICY: u16 = 1 << 3; + + /// Allows post-change notifications outside of the `target()` routine. + pub const ASYNC_NOTIFICATION: u16 = 1 << 4; + + /// Ensure CPU starts at a valid frequency from the driver's freq-table. + pub const NEED_INITIAL_FREQ_CHECK: u16 = 1 << 5; + + /// Disallow governors with `dynamic_switching` capability. + pub const NO_AUTO_DYNAMIC_SWITCHING: u16 = 1 << 6; +} + +/// Relations from the C code. +const CPUFREQ_RELATION_L: u32 = 0; +const CPUFREQ_RELATION_H: u32 = 1; +const CPUFREQ_RELATION_C: u32 = 2; + +/// Can be used with any of the above values. +const CPUFREQ_RELATION_E: u32 = 1 << 2; + +/// CPU frequency selection relations. +/// +/// CPU frequency selection relations, each optionally marked as "efficient". +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum Relation { + /// Select the lowest frequency at or above target. + Low(bool), + /// Select the highest frequency below or at target. + High(bool), + /// Select the closest frequency to the target. + Close(bool), +} + +impl Relation { + // Construct from a C-compatible `u32` value. + fn new(val: u32) -> Result { + let efficient = val & CPUFREQ_RELATION_E != 0; + + Ok(match val & !CPUFREQ_RELATION_E { + CPUFREQ_RELATION_L => Self::Low(efficient), + CPUFREQ_RELATION_H => Self::High(efficient), + CPUFREQ_RELATION_C => Self::Close(efficient), + _ => return Err(EINVAL), + }) + } +} + +impl From for u32 { + // Convert to a C-compatible `u32` value. + fn from(rel: Relation) -> Self { + let (mut val, efficient) = match rel { + Relation::Low(e) => (CPUFREQ_RELATION_L, e), + Relation::High(e) => (CPUFREQ_RELATION_H, e), + Relation::Close(e) => (CPUFREQ_RELATION_C, e), + }; + + if efficient { + val |= CPUFREQ_RELATION_E; + } + + val + } +} + +/// Policy data. +/// +/// Rust abstraction for the C `struct cpufreq_policy_data`. +/// +/// # Invariants +/// +/// A [`PolicyData`] instance always corresponds to a valid C `struct cpufreq_policy_data`. +/// +/// The callers must ensure that the `struct cpufreq_policy_data` is valid for access and remains +/// valid for the lifetime of the returned reference. +#[repr(transparent)] +pub struct PolicyData(Opaque); + +impl PolicyData { + /// Creates a mutable reference to an existing `struct cpufreq_policy_data` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime + /// of the returned reference. + #[inline] + pub unsafe fn from_raw_mut<'a>(ptr: *mut bindings::cpufreq_policy_data) -> &'a mut Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the + // lifetime of the returned reference. + unsafe { &mut *ptr.cast() } + } + + /// Returns a raw pointer to the underlying C `cpufreq_policy_data`. + #[inline] + pub fn as_raw(&self) -> *mut bindings::cpufreq_policy_data { + let this: *const Self = self; + this.cast_mut().cast() + } + + /// Wrapper for `cpufreq_generic_frequency_table_verify`. + #[inline] + pub fn generic_verify(&self) -> Result { + // SAFETY: By the type invariant, the pointer stored in `self` is valid. + to_result(unsafe { bindings::cpufreq_generic_frequency_table_verify(self.as_raw()) }) + } +} + +/// The frequency table index. +/// +/// Represents index with a frequency table. +/// +/// # Invariants +/// +/// The index must correspond to a valid entry in the [`Table`] it is used for. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct TableIndex(usize); + +impl TableIndex { + /// Creates an instance of [`TableIndex`]. + /// + /// # Safety + /// + /// The caller must ensure that `index` correspond to a valid entry in the [`Table`] it is used + /// for. + pub unsafe fn new(index: usize) -> Self { + // INVARIANT: The caller ensures that `index` correspond to a valid entry in the [`Table`]. + Self(index) + } +} + +impl From for usize { + #[inline] + fn from(index: TableIndex) -> Self { + index.0 + } +} + +/// CPU frequency table. +/// +/// Rust abstraction for the C `struct cpufreq_frequency_table`. +/// +/// # Invariants +/// +/// A [`Table`] instance always corresponds to a valid C `struct cpufreq_frequency_table`. +/// +/// The callers must ensure that the `struct cpufreq_frequency_table` is valid for access and +/// remains valid for the lifetime of the returned reference. +/// +/// ## Examples +/// +/// The following example demonstrates how to read a frequency value from [`Table`]. +/// +/// ``` +/// use kernel::cpufreq::{Policy, TableIndex}; +/// +/// fn show_freq(policy: &Policy) -> Result { +/// let table = policy.freq_table()?; +/// +/// // SAFETY: Index is a valid entry in the table. +/// let index = unsafe { TableIndex::new(0) }; +/// +/// pr_info!("The frequency at index 0 is: {:?}\n", table.freq(index)?); +/// pr_info!("The flags at index 0 is: {}\n", table.flags(index)); +/// pr_info!("The data at index 0 is: {}\n", table.data(index)); +/// Ok(()) +/// } +/// ``` +#[repr(transparent)] +pub struct Table(Opaque); + +impl Table { + /// Creates a reference to an existing C `struct cpufreq_frequency_table` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime + /// of the returned reference. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *const bindings::cpufreq_frequency_table) -> &'a Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the + // lifetime of the returned reference. + unsafe { &*ptr.cast() } + } + + /// Returns the raw mutable pointer to the C `struct cpufreq_frequency_table`. + #[inline] + pub fn as_raw(&self) -> *mut bindings::cpufreq_frequency_table { + let this: *const Self = self; + this.cast_mut().cast() + } + + /// Returns frequency at `index` in the [`Table`]. + #[inline] + pub fn freq(&self, index: TableIndex) -> Result { + // SAFETY: By the type invariant, the pointer stored in `self` is valid and `index` is + // guaranteed to be valid by its safety requirements. + Ok(Hertz::from_khz(unsafe { + (*self.as_raw().add(index.into())).frequency.try_into()? + })) + } + + /// Returns flags at `index` in the [`Table`]. + #[inline] + pub fn flags(&self, index: TableIndex) -> u32 { + // SAFETY: By the type invariant, the pointer stored in `self` is valid and `index` is + // guaranteed to be valid by its safety requirements. + unsafe { (*self.as_raw().add(index.into())).flags } + } + + /// Returns data at `index` in the [`Table`]. + #[inline] + pub fn data(&self, index: TableIndex) -> u32 { + // SAFETY: By the type invariant, the pointer stored in `self` is valid and `index` is + // guaranteed to be valid by its safety requirements. + unsafe { (*self.as_raw().add(index.into())).driver_data } + } +} + +/// CPU frequency table owned and pinned in memory, created from a [`TableBuilder`]. +pub struct TableBox { + entries: Pin>, +} + +impl TableBox { + /// Constructs a new [`TableBox`] from a [`KVec`] of entries. + /// + /// # Errors + /// + /// Returns `EINVAL` if the entries list is empty. + #[inline] + fn new(entries: KVec) -> Result { + if entries.is_empty() { + return Err(EINVAL); + } + + Ok(Self { + // Pin the entries to memory, since we are passing its pointer to the C code. + entries: Pin::new(entries), + }) + } + + /// Returns a raw pointer to the underlying C `cpufreq_frequency_table`. + #[inline] + fn as_raw(&self) -> *const bindings::cpufreq_frequency_table { + // The pointer is valid until the table gets dropped. + self.entries.as_ptr() + } +} + +impl Deref for TableBox { + type Target = Table; + + fn deref(&self) -> &Self::Target { + // SAFETY: The caller owns TableBox, it is safe to deref. + unsafe { Self::Target::from_raw(self.as_raw()) } + } +} + +/// CPU frequency table builder. +/// +/// This is used by the CPU frequency drivers to build a frequency table dynamically. +/// +/// ## Examples +/// +/// The following example demonstrates how to create a CPU frequency table. +/// +/// ``` +/// use kernel::cpufreq::{TableBuilder, TableIndex}; +/// use kernel::clk::Hertz; +/// +/// let mut builder = TableBuilder::new(); +/// +/// // Adds few entries to the table. +/// builder.add(Hertz::from_mhz(700), 0, 1).unwrap(); +/// builder.add(Hertz::from_mhz(800), 2, 3).unwrap(); +/// builder.add(Hertz::from_mhz(900), 4, 5).unwrap(); +/// builder.add(Hertz::from_ghz(1), 6, 7).unwrap(); +/// +/// let table = builder.to_table().unwrap(); +/// +/// // SAFETY: Index values correspond to valid entries in the table. +/// let (index0, index2) = unsafe { (TableIndex::new(0), TableIndex::new(2)) }; +/// +/// assert_eq!(table.freq(index0), Ok(Hertz::from_mhz(700))); +/// assert_eq!(table.flags(index0), 0); +/// assert_eq!(table.data(index0), 1); +/// +/// assert_eq!(table.freq(index2), Ok(Hertz::from_mhz(900))); +/// assert_eq!(table.flags(index2), 4); +/// assert_eq!(table.data(index2), 5); +/// ``` +#[derive(Default)] +#[repr(transparent)] +pub struct TableBuilder { + entries: KVec, +} + +impl TableBuilder { + /// Creates a new instance of [`TableBuilder`]. + #[inline] + pub fn new() -> Self { + Self { + entries: KVec::new(), + } + } + + /// Adds a new entry to the table. + pub fn add(&mut self, freq: Hertz, flags: u32, driver_data: u32) -> Result { + // Adds the new entry at the end of the vector. + Ok(self.entries.push( + bindings::cpufreq_frequency_table { + flags, + driver_data, + frequency: freq.as_khz() as u32, + }, + GFP_KERNEL, + )?) + } + + /// Consumes the [`TableBuilder`] and returns [`TableBox`]. + pub fn to_table(mut self) -> Result { + // Add last entry to the table. + self.add(Hertz(c_ulong::MAX), 0, 0)?; + + TableBox::new(self.entries) + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index ea589254b4ac..133ebee4f9d3 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -44,6 +44,8 @@ pub mod build_assert; pub mod clk; pub mod cpu; +#[cfg(CONFIG_CPU_FREQ)] +pub mod cpufreq; pub mod cpumask; pub mod cred; pub mod device; From 6ebdd7c9317737123b260a24e2780018008f1295 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 7 May 2025 15:32:25 +0530 Subject: [PATCH 29/32] rust: cpufreq: Extend abstractions for policy and driver ops Extend the cpufreq abstractions to include support for policy handling and driver operations. Signed-off-by: Viresh Kumar --- rust/kernel/cpufreq.rs | 463 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 461 insertions(+), 2 deletions(-) diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index 94fa63bda75a..1bc208b91822 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -10,13 +10,25 @@ use crate::{ clk::Hertz, - error::{code::*, to_result, Result}, + cpumask, + device::Device, + error::{code::*, from_err_ptr, to_result, Result, VTABLE_DEFAULT_ERROR}, ffi::c_ulong, prelude::*, + types::ForeignOwnable, types::Opaque, }; -use core::{ops::Deref, pin::Pin}; +#[cfg(CONFIG_COMMON_CLK)] +use crate::clk::Clk; + +use core::{ + ops::{Deref, DerefMut}, + pin::Pin, + ptr, +}; + +use macros::vtable; /// Default transition latency value in nanoseconds. pub const ETERNAL_LATENCY_NS: u32 = bindings::CPUFREQ_ETERNAL as u32; @@ -363,3 +375,450 @@ pub fn to_table(mut self) -> Result { TableBox::new(self.entries) } } + +/// CPU frequency policy. +/// +/// Rust abstraction for the C `struct cpufreq_policy`. +/// +/// # Invariants +/// +/// A [`Policy`] instance always corresponds to a valid C `struct cpufreq_policy`. +/// +/// The callers must ensure that the `struct cpufreq_policy` is valid for access and remains valid +/// for the lifetime of the returned reference. +/// +/// ## Examples +/// +/// The following example demonstrates how to create a CPU frequency table. +/// +/// ``` +/// use kernel::cpufreq::{ETERNAL_LATENCY_NS, Policy}; +/// +/// fn update_policy(policy: &mut Policy) { +/// policy +/// .set_dvfs_possible_from_any_cpu(true) +/// .set_fast_switch_possible(true) +/// .set_transition_latency_ns(ETERNAL_LATENCY_NS); +/// +/// pr_info!("The policy details are: {:?}\n", (policy.cpu(), policy.cur())); +/// } +/// ``` +#[repr(transparent)] +pub struct Policy(Opaque); + +impl Policy { + /// Creates a reference to an existing `struct cpufreq_policy` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime + /// of the returned reference. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *const bindings::cpufreq_policy) -> &'a Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the + // lifetime of the returned reference. + unsafe { &*ptr.cast() } + } + + /// Creates a mutable reference to an existing `struct cpufreq_policy` pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime + /// of the returned reference. + #[inline] + pub unsafe fn from_raw_mut<'a>(ptr: *mut bindings::cpufreq_policy) -> &'a mut Self { + // SAFETY: Guaranteed by the safety requirements of the function. + // + // INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the + // lifetime of the returned reference. + unsafe { &mut *ptr.cast() } + } + + /// Returns a raw mutable pointer to the C `struct cpufreq_policy`. + #[inline] + fn as_raw(&self) -> *mut bindings::cpufreq_policy { + let this: *const Self = self; + this.cast_mut().cast() + } + + #[inline] + fn as_ref(&self) -> &bindings::cpufreq_policy { + // SAFETY: By the type invariant, the pointer stored in `self` is valid. + unsafe { &*self.as_raw() } + } + + #[inline] + fn as_mut_ref(&mut self) -> &mut bindings::cpufreq_policy { + // SAFETY: By the type invariant, the pointer stored in `self` is valid. + unsafe { &mut *self.as_raw() } + } + + /// Returns the primary CPU for the [`Policy`]. + #[inline] + pub fn cpu(&self) -> u32 { + self.as_ref().cpu + } + + /// Returns the minimum frequency for the [`Policy`]. + #[inline] + pub fn min(&self) -> Hertz { + Hertz::from_khz(self.as_ref().min as usize) + } + + /// Set the minimum frequency for the [`Policy`]. + #[inline] + pub fn set_min(&mut self, min: Hertz) -> &mut Self { + self.as_mut_ref().min = min.as_khz() as u32; + self + } + + /// Returns the maximum frequency for the [`Policy`]. + #[inline] + pub fn max(&self) -> Hertz { + Hertz::from_khz(self.as_ref().max as usize) + } + + /// Set the maximum frequency for the [`Policy`]. + #[inline] + pub fn set_max(&mut self, max: Hertz) -> &mut Self { + self.as_mut_ref().max = max.as_khz() as u32; + self + } + + /// Returns the current frequency for the [`Policy`]. + #[inline] + pub fn cur(&self) -> Hertz { + Hertz::from_khz(self.as_ref().cur as usize) + } + + /// Returns the suspend frequency for the [`Policy`]. + #[inline] + pub fn suspend_freq(&self) -> Hertz { + Hertz::from_khz(self.as_ref().suspend_freq as usize) + } + + /// Sets the suspend frequency for the [`Policy`]. + #[inline] + pub fn set_suspend_freq(&mut self, freq: Hertz) -> &mut Self { + self.as_mut_ref().suspend_freq = freq.as_khz() as u32; + self + } + + /// Provides a wrapper to the generic suspend routine. + #[inline] + pub fn generic_suspend(&mut self) -> Result { + // SAFETY: By the type invariant, the pointer stored in `self` is valid. + to_result(unsafe { bindings::cpufreq_generic_suspend(self.as_mut_ref()) }) + } + + /// Provides a wrapper to the generic get routine. + #[inline] + pub fn generic_get(&self) -> Result { + // SAFETY: By the type invariant, the pointer stored in `self` is valid. + Ok(unsafe { bindings::cpufreq_generic_get(self.cpu()) }) + } + + /// Provides a wrapper to the register with energy model using the OPP core. + #[cfg(CONFIG_PM_OPP)] + #[inline] + pub fn register_em_opp(&mut self) { + // SAFETY: By the type invariant, the pointer stored in `self` is valid. + unsafe { bindings::cpufreq_register_em_with_opp(self.as_mut_ref()) }; + } + + /// Gets [`cpumask::Cpumask`] for a cpufreq [`Policy`]. + #[inline] + pub fn cpus(&mut self) -> &mut cpumask::Cpumask { + // SAFETY: The pointer to `cpus` is valid for writing and remains valid for the lifetime of + // the returned reference. + unsafe { cpumask::CpumaskVar::as_mut_ref(&mut self.as_mut_ref().cpus) } + } + + /// Sets clock for the [`Policy`]. + /// + /// # Safety + /// + /// The caller must guarantee that the returned [`Clk`] is not dropped while it is getting used + /// by the C code. + #[cfg(CONFIG_COMMON_CLK)] + pub unsafe fn set_clk(&mut self, dev: &Device, name: Option<&CStr>) -> Result { + let clk = Clk::get(dev, name)?; + self.as_mut_ref().clk = clk.as_raw(); + Ok(clk) + } + + /// Allows / disallows frequency switching code to run on any CPU. + #[inline] + pub fn set_dvfs_possible_from_any_cpu(&mut self, val: bool) -> &mut Self { + self.as_mut_ref().dvfs_possible_from_any_cpu = val; + self + } + + /// Returns if fast switching of frequencies is possible or not. + #[inline] + pub fn fast_switch_possible(&self) -> bool { + self.as_ref().fast_switch_possible + } + + /// Enables / disables fast frequency switching. + #[inline] + pub fn set_fast_switch_possible(&mut self, val: bool) -> &mut Self { + self.as_mut_ref().fast_switch_possible = val; + self + } + + /// Sets transition latency (in nanoseconds) for the [`Policy`]. + #[inline] + pub fn set_transition_latency_ns(&mut self, latency_ns: u32) -> &mut Self { + self.as_mut_ref().cpuinfo.transition_latency = latency_ns; + self + } + + /// Sets cpuinfo `min_freq`. + #[inline] + pub fn set_cpuinfo_min_freq(&mut self, min_freq: Hertz) -> &mut Self { + self.as_mut_ref().cpuinfo.min_freq = min_freq.as_khz() as u32; + self + } + + /// Sets cpuinfo `max_freq`. + #[inline] + pub fn set_cpuinfo_max_freq(&mut self, max_freq: Hertz) -> &mut Self { + self.as_mut_ref().cpuinfo.max_freq = max_freq.as_khz() as u32; + self + } + + /// Set `transition_delay_us`, i.e. the minimum time between successive frequency change + /// requests. + #[inline] + pub fn set_transition_delay_us(&mut self, transition_delay_us: u32) -> &mut Self { + self.as_mut_ref().transition_delay_us = transition_delay_us; + self + } + + /// Returns reference to the CPU frequency [`Table`] for the [`Policy`]. + pub fn freq_table(&self) -> Result<&Table> { + if self.as_ref().freq_table.is_null() { + return Err(EINVAL); + } + + // SAFETY: The `freq_table` is guaranteed to be valid for reading and remains valid for the + // lifetime of the returned reference. + Ok(unsafe { Table::from_raw(self.as_ref().freq_table) }) + } + + /// Sets the CPU frequency [`Table`] for the [`Policy`]. + /// + /// # Safety + /// + /// The caller must guarantee that the [`Table`] is not dropped while it is getting used by the + /// C code. + #[inline] + pub unsafe fn set_freq_table(&mut self, table: &Table) -> &mut Self { + self.as_mut_ref().freq_table = table.as_raw(); + self + } + + /// Returns the [`Policy`]'s private data. + pub fn data(&mut self) -> Option<::Borrowed<'_>> { + if self.as_ref().driver_data.is_null() { + None + } else { + // SAFETY: The data is earlier set from [`set_data`]. + Some(unsafe { T::borrow(self.as_ref().driver_data) }) + } + } + + /// Sets the private data of the [`Policy`] using a foreign-ownable wrapper. + /// + /// # Errors + /// + /// Returns `EBUSY` if private data is already set. + fn set_data(&mut self, data: T) -> Result { + if self.as_ref().driver_data.is_null() { + // Transfer the ownership of the data to the foreign interface. + self.as_mut_ref().driver_data = ::into_foreign(data) as _; + Ok(()) + } else { + Err(EBUSY) + } + } + + /// Clears and returns ownership of the private data. + fn clear_data(&mut self) -> Option { + if self.as_ref().driver_data.is_null() { + None + } else { + let data = Some( + // SAFETY: The data is earlier set by us from [`set_data`]. It is safe to take + // back the ownership of the data from the foreign interface. + unsafe { ::from_foreign(self.as_ref().driver_data) }, + ); + self.as_mut_ref().driver_data = ptr::null_mut(); + data + } + } +} + +/// CPU frequency policy created from a CPU number. +/// +/// This struct represents the CPU frequency policy obtained for a specific CPU, providing safe +/// access to the underlying `cpufreq_policy` and ensuring proper cleanup when the `PolicyCpu` is +/// dropped. +struct PolicyCpu<'a>(&'a mut Policy); + +impl<'a> PolicyCpu<'a> { + fn from_cpu(cpu: u32) -> Result { + // SAFETY: It is safe to call `cpufreq_cpu_get` for any valid CPU. + let ptr = from_err_ptr(unsafe { bindings::cpufreq_cpu_get(cpu) })?; + + Ok(Self( + // SAFETY: The `ptr` is guaranteed to be valid and remains valid for the lifetime of + // the returned reference. + unsafe { Policy::from_raw_mut(ptr) }, + )) + } +} + +impl<'a> Deref for PolicyCpu<'a> { + type Target = Policy; + + fn deref(&self) -> &Self::Target { + self.0 + } +} + +impl<'a> DerefMut for PolicyCpu<'a> { + fn deref_mut(&mut self) -> &mut Policy { + self.0 + } +} + +impl<'a> Drop for PolicyCpu<'a> { + fn drop(&mut self) { + // SAFETY: The underlying pointer is guaranteed to be valid for the lifetime of `self`. + unsafe { bindings::cpufreq_cpu_put(self.0.as_raw()) }; + } +} + +/// CPU frequency driver. +/// +/// Implement this trait to provide a CPU frequency driver and its callbacks. +/// +/// Reference: +#[vtable] +pub trait Driver { + /// Driver's name. + const NAME: &'static CStr; + + /// Driver's flags. + const FLAGS: u16; + + /// Boost support. + const BOOST_ENABLED: bool; + + /// Policy specific data. + /// + /// Require that `PData` implements `ForeignOwnable`. We guarantee to never move the underlying + /// wrapped data structure. + type PData: ForeignOwnable; + + /// Driver's `init` callback. + fn init(policy: &mut Policy) -> Result; + + /// Driver's `exit` callback. + fn exit(_policy: &mut Policy, _data: Option) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `online` callback. + fn online(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `offline` callback. + fn offline(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `suspend` callback. + fn suspend(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `resume` callback. + fn resume(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `ready` callback. + fn ready(_policy: &mut Policy) { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `verify` callback. + fn verify(data: &mut PolicyData) -> Result; + + /// Driver's `setpolicy` callback. + fn setpolicy(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `target` callback. + fn target(_policy: &mut Policy, _target_freq: u32, _relation: Relation) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `target_index` callback. + fn target_index(_policy: &mut Policy, _index: TableIndex) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `fast_switch` callback. + fn fast_switch(_policy: &mut Policy, _target_freq: u32) -> u32 { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `adjust_perf` callback. + fn adjust_perf(_policy: &mut Policy, _min_perf: usize, _target_perf: usize, _capacity: usize) { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `get_intermediate` callback. + fn get_intermediate(_policy: &mut Policy, _index: TableIndex) -> u32 { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `target_intermediate` callback. + fn target_intermediate(_policy: &mut Policy, _index: TableIndex) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `get` callback. + fn get(_policy: &mut Policy) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `update_limits` callback. + fn update_limits(_policy: &mut Policy) { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `bios_limit` callback. + fn bios_limit(_policy: &mut Policy, _limit: &mut u32) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `set_boost` callback. + fn set_boost(_policy: &mut Policy, _state: i32) -> Result { + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Driver's `register_em` callback. + fn register_em(_policy: &mut Policy) { + build_error!(VTABLE_DEFAULT_ERROR) + } +} From c6af9a1191d042839e56abff69e8b0302d117988 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 24 Jan 2024 12:36:33 +0530 Subject: [PATCH 30/32] rust: cpufreq: Extend abstractions for driver registration Extend the cpufreq abstractions to support driver registration from Rust. Reviewed-by: Danilo Krummrich Signed-off-by: Viresh Kumar --- rust/kernel/cpufreq.rs | 503 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 500 insertions(+), 3 deletions(-) diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index 1bc208b91822..09b856bb297b 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -11,9 +11,10 @@ use crate::{ clk::Hertz, cpumask, - device::Device, - error::{code::*, from_err_ptr, to_result, Result, VTABLE_DEFAULT_ERROR}, - ffi::c_ulong, + device::{Bound, Device}, + devres::Devres, + error::{code::*, from_err_ptr, from_result, to_result, Result, VTABLE_DEFAULT_ERROR}, + ffi::{c_char, c_ulong}, prelude::*, types::ForeignOwnable, types::Opaque, @@ -23,6 +24,9 @@ use crate::clk::Clk; use core::{ + cell::UnsafeCell, + marker::PhantomData, + mem::MaybeUninit, ops::{Deref, DerefMut}, pin::Pin, ptr, @@ -30,6 +34,9 @@ use macros::vtable; +/// Maximum length of CPU frequency driver's name. +const CPUFREQ_NAME_LEN: usize = bindings::CPUFREQ_NAME_LEN as usize; + /// Default transition latency value in nanoseconds. pub const ETERNAL_LATENCY_NS: u32 = bindings::CPUFREQ_ETERNAL as u32; @@ -822,3 +829,493 @@ fn register_em(_policy: &mut Policy) { build_error!(VTABLE_DEFAULT_ERROR) } } + +/// CPU frequency driver Registration. +/// +/// ## Examples +/// +/// The following example demonstrates how to register a cpufreq driver. +/// +/// ``` +/// use kernel::{ +/// cpufreq, +/// c_str, +/// device::{Core, Device}, +/// macros::vtable, +/// of, platform, +/// sync::Arc, +/// }; +/// struct SampleDevice; +/// +/// #[derive(Default)] +/// struct SampleDriver; +/// +/// #[vtable] +/// impl cpufreq::Driver for SampleDriver { +/// const NAME: &'static CStr = c_str!("cpufreq-sample"); +/// const FLAGS: u16 = cpufreq::flags::NEED_INITIAL_FREQ_CHECK | cpufreq::flags::IS_COOLING_DEV; +/// const BOOST_ENABLED: bool = true; +/// +/// type PData = Arc; +/// +/// fn init(policy: &mut cpufreq::Policy) -> Result { +/// // Initialize here +/// Ok(Arc::new(SampleDevice, GFP_KERNEL)?) +/// } +/// +/// fn exit(_policy: &mut cpufreq::Policy, _data: Option) -> Result { +/// Ok(()) +/// } +/// +/// fn suspend(policy: &mut cpufreq::Policy) -> Result { +/// policy.generic_suspend() +/// } +/// +/// fn verify(data: &mut cpufreq::PolicyData) -> Result { +/// data.generic_verify() +/// } +/// +/// fn target_index(policy: &mut cpufreq::Policy, index: cpufreq::TableIndex) -> Result { +/// // Update CPU frequency +/// Ok(()) +/// } +/// +/// fn get(policy: &mut cpufreq::Policy) -> Result { +/// policy.generic_get() +/// } +/// } +/// +/// impl platform::Driver for SampleDriver { +/// type IdInfo = (); +/// const OF_ID_TABLE: Option> = None; +/// +/// fn probe( +/// pdev: &platform::Device, +/// _id_info: Option<&Self::IdInfo>, +/// ) -> Result>> { +/// cpufreq::Registration::::new_foreign_owned(pdev.as_ref())?; +/// Ok(KBox::new(Self {}, GFP_KERNEL)?.into()) +/// } +/// } +/// ``` +#[repr(transparent)] +pub struct Registration(KBox>, PhantomData); + +/// SAFETY: `Registration` doesn't offer any methods or access to fields when shared between threads +/// or CPUs, so it is safe to share it. +unsafe impl Sync for Registration {} + +#[allow(clippy::non_send_fields_in_send_ty)] +/// SAFETY: Registration with and unregistration from the cpufreq subsystem can happen from any +/// thread. +unsafe impl Send for Registration {} + +impl Registration { + const VTABLE: bindings::cpufreq_driver = bindings::cpufreq_driver { + name: Self::copy_name(T::NAME), + boost_enabled: T::BOOST_ENABLED, + flags: T::FLAGS, + + // Initialize mandatory callbacks. + init: Some(Self::init_callback), + verify: Some(Self::verify_callback), + + // Initialize optional callbacks based on the traits of `T`. + setpolicy: if T::HAS_SETPOLICY { + Some(Self::setpolicy_callback) + } else { + None + }, + target: if T::HAS_TARGET { + Some(Self::target_callback) + } else { + None + }, + target_index: if T::HAS_TARGET_INDEX { + Some(Self::target_index_callback) + } else { + None + }, + fast_switch: if T::HAS_FAST_SWITCH { + Some(Self::fast_switch_callback) + } else { + None + }, + adjust_perf: if T::HAS_ADJUST_PERF { + Some(Self::adjust_perf_callback) + } else { + None + }, + get_intermediate: if T::HAS_GET_INTERMEDIATE { + Some(Self::get_intermediate_callback) + } else { + None + }, + target_intermediate: if T::HAS_TARGET_INTERMEDIATE { + Some(Self::target_intermediate_callback) + } else { + None + }, + get: if T::HAS_GET { + Some(Self::get_callback) + } else { + None + }, + update_limits: if T::HAS_UPDATE_LIMITS { + Some(Self::update_limits_callback) + } else { + None + }, + bios_limit: if T::HAS_BIOS_LIMIT { + Some(Self::bios_limit_callback) + } else { + None + }, + online: if T::HAS_ONLINE { + Some(Self::online_callback) + } else { + None + }, + offline: if T::HAS_OFFLINE { + Some(Self::offline_callback) + } else { + None + }, + exit: if T::HAS_EXIT { + Some(Self::exit_callback) + } else { + None + }, + suspend: if T::HAS_SUSPEND { + Some(Self::suspend_callback) + } else { + None + }, + resume: if T::HAS_RESUME { + Some(Self::resume_callback) + } else { + None + }, + ready: if T::HAS_READY { + Some(Self::ready_callback) + } else { + None + }, + set_boost: if T::HAS_SET_BOOST { + Some(Self::set_boost_callback) + } else { + None + }, + register_em: if T::HAS_REGISTER_EM { + Some(Self::register_em_callback) + } else { + None + }, + // SAFETY: All zeros is a valid value for `bindings::cpufreq_driver`. + ..unsafe { MaybeUninit::zeroed().assume_init() } + }; + + const fn copy_name(name: &'static CStr) -> [c_char; CPUFREQ_NAME_LEN] { + let src = name.as_bytes_with_nul(); + let mut dst = [0; CPUFREQ_NAME_LEN]; + + build_assert!(src.len() <= CPUFREQ_NAME_LEN); + + let mut i = 0; + while i < src.len() { + dst[i] = src[i]; + i += 1; + } + + dst + } + + /// Registers a CPU frequency driver with the cpufreq core. + pub fn new() -> Result { + // We can't use `&Self::VTABLE` directly because the cpufreq core modifies some fields in + // the C `struct cpufreq_driver`, which requires a mutable reference. + let mut drv = KBox::new(UnsafeCell::new(Self::VTABLE), GFP_KERNEL)?; + + // SAFETY: `drv` is guaranteed to be valid for the lifetime of `Registration`. + to_result(unsafe { bindings::cpufreq_register_driver(drv.get_mut()) })?; + + Ok(Self(drv, PhantomData)) + } + + /// Same as [`Registration::new`], but does not return a [`Registration`] instance. + /// + /// Instead the [`Registration`] is owned by [`Devres`] and will be revoked / dropped, once the + /// device is detached. + pub fn new_foreign_owned(dev: &Device) -> Result { + Devres::new_foreign_owned(dev, Self::new()?, GFP_KERNEL) + } +} + +/// CPU frequency driver callbacks. +impl Registration { + /// Driver's `init` callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn init_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + + let data = T::init(policy)?; + policy.set_data(data)?; + Ok(0) + }) + } + + /// Driver's `exit` callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn exit_callback(ptr: *mut bindings::cpufreq_policy) { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + + let data = policy.clear_data(); + let _ = T::exit(policy, data); + } + + /// Driver's `online` callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn online_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::online(policy).map(|()| 0) + }) + } + + /// Driver's `offline` callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn offline_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::offline(policy).map(|()| 0) + }) + } + + /// Driver's `suspend` callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn suspend_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::suspend(policy).map(|()| 0) + }) + } + + /// Driver's `resume` callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn resume_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::resume(policy).map(|()| 0) + }) + } + + /// Driver's `ready` callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn ready_callback(ptr: *mut bindings::cpufreq_policy) { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::ready(policy); + } + + /// Driver's `verify` callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn verify_callback(ptr: *mut bindings::cpufreq_policy_data) -> kernel::ffi::c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let data = unsafe { PolicyData::from_raw_mut(ptr) }; + T::verify(data).map(|()| 0) + }) + } + + /// Driver's `setpolicy` callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn setpolicy_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::setpolicy(policy).map(|()| 0) + }) + } + + /// Driver's `target` callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn target_callback( + ptr: *mut bindings::cpufreq_policy, + target_freq: u32, + relation: u32, + ) -> kernel::ffi::c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::target(policy, target_freq, Relation::new(relation)?).map(|()| 0) + }) + } + + /// Driver's `target_index` callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn target_index_callback( + ptr: *mut bindings::cpufreq_policy, + index: u32, + ) -> kernel::ffi::c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + + // SAFETY: The C code guarantees that `index` corresponds to a valid entry in the + // frequency table. + let index = unsafe { TableIndex::new(index as usize) }; + + T::target_index(policy, index).map(|()| 0) + }) + } + + /// Driver's `fast_switch` callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn fast_switch_callback( + ptr: *mut bindings::cpufreq_policy, + target_freq: u32, + ) -> kernel::ffi::c_uint { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::fast_switch(policy, target_freq) + } + + /// Driver's `adjust_perf` callback. + extern "C" fn adjust_perf_callback( + cpu: u32, + min_perf: usize, + target_perf: usize, + capacity: usize, + ) { + if let Ok(mut policy) = PolicyCpu::from_cpu(cpu) { + T::adjust_perf(&mut policy, min_perf, target_perf, capacity); + } + } + + /// Driver's `get_intermediate` callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn get_intermediate_callback( + ptr: *mut bindings::cpufreq_policy, + index: u32, + ) -> kernel::ffi::c_uint { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + + // SAFETY: The C code guarantees that `index` corresponds to a valid entry in the + // frequency table. + let index = unsafe { TableIndex::new(index as usize) }; + + T::get_intermediate(policy, index) + } + + /// Driver's `target_intermediate` callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn target_intermediate_callback( + ptr: *mut bindings::cpufreq_policy, + index: u32, + ) -> kernel::ffi::c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + + // SAFETY: The C code guarantees that `index` corresponds to a valid entry in the + // frequency table. + let index = unsafe { TableIndex::new(index as usize) }; + + T::target_intermediate(policy, index).map(|()| 0) + }) + } + + /// Driver's `get` callback. + extern "C" fn get_callback(cpu: u32) -> kernel::ffi::c_uint { + PolicyCpu::from_cpu(cpu).map_or(0, |mut policy| T::get(&mut policy).map_or(0, |f| f)) + } + + /// Driver's `update_limit` callback. + extern "C" fn update_limits_callback(ptr: *mut bindings::cpufreq_policy) { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::update_limits(policy); + } + + /// Driver's `bios_limit` callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn bios_limit_callback(cpu: i32, limit: *mut u32) -> kernel::ffi::c_int { + from_result(|| { + let mut policy = PolicyCpu::from_cpu(cpu as u32)?; + + // SAFETY: `limit` is guaranteed by the C code to be valid. + T::bios_limit(&mut policy, &mut (unsafe { *limit })).map(|()| 0) + }) + } + + /// Driver's `set_boost` callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn set_boost_callback( + ptr: *mut bindings::cpufreq_policy, + state: i32, + ) -> kernel::ffi::c_int { + from_result(|| { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::set_boost(policy, state).map(|()| 0) + }) + } + + /// Driver's `register_em` callback. + /// + /// SAFETY: Called from C. Inputs must be valid pointers. + extern "C" fn register_em_callback(ptr: *mut bindings::cpufreq_policy) { + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::register_em(policy); + } +} + +impl Drop for Registration { + /// Unregisters with the cpufreq core. + fn drop(&mut self) { + // SAFETY: `self.0` is guaranteed to be valid for the lifetime of `Registration`. + unsafe { bindings::cpufreq_unregister_driver(self.0.get_mut()) }; + } +} From 14f47156cf390606eb719da9ad1058f87af0a291 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Tue, 7 Jan 2025 12:53:32 +0530 Subject: [PATCH 31/32] rust: opp: Extend OPP abstractions with cpufreq support Extend the OPP abstractions to include support for interacting with the cpufreq core, including the ability to retrieve frequency tables from OPP table. Signed-off-by: Viresh Kumar --- rust/kernel/opp.rs | 70 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/rust/kernel/opp.rs b/rust/kernel/opp.rs index 5293d878cb55..212555dacd45 100644 --- a/rust/kernel/opp.rs +++ b/rust/kernel/opp.rs @@ -19,6 +19,69 @@ types::{ARef, AlwaysRefCounted, Opaque}, }; +#[cfg(CONFIG_CPU_FREQ)] +/// Frequency table implementation. +mod freq { + use super::*; + use crate::cpufreq; + use core::ops::Deref; + + /// OPP frequency table. + /// + /// A [`cpufreq::Table`] created from [`Table`]. + pub struct FreqTable { + dev: ARef, + ptr: *mut bindings::cpufreq_frequency_table, + } + + impl FreqTable { + /// Creates a new instance of [`FreqTable`] from [`Table`]. + pub(crate) fn new(table: &Table) -> Result { + let mut ptr: *mut bindings::cpufreq_frequency_table = ptr::null_mut(); + + // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety + // requirements. + to_result(unsafe { + bindings::dev_pm_opp_init_cpufreq_table(table.dev.as_raw(), &mut ptr) + })?; + + Ok(Self { + dev: table.dev.clone(), + ptr, + }) + } + + /// Returns a reference to the underlying [`cpufreq::Table`]. + #[inline] + fn table(&self) -> &cpufreq::Table { + // SAFETY: The `ptr` is guaranteed by the C code to be valid. + unsafe { cpufreq::Table::from_raw(self.ptr) } + } + } + + impl Deref for FreqTable { + type Target = cpufreq::Table; + + #[inline] + fn deref(&self) -> &Self::Target { + self.table() + } + } + + impl Drop for FreqTable { + fn drop(&mut self) { + // SAFETY: The pointer was created via `dev_pm_opp_init_cpufreq_table`, and is only + // freed here. + unsafe { + bindings::dev_pm_opp_free_cpufreq_table(self.dev.as_raw(), &mut self.as_raw()) + }; + } + } +} + +#[cfg(CONFIG_CPU_FREQ)] +pub use freq::FreqTable; + use core::{marker::PhantomData, ptr}; use macros::vtable; @@ -753,6 +816,13 @@ pub fn adjust_voltage( }) } + /// Creates [`FreqTable`] from [`Table`]. + #[cfg(CONFIG_CPU_FREQ)] + #[inline] + pub fn cpufreq_table(&mut self) -> Result { + FreqTable::new(self) + } + /// Configures device with [`OPP`] matching the frequency value. #[inline] pub fn set_rate(&self, freq: Hertz) -> Result { From 06149d8f2216894cee86106c701d13141948f159 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 1 Feb 2024 10:36:24 +0530 Subject: [PATCH 32/32] cpufreq: Add Rust-based cpufreq-dt driver Introduce a Rust-based implementation of the cpufreq-dt driver, covering most of the functionality provided by the existing C version. Some features, such as retrieving platform data from `cpufreq-dt-platdev.c`, are still pending. The driver has been tested with QEMU, and frequency scaling works as expected. Signed-off-by: Viresh Kumar --- drivers/cpufreq/Kconfig | 12 ++ drivers/cpufreq/Makefile | 1 + drivers/cpufreq/rcpufreq_dt.rs | 226 +++++++++++++++++++++++++++++++++ 3 files changed, 239 insertions(+) create mode 100644 drivers/cpufreq/rcpufreq_dt.rs diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index d64b07ec48e5..78702a08364f 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -217,6 +217,18 @@ config CPUFREQ_DT If in doubt, say N. +config CPUFREQ_DT_RUST + tristate "Rust based Generic DT based cpufreq driver" + depends on HAVE_CLK && OF && RUST + select CPUFREQ_DT_PLATDEV + select PM_OPP + help + This adds a Rust based generic DT based cpufreq driver for frequency + management. It supports both uniprocessor (UP) and symmetric + multiprocessor (SMP) systems. + + If in doubt, say N. + config CPUFREQ_VIRT tristate "Virtual cpufreq driver" depends on GENERIC_ARCH_TOPOLOGY diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index 22ab45209f9b..d38526b8e063 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -15,6 +15,7 @@ obj-$(CONFIG_CPU_FREQ_GOV_COMMON) += cpufreq_governor.o obj-$(CONFIG_CPU_FREQ_GOV_ATTR_SET) += cpufreq_governor_attr_set.o obj-$(CONFIG_CPUFREQ_DT) += cpufreq-dt.o +obj-$(CONFIG_CPUFREQ_DT_RUST) += rcpufreq_dt.o obj-$(CONFIG_CPUFREQ_DT_PLATDEV) += cpufreq-dt-platdev.o obj-$(CONFIG_CPUFREQ_VIRT) += virtual-cpufreq.o diff --git a/drivers/cpufreq/rcpufreq_dt.rs b/drivers/cpufreq/rcpufreq_dt.rs new file mode 100644 index 000000000000..94ed81644fe1 --- /dev/null +++ b/drivers/cpufreq/rcpufreq_dt.rs @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust based implementation of the cpufreq-dt driver. + +use kernel::{ + c_str, + clk::Clk, + cpu, cpufreq, + cpumask::CpumaskVar, + device::{Core, Device}, + error::code::*, + fmt, + macros::vtable, + module_platform_driver, of, opp, platform, + prelude::*, + str::CString, + sync::Arc, +}; + +/// Finds exact supply name from the OF node. +fn find_supply_name_exact(dev: &Device, name: &str) -> Option { + let prop_name = CString::try_from_fmt(fmt!("{}-supply", name)).ok()?; + dev.property_present(&prop_name) + .then(|| CString::try_from_fmt(fmt!("{name}")).ok()) + .flatten() +} + +/// Finds supply name for the CPU from DT. +fn find_supply_names(dev: &Device, cpu: u32) -> Option> { + // Try "cpu0" for older DTs, fallback to "cpu". + let name = (cpu == 0) + .then(|| find_supply_name_exact(dev, "cpu0")) + .flatten() + .or_else(|| find_supply_name_exact(dev, "cpu"))?; + + let mut list = KVec::with_capacity(1, GFP_KERNEL).ok()?; + list.push(name, GFP_KERNEL).ok()?; + + Some(list) +} + +/// Represents the cpufreq dt device. +struct CPUFreqDTDevice { + opp_table: opp::Table, + freq_table: opp::FreqTable, + _mask: CpumaskVar, + _token: Option, + _clk: Clk, +} + +#[derive(Default)] +struct CPUFreqDTDriver; + +#[vtable] +impl opp::ConfigOps for CPUFreqDTDriver {} + +#[vtable] +impl cpufreq::Driver for CPUFreqDTDriver { + const NAME: &'static CStr = c_str!("cpufreq-dt"); + const FLAGS: u16 = cpufreq::flags::NEED_INITIAL_FREQ_CHECK | cpufreq::flags::IS_COOLING_DEV; + const BOOST_ENABLED: bool = true; + + type PData = Arc; + + fn init(policy: &mut cpufreq::Policy) -> Result { + let cpu = policy.cpu(); + // SAFETY: The CPU device is only used during init; it won't get hot-unplugged. The cpufreq + // core registers with CPU notifiers and the cpufreq core/driver won't use the CPU device, + // once the CPU is hot-unplugged. + let dev = unsafe { cpu::from_cpu(cpu)? }; + let mut mask = CpumaskVar::new_zero(GFP_KERNEL)?; + + mask.set(cpu); + + let token = find_supply_names(dev, cpu) + .map(|names| { + opp::Config::::new() + .set_regulator_names(names)? + .set(dev) + }) + .transpose()?; + + // Get OPP-sharing information from "operating-points-v2" bindings. + let fallback = match opp::Table::of_sharing_cpus(dev, &mut mask) { + Ok(()) => false, + Err(e) if e == ENOENT => { + // "operating-points-v2" not supported. If the platform hasn't + // set sharing CPUs, fallback to all CPUs share the `Policy` + // for backward compatibility. + opp::Table::sharing_cpus(dev, &mut mask).is_err() + } + Err(e) => return Err(e), + }; + + // Initialize OPP tables for all policy cpus. + // + // For platforms not using "operating-points-v2" bindings, we do this + // before updating policy cpus. Otherwise, we will end up creating + // duplicate OPPs for the CPUs. + // + // OPPs might be populated at runtime, don't fail for error here unless + // it is -EPROBE_DEFER. + let mut opp_table = match opp::Table::from_of_cpumask(dev, &mut mask) { + Ok(table) => table, + Err(e) => { + if e == EPROBE_DEFER { + return Err(e); + } + + // The table is added dynamically ? + opp::Table::from_dev(dev)? + } + }; + + // The OPP table must be initialized, statically or dynamically, by this point. + opp_table.opp_count()?; + + // Set sharing cpus for fallback scenario. + if fallback { + mask.setall(); + opp_table.set_sharing_cpus(&mut mask)?; + } + + let mut transition_latency = opp_table.max_transition_latency_ns() as u32; + if transition_latency == 0 { + transition_latency = cpufreq::ETERNAL_LATENCY_NS; + } + + policy + .set_dvfs_possible_from_any_cpu(true) + .set_suspend_freq(opp_table.suspend_freq()) + .set_transition_latency_ns(transition_latency); + + let freq_table = opp_table.cpufreq_table()?; + // SAFETY: The `freq_table` is not dropped while it is getting used by the C code. + unsafe { policy.set_freq_table(&freq_table) }; + + // SAFETY: The returned `clk` is not dropped while it is getting used by the C code. + let clk = unsafe { policy.set_clk(dev, None)? }; + + mask.copy(policy.cpus()); + + Ok(Arc::new( + CPUFreqDTDevice { + opp_table, + freq_table, + _mask: mask, + _token: token, + _clk: clk, + }, + GFP_KERNEL, + )?) + } + + fn exit(_policy: &mut cpufreq::Policy, _data: Option) -> Result { + Ok(()) + } + + fn online(_policy: &mut cpufreq::Policy) -> Result { + // We did light-weight tear down earlier, nothing to do here. + Ok(()) + } + + fn offline(_policy: &mut cpufreq::Policy) -> Result { + // Preserve policy->data and don't free resources on light-weight + // tear down. + Ok(()) + } + + fn suspend(policy: &mut cpufreq::Policy) -> Result { + policy.generic_suspend() + } + + fn verify(data: &mut cpufreq::PolicyData) -> Result { + data.generic_verify() + } + + fn target_index(policy: &mut cpufreq::Policy, index: cpufreq::TableIndex) -> Result { + let Some(data) = policy.data::() else { + return Err(ENOENT); + }; + + let freq = data.freq_table.freq(index)?; + data.opp_table.set_rate(freq) + } + + fn get(policy: &mut cpufreq::Policy) -> Result { + policy.generic_get() + } + + fn set_boost(_policy: &mut cpufreq::Policy, _state: i32) -> Result { + Ok(()) + } + + fn register_em(policy: &mut cpufreq::Policy) { + policy.register_em_opp() + } +} + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + ::IdInfo, + [(of::DeviceId::new(c_str!("operating-points-v2")), ())] +); + +impl platform::Driver for CPUFreqDTDriver { + type IdInfo = (); + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe( + pdev: &platform::Device, + _id_info: Option<&Self::IdInfo>, + ) -> Result>> { + cpufreq::Registration::::new_foreign_owned(pdev.as_ref())?; + Ok(KBox::new(Self {}, GFP_KERNEL)?.into()) + } +} + +module_platform_driver! { + type: CPUFreqDTDriver, + name: "cpufreq-dt", + author: "Viresh Kumar ", + description: "Generic CPUFreq DT driver", + license: "GPL v2", +}