github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-10-22 12:52:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Driver core changes for 7.4-rc1

Browse Source 
- core:
 
   - Add TAINT_FORCED_BIND to taint the kernel and driver module before manual
     sysfs bind/unbind operations; make add_taint_module() available to
     driver core and update taint decoding tools and documentation
 
   - Constify device_show/store_*() attribute parameters and adapt x86 MCE and
     perf users to prepare for const dev_ext_attribute instances
 
   - Fix UAF in NUMA access node registration failure handling by dropping
     the device reference with put_device() instead of directly freeing the node
 
 - debugfs:
 
   - Make debugfs_create_str() read-only to eliminate double-free races between
     concurrent writers; reject writable modes; convert interconnect and
     SoundWire users to local file operations with mutex-protected strings
 
 - kobject:
 
   - Fix out-of-bounds access in kobject_action_type() when parsing action
     strings containing embedded NUL bytes
 
   - Fix the kobject_set_name() reference in kset_register() kernel-doc
 
 - Rust:
 
   - DMA:
 
     - Tie Coherent, CoherentBox and CoherentHandle to the device's bound
       lifetime to prevent retaining DMA allocations beyond driver unbind;
       propagate the lifetime through nova-core and the DMA sample driver
 
   - I/O:
 
     - Require explicit base types for registers and replace relative registers
       with typed subregion projections; reimplement register!() as a proc
       macro, support registers using existing types without inline bitfields,
       and convert nova-core, Tyr and the PCI sample; add AsRepr/AsReprMut and
       transmute helpers with deferred size checks for representation
       conversions
 
     - Wrap ResourceSize in a transparent newtype with explicit conversions,
       including checked conversion to usize
 
   - PCI:
 
     - Add typed PCIe extended capability lookup and SR-IOV accessors for the
       First VF Offset and decoded VF BAR addresses and widths, keeping raw
       SR-IOV registers private to preserve PCI core ownership of their state
 
   - scatterlist:
 
     - Honor the device's maximum segment size as well as the DMA mapping limit
       when constructing SG tables
 
     - Return u32 from SGEntry::dma_len() to match the C segment length type
 
   - misc:
 
     - Validate auxiliary DeviceId name length explicitly to improve
       compile-time diagnostics for oversized device ID table entries
 
     - Drop unnecessary 'static bounds from debugfs ScopedDir file creation
       methods to support data containing non-'static references, including DMA
       buffers
 
 - misc:
 
   - Add Johan Hovold as driver core maintainer
 
   - Simplify platform_device_alloc() by returning early on allocation failure
 
   - Fix typos in driver core and kernfs comments
 -----BEGIN PGP SIGNATURE-----
 
 iJEEABYKADkWIQS2q/xV6QjXAdC7k+1FlHeO1qrKLgUCatNacRsUgAAAAAAEAA5t
 YW51MiwyLjUrMS4xMiwwLDMACgkQRZR3jtaqyi7YegD8ChAjvp7WireJHjodlAMA
 j2Ty0NLjR8715/Ml/QNxnTUA/1/Auj3pXS5/VesKPp+PU6VVB4IdjjHzuzSY+7sz
 y7wN
 =GYmP
 -----END PGP SIGNATURE-----

Merge tag 'driver-core-7.4-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core

Pull driver core updates from Danilo Krummrich:
 "Core:

   - Add TAINT_FORCED_BIND to taint the kernel and driver module before
     manual sysfs bind/unbind operations; make add_taint_module()
     available to driver core and update taint decoding tools and
     documentation

   - Constify device_show/store_*() attribute parameters and adapt x86
     MCE and perf users to prepare for const dev_ext_attribute instances

   - Fix UAF in NUMA access node registration failure handling by
     dropping the device reference with put_device() instead of directly
     freeing the node

  debugfs:

   - Make debugfs_create_str() read-only to eliminate double-free races
     between concurrent writers; reject writable modes; convert
     interconnect and SoundWire users to local file operations with
     mutex-protected strings

  kobject:

   - Fix out-of-bounds access in kobject_action_type() when parsing
     action strings containing embedded NUL bytes

   - Fix the kobject_set_name() reference in kset_register() kernel-doc

  Rust:

   - DMA:

      - Tie Coherent, CoherentBox and CoherentHandle to the device's
        bound lifetime to prevent retaining DMA allocations beyond
        driver unbind; propagate the lifetime through nova-core and the
        DMA sample driver

   - I/O:

      - Require explicit base types for registers and replace relative
        registers with typed subregion projections; reimplement
        register!() as a proc macro, support registers using existing
        types without inline bitfields, and convert nova-core, Tyr and
        the PCI sample; add AsRepr/AsReprMut and transmute helpers with
        deferred size checks for representation conversions

      - Wrap ResourceSize in a transparent newtype with explicit
        conversions, including checked conversion to usize

   - PCI:

      - Add typed PCIe extended capability lookup and SR-IOV accessors
        for the First VF Offset and decoded VF BAR addresses and widths,
        keeping raw SR-IOV registers private to preserve PCI core
        ownership of their state

   - scatterlist:

      - Honor the device's maximum segment size as well as the DMA
        mapping limit when constructing SG tables

      - Return u32 from SGEntry::dma_len() to match the C segment length
        type

   - misc:

      - Validate auxiliary DeviceId name length explicitly to improve
        compile-time diagnostics for oversized device ID table entries

      - Drop unnecessary 'static bounds from debugfs ScopedDir file
        creation methods to support data containing non-'static
        references, including DMA buffers

  Misc:

   - Add Johan Hovold as driver core maintainer

   - Simplify platform_device_alloc() by returning early on allocation
     failure

   - Fix typos in driver core and kernfs comments"

* tag 'driver-core-7.4-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core: (41 commits)
  rust: pci: add extended capability and SR-IOV support
  kobject: Fix the kobj_set_name reference in kset_register() kernel-doc
  rust: io: convert ResourceSize into a transparent newtype
  rust: scatterlist: return u32 from SGEntry::dma_len()
  MAINTAINERS: add myself as driver core maintainer
  debugfs: make debugfs_create_str() read-only
  soundwire: debugfs: replace writable string helper
  interconnect: debugfs: replace writable string helper
  docs: admin-guide: Handle TAINT_FORCED_BIND when parsing /proc/sys/kernel/tainted
  driver core: platform: Simplify platform_device_alloc()
  kobject: fix out-of-bounds access in kobject_action_type()
  rust: scatterlist: honor the device's maximum segment size
  driver core: add TAINT_FORCED_BIND for when userspace manually messes with devices and drivers
  module: fix up documentation for add_taint() and add_taint_module()
  module: pull out add_taint_module() to be public
  driver core: Constify the signature of device_show/store_*()
  drivers: perf: Use __DEVICE_ATTR() macro to initialize dev_ext_attribute
  x86/mce: Use __DEVICE_ATTR() macro to initialize dev_ext_attribute
  rust: auxiliary: validate DeviceId name length
  driver core: fix typos in comments
  ...
This commit is contained in:
Linus Torvalds 2026-10-21 18:53:14 -10:00
commit 7be4bc52fd
96 changed files with 2170 additions and 1449 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
Documentation/admin-guide/tainted-kernels.rst
View File

@ -74,35 +74,36 @@ a particular type of taint. It's best to leave that to the aforementioned
script, but if you need something quick you can use this shell command to check
which bits are set::
$ for i in $(seq 20); do echo $(($i-1)) $(($(cat /proc/sys/kernel/tainted)>>($i-1)&1));done
$ for i in $(seq 0 20); do echo $i $(($(cat /proc/sys/kernel/tainted)>>$i&1));done
Table for decoding tainted state
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
=== === ====== ========================================================
Bit Log Number Reason that got the kernel tainted
=== === ====== ========================================================
0 G/P 1 proprietary module was loaded
1 _/F 2 module was force loaded
2 _/S 4 kernel running on an out of specification system
3 _/R 8 module was force unloaded
4 _/M 16 processor reported a Machine Check Exception (MCE)
5 _/B 32 bad page referenced or some unexpected page flags
6 _/U 64 taint requested by userspace application
7 _/D 128 kernel died recently, i.e. there was an OOPS or BUG
8 _/A 256 ACPI table overridden by user
9 _/W 512 kernel issued warning
10 _/C 1024 staging driver was loaded
11 _/I 2048 workaround for bug in platform firmware applied
12 _/O 4096 externally-built ("out-of-tree") module was loaded
13 _/E 8192 unsigned module was loaded
14 _/L 16384 soft lockup occurred
15 _/K 32768 kernel has been live patched
16 _/X 65536 auxiliary taint, defined for and used by distros
17 _/T 131072 kernel was built with the struct randomization plugin
18 _/N 262144 an in-kernel test has been run
19 _/J 524288 userspace used a mutating debug operation in fwctl
=== === ====== ========================================================
=== === ======= ========================================================
Bit Log Number Reason that got the kernel tainted
=== === ======= ========================================================
0 G/P 1 proprietary module was loaded
1 _/F 2 module was force loaded
2 _/S 4 kernel running on an out of specification system
3 _/R 8 module was force unloaded
4 _/M 16 processor reported a Machine Check Exception (MCE)
5 _/B 32 bad page referenced or some unexpected page flags
6 _/U 64 taint requested by userspace application
7 _/D 128 kernel died recently, i.e. there was an OOPS or BUG
8 _/A 256 ACPI table overridden by user
9 _/W 512 kernel issued warning
10 _/C 1024 staging driver was loaded
11 _/I 2048 workaround for bug in platform firmware applied
12 _/O 4096 externally-built ("out-of-tree") module was loaded
13 _/E 8192 unsigned module was loaded
14 _/L 16384 soft lockup occurred
15 _/K 32768 kernel has been live patched
16 _/X 65536 auxiliary taint, defined for and used by distros
17 _/T 131072 kernel was built with the struct randomization plugin
18 _/N 262144 an in-kernel test has been run
19 _/J 524288 userspace used a mutating debug operation in fwctl
20 _/Y 1048576 device was manually bound or unbound from a driver
=== === ======= ========================================================
Note: The character ``_`` is representing a blank in this table to make reading
easier.
@ -189,3 +190,6 @@ More detailed explanation for tainting
19) ``J`` if userspace opened /dev/fwctl/* and performed a FWTCL_RPC_DEBUG_WRITE
to use the devices debugging features. Device debugging features could
cause the device to malfunction in undefined ways.
20) ``Y`` If userspace wrote to the `bind` or `unbind` sysfs files and
successfully bound or removed a device from a driver.

2
MAINTAINERS
View File

@ -7526,6 +7526,7 @@ F: rust/kernel/io.rs
F: rust/kernel/io/
F: rust/kernel/irq.rs
F: rust/kernel/irq/
F: rust/macros/io/
DEVICE RESOURCE MANAGEMENT HELPERS
M: Hans de Goede <hansg@kernel.org>
@ -7988,6 +7989,7 @@ DRIVER CORE, KOBJECTS, DEBUGFS AND SYSFS
M: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
M: "Rafael J. Wysocki" <rafael@kernel.org>
M: Danilo Krummrich <dakr@kernel.org>
M: Johan Hovold <johan@kernel.org>
L: driver-core@lists.linux.dev
S: Supported
T: git git://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git

2
arch/powerpc/perf/hv-24x7.c
View File

@ -462,7 +462,7 @@ static struct attribute *device_str_attr_create_(char *name, char *str)
attr->var = str;
attr->attr.attr.name = name;
attr->attr.attr.mode = 0444;
attr->attr.show = device_show_string;
attr->attr.show_const = device_show_string;
return &attr->attr.attr;
}

6
arch/x86/kernel/cpu/mce/core.c
View File

@ -2614,17 +2614,17 @@ static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce);
static DEVICE_BOOL_ATTR(print_all, 0644, mca_cfg.print_all);
static struct dev_ext_attribute dev_attr_check_interval = {
__ATTR(check_interval, 0644, device_show_int, store_int_with_restart),
__DEVICE_ATTR(check_interval, 0644, device_show_int, store_int_with_restart),
&check_interval
};
static struct dev_ext_attribute dev_attr_ignore_ce = {
__ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce),
__DEVICE_ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce),
&mca_cfg.ignore_ce
};
static struct dev_ext_attribute dev_attr_cmci_disabled = {
__ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled),
__DEVICE_ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled),
&mca_cfg.cmci_disabled
};

3
drivers/base/bus.c
View File

@ -19,6 +19,7 @@
#include <linux/string.h>
#include <linux/mutex.h>
#include <linux/sysfs.h>
#include <linux/panic.h>
#include "base.h"
#include "power/power.h"
@ -241,6 +242,7 @@ static ssize_t unbind_store(struct device_driver *drv, const char *buf,
dev = bus_find_device_by_name(bus, NULL, buf);
if (dev && dev->driver == drv) {
add_taint_module(drv->owner, TAINT_FORCED_BIND, LOCKDEP_STILL_OK);
device_driver_detach(dev);
err = count;
}
@ -264,6 +266,7 @@ static ssize_t bind_store(struct device_driver *drv, const char *buf,
dev = bus_find_device_by_name(bus, NULL, buf);
if (dev && driver_match_device(drv, dev)) {
add_taint_module(drv->owner, TAINT_FORCED_BIND, LOCKDEP_STILL_OK);
err = device_driver_attach(drv, dev);
if (!err) {
/* success */

30
drivers/base/core.c
View File

@ -2517,13 +2517,13 @@ static const struct sysfs_ops dev_sysfs_ops = {
.store = dev_attr_store,
};
#define to_ext_attr(x) container_of(x, struct dev_ext_attribute, attr)
#define to_ext_attr(x) container_of_const(x, struct dev_ext_attribute, attr)
ssize_t device_store_ulong(struct device *dev,
struct device_attribute *attr,
const struct device_attribute *attr,
const char *buf, size_t size)
{
struct dev_ext_attribute *ea = to_ext_attr(attr);
const struct dev_ext_attribute *ea = to_ext_attr(attr);
int ret;
unsigned long new;
@ -2537,19 +2537,19 @@ ssize_t device_store_ulong(struct device *dev,
EXPORT_SYMBOL_GPL(device_store_ulong);
ssize_t device_show_ulong(struct device *dev,
struct device_attribute *attr,
const struct device_attribute *attr,
char *buf)
{
struct dev_ext_attribute *ea = to_ext_attr(attr);
const struct dev_ext_attribute *ea = to_ext_attr(attr);
return sysfs_emit(buf, "%lx\n", *(unsigned long *)(ea->var));
}
EXPORT_SYMBOL_GPL(device_show_ulong);
ssize_t device_store_int(struct device *dev,
struct device_attribute *attr,
const struct device_attribute *attr,
const char *buf, size_t size)
{
struct dev_ext_attribute *ea = to_ext_attr(attr);
const struct dev_ext_attribute *ea = to_ext_attr(attr);
int ret;
long new;
@ -2566,19 +2566,19 @@ ssize_t device_store_int(struct device *dev,
EXPORT_SYMBOL_GPL(device_store_int);
ssize_t device_show_int(struct device *dev,
struct device_attribute *attr,
const struct device_attribute *attr,
char *buf)
{
struct dev_ext_attribute *ea = to_ext_attr(attr);
const struct dev_ext_attribute *ea = to_ext_attr(attr);
return sysfs_emit(buf, "%d\n", *(int *)(ea->var));
}
EXPORT_SYMBOL_GPL(device_show_int);
ssize_t device_store_bool(struct device *dev, struct device_attribute *attr,
ssize_t device_store_bool(struct device *dev, const struct device_attribute *attr,
const char *buf, size_t size)
{
struct dev_ext_attribute *ea = to_ext_attr(attr);
const struct dev_ext_attribute *ea = to_ext_attr(attr);
if (kstrtobool(buf, ea->var) < 0)
return -EINVAL;
@ -2587,19 +2587,19 @@ ssize_t device_store_bool(struct device *dev, struct device_attribute *attr,
}
EXPORT_SYMBOL_GPL(device_store_bool);
ssize_t device_show_bool(struct device *dev, struct device_attribute *attr,
ssize_t device_show_bool(struct device *dev, const struct device_attribute *attr,
char *buf)
{
struct dev_ext_attribute *ea = to_ext_attr(attr);
const struct dev_ext_attribute *ea = to_ext_attr(attr);
return sysfs_emit(buf, "%d\n", *(bool *)(ea->var));
}
EXPORT_SYMBOL_GPL(device_show_bool);
ssize_t device_show_string(struct device *dev,
struct device_attribute *attr, char *buf)
const struct device_attribute *attr, char *buf)
{
struct dev_ext_attribute *ea = to_ext_attr(attr);
const struct dev_ext_attribute *ea = to_ext_attr(attr);
return sysfs_emit(buf, "%s\n", (char *)ea->var);
}

17
drivers/base/node.c
View File

@ -167,20 +167,19 @@ static struct node_access_nodes *node_init_node_access(struct node *node,
dev->parent = &node->dev;
dev->release = node_access_release;
dev->groups = node_access_node_groups;
if (dev_set_name(dev, "access%u", access))
goto free;
if (dev_set_name(dev, "access%u", access)) {
kfree(access_node);
return NULL;
}
if (device_register(dev))
goto free_name;
if (device_register(dev)) {
put_device(dev);
return NULL;
}
pm_runtime_no_callbacks(dev);
list_add_tail(&access_node->list_node, &node->access_list);
return access_node;
free_name:
kfree_const(dev->kobj.name);
free:
kfree(access_node);
return NULL;
}
#ifdef CONFIG_HMEM_REPORTING

19
drivers/base/platform.c
View File

@ -632,16 +632,17 @@ struct platform_device *platform_device_alloc(const char *name, int id)
struct platform_object *pa;
pa = kzalloc(sizeof(*pa) + strlen(name) + 1, GFP_KERNEL);
if (pa) {
strcpy(pa->name, name);
pa->pdev.name = pa->name;
pa->pdev.id = id;
device_initialize(&pa->pdev.dev);
pa->pdev.dev.release = platform_device_release;
setup_pdev_dma_masks(&pa->pdev);
}
if (!pa)
return NULL;
return pa ? &pa->pdev : NULL;
strcpy(pa->name, name);
pa->pdev.name = pa->name;
pa->pdev.id = id;
device_initialize(&pa->pdev.dev);
pa->pdev.dev.release = platform_device_release;
setup_pdev_dma_masks(&pa->pdev);
return &pa->pdev;
}
EXPORT_SYMBOL_GPL(platform_device_alloc);

2
drivers/base/property.c
View File

@ -705,7 +705,7 @@ bool fwnode_name_eq(const struct fwnode_handle *fwnode, const char *name)
EXPORT_SYMBOL_GPL(fwnode_name_eq);
/**
* fwnode_get_parent - Return parent firwmare node
* fwnode_get_parent - Return parent firmware node
* @fwnode: Firmware whose parent is retrieved
*
* The caller is responsible for calling fwnode_handle_put() on the returned

2
drivers/base/transport_class.c
View File

@ -7,7 +7,7 @@
*
* The basic idea here is to allow any "device controller" (which
* would most often be a Host Bus Adapter to use the services of one
* or more tranport classes for performing transport specific
* or more transport classes for performing transport specific
* services. Transport specific services are things that the generic
* command layer doesn't want to know about (speed settings, line
* condidtioning, etc), but which the user might be interested in.

1
drivers/gpu/drm/tyr/driver.rs
View File

@ -46,6 +46,7 @@
};
pub(crate) type IoMem<'a> = kernel::io::mem::IoMem<'a, SZ_2M>;
pub(crate) type TyrRegisters = kernel::io::Region<SZ_2M>;
pub(crate) struct TyrDrmDriver;

5
drivers/gpu/drm/tyr/fw.rs
View File

@ -39,7 +39,8 @@
use crate::{
driver::{
IoMem,
TyrDrmDevice, //
TyrDrmDevice,
TyrRegisters, //
},
fw::parser::{
FwParser,
@ -101,6 +102,8 @@ fn from(value: CacheMode) -> Self {
}
register! {
base: TyrRegisters;
#[allow(non_upper_case_globals)]
pub(super) SectionFlags(u32) @ 0x0 {
0:0 read => bool;

45
drivers/gpu/drm/tyr/regs.rs
View File

@ -57,7 +57,11 @@ pub(crate) mod gpu_control {
uapi, //
};
use crate::driver::TyrRegisters;
register! {
base: TyrRegisters;
/// GPU identification register.
pub(crate) GPU_ID(u32) @ 0x0 {
/// Status of the GPU release.
@ -315,6 +319,8 @@ fn from(mode: FlushMode) -> Self {
}
register! {
base: TyrRegisters;
/// GPU command register.
///
/// Use the constructor methods to create commands:
@ -380,6 +386,8 @@ pub(crate) fn clear_fault() -> Self {
}
register! {
base: TyrRegisters;
/// GPU status register. Read only.
pub(crate) GPU_STATUS(u32) @ 0x34 {
/// GPU active, a 1-bit boolean flag.
@ -463,6 +471,8 @@ fn from(access: AccessType) -> Self {
}
register! {
base: TyrRegisters;
/// GPU fault status register. Read only.
pub(crate) GPU_FAULTSTATUS(u32) @ 0x3c {
/// Exception type.
@ -768,6 +778,8 @@ fn from(mode: CoherencyMode) -> Self {
}
register! {
base: TyrRegisters;
/// Coherency enable. An index of which coherency protocols should be used.
/// This register only selects the protocol for coherency messages on the
/// interconnect. This is not to enable or disable coherency controlled by MMU.
@ -808,6 +820,8 @@ fn from(mode: McuControlMode) -> Self {
}
register! {
base: TyrRegisters;
/// MCU control.
pub(crate) MCU_CONTROL(u32) @ 0x700 {
/// Request MCU state change.
@ -849,6 +863,8 @@ fn from(status: McuStatus) -> Self {
}
register! {
base: TyrRegisters;
/// MCU status. Read only.
pub(crate) MCU_STATUS(u32) @ 0x704 {
/// Read current state of MCU.
@ -862,7 +878,11 @@ fn from(status: McuStatus) -> Self {
pub(crate) mod job_control {
use kernel::register;
use crate::driver::TyrRegisters;
register! {
base: TyrRegisters;
/// Raw status of job interrupts.
///
/// Write to this register to trigger these interrupts.
@ -912,7 +932,11 @@ pub(crate) mod job_control {
pub(crate) mod mmu_control {
use kernel::register;
use crate::driver::TyrRegisters;
register! {
base: TyrRegisters;
/// IRQ sources raw status.
///
/// This register contains the raw unmasked interrupt sources for MMU status and exception
@ -966,9 +990,10 @@ pub(crate) mod mmu_as_control {
prelude::*,
register, //
};
use pin_init::Zeroable;
use crate::driver::TyrRegisters;
/// Maximum number of hardware address space slots.
/// The actual number of slots available is usually lower.
pub(crate) const MAX_AS: usize = 16;
@ -977,6 +1002,8 @@ pub(crate) mod mmu_as_control {
const STRIDE: usize = 0x40;
register! {
base: TyrRegisters;
/// Translation table base address. A 64-bit pointer.
///
/// This field contains the address of the top level of a translation table structure.
@ -1104,6 +1131,8 @@ fn from(val: MemoryType) -> Self {
}
register! {
base: TyrRegisters;
/// Stage 1 memory attributes (8-bit bitfield).
///
/// This is not an actual register, but a bitfield definition used by the MEMATTR
@ -1137,6 +1166,8 @@ fn from(val: MMU_MEMATTR_STAGE1) -> Self {
}
register! {
base: TyrRegisters;
/// Memory attributes.
///
/// Each address space can configure up to 8 different memory attribute profiles.
@ -1292,6 +1323,8 @@ pub(crate) fn from_mair(mair: u64) -> Self {
}
register! {
base: TyrRegisters;
/// Lock region address for each address space.
pub(crate) LOCKADDR(u64)[MAX_AS, stride = STRIDE] @ 0x2410 {
/// Lock region size.
@ -1353,6 +1386,8 @@ fn from(cmd: MmuCommand) -> Self {
}
register! {
base: TyrRegisters;
/// MMU command register for each address space. Write only.
pub(crate) COMMAND(u32)[MAX_AS, stride = STRIDE] @ 0x2418 {
7:0 command ?=> MmuCommand;
@ -1480,6 +1515,8 @@ fn from(access: MmuAccessType) -> Self {
}
register! {
base: TyrRegisters;
/// Fault status register for each address space. Read only.
pub(crate) FAULTSTATUS(u32)[MAX_AS, stride = STRIDE] @ 0x241c {
/// Exception type.
@ -1705,6 +1742,8 @@ fn from(sh: PtwShareability) -> Self {
}
register! {
base: TyrRegisters;
/// Translation configuration and control.
pub(crate) TRANSCFG(u64)[MAX_AS, stride = STRIDE] @ 0x2430 {
/// Address space mode.
@ -1760,6 +1799,8 @@ fn from(sh: PtwShareability) -> Self {
pub(crate) mod doorbell_block {
use kernel::register;
use crate::driver::TyrRegisters;
/// Number of doorbells available.
pub(crate) const NUM_DOORBELLS: usize = 64;
@ -1770,6 +1811,8 @@ pub(crate) mod doorbell_block {
const STRIDE: usize = 0x10000;
register! {
base: TyrRegisters;
/// Doorbell request register. Write-only.
pub(crate) DOORBELL(u32)[NUM_DOORBELLS, stride = STRIDE] @ 0x80000 {
/// Doorbell set. Writing 1 triggers the doorbell.

1
drivers/gpu/drm/tyr/vm.rs
View File

@ -644,7 +644,6 @@ fn sm_step_map<'op>(
// Expressly convert to u64 to work with arm 32-bit builds.
#[allow(clippy::useless_conversion)]
let mut paddr = u64::from(sgt_entry.dma_address());
#[allow(clippy::useless_conversion)]
let mut sgt_entry_length = u64::from(sgt_entry.dma_len());
if bytes_left_to_map == 0 {

1
drivers/gpu/nova-core/driver.rs
View File

@ -37,6 +37,7 @@ pub(crate) struct NovaCore<'bound> {
const BAR0_SIZE: usize = SZ_16M;
pub(crate) type Bar0<'a> = &'a pci::Bar<'a, BAR0_SIZE>;
pub(crate) type NovaRegisters = kernel::io::Region<BAR0_SIZE>;
kernel::pci_device_table!(
PCI_TABLE,

163
drivers/gpu/nova-core/falcon.rs
View File

@ -14,13 +14,12 @@
io::{
io_project,
poll::read_poll_timeout,
register::{
RegisterBase,
WithBase, //
},
register::Array,
Io,
Mmio, //
},
prelude::*,
sizes::SZ_4K,
time::Delta,
};
@ -165,18 +164,25 @@ pub(crate) enum FalconFbifMemType with From<Bounded<u32, 1>> {
}
}
/// Type used to represent the `PFALCON` registers address base for a given falcon engine.
pub(crate) struct PFalconBase(());
const PFALCON_REGION_SIZE: usize = SZ_4K;
const PFALCON2_REGION_SIZE: usize = SZ_4K;
/// Type used to represent the `PFALCON2` registers address base for a given falcon engine.
pub(crate) struct PFalcon2Base(());
/// Type used to represent the `PFALCON` registers.
#[repr(align(4))]
#[derive(FromBytes, IntoBytes)]
pub(crate) struct PFalconRegisters([u8; PFALCON_REGION_SIZE]);
/// Type used to represent the `PFALCON2` registers.
#[repr(align(4))]
#[derive(FromBytes, IntoBytes)]
pub(crate) struct PFalcon2Registers([u8; PFALCON2_REGION_SIZE]);
/// Trait defining the parameters of a given Falcon engine.
///
/// Each engine provides one base for `PFALCON` and `PFALCON2` registers.
pub(crate) trait FalconEngine:
Send + Sync + RegisterBase<PFalconBase> + RegisterBase<PFalcon2Base> + Sized
{
pub(crate) trait FalconEngine: Send + Sync + Sized {
fn pfalcon(io: Bar0<'_>) -> Mmio<'_, PFalconRegisters>;
fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, PFalcon2Registers>;
}
/// Represents a portion of the firmware to be loaded into a particular memory (e.g. IMEM or DMEM)
@ -358,6 +364,9 @@ pub(crate) struct Falcon<'a, E: FalconEngine> {
hal: KBox<dyn FalconHal<E>>,
dev: &'a device::Device<device::Bound>,
bar: Bar0<'a>,
// TODO: make private
pub(crate) pfalcon: Mmio<'a, PFalconRegisters>,
pfalcon2: Mmio<'a, PFalcon2Registers>,
}
impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
@ -371,19 +380,19 @@ pub(crate) fn new(
hal: hal::falcon_hal(chipset)?,
dev,
bar,
pfalcon: E::pfalcon(bar),
pfalcon2: E::pfalcon2(bar),
})
}
/// Resets DMA-related registers.
pub(crate) fn dma_reset(&self) {
self.bar.update(regs::NV_PFALCON_FBIF_CTL::of::<E>(), |v| {
self.pfalcon.update(regs::NV_PFALCON_FBIF_CTL, |v| {
v.with_allow_phys_no_ctx(true)
});
self.bar.write(
WithBase::of::<E>(),
regs::NV_PFALCON_FALCON_DMACTL::zeroed(),
);
self.pfalcon
.write_reg(regs::NV_PFALCON_FALCON_DMACTL::zeroed());
}
/// Reset the controller, select the falcon core, and wait for memory scrubbing to complete.
@ -392,10 +401,9 @@ pub(crate) fn reset(&self) -> Result {
self.hal.select_core(self)?;
self.hal.reset_wait_mem_scrubbing(self)?;
self.bar.write(
WithBase::of::<E>(),
regs::NV_PFALCON_FALCON_RM::from(self.bar.read(regs::NV_PMC_BOOT_0).into_raw()),
);
self.pfalcon.write_reg(regs::NV_PFALCON_FALCON_RM::from(
self.bar.read(regs::NV_PMC_BOOT_0).into_raw(),
));
Ok(())
}
@ -413,8 +421,8 @@ fn pio_wr_imem_slice(&self, load_offsets: FalconPioImemLoadTarget<'_>) -> Result
return Err(EINVAL);
}
self.bar.write(
WithBase::of::<E>().at(Self::PIO_PORT),
self.pfalcon.write(
Array::at(Self::PIO_PORT),
regs::NV_PFALCON_FALCON_IMEMC::zeroed()
.with_secure(load_offsets.secure)
.with_aincw(true)
@ -424,14 +432,14 @@ fn pio_wr_imem_slice(&self, load_offsets: FalconPioImemLoadTarget<'_>) -> Result
for (n, block) in load_offsets.data.chunks(MEM_BLOCK_ALIGNMENT).enumerate() {
let n = u16::try_from(n)?;
let tag: u16 = load_offsets.start_tag.checked_add(n).ok_or(ERANGE)?;
self.bar.write(
WithBase::of::<E>().at(Self::PIO_PORT),
self.pfalcon.write(
Array::at(Self::PIO_PORT),
regs::NV_PFALCON_FALCON_IMEMT::zeroed().with_tag(tag),
);
for word in block.chunks_exact(4) {
let w = [word[0], word[1], word[2], word[3]];
self.bar.write(
WithBase::of::<E>().at(Self::PIO_PORT),
self.pfalcon.write(
Array::at(Self::PIO_PORT),
regs::NV_PFALCON_FALCON_IMEMD::zeroed().with_data(u32::from_le_bytes(w)),
);
}
@ -450,8 +458,8 @@ fn pio_wr_dmem_slice(&self, load_offsets: FalconPioDmemLoadTarget<'_>) -> Result
return Err(EINVAL);
}
self.bar.write(
WithBase::of::<E>().at(Self::PIO_PORT),
self.pfalcon.write(
Array::at(Self::PIO_PORT),
regs::NV_PFALCON_FALCON_DMEMC::zeroed()
.with_aincw(true)
.with_offs(load_offsets.dst_start),
@ -459,8 +467,8 @@ fn pio_wr_dmem_slice(&self, load_offsets: FalconPioDmemLoadTarget<'_>) -> Result
for word in load_offsets.data.chunks_exact(4) {
let w = [word[0], word[1], word[2], word[3]];
self.bar.write(
WithBase::of::<E>().at(Self::PIO_PORT),
self.pfalcon.write(
Array::at(Self::PIO_PORT),
regs::NV_PFALCON_FALCON_DMEMD::zeroed().with_data(u32::from_le_bytes(w)),
);
}
@ -473,14 +481,12 @@ pub(crate) fn pio_load<F: FalconFirmware<Target = E> + FalconPioLoadable>(
&self,
fw: &F,
) -> Result {
self.bar.update(regs::NV_PFALCON_FBIF_CTL::of::<E>(), |v| {
self.pfalcon.update(regs::NV_PFALCON_FBIF_CTL, |v| {
v.with_allow_phys_no_ctx(true)
});
self.bar.write(
WithBase::of::<E>(),
regs::NV_PFALCON_FALCON_DMACTL::zeroed(),
);
self.pfalcon
.write_reg(regs::NV_PFALCON_FALCON_DMACTL::zeroed());
if let Some(imem_ns) = fw.imem_ns_load_params() {
self.pio_wr_imem_slice(imem_ns)?;
@ -492,10 +498,8 @@ pub(crate) fn pio_load<F: FalconFirmware<Target = E> + FalconPioLoadable>(
self.hal.program_brom(self, &fw.brom_params());
self.bar.write(
WithBase::of::<E>(),
regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()),
);
self.pfalcon
.write_reg(regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()));
Ok(())
}
@ -506,7 +510,7 @@ pub(crate) fn pio_load<F: FalconFirmware<Target = E> + FalconPioLoadable>(
/// `sec` is set if the loaded firmware is expected to run in secure mode.
fn dma_wr(
&self,
dma_obj: &Coherent<[u8]>,
dma_obj: &Coherent<'_, [u8]>,
target_mem: FalconMem,
load_offsets: FalconDmaLoadTarget,
) -> Result {
@ -547,16 +551,13 @@ fn dma_wr(
// Set up the base source DMA address.
self.bar.write(
WithBase::of::<E>(),
regs::NV_PFALCON_FALCON_DMATRFBASE::zeroed().with_base(
self.pfalcon
.write_reg(regs::NV_PFALCON_FALCON_DMATRFBASE::zeroed().with_base(
// CAST: `as u32` is used on purpose since we do want to strip the upper bits,
// which will be written to `NV_PFALCON_FALCON_DMATRFBASE1`.
(dma_address >> 8) as u32,
),
);
self.bar.write(
WithBase::of::<E>(),
));
self.pfalcon.write_reg(
regs::NV_PFALCON_FALCON_DMATRFBASE1::zeroed().try_with_base(dma_address >> 40)?,
);
@ -566,23 +567,21 @@ fn dma_wr(
for pos in (0..num_transfers).map(|i| i * DMA_LEN) {
// Perform a transfer of size `DMA_LEN`.
self.bar.write(
WithBase::of::<E>(),
self.pfalcon.write_reg(
regs::NV_PFALCON_FALCON_DMATRFMOFFS::zeroed()
.try_with_offs(load_offsets.dst_start + pos)?,
);
self.bar.write(
WithBase::of::<E>(),
self.pfalcon.write_reg(
regs::NV_PFALCON_FALCON_DMATRFFBOFFS::zeroed().with_offs(src_start + pos),
);
self.bar.write(WithBase::of::<E>(), cmd);
self.pfalcon.write_reg(cmd);
// Wait for the transfer to complete.
// TIMEOUT: arbitrarily large value, no DMA transfer to the falcon's small memories
// should ever take that long.
read_poll_timeout(
|| Ok(self.bar.read(regs::NV_PFALCON_FALCON_DMATRFCMD::of::<E>())),
|| Ok(self.pfalcon.read(regs::NV_PFALCON_FALCON_DMATRFCMD)),
|r| r.idle(),
Delta::ZERO,
Delta::from_secs(2),
@ -614,8 +613,8 @@ fn dma_load<F: FalconFirmware<Target = E> + FalconDmaLoadable>(&self, fw: &F) ->
};
self.dma_reset();
self.bar
.update(regs::NV_PFALCON_FBIF_TRANSCFG::of::<E>().at(0), |v| {
self.pfalcon
.update(regs::NV_PFALCON_FBIF_TRANSCFG::at(0), |v| {
v.with_target(FalconFbifTarget::CoherentSysmem)
.with_mem_type(FalconFbifMemType::Physical)
});
@ -626,10 +625,8 @@ fn dma_load<F: FalconFirmware<Target = E> + FalconDmaLoadable>(&self, fw: &F) ->
self.hal.program_brom(self, &fw.brom_params());
// Set `BootVec` to start of non-secure code.
self.bar.write(
WithBase::of::<E>(),
regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()),
);
self.pfalcon
.write_reg(regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()));
Ok(())
}
@ -638,7 +635,7 @@ fn dma_load<F: FalconFirmware<Target = E> + FalconDmaLoadable>(&self, fw: &F) ->
pub(crate) fn wait_till_halted(&self) -> Result<()> {
// TIMEOUT: arbitrarily large value, firmwares should complete in less than 2 seconds.
read_poll_timeout(
|| Ok(self.bar.read(regs::NV_PFALCON_FALCON_CPUCTL::of::<E>())),
|| Ok(self.pfalcon.read(regs::NV_PFALCON_FALCON_CPUCTL)),
|r| r.halted(),
Delta::ZERO,
Delta::from_secs(2),
@ -649,19 +646,13 @@ pub(crate) fn wait_till_halted(&self) -> Result<()> {
/// Start the falcon CPU.
pub(crate) fn start(&self) -> Result<()> {
match self
.bar
.read(regs::NV_PFALCON_FALCON_CPUCTL::of::<E>())
.alias_en()
{
true => self.bar.write(
WithBase::of::<E>(),
regs::NV_PFALCON_FALCON_CPUCTL_ALIAS::zeroed().with_startcpu(true),
),
false => self.bar.write(
WithBase::of::<E>(),
regs::NV_PFALCON_FALCON_CPUCTL::zeroed().with_startcpu(true),
),
match self.pfalcon.read(regs::NV_PFALCON_FALCON_CPUCTL).alias_en() {
true => self
.pfalcon
.write_reg(regs::NV_PFALCON_FALCON_CPUCTL_ALIAS::zeroed().with_startcpu(true)),
false => self
.pfalcon
.write_reg(regs::NV_PFALCON_FALCON_CPUCTL::zeroed().with_startcpu(true)),
}
Ok(())
@ -670,32 +661,24 @@ pub(crate) fn start(&self) -> Result<()> {
/// Writes values to the mailbox registers if provided.
pub(crate) fn write_mailboxes(&self, mbox0: Option<u32>, mbox1: Option<u32>) {
if let Some(mbox0) = mbox0 {
self.bar.write(
WithBase::of::<E>(),
regs::NV_PFALCON_FALCON_MAILBOX0::zeroed().with_value(mbox0),
);
self.pfalcon
.write_reg(regs::NV_PFALCON_FALCON_MAILBOX0::zeroed().with_value(mbox0));
}
if let Some(mbox1) = mbox1 {
self.bar.write(
WithBase::of::<E>(),
regs::NV_PFALCON_FALCON_MAILBOX1::zeroed().with_value(mbox1),
);
self.pfalcon
.write_reg(regs::NV_PFALCON_FALCON_MAILBOX1::zeroed().with_value(mbox1));
}
}
/// Reads the value from `mbox0` register.
pub(crate) fn read_mailbox0(&self) -> u32 {
self.bar
.read(regs::NV_PFALCON_FALCON_MAILBOX0::of::<E>())
.value()
self.pfalcon.read(regs::NV_PFALCON_FALCON_MAILBOX0).value()
}
/// Reads the value from `mbox1` register.
pub(crate) fn read_mailbox1(&self) -> u32 {
self.bar
.read(regs::NV_PFALCON_FALCON_MAILBOX1::of::<E>())
.value()
self.pfalcon.read(regs::NV_PFALCON_FALCON_MAILBOX1).value()
}
/// Reads values from both mailbox registers.
@ -760,9 +743,7 @@ pub(crate) fn load<F: FalconFirmware<Target = E> + FalconDmaLoadable>(&self, fw:
/// Write the application version to the OS register.
pub(crate) fn write_os_version(&self, app_version: u32) {
self.bar.write(
WithBase::of::<E>(),
regs::NV_PFALCON_FALCON_OS::zeroed().with_value(app_version),
);
self.pfalcon
.write_reg(regs::NV_PFALCON_FALCON_OS::zeroed().with_value(app_version));
}
}

63
drivers/gpu/nova-core/falcon/fsp.rs
View File

@ -8,13 +8,12 @@
use kernel::{
io::{
io_project,
poll::read_poll_timeout,
register::{
Array,
RegisterBase,
WithBase, //
},
Io, //
register,
register::Array,
Io,
Mmio, //
},
prelude::*,
sizes::SZ_1K,
@ -22,11 +21,13 @@
};
use crate::{
driver::{
Bar0,
NovaRegisters, //
},
falcon::{
Falcon,
FalconEngine,
PFalcon2Base,
PFalconBase, //
FalconEngine, //
},
num,
regs, //
@ -41,15 +42,24 @@
/// Type specifying the `Fsp` falcon engine. Cannot be instantiated.
pub(crate) struct Fsp(());
impl RegisterBase<PFalconBase> for Fsp {
const BASE: usize = 0x8f2000;
register! {
base: NovaRegisters;
PFALCON: super::PFalconRegisters @ 0x8f2000;
PFALCON2: super::PFalcon2Registers @ 0x8f3000;
}
impl RegisterBase<PFalcon2Base> for Fsp {
const BASE: usize = 0x8f3000;
}
impl FalconEngine for Fsp {
#[inline]
fn pfalcon(io: Bar0<'_>) -> Mmio<'_, super::PFalconRegisters> {
io_project!(io, build: PFALCON)
}
impl FalconEngine for Fsp {}
#[inline]
fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, super::PFalcon2Registers> {
io_project!(io, build: PFALCON2)
}
}
impl<'a> Falcon<'a, Fsp> {
/// Writes `data` to FSP external memory at offset `0`.
@ -62,19 +72,15 @@ fn write_emem(&mut self, data: &[u8]) -> Result {
}
// Begin a write burst at offset `0`, auto-incrementing on each write.
self.bar.write(
WithBase::of::<Fsp>(),
regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincw(true),
);
self.pfalcon
.write_reg(regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincw(true));
for chunk in data.chunks_exact(4) {
let value = u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]);
// Write the next 32-bit `value`; hardware advances the offset.
self.bar.write(
WithBase::of::<Fsp>(),
regs::NV_PFALCON_FALCON_EMEMD::zeroed().with_data(value),
);
self.pfalcon
.write_reg(regs::NV_PFALCON_FALCON_EMEMD::zeroed().with_data(value));
}
Ok(())
@ -90,17 +96,12 @@ fn read_emem(&mut self, data: &mut [u8]) -> Result {
}
// Begin a read burst at offset `0`, auto-incrementing on each read.
self.bar.write(
WithBase::of::<Fsp>(),
regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincr(true),
);
self.pfalcon
.write_reg(regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincr(true));
for chunk in data.chunks_exact_mut(4) {
// Read the next 32-bit word; hardware advances the offset.
let value = self
.bar
.read(regs::NV_PFALCON_FALCON_EMEMD::of::<Fsp>())
.data();
let value = self.pfalcon.read(regs::NV_PFALCON_FALCON_EMEMD).data();
chunk.copy_from_slice(&value.to_le_bytes());
}

51
drivers/gpu/nova-core/falcon/gsp.rs
View File

@ -2,23 +2,24 @@
use kernel::{
io::{
io_project,
poll::read_poll_timeout,
register::{
RegisterBase,
WithBase, //
},
register,
Io,
Mmio, //
},
prelude::*,
time::Delta, //
};
use crate::{
driver::{
Bar0,
NovaRegisters, //
},
falcon::{
Falcon,
FalconEngine,
PFalcon2Base,
PFalconBase, //
FalconEngine, //
},
regs,
};
@ -26,24 +27,31 @@
/// Type specifying the `Gsp` falcon engine. Cannot be instantiated.
pub(crate) struct Gsp(());
impl RegisterBase<PFalconBase> for Gsp {
const BASE: usize = 0x00110000;
register! {
base: NovaRegisters;
PFALCON: super::PFalconRegisters @ 0x00110000;
PFALCON2: super::PFalcon2Registers @ 0x00111000;
}
impl RegisterBase<PFalcon2Base> for Gsp {
const BASE: usize = 0x00111000;
}
impl FalconEngine for Gsp {
#[inline]
fn pfalcon(io: Bar0<'_>) -> Mmio<'_, super::PFalconRegisters> {
io_project!(io, build: PFALCON)
}
impl FalconEngine for Gsp {}
#[inline]
fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, super::PFalcon2Registers> {
io_project!(io, build: PFALCON2)
}
}
impl<'a> Falcon<'a, Gsp> {
/// Clears the SWGEN0 bit in the Falcon's IRQ status clear register to
/// allow GSP to signal CPU for processing new messages in message queue.
pub(crate) fn clear_swgen0_intr(&self) {
self.bar.write(
WithBase::of::<Gsp>(),
regs::NV_PFALCON_FALCON_IRQSCLR::zeroed().with_swgen0(true),
);
self.pfalcon
.write_reg(regs::NV_PFALCON_FALCON_IRQSCLR::zeroed().with_swgen0(true));
}
/// Checks if GSP reload/resume has completed during the boot process.
@ -59,8 +67,8 @@ pub(crate) fn check_reload_completed(&self, timeout: Delta) -> Result<bool> {
/// Returns whether the RISC-V branch privilege lockdown bit is set.
pub(crate) fn riscv_branch_privilege_lockdown(&self) -> bool {
self.bar
.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<Gsp>())
self.pfalcon
.read(regs::NV_PFALCON_FALCON_HWCFG2)
.riscv_br_priv_lockdown()
}
@ -71,10 +79,7 @@ pub(crate) fn priv_target_mask_released(&self) -> bool {
const LOCKED_PATTERN: u32 = 0xbadf_4100;
const LOCKED_MASK: u32 = 0xffff_ff00;
let hwcfg2 = self
.bar
.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<Gsp>())
.into_raw();
let hwcfg2 = self.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2).into_raw();
hwcfg2 != 0 && (hwcfg2 & LOCKED_MASK) != LOCKED_PATTERN
}

59
drivers/gpu/nova-core/falcon/hal/ga102.rs
View File

@ -6,11 +6,9 @@
device,
io::{
poll::read_poll_timeout,
register::{
Array,
WithBase, //
},
Io, //
register::Array,
Io,
Mmio, //
},
prelude::*,
time::Delta, //
@ -24,6 +22,7 @@
FalconBromParams,
FalconEngine,
FalconModSelAlgo,
PFalcon2Registers,
PeregrineCoreSelect, //
},
regs,
@ -31,17 +30,16 @@
use super::FalconHal;
fn select_core_ga102<E: FalconEngine>(bar: Bar0<'_>) -> Result {
let bcr_ctrl = bar.read(regs::NV_PRISCV_RISCV_BCR_CTRL::of::<E>());
fn select_core_ga102(pfalcon2: Mmio<'_, PFalcon2Registers>) -> Result {
let bcr_ctrl = pfalcon2.read(regs::NV_PRISCV_RISCV_BCR_CTRL);
if bcr_ctrl.core_select() != PeregrineCoreSelect::Falcon {
bar.write(
WithBase::of::<E>(),
pfalcon2.write_reg(
regs::NV_PRISCV_RISCV_BCR_CTRL::zeroed().with_core_select(PeregrineCoreSelect::Falcon),
);
// TIMEOUT: falcon core should take less than 10ms to report being enabled.
read_poll_timeout(
|| Ok(bar.read(regs::NV_PRISCV_RISCV_BCR_CTRL::of::<E>())),
|| Ok(pfalcon2.read(regs::NV_PRISCV_RISCV_BCR_CTRL)),
|r| r.valid(),
Delta::ZERO,
Delta::from_millis(10),
@ -86,24 +84,20 @@ fn signature_reg_fuse_version_ga102(
Ok(u16::BITS - reg_fuse_version.leading_zeros())
}
fn program_brom_ga102<E: FalconEngine>(bar: Bar0<'_>, params: &FalconBromParams) {
bar.write(
WithBase::of::<E>().at(0),
fn program_brom_ga102(pfalcon2: Mmio<'_, PFalcon2Registers>, params: &FalconBromParams) {
pfalcon2.write(
Array::at(0),
regs::NV_PFALCON2_FALCON_BROM_PARAADDR::zeroed().with_value(params.pkc_data_offset),
);
bar.write(
WithBase::of::<E>(),
pfalcon2.write_reg(
regs::NV_PFALCON2_FALCON_BROM_ENGIDMASK::zeroed()
.with_value(u32::from(params.engine_id_mask)),
);
bar.write(
WithBase::of::<E>(),
pfalcon2.write_reg(
regs::NV_PFALCON2_FALCON_BROM_CURR_UCODE_ID::zeroed().with_ucode_id(params.ucode_id),
);
bar.write(
WithBase::of::<E>(),
regs::NV_PFALCON2_FALCON_MOD_SEL::zeroed().with_algo(FalconModSelAlgo::Rsa3k),
);
pfalcon2
.write_reg(regs::NV_PFALCON2_FALCON_MOD_SEL::zeroed().with_algo(FalconModSelAlgo::Rsa3k));
}
pub(super) struct Ga102<E: FalconEngine>(PhantomData<E>);
@ -116,7 +110,7 @@ pub(super) fn new() -> Self {
impl<E: FalconEngine> FalconHal<E> for Ga102<E> {
fn select_core(&self, falcon: &Falcon<'_, E>) -> Result {
select_core_ga102::<E>(falcon.bar)
select_core_ga102(falcon.pfalcon2)
}
fn signature_reg_fuse_version(
@ -129,27 +123,24 @@ fn signature_reg_fuse_version(
}
fn program_brom(&self, falcon: &Falcon<'_, E>, params: &FalconBromParams) {
program_brom_ga102::<E>(falcon.bar, params);
program_brom_ga102(falcon.pfalcon2, params);
}
fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool {
falcon
.bar
.read(regs::NV_PRISCV_RISCV_CPUCTL::of::<E>())
.pfalcon2
.read(regs::NV_PRISCV_RISCV_CPUCTL)
.active_stat()
}
fn is_riscv_halted(&self, falcon: &Falcon<'_, E>) -> Result<bool> {
Ok(falcon
.bar
.read(regs::NV_PRISCV_RISCV_CPUCTL::of::<E>())
.halted())
Ok(falcon.pfalcon2.read(regs::NV_PRISCV_RISCV_CPUCTL).halted())
}
fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result {
// TIMEOUT: memory scrubbing should complete in less than 20ms.
read_poll_timeout(
|| Ok(falcon.bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<E>())),
|| Ok(falcon.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2)),
|r| r.mem_scrubbing_done(),
Delta::ZERO,
Delta::from_millis(20),
@ -158,20 +149,18 @@ fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result {
}
fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result {
let bar = falcon.bar;
let _ = bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<E>());
let _ = falcon.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2);
// According to OpenRM's `kflcnPreResetWait_GA102` documentation, HW sometimes does not set
// RESET_READY so a non-failing timeout is used.
let _ = read_poll_timeout(
|| Ok(bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<E>())),
|| Ok(falcon.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2)),
|r| r.reset_ready(),
Delta::ZERO,
Delta::from_micros(150),
);
regs::NV_PFALCON_FALCON_ENGINE::reset_engine::<E>(bar);
regs::NV_PFALCON_FALCON_ENGINE::reset_engine(falcon.pfalcon);
self.reset_wait_mem_scrubbing(falcon)?;
Ok(())

9
drivers/gpu/nova-core/falcon/hal/tu102.rs
View File

@ -5,7 +5,6 @@
use kernel::{
io::{
poll::read_poll_timeout,
register::WithBase,
Io, //
},
prelude::*,
@ -50,8 +49,8 @@ fn program_brom(&self, _falcon: &Falcon<'_, E>, _params: &FalconBromParams) {}
fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool {
falcon
.bar
.read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS::of::<E>())
.pfalcon2
.read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS)
.active_stat()
}
@ -62,7 +61,7 @@ fn is_riscv_halted(&self, _falcon: &Falcon<'_, E>) -> Result<bool> {
fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result {
// TIMEOUT: memory scrubbing should complete in less than 10ms.
read_poll_timeout(
|| Ok(falcon.bar.read(regs::NV_PFALCON_FALCON_DMACTL::of::<E>())),
|| Ok(falcon.pfalcon.read(regs::NV_PFALCON_FALCON_DMACTL)),
|r| r.mem_scrubbing_done(),
Delta::ZERO,
Delta::from_millis(10),
@ -71,7 +70,7 @@ fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result {
}
fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result {
regs::NV_PFALCON_FALCON_ENGINE::reset_engine::<E>(falcon.bar);
regs::NV_PFALCON_FALCON_ENGINE::reset_engine(falcon.pfalcon);
self.reset_wait_mem_scrubbing(falcon)?;
Ok(())

37
drivers/gpu/nova-core/falcon/sec2.rs
View File

@ -1,22 +1,37 @@
// SPDX-License-Identifier: GPL-2.0
use kernel::io::register::RegisterBase;
use kernel::io::{
io_project,
register,
Mmio, //
};
use crate::falcon::{
FalconEngine,
PFalcon2Base,
PFalconBase, //
use crate::{
driver::{
Bar0,
NovaRegisters, //
},
falcon::FalconEngine, //
};
/// Type specifying the `Sec2` falcon engine. Cannot be instantiated.
pub(crate) struct Sec2(());
impl RegisterBase<PFalconBase> for Sec2 {
const BASE: usize = 0x00840000;
register! {
base: NovaRegisters;
PFALCON: super::PFalconRegisters @ 0x00840000;
PFALCON2: super::PFalcon2Registers @ 0x00841000;
}
impl RegisterBase<PFalcon2Base> for Sec2 {
const BASE: usize = 0x00841000;
}
impl FalconEngine for Sec2 {
#[inline]
fn pfalcon(io: Bar0<'_>) -> Mmio<'_, super::PFalconRegisters> {
io_project!(io, build: PFALCON)
}
impl FalconEngine for Sec2 {}
#[inline]
fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, super::PFalcon2Registers> {
io_project!(io, build: PFALCON2)
}
}

4
drivers/gpu/nova-core/fb.rs
View File

@ -49,7 +49,7 @@ pub(crate) struct SysmemFlush<'sys> {
device: &'sys device::Device,
bar: Bar0<'sys>,
/// Keep the page alive as long as we need it.
page: CoherentHandle,
page: CoherentHandle<'sys>,
}
impl<'sys> SysmemFlush<'sys> {
@ -177,7 +177,7 @@ impl FbRanges {
pub(crate) fn new(
chipset: Chipset,
bar: Bar0<'_>,
gsp_fw: &GspFirmware,
gsp_fw: &GspFirmware<'_>,
vgpu_state: VgpuState,
) -> Result<Self> {
let hal = hal::fb_hal(chipset);

59
drivers/gpu/nova-core/fb/hal/gb100.rs
View File

@ -5,11 +5,10 @@
use kernel::{
io::{
register::{
RegisterBase,
WithBase, //
},
Io, //
io_project,
register,
Io,
Mmio, //
},
num::Bounded,
prelude::*,
@ -21,7 +20,10 @@
};
use crate::{
driver::Bar0,
driver::{
Bar0,
NovaRegisters, //
},
fb::{
hal::FbHal,
regs, //
@ -31,17 +33,26 @@
struct Gb100;
impl RegisterBase<regs::Hshub0Base> for Gb100 {
const BASE: usize = 0x0087_0000;
register! {
base: NovaRegisters;
HSHUB0: regs::Hshub0Registers @ 0x0087_0000;
}
fn read_sysmem_flush_page_gb100(bar: Bar0<'_>) -> u64 {
#[inline]
fn hshub0(bar: Bar0<'_>) -> Mmio<'_, regs::Hshub0Registers> {
io_project!(bar, build: HSHUB0)
}
fn read_sysmem_flush_page_gb100(hshub0: Mmio<'_, regs::Hshub0Registers>) -> u64 {
let lo = u64::from(
bar.read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::<Gb100>())
hshub0
.read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO)
.adr(),
);
let hi = u64::from(
bar.read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::<Gb100>())
hshub0
.read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI)
.adr(),
);
@ -52,7 +63,7 @@ fn read_sysmem_flush_page_gb100(bar: Bar0<'_>) -> u64 {
///
/// Both the primary and EG (egress) register pairs must be programmed to the same address,
/// as required by hardware.
fn write_sysmem_flush_page_gb100(bar: Bar0<'_>, addr: Bounded<u64, 52>) {
fn write_sysmem_flush_page_gb100(hshub0: Mmio<'_, regs::Hshub0Registers>, addr: Bounded<u64, 52>) {
// CAST: lower 32 bits. Hardware ignores bits 7:0.
let addr_lo = *addr as u32;
let addr_hi = addr.shr::<32, 20>().cast::<u32>();
@ -60,24 +71,12 @@ fn write_sysmem_flush_page_gb100(bar: Bar0<'_>, addr: Bounded<u64, 52>) {
// Write HI first. The hardware will trigger the flush on the LO write.
// Primary HSHUB pair.
bar.write(
regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::<Gb100>(),
regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi),
);
bar.write(
regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::<Gb100>(),
regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo),
);
hshub0.write_reg(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi));
hshub0.write_reg(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo));
// EG (egress) pair -- must match the primary pair.
bar.write(
regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI::of::<Gb100>(),
regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi),
);
bar.write(
regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO::of::<Gb100>(),
regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo),
);
hshub0.write_reg(regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi));
hshub0.write_reg(regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo));
}
// This PMU reservation size is r570-specific.
@ -88,13 +87,13 @@ pub(super) const fn pmu_reserved_size_gb100() -> u32 {
impl FbHal for Gb100 {
fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 {
read_sysmem_flush_page_gb100(bar)
read_sysmem_flush_page_gb100(hshub0(bar))
}
fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result {
let addr = Bounded::<u64, 52>::try_new(addr).ok_or(EINVAL)?;
write_sysmem_flush_page_gb100(bar, addr);
write_sysmem_flush_page_gb100(hshub0(bar), addr);
Ok(())
}

34
drivers/gpu/nova-core/fb/regs.rs
View File

@ -2,12 +2,20 @@
use kernel::{
io::register,
sizes::SizeConstants, //
prelude::*,
sizes::{
SizeConstants,
SZ_4K, //
}, //
};
use crate::driver::NovaRegisters;
// PDISP
register! {
base: NovaRegisters;
pub(super) NV_PDISP_VGA_WORKSPACE_BASE(u32) @ 0x00625f04 {
/// VGA workspace base address divided by 0x10000.
31:8 addr;
@ -30,6 +38,8 @@ pub(super) fn vga_workspace_addr(self) -> Option<u64> {
// PFB
register! {
base: NovaRegisters;
/// Low bits of the physical system memory address used by the GPU to perform sysmembar
/// operations (see [`crate::fb::SysmemFlush`]).
pub(super) NV_PFB_NISO_FLUSH_SYSMEM_ADDR(u32) @ 0x00100c10 {
@ -59,34 +69,42 @@ pub(super) fn vga_workspace_addr(self) -> Option<u64> {
}
}
/// Base of the GB10x HSHUB0 register window (`NV_HSHUB0_PRIV_BASE` in Open RM).
const HSHUB0_REGION_SIZE: usize = SZ_4K;
/// The GB10x HSHUB0 register window (Base defined as `NV_HSHUB0_PRIV_BASE` in Open RM).
///
/// The base is provided by the GB10x framebuffer HAL.
pub(super) struct Hshub0Base(());
#[repr(align(4))]
#[derive(FromBytes, IntoBytes)]
pub(super) struct Hshub0Registers([u8; HSHUB0_REGION_SIZE]);
register! {
base: Hshub0Registers;
// GB10x sysmem flush registers, relative to the HSHUB0 base. GB10x routes sysmembar
// through a primary and an EG (egress) pair that must both be programmed to the same
// address. Hardware ignores bits 7:0 of each LO register. The boot path uses a fixed
// HSHUB0 base, so the multiple runtime-discovered HSHUB bases are not needed here.
pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x00000e50 {
pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x00000e50 {
31:0 adr => u32;
}
pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x00000e54 {
pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x00000e54 {
19:0 adr;
}
pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x000006c0 {
pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x000006c0 {
31:0 adr => u32;
}
pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x000006c4 {
pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x000006c4 {
19:0 adr;
}
}
register! {
base: NovaRegisters;
// GB20x FBHUB0 sysmem flush registers. Unlike the older
// NV_PFB_NISO_FLUSH_SYSMEM_ADDR registers, which encode the address with an
// 8-bit right-shift, these take the raw address split into lower and upper
@ -101,6 +119,8 @@ pub(super) fn vga_workspace_addr(self) -> Option<u64> {
}
register! {
base: NovaRegisters;
/// Low bits of the physical system memory address used by the GPU to perform
/// sysmembar operations on Hopper.
///

2
drivers/gpu/nova-core/firmware/booter.rs
View File

@ -186,7 +186,7 @@ pub(crate) fn run<T>(
&self,
dev: &device::Device<device::Bound>,
sec2_falcon: &Falcon<'_, Sec2>,
wpr_meta: &Coherent<T>,
wpr_meta: &Coherent<'_, T>,
) -> Result {
sec2_falcon.reset()?;
sec2_falcon.load(self)?;

8
drivers/gpu/nova-core/firmware/fsp.rs
View File

@ -39,15 +39,15 @@ pub(crate) struct FmcSignatures {
pub(crate) signature: [u8; FSP_SIG_SIZE],
}
pub(crate) struct FspFirmware {
pub(crate) struct FspFirmware<'a> {
/// FMC firmware image data
pub(crate) fmc_image: Coherent<[u8]>,
pub(crate) fmc_image: Coherent<'a, [u8]>,
/// FMC firmware signatures.
pub(crate) fmc_sigs: KBox<FmcSignatures>,
}
impl FspFirmware {
pub(crate) fn new(dev: &device::Device<device::Bound>, chipset: Chipset) -> Result<Self> {
impl<'a> FspFirmware<'a> {
pub(crate) fn new(dev: &'a device::Device<device::Bound>, chipset: Chipset) -> Result<Self> {
let fw = request_tlv(dev, chipset, "fmc")?;
let tlv = Tlv::new(fw.data())?;
dev_dbg!(dev, "loaded fsp firmware v{}\n", tlv.get_string(b"VERS")?);

30
drivers/gpu/nova-core/firmware/fwsec/bootloader.rs
View File

@ -12,7 +12,10 @@
Device, //
},
dma::Coherent,
io::{register::WithBase, Io},
io::{
register::Array,
Io, //
},
prelude::*,
ptr::{
Alignable,
@ -23,7 +26,6 @@
};
use crate::{
driver::Bar0,
falcon::{
self,
gsp::Gsp,
@ -98,9 +100,9 @@ unsafe impl AsBytes for BootloaderDmemDescV2 {}
/// Wrapper for [`FwsecFirmware`] that includes the bootloader performing the actual load
/// operation.
pub(crate) struct FwsecFirmwareWithBl {
pub(crate) struct FwsecFirmwareWithBl<'a> {
/// DMA object the bootloader will copy the firmware from.
_firmware_dma: Coherent<[u8]>,
_firmware_dma: Coherent<'a, [u8]>,
/// Code of the bootloader to be loaded into non-secure IMEM.
ucode: KVec<u8>,
/// Descriptor to be loaded into DMEM for the bootloader to read.
@ -113,12 +115,12 @@ pub(crate) struct FwsecFirmwareWithBl {
start_tag: u16,
}
impl FwsecFirmwareWithBl {
impl<'a> FwsecFirmwareWithBl<'a> {
/// Loads the bootloader firmware for `dev` and `chipset`, and wrap `firmware` so it can be
/// loaded using it.
pub(crate) fn new(
firmware: FwsecFirmware,
dev: &Device<device::Bound>,
dev: &'a Device<device::Bound>,
chipset: Chipset,
) -> Result<Self> {
let fw = request_tlv(dev, chipset, "gen_bootloader")?;
@ -235,12 +237,7 @@ pub(crate) fn new(
///
/// The bootloader will load the FWSEC firmware and then execute it. This function returns
/// after FWSEC has reached completion.
pub(crate) fn run(
&self,
dev: &Device<device::Bound>,
falcon: &Falcon<'_, Gsp>,
bar: Bar0<'_>,
) -> Result<()> {
pub(crate) fn run(&self, dev: &Device<device::Bound>, falcon: &Falcon<'_, Gsp>) -> Result<()> {
// Reset falcon, load the firmware, and run it.
falcon
.reset()
@ -250,9 +247,8 @@ pub(crate) fn run(
.inspect_err(|e| dev_err!(dev, "Failed to load FWSEC firmware: {:?}\n", e))?;
// Configure DMA index for the bootloader to fetch the FWSEC firmware from system memory.
bar.update(
regs::NV_PFALCON_FBIF_TRANSCFG::of::<Gsp>()
.try_at(usize::from_safe_cast(self.dmem_desc.ctx_dma))
falcon.pfalcon.update(
regs::NV_PFALCON_FBIF_TRANSCFG::try_at(usize::from_safe_cast(self.dmem_desc.ctx_dma))
.ok_or(EINVAL)?,
|v| {
v.with_target(FalconFbifTarget::CoherentSysmem)
@ -272,7 +268,7 @@ pub(crate) fn run(
}
}
impl FalconFirmware for FwsecFirmwareWithBl {
impl FalconFirmware for FwsecFirmwareWithBl<'_> {
type Target = Gsp;
fn brom_params(&self) -> FalconBromParams {
@ -286,7 +282,7 @@ fn boot_addr(&self) -> u32 {
}
}
impl FalconPioLoadable for FwsecFirmwareWithBl {
impl FalconPioLoadable for FwsecFirmwareWithBl<'_> {
fn imem_sec_load_params(&self) -> Option<FalconPioImemLoadTarget<'_>> {
None
}

14
drivers/gpu/nova-core/firmware/gsp.rs
View File

@ -44,7 +44,7 @@
/// Each page is 4KB, each entry is 8 bytes (64-bit DMA address).
/// Also known as "Radix3" firmware.
#[pin_data]
pub(crate) struct GspFirmware {
pub(crate) struct GspFirmware<'a> {
/// The GSP firmware inside a [`VVec`], device-mapped via a SG table.
#[pin]
fw: SGTable<Owned<VVec<u8>>>,
@ -55,19 +55,19 @@ pub(crate) struct GspFirmware {
#[pin]
level1: SGTable<Owned<VVec<u8>>>,
/// Level 0 page table (single 4KB page) with one entry: DMA address of first level 1 page.
level0: Coherent<[u64]>,
level0: Coherent<'a, [u64]>,
/// Size in bytes of the firmware contained in [`Self::fw`].
pub(crate) size: usize,
/// Device-mapped GSP signatures matching the GPU's [`Chipset`].
pub(crate) signatures: Coherent<[u8]>,
pub(crate) signatures: Coherent<'a, [u8]>,
/// GSP bootloader, verifies the GSP firmware before loading and running it.
pub(crate) bootloader: RiscvFirmware,
pub(crate) bootloader: RiscvFirmware<'a>,
}
impl GspFirmware {
impl<'a> GspFirmware<'a> {
/// Loads the GSP firmware binaries, map them into `dev`'s address-space, and creates the page
/// tables expected by the GSP bootloader to load it.
pub(crate) fn new<'a>(
pub(crate) fn new(
dev: &'a device::Device<device::Bound>,
chipset: Chipset,
) -> impl PinInit<Self, Error> + 'a {
@ -120,7 +120,7 @@ pub(crate) fn new<'a>(
// Create level 0 page table data and fill its first entry with the level 1
// table.
let mut level0 = CoherentBox::<[u64]>::zeroed_slice(
let mut level0 = CoherentBox::<'_, [u64]>::zeroed_slice(
dev,
GSP_PAGE_SIZE / size_of::<u64>(),
GFP_KERNEL

8
drivers/gpu/nova-core/firmware/riscv.rs
View File

@ -13,7 +13,7 @@
use crate::firmware::tlv::Tlv;
/// A parsed firmware for a RISC-V core, ready to be loaded and run.
pub(crate) struct RiscvFirmware {
pub(crate) struct RiscvFirmware<'a> {
/// Offset at which the code starts in the firmware image.
pub(crate) code_offset: u32,
/// Offset at which the data starts in the firmware image.
@ -23,12 +23,12 @@ pub(crate) struct RiscvFirmware {
/// Application version.
pub(crate) app_version: u32,
/// Device-mapped firmware image.
pub(crate) ucode: Coherent<[u8]>,
pub(crate) ucode: Coherent<'a, [u8]>,
}
impl RiscvFirmware {
impl<'a> RiscvFirmware<'a> {
/// Parses the RISC-V firmware image contained in `fw`.
pub(crate) fn new(dev: &device::Device<device::Bound>, fw: &Firmware) -> Result<Self> {
pub(crate) fn new(dev: &'a device::Device<device::Bound>, fw: &Firmware) -> Result<Self> {
let tlv = Tlv::new(fw.data())?;
dev_dbg!(
dev,

20
drivers/gpu/nova-core/fsp.rs
View File

@ -267,7 +267,7 @@ fn frts_vidmem_offset(hal: &dyn hal::FspHal, fb_info: &FbSizes) -> Result<u64> {
/// Returns an in-place initializer for [`FspCotMessage`].
fn new<'a>(
fb_info: &FbSizes,
fsp_fw: &'a FspFirmware,
fsp_fw: &'a FspFirmware<'_>,
args: &'a FmcBootArgs<'_>,
) -> Result<impl Init<Self> + 'a> {
let hal = hal::fsp_hal(args.chipset).ok_or(ENOTSUPP)?;
@ -345,28 +345,28 @@ impl MessageToFsp for FspPrcMessage {
/// Bundled arguments for FMC boot via FSP Chain of Trust.
pub(crate) struct FmcBootArgs<'a> {
chipset: Chipset,
fmc_boot_params: Coherent<GspFmcBootParams>,
fmc_boot_params: Coherent<'a, GspFmcBootParams>,
resume: bool,
// Additional dependencies required to be kept alive for FMC boot.
_wpr_meta: Coherent<GspFwWprMeta>,
_libos: &'a Coherent<[LibosMemoryRegionInitArgument]>,
_wpr_meta: Coherent<'a, GspFwWprMeta>,
_libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>,
}
impl<'a> FmcBootArgs<'a> {
/// Builds FMC boot arguments, allocating the DMA-coherent boot parameter
/// structure that FSP will read.
pub(crate) fn new(
dev: &device::Device<device::Bound>,
dev: &'a device::Device<device::Bound>,
chipset: Chipset,
wpr_meta: Coherent<GspFwWprMeta>,
libos: &'a Coherent<[LibosMemoryRegionInitArgument]>,
wpr_meta: Coherent<'a, GspFwWprMeta>,
libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>,
resume: bool,
) -> Result<Self> {
let init = GspFmcBootParams::new(wpr_meta.dma_address(), libos.dma_address());
Ok(Self {
chipset,
fmc_boot_params: Coherent::<GspFmcBootParams>::init(dev, GFP_KERNEL, init)?,
fmc_boot_params: Coherent::init(dev, GFP_KERNEL, init)?,
resume,
_wpr_meta: wpr_meta,
_libos: libos,
@ -374,7 +374,7 @@ pub(crate) fn new(
}
/// Returns the FMC boot parameters allocation.
pub(crate) fn boot_params(&self) -> &Coherent<GspFmcBootParams> {
pub(crate) fn boot_params(&self) -> &Coherent<'_, GspFmcBootParams> {
&self.fmc_boot_params
}
}
@ -386,7 +386,7 @@ pub(crate) fn boot_params(&self) -> &Coherent<GspFmcBootParams> {
/// Chain of Trust boot.
pub(crate) struct Fsp<'a> {
falcon: Falcon<'a, FspEngine>,
fsp_fw: FspFirmware,
fsp_fw: FspFirmware<'a>,
}
impl<'a> Fsp<'a> {

4
drivers/gpu/nova-core/gpu.rs
View File

@ -272,9 +272,9 @@ struct GspResources<'gpu> {
vgpu: VgpuManager,
/// GSP runtime data.
#[pin]
gsp: Gsp,
gsp: Gsp<'gpu>,
/// GSP unload firmware bundle, if any.
unload_bundle: Option<gsp::UnloadBundle>,
unload_bundle: Option<gsp::UnloadBundle<'gpu>>,
}
/// Structure holding the resources required to operate the GPU.

30
drivers/gpu/nova-core/gsp.rs
View File

@ -115,11 +115,11 @@ fn init(view: CoherentView<'_, Self>, start: DmaAddress) -> Result<()> {
/// then pp points to index into the buffer where the next logging entry will
/// be written. Therefore, the logging data is valid if:
/// 1 <= pp < sizeof(buffer)/sizeof(u64)
struct LogBuffer(Coherent<[u8; LOG_BUFFER_SIZE]>);
struct LogBuffer<'a>(Coherent<'a, [u8; LOG_BUFFER_SIZE]>);
impl LogBuffer {
impl<'a> LogBuffer<'a> {
/// Creates a new `LogBuffer` mapped on `dev`.
fn new(dev: &device::Device<device::Bound>) -> Result<Self> {
fn new(dev: &'a device::Device<device::Bound>) -> Result<Self> {
let obj = Self(Coherent::zeroed(dev, GFP_KERNEL)?);
let start_addr = obj.0.dma_address();
@ -135,33 +135,33 @@ fn new(dev: &device::Device<device::Bound>) -> Result<Self> {
}
}
struct LogBuffers {
struct LogBuffers<'a> {
/// Init log buffer.
loginit: LogBuffer,
loginit: LogBuffer<'a>,
/// Interrupts log buffer.
logintr: LogBuffer,
logintr: LogBuffer<'a>,
/// RM log buffer.
logrm: LogBuffer,
logrm: LogBuffer<'a>,
}
/// GSP runtime data.
#[pin_data]
pub(crate) struct Gsp {
pub(crate) struct Gsp<'gsp> {
/// Libos arguments.
pub(crate) libos: Coherent<[LibosMemoryRegionInitArgument]>,
pub(crate) libos: Coherent<'gsp, [LibosMemoryRegionInitArgument]>,
/// Log buffers, optionally exposed via debugfs.
#[pin]
logs: debugfs::Scope<LogBuffers>,
logs: debugfs::Scope<LogBuffers<'gsp>>,
/// Command queue.
#[pin]
pub(crate) cmdq: Cmdq,
pub(crate) cmdq: Cmdq<'gsp>,
/// RM arguments.
rmargs: Coherent<GspArgumentsPadded>,
rmargs: Coherent<'gsp, GspArgumentsPadded>,
}
impl Gsp {
impl<'gsp> Gsp<'gsp> {
// Creates an in-place initializer for a `Gsp` manager for `pdev`.
pub(crate) fn new(pdev: &pci::Device<device::Bound>) -> impl PinInit<Self, Error> + '_ {
pub(crate) fn new(pdev: &'gsp pci::Device<device::Bound>) -> impl PinInit<Self, Error> + 'gsp {
pin_init::pin_init_scope(move || {
let dev = pdev.as_ref();
@ -223,4 +223,4 @@ pub(crate) fn get_static_info(&self, bar: Bar0<'_>) -> Result<commands::GetGspSt
}
/// Opaque bundle required to unload the GSP. Created by [`Gsp::boot`], consumed by [`Gsp::unload`].
pub(crate) struct UnloadBundle(KBox<dyn hal::UnloadBundle>);
pub(crate) struct UnloadBundle<'a>(KBox<dyn hal::UnloadBundle + 'a>);

10
drivers/gpu/nova-core/gsp/boot.rs
View File

@ -22,7 +22,7 @@
},
};
impl super::Gsp {
impl<'gsp> super::Gsp<'gsp> {
/// Attempt to boot the GSP.
///
/// This is a GPU-dependent and complex procedure that involves loading firmware files from
@ -33,8 +33,8 @@ impl super::Gsp {
/// [`Self::unload`]) returned.
pub(crate) fn boot(
self: Pin<&mut Self>,
mut ctx: super::GspBootContext<'_, '_>,
) -> Result<Option<super::UnloadBundle>> {
mut ctx: super::GspBootContext<'_, 'gsp>,
) -> Result<Option<super::UnloadBundle<'gsp>>> {
let pdev = ctx.pdev;
let bar = ctx.bar;
let chipset = ctx.chipset;
@ -88,7 +88,7 @@ pub(crate) fn boot(
/// Shut down the GSP and wait until it is offline.
fn shutdown_gsp(
cmdq: &Cmdq,
cmdq: &Cmdq<'_>,
bar: Bar0<'_>,
gsp_falcon: &Falcon<'_, Gsp>,
mode: commands::PowerStateLevel,
@ -113,7 +113,7 @@ fn shutdown_gsp(
pub(crate) fn unload(
&self,
mut ctx: super::GspBootContext<'_, '_>,
unload_bundle: Option<super::UnloadBundle>,
unload_bundle: Option<super::UnloadBundle<'_>>,
) -> Result {
let dev = ctx.dev();

35
drivers/gpu/nova-core/gsp/cmdq.rs
View File

@ -25,10 +25,7 @@
new_mutex,
prelude::*,
ptr,
sync::{
aref::ARef,
Mutex, //
},
sync::Mutex,
time::Delta,
transmute::{
AsBytes,
@ -230,19 +227,19 @@ unsafe impl FromBytes for GspMem {}
/// pointer and the GSP read pointer. This region is returned by [`Self::driver_write_area`].
/// * The driver owns (i.e. can read from) the part of the GSP message queue between the CPU read
/// pointer and the GSP write pointer. This region is returned by [`Self::driver_read_area`].
struct DmaGspMem(Coherent<GspMem>);
struct DmaGspMem<'a>(Coherent<'a, GspMem>);
impl DmaGspMem {
impl<'a> DmaGspMem<'a> {
/// Allocate a new instance and map it for `dev`.
fn new(dev: &device::Device<device::Bound>) -> Result<Self> {
fn new(dev: &'a device::Device<device::Bound>) -> Result<Self> {
const MSGQ_SIZE: u32 = num::usize_into_u32::<{ size_of::<Msgq>() }>();
const RX_HDR_OFF: u32 = num::usize_into_u32::<{ mem::offset_of!(Msgq, rx) }>();
let mut gsp_mem = CoherentBox::<GspMem>::zeroed(dev, GFP_KERNEL)?;
let mut gsp_mem = CoherentBox::<'_, GspMem>::zeroed(dev, GFP_KERNEL)?;
gsp_mem.cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES);
gsp_mem.cpuq.rx = MsgqRxHeader::new();
let gsp_mem: Coherent<_> = gsp_mem.into();
let gsp_mem: Coherent<'_, _> = gsp_mem.into();
PteArray::init(io_project!(gsp_mem, .ptes), gsp_mem.dma_address())?;
Ok(Self(gsp_mem))
@ -483,15 +480,15 @@ struct GspMessage<'a> {
/// Provides the ability to send commands and receive messages from the GSP using a shared memory
/// area.
#[pin_data]
pub(crate) struct Cmdq {
pub(crate) struct Cmdq<'cmdq> {
/// Inner mutex-protected state.
#[pin]
inner: Mutex<CmdqInner>,
inner: Mutex<CmdqInner<'cmdq>>,
/// DMA address of the command queue's shared memory region.
pub(super) dma_addr: DmaAddress,
}
impl Cmdq {
impl<'cmdq> Cmdq<'cmdq> {
/// Offset of the data after the PTEs.
const POST_PTE_OFFSET: usize = core::mem::offset_of!(GspMem, cpuq);
@ -512,14 +509,16 @@ impl Cmdq {
pub(super) const RECEIVE_TIMEOUT: Delta = Delta::from_secs(5);
/// Creates a new command queue for `dev`.
pub(crate) fn new(dev: &device::Device<device::Bound>) -> impl PinInit<Self, Error> + '_ {
pub(crate) fn new(
dev: &'cmdq device::Device<device::Bound>,
) -> impl PinInit<Self, Error> + 'cmdq {
pin_init_scope(move || {
let gsp_mem = DmaGspMem::new(dev)?;
Ok(try_pin_init!(Self {
dma_addr: gsp_mem.0.dma_address(),
inner <- new_mutex!(CmdqInner {
dev: dev.into(),
dev,
gsp_mem,
seq: 0,
}),
@ -610,16 +609,16 @@ pub(crate) fn receive_msg<M: MessageFromGsp>(&self, timeout: Delta) -> Result<M>
}
/// Inner mutex protected state of [`Cmdq`].
struct CmdqInner {
struct CmdqInner<'a> {
/// Device this command queue belongs to.
dev: ARef<device::Device>,
dev: &'a device::Device,
/// Current command sequence number.
seq: u32,
/// Memory area shared with the GSP for communicating commands and messages.
gsp_mem: DmaGspMem,
gsp_mem: DmaGspMem<'a>,
}
impl CmdqInner {
impl CmdqInner<'_> {
/// Timeout for waiting for space on the command queue.
const ALLOCATE_TIMEOUT: Delta = Delta::from_secs(1);

2
drivers/gpu/nova-core/gsp/commands.rs
View File

@ -187,7 +187,7 @@ fn read(
}
/// Waits for GSP initialization to complete.
pub(crate) fn wait_gsp_init_done(cmdq: &Cmdq) -> Result {
pub(crate) fn wait_gsp_init_done(cmdq: &Cmdq<'_>) -> Result {
loop {
match cmdq.receive_msg::<GspInitDone>(Cmdq::RECEIVE_TIMEOUT) {
Ok(_) => break Ok(()),

12
drivers/gpu/nova-core/gsp/fw.rs
View File

@ -179,7 +179,7 @@ impl GspFwWprMeta {
/// Returns an initializer for a `GspFwWprMeta` suitable for booting `gsp_firmware` using the
/// framebuffer ranges `ranges`.
pub(crate) fn from_ranges<'a>(
gsp_firmware: &'a GspFirmware,
gsp_firmware: &'a GspFirmware<'_>,
ranges: &'a FbRanges,
) -> impl Init<Self> + 'a {
let init_inner = init!(bindings::GspFwWprMeta {
@ -231,7 +231,7 @@ pub(crate) fn from_ranges<'a>(
///
/// The region offsets are left at zero: the ACR ucode computes them when it sets up WPR2.
pub(crate) fn from_sizes<'a>(
gsp_firmware: &'a GspFirmware,
gsp_firmware: &'a GspFirmware<'_>,
sizes: &'a FbSizes,
) -> impl Init<Self> + 'a {
/// VGA workspace size to reserve at the end of the framebuffer, in bytes.
@ -665,7 +665,7 @@ unsafe impl FromBytes for LibosMemoryRegionInitArgument {}
impl LibosMemoryRegionInitArgument {
pub(crate) fn new<'a, A: AsBytes + FromBytes + KnownSize + ?Sized>(
name: &'static str,
obj: &'a Coherent<A>,
obj: &'a Coherent<'_, A>,
) -> impl Init<Self> + 'a {
/// Generates the `ID8` identifier required for some GSP objects.
fn id8(name: &str) -> u64 {
@ -897,7 +897,7 @@ pub(crate) struct GspArgumentsCached {
impl GspArgumentsCached {
/// Creates the arguments for starting the GSP up using `cmdq` as its command queue.
pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ {
pub(crate) fn new<'a, 'b>(cmdq: &'a Cmdq<'b>) -> impl Init<Self> + use<'a, 'b> {
let init_inner = init!(bindings::GSP_ARGUMENTS_CACHED {
messageQueueInitArguments <- MessageQueueInitArguments::new(cmdq),
bDmemStack: 1,
@ -924,7 +924,7 @@ pub(crate) struct GspArgumentsPadded {
}
impl GspArgumentsPadded {
pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ {
pub(crate) fn new<'a, 'b>(cmdq: &'a Cmdq<'b>) -> impl Init<Self> + use<'a, 'b> {
init!(GspArgumentsPadded {
inner <- GspArgumentsCached::new(cmdq),
..Zeroable::init_zeroed()
@ -944,7 +944,7 @@ unsafe impl FromBytes for GspArgumentsPadded {}
impl MessageQueueInitArguments {
/// Creates a new init arguments structure for `cmdq`.
fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ {
fn new<'a, 'b>(cmdq: &'a Cmdq<'b>) -> impl Init<Self> + use<'a, 'b> {
init!(MessageQueueInitArguments {
sharedMemPhysAddr: cmdq.dma_addr,
pageTableEntryCount: num::usize_into_u32::<{ Cmdq::NUM_PTES }>(),

14
drivers/gpu/nova-core/gsp/hal.rs
View File

@ -35,12 +35,12 @@ pub(super) trait GspHal: Send {
///
/// Upon success, returns the [`crate::gsp::UnloadBundle`] to use with [`Gsp::unload`], if one
/// could be created.
fn boot(
fn boot<'gpu>(
&self,
gsp: &Gsp,
ctx: &mut GspBootContext<'_, '_>,
gsp_fw: &GspFirmware,
) -> Result<Option<crate::gsp::UnloadBundle>>;
gsp: &Gsp<'gpu>,
ctx: &mut GspBootContext<'_, 'gpu>,
gsp_fw: &GspFirmware<'gpu>,
) -> Result<Option<super::UnloadBundle<'gpu>>>;
/// Performs HAL-specific post-GSP boot tasks.
///
@ -48,9 +48,9 @@ fn boot(
/// after the initialization commands have been pushed onto its queue.
fn post_boot(
&self,
_gsp: &Gsp,
_gsp: &Gsp<'_>,
_ctx: &mut GspBootContext<'_, '_>,
_gsp_fw: &GspFirmware,
_gsp_fw: &GspFirmware<'_>,
) -> Result {
Ok(())
}

16
drivers/gpu/nova-core/gsp/hal/gh100.rs
View File

@ -58,7 +58,7 @@ fn combined_addr(&self) -> u64 {
fn lockdown_released_or_error(
&self,
gsp_falcon: &Falcon<'_, GspEngine>,
fmc_boot_params: &Coherent<GspFmcBootParams>,
fmc_boot_params: &Coherent<'_, GspFmcBootParams>,
) -> bool {
// GSP-FMC normally clears the boot parameters address from the mailboxes early during
// boot. If the address is still there, keep polling rather than treating it as an error.
@ -75,7 +75,7 @@ fn lockdown_released_or_error(
fn wait_for_gsp_lockdown_release(
dev: &device::Device<device::Bound>,
gsp_falcon: &Falcon<'_, GspEngine>,
fmc_boot_params: &Coherent<GspFmcBootParams>,
fmc_boot_params: &Coherent<'_, GspFmcBootParams>,
) -> Result {
dev_dbg!(dev, "Waiting for GSP lockdown release\n");
@ -141,12 +141,12 @@ impl GspHal for Gh100 {
///
/// This path uses FSP to establish a chain of trust and boot GSP-FMC. FSP handles
/// the GSP boot internally - no manual GSP reset/boot is needed.
fn boot(
fn boot<'gpu>(
&self,
gsp: &Gsp,
ctx: &mut GspBootContext<'_, '_>,
gsp_fw: &GspFirmware,
) -> Result<Option<crate::gsp::UnloadBundle>> {
gsp: &Gsp<'gpu>,
ctx: &mut GspBootContext<'_, 'gpu>,
gsp_fw: &GspFirmware<'gpu>,
) -> Result<Option<crate::gsp::UnloadBundle<'gpu>>> {
let dev = ctx.dev();
let chipset = ctx.chipset;
let gsp_falcon = ctx.gsp_falcon;
@ -159,7 +159,7 @@ fn boot(
let args = FmcBootArgs::new(dev, chipset, wpr_meta, &gsp.libos, false)?;
let unload_bundle = crate::gsp::UnloadBundle(
KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox<dyn UnloadBundle>
KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox<dyn UnloadBundle + 'gpu>
);
// Wait for the GSP RISC-V core to halt in case of error. We create this guard after `args`

41
drivers/gpu/nova-core/gsp/hal/tu102.rs
View File

@ -52,34 +52,33 @@
//
// Since there are two variants of the prepared firmware (with and without a bootloader), this type
// abstracts the difference.
enum FwsecUnloadFirmware {
enum FwsecUnloadFirmware<'a> {
WithoutBl(FwsecFirmware),
WithBl(FwsecFirmwareWithBl),
WithBl(FwsecFirmwareWithBl<'a>),
}
impl FwsecUnloadFirmware {
impl FwsecUnloadFirmware<'_> {
/// Runs the FWSEC SB firmware.
fn run(
&self,
dev: &device::Device<device::Bound>,
bar: Bar0<'_>,
gsp_falcon: &Falcon<'_, GspEngine>,
) -> Result {
match self {
Self::WithoutBl(fw) => fw.run(dev, gsp_falcon),
Self::WithBl(fw) => fw.run(dev, gsp_falcon, bar),
Self::WithBl(fw) => fw.run(dev, gsp_falcon),
}
}
}
// Contains the firmware required to fully reset GSP on chipsets where the GSP is started using
// FWSEC/Booter.
struct Sec2UnloadBundle {
fwsec_sb: FwsecUnloadFirmware,
struct Sec2UnloadBundle<'a> {
fwsec_sb: FwsecUnloadFirmware<'a>,
booter_unloader: BooterFirmware,
}
impl UnloadBundle for Sec2UnloadBundle {
impl UnloadBundle for Sec2UnloadBundle<'_> {
fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result {
let dev = ctx.dev();
let bar = ctx.bar;
@ -88,7 +87,7 @@ fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result {
// Log errors but keep going if it fails.
let fwsec_sb_res = self
.fwsec_sb
.run(dev, bar, ctx.gsp_falcon)
.run(dev, ctx.gsp_falcon)
.inspect_err(|e| dev_err!(dev, "FWSEC-SB failed to run: {:?}\n", e));
// Remove WPR2 region if set.
@ -168,7 +167,7 @@ fn run_fwsec_frts(
if self.needs_fwsec_bootloader {
let fwsec_frts_bl = FwsecFirmwareWithBl::new(fwsec_frts, dev, chipset)?;
// Load and run the bootloader, which will load FWSEC-FRTS and run it.
fwsec_frts_bl.run(dev, falcon, bar)?;
fwsec_frts_bl.run(dev, falcon)?;
} else {
// Load and run FWSEC-FRTS directly.
fwsec_frts.run(dev, falcon)?;
@ -213,14 +212,14 @@ fn run_fwsec_frts(
}
/// Load and prepare the resources required to properly reset the GSP after it has been stopped.
fn build_unload_bundle(
fn build_unload_bundle<'gpu>(
&self,
dev: &device::Device<device::Bound>,
dev: &'gpu device::Device<device::Bound>,
chipset: Chipset,
bios: &Vbios,
gsp_falcon: &Falcon<'_, GspEngine>,
sec2_falcon: &Falcon<'_, Sec2>,
) -> Result<crate::gsp::UnloadBundle> {
) -> Result<crate::gsp::UnloadBundle<'gpu>> {
// Load the FWSEC SB firmware, as well as its bootloader if required.
let fwsec_sb = FwsecFirmware::new(dev, gsp_falcon, bios, FwsecCommand::Sb)?;
let fwsec_sb = if self.needs_fwsec_bootloader {
@ -241,18 +240,18 @@ fn build_unload_bundle(
},
GFP_KERNEL,
)
.map(|b| crate::gsp::UnloadBundle(b))
.map(|b| crate::gsp::UnloadBundle(b as KBox<dyn UnloadBundle + 'gpu>))
.map_err(Into::into)
}
}
impl GspHal for Tu102 {
fn boot(
fn boot<'gpu>(
&self,
gsp: &Gsp,
ctx: &mut GspBootContext<'_, '_>,
gsp_fw: &GspFirmware,
) -> Result<Option<crate::gsp::UnloadBundle>> {
gsp: &Gsp<'gpu>,
ctx: &mut GspBootContext<'_, 'gpu>,
gsp_fw: &GspFirmware<'gpu>,
) -> Result<Option<crate::gsp::UnloadBundle<'gpu>>> {
let dev = ctx.dev();
let bar = ctx.bar;
let chipset = ctx.chipset;
@ -317,9 +316,9 @@ fn boot(
fn post_boot(
&self,
gsp: &Gsp,
gsp: &Gsp<'_>,
ctx: &mut GspBootContext<'_, '_>,
gsp_fw: &GspFirmware,
gsp_fw: &GspFirmware<'_>,
) -> Result {
GspSequencer::run(&gsp.cmdq, ctx, &gsp.libos, gsp_fw.bootloader.app_version)?;

9
drivers/gpu/nova-core/gsp/regs.rs
View File

@ -2,11 +2,16 @@
use kernel::io::register;
use crate::regs::NV_PBUS_SW_SCRATCH;
use crate::{
driver::NovaRegisters,
regs::NV_PBUS_SW_SCRATCH, //
};
// PGSP
register! {
base: NovaRegisters;
pub(super) NV_PGSP_QUEUE_HEAD(u32) @ 0x00110c00 {
31:0 address;
}
@ -15,6 +20,8 @@
// PBUS
register! {
base: NovaRegisters;
/// Scratch register 0xe used as FRTS firmware error code.
pub(super) NV_PBUS_SW_SCRATCH_0E_FRTS_ERR(u32) => NV_PBUS_SW_SCRATCH[0xe] {
31:16 frts_err_code;

6
drivers/gpu/nova-core/gsp/sequencer.rs
View File

@ -138,7 +138,7 @@ pub(crate) struct GspSequencer<'a> {
/// GSP falcon for core operations.
gsp_falcon: &'a Falcon<'a, Gsp>,
/// LibOS memory region init arguments.
libos: &'a Coherent<[LibosMemoryRegionInitArgument]>,
libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>,
/// Bootloader application version.
bootloader_app_version: u32,
/// Device for logging.
@ -338,9 +338,9 @@ fn next(&mut self) -> Option<Self::Item> {
impl<'a> GspSequencer<'a> {
pub(crate) fn run(
cmdq: &Cmdq,
cmdq: &Cmdq<'_>,
ctx: &'a GspBootContext<'_, '_>,
libos: &'a Coherent<[LibosMemoryRegionInitArgument]>,
libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>,
bootloader_app_version: u32,
) -> Result {
let seq_info = loop {

115
drivers/gpu/nova-core/regs.rs
View File

@ -4,8 +4,8 @@
use kernel::{
io::{
register,
register::WithBase,
Io, //
Io,
Mmio, //
},
prelude::*,
sizes::SizeConstants,
@ -13,19 +13,18 @@
};
use crate::{
driver::Bar0,
driver::NovaRegisters,
falcon::{
DmaTrfCmdSize,
FalconCoreRev,
FalconCoreRevSubversion,
FalconEngine,
FalconFbifMemType,
FalconFbifTarget,
FalconMem,
FalconModSelAlgo,
FalconSecurityModel,
PFalcon2Base,
PFalconBase,
PFalcon2Registers,
PFalconRegisters,
PeregrineCoreSelect, //
},
gpu::{
@ -37,6 +36,8 @@
// PMC
register! {
base: NovaRegisters;
/// Basic revision information about the GPU.
pub(crate) NV_PMC_BOOT_0(u32) @ 0x00000000 {
/// Lower bits of the architecture.
@ -108,6 +109,8 @@ fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result {
// PBUS
register! {
base: NovaRegisters;
pub(crate) NV_PBUS_SW_SCRATCH(u32)[64] @ 0x00001400 {}
}
@ -121,6 +124,8 @@ fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result {
// number.
register! {
base: NovaRegisters;
/// Boot Sequence Interface (BSI) register used to determine
/// if GSP reload/resume has completed during the boot process.
pub(crate) NV_PGC6_BSI_SECURE_SCRATCH_14(u32) @ 0x001180f8 {
@ -175,6 +180,8 @@ pub(crate) fn usable_fb_size(self) -> u64 {
pub(crate) const NV_FUSE_OPT_FPF_SIZE: usize = 16;
register! {
base: NovaRegisters;
pub(crate) NV_FUSE_OPT_FPF_NVDEC_UCODE1_VERSION(u32)[NV_FUSE_OPT_FPF_SIZE] @ 0x00824100 {
15:0 data => u16;
}
@ -191,30 +198,32 @@ pub(crate) fn usable_fb_size(self) -> u64 {
// PFALCON
register! {
pub(crate) NV_PFALCON_FALCON_IRQSCLR(u32) @ PFalconBase + 0x00000004 {
base: PFalconRegisters;
pub(crate) NV_PFALCON_FALCON_IRQSCLR(u32) @ 0x00000004 {
6:6 swgen0 => bool;
4:4 halt => bool;
}
pub(crate) NV_PFALCON_FALCON_MAILBOX0(u32) @ PFalconBase + 0x00000040 {
pub(crate) NV_PFALCON_FALCON_MAILBOX0(u32) @ 0x00000040 {
31:0 value => u32;
}
pub(crate) NV_PFALCON_FALCON_MAILBOX1(u32) @ PFalconBase + 0x00000044 {
pub(crate) NV_PFALCON_FALCON_MAILBOX1(u32) @ 0x00000044 {
31:0 value => u32;
}
/// Used to store version information about the firmware running
/// on the Falcon processor.
pub(crate) NV_PFALCON_FALCON_OS(u32) @ PFalconBase + 0x00000080 {
pub(crate) NV_PFALCON_FALCON_OS(u32) @ 0x00000080 {
31:0 value => u32;
}
pub(crate) NV_PFALCON_FALCON_RM(u32) @ PFalconBase + 0x00000084 {
pub(crate) NV_PFALCON_FALCON_RM(u32) @ 0x00000084 {
31:0 value => u32;
}
pub(crate) NV_PFALCON_FALCON_HWCFG2(u32) @ PFalconBase + 0x000000f4 {
pub(crate) NV_PFALCON_FALCON_HWCFG2(u32) @ 0x000000f4 {
/// Signal indicating that reset is completed (GA102+).
31:31 reset_ready => bool;
/// RISC-V branch privilege lockdown bit.
@ -224,17 +233,17 @@ pub(crate) fn usable_fb_size(self) -> u64 {
10:10 riscv => bool;
}
pub(crate) NV_PFALCON_FALCON_CPUCTL(u32) @ PFalconBase + 0x00000100 {
pub(crate) NV_PFALCON_FALCON_CPUCTL(u32) @ 0x00000100 {
6:6 alias_en => bool;
4:4 halted => bool;
1:1 startcpu => bool;
}
pub(crate) NV_PFALCON_FALCON_BOOTVEC(u32) @ PFalconBase + 0x00000104 {
pub(crate) NV_PFALCON_FALCON_BOOTVEC(u32) @ 0x00000104 {
31:0 value => u32;
}
pub(crate) NV_PFALCON_FALCON_DMACTL(u32) @ PFalconBase + 0x0000010c {
pub(crate) NV_PFALCON_FALCON_DMACTL(u32) @ 0x0000010c {
7:7 secure_stat => bool;
6:3 dmaq_num;
2:2 imem_scrubbing => bool;
@ -242,15 +251,15 @@ pub(crate) fn usable_fb_size(self) -> u64 {
0:0 require_ctx => bool;
}
pub(crate) NV_PFALCON_FALCON_DMATRFBASE(u32) @ PFalconBase + 0x00000110 {
pub(crate) NV_PFALCON_FALCON_DMATRFBASE(u32) @ 0x00000110 {
31:0 base => u32;
}
pub(crate) NV_PFALCON_FALCON_DMATRFMOFFS(u32) @ PFalconBase + 0x00000114 {
pub(crate) NV_PFALCON_FALCON_DMATRFMOFFS(u32) @ 0x00000114 {
23:0 offs;
}
pub(crate) NV_PFALCON_FALCON_DMATRFCMD(u32) @ PFalconBase + 0x00000118 {
pub(crate) NV_PFALCON_FALCON_DMATRFCMD(u32) @ 0x00000118 {
16:16 set_dmtag;
14:12 ctxdma;
10:8 size ?=> DmaTrfCmdSize;
@ -261,15 +270,15 @@ pub(crate) fn usable_fb_size(self) -> u64 {
0:0 full => bool;
}
pub(crate) NV_PFALCON_FALCON_DMATRFFBOFFS(u32) @ PFalconBase + 0x0000011c {
pub(crate) NV_PFALCON_FALCON_DMATRFFBOFFS(u32) @ 0x0000011c {
31:0 offs => u32;
}
pub(crate) NV_PFALCON_FALCON_DMATRFBASE1(u32) @ PFalconBase + 0x00000128 {
pub(crate) NV_PFALCON_FALCON_DMATRFBASE1(u32) @ 0x00000128 {
8:0 base;
}
pub(crate) NV_PFALCON_FALCON_HWCFG1(u32) @ PFalconBase + 0x0000012c {
pub(crate) NV_PFALCON_FALCON_HWCFG1(u32) @ 0x0000012c {
/// Core revision subversion.
7:6 core_rev_subversion => FalconCoreRevSubversion;
/// Security model.
@ -278,12 +287,12 @@ pub(crate) fn usable_fb_size(self) -> u64 {
3:0 core_rev ?=> FalconCoreRev;
}
pub(crate) NV_PFALCON_FALCON_CPUCTL_ALIAS(u32) @ PFalconBase + 0x00000130 {
pub(crate) NV_PFALCON_FALCON_CPUCTL_ALIAS(u32) @ 0x00000130 {
1:1 startcpu => bool;
}
/// IMEM access control register. Up to 4 ports are available for IMEM access.
pub(crate) NV_PFALCON_FALCON_IMEMC(u32)[4, stride = 16] @ PFalconBase + 0x00000180 {
pub(crate) NV_PFALCON_FALCON_IMEMC(u32)[4, stride = 16] @ 0x00000180 {
/// Access secure IMEM.
28:28 secure => bool;
/// Auto-increment on write.
@ -294,17 +303,17 @@ pub(crate) fn usable_fb_size(self) -> u64 {
/// IMEM data register. Reading/writing this register accesses IMEM at the address
/// specified by the corresponding IMEMC register.
pub(crate) NV_PFALCON_FALCON_IMEMD(u32)[4, stride = 16] @ PFalconBase + 0x00000184 {
pub(crate) NV_PFALCON_FALCON_IMEMD(u32)[4, stride = 16] @ 0x00000184 {
31:0 data;
}
/// IMEM tag register. Used to set the tag for the current IMEM block.
pub(crate) NV_PFALCON_FALCON_IMEMT(u32)[4, stride = 16] @ PFalconBase + 0x00000188 {
pub(crate) NV_PFALCON_FALCON_IMEMT(u32)[4, stride = 16] @ 0x00000188 {
15:0 tag;
}
/// DMEM access control register. Up to 8 ports are available for DMEM access.
pub(crate) NV_PFALCON_FALCON_DMEMC(u32)[8, stride = 8] @ PFalconBase + 0x000001c0 {
pub(crate) NV_PFALCON_FALCON_DMEMC(u32)[8, stride = 8] @ 0x000001c0 {
/// Auto-increment on write.
24:24 aincw => bool;
/// DMEM block and word offset.
@ -313,29 +322,29 @@ pub(crate) fn usable_fb_size(self) -> u64 {
/// DMEM data register. Reading/writing this register accesses DMEM at the address
/// specified by the corresponding DMEMC register.
pub(crate) NV_PFALCON_FALCON_DMEMD(u32)[8, stride = 8] @ PFalconBase + 0x000001c4 {
pub(crate) NV_PFALCON_FALCON_DMEMD(u32)[8, stride = 8] @ 0x000001c4 {
31:0 data;
}
/// Actually known as `NV_PSEC_FALCON_ENGINE` and `NV_PGSP_FALCON_ENGINE` depending on the
/// falcon instance.
pub(crate) NV_PFALCON_FALCON_ENGINE(u32) @ PFalconBase + 0x000003c0 {
pub(crate) NV_PFALCON_FALCON_ENGINE(u32) @ 0x000003c0 {
0:0 reset => bool;
}
pub(crate) NV_PFALCON_FBIF_TRANSCFG(u32)[8] @ PFalconBase + 0x00000600 {
pub(crate) NV_PFALCON_FBIF_TRANSCFG(u32)[8] @ 0x00000600 {
2:2 mem_type => FalconFbifMemType;
1:0 target ?=> FalconFbifTarget;
}
pub(crate) NV_PFALCON_FBIF_CTL(u32) @ PFalconBase + 0x00000624 {
pub(crate) NV_PFALCON_FBIF_CTL(u32) @ 0x00000624 {
7:7 allow_phys_no_ctx => bool;
}
// Falcon EMEM PIO registers (used by FSP on Hopper/Blackwell).
// These provide the falcon external memory communication interface.
pub(crate) NV_PFALCON_FALCON_EMEMC(u32) @ PFalconBase + 0x00000ac0 {
pub(crate) NV_PFALCON_FALCON_EMEMC(u32) @ 0x00000ac0 {
/// EMEM byte offset (4-byte aligned) within the block.
7:2 offs;
/// EMEM block to access.
@ -346,7 +355,7 @@ pub(crate) fn usable_fb_size(self) -> u64 {
25:25 aincr => bool;
}
pub(crate) NV_PFALCON_FALCON_EMEMD(u32) @ PFalconBase + 0x00000ac4 {
pub(crate) NV_PFALCON_FALCON_EMEMD(u32) @ 0x00000ac4 {
31:0 data => u32;
}
}
@ -372,13 +381,13 @@ pub(crate) fn with_falcon_mem(self, mem: FalconMem) -> Self {
impl NV_PFALCON_FALCON_ENGINE {
/// Resets the falcon
pub(crate) fn reset_engine<E: FalconEngine>(bar: Bar0<'_>) {
bar.update(Self::of::<E>(), |r| r.with_reset(true));
pub(crate) fn reset_engine(pfalcon: Mmio<'_, PFalconRegisters>) {
pfalcon.update(NV_PFALCON_FALCON_ENGINE, |r| r.with_reset(true));
// TIMEOUT: falcon engine should not take more than 10us to reset.
time::delay::fsleep(time::Delta::from_micros(10));
bar.update(Self::of::<E>(), |r| r.with_reset(false));
pfalcon.update(NV_PFALCON_FALCON_ENGINE, |r| r.with_reset(false));
}
}
@ -392,21 +401,23 @@ pub(crate) fn mem_scrubbing_done(self) -> bool {
/* PFALCON2 */
register! {
pub(crate) NV_PFALCON2_FALCON_MOD_SEL(u32) @ PFalcon2Base + 0x00000180 {
base: PFalcon2Registers;
pub(crate) NV_PFALCON2_FALCON_MOD_SEL(u32) @ 0x00000180 {
7:0 algo ?=> FalconModSelAlgo;
}
pub(crate) NV_PFALCON2_FALCON_BROM_CURR_UCODE_ID(u32) @ PFalcon2Base + 0x00000198 {
pub(crate) NV_PFALCON2_FALCON_BROM_CURR_UCODE_ID(u32) @ 0x00000198 {
7:0 ucode_id => u8;
}
pub(crate) NV_PFALCON2_FALCON_BROM_ENGIDMASK(u32) @ PFalcon2Base + 0x0000019c {
pub(crate) NV_PFALCON2_FALCON_BROM_ENGIDMASK(u32) @ 0x0000019c {
31:0 value => u32;
}
/// OpenRM defines this as a register array, but doesn't specify its size and only uses its
/// first element. Be conservative until we know the actual size or need to use more registers.
pub(crate) NV_PFALCON2_FALCON_BROM_PARAADDR(u32)[1] @ PFalcon2Base + 0x00000210 {
pub(crate) NV_PFALCON2_FALCON_BROM_PARAADDR(u32)[1] @ 0x00000210 {
31:0 value => u32;
}
}
@ -414,21 +425,23 @@ pub(crate) fn mem_scrubbing_done(self) -> bool {
// PRISCV
register! {
base: PFalcon2Registers;
/// RISC-V status register for debug (Turing and GA100 only).
/// Reflects current RISC-V core status.
pub(crate) NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS(u32) @ PFalcon2Base + 0x00000240 {
pub(crate) NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS(u32) @ 0x00000240 {
/// RISC-V core active/inactive status.
0:0 active_stat => bool;
}
/// GA102 and later.
pub(crate) NV_PRISCV_RISCV_CPUCTL(u32) @ PFalcon2Base + 0x00000388 {
pub(crate) NV_PRISCV_RISCV_CPUCTL(u32) @ 0x00000388 {
7:7 active_stat => bool;
4:4 halted => bool;
}
/// GA102 and later.
pub(crate) NV_PRISCV_RISCV_BCR_CTRL(u32) @ PFalcon2Base + 0x00000668 {
pub(crate) NV_PRISCV_RISCV_BCR_CTRL(u32) @ 0x00000668 {
8:8 br_fetch => bool;
4:4 core_select => PeregrineCoreSelect;
0:0 valid => bool;
@ -439,6 +452,8 @@ pub(crate) fn mem_scrubbing_done(self) -> bool {
// These registers manage falcon EMEM communication queues.
register! {
base: NovaRegisters;
pub(crate) NV_PFSP_QUEUE_HEAD(u32)[8] @ 0x008f2c00 {
31:0 address => u32;
}
@ -462,9 +477,13 @@ pub(crate) fn mem_scrubbing_done(self) -> bool {
pub(crate) mod gm107 {
use kernel::io::register;
use crate::driver::NovaRegisters;
// FUSE
register! {
base: NovaRegisters;
pub(crate) NV_FUSE_STATUS_OPT_DISPLAY(u32) @ 0x00021c04 {
0:0 display_disabled => bool;
}
@ -474,9 +493,13 @@ pub(crate) mod gm107 {
pub(crate) mod ga100 {
use kernel::io::register;
use crate::driver::NovaRegisters;
// FUSE
register! {
base: NovaRegisters;
pub(crate) NV_FUSE_STATUS_OPT_DISPLAY(u32) @ 0x00820c04 {
0:0 display_disabled => bool;
}
@ -488,9 +511,13 @@ pub(crate) mod ga100 {
pub(crate) mod gh100 {
use kernel::io::register;
use crate::driver::NovaRegisters;
// PTHERM
register! {
base: NovaRegisters;
pub(crate) NV_THERM_I2CS_SCRATCH(u32) @ 0x000200bc {
31:0 data;
}
@ -505,9 +532,13 @@ pub(crate) mod gh100 {
pub(crate) mod gb202 {
use kernel::io::register;
use crate::driver::NovaRegisters;
// PTHERM
register! {
base: NovaRegisters;
pub(crate) NV_THERM_I2CS_SCRATCH(u32) @ 0x00ad00bc {
31:0 data;
}

11
drivers/gpu/nova-core/vbios.rs
View File

@ -16,7 +16,10 @@
};
use crate::{
driver::Bar0,
driver::{
Bar0,
NovaRegisters, //
},
firmware::{
fwsec::Bcrt30Rsa3kSignature,
FalconUCodeDesc,
@ -92,12 +95,16 @@ impl<'a> VbiosIterator<'a> {
fn rom_offset(dev: &device::Device, bar0: Bar0<'_>) -> Result<usize> {
// IFR Header in VBIOS.
register! {
base: NovaRegisters;
NV_PBUS_IFR_FMT_FIXED0(u32) @ 0x300000 {
31:0 signature;
}
}
register! {
base: NovaRegisters;
NV_PBUS_IFR_FMT_FIXED1(u32) @ 0x300004 {
30:16 fixed_data_size;
15:8 version => u8;
@ -105,6 +112,8 @@ fn rom_offset(dev: &device::Device, bar0: Bar0<'_>) -> Result<usize> {
}
register! {
base: NovaRegisters;
NV_PBUS_IFR_FMT_FIXED2(u32) @ 0x300008 {
19:0 total_data_size;
}

81
drivers/interconnect/debugfs-client.c
View File

@ -5,6 +5,7 @@
#include <linux/debugfs.h>
#include <linux/interconnect.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include "internal.h"
@ -36,6 +37,59 @@ struct debugfs_path {
struct list_head list;
};
static ssize_t icc_node_read(struct file *file, char __user *user_buf,
size_t count, loff_t *ppos)
{
char **node = file->private_data;
char *copy;
size_t len;
ssize_t ret;
scoped_guard(mutex, &debugfs_lock) {
copy = kstrdup(*node ?: "", GFP_KERNEL);
}
if (!copy)
return -ENOMEM;
len = strlen(copy);
copy[len++] = '\n';
ret = simple_read_from_buffer(user_buf, count, ppos, copy, len);
kfree(copy);
return ret;
}
static ssize_t icc_node_write(struct file *file, const char __user *user_buf,
size_t count, loff_t *ppos)
{
char **node = file->private_data;
char *old, *new;
if (*ppos)
return -EINVAL;
if (count + 1 > PAGE_SIZE)
return -E2BIG;
new = memdup_user_nul(user_buf, count);
if (IS_ERR(new))
return PTR_ERR(new);
strim(new);
scoped_guard(mutex, &debugfs_lock) {
old = *node;
*node = new;
}
kfree(old);
return count;
}
static const struct file_operations icc_node_fops = {
.open = simple_open,
.read = icc_node_read,
.write = icc_node_write,
.llseek = default_llseek,
};
static struct icc_path *get_path(const char *src, const char *dst)
{
struct debugfs_path *path;
@ -54,26 +108,19 @@ static int icc_get_set(void *data, u64 val)
char *src, *dst;
int ret = 0;
mutex_lock(&debugfs_lock);
rcu_read_lock();
src = rcu_dereference(src_node);
dst = rcu_dereference(dst_node);
guard(mutex)(&debugfs_lock);
/*
* If we've already looked up a path, then use the existing one instead
* of calling icc_get() again. This allows for updating previous BW
* votes when "get" is written to multiple times for multiple paths.
*/
cur_path = get_path(src, dst);
if (cur_path) {
rcu_read_unlock();
cur_path = get_path(src_node, dst_node);
if (cur_path)
goto out;
}
src = kstrdup(src, GFP_ATOMIC);
dst = kstrdup(dst, GFP_ATOMIC);
rcu_read_unlock();
src = kstrdup(src_node, GFP_KERNEL);
dst = kstrdup(dst_node, GFP_KERNEL);
if (!src || !dst) {
ret = -ENOMEM;
@ -105,7 +152,6 @@ static int icc_get_set(void *data, u64 val)
kfree(src);
kfree(dst);
out:
mutex_unlock(&debugfs_lock);
return ret;
}
@ -115,7 +161,7 @@ static int icc_commit_set(void *data, u64 val)
{
int ret;
mutex_lock(&debugfs_lock);
guard(mutex)(&debugfs_lock);
if (!cur_path) {
ret = -EINVAL;
@ -130,7 +176,6 @@ static int icc_commit_set(void *data, u64 val)
icc_set_tag(cur_path, tag);
ret = icc_set_bw(cur_path, avg_bw, peak_bw);
out:
mutex_unlock(&debugfs_lock);
return ret;
}
@ -162,8 +207,10 @@ int icc_debugfs_client_init(struct dentry *icc_dir)
client_dir = debugfs_create_dir("test_client", icc_dir);
debugfs_create_str("src_node", 0600, client_dir, &src_node);
debugfs_create_str("dst_node", 0600, client_dir, &dst_node);
debugfs_create_file("src_node", 0600, client_dir, &src_node,
&icc_node_fops);
debugfs_create_file("dst_node", 0600, client_dir, &dst_node,
&icc_node_fops);
debugfs_create_file("get", 0200, client_dir, NULL, &icc_get_fops);
debugfs_create_u32("avg_bw", 0600, client_dir, &avg_bw);
debugfs_create_u32("peak_bw", 0600, client_dir, &peak_bw);

2
drivers/perf/alibaba_uncore_drw_pmu.c
View File

@ -135,7 +135,7 @@ static ssize_t ali_drw_pmu_event_show(struct device *dev,
#define ALI_DRW_PMU_ATTR(_name, _func, _config) \
(&((struct dev_ext_attribute[]) { \
{ __ATTR(_name, 0444, _func, NULL), (void *)_config } \
{ __DEVICE_ATTR(_name, 0444, _func, NULL), (void *)_config } \
})[0].attr.attr)
#define ALI_DRW_PMU_FORMAT_ATTR(_name, _config) \

2
drivers/perf/arm-cci.c
View File

@ -132,7 +132,7 @@ static ssize_t __maybe_unused cci_pmu_event_show(struct device *dev,
#define CCI_EXT_ATTR_ENTRY(_name, _func, _config) \
&((struct dev_ext_attribute[]) { \
{ __ATTR(_name, S_IRUGO, _func, NULL), (void *)_config } \
{ __DEVICE_ATTR(_name, S_IRUGO, _func, NULL), (void *)_config } \
})[0].attr.attr
#define CCI_FORMAT_EXT_ATTR_ENTRY(_name, _config) \

2
drivers/perf/arm-ccn.c
View File

@ -216,7 +216,7 @@ static void arm_ccn_pmu_config_set(u64 *config, u32 node_xp, u32 type, u32 port)
#define CCN_FORMAT_ATTR(_name, _config) \
struct dev_ext_attribute arm_ccn_pmu_format_attr_##_name = \
{ __ATTR(_name, S_IRUGO, device_show_string, \
{ __DEVICE_ATTR(_name, S_IRUGO, device_show_string, \
NULL), _config }
static CCN_FORMAT_ATTR(node, "config:0-7");

2
drivers/perf/arm_cspmu/arm_cspmu.h
View File

@ -23,7 +23,7 @@
#define ARM_CSPMU_EXT_ATTR(_name, _func, _config) \
(&((struct dev_ext_attribute[]){ \
{ \
.attr = __ATTR(_name, 0444, _func, NULL), \
.attr = __DEVICE_ATTR(_name, 0444, _func, NULL),\
.var = (void *)_config \
} \
})[0].attr.attr)

2
drivers/perf/arm_dsu_pmu.c
View File

@ -69,7 +69,7 @@
#define DSU_EXT_ATTR(_name, _func, _config) \
(&((struct dev_ext_attribute[]) { \
{ \
.attr = __ATTR(_name, 0444, _func, NULL), \
.attr = __DEVICE_ATTR(_name, 0444, _func, NULL),\
.var = (void *)_config \
} \
})[0].attr.attr)

6
drivers/perf/arm_spe_pmu.c
View File

@ -172,9 +172,9 @@ static ssize_t arm_spe_pmu_cap_show_hex(struct device *dev,
return sysfs_emit(buf, "0x%llx\n", arm_spe_pmu_cap_get(spe_pmu, cap));
}
#define SPE_EXT_ATTR_ENTRY(_name, _func, _var) \
&((struct dev_ext_attribute[]) { \
{ __ATTR(_name, S_IRUGO, _func, NULL), (void *)_var } \
#define SPE_EXT_ATTR_ENTRY(_name, _func, _var) \
&((struct dev_ext_attribute[]) { \
{ __DEVICE_ATTR(_name, S_IRUGO, _func, NULL), (void *)_var } \
})[0].attr.attr
#define SPE_CAP_EXT_ATTR_ENTRY(_name, _var) \

10
drivers/perf/cxl_pmu.c
View File

@ -228,11 +228,11 @@ static int cxl_pmu_parse_caps(struct device *dev, struct cxl_pmu_info *info)
}
#define CXL_PMU_FORMAT_ATTR(_name, _format)\
(&((struct dev_ext_attribute[]) { \
{ \
.attr = __ATTR(_name, 0444, device_show_string, NULL), \
.var = (void *)_format \
} \
(&((struct dev_ext_attribute[]) { \
{ \
.attr = __DEVICE_ATTR(_name, 0444, device_show_string, NULL), \
.var = (void *)_format \
} \
})[0].attr.attr)
enum {

2
drivers/perf/fsl_imx8_ddr_perf.c
View File

@ -214,7 +214,7 @@ static ssize_t ddr_perf_filter_cap_show(struct device *dev,
#define PERF_EXT_ATTR_ENTRY(_name, _func, _var) \
(&((struct dev_ext_attribute) { \
__ATTR(_name, 0444, _func, NULL), (void *)_var \
__DEVICE_ATTR(_name, 0444, _func, NULL), (void *)_var \
}).attr.attr)
#define PERF_FILTER_EXT_ATTR_ENTRY(_name, _var) \

2
drivers/perf/fujitsu_uncore_pmu.c
View File

@ -291,7 +291,7 @@ static void fujitsu_uncore_event_read(struct perf_event *event)
#define UNCORE_PMU_FORMAT_ATTR(_name, _config) \
(&((struct dev_ext_attribute[]) { \
{ .attr = __ATTR(_name, 0444, device_show_string, NULL), \
{ .attr = __DEVICE_ATTR(_name, 0444, device_show_string, NULL), \
.var = (void *)_config, } \
})[0].attr.attr)

2
drivers/perf/hisilicon/hisi_pcie_pmu.c
View File

@ -110,7 +110,7 @@ static ssize_t hisi_pcie_event_sysfs_show(struct device *dev, struct device_attr
#define HISI_PCIE_PMU_FORMAT_ATTR(_name, _format) \
(&((struct dev_ext_attribute[]){ \
{ .attr = __ATTR(_name, 0444, device_show_string, NULL), \
{ .attr = __DEVICE_ATTR(_name, 0444, device_show_string, NULL),\
.var = (void *)_format } \
})[0].attr.attr)

6
drivers/perf/hisilicon/hisi_uncore_pmu.h
View File

@ -27,9 +27,9 @@
#define HISI_MAX_COUNTERS 0x18
#define to_hisi_pmu(p) (container_of(p, struct hisi_pmu, pmu))
#define HISI_PMU_ATTR(_name, _func, _config) \
(&((struct dev_ext_attribute[]) { \
{ __ATTR(_name, 0444, _func, NULL), (void *)_config } \
#define HISI_PMU_ATTR(_name, _func, _config) \
(&((struct dev_ext_attribute[]) { \
{ __DEVICE_ATTR(_name, 0444, _func, NULL), (void *)_config } \
})[0].attr.attr)
#define HISI_PMU_FORMAT_ATTR(_name, _config) \

6
drivers/perf/hisilicon/hns3_pmu.c
View File

@ -405,9 +405,9 @@ static ssize_t hns3_pmu_filter_mode_show(struct device *dev,
return len;
}
#define HNS3_PMU_ATTR(_name, _func, _config) \
(&((struct dev_ext_attribute[]) { \
{ __ATTR(_name, 0444, _func, NULL), (void *)_config } \
#define HNS3_PMU_ATTR(_name, _func, _config) \
(&((struct dev_ext_attribute[]) { \
{ __DEVICE_ATTR(_name, 0444, _func, NULL), (void *)_config } \
})[0].attr.attr)
#define HNS3_PMU_FORMAT_ATTR(_name, _format) \

2
drivers/perf/nvidia_t410_c2c_pmu.c
View File

@ -545,7 +545,7 @@ static struct attribute_group nv_c2c_pmu_peer_attr_group = {
#define NV_C2C_PMU_EXT_ATTR(_name, _func, _config) \
(&((struct dev_ext_attribute[]){ \
{ \
.attr = __ATTR(_name, 0444, _func, NULL), \
.attr = __DEVICE_ATTR(_name, 0444, _func, NULL),\
.var = (void *)_config \
} \
})[0].attr.attr)

12
drivers/perf/nvidia_t410_cmem_latency_pmu.c
View File

@ -435,12 +435,12 @@ static struct attribute_group cmem_lat_pmu_identifier_attr_group = {
/* Format attributes. */
#define NV_PMU_EXT_ATTR(_name, _func, _config) \
(&((struct dev_ext_attribute[]){ \
{ \
.attr = __ATTR(_name, 0444, _func, NULL), \
.var = (void *)_config \
} \
#define NV_PMU_EXT_ATTR(_name, _func, _config) \
(&((struct dev_ext_attribute[]){ \
{ \
.attr = __DEVICE_ATTR(_name, 0444, _func, NULL), \
.var = (void *)_config \
} \
})[0].attr.attr)
static struct attribute *cmem_lat_pmu_formats[] = {

8
drivers/perf/qcom_l3_pmu.c
View File

@ -609,10 +609,10 @@ static void qcom_l3_cache__event_read(struct perf_event *event)
/* formats */
#define L3CACHE_PMU_FORMAT_ATTR(_name, _config) \
(&((struct dev_ext_attribute[]) { \
{ .attr = __ATTR(_name, 0444, device_show_string, NULL), \
.var = (void *) _config, } \
#define L3CACHE_PMU_FORMAT_ATTR(_name, _config) \
(&((struct dev_ext_attribute[]) { \
{ .attr = __DEVICE_ATTR(_name, 0444, device_show_string, NULL), \
.var = (void *) _config, } \
})[0].attr.attr)
static struct attribute *qcom_l3_cache_pmu_formats[] = {

2
drivers/perf/starfive_starlink_pmu.c
View File

@ -51,7 +51,7 @@
#define STARLINK_FORMAT_ATTR(_name, _config) \
(&((struct dev_ext_attribute[]) { \
{ .attr = __ATTR(_name, 0444, starlink_pmu_sysfs_format_show, NULL), \
{ .attr = __DEVICE_ATTR(_name, 0444, starlink_pmu_sysfs_format_show, NULL), \
.var = (void *)_config, } \
})[0].attr.attr)

2
drivers/perf/xgene_pmu.c
View File

@ -164,7 +164,7 @@ enum xgene_pmu_dev_type {
*/
#define XGENE_PMU_FORMAT_ATTR(_name, _config) \
(&((struct dev_ext_attribute[]) { \
{ .attr = __ATTR(_name, S_IRUGO, device_show_string, NULL), \
{ .attr = __DEVICE_ATTR(_name, S_IRUGO, device_show_string, NULL), \
.var = (void *) _config, } \
})[0].attr.attr)

63
drivers/soundwire/debugfs.c
View File

@ -145,6 +145,38 @@ static u32 start_addr;
static size_t num_bytes;
static u8 read_buffer[MAX_CMD_BYTES];
static char *firmware_file;
static DEFINE_MUTEX(firmware_file_lock);
static ssize_t firmware_file_write(struct file *file,
const char __user *user_buf,
size_t count, loff_t *ppos)
{
char *old, *new;
if (*ppos)
return -EINVAL;
if (count + 1 > PAGE_SIZE)
return -E2BIG;
new = memdup_user_nul(user_buf, count);
if (IS_ERR(new))
return PTR_ERR(new);
strim(new);
scoped_guard(mutex, &firmware_file_lock) {
old = firmware_file;
firmware_file = new;
}
kfree(old);
return count;
}
static const struct file_operations firmware_file_fops = {
.open = simple_open,
.write = firmware_file_write,
.llseek = default_llseek,
};
static int set_command(void *data, u64 value)
{
@ -246,6 +278,7 @@ static int cmd_go(void *data, u64 value)
{
const struct firmware *fw = NULL;
struct sdw_slave *slave = data;
char *fw_name = NULL;
ktime_t start_t;
ktime_t finish_t;
int ret;
@ -265,15 +298,24 @@ static int cmd_go(void *data, u64 value)
}
if (cmd == 0) {
ret = request_firmware(&fw, firmware_file, &slave->dev);
scoped_guard(mutex, &firmware_file_lock) {
if (firmware_file)
fw_name = kstrdup(firmware_file, GFP_KERNEL);
}
if (!fw_name) {
ret = -ENOMEM;
goto out;
}
ret = request_firmware(&fw, fw_name, &slave->dev);
if (ret < 0) {
dev_err(&slave->dev, "firmware %s not found\n", firmware_file);
dev_err(&slave->dev, "firmware %s not found\n", fw_name);
goto out;
}
if (fw->size < num_bytes) {
dev_err(&slave->dev,
"firmware %s: firmware size %zd, desired %zd\n",
firmware_file, fw->size, num_bytes);
fw_name, fw->size, num_bytes);
goto out;
}
}
@ -306,6 +348,7 @@ static int cmd_go(void *data, u64 value)
out:
if (fw)
release_firmware(fw);
kfree(fw_name);
pm_runtime_mark_last_busy(&slave->dev);
pm_runtime_put(&slave->dev);
@ -358,7 +401,8 @@ void sdw_slave_debugfs_init(struct sdw_slave *slave)
debugfs_create_file("read_buffer", 0400, d, slave, &read_buffer_fops);
if (firmware_file)
debugfs_create_str("firmware_file", 0200, d, &firmware_file);
debugfs_create_file("firmware_file", 0200, d, NULL,
&firmware_file_fops);
slave->debugfs = d;
}
@ -370,8 +414,10 @@ void sdw_slave_debugfs_exit(struct sdw_slave *slave)
void sdw_debugfs_init(void)
{
if (!firmware_file)
firmware_file = kstrdup("", GFP_KERNEL);
scoped_guard(mutex, &firmware_file_lock) {
if (!firmware_file)
firmware_file = kstrdup("", GFP_KERNEL);
}
sdw_debugfs_root = debugfs_create_dir("soundwire", NULL);
}
@ -379,6 +425,7 @@ void sdw_debugfs_init(void)
void sdw_debugfs_exit(void)
{
debugfs_remove_recursive(sdw_debugfs_root);
kfree(firmware_file);
firmware_file = NULL;
scoped_guard(mutex, &firmware_file_lock) {
kfree(firmware_file);
}
}

81
fs/debugfs/file.c
View File

@ -1049,98 +1049,37 @@ ssize_t debugfs_read_file_str(struct file *file, char __user *user_buf,
return ret;
}
static ssize_t debugfs_write_file_str(struct file *file, const char __user *user_buf,
size_t count, loff_t *ppos)
{
struct dentry *dentry = F_DENTRY(file);
char *old, *new = NULL;
int pos = *ppos;
int r;
r = debugfs_file_get(dentry);
if (unlikely(r))
return r;
old = *(char **)file->private_data;
/* only allow strict concatenation */
r = -EINVAL;
if (pos && pos != strlen(old))
goto error;
r = -E2BIG;
if (pos + count + 1 > PAGE_SIZE)
goto error;
r = -ENOMEM;
new = kmalloc(pos + count + 1, GFP_KERNEL);
if (!new)
goto error;
if (pos)
memcpy(new, old, pos);
r = -EFAULT;
if (copy_from_user(new + pos, user_buf, count))
goto error;
new[pos + count] = '\0';
strim(new);
rcu_assign_pointer(*(char __rcu **)file->private_data, new);
synchronize_rcu();
kfree(old);
debugfs_file_put(dentry);
return count;
error:
kfree(new);
debugfs_file_put(dentry);
return r;
}
static const struct file_operations fops_str = {
.read = debugfs_read_file_str,
.write = debugfs_write_file_str,
.open = simple_open,
.llseek = default_llseek,
};
static const struct file_operations fops_str_ro = {
.read = debugfs_read_file_str,
.open = simple_open,
.llseek = default_llseek,
};
static const struct file_operations fops_str_wo = {
.write = debugfs_write_file_str,
.open = simple_open,
.llseek = default_llseek,
};
/**
* debugfs_create_str - create a debugfs file that is used to read and write a string value
* debugfs_create_str - create a debugfs file that is used to read a string value
* @name: a pointer to a string containing the name of the file to create.
* @mode: the permission that the file should have
* @parent: a pointer to the parent dentry for this file. This should be a
* directory dentry if set. If this parameter is %NULL, then the
* file will be created in the root of the debugfs filesystem.
* @value: a pointer to the variable that the file should read to and write
* from. This pointer and the string it points to must not be %NULL.
* @value: a pointer to the variable that the file should read from. This
* pointer and the string it points to must not be %NULL.
*
* This function creates a file in debugfs with the given name that
* contains the value of the variable @value. If the @mode variable is so
* set, it can be read from, and written to.
* contains the value of the variable @value. The file can be read from.
* Writable files are not supported; if @mode contains write permission bits,
* no file is created.
*/
void debugfs_create_str(const char *name, umode_t mode,
struct dentry *parent, char **value)
{
if (WARN_ON(!value || !*value))
return;
if (WARN(mode & 0222,
"%s() does not support writable files\n", __func__))
return;
debugfs_create_mode_unsafe(name, mode, parent, value, &fops_str,
&fops_str_ro, &fops_str_wo);
debugfs_create_file_unsafe(name, mode, parent, value, &fops_str);
}
EXPORT_SYMBOL_GPL(debugfs_create_str);

14
include/linux/device.h
View File

@ -129,19 +129,19 @@ struct dev_ext_attribute {
void *var;
};
ssize_t device_show_ulong(struct device *dev, struct device_attribute *attr,
ssize_t device_show_ulong(struct device *dev, const struct device_attribute *attr,
char *buf);
ssize_t device_store_ulong(struct device *dev, struct device_attribute *attr,
ssize_t device_store_ulong(struct device *dev, const struct device_attribute *attr,
const char *buf, size_t count);
ssize_t device_show_int(struct device *dev, struct device_attribute *attr,
ssize_t device_show_int(struct device *dev, const struct device_attribute *attr,
char *buf);
ssize_t device_store_int(struct device *dev, struct device_attribute *attr,
ssize_t device_store_int(struct device *dev, const struct device_attribute *attr,
const char *buf, size_t count);
ssize_t device_show_bool(struct device *dev, struct device_attribute *attr,
ssize_t device_show_bool(struct device *dev, const struct device_attribute *attr,
char *buf);
ssize_t device_store_bool(struct device *dev, struct device_attribute *attr,
ssize_t device_store_bool(struct device *dev, const struct device_attribute *attr,
const char *buf, size_t count);
ssize_t device_show_string(struct device *dev, struct device_attribute *attr,
ssize_t device_show_string(struct device *dev, const struct device_attribute *attr,
char *buf);
typedef ssize_t __device_show_handler_const(struct device *dev, const struct device_attribute *attr,

2
include/linux/kernfs.h
View File

@ -309,7 +309,7 @@ struct kernfs_ops {
* write() is bounced through kernel buffer. If atomic_write_len
* is not set, a write larger than PAGE_SIZE results in partial
* operations of PAGE_SIZE chunks. If atomic_write_len is set,
* writes upto the specified size are executed atomically but
* writes up to the specified size are executed atomically but
* larger ones are rejected with -E2BIG.
*/
size_t atomic_write_len;

10
include/linux/module.h
View File

@ -29,6 +29,7 @@
#include <linux/srcu.h>
#include <linux/static_call_types.h>
#include <linux/dynamic_debug.h>
#include <linux/panic.h>
#include <linux/percpu.h>
#include <asm/module.h>
@ -770,6 +771,9 @@ static inline bool is_livepatch_module(struct module *mod)
void module_for_each_mod(int(*func)(struct module *mod, void *data), void *data);
void add_taint_module(struct module *mod, unsigned flag,
enum lockdep_ok lockdep_ok);
#else /* !CONFIG_MODULES... */
static inline struct module *__module_address(unsigned long addr)
@ -877,6 +881,12 @@ static inline bool module_is_coming(struct module *mod)
static inline void module_for_each_mod(int(*func)(struct module *mod, void *data), void *data)
{
}
static inline void add_taint_module(struct module *mod, unsigned flag,
enum lockdep_ok lockdep_ok)
{
add_taint(flag, lockdep_ok);
}
#endif /* CONFIG_MODULES */
#ifdef CONFIG_SYSFS

3
include/linux/panic.h
View File

@ -88,7 +88,8 @@ static inline void set_arch_panic_timeout(int timeout, int arch_default_timeout)
#define TAINT_RANDSTRUCT 17
#define TAINT_TEST 18
#define TAINT_FWCTL 19
#define TAINT_FLAGS_COUNT 20
#define TAINT_FORCED_BIND 20
#define TAINT_FLAGS_COUNT 21
#define TAINT_FLAGS_MAX ((1UL << TAINT_FLAGS_COUNT) - 1)
struct taint_flag {

3
include/trace/events/module.h
View File

@ -26,7 +26,8 @@ struct module;
{ (1UL << TAINT_OOT_MODULE), "O" }, \
{ (1UL << TAINT_FORCED_MODULE), "F" }, \
{ (1UL << TAINT_CRAP), "C" }, \
{ (1UL << TAINT_UNSIGNED_MODULE), "E" })
{ (1UL << TAINT_UNSIGNED_MODULE), "E" }, \
{ (1UL << TAINT_FORCED_BIND), "Y" })
TRACE_EVENT(module_load,

17
kernel/module/main.c
View File

@ -191,11 +191,22 @@ static inline int strong_try_module_get(struct module *mod)
return -ENOENT;
}
static inline void add_taint_module(struct module *mod, unsigned flag,
enum lockdep_ok lockdep_ok)
/**
* add_taint_module: add a taint flag if not already set for a specific module
* @mod: pointer to the module that caused the problem
* @flag: one of the TAINT_* constants.
* @lockdep_ok: whether lock debugging is still OK.
*
* If something bad has gone wrong, you'll want
* @lockdep_ok = LOCKDEP_NOW_UNRELIABLE, but for some
* noteworthy-but-not-corrupting cases, it can be set to LOCKDEP_STILL_OK.
*/
void add_taint_module(struct module *mod, unsigned flag,
enum lockdep_ok lockdep_ok)
{
add_taint(flag, lockdep_ok);
set_bit(flag, &mod->taints);
if (mod)
set_bit(flag, &mod->taints);
}
/*

10
kernel/panic.c
View File

@ -826,6 +826,7 @@ const struct taint_flag taint_flags[TAINT_FLAGS_COUNT] = {
TAINT_FLAG(RANDSTRUCT, 'T', ' '),
TAINT_FLAG(TEST, 'N', ' '),
TAINT_FLAG(FWCTL, 'J', ' '),
TAINT_FLAG(FORCED_BIND, 'Y', ' '),
};
#undef TAINT_FLAG
@ -862,9 +863,9 @@ static void print_tainted_seq(struct seq_buf *s, bool verbose)
* exact size is allocated dynamically; the initial buffer remains
* as a fallback if allocation fails.
*
* The verbose taint string currently requires up to 327 characters.
* The verbose taint string currently requires up to 344 characters.
*/
#define INIT_TAINT_BUF_MAX 350
#define INIT_TAINT_BUF_MAX 370
static char init_taint_buf[INIT_TAINT_BUF_MAX] __initdata;
static char *taint_buf __refdata = init_taint_buf;
@ -948,8 +949,9 @@ unsigned long get_taint(void)
* @flag: one of the TAINT_* constants.
* @lockdep_ok: whether lock debugging is still OK.
*
* If something bad has gone wrong, you'll want @lockdebug_ok = false, but for
* some notewortht-but-not-corrupting cases, it can be set to true.
* If something bad has gone wrong, you'll want
* @lockdep_ok = LOCKDEP_NOW_UNRELIABLE, but for some
* noteworthy-but-not-corrupting cases, it can be set to LOCKDEP_STILL_OK.
*/
void add_taint(unsigned flag, enum lockdep_ok lockdep_ok)
{

2
lib/kobject.c
View File

@ -855,7 +855,7 @@ EXPORT_SYMBOL_GPL(kobj_sysfs_ops);
* kset_register() - Initialize and add a kset.
* @k: kset.
*
* NOTE: On error, the kset.kobj.name allocated by() kobj_set_name()
* NOTE: On error, the kset.kobj.name allocated by kobject_set_name()
* is freed, it can not be used any more.
*/
int kset_register(struct kset *k)

2
lib/kobject_uevent.c
View File

@ -83,7 +83,7 @@ static int kobject_action_type(const char *buf, size_t count,
for (action = 0; action < ARRAY_SIZE(kobject_actions); action++) {
if (strncmp(kobject_actions[action], buf, count_first) != 0)
continue;
if (kobject_actions[action][count_first] != '\0')
if (strlen(kobject_actions[action]) != count_first)
continue;
if (args)
*args = args_start;

5
rust/helpers/dma.c
View File

@ -49,3 +49,8 @@ __rust_helper void rust_helper_dma_set_max_seg_size(struct device *dev,
{
dma_set_max_seg_size(dev, size);
}
__rust_helper unsigned int rust_helper_dma_get_max_seg_size(struct device *dev)
{
return dma_get_max_seg_size(dev);
}

5
rust/helpers/pci.c
View File

@ -29,6 +29,11 @@ __rust_helper unsigned int rust_helper_pci_irq_type(struct pci_dev *pdev)
return pci_irq_type(pdev);
}
__rust_helper u32 rust_helper_pci_ext_cap_next(u32 header)
{
return PCI_EXT_CAP_NEXT(header);
}
#ifndef CONFIG_PCI_IOV
__rust_helper unsigned int
rust_helper_pci_sriov_get_totalvfs(struct pci_dev *pdev)

8
rust/kernel/auxiliary.rs
View File

@ -137,10 +137,18 @@ macro_rules! module_auxiliary_driver {
impl DeviceId {
/// Create a new [`DeviceId`] from name.
///
/// This is only intended to be called in const context, when constructing a
/// device ID table, where exceeding `AUXILIARY_NAME_SIZE` is a compile time error.
pub const fn new(modname: &'static CStr, name: &'static CStr) -> Self {
let name = name.to_bytes_with_nul();
let modname = modname.to_bytes_with_nul();
assert!(
modname.len().saturating_add(name.len()) <= bindings::AUXILIARY_NAME_SIZE as usize,
"auxiliary device ID is too long"
);
let mut id: bindings::auxiliary_device_id = pin_init::zeroed();
let mut i = 0;
while i < modname.len() {

9
rust/kernel/bitfield.rs
View File

@ -345,6 +345,15 @@ fn from(val: $storage) -> $name {
Self::from_raw(val)
}
}
// SAFETY: `$name` is transparent over `$storage` and `$storage` has no interior mutability.
unsafe impl $crate::mem::AsRepr for $name {
// Normalize `$storage` to the canonical repr type in case it is signed.
type Repr = <$storage as $crate::mem::AsRepr>::Repr;
}
// SAFETY: `$name` is transparent over `$storage`.
unsafe impl $crate::mem::AsReprMut for $name {}
};
// Definitions requiring knowledge of individual fields: private and public field accessors,

26
rust/kernel/debugfs.rs
View File

@ -538,7 +538,7 @@ pub fn dir<'dir2>(&'dir2 self, name: &CStr) -> ScopedDir<'data, 'dir2> {
}
}
fn create_file<T: Sync>(&self, name: &CStr, data: &'data T, vtable: &'static FileOps<T>) {
fn create_file<T: Sync>(&self, name: &CStr, data: &'data T, vtable: &FileOps<T>) {
#[cfg(CONFIG_DEBUG_FS)]
core::mem::forget(Entry::file(name, &self.entry, data, vtable));
}
@ -550,7 +550,7 @@ fn create_file<T: Sync>(&self, name: &CStr, data: &'data T, vtable: &'static Fil
/// This function does not produce an owning handle to the file. The created
/// file is removed when the [`Scope`] that this directory belongs
/// to is dropped.
pub fn read_only_file<T: Writer + Send + Sync + 'static>(&self, name: &CStr, data: &'data T) {
pub fn read_only_file<T: Writer + Send + Sync>(&self, name: &CStr, data: &'data T) {
self.create_file(name, data, &T::FILE_OPS)
}
@ -560,11 +560,7 @@ pub fn read_only_file<T: Writer + Send + Sync + 'static>(&self, name: &CStr, dat
///
/// This function does not produce an owning handle to the file. The created file is removed
/// when the [`Scope`] that this directory belongs to is dropped.
pub fn read_binary_file<T: BinaryWriter + Send + Sync + 'static>(
&self,
name: &CStr,
data: &'data T,
) {
pub fn read_binary_file<T: BinaryWriter + Send + Sync>(&self, name: &CStr, data: &'data T) {
self.create_file(name, data, &T::FILE_OPS)
}
@ -596,11 +592,7 @@ pub fn read_callback_file<T, F>(&self, name: &CStr, data: &'data T, _f: &'static
/// This function does not produce an owning handle to the file. The created
/// file is removed when the [`Scope`] that this directory belongs
/// to is dropped.
pub fn read_write_file<T: Writer + Reader + Send + Sync + 'static>(
&self,
name: &CStr,
data: &'data T,
) {
pub fn read_write_file<T: Writer + Reader + Send + Sync>(&self, name: &CStr, data: &'data T) {
let vtable = &<T as ReadWriteFile<_>>::FILE_OPS;
self.create_file(name, data, vtable)
}
@ -612,7 +604,7 @@ pub fn read_write_file<T: Writer + Reader + Send + Sync + 'static>(
///
/// This function does not produce an owning handle to the file. The created file is removed
/// when the [`Scope`] that this directory belongs to is dropped.
pub fn read_write_binary_file<T: BinaryWriter + BinaryReader + Send + Sync + 'static>(
pub fn read_write_binary_file<T: BinaryWriter + BinaryReader + Send + Sync>(
&self,
name: &CStr,
data: &'data T,
@ -655,7 +647,7 @@ pub fn read_write_callback_file<T, F, W>(
/// This function does not produce an owning handle to the file. The created
/// file is removed when the [`Scope`] that this directory belongs
/// to is dropped.
pub fn write_only_file<T: Reader + Send + Sync + 'static>(&self, name: &CStr, data: &'data T) {
pub fn write_only_file<T: Reader + Send + Sync>(&self, name: &CStr, data: &'data T) {
let vtable = &<T as WriteFile<_>>::FILE_OPS;
self.create_file(name, data, vtable)
}
@ -666,11 +658,7 @@ pub fn write_only_file<T: Reader + Send + Sync + 'static>(&self, name: &CStr, da
///
/// This function does not produce an owning handle to the file. The created file is removed
/// when the [`Scope`] that this directory belongs to is dropped.
pub fn write_binary_file<T: BinaryReader + Send + Sync + 'static>(
&self,
name: &CStr,
data: &'data T,
) {
pub fn write_binary_file<T: BinaryReader + Send + Sync>(&self, name: &CStr, data: &'data T) {
self.create_file(name, data, &T::FILE_OPS)
}

4
rust/kernel/debugfs/entry.rs
View File

@ -74,7 +74,7 @@ pub(crate) unsafe fn dynamic_file<T>(
parent.as_ptr(),
core::ptr::from_ref(data) as *mut c_void,
core::ptr::null(),
&**file_ops,
file_ops.fops(),
)
};
@ -127,7 +127,7 @@ pub(crate) fn file<T>(
parent.as_ptr(),
core::ptr::from_ref(data) as *mut c_void,
core::ptr::null(),
&**file_ops,
file_ops.fops(),
)
};

29
rust/kernel/debugfs/file_ops.rs
View File

@ -20,9 +20,6 @@
use core::marker::PhantomData;
#[cfg(CONFIG_DEBUG_FS)]
use core::ops::Deref;
/// # Invariant
///
/// `FileOps<T>` will always contain an `operations` which is safe to use for a file backed
@ -30,7 +27,7 @@
/// into a reference.
pub(super) struct FileOps<T> {
#[cfg(CONFIG_DEBUG_FS)]
operations: bindings::file_operations,
operations: &'static bindings::file_operations,
#[cfg(CONFIG_DEBUG_FS)]
mode: u16,
_phantom: PhantomData<T>,
@ -41,7 +38,7 @@ impl<T> FileOps<T> {
///
/// The caller asserts that the provided `operations` is safe to use for a file whose
/// inode has a pointer to `T` in its private data that is safe to convert into a reference.
const unsafe fn new(operations: bindings::file_operations, mode: u16) -> Self {
const unsafe fn new(operations: &'static bindings::file_operations, mode: u16) -> Self {
Self {
#[cfg(CONFIG_DEBUG_FS)]
operations,
@ -65,11 +62,11 @@ pub(super) const fn adapt(&self) -> &FileOps<T::Inner> {
}
#[cfg(CONFIG_DEBUG_FS)]
impl<T> Deref for FileOps<T> {
type Target = bindings::file_operations;
fn deref(&self) -> &Self::Target {
&self.operations
impl<T> FileOps<T> {
/// Returns a `'static` reference to the inner `file_operations`.
#[inline]
pub(crate) fn fops(&self) -> &'static bindings::file_operations {
self.operations
}
}
@ -130,7 +127,7 @@ pub(crate) trait ReadFile<T> {
impl<T: Writer + Sync> ReadFile<T> for T {
const FILE_OPS: FileOps<T> = {
let operations = bindings::file_operations {
let operations = &bindings::file_operations {
read: Some(bindings::seq_read),
llseek: Some(bindings::seq_lseek),
release: Some(bindings::single_release),
@ -181,7 +178,7 @@ pub(crate) trait ReadWriteFile<T> {
impl<T: Writer + Reader + Sync> ReadWriteFile<T> for T {
const FILE_OPS: FileOps<T> = {
let operations = bindings::file_operations {
let operations = &bindings::file_operations {
open: Some(writer_open::<T>),
read: Some(bindings::seq_read),
write: Some(write::<T>),
@ -238,7 +235,7 @@ pub(crate) trait WriteFile<T> {
impl<T: Reader + Sync> WriteFile<T> for T {
const FILE_OPS: FileOps<T> = {
let operations = bindings::file_operations {
let operations = &bindings::file_operations {
open: Some(write_only_open),
write: Some(write_only_write::<T>),
llseek: Some(bindings::noop_llseek),
@ -290,7 +287,7 @@ pub(crate) trait BinaryReadFile<T> {
impl<T: BinaryWriter + Sync> BinaryReadFile<T> for T {
const FILE_OPS: FileOps<T> = {
let operations = bindings::file_operations {
let operations = &bindings::file_operations {
read: Some(blob_read::<T>),
llseek: Some(bindings::default_llseek),
open: Some(bindings::simple_open),
@ -344,7 +341,7 @@ pub(crate) trait BinaryWriteFile<T> {
impl<T: BinaryReader + Sync> BinaryWriteFile<T> for T {
const FILE_OPS: FileOps<T> = {
let operations = bindings::file_operations {
let operations = &bindings::file_operations {
write: Some(blob_write::<T>),
llseek: Some(bindings::default_llseek),
open: Some(bindings::simple_open),
@ -368,7 +365,7 @@ pub(crate) trait BinaryReadWriteFile<T> {
impl<T: BinaryWriter + BinaryReader + Sync> BinaryReadWriteFile<T> for T {
const FILE_OPS: FileOps<T> = {
let operations = bindings::file_operations {
let operations = &bindings::file_operations {
read: Some(blob_read::<T>),
write: Some(blob_write::<T>),
llseek: Some(bindings::default_llseek),

3
rust/kernel/device_id.rs
View File

@ -146,8 +146,7 @@ impl<T: RawDeviceId, const N: usize> IdArray<T, (), N> {
/// If the device implements [`RawDeviceIdIndex`], consider using [`IdArray::new`] instead.
pub const fn new_without_index(ids: [T; N]) -> Self {
// SAFETY: `T` is layout-wise compatible with `T::RawType`, so is the array of them.
let raw_ids: [MaybeUninit<T::RawType>; N] = unsafe { core::mem::transmute_copy(&ids) };
core::mem::forget(ids);
let raw_ids: [MaybeUninit<T::RawType>; N] = unsafe { crate::mem::transmute(ids) };
Self {
ids: raw_ids,

141
rust/kernel/dma.rs
View File

@ -24,7 +24,6 @@
},
prelude::*,
ptr::KnownSize,
sync::aref::ARef,
transmute::{
AsBytes,
FromBytes, //
@ -223,7 +222,7 @@ pub const fn value(&self) -> u64 {
///
/// # fn test(dev: &Device<Bound>) -> Result {
/// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN;
/// let c: Coherent<[u64]> =
/// let c: Coherent<'_, [u64]> =
/// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, attribs)?;
/// # Ok::<(), Error>(()) }
/// ```
@ -390,9 +389,9 @@ fn from(direction: DataDirection) -> Self {
/// };
///
/// # fn test(dev: &Device<Bound>) -> Result {
/// let mut dmem: CoherentBox<u64> = CoherentBox::zeroed(dev, GFP_KERNEL)?;
/// let mut dmem: CoherentBox<'_, u64> = CoherentBox::zeroed(dev, GFP_KERNEL)?;
/// *dmem = 42;
/// let dmem: Coherent<u64> = dmem.into();
/// let dmem: Coherent<'_, u64> = dmem.into();
/// # Ok::<(), Error>(()) }
/// ```
///
@ -410,18 +409,18 @@ fn from(direction: DataDirection) -> Self {
/// };
///
/// # fn test(dev: &Device<Bound>) -> Result {
/// let mut dmem: CoherentBox<[u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?;
/// let mut dmem: CoherentBox<'_, [u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?;
/// dmem.fill(42);
/// let dmem: Coherent<[u64]> = dmem.into();
/// let dmem: Coherent<'_, [u64]> = dmem.into();
/// # Ok::<(), Error>(()) }
/// ```
pub struct CoherentBox<T: KnownSize + ?Sized>(Coherent<T>);
pub struct CoherentBox<'a, T: KnownSize + ?Sized>(Coherent<'a, T>);
impl<T: AsBytes + FromBytes> CoherentBox<[T]> {
impl<'a, T: AsBytes + FromBytes> CoherentBox<'a, [T]> {
/// [`CoherentBox`] variant of [`Coherent::zeroed_slice_with_attrs`].
#[inline]
pub fn zeroed_slice_with_attrs(
dev: &device::Device<Bound>,
dev: &'a device::Device<Bound>,
count: usize,
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
@ -432,7 +431,7 @@ pub fn zeroed_slice_with_attrs(
/// Same as [CoherentBox::zeroed_slice_with_attrs], but with `dma::Attrs(0)`.
#[inline]
pub fn zeroed_slice(
dev: &device::Device<Bound>,
dev: &'a device::Device<Bound>,
count: usize,
gfp_flags: kernel::alloc::Flags,
) -> Result<Self> {
@ -480,14 +479,14 @@ pub fn init_at<E>(&mut self, i: usize, init: impl Init<T, E>) -> Result
///
/// # fn test(dev: &Device<Bound>) -> Result {
/// let data = [0u8, 1u8, 2u8, 3u8];
/// let c: CoherentBox<[u8]> =
/// let c: CoherentBox<'_, [u8]> =
/// CoherentBox::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
///
/// assert_eq!(c.deref(), &data);
/// # Ok::<(), Error>(()) }
/// ```
pub fn from_slice_with_attrs(
dev: &device::Device<Bound>,
dev: &'a device::Device<Bound>,
data: &[T],
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
@ -512,7 +511,7 @@ pub fn from_slice_with_attrs(
/// `dma_attrs` is 0 by default.
#[inline]
pub fn from_slice(
dev: &device::Device<Bound>,
dev: &'a device::Device<Bound>,
data: &[T],
gfp_flags: kernel::alloc::Flags,
) -> Result<Self>
@ -523,11 +522,11 @@ pub fn from_slice(
}
}
impl<T: AsBytes + FromBytes> CoherentBox<T> {
impl<'a, T: AsBytes + FromBytes> CoherentBox<'a, T> {
/// Same as [`CoherentBox::zeroed_slice_with_attrs`], but for a single element.
#[inline]
pub fn zeroed_with_attrs(
dev: &device::Device<Bound>,
dev: &'a device::Device<Bound>,
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
) -> Result<Self> {
@ -536,12 +535,12 @@ pub fn zeroed_with_attrs(
/// Same as [`CoherentBox::zeroed_slice`], but for a single element.
#[inline]
pub fn zeroed(dev: &device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> {
pub fn zeroed(dev: &'a device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> {
Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0))
}
}
impl<T: KnownSize + ?Sized> Deref for CoherentBox<T> {
impl<T: KnownSize + ?Sized> Deref for CoherentBox<'_, T> {
type Target = T;
#[inline]
@ -554,7 +553,7 @@ fn deref(&self) -> &Self::Target {
}
}
impl<T: AsBytes + FromBytes + KnownSize + ?Sized> DerefMut for CoherentBox<T> {
impl<T: AsBytes + FromBytes + KnownSize + ?Sized> DerefMut for CoherentBox<'_, T> {
#[inline]
fn deref_mut(&mut self) -> &mut Self::Target {
// SAFETY:
@ -565,9 +564,9 @@ fn deref_mut(&mut self) -> &mut Self::Target {
}
}
impl<T: AsBytes + FromBytes + KnownSize + ?Sized> From<CoherentBox<T>> for Coherent<T> {
impl<'a, T: AsBytes + FromBytes + KnownSize + ?Sized> From<CoherentBox<'a, T>> for Coherent<'a, T> {
#[inline]
fn from(value: CoherentBox<T>) -> Self {
fn from(value: CoherentBox<'a, T>) -> Self {
value.0
}
}
@ -588,26 +587,20 @@ fn from(value: CoherentBox<T>) -> Self {
/// to an allocated region of coherent memory and `dma_addr` is the DMA address base of the
/// region.
/// - The size in bytes of the allocation is equal to size information via pointer.
// TODO
//
// DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness
// reasons DMA allocation would need to be embedded in a `Devres` container, in order to ensure
// that device resources can never survive device unbind.
//
// However, it is neither desirable nor necessary to protect the allocated memory of the DMA
// allocation from surviving device unbind; it would require RCU read side critical sections to
// access the memory, which may require subsequent unnecessary copies.
//
// Hence, find a way to revoke the device resources of a `Coherent`, but not the
// entire `Coherent` including the allocated memory itself.
pub struct Coherent<T: KnownSize + ?Sized> {
dev: ARef<device::Device>,
// The lifetime parameter ties DMA allocations to the device's bound scope, ensuring they are freed
// before the device is unbound under normal circumstances. However, if a `Coherent` is leaked (e.g.
// via `mem::forget`), device resources such as IOMMU mappings will not be released. Making all
// constructors `unsafe` to prevent this is considered too restrictive for the common case; this
// soundness hole is accepted for now.
pub struct Coherent<'a, T: KnownSize + ?Sized> {
dev: &'a device::Device<Bound>,
dma_addr: DmaAddress,
cpu_addr: NonNull<T>,
dma_attrs: Attrs,
}
impl<T: KnownSize + ?Sized> Coherent<T> {
impl<T: KnownSize + ?Sized> Coherent<'_, T> {
/// Returns the size in bytes of this allocation.
#[inline]
pub fn size(&self) -> usize {
@ -663,10 +656,10 @@ pub unsafe fn as_mut(&self) -> &mut T {
}
}
impl<T: AsBytes + FromBytes> Coherent<T> {
impl<'a, T: AsBytes + FromBytes> Coherent<'a, T> {
/// Allocates a region of `T` of coherent memory.
fn alloc_with_attrs(
dev: &device::Device<Bound>,
dev: &'a device::Device<Bound>,
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
) -> Result<Self> {
@ -692,9 +685,9 @@ fn alloc_with_attrs(
// INVARIANT:
// - We just successfully allocated a coherent region which is adequately sized for `T`,
// hence the cpu address is valid.
// - We also hold a refcounted reference to the device.
// - `dev` is a valid reference to a bound device that outlives this allocation.
Ok(Self {
dev: dev.into(),
dev,
dma_addr,
cpu_addr,
dma_attrs,
@ -716,13 +709,13 @@ fn alloc_with_attrs(
/// };
///
/// # fn test(dev: &Device<Bound>) -> Result {
/// let c: Coherent<[u64; 4]> =
/// let c: Coherent<'_, [u64; 4]> =
/// Coherent::zeroed_with_attrs(dev, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
/// # Ok::<(), Error>(()) }
/// ```
#[inline]
pub fn zeroed_with_attrs(
dev: &device::Device<Bound>,
dev: &'a device::Device<Bound>,
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
) -> Result<Self> {
@ -732,14 +725,14 @@ pub fn zeroed_with_attrs(
/// Performs the same functionality as [`Coherent::zeroed_with_attrs`], except the
/// `dma_attrs` is 0 by default.
#[inline]
pub fn zeroed(dev: &device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> {
pub fn zeroed(dev: &'a device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> {
Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0))
}
/// Same as [`Coherent::zeroed_with_attrs`], but instead of a zero-initialization the memory is
/// initialized with `init`.
pub fn init_with_attrs<E>(
dev: &device::Device<Bound>,
dev: &'a device::Device<Bound>,
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
init: impl Init<T, E>,
@ -764,7 +757,7 @@ pub fn init_with_attrs<E>(
/// with `init`.
#[inline]
pub fn init<E>(
dev: &device::Device<Bound>,
dev: &'a device::Device<Bound>,
gfp_flags: kernel::alloc::Flags,
init: impl Init<T, E>,
) -> Result<Self>
@ -776,11 +769,11 @@ pub fn init<E>(
/// Allocates a region of `[T; len]` of coherent memory.
fn alloc_slice_with_attrs(
dev: &device::Device<Bound>,
dev: &'a device::Device<Bound>,
len: usize,
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
) -> Result<Coherent<[T]>> {
) -> Result<Coherent<'a, [T]>> {
const {
assert!(
core::mem::size_of::<T>() > 0,
@ -809,9 +802,9 @@ fn alloc_slice_with_attrs(
// INVARIANT:
// - We just successfully allocated a coherent region which is adequately sized for
// `[T; len]`, hence the cpu address is valid.
// - We also hold a refcounted reference to the device.
// - `dev` is a valid reference to a bound device that outlives this allocation.
Ok(Coherent {
dev: dev.into(),
dev,
dma_addr,
cpu_addr,
dma_attrs,
@ -836,17 +829,17 @@ fn alloc_slice_with_attrs(
/// };
///
/// # fn test(dev: &Device<Bound>) -> Result {
/// let c: Coherent<[u64]> =
/// let c: Coherent<'_, [u64]> =
/// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
/// # Ok::<(), Error>(()) }
/// ```
#[inline]
pub fn zeroed_slice_with_attrs(
dev: &device::Device<Bound>,
dev: &'a device::Device<Bound>,
len: usize,
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
) -> Result<Coherent<[T]>> {
) -> Result<Coherent<'a, [T]>> {
Coherent::alloc_slice_with_attrs(dev, len, gfp_flags | __GFP_ZERO, dma_attrs)
}
@ -854,10 +847,10 @@ pub fn zeroed_slice_with_attrs(
/// `dma_attrs` is 0 by default.
#[inline]
pub fn zeroed_slice(
dev: &device::Device<Bound>,
dev: &'a device::Device<Bound>,
len: usize,
gfp_flags: kernel::alloc::Flags,
) -> Result<Coherent<[T]>> {
) -> Result<Coherent<'a, [T]>> {
Self::zeroed_slice_with_attrs(dev, len, gfp_flags, Attrs(0))
}
@ -876,18 +869,18 @@ pub fn zeroed_slice(
/// # fn test(dev: &Device<Bound>) -> Result {
/// let data = [0u8, 1u8, 2u8, 3u8];
/// // `c` has the same content as `data`.
/// let c: Coherent<[u8]> =
/// let c: Coherent<'_, [u8]> =
/// Coherent::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
///
/// # Ok::<(), Error>(()) }
/// ```
#[inline]
pub fn from_slice_with_attrs(
dev: &device::Device<Bound>,
dev: &'a device::Device<Bound>,
data: &[T],
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
) -> Result<Coherent<[T]>>
) -> Result<Coherent<'a, [T]>>
where
T: Copy,
{
@ -898,10 +891,10 @@ pub fn from_slice_with_attrs(
/// `dma_attrs` is 0 by default.
#[inline]
pub fn from_slice(
dev: &device::Device<Bound>,
dev: &'a device::Device<Bound>,
data: &[T],
gfp_flags: kernel::alloc::Flags,
) -> Result<Coherent<[T]>>
) -> Result<Coherent<'a, [T]>>
where
T: Copy,
{
@ -909,7 +902,7 @@ pub fn from_slice(
}
}
impl<T> Coherent<[T]> {
impl<T> Coherent<'_, [T]> {
/// Returns the number of elements `T` in this allocation.
///
/// Note that this is not the size of the allocation in bytes, which is provided by
@ -922,10 +915,10 @@ pub fn len(&self) -> usize {
}
/// Note that the device configured to do DMA must be halted before this object is dropped.
impl<T: KnownSize + ?Sized> Drop for Coherent<T> {
impl<T: KnownSize + ?Sized> Drop for Coherent<'_, T> {
fn drop(&mut self) {
let size = T::size(self.cpu_addr.as_ptr());
// SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`.
// SAFETY: Device pointer is guaranteed as valid by the lifetime of this `Coherent`.
// The cpu address, and the dma address are valid due to the type invariants on
// `Coherent`.
unsafe {
@ -942,15 +935,15 @@ fn drop(&mut self) {
// SAFETY: It is safe to send a `Coherent` to another thread if `T`
// can be sent to another thread.
unsafe impl<T: KnownSize + Send + ?Sized> Send for Coherent<T> {}
unsafe impl<T: KnownSize + Send + ?Sized> Send for Coherent<'_, T> {}
// SAFETY: Sharing `&Coherent` across threads is safe if `T` is `Sync`, because all
// methods that access the buffer contents (`field_read`, `field_write`, `as_slice`,
// `as_slice_mut`) are `unsafe`, and callers are responsible for ensuring no data races occur.
// The safe methods only return metadata or raw pointers whose use requires `unsafe`.
unsafe impl<T: KnownSize + ?Sized + AsBytes + FromBytes + Sync> Sync for Coherent<T> {}
unsafe impl<T: KnownSize + ?Sized + AsBytes + FromBytes + Sync> Sync for Coherent<'_, T> {}
impl<T: KnownSize + AsBytes + ?Sized> debugfs::BinaryWriter for Coherent<T> {
impl<T: KnownSize + AsBytes + ?Sized> debugfs::BinaryWriter for Coherent<'_, T> {
fn write_to_slice(
&self,
writer: &mut UserSliceWriter,
@ -996,15 +989,15 @@ fn write_to_slice(
/// - `size` is the allocation size in bytes as passed to `dma_alloc_attrs`.
/// - `dma_attrs` contains the attributes used for the allocation, always including
/// `DMA_ATTR_NO_KERNEL_MAPPING`.
pub struct CoherentHandle {
dev: ARef<device::Device>,
pub struct CoherentHandle<'a> {
dev: &'a device::Device<Bound>,
dma_addr: DmaAddress,
cpu_handle: NonNull<c_void>,
size: usize,
dma_attrs: Attrs,
}
impl CoherentHandle {
impl<'a> CoherentHandle<'a> {
/// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping.
///
/// Additional DMA attributes may be passed via `dma_attrs`; `DMA_ATTR_NO_KERNEL_MAPPING` is
@ -1012,7 +1005,7 @@ impl CoherentHandle {
///
/// Returns `EINVAL` if `size` is zero, `ENOMEM` if the allocation fails.
pub fn alloc_with_attrs(
dev: &device::Device<Bound>,
dev: &'a device::Device<Bound>,
size: usize,
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
@ -1038,9 +1031,9 @@ pub fn alloc_with_attrs(
// INVARIANT: `cpu_handle` is the opaque handle from a successful `dma_alloc_attrs` call
// with `DMA_ATTR_NO_KERNEL_MAPPING`, `dma_addr` is the corresponding DMA address,
// and we hold a refcounted reference to the device.
// and `dev` is a valid reference to a bound device that outlives this allocation.
Ok(Self {
dev: dev.into(),
dev,
dma_addr,
cpu_handle,
size,
@ -1051,7 +1044,7 @@ pub fn alloc_with_attrs(
/// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping.
#[inline]
pub fn alloc(
dev: &device::Device<Bound>,
dev: &'a device::Device<Bound>,
size: usize,
gfp_flags: kernel::alloc::Flags,
) -> Result<Self> {
@ -1073,7 +1066,7 @@ pub fn size(&self) -> usize {
}
}
impl Drop for CoherentHandle {
impl Drop for CoherentHandle<'_> {
fn drop(&mut self) {
// SAFETY: All values are valid by the type invariants on `CoherentHandle`.
// `cpu_handle` is the opaque handle from `dma_alloc_attrs` and is passed back unchanged.
@ -1091,12 +1084,12 @@ fn drop(&mut self) {
// SAFETY: `CoherentHandle` only holds a device reference, a DMA address, an opaque CPU handle,
// and a size. None of these are tied to a specific thread.
unsafe impl Send for CoherentHandle {}
unsafe impl Send for CoherentHandle<'_> {}
// SAFETY: `CoherentHandle` provides no CPU access to the underlying allocation. The only
// operations on `&CoherentHandle` are reading the DMA address and size, both of which are
// plain `Copy` values.
unsafe impl Sync for CoherentHandle {}
unsafe impl Sync for CoherentHandle<'_> {}
/// View type for `Coherent`.
///
@ -1236,7 +1229,7 @@ fn as_view(self) -> CoherentView<'a, Self::Target> {
}
}
impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent<T> {
impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent<'_, T> {
type Backend = CoherentIoBackend;
type Target = T;

255
rust/kernel/io.rs
View File

@ -6,11 +6,17 @@
use core::{
marker::PhantomData,
mem::MaybeUninit, //
mem::MaybeUninit,
num::TryFromIntError, //
};
use crate::{
bindings,
fmt,
mem::{
AsRepr,
AsReprMut, //
},
prelude::*,
ptr::{
Alignment,
@ -35,11 +41,73 @@
/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures.
pub type PhysAddr = bindings::phys_addr_t;
/// Resource Size type.
/// Resource size type.
///
/// This is a type alias to either `u32` or `u64` depending on the config option
/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures.
pub type ResourceSize = bindings::resource_size_t;
/// This wraps either `u32` or `u64` depending on the config option
/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a `u64` even on 32-bit architectures.
///
/// # Examples
///
/// ```
/// use kernel::io::ResourceSize;
///
/// let size = ResourceSize::from_raw(0x1000);
/// assert_eq!(size.into_raw(), 0x1000);
///
/// // Round-trips through the raw C type.
/// let raw: kernel::bindings::resource_size_t = size.into();
/// assert_eq!(ResourceSize::from(raw), size);
///
/// // Fallible conversion to `usize` (can truncate on 32-bit).
/// assert_eq!(usize::try_from(size)?, 0x1000);
/// # Ok::<(), core::num::TryFromIntError>(())
/// ```
#[repr(transparent)]
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct ResourceSize(bindings::resource_size_t);
impl ResourceSize {
/// Creates a resource size from the raw C type.
#[inline]
pub const fn from_raw(value: bindings::resource_size_t) -> Self {
Self(value)
}
/// Turns this resource size into the raw C type.
#[inline]
pub const fn into_raw(self) -> bindings::resource_size_t {
self.0
}
}
impl fmt::Debug for ResourceSize {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:#x}", self.0)
}
}
impl From<bindings::resource_size_t> for ResourceSize {
#[inline]
fn from(value: bindings::resource_size_t) -> Self {
Self::from_raw(value)
}
}
impl From<ResourceSize> for bindings::resource_size_t {
#[inline]
fn from(value: ResourceSize) -> Self {
value.into_raw()
}
}
impl TryFrom<ResourceSize> for usize {
type Error = TryFromIntError;
#[inline]
fn try_from(value: ResourceSize) -> Result<Self, Self::Error> {
Self::try_from(value.into_raw())
}
}
/// Untyped I/O region.
///
@ -226,6 +294,17 @@ fn io_view<'a, IO: Io<'a>, U>(
Ok(unsafe { IO::Backend::project_view(view, projected_ptr) })
}
/// Returns the primitive view of a I/O view.
#[inline]
fn io_view_as_repr<'a, IO: Io<'a, Target = T>, T: AsRepr>(
this: IO,
) -> <IO::Backend as IoBackend>::View<'a, T::Repr> {
let view = this.as_view();
// SAFETY: `AsRepr` guarantees layout compatibility.
unsafe { IO::Backend::project_view(view, IO::Backend::as_ptr(view).cast::<T::Repr>()) }
}
/// I/O backends.
///
/// This is an abstract representation to be implemented by arbitrary I/O
@ -353,15 +432,12 @@ fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) {
///
/// - The valid `Base` to operate on. For most registers, this should be [`Region`].
/// - The offset to access (returned by [`IoLoc::offset`]),
/// - The width of the access (determined by [`IoLoc::IoType`]),
/// - The type `T` in which the raw data is returned or provided.
/// - The type `T` in which the data is returned or provided.
///
/// `T` and `IoLoc::IoType` may differ: for instance, a typed register has `T` = the register type
/// with its bitfields, and `IoType` = its backing primitive (e.g. `u32`).
/// `T` is not necessarily the type for underlying I/O operation. Methods that take `IoLoc` have `T:
/// AsRepr` bound and the `<T as AsRepr>::Repr` type would be used to perform I/O and converted to
/// `T` instead.
pub trait IoLoc<Base: ?Sized, T> {
/// Size ([`u8`], [`u16`], etc) of the I/O performed on the returned [`offset`](IoLoc::offset).
type IoType: Into<T> + From<T>;
/// Consumes `self` and returns the offset of this location.
fn offset(self) -> usize;
}
@ -372,8 +448,6 @@ macro_rules! impl_usize_ioloc {
($($ty:ty),*) => {
$(
impl<const SIZE: usize> IoLoc<Region<SIZE>, $ty> for usize {
type IoType = $ty;
#[inline(always)]
fn offset(self) -> usize {
self
@ -498,10 +572,10 @@ fn try_cast<U>(self) -> Result<<Self::Backend as IoBackend>::View<'a, U>>
#[inline]
fn read_val(self) -> Self::Target
where
Self::Backend: IoCapable<Self::Target>,
Self::Target: Sized,
Self::Target: AsReprMut,
Self::Backend: IoCapable<<Self::Target as AsRepr>::Repr>,
{
Self::Backend::io_read(self.as_view())
Self::Target::from_repr(Self::Backend::io_read(io_view_as_repr(self)))
}
/// Write a value to I/O.
@ -520,10 +594,10 @@ fn read_val(self) -> Self::Target
#[inline]
fn write_val(self, value: Self::Target)
where
Self::Backend: IoCapable<Self::Target>,
Self::Target: Sized,
Self::Target: AsRepr,
Self::Backend: IoCapable<<Self::Target as AsRepr>::Repr>,
{
Self::Backend::io_write(self.as_view(), value)
Self::Backend::io_write(io_view_as_repr(self), Self::Target::into_repr(value))
}
/// Copy-read from I/O memory.
@ -645,7 +719,7 @@ fn copy_to_slice(self, data: &mut [u8])
#[inline(always)]
fn try_read8(self, offset: usize) -> Result<u8>
where
usize: IoLoc<Self::Target, u8, IoType = u8>,
usize: IoLoc<Self::Target, u8>,
Self::Backend: IoCapable<u8>,
{
self.try_read(offset)
@ -655,7 +729,7 @@ fn try_read8(self, offset: usize) -> Result<u8>
#[inline(always)]
fn try_read16(self, offset: usize) -> Result<u16>
where
usize: IoLoc<Self::Target, u16, IoType = u16>,
usize: IoLoc<Self::Target, u16>,
Self::Backend: IoCapable<u16>,
{
self.try_read(offset)
@ -665,7 +739,7 @@ fn try_read16(self, offset: usize) -> Result<u16>
#[inline(always)]
fn try_read32(self, offset: usize) -> Result<u32>
where
usize: IoLoc<Self::Target, u32, IoType = u32>,
usize: IoLoc<Self::Target, u32>,
Self::Backend: IoCapable<u32>,
{
self.try_read(offset)
@ -675,7 +749,7 @@ fn try_read32(self, offset: usize) -> Result<u32>
#[inline(always)]
fn try_read64(self, offset: usize) -> Result<u64>
where
usize: IoLoc<Self::Target, u64, IoType = u64>,
usize: IoLoc<Self::Target, u64>,
Self::Backend: IoCapable<u64>,
{
self.try_read(offset)
@ -685,7 +759,7 @@ fn try_read64(self, offset: usize) -> Result<u64>
#[inline(always)]
fn try_write8(self, value: u8, offset: usize) -> Result
where
usize: IoLoc<Self::Target, u8, IoType = u8>,
usize: IoLoc<Self::Target, u8>,
Self::Backend: IoCapable<u8>,
{
self.try_write(offset, value)
@ -695,7 +769,7 @@ fn try_write8(self, value: u8, offset: usize) -> Result
#[inline(always)]
fn try_write16(self, value: u16, offset: usize) -> Result
where
usize: IoLoc<Self::Target, u16, IoType = u16>,
usize: IoLoc<Self::Target, u16>,
Self::Backend: IoCapable<u16>,
{
self.try_write(offset, value)
@ -705,7 +779,7 @@ fn try_write16(self, value: u16, offset: usize) -> Result
#[inline(always)]
fn try_write32(self, value: u32, offset: usize) -> Result
where
usize: IoLoc<Self::Target, u32, IoType = u32>,
usize: IoLoc<Self::Target, u32>,
Self::Backend: IoCapable<u32>,
{
self.try_write(offset, value)
@ -715,7 +789,7 @@ fn try_write32(self, value: u32, offset: usize) -> Result
#[inline(always)]
fn try_write64(self, value: u64, offset: usize) -> Result
where
usize: IoLoc<Self::Target, u64, IoType = u64>,
usize: IoLoc<Self::Target, u64>,
Self::Backend: IoCapable<u64>,
{
self.try_write(offset, value)
@ -727,7 +801,7 @@ fn try_write64(self, value: u64, offset: usize) -> Result
#[inline(always)]
fn read8(self, offset: usize) -> u8
where
usize: IoLoc<Self::Target, u8, IoType = u8>,
usize: IoLoc<Self::Target, u8>,
Self::Backend: IoCapable<u8>,
{
self.read(offset)
@ -739,7 +813,7 @@ fn read8(self, offset: usize) -> u8
#[inline(always)]
fn read16(self, offset: usize) -> u16
where
usize: IoLoc<Self::Target, u16, IoType = u16>,
usize: IoLoc<Self::Target, u16>,
Self::Backend: IoCapable<u16>,
{
self.read(offset)
@ -751,7 +825,7 @@ fn read16(self, offset: usize) -> u16
#[inline(always)]
fn read32(self, offset: usize) -> u32
where
usize: IoLoc<Self::Target, u32, IoType = u32>,
usize: IoLoc<Self::Target, u32>,
Self::Backend: IoCapable<u32>,
{
self.read(offset)
@ -763,7 +837,7 @@ fn read32(self, offset: usize) -> u32
#[inline(always)]
fn read64(self, offset: usize) -> u64
where
usize: IoLoc<Self::Target, u64, IoType = u64>,
usize: IoLoc<Self::Target, u64>,
Self::Backend: IoCapable<u64>,
{
self.read(offset)
@ -775,7 +849,7 @@ fn read64(self, offset: usize) -> u64
#[inline(always)]
fn write8(self, value: u8, offset: usize)
where
usize: IoLoc<Self::Target, u8, IoType = u8>,
usize: IoLoc<Self::Target, u8>,
Self::Backend: IoCapable<u8>,
{
self.write(offset, value)
@ -787,7 +861,7 @@ fn write8(self, value: u8, offset: usize)
#[inline(always)]
fn write16(self, value: u16, offset: usize)
where
usize: IoLoc<Self::Target, u16, IoType = u16>,
usize: IoLoc<Self::Target, u16>,
Self::Backend: IoCapable<u16>,
{
self.write(offset, value)
@ -799,7 +873,7 @@ fn write16(self, value: u16, offset: usize)
#[inline(always)]
fn write32(self, value: u32, offset: usize)
where
usize: IoLoc<Self::Target, u32, IoType = u32>,
usize: IoLoc<Self::Target, u32>,
Self::Backend: IoCapable<u32>,
{
self.write(offset, value)
@ -811,7 +885,7 @@ fn write32(self, value: u32, offset: usize)
#[inline(always)]
fn write64(self, value: u64, offset: usize)
where
usize: IoLoc<Self::Target, u64, IoType = u64>,
usize: IoLoc<Self::Target, u64>,
Self::Backend: IoCapable<u64>,
{
self.write(offset, value)
@ -843,11 +917,11 @@ fn write64(self, value: u64, offset: usize)
#[inline(always)]
fn try_read<T, L>(self, location: L) -> Result<T>
where
T: AsReprMut,
L: IoLoc<Self::Target, T>,
Self::Backend: IoCapable<L::IoType>,
Self::Backend: IoCapable<<T as AsRepr>::Repr>,
{
let view = io_view::<Self, L::IoType>(self, location.offset())?;
Ok(Self::Backend::io_read(view).into())
Ok(io_read!(self, try: location))
}
/// Generic fallible write with runtime bounds check.
@ -876,12 +950,11 @@ fn try_read<T, L>(self, location: L) -> Result<T>
#[inline(always)]
fn try_write<T, L>(self, location: L, value: T) -> Result
where
T: AsRepr,
L: IoLoc<Self::Target, T>,
Self::Backend: IoCapable<L::IoType>,
Self::Backend: IoCapable<<T as AsRepr>::Repr>,
{
let view = io_view::<Self, L::IoType>(self, location.offset())?;
let io_value = value.into();
Self::Backend::io_write(view, io_value);
io_write!(self, try: location, value);
Ok(())
}
@ -900,6 +973,8 @@ fn try_write<T, L>(self, location: L, value: T) -> Result
/// };
///
/// register! {
/// base: Region;
///
/// VERSION(u32) @ 0x100 {
/// 15:8 major;
/// 7:0 minor;
@ -920,9 +995,10 @@ fn try_write<T, L>(self, location: L, value: T) -> Result
#[inline(always)]
fn try_write_reg<T, L, V>(self, value: V) -> Result
where
T: AsRepr,
L: IoLoc<Self::Target, T>,
V: LocatedRegister<Self::Target, Location = L, Value = T>,
Self::Backend: IoCapable<L::IoType>,
Self::Backend: IoCapable<<T as AsRepr>::Repr>,
{
let (location, value) = value.into_io_op();
@ -954,16 +1030,13 @@ fn try_write_reg<T, L, V>(self, value: V) -> Result
#[inline(always)]
fn try_update<T, L, F>(self, location: L, f: F) -> Result
where
T: AsReprMut,
L: IoLoc<Self::Target, T>,
Self::Backend: IoCapable<L::IoType>,
Self::Backend: IoCapable<<T as AsRepr>::Repr>,
F: FnOnce(T) -> T,
{
let view = io_view::<Self, L::IoType>(self, location.offset())?;
let value: T = Self::Backend::io_read(view).into();
let io_value = f(value).into();
Self::Backend::io_write(view, io_value);
let view = io_project!(self, try: location);
view.write_val(f(view.read_val()));
Ok(())
}
@ -991,11 +1064,11 @@ fn try_update<T, L, F>(self, location: L, f: F) -> Result
#[inline(always)]
fn read<T, L>(self, location: L) -> T
where
T: AsReprMut,
L: IoLoc<Self::Target, T>,
Self::Backend: IoCapable<L::IoType>,
Self::Backend: IoCapable<<T as AsRepr>::Repr>,
{
let view = io_view_assert::<Self, L::IoType>(self, location.offset());
Self::Backend::io_read(view).into()
io_read!(self, build: location)
}
/// Generic infallible write with compile-time bounds check.
@ -1022,12 +1095,11 @@ fn read<T, L>(self, location: L) -> T
#[inline(always)]
fn write<T, L>(self, location: L, value: T)
where
T: AsRepr,
L: IoLoc<Self::Target, T>,
Self::Backend: IoCapable<L::IoType>,
Self::Backend: IoCapable<<T as AsRepr>::Repr>,
{
let view = io_view_assert::<Self, L::IoType>(self, location.offset());
let io_value = value.into();
Self::Backend::io_write(view, io_value);
io_write!(self, build: location, value);
}
/// Generic infallible write of a fully-located register value.
@ -1045,6 +1117,8 @@ fn write<T, L>(self, location: L, value: T)
/// };
///
/// register! {
/// base: Region<0x1000>;
///
/// VERSION(u32) @ 0x100 {
/// 15:8 major;
/// 7:0 minor;
@ -1064,9 +1138,10 @@ fn write<T, L>(self, location: L, value: T)
#[inline(always)]
fn write_reg<T, L, V>(self, value: V)
where
T: AsRepr,
L: IoLoc<Self::Target, T>,
V: LocatedRegister<Self::Target, Location = L, Value = T>,
Self::Backend: IoCapable<L::IoType>,
Self::Backend: IoCapable<<T as AsRepr>::Repr>,
{
let (location, value) = value.into_io_op();
@ -1098,14 +1173,13 @@ fn write_reg<T, L, V>(self, value: V)
#[inline(always)]
fn update<T, L, F>(self, location: L, f: F)
where
T: AsReprMut,
L: IoLoc<Self::Target, T>,
Self::Backend: IoCapable<L::IoType>,
Self::Backend: IoCapable<<T as AsRepr>::Repr>,
F: FnOnce(T) -> T,
{
let view = io_view_assert::<Self, L::IoType>(self, location.offset());
let value: T = Self::Backend::io_read(view).into();
let io_value = f(value).into();
Self::Backend::io_write(view, io_value);
let view = io_project!(self, build: location);
view.write_val(f(view.read_val()));
}
}
@ -1649,6 +1723,25 @@ pub unsafe fn project_view<U: ?Sized + KnownSize>(
// SAFETY: Per safety requirement.
unsafe { T::Backend::project_view::<T::Target, _>(self.0, ptr) }
}
#[inline(always)]
pub fn try_project_loc<U, L>(
self,
location: L,
) -> Result<<T::Backend as IoBackend>::View<'a, U>>
where
L: IoLoc<T::Target, U>,
{
io_view::<_, U>(self.0, location.offset())
}
#[inline(always)]
pub fn project_loc<U, L>(self, location: L) -> <T::Backend as IoBackend>::View<'a, U>
where
L: IoLoc<T::Target, U>,
{
io_view_assert::<_, U>(self.0, location.offset())
}
}
/// Project an I/O type to a subview of it.
@ -1656,26 +1749,54 @@ pub unsafe fn project_view<U: ?Sized + KnownSize>(
/// The syntax is of form `io_project!(io, proj)` where `io` is an expression to a type that
/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!).
///
/// `io_project!` can also project to a subview of registers defined with [`register!`] macro.
/// Register projection has syntax `io_project!(io, try: REGISTER)` for fallible projection and
/// `io_project!(io, build: REGISTER)` for infallible projection.
///
/// # Examples
///
/// ```
/// use kernel::io::{
/// io_project,
/// register,
/// Mmio,
/// };
/// #[repr(C)]
/// struct MyStruct { field: u32, }
///
/// register! {
/// base: MyStruct;
/// FIELD(u32) @ 0 {
/// 31:0 val;
/// }
/// }
///
/// # fn test(mmio: Mmio<'_, [MyStruct]>) -> Result {
/// // let mmio: Mmio<[MyStruct]>;
/// let field: Mmio<'_, u32> = io_project!(mmio, [try: 1].field);
/// let whole: Mmio<'_, MyStruct> = io_project!(mmio, [try: 2]);
/// let nested: Mmio<'_, u32> = io_project!(whole, .field);
/// let reg: Mmio<'_, FIELD> = io_project!(whole, build: FIELD);
/// # Ok::<(), Error>(()) }
/// ```
#[macro_export]
#[doc(hidden)]
macro_rules! io_project {
// Register projection
($io:expr, try: $ioloc:expr) => {{
#[allow(unused)]
use $crate::io::IoBase as _;
let view = $crate::io::ProjectHelper($io.as_view());
view.try_project_loc($ioloc)?
}};
($io:expr, build: $ioloc:expr) => {{
#[allow(unused)]
use $crate::io::IoBase as _;
let view = $crate::io::ProjectHelper($io.as_view());
view.project_loc($ioloc)
}};
// Field or index projection
($io:expr, $($proj:tt)*) => {{
#[allow(unused)]
use $crate::io::IoBase as _;
@ -1746,6 +1867,12 @@ macro_rules! io_write {
(@parse [$io:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => {
$crate::io_write!(@parse [$io] [$($proj)* [$flavor: $index]] [$($rest)*])
};
(@parse [$io:expr] [] [try: $ioloc:expr, $($rest:tt)*]) => {
$crate::io_write!(@parse [$io] [try: $ioloc] [, $($rest)*])
};
(@parse [$io:expr] [] [build: $ioloc:expr, $($rest:tt)*]) => {
$crate::io_write!(@parse [$io] [build: $ioloc] [, $($rest)*])
};
($io:expr, $($rest:tt)*) => {
$crate::io_write!(@parse [$io] [] [$($rest)*])
};

803
rust/kernel/io/register.rs
View File

@ -8,14 +8,19 @@
//!
//! Note: most of the items in this module are public so they can be referenced by the macro, but
//! most are not to be used directly by users. Outside of the `register!` macro itself, the only
//! items you might want to import from this module are [`WithBase`] and [`Array`].
//! item you might want to import from this module is [`Array`].
//!
//! # Simple example
//!
//! ```no_run
//! use kernel::io::register;
//! use kernel::io::{
//! register,
//! Region,
//! };
//!
//! register! {
//! base: Region<0x1000>;
//!
//! /// Basic information about the chip.
//! pub BOOT_0(u32) @ 0x00000100 {
//! /// Vendor ID.
@ -55,11 +60,14 @@
//! register,
//! Io,
//! IoLoc,
//! Region,
//! },
//! num::Bounded,
//! };
//! # use kernel::io::{Mmio, Region};
//! # use kernel::io::Mmio;
//! # register! {
//! # base: Region<0x1000>;
//! #
//! # pub BOOT_0(u32) @ 0x00000100 {
//! # 15:8 vendor_id;
//! # 7:4 major_revision;
@ -113,149 +121,50 @@
io::IoLoc, //
};
use super::Region;
/// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when
/// passing a [`FixedIoLoc`] value.
impl<Base: ?Sized, T> IoLoc<Base, T> for ()
where
T: FixedIoLoc<Base>,
{
#[inline(always)]
fn offset(self) -> usize {
T::LOCATION.offset()
}
}
/// Trait implemented by all registers.
pub trait Register: Sized {
/// Backing primitive type of the register.
type Storage: Into<Self> + From<Self>;
// Provides a `IoLoc` impl that for a fixed offset.
#[doc(hidden)]
pub struct OffsetLoc<Base: ?Sized, T>(usize, PhantomData<(T, Base)>);
impl<Base: ?Sized, T> OffsetLoc<Base, T> {
#[inline]
pub const fn new(offset: usize) -> Self {
Self(offset, PhantomData)
}
#[inline]
pub const fn const_offset(self) -> usize {
self.0
}
}
impl<Base: ?Sized, T> IoLoc<Base, T> for OffsetLoc<Base, T> {
#[inline(always)]
fn offset(self) -> usize {
self.0
}
}
/// Trait implemented by arrays of registers.
pub trait RegisterArray: Sized {
/// Base type for this register.
type Base: ?Sized;
/// Start offset of the register.
///
/// The interpretation of this offset depends on the type of the register.
const OFFSET: usize;
}
/// Trait implemented by registers with a fixed offset.
pub trait FixedRegister: Register {}
/// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when
/// passing a [`FixedRegister`] value.
impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for ()
where
T: FixedRegister,
{
type IoType = T::Storage;
#[inline(always)]
fn offset(self) -> usize {
T::OFFSET
}
}
/// A [`FixedRegister`] carries its location in its type. Thus `FixedRegister` values can be used
/// as an [`IoLoc`].
impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for T
where
T: FixedRegister,
{
type IoType = T::Storage;
#[inline(always)]
fn offset(self) -> usize {
T::OFFSET
}
}
/// Location of a fixed register.
pub struct FixedRegisterLoc<T: FixedRegister>(PhantomData<T>);
impl<T: FixedRegister> FixedRegisterLoc<T> {
/// Returns the location of `T`.
#[inline(always)]
// We do not implement `Default` so we can be const.
#[expect(clippy::new_without_default)]
pub const fn new() -> Self {
Self(PhantomData)
}
}
impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for FixedRegisterLoc<T>
where
T: FixedRegister,
{
type IoType = T::Storage;
#[inline(always)]
fn offset(self) -> usize {
T::OFFSET
}
}
/// Trait providing a base address to be added to the offset of a relative register to obtain
/// its actual offset.
///
/// The `T` generic argument is used to distinguish which base to use, in case a type provides
/// several bases. It is given to the `register!` macro to restrict the use of the register to
/// implementors of this particular variant.
pub trait RegisterBase<T> {
/// Base address to which register offsets are added.
const BASE: usize;
}
/// Trait implemented by all registers that are relative to a base.
pub trait WithBase {
/// Family of bases applicable to this register.
type BaseFamily;
/// Returns the absolute location of this type when using `B` as its base.
#[inline(always)]
fn of<B: RegisterBase<Self::BaseFamily>>() -> RelativeRegisterLoc<Self, B>
where
Self: Register,
{
RelativeRegisterLoc::new()
}
}
/// Trait implemented by relative registers.
pub trait RelativeRegister: Register + WithBase {}
/// Location of a relative register.
///
/// This can either be an immediately accessible regular [`RelativeRegister`], or a
/// [`RelativeRegisterArray`] that needs one additional resolution through
/// [`RelativeRegisterLoc::at`].
pub struct RelativeRegisterLoc<T: WithBase, B: ?Sized>(PhantomData<T>, PhantomData<B>);
impl<T, B> RelativeRegisterLoc<T, B>
where
T: Register + WithBase,
B: RegisterBase<T::BaseFamily> + ?Sized,
{
/// Returns the location of a relative register or register array.
#[inline(always)]
// We do not implement `Default` so we can be const.
#[expect(clippy::new_without_default)]
pub const fn new() -> Self {
Self(PhantomData, PhantomData)
}
// Returns the absolute offset of the relative register using base `B`.
//
// This is implemented as a private const method so it can be reused by the [`IoLoc`]
// implementations of both [`RelativeRegisterLoc`] and [`RelativeRegisterArrayLoc`].
#[inline]
const fn offset(self) -> usize {
B::BASE + T::OFFSET
}
}
impl<const SIZE: usize, T, B> IoLoc<Region<SIZE>, T> for RelativeRegisterLoc<T, B>
where
T: RelativeRegister,
B: RegisterBase<T::BaseFamily> + ?Sized,
{
type IoType = T::Storage;
#[inline(always)]
fn offset(self) -> usize {
RelativeRegisterLoc::offset(self)
}
}
/// Trait implemented by arrays of registers.
pub trait RegisterArray: Register {
/// Number of elements in the registers array.
const SIZE: usize;
/// Number of bytes between the start of elements in the registers array.
@ -285,12 +194,10 @@ pub fn try_new(idx: usize) -> Option<Self> {
}
}
impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for RegisterArrayLoc<T>
impl<Base: ?Sized, T> IoLoc<Base, T> for RegisterArrayLoc<T>
where
T: RegisterArray,
T: RegisterArray<Base = Base>,
{
type IoType = T::Storage;
#[inline(always)]
fn offset(self) -> usize {
T::OFFSET + self.0 * T::STRIDE
@ -318,71 +225,15 @@ fn try_at(idx: usize) -> Option<RegisterArrayLoc<Self>>
}
}
/// Trait implemented by arrays of relative registers.
pub trait RelativeRegisterArray: RegisterArray + WithBase {}
/// Trait implemented by types that indicate there is a fixed I/O location for this given type.
///
/// Implementors can be used with [`Io::write_reg`](super::Io::write_reg).
pub trait FixedIoLoc<Base: ?Sized>: Sized {
/// Type of [`FixedIoLoc::LOCATION`].
type Location: IoLoc<Base, Self>;
/// Location of a relative array register.
pub struct RelativeRegisterArrayLoc<
T: RelativeRegisterArray,
B: RegisterBase<T::BaseFamily> + ?Sized,
>(RelativeRegisterLoc<T, B>, usize);
impl<T, B> RelativeRegisterArrayLoc<T, B>
where
T: RelativeRegisterArray,
B: RegisterBase<T::BaseFamily> + ?Sized,
{
/// Returns the location of register `T` from the base `B` at index `idx`, with build-time
/// validation.
#[inline(always)]
pub fn new(idx: usize) -> Self {
build_assert!(idx < T::SIZE);
Self(RelativeRegisterLoc::new(), idx)
}
/// Attempts to return the location of register `T` from the base `B` at index `idx`, with
/// runtime validation.
#[inline(always)]
pub fn try_new(idx: usize) -> Option<Self> {
if idx < T::SIZE {
Some(Self(RelativeRegisterLoc::new(), idx))
} else {
None
}
}
}
/// Methods exclusive to [`RelativeRegisterLoc`]s created with a [`RelativeRegisterArray`].
impl<T, B> RelativeRegisterLoc<T, B>
where
T: RelativeRegisterArray,
B: RegisterBase<T::BaseFamily> + ?Sized,
{
/// Returns the location of the register at position `idx`, with build-time validation.
#[inline(always)]
pub fn at(self, idx: usize) -> RelativeRegisterArrayLoc<T, B> {
RelativeRegisterArrayLoc::new(idx)
}
/// Returns the location of the register at position `idx`, with runtime validation.
#[inline(always)]
pub fn try_at(self, idx: usize) -> Option<RelativeRegisterArrayLoc<T, B>> {
RelativeRegisterArrayLoc::try_new(idx)
}
}
impl<const SIZE: usize, T, B> IoLoc<Region<SIZE>, T> for RelativeRegisterArrayLoc<T, B>
where
T: RelativeRegisterArray,
B: RegisterBase<T::BaseFamily> + ?Sized,
{
type IoType = T::Storage;
#[inline(always)]
fn offset(self) -> usize {
self.0.offset() + self.1 * T::STRIDE
}
/// Location of this type within given base.
const LOCATION: Self::Location;
}
/// Trait implemented by items that contain both a register value and the absolute I/O location at
@ -390,8 +241,8 @@ fn offset(self) -> usize {
///
/// Implementors can be used with [`Io::write_reg`](super::Io::write_reg).
pub trait LocatedRegister<Base: ?Sized> {
/// Register value to write.
type Value: Register;
/// Value to write.
type Value;
/// Full location information at which to write the value.
type Location: IoLoc<Base, Self::Value>;
@ -400,27 +251,38 @@ pub trait LocatedRegister<Base: ?Sized> {
fn into_io_op(self) -> (Self::Location, Self::Value);
}
impl<const SIZE: usize, T> LocatedRegister<Region<SIZE>> for T
impl<Base: ?Sized, T> LocatedRegister<Base> for T
where
T: FixedRegister,
T: FixedIoLoc<Base>,
{
type Location = FixedRegisterLoc<Self::Value>;
type Location = T::Location;
type Value = T;
#[inline(always)]
fn into_io_op(self) -> (FixedRegisterLoc<T>, T) {
(FixedRegisterLoc::new(), self)
fn into_io_op(self) -> (T::Location, T) {
(T::LOCATION, self)
}
}
/// Helper function for register element alias implementation.
///
/// This is used to enforce base matching and provide bounds checking.
#[doc(hidden)]
#[inline(always)] // for const eval only
pub const fn element_alias_offset<Base: ?Sized, Alias: RegisterArray<Base = Base>>(
idx: usize,
) -> usize {
assert!(idx < Alias::SIZE);
Alias::OFFSET + idx * Alias::STRIDE
}
/// Defines a dedicated type for a register, including getter and setter methods for its fields and
/// methods to read and write it from an [`Io`](kernel::io::Io) region.
///
/// This documentation focuses on how to declare registers. See the [module-level
/// documentation](mod@kernel::io::register) for examples of how to access them.
///
/// There are 4 possible kinds of registers: fixed offset registers, relative registers, arrays of
/// registers, and relative arrays of registers.
/// Registers can either be fixed offset registers or arrays of registers.
///
/// ## Fixed offset registers
///
@ -444,11 +306,14 @@ fn into_io_op(self) -> (FixedRegisterLoc<T>, T) {
/// io::{
/// register,
/// Io,
/// Region,
/// },
/// };
/// # use kernel::io::{Mmio, Region};
/// # use kernel::io::Mmio;
///
/// register! {
/// base: Region<0x1000>;
///
/// FIXED_REG(u32) @ 0x100 {
/// 15:8 high_byte;
/// 7:0 low_byte;
@ -479,9 +344,14 @@ fn into_io_op(self) -> (FixedRegisterLoc<T>, T) {
/// the context:
///
/// ```no_run
/// use kernel::io::register;
/// use kernel::io::{
/// register,
/// Region,
/// };
///
/// register! {
/// base: Region<0x1000>;
///
/// /// Scratch register.
/// pub SCRATCH(u32) @ 0x00000200 {
/// 31:0 value;
@ -497,113 +367,45 @@ fn into_io_op(self) -> (FixedRegisterLoc<T>, T) {
/// In this example, `SCRATCH_BOOT_STATUS` uses the same I/O address as `SCRATCH`, while providing
/// its own `completed` field.
///
/// ## Relative registers
///
/// Relative registers can be instantiated several times at a relative offset of a group of bases.
/// For instance, imagine the following I/O space:
///
/// ```text
/// +-----------------------------+
/// | ... |
/// | |
/// 0x100--->+------------CPU0-------------+
/// | |
/// 0x110--->+-----------------------------+
/// | CPU_CTL |
/// +-----------------------------+
/// | ... |
/// | |
/// | |
/// 0x200--->+------------CPU1-------------+
/// | |
/// 0x210--->+-----------------------------+
/// | CPU_CTL |
/// +-----------------------------+
/// | ... |
/// +-----------------------------+
/// ```
///
/// `CPU0` and `CPU1` both have a `CPU_CTL` register that starts at offset `0x10` of their I/O
/// space segment. Since both instances of `CPU_CTL` share the same layout, we don't want to define
/// them twice and would prefer a way to select which one to use from a single definition.
///
/// This can be done using the `Base + Offset` syntax when specifying the register's address:
///
/// ```ignore
/// register! {
/// pub RELATIVE_REG(u32) @ Base + 0x80 {
/// ...
/// }
/// }
/// ```
///
/// This creates a register with an offset of `0x80` from a given base.
///
/// `Base` is an arbitrary type (typically a ZST) to be used as a generic parameter of the
/// [`RegisterBase`] trait to provide the base as a constant, i.e. each type providing a base for
/// this register needs to implement `RegisterBase<Base>`.
///
/// The location of relative registers can be built using the [`WithBase::of`] method to specify
/// its base. All relative registers implement [`WithBase`].
///
/// Here is the above layout translated into code:
/// If you do not wish to have a bitfield defined, you can also create a register using an existing
/// type.
///
/// ```no_run
/// use kernel::{
/// io::{
/// register,
/// register::{
/// RegisterBase,
/// WithBase,
/// },
/// Io,
/// },
/// };
/// # use kernel::io::{Mmio, Region};
///
/// // Type used to identify the base.
/// pub struct CpuCtlBase;
///
/// // ZST describing `CPU0`.
/// struct Cpu0;
/// impl RegisterBase<CpuCtlBase> for Cpu0 {
/// const BASE: usize = 0x100;
/// }
///
/// // ZST describing `CPU1`.
/// struct Cpu1;
/// impl RegisterBase<CpuCtlBase> for Cpu1 {
/// const BASE: usize = 0x200;
/// }
///
/// // This makes `CPU_CTL` accessible from all implementors of `RegisterBase<CpuCtlBase>`.
/// # use kernel::io::*;
/// register! {
/// /// CPU core control.
/// pub CPU_CTL(u32) @ CpuCtlBase + 0x10 {
/// 0:0 start;
/// base: Region<0x1000>;
///
/// /// UART RX register.
/// pub UART_RX: u8 @ 0x100;
/// }
/// ```
///
/// In case there is a fixed register associated with a specific type in the base, you can apply
/// `#[unique]` attribute which enables `write_reg` shorthand. This is automatically applied to
/// bitfields instantiated via the `register!` macro.
///
/// This should only be used when types meaningfully represent a register. For example, in the
/// previous `UART_RX` example, even if only a single register is defined with `u8` type, it is a
/// bad idea to annotate it with `#[unique]`.
///
/// ```no_run
/// # use kernel::{bitfield, io::*};
///
/// bitfield! {
/// pub struct Reset(u32) {
/// 0:0 reset;
/// }
/// }
///
/// # fn test(io: Mmio<'_, Region<0x1000>>) {
/// // Read the status of `Cpu0`.
/// let cpu0_started = io.read(CPU_CTL::of::<Cpu0>());
///
/// // Stop `Cpu0`.
/// io.write(WithBase::of::<Cpu0>(), CPU_CTL::zeroed());
/// # }
///
/// // Aliases can also be defined for relative register.
/// register! {
/// /// Alias to CPU core control.
/// pub CPU_CTL_ALIAS(u32) => CpuCtlBase + CPU_CTL {
/// /// Start the aliased CPU core.
/// 1:1 alias_start;
/// }
/// base: Region<0x1000>;
///
/// pub RESET: #[unique] Reset @ 0x100;
/// }
///
/// # fn test2(io: Mmio<'_, Region<0x1000>>) {
/// // Start the aliased `CPU0`, leaving its other fields untouched.
/// io.update(CPU_CTL_ALIAS::of::<Cpu0>(), |r| r.with_alias_start(true));
/// # fn test(mmio: Mmio<'_, Region<0x1000>>) {
/// // let mmio: Mmio<'_, Region<0x1000>>;
/// mmio.write_reg(Reset::zeroed().with_const_reset::<1>());
/// # }
/// ```
///
@ -636,15 +438,18 @@ fn into_io_op(self) -> (FixedRegisterLoc<T>, T) {
/// register,
/// register::Array,
/// Io,
/// Region,
/// },
/// };
/// # use kernel::io::{Mmio, Region};
/// # use kernel::io::Mmio;
/// # fn get_scratch_idx() -> usize {
/// # 0x15
/// # }
///
/// // Array of 64 consecutive registers with the same layout starting at offset `0x80`.
/// register! {
/// base: Region<0x1000>;
///
/// /// Scratch registers.
/// pub SCRATCH(u32)[64] @ 0x00000080 {
/// 31:0 value;
@ -670,6 +475,8 @@ fn into_io_op(self) -> (FixedRegisterLoc<T>, T) {
/// // Alias to a specific register in an array.
/// // Here `SCRATCH[8]` is used to convey the firmware exit code.
/// register! {
/// base: Region<0x1000>;
///
/// /// Firmware exit status code.
/// pub FIRMWARE_STATUS(u32) => SCRATCH[8] {
/// 7:0 status;
@ -682,6 +489,8 @@ fn into_io_op(self) -> (FixedRegisterLoc<T>, T) {
/// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the
/// // registers of the two declarations below are interleaved.
/// register! {
/// base: Region<0x1000>;
///
/// /// Scratch registers bank 0.
/// pub SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ 0x000000c0 {
/// 31:0 value;
@ -696,332 +505,88 @@ fn into_io_op(self) -> (FixedRegisterLoc<T>, T) {
/// # }
/// ```
///
/// ## Relative arrays of registers
/// ## Relative registers
///
/// Combining the two features described in the sections above, arrays of registers accessible from
/// a base can also be defined:
/// There are cases where a register region is subdivided into small subregions, and you may wish to
/// have your register definition be relative to these subregions. This may be needed, for example,
/// if these subregions are instantiated several times, or you just want it for encapsulation
/// purpose.
///
/// ```ignore
/// register! {
/// pub RELATIVE_REGISTER_ARRAY(u8)[10, stride = 4] @ Base + 0x100 {
/// ...
/// }
/// }
/// For instance, imagine the following I/O space:
///
/// ```text
/// +-----------------------------+
/// | ... |
/// | |
/// 0x100--->+------------CPU0-------------+
/// | |
/// 0x110--->+-----------------------------+
/// | CPU_CTL |
/// +-----------------------------+
/// | ... |
/// | |
/// | |
/// 0x200--->+------------CPU1-------------+
/// | |
/// 0x210--->+-----------------------------+
/// | CPU_CTL |
/// +-----------------------------+
/// | ... |
/// +-----------------------------+
/// ```
///
/// Like relative registers, they implement the [`WithBase`] trait. However the return value of
/// [`WithBase::of`] cannot be used directly as a location and must be further specified using the
/// [`at`](RelativeRegisterLoc::at) method.
/// `CPU0` and `CPU1` both have a `CPU_CTL` register that starts at offset `0x10` of their I/O
/// space segment. Since both instances of `CPU_CTL` share the same layout, we don't want to define
/// them twice and would prefer a way to select which one to use from a single definition.
///
/// This can be done by defining a new type for the subregion, and then defining registers that use
/// the new type as the base:
///
/// ```no_run
/// use kernel::{
/// io::{
/// io_project,
/// register,
/// register::{
/// RegisterBase,
/// WithBase,
/// },
/// Io,
/// Region,
/// },
/// };
/// # use kernel::io::{Mmio, Region};
/// # fn get_scratch_idx() -> usize {
/// # 0x15
/// # }
/// # use kernel::io::Mmio;
///
/// // Type used as parameter of `RegisterBase` to specify the base.
/// pub struct CpuCtlBase;
/// // Subregion type. Make sure it has adequate size and alignment.
/// #[repr(align(4))]
/// #[derive(FromBytes, IntoBytes)]
/// pub struct CpuCtl([u8; 0x100]);
///
/// // ZST describing `CPU0`.
/// struct Cpu0;
/// impl RegisterBase<CpuCtlBase> for Cpu0 {
/// const BASE: usize = 0x100;
/// }
///
/// // ZST describing `CPU1`.
/// struct Cpu1;
/// impl RegisterBase<CpuCtlBase> for Cpu1 {
/// const BASE: usize = 0x200;
/// }
///
/// // 64 per-cpu scratch registers, arranged as a contiguous array.
/// register! {
/// /// Per-CPU scratch registers.
/// pub CPU_SCRATCH(u32)[64] @ CpuCtlBase + 0x00000080 {
/// 31:0 value;
/// }
/// base: Region<0x1000>;
///
/// // Subregions can just be defined like normal registers.
/// CPU0: CpuCtl @ 0x100;
/// CPU1: CpuCtl @ 0x200;
/// }
///
/// # fn test(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> {
/// // Read scratch register 0 of CPU0.
/// let scratch = io.read(CPU_SCRATCH::of::<Cpu0>().at(0));
///
/// // Write the retrieved value into scratch register 15 of CPU1.
/// io.write(WithBase::of::<Cpu1>().at(15), scratch);
///
/// // This won't build.
/// // let cpu0_scratch_128 = io.read(CPU_SCRATCH::of::<Cpu0>().at(128)).value();
///
/// // Runtime-obtained array index.
/// let scratch_idx = get_scratch_idx();
/// // Access on a runtime index returns an error if it is out-of-bounds.
/// let cpu0_scratch = io.read(
/// CPU_SCRATCH::of::<Cpu0>().try_at(scratch_idx).ok_or(EINVAL)?
/// ).value();
/// # Ok(())
/// # }
///
/// // Alias to `SCRATCH[8]` used to convey the firmware exit code.
/// // Then you can define new registers on the subregion.
/// register! {
/// /// Per-CPU firmware exit status code.
/// pub CPU_FIRMWARE_STATUS(u32) => CpuCtlBase + CPU_SCRATCH[8] {
/// 7:0 status;
/// base: CpuCtl;
///
/// /// CPU core control.
/// pub CPU_CTL(u32) @ 0x10 {
/// 0:0 start;
/// }
/// }
///
/// // Non-contiguous relative register arrays can be defined by adding a stride parameter.
/// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the
/// // registers of the two declarations below are interleaved.
/// register! {
/// /// Scratch registers bank 0.
/// pub CPU_SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d00 {
/// 31:0 value;
/// }
/// # fn test(io: Mmio<'_, Region<0x1000>>) {
/// // Read the status of `Cpu0`.
/// let cpu0_started = io_project!(io, build: CPU0).read(CPU_CTL);
///
/// /// Scratch registers bank 1.
/// pub CPU_SCRATCH_INTERLEAVED_1(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d04 {
/// 31:0 value;
/// }
/// }
///
/// # fn test2(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> {
/// let cpu0_status = io.read(CPU_FIRMWARE_STATUS::of::<Cpu0>()).status();
/// # Ok(())
/// // Stop `Cpu0`.
/// io_project!(io, build: CPU0).write_reg(CPU_CTL::zeroed());
/// # }
/// ```
#[macro_export]
macro_rules! register {
// Entry point for the macro, allowing multiple registers to be defined in one call.
// It matches all possible register declaration patterns to dispatch them to corresponding
// `@reg` rule that defines a single register.
//
// TODO: change `alias:ident` to `alias:path` once relative registers are replaced by I/O
// projections.
(
$(
$(#[$attr:meta])* $vis:vis $name:ident ($storage:ty)
$([ $size:expr $(, stride = $stride:expr)? ])?
$(@ $($base:ident +)? $offset:literal)?
$(=> $alias:ident $(+ $alias_offset:ident)? $([$alias_idx:expr])? )?
{ $($fields:tt)* }
)*
) => {
$(
$crate::register!(
@reg $(#[$attr])* $vis $name ($storage) $([$size $(, stride = $stride)?])?
$(@ $($base +)? $offset)?
$(=> $alias $(+ $alias_offset)? $([$alias_idx])? )?
{ $($fields)* }
);
)*
};
// All the rules below are private helpers.
// Creates a register at a fixed offset of the MMIO space.
(
@reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $offset:literal
{ $($fields:tt)* }
) => {
$crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
$crate::register!(@io_base $name($storage) @ $offset);
$crate::register!(@io_fixed $(#[$attr])* $vis $name);
};
// Creates an alias register of fixed offset register `alias` with its own fields.
(
@reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path
{ $($fields:tt)* }
) => {
$crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
$crate::register!(
@io_base $name($storage) @
<$alias as $crate::io::register::Register>::OFFSET
);
$crate::register!(@io_fixed $(#[$attr])* $vis $name);
};
// Creates a register at a relative offset from a base address provider.
(
@reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $base:ident + $offset:literal
{ $($fields:tt)* }
) => {
$crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
$crate::register!(@io_base $name($storage) @ $offset);
$crate::register!(@io_relative $name @ $base);
};
// Creates an alias register of relative offset register `alias` with its own fields.
(
@reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $base:ident + $alias:ident
{ $($fields:tt)* }
) => {
$crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
$crate::register!(
@io_base $name($storage) @ <$alias as $crate::io::register::Register>::OFFSET
);
$crate::register!(@io_relative $name @ $base);
};
// Creates an array of registers at a fixed offset of the MMIO space.
(
@reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty)
[ $size:expr, stride = $stride:expr ] @ $offset:literal { $($fields:tt)* }
) => {
$crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride);
$crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
$crate::register!(@io_base $name($storage) @ $offset);
$crate::register!(@io_array $name [ $size, stride = $stride ]);
};
// Shortcut for contiguous array of registers (stride == size of element).
(
@reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ] @ $offset:literal
{ $($fields:tt)* }
) => {
$crate::register!(
@reg $(#[$attr])* $vis $name($storage)
[ $size, stride = ::core::mem::size_of::<$storage>() ]
@ $offset { $($fields)* }
);
};
// Creates an alias of register `idx` of array of registers `alias` with its own fields.
(
@reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path [ $idx:expr ]
{ $($fields:tt)* }
) => {
$crate::build_assert::static_assert!(
$idx < <$alias as $crate::io::register::RegisterArray>::SIZE
);
$crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
$crate::register!(
@io_base $name($storage) @
<$alias as $crate::io::register::Register>::OFFSET
+ $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE
);
$crate::register!(@io_fixed $(#[$attr])* $vis $name);
};
// Creates an array of registers at a relative offset from a base address provider.
(
@reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty)
[ $size:expr, stride = $stride:expr ]
@ $base:ident + $offset:literal { $($fields:tt)* }
) => {
$crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride);
$crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
$crate::register!(@io_base $name($storage) @ $offset);
$crate::register!(@io_relative_array $name [ $size, stride = $stride ] @ $base);
};
// Shortcut for contiguous array of relative registers (stride == size of element).
(
@reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ]
@ $base:ident + $offset:literal { $($fields:tt)* }
) => {
$crate::register!(
@reg $(#[$attr])* $vis $name($storage)
[ $size, stride = ::core::mem::size_of::<$storage>() ]
@ $base + $offset { $($fields)* }
);
};
// Creates an alias of register `idx` of relative array of registers `alias` with its own
// fields.
(
@reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty)
=> $base:ident + $alias:ident [ $idx:expr ] { $($fields:tt)* }
) => {
$crate::build_assert::static_assert!(
$idx < <$alias as $crate::io::register::RegisterArray>::SIZE
);
$crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
$crate::register!(
@io_base $name($storage) @
<$alias as $crate::io::register::Register>::OFFSET +
$idx * <$alias as $crate::io::register::RegisterArray>::STRIDE
);
$crate::register!(@io_relative $name @ $base);
};
// Generates the bitfield for the register.
//
// `#[allow(non_camel_case_types)]` is added since register names typically use
// `SCREAMING_CASE`.
(
@bitfield $(#[$attr:meta])* $vis:vis struct $name:ident($storage:ty) { $($fields:tt)* }
) => {
$crate::bitfield!(
#[allow(non_camel_case_types)]
$(#[$attr])* $vis struct $name($storage) { $($fields)* }
);
};
// Implementations shared by all registers types.
(@io_base $name:ident($storage:ty) @ $offset:expr) => {
impl $crate::io::register::Register for $name {
type Storage = $storage;
const OFFSET: usize = $offset;
}
};
// Implementations of fixed registers.
(@io_fixed $(#[$attr:meta])* $vis:vis $name:ident) => {
impl $crate::io::register::FixedRegister for $name {}
$(#[$attr])*
$vis const $name: $crate::io::register::FixedRegisterLoc<$name> =
$crate::io::register::FixedRegisterLoc::<$name>::new();
};
// Implementations of relative registers.
(@io_relative $name:ident @ $base:ident) => {
impl $crate::io::register::WithBase for $name {
type BaseFamily = $base;
}
impl $crate::io::register::RelativeRegister for $name {}
};
// Implementations of register arrays.
(@io_array $name:ident [ $size:expr, stride = $stride:expr ]) => {
impl $crate::io::register::Array for $name {}
impl $crate::io::register::RegisterArray for $name {
const SIZE: usize = $size;
const STRIDE: usize = $stride;
}
};
// Implementations of relative array registers.
(
@io_relative_array $name:ident [ $size:expr, stride = $stride:expr ] @ $base:ident
) => {
impl $crate::io::register::WithBase for $name {
type BaseFamily = $base;
}
impl $crate::io::register::RegisterArray for $name {
const SIZE: usize = $size;
const STRIDE: usize = $stride;
}
impl $crate::io::register::RelativeRegisterArray for $name {}
($($tt:tt)*) => {
$crate::macros::register!($($tt)*);
};
}

6
rust/kernel/io/resource.rs
View File

@ -58,7 +58,7 @@ fn drop(&mut self) {
};
// SAFETY: Safe as per the invariant of `Region`.
unsafe { release_fn(start, size) };
unsafe { release_fn(start, size.into_raw()) };
}
}
@ -114,7 +114,7 @@ pub fn request_region(
bindings::__request_region(
self.0.get(),
start,
size,
size.into_raw(),
name.as_char_ptr(),
flags.0 as c_int,
)
@ -130,7 +130,7 @@ pub fn request_region(
pub fn size(&self) -> ResourceSize {
let inner = self.0.get();
// SAFETY: Safe as per the invariants of `Resource`.
unsafe { bindings::resource_size(inner) }
ResourceSize::from_raw(unsafe { bindings::resource_size(inner) })
}
/// Returns the start address of the resource.

230
rust/kernel/mem.rs
View File

@ -4,6 +4,7 @@
// TODO: `Alignment` should be moved here as well, Rust moved it upstream in 1.96.
pub use crate::ptr::Alignment;
use crate::prelude::*;
/// Trait indicating that [`Align<N>`] is a valid alignment.
pub trait ValidAlign {
@ -68,3 +69,232 @@ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
write!(f, "Align<{N}>")
}
}
/// Transmute between two types.
///
/// Use this instead of [`core::mem::transmute`] when it is known that sizes are identical but this
/// cannot be proven by the compiler.
///
/// This is equivalent to Rust's `transmute_unchecked` intrinsics.
///
/// # Safety
///
/// All safety requirements of [`core::mem::transmute`] apply, plus that the size `Src` and `Dst`
/// must match.
///
/// # Examples
///
/// This can be used when types are known to have the same size, but only at runtime.
///
/// ```no_run
/// # use core::any::TypeId;
/// fn to_u32<T: 'static>(v: T) -> Option<u32> {
/// if TypeId::of::<T>() != TypeId::of::<u32>() {
/// return None;
/// }
///
/// // `core::mem::transmute` won't work here.
/// // SAFETY: We've checked that `T` is `u32`!
/// Some(unsafe { kernel::mem::transmute_unchecked(v) })
/// }
///
/// to_u32(1u32);
/// ```
#[inline(always)]
pub const unsafe fn transmute_unchecked<Src, Dst>(val: Src) -> Dst {
// SAFETY: This is identical to `transmute` except that we bypassed the size check; which is
// true per safety requirement.
unsafe { core::mem::transmute_copy(&core::mem::ManuallyDrop::new(val)) }
}
/// Version of `transmute` that performs size check at monomorphization-time.
///
/// Use this instead of [`core::mem::transmute`] when it is known that sizes are identical but this
/// cannot be proven by the compiler during type checking and can be proven during monomorphization.
///
/// The signature is equivalent to Rust standard library's unstable `transmute_neo` and that of
/// [RFC 3844](https://github.com/rust-lang/rfcs/pull/3844).
///
/// # Safety
///
/// Same as [`core::mem::transmute`].
///
/// # Examples
///
/// This is typically used in generic code where it's known that type will have the same size, but
/// the compiler cannot prove it generically.
///
/// ```no_run
/// trait IsU32 {}
/// impl IsU32 for u32 {}
///
/// fn to_u32<T: IsU32>(v: T) -> u32 {
/// // `core::mem::transmute` won't work here.
/// // SAFETY: We know that `v` is u32!
/// unsafe { kernel::mem::transmute(v) }
/// }
///
/// to_u32(1u32);
/// ```
#[inline(always)]
pub const unsafe fn transmute<Src, Dst>(val: Src) -> Dst {
const_assert!(size_of::<Src>() == size_of::<Dst>());
// SAFETY: Size is checked above. Other safety requirements follow those of the function.
unsafe { transmute_unchecked(val) }
}
/// Safely transmutes a value of one type to a value of another type of the same size.
///
/// The sizes are checked during monomorphization.
///
/// This can be considered as generic version of [`zerocopy::transmute!`] macro that defers the size
/// check and thus can be used in more cases.
///
/// # Examples
///
/// ```no_run
/// fn to_u32<T: FromBytes + IntoBytes>(v: T) -> u32 {
/// // `zerocopy::transmute!` won't work here.
/// kernel::mem::safe_transmute(v)
/// }
///
/// to_u32(1i32);
/// ```
#[inline(always)]
pub const fn safe_transmute<Src: IntoBytes, Dst: FromBytes>(val: Src) -> Dst {
// SAFETY: `transmute` is safe with `IntoBytes` and `FromBytes` bounds.
unsafe { transmute(val) }
}
/// Type that is layout-compatible with a primitive representation.
///
/// # Safety
///
/// - [`Self`] must have the same size and alignment as [`Self::Repr`].
/// - [`Self`] must be [transmutable] to [`Self::Repr`].
/// - Neither [`Self`] nor [`Self::Repr`] contains interior mutability.
///
/// The above basically says that `&Self` can be transmuted to `&Self::Repr`.
///
/// [transmutable]: core::mem::transmute
pub unsafe trait AsRepr: Sized {
/// Primitive representation of this type.
type Repr;
/// Convert from [`&Self`](Self) to [`&Self::Repr`](AsRepr::Repr).
#[inline(always)]
fn as_repr(this: &Self) -> &Self::Repr {
// SAFETY: Per safety requirement of the trait.
unsafe { core::mem::transmute(this) }
}
/// Convert from [`Self`] to [`Self::Repr`].
#[inline(always)]
fn into_repr(this: Self) -> Self::Repr {
// SAFETY: Per safety requirement of the trait.
unsafe { transmute(this) }
}
/// Convert from [`Self::Repr`] to [`Self`].
///
/// # Safety
///
/// `repr` must be a valid bit pattern of [`Self`] and satisfy type-specific invariants of it.
///
/// Alternatively, if `repr` is previously obtained using [`Self::into_repr`], and each
/// `from_repr_unchecked` should correspond to a unique `into_repr` call, then it is safe to
/// call as well (this means that we're undoing a `into_repr` call getting the exact bytes
/// back).
///
/// No guarantee is made if the result of a `into_repr` is passed to multiple
/// `from_repr_unchecked` (i.e. copies are made), to allow for cases where `Repr` is a pointer
/// and the user of the API wants ownership transfer. Users that want the ability to call
/// `from_repr_unchecked` after copying can require `Copy` bound explicitly.
#[inline(always)]
unsafe fn from_repr_unchecked(repr: Self::Repr) -> Self {
// SAFETY: Per safety requirement, `repr` is valid repr of `Self`, or it is previously from
// `into_repr`, in which case we're undoing the transmute so it is also safe.
unsafe { transmute(repr) }
}
}
/// Type that is bi-directionally transmutable with a primitive representation.
///
/// # Safety
///
/// - [`Self`] must be [transmutable] from [`Self::Repr`].
///
/// [transmutable]: core::mem::transmute
/// [`Self::Repr`]: AsRepr::Repr
pub unsafe trait AsReprMut: AsRepr {
/// Convert from `&mut Self` to [`&mut Self::Repr`](AsRepr::Repr).
#[inline(always)]
fn as_repr_mut(this: &mut Self) -> &mut Self::Repr {
// SAFETY: Per safety requirement of the trait.
unsafe { core::mem::transmute(this) }
}
/// Convert from [`Self::Repr`](AsRepr::Repr) to `Self`.
#[inline(always)]
fn from_repr(repr: Self::Repr) -> Self {
// SAFETY: Per safety requirement of the trait.
unsafe { transmute(repr) }
}
}
// SAFETY: `bool` has the same size and alignment as `u8`, and Rust guarantees that `bool` has
// only two valid bit patterns: 0 (`false`) and 1 (`true`). Thus `bool` can be transmuted to `u8`.
// Neither types contain interior mutability.
unsafe impl AsRepr for bool {
type Repr = u8;
}
// SAFETY: `*mut T` has the same size and alignment with `*const c_void`, and thus `*mut T` is
// transmutable to `*const c_void`. Neither types contain interior mutability.
unsafe impl<T> AsRepr for *mut T {
type Repr = *const c_void;
}
// SAFETY: `*mut T` is transmutable from `*const c_void`.
unsafe impl<T> AsReprMut for *mut T {}
// SAFETY: `*const T` has the same size and alignment with `*const c_void`, and is transmutable to
// `*const c_void`. Neither types contain interior mutability.
unsafe impl<T> AsRepr for *const T {
type Repr = *const c_void;
}
// SAFETY: `*const T` is transmutable from `*const c_void`.
unsafe impl<T> AsReprMut for *const T {}
macro_rules! int_impl {
($($unsigned:ident $signed:ident ,)*) => {$(
// SAFETY: `$unsigned` has the same size and alignment with itself, and is transmutable to
// itself. It does not contain interior mutability.
unsafe impl AsRepr for $unsigned {
type Repr = $unsigned;
}
// SAFETY: `$unsigned` is transmutable from itself.
unsafe impl AsReprMut for $unsigned {}
// SAFETY: `$signed` has the same size and alignment with `$unsigned`, and is transmutable
// to it Neither types contain interior mutability.
unsafe impl AsRepr for $signed {
type Repr = $unsigned;
}
// SAFETY: `$signed` is transmutable from `$unsigned`.
unsafe impl AsReprMut for $signed {}
)*};
}
int_impl! {
u8 i8,
u16 i16,
u32 i32,
u64 i64,
// `usize` is not normalized to particular integer for portability.
usize isize,
}

8
rust/kernel/pci.rs
View File

@ -32,10 +32,18 @@
},
};
mod cap;
mod id;
mod io;
mod irq;
pub use self::cap::{
ExtCapId,
ExtCapability,
ExtSriovCapability,
ExtSriovRegs,
ExtSriovVfBar, //
};
pub use self::id::{
Class,
ClassMask,

292
rust/kernel/pci/cap.rs Normal file
View File

@ -0,0 +1,292 @@
// SPDX-License-Identifier: GPL-2.0
//! PCI extended capability support.
use super::{
io::ConfigSpaceBackend,
ConfigSpace,
Extended, //
};
use crate::{
bindings,
io::{
Io,
IoBackend,
Region, //
},
num::{casts, Bounded},
prelude::*,
};
/// Number of VF BAR register slots in an SR-IOV capability.
const NUM_VF_BARS: usize = casts::u32_as_usize(bindings::PCI_SRIOV_NUM_BARS);
/// PCI extended capability IDs.
#[repr(transparent)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ExtCapId(u16);
impl ExtCapId {
/// Single Root I/O Virtualization.
pub const SRIOV: Self = Self(casts::u32_into_u16::<{ bindings::PCI_EXT_CAP_ID_SRIOV }>());
/// Returns the raw PCIe extended capability ID.
#[inline]
const fn as_raw(self) -> u16 {
self.0
}
}
/// A typed PCI extended capability register layout.
///
/// Implementors describe the register layout of one extended capability. The layout must start at
/// the extended capability header, and [`Self::ID`] must identify that layout.
pub trait ExtCapability: FromBytes + IntoBytes {
/// PCI extended capability ID for this register layout.
const ID: ExtCapId;
}
impl<'a> ConfigSpace<'a, Extended> {
/// Finds and projects an extended capability into its typed register layout.
///
/// Returns [`None`] if the device does not implement the capability.
///
/// Returns an error if the capability is present but its register span is too small or
/// insufficiently aligned for `C`.
///
/// # Examples
///
/// ```no_run
/// use kernel::{
/// device::Bound,
/// pci,
/// prelude::*,
/// };
///
/// fn probe_sriov(pdev: &pci::Device<Bound>) -> Result {
/// let Some(sriov) = pdev
/// .config_space_extended()?
/// .find_ext_capability::<pci::ExtSriovRegs>()?
/// else {
/// return Ok(());
/// };
///
/// let first_vf_offset = sriov.first_vf_offset();
/// let mut vf_bars = sriov.vf_bars()?;
/// let bar0 = vf_bars.next().ok_or(EINVAL)?;
/// let bar1 = vf_bars.next().ok_or(EINVAL)?;
/// let bar2 = vf_bars.next().ok_or(EINVAL)?;
///
/// Ok(())
/// }
/// ```
pub fn find_ext_capability<C: ExtCapability>(&self) -> Result<Option<ConfigSpace<'a, C>>> {
let offset = usize::from(
// SAFETY: `self.pdev` is valid by the type invariant of `ConfigSpace`.
unsafe {
bindings::pci_find_ext_capability(self.pdev.as_raw(), i32::from(C::ID.as_raw()))
},
);
if offset == 0 {
return Ok(None);
}
let size = self.calculate_ext_cap_size(offset)?;
let base = ConfigSpaceBackend::as_ptr(*self)
.cast::<u8>()
.wrapping_add(offset);
let ptr = Region::<0>::ptr_try_from_raw_parts_mut(base, size)?;
// SAFETY: `offset` was returned by `pci_find_ext_capability`, and
// `calculate_ext_cap_size` bounds `ptr` at the next capability or the end of the extended
// configuration space. `ptr_try_from_raw_parts_mut` verified the region layout.
let capability = unsafe { ConfigSpaceBackend::project_view(*self, ptr) };
capability.try_cast::<C>().map(Some)
}
/// Calculates the size of the extended capability at `offset`.
///
/// The capability extends to the next extended capability, or to the end of the extended
/// configuration space if it is the last one. `offset` must be a DWORD-aligned offset within
/// the extended configuration space returned by `pci_find_ext_capability`. Returns an error if
/// the capability header is outside the extended configuration space.
fn calculate_ext_cap_size(&self, offset: usize) -> Result<usize> {
let header = self.try_read32(offset)?;
// SAFETY: Pure bit manipulation, no preconditions.
let next = casts::u32_as_usize(unsafe { bindings::pci_ext_cap_next(header) });
Ok(if next > offset {
next - offset
} else {
self.size() - offset
})
}
}
/// SR-IOV register layout per PCIe spec (64 bytes starting at cap offset).
///
/// The raw registers are private because PCI core owns SR-IOV state management. Drivers should use
/// PCI core APIs for operations such as enabling VFs and querying their topology.
#[repr(C)]
#[derive(FromBytes, IntoBytes)]
pub struct ExtSriovRegs {
/// Extended capability header.
_header: u32,
/// SR-IOV capabilities.
_cap: u32,
/// SR-IOV control.
_ctrl: u16,
/// SR-IOV status.
_status: u16,
/// Initial VFs.
_initial_vfs: u16,
/// Total VFs.
_total_vfs: u16,
/// Number of VFs.
_num_vfs: u16,
/// Function dependency link.
_func_dep_link: u8,
_reserved_0: u8,
/// First VF offset.
vf_offset: u16,
/// VF stride.
_vf_stride: u16,
_reserved_1: u16,
/// VF device ID.
_vf_device_id: u16,
/// Supported page sizes.
_supported_page_sizes: u32,
/// System page size.
_system_page_size: u32,
/// VF BARs (BAR0–BAR5).
vf_bar: [u32; NUM_VF_BARS],
/// VF migration state array offset.
_migration_state: u32,
}
impl ExtCapability for ExtSriovRegs {
const ID: ExtCapId = ExtCapId::SRIOV;
}
/// A typed view of an SR-IOV extended capability.
pub type ExtSriovCapability<'a> = ConfigSpace<'a, ExtSriovRegs>;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum BarMemoryType {
Bits32,
Bits64,
}
impl TryFrom<Bounded<u32, 2>> for BarMemoryType {
type Error = Error;
fn try_from(value: Bounded<u32, 2>) -> Result<Self> {
match value.get() {
0b00 => Ok(Self::Bits32),
0b10 => Ok(Self::Bits64),
_ => Err(EINVAL),
}
}
}
impl From<BarMemoryType> for Bounded<u32, 2> {
fn from(value: BarMemoryType) -> Self {
match value {
BarMemoryType::Bits32 => Self::new::<0b00>(),
BarMemoryType::Bits64 => Self::new::<0b10>(),
}
}
}
// TODO: Move the common BAR encoding definitions into the PCI core abstraction once it provides
// BAR register decoding.
crate::bitfield! {
/// Low DWORD of an SR-IOV VF BAR.
struct BarLow(u32) {
/// Base address bits 31:4.
31:4 address;
/// Whether the address range is prefetchable.
3:3 prefetchable => bool;
/// Memory BAR type.
2:1 memory_type ?=> BarMemoryType;
/// Whether this is an I/O-space BAR.
0:0 io_space => bool;
}
}
/// A decoded VF BAR register encoding.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ExtSriovVfBar {
/// The PCI bus address encoded by the BAR, without PCI attribute bits.
///
/// This is not necessarily the CPU resource address.
pub address: u64,
/// Whether the BAR is 64-bit.
pub is_64bit: bool,
}
impl ExtSriovCapability<'_> {
/// Returns the current PCIe First VF Offset in Routing ID space.
///
/// This value may change when NumVFs changes. Drivers should use PCI core helpers when they
/// need to calculate a VF BDF; this accessor is intended for interfaces that require the First
/// VF Offset itself.
#[inline]
pub fn first_vf_offset(&self) -> u16 {
crate::io_read!(*self, .vf_offset)
}
/// Returns an iterator over decoded VF BAR register encodings.
///
/// All six raw VF BAR register slots are read and decoded up front. A 32-bit encoding yields
/// one entry; a 64-bit encoding combines two slots into one entry.
///
/// A zero-valued low DWORD is yielded as a 32-bit BAR at address zero; this method does not
/// probe whether a BAR is implemented.
///
/// Returns [`EINVAL`] and logs an error if a BAR low DWORD does not encode a 32-bit or 64-bit
/// memory BAR, or if a 64-bit encoding has no upper DWORD.
pub fn vf_bars(&self) -> Result<impl Iterator<Item = ExtSriovVfBar>> {
let slots: [u32; NUM_VF_BARS] =
core::array::from_fn(|slot| crate::io_read!(*self, .vf_bar[panic: slot]));
let mut slots = slots.into_iter();
let mut bars = [None; NUM_VF_BARS];
let mut count = 0;
let mut decode = || {
while let Some(low) = slots.next().map(BarLow::from) {
// SR-IOV VF BARs describe memory-space apertures; an I/O-space encoding is not
// valid for these registers.
if low.io_space() {
return Err(EINVAL);
}
let low_address = u64::from(low.address()) << BarLow::ADDRESS_SHIFT;
let bar = match low.memory_type()? {
BarMemoryType::Bits64 => ExtSriovVfBar {
address: (u64::from(slots.next().ok_or(EINVAL)?) << 32) | low_address,
is_64bit: true,
},
BarMemoryType::Bits32 => ExtSriovVfBar {
address: low_address,
is_64bit: false,
},
};
bars[count] = Some(bar);
count += 1;
}
Ok(())
};
decode().inspect_err(|_| {
dev_err!(self.pdev, "invalid VF BAR encoding in SR-IOV capability\n");
})?;
Ok(bars.into_iter().flatten())
}
}

23
rust/kernel/scatterlist.rs
View File

@ -34,12 +34,12 @@
bindings,
device::{Bound, Device},
devres::Devres,
dma, error,
io::ResourceSize,
dma,
error,
page,
prelude::*,
sync::aref::ARef,
types::Opaque,
types::Opaque, //
};
use core::{ops::Deref, ptr::NonNull};
@ -90,10 +90,9 @@ pub fn dma_address(&self) -> dma::DmaAddress {
/// Returns the length of this SG entry in bytes.
#[inline]
pub fn dma_len(&self) -> ResourceSize {
#[allow(clippy::useless_conversion)]
pub fn dma_len(&self) -> u32 {
// SAFETY: `self.as_raw()` is a valid pointer to a `struct scatterlist`.
unsafe { bindings::sg_dma_len(self.as_raw()) }.into()
unsafe { bindings::sg_dma_len(self.as_raw()) }
}
}
@ -350,15 +349,23 @@ fn new(
page_vec.push(page.as_ptr(), flags)?;
}
// Cap segments at both the DMA mapping-path limit and the device's declared
// max segment size.
//
// `dma_max_mapping_size` returns `size_t`, but `sg_alloc_table_from_pages_segment()` takes
// an `unsigned int`.
//
// SAFETY: `dev.as_raw()` is a valid pointer to a `struct device`.
let max_segment = match unsafe { bindings::dma_max_mapping_size(dev.as_raw()) } {
let max_mapping_size = match unsafe { bindings::dma_max_mapping_size(dev.as_raw()) } {
0 => u32::MAX,
max_segment => u32::try_from(max_segment).unwrap_or(u32::MAX),
max_mapping_size => u32::try_from(max_mapping_size).unwrap_or(u32::MAX),
};
// SAFETY: `dev.as_raw()` is a valid pointer to a `struct device`.
let max_seg_size = unsafe { bindings::dma_get_max_seg_size(dev.as_raw()) };
let max_segment = max_mapping_size.min(max_seg_size);
Ok(try_pin_init!(&this in Self {
// SAFETY:
// - `page_vec` is a `KVec` of valid `struct page *` obtained from `pages`.

4
rust/kernel/sync/atomic.rs
View File

@ -140,7 +140,7 @@ pub unsafe trait AtomicAdd<Rhs = Self>: AtomicType {
const fn into_repr<T: AtomicType>(v: T) -> T::Repr {
// SAFETY: Per the safety requirement of `AtomicType`, `T` is round-trip transmutable to
// `T::Repr`, therefore the transmute operation is sound.
unsafe { core::mem::transmute_copy(&v) }
unsafe { crate::mem::transmute(v) }
}
/// # Safety
@ -149,7 +149,7 @@ const fn into_repr<T: AtomicType>(v: T) -> T::Repr {
#[inline(always)]
const unsafe fn from_repr<T: AtomicType>(r: T::Repr) -> T {
// SAFETY: Per the safety requirement of the function, the transmute operation is sound.
unsafe { core::mem::transmute_copy(&r) }
unsafe { crate::mem::transmute(r) }
}
impl<T: AtomicType> Atomic<T> {

4
rust/kernel/uaccess.rs
View File

@ -520,14 +520,14 @@ pub fn write_slice(&mut self, data: &[u8]) -> Result {
///
/// fn copy_dma_to_user(
/// mut writer: UserSliceWriter,
/// alloc: &Coherent<[u8]>,
/// alloc: &Coherent<'_, [u8]>,
/// ) -> Result {
/// writer.write_dma(alloc, 0, 256)
/// }
/// ```
pub fn write_dma<T: KnownSize + AsBytes + ?Sized>(
&mut self,
alloc: &Coherent<T>,
alloc: &Coherent<'_, T>,
offset: usize,
count: usize,
) -> Result {

3
rust/macros/io/mod.rs Normal file
View File

@ -0,0 +1,3 @@
// SPDX-License-Identifier: GPL-2.0
pub(crate) mod register;

296
rust/macros/io/register.rs Normal file
View File

@ -0,0 +1,296 @@
// SPDX-License-Identifier: GPL-2.0
//! Documentation and usage example of the macro can be found at `rust/kernel/io/register.rs`.
use proc_macro2::{
Group,
Literal,
Span,
TokenStream, //
};
use quote::{
quote,
quote_spanned, //
};
use syn::{
bracketed,
parenthesized,
parse::Parse,
parse_quote,
spanned::Spanned,
token,
Attribute,
Error,
Expr,
Ident,
Path,
Result,
Token,
Type,
Visibility, //
};
mod kw {
syn::custom_keyword!(base);
syn::custom_keyword!(stride);
}
/// Definition of a register array.
///
/// Specify a size, and optionally a stride. Syntax is of form `[EXPR $(, stride = EXPR)?]`.
struct RegArrayDef {
size: Expr,
stride: Option<Expr>,
}
/// Offset of a register.
///
/// Can be either of form
/// * `@ offset` for fixed offset
/// * `=> alias` for alias of register `alias`.
/// * `=> alias[idx]` for alias of register array `alias`'s `idx`-th element.
enum RegOffset {
/// Register is located at fixed address.
Fixed { offset: Literal },
/// Register is an alias of a fixed register.
Alias { alias: Path },
/// Register is an alias of an element of a register array.
ElementAlias { alias: Path, idx: Expr },
}
/// Definition of a single register.
struct Reg {
attrs: Vec<Attribute>,
vis: Visibility,
name: Ident,
unique: bool,
ty: Type,
array: Option<RegArrayDef>,
offset: RegOffset,
bitfield: Option<(Type, Group)>,
}
impl Parse for Reg {
fn parse(input: syn::parse::ParseStream<'_>) -> Result<Self> {
let attrs = input.call(Attribute::parse_outer)?;
let vis = input.parse()?;
let name = input.parse()?;
let lh = input.lookahead1();
let (unique, ty, bitfield_storage) = if lh.peek(Token![:]) {
let _: Token![:] = input.parse()?;
let mut attrs = input.call(Attribute::parse_outer)?;
let unique = attrs
.extract_if(.., |attr| attr.path().is_ident("unique"))
.count()
!= 0;
if !attrs.is_empty() {
Err(Error::new_spanned(&attrs[0], "unexpected attributes"))?
}
(unique, input.parse()?, None)
} else if lh.peek(token::Paren) {
let content;
parenthesized!(content in input);
let bitfield_storage = Some(content.parse()?);
// For bitfields, bitfield macro will generate a type with the same name as `name`.
(true, parse_quote!(#name), bitfield_storage)
} else {
Err(lh.error())?
};
let array = if input.peek(token::Bracket) {
let content;
bracketed!(content in input);
let size = content.parse()?;
let stride = if content.peek(Token![,]) {
let _: Token![,] = content.parse()?;
let _: kw::stride = content.parse()?;
let _: Token![=] = content.parse()?;
Some(content.parse()?)
} else {
None
};
Some(RegArrayDef { size, stride })
} else {
None
};
// Parse offset and the base it's relative to.
let lh = input.lookahead1();
let offset = if lh.peek(Token![@]) {
let _: Token![@] = input.parse()?;
RegOffset::Fixed {
offset: input.parse()?,
}
} else if lh.peek(Token![=>]) {
let _: Token![=>] = input.parse()?;
let alias: Path = input.parse()?;
if input.peek(token::Bracket) {
let content;
bracketed!(content in input);
RegOffset::ElementAlias {
alias,
idx: content.parse()?,
}
} else {
RegOffset::Alias { alias }
}
} else {
Err(lh.error())?
};
let bitfield = if let Some(storage) = bitfield_storage {
let lh = input.lookahead1();
let args = if lh.peek(token::Brace) {
input.parse()?
} else {
Err(lh.error())?
};
Some((storage, args))
} else {
let _: Token![;] = input.parse()?;
None
};
Ok(Self {
attrs,
vis,
name,
unique,
ty,
array,
offset,
bitfield,
})
}
}
pub(crate) struct RegDef {
base: Type,
regs: Vec<Reg>,
}
impl Parse for RegDef {
fn parse(input: syn::parse::ParseStream<'_>) -> Result<Self> {
let _: kw::base = input.parse().map_err(|e| {
Error::new(
e.span(),
"a base type needs to be specified for `register!` invocation with `base: ty;`",
)
})?;
let _: Token![:] = input.parse()?;
let base = input.parse()?;
let _: Token![;] = input.parse()?;
let mut regs = Vec::new();
while !input.is_empty() {
regs.push(input.parse()?);
}
Ok(RegDef { base, regs })
}
}
pub(crate) fn register(def: RegDef) -> Result<TokenStream> {
let mut outputs = TokenStream::new();
let base = &def.base;
for reg in def.regs {
let Reg {
attrs,
vis,
name,
unique,
ty,
array,
offset,
bitfield,
} = reg;
// Use register name's span for generated code, so error messages (if any) can point to it
// instead of the entire register allocation.
let span = name.span().resolved_at(Span::mixed_site());
let offset = match offset {
RegOffset::Fixed { offset } => quote!(#offset),
RegOffset::Alias { alias } => {
quote_spanned!(alias.span().resolved_at(span) =>
::kernel::io::register::OffsetLoc::<#base, _>::const_offset(#alias)
)
}
RegOffset::ElementAlias { alias, idx } => {
quote_spanned!(alias.span().resolved_at(span) =>
::kernel::io::register::element_alias_offset::<#base, #alias>(#idx)
)
}
};
if let Some((storage, args)) = &bitfield {
outputs.extend(quote_spanned!(span =>
::kernel::bitfield!(
// `#[allow(non_camel_case_types)]` is added since register names typically use
// `SCREAMING_CASE`.
#[allow(non_camel_case_types)]
#(#attrs)* #vis struct #name(#storage) #args
);
));
}
match array {
None => {
if unique {
outputs.extend(quote!(
impl ::kernel::io::register::FixedIoLoc<#base> for #ty {
type Location = ::kernel::io::register::OffsetLoc<#base, #ty>;
const LOCATION: Self::Location = #name;
}
))
}
outputs.extend(quote_spanned!(span =>
#(#attrs)* #vis const #name: ::kernel::io::register::OffsetLoc<#base, #ty> =
::kernel::io::register::OffsetLoc::new(#offset);
));
}
Some(def) => {
if bitfield.is_none() {
Err(Error::new_spanned(
&ty,
"defining without bitfield is not yet supported for this type of register",
))?
}
let size = &def.size;
let stride = if let Some(stride) = &def.stride {
outputs.extend(quote_spanned!(stride.span().resolved_at(span) =>
::kernel::build_assert::static_assert!(
::core::mem::size_of::<#ty>() <= #stride
);
));
quote!(#stride)
} else {
quote_spanned!(span => ::core::mem::size_of::<#ty>())
};
outputs.extend(quote_spanned!(span =>
impl ::kernel::io::register::Array for #name {}
impl ::kernel::io::register::RegisterArray for #name {
type Base = #base;
const OFFSET: usize = #offset;
const SIZE: usize = #size;
const STRIDE: usize = #stride;
}
));
}
};
}
Ok(outputs)
}

9
rust/macros/lib.rs
View File

@ -14,6 +14,7 @@
mod fmt;
mod for_lt;
mod helpers;
mod io;
mod kunit;
mod module;
mod paste;
@ -461,6 +462,14 @@ pub fn paste(input: TokenStream) -> TokenStream {
.into()
}
#[doc(hidden)] // Documented in `kernel` crate.
#[proc_macro]
pub fn register(input: TokenStream) -> TokenStream {
io::register::register(parse_macro_input!(input))
.unwrap_or_else(|e| e.into_compile_error())
.into()
}
/// Registers a KUnit test suite and its test cases using a user-space like syntax.
///
/// This macro should be used on modules. If `CONFIG_KUNIT` (in `.config`) is `n`, the target module

30
samples/rust/rust_dma.rs
View File

@ -5,7 +5,10 @@
//! To make this driver probe, QEMU must be run with `-device pci-testdev`.
use kernel::{
device::Core,
device::{
Bound,
Core, //
},
dma::{
Coherent,
DataDirection,
@ -23,14 +26,15 @@
scatterlist::{
Owned,
SGTable, //
},
sync::aref::ARef, //
}, //
};
struct DmaSampleDriver;
#[pin_data(PinnedDrop)]
struct DmaSampleDriver {
pdev: ARef<pci::Device>,
ca: Coherent<[MyStruct]>,
struct DmaSampleData<'bound> {
pdev: &'bound pci::Device<Bound>,
ca: Coherent<'bound, [MyStruct]>,
#[pin]
sgt: SGTable<Owned<VVec<u8>>>,
}
@ -67,13 +71,13 @@ unsafe impl kernel::transmute::FromBytes for MyStruct {}
impl pci::Driver for DmaSampleDriver {
type IdInfo = ();
type Data<'bound> = Self;
type Data<'bound> = DmaSampleData<'bound>;
const ID_TABLE: pci::IdTable<Self::IdInfo> = &PCI_TABLE;
fn probe<'bound>(
pdev: &'bound pci::Device<Core<'_>>,
_info: Option<&'bound Self::IdInfo>,
) -> impl PinInit<Self, Error> + 'bound {
) -> impl PinInit<Self::Data<'bound>, Error> + 'bound {
pin_init::pin_init_scope(move || {
dev_info!(pdev, "Probe DMA test driver.\n");
@ -82,7 +86,7 @@ fn probe<'bound>(
// SAFETY: There are no concurrent calls to DMA allocation and mapping primitives.
unsafe { pdev.dma_set_mask_and_coherent(mask)? };
let ca: Coherent<[MyStruct]> =
let ca: Coherent<'_, [MyStruct]> =
Coherent::zeroed_slice(pdev.as_ref(), TEST_VALUES.len(), GFP_KERNEL)?;
for (i, value) in TEST_VALUES.into_iter().enumerate() {
@ -94,8 +98,8 @@ fn probe<'bound>(
let sgt = SGTable::new(pdev.as_ref(), pages, DataDirection::ToDevice, GFP_KERNEL);
Ok(try_pin_init!(Self {
pdev: pdev.into(),
Ok(try_pin_init!(Self::Data {
pdev,
ca,
sgt <- sgt,
}))
@ -103,7 +107,7 @@ fn probe<'bound>(
}
}
impl DmaSampleDriver {
impl DmaSampleData<'_> {
fn check_dma(&self) {
for (i, value) in TEST_VALUES.into_iter().enumerate() {
let val0 = io_read!(self.ca, [panic: i].h);
@ -116,7 +120,7 @@ fn check_dma(&self) {
}
#[pinned_drop]
impl PinnedDrop for DmaSampleDriver {
impl PinnedDrop for DmaSampleData<'_> {
fn drop(self: Pin<&mut Self>) {
dev_info!(self.pdev, "Unload DMA test driver.\n");

4
samples/rust/rust_driver_pci.rs
View File

@ -23,6 +23,8 @@ mod regs {
use super::*;
register! {
base: kernel::io::Region<END>;
pub(super) TEST(u8) @ 0x0 {
7:0 index => TestIndex;
}
@ -102,6 +104,8 @@ fn config_space(pdev: &pci::Device<Bound>) {
// Some PCI configuration space registers.
register! {
base: pci::Normal;
VENDOR_ID(u16) @ 0x0 {
15:0 vendor_id;
}

8
tools/debugging/kernel-chktaint
View File

@ -211,6 +211,14 @@ else
addout "J"
echo " * fwctl's mutating debug interface was used (#19)"
fi
T=`expr $T / 2`
if [ `expr $T % 2` -eq 0 ]; then
addout " "
else
addout "Y"
echo " * device was manually bound or unbound from a driver (#20)"
fi
echo "Raw taint value as int/string: $taint/'$out'"
# report on any tainted loadable modules