From fea3a2dd7d3fc1936211ced5f84420e610435730 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Wed, 3 Jun 2026 15:42:30 -0400 Subject: [PATCH 001/137] rust: drm: gem: shmem: Fix Default implementation for ObjectConfig I completely forgot when coming up with this type that #[derive(Default)] only works if all generics mentioned in the type implement Default (and T usually doesn't). This being said: We don't use `T` for anything besides using it for a reference type, so whether or not it implements `Default` shouldn't actually need to matter. So, fix this by just manually implementing Default instead of deriving it. Signed-off-by: Lyude Paul Link: https://patch.msgid.link/20260603195210.693856-2-lyude@redhat.com Signed-off-by: Alice Ryhl --- rust/kernel/drm/gem/shmem.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 34af402899a0..084b798ce795 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -42,7 +42,6 @@ /// /// This is used with [`Object::new()`] to control various properties that can only be set when /// initially creating a shmem-backed GEM object. -#[derive(Default)] pub struct ObjectConfig<'a, T: DriverObject, C: DeviceContext = Registered> { /// Whether to set the write-combine map flag. pub map_wc: bool, @@ -53,6 +52,16 @@ pub struct ObjectConfig<'a, T: DriverObject, C: DeviceContext = Registered> { pub parent_resv_obj: Option<&'a Object>, } +impl<'a, T: DriverObject, C: DeviceContext> Default for ObjectConfig<'a, T, C> { + #[inline(always)] + fn default() -> Self { + Self { + map_wc: false, + parent_resv_obj: None, + } + } +} + /// A shmem-backed GEM object. /// /// # Invariants From 56006044df9c65e8e179b236851bcdf4406f2164 Mon Sep 17 00:00:00 2001 From: Zhi Wang Date: Thu, 4 Jun 2026 14:43:32 +0300 Subject: [PATCH 002/137] gpu: nova-core: factor out common FSP message header Extract common MCTP + NVDM headers into FspMessageHeader, rename FspMessage to FspCotMessage, and update FspResponse to use the shared header. This prepares for adding new FSP message types. Signed-off-by: Zhi Wang Link: https://patch.msgid.link/20260604114339.1565660-3-zhiw@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fsp.rs | 56 +++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index 8fc243c66e35..78b90bfbfba4 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -57,12 +57,35 @@ struct NvdmPayloadCommandResponse { error_code: u32, } +/// Common MCTP and NVDM headers shared by all FSP messages. +#[repr(C, packed)] +#[derive(Clone, Copy)] +struct FspMessageHeader { + mctp_header: MctpHeader, + nvdm_header: NvdmHeader, +} + +// SAFETY: FspMessageHeader is a packed C struct with only integral fields. +unsafe impl AsBytes for FspMessageHeader {} + +// SAFETY: FspMessageHeader is a packed C struct with only integral fields. +unsafe impl FromBytes for FspMessageHeader {} + +impl FspMessageHeader { + /// Construct a standard FSP message header for the given NVDM type. + fn new(nvdm_type: NvdmType) -> Self { + Self { + mctp_header: MctpHeader::single_packet(), + nvdm_header: NvdmHeader::new(nvdm_type), + } + } +} + /// Complete FSP response structure with MCTP and NVDM headers. #[repr(C, packed)] #[derive(Clone, Copy)] struct FspResponse { - mctp_header: MctpHeader, - nvdm_header: NvdmHeader, + header: FspMessageHeader, response: NvdmPayloadCommandResponse, } @@ -94,17 +117,16 @@ struct NvdmPayloadCot { gsp_boot_args_sysmem_offset: u64, } -/// Complete FSP message structure with MCTP and NVDM headers. +/// Complete FSP COT (Chain of Trust) message structure. #[repr(C)] #[derive(Clone, Copy)] -struct FspMessage { - mctp_header: MctpHeader, - nvdm_header: NvdmHeader, +struct FspCotMessage { + header: FspMessageHeader, cot: NvdmPayloadCot, } -impl FspMessage { - /// Returns an in-place initializer for [`FspMessage`]. +impl FspCotMessage { + /// Returns an in-place initializer for [`FspCotMessage`]. fn new<'a>( fb_layout: &FbLayout, fsp_fw: &'a FspFirmware, @@ -131,8 +153,7 @@ fn new<'a>( let size = num::usize_into_u16::<{ core::mem::size_of::() }>(); Ok(init!(Self { - mctp_header: MctpHeader::single_packet(), - nvdm_header: NvdmHeader::new(NvdmType::Cot), + header: FspMessageHeader::new(NvdmType::Cot), // The payload is packed, so we cannot use `init!`. Initialize it member-by-member using // `chain`. cot <- pin_init::init_zeroed(), @@ -153,11 +174,11 @@ fn new<'a>( } } -// SAFETY: `FspMessage` is `#[repr(C)]` with no padding, so all of its +// SAFETY: `FspCotMessage` is `#[repr(C)]` with no padding, so all of its // bytes are initialized. -unsafe impl AsBytes for FspMessage {} +unsafe impl AsBytes for FspCotMessage {} -impl MessageToFsp for FspMessage { +impl MessageToFsp for FspCotMessage { const NVDM_TYPE: NvdmType = NvdmType::Cot; } @@ -251,8 +272,8 @@ fn send_sync_fsp(&mut self, dev: &device::Device, bar: Bar0<'_>, msg: &M) -> EIO })?; - let mctp_header = response.mctp_header; - let nvdm_header = response.nvdm_header; + let mctp_header = response.header.mctp_header; + let nvdm_header = response.header.nvdm_header; let command_nvdm_type = response.response.command_nvdm_type; let error_code = response.response.error_code; @@ -310,7 +331,10 @@ pub(crate) fn boot_fmc( ) -> Result { dev_dbg!(dev, "Starting FSP boot sequence for {}\n", args.chipset); - let msg = KBox::init(FspMessage::new(fb_layout, &self.fsp_fw, args)?, GFP_KERNEL)?; + let msg = KBox::init( + FspCotMessage::new(fb_layout, &self.fsp_fw, args)?, + GFP_KERNEL, + )?; self.send_sync_fsp(dev, bar, &*msg)?; From 31522a902f2ae18aa4dcb48346dd61c5779a6f29 Mon Sep 17 00:00:00 2001 From: Zhi Wang Date: Thu, 4 Jun 2026 14:43:33 +0300 Subject: [PATCH 003/137] gpu: nova-core: return FSP response buffer to caller Change send_sync_fsp() to return the raw response buffer after validating the common MCTP/NVDM headers and error code. This allows callers to perform protocol-specific parsing on the response payload, which is needed for the upcoming PRC protocol support. For the existing COT caller, the response buffer is unused. Signed-off-by: Zhi Wang Link: https://patch.msgid.link/20260604114339.1565660-4-zhiw@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fsp.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index 78b90bfbfba4..5fd2e9e277b1 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -257,7 +257,8 @@ pub(crate) fn wait_secure_boot( } /// Sends a message to FSP and waits for the response. - fn send_sync_fsp(&mut self, dev: &device::Device, bar: Bar0<'_>, msg: &M) -> Result + /// Returns the full response buffer on success. + fn send_sync_fsp(&mut self, dev: &device::Device, bar: Bar0<'_>, msg: &M) -> Result> where M: MessageToFsp, { @@ -315,7 +316,7 @@ fn send_sync_fsp(&mut self, dev: &device::Device, bar: Bar0<'_>, msg: &M) -> return Err(EIO); } - Ok(()) + Ok(response_buf) } /// Boots GSP FMC via FSP Chain of Trust. @@ -336,7 +337,7 @@ pub(crate) fn boot_fmc( GFP_KERNEL, )?; - self.send_sync_fsp(dev, bar, &*msg)?; + let _response_buf = self.send_sync_fsp(dev, bar, &*msg)?; dev_dbg!(dev, "FSP Chain of Trust completed successfully\n"); Ok(()) From bfd90545ad9ef8bed2ac7c157fa9db7604befb27 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Sun, 7 Jun 2026 16:05:39 -0700 Subject: [PATCH 004/137] gpu: nova-core: don't declare booter firmware for FSP chipsets The module firmware table lists booter_load and booter_unload for every chipset, but Hopper and Blackwell boot the GSP through FSP and never load the SEC2 booter. Those modinfo entries point at firmware files that are not shipped for FSP-based chipsets, so initramfs tooling looks for images that are never used. Declare the booter only for chipsets that boot via it, matching how the FMC and FWSEC bootloader images are already gated on chipset capabilities. Signed-off-by: John Hubbard Reviewed-by: Danilo Krummrich Reviewed-by: Timur Tabi Link: https://patch.msgid.link/20260607230539.144382-1-jhubbard@nvidia.com [acourbot: make comment on FMC/Booter choice a bit more precise.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/firmware.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index 366d3b76360e..279fbacd0b8e 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -427,19 +427,20 @@ const fn make_entry_chipset(self, chipset: gpu::Chipset) -> Self { let name = chipset.name(); let this = self - .make_entry_file(name, "booter_load") - .make_entry_file(name, "booter_unload") .make_entry_file(name, "bootloader") .make_entry_file(name, "gsp"); - let this = if chipset.needs_fwsec_bootloader() { - this.make_entry_file(name, "gen_bootloader") + // FSP-based chipsets (Hopper, Blackwell and later) boot the GSP via the FMC image loaded by + // FSP. Older chipsets use the SEC2 booter instead. + let this = if chipset.uses_fsp() { + this.make_entry_file(name, "fmc") } else { - this + this.make_entry_file(name, "booter_load") + .make_entry_file(name, "booter_unload") }; - if chipset.uses_fsp() { - this.make_entry_file(name, "fmc") + if chipset.needs_fwsec_bootloader() { + this.make_entry_file(name, "gen_bootloader") } else { this } From 9eaff547805f8556992a9474465001c3e128b7bd Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Sun, 31 May 2026 21:37:27 +0900 Subject: [PATCH 005/137] gpu: nova-core: gsp: tu102: keep unloading if FWSEC-SB fails On Turing and Ampere, resetting the GSP involves running two firmware images: FWSEC-SB and Booter Unloader. They are independent from one another, and we should do whatever is possible to restore the GSP's unloaded state even if a failure occurs along the way. Thus, keep going and run Booter Unloader even if the execution of FWSEC-SB failed. Fixes: adb99ce3cc78 ("gpu: nova-core: run Booter Unloader and FWSEC-SB upon unbinding") Reported-by: Sashiko Closes: https://sashiko.dev/#/patchset/20260529-nova-unload-v7-0-678f39209e00%40nvidia.com?part=3 Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260531-nova-unload-fix-v1-1-c8dcdc769b53@nvidia.com [acourbot: log Booter Unloader errors.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/hal/tu102.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index 2f6301af7113..eb7166148cc9 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -134,11 +134,19 @@ fn run( sec2_falcon: &Falcon, ) -> Result { // Run FWSEC-SB to reset the GSP falcon to its pre-libos state. - self.fwsec_sb.run(dev, bar, gsp_falcon)?; + // Log errors but keep going if it fails. + let fwsec_sb_res = self + .fwsec_sb + .run(dev, bar, gsp_falcon) + .inspect_err(|e| dev_err!(dev, "FWSEC-SB failed to run: {:?}\n", e)); // Remove WPR2 region if set. let wpr2_hi = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI); - if wpr2_hi.is_wpr2_set() { + let booter_unloader_res = (|| { + if !wpr2_hi.is_wpr2_set() { + return Ok(()); + } + sec2_falcon.reset(bar)?; sec2_falcon.load(dev, bar, &self.booter_unloader)?; @@ -160,9 +168,12 @@ fn run( ); return Err(EBUSY); } - } - Ok(()) + Ok(()) + })() + .inspect_err(|e| dev_err!(dev, "Booter Unloader failed to run: {:?}\n", e)); + + fwsec_sb_res.and(booter_unloader_res) } } From 848bf57e98e1678ce7a49eb4e0bf0502da95dc07 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Wed, 3 Jun 2026 16:50:34 -0700 Subject: [PATCH 006/137] gpu: nova-core: clean up FSP FRTS comments Two comments in the FSP Chain of Trust message setup had drifted from the code. One referred to a variable name that no longer exists, and another described the unused sysmem FRTS fields as future work rather than explaining why they are zero. Update both to describe the code as it stands. Signed-off-by: John Hubbard Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260603235034.131354-3-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fsp.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index 5fd2e9e277b1..d949c03dd304 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -132,7 +132,8 @@ fn new<'a>( fsp_fw: &'a FspFirmware, args: &'a FmcBootArgs, ) -> Result + 'a> { - // frts_offset is relative to FB end: FRTS_location = FB_END - frts_offset + // frts_vidmem_offset is measured from the end of FB, so FRTS sits at + // (end of FB) - frts_vidmem_offset. let frts_vidmem_offset = if !args.resume { let frts_reserved_size = fb_layout.heap.len() + u64::from(fb_layout.pmu_reserved_size); @@ -164,8 +165,8 @@ fn new<'a>( msg.cot.gsp_fmc_sysmem_offset = fsp_fw.fmc_image.dma_handle(); msg.cot.frts_vidmem_offset = frts_vidmem_offset; msg.cot.frts_vidmem_size = frts_size; - // frts_sysmem_* intentionally left at zero for now, but will be needed for e.g. - // systems without VRAM. + // frts_sysmem_* are left at zero because this path places FRTS in vidmem. The sysmem + // fields point to an FRTS buffer in sysmem instead, for systems without VRAM. msg.cot.gsp_boot_args_sysmem_offset = args.fmc_boot_params.dma_handle(); msg.cot.sigs = *fsp_fw.fmc_sigs; From 3e4bac7b8ca7688fb3aa9c0bf005a5a4256ad578 Mon Sep 17 00:00:00 2001 From: Antonin Malzieu Ridolfi Date: Sun, 7 Jun 2026 14:10:31 +0000 Subject: [PATCH 007/137] gpu: nova-core: gsp: Move gsp register definition into gsp module Split the gsp register definitions grouped in nova root register file to the gsp module which actually use them. Suggested-by: Alexandre Courbot Suggested-by: Danilo Krummrich Signed-off-by: Antonin Malzieu Ridolfi Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260607140949.152575-1-dev@nanonej.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/gsp.rs | 1 + drivers/gpu/nova-core/gsp/cmdq.rs | 3 ++- drivers/gpu/nova-core/gsp/regs.rs | 11 +++++++++++ drivers/gpu/nova-core/regs.rs | 8 -------- 4 files changed, 14 insertions(+), 9 deletions(-) create mode 100644 drivers/gpu/nova-core/gsp/regs.rs diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index 69175ca3315c..385b4c09582b 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -22,6 +22,7 @@ pub(crate) mod cmdq; pub(crate) mod commands; mod fw; +mod regs; mod sequencer; pub(crate) use fw::{ diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index 0bc5a95a9cd7..495d07d65c39 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -51,10 +51,11 @@ GSP_PAGE_SIZE, // }, num, - regs, sbuffer::SBufferIter, // }; +use super::regs; + /// Marker type representing the absence of a reply for a command. Commands using this as their /// reply type are sent using [`Cmdq::send_command_no_wait`]. pub(crate) struct NoReply; diff --git a/drivers/gpu/nova-core/gsp/regs.rs b/drivers/gpu/nova-core/gsp/regs.rs new file mode 100644 index 000000000000..a76dea3c3ab0 --- /dev/null +++ b/drivers/gpu/nova-core/gsp/regs.rs @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 + +use kernel::io::register; + +// PGSP + +register! { + pub(super) NV_PGSP_QUEUE_HEAD(u32) @ 0x00110c00 { + 31:0 address; + } +} diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index 0f49c1ab83ad..73339a0cff99 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -227,14 +227,6 @@ pub(crate) fn is_wpr2_set(self) -> bool { } } -// PGSP - -register! { - pub(crate) NV_PGSP_QUEUE_HEAD(u32) @ 0x00110c00 { - 31:0 address; - } -} - // PGC6 register space. // // `GC6` is a GPU low-power state where VRAM is in self-refresh and the GPU is powered down (except From e453072df2547d547d04cde60241200f34143af3 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 10 Jun 2026 14:57:14 +0100 Subject: [PATCH 008/137] gpu: nova-core: remove imports available from prelude No functional changes intended. Signed-off-by: Gary Guo Link: https://patch.msgid.link/20260610135716.1013688-1-gary@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/firmware.rs | 4 +--- drivers/gpu/nova-core/firmware/fwsec/bootloader.rs | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index 279fbacd0b8e..15a61edaaa82 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -468,11 +468,9 @@ pub(crate) const fn create( /// that scheme before nova-core becomes stable, which means this module will eventually be /// removed. mod elf { - use core::mem::size_of; - use kernel::{ bindings, - str::CStr, + prelude::*, transmute::FromBytes, // }; diff --git a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs index 039920dc340b..ac1558a83b83 100644 --- a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs +++ b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs @@ -7,7 +7,6 @@ //! be loaded using PIO. use kernel::{ - alloc::KVec, device::{ self, Device, // From 550dc7536644db2d67c6f8cf525bba682fba08d9 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 10 Jun 2026 14:57:15 +0100 Subject: [PATCH 009/137] gpu: nova-core: use `c"literal"` instead of `c_str!()` No functional changes intended. Signed-off-by: Gary Guo Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260610135716.1013688-2-gary@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/nova_core.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 9f0199f7b38c..735b8e17c6b6 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -54,7 +54,7 @@ struct NovaCoreModule { impl InPlaceModule for NovaCoreModule { fn init(module: &'static kernel::ThisModule) -> impl PinInit { - let dir = debugfs::Dir::new(kernel::c_str!("nova-core")); + let dir = debugfs::Dir::new(c"nova-core"); // SAFETY: We are the only driver code running during init, so there // cannot be any concurrent access to `DEBUGFS_ROOT`. From 5a22c80ae8f942d4b560d6a710a037f424e33b9a Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Fri, 12 Jun 2026 15:43:33 -0400 Subject: [PATCH 010/137] rust: drm: gem: shmem: Add DmaResvGuard helper Just a temporary holdover to make locking/unlocking the dma_resv lock much easier. Signed-off-by: Lyude Paul Co-Authored-By: Alexandre Courbot Signed-off-by: Alexandre Courbot Reviewed-by: Alice Ryhl Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260612194436.585385-2-lyude@redhat.com --- rust/kernel/drm/gem/shmem.rs | 39 ++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 084b798ce795..090c5d869fdb 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -22,7 +22,10 @@ error::to_result, prelude::*, sync::aref::ARef, - types::Opaque, // + types::{ + NotThreadSafe, + Opaque, // + }, }; use core::{ marker::PhantomData, @@ -30,7 +33,10 @@ Deref, DerefMut, // }, - ptr::NonNull, // + ptr::{ + self, + NonNull, // + }, }; use gem::{ BaseObjectPrivate, @@ -244,3 +250,32 @@ impl driver::AllocImpl for Object { dumb_map_offset: None, }; } + +/// Private helper-type for holding the `dma_resv` object for a GEM shmem object. +/// +/// When this is dropped, the `dma_resv` lock is dropped as well. +/// +// TODO: This should be replace with a WwMutex equivalent once we have such bindings in the kernel. +struct DmaResvGuard<'a, T: DriverObject, C: DeviceContext = Registered>( + &'a Object, + NotThreadSafe, +); + +impl<'a, T: DriverObject, C: DeviceContext> DmaResvGuard<'a, T, C> { + #[inline] + #[expect(unused)] + fn new(obj: &'a Object) -> Self { + // SAFETY: This lock is initialized throughout the lifetime of `object`. + unsafe { bindings::dma_resv_lock(obj.raw_dma_resv(), ptr::null_mut()) }; + + Self(obj, NotThreadSafe) + } +} + +impl<'a, T: DriverObject, C: DeviceContext> Drop for DmaResvGuard<'a, T, C> { + #[inline] + fn drop(&mut self) { + // SAFETY: We are releasing the lock grabbed during the creation of this object. + unsafe { bindings::dma_resv_unlock(self.0.raw_dma_resv()) }; + } +} From d055768429b3a49090e5f633fa45d7b964fc23ec Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Fri, 12 Jun 2026 15:43:34 -0400 Subject: [PATCH 011/137] rust: drm: gem: shmem: Add vmap functions One of the more obvious use cases for gem shmem objects is the ability to create mappings into their contents. So, let's hook this up in our rust bindings. Signed-off-by: Lyude Paul Reviewed-by: Alexandre Courbot Reviewed-by: Alice Ryhl Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260612194436.585385-3-lyude@redhat.com --- rust/kernel/drm/gem/shmem.rs | 338 ++++++++++++++++++++++++++++++++++- 1 file changed, 337 insertions(+), 1 deletion(-) diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 090c5d869fdb..a38c98add3d1 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -20,6 +20,11 @@ Registered, // }, error::to_result, + io::{ + Io, + IoCapable, + IoKnownSize, // + }, prelude::*, sync::aref::ARef, types::{ @@ -28,7 +33,9 @@ }, }; use core::{ + ffi::c_void, marker::PhantomData, + mem::MaybeUninit, // ops::{ Deref, DerefMut, // @@ -39,6 +46,7 @@ }, }; use gem::{ + BaseObject, BaseObjectPrivate, DriverObject, IntoGEMObject, // @@ -200,6 +208,79 @@ extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { // SAFETY: We're recovering the Kbox<> we created in gem_create_object() let _ = unsafe { KBox::from_raw(this) }; } + + /// Attempt to create a vmap from the gem object, and confirm the size of said vmap. + fn make_vmap<'a, R, const SIZE: usize>(&'a self) -> Result> + where + R: Deref + From<&'a Self>, + { + // INVARIANT: We check here that the gem object is at least as large as `SIZE`. + if self.size() < SIZE { + return Err(ENOSPC); + } + + let mut map: MaybeUninit = MaybeUninit::uninit(); + let guard = DmaResvGuard::new(self); + + // SAFETY: `drm_gem_shmem_vmap()` can be called with the DMA reservation lock held. + to_result(unsafe { + bindings::drm_gem_shmem_vmap_locked(self.as_raw_shmem(), map.as_mut_ptr()) + })?; + + // Drop the guard explicitly here, since we may need to call `raw_vunmap()` (which + // re-acquires the lock). + drop(guard); + + // SAFETY: The call to `drm_gem_shmem_vmap_locked()` succeeded above, so we are guaranteed + // that map is properly initialized. + let map = unsafe { map.assume_init() }; + + // XXX: We don't currently support iomem allocations + if map.is_iomem { + // SAFETY: The vmap operation above succeeded, guaranteeing that `map` points to a valid + // memory mapping. + unsafe { self.raw_vunmap(map) }; + + Err(ENOTSUPP) + } else { + Ok(VMap { + // INVARIANT: `addr` remains valid for as long as `owner` does, which extends to the + // lifetime of `VMap` itself. + // SAFETY: We checked that this is not an iomem allocation, making it safe to read + // vaddr. + addr: unsafe { map.__bindgen_anon_1.vaddr }, + owner: self.into(), + }) + } + } + + /// Unmap a vmap from the gem object. + /// + /// # Safety + /// + /// - The caller promises that `map` is a valid vmap on this gem object. + /// - The caller promises that the memory pointed to by map will no longer be accesed through + /// this instance. + unsafe fn raw_vunmap(&self, mut map: bindings::iosys_map) { + let _guard = DmaResvGuard::new(self); + + // SAFETY: + // - This function is safe to call with the DMA reservation lock held. + // - The caller promises that `map` is a valid vmap on this gem object. + unsafe { bindings::drm_gem_shmem_vunmap_locked(self.as_raw_shmem(), &mut map) }; + } + + /// Creates and returns a virtual kernel memory mapping for this object. + #[inline] + pub fn vmap(&self) -> Result> { + self.make_vmap() + } + + /// Creates and returns an owned reference to a virtual kernel memory mapping for this object. + #[inline] + pub fn owned_vmap(&self) -> Result> { + self.make_vmap() + } } impl Deref for Object { @@ -263,7 +344,6 @@ struct DmaResvGuard<'a, T: DriverObject, C: DeviceContext = Registered>( impl<'a, T: DriverObject, C: DeviceContext> DmaResvGuard<'a, T, C> { #[inline] - #[expect(unused)] fn new(obj: &'a Object) -> Self { // SAFETY: This lock is initialized throughout the lifetime of `object`. unsafe { bindings::dma_resv_lock(obj.raw_dma_resv(), ptr::null_mut()) }; @@ -279,3 +359,259 @@ fn drop(&mut self) { unsafe { bindings::dma_resv_unlock(self.0.raw_dma_resv()) }; } } + +/// A reference to a virtual mapping for an shmem-based GEM object in kernel address space. +/// +/// # Invariants +/// +/// - The size of `owner` is >= SIZE. +/// - The memory pointed to by `addr` remains valid at least until this object is dropped. +pub struct VMap +where + D: DriverObject, + C: DeviceContext, + R: Deref>, +{ + addr: *mut c_void, + owner: R, +} + +/// An alias type for a reference to a shmem-based GEM object's VMap. +pub type VMapRef<'a, D, C, const SIZE: usize = 0> = VMap, C, SIZE>; + +/// An alias type for an owned reference to a shmem-based GEM object's VMap. +pub type VMapOwned = VMap>, C, SIZE>; + +impl VMap +where + D: DriverObject, + C: DeviceContext, + R: Deref>, +{ + /// Borrows a reference to the object that owns this virtual mapping. + #[inline] + pub fn owner(&self) -> &Object { + &self.owner + } +} + +impl Drop for VMap +where + D: DriverObject, + C: DeviceContext, + R: Deref>, +{ + #[inline] + fn drop(&mut self) { + // SAFETY: + // - Our existence is proof that this map was previously created using self.owner. + // - Since we are in Drop, we are guaranteed that no one will access the memory + // through this mapping after calling this. + unsafe { + self.owner.raw_vunmap(bindings::iosys_map { + is_iomem: false, + __bindgen_anon_1: bindings::iosys_map__bindgen_ty_1 { vaddr: self.addr }, + }) + }; + } +} + +// SAFETY: `addr` points to a valid memory address for as long as `owner` exists, meaning that so +// long as `owner` is `Send` so is `VMap`. +unsafe impl Send for VMap +where + D: DriverObject, + C: DeviceContext, + R: Deref> + Send, +{ +} + +// SAFETY: `addr` points to a valid memory address for as long as `owner` exists, meaning that so +// long as `owner` is `Sync` so is `VMap`. +unsafe impl Sync for VMap +where + D: DriverObject, + C: DeviceContext, + R: Deref> + Sync, +{ +} + +impl Io for VMap +where + D: DriverObject, + C: DeviceContext, + R: Deref>, +{ + #[inline] + fn addr(&self) -> usize { + self.addr as usize + } + + #[inline] + fn maxsize(&self) -> usize { + self.owner.size() + } +} + +impl IoKnownSize for VMap +where + D: DriverObject, + C: DeviceContext, + R: Deref>, +{ + const MIN_SIZE: usize = SIZE; +} + +macro_rules! impl_vmap_io_capable { + ($ty:ty) => { + impl IoCapable<$ty> for VMap + where + D: DriverObject, + C: DeviceContext, + R: Deref>, + { + #[inline] + unsafe fn io_read(&self, address: usize) -> $ty { + let ptr = address as *mut $ty; + + // SAFETY: The safety contract of `io_read` guarantees that address is a valid + // address within the bounds of `Self` of at least the size of $ty, and is properly + // aligned. + unsafe { ptr::read_volatile(ptr) } + } + + #[inline] + unsafe fn io_write(&self, value: $ty, address: usize) { + let ptr = address as *mut $ty; + + // SAFETY: The safety contract of `io_write` guarantees that address is a valid + // address within the bounds of `Self` of at least the size of $ty, and is properly + // aligned. + unsafe { ptr::write_volatile(ptr, value) } + } + } + }; +} + +impl_vmap_io_capable!(u8); +impl_vmap_io_capable!(u16); +impl_vmap_io_capable!(u32); +#[cfg(CONFIG_64BIT)] +impl_vmap_io_capable!(u64); + +#[kunit_tests(rust_drm_gem_shmem)] +mod tests { + use super::*; + use crate::{ + drm::{ + self, + UnregisteredDevice, // + }, + faux, + page::PAGE_SIZE, // + }; + + // The bare minimum needed to create a fake drm driver for kunit + + #[pin_data] + struct KunitData {} + struct KunitDriver; + struct KunitFile; + #[pin_data] + struct KunitObject {} + + const INFO: drm::DriverInfo = drm::DriverInfo { + major: 0, + minor: 0, + patchlevel: 0, + name: c"kunit", + desc: c"Kunit", + }; + + impl drm::file::DriverFile for KunitFile { + type Driver = KunitDriver; + + fn open(_dev: &drm::Device) -> Result>> { + Ok(KBox::new(Self, GFP_KERNEL)?.into()) + } + } + + impl gem::DriverObject for KunitObject { + type Driver = KunitDriver; + type Args = (); + + fn new( + _dev: &drm::Device, + _size: usize, + _args: Self::Args, + ) -> impl PinInit { + try_pin_init!(KunitObject {}) + } + } + + #[vtable] + impl drm::Driver for KunitDriver { + type Data = KunitData; + type File = KunitFile; + type Object = Object; + + const INFO: drm::DriverInfo = INFO; + const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor] = &[]; + } + + fn create_drm_dev() -> Result<(faux::Registration, UnregisteredDevice)> { + // Create a faux DRM device so we can test gem object creation. + let data = try_pin_init!(KunitData {}); + let dev = faux::Registration::new(c"Kunit", None)?; + let drm = UnregisteredDevice::new(dev.as_ref(), data)?; + + Ok((dev, drm)) + } + + #[test] + fn compile_time_vmap_sizes() -> Result { + let (_dev, drm) = create_drm_dev()?; + + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + + // Try creating a normal vmap + obj.vmap::()?; + + // Try creating a vmap that's smaller then the size we specified + let vmap = obj.vmap::<{ PAGE_SIZE - 100 }>()?; + + // Verify the owner matches + assert!(ptr::eq(vmap.owner(), obj.deref())); + + // Verify the max size matches the actual object size + assert_eq!(vmap.maxsize(), PAGE_SIZE); + + // Make sure creating a vmap that's too large fails + assert!(obj.vmap::<{ PAGE_SIZE + 200 }>().is_err()); + + Ok(()) + } + + #[test] + fn vmap_io() -> Result { + let (_dev, drm) = create_drm_dev()?; + + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + + let vmap = obj.vmap::()?; + + vmap.write8(0xDE, 0x0); + assert_eq!(vmap.read8(0x0), 0xDE); + vmap.write32(0xFEDCBA98, 0x20); + + assert_eq!(vmap.read32(0x20), 0xFEDCBA98); + + // Ensure the ordering in memory is correct + let expected = 0xFEDCBA98_u32.to_ne_bytes().into_iter(); + for (offset, expected) in (0x20..=0x23).zip(expected) { + assert_eq!(vmap.read8(offset), expected); + } + + Ok(()) + } +} From 20003c1a1fd80ae1b1742ff54297b67f7f21b77f Mon Sep 17 00:00:00 2001 From: Deborah Brouwer Date: Wed, 10 Jun 2026 17:01:26 -0700 Subject: [PATCH 012/137] rust: drm: gpuvm: update DriverGpuVm for DeviceContext Since the introduction of DeviceContext, there is no longer a single driver object type to equate with the GPUVM object type. Instead of threading DeviceContext through GPUVM, remove the strict identity between DriverGpuVm::Object and drm::Driver::Object and instead tighten the requirement that the DriverGpuVm::Object be an allocatable GEM object associated with the same DRM driver. Also, make GpuVm::new() generic over DeviceContext so it can accept a drm::Device. Fixes: 0023a1e8d01a ("rust/drm/gem: Use DeviceContext with GEM objects") Signed-off-by: Deborah Brouwer Reviewed-by: Alice Ryhl Reviewed-by: Sami Tolvanen Link: https://patch.msgid.link/20260610-gpuvm_device_context_v1-v1-1-01a890b17448@collabora.com Signed-off-by: Danilo Krummrich --- rust/kernel/drm/gpuvm/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs index ae58f6f667c1..a625fcd9b5f2 100644 --- a/rust/kernel/drm/gpuvm/mod.rs +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -116,9 +116,9 @@ const fn vtable() -> &'static bindings::drm_gpuvm_ops { /// Creates a GPUVM instance. #[expect(clippy::new_ret_no_self)] - pub fn new( + pub fn new( name: &'static CStr, - dev: &drm::Device, + dev: &drm::Device, r_obj: &T::Object, range: Range, reserve_range: Range, @@ -252,10 +252,10 @@ fn raw_resv(&self) -> *mut bindings::dma_resv { /// The manager for a GPUVM. pub trait DriverGpuVm: Sized + Send { /// Parent `Driver` for this object. - type Driver: drm::Driver; + type Driver: drm::Driver; /// The kind of GEM object stored in this GPUVM. - type Object: IntoGEMObject; + type Object: drm::driver::AllocImpl; /// Data stored with each [`struct drm_gpuva`](struct@GpuVa). type VaData; From 5f7410aa26524101d34b627fbe16670b1514962c Mon Sep 17 00:00:00 2001 From: Boris Brezillon Date: Wed, 10 Jun 2026 17:04:31 -0700 Subject: [PATCH 013/137] rust: drm: gpuvm: add SmContext lifetime bound If a DriverGpuVm implementation is lifetime-parameterized, its SmContext<'ctx> type may depend on lifetimes carried by that driver implementation. In this case, SmContext<'ctx> is only valid if the driver implementation outlives 'ctx. Add a Self: 'ctx bound to DriverGpuVm::SmContext<'ctx> to express that requirement. Then propagate the corresponding T: 'ctx bound to the GPUVM state machine helper types that store T::SmContext<'ctx>. This allows drivers to provide lifetime-parameterized implementations of DriverGpuVm. Signed-off-by: Boris Brezillon Signed-off-by: Deborah Brouwer Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20260610-gpuvm_smcontext_lifetime_bound_v1-v1-1-531e7d2ee7b4@collabora.com Signed-off-by: Danilo Krummrich --- rust/kernel/drm/gpuvm/mod.rs | 4 +++- rust/kernel/drm/gpuvm/sm_ops.rs | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs index a625fcd9b5f2..20a08b3defeb 100644 --- a/rust/kernel/drm/gpuvm/mod.rs +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -264,7 +264,9 @@ pub trait DriverGpuVm: Sized + Send { type VmBoData; /// The private data passed to callbacks. - type SmContext<'ctx>; + type SmContext<'ctx> + where + Self: 'ctx; /// Indicates that a new mapping should be created. fn sm_step_map<'op, 'ctx>( diff --git a/rust/kernel/drm/gpuvm/sm_ops.rs b/rust/kernel/drm/gpuvm/sm_ops.rs index 69a8e5ab2821..742c151b2540 100644 --- a/rust/kernel/drm/gpuvm/sm_ops.rs +++ b/rust/kernel/drm/gpuvm/sm_ops.rs @@ -3,7 +3,7 @@ use super::*; /// The actual data that gets threaded through the callbacks. -struct SmData<'a, 'ctx, T: DriverGpuVm> { +struct SmData<'a, 'ctx, T: DriverGpuVm + 'ctx> { gpuvm: &'a mut UniqueRefGpuVm, user_context: &'a mut T::SmContext<'ctx>, } @@ -20,7 +20,7 @@ struct SmMapData<'a, 'ctx, T: DriverGpuVm> { } /// The argument for [`UniqueRefGpuVm::sm_map`]. -pub struct OpMapRequest<'a, 'ctx, T: DriverGpuVm> { +pub struct OpMapRequest<'a, 'ctx, T: DriverGpuVm + 'ctx> { /// Address in GPU virtual address space. pub addr: u64, /// Length of mapping to create. From 21baef62022fbcca539fca2171e2d3ff15fcb9d3 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Wed, 10 Jun 2026 18:18:59 -0700 Subject: [PATCH 014/137] gpu: nova-core: Blackwell: use absolute FBHUB0 flush registers The GB20x sysmem flush registers were defined relative to an Fbhub0Base register window, but there is exactly one FBHUB0 base, so expressing them as base-plus-offset only adds indirection. Rename these to FBHUB0 and give them their fixed absolute addresses, dropping the base struct and its RegisterBase impl. No functional changes. Signed-off-by: John Hubbard Link: https://patch.msgid.link/20260611011901.84517-2-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fb/hal/gb202.rs | 26 +++++++------------------- drivers/gpu/nova-core/regs.rs | 19 ++++++++----------- 2 files changed, 15 insertions(+), 30 deletions(-) diff --git a/drivers/gpu/nova-core/fb/hal/gb202.rs b/drivers/gpu/nova-core/fb/hal/gb202.rs index 038d1278c634..6747ba6c9c13 100644 --- a/drivers/gpu/nova-core/fb/hal/gb202.rs +++ b/drivers/gpu/nova-core/fb/hal/gb202.rs @@ -4,13 +4,7 @@ //! Blackwell GB20x framebuffer HAL. use kernel::{ - io::{ - register::{ - RegisterBase, - WithBase, // - }, - Io, // - }, + io::Io, num::Bounded, prelude::*, sizes::SizeConstants, // @@ -24,17 +18,13 @@ struct Gb202; -impl RegisterBase for Gb202 { - const BASE: usize = 0x008a_0000; -} - fn read_sysmem_flush_page_gb202(bar: Bar0<'_>) -> u64 { let lo = u64::from( - bar.read(regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::()) + bar.read(regs::NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_LO) .adr(), ); let hi = u64::from( - bar.read(regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::()) + bar.read(regs::NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_HI) .adr(), ); @@ -44,15 +34,13 @@ fn read_sysmem_flush_page_gb202(bar: Bar0<'_>) -> u64 { /// Write the sysmem flush page address through the GB20x FBHUB0 registers. fn write_sysmem_flush_page_gb202(bar: Bar0<'_>, addr: Bounded) { // Write HI first. The hardware will trigger the flush on the LO write. - bar.write( - regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::(), - regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed() + bar.write_reg( + regs::NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed() .with_adr(addr.shr::<32, 20>().cast::()), ); - bar.write( - regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::(), + bar.write_reg( // CAST: lower 32 bits. Hardware ignores bits 7:0. - regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(*addr as u32), + regs::NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(*addr as u32), ); } diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index 73339a0cff99..5ab7ccfb9855 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -153,11 +153,6 @@ fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { /// The base is provided by the GB10x framebuffer HAL. pub(crate) struct Hshub0Base(()); -/// Base of the GB20x FBHUB0 register window (`NV_FBHUB0_PRI_BASE` in Open RM). -/// -/// The base is provided by the GB20x framebuffer HAL. -pub(crate) struct Fbhub0Base(()); - register! { // GB10x sysmem flush registers, relative to the HSHUB0 base. GB10x routes sysmembar // through a primary and an EG (egress) pair that must both be programmed to the same @@ -178,16 +173,18 @@ fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { pub(crate) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x000006c4 { 19:0 adr; } +} - // GB20x sysmem flush registers, relative to the FBHUB0 base. Unlike the older - // NV_PFB_NISO_FLUSH_SYSMEM_ADDR registers which encode the address with an 8-bit - // right-shift, these take the raw address split into lower and upper halves. Hardware - // ignores bits 7:0 of the LO register. - pub(crate) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Fbhub0Base + 0x00001d58 { +register! { + // GB20x FBHUB0 sysmem flush registers. Unlike the older + // NV_PFB_NISO_FLUSH_SYSMEM_ADDR registers, which encode the address with an + // 8-bit right-shift, these take the raw address split into lower and upper + // halves. Hardware ignores bits 7:0 of the LO register. + pub(crate) NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x008a1d58 { 31:0 adr => u32; } - pub(crate) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Fbhub0Base + 0x00001d5c { + pub(crate) NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x008a1d5c { 19:0 adr; } } From 84d58754370fc7bb8ff7af5213bb9aa967a38be2 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Wed, 10 Jun 2026 18:19:00 -0700 Subject: [PATCH 015/137] gpu: nova-core: Hopper: use correct sysmem flush registers Hopper has its own FBHUB sysmem flush page registers, but the Hopper framebuffer HAL delegates to the Ampere NISO path, which encodes the address with an 8-bit right-shift. That programs the wrong value into the wrong registers, so the GPU's sysmembar flush targets the wrong system memory address. Add Hopper's FBHUB flush registers and program them directly from the Hopper HAL. This has not yet been tested on real Hopper hardware (that's true for nova-core in general). Signed-off-by: John Hubbard Link: https://patch.msgid.link/20260611011901.84517-3-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fb/hal/gh100.rs | 31 ++++++++++++++++++++++++--- drivers/gpu/nova-core/regs.rs | 19 ++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/nova-core/fb/hal/gh100.rs b/drivers/gpu/nova-core/fb/hal/gh100.rs index 5450c7254dad..d39fe99537ed 100644 --- a/drivers/gpu/nova-core/fb/hal/gh100.rs +++ b/drivers/gpu/nova-core/fb/hal/gh100.rs @@ -2,24 +2,49 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use kernel::{ + io::Io, + num::Bounded, prelude::*, sizes::SizeConstants, // }; use crate::{ driver::Bar0, - fb::hal::FbHal, // + fb::hal::FbHal, + regs, // }; struct Gh100; +fn read_sysmem_flush_page_gh100(bar: Bar0<'_>) -> u64 { + let lo = u64::from(bar.read(regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO).adr()); + let hi = u64::from(bar.read(regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI).adr()); + + (hi << 32) | lo +} + +/// Write the sysmem flush page address through the Hopper FBHUB registers. +fn write_sysmem_flush_page_gh100(bar: Bar0<'_>, addr: Bounded) { + // Write HI first. The hardware will trigger the flush on the LO write. + bar.write_reg( + regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed() + .with_adr(addr.shr::<32, 20>().cast::()), + ); + bar.write_reg( + // CAST: lower 32 bits. Hardware ignores bits 7:0. + regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(*addr as u32), + ); +} + impl FbHal for Gh100 { fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 { - super::ga100::read_sysmem_flush_page_ga100(bar) + read_sysmem_flush_page_gh100(bar) } fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result { - super::ga100::write_sysmem_flush_page_ga100(bar, addr); + let addr = Bounded::::try_new(addr).ok_or(EINVAL)?; + + write_sysmem_flush_page_gh100(bar, addr); Ok(()) } diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index 5ab7ccfb9855..7982778fd6cb 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -189,6 +189,25 @@ fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { } } +register! { + /// Low bits of the physical system memory address used by the GPU to perform + /// sysmembar operations on Hopper. + /// + /// Like the GB20x FBHUB0 registers, and unlike the Ampere + /// `NV_PFB_NISO_FLUSH_SYSMEM_ADDR` registers (which encode the address with an + /// 8-bit right-shift), these take the raw address split into lower and upper + /// halves. Hardware ignores bits 7:0 of the LO register. + pub(crate) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x00100a34 { + 31:0 adr => u32; + } + + /// High bits of the physical system memory address used by the GPU to perform + /// sysmembar operations on Hopper. + pub(crate) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x00100a38 { + 19:0 adr; + } +} + impl NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE { /// Returns the usable framebuffer size, in bytes. pub(crate) fn usable_fb_size(self) -> u64 { From 746e0e1cc93d15925a4141779c8ee789ff5ea392 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Wed, 10 Jun 2026 18:19:01 -0700 Subject: [PATCH 016/137] gpu: nova-core: fb: fix Blackwell flush address composition The Blackwell sysmem flush read helper composed the 64-bit address as lo | (hi << 32). Write it as (hi << 32) | lo, the order humans expect to read, matching the bit layout and the new Hopper helper. Signed-off-by: John Hubbard Link: https://patch.msgid.link/20260611011901.84517-4-jhubbard@nvidia.com [acourbot: separate unrelated changes into their own patch.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fb/hal/gb202.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/fb/hal/gb202.rs b/drivers/gpu/nova-core/fb/hal/gb202.rs index 6747ba6c9c13..b78e0970f66d 100644 --- a/drivers/gpu/nova-core/fb/hal/gb202.rs +++ b/drivers/gpu/nova-core/fb/hal/gb202.rs @@ -28,7 +28,7 @@ fn read_sysmem_flush_page_gb202(bar: Bar0<'_>) -> u64 { .adr(), ); - lo | (hi << 32) + (hi << 32) | lo } /// Write the sysmem flush page address through the GB20x FBHUB0 registers. From 42c7006927a99f3f940e721dbb269168535258be Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Wed, 10 Jun 2026 18:19:01 -0700 Subject: [PATCH 017/137] gpu: nova-core: fb: remove duplicated SysmemFlush doc link To prevent some copy-paste boilerplate doc comments, remove the second copy of "see [`crate::fb::SysmemFlush`]" from regs.rs. Signed-off-by: John Hubbard Link: https://patch.msgid.link/20260611011901.84517-4-jhubbard@nvidia.com [acourbot: separate unrelated changes into their own patch.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/regs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index 7982778fd6cb..3f16365d3a0e 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -126,7 +126,7 @@ fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { } /// High bits of the physical system memory address used by the GPU to perform sysmembar - /// operations (see [`crate::fb::SysmemFlush`]). + /// operations. pub(crate) NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x00100c40 { 23:0 adr_63_40; } From 9b81ca3c96d217891a612c1d72ac17cb356ad4ac Mon Sep 17 00:00:00 2001 From: Zhi Wang Date: Thu, 4 Jun 2026 14:43:35 +0300 Subject: [PATCH 018/137] gpu: nova-core: add FSP and PRC protocol documentation Add documentation for the Foundation Security Processor (FSP) interface covering the simplified Hopper/Blackwell boot flow, the Chain of Trust (COT) message protocol, the MCTP/NVDM message format, and the Product Reconfiguration Control (PRC) protocol used to query device configuration knobs such as vGPU mode. Signed-off-by: Zhi Wang Link: https://patch.msgid.link/20260604114339.1565660-6-zhiw@nvidia.com Signed-off-by: Alexandre Courbot --- Documentation/gpu/nova/core/fsp.rst | 142 ++++++++++++++++++++++++++++ Documentation/gpu/nova/index.rst | 1 + 2 files changed, 143 insertions(+) create mode 100644 Documentation/gpu/nova/core/fsp.rst diff --git a/Documentation/gpu/nova/core/fsp.rst b/Documentation/gpu/nova/core/fsp.rst new file mode 100644 index 000000000000..52d618d22bb8 --- /dev/null +++ b/Documentation/gpu/nova/core/fsp.rst @@ -0,0 +1,142 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=================================================== +FSP (Foundation Security Processor) and Secure Boot +=================================================== +This document describes the role of the FSP in the GPU boot sequence on +Hopper and Blackwell GPUs, and how it differs from the earlier Ampere boot +flow. It also provides a brief overview of the PRC (Product Reconfiguration +Control) protocol used to query device configuration through FSP. As with +other documents in this directory, the information is subject to change and +is intended to help developers understand the corresponding kernel code. + +What is FSP? +============ +The Foundation Security Processor (FSP) is the GPU's Internal Root of Trust +(IROT). It is a dedicated security processor that boots from immutable ROM +(Boot ROM) inside the GPU and is responsible for establishing the Chain of +Trust before any other firmware is allowed to run. + +FSP runs independently of the host CPU and starts executing as soon as the +GPU is powered on. By the time the nova-core driver is loaded, FSP has +already completed its own secure boot and is ready to accept commands from +the driver. + +Simplified boot flow (Hopper/Blackwell) +======================================= +Starting with Hopper, the boot flow is significantly simplified compared to +earlier GPU generations like Ampere. + +On an **Ampere** GPU, the boot verification chain involves multiple Falcon +engines and multiple ucode stages (see falcon.rst for details):: + + Hardware BROM (SEC2) + -> HS Booter (SEC2) + -> LS GSP-RM (GSP) + +The driver must extract ucode from VBIOS, manage SEC2 and GSP, and +orchestrate the Booter to load GSP-RM. This involves FWSEC-FRTS, devinit, +and the Booter stages. + +On **Hopper/Blackwell** GPUs, FSP replaces this multi-stage process with a +single message-driven interface:: + + FSP (hardware root of trust, boots from ROM) + -> FMC (Falcon Microcontroller, verified by FSP) + -> GSP-RM (verified and loaded by FMC) + +The driver only needs to: + +1. Wait for FSP to complete its own secure boot (polling a scratch register). +2. Send a Chain of Trust (COT) message to FSP with the FMC firmware location, + cryptographic signatures, and GSP boot parameters. +3. FSP authenticates the FMC firmware and boots it, FMC in turn loads GSP-RM. + +There is no SEC2 involvement, no Booter ucode, and no FWSEC-FRTS stage. The +entire secure boot is driven by a single FSP message exchange. + +Chain of Trust (COT) protocol +============================= +The Chain of Trust establishes a cryptographically enforced boot sequence, +ensuring the GPU reaches a known, trusted state. + +The driver communicates with FSP using a message queue (Falcon MSGQ +interface). Each message consists of an MCTP (Management Component Transport +Protocol) transport header and an NVDM (NVIDIA Vendor Defined Message) header, +followed by a protocol-specific payload. + +For Chain of Trust, the payload includes: + +- The system memory address of the FMC firmware image. +- Cryptographic material: a SHA-384 hash, RSA-3K public key, and RSA-3K + signature extracted from the FMC ELF firmware. +- FRTS (Firmware Runtime Services) region information (vidmem offset and size). +- The system memory address of the GSP boot arguments structure. + +FSP verifies the signature against the provided public key and hash, and if +verification succeeds, boots the FMC. The FMC then authenticates and launches +GSP-RM. + +The message flow is:: + + nova-core FSP + | | + | 1. Poll scratch register | + | (wait for FSP boot complete) | + | | + | 2. COT message ------------> | + | (FMC addr, signatures, | + | boot params) | + | | + | |--- Verify FMC signature + | |--- Boot FMC + | |--- FMC loads GSP-RM + | | + | 3. COT response <------------ | + | (success/error) | + | | + +FSP message format +================== +All FSP messages share a common header format consisting of two 32-bit words: + +**MCTP header** (Management Component Transport Protocol): + +- Bit 31: SOM (Start of Message) +- Bit 30: EOM (End of Message) +- Bits 29:28: Packet sequence number +- Bits 23:16: Source Endpoint ID + +**NVDM header** (NVIDIA Vendor Defined Message): + +- Bits 6:0: MCTP message type (0x7e = vendor-defined PCI) +- Bits 23:8: PCI vendor ID (0x10de = NVIDIA) +- Bits 31:24: NVDM type (0x14 = COT, 0x13 = PRC, 0x15 = FSP response) + +PRC (Product Reconfiguration Control) protocol +=============================================== +PRC is an API system exposed through FSP's Management Partition that allows +querying and modifying device configuration without firmware updates. + +Configuration parameters are called "knobs". Each knob has a unique object +ID and controls a specific device behavior. Examples include vGPU mode, ECC +enable, confidential computing mode, and NVLINK configuration. + +Each knob has two values: + +- **Active**: the currently effective value for this boot cycle. +- **Persistent**: the value stored in InfoROM, applied on subsequent boots. + +The nova-core driver uses PRC to read the vGPU mode knob (object ID 0x29) +during early boot, before firmware loading, to determine whether the GPU +should operate in vGPU mode. + +The PRC message format follows the same MCTP/NVDM header structure as COT, +with NVDM type 0x13. The payload contains: + +- A sub-command (e.g., 0x0c for read). +- Flags indicating which value to read (bit 0 = persistent, bit 1 = active). +- The knob object ID. + +The response includes the common FSP response header (with error status) +followed by the knob's 16-bit state value. diff --git a/Documentation/gpu/nova/index.rst b/Documentation/gpu/nova/index.rst index e39cb3163581..1783513cbd05 100644 --- a/Documentation/gpu/nova/index.rst +++ b/Documentation/gpu/nova/index.rst @@ -30,5 +30,6 @@ vGPU manager VFIO driver and the nova-drm driver. core/todo core/vbios core/devinit + core/fsp core/fwsec core/falcon From e655873885063245fd7f49f81cebfdfdef66a59d Mon Sep 17 00:00:00 2001 From: Zhi Wang Date: Thu, 4 Jun 2026 14:43:36 +0300 Subject: [PATCH 019/137] gpu: nova-core: consolidate GSP boot parameters into GspBootContext The GspHal trait methods boot() and post_boot() accept a long list of individual parameters (dev, bar, chipset, gsp_falcon, sec2_falcon) that are threaded through the entire GSP boot call chain. This makes the signatures unwieldy and difficult to extend as new boot-time context (e.g. vGPU state) is introduced. Introduce a GspBootContext struct that bundles the common boot parameters into a single object, and refactor the GspHal trait to accept &GspBootContext instead of individual arguments. The struct also exposes a dev() helper with proper lifetime annotation so that HAL implementations can extract the device reference without reborrowing constraints. Update both TU102 and GH100 HAL implementations to extract their required parameters from the context struct, and simplify the call sites in Gsp::boot() accordingly. Signed-off-by: Zhi Wang Link: https://patch.msgid.link/20260604114339.1565660-7-zhiw@nvidia.com [acourbot: pass `GspBootContext` by value to `Gsp::boot`.] [acourbot: deconstruct `GspBootContext` in `Gsp::boot` to simplify diff.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gpu.rs | 11 +++++++-- drivers/gpu/nova-core/gsp.rs | 22 ++++++++++++++++++ drivers/gpu/nova-core/gsp/boot.rs | 25 ++++++--------------- drivers/gpu/nova-core/gsp/hal.rs | 23 +++++-------------- drivers/gpu/nova-core/gsp/hal/gh100.rs | 14 +++++++----- drivers/gpu/nova-core/gsp/hal/tu102.rs | 31 +++++++++++--------------- 6 files changed, 64 insertions(+), 62 deletions(-) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index b3c91731db45..6a9572107cf3 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -23,7 +23,8 @@ fb::SysmemFlush, gsp::{ self, - Gsp, // + Gsp, + GspBootContext, // }, regs, }; @@ -323,7 +324,13 @@ pub(crate) fn new( // This member must be initialized last, so the `UnloadBundle` can never be dropped from // outside of the constructed `Gpu`, ensuring that the unload sequence is properly run // in case of failure. - unload_bundle: gsp.boot(pdev, bar, spec.chipset, gsp_falcon, sec2_falcon)?, + unload_bundle: gsp.boot(GspBootContext { + pdev, + bar, + chipset: spec.chipset, + gsp_falcon, + sec2_falcon, + })?, bar, }) } diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index 385b4c09582b..3876208779ad 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -32,6 +32,13 @@ }; use crate::{ + driver::Bar0, + falcon::{ + gsp::Gsp as GspFalcon, + sec2::Sec2 as Sec2Falcon, + Falcon, // + }, + gpu::Chipset, gsp::cmdq::Cmdq, gsp::fw::{ GspArgumentsPadded, @@ -43,6 +50,21 @@ pub(crate) const GSP_PAGE_SHIFT: usize = 12; pub(crate) const GSP_PAGE_SIZE: usize = 1 << GSP_PAGE_SHIFT; +/// Common context for the GSP boot process. +pub(crate) struct GspBootContext<'a> { + pub(crate) pdev: &'a pci::Device, + pub(crate) bar: Bar0<'a>, + pub(crate) chipset: Chipset, + pub(crate) gsp_falcon: &'a Falcon, + pub(crate) sec2_falcon: &'a Falcon, +} + +impl<'a> GspBootContext<'a> { + pub(crate) fn dev(&self) -> &'a device::Device { + self.pdev.as_ref() + } +} + /// Number of GSP pages to use in a RM log buffer. const RM_LOG_BUFFER_NUM_PAGES: usize = 0x10; const LOG_BUFFER_SIZE: usize = RM_LOG_BUFFER_NUM_PAGES * GSP_PAGE_SIZE; diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 8afb62d689cb..e380334e937b 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -6,7 +6,6 @@ device, dma::Coherent, io::poll::read_poll_timeout, - pci, prelude::*, time::Delta, types::ScopeGuard, // @@ -24,7 +23,6 @@ gsp::GspFirmware, FIRMWARE_VERSION, // }, - gpu::Chipset, gsp::{ cmdq::Cmdq, commands, @@ -103,12 +101,12 @@ impl super::Gsp { /// [`Self::unload`]) returned. pub(crate) fn boot( self: Pin<&mut Self>, - pdev: &pci::Device, - bar: Bar0<'_>, - chipset: Chipset, - gsp_falcon: &Falcon, - sec2_falcon: &Falcon, + ctx: super::GspBootContext<'_>, ) -> Result> { + let pdev = ctx.pdev; + let bar = ctx.bar; + let chipset = ctx.chipset; + let gsp_falcon = ctx.gsp_falcon; let dev = pdev.as_ref(); let hal = super::hal::gsp_hal(chipset); @@ -120,16 +118,7 @@ pub(crate) fn boot( let wpr_meta = Coherent::init(dev, GFP_KERNEL, GspFwWprMeta::new(&gsp_fw, &fb_layout))?; // Perform the chipset-specific boot sequence, and retrieve the unload bundle. - let unload_guard = hal.boot( - &self, - dev, - bar, - chipset, - &fb_layout, - &wpr_meta, - gsp_falcon, - sec2_falcon, - )?; + let unload_guard = hal.boot(&self, &ctx, &fb_layout, &wpr_meta)?; gsp_falcon.write_os_version(bar, gsp_fw.bootloader.app_version); @@ -148,7 +137,7 @@ pub(crate) fn boot( self.cmdq .send_command_no_wait(bar, commands::SetRegistry::new())?; - hal.post_boot(&self, dev, bar, &gsp_fw, gsp_falcon, sec2_falcon)?; + hal.post_boot(&self, &ctx, &gsp_fw)?; // Wait until GSP is fully initialized. commands::wait_gsp_init_done(&self.cmdq)?; diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs index 04f004856c60..51a277fe97bb 100644 --- a/drivers/gpu/nova-core/gsp/hal.rs +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -4,11 +4,10 @@ mod gh100; mod tu102; -use kernel::prelude::*; - use kernel::{ device, - dma::Coherent, // + dma::Coherent, + prelude::*, // }; use crate::{ @@ -27,6 +26,7 @@ gsp::{ boot::BootUnloadGuard, Gsp, + GspBootContext, GspFwWprMeta, // }, }; @@ -53,32 +53,19 @@ pub(super) trait GspHal: Send { /// /// Upon success, returns a guard that runs the GSP unload sequence if GSP boot does not /// complete. - #[allow(clippy::too_many_arguments)] fn boot<'a>( &self, gsp: &'a Gsp, - dev: &'a device::Device, - bar: Bar0<'a>, - chipset: Chipset, + ctx: &GspBootContext<'a>, fb_layout: &FbLayout, wpr_meta: &Coherent, - gsp_falcon: &'a Falcon, - sec2_falcon: &'a Falcon, ) -> Result>; /// Performs HAL-specific post-GSP boot tasks. /// /// This method is called by the GSP boot code after the GSP is confirmed to be running, and /// after the initialization commands have been pushed onto its queue. - fn post_boot( - &self, - _gsp: &Gsp, - _dev: &device::Device, - _bar: Bar0<'_>, - _gsp_fw: &GspFirmware, - _gsp_falcon: &Falcon, - _sec2_falcon: &Falcon, - ) -> Result { + fn post_boot(&self, _gsp: &Gsp, _ctx: &GspBootContext<'_>, _gsp_fw: &GspFirmware) -> Result { Ok(()) } } diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index 98f5ce197d13..c9fdc8cacedc 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -26,7 +26,6 @@ FmcBootArgs, Fsp, // }, - gpu::Chipset, gsp::{ boot::BootUnloadGuard, hal::{ @@ -34,6 +33,7 @@ UnloadBundle, // }, Gsp, + GspBootContext, GspFwWprMeta, // }, }; @@ -152,14 +152,16 @@ impl GspHal for Gh100 { fn boot<'a>( &self, gsp: &'a Gsp, - dev: &'a device::Device, - bar: Bar0<'a>, - chipset: Chipset, + ctx: &GspBootContext<'a>, fb_layout: &FbLayout, wpr_meta: &Coherent, - gsp_falcon: &'a Falcon, - sec2_falcon: &'a Falcon, ) -> Result> { + let dev = ctx.dev(); + let bar = ctx.bar; + let chipset = ctx.chipset; + let gsp_falcon = ctx.gsp_falcon; + let sec2_falcon = ctx.sec2_falcon; + let fsp_fw = FspFirmware::new(dev, chipset, FIRMWARE_VERSION)?; let unload_bundle = crate::gsp::UnloadBundle( diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index eb7166148cc9..f8a8541704ee 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -42,6 +42,7 @@ GspSequencerParams, // }, Gsp, + GspBootContext, GspFwWprMeta, // }, regs, @@ -269,14 +270,16 @@ impl GspHal for Tu102 { fn boot<'a>( &self, gsp: &'a Gsp, - dev: &'a device::Device, - bar: Bar0<'a>, - chipset: Chipset, + ctx: &GspBootContext<'a>, fb_layout: &FbLayout, wpr_meta: &Coherent, - gsp_falcon: &'a Falcon, - sec2_falcon: &'a Falcon, ) -> Result> { + let dev = ctx.dev(); + let bar = ctx.bar; + let chipset = ctx.chipset; + let gsp_falcon = ctx.gsp_falcon; + let sec2_falcon = ctx.sec2_falcon; + let bios = Vbios::new(dev, bar)?; // Try and prepare the unload bundle. @@ -332,23 +335,15 @@ fn boot<'a>( Ok(unload_guard) } - fn post_boot( - &self, - gsp: &Gsp, - dev: &device::Device, - bar: Bar0<'_>, - gsp_fw: &GspFirmware, - gsp_falcon: &Falcon, - sec2_falcon: &Falcon, - ) -> Result { + fn post_boot(&self, gsp: &Gsp, ctx: &GspBootContext<'_>, gsp_fw: &GspFirmware) -> Result { // Create and run the GSP sequencer. let seq_params = GspSequencerParams { bootloader_app_version: gsp_fw.bootloader.app_version, libos_dma_handle: gsp.libos.dma_handle(), - gsp_falcon, - sec2_falcon, - dev, - bar, + gsp_falcon: ctx.gsp_falcon, + sec2_falcon: ctx.sec2_falcon, + dev: ctx.dev(), + bar: ctx.bar, }; GspSequencer::run(&gsp.cmdq, seq_params)?; From 2418aea12bf6e5bf8138bde50da353bf7e163d50 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 15 Jun 2026 23:40:44 +0900 Subject: [PATCH 020/137] gpu: nova-core: falcon: gsp: move PRIV target mask constants Small cleanup to move these constants which are only used once closer to their use location. Signed-off-by: Eliot Courtney Reviewed-by: Alistair Popple Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260615-blackwell-fixes-v1-4-f2853e49ff7d@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/falcon/gsp.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/nova-core/falcon/gsp.rs b/drivers/gpu/nova-core/falcon/gsp.rs index d1f6f7fcffff..f788b87bd951 100644 --- a/drivers/gpu/nova-core/falcon/gsp.rs +++ b/drivers/gpu/nova-core/falcon/gsp.rs @@ -24,10 +24,6 @@ regs, }; -/// Pattern returned by GSP register reads while the PRIV target mask still blocks CPU access. -const GSP_TARGET_MASK_LOCKED_PATTERN: u32 = 0xbadf_4100; -const GSP_TARGET_MASK_LOCKED_MASK: u32 = 0xffff_ff00; - /// Type specifying the `Gsp` falcon engine. Cannot be instantiated. pub(crate) struct Gsp(()); @@ -70,10 +66,15 @@ pub(crate) fn riscv_branch_privilege_lockdown(&self, bar: Bar0<'_>) -> bool { /// Returns whether GSP registers can be read by the CPU. pub(crate) fn priv_target_mask_released(&self, bar: Bar0<'_>) -> bool { + /// Pattern returned by GSP register reads while the PRIV target mask still blocks CPU + /// access. The low byte varies; the upper 24 bits are fixed. + const LOCKED_PATTERN: u32 = 0xbadf_4100; + const LOCKED_MASK: u32 = 0xffff_ff00; + let hwcfg2 = bar .read(regs::NV_PFALCON_FALCON_HWCFG2::of::()) .into_raw(); - hwcfg2 != 0 && (hwcfg2 & GSP_TARGET_MASK_LOCKED_MASK) != GSP_TARGET_MASK_LOCKED_PATTERN + hwcfg2 != 0 && (hwcfg2 & LOCKED_MASK) != LOCKED_PATTERN } } From 44396428978789e148172d083e90e380d38fb538 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 15 Jun 2026 23:40:46 +0900 Subject: [PATCH 021/137] gpu: nova-core: fsp: move FMC firmware loading into wait_secure_boot `FspFirmware` is constructed and immediately passed into `Fsp`. It makes sense for `Fsp` to ask to load its firmware, so move it there. Signed-off-by: Eliot Courtney Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260615-blackwell-fixes-v1-6-f2853e49ff7d@nvidia.com [acourbot: fix minor merge conflict.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fsp.rs | 11 +++++++---- drivers/gpu/nova-core/gsp/hal/gh100.rs | 8 +------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index d949c03dd304..4b97d1fb505e 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -31,9 +31,12 @@ Falcon, // }, fb::FbLayout, - firmware::fsp::{ - FmcSignatures, - FspFirmware, // + firmware::{ + fsp::{ + FmcSignatures, + FspFirmware, // + }, + FIRMWARE_VERSION, // }, gpu::Chipset, gsp::GspFmcBootParams, @@ -236,13 +239,13 @@ pub(crate) fn wait_secure_boot( dev: &device::Device, bar: Bar0<'_>, chipset: Chipset, - fsp_fw: FspFirmware, ) -> Result { /// FSP secure boot completion timeout in milliseconds. const FSP_SECURE_BOOT_TIMEOUT_MS: i64 = 5000; let hal = hal::fsp_hal(chipset).ok_or(ENOTSUPP)?; let falcon = Falcon::::new(dev, chipset)?; + let fsp_fw = FspFirmware::new(dev, chipset, FIRMWARE_VERSION)?; read_poll_timeout( || Ok(hal.fsp_boot_status(bar)), diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index c9fdc8cacedc..2187e11168b2 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -18,10 +18,6 @@ Falcon, // }, fb::FbLayout, - firmware::{ - fsp::FspFirmware, - FIRMWARE_VERSION, // - }, fsp::{ FmcBootArgs, Fsp, // @@ -162,8 +158,6 @@ fn boot<'a>( let gsp_falcon = ctx.gsp_falcon; let sec2_falcon = ctx.sec2_falcon; - let fsp_fw = FspFirmware::new(dev, chipset, FIRMWARE_VERSION)?; - let unload_bundle = crate::gsp::UnloadBundle( KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox ); @@ -172,7 +166,7 @@ fn boot<'a>( let unload_guard = BootUnloadGuard::new(gsp, dev, bar, gsp_falcon, sec2_falcon, Some(unload_bundle)); - let mut fsp = Fsp::wait_secure_boot(dev, bar, chipset, fsp_fw)?; + let mut fsp = Fsp::wait_secure_boot(dev, bar, chipset)?; let args = FmcBootArgs::new( dev, From b3e079288bba7a0585a4ceac3a50a32dd712e136 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 17 Jun 2026 22:24:43 +0900 Subject: [PATCH 022/137] gpu: nova-core: move GSP unload state to a pinned Gpu subobject `Gpu` currently owns the state needed to unload the GSP directly. This means that `unload_bundle` has to be the last initialized field: once GSP boot succeeds, any later initialization failure would leave `Gpu` partially initialized, and its `PinnedDrop` implementation would not run. This prevents adding fallible `Gpu` fields that need to query the GSP after it has booted. Move the GSP state and unload bundle into a dedicated pinned `GspResources` object. Once that subobject has been initialized, its `PinnedDrop` implementation will run even if initialization of a later `Gpu` field fails, ensuring that the GSP unload sequence is executed. Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260617-boot-vram-v3-1-20b9ec5fe9f2@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gpu.rs | 104 ++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 44 deletions(-) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 6a9572107cf3..acee9a3ab37f 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -263,35 +263,62 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { } } -/// Structure holding the resources required to operate the GPU. +/// Self-contained resources to operate and drop the GSP. #[pin_data(PinnedDrop)] -pub(crate) struct Gpu<'gpu> { +struct GspResources<'gpu> { /// Device owning the GPU. device: &'gpu device::Device, - spec: Spec, /// MMIO mapping of PCI BAR 0. bar: Bar0<'gpu>, - /// System memory page required for flushing all pending GPU-side memory writes done through - /// PCIE into system memory, via sysmembar (A GPU-initiated HW memory-barrier operation). - sysmem_flush: SysmemFlush<'gpu>, /// GSP falcon instance, used for GSP boot up and cleanup. gsp_falcon: Falcon, /// SEC2 falcon instance, used for GSP boot up and cleanup. sec2_falcon: Falcon, - /// GSP runtime data. Temporarily an empty placeholder. + /// GSP runtime data. #[pin] gsp: Gsp, /// GSP unload firmware bundle, if any. unload_bundle: Option, } +/// Structure holding the resources required to operate the GPU. +#[pin_data] +pub(crate) struct Gpu<'gpu> { + spec: Spec, + /// GSP and its resources. + #[pin] + gsp_resources: GspResources<'gpu>, + /// System memory page required for flushing all pending GPU-side memory writes done through + /// PCIE into system memory, via sysmembar (A GPU-initiated HW memory-barrier operation). + /// + /// Must be kept declared *after* `gsp_resources`, as the latter's `PinnedDrop` implementation + /// requires the sysmem flush page to be in place. + sysmem_flush: SysmemFlush<'gpu>, +} + +#[pinned_drop] +impl PinnedDrop for GspResources<'_> { + fn drop(self: Pin<&mut Self>) { + let this = self.project(); + let device = *this.device; + let bar = *this.bar; + let bundle = this.unload_bundle.take(); + + let _ = this + .gsp + .as_ref() + .get_ref() + .unload(device, bar, &*this.gsp_falcon, &*this.sec2_falcon, bundle) + .inspect_err(|e| dev_err!(device, "failed to unload GSP: {:?}\n", e)); + } +} + impl<'gpu> Gpu<'gpu> { pub(crate) fn new( pdev: &'gpu pci::Device>, bar: Bar0<'gpu>, ) -> impl PinInit + 'gpu { try_pin_init!(Self { - device: pdev.as_ref(), spec: Spec::new(pdev.as_ref(), bar).inspect(|spec| { dev_info!(pdev,"NVIDIA ({})\n", spec); })?, @@ -309,46 +336,35 @@ pub(crate) fn new( .inspect_err(|_| dev_err!(pdev, "GFW boot did not complete\n"))?; }, + // Initialize this early because `gsp_resources` depends on it. sysmem_flush: SysmemFlush::register(pdev.as_ref(), bar, spec.chipset)?, - gsp_falcon: Falcon::new( - pdev.as_ref(), - spec.chipset, - ) - .inspect(|falcon| falcon.clear_swgen0_intr(bar))?, + gsp_resources <- try_pin_init!(GspResources { + device: pdev.as_ref(), - sec2_falcon: Falcon::new(pdev.as_ref(), spec.chipset)?, - - gsp <- Gsp::new(pdev), - - // This member must be initialized last, so the `UnloadBundle` can never be dropped from - // outside of the constructed `Gpu`, ensuring that the unload sequence is properly run - // in case of failure. - unload_bundle: gsp.boot(GspBootContext { - pdev, bar, - chipset: spec.chipset, - gsp_falcon, - sec2_falcon, - })?, - bar, + + gsp_falcon: Falcon::new( + pdev.as_ref(), + spec.chipset, + ) + .inspect(|falcon| falcon.clear_swgen0_intr(bar))?, + + sec2_falcon: Falcon::new(pdev.as_ref(), spec.chipset)?, + + gsp <- Gsp::new(pdev), + + // This member must be initialized last, so the `UnloadBundle` can never be dropped + // from outside of the constructed `GspResources`, ensuring that the unload sequence + // is properly run in case of failure. + unload_bundle: gsp.boot(GspBootContext { + pdev, + bar, + chipset: spec.chipset, + gsp_falcon, + sec2_falcon, + })?, + }), }) } } - -#[pinned_drop] -impl PinnedDrop for Gpu<'_> { - fn drop(self: Pin<&mut Self>) { - let this = self.project(); - let device = *this.device; - let bar = *this.bar; - let bundle = this.unload_bundle.take(); - - let _ = this - .gsp - .as_ref() - .get_ref() - .unload(device, bar, &*this.gsp_falcon, &*this.sec2_falcon, bundle) - .inspect_err(|e| dev_err!(device, "failed to unload GSP: {:?}\n", e)); - } -} From f0c1bb8ead8a9790e7d0339354598b50a9c6789a Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 17 Jun 2026 22:24:44 +0900 Subject: [PATCH 023/137] gpu: nova-core: move GPU static information acquisition to a GSP method The GSP static information is useful during regular driver runtime; however it is currently obtained from `Gsp::boot`, with no elegant way to pass it back to the caller. Solve this by moving the code acquiring it to a dedicated method of `Gsp` that can be called as soon as the `Gsp` is booted. This allows us to obtain and display the static information from the `Gpu` constructor, and to store the static information for later use. Its location at the end of `Gsp::boot` was a bit out-of-place anyway: technically, the GSP is considered booted after we have received the `GspInitDone` message, so anything that happens afterwards is not part of the boot sequence anymore. Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260617-boot-vram-v3-2-20b9ec5fe9f2@nvidia.com [acourbot: add documentation to `get_static_info` method.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gpu.rs | 14 ++++++++++++++ drivers/gpu/nova-core/gsp.rs | 15 +++++++++++---- drivers/gpu/nova-core/gsp/boot.rs | 7 ------- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index acee9a3ab37f..a34114d3afcb 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -23,6 +23,7 @@ fb::SysmemFlush, gsp::{ self, + commands::GetGspStaticInfoReply, Gsp, GspBootContext, // }, @@ -285,6 +286,8 @@ struct GspResources<'gpu> { #[pin_data] pub(crate) struct Gpu<'gpu> { spec: Spec, + /// Static GPU information as provided by the GSP. + gsp_static_info: GetGspStaticInfoReply, /// GSP and its resources. #[pin] gsp_resources: GspResources<'gpu>, @@ -365,6 +368,17 @@ pub(crate) fn new( sec2_falcon, })?, }), + + gsp_static_info: { + // Obtain and display basic GPU information. + let info = gsp_resources.gsp.get_static_info(bar)?; + match info.gpu_name() { + Ok(name) => dev_info!(pdev, "GPU name: {}\n", name), + Err(e) => dev_warn!(pdev, "GPU name unavailable: {:?}\n", e), + } + + info + } }) } } diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index 3876208779ad..73e93403601c 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -39,10 +39,12 @@ Falcon, // }, gpu::Chipset, - gsp::cmdq::Cmdq, - gsp::fw::{ - GspArgumentsPadded, - LibosMemoryRegionInitArgument, // + gsp::{ + cmdq::Cmdq, + fw::{ + GspArgumentsPadded, + LibosMemoryRegionInitArgument, // + }, }, num, }; @@ -208,6 +210,11 @@ pub(crate) fn new(pdev: &pci::Device) -> impl PinInit) -> Result { + self.cmdq.send_command(bar, commands::GetGspStaticInfo) + } } /// Opaque bundle required to unload the GSP. Created by [`Gsp::boot`], consumed by [`Gsp::unload`]. diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index e380334e937b..bb2000b7a78b 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -142,13 +142,6 @@ pub(crate) fn boot( // Wait until GSP is fully initialized. commands::wait_gsp_init_done(&self.cmdq)?; - // Obtain and display basic GPU information. - let info = self.cmdq.send_command(bar, commands::GetGspStaticInfo)?; - match info.gpu_name() { - Ok(name) => dev_info!(pdev, "GPU name: {}\n", name), - Err(e) => dev_warn!(pdev, "GPU name unavailable: {:?}\n", e), - } - Ok(unload_guard.dismiss()) } From 917e43d72e164795af907d90a43183bdb392da56 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 17 Jun 2026 22:24:45 +0900 Subject: [PATCH 024/137] gpu: nova-core: gsp: Extract and display usable FB regions from GSP Add usable_fb_regions() to GspStaticConfigInfo to extract the usable FB regions from GSP's fbRegionInfoParams. Usable regions are those that are not reserved or protected. The extracted regions are stored in GetGspStaticInfoReply and exposed for use by the memory subsystem. Display the regions and their total size upon device probe. [acourbot: expose all regions as a KVec, display usable regions and total usable VRAM.] Signed-off-by: Joel Fernandes Reviewed-by: Eliot Courtney Reviewed-by: Danilo Krummrich Link: https://patch.msgid.link/20260617-boot-vram-v3-3-20b9ec5fe9f2@nvidia.com [acourbot: replace dev_info!() with dev_dbg!().] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gpu.rs | 18 ++++++++++- drivers/gpu/nova-core/gsp/commands.rs | 13 ++++++-- drivers/gpu/nova-core/gsp/fw/commands.rs | 40 +++++++++++++++++++++++- 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index a34114d3afcb..4d76be429e75 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -9,7 +9,8 @@ io::Io, num::Bounded, pci, - prelude::*, // + prelude::*, + sizes::SizeConstants, // }; use crate::{ @@ -377,6 +378,21 @@ pub(crate) fn new( Err(e) => dev_warn!(pdev, "GPU name unavailable: {:?}\n", e), } + if !info.usable_fb_regions.is_empty() { + dev_dbg!(pdev, "Usable FB regions:\n"); + for region in &info.usable_fb_regions { + dev_dbg!(pdev, " - {:#x?}\n", region); + } + + dev_dbg!( + pdev, + "Total usable VRAM: {} MiB\n", + info.usable_fb_regions.iter().fold(0u64, |res, region| res + .saturating_add(region.end - region.start)) + / u64::SZ_1M + ); + } + info } }) diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index f84de9f4f045..86a3747cd31c 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -5,6 +5,7 @@ array, convert::Infallible, ffi::FromBytesUntilNulError, + ops::Range, str::Utf8Error, // }; @@ -191,22 +192,30 @@ fn init(&self) -> impl Init { } } -/// The reply from the GSP to the [`GetGspInfo`] command. +/// The reply from the GSP to the [`GetGspStaticInfo`] command. pub(crate) struct GetGspStaticInfoReply { gpu_name: [u8; 64], + /// Usable FB (VRAM) regions for driver memory allocation. + pub(crate) usable_fb_regions: KVec>, } impl MessageFromGsp for GetGspStaticInfoReply { const FUNCTION: MsgFunction = MsgFunction::GetGspStaticInfo; type Message = fw::commands::GspStaticConfigInfo; - type InitError = Infallible; + type InitError = Error; fn read( msg: &Self::Message, _sbuffer: &mut SBufferIter>, ) -> Result { + let mut usable_fb_regions = KVec::new(); + for region in msg.usable_fb_regions() { + usable_fb_regions.push(region, GFP_KERNEL)?; + } + Ok(GetGspStaticInfoReply { gpu_name: msg.gpu_name_str(), + usable_fb_regions, }) } } diff --git a/drivers/gpu/nova-core/gsp/fw/commands.rs b/drivers/gpu/nova-core/gsp/fw/commands.rs index 7bcc41fc7fa0..ebdc12bcd4e3 100644 --- a/drivers/gpu/nova-core/gsp/fw/commands.rs +++ b/drivers/gpu/nova-core/gsp/fw/commands.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +use core::ops::Range; + use kernel::{ device, pci, @@ -13,7 +15,8 @@ use crate::{ gpu::Chipset, - gsp::GSP_PAGE_SIZE, // + gsp::GSP_PAGE_SIZE, + num::IntoSafeCast, // }; use super::bindings; @@ -129,6 +132,41 @@ impl GspStaticConfigInfo { pub(crate) fn gpu_name_str(&self) -> [u8; 64] { self.0.gpuNameString } + + /// Returns an iterator over valid FB regions from GSP firmware data. + fn fb_regions( + &self, + ) -> impl Iterator { + let fb_info = &self.0.fbRegionInfoParams; + fb_info + .fbRegion + .iter() + .take(fb_info.numFBRegions.into_safe_cast()) + .filter(|reg| reg.limit >= reg.base) + } + + /// Iterates over usable FB regions from GSP firmware data. + /// + /// Each yielded region is a [`Range`] suitable for driver memory allocation. + /// Usable regions are those that satisfy all the following properties: + /// - Are not reserved for firmware internal use. + /// - Are not protected (hardware-enforced access restrictions). + /// - Support compression (can use GPU memory compression for bandwidth). + /// - Support ISO (isochronous memory for display requiring guaranteed bandwidth). + pub(crate) fn usable_fb_regions(&self) -> impl Iterator> + '_ { + self.fb_regions().filter_map(|reg| { + // Filter: not reserved, not protected, supports compression and ISO. + if reg.reserved == 0 + && reg.bProtected == 0 + && reg.supportCompressed != 0 + && reg.supportISO != 0 + { + reg.limit.checked_add(1).map(|end| reg.base..end) + } else { + None + } + }) + } } // SAFETY: Padding is explicit and will not contain uninitialized data. From 9102e655ea7285c1bc329669865ba8a38cdb69e6 Mon Sep 17 00:00:00 2001 From: Antonin Malzieu Ridolfi Date: Wed, 17 Jun 2026 01:48:12 +0200 Subject: [PATCH 025/137] gpu: nova-core: fb: Move PDISP register definition Move PDISP register definition into fb module and update register visibility. Signed-off-by: Antonin Malzieu Ridolfi Link: https://patch.msgid.link/20260617-nova-core-regs-split-v1-1-4c7dc4450ea7@nanonej.com [acourbot: fix rustfmt issue.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fb.rs | 4 ++-- drivers/gpu/nova-core/fb/regs.rs | 25 +++++++++++++++++++++++++ drivers/gpu/nova-core/regs.rs | 22 ---------------------- 3 files changed, 27 insertions(+), 24 deletions(-) create mode 100644 drivers/gpu/nova-core/fb/regs.rs diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 725e428154cf..273cff752fae 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -23,11 +23,11 @@ firmware::gsp::GspFirmware, gpu::Chipset, gsp, - num::FromSafeCast, - regs, // + num::FromSafeCast, // }; mod hal; +mod regs; /// Type holding the sysmem flush memory page, a page of memory to be written into the /// `NV_PFB_NISO_FLUSH_SYSMEM_ADDR*` registers and used to maintain memory coherency. diff --git a/drivers/gpu/nova-core/fb/regs.rs b/drivers/gpu/nova-core/fb/regs.rs new file mode 100644 index 000000000000..b2ec02f584be --- /dev/null +++ b/drivers/gpu/nova-core/fb/regs.rs @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 + +use kernel::io::register; + +// PDISP + +register! { + pub(super) NV_PDISP_VGA_WORKSPACE_BASE(u32) @ 0x00625f04 { + /// VGA workspace base address divided by 0x10000. + 31:8 addr; + /// Set if the `addr` field is valid. + 3:3 status_valid => bool; + } +} + +impl NV_PDISP_VGA_WORKSPACE_BASE { + /// Returns the base address of the VGA workspace, or `None` if none exists. + pub(super) fn vga_workspace_addr(self) -> Option { + if self.status_valid() { + Some(u64::from(self.addr()) << 16) + } else { + None + } + } +} diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index 3f16365d3a0e..397124f245ee 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -302,28 +302,6 @@ pub(crate) fn usable_fb_size(self) -> u64 { } } -// PDISP - -register! { - pub(crate) NV_PDISP_VGA_WORKSPACE_BASE(u32) @ 0x00625f04 { - /// VGA workspace base address divided by 0x10000. - 31:8 addr; - /// Set if the `addr` field is valid. - 3:3 status_valid => bool; - } -} - -impl NV_PDISP_VGA_WORKSPACE_BASE { - /// Returns the base address of the VGA workspace, or `None` if none exists. - pub(crate) fn vga_workspace_addr(self) -> Option { - if self.status_valid() { - Some(u64::from(self.addr()) << 16) - } else { - None - } - } -} - // FUSE pub(crate) const NV_FUSE_OPT_FPF_SIZE: usize = 16; From 52aab89ad32dfa50ea6874cb5013e6f75894320f Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 10 Jun 2026 15:13:58 +0100 Subject: [PATCH 026/137] drm/tyr: remove imports available from prelude No functional changes intended. Signed-off-by: Gary Guo Acked-by: Deborah Brouwer Link: https://patch.msgid.link/20260610141359.1033755-1-gary@kernel.org Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/regs.rs | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/tyr/regs.rs b/drivers/gpu/drm/tyr/regs.rs index 562023e5df2f..831357a8ef87 100644 --- a/drivers/gpu/drm/tyr/regs.rs +++ b/drivers/gpu/drm/tyr/regs.rs @@ -48,17 +48,12 @@ pub(crate) fn read_u64_no_tearing(lo_read: impl Fn() -> u32, hi_read: impl Fn() /// These registers correspond to the GPU_CONTROL register page. /// They are involved in GPU configuration and control. pub(crate) mod gpu_control { - use core::convert::TryFrom; use kernel::{ - error::{ - code::EINVAL, - Error, // - }, num::Bounded, + prelude::*, register, uapi, // }; - use pin_init::Zeroable; register! { /// GPU identification register. @@ -964,14 +959,9 @@ pub(crate) mod mmu_control { /// /// This array contains 16 instances of the MMU_AS_CONTROL register page. pub(crate) mod mmu_as_control { - use core::convert::TryFrom; - use kernel::{ - error::{ - code::EINVAL, - Error, // - }, num::Bounded, + prelude::*, register, // }; From 616c229ab010a31c5d1f793b925c7fb2eaad664c Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Fri, 12 Jun 2026 15:43:36 -0400 Subject: [PATCH 027/137] rust: drm: gem: Introduce shmem::Object::sg_table() In order to do this, we need to be careful to ensure that any interface we expose for scatterlists ensures that any mappings created from one are destroyed on driver-unbind. To do this, we introduce a Devres resource into shmem::Object that we use in order to ensure that we release any SGTable mappings on driver-unbind. There's some other slightly unfortunate caveats of this: * Drivers don't have explicit control at the moment over when unmapping happens (which is exactly the same as the C side atm, so it might not be a problem). * We can't just return `SGTableMap` to the user through an Arc to attempt to fix the last caveat - because that implies the gem object would need to hold a reference count to the scatterlist mapping, which just leaves us with the same problem. Signed-off-by: Lyude Paul Reviewed-by: Alexandre Courbot Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260612194436.585385-5-lyude@redhat.com --- rust/kernel/drm/gem/shmem.rs | 174 +++++++++++++++++++++++++++++++++-- 1 file changed, 164 insertions(+), 10 deletions(-) diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index a38c98add3d1..3ee19ef6264e 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -11,6 +11,11 @@ use crate::{ container_of, + device::{ + self, + Bound, // + }, + devres::*, drm::{ driver, gem, @@ -19,14 +24,23 @@ DeviceContext, Registered, // }, - error::to_result, + error::{ + from_err_ptr, + to_result, // + }, io::{ Io, IoCapable, IoKnownSize, // }, prelude::*, - sync::aref::ARef, + scatterlist, + sync::{ + aref::ARef, + new_mutex, + Mutex, + SetOnce, // + }, types::{ NotThreadSafe, Opaque, // @@ -35,7 +49,10 @@ use core::{ ffi::c_void, marker::PhantomData, - mem::MaybeUninit, // + mem::{ + ManuallyDrop, + MaybeUninit, // + }, ops::{ Deref, DerefMut, // @@ -90,6 +107,11 @@ pub struct Object { obj: Opaque, /// Parent object that owns this object's DMA reservation object. parent_resv_obj: Option>>, + /// Devres object for unmapping any SGTable on driver-unbind. + sgt_res: ManuallyDrop>>>, + #[pin] + /// Lock for protecting initialization of `sgt_res`. + sgt_lock: Mutex<()>, #[pin] inner: T, _ctx: PhantomData, @@ -148,6 +170,8 @@ pub fn new( try_pin_init!(Self { obj <- Opaque::init_zeroed(), parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), + sgt_res: ManuallyDrop::new(SetOnce::new()), + sgt_lock <- new_mutex!(()), inner <- T::new(dev, size, args), _ctx: PhantomData::, }), @@ -192,18 +216,26 @@ extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { // - DRM always passes a valid gem object here // - We used drm_gem_shmem_create() in our create_gem_object callback, so we know that // `obj` is contained within a drm_gem_shmem_object - let this = unsafe { container_of!(obj, bindings::drm_gem_shmem_object, base) }; - - // SAFETY: - // - We're in free_callback - so this function is safe to call. - // - We won't be using the gem resources on `this` after this call. - unsafe { bindings::drm_gem_shmem_release(this) }; + let base = unsafe { container_of!(obj, bindings::drm_gem_shmem_object, base) }; // SAFETY: // - We verified above that `obj` is valid, which makes `this` valid // - This function is set in AllocOps, so we know that `this` is contained within a // `Object` - let this = unsafe { container_of!(Opaque::cast_from(this), Self, obj) }.cast_mut(); + let this = unsafe { container_of!(Opaque::cast_from(base), Self, obj) }.cast_mut(); + + // We need to drop `sgt_res` first, since doing so requires that the GEM object is still + // alive. + // SAFETY: + // - We verified above that `this` is valid. + // - We are in free_callback, guaranteeing we have exclusive access to `this` and that + // `sgt_res` will not be used after dropping it here. + unsafe { ManuallyDrop::drop(&mut (*this).sgt_res) }; + + // SAFETY: + // - We're in free_callback - so this function is safe to call. + // - We won't be using the gem resources on `this` after this call. + unsafe { bindings::drm_gem_shmem_release(base) }; // SAFETY: We're recovering the Kbox<> we created in gem_create_object() let _ = unsafe { KBox::from_raw(this) }; @@ -281,6 +313,46 @@ pub fn vmap(&self) -> Result> { pub fn owned_vmap(&self) -> Result> { self.make_vmap() } + + /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA + /// pages for this object. + /// + /// This will pin the object in memory. It is expected that `dev` should be a pointer to the + /// same [`device::Device`] which `self` belongs to, otherwise this function will return + /// `Err(EINVAL)`. + pub fn sg_table<'a>( + &'a self, + dev: &'a device::Device, + ) -> Result<&'a scatterlist::SGTable> { + if dev.as_raw() != self.dev().as_ref().as_raw() { + return Err(EINVAL); + } + + let sgt_res = 'out: { + // Fast path: sgt_res is already initialized + if let Some(sgt_res) = self.sgt_res.as_ref() { + break 'out sgt_res; + } + + // Slow path: Grab the lock and see if we need to initialize sgt_res. + let _guard = self.sgt_lock.lock(); + + // If someone initialized it while we were waiting, we can exit early. + if let Some(sgt_res) = self.sgt_res.as_ref() { + break 'out sgt_res; + } + + // If not, finish initializing and return. `populate()` cannot return false, as + // `sgt_res` must be unpopulated, and we must hold `sgt_lock` to reach this point. + self.sgt_res + .populate(Devres::new(dev, SGTableMap::new(self))?); + + // SAFETY: We just populated sgt_res above. + unsafe { self.sgt_res.as_ref().unwrap_unchecked() } + }; + + Ok(sgt_res.access(dev)?) + } } impl Deref for Object { @@ -499,6 +571,64 @@ unsafe fn io_write(&self, value: $ty, address: usize) { #[cfg(CONFIG_64BIT)] impl_vmap_io_capable!(u64); +/// A reference to a GEM object that is known to have a mapped [`SGTable`]. +/// +/// This is used by the Rust bindings with [`Devres`] in order to ensure that mappings for SGTables +/// on GEM shmem objects are revoked on driver-unbind. +/// +/// # Invariants +/// +/// - `self.obj` always points to a valid GEM object. +/// - This object is proof that `self.obj.owner.sgt_res` has an initialized and valid pointer to an +/// [`SGTable`]. +/// +/// [`SGTable`]: scatterlist::SGTable +pub struct SGTableMap { + obj: NonNull>, +} + +impl Deref for SGTableMap { + type Target = scatterlist::SGTable; + + fn deref(&self) -> &Self::Target { + // SAFETY: + // - The NonNull is guaranteed to be valid via our type invariants. + // - The sgt field is guaranteed to be initialized and valid via our type invariants. + unsafe { scatterlist::SGTable::from_raw((*self.obj.as_ref().as_raw_shmem()).sgt) } + } +} + +impl Drop for SGTableMap { + fn drop(&mut self) { + // SAFETY: `obj` is always valid via our type invariants + let obj = unsafe { self.obj.as_ref() }; + let _lock = DmaResvGuard::new(obj); + + // SAFETY: We acquired the lock needed for calling this function above + unsafe { bindings::__drm_gem_shmem_free_sgt_locked(obj.as_raw_shmem()) }; + } +} + +impl SGTableMap { + fn new(obj: &Object) -> impl Init { + // INVARIANT: + // - We call drm_gem_shmem_get_pages_sgt below and check whether or not it succeeds, + // fulfilling the invariant of SGTableMap that the object's `sgt` field is initialized. + // SAFETY: + // - `obj` is fully initialized, making this function safe to call. + from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(obj.as_raw_shmem()) })?; + + Ok(Self { obj: obj.into() }) + } +} + +// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object +// it points to is guaranteed to be thread-safe. +unsafe impl Send for SGTableMap {} +// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object +// it points to is guaranteed to be thread-safe. +unsafe impl Sync for SGTableMap {} + #[kunit_tests(rust_drm_gem_shmem)] mod tests { use super::*; @@ -614,4 +744,28 @@ fn vmap_io() -> Result { Ok(()) } + + // TODO: I would love to actually test the success paths of sg_table(), but that would require + // also implementing dummy dma_ops so that trying to create a mapping doesn't explode. So, leave + // that for someone else. + + // Ensures that passing the wrong device to sg_table() fails as we expect, and also ensure it + // skips initializing `sgt_res` since we could otherwise create `sgt_res` with the wrong device + // bound to it. + #[test] + fn fail_sg_table_on_wrong_dev() -> Result { + let (_dev, drm) = create_drm_dev()?; + let wrong_dev = faux::Registration::new(c"EvilKunit", None)?; + + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + + assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); + + // If sgt_res was not initialized mistakenly with the wrong device, this should still fail. + assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); + + // TODO: Someday, we should test that creating an sg_table here still succeeds. + + Ok(()) + } } From fa8cc4e3067f958ea2057f37a8a6f9c6b10a9c03 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Fri, 12 Jun 2026 15:43:35 -0400 Subject: [PATCH 028/137] rust: faux: Allow retrieving a bound Device When writing up some rust code that used faux devices for unit testing, I noticed that we never actually added the Bound device context to faux::Registration's AsRef implementation. This being said: the Registration object itself is proof that a driver is bound to the device - so this should be safe. Signed-off-by: Lyude Paul Reviewed-by: Alexandre Courbot Reviewed-by: Alice Ryhl Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260612194436.585385-4-lyude@redhat.com Signed-off-by: Danilo Krummrich --- rust/kernel/faux.rs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/rust/kernel/faux.rs b/rust/kernel/faux.rs index 43b4974f48cd..36c92ae2943c 100644 --- a/rust/kernel/faux.rs +++ b/rust/kernel/faux.rs @@ -25,7 +25,8 @@ /// /// # Invariants /// -/// `self.0` always holds a valid pointer to an initialized and registered [`struct faux_device`]. +/// - `self.0` always holds a valid pointer to an initialized and registered [`struct faux_device`]. +/// - This object is proof that the object described by this `Registration` is bound to a device. /// /// [`struct faux_device`]: srctree/include/linux/device/faux.h pub struct Registration(NonNull); @@ -59,10 +60,17 @@ fn as_raw(&self) -> *mut bindings::faux_device { } } -impl AsRef for Registration { - fn as_ref(&self) -> &device::Device { - // SAFETY: The underlying `device` in `faux_device` is guaranteed by the C API to be - // a valid initialized `device`. +impl AsRef> for Registration { + fn as_ref(&self) -> &device::Device { + // SAFETY: + // - The underlying `device` in `faux_device` is guaranteed by the C API to be a valid + // initialized `device`. + // - `faux_match()` always returns 1, and probe runs synchronously + // (PROBE_FORCE_SYNCHRONOUS). + // - `suppress_bind_attrs = true` on faux_driver prevents userspace-triggered unbind via + // sysfs. + // - `mem::forget(Registration)` is not a problem; if the `Registration` is leaked, the faux + // device stays bound forever. unsafe { device::Device::from_raw(addr_of_mut!((*self.as_raw()).dev)) } } } From ca524e273c43c990756cac471a4cb48d219480dd Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Mon, 22 Jun 2026 22:30:07 +0900 Subject: [PATCH 029/137] gpu: build nova-core and nova-drm from drivers/gpu/Makefile A dependency between nova-core and nova-drm is about to be introduced, which requires nova-core to be built first. As this is not easily doable from separate directories, move both build targets to the first common parent, `drivers/gpu/Makefile`. Suggested-by: Miguel Ojeda Signed-off-by: Alexandre Courbot Link: https://patch.msgid.link/20260622-nova-exports-v5-1-6191773fc977@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/Makefile | 12 +++++++++++- drivers/gpu/drm/Makefile | 2 +- drivers/gpu/drm/nova/Makefile | 4 +--- drivers/gpu/nova-core/Makefile | 4 +--- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/Makefile b/drivers/gpu/Makefile index b4e5e338efa2..45e0941324fb 100644 --- a/drivers/gpu/Makefile +++ b/drivers/gpu/Makefile @@ -7,4 +7,14 @@ obj-$(CONFIG_GPU_BUDDY) += buddy.o obj-y += host1x/ drm/ vga/ tests/ obj-$(CONFIG_IMX_IPUV3_CORE) += ipu-v3/ obj-$(CONFIG_TRACE_GPU_MEM) += trace/ -obj-$(CONFIG_NOVA_CORE) += nova-core/ + +# nova-core and nova-drm are built from this Makefile so nova-drm's dependency +# on nova-core can be expressed as a plain Make prerequisite rather than a +# recursive sub-make. This is a temporary workaround until the Rust build +# system supports cross-crate dependencies natively. + +obj-$(CONFIG_NOVA_CORE) += nova-core.o +nova-core-y := nova-core/nova_core.o + +obj-$(CONFIG_DRM_NOVA) += nova-drm.o +nova-drm-y := drm/nova/nova.o diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index e97faabcd783..e635fcffd379 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -186,7 +186,7 @@ obj-$(CONFIG_DRM_VMWGFX)+= vmwgfx/ obj-$(CONFIG_DRM_VGEM) += vgem/ obj-$(CONFIG_DRM_VKMS) += vkms/ obj-$(CONFIG_DRM_NOUVEAU) +=nouveau/ -obj-$(CONFIG_DRM_NOVA) += nova/ +# nova-drm is built from drivers/gpu/Makefile together with nova-core. obj-$(CONFIG_DRM_EXYNOS) +=exynos/ obj-$(CONFIG_DRM_ROCKCHIP) +=rockchip/ obj-$(CONFIG_DRM_GMA500) += gma500/ diff --git a/drivers/gpu/drm/nova/Makefile b/drivers/gpu/drm/nova/Makefile index f8527b2b7b4a..b9fad3956358 100644 --- a/drivers/gpu/drm/nova/Makefile +++ b/drivers/gpu/drm/nova/Makefile @@ -1,4 +1,2 @@ # SPDX-License-Identifier: GPL-2.0 - -obj-$(CONFIG_DRM_NOVA) += nova-drm.o -nova-drm-y := nova.o +# nova-drm is built from drivers/gpu/Makefile. diff --git a/drivers/gpu/nova-core/Makefile b/drivers/gpu/nova-core/Makefile index 4ae544f808f4..4c15729704a1 100644 --- a/drivers/gpu/nova-core/Makefile +++ b/drivers/gpu/nova-core/Makefile @@ -1,4 +1,2 @@ # SPDX-License-Identifier: GPL-2.0 - -obj-$(CONFIG_NOVA_CORE) += nova-core.o -nova-core-y := nova_core.o +# nova-core is built from drivers/gpu/Makefile. From 3b7b7ad78fd2adae8d9a016677b6dbbb9c9632a2 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Mon, 22 Jun 2026 22:30:08 +0900 Subject: [PATCH 030/137] gpu: nova-core: export Rust symbols for nova-drm Export nova-core's Rust symbols so nova-drm can resolve references to it when loaded as a module. This is done by generating declarations and EXPORT_SYMBOL_RUST_GPL() calls for Rust symbols referenced by nova-drm, and compiling them into the module as `nova_core_exports.o`. `nova_core_exports.o` declares every Rust symbol as `extern int`. Running `gendwarfksyms` on it would compute CRCs from those placeholder types instead of the real Rust ones, so make MODVERSIONS use this shim only for the export list, and derive CRCs from `nova_core.o`. This patch is intended to be a workaround until the build system supports Rust cross-crate dependencies natively. Signed-off-by: Alexandre Courbot Link: https://patch.msgid.link/20260622-nova-exports-v5-2-6191773fc977@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/Makefile | 40 ++++++++++++++++++++++- drivers/gpu/nova-core/.gitignore | 1 + drivers/gpu/nova-core/nova_core_exports.c | 15 +++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/nova-core/.gitignore create mode 100644 drivers/gpu/nova-core/nova_core_exports.c diff --git a/drivers/gpu/Makefile b/drivers/gpu/Makefile index 45e0941324fb..67d51b7f3f55 100644 --- a/drivers/gpu/Makefile +++ b/drivers/gpu/Makefile @@ -14,7 +14,45 @@ obj-$(CONFIG_TRACE_GPU_MEM) += trace/ # system supports cross-crate dependencies natively. obj-$(CONFIG_NOVA_CORE) += nova-core.o -nova-core-y := nova-core/nova_core.o +nova-core-y := nova-core/nova_core.o nova-core/nova_core_exports.o obj-$(CONFIG_DRM_NOVA) += nova-drm.o nova-drm-y := drm/nova/nova.o + +# Export Rust symbols from nova-core only if nova-drm actually references them. +nova-core-export-deps := $(if $(CONFIG_DRM_NOVA),$(obj)/drm/nova/nova.o) + +rust_needed_exports = \ + { $(if $(strip $(2)),$(NM) -u $(2);,) echo "__DEFINED_RUST_SYMBOLS__"; \ + $(NM) -p --defined-only $(1); } | \ + awk -v fmt='$(3)' ' \ + /^__DEFINED_RUST_SYMBOLS__$$/ { defs = 1; next } \ + !defs { if ($$NF ~ /^_R/) needed[$$NF] = 1; next } \ + defs && $$2 ~ /(T|R|D|B)/ && $$3 ~ /^_R/ && \ + $$3 !~ /_(init|cleanup)_module$$/ && \ + $$3 !~ /__(pfx|cfi|odr_asan)/ && \ + $$3 in needed { printf fmt, $$3 } \ + ' + +quiet_cmd_exports = EXPORTS $@ + cmd_exports = \ + $(call rust_needed_exports,$<,$(nova-core-export-deps),EXPORT_SYMBOL_RUST_GPL(%s);\n) > $@ + +$(obj)/nova-core/exports_nova_core_generated.h: $(obj)/nova-core/nova_core.o $(nova-core-export-deps) FORCE + $(call if_changed,exports) + +targets += nova-core/exports_nova_core_generated.h + +$(obj)/nova-core/nova_core_exports.o: $(obj)/nova-core/exports_nova_core_generated.h +CFLAGS_nova-core/nova_core_exports.o := -I $(objtree)/$(obj)/nova-core + +ifdef CONFIG_MODVERSIONS +# The C export shim declares Rust symbols as `extern int`, so reuse its export +# list but generate symbol CRCs from the Rust object instead of the shim's DWARF. +$(obj)/nova-core/nova_core_exports.o: private cmd_gensymtypes_c = \ + $(call getexportsymbols,\1) | \ + $(objtree)/scripts/gendwarfksyms/gendwarfksyms \ + $(if $(KBUILD_GENDWARFKSYMS_STABLE), --stable) \ + $(if $(KBUILD_SYMTYPES), --symtypes $(@:.o=.symtypes),) \ + $(obj)/nova-core/nova_core.o +endif diff --git a/drivers/gpu/nova-core/.gitignore b/drivers/gpu/nova-core/.gitignore new file mode 100644 index 000000000000..7cc8318c76b1 --- /dev/null +++ b/drivers/gpu/nova-core/.gitignore @@ -0,0 +1 @@ +exports_nova_core_generated.h diff --git a/drivers/gpu/nova-core/nova_core_exports.c b/drivers/gpu/nova-core/nova_core_exports.c new file mode 100644 index 000000000000..6e80ca9792ee --- /dev/null +++ b/drivers/gpu/nova-core/nova_core_exports.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +/* + * Exports Rust symbols from the `nova_core` crate for use by dependent modules. + * + * This is a workaround until the build system supports Rust cross-module + * dependencies natively. + */ + +#include + +#define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym) + +#include "exports_nova_core_generated.h" From 0dc79ddc9f6f5dde8c3a78395f5f10c1cf82b2df Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Mon, 22 Jun 2026 22:30:09 +0900 Subject: [PATCH 031/137] gpu: nova-core: emit Rust metadata for nova-drm Emit nova-core's crate metadata (libnova_core.rmeta) so that nova-drm can import nova-core's types and functions at compile time. This is intended to be a workaround until the build system supports Rust cross-crate dependencies natively. Signed-off-by: Alexandre Courbot Link: https://patch.msgid.link/20260622-nova-exports-v5-3-6191773fc977@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/Makefile b/drivers/gpu/Makefile index 67d51b7f3f55..13c96aa57033 100644 --- a/drivers/gpu/Makefile +++ b/drivers/gpu/Makefile @@ -56,3 +56,7 @@ $(obj)/nova-core/nova_core_exports.o: private cmd_gensymtypes_c = \ $(if $(KBUILD_SYMTYPES), --symtypes $(@:.o=.symtypes),) \ $(obj)/nova-core/nova_core.o endif + +# Output nova-core's crate metadata for use by nova-drm at compile time. +RUSTFLAGS_nova-core/nova_core.o += \ + --emit=metadata=$(objtree)/$(obj)/nova-core/libnova_core.rmeta From f1bd7119ac4c98fc2f0ddf5a6d851de66bc5f62f Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Mon, 22 Jun 2026 22:30:10 +0900 Subject: [PATCH 032/137] gpu: drm: nova: depend on nova-core and use its symbols Make nova-core a build dependency of nova-drm, so its crate metadata is available and up-to-date when the latter is built. This is intended to be a workaround until the build system supports Rust cross-crate dependencies natively. Signed-off-by: Alexandre Courbot Link: https://patch.msgid.link/20260622-nova-exports-v5-4-6191773fc977@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/Makefile b/drivers/gpu/Makefile index 13c96aa57033..e372fc02139f 100644 --- a/drivers/gpu/Makefile +++ b/drivers/gpu/Makefile @@ -60,3 +60,7 @@ endif # Output nova-core's crate metadata for use by nova-drm at compile time. RUSTFLAGS_nova-core/nova_core.o += \ --emit=metadata=$(objtree)/$(obj)/nova-core/libnova_core.rmeta + +# Allow nova-drm to import nova-core's types. +$(obj)/drm/nova/nova.o: $(obj)/nova-core/nova_core.o +RUSTFLAGS_drm/nova/nova.o := -L $(objtree)/$(obj)/nova-core --extern nova_core From 23d66dbab84e8518943563df2ced14aaab28b77a Mon Sep 17 00:00:00 2001 From: Tim Kovalenko Date: Fri, 26 Jun 2026 02:24:47 +0000 Subject: [PATCH 033/137] gpu: nova-core: falcon: store bar and dev in falcon Store the bound device and `BAR0` mapping in `Falcon` instead of passing them through every `Falcon` operation. This simplifies the `Falcon` API and removes repeated `dev`/`bar` plumbing from reset, load, boot, mailbox, DMA, and GSP/FSP-specific Falcon helpers. `FalconHal` now receives a reference to a `Falcon` and uses its methods and members instead of passing them individually. Suggested-by: Alexandre Courbot Link: https://rust-for-linux.zulipchat.com/#narrow/channel/509436-Nova/topic/Storing.20driver-bound.20references.20into.20sub-devices/near/599137882 Signed-off-by: Tim Kovalenko Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260625-drm-bar-refactor-v2-1-9db6b890d92e@proton.me Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/falcon.rs | 195 ++++++++---------- drivers/gpu/nova-core/falcon/fsp.rs | 44 ++-- drivers/gpu/nova-core/falcon/gsp.rs | 21 +- drivers/gpu/nova-core/falcon/hal.rs | 14 +- drivers/gpu/nova-core/falcon/hal/ga102.rs | 29 +-- drivers/gpu/nova-core/falcon/hal/tu102.rs | 24 +-- drivers/gpu/nova-core/firmware/booter.rs | 25 +-- drivers/gpu/nova-core/firmware/fwsec.rs | 19 +- .../nova-core/firmware/fwsec/bootloader.rs | 15 +- drivers/gpu/nova-core/fsp.rs | 23 +-- drivers/gpu/nova-core/gpu.rs | 9 +- drivers/gpu/nova-core/gsp.rs | 4 +- drivers/gpu/nova-core/gsp/boot.rs | 22 +- drivers/gpu/nova-core/gsp/hal.rs | 4 +- drivers/gpu/nova-core/gsp/hal/gh100.rs | 32 ++- drivers/gpu/nova-core/gsp/hal/tu102.rs | 68 +++--- drivers/gpu/nova-core/gsp/sequencer.rs | 31 ++- 17 files changed, 263 insertions(+), 316 deletions(-) diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index 94c7696a6493..78948cc8bff3 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -5,10 +5,7 @@ use hal::FalconHal; use kernel::{ - device::{ - self, - Device, // - }, + device, dma::{ Coherent, CoherentBox, @@ -24,7 +21,6 @@ Io, }, prelude::*, - sync::aref::ARef, time::Delta, }; @@ -358,41 +354,47 @@ pub(crate) trait FalconFirmware { } /// Contains the base parameters common to all Falcon instances. -pub(crate) struct Falcon { +pub(crate) struct Falcon<'a, E: FalconEngine> { hal: KBox>, - dev: ARef, + dev: &'a device::Device, + bar: Bar0<'a>, } -impl Falcon { +impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { /// Create a new falcon instance. - pub(crate) fn new(dev: &device::Device, chipset: Chipset) -> Result { + pub(crate) fn new( + dev: &'a device::Device, + chipset: Chipset, + bar: Bar0<'a>, + ) -> Result { Ok(Self { hal: hal::falcon_hal(chipset)?, - dev: dev.into(), + dev, + bar, }) } /// Resets DMA-related registers. - pub(crate) fn dma_reset(&self, bar: Bar0<'_>) { - bar.update(regs::NV_PFALCON_FBIF_CTL::of::(), |v| { + pub(crate) fn dma_reset(&self) { + self.bar.update(regs::NV_PFALCON_FBIF_CTL::of::(), |v| { v.with_allow_phys_no_ctx(true) }); - bar.write( + self.bar.write( WithBase::of::(), regs::NV_PFALCON_FALCON_DMACTL::zeroed(), ); } /// Reset the controller, select the falcon core, and wait for memory scrubbing to complete. - pub(crate) fn reset(&self, bar: Bar0<'_>) -> Result { - self.hal.reset_eng(bar)?; - self.hal.select_core(self, bar)?; - self.hal.reset_wait_mem_scrubbing(bar)?; + pub(crate) fn reset(&self) -> Result { + self.hal.reset_eng(self)?; + self.hal.select_core(self)?; + self.hal.reset_wait_mem_scrubbing(self)?; - bar.write( + self.bar.write( WithBase::of::(), - regs::NV_PFALCON_FALCON_RM::from(bar.read(regs::NV_PMC_BOOT_0).into_raw()), + regs::NV_PFALCON_FALCON_RM::from(self.bar.read(regs::NV_PMC_BOOT_0).into_raw()), ); Ok(()) @@ -404,18 +406,14 @@ pub(crate) fn reset(&self, bar: Bar0<'_>) -> Result { /// Write a slice to Falcon IMEM memory using programmed I/O (PIO). /// /// Returns `EINVAL` if `img.len()` is not a multiple of 4. - fn pio_wr_imem_slice( - &self, - bar: Bar0<'_>, - load_offsets: FalconPioImemLoadTarget<'_>, - ) -> Result { + fn pio_wr_imem_slice(&self, load_offsets: FalconPioImemLoadTarget<'_>) -> Result { // Rejecting misaligned images here allows us to avoid checking // inside the loops. if load_offsets.data.len() % 4 != 0 { return Err(EINVAL); } - bar.write( + self.bar.write( WithBase::of::().at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_IMEMC::zeroed() .with_secure(load_offsets.secure) @@ -426,13 +424,13 @@ fn pio_wr_imem_slice( for (n, block) in load_offsets.data.chunks(MEM_BLOCK_ALIGNMENT).enumerate() { let n = u16::try_from(n)?; let tag: u16 = load_offsets.start_tag.checked_add(n).ok_or(ERANGE)?; - bar.write( + self.bar.write( WithBase::of::().at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_IMEMT::zeroed().with_tag(tag), ); for word in block.chunks_exact(4) { let w = [word[0], word[1], word[2], word[3]]; - bar.write( + self.bar.write( WithBase::of::().at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_IMEMD::zeroed().with_data(u32::from_le_bytes(w)), ); @@ -445,18 +443,14 @@ fn pio_wr_imem_slice( /// Write a slice to Falcon DMEM memory using programmed I/O (PIO). /// /// Returns `EINVAL` if `img.len()` is not a multiple of 4. - fn pio_wr_dmem_slice( - &self, - bar: Bar0<'_>, - load_offsets: FalconPioDmemLoadTarget<'_>, - ) -> Result { + fn pio_wr_dmem_slice(&self, load_offsets: FalconPioDmemLoadTarget<'_>) -> Result { // Rejecting misaligned images here allows us to avoid checking // inside the loops. if load_offsets.data.len() % 4 != 0 { return Err(EINVAL); } - bar.write( + self.bar.write( WithBase::of::().at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_DMEMC::zeroed() .with_aincw(true) @@ -465,7 +459,7 @@ fn pio_wr_dmem_slice( for word in load_offsets.data.chunks_exact(4) { let w = [word[0], word[1], word[2], word[3]]; - bar.write( + self.bar.write( WithBase::of::().at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_DMEMD::zeroed().with_data(u32::from_le_bytes(w)), ); @@ -477,29 +471,28 @@ fn pio_wr_dmem_slice( /// Perform a PIO copy into `IMEM` and `DMEM` of `fw`, and prepare the falcon to run it. pub(crate) fn pio_load + FalconPioLoadable>( &self, - bar: Bar0<'_>, fw: &F, ) -> Result { - bar.update(regs::NV_PFALCON_FBIF_CTL::of::(), |v| { + self.bar.update(regs::NV_PFALCON_FBIF_CTL::of::(), |v| { v.with_allow_phys_no_ctx(true) }); - bar.write( + self.bar.write( WithBase::of::(), regs::NV_PFALCON_FALCON_DMACTL::zeroed(), ); if let Some(imem_ns) = fw.imem_ns_load_params() { - self.pio_wr_imem_slice(bar, imem_ns)?; + self.pio_wr_imem_slice(imem_ns)?; } if let Some(imem_sec) = fw.imem_sec_load_params() { - self.pio_wr_imem_slice(bar, imem_sec)?; + self.pio_wr_imem_slice(imem_sec)?; } - self.pio_wr_dmem_slice(bar, fw.dmem_load_params())?; + self.pio_wr_dmem_slice(fw.dmem_load_params())?; - self.hal.program_brom(self, bar, &fw.brom_params()); + self.hal.program_brom(self, &fw.brom_params()); - bar.write( + self.bar.write( WithBase::of::(), regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()), ); @@ -513,7 +506,6 @@ pub(crate) fn pio_load + FalconPioLoadable>( /// `sec` is set if the loaded firmware is expected to run in secure mode. fn dma_wr( &self, - bar: Bar0<'_>, dma_obj: &Coherent<[u8]>, target_mem: FalconMem, load_offsets: FalconDmaLoadTarget, @@ -571,7 +563,7 @@ fn dma_wr( // Set up the base source DMA address. - bar.write( + self.bar.write( WithBase::of::(), regs::NV_PFALCON_FALCON_DMATRFBASE::zeroed().with_base( // CAST: `as u32` is used on purpose since we do want to strip the upper bits, @@ -579,7 +571,7 @@ fn dma_wr( (dma_start >> 8) as u32, ), ); - bar.write( + self.bar.write( WithBase::of::(), regs::NV_PFALCON_FALCON_DMATRFBASE1::zeroed().try_with_base(dma_start >> 40)?, ); @@ -590,23 +582,23 @@ fn dma_wr( for pos in (0..num_transfers).map(|i| i * DMA_LEN) { // Perform a transfer of size `DMA_LEN`. - bar.write( + self.bar.write( WithBase::of::(), regs::NV_PFALCON_FALCON_DMATRFMOFFS::zeroed() .try_with_offs(load_offsets.dst_start + pos)?, ); - bar.write( + self.bar.write( WithBase::of::(), regs::NV_PFALCON_FALCON_DMATRFFBOFFS::zeroed().with_offs(src_start + pos), ); - bar.write(WithBase::of::(), cmd); + self.bar.write(WithBase::of::(), cmd); // Wait for the transfer to complete. // TIMEOUT: arbitrarily large value, no DMA transfer to the falcon's small memories // should ever take that long. read_poll_timeout( - || Ok(bar.read(regs::NV_PFALCON_FALCON_DMATRFCMD::of::())), + || Ok(self.bar.read(regs::NV_PFALCON_FALCON_DMATRFCMD::of::())), |r| r.idle(), Delta::ZERO, Delta::from_secs(2), @@ -617,12 +609,7 @@ fn dma_wr( } /// Perform a DMA load into `IMEM` and `DMEM` of `fw`, and prepare the falcon to run it. - fn dma_load + FalconDmaLoadable>( - &self, - dev: &Device, - bar: Bar0<'_>, - fw: &F, - ) -> Result { + fn dma_load + FalconDmaLoadable>(&self, fw: &F) -> Result { // DMA object with firmware content as the source of the DMA engine. let dma_obj = { let fw_slice = fw.as_slice(); @@ -630,7 +617,7 @@ fn dma_load + FalconDmaLoadable>( // DMA copies are done in chunks of `MEM_BLOCK_ALIGNMENT`, so pad the length // accordingly and fill with `0`. let mut dma_obj = CoherentBox::zeroed_slice( - dev, + self.dev, fw_slice.len().next_multiple_of(MEM_BLOCK_ALIGNMENT), GFP_KERNEL, )?; @@ -642,24 +629,20 @@ fn dma_load + FalconDmaLoadable>( dma_obj.into() }; - self.dma_reset(bar); - bar.update(regs::NV_PFALCON_FBIF_TRANSCFG::of::().at(0), |v| { - v.with_target(FalconFbifTarget::CoherentSysmem) - .with_mem_type(FalconFbifMemType::Physical) - }); + self.dma_reset(); + self.bar + .update(regs::NV_PFALCON_FBIF_TRANSCFG::of::().at(0), |v| { + v.with_target(FalconFbifTarget::CoherentSysmem) + .with_mem_type(FalconFbifMemType::Physical) + }); - self.dma_wr( - bar, - &dma_obj, - FalconMem::ImemSecure, - fw.imem_sec_load_params(), - )?; - self.dma_wr(bar, &dma_obj, FalconMem::Dmem, fw.dmem_load_params())?; + self.dma_wr(&dma_obj, FalconMem::ImemSecure, fw.imem_sec_load_params())?; + self.dma_wr(&dma_obj, FalconMem::Dmem, fw.dmem_load_params())?; - self.hal.program_brom(self, bar, &fw.brom_params()); + self.hal.program_brom(self, &fw.brom_params()); // Set `BootVec` to start of non-secure code. - bar.write( + self.bar.write( WithBase::of::(), regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()), ); @@ -668,10 +651,10 @@ fn dma_load + FalconDmaLoadable>( } /// Wait until the falcon CPU is halted. - pub(crate) fn wait_till_halted(&self, bar: Bar0<'_>) -> Result<()> { + pub(crate) fn wait_till_halted(&self) -> Result<()> { // TIMEOUT: arbitrarily large value, firmwares should complete in less than 2 seconds. read_poll_timeout( - || Ok(bar.read(regs::NV_PFALCON_FALCON_CPUCTL::of::())), + || Ok(self.bar.read(regs::NV_PFALCON_FALCON_CPUCTL::of::())), |r| r.halted(), Delta::ZERO, Delta::from_secs(2), @@ -681,16 +664,17 @@ pub(crate) fn wait_till_halted(&self, bar: Bar0<'_>) -> Result<()> { } /// Start the falcon CPU. - pub(crate) fn start(&self, bar: Bar0<'_>) -> Result<()> { - match bar + pub(crate) fn start(&self) -> Result<()> { + match self + .bar .read(regs::NV_PFALCON_FALCON_CPUCTL::of::()) .alias_en() { - true => bar.write( + true => self.bar.write( WithBase::of::(), regs::NV_PFALCON_FALCON_CPUCTL_ALIAS::zeroed().with_startcpu(true), ), - false => bar.write( + false => self.bar.write( WithBase::of::(), regs::NV_PFALCON_FALCON_CPUCTL::zeroed().with_startcpu(true), ), @@ -700,16 +684,16 @@ pub(crate) fn start(&self, bar: Bar0<'_>) -> Result<()> { } /// Writes values to the mailbox registers if provided. - pub(crate) fn write_mailboxes(&self, bar: Bar0<'_>, mbox0: Option, mbox1: Option) { + pub(crate) fn write_mailboxes(&self, mbox0: Option, mbox1: Option) { if let Some(mbox0) = mbox0 { - bar.write( + self.bar.write( WithBase::of::(), regs::NV_PFALCON_FALCON_MAILBOX0::zeroed().with_value(mbox0), ); } if let Some(mbox1) = mbox1 { - bar.write( + self.bar.write( WithBase::of::(), regs::NV_PFALCON_FALCON_MAILBOX1::zeroed().with_value(mbox1), ); @@ -717,21 +701,23 @@ pub(crate) fn write_mailboxes(&self, bar: Bar0<'_>, mbox0: Option, mbox1: O } /// Reads the value from `mbox0` register. - pub(crate) fn read_mailbox0(&self, bar: Bar0<'_>) -> u32 { - bar.read(regs::NV_PFALCON_FALCON_MAILBOX0::of::()) + pub(crate) fn read_mailbox0(&self) -> u32 { + self.bar + .read(regs::NV_PFALCON_FALCON_MAILBOX0::of::()) .value() } /// Reads the value from `mbox1` register. - pub(crate) fn read_mailbox1(&self, bar: Bar0<'_>) -> u32 { - bar.read(regs::NV_PFALCON_FALCON_MAILBOX1::of::()) + pub(crate) fn read_mailbox1(&self) -> u32 { + self.bar + .read(regs::NV_PFALCON_FALCON_MAILBOX1::of::()) .value() } /// Reads values from both mailbox registers. - pub(crate) fn read_mailboxes(&self, bar: Bar0<'_>) -> (u32, u32) { - let mbox0 = self.read_mailbox0(bar); - let mbox1 = self.read_mailbox1(bar); + pub(crate) fn read_mailboxes(&self) -> (u32, u32) { + let mbox0 = self.read_mailbox0(); + let mbox1 = self.read_mailbox1(); (mbox0, mbox1) } @@ -743,54 +729,43 @@ pub(crate) fn read_mailboxes(&self, bar: Bar0<'_>) -> (u32, u32) { /// /// Wait up to two seconds for the firmware to complete, and return its exit status read from /// the `MBOX0` and `MBOX1` registers. - pub(crate) fn boot( - &self, - bar: Bar0<'_>, - mbox0: Option, - mbox1: Option, - ) -> Result<(u32, u32)> { - self.write_mailboxes(bar, mbox0, mbox1); - self.start(bar)?; - self.wait_till_halted(bar)?; - Ok(self.read_mailboxes(bar)) + pub(crate) fn boot(&self, mbox0: Option, mbox1: Option) -> Result<(u32, u32)> { + self.write_mailboxes(mbox0, mbox1); + self.start()?; + self.wait_till_halted()?; + Ok(self.read_mailboxes()) } /// Returns the fused version of the signature to use in order to run a HS firmware on this /// falcon instance. `engine_id_mask` and `ucode_id` are obtained from the firmware header. pub(crate) fn signature_reg_fuse_version( &self, - bar: Bar0<'_>, engine_id_mask: u16, ucode_id: u8, ) -> Result { self.hal - .signature_reg_fuse_version(self, bar, engine_id_mask, ucode_id) + .signature_reg_fuse_version(self, engine_id_mask, ucode_id) } /// Check if the RISC-V core is active. /// /// Returns `true` if the RISC-V core is active, `false` otherwise. - pub(crate) fn is_riscv_active(&self, bar: Bar0<'_>) -> bool { - self.hal.is_riscv_active(bar) + pub(crate) fn is_riscv_active(&self) -> bool { + self.hal.is_riscv_active(self) } /// Load a firmware image into Falcon memory, using the preferred method for the current /// chipset. - pub(crate) fn load + FalconDmaLoadable>( - &self, - dev: &Device, - bar: Bar0<'_>, - fw: &F, - ) -> Result { + pub(crate) fn load + FalconDmaLoadable>(&self, fw: &F) -> Result { match self.hal.load_method() { - LoadMethod::Dma => self.dma_load(dev, bar, fw), - LoadMethod::Pio => self.pio_load(bar, &fw.try_as_pio_loadable()?), + LoadMethod::Dma => self.dma_load(fw), + LoadMethod::Pio => self.pio_load(&fw.try_as_pio_loadable()?), } } /// Write the application version to the OS register. - pub(crate) fn write_os_version(&self, bar: Bar0<'_>, app_version: u32) { - bar.write( + pub(crate) fn write_os_version(&self, app_version: u32) { + self.bar.write( WithBase::of::(), regs::NV_PFALCON_FALCON_OS::zeroed().with_value(app_version), ); diff --git a/drivers/gpu/nova-core/falcon/fsp.rs b/drivers/gpu/nova-core/falcon/fsp.rs index 52cdb84ef0e8..53b1079843ae 100644 --- a/drivers/gpu/nova-core/falcon/fsp.rs +++ b/drivers/gpu/nova-core/falcon/fsp.rs @@ -21,7 +21,6 @@ }; use crate::{ - driver::Bar0, falcon::{ Falcon, FalconEngine, @@ -48,18 +47,18 @@ impl RegisterBase for Fsp { impl FalconEngine for Fsp {} -impl Falcon { +impl<'a> Falcon<'a, Fsp> { /// Writes `data` to FSP external memory at offset `0`. /// /// `data` is interpreted as little-endian 32-bit words. Returns `EINVAL` /// if the `data` length is not 4-byte aligned. - fn write_emem(&mut self, bar: Bar0<'_>, data: &[u8]) -> Result { + fn write_emem(&mut self, data: &[u8]) -> Result { if data.len() % 4 != 0 { return Err(EINVAL); } // Begin a write burst at offset `0`, auto-incrementing on each write. - bar.write( + self.bar.write( WithBase::of::(), regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincw(true), ); @@ -68,7 +67,7 @@ fn write_emem(&mut self, bar: Bar0<'_>, data: &[u8]) -> Result { let value = u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); // Write the next 32-bit `value`; hardware advances the offset. - bar.write( + self.bar.write( WithBase::of::(), regs::NV_PFALCON_FALCON_EMEMD::zeroed().with_data(value), ); @@ -81,20 +80,23 @@ fn write_emem(&mut self, bar: Bar0<'_>, data: &[u8]) -> Result { /// /// `data` is stored as little-endian 32-bit words. Returns `EINVAL` if /// the `data` length is not 4-byte aligned. - fn read_emem(&mut self, bar: Bar0<'_>, data: &mut [u8]) -> Result { + fn read_emem(&mut self, data: &mut [u8]) -> Result { if data.len() % 4 != 0 { return Err(EINVAL); } // Begin a read burst at offset `0`, auto-incrementing on each read. - bar.write( + self.bar.write( WithBase::of::(), regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincr(true), ); for chunk in data.chunks_exact_mut(4) { // Read the next 32-bit word; hardware advances the offset. - let value = bar.read(regs::NV_PFALCON_FALCON_EMEMD::of::()).data(); + let value = self + .bar + .read(regs::NV_PFALCON_FALCON_EMEMD::of::()) + .data(); chunk.copy_from_slice(&value.to_le_bytes()); } @@ -107,9 +109,9 @@ fn read_emem(&mut self, bar: Bar0<'_>, data: &mut [u8]) -> Result { /// /// The FSP message queue is not circular. Pointers are reset to 0 after each /// message exchange, so `tail >= head` is always true when data is present. - fn poll_msgq(&self, bar: Bar0<'_>) -> u32 { - let head = bar.read(regs::NV_PFSP_MSGQ_HEAD::at(0)).val(); - let tail = bar.read(regs::NV_PFSP_MSGQ_TAIL::at(0)).val(); + fn poll_msgq(&self) -> u32 { + let head = self.bar.read(regs::NV_PFSP_MSGQ_HEAD::at(0)).val(); + let tail = self.bar.read(regs::NV_PFSP_MSGQ_TAIL::at(0)).val(); if head == tail { return 0; @@ -122,20 +124,20 @@ fn poll_msgq(&self, bar: Bar0<'_>) -> u32 { /// Writes `packet` to FSP EMEM and updates the queue pointers to notify FSP. /// /// Returns `EINVAL` if `packet` is empty or its length is not 4-byte aligned. - pub(crate) fn send_msg(&mut self, bar: Bar0<'_>, packet: &[u8]) -> Result { + pub(crate) fn send_msg(&mut self, packet: &[u8]) -> Result { if packet.is_empty() { return Err(EINVAL); } - self.write_emem(bar, packet)?; + self.write_emem(packet)?; // Update queue pointers. TAIL points at the last DWORD written. let tail_offset = u32::try_from(packet.len() - 4).map_err(|_| EINVAL)?; - bar.write( + self.bar.write( Array::at(0), regs::NV_PFSP_QUEUE_TAIL::zeroed().with_address(tail_offset), ); - bar.write( + self.bar.write( Array::at(0), regs::NV_PFSP_QUEUE_HEAD::zeroed().with_address(0), ); @@ -148,9 +150,9 @@ pub(crate) fn send_msg(&mut self, bar: Bar0<'_>, packet: &[u8]) -> Result { /// /// Returns `ETIMEDOUT` if no message was available until timeout, or a regular error code if a /// memory allocation error occurred. - pub(crate) fn recv_msg(&mut self, bar: Bar0<'_>) -> Result> { + pub(crate) fn recv_msg(&mut self) -> Result> { let msg_size = read_poll_timeout( - || Ok(self.poll_msgq(bar)), + || Ok(self.poll_msgq()), |&size| size > 0, Delta::from_millis(10), Delta::from_millis(FSP_MSG_TIMEOUT_MS), @@ -160,11 +162,13 @@ pub(crate) fn recv_msg(&mut self, bar: Bar0<'_>) -> Result> { let mut buffer = KVec::::new(); buffer.resize(msg_size, 0, GFP_KERNEL)?; - self.read_emem(bar, &mut buffer)?; + self.read_emem(&mut buffer)?; // Reset message queue pointers after reading. - bar.write(Array::at(0), regs::NV_PFSP_MSGQ_TAIL::zeroed().with_val(0)); - bar.write(Array::at(0), regs::NV_PFSP_MSGQ_HEAD::zeroed().with_val(0)); + self.bar + .write(Array::at(0), regs::NV_PFSP_MSGQ_TAIL::zeroed().with_val(0)); + self.bar + .write(Array::at(0), regs::NV_PFSP_MSGQ_HEAD::zeroed().with_val(0)); Ok(buffer) } diff --git a/drivers/gpu/nova-core/falcon/gsp.rs b/drivers/gpu/nova-core/falcon/gsp.rs index f788b87bd951..ae32f401aeb0 100644 --- a/drivers/gpu/nova-core/falcon/gsp.rs +++ b/drivers/gpu/nova-core/falcon/gsp.rs @@ -14,7 +14,6 @@ }; use crate::{ - driver::Bar0, falcon::{ Falcon, FalconEngine, @@ -37,20 +36,20 @@ impl RegisterBase for Gsp { impl FalconEngine for Gsp {} -impl Falcon { +impl<'a> Falcon<'a, Gsp> { /// Clears the SWGEN0 bit in the Falcon's IRQ status clear register to /// allow GSP to signal CPU for processing new messages in message queue. - pub(crate) fn clear_swgen0_intr(&self, bar: Bar0<'_>) { - bar.write( + pub(crate) fn clear_swgen0_intr(&self) { + self.bar.write( WithBase::of::(), regs::NV_PFALCON_FALCON_IRQSCLR::zeroed().with_swgen0(true), ); } /// Checks if GSP reload/resume has completed during the boot process. - pub(crate) fn check_reload_completed(&self, bar: Bar0<'_>, timeout: Delta) -> Result { + pub(crate) fn check_reload_completed(&self, timeout: Delta) -> Result { read_poll_timeout( - || Ok(bar.read(regs::NV_PGC6_BSI_SECURE_SCRATCH_14)), + || Ok(self.bar.read(regs::NV_PGC6_BSI_SECURE_SCRATCH_14)), |val| val.boot_stage_3_handoff(), Delta::ZERO, timeout, @@ -59,19 +58,21 @@ pub(crate) fn check_reload_completed(&self, bar: Bar0<'_>, timeout: Delta) -> Re } /// Returns whether the RISC-V branch privilege lockdown bit is set. - pub(crate) fn riscv_branch_privilege_lockdown(&self, bar: Bar0<'_>) -> bool { - bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::()) + pub(crate) fn riscv_branch_privilege_lockdown(&self) -> bool { + self.bar + .read(regs::NV_PFALCON_FALCON_HWCFG2::of::()) .riscv_br_priv_lockdown() } /// Returns whether GSP registers can be read by the CPU. - pub(crate) fn priv_target_mask_released(&self, bar: Bar0<'_>) -> bool { + pub(crate) fn priv_target_mask_released(&self) -> bool { /// Pattern returned by GSP register reads while the PRIV target mask still blocks CPU /// access. The low byte varies; the upper 24 bits are fixed. const LOCKED_PATTERN: u32 = 0xbadf_4100; const LOCKED_MASK: u32 = 0xffff_ff00; - let hwcfg2 = bar + let hwcfg2 = self + .bar .read(regs::NV_PFALCON_FALCON_HWCFG2::of::()) .into_raw(); diff --git a/drivers/gpu/nova-core/falcon/hal.rs b/drivers/gpu/nova-core/falcon/hal.rs index 89b56823906b..ee4a017f3a4c 100644 --- a/drivers/gpu/nova-core/falcon/hal.rs +++ b/drivers/gpu/nova-core/falcon/hal.rs @@ -3,7 +3,6 @@ use kernel::prelude::*; use crate::{ - driver::Bar0, falcon::{ Falcon, FalconBromParams, @@ -34,7 +33,7 @@ pub(crate) enum LoadMethod { /// registers. pub(crate) trait FalconHal: Send + Sync { /// Activates the Falcon core if the engine is a risvc/falcon dual engine. - fn select_core(&self, _falcon: &Falcon, _bar: Bar0<'_>) -> Result { + fn select_core(&self, _falcon: &Falcon<'_, E>) -> Result { Ok(()) } @@ -42,24 +41,23 @@ fn select_core(&self, _falcon: &Falcon, _bar: Bar0<'_>) -> Result { /// falcon instance. `engine_id_mask` and `ucode_id` are obtained from the firmware header. fn signature_reg_fuse_version( &self, - falcon: &Falcon, - bar: Bar0<'_>, + falcon: &Falcon<'_, E>, engine_id_mask: u16, ucode_id: u8, ) -> Result; /// Program the boot ROM registers prior to starting a secure firmware. - fn program_brom(&self, falcon: &Falcon, bar: Bar0<'_>, params: &FalconBromParams); + fn program_brom(&self, falcon: &Falcon<'_, E>, params: &FalconBromParams); /// Check if the RISC-V core is active. /// Returns `true` if the RISC-V core is active, `false` otherwise. - fn is_riscv_active(&self, bar: Bar0<'_>) -> bool; + fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool; /// Wait for memory scrubbing to complete. - fn reset_wait_mem_scrubbing(&self, bar: Bar0<'_>) -> Result; + fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result; /// Reset the falcon engine. - fn reset_eng(&self, bar: Bar0<'_>) -> Result; + fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result; /// Returns the method used to load data into the falcon's memory. /// diff --git a/drivers/gpu/nova-core/falcon/hal/ga102.rs b/drivers/gpu/nova-core/falcon/hal/ga102.rs index cf6ce47e6b25..fe821ded5fa1 100644 --- a/drivers/gpu/nova-core/falcon/hal/ga102.rs +++ b/drivers/gpu/nova-core/falcon/hal/ga102.rs @@ -115,33 +115,34 @@ pub(super) fn new() -> Self { } impl FalconHal for Ga102 { - fn select_core(&self, _falcon: &Falcon, bar: Bar0<'_>) -> Result { - select_core_ga102::(bar) + fn select_core(&self, falcon: &Falcon<'_, E>) -> Result { + select_core_ga102::(falcon.bar) } fn signature_reg_fuse_version( &self, - falcon: &Falcon, - bar: Bar0<'_>, + falcon: &Falcon<'_, E>, engine_id_mask: u16, ucode_id: u8, ) -> Result { - signature_reg_fuse_version_ga102(&falcon.dev, bar, engine_id_mask, ucode_id) + signature_reg_fuse_version_ga102(falcon.dev, falcon.bar, engine_id_mask, ucode_id) } - fn program_brom(&self, _falcon: &Falcon, bar: Bar0<'_>, params: &FalconBromParams) { - program_brom_ga102::(bar, params); + fn program_brom(&self, falcon: &Falcon<'_, E>, params: &FalconBromParams) { + program_brom_ga102::(falcon.bar, params); } - fn is_riscv_active(&self, bar: Bar0<'_>) -> bool { - bar.read(regs::NV_PRISCV_RISCV_CPUCTL::of::()) + fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool { + falcon + .bar + .read(regs::NV_PRISCV_RISCV_CPUCTL::of::()) .active_stat() } - fn reset_wait_mem_scrubbing(&self, bar: Bar0<'_>) -> Result { + fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result { // TIMEOUT: memory scrubbing should complete in less than 20ms. read_poll_timeout( - || Ok(bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::())), + || Ok(falcon.bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::())), |r| r.mem_scrubbing_done(), Delta::ZERO, Delta::from_millis(20), @@ -149,7 +150,9 @@ fn reset_wait_mem_scrubbing(&self, bar: Bar0<'_>) -> Result { .map(|_| ()) } - fn reset_eng(&self, bar: Bar0<'_>) -> Result { + fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result { + let bar = falcon.bar; + let _ = bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::()); // According to OpenRM's `kflcnPreResetWait_GA102` documentation, HW sometimes does not set @@ -162,7 +165,7 @@ fn reset_eng(&self, bar: Bar0<'_>) -> Result { ); regs::NV_PFALCON_FALCON_ENGINE::reset_engine::(bar); - self.reset_wait_mem_scrubbing(bar)?; + self.reset_wait_mem_scrubbing(falcon)?; Ok(()) } diff --git a/drivers/gpu/nova-core/falcon/hal/tu102.rs b/drivers/gpu/nova-core/falcon/hal/tu102.rs index 3aaee3869312..34bf9f3f44c7 100644 --- a/drivers/gpu/nova-core/falcon/hal/tu102.rs +++ b/drivers/gpu/nova-core/falcon/hal/tu102.rs @@ -13,7 +13,6 @@ }; use crate::{ - driver::Bar0, falcon::{ hal::LoadMethod, Falcon, @@ -34,31 +33,32 @@ pub(super) fn new() -> Self { } impl FalconHal for Tu102 { - fn select_core(&self, _falcon: &Falcon, _bar: Bar0<'_>) -> Result { + fn select_core(&self, _falcon: &Falcon<'_, E>) -> Result { Ok(()) } fn signature_reg_fuse_version( &self, - _falcon: &Falcon, - _bar: Bar0<'_>, + _falcon: &Falcon<'_, E>, _engine_id_mask: u16, _ucode_id: u8, ) -> Result { Ok(0) } - fn program_brom(&self, _falcon: &Falcon, _bar: Bar0<'_>, _params: &FalconBromParams) {} + fn program_brom(&self, _falcon: &Falcon<'_, E>, _params: &FalconBromParams) {} - fn is_riscv_active(&self, bar: Bar0<'_>) -> bool { - bar.read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS::of::()) + fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool { + falcon + .bar + .read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS::of::()) .active_stat() } - fn reset_wait_mem_scrubbing(&self, bar: Bar0<'_>) -> Result { + fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result { // TIMEOUT: memory scrubbing should complete in less than 10ms. read_poll_timeout( - || Ok(bar.read(regs::NV_PFALCON_FALCON_DMACTL::of::())), + || Ok(falcon.bar.read(regs::NV_PFALCON_FALCON_DMACTL::of::())), |r| r.mem_scrubbing_done(), Delta::ZERO, Delta::from_millis(10), @@ -66,9 +66,9 @@ fn reset_wait_mem_scrubbing(&self, bar: Bar0<'_>) -> Result { .map(|_| ()) } - fn reset_eng(&self, bar: Bar0<'_>) -> Result { - regs::NV_PFALCON_FALCON_ENGINE::reset_engine::(bar); - self.reset_wait_mem_scrubbing(bar)?; + fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result { + regs::NV_PFALCON_FALCON_ENGINE::reset_engine::(falcon.bar); + self.reset_wait_mem_scrubbing(falcon)?; Ok(()) } diff --git a/drivers/gpu/nova-core/firmware/booter.rs b/drivers/gpu/nova-core/firmware/booter.rs index d9313ac361af..acb7f4d8a532 100644 --- a/drivers/gpu/nova-core/firmware/booter.rs +++ b/drivers/gpu/nova-core/firmware/booter.rs @@ -15,7 +15,6 @@ }; use crate::{ - driver::Bar0, falcon::{ sec2::Sec2, Falcon, @@ -293,8 +292,7 @@ pub(crate) fn new( kind: BooterKind, chipset: Chipset, ver: &str, - falcon: &Falcon<::Target>, - bar: Bar0<'_>, + falcon: &Falcon<'_, ::Target>, ) -> Result { let fw_name = match kind { BooterKind::Loader => "booter_load", @@ -339,11 +337,8 @@ pub(crate) fn new( } else { // Obtain the version from the fuse register, and extract the corresponding // signature. - let reg_fuse_version = falcon.signature_reg_fuse_version( - bar, - brom_params.engine_id_mask, - brom_params.ucode_id, - )?; + let reg_fuse_version = falcon + .signature_reg_fuse_version(brom_params.engine_id_mask, brom_params.ucode_id)?; // `0` means the last signature should be used. const FUSE_VERSION_USE_LAST_SIG: u32 = 0; @@ -405,18 +400,14 @@ pub(crate) fn new( pub(crate) fn run( &self, dev: &device::Device, - bar: Bar0<'_>, - sec2_falcon: &Falcon, + sec2_falcon: &Falcon<'_, Sec2>, wpr_meta: &Coherent, ) -> Result { - sec2_falcon.reset(bar)?; - sec2_falcon.load(dev, bar, self)?; + sec2_falcon.reset()?; + sec2_falcon.load(self)?; let wpr_handle = wpr_meta.dma_handle(); - let (mbox0, mbox1) = sec2_falcon.boot( - bar, - Some(wpr_handle as u32), - Some((wpr_handle >> 32) as u32), - )?; + let (mbox0, mbox1) = + sec2_falcon.boot(Some(wpr_handle as u32), Some((wpr_handle >> 32) as u32))?; dev_dbg!(dev, "SEC2 MBOX0: {:#x}, MBOX1: {:#x}\n", mbox0, mbox1); if mbox0 != 0 { diff --git a/drivers/gpu/nova-core/firmware/fwsec.rs b/drivers/gpu/nova-core/firmware/fwsec.rs index 199ae2adb664..95e0dd77746b 100644 --- a/drivers/gpu/nova-core/firmware/fwsec.rs +++ b/drivers/gpu/nova-core/firmware/fwsec.rs @@ -27,7 +27,6 @@ }; use crate::{ - driver::Bar0, falcon::{ gsp::Gsp, Falcon, @@ -320,8 +319,7 @@ impl FwsecFirmware { /// command. pub(crate) fn new( dev: &Device, - falcon: &Falcon, - bar: Bar0<'_>, + falcon: &Falcon<'_, Gsp>, bios: &Vbios, cmd: FwsecCommand, ) -> Result { @@ -337,7 +335,7 @@ pub(crate) fn new( .ok_or(EINVAL)?; let desc_sig_versions = u32::from(desc.signature_versions()); let reg_fuse_version = - falcon.signature_reg_fuse_version(bar, desc.engine_id_mask(), desc.ucode_id())?; + falcon.signature_reg_fuse_version(desc.engine_id_mask(), desc.ucode_id())?; dev_dbg!( dev, "desc_sig_versions: {:#x}, reg_fuse_version: {}\n", @@ -390,21 +388,16 @@ pub(crate) fn new( /// This must only be called on chipsets that do not need the FWSEC bootloader (i.e., where /// [`Chipset::needs_fwsec_bootloader()`](crate::gpu::Chipset::needs_fwsec_bootloader) returns /// `false`). On chipsets that do, use [`bootloader::FwsecFirmwareWithBl`] instead. - pub(crate) fn run( - &self, - dev: &Device, - falcon: &Falcon, - bar: Bar0<'_>, - ) -> Result<()> { + pub(crate) fn run(&self, dev: &Device, falcon: &Falcon<'_, Gsp>) -> Result<()> { // Reset falcon, load the firmware, and run it. falcon - .reset(bar) + .reset() .inspect_err(|e| dev_err!(dev, "Failed to reset GSP falcon: {:?}\n", e))?; falcon - .load(dev, bar, self) + .load(self) .inspect_err(|e| dev_err!(dev, "Failed to load FWSEC firmware: {:?}\n", e))?; let (mbox0, _) = falcon - .boot(bar, Some(0), None) + .boot(Some(0), None) .inspect_err(|e| dev_err!(dev, "Failed to boot FWSEC firmware: {:?}\n", e))?; if mbox0 != 0 { dev_err!(dev, "FWSEC firmware returned error {}\n", mbox0); diff --git a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs index ac1558a83b83..d9fafd2eea5b 100644 --- a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs +++ b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs @@ -12,10 +12,7 @@ Device, // }, dma::Coherent, - io::{ - register::WithBase, // - Io, - }, + io::{register::WithBase, Io}, prelude::*, ptr::{ Alignable, @@ -50,7 +47,7 @@ FIRMWARE_VERSION, // }, gpu::Chipset, - num::FromSafeCast, + num::FromSafeCast, // regs, }; @@ -278,15 +275,15 @@ pub(crate) fn new( pub(crate) fn run( &self, dev: &Device, - falcon: &Falcon, + falcon: &Falcon<'_, Gsp>, bar: Bar0<'_>, ) -> Result<()> { // Reset falcon, load the firmware, and run it. falcon - .reset(bar) + .reset() .inspect_err(|e| dev_err!(dev, "Failed to reset GSP falcon: {:?}\n", e))?; falcon - .pio_load(bar, self) + .pio_load(self) .inspect_err(|e| dev_err!(dev, "Failed to load FWSEC firmware: {:?}\n", e))?; // Configure DMA index for the bootloader to fetch the FWSEC firmware from system memory. @@ -301,7 +298,7 @@ pub(crate) fn run( ); let (mbox0, _) = falcon - .boot(bar, Some(0), None) + .boot(Some(0), None) .inspect_err(|e| dev_err!(dev, "Failed to boot FWSEC firmware: {:?}\n", e))?; if mbox0 != 0 { dev_err!(dev, "FWSEC firmware returned error {}\n", mbox0); diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index 4b97d1fb505e..574e1627e63c 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -224,27 +224,27 @@ pub(crate) fn boot_params_dma_handle(&self) -> u64 { /// An `Fsp` is produced by [`Fsp::wait_secure_boot`], which only returns once FSP secure boot /// has completed. It owns the FSP falcon and the FMC firmware, which are used for the subsequent /// Chain of Trust boot. -pub(crate) struct Fsp { - falcon: Falcon, +pub(crate) struct Fsp<'a> { + falcon: Falcon<'a, FspEngine>, fsp_fw: FspFirmware, } -impl Fsp { +impl<'a> Fsp<'a> { /// Waits for FSP secure boot completion, then returns the [`Fsp`] interface. /// /// Polls the thermal scratch register until FSP signals boot completion or the timeout /// elapses. Returning an [`Fsp`] only on success guarantees, at the API level, that the /// interface is not used before secure boot has completed. pub(crate) fn wait_secure_boot( - dev: &device::Device, - bar: Bar0<'_>, + dev: &'a device::Device, + bar: Bar0<'a>, chipset: Chipset, - ) -> Result { + ) -> Result> { /// FSP secure boot completion timeout in milliseconds. const FSP_SECURE_BOOT_TIMEOUT_MS: i64 = 5000; let hal = hal::fsp_hal(chipset).ok_or(ENOTSUPP)?; - let falcon = Falcon::::new(dev, chipset)?; + let falcon = Falcon::::new(dev, chipset, bar)?; let fsp_fw = FspFirmware::new(dev, chipset, FIRMWARE_VERSION)?; read_poll_timeout( @@ -262,13 +262,13 @@ pub(crate) fn wait_secure_boot( /// Sends a message to FSP and waits for the response. /// Returns the full response buffer on success. - fn send_sync_fsp(&mut self, dev: &device::Device, bar: Bar0<'_>, msg: &M) -> Result> + fn send_sync_fsp(&mut self, dev: &device::Device, msg: &M) -> Result> where M: MessageToFsp, { - self.falcon.send_msg(bar, msg.as_bytes())?; + self.falcon.send_msg(msg.as_bytes())?; - let response_buf = self.falcon.recv_msg(bar).inspect_err(|e| { + let response_buf = self.falcon.recv_msg().inspect_err(|e| { dev_err!(dev, "FSP response error: {:?}\n", e); })?; @@ -330,7 +330,6 @@ fn send_sync_fsp(&mut self, dev: &device::Device, bar: Bar0<'_>, msg: &M) -> pub(crate) fn boot_fmc( &mut self, dev: &device::Device, - bar: Bar0<'_>, fb_layout: &FbLayout, args: &FmcBootArgs, ) -> Result { @@ -341,7 +340,7 @@ pub(crate) fn boot_fmc( GFP_KERNEL, )?; - let _response_buf = self.send_sync_fsp(dev, bar, &*msg)?; + let _response_buf = self.send_sync_fsp(dev, &*msg)?; dev_dbg!(dev, "FSP Chain of Trust completed successfully\n"); Ok(()) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 4d76be429e75..43c3f4f8df71 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -273,9 +273,9 @@ struct GspResources<'gpu> { /// MMIO mapping of PCI BAR 0. bar: Bar0<'gpu>, /// GSP falcon instance, used for GSP boot up and cleanup. - gsp_falcon: Falcon, + gsp_falcon: Falcon<'gpu, GspFalcon>, /// SEC2 falcon instance, used for GSP boot up and cleanup. - sec2_falcon: Falcon, + sec2_falcon: Falcon<'gpu, Sec2Falcon>, /// GSP runtime data. #[pin] gsp: Gsp, @@ -351,10 +351,11 @@ pub(crate) fn new( gsp_falcon: Falcon::new( pdev.as_ref(), spec.chipset, + bar ) - .inspect(|falcon| falcon.clear_swgen0_intr(bar))?, + .inspect(|falcon| falcon.clear_swgen0_intr())?, - sec2_falcon: Falcon::new(pdev.as_ref(), spec.chipset)?, + sec2_falcon: Falcon::new(pdev.as_ref(), spec.chipset, bar)?, gsp <- Gsp::new(pdev), diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index 73e93403601c..b4ac4156056e 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -57,8 +57,8 @@ pub(crate) struct GspBootContext<'a> { pub(crate) pdev: &'a pci::Device, pub(crate) bar: Bar0<'a>, pub(crate) chipset: Chipset, - pub(crate) gsp_falcon: &'a Falcon, - pub(crate) sec2_falcon: &'a Falcon, + pub(crate) gsp_falcon: &'a Falcon<'a, GspFalcon>, + pub(crate) sec2_falcon: &'a Falcon<'a, Sec2Falcon>, } impl<'a> GspBootContext<'a> { diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index bb2000b7a78b..ab0491b57944 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -37,8 +37,8 @@ pub(super) struct BootUnloadArgs<'a> { gsp: &'a super::Gsp, dev: &'a device::Device, bar: Bar0<'a>, - gsp_falcon: &'a Falcon, - sec2_falcon: &'a Falcon, + gsp_falcon: &'a Falcon<'a, Gsp>, + sec2_falcon: &'a Falcon<'a, Sec2>, unload_bundle: Option, } @@ -56,8 +56,8 @@ pub(super) fn new( gsp: &'a super::Gsp, dev: &'a device::Device, bar: Bar0<'a>, - gsp_falcon: &'a Falcon, - sec2_falcon: &'a Falcon, + gsp_falcon: &'a Falcon<'a, Gsp>, + sec2_falcon: &'a Falcon<'a, Sec2>, unload_bundle: Option, ) -> Self { Self { @@ -120,17 +120,17 @@ pub(crate) fn boot( // Perform the chipset-specific boot sequence, and retrieve the unload bundle. let unload_guard = hal.boot(&self, &ctx, &fb_layout, &wpr_meta)?; - gsp_falcon.write_os_version(bar, gsp_fw.bootloader.app_version); + gsp_falcon.write_os_version(gsp_fw.bootloader.app_version); // Poll for RISC-V to become active before continuing. read_poll_timeout( - || Ok(gsp_falcon.is_riscv_active(bar)), + || Ok(gsp_falcon.is_riscv_active()), |val: &bool| *val, Delta::from_millis(10), Delta::from_secs(5), )?; - dev_dbg!(pdev, "RISC-V active? {}\n", gsp_falcon.is_riscv_active(bar),); + dev_dbg!(pdev, "RISC-V active? {}\n", gsp_falcon.is_riscv_active(),); self.cmdq .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev, chipset))?; @@ -149,7 +149,7 @@ pub(crate) fn boot( fn shutdown_gsp( cmdq: &Cmdq, bar: Bar0<'_>, - gsp_falcon: &Falcon, + gsp_falcon: &Falcon<'_, Gsp>, mode: commands::PowerStateLevel, ) -> Result { // Command to shut the GSP down. @@ -158,7 +158,7 @@ fn shutdown_gsp( // Wait until GSP signals it is suspended. const LIBOS_INTERRUPT_PROCESSOR_SUSPENDED: u32 = bits::bit_u32(31); read_poll_timeout( - || Ok(gsp_falcon.read_mailbox0(bar)), + || Ok(gsp_falcon.read_mailbox0()), |&mb0| mb0 & LIBOS_INTERRUPT_PROCESSOR_SUSPENDED != 0, Delta::from_millis(10), Delta::from_secs(5), @@ -173,8 +173,8 @@ pub(crate) fn unload( &self, dev: &device::Device, bar: Bar0<'_>, - gsp_falcon: &Falcon, - sec2_falcon: &Falcon, + gsp_falcon: &Falcon<'_, Gsp>, + sec2_falcon: &Falcon<'_, Sec2>, unload_bundle: Option, ) -> Result { // Shut down the GSP. Keep going even in case of error. diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs index 51a277fe97bb..d3e47ef206de 100644 --- a/drivers/gpu/nova-core/gsp/hal.rs +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -42,8 +42,8 @@ fn run( &self, dev: &device::Device, bar: Bar0<'_>, - gsp_falcon: &Falcon, - sec2_falcon: &Falcon, + gsp_falcon: &Falcon<'_, GspEngine>, + sec2_falcon: &Falcon<'_, Sec2>, ) -> Result; } diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index 2187e11168b2..1d06405a32f6 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -42,10 +42,10 @@ struct GspMbox { impl GspMbox { /// Reads both mailboxes from the GSP falcon. - fn read(gsp_falcon: &Falcon, bar: Bar0<'_>) -> Self { + fn read(gsp_falcon: &Falcon<'_, GspEngine>) -> Self { Self { - mbox0: gsp_falcon.read_mailbox0(bar), - mbox1: gsp_falcon.read_mailbox1(bar), + mbox0: gsp_falcon.read_mailbox0(), + mbox1: gsp_falcon.read_mailbox1(), } } @@ -60,8 +60,7 @@ fn combined_addr(&self) -> u64 { /// either condition should stop the poll loop. fn lockdown_released_or_error( &self, - gsp_falcon: &Falcon, - bar: Bar0<'_>, + gsp_falcon: &Falcon<'_, GspEngine>, fmc_boot_params_addr: u64, ) -> bool { // GSP-FMC normally clears the boot parameters address from the mailboxes early during @@ -71,15 +70,14 @@ fn lockdown_released_or_error( return self.combined_addr() != fmc_boot_params_addr; } - !gsp_falcon.riscv_branch_privilege_lockdown(bar) + !gsp_falcon.riscv_branch_privilege_lockdown() } } /// Waits for GSP lockdown to be released after FSP Chain of Trust. fn wait_for_gsp_lockdown_release( dev: &device::Device, - bar: Bar0<'_>, - gsp_falcon: &Falcon, + gsp_falcon: &Falcon<'_, GspEngine>, fmc_boot_params_addr: u64, ) -> Result { dev_dbg!(dev, "Waiting for GSP lockdown release\n"); @@ -88,14 +86,14 @@ fn wait_for_gsp_lockdown_release( || { // While the PRIV target mask is still locked to FSP, GSP register and mailbox reads // are not meaningful. Wait until HWCFG2 says the CPU can read them. - Ok(match gsp_falcon.priv_target_mask_released(bar) { + Ok(match gsp_falcon.priv_target_mask_released() { false => None, - true => Some(GspMbox::read(gsp_falcon, bar)), + true => Some(GspMbox::read(gsp_falcon)), }) }, |mbox| match mbox { None => false, - Some(mbox) => mbox.lockdown_released_or_error(gsp_falcon, bar, fmc_boot_params_addr), + Some(mbox) => mbox.lockdown_released_or_error(gsp_falcon, fmc_boot_params_addr), }, Delta::from_millis(10), Delta::from_secs(30), @@ -122,13 +120,13 @@ impl UnloadBundle for FspUnloadBundle { fn run( &self, dev: &device::Device, - bar: Bar0<'_>, - gsp_falcon: &Falcon, - _sec2_falcon: &Falcon, + _bar: Bar0<'_>, + gsp_falcon: &Falcon<'_, GspEngine>, + _sec2_falcon: &Falcon<'_, Sec2>, ) -> Result { // GSP falcon does most of the work of resetting, so just wait for it to finish. read_poll_timeout( - || Ok(gsp_falcon.is_riscv_active(bar)), + || Ok(gsp_falcon.is_riscv_active()), |&active| !active, Delta::from_millis(10), Delta::from_secs(5), @@ -176,9 +174,9 @@ fn boot<'a>( false, )?; - fsp.boot_fmc(dev, bar, fb_layout, &args)?; + fsp.boot_fmc(dev, fb_layout, &args)?; - wait_for_gsp_lockdown_release(dev, bar, gsp_falcon, args.boot_params_dma_handle())?; + wait_for_gsp_lockdown_release(dev, gsp_falcon, args.boot_params_dma_handle())?; Ok(unload_guard) } diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index f8a8541704ee..ff71b45b5432 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -62,12 +62,11 @@ impl FwsecUnloadFirmware { /// Loads the FWSEC SB firmware, as well as its bootloader if `chipset` requires it. fn new( dev: &device::Device, - bar: Bar0<'_>, chipset: Chipset, bios: &Vbios, - gsp_falcon: &Falcon, + gsp_falcon: &Falcon<'_, GspEngine>, ) -> Result { - let fwsec_sb = FwsecFirmware::new(dev, gsp_falcon, bar, bios, FwsecCommand::Sb)?; + let fwsec_sb = FwsecFirmware::new(dev, gsp_falcon, bios, FwsecCommand::Sb)?; Ok(if chipset.needs_fwsec_bootloader() { Self::WithBl(FwsecFirmwareWithBl::new(fwsec_sb, dev, chipset)?) @@ -81,10 +80,10 @@ fn run( &self, dev: &device::Device, bar: Bar0<'_>, - gsp_falcon: &Falcon, + gsp_falcon: &Falcon<'_, GspEngine>, ) -> Result { match self { - Self::WithoutBl(fw) => fw.run(dev, gsp_falcon, bar), + Self::WithoutBl(fw) => fw.run(dev, gsp_falcon), Self::WithBl(fw) => fw.run(dev, gsp_falcon, bar), } } @@ -101,22 +100,20 @@ impl Sec2UnloadBundle { /// Load and prepare the resources required to properly reset the GSP after it has been stopped. fn build( dev: &device::Device, - bar: Bar0<'_>, chipset: Chipset, bios: &Vbios, - gsp_falcon: &Falcon, - sec2_falcon: &Falcon, + gsp_falcon: &Falcon<'_, GspEngine>, + sec2_falcon: &Falcon<'_, Sec2>, ) -> Result> { KBox::new( Self { - fwsec_sb: FwsecUnloadFirmware::new(dev, bar, chipset, bios, gsp_falcon)?, + fwsec_sb: FwsecUnloadFirmware::new(dev, chipset, bios, gsp_falcon)?, booter_unloader: BooterFirmware::new( dev, BooterKind::Unloader, chipset, FIRMWARE_VERSION, sec2_falcon, - bar, )?, }, GFP_KERNEL, @@ -131,8 +128,8 @@ fn run( &self, dev: &device::Device, bar: Bar0<'_>, - gsp_falcon: &Falcon, - sec2_falcon: &Falcon, + gsp_falcon: &Falcon<'_, GspEngine>, + sec2_falcon: &Falcon<'_, Sec2>, ) -> Result { // Run FWSEC-SB to reset the GSP falcon to its pre-libos state. // Log errors but keep going if it fails. @@ -148,13 +145,12 @@ fn run( return Ok(()); } - sec2_falcon.reset(bar)?; - sec2_falcon.load(dev, bar, &self.booter_unloader)?; + sec2_falcon.reset()?; + sec2_falcon.load(&self.booter_unloader)?; // Sentinel value to confirm that Booter Unloader has run. const MAILBOX_SENTINEL: u32 = 0xff; - let (mbox0, _) = - sec2_falcon.boot(bar, Some(MAILBOX_SENTINEL), Some(MAILBOX_SENTINEL))?; + let (mbox0, _) = sec2_falcon.boot(Some(MAILBOX_SENTINEL), Some(MAILBOX_SENTINEL))?; if mbox0 != 0 { dev_err!(dev, "Booter Unloader returned error 0x{:x}\n", mbox0); return Err(EINVAL); @@ -183,7 +179,7 @@ fn run( fn run_fwsec_frts( dev: &device::Device, chipset: Chipset, - falcon: &Falcon, + falcon: &Falcon<'_, GspEngine>, bar: Bar0<'_>, bios: &Vbios, fb_layout: &FbLayout, @@ -202,7 +198,6 @@ fn run_fwsec_frts( let fwsec_frts = FwsecFirmware::new( dev, falcon, - bar, bios, FwsecCommand::Frts { frts_addr: fb_layout.frts.start, @@ -216,7 +211,7 @@ fn run_fwsec_frts( fwsec_frts_bl.run(dev, falcon, bar)?; } else { // Load and run FWSEC-FRTS directly. - fwsec_frts.run(dev, falcon, bar)?; + fwsec_frts.run(dev, falcon)?; } // SCRATCH_E contains the error code for FWSEC-FRTS. @@ -286,18 +281,17 @@ fn boot<'a>( // // If the unload bundle creation fails, the GPU will need to be reset before the driver can // be probed again. - let unload_bundle = - Sec2UnloadBundle::build(dev, bar, chipset, &bios, gsp_falcon, sec2_falcon) - .inspect_err(|e| { - dev_warn!(dev, "Failed to prepare unload firmware: {:?}\n", e); - dev_warn!(dev, "The GSP won't be able to unload properly on unbind.\n"); - dev_warn!( - dev, - "The GPU will need to be reset before the driver can bind again.\n" - ); - }) - .ok() - .map(crate::gsp::UnloadBundle); + let unload_bundle = Sec2UnloadBundle::build(dev, chipset, &bios, gsp_falcon, sec2_falcon) + .inspect_err(|e| { + dev_warn!(dev, "Failed to prepare unload firmware: {:?}\n", e); + dev_warn!(dev, "The GSP won't be able to unload properly on unbind.\n"); + dev_warn!( + dev, + "The GPU will need to be reset before the driver can bind again.\n" + ); + }) + .ok() + .map(crate::gsp::UnloadBundle); // Wrap the unload bundle into a drop guard so it is automatically run upon failure. let unload_guard = @@ -308,13 +302,10 @@ fn boot<'a>( run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, fb_layout)?; } - gsp_falcon.reset(bar)?; + gsp_falcon.reset()?; let libos_handle = gsp.libos.dma_handle(); - let (mbox0, mbox1) = gsp_falcon.boot( - bar, - Some(libos_handle as u32), - Some((libos_handle >> 32) as u32), - )?; + let (mbox0, mbox1) = + gsp_falcon.boot(Some(libos_handle as u32), Some((libos_handle >> 32) as u32))?; dev_dbg!(dev, "GSP MBOX0: {:#x}, MBOX1: {:#x}\n", mbox0, mbox1); dev_dbg!( @@ -328,9 +319,8 @@ fn boot<'a>( chipset, FIRMWARE_VERSION, sec2_falcon, - bar, )? - .run(dev, bar, sec2_falcon, wpr_meta)?; + .run(dev, sec2_falcon, wpr_meta)?; Ok(unload_guard) } diff --git a/drivers/gpu/nova-core/gsp/sequencer.rs b/drivers/gpu/nova-core/gsp/sequencer.rs index e0850d21adca..13983d42b12b 100644 --- a/drivers/gpu/nova-core/gsp/sequencer.rs +++ b/drivers/gpu/nova-core/gsp/sequencer.rs @@ -133,9 +133,9 @@ pub(crate) struct GspSequencer<'a> { /// `Bar0` for register access. bar: Bar0<'a>, /// SEC2 falcon for core operations. - sec2_falcon: &'a Falcon, + sec2_falcon: &'a Falcon<'a, Sec2>, /// GSP falcon for core operations. - gsp_falcon: &'a Falcon, + gsp_falcon: &'a Falcon<'a, Gsp>, /// LibOS DMA handle address. libos_dma_handle: u64, /// Bootloader application version. @@ -213,16 +213,16 @@ fn run(&self, seq: &GspSequencer<'_>) -> Result { GspSeqCmd::DelayUs(cmd) => cmd.run(seq), GspSeqCmd::RegStore(cmd) => cmd.run(seq), GspSeqCmd::CoreReset => { - seq.gsp_falcon.reset(seq.bar)?; - seq.gsp_falcon.dma_reset(seq.bar); + seq.gsp_falcon.reset()?; + seq.gsp_falcon.dma_reset(); Ok(()) } GspSeqCmd::CoreStart => { - seq.gsp_falcon.start(seq.bar)?; + seq.gsp_falcon.start()?; Ok(()) } GspSeqCmd::CoreWaitForHalt => { - seq.gsp_falcon.wait_till_halted(seq.bar)?; + seq.gsp_falcon.wait_till_halted()?; Ok(()) } GspSeqCmd::CoreResume => { @@ -231,35 +231,32 @@ fn run(&self, seq: &GspSequencer<'_>) -> Result { // sequencer will start both. // Reset the GSP to prepare it for resuming. - seq.gsp_falcon.reset(seq.bar)?; + seq.gsp_falcon.reset()?; // Write the libOS DMA handle to GSP mailboxes. seq.gsp_falcon.write_mailboxes( - seq.bar, Some(seq.libos_dma_handle as u32), Some((seq.libos_dma_handle >> 32) as u32), ); // Start the SEC2 falcon which will trigger GSP-RM to resume on the GSP. - seq.sec2_falcon.start(seq.bar)?; + seq.sec2_falcon.start()?; // Poll until GSP-RM reload/resume has completed (up to 2 seconds). - seq.gsp_falcon - .check_reload_completed(seq.bar, Delta::from_secs(2))?; + seq.gsp_falcon.check_reload_completed(Delta::from_secs(2))?; // Verify SEC2 completed successfully by checking its mailbox for errors. - let mbox0 = seq.sec2_falcon.read_mailbox0(seq.bar); + let mbox0 = seq.sec2_falcon.read_mailbox0(); if mbox0 != 0 { dev_err!(seq.dev, "Sequencer: sec2 errors: {:?}\n", mbox0); return Err(EIO); } // Configure GSP with the bootloader version. - seq.gsp_falcon - .write_os_version(seq.bar, seq.bootloader_app_version); + seq.gsp_falcon.write_os_version(seq.bootloader_app_version); // Verify the GSP's RISC-V core is active indicating successful GSP boot. - if !seq.gsp_falcon.is_riscv_active(seq.bar) { + if !seq.gsp_falcon.is_riscv_active() { dev_err!(seq.dev, "Sequencer: RISC-V core is not active\n"); return Err(EIO); } @@ -345,9 +342,9 @@ pub(crate) struct GspSequencerParams<'a> { /// LibOS DMA handle address. pub(crate) libos_dma_handle: u64, /// GSP falcon for core operations. - pub(crate) gsp_falcon: &'a Falcon, + pub(crate) gsp_falcon: &'a Falcon<'a, Gsp>, /// SEC2 falcon for core operations. - pub(crate) sec2_falcon: &'a Falcon, + pub(crate) sec2_falcon: &'a Falcon<'a, Sec2>, /// Device for logging. pub(crate) dev: &'a device::Device, /// BAR0 for register access. From 431f10ba13a964c146ae05728e42e4074bf735ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Antinori?= Date: Mon, 29 Jun 2026 11:20:05 -0300 Subject: [PATCH 034/137] gpu: nova-core: vbios: parse structs via zerocopy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the unsafe `kernel::transmute::FromBytes` trait implementation for the `FalconUCodeDescV3`, `PcirStruct`, `BitHeader`, `BitToken`, `NpdeStruct`, `PciRomHeader`, `PmuLookupTableEntry` and `PmuLookupTableHeader` structs with the derivable `zerocopy::FromBytes` trait. This change eliminates the manual unsafe implementations in favor of a derivable trait. When this trait is derived, validity checks are performed at compile time to ensure that the type can safely implement `FromBytes`. Suggested-by: Miguel Ojeda Link: https://github.com/Rust-for-Linux/linux/issues/1241 Signed-off-by: Nicolás Antinori Link: https://patch.msgid.link/20260629142007.269873-1-nico.antinori.7@gmail.com [acourbot: add `vbios:` prefix to commit title.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/firmware.rs | 6 +-- drivers/gpu/nova-core/vbios.rs | 64 +++++++++---------------------- 2 files changed, 20 insertions(+), 50 deletions(-) diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index 80e948bf7511..a94820a3b335 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -88,7 +88,7 @@ pub(crate) struct FalconUCodeDescV2 { /// Structure used to describe some firmwares, notably FWSEC-FRTS. #[repr(C)] -#[derive(Debug, Clone)] +#[derive(Debug, Clone, FromBytes)] pub(crate) struct FalconUCodeDescV3 { /// Header defined by `NV_BIT_FALCON_UCODE_DESC_HEADER_VDESC*` in OpenRM. hdr: u32, @@ -119,10 +119,6 @@ pub(crate) struct FalconUCodeDescV3 { _reserved: u16, } -// SAFETY: all bit patterns are valid for this type, and it doesn't use -// interior mutability. -unsafe impl FromBytes for FalconUCodeDescV3 {} - /// Enum wrapping the different versions of Falcon microcode descriptors. /// /// This allows handling both V2 and V3 descriptor formats through a diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index c6e6bfcd6a1f..c03650ee5226 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -13,11 +13,8 @@ register, sizes::SZ_4K, sync::aref::ARef, - transmute::FromBytes, }; -use zerocopy::FromBytes as _; - use crate::{ driver::Bar0, firmware::{ @@ -359,7 +356,7 @@ pub(crate) fn fwsec_image(&self) -> &FwSecBiosImage { } /// PCI Data Structure as defined in PCI Firmware Specification -#[derive(Debug, Clone)] +#[derive(Debug, Clone, FromBytes)] #[repr(C)] struct PcirStruct { /// PCI Data Structure signature ("PCIR" or "NPDS") @@ -388,15 +385,12 @@ struct PcirStruct { max_runtime_image_len: u16, } -// SAFETY: all bit patterns are valid for `PcirStruct`. -unsafe impl FromBytes for PcirStruct {} - impl PcirStruct { /// The bit in `last_image` that indicates the last image. const LAST_IMAGE_BIT_MASK: u8 = 0x80; fn new(dev: &device::Device, data: &[u8]) -> Result { - let (pcir, _) = PcirStruct::from_bytes_copy_prefix(data).ok_or(EINVAL)?; + let (pcir, _) = PcirStruct::read_from_prefix(data).map_err(|_| EINVAL)?; // Signature should be "PCIR" (0x52494350) or "NPDS" (0x5344504e). if &pcir.signature != b"PCIR" && &pcir.signature != b"NPDS" { @@ -432,7 +426,7 @@ fn image_size_bytes(&self) -> usize { /// This is the head of the BIT table, that is used to locate the Falcon data. The BIT table (with /// its header) is in the [`PciAtBiosImage`] and the falcon data it is pointing to is in the /// [`FwSecBiosImage`]. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, FromBytes)] #[repr(C)] struct BitHeader { /// 0h: BIT Header Identifier (BMP=0x7FFF/BIT=0xB8FF) @@ -451,12 +445,9 @@ struct BitHeader { checksum: u8, } -// SAFETY: all bit patterns are valid for `BitHeader`. -unsafe impl FromBytes for BitHeader {} - impl BitHeader { fn new(data: &[u8]) -> Result { - let (header, _) = BitHeader::from_bytes_copy_prefix(data).ok_or(EINVAL)?; + let (header, _) = BitHeader::read_from_prefix(data).map_err(|_| EINVAL)?; // Check header ID and signature if header.id != 0xB8FF || &header.signature != b"BIT\0" { @@ -468,7 +459,7 @@ fn new(data: &[u8]) -> Result { } /// BIT Token Entry: Records in the BIT table followed by the BIT header. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, FromBytes)] #[repr(C)] struct BitToken { /// 00h: Token identifier @@ -481,9 +472,6 @@ struct BitToken { data_offset: u16, } -// SAFETY: all bit patterns are valid for `BitToken`. -unsafe impl FromBytes for BitToken {} - impl BitToken { /// BIT token ID for Falcon data. const ID_FALCON_DATA: u8 = 0x70; @@ -508,7 +496,7 @@ fn from_id(image: &PciAtBiosImage, token_id: u8) -> Result { .and_then(|data| data.get(..entry_size)) .ok_or(EINVAL)?; - let (token, _) = BitToken::from_bytes_copy_prefix(entry).ok_or(EINVAL)?; + let (token, _) = BitToken::read_from_prefix(entry).map_err(|_| EINVAL)?; // Check if this token has the requested ID if token.id == token_id { @@ -525,7 +513,7 @@ fn from_id(image: &PciAtBiosImage, token_id: u8) -> Result { /// /// This header is at the beginning of every image in the set of images in the ROM. It contains a /// pointer to the PCI Data Structure which describes the image. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, FromBytes)] #[repr(C)] struct PciRomHeader { /// 00h: Signature (0xAA55) @@ -536,13 +524,10 @@ struct PciRomHeader { pci_data_struct_offset: u16, } -// SAFETY: all bit patterns are valid for `PciRomHeader`. -unsafe impl FromBytes for PciRomHeader {} - impl PciRomHeader { fn new(dev: &device::Device, data: &[u8]) -> Result { - let (rom_header, _) = PciRomHeader::from_bytes_copy_prefix(data) - .ok_or(EINVAL) + let (rom_header, _) = PciRomHeader::read_from_prefix(data) + .map_err(|_| EINVAL) .inspect_err(|_| dev_err!(dev, "Not enough data for ROM header\n"))?; // Check for valid ROM signatures. @@ -564,7 +549,7 @@ fn new(dev: &device::Device, data: &[u8]) -> Result { /// PCI Data Structure. It contains some fields that are redundant with the PCI Data Structure, but /// are needed for traversing the BIOS images. It is expected to be present in all BIOS images /// except for NBSI images. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, FromBytes)] #[repr(C)] struct NpdeStruct { /// 00h: Signature ("NPDE") @@ -579,15 +564,12 @@ struct NpdeStruct { last_image: u8, } -// SAFETY: all bit patterns are valid for `NpdeStruct`. -unsafe impl FromBytes for NpdeStruct {} - impl NpdeStruct { /// The bit in `last_image` that indicates the last image. const LAST_IMAGE_BIT_MASK: u8 = 0x80; fn new(dev: &device::Device, data: &[u8]) -> Option { - let (npde, _) = NpdeStruct::from_bytes_copy_prefix(data)?; + let (npde, _) = NpdeStruct::read_from_prefix(data).ok()?; // Signature should be "NPDE" (0x4544504E). if &npde.signature != b"NPDE" { @@ -784,7 +766,7 @@ fn falcon_data_offset(&self, dev: &device::Device) -> Result { let data = &self.base.data; let (ptr, _) = data .get(offset..) - .and_then(u32::from_bytes_copy_prefix) + .and_then(|p| u32::read_from_prefix(p).ok()) .ok_or(EINVAL)?; usize::from_safe_cast(ptr) @@ -814,6 +796,7 @@ fn try_from(base: BiosImage) -> Result { /// The [`PmuLookupTableEntry`] structure is a single entry in the [`PmuLookupTable`]. /// /// See the [`PmuLookupTable`] description for more information. +#[derive(FromBytes)] #[repr(C, packed)] struct PmuLookupTableEntry { application_id: u8, @@ -821,9 +804,6 @@ struct PmuLookupTableEntry { data: u32, } -// SAFETY: all bit patterns are valid for `PmuLookupTableEntry`. -unsafe impl FromBytes for PmuLookupTableEntry {} - impl PmuLookupTableEntry { /// PMU lookup table application ID for firmware security license ucode. #[expect(dead_code)] @@ -836,6 +816,7 @@ impl PmuLookupTableEntry { } #[repr(C)] +#[derive(FromBytes)] struct PmuLookupTableHeader { version: u8, header_len: u8, @@ -843,9 +824,6 @@ struct PmuLookupTableHeader { entry_count: u8, } -// SAFETY: all bit patterns are valid for `PmuLookupTableHeader`. -unsafe impl FromBytes for PmuLookupTableHeader {} - /// The [`PmuLookupTableEntry`] structure is used to find the [`PmuLookupTableEntry`] for a given /// application ID. /// @@ -857,7 +835,7 @@ struct PmuLookupTable { impl PmuLookupTable { fn new(dev: &device::Device, data: &[u8]) -> Result { - let (header, _) = PmuLookupTableHeader::from_bytes_copy_prefix(data).ok_or(EINVAL)?; + let (header, _) = PmuLookupTableHeader::read_from_prefix(data).map_err(|_| EINVAL)?; let header_len = usize::from(header.header_len); let entry_len = usize::from(header.entry_len); @@ -872,8 +850,8 @@ fn new(dev: &device::Device, data: &[u8]) -> Result { let mut entries = KVVec::with_capacity(entry_count, GFP_KERNEL)?; for i in 0..entry_count { - let (entry, _) = PmuLookupTableEntry::from_bytes_copy_prefix(&data[i * entry_len..]) - .ok_or(EINVAL)?; + let (entry, _) = PmuLookupTableEntry::read_from_prefix(&data[i * entry_len..]) + .map_err(|_| EINVAL)?; entries.push(entry, GFP_KERNEL)?; } @@ -929,15 +907,11 @@ pub(crate) fn header(&self) -> Result { let ver = data.get(1).copied().ok_or(EINVAL)?; match ver { 2 => { - let v2 = FalconUCodeDescV2::read_from_prefix(data) - .map_err(|_| EINVAL)? - .0; + let (v2, _) = FalconUCodeDescV2::read_from_prefix(data).map_err(|_| EINVAL)?; Ok(FalconUCodeDesc::V2(v2)) } 3 => { - let v3 = FalconUCodeDescV3::from_bytes_copy_prefix(data) - .ok_or(EINVAL)? - .0; + let (v3, _) = FalconUCodeDescV3::read_from_prefix(data).map_err(|_| EINVAL)?; Ok(FalconUCodeDesc::V3(v3)) } _ => { From 24d2581fd911d34f88153af59d3b0d6bc5f07adf Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Fri, 12 Jun 2026 13:34:00 +0100 Subject: [PATCH 035/137] gpu: nova-core: remove `#[allow(non_snake_case)]` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit 5423ef9d4db8 ("rust: pin-init: internal: suppress `non_snake_case` lint in `[pin_]init!`"), mere use of the non-snake-case identifiers would not cause the warning to be generated. Thus remove these allows. Signed-off-by: Gary Guo Acked-by: Alexandre Courbot Reviewed-by: Onur Özkan Link: https://patch.msgid.link/20260612123401.2684025-1-gary@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/gsp/fw.rs | 8 -------- drivers/gpu/nova-core/gsp/fw/commands.rs | 2 -- 2 files changed, 10 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 4db0cfa4dc4d..d96ea0b216b4 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -219,7 +219,6 @@ pub(crate) fn new<'a>( gsp_firmware: &'a GspFirmware, fb_layout: &'a FbLayout, ) -> impl Init + 'a { - #[allow(non_snake_case)] let init_inner = init!(bindings::GspFwWprMeta { // CAST: we want to store the bits of `GSP_FW_WPR_META_MAGIC` unmodified. magic: bindings::GSP_FW_WPR_META_MAGIC as u64, @@ -674,7 +673,6 @@ fn id8(name: &str) -> u64 { u64::from_ne_bytes(bytes) } - #[allow(non_snake_case)] let init_inner = init!(bindings::LibosMemoryRegionInitArgument { id8: id8(name), pa: obj.dma_handle(), @@ -793,7 +791,6 @@ impl GspMsgElement { /// * `sequence` - Sequence number of the message. /// * `cmd_size` - Size of the command (not including the message element), in bytes. /// * `function` - Function of the message. - #[allow(non_snake_case)] pub(crate) fn init( sequence: u32, cmd_size: usize, @@ -876,7 +873,6 @@ pub(crate) struct GspArgumentsCached { impl GspArgumentsCached { /// Creates the arguments for starting the GSP up using `cmdq` as its command queue. pub(crate) fn new(cmdq: &Cmdq) -> impl Init + '_ { - #[allow(non_snake_case)] let init_inner = init!(bindings::GSP_ARGUMENTS_CACHED { messageQueueInitArguments <- MessageQueueInitArguments::new(cmdq), bDmemStack: 1, @@ -923,7 +919,6 @@ unsafe impl FromBytes for GspArgumentsPadded {} impl MessageQueueInitArguments { /// Creates a new init arguments structure for `cmdq`. - #[allow(non_snake_case)] fn new(cmdq: &Cmdq) -> impl Init + '_ { init!(MessageQueueInitArguments { sharedMemPhysAddr: cmdq.dma_handle, @@ -947,7 +942,6 @@ pub(crate) enum GspDmaTarget { impl GspAcrBootGspRmParams { fn new(target: GspDmaTarget, wpr_meta_addr: u64) -> impl Init { - #[allow(non_snake_case)] let params = init!(Self { target: target as u32, gspRmDescSize: num::usize_into_u32::<{ size_of::() }>(), @@ -966,7 +960,6 @@ fn new(target: GspDmaTarget, wpr_meta_addr: u64) -> impl Init { impl GspRmParams { fn new(target: GspDmaTarget, libos_addr: u64) -> impl Init { - #[allow(non_snake_case)] let params = init!(Self { target: target as u32, bootArgsOffset: libos_addr, @@ -986,7 +979,6 @@ unsafe impl FromBytes for GspFmcBootParams {} impl GspFmcBootParams { pub(crate) fn new(wpr_meta_addr: u64, libos_addr: u64) -> impl Init { - #[allow(non_snake_case)] let init = init!(Self { // Blackwell FSP obtains WPR info from other sources, so // wprCarveoutOffset and wprCarveoutSize are left zero. diff --git a/drivers/gpu/nova-core/gsp/fw/commands.rs b/drivers/gpu/nova-core/gsp/fw/commands.rs index ebdc12bcd4e3..6dc31d1bf5ae 100644 --- a/drivers/gpu/nova-core/gsp/fw/commands.rs +++ b/drivers/gpu/nova-core/gsp/fw/commands.rs @@ -30,7 +30,6 @@ pub(crate) struct GspSetSystemInfo { impl GspSetSystemInfo { /// Returns an in-place initializer for the `GspSetSystemInfo` command. - #[allow(non_snake_case)] pub(crate) fn init<'a>( dev: &'a pci::Device, chipset: Chipset, @@ -102,7 +101,6 @@ pub(crate) struct PackedRegistryTable { } impl PackedRegistryTable { - #[allow(non_snake_case)] pub(crate) fn init(num_entries: u32, size: u32) -> impl Init { type InnerPackedRegistryTable = bindings::PACKED_REGISTRY_TABLE; let init_inner = init!(InnerPackedRegistryTable { From 064375c89bd100809e04f6cfb01f40bca3c20af6 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 1 Jul 2026 09:15:30 +0900 Subject: [PATCH 036/137] gpu: nova-core: convert to kernel bitfield macro Replace uses of the Nova-local `bitfield!` macro with the kernel one. Reviewed-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260701-nova-bitfield-v2-1-2e949bf1836c@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fsp.rs | 3 +- drivers/gpu/nova-core/gsp/fw.rs | 11 ++-- drivers/gpu/nova-core/mctp.rs | 86 +++++++++++++++--------------- drivers/gpu/nova-core/nova_core.rs | 3 -- 4 files changed, 51 insertions(+), 52 deletions(-) diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index 574e1627e63c..f0c595175c9c 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -11,6 +11,7 @@ device, dma::Coherent, io::poll::read_poll_timeout, + num::TryIntoBounded, prelude::*, ptr::{ Alignable, @@ -300,7 +301,7 @@ fn send_sync_fsp(&mut self, dev: &device::Device, msg: &M) -> Result return Err(EIO); } - if command_nvdm_type != u8::from(M::NVDM_TYPE).into() { + if command_nvdm_type.try_into_bounded() != Some(M::NVDM_TYPE.into()) { dev_err!( dev, "Expected NVDM type {:?} in reply, got {:#x}\n", diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index d96ea0b216b4..2590931262af 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -10,6 +10,7 @@ use core::ops::Range; use kernel::{ + bitfield, dma::Coherent, prelude::*, ptr::{ @@ -740,8 +741,8 @@ unsafe impl AsBytes for MsgqRxHeader {} bitfield! { struct MsgHeaderVersion(u32) { - 31:24 major as u8; - 23:16 minor as u8; + 31:24 major; + 23:16 minor; } } @@ -750,9 +751,9 @@ impl MsgHeaderVersion { const MINOR_TOT: u8 = 0; fn new() -> Self { - Self::default() - .set_major(Self::MAJOR_TOT) - .set_minor(Self::MINOR_TOT) + Self::zeroed() + .with_major(Self::MAJOR_TOT) + .with_minor(Self::MINOR_TOT) } } diff --git a/drivers/gpu/nova-core/mctp.rs b/drivers/gpu/nova-core/mctp.rs index 482786e07bc7..acc2abbd4b0c 100644 --- a/drivers/gpu/nova-core/mctp.rs +++ b/drivers/gpu/nova-core/mctp.rs @@ -7,55 +7,51 @@ //! Data Model) messages between the kernel driver and GPU firmware processors //! such as FSP and GSP. -use kernel::pci::Vendor; +use kernel::{ + bitfield, + pci::Vendor, + prelude::*, // +}; -/// NVDM message type identifiers carried over MCTP. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -#[repr(u8)] -pub(crate) enum NvdmType { - #[default] - /// Chain of Trust boot message. - Cot = 0x14, - /// FSP command response. - FspResponse = 0x15, -} +use crate::{ + bounded_enum, + num, // +}; -impl TryFrom for NvdmType { - type Error = u8; - - fn try_from(value: u8) -> Result { - match value { - x if x == u8::from(Self::Cot) => Ok(Self::Cot), - x if x == u8::from(Self::FspResponse) => Ok(Self::FspResponse), - _ => Err(value), - } - } -} - -impl From for u8 { - fn from(value: NvdmType) -> Self { - value as u8 +bounded_enum! { + /// NVDM message type identifiers carried over MCTP. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(crate) enum NvdmType with TryFrom> { + /// Chain of Trust boot message. + Cot = 0x14, + /// FSP command response. + FspResponse = 0x15, } } bitfield! { - pub(crate) struct MctpHeader(u32), "MCTP transport header for NVIDIA firmware messages." { - 31:31 som as bool, "Start-of-message bit."; - 30:30 eom as bool, "End-of-message bit."; - 29:28 seq as u8, "Packet sequence number."; - 23:16 seid as u8, "Source endpoint ID."; + /// MCTP transport header for NVIDIA firmware messages. + pub(crate) struct MctpHeader(u32) { + /// Start-of-message bit. + 31:31 som; + /// End-of-message bit. + 30:30 eom; + /// Packet sequence number. + 29:28 seq; + /// Source endpoint ID. + 23:16 seid; } } impl MctpHeader { /// Builds a single-packet MCTP header (`SOM=1`, `EOM=1`, `SEQ=0`, `SEID=0`). pub(crate) fn single_packet() -> Self { - Self::default().set_som(true).set_eom(true) + Self::zeroed().with_som(true).with_eom(true) } /// Returns whether this is a complete single-packet message (`SOM=1` and `EOM=1`). pub(crate) fn is_single_packet(self) -> bool { - self.som() && self.eom() + self.som().into_bool() && self.eom().into_bool() } } @@ -63,26 +59,30 @@ pub(crate) fn is_single_packet(self) -> bool { const MSG_TYPE_VENDOR_PCI: u8 = 0x7e; bitfield! { - pub(crate) struct NvdmHeader(u32), "NVIDIA Vendor-Defined Message header over MCTP." { - 31:24 nvdm_type as u8 ?=> NvdmType, "NVDM message type."; - 23:8 vendor_id as u16, "PCI vendor ID."; - 6:0 msg_type as u8, "MCTP vendor-defined message type."; + /// NVIDIA Vendor-Defined Message header over MCTP. + pub(crate) struct NvdmHeader(u32) { + /// NVDM message type. + 31:24 nvdm_type ?=> NvdmType; + /// PCI vendor ID. + 23:8 vendor_id; + /// MCTP vendor-defined message type. + 6:0 msg_type; } } impl NvdmHeader { /// Builds an NVDM header for the given message type. pub(crate) fn new(nvdm_type: NvdmType) -> Self { - Self::default() - .set_msg_type(MSG_TYPE_VENDOR_PCI) - .set_vendor_id(Vendor::NVIDIA.as_raw()) - .set_nvdm_type(nvdm_type) + Self::zeroed() + .with_const_msg_type::<{ num::u8_as_u32(MSG_TYPE_VENDOR_PCI) }>() + .with_vendor_id(Vendor::NVIDIA.as_raw()) + .with_nvdm_type(nvdm_type) } /// Validates this header against the expected NVIDIA NVDM format and type. pub(crate) fn validate(self, expected_type: NvdmType) -> bool { - self.msg_type() == MSG_TYPE_VENDOR_PCI - && self.vendor_id() == Vendor::NVIDIA.as_raw() + u8::from(self.msg_type()) == MSG_TYPE_VENDOR_PCI + && u16::from(self.vendor_id()) == Vendor::NVIDIA.as_raw() && matches!(self.nvdm_type(), Ok(nvdm_type) if nvdm_type == expected_type) } } diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 735b8e17c6b6..a61406ba5c0b 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -10,9 +10,6 @@ InPlaceModule, // }; -#[macro_use] -mod bitfield; - mod driver; mod falcon; mod fb; From a73a398a68ca9b9e5116a617562471f16b8310c4 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 1 Jul 2026 09:15:31 +0900 Subject: [PATCH 037/137] gpu: nova-core: remove local bitfield macro This module is now orphaned code, superseded by a kernel-global implementation, so remove it. Reviewed-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260701-nova-bitfield-v2-2-2e949bf1836c@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/bitfield.rs | 329 ------------------------------ 1 file changed, 329 deletions(-) delete mode 100644 drivers/gpu/nova-core/bitfield.rs diff --git a/drivers/gpu/nova-core/bitfield.rs b/drivers/gpu/nova-core/bitfield.rs deleted file mode 100644 index 660c3911402d..000000000000 --- a/drivers/gpu/nova-core/bitfield.rs +++ /dev/null @@ -1,329 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -//! Bitfield library for Rust structures -//! -//! Support for defining bitfields in Rust structures. Also used by the [`register!`] macro. - -/// Defines a struct with accessors to access bits within an inner unsigned integer. -/// -/// # Syntax -/// -/// ```rust -/// use nova_core::bitfield; -/// -/// #[derive(Debug, Clone, Copy, Default)] -/// enum Mode { -/// #[default] -/// Low = 0, -/// High = 1, -/// Auto = 2, -/// } -/// -/// impl TryFrom for Mode { -/// type Error = u8; -/// fn try_from(value: u8) -> Result { -/// match value { -/// 0 => Ok(Mode::Low), -/// 1 => Ok(Mode::High), -/// 2 => Ok(Mode::Auto), -/// _ => Err(value), -/// } -/// } -/// } -/// -/// impl From for u8 { -/// fn from(mode: Mode) -> u8 { -/// mode as u8 -/// } -/// } -/// -/// #[derive(Debug, Clone, Copy, Default)] -/// enum State { -/// #[default] -/// Inactive = 0, -/// Active = 1, -/// } -/// -/// impl From for State { -/// fn from(value: bool) -> Self { -/// if value { State::Active } else { State::Inactive } -/// } -/// } -/// -/// impl From for bool { -/// fn from(state: State) -> bool { -/// match state { -/// State::Inactive => false, -/// State::Active => true, -/// } -/// } -/// } -/// -/// bitfield! { -/// pub struct ControlReg(u32) { -/// 7:7 state as bool => State; -/// 3:0 mode as u8 ?=> Mode; -/// } -/// } -/// ``` -/// -/// This generates a struct with: -/// - Field accessors: `mode()`, `state()`, etc. -/// - Field setters: `set_mode()`, `set_state()`, etc. (supports chaining with builder pattern). -/// Note that the compiler will error out if the size of the setter's arg exceeds the -/// struct's storage size. -/// - Debug and Default implementations. -/// -/// Note: Field accessors and setters inherit the same visibility as the struct itself. -/// In the example above, both `mode()` and `set_mode()` methods will be `pub`. -/// -/// Fields are defined as follows: -/// -/// - `as ` simply returns the field value casted to , typically `u32`, `u16`, `u8` or -/// `bool`. Note that `bool` fields must have a range of 1 bit. -/// - `as => ` calls ``'s `From::<>` implementation and returns -/// the result. -/// - `as ?=> ` calls ``'s `TryFrom::<>` implementation -/// and returns the result. This is useful with fields for which not all values are valid. -macro_rules! bitfield { - // Main entry point - defines the bitfield struct with fields - ($vis:vis struct $name:ident($storage:ty) $(, $comment:literal)? { $($fields:tt)* }) => { - bitfield!(@core $vis $name $storage $(, $comment)? { $($fields)* }); - }; - - // All rules below are helpers. - - // Defines the wrapper `$name` type, as well as its relevant implementations (`Debug`, - // `Default`, and conversion to the value type) and field accessor methods. - (@core $vis:vis $name:ident $storage:ty $(, $comment:literal)? { $($fields:tt)* }) => { - $( - #[doc=$comment] - )? - #[repr(transparent)] - #[derive(Clone, Copy)] - $vis struct $name($storage); - - impl ::core::convert::From<$name> for $storage { - fn from(val: $name) -> $storage { - val.0 - } - } - - bitfield!(@fields_dispatcher $vis $name $storage { $($fields)* }); - }; - - // Captures the fields and passes them to all the implementers that require field information. - // - // Used to simplify the matching rules for implementers, so they don't need to match the entire - // complex fields rule even though they only make use of part of it. - (@fields_dispatcher $vis:vis $name:ident $storage:ty { - $($hi:tt:$lo:tt $field:ident as $type:tt - $(?=> $try_into_type:ty)? - $(=> $into_type:ty)? - $(, $comment:literal)? - ; - )* - } - ) => { - bitfield!(@field_accessors $vis $name $storage { - $( - $hi:$lo $field as $type - $(?=> $try_into_type)? - $(=> $into_type)? - $(, $comment)? - ; - )* - }); - bitfield!(@debug $name { $($field;)* }); - bitfield!(@default $name { $($field;)* }); - }; - - // Defines all the field getter/setter methods for `$name`. - ( - @field_accessors $vis:vis $name:ident $storage:ty { - $($hi:tt:$lo:tt $field:ident as $type:tt - $(?=> $try_into_type:ty)? - $(=> $into_type:ty)? - $(, $comment:literal)? - ; - )* - } - ) => { - $( - bitfield!(@check_field_bounds $hi:$lo $field as $type); - )* - - #[allow(dead_code)] - impl $name { - $( - bitfield!(@field_accessor $vis $name $storage, $hi:$lo $field as $type - $(?=> $try_into_type)? - $(=> $into_type)? - $(, $comment)? - ; - ); - )* - } - }; - - // Boolean fields must have `$hi == $lo`. - (@check_field_bounds $hi:tt:$lo:tt $field:ident as bool) => { - #[allow(clippy::eq_op)] - const _: () = { - ::kernel::build_assert::build_assert!( - $hi == $lo, - concat!("boolean field `", stringify!($field), "` covers more than one bit") - ); - }; - }; - - // Non-boolean fields must have `$hi >= $lo`. - (@check_field_bounds $hi:tt:$lo:tt $field:ident as $type:tt) => { - #[allow(clippy::eq_op)] - const _: () = { - ::kernel::build_assert::build_assert!( - $hi >= $lo, - concat!("field `", stringify!($field), "`'s MSB is smaller than its LSB") - ); - }; - }; - - // Catches fields defined as `bool` and convert them into a boolean value. - ( - @field_accessor $vis:vis $name:ident $storage:ty, $hi:tt:$lo:tt $field:ident as bool - => $into_type:ty $(, $comment:literal)?; - ) => { - bitfield!( - @leaf_accessor $vis $name $storage, $hi:$lo $field - { |f| <$into_type>::from(f != 0) } - bool $into_type => $into_type $(, $comment)?; - ); - }; - - // Shortcut for fields defined as `bool` without the `=>` syntax. - ( - @field_accessor $vis:vis $name:ident $storage:ty, $hi:tt:$lo:tt $field:ident as bool - $(, $comment:literal)?; - ) => { - bitfield!( - @field_accessor $vis $name $storage, $hi:$lo $field as bool => bool $(, $comment)?; - ); - }; - - // Catches the `?=>` syntax for non-boolean fields. - ( - @field_accessor $vis:vis $name:ident $storage:ty, $hi:tt:$lo:tt $field:ident as $type:tt - ?=> $try_into_type:ty $(, $comment:literal)?; - ) => { - bitfield!(@leaf_accessor $vis $name $storage, $hi:$lo $field - { |f| <$try_into_type>::try_from(f as $type) } $type $try_into_type => - ::core::result::Result< - $try_into_type, - <$try_into_type as ::core::convert::TryFrom<$type>>::Error - > - $(, $comment)?;); - }; - - // Catches the `=>` syntax for non-boolean fields. - ( - @field_accessor $vis:vis $name:ident $storage:ty, $hi:tt:$lo:tt $field:ident as $type:tt - => $into_type:ty $(, $comment:literal)?; - ) => { - bitfield!(@leaf_accessor $vis $name $storage, $hi:$lo $field - { |f| <$into_type>::from(f as $type) } $type $into_type => $into_type $(, $comment)?;); - }; - - // Shortcut for non-boolean fields defined without the `=>` or `?=>` syntax. - ( - @field_accessor $vis:vis $name:ident $storage:ty, $hi:tt:$lo:tt $field:ident as $type:tt - $(, $comment:literal)?; - ) => { - bitfield!( - @field_accessor $vis $name $storage, $hi:$lo $field as $type => $type $(, $comment)?; - ); - }; - - // Generates the accessor methods for a single field. - ( - @leaf_accessor $vis:vis $name:ident $storage:ty, $hi:tt:$lo:tt $field:ident - { $process:expr } $prim_type:tt $to_type:ty => $res_type:ty $(, $comment:literal)?; - ) => { - ::kernel::macros::paste!( - const [<$field:upper _RANGE>]: ::core::ops::RangeInclusive = $lo..=$hi; - const [<$field:upper _MASK>]: $storage = { - // Generate mask for shifting - match ::core::mem::size_of::<$storage>() { - 1 => ::kernel::bits::genmask_u8($lo..=$hi) as $storage, - 2 => ::kernel::bits::genmask_u16($lo..=$hi) as $storage, - 4 => ::kernel::bits::genmask_u32($lo..=$hi) as $storage, - 8 => ::kernel::bits::genmask_u64($lo..=$hi) as $storage, - _ => ::kernel::build_error!("Unsupported storage type size") - } - }; - const [<$field:upper _SHIFT>]: u32 = $lo; - ); - - $( - #[doc="Returns the value of this field:"] - #[doc=$comment] - )? - #[inline(always)] - $vis fn $field(self) -> $res_type { - ::kernel::macros::paste!( - const MASK: $storage = $name::[<$field:upper _MASK>]; - const SHIFT: u32 = $name::[<$field:upper _SHIFT>]; - ); - let field = ((self.0 & MASK) >> SHIFT); - - $process(field) - } - - ::kernel::macros::paste!( - $( - #[doc="Sets the value of this field:"] - #[doc=$comment] - )? - #[inline(always)] - $vis fn [](mut self, value: $to_type) -> Self { - const MASK: $storage = $name::[<$field:upper _MASK>]; - const SHIFT: u32 = $name::[<$field:upper _SHIFT>]; - let value = ($storage::from($prim_type::from(value)) << SHIFT) & MASK; - self.0 = (self.0 & !MASK) | value; - - self - } - ); - }; - - // Generates the `Debug` implementation for `$name`. - (@debug $name:ident { $($field:ident;)* }) => { - impl ::kernel::fmt::Debug for $name { - fn fmt(&self, f: &mut ::kernel::fmt::Formatter<'_>) -> ::kernel::fmt::Result { - f.debug_struct(stringify!($name)) - .field("", &::kernel::prelude::fmt!("{:#x}", &self.0)) - $( - .field(stringify!($field), &self.$field()) - )* - .finish() - } - } - }; - - // Generates the `Default` implementation for `$name`. - (@default $name:ident { $($field:ident;)* }) => { - /// Returns a value for the bitfield where all fields are set to their default value. - impl ::core::default::Default for $name { - fn default() -> Self { - let value = Self(Default::default()); - - ::kernel::macros::paste!( - $( - let value = value.[](Default::default()); - )* - ); - - value - } - } - }; -} From 78900738d981c97aad929e33620340d0d9bcfc82 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 1 Jul 2026 15:40:51 +0900 Subject: [PATCH 038/137] gpu: nova: fix rust-analyzer generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rust-analyzer generator script recognizes a crate when its corresponding `.o` appears in the Rust source file's immediate `Makefile` or `Kbuild` file. Commit ca524e273c43 ("gpu: build nova-core and nova-drm from drivers/gpu/Makefile") moves the build rules for `nova-core` and `nova-drm` into `drivers/gpu/Makefile`, which results in the generator script ignoring these crates. Fix this by naming the crates' module as a comment in their respective `Makefile`; this is enough for the script to pick them up and restore `rust-analyzer` functionality on them. Fixes: ca524e273c43 ("gpu: build nova-core and nova-drm from drivers/gpu/Makefile") Signed-off-by: Alexandre Courbot Tested-by: Timur Tabi Reviewed-by: Onur Özkan Link: https://patch.msgid.link/20260701-nova-rust-analyzer-v1-1-5209f486f10d@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/Makefile | 1 + drivers/gpu/nova-core/Makefile | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/gpu/drm/nova/Makefile b/drivers/gpu/drm/nova/Makefile index b9fad3956358..6355f7502c48 100644 --- a/drivers/gpu/drm/nova/Makefile +++ b/drivers/gpu/drm/nova/Makefile @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 # nova-drm is built from drivers/gpu/Makefile. +# nova.o (rust-analyzer marker - DO NOT REMOVE). diff --git a/drivers/gpu/nova-core/Makefile b/drivers/gpu/nova-core/Makefile index 4c15729704a1..216329760a5b 100644 --- a/drivers/gpu/nova-core/Makefile +++ b/drivers/gpu/nova-core/Makefile @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 # nova-core is built from drivers/gpu/Makefile. +# nova_core.o (rust-analyzer marker - DO NOT REMOVE). From 2493c4416542ce1759d6dcd0a2b1f0ed18ab8c35 Mon Sep 17 00:00:00 2001 From: Zhi Wang Date: Wed, 1 Jul 2026 09:26:17 +0300 Subject: [PATCH 039/137] gpu: nova-core: fsp: rename FSP response header type FSP message handling currently uses FspResponse for the common response prefix containing the MCTP header, NVDM header and command response payload. That name is too broad once other FSP response formats reuse the same prefix and append protocol-specific payloads. Rename it to FspResponseHeader so subsequent response structures can embed the common header without overloading the meaning of FspResponse. Suggested-by: Alexandre Courbot Link: https://lore.kernel.org/all/DJMBI9CN2Z67.2T02SR8TAWEC5@nvidia.com/ Signed-off-by: Zhi Wang Link: https://patch.msgid.link/20260701062622.3499033-4-zhiw@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fsp.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index f0c595175c9c..08f4acef09f6 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -85,16 +85,16 @@ fn new(nvdm_type: NvdmType) -> Self { } } -/// Complete FSP response structure with MCTP and NVDM headers. +/// Common FSP response header with MCTP, NVDM and command response payloads. #[repr(C, packed)] #[derive(Clone, Copy)] -struct FspResponse { +struct FspResponseHeader { header: FspMessageHeader, response: NvdmPayloadCommandResponse, } -// SAFETY: FspResponse is a packed C struct with only integral fields. -unsafe impl FromBytes for FspResponse {} +// SAFETY: FspResponseHeader is a packed C struct with only integral fields. +unsafe impl FromBytes for FspResponseHeader {} /// Trait implemented by types representing a message to send to FSP. /// @@ -273,10 +273,11 @@ fn send_sync_fsp(&mut self, dev: &device::Device, msg: &M) -> Result dev_err!(dev, "FSP response error: {:?}\n", e); })?; - let (response, _) = FspResponse::from_bytes_prefix(&response_buf[..]).ok_or_else(|| { - dev_err!(dev, "FSP response too small: {}\n", response_buf.len()); - EIO - })?; + let (response, _) = + FspResponseHeader::from_bytes_prefix(&response_buf[..]).ok_or_else(|| { + dev_err!(dev, "FSP response too small: {}\n", response_buf.len()); + EIO + })?; let mctp_header = response.header.mctp_header; let nvdm_header = response.header.nvdm_header; From d85845b64c0020b2812243a22fa79d57cc1c1e38 Mon Sep 17 00:00:00 2001 From: Zhi Wang Date: Wed, 1 Jul 2026 09:26:20 +0300 Subject: [PATCH 040/137] gpu: nova-core: build SetRegistry entries dynamically The GSP SetRegistry command currently stores its registry entries in a fixed-size array. That makes every additional runtime-dependent registry object require reshaping the command data structure at the same time as the feature that needs the new entry. Keep the existing registry contents unchanged, but store them in a KVec so SetRegistry can be constructed dynamically. The constructor now returns a Result to propagate allocation failures while the command payload layout is still computed from the final entry list. Cc: Alexandre Courbot Signed-off-by: Zhi Wang Link: https://patch.msgid.link/20260701062622.3499033-7-zhiw@nvidia.com [acourbot: remove orphan comment.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/boot.rs | 2 +- drivers/gpu/nova-core/gsp/commands.rs | 76 +++++++++++++++------------ 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index ab0491b57944..152093aafae6 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -135,7 +135,7 @@ pub(crate) fn boot( self.cmdq .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev, chipset))?; self.cmdq - .send_command_no_wait(bar, commands::SetRegistry::new())?; + .send_command_no_wait(bar, commands::SetRegistry::new()?)?; hal.post_boot(&self, &ctx, &gsp_fw)?; diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index 86a3747cd31c..08380de39048 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -67,37 +67,44 @@ struct RegistryEntry { /// The `SetRegistry` command. pub(crate) struct SetRegistry { - entries: [RegistryEntry; Self::NUM_ENTRIES], + entries: KVec, } impl SetRegistry { - // For now we hard-code the registry entries. Future work will allow others to - // be added as module parameters. - const NUM_ENTRIES: usize = 3; - /// Creates a new `SetRegistry` command, using a set of hardcoded entries. - pub(crate) fn new() -> Self { - Self { - entries: [ - // RMSecBusResetEnable - enables PCI secondary bus reset - RegistryEntry { - key: "RMSecBusResetEnable", - value: 1, - }, - // RMForcePcieConfigSave - forces GSP-RM to preserve PCI configuration registers on - // any PCI reset. - RegistryEntry { - key: "RMForcePcieConfigSave", - value: 1, - }, - // RMDevidCheckIgnore - allows GSP-RM to boot even if the PCI dev ID is not found - // in the internal product name database. - RegistryEntry { - key: "RMDevidCheckIgnore", - value: 1, - }, - ], - } + pub(crate) fn new() -> Result { + let mut entries = KVec::new(); + + // RMSecBusResetEnable - enables PCI secondary bus reset + entries.push( + RegistryEntry { + key: "RMSecBusResetEnable", + value: 1, + }, + GFP_KERNEL, + )?; + + // RMForcePcieConfigSave - forces GSP-RM to preserve PCI configuration registers on + // any PCI reset. + entries.push( + RegistryEntry { + key: "RMForcePcieConfigSave", + value: 1, + }, + GFP_KERNEL, + )?; + + // RMDevidCheckIgnore - allows GSP-RM to boot even if the PCI dev ID is not found + // in the internal product name database. + entries.push( + RegistryEntry { + key: "RMDevidCheckIgnore", + value: 1, + }, + GFP_KERNEL, + )?; + + Ok(Self { entries }) } } @@ -108,15 +115,18 @@ impl CommandToGsp for SetRegistry { type InitError = Infallible; fn init(&self) -> impl Init { - Self::Command::init(Self::NUM_ENTRIES as u32, self.variable_payload_len() as u32) + Self::Command::init( + self.entries.len() as u32, + self.variable_payload_len() as u32, + ) } fn variable_payload_len(&self) -> usize { let mut key_size = 0; - for i in 0..Self::NUM_ENTRIES { - key_size += self.entries[i].key.len() + 1; // +1 for NULL terminator + for entry in self.entries.iter() { + key_size += entry.key.len() + 1; // +1 for NULL terminator } - Self::NUM_ENTRIES * size_of::() + key_size + self.entries.len() * size_of::() + key_size } fn init_variable_payload( @@ -124,12 +134,12 @@ fn init_variable_payload( dst: &mut SBufferIter>, ) -> Result { let string_data_start_offset = size_of::() - + Self::NUM_ENTRIES * size_of::(); + + self.entries.len() * size_of::(); // Array for string data. let mut string_data = KVec::new(); - for entry in self.entries.iter().take(Self::NUM_ENTRIES) { + for entry in self.entries.iter() { dst.write_all( fw::commands::PackedRegistryEntry::new( (string_data_start_offset + string_data.len()) as u32, From 426c92ca1bdd33dcbc01d6d66bb5bb4a356f2c54 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 6 Jul 2026 13:44:14 +0100 Subject: [PATCH 041/137] rust: io: add dynamically-sized `Region` type Currently many I/O related structs carry a `SIZE` parameter to denote the minimum size of the I/O region, while they also carry a field indicating the actual size. Proliferation of the pattern creates a lot of duplicated code, and makes it hard to create typed views of I/O. Introduce a `Region` type that carries the `SIZE` parameter. It is a wrapper of `[u8]`, which makes it dynamically sized with a metadata of `usize`. This way, pointers to `Region` naturally carry size information. This type is required to be 4-byte aligned. Expose the minimum size information via `MIN_SIZE` constant of the `KnownSize` trait. Similarly, expose the minimum alignment information via `KnownSize::MIN_ALIGN`. With these changes, it is possible to add an associated type to `Io` trait to represent the type of I/O region. For untyped regions, this is the newly added `Region` type. Remove `IoKnownSize` as it is no longer necessary. Use the same mechanism to indicate minimum size of PCI config spaces. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260706-io_projection-v6-1-72cd5d055d54@garyguo.net [ Add brief explanation on MIN_ALIGN. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/devres.rs | 6 +- rust/kernel/io.rs | 130 ++++++++++++++++++++++++++++-------------- rust/kernel/lib.rs | 3 + rust/kernel/pci.rs | 1 - rust/kernel/pci/io.rs | 40 ++++++------- rust/kernel/ptr.rs | 12 ++++ 6 files changed, 118 insertions(+), 74 deletions(-) diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index 11ce500e9b76..ed30ccc6e68e 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -68,7 +68,6 @@ struct Inner { /// devres::Devres, /// io::{ /// Io, -/// IoKnownSize, /// Mmio, /// MmioRaw, /// PhysAddr, // @@ -297,10 +296,7 @@ pub fn device(&self) -> &Device { /// use kernel::{ /// device::Core, /// devres::Devres, - /// io::{ - /// Io, - /// IoKnownSize, // - /// }, + /// io::Io, /// pci, // /// }; /// diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index fcc7678fd9e3..b4cfa3588098 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -6,7 +6,11 @@ use crate::{ bindings, - prelude::*, // + prelude::*, + ptr::{ + Alignment, + KnownSize, // + }, // }; pub mod mem; @@ -31,6 +35,58 @@ /// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. pub type ResourceSize = bindings::resource_size_t; +/// Untyped I/O region. +/// +/// This type can be used when an I/O region without known type information has a compile-time known +/// minimum size (and a runtime known actual size). +/// +/// # Invariants +/// +/// - Size of the region is at least as large as the `SIZE` generic parameter. +/// - Size of the region is multiple of 4. +#[repr(C, align(4))] +pub struct Region { + inner: [u8], +} + +impl Region { + /// Create a raw mutable pointer from given base address and size. + /// + /// `size` should be at least as large as the minimum size `SIZE`, and `base` and `size` should + /// be 4-byte aligned to uphold the type invariant. + /// + /// Just like other methods on raw pointers, it is not unsafe to create a raw pointer + /// that does not uphold the type invariants. However such pointers are not valid. + #[inline] + pub fn ptr_from_raw_parts_mut(base: *mut u8, size: usize) -> *mut Self { + core::ptr::slice_from_raw_parts_mut(base, size) as *mut Region + } + + /// Create a raw mutable pointer from given base address and size. + /// + /// The alignment of `base` is checked, and `size` is checked against the minimum size specified + /// via const generics. + #[inline] + pub fn ptr_try_from_raw_parts_mut(base: *mut u8, size: usize) -> Result<*mut Self> { + if size < SIZE || base.align_offset(4) != 0 || !size.is_multiple_of(4) { + return Err(EINVAL); + } + + Ok(Self::ptr_from_raw_parts_mut(base, size)) + } +} + +impl KnownSize for Region { + const MIN_SIZE: usize = SIZE; + // Alignment of 4 is the most common; different base types can be added once required. + const MIN_ALIGN: Alignment = Alignment::new::<4>(); + + #[inline(always)] + fn size(p: *const Self) -> usize { + (p as *const [u8]).len() + } +} + /// Raw representation of an MMIO region. /// /// By itself, the existence of an instance of this structure does not provide any guarantees that @@ -85,7 +141,6 @@ pub fn maxsize(&self) -> usize { /// ffi::c_void, /// io::{ /// Io, -/// IoKnownSize, /// Mmio, /// MmioRaw, /// PhysAddr, @@ -241,12 +296,25 @@ fn offset(self) -> usize { /// For MMIO regions, all widths (u8, u16, u32, and u64 on 64-bit systems) are typically /// supported. For PCI configuration space, u8, u16, and u32 are supported but u64 is not. pub trait Io { + /// Type of this I/O region. For untyped regions, [`Region`] can be used. + type Target: ?Sized + KnownSize; + /// Returns the base address of this mapping. fn addr(&self) -> usize; /// Returns the maximum size of this mapping. fn maxsize(&self) -> usize; + /// Returns the absolute I/O address for a given `offset`, + /// performing compile-time bound checks. + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] + fn io_addr_assert(&self, offset: usize) -> usize { + build_assert!(offset_valid::(offset, Self::Target::MIN_SIZE)); + + self.addr() + offset + } + /// Returns the absolute I/O address for a given `offset`, /// performing runtime bound checks. #[inline] @@ -336,7 +404,7 @@ fn try_write64(&self, value: u64, offset: usize) -> Result #[inline(always)] fn read8(&self, offset: usize) -> u8 where - Self: IoKnownSize + IoCapable, + Self: IoCapable, { self.read(offset) } @@ -345,7 +413,7 @@ fn read8(&self, offset: usize) -> u8 #[inline(always)] fn read16(&self, offset: usize) -> u16 where - Self: IoKnownSize + IoCapable, + Self: IoCapable, { self.read(offset) } @@ -354,7 +422,7 @@ fn read16(&self, offset: usize) -> u16 #[inline(always)] fn read32(&self, offset: usize) -> u32 where - Self: IoKnownSize + IoCapable, + Self: IoCapable, { self.read(offset) } @@ -363,7 +431,7 @@ fn read32(&self, offset: usize) -> u32 #[inline(always)] fn read64(&self, offset: usize) -> u64 where - Self: IoKnownSize + IoCapable, + Self: IoCapable, { self.read(offset) } @@ -372,7 +440,7 @@ fn read64(&self, offset: usize) -> u64 #[inline(always)] fn write8(&self, value: u8, offset: usize) where - Self: IoKnownSize + IoCapable, + Self: IoCapable, { self.write(offset, value) } @@ -381,7 +449,7 @@ fn write8(&self, value: u8, offset: usize) #[inline(always)] fn write16(&self, value: u16, offset: usize) where - Self: IoKnownSize + IoCapable, + Self: IoCapable, { self.write(offset, value) } @@ -390,7 +458,7 @@ fn write16(&self, value: u16, offset: usize) #[inline(always)] fn write32(&self, value: u32, offset: usize) where - Self: IoKnownSize + IoCapable, + Self: IoCapable, { self.write(offset, value) } @@ -399,7 +467,7 @@ fn write32(&self, value: u32, offset: usize) #[inline(always)] fn write64(&self, value: u64, offset: usize) where - Self: IoKnownSize + IoCapable, + Self: IoCapable, { self.write(offset, value) } @@ -582,7 +650,7 @@ fn try_update(&self, location: L, f: F) -> Result fn read(&self, location: L) -> T where L: IoLoc, - Self: IoKnownSize + IoCapable, + Self: IoCapable, { let address = self.io_addr_assert::(location.offset()); @@ -614,7 +682,7 @@ fn read(&self, location: L) -> T fn write(&self, location: L, value: T) where L: IoLoc, - Self: IoKnownSize + IoCapable, + Self: IoCapable, { let address = self.io_addr_assert::(location.offset()); let io_value = value.into(); @@ -658,7 +726,7 @@ fn write_reg(&self, value: V) where L: IoLoc, V: LocatedRegister, - Self: IoKnownSize + IoCapable, + Self: IoCapable, { let (location, value) = value.into_io_op(); @@ -690,7 +758,7 @@ fn write_reg(&self, value: V) fn update(&self, location: L, f: F) where L: IoLoc, - Self: IoKnownSize + IoCapable + Sized, + Self: IoCapable + Sized, F: FnOnce(T) -> T, { let address = self.io_addr_assert::(location.offset()); @@ -704,28 +772,6 @@ fn update(&self, location: L, f: F) } } -/// Trait for types with a known size at compile time. -/// -/// This trait is implemented by I/O backends that have a compile-time known size, -/// enabling the use of infallible I/O accessors with compile-time bounds checking. -/// -/// Types implementing this trait can use the infallible methods in [`Io`] trait -/// (e.g., `read8`, `write32`), which require `Self: IoKnownSize` bound. -pub trait IoKnownSize: Io { - /// Minimum usable size of this region. - const MIN_SIZE: usize; - - /// Returns the absolute I/O address for a given `offset`, - /// performing compile-time bound checks. - // Always inline to optimize out error path of `build_assert`. - #[inline(always)] - fn io_addr_assert(&self, offset: usize) -> usize { - build_assert!(offset_valid::(offset, Self::MIN_SIZE)); - - self.addr() + offset - } -} - /// Implements [`IoCapable`] on `$mmio` for `$ty` using `$read_fn` and `$write_fn`. macro_rules! impl_mmio_io_capable { ($mmio:ident, $(#[$attr:meta])* $ty:ty, $read_fn:ident, $write_fn:ident) => { @@ -758,6 +804,8 @@ unsafe fn io_write(&self, value: $ty, address: usize) { ); impl Io for Mmio { + type Target = Region; + /// Returns the base address of this mapping. #[inline] fn addr(&self) -> usize { @@ -771,10 +819,6 @@ fn maxsize(&self) -> usize { } } -impl IoKnownSize for Mmio { - const MIN_SIZE: usize = SIZE; -} - impl Mmio { /// Converts an `MmioRaw` into an `Mmio` instance, providing the accessors to the MMIO mapping. /// @@ -798,6 +842,8 @@ pub unsafe fn from_raw(raw: &MmioRaw) -> &Self { pub struct RelaxedMmio(Mmio); impl Io for RelaxedMmio { + type Target = Region; + #[inline] fn addr(&self) -> usize { self.0.addr() @@ -809,10 +855,6 @@ fn maxsize(&self) -> usize { } } -impl IoKnownSize for RelaxedMmio { - const MIN_SIZE: usize = SIZE; -} - impl Mmio { /// Returns a [`RelaxedMmio`] reference that performs relaxed I/O operations. /// diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 9512af7156df..68f4d9a3425d 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -16,6 +16,9 @@ // Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on // the unstable features in use. // +// Stable since Rust 1.87.0. +#![feature(unsigned_is_multiple_of)] +// // Stable since Rust 1.89.0. #![feature(generic_arg_infer)] // diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index 5071cae6543f..c6d6bd8f251d 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -43,7 +43,6 @@ pub use self::io::{ Bar, ConfigSpace, - ConfigSpaceKind, ConfigSpaceSize, Extended, Normal, // diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs index 0461e01aaa20..b4996aa059d8 100644 --- a/rust/kernel/pci/io.rs +++ b/rust/kernel/pci/io.rs @@ -10,11 +10,12 @@ io::{ Io, IoCapable, - IoKnownSize, Mmio, - MmioRaw, // + MmioRaw, + Region, // }, - prelude::*, // + prelude::*, + ptr::KnownSize, // }; use core::{ marker::PhantomData, @@ -46,28 +47,21 @@ pub const fn into_raw(self) -> usize { } } -/// Marker type for normal (256-byte) PCI configuration space. -pub struct Normal; +/// Alias for normal (256-byte) PCI configuration space. +pub type Normal = Region<256>; -/// Marker type for extended (4096-byte) PCIe configuration space. -pub struct Extended; +/// Alias for extended (4096-byte) PCIe configuration space. +pub type Extended = Region<4096>; /// Trait for PCI configuration space size markers. /// /// This trait is implemented by [`Normal`] and [`Extended`] to provide /// compile-time knowledge of the configuration space size. -pub trait ConfigSpaceKind { - /// The size of this configuration space in bytes. - const SIZE: usize; -} +pub trait ConfigSpaceKind: KnownSize {} -impl ConfigSpaceKind for Normal { - const SIZE: usize = 256; -} +impl ConfigSpaceKind for Normal {} -impl ConfigSpaceKind for Extended { - const SIZE: usize = 4096; -} +impl ConfigSpaceKind for Extended {} /// The PCI configuration space of a device. /// @@ -77,7 +71,7 @@ impl ConfigSpaceKind for Extended { /// The generic parameter `S` indicates the maximum size of the configuration space. /// Use [`Normal`] for 256-byte legacy configuration space or [`Extended`] for /// 4096-byte PCIe extended configuration space (default). -pub struct ConfigSpace<'a, S: ConfigSpaceKind = Extended> { +pub struct ConfigSpace<'a, S: ?Sized + ConfigSpaceKind = Extended> { pub(crate) pdev: &'a Device, _marker: PhantomData, } @@ -85,7 +79,7 @@ pub struct ConfigSpace<'a, S: ConfigSpaceKind = Extended> { /// Implements [`IoCapable`] on [`ConfigSpace`] for `$ty` using `$read_fn` and `$write_fn`. macro_rules! impl_config_space_io_capable { ($ty:ty, $read_fn:ident, $write_fn:ident) => { - impl<'a, S: ConfigSpaceKind> IoCapable<$ty> for ConfigSpace<'a, S> { + impl<'a, S: ?Sized + ConfigSpaceKind> IoCapable<$ty> for ConfigSpace<'a, S> { unsafe fn io_read(&self, address: usize) -> $ty { let mut val: $ty = 0; @@ -118,7 +112,9 @@ unsafe fn io_write(&self, value: $ty, address: usize) { impl_config_space_io_capable!(u16, pci_read_config_word, pci_write_config_word); impl_config_space_io_capable!(u32, pci_read_config_dword, pci_write_config_dword); -impl<'a, S: ConfigSpaceKind> Io for ConfigSpace<'a, S> { +impl<'a, S: ?Sized + ConfigSpaceKind> Io for ConfigSpace<'a, S> { + type Target = S; + /// Returns the base address of the I/O region. It is always 0 for configuration space. #[inline] fn addr(&self) -> usize { @@ -132,10 +128,6 @@ fn maxsize(&self) -> usize { } } -impl<'a, S: ConfigSpaceKind> IoKnownSize for ConfigSpace<'a, S> { - const MIN_SIZE: usize = S::SIZE; -} - /// A PCI BAR to perform I/O-Operations on. /// /// I/O backend assumes that the device is little-endian and will automatically diff --git a/rust/kernel/ptr.rs b/rust/kernel/ptr.rs index 3f3e529e9f58..82acb531b17b 100644 --- a/rust/kernel/ptr.rs +++ b/rust/kernel/ptr.rs @@ -235,11 +235,20 @@ fn align_up(self, alignment: Alignment) -> Option { /// /// This is a generalization of [`size_of`] that works for dynamically sized types. pub trait KnownSize { + /// Minimum size of this type known at compile-time. + const MIN_SIZE: usize; + + /// Minimum alignment of this type known at compile-time. + const MIN_ALIGN: Alignment; + /// Get the size of an object of this type in bytes, with the metadata of the given pointer. fn size(p: *const Self) -> usize; } impl KnownSize for T { + const MIN_SIZE: usize = size_of::(); + const MIN_ALIGN: Alignment = Alignment::of::(); + #[inline(always)] fn size(_: *const Self) -> usize { size_of::() @@ -247,6 +256,9 @@ fn size(_: *const Self) -> usize { } impl KnownSize for [T] { + const MIN_SIZE: usize = 0; + const MIN_ALIGN: Alignment = Alignment::of::(); + #[inline(always)] fn size(p: *const Self) -> usize { p.len() * size_of::() From 65f6abf9fa81617c8455a28b5f07269d883d080a Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 6 Jul 2026 13:44:15 +0100 Subject: [PATCH 042/137] rust: io: add missing safety requirement in `IoCapable` methods The current safety comment on `io_read`/`io_write` does not cover the topic about alignment. Add it so it can be relied on by implementor of `IoCapable`. Expand the check performed by `Io` by taking `self.addr()` into consideration when checking if `offset` is aligned. For the compile-time `io_addr_assert` check, check using the known minimum alignment of `Io::Target` and the accessed type. While at it, fix the alignment check to use `align_of` instead of `size_of`. The values match for all primitives (including u64, given that we do not provide u64 accessor on 32-bit platforms), but are not necessarily true for custom types. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260706-io_projection-v6-2-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index b4cfa3588098..c9597065a776 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -196,13 +196,14 @@ pub fn maxsize(&self) -> usize { #[repr(transparent)] pub struct Mmio(MmioRaw); -/// Checks whether an access of type `U` at the given `offset` +/// Checks whether an access of type `U` at the given `base` and the given `offset` /// is valid within this region. +/// +/// The `base` is used for alignment checking only. This can be set to 0 to skip the check. #[inline] -const fn offset_valid(offset: usize, size: usize) -> bool { - let type_size = core::mem::size_of::(); - if let Some(end) = offset.checked_add(type_size) { - end <= size && offset % type_size == 0 +const fn offset_valid(base: usize, offset: usize, size: usize) -> bool { + if let Some(end) = offset.checked_add(size_of::()) { + end <= size && (base.wrapping_add(offset) % align_of::() == 0) } else { false } @@ -221,14 +222,16 @@ pub trait IoCapable { /// /// # Safety /// - /// The range `[address..address + size_of::()]` must be within the bounds of `Self`. + /// - The range `[address..address + size_of::()]` must be within the bounds of `Self`. + /// - `address` must be aligned. unsafe fn io_read(&self, address: usize) -> T; /// Performs an I/O write of `value` at `address`. /// /// # Safety /// - /// The range `[address..address + size_of::()]` must be within the bounds of `Self`. + /// - The range `[address..address + size_of::()]` must be within the bounds of `Self`. + /// - `address` must be aligned. unsafe fn io_write(&self, value: T, address: usize); } @@ -310,7 +313,11 @@ pub trait Io { // Always inline to optimize out error path of `build_assert`. #[inline(always)] fn io_addr_assert(&self, offset: usize) -> usize { - build_assert!(offset_valid::(offset, Self::Target::MIN_SIZE)); + // We cannot check alignment with `offset_valid` using `self.addr()`. So set 0 for it and + // ensure alignment by checking that the alignment of `U` is smaller or equal to the + // alignment of `Self::Target`. + const_assert!(Alignment::of::().as_usize() <= Self::Target::MIN_ALIGN.as_usize()); + build_assert!(offset_valid::(0, offset, Self::Target::MIN_SIZE)); self.addr() + offset } @@ -319,7 +326,7 @@ fn io_addr_assert(&self, offset: usize) -> usize { /// performing runtime bound checks. #[inline] fn io_addr(&self, offset: usize) -> Result { - if !offset_valid::(offset, self.maxsize()) { + if !offset_valid::(self.addr(), offset, self.maxsize()) { return Err(EINVAL); } From 6461c5776bf0f546cfeaf2a72f1a2f7de27bfe0d Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 6 Jul 2026 13:44:16 +0100 Subject: [PATCH 043/137] rust: io: restrict untyped IO access and `register!` to `Region` Currently the `Io` trait exposes a bunch of untyped IO accesses, but if the `Io` region itself is typed, then it might be weird to have let io: Mmio = /* ... */; io.read8(1); while not unsound, it is surely strange. Thus, restrict the untyped methods and also the register macro to `Region` type only. Implement it by adding a generic type to `IoLoc` indicating allowed base types. This also paves the way to add typed register blocks in the future; for example, we could use this mechanism to block driver A's `register!()` generated macro from being used on driver B's MMIO. The same mechanism could be used for relative IO registers. These are future opportunities, and for now restrict everything to require `IoLoc, _>`. Suggested-by: Alexandre Courbot Link: https://lore.kernel.org/rust-for-linux/DHLB3RO3OSF5.2R7F27U99BKLN@nvidia.com/ Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260706-io_projection-v6-3-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 49 +++++++++++++++++++++++++------------- rust/kernel/io/register.rs | 20 +++++++++------- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index c9597065a776..b0dac2a54a24 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -244,15 +244,16 @@ pub trait IoCapable { /// (for primitive types like [`u32`]) and typed ones (like those generated by the [`register!`] /// macro). /// -/// An `IoLoc` carries three pieces of information: +/// An `IoLoc` carries the following pieces of information: /// +/// - The valid `Base` to operate on. For most registers, this should be [`Region`]. /// - The offset to access (returned by [`IoLoc::offset`]), /// - The width of the access (determined by [`IoLoc::IoType`]), /// - The type `T` in which the raw data is returned or provided. /// /// `T` and `IoLoc::IoType` may differ: for instance, a typed register has `T` = the register type /// with its bitfields, and `IoType` = its backing primitive (e.g. `u32`). -pub trait IoLoc { +pub trait IoLoc { /// Size ([`u8`], [`u16`], etc) of the I/O performed on the returned [`offset`](IoLoc::offset). type IoType: Into + From; @@ -260,12 +261,12 @@ pub trait IoLoc { fn offset(self) -> usize; } -/// Implements [`IoLoc<$ty>`] for [`usize`], allowing [`usize`] to be used as a parameter of -/// [`Io::read`] and [`Io::write`]. +/// Implements [`IoLoc, $ty>`] for [`usize`], allowing [`usize`] to be used as a +/// parameter of [`Io::read`] and [`Io::write`]. macro_rules! impl_usize_ioloc { ($($ty:ty),*) => { $( - impl IoLoc<$ty> for usize { + impl IoLoc, $ty> for usize { type IoType = $ty; #[inline(always)] @@ -339,6 +340,7 @@ fn io_addr(&self, offset: usize) -> Result { #[inline(always)] fn try_read8(&self, offset: usize) -> Result where + usize: IoLoc, Self: IoCapable, { self.try_read(offset) @@ -348,6 +350,7 @@ fn try_read8(&self, offset: usize) -> Result #[inline(always)] fn try_read16(&self, offset: usize) -> Result where + usize: IoLoc, Self: IoCapable, { self.try_read(offset) @@ -357,6 +360,7 @@ fn try_read16(&self, offset: usize) -> Result #[inline(always)] fn try_read32(&self, offset: usize) -> Result where + usize: IoLoc, Self: IoCapable, { self.try_read(offset) @@ -366,6 +370,7 @@ fn try_read32(&self, offset: usize) -> Result #[inline(always)] fn try_read64(&self, offset: usize) -> Result where + usize: IoLoc, Self: IoCapable, { self.try_read(offset) @@ -375,6 +380,7 @@ fn try_read64(&self, offset: usize) -> Result #[inline(always)] fn try_write8(&self, value: u8, offset: usize) -> Result where + usize: IoLoc, Self: IoCapable, { self.try_write(offset, value) @@ -384,6 +390,7 @@ fn try_write8(&self, value: u8, offset: usize) -> Result #[inline(always)] fn try_write16(&self, value: u16, offset: usize) -> Result where + usize: IoLoc, Self: IoCapable, { self.try_write(offset, value) @@ -393,6 +400,7 @@ fn try_write16(&self, value: u16, offset: usize) -> Result #[inline(always)] fn try_write32(&self, value: u32, offset: usize) -> Result where + usize: IoLoc, Self: IoCapable, { self.try_write(offset, value) @@ -402,6 +410,7 @@ fn try_write32(&self, value: u32, offset: usize) -> Result #[inline(always)] fn try_write64(&self, value: u64, offset: usize) -> Result where + usize: IoLoc, Self: IoCapable, { self.try_write(offset, value) @@ -411,6 +420,7 @@ fn try_write64(&self, value: u64, offset: usize) -> Result #[inline(always)] fn read8(&self, offset: usize) -> u8 where + usize: IoLoc, Self: IoCapable, { self.read(offset) @@ -420,6 +430,7 @@ fn read8(&self, offset: usize) -> u8 #[inline(always)] fn read16(&self, offset: usize) -> u16 where + usize: IoLoc, Self: IoCapable, { self.read(offset) @@ -429,6 +440,7 @@ fn read16(&self, offset: usize) -> u16 #[inline(always)] fn read32(&self, offset: usize) -> u32 where + usize: IoLoc, Self: IoCapable, { self.read(offset) @@ -438,6 +450,7 @@ fn read32(&self, offset: usize) -> u32 #[inline(always)] fn read64(&self, offset: usize) -> u64 where + usize: IoLoc, Self: IoCapable, { self.read(offset) @@ -447,6 +460,7 @@ fn read64(&self, offset: usize) -> u64 #[inline(always)] fn write8(&self, value: u8, offset: usize) where + usize: IoLoc, Self: IoCapable, { self.write(offset, value) @@ -456,6 +470,7 @@ fn write8(&self, value: u8, offset: usize) #[inline(always)] fn write16(&self, value: u16, offset: usize) where + usize: IoLoc, Self: IoCapable, { self.write(offset, value) @@ -465,6 +480,7 @@ fn write16(&self, value: u16, offset: usize) #[inline(always)] fn write32(&self, value: u32, offset: usize) where + usize: IoLoc, Self: IoCapable, { self.write(offset, value) @@ -474,6 +490,7 @@ fn write32(&self, value: u32, offset: usize) #[inline(always)] fn write64(&self, value: u64, offset: usize) where + usize: IoLoc, Self: IoCapable, { self.write(offset, value) @@ -504,7 +521,7 @@ fn write64(&self, value: u64, offset: usize) #[inline(always)] fn try_read(&self, location: L) -> Result where - L: IoLoc, + L: IoLoc, Self: IoCapable, { let address = self.io_addr::(location.offset())?; @@ -538,7 +555,7 @@ fn try_read(&self, location: L) -> Result #[inline(always)] fn try_write(&self, location: L, value: T) -> Result where - L: IoLoc, + L: IoLoc, Self: IoCapable, { let address = self.io_addr::(location.offset())?; @@ -584,8 +601,8 @@ fn try_write(&self, location: L, value: T) -> Result #[inline(always)] fn try_write_reg(&self, value: V) -> Result where - L: IoLoc, - V: LocatedRegister, + L: IoLoc, + V: LocatedRegister, Self: IoCapable, { let (location, value) = value.into_io_op(); @@ -617,7 +634,7 @@ fn try_write_reg(&self, value: V) -> Result #[inline(always)] fn try_update(&self, location: L, f: F) -> Result where - L: IoLoc, + L: IoLoc, Self: IoCapable, F: FnOnce(T) -> T, { @@ -656,7 +673,7 @@ fn try_update(&self, location: L, f: F) -> Result #[inline(always)] fn read(&self, location: L) -> T where - L: IoLoc, + L: IoLoc, Self: IoCapable, { let address = self.io_addr_assert::(location.offset()); @@ -688,7 +705,7 @@ fn read(&self, location: L) -> T #[inline(always)] fn write(&self, location: L, value: T) where - L: IoLoc, + L: IoLoc, Self: IoCapable, { let address = self.io_addr_assert::(location.offset()); @@ -731,8 +748,8 @@ fn write(&self, location: L, value: T) #[inline(always)] fn write_reg(&self, value: V) where - L: IoLoc, - V: LocatedRegister, + L: IoLoc, + V: LocatedRegister, Self: IoCapable, { let (location, value) = value.into_io_op(); @@ -764,8 +781,8 @@ fn write_reg(&self, value: V) #[inline(always)] fn update(&self, location: L, f: F) where - L: IoLoc, - Self: IoCapable + Sized, + L: IoLoc, + Self: IoCapable, F: FnOnce(T) -> T, { let address = self.io_addr_assert::(location.offset()); diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index f924c7c7c1db..3122b17098ee 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -113,6 +113,8 @@ io::IoLoc, // }; +use super::Region; + /// Trait implemented by all registers. pub trait Register: Sized { /// Backing primitive type of the register. @@ -129,7 +131,7 @@ pub trait FixedRegister: Register {} /// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when /// passing a [`FixedRegister`] value. -impl IoLoc for () +impl IoLoc, T> for () where T: FixedRegister, { @@ -143,7 +145,7 @@ fn offset(self) -> usize { /// A [`FixedRegister`] carries its location in its type. Thus `FixedRegister` values can be used /// as an [`IoLoc`]. -impl IoLoc for T +impl IoLoc, T> for T where T: FixedRegister, { @@ -168,7 +170,7 @@ pub const fn new() -> Self { } } -impl IoLoc for FixedRegisterLoc +impl IoLoc, T> for FixedRegisterLoc where T: FixedRegister, { @@ -239,7 +241,7 @@ const fn offset(self) -> usize { } } -impl IoLoc for RelativeRegisterLoc +impl IoLoc, T> for RelativeRegisterLoc where T: RelativeRegister, B: RegisterBase + ?Sized, @@ -283,7 +285,7 @@ pub fn try_new(idx: usize) -> Option { } } -impl IoLoc for RegisterArrayLoc +impl IoLoc, T> for RegisterArrayLoc where T: RegisterArray, { @@ -370,7 +372,7 @@ pub fn try_at(self, idx: usize) -> Option> { } } -impl IoLoc for RelativeRegisterArrayLoc +impl IoLoc, T> for RelativeRegisterArrayLoc where T: RelativeRegisterArray, B: RegisterBase + ?Sized, @@ -387,18 +389,18 @@ fn offset(self) -> usize { /// which to write it. /// /// Implementors can be used with [`Io::write_reg`](super::Io::write_reg). -pub trait LocatedRegister { +pub trait LocatedRegister { /// Register value to write. type Value: Register; /// Full location information at which to write the value. - type Location: IoLoc; + type Location: IoLoc; /// Consumes `self` and returns a `(location, value)` tuple describing a valid I/O write /// operation. fn into_io_op(self) -> (Self::Location, Self::Value); } -impl LocatedRegister for T +impl LocatedRegister> for T where T: FixedRegister, { From 46b1b54139c3e24b80eefc8da09c2f731ecc7e73 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 6 Jul 2026 13:44:17 +0100 Subject: [PATCH 044/137] rust: io: implement `Io` on reference types instead Currently, `Io` is implemented on owned I/O objects (e.g. `Bar`). This is going to change with I/O projections, as then `Io` needs to work both for owned objects and views of them. Views are themselves reference-like (however they obviously cannot be references, because they belong to a different address space). To facilitate the change, change `Io` to be implemented on reference types for the owned I/O objects, and make methods take `self` instead of `&self`. When I/O views are implemented, we can then naturally implement `Io` for these objects. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260706-io_projection-v6-4-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 82 ++++++++++++++++++++++--------------------- rust/kernel/pci/io.rs | 12 +++---- 2 files changed, 48 insertions(+), 46 deletions(-) diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index b0dac2a54a24..a2aa6173ce87 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -224,7 +224,7 @@ pub trait IoCapable { /// /// - The range `[address..address + size_of::()]` must be within the bounds of `Self`. /// - `address` must be aligned. - unsafe fn io_read(&self, address: usize) -> T; + unsafe fn io_read(self, address: usize) -> T; /// Performs an I/O write of `value` at `address`. /// @@ -232,7 +232,7 @@ pub trait IoCapable { /// /// - The range `[address..address + size_of::()]` must be within the bounds of `Self`. /// - `address` must be aligned. - unsafe fn io_write(&self, value: T, address: usize); + unsafe fn io_write(self, value: T, address: usize); } /// Describes a given I/O location: its offset, width, and type to convert the raw value from and @@ -295,25 +295,27 @@ fn offset(self) -> usize { /// Which I/O methods are available depends on which [`IoCapable`] traits /// are implemented for the type. /// +/// This should be implemented on cheaply copyable handles, such as references or view types. +/// /// # Examples /// /// For MMIO regions, all widths (u8, u16, u32, and u64 on 64-bit systems) are typically /// supported. For PCI configuration space, u8, u16, and u32 are supported but u64 is not. -pub trait Io { +pub trait Io: Copy { /// Type of this I/O region. For untyped regions, [`Region`] can be used. type Target: ?Sized + KnownSize; /// Returns the base address of this mapping. - fn addr(&self) -> usize; + fn addr(self) -> usize; /// Returns the maximum size of this mapping. - fn maxsize(&self) -> usize; + fn maxsize(self) -> usize; /// Returns the absolute I/O address for a given `offset`, /// performing compile-time bound checks. // Always inline to optimize out error path of `build_assert`. #[inline(always)] - fn io_addr_assert(&self, offset: usize) -> usize { + fn io_addr_assert(self, offset: usize) -> usize { // We cannot check alignment with `offset_valid` using `self.addr()`. So set 0 for it and // ensure alignment by checking that the alignment of `U` is smaller or equal to the // alignment of `Self::Target`. @@ -326,7 +328,7 @@ fn io_addr_assert(&self, offset: usize) -> usize { /// Returns the absolute I/O address for a given `offset`, /// performing runtime bound checks. #[inline] - fn io_addr(&self, offset: usize) -> Result { + fn io_addr(self, offset: usize) -> Result { if !offset_valid::(self.addr(), offset, self.maxsize()) { return Err(EINVAL); } @@ -338,7 +340,7 @@ fn io_addr(&self, offset: usize) -> Result { /// Fallible 8-bit read with runtime bounds check. #[inline(always)] - fn try_read8(&self, offset: usize) -> Result + fn try_read8(self, offset: usize) -> Result where usize: IoLoc, Self: IoCapable, @@ -348,7 +350,7 @@ fn try_read8(&self, offset: usize) -> Result /// Fallible 16-bit read with runtime bounds check. #[inline(always)] - fn try_read16(&self, offset: usize) -> Result + fn try_read16(self, offset: usize) -> Result where usize: IoLoc, Self: IoCapable, @@ -358,7 +360,7 @@ fn try_read16(&self, offset: usize) -> Result /// Fallible 32-bit read with runtime bounds check. #[inline(always)] - fn try_read32(&self, offset: usize) -> Result + fn try_read32(self, offset: usize) -> Result where usize: IoLoc, Self: IoCapable, @@ -368,7 +370,7 @@ fn try_read32(&self, offset: usize) -> Result /// Fallible 64-bit read with runtime bounds check. #[inline(always)] - fn try_read64(&self, offset: usize) -> Result + fn try_read64(self, offset: usize) -> Result where usize: IoLoc, Self: IoCapable, @@ -378,7 +380,7 @@ fn try_read64(&self, offset: usize) -> Result /// Fallible 8-bit write with runtime bounds check. #[inline(always)] - fn try_write8(&self, value: u8, offset: usize) -> Result + fn try_write8(self, value: u8, offset: usize) -> Result where usize: IoLoc, Self: IoCapable, @@ -388,7 +390,7 @@ fn try_write8(&self, value: u8, offset: usize) -> Result /// Fallible 16-bit write with runtime bounds check. #[inline(always)] - fn try_write16(&self, value: u16, offset: usize) -> Result + fn try_write16(self, value: u16, offset: usize) -> Result where usize: IoLoc, Self: IoCapable, @@ -398,7 +400,7 @@ fn try_write16(&self, value: u16, offset: usize) -> Result /// Fallible 32-bit write with runtime bounds check. #[inline(always)] - fn try_write32(&self, value: u32, offset: usize) -> Result + fn try_write32(self, value: u32, offset: usize) -> Result where usize: IoLoc, Self: IoCapable, @@ -408,7 +410,7 @@ fn try_write32(&self, value: u32, offset: usize) -> Result /// Fallible 64-bit write with runtime bounds check. #[inline(always)] - fn try_write64(&self, value: u64, offset: usize) -> Result + fn try_write64(self, value: u64, offset: usize) -> Result where usize: IoLoc, Self: IoCapable, @@ -418,7 +420,7 @@ fn try_write64(&self, value: u64, offset: usize) -> Result /// Infallible 8-bit read with compile-time bounds check. #[inline(always)] - fn read8(&self, offset: usize) -> u8 + fn read8(self, offset: usize) -> u8 where usize: IoLoc, Self: IoCapable, @@ -428,7 +430,7 @@ fn read8(&self, offset: usize) -> u8 /// Infallible 16-bit read with compile-time bounds check. #[inline(always)] - fn read16(&self, offset: usize) -> u16 + fn read16(self, offset: usize) -> u16 where usize: IoLoc, Self: IoCapable, @@ -438,7 +440,7 @@ fn read16(&self, offset: usize) -> u16 /// Infallible 32-bit read with compile-time bounds check. #[inline(always)] - fn read32(&self, offset: usize) -> u32 + fn read32(self, offset: usize) -> u32 where usize: IoLoc, Self: IoCapable, @@ -448,7 +450,7 @@ fn read32(&self, offset: usize) -> u32 /// Infallible 64-bit read with compile-time bounds check. #[inline(always)] - fn read64(&self, offset: usize) -> u64 + fn read64(self, offset: usize) -> u64 where usize: IoLoc, Self: IoCapable, @@ -458,7 +460,7 @@ fn read64(&self, offset: usize) -> u64 /// Infallible 8-bit write with compile-time bounds check. #[inline(always)] - fn write8(&self, value: u8, offset: usize) + fn write8(self, value: u8, offset: usize) where usize: IoLoc, Self: IoCapable, @@ -468,7 +470,7 @@ fn write8(&self, value: u8, offset: usize) /// Infallible 16-bit write with compile-time bounds check. #[inline(always)] - fn write16(&self, value: u16, offset: usize) + fn write16(self, value: u16, offset: usize) where usize: IoLoc, Self: IoCapable, @@ -478,7 +480,7 @@ fn write16(&self, value: u16, offset: usize) /// Infallible 32-bit write with compile-time bounds check. #[inline(always)] - fn write32(&self, value: u32, offset: usize) + fn write32(self, value: u32, offset: usize) where usize: IoLoc, Self: IoCapable, @@ -488,7 +490,7 @@ fn write32(&self, value: u32, offset: usize) /// Infallible 64-bit write with compile-time bounds check. #[inline(always)] - fn write64(&self, value: u64, offset: usize) + fn write64(self, value: u64, offset: usize) where usize: IoLoc, Self: IoCapable, @@ -519,7 +521,7 @@ fn write64(&self, value: u64, offset: usize) /// } /// ``` #[inline(always)] - fn try_read(&self, location: L) -> Result + fn try_read(self, location: L) -> Result where L: IoLoc, Self: IoCapable, @@ -553,7 +555,7 @@ fn try_read(&self, location: L) -> Result /// } /// ``` #[inline(always)] - fn try_write(&self, location: L, value: T) -> Result + fn try_write(self, location: L, value: T) -> Result where L: IoLoc, Self: IoCapable, @@ -599,7 +601,7 @@ fn try_write(&self, location: L, value: T) -> Result /// } /// ``` #[inline(always)] - fn try_write_reg(&self, value: V) -> Result + fn try_write_reg(self, value: V) -> Result where L: IoLoc, V: LocatedRegister, @@ -632,7 +634,7 @@ fn try_write_reg(&self, value: V) -> Result /// } /// ``` #[inline(always)] - fn try_update(&self, location: L, f: F) -> Result + fn try_update(self, location: L, f: F) -> Result where L: IoLoc, Self: IoCapable, @@ -671,7 +673,7 @@ fn try_update(&self, location: L, f: F) -> Result /// } /// ``` #[inline(always)] - fn read(&self, location: L) -> T + fn read(self, location: L) -> T where L: IoLoc, Self: IoCapable, @@ -703,7 +705,7 @@ fn read(&self, location: L) -> T /// } /// ``` #[inline(always)] - fn write(&self, location: L, value: T) + fn write(self, location: L, value: T) where L: IoLoc, Self: IoCapable, @@ -746,7 +748,7 @@ fn write(&self, location: L, value: T) /// } /// ``` #[inline(always)] - fn write_reg(&self, value: V) + fn write_reg(self, value: V) where L: IoLoc, V: LocatedRegister, @@ -779,7 +781,7 @@ fn write_reg(&self, value: V) /// } /// ``` #[inline(always)] - fn update(&self, location: L, f: F) + fn update(self, location: L, f: F) where L: IoLoc, Self: IoCapable, @@ -800,13 +802,13 @@ fn update(&self, location: L, f: F) macro_rules! impl_mmio_io_capable { ($mmio:ident, $(#[$attr:meta])* $ty:ty, $read_fn:ident, $write_fn:ident) => { $(#[$attr])* - impl IoCapable<$ty> for $mmio { - unsafe fn io_read(&self, address: usize) -> $ty { + impl IoCapable<$ty> for &$mmio { + unsafe fn io_read(self, address: usize) -> $ty { // SAFETY: By the trait invariant `address` is a valid address for MMIO operations. unsafe { bindings::$read_fn(address as *const c_void) } } - unsafe fn io_write(&self, value: $ty, address: usize) { + unsafe fn io_write(self, value: $ty, address: usize) { // SAFETY: By the trait invariant `address` is a valid address for MMIO operations. unsafe { bindings::$write_fn(value, address as *mut c_void) } } @@ -827,18 +829,18 @@ unsafe fn io_write(&self, value: $ty, address: usize) { writeq ); -impl Io for Mmio { +impl<'a, const SIZE: usize> Io for &'a Mmio { type Target = Region; /// Returns the base address of this mapping. #[inline] - fn addr(&self) -> usize { + fn addr(self) -> usize { self.0.addr() } /// Returns the maximum size of this mapping. #[inline] - fn maxsize(&self) -> usize { + fn maxsize(self) -> usize { self.0.maxsize() } } @@ -865,16 +867,16 @@ pub unsafe fn from_raw(raw: &MmioRaw) -> &Self { #[repr(transparent)] pub struct RelaxedMmio(Mmio); -impl Io for RelaxedMmio { +impl<'a, const SIZE: usize> Io for &'a RelaxedMmio { type Target = Region; #[inline] - fn addr(&self) -> usize { + fn addr(self) -> usize { self.0.addr() } #[inline] - fn maxsize(&self) -> usize { + fn maxsize(self) -> usize { self.0.maxsize() } } diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs index b4996aa059d8..505305cd9b86 100644 --- a/rust/kernel/pci/io.rs +++ b/rust/kernel/pci/io.rs @@ -79,8 +79,8 @@ pub struct ConfigSpace<'a, S: ?Sized + ConfigSpaceKind = Extended> { /// Implements [`IoCapable`] on [`ConfigSpace`] for `$ty` using `$read_fn` and `$write_fn`. macro_rules! impl_config_space_io_capable { ($ty:ty, $read_fn:ident, $write_fn:ident) => { - impl<'a, S: ?Sized + ConfigSpaceKind> IoCapable<$ty> for ConfigSpace<'a, S> { - unsafe fn io_read(&self, address: usize) -> $ty { + impl<'a, S: ?Sized + ConfigSpaceKind> IoCapable<$ty> for &ConfigSpace<'a, S> { + unsafe fn io_read(self, address: usize) -> $ty { let mut val: $ty = 0; // Return value from C function is ignored in infallible accessors. @@ -94,7 +94,7 @@ unsafe fn io_read(&self, address: usize) -> $ty { val } - unsafe fn io_write(&self, value: $ty, address: usize) { + unsafe fn io_write(self, value: $ty, address: usize) { // Return value from C function is ignored in infallible accessors. let _ret = // SAFETY: By the type invariant `self.pdev` is a valid address. @@ -112,18 +112,18 @@ unsafe fn io_write(&self, value: $ty, address: usize) { impl_config_space_io_capable!(u16, pci_read_config_word, pci_write_config_word); impl_config_space_io_capable!(u32, pci_read_config_dword, pci_write_config_dword); -impl<'a, S: ?Sized + ConfigSpaceKind> Io for ConfigSpace<'a, S> { +impl<'a, S: ?Sized + ConfigSpaceKind> Io for &ConfigSpace<'a, S> { type Target = S; /// Returns the base address of the I/O region. It is always 0 for configuration space. #[inline] - fn addr(&self) -> usize { + fn addr(self) -> usize { 0 } /// Returns the maximum size of the configuration space. #[inline] - fn maxsize(&self) -> usize { + fn maxsize(self) -> usize { self.pdev.cfg_size().into_raw() } } From 9734e905119c5f7d7af9dd3e483f9a0d9ee12187 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 6 Jul 2026 13:44:18 +0100 Subject: [PATCH 045/137] rust: io: generalize `MmioRaw` to pointer to arbitrary type Conceptually, `MmioRaw` is just `__iomem *`, so it should work for any types. Update the existing use case where it represents a region of compile-time known minimum size and run-time known actual size to use the dynamic-sized type `Region` instead. Rename `maxsize` method to reflect that it is the actual size (not a bound) of the region. Implement `Clone` and `Copy` manually, which cannot be derived due to the generic parameter. The use of raw pointers also cause the `Send` and `Sync` auto trait implementation to be lost, so add them back by manual implementation. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Reviewed-by: Daniel Almeida Suggested-by: Danilo Krummrich Link: https://rust-for-linux.zulipchat.com/#narrow/channel/288089-General/topic/Generic.20I.2FO.20backends/near/571198078 Link: https://patch.msgid.link/20260706-io_projection-v6-5-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/devres.rs | 7 +++-- rust/kernel/io.rs | 71 +++++++++++++++++++++++++++++++------------ rust/kernel/io/mem.rs | 5 ++- rust/kernel/pci/io.rs | 4 +-- 4 files changed, 59 insertions(+), 28 deletions(-) diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index ed30ccc6e68e..d0c677fd7932 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -70,14 +70,15 @@ struct Inner { /// Io, /// Mmio, /// MmioRaw, -/// PhysAddr, // +/// PhysAddr, +/// Region, // /// }, /// prelude::*, /// }; /// use core::ops::Deref; /// /// // See also [`pci::Bar`] for a real example. -/// struct IoMem(MmioRaw); +/// struct IoMem(MmioRaw>); /// /// impl IoMem { /// /// # Safety @@ -92,7 +93,7 @@ struct Inner { /// return Err(ENOMEM); /// } /// -/// Ok(IoMem(MmioRaw::new(addr as usize, SIZE)?)) +/// Ok(IoMem(MmioRaw::new_region(addr as usize, SIZE)?)) /// } /// } /// diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index a2aa6173ce87..3013d8cf39e4 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -89,37 +89,67 @@ fn size(p: *const Self) -> usize { /// Raw representation of an MMIO region. /// +/// `MmioRaw` is equivalent to `T __iomem *` in C. +/// /// By itself, the existence of an instance of this structure does not provide any guarantees that /// the represented MMIO region does exist or is properly mapped. /// /// Instead, the bus specific MMIO implementation must convert this raw representation into an /// `Mmio` instance providing the actual memory accessors. Only by the conversion into an `Mmio` /// structure any guarantees are given. -pub struct MmioRaw { - addr: usize, - maxsize: usize, +pub struct MmioRaw { + /// Pointer is in I/O address space. + /// + /// The provenance does not matter, only the address and metadata do. + ptr: *mut T, } -impl MmioRaw { - /// Returns a new `MmioRaw` instance on success, an error otherwise. - pub fn new(addr: usize, maxsize: usize) -> Result { - if maxsize < SIZE { - return Err(EINVAL); - } - - Ok(Self { addr, maxsize }) +impl Copy for MmioRaw {} +impl Clone for MmioRaw { + #[inline] + fn clone(&self) -> Self { + *self } +} +// SAFETY: `MmioRaw` is just an address, so is thread-safe. +unsafe impl Send for MmioRaw {} +// SAFETY: `MmioRaw` is just an address, so is thread-safe. +unsafe impl Sync for MmioRaw {} + +impl MmioRaw { + /// Create a `MmioRaw` from address. + #[inline] + pub fn new(addr: usize) -> Self { + Self { + ptr: core::ptr::without_provenance_mut(addr), + } + } +} + +impl MmioRaw> { + /// Create a `MmioRaw` representing a I/O region with given size. + /// + /// The size is checked against the minimum size specified via const generics. + #[inline] + pub fn new_region(addr: usize, size: usize) -> Result { + Ok(Self { + ptr: Region::ptr_try_from_raw_parts_mut(core::ptr::without_provenance_mut(addr), size)?, + }) + } +} + +impl MmioRaw { /// Returns the base address of the MMIO region. #[inline] pub fn addr(&self) -> usize { - self.addr + self.ptr.addr() } - /// Returns the maximum size of the MMIO region. + /// Returns the size of the MMIO region. #[inline] - pub fn maxsize(&self) -> usize { - self.maxsize + pub fn size(&self) -> usize { + KnownSize::size(self.ptr) } } @@ -144,12 +174,13 @@ pub fn maxsize(&self) -> usize { /// Mmio, /// MmioRaw, /// PhysAddr, +/// Region, /// }, /// }; /// use core::ops::Deref; /// /// // See also `pci::Bar` for a real example. -/// struct IoMem(MmioRaw); +/// struct IoMem(MmioRaw>); /// /// impl IoMem { /// /// # Safety @@ -164,7 +195,7 @@ pub fn maxsize(&self) -> usize { /// return Err(ENOMEM); /// } /// -/// Ok(IoMem(MmioRaw::new(addr as usize, SIZE)?)) +/// Ok(IoMem(MmioRaw::new_region(addr as usize, SIZE)?)) /// } /// } /// @@ -194,7 +225,7 @@ pub fn maxsize(&self) -> usize { /// # } /// ``` #[repr(transparent)] -pub struct Mmio(MmioRaw); +pub struct Mmio(MmioRaw>); /// Checks whether an access of type `U` at the given `base` and the given `offset` /// is valid within this region. @@ -841,7 +872,7 @@ fn addr(self) -> usize { /// Returns the maximum size of this mapping. #[inline] fn maxsize(self) -> usize { - self.0.maxsize() + self.0.size() } } @@ -852,7 +883,7 @@ impl Mmio { /// /// Callers must ensure that `addr` is the start of a valid I/O mapped memory region of size /// `maxsize`. - pub unsafe fn from_raw(raw: &MmioRaw) -> &Self { + pub unsafe fn from_raw(raw: &MmioRaw>) -> &Self { // SAFETY: `Mmio` is a transparent wrapper around `MmioRaw`. unsafe { &*core::ptr::from_ref(raw).cast() } } diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs index fc2a3e24f8d5..9e15bc8fde78 100644 --- a/rust/kernel/io/mem.rs +++ b/rust/kernel/io/mem.rs @@ -229,7 +229,7 @@ fn deref(&self) -> &Self::Target { /// start of the I/O memory mapped region. pub struct IoMem<'a, const SIZE: usize = 0> { dev: &'a Device, - io: MmioRaw, + io: MmioRaw>, } impl<'a, const SIZE: usize> IoMem<'a, SIZE> { @@ -264,8 +264,7 @@ fn ioremap(dev: &'a Device, resource: &Resource) -> Result { return Err(ENOMEM); } - let io = MmioRaw::new(addr as usize, size)?; - + let io = MmioRaw::new_region(addr as usize, size)?; Ok(IoMem { dev, io }) } diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs index 505305cd9b86..42f840d64a6f 100644 --- a/rust/kernel/pci/io.rs +++ b/rust/kernel/pci/io.rs @@ -139,7 +139,7 @@ fn maxsize(self) -> usize { /// memory mapped PCI BAR and its size. pub struct Bar<'a, const SIZE: usize = 0> { pdev: &'a Device, - io: MmioRaw, + io: MmioRaw>, num: i32, } @@ -179,7 +179,7 @@ pub(super) fn new( return Err(ENOMEM); } - let io = match MmioRaw::new(ioptr, len as usize) { + let io = match MmioRaw::new_region(ioptr, len as usize) { Ok(io) => io, Err(err) => { // SAFETY: From 691c75967d44bef006e4d4e783baa88470b33ea5 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 6 Jul 2026 13:44:19 +0100 Subject: [PATCH 046/137] rust: io: rename `Mmio` to `MmioOwned` Most users would more commonly reach out to a view of `Mmio` rather than an owned instance of `Mmio`. Only implementor of `Io` like `Bar` or `IoMem` would need the owned version. Thus, rename `Mmio` to `MmioOwned` so that the name `Mmio` can be used for the view type instead. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Reviewed-by: Daniel Almeida Suggested-by: Danilo Krummrich Link: https://rust-for-linux.zulipchat.com/#narrow/channel/288089-General/topic/Generic.20I.2FO.20backends/near/571198078 Link: https://patch.msgid.link/20260706-io_projection-v6-6-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/devres.rs | 6 +-- rust/kernel/io.rs | 77 +++++++++++++++++++------------------- rust/kernel/io/mem.rs | 8 ++-- rust/kernel/io/poll.rs | 8 ++-- rust/kernel/io/register.rs | 24 ++++++------ rust/kernel/pci/io.rs | 6 +-- 6 files changed, 65 insertions(+), 64 deletions(-) diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index d0c677fd7932..aed0c994fd30 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -68,7 +68,7 @@ struct Inner { /// devres::Devres, /// io::{ /// Io, -/// Mmio, +/// MmioOwned, /// MmioRaw, /// PhysAddr, /// Region, // @@ -105,11 +105,11 @@ struct Inner { /// } /// /// impl Deref for IoMem { -/// type Target = Mmio; +/// type Target = MmioOwned; /// /// fn deref(&self) -> &Self::Target { /// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`. -/// unsafe { Mmio::from_raw(&self.0) } +/// unsafe { MmioOwned::from_raw(&self.0) } /// } /// } /// # fn no_run(dev: &Device) -> Result<(), Error> { diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 3013d8cf39e4..ec4ac42aa25d 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -95,8 +95,8 @@ fn size(p: *const Self) -> usize { /// the represented MMIO region does exist or is properly mapped. /// /// Instead, the bus specific MMIO implementation must convert this raw representation into an -/// `Mmio` instance providing the actual memory accessors. Only by the conversion into an `Mmio` -/// structure any guarantees are given. +/// `MmioOwned` instance providing the actual memory accessors. Only by the conversion into an +/// `MmioOwned` structure any guarantees are given. pub struct MmioRaw { /// Pointer is in I/O address space. /// @@ -171,7 +171,7 @@ pub fn size(&self) -> usize { /// ffi::c_void, /// io::{ /// Io, -/// Mmio, +/// MmioOwned, /// MmioRaw, /// PhysAddr, /// Region, @@ -207,11 +207,11 @@ pub fn size(&self) -> usize { /// } /// /// impl Deref for IoMem { -/// type Target = Mmio; +/// type Target = MmioOwned; /// /// fn deref(&self) -> &Self::Target { /// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`. -/// unsafe { Mmio::from_raw(&self.0) } +/// unsafe { MmioOwned::from_raw(&self.0) } /// } /// } /// @@ -225,7 +225,7 @@ pub fn size(&self) -> usize { /// # } /// ``` #[repr(transparent)] -pub struct Mmio(MmioRaw>); +pub struct MmioOwned(MmioRaw>); /// Checks whether an access of type `U` at the given `base` and the given `offset` /// is valid within this region. @@ -538,10 +538,10 @@ fn write64(self, value: u64, offset: usize) /// ```no_run /// use kernel::io::{ /// Io, - /// Mmio, + /// MmioOwned, /// }; /// - /// fn do_reads(io: &Mmio) -> Result { + /// fn do_reads(io: &MmioOwned) -> Result { /// // 32-bit read from address `0x10`. /// let v: u32 = io.try_read(0x10)?; /// @@ -572,10 +572,10 @@ fn try_read(self, location: L) -> Result /// ```no_run /// use kernel::io::{ /// Io, - /// Mmio, + /// MmioOwned, /// }; /// - /// fn do_writes(io: &Mmio) -> Result { + /// fn do_writes(io: &MmioOwned) -> Result { /// // 32-bit write of value `1` at address `0x10`. /// io.try_write(0x10, 1u32)?; /// @@ -610,7 +610,7 @@ fn try_write(self, location: L, value: T) -> Result /// use kernel::io::{ /// register, /// Io, - /// Mmio, + /// MmioOwned, /// }; /// /// register! { @@ -626,7 +626,7 @@ fn try_write(self, location: L, value: T) -> Result /// } /// } /// - /// fn do_write_reg(io: &Mmio) -> Result { + /// fn do_write_reg(io: &MmioOwned) -> Result { /// /// io.try_write_reg(VERSION::new(1, 0)) /// } @@ -655,10 +655,10 @@ fn try_write_reg(self, value: V) -> Result /// ```no_run /// use kernel::io::{ /// Io, - /// Mmio, + /// MmioOwned, /// }; /// - /// fn do_update(io: &Mmio<0x1000>) -> Result { + /// fn do_update(io: &MmioOwned<0x1000>) -> Result { /// io.try_update(0x10, |v: u32| { /// v + 1 /// }) @@ -692,10 +692,10 @@ fn try_update(self, location: L, f: F) -> Result /// ```no_run /// use kernel::io::{ /// Io, - /// Mmio, + /// MmioOwned, /// }; /// - /// fn do_reads(io: &Mmio<0x1000>) { + /// fn do_reads(io: &MmioOwned<0x1000>) { /// // 32-bit read from address `0x10`. /// let v: u32 = io.read(0x10); /// @@ -724,10 +724,10 @@ fn read(self, location: L) -> T /// ```no_run /// use kernel::io::{ /// Io, - /// Mmio, + /// MmioOwned, /// }; /// - /// fn do_writes(io: &Mmio<0x1000>) { + /// fn do_writes(io: &MmioOwned<0x1000>) { /// // 32-bit write of value `1` at address `0x10`. /// io.write(0x10, 1u32); /// @@ -758,7 +758,7 @@ fn write(self, location: L, value: T) /// use kernel::io::{ /// register, /// Io, - /// Mmio, + /// MmioOwned, /// }; /// /// register! { @@ -774,7 +774,7 @@ fn write(self, location: L, value: T) /// } /// } /// - /// fn do_write_reg(io: &Mmio<0x1000>) { + /// fn do_write_reg(io: &MmioOwned<0x1000>) { /// io.write_reg(VERSION::new(1, 0)); /// } /// ``` @@ -802,10 +802,10 @@ fn write_reg(self, value: V) /// ```no_run /// use kernel::io::{ /// Io, - /// Mmio, + /// MmioOwned, /// }; /// - /// fn do_update(io: &Mmio<0x1000>) { + /// fn do_update(io: &MmioOwned<0x1000>) { /// io.update(0x10, |v: u32| { /// v + 1 /// }) @@ -848,19 +848,19 @@ unsafe fn io_write(self, value: $ty, address: usize) { } // MMIO regions support 8, 16, and 32-bit accesses. -impl_mmio_io_capable!(Mmio, u8, readb, writeb); -impl_mmio_io_capable!(Mmio, u16, readw, writew); -impl_mmio_io_capable!(Mmio, u32, readl, writel); +impl_mmio_io_capable!(MmioOwned, u8, readb, writeb); +impl_mmio_io_capable!(MmioOwned, u16, readw, writew); +impl_mmio_io_capable!(MmioOwned, u32, readl, writel); // MMIO regions on 64-bit systems also support 64-bit accesses. impl_mmio_io_capable!( - Mmio, + MmioOwned, #[cfg(CONFIG_64BIT)] u64, readq, writeq ); -impl<'a, const SIZE: usize> Io for &'a Mmio { +impl<'a, const SIZE: usize> Io for &'a MmioOwned { type Target = Region; /// Returns the base address of this mapping. @@ -876,27 +876,28 @@ fn maxsize(self) -> usize { } } -impl Mmio { - /// Converts an `MmioRaw` into an `Mmio` instance, providing the accessors to the MMIO mapping. +impl MmioOwned { + /// Converts an `MmioRaw` into an `MmioOwned` instance, providing the accessors to the MMIO + /// mapping. /// /// # Safety /// /// Callers must ensure that `addr` is the start of a valid I/O mapped memory region of size /// `maxsize`. pub unsafe fn from_raw(raw: &MmioRaw>) -> &Self { - // SAFETY: `Mmio` is a transparent wrapper around `MmioRaw`. + // SAFETY: `MmioOwned` is a transparent wrapper around `MmioRaw`. unsafe { &*core::ptr::from_ref(raw).cast() } } } -/// [`Mmio`] wrapper using relaxed accessors. +/// [`MmioOwned`] wrapper using relaxed accessors. /// /// This type provides an implementation of [`Io`] that uses relaxed I/O MMIO operands instead of /// the regular ones. /// -/// See [`Mmio::relaxed`] for a usage example. +/// See [`MmioOwned::relaxed`] for a usage example. #[repr(transparent)] -pub struct RelaxedMmio(Mmio); +pub struct RelaxedMmio(MmioOwned); impl<'a, const SIZE: usize> Io for &'a RelaxedMmio { type Target = Region; @@ -912,7 +913,7 @@ fn maxsize(self) -> usize { } } -impl Mmio { +impl MmioOwned { /// Returns a [`RelaxedMmio`] reference that performs relaxed I/O operations. /// /// Relaxed accessors do not provide ordering guarantees with respect to DMA or memory accesses @@ -923,19 +924,19 @@ impl Mmio { /// ```no_run /// use kernel::io::{ /// Io, - /// Mmio, + /// MmioOwned, /// RelaxedMmio, /// }; /// - /// fn do_io(io: &Mmio<0x100>) { + /// fn do_io(io: &MmioOwned<0x100>) { /// // The access is performed using `readl_relaxed` instead of `readl`. /// let v = io.relaxed().read32(0x10); /// } /// /// ``` pub fn relaxed(&self) -> &RelaxedMmio { - // SAFETY: `RelaxedMmio` is `#[repr(transparent)]` over `Mmio`, so `Mmio` and - // `RelaxedMmio` have identical layout. + // SAFETY: `RelaxedMmio` is `#[repr(transparent)]` over `MmioOwned`, so `MmioOwned` + // and `RelaxedMmio` have identical layout. unsafe { core::mem::transmute(self) } } } diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs index 9e15bc8fde78..8f6c257c5b8e 100644 --- a/rust/kernel/io/mem.rs +++ b/rust/kernel/io/mem.rs @@ -16,7 +16,7 @@ Region, Resource, // }, - Mmio, + MmioOwned, MmioRaw, // }, prelude::*, @@ -211,7 +211,7 @@ pub fn into_devres(self) -> Result>> { } impl Deref for ExclusiveIoMem<'_, SIZE> { - type Target = Mmio; + type Target = MmioOwned; fn deref(&self) -> &Self::Target { &self.iomem @@ -291,10 +291,10 @@ fn drop(&mut self) { } impl Deref for IoMem<'_, SIZE> { - type Target = Mmio; + type Target = MmioOwned; fn deref(&self) -> &Self::Target { // SAFETY: Safe as by the invariant of `IoMem`. - unsafe { Mmio::from_raw(&self.io) } + unsafe { MmioOwned::from_raw(&self.io) } } } diff --git a/rust/kernel/io/poll.rs b/rust/kernel/io/poll.rs index 75d1b3e8596c..79828a8006b5 100644 --- a/rust/kernel/io/poll.rs +++ b/rust/kernel/io/poll.rs @@ -47,14 +47,14 @@ /// ```no_run /// use kernel::io::{ /// Io, -/// Mmio, +/// MmioOwned, /// poll::read_poll_timeout, // /// }; /// use kernel::time::Delta; /// /// const HW_READY: u16 = 0x01; /// -/// fn wait_for_hardware(io: &Mmio) -> Result { +/// fn wait_for_hardware(io: &MmioOwned) -> Result { /// read_poll_timeout( /// // The `op` closure reads the value of a specific status register. /// || io.try_read16(0x1000), @@ -134,14 +134,14 @@ pub fn read_poll_timeout( /// ```no_run /// use kernel::io::{ /// Io, -/// Mmio, +/// MmioOwned, /// poll::read_poll_timeout_atomic, // /// }; /// use kernel::time::Delta; /// /// const HW_READY: u16 = 0x01; /// -/// fn wait_for_hardware(io: &Mmio) -> Result { +/// fn wait_for_hardware(io: &MmioOwned) -> Result { /// read_poll_timeout_atomic( /// // The `op` closure reads the value of a specific status register. /// || io.try_read16(0x1000), diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index 3122b17098ee..43284d9fba96 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -58,7 +58,7 @@ //! }, //! num::Bounded, //! }; -//! # use kernel::io::Mmio; +//! # use kernel::io::MmioOwned; //! # register! { //! # pub BOOT_0(u32) @ 0x00000100 { //! # 15:8 vendor_id; @@ -66,7 +66,7 @@ //! # 3:0 minor_revision; //! # } //! # } -//! # fn test(io: &Mmio<0x1000>) { +//! # fn test(io: &MmioOwned<0x1000>) { //! # fn obtain_vendor_id() -> u8 { 0xff } //! //! // Read from the register's defined offset (0x100). @@ -446,7 +446,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// Io, /// }, /// }; -/// # use kernel::io::Mmio; +/// # use kernel::io::MmioOwned; /// /// register! { /// FIXED_REG(u32) @ 0x100 { @@ -455,7 +455,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// } /// } /// -/// # fn test(io: &Mmio<0x1000>) { +/// # fn test(io: &MmioOwned<0x1000>) { /// let val = io.read(FIXED_REG); /// /// // Write from an already-existing value. @@ -559,7 +559,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// Io, /// }, /// }; -/// # use kernel::io::Mmio; +/// # use kernel::io::MmioOwned; /// /// // Type used to identify the base. /// pub struct CpuCtlBase; @@ -584,7 +584,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// } /// } /// -/// # fn test(io: Mmio<0x1000>) { +/// # fn test(io: MmioOwned<0x1000>) { /// // Read the status of `Cpu0`. /// let cpu0_started = io.read(CPU_CTL::of::()); /// @@ -601,7 +601,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// } /// } /// -/// # fn test2(io: Mmio<0x1000>) { +/// # fn test2(io: MmioOwned<0x1000>) { /// // Start the aliased `CPU0`, leaving its other fields untouched. /// io.update(CPU_CTL_ALIAS::of::(), |r| r.with_alias_start(true)); /// # } @@ -638,7 +638,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// Io, /// }, /// }; -/// # use kernel::io::Mmio; +/// # use kernel::io::MmioOwned; /// # fn get_scratch_idx() -> usize { /// # 0x15 /// # } @@ -651,7 +651,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// } /// } /// -/// # fn test(io: &Mmio<0x1000>) +/// # fn test(io: &MmioOwned<0x1000>) /// # -> Result<(), Error>{ /// // Read scratch register 0, i.e. I/O address `0x80`. /// let scratch_0 = io.read(SCRATCH::at(0)).value(); @@ -724,7 +724,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// Io, /// }, /// }; -/// # use kernel::io::Mmio; +/// # use kernel::io::MmioOwned; /// # fn get_scratch_idx() -> usize { /// # 0x15 /// # } @@ -752,7 +752,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// } /// } /// -/// # fn test(io: &Mmio<0x1000>) -> Result<(), Error> { +/// # fn test(io: &MmioOwned<0x1000>) -> Result<(), Error> { /// // Read scratch register 0 of CPU0. /// let scratch = io.read(CPU_SCRATCH::of::().at(0)); /// @@ -794,7 +794,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// } /// } /// -/// # fn test2(io: &Mmio<0x1000>) -> Result<(), Error> { +/// # fn test2(io: &MmioOwned<0x1000>) -> Result<(), Error> { /// let cpu0_status = io.read(CPU_FIRMWARE_STATUS::of::()).status(); /// # Ok(()) /// # } diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs index 42f840d64a6f..e0acb62f58a2 100644 --- a/rust/kernel/pci/io.rs +++ b/rust/kernel/pci/io.rs @@ -10,7 +10,7 @@ io::{ Io, IoCapable, - Mmio, + MmioOwned, MmioRaw, Region, // }, @@ -242,11 +242,11 @@ fn drop(&mut self) { } impl Deref for Bar<'_, SIZE> { - type Target = Mmio; + type Target = MmioOwned; fn deref(&self) -> &Self::Target { // SAFETY: By the type invariant of `Self`, the MMIO range in `self.io` is properly mapped. - unsafe { Mmio::from_raw(&self.io) } + unsafe { MmioOwned::from_raw(&self.io) } } } From 9f64c84af008b8e01309ec6fe04bc772fbf23ea5 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 6 Jul 2026 13:44:20 +0100 Subject: [PATCH 047/137] rust: io: implement `Mmio` as view type Implement `Mmio` as view type and convert `RelaxedMmio` to view type as well. I/O implementations of `MmioOwned` are changed to delegate to the `Mmio` view type. All existing users of `MmioOwned` in the documentation which do not actually reflect the owning semantics is converted. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Tested-by: Daniel Almeida Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260706-io_projection-v6-7-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 176 ++++++++++++++++++++++++++++--------- rust/kernel/io/poll.rs | 10 ++- rust/kernel/io/register.rs | 24 ++--- 3 files changed, 153 insertions(+), 57 deletions(-) diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index ec4ac42aa25d..081b4613bc12 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -4,6 +4,10 @@ //! //! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h) +use core::{ + marker::PhantomData, // +}; + use crate::{ bindings, prelude::*, @@ -538,10 +542,11 @@ fn write64(self, value: u64, offset: usize) /// ```no_run /// use kernel::io::{ /// Io, - /// MmioOwned, + /// Mmio, + /// Region, /// }; /// - /// fn do_reads(io: &MmioOwned) -> Result { + /// fn do_reads(io: Mmio<'_, Region>) -> Result { /// // 32-bit read from address `0x10`. /// let v: u32 = io.try_read(0x10)?; /// @@ -572,10 +577,11 @@ fn try_read(self, location: L) -> Result /// ```no_run /// use kernel::io::{ /// Io, - /// MmioOwned, + /// Mmio, + /// Region, /// }; /// - /// fn do_writes(io: &MmioOwned) -> Result { + /// fn do_writes(io: Mmio<'_, Region>) -> Result { /// // 32-bit write of value `1` at address `0x10`. /// io.try_write(0x10, 1u32)?; /// @@ -610,7 +616,8 @@ fn try_write(self, location: L, value: T) -> Result /// use kernel::io::{ /// register, /// Io, - /// MmioOwned, + /// Mmio, + /// Region, /// }; /// /// register! { @@ -626,7 +633,7 @@ fn try_write(self, location: L, value: T) -> Result /// } /// } /// - /// fn do_write_reg(io: &MmioOwned) -> Result { + /// fn do_write_reg(io: Mmio<'_, Region>) -> Result { /// /// io.try_write_reg(VERSION::new(1, 0)) /// } @@ -655,10 +662,11 @@ fn try_write_reg(self, value: V) -> Result /// ```no_run /// use kernel::io::{ /// Io, - /// MmioOwned, + /// Mmio, + /// Region, /// }; /// - /// fn do_update(io: &MmioOwned<0x1000>) -> Result { + /// fn do_update(io: Mmio<'_, Region<0x1000>>) -> Result { /// io.try_update(0x10, |v: u32| { /// v + 1 /// }) @@ -692,10 +700,11 @@ fn try_update(self, location: L, f: F) -> Result /// ```no_run /// use kernel::io::{ /// Io, - /// MmioOwned, + /// Mmio, + /// Region, /// }; /// - /// fn do_reads(io: &MmioOwned<0x1000>) { + /// fn do_reads(io: Mmio<'_, Region<0x1000>>) { /// // 32-bit read from address `0x10`. /// let v: u32 = io.read(0x10); /// @@ -724,10 +733,11 @@ fn read(self, location: L) -> T /// ```no_run /// use kernel::io::{ /// Io, - /// MmioOwned, + /// Mmio, + /// Region, /// }; /// - /// fn do_writes(io: &MmioOwned<0x1000>) { + /// fn do_writes(io: Mmio<'_, Region<0x1000>>) { /// // 32-bit write of value `1` at address `0x10`. /// io.write(0x10, 1u32); /// @@ -758,7 +768,8 @@ fn write(self, location: L, value: T) /// use kernel::io::{ /// register, /// Io, - /// MmioOwned, + /// Mmio, + /// Region, /// }; /// /// register! { @@ -774,7 +785,7 @@ fn write(self, location: L, value: T) /// } /// } /// - /// fn do_write_reg(io: &MmioOwned<0x1000>) { + /// fn do_write_reg(io: Mmio<'_, Region<0x1000>>) { /// io.write_reg(VERSION::new(1, 0)); /// } /// ``` @@ -802,10 +813,11 @@ fn write_reg(self, value: V) /// ```no_run /// use kernel::io::{ /// Io, - /// MmioOwned, + /// Mmio, + /// Region, /// }; /// - /// fn do_update(io: &MmioOwned<0x1000>) { + /// fn do_update(io: Mmio<'_, Region<0x1000>>) { /// io.update(0x10, |v: u32| { /// v + 1 /// }) @@ -829,16 +841,72 @@ fn update(self, location: L, f: F) } } +/// A view of memory-mapped I/O region. +/// +/// # Invariant +/// +/// `ptr` points to a valid and aligned memory-mapped I/O region for the duration lifetime `'a`. +pub struct Mmio<'a, T: ?Sized> { + ptr: *mut T, + phantom: PhantomData<&'a ()>, +} + +impl Copy for Mmio<'_, T> {} +impl Clone for Mmio<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T: ?Sized> Mmio<'a, T> { + /// Create a `Mmio`, providing the accessors to the MMIO mapping. + /// + /// # Safety + /// + /// `raw` represents a valid and aligned memory-mapped I/O region while `'a` is alive. + #[inline] + pub unsafe fn from_raw(raw: MmioRaw) -> Self { + // INVARIANT: Per safety requirement. + Self { + ptr: raw.ptr, + phantom: PhantomData, + } + } +} + +// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl Send for Mmio<'_, T> {} + +// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl Sync for Mmio<'_, T> {} + +impl Io for Mmio<'_, T> { + type Target = T; + + #[inline] + fn addr(self) -> usize { + self.ptr.addr() + } + + #[inline] + fn maxsize(self) -> usize { + KnownSize::size(self.ptr) + } +} + /// Implements [`IoCapable`] on `$mmio` for `$ty` using `$read_fn` and `$write_fn`. macro_rules! impl_mmio_io_capable { ($mmio:ident, $(#[$attr:meta])* $ty:ty, $read_fn:ident, $write_fn:ident) => { $(#[$attr])* - impl IoCapable<$ty> for &$mmio { + impl IoCapable<$ty> for $mmio<'_, T> { + #[inline] unsafe fn io_read(self, address: usize) -> $ty { // SAFETY: By the trait invariant `address` is a valid address for MMIO operations. unsafe { bindings::$read_fn(address as *const c_void) } } + #[inline] unsafe fn io_write(self, value: $ty, address: usize) { // SAFETY: By the trait invariant `address` is a valid address for MMIO operations. unsafe { bindings::$write_fn(value, address as *mut c_void) } @@ -848,17 +916,12 @@ unsafe fn io_write(self, value: $ty, address: usize) { } // MMIO regions support 8, 16, and 32-bit accesses. -impl_mmio_io_capable!(MmioOwned, u8, readb, writeb); -impl_mmio_io_capable!(MmioOwned, u16, readw, writew); -impl_mmio_io_capable!(MmioOwned, u32, readl, writel); +impl_mmio_io_capable!(Mmio, u8, readb, writeb); +impl_mmio_io_capable!(Mmio, u16, readw, writew); +impl_mmio_io_capable!(Mmio, u32, readl, writel); // MMIO regions on 64-bit systems also support 64-bit accesses. -impl_mmio_io_capable!( - MmioOwned, - #[cfg(CONFIG_64BIT)] - u64, - readq, - writeq -); +#[cfg(CONFIG_64BIT)] +impl_mmio_io_capable!(Mmio, u64, readq, writeq); impl<'a, const SIZE: usize> Io for &'a MmioOwned { type Target = Region; @@ -876,6 +939,23 @@ fn maxsize(self) -> usize { } } +impl<'a, const SIZE: usize, T> IoCapable for &'a MmioOwned +where + Mmio<'a, Region>: IoCapable, +{ + #[inline] + unsafe fn io_read(self, address: usize) -> T { + // SAFETY: Per safety requirement. + unsafe { self.as_view().io_read(address) } + } + + #[inline] + unsafe fn io_write(self, value: T, address: usize) { + // SAFETY: Per safety requirement. + unsafe { self.as_view().io_write(value, address) } + } +} + impl MmioOwned { /// Converts an `MmioRaw` into an `MmioOwned` instance, providing the accessors to the MMIO /// mapping. @@ -888,19 +968,33 @@ pub unsafe fn from_raw(raw: &MmioRaw>) -> &Self { // SAFETY: `MmioOwned` is a transparent wrapper around `MmioRaw`. unsafe { &*core::ptr::from_ref(raw).cast() } } + + /// Return a view that covers the full region. + #[inline] + pub fn as_view(&self) -> Mmio<'_, Region> { + // SAFETY: `Mmio` has same invariant as `MmioOwned`. + unsafe { Mmio::from_raw(self.0) } + } } -/// [`MmioOwned`] wrapper using relaxed accessors. +/// [`Mmio`] but using relaxed accessors. /// /// This type provides an implementation of [`Io`] that uses relaxed I/O MMIO operands instead of /// the regular ones. /// -/// See [`MmioOwned::relaxed`] for a usage example. -#[repr(transparent)] -pub struct RelaxedMmio(MmioOwned); +/// See [`Mmio::relaxed`] for a usage example. +pub struct RelaxedMmio<'a, T: ?Sized>(Mmio<'a, T>); -impl<'a, const SIZE: usize> Io for &'a RelaxedMmio { - type Target = Region; +impl Copy for RelaxedMmio<'_, T> {} +impl Clone for RelaxedMmio<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl Io for RelaxedMmio<'_, T> { + type Target = T; #[inline] fn addr(self) -> usize { @@ -913,8 +1007,8 @@ fn maxsize(self) -> usize { } } -impl MmioOwned { - /// Returns a [`RelaxedMmio`] reference that performs relaxed I/O operations. +impl<'a, T: ?Sized> Mmio<'a, T> { + /// Returns a [`RelaxedMmio`] that performs relaxed I/O operations. /// /// Relaxed accessors do not provide ordering guarantees with respect to DMA or memory accesses /// and can be used when such ordering is not required. @@ -924,20 +1018,20 @@ impl MmioOwned { /// ```no_run /// use kernel::io::{ /// Io, - /// MmioOwned, + /// Mmio, + /// Region, /// RelaxedMmio, /// }; /// - /// fn do_io(io: &MmioOwned<0x100>) { + /// fn do_io(io: Mmio<'_, Region<0x100>>) { /// // The access is performed using `readl_relaxed` instead of `readl`. /// let v = io.relaxed().read32(0x10); /// } /// /// ``` - pub fn relaxed(&self) -> &RelaxedMmio { - // SAFETY: `RelaxedMmio` is `#[repr(transparent)]` over `MmioOwned`, so `MmioOwned` - // and `RelaxedMmio` have identical layout. - unsafe { core::mem::transmute(self) } + #[inline] + pub fn relaxed(self) -> RelaxedMmio<'a, T> { + RelaxedMmio(self) } } diff --git a/rust/kernel/io/poll.rs b/rust/kernel/io/poll.rs index 79828a8006b5..d75f2fcf46f2 100644 --- a/rust/kernel/io/poll.rs +++ b/rust/kernel/io/poll.rs @@ -47,14 +47,15 @@ /// ```no_run /// use kernel::io::{ /// Io, -/// MmioOwned, +/// Mmio, +/// Region, /// poll::read_poll_timeout, // /// }; /// use kernel::time::Delta; /// /// const HW_READY: u16 = 0x01; /// -/// fn wait_for_hardware(io: &MmioOwned) -> Result { +/// fn wait_for_hardware(io: Mmio<'_, Region>) -> Result { /// read_poll_timeout( /// // The `op` closure reads the value of a specific status register. /// || io.try_read16(0x1000), @@ -134,14 +135,15 @@ pub fn read_poll_timeout( /// ```no_run /// use kernel::io::{ /// Io, -/// MmioOwned, +/// Mmio, +/// Region, /// poll::read_poll_timeout_atomic, // /// }; /// use kernel::time::Delta; /// /// const HW_READY: u16 = 0x01; /// -/// fn wait_for_hardware(io: &MmioOwned) -> Result { +/// fn wait_for_hardware(io: Mmio<'_, Region>) -> Result { /// read_poll_timeout_atomic( /// // The `op` closure reads the value of a specific status register. /// || io.try_read16(0x1000), diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index 43284d9fba96..80e638a892d7 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -58,7 +58,7 @@ //! }, //! num::Bounded, //! }; -//! # use kernel::io::MmioOwned; +//! # use kernel::io::{Mmio, Region}; //! # register! { //! # pub BOOT_0(u32) @ 0x00000100 { //! # 15:8 vendor_id; @@ -66,7 +66,7 @@ //! # 3:0 minor_revision; //! # } //! # } -//! # fn test(io: &MmioOwned<0x1000>) { +//! # fn test(io: Mmio<'_, Region<0x1000>>) { //! # fn obtain_vendor_id() -> u8 { 0xff } //! //! // Read from the register's defined offset (0x100). @@ -446,7 +446,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// Io, /// }, /// }; -/// # use kernel::io::MmioOwned; +/// # use kernel::io::{Mmio, Region}; /// /// register! { /// FIXED_REG(u32) @ 0x100 { @@ -455,7 +455,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// } /// } /// -/// # fn test(io: &MmioOwned<0x1000>) { +/// # fn test(io: Mmio<'_, Region<0x1000>>) { /// let val = io.read(FIXED_REG); /// /// // Write from an already-existing value. @@ -559,7 +559,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// Io, /// }, /// }; -/// # use kernel::io::MmioOwned; +/// # use kernel::io::{Mmio, Region}; /// /// // Type used to identify the base. /// pub struct CpuCtlBase; @@ -584,7 +584,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// } /// } /// -/// # fn test(io: MmioOwned<0x1000>) { +/// # fn test(io: Mmio<'_, Region<0x1000>>) { /// // Read the status of `Cpu0`. /// let cpu0_started = io.read(CPU_CTL::of::()); /// @@ -601,7 +601,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// } /// } /// -/// # fn test2(io: MmioOwned<0x1000>) { +/// # fn test2(io: Mmio<'_, Region<0x1000>>) { /// // Start the aliased `CPU0`, leaving its other fields untouched. /// io.update(CPU_CTL_ALIAS::of::(), |r| r.with_alias_start(true)); /// # } @@ -638,7 +638,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// Io, /// }, /// }; -/// # use kernel::io::MmioOwned; +/// # use kernel::io::{Mmio, Region}; /// # fn get_scratch_idx() -> usize { /// # 0x15 /// # } @@ -651,7 +651,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// } /// } /// -/// # fn test(io: &MmioOwned<0x1000>) +/// # fn test(io: Mmio<'_, Region<0x1000>>) /// # -> Result<(), Error>{ /// // Read scratch register 0, i.e. I/O address `0x80`. /// let scratch_0 = io.read(SCRATCH::at(0)).value(); @@ -724,7 +724,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// Io, /// }, /// }; -/// # use kernel::io::MmioOwned; +/// # use kernel::io::{Mmio, Region}; /// # fn get_scratch_idx() -> usize { /// # 0x15 /// # } @@ -752,7 +752,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// } /// } /// -/// # fn test(io: &MmioOwned<0x1000>) -> Result<(), Error> { +/// # fn test(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { /// // Read scratch register 0 of CPU0. /// let scratch = io.read(CPU_SCRATCH::of::().at(0)); /// @@ -794,7 +794,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// } /// } /// -/// # fn test2(io: &MmioOwned<0x1000>) -> Result<(), Error> { +/// # fn test2(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { /// let cpu0_status = io.read(CPU_FIRMWARE_STATUS::of::()).status(); /// # Ok(()) /// # } From 6e5f28968d7b3f2137e99528579e6109acb4d4c8 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 6 Jul 2026 13:44:21 +0100 Subject: [PATCH 048/137] rust: pci: io: make `ConfigSpace` a view In order to support I/O projection, we are splitting I/O types into two categories: owned objects and views. Owned objects have a specific type that is related to setting up and tearing down, while views can have their type changed with I/O projection. Things like `IoMem` or `Bar` are owned objects, which requires setting up mapping and cleaning up on drop. On the other side, `ConfigSpace` is really just a view, as the resource is associated with the `pci::Device`. Remove the `ConfigSpaceKind` bound on `ConfigSpace` and make it a generic view. This means that `ConfigSpace` object now represents a subregion and therefore encodes offset (as address of pointers) and size (as metadata of pointers) itself. The full region case is still supported with offset 0 and size of `cfg_size`. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260706-io_projection-v6-8-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/pci/io.rs | 64 ++++++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs index e0acb62f58a2..89f4bb483a7f 100644 --- a/rust/kernel/pci/io.rs +++ b/rust/kernel/pci/io.rs @@ -18,7 +18,6 @@ ptr::KnownSize, // }; use core::{ - marker::PhantomData, ops::Deref, // }; @@ -53,33 +52,42 @@ pub const fn into_raw(self) -> usize { /// Alias for extended (4096-byte) PCIe configuration space. pub type Extended = Region<4096>; -/// Trait for PCI configuration space size markers. -/// -/// This trait is implemented by [`Normal`] and [`Extended`] to provide -/// compile-time knowledge of the configuration space size. -pub trait ConfigSpaceKind: KnownSize {} - -impl ConfigSpaceKind for Normal {} - -impl ConfigSpaceKind for Extended {} - -/// The PCI configuration space of a device. +/// A view of PCI configuration space of a device. /// /// Provides typed read and write accessors for configuration registers /// using the standard `pci_read_config_*` and `pci_write_config_*` helpers. /// -/// The generic parameter `S` indicates the maximum size of the configuration space. -/// Use [`Normal`] for 256-byte legacy configuration space or [`Extended`] for -/// 4096-byte PCIe extended configuration space (default). -pub struct ConfigSpace<'a, S: ?Sized + ConfigSpaceKind = Extended> { +/// The generic parameter `T` is the type of the view. The full configuration space is also a +/// special type of view; in such cases, `T` can be [`Normal`] for 256-byte legacy configuration +/// space or [`Extended`] for 4096-byte PCIe extended configuration space (default). +/// +/// # Invariants +/// +/// `ptr` is aligned and range `ptr..ptr + KnownSize::size(ptr)` is within +/// `0..pdev.cfg_size().into_raw()`. +pub struct ConfigSpace<'a, T: ?Sized = Extended> { pub(crate) pdev: &'a Device, - _marker: PhantomData, + ptr: *mut T, } +impl Copy for ConfigSpace<'_, T> {} +impl Clone for ConfigSpace<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +// SAFETY: `ConfigSpace<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl Send for ConfigSpace<'_, T> {} + +// SAFETY: `ConfigSpace<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl Sync for ConfigSpace<'_, T> {} + /// Implements [`IoCapable`] on [`ConfigSpace`] for `$ty` using `$read_fn` and `$write_fn`. macro_rules! impl_config_space_io_capable { ($ty:ty, $read_fn:ident, $write_fn:ident) => { - impl<'a, S: ?Sized + ConfigSpaceKind> IoCapable<$ty> for &ConfigSpace<'a, S> { + impl<'a, T: ?Sized> IoCapable<$ty> for ConfigSpace<'a, T> { unsafe fn io_read(self, address: usize) -> $ty { let mut val: $ty = 0; @@ -112,19 +120,17 @@ unsafe fn io_write(self, value: $ty, address: usize) { impl_config_space_io_capable!(u16, pci_read_config_word, pci_write_config_word); impl_config_space_io_capable!(u32, pci_read_config_dword, pci_write_config_dword); -impl<'a, S: ?Sized + ConfigSpaceKind> Io for &ConfigSpace<'a, S> { - type Target = S; +impl<'a, T: ?Sized + KnownSize> Io for ConfigSpace<'a, T> { + type Target = T; - /// Returns the base address of the I/O region. It is always 0 for configuration space. #[inline] fn addr(self) -> usize { - 0 + self.ptr.addr() } - /// Returns the maximum size of the configuration space. #[inline] fn maxsize(self) -> usize { - self.pdev.cfg_size().into_raw() + KnownSize::size(self.ptr) } } @@ -281,23 +287,25 @@ pub fn cfg_size(&self) -> ConfigSpaceSize { } } - /// Return an initialized normal (256-byte) config space object. + /// Return a view of the normal (256-byte) config space. pub fn config_space<'a>(&'a self) -> ConfigSpace<'a, Normal> { + // INVARIANT: null is aligned and the range is within config space. ConfigSpace { pdev: self, - _marker: PhantomData, + ptr: Normal::ptr_from_raw_parts_mut(core::ptr::null_mut(), self.cfg_size().into_raw()), } } - /// Return an initialized extended (4096-byte) config space object. + /// Return a view of the extended (4096-byte) config space. pub fn config_space_extended<'a>(&'a self) -> Result> { if self.cfg_size() != ConfigSpaceSize::Extended { return Err(EINVAL); } + // INVARIANT: null is aligned and we just checked the `cfg_size`. Ok(ConfigSpace { pdev: self, - _marker: PhantomData, + ptr: Extended::ptr_from_raw_parts_mut(core::ptr::null_mut(), 4096), }) } } From e0454ec1220c29178c13c209197f29f29e324d7f Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 6 Jul 2026 13:44:22 +0100 Subject: [PATCH 049/137] rust: io: use view types instead of addresses for `Io` Currently, `io_read` and `io_write` methods require the exact type of `Io` plus an address. This means that they need to be monomorphized for each different `Io` instance. This also means that multiple I/O implementors for the same I/O kind needs to duplicate implementation (e.g. `Mmio` and `MmioOwned`). Create a new `IoBackend` trait and define these operations on it instead. The operations are just going to receive a view type and operate on them. This has the additional advantage that the invariants can be moved from the trait (and guaranteed via `unsafe`) to type invariants on the canonical view types of the backends, so `io_read` and `io_write` can be safe. Note that a view type is needed; addresses are insufficient in this design, as they do not carry sufficient information. For example, `ConfigSpace` needs `&pci::Device` in addition to the address. `io_addr_assert` and `io_addr` are renamed to `io_view*` to reflect that they operate on views now, and make them standalone functions so they cannot be used by users to cast types outside io.rs. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260706-io_projection-v6-9-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 388 ++++++++++++++++++++++-------------------- rust/kernel/pci/io.rs | 70 +++++--- 2 files changed, 249 insertions(+), 209 deletions(-) diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 081b4613bc12..82a3369ae110 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -244,6 +244,81 @@ const fn offset_valid(base: usize, offset: usize, size: usize) -> bool { } } +/// Returns a view for a given `offset`, performing compile-time bound checks. +// Always inline to optimize out error path of `build_assert`. +#[inline(always)] +fn io_view_assert<'a, IO: Io<'a>, U>( + this: IO, + offset: usize, +) -> ::View<'a, U> { + // We cannot check alignment with `offset_valid` using `ptr.addr()`. So set 0 for it and + // ensure alignment by checking that the alignment of `U` is smaller or equal to the + // alignment of `IO::Target`. + const_assert!(Alignment::of::().as_usize() <= IO::Target::MIN_ALIGN.as_usize()); + build_assert!(offset_valid::(0, offset, IO::Target::MIN_SIZE)); + + let view = this.as_view(); + let ptr = IO::Backend::as_ptr(view); + let projected_ptr = ptr.cast::().wrapping_byte_add(offset); + // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a + // valid projection. + unsafe { IO::Backend::project_view(view, projected_ptr) } +} + +/// Returns a view for a given `offset`, performing runtime bound checks. +#[inline] +fn io_view<'a, IO: Io<'a>, U>( + this: IO, + offset: usize, +) -> Result<::View<'a, U>> { + let view = this.as_view(); + let ptr = IO::Backend::as_ptr(view); + + if !offset_valid::(ptr.addr(), offset, KnownSize::size(ptr)) { + return Err(EINVAL); + } + + let projected_ptr = ptr.cast::().wrapping_byte_add(offset); + // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a + // valid projection. + Ok(unsafe { IO::Backend::project_view(view, projected_ptr) }) +} + +/// I/O backends. +/// +/// This is an abstract representation to be implemented by arbitrary I/O +/// backends (e.g. MMIO, PCI config space, etc.). +/// +/// The base trait only defines the projection operations; which I/O methods are available depends +/// on which [`IoCapable`] traits are implemented for the type. For example, for MMIO regions, +/// all widths (u8, u16, u32, and u64 on 64-bit systems) are typically supported. For PCI +/// configuration space, u8, u16, and u32 are supported but u64 is not. +/// +/// This trait is separate from the `Io` trait as multiple different I/O types may share the same +/// operation. +pub trait IoBackend { + /// View type for this I/O backend. + type View<'a, T: ?Sized + KnownSize>: Io<'a, Backend = Self, Target = T>; + + /// Convert a `view` to a raw pointer for projection. + /// + /// The returned pointer is private implementation detail of the backend; it is likely not + /// valid. It should not be dereferenced. + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T; + + /// Project `view` to its subregion indicated by `ptr`. + /// + /// If input `view` is valid, returned view must also be valid. + /// + /// # Safety + /// + /// `ptr` must be a projection of `Self::as_ptr(view)`. + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U>; +} + /// Trait indicating that an I/O backend supports operations of a certain type and providing an /// implementation for these operations. /// @@ -252,22 +327,12 @@ const fn offset_valid(base: usize, offset: usize, size: usize) -> bool { /// For example, a PCI configuration space may implement `IoCapable`, `IoCapable`, /// and `IoCapable`, but not `IoCapable`, while an MMIO region on a 64-bit /// system might implement all four. -pub trait IoCapable { - /// Performs an I/O read of type `T` at `address` and returns the result. - /// - /// # Safety - /// - /// - The range `[address..address + size_of::()]` must be within the bounds of `Self`. - /// - `address` must be aligned. - unsafe fn io_read(self, address: usize) -> T; +pub trait IoCapable: IoBackend { + /// Performs an I/O read of type `T` at `view` and returns the result. + fn io_read<'a>(view: Self::View<'a, T>) -> T; - /// Performs an I/O write of `value` at `address`. - /// - /// # Safety - /// - /// - The range `[address..address + size_of::()]` must be within the bounds of `Self`. - /// - `address` must be aligned. - unsafe fn io_write(self, value: T, address: usize); + /// Performs an I/O write of `value` at `view`. + fn io_write<'a>(view: Self::View<'a, T>, value: T); } /// Describes a given I/O location: its offset, width, and type to convert the raw value from and @@ -319,66 +384,30 @@ fn offset(self) -> usize { /// Types implementing this trait (e.g. MMIO BARs or PCI config regions) /// can perform I/O operations on regions of memory. /// -/// This is an abstract representation to be implemented by arbitrary I/O -/// backends (e.g. MMIO, PCI config space, etc.). -/// /// The [`Io`] trait provides: -/// - Base address and size information +/// - Method to convert into [`IoBackend::View`]. /// - Helper methods for offset validation and address calculation /// - Fallible (runtime checked) accessors for different data widths /// -/// Which I/O methods are available depends on which [`IoCapable`] traits -/// are implemented for the type. +/// Which I/O methods are available depends on the associated [`IoBackend`] implementation. /// /// This should be implemented on cheaply copyable handles, such as references or view types. -/// -/// # Examples -/// -/// For MMIO regions, all widths (u8, u16, u32, and u64 on 64-bit systems) are typically -/// supported. For PCI configuration space, u8, u16, and u32 are supported but u64 is not. -pub trait Io: Copy { +pub trait Io<'a>: Copy { + /// Type that defines all I/O operations. + type Backend: IoBackend; + /// Type of this I/O region. For untyped regions, [`Region`] can be used. type Target: ?Sized + KnownSize; - /// Returns the base address of this mapping. - fn addr(self) -> usize; - - /// Returns the maximum size of this mapping. - fn maxsize(self) -> usize; - - /// Returns the absolute I/O address for a given `offset`, - /// performing compile-time bound checks. - // Always inline to optimize out error path of `build_assert`. - #[inline(always)] - fn io_addr_assert(self, offset: usize) -> usize { - // We cannot check alignment with `offset_valid` using `self.addr()`. So set 0 for it and - // ensure alignment by checking that the alignment of `U` is smaller or equal to the - // alignment of `Self::Target`. - const_assert!(Alignment::of::().as_usize() <= Self::Target::MIN_ALIGN.as_usize()); - build_assert!(offset_valid::(0, offset, Self::Target::MIN_SIZE)); - - self.addr() + offset - } - - /// Returns the absolute I/O address for a given `offset`, - /// performing runtime bound checks. - #[inline] - fn io_addr(self, offset: usize) -> Result { - if !offset_valid::(self.addr(), offset, self.maxsize()) { - return Err(EINVAL); - } - - // Probably no need to check, since the safety requirements of `Self::new` guarantee that - // this can't overflow. - self.addr().checked_add(offset).ok_or(EINVAL) - } + /// Return a view that covers the full region. + fn as_view(self) -> ::View<'a, Self::Target>; /// Fallible 8-bit read with runtime bounds check. #[inline(always)] fn try_read8(self, offset: usize) -> Result where usize: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { self.try_read(offset) } @@ -388,7 +417,7 @@ fn try_read8(self, offset: usize) -> Result fn try_read16(self, offset: usize) -> Result where usize: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { self.try_read(offset) } @@ -398,7 +427,7 @@ fn try_read16(self, offset: usize) -> Result fn try_read32(self, offset: usize) -> Result where usize: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { self.try_read(offset) } @@ -408,7 +437,7 @@ fn try_read32(self, offset: usize) -> Result fn try_read64(self, offset: usize) -> Result where usize: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { self.try_read(offset) } @@ -418,7 +447,7 @@ fn try_read64(self, offset: usize) -> Result fn try_write8(self, value: u8, offset: usize) -> Result where usize: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { self.try_write(offset, value) } @@ -428,7 +457,7 @@ fn try_write8(self, value: u8, offset: usize) -> Result fn try_write16(self, value: u16, offset: usize) -> Result where usize: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { self.try_write(offset, value) } @@ -438,7 +467,7 @@ fn try_write16(self, value: u16, offset: usize) -> Result fn try_write32(self, value: u32, offset: usize) -> Result where usize: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { self.try_write(offset, value) } @@ -448,7 +477,7 @@ fn try_write32(self, value: u32, offset: usize) -> Result fn try_write64(self, value: u64, offset: usize) -> Result where usize: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { self.try_write(offset, value) } @@ -458,7 +487,7 @@ fn try_write64(self, value: u64, offset: usize) -> Result fn read8(self, offset: usize) -> u8 where usize: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { self.read(offset) } @@ -468,7 +497,7 @@ fn read8(self, offset: usize) -> u8 fn read16(self, offset: usize) -> u16 where usize: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { self.read(offset) } @@ -478,7 +507,7 @@ fn read16(self, offset: usize) -> u16 fn read32(self, offset: usize) -> u32 where usize: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { self.read(offset) } @@ -488,7 +517,7 @@ fn read32(self, offset: usize) -> u32 fn read64(self, offset: usize) -> u64 where usize: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { self.read(offset) } @@ -498,7 +527,7 @@ fn read64(self, offset: usize) -> u64 fn write8(self, value: u8, offset: usize) where usize: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { self.write(offset, value) } @@ -508,7 +537,7 @@ fn write8(self, value: u8, offset: usize) fn write16(self, value: u16, offset: usize) where usize: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { self.write(offset, value) } @@ -518,7 +547,7 @@ fn write16(self, value: u16, offset: usize) fn write32(self, value: u32, offset: usize) where usize: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { self.write(offset, value) } @@ -528,7 +557,7 @@ fn write32(self, value: u32, offset: usize) fn write64(self, value: u64, offset: usize) where usize: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { self.write(offset, value) } @@ -560,12 +589,10 @@ fn write64(self, value: u64, offset: usize) fn try_read(self, location: L) -> Result where L: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { - let address = self.io_addr::(location.offset())?; - - // SAFETY: `address` has been validated by `io_addr`. - Ok(unsafe { self.io_read(address) }.into()) + let view = io_view::(self, location.offset())?; + Ok(Self::Backend::io_read(view).into()) } /// Generic fallible write with runtime bounds check. @@ -595,14 +622,11 @@ fn try_read(self, location: L) -> Result fn try_write(self, location: L, value: T) -> Result where L: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { - let address = self.io_addr::(location.offset())?; + let view = io_view::(self, location.offset())?; let io_value = value.into(); - - // SAFETY: `address` has been validated by `io_addr`. - unsafe { self.io_write(io_value, address) } - + Self::Backend::io_write(view, io_value); Ok(()) } @@ -643,7 +667,7 @@ fn try_write_reg(self, value: V) -> Result where L: IoLoc, V: LocatedRegister, - Self: IoCapable, + Self::Backend: IoCapable, { let (location, value) = value.into_io_op(); @@ -676,17 +700,14 @@ fn try_write_reg(self, value: V) -> Result fn try_update(self, location: L, f: F) -> Result where L: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, F: FnOnce(T) -> T, { - let address = self.io_addr::(location.offset())?; + let view = io_view::(self, location.offset())?; - // SAFETY: `address` has been validated by `io_addr`. - let value: T = unsafe { self.io_read(address) }.into(); + let value: T = Self::Backend::io_read(view).into(); let io_value = f(value).into(); - - // SAFETY: `address` has been validated by `io_addr`. - unsafe { self.io_write(io_value, address) } + Self::Backend::io_write(view, io_value); Ok(()) } @@ -716,12 +737,10 @@ fn try_update(self, location: L, f: F) -> Result fn read(self, location: L) -> T where L: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { - let address = self.io_addr_assert::(location.offset()); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_read(address) }.into() + let view = io_view_assert::(self, location.offset()); + Self::Backend::io_read(view).into() } /// Generic infallible write with compile-time bounds check. @@ -749,13 +768,11 @@ fn read(self, location: L) -> T fn write(self, location: L, value: T) where L: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, { - let address = self.io_addr_assert::(location.offset()); + let view = io_view_assert::(self, location.offset()); let io_value = value.into(); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_write(io_value, address) } + Self::Backend::io_write(view, io_value); } /// Generic infallible write of a fully-located register value. @@ -794,7 +811,7 @@ fn write_reg(self, value: V) where L: IoLoc, V: LocatedRegister, - Self: IoCapable, + Self::Backend: IoCapable, { let (location, value) = value.into_io_op(); @@ -827,17 +844,13 @@ fn write_reg(self, value: V) fn update(self, location: L, f: F) where L: IoLoc, - Self: IoCapable, + Self::Backend: IoCapable, F: FnOnce(T) -> T, { - let address = self.io_addr_assert::(location.offset()); - - // SAFETY: `address` has been validated by `io_addr_assert`. - let value: T = unsafe { self.io_read(address) }.into(); + let view = io_view_assert::(self, location.offset()); + let value: T = Self::Backend::io_read(view).into(); let io_value = f(value).into(); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_write(io_value, address) } + Self::Backend::io_write(view, io_value); } } @@ -881,78 +894,78 @@ unsafe impl Send for Mmio<'_, T> {} // SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory. unsafe impl Sync for Mmio<'_, T> {} -impl Io for Mmio<'_, T> { +impl<'a, T: ?Sized + KnownSize> Io<'a> for Mmio<'a, T> { + type Backend = MmioBackend; type Target = T; #[inline] - fn addr(self) -> usize { - self.ptr.addr() - } - - #[inline] - fn maxsize(self) -> usize { - KnownSize::size(self.ptr) + fn as_view(self) -> Mmio<'a, T> { + self } } -/// Implements [`IoCapable`] on `$mmio` for `$ty` using `$read_fn` and `$write_fn`. +/// I/O Backend for memory-mapped I/O. +pub struct MmioBackend; + +impl IoBackend for MmioBackend { + type View<'a, T: ?Sized + KnownSize> = Mmio<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + _view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid + // memory-mapped I/O region. + Mmio { + ptr, + phantom: PhantomData, + } + } +} + +/// Implements [`IoCapable`] on `$backend` for `$ty` using `$read_fn` and `$write_fn`. macro_rules! impl_mmio_io_capable { - ($mmio:ident, $(#[$attr:meta])* $ty:ty, $read_fn:ident, $write_fn:ident) => { - $(#[$attr])* - impl IoCapable<$ty> for $mmio<'_, T> { + ($backend: ident, $ty:ty, $read_fn:ident, $write_fn:ident) => { + impl IoCapable<$ty> for $backend { #[inline] - unsafe fn io_read(self, address: usize) -> $ty { - // SAFETY: By the trait invariant `address` is a valid address for MMIO operations. - unsafe { bindings::$read_fn(address as *const c_void) } + fn io_read(view: <$backend as IoBackend>::View<'_, $ty>) -> $ty { + // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both + // `MmioBackend` and `RelaxedMmioBackend`. + unsafe { bindings::$read_fn($backend::as_ptr(view).cast_const().cast()) } } #[inline] - unsafe fn io_write(self, value: $ty, address: usize) { - // SAFETY: By the trait invariant `address` is a valid address for MMIO operations. - unsafe { bindings::$write_fn(value, address as *mut c_void) } + fn io_write(view: <$backend as IoBackend>::View<'_, $ty>, value: $ty) { + // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both + // `MmioBackend` and `RelaxedMmioBackend`. + unsafe { bindings::$write_fn(value, $backend::as_ptr(view).cast()) } } } }; } // MMIO regions support 8, 16, and 32-bit accesses. -impl_mmio_io_capable!(Mmio, u8, readb, writeb); -impl_mmio_io_capable!(Mmio, u16, readw, writew); -impl_mmio_io_capable!(Mmio, u32, readl, writel); +impl_mmio_io_capable!(MmioBackend, u8, readb, writeb); +impl_mmio_io_capable!(MmioBackend, u16, readw, writew); +impl_mmio_io_capable!(MmioBackend, u32, readl, writel); // MMIO regions on 64-bit systems also support 64-bit accesses. #[cfg(CONFIG_64BIT)] -impl_mmio_io_capable!(Mmio, u64, readq, writeq); +impl_mmio_io_capable!(MmioBackend, u64, readq, writeq); -impl<'a, const SIZE: usize> Io for &'a MmioOwned { +impl<'a, const SIZE: usize> Io<'a> for &'a MmioOwned { + type Backend = MmioBackend; type Target = Region; - /// Returns the base address of this mapping. #[inline] - fn addr(self) -> usize { - self.0.addr() - } - - /// Returns the maximum size of this mapping. - #[inline] - fn maxsize(self) -> usize { - self.0.size() - } -} - -impl<'a, const SIZE: usize, T> IoCapable for &'a MmioOwned -where - Mmio<'a, Region>: IoCapable, -{ - #[inline] - unsafe fn io_read(self, address: usize) -> T { - // SAFETY: Per safety requirement. - unsafe { self.as_view().io_read(address) } - } - - #[inline] - unsafe fn io_write(self, value: T, address: usize) { - // SAFETY: Per safety requirement. - unsafe { self.as_view().io_write(value, address) } + fn as_view(self) -> Mmio<'a, Self::Target> { + // SAFETY: `Mmio` has same invariant as `MmioOwned` + unsafe { Mmio::from_raw(self.0) } } } @@ -968,13 +981,6 @@ pub unsafe fn from_raw(raw: &MmioRaw>) -> &Self { // SAFETY: `MmioOwned` is a transparent wrapper around `MmioRaw`. unsafe { &*core::ptr::from_ref(raw).cast() } } - - /// Return a view that covers the full region. - #[inline] - pub fn as_view(&self) -> Mmio<'_, Region> { - // SAFETY: `Mmio` has same invariant as `MmioOwned`. - unsafe { Mmio::from_raw(self.0) } - } } /// [`Mmio`] but using relaxed accessors. @@ -993,17 +999,34 @@ fn clone(&self) -> Self { } } -impl Io for RelaxedMmio<'_, T> { - type Target = T; +/// I/O Backend for memory-mapped I/O, with relaxed access semantics. +pub struct RelaxedMmioBackend; + +impl IoBackend for RelaxedMmioBackend { + type View<'a, T: ?Sized + KnownSize> = RelaxedMmio<'a, T>; #[inline] - fn addr(self) -> usize { - self.0.addr() + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + MmioBackend::as_ptr(view.0) } #[inline] - fn maxsize(self) -> usize { - self.0.maxsize() + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // SAFETY: Per safety requirement. + RelaxedMmio(unsafe { MmioBackend::project_view(view.0, ptr) }) + } +} + +impl<'a, T: ?Sized + KnownSize> Io<'a> for RelaxedMmio<'a, T> { + type Backend = RelaxedMmioBackend; + type Target = T; + + #[inline] + fn as_view(self) -> RelaxedMmio<'a, T> { + self } } @@ -1036,14 +1059,9 @@ pub fn relaxed(self) -> RelaxedMmio<'a, T> { } // MMIO regions support 8, 16, and 32-bit accesses. -impl_mmio_io_capable!(RelaxedMmio, u8, readb_relaxed, writeb_relaxed); -impl_mmio_io_capable!(RelaxedMmio, u16, readw_relaxed, writew_relaxed); -impl_mmio_io_capable!(RelaxedMmio, u32, readl_relaxed, writel_relaxed); +impl_mmio_io_capable!(RelaxedMmioBackend, u8, readb_relaxed, writeb_relaxed); +impl_mmio_io_capable!(RelaxedMmioBackend, u16, readw_relaxed, writew_relaxed); +impl_mmio_io_capable!(RelaxedMmioBackend, u32, readl_relaxed, writel_relaxed); // MMIO regions on 64-bit systems also support 64-bit accesses. -impl_mmio_io_capable!( - RelaxedMmio, - #[cfg(CONFIG_64BIT)] - u64, - readq_relaxed, - writeq_relaxed -); +#[cfg(CONFIG_64BIT)] +impl_mmio_io_capable!(RelaxedMmioBackend, u64, readq_relaxed, writeq_relaxed); diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs index 89f4bb483a7f..e67c1e3694fb 100644 --- a/rust/kernel/pci/io.rs +++ b/rust/kernel/pci/io.rs @@ -9,6 +9,7 @@ devres::Devres, io::{ Io, + IoBackend, IoCapable, MmioOwned, MmioRaw, @@ -84,32 +85,57 @@ unsafe impl Send for ConfigSpace<'_, T> {} // SAFETY: `ConfigSpace<'_, T>` is conceptually `&T` but in I/O memory. unsafe impl Sync for ConfigSpace<'_, T> {} +/// I/O Backend for PCI configuration space. +pub struct ConfigSpaceBackend; + +impl IoBackend for ConfigSpaceBackend { + type View<'a, T: ?Sized + KnownSize> = ConfigSpace<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: ConfigSpace<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement. + ConfigSpace { + pdev: view.pdev, + ptr, + } + } +} + /// Implements [`IoCapable`] on [`ConfigSpace`] for `$ty` using `$read_fn` and `$write_fn`. macro_rules! impl_config_space_io_capable { ($ty:ty, $read_fn:ident, $write_fn:ident) => { - impl<'a, T: ?Sized> IoCapable<$ty> for ConfigSpace<'a, T> { - unsafe fn io_read(self, address: usize) -> $ty { + impl IoCapable<$ty> for ConfigSpaceBackend { + fn io_read(view: ConfigSpace<'_, $ty>) -> $ty { + // CAST: The offset is cast to `i32` because the C functions expect a 32-bit + // signed offset parameter. PCI configuration space size is at most 4096 bytes, + // so the value always fits within `i32` without truncation or sign change. + let addr = view.ptr.addr() as i32; + let mut val: $ty = 0; // Return value from C function is ignored in infallible accessors. - let _ret = - // SAFETY: By the type invariant `self.pdev` is a valid address. - // CAST: The offset is cast to `i32` because the C functions expect a 32-bit - // signed offset parameter. PCI configuration space size is at most 4096 bytes, - // so the value always fits within `i32` without truncation or sign change. - unsafe { bindings::$read_fn(self.pdev.as_raw(), address as i32, &mut val) }; - + // SAFETY: By the type invariant `pdev` is a valid address. + let _ = unsafe { bindings::$read_fn(view.pdev.as_raw(), addr, &mut val) }; val } - unsafe fn io_write(self, value: $ty, address: usize) { + fn io_write(view: ConfigSpace<'_, $ty>, value: $ty) { + // CAST: The offset is cast to `i32` because the C functions expect a 32-bit + // signed offset parameter. PCI configuration space size is at most 4096 bytes, + // so the value always fits within `i32` without truncation or sign change. + let addr = view.ptr.addr() as i32; + // Return value from C function is ignored in infallible accessors. - let _ret = - // SAFETY: By the type invariant `self.pdev` is a valid address. - // CAST: The offset is cast to `i32` because the C functions expect a 32-bit - // signed offset parameter. PCI configuration space size is at most 4096 bytes, - // so the value always fits within `i32` without truncation or sign change. - unsafe { bindings::$write_fn(self.pdev.as_raw(), address as i32, value) }; + // SAFETY: By the type invariant `pdev` is a valid address. + let _ = unsafe { bindings::$write_fn(view.pdev.as_raw(), addr, value) }; } } }; @@ -120,17 +146,13 @@ unsafe fn io_write(self, value: $ty, address: usize) { impl_config_space_io_capable!(u16, pci_read_config_word, pci_write_config_word); impl_config_space_io_capable!(u32, pci_read_config_dword, pci_write_config_dword); -impl<'a, T: ?Sized + KnownSize> Io for ConfigSpace<'a, T> { +impl<'a, T: ?Sized + KnownSize> Io<'a> for ConfigSpace<'a, T> { + type Backend = ConfigSpaceBackend; type Target = T; #[inline] - fn addr(self) -> usize { - self.ptr.addr() - } - - #[inline] - fn maxsize(self) -> usize { - KnownSize::size(self.ptr) + fn as_view(self) -> ConfigSpace<'a, T> { + self } } From 0adc93b85374277514e5145970037e3a287b62dd Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 6 Jul 2026 13:44:23 +0100 Subject: [PATCH 050/137] pwm: th1520: remove unnecessary `deref` `Deref` is automatic and should normally not be used directly. Also, `IoMem` is going to be implementing `Io` directly, so it will no longer to be implementing `Deref`. Reported-by: Andreas Hindborg Link: https://rust-for-linux.zulipchat.com/#narrow/channel/291565-Help/topic/.E2.9C.94.20Projection.20in.20dma.20bus.20address.20space/near/606672061 Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260706-io_projection-v6-10-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich --- drivers/pwm/pwm_th1520.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/drivers/pwm/pwm_th1520.rs b/drivers/pwm/pwm_th1520.rs index 3e3fa51ccef9..022338d17218 100644 --- a/drivers/pwm/pwm_th1520.rs +++ b/drivers/pwm/pwm_th1520.rs @@ -20,7 +20,6 @@ //! this method is not used in this driver. //! -use core::ops::Deref; use kernel::{ clk::Clk, device::{Bound, Core, Device}, @@ -213,8 +212,7 @@ fn read_waveform( ) -> Result { let data = chip.drvdata(); let hwpwm = pwm.hwpwm(); - let iomem_accessor = data.iomem.access(parent_dev)?; - let iomap = iomem_accessor.deref(); + let iomap = data.iomem.access(parent_dev)?; let ctrl = iomap.try_read32(th1520_pwm_ctrl(hwpwm))?; let period_cycles = iomap.try_read32(th1520_pwm_per(hwpwm))?; @@ -248,8 +246,7 @@ fn write_waveform( ) -> Result { let data = chip.drvdata(); let hwpwm = pwm.hwpwm(); - let iomem_accessor = data.iomem.access(parent_dev)?; - let iomap = iomem_accessor.deref(); + let iomap = data.iomem.access(parent_dev)?; let duty_cycles = iomap.try_read32(th1520_pwm_fp(hwpwm))?; let was_enabled = duty_cycles != 0; From bed01ca9e9cf8f8fea5352c07fa206cc3c106045 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 6 Jul 2026 13:44:24 +0100 Subject: [PATCH 051/137] rust: io: remove `MmioOwned` `Io` trait is now very easy to implement. Thus, implement it on `Bar` and `IoMem` directly and remove the `MmioOwned` struct. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Reviewed-by: Daniel Almeida Suggested-by: Danilo Krummrich Link: https://rust-for-linux.zulipchat.com/#narrow/channel/288089-General/topic/Generic.20I.2FO.20backends/near/571198078 Link: https://patch.msgid.link/20260706-io_projection-v6-11-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/devres.rs | 12 +++-- rust/kernel/io.rs | 103 +----------------------------------------- rust/kernel/io/mem.rs | 26 ++++++----- rust/kernel/pci/io.rs | 16 +++---- 4 files changed, 32 insertions(+), 125 deletions(-) diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index aed0c994fd30..3545ffc5345d 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -68,8 +68,9 @@ struct Inner { /// devres::Devres, /// io::{ /// Io, -/// MmioOwned, +/// Mmio, /// MmioRaw, +/// MmioBackend, /// PhysAddr, /// Region, // /// }, @@ -104,12 +105,13 @@ struct Inner { /// } /// } /// -/// impl Deref for IoMem { -/// type Target = MmioOwned; +/// impl<'a, const SIZE: usize> Io<'a> for &'a IoMem { +/// type Backend = MmioBackend; +/// type Target = Region; /// -/// fn deref(&self) -> &Self::Target { +/// fn as_view(self) -> Mmio<'a, Region> { /// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`. -/// unsafe { MmioOwned::from_raw(&self.0) } +/// unsafe { Mmio::from_raw(self.0) } /// } /// } /// # fn no_run(dev: &Device) -> Result<(), Error> { diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 82a3369ae110..729b64a385c3 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -99,8 +99,8 @@ fn size(p: *const Self) -> usize { /// the represented MMIO region does exist or is properly mapped. /// /// Instead, the bus specific MMIO implementation must convert this raw representation into an -/// `MmioOwned` instance providing the actual memory accessors. Only by the conversion into an -/// `MmioOwned` structure any guarantees are given. +/// `Mmio` instance providing the actual memory accessors. Only by the conversion into an `Mmio` +/// structure any guarantees are given. pub struct MmioRaw { /// Pointer is in I/O address space. /// @@ -157,80 +157,6 @@ pub fn size(&self) -> usize { } } -/// IO-mapped memory region. -/// -/// The creator (usually a subsystem / bus such as PCI) is responsible for creating the -/// mapping, performing an additional region request etc. -/// -/// # Invariant -/// -/// `addr` is the start and `maxsize` the length of valid I/O mapped memory region of size -/// `maxsize`. -/// -/// # Examples -/// -/// ```no_run -/// use kernel::{ -/// bindings, -/// ffi::c_void, -/// io::{ -/// Io, -/// MmioOwned, -/// MmioRaw, -/// PhysAddr, -/// Region, -/// }, -/// }; -/// use core::ops::Deref; -/// -/// // See also `pci::Bar` for a real example. -/// struct IoMem(MmioRaw>); -/// -/// impl IoMem { -/// /// # Safety -/// /// -/// /// [`paddr`, `paddr` + `SIZE`) must be a valid MMIO region that is mappable into the CPUs -/// /// virtual address space. -/// unsafe fn new(paddr: usize) -> Result{ -/// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is -/// // valid for `ioremap`. -/// let addr = unsafe { bindings::ioremap(paddr as PhysAddr, SIZE) }; -/// if addr.is_null() { -/// return Err(ENOMEM); -/// } -/// -/// Ok(IoMem(MmioRaw::new_region(addr as usize, SIZE)?)) -/// } -/// } -/// -/// impl Drop for IoMem { -/// fn drop(&mut self) { -/// // SAFETY: `self.0.addr()` is guaranteed to be properly mapped by `Self::new`. -/// unsafe { bindings::iounmap(self.0.addr() as *mut c_void); }; -/// } -/// } -/// -/// impl Deref for IoMem { -/// type Target = MmioOwned; -/// -/// fn deref(&self) -> &Self::Target { -/// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`. -/// unsafe { MmioOwned::from_raw(&self.0) } -/// } -/// } -/// -///# fn no_run() -> Result<(), Error> { -/// // SAFETY: Invalid usage for example purposes. -/// let iomem = unsafe { IoMem::<{ core::mem::size_of::() }>::new(0xBAAAAAAD)? }; -/// iomem.write32(0x42, 0x0); -/// assert!(iomem.try_write32(0x42, 0x0).is_ok()); -/// assert!(iomem.try_write32(0x42, 0x4).is_err()); -/// # Ok(()) -/// # } -/// ``` -#[repr(transparent)] -pub struct MmioOwned(MmioRaw>); - /// Checks whether an access of type `U` at the given `base` and the given `offset` /// is valid within this region. /// @@ -958,31 +884,6 @@ fn io_write(view: <$backend as IoBackend>::View<'_, $ty>, value: $ty) { #[cfg(CONFIG_64BIT)] impl_mmio_io_capable!(MmioBackend, u64, readq, writeq); -impl<'a, const SIZE: usize> Io<'a> for &'a MmioOwned { - type Backend = MmioBackend; - type Target = Region; - - #[inline] - fn as_view(self) -> Mmio<'a, Self::Target> { - // SAFETY: `Mmio` has same invariant as `MmioOwned` - unsafe { Mmio::from_raw(self.0) } - } -} - -impl MmioOwned { - /// Converts an `MmioRaw` into an `MmioOwned` instance, providing the accessors to the MMIO - /// mapping. - /// - /// # Safety - /// - /// Callers must ensure that `addr` is the start of a valid I/O mapped memory region of size - /// `maxsize`. - pub unsafe fn from_raw(raw: &MmioRaw>) -> &Self { - // SAFETY: `MmioOwned` is a transparent wrapper around `MmioRaw`. - unsafe { &*core::ptr::from_ref(raw).cast() } - } -} - /// [`Mmio`] but using relaxed accessors. /// /// This type provides an implementation of [`Io`] that uses relaxed I/O MMIO operands instead of diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs index 8f6c257c5b8e..d9b3189d09b4 100644 --- a/rust/kernel/io/mem.rs +++ b/rust/kernel/io/mem.rs @@ -2,8 +2,6 @@ //! Generic memory-mapped IO. -use core::ops::Deref; - use crate::{ device::{ Bound, @@ -16,7 +14,9 @@ Region, Resource, // }, - MmioOwned, + Io, + Mmio, + MmioBackend, MmioRaw, // }, prelude::*, @@ -210,11 +210,13 @@ pub fn into_devres(self) -> Result>> { } } -impl Deref for ExclusiveIoMem<'_, SIZE> { - type Target = MmioOwned; +impl<'a, const SIZE: usize> Io<'a> for &'a ExclusiveIoMem<'_, SIZE> { + type Backend = MmioBackend; + type Target = super::Region; - fn deref(&self) -> &Self::Target { - &self.iomem + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { + self.iomem.as_view() } } @@ -290,11 +292,13 @@ fn drop(&mut self) { } } -impl Deref for IoMem<'_, SIZE> { - type Target = MmioOwned; +impl<'a, const SIZE: usize> Io<'a> for &'a IoMem<'_, SIZE> { + type Backend = MmioBackend; + type Target = super::Region; - fn deref(&self) -> &Self::Target { + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { // SAFETY: Safe as by the invariant of `IoMem`. - unsafe { MmioOwned::from_raw(&self.io) } + unsafe { Mmio::from_raw(self.io) } } } diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs index e67c1e3694fb..4be33ecb4192 100644 --- a/rust/kernel/pci/io.rs +++ b/rust/kernel/pci/io.rs @@ -11,16 +11,14 @@ Io, IoBackend, IoCapable, - MmioOwned, + Mmio, + MmioBackend, MmioRaw, Region, // }, prelude::*, ptr::KnownSize, // }; -use core::{ - ops::Deref, // -}; /// Represents the size of a PCI configuration space. /// @@ -269,12 +267,14 @@ fn drop(&mut self) { } } -impl Deref for Bar<'_, SIZE> { - type Target = MmioOwned; +impl<'a, const SIZE: usize> Io<'a> for &'a Bar<'_, SIZE> { + type Backend = MmioBackend; + type Target = crate::io::Region; - fn deref(&self) -> &Self::Target { + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { // SAFETY: By the type invariant of `Self`, the MMIO range in `self.io` is properly mapped. - unsafe { MmioOwned::from_raw(&self.io) } + unsafe { Mmio::from_raw(self.io) } } } From 9b36c13cbd4fb761212b1ea9a2e89f7df2d3c9f8 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 6 Jul 2026 13:44:25 +0100 Subject: [PATCH 052/137] rust: io: move `Io` methods to extension trait `Io` trait now has a single required method with many more provided methods. Provided methods may want to rely on their implementations to not be arbitrarily overridden by implementers for correctness or soundness. A good example is the `size` method, it may be relied by unsafe code and thus must be consistent with the metadata obtained from `as_ptr`. Thus, create a new trait to host `size` method, extract existing provided methods to the new trait, and provide a blanket implementation. This pattern is used extensively in userspace Rust libraries e.g. `tokio` where `AsyncRead` has minimum methods and `AsyncReadExt` is what users mostly interact with. To avoid changing all user imports, the base trait is renamed to `IoBase` and the newly added trait takes the existing `Io` name. Reviewed-by: Alexandre Courbot Suggested-by: Danilo Krummrich Signed-off-by: Gary Guo Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260706-io_projection-v6-12-72cd5d055d54@garyguo.net [ Add comment explaining the purpose of the Io blanket implementation. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/devres.rs | 3 ++- rust/kernel/io.rs | 36 ++++++++++++++++++++++++++---------- rust/kernel/io/mem.rs | 6 +++--- rust/kernel/pci/io.rs | 6 +++--- 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index 3545ffc5345d..6e0b845b229b 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -68,6 +68,7 @@ struct Inner { /// devres::Devres, /// io::{ /// Io, +/// IoBase, /// Mmio, /// MmioRaw, /// MmioBackend, @@ -105,7 +106,7 @@ struct Inner { /// } /// } /// -/// impl<'a, const SIZE: usize> Io<'a> for &'a IoMem { +/// impl<'a, const SIZE: usize> IoBase<'a> for &'a IoMem { /// type Backend = MmioBackend; /// type Target = Region; /// diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 729b64a385c3..a9ff9e2c9f5c 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -224,7 +224,7 @@ fn io_view<'a, IO: Io<'a>, U>( /// operation. pub trait IoBackend { /// View type for this I/O backend. - type View<'a, T: ?Sized + KnownSize>: Io<'a, Backend = Self, Target = T>; + type View<'a, T: ?Sized + KnownSize>: IoBase<'a, Backend = Self, Target = T>; /// Convert a `view` to a raw pointer for projection. /// @@ -310,15 +310,12 @@ fn offset(self) -> usize { /// Types implementing this trait (e.g. MMIO BARs or PCI config regions) /// can perform I/O operations on regions of memory. /// -/// The [`Io`] trait provides: -/// - Method to convert into [`IoBackend::View`]. -/// - Helper methods for offset validation and address calculation -/// - Fallible (runtime checked) accessors for different data widths -/// -/// Which I/O methods are available depends on the associated [`IoBackend`] implementation. +/// This trait defines which backend shall be used for I/O operations and provides a method to +/// convert into [`IoBackend::View`]. Users should use the [`Io`] trait which provides the actual +/// methods to perform I/O operations. /// /// This should be implemented on cheaply copyable handles, such as references or view types. -pub trait Io<'a>: Copy { +pub trait IoBase<'a>: Copy { /// Type that defines all I/O operations. type Backend: IoBackend; @@ -327,6 +324,21 @@ pub trait Io<'a>: Copy { /// Return a view that covers the full region. fn as_view(self) -> ::View<'a, Self::Target>; +} + +/// Extension trait to provide I/O operation methods to types that implement [`IoBase`]. +/// +/// This trait provides: +/// - Helper methods for offset validation and address calculation +/// - Fallible (runtime checked) accessors for different data widths +/// +/// Which I/O methods are available depends on the associated [`IoBackend`] implementation. +pub trait Io<'a>: IoBase<'a> { + /// Returns the size of this I/O region. + #[inline] + fn size(self) -> usize { + KnownSize::size(Self::Backend::as_ptr(self.as_view())) + } /// Fallible 8-bit read with runtime bounds check. #[inline(always)] @@ -780,6 +792,10 @@ fn update(self, location: L, f: F) } } +// Blanket implementation ensures that provided methods cannot be arbitrarily overridden by +// implementers, which is relied upon for correctness and soundness. +impl<'a, T: IoBase<'a>> Io<'a> for T {} + /// A view of memory-mapped I/O region. /// /// # Invariant @@ -820,7 +836,7 @@ unsafe impl Send for Mmio<'_, T> {} // SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory. unsafe impl Sync for Mmio<'_, T> {} -impl<'a, T: ?Sized + KnownSize> Io<'a> for Mmio<'a, T> { +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for Mmio<'a, T> { type Backend = MmioBackend; type Target = T; @@ -921,7 +937,7 @@ unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( } } -impl<'a, T: ?Sized + KnownSize> Io<'a> for RelaxedMmio<'a, T> { +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for RelaxedMmio<'a, T> { type Backend = RelaxedMmioBackend; type Target = T; diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs index d9b3189d09b4..e95b769ebe47 100644 --- a/rust/kernel/io/mem.rs +++ b/rust/kernel/io/mem.rs @@ -14,7 +14,7 @@ Region, Resource, // }, - Io, + IoBase, Mmio, MmioBackend, MmioRaw, // @@ -210,7 +210,7 @@ pub fn into_devres(self) -> Result>> { } } -impl<'a, const SIZE: usize> Io<'a> for &'a ExclusiveIoMem<'_, SIZE> { +impl<'a, const SIZE: usize> IoBase<'a> for &'a ExclusiveIoMem<'_, SIZE> { type Backend = MmioBackend; type Target = super::Region; @@ -292,7 +292,7 @@ fn drop(&mut self) { } } -impl<'a, const SIZE: usize> Io<'a> for &'a IoMem<'_, SIZE> { +impl<'a, const SIZE: usize> IoBase<'a> for &'a IoMem<'_, SIZE> { type Backend = MmioBackend; type Target = super::Region; diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs index 4be33ecb4192..4d1d0afdc491 100644 --- a/rust/kernel/pci/io.rs +++ b/rust/kernel/pci/io.rs @@ -8,8 +8,8 @@ device, devres::Devres, io::{ - Io, IoBackend, + IoBase, IoCapable, Mmio, MmioBackend, @@ -144,7 +144,7 @@ fn io_write(view: ConfigSpace<'_, $ty>, value: $ty) { impl_config_space_io_capable!(u16, pci_read_config_word, pci_write_config_word); impl_config_space_io_capable!(u32, pci_read_config_dword, pci_write_config_dword); -impl<'a, T: ?Sized + KnownSize> Io<'a> for ConfigSpace<'a, T> { +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for ConfigSpace<'a, T> { type Backend = ConfigSpaceBackend; type Target = T; @@ -267,7 +267,7 @@ fn drop(&mut self) { } } -impl<'a, const SIZE: usize> Io<'a> for &'a Bar<'_, SIZE> { +impl<'a, const SIZE: usize> IoBase<'a> for &'a Bar<'_, SIZE> { type Backend = MmioBackend; type Target = crate::io::Region; From 1f989555fe5c823de108d55603ca3cb30053fb45 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 6 Jul 2026 13:44:26 +0100 Subject: [PATCH 053/137] rust: io: add projection macro and methods Add an `io_project!()` macro to allow projection from `Io` to a subview of it, using the pointer projection mechanism to perform compile-time checks. For cases where type-casting is required, the `try_cast()` function may be used where the size and alignment checks are performed at runtime. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260706-io_projection-v6-13-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 128 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index a9ff9e2c9f5c..9f6515a717de 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -49,6 +49,7 @@ /// - Size of the region is at least as large as the `SIZE` generic parameter. /// - Size of the region is multiple of 4. #[repr(C, align(4))] +#[derive(FromBytes)] pub struct Region { inner: [u8], } @@ -91,6 +92,19 @@ fn size(p: *const Self) -> usize { } } +// SAFETY: +// - Values read from I/O are always treated as initialized. +// - Per type invariant the size is multiple of 4 and the type is 4-byte aligned, so it is padding +// free. +// +// This cannot be derived as `derive(IntoBytes)` as the padding free property comes from type +// invariant which the macro does not know. +unsafe impl IntoBytes for Region { + #[inline] + #[allow(unused)] // Rust 1.87+ stops requiring this and will emit unused warnings. + fn only_derive_is_allowed_to_implement_this_trait() {} +} + /// Raw representation of an MMIO region. /// /// `MmioRaw` is equivalent to `T __iomem *` in C. @@ -340,6 +354,51 @@ fn size(self) -> usize { KnownSize::size(Self::Backend::as_ptr(self.as_view())) } + /// Try to convert into a different typed I/O view. + /// + /// A runtime check is performed to ensure that the target type is of same or smaller size to + /// current type, and the current view is properly aligned for the target type. Returns + /// `Err(EINVAL)` if the runtime check fails. + /// + /// # Examples + /// + /// ```no_run + /// use kernel::io::{ + /// io_project, + /// Mmio, + /// Io, + /// Region, + /// }; + /// #[derive(FromBytes, IntoBytes)] + /// #[repr(C)] + /// struct MyStruct { field: u32, } + /// + /// # fn test(mmio: &Mmio<'_, Region>) -> Result { + /// // let mmio: Mmio<'_, Region>; + /// let whole: Mmio<'_, MyStruct> = mmio.try_cast()?; + /// # Ok::<(), Error>(()) } + /// ``` + #[inline] + fn try_cast(self) -> Result<::View<'a, U>> + where + Self::Target: FromBytes + IntoBytes, + U: FromBytes + IntoBytes, + { + let view = self.as_view(); + let ptr = Self::Backend::as_ptr(view); + + if size_of::() > KnownSize::size(ptr) { + return Err(EINVAL); + } + + if ptr.addr() % align_of::() != 0 { + return Err(EINVAL); + } + + // SAFETY: We have checked bounds and alignment, so this is a valid projection. + Ok(unsafe { Self::Backend::project_view(view, ptr.cast()) }) + } + /// Fallible 8-bit read with runtime bounds check. #[inline(always)] fn try_read8(self, offset: usize) -> Result @@ -982,3 +1041,72 @@ pub fn relaxed(self) -> RelaxedMmio<'a, T> { // MMIO regions on 64-bit systems also support 64-bit accesses. #[cfg(CONFIG_64BIT)] impl_mmio_io_capable!(RelaxedMmioBackend, u64, readq_relaxed, writeq_relaxed); + +// This helper turns associated functions to methods so it can be invoked in macro. +// Used by `io_project!()` only. +#[doc(hidden)] +#[derive(Clone, Copy)] +pub struct ProjectHelper(pub T); + +impl<'a, T> ProjectHelper +where + T: Io<'a, Backend: IoBackend = T>>, +{ + // These helper methods must not have symbols present in the binary to avoid confusion. + #[inline(always)] + pub fn as_ptr(self) -> *mut T::Target { + T::Backend::as_ptr(self.0) + } + + /// # Safety + /// + /// Same as `IoBackend::project_view` + #[inline(always)] + pub unsafe fn project_view( + self, + ptr: *mut U, + ) -> ::View<'a, U> { + // SAFETY: Per safety requirement. + unsafe { T::Backend::project_view::(self.0, ptr) } + } +} + +/// Project an I/O type to a subview of it. +/// +/// The syntax is of form `io_project!(io, proj)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!). +/// +/// # Examples +/// +/// ``` +/// use kernel::io::{ +/// io_project, +/// Mmio, +/// }; +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<[MyStruct]>; +/// let field: Mmio<'_, u32> = io_project!(mmio, [try: 1].field); +/// let whole: Mmio<'_, MyStruct> = io_project!(mmio, [try: 2]); +/// let nested: Mmio<'_, u32> = io_project!(whole, .field); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_project { + ($io:expr, $($proj:tt)*) => {{ + #[allow(unused)] + use $crate::io::IoBase as _; + let view = $crate::io::ProjectHelper($io.as_view()); + let ptr = $crate::ptr::project!( + mut view.as_ptr(), $($proj)* + ); + #[allow(unused_unsafe)] + // SAFETY: `ptr` is a projection. + unsafe { view.project_view(ptr) } + }}; +} +#[doc(inline)] +pub use crate::io_project; From 2fabff7807853804af2ae691b263c3ac5cc9a636 Mon Sep 17 00:00:00 2001 From: Laura Nao Date: Mon, 6 Jul 2026 13:44:27 +0100 Subject: [PATCH 054/137] rust: io: add I/O backend for system memory with volatile access Add `SysMem`, an `Io` trait implementation for kernel virtual address ranges. It uses volatile accessors to provide safe access to shared memory that may be concurrently accessed by external hardware. Implement `IoCapable` for `u8`, `u16`, `u32`, and `u64` (for 64-bit system). This can be used instead of `Coherent` for cases where a different layer takes care of mapping the system memory to the device (e.g. dma-buf or GPUVM). Signed-off-by: Laura Nao [ Rebased and adapted on top of I/O rework. - Gary ] Co-developed-by: Gary Guo Signed-off-by: Gary Guo Reviewed-by: Alexandre Courbot Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260706-io_projection-v6-14-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 122 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 9f6515a717de..c212de8860ac 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -1042,6 +1042,128 @@ pub fn relaxed(self) -> RelaxedMmio<'a, T> { #[cfg(CONFIG_64BIT)] impl_mmio_io_capable!(RelaxedMmioBackend, u64, readq_relaxed, writeq_relaxed); +/// I/O Backend for system memory. +pub struct SysMemBackend; + +impl IoBackend for SysMemBackend { + type View<'a, T: ?Sized + KnownSize> = SysMem<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + _view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid + // kernel accessible memory region. + SysMem { + ptr, + phantom: PhantomData, + } + } +} + +/// Implements [`IoCapable`] on `SysMemBackend` for `$ty` using `read_volatile` and +/// `write_volatile`. +macro_rules! impl_sysmem_io_capable { + ($ty:ty) => { + impl IoCapable<$ty> for SysMemBackend { + #[inline] + fn io_read(view: SysMem<'_, $ty>) -> $ty { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using read_volatile() here so that race with hardware is well-defined. + // - Using read_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + // - The macro is only used on primitives so all bit patterns are valid. + unsafe { view.ptr.read_volatile() } + } + + #[inline] + fn io_write(view: SysMem<'_, $ty>, value: $ty) { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using write_volatile() here so that race with hardware is well-defined. + // - Using write_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + unsafe { view.ptr.write_volatile(value) } + } + } + }; +} + +impl_sysmem_io_capable!(u8); +impl_sysmem_io_capable!(u16); +impl_sysmem_io_capable!(u32); +#[cfg(CONFIG_64BIT)] +impl_sysmem_io_capable!(u64); + +/// A view of a system memory region. +/// +/// Provides `Io` trait implementation for kernel virtual address ranges, +/// using volatile read/write to safely access shared memory that may be +/// concurrently accessed by external hardware. +/// +/// # Invariants +/// +/// `self.ptr.addr() .. self.ptr.addr() + KnownSize::size(self.ptr)` is valid and aligned kernel +/// accessible memory region for the lifetime `'a`. +pub struct SysMem<'a, T: ?Sized> { + ptr: *mut T, + phantom: PhantomData<&'a ()>, +} + +impl Copy for SysMem<'_, T> {} +impl Clone for SysMem<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +// SAFETY: `SysMem<'_, T>` is conceptually `&T`. +unsafe impl Send for SysMem<'_, T> {} + +// SAFETY: `SysMem<'_, T>` is conceptually `&T`. +unsafe impl Sync for SysMem<'_, T> {} + +impl<'a, T: ?Sized> SysMem<'a, T> { + /// Create a `SysMem` from a raw pointer. + /// + /// # Safety + /// + /// `ptr.addr() .. ptr.addr() + KnownSize::size(ptr)` must be valid and aligned kernel + /// accessible memory region for the lifetime `'a`. + #[inline] + pub unsafe fn new(ptr: *mut T) -> Self { + // INVARIANT: Per safety requirement. + Self { + ptr, + phantom: PhantomData, + } + } + + /// Obtain the raw pointer to the memory. + #[inline] + pub fn as_ptr(self) -> *mut T { + self.ptr + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for SysMem<'a, T> { + type Backend = SysMemBackend; + type Target = T; + + #[inline] + fn as_view(self) -> ::View<'a, Self::Target> { + self + } +} + // This helper turns associated functions to methods so it can be invoked in macro. // Used by `io_project!()` only. #[doc(hidden)] From 1d409d1e7a874b3aeff8908292bc26ba4113cc06 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 6 Jul 2026 13:44:28 +0100 Subject: [PATCH 055/137] rust: io: implement a view type for `Coherent` Implement a `CoherentView` type which is a view of `Coherent`. To be able to give out DMA handles, the view type contains both CPU and DMA pointers, and the projection method projects both at once. Delegate most of the `Io` implementation to `SysMemBackend`. Provide a method to erase the DMA handle and give out a `SysMem` view, if the user does not need the `dma_handle`. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260706-io_projection-v6-15-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/dma.rs | 138 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 136 insertions(+), 2 deletions(-) diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 200def84fb69..1535bc6eec64 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -14,14 +14,21 @@ }, error::to_result, fs::file, + io::{ + IoBackend, + IoBase, + IoCapable, + SysMem, + SysMemBackend, // + }, prelude::*, ptr::KnownSize, sync::aref::ARef, transmute::{ AsBytes, FromBytes, // - }, // - uaccess::UserSliceWriter, + }, + uaccess::UserSliceWriter, // }; use core::{ ops::{ @@ -1133,6 +1140,133 @@ unsafe impl Send for CoherentHandle {} // plain `Copy` values. unsafe impl Sync for CoherentHandle {} +/// View type for `Coherent`. +/// +/// This is same as [`SysMem`] but with additional information that allows handing out a DMA handle. +pub struct CoherentView<'a, T: ?Sized> { + cpu_addr: SysMem<'a, T>, + dma_handle: DmaAddress, +} + +impl Copy for CoherentView<'_, T> {} +impl Clone for CoherentView<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T: ?Sized> CoherentView<'a, T> { + /// Erase the DMA handle information and obtain a [`SysMem`] view of the same memory region. + #[inline] + pub fn as_sys_mem(self) -> SysMem<'a, T> { + self.cpu_addr + } + + /// Returns a DMA handle which may be given to the device as the DMA address base of the region. + #[inline] + pub fn dma_handle(self) -> DmaAddress { + self.dma_handle + } + + /// Returns a reference to the data in the region. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// reference is live. + /// * Callers must ensure that this call does not race with a write (including call to `as_mut`) + /// to the same region while the returned reference is live. + #[inline] + pub unsafe fn as_ref(self) -> &'a T { + // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per + // safety requirement. + unsafe { &*self.cpu_addr.as_ptr() } + } + + /// Returns a mutable reference to the data in the region. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// reference is live. + /// * Callers must ensure that this call does not race with a read (including call to `as_ref`) + /// or write (including call to `as_mut`) to the same region while the returned reference is + /// live. + #[inline] + pub unsafe fn as_mut(self) -> &'a mut T { + // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per + // safety requirement. + unsafe { &mut *self.cpu_addr.as_ptr() } + } +} + +/// `IoBackend` implementation for `Coherent`. +pub struct CoherentIoBackend; + +impl IoBackend for CoherentIoBackend { + type View<'a, T: ?Sized + KnownSize> = CoherentView<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + SysMemBackend::as_ptr(view.cpu_addr) + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + let offset = ptr.addr() - view.cpu_addr.as_ptr().addr(); + // CAST: The offset DMA address can never overflow. + let dma_handle = view.dma_handle + offset as DmaAddress; + CoherentView { + dma_handle, + // SAFETY: Per safety requirement. + cpu_addr: unsafe { SysMemBackend::project_view(view.cpu_addr, ptr) }, + } + } +} + +impl IoCapable for CoherentIoBackend +where + SysMemBackend: IoCapable, +{ + #[inline] + fn io_read<'a>(view: Self::View<'a, T>) -> T { + SysMemBackend::io_read(view.cpu_addr) + } + + #[inline] + fn io_write<'a>(view: Self::View<'a, T>, value: T) { + SysMemBackend::io_write(view.cpu_addr, value) + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for CoherentView<'a, T> { + type Backend = CoherentIoBackend; + type Target = T; + + #[inline] + fn as_view(self) -> CoherentView<'a, Self::Target> { + self + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent { + type Backend = CoherentIoBackend; + type Target = T; + + #[inline] + fn as_view(self) -> CoherentView<'a, Self::Target> { + CoherentView { + // SAFETY: `cpu_addr` is valid and aligned kernel accessible memory. + cpu_addr: unsafe { SysMem::new(self.cpu_addr.as_ptr()) }, + dma_handle: self.dma_handle, + } + } +} + /// Reads a field of an item from an allocated region of structs. /// /// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating From 89814c42c19ea63600f7235156ae665f6bf8b369 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 6 Jul 2026 13:44:29 +0100 Subject: [PATCH 056/137] rust: io: add `read_val` and `write_val` functions on `Io` Provide `read_val` and `write_val` that allow I/O views to be accessed when they're narrowed down to just views of primitives. This is used to provide `io_read!` and `io_write!` macros, which are generalized version of current `dma_read!` and `dma_write!` macro that work for all types that implement `Io`. Note though `io_read!` and `io_write!` only works if backend implements `IoCapable` for the type; which is typically only implemented for atomically accessible primitives. `dma_read!` and `dma_write!` currently supports them via `read_volatile` and `write_volatile`; this can be undesirable for aggregates as LLVM may turn them to multiple instructions to access parts and re-assemble, even if they could be combined to a single instruction. Thus, `io_read!()` and `io_write!()` does not fully replace `dma_read!()` and `dma_write!()` in this scenario. The ability to read/write aggregates (when atomicity is of no concern) is better served with copying primitives (e.g. memcpy_{from,to}io). Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260706-io_projection-v6-16-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 106 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index c212de8860ac..3f430bce61e5 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -399,6 +399,50 @@ fn try_cast(self) -> Result<::View<'a, U>> Ok(unsafe { Self::Backend::project_view(view, ptr.cast()) }) } + /// Read a value from I/O. + /// + /// This only works for primitives supported by the I/O backend. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_read_val(mmio: Mmio<'_, u32>) { + /// // let mmio: Mmio<'_, u32>; + /// let val: u32 = mmio.read_val(); + /// # } + /// ``` + #[inline] + fn read_val(self) -> Self::Target + where + Self::Backend: IoCapable, + Self::Target: Sized, + { + Self::Backend::io_read(self.as_view()) + } + + /// Write a value to I/O. + /// + /// This only works for primitives supported by the I/O backend. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_write_val(mmio: Mmio<'_, u32>) { + /// // let mmio: Mmio<'_, u32>; + /// mmio.write_val(1u32); + /// # } + /// ``` + #[inline] + fn write_val(self, value: Self::Target) + where + Self::Backend: IoCapable, + Self::Target: Sized, + { + Self::Backend::io_write(self.as_view(), value) + } + /// Fallible 8-bit read with runtime bounds check. #[inline(always)] fn try_read8(self, offset: usize) -> Result @@ -1232,3 +1276,65 @@ macro_rules! io_project { } #[doc(inline)] pub use crate::io_project; + +/// Read from I/O memory. +/// +/// The syntax is of form `io_read!(io, proj)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!). +/// +/// # Examples +/// +/// ``` +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<'_, [MyStruct]>; +/// let field: u32 = kernel::io::io_read!(mmio, [try: 2].field); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_read { + ($io:expr, $($proj:tt)*) => { + $crate::io::Io::read_val($crate::io_project!($io, $($proj)*)) + }; +} +#[doc(inline)] +pub use crate::io_read; + +/// Writes to I/O memory. +/// +/// The syntax is of form `io_write!(io, proj, val)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!), +/// and `val` is the value to be written to the projected location. +/// +/// # Examples +/// +/// ``` +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<'_, [MyStruct]>; +/// kernel::io::io_write!(mmio, [try: 2].field, 10); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_write { + (@parse [$io:expr] [$($proj:tt)*] [, $val:expr]) => { + $crate::io::Io::write_val($crate::io_project!($io, $($proj)*), $val) + }; + (@parse [$io:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [$($proj)* .$field] [$($rest)*]) + }; + (@parse [$io:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [$($proj)* [$flavor: $index]] [$($rest)*]) + }; + ($io:expr, $($rest:tt)*) => { + $crate::io_write!(@parse [$io] [] [$($rest)*]) + }; +} +#[doc(inline)] +pub use crate::io_write; From 0722567f5085bfc48d8b01c5c759745997e94785 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 6 Jul 2026 13:44:30 +0100 Subject: [PATCH 057/137] gpu: nova-core: use I/O projection for cleaner encapsulation Use `io_project!` for PTE array and message queues to restore the proper encapsulation. The remaining `dma_read!` and `dma_write!` is now only acting on primitives; thus replace by `io_read!` and `io_write!`. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Link: https://patch.msgid.link/20260706-io_projection-v6-17-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/gsp.rs | 53 ++++++++++---------- drivers/gpu/nova-core/gsp/cmdq.rs | 66 ++++++++++++++----------- drivers/gpu/nova-core/gsp/fw.rs | 82 +++++++++++-------------------- 3 files changed, 90 insertions(+), 111 deletions(-) diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index 69175ca3315c..cfa7553cd820 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -9,14 +9,16 @@ dma::{ Coherent, CoherentBox, + CoherentView, DmaAddress, // }, + io::{ + io_project, + io_write, + Io, // + }, pci, - prelude::*, - transmute::{ - AsBytes, - FromBytes, // - }, // + prelude::*, // }; pub(crate) mod cmdq; @@ -48,21 +50,21 @@ /// Array of page table entries, as understood by the GSP bootloader. #[repr(C)] +#[derive(FromBytes, IntoBytes)] struct PteArray([u64; NUM_ENTRIES]); -/// SAFETY: arrays of `u64` implement `FromBytes` and we are but a wrapper around one. -unsafe impl FromBytes for PteArray {} - -/// SAFETY: arrays of `u64` implement `AsBytes` and we are but a wrapper around one. -unsafe impl AsBytes for PteArray {} - impl PteArray { - /// Returns the page table entry for `index`, for a mapping starting at `start`. - // TODO: Replace with `IoView` projection once available. - fn entry(start: DmaAddress, index: usize) -> Result { - start - .checked_add(num::usize_as_u64(index) << GSP_PAGE_SHIFT) - .ok_or(EOVERFLOW) + /// Initialize a new page table array mapping `NUM_PAGES` GSP pages starting at address `start`. + fn init(view: CoherentView<'_, Self>, start: DmaAddress) -> Result<()> { + for i in 0..NUM_PAGES { + io_write!(view, .0[build: i], + start + .checked_add(num::usize_as_u64(i) << GSP_PAGE_SHIFT) + .ok_or(EOVERFLOW)? + ); + } + + Ok(()) } } @@ -89,17 +91,12 @@ fn new(dev: &device::Device) -> Result { let start_addr = obj.0.dma_handle(); - // SAFETY: `obj` has just been created and we are its sole user. - let pte_region = unsafe { - &mut obj.0.as_mut()[size_of::()..][..RM_LOG_BUFFER_NUM_PAGES * size_of::()] - }; - - // Write values one by one to avoid an on-stack instance of `PteArray`. - for (i, chunk) in pte_region.chunks_exact_mut(size_of::()).enumerate() { - let pte_value = PteArray::<0>::entry(start_addr, i)?; - - chunk.copy_from_slice(&pte_value.to_ne_bytes()); - } + let pte_view = io_project!( + obj.0, + [build: size_of::()..][build: ..RM_LOG_BUFFER_NUM_PAGES * size_of::()] + ) + .try_cast::>()?; + PteArray::init(pte_view, start_addr)?; Ok(obj) } diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index 070de0731e95..c34b48961496 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -2,16 +2,23 @@ mod continuation; -use core::mem; +use core::{ + mem, + sync::atomic::{ + fence, + Ordering, // + }, +}; use kernel::{ device, dma::{ Coherent, + CoherentBox, DmaAddress, // }, - dma_write, io::{ + io_project, poll::read_poll_timeout, Io, // }, @@ -171,20 +178,18 @@ struct MsgqData { #[repr(C)] // There is no struct defined for this in the open-gpu-kernel-source headers. // Instead it is defined by code in `GspMsgQueuesInit()`. -// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module. -pub(super) struct Msgq { +struct Msgq { /// Header for sending messages, including the write pointer. - pub(super) tx: MsgqTxHeader, + tx: MsgqTxHeader, /// Header for receiving messages, including the read pointer. - pub(super) rx: MsgqRxHeader, + rx: MsgqRxHeader, /// The message queue proper. msgq: MsgqData, } /// Structure shared between the driver and the GSP and containing the command and message queues. #[repr(C)] -// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module. -pub(super) struct GspMem { +struct GspMem { /// Self-mapping page table entries. ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>, /// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the @@ -192,13 +197,13 @@ pub(super) struct GspMem { /// index into the GSP queue. /// /// This member is read-only for the GSP. - pub(super) cpuq: Msgq, + cpuq: Msgq, /// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the /// write and read pointers that the GSP updates. This means that the read pointer here is an /// index into the CPU queue. /// /// This member is read-only for the driver. - pub(super) gspq: Msgq, + gspq: Msgq, } impl GspMem { @@ -232,20 +237,12 @@ fn new(dev: &device::Device) -> Result { const MSGQ_SIZE: u32 = num::usize_into_u32::<{ size_of::() }>(); const RX_HDR_OFF: u32 = num::usize_into_u32::<{ mem::offset_of!(Msgq, rx) }>(); - let gsp_mem = Coherent::::zeroed(dev, GFP_KERNEL)?; + let mut gsp_mem = CoherentBox::::zeroed(dev, GFP_KERNEL)?; + gsp_mem.cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES); + gsp_mem.cpuq.rx = MsgqRxHeader::new(); - let start = gsp_mem.dma_handle(); - // Write values one by one to avoid an on-stack instance of `PteArray`. - for i in 0..GspMem::PTE_ARRAY_SIZE { - dma_write!(gsp_mem, .ptes.0[build: i], PteArray::<0>::entry(start, i)?); - } - - dma_write!( - gsp_mem, - .cpuq.tx, - MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES) - ); - dma_write!(gsp_mem, .cpuq.rx, MsgqRxHeader::new()); + let gsp_mem: Coherent<_> = gsp_mem.into(); + PteArray::init(io_project!(gsp_mem, .ptes), gsp_mem.dma_handle())?; Ok(Self(gsp_mem)) } @@ -406,7 +403,7 @@ fn allocate_command(&mut self, size: usize, timeout: Delta) -> Result u32 { - super::fw::gsp_mem::gsp_write_ptr(&self.0) + MsgqTxHeader::write_ptr(io_project!(self.0, .gspq.tx)) % MSGQ_NUM_PAGES } // Returns the index of the memory page the GSP will read the next command from. @@ -415,7 +412,7 @@ fn gsp_write_ptr(&self) -> u32 { // // - The returned value is within `0..MSGQ_NUM_PAGES`. fn gsp_read_ptr(&self) -> u32 { - super::fw::gsp_mem::gsp_read_ptr(&self.0) + MsgqRxHeader::read_ptr(io_project!(self.0, .gspq.rx)) % MSGQ_NUM_PAGES } // Returns the index of the memory page the CPU can read the next message from. @@ -424,12 +421,18 @@ fn gsp_read_ptr(&self) -> u32 { // // - The returned value is within `0..MSGQ_NUM_PAGES`. fn cpu_read_ptr(&self) -> u32 { - super::fw::gsp_mem::cpu_read_ptr(&self.0) + MsgqRxHeader::read_ptr(io_project!(self.0, .cpuq.rx)) % MSGQ_NUM_PAGES } // Informs the GSP that it can send `elem_count` new pages into the message queue. fn advance_cpu_read_ptr(&mut self, elem_count: u32) { - super::fw::gsp_mem::advance_cpu_read_ptr(&self.0, elem_count) + let rx = io_project!(self.0, .cpuq.rx); + let rptr = MsgqRxHeader::read_ptr(rx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; + + // Ensure read pointer is properly ordered. + fence(Ordering::SeqCst); + + MsgqRxHeader::set_read_ptr(rx, rptr) } // Returns the index of the memory page the CPU can write the next command to. @@ -438,12 +441,17 @@ fn advance_cpu_read_ptr(&mut self, elem_count: u32) { // // - The returned value is within `0..MSGQ_NUM_PAGES`. fn cpu_write_ptr(&self) -> u32 { - super::fw::gsp_mem::cpu_write_ptr(&self.0) + MsgqTxHeader::write_ptr(io_project!(self.0, .cpuq.tx)) % MSGQ_NUM_PAGES } // Informs the GSP that it can process `elem_count` new pages from the command queue. fn advance_cpu_write_ptr(&mut self, elem_count: u32) { - super::fw::gsp_mem::advance_cpu_write_ptr(&self.0, elem_count) + let tx = io_project!(self.0, .cpuq.tx); + let wptr = MsgqTxHeader::write_ptr(tx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; + MsgqTxHeader::set_write_ptr(tx, wptr); + + // Ensure all command data is visible before triggering the GSP read. + fence(Ordering::SeqCst); } } diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 4db0cfa4dc4d..b0e7de328eaf 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -10,7 +10,14 @@ use core::ops::Range; use kernel::{ - dma::Coherent, + dma::{ + Coherent, + CoherentView, // + }, + io::{ + io_read, + io_write, // + }, prelude::*, ptr::{ Alignable, @@ -44,59 +51,6 @@ }, }; -// TODO: Replace with `IoView` projections once available. -pub(super) mod gsp_mem { - use core::sync::atomic::{ - fence, - Ordering, // - }; - - use kernel::{ - dma::Coherent, - dma_read, - dma_write, // - }; - - use crate::gsp::cmdq::{ - GspMem, - MSGQ_NUM_PAGES, // - }; - - pub(in crate::gsp) fn gsp_write_ptr(qs: &Coherent) -> u32 { - dma_read!(qs, .gspq.tx.0.writePtr) % MSGQ_NUM_PAGES - } - - pub(in crate::gsp) fn gsp_read_ptr(qs: &Coherent) -> u32 { - dma_read!(qs, .gspq.rx.0.readPtr) % MSGQ_NUM_PAGES - } - - pub(in crate::gsp) fn cpu_read_ptr(qs: &Coherent) -> u32 { - dma_read!(qs, .cpuq.rx.0.readPtr) % MSGQ_NUM_PAGES - } - - pub(in crate::gsp) fn advance_cpu_read_ptr(qs: &Coherent, count: u32) { - let rptr = cpu_read_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES; - - // Ensure read pointer is properly ordered. - fence(Ordering::SeqCst); - - dma_write!(qs, .cpuq.rx.0.readPtr, rptr); - } - - pub(in crate::gsp) fn cpu_write_ptr(qs: &Coherent) -> u32 { - dma_read!(qs, .cpuq.tx.0.writePtr) % MSGQ_NUM_PAGES - } - - pub(in crate::gsp) fn advance_cpu_write_ptr(qs: &Coherent, count: u32) { - let wptr = cpu_write_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES; - - dma_write!(qs, .cpuq.tx.0.writePtr, wptr); - - // Ensure all command data is visible before triggering the GSP read. - fence(Ordering::SeqCst); - } -} - /// Maximum size of a single GSP message queue element in bytes. pub(crate) const GSP_MSG_QUEUE_ELEMENT_SIZE_MAX: usize = num::u32_as_usize(bindings::GSP_MSG_QUEUE_ELEMENT_SIZE_MAX); @@ -720,6 +674,16 @@ pub(crate) fn new(msgq_size: u32, rx_hdr_offset: u32, msg_count: u32) -> Self { entryOff: num::usize_into_u32::(), }) } + + /// Returns the value of the write pointer for this queue. + pub(crate) fn write_ptr(this: CoherentView<'_, Self>) -> u32 { + io_read!(this, .0.writePtr) + } + + /// Sets the value of the write pointer for this queue. + pub(crate) fn set_write_ptr(this: CoherentView<'_, Self>, val: u32) { + io_write!(this, .0.writePtr, val) + } } // SAFETY: Padding is explicit and does not contain uninitialized data. @@ -735,6 +699,16 @@ impl MsgqRxHeader { pub(crate) fn new() -> Self { Self(Default::default()) } + + /// Returns the value of the read pointer for this queue. + pub(crate) fn read_ptr(this: CoherentView<'_, Self>) -> u32 { + io_read!(this, .0.readPtr) + } + + /// Sets the value of the read pointer for this queue. + pub(crate) fn set_read_ptr(this: CoherentView<'_, Self>, val: u32) { + io_write!(this, .0.readPtr, val) + } } // SAFETY: Padding is explicit and does not contain uninitialized data. From 6ff7d69b7e6e0b09d53ffde472760f904ea5714f Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 6 Jul 2026 13:44:31 +0100 Subject: [PATCH 058/137] rust: dma: drop `dma_read!` and `dma_write!` API The primitive read/write use case is covered by the `io_read!` and `io_write!` macro. The non-primitive use case was finicky; they should either be achieved using `CoherentBox` or `as_ref()/as_mut()` to assert the lack of concurrent access, or should be using memcpy-like APIs to express the non-atomic and tearable nature. Reviewed-by: Andreas Hindborg Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260706-io_projection-v6-18-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/dma.rs | 128 --------------------------------------- samples/rust/rust_dma.rs | 11 +++- 2 files changed, 8 insertions(+), 131 deletions(-) diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 1535bc6eec64..6e7ea3b72f2f 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -661,52 +661,6 @@ pub unsafe fn as_mut(&self) -> &mut T { // SAFETY: per safety requirement. unsafe { &mut *self.as_mut_ptr() } } - - /// Reads the value of `field` and ensures that its type is [`FromBytes`]. - /// - /// # Safety - /// - /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is - /// validated beforehand. - /// - /// Public but hidden since it should only be used from [`dma_read`] macro. - #[doc(hidden)] - pub unsafe fn field_read(&self, field: *const F) -> F { - // SAFETY: - // - By the safety requirements field is valid. - // - Using read_volatile() here is not sound as per the usual rules, the usage here is - // a special exception with the following notes in place. When dealing with a potential - // race from a hardware or code outside kernel (e.g. user-space program), we need that - // read on a valid memory is not UB. Currently read_volatile() is used for this, and the - // rationale behind is that it should generate the same code as READ_ONCE() which the - // kernel already relies on to avoid UB on data races. Note that the usage of - // read_volatile() is limited to this particular case, it cannot be used to prevent - // the UB caused by racing between two kernel functions nor do they provide atomicity. - unsafe { field.read_volatile() } - } - - /// Writes a value to `field` and ensures that its type is [`AsBytes`]. - /// - /// # Safety - /// - /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is - /// validated beforehand. - /// - /// Public but hidden since it should only be used from [`dma_write`] macro. - #[doc(hidden)] - pub unsafe fn field_write(&self, field: *mut F, val: F) { - // SAFETY: - // - By the safety requirements field is valid. - // - Using write_volatile() here is not sound as per the usual rules, the usage here is - // a special exception with the following notes in place. When dealing with a potential - // race from a hardware or code outside kernel (e.g. user-space program), we need that - // write on a valid memory is not UB. Currently write_volatile() is used for this, and the - // rationale behind is that it should generate the same code as WRITE_ONCE() which the - // kernel already relies on to avoid UB on data races. Note that the usage of - // write_volatile() is limited to this particular case, it cannot be used to prevent - // the UB caused by racing between two kernel functions nor do they provide atomicity. - unsafe { field.write_volatile(val) } - } } impl Coherent { @@ -1266,85 +1220,3 @@ fn as_view(self) -> CoherentView<'a, Self::Target> { } } } - -/// Reads a field of an item from an allocated region of structs. -/// -/// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating -/// to a [`Coherent`] and `proj` is a [projection specification](kernel::ptr::project!). -/// -/// # Examples -/// -/// ``` -/// use kernel::device::Device; -/// use kernel::dma::{attrs::*, Coherent}; -/// -/// struct MyStruct { field: u32, } -/// -/// // SAFETY: All bit patterns are acceptable values for `MyStruct`. -/// unsafe impl kernel::transmute::FromBytes for MyStruct{}; -/// // SAFETY: Instances of `MyStruct` have no uninitialized portions. -/// unsafe impl kernel::transmute::AsBytes for MyStruct{}; -/// -/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result { -/// let whole = kernel::dma_read!(alloc, [try: 2]); -/// let field = kernel::dma_read!(alloc, [panic: 1].field); -/// # Ok::<(), Error>(()) } -/// ``` -#[macro_export] -macro_rules! dma_read { - ($dma:expr, $($proj:tt)*) => {{ - let dma = &$dma; - let ptr = $crate::ptr::project!( - $crate::dma::Coherent::as_ptr(dma), $($proj)* - ); - // SAFETY: The pointer created by the projection is within the DMA region. - unsafe { $crate::dma::Coherent::field_read(dma, ptr) } - }}; -} - -/// Writes to a field of an item from an allocated region of structs. -/// -/// The syntax is of the form `kernel::dma_write!(dma, proj, val)` where `dma` is an expression -/// evaluating to a [`Coherent`], `proj` is a -/// [projection specification](kernel::ptr::project!), and `val` is the value to be written to the -/// projected location. -/// -/// # Examples -/// -/// ``` -/// use kernel::device::Device; -/// use kernel::dma::{attrs::*, Coherent}; -/// -/// struct MyStruct { member: u32, } -/// -/// // SAFETY: All bit patterns are acceptable values for `MyStruct`. -/// unsafe impl kernel::transmute::FromBytes for MyStruct{}; -/// // SAFETY: Instances of `MyStruct` have no uninitialized portions. -/// unsafe impl kernel::transmute::AsBytes for MyStruct{}; -/// -/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result { -/// kernel::dma_write!(alloc, [try: 2].member, 0xf); -/// kernel::dma_write!(alloc, [panic: 1], MyStruct { member: 0xf }); -/// # Ok::<(), Error>(()) } -/// ``` -#[macro_export] -macro_rules! dma_write { - (@parse [$dma:expr] [$($proj:tt)*] [, $val:expr]) => {{ - let dma = &$dma; - let ptr = $crate::ptr::project!( - mut $crate::dma::Coherent::as_mut_ptr(dma), $($proj)* - ); - let val = $val; - // SAFETY: The pointer created by the projection is within the DMA region. - unsafe { $crate::dma::Coherent::field_write(dma, ptr, val) } - }}; - (@parse [$dma:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => { - $crate::dma_write!(@parse [$dma] [$($proj)* .$field] [$($rest)*]) - }; - (@parse [$dma:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => { - $crate::dma_write!(@parse [$dma] [$($proj)* [$flavor: $index]] [$($rest)*]) - }; - ($dma:expr, $($rest:tt)*) => { - $crate::dma_write!(@parse [$dma] [] [$($rest)*]) - }; -} diff --git a/samples/rust/rust_dma.rs b/samples/rust/rust_dma.rs index 5046b4628d0e..4af46e99d2dd 100644 --- a/samples/rust/rust_dma.rs +++ b/samples/rust/rust_dma.rs @@ -12,6 +12,10 @@ Device, DmaMask, // }, + io::{ + io_project, + io_read, // + }, page, pci, prelude::*, scatterlist::{Owned, SGTable}, @@ -77,7 +81,8 @@ fn probe<'bound>( Coherent::zeroed_slice(pdev.as_ref(), TEST_VALUES.len(), GFP_KERNEL)?; for (i, value) in TEST_VALUES.into_iter().enumerate() { - kernel::dma_write!(ca, [try: i], MyStruct::new(value.0, value.1)); + // SAFETY: `ca` is not yet shared with device or other threads. + unsafe { *io_project!(ca, [panic: i]).as_mut() = MyStruct::new(value.0, value.1) }; } let size = 4 * page::PAGE_SIZE; @@ -97,8 +102,8 @@ fn probe<'bound>( impl DmaSampleDriver { fn check_dma(&self) { for (i, value) in TEST_VALUES.into_iter().enumerate() { - let val0 = kernel::dma_read!(self.ca, [panic: i].h); - let val1 = kernel::dma_read!(self.ca, [panic: i].b); + let val0 = io_read!(self.ca, [panic: i].h); + let val1 = io_read!(self.ca, [panic: i].b); assert_eq!(val0, value.0); assert_eq!(val1, value.1); From e7219e53c525db87b43f4a9064d0e6331d7dc710 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 6 Jul 2026 13:44:32 +0100 Subject: [PATCH 059/137] rust: io: add copying methods One feature that was lost from the old `dma_read!` and `dma_write!` when moving to `io_read!` and `io_write!` was the ability to read/write a large structs. However, the semantics was unclear to begin with, as there was no guarantee about their atomicity even for structs that were small enough to fit in u32. Re-introduce the capability in the form of copying methods. dma_read!(foo, bar) -> io_project!(foo, bar).copy_read() dma_write!(foo, bar, baz) -> io_project!(foo, bar).copy_write(baz) Model these semantics after memcpy so user has clear expectation of lack of atomicity. As an additional benefit of this change, this now works for MMIO as well by mapping them to `memcpy_{from,to}io`. For slices which is DST so the `copy_read` and `copy_write` API above can't work, add `copy_from_slice` and `copy_to_slice` to copy from/to normal memory. Signed-off-by: Gary Guo Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260706-io_projection-v6-19-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich --- rust/helpers/io.c | 13 ++ rust/kernel/dma.rs | 25 ++++ rust/kernel/io.rs | 262 ++++++++++++++++++++++++++++++++++++++- samples/rust/rust_dma.rs | 7 +- 4 files changed, 303 insertions(+), 4 deletions(-) diff --git a/rust/helpers/io.c b/rust/helpers/io.c index 397810864a24..7ed9a4f77f1b 100644 --- a/rust/helpers/io.c +++ b/rust/helpers/io.c @@ -19,6 +19,19 @@ __rust_helper void rust_helper_iounmap(void __iomem *addr) iounmap(addr); } +__rust_helper void rust_helper_memcpy_fromio(void *dst, + const volatile void __iomem *src, + size_t count) +{ + memcpy_fromio(dst, src, count); +} + +__rust_helper void rust_helper_memcpy_toio(volatile void __iomem *dst, + const void *src, size_t count) +{ + memcpy_toio(dst, src, count); +} + __rust_helper u8 rust_helper_readb(const void __iomem *addr) { return readb(addr); diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 6e7ea3b72f2f..e275f2562a5b 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -18,6 +18,7 @@ IoBackend, IoBase, IoCapable, + IoCopyable, SysMem, SysMemBackend, // }, @@ -1197,6 +1198,30 @@ fn io_write<'a>(view: Self::View<'a, T>, value: T) { } } +impl IoCopyable for CoherentIoBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // SAFETY: Per safety requirement. + unsafe { SysMemBackend::copy_from_io(view.cpu_addr, buffer) } + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // SAFETY: Per safety requirement. + unsafe { SysMemBackend::copy_to_io(view.cpu_addr, buffer) } + } + + #[inline] + fn copy_read(view: Self::View<'_, T>) -> T { + SysMemBackend::copy_read(view.cpu_addr) + } + + #[inline] + fn copy_write(view: Self::View<'_, T>, value: T) { + SysMemBackend::copy_write(view.cpu_addr, value) + } +} + impl<'a, T: ?Sized + KnownSize> IoBase<'a> for CoherentView<'a, T> { type Backend = CoherentIoBackend; type Target = T; diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 3f430bce61e5..dbaa88898c3b 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -5,7 +5,8 @@ //! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h) use core::{ - marker::PhantomData, // + marker::PhantomData, + mem::MaybeUninit, // }; use crate::{ @@ -275,6 +276,69 @@ pub trait IoCapable: IoBackend { fn io_write<'a>(view: Self::View<'a, T>, value: T); } +/// Trait indicating that an I/O backend supports memory copy operations. +pub trait IoCopyable: IoBackend { + /// Copy contents of `view` to `buffer`. + /// + /// # Safety + /// + /// - `buffer` is valid for volatile write for `view.size()` bytes. + /// - `buffer` should not overlap with `view`. + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8); + + /// Copy contents from `buffer` to `view`. + /// + /// # Safety + /// + /// - `buffer` is valid for volatile read for `view.size()` bytes. + /// - `buffer` should not overlap with `view`. + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8); + + /// Copy from `view` and return the value. + #[inline] + fn copy_read(view: Self::View<'_, T>) -> T { + // Project `self` to `[u8]`. + let ptr = Self::as_ptr(view); + // SAFETY: This is a identity projection. + let slice_view = unsafe { + Self::project_view( + view, + core::ptr::slice_from_raw_parts_mut::(ptr.cast(), size_of::()), + ) + }; + + let mut buf = MaybeUninit::::uninit(); + // SAFETY: + // - `buf.as_mut_ptr()` is valid for write for `size_of::()` bytes. + // - `buf` is local so `buf.as_mut_ptr()` cannot overlap with `slice_view`. + unsafe { Self::copy_from_io(slice_view, buf.as_mut_ptr().cast()) }; + // SAFETY: `T: FromBytes` guarantee that all bit patterns are valid. + unsafe { buf.assume_init() } + } + + /// Copy `value` to `view`. + /// + /// Destructor of `value` will not be executed, consistent with [`zerocopy::transmute`]. + #[inline] + fn copy_write(view: Self::View<'_, T>, value: T) { + // Project `self` to `[u8]`. + let ptr = Self::as_ptr(view); + // SAFETY: This is a identity projection. + let slice_view = unsafe { + Self::project_view( + view, + core::ptr::slice_from_raw_parts_mut::(ptr.cast(), size_of::()), + ) + }; + + // SAFETY: + // - `&raw const value` is valid for read for `size_of::()` bytes. + // - `value` is local so `&raw const value` cannot overlap with `slice_view`. + unsafe { Self::copy_to_io(slice_view, (&raw const value).cast()) }; + core::mem::forget(value); + } +} + /// Describes a given I/O location: its offset, width, and type to convert the raw value from and /// into. /// @@ -354,6 +418,24 @@ fn size(self) -> usize { KnownSize::size(Self::Backend::as_ptr(self.as_view())) } + /// Returns the length of the slice in number of elements. + #[inline] + fn len(self) -> usize + where + Self: Io<'a, Target = [T]>, + { + Self::Backend::as_ptr(self.as_view()).len() + } + + /// Returns `true` if the slice has a length of 0. + #[inline] + fn is_empty(self) -> bool + where + Self: Io<'a, Target = [T]>, + { + self.len() == 0 + } + /// Try to convert into a different typed I/O view. /// /// A runtime check is performed to ensure that the target type is of same or smaller size to @@ -443,6 +525,121 @@ fn write_val(self, value: Self::Target) Self::Backend::io_write(self.as_view(), value) } + /// Copy-read from I/O memory. + /// + /// This is equivalent to reading from the I/O memory with byte-wise copy, although the actual + /// implementation might be more efficient. There is no atomicity guarantee. Note that for some + /// backends (e.g. `Mmio`), this can read different value compared to [`read_val`] as + /// byte-swapping is not performed. + /// + /// [`read_val`]: Io::read_val + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_read(mmio: Mmio<'_, [u8; 6]>) { + /// // let mmio: Mmio<'_, [u8; 6]>; + /// let val: [u8; 6] = mmio.copy_read(); + /// # } + /// ``` + #[inline] + fn copy_read(self) -> Self::Target + where + Self::Backend: IoCopyable, + Self::Target: Sized + FromBytes, + { + Self::Backend::copy_read(self.as_view()) + } + + /// Copy-write to I/O memory. + /// + /// This is equivalent to writing to the I/O memory with byte-wise copy, although the actual + /// implementation might be more efficient. There is no atomicity guarantee. Note that for some + /// backends (e.g. `Mmio`), this can write different value compared to [`write_val`] as + /// byte-swapping is not performed. + /// + /// [`write_val`]: Io::write_val + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8; 6]>) { + /// // let mmio: Mmio<'_, [u8; 6]>; + /// mmio.copy_write([0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]); + /// # } + /// ``` + #[inline] + fn copy_write(self, value: Self::Target) + where + Self::Backend: IoCopyable, + Self::Target: Sized + IntoBytes, + { + Self::Backend::copy_write(self.as_view(), value); + } + + /// Copy bytes from `data` to I/O memory. + /// + /// # Panics + /// + /// This function will panic if the length of `self` differs from the length of `data`, similar + /// to [`[u8]::copy_from_slice`]. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) { + /// // let mmio: Mmio<'_, [u8]>; + /// mmio.copy_from_slice(&[0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]); + /// # } + /// ``` + #[inline] + fn copy_from_slice(self, data: &[u8]) + where + Self::Backend: IoCopyable, + Self: Io<'a, Target = [u8]>, + { + assert_eq!(self.len(), data.len()); + + // SAFETY: `data.as_ptr()` is valid for read for `self.size()` bytes. + unsafe { + Self::Backend::copy_to_io(self.as_view(), data.as_ptr()); + } + } + + /// Copy bytes from I/O memory to `data`. + /// + /// # Panics + /// + /// This function will panic if the length of `self` differs from the length of `data`, similar + /// to [`[u8]::copy_from_slice`]. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) { + /// // let mmio: Mmio<'_, [u8]>; + /// let mut buf = [0; 6]; + /// mmio.copy_to_slice(&mut buf); + /// # } + /// ``` + #[inline] + fn copy_to_slice(self, data: &mut [u8]) + where + Self::Backend: IoCopyable, + Self: Io<'a, Target = [u8]>, + { + assert_eq!(self.len(), data.len()); + + // SAFETY: `data.as_mut_ptr()` is valid for write for `self.size()` bytes. + unsafe { + Self::Backend::copy_from_io(self.as_view(), data.as_mut_ptr()); + } + } + /// Fallible 8-bit read with runtime bounds check. #[inline(always)] fn try_read8(self, offset: usize) -> Result @@ -1003,6 +1200,28 @@ fn io_write(view: <$backend as IoBackend>::View<'_, $ty>, value: $ty) { #[cfg(CONFIG_64BIT)] impl_mmio_io_capable!(MmioBackend, u64, readq, writeq); +impl IoCopyable for MmioBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // SAFETY: + // - `view.ptr` is valid MMIO memory for `view.size()` bytes. + // - `buffer` is valid for write for `view.size()` bytes. + unsafe { + bindings::memcpy_fromio(buffer.cast(), view.ptr.cast(), view.size()); + } + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // SAFETY: + // - `view.ptr` is valid MMIO memory for `view.size()` bytes. + // - `buffer` is valid for read for `view.size()` bytes. + unsafe { + bindings::memcpy_toio(view.ptr.cast(), buffer.cast(), view.size()); + } + } +} + /// [`Mmio`] but using relaxed accessors. /// /// This type provides an implementation of [`Io`] that uses relaxed I/O MMIO operands instead of @@ -1146,6 +1365,47 @@ fn io_write(view: SysMem<'_, $ty>, value: $ty) { #[cfg(CONFIG_64BIT)] impl_sysmem_io_capable!(u64); +impl IoCopyable for SysMemBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile. + // SAFETY: + // - `view.ptr` is in CPU address space and valid for read. + // - `buffer` is valid for write for `view.size()` bytes which is equal to `view.ptr.len()`. + unsafe { bindings::memcpy(buffer.cast(), view.ptr.cast(), view.ptr.len()) }; + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile. + // SAFETY: + // - `view.ptr` is in CPU address space and valid for write. + // - `buffer` is valid for read for `view.size()` bytes which is equal to `view.ptr.len()`. + unsafe { bindings::memcpy(view.ptr.cast(), buffer.cast(), view.ptr.len()) }; + } + + #[inline] + fn copy_read(view: Self::View<'_, T>) -> T { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using read_volatile() here so that race with hardware is well-defined. + // - Using read_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + // - `T: FromBytes` so all bit patterns are valid. + unsafe { view.ptr.read_volatile() } + } + + #[inline] + fn copy_write(view: Self::View<'_, T>, value: T) { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using write_volatile() here so that race with hardware is well-defined. + // - Using write_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + unsafe { view.ptr.write_volatile(value) } + } +} + /// A view of a system memory region. /// /// Provides `Io` trait implementation for kernel virtual address ranges, diff --git a/samples/rust/rust_dma.rs b/samples/rust/rust_dma.rs index 4af46e99d2dd..b629acc6d915 100644 --- a/samples/rust/rust_dma.rs +++ b/samples/rust/rust_dma.rs @@ -14,7 +14,8 @@ }, io::{ io_project, - io_read, // + io_read, + Io, // }, page, pci, prelude::*, @@ -38,6 +39,7 @@ struct DmaSampleDriver { (0xcd, 0xef), ]; +#[derive(FromBytes, IntoBytes)] struct MyStruct { h: u32, b: u32, @@ -81,8 +83,7 @@ fn probe<'bound>( Coherent::zeroed_slice(pdev.as_ref(), TEST_VALUES.len(), GFP_KERNEL)?; for (i, value) in TEST_VALUES.into_iter().enumerate() { - // SAFETY: `ca` is not yet shared with device or other threads. - unsafe { *io_project!(ca, [panic: i]).as_mut() = MyStruct::new(value.0, value.1) }; + io_project!(ca, [panic: i]).copy_write(MyStruct::new(value.0, value.1)); } let size = 4 * page::PAGE_SIZE; From 11a4784f902ebf3e674dbaf07dbec9a37aabb5e4 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 6 Jul 2026 13:44:33 +0100 Subject: [PATCH 060/137] rust: io: implement `IoSysMap` Add an enum as sum type for `Mmio` and `SysMem`. This serves similar purpose of `iosys_map`. Thanks to Rust's type system, all of projection and struct read/write can be handled by the generic I/O projection mechanism (i.e. `io_project!`, `io_read!, `io_write!`) for free, and there is no need to provide things like `iosys_map_rd_field` or `iosys_map_wr_field`. An enum type also makes it very easy to construct or destruct. This could be made more generic by implementing on a general purpose sum type like `Either`; however this is kept specific unless a need arises that warrants this to be generic over other I/O backends. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Link: https://patch.msgid.link/20260706-io_projection-v6-20-72cd5d055d54@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 137 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index dbaa88898c3b..95f46bb75f9e 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -1468,6 +1468,143 @@ fn as_view(self) -> ::View<'a, Self::Target> { } } +/// I/O Backend for [`IoSysMap`]. +pub struct IoSysMapBackend; + +/// Either [`Mmio`] or [`SysMem`]. +/// +/// This can be used when a piece of logic may wish to handle both MMIO or system memory but does +/// not want or cannot be generic over I/O backends. This serves a similar purpose to +/// [`include/linux/iosys-map.h`] in C. +/// +/// This type can be used like any other types that implements [`Io`]; this also include +/// [`io_project!`], [`io_read!`], [`io_write!`]. +/// +/// [`include/linux/iosys-map.h`]: srctree/include/linux/iosys-map.h +pub enum IoSysMap<'a, T: ?Sized> { + /// The view is I/O memory. + Io(Mmio<'a, T>), + /// The view is system memory. + Sys(SysMem<'a, T>), +} + +impl Copy for IoSysMap<'_, T> {} +impl Clone for IoSysMap<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T: ?Sized> From> for IoSysMap<'a, T> { + #[inline] + fn from(value: Mmio<'a, T>) -> Self { + IoSysMap::Io(value) + } +} + +impl<'a, T: ?Sized> From> for IoSysMap<'a, T> { + #[inline] + fn from(value: SysMem<'a, T>) -> Self { + IoSysMap::Sys(value) + } +} + +impl IoBackend for IoSysMapBackend { + type View<'a, T: ?Sized + KnownSize> = IoSysMap<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + match view { + IoSysMap::Io(l) => MmioBackend::as_ptr(l), + IoSysMap::Sys(r) => SysMemBackend::as_ptr(r), + } + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => IoSysMap::Io(unsafe { MmioBackend::project_view(l, ptr) }), + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => IoSysMap::Sys(unsafe { SysMemBackend::project_view(r, ptr) }), + } + } +} + +impl IoCapable for IoSysMapBackend +where + MmioBackend: IoCapable, + SysMemBackend: IoCapable, +{ + #[inline] + fn io_read(view: Self::View<'_, T>) -> T { + match view { + IoSysMap::Io(l) => MmioBackend::io_read(l), + IoSysMap::Sys(r) => SysMemBackend::io_read(r), + } + } + + #[inline] + fn io_write<'a>(view: Self::View<'a, T>, value: T) { + match view { + IoSysMap::Io(l) => MmioBackend::io_write(l, value), + IoSysMap::Sys(r) => SysMemBackend::io_write(r, value), + } + } +} + +impl IoCopyable for IoSysMapBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => unsafe { MmioBackend::copy_from_io(l, buffer) }, + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_from_io(r, buffer) }, + } + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => unsafe { MmioBackend::copy_to_io(l, buffer) }, + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_to_io(r, buffer) }, + } + } + + #[inline] + fn copy_read(view: Self::View<'_, T>) -> T { + match view { + IoSysMap::Io(l) => MmioBackend::copy_read(l), + IoSysMap::Sys(r) => SysMemBackend::copy_read(r), + } + } + + #[inline] + fn copy_write(view: Self::View<'_, T>, value: T) { + match view { + IoSysMap::Io(l) => MmioBackend::copy_write(l, value), + IoSysMap::Sys(r) => SysMemBackend::copy_write(r, value), + } + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for IoSysMap<'a, T> { + type Backend = IoSysMapBackend; + type Target = T; + + #[inline] + fn as_view(self) -> IoSysMap<'a, T> { + self + } +} + // This helper turns associated functions to methods so it can be invoked in macro. // Used by `io_project!()` only. #[doc(hidden)] From 68b151bc6145dea3db5598ebaf4b776cd205e395 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 28 Jun 2026 16:53:21 +0200 Subject: [PATCH 061/137] rust: drm: ioctl: fix unbounded lifetimes in ioctl handler arguments References to dev, data, and file in the declare_drm_ioctls! macro are created via unsafe pointer dereferences, producing unbounded lifetimes. If an ioctl handler explicitly annotates its parameters with 'static, the compiler accepts this, allowing the handler to stash references that outlive the ioctl call. Fix this by adding a higher-ranked function pointer coercion that enforces the handler accepts universally quantified lifetimes: let _: for<'a> fn(&'a _, &'a mut _, &'a _) -> _ = $func; Since the handler must be coercible to a function pointer accepting any lifetime 'a, it can no longer demand 'static on any parameter. Cc: stable@vger.kernel.org Fixes: 9a69570682b1 ("rust: drm: ioctl: Add DRM ioctl abstraction") Reported-by: sashiko-bot@kernel.org Closes: https://lore.kernel.org/all/20260620011346.A47D01F000E9@smtp.kernel.org/ Suggested-by: Gary Guo Reviewed-by: Alexandre Courbot Reviewed-by: Lyude Paul Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260628145406.2107056-2-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/drm/ioctl.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rust/kernel/drm/ioctl.rs b/rust/kernel/drm/ioctl.rs index cf328101dde4..ccf4150d83b6 100644 --- a/rust/kernel/drm/ioctl.rs +++ b/rust/kernel/drm/ioctl.rs @@ -135,6 +135,12 @@ macro_rules! declare_drm_ioctls { // dev/file match the current driver these ioctls are being declared // for, and it's not clear how to enforce this within the type system. let dev = $crate::drm::device::Device::from_raw(raw_dev); + + // Enforce that the handler accepts higher-ranked + // lifetimes, preventing it from requiring 'static + // references that could escape this scope. + let _: for<'a> fn(&'a _, &'a mut _, &'a _) -> _ = $func; + // SAFETY: The ioctl argument has size `_IOC_SIZE(cmd)`, which we // asserted above matches the size of this type, and all bit patterns of // UAPI structs must be valid. From fd0f827c532b976d38e3cbbda5d3fa60a82ce8d5 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 28 Jun 2026 16:53:22 +0200 Subject: [PATCH 062/137] rust: drm: rename Uninit DeviceContext to Normal Rename the Uninit DeviceContext to Normal to better reflect its purpose as the general-purpose, reference-counted device context. The Uninit name was a leftover from when DRM device private data initialization was planned to split across UnregisteredDevice::new() and Registration::new(); with the subsequent introduction of RegistrationData, this distinction is no longer needed. This also simplifies the DeviceContext documentation, trimming the multi-stage initialization description that no longer applies. Subsequent patches will refine the semantics of the Registered context accordingly. No functional change. Reviewed-by: Lyude Paul Reviewed-by: Alexandre Courbot Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260628145406.2107056-3-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/drm/device.rs | 92 ++++++++++++--------------------------- rust/kernel/drm/mod.rs | 2 +- 2 files changed, 28 insertions(+), 66 deletions(-) diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 7ad124327a83..35ff9c6942d8 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -74,36 +74,22 @@ macro_rules! drm_legacy_fields { /// A trait implemented by all possible contexts a [`Device`] can be used in. /// -/// Setting up a new [`Device`] is a multi-stage process. Each step of the process that a user -/// interacts with in Rust has a respective [`DeviceContext`] typestate. For example, -/// `Device` would be a [`Device`] that reached the [`Registered`] [`DeviceContext`]. +/// A [`Device`] can be in one of two contexts: /// -/// Each stage of this process is described below: -/// -/// ```text -/// 1 2 3 -/// +--------------+ +------------------+ +-----------------------+ -/// |Device created| → |Device initialized| → |Registered w/ userspace| -/// +--------------+ +------------------+ +-----------------------+ -/// (Uninit) (Registered) -/// ``` -/// -/// 1. The [`Device`] is in the [`Uninit`] context and is not guaranteed to be initialized or -/// registered with userspace. Only a limited subset of DRM core functionality is available. -/// 2. The [`Device`] is guaranteed to be fully initialized, but is not guaranteed to be registered -/// with userspace. All DRM core functionality which doesn't interact with userspace is -/// available. We currently don't have a context for representing this. -/// 3. The [`Device`] is guaranteed to be fully initialized, and is guaranteed to have been -/// registered with userspace at some point - thus putting it in the [`Registered`] context. -/// -/// An important caveat of [`DeviceContext`] which must be kept in mind: when used as a typestate -/// for a reference type, it can only guarantee that a [`Device`] reached a particular stage in the -/// initialization process _at the time the reference was taken_. No guarantee is made in regards to -/// what stage of the process the [`Device`] is currently in. This means for instance that a -/// `&Device` may actually be registered with userspace, it just wasn't known to be -/// registered at the time the reference was taken. +/// - [`Normal`]: The general-purpose, reference-counted context. A [`Device`] in this context may +/// or may not be registered with userspace. +/// - [`Registered`]: The device has been registered with userspace at some point. pub trait DeviceContext: Sealed + Send + Sync + 'static {} +/// The general-purpose, reference-counted [`DeviceContext`]. +/// +/// A [`Device`] in this context may or may not be registered with userspace. This context is used +/// for reference-counted device handles and during device setup via [`UnregisteredDevice`]. +pub struct Normal; + +impl Sealed for Normal {} +impl DeviceContext for Normal {} + /// The [`DeviceContext`] of a [`Device`] that was registered with userspace at some point. /// /// This represents a [`Device`] which is guaranteed to have been registered with userspace at @@ -121,20 +107,6 @@ pub trait DeviceContext: Sealed + Send + Sync + 'static {} impl Sealed for Registered {} impl DeviceContext for Registered {} -/// The [`DeviceContext`] of a [`Device`] that may be unregistered and partly uninitialized. -/// -/// A [`Device`] in this context is only guaranteed to be partly initialized, and may or may not -/// be registered with userspace. Thus operations which depend on the [`Device`] being fully -/// initialized, or which depend on the [`Device`] being registered with userspace are not -/// available through this [`DeviceContext`]. -/// -/// A [`Device`] in this context can be used to create a -/// [`Registration`](drm::driver::Registration). -pub struct Uninit; - -impl Sealed for Uninit {} -impl DeviceContext for Uninit {} - /// A [`Device`] which is known at compile-time to be unregistered with userspace. /// /// This type allows performing operations which are only safe to do before userspace registration, @@ -147,10 +119,10 @@ impl DeviceContext for Uninit {} /// /// The device in `self.0` is guaranteed to be a newly created [`Device`] that has not yet been /// registered with userspace until this type is dropped. -pub struct UnregisteredDevice(ARef>, NotThreadSafe); +pub struct UnregisteredDevice(ARef>, NotThreadSafe); impl Deref for UnregisteredDevice { - type Target = Device; + type Target = Device; fn deref(&self) -> &Self::Target { &self.0 @@ -178,15 +150,13 @@ const fn compute_features() -> u32 { master_drop: None, debugfs_init: None, - // Ignore the Uninit DeviceContext below. It is only provided because it is required by the - // compiler, and it is not actually used by these functions. - gem_create_object: T::Object::::ALLOC_OPS.gem_create_object, - prime_handle_to_fd: T::Object::::ALLOC_OPS.prime_handle_to_fd, - prime_fd_to_handle: T::Object::::ALLOC_OPS.prime_fd_to_handle, - gem_prime_import: T::Object::::ALLOC_OPS.gem_prime_import, - gem_prime_import_sg_table: T::Object::::ALLOC_OPS.gem_prime_import_sg_table, - dumb_create: T::Object::::ALLOC_OPS.dumb_create, - dumb_map_offset: T::Object::::ALLOC_OPS.dumb_map_offset, + gem_create_object: T::Object::::ALLOC_OPS.gem_create_object, + prime_handle_to_fd: T::Object::::ALLOC_OPS.prime_handle_to_fd, + prime_fd_to_handle: T::Object::::ALLOC_OPS.prime_fd_to_handle, + gem_prime_import: T::Object::::ALLOC_OPS.gem_prime_import, + gem_prime_import_sg_table: T::Object::::ALLOC_OPS.gem_prime_import_sg_table, + dumb_create: T::Object::::ALLOC_OPS.dumb_create, + dumb_map_offset: T::Object::::ALLOC_OPS.dumb_map_offset, show_fdinfo: None, fbdev_probe: None, @@ -211,7 +181,7 @@ const fn compute_features() -> u32 { pub fn new(dev: &device::Device, data: impl PinInit) -> Result { // `__drm_dev_alloc` uses `kmalloc()` to allocate memory, hence ensure a `kmalloc()` // compatible `Layout`. - let layout = Kmalloc::aligned_layout(Layout::new::>()); + let layout = Kmalloc::aligned_layout(Layout::new::>()); // Use a temporary vtable without a `release` callback until `data` is initialized, so // init failure can release the DRM device without dropping uninitialized fields. @@ -223,12 +193,12 @@ pub fn new(dev: &device::Device, data: impl PinInit) -> Result = unsafe { + let raw_drm: *mut Device = unsafe { bindings::__drm_dev_alloc( dev.as_raw(), &alloc_vtable, layout.size(), - mem::offset_of!(Device, dev), + mem::offset_of!(Device, dev), ) } .cast(); @@ -264,16 +234,8 @@ pub fn new(dev: &device::Device, data: impl PinInit) -> Result Date: Sun, 28 Jun 2026 16:53:23 +0200 Subject: [PATCH 063/137] rust: faux: add Device type with AsBusDevice support Add a faux::Device type that wraps struct faux_device and implements AsBusDevice, enabling faux devices to be used as parent devices for subsystems that require a bus device, such as DRM. Update Registration to return &faux::Device via AsRef. Reviewed-by: Alexandre Courbot Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260628145406.2107056-4-dakr@kernel.org [ Drop redundant 'struct device' invariant; implied by valid struct faux_device. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/drm/gem/shmem.rs | 11 +++-- rust/kernel/faux.rs | 69 +++++++++++++++++++++++++++----- samples/rust/rust_driver_faux.rs | 3 +- 3 files changed, 68 insertions(+), 15 deletions(-) diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 3ee19ef6264e..52de59b14dad 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -692,10 +692,12 @@ impl drm::Driver for KunitDriver { fn create_drm_dev() -> Result<(faux::Registration, UnregisteredDevice)> { // Create a faux DRM device so we can test gem object creation. let data = try_pin_init!(KunitData {}); - let dev = faux::Registration::new(c"Kunit", None)?; - let drm = UnregisteredDevice::new(dev.as_ref(), data)?; + let reg = faux::Registration::new(c"Kunit", None)?; + let fdev = reg.as_ref(); + let dev = fdev.as_ref(); + let drm = UnregisteredDevice::new(dev, data)?; - Ok((dev, drm)) + Ok((reg, drm)) } #[test] @@ -755,7 +757,8 @@ fn vmap_io() -> Result { #[test] fn fail_sg_table_on_wrong_dev() -> Result { let (_dev, drm) = create_drm_dev()?; - let wrong_dev = faux::Registration::new(c"EvilKunit", None)?; + let reg = faux::Registration::new(c"EvilKunit", None)?; + let wrong_dev = reg.as_ref(); let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; diff --git a/rust/kernel/faux.rs b/rust/kernel/faux.rs index 36c92ae2943c..cd4198fbb232 100644 --- a/rust/kernel/faux.rs +++ b/rust/kernel/faux.rs @@ -9,15 +9,63 @@ use crate::{ bindings, device, - prelude::*, // + prelude::*, + types::Opaque, // }; -use core::ptr::{ - addr_of_mut, - null, - null_mut, - NonNull, // +use core::{ + marker::PhantomData, + ptr::{ + null, + null_mut, + NonNull, // + }, }; +/// A faux device. +/// +/// A faux device is a virtual device backed by the faux bus, primarily used for scenarios where a +/// real hardware device is not available or for testing. +/// +/// # Invariants +/// +/// The underlying `struct faux_device` is valid. +#[repr(transparent)] +pub struct Device( + Opaque, + PhantomData, +); + +impl Device { + #[inline] + fn as_raw(&self) -> *mut bindings::faux_device { + self.0.get() + } + + /// # Safety + /// + /// `ptr` must be a valid pointer to a `struct faux_device`. + #[inline] + unsafe fn from_raw<'a>(ptr: *mut bindings::faux_device) -> &'a Self { + // SAFETY: `Device` is a transparent wrapper of `Opaque`. + unsafe { &*ptr.cast() } + } +} + +impl AsRef> for Device { + #[inline] + fn as_ref(&self) -> &device::Device { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct faux_device`. `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(&raw mut (*self.as_raw()).dev) } + } +} + +// SAFETY: `faux::Device` is a transparent wrapper of `struct faux_device`. +// The offset is guaranteed to point to a valid device field inside `faux::Device`. +unsafe impl device::AsBusDevice for Device { + const OFFSET: usize = core::mem::offset_of!(bindings::faux_device, dev); +} + /// The registration of a faux device. /// /// This type represents the registration of a [`struct faux_device`]. When an instance of this type @@ -60,10 +108,11 @@ fn as_raw(&self) -> *mut bindings::faux_device { } } -impl AsRef> for Registration { - fn as_ref(&self) -> &device::Device { +impl AsRef> for Registration { + #[inline] + fn as_ref(&self) -> &Device { // SAFETY: - // - The underlying `device` in `faux_device` is guaranteed by the C API to be a valid + // - The underlying `struct faux_device` is guaranteed by the C API to be a valid // initialized `device`. // - `faux_match()` always returns 1, and probe runs synchronously // (PROBE_FORCE_SYNCHRONOUS). @@ -71,7 +120,7 @@ fn as_ref(&self) -> &device::Device { // sysfs. // - `mem::forget(Registration)` is not a problem; if the `Registration` is leaked, the faux // device stays bound forever. - unsafe { device::Device::from_raw(addr_of_mut!((*self.as_raw()).dev)) } + unsafe { Device::from_raw(self.as_raw()) } } } diff --git a/samples/rust/rust_driver_faux.rs b/samples/rust/rust_driver_faux.rs index 99876c8e3743..27b6d3e2bb44 100644 --- a/samples/rust/rust_driver_faux.rs +++ b/samples/rust/rust_driver_faux.rs @@ -25,8 +25,9 @@ fn init(_module: &'static ThisModule) -> Result { pr_info!("Initialising Rust Faux Device Sample\n"); let reg = faux::Registration::new(c"rust-faux-sample-device", None)?; + let fdev = reg.as_ref(); - dev_info!(reg, "Hello from faux device!\n"); + dev_info!(fdev, "Hello from faux device!\n"); Ok(Self { _reg: reg }) } From 49e27d58a06d7edda4791b9fa818a536dd34b7e5 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 28 Jun 2026 16:53:24 +0200 Subject: [PATCH 064/137] rust: drm: Add Driver::ParentDevice associated type Add a ParentDevice associated type to the Driver trait, allowing each DRM driver to declare its parent bus device type (e.g. auxiliary::Device, platform::Device). Change UnregisteredDevice::new() to take &T::ParentDevice, ensuring at the type level that the DRM device's parent matches the declared bus device type. Reviewed-by: Lyude Paul Reviewed-by: Alexandre Courbot Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260628145406.2107056-5-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/driver.rs | 8 ++++++-- drivers/gpu/drm/tyr/driver.rs | 6 ++++-- rust/kernel/drm/device.rs | 7 +++++-- rust/kernel/drm/driver.rs | 3 +++ rust/kernel/drm/gem/shmem.rs | 4 ++-- 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index 48933d86ddda..c5b0313006bd 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -2,7 +2,10 @@ use kernel::{ auxiliary, - device::Core, + device::{ + Core, + DeviceContext, // + }, drm::{ self, gem, @@ -62,7 +65,7 @@ fn probe<'bound>( ) -> impl PinInit, Error> + 'bound { let data = try_pin_init!(NovaData { adev: adev.into() }); - let drm = drm::UnregisteredDevice::::new(adev.as_ref(), data)?; + let drm = drm::UnregisteredDevice::::new(adev, data)?; let drm = drm::Registration::new_foreign_owned(drm, adev.as_ref(), 0)?; Ok(Nova { drm: drm.into() }) @@ -74,6 +77,7 @@ impl drm::Driver for NovaDriver { type Data = NovaData; type File = File; type Object = gem::Object; + type ParentDevice = auxiliary::Device; const INFO: drm::DriverInfo = INFO; diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index d063bc664cc1..338c25ccc151 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -7,7 +7,8 @@ }, device::{ Core, - Device, // + Device, + DeviceContext, // }, dma::{ Device as DmaDevice, @@ -148,7 +149,7 @@ fn probe<'bound>( gpu_info, }); - let tdev = drm::UnregisteredDevice::::new(pdev.as_ref(), data)?; + let tdev = drm::UnregisteredDevice::::new(pdev, data)?; let tdev = drm::driver::Registration::new_foreign_owned(tdev, pdev.as_ref(), 0)?; let driver = TyrPlatformDriverData { @@ -182,6 +183,7 @@ impl drm::Driver for TyrDrmDriver { type Data = TyrDrmDeviceData; type File = TyrDrmFileData; type Object = drm::gem::shmem::Object; + type ParentDevice = platform::Device; const INFO: drm::DriverInfo = INFO; const FEAT_RENDER: bool = true; diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 35ff9c6942d8..343c0ef63c6c 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -178,7 +178,10 @@ const fn compute_features() -> u32 { /// Create a new `UnregisteredDevice` for a `drm::Driver`. /// /// This can be used to create a [`Registration`](kernel::drm::Registration). - pub fn new(dev: &device::Device, data: impl PinInit) -> Result { + pub fn new( + dev: &T::ParentDevice, + data: impl PinInit, + ) -> Result { // `__drm_dev_alloc` uses `kmalloc()` to allocate memory, hence ensure a `kmalloc()` // compatible `Layout`. let layout = Kmalloc::aligned_layout(Layout::new::>()); @@ -195,7 +198,7 @@ pub fn new(dev: &device::Device, data: impl PinInit) -> Result = unsafe { bindings::__drm_dev_alloc( - dev.as_raw(), + dev.as_ref().as_raw(), &alloc_vtable, layout.size(), mem::offset_of!(Device, dev), diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 25f7e233884d..802e7fc13e30 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -116,6 +116,9 @@ pub trait Driver { /// The type used to represent a DRM File (client) type File: drm::file::DriverFile; + /// The bus device type of the parent device that the DRM device is associated with. + type ParentDevice: device::AsBusDevice; + /// Driver metadata const INFO: DriverInfo; diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 52de59b14dad..cbcfc7e4edb6 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -684,6 +684,7 @@ impl drm::Driver for KunitDriver { type Data = KunitData; type File = KunitFile; type Object = Object; + type ParentDevice = faux::Device; const INFO: drm::DriverInfo = INFO; const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor] = &[]; @@ -694,8 +695,7 @@ fn create_drm_dev() -> Result<(faux::Registration, UnregisteredDevice Date: Sun, 28 Jun 2026 16:53:25 +0200 Subject: [PATCH 065/137] rust: drm: change default DeviceContext to Normal Change the default DeviceContext from Registered to Normal for drm::Device, gem::Object, gem::shmem::Object and gem::shmem::ObjectConfig. Normal is the general-purpose, reference-counted context suitable for most uses; Registered represents a device that was registered with userspace and will become a non-owning context obtained through a RegistrationGuard. Update the create_handle/lookup_handle bounds from Object to Object to match the new default context of GEM objects, and update the driver device type aliases (NovaDevice, TyrDrmDevice) to default to Normal. Reviewed-by: Lyude Paul Reviewed-by: Alexandre Courbot Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260628145406.2107056-6-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/driver.rs | 2 +- drivers/gpu/drm/tyr/driver.rs | 2 +- rust/kernel/drm/device.rs | 2 +- rust/kernel/drm/gem/mod.rs | 7 ++++--- rust/kernel/drm/gem/shmem.rs | 10 +++++----- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index c5b0313006bd..8ddb81fd0c87 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -26,7 +26,7 @@ pub(crate) struct Nova { } /// Convienence type alias for the DRM device type for this driver -pub(crate) type NovaDevice = drm::Device; +pub(crate) type NovaDevice = drm::Device; #[pin_data] pub(crate) struct NovaData { diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index 338c25ccc151..180631daff02 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -47,7 +47,7 @@ pub(crate) struct TyrDrmDriver; /// Convenience type alias for the DRM device type for this driver. -pub(crate) type TyrDrmDevice = drm::Device; +pub(crate) type TyrDrmDevice = drm::Device; pub(crate) struct TyrPlatformDriver; diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 343c0ef63c6c..458519d62a46 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -246,7 +246,7 @@ pub fn new( /// * The data layout of `Self` remains the same across all implementations of `C`. /// * Any invariants for `C` also apply. #[repr(C)] -pub struct Device { +pub struct Device { dev: Opaque, data: T::Data, _ctx: PhantomData, diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index 48fa6e96dfe7..6a688568afbb 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -10,6 +10,7 @@ self, device::{ DeviceContext, + Normal, Registered, // }, driver::{ @@ -183,7 +184,7 @@ fn size(&self) -> usize { fn create_handle(&self, file: &drm::File) -> Result where Self: AllocImpl, - D: drm::Driver = Self, File = F>, + D: drm::Driver = Self, File = F>, F: drm::file::DriverFile, { let mut handle: u32 = 0; @@ -198,7 +199,7 @@ fn create_handle(&self, file: &drm::File) -> Result fn lookup_handle(file: &drm::File, handle: u32) -> Result> where Self: AllocImpl, - D: drm::Driver = Self, File = F>, + D: drm::Driver = Self, File = F>, F: drm::file::DriverFile, { // SAFETY: The arguments are all valid per the type invariants. @@ -254,7 +255,7 @@ impl BaseObjectPrivate for T {} /// * Any type invariants of `Ctx` apply to the parent DRM device for this GEM object. #[repr(C)] #[pin_data] -pub struct Object { +pub struct Object { obj: Opaque, #[pin] data: T, diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index cbcfc7e4edb6..5ffa1355ecf2 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -22,7 +22,7 @@ private::Sealed, Device, DeviceContext, - Registered, // + Normal, // }, error::{ from_err_ptr, @@ -73,7 +73,7 @@ /// /// This is used with [`Object::new()`] to control various properties that can only be set when /// initially creating a shmem-backed GEM object. -pub struct ObjectConfig<'a, T: DriverObject, C: DeviceContext = Registered> { +pub struct ObjectConfig<'a, T: DriverObject, C: DeviceContext = Normal> { /// Whether to set the write-combine map flag. pub map_wc: bool, @@ -102,7 +102,7 @@ fn default() -> Self { /// - Any type invariants of `C` apply to the parent DRM device for this GEM object. #[repr(C)] #[pin_data] -pub struct Object { +pub struct Object { #[pin] obj: Opaque, /// Parent object that owns this object's DMA reservation object. @@ -409,7 +409,7 @@ impl driver::AllocImpl for Object { /// When this is dropped, the `dma_resv` lock is dropped as well. /// // TODO: This should be replace with a WwMutex equivalent once we have such bindings in the kernel. -struct DmaResvGuard<'a, T: DriverObject, C: DeviceContext = Registered>( +struct DmaResvGuard<'a, T: DriverObject, C: DeviceContext = Normal>( &'a Object, NotThreadSafe, ); @@ -438,7 +438,7 @@ fn drop(&mut self) { /// /// - The size of `owner` is >= SIZE. /// - The memory pointed to by `addr` remains valid at least until this object is dropped. -pub struct VMap +pub struct VMap where D: DriverObject, C: DeviceContext, From 506a7d63dab00f0f279869c956b986749292623a Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 28 Jun 2026 16:53:26 +0200 Subject: [PATCH 066/137] rust: drm: restrict AlwaysRefCounted to Normal Device context Restrict the AlwaysRefCounted implementation for drm::Device to the Normal context. Registered devices represent a non-owning view of a device within a RegistrationGuard scope and must not be independently reference-counted. Reviewed-by: Lyude Paul Reviewed-by: Alexandre Courbot Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260628145406.2107056-7-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/drm/device.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 458519d62a46..312850f125af 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -85,6 +85,9 @@ pub trait DeviceContext: Sealed + Send + Sync + 'static {} /// /// A [`Device`] in this context may or may not be registered with userspace. This context is used /// for reference-counted device handles and during device setup via [`UnregisteredDevice`]. +/// +/// [`AlwaysRefCounted`] is only implemented for `Device`, making this the required +/// context for [`ARef`]-based device handles. pub struct Normal; impl Sealed for Normal {} @@ -327,7 +330,7 @@ fn deref(&self) -> &Self::Target { // SAFETY: DRM device objects are always reference counted and the get/put functions // satisfy the requirements. -unsafe impl AlwaysRefCounted for Device { +unsafe impl AlwaysRefCounted for Device { fn inc_ref(&self) { // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. unsafe { bindings::drm_dev_get(self.as_raw()) }; @@ -357,12 +360,10 @@ unsafe impl Send for Device {} // by the synchronization in `struct drm_device`. unsafe impl Sync for Device {} -impl WorkItem for Device +impl WorkItem for Device where - T: drm::Driver, T::Data: WorkItem>, T::Data: HasWork, - C: DeviceContext, { type Pointer = ARef; From ec8b2cc27c766fab80f70b02d196b7c3be7d10ce Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 28 Jun 2026 16:53:27 +0200 Subject: [PATCH 067/137] rust: drm: restrict AlwaysRefCounted to Normal GEM Object context Restrict AlwaysRefCounted for gem::Object and gem::shmem::Object to the Normal context, since only Normal objects should be independently reference-counted. To avoid cascading through IntoGEMObject (which had AlwaysRefCounted as a supertrait), remove AlwaysRefCounted from IntoGEMObject's supertraits and instead add it as an explicit bound on lookup_handle(), which is the only BaseObject method that returns an ARef. Since Object::new() and shmem::Object::new() return ARef, move them to Normal-only impl blocks. Similarly, simplify ObjectConfig and shmem's parent_resv_obj field to the Normal context. Remove the DeviceContext generic from DriverObject::new() and Driver::Object, since GEM objects can only be constructed in the Normal context. Simplify DriverAllocImpl accordingly. Reviewed-by: Lyude Paul Reviewed-by: Alexandre Courbot Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260628145406.2107056-8-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/driver.rs | 2 +- drivers/gpu/drm/nova/gem.rs | 18 ++--- drivers/gpu/drm/tyr/driver.rs | 2 +- drivers/gpu/drm/tyr/gem.rs | 11 +-- rust/kernel/drm/device.rs | 14 ++-- rust/kernel/drm/driver.rs | 2 +- rust/kernel/drm/gem/mod.rs | 97 ++++++++++++------------- rust/kernel/drm/gem/shmem.rs | 129 +++++++++++++++++---------------- 8 files changed, 130 insertions(+), 145 deletions(-) diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index 8ddb81fd0c87..e3c54303d70e 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -76,7 +76,7 @@ fn probe<'bound>( impl drm::Driver for NovaDriver { type Data = NovaData; type File = File; - type Object = gem::Object; + type Object = gem::Object; type ParentDevice = auxiliary::Device; const INFO: drm::DriverInfo = INFO; diff --git a/drivers/gpu/drm/nova/gem.rs b/drivers/gpu/drm/nova/gem.rs index 9d8ff7de2c0f..2b6fe9dc0bfa 100644 --- a/drivers/gpu/drm/nova/gem.rs +++ b/drivers/gpu/drm/nova/gem.rs @@ -2,7 +2,10 @@ use kernel::{ drm, - drm::{gem, gem::BaseObject, DeviceContext}, + drm::{ + gem, + gem::BaseObject, // + }, page, prelude::*, sync::aref::ARef, @@ -21,27 +24,20 @@ impl gem::DriverObject for NovaObject { type Driver = NovaDriver; type Args = (); - fn new( - _dev: &NovaDevice, - _size: usize, - _args: Self::Args, - ) -> impl PinInit { + fn new(_dev: &NovaDevice, _size: usize, _args: Self::Args) -> impl PinInit { try_pin_init!(NovaObject {}) } } impl NovaObject { /// Create a new DRM GEM object. - pub(crate) fn new( - dev: &NovaDevice, - size: usize, - ) -> Result>> { + pub(crate) fn new(dev: &NovaDevice, size: usize) -> Result>> { if size == 0 { return Err(EINVAL); } let aligned_size = page::page_align(size).ok_or(EINVAL)?; - gem::Object::::new(dev, aligned_size, ()) + gem::Object::::new(dev, aligned_size, ()) } /// Look up a GEM object handle for a `File` and return an `ObjectRef` for it. diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index 180631daff02..7f082de6d6dc 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -182,7 +182,7 @@ fn drop(self: Pin<&mut Self>) {} impl drm::Driver for TyrDrmDriver { type Data = TyrDrmDeviceData; type File = TyrDrmFileData; - type Object = drm::gem::shmem::Object; + type Object = drm::gem::shmem::Object; type ParentDevice = platform::Device; const INFO: drm::DriverInfo = INFO; diff --git a/drivers/gpu/drm/tyr/gem.rs b/drivers/gpu/drm/tyr/gem.rs index c6d4d6f9bae3..1640a161754b 100644 --- a/drivers/gpu/drm/tyr/gem.rs +++ b/drivers/gpu/drm/tyr/gem.rs @@ -5,10 +5,7 @@ //! DRM's GEM subsystem with shmem backing. use kernel::{ - drm::{ - gem, - DeviceContext, // - }, + drm::gem, prelude::*, // }; @@ -33,11 +30,7 @@ impl gem::DriverObject for BoData { type Driver = TyrDrmDriver; type Args = BoCreateArgs; - fn new( - _dev: &TyrDrmDevice, - _size: usize, - args: BoCreateArgs, - ) -> impl PinInit { + fn new(_dev: &TyrDrmDevice, _size: usize, args: BoCreateArgs) -> impl PinInit { try_pin_init!(Self { flags: args.flags }) } } diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 312850f125af..eb8146ea1c98 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -153,13 +153,13 @@ const fn compute_features() -> u32 { master_drop: None, debugfs_init: None, - gem_create_object: T::Object::::ALLOC_OPS.gem_create_object, - prime_handle_to_fd: T::Object::::ALLOC_OPS.prime_handle_to_fd, - prime_fd_to_handle: T::Object::::ALLOC_OPS.prime_fd_to_handle, - gem_prime_import: T::Object::::ALLOC_OPS.gem_prime_import, - gem_prime_import_sg_table: T::Object::::ALLOC_OPS.gem_prime_import_sg_table, - dumb_create: T::Object::::ALLOC_OPS.dumb_create, - dumb_map_offset: T::Object::::ALLOC_OPS.dumb_map_offset, + gem_create_object: T::Object::ALLOC_OPS.gem_create_object, + prime_handle_to_fd: T::Object::ALLOC_OPS.prime_handle_to_fd, + prime_fd_to_handle: T::Object::ALLOC_OPS.prime_fd_to_handle, + gem_prime_import: T::Object::ALLOC_OPS.gem_prime_import, + gem_prime_import_sg_table: T::Object::ALLOC_OPS.gem_prime_import_sg_table, + dumb_create: T::Object::ALLOC_OPS.dumb_create, + dumb_map_offset: T::Object::ALLOC_OPS.dumb_map_offset, show_fdinfo: None, fbdev_probe: None, diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 802e7fc13e30..5152a18a8312 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -111,7 +111,7 @@ pub trait Driver { type Data: Sync + Send; /// The type used to manage memory for this driver. - type Object: AllocImpl; + type Object: AllocImpl; /// The type used to represent a DRM File (client) type File: drm::file::DriverFile; diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index 6a688568afbb..b03b5f9ccd7e 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -10,8 +10,7 @@ self, device::{ DeviceContext, - Normal, - Registered, // + Normal, // }, driver::{ AllocImpl, @@ -82,8 +81,7 @@ unsafe fn dec_ref(obj: core::ptr::NonNull) { /// A type alias for retrieving the current [`AllocImpl`] for a given [`DriverObject`]. /// /// [`Driver`]: drm::Driver -pub type DriverAllocImpl = - <::Driver as drm::Driver>::Object; +pub type DriverAllocImpl = <::Driver as drm::Driver>::Object; /// GEM object functions, which must be implemented by drivers. pub trait DriverObject: Sync + Send + Sized + 'static { @@ -94,8 +92,8 @@ pub trait DriverObject: Sync + Send + Sized + 'static { type Args; /// Create a new driver data object for a GEM object of a given size. - fn new( - dev: &drm::Device, + fn new( + dev: &drm::Device, size: usize, args: Self::Args, ) -> impl PinInit; @@ -110,7 +108,7 @@ fn close(_obj: &DriverAllocImpl, _file: &DriverFile) {} } /// Trait that represents a GEM object subtype -pub trait IntoGEMObject: Sized + super::private::Sealed + AlwaysRefCounted { +pub trait IntoGEMObject: Sized + super::private::Sealed { /// Returns a reference to the raw `drm_gem_object` structure, which must be valid as long as /// this owning object is valid. fn as_raw(&self) -> *mut bindings::drm_gem_object; @@ -184,7 +182,7 @@ fn size(&self) -> usize { fn create_handle(&self, file: &drm::File) -> Result where Self: AllocImpl, - D: drm::Driver = Self, File = F>, + D: drm::Driver, F: drm::file::DriverFile, { let mut handle: u32 = 0; @@ -198,8 +196,8 @@ fn create_handle(&self, file: &drm::File) -> Result /// Looks up an object by its handle for a given `File`. fn lookup_handle(file: &drm::File, handle: u32) -> Result> where - Self: AllocImpl, - D: drm::Driver = Self, File = F>, + Self: AllocImpl + AlwaysRefCounted, + D: drm::Driver, F: drm::file::DriverFile, { // SAFETY: The arguments are all valid per the type invariants. @@ -281,12 +279,43 @@ impl Object { rss: None, }; + /// Returns the `Device` that owns this GEM object. + pub fn dev(&self) -> &drm::Device { + // SAFETY: + // - `struct drm_gem_object.dev` is initialized and valid for as long as the GEM + // object lives. + // - The device we used for creating the gem object is passed as &drm::Device to + // Object::::new(), so we know that `T::Driver` is the right generic parameter to use + // here. + // - Any type invariants of `Ctx` are upheld by using the same `Ctx` for the `Device` we + // return. + unsafe { drm::Device::from_raw((*self.as_raw()).dev) } + } + + fn as_raw(&self) -> *mut bindings::drm_gem_object { + self.obj.get() + } + + extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { + let ptr: *mut Opaque = obj.cast(); + + // SAFETY: All of our objects are of type `Object`. + let this = unsafe { crate::container_of!(ptr, Self, obj) }; + + // SAFETY: The C code only ever calls this callback with a valid pointer to a `struct + // drm_gem_object`. + unsafe { bindings::drm_gem_object_release(obj) }; + + // SAFETY: All of our objects are allocated via `KBox`, and we're in the + // free callback which guarantees this object has zero remaining references, + // so we can drop it. + let _ = unsafe { KBox::from_raw(this) }; + } +} + +impl Object { /// Create a new GEM object. - pub fn new( - dev: &drm::Device, - size: usize, - args: T::Args, - ) -> Result> { + pub fn new(dev: &drm::Device, size: usize, args: T::Args) -> Result> { let obj: Pin> = KBox::pin_init( try_pin_init!(Self { obj: Opaque::new(bindings::drm_gem_object::default()), @@ -322,46 +351,12 @@ pub fn new( // SAFETY: We take over the initial reference count from `drm_gem_object_init()`. Ok(unsafe { ARef::from_raw(ptr) }) } - - /// Returns the `Device` that owns this GEM object. - pub fn dev(&self) -> &drm::Device { - // SAFETY: - // - `struct drm_gem_object.dev` is initialized and valid for as long as the GEM - // object lives. - // - The device we used for creating the gem object is passed as &drm::Device to - // Object::::new(), so we know that `T::Driver` is the right generic parameter to use - // here. - // - Any type invariants of `Ctx` are upheld by using the same `Ctx` for the `Device` we - // return. - unsafe { drm::Device::from_raw((*self.as_raw()).dev) } - } - - fn as_raw(&self) -> *mut bindings::drm_gem_object { - self.obj.get() - } - - extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { - let ptr: *mut Opaque = obj.cast(); - - // SAFETY: All of our objects are of type `Object`. - let this = unsafe { crate::container_of!(ptr, Self, obj) }; - - // SAFETY: The C code only ever calls this callback with a valid pointer to a `struct - // drm_gem_object`. - unsafe { bindings::drm_gem_object_release(obj) }; - - // SAFETY: All of our objects are allocated via `KBox`, and we're in the - // free callback which guarantees this object has zero remaining references, - // so we can drop it. - let _ = unsafe { KBox::from_raw(this) }; - } } impl_aref_for_gem_obj! { - impl for Object + impl for Object where - T: DriverObject, - C: DeviceContext + T: DriverObject } impl super::private::Sealed for Object {} diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 5ffa1355ecf2..cf8410e0f228 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -73,17 +73,17 @@ /// /// This is used with [`Object::new()`] to control various properties that can only be set when /// initially creating a shmem-backed GEM object. -pub struct ObjectConfig<'a, T: DriverObject, C: DeviceContext = Normal> { +pub struct ObjectConfig<'a, T: DriverObject> { /// Whether to set the write-combine map flag. pub map_wc: bool, /// Reuse the DMA reservation from another GEM object. /// /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. - pub parent_resv_obj: Option<&'a Object>, + pub parent_resv_obj: Option<&'a Object>, } -impl<'a, T: DriverObject, C: DeviceContext> Default for ObjectConfig<'a, T, C> { +impl<'a, T: DriverObject> Default for ObjectConfig<'a, T> { #[inline(always)] fn default() -> Self { Self { @@ -106,7 +106,7 @@ pub struct Object { #[pin] obj: Opaque, /// Parent object that owns this object's DMA reservation object. - parent_resv_obj: Option>>, + parent_resv_obj: Option>>, /// Devres object for unmapping any SGTable on driver-unbind. sgt_res: ManuallyDrop>>>, #[pin] @@ -118,10 +118,9 @@ pub struct Object { } super::impl_aref_for_gem_obj! { - impl for Object + impl for Object where - T: DriverObject, - C: DeviceContext + T: DriverObject } // SAFETY: All GEM objects are thread-safe. @@ -157,54 +156,6 @@ fn as_raw_shmem(&self) -> *mut bindings::drm_gem_shmem_object { self.obj.get() } - /// Create a new shmem-backed DRM object of the given size. - /// - /// Additional config options can be specified using `config`. - pub fn new( - dev: &Device, - size: usize, - config: ObjectConfig<'_, T, C>, - args: T::Args, - ) -> Result> { - let new: Pin> = KBox::try_pin_init( - try_pin_init!(Self { - obj <- Opaque::init_zeroed(), - parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), - sgt_res: ManuallyDrop::new(SetOnce::new()), - sgt_lock <- new_mutex!(()), - inner <- T::new(dev, size, args), - _ctx: PhantomData::, - }), - GFP_KERNEL, - )?; - - // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. - unsafe { (*new.as_raw()).funcs = &Self::VTABLE }; - - // SAFETY: The arguments are all valid via the type invariants. - to_result(unsafe { bindings::drm_gem_shmem_init(dev.as_raw(), new.as_raw_shmem(), size) })?; - - // SAFETY: We never move out of `self`. - let new = KBox::into_raw(unsafe { Pin::into_inner_unchecked(new) }); - - // SAFETY: We're taking over the owned refcount from `drm_gem_shmem_init`. - let obj = unsafe { ARef::from_raw(NonNull::new_unchecked(new)) }; - - // Start filling out values from `config` - if let Some(parent_resv) = config.parent_resv_obj { - // SAFETY: We have yet to expose the new gem object outside of this function, so it is - // safe to modify this field. - unsafe { (*obj.obj.get()).base.resv = parent_resv.raw_dma_resv() }; - } - - // SAFETY: We have yet to expose this object outside of this function, so we're guaranteed - // to have exclusive access - thus making this safe to hold a mutable reference to. - let shmem = unsafe { &mut *obj.as_raw_shmem() }; - shmem.set_map_wc(config.map_wc); - - Ok(obj) - } - /// Returns the `Device` that owns this GEM object. pub fn dev(&self) -> &Device { // SAFETY: `dev` will have been initialized in `Self::new()` by `drm_gem_shmem_init()`. @@ -308,12 +259,6 @@ pub fn vmap(&self) -> Result> { self.make_vmap() } - /// Creates and returns an owned reference to a virtual kernel memory mapping for this object. - #[inline] - pub fn owned_vmap(&self) -> Result> { - self.make_vmap() - } - /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA /// pages for this object. /// @@ -355,6 +300,62 @@ pub fn sg_table<'a>( } } +impl Object { + /// Create a new shmem-backed DRM object of the given size. + /// + /// Additional config options can be specified using `config`. + pub fn new( + dev: &Device, + size: usize, + config: ObjectConfig<'_, T>, + args: T::Args, + ) -> Result> { + let new: Pin> = KBox::try_pin_init( + try_pin_init!(Self { + obj <- Opaque::init_zeroed(), + parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), + sgt_res: ManuallyDrop::new(SetOnce::new()), + sgt_lock <- new_mutex!(()), + inner <- T::new(dev, size, args), + _ctx: PhantomData, + }), + GFP_KERNEL, + )?; + + // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. + unsafe { (*new.as_raw()).funcs = &Self::VTABLE }; + + // SAFETY: The arguments are all valid via the type invariants. + to_result(unsafe { bindings::drm_gem_shmem_init(dev.as_raw(), new.as_raw_shmem(), size) })?; + + // SAFETY: We never move out of `self`. + let new = KBox::into_raw(unsafe { Pin::into_inner_unchecked(new) }); + + // SAFETY: We're taking over the owned refcount from `drm_gem_shmem_init`. + let obj = unsafe { ARef::from_raw(NonNull::new_unchecked(new)) }; + + // Start filling out values from `config` + if let Some(parent_resv) = config.parent_resv_obj { + // SAFETY: We have yet to expose the new gem object outside of this function, so it is + // safe to modify this field. + unsafe { (*obj.obj.get()).base.resv = parent_resv.raw_dma_resv() }; + } + + // SAFETY: We have yet to expose this object outside of this function, so we're guaranteed + // to have exclusive access - thus making this safe to hold a mutable reference to. + let shmem = unsafe { &mut *obj.as_raw_shmem() }; + shmem.set_map_wc(config.map_wc); + + Ok(obj) + } + + /// Creates and returns an owned reference to a virtual kernel memory mapping for this object. + #[inline] + pub fn owned_vmap(&self) -> Result> { + self.make_vmap() + } +} + impl Deref for Object { type Target = T; @@ -670,8 +671,8 @@ impl gem::DriverObject for KunitObject { type Driver = KunitDriver; type Args = (); - fn new( - _dev: &drm::Device, + fn new( + _dev: &drm::Device, _size: usize, _args: Self::Args, ) -> impl PinInit { @@ -683,7 +684,7 @@ fn new( impl drm::Driver for KunitDriver { type Data = KunitData; type File = KunitFile; - type Object = Object; + type Object = Object; type ParentDevice = faux::Device; const INFO: drm::DriverInfo = INFO; From 7f994b8912eba190ff58e9c8a378d02d13c9eb8a Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 28 Jun 2026 16:53:28 +0200 Subject: [PATCH 068/137] rust: drm/gem: remove DeviceContext from shmem::Object Now that AlwaysRefCounted is restricted to the Normal GEM Object context, there is no use for instantiating Object with a non-Normal context. Remove the DeviceContext generic parameter from shmem::Object and all associated types (VMap, VMapRef, VMapOwned, DmaResvGuard, SGTableMap), simplifying the API. Reviewed-by: Alexandre Courbot Reviewed-by: Lyude Paul Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260628145406.2107056-9-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/drm/gem/shmem.rs | 121 +++++++++++++++-------------------- 1 file changed, 51 insertions(+), 70 deletions(-) diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index cf8410e0f228..e0ef47352e88 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -20,9 +20,7 @@ driver, gem, private::Sealed, - Device, - DeviceContext, - Normal, // + Device, // }, error::{ from_err_ptr, @@ -48,7 +46,6 @@ }; use core::{ ffi::c_void, - marker::PhantomData, mem::{ ManuallyDrop, MaybeUninit, // @@ -99,22 +96,20 @@ fn default() -> Self { /// /// - `obj` contains a valid initialized `struct drm_gem_shmem_object` for the lifetime of this /// object. -/// - Any type invariants of `C` apply to the parent DRM device for this GEM object. #[repr(C)] #[pin_data] -pub struct Object { +pub struct Object { #[pin] obj: Opaque, /// Parent object that owns this object's DMA reservation object. parent_resv_obj: Option>>, /// Devres object for unmapping any SGTable on driver-unbind. - sgt_res: ManuallyDrop>>>, + sgt_res: ManuallyDrop>>>, #[pin] /// Lock for protecting initialization of `sgt_res`. sgt_lock: Mutex<()>, #[pin] inner: T, - _ctx: PhantomData, } super::impl_aref_for_gem_obj! { @@ -124,12 +119,12 @@ impl for Object } // SAFETY: All GEM objects are thread-safe. -unsafe impl Send for Object {} +unsafe impl Send for Object {} // SAFETY: All GEM objects are thread-safe. -unsafe impl Sync for Object {} +unsafe impl Sync for Object {} -impl Object { +impl Object { /// `drm_gem_object_funcs` vtable suitable for GEM shmem objects. const VTABLE: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { free: Some(Self::free_callback), @@ -157,7 +152,7 @@ fn as_raw_shmem(&self) -> *mut bindings::drm_gem_shmem_object { } /// Returns the `Device` that owns this GEM object. - pub fn dev(&self) -> &Device { + pub fn dev(&self) -> &Device { // SAFETY: `dev` will have been initialized in `Self::new()` by `drm_gem_shmem_init()`. unsafe { Device::from_raw((*self.as_raw()).dev) } } @@ -171,8 +166,8 @@ extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { // SAFETY: // - We verified above that `obj` is valid, which makes `this` valid - // - This function is set in AllocOps, so we know that `this` is contained within a - // `Object` + // - This function is set in AllocOps, so we know that `this` is contained within an + // `Object` let this = unsafe { container_of!(Opaque::cast_from(base), Self, obj) }.cast_mut(); // We need to drop `sgt_res` first, since doing so requires that the GEM object is still @@ -193,7 +188,7 @@ extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { } /// Attempt to create a vmap from the gem object, and confirm the size of said vmap. - fn make_vmap<'a, R, const SIZE: usize>(&'a self) -> Result> + fn make_vmap<'a, R, const SIZE: usize>(&'a self) -> Result> where R: Deref + From<&'a Self>, { @@ -255,7 +250,7 @@ unsafe fn raw_vunmap(&self, mut map: bindings::iosys_map) { /// Creates and returns a virtual kernel memory mapping for this object. #[inline] - pub fn vmap(&self) -> Result> { + pub fn vmap(&self) -> Result> { self.make_vmap() } @@ -298,9 +293,7 @@ pub fn sg_table<'a>( Ok(sgt_res.access(dev)?) } -} -impl Object { /// Create a new shmem-backed DRM object of the given size. /// /// Additional config options can be specified using `config`. @@ -317,7 +310,6 @@ pub fn new( sgt_res: ManuallyDrop::new(SetOnce::new()), sgt_lock <- new_mutex!(()), inner <- T::new(dev, size, args), - _ctx: PhantomData, }), GFP_KERNEL, )?; @@ -351,12 +343,12 @@ pub fn new( /// Creates and returns an owned reference to a virtual kernel memory mapping for this object. #[inline] - pub fn owned_vmap(&self) -> Result> { + pub fn owned_vmap(&self) -> Result> { self.make_vmap() } } -impl Deref for Object { +impl Deref for Object { type Target = T; fn deref(&self) -> &Self::Target { @@ -364,15 +356,15 @@ fn deref(&self) -> &Self::Target { } } -impl DerefMut for Object { +impl DerefMut for Object { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.inner } } -impl Sealed for Object {} +impl Sealed for Object {} -impl gem::IntoGEMObject for Object { +impl gem::IntoGEMObject for Object { fn as_raw(&self) -> *mut bindings::drm_gem_object { // SAFETY: // - Our immutable reference is proof that this is safe to dereference. @@ -391,7 +383,7 @@ unsafe fn from_raw<'a>(obj: *mut bindings::drm_gem_object) -> &'a Self { } } -impl driver::AllocImpl for Object { +impl driver::AllocImpl for Object { type Driver = T::Driver; const ALLOC_OPS: driver::AllocOps = driver::AllocOps { @@ -410,14 +402,11 @@ impl driver::AllocImpl for Object { /// When this is dropped, the `dma_resv` lock is dropped as well. /// // TODO: This should be replace with a WwMutex equivalent once we have such bindings in the kernel. -struct DmaResvGuard<'a, T: DriverObject, C: DeviceContext = Normal>( - &'a Object, - NotThreadSafe, -); +struct DmaResvGuard<'a, T: DriverObject>(&'a Object, NotThreadSafe); -impl<'a, T: DriverObject, C: DeviceContext> DmaResvGuard<'a, T, C> { +impl<'a, T: DriverObject> DmaResvGuard<'a, T> { #[inline] - fn new(obj: &'a Object) -> Self { + fn new(obj: &'a Object) -> Self { // SAFETY: This lock is initialized throughout the lifetime of `object`. unsafe { bindings::dma_resv_lock(obj.raw_dma_resv(), ptr::null_mut()) }; @@ -425,7 +414,7 @@ fn new(obj: &'a Object) -> Self { } } -impl<'a, T: DriverObject, C: DeviceContext> Drop for DmaResvGuard<'a, T, C> { +impl<'a, T: DriverObject> Drop for DmaResvGuard<'a, T> { #[inline] fn drop(&mut self) { // SAFETY: We are releasing the lock grabbed during the creation of this object. @@ -439,40 +428,37 @@ fn drop(&mut self) { /// /// - The size of `owner` is >= SIZE. /// - The memory pointed to by `addr` remains valid at least until this object is dropped. -pub struct VMap +pub struct VMap where D: DriverObject, - C: DeviceContext, - R: Deref>, + R: Deref>, { addr: *mut c_void, owner: R, } /// An alias type for a reference to a shmem-based GEM object's VMap. -pub type VMapRef<'a, D, C, const SIZE: usize = 0> = VMap, C, SIZE>; +pub type VMapRef<'a, D, const SIZE: usize = 0> = VMap, SIZE>; /// An alias type for an owned reference to a shmem-based GEM object's VMap. -pub type VMapOwned = VMap>, C, SIZE>; +pub type VMapOwned = VMap>, SIZE>; -impl VMap +impl VMap where D: DriverObject, - C: DeviceContext, - R: Deref>, + R: Deref>, { /// Borrows a reference to the object that owns this virtual mapping. #[inline] - pub fn owner(&self) -> &Object { + pub fn owner(&self) -> &Object { &self.owner } } -impl Drop for VMap +impl Drop for VMap where D: DriverObject, - C: DeviceContext, - R: Deref>, + R: Deref>, { #[inline] fn drop(&mut self) { @@ -491,29 +477,26 @@ fn drop(&mut self) { // SAFETY: `addr` points to a valid memory address for as long as `owner` exists, meaning that so // long as `owner` is `Send` so is `VMap`. -unsafe impl Send for VMap +unsafe impl Send for VMap where D: DriverObject, - C: DeviceContext, - R: Deref> + Send, + R: Deref> + Send, { } // SAFETY: `addr` points to a valid memory address for as long as `owner` exists, meaning that so // long as `owner` is `Sync` so is `VMap`. -unsafe impl Sync for VMap +unsafe impl Sync for VMap where D: DriverObject, - C: DeviceContext, - R: Deref> + Sync, + R: Deref> + Sync, { } -impl Io for VMap +impl Io for VMap where D: DriverObject, - C: DeviceContext, - R: Deref>, + R: Deref>, { #[inline] fn addr(&self) -> usize { @@ -526,22 +509,20 @@ fn maxsize(&self) -> usize { } } -impl IoKnownSize for VMap +impl IoKnownSize for VMap where D: DriverObject, - C: DeviceContext, - R: Deref>, + R: Deref>, { const MIN_SIZE: usize = SIZE; } macro_rules! impl_vmap_io_capable { ($ty:ty) => { - impl IoCapable<$ty> for VMap + impl IoCapable<$ty> for VMap where D: DriverObject, - C: DeviceContext, - R: Deref>, + R: Deref>, { #[inline] unsafe fn io_read(&self, address: usize) -> $ty { @@ -584,11 +565,11 @@ unsafe fn io_write(&self, value: $ty, address: usize) { /// [`SGTable`]. /// /// [`SGTable`]: scatterlist::SGTable -pub struct SGTableMap { - obj: NonNull>, +pub struct SGTableMap { + obj: NonNull>, } -impl Deref for SGTableMap { +impl Deref for SGTableMap { type Target = scatterlist::SGTable; fn deref(&self) -> &Self::Target { @@ -599,7 +580,7 @@ fn deref(&self) -> &Self::Target { } } -impl Drop for SGTableMap { +impl Drop for SGTableMap { fn drop(&mut self) { // SAFETY: `obj` is always valid via our type invariants let obj = unsafe { self.obj.as_ref() }; @@ -610,8 +591,8 @@ fn drop(&mut self) { } } -impl SGTableMap { - fn new(obj: &Object) -> impl Init { +impl SGTableMap { + fn new(obj: &Object) -> impl Init { // INVARIANT: // - We call drm_gem_shmem_get_pages_sgt below and check whether or not it succeeds, // fulfilling the invariant of SGTableMap that the object's `sgt` field is initialized. @@ -625,10 +606,10 @@ fn new(obj: &Object) -> impl Init { // SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object // it points to is guaranteed to be thread-safe. -unsafe impl Send for SGTableMap {} +unsafe impl Send for SGTableMap {} // SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object // it points to is guaranteed to be thread-safe. -unsafe impl Sync for SGTableMap {} +unsafe impl Sync for SGTableMap {} #[kunit_tests(rust_drm_gem_shmem)] mod tests { @@ -705,7 +686,7 @@ fn create_drm_dev() -> Result<(faux::Registration, UnregisteredDevice Result { let (_dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; // Try creating a normal vmap obj.vmap::()?; @@ -729,7 +710,7 @@ fn compile_time_vmap_sizes() -> Result { fn vmap_io() -> Result { let (_dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; let vmap = obj.vmap::()?; @@ -761,7 +742,7 @@ fn fail_sg_table_on_wrong_dev() -> Result { let reg = faux::Registration::new(c"EvilKunit", None)?; let wrong_dev = reg.as_ref(); - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); From 1c8a1f88ac32a987643bf2d534ef4aa2d3da3aeb Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 28 Jun 2026 16:53:29 +0200 Subject: [PATCH 069/137] rust: drm: split Deref for Device context typestates Split the Deref implementation for drm::Device by context: - Device (Normal) dereferences to T::Data. - Device dereferences to Device (Normal). Reviewed-by: Lyude Paul Reviewed-by: Alexandre Courbot Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260628145406.2107056-10-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/drm/device.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index eb8146ea1c98..f5342de190f4 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -79,6 +79,9 @@ macro_rules! drm_legacy_fields { /// - [`Normal`]: The general-purpose, reference-counted context. A [`Device`] in this context may /// or may not be registered with userspace. /// - [`Registered`]: The device has been registered with userspace at some point. +/// +/// `Device` dereferences to `Device` ([`Normal`]), so any method available on a +/// [`Normal`] device is also available on a [`Registered`] one. pub trait DeviceContext: Sealed + Send + Sync + 'static {} /// The general-purpose, reference-counted [`DeviceContext`]. @@ -320,7 +323,7 @@ pub(crate) unsafe fn assume_ctx(&self) -> &Device Deref for Device { +impl Deref for Device { type Target = T::Data; fn deref(&self) -> &Self::Target { @@ -328,6 +331,17 @@ fn deref(&self) -> &Self::Target { } } +impl Deref for Device { + type Target = Device; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: The caller holds a `Device`, which guarantees all invariants + // of the weaker `Normal` context. + unsafe { self.assume_ctx() } + } +} + // SAFETY: DRM device objects are always reference counted and the get/put functions // satisfy the requirements. unsafe impl AlwaysRefCounted for Device { From 499eb35cd4776b7c1ac8c6bb2ea6e693b7519a11 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 28 Jun 2026 16:53:30 +0200 Subject: [PATCH 070/137] rust: drm: pin ioctl Device reference to Normal context Explicitly annotate the Device reference produced by from_raw() in the ioctl dispatch macro as Device<_, Normal>. Without this annotation, the context is inferred from the handler's first parameter type, which would allow a handler declaring &Device to obtain a Registered reference without runtime proof via RegistrationGuard. Reviewed-by: Lyude Paul Reviewed-by: Alexandre Courbot Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260628145406.2107056-11-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/drm/ioctl.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/kernel/drm/ioctl.rs b/rust/kernel/drm/ioctl.rs index ccf4150d83b6..6f5a9877bdae 100644 --- a/rust/kernel/drm/ioctl.rs +++ b/rust/kernel/drm/ioctl.rs @@ -134,7 +134,8 @@ macro_rules! declare_drm_ioctls { // FIXME: Currently there is nothing enforcing that the types of the // dev/file match the current driver these ioctls are being declared // for, and it's not clear how to enforce this within the type system. - let dev = $crate::drm::device::Device::from_raw(raw_dev); + let dev: &$crate::drm::device::Device<_, $crate::drm::Normal> = + $crate::drm::device::Device::from_raw(raw_dev); // Enforce that the handler accepts higher-ranked // lifetimes, preventing it from requiring 'static From 86b20b11505dcdfa5dfe108ac57220b8e3ab9d6d Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 28 Jun 2026 16:53:31 +0200 Subject: [PATCH 071/137] rust: drm: add Ioctl device context typestate Add the Ioctl DeviceContext for DRM devices that have been registered with userspace previously. A Device has been registered at some point, but may be concurrently unregistering or already unregistered. drm_dev_enter() can guard against this, ensuring the device remains registered for the duration of the critical section. This typestate will be used in ioctl dispatch context where registration is guaranteed by the DRM core, and RegistrationGuard can safely be acquired. Reviewed-by: Lyude Paul Reviewed-by: Alexandre Courbot Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260628145406.2107056-12-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/drm/device.rs | 34 +++++++++++++++++++++++++++++++--- rust/kernel/drm/mod.rs | 1 + 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index f5342de190f4..42a068421c27 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -74,14 +74,16 @@ macro_rules! drm_legacy_fields { /// A trait implemented by all possible contexts a [`Device`] can be used in. /// -/// A [`Device`] can be in one of two contexts: +/// A [`Device`] can be in one of the following contexts: /// /// - [`Normal`]: The general-purpose, reference-counted context. A [`Device`] in this context may /// or may not be registered with userspace. +/// - [`Ioctl`]: The device has been registered with userspace at some point; used in ioctl +/// dispatch context. /// - [`Registered`]: The device has been registered with userspace at some point. /// -/// `Device` dereferences to `Device` ([`Normal`]), so any method available on a -/// [`Normal`] device is also available on a [`Registered`] one. +/// Both `Device` and `Device` dereference to `Device` ([`Normal`]), +/// so any method available on a [`Normal`] device is also available in the other contexts. pub trait DeviceContext: Sealed + Send + Sync + 'static {} /// The general-purpose, reference-counted [`DeviceContext`]. @@ -113,6 +115,21 @@ impl DeviceContext for Normal {} impl Sealed for Registered {} impl DeviceContext for Registered {} +/// The [`DeviceContext`] of a [`Device`] that has been registered with userspace previously. +/// +/// A [`Device`] in this context has been registered at some point, but may be concurrently +/// unregistering or already unregistered. `drm_dev_enter()` can guard against this, ensuring the +/// device remains registered for the duration of the critical section. +/// +/// # Invariants +/// +/// A [`Device`] in this context has been registered with userspace via `drm_dev_register()` at +/// some point. +pub struct Ioctl; + +impl Sealed for Ioctl {} +impl DeviceContext for Ioctl {} + /// A [`Device`] which is known at compile-time to be unregistered with userspace. /// /// This type allows performing operations which are only safe to do before userspace registration, @@ -342,6 +359,17 @@ fn deref(&self) -> &Self::Target { } } +impl Deref for Device { + type Target = Device; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: The caller holds a `Device`, which guarantees all invariants + // of the weaker `Normal` context. + unsafe { self.assume_ctx() } + } +} + // SAFETY: DRM device objects are always reference counted and the get/put functions // satisfy the requirements. unsafe impl AlwaysRefCounted for Device { diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs index e5bfaf130342..a6693d2b84b8 100644 --- a/rust/kernel/drm/mod.rs +++ b/rust/kernel/drm/mod.rs @@ -11,6 +11,7 @@ pub use self::device::Device; pub use self::device::DeviceContext; +pub use self::device::Ioctl; pub use self::device::Normal; pub use self::device::Registered; pub use self::device::UnregisteredDevice; From 2455d5f2d5e879b42fe6ab3207b7e3f4b9c78383 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 28 Jun 2026 16:53:32 +0200 Subject: [PATCH 072/137] rust: drm: Add RegistrationGuard for drm_dev_enter/exit critical sections DRM ioctls do not guarantee that the parent bus device is still bound. However, since DRM device registration is managed through Devres, using drm_dev_unplug() on unregistration ensures that between drm_dev_enter() and drm_dev_exit() the parent device must be bound. Add RegistrationGuard, a guard object representing a drm_dev_enter/exit SRCU critical section that dereferences to &Device. The guard is obtained from Device and proves at runtime that the device is still registered. Switch Registration::drop from drm_dev_unregister() to drm_dev_unplug() to provide the SRCU barrier that RegistrationGuard's safety argument relies on. Reviewed-by: Lyude Paul Reviewed-by: Alexandre Courbot Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260628145406.2107056-13-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/drm/device.rs | 85 ++++++++++++++++++++++++++++++++++----- rust/kernel/drm/driver.rs | 10 ++++- rust/kernel/drm/mod.rs | 1 + 3 files changed, 83 insertions(+), 13 deletions(-) diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 42a068421c27..97e2b3de78bc 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -80,7 +80,8 @@ macro_rules! drm_legacy_fields { /// or may not be registered with userspace. /// - [`Ioctl`]: The device has been registered with userspace at some point; used in ioctl /// dispatch context. -/// - [`Registered`]: The device has been registered with userspace at some point. +/// - [`Registered`]: The device is currently registered with userspace and the parent bus device +/// is bound. /// /// Both `Device` and `Device` dereference to `Device` ([`Normal`]), /// so any method available on a [`Normal`] device is also available in the other contexts. @@ -98,18 +99,15 @@ pub trait DeviceContext: Sealed + Send + Sync + 'static {} impl Sealed for Normal {} impl DeviceContext for Normal {} -/// The [`DeviceContext`] of a [`Device`] that was registered with userspace at some point. +/// The [`DeviceContext`] of a [`Device`] that is currently registered with userspace. /// -/// This represents a [`Device`] which is guaranteed to have been registered with userspace at -/// some point in time. Such a DRM device is guaranteed to have been fully-initialized. -/// -/// Note: A device in this context is not guaranteed to remain registered with userspace for its -/// entire lifetime, as this is impossible to guarantee at compile-time. +/// A [`Device`] in this context is guaranteed to be registered and its parent bus device is +/// guaranteed to be bound. This is enforced at runtime by [`RegistrationGuard`], which holds a +/// `drm_dev_enter()` / `drm_dev_exit()` SRCU critical section. /// /// # Invariants /// -/// A [`Device`] in this [`DeviceContext`] is guaranteed to have been registered with userspace -/// at some point in time. +/// The parent bus device is bound for the duration of any reference to a `Device`. pub struct Registered; impl Sealed for Registered {} @@ -260,8 +258,8 @@ pub fn new( /// A typed DRM device with a specific [`drm::Driver`] implementation and [`DeviceContext`]. /// -/// A device in the [`Registered`] context is guaranteed to have been registered with userspace -/// at some point. The [`Normal`] context is the general-purpose, reference-counted context. +/// A device in the [`Registered`] context is currently registered with userspace and its parent +/// bus device is bound. The [`Normal`] context is the general-purpose, reference-counted context. /// /// # Invariants /// @@ -340,6 +338,71 @@ pub(crate) unsafe fn assume_ctx(&self) -> &Device Device { + /// Guard against the parent bus device being unbound. + /// + /// Returns a [`RegistrationGuard`] if the device has not been unplugged, [`None`] otherwise. + /// + /// While [`RegistrationGuard`] is held the parent device is guaranteed to be bound. + #[must_use] + pub fn registration_guard(&self) -> Option> { + let mut idx: i32 = 0; + // SAFETY: `self.as_raw()` is a valid pointer to a `struct drm_device`. + if unsafe { bindings::drm_dev_enter(self.as_raw(), &mut idx) } { + // INVARIANT: + // - `idx` is the SRCU index from the successful `drm_dev_enter()` above. + // - The parent bus device is bound: `drm_dev_enter()` succeeded, meaning + // `drm_dev_unplug()` has not completed; since it is only called from + // `Registration::drop()` during parent unbind, the parent is still bound. + Some(RegistrationGuard { + // SAFETY: See INVARIANT above; the `Registered` context invariant holds. + dev: unsafe { self.assume_ctx() }, + idx, + _not_send: NotThreadSafe, + }) + } else { + None + } + } +} + +/// A guard proving the DRM device is registered and the parent bus device is bound. +/// +/// The guard dereferences to [`Device`], providing access to the DRM device with +/// the guarantee that the parent bus device is bound for the entire duration of the critical +/// section. +/// +/// Internally this is backed by a `drm_dev_enter()` / `drm_dev_exit()` SRCU critical section. +/// +/// # Invariants +/// +/// - `idx` is the SRCU read lock index returned by a successful `drm_dev_enter()` call. +/// - The parent bus device of `dev` is bound for the lifetime of this guard. +#[must_use] +pub struct RegistrationGuard<'a, T: drm::Driver> { + dev: &'a Device, + idx: i32, + _not_send: NotThreadSafe, +} + +impl Deref for RegistrationGuard<'_, T> { + type Target = Device; + + #[inline] + fn deref(&self) -> &Self::Target { + self.dev + } +} + +impl Drop for RegistrationGuard<'_, T> { + #[inline] + fn drop(&mut self) { + // SAFETY: `self.idx` was returned by a successful `drm_dev_enter()` call, as guaranteed + // by the type invariants of `RegistrationGuard`. + unsafe { bindings::drm_dev_exit(self.idx) }; + } +} + impl Deref for Device { type Target = T::Data; diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 5152a18a8312..3cda8dceb498 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -199,8 +199,14 @@ unsafe impl Send for Registration {} impl Drop for Registration { fn drop(&mut self) { + // Use `drm_dev_unplug` rather than `drm_dev_unregister` to ensure that existing + // `drm_dev_enter()` critical sections complete before unregistration proceeds. This + // is required for the safety of `RegistrationGuard`, which relies on the SRCU barrier in + // `drm_dev_unplug()` to guarantee that the parent device is still bound within the + // critical section. + // // SAFETY: Safe by the invariant of `ARef>`. The existence of this - // `Registration` also guarantees the this `drm::Device` is actually registered. - unsafe { bindings::drm_dev_unregister(self.0.as_raw()) }; + // `Registration` also guarantees that this `drm::Device` is actually registered. + unsafe { bindings::drm_dev_unplug(self.0.as_raw()) }; } } diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs index a6693d2b84b8..fd6ed35bc35a 100644 --- a/rust/kernel/drm/mod.rs +++ b/rust/kernel/drm/mod.rs @@ -14,6 +14,7 @@ pub use self::device::Ioctl; pub use self::device::Normal; pub use self::device::Registered; +pub use self::device::RegistrationGuard; pub use self::device::UnregisteredDevice; pub use self::driver::Driver; pub use self::driver::DriverInfo; From 478da53e5b682f0c39a4c6c3eeb09c0131342e8a Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 28 Jun 2026 16:53:33 +0200 Subject: [PATCH 073/137] rust: drm: Wrap ioctl dispatch in RegistrationGuard Make Ioctl handlers receive a &Device reference, proving at the type level that the device is registered and its parent bus device is bound. This is achieved by calling registration_guard() on the Device obtained in ioctl dispatch context. If the device has been unplugged, the ioctl returns -ENODEV without calling the handler. To resolve the driver type parameter T for type inference, which the compiler cannot propagate through method resolution and associated-type projections alone, a dead-code closure and a helper function are used as a type-inference anchor. Reviewed-by: Lyude Paul Reviewed-by: Alexandre Courbot Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260628145406.2107056-14-dakr@kernel.org [ Use imperative mood in commit message; clarify __dev_ctx_cast() doc comment to reflect Ioctl-to-Registered cast. - Danilo ] Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/file.rs | 12 ++++++---- drivers/gpu/drm/tyr/file.rs | 7 ++++-- rust/kernel/drm/ioctl.rs | 45 +++++++++++++++++++++++++++++++++--- 3 files changed, 55 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/nova/file.rs b/drivers/gpu/drm/nova/file.rs index a3b7bd36792c..19fb89b28984 100644 --- a/drivers/gpu/drm/nova/file.rs +++ b/drivers/gpu/drm/nova/file.rs @@ -4,7 +4,11 @@ use crate::gem::NovaObject; use kernel::{ alloc::flags::*, - drm::{self, gem::BaseObject}, + drm::{ + self, + gem::BaseObject, + Registered, // + }, pci, prelude::*, uapi, @@ -23,7 +27,7 @@ fn open(_dev: &NovaDevice) -> Result>> { impl File { /// IOCTL: get_param: Query GPU / driver metadata. pub(crate) fn get_param( - dev: &NovaDevice, + dev: &NovaDevice, getparam: &mut uapi::drm_nova_getparam, _file: &drm::File, ) -> Result { @@ -43,7 +47,7 @@ pub(crate) fn get_param( /// IOCTL: gem_create: Create a new DRM GEM object. pub(crate) fn gem_create( - dev: &NovaDevice, + dev: &NovaDevice, req: &mut uapi::drm_nova_gem_create, file: &drm::File, ) -> Result { @@ -56,7 +60,7 @@ pub(crate) fn gem_create( /// IOCTL: gem_info: Query GEM metadata. pub(crate) fn gem_info( - _dev: &NovaDevice, + _dev: &NovaDevice, req: &mut uapi::drm_nova_gem_info, file: &drm::File, ) -> Result { diff --git a/drivers/gpu/drm/tyr/file.rs b/drivers/gpu/drm/tyr/file.rs index 31411da203c5..fb9233eae01c 100644 --- a/drivers/gpu/drm/tyr/file.rs +++ b/drivers/gpu/drm/tyr/file.rs @@ -1,7 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 or MIT use kernel::{ - drm, + drm::{ + self, + Registered, // + }, prelude::*, uaccess::UserSlice, uapi, // @@ -28,7 +31,7 @@ fn open(_dev: &drm::Device) -> Result>> { impl TyrDrmFileData { pub(crate) fn dev_query( - ddev: &TyrDrmDevice, + ddev: &TyrDrmDevice, devquery: &mut uapi::drm_panthor_dev_query, _file: &TyrDrmFile, ) -> Result { diff --git a/rust/kernel/drm/ioctl.rs b/rust/kernel/drm/ioctl.rs index 6f5a9877bdae..c70ad5e2e5a1 100644 --- a/rust/kernel/drm/ioctl.rs +++ b/rust/kernel/drm/ioctl.rs @@ -70,6 +70,18 @@ pub mod internal { pub use bindings::drm_device; pub use bindings::drm_file; pub use bindings::drm_ioctl_desc; + + /// Cast an [`Ioctl`] DRM device pointer to [`Registered`], preserving the driver type + /// parameter `T`. + /// + /// Used by [`declare_drm_ioctls!`] to anchor type inference. + #[doc(hidden)] + #[inline] + pub const fn __dev_ctx_cast( + ptr: *const crate::drm::Device, + ) -> *const crate::drm::Device { + ptr.cast() + } } /// Declare the DRM ioctls for a driver. @@ -82,7 +94,7 @@ pub mod internal { /// `user_callback` should have the following prototype: /// /// ```ignore -/// fn foo(device: &kernel::drm::Device, +/// fn foo(device: &kernel::drm::Device, /// data: &mut uapi::argument_type, /// file: &kernel::drm::File, /// ) -> Result @@ -131,17 +143,44 @@ macro_rules! declare_drm_ioctls { // - The DRM device must have been registered when we're called through // an IOCTL. // + // INVARIANT: The `Ioctl` context requires that the device has been + // registered via `drm_dev_register()` at some point; the DRM core + // guarantees this for ioctl dispatch callbacks. + // // FIXME: Currently there is nothing enforcing that the types of the // dev/file match the current driver these ioctls are being declared // for, and it's not clear how to enforce this within the type system. - let dev: &$crate::drm::device::Device<_, $crate::drm::Normal> = + let dev: &$crate::drm::device::Device<_, $crate::drm::Ioctl> = $crate::drm::device::Device::from_raw(raw_dev); + // Type-inference anchor: the closure is never called but ties `dev`'s + // type to `$func`'s first parameter, which the compiler cannot infer + // through method resolution and associated-type projections alone. + #[allow(unreachable_code)] + let _ = || { + let __ptr = $crate::drm::ioctl::internal::__dev_ctx_cast( + ::core::ptr::from_ref(dev), + ); + + $func( + // SAFETY: This closure is never executed; the dereference + // exists purely to unify the type parameter with `$func`. + // The pointer is valid regardless. + unsafe { &*__ptr }, + unreachable!(), + unreachable!(), + ) + }; + // Enforce that the handler accepts higher-ranked // lifetimes, preventing it from requiring 'static // references that could escape this scope. let _: for<'a> fn(&'a _, &'a mut _, &'a _) -> _ = $func; + let Some(guard) = dev.registration_guard() else { + return $crate::error::code::ENODEV.to_errno(); + }; + // SAFETY: The ioctl argument has size `_IOC_SIZE(cmd)`, which we // asserted above matches the size of this type, and all bit patterns of // UAPI structs must be valid. @@ -154,7 +193,7 @@ macro_rules! declare_drm_ioctls { // SAFETY: This is just the DRM file structure let file = unsafe { $crate::drm::File::from_raw(raw_file) }; - match $func(dev, data, file) { + match $func(&*guard, data, file) { Err(e) => e.to_errno(), Ok(i) => i.try_into() .unwrap_or($crate::error::code::ERANGE.to_errno()), From 47f600d40bc6bf7bcc1f28f8c7fa3e3a7aa445eb Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 28 Jun 2026 16:53:34 +0200 Subject: [PATCH 074/137] rust: drm: return ParentDevice from Device AsRef Change AsRef for drm::Device to return &T::ParentDevice instead of &device::Device, and restrict it to the Normal context. Device still gets this through Deref coercion. This provides access to the typed parent bus device rather than the raw base device. Reviewed-by: Lyude Paul Reviewed-by: Alexandre Courbot Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260628145406.2107056-15-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/drm/device.rs | 10 +++++++--- rust/kernel/drm/driver.rs | 3 ++- rust/kernel/drm/gem/shmem.rs | 3 ++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 97e2b3de78bc..d4521ef8ed80 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -450,11 +450,15 @@ unsafe fn dec_ref(obj: NonNull) { } } -impl AsRef for Device { - fn as_ref(&self) -> &device::Device { +impl AsRef> for Device { + fn as_ref(&self) -> &T::ParentDevice { // SAFETY: `bindings::drm_device::dev` is valid as long as the DRM device itself is valid, // which is guaranteed by the type invariant. - unsafe { device::Device::from_raw((*self.as_raw()).dev) } + let dev = unsafe { device::Device::from_raw((*self.as_raw()).dev) }; + + // SAFETY: The DRM device was constructed in `UnregisteredDevice::new()` with a parent + // device of type `T::ParentDevice`, hence `dev` is contained in a `T::ParentDevice`. + unsafe { device::AsBusDevice::from_device(dev) } } } diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 3cda8dceb498..9ba2eba84191 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -170,7 +170,8 @@ pub fn new_foreign_owned<'a>( where T: 'static, { - if drm.as_ref().as_raw() != dev.as_raw() { + let parent = drm.as_ref(); + if parent.as_ref().as_raw() != dev.as_raw() { return Err(EINVAL); } diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index e0ef47352e88..c1d82a04878b 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -264,7 +264,8 @@ pub fn sg_table<'a>( &'a self, dev: &'a device::Device, ) -> Result<&'a scatterlist::SGTable> { - if dev.as_raw() != self.dev().as_ref().as_raw() { + let parent = self.dev().as_ref(); + if dev.as_raw() != parent.as_ref().as_raw() { return Err(EINVAL); } From 453197b7cc320c5a7fc289bafff028bf6c550ceb Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 28 Jun 2026 16:53:35 +0200 Subject: [PATCH 075/137] rust: drm: add AsRef> for Device Implement AsRef> for Device, providing access to the bound parent bus device for registered DRM devices. Since a Device guarantees that the parent bus device is bound, the conversion to T::ParentDevice is safe. Reviewed-by: Lyude Paul Reviewed-by: Alexandre Courbot Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260628145406.2107056-16-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/drm/device.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index d4521ef8ed80..fb3724c09f27 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -462,6 +462,20 @@ fn as_ref(&self) -> &T::ParentDevice { } } +impl AsRef> for Device { + #[inline] + fn as_ref(&self) -> &T::ParentDevice { + let dev = (**self).as_ref().as_ref(); + + // SAFETY: A `Device` guarantees that the parent device is bound. + let dev = unsafe { dev.as_bound() }; + + // SAFETY: The DRM device was constructed in `UnregisteredDevice::new()` with a parent + // device of type `T::ParentDevice`, hence `dev` is contained in a `T::ParentDevice`. + unsafe { device::AsBusDevice::from_device(dev) } + } +} + // SAFETY: A `drm::Device` can be released from any thread. unsafe impl Send for Device {} From eb197f7d60f00d0f5b1b3505dfc86a7e36045a3e Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 28 Jun 2026 16:53:36 +0200 Subject: [PATCH 076/137] drm: fix race between partial drm_dev_register() failure and ioctl If drm_dev_register() fails after registering a minor (e.g. render minor registered, primary minor fails), userspace could have opened the first minor and entered a drm_dev_enter() critical section. Since the unplugged flag was never set, the ioctl proceeds while the error path tears down device resources. Fix this by introducing drm_dev_synchronize_unplug(), which sets the unplugged flag and waits for the SRCU barrier, ensuring all in-flight drm_dev_enter() critical sections complete before cleanup proceeds; call it on the error path of drm_dev_register(). Fixes: bee330f3d672 ("drm: Use srcu to protect drm_device.unplugged") Cc: stable@vger.kernel.org Reported-by: sashiko-bot@kernel.org Closes: https://lore.kernel.org/all/20260620190648.2E9F61F000E9@smtp.kernel.org/ Reviewed-by: Alexandre Courbot Reviewed-by: Lyude Paul Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260628145406.2107056-17-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/drm_drv.c | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c index 675675480da4..e890052061f3 100644 --- a/drivers/gpu/drm/drm_drv.c +++ b/drivers/gpu/drm/drm_drv.c @@ -473,6 +473,22 @@ void drm_dev_exit(int idx) } EXPORT_SYMBOL(drm_dev_exit); +/* + * Mark the device as unplugged and wait for any in-flight drm_dev_enter() + * critical sections to complete. + */ +static void drm_dev_synchronize_unplug(struct drm_device *dev) +{ + /* + * After synchronizing any critical read section is guaranteed to see + * the new value of ->unplugged, and any critical section which might + * still have seen the old value of ->unplugged is guaranteed to have + * finished. + */ + dev->unplugged = true; + synchronize_srcu(&drm_unplug_srcu); +} + /** * drm_dev_unplug - unplug a DRM device * @dev: DRM device @@ -485,15 +501,7 @@ EXPORT_SYMBOL(drm_dev_exit); */ void drm_dev_unplug(struct drm_device *dev) { - /* - * After synchronizing any critical read section is guaranteed to see - * the new value of ->unplugged, and any critical section which might - * still have seen the old value of ->unplugged is guaranteed to have - * finished. - */ - dev->unplugged = true; - synchronize_srcu(&drm_unplug_srcu); - + drm_dev_synchronize_unplug(dev); drm_dev_unregister(dev); /* Clear all CPU mappings pointing to this device */ @@ -1091,6 +1099,7 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags) goto err_minors; dev->registered = true; + dev->unplugged = false; if (driver->load) { ret = driver->load(dev, flags); @@ -1118,6 +1127,13 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags) if (dev->driver->unload) dev->driver->unload(dev); err_minors: + /* + * If a minor was registered before the failure, userspace could have + * opened it and entered a drm_dev_enter() critical section. Ensure all + * such sections complete before we clean up. + */ + drm_dev_synchronize_unplug(dev); + remove_compat_control_link(dev); drm_minor_unregister(dev, DRM_MINOR_ACCEL); drm_minor_unregister(dev, DRM_MINOR_PRIMARY); From e15b88223dc1becdc8c0d3d88795c5eb06518347 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 28 Jun 2026 16:53:37 +0200 Subject: [PATCH 077/137] rust: drm: Add RegistrationData to drm::Driver Add a RegistrationData GAT (Generic Associated Type) to drm::Driver. The lifetime parameter is tied to the parent bus device binding scope. Registration<'a, T> takes ownership of the data via Pin>, storing it with its real lifetime. The pointer is written to drm::Device before drm_dev_register() to ensure it is already in place when ioctls arrive. Device::registration_data_with() provides access with the lifetime shortened from 'static via a pointer cast. Since Registration::drop() calls drm_dev_unplug(), which performs an SRCU barrier waiting for all drm_dev_enter() critical sections to complete, the data is guaranteed to remain valid for the duration of any RegistrationGuard. Reviewed-by: Lyude Paul Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260628145406.2107056-18-dakr@kernel.org [ Move registration_data_unchecked() to Device impl block. - Danilo ] Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/driver.rs | 15 ++++-- drivers/gpu/drm/tyr/driver.rs | 15 ++++-- rust/kernel/drm/device.rs | 44 ++++++++++++++++ rust/kernel/drm/driver.rs | 95 +++++++++++++++++++--------------- rust/kernel/drm/gem/shmem.rs | 1 + 5 files changed, 120 insertions(+), 50 deletions(-) diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index e3c54303d70e..bd2a55405db8 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -20,9 +20,10 @@ pub(crate) struct NovaDriver; -pub(crate) struct Nova { +pub(crate) struct Nova<'bound> { #[expect(unused)] drm: ARef>, + _reg: drm::Registration<'bound, NovaDriver>, } /// Convienence type alias for the DRM device type for this driver @@ -56,7 +57,7 @@ pub(crate) struct NovaData { impl auxiliary::Driver for NovaDriver { type IdInfo = (); - type Data<'bound> = Nova; + type Data<'bound> = Nova<'bound>; const ID_TABLE: auxiliary::IdTable = &AUX_TABLE; fn probe<'bound>( @@ -66,15 +67,21 @@ fn probe<'bound>( let data = try_pin_init!(NovaData { adev: adev.into() }); let drm = drm::UnregisteredDevice::::new(adev, data)?; - let drm = drm::Registration::new_foreign_owned(drm, adev.as_ref(), 0)?; + // SAFETY: `reg` is stored in `Nova` and dropped when the driver is unbound; it is + // never forgotten. + let reg = unsafe { drm::Registration::new(adev.as_ref(), drm, (), 0)? }; - Ok(Nova { drm: drm.into() }) + Ok(Nova { + drm: reg.device().into(), + _reg: reg, + }) } } #[vtable] impl drm::Driver for NovaDriver { type Data = NovaData; + type RegistrationData<'a> = (); type File = File; type Object = gem::Object; type ParentDevice = auxiliary::Device; diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index 7f082de6d6dc..8348c6cd3929 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -52,8 +52,9 @@ pub(crate) struct TyrPlatformDriver; #[pin_data(PinnedDrop)] -pub(crate) struct TyrPlatformDriverData { +pub(crate) struct TyrPlatformDriverData<'bound> { _device: ARef, + _reg: drm::Registration<'bound, TyrDrmDriver>, } #[pin_data] @@ -98,7 +99,7 @@ fn issue_soft_reset(dev: &Device, iomem: &IoMem<'_>) -> Result { impl platform::Driver for TyrPlatformDriver { type IdInfo = (); - type Data<'bound> = TyrPlatformDriverData; + type Data<'bound> = TyrPlatformDriverData<'bound>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); fn probe<'bound>( @@ -150,10 +151,13 @@ fn probe<'bound>( }); let tdev = drm::UnregisteredDevice::::new(pdev, data)?; - let tdev = drm::driver::Registration::new_foreign_owned(tdev, pdev.as_ref(), 0)?; + // SAFETY: `reg` is stored in `TyrPlatformDriverData` and dropped when the driver is + // unbound; it is never forgotten. + let reg = unsafe { drm::Registration::new(pdev.as_ref(), tdev, (), 0)? }; let driver = TyrPlatformDriverData { - _device: tdev.into(), + _device: reg.device().into(), + _reg: reg, }; // We need this to be dev_info!() because dev_dbg!() does not work at @@ -164,7 +168,7 @@ fn probe<'bound>( } #[pinned_drop] -impl PinnedDrop for TyrPlatformDriverData { +impl PinnedDrop for TyrPlatformDriverData<'_> { fn drop(self: Pin<&mut Self>) {} } @@ -181,6 +185,7 @@ fn drop(self: Pin<&mut Self>) {} #[vtable] impl drm::Driver for TyrDrmDriver { type Data = TyrDrmDeviceData; + type RegistrationData<'a> = (); type File = TyrDrmFileData; type Object = drm::gem::shmem::Object; type ParentDevice = platform::Device; diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index fb3724c09f27..f43c6887ad23 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -32,6 +32,7 @@ }; use core::{ alloc::Layout, + cell::UnsafeCell, marker::PhantomData, mem, ops::Deref, @@ -247,6 +248,9 @@ pub fn new( // SAFETY: `drm_dev` is still private to this function. unsafe { (*drm_dev).driver = const { &Self::VTABLE } }; + // SAFETY: `raw_drm` is valid; no concurrent access before registration. + unsafe { (*raw_drm.as_ptr()).registration_data = UnsafeCell::new(NonNull::dangling()) }; + // SAFETY: The reference count is one, and now we take ownership of that reference as a // `drm::Device`. // INVARIANT: We just created the device above, but have yet to call `drm_dev_register`. @@ -270,6 +274,7 @@ pub fn new( pub struct Device { dev: Opaque, data: T::Data, + pub(super) registration_data: UnsafeCell>>, _ctx: PhantomData, } @@ -385,6 +390,45 @@ pub struct RegistrationGuard<'a, T: drm::Driver> { _not_send: NotThreadSafe, } +impl Device { + /// Returns a reference to the registration data with lifetime shortened from `'static`. + /// + /// # Safety + /// + /// The returned reference must not be exposed to code that can choose a concrete lifetime for + /// it, as that would be unsound for types that are invariant over their lifetime parameter + /// (e.g. it must be passed through an HRTB-bounded closure). + #[inline] + unsafe fn registration_data_unchecked(&self) -> &T::RegistrationData<'_> { + // SAFETY: + // - `Registered` guarantees the parent bus device is bound, hence the pointer is valid. + // - The pointer cast from `Of<'static>` to `Of<'_>` is layout-compatible since lifetimes + // are erased at runtime. + // - Caller guarantees the reference is only used behind an HRTB, making the lifetime + // shortening sound regardless of variance. + unsafe { (*self.registration_data.get()).cast::<_>().as_ref() } + } + + /// Access the registration data through a closure, with the lifetime tied to the closure + /// scope. + /// + /// The data is owned by [`Registration`](drm::Registration) and is guaranteed to remain valid + /// as long as the device is registered, since [`Registration`](drm::Registration)'s `drop` + /// calls `drm_dev_unplug()` which waits for all `drm_dev_enter()` critical sections to + /// complete. + #[inline] + pub fn registration_data_with(&self, f: F) -> R + where + F: for<'a> FnOnce(&'a T::RegistrationData<'a>) -> R, + { + // SAFETY: `Registered` guarantees the device is registered and the parent bus device is + // bound. The closure's HRTB `for<'a>` prevents the caller from smuggling in references + // with a concrete short lifetime, satisfying the lifetime requirement of + // `registration_data_unchecked`. + f(unsafe { self.registration_data_unchecked() }) + } +} + impl Deref for RegistrationGuard<'_, T> { type Target = Device; diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 9ba2eba84191..08b2a318cf02 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -7,16 +7,12 @@ use crate::{ bindings, device, - devres, drm, error::to_result, prelude::*, sync::aref::ARef, // }; -use core::{ - mem, - ptr::NonNull, // -}; +use core::ptr::NonNull; /// Driver use the GEM memory manager. This should be set for all modern drivers. pub(crate) const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM; @@ -110,6 +106,14 @@ pub trait Driver { /// Context data associated with the DRM driver type Data: Sync + Send; + /// Data owned by the [`Registration`] and accessible within a + /// [`RegistrationGuard`](drm::RegistrationGuard) critical section via + /// [`Device::registration_data_with()`](drm::Device::registration_data_with). + /// + /// The lifetime parameter is tied to the [`Registration`] scope, which is enclosed in the + /// parent bus device binding scope but may be shorter. + type RegistrationData<'a>: Send + Sync + 'a; + /// The type used to manage memory for this driver. type Object: AllocImpl; @@ -139,66 +143,72 @@ pub trait Driver { /// The registration type of a `drm::Device`. /// /// Once the `Registration` structure is dropped, the device is unregistered. -pub struct Registration(ARef>); +pub struct Registration<'a, T: Driver> { + drm: ARef>, + _reg_data: Pin>>, +} -impl Registration { - fn new(drm: drm::UnregisteredDevice, flags: usize) -> Result { - // SAFETY: `drm.as_raw()` is valid by the invariants of `drm::Device`. - to_result(unsafe { bindings::drm_dev_register(drm.as_raw(), flags) })?; - - // SAFETY: We just called `drm_dev_register` above - let new = NonNull::from(unsafe { drm.assume_ctx() }); - - // Leak the ARef from UnregisteredDevice in preparation for transferring its ownership. - mem::forget(drm); - - // SAFETY: `drm`'s `Drop` constructor was never called, ensuring that there remains at least - // one reference to the device - which we take ownership over here. - let new = unsafe { ARef::from_raw(new) }; - - Ok(Self(new)) - } - - /// Registers a new [`UnregisteredDevice`](drm::UnregisteredDevice) with userspace. +impl<'a, T: Driver> Registration<'a, T> { + /// Register a new [`UnregisteredDevice`](drm::UnregisteredDevice) with userspace. /// - /// Ownership of the [`Registration`] object is passed to [`devres::register`]. - pub fn new_foreign_owned<'a>( - drm: drm::UnregisteredDevice, + /// # Safety + /// + /// The caller must not `mem::forget()` the returned [`Registration`] or otherwise prevent its + /// [`Drop`] implementation from running, since the registration data may contain borrowed + /// references that become invalid after `'a` ends. + pub unsafe fn new( dev: &'a device::Device, + drm: drm::UnregisteredDevice, + reg_data: impl PinInit, E>, flags: usize, - ) -> Result<&'a drm::Device> + ) -> Result where - T: 'static, + Error: From, { let parent = drm.as_ref(); if parent.as_ref().as_raw() != dev.as_raw() { return Err(EINVAL); } - let reg = Registration::::new(drm, flags)?; - let drm = NonNull::from(reg.device()); + let reg_data: Pin>> = KBox::pin_init(reg_data, GFP_KERNEL)?; - devres::register(dev, reg, GFP_KERNEL)?; + // Store the registration data pointer in the device before registration, so that it is + // visible once ioctls can be called. + let ptr: NonNull> = + NonNull::from(Pin::get_ref(reg_data.as_ref())).cast(); - // SAFETY: Since `reg` was passed to devres::register(), the device now owns the lifetime - // of the DRM registration - ensuring that this references lives for at least as long as 'a. - Ok(unsafe { drm.as_ref() }) + // SAFETY: No concurrent access; the device is not yet registered. + unsafe { *drm.registration_data.get() = ptr }; + + // SAFETY: `drm` is a valid, initialized but not yet registered DRM device. + let ret = unsafe { bindings::drm_dev_register(drm.as_raw(), flags) }; + if let Err(e) = to_result(ret) { + // SAFETY: `drm_dev_register()` synchronizes SRCU on failure, so no concurrent + // access to `registration_data` is possible at this point. + unsafe { *drm.registration_data.get() = NonNull::dangling() }; + return Err(e); + } + + Ok(Self { + drm: (&*drm).into(), + _reg_data: reg_data, + }) } /// Returns a reference to the `Device` instance for this registration. pub fn device(&self) -> &drm::Device { - &self.0 + &self.drm } } // SAFETY: `Registration` doesn't offer any methods or access to fields when shared between // threads, hence it's safe to share it. -unsafe impl Sync for Registration {} +unsafe impl Sync for Registration<'_, T> {} // SAFETY: Registration with and unregistration from the DRM subsystem can happen from any thread. -unsafe impl Send for Registration {} +unsafe impl Send for Registration<'_, T> {} -impl Drop for Registration { +impl Drop for Registration<'_, T> { fn drop(&mut self) { // Use `drm_dev_unplug` rather than `drm_dev_unregister` to ensure that existing // `drm_dev_enter()` critical sections complete before unregistration proceeds. This @@ -208,6 +218,9 @@ fn drop(&mut self) { // // SAFETY: Safe by the invariant of `ARef>`. The existence of this // `Registration` also guarantees that this `drm::Device` is actually registered. - unsafe { bindings::drm_dev_unplug(self.0.as_raw()) }; + unsafe { bindings::drm_dev_unplug(self.drm.as_raw()) }; + // After drm_dev_unplug(), the SRCU barrier guarantees that all RegistrationGuard critical + // sections have completed, so no one holds a reference to reg_data anymore. + // reg_data is dropped here automatically. } } diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index c1d82a04878b..60dca8871b87 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -665,6 +665,7 @@ fn new( #[vtable] impl drm::Driver for KunitDriver { type Data = KunitData; + type RegistrationData<'a> = (); type File = KunitFile; type Object = Object; type ParentDevice = faux::Device; From 3ba210061c2960380007e6475d1c1119216f5c83 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 28 Jun 2026 16:53:38 +0200 Subject: [PATCH 078/137] rust: drm: Pass registration data to ioctl handlers Pass registration data to ioctl handlers via drm::Device::registration_data_with(). The closure's HRTB ties the lifetime to the closure scope, and the pointer cast shortens it from 'static internally. The reference is valid for the duration of the drm_dev_enter/exit critical section held by RegistrationGuard. Reviewed-by: Lyude Paul Reviewed-by: Alexandre Courbot Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260628145406.2107056-19-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/file.rs | 3 +++ drivers/gpu/drm/tyr/file.rs | 1 + rust/kernel/drm/ioctl.rs | 8 ++++++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/nova/file.rs b/drivers/gpu/drm/nova/file.rs index 19fb89b28984..208be4e38188 100644 --- a/drivers/gpu/drm/nova/file.rs +++ b/drivers/gpu/drm/nova/file.rs @@ -28,6 +28,7 @@ impl File { /// IOCTL: get_param: Query GPU / driver metadata. pub(crate) fn get_param( dev: &NovaDevice, + _reg_data: &(), getparam: &mut uapi::drm_nova_getparam, _file: &drm::File, ) -> Result { @@ -48,6 +49,7 @@ pub(crate) fn get_param( /// IOCTL: gem_create: Create a new DRM GEM object. pub(crate) fn gem_create( dev: &NovaDevice, + _reg_data: &(), req: &mut uapi::drm_nova_gem_create, file: &drm::File, ) -> Result { @@ -61,6 +63,7 @@ pub(crate) fn gem_create( /// IOCTL: gem_info: Query GEM metadata. pub(crate) fn gem_info( _dev: &NovaDevice, + _reg_data: &(), req: &mut uapi::drm_nova_gem_info, file: &drm::File, ) -> Result { diff --git a/drivers/gpu/drm/tyr/file.rs b/drivers/gpu/drm/tyr/file.rs index fb9233eae01c..b686041d5d6b 100644 --- a/drivers/gpu/drm/tyr/file.rs +++ b/drivers/gpu/drm/tyr/file.rs @@ -32,6 +32,7 @@ fn open(_dev: &drm::Device) -> Result>> { impl TyrDrmFileData { pub(crate) fn dev_query( ddev: &TyrDrmDevice, + _reg_data: &(), devquery: &mut uapi::drm_panthor_dev_query, _file: &TyrDrmFile, ) -> Result { diff --git a/rust/kernel/drm/ioctl.rs b/rust/kernel/drm/ioctl.rs index c70ad5e2e5a1..64af9eacc306 100644 --- a/rust/kernel/drm/ioctl.rs +++ b/rust/kernel/drm/ioctl.rs @@ -95,6 +95,7 @@ pub const fn __dev_ctx_cast( /// /// ```ignore /// fn foo(device: &kernel::drm::Device, +/// reg_data: &Self::RegistrationData<'_>, /// data: &mut uapi::argument_type, /// file: &kernel::drm::File, /// ) -> Result @@ -169,13 +170,14 @@ macro_rules! declare_drm_ioctls { unsafe { &*__ptr }, unreachable!(), unreachable!(), + unreachable!(), ) }; // Enforce that the handler accepts higher-ranked // lifetimes, preventing it from requiring 'static // references that could escape this scope. - let _: for<'a> fn(&'a _, &'a mut _, &'a _) -> _ = $func; + let _: for<'a> fn(&'a _, &'a _, &'a mut _, &'a _) -> _ = $func; let Some(guard) = dev.registration_guard() else { return $crate::error::code::ENODEV.to_errno(); @@ -193,7 +195,9 @@ macro_rules! declare_drm_ioctls { // SAFETY: This is just the DRM file structure let file = unsafe { $crate::drm::File::from_raw(raw_file) }; - match $func(&*guard, data, file) { + match guard.registration_data_with(|reg_data| { + $func(&*guard, reg_data, data, file) + }) { Err(e) => e.to_errno(), Ok(i) => i.try_into() .unwrap_or($crate::error::code::ERANGE.to_errno()), From 354a8f8b098b29d7c6064a12958cb289421a09d1 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 28 Jun 2026 16:53:39 +0200 Subject: [PATCH 079/137] drm: nova: Use drm::Device to access the parent bus device The get_param ioctl needs access to the parent auxiliary device. Since ioctl handlers run inside a RegistrationGuard, accept &NovaDevice to obtain &auxiliary::Device via as_ref() directly. This removes the need for drm::Device data, hence set it to (). Reviewed-by: Lyude Paul Reviewed-by: Alexandre Courbot Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260628145406.2107056-20-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/driver.rs | 11 ++--------- drivers/gpu/drm/nova/file.rs | 7 ++++--- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index bd2a55405db8..739690bc2db5 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -29,11 +29,6 @@ pub(crate) struct Nova<'bound> { /// Convienence type alias for the DRM device type for this driver pub(crate) type NovaDevice = drm::Device; -#[pin_data] -pub(crate) struct NovaData { - pub(crate) adev: ARef, -} - const INFO: drm::DriverInfo = drm::DriverInfo { major: 0, minor: 0, @@ -64,9 +59,7 @@ fn probe<'bound>( adev: &'bound auxiliary::Device>, _info: &'bound Self::IdInfo, ) -> impl PinInit, Error> + 'bound { - let data = try_pin_init!(NovaData { adev: adev.into() }); - - let drm = drm::UnregisteredDevice::::new(adev, data)?; + let drm = drm::UnregisteredDevice::::new(adev, Ok(()))?; // SAFETY: `reg` is stored in `Nova` and dropped when the driver is unbound; it is // never forgotten. let reg = unsafe { drm::Registration::new(adev.as_ref(), drm, (), 0)? }; @@ -80,7 +73,7 @@ fn probe<'bound>( #[vtable] impl drm::Driver for NovaDriver { - type Data = NovaData; + type Data = (); type RegistrationData<'a> = (); type File = File; type Object = gem::Object; diff --git a/drivers/gpu/drm/nova/file.rs b/drivers/gpu/drm/nova/file.rs index 208be4e38188..298c02bacb4b 100644 --- a/drivers/gpu/drm/nova/file.rs +++ b/drivers/gpu/drm/nova/file.rs @@ -4,6 +4,8 @@ use crate::gem::NovaObject; use kernel::{ alloc::flags::*, + auxiliary, + device::Bound, drm::{ self, gem::BaseObject, @@ -32,9 +34,8 @@ pub(crate) fn get_param( getparam: &mut uapi::drm_nova_getparam, _file: &drm::File, ) -> Result { - let adev = &dev.adev; - let parent = adev.parent(); - let pdev: &pci::Device = parent.try_into()?; + let adev: &auxiliary::Device = dev.as_ref(); + let pdev: &pci::Device = adev.parent().try_into()?; let value = match getparam.param as u32 { uapi::NOVA_GETPARAM_VRAM_BAR_SIZE => pdev.resource_len(1)?, From 9a80ddbf66e4fb549fe1e94858da659619333dda Mon Sep 17 00:00:00 2001 From: Yilin Chen <1479826151@qq.com> Date: Tue, 7 Jul 2026 16:33:33 +0000 Subject: [PATCH 080/137] rust: drm: fix GEM object pointer safety docs IntoGEMObject::from_raw() receives a pointer to struct drm_gem_object, not a pointer to Self. The previous documentation used Self even though the function argument is the embedded GEM object pointer. However, the pointer must not be any arbitrary valid drm_gem_object. The implementations recover Self with container_of(), so the GEM object must be embedded in a valid Self instance. This patch documents that requirement explicitly. Assisted-by: Codex:GPT-5 Signed-off-by: Yilin Chen <1479826151@qq.com> Link: https://patch.msgid.link/tencent_4426892E62B77DEA2AE898E899A871940005@qq.com Signed-off-by: Alice Ryhl --- rust/kernel/drm/gem/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index b03b5f9ccd7e..80d8f524f9d5 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -117,7 +117,8 @@ pub trait IntoGEMObject: Sized + super::private::Sealed { /// /// # Safety /// - /// - `self_ptr` must be a valid pointer to `Self`. + /// - `self_ptr` must be a valid pointer to the `struct drm_gem_object` embedded in a + /// valid instance of `Self`. /// - The caller promises that holding the immutable reference returned by this function does /// not violate rust's data aliasing rules and remains valid throughout the lifetime of `'a`. unsafe fn from_raw<'a>(self_ptr: *mut bindings::drm_gem_object) -> &'a Self; From b59ec72fec247b90dc26f17c5b1ee9f1e0fe334c Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Thu, 11 Jun 2026 22:17:21 +0000 Subject: [PATCH 081/137] rust: drm: gpuvm: require Send + Sync for the driver's associated data DriverGpuVm permitted !Send/!Sync associated data on an abstraction whose handles are shared and dropped across threads: obtain() runs from many threads and the VA API performs deferred cross-thread drops. That is unsound. Require Send + Sync on the trait and its associated data so the GpuVm and UniqueRefGpuVm handle impls need no per-impl bounds. Fixes: 82b78182eacf ("rust: drm: add base GPUVM immediate mode abstraction") Signed-off-by: Sami Tolvanen Link: https://patch.msgid.link/20260611-gpuvm-sync-send-v4-1-6c7f4ab2778a@google.com Signed-off-by: Alice Ryhl --- rust/kernel/drm/gpuvm/mod.rs | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs index 20a08b3defeb..d9d43d719761 100644 --- a/rust/kernel/drm/gpuvm/mod.rs +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -72,10 +72,12 @@ pub struct GpuVm { data: UnsafeCell, } -// SAFETY: The GPUVM api does not assume that it is tied to a specific thread. The destructor will -// drop the `data` field, which is okay because it is guaranteed `Send` by the `DriverGpuVm` trait. +// SAFETY: It is safe to send a `GpuVm` to another thread: all data reachable through it +// (`T`, `T::VmBoData`, and the GEM `T::Object`) is `Send` by the `DriverGpuVm` bounds. unsafe impl Send for GpuVm {} -// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel. +// SAFETY: It is safe to share a `&GpuVm` between threads: `&self` methods only alias data +// that is `Sync` by the `DriverGpuVm` bounds, and any thread may drop that data, or upgrade the +// reference and ultimately drop `T`, which the same bounds make `Send`. unsafe impl Sync for GpuVm {} // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. @@ -250,18 +252,22 @@ fn raw_resv(&self) -> *mut bindings::dma_resv { } /// The manager for a GPUVM. -pub trait DriverGpuVm: Sized + Send { +pub trait DriverGpuVm: Sized + Send + Sync { /// Parent `Driver` for this object. type Driver: drm::Driver; /// The kind of GEM object stored in this GPUVM. - type Object: drm::driver::AllocImpl; + type Object: drm::driver::AllocImpl + Send + Sync; /// Data stored with each [`struct drm_gpuva`](struct@GpuVa). - type VaData; + /// + /// Only `Send` is required: the data has a single owner at all times, moving + /// between threads by value (handed back as a [`GpuVaRemoved`]) but never + /// accessed by two threads concurrently. + type VaData: Send; /// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo). - type VmBoData; + type VmBoData: Send + Sync; /// The private data passed to callbacks. type SmContext<'ctx> @@ -298,12 +304,10 @@ fn sm_step_remap<'op, 'ctx>( /// # Invariants /// /// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference. +// `Send`/`Sync` derive from `ARef>`; the trait bounds make them correct for the unique +// handle's `&mut T` access. pub struct UniqueRefGpuVm(ARef>); -// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel, and -// concurrent access to `data` is safe due to the `T: Sync` requirement. -unsafe impl Sync for UniqueRefGpuVm {} - impl UniqueRefGpuVm { /// Access the data owned by this `UniqueRefGpuVm` immutably. #[inline] From 727dc02ec6b9c1c30c99d51a4db965f0bb27b12f Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Thu, 11 Jun 2026 22:17:22 +0000 Subject: [PATCH 082/137] rust: drm: gpuvm: implement Send and Sync for GpuVaAlloc and GpuVmBo Moving a GpuVaAlloc or GpuVmBo between threads currently forces drivers to write their own unsafe Send and Sync impls. Provide the markers in the abstraction instead. GpuVaAlloc wraps only uninitialised memory and exposes none of it. GpuVmBo hands out the driver data and GEM object by shared reference and drops them in its deferred put; the DriverGpuVm trait already guarantees both are Send + Sync, so both impls are unconditional. Signed-off-by: Sami Tolvanen Link: https://patch.msgid.link/20260611-gpuvm-sync-send-v4-2-6c7f4ab2778a@google.com Signed-off-by: Alice Ryhl --- rust/kernel/drm/gpuvm/va.rs | 8 ++++++++ rust/kernel/drm/gpuvm/vm_bo.rs | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/rust/kernel/drm/gpuvm/va.rs b/rust/kernel/drm/gpuvm/va.rs index 0b09fe44ab39..b108ec7aa1bc 100644 --- a/rust/kernel/drm/gpuvm/va.rs +++ b/rust/kernel/drm/gpuvm/va.rs @@ -104,6 +104,14 @@ pub fn vm_bo(&self) -> &GpuVmBo { /// The memory is zeroed. pub struct GpuVaAlloc(KBox>>); +// SAFETY: A `GpuVaAlloc` is an owned, uninitialised allocation with no live `T::VaData` and no +// thread-bound state. +unsafe impl Send for GpuVaAlloc {} + +// SAFETY: A `GpuVaAlloc` has no `&self` method that reaches its contents, so a shared +// `&GpuVaAlloc` cannot access the allocation. +unsafe impl Sync for GpuVaAlloc {} + impl GpuVaAlloc { /// Pre-allocate a [`GpuVa`] object. pub fn new(flags: AllocFlags) -> Result, AllocError> { diff --git a/rust/kernel/drm/gpuvm/vm_bo.rs b/rust/kernel/drm/gpuvm/vm_bo.rs index c064ac63897b..a30f838c11b8 100644 --- a/rust/kernel/drm/gpuvm/vm_bo.rs +++ b/rust/kernel/drm/gpuvm/vm_bo.rs @@ -19,6 +19,15 @@ pub struct GpuVmBo { data: T::VmBoData, } +// SAFETY: It is safe to send a `GpuVmBo` to another thread: dropping it there drops +// `T::VmBoData` and the GEM `T::Object`, both `Send` by the `DriverGpuVm` bounds. +unsafe impl Send for GpuVmBo {} + +// SAFETY: It is safe to share a `&GpuVmBo` between threads: it effectively shares +// `&T::VmBoData` and the GEM `&T::Object` (both `Sync`), and any thread may upgrade to an +// `ARef` and ultimately drop them (both `Send`), per the `DriverGpuVm` bounds. +unsafe impl Sync for GpuVmBo {} + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`. unsafe impl AlwaysRefCounted for GpuVmBo { fn inc_ref(&self) { From 6cb3fdc9f5aeb0c5c0f68a28e9370cbe37f555f2 Mon Sep 17 00:00:00 2001 From: Younes Akhouayri Date: Sun, 12 Jul 2026 15:39:46 +0200 Subject: [PATCH 083/137] rust: drm: Fix typo in FEAT_RENDER documentation Correct the spelling of "privilege" in the DRIVER_RENDER Rustdoc. Signed-off-by: Younes Akhouayri Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20260712-docs-drm-feat-render-rustdoc-typo-v1-1-c9df1cbbce4b@younes.io Signed-off-by: Danilo Krummrich --- rust/kernel/drm/driver.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 08b2a318cf02..74f6ed690d8b 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -132,7 +132,7 @@ pub trait Driver { /// Sets the `DRIVER_RENDER` feature for this driver. /// /// When enabled, the driver exposes `/dev/dri/renderDXX` render nodes to - /// userspace. The render node is an alternate low-priviledge way to access + /// userspace. The render node is an alternate low-privilege way to access /// the driver, which is enforced on a per-ioctl level. Userspace processes /// that open the render node can only invoke ioctls explicitly listed as /// usable from the render node (i.e. marked DRM_RENDER_ALLOW), whereas From d3cac8a343241a547445e8a651f1d4ecc276b828 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Thu, 9 Jul 2026 15:52:54 +0900 Subject: [PATCH 084/137] gpu: nova-core: gsp: sequencer: use GspBootContext `GspBootContext` contains all the resources currently carried by `GspSequencerParams`, so replace the latter with the former for better integration with the boot process and less code. Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260709-nova-bootcontext-v6-1-520cbf8b9b50@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/hal/tu102.rs | 21 +++++---------- drivers/gpu/nova-core/gsp/sequencer.rs | 36 ++++++++++---------------- 2 files changed, 20 insertions(+), 37 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index ff71b45b5432..87ceb8878f01 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -37,10 +37,7 @@ GspHal, UnloadBundle, // }, - sequencer::{ - GspSequencer, - GspSequencerParams, // - }, + sequencer::GspSequencer, Gsp, GspBootContext, GspFwWprMeta, // @@ -326,16 +323,12 @@ fn boot<'a>( } fn post_boot(&self, gsp: &Gsp, ctx: &GspBootContext<'_>, gsp_fw: &GspFirmware) -> Result { - // Create and run the GSP sequencer. - let seq_params = GspSequencerParams { - bootloader_app_version: gsp_fw.bootloader.app_version, - libos_dma_handle: gsp.libos.dma_handle(), - gsp_falcon: ctx.gsp_falcon, - sec2_falcon: ctx.sec2_falcon, - dev: ctx.dev(), - bar: ctx.bar, - }; - GspSequencer::run(&gsp.cmdq, seq_params)?; + GspSequencer::run( + &gsp.cmdq, + ctx, + gsp.libos.dma_handle(), + gsp_fw.bootloader.app_version, + )?; Ok(()) } diff --git a/drivers/gpu/nova-core/gsp/sequencer.rs b/drivers/gpu/nova-core/gsp/sequencer.rs index 13983d42b12b..f55205bd61f3 100644 --- a/drivers/gpu/nova-core/gsp/sequencer.rs +++ b/drivers/gpu/nova-core/gsp/sequencer.rs @@ -31,6 +31,7 @@ MessageFromGsp, // }, fw, + GspBootContext, // }, num::FromSafeCast, sbuffer::SBufferIter, @@ -335,24 +336,13 @@ fn iter(&self) -> GspSeqIter<'_> { } } -/// Parameters for running the GSP sequencer. -pub(crate) struct GspSequencerParams<'a> { - /// Bootloader application version. - pub(crate) bootloader_app_version: u32, - /// LibOS DMA handle address. - pub(crate) libos_dma_handle: u64, - /// GSP falcon for core operations. - pub(crate) gsp_falcon: &'a Falcon<'a, Gsp>, - /// SEC2 falcon for core operations. - pub(crate) sec2_falcon: &'a Falcon<'a, Sec2>, - /// Device for logging. - pub(crate) dev: &'a device::Device, - /// BAR0 for register access. - pub(crate) bar: Bar0<'a>, -} - impl<'a> GspSequencer<'a> { - pub(crate) fn run(cmdq: &Cmdq, params: GspSequencerParams<'a>) -> Result { + pub(crate) fn run( + cmdq: &Cmdq, + ctx: &'a GspBootContext<'_>, + libos_dma_handle: u64, + bootloader_app_version: u32, + ) -> Result { let seq_info = loop { match cmdq.receive_msg::(Cmdq::RECEIVE_TIMEOUT) { Ok(seq_info) => break seq_info, @@ -363,12 +353,12 @@ pub(crate) fn run(cmdq: &Cmdq, params: GspSequencerParams<'a>) -> Result { let sequencer = GspSequencer { seq_info, - bar: params.bar, - sec2_falcon: params.sec2_falcon, - gsp_falcon: params.gsp_falcon, - libos_dma_handle: params.libos_dma_handle, - bootloader_app_version: params.bootloader_app_version, - dev: params.dev, + bar: ctx.bar, + sec2_falcon: ctx.sec2_falcon, + gsp_falcon: ctx.gsp_falcon, + libos_dma_handle, + bootloader_app_version, + dev: ctx.dev(), }; dev_dbg!(sequencer.dev, "Running CPU Sequencer commands\n"); From 686a7fd1f7183c9fbd425ba50cf8dc37e020ec73 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Thu, 9 Jul 2026 15:52:55 +0900 Subject: [PATCH 085/137] gpu: nova-core: gsp: sequencer: do not store sequence into GspSequencer The sequence is currently stored in the `GspSequencer` even though its lifetime is limited to `GspSequencer::run`. This object-oriented design does not play well with the borrow-checker, as `GspSequencer::iter` borrows the `GspSequencer`, which makes it difficult to introduce mutable references in `GspBootContext`, as we want to do in order to make the `Falcon` references mutable. Thus, store the sequence locally in `GspSequencer::run`, and move iterator creation to `GspSeqIter::new` so it no longer needs to borrow the whole `GspSequencer`. Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260709-nova-bootcontext-v6-2-520cbf8b9b50@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/sequencer.rs | 35 +++++++++++--------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/sequencer.rs b/drivers/gpu/nova-core/gsp/sequencer.rs index f55205bd61f3..ddce32cc4e30 100644 --- a/drivers/gpu/nova-core/gsp/sequencer.rs +++ b/drivers/gpu/nova-core/gsp/sequencer.rs @@ -129,8 +129,6 @@ pub(crate) fn new(data: &[u8], dev: &device::Device) -> Result<(Self, usize)> { /// GSP Sequencer for executing firmware commands during boot. pub(crate) struct GspSequencer<'a> { - /// Sequencer information with command data. - seq_info: GspSequence, /// `Bar0` for register access. bar: Bar0<'a>, /// SEC2 falcon for core operations. @@ -268,7 +266,7 @@ fn run(&self, seq: &GspSequencer<'_>) -> Result { } /// Iterator over GSP sequencer commands. -pub(crate) struct GspSeqIter<'a> { +struct GspSeqIter<'a> { /// Command data buffer. cmd_data: &'a [u8], /// Current position in the buffer. @@ -281,6 +279,18 @@ pub(crate) struct GspSeqIter<'a> { dev: &'a device::Device, } +impl<'a> GspSeqIter<'a> { + fn new(seq: &'a GspSequence, dev: &'a device::Device) -> Self { + Self { + cmd_data: &seq.cmd_data, + current_offset: 0, + total_cmds: seq.cmd_index, + cmds_processed: 0, + dev, + } + } +} + impl<'a> Iterator for GspSeqIter<'a> { type Item = Result; @@ -322,20 +332,6 @@ fn next(&mut self) -> Option { } } -impl<'a> GspSequencer<'a> { - fn iter(&self) -> GspSeqIter<'_> { - let cmd_data = &self.seq_info.cmd_data[..]; - - GspSeqIter { - cmd_data, - current_offset: 0, - total_cmds: self.seq_info.cmd_index, - cmds_processed: 0, - dev: self.dev, - } - } -} - impl<'a> GspSequencer<'a> { pub(crate) fn run( cmdq: &Cmdq, @@ -352,7 +348,6 @@ pub(crate) fn run( }; let sequencer = GspSequencer { - seq_info, bar: ctx.bar, sec2_falcon: ctx.sec2_falcon, gsp_falcon: ctx.gsp_falcon, @@ -363,14 +358,14 @@ pub(crate) fn run( dev_dbg!(sequencer.dev, "Running CPU Sequencer commands\n"); - for cmd_result in sequencer.iter() { + for cmd_result in GspSeqIter::new(&seq_info, sequencer.dev) { match cmd_result { Ok(cmd) => cmd.run(&sequencer)?, Err(e) => { dev_err!( sequencer.dev, "Error running command at index {}\n", - sequencer.seq_info.cmd_index + seq_info.cmd_index ); return Err(e); } From ebf164beb9aedac6627d5a2a7da0d1a5807161c3 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Thu, 9 Jul 2026 15:52:56 +0900 Subject: [PATCH 086/137] gpu: nova-core: gsp: replace BootUnloadGuard with local handlers When adding the GSP unload capability, we introduced `BootUnloadGuard` to automatically call `Gsp::unload` whenever an error occurred during the boot process, in order to try to reset the GSP to a valid state. This approach is not well-suited to the errors that may occur in HALs: by definition, an error occurring in the HAL means that the GSP is not booted; yet the first thing that `Gsp::unload` does is queue a shutdown message to the GSP, which will inevitably result in a timeout when done from a HAL. Furthermore, `BootUnloadGuard` is problematic because it holds additional references to the boot context, notably the `Falcon`s. These extra references stand in the way of making some of the `Falcon`'s methods mutable, since those methods would require exclusive access. As this behavior is only needed in one place, introducing dedicated types for it is distracting and unnecessary. Thus, remove `BootUnloadGuard` and adopt a two-level error handling strategy: - HALs are free to handle their errors as they see fit (most likely, by running their unload bundle if it is ready by the time of the error), - `Gsp::boot` uses a `ScopeGuard` that runs `Gsp::unload`, since the GSP should be up and running by the time `GspHal::boot` has returned. Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260709-nova-bootcontext-v6-3-520cbf8b9b50@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/boot.rs | 67 +++----------------------- drivers/gpu/nova-core/gsp/hal.rs | 13 +++-- drivers/gpu/nova-core/gsp/hal/gh100.rs | 27 ++++++----- drivers/gpu/nova-core/gsp/hal/tu102.rs | 23 +++++---- 4 files changed, 41 insertions(+), 89 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 152093aafae6..f62079709b6f 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -30,66 +30,6 @@ }, }; -/// Arguments required to call [`Gsp::unload`](super::Gsp::unload). -/// -/// Stored as their own type to avoid repeating a long and tedious list in [`BootUnloadGuard`]. -pub(super) struct BootUnloadArgs<'a> { - gsp: &'a super::Gsp, - dev: &'a device::Device, - bar: Bar0<'a>, - gsp_falcon: &'a Falcon<'a, Gsp>, - sec2_falcon: &'a Falcon<'a, Sec2>, - unload_bundle: Option, -} - -/// Guard that calls [`Gsp::unload`](super::Gsp::unload) with a -/// [`UnloadBundle`](super::UnloadBundle) when dropped. -/// -/// Used to ensure the `UnloadBundle` is run during failure paths. -pub(super) struct BootUnloadGuard<'a> { - guard: ScopeGuard, fn(BootUnloadArgs<'a>)>, -} - -impl<'a> BootUnloadGuard<'a> { - /// Wraps `unload_bundle` into a guard that executes it when dropped. - pub(super) fn new( - gsp: &'a super::Gsp, - dev: &'a device::Device, - bar: Bar0<'a>, - gsp_falcon: &'a Falcon<'a, Gsp>, - sec2_falcon: &'a Falcon<'a, Sec2>, - unload_bundle: Option, - ) -> Self { - Self { - guard: ScopeGuard::new_with_data( - BootUnloadArgs { - gsp, - dev, - bar, - gsp_falcon, - sec2_falcon, - unload_bundle, - }, - |args| { - let _ = super::Gsp::unload( - args.gsp, - args.dev, - args.bar, - args.gsp_falcon, - args.sec2_falcon, - args.unload_bundle, - ); - }, - ), - } - } - - /// Disarms the guard and returns the [`UnloadBundle`](super::UnloadBundle) it contains. - pub(super) fn dismiss(self) -> Option { - self.guard.dismiss().unload_bundle - } -} - impl super::Gsp { /// Attempt to boot the GSP. /// @@ -107,6 +47,7 @@ pub(crate) fn boot( let bar = ctx.bar; let chipset = ctx.chipset; let gsp_falcon = ctx.gsp_falcon; + let sec2_falcon = ctx.sec2_falcon; let dev = pdev.as_ref(); let hal = super::hal::gsp_hal(chipset); @@ -118,7 +59,11 @@ pub(crate) fn boot( let wpr_meta = Coherent::init(dev, GFP_KERNEL, GspFwWprMeta::new(&gsp_fw, &fb_layout))?; // Perform the chipset-specific boot sequence, and retrieve the unload bundle. - let unload_guard = hal.boot(&self, &ctx, &fb_layout, &wpr_meta)?; + let unload_bundle = hal.boot(&self, &ctx, &fb_layout, &wpr_meta)?; + + let unload_guard = ScopeGuard::new_with_data(unload_bundle, |unload_bundle| { + let _ = self.unload(dev, bar, gsp_falcon, sec2_falcon, unload_bundle); + }); gsp_falcon.write_os_version(gsp_fw.bootloader.app_version); diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs index d3e47ef206de..851d1f24c137 100644 --- a/drivers/gpu/nova-core/gsp/hal.rs +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -24,7 +24,6 @@ Chipset, // }, gsp::{ - boot::BootUnloadGuard, Gsp, GspBootContext, GspFwWprMeta, // @@ -51,15 +50,15 @@ fn run( pub(super) trait GspHal: Send { /// Performs the GSP boot process, loading and running the required firmwares as needed. /// - /// Upon success, returns a guard that runs the GSP unload sequence if GSP boot does not - /// complete. - fn boot<'a>( + /// Upon success, returns the [`crate::gsp::UnloadBundle`] to use with [`Gsp::unload`], if one + /// could be created. + fn boot( &self, - gsp: &'a Gsp, - ctx: &GspBootContext<'a>, + gsp: &Gsp, + ctx: &GspBootContext<'_>, fb_layout: &FbLayout, wpr_meta: &Coherent, - ) -> Result>; + ) -> Result>; /// Performs HAL-specific post-GSP boot tasks. /// diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index 1d06405a32f6..5a39d087c904 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -7,7 +7,8 @@ device, dma::Coherent, io::poll::read_poll_timeout, - time::Delta, // + time::Delta, + types::ScopeGuard, // }; use crate::{ @@ -23,7 +24,6 @@ Fsp, // }, gsp::{ - boot::BootUnloadGuard, hal::{ GspHal, UnloadBundle, // @@ -143,13 +143,13 @@ impl GspHal for Gh100 { /// /// This path uses FSP to establish a chain of trust and boot GSP-FMC. FSP handles /// the GSP boot internally - no manual GSP reset/boot is needed. - fn boot<'a>( + fn boot( &self, - gsp: &'a Gsp, - ctx: &GspBootContext<'a>, + gsp: &Gsp, + ctx: &GspBootContext<'_>, fb_layout: &FbLayout, wpr_meta: &Coherent, - ) -> Result> { + ) -> Result> { let dev = ctx.dev(); let bar = ctx.bar; let chipset = ctx.chipset; @@ -160,10 +160,6 @@ fn boot<'a>( KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox ); - // Wrap the unload bundle into a drop guard so it is automatically run upon failure. - let unload_guard = - BootUnloadGuard::new(gsp, dev, bar, gsp_falcon, sec2_falcon, Some(unload_bundle)); - let mut fsp = Fsp::wait_secure_boot(dev, bar, chipset)?; let args = FmcBootArgs::new( @@ -174,11 +170,20 @@ fn boot<'a>( false, )?; + // Wait for the GSP RISC-V core to halt in case of error. We create this guard after `args` + // to make sure that boot args are kept alive until halt, in case they are still being + // accessed. + let unload_guard = ScopeGuard::new_with_data(unload_bundle, |unload_bundle| { + let _ = unload_bundle.0.run(dev, bar, gsp_falcon, sec2_falcon); + }); + fsp.boot_fmc(dev, fb_layout, &args)?; + // Wait for GSP-FMC to release the GSP lockdown, indicating that `args` is not accessed + // anymore. wait_for_gsp_lockdown_release(dev, gsp_falcon, args.boot_params_dma_handle())?; - Ok(unload_guard) + Ok(Some(unload_guard.dismiss())) } } diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index 87ceb8878f01..f78e2489f5a6 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -6,7 +6,8 @@ use kernel::{ device, dma::Coherent, - io::Io, // + io::Io, + types::ScopeGuard, // }; use crate::{ @@ -32,7 +33,6 @@ }, gpu::Chipset, gsp::{ - boot::BootUnloadGuard, hal::{ GspHal, UnloadBundle, // @@ -259,13 +259,13 @@ fn run_fwsec_frts( struct Tu102; impl GspHal for Tu102 { - fn boot<'a>( + fn boot( &self, - gsp: &'a Gsp, - ctx: &GspBootContext<'a>, + gsp: &Gsp, + ctx: &GspBootContext<'_>, fb_layout: &FbLayout, wpr_meta: &Coherent, - ) -> Result> { + ) -> Result> { let dev = ctx.dev(); let bar = ctx.bar; let chipset = ctx.chipset; @@ -290,9 +290,12 @@ fn boot<'a>( .ok() .map(crate::gsp::UnloadBundle); - // Wrap the unload bundle into a drop guard so it is automatically run upon failure. - let unload_guard = - BootUnloadGuard::new(gsp, dev, bar, gsp_falcon, sec2_falcon, unload_bundle); + // Run the unload bundle to try and recover the GSP if an error occurs. + let unload_guard = ScopeGuard::new_with_data(unload_bundle, |unload_bundle| { + if let Some(unload_bundle) = unload_bundle { + let _ = unload_bundle.0.run(dev, bar, gsp_falcon, sec2_falcon); + } + }); // FWSEC-FRTS is not executed on chips where the FRTS region size is 0 (e.g. GA100). if !fb_layout.frts.is_empty() { @@ -319,7 +322,7 @@ fn boot<'a>( )? .run(dev, sec2_falcon, wpr_meta)?; - Ok(unload_guard) + Ok(unload_guard.dismiss()) } fn post_boot(&self, gsp: &Gsp, ctx: &GspBootContext<'_>, gsp_fw: &GspFirmware) -> Result { From 3099edaaabe97d9cbe604083a9badde70b05221e Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Thu, 9 Jul 2026 15:52:57 +0900 Subject: [PATCH 087/137] gpu: nova-core: gsp: pass GspBootContext to unload methods `GspBootContext` contains the resources required to boot the GSP. As it turns out, this is also the context required for unloading it. Reflect that fact by replacing the arguments of `Gsp::unload` and `UnloadBundle::run` with the `GspBootContext`. This symmetry between `Gsp::boot` and `Gsp::unload` will also be convenient when we want to make these methods generic over the boot context corresponding to the boot method used. Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260709-nova-bootcontext-v6-4-520cbf8b9b50@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gpu.rs | 19 ++++++++++++++--- drivers/gpu/nova-core/gsp/boot.rs | 28 ++++++++++++-------------- drivers/gpu/nova-core/gsp/hal.rs | 15 +------------- drivers/gpu/nova-core/gsp/hal/gh100.rs | 17 ++++------------ drivers/gpu/nova-core/gsp/hal/tu102.rs | 23 ++++++++++----------- 5 files changed, 45 insertions(+), 57 deletions(-) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 43c3f4f8df71..32bfa0be2357 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -269,7 +269,9 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { #[pin_data(PinnedDrop)] struct GspResources<'gpu> { /// Device owning the GPU. - device: &'gpu device::Device, + device: &'gpu pci::Device, + /// Details about the chipset. + spec: Spec, /// MMIO mapping of PCI BAR 0. bar: Bar0<'gpu>, /// GSP falcon instance, used for GSP boot up and cleanup. @@ -312,7 +314,16 @@ fn drop(self: Pin<&mut Self>) { .gsp .as_ref() .get_ref() - .unload(device, bar, &*this.gsp_falcon, &*this.sec2_falcon, bundle) + .unload( + GspBootContext { + pdev: device, + bar, + chipset: this.spec.chipset, + gsp_falcon: &*this.gsp_falcon, + sec2_falcon: &*this.sec2_falcon, + }, + bundle, + ) .inspect_err(|e| dev_err!(device, "failed to unload GSP: {:?}\n", e)); } } @@ -344,7 +355,9 @@ pub(crate) fn new( sysmem_flush: SysmemFlush::register(pdev.as_ref(), bar, spec.chipset)?, gsp_resources <- try_pin_init!(GspResources { - device: pdev.as_ref(), + device: pdev, + + spec: *spec, bar, diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index f62079709b6f..a23219a79355 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -3,7 +3,6 @@ use kernel::{ bits, - device, dma::Coherent, io::poll::read_poll_timeout, prelude::*, @@ -15,7 +14,6 @@ driver::Bar0, falcon::{ gsp::Gsp, - sec2::Sec2, Falcon, // }, fb::FbLayout, @@ -47,7 +45,6 @@ pub(crate) fn boot( let bar = ctx.bar; let chipset = ctx.chipset; let gsp_falcon = ctx.gsp_falcon; - let sec2_falcon = ctx.sec2_falcon; let dev = pdev.as_ref(); let hal = super::hal::gsp_hal(chipset); @@ -61,9 +58,11 @@ pub(crate) fn boot( // Perform the chipset-specific boot sequence, and retrieve the unload bundle. let unload_bundle = hal.boot(&self, &ctx, &fb_layout, &wpr_meta)?; - let unload_guard = ScopeGuard::new_with_data(unload_bundle, |unload_bundle| { - let _ = self.unload(dev, bar, gsp_falcon, sec2_falcon, unload_bundle); - }); + let unload_guard = + ScopeGuard::new_with_data((ctx, unload_bundle), |(ctx, unload_bundle)| { + let _ = self.unload(ctx, unload_bundle); + }); + let ctx = &unload_guard.0; gsp_falcon.write_os_version(gsp_fw.bootloader.app_version); @@ -82,12 +81,12 @@ pub(crate) fn boot( self.cmdq .send_command_no_wait(bar, commands::SetRegistry::new()?)?; - hal.post_boot(&self, &ctx, &gsp_fw)?; + hal.post_boot(&self, ctx, &gsp_fw)?; // Wait until GSP is fully initialized. commands::wait_gsp_init_done(&self.cmdq)?; - Ok(unload_guard.dismiss()) + Ok(unload_guard.dismiss().1) } /// Shut down the GSP and wait until it is offline. @@ -116,17 +115,16 @@ fn shutdown_gsp( /// This stops all activity on the GSP. pub(crate) fn unload( &self, - dev: &device::Device, - bar: Bar0<'_>, - gsp_falcon: &Falcon<'_, Gsp>, - sec2_falcon: &Falcon<'_, Sec2>, + ctx: super::GspBootContext<'_>, unload_bundle: Option, ) -> Result { + let dev = ctx.dev(); + // Shut down the GSP. Keep going even in case of error. let mut res = Self::shutdown_gsp( &self.cmdq, - bar, - gsp_falcon, + ctx.bar, + ctx.gsp_falcon, commands::PowerStateLevel::Level0, ) .inspect_err(|e| dev_err!(dev, "GSP shutdown failed: {:?}\n", e)); @@ -136,7 +134,7 @@ pub(crate) fn unload( res = res.and( unload_bundle .0 - .run(dev, bar, gsp_falcon, sec2_falcon) + .run(&ctx) .inspect_err(|e| dev_err!(dev, "Unload bundle failed: {:?}\n", e)), ); } else { diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs index 851d1f24c137..849ca224085b 100644 --- a/drivers/gpu/nova-core/gsp/hal.rs +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -5,18 +5,11 @@ mod tu102; use kernel::{ - device, dma::Coherent, prelude::*, // }; use crate::{ - driver::Bar0, - falcon::{ - gsp::Gsp as GspEngine, - sec2::Sec2, - Falcon, // - }, fb::FbLayout, firmware::gsp::GspFirmware, gpu::{ @@ -37,13 +30,7 @@ /// required for unloading is prepared at load time, and stored here until it needs to be run. pub(super) trait UnloadBundle: Send { /// Performs the steps required to properly reset the GSP after it has been stopped. - fn run( - &self, - dev: &device::Device, - bar: Bar0<'_>, - gsp_falcon: &Falcon<'_, GspEngine>, - sec2_falcon: &Falcon<'_, Sec2>, - ) -> Result; + fn run(&self, ctx: &GspBootContext<'_>) -> Result; } /// Trait implemented by GSP HALs. diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index 5a39d087c904..b991f160abcc 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -12,10 +12,8 @@ }; use crate::{ - driver::Bar0, falcon::{ gsp::Gsp as GspEngine, - sec2::Sec2, Falcon, // }, fb::FbLayout, @@ -117,22 +115,16 @@ fn wait_for_gsp_lockdown_release( struct FspUnloadBundle; impl UnloadBundle for FspUnloadBundle { - fn run( - &self, - dev: &device::Device, - _bar: Bar0<'_>, - gsp_falcon: &Falcon<'_, GspEngine>, - _sec2_falcon: &Falcon<'_, Sec2>, - ) -> Result { + fn run(&self, ctx: &GspBootContext<'_>) -> Result { // GSP falcon does most of the work of resetting, so just wait for it to finish. read_poll_timeout( - || Ok(gsp_falcon.is_riscv_active()), + || Ok(ctx.gsp_falcon.is_riscv_active()), |&active| !active, Delta::from_millis(10), Delta::from_secs(5), ) .map(|_| ()) - .inspect_err(|_| dev_err!(dev, "GSP falcon failed to halt\n")) + .inspect_err(|_| dev_err!(ctx.dev(), "GSP falcon failed to halt\n")) } } @@ -154,7 +146,6 @@ fn boot( let bar = ctx.bar; let chipset = ctx.chipset; let gsp_falcon = ctx.gsp_falcon; - let sec2_falcon = ctx.sec2_falcon; let unload_bundle = crate::gsp::UnloadBundle( KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox @@ -174,7 +165,7 @@ fn boot( // to make sure that boot args are kept alive until halt, in case they are still being // accessed. let unload_guard = ScopeGuard::new_with_data(unload_bundle, |unload_bundle| { - let _ = unload_bundle.0.run(dev, bar, gsp_falcon, sec2_falcon); + let _ = unload_bundle.0.run(ctx); }); fsp.boot_fmc(dev, fb_layout, &args)?; diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index f78e2489f5a6..c0956fb1c9cf 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -121,18 +121,15 @@ fn build( } impl UnloadBundle for Sec2UnloadBundle { - fn run( - &self, - dev: &device::Device, - bar: Bar0<'_>, - gsp_falcon: &Falcon<'_, GspEngine>, - sec2_falcon: &Falcon<'_, Sec2>, - ) -> Result { + fn run(&self, ctx: &GspBootContext<'_>) -> Result { + let dev = ctx.dev(); + let bar = ctx.bar; + // Run FWSEC-SB to reset the GSP falcon to its pre-libos state. // Log errors but keep going if it fails. let fwsec_sb_res = self .fwsec_sb - .run(dev, bar, gsp_falcon) + .run(dev, bar, ctx.gsp_falcon) .inspect_err(|e| dev_err!(dev, "FWSEC-SB failed to run: {:?}\n", e)); // Remove WPR2 region if set. @@ -142,12 +139,14 @@ fn run( return Ok(()); } - sec2_falcon.reset()?; - sec2_falcon.load(&self.booter_unloader)?; + ctx.sec2_falcon.reset()?; + ctx.sec2_falcon.load(&self.booter_unloader)?; // Sentinel value to confirm that Booter Unloader has run. const MAILBOX_SENTINEL: u32 = 0xff; - let (mbox0, _) = sec2_falcon.boot(Some(MAILBOX_SENTINEL), Some(MAILBOX_SENTINEL))?; + let (mbox0, _) = ctx + .sec2_falcon + .boot(Some(MAILBOX_SENTINEL), Some(MAILBOX_SENTINEL))?; if mbox0 != 0 { dev_err!(dev, "Booter Unloader returned error 0x{:x}\n", mbox0); return Err(EINVAL); @@ -293,7 +292,7 @@ fn boot( // Run the unload bundle to try and recover the GSP if an error occurs. let unload_guard = ScopeGuard::new_with_data(unload_bundle, |unload_bundle| { if let Some(unload_bundle) = unload_bundle { - let _ = unload_bundle.0.run(dev, bar, gsp_falcon, sec2_falcon); + let _ = unload_bundle.0.run(ctx); } }); From 92faa16f341dd2bc43eaab4f32dc8511ea70f3d3 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Thu, 9 Jul 2026 15:52:58 +0900 Subject: [PATCH 088/137] gpu: nova-core: gsp: centralize missing unload bundle warnings The warning emitted when the unload bundle cannot be constructed is valid regardless of the boot method, but it was local to `Tu102`. Move it to `Gsp::boot` so it applies to all boot methods. Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260709-nova-bootcontext-v6-5-520cbf8b9b50@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/boot.rs | 10 +++++++++- drivers/gpu/nova-core/gsp/hal/tu102.rs | 9 +-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index a23219a79355..75488a8e3c0b 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -56,7 +56,15 @@ pub(crate) fn boot( let wpr_meta = Coherent::init(dev, GFP_KERNEL, GspFwWprMeta::new(&gsp_fw, &fb_layout))?; // Perform the chipset-specific boot sequence, and retrieve the unload bundle. - let unload_bundle = hal.boot(&self, &ctx, &fb_layout, &wpr_meta)?; + let unload_bundle = hal.boot(&self, &ctx, &fb_layout, &wpr_meta)?.or_else(|| { + dev_warn!(dev, "The GSP won't be able to unload properly on unbind.\n"); + dev_warn!( + dev, + "The GPU will need to be reset before the driver can bind again.\n" + ); + + None + }); let unload_guard = ScopeGuard::new_with_data((ctx, unload_bundle), |(ctx, unload_bundle)| { diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index c0956fb1c9cf..ef465b99af05 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -278,14 +278,7 @@ fn boot( // If the unload bundle creation fails, the GPU will need to be reset before the driver can // be probed again. let unload_bundle = Sec2UnloadBundle::build(dev, chipset, &bios, gsp_falcon, sec2_falcon) - .inspect_err(|e| { - dev_warn!(dev, "Failed to prepare unload firmware: {:?}\n", e); - dev_warn!(dev, "The GSP won't be able to unload properly on unbind.\n"); - dev_warn!( - dev, - "The GPU will need to be reset before the driver can bind again.\n" - ); - }) + .inspect_err(|e| dev_warn!(dev, "Failed to prepare unload firmware: {:?}\n", e)) .ok() .map(crate::gsp::UnloadBundle); From 436f3c4ac4072209bdd52e8762d4aaf014e073be Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Thu, 9 Jul 2026 15:52:59 +0900 Subject: [PATCH 089/137] gpu: nova-core: gsp: fold TU102 unload bundle construction into HAL method The construction of the unload bundle is currently a bit convoluted and could be done in one function instead of two. Additionally, turn that function into a method of `Tu102`. A following patch will turn the "use FWSEC bootloader" property into a flag of the TU102 HAL itself, and making this a method will allow the code to access it instead of querying `Chipset`. Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260709-nova-bootcontext-v6-6-520cbf8b9b50@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/hal/tu102.rs | 85 ++++++++++++-------------- 1 file changed, 39 insertions(+), 46 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index ef465b99af05..c8c7a4f45809 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -56,22 +56,6 @@ enum FwsecUnloadFirmware { } impl FwsecUnloadFirmware { - /// Loads the FWSEC SB firmware, as well as its bootloader if `chipset` requires it. - fn new( - dev: &device::Device, - chipset: Chipset, - bios: &Vbios, - gsp_falcon: &Falcon<'_, GspEngine>, - ) -> Result { - let fwsec_sb = FwsecFirmware::new(dev, gsp_falcon, bios, FwsecCommand::Sb)?; - - Ok(if chipset.needs_fwsec_bootloader() { - Self::WithBl(FwsecFirmwareWithBl::new(fwsec_sb, dev, chipset)?) - } else { - Self::WithoutBl(fwsec_sb) - }) - } - /// Runs the FWSEC SB firmware. fn run( &self, @@ -93,33 +77,6 @@ struct Sec2UnloadBundle { booter_unloader: BooterFirmware, } -impl Sec2UnloadBundle { - /// Load and prepare the resources required to properly reset the GSP after it has been stopped. - fn build( - dev: &device::Device, - chipset: Chipset, - bios: &Vbios, - gsp_falcon: &Falcon<'_, GspEngine>, - sec2_falcon: &Falcon<'_, Sec2>, - ) -> Result> { - KBox::new( - Self { - fwsec_sb: FwsecUnloadFirmware::new(dev, chipset, bios, gsp_falcon)?, - booter_unloader: BooterFirmware::new( - dev, - BooterKind::Unloader, - chipset, - FIRMWARE_VERSION, - sec2_falcon, - )?, - }, - GFP_KERNEL, - ) - .map(|b| b as KBox) - .map_err(Into::into) - } -} - impl UnloadBundle for Sec2UnloadBundle { fn run(&self, ctx: &GspBootContext<'_>) -> Result { let dev = ctx.dev(); @@ -257,6 +214,42 @@ fn run_fwsec_frts( struct Tu102; +impl Tu102 { + /// Load and prepare the resources required to properly reset the GSP after it has been stopped. + fn build_unload_bundle( + &self, + dev: &device::Device, + chipset: Chipset, + bios: &Vbios, + gsp_falcon: &Falcon<'_, GspEngine>, + sec2_falcon: &Falcon<'_, Sec2>, + ) -> Result { + // Load the FWSEC SB firmware, as well as its bootloader if required. + let fwsec_sb = FwsecFirmware::new(dev, gsp_falcon, bios, FwsecCommand::Sb)?; + let fwsec_sb = if chipset.needs_fwsec_bootloader() { + FwsecUnloadFirmware::WithBl(FwsecFirmwareWithBl::new(fwsec_sb, dev, chipset)?) + } else { + FwsecUnloadFirmware::WithoutBl(fwsec_sb) + }; + + KBox::new( + Sec2UnloadBundle { + fwsec_sb, + booter_unloader: BooterFirmware::new( + dev, + BooterKind::Unloader, + chipset, + FIRMWARE_VERSION, + sec2_falcon, + )?, + }, + GFP_KERNEL, + ) + .map(|b| crate::gsp::UnloadBundle(b)) + .map_err(Into::into) + } +} + impl GspHal for Tu102 { fn boot( &self, @@ -277,10 +270,10 @@ fn boot( // // If the unload bundle creation fails, the GPU will need to be reset before the driver can // be probed again. - let unload_bundle = Sec2UnloadBundle::build(dev, chipset, &bios, gsp_falcon, sec2_falcon) + let unload_bundle = self + .build_unload_bundle(dev, chipset, &bios, gsp_falcon, sec2_falcon) .inspect_err(|e| dev_warn!(dev, "Failed to prepare unload firmware: {:?}\n", e)) - .ok() - .map(crate::gsp::UnloadBundle); + .ok(); // Run the unload bundle to try and recover the GSP if an error occurs. let unload_guard = ScopeGuard::new_with_data(unload_bundle, |unload_bundle| { From 7567e62b9c609baa6bd6e332f7d3f0cf28c7ef43 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Thu, 9 Jul 2026 15:53:00 +0900 Subject: [PATCH 090/137] gpu: nova-core: gsp: turn FWSEC execution into HAL method Turn the `run_fwsec_frts` function into a method of `Tu102`. A following patch will turn the "use FWSEC bootloader" property into a flag of the TU102 HAL itself, and making this a method will allow the code to access it instead of querying `Chipset`. Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260709-nova-bootcontext-v6-7-520cbf8b9b50@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/hal/tu102.rs | 173 +++++++++++++------------ 1 file changed, 87 insertions(+), 86 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index c8c7a4f45809..066f97c9908a 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -127,94 +127,95 @@ fn run(&self, ctx: &GspBootContext<'_>) -> Result { } } -/// Helper function to load and run the FWSEC-FRTS firmware and confirm that it has properly -/// created the WPR2 region. -fn run_fwsec_frts( - dev: &device::Device, - chipset: Chipset, - falcon: &Falcon<'_, GspEngine>, - bar: Bar0<'_>, - bios: &Vbios, - fb_layout: &FbLayout, -) -> Result { - // Check that the WPR2 region does not already exist - if it does, we cannot run - // FWSEC-FRTS until the GPU is reset. - if bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound() != 0 { - dev_err!( - dev, - "WPR2 region already exists - GPU needs to be reset to proceed\n" - ); - return Err(EBUSY); - } - - // FWSEC-FRTS will create the WPR2 region. - let fwsec_frts = FwsecFirmware::new( - dev, - falcon, - bios, - FwsecCommand::Frts { - frts_addr: fb_layout.frts.start, - frts_size: fb_layout.frts.len(), - }, - )?; - - if chipset.needs_fwsec_bootloader() { - let fwsec_frts_bl = FwsecFirmwareWithBl::new(fwsec_frts, dev, chipset)?; - // Load and run the bootloader, which will load FWSEC-FRTS and run it. - fwsec_frts_bl.run(dev, falcon, bar)?; - } else { - // Load and run FWSEC-FRTS directly. - fwsec_frts.run(dev, falcon)?; - } - - // SCRATCH_E contains the error code for FWSEC-FRTS. - let frts_status = bar - .read(regs::NV_PBUS_SW_SCRATCH_0E_FRTS_ERR) - .frts_err_code(); - if frts_status != 0 { - dev_err!( - dev, - "FWSEC-FRTS returned with error code {:#x}\n", - frts_status - ); - - return Err(EIO); - } - - // Check that the WPR2 region has been created as we requested. - let (wpr2_lo, wpr2_hi) = ( - bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO).lower_bound(), - bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound(), - ); - - match (wpr2_lo, wpr2_hi) { - (_, 0) => { - dev_err!(dev, "WPR2 region not created after running FWSEC-FRTS\n"); - - Err(EIO) - } - (wpr2_lo, _) if wpr2_lo != fb_layout.frts.start => { - dev_err!( - dev, - "WPR2 region created at unexpected address {:#x}; expected {:#x}\n", - wpr2_lo, - fb_layout.frts.start, - ); - - Err(EIO) - } - (wpr2_lo, wpr2_hi) => { - dev_dbg!(dev, "WPR2: {:#x}-{:#x}\n", wpr2_lo, wpr2_hi); - dev_dbg!(dev, "GPU instance built\n"); - - Ok(()) - } - } -} - struct Tu102; impl Tu102 { + /// Helper method to load and run the FWSEC-FRTS firmware and confirm that it has properly + /// created the WPR2 region. + fn run_fwsec_frts( + &self, + dev: &device::Device, + chipset: Chipset, + falcon: &Falcon<'_, GspEngine>, + bar: Bar0<'_>, + bios: &Vbios, + fb_layout: &FbLayout, + ) -> Result { + // Check that the WPR2 region does not already exist - if it does, we cannot run + // FWSEC-FRTS until the GPU is reset. + if bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound() != 0 { + dev_err!( + dev, + "WPR2 region already exists - GPU needs to be reset to proceed\n" + ); + return Err(EBUSY); + } + + // FWSEC-FRTS will create the WPR2 region. + let fwsec_frts = FwsecFirmware::new( + dev, + falcon, + bios, + FwsecCommand::Frts { + frts_addr: fb_layout.frts.start, + frts_size: fb_layout.frts.len(), + }, + )?; + + if chipset.needs_fwsec_bootloader() { + let fwsec_frts_bl = FwsecFirmwareWithBl::new(fwsec_frts, dev, chipset)?; + // Load and run the bootloader, which will load FWSEC-FRTS and run it. + fwsec_frts_bl.run(dev, falcon, bar)?; + } else { + // Load and run FWSEC-FRTS directly. + fwsec_frts.run(dev, falcon)?; + } + + // SCRATCH_E contains the error code for FWSEC-FRTS. + let frts_status = bar + .read(regs::NV_PBUS_SW_SCRATCH_0E_FRTS_ERR) + .frts_err_code(); + if frts_status != 0 { + dev_err!( + dev, + "FWSEC-FRTS returned with error code {:#x}\n", + frts_status + ); + + return Err(EIO); + } + + // Check that the WPR2 region has been created as we requested. + let (wpr2_lo, wpr2_hi) = ( + bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO).lower_bound(), + bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound(), + ); + + match (wpr2_lo, wpr2_hi) { + (_, 0) => { + dev_err!(dev, "WPR2 region not created after running FWSEC-FRTS\n"); + + Err(EIO) + } + (wpr2_lo, _) if wpr2_lo != fb_layout.frts.start => { + dev_err!( + dev, + "WPR2 region created at unexpected address {:#x}; expected {:#x}\n", + wpr2_lo, + fb_layout.frts.start, + ); + + Err(EIO) + } + (wpr2_lo, wpr2_hi) => { + dev_dbg!(dev, "WPR2: {:#x}-{:#x}\n", wpr2_lo, wpr2_hi); + dev_dbg!(dev, "GPU instance built\n"); + + Ok(()) + } + } + } + /// Load and prepare the resources required to properly reset the GSP after it has been stopped. fn build_unload_bundle( &self, @@ -284,7 +285,7 @@ fn boot( // FWSEC-FRTS is not executed on chips where the FRTS region size is 0 (e.g. GA100). if !fb_layout.frts.is_empty() { - run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, fb_layout)?; + self.run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, fb_layout)?; } gsp_falcon.reset()?; From bb2d007399d89bd900ec398bf5e29093f982ddcc Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Thu, 9 Jul 2026 15:53:01 +0900 Subject: [PATCH 091/137] gpu: nova-core: gsp: make use of FWSEC bootloader a property of the TU102 HAL By being in the TU102 HAL, we already know that the GSP boot method is the SEC2 Booter, so the only variable is whether the FWSEC bootloader is used or not. Since `Chipset` also includes the variants that boot FSP, querying it for that information introduces a potential code path (a chipset that boots via FSP) that the current code doesn't handle. Turn the use of the FWSEC bootloader into a property of the `Tu102` HAL, and give GA102+ chipsets their own instance with that property set to `false`. This removes the invalid code path and the only use of `Chipset` is now to load the correct firmware files. This also removes some uses of the `Chipset::needs_fwsec_bootloader` method and prepares the ground for removing it. Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260709-nova-bootcontext-v6-8-520cbf8b9b50@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/hal.rs | 5 ++++- drivers/gpu/nova-core/gsp/hal/ga102.rs | 14 ++++++++++++++ drivers/gpu/nova-core/gsp/hal/tu102.rs | 15 +++++++++++---- 3 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 drivers/gpu/nova-core/gsp/hal/ga102.rs diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs index 849ca224085b..eddb4e8bf510 100644 --- a/drivers/gpu/nova-core/gsp/hal.rs +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +mod ga102; mod gh100; mod tu102; @@ -59,7 +60,9 @@ fn post_boot(&self, _gsp: &Gsp, _ctx: &GspBootContext<'_>, _gsp_fw: &GspFirmware /// Returns the GSP HAL to be used for `chipset`. pub(super) fn gsp_hal(chipset: Chipset) -> &'static dyn GspHal { match chipset.arch() { - Architecture::Turing | Architecture::Ampere | Architecture::Ada => tu102::TU102_HAL, + Architecture::Turing => tu102::TU102_HAL, + Architecture::Ampere if matches!(chipset, Chipset::GA100) => tu102::TU102_HAL, + Architecture::Ampere | Architecture::Ada => ga102::GA102_HAL, Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { gh100::GH100_HAL } diff --git a/drivers/gpu/nova-core/gsp/hal/ga102.rs b/drivers/gpu/nova-core/gsp/hal/ga102.rs new file mode 100644 index 000000000000..ceb3eb39d138 --- /dev/null +++ b/drivers/gpu/nova-core/gsp/hal/ga102.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use crate::gsp::hal::{ + tu102::Tu102, + GspHal, // +}; + +/// The GA102 HAL is like the TU102 one, except it doesn't use the bootloader. +const GA102: Tu102 = Tu102 { + needs_fwsec_bootloader: false, +}; + +pub(super) const GA102_HAL: &dyn GspHal = &GA102; diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index 066f97c9908a..8480e2eb456f 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -127,7 +127,10 @@ fn run(&self, ctx: &GspBootContext<'_>) -> Result { } } -struct Tu102; +pub(super) struct Tu102 { + /// If `true`, then the FWSEC-FRTS bootloader will be used to load the actual firmware. + pub(super) needs_fwsec_bootloader: bool, +} impl Tu102 { /// Helper method to load and run the FWSEC-FRTS firmware and confirm that it has properly @@ -162,7 +165,7 @@ fn run_fwsec_frts( }, )?; - if chipset.needs_fwsec_bootloader() { + if self.needs_fwsec_bootloader { let fwsec_frts_bl = FwsecFirmwareWithBl::new(fwsec_frts, dev, chipset)?; // Load and run the bootloader, which will load FWSEC-FRTS and run it. fwsec_frts_bl.run(dev, falcon, bar)?; @@ -227,7 +230,7 @@ fn build_unload_bundle( ) -> Result { // Load the FWSEC SB firmware, as well as its bootloader if required. let fwsec_sb = FwsecFirmware::new(dev, gsp_falcon, bios, FwsecCommand::Sb)?; - let fwsec_sb = if chipset.needs_fwsec_bootloader() { + let fwsec_sb = if self.needs_fwsec_bootloader { FwsecUnloadFirmware::WithBl(FwsecFirmwareWithBl::new(fwsec_sb, dev, chipset)?) } else { FwsecUnloadFirmware::WithoutBl(fwsec_sb) @@ -323,5 +326,9 @@ fn post_boot(&self, gsp: &Gsp, ctx: &GspBootContext<'_>, gsp_fw: &GspFirmware) - } } -const TU102: Tu102 = Tu102; +/// The TU102 HAL requires the use of the FWSEC bootloader. +const TU102: Tu102 = Tu102 { + needs_fwsec_bootloader: true, +}; + pub(super) const TU102_HAL: &dyn GspHal = &TU102; From 691f6bc8915fec0d0b26871fc63d3b08a1efa0b7 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Thu, 9 Jul 2026 15:53:02 +0900 Subject: [PATCH 092/137] gpu: nova-core: move GSP firmware files decision to GSP HAL The files to give the `ModInfoBuilder` depend on the chipset; this is HAL information, so move it there. Doing so lets us remove the `uses_fsp` and `needs_fwsec_bootloader` ad-hoc methods of `Chipset`. Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260709-nova-bootcontext-v6-9-520cbf8b9b50@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/firmware.rs | 25 +++++++++++-------------- drivers/gpu/nova-core/firmware/fwsec.rs | 5 ++--- drivers/gpu/nova-core/gpu.rs | 16 ---------------- drivers/gpu/nova-core/gsp.rs | 1 + drivers/gpu/nova-core/gsp/hal.rs | 19 +++++++++++++++++++ 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index a94820a3b335..20eff987c5d6 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -21,6 +21,7 @@ FalconFirmware, // }, gpu, + gsp::boot_firmware_files, num::{ FromSafeCast, IntoSafeCast, // @@ -419,24 +420,20 @@ const fn make_entry_file(self, chipset: &str, fw: &str) -> Self { const fn make_entry_chipset(self, chipset: gpu::Chipset) -> Self { let name = chipset.name(); - let this = self + // GSP firmware files are always present. + let mut this = self .make_entry_file(name, "bootloader") .make_entry_file(name, "gsp"); - // FSP-based chipsets (Hopper, Blackwell and later) boot the GSP via the FMC image loaded by - // FSP. Older chipsets use the SEC2 booter instead. - let this = if chipset.uses_fsp() { - this.make_entry_file(name, "fmc") - } else { - this.make_entry_file(name, "booter_load") - .make_entry_file(name, "booter_unload") - }; - - if chipset.needs_fwsec_bootloader() { - this.make_entry_file(name, "gen_bootloader") - } else { - this + // Add the firmware files specific to the GSP boot method of `chipset`. + let boot_files = boot_firmware_files(chipset); + let mut i = 0; + while i < boot_files.len() { + this = this.make_entry_file(name, boot_files[i]); + i += 1; } + + this } pub(crate) const fn create( diff --git a/drivers/gpu/nova-core/firmware/fwsec.rs b/drivers/gpu/nova-core/firmware/fwsec.rs index 95e0dd77746b..7a931f22f629 100644 --- a/drivers/gpu/nova-core/firmware/fwsec.rs +++ b/drivers/gpu/nova-core/firmware/fwsec.rs @@ -385,9 +385,8 @@ pub(crate) fn new( /// Loads the FWSEC firmware into `falcon` and execute it. /// - /// This must only be called on chipsets that do not need the FWSEC bootloader (i.e., where - /// [`Chipset::needs_fwsec_bootloader()`](crate::gpu::Chipset::needs_fwsec_bootloader) returns - /// `false`). On chipsets that do, use [`bootloader::FwsecFirmwareWithBl`] instead. + /// This must only be called on chipsets that do not need the FWSEC bootloader. On chipsets + /// where the bootloader is required, use [`bootloader::FwsecFirmwareWithBl`] instead. pub(crate) fn run(&self, dev: &Device, falcon: &Falcon<'_, Gsp>) -> Result<()> { // Reset falcon, load the firmware, and run it. falcon diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 32bfa0be2357..c04706b60ba8 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -133,22 +133,6 @@ pub(crate) const fn arch(self) -> Architecture { } } - /// Returns `true` if this chipset requires the PIO-loaded bootloader in order to boot FWSEC. - /// - /// This includes all chipsets < GA102. - pub(crate) const fn needs_fwsec_bootloader(self) -> bool { - matches!(self.arch(), Architecture::Turing) || matches!(self, Self::GA100) - } - - /// Returns `true` if this chipset boots via FSP (Hopper and later), which requires the FMC - /// firmware image. - pub(crate) const fn uses_fsp(self) -> bool { - matches!( - self.arch(), - Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x - ) - } - /// Returns the address range of the PCI config mirror space. pub(crate) fn pci_config_mirror_range(self) -> Range { hal::gpu_hal(self).pci_config_mirror_range() diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index 9e8491dfbebe..c3537e071933 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -32,6 +32,7 @@ GspFwWprMeta, LibosParams, // }; +pub(crate) use hal::boot_firmware_files; use crate::{ driver::Bar0, diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs index eddb4e8bf510..9da078dd1059 100644 --- a/drivers/gpu/nova-core/gsp/hal.rs +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -57,6 +57,25 @@ fn post_boot(&self, _gsp: &Gsp, _ctx: &GspBootContext<'_>, _gsp_fw: &GspFirmware } } +/// Returns the names of the firmware files required to boot the GSP of `chipset`, in addition to +/// the "bootloader" and "gsp" images required by all chipsets. +pub(crate) const fn boot_firmware_files(chipset: Chipset) -> &'static [&'static str] { + match chipset.arch() { + // Turing chipsets boot the GSP via the SEC2 Booter, and require the FWSEC bootloader. + Architecture::Turing => &["booter_load", "booter_unload", "gen_bootloader"], + // GA100 also requires the FWSEC bootloader. + Architecture::Ampere if matches!(chipset, Chipset::GA100) => { + &["booter_load", "booter_unload", "gen_bootloader"] + } + // Other Ampere chipsets, as well as Ada chipsets, run FWSEC directly. + Architecture::Ampere | Architecture::Ada => &["booter_load", "booter_unload"], + // Hopper and later chipsets boot the GSP via the FMC image loaded by FSP. + Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { + &["fmc"] + } + } +} + /// Returns the GSP HAL to be used for `chipset`. pub(super) fn gsp_hal(chipset: Chipset) -> &'static dyn GspHal { match chipset.arch() { From f1aab4b1ab8298b42b89e31990d8f0e52b6104a5 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Thu, 9 Jul 2026 15:53:03 +0900 Subject: [PATCH 093/137] gpu: nova-core: avoid repeated calls to pci::Device::as_ref Add a local `Device` reference created from the `pci::Device` in the `Gpu` constructor to avoid repeatedly calling `as_ref`. Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260709-nova-bootcontext-v6-10-520cbf8b9b50@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gpu.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index c04706b60ba8..fc90069bc2fe 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -317,9 +317,11 @@ pub(crate) fn new( pdev: &'gpu pci::Device>, bar: Bar0<'gpu>, ) -> impl PinInit + 'gpu { + let dev = pdev.as_ref(); + try_pin_init!(Self { - spec: Spec::new(pdev.as_ref(), bar).inspect(|spec| { - dev_info!(pdev,"NVIDIA ({})\n", spec); + spec: Spec::new(dev, bar).inspect(|spec| { + dev_info!(dev,"NVIDIA ({})\n", spec); })?, // We must wait for GFW_BOOT completion before doing any significant setup on the GPU. @@ -332,11 +334,11 @@ pub(crate) fn new( unsafe { pdev.dma_set_mask_and_coherent(dma_mask)? }; hal.wait_gfw_boot_completion(bar) - .inspect_err(|_| dev_err!(pdev, "GFW boot did not complete\n"))?; + .inspect_err(|_| dev_err!(dev, "GFW boot did not complete\n"))?; }, // Initialize this early because `gsp_resources` depends on it. - sysmem_flush: SysmemFlush::register(pdev.as_ref(), bar, spec.chipset)?, + sysmem_flush: SysmemFlush::register(dev, bar, spec.chipset)?, gsp_resources <- try_pin_init!(GspResources { device: pdev, @@ -346,13 +348,13 @@ pub(crate) fn new( bar, gsp_falcon: Falcon::new( - pdev.as_ref(), + dev, spec.chipset, bar ) .inspect(|falcon| falcon.clear_swgen0_intr())?, - sec2_falcon: Falcon::new(pdev.as_ref(), spec.chipset, bar)?, + sec2_falcon: Falcon::new(dev, spec.chipset, bar)?, gsp <- Gsp::new(pdev), @@ -372,18 +374,18 @@ pub(crate) fn new( // Obtain and display basic GPU information. let info = gsp_resources.gsp.get_static_info(bar)?; match info.gpu_name() { - Ok(name) => dev_info!(pdev, "GPU name: {}\n", name), - Err(e) => dev_warn!(pdev, "GPU name unavailable: {:?}\n", e), + Ok(name) => dev_info!(dev, "GPU name: {}\n", name), + Err(e) => dev_warn!(dev, "GPU name unavailable: {:?}\n", e), } if !info.usable_fb_regions.is_empty() { - dev_dbg!(pdev, "Usable FB regions:\n"); + dev_dbg!(dev, "Usable FB regions:\n"); for region in &info.usable_fb_regions { - dev_dbg!(pdev, " - {:#x?}\n", region); + dev_dbg!(dev, " - {:#x?}\n", region); } dev_dbg!( - pdev, + dev, "Total usable VRAM: {} MiB\n", info.usable_fb_regions.iter().fold(0u64, |res, region| res .saturating_add(region.end - region.start)) From 43f890d85c03ced99b5ecad5877fb654f0a3f4c4 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Thu, 9 Jul 2026 15:53:04 +0900 Subject: [PATCH 094/137] gpu: nova-core: gsp: pass GspBootContext mutably We want to move the `Fsp` instance into `Gpu`, which will require passing it as a mutable reference in `GspBootContext`, since `Fsp::boot_fmc` is a mutable method. In order to use the mutable references it contains, `GspBootContext` must also be mutable. We will also follow up by making some methods of the `Falcon`s mutable, which also requires passing them as mutable references. Thus, make the `GspBootContext` passed to `Gsp::boot` and `Gsp::unload` mutable, and pass mutable references to it to the GSP boot HAL methods. Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260709-nova-bootcontext-v6-11-520cbf8b9b50@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/boot.rs | 28 ++++++++++++++------------ drivers/gpu/nova-core/gsp/hal.rs | 11 +++++++--- drivers/gpu/nova-core/gsp/hal/gh100.rs | 4 ++-- drivers/gpu/nova-core/gsp/hal/tu102.rs | 6 +++--- 4 files changed, 28 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 75488a8e3c0b..17e589dbd483 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -39,7 +39,7 @@ impl super::Gsp { /// [`Self::unload`]) returned. pub(crate) fn boot( self: Pin<&mut Self>, - ctx: super::GspBootContext<'_>, + mut ctx: super::GspBootContext<'_>, ) -> Result> { let pdev = ctx.pdev; let bar = ctx.bar; @@ -56,21 +56,23 @@ pub(crate) fn boot( let wpr_meta = Coherent::init(dev, GFP_KERNEL, GspFwWprMeta::new(&gsp_fw, &fb_layout))?; // Perform the chipset-specific boot sequence, and retrieve the unload bundle. - let unload_bundle = hal.boot(&self, &ctx, &fb_layout, &wpr_meta)?.or_else(|| { - dev_warn!(dev, "The GSP won't be able to unload properly on unbind.\n"); - dev_warn!( - dev, - "The GPU will need to be reset before the driver can bind again.\n" - ); + let unload_bundle = hal + .boot(&self, &mut ctx, &fb_layout, &wpr_meta)? + .or_else(|| { + dev_warn!(dev, "The GSP won't be able to unload properly on unbind.\n"); + dev_warn!( + dev, + "The GPU will need to be reset before the driver can bind again.\n" + ); - None - }); + None + }); - let unload_guard = + let mut unload_guard = ScopeGuard::new_with_data((ctx, unload_bundle), |(ctx, unload_bundle)| { let _ = self.unload(ctx, unload_bundle); }); - let ctx = &unload_guard.0; + let ctx = &mut unload_guard.0; gsp_falcon.write_os_version(gsp_fw.bootloader.app_version); @@ -123,7 +125,7 @@ fn shutdown_gsp( /// This stops all activity on the GSP. pub(crate) fn unload( &self, - ctx: super::GspBootContext<'_>, + mut ctx: super::GspBootContext<'_>, unload_bundle: Option, ) -> Result { let dev = ctx.dev(); @@ -142,7 +144,7 @@ pub(crate) fn unload( res = res.and( unload_bundle .0 - .run(&ctx) + .run(&mut ctx) .inspect_err(|e| dev_err!(dev, "Unload bundle failed: {:?}\n", e)), ); } else { diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs index 9da078dd1059..7ebdeafc1432 100644 --- a/drivers/gpu/nova-core/gsp/hal.rs +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -31,7 +31,7 @@ /// required for unloading is prepared at load time, and stored here until it needs to be run. pub(super) trait UnloadBundle: Send { /// Performs the steps required to properly reset the GSP after it has been stopped. - fn run(&self, ctx: &GspBootContext<'_>) -> Result; + fn run(&self, ctx: &mut GspBootContext<'_>) -> Result; } /// Trait implemented by GSP HALs. @@ -43,7 +43,7 @@ pub(super) trait GspHal: Send { fn boot( &self, gsp: &Gsp, - ctx: &GspBootContext<'_>, + ctx: &mut GspBootContext<'_>, fb_layout: &FbLayout, wpr_meta: &Coherent, ) -> Result>; @@ -52,7 +52,12 @@ fn boot( /// /// This method is called by the GSP boot code after the GSP is confirmed to be running, and /// after the initialization commands have been pushed onto its queue. - fn post_boot(&self, _gsp: &Gsp, _ctx: &GspBootContext<'_>, _gsp_fw: &GspFirmware) -> Result { + fn post_boot( + &self, + _gsp: &Gsp, + _ctx: &mut GspBootContext<'_>, + _gsp_fw: &GspFirmware, + ) -> Result { Ok(()) } } diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index b991f160abcc..067bb01903e5 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -115,7 +115,7 @@ fn wait_for_gsp_lockdown_release( struct FspUnloadBundle; impl UnloadBundle for FspUnloadBundle { - fn run(&self, ctx: &GspBootContext<'_>) -> Result { + fn run(&self, ctx: &mut GspBootContext<'_>) -> Result { // GSP falcon does most of the work of resetting, so just wait for it to finish. read_poll_timeout( || Ok(ctx.gsp_falcon.is_riscv_active()), @@ -138,7 +138,7 @@ impl GspHal for Gh100 { fn boot( &self, gsp: &Gsp, - ctx: &GspBootContext<'_>, + ctx: &mut GspBootContext<'_>, fb_layout: &FbLayout, wpr_meta: &Coherent, ) -> Result> { diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index 8480e2eb456f..4e2f48c27368 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -78,7 +78,7 @@ struct Sec2UnloadBundle { } impl UnloadBundle for Sec2UnloadBundle { - fn run(&self, ctx: &GspBootContext<'_>) -> Result { + fn run(&self, ctx: &mut GspBootContext<'_>) -> Result { let dev = ctx.dev(); let bar = ctx.bar; @@ -258,7 +258,7 @@ impl GspHal for Tu102 { fn boot( &self, gsp: &Gsp, - ctx: &GspBootContext<'_>, + ctx: &mut GspBootContext<'_>, fb_layout: &FbLayout, wpr_meta: &Coherent, ) -> Result> { @@ -314,7 +314,7 @@ fn boot( Ok(unload_guard.dismiss()) } - fn post_boot(&self, gsp: &Gsp, ctx: &GspBootContext<'_>, gsp_fw: &GspFirmware) -> Result { + fn post_boot(&self, gsp: &Gsp, ctx: &mut GspBootContext<'_>, gsp_fw: &GspFirmware) -> Result { GspSequencer::run( &gsp.cmdq, ctx, From 320608e5bc887b03a67c26396bc698f556906db0 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Thu, 9 Jul 2026 15:53:05 +0900 Subject: [PATCH 095/137] gpu: nova-core: gsp: separate context and GPU lifetimes in GspBootContext `Falcon` instances retain references tied to the lifetime of the bound GPU. `GspBootContext` currently uses that same lifetime for its own borrows of the `Falcon` instances and other references. But these lifetimes are independent: the references captured by a `Falcon` remain valid for the GPU lifetime, while the context only borrows the `Falcon` for the duration of a boot or unload operation. This distinction is hidden for shared references by covariance, but cannot be ignored anymore if the context carries mutable references to GPU subdevices, as will happen for the `Fsp` and the `Falcon`s. Thus, give `GspBootContext` separate lifetimes for its subdevice borrows and the GPU resources captured by those subdevices, and update its users accordingly. Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260709-nova-bootcontext-v6-12-520cbf8b9b50@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp.rs | 19 ++++++++++++------- drivers/gpu/nova-core/gsp/boot.rs | 4 ++-- drivers/gpu/nova-core/gsp/hal.rs | 6 +++--- drivers/gpu/nova-core/gsp/hal/gh100.rs | 4 ++-- drivers/gpu/nova-core/gsp/hal/tu102.rs | 11 ++++++++--- drivers/gpu/nova-core/gsp/sequencer.rs | 2 +- 6 files changed, 28 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index c3537e071933..f4262d5eaf18 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -56,16 +56,21 @@ pub(crate) const GSP_PAGE_SIZE: usize = 1 << GSP_PAGE_SHIFT; /// Common context for the GSP boot process. -pub(crate) struct GspBootContext<'a> { - pub(crate) pdev: &'a pci::Device, - pub(crate) bar: Bar0<'a>, +/// +/// It carries two distinct lifetimes: +/// +/// - `'gpu` is the lifetime of the bound GPU device, as captured by the GPU subdevices. +/// - `'ctx` is a shorter lifetime during which this context borrows those subdevices. +pub(crate) struct GspBootContext<'ctx, 'gpu> { + pub(crate) pdev: &'gpu pci::Device, + pub(crate) bar: Bar0<'gpu>, pub(crate) chipset: Chipset, - pub(crate) gsp_falcon: &'a Falcon<'a, GspFalcon>, - pub(crate) sec2_falcon: &'a Falcon<'a, Sec2Falcon>, + pub(crate) gsp_falcon: &'ctx Falcon<'gpu, GspFalcon>, + pub(crate) sec2_falcon: &'ctx Falcon<'gpu, Sec2Falcon>, } -impl<'a> GspBootContext<'a> { - pub(crate) fn dev(&self) -> &'a device::Device { +impl<'ctx, 'gpu> GspBootContext<'ctx, 'gpu> { + pub(crate) fn dev(&self) -> &'gpu device::Device { self.pdev.as_ref() } } diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 17e589dbd483..6b7d00205000 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -39,7 +39,7 @@ impl super::Gsp { /// [`Self::unload`]) returned. pub(crate) fn boot( self: Pin<&mut Self>, - mut ctx: super::GspBootContext<'_>, + mut ctx: super::GspBootContext<'_, '_>, ) -> Result> { let pdev = ctx.pdev; let bar = ctx.bar; @@ -125,7 +125,7 @@ fn shutdown_gsp( /// This stops all activity on the GSP. pub(crate) fn unload( &self, - mut ctx: super::GspBootContext<'_>, + mut ctx: super::GspBootContext<'_, '_>, unload_bundle: Option, ) -> Result { let dev = ctx.dev(); diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs index 7ebdeafc1432..34b4bb82a999 100644 --- a/drivers/gpu/nova-core/gsp/hal.rs +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -31,7 +31,7 @@ /// required for unloading is prepared at load time, and stored here until it needs to be run. pub(super) trait UnloadBundle: Send { /// Performs the steps required to properly reset the GSP after it has been stopped. - fn run(&self, ctx: &mut GspBootContext<'_>) -> Result; + fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result; } /// Trait implemented by GSP HALs. @@ -43,7 +43,7 @@ pub(super) trait GspHal: Send { fn boot( &self, gsp: &Gsp, - ctx: &mut GspBootContext<'_>, + ctx: &mut GspBootContext<'_, '_>, fb_layout: &FbLayout, wpr_meta: &Coherent, ) -> Result>; @@ -55,7 +55,7 @@ fn boot( fn post_boot( &self, _gsp: &Gsp, - _ctx: &mut GspBootContext<'_>, + _ctx: &mut GspBootContext<'_, '_>, _gsp_fw: &GspFirmware, ) -> Result { Ok(()) diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index 067bb01903e5..d134ad052308 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -115,7 +115,7 @@ fn wait_for_gsp_lockdown_release( struct FspUnloadBundle; impl UnloadBundle for FspUnloadBundle { - fn run(&self, ctx: &mut GspBootContext<'_>) -> Result { + fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result { // GSP falcon does most of the work of resetting, so just wait for it to finish. read_poll_timeout( || Ok(ctx.gsp_falcon.is_riscv_active()), @@ -138,7 +138,7 @@ impl GspHal for Gh100 { fn boot( &self, gsp: &Gsp, - ctx: &mut GspBootContext<'_>, + ctx: &mut GspBootContext<'_, '_>, fb_layout: &FbLayout, wpr_meta: &Coherent, ) -> Result> { diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index 4e2f48c27368..29bb17171f56 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -78,7 +78,7 @@ struct Sec2UnloadBundle { } impl UnloadBundle for Sec2UnloadBundle { - fn run(&self, ctx: &mut GspBootContext<'_>) -> Result { + fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result { let dev = ctx.dev(); let bar = ctx.bar; @@ -258,7 +258,7 @@ impl GspHal for Tu102 { fn boot( &self, gsp: &Gsp, - ctx: &mut GspBootContext<'_>, + ctx: &mut GspBootContext<'_, '_>, fb_layout: &FbLayout, wpr_meta: &Coherent, ) -> Result> { @@ -314,7 +314,12 @@ fn boot( Ok(unload_guard.dismiss()) } - fn post_boot(&self, gsp: &Gsp, ctx: &mut GspBootContext<'_>, gsp_fw: &GspFirmware) -> Result { + fn post_boot( + &self, + gsp: &Gsp, + ctx: &mut GspBootContext<'_, '_>, + gsp_fw: &GspFirmware, + ) -> Result { GspSequencer::run( &gsp.cmdq, ctx, diff --git a/drivers/gpu/nova-core/gsp/sequencer.rs b/drivers/gpu/nova-core/gsp/sequencer.rs index ddce32cc4e30..422a74f9ecbd 100644 --- a/drivers/gpu/nova-core/gsp/sequencer.rs +++ b/drivers/gpu/nova-core/gsp/sequencer.rs @@ -335,7 +335,7 @@ fn next(&mut self) -> Option { impl<'a> GspSequencer<'a> { pub(crate) fn run( cmdq: &Cmdq, - ctx: &'a GspBootContext<'_>, + ctx: &'a GspBootContext<'_, '_>, libos_dma_handle: u64, bootloader_app_version: u32, ) -> Result { From ce2d97f714c8a37cac506a41e52da45b09555b81 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Thu, 9 Jul 2026 15:53:06 +0900 Subject: [PATCH 096/137] gpu: nova-core: store Fsp instance in Gpu The `Fsp` instance was only used in the Hopper+ boot path, and consequently built locally (and immediately dropped) in it. This worked well as a temporary measure, but the FSP is a GPU sub-device, so its lifetime should match the GPU rather than a single boot invocation. It will also be needed in other parts of the driver, for instance vGPU. Thus, create the `Fsp` instance in the `Gpu` constructor and store it there, passing it to the GSP boot as a mutable reference using `GspBootContext`. This makes the `Fsp` available even after the GSP is booted. Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260709-nova-bootcontext-v6-13-520cbf8b9b50@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fsp.rs | 21 +++++++++++++++++++-- drivers/gpu/nova-core/gpu.rs | 9 +++++++++ drivers/gpu/nova-core/gsp.rs | 2 ++ drivers/gpu/nova-core/gsp/hal/gh100.rs | 19 ++++++++----------- 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index 08f4acef09f6..afbd75879d16 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -231,20 +231,37 @@ pub(crate) struct Fsp<'a> { } impl<'a> Fsp<'a> { + /// Attempts to create a `Fsp` instance. + /// + /// This can involve waiting for FSP secure boot completion, but should be instantaneous in + /// practice. + /// + /// If `chipset` doesn't support FSP, `Ok(None)` is returned. + pub(crate) fn try_new( + dev: &'a device::Device, + bar: Bar0<'a>, + chipset: Chipset, + ) -> Result> { + match hal::fsp_hal(chipset) { + None => Ok(None), + Some(hal) => Self::wait_secure_boot(dev, bar, chipset, hal).map(Option::Some), + } + } + /// Waits for FSP secure boot completion, then returns the [`Fsp`] interface. /// /// Polls the thermal scratch register until FSP signals boot completion or the timeout /// elapses. Returning an [`Fsp`] only on success guarantees, at the API level, that the /// interface is not used before secure boot has completed. - pub(crate) fn wait_secure_boot( + fn wait_secure_boot( dev: &'a device::Device, bar: Bar0<'a>, chipset: Chipset, + hal: &'static dyn hal::FspHal, ) -> Result> { /// FSP secure boot completion timeout in milliseconds. const FSP_SECURE_BOOT_TIMEOUT_MS: i64 = 5000; - let hal = hal::fsp_hal(chipset).ok_or(ENOTSUPP)?; let falcon = Falcon::::new(dev, chipset, bar)?; let fsp_fw = FspFirmware::new(dev, chipset, FIRMWARE_VERSION)?; diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index fc90069bc2fe..442c0979f9c6 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -22,6 +22,7 @@ Falcon, // }, fb::SysmemFlush, + fsp::Fsp, gsp::{ self, commands::GetGspStaticInfoReply, @@ -262,6 +263,10 @@ struct GspResources<'gpu> { gsp_falcon: Falcon<'gpu, GspFalcon>, /// SEC2 falcon instance, used for GSP boot up and cleanup. sec2_falcon: Falcon<'gpu, Sec2Falcon>, + /// FSP instance, if on an arch that supports it. + // TODO: use different resource types for each boot method, and make the relevant Gsp methods + // generic against them. + fsp: Option>, /// GSP runtime data. #[pin] gsp: Gsp, @@ -305,6 +310,7 @@ fn drop(self: Pin<&mut Self>) { chipset: this.spec.chipset, gsp_falcon: &*this.gsp_falcon, sec2_falcon: &*this.sec2_falcon, + fsp: this.fsp.as_mut(), }, bundle, ) @@ -356,6 +362,8 @@ pub(crate) fn new( sec2_falcon: Falcon::new(dev, spec.chipset, bar)?, + fsp: Fsp::try_new(dev, bar, spec.chipset)?, + gsp <- Gsp::new(pdev), // This member must be initialized last, so the `UnloadBundle` can never be dropped @@ -367,6 +375,7 @@ pub(crate) fn new( chipset: spec.chipset, gsp_falcon, sec2_falcon, + fsp: fsp.as_mut(), })?, }), diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index f4262d5eaf18..f38630026e5d 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -41,6 +41,7 @@ sec2::Sec2 as Sec2Falcon, Falcon, // }, + fsp::Fsp, gpu::Chipset, gsp::{ cmdq::Cmdq, @@ -67,6 +68,7 @@ pub(crate) struct GspBootContext<'ctx, 'gpu> { pub(crate) chipset: Chipset, pub(crate) gsp_falcon: &'ctx Falcon<'gpu, GspFalcon>, pub(crate) sec2_falcon: &'ctx Falcon<'gpu, Sec2Falcon>, + pub(crate) fsp: Option<&'ctx mut Fsp<'gpu>>, } impl<'ctx, 'gpu> GspBootContext<'ctx, 'gpu> { diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index d134ad052308..ad04904b5af4 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -17,10 +17,7 @@ Falcon, // }, fb::FbLayout, - fsp::{ - FmcBootArgs, - Fsp, // - }, + fsp::FmcBootArgs, gsp::{ hal::{ GspHal, @@ -143,7 +140,6 @@ fn boot( wpr_meta: &Coherent, ) -> Result> { let dev = ctx.dev(); - let bar = ctx.bar; let chipset = ctx.chipset; let gsp_falcon = ctx.gsp_falcon; @@ -151,8 +147,6 @@ fn boot( KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox ); - let mut fsp = Fsp::wait_secure_boot(dev, bar, chipset)?; - let args = FmcBootArgs::new( dev, chipset, @@ -164,9 +158,12 @@ fn boot( // Wait for the GSP RISC-V core to halt in case of error. We create this guard after `args` // to make sure that boot args are kept alive until halt, in case they are still being // accessed. - let unload_guard = ScopeGuard::new_with_data(unload_bundle, |unload_bundle| { - let _ = unload_bundle.0.run(ctx); - }); + let mut unload_guard = + ScopeGuard::new_with_data((unload_bundle, ctx), |(unload_bundle, ctx)| { + let _ = unload_bundle.0.run(ctx); + }); + + let fsp = unload_guard.1.fsp.as_mut().ok_or(ENODEV)?; fsp.boot_fmc(dev, fb_layout, &args)?; @@ -174,7 +171,7 @@ fn boot( // anymore. wait_for_gsp_lockdown_release(dev, gsp_falcon, args.boot_params_dma_handle())?; - Ok(Some(unload_guard.dismiss())) + Ok(Some(unload_guard.dismiss().0)) } } From 22e77d81d0a9ab3aee1c5538b3f2f8a66930bfa7 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Thu, 16 Jul 2026 15:25:44 +0100 Subject: [PATCH 097/137] rust: drm: fix non-const `read8` in unit test With CONFIG_CC_OPTIMIZE_FOR_SIZE, the address validity check in non-const `read8` invocaction is not optimized away, leading to build failure. Fixes: d055768429b3 ("rust: drm: gem: shmem: Add vmap functions") Signed-off-by: Gary Guo Link: https://patch.msgid.link/20260716142545.3622278-2-gary@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/drm/gem/shmem.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 580e0808d6c0..a687d46d170d 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -688,7 +688,7 @@ fn vmap_io() -> Result { // Ensure the ordering in memory is correct let expected = 0xFEDCBA98_u32.to_ne_bytes().into_iter(); for (offset, expected) in (0x20..=0x23).zip(expected) { - assert_eq!(vmap.read8(offset), expected); + assert_eq!(vmap.try_read8(offset).unwrap(), expected); } Ok(()) From 5f5237410773783c066c6c05bd502a34a95c6e8a Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Fri, 3 Jul 2026 19:22:05 +0900 Subject: [PATCH 098/137] gpu: nova-core: fsp: limit FSP receive message allocation size Currently, the FSP receive message code will try to allocate whatever was sent without checking it at all. But the actual size allowed is limited to 1024 anyway, so reject any messages over that size as bogus. Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260703-blackwell-fixes-v2-1-8e3d8bc32bb9@nvidia.com [acourbot: use `SZ_1K` constant for size.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/falcon/fsp.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/gpu/nova-core/falcon/fsp.rs b/drivers/gpu/nova-core/falcon/fsp.rs index 53b1079843ae..9d7322de1ce4 100644 --- a/drivers/gpu/nova-core/falcon/fsp.rs +++ b/drivers/gpu/nova-core/falcon/fsp.rs @@ -17,6 +17,7 @@ Io, // }, prelude::*, + sizes::SZ_1K, time::Delta, }; @@ -34,6 +35,9 @@ /// FSP message timeout in milliseconds. const FSP_MSG_TIMEOUT_MS: i64 = 2000; +/// Size of the FSP EMEM channel 0 that we can use. +const FSP_EMEM_CHANNEL_0_SIZE: usize = SZ_1K; + /// Type specifying the `Fsp` falcon engine. Cannot be instantiated. pub(crate) struct Fsp(()); @@ -159,6 +163,11 @@ pub(crate) fn recv_msg(&mut self) -> Result> { ) .map(num::u32_as_usize)?; + // Don't blindly allocate more than the maximum we expect from FSP. + if msg_size > FSP_EMEM_CHANNEL_0_SIZE { + return Err(EMSGSIZE); + } + let mut buffer = KVec::::new(); buffer.resize(msg_size, 0, GFP_KERNEL)?; From ed33ea9390dd40eabb25069eaaed3d011143fe58 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Fri, 3 Jul 2026 19:22:06 +0900 Subject: [PATCH 099/137] gpu: nova-core: fsp: catch bogus queue pointer issues Currently, `poll_msgq` will report a message of size 4 if the queue pointers are broken. It's easy to catch this if it occurs, so have `poll_msgq` return an error in this case. Signed-off-by: Eliot Courtney Reviewed-by: Alistair Popple Link: https://patch.msgid.link/20260703-blackwell-fixes-v2-2-8e3d8bc32bb9@nvidia.com [acourbot: explicitly mention the error, add paragraph separator.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/falcon/fsp.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/nova-core/falcon/fsp.rs b/drivers/gpu/nova-core/falcon/fsp.rs index 9d7322de1ce4..0437180b8829 100644 --- a/drivers/gpu/nova-core/falcon/fsp.rs +++ b/drivers/gpu/nova-core/falcon/fsp.rs @@ -111,18 +111,22 @@ fn read_emem(&mut self, data: &mut [u8]) -> Result { /// /// Returns the size of available data in bytes, or 0 if no data is available. /// + /// Returns [`EIO`] if the queue pointers are bogus (`tail < head`). + /// /// The FSP message queue is not circular. Pointers are reset to 0 after each /// message exchange, so `tail >= head` is always true when data is present. - fn poll_msgq(&self) -> u32 { + fn poll_msgq(&self) -> Result { let head = self.bar.read(regs::NV_PFSP_MSGQ_HEAD::at(0)).val(); let tail = self.bar.read(regs::NV_PFSP_MSGQ_TAIL::at(0)).val(); if head == tail { - return 0; + Ok(0) + } else { + // TAIL points at the last DWORD written, so the size is `tail - head + 4`. + tail.checked_sub(head) + .and_then(|delta| delta.checked_add(4)) + .ok_or(EIO) } - - // TAIL points at last DWORD written, so add 4 to get total size. - tail.saturating_sub(head).saturating_add(4) } /// Writes `packet` to FSP EMEM and updates the queue pointers to notify FSP. @@ -156,7 +160,7 @@ pub(crate) fn send_msg(&mut self, packet: &[u8]) -> Result { /// memory allocation error occurred. pub(crate) fn recv_msg(&mut self) -> Result> { let msg_size = read_poll_timeout( - || Ok(self.poll_msgq()), + || self.poll_msgq(), |&size| size > 0, Delta::from_millis(10), Delta::from_millis(FSP_MSG_TIMEOUT_MS), From d76956f7b79531e6801fe293259e93d0df1b96a7 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Fri, 3 Jul 2026 19:22:07 +0900 Subject: [PATCH 100/137] gpu: nova-core: gsp: ensure lifetime for FMC boot DMA allocations Currently, `FmcBootArgs` takes DMA handles directly, rather than references to the `Coherent` for them. This is error prone, so instead store lifetime'd references to the `Coherent` allocation. Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260703-blackwell-fixes-v2-3-8e3d8bc32bb9@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fsp.rs | 32 ++++++++++++++++---------- drivers/gpu/nova-core/gsp.rs | 6 ++--- drivers/gpu/nova-core/gsp/hal/gh100.rs | 19 ++++++--------- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index afbd75879d16..1475485bded3 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -40,7 +40,11 @@ FIRMWARE_VERSION, // }, gpu::Chipset, - gsp::GspFmcBootParams, + gsp::{ + GspFmcBootParams, + GspFwWprMeta, + LibosMemoryRegionInitArgument, // + }, mctp::{ MctpHeader, NvdmHeader, @@ -134,7 +138,7 @@ impl FspCotMessage { fn new<'a>( fb_layout: &FbLayout, fsp_fw: &'a FspFirmware, - args: &'a FmcBootArgs, + args: &'a FmcBootArgs<'_>, ) -> Result + 'a> { // frts_vidmem_offset is measured from the end of FB, so FRTS sits at // (end of FB) - frts_vidmem_offset. @@ -188,35 +192,39 @@ impl MessageToFsp for FspCotMessage { } /// Bundled arguments for FMC boot via FSP Chain of Trust. -pub(crate) struct FmcBootArgs { +pub(crate) struct FmcBootArgs<'a> { chipset: Chipset, fmc_boot_params: Coherent, resume: bool, + // Additional dependencies required to be kept alive for FMC boot. + _wpr_meta: &'a Coherent, + _libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, } -impl FmcBootArgs { +impl<'a> FmcBootArgs<'a> { /// Builds FMC boot arguments, allocating the DMA-coherent boot parameter /// structure that FSP will read. pub(crate) fn new( dev: &device::Device, chipset: Chipset, - wpr_meta_addr: u64, - libos_addr: u64, + wpr_meta: &'a Coherent, + libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, resume: bool, ) -> Result { - let init = GspFmcBootParams::new(wpr_meta_addr, libos_addr); + let init = GspFmcBootParams::new(wpr_meta.dma_handle(), libos.dma_handle()); Ok(Self { chipset, fmc_boot_params: Coherent::::init(dev, GFP_KERNEL, init)?, resume, + _wpr_meta: wpr_meta, + _libos: libos, }) } - /// DMA address of the FMC boot parameters, needed after boot for lockdown - /// release polling. - pub(crate) fn boot_params_dma_handle(&self) -> u64 { - self.fmc_boot_params.dma_handle() + /// Returns the FMC boot parameters allocation. + pub(crate) fn boot_params(&self) -> &Coherent { + &self.fmc_boot_params } } @@ -350,7 +358,7 @@ pub(crate) fn boot_fmc( &mut self, dev: &device::Device, fb_layout: &FbLayout, - args: &FmcBootArgs, + args: &FmcBootArgs<'_>, ) -> Result { dev_dbg!(dev, "Starting FSP boot sequence for {}\n", args.chipset); diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index f38630026e5d..c7b18c44c03d 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -30,6 +30,7 @@ pub(crate) use fw::{ GspFmcBootParams, GspFwWprMeta, + LibosMemoryRegionInitArgument, LibosParams, // }; pub(crate) use hal::boot_firmware_files; @@ -45,10 +46,7 @@ gpu::Chipset, gsp::{ cmdq::Cmdq, - fw::{ - GspArgumentsPadded, - LibosMemoryRegionInitArgument, // - }, + fw::GspArgumentsPadded, // }, num, }; diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index ad04904b5af4..ee086f1c2876 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -25,6 +25,7 @@ }, Gsp, GspBootContext, + GspFmcBootParams, GspFwWprMeta, // }, }; @@ -56,13 +57,13 @@ fn combined_addr(&self) -> u64 { fn lockdown_released_or_error( &self, gsp_falcon: &Falcon<'_, GspEngine>, - fmc_boot_params_addr: u64, + fmc_boot_params: &Coherent, ) -> bool { // GSP-FMC normally clears the boot parameters address from the mailboxes early during // boot. If the address is still there, keep polling rather than treating it as an error. // Any other non-zero mailbox0 value is a GSP-FMC error code. if self.mbox0 != 0 { - return self.combined_addr() != fmc_boot_params_addr; + return self.combined_addr() != fmc_boot_params.dma_handle(); } !gsp_falcon.riscv_branch_privilege_lockdown() @@ -73,7 +74,7 @@ fn lockdown_released_or_error( fn wait_for_gsp_lockdown_release( dev: &device::Device, gsp_falcon: &Falcon<'_, GspEngine>, - fmc_boot_params_addr: u64, + fmc_boot_params: &Coherent, ) -> Result { dev_dbg!(dev, "Waiting for GSP lockdown release\n"); @@ -88,7 +89,7 @@ fn wait_for_gsp_lockdown_release( }, |mbox| match mbox { None => false, - Some(mbox) => mbox.lockdown_released_or_error(gsp_falcon, fmc_boot_params_addr), + Some(mbox) => mbox.lockdown_released_or_error(gsp_falcon, fmc_boot_params), }, Delta::from_millis(10), Delta::from_secs(30), @@ -147,13 +148,7 @@ fn boot( KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox ); - let args = FmcBootArgs::new( - dev, - chipset, - wpr_meta.dma_handle(), - gsp.libos.dma_handle(), - false, - )?; + let args = FmcBootArgs::new(dev, chipset, wpr_meta, &gsp.libos, false)?; // Wait for the GSP RISC-V core to halt in case of error. We create this guard after `args` // to make sure that boot args are kept alive until halt, in case they are still being @@ -169,7 +164,7 @@ fn boot( // Wait for GSP-FMC to release the GSP lockdown, indicating that `args` is not accessed // anymore. - wait_for_gsp_lockdown_release(dev, gsp_falcon, args.boot_params_dma_handle())?; + wait_for_gsp_lockdown_release(dev, gsp_falcon, args.boot_params())?; Ok(Some(unload_guard.dismiss().0)) } From 9c96c8c2caf8e6095feff63c3e5bc05a95feebc6 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Fri, 3 Jul 2026 19:22:08 +0900 Subject: [PATCH 101/137] gpu: nova-core: gsp: ensure LibOS DMA allocation lives long enough Currently, `GspSequencer` stores a raw DMA handle. Instead, store a reference to `Coherent` to statically ensure that the allocation lives long enough. Signed-off-by: Eliot Courtney Reviewed-by: Alistair Popple Link: https://patch.msgid.link/20260703-blackwell-fixes-v2-4-8e3d8bc32bb9@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/hal/tu102.rs | 7 +------ drivers/gpu/nova-core/gsp/sequencer.rs | 18 +++++++++++------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index 29bb17171f56..648657e248da 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -320,12 +320,7 @@ fn post_boot( ctx: &mut GspBootContext<'_, '_>, gsp_fw: &GspFirmware, ) -> Result { - GspSequencer::run( - &gsp.cmdq, - ctx, - gsp.libos.dma_handle(), - gsp_fw.bootloader.app_version, - )?; + GspSequencer::run(&gsp.cmdq, ctx, &gsp.libos, gsp_fw.bootloader.app_version)?; Ok(()) } diff --git a/drivers/gpu/nova-core/gsp/sequencer.rs b/drivers/gpu/nova-core/gsp/sequencer.rs index 422a74f9ecbd..5e1ec7e59ab0 100644 --- a/drivers/gpu/nova-core/gsp/sequencer.rs +++ b/drivers/gpu/nova-core/gsp/sequencer.rs @@ -6,6 +6,7 @@ use kernel::{ device, + dma::Coherent, io::{ poll::read_poll_timeout, Io, // @@ -31,7 +32,8 @@ MessageFromGsp, // }, fw, - GspBootContext, // + GspBootContext, + LibosMemoryRegionInitArgument, // }, num::FromSafeCast, sbuffer::SBufferIter, @@ -135,8 +137,8 @@ pub(crate) struct GspSequencer<'a> { sec2_falcon: &'a Falcon<'a, Sec2>, /// GSP falcon for core operations. gsp_falcon: &'a Falcon<'a, Gsp>, - /// LibOS DMA handle address. - libos_dma_handle: u64, + /// LibOS memory region init arguments. + libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, /// Bootloader application version. bootloader_app_version: u32, /// Device for logging. @@ -232,10 +234,12 @@ fn run(&self, seq: &GspSequencer<'_>) -> Result { // Reset the GSP to prepare it for resuming. seq.gsp_falcon.reset()?; + let libos_dma_handle = seq.libos.dma_handle(); + // Write the libOS DMA handle to GSP mailboxes. seq.gsp_falcon.write_mailboxes( - Some(seq.libos_dma_handle as u32), - Some((seq.libos_dma_handle >> 32) as u32), + Some(libos_dma_handle as u32), + Some((libos_dma_handle >> 32) as u32), ); // Start the SEC2 falcon which will trigger GSP-RM to resume on the GSP. @@ -336,7 +340,7 @@ impl<'a> GspSequencer<'a> { pub(crate) fn run( cmdq: &Cmdq, ctx: &'a GspBootContext<'_, '_>, - libos_dma_handle: u64, + libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, bootloader_app_version: u32, ) -> Result { let seq_info = loop { @@ -351,7 +355,7 @@ pub(crate) fn run( bar: ctx.bar, sec2_falcon: ctx.sec2_falcon, gsp_falcon: ctx.gsp_falcon, - libos_dma_handle, + libos, bootloader_app_version, dev: ctx.dev(), }; From 5557c238eb0f97169edda1d0776207e3d61f4f16 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Fri, 3 Jul 2026 19:22:13 +0900 Subject: [PATCH 102/137] gpu: nova-core: correct RISC-V HALTED field This uses the incorrect value, so update it. Fixes: bb58d1aee608 ("gpu: nova-core: falcon: Add support to check if RISC-V is active") Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260703-blackwell-fixes-v2-9-8e3d8bc32bb9@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/regs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index 397124f245ee..49591c3dcfa7 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -556,7 +556,7 @@ pub(crate) fn mem_scrubbing_done(self) -> bool { /// GA102 and later. pub(crate) NV_PRISCV_RISCV_CPUCTL(u32) @ PFalcon2Base + 0x00000388 { 7:7 active_stat => bool; - 0:0 halted => bool; + 4:4 halted => bool; } /// GA102 and later. From 71d4e7233f235871b13553e504e591ace6b54373 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Fri, 3 Jul 2026 19:22:14 +0900 Subject: [PATCH 103/137] gpu: nova-core: wait for RISC-V HALTED on FSP unload Currently the code waits for "not active" but this is not the same as halted as there are more than two states. Match openrm here and wait for halted instead. Fixes: c7fea1f70944 ("gpu: nova-core: add non-sec2 unload path") Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260703-blackwell-fixes-v2-10-8e3d8bc32bb9@nvidia.com [acourbot: s/imply/guarantee.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/falcon.rs | 11 +++++++++++ drivers/gpu/nova-core/falcon/hal.rs | 5 +++++ drivers/gpu/nova-core/falcon/hal/ga102.rs | 7 +++++++ drivers/gpu/nova-core/falcon/hal/tu102.rs | 4 ++++ drivers/gpu/nova-core/gsp/hal/gh100.rs | 11 +++++++++-- 5 files changed, 36 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index 78948cc8bff3..a91cbdd5d636 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -749,11 +749,22 @@ pub(crate) fn signature_reg_fuse_version( /// Check if the RISC-V core is active. /// + /// Note that this does not guarantee that the RISC-V core is halted if it returns `false`. + /// /// Returns `true` if the RISC-V core is active, `false` otherwise. pub(crate) fn is_riscv_active(&self) -> bool { self.hal.is_riscv_active(self) } + /// Checks whether the RISC-V core is halted. + /// + /// Note that this does not guarantee that the RISC-V core is active if it returns `false`. + /// + /// Returns [`ENOTSUPP`] if the status is not available. + pub(crate) fn is_riscv_halted(&self) -> Result { + self.hal.is_riscv_halted(self) + } + /// Load a firmware image into Falcon memory, using the preferred method for the current /// chipset. pub(crate) fn load + FalconDmaLoadable>(&self, fw: &F) -> Result { diff --git a/drivers/gpu/nova-core/falcon/hal.rs b/drivers/gpu/nova-core/falcon/hal.rs index ee4a017f3a4c..7e532889a1f4 100644 --- a/drivers/gpu/nova-core/falcon/hal.rs +++ b/drivers/gpu/nova-core/falcon/hal.rs @@ -53,6 +53,11 @@ fn signature_reg_fuse_version( /// Returns `true` if the RISC-V core is active, `false` otherwise. fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool; + /// Checks whether the RISC-V core is halted. + /// + /// Returns [`ENOTSUPP`] if the chipset does not expose RISC-V halt status. + fn is_riscv_halted(&self, falcon: &Falcon<'_, E>) -> Result; + /// Wait for memory scrubbing to complete. fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result; diff --git a/drivers/gpu/nova-core/falcon/hal/ga102.rs b/drivers/gpu/nova-core/falcon/hal/ga102.rs index fe821ded5fa1..7600ee07ca2e 100644 --- a/drivers/gpu/nova-core/falcon/hal/ga102.rs +++ b/drivers/gpu/nova-core/falcon/hal/ga102.rs @@ -139,6 +139,13 @@ fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool { .active_stat() } + fn is_riscv_halted(&self, falcon: &Falcon<'_, E>) -> Result { + Ok(falcon + .bar + .read(regs::NV_PRISCV_RISCV_CPUCTL::of::()) + .halted()) + } + fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result { // TIMEOUT: memory scrubbing should complete in less than 20ms. read_poll_timeout( diff --git a/drivers/gpu/nova-core/falcon/hal/tu102.rs b/drivers/gpu/nova-core/falcon/hal/tu102.rs index 34bf9f3f44c7..5291598fedf7 100644 --- a/drivers/gpu/nova-core/falcon/hal/tu102.rs +++ b/drivers/gpu/nova-core/falcon/hal/tu102.rs @@ -55,6 +55,10 @@ fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool { .active_stat() } + fn is_riscv_halted(&self, _falcon: &Falcon<'_, E>) -> Result { + Err(ENOTSUPP) + } + fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result { // TIMEOUT: memory scrubbing should complete in less than 10ms. read_poll_timeout( diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index ee086f1c2876..22b60f9233de 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -116,8 +116,15 @@ impl UnloadBundle for FspUnloadBundle { fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result { // GSP falcon does most of the work of resetting, so just wait for it to finish. read_poll_timeout( - || Ok(ctx.gsp_falcon.is_riscv_active()), - |&active| !active, + || { + // GSP register reads are not meaningful until the PRIV target mask is released. + if !ctx.gsp_falcon.priv_target_mask_released() { + return Ok(false); + } + + ctx.gsp_falcon.is_riscv_halted() + }, + |&halted| halted, Delta::from_millis(10), Delta::from_secs(5), ) From 7cffd051ae546e8644bf9bf7410b63541f666845 Mon Sep 17 00:00:00 2001 From: Zhi Wang Date: Wed, 22 Jul 2026 10:39:08 +0300 Subject: [PATCH 104/137] PCI/IOV: Return unsigned int from pci_sriov_get_totalvfs() pci_sriov_get_totalvfs() reports a VF count, not an errno-style status. It returns 0 when SR-IOV is unavailable or the device is not a PF, and otherwise returns the PF's driver_max_VFs value. driver_max_VFs is stored as a u16 in struct pci_sriov. It is derived from the SR-IOV TotalVFs field or from a driver-provided limit, so the implementation cannot return a negative value. Change the declaration, CONFIG_PCI_IOV stub, and implementation to return unsigned int. Suggested-by: Alexandre Courbot Reviewed-by: Alexandre Courbot Acked-by: Bjorn Helgaas Cc: Bjorn Helgaas Cc: David Laight Cc: Gary Guo Cc: linux-pci@vger.kernel.org Link: https://lore.kernel.org/all/DJHPRE4TGGT8.BUTMYOF5YE05@nvidia.com/ Signed-off-by: Zhi Wang Link: https://patch.msgid.link/20260722073913.1807677-2-zhiw@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/pci/iov.c | 2 +- include/linux/pci.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c index b0d24839c084..9d408fb8ac25 100644 --- a/drivers/pci/iov.c +++ b/drivers/pci/iov.c @@ -1283,7 +1283,7 @@ EXPORT_SYMBOL_GPL(pci_sriov_set_totalvfs); * SRIOV capability value of TotalVFs or the value of driver_max_VFs * if the driver reduced it. Otherwise 0. */ -int pci_sriov_get_totalvfs(struct pci_dev *dev) +unsigned int pci_sriov_get_totalvfs(struct pci_dev *dev) { if (!dev->is_physfn) return 0; diff --git a/include/linux/pci.h b/include/linux/pci.h index ebb5b9d76360..2b9c61de5f67 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -2569,7 +2569,7 @@ void pci_iov_remove_virtfn(struct pci_dev *dev, int id); int pci_num_vf(struct pci_dev *dev); int pci_vfs_assigned(struct pci_dev *dev); int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs); -int pci_sriov_get_totalvfs(struct pci_dev *dev); +unsigned int pci_sriov_get_totalvfs(struct pci_dev *dev); int pci_sriov_configure_simple(struct pci_dev *dev, int nr_virtfn); resource_size_t pci_iov_resource_size(const struct pci_dev *dev, int resno); int pci_iov_vf_bar_set_size(struct pci_dev *dev, int resno, int size); @@ -2622,7 +2622,7 @@ static inline int pci_vfs_assigned(struct pci_dev *dev) { return 0; } static inline int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs) { return 0; } -static inline int pci_sriov_get_totalvfs(struct pci_dev *dev) +static inline unsigned int pci_sriov_get_totalvfs(struct pci_dev *dev) { return 0; } #define pci_sriov_configure_simple NULL static inline resource_size_t pci_iov_resource_size(const struct pci_dev *dev, From 6afbbc27f582b873eb42744b09d57aeefc9e5628 Mon Sep 17 00:00:00 2001 From: Zhi Wang Date: Wed, 22 Jul 2026 10:39:09 +0300 Subject: [PATCH 105/137] rust: pci: add sriov_get_totalvfs() helper Expose pci_sriov_get_totalvfs() to Rust PCI drivers so they can query how many SR-IOV VFs a device supports. Use a conditional C helper because the !CONFIG_PCI_IOV version of pci_sriov_get_totalvfs() is a static inline function and is therefore not emitted into the Rust bindings. Return Option> so Rust callers must handle the zero value that represents unavailable SR-IOV. Reviewed-by: Alexandre Courbot Cc: Alexandre Courbot Cc: Bjorn Helgaas Cc: David Laight Cc: Gary Guo Cc: linux-pci@vger.kernel.org Link: https://lore.kernel.org/all/DJHPRE4TGGT8.BUTMYOF5YE05@nvidia.com/ Signed-off-by: Zhi Wang Link: https://patch.msgid.link/20260722073913.1807677-3-zhiw@nvidia.com Signed-off-by: Danilo Krummrich --- rust/helpers/pci.c | 8 ++++++++ rust/kernel/pci.rs | 13 +++++++++++++ 2 files changed, 21 insertions(+) diff --git a/rust/helpers/pci.c b/rust/helpers/pci.c index e44905317d75..4ebf256dff23 100644 --- a/rust/helpers/pci.c +++ b/rust/helpers/pci.c @@ -24,6 +24,14 @@ __rust_helper bool rust_helper_dev_is_pci(const struct device *dev) return dev_is_pci(dev); } +#ifndef CONFIG_PCI_IOV +__rust_helper unsigned int +rust_helper_pci_sriov_get_totalvfs(struct pci_dev *pdev) +{ + return pci_sriov_get_totalvfs(pdev); +} +#endif + #ifndef CONFIG_PCI_MSI __rust_helper int rust_helper_pci_alloc_irq_vectors(struct pci_dev *dev, unsigned int min_vecs, diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index c6d6bd8f251d..9f19ccd5905c 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -25,6 +25,7 @@ use core::{ marker::PhantomData, mem::offset_of, + num::NonZero, ptr::{ addr_of_mut, NonNull, // @@ -452,6 +453,18 @@ pub fn pci_class(&self) -> Class { } impl<'a> Device> { + /// Returns the total number of VFs, or [`None`] if SR-IOV is not available. + #[inline] + pub fn sriov_get_totalvfs(&self) -> Option> { + // SAFETY: `self.as_raw()` is a valid pointer to a `struct pci_dev`. + let total_vfs = unsafe { bindings::pci_sriov_get_totalvfs(self.as_raw()) }; + + // CAST: The C function returns `unsigned int`, but the value originates + // from TotalVFs/driver_max_VFs (which are defined as `u16`), so this cast + // cannot truncate. + NonZero::new(total_vfs as u16) + } + /// Enable memory resources for this device. pub fn enable_device_mem(&self) -> Result { // SAFETY: `self.as_raw` is guaranteed to be a pointer to a valid `struct pci_dev`. From 29073113cf0044fd05dd0b4318421e91b8de8542 Mon Sep 17 00:00:00 2001 From: Zhi Wang Date: Wed, 22 Jul 2026 10:39:10 +0300 Subject: [PATCH 106/137] gpu: nova-core: read vGPU mode from FSP via PRC protocol vGPU boot needs to know whether firmware reports vGPU mode as active. FSP's Management Partition exposes PRC (Product Reconfiguration Control) as an API for reading device configuration knobs without firmware updates. The vGPU mode knob is one such configuration value. Add typed PRC request and response payloads for the vGPU mode object, add the PRC NVDM type, and parse the returned knob value into VgpuMode. Signed-off-by: Zhi Wang Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260722073913.1807677-4-zhiw@nvidia.com [ FspPrcMessage is small and short-lived; stack-allocate it instead of using KBox. - Danilo ] Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/fsp.rs | 170 ++++++++++++++++++++++++++++++++++ drivers/gpu/nova-core/mctp.rs | 2 + 2 files changed, 172 insertions(+) diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index 1475485bded3..e63c8869379c 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -56,6 +56,56 @@ mod hal; +/// PRC message sub-command. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +enum PrcMessageSubcmd { + /// Read a PRC knob value. + Read = 0x0c, +} + +impl From for u8 { + fn from(value: PrcMessageSubcmd) -> Self { + value as u8 + } +} + +/// PRC object identifier. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +enum PrcObjectId { + /// vGPU mode configuration knob. + VgpuMode = 0x29, +} + +impl From for u8 { + fn from(value: PrcObjectId) -> Self { + value as u8 + } +} + +kernel::impl_flags!( + /// PRC request flags. + #[derive(Clone, Copy, Default, PartialEq, Eq)] + struct PrcFlags(u8); + + /// Individual PRC request flag. + #[derive(Clone, Copy, PartialEq, Eq)] + enum PrcFlag { + /// Request the active knob value for the current boot. + Active = 1 << 1, + } +); + +/// vGPU operating mode as reported by FSP via the PRC protocol. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum VgpuMode { + /// vGPU support is disabled on this GPU. + Disabled, + /// vGPU support is enabled on this GPU. + Enabled, +} + /// FSP command response payload (`NVDM_PAYLOAD_COMMAND_RESPONSE`). #[repr(C, packed)] #[derive(Clone, Copy)] @@ -65,6 +115,62 @@ struct NvdmPayloadCommandResponse { error_code: u32, } +/// PRC message payload. +/// +/// Sent to FSP to query or modify a device configuration knob. +#[repr(C, packed)] +#[derive(Clone, Copy)] +struct NvdmPayloadPrc { + sub_message_id: u8, + flags: u8, + object_id: u8, + reserved: u8, +} + +impl NvdmPayloadPrc { + /// Constructs a PRC payload from typed protocol fields. + fn new(subcmd: PrcMessageSubcmd, object_id: PrcObjectId, flags: PrcFlags) -> Self { + Self { + sub_message_id: subcmd.into(), + flags: flags.into(), + object_id: object_id.into(), + reserved: 0, + } + } +} + +// SAFETY: NvdmPayloadPrc is a packed C struct with only integral fields. +unsafe impl AsBytes for NvdmPayloadPrc {} + +/// PRC response payload containing the knob state value. +#[repr(C, packed)] +#[derive(Clone, Copy)] +struct NvdmPayloadPrcResponse { + value_low: u8, + value_high: u8, + reserved1: u8, + reserved2: u8, +} + +impl NvdmPayloadPrcResponse { + /// Returns the PRC knob value as a little-endian 16-bit integer. + fn value(self) -> u16 { + u16::from(self.value_low) | (u16::from(self.value_high) << 8) + } +} + +impl TryFrom for VgpuMode { + type Error = kernel::error::Error; + + fn try_from(value: NvdmPayloadPrcResponse) -> Result { + match value.value() { + 0 => Ok(VgpuMode::Disabled), + 1 => Ok(VgpuMode::Enabled), + _ => Err(EINVAL), + } + } +} + /// Common MCTP and NVDM headers shared by all FSP messages. #[repr(C, packed)] #[derive(Clone, Copy)] @@ -100,6 +206,17 @@ struct FspResponseHeader { // SAFETY: FspResponseHeader is a packed C struct with only integral fields. unsafe impl FromBytes for FspResponseHeader {} +/// Complete FSP PRC response including the knob state payload. +#[repr(C, packed)] +#[derive(Clone, Copy)] +struct FspPrcResponse { + header: FspResponseHeader, + prc_data: NvdmPayloadPrcResponse, +} + +// SAFETY: FspPrcResponse is a packed C struct with only integral fields. +unsafe impl FromBytes for FspPrcResponse {} + /// Trait implemented by types representing a message to send to FSP. /// /// This provides [`Fsp::send_sync_fsp`] with the information it needs to send @@ -187,10 +304,35 @@ fn new<'a>( // bytes are initialized. unsafe impl AsBytes for FspCotMessage {} +/// Complete FSP PRC message. +#[repr(C, packed)] +#[derive(Clone, Copy)] +struct FspPrcMessage { + header: FspMessageHeader, + prc: NvdmPayloadPrc, +} + +impl FspPrcMessage { + /// Constructs a PRC message. + fn new(subcmd: PrcMessageSubcmd, object_id: PrcObjectId, flags: PrcFlags) -> Self { + Self { + header: FspMessageHeader::new(NvdmType::Prc), + prc: NvdmPayloadPrc::new(subcmd, object_id, flags), + } + } +} + +// SAFETY: FspPrcMessage is a packed C struct with only integral fields. +unsafe impl AsBytes for FspPrcMessage {} + impl MessageToFsp for FspCotMessage { const NVDM_TYPE: NvdmType = NvdmType::Cot; } +impl MessageToFsp for FspPrcMessage { + const NVDM_TYPE: NvdmType = NvdmType::Prc; +} + /// Bundled arguments for FMC boot via FSP Chain of Trust. pub(crate) struct FmcBootArgs<'a> { chipset: Chipset, @@ -350,6 +492,34 @@ fn send_sync_fsp(&mut self, dev: &device::Device, msg: &M) -> Result Ok(response_buf) } + /// Reads the active vGPU mode from FSP using the PRC protocol. + /// + /// Queries FSP's Management Partition for the active vGPU mode knob value. + #[expect(dead_code)] + pub(crate) fn read_vgpu_mode( + &mut self, + dev: &device::Device, + ) -> Result { + let msg = FspPrcMessage::new( + PrcMessageSubcmd::Read, + PrcObjectId::VgpuMode, + PrcFlags::from(PrcFlag::Active), + ); + + let response_buf = self.send_sync_fsp(dev, &msg)?; + let (prc_response, _) = + FspPrcResponse::from_bytes_prefix(&response_buf[..]).ok_or_else(|| { + dev_err!(dev, "PRC response too small: {}\n", response_buf.len()); + EIO + })?; + + let prc_data = prc_response.prc_data; + + VgpuMode::try_from(prc_data).inspect_err(|_| { + dev_err!(dev, "Unexpected vGPU mode value: {:#x}\n", prc_data.value()); + }) + } + /// Boots GSP FMC via FSP Chain of Trust. /// /// Builds the CoT message from the pre-configured [`FmcBootArgs`], sends it diff --git a/drivers/gpu/nova-core/mctp.rs b/drivers/gpu/nova-core/mctp.rs index acc2abbd4b0c..90c642c91a72 100644 --- a/drivers/gpu/nova-core/mctp.rs +++ b/drivers/gpu/nova-core/mctp.rs @@ -22,6 +22,8 @@ /// NVDM message type identifiers carried over MCTP. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum NvdmType with TryFrom> { + /// PRC (Product Reconfiguration Control) message. + Prc = 0x13, /// Chain of Trust boot message. Cot = 0x14, /// FSP command response. From c0ce158096d4c2e88d270777588bf5ea7cee5f5c Mon Sep 17 00:00:00 2001 From: Zhi Wang Date: Wed, 22 Jul 2026 10:39:11 +0300 Subject: [PATCH 107/137] gpu: nova-core: detect and store vGPU state GSP boot needs a stable view of vGPU state before it starts building the boot-time data structures that depend on SR-IOV and firmware policy. That state must be derived once from the PCI VF count and the FSP PRC vGPU mode knob before booting GSP. Add VgpuManager to detect and retain the vGPU state during GPU construction. Keep the manager separate from the detected state because later vGPU milestones will add vGPU resources and lifecycle state to it. Keep the vGPU capability gate local to the vGPU module with per-chip HAL modules. Treat failures to detect the optional vGPU state as disabled so they do not prevent a bare-metal probe, and log both the failure and the detected state where the manager is constructed. Cc: Alexandre Courbot Signed-off-by: Zhi Wang Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260722073913.1807677-5-zhiw@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/fsp.rs | 1 - drivers/gpu/nova-core/gpu.rs | 7 ++ drivers/gpu/nova-core/gsp.rs | 3 + drivers/gpu/nova-core/nova_core.rs | 1 + drivers/gpu/nova-core/vgpu.rs | 93 +++++++++++++++++++++++++ drivers/gpu/nova-core/vgpu/hal.rs | 25 +++++++ drivers/gpu/nova-core/vgpu/hal/gb202.rs | 15 ++++ drivers/gpu/nova-core/vgpu/hal/tu102.rs | 15 ++++ 8 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/nova-core/vgpu.rs create mode 100644 drivers/gpu/nova-core/vgpu/hal.rs create mode 100644 drivers/gpu/nova-core/vgpu/hal/gb202.rs create mode 100644 drivers/gpu/nova-core/vgpu/hal/tu102.rs diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index e63c8869379c..ba4544210e40 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -495,7 +495,6 @@ fn send_sync_fsp(&mut self, dev: &device::Device, msg: &M) -> Result /// Reads the active vGPU mode from FSP using the PRC protocol. /// /// Queries FSP's Management Partition for the active vGPU mode knob value. - #[expect(dead_code)] pub(crate) fn read_vgpu_mode( &mut self, dev: &device::Device, diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 442c0979f9c6..42a4cd7971fa 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -30,6 +30,7 @@ GspBootContext, // }, regs, + vgpu::VgpuManager, // }; mod hal; @@ -267,6 +268,8 @@ struct GspResources<'gpu> { // TODO: use different resource types for each boot method, and make the relevant Gsp methods // generic against them. fsp: Option>, + /// vGPU state detected before GSP boot. + vgpu: VgpuManager, /// GSP runtime data. #[pin] gsp: Gsp, @@ -311,6 +314,7 @@ fn drop(self: Pin<&mut Self>) { gsp_falcon: &*this.gsp_falcon, sec2_falcon: &*this.sec2_falcon, fsp: this.fsp.as_mut(), + vgpu: &*this.vgpu, }, bundle, ) @@ -364,6 +368,8 @@ pub(crate) fn new( fsp: Fsp::try_new(dev, bar, spec.chipset)?, + vgpu: VgpuManager::new(pdev, spec.chipset, fsp.as_mut()), + gsp <- Gsp::new(pdev), // This member must be initialized last, so the `UnloadBundle` can never be dropped @@ -376,6 +382,7 @@ pub(crate) fn new( gsp_falcon, sec2_falcon, fsp: fsp.as_mut(), + vgpu, })?, }), diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index c7b18c44c03d..d3c97da5848c 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -49,6 +49,7 @@ fw::GspArgumentsPadded, // }, num, + vgpu::VgpuManager, // }; pub(crate) const GSP_PAGE_SHIFT: usize = 12; @@ -67,6 +68,8 @@ pub(crate) struct GspBootContext<'ctx, 'gpu> { pub(crate) gsp_falcon: &'ctx Falcon<'gpu, GspFalcon>, pub(crate) sec2_falcon: &'ctx Falcon<'gpu, Sec2Falcon>, pub(crate) fsp: Option<&'ctx mut Fsp<'gpu>>, + #[expect(dead_code)] + pub(crate) vgpu: &'ctx VgpuManager, } impl<'ctx, 'gpu> GspBootContext<'ctx, 'gpu> { diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index a61406ba5c0b..35a8b1214b0e 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -23,6 +23,7 @@ mod regs; mod sbuffer; mod vbios; +mod vgpu; pub(crate) const MODULE_NAME: &core::ffi::CStr = ::NAME; diff --git a/drivers/gpu/nova-core/vgpu.rs b/drivers/gpu/nova-core/vgpu.rs new file mode 100644 index 000000000000..71fd0fe879c2 --- /dev/null +++ b/drivers/gpu/nova-core/vgpu.rs @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0 + +use core::num::NonZero; + +use kernel::{ + device, + pci, + prelude::*, // +}; + +use crate::{ + fsp::{ + Fsp, + VgpuMode, // + }, + gpu::Chipset, // +}; + +mod hal; + +/// vGPU state detected during GPU construction. +#[derive(Debug, Clone, Copy)] +pub(crate) enum VgpuState { + /// vGPU mode is not enabled for this boot. + Disabled, + /// vGPU mode is enabled for this boot. + Enabled { + /// Total number of SR-IOV VFs supported by this device. + #[expect(dead_code)] + total_vfs: NonZero, + }, +} + +/// vGPU state manager. +pub(crate) struct VgpuManager { + state: VgpuState, +} + +impl VgpuManager { + /// Creates a vGPU manager by querying SR-IOV and the FSP PRC vGPU knob. + pub(crate) fn new( + pdev: &pci::Device>, + chipset: Chipset, + fsp: Option<&mut Fsp<'_>>, + ) -> Self { + let state = Self::detect_state(pdev, chipset, fsp).unwrap_or_else(|e| { + dev_warn!( + pdev, + "vGPU state detection failed: {:?}; disabling vGPU\n", + e + ); + VgpuState::Disabled + }); + dev_dbg!(pdev, "vGPU state: {:?}\n", state); + + Self { state } + } + + /// Detects the vGPU state from the chipset, SR-IOV capability and FSP PRC knob. + fn detect_state( + pdev: &pci::Device>, + chipset: Chipset, + fsp: Option<&mut Fsp<'_>>, + ) -> Result { + if !hal::vgpu_hal(chipset).supports_vgpu() { + return Ok(VgpuState::Disabled); + } + + let Some(total_vfs) = pdev.sriov_get_totalvfs() else { + return Ok(VgpuState::Disabled); + }; + + if total_vfs.get() < 2 { + // The current vGPU path does not support single-VF SR-IOV devices yet. + // Treat one total VF as vGPU-disabled for now; single-VF support can relax + // this gate once the manager handles that topology. + return Ok(VgpuState::Disabled); + } + + let fsp = fsp.ok_or(ENODEV)?; + + match fsp.read_vgpu_mode(pdev.as_ref())? { + VgpuMode::Enabled => Ok(VgpuState::Enabled { total_vfs }), + VgpuMode::Disabled => Ok(VgpuState::Disabled), + } + } + + /// Returns the detected vGPU state for this boot. + #[expect(dead_code)] + pub(crate) fn state(&self) -> VgpuState { + self.state + } +} diff --git a/drivers/gpu/nova-core/vgpu/hal.rs b/drivers/gpu/nova-core/vgpu/hal.rs new file mode 100644 index 000000000000..456f8fe27582 --- /dev/null +++ b/drivers/gpu/nova-core/vgpu/hal.rs @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 + +use crate::gpu::{ + Architecture, + Chipset, // +}; + +mod gb202; +mod tu102; + +pub(super) trait VgpuHal { + /// Returns whether this chipset can support vGPU. + fn supports_vgpu(&self) -> bool; +} + +pub(super) fn vgpu_hal(chipset: Chipset) -> &'static dyn VgpuHal { + match chipset.arch() { + Architecture::BlackwellGB20x => gb202::GB202_HAL, + Architecture::Turing + | Architecture::Ampere + | Architecture::Hopper + | Architecture::Ada + | Architecture::BlackwellGB10x => tu102::TU102_HAL, + } +} diff --git a/drivers/gpu/nova-core/vgpu/hal/gb202.rs b/drivers/gpu/nova-core/vgpu/hal/gb202.rs new file mode 100644 index 000000000000..3add8af26616 --- /dev/null +++ b/drivers/gpu/nova-core/vgpu/hal/gb202.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use crate::vgpu::hal::VgpuHal; + +struct Gb202; + +impl VgpuHal for Gb202 { + fn supports_vgpu(&self) -> bool { + true + } +} + +const GB202: Gb202 = Gb202; +pub(super) const GB202_HAL: &dyn VgpuHal = &GB202; diff --git a/drivers/gpu/nova-core/vgpu/hal/tu102.rs b/drivers/gpu/nova-core/vgpu/hal/tu102.rs new file mode 100644 index 000000000000..baeea3ac5754 --- /dev/null +++ b/drivers/gpu/nova-core/vgpu/hal/tu102.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use crate::vgpu::hal::VgpuHal; + +struct Tu102; + +impl VgpuHal for Tu102 { + fn supports_vgpu(&self) -> bool { + false + } +} + +const TU102: Tu102 = Tu102; +pub(super) const TU102_HAL: &dyn VgpuHal = &TU102; From f6f0d2d461fdd58e2f920e9efc8c7f482f81fb2e Mon Sep 17 00:00:00 2001 From: Zhi Wang Date: Wed, 22 Jul 2026 10:39:12 +0300 Subject: [PATCH 108/137] gpu: nova-core: set RMSetSriovMode for vGPU The GSP registry setup needs to advertise SR-IOV mode when nova-core boots GSP for an enabled vGPU configuration. Without the registry entry, GSP-RM is not told to initialize in the mode required by NVIDIA vGPU. Append RMSetSriovMode to the SetRegistry command when the vGPU state detected before GSP boot is enabled. Keep the existing registry entries unchanged for non-vGPU boots. Cc: Alexandre Courbot Signed-off-by: Zhi Wang Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260722073913.1807677-6-zhiw@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/gsp.rs | 1 - drivers/gpu/nova-core/gsp/boot.rs | 2 +- drivers/gpu/nova-core/gsp/commands.rs | 14 +++++++++++++- drivers/gpu/nova-core/vgpu.rs | 1 - 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index d3c97da5848c..b403dc3515a5 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -68,7 +68,6 @@ pub(crate) struct GspBootContext<'ctx, 'gpu> { pub(crate) gsp_falcon: &'ctx Falcon<'gpu, GspFalcon>, pub(crate) sec2_falcon: &'ctx Falcon<'gpu, Sec2Falcon>, pub(crate) fsp: Option<&'ctx mut Fsp<'gpu>>, - #[expect(dead_code)] pub(crate) vgpu: &'ctx VgpuManager, } diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 6b7d00205000..a2e8342e7760 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -89,7 +89,7 @@ pub(crate) fn boot( self.cmdq .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev, chipset))?; self.cmdq - .send_command_no_wait(bar, commands::SetRegistry::new()?)?; + .send_command_no_wait(bar, commands::SetRegistry::new(ctx.vgpu.state())?)?; hal.post_boot(&self, ctx, &gsp_fw)?; diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index 08380de39048..134f34b19174 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -34,6 +34,7 @@ }, }, sbuffer::SBufferIter, + vgpu::VgpuState, // }; /// The `GspSetSystemInfo` command. @@ -72,7 +73,7 @@ pub(crate) struct SetRegistry { impl SetRegistry { /// Creates a new `SetRegistry` command, using a set of hardcoded entries. - pub(crate) fn new() -> Result { + pub(crate) fn new(vgpu_state: VgpuState) -> Result { let mut entries = KVec::new(); // RMSecBusResetEnable - enables PCI secondary bus reset @@ -104,6 +105,17 @@ pub(crate) fn new() -> Result { GFP_KERNEL, )?; + if matches!(vgpu_state, VgpuState::Enabled { .. }) { + // RMSetSriovMode - required when vGPU is enabled. + entries.push( + RegistryEntry { + key: "RMSetSriovMode", + value: 1, + }, + GFP_KERNEL, + )?; + } + Ok(Self { entries }) } } diff --git a/drivers/gpu/nova-core/vgpu.rs b/drivers/gpu/nova-core/vgpu.rs index 71fd0fe879c2..5359847e2eb9 100644 --- a/drivers/gpu/nova-core/vgpu.rs +++ b/drivers/gpu/nova-core/vgpu.rs @@ -86,7 +86,6 @@ fn detect_state( } /// Returns the detected vGPU state for this boot. - #[expect(dead_code)] pub(crate) fn state(&self) -> VgpuState { self.state } From 6dcbb4b1320fa91fee349462a52bb69135f2e45e Mon Sep 17 00:00:00 2001 From: Zhi Wang Date: Wed, 22 Jul 2026 10:39:13 +0300 Subject: [PATCH 109/137] gpu: nova-core: reserve vGPU WPR2 heap GSP-RM needs a larger WPR2 heap when booting in vGPU mode. The heap size is firmware-dependent, so it should come from the generated firmware bindings instead of being open-coded in nova-core. Pass the detected vGPU state into the framebuffer layout calculation. Keep baremetal boots on the existing heap sizing path, and use the 570.144 vGPU default heap binding only when vGPU is enabled. The same state match also sets the VF partition count, so disabled and invalid 0/1-VF states do not enter the vGPU heap path. Cc: Alexandre Courbot Signed-off-by: Zhi Wang Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260722073913.1807677-7-zhiw@nvidia.com [ Use checked arithmetic to calculate wpr2_heap_addr. - Danilo ] Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/fb.rs | 31 +++++++++++++++---- drivers/gpu/nova-core/gsp/boot.rs | 2 +- drivers/gpu/nova-core/gsp/fw.rs | 5 +++ .../gpu/nova-core/gsp/fw/r570_144/bindings.rs | 1 + drivers/gpu/nova-core/vgpu.rs | 1 - 5 files changed, 32 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 273cff752fae..365db7abf7db 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -23,7 +23,8 @@ firmware::gsp::GspFirmware, gpu::Chipset, gsp, - num::FromSafeCast, // + num::FromSafeCast, + vgpu::VgpuState, // }; mod hal; @@ -171,7 +172,12 @@ pub(crate) struct FbLayout { impl FbLayout { /// Computes the FB layout for `chipset` required to run the `gsp_fw` GSP firmware. - pub(crate) fn new(chipset: Chipset, bar: Bar0<'_>, gsp_fw: &GspFirmware) -> Result { + pub(crate) fn new( + chipset: Chipset, + bar: Bar0<'_>, + gsp_fw: &GspFirmware, + vgpu_state: VgpuState, + ) -> Result { let hal = hal::fb_hal(chipset); let fb = { @@ -234,11 +240,24 @@ pub(crate) fn new(chipset: Chipset, bar: Bar0<'_>, gsp_fw: &GspFirmware) -> Resu FbRange(elf_addr..elf_addr + elf_size) }; + let (vf_partition_count, wpr2_heap_size) = match vgpu_state { + VgpuState::Disabled => ( + 0, + gsp::LibosParams::from_chipset(chipset).wpr_heap_size(chipset, fb.end)?, + ), + VgpuState::Enabled { total_vfs } => ( + u8::try_from(total_vfs.get()).map_err(|_| EINVAL)?, + gsp::LibosParams::vgpu_wpr_heap_size(), + ), + }; + let wpr2_heap = { const WPR2_HEAP_DOWN_ALIGN: Alignment = Alignment::new::(); - let wpr2_heap_size = - gsp::LibosParams::from_chipset(chipset).wpr_heap_size(chipset, fb.end)?; - let wpr2_heap_addr = (elf.start - wpr2_heap_size).align_down(WPR2_HEAP_DOWN_ALIGN); + let wpr2_heap_addr = elf + .start + .checked_sub(wpr2_heap_size) + .ok_or(EOVERFLOW)? + .align_down(WPR2_HEAP_DOWN_ALIGN); FbRange(wpr2_heap_addr..(elf.start).align_down(WPR2_HEAP_DOWN_ALIGN)) }; @@ -265,7 +284,7 @@ pub(crate) fn new(chipset: Chipset, bar: Bar0<'_>, gsp_fw: &GspFirmware) -> Resu wpr2_heap, wpr2, heap, - vf_partition_count: 0, + vf_partition_count, pmu_reserved_size: hal.pmu_reserved_size(), }) } diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index a2e8342e7760..727b8ae4bcb7 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -50,7 +50,7 @@ pub(crate) fn boot( let gsp_fw = KBox::pin_init(GspFirmware::new(dev, chipset, FIRMWARE_VERSION), GFP_KERNEL)?; - let fb_layout = FbLayout::new(chipset, bar, &gsp_fw)?; + let fb_layout = FbLayout::new(chipset, bar, &gsp_fw, ctx.vgpu.state())?; dev_dbg!(dev, "{:#x?}\n", fb_layout); let wpr_meta = Coherent::init(dev, GFP_KERNEL, GspFwWprMeta::new(&gsp_fw, &fb_layout))?; diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 166b0c69edb7..6e8e7d822ef1 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -132,6 +132,11 @@ pub(crate) fn from_chipset(chipset: Chipset) -> &'static LibosParams { } } + /// Returns the WPR heap size to reserve when vGPU is enabled. + pub(crate) fn vgpu_wpr_heap_size() -> u64 { + u64::from(bindings::GSP_FW_HEAP_SIZE_VGPU_DEFAULT) + } + /// Returns the amount of memory (in bytes) to allocate for the WPR heap for a framebuffer size /// of `fb_size` (in bytes) for `chipset`. pub(crate) fn wpr_heap_size(&self, chipset: Chipset, fb_size: u64) -> Result { diff --git a/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs b/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs index ea350f9b2cc4..afe3e007f088 100644 --- a/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs +++ b/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs @@ -40,6 +40,7 @@ fn fmt(&self, fmt: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { pub const GSP_FW_HEAP_PARAM_BASE_RM_SIZE_GH100: u32 = 14680064; pub const GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB: u32 = 98304; pub const GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE: u32 = 100663296; +pub const GSP_FW_HEAP_SIZE_VGPU_DEFAULT: u32 = 609222656; pub const GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MIN_MB: u32 = 64; pub const GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MAX_MB: u32 = 256; pub const GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MIN_MB: u32 = 88; diff --git a/drivers/gpu/nova-core/vgpu.rs b/drivers/gpu/nova-core/vgpu.rs index 5359847e2eb9..6b7e045acea8 100644 --- a/drivers/gpu/nova-core/vgpu.rs +++ b/drivers/gpu/nova-core/vgpu.rs @@ -26,7 +26,6 @@ pub(crate) enum VgpuState { /// vGPU mode is enabled for this boot. Enabled { /// Total number of SR-IOV VFs supported by this device. - #[expect(dead_code)] total_vfs: NonZero, }, } From 93b9511a3bba7f31d95502e5f912f0a476b0cf4a Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Thu, 23 Jul 2026 15:54:46 +0900 Subject: [PATCH 110/137] gpu: nova-core: fix packed registry table size `PACKED_REGISTRY_TABLE::size` describes the entire table, including its fixed-size header. `SetRegistry` currently initializes it with only the variable payload length, omitting the 8 bytes header. Fix this by using `CommandToGsp::size` to obtain the actual command size, including its header. Fixes: 19b0a6e7c2be ("gpu: nova-core: gsp: Add SetRegistry command") Reported-by: Sashiko Closes: https://lore.kernel.org/r/20260722075253.B6DDB1F00A3D@smtp.kernel.org Reviewed-by: Danilo Krummrich Link: https://patch.msgid.link/20260723-nova-registry-size-fix-v1-1-8f471ba00ab4@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/commands.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index 134f34b19174..ffc25fd8c47b 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -127,10 +127,7 @@ impl CommandToGsp for SetRegistry { type InitError = Infallible; fn init(&self) -> impl Init { - Self::Command::init( - self.entries.len() as u32, - self.variable_payload_len() as u32, - ) + Self::Command::init(self.entries.len() as u32, self.size() as u32) } fn variable_payload_len(&self) -> usize { From 7c9953b8681db62b1cc22afbfe0d6da9445b1e1c Mon Sep 17 00:00:00 2001 From: Antonin Malzieu Ridolfi Date: Mon, 27 Jul 2026 17:51:51 +0200 Subject: [PATCH 111/137] gpu: nova-core: Add function to query WPR2 range Create new function abstracting WPR2 region range query. Refactor gsp hal tu102 to query the WPR2 region range using this new function. Suggested-by: Alexandre Courbot Signed-off-by: Antonin Malzieu Ridolfi Link: https://patch.msgid.link/20260727-nova-core-regs-split-v2-1-21b5e6e32ea5@nanonej.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fb.rs | 14 +++++++ drivers/gpu/nova-core/gsp/hal/tu102.rs | 54 ++++++++++++-------------- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 365db7abf7db..8934efc5f436 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -289,3 +289,17 @@ pub(crate) fn new( }) } } + +/// Reads the WPR2 memory region registers and returns the range if set. +/// Returns `None` if the WPR2 region is not set. +pub(crate) fn wpr2_range(bar: Bar0<'_>) -> Option> { + let wpr2_hi = bar.read(crate::regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI); + + if !wpr2_hi.is_wpr2_set() { + return None; + } + + let wpr2_lo = bar.read(crate::regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO); + + Some(wpr2_lo.lower_bound()..wpr2_hi.higher_bound()) +} diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index 648657e248da..03861add8e20 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -17,7 +17,10 @@ sec2::Sec2, Falcon, // }, - fb::FbLayout, + fb::{ + wpr2_range, + FbLayout, // + }, firmware::{ booter::{ BooterFirmware, @@ -90,9 +93,8 @@ fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result { .inspect_err(|e| dev_err!(dev, "FWSEC-SB failed to run: {:?}\n", e)); // Remove WPR2 region if set. - let wpr2_hi = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI); let booter_unloader_res = (|| { - if !wpr2_hi.is_wpr2_set() { + if wpr2_range(bar).is_none() { return Ok(()); } @@ -110,8 +112,7 @@ fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result { } // Confirm that the WPR2 region has been removed. - let wpr2_hi = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI); - if wpr2_hi.is_wpr2_set() { + if wpr2_range(bar).is_some() { dev_err!( dev, "WPR2 region still set after Booter Unloader returned\n" @@ -146,7 +147,7 @@ fn run_fwsec_frts( ) -> Result { // Check that the WPR2 region does not already exist - if it does, we cannot run // FWSEC-FRTS until the GPU is reset. - if bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound() != 0 { + if wpr2_range(bar).is_some() { dev_err!( dev, "WPR2 region already exists - GPU needs to be reset to proceed\n" @@ -189,34 +190,27 @@ fn run_fwsec_frts( } // Check that the WPR2 region has been created as we requested. - let (wpr2_lo, wpr2_hi) = ( - bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO).lower_bound(), - bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound(), - ); + let Some(wpr2_range) = wpr2_range(bar) else { + dev_err!(dev, "WPR2 region not created after running FWSEC-FRTS\n"); - match (wpr2_lo, wpr2_hi) { - (_, 0) => { - dev_err!(dev, "WPR2 region not created after running FWSEC-FRTS\n"); + return Err(EIO); + }; - Err(EIO) - } - (wpr2_lo, _) if wpr2_lo != fb_layout.frts.start => { - dev_err!( - dev, - "WPR2 region created at unexpected address {:#x}; expected {:#x}\n", - wpr2_lo, - fb_layout.frts.start, - ); + if wpr2_range.start != fb_layout.frts.start { + dev_err!( + dev, + "WPR2 region created at unexpected address {:#x}; expected {:#x}\n", + wpr2_range.start, + fb_layout.frts.start, + ); - Err(EIO) - } - (wpr2_lo, wpr2_hi) => { - dev_dbg!(dev, "WPR2: {:#x}-{:#x}\n", wpr2_lo, wpr2_hi); - dev_dbg!(dev, "GPU instance built\n"); - - Ok(()) - } + return Err(EIO); } + + dev_dbg!(dev, "WPR2: {:#x}-{:#x}\n", wpr2_range.start, wpr2_range.end); + dev_dbg!(dev, "GPU instance built\n"); + + Ok(()) } /// Load and prepare the resources required to properly reset the GSP after it has been stopped. From 81aa6f190aa0e9ff752fcd0a9eda676458e23133 Mon Sep 17 00:00:00 2001 From: Antonin Malzieu Ridolfi Date: Mon, 27 Jul 2026 17:51:52 +0200 Subject: [PATCH 112/137] gpu: nova-core: Move PFB registers definitions Move PFB registers definitions into fb module and update registers visibility. Suggested-by: Alexandre Courbot Suggested-by: Danilo Krummrich Signed-off-by: Antonin Malzieu Ridolfi Link: https://patch.msgid.link/20260727-nova-core-regs-split-v2-2-21b5e6e32ea5@nanonej.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fb.rs | 4 +- drivers/gpu/nova-core/fb/hal/ga100.rs | 8 +- drivers/gpu/nova-core/fb/hal/gb100.rs | 6 +- drivers/gpu/nova-core/fb/hal/gb202.rs | 6 +- drivers/gpu/nova-core/fb/hal/gh100.rs | 6 +- drivers/gpu/nova-core/fb/hal/tu102.rs | 8 +- drivers/gpu/nova-core/fb/regs.rs | 132 +++++++++++++++++++++++++- drivers/gpu/nova-core/regs.rs | 127 ------------------------- 8 files changed, 155 insertions(+), 142 deletions(-) diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 8934efc5f436..9e475efb1150 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -293,13 +293,13 @@ pub(crate) fn new( /// Reads the WPR2 memory region registers and returns the range if set. /// Returns `None` if the WPR2 region is not set. pub(crate) fn wpr2_range(bar: Bar0<'_>) -> Option> { - let wpr2_hi = bar.read(crate::regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI); + let wpr2_hi = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI); if !wpr2_hi.is_wpr2_set() { return None; } - let wpr2_lo = bar.read(crate::regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO); + let wpr2_lo = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO); Some(wpr2_lo.lower_bound()..wpr2_hi.higher_bound()) } diff --git a/drivers/gpu/nova-core/fb/hal/ga100.rs b/drivers/gpu/nova-core/fb/hal/ga100.rs index 3cc1caf361c7..d13c9a826eef 100644 --- a/drivers/gpu/nova-core/fb/hal/ga100.rs +++ b/drivers/gpu/nova-core/fb/hal/ga100.rs @@ -9,8 +9,10 @@ use crate::{ driver::Bar0, - fb::hal::FbHal, - regs, // + fb::{ + hal::FbHal, + regs, // + }, }; use super::tu102::FLUSH_SYSMEM_ADDR_SHIFT; @@ -41,7 +43,7 @@ pub(super) fn write_sysmem_flush_page_ga100(bar: Bar0<'_>, addr: u64) { } pub(super) fn display_enabled_ga100(bar: Bar0<'_>) -> bool { - !bar.read(regs::ga100::NV_FUSE_STATUS_OPT_DISPLAY) + !bar.read(crate::regs::ga100::NV_FUSE_STATUS_OPT_DISPLAY) .display_disabled() } diff --git a/drivers/gpu/nova-core/fb/hal/gb100.rs b/drivers/gpu/nova-core/fb/hal/gb100.rs index 6e0eba101ca1..ec55ec3fc7e1 100644 --- a/drivers/gpu/nova-core/fb/hal/gb100.rs +++ b/drivers/gpu/nova-core/fb/hal/gb100.rs @@ -22,9 +22,11 @@ use crate::{ driver::Bar0, - fb::hal::FbHal, + fb::{ + hal::FbHal, + regs, // + }, num::usize_into_u32, - regs, // }; struct Gb100; diff --git a/drivers/gpu/nova-core/fb/hal/gb202.rs b/drivers/gpu/nova-core/fb/hal/gb202.rs index b78e0970f66d..69ba35d2ea08 100644 --- a/drivers/gpu/nova-core/fb/hal/gb202.rs +++ b/drivers/gpu/nova-core/fb/hal/gb202.rs @@ -12,8 +12,10 @@ use crate::{ driver::Bar0, - fb::hal::FbHal, - regs, // + fb::{ + hal::FbHal, + regs, // + }, }; struct Gb202; diff --git a/drivers/gpu/nova-core/fb/hal/gh100.rs b/drivers/gpu/nova-core/fb/hal/gh100.rs index d39fe99537ed..2867ae058d0a 100644 --- a/drivers/gpu/nova-core/fb/hal/gh100.rs +++ b/drivers/gpu/nova-core/fb/hal/gh100.rs @@ -10,8 +10,10 @@ use crate::{ driver::Bar0, - fb::hal::FbHal, - regs, // + fb::{ + hal::FbHal, + regs, // + }, }; struct Gh100; diff --git a/drivers/gpu/nova-core/fb/hal/tu102.rs b/drivers/gpu/nova-core/fb/hal/tu102.rs index f629e8e9d5d5..541f163b52d3 100644 --- a/drivers/gpu/nova-core/fb/hal/tu102.rs +++ b/drivers/gpu/nova-core/fb/hal/tu102.rs @@ -9,8 +9,10 @@ use crate::{ driver::Bar0, - fb::hal::FbHal, - regs, // + fb::{ + hal::FbHal, + regs, // + }, }; /// Shift applied to the sysmem address before it is written into `NV_PFB_NISO_FLUSH_SYSMEM_ADDR`, @@ -31,7 +33,7 @@ pub(super) fn write_sysmem_flush_page_gm107(bar: Bar0<'_>, addr: u64) -> Result } pub(super) fn display_enabled_gm107(bar: Bar0<'_>) -> bool { - !bar.read(regs::gm107::NV_FUSE_STATUS_OPT_DISPLAY) + !bar.read(crate::regs::gm107::NV_FUSE_STATUS_OPT_DISPLAY) .display_disabled() } diff --git a/drivers/gpu/nova-core/fb/regs.rs b/drivers/gpu/nova-core/fb/regs.rs index b2ec02f584be..95adbe124a30 100644 --- a/drivers/gpu/nova-core/fb/regs.rs +++ b/drivers/gpu/nova-core/fb/regs.rs @@ -1,6 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 -use kernel::io::register; +use kernel::{ + io::register, + sizes::SizeConstants, // +}; // PDISP @@ -23,3 +26,130 @@ pub(super) fn vga_workspace_addr(self) -> Option { } } } + +// PFB + +register! { + /// Low bits of the physical system memory address used by the GPU to perform sysmembar + /// operations (see [`crate::fb::SysmemFlush`]). + pub(super) NV_PFB_NISO_FLUSH_SYSMEM_ADDR(u32) @ 0x00100c10 { + 31:0 adr_39_08; + } + + /// High bits of the physical system memory address used by the GPU to perform sysmembar + /// operations. + pub(super) NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x00100c40 { + 23:0 adr_63_40; + } + + pub(super) NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE(u32) @ 0x00100ce0 { + 30:30 ecc_mode_enabled => bool; + 9:4 lower_mag; + 3:0 lower_scale; + } + + pub(super) NV_PFB_PRI_MMU_WPR2_ADDR_LO(u32) @ 0x001fa824 { + /// Bits 12..40 of the lower (inclusive) bound of the WPR2 region. + 31:4 lo_val; + } + + pub(super) NV_PFB_PRI_MMU_WPR2_ADDR_HI(u32) @ 0x001fa828 { + /// Bits 12..40 of the higher (exclusive) bound of the WPR2 region. + 31:4 hi_val; + } +} + +/// Base of the GB10x HSHUB0 register window (`NV_HSHUB0_PRIV_BASE` in Open RM). +/// +/// The base is provided by the GB10x framebuffer HAL. +pub(super) struct Hshub0Base(()); + +register! { + // GB10x sysmem flush registers, relative to the HSHUB0 base. GB10x routes sysmembar + // through a primary and an EG (egress) pair that must both be programmed to the same + // address. Hardware ignores bits 7:0 of each LO register. The boot path uses a fixed + // HSHUB0 base, so the multiple runtime-discovered HSHUB bases are not needed here. + pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x00000e50 { + 31:0 adr => u32; + } + + pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x00000e54 { + 19:0 adr; + } + + pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x000006c0 { + 31:0 adr => u32; + } + + pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x000006c4 { + 19:0 adr; + } +} + +register! { + // GB20x FBHUB0 sysmem flush registers. Unlike the older + // NV_PFB_NISO_FLUSH_SYSMEM_ADDR registers, which encode the address with an + // 8-bit right-shift, these take the raw address split into lower and upper + // halves. Hardware ignores bits 7:0 of the LO register. + pub(super) NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x008a1d58 { + 31:0 adr => u32; + } + + pub(super) NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x008a1d5c { + 19:0 adr; + } +} + +register! { + /// Low bits of the physical system memory address used by the GPU to perform + /// sysmembar operations on Hopper. + /// + /// Like the GB20x FBHUB0 registers, and unlike the Ampere + /// `NV_PFB_NISO_FLUSH_SYSMEM_ADDR` registers (which encode the address with an + /// 8-bit right-shift), these take the raw address split into lower and upper + /// halves. Hardware ignores bits 7:0 of the LO register. + pub(super) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x00100a34 { + 31:0 adr => u32; + } + + /// High bits of the physical system memory address used by the GPU to perform + /// sysmembar operations on Hopper. + pub(super) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x00100a38 { + 19:0 adr; + } +} + +impl NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE { + /// Returns the usable framebuffer size, in bytes. + pub(super) fn usable_fb_size(self) -> u64 { + let size = (u64::from(self.lower_mag()) << u64::from(self.lower_scale())) * u64::SZ_1M; + + if self.ecc_mode_enabled() { + // Remove the amount of memory reserved for ECC (one per 16 units). + size / 16 * 15 + } else { + size + } + } +} + +impl NV_PFB_PRI_MMU_WPR2_ADDR_LO { + /// Returns the lower (inclusive) bound of the WPR2 region. + pub(super) fn lower_bound(self) -> u64 { + u64::from(self.lo_val()) << 12 + } +} + +impl NV_PFB_PRI_MMU_WPR2_ADDR_HI { + /// Returns the higher (exclusive) bound of the WPR2 region. + /// + /// A value of zero means the WPR2 region is not set. + pub(super) fn higher_bound(self) -> u64 { + u64::from(self.hi_val()) << 12 + } + + /// Returns whether the WPR2 region is currently set. + pub(super) fn is_wpr2_set(self) -> bool { + self.hi_val() != 0 + } +} diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index 49591c3dcfa7..d58dc6dd0f04 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -116,133 +116,6 @@ fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { } } -// PFB - -register! { - /// Low bits of the physical system memory address used by the GPU to perform sysmembar - /// operations (see [`crate::fb::SysmemFlush`]). - pub(crate) NV_PFB_NISO_FLUSH_SYSMEM_ADDR(u32) @ 0x00100c10 { - 31:0 adr_39_08; - } - - /// High bits of the physical system memory address used by the GPU to perform sysmembar - /// operations. - pub(crate) NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x00100c40 { - 23:0 adr_63_40; - } - - pub(crate) NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE(u32) @ 0x00100ce0 { - 30:30 ecc_mode_enabled => bool; - 9:4 lower_mag; - 3:0 lower_scale; - } - - pub(crate) NV_PFB_PRI_MMU_WPR2_ADDR_LO(u32) @ 0x001fa824 { - /// Bits 12..40 of the lower (inclusive) bound of the WPR2 region. - 31:4 lo_val; - } - - pub(crate) NV_PFB_PRI_MMU_WPR2_ADDR_HI(u32) @ 0x001fa828 { - /// Bits 12..40 of the higher (exclusive) bound of the WPR2 region. - 31:4 hi_val; - } -} - -/// Base of the GB10x HSHUB0 register window (`NV_HSHUB0_PRIV_BASE` in Open RM). -/// -/// The base is provided by the GB10x framebuffer HAL. -pub(crate) struct Hshub0Base(()); - -register! { - // GB10x sysmem flush registers, relative to the HSHUB0 base. GB10x routes sysmembar - // through a primary and an EG (egress) pair that must both be programmed to the same - // address. Hardware ignores bits 7:0 of each LO register. The boot path uses a fixed - // HSHUB0 base, so the multiple runtime-discovered HSHUB bases are not needed here. - pub(crate) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x00000e50 { - 31:0 adr => u32; - } - - pub(crate) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x00000e54 { - 19:0 adr; - } - - pub(crate) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x000006c0 { - 31:0 adr => u32; - } - - pub(crate) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x000006c4 { - 19:0 adr; - } -} - -register! { - // GB20x FBHUB0 sysmem flush registers. Unlike the older - // NV_PFB_NISO_FLUSH_SYSMEM_ADDR registers, which encode the address with an - // 8-bit right-shift, these take the raw address split into lower and upper - // halves. Hardware ignores bits 7:0 of the LO register. - pub(crate) NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x008a1d58 { - 31:0 adr => u32; - } - - pub(crate) NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x008a1d5c { - 19:0 adr; - } -} - -register! { - /// Low bits of the physical system memory address used by the GPU to perform - /// sysmembar operations on Hopper. - /// - /// Like the GB20x FBHUB0 registers, and unlike the Ampere - /// `NV_PFB_NISO_FLUSH_SYSMEM_ADDR` registers (which encode the address with an - /// 8-bit right-shift), these take the raw address split into lower and upper - /// halves. Hardware ignores bits 7:0 of the LO register. - pub(crate) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x00100a34 { - 31:0 adr => u32; - } - - /// High bits of the physical system memory address used by the GPU to perform - /// sysmembar operations on Hopper. - pub(crate) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x00100a38 { - 19:0 adr; - } -} - -impl NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE { - /// Returns the usable framebuffer size, in bytes. - pub(crate) fn usable_fb_size(self) -> u64 { - let size = (u64::from(self.lower_mag()) << u64::from(self.lower_scale())) * u64::SZ_1M; - - if self.ecc_mode_enabled() { - // Remove the amount of memory reserved for ECC (one per 16 units). - size / 16 * 15 - } else { - size - } - } -} - -impl NV_PFB_PRI_MMU_WPR2_ADDR_LO { - /// Returns the lower (inclusive) bound of the WPR2 region. - pub(crate) fn lower_bound(self) -> u64 { - u64::from(self.lo_val()) << 12 - } -} - -impl NV_PFB_PRI_MMU_WPR2_ADDR_HI { - /// Returns the higher (exclusive) bound of the WPR2 region. - /// - /// A value of zero means the WPR2 region is not set. - pub(crate) fn higher_bound(self) -> u64 { - u64::from(self.hi_val()) << 12 - } - - /// Returns whether the WPR2 region is currently set. - pub(crate) fn is_wpr2_set(self) -> bool { - self.hi_val() != 0 - } -} - // PGC6 register space. // // `GC6` is a GPU low-power state where VRAM is in self-refresh and the GPU is powered down (except From 849044a381323b141d81f6bcba628cbeb0bcff31 Mon Sep 17 00:00:00 2001 From: Antonin Malzieu Ridolfi Date: Mon, 27 Jul 2026 17:51:53 +0200 Subject: [PATCH 113/137] gpu: nova-core: Move one PBUS register definition Move NV_PBUS_SW_SCRATCH_0E_FRTS_ERR register definition into gsp module and update registers visibility. Suggested-by: Alexandre Courbot Suggested-by: Danilo Krummrich Signed-off-by: Antonin Malzieu Ridolfi Link: https://patch.msgid.link/20260727-nova-core-regs-split-v2-3-21b5e6e32ea5@nanonej.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/hal/tu102.rs | 2 +- drivers/gpu/nova-core/gsp/regs.rs | 11 +++++++++++ drivers/gpu/nova-core/regs.rs | 5 ----- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index 03861add8e20..03133f723faf 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -40,12 +40,12 @@ GspHal, UnloadBundle, // }, + regs, sequencer::GspSequencer, Gsp, GspBootContext, GspFwWprMeta, // }, - regs, vbios::Vbios, // }; diff --git a/drivers/gpu/nova-core/gsp/regs.rs b/drivers/gpu/nova-core/gsp/regs.rs index a76dea3c3ab0..9a48aa87e7fb 100644 --- a/drivers/gpu/nova-core/gsp/regs.rs +++ b/drivers/gpu/nova-core/gsp/regs.rs @@ -2,6 +2,8 @@ use kernel::io::register; +use crate::regs::NV_PBUS_SW_SCRATCH; + // PGSP register! { @@ -9,3 +11,12 @@ 31:0 address; } } + +// PBUS + +register! { + /// Scratch register 0xe used as FRTS firmware error code. + pub(super) NV_PBUS_SW_SCRATCH_0E_FRTS_ERR(u32) => NV_PBUS_SW_SCRATCH[0xe] { + 31:16 frts_err_code; + } +} diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index d58dc6dd0f04..caeef4d85874 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -109,11 +109,6 @@ fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { register! { pub(crate) NV_PBUS_SW_SCRATCH(u32)[64] @ 0x00001400 {} - - /// Scratch register 0xe used as FRTS firmware error code. - pub(crate) NV_PBUS_SW_SCRATCH_0E_FRTS_ERR(u32) => NV_PBUS_SW_SCRATCH[0xe] { - 31:16 frts_err_code; - } } // PGC6 register space. From 233f147985c887588c95ad85543e51c726fe2de9 Mon Sep 17 00:00:00 2001 From: Deborah Brouwer Date: Fri, 3 Jul 2026 17:28:44 -0700 Subject: [PATCH 114/137] rust: iommu: add device lifetime to IoPageTable Currently, using a raw IoPageTable is unsafe because the returned IoPageTable is not tied to the device driver binding lifetime. Since device drivers now receive a lifetime parameter <'bound> representing the interval during which a device driver is bound to its bus device, add a lifetime parameter to IoPageTable. This ensures that the returned IoPageTable cannot outlive the bus device binding. Also remove the option to create a page table as a device resource since currently Devres is not compatible with resources that have a lifetime parameter. This option can be restored once the lifetime-aware wrapper for devres is available and if a use-case appears for it. Suggested-by: Boris Brezillon Signed-off-by: Deborah Brouwer Reviewed-by: Danilo Krummrich Reviewed-by: Alice Ryhl Acked-by: Joerg Roedel Link: https://patch.msgid.link/20260703-pgtable_lt_b4-v3-1-e738e1f513a4@collabora.com Signed-off-by: Alice Ryhl --- rust/kernel/iommu/pgtable.rs | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/rust/kernel/iommu/pgtable.rs b/rust/kernel/iommu/pgtable.rs index c88e38fd938a..f5f2706d72fb 100644 --- a/rust/kernel/iommu/pgtable.rs +++ b/rust/kernel/iommu/pgtable.rs @@ -16,7 +16,6 @@ Bound, Device, // }, - devres::Devres, error::to_result, io::PhysAddr, prelude::*, // @@ -59,15 +58,16 @@ pub struct Config { /// # Invariants /// /// The pointer references a valid io page table. -pub struct IoPageTable { +pub struct IoPageTable<'a, F: IoPageTableFmt> { ptr: NonNull, + _dev: PhantomData<&'a Device>, _marker: PhantomData, } // SAFETY: `struct io_pgtable_ops` is not restricted to a single thread. -unsafe impl Send for IoPageTable {} +unsafe impl Send for IoPageTable<'_, F> {} // SAFETY: `struct io_pgtable_ops` may be accessed concurrently. -unsafe impl Sync for IoPageTable {} +unsafe impl Sync for IoPageTable<'_, F> {} /// The format used by this page table. pub trait IoPageTableFmt: 'static { @@ -75,25 +75,10 @@ pub trait IoPageTableFmt: 'static { const FORMAT: io_pgtable_fmt; } -impl IoPageTable { - /// Create a new `IoPageTable` as a device resource. - #[inline] - pub fn new( - dev: &Device, - config: Config, - ) -> impl PinInit>, Error> + '_ { - // SAFETY: Devres ensures that the value is dropped during device unbind. - Devres::new(dev, unsafe { Self::new_raw(dev, config) }) - } - +impl<'a, F: IoPageTableFmt> IoPageTable<'a, F> { /// Create a new `IoPageTable`. - /// - /// # Safety - /// - /// If successful, then the returned `IoPageTable` must be dropped before the device is - /// unbound. #[inline] - pub unsafe fn new_raw(dev: &Device, config: Config) -> Result> { + pub fn new(dev: &'a Device, config: Config) -> Result> { let mut raw_cfg = bindings::io_pgtable_cfg { quirks: config.quirks, pgsize_bitmap: config.pgsize_bitmap, @@ -118,6 +103,7 @@ pub unsafe fn new_raw(dev: &Device, config: Config) -> Result Drop for IoPageTable { +impl Drop for IoPageTable<'_, F> { fn drop(&mut self) { // SAFETY: The caller of `Self::ttbr()` promised that the page table is not live when this // destructor runs. @@ -255,7 +241,7 @@ impl IoPageTableFmt for ARM64LPAES1 { const FORMAT: io_pgtable_fmt = bindings::io_pgtable_fmt_ARM_64_LPAE_S1 as io_pgtable_fmt; } -impl IoPageTable { +impl IoPageTable<'_, ARM64LPAES1> { /// Access the `ttbr` field of the configuration. /// /// This is the physical address of the page table, which may be passed to the device that From 3e8d932a48edefc8a062700e7f2926e6d86645d4 Mon Sep 17 00:00:00 2001 From: Deborah Brouwer Date: Tue, 28 Jul 2026 11:39:23 -0700 Subject: [PATCH 115/137] drm/tyr: add resources to RegistrationData Currently Tyr is not storing any resources in its drm::Driver RegistrationData. Move Tyr's device-private resources and gpu information from drm::Driver::Data to drm::Driver::RegistrationData. This allows Tyr to access this data safely within the lifetime of its binding to its parent platform device and while registered with userspace. Reviewed-by: Daniel Almeida Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260728-fw-boot-b4-v10-1-9187aefa3f2f@collabora.com Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/driver.rs | 42 +++++++++++++++++------------------ drivers/gpu/drm/tyr/file.rs | 11 ++++----- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index 8348c6cd3929..728a8388d591 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -6,6 +6,7 @@ OptionalClk, // }, device::{ + Bound, Core, Device, DeviceContext, // @@ -27,10 +28,7 @@ regulator, regulator::Regulator, sizes::SZ_2M, - sync::{ - aref::ARef, - Mutex, // - }, + sync::Mutex, time, // }; @@ -53,13 +51,17 @@ #[pin_data(PinnedDrop)] pub(crate) struct TyrPlatformDriverData<'bound> { - _device: ARef, _reg: drm::Registration<'bound, TyrDrmDriver>, } +/// Data owned by the DRM [`Registration`]. +/// +/// This data can have references tied to the parent platform device binding scope +/// and is accessible only while the DRM device is registered with userspace. #[pin_data] -pub(crate) struct TyrDrmDeviceData { - pub(crate) pdev: ARef, +pub(crate) struct TyrDrmRegistrationData<'drm> { + /// Parent platform device. + pub(crate) pdev: &'drm platform::Device, #[pin] clks: Mutex, @@ -67,9 +69,10 @@ pub(crate) struct TyrDrmDeviceData { #[pin] regulators: Mutex, - /// Some information on the GPU. - /// - /// This is mainly queried by userspace, i.e.: Mesa. + /// GPU MMIO register mapping. + pub(crate) iomem: IoMem<'drm>, + + /// GPU information read from hardware during probe. pub(crate) gpu_info: GpuInfo, } @@ -134,10 +137,10 @@ fn probe<'bound>( // other threads of execution. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(pa_bits)?)? }; - let platform: ARef = pdev.into(); + let unreg_dev = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - let data = try_pin_init!(TyrDrmDeviceData { - pdev: platform.clone(), + let reg_data = try_pin_init!(TyrDrmRegistrationData { + pdev, clks <- new_mutex!(Clocks { core: core_clk, stacks: stacks_clk, @@ -147,18 +150,15 @@ fn probe<'bound>( _mali: mali_regulator, _sram: sram_regulator, }), + iomem, gpu_info, }); - let tdev = drm::UnregisteredDevice::::new(pdev, data)?; // SAFETY: `reg` is stored in `TyrPlatformDriverData` and dropped when the driver is // unbound; it is never forgotten. - let reg = unsafe { drm::Registration::new(pdev.as_ref(), tdev, (), 0)? }; + let reg = unsafe { drm::Registration::new(pdev.as_ref(), unreg_dev, reg_data, 0)? }; - let driver = TyrPlatformDriverData { - _device: reg.device().into(), - _reg: reg, - }; + let driver = TyrPlatformDriverData { _reg: reg }; // We need this to be dev_info!() because dev_dbg!() does not work at // all in Rust for now, and we need to see whether probe succeeded. @@ -184,8 +184,8 @@ fn drop(self: Pin<&mut Self>) {} #[vtable] impl drm::Driver for TyrDrmDriver { - type Data = TyrDrmDeviceData; - type RegistrationData<'a> = (); + type Data = (); + type RegistrationData<'drm> = TyrDrmRegistrationData<'drm>; type File = TyrDrmFileData; type Object = drm::gem::shmem::Object; type ParentDevice = platform::Device; diff --git a/drivers/gpu/drm/tyr/file.rs b/drivers/gpu/drm/tyr/file.rs index b686041d5d6b..9f60a90d4948 100644 --- a/drivers/gpu/drm/tyr/file.rs +++ b/drivers/gpu/drm/tyr/file.rs @@ -12,7 +12,8 @@ use crate::driver::{ TyrDrmDevice, - TyrDrmDriver, // + TyrDrmDriver, + TyrDrmRegistrationData, // }; #[pin_data] @@ -31,15 +32,15 @@ fn open(_dev: &drm::Device) -> Result>> { impl TyrDrmFileData { pub(crate) fn dev_query( - ddev: &TyrDrmDevice, - _reg_data: &(), + _ddev: &TyrDrmDevice, + reg_data: &TyrDrmRegistrationData<'_>, devquery: &mut uapi::drm_panthor_dev_query, _file: &TyrDrmFile, ) -> Result { if devquery.pointer == 0 { match devquery.type_ { uapi::drm_panthor_dev_query_type_DRM_PANTHOR_DEV_QUERY_GPU_INFO => { - devquery.size = core::mem::size_of_val(&ddev.gpu_info) as u32; + devquery.size = core::mem::size_of_val(®_data.gpu_info) as u32; Ok(0) } _ => Err(EINVAL), @@ -53,7 +54,7 @@ pub(crate) fn dev_query( ) .writer(); - writer.write(&ddev.gpu_info)?; + writer.write(®_data.gpu_info)?; Ok(0) } From ef0ef3184c4d4f21a9f774d8de7e3a49ba40a4d5 Mon Sep 17 00:00:00 2001 From: Boris Brezillon Date: Tue, 28 Jul 2026 11:39:24 -0700 Subject: [PATCH 116/137] drm/tyr: add a generic slot manager Introduce a generic slot manager to dynamically allocate limited hardware slots to software "seats". It can be used for both address space (AS) and command stream group (CSG) slots. The slot manager initially assigns seats to its free slots. It will continue to reuse the same slot for a seat, as long as another seat does not start to use the slot in the interim. When contention arises because all of the slots are allocated, the slot manager will lazily evict and reuse slots that have become idle (if any). The seat state is protected using the LockedBy pattern with the same lock that guards the SlotManager. This ensures the seat state stays consistent across slot operations. Hardware specific behaviour is controlled through the SlotManager's specific manager type that implements the `SlotOperations` trait. Signed-off-by: Boris Brezillon Co-developed-by: Deborah Brouwer Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260728-fw-boot-b4-v10-2-9187aefa3f2f@collabora.com Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/slot.rs | 405 ++++++++++++++++++++++++++++++++++++ drivers/gpu/drm/tyr/tyr.rs | 1 + 2 files changed, 406 insertions(+) create mode 100644 drivers/gpu/drm/tyr/slot.rs diff --git a/drivers/gpu/drm/tyr/slot.rs b/drivers/gpu/drm/tyr/slot.rs new file mode 100644 index 000000000000..845846e07ee5 --- /dev/null +++ b/drivers/gpu/drm/tyr/slot.rs @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +//! Slot management abstraction for limited hardware resources. +//! +//! This module provides a generic [`SlotManager`] that assigns limited hardware +//! slots to logical "seats". A seat represents an entity (such as a virtual memory +//! (VM) address space) that needs access to a hardware slot. +//! +//! The [`SlotManager`] tracks slot allocation using sequence numbers (seqno) to detect +//! when a seat's binding has been invalidated. When a seat requests activation, +//! the manager will either reuse the seat's existing slot (if still valid), +//! allocate a free slot (if any are available), or evict the oldest idle slot if any +//! slots are idle. +//! +//! Hardware-specific behavior is customized by implementing the [`SlotOperations`] +//! trait, which allows callbacks when slots are activated or evicted. +//! +//! This is currently used for managing address space slots in the GPU, and it will +//! also be used to manage Command Stream Group (CSG) interface slots in the future. +//! +//! [SlotOperations]: crate::slot::SlotOperations +//! [SlotManager]: crate::slot::SlotManager +#![expect(dead_code)] + +use core::{ + mem, + ops::{ + Deref, + DerefMut, // + }, // +}; + +use kernel::{ + prelude::*, + sync::LockedBy, // +}; + +/// Seat information. +/// +/// This can't be accessed directly by the element embedding a `Seat`, +/// but is used by the generic slot manager logic to control residency +/// of a certain object on a hardware slot. +pub(crate) struct SeatInfo { + /// Slot used by this seat. + /// + /// This index is only valid if the slot pointed to by this index + /// has its `SlotInfo::seqno` match `SeatInfo::seqno`. Otherwise, + /// it means the object has been evicted from the hardware slot, + /// and a new slot needs to be acquired to make this object + /// resident again. + slot: u8, + + /// Sequence number encoding the last time this seat was active. + /// We also use it to check if a slot is still bound to a seat. + seqno: u64, +} + +/// Seat state. +/// +/// This is meant to be embedded in the object that wants to acquire +/// hardware slots. It also starts in the `Seat::NoSeat` state, and +/// the slot manager will change the object value when an active/evict +/// request is issued. +#[derive(Default)] +pub(crate) enum Seat { + #[expect(clippy::enum_variant_names)] + /// Resource is not resident. + /// + /// All objects start with a seat in the `Seat::NoSeat` state. The seat also + /// gets back to that state if the user requests eviction. It + /// can also end up in that state next time an operation is done + /// on a `Seat::Idle` seat and the slot manager finds out this + /// object has been evicted from the slot. + #[default] + NoSeat, + + /// Resource is actively used and resident. + /// + /// When a seat is in the `Seat::Active` state, it can't be evicted, and the + /// slot pointed to by `SeatInfo::slot` is guaranteed to be reserved + /// for this object as long as the seat stays active. + Active(SeatInfo), + + /// Resource is idle and might or might not be resident. + /// + /// When a seat is in the`Seat::Idle` state, we can't know for sure if the + /// object is resident or evicted until the next request we issue + /// to the slot manager. This tells the slot manager it can + /// reclaim the underlying slot if needed. + /// In order for the hardware to use this object again, the seat + /// needs to be turned into an `Seat::Active` state again + /// with a `SlotManager::activate()` call. + Idle(SeatInfo), +} + +impl Seat { + /// Get the slot index this seat is pointing to. + /// + /// If the seat is not `Seat::Active` we can't trust the + /// `SeatInfo`. In that case `None` is returned, otherwise + /// `Some(SeatInfo::slot)` is returned. + pub(crate) fn slot(&self) -> Option { + match self { + Self::Active(info) => Some(info.slot), + _ => None, + } + } +} + +/// Information related to a slot. +struct SlotInfo { + /// Type specific data attached to a slot. + slot_data: D, + + /// Sequence number from when this slot was last activated. + seqno: u64, +} + +/// Slot state. +#[derive(Default)] +enum Slot { + /// Slot is free. + #[default] + Free, + + /// Slot is active. + Active(SlotInfo), + + /// Slot is idle. + Idle(SlotInfo), +} + +pub(crate) type LockedSeat = LockedBy>; + +/// Trait describing the slot-related operations. +pub(crate) trait SlotOperations: Sized { + /// Implementation-specific data associated with each slot. + type SlotData; + + /// Returns the seat belonging to this slot data. + fn seat(slot_data: &Self::SlotData) -> &LockedSeat; + + /// Called when a slot is being activated for a seat. + fn activate(&mut self, _slot_idx: usize, _slot_data: &Self::SlotData) -> Result { + Ok(()) + } + + /// Called when a slot is being evicted and freed. + fn evict(&mut self, _slot_idx: usize, _slot_data: &Self::SlotData) -> Result { + Ok(()) + } +} + +/// A generic slot manager that provides access to a limited number of hardware slots. +pub(crate) struct SlotManager, const MAX_SLOTS: usize> { + /// A specific implementation of the generic slot manager. + manager: T, + + /// Number of slots actually available. + slot_count: usize, + + /// Slot array used to track the state of each slot. + slots: [Slot; MAX_SLOTS], + + /// Sequence number incremented each time a Seat is successfully activated + use_seqno: u64, +} + +impl, const MAX_SLOTS: usize> SlotManager { + /// Creates a specific instance of a slot manager. + pub(crate) fn new(manager: T, slot_count: usize) -> Result { + if slot_count == 0 { + return Err(EINVAL); + } + if slot_count > MAX_SLOTS { + return Err(EINVAL); + } + // Since the slot index is stored in SeatInfo as a u8, the maximum number of slots is 256. + if slot_count > u8::MAX as usize + 1 { + return Err(EINVAL); + } + + Ok(Self { + manager, + slot_count, + slots: [const { Slot::Free }; MAX_SLOTS], + use_seqno: 1, + }) + } + + /// Records a newly activated slot for the given seat. + /// The slot manager takes ownership of the hardware-specific slot data. + fn record_active_slot(&mut self, slot_idx: usize, slot_data: T::SlotData) { + let cur_seqno = self.use_seqno; + + *T::seat(&slot_data).access_mut(self) = Seat::Active(SeatInfo { + slot: slot_idx as u8, + seqno: cur_seqno, + }); + + self.slots[slot_idx] = Slot::Active(SlotInfo { + slot_data, + seqno: cur_seqno, + }); + + self.use_seqno += 1; + } + + /// Reactivates an active/idle slot for a given seat without reprogramming the hardware. + /// The SlotManager reuses the existing slot_data. This ensures that the hardware-specific + /// information is not changed between subsequent uses. It also ensures that resources + /// owned by the existing slot_data remain alive while the hardware is configured to use them. + fn reactivate_slot(&mut self, slot_idx: usize, slot_data: &T::SlotData) -> Result { + let cur_seqno = self.use_seqno; + + let mut slot_info = match mem::take(&mut self.slots[slot_idx]) { + Slot::Active(slot_info) | Slot::Idle(slot_info) => slot_info, + Slot::Free => { + *T::seat(slot_data).access_mut(self) = Seat::NoSeat; + return Err(EINVAL); + } + }; + + *T::seat(slot_data).access_mut(self) = Seat::Active(SeatInfo { + slot: slot_idx as u8, + seqno: cur_seqno, + }); + + slot_info.seqno = cur_seqno; + self.slots[slot_idx] = Slot::Active(slot_info); + + self.use_seqno += 1; + + Ok(()) + } + + /// Activates a slot for the given seat. + fn activate_slot(&mut self, slot_idx: usize, slot_data: T::SlotData) -> Result { + self.manager.activate(slot_idx, &slot_data)?; + self.record_active_slot(slot_idx, slot_data); + Ok(()) + } + + /// Finds a slot for the given seat. A free slot is preferred, but if none + /// are available, the oldest idle slot is evicted and reused. Otherwise, if + /// there are no free or idle slots, return [`EBUSY`]. + fn allocate_slot(&mut self, slot_data: T::SlotData) -> Result { + let slots = &self.slots[..self.slot_count]; + + let mut idle_slot_idx = None; + let mut idle_slot_seqno: u64 = 0; + + for (slot_idx, slot) in slots.iter().enumerate() { + match slot { + Slot::Free => { + return self.activate_slot(slot_idx, slot_data); + } + Slot::Idle(slot_info) => { + if idle_slot_idx.is_none() || slot_info.seqno < idle_slot_seqno { + idle_slot_idx = Some(slot_idx); + idle_slot_seqno = slot_info.seqno; + } + } + Slot::Active(_) => (), + } + } + + match idle_slot_idx { + Some(slot_idx) => { + // Lazily evict idle slot just before it is reused. + if let Slot::Idle(slot_info) = &self.slots[slot_idx] { + self.manager.evict(slot_idx, &slot_info.slot_data)?; + mem::take(&mut self.slots[slot_idx]); + } + self.activate_slot(slot_idx, slot_data) + } + None => Err(EBUSY), + } + } + + /// Converts an active slot and its seat to idle state. + fn idle_slot(&mut self, slot_idx: usize, locked_seat: &LockedSeat) -> Result { + let slot = mem::take(&mut self.slots[slot_idx]); + + self.slots[slot_idx] = match slot { + // If the slot was active, make it idle. + Slot::Active(slot_info) => Slot::Idle(slot_info), + + // Preserve an already-idle slot. + Slot::Idle(slot_info) => Slot::Idle(slot_info), + + // A free slot remains free. + Slot::Free => Slot::Free, + }; + + // If the seat was active, make it idle, or keep it idle if it was already idle. + *locked_seat.access_mut(self) = match locked_seat.access(self) { + Seat::Active(seat_info) | Seat::Idle(seat_info) => Seat::Idle(SeatInfo { + slot: seat_info.slot, + seqno: seat_info.seqno, + }), + Seat::NoSeat => Seat::NoSeat, + }; + Ok(()) + } + + /// Evicts an active or idle slot: calls the eviction callback and marks the slot as free + /// and the seat as NoSeat. + fn evict_slot(&mut self, slot_idx: usize, locked_seat: &LockedSeat) -> Result { + match &self.slots[slot_idx] { + Slot::Active(slot_info) | Slot::Idle(slot_info) => { + // If hardware eviction fails (e.g. times out), the slot retains + // its SlotData so that any resources still referenced by the hardware + // will remain alive. This prevents use-after-free errors. + self.manager.evict(slot_idx, &slot_info.slot_data)?; + mem::take(&mut self.slots[slot_idx]); + } + _ => (), + } + + *locked_seat.access_mut(self) = Seat::NoSeat; + Ok(()) + } + + /// Checks that the seat state matches the slot's state. + /// If they don't match, the seat is stale and is reset to `NoSeat`. + fn check_seat(&mut self, locked_seat: &LockedSeat) { + let (slot_idx, seat_seqno, is_active) = match locked_seat.access(self) { + Seat::Active(seat_info) => (seat_info.slot as usize, seat_info.seqno, true), + Seat::Idle(seat_info) => (seat_info.slot as usize, seat_info.seqno, false), + _ => return, + }; + + let valid = if is_active { + !kernel::warn_on!(!matches!( + &self.slots[slot_idx], + Slot::Active(slot_info) if slot_info.seqno == seat_seqno + )) + } else { + matches!( + &self.slots[slot_idx], + Slot::Idle(slot_info) if slot_info.seqno == seat_seqno + ) + }; + + if !valid { + *locked_seat.access_mut(self) = Seat::NoSeat; + } + } + + /// Activates a resource on any available/reclaimable slot. + pub(crate) fn activate(&mut self, slot_data: T::SlotData) -> Result { + self.check_seat(T::seat(&slot_data)); + + // Copy out only the slot index so the borrow of slot_data ends here. + let slot_idx = match T::seat(&slot_data).access(self) { + Seat::Active(seat_info) | Seat::Idle(seat_info) => Some(seat_info.slot as usize), + Seat::NoSeat => None, + }; + + match slot_idx { + Some(slot_idx) => self.reactivate_slot(slot_idx, &slot_data), + None => self.allocate_slot(slot_data), + } + } + + /// Flag a resource as idle. This method will be used for user VM support. + #[expect(dead_code)] + pub(crate) fn idle(&mut self, locked_seat: &LockedSeat) -> Result { + self.check_seat(locked_seat); + if let Seat::Active(seat_info) = locked_seat.access(self) { + self.idle_slot(seat_info.slot as usize, locked_seat)?; + } + Ok(()) + } + + /// Evict a resource from its slot. + pub(crate) fn evict(&mut self, locked_seat: &LockedSeat) -> Result { + self.check_seat(locked_seat); + + match locked_seat.access(self) { + Seat::Active(seat_info) | Seat::Idle(seat_info) => { + let slot_idx = seat_info.slot as usize; + self.evict_slot(slot_idx, locked_seat)?; + } + _ => (), + } + + Ok(()) + } +} + +impl, const MAX_SLOTS: usize> Deref for SlotManager { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.manager + } +} + +impl, const MAX_SLOTS: usize> DerefMut for SlotManager { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.manager + } +} diff --git a/drivers/gpu/drm/tyr/tyr.rs b/drivers/gpu/drm/tyr/tyr.rs index 95cda7b0962f..7c9a8063b3b9 100644 --- a/drivers/gpu/drm/tyr/tyr.rs +++ b/drivers/gpu/drm/tyr/tyr.rs @@ -12,6 +12,7 @@ mod gem; mod gpu; mod regs; +mod slot; kernel::module_platform_driver! { type: TyrPlatformDriver, From ae047468e047a67d5b6d4c1988da613d77f7a9dc Mon Sep 17 00:00:00 2001 From: Boris Brezillon Date: Tue, 28 Jul 2026 11:39:25 -0700 Subject: [PATCH 117/137] drm/tyr: add Memory Management Unit (MMU) support Add Memory Management Unit (MMU) support in Tyr. The MMU module wraps a SlotManager instance to allocate MMU address-space slots for use by virtual memory (VM) address spaces. The MMU's SlotManager uses an AddressSpaceManager to handle the hardware-specific callbacks. For example, the AddressSpaceManager activates and evicts VMs from slots by writing commands to the MMU registers. Add an implementation block for the MMU's MEMATTR register to provide a method for translating the Memory Attribute Indirection Register (MAIR) format from the pagetable configuration to a format understood by the MMU. Create an mmu instance during probe, it will be used by subsequent patches in this series. Wrap the iomem stored in TyrDrmRegistrationData in an Arc. The iomem is stored in the mmu through its AddressSpaceManager. In anticipation of the iomem also being stored in the firmware object, set up shared ownership of the iomem now. Update Kconfig to add the new MMU and IOMMU dependencies required by this MMU module. Signed-off-by: Boris Brezillon Co-developed-by: Deborah Brouwer Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260728-fw-boot-b4-v10-3-9187aefa3f2f@collabora.com Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/Kconfig | 3 + drivers/gpu/drm/tyr/driver.rs | 13 +- drivers/gpu/drm/tyr/mmu.rs | 121 ++++++ drivers/gpu/drm/tyr/mmu/address_space.rs | 511 +++++++++++++++++++++++ drivers/gpu/drm/tyr/regs.rs | 135 +++++- drivers/gpu/drm/tyr/slot.rs | 1 - drivers/gpu/drm/tyr/tyr.rs | 1 + 7 files changed, 780 insertions(+), 5 deletions(-) create mode 100644 drivers/gpu/drm/tyr/mmu.rs create mode 100644 drivers/gpu/drm/tyr/mmu/address_space.rs diff --git a/drivers/gpu/drm/tyr/Kconfig b/drivers/gpu/drm/tyr/Kconfig index 51a68ef8212c..61a2fd6f961a 100644 --- a/drivers/gpu/drm/tyr/Kconfig +++ b/drivers/gpu/drm/tyr/Kconfig @@ -5,9 +5,12 @@ config DRM_TYR depends on DRM=y depends on RUST depends on ARM || ARM64 || COMPILE_TEST + depends on MMU depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_LPAE depends on COMMON_CLK + depends on IOMMU_SUPPORT default n + select IOMMU_IO_PGTABLE_LPAE select RUST_DRM_GEM_SHMEM_HELPER help Rust DRM driver for ARM Mali CSF-based GPUs. diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index 728a8388d591..92c9d98e3aab 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -28,7 +28,10 @@ regulator, regulator::Regulator, sizes::SZ_2M, - sync::Mutex, + sync::{ + Arc, + Mutex, // + }, time, // }; @@ -37,6 +40,7 @@ gem::BoData, gpu, gpu::GpuInfo, + mmu::Mmu, regs::gpu_control::*, // }; @@ -70,7 +74,7 @@ pub(crate) struct TyrDrmRegistrationData<'drm> { regulators: Mutex, /// GPU MMIO register mapping. - pub(crate) iomem: IoMem<'drm>, + pub(crate) iomem: Arc>, /// GPU information read from hardware during probe. pub(crate) gpu_info: GpuInfo, @@ -121,7 +125,8 @@ fn probe<'bound>( let sram_regulator = Regulator::::get(pdev.as_ref(), c"sram")?; let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; - let iomem = request.iomap_sized::()?; + + let iomem = Arc::new(request.iomap_sized::()?, GFP_KERNEL)?; issue_soft_reset(pdev.as_ref(), &iomem)?; gpu::l2_power_on(pdev.as_ref(), &iomem)?; @@ -139,6 +144,8 @@ fn probe<'bound>( let unreg_dev = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + let _mmu = Mmu::new(pdev.as_ref(), iomem.as_arc_borrow(), &gpu_info)?; + let reg_data = try_pin_init!(TyrDrmRegistrationData { pdev, clks <- new_mutex!(Clocks { diff --git a/drivers/gpu/drm/tyr/mmu.rs b/drivers/gpu/drm/tyr/mmu.rs new file mode 100644 index 000000000000..83e022f9168b --- /dev/null +++ b/drivers/gpu/drm/tyr/mmu.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +//! Memory Management Unit (MMU) module. +//! +//! The GPU MMU provides a limited number of memory address spaces for use by command streams. +//! The MMU translates virtual addresses to physical addresses and manages memory configuration +//! and access permissions. +//! +//! This MMU module is essentially a locked wrapper around a [`SlotManager`] instance. +//! The [`SlotManager`] manages the assignment of virtual address spaces to hardware address-space +//! (AS) slots. MMU commands such as updates and flushes are carried out by the +//! [`AddressSpaceManager`] which actually writes to the MMU registers. +#![expect(dead_code)] + +use core::ops::Range; + +use kernel::{ + device::{ + Bound, + Device, // + }, + new_mutex, + prelude::*, + sync::{ + Arc, + ArcBorrow, + Mutex, // + }, // +}; + +use crate::{ + driver::IoMem, + gpu::GpuInfo, + mmu::address_space::{ + AddressSpaceManager, + VmAsData, // + }, + regs::{ + gpu_control::AS_PRESENT, + MAX_AS, // + }, + slot::SlotManager, // +}; + +pub(crate) mod address_space; + +pub(crate) type AsSlotManager<'drm> = SlotManager, MAX_AS>; + +/// Locked wrapper for carrying out virtual memory (VM) operations on the MMU. +#[pin_data] +pub(crate) struct Mmu<'drm> { + /// Slot Manager instance used to allocate hardware slots and write to MMU registers. + #[pin] + pub(crate) as_manager: Mutex>, +} + +impl<'drm> Mmu<'drm> { + /// Create an MMU component for this device. + pub(crate) fn new( + dev: &'drm Device, + iomem: ArcBorrow<'_, IoMem<'drm>>, + gpu_info: &GpuInfo, + ) -> Result>> { + let present = AS_PRESENT::from_raw(gpu_info.as_present).present().get(); + let slot_count = present.count_ones().try_into()?; + + let address_space_manager = AddressSpaceManager::new(dev, iomem.into(), present)?; + let as_slot_manager = + SlotManager::new(address_space_manager, slot_count).inspect_err(|e| { + dev_err!( + dev, + "Failed to initialize MMU slot manager with {} slots: {:?}", + slot_count, + e + ); + })?; + let mmu_init = try_pin_init!(Self{ + as_manager <- new_mutex!(as_slot_manager), + }); + Arc::pin_init(mmu_init, GFP_KERNEL) + } + + /// Assign a VM to an AS slot, provide a translation table, + /// and update the MMU to make the VM resident. + pub(crate) fn activate_vm(&self, vm_as_data: ArcBorrow<'_, VmAsData<'drm>>) -> Result { + self.as_manager.lock().activate_vm(vm_as_data) + } + + /// Evict a VM from its AS slot and flush the MMU. + pub(crate) fn deactivate_vm(&self, vm_as_data: &VmAsData<'drm>) -> Result { + self.as_manager.lock().deactivate_vm(vm_as_data) + } + + /// Flush MMU translation caches after a VM update. + pub(crate) fn flush_vm(&self, vm_as_data: &VmAsData<'drm>) -> Result { + self.as_manager.lock().flush_vm(vm_as_data) + } + + /// Flags the start of a VM update. + /// + /// If the VM is resident, any GPU access on the memory range being + /// updated will be blocked until `Mmu::end_vm_update()` is called. + /// This guarantees the atomicity of a VM update. + /// If the VM is not resident, this is a NOP. + pub(crate) fn start_vm_update( + &self, + vm_as_data: &VmAsData<'drm>, + region: &Range, + ) -> Result { + self.as_manager.lock().start_vm_update(vm_as_data, region) + } + + /// Flags the end of a VM update. + /// + /// If the VM is resident, this will let GPU accesses on the updated + /// range go through, in case any of them were blocked. + /// If the VM is not resident, this is a NOP. + pub(crate) fn end_vm_update(&self, vm_as_data: &VmAsData<'drm>) -> Result { + self.as_manager.lock().end_vm_update(vm_as_data) + } +} diff --git a/drivers/gpu/drm/tyr/mmu/address_space.rs b/drivers/gpu/drm/tyr/mmu/address_space.rs new file mode 100644 index 000000000000..76e3d2df6afd --- /dev/null +++ b/drivers/gpu/drm/tyr/mmu/address_space.rs @@ -0,0 +1,511 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +//! Address space module. +//! +//! This module handles the hardware interaction for MMU operations through +//! MMIO register access. +//! + +use core::ops::Range; + +use kernel::{ + device::{ + Bound, + Device, // + }, // + error::Result, + io::{ + poll, + register::Array, + Io, // + }, + iommu::pgtable::{ + Config, + IoPageTable, + ARM64LPAES1, // + }, + num::Bounded, + prelude::*, + sizes::{ + SZ_2M, + SZ_4K, // + }, + sync::{ + Arc, + ArcBorrow, + LockedBy, // + }, + time::Delta, // +}; + +use crate::{ + driver::IoMem, + mmu::{ + AsSlotManager, + Mmu, // + }, + regs::{ + mmu_control::mmu_as_control, + mmu_control::mmu_as_control::*, + MAX_AS, // + }, + slot::{ + LockedSeat, + Seat, + SlotOperations, // + }, // +}; + +/// Address space configuration values to be written to MMU registers. +#[derive(Clone, Copy)] +struct AddressSpaceConfig { + /// Translation configuration. Configures how the MMU walks the page table for this + /// address space. + transcfg: u64, + + /// Translation table base address. The address of the page table. + transtab: u64, + + /// Memory attributes such as cacheability. + memattr: u64, +} + +/// Virtual memory (VM) address space data for use in MMU operations. +#[pin_data] +pub(crate) struct VmAsData<'drm> { + /// This address-space seat tracks this VM's binding to a hardware address space slot. + /// It can only be accessed when holding the `Mmu::as_manager` lock. + as_seat: LockedSeat, MAX_AS>, + + /// Virtual address bits for this address space. + va_bits: u8, + + /// The page table which maps GPU virtual addresses to physical addresses for this VM. + #[pin] + pub(crate) page_table: IoPageTable<'drm, ARM64LPAES1>, +} + +impl<'drm> VmAsData<'drm> { + /// Creates VM address space data by initializing all of its fields. + pub(crate) fn new<'a>( + mmu: &'a Mmu<'drm>, + dev: &'drm Device, + va_bits: u32, + pa_bits: u32, + ) -> impl pin_init::PinInit, Error> + 'a { + let pt_config = Config { + quirks: 0, + pgsize_bitmap: SZ_4K | SZ_2M, + ias: va_bits, + oas: pa_bits, + coherent_walk: false, + }; + + let page_table_init = IoPageTable::new(dev, pt_config); + + try_pin_init!(Self { + as_seat: LockedBy::new(&mmu.as_manager, Seat::NoSeat), + va_bits: va_bits as u8, + page_table <- page_table_init, + }? Error) + } + + /// Computes the hardware configuration for this address space. + fn as_config(&self) -> Result { + let pt = &self.page_table; + // The hardware computes the valid input address range as: + // INA_BITS_VALID = min(HW_INA_BITS, 55 - INA_BITS) + // To configure our desired va_bits, we solve for INA_BITS: + // INA_BITS = 55 - va_bits + // This assumes HW_INA_BITS (hardware capability) >= va_bits. + let field = 55u64.checked_sub(self.va_bits.into()).ok_or(EINVAL)?; + let ina_bits = + match mmu_as_control::InaBits::try_from(Bounded::try_new(field).ok_or(EINVAL)?)? { + mmu_as_control::InaBits::Reset => return Err(EINVAL), + bits => bits, + }; + + let transcfg = mmu_as_control::TRANSCFG::zeroed() + .with_ptw_memattr(mmu_as_control::PtwMemattr::WriteBack) + .with_r_allocate(true) + .with_mode(mmu_as_control::AddressSpaceMode::Aarch64_4K) + .with_ina_bits(ina_bits) + .into_raw(); + + Ok(AddressSpaceConfig { + transcfg, + // SAFETY: The SlotManager holds an `Arc` as SlotData while this + // TTBR is programmed and stores that Arc in the active slot before + // returning. Eviction flushes and disables the slot before releasing + // the Arc; if eviction fails, the slot retains it. Therefore the page + // table cannot be dropped while the GPU is using it. + transtab: unsafe { pt.ttbr() }, + memattr: MEMATTR::from_mair(pt.mair()).into_raw(), + }) + } +} + +/// Coordinates all hardware-level address space operations through MMIO register +/// operations including enabling, disabling, flushing, and updating address spaces. +pub(crate) struct AddressSpaceManager<'drm> { + /// Parent device used for logging. + dev: &'drm Device, + + /// Memory-mapped I/O region for GPU register access. + iomem: Arc>, + + /// Bitmask of present address space slots from GPU_AS_PRESENT register. + as_present: u32, +} + +impl<'drm> AddressSpaceManager<'drm> { + /// Creates a new address space manager. + /// + /// Initializes the manager with references to the platform device and + /// I/O memory region, along with the bitmask of available AS slots. + pub(super) fn new( + dev: &'drm Device, + iomem: Arc>, + as_present: u32, + ) -> Result> { + if as_present.trailing_ones() != as_present.count_ones() { + dev_err!( + dev, + "Sparse AS_PRESENT mask is unsupported: {:#x}", + as_present + ); + return Err(EINVAL); + } + Ok(Self { + dev, + iomem, + as_present, + }) + } + + /// Validates that an AS slot number is within range and present in hardware. + /// + /// Checks that the slot index is less than [`MAX_AS`] and that + /// the corresponding bit is set in the `as_present` mask read from the GPU. + /// + /// Returns [`EINVAL`] if the slot is out of range or not present in hardware. + fn validate_as_slot(&self, as_nr: usize) -> Result { + if as_nr >= MAX_AS { + dev_err!( + self.dev, + "AS slot {} out of valid range (max {})", + as_nr, + MAX_AS + ); + return Err(EINVAL); + } + + if (self.as_present & (1 << as_nr)) == 0 { + dev_err!( + self.dev, + "AS slot {} not present in hardware (AS_PRESENT={:#x})", + as_nr, + self.as_present + ); + return Err(EINVAL); + } + Ok(()) + } + + /// Waits for an AS slot to become ready (not active). + /// + /// Returns an error if polling times out after 10ms or if register access fails. + fn as_wait_ready(&self, as_nr: usize) -> Result { + let io = &*self.iomem; + let op = || { + let status_reg = STATUS::try_at(as_nr).ok_or(EINVAL)?; + Ok(io.read(status_reg)) + }; + let cond = |status: &STATUS| -> bool { !status.active_ext() }; + poll::read_poll_timeout(op, cond, Delta::from_micros(50), Delta::from_millis(10))?; + + Ok(()) + } + + /// Sends a command to an AS slot. + /// + /// Returns an error if waiting for ready times out or if register write fails. + fn as_send_cmd(&mut self, as_nr: usize, cmd: MmuCommand) -> Result { + self.as_wait_ready(as_nr)?; + let io = &*self.iomem; + let command_reg = COMMAND::try_at(as_nr).ok_or(EINVAL)?; + io.write(command_reg, COMMAND::zeroed().with_command(cmd)); + Ok(()) + } + + /// Sends a command to an AS slot and waits for completion. + /// + /// Returns an error if sending the command fails or if waiting for completion times out. + fn as_send_cmd_and_wait(&mut self, as_nr: usize, cmd: MmuCommand) -> Result { + self.as_send_cmd(as_nr, cmd)?; + self.as_wait_ready(as_nr)?; + Ok(()) + } + + /// Enables an AS slot with the provided configuration. + /// + /// Returns an error if the slot is invalid or if register writes/commands fail. + fn as_enable(&mut self, as_nr: usize, as_config: &AddressSpaceConfig) -> Result { + self.validate_as_slot(as_nr)?; + + let io = &*self.iomem; + + let transtab = as_config.transtab; + io.write( + TRANSTAB_LO::try_at(as_nr).ok_or(EINVAL)?, + TRANSTAB_LO::from_raw(transtab as u32), + ); + io.write( + TRANSTAB_HI::try_at(as_nr).ok_or(EINVAL)?, + TRANSTAB_HI::from_raw((transtab >> 32) as u32), + ); + + let transcfg = as_config.transcfg; + io.write( + TRANSCFG_LO::try_at(as_nr).ok_or(EINVAL)?, + TRANSCFG_LO::from_raw(transcfg as u32), + ); + io.write( + TRANSCFG_HI::try_at(as_nr).ok_or(EINVAL)?, + TRANSCFG_HI::from_raw((transcfg >> 32) as u32), + ); + + let memattr = as_config.memattr; + io.write( + MEMATTR_LO::try_at(as_nr).ok_or(EINVAL)?, + MEMATTR_LO::from_raw(memattr as u32), + ); + io.write( + MEMATTR_HI::try_at(as_nr).ok_or(EINVAL)?, + MEMATTR_HI::from_raw((memattr >> 32) as u32), + ); + + self.as_send_cmd_and_wait(as_nr, MmuCommand::Update)?; + + Ok(()) + } + + /// Disables an AS slot and clears its configuration. + /// + /// Returns an error if the slot is invalid or if register writes/commands fail. + fn as_disable(&mut self, as_nr: usize) -> Result { + self.validate_as_slot(as_nr)?; + + // Flush AS before disabling + self.as_send_cmd_and_wait(as_nr, MmuCommand::FlushMem)?; + + let io = &*self.iomem; + + io.write( + TRANSTAB_LO::try_at(as_nr).ok_or(EINVAL)?, + TRANSTAB_LO::from_raw(0), + ); + io.write( + TRANSTAB_HI::try_at(as_nr).ok_or(EINVAL)?, + TRANSTAB_HI::from_raw(0), + ); + + io.write( + MEMATTR_LO::try_at(as_nr).ok_or(EINVAL)?, + MEMATTR_LO::from_raw(0), + ); + io.write( + MEMATTR_HI::try_at(as_nr).ok_or(EINVAL)?, + MEMATTR_HI::from_raw(0), + ); + + let transcfg = TRANSCFG::zeroed() + .with_mode(AddressSpaceMode::Unmapped) + .into_raw(); + + io.write( + TRANSCFG_LO::try_at(as_nr).ok_or(EINVAL)?, + TRANSCFG_LO::from_raw(transcfg as u32), + ); + io.write( + TRANSCFG_HI::try_at(as_nr).ok_or(EINVAL)?, + TRANSCFG_HI::from_raw((transcfg >> 32) as u32), + ); + + self.as_send_cmd_and_wait(as_nr, MmuCommand::Update)?; + + Ok(()) + } + + /// Locks a region of the translation tables for an atomic update. + /// + /// Programs the MMU [`LOCKADDR`] register for the given address space and issues + /// the lock command. The hardware rounds the requested range up to a + /// power-of-two region aligned to its size. + /// + /// Returns an error if the slot is invalid or if register writes/commands fail. + fn as_start_update(&mut self, as_nr: usize, region: &Range) -> Result { + self.validate_as_slot(as_nr)?; + + // Avoid both an empty range and an inverted range. + if region.start >= region.end { + return Err(EINVAL); + } + + // The lock operates on full 64-byte cache lines of translation table entries. + // Since each translation table entry (TTE) is 8 bytes, a cache line has 8 TTEs. + // Since each TTE maps one page, the minimum locked region size will be 8 pages. + // + // With 4KiB pages (Aarch64_4K mode), the minimum locked region is 32KiB. + let lock_region_min_size: u64 = 4096 * 8; + + // Count the number of trailing zero bits (zeros at the right/least-significant + // end of the binary representation). For a power-of-two value, this equals the + // base-2 exponent (e.g., 32 KiB = 2^15 → 15). + let lock_region_min_size_log2 = lock_region_min_size.trailing_zeros() as u8; + + // XOR the first and last addresses to identify which bits differ between them. + // The highest set bit in the result determines the exponent of the smallest + // power-of-two region that can contain both addresses. + // + // Example: + // addr_xor = 0x1000 ^ 0x2FFF = 0x3FFF + // highest set bit in 0x3FFF is bit 13 + // minimum region size = 2^(13 + 1) = 16 KiB + let addr_xor = region.start ^ (region.end - 1); + let region_size_log2 = 64 - addr_xor.leading_zeros() as u8; + + let lock_region_log2 = core::cmp::max(region_size_log2, lock_region_min_size_log2); + + let lock_region_size = 1u64.checked_shl(lock_region_log2.into()).ok_or(EINVAL)?; + // Align the LOCKADDR base address down to the lock region size (1 << lock_region_log2). + // + // The MMU ignores the low lock_region_log2 bits of LOCKADDR base, so ensure + // they are cleared in software to avoid ambiguity. + // + // Example: + // lock_region_log2 = 14 (16 KiB) + // region.start = 0x1000 + // lockaddr_base = 0x1000 & ~(0x3FFF) = 0x0000 + let lockaddr_base = region.start & !(lock_region_size - 1); + + // The LOCKADDR size field encodes the lock region size as log2(size) - 1, + // per the hardware definition. For example, a 32 KiB region is encoded as 14 + // because log2(32 KiB) = 15. + let lockaddr_size = lock_region_log2 - 1; + + let io = &*self.iomem; + + // The LOCKADDR base field stores address bits 63:12, so remove the low 12 bits + // before passing this value to the register macro helper. + // These bits are guaranteed to be zero anyway because of the minimum + // size of the locked region. + let lockaddr_base_field = lockaddr_base >> 12; + let lockaddr_val = LOCKADDR::zeroed() + .try_with_size(lockaddr_size)? + .try_with_base(lockaddr_base_field)? + .into_raw(); + + io.write( + LOCKADDR_LO::try_at(as_nr).ok_or(EINVAL)?, + LOCKADDR_LO::from_raw(lockaddr_val as u32), + ); + io.write( + LOCKADDR_HI::try_at(as_nr).ok_or(EINVAL)?, + LOCKADDR_HI::from_raw((lockaddr_val >> 32) as u32), + ); + + self.as_send_cmd_and_wait(as_nr, MmuCommand::Lock) + } + + /// Completes an atomic translation table update. + /// + /// Returns an error if the slot is invalid or if the flush command fails. + fn as_end_update(&mut self, as_nr: usize) -> Result { + self.validate_as_slot(as_nr)?; + self.as_send_cmd_and_wait(as_nr, MmuCommand::FlushPt)?; + Ok(()) + } + + /// Flushes the translation table cache for an AS slot. + /// + /// Returns an error if the slot is invalid or if the flush command fails. + fn as_flush(&mut self, as_nr: usize) -> Result { + self.validate_as_slot(as_nr)?; + self.as_send_cmd_and_wait(as_nr, MmuCommand::FlushPt) + } +} + +impl<'drm> SlotOperations for AddressSpaceManager<'drm> { + /// VM address space data associated with a hardware slot. + type SlotData = Arc>; + + fn seat(slot_data: &Self::SlotData) -> &LockedSeat { + &slot_data.as_seat + } + + /// Activates a VM in a hardware slot. + fn activate(&mut self, slot_idx: usize, slot_data: &Self::SlotData) -> Result { + let as_config = slot_data.as_config()?; + self.as_enable(slot_idx, &as_config) + } + + /// Evicts a VM from a hardware slot. + fn evict(&mut self, slot_idx: usize, _slot_data: &Self::SlotData) -> Result { + self.as_flush(slot_idx)?; + self.as_disable(slot_idx)?; + Ok(()) + } +} + +impl<'drm> AsSlotManager<'drm> { + /// Locks a region for translation table updates if the VM has an active slot. + pub(super) fn start_vm_update( + &mut self, + vm_as_data: &VmAsData<'drm>, + region: &Range, + ) -> Result { + let seat = vm_as_data.as_seat.access(self); + match seat.slot() { + Some(slot) => { + let as_nr = slot as usize; + self.as_start_update(as_nr, region) + } + _ => Ok(()), + } + } + + /// Completes translation table updates and unlocks the region. + pub(super) fn end_vm_update(&mut self, vm_as_data: &VmAsData<'drm>) -> Result { + let seat = vm_as_data.as_seat.access(self); + match seat.slot() { + Some(slot) => { + let as_nr = slot as usize; + self.as_end_update(as_nr) + } + _ => Ok(()), + } + } + + /// Flushes the translation table cache if the VM has an active slot. + pub(super) fn flush_vm(&mut self, vm_as_data: &VmAsData<'drm>) -> Result { + let seat = vm_as_data.as_seat.access(self); + match seat.slot() { + Some(slot) => { + let as_nr = slot as usize; + self.as_flush(as_nr) + } + _ => Ok(()), + } + } + + /// Activates a VM by assigning it to a hardware slot. + pub(super) fn activate_vm(&mut self, vm_as_data: ArcBorrow<'_, VmAsData<'drm>>) -> Result { + self.activate(vm_as_data.into()) + } + + /// Deactivates a VM by evicting it from its hardware slot. + pub(super) fn deactivate_vm(&mut self, vm_as_data: &VmAsData<'drm>) -> Result { + self.evict(&vm_as_data.as_seat) + } +} diff --git a/drivers/gpu/drm/tyr/regs.rs b/drivers/gpu/drm/tyr/regs.rs index 831357a8ef87..a62724378ced 100644 --- a/drivers/gpu/drm/tyr/regs.rs +++ b/drivers/gpu/drm/tyr/regs.rs @@ -25,7 +25,7 @@ // // Nevertheless, it is useful to have most of them defined, like the C driver // does. -#![allow(dead_code)] +#![expect(dead_code)] /// Combine two 32-bit values into a single 64-bit value. pub(crate) fn join_u64(lo: u32, hi: u32) -> u64 { @@ -45,6 +45,8 @@ pub(crate) fn read_u64_no_tearing(lo_read: impl Fn() -> u32, hi_read: impl Fn() } } +pub(crate) use mmu_control::mmu_as_control::MAX_AS; + /// These registers correspond to the GPU_CONTROL register page. /// They are involved in GPU configuration and control. pub(crate) mod gpu_control { @@ -965,6 +967,8 @@ pub(crate) mod mmu_as_control { register, // }; + use pin_init::Zeroable; + /// Maximum number of hardware address space slots. /// The actual number of slots available is usually lower. pub(crate) const MAX_AS: usize = 16; @@ -1158,7 +1162,136 @@ fn from(val: MMU_MEMATTR_STAGE1) -> Self { pub(crate) MEMATTR_HI(u32)[MAX_AS, stride = STRIDE] @ 0x240c { 31:0 value; } + } + impl MEMATTR { + /// Outer cache-policy nibble indicating device memory. + const ARM_MAIR_DEVICE_MEMORY: u8 = 0x0; + + /// In the ARM Architecture Reference Manual, the MAIR encoding for Normal memory + /// uses the format `0bxxRW` where: + /// - `W` (bit 0) = Write-Allocate policy + /// - `R` (bit 1) = Read-Allocate policy + /// E.g., `0b0011` would allow both read and write allocation on a cache miss. + /// + /// ARM MAIR Write-Allocate bit (bit 0 of a cache policy nibble). + const ARM_MAIR_WRITE_ALLOCATE: u8 = 0x1; + /// ARM MAIR Read-Allocate bit (bit 1 of a cache policy nibble). + const ARM_MAIR_READ_ALLOCATE: u8 = 0x2; + + /// Write-back policy bit. For cacheable encodings, it is necessary but not + /// sufficient to set bit 2 of the cache policy nibble. Bit 2 does not + /// definitively determine write back because bit 2 is also set in `0b0100` + /// which encodes Normal non-cacheable memory. + const ARM_MAIR_WRITE_BACK_BIT: u8 = 0x4; + + /// Complete cache-policy nibble encoding for Normal Non-cacheable memory. + const ARM_MAIR_NON_CACHEABLE: u8 = 0x4; + + /// Mask for the inner cache policy nibble in MAIR attribute bytes. + const ARM_MAIR_INNER_MASK: u8 = 0x0f; + + /// Check if a MAIR attribute byte represents device memory. + /// + /// Device memory (memory-mapped I/O, registers) cannot be cached because + /// reading and writing to this memory may have side effects. + fn is_device_memory(mair_attr: u8) -> bool { + // In AArch64 MAIR, outer nibble only is 0 for device memory. + (mair_attr >> 4) == Self::ARM_MAIR_DEVICE_MEMORY + } + + /// Check if normal memory is fully write-back cacheable. + /// + /// ARM MAIR has two cache policy levels (outer [7:4] and inner [3:0]). + /// For memory to be truly write-back, BOTH levels must have the write-back bit set. + /// If only one level is write-back, treat it as non-cacheable for GPU purposes. + fn is_writeback_cacheable(mair_attr: u8) -> bool { + let outer = mair_attr >> 4; + let inner = mair_attr & Self::ARM_MAIR_INNER_MASK; + + outer != Self::ARM_MAIR_NON_CACHEABLE + && inner != Self::ARM_MAIR_NON_CACHEABLE + && (outer & Self::ARM_MAIR_WRITE_BACK_BIT) != 0 + && (inner & Self::ARM_MAIR_WRITE_BACK_BIT) != 0 + } + + // Helper to encode a MEMATTR attribute from its individual fields. + fn encode_attribute( + alloc_w: bool, + alloc_r: bool, + alloc_sel: AllocPolicySelect, + coherency: Coherency, + memory_type: MemoryType, + ) -> MMU_MEMATTR_STAGE1 { + MMU_MEMATTR_STAGE1::zeroed() + .with_alloc_w(alloc_w) + .with_alloc_r(alloc_r) + .with_alloc_sel(alloc_sel) + .with_coherency(coherency) + .with_memory_type(memory_type) + } + + /// Convert one MAIR attribute byte into a MEMATTR attribute. + // TODO: Add a `coherent` parameter like panthor's mair_to_memattr(). + // For now, assume a non-coherent system and always encode write-back + // memory with MidgardInnerDomain coherency. + fn attribute_from_mair(mair_attr: u8) -> MMU_MEMATTR_STAGE1 { + // Device memory or non-write-back normal memory + if Self::is_device_memory(mair_attr) || !Self::is_writeback_cacheable(mair_attr) { + return Self::encode_attribute( + false, + false, + AllocPolicySelect::Alloc, + Coherency::MidgardInnerDomain, + MemoryType::NonCacheable, + ); + } + + // Write-back cacheable normal memory + let inner: u8 = mair_attr & Self::ARM_MAIR_INNER_MASK; + Self::encode_attribute( + (inner & Self::ARM_MAIR_WRITE_ALLOCATE) != 0, + (inner & Self::ARM_MAIR_READ_ALLOCATE) != 0, + AllocPolicySelect::Alloc, + Coherency::MidgardInnerDomain, + MemoryType::WriteBack, + ) + } + + /// Write one converted MAIR attribute into a corresponding MEMATTR slot. + fn with_encoded_attribute(self, index: usize, attr: MMU_MEMATTR_STAGE1) -> Self { + debug_assert!(index < 8); + + let shift = index * 8; + let mask = !(0xffu64 << shift); + let raw = (self.into_raw() & mask) | ((u64::from(attr.into_raw())) << shift); + + Self::from_raw(raw) + } + + /// Convert an AArch64 MAIR value into the GPU MEMATTR register encoding. + /// + /// Both MAIR and MEMATTR are 64-bit values with eight 8-bit memory + /// attribute entries, but the bits do not map directly. The GPU MEMATTR encoding + /// is less detailed than the MAIR encoding, so MAIR is converted to MEMATTR + /// conservatively as follows: + /// + /// 1. Device memory, or Normal Memory that is not write-back cacheable, is encoded + /// as GPU `NonCacheable` + /// + /// 2. Normal memory that is write-back cacheable is encoded as GPU `WriteBack`, + /// and the inner allocation hints are preserved. + pub(crate) fn from_mair(mair: u64) -> Self { + mair.to_le_bytes() + .into_iter() + .enumerate() + .fold(Self::zeroed(), |acc, (i, attr)| { + acc.with_encoded_attribute(i, Self::attribute_from_mair(attr)) + }) + } + } + + register! { /// Lock region address for each address space. pub(crate) LOCKADDR(u64)[MAX_AS, stride = STRIDE] @ 0x2410 { /// Lock region size. diff --git a/drivers/gpu/drm/tyr/slot.rs b/drivers/gpu/drm/tyr/slot.rs index 845846e07ee5..d194ead53f71 100644 --- a/drivers/gpu/drm/tyr/slot.rs +++ b/drivers/gpu/drm/tyr/slot.rs @@ -20,7 +20,6 @@ //! //! [SlotOperations]: crate::slot::SlotOperations //! [SlotManager]: crate::slot::SlotManager -#![expect(dead_code)] use core::{ mem, diff --git a/drivers/gpu/drm/tyr/tyr.rs b/drivers/gpu/drm/tyr/tyr.rs index 7c9a8063b3b9..79045d0135a8 100644 --- a/drivers/gpu/drm/tyr/tyr.rs +++ b/drivers/gpu/drm/tyr/tyr.rs @@ -11,6 +11,7 @@ mod file; mod gem; mod gpu; +mod mmu; mod regs; mod slot; From f2dc32d5a133ed6e7b7295247e9477ec57eb93cd Mon Sep 17 00:00:00 2001 From: Boris Brezillon Date: Tue, 28 Jul 2026 11:39:26 -0700 Subject: [PATCH 118/137] drm/tyr: add GPU virtual memory (VM) support Add GPU virtual address space management using the DRM GPUVM framework. Each virtual memory (VM) space is backed by ARM64 LPAE Stage 1 page tables and can be mapped into hardware address space (AS) slots for GPU execution. The implementation provides memory isolation and virtual address allocation. VMs support mapping GEM buffer objects with configurable protection flags (readonly, noexec, uncached) and handle both 4KB and 2MB page sizes. A new_dummy_object() helper is provided to create a dummy GEM object for use as a GPUVM root. The vm module integrates with the MMU for address space activation and provides map/unmap/remap operations with page table synchronization. Signed-off-by: Boris Brezillon Co-developed-by: Daniel Almeida Signed-off-by: Daniel Almeida Co-developed-by: Deborah Brouwer Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260728-fw-boot-b4-v10-4-9187aefa3f2f@collabora.com [aliceryhl: fix integer cast on 32-bit arm] Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/Kconfig | 1 + drivers/gpu/drm/tyr/driver.rs | 4 +- drivers/gpu/drm/tyr/gem.rs | 26 +- drivers/gpu/drm/tyr/mmu.rs | 1 - drivers/gpu/drm/tyr/tyr.rs | 1 + drivers/gpu/drm/tyr/vm.rs | 951 ++++++++++++++++++++++++++++++++++ 6 files changed, 979 insertions(+), 5 deletions(-) create mode 100644 drivers/gpu/drm/tyr/vm.rs diff --git a/drivers/gpu/drm/tyr/Kconfig b/drivers/gpu/drm/tyr/Kconfig index 61a2fd6f961a..79ea4bb214de 100644 --- a/drivers/gpu/drm/tyr/Kconfig +++ b/drivers/gpu/drm/tyr/Kconfig @@ -12,6 +12,7 @@ config DRM_TYR default n select IOMMU_IO_PGTABLE_LPAE select RUST_DRM_GEM_SHMEM_HELPER + select RUST_DRM_GPUVM help Rust DRM driver for ARM Mali CSF-based GPUs. diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index 92c9d98e3aab..f487b3996293 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -37,7 +37,7 @@ use crate::{ file::TyrDrmFileData, - gem::BoData, + gem::Bo, gpu, gpu::GpuInfo, mmu::Mmu, @@ -194,7 +194,7 @@ impl drm::Driver for TyrDrmDriver { type Data = (); type RegistrationData<'drm> = TyrDrmRegistrationData<'drm>; type File = TyrDrmFileData; - type Object = drm::gem::shmem::Object; + type Object = Bo; type ParentDevice = platform::Device; const INFO: drm::DriverInfo = INFO; diff --git a/drivers/gpu/drm/tyr/gem.rs b/drivers/gpu/drm/tyr/gem.rs index 1640a161754b..c28be61a01bb 100644 --- a/drivers/gpu/drm/tyr/gem.rs +++ b/drivers/gpu/drm/tyr/gem.rs @@ -5,8 +5,12 @@ //! DRM's GEM subsystem with shmem backing. use kernel::{ - drm::gem, - prelude::*, // + drm::gem::{ + self, + shmem, // + }, + prelude::*, + sync::aref::ARef, // }; use crate::driver::{ @@ -34,3 +38,21 @@ fn new(_dev: &TyrDrmDevice, _size: usize, args: BoCreateArgs) -> impl PinInit; + +/// Creates a dummy GEM object to serve as the root of a GPUVM. +pub(crate) fn new_dummy_object(ddev: &TyrDrmDevice) -> Result> { + let bo = Bo::new( + ddev, + 4096, + shmem::ObjectConfig { + map_wc: true, + parent_resv_obj: None, + }, + BoCreateArgs { flags: 0 }, + )?; + + Ok(bo) +} diff --git a/drivers/gpu/drm/tyr/mmu.rs b/drivers/gpu/drm/tyr/mmu.rs index 83e022f9168b..5e88c9e6a4cd 100644 --- a/drivers/gpu/drm/tyr/mmu.rs +++ b/drivers/gpu/drm/tyr/mmu.rs @@ -10,7 +10,6 @@ //! The [`SlotManager`] manages the assignment of virtual address spaces to hardware address-space //! (AS) slots. MMU commands such as updates and flushes are carried out by the //! [`AddressSpaceManager`] which actually writes to the MMU registers. -#![expect(dead_code)] use core::ops::Range; diff --git a/drivers/gpu/drm/tyr/tyr.rs b/drivers/gpu/drm/tyr/tyr.rs index 79045d0135a8..92f6885cdaae 100644 --- a/drivers/gpu/drm/tyr/tyr.rs +++ b/drivers/gpu/drm/tyr/tyr.rs @@ -14,6 +14,7 @@ mod mmu; mod regs; mod slot; +mod vm; kernel::module_platform_driver! { type: TyrPlatformDriver, diff --git a/drivers/gpu/drm/tyr/vm.rs b/drivers/gpu/drm/tyr/vm.rs new file mode 100644 index 000000000000..aa7c1522b9d7 --- /dev/null +++ b/drivers/gpu/drm/tyr/vm.rs @@ -0,0 +1,951 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +//! GPU virtual memory management using the DRM GPUVM framework. +//! +//! This module manages GPU virtual address spaces, providing memory isolation and +//! the illusion of owning the entire virtual address (VA) range, similar to CPU virtual memory. +//! Each virtual memory (VM) area is backed by ARM64 LPAE Stage 1 page tables and can be +//! mapped into hardware address space (AS) slots for GPU execution. +#![expect(dead_code)] + +use core::marker::PhantomData; +use core::ops::Range; + +use kernel::{ + device::{ + Bound, + Device, // + }, + drm::{ + gem::BaseObject, + gpuvm::{ + DriverGpuVm, + GpuVaAlloc, + GpuVm, + GpuVmBo, + OpMap, + OpMapRequest, + OpMapped, + OpRemap, + OpRemapped, + OpUnmap, + OpUnmapped, + UniqueRefGpuVm, // + }, // + }, + fmt, + impl_flags, + io::PhysAddr, + iommu::pgtable::{ + prot, + IoPageTable, + ARM64LPAES1, // + }, + new_mutex, + prelude::*, + sizes::{ + SZ_1G, + SZ_2M, + SZ_4K, // + }, + sync::{ + aref::ARef, + Arc, + ArcBorrow, + Mutex, // + }, + uapi, // +}; + +use crate::{ + driver::{ + TyrDrmDevice, + TyrDrmDriver, // + }, + gem, + gem::Bo, + gpu::GpuInfo, + mmu::{ + address_space::VmAsData, + Mmu, // + }, + regs::gpu_control::MMU_FEATURES, +}; + +impl_flags!( + /// Flags controlling virtual memory mapping behavior. + /// + /// These flags control access permissions and caching behavior for GPU virtual + /// memory mappings. + #[derive(Debug, Clone, Default, Copy, PartialEq, Eq)] + pub(crate) struct VmMapFlags(u32); + + /// Individual flags that can be combined in [`VmMapFlags`]. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(crate) enum VmFlag { + /// Map as read-only. + Readonly = uapi::drm_panthor_vm_bind_op_flags_DRM_PANTHOR_VM_BIND_OP_MAP_READONLY as u32, + /// Map as non-executable. + Noexec = uapi::drm_panthor_vm_bind_op_flags_DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC as u32, + /// Map as uncached. + Uncached = uapi::drm_panthor_vm_bind_op_flags_DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED as u32, + } +); + +impl VmMapFlags { + /// Convert the flags to `pgtable::prot`. + fn to_prot(self) -> u32 { + let mut prot = 0; + + if self.contains(VmFlag::Readonly) { + prot |= prot::READ; + } else { + prot |= prot::READ | prot::WRITE; + } + + if self.contains(VmFlag::Noexec) { + prot |= prot::NOEXEC; + } + + if !self.contains(VmFlag::Uncached) { + prot |= prot::CACHE; + } + + prot + } +} + +impl fmt::Display for VmMapFlags { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut first = true; + + if self.contains(VmFlag::Readonly) { + write!(f, "READONLY")?; + first = false; + } + if self.contains(VmFlag::Noexec) { + if !first { + write!(f, " | ")?; + } + write!(f, "NOEXEC")?; + first = false; + } + + if self.contains(VmFlag::Uncached) { + if !first { + write!(f, " | ")?; + } + write!(f, "UNCACHED")?; + } + + Ok(()) + } +} + +impl TryFrom for VmMapFlags { + type Error = Error; + + fn try_from(value: u32) -> Result { + let valid = VmFlag::Readonly as u32 | VmFlag::Noexec as u32 | VmFlag::Uncached as u32; + + if value & !valid != 0 { + return Err(EINVAL); + } + Ok(Self(value)) + } +} + +/// Arguments for a virtual memory map operation. +struct VmMapArgs<'drm> { + /// Access permissions and caching behavior for the mapping. + flags: VmMapFlags, + /// GEM buffer object registered with the GPUVM framework. + vm_bo: ARef>>, + /// Offset in bytes from the start of the buffer object. + bo_offset: u64, +} + +/// Type of virtual memory operation. +enum VmOpType<'drm> { + /// Map a GEM buffer object into the virtual address space. + Map(VmMapArgs<'drm>), + /// Unmap a region from the virtual address space. + Unmap, +} + +/// Preallocated resources needed to execute a VM operation. +/// +/// VM operations may require allocating new GPUVA objects to track mappings. +/// To avoid allocation failures during the operation, preallocate the +/// maximum number of GPUVAs that might be needed. +struct VmOpResources<'drm> { + /// Preallocated GPUVA objects for remap operations. + /// + /// Partial unmap requests or map requests overlapping existing mappings + /// will trigger a remap call, which needs to register up to three VA + /// objects (one for the new mapping, and two for the previous and next + /// mappings). + preallocated_gpuvas: [Option>>; 3], +} + +/// Request to execute a virtual memory operation. +struct VmOpRequest<'drm> { + /// Request type. + op_type: VmOpType<'drm>, + + /// Region of the virtual address space covered by this request. + region: Range, +} + +/// Arguments for a page table map operation. +struct PtMapArgs { + /// Memory protection flags describing allowed accesses for this mapping. + /// + /// This is directly derived from [`VmMapFlags`] via [`VmMapFlags::to_prot`]. + prot: u32, +} + +/// Type of page table operation. +enum PtOpType { + /// Map pages into the page table. + Map(PtMapArgs), + /// Unmap pages from the page table. + Unmap, +} + +/// Context for updating the GPU page table. +/// +/// This context is created when beginning a page table update operation and +/// automatically flushes changes when dropped. It ensures that the +/// Memory Management Unit (MMU) state is properly managed and Translation +/// Lookaside Buffer (TLB) entries are flushed. +pub(crate) struct PtUpdateContext<'ctx, 'drm> { + /// Device used for DMA-mapping GEM shmem SG tables. + dev: &'ctx Device, + + /// Page table. + pt: &'ctx IoPageTable<'drm, ARM64LPAES1>, + + /// MMU manager. + mmu: &'ctx Mmu<'drm>, + + /// Reference to the address space data to pass to the MMU functions. + as_data: &'ctx VmAsData<'drm>, + + /// Region of the virtual address space covered by this request. + region: Range, + + /// Operation type. + op_type: PtOpType, + + /// Preallocated resources that can be used when executing the request. + resources: &'ctx mut VmOpResources<'drm>, +} + +impl<'ctx, 'drm> PtUpdateContext<'ctx, 'drm> { + /// Creates a new page table update context. + /// + /// This prepares the MMU for a page table update. + /// The context will automatically flush the TLB and + /// complete the update when dropped. + fn new( + dev: &'ctx Device, + pt: &'ctx IoPageTable<'drm, ARM64LPAES1>, + mmu: &'ctx Mmu<'drm>, + as_data: &'ctx VmAsData<'drm>, + region: Range, + op_type: PtOpType, + resources: &'ctx mut VmOpResources<'drm>, + ) -> Result> { + mmu.start_vm_update(as_data, ®ion)?; + + Ok(Self { + dev, + pt, + mmu, + as_data, + region, + op_type, + resources, + }) + } + + /// Finds one of our pre-allocated VAs. + fn preallocated_gpuva(&mut self) -> Result>> { + self.resources + .preallocated_gpuvas + .iter_mut() + .find_map(|f| f.take()) + .ok_or(EINVAL) + } + + /// Returns an unused GPUVA object to the preallocated pool. + /// If the pool is already full, the unused allocation is simply dropped. + fn return_preallocated_gpuva(&mut self, gpuva: GpuVaAlloc>) { + if let Some(slot) = self + .resources + .preallocated_gpuvas + .iter_mut() + .find(|slot| slot.is_none()) + { + *slot = Some(gpuva); + } + } +} + +impl Drop for PtUpdateContext<'_, '_> { + fn drop(&mut self) { + if let Err(e) = self.mmu.end_vm_update(self.as_data) { + dev_err!(self.dev, "Failed to end VM update {:?}", e); + } + + if let Err(e) = self.mmu.flush_vm(self.as_data) { + dev_err!(self.dev, "Failed to flush VM {:?}", e); + } + } +} + +/// Driver implementation for the GPUVM framework. +/// +/// Implements [`DriverGpuVm`] to provide VM operation callbacks (map, unmap, remap) +/// and associated types for buffer objects, virtual addresses, and contexts. +pub(crate) struct GpuVmData<'drm> { + _phantom: PhantomData<&'drm ()>, +} + +/// GPU virtual address space. +/// +/// Each VM can be mapped into a hardware address space slot. +#[pin_data] +pub(crate) struct Vm<'drm> { + /// Data referenced by an AS when the VM is active + as_data: Arc>, + /// MMU manager. + mmu: Arc>, + /// Parent device used for DMA mapping and page-table operations. + dev: &'drm Device, + /// DRM GPUVM core for managing virtual address space. + #[pin] + gpuvm_unique: Mutex>>, + /// Non-core part of the GPUVM. Can be used for stuff that doesn't modify the + /// internal mapping tree, like GpuVm::obtain() + gpuvm: ARef>>, + /// VA range for this VM. + va_range: Range, +} + +impl<'drm> Vm<'drm> { + /// Creates a new GPU virtual address space. + /// + /// The VM is initialized with a page table configured according to the GPU's + /// address translation capabilities and registered with the GPUVM framework. + pub(crate) fn new( + dev: &'drm Device, + ddev: &TyrDrmDevice, + mmu: ArcBorrow<'_, Mmu<'drm>>, + gpu_info: &GpuInfo, + ) -> Result>> { + let mmu_features = MMU_FEATURES::from_raw(gpu_info.mmu_features); + let va_bits = mmu_features.va_bits().get(); + let pa_bits = mmu_features.pa_bits().get(); + + let range = 0..(1u64 << va_bits); + let reserve_range = 0..0u64; + + // dummy_obj is used to initialize the GPUVM tree. + let dummy_obj = gem::new_dummy_object(ddev).inspect_err(|e| { + dev_err!(dev, "Failed to create dummy GEM object: {:?}", e); + })?; + + let gpuvm_unique = GpuVm::new::( + c"Tyr::GpuVm", + ddev, + &*dummy_obj, + range.clone(), + reserve_range, + GpuVmData::<'drm> { + _phantom: PhantomData::<&()>, + }, + ) + .inspect_err(|e| { + dev_err!(dev, "Failed to create GpuVm: {:?}", e); + })?; + let gpuvm = ARef::from(&*gpuvm_unique); + + let as_data = Arc::pin_init(VmAsData::new(&mmu, dev, va_bits, pa_bits), GFP_KERNEL)?; + + let vm = Arc::pin_init( + pin_init!(Self{ + as_data, + dev, + mmu: mmu.into(), + gpuvm, + gpuvm_unique <- new_mutex!(gpuvm_unique), + va_range: range, + }), + GFP_KERNEL, + )?; + + Ok(vm) + } + + /// Returns the parent device used by this VM for DMA mapping and page-table operations. + pub(crate) fn dev(&self) -> &'drm Device { + self.dev + } + + /// Activate the VM in a hardware address space slot. + pub(crate) fn activate(&self) -> Result { + self.mmu + .activate_vm(self.as_data.as_arc_borrow()) + .inspect_err(|e| { + dev_err!(self.dev, "Failed to activate VM: {:?}", e); + }) + } + + /// Deactivate the VM by evicting it from its address space slot. + fn deactivate(&self) -> Result { + self.mmu.deactivate_vm(&self.as_data).inspect_err(|e| { + dev_err!(self.dev, "Failed to deactivate VM: {:?}", e); + }) + } + + /// Kills the VM by deactivating it and unmapping all regions. + pub(crate) fn kill(&self) { + // TODO: Turn the VM into a state where it can't be used. + let _ = self.deactivate(); + let _ = self + .unmap_range(self.va_range.start, self.va_range.end - self.va_range.start) + .inspect_err(|e| { + dev_err!(self.dev, "Failed to unmap range during deactivate: {:?}", e); + }); + } + + /// Executes a virtual memory operation. + /// + /// This handles both map and unmap operations by coordinating between the + /// GPUVM framework and the hardware page table. + fn exec_op<'a>( + &self, + gpuvm_unique: &mut UniqueRefGpuVm>, + req: VmOpRequest<'drm>, + resources: &'a mut VmOpResources<'drm>, + ) -> Result { + let pt = &self.as_data.page_table; + + match req.op_type { + VmOpType::Map(args) => { + let mut pt_upd = PtUpdateContext::new( + self.dev, + pt, + &self.mmu, + &self.as_data, + req.region, + PtOpType::Map(PtMapArgs { + prot: args.flags.to_prot(), + }), + resources, + )?; + + gpuvm_unique.sm_map(OpMapRequest { + addr: pt_upd.region.start, + range: pt_upd.region.end - pt_upd.region.start, + gem_offset: args.bo_offset, + vm_bo: &args.vm_bo, + context: &mut pt_upd, + }) + //PtUpdateContext drops here flushing the page table + } + VmOpType::Unmap => { + let mut pt_upd = PtUpdateContext::new( + self.dev, + pt, + &self.mmu, + &self.as_data, + req.region, + PtOpType::Unmap, + resources, + )?; + + gpuvm_unique.sm_unmap( + pt_upd.region.start, + pt_upd.region.end - pt_upd.region.start, + &mut pt_upd, + ) + //PtUpdateContext drops here flushing the page table + } + } + } + + /// Maps a GEM buffer object range into the VM at the specified virtual address. + /// + /// This creates a mapping from GPU virtual address `va` to the physical pages + /// backing the GEM object, starting at `bo_offset` bytes into the object and + /// spanning `map_size` bytes. The mapping respects the access permissions and + /// caching behavior specified in `flags`. + pub(crate) fn map_bo_range( + &self, + bo: &Bo, + bo_offset: u64, + map_size: u64, + va: u64, + flags: VmMapFlags, + ) -> Result { + if map_size == 0 + || va % SZ_4K as u64 != 0 + || bo_offset % SZ_4K as u64 != 0 + || map_size % SZ_4K as u64 != 0 + { + return Err(EINVAL); + } + + let bo_size = u64::try_from(bo.size()).map_err(|_| EOVERFLOW)?; + let bo_end = bo_offset.checked_add(map_size).ok_or(EINVAL)?; + + if bo_end > bo_size { + dev_err!( + self.dev, + "BO mapping range {:#x}..{:#x} exceeds BO size {:#x}", + bo_offset, + bo_end, + bo_size + ); + return Err(EINVAL); + } + + let va_end: u64 = va.checked_add(map_size).ok_or(EINVAL)?; + + let req = VmOpRequest { + op_type: VmOpType::Map(VmMapArgs { + vm_bo: self.gpuvm.obtain(bo, ())?, + flags, + bo_offset, + }), + region: va..va_end, + }; + let mut resources = VmOpResources { + preallocated_gpuvas: [ + Some(GpuVaAlloc::>::new(GFP_KERNEL)?), + Some(GpuVaAlloc::>::new(GFP_KERNEL)?), + Some(GpuVaAlloc::>::new(GFP_KERNEL)?), + ], + }; + let result = { + let mut gpuvm_unique = self.gpuvm_unique.lock(); + self.exec_op(gpuvm_unique.as_mut().get_mut(), req, &mut resources) + }; + // We flush the defer cleanup list now. Things will be different in + // the asynchronous VM_BIND path, where we want the cleanup to + // happen outside the DMA signalling path. + self.gpuvm.deferred_cleanup(); + result + } + + /// Unmaps a virtual address range from the VM. + /// + /// This removes any existing mappings in the specified range, freeing the + /// virtual address space for reuse. + pub(crate) fn unmap_range(&self, va: u64, size: u64) -> Result { + if size == 0 || va % SZ_4K as u64 != 0 || size % SZ_4K as u64 != 0 { + return Err(EINVAL); + } + + let end = va.checked_add(size).ok_or(EINVAL)?; + + if va < self.va_range.start || end > self.va_range.end { + dev_err!( + self.dev, + "Unmap range {:#x}..{:#x} exceeds VM range {:#x}..{:#x}", + va, + end, + self.va_range.start, + self.va_range.end + ); + return Err(EINVAL); + } + + let req = VmOpRequest { + op_type: VmOpType::Unmap, + region: va..end, + }; + + let full_vm = va == self.va_range.start && end == self.va_range.end; + + let mut resources = VmOpResources { + preallocated_gpuvas: if full_vm { + // Unmapping the entire VM cannot split an existing mapping, + // so no GPUVA objects are needed for remap operations. + [None, None, None] + } else { + [ + Some(GpuVaAlloc::>::new(GFP_KERNEL)?), + Some(GpuVaAlloc::>::new(GFP_KERNEL)?), + Some(GpuVaAlloc::>::new(GFP_KERNEL)?), + ] + }, + }; + let result = { + let mut gpuvm_unique = self.gpuvm_unique.lock(); + self.exec_op(gpuvm_unique.as_mut().get_mut(), req, &mut resources) + }; + // We flush the defer cleanup list now. Things will be different in + // the asynchronous VM_BIND path, where we want the cleanup to + // happen outside the DMA signalling path. + self.gpuvm.deferred_cleanup(); + result + } +} + +impl<'drm> DriverGpuVm for GpuVmData<'drm> { + type Driver = TyrDrmDriver; + type Object = Bo; + type VmBoData = (); + type VaData = (); + type SmContext<'ctx> + = PtUpdateContext<'ctx, 'drm> + where + Self: 'ctx; + + /// Create a new mapping. + fn sm_step_map<'op>( + &mut self, + op: OpMap<'op, Self>, + context: &mut Self::SmContext<'_>, + ) -> Result, Error> { + let start_iova = op.addr(); + let mut iova = start_iova; + let mut bytes_left_to_map = op.length(); + let mut gem_offset = op.gem_offset(); + + // Make sure that the end of the requested GEM range doesn't run past the + // end of the GEM buffer itself. + let gem_range_end = op.gem_offset().checked_add(op.length()).ok_or(EINVAL)?; + + if gem_range_end > op.obj().size() as u64 { + dev_err!( + context.dev, + "Requested GEM range ends at {} which is beyond the GEM buffer size {}", + gem_range_end, + op.obj().size() + ); + return Err(EINVAL); + } + + let sgt = op.obj().sg_table(context.dev).inspect_err(|e| { + dev_err!(context.dev, "Failed to get sg_table: {:?}", e); + })?; + let prot = match &context.op_type { + PtOpType::Map(args) => args.prot, + _ => { + return Err(EINVAL); + } + }; + + for sgt_entry in sgt.iter() { + // Expressly convert to u64 to work with arm 32-bit builds. + #[allow(clippy::useless_conversion)] + let mut paddr = u64::from(sgt_entry.dma_address()); + #[allow(clippy::useless_conversion)] + let mut sgt_entry_length = u64::from(sgt_entry.dma_len()); + + if bytes_left_to_map == 0 { + break; + } + + if gem_offset > 0 { + // Skip the entire SGT entry if the gem_offset exceeds its length. + let skip = u64::min(sgt_entry_length, gem_offset); + paddr += skip; + sgt_entry_length -= skip; + gem_offset -= skip; + } + + if sgt_entry_length == 0 { + continue; + } + + let len = u64::min(sgt_entry_length, bytes_left_to_map); + + let segment_mapped = match pt_map(context.dev, context.pt, iova, paddr, len, prot) { + Ok(segment_mapped) => segment_mapped, + Err(e) => { + // clean up any successful mappings from previous SGT entries. + let total_mapped = iova - start_iova; + if total_mapped > 0 { + let _ = pt_unmap( + context.dev, + context.pt, + start_iova..(start_iova + total_mapped), + ); + } + return Err(e); + } + }; + + bytes_left_to_map -= segment_mapped; + iova += segment_mapped; + } + + if bytes_left_to_map != 0 { + let total_mapped = iova - start_iova; + + if total_mapped > 0 { + let _ = pt_unmap(context.dev, context.pt, start_iova..iova); + } + + dev_err!( + context.dev, + "SG table is too small for requested mapping: {} bytes remain", + bytes_left_to_map + ); + + return Err(EINVAL); + } + + let gpuva = context.preallocated_gpuva()?; + let op = op.insert(gpuva, pin_init::init_zeroed()); + + Ok(op) + } + + /// Indicates that an existing mapping should be removed. + fn sm_step_unmap<'op>( + &mut self, + op: OpUnmap<'op, Self>, + context: &mut Self::SmContext<'_>, + ) -> Result, Error> { + let start_iova = op.va().addr(); + let length = op.va().length(); + + let region = start_iova..(start_iova + length); + pt_unmap(context.dev, context.pt, region.clone()).inspect_err(|e| { + dev_err!( + context.dev, + "Failed to unmap region {:#x}..{:#x}: {:?}", + region.start, + region.end, + e + ); + })?; + + let (op_unmapped, _va_removed) = op.remove(); + + Ok(op_unmapped) + } + + /// Split up an existing mapping. + fn sm_step_remap<'op>( + &mut self, + op: OpRemap<'op, Self>, + context: &mut Self::SmContext<'_>, + ) -> Result, Error> { + let unmap_start = if let Some(prev) = op.prev() { + prev.addr() + prev.length() + } else { + op.va_to_unmap().addr() + }; + + let unmap_end = if let Some(next) = op.next() { + next.addr() + } else { + op.va_to_unmap().addr() + op.va_to_unmap().length() + }; + + let unmap_length = unmap_end - unmap_start; + + if unmap_length > 0 { + let region = unmap_start..(unmap_start + unmap_length); + pt_unmap(context.dev, context.pt, region.clone()).inspect_err(|e| { + dev_err!( + context.dev, + "Failed to unmap remap region {:#x}..{:#x}: {:?}", + region.start, + region.end, + e + ); + })?; + } + + let prev_va = context.preallocated_gpuva()?; + let next_va = context.preallocated_gpuva()?; + + let (op_remapped, remap_ret) = op.remap( + [prev_va, next_va], + pin_init::init_zeroed(), + pin_init::init_zeroed(), + ); + + if let Some(unused_va) = remap_ret.unused_va { + context.return_preallocated_gpuva(unused_va); + } + + Ok(op_remapped) + } +} + +/// This function selects the largest supported block size (currently 4KB or 2MB) +/// that can be used for a mapping at the given address and size, respecting alignment constraints. +/// +/// We can map multiple pages at once but we can't exceed the size of the +/// table entry itself. So, if mapping 4KB pages, figure out how many pages +/// can be mapped before we hit the 2MB boundary. Or, if mapping 2MB pages, +/// figure out how many pages can be mapped before hitting the 1GB boundary +/// Returns the page size (4KB or 2MB) and the number of pages that can be mapped at that size. +fn get_pgsize(addr: u64, size: u64) -> (u64, u64) { + // Get the distance to the next boundary of 2MB block + let blk_offset_2m = addr.wrapping_neg() % (SZ_2M as u64); + + // Use 4K blocks if the address is not 2MB aligned, or we have less than 2MB to map + if blk_offset_2m != 0 || size < SZ_2M as u64 { + let pgcount = if blk_offset_2m == 0 { + size / SZ_4K as u64 + } else { + u64::min(blk_offset_2m, size) / SZ_4K as u64 + }; + return (SZ_4K as u64, pgcount); + } + + let blk_offset_1g = addr.wrapping_neg() % (SZ_1G as u64); + let blk_offset = if blk_offset_1g == 0 { + SZ_1G as u64 + } else { + blk_offset_1g + }; + let pgcount = u64::min(blk_offset, size) / SZ_2M as u64; + + (SZ_2M as u64, pgcount) +} + +/// Maps a physical address range into the page table at the specified virtual address. +/// +/// This function maps `len` bytes of physical memory starting at `paddr` to the +/// virtual address `iova`, using the protection flags specified in `prot`. It +/// automatically selects optimal page sizes to minimize page table overhead. +/// +/// If the mapping fails partway through, all successfully mapped pages are +/// unmapped before returning an error. +/// +/// Returns the number of bytes successfully mapped. +fn pt_map( + dev: &Device, + pt: &IoPageTable<'_, ARM64LPAES1>, + iova: u64, + paddr: u64, + len: u64, + prot: u32, +) -> Result { + let mut segment_mapped = 0u64; + while segment_mapped < len { + let remaining = len - segment_mapped; + let curr_iova = iova + segment_mapped; + let curr_paddr = paddr + segment_mapped; + + let (pgsize, pgcount) = get_pgsize(curr_iova | curr_paddr, remaining); + + // On 32-bit systems, usize is only 32 bits, so check that + // the iova can be converted without truncation. + let curr_iova = match usize::try_from(curr_iova) { + Ok(curr_iova) => curr_iova, + Err(_) => { + dev_err!( + dev, + "curr_iova {:#x} cannot be represented as usize (max {:#x})", + curr_iova, + usize::MAX + ); + + if segment_mapped > 0 { + let _ = pt_unmap(dev, pt, iova..(iova + segment_mapped)); + } + + return Err(EOVERFLOW); + } + }; + + // SAFETY: + // No other io-pgtable operation can currently access this range because Tyr holds + // the gpuvm_unique mutex for the entire sm_map() operation. + // The addresses being mapped won't overlap any existing mappings in this + // page table because drm_gpuvm_sm_map() checks each requested mapping and either unmaps + // or remaps any overlap before creating the new mapping. + let (mapped, result) = unsafe { + pt.map_pages( + curr_iova, + curr_paddr as PhysAddr, + pgsize as usize, + pgcount as usize, + prot, + GFP_KERNEL, + ) + }; + + if let Err(e) = result { + // If map_pages fails, mapped will be zero because the ARM LPAE backend + // only updates the mapped value after the entire request succeeds. + dev_err!(dev, "pt.map_pages failed at iova {:#x}: {:?}", curr_iova, e); + if segment_mapped > 0 { + let _ = pt_unmap(dev, pt, iova..(iova + segment_mapped)); + } + return Err(e); + } + + if mapped == 0 { + dev_err!(dev, "Failed to map any pages at iova {:#x}", curr_iova); + if segment_mapped > 0 { + let _ = pt_unmap(dev, pt, iova..(iova + segment_mapped)); + } + return Err(ENOMEM); + } + + segment_mapped += mapped as u64; + } + + Ok(segment_mapped) +} + +/// Unmaps a virtual address range from the page table. +/// +/// This function removes all page table entries in the specified range, +/// automatically handling different page sizes that may be present. +fn pt_unmap(dev: &Device, pt: &IoPageTable<'_, ARM64LPAES1>, range: Range) -> Result { + let mut iova = range.start; + let mut bytes_left_to_unmap = range.end - range.start; + + while bytes_left_to_unmap > 0 { + // It is fine to use just the iova to determine the page size + // because if the actual mapping was represented with smaller page sizes, + // (e.g. because the physical address was not 2MiB aligned) + // the ARM LPAE backend will notice and handle the lower-level table correctly. + let (pgsize, pgcount) = get_pgsize(iova, bytes_left_to_unmap); + + // On 32-bit systems, usize is only 32 bits, so check that + // the iova can be converted without truncation. + let iova_usize = usize::try_from(iova).map_err(|_| { + dev_err!( + dev, + "IOVA {:#x} cannot be represented as usize (max {:#x})", + iova, + usize::MAX + ); + EOVERFLOW + })?; + + // SAFETY: + // No other io-pgtable operation can currently access this range because Tyr holds + // the gpuvm_unique mutex for the entire sm_unmap() operation. + // We know that this page table has one or more consecutive mappings + // starting at `iova` with the total size of `pgcount * pgsize` because + // gpuvm callbacks provide exactly the range that was previously mapped. + let unmapped = unsafe { pt.unmap_pages(iova_usize, pgsize as usize, pgcount as usize) }; + + if unmapped == 0 { + dev_err!(dev, "Failed to unmap any bytes at iova {:#x}", iova_usize); + return Err(EINVAL); + } + + bytes_left_to_unmap -= unmapped as u64; + iova += unmapped as u64; + } + + Ok(()) +} From e38e457b590bcb4bb09567a2c0b70eaf9568f778 Mon Sep 17 00:00:00 2001 From: Deborah Brouwer Date: Tue, 28 Jul 2026 11:39:27 -0700 Subject: [PATCH 119/137] drm/tyr: add a kernel buffer object Introduce a buffer object type (KernelBo) for internal driver allocations that are managed by the kernel rather than userspace. KernelBo wraps a GEM shmem object and automatically handles GPU virtual address space mapping during creation and unmapping on drop. This provides a safe and convenient way for the driver to both allocate and clean up internal buffers for kernel-managed resources. Co-developed-by: Boris Brezillon Signed-off-by: Boris Brezillon Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260728-fw-boot-b4-v10-5-9187aefa3f2f@collabora.com Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/gem.rs | 113 +++++++++++++++++++++++++++++++++++-- 1 file changed, 109 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/tyr/gem.rs b/drivers/gpu/drm/tyr/gem.rs index c28be61a01bb..69e1c75e59a5 100644 --- a/drivers/gpu/drm/tyr/gem.rs +++ b/drivers/gpu/drm/tyr/gem.rs @@ -4,18 +4,29 @@ //! This module provides buffer object (BO) management functionality using //! DRM's GEM subsystem with shmem backing. +use core::ops::Range; + use kernel::{ drm::gem::{ self, shmem, // }, prelude::*, - sync::aref::ARef, // + sync::{ + aref::ARef, + Arc, // + }, // }; -use crate::driver::{ - TyrDrmDevice, - TyrDrmDriver, // +use crate::{ + driver::{ + TyrDrmDevice, + TyrDrmDriver, // + }, + vm::{ + Vm, + VmMapFlags, // + }, }; /// Tyr's DriverObject type for GEM objects. @@ -56,3 +67,97 @@ pub(crate) fn new_dummy_object(ddev: &TyrDrmDevice) -> Result> { Ok(bo) } + +/// Specifies how to choose a GPU virtual address for a [`KernelBo`]. +/// An automatic VA allocation strategy will be added in the future. +pub(crate) enum KernelBoVaAlloc { + /// Explicit VA address specified by the caller. + #[expect(dead_code)] + Explicit(u64), +} + +/// A kernel-owned buffer object with automatic GPU virtual address mapping. +/// +/// This structure represents a buffer object that is created and managed entirely +/// by the kernel driver, as opposed to userspace-created GEM objects. It combines +/// a GEM object with automatic GPU virtual address (VA) space mapping and cleanup. +/// +/// When dropped, the buffer is automatically unmapped from the GPU VA space. +pub(crate) struct KernelBo<'drm> { + /// The underlying GEM buffer object. + bo: ARef, + /// The GPU VM this buffer is mapped into. + vm: Arc>, + /// The GPU VA range occupied by this buffer. + va_range: Range, +} + +impl<'drm> KernelBo<'drm> { + /// Creates a new kernel-owned buffer object and maps it into GPU VA space. + /// + /// This function allocates a new shmem-backed GEM object and immediately maps + /// it into the specified GPU virtual memory space. The mapping is automatically + /// cleaned up when the [`KernelBo`] is dropped. + #[expect(dead_code)] + pub(crate) fn new( + ddev: &TyrDrmDevice, + vm: Arc>, + size: u64, + va_alloc: KernelBoVaAlloc, + flags: VmMapFlags, + ) -> Result { + if size == 0 { + dev_err!(vm.dev(), "Cannot create KernelBo with size 0"); + return Err(EINVAL); + } + + let KernelBoVaAlloc::Explicit(va) = va_alloc; + + let bo_size = usize::try_from(size).map_err(|_| EOVERFLOW)?; + let va_end = va.checked_add(size).ok_or(EINVAL)?; + + let bo = Bo::new( + ddev, + bo_size, + shmem::ObjectConfig { + map_wc: true, + parent_resv_obj: None, + }, + BoCreateArgs { flags: 0 }, + )?; + + vm.map_bo_range(&bo, 0, size, va, flags)?; + + Ok(KernelBo { + bo, + vm, + va_range: va..va_end, + }) + } + + #[expect(dead_code)] + pub(crate) fn bo(&self) -> &Bo { + &self.bo + } +} + +impl Drop for KernelBo<'_> { + fn drop(&mut self) { + let va = self.va_range.start; + let size = self.va_range.end - self.va_range.start; + + if let Err(e) = self.vm.unmap_range(va, size) { + // If unmap_range fails, it is still safe to drop the + // KernelBo and its ARef to the GEM buffer object because + // GPUVM also holds a reference to the GEM buffer object. + // The physical pages won't be freed or reallocated. + dev_err!( + self.vm.dev(), + "Failed to unmap KernelBo range {:#x}..{:#x}: {:?}", + self.va_range.start, + self.va_range.end, + e + ); + } + } +} From aa9ce820136062498e36a3fceeaf172561323fc3 Mon Sep 17 00:00:00 2001 From: Daniel Almeida Date: Tue, 28 Jul 2026 11:39:28 -0700 Subject: [PATCH 120/137] drm/tyr: add parser for firmware binary Add a parser for the Mali CSF GPU firmware binary format. The firmware consists of a header followed by entries describing how to load firmware sections into the MCU's memory. The parser extracts section metadata including virtual address ranges, data byte offsets within the binary, and section flags controlling permissions and cache modes. It validates the basic firmware structure and alignment and ignores protected-mode sections for now. Signed-off-by: Daniel Almeida Co-developed-by: Beata Michalska Signed-off-by: Beata Michalska Co-developed-by: Boris Brezillon Signed-off-by: Boris Brezillon Co-developed-by: Deborah Brouwer Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260728-fw-boot-b4-v10-6-9187aefa3f2f@collabora.com Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/fw/parser.rs | 588 +++++++++++++++++++++++++++++++ 1 file changed, 588 insertions(+) create mode 100644 drivers/gpu/drm/tyr/fw/parser.rs diff --git a/drivers/gpu/drm/tyr/fw/parser.rs b/drivers/gpu/drm/tyr/fw/parser.rs new file mode 100644 index 000000000000..c4d0ad1d7899 --- /dev/null +++ b/drivers/gpu/drm/tyr/fw/parser.rs @@ -0,0 +1,588 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +//! Firmware binary parser for Mali CSF (Command Stream Frontend) GPU. +//! +//! This module implements a parser for the Mali GPU firmware binary format. The firmware +//! file contains a header followed by a sequence of entries, each describing how to load +//! firmware sections into the MCU (Microcontroller Unit) memory. The parser extracts section +//! metadata including: +//! - Virtual address ranges where sections should be mapped +//! - Data ranges (byte offsets) within the firmware binary +//! - Section flags (permissions, cache modes) + +use core::{ + mem::size_of, + ops::Range, // +}; + +use kernel::{ + bits::bit_u32, + device::Device, + prelude::*, + sizes::SZ_4K, // +}; + +use crate::{ + fw::{ + CacheMode, + SectionFlags, + CSF_MCU_SHARED_REGION_START, // + }, + vm::{ + VmFlag, + VmMapFlags, // + }, // +}; + +/// A parsed firmware section ready for loading into MCU memory. +/// +/// Represents a single firmware section extracted from the firmware binary, containing +/// all information needed to map the section's data into the MCU's virtual address space. +pub(super) struct ParsedSection { + /// Byte offset range within the firmware binary where this section's data resides. + pub(super) data_range: Range, + /// MCU virtual address range where this section should be mapped. + pub(super) va: Range, + /// Memory protection and caching flags for the mapping. + pub(super) vm_map_flags: VmMapFlags, +} + +/// A bare-bones `std::io::Cursor<[u8]>` clone to keep track of the current position in the +/// firmware binary. +/// +/// Provides methods to sequentially read primitive types and byte arrays from the firmware +/// binary while maintaining the current read position. +struct Cursor<'a> { + dev: &'a Device, + data: &'a [u8], + pos: usize, +} + +impl<'a> Cursor<'a> { + fn new(dev: &'a Device, data: &'a [u8]) -> Self { + Self { dev, data, pos: 0 } + } + + fn len(&self) -> usize { + self.data.len() + } + + fn pos(&self) -> usize { + self.pos + } + + /// Returns a view into the cursor's data. + /// + /// This spawns a new cursor, leaving the current cursor unchanged. + fn view(&self, range: Range) -> Result> { + if range.start < self.pos || range.end > self.data.len() { + dev_err!( + self.dev, + "Invalid cursor range {:?} for data of length {}", + range, + self.data.len() + ); + + Err(EINVAL) + } else { + Ok(Self { + dev: self.dev, + data: &self.data[range], + pos: 0, + }) + } + } + + /// Reads a slice of bytes from the current position and advances the cursor. + /// + /// Returns an error if the read would exceed the data bounds. + fn read(&mut self, nbytes: usize) -> Result<&[u8]> { + let start = self.pos; + let end = start + nbytes; + + if end > self.data.len() { + dev_err!( + self.dev, + "Invalid firmware file: read of size {} at position {} is out of bounds", + nbytes, + start, + ); + return Err(EINVAL); + } + + self.pos += nbytes; + Ok(&self.data[start..end]) + } + + /// Reads a little-endian `u8` from the current position and advances the cursor. + fn read_u8(&mut self) -> Result { + let bytes = self.read(size_of::())?; + Ok(bytes[0]) + } + + /// Reads a little-endian `u16` from the current position and advances the cursor. + fn read_u16(&mut self) -> Result { + let bytes: [u8; 2] = self + .read(size_of::())? + .try_into() + .map_err(|_| EINVAL)?; + + Ok(u16::from_le_bytes(bytes)) + } + + /// Reads a little-endian `u32` from the current position and advances the cursor. + fn read_u32(&mut self) -> Result { + let bytes: [u8; 4] = self + .read(size_of::())? + .try_into() + .map_err(|_| EINVAL)?; + + Ok(u32::from_le_bytes(bytes)) + } + + /// Advances the cursor position by the specified number of bytes. + /// + /// Returns an error if the advance would exceed the data bounds. + fn advance(&mut self, nbytes: usize) -> Result { + if self.pos + nbytes > self.data.len() { + dev_err!( + self.dev, + "Invalid firmware file: advance of size {} at position {} is out of bounds", + nbytes, + self.pos, + ); + return Err(EINVAL); + } + self.pos += nbytes; + Ok(()) + } +} + +/// Parser for Mali CSF GPU firmware binaries. +/// +/// Parses the firmware binary format, extracting section metadata including virtual +/// address ranges, data offsets, and memory protection flags needed to load firmware +/// into the MCU's memory. +pub(super) struct FwParser<'a> { + cursor: Cursor<'a>, +} + +impl<'a> FwParser<'a> { + /// Creates a new firmware parser for the given firmware binary data. + pub(super) fn new(dev: &'a Device, data: &'a [u8]) -> Self { + Self { + cursor: Cursor::new(dev, data), + } + } + + /// Parses the firmware binary and returns a collection of parsed sections. + /// + /// This method validates the firmware header and iterates through all entries + /// in the binary, extracting section information needed for loading. + pub(super) fn parse(&mut self) -> Result> { + let fw_header = self.parse_fw_header()?; + let header_end = fw_header.size as usize; + + let mut parsed_sections = KVec::new(); + while self.cursor.pos() < header_end { + let entry_section = self.parse_entry(header_end)?; + + if let Some(inner) = entry_section.inner { + parsed_sections.push(inner, GFP_KERNEL)?; + } + } + + if parsed_sections.is_empty() { + dev_err!(self.cursor.dev, "Firmware contains no loadable sections"); + return Err(EINVAL); + } + + Ok(parsed_sections) + } + + fn parse_fw_header(&mut self) -> Result { + let fw_header: FirmwareHeader = match FirmwareHeader::new(&mut self.cursor) { + Ok(fw_header) => fw_header, + Err(e) => { + dev_err!(self.cursor.dev, "Invalid firmware file: {}", e.to_errno()); + return Err(e); + } + }; + + if fw_header.size as usize > self.cursor.len() { + dev_err!(self.cursor.dev, "Firmware image is truncated"); + return Err(EINVAL); + } + Ok(fw_header) + } + + fn parse_entry(&mut self, header_end: usize) -> Result { + let entry_start = self.cursor.pos(); + + let entry_header_end = entry_start + .checked_add(size_of::()) + .ok_or(EINVAL)?; + + if entry_header_end > header_end { + dev_err!( + self.cursor.dev, + "Firmware entry header at {:#x} exceeds header region ending at {:#x}", + entry_start, + header_end + ); + return Err(EINVAL); + } + + let entry_section = EntrySection { + entry_hdr: EntryHeader(self.cursor.read_u32()?), + inner: None, + }; + + let firmware_size = self.cursor.len(); + let entry_size = entry_section.entry_hdr.size() as usize; + + if self.cursor.pos() % size_of::() != 0 + || entry_size % size_of::() != 0 + || entry_size < size_of::() + { + dev_err!( + self.cursor.dev, + "Firmware entry isn't 32 bit aligned, offset={:#x} size={:#x}", + self.cursor.pos() - size_of::(), + entry_size + ); + return Err(EINVAL); + } + + let entry_end = entry_start.checked_add(entry_size).ok_or(EINVAL)?; + + if entry_end > header_end { + dev_err!( + self.cursor.dev, + "Firmware entry at {:#x} extends beyond header region ending at {:#x}", + entry_start, + header_end + ); + return Err(EINVAL); + } + + let section_hdr_size = entry_size - size_of::(); + + let entry_section = { + let mut entry_cursor = self.cursor.view(self.cursor.pos()..entry_end)?; + + match entry_section.entry_hdr.entry_type() { + Ok(EntryType::Iface) => Ok(EntrySection { + entry_hdr: entry_section.entry_hdr, + inner: Self::parse_section_entry(&mut entry_cursor, firmware_size)?, + }), + Ok( + EntryType::Config + | EntryType::FutfTest + | EntryType::TraceBuffer + | EntryType::TimelineMetadata + | EntryType::BuildInfoMetadata, + ) => Ok(entry_section), + + Err(_) => { + if entry_section.entry_hdr.optional() { + Ok(entry_section) + } else { + dev_err!( + self.cursor.dev, + "Failed to handle firmware entry type: {}", + entry_section.entry_hdr.entry_type_raw() + ); + Err(EINVAL) + } + } + } + }; + + if entry_section.is_ok() { + self.cursor.advance(section_hdr_size)?; + } + + entry_section + } + + fn parse_section_entry( + entry_cursor: &mut Cursor<'_>, + firmware_size: usize, + ) -> Result> { + let section_hdr: SectionHeader = SectionHeader::new(entry_cursor)?; + + if section_hdr.data.end < section_hdr.data.start { + dev_err!( + entry_cursor.dev, + "Firmware corrupted, data.end < data.start (0x{:x} < 0x{:x})", + section_hdr.data.end, + section_hdr.data.start + ); + return Err(EINVAL); + } + + if section_hdr.data.end as usize > firmware_size { + dev_err!( + entry_cursor.dev, + "Firmware data range {:#x}..{:#x} exceeds firmware size {:#x}", + section_hdr.data.start, + section_hdr.data.end, + firmware_size, + ); + return Err(EINVAL); + } + + if section_hdr.va.start as usize % SZ_4K != 0 || section_hdr.va.end as usize % SZ_4K != 0 { + dev_err!( + entry_cursor.dev, + "Firmware virtual address range {:#x}..{:#x} is not page aligned", + section_hdr.va.start, + section_hdr.va.end + ); + return Err(EINVAL); + } + + if section_hdr.section_flags.prot() { + dev_dbg!( + entry_cursor.dev, + "Firmware protected mode entry not supported, ignoring" + ); + return Ok(None); + } + + if section_hdr.va.start == CSF_MCU_SHARED_REGION_START + && !section_hdr.section_flags.shared() + { + dev_err!( + entry_cursor.dev, + "Interface at 0x{:x} must be shared", + CSF_MCU_SHARED_REGION_START + ); + return Err(EINVAL); + } + + if section_hdr.va.is_empty() { + return Ok(None); + } + + let mut vm_map_flags = VmMapFlags::empty(); + + if !section_hdr.section_flags.write() { + vm_map_flags |= VmFlag::Readonly; + } + + if !section_hdr.section_flags.exec() { + vm_map_flags |= VmFlag::Noexec; + } + + // TODO: As in Panthor, map coherent firmware sections uncached until the VM + // supports a coherent mapping attribute. + if section_hdr.section_flags.cache_mode() != CacheMode::Cached { + vm_map_flags |= VmFlag::Uncached; + } + + Ok(Some(ParsedSection { + data_range: section_hdr.data.clone(), + va: section_hdr.va, + vm_map_flags, + })) + } +} + +/// Firmware binary header containing version and size information. +/// +/// The header is located at the beginning of the firmware binary and contains +/// a magic value for validation, version information, and the total size of +/// all structured headers that follow. +#[expect(dead_code)] +struct FirmwareHeader { + /// Magic value to check binary validity. + magic: u32, + + /// Minor firmware version. + minor: u8, + + /// Major firmware version. + major: u8, + + /// Padding. Must be set to zero. + _padding1: u16, + + /// Firmware version hash. + version_hash: u32, + + /// Padding. Must be set to zero. + _padding2: u32, + + /// Total size of all the structured data headers at beginning of firmware binary. + size: u32, +} + +impl FirmwareHeader { + const FW_BINARY_MAGIC: u32 = 0xc3f13a6e; + const FW_BINARY_MAJOR_MAX: u8 = 0; + + /// Reads and validates a firmware header from the cursor. + /// + /// Verifies the magic value, version compatibility, and padding fields. + fn new(cursor: &mut Cursor<'_>) -> Result { + let magic = cursor.read_u32()?; + if magic != Self::FW_BINARY_MAGIC { + dev_err!(cursor.dev, "Invalid firmware magic"); + return Err(EINVAL); + } + + let minor = cursor.read_u8()?; + let major = cursor.read_u8()?; + + if major > Self::FW_BINARY_MAJOR_MAX { + dev_err!( + cursor.dev, + "Unsupported firmware binary header version {}.{} (expected {}.x)", + major, + minor, + Self::FW_BINARY_MAJOR_MAX + ); + return Err(EINVAL); + } + + let padding1 = cursor.read_u16()?; + let version_hash = cursor.read_u32()?; + let padding2 = cursor.read_u32()?; + let size = cursor.read_u32()?; + + if padding1 != 0 || padding2 != 0 { + dev_err!( + cursor.dev, + "Invalid firmware file: header padding is not zero" + ); + return Err(EINVAL); + } + + let fw_header = Self { + magic, + minor, + major, + _padding1: padding1, + version_hash, + _padding2: padding2, + size, + }; + + Ok(fw_header) + } +} + +/// Firmware section header for loading binary sections into MCU memory. +#[derive(Debug)] +struct SectionHeader { + section_flags: SectionFlags, + /// MCU virtual range to map this binary section to. + va: Range, + /// References the data in the FW binary. + data: Range, +} + +impl SectionHeader { + /// Reads and validates a section header from the cursor. + /// + /// Parses section flags, virtual address range, and data range from the firmware binary. + fn new(cursor: &mut Cursor<'_>) -> Result { + let section_flags = SectionFlags::try_from_fw(cursor.read_u32()?)?; + + let va_start = cursor.read_u32()?; + let va_end = cursor.read_u32()?; + + let va = va_start..va_end; + + if va.end < va.start { + dev_err!( + cursor.dev, + "Invalid firmware file: VA end precedes start at pos {}", + cursor.pos(), + ); + return Err(EINVAL); + } + + let data_start = cursor.read_u32()?; + let data_end = cursor.read_u32()?; + let data = data_start..data_end; + + Ok(Self { + section_flags, + va, + data, + }) + } +} + +/// A firmware entry containing a header and optional parsed section data. +/// +/// Represents a single entry in the firmware binary, which may contain loadable +/// section data or metadata that doesn't require loading. +struct EntrySection { + entry_hdr: EntryHeader, + inner: Option, +} + +/// Header for a firmware entry, packed into a single u32. +/// +/// The entry header encodes the entry type, size, and optional flag in a +/// 32-bit value with the following layout: +/// - Bits 0-7: Entry type +/// - Bits 8-15: Size in bytes +/// - Bit 31: Optional flag +struct EntryHeader(u32); + +impl EntryHeader { + fn entry_type_raw(&self) -> u8 { + (self.0 & 0xff) as u8 + } + + fn entry_type(&self) -> Result { + let v = self.entry_type_raw(); + EntryType::try_from(v) + } + + fn optional(&self) -> bool { + self.0 & bit_u32(31) != 0 + } + + fn size(&self) -> u32 { + self.0 >> 8 & 0xff + } +} + +#[derive(Clone, Copy, Debug)] +#[repr(u8)] +enum EntryType { + /// Host <-> FW interface. + Iface = 0, + /// FW config. + Config = 1, + /// Unit tests. + FutfTest = 2, + /// Trace buffer interface. + TraceBuffer = 3, + /// Timeline metadata interface. + TimelineMetadata = 4, + /// Metadata about how the FW binary was built. + BuildInfoMetadata = 6, +} + +impl TryFrom for EntryType { + type Error = Error; + + fn try_from(value: u8) -> Result { + match value { + 0 => Ok(EntryType::Iface), + 1 => Ok(EntryType::Config), + 2 => Ok(EntryType::FutfTest), + 3 => Ok(EntryType::TraceBuffer), + 4 => Ok(EntryType::TimelineMetadata), + 6 => Ok(EntryType::BuildInfoMetadata), + _ => Err(EINVAL), + } + } +} From 44e7e7f7cffb10a93bb88e7cb59b7b8b3e2deb1c Mon Sep 17 00:00:00 2001 From: Deborah Brouwer Date: Tue, 28 Jul 2026 11:39:29 -0700 Subject: [PATCH 121/137] drm/tyr: add Microcontroller Unit (MCU) booting Add a firmware module to load, parse, and map the MCU firmware sections into shared GEM memory at the required virtual addresses accessible by the GPU. Create a firmware instance during probe and store it inside the TyrDrmRegistrationData to keep it alive after probe. Use the firmware instance to boot the MCU. Remove the dead-code annotations from the MMU, VM, slot manager, and kernel BO code now that these paths are used by the firmware module. Update Kconfig to add the RUST_FW_LOADER_ABSTRACTIONS dependency required by this module. Co-developed-by: Boris Brezillon Signed-off-by: Boris Brezillon Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260728-fw-boot-b4-v10-7-9187aefa3f2f@collabora.com Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/Kconfig | 1 + drivers/gpu/drm/tyr/driver.rs | 23 ++- drivers/gpu/drm/tyr/fw.rs | 321 ++++++++++++++++++++++++++++++++++ drivers/gpu/drm/tyr/gem.rs | 3 - drivers/gpu/drm/tyr/tyr.rs | 1 + drivers/gpu/drm/tyr/vm.rs | 1 - 6 files changed, 341 insertions(+), 9 deletions(-) create mode 100644 drivers/gpu/drm/tyr/fw.rs diff --git a/drivers/gpu/drm/tyr/Kconfig b/drivers/gpu/drm/tyr/Kconfig index 79ea4bb214de..8f13e49f11f9 100644 --- a/drivers/gpu/drm/tyr/Kconfig +++ b/drivers/gpu/drm/tyr/Kconfig @@ -13,6 +13,7 @@ config DRM_TYR select IOMMU_IO_PGTABLE_LPAE select RUST_DRM_GEM_SHMEM_HELPER select RUST_DRM_GPUVM + select RUST_FW_LOADER_ABSTRACTIONS help Rust DRM driver for ARM Mali CSF-based GPUs. diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index f487b3996293..d78ad9d292ff 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -37,6 +37,7 @@ use crate::{ file::TyrDrmFileData, + fw::Firmware, gem::Bo, gpu, gpu::GpuInfo, @@ -67,6 +68,9 @@ pub(crate) struct TyrDrmRegistrationData<'drm> { /// Parent platform device. pub(crate) pdev: &'drm platform::Device, + /// Firmware sections. + pub(crate) fw: Firmware<'drm>, + #[pin] clks: Mutex, @@ -144,10 +148,21 @@ fn probe<'bound>( let unreg_dev = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - let _mmu = Mmu::new(pdev.as_ref(), iomem.as_arc_borrow(), &gpu_info)?; + let mmu = Mmu::new(pdev.as_ref(), iomem.as_arc_borrow(), &gpu_info)?; - let reg_data = try_pin_init!(TyrDrmRegistrationData { + let firmware = Firmware::new( + pdev.as_ref(), + iomem.clone(), + &unreg_dev, + mmu.as_arc_borrow(), + &gpu_info, + )?; + + firmware.boot()?; + + let reg_data = pin_init!(TyrDrmRegistrationData { pdev, + fw: firmware, clks <- new_mutex!(Clocks { core: core_clk, stacks: stacks_clk, @@ -167,9 +182,7 @@ fn probe<'bound>( let driver = TyrPlatformDriverData { _reg: reg }; - // We need this to be dev_info!() because dev_dbg!() does not work at - // all in Rust for now, and we need to see whether probe succeeded. - dev_info!(pdev, "Tyr initialized correctly.\n"); + dev_dbg!(pdev, "Tyr initialized correctly."); Ok(driver) } } diff --git a/drivers/gpu/drm/tyr/fw.rs b/drivers/gpu/drm/tyr/fw.rs new file mode 100644 index 000000000000..47d25c901bd0 --- /dev/null +++ b/drivers/gpu/drm/tyr/fw.rs @@ -0,0 +1,321 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +//! Firmware loading and management for Mali CSF GPU. +//! +//! This module handles loading the Mali GPU firmware binary, parsing it into sections, +//! and mapping those sections into the MCU's virtual address space. Each firmware section +//! has specific properties (read/write/execute permissions, cache modes) and must be loaded +//! at specific virtual addresses expected by the MCU. +//! +//! See [`Firmware`] for the main firmware management interface and [`Section`] for +//! individual firmware sections. +//! +//! [`Firmware`]: crate::fw::Firmware +//! [`Section`]: crate::fw::Section + +use kernel::{ + device::{ + Bound, + Device, // + }, + drm::{ + gem::BaseObject, // + }, + io::{ + poll, + Io, // + }, + num::Bounded, + prelude::*, + register, + str::CString, + sync::{ + Arc, + ArcBorrow, // + }, + time, // +}; + +use crate::{ + driver::{ + IoMem, + TyrDrmDevice, // + }, + fw::parser::{ + FwParser, + ParsedSection, // + }, + gem, + gem::{ + KernelBo, + KernelBoVaAlloc, // + }, + gpu::GpuInfo, + + mmu::Mmu, + regs::{ + gpu_control::{ + McuControlMode, + McuStatus, + GPU_ID, + MCU_CONTROL, + MCU_STATUS, // + }, // + job_control::{ + JOB_IRQ_CLEAR, + JOB_IRQ_RAWSTAT, // + }, // + }, + vm::Vm, // +}; + +mod parser; + +pub(super) const CSF_MCU_SHARED_REGION_START: u32 = 0x04000000; + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u8)] +pub(super) enum CacheMode { + None = 0, + Cached = 1, + UncachedCoherent = 2, + CachedCoherent = 3, +} + +impl From> for CacheMode { + fn from(value: Bounded) -> Self { + match value.get() { + 0 => Self::None, + 1 => Self::Cached, + 2 => Self::UncachedCoherent, + 3 => Self::CachedCoherent, + _ => unreachable!(), + } + } +} + +impl From for Bounded { + fn from(value: CacheMode) -> Self { + Bounded::try_new(value as u32).unwrap() + } +} + +register! { + #[allow(non_upper_case_globals)] + pub(super) SectionFlags(u32) @ 0x0 { + 0:0 read => bool; + 1:1 write => bool; + 2:2 exec => bool; + 4:3 cache_mode => CacheMode; + 5:5 prot => bool; + 30:30 shared => bool; + 31:31 zero => bool; + } +} + +impl SectionFlags { + const VALID_MASK: u32 = Self::READ_MASK + | Self::WRITE_MASK + | Self::EXEC_MASK + | Self::CACHE_MODE_MASK + | Self::PROT_MASK + | Self::SHARED_MASK + | Self::ZERO_MASK; + + fn try_from_fw(value: u32) -> Result { + if value & !Self::VALID_MASK != 0 { + Err(EINVAL) + } else { + Ok(Self::from_raw(value)) + } + } +} + +/// A parsed section of the firmware binary. +struct Section<'drm> { + // Raw firmware section data for reset purposes + #[expect(dead_code)] + data: KVec, + + // Keep the BO backing this firmware section so that both the + // GPU mapping and CPU mapping remain valid until the Section is dropped. + #[expect(dead_code)] + mem: gem::KernelBo<'drm>, +} + +/// Loaded firmware with sections mapped into MCU VM. +pub(crate) struct Firmware<'drm> { + /// Iomem need to access registers. + iomem: Arc>, + + /// MCU VM. + vm: Arc>, + + /// List of firmware sections. + #[expect(dead_code)] + sections: KVec>, +} + +impl<'drm> Drop for Firmware<'drm> { + fn drop(&mut self) { + // Stop the MCU before releasing its firmware mappings and memory. + let _ = self.stop(); + + // AS slots retain a VM ref, we need to kill the circular ref manually. + self.vm.kill(); + } +} + +impl<'drm> Firmware<'drm> { + fn init_section_mem(dev: &Device, mem: &mut KernelBo<'drm>, data: &KVec) -> Result { + if data.is_empty() { + return Ok(()); + } + + let vmap = mem.bo().vmap::<0>()?; + let size = mem.bo().size(); + + if data.len() > size { + dev_err!(dev, "fw section {} bigger than BO {}", data.len(), size); + return Err(EINVAL); + } + + for (i, &byte) in data.iter().enumerate() { + vmap.try_write8(byte, i)?; + } + + Ok(()) + } + + fn request(ddev: &TyrDrmDevice, gpu_info: &GpuInfo) -> Result { + let gpu_id = GPU_ID::from_raw(gpu_info.gpu_id); + + let path = CString::try_from_fmt(fmt!( + "arm/mali/arch{}.{}/mali_csffw.bin", + gpu_id.arch_major().get(), + gpu_id.arch_minor().get() + ))?; + + kernel::firmware::Firmware::request(&path, ddev.as_ref().as_ref()) + } + + fn load( + dev: &Device, + ddev: &TyrDrmDevice, + gpu_info: &GpuInfo, + ) -> Result<(kernel::firmware::Firmware, KVec)> { + let fw = Self::request(ddev, gpu_info)?; + let mut parser = FwParser::new(dev, fw.data()); + + let parsed_sections = parser.parse()?; + + Ok((fw, parsed_sections)) + } + + /// Load firmware and map sections into MCU VM. + pub(crate) fn new( + dev: &'drm Device, + iomem: Arc>, + ddev: &TyrDrmDevice, + mmu: ArcBorrow<'_, Mmu<'drm>>, + gpu_info: &GpuInfo, + ) -> Result> { + let vm = Vm::new(dev, ddev, mmu, gpu_info)?; + vm.activate()?; + + let result = (|| { + let (fw, parsed_sections) = Self::load(dev, ddev, gpu_info)?; + let mut sections = KVec::new(); + for parsed in parsed_sections { + let size = u64::from(parsed.va.end.checked_sub(parsed.va.start).ok_or(EINVAL)?); + + let va = u64::from(parsed.va.start); + + let mut mem = KernelBo::new( + ddev, + vm.clone(), + size, + KernelBoVaAlloc::Explicit(va), + parsed.vm_map_flags, + )?; + + let section_start = parsed.data_range.start as usize; + let section_end = parsed.data_range.end as usize; + let mut data = KVec::new(); + + // Ensure that the firmware slice is not out of bounds. + let fw_data = fw.data(); + let bytes = fw_data.get(section_start..section_end).ok_or(EINVAL)?; + data.extend_from_slice(bytes, GFP_KERNEL)?; + + Self::init_section_mem(dev, &mut mem, &data)?; + + sections.push(Section { data, mem }, GFP_KERNEL)?; + } + + Ok(Firmware { + iomem, + vm: vm.clone(), + sections, + }) + })(); + + if result.is_err() { + vm.kill(); + } + + result + } + + pub(crate) fn boot(&self) -> Result { + let io = &self.iomem; + + // Discard any stale global interrupt. + io.write_reg(JOB_IRQ_CLEAR::zeroed().with_glb(true)); + + io.write_reg(MCU_CONTROL::zeroed().with_req(McuControlMode::Auto)); + + if let Err(e) = poll::read_poll_timeout( + || Ok((io.read(MCU_STATUS), io.read(JOB_IRQ_RAWSTAT))), + |(mcu_status, irq_rawstat)| { + mcu_status.value() == McuStatus::Enabled && irq_rawstat.glb() + }, + time::Delta::from_millis(1), + time::Delta::from_millis(100), + ) { + let status = io.read(MCU_STATUS); + dev_err!( + self.vm.dev(), + "MCU failed to boot, status: {:?}", + status.value() + ); + return Err(e); + } + + io.write_reg(JOB_IRQ_CLEAR::zeroed().with_glb(true)); + + Ok(()) + } + + fn stop(&self) -> Result { + let io = &self.iomem; + io.write_reg(MCU_CONTROL::zeroed().with_req(McuControlMode::Disable)); + + if let Err(e) = poll::read_poll_timeout( + || Ok(io.read(MCU_STATUS)), + |status| status.value() == McuStatus::Disabled, + time::Delta::from_micros(10), + time::Delta::from_millis(100), + ) { + let status = io.read(MCU_STATUS); + dev_err!( + self.vm.dev(), + "MCU failed to stop, status: {:?}", + status.value() + ); + return Err(e); + } + + Ok(()) + } +} diff --git a/drivers/gpu/drm/tyr/gem.rs b/drivers/gpu/drm/tyr/gem.rs index 69e1c75e59a5..3bf3787f5c3f 100644 --- a/drivers/gpu/drm/tyr/gem.rs +++ b/drivers/gpu/drm/tyr/gem.rs @@ -72,7 +72,6 @@ pub(crate) fn new_dummy_object(ddev: &TyrDrmDevice) -> Result> { /// An automatic VA allocation strategy will be added in the future. pub(crate) enum KernelBoVaAlloc { /// Explicit VA address specified by the caller. - #[expect(dead_code)] Explicit(u64), } @@ -98,7 +97,6 @@ impl<'drm> KernelBo<'drm> { /// This function allocates a new shmem-backed GEM object and immediately maps /// it into the specified GPU virtual memory space. The mapping is automatically /// cleaned up when the [`KernelBo`] is dropped. - #[expect(dead_code)] pub(crate) fn new( ddev: &TyrDrmDevice, vm: Arc>, @@ -135,7 +133,6 @@ pub(crate) fn new( }) } - #[expect(dead_code)] pub(crate) fn bo(&self) -> &Bo { &self.bo } diff --git a/drivers/gpu/drm/tyr/tyr.rs b/drivers/gpu/drm/tyr/tyr.rs index 92f6885cdaae..e7ec450bdc9c 100644 --- a/drivers/gpu/drm/tyr/tyr.rs +++ b/drivers/gpu/drm/tyr/tyr.rs @@ -9,6 +9,7 @@ mod driver; mod file; +mod fw; mod gem; mod gpu; mod mmu; diff --git a/drivers/gpu/drm/tyr/vm.rs b/drivers/gpu/drm/tyr/vm.rs index aa7c1522b9d7..c5e307b1e241 100644 --- a/drivers/gpu/drm/tyr/vm.rs +++ b/drivers/gpu/drm/tyr/vm.rs @@ -6,7 +6,6 @@ //! the illusion of owning the entire virtual address (VA) range, similar to CPU virtual memory. //! Each virtual memory (VM) area is backed by ARM64 LPAE Stage 1 page tables and can be //! mapped into hardware address space (AS) slots for GPU execution. -#![expect(dead_code)] use core::marker::PhantomData; use core::ops::Range; From 5c9deba5578db5a3ea05be8a3e4a59ecb08ee76f Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Tue, 4 Aug 2026 14:41:12 +0900 Subject: [PATCH 122/137] gpu: nova-core: correct FRTS vidmem offset calculation Currently, the frts vidmem offset is calculated based on the non-wpr heap size and pmu reservation size, but this is not right. The layout actually looks like this: | non-wpr heap | WPR2 .. FRTS | PMU reserved | ... | VGA workspace | It's just by coincidence + generous alignment that the values happened to match. Instead, define a per-architecture reserved size at the end of the framebuffer and use this plus the PMU reserved size to calculate the frts vidmem offset. Fixes: d317e4585fa3 ("gpu: nova-core: Hopper/Blackwell: add FSP Chain of Trust boot") Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260804-blackwell-fixes-v4-1-ac858b6a1935@nvidia.com [acourbot: add comment clarifying reason for testing pmu_reserved_size.] [acourbot: make fb_end_reserved_size() return u64.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fb/hal/gb100.rs | 1 + drivers/gpu/nova-core/fb/hal/gb202.rs | 1 + drivers/gpu/nova-core/fsp.rs | 28 ++++++++++++++++++-------- drivers/gpu/nova-core/fsp/hal.rs | 4 ++++ drivers/gpu/nova-core/fsp/hal/gb100.rs | 6 ++++++ drivers/gpu/nova-core/fsp/hal/gb202.rs | 9 ++++++++- drivers/gpu/nova-core/fsp/hal/gh100.rs | 9 ++++++++- 7 files changed, 48 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/nova-core/fb/hal/gb100.rs b/drivers/gpu/nova-core/fb/hal/gb100.rs index ec55ec3fc7e1..7e5b0e3ffc67 100644 --- a/drivers/gpu/nova-core/fb/hal/gb100.rs +++ b/drivers/gpu/nova-core/fb/hal/gb100.rs @@ -80,6 +80,7 @@ fn write_sysmem_flush_page_gb100(bar: Bar0<'_>, addr: Bounded) { ); } +// This PMU reservation size is r570-specific. pub(super) const fn pmu_reserved_size_gb100() -> u32 { usize_into_u32::<{ const_align_up(SZ_8M + SZ_16M + SZ_4K, Alignment::new::()).unwrap() }>( ) diff --git a/drivers/gpu/nova-core/fb/hal/gb202.rs b/drivers/gpu/nova-core/fb/hal/gb202.rs index 69ba35d2ea08..c590e5b1269c 100644 --- a/drivers/gpu/nova-core/fb/hal/gb202.rs +++ b/drivers/gpu/nova-core/fb/hal/gb202.rs @@ -73,6 +73,7 @@ fn pmu_reserved_size(&self) -> u32 { fn non_wpr_heap_size(&self) -> u32 { // Non-WPR heap for GB20x (see Open RM: kgspGetNonWprHeapSize, GB202+). + // This size is r570-specific. u32::SZ_2M + u32::SZ_128K } diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index ba4544210e40..17d100a085f0 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -251,20 +251,32 @@ struct FspCotMessage { } impl FspCotMessage { + /// Computes the FRTS vidmem offset for the Chain-of-Trust message. It is measured backwards + /// from the end of the framebuffer. + fn frts_vidmem_offset(hal: &dyn hal::FspHal, fb_layout: &FbLayout) -> Result { + let mut offset = hal.fb_end_reserved_size(); + + // As per OpenRM's `kfspPrepareBootCommands_GH100`. + if fb_layout.pmu_reserved_size != 0 { + offset = (offset + u64::from(fb_layout.pmu_reserved_size)) + // The 2 MiB alignment is r570-specific. + .align_up(Alignment::new::()) + .ok_or(EINVAL)?; + } + + Ok(offset) + } + /// Returns an in-place initializer for [`FspCotMessage`]. fn new<'a>( fb_layout: &FbLayout, fsp_fw: &'a FspFirmware, args: &'a FmcBootArgs<'_>, ) -> Result + 'a> { - // frts_vidmem_offset is measured from the end of FB, so FRTS sits at - // (end of FB) - frts_vidmem_offset. - let frts_vidmem_offset = if !args.resume { - let frts_reserved_size = fb_layout.heap.len() + u64::from(fb_layout.pmu_reserved_size); + let hal = hal::fsp_hal(args.chipset).ok_or(ENOTSUPP)?; - frts_reserved_size - .align_up(Alignment::new::()) - .ok_or(EINVAL)? + let frts_vidmem_offset = if !args.resume { + Self::frts_vidmem_offset(hal, fb_layout)? } else { 0 }; @@ -275,7 +287,7 @@ fn new<'a>( 0 }; - let version = hal::fsp_hal(args.chipset).ok_or(ENOTSUPP)?.cot_version(); + let version = hal.cot_version(); let size = num::usize_into_u16::<{ core::mem::size_of::() }>(); Ok(init!(Self { diff --git a/drivers/gpu/nova-core/fsp/hal.rs b/drivers/gpu/nova-core/fsp/hal.rs index b6f2624bb13d..eaf5837ac5a8 100644 --- a/drivers/gpu/nova-core/fsp/hal.rs +++ b/drivers/gpu/nova-core/fsp/hal.rs @@ -19,6 +19,10 @@ pub(super) trait FspHal { /// Returns the FSP Chain of Trust protocol version this chipset advertises. fn cot_version(&self) -> u16; + + // TODO: consider moving this into the TLV firmware metadata when ready + /// Returns the size reserved at the end of the framebuffer, in bytes. + fn fb_end_reserved_size(&self) -> u64; } /// Returns the FSP HAL, or `None` if the architecture doesn't support FSP. diff --git a/drivers/gpu/nova-core/fsp/hal/gb100.rs b/drivers/gpu/nova-core/fsp/hal/gb100.rs index 42f5ecfc6400..7cf53aa3d1ff 100644 --- a/drivers/gpu/nova-core/fsp/hal/gb100.rs +++ b/drivers/gpu/nova-core/fsp/hal/gb100.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +use kernel::sizes::SizeConstants; + use crate::{ driver::Bar0, fsp::hal::FspHal, // @@ -17,6 +19,10 @@ fn fsp_boot_status(&self, bar: Bar0<'_>) -> u32 { fn cot_version(&self) -> u16 { 2 } + + fn fb_end_reserved_size(&self) -> u64 { + u64::SZ_2M + u64::SZ_128K + } } const GB100: Gb100 = Gb100; diff --git a/drivers/gpu/nova-core/fsp/hal/gb202.rs b/drivers/gpu/nova-core/fsp/hal/gb202.rs index 1091b169a645..e380bbc5d58d 100644 --- a/drivers/gpu/nova-core/fsp/hal/gb202.rs +++ b/drivers/gpu/nova-core/fsp/hal/gb202.rs @@ -1,7 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -use kernel::io::Io; +use kernel::{ + io::Io, + sizes::SizeConstants, // +}; use crate::{ driver::Bar0, @@ -21,6 +24,10 @@ fn fsp_boot_status(&self, bar: Bar0<'_>) -> u32 { fn cot_version(&self) -> u16 { 2 } + + fn fb_end_reserved_size(&self) -> u64 { + u64::SZ_2M + u64::SZ_128K + } } const GB202: Gb202 = Gb202; diff --git a/drivers/gpu/nova-core/fsp/hal/gh100.rs b/drivers/gpu/nova-core/fsp/hal/gh100.rs index 291acaf2845a..9a8563799da8 100644 --- a/drivers/gpu/nova-core/fsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/fsp/hal/gh100.rs @@ -1,7 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -use kernel::io::Io; +use kernel::{ + io::Io, + sizes::SizeConstants, // +}; use crate::{ driver::Bar0, @@ -26,6 +29,10 @@ fn fsp_boot_status(&self, bar: Bar0<'_>) -> u32 { fn cot_version(&self) -> u16 { 1 } + + fn fb_end_reserved_size(&self) -> u64 { + u64::SZ_2M + } } const GH100: Gh100 = Gh100; From b2bc53d34435043001223cbbd9429b0b6d71bc59 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Tue, 4 Aug 2026 14:41:13 +0900 Subject: [PATCH 123/137] gpu: nova-core: rename heap size field This field is called non_wpr_heap_size everywhere else. Unify the name to make it more obvious which heap it is. Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260804-blackwell-fixes-v4-2-ac858b6a1935@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fb.rs | 10 +++++----- drivers/gpu/nova-core/gsp/fw.rs | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 9e475efb1150..4a1be29cf5fb 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -164,7 +164,7 @@ pub(crate) struct FbLayout { pub(crate) wpr2_heap: FbRange, /// WPR2 region range, starting with an instance of `GspFwWprMeta`. pub(crate) wpr2: FbRange, - pub(crate) heap: FbRange, + pub(crate) non_wpr_heap: FbRange, pub(crate) vf_partition_count: u8, /// PMU reserved memory size, in bytes. pub(crate) pmu_reserved_size: u32, @@ -270,9 +270,9 @@ pub(crate) fn new( FbRange(wpr2_addr..frts.end) }; - let heap = { - let heap_size = u64::from(hal.non_wpr_heap_size()); - FbRange(wpr2.start - heap_size..wpr2.start) + let non_wpr_heap = { + let non_wpr_heap_size = u64::from(hal.non_wpr_heap_size()); + FbRange(wpr2.start - non_wpr_heap_size..wpr2.start) }; Ok(Self { @@ -283,7 +283,7 @@ pub(crate) fn new( elf, wpr2_heap, wpr2, - heap, + non_wpr_heap, vf_partition_count, pmu_reserved_size: hal.pmu_reserved_size(), }) diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 6e8e7d822ef1..a7637bf3f3df 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -196,9 +196,9 @@ pub(crate) fn new<'a>( sizeOfSignature: u64::from_safe_cast(gsp_firmware.signatures.size()), }, }, - gspFwRsvdStart: fb_layout.heap.start, - nonWprHeapOffset: fb_layout.heap.start, - nonWprHeapSize: fb_layout.heap.end - fb_layout.heap.start, + gspFwRsvdStart: fb_layout.non_wpr_heap.start, + nonWprHeapOffset: fb_layout.non_wpr_heap.start, + nonWprHeapSize: fb_layout.non_wpr_heap.end - fb_layout.non_wpr_heap.start, gspFwWprStart: fb_layout.wpr2.start, gspFwHeapOffset: fb_layout.wpr2_heap.start, gspFwHeapSize: fb_layout.wpr2_heap.end - fb_layout.wpr2_heap.start, From 744e168d62fc50b362a4835f77feafd417b965a1 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Tue, 4 Aug 2026 14:41:14 +0900 Subject: [PATCH 124/137] gpu: nova-core: return non-WPR heap size as u64 from HALs This is always immediately widened to u64, so just return it as a u64 from the beginning. Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260804-blackwell-fixes-v4-3-ac858b6a1935@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fb.rs | 2 +- drivers/gpu/nova-core/fb/hal.rs | 2 +- drivers/gpu/nova-core/fb/hal/ga100.rs | 2 +- drivers/gpu/nova-core/fb/hal/ga102.rs | 2 +- drivers/gpu/nova-core/fb/hal/gb100.rs | 4 ++-- drivers/gpu/nova-core/fb/hal/gb202.rs | 4 ++-- drivers/gpu/nova-core/fb/hal/gh100.rs | 4 ++-- drivers/gpu/nova-core/fb/hal/tu102.rs | 6 +++--- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 4a1be29cf5fb..86d2bdaab7f9 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -271,7 +271,7 @@ pub(crate) fn new( }; let non_wpr_heap = { - let non_wpr_heap_size = u64::from(hal.non_wpr_heap_size()); + let non_wpr_heap_size = hal.non_wpr_heap_size(); FbRange(wpr2.start - non_wpr_heap_size..wpr2.start) }; diff --git a/drivers/gpu/nova-core/fb/hal.rs b/drivers/gpu/nova-core/fb/hal.rs index 714f0b51cd8f..99363cfb116b 100644 --- a/drivers/gpu/nova-core/fb/hal.rs +++ b/drivers/gpu/nova-core/fb/hal.rs @@ -37,7 +37,7 @@ pub(crate) trait FbHal { fn pmu_reserved_size(&self) -> u32; /// Returns the non-WPR heap size for this chipset, in bytes. - fn non_wpr_heap_size(&self) -> u32; + fn non_wpr_heap_size(&self) -> u64; /// Returns the FRTS size, in bytes. fn frts_size(&self) -> u64; diff --git a/drivers/gpu/nova-core/fb/hal/ga100.rs b/drivers/gpu/nova-core/fb/hal/ga100.rs index d13c9a826eef..a81b3e42ae41 100644 --- a/drivers/gpu/nova-core/fb/hal/ga100.rs +++ b/drivers/gpu/nova-core/fb/hal/ga100.rs @@ -74,7 +74,7 @@ fn pmu_reserved_size(&self) -> u32 { super::tu102::pmu_reserved_size_tu102() } - fn non_wpr_heap_size(&self) -> u32 { + fn non_wpr_heap_size(&self) -> u64 { super::tu102::non_wpr_heap_size_tu102() } diff --git a/drivers/gpu/nova-core/fb/hal/ga102.rs b/drivers/gpu/nova-core/fb/hal/ga102.rs index 44a2cf8a00f1..a5eeddb8a1ae 100644 --- a/drivers/gpu/nova-core/fb/hal/ga102.rs +++ b/drivers/gpu/nova-core/fb/hal/ga102.rs @@ -41,7 +41,7 @@ fn pmu_reserved_size(&self) -> u32 { super::tu102::pmu_reserved_size_tu102() } - fn non_wpr_heap_size(&self) -> u32 { + fn non_wpr_heap_size(&self) -> u64 { super::tu102::non_wpr_heap_size_tu102() } diff --git a/drivers/gpu/nova-core/fb/hal/gb100.rs b/drivers/gpu/nova-core/fb/hal/gb100.rs index 7e5b0e3ffc67..d9e4d62ae632 100644 --- a/drivers/gpu/nova-core/fb/hal/gb100.rs +++ b/drivers/gpu/nova-core/fb/hal/gb100.rs @@ -111,9 +111,9 @@ fn pmu_reserved_size(&self) -> u32 { pmu_reserved_size_gb100() } - fn non_wpr_heap_size(&self) -> u32 { + fn non_wpr_heap_size(&self) -> u64 { // Non-WPR heap for GB10x (see Open RM: kgspGetNonWprHeapSize, GB100/GB102). - u32::SZ_2M + u64::SZ_2M } fn frts_size(&self) -> u64 { diff --git a/drivers/gpu/nova-core/fb/hal/gb202.rs b/drivers/gpu/nova-core/fb/hal/gb202.rs index c590e5b1269c..4341ecf36188 100644 --- a/drivers/gpu/nova-core/fb/hal/gb202.rs +++ b/drivers/gpu/nova-core/fb/hal/gb202.rs @@ -71,10 +71,10 @@ fn pmu_reserved_size(&self) -> u32 { super::gb100::pmu_reserved_size_gb100() } - fn non_wpr_heap_size(&self) -> u32 { + fn non_wpr_heap_size(&self) -> u64 { // Non-WPR heap for GB20x (see Open RM: kgspGetNonWprHeapSize, GB202+). // This size is r570-specific. - u32::SZ_2M + u32::SZ_128K + u64::SZ_2M + u64::SZ_128K } fn frts_size(&self) -> u64 { diff --git a/drivers/gpu/nova-core/fb/hal/gh100.rs b/drivers/gpu/nova-core/fb/hal/gh100.rs index 2867ae058d0a..cf8e6403ef98 100644 --- a/drivers/gpu/nova-core/fb/hal/gh100.rs +++ b/drivers/gpu/nova-core/fb/hal/gh100.rs @@ -63,9 +63,9 @@ fn pmu_reserved_size(&self) -> u32 { super::tu102::pmu_reserved_size_tu102() } - fn non_wpr_heap_size(&self) -> u32 { + fn non_wpr_heap_size(&self) -> u64 { // Non-WPR heap for Hopper (see Open RM: kgspCalculateFbLayout_GH100). - u32::SZ_2M + u64::SZ_2M } fn frts_size(&self) -> u64 { diff --git a/drivers/gpu/nova-core/fb/hal/tu102.rs b/drivers/gpu/nova-core/fb/hal/tu102.rs index 541f163b52d3..bf5967cbffd3 100644 --- a/drivers/gpu/nova-core/fb/hal/tu102.rs +++ b/drivers/gpu/nova-core/fb/hal/tu102.rs @@ -46,8 +46,8 @@ pub(super) const fn pmu_reserved_size_tu102() -> u32 { 0 } -pub(super) const fn non_wpr_heap_size_tu102() -> u32 { - u32::SZ_1M +pub(super) const fn non_wpr_heap_size_tu102() -> u64 { + u64::SZ_1M } pub(super) const fn frts_size_tu102() -> u64 { @@ -77,7 +77,7 @@ fn pmu_reserved_size(&self) -> u32 { pmu_reserved_size_tu102() } - fn non_wpr_heap_size(&self) -> u32 { + fn non_wpr_heap_size(&self) -> u64 { non_wpr_heap_size_tu102() } From 6e46097f4d616a0f81083ae2c112efbb1473539e Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Tue, 4 Aug 2026 14:41:15 +0900 Subject: [PATCH 125/137] gpu: nova-core: split FbLayout into FSP and non-FSP versions `FbLayout` is currently used for both pre and post FSP architectures. It contains ranges for each region of framebuffer, but on post FSP architectures, only the size is actually used. The region locations are decided by ACR, which runs as part of the GSP-FMC, not by the driver. The driver only provides the sizes. So, for post FSP architectures `FbLayout` contains essentially guesses for the offsets. Instead, make separate types so that we only store the information that's actually needed, rather than keeping around offsets that may not be correct. Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260804-blackwell-fixes-v4-4-ac858b6a1935@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fb.rs | 65 ++++++++++++++++---- drivers/gpu/nova-core/fsp.rs | 21 +++---- drivers/gpu/nova-core/gsp/boot.rs | 28 +++------ drivers/gpu/nova-core/gsp/fw.rs | 85 ++++++++++++++++++++------ drivers/gpu/nova-core/gsp/hal.rs | 12 +--- drivers/gpu/nova-core/gsp/hal/gh100.rs | 17 ++++-- drivers/gpu/nova-core/gsp/hal/tu102.rs | 32 ++++++---- 7 files changed, 170 insertions(+), 90 deletions(-) diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 86d2bdaab7f9..77447a6567a4 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -149,7 +149,7 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { /// /// Contains ranges of GPU memory reserved for a given purpose during the GSP boot process. #[derive(Debug)] -pub(crate) struct FbLayout { +pub(crate) struct FbRanges { /// Range of the framebuffer. Starts at `0`. pub(crate) fb: FbRange, /// VGA workspace, small area of reserved memory at the end of the framebuffer. @@ -164,14 +164,16 @@ pub(crate) struct FbLayout { pub(crate) wpr2_heap: FbRange, /// WPR2 region range, starting with an instance of `GspFwWprMeta`. pub(crate) wpr2: FbRange, + /// Non-WPR heap, located just below WPR2. pub(crate) non_wpr_heap: FbRange, + /// Number of VF partitions. pub(crate) vf_partition_count: u8, /// PMU reserved memory size, in bytes. pub(crate) pmu_reserved_size: u32, } -impl FbLayout { - /// Computes the FB layout for `chipset` required to run the `gsp_fw` GSP firmware. +impl FbRanges { + /// Computes concrete framebuffer ranges required on non-FSP booting architectures. pub(crate) fn new( chipset: Chipset, bar: Bar0<'_>, @@ -240,16 +242,7 @@ pub(crate) fn new( FbRange(elf_addr..elf_addr + elf_size) }; - let (vf_partition_count, wpr2_heap_size) = match vgpu_state { - VgpuState::Disabled => ( - 0, - gsp::LibosParams::from_chipset(chipset).wpr_heap_size(chipset, fb.end)?, - ), - VgpuState::Enabled { total_vfs } => ( - u8::try_from(total_vfs.get()).map_err(|_| EINVAL)?, - gsp::LibosParams::vgpu_wpr_heap_size(), - ), - }; + let (vf_partition_count, wpr2_heap_size) = wpr2_heap_params(chipset, vgpu_state, fb.end)?; let wpr2_heap = { const WPR2_HEAP_DOWN_ALIGN: Alignment = Alignment::new::(); @@ -303,3 +296,49 @@ pub(crate) fn wpr2_range(bar: Bar0<'_>) -> Option> { Some(wpr2_lo.lower_bound()..wpr2_hi.higher_bound()) } + +/// Computes the number of VF partitions and the WPR2 heap size from the vGPU state. +fn wpr2_heap_params(chipset: Chipset, vgpu_state: VgpuState, fb_size: u64) -> Result<(u8, u64)> { + Ok(match vgpu_state { + VgpuState::Disabled => ( + 0, + gsp::LibosParams::from_chipset(chipset).wpr_heap_size(chipset, fb_size)?, + ), + VgpuState::Enabled { total_vfs } => ( + u8::try_from(total_vfs.get()).map_err(|_| EINVAL)?, + gsp::LibosParams::vgpu_wpr_heap_size(), + ), + }) +} + +/// Framebuffer region sizes needed for GSP-FMC boot. +#[derive(Debug)] +pub(crate) struct FbSizes { + /// FRTS size, in bytes. + pub(crate) frts_size: u64, + /// WPR2 heap size, in bytes. + pub(crate) wpr2_heap_size: u64, + /// Non-WPR heap size, in bytes. + pub(crate) non_wpr_heap_size: u64, + /// PMU reserved memory size, in bytes. + pub(crate) pmu_reserved_size: u32, + /// Number of VF partitions. + pub(crate) vf_partition_count: u8, +} + +impl FbSizes { + /// Computes the framebuffer region sizes for GSP-FMC boot. + pub(crate) fn new(chipset: Chipset, bar: Bar0<'_>, vgpu_state: VgpuState) -> Result { + let hal = hal::fb_hal(chipset); + let fb_size = hal.vidmem_size(bar); + let (vf_partition_count, wpr2_heap_size) = wpr2_heap_params(chipset, vgpu_state, fb_size)?; + + Ok(Self { + frts_size: hal.frts_size(), + wpr2_heap_size, + non_wpr_heap_size: hal.non_wpr_heap_size(), + pmu_reserved_size: hal.pmu_reserved_size(), + vf_partition_count, + }) + } +} diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index 17d100a085f0..2c9b050f6139 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -31,7 +31,7 @@ fsp::Fsp as FspEngine, Falcon, // }, - fb::FbLayout, + fb::FbSizes, firmware::{ fsp::{ FmcSignatures, @@ -253,12 +253,12 @@ struct FspCotMessage { impl FspCotMessage { /// Computes the FRTS vidmem offset for the Chain-of-Trust message. It is measured backwards /// from the end of the framebuffer. - fn frts_vidmem_offset(hal: &dyn hal::FspHal, fb_layout: &FbLayout) -> Result { + fn frts_vidmem_offset(hal: &dyn hal::FspHal, fb_info: &FbSizes) -> Result { let mut offset = hal.fb_end_reserved_size(); // As per OpenRM's `kfspPrepareBootCommands_GH100`. - if fb_layout.pmu_reserved_size != 0 { - offset = (offset + u64::from(fb_layout.pmu_reserved_size)) + if fb_info.pmu_reserved_size != 0 { + offset = (offset + u64::from(fb_info.pmu_reserved_size)) // The 2 MiB alignment is r570-specific. .align_up(Alignment::new::()) .ok_or(EINVAL)?; @@ -269,20 +269,20 @@ fn frts_vidmem_offset(hal: &dyn hal::FspHal, fb_layout: &FbLayout) -> Result( - fb_layout: &FbLayout, + fb_info: &FbSizes, fsp_fw: &'a FspFirmware, args: &'a FmcBootArgs<'_>, ) -> Result + 'a> { let hal = hal::fsp_hal(args.chipset).ok_or(ENOTSUPP)?; let frts_vidmem_offset = if !args.resume { - Self::frts_vidmem_offset(hal, fb_layout)? + Self::frts_vidmem_offset(hal, fb_info)? } else { 0 }; let frts_size: u32 = if !args.resume { - fb_layout.frts.len().try_into()? + fb_info.frts_size.try_into()? } else { 0 }; @@ -538,15 +538,12 @@ pub(crate) fn read_vgpu_mode( pub(crate) fn boot_fmc( &mut self, dev: &device::Device, - fb_layout: &FbLayout, + fb_info: &FbSizes, args: &FmcBootArgs<'_>, ) -> Result { dev_dbg!(dev, "Starting FSP boot sequence for {}\n", args.chipset); - let msg = KBox::init( - FspCotMessage::new(fb_layout, &self.fsp_fw, args)?, - GFP_KERNEL, - )?; + let msg = KBox::init(FspCotMessage::new(fb_info, &self.fsp_fw, args)?, GFP_KERNEL)?; let _response_buf = self.send_sync_fsp(dev, &*msg)?; diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 727b8ae4bcb7..97f6e7ef4ead 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -3,7 +3,6 @@ use kernel::{ bits, - dma::Coherent, io::poll::read_poll_timeout, prelude::*, time::Delta, @@ -16,15 +15,13 @@ gsp::Gsp, Falcon, // }, - fb::FbLayout, firmware::{ gsp::GspFirmware, FIRMWARE_VERSION, // }, gsp::{ cmdq::Cmdq, - commands, - GspFwWprMeta, // + commands, // }, }; @@ -50,23 +47,16 @@ pub(crate) fn boot( let gsp_fw = KBox::pin_init(GspFirmware::new(dev, chipset, FIRMWARE_VERSION), GFP_KERNEL)?; - let fb_layout = FbLayout::new(chipset, bar, &gsp_fw, ctx.vgpu.state())?; - dev_dbg!(dev, "{:#x?}\n", fb_layout); - - let wpr_meta = Coherent::init(dev, GFP_KERNEL, GspFwWprMeta::new(&gsp_fw, &fb_layout))?; - // Perform the chipset-specific boot sequence, and retrieve the unload bundle. - let unload_bundle = hal - .boot(&self, &mut ctx, &fb_layout, &wpr_meta)? - .or_else(|| { - dev_warn!(dev, "The GSP won't be able to unload properly on unbind.\n"); - dev_warn!( - dev, - "The GPU will need to be reset before the driver can bind again.\n" - ); + let unload_bundle = hal.boot(&self, &mut ctx, &gsp_fw)?.or_else(|| { + dev_warn!(dev, "The GSP won't be able to unload properly on unbind.\n"); + dev_warn!( + dev, + "The GPU will need to be reset before the driver can bind again.\n" + ); - None - }); + None + }); let mut unload_guard = ScopeGuard::new_with_data((ctx, unload_bundle), |(ctx, unload_bundle)| { diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index a7637bf3f3df..998293656794 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -36,7 +36,10 @@ }; use crate::{ - fb::FbLayout, + fb::{ + FbRanges, + FbSizes, // + }, firmware::gsp::GspFirmware, gpu::{ Architecture, @@ -174,10 +177,10 @@ unsafe impl FromBytes for GspFwWprMeta {} impl GspFwWprMeta { /// Returns an initializer for a `GspFwWprMeta` suitable for booting `gsp_firmware` using the - /// `fb_layout` layout. - pub(crate) fn new<'a>( + /// framebuffer ranges `ranges`. + pub(crate) fn from_ranges<'a>( gsp_firmware: &'a GspFirmware, - fb_layout: &'a FbLayout, + ranges: &'a FbRanges, ) -> impl Init + 'a { let init_inner = init!(bindings::GspFwWprMeta { // CAST: we want to store the bits of `GSP_FW_WPR_META_MAGIC` unmodified. @@ -196,25 +199,67 @@ pub(crate) fn new<'a>( sizeOfSignature: u64::from_safe_cast(gsp_firmware.signatures.size()), }, }, - gspFwRsvdStart: fb_layout.non_wpr_heap.start, - nonWprHeapOffset: fb_layout.non_wpr_heap.start, - nonWprHeapSize: fb_layout.non_wpr_heap.end - fb_layout.non_wpr_heap.start, - gspFwWprStart: fb_layout.wpr2.start, - gspFwHeapOffset: fb_layout.wpr2_heap.start, - gspFwHeapSize: fb_layout.wpr2_heap.end - fb_layout.wpr2_heap.start, - gspFwOffset: fb_layout.elf.start, - bootBinOffset: fb_layout.boot.start, - frtsOffset: fb_layout.frts.start, - frtsSize: fb_layout.frts.end - fb_layout.frts.start, - gspFwWprEnd: fb_layout + gspFwRsvdStart: ranges.non_wpr_heap.start, + nonWprHeapOffset: ranges.non_wpr_heap.start, + nonWprHeapSize: ranges.non_wpr_heap.len(), + gspFwWprStart: ranges.wpr2.start, + gspFwHeapOffset: ranges.wpr2_heap.start, + gspFwHeapSize: ranges.wpr2_heap.len(), + gspFwOffset: ranges.elf.start, + bootBinOffset: ranges.boot.start, + frtsOffset: ranges.frts.start, + frtsSize: ranges.frts.len(), + gspFwWprEnd: ranges .vga_workspace .start .align_down(Alignment::new::()), - gspFwHeapVfPartitionCount: fb_layout.vf_partition_count, - fbSize: fb_layout.fb.end - fb_layout.fb.start, - vgaWorkspaceOffset: fb_layout.vga_workspace.start, - vgaWorkspaceSize: fb_layout.vga_workspace.end - fb_layout.vga_workspace.start, - pmuReservedSize: fb_layout.pmu_reserved_size, + gspFwHeapVfPartitionCount: ranges.vf_partition_count, + fbSize: ranges.fb.len(), + vgaWorkspaceOffset: ranges.vga_workspace.start, + vgaWorkspaceSize: ranges.vga_workspace.len(), + pmuReservedSize: ranges.pmu_reserved_size, + ..Zeroable::init_zeroed() + }); + + init!(GspFwWprMeta { + inner <- init_inner, + }) + } + + /// Returns an initializer for a `GspFwWprMeta` suitable for booting `gsp_firmware` using the + /// framebuffer region sizes `sizes`. + /// + /// The region offsets are left at zero: the ACR ucode computes them when it sets up WPR2. + pub(crate) fn from_sizes<'a>( + gsp_firmware: &'a GspFirmware, + sizes: &'a FbSizes, + ) -> impl Init + 'a { + /// VGA workspace size to reserve at the end of the framebuffer, in bytes. + const VGA_WORKSPACE_SIZE: u64 = u64::SZ_128K; + + let init_inner = init!(bindings::GspFwWprMeta { + // CAST: we want to store the bits of `GSP_FW_WPR_META_MAGIC` unmodified. + magic: bindings::GSP_FW_WPR_META_MAGIC as u64, + revision: u64::from(bindings::GSP_FW_WPR_META_REVISION), + sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_handle(), + sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size), + sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_handle(), + sizeOfBootloader: u64::from_safe_cast(gsp_firmware.bootloader.ucode.size()), + bootloaderCodeOffset: u64::from(gsp_firmware.bootloader.code_offset), + bootloaderDataOffset: u64::from(gsp_firmware.bootloader.data_offset), + bootloaderManifestOffset: u64::from(gsp_firmware.bootloader.manifest_offset), + __bindgen_anon_1: GspFwWprMetaBootResumeInfo { + __bindgen_anon_1: GspFwWprMetaBootInfo { + sysmemAddrOfSignature: gsp_firmware.signatures.dma_handle(), + sizeOfSignature: u64::from_safe_cast(gsp_firmware.signatures.size()), + }, + }, + nonWprHeapSize: sizes.non_wpr_heap_size, + gspFwHeapSize: sizes.wpr2_heap_size, + frtsSize: sizes.frts_size, + gspFwHeapVfPartitionCount: sizes.vf_partition_count, + vgaWorkspaceSize: VGA_WORKSPACE_SIZE, + pmuReservedSize: sizes.pmu_reserved_size, ..Zeroable::init_zeroed() }); diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs index 34b4bb82a999..11e436651a69 100644 --- a/drivers/gpu/nova-core/gsp/hal.rs +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -5,13 +5,9 @@ mod gh100; mod tu102; -use kernel::{ - dma::Coherent, - prelude::*, // -}; +use kernel::prelude::*; use crate::{ - fb::FbLayout, firmware::gsp::GspFirmware, gpu::{ Architecture, @@ -19,8 +15,7 @@ }, gsp::{ Gsp, - GspBootContext, - GspFwWprMeta, // + GspBootContext, // }, }; @@ -44,8 +39,7 @@ fn boot( &self, gsp: &Gsp, ctx: &mut GspBootContext<'_, '_>, - fb_layout: &FbLayout, - wpr_meta: &Coherent, + gsp_fw: &GspFirmware, ) -> Result>; /// Performs HAL-specific post-GSP boot tasks. diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index 22b60f9233de..be10d278f567 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -16,7 +16,8 @@ gsp::Gsp as GspEngine, Falcon, // }, - fb::FbLayout, + fb::FbSizes, + firmware::gsp::GspFirmware, fsp::FmcBootArgs, gsp::{ hal::{ @@ -144,19 +145,23 @@ fn boot( &self, gsp: &Gsp, ctx: &mut GspBootContext<'_, '_>, - fb_layout: &FbLayout, - wpr_meta: &Coherent, + gsp_fw: &GspFirmware, ) -> Result> { let dev = ctx.dev(); let chipset = ctx.chipset; let gsp_falcon = ctx.gsp_falcon; + let fb_sizes = FbSizes::new(chipset, ctx.bar, ctx.vgpu.state())?; + dev_dbg!(dev, "{:#x?}\n", fb_sizes); + + let wpr_meta = + Coherent::init(dev, GFP_KERNEL, GspFwWprMeta::from_sizes(gsp_fw, &fb_sizes))?; + let args = FmcBootArgs::new(dev, chipset, &wpr_meta, &gsp.libos, false)?; + let unload_bundle = crate::gsp::UnloadBundle( KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox ); - let args = FmcBootArgs::new(dev, chipset, wpr_meta, &gsp.libos, false)?; - // Wait for the GSP RISC-V core to halt in case of error. We create this guard after `args` // to make sure that boot args are kept alive until halt, in case they are still being // accessed. @@ -167,7 +172,7 @@ fn boot( let fsp = unload_guard.1.fsp.as_mut().ok_or(ENODEV)?; - fsp.boot_fmc(dev, fb_layout, &args)?; + fsp.boot_fmc(dev, &fb_sizes, &args)?; // Wait for GSP-FMC to release the GSP lockdown, indicating that `args` is not accessed // anymore. diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index 03133f723faf..e3c365cf4a68 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -19,7 +19,7 @@ }, fb::{ wpr2_range, - FbLayout, // + FbRanges, // }, firmware::{ booter::{ @@ -143,7 +143,7 @@ fn run_fwsec_frts( falcon: &Falcon<'_, GspEngine>, bar: Bar0<'_>, bios: &Vbios, - fb_layout: &FbLayout, + fb_ranges: &FbRanges, ) -> Result { // Check that the WPR2 region does not already exist - if it does, we cannot run // FWSEC-FRTS until the GPU is reset. @@ -161,8 +161,8 @@ fn run_fwsec_frts( falcon, bios, FwsecCommand::Frts { - frts_addr: fb_layout.frts.start, - frts_size: fb_layout.frts.len(), + frts_addr: fb_ranges.frts.start, + frts_size: fb_ranges.frts.len(), }, )?; @@ -196,12 +196,12 @@ fn run_fwsec_frts( return Err(EIO); }; - if wpr2_range.start != fb_layout.frts.start { + if wpr2_range.start != fb_ranges.frts.start { dev_err!( dev, "WPR2 region created at unexpected address {:#x}; expected {:#x}\n", wpr2_range.start, - fb_layout.frts.start, + fb_ranges.frts.start, ); return Err(EIO); @@ -253,8 +253,7 @@ fn boot( &self, gsp: &Gsp, ctx: &mut GspBootContext<'_, '_>, - fb_layout: &FbLayout, - wpr_meta: &Coherent, + gsp_fw: &GspFirmware, ) -> Result> { let dev = ctx.dev(); let bar = ctx.bar; @@ -262,6 +261,17 @@ fn boot( let gsp_falcon = ctx.gsp_falcon; let sec2_falcon = ctx.sec2_falcon; + let fb_ranges = FbRanges::new(chipset, bar, gsp_fw, ctx.vgpu.state())?; + dev_dbg!(dev, "{:#x?}\n", fb_ranges); + + // Declared before the unload guard so that if Booter fails while running, SEC2 is reset + // by the guard before this allocation is freed. + let wpr_meta = Coherent::init( + dev, + GFP_KERNEL, + GspFwWprMeta::from_ranges(gsp_fw, &fb_ranges), + )?; + let bios = Vbios::new(dev, bar)?; // Try and prepare the unload bundle. @@ -281,8 +291,8 @@ fn boot( }); // FWSEC-FRTS is not executed on chips where the FRTS region size is 0 (e.g. GA100). - if !fb_layout.frts.is_empty() { - self.run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, fb_layout)?; + if !fb_ranges.frts.is_empty() { + self.run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, &fb_ranges)?; } gsp_falcon.reset()?; @@ -303,7 +313,7 @@ fn boot( FIRMWARE_VERSION, sec2_falcon, )? - .run(dev, sec2_falcon, wpr_meta)?; + .run(dev, sec2_falcon, &wpr_meta)?; Ok(unload_guard.dismiss()) } From 528aef3a4bdd85137de9ec76bd02eacdf8160bb5 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Tue, 4 Aug 2026 14:41:16 +0900 Subject: [PATCH 126/137] gpu: nova-core: pass WPR metadata ownership to FmcBootArgs `FmcBootArgs` logically owns this, so pass ownership to it instead of storing a reference. Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260804-blackwell-fixes-v4-5-ac858b6a1935@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fsp.rs | 4 ++-- drivers/gpu/nova-core/gsp/hal/gh100.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index 2c9b050f6139..6263277a7614 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -351,7 +351,7 @@ pub(crate) struct FmcBootArgs<'a> { fmc_boot_params: Coherent, resume: bool, // Additional dependencies required to be kept alive for FMC boot. - _wpr_meta: &'a Coherent, + _wpr_meta: Coherent, _libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, } @@ -361,7 +361,7 @@ impl<'a> FmcBootArgs<'a> { pub(crate) fn new( dev: &device::Device, chipset: Chipset, - wpr_meta: &'a Coherent, + wpr_meta: Coherent, libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, resume: bool, ) -> Result { diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index be10d278f567..b16c2f6f82a0 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -156,15 +156,15 @@ fn boot( let wpr_meta = Coherent::init(dev, GFP_KERNEL, GspFwWprMeta::from_sizes(gsp_fw, &fb_sizes))?; - let args = FmcBootArgs::new(dev, chipset, &wpr_meta, &gsp.libos, false)?; + let args = FmcBootArgs::new(dev, chipset, wpr_meta, &gsp.libos, false)?; let unload_bundle = crate::gsp::UnloadBundle( KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox ); // Wait for the GSP RISC-V core to halt in case of error. We create this guard after `args` - // to make sure that boot args are kept alive until halt, in case they are still being - // accessed. + // to make sure that the boot args and the WPR metadata they own are kept alive until halt, + // in case they are still being accessed. let mut unload_guard = ScopeGuard::new_with_data((unload_bundle, ctx), |(unload_bundle, ctx)| { let _ = unload_bundle.0.run(ctx); From 60782beb112ee1514ea7888adc9a53dc370fc522 Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Fri, 31 Jul 2026 15:10:10 -0500 Subject: [PATCH 127/137] rust: alloc: add Vec::zeroed method Add a constructor for kernel Vec that allocates a vector of a given length with all elements zero-initialized. Memory is allocated with the __GFP_ZERO flag, matching the existing KBox::zeroed() pattern. Signed-off-by: Timur Tabi Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260731201017.2580713-2-ttabi@nvidia.com Co-developed-by: Danilo Krummrich Signed-off-by: Danilo Krummrich --- rust/kernel/alloc/kvec.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index f7af62835aa8..c7546b9da4fa 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -9,6 +9,7 @@ Vmalloc, VmallocPageIter, // }, + flags::__GFP_ZERO, layout::ArrayLayout, AllocError, Allocator, @@ -51,6 +52,8 @@ }, // }; +use pin_init::Zeroable; + mod errors; pub use self::errors::{InsertError, PushError, RemoveError}; @@ -532,6 +535,30 @@ pub fn with_capacity(capacity: usize, flags: Flags) -> Result Ok(v) } + /// Creates a new [`Vec`] with `n` zero-initialized elements. + /// + /// # Examples + /// + /// ``` + /// let v = KVec::::zeroed(20, GFP_KERNEL)?; + /// + /// assert!(v.iter().all(|&x| x == 0)); + /// # Ok::<(), Error>(()) + /// ``` + pub fn zeroed(n: usize, flags: Flags) -> Result + where + T: Zeroable, + { + let mut v = Self::with_capacity(n, flags | __GFP_ZERO)?; + + // SAFETY: + // - `n <= capacity - len`: `with_capacity(n)` guarantees capacity >= n, len is 0. + // - All elements in `[0, n)` are initialized: `__GFP_ZERO` zeroes the allocation, + // and `T: Zeroable` guarantees all-zeroes is a valid bit pattern. + unsafe { v.inc_len(n) }; + Ok(v) + } + /// Creates a `Vec` from a pointer, a length and a capacity using the allocator `A`. /// /// # Examples From e6c2c62655211e2ad507209b02b99db6dec05630 Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Fri, 31 Jul 2026 15:10:11 -0500 Subject: [PATCH 128/137] rust: firmware: add request_into_buf() Add request_into_buf(), a Rust wrapper around the request_firmware_into_buf() function. This variant loads the firmware image directly into a caller-provided buffer rather than a kernel-allocated one. Signed-off-by: Timur Tabi Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260731201017.2580713-3-ttabi@nvidia.com [ Declare fw as *const to match the FFI out-parameter type and pass &raw mut directly, removing the redundant cast chain. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/firmware.rs | 46 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/rust/kernel/firmware.rs b/rust/kernel/firmware.rs index 71168d8004e2..6a6b392ffc4e 100644 --- a/rust/kernel/firmware.rs +++ b/rust/kernel/firmware.rs @@ -7,9 +7,9 @@ use crate::{ bindings, device::Device, - error::Error, - error::Result, + error::to_result, ffi, + prelude::*, str::{CStr, CStrExt as _}, }; use core::ptr::NonNull; @@ -120,6 +120,48 @@ fn drop(&mut self) { } } +/// Load firmware directly into the caller-provided `buf`. +/// +/// On success the firmware image has been copied into `buf`; the caller accesses the data +/// through `buf` itself. +/// +/// This is intentionally a stand-alone function rather than a `Firmware` constructor. For +/// the `into_buf` path, the firmware data lives in the caller's `buf`, not in a +/// kernel-owned buffer, so returning a `Firmware` would expose `Firmware::data()` as a +/// second handle aliasing `buf` (and `release_firmware()` does not free `buf` anyway). +pub fn request_into_buf(name: &CStr, dev: &Device, buf: &mut [u8]) -> Result { + // `as_mut_ptr()` on an empty slice returns a non-NULL pointer to + // memory which the loader does not own. Passing that pointer with `size == 0` + // makes the loader believe that it is buffer it allocated itself, so when + // `release_firmware()` is called, it will vfree the pointer and trigger a + // bug. Reject empty slices to avoid this situation. + if buf.is_empty() { + return Err(EINVAL); + } + + let mut fw: *const bindings::firmware = core::ptr::null(); + + // SAFETY: `&raw mut fw` is a valid pointer to a NULL initialized `bindings::firmware` pointer. + // `name` and `dev` are valid as by their type invariants. `buf` is a valid writable + // buffer of `buf.len()` bytes. + to_result(unsafe { + bindings::request_firmware_into_buf( + &raw mut fw, + name.as_char_ptr(), + dev.as_raw(), + buf.as_mut_ptr().cast(), + buf.len(), + ) + })?; + + // The firmware bytes are now in `buf`, which the caller owns, so we don't need + // the kernel to hang on to it any more. + // SAFETY: `fw` is a valid pointer returned by `request_firmware_into_buf`. + unsafe { bindings::release_firmware(fw) }; + + Ok(()) +} + // SAFETY: `Firmware` only holds a pointer to a C `struct firmware`, which is safe to be used from // any thread. unsafe impl Send for Firmware {} From 33f40117c0c53c88536936f14f87fc06279f1f2c Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Fri, 31 Jul 2026 15:10:12 -0500 Subject: [PATCH 129/137] gpu: nova-core: add TLV parser for firmware files TLV (type, length, value) files are the new image format used by Nova to encapsulate firmware images and their metadata. Unlike the firmware files for previous versions of the firmware, TLV filenames are not versioned, and they have a .tlv suffix. Add function request_tlv() to load TLV firmware images. Add the Tlv struct and supporting types for parsing TLV firmware images. TLV files begin with a 4-byte magic header, which must be "NVFW" for Nvidia firmware files. This is followed by a sequence of blocks each containing a 4-byte ASCII tag, a 4-byte little-endian length, and a payload padded to a 4-byte boundary. Tlv::new() validates the entire image up front, so that the iterator can subsequently yield blocks without fallible parsing. Also add accessor methods for the various encoded types that will be used by the driver. Signed-off-by: Timur Tabi Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260731201017.2580713-4-ttabi@nvidia.com [ Drop unnecessary payload.is_empty() check in Tlv::new(), use EINVAL instead of ENODATA in Tlv::get_bytes() and add a corresponding TODO comment. - Danilo ] Signed-off-by: Danilo Krummrich --- Documentation/gpu/nova/core/tlv.rst | 184 ++++++++++++++++++ Documentation/gpu/nova/index.rst | 1 + drivers/gpu/nova-core/firmware.rs | 1 + drivers/gpu/nova-core/firmware/tlv.rs | 264 ++++++++++++++++++++++++++ 4 files changed, 450 insertions(+) create mode 100644 Documentation/gpu/nova/core/tlv.rst create mode 100644 drivers/gpu/nova-core/firmware/tlv.rs diff --git a/Documentation/gpu/nova/core/tlv.rst b/Documentation/gpu/nova/core/tlv.rst new file mode 100644 index 000000000000..3ce508e9545a --- /dev/null +++ b/Documentation/gpu/nova/core/tlv.rst @@ -0,0 +1,184 @@ +.. SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +================================== +TLV Tags in Nova Firmware Images +================================== + +Nova firmware images use a Type-Length-Value (TLV) format to encapsulate +firmware components and metadata. The TLV file begins with a 4-byte "magic" +header that contains the string "NVFW". Following the header is a sequence of +TLV blocks. + +Each block consists of a 4-byte tag of ASCII characters, a 4-byte length +encoded as a little-endian unsigned integer, and a sequence of bytes, the size +of which is equal to the length rounded up to the next multiple of 4. + +The driver code that reads the TLV and uses its contents is called the parser. +It is the responsibility of the parser to handle missing or malformed tags, +lengths, and values in the TLV. + +:: + + +------+------+------+------+ + | 'N' | 'V' | 'F' | 'W' | Magic header + +------+------+------+------+ + | Tag (4 bytes, ASCII) | TLV block 0 + +---------------------------+ + | Length (4 bytes, LE) | + +---------------------------+ + | | + | Value (length bytes, | + | padded to 4-byte align) | + | | + +---------------------------+ + | Tag (4 bytes, ASCII) | TLV block 1 + +---------------------------+ + | Length (4 bytes, LE) | + +---------------------------+ + | | + | Value (length bytes, | + | padded to 4-byte align) | + | | + +---------------------------+ + | ... | More TLV blocks + +---------------------------+ + +Tags and Length +=============== +TLV tags are always four-character words, with all letters being upper case. +Duplicate tags are not allowed. + +A TLV file may contain additional tags not described in this document. + +Values +====== +Values are one of four types. The type is not encoded in the format; rather, +the parser expects a given tag to have a value of a given type. + +1) Integers, encoded in 32-bit or 64-bit little-endian format. +2) Strings, encoded as-is and required to be only printable ASCII characters + and without a null terminator. +3) An array of bytes, for binary data. +4) Boolean, encoded as single byte, with a value of 0 for False or 1 for True. + +Common Tags +=========== +These tags are shared across firmware types and carry the same meaning +wherever they appear. Unlike the firmware-specific tags below, a common tag +is reserved: its meaning is fixed and may never be redefined for a particular +firmware type. + +``VERS`` (string) + Human-readable firmware version string. Present in all TLV files. + +A TLV image must contain either a single ``BLOB`` tag (firmware embedded +inline) or a ``SIZE``/``FILE`` pair (firmware stored in a separate file). + +``BLOB`` (bytes) + If the firmware microcode binary is stored in the TLV, this tag contains + the actual firmware image bytes. + +``FILE`` (string) + If the firmware binary is stored as a separate file, this tag contains the + name of that file, which is required to be in the same directory as the TLV, + so no paths are allowed in the filename. This tag is always paired with + ``SIZE``, so as to allow the driver to pre-allocate the buffer before + loading the file. + +``SIZE`` (u32) + Total size in bytes of the firmware image to be loaded from the companion + file named by ``FILE``. This tag is mandatory if ``FILE`` exists, so the + size of the firmware image must be known when the TLV is created. If the + firmware image is updated and its size changes, then the TLV must be + updated with it. + +GSP Firmware Tags +================= +``SIGN`` (bytes) + Cryptographic signature for the GSP firmware. + +``BLID`` (string) + The build ID, extracted from the ".note.gnu.build-id" section. + +Booter Firmware Tags +==================== +``DAOF`` (u32) - ``os_data_offset`` + OS data section offset within the firmware image (absolute byte offset). + Maps to the DMEM load source. + +``DASZ`` (u32) - ``os_data_size`` + OS data section size in bytes. + +``CDOF`` (u32) - ``os_code_offset`` + OS code section offset within the firmware image (absolute byte offset). + Maps to the non-secure IMEM load source. + +``CDSZ`` (u32) - ``os_code_size`` + OS code section size in bytes. + +``PLOC`` (u32) - ``patch_loc`` + Signature patch location -- byte offset within the firmware image where the + selected signature should be written. + +``FUSE`` (u32) - ``fuse_version`` + Fuse version of the firmware, used with the hardware fuse register to + select the correct signature index. + +``ENID`` (u32) - ``engine_id`` + Engine ID mask identifying the falcon engine this firmware targets. + +``UCID`` (u32) - ``ucode_id`` + Microcode ID used together with the engine ID to query hardware signature + fuse registers. + +``A0CO`` (u32) - ``app0_code_offset`` + App0 code offset -- start of the secure code region within the firmware + image. Used as the IMEM secure section source. + +``A0CS`` (u32) - ``app0_code_size`` + App0 code size in bytes. + +``NSIG`` (u32) - ``num_sigs`` + Number of signatures included in the ``SIGN`` tag. + +``SIGN`` (bytes) + Concatenated array of firmware signatures. The size of each signature is + the total length of the ``SIGN`` value divided by ``NSIG``. The correct + signature is selected using the fuse-version-derived index. + +Generic Bootloader Tags +======================= +``CDSZ`` (u32) - ``code_size`` + Size in bytes of the bootloader code to copy from the ``BLOB`` tag and + PIO-load into falcon IMEM. + +``STRT`` (u32) - ``start_tag`` + Start tag identifying the IMEM block where execution begins. The falcon + boot address is derived as ``start_tag << 8``. + +GSP Bootloader Tags +=================== +``CDOF`` (u32) - ``code_offset`` + Offset within the firmware image at which the code section starts. + +``DAOF`` (u32) - ``data_offset`` + Offset within the firmware image at which the data section starts. + +``MFOF`` (u32) - ``manifest_offset`` + Offset within the firmware image at which the manifest starts. + +``APPV`` (u32) - ``app_version`` + Application version of the firmware. + +FMC Firmware Tags +================= +``HASH`` (bytes) + SHA-384 hash of the FMC firmware, exactly 48 bytes long. + +``PKEY`` (bytes) + Public key used to verify the FMC firmware. At most 384 bytes (RSA-3072), + but may be shorter. + +``SIGN`` (bytes) + Signature of the FMC firmware. At most 384 bytes (RSA-3072), but may + be shorter. diff --git a/Documentation/gpu/nova/index.rst b/Documentation/gpu/nova/index.rst index 1783513cbd05..2afa58e8f08d 100644 --- a/Documentation/gpu/nova/index.rst +++ b/Documentation/gpu/nova/index.rst @@ -33,3 +33,4 @@ vGPU manager VFIO driver and the nova-drm driver. core/fsp core/fwsec core/falcon + core/tlv diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index 20eff987c5d6..2075b68b364a 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -33,6 +33,7 @@ pub(crate) mod fwsec; pub(crate) mod gsp; pub(crate) mod riscv; +pub(crate) mod tlv; pub(crate) const FIRMWARE_VERSION: &str = "570.144"; diff --git a/drivers/gpu/nova-core/firmware/tlv.rs b/drivers/gpu/nova-core/firmware/tlv.rs new file mode 100644 index 000000000000..02150459c279 --- /dev/null +++ b/drivers/gpu/nova-core/firmware/tlv.rs @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use kernel::{ + device, + firmware, + prelude::*, + str::CString, // +}; + +use crate::{ + gpu, + num::*, // +}; + +/// Requests the GPU firmware TLV `name` suitable for `chipset`. +#[expect(dead_code)] +pub(crate) fn request_tlv( + dev: &device::Device, + chipset: gpu::Chipset, + name: &str, +) -> Result { + let chip_name = chipset.name(); + + let filename = CString::try_from_fmt(fmt!("nvidia/{chip_name}/gsp/{name}.tlv"))?; + + dev_dbg!(dev, "loading firmware image {:?}\n", &filename); + + firmware::Firmware::request(&filename, dev) +} + +struct TlvBlock<'a> { + tag: [u8; 4], + value: &'a [u8], +} + +/// On-wire TLV block header: 4-byte ASCII tag + little-endian payload length (bytes, excluding +/// padding to a 4-byte boundary). +struct TlvBlockHeader { + tag: [u8; 4], + length: usize, +} + +impl TlvBlockHeader { + const SIZE: usize = size_of::<[u8; 4]>() + size_of::(); + + /// Parses the first [`Self::SIZE`] bytes of `hdr` (caller may pass a longer slice). + fn parse(hdr: &[u8]) -> Option { + let hdr = hdr.get(..Self::SIZE)?; + let tag = <[u8; 4]>::try_from(hdr.get(..4)?).ok()?; + if !tag.is_ascii() { + return None; + } + let len_arr = <[u8; 4]>::try_from(hdr.get(4..Self::SIZE)?).ok()?; + let length = u32_as_usize(u32::from_le_bytes(len_arr)); + Some(Self { tag, length }) + } +} + +/// Iterator over the [`TlvBlock`]s of a [`Tlv`]. +/// +/// # Invariants +/// +/// `pos` is a byte offset into `tlv.data` that always lies on a block boundary (in the sense +/// of the [`Tlv`] invariant): it is either the start of a well-formed block, or equal to +/// `tlv.data.len()` (end of iteration). +struct TlvIter<'tlv, 'a> { + tlv: &'tlv Tlv<'a>, + pos: usize, +} + +impl<'tlv, 'a> Iterator for TlvIter<'tlv, 'a> { + type Item = TlvBlock<'a>; + + /// Returns the block starting at `self.pos` and advances the cursor past it, or [`None`] + /// once the cursor reaches the end of the data or encounters an error. + /// + /// Note that errors cannot actually occur because the data is validated in the constructor. + fn next(&mut self) -> Option { + if self.pos >= self.tlv.data.len() { + return None; + } + + let tail = self.tlv.data.get(self.pos..)?; + + let hdr = tail.get(..TlvBlockHeader::SIZE)?; + let header = TlvBlockHeader::parse(hdr)?; + + let stored_size = header.length.checked_next_multiple_of(4)?; + let advance = TlvBlockHeader::SIZE.checked_add(stored_size)?; + let payload_end = TlvBlockHeader::SIZE.checked_add(header.length)?; + + let value = tail + .get(..advance)? + .get(TlvBlockHeader::SIZE..payload_end)?; + + // INVARIANT: by the `Tlv` invariant the block at `self.pos` occupies exactly `advance` + // bytes, so `self.pos + advance` is the next block boundary (or `data.len()`). + self.pos = self.pos.checked_add(advance)?; + + Some(TlvBlock { + tag: header.tag, + value, + }) + } +} + +/// The post-header part of a validated TLV (type, length, value) firmware image. +/// +/// TLV firmware images start with a 4-byte "NVFW" magic header, followed by a sequence of +/// blocks. Each block has a 4-byte type tag, a 4-byte length field, and a data payload +/// (value) whose stored size is the length rounded up to the nearest multiple of 4. +/// +/// [`Self::new`] checks the magic header and walks every block: tags must be ASCII, +/// lengths and padding must fit without overflow, and the byte stream after `NVFW` must +/// be exactly partitionable into blocks (no trailing partial header or slack). After +/// that, [`TlvIter`] only signals end-of-stream via [`None`], not parse failure. +/// +/// Although the spec forbids duplicate tags, neither the constructor nor the iterator +/// enforces this restriction. Instead, duplicate tags are simply ignored. +/// +/// # Invariants +/// +/// `data` is a validated TLV payload (the bytes *after* the `NVFW` magic): it is the exact +/// concatenation of zero or more well-formed blocks, with no trailing partial header or slack. +/// Consequently, any offset `o` into `data` that is a block boundary and satisfies +/// `o < data.len()` is the start of a complete block whose header parses and whose stored +/// extent (`TlvBlockHeader::SIZE + header.length.next_multiple_of(4)` bytes) lies within +/// `data`. `data.len()` is itself a boundary. +pub(crate) struct Tlv<'a> { + data: &'a [u8], +} + +#[expect(dead_code)] +impl<'a> Tlv<'a> { + const MAGIC: &'static [u8; 4] = b"NVFW"; + + /// Parses `data` as a TLV firmware image, returning [`EINVAL`] if the image is malformed. + pub(crate) fn new(data: &'a [u8]) -> Result { + // Verify that the magic bytes exist and are the correct value + let magic_len = Self::MAGIC.len(); + if data + .get(..magic_len) + .is_none_or(|magic| magic != Self::MAGIC) + { + return Err(EINVAL); + } + + // The payload is the contiguous sequence of TLV blocks after the magic. + let payload = data.get(magic_len..).ok_or(EINVAL)?; + + // The spec says every TLV must have a VERS tag. + let mut has_vers = false; + + let mut rest = payload; + while !rest.is_empty() { + // Validate and extract the header (type, length). + let Some(header): Option = rest + .get(..TlvBlockHeader::SIZE) + .and_then(TlvBlockHeader::parse) + else { + return Err(EINVAL); + }; + + has_vers |= header.tag == *b"VERS"; + + // The `length` field of a TLV block contains the actual byte length of the + // value, but each TLV block is aligned to a 4-byte boundary. + let Some(stored_size) = header.length.checked_next_multiple_of(4) else { + return Err(EINVAL); + }; + + let length = TlvBlockHeader::SIZE + .checked_add(stored_size) + .ok_or(EINVAL)?; + + rest = rest.split_at_checked(length).ok_or(EINVAL)?.1; + } + + if !has_vers { + return Err(EINVAL); + } + + // INVARIANT: the loop above walked `payload` block-by-block. For each block, the + // header is parsed (`TlvBlockHeader::parse` rejects non-ASCII tags), and the + // stored extent (`SIZE + length.next_multiple_of(4)`) is computed without + // overflow and split off `rest` only when it fits. The loop ends only when `rest` + // is empty, so the byte stream is an exact concatenation of blocks with no + // trailing partial header or slack. + Ok(Self { data: payload }) + } + + fn iter(&self) -> TlvIter<'_, 'a> { + // INVARIANT: 0 is a block boundary, either the start of the first block, + // or `data.len()` when `data` is empty. + TlvIter { tlv: self, pos: 0 } + } + + fn find(&self, tag: &[u8; 4]) -> Result> { + self.iter().find(|b| b.tag == *tag).ok_or(EINVAL) + } + + /// Return a slice of bytes. + /// + /// Returns `EINVAL` if the value is empty. + pub(crate) fn get_bytes(&self, tag: &[u8; 4]) -> Result<&'a [u8]> { + let tlv = self.find(tag)?; + + // Treat empty value as an error, to avoid trying to parse nothing. + if tlv.value.is_empty() { + return Err(EINVAL); // TODO: Use ENODATA once available. + } + + Ok(tlv.value) + } + + /// Return a little-endian u32. + pub(crate) fn get_u32(&self, tag: &[u8; 4]) -> Result { + let tlv = self.find(tag)?; + + tlv.value + .try_into() + .ok() + .map(u32::from_le_bytes) + .ok_or(EINVAL) + } + + /// Return a string value. + pub(crate) fn get_string(&self, tag: &[u8; 4]) -> Result<&'a str> { + let tlv = self.find(tag)?; + + let bytes = tlv.value; + + // Strings can only contain printable ASCII characters. + if bytes.iter().any(|&b| !(32..127).contains(&b)) { + return Err(EINVAL); + } + + core::str::from_utf8(bytes).map_err(|_| EINVAL) + } + + /// Obtain the nth signature from a SIGN tag. If `index` is None, + /// then return the last signature. + pub(crate) fn get_signature(&self, index: Option) -> Result<&'a [u8]> { + let num_sigs: usize = match self.get_u32(b"NSIG")? { + 0 => return Err(EINVAL), + n => n.into_safe_cast(), + }; + + let sig_bytes = self.get_bytes(b"SIGN")?; + + // Ensure that sig_bytes can be divided evenly into chunks. + if sig_bytes.len() % num_sigs != 0 { + return Err(EINVAL); + } + + // num_sigs cannot be 0, and sig_bytes cannot be empty, so this cannot panic. + let sig_size = sig_bytes.len() / num_sigs; + + let index = index.unwrap_or(num_sigs - 1); + + sig_bytes.chunks_exact(sig_size).nth(index).ok_or(EINVAL) + } +} From ad5157827522aedde98bd0323076cd7f6190fbd0 Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Fri, 31 Jul 2026 15:10:13 -0500 Subject: [PATCH 130/137] gpu: nova-core: transition booter to TLV images Switch the booter firmware loader from the legacy binary format to the TLV format. This change requires the new TLV versions of the r570.144 firmware images. The new TLV format has all of the metadata needed by Nova encoded as separate tags, eliminating the need to parse legacy firmware headers such as HsHeaderV2 and HsSignatureParams. All of the structs and code for parsing those headers is therefore deleted. Signed-off-by: Timur Tabi Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260731201017.2580713-5-ttabi@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/firmware.rs | 15 +- drivers/gpu/nova-core/firmware/booter.rs | 323 ++++------------------- drivers/gpu/nova-core/firmware/tlv.rs | 2 - drivers/gpu/nova-core/gsp/hal/tu102.rs | 13 +- 4 files changed, 59 insertions(+), 294 deletions(-) diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index 2075b68b364a..bd47ebbb013e 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -22,10 +22,7 @@ }, gpu, gsp::boot_firmware_files, - num::{ - FromSafeCast, - IntoSafeCast, // - }, + num::IntoSafeCast, // }; pub(crate) mod booter; @@ -389,16 +386,6 @@ fn new(fw: &'a firmware::Firmware) -> Result { .map(|hdr| Self { hdr, fw }) .ok_or(EINVAL) } - - /// Returns the data payload of the firmware, or `None` if the data range is out of bounds of - /// the firmware image. - fn data(&self) -> Option<&[u8]> { - let fw_start = usize::from_safe_cast(self.hdr.data_offset); - let fw_size = usize::from_safe_cast(self.hdr.data_size); - let fw_end = fw_start.checked_add(fw_size)?; - - self.fw.get(fw_start..fw_end) - } } pub(crate) struct ModInfoBuilder(firmware::ModInfoBuilder); diff --git a/drivers/gpu/nova-core/firmware/booter.rs b/drivers/gpu/nova-core/firmware/booter.rs index acb7f4d8a532..6e7d688deadd 100644 --- a/drivers/gpu/nova-core/firmware/booter.rs +++ b/drivers/gpu/nova-core/firmware/booter.rs @@ -10,8 +10,7 @@ use kernel::{ device, dma::Coherent, - prelude::*, - transmute::FromBytes, // + prelude::*, // }; use crate::{ @@ -24,224 +23,19 @@ FalconFirmware, // }, firmware::{ - BinFirmware, + tlv::{ + request_tlv, // + Tlv, + }, FirmwareObject, FirmwareSignature, Signed, Unsigned, // }, gpu::Chipset, - num::{ - FromSafeCast, - IntoSafeCast, // - }, + num::IntoSafeCast, }; -/// Local convenience function to return a copy of `S` by reinterpreting the bytes starting at -/// `offset` in `slice`. -fn frombytes_at(slice: &[u8], offset: usize) -> Result { - let end = offset.checked_add(size_of::()).ok_or(EINVAL)?; - slice - .get(offset..end) - .and_then(S::from_bytes_copy) - .ok_or(EINVAL) -} - -/// Heavy-Secured firmware header. -/// -/// Such firmwares have an application-specific payload that needs to be patched with a given -/// signature. -#[repr(C)] -#[derive(Debug, Clone)] -struct HsHeaderV2 { - /// Offset to the start of the signatures. - sig_prod_offset: u32, - /// Size in bytes of the signatures. - sig_prod_size: u32, - /// Offset to a `u32` containing the location at which to patch the signature in the microcode - /// image. - patch_loc_offset: u32, - /// Offset to a `u32` containing the index of the signature to patch. - patch_sig_offset: u32, - /// Start offset to the signature metadata. - meta_data_offset: u32, - /// Size in bytes of the signature metadata. - meta_data_size: u32, - /// Offset to a `u32` containing the number of signatures in the signatures section. - num_sig_offset: u32, - /// Offset of the application-specific header. - header_offset: u32, - /// Size in bytes of the application-specific header. - header_size: u32, -} - -// SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. -unsafe impl FromBytes for HsHeaderV2 {} - -/// Heavy-Secured Firmware image container. -/// -/// This provides convenient access to the fields of [`HsHeaderV2`] that are actually indices to -/// read from in the firmware data. -struct HsFirmwareV2<'a> { - hdr: HsHeaderV2, - fw: &'a [u8], -} - -impl<'a> HsFirmwareV2<'a> { - /// Interprets the header of `bin_fw` as a [`HsHeaderV2`] and returns an instance of - /// `HsFirmwareV2` for further parsing. - /// - /// Fails if the header pointed at by `bin_fw` is not within the bounds of the firmware image. - fn new(bin_fw: &BinFirmware<'a>) -> Result { - frombytes_at::(bin_fw.fw, bin_fw.hdr.header_offset.into_safe_cast()) - .map(|hdr| Self { hdr, fw: bin_fw.fw }) - } - - /// Returns the location at which the signatures should be patched in the microcode image. - /// - /// Fails if the offset of the patch location is outside the bounds of the firmware - /// image. - fn patch_location(&self) -> Result { - frombytes_at::(self.fw, self.hdr.patch_loc_offset.into_safe_cast()) - } - - /// Returns an iterator to the signatures of the firmware. The iterator can be empty if the - /// firmware is unsigned. - /// - /// Fails if the pointed signatures are outside the bounds of the firmware image. - fn signatures_iter(&'a self) -> Result>> { - let num_sig = frombytes_at::(self.fw, self.hdr.num_sig_offset.into_safe_cast())?; - let iter = match self.hdr.sig_prod_size.checked_div(num_sig) { - // If there are no signatures, return an iterator that will yield zero elements. - None => (&[] as &[u8]).chunks_exact(1), - Some(sig_size) => { - let patch_sig = - frombytes_at::(self.fw, self.hdr.patch_sig_offset.into_safe_cast())?; - - let signatures_start = self - .hdr - .sig_prod_offset - .checked_add(patch_sig) - .map(usize::from_safe_cast) - .ok_or(EINVAL)?; - - let signatures_end = signatures_start - .checked_add(usize::from_safe_cast(self.hdr.sig_prod_size)) - .ok_or(EINVAL)?; - - self.fw - // Get signatures range. - .get(signatures_start..signatures_end) - .ok_or(EINVAL)? - .chunks_exact(sig_size.into_safe_cast()) - } - }; - - // Map the byte slices into signatures. - Ok(iter.map(BooterSignature)) - } -} - -/// Signature parameters, as defined in the firmware. -#[repr(C)] -struct HsSignatureParams { - /// Fuse version to use. - fuse_ver: u32, - /// Mask of engine IDs this firmware applies to. - engine_id_mask: u32, - /// ID of the microcode. - ucode_id: u32, -} - -// SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. -unsafe impl FromBytes for HsSignatureParams {} - -impl HsSignatureParams { - /// Returns the signature parameters contained in `hs_fw`. - /// - /// Fails if the meta data parameter of `hs_fw` is outside the bounds of the firmware image, or - /// if its size doesn't match that of [`HsSignatureParams`]. - fn new(hs_fw: &HsFirmwareV2<'_>) -> Result { - let start = usize::from_safe_cast(hs_fw.hdr.meta_data_offset); - let end = start - .checked_add(hs_fw.hdr.meta_data_size.into_safe_cast()) - .ok_or(EINVAL)?; - - hs_fw - .fw - .get(start..end) - .and_then(Self::from_bytes_copy) - .ok_or(EINVAL) - } -} - -/// Header for code and data load offsets. -#[repr(C)] -#[derive(Debug, Clone)] -struct HsLoadHeaderV2 { - // Offset at which the code starts. - os_code_offset: u32, - // Total size of the code, for all apps. - os_code_size: u32, - // Offset at which the data starts. - os_data_offset: u32, - // Size of the data. - os_data_size: u32, - // Number of apps following this header. Each app is described by a [`HsLoadHeaderV2App`]. - num_apps: u32, -} - -// SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. -unsafe impl FromBytes for HsLoadHeaderV2 {} - -impl HsLoadHeaderV2 { - /// Returns the load header contained in `hs_fw`. - /// - /// Fails if the header pointed at by `hs_fw` is not within the bounds of the firmware image. - fn new(hs_fw: &HsFirmwareV2<'_>) -> Result { - frombytes_at::(hs_fw.fw, hs_fw.hdr.header_offset.into_safe_cast()) - } -} - -/// Header for app code loader. -#[repr(C)] -#[derive(Debug, Clone)] -struct HsLoadHeaderV2App { - /// Offset at which to load the app code. - offset: u32, - /// Length in bytes of the app code. - len: u32, -} - -// SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. -unsafe impl FromBytes for HsLoadHeaderV2App {} - -impl HsLoadHeaderV2App { - /// Returns the [`HsLoadHeaderV2App`] for app `idx` of `hs_fw`. - /// - /// Fails if `idx` is larger than the number of apps declared in `hs_fw`, or if the header is - /// not within the bounds of the firmware image. - fn new(hs_fw: &HsFirmwareV2<'_>, idx: u32) -> Result { - let load_hdr = HsLoadHeaderV2::new(hs_fw)?; - if idx >= load_hdr.num_apps { - Err(EINVAL) - } else { - frombytes_at::( - hs_fw.fw, - usize::from_safe_cast(hs_fw.hdr.header_offset) - // Skip the load header... - .checked_add(size_of::()) - // ... and jump to app header `idx`. - .and_then(|offset| { - offset - .checked_add(usize::from_safe_cast(idx).checked_mul(size_of::())?) - }) - .ok_or(EINVAL)?, - ) - } - } -} - /// Signature for Booter firmware. Their size is encoded into the header and not known a compile /// time, so we just wrap a byte slices on which we can implement [`FirmwareSignature`]. struct BooterSignature<'a>(&'a [u8]); @@ -291,85 +85,76 @@ pub(crate) fn new( dev: &device::Device, kind: BooterKind, chipset: Chipset, - ver: &str, falcon: &Falcon<'_, ::Target>, ) -> Result { let fw_name = match kind { BooterKind::Loader => "booter_load", BooterKind::Unloader => "booter_unload", }; - let fw = super::request_firmware(dev, chipset, fw_name, ver)?; - let bin_fw = BinFirmware::new(&fw)?; + let fw = request_tlv(dev, chipset, fw_name)?; + let tlv = Tlv::new(fw.data())?; + dev_dbg!( + dev, + "loaded {} firmware v{}\n", + fw_name, + tlv.get_string(b"VERS")? + ); - // The binary firmware embeds a Heavy-Secured firmware. - let hs_fw = HsFirmwareV2::new(&bin_fw)?; + let os_data_offset = tlv.get_u32(b"DAOF")?; + let os_data_size = tlv.get_u32(b"DASZ")?; + let os_code_offset = tlv.get_u32(b"CDOF")?; + let os_code_size = tlv.get_u32(b"CDSZ")?; + let patch_loc = tlv.get_u32(b"PLOC")?; + let fuse_version: usize = tlv.get_u32(b"FUSE")?.into_safe_cast(); + let engine_id = tlv.get_u32(b"ENID")?; + let ucode_id = tlv.get_u32(b"UCID")?; + let app0_code_offset = tlv.get_u32(b"A0CO")?; + let app0_code_size = tlv.get_u32(b"A0CS")?; - // The Heavy-Secured firmware embeds a firmware load descriptor. - let load_hdr = HsLoadHeaderV2::new(&hs_fw)?; - - // Offset in `ucode` where to patch the signature. - let patch_loc = hs_fw.patch_location()?; - - let sig_params = HsSignatureParams::new(&hs_fw)?; let brom_params = FalconBromParams { - // `load_hdr.os_data_offset` is an absolute index, but `pkc_data_offset` is from the + // `os_data_offset` is an absolute index, but `pkc_data_offset` is from the // signature patch location. - pkc_data_offset: patch_loc - .checked_sub(load_hdr.os_data_offset) - .ok_or(EINVAL)?, - engine_id_mask: u16::try_from(sig_params.engine_id_mask).map_err(|_| EINVAL)?, - ucode_id: u8::try_from(sig_params.ucode_id).map_err(|_| EINVAL)?, + pkc_data_offset: patch_loc.checked_sub(os_data_offset).ok_or(EINVAL)?, + engine_id_mask: u16::try_from(engine_id).map_err(|_| EINVAL)?, + ucode_id: u8::try_from(ucode_id).map_err(|_| EINVAL)?, }; - let app0 = HsLoadHeaderV2App::new(&hs_fw, 0)?; - // Object containing the firmware microcode to be signature-patched. - let ucode = bin_fw - .data() - .ok_or(EINVAL) + let ucode = tlv + .get_bytes(b"BLOB") .and_then(FirmwareObject::::new_booter)?; - let ucode_signed = { - let mut signatures = hs_fw.signatures_iter()?.peekable(); + // Obtain the version from the fuse register, and extract the corresponding + // signature. + let reg_fuse_version: usize = falcon + .signature_reg_fuse_version(brom_params.engine_id_mask, brom_params.ucode_id)? + .into_safe_cast(); - if signatures.peek().is_none() { - // If there are no signatures, then the firmware is unsigned. - ucode.no_patch_signature() - } else { - // Obtain the version from the fuse register, and extract the corresponding - // signature. - let reg_fuse_version = falcon - .signature_reg_fuse_version(brom_params.engine_id_mask, brom_params.ucode_id)?; + const FUSE_VERSION_USE_LAST_SIG: usize = 0; - // `0` means the last signature should be used. - const FUSE_VERSION_USE_LAST_SIG: u32 = 0; - let signature = match reg_fuse_version { - FUSE_VERSION_USE_LAST_SIG => signatures.last(), - // Otherwise hardware fuse version needs to be subtracted to obtain the index. - reg_fuse_version => { - let Some(idx) = sig_params.fuse_ver.checked_sub(reg_fuse_version) else { - dev_err!(dev, "invalid fuse version for Booter firmware\n"); - return Err(EINVAL); - }; - signatures.nth(idx.into_safe_cast()) - } - } - .ok_or(EINVAL)?; - - ucode.patch_signature(&signature, patch_loc.into_safe_cast())? - } + let index = match reg_fuse_version { + // `0` means the last signature should be used. + FUSE_VERSION_USE_LAST_SIG => None, + // Otherwise, hardware fuse version needs to be subtracted to obtain the index. + _ => Some(fuse_version.checked_sub(reg_fuse_version).ok_or(EINVAL)?), }; + // Extract the nth signature. Booter is always signed. + let sig_chunk = tlv.get_signature(index)?; + + let signature = BooterSignature(sig_chunk); + let ucode_signed = ucode.patch_signature(&signature, patch_loc.into_safe_cast())?; + // There are two versions of Booter, one for Turing/GA100, and another for // GA102+. The extraction of the IMEM sections differs between the two // versions. Unfortunately, the file names are the same, and the headers // don't indicate the versions. The only way to differentiate is by the Chipset. let (imem_sec_dst_start, imem_ns_load_target) = if chipset <= Chipset::GA100 { ( - app0.offset, + app0_code_offset, Some(FalconDmaLoadTarget { src_start: 0, - dst_start: load_hdr.os_code_offset, - len: load_hdr.os_code_size, + dst_start: os_code_offset, + len: os_code_size, }), ) } else { @@ -378,15 +163,15 @@ pub(crate) fn new( Ok(Self { imem_sec_load_target: FalconDmaLoadTarget { - src_start: app0.offset, + src_start: app0_code_offset, dst_start: imem_sec_dst_start, - len: app0.len, + len: app0_code_size, }, imem_ns_load_target, dmem_load_target: FalconDmaLoadTarget { - src_start: load_hdr.os_data_offset, + src_start: os_data_offset, dst_start: 0, - len: load_hdr.os_data_size, + len: os_data_size, }, brom_params, ucode: ucode_signed, diff --git a/drivers/gpu/nova-core/firmware/tlv.rs b/drivers/gpu/nova-core/firmware/tlv.rs index 02150459c279..7b879f13a61e 100644 --- a/drivers/gpu/nova-core/firmware/tlv.rs +++ b/drivers/gpu/nova-core/firmware/tlv.rs @@ -14,7 +14,6 @@ }; /// Requests the GPU firmware TLV `name` suitable for `chipset`. -#[expect(dead_code)] pub(crate) fn request_tlv( dev: &device::Device, chipset: gpu::Chipset, @@ -131,7 +130,6 @@ pub(crate) struct Tlv<'a> { data: &'a [u8], } -#[expect(dead_code)] impl<'a> Tlv<'a> { const MAGIC: &'static [u8; 4] = b"NVFW"; diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index e3c365cf4a68..2ae3140cb0fe 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -31,8 +31,7 @@ FwsecCommand, FwsecFirmware, // }, - gsp::GspFirmware, - FIRMWARE_VERSION, // + gsp::GspFirmware, // }, gpu::Chipset, gsp::{ @@ -237,7 +236,6 @@ fn build_unload_bundle( dev, BooterKind::Unloader, chipset, - FIRMWARE_VERSION, sec2_falcon, )?, }, @@ -306,14 +304,11 @@ fn boot( "Using SEC2 to load and run the booter_load firmware...\n" ); - BooterFirmware::new( + BooterFirmware::new(dev, BooterKind::Loader, chipset, sec2_falcon)?.run( dev, - BooterKind::Loader, - chipset, - FIRMWARE_VERSION, sec2_falcon, - )? - .run(dev, sec2_falcon, &wpr_meta)?; + &wpr_meta, + )?; Ok(unload_guard.dismiss()) } From ccdcefb71acd3664a0e38ebdb5fa7b56a1a546a4 Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Fri, 31 Jul 2026 15:10:14 -0500 Subject: [PATCH 131/137] gpu: nova-core: transition gsp to TLV images Switch the GSP firmware loader from the legacy binary format to the TLV format. This change requires the new TLV versions of the r570.144 firmware images. Unlike the other TLV firmware images, gsp.tlv contains a pointer to the actual GSP firmware file instead of its contents. This allows each small gsp.tlv file to contain the distinct metadata for each GPU while still allowing the very large gsp.bin to be shared by all GPUs. One key piece of metadata is the signature. The legacy GSP firmware image is an ELF file that contains multiple sections that needed to be parsed, and the driver needed to determine which section is relevant for the GPU. Instead, gsp.tlv contains the pre-processed metadata, so all the driver needs to do is to extract it. Signed-off-by: Timur Tabi Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260731201017.2580713-6-ttabi@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/firmware.rs | 23 -------- drivers/gpu/nova-core/firmware/gsp.rs | 59 +++++++------------ drivers/gpu/nova-core/firmware/riscv.rs | 75 ++++++------------------- drivers/gpu/nova-core/gsp/boot.rs | 7 +-- 4 files changed, 38 insertions(+), 126 deletions(-) diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index bd47ebbb013e..3aa0137cba53 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -365,29 +365,6 @@ struct BinHdr { // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. unsafe impl FromBytes for BinHdr {} -// A firmware blob starting with a `BinHdr`. -struct BinFirmware<'a> { - hdr: BinHdr, - fw: &'a [u8], -} - -impl<'a> BinFirmware<'a> { - /// Interpret `fw` as a firmware image starting with a [`BinHdr`], and returns the - /// corresponding [`BinFirmware`] that can be used to extract its payload. - fn new(fw: &'a firmware::Firmware) -> Result { - const BIN_MAGIC: u32 = 0x10de; - let fw = fw.data(); - - fw.get(0..size_of::()) - // Extract header. - .and_then(BinHdr::from_bytes_copy) - // Validate header. - .filter(|hdr| hdr.bin_magic == BIN_MAGIC) - .map(|hdr| Self { hdr, fw }) - .ok_or(EINVAL) - } -} - pub(crate) struct ModInfoBuilder(firmware::ModInfoBuilder); impl ModInfoBuilder { diff --git a/drivers/gpu/nova-core/firmware/gsp.rs b/drivers/gpu/nova-core/firmware/gsp.rs index 99a302bae567..55a8c513e3a9 100644 --- a/drivers/gpu/nova-core/firmware/gsp.rs +++ b/drivers/gpu/nova-core/firmware/gsp.rs @@ -8,22 +8,24 @@ DataDirection, DmaAddress, // }, + firmware, prelude::*, scatterlist::{ Owned, SGTable, // }, + str::CString, }; use crate::{ firmware::{ - elf, riscv::RiscvFirmware, // + tlv::{ + request_tlv, // + Tlv, + }, }, - gpu::{ - Architecture, - Chipset, // - }, + gpu::Chipset, gsp::GSP_PAGE_SIZE, num::FromSafeCast, }; @@ -63,43 +65,26 @@ pub(crate) struct GspFirmware { } impl GspFirmware { - fn find_gsp_sigs_section(chipset: Chipset) -> &'static str { - match chipset.arch() { - Architecture::Turing if matches!(chipset, Chipset::TU116 | Chipset::TU117) => { - ".fwsignature_tu11x" - } - Architecture::Turing => ".fwsignature_tu10x", - Architecture::Ampere if chipset == Chipset::GA100 => ".fwsignature_ga100", - Architecture::Ampere => ".fwsignature_ga10x", - Architecture::Ada => ".fwsignature_ad10x", - Architecture::Hopper => ".fwsignature_gh10x", - Architecture::BlackwellGB10x => ".fwsignature_gb10x", - Architecture::BlackwellGB20x => ".fwsignature_gb20x", - } - } - /// Loads the GSP firmware binaries, map them into `dev`'s address-space, and creates the page /// tables expected by the GSP bootloader to load it. pub(crate) fn new<'a>( dev: &'a device::Device, chipset: Chipset, - ver: &'a str, ) -> impl PinInit + 'a { pin_init::pin_init_scope(move || { - let firmware = super::request_firmware(dev, chipset, "gsp", ver)?; + let firmware = request_tlv(dev, chipset, "gsp")?; + let tlv = Tlv::new(firmware.data())?; + dev_dbg!(dev, "loaded gsp firmware v{}\n", tlv.get_string(b"VERS")?); - let fw_section = elf::elf_section(firmware.data(), ".fwimage").ok_or(EINVAL)?; + let size = usize::from_safe_cast(tlv.get_u32(b"SIZE")?); + let mut fw_vvec = VVec::zeroed(size, GFP_KERNEL).map_err(|_| ENOMEM)?; - let size = fw_section.len(); + let chip_name = chipset.name(); + let file = tlv.get_string(b"FILE")?; + let filename = CString::try_from_fmt(fmt!("nvidia/{chip_name}/gsp/{file}"))?; + firmware::request_into_buf(&filename, dev, fw_vvec.as_mut_slice())?; - // Move the firmware into a vmalloc'd vector and map it into the device address - // space. - let fw_vvec = VVec::with_capacity(fw_section.len(), GFP_KERNEL) - .and_then(|mut v| { - v.extend_from_slice(fw_section, GFP_KERNEL)?; - Ok(v) - }) - .map_err(|_| ENOMEM)?; + let signatures = Coherent::from_slice(dev, tlv.get_bytes(b"SIGN")?, GFP_KERNEL)?; Ok(try_pin_init!(Self { fw <- SGTable::new(dev, fw_vvec, DataDirection::ToDevice, GFP_KERNEL), @@ -145,15 +130,9 @@ pub(crate) fn new<'a>( level0.into() }, size, - signatures: { - let sigs_section = Self::find_gsp_sigs_section(chipset); - - elf::elf_section(firmware.data(), sigs_section) - .ok_or(EINVAL) - .and_then(|data| Coherent::from_slice(dev, data, GFP_KERNEL))? - }, + signatures, bootloader: { - let bl = super::request_firmware(dev, chipset, "bootloader", ver)?; + let bl = request_tlv(dev, chipset, "gsp_bootloader")?; RiscvFirmware::new(dev, &bl)? }, diff --git a/drivers/gpu/nova-core/firmware/riscv.rs b/drivers/gpu/nova-core/firmware/riscv.rs index 2afa7f36404e..1403f05a7305 100644 --- a/drivers/gpu/nova-core/firmware/riscv.rs +++ b/drivers/gpu/nova-core/firmware/riscv.rs @@ -7,53 +7,10 @@ device, dma::Coherent, firmware::Firmware, - prelude::*, - transmute::FromBytes, // + prelude::*, // }; -use crate::{ - firmware::BinFirmware, - num::FromSafeCast, // -}; - -/// Descriptor for microcode running on a RISC-V core. -#[repr(C)] -#[derive(Debug)] -struct RmRiscvUCodeDesc { - version: u32, - bootloader_offset: u32, - bootloader_size: u32, - bootloader_param_offset: u32, - bootloader_param_size: u32, - riscv_elf_offset: u32, - riscv_elf_size: u32, - app_version: u32, - manifest_offset: u32, - manifest_size: u32, - monitor_data_offset: u32, - monitor_data_size: u32, - monitor_code_offset: u32, - monitor_code_size: u32, -} - -// SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. -unsafe impl FromBytes for RmRiscvUCodeDesc {} - -impl RmRiscvUCodeDesc { - /// Interprets the header of `bin_fw` as a [`RmRiscvUCodeDesc`] and returns it. - /// - /// Fails if the header pointed at by `bin_fw` is not within the bounds of the firmware image. - fn new(bin_fw: &BinFirmware<'_>) -> Result { - let offset = usize::from_safe_cast(bin_fw.hdr.header_offset); - let end = offset.checked_add(size_of::()).ok_or(EINVAL)?; - - bin_fw - .fw - .get(offset..end) - .and_then(Self::from_bytes_copy) - .ok_or(EINVAL) - } -} +use crate::firmware::tlv::Tlv; /// A parsed firmware for a RISC-V core, ready to be loaded and run. pub(crate) struct RiscvFirmware { @@ -72,24 +29,26 @@ pub(crate) struct RiscvFirmware { impl RiscvFirmware { /// Parses the RISC-V firmware image contained in `fw`. pub(crate) fn new(dev: &device::Device, fw: &Firmware) -> Result { - let bin_fw = BinFirmware::new(fw)?; + let tlv = Tlv::new(fw.data())?; + dev_dbg!( + dev, + "loaded gsp bootloader firmware v{}\n", + tlv.get_string(b"VERS")? + ); - let riscv_desc = RmRiscvUCodeDesc::new(&bin_fw)?; + let code_offset = tlv.get_u32(b"CDOF")?; + let data_offset = tlv.get_u32(b"DAOF")?; + let manifest_offset = tlv.get_u32(b"MFOF")?; + let app_version = tlv.get_u32(b"APPV")?; - let ucode = { - let start = usize::from_safe_cast(bin_fw.hdr.data_offset); - let len = usize::from_safe_cast(bin_fw.hdr.data_size); - let end = start.checked_add(len).ok_or(EINVAL)?; - - Coherent::from_slice(dev, fw.data().get(start..end).ok_or(EINVAL)?, GFP_KERNEL)? - }; + let ucode = Coherent::from_slice(dev, tlv.get_bytes(b"BLOB")?, GFP_KERNEL)?; Ok(Self { ucode, - code_offset: riscv_desc.monitor_code_offset, - data_offset: riscv_desc.monitor_data_offset, - manifest_offset: riscv_desc.manifest_offset, - app_version: riscv_desc.app_version, + code_offset, + data_offset, + manifest_offset, + app_version, }) } } diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 97f6e7ef4ead..e03700ee7bea 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -15,10 +15,7 @@ gsp::Gsp, Falcon, // }, - firmware::{ - gsp::GspFirmware, - FIRMWARE_VERSION, // - }, + firmware::gsp::GspFirmware, gsp::{ cmdq::Cmdq, commands, // @@ -45,7 +42,7 @@ pub(crate) fn boot( let dev = pdev.as_ref(); let hal = super::hal::gsp_hal(chipset); - let gsp_fw = KBox::pin_init(GspFirmware::new(dev, chipset, FIRMWARE_VERSION), GFP_KERNEL)?; + let gsp_fw = KBox::pin_init(GspFirmware::new(dev, chipset), GFP_KERNEL)?; // Perform the chipset-specific boot sequence, and retrieve the unload bundle. let unload_bundle = hal.boot(&self, &mut ctx, &gsp_fw)?.or_else(|| { From ebc053cf0b4c8ed6bff9a0de6b25f819473ba83a Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Fri, 31 Jul 2026 15:10:15 -0500 Subject: [PATCH 132/137] gpu: nova-core: transition gen_bootloader to TLV images Switch the generic bootloader firmware loader from the legacy binary format to the TLV format. This change requires the new TLV versions of the r570.144 firmware images. Signed-off-by: Timur Tabi Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260731201017.2580713-7-ttabi@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/firmware.rs | 24 +----- .../nova-core/firmware/fwsec/bootloader.rs | 75 +++++-------------- 2 files changed, 20 insertions(+), 79 deletions(-) diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index 3aa0137cba53..c285e57268f0 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -11,8 +11,7 @@ device, firmware, prelude::*, - str::CString, - transmute::FromBytes, // + str::CString, // }; use crate::{ @@ -344,27 +343,6 @@ fn no_patch_signature(self) -> FirmwareObject { } } -/// Header common to most firmware files. -#[repr(C)] -#[derive(Debug, Clone)] -struct BinHdr { - /// Magic number, must be `0x10de`. - bin_magic: u32, - /// Version of the header. - bin_ver: u32, - /// Size in bytes of the binary (to be ignored). - bin_size: u32, - /// Offset of the start of the application-specific header. - header_offset: u32, - /// Offset of the start of the data payload. - data_offset: u32, - /// Size in bytes of the data payload. - data_size: u32, -} - -// SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. -unsafe impl FromBytes for BinHdr {} - pub(crate) struct ModInfoBuilder(firmware::ModInfoBuilder); impl ModInfoBuilder { diff --git a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs index d9fafd2eea5b..30d247b59848 100644 --- a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs +++ b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs @@ -19,10 +19,7 @@ Alignment, // }, sizes, - transmute::{ - AsBytes, - FromBytes, // - }, + transmute::AsBytes, }; use crate::{ @@ -42,38 +39,16 @@ }, firmware::{ fwsec::FwsecFirmware, - request_firmware, - BinHdr, - FIRMWARE_VERSION, // + tlv::{ + request_tlv, // + Tlv, + }, }, gpu::Chipset, num::FromSafeCast, // regs, }; -/// Descriptor used by RM to figure out the requirements of the boot loader. -/// -/// Most of its fields appear to be legacy and carry incorrect values, so they are left unused. -#[repr(C)] -#[derive(Debug, Clone)] -struct BootloaderDesc { - /// Starting tag of bootloader. - start_tag: u32, - /// DMEM load offset - unused here as we always load at offset `0`. - _dmem_load_off: u32, - /// Offset of code section in the image. Unused as there is only one section in the bootloader - /// binary. - _code_off: u32, - /// Size of code section in the image. - code_size: u32, - /// Offset of data section in the image. Unused as we build the data section ourselves. - _data_off: u32, - /// Size of data section in the image. Unused as we build the data section ourselves. - _data_size: u32, -} -// SAFETY: any byte sequence is valid for this struct. -unsafe impl FromBytes for BootloaderDesc {} - /// Structure used by the boot-loader to load the rest of the code. /// /// This has to be filled by the GPU driver and copied into DMEM at offset @@ -146,38 +121,24 @@ pub(crate) fn new( dev: &Device, chipset: Chipset, ) -> Result { - let fw = request_firmware(dev, chipset, "gen_bootloader", FIRMWARE_VERSION)?; - let hdr = fw - .data() - .get(0..size_of::()) - .and_then(BinHdr::from_bytes_copy) - .ok_or(EINVAL)?; - - let desc = { - let desc_offset = usize::from_safe_cast(hdr.header_offset); - - fw.data() - .get(desc_offset..) - .and_then(BootloaderDesc::from_bytes_copy_prefix) - .ok_or(EINVAL)? - .0 - }; + let fw = request_tlv(dev, chipset, "gen_bootloader")?; + let tlv = Tlv::new(fw.data())?; + dev_dbg!( + dev, + "loaded generic bootloader firmware v{}\n", + tlv.get_string(b"VERS")? + ); let ucode = { - let ucode_start = usize::from_safe_cast(hdr.data_offset); - let code_size = usize::from_safe_cast(desc.code_size); - // Align to falcon block size (256 bytes). + let blob = tlv.get_bytes(b"BLOB")?; + let code_size = usize::from_safe_cast(tlv.get_u32(b"CDSZ")?); + let code = blob.get(..code_size).ok_or(EINVAL)?; let aligned_code_size = code_size .align_up(Alignment::new::<{ falcon::MEM_BLOCK_ALIGNMENT }>()) .ok_or(EINVAL)?; let mut ucode = KVec::with_capacity(aligned_code_size, GFP_KERNEL)?; - ucode.extend_from_slice( - fw.data() - .get(ucode_start..ucode_start + code_size) - .ok_or(EINVAL)?, - GFP_KERNEL, - )?; + ucode.extend_from_slice(code, GFP_KERNEL)?; ucode.resize(aligned_code_size, 0, GFP_KERNEL)?; ucode @@ -258,13 +219,15 @@ pub(crate) fn new( .checked_sub(ucode.len()) .ok_or(EOVERFLOW)?; + let start_tag = u16::try_from(tlv.get_u32(b"STRT")?)?; + Ok(Self { _firmware_dma: firmware_dma, ucode, dmem_desc, brom_params: firmware.brom_params(), imem_dst_start: u16::try_from(imem_dst_start)?, - start_tag: u16::try_from(desc.start_tag)?, + start_tag, }) } From 08994a0f2936532f1e61a881fc0cc1b731773cf7 Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Fri, 31 Jul 2026 15:10:16 -0500 Subject: [PATCH 133/137] gpu: nova-core: transition fsp to TLV images Switch the FSP firmware loaders from the legacy ELF32 format to the new TLV format. This change requires the new TLV versions of the r570.144 firmware images. Because we are no longer loading ELF images, we can also delete the ELF parser. Also remove function request_firmware() as this was the last user. Signed-off-by: Timur Tabi Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260731201017.2580713-8-ttabi@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/firmware.rs | 222 +------------------------- drivers/gpu/nova-core/firmware/fsp.rs | 88 +++++----- drivers/gpu/nova-core/fsp.rs | 11 +- 3 files changed, 46 insertions(+), 275 deletions(-) diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index c285e57268f0..d58a3d5770d5 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -8,10 +8,8 @@ use core::ops::Deref; use kernel::{ - device, firmware, - prelude::*, - str::CString, // + prelude::*, // }; use crate::{ @@ -33,19 +31,6 @@ pub(crate) const FIRMWARE_VERSION: &str = "570.144"; -/// Requests the GPU firmware `name` suitable for `chipset`, with version `ver`. -fn request_firmware( - dev: &device::Device, - chipset: gpu::Chipset, - name: &str, - ver: &str, -) -> Result { - let chip_name = chipset.name(); - - CString::try_from_fmt(fmt!("nvidia/{chip_name}/gsp/{name}-{ver}.bin")) - .and_then(|path| firmware::Firmware::request(&path, dev)) -} - /// Structure used to describe some firmwares, notably FWSEC-FRTS. #[repr(C)] #[derive(Debug, Clone, FromBytes)] @@ -393,208 +378,3 @@ pub(crate) const fn create( this.0 } } - -/// Ad-hoc and temporary module to extract sections from ELF images. -/// -/// Some firmware images are currently packaged as ELF files, where sections names are used as keys -/// to specific and related bits of data. Future firmware versions are scheduled to move away from -/// that scheme before nova-core becomes stable, which means this module will eventually be -/// removed. -mod elf { - use kernel::{ - bindings, - prelude::*, - transmute::FromBytes, // - }; - - /// Trait to abstract over ELF header differences. - trait ElfHeader: FromBytes { - fn shnum(&self) -> u16; - fn shoff(&self) -> u64; - fn shstrndx(&self) -> u16; - } - - /// Trait to abstract over ELF section-header differences. - trait ElfSectionHeader: FromBytes { - fn name(&self) -> u32; - fn offset(&self) -> u64; - fn size(&self) -> u64; - } - - /// Trait describing a matching ELF header and section-header format. - trait ElfFormat { - type Header: ElfHeader; - type SectionHeader: ElfSectionHeader; - } - - /// Newtype to provide a [`FromBytes`] implementation. - #[repr(transparent)] - struct Elf64Hdr(bindings::elf64_hdr); - // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. - unsafe impl FromBytes for Elf64Hdr {} - - impl ElfHeader for Elf64Hdr { - fn shnum(&self) -> u16 { - self.0.e_shnum - } - - fn shoff(&self) -> u64 { - self.0.e_shoff - } - - fn shstrndx(&self) -> u16 { - self.0.e_shstrndx - } - } - - #[repr(transparent)] - struct Elf64SHdr(bindings::elf64_shdr); - // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. - unsafe impl FromBytes for Elf64SHdr {} - - impl ElfSectionHeader for Elf64SHdr { - fn name(&self) -> u32 { - self.0.sh_name - } - - fn offset(&self) -> u64 { - self.0.sh_offset - } - - fn size(&self) -> u64 { - self.0.sh_size - } - } - - struct Elf64Format; - - impl ElfFormat for Elf64Format { - type Header = Elf64Hdr; - type SectionHeader = Elf64SHdr; - } - - /// Newtype to provide [`FromBytes`] and [`ElfHeader`] implementations for ELF32. - #[repr(transparent)] - struct Elf32Hdr(bindings::elf32_hdr); - // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. - unsafe impl FromBytes for Elf32Hdr {} - - impl ElfHeader for Elf32Hdr { - fn shnum(&self) -> u16 { - self.0.e_shnum - } - - fn shoff(&self) -> u64 { - u64::from(self.0.e_shoff) - } - - fn shstrndx(&self) -> u16 { - self.0.e_shstrndx - } - } - - /// Newtype to provide [`FromBytes`] and [`ElfSectionHeader`] implementations for ELF32. - #[repr(transparent)] - struct Elf32SHdr(bindings::elf32_shdr); - // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. - unsafe impl FromBytes for Elf32SHdr {} - - impl ElfSectionHeader for Elf32SHdr { - fn name(&self) -> u32 { - self.0.sh_name - } - - fn offset(&self) -> u64 { - u64::from(self.0.sh_offset) - } - - fn size(&self) -> u64 { - u64::from(self.0.sh_size) - } - } - - struct Elf32Format; - - impl ElfFormat for Elf32Format { - type Header = Elf32Hdr; - type SectionHeader = Elf32SHdr; - } - - /// Returns a NULL-terminated string from the ELF image at `offset`. - fn elf_str(elf: &[u8], offset: u64) -> Option<&str> { - let idx = usize::try_from(offset).ok()?; - let bytes = elf.get(idx..)?; - CStr::from_bytes_until_nul(bytes).ok()?.to_str().ok() - } - - fn elf_section_generic<'a, F>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> - where - F: ElfFormat, - { - let hdr = F::Header::from_bytes(elf.get(0..size_of::())?)?; - - let shdr_num = usize::from(hdr.shnum()); - let shdr_start = usize::try_from(hdr.shoff()).ok()?; - let shdr_end = shdr_num - .checked_mul(size_of::()) - .and_then(|v| v.checked_add(shdr_start))?; - - // Get all the section headers as an iterator over byte chunks. - let shdr_bytes = elf.get(shdr_start..shdr_end)?; - let mut shdr_iter = shdr_bytes.chunks_exact(size_of::()); - - // Get the strings table. - let strhdr = shdr_iter - .clone() - .nth(usize::from(hdr.shstrndx())) - .and_then(F::SectionHeader::from_bytes)?; - - // Find the section which name matches `name` and return it. - shdr_iter.find_map(|sh_bytes| { - let sh = F::SectionHeader::from_bytes(sh_bytes)?; - let name_offset = strhdr.offset().checked_add(u64::from(sh.name()))?; - let section_name = elf_str(elf, name_offset)?; - - if section_name != name { - return None; - } - - let start = usize::try_from(sh.offset()).ok()?; - let end = usize::try_from(sh.size()) - .ok() - .and_then(|sz| start.checked_add(sz))?; - - elf.get(start..end) - }) - } - - /// Extract the section with name `name` from the ELF64 image `elf`. - fn elf64_section<'a>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> { - elf_section_generic::(elf, name) - } - - /// Extract the section with name `name` from the ELF32 image `elf`. - fn elf32_section<'a>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> { - elf_section_generic::(elf, name) - } - - /// Automatically detects ELF32 vs ELF64 based on the ELF header. - pub(super) fn elf_section<'a>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> { - // ELF identification: a 4-byte magic followed by a class byte (32- vs 64-bit). - const ELFMAG: &[u8] = b"\x7fELF"; - const SELFMAG: usize = ELFMAG.len(); - const EI_CLASS: usize = 4; - const ELFCLASS32: u8 = 1; - const ELFCLASS64: u8 = 2; - - if elf.get(0..SELFMAG) != Some(ELFMAG) { - return None; - } - - match *elf.get(EI_CLASS)? { - ELFCLASS32 => elf32_section(elf, name), - ELFCLASS64 => elf64_section(elf, name), - _ => None, - } - } -} diff --git a/drivers/gpu/nova-core/firmware/fsp.rs b/drivers/gpu/nova-core/firmware/fsp.rs index 6eaf1c684b9d..5462e318410a 100644 --- a/drivers/gpu/nova-core/firmware/fsp.rs +++ b/drivers/gpu/nova-core/firmware/fsp.rs @@ -6,12 +6,14 @@ use kernel::{ device, dma::Coherent, - firmware::Firmware, prelude::*, // }; use crate::{ - firmware::elf, + firmware::tlv::{ + request_tlv, // + Tlv, + }, gpu::Chipset, // }; @@ -19,11 +21,11 @@ const FSP_HASH_SIZE: usize = 48; /// Maximum size of the FSP public key (RSA-3072), in bytes. /// -/// The FMC ELF `publickey` section may be shorter, so the remaining bytes are zero-padded. +/// The FMC `PKEY` tag may be shorter, so the remaining bytes are zero-padded. const FSP_PKEY_SIZE: usize = 384; /// Maximum size of the FSP signature (RSA-3072), in bytes. /// -/// The FMC ELF `signature` section may be shorter, so the remaining bytes are zero-padded. +/// The FMC `SIGN` tag may be shorter, so the remaining bytes are zero-padded. const FSP_SIG_SIZE: usize = 384; /// Structure to hold FMC signatures. @@ -38,64 +40,35 @@ pub(crate) struct FmcSignatures { } pub(crate) struct FspFirmware { - /// FMC firmware image data (only the "image" ELF section). + /// FMC firmware image data pub(crate) fmc_image: Coherent<[u8]>, /// FMC firmware signatures. pub(crate) fmc_sigs: KBox, } impl FspFirmware { - pub(crate) fn new( - dev: &device::Device, - chipset: Chipset, - ver: &str, - ) -> Result { - let fw = super::request_firmware(dev, chipset, "fmc", ver)?; + pub(crate) fn new(dev: &device::Device, chipset: Chipset) -> Result { + let fw = request_tlv(dev, chipset, "fmc")?; + let tlv = Tlv::new(fw.data())?; + dev_dbg!(dev, "loaded fsp firmware v{}\n", tlv.get_string(b"VERS")?); - // FSP expects only the "image" section, not the entire ELF file. - let fmc_image_data = elf::elf_section(fw.data(), "image").ok_or_else(|| { - dev_err!(dev, "FMC ELF file missing 'image' section\n"); - EINVAL - })?; + let fmc_image_data = tlv.get_bytes(b"BLOB")?; let fmc_image = Coherent::from_slice(dev, fmc_image_data, GFP_KERNEL)?; Ok(Self { fmc_image, - fmc_sigs: Self::extract_fmc_signatures(&fw, dev)?, + fmc_sigs: Self::extract_fmc_signatures(&tlv, dev)?, }) } /// Extract FMC firmware signatures for Chain of Trust verification. /// - /// Extracts real cryptographic signatures from FMC ELF32 firmware sections. + /// Extracts real cryptographic signatures from FMC TLV firmware tags. /// Returns signatures in a heap-allocated structure to prevent stack overflow. - fn extract_fmc_signatures( - fmc_fw: &Firmware, - dev: &device::Device, - ) -> Result> { - let get_section = |name: &str, max_len: usize| { - elf::elf_section(fmc_fw.data(), name) - .ok_or(EINVAL) - .inspect_err(|_| dev_err!(dev, "FMC firmware missing '{}' section\n", name)) - .and_then(|section| { - if section.len() > max_len { - dev_err!( - dev, - "FMC {} section size {} > maximum {}\n", - name, - section.len(), - max_len - ); - Err(EINVAL) - } else { - Ok(section) - } - }) - }; - - let hash_section = get_section("hash", FSP_HASH_SIZE)?; - let pkey_section = get_section("publickey", FSP_PKEY_SIZE)?; - let sig_section = get_section("signature", FSP_SIG_SIZE)?; + fn extract_fmc_signatures(tlv: &Tlv<'_>, dev: &device::Device) -> Result> { + let hash_section = tlv.get_bytes(b"HASH")?; + let pkey_section = tlv.get_bytes(b"PKEY")?; + let sig_section = tlv.get_bytes(b"SIGN")?; // The hash section is a SHA-384 output: it must be exactly FSP_HASH_SIZE bytes. if hash_section.len() != FSP_HASH_SIZE { @@ -108,15 +81,36 @@ fn extract_fmc_signatures( return Err(EINVAL); } + // The key and signature sections are zero-padded to a fixed maximum, so they may be + // shorter, but must not exceed the destination buffers. + if pkey_section.len() > FSP_PKEY_SIZE { + dev_err!( + dev, + "FMC public key section size {} > maximum {}\n", + pkey_section.len(), + FSP_PKEY_SIZE + ); + return Err(EINVAL); + } + if sig_section.len() > FSP_SIG_SIZE { + dev_err!( + dev, + "FMC signature section size {} > maximum {}\n", + sig_section.len(), + FSP_SIG_SIZE + ); + return Err(EINVAL); + } + // Initialize the signatures in place to avoid building the large `FmcSignatures` on the // stack, then fill each section from the firmware. let signatures = KBox::init( pin_init::init_zeroed::().chain(|sigs| { // PANIC: src and dst lengths are both FSP_HASH_SIZE (verified above). sigs.hash384.copy_from_slice(hash_section); - // PANIC: dst is sliced to src.len(); src.len() <= FSP_PKEY_SIZE per `get_section`. + // PANIC: dst is sliced to src.len(); src.len() <= FSP_PKEY_SIZE (verified above). sigs.public_key[..pkey_section.len()].copy_from_slice(pkey_section); - // PANIC: dst is sliced to src.len(); src.len() <= FSP_SIG_SIZE per `get_section`. + // PANIC: dst is sliced to src.len(); src.len() <= FSP_SIG_SIZE (verified above). sigs.signature[..sig_section.len()].copy_from_slice(sig_section); Ok(()) }), diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index 6263277a7614..2a4d8ff53dbe 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -32,12 +32,9 @@ Falcon, // }, fb::FbSizes, - firmware::{ - fsp::{ - FmcSignatures, - FspFirmware, // - }, - FIRMWARE_VERSION, // + firmware::fsp::{ + FmcSignatures, + FspFirmware, // }, gpu::Chipset, gsp::{ @@ -425,7 +422,7 @@ fn wait_secure_boot( const FSP_SECURE_BOOT_TIMEOUT_MS: i64 = 5000; let falcon = Falcon::::new(dev, chipset, bar)?; - let fsp_fw = FspFirmware::new(dev, chipset, FIRMWARE_VERSION)?; + let fsp_fw = FspFirmware::new(dev, chipset)?; read_poll_timeout( || Ok(hal.fsp_boot_status(bar)), From 7923d0bc8ede8786f83b15bce5876cf112c347a6 Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Fri, 31 Jul 2026 15:10:17 -0500 Subject: [PATCH 134/137] gpu: nova-core: update firmware module info for TLV images Now that nova-core loads the TLV firmware images, update the firmware module info to specify those files. Also remove FIRMWARE_VERSION as it is no longer used. Signed-off-by: Timur Tabi Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260731201017.2580713-9-ttabi@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/firmware.rs | 12 ++++-------- drivers/gpu/nova-core/gsp/hal.rs | 8 ++++---- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index d58a3d5770d5..b49613a90bf0 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -29,8 +29,6 @@ pub(crate) mod riscv; pub(crate) mod tlv; -pub(crate) const FIRMWARE_VERSION: &str = "570.144"; - /// Structure used to describe some firmwares, notably FWSEC-FRTS. #[repr(C)] #[derive(Debug, Clone, FromBytes)] @@ -338,10 +336,7 @@ const fn make_entry_file(self, chipset: &str, fw: &str) -> Self { .push("nvidia/") .push(chipset) .push("/gsp/") - .push(fw) - .push("-") - .push(FIRMWARE_VERSION) - .push(".bin"), + .push(fw), ) } @@ -350,8 +345,9 @@ const fn make_entry_chipset(self, chipset: gpu::Chipset) -> Self { // GSP firmware files are always present. let mut this = self - .make_entry_file(name, "bootloader") - .make_entry_file(name, "gsp"); + .make_entry_file(name, "gsp_bootloader.tlv") + .make_entry_file(name, "gsp.tlv") + .make_entry_file(name, "gsp.bin"); // Add the firmware files specific to the GSP boot method of `chipset`. let boot_files = boot_firmware_files(chipset); diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs index 11e436651a69..5850fa0fe0e9 100644 --- a/drivers/gpu/nova-core/gsp/hal.rs +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -61,16 +61,16 @@ fn post_boot( pub(crate) const fn boot_firmware_files(chipset: Chipset) -> &'static [&'static str] { match chipset.arch() { // Turing chipsets boot the GSP via the SEC2 Booter, and require the FWSEC bootloader. - Architecture::Turing => &["booter_load", "booter_unload", "gen_bootloader"], + Architecture::Turing => &["booter_load.tlv", "booter_unload.tlv", "gen_bootloader.tlv"], // GA100 also requires the FWSEC bootloader. Architecture::Ampere if matches!(chipset, Chipset::GA100) => { - &["booter_load", "booter_unload", "gen_bootloader"] + &["booter_load.tlv", "booter_unload.tlv", "gen_bootloader.tlv"] } // Other Ampere chipsets, as well as Ada chipsets, run FWSEC directly. - Architecture::Ampere | Architecture::Ada => &["booter_load", "booter_unload"], + Architecture::Ampere | Architecture::Ada => &["booter_load.tlv", "booter_unload.tlv"], // Hopper and later chipsets boot the GSP via the FMC image loaded by FSP. Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { - &["fmc"] + &["fmc.tlv"] } } } From d1dc8faf1152e39070250df714b2d544c8ae32bb Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 5 Aug 2026 14:01:45 +0900 Subject: [PATCH 135/137] gpu: nova-core: falcon: remove unnecessary check The `try_with_base` call performed on `NV_PFALCON_FALCON_DMATRFBASE1` already returns `EOVERFLOW` if the address is too large for the register, making this check redundant. Signed-off-by: Alexandre Courbot Link: https://patch.msgid.link/20260805-falcon-dma-projections-v2-1-4cc9f3f13ee9@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/falcon.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index a91cbdd5d636..cd05985f5ee6 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -9,8 +9,7 @@ dma::{ Coherent, CoherentBox, - DmaAddress, - DmaMask, // + DmaAddress, // }, io::{ poll::read_poll_timeout, @@ -534,12 +533,6 @@ fn dma_wr( return Err(EINVAL); } - // The DMATRFBASE/1 register pair only supports a 49-bit address. - if dma_start > DmaMask::new::<49>().value() { - dev_err!(self.dev, "DMA address {:#x} exceeds 49 bits\n", dma_start); - return Err(ERANGE); - } - // DMA transfers can only be done in units of 256 bytes. Compute how many such transfers we // need to perform. let num_transfers = load_offsets.len.div_ceil(DMA_LEN); From 91645a52ebf2e22f0bd37c142b79238617d38758 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 5 Aug 2026 14:01:46 +0900 Subject: [PATCH 136/137] rust: dma: rename dma_handle to dma_address The `dma_handle` naming is inherited from the C API, but what this really describes is the device DMA address; everything named `dma_handle` is actually a `dma_addr_t`. This naming introduces some confusion on the Rust API side, as handles are supposed to be opaque tokens, yet we were doing address computation on values returned by `dma_handle`. Rename `dma_handle` to `dma_address` while nova-core is still its only user. Suggested-by: John Hubbard Suggested-by: Danilo Krummrich Link: https://lore.kernel.org/all/DK75LUA4NLGI.3P29AIZQE20V2@kernel.org/ Signed-off-by: Alexandre Courbot Reviewed-by: Robin Murphy Link: https://patch.msgid.link/20260805-falcon-dma-projections-v2-2-4cc9f3f13ee9@nvidia.com [ Rebase and fix up build failures due to newly introduced dma_handle() calls. - Danilo ] Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/falcon.rs | 8 +-- drivers/gpu/nova-core/fb.rs | 4 +- drivers/gpu/nova-core/firmware/booter.rs | 8 ++- .../nova-core/firmware/fwsec/bootloader.rs | 4 +- drivers/gpu/nova-core/firmware/gsp.rs | 6 +- drivers/gpu/nova-core/fsp.rs | 6 +- drivers/gpu/nova-core/gsp.rs | 2 +- drivers/gpu/nova-core/gsp/cmdq.rs | 8 +-- drivers/gpu/nova-core/gsp/fw.rs | 16 ++--- drivers/gpu/nova-core/gsp/hal/gh100.rs | 2 +- drivers/gpu/nova-core/gsp/hal/tu102.rs | 8 ++- drivers/gpu/nova-core/gsp/sequencer.rs | 8 +-- rust/kernel/dma.rs | 70 +++++++++---------- 13 files changed, 77 insertions(+), 73 deletions(-) diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index cd05985f5ee6..a281d316ebfd 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -499,7 +499,7 @@ pub(crate) fn pio_load + FalconPioLoadable>( Ok(()) } - /// Perform a DMA write according to `load_offsets` from `dma_handle` into the falcon's + /// Perform a DMA write according to `load_offsets` from `dma_obj` into the falcon's /// `target_mem`. /// /// `sec` is set if the loaded firmware is expected to run in secure mode. @@ -514,14 +514,14 @@ fn dma_wr( // For IMEM, we want to use the start offset as a virtual address tag for each page, since // code addresses in the firmware (and the boot vector) are virtual. // - // For DMEM we can fold the start offset into the DMA handle. + // For DMEM we can fold the start offset into the DMA address. let (src_start, dma_start) = match target_mem { FalconMem::ImemSecure | FalconMem::ImemNonSecure => { - (load_offsets.src_start, dma_obj.dma_handle()) + (load_offsets.src_start, dma_obj.dma_address()) } FalconMem::Dmem => ( 0, - dma_obj.dma_handle() + DmaAddress::from(load_offsets.src_start), + dma_obj.dma_address() + DmaAddress::from(load_offsets.src_start), ), }; if dma_start % DmaAddress::from(DMA_LEN) > 0 { diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 77447a6567a4..1576399389b1 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -61,7 +61,7 @@ pub(crate) fn register( ) -> Result { let page = CoherentHandle::alloc(dev, kernel::page::PAGE_SIZE, GFP_KERNEL)?; - hal::fb_hal(chipset).write_sysmem_flush_page(bar, page.dma_handle())?; + hal::fb_hal(chipset).write_sysmem_flush_page(bar, page.dma_address())?; Ok(Self { chipset, @@ -76,7 +76,7 @@ impl Drop for SysmemFlush<'_> { fn drop(&mut self) { let hal = hal::fb_hal(self.chipset); - if hal.read_sysmem_flush_page(self.bar) == self.page.dma_handle() { + if hal.read_sysmem_flush_page(self.bar) == self.page.dma_address() { let _ = hal.write_sysmem_flush_page(self.bar, 0).inspect_err(|e| { dev_warn!( &self.device, diff --git a/drivers/gpu/nova-core/firmware/booter.rs b/drivers/gpu/nova-core/firmware/booter.rs index 6e7d688deadd..dc071edba331 100644 --- a/drivers/gpu/nova-core/firmware/booter.rs +++ b/drivers/gpu/nova-core/firmware/booter.rs @@ -190,9 +190,11 @@ pub(crate) fn run( ) -> Result { sec2_falcon.reset()?; sec2_falcon.load(self)?; - let wpr_handle = wpr_meta.dma_handle(); - let (mbox0, mbox1) = - sec2_falcon.boot(Some(wpr_handle as u32), Some((wpr_handle >> 32) as u32))?; + let wpr_dma_address = wpr_meta.dma_address(); + let (mbox0, mbox1) = sec2_falcon.boot( + Some(wpr_dma_address as u32), + Some((wpr_dma_address >> 32) as u32), + )?; dev_dbg!(dev, "SEC2 MBOX0: {:#x}, MBOX1: {:#x}\n", mbox0, mbox1); if mbox0 != 0 { diff --git a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs index 30d247b59848..ec4d92317a93 100644 --- a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs +++ b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs @@ -191,7 +191,7 @@ pub(crate) fn new( reserved: [0; 4], signature: [0; 4], ctx_dma: FALCON_DMAIDX_PHYS_SYS_NCOH, - code_dma_base: firmware_dma.dma_handle(), + code_dma_base: firmware_dma.dma_address(), // `dst_start` is also valid as the source offset since the firmware DMA object is // a mirror image of the target IMEM layout. non_sec_code_off: imem_ns.dst_start, @@ -203,7 +203,7 @@ pub(crate) fn new( code_entry_point: 0, // Start of data section is the added padding + the DMEM `src_start` field. data_dma_base: firmware_dma - .dma_handle() + .dma_address() .checked_add(u64::from_safe_cast(align_padding)) .and_then(|offset| offset.checked_add(dmem.src_start.into())) .ok_or(EOVERFLOW)?, diff --git a/drivers/gpu/nova-core/firmware/gsp.rs b/drivers/gpu/nova-core/firmware/gsp.rs index 55a8c513e3a9..e8f9491e84cc 100644 --- a/drivers/gpu/nova-core/firmware/gsp.rs +++ b/drivers/gpu/nova-core/firmware/gsp.rs @@ -140,9 +140,9 @@ pub(crate) fn new<'a>( }) } - /// Returns the DMA handle of the radix3 level 0 page table. - pub(crate) fn radix3_dma_handle(&self) -> DmaAddress { - self.level0.dma_handle() + /// Returns the DMA address of the radix3 level 0 page table. + pub(crate) fn radix3_dma_address(&self) -> DmaAddress { + self.level0.dma_address() } } diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index 2a4d8ff53dbe..ab685fb4168f 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -296,12 +296,12 @@ fn new<'a>( .chain(move |msg| { msg.cot.version = version; msg.cot.size = size; - msg.cot.gsp_fmc_sysmem_offset = fsp_fw.fmc_image.dma_handle(); + msg.cot.gsp_fmc_sysmem_offset = fsp_fw.fmc_image.dma_address(); msg.cot.frts_vidmem_offset = frts_vidmem_offset; msg.cot.frts_vidmem_size = frts_size; // frts_sysmem_* are left at zero because this path places FRTS in vidmem. The sysmem // fields point to an FRTS buffer in sysmem instead, for systems without VRAM. - msg.cot.gsp_boot_args_sysmem_offset = args.fmc_boot_params.dma_handle(); + msg.cot.gsp_boot_args_sysmem_offset = args.fmc_boot_params.dma_address(); msg.cot.sigs = *fsp_fw.fmc_sigs; Ok(()) @@ -362,7 +362,7 @@ pub(crate) fn new( libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, resume: bool, ) -> Result { - let init = GspFmcBootParams::new(wpr_meta.dma_handle(), libos.dma_handle()); + let init = GspFmcBootParams::new(wpr_meta.dma_address(), libos.dma_address()); Ok(Self { chipset, diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index b403dc3515a5..13f361406a6c 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -122,7 +122,7 @@ impl LogBuffer { fn new(dev: &device::Device) -> Result { let obj = Self(Coherent::zeroed(dev, GFP_KERNEL)?); - let start_addr = obj.0.dma_handle(); + let start_addr = obj.0.dma_address(); let pte_view = io_project!( obj.0, diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index cd844fe48f05..f0f28b6ded7a 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -243,7 +243,7 @@ fn new(dev: &device::Device) -> Result { gsp_mem.cpuq.rx = MsgqRxHeader::new(); let gsp_mem: Coherent<_> = gsp_mem.into(); - PteArray::init(io_project!(gsp_mem, .ptes), gsp_mem.dma_handle())?; + PteArray::init(io_project!(gsp_mem, .ptes), gsp_mem.dma_address())?; Ok(Self(gsp_mem)) } @@ -487,8 +487,8 @@ pub(crate) struct Cmdq { /// Inner mutex-protected state. #[pin] inner: Mutex, - /// DMA handle of the command queue's shared memory region. - pub(super) dma_handle: DmaAddress, + /// DMA address of the command queue's shared memory region. + pub(super) dma_addr: DmaAddress, } impl Cmdq { @@ -517,7 +517,7 @@ pub(crate) fn new(dev: &device::Device) -> impl PinInit( // CAST: we want to store the bits of `GSP_FW_WPR_META_MAGIC` unmodified. magic: bindings::GSP_FW_WPR_META_MAGIC as u64, revision: u64::from(bindings::GSP_FW_WPR_META_REVISION), - sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_handle(), + sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_address(), sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size), - sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_handle(), + sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_address(), sizeOfBootloader: u64::from_safe_cast(gsp_firmware.bootloader.ucode.size()), bootloaderCodeOffset: u64::from(gsp_firmware.bootloader.code_offset), bootloaderDataOffset: u64::from(gsp_firmware.bootloader.data_offset), bootloaderManifestOffset: u64::from(gsp_firmware.bootloader.manifest_offset), __bindgen_anon_1: GspFwWprMetaBootResumeInfo { __bindgen_anon_1: GspFwWprMetaBootInfo { - sysmemAddrOfSignature: gsp_firmware.signatures.dma_handle(), + sysmemAddrOfSignature: gsp_firmware.signatures.dma_address(), sizeOfSignature: u64::from_safe_cast(gsp_firmware.signatures.size()), }, }, @@ -241,16 +241,16 @@ pub(crate) fn from_sizes<'a>( // CAST: we want to store the bits of `GSP_FW_WPR_META_MAGIC` unmodified. magic: bindings::GSP_FW_WPR_META_MAGIC as u64, revision: u64::from(bindings::GSP_FW_WPR_META_REVISION), - sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_handle(), + sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_address(), sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size), - sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_handle(), + sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_address(), sizeOfBootloader: u64::from_safe_cast(gsp_firmware.bootloader.ucode.size()), bootloaderCodeOffset: u64::from(gsp_firmware.bootloader.code_offset), bootloaderDataOffset: u64::from(gsp_firmware.bootloader.data_offset), bootloaderManifestOffset: u64::from(gsp_firmware.bootloader.manifest_offset), __bindgen_anon_1: GspFwWprMetaBootResumeInfo { __bindgen_anon_1: GspFwWprMetaBootInfo { - sysmemAddrOfSignature: gsp_firmware.signatures.dma_handle(), + sysmemAddrOfSignature: gsp_firmware.signatures.dma_address(), sizeOfSignature: u64::from_safe_cast(gsp_firmware.signatures.size()), }, }, @@ -680,7 +680,7 @@ fn id8(name: &str) -> u64 { let init_inner = init!(bindings::LibosMemoryRegionInitArgument { id8: id8(name), - pa: obj.dma_handle(), + pa: obj.dma_address(), size: num::usize_as_u64(obj.size()), kind: num::u32_into_u8::< { bindings::LibosMemoryRegionKind_LIBOS_MEMORY_REGION_CONTIGUOUS }, @@ -946,7 +946,7 @@ impl MessageQueueInitArguments { /// Creates a new init arguments structure for `cmdq`. fn new(cmdq: &Cmdq) -> impl Init + '_ { init!(MessageQueueInitArguments { - sharedMemPhysAddr: cmdq.dma_handle, + sharedMemPhysAddr: cmdq.dma_addr, pageTableEntryCount: num::usize_into_u32::<{ Cmdq::NUM_PTES }>(), cmdQueueOffset: num::usize_as_u64(Cmdq::CMDQ_OFFSET), statQueueOffset: num::usize_as_u64(Cmdq::STATQ_OFFSET), diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index b16c2f6f82a0..e283429a95dd 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -64,7 +64,7 @@ fn lockdown_released_or_error( // boot. If the address is still there, keep polling rather than treating it as an error. // Any other non-zero mailbox0 value is a GSP-FMC error code. if self.mbox0 != 0 { - return self.combined_addr() != fmc_boot_params.dma_handle(); + return self.combined_addr() != fmc_boot_params.dma_address(); } !gsp_falcon.riscv_branch_privilege_lockdown() diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index 2ae3140cb0fe..a5c0ca355493 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -294,9 +294,11 @@ fn boot( } gsp_falcon.reset()?; - let libos_handle = gsp.libos.dma_handle(); - let (mbox0, mbox1) = - gsp_falcon.boot(Some(libos_handle as u32), Some((libos_handle >> 32) as u32))?; + let libos_dma_address = gsp.libos.dma_address(); + let (mbox0, mbox1) = gsp_falcon.boot( + Some(libos_dma_address as u32), + Some((libos_dma_address >> 32) as u32), + )?; dev_dbg!(dev, "GSP MBOX0: {:#x}, MBOX1: {:#x}\n", mbox0, mbox1); dev_dbg!( diff --git a/drivers/gpu/nova-core/gsp/sequencer.rs b/drivers/gpu/nova-core/gsp/sequencer.rs index 5e1ec7e59ab0..bcad1421953a 100644 --- a/drivers/gpu/nova-core/gsp/sequencer.rs +++ b/drivers/gpu/nova-core/gsp/sequencer.rs @@ -234,12 +234,12 @@ fn run(&self, seq: &GspSequencer<'_>) -> Result { // Reset the GSP to prepare it for resuming. seq.gsp_falcon.reset()?; - let libos_dma_handle = seq.libos.dma_handle(); + let libos_dma_address = seq.libos.dma_address(); - // Write the libOS DMA handle to GSP mailboxes. + // Write the libOS DMA address to GSP mailboxes. seq.gsp_falcon.write_mailboxes( - Some(libos_dma_handle as u32), - Some((libos_dma_handle >> 32) as u32), + Some(libos_dma_address as u32), + Some((libos_dma_address >> 32) as u32), ); // Start the SEC2 falcon which will trigger GSP-RM to resume on the GSP. diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index e275f2562a5b..4258ff7ff525 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -585,7 +585,7 @@ fn from(value: CoherentBox) -> Self { /// # Invariants /// /// - For the lifetime of an instance of [`Coherent`], the `cpu_addr` is a valid pointer -/// to an allocated region of coherent memory and `dma_handle` is the DMA address base of the +/// to an allocated region of coherent memory and `dma_addr` is the DMA address base of the /// region. /// - The size in bytes of the allocation is equal to size information via pointer. // TODO @@ -602,7 +602,7 @@ fn from(value: CoherentBox) -> Self { // entire `Coherent` including the allocated memory itself. pub struct Coherent { dev: ARef, - dma_handle: DmaAddress, + dma_addr: DmaAddress, cpu_addr: NonNull, dma_attrs: Attrs, } @@ -627,11 +627,10 @@ pub fn as_mut_ptr(&self) -> *mut T { self.cpu_addr.as_ptr() } - /// Returns a DMA handle which may be given to the device as the DMA address base of - /// the region. + /// Returns a DMA address which may be given to the device as the base of the region. #[inline] - pub fn dma_handle(&self) -> DmaAddress { - self.dma_handle + pub fn dma_address(&self) -> DmaAddress { + self.dma_addr } /// Returns a reference to the data in the region. @@ -678,13 +677,13 @@ fn alloc_with_attrs( ); } - let mut dma_handle = 0; + let mut dma_addr = 0; // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. let addr = unsafe { bindings::dma_alloc_attrs( dev.as_raw(), core::mem::size_of::(), - &mut dma_handle, + &mut dma_addr, gfp_flags.as_raw(), dma_attrs.as_raw(), ) @@ -696,7 +695,7 @@ fn alloc_with_attrs( // - We also hold a refcounted reference to the device. Ok(Self { dev: dev.into(), - dma_handle, + dma_addr, cpu_addr, dma_attrs, }) @@ -795,13 +794,13 @@ fn alloc_slice_with_attrs( } let size = core::mem::size_of::().checked_mul(len).ok_or(ENOMEM)?; - let mut dma_handle = 0; + let mut dma_addr = 0; // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. let addr = unsafe { bindings::dma_alloc_attrs( dev.as_raw(), size, - &mut dma_handle, + &mut dma_addr, gfp_flags.as_raw(), dma_attrs.as_raw(), ) @@ -813,7 +812,7 @@ fn alloc_slice_with_attrs( // - We also hold a refcounted reference to the device. Ok(Coherent { dev: dev.into(), - dma_handle, + dma_addr, cpu_addr, dma_attrs, }) @@ -927,14 +926,14 @@ impl Drop for Coherent { fn drop(&mut self) { let size = T::size(self.cpu_addr.as_ptr()); // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. - // The cpu address, and the dma handle are valid due to the type invariants on + // The cpu address, and the dma address are valid due to the type invariants on // `Coherent`. unsafe { bindings::dma_free_attrs( self.dev.as_raw(), size, self.cpu_addr.as_ptr().cast(), - self.dma_handle, + self.dma_addr, self.dma_attrs.as_raw(), ) } @@ -989,13 +988,13 @@ fn write_to_slice( /// /// - `cpu_handle` holds the opaque handle returned by `dma_alloc_attrs` with /// `DMA_ATTR_NO_KERNEL_MAPPING` set, and is only valid for passing back to `dma_free_attrs`. -/// - `dma_handle` is the corresponding bus address for device DMA. +/// - `dma_addr` is the corresponding bus address for device DMA. /// - `size` is the allocation size in bytes as passed to `dma_alloc_attrs`. /// - `dma_attrs` contains the attributes used for the allocation, always including /// `DMA_ATTR_NO_KERNEL_MAPPING`. pub struct CoherentHandle { dev: ARef, - dma_handle: DmaAddress, + dma_addr: DmaAddress, cpu_handle: NonNull, size: usize, dma_attrs: Attrs, @@ -1019,13 +1018,13 @@ pub fn alloc_with_attrs( } let dma_attrs = dma_attrs | Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING); - let mut dma_handle = 0; + let mut dma_addr = 0; // SAFETY: `dev.as_raw()` is valid by the type invariant on `device::Device`. let cpu_handle = unsafe { bindings::dma_alloc_attrs( dev.as_raw(), size, - &mut dma_handle, + &mut dma_addr, gfp_flags.as_raw(), dma_attrs.as_raw(), ) @@ -1034,11 +1033,11 @@ pub fn alloc_with_attrs( let cpu_handle = NonNull::new(cpu_handle).ok_or(ENOMEM)?; // INVARIANT: `cpu_handle` is the opaque handle from a successful `dma_alloc_attrs` call - // with `DMA_ATTR_NO_KERNEL_MAPPING`, `dma_handle` is the corresponding DMA address, + // with `DMA_ATTR_NO_KERNEL_MAPPING`, `dma_addr` is the corresponding DMA address, // and we hold a refcounted reference to the device. Ok(Self { dev: dev.into(), - dma_handle, + dma_addr, cpu_handle, size, dma_attrs, @@ -1055,12 +1054,12 @@ pub fn alloc( Self::alloc_with_attrs(dev, size, gfp_flags, Attrs(0)) } - /// Returns the DMA handle for this allocation. + /// Returns the DMA address for this allocation. /// /// This address can be programmed into device hardware for DMA access. #[inline] - pub fn dma_handle(&self) -> DmaAddress { - self.dma_handle + pub fn dma_address(&self) -> DmaAddress { + self.dma_addr } /// Returns the size in bytes of this allocation. @@ -1079,28 +1078,29 @@ fn drop(&mut self) { self.dev.as_raw(), self.size, self.cpu_handle.as_ptr(), - self.dma_handle, + self.dma_addr, self.dma_attrs.as_raw(), ) } } } -// SAFETY: `CoherentHandle` only holds a device reference, a DMA handle, an opaque CPU handle, +// SAFETY: `CoherentHandle` only holds a device reference, a DMA address, an opaque CPU handle, // and a size. None of these are tied to a specific thread. unsafe impl Send for CoherentHandle {} // SAFETY: `CoherentHandle` provides no CPU access to the underlying allocation. The only -// operations on `&CoherentHandle` are reading the DMA handle and size, both of which are +// operations on `&CoherentHandle` are reading the DMA address and size, both of which are // plain `Copy` values. unsafe impl Sync for CoherentHandle {} /// View type for `Coherent`. /// -/// This is same as [`SysMem`] but with additional information that allows handing out a DMA handle. +/// This is same as [`SysMem`] but with additional information that allows handing out a DMA +/// address. pub struct CoherentView<'a, T: ?Sized> { cpu_addr: SysMem<'a, T>, - dma_handle: DmaAddress, + dma_addr: DmaAddress, } impl Copy for CoherentView<'_, T> {} @@ -1112,16 +1112,16 @@ fn clone(&self) -> Self { } impl<'a, T: ?Sized> CoherentView<'a, T> { - /// Erase the DMA handle information and obtain a [`SysMem`] view of the same memory region. + /// Erase the DMA address information and obtain a [`SysMem`] view of the same memory region. #[inline] pub fn as_sys_mem(self) -> SysMem<'a, T> { self.cpu_addr } - /// Returns a DMA handle which may be given to the device as the DMA address base of the region. + /// Returns the DMA address which may be given to the device as base of the region. #[inline] - pub fn dma_handle(self) -> DmaAddress { - self.dma_handle + pub fn dma_address(self) -> DmaAddress { + self.dma_addr } /// Returns a reference to the data in the region. @@ -1174,9 +1174,9 @@ unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( ) -> Self::View<'a, U> { let offset = ptr.addr() - view.cpu_addr.as_ptr().addr(); // CAST: The offset DMA address can never overflow. - let dma_handle = view.dma_handle + offset as DmaAddress; + let dma_addr = view.dma_addr + offset as DmaAddress; CoherentView { - dma_handle, + dma_addr, // SAFETY: Per safety requirement. cpu_addr: unsafe { SysMemBackend::project_view(view.cpu_addr, ptr) }, } @@ -1241,7 +1241,7 @@ fn as_view(self) -> CoherentView<'a, Self::Target> { CoherentView { // SAFETY: `cpu_addr` is valid and aligned kernel accessible memory. cpu_addr: unsafe { SysMem::new(self.cpu_addr.as_ptr()) }, - dma_handle: self.dma_handle, + dma_addr: self.dma_addr, } } } From 4c9ba407018e8deb06dbc643112bac8f40404f95 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 5 Aug 2026 14:01:47 +0900 Subject: [PATCH 137/137] gpu: nova-core: falcon: use I/O projection to check transfer bounds The DMA transfer routine was computing the start of the DMA area by taking the address of the coherent allocation, and then adding the transfer's start offset. It then checked manually that the upper bound was valid. Convert this to an I/O projection of the same region, which returns `ERANGE` if the passed range does not fit within the coherent allocation. This removes the need to perform arithmetic on DMA addresses and to explicitly check for the bounds' validity. Signed-off-by: Alexandre Courbot Link: https://patch.msgid.link/20260805-falcon-dma-projections-v2-3-4cc9f3f13ee9@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/falcon.rs | 55 ++++++++++++++------------------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index a281d316ebfd..65cb12d26e2b 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -12,6 +12,7 @@ DmaAddress, // }, io::{ + io_project, poll::read_poll_timeout, register::{ RegisterBase, @@ -511,20 +512,31 @@ fn dma_wr( ) -> Result { const DMA_LEN: u32 = num::usize_into_u32::<{ MEM_BLOCK_ALIGNMENT }>(); + // DMA transfers can only be done in units of 256 bytes. Compute how many such transfers we + // need to perform. + let num_transfers = load_offsets.len.div_ceil(DMA_LEN); + // For IMEM, we want to use the start offset as a virtual address tag for each page, since // code addresses in the firmware (and the boot vector) are virtual. // - // For DMEM we can fold the start offset into the DMA address. + // For DMEM, the start offset is folded into the DMA address. let (src_start, dma_start) = match target_mem { - FalconMem::ImemSecure | FalconMem::ImemNonSecure => { - (load_offsets.src_start, dma_obj.dma_address()) - } - FalconMem::Dmem => ( - 0, - dma_obj.dma_address() + DmaAddress::from(load_offsets.src_start), - ), + FalconMem::ImemSecure | FalconMem::ImemNonSecure => (load_offsets.src_start, 0), + FalconMem::Dmem => (0, usize::from_safe_cast(load_offsets.src_start)), }; - if dma_start % DmaAddress::from(DMA_LEN) > 0 { + + let dma_address = { + // Upper limit of transfer is `(num_transfers * DMA_LEN) + load_offsets.src_start`. + let dma_end = num_transfers + .checked_mul(DMA_LEN) + .and_then(|size| size.checked_add(load_offsets.src_start)) + .map(usize::from_safe_cast) + .ok_or(EOVERFLOW)?; + + io_project!(dma_obj, [try: dma_start..dma_end]).dma_address() + }; + + if dma_address % DmaAddress::from(DMA_LEN) > 0 { dev_err!( self.dev, "DMA transfer start addresses must be a multiple of {}\n", @@ -533,27 +545,6 @@ fn dma_wr( return Err(EINVAL); } - // DMA transfers can only be done in units of 256 bytes. Compute how many such transfers we - // need to perform. - let num_transfers = load_offsets.len.div_ceil(DMA_LEN); - - // Check that the area we are about to transfer is within the bounds of the DMA object. - // Upper limit of transfer is `(num_transfers * DMA_LEN) + load_offsets.src_start`. - match num_transfers - .checked_mul(DMA_LEN) - .and_then(|size| size.checked_add(load_offsets.src_start)) - { - None => { - dev_err!(self.dev, "DMA transfer length overflow\n"); - return Err(EOVERFLOW); - } - Some(upper_bound) if usize::from_safe_cast(upper_bound) > dma_obj.size() => { - dev_err!(self.dev, "DMA transfer goes beyond range of DMA object\n"); - return Err(EINVAL); - } - Some(_) => (), - }; - // Set up the base source DMA address. self.bar.write( @@ -561,12 +552,12 @@ fn dma_wr( regs::NV_PFALCON_FALCON_DMATRFBASE::zeroed().with_base( // CAST: `as u32` is used on purpose since we do want to strip the upper bits, // which will be written to `NV_PFALCON_FALCON_DMATRFBASE1`. - (dma_start >> 8) as u32, + (dma_address >> 8) as u32, ), ); self.bar.write( WithBase::of::(), - regs::NV_PFALCON_FALCON_DMATRFBASE1::zeroed().try_with_base(dma_start >> 40)?, + regs::NV_PFALCON_FALCON_DMATRFBASE1::zeroed().try_with_base(dma_address >> 40)?, ); let cmd = regs::NV_PFALCON_FALCON_DMATRFCMD::zeroed()