From 6556f7556d622dbbc5c936df9d39c2f33600adfb Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Mon, 10 Aug 2026 13:12:43 -0500 Subject: [PATCH 001/110] gpu: nova-core: write initial RPCs before booting GSP-RM Make sure that the two setup RPCs, SetSystemInfo and SetRegistry, are in the command queue before booting GSP-RM. These message are expected to already be present, because GSP-RM will process the queue early in boot but will silently ignore the error if absent. GSP-RM then continues booting, but since the registry is not set up, it fails to initialize. Fixes: 0e42ec83d46a ("gpu: nova-core: gsp: shuffle boot code a bit to keep chipset-specific parts close") Signed-off-by: Timur Tabi Reviewed-by: Alexandre Courbot Reviewed-by: John Hubbard Link: https://patch.msgid.link/20260810181243.3036669-1-ttabi@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/boot.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index e03700ee7bea..874f6e9499f0 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -44,6 +44,11 @@ pub(crate) fn boot( let gsp_fw = KBox::pin_init(GspFirmware::new(dev, chipset), GFP_KERNEL)?; + self.cmdq + .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev, chipset))?; + self.cmdq + .send_command_no_wait(bar, commands::SetRegistry::new(ctx.vgpu.state())?)?; + // Perform the chipset-specific boot sequence, and retrieve the unload bundle. let unload_bundle = hal.boot(&self, &mut ctx, &gsp_fw)?.or_else(|| { dev_warn!(dev, "The GSP won't be able to unload properly on unbind.\n"); @@ -73,11 +78,6 @@ pub(crate) fn boot( dev_dbg!(pdev, "RISC-V active? {}\n", gsp_falcon.is_riscv_active(),); - self.cmdq - .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev, chipset))?; - self.cmdq - .send_command_no_wait(bar, commands::SetRegistry::new(ctx.vgpu.state())?)?; - hal.post_boot(&self, ctx, &gsp_fw)?; // Wait until GSP is fully initialized. From d53bc149eea6213e9c4a4604df5521db7b7296ca Mon Sep 17 00:00:00 2001 From: Antonin Malzieu Ridolfi Date: Mon, 17 Aug 2026 18:51:08 +0200 Subject: [PATCH 002/110] gpu: nova-core: Extract PMC registers definitions Move PMC register definitions (NV_PMC_BOOT_0, NV_PMC_BOOT_42) and their associated implementations from the root regs.rs file into the gpu module that own them, in the new gpu/regs.rs file. This follows the same pattern established by previous commits for GSP, PDISP, PFB, and PBUS registers: register definitions move to the module that owns them, visibility changes to pub(super), and cross-module access is provided via pub(crate) helper functions. Since falcon.rs (outside the gpu module) also reads NV_PMC_BOOT_0 for its raw value, a pub(crate) helper function boot_0_raw() is added in gpu.rs to provide that access without exposing the register type directly. Suggested-by: Alexandre Courbot Suggested-by: Danilo Krummrich Signed-off-by: Antonin Malzieu Ridolfi Link: https://patch.msgid.link/20260817-b4-extract-pmc-registers-to-gpu-mod-v2-1-9dd1d599deb4@nanonej.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/falcon.rs | 2 +- drivers/gpu/nova-core/gpu.rs | 7 ++- drivers/gpu/nova-core/gpu/regs.rs | 82 +++++++++++++++++++++++++++++++ drivers/gpu/nova-core/regs.rs | 76 ---------------------------- 4 files changed, 89 insertions(+), 78 deletions(-) create mode 100644 drivers/gpu/nova-core/gpu/regs.rs diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index 65cb12d26e2b..11c1008b7d1e 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -394,7 +394,7 @@ pub(crate) fn reset(&self) -> Result { self.bar.write( WithBase::of::(), - regs::NV_PFALCON_FALCON_RM::from(self.bar.read(regs::NV_PMC_BOOT_0).into_raw()), + regs::NV_PFALCON_FALCON_RM::from(crate::gpu::boot_0_raw(self.bar)), ); Ok(()) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 42a4cd7971fa..9e4232645a7e 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -29,11 +29,11 @@ Gsp, GspBootContext, // }, - regs, vgpu::VgpuManager, // }; mod hal; +mod regs; macro_rules! define_chipset { ({ $($variant:ident = $value:expr),* $(,)* }) => @@ -414,3 +414,8 @@ pub(crate) fn new( }) } } + +/// Reads the boot0 register and returns its raw value. +pub(crate) fn boot_0_raw(bar: Bar0<'_>) -> u32 { + bar.read(regs::NV_PMC_BOOT_0).into_raw() +} diff --git a/drivers/gpu/nova-core/gpu/regs.rs b/drivers/gpu/nova-core/gpu/regs.rs new file mode 100644 index 000000000000..1c4db9625250 --- /dev/null +++ b/drivers/gpu/nova-core/gpu/regs.rs @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0 + +use kernel::{ + io::register, + prelude::*, // +}; + +use super::{ + Architecture, + Chipset, // +}; + +// PMC + +register! { + /// Basic revision information about the GPU. + pub(super) NV_PMC_BOOT_0(u32) @ 0x00000000 { + /// Lower bits of the architecture. + 28:24 architecture_0; + /// Implementation version of the architecture. + 23:20 implementation; + /// MSB of the architecture. + 8:8 architecture_1; + /// Major revision of the chip. + 7:4 major_revision; + /// Minor revision of the chip. + 3:0 minor_revision; + } + + /// Extended architecture information. + pub(super) NV_PMC_BOOT_42(u32) @ 0x00000a00 { + /// Architecture value. + 29:24 architecture ?=> Architecture; + /// Implementation version of the architecture. + 23:20 implementation; + /// Major revision of the chip. + 19:16 major_revision; + /// Minor revision of the chip. + 15:12 minor_revision; + } +} + +impl NV_PMC_BOOT_0 { + pub(super) fn is_older_than_fermi(self) -> bool { + // From https://github.com/NVIDIA/open-gpu-doc/tree/master/manuals : + const NV_PMC_BOOT_0_ARCHITECTURE_GF100: u32 = 0xc; + + // Older chips left arch1 zeroed out. That, combined with an arch0 value that is less than + // GF100, means "older than Fermi". + self.architecture_1() == 0 && self.architecture_0() < NV_PMC_BOOT_0_ARCHITECTURE_GF100 + } +} + +impl NV_PMC_BOOT_42 { + /// Combines `architecture` and `implementation` to obtain a code unique to the chipset. + pub(super) fn chipset(self) -> Result { + self.architecture() + .map(|arch| { + ((arch as u32) << Self::IMPLEMENTATION_RANGE.len()) + | u32::from(self.implementation()) + }) + .and_then(Chipset::try_from) + } + + /// Returns the raw architecture value from the register. + fn architecture_raw(self) -> u8 { + ((self.into_raw() >> Self::ARCHITECTURE_RANGE.start()) + & ((1 << Self::ARCHITECTURE_RANGE.len()) - 1)) as u8 + } +} + +impl kernel::fmt::Display for NV_PMC_BOOT_42 { + fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + write!( + f, + "boot42 = 0x{:08x} (architecture 0x{:x}, implementation 0x{:x})", + self.inner, + self.architecture_raw(), + self.implementation() + ) + } +} diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index caeef4d85874..3422b49df7a7 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -7,7 +7,6 @@ register::WithBase, Io, // }, - prelude::*, sizes::SizeConstants, time, // }; @@ -28,83 +27,8 @@ PFalconBase, PeregrineCoreSelect, // }, - gpu::{ - Architecture, - Chipset, // - }, }; -// PMC - -register! { - /// Basic revision information about the GPU. - pub(crate) NV_PMC_BOOT_0(u32) @ 0x00000000 { - /// Lower bits of the architecture. - 28:24 architecture_0; - /// Implementation version of the architecture. - 23:20 implementation; - /// MSB of the architecture. - 8:8 architecture_1; - /// Major revision of the chip. - 7:4 major_revision; - /// Minor revision of the chip. - 3:0 minor_revision; - } - - /// Extended architecture information. - pub(crate) NV_PMC_BOOT_42(u32) @ 0x00000a00 { - /// Architecture value. - 29:24 architecture ?=> Architecture; - /// Implementation version of the architecture. - 23:20 implementation; - /// Major revision of the chip. - 19:16 major_revision; - /// Minor revision of the chip. - 15:12 minor_revision; - } -} - -impl NV_PMC_BOOT_0 { - pub(crate) fn is_older_than_fermi(self) -> bool { - // From https://github.com/NVIDIA/open-gpu-doc/tree/master/manuals : - const NV_PMC_BOOT_0_ARCHITECTURE_GF100: u32 = 0xc; - - // Older chips left arch1 zeroed out. That, combined with an arch0 value that is less than - // GF100, means "older than Fermi". - self.architecture_1() == 0 && self.architecture_0() < NV_PMC_BOOT_0_ARCHITECTURE_GF100 - } -} - -impl NV_PMC_BOOT_42 { - /// Combines `architecture` and `implementation` to obtain a code unique to the chipset. - pub(crate) fn chipset(self) -> Result { - self.architecture() - .map(|arch| { - ((arch as u32) << Self::IMPLEMENTATION_RANGE.len()) - | u32::from(self.implementation()) - }) - .and_then(Chipset::try_from) - } - - /// Returns the raw architecture value from the register. - fn architecture_raw(self) -> u8 { - ((self.into_raw() >> Self::ARCHITECTURE_RANGE.start()) - & ((1 << Self::ARCHITECTURE_RANGE.len()) - 1)) as u8 - } -} - -impl kernel::fmt::Display for NV_PMC_BOOT_42 { - fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - write!( - f, - "boot42 = 0x{:08x} (architecture 0x{:x}, implementation 0x{:x})", - self.inner, - self.architecture_raw(), - self.implementation() - ) - } -} - // PBUS register! { From b705c185105762676aa6ec16cf976101df87cc35 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Thu, 23 Jul 2026 22:54:37 +0900 Subject: [PATCH 003/110] gpu: nova-core: fix incorrect naming/framing of GSP-FMC firmware The GSP-FMC firmware code lived in `firmware/fsp.rs` and described itself as running on the FSP. This is incorrect: GSP-FMC is loaded by the FSP, but runs on the GSP as the loader and verifier of GSP-RM. FSP secure-boots on its own before the driver even starts. Correct the naming and location of the GSP-FMC code by: - Renaming `firmware/fsp.rs` to `firmware/gsp_fmc.rs` (as GSP-FMC is not the FSP firmware), - Changing the prefix of some declarations in that file to `FMC` or `GspFmc` (interestingly some were already correct), - Rewording the `fsp.rs` module documentation to describe the actual boot flow. Reviewed-by: Danilo Krummrich Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260825-gsp-fmc-rename-v1-1-f82526f01d7d@nvidia.com [acourbot: drop the documentation change to be picked up by Zhi, reword commit message a bit.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/firmware.rs | 2 +- .../nova-core/firmware/{fsp.rs => gsp_fmc.rs} | 55 ++++++++++--------- drivers/gpu/nova-core/fsp.rs | 29 +++++----- 3 files changed, 47 insertions(+), 39 deletions(-) rename drivers/gpu/nova-core/firmware/{fsp.rs => gsp_fmc.rs} (71%) diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index b49613a90bf0..c16fee6e2b2a 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -23,9 +23,9 @@ }; pub(crate) mod booter; -pub(crate) mod fsp; pub(crate) mod fwsec; pub(crate) mod gsp; +pub(crate) mod gsp_fmc; pub(crate) mod riscv; pub(crate) mod tlv; diff --git a/drivers/gpu/nova-core/firmware/fsp.rs b/drivers/gpu/nova-core/firmware/gsp_fmc.rs similarity index 71% rename from drivers/gpu/nova-core/firmware/fsp.rs rename to drivers/gpu/nova-core/firmware/gsp_fmc.rs index 5462e318410a..fbb0af133f24 100644 --- a/drivers/gpu/nova-core/firmware/fsp.rs +++ b/drivers/gpu/nova-core/firmware/gsp_fmc.rs @@ -1,7 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -//! FSP is a hardware unit that runs FMC firmware. +//! GSP-FMC (First Mutable Code) is loaded by FSP into GSP to serve as the loader and verifier of +//! GSP-RM. use kernel::{ device, @@ -17,16 +18,16 @@ gpu::Chipset, // }; -/// Size of the FSP SHA-384 hash, in bytes. -const FSP_HASH_SIZE: usize = 48; -/// Maximum size of the FSP public key (RSA-3072), in bytes. +/// Size of the GSP-FMC SHA-384 hash, in bytes. +const FMC_HASH_SIZE: usize = 48; +/// Maximum size of the GSP-FMC public key (RSA-3072), in bytes. /// -/// The FMC `PKEY` tag may be shorter, so the remaining bytes are zero-padded. -const FSP_PKEY_SIZE: usize = 384; -/// Maximum size of the FSP signature (RSA-3072), in bytes. +/// The `PKEY` tag may be shorter, so the remaining bytes are zero-padded. +const FMC_PKEY_SIZE: usize = 384; +/// Maximum size of the GSP-FMC signature (RSA-3072), in bytes. /// -/// The FMC `SIGN` tag may be shorter, so the remaining bytes are zero-padded. -const FSP_SIG_SIZE: usize = 384; +/// The `SIGN` tag may be shorter, so the remaining bytes are zero-padded. +const FMC_SIG_SIZE: usize = 384; /// Structure to hold FMC signatures. /// @@ -34,23 +35,27 @@ #[derive(Debug, Clone, Copy, Zeroable)] #[repr(C)] pub(crate) struct FmcSignatures { - pub(crate) hash384: [u8; FSP_HASH_SIZE], - pub(crate) public_key: [u8; FSP_PKEY_SIZE], - pub(crate) signature: [u8; FSP_SIG_SIZE], + pub(crate) hash384: [u8; FMC_HASH_SIZE], + pub(crate) public_key: [u8; FMC_PKEY_SIZE], + pub(crate) signature: [u8; FMC_SIG_SIZE], } -pub(crate) struct FspFirmware { +pub(crate) struct GspFmcFirmware { /// FMC firmware image data pub(crate) fmc_image: Coherent<[u8]>, /// FMC firmware signatures. pub(crate) fmc_sigs: KBox, } -impl FspFirmware { +impl GspFmcFirmware { pub(crate) fn new(dev: &device::Device, chipset: Chipset) -> Result { let fw = request_tlv(dev, chipset, "fmc")?; let tlv = Tlv::new(fw.data())?; - dev_dbg!(dev, "loaded fsp firmware v{}\n", tlv.get_string(b"VERS")?); + dev_dbg!( + dev, + "loaded GSP-FMC firmware v{}\n", + tlv.get_string(b"VERS")? + ); let fmc_image_data = tlv.get_bytes(b"BLOB")?; let fmc_image = Coherent::from_slice(dev, fmc_image_data, GFP_KERNEL)?; @@ -70,34 +75,34 @@ fn extract_fmc_signatures(tlv: &Tlv<'_>, dev: &device::Device) -> Result FSP_PKEY_SIZE { + if pkey_section.len() > FMC_PKEY_SIZE { dev_err!( dev, "FMC public key section size {} > maximum {}\n", pkey_section.len(), - FSP_PKEY_SIZE + FMC_PKEY_SIZE ); return Err(EINVAL); } - if sig_section.len() > FSP_SIG_SIZE { + if sig_section.len() > FMC_SIG_SIZE { dev_err!( dev, "FMC signature section size {} > maximum {}\n", sig_section.len(), - FSP_SIG_SIZE + FMC_SIG_SIZE ); return Err(EINVAL); } @@ -106,11 +111,11 @@ fn extract_fmc_signatures(tlv: &Tlv<'_>, dev: &device::Device) -> Result().chain(|sigs| { - // PANIC: src and dst lengths are both FSP_HASH_SIZE (verified above). + // PANIC: src and dst lengths are both `FMC_HASH_SIZE` (verified above). sigs.hash384.copy_from_slice(hash_section); - // PANIC: dst is sliced to src.len(); src.len() <= FSP_PKEY_SIZE (verified above). + // PANIC: dst is sliced to src.len(); src.len() <= `FMC_PKEY_SIZE` (verified above). sigs.public_key[..pkey_section.len()].copy_from_slice(pkey_section); - // PANIC: dst is sliced to src.len(); src.len() <= FSP_SIG_SIZE (verified above). + // PANIC: dst is sliced to src.len(); src.len() <= `FMC_SIG_SIZE` (verified above). sigs.signature[..sig_section.len()].copy_from_slice(sig_section); Ok(()) }), diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index ab685fb4168f..d2be19b8bff9 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -3,9 +3,12 @@ //! FSP (Foundation Security Processor) interface for Hopper/Blackwell GPUs. //! -//! Hopper/Blackwell use a simplified firmware boot sequence: FMC, then FSP, then GSP. +//! Hopper/Blackwell use a simplified firmware boot sequence: FSP secure-boots independently before +//! the driver starts. The driver then sends FSP a Chain-of-Trust request containing the GSP-FMC +//! image. FSP authenticates the image and launches GSP-FMC on the GSP RISC-V core; GSP-FMC +//! subsequently authenticates and boots GSP-RM. +//! //! Unlike Turing/Ampere/Ada, there is no SEC2 (Security Engine 2) usage. -//! FSP handles secure boot directly using FMC firmware and Chain of Trust. use kernel::{ device, @@ -32,9 +35,9 @@ Falcon, // }, fb::FbSizes, - firmware::fsp::{ + firmware::gsp_fmc::{ FmcSignatures, - FspFirmware, // + GspFmcFirmware, // }, gpu::Chipset, gsp::{ @@ -267,7 +270,7 @@ fn frts_vidmem_offset(hal: &dyn hal::FspHal, fb_info: &FbSizes) -> Result { /// Returns an in-place initializer for [`FspCotMessage`]. fn new<'a>( fb_info: &FbSizes, - fsp_fw: &'a FspFirmware, + fmc_fw: &'a GspFmcFirmware, args: &'a FmcBootArgs<'_>, ) -> Result + 'a> { let hal = hal::fsp_hal(args.chipset).ok_or(ENOTSUPP)?; @@ -296,13 +299,13 @@ fn new<'a>( .chain(move |msg| { msg.cot.version = version; msg.cot.size = size; - msg.cot.gsp_fmc_sysmem_offset = fsp_fw.fmc_image.dma_address(); + msg.cot.gsp_fmc_sysmem_offset = fmc_fw.fmc_image.dma_address(); msg.cot.frts_vidmem_offset = frts_vidmem_offset; msg.cot.frts_vidmem_size = frts_size; // frts_sysmem_* are left at zero because this path places FRTS in vidmem. The sysmem // fields point to an FRTS buffer in sysmem instead, for systems without VRAM. msg.cot.gsp_boot_args_sysmem_offset = args.fmc_boot_params.dma_address(); - msg.cot.sigs = *fsp_fw.fmc_sigs; + msg.cot.sigs = *fmc_fw.fmc_sigs; Ok(()) })) @@ -381,12 +384,12 @@ pub(crate) fn boot_params(&self) -> &Coherent { /// FSP interface for Hopper/Blackwell GPUs. /// -/// An `Fsp` is produced by [`Fsp::wait_secure_boot`], which only returns once FSP secure boot -/// has completed. It owns the FSP falcon and the FMC firmware, which are used for the subsequent +/// An `Fsp` is produced by [`Fsp::wait_secure_boot`], which only returns once FSP secure boot has +/// completed. It owns the FSP falcon and the GSP-FMC firmware, which are used for the subsequent /// Chain of Trust boot. pub(crate) struct Fsp<'a> { falcon: Falcon<'a, FspEngine>, - fsp_fw: FspFirmware, + fmc_fw: GspFmcFirmware, } impl<'a> Fsp<'a> { @@ -422,7 +425,7 @@ fn wait_secure_boot( const FSP_SECURE_BOOT_TIMEOUT_MS: i64 = 5000; let falcon = Falcon::::new(dev, chipset, bar)?; - let fsp_fw = FspFirmware::new(dev, chipset)?; + let fmc_fw = GspFmcFirmware::new(dev, chipset)?; read_poll_timeout( || Ok(hal.fsp_boot_status(bar)), @@ -434,7 +437,7 @@ fn wait_secure_boot( dev_err!(dev, "FSP secure boot completion error: {:?}\n", e); })?; - Ok(Fsp { falcon, fsp_fw }) + Ok(Fsp { falcon, fmc_fw }) } /// Sends a message to FSP and waits for the response. @@ -540,7 +543,7 @@ pub(crate) fn boot_fmc( ) -> Result { dev_dbg!(dev, "Starting FSP boot sequence for {}\n", args.chipset); - let msg = KBox::init(FspCotMessage::new(fb_info, &self.fsp_fw, args)?, GFP_KERNEL)?; + let msg = KBox::init(FspCotMessage::new(fb_info, &self.fmc_fw, args)?, GFP_KERNEL)?; let _response_buf = self.send_sync_fsp(dev, &*msg)?; From 9e270c4fd863918883977615d7fc54c9b9d018c8 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 21:59:32 +0100 Subject: [PATCH 004/110] gpu: nova-core: fix barrier usage in CPU->GSP messaging path In the CPU->GSP messaging path, the code reads the read pointer from GSP, writes the command, advances the write pointer, and then notifies the GSP. A LOAD->STORE ordering is needed after reading the read pointer from GSP and writing the command. Control dependency exists here which provide the needed ordering, but it's best to avoid depending on it. A STORE->STORE ordering is needed after the command write and before the write pointer advance. This is currently incorrectly done after the write pointer advance (and before GSP notification), but this can cause issue if GSP is still processing ring buffer, as it may observe the write pointer advance before command write. Thus move this barrier to be before the write pointer advance. Note that barriers are not needed between write pointer advance and GSP notification, as MMIO accessors already carries the required barrier. Signed-off-by: Gary Guo Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260901-rust-barrier-v4-1-94427f445310@garyguo.net Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/gsp/cmdq.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index 6da728201281..bfd61e678802 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -27,6 +27,11 @@ ptr, sync::{ aref::ARef, + barrier::{ + dma_mb, + Full, + Write, // + }, Mutex, // }, time::Delta, @@ -413,7 +418,12 @@ fn gsp_write_ptr(&self) -> u32 { // // - The returned value is within `0..MSGQ_NUM_PAGES`. fn gsp_read_ptr(&self) -> u32 { - MsgqRxHeader::read_ptr(io_project!(self.0, .gspq.rx)) % MSGQ_NUM_PAGES + let ptr = MsgqRxHeader::read_ptr(io_project!(self.0, .gspq.rx)) % MSGQ_NUM_PAGES; + + // ORDERING: LOAD->STORE ordering needed to order `gsp_read_ptr` read before data write. + dma_mb(Full); + + ptr } // Returns the index of the memory page the CPU can read the next message from. @@ -447,12 +457,12 @@ fn cpu_write_ptr(&self) -> u32 { // Informs the GSP that it can process `elem_count` new pages from the command queue. fn advance_cpu_write_ptr(&mut self, elem_count: u32) { + // ORDERING: STORE->STORE ordering needed to order `cpu_write_ptr` write after data write. + dma_mb(Write); + let tx = io_project!(self.0, .cpuq.tx); let wptr = MsgqTxHeader::write_ptr(tx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; MsgqTxHeader::set_write_ptr(tx, wptr); - - // Ensure all command data is visible before triggering the GSP read. - fence(Ordering::SeqCst); } } From 6cb331644c441ff4101a4f8726283a6ed0d5947e Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 21:59:33 +0100 Subject: [PATCH 005/110] gpu: nova-core: fix barrier usage in GSP->CPU messaging path In the GSP->CPU messaging path, the code reads the write pointer from GSP, reads the response and advances the read pointer. A LOAD->LOAD ordering is required after the write pointer read and the data read. Add it as this is currently missing. A LOAD->STORE ordering is required after the data read and the advance of read pointer. Currently a Rust `SeqCst` barrier is used, which roughly maps to `smp_mb(Full)`; this however does not order DMA operations (notably on ARM, the generate barrier orders inner shareable and not outer shareable, which is ordered by `dma_mb`). This ordering does not need to be in between read pointer read and write, because it's for ordering between the ring buffer data and the pointer; the RMW operation does not internally need a barrier (nor it has to be atomic, as CPU pointers are updated by CPU only), so move it to before the RMW sequence for clarity. Signed-off-by: Gary Guo Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260901-rust-barrier-v4-2-94427f445310@garyguo.net Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/gsp/cmdq.rs | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index bfd61e678802..14a711307654 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -2,13 +2,7 @@ mod continuation; -use core::{ - mem, - sync::atomic::{ - fence, - Ordering, // - }, -}; +use core::mem; use kernel::{ device, @@ -30,6 +24,7 @@ barrier::{ dma_mb, Full, + Read, Write, // }, Mutex, // @@ -409,7 +404,12 @@ fn allocate_command(&mut self, size: usize, timeout: Delta) -> Result u32 { - MsgqTxHeader::write_ptr(io_project!(self.0, .gspq.tx)) % MSGQ_NUM_PAGES + let ptr = MsgqTxHeader::write_ptr(io_project!(self.0, .gspq.tx)) % MSGQ_NUM_PAGES; + + // ORDERING: LOAD->LOAD ordering needed to order `gsp_write_ptr` read before data read. + dma_mb(Read); + + ptr } // Returns the index of the memory page the GSP will read the next command from. @@ -437,12 +437,11 @@ fn cpu_read_ptr(&self) -> u32 { // Informs the GSP that it can send `elem_count` new pages into the message queue. fn advance_cpu_read_ptr(&mut self, elem_count: u32) { + // ORDERING: LOAD->STORE ordering needed to order `cpu_read_ptr` write after data read. + dma_mb(Full); + let rx = io_project!(self.0, .cpuq.rx); let rptr = MsgqRxHeader::read_ptr(rx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; - - // Ensure read pointer is properly ordered. - fence(Ordering::SeqCst); - MsgqRxHeader::set_read_ptr(rx, rptr) } From 63b2578517df761ec8aed607b29e24c0e2655410 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:25 +0100 Subject: [PATCH 006/110] rust: io: register: reimplement as proc macro The existing `register!` macro is implemented as a declarative macro. Reimplement it as proc macro instead, with no functional changes intended. The old implementation produces unhelpful diagnostics when things go wrong. For example, for code like register! { pub(crate) TESTREG(u32) { 31:0 data; } } which misses out the "@ offset" part of the specification, and the following error is produced: error: no rules expected `{` --> test.rs:42:5 | 42 | / register! { 43 | | pub(crate) TESTREG(u32) { 44 | | 31:0 data; ... | 100 | | } | |_____^ no rules expected this token in macro call which isn't very helpful. With the proc macro implementation, the following error is produced: error: expected `@` or `=>` --> tests.rs:43:33 | 43 | pub(crate) TESTREG(u32) { | ^ which is much more helpful. Apart from diagnostics, proc macro also has a benefit of not having follow-set restrictions, which makes syntax like register!(name: ty @ offset); possible; declarative macro will reject this as `@` is not in the follow-set of "ty" metavariable kind. Signed-off-by: Gary Guo Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-1-5552b1d59525@garyguo.net [ Fix typo in module-level doc comment and fix ArrayDef syntax description to match the actual macro syntax. - Danilo ] Signed-off-by: Danilo Krummrich --- MAINTAINERS | 1 + rust/kernel/io/register.rs | 224 +----------------------------- rust/macros/io/mod.rs | 3 + rust/macros/io/register.rs | 276 +++++++++++++++++++++++++++++++++++++ rust/macros/lib.rs | 9 ++ 5 files changed, 291 insertions(+), 222 deletions(-) create mode 100644 rust/macros/io/mod.rs create mode 100644 rust/macros/io/register.rs diff --git a/MAINTAINERS b/MAINTAINERS index 3a19da74d00c..d92c7d834693 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7494,6 +7494,7 @@ F: rust/kernel/io.rs F: rust/kernel/io/ F: rust/kernel/irq.rs F: rust/kernel/irq/ +F: rust/macros/io/ DEVICE RESOURCE MANAGEMENT HELPERS M: Hans de Goede diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index 03dfd2ff48c7..6a19552ffb95 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -801,227 +801,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// ``` #[macro_export] macro_rules! register { - // Entry point for the macro, allowing multiple registers to be defined in one call. - // It matches all possible register declaration patterns to dispatch them to corresponding - // `@reg` rule that defines a single register. - // - // TODO: change `alias:ident` to `alias:path` once relative registers are replaced by I/O - // projections. - ( - $( - $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) - $([ $size:expr $(, stride = $stride:expr)? ])? - $(@ $($base:ident +)? $offset:literal)? - $(=> $alias:ident $(+ $alias_offset:ident)? $([$alias_idx:expr])? )? - { $($fields:tt)* } - )* - ) => { - $( - $crate::register!( - @reg $(#[$attr])* $vis $name ($storage) $([$size $(, stride = $stride)?])? - $(@ $($base +)? $offset)? - $(=> $alias $(+ $alias_offset)? $([$alias_idx])? )? - { $($fields)* } - ); - )* - }; - - // All the rules below are private helpers. - - // Creates a register at a fixed offset of the MMIO space. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $offset:literal - { $($fields:tt)* } - ) => { - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!(@io_fixed $(#[$attr])* $vis $name); - }; - - // Creates an alias register of fixed offset register `alias` with its own fields. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path - { $($fields:tt)* } - ) => { - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!( - @io_base $name($storage) @ - <$alias as $crate::io::register::Register>::OFFSET - ); - $crate::register!(@io_fixed $(#[$attr])* $vis $name); - }; - - // Creates a register at a relative offset from a base address provider. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $base:ident + $offset:literal - { $($fields:tt)* } - ) => { - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!(@io_relative $name @ $base); - }; - - // Creates an alias register of relative offset register `alias` with its own fields. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $base:ident + $alias:ident - { $($fields:tt)* } - ) => { - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!( - @io_base $name($storage) @ <$alias as $crate::io::register::Register>::OFFSET - ); - $crate::register!(@io_relative $name @ $base); - }; - - // Creates an array of registers at a fixed offset of the MMIO space. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) - [ $size:expr, stride = $stride:expr ] @ $offset:literal { $($fields:tt)* } - ) => { - $crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride); - - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!(@io_array $name [ $size, stride = $stride ]); - }; - - // Shortcut for contiguous array of registers (stride == size of element). - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ] @ $offset:literal - { $($fields:tt)* } - ) => { - $crate::register!( - @reg $(#[$attr])* $vis $name($storage) - [ $size, stride = ::core::mem::size_of::<$storage>() ] - @ $offset { $($fields)* } - ); - }; - - // Creates an alias of register `idx` of array of registers `alias` with its own fields. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path [ $idx:expr ] - { $($fields:tt)* } - ) => { - $crate::build_assert::static_assert!( - $idx < <$alias as $crate::io::register::RegisterArray>::SIZE - ); - - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!( - @io_base $name($storage) @ - <$alias as $crate::io::register::Register>::OFFSET - + $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE - ); - $crate::register!(@io_fixed $(#[$attr])* $vis $name); - }; - - // Creates an array of registers at a relative offset from a base address provider. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) - [ $size:expr, stride = $stride:expr ] - @ $base:ident + $offset:literal { $($fields:tt)* } - ) => { - $crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride); - - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!(@io_relative_array $name [ $size, stride = $stride ] @ $base); - }; - - // Shortcut for contiguous array of relative registers (stride == size of element). - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ] - @ $base:ident + $offset:literal { $($fields:tt)* } - ) => { - $crate::register!( - @reg $(#[$attr])* $vis $name($storage) - [ $size, stride = ::core::mem::size_of::<$storage>() ] - @ $base + $offset { $($fields)* } - ); - }; - - // Creates an alias of register `idx` of relative array of registers `alias` with its own - // fields. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) - => $base:ident + $alias:ident [ $idx:expr ] { $($fields:tt)* } - ) => { - $crate::build_assert::static_assert!( - $idx < <$alias as $crate::io::register::RegisterArray>::SIZE - ); - - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!( - @io_base $name($storage) @ - <$alias as $crate::io::register::Register>::OFFSET + - $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE - ); - $crate::register!(@io_relative $name @ $base); - }; - - // Generates the bitfield for the register. - // - // `#[allow(non_camel_case_types)]` is added since register names typically use - // `SCREAMING_CASE`. - ( - @bitfield $(#[$attr:meta])* $vis:vis struct $name:ident($storage:ty) { $($fields:tt)* } - ) => { - $crate::bitfield!( - #[allow(non_camel_case_types)] - $(#[$attr])* $vis struct $name($storage) { $($fields)* } - ); - }; - - // Implementations shared by all registers types. - (@io_base $name:ident($storage:ty) @ $offset:expr) => { - impl $crate::io::register::Register for $name { - type Storage = $storage; - - const OFFSET: usize = $offset; - } - }; - - // Implementations of fixed registers. - (@io_fixed $(#[$attr:meta])* $vis:vis $name:ident) => { - impl $crate::io::register::FixedRegister for $name {} - - $(#[$attr])* - $vis const $name: $crate::io::register::FixedRegisterLoc<$name> = - $crate::io::register::FixedRegisterLoc::<$name>::new(); - }; - - // Implementations of relative registers. - (@io_relative $name:ident @ $base:ident) => { - impl $crate::io::register::WithBase for $name { - type BaseFamily = $base; - } - - impl $crate::io::register::RelativeRegister for $name {} - }; - - // Implementations of register arrays. - (@io_array $name:ident [ $size:expr, stride = $stride:expr ]) => { - impl $crate::io::register::Array for $name {} - - impl $crate::io::register::RegisterArray for $name { - const SIZE: usize = $size; - const STRIDE: usize = $stride; - } - }; - - // Implementations of relative array registers. - ( - @io_relative_array $name:ident [ $size:expr, stride = $stride:expr ] @ $base:ident - ) => { - impl $crate::io::register::WithBase for $name { - type BaseFamily = $base; - } - - impl $crate::io::register::RegisterArray for $name { - const SIZE: usize = $size; - const STRIDE: usize = $stride; - } - - impl $crate::io::register::RelativeRegisterArray for $name {} + ($($tt:tt)*) => { + $crate::macros::register!($($tt)*); }; } diff --git a/rust/macros/io/mod.rs b/rust/macros/io/mod.rs new file mode 100644 index 000000000000..87f7742f4619 --- /dev/null +++ b/rust/macros/io/mod.rs @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: GPL-2.0 + +pub(crate) mod register; diff --git a/rust/macros/io/register.rs b/rust/macros/io/register.rs new file mode 100644 index 000000000000..cc5bcad31ae8 --- /dev/null +++ b/rust/macros/io/register.rs @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Documentation and usage example of the macro can be found at `rust/kernel/io/register.rs`. + +use proc_macro2::{ + Group, + Literal, + Span, + TokenStream, // +}; +use quote::{ + quote, + quote_spanned, // +}; +use syn::{ + bracketed, + parenthesized, + parse::Parse, + spanned::Spanned, + token, + Attribute, + Expr, + Ident, + Path, + Result, + Token, + Type, + Visibility, // +}; + +mod kw { + syn::custom_keyword!(stride); +} + +/// Definition of a register array. +/// +/// Specify a size, and optionally a stride. Syntax is of form `[EXPR $(, stride = EXPR)?]`. +struct RegArrayDef { + size: Expr, + stride: Option, +} + +/// Offset of a register. +/// +/// Can be either of form +/// * `@ offset` for fixed offset +/// * `=> alias` for alias of register `alias`. +/// * `=> alias[idx]` for alias of register array `alias`'s `idx`-th element. +enum RegOffset { + /// Register is located at fixed address. + Fixed { offset: Literal }, + /// Register is an alias of a fixed register. + Alias { alias: Path }, + /// Register is an alias of an element of a register array. + ElementAlias { alias: Path, idx: Expr }, +} + +/// Definition of a single register. +struct Reg { + attrs: Vec, + vis: Visibility, + name: Ident, + storage: Type, + array: Option, + relative_base: Option, + offset: RegOffset, + bitfield_args: Group, +} + +impl Parse for Reg { + fn parse(input: syn::parse::ParseStream<'_>) -> Result { + let attrs = input.call(Attribute::parse_outer)?; + let vis = input.parse()?; + let name = input.parse()?; + let storage = { + let content; + parenthesized!(content in input); + content.parse()? + }; + let array = if input.peek(token::Bracket) { + let content; + bracketed!(content in input); + let size = content.parse()?; + let stride = if content.peek(Token![,]) { + let _: Token![,] = content.parse()?; + let _: kw::stride = content.parse()?; + let _: Token![=] = content.parse()?; + Some(content.parse()?) + } else { + None + }; + Some(RegArrayDef { size, stride }) + } else { + None + }; + + // Parse offset and the base it's relative to. + let lh = input.lookahead1(); + let mut relative_base = None; + let offset = if lh.peek(Token![@]) { + let _: Token![@] = input.parse()?; + + if input.peek(Ident) { + relative_base = Some(input.parse()?); + let _: Token![+] = input.parse()?; + } + + RegOffset::Fixed { + offset: input.parse()?, + } + } else if lh.peek(Token![=>]) { + let _: Token![=>] = input.parse()?; + let mut alias: Path = input.parse()?; + if input.peek(Token![+]) { + let _: Token![+] = input.parse()?; + relative_base = Some(alias); + alias = input.parse()?; + } + + if input.peek(token::Bracket) { + let content; + bracketed!(content in input); + RegOffset::ElementAlias { + alias, + idx: content.parse()?, + } + } else { + RegOffset::Alias { alias } + } + } else { + Err(lh.error())? + }; + + let lh = input.lookahead1(); + let bitfield_args = if lh.peek(token::Brace) { + input.parse()? + } else { + Err(lh.error())? + }; + + Ok(Self { + attrs, + vis, + name, + storage, + array, + relative_base, + offset, + bitfield_args, + }) + } +} + +pub(crate) struct RegDef { + regs: Vec, +} + +impl Parse for RegDef { + fn parse(input: syn::parse::ParseStream<'_>) -> Result { + let mut regs = Vec::new(); + while !input.is_empty() { + regs.push(input.parse()?); + } + Ok(RegDef { regs }) + } +} + +pub(crate) fn register(def: RegDef) -> Result { + let mut outputs = TokenStream::new(); + + for reg in def.regs { + let Reg { + attrs, + vis, + name, + storage, + array, + relative_base, + offset, + bitfield_args, + } = reg; + + // Use register name's span for generated code, so error messages (if any) can point to it + // instead of the entire register allocation. + let span = name.span().resolved_at(Span::mixed_site()); + + let offset = match offset { + RegOffset::Fixed { offset } => quote!(#offset), + RegOffset::Alias { alias } => { + quote_spanned!(alias.span().resolved_at(span) => + <#alias as ::kernel::io::register::Register>::OFFSET + ) + } + RegOffset::ElementAlias { alias, idx } => { + outputs.extend(quote_spanned!(idx.span().resolved_at(span) => + ::kernel::build_assert::static_assert!( + (#idx) < <#alias as ::kernel::io::register::RegisterArray>::SIZE + ); + )); + quote_spanned!(alias.span().resolved_at(span) => + <#alias as ::kernel::io::register::Register>::OFFSET + + (#idx) * <#alias as ::kernel::io::register::RegisterArray>::STRIDE + ) + } + }; + + outputs.extend(quote_spanned!(span => + ::kernel::bitfield!( + // `#[allow(non_camel_case_types)]` is added since register names typically use + // `SCREAMING_CASE`. + #[allow(non_camel_case_types)] + #(#attrs)* #vis struct #name(#storage) #bitfield_args + ); + + impl ::kernel::io::register::Register for #name { + type Storage = #storage; + + const OFFSET: usize = #offset; + } + )); + + match array { + None => match relative_base { + None => outputs.extend(quote_spanned!(span => + impl ::kernel::io::register::FixedRegister for #name {} + + #(#attrs)* #vis const #name: ::kernel::io::register::FixedRegisterLoc<#name> = + ::kernel::io::register::FixedRegisterLoc::<#name>::new(); + )), + Some(relative_base) => outputs.extend(quote_spanned!(span => + impl ::kernel::io::register::WithBase for #name { + type BaseFamily = #relative_base; + } + + impl ::kernel::io::register::RelativeRegister for #name {} + )), + }, + + Some(def) => { + let size = &def.size; + let stride = if let Some(stride) = &def.stride { + outputs.extend(quote_spanned!(stride.span().resolved_at(span) => + ::kernel::build_assert::static_assert!( + ::core::mem::size_of::<#storage>() <= #stride + ); + )); + quote!(#stride) + } else { + quote_spanned!(span => ::core::mem::size_of::<#storage>()) + }; + + outputs.extend(quote_spanned!(span => + impl ::kernel::io::register::RegisterArray for #name { + const SIZE: usize = #size; + const STRIDE: usize = #stride; + } + )); + + match relative_base { + None => outputs.extend(quote_spanned!(span => + impl ::kernel::io::register::Array for #name {} + )), + Some(relative_base) => outputs.extend(quote_spanned!(span => + impl ::kernel::io::register::WithBase for #name { + type BaseFamily = #relative_base; + } + + impl ::kernel::io::register::RelativeRegisterArray for #name {} + )), + } + } + }; + } + + Ok(outputs) +} diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index 24f96feaeb34..9b76efe1476f 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -19,6 +19,7 @@ mod fmt; mod for_lt; mod helpers; +mod io; mod kunit; mod module; mod paste; @@ -481,6 +482,14 @@ pub fn paste(input: TokenStream) -> TokenStream { .into() } +#[doc(hidden)] // Documented in `kernel` crate. +#[proc_macro] +pub fn register(input: TokenStream) -> TokenStream { + io::register::register(parse_macro_input!(input)) + .unwrap_or_else(|e| e.into_compile_error()) + .into() +} + /// Registers a KUnit test suite and its test cases using a user-space like syntax. /// /// This macro should be used on modules. If `CONFIG_KUNIT` (in `.config`) is `n`, the target module From 9dc6cedcd971d084a64a9bd11d9c822e1125b798 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sat, 29 Aug 2026 14:42:54 +0200 Subject: [PATCH 007/110] rust: debugfs: drop 'static bound from ScopedDir file creation methods Drop the T: 'static bound from ScopedDir's file creation methods (read_binary_file(), read_only_file(), etc.) to support registering debugfs files backed by types that contain non-'static references, such as dma::Coherent<'a, T>. The previous 'static bound existed because ScopedDir::create_file() took &'static FileOps, and &'static requires T: 'static for well- formedness. However, this was overly conservative: the file_operations pointer passed to the C debugfs API just needs to be 'static, not the entire FileOps. Store &'static bindings::file_operations in FileOps instead of the file_operations by value. In each trait impl, take a reference to the file_operations struct within the const block; since bindings::file_operations does not mention T, the reference is promoted to 'static regardless of T's lifetime parameters. Replace the Deref impl with an explicit fops() method that returns the stored &'static reference. Link: https://lore.kernel.org/driver-core/20260830193824.471089-2-dakr@kernel.org Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260830193824.471089-2-dakr@kernel.org [ Remove unrelated change from Self to T. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/debugfs.rs | 26 +++++++------------------- rust/kernel/debugfs/entry.rs | 4 ++-- rust/kernel/debugfs/file_ops.rs | 29 +++++++++++++---------------- 3 files changed, 22 insertions(+), 37 deletions(-) diff --git a/rust/kernel/debugfs.rs b/rust/kernel/debugfs.rs index d7b8014a6474..2beb55d444ca 100644 --- a/rust/kernel/debugfs.rs +++ b/rust/kernel/debugfs.rs @@ -538,7 +538,7 @@ pub fn dir<'dir2>(&'dir2 self, name: &CStr) -> ScopedDir<'data, 'dir2> { } } - fn create_file(&self, name: &CStr, data: &'data T, vtable: &'static FileOps) { + fn create_file(&self, name: &CStr, data: &'data T, vtable: &FileOps) { #[cfg(CONFIG_DEBUG_FS)] core::mem::forget(Entry::file(name, &self.entry, data, vtable)); } @@ -550,7 +550,7 @@ fn create_file(&self, name: &CStr, data: &'data T, vtable: &'static Fil /// This function does not produce an owning handle to the file. The created /// file is removed when the [`Scope`] that this directory belongs /// to is dropped. - pub fn read_only_file(&self, name: &CStr, data: &'data T) { + pub fn read_only_file(&self, name: &CStr, data: &'data T) { self.create_file(name, data, &T::FILE_OPS) } @@ -560,11 +560,7 @@ pub fn read_only_file(&self, name: &CStr, dat /// /// This function does not produce an owning handle to the file. The created file is removed /// when the [`Scope`] that this directory belongs to is dropped. - pub fn read_binary_file( - &self, - name: &CStr, - data: &'data T, - ) { + pub fn read_binary_file(&self, name: &CStr, data: &'data T) { self.create_file(name, data, &T::FILE_OPS) } @@ -596,11 +592,7 @@ pub fn read_callback_file(&self, name: &CStr, data: &'data T, _f: &'static /// This function does not produce an owning handle to the file. The created /// file is removed when the [`Scope`] that this directory belongs /// to is dropped. - pub fn read_write_file( - &self, - name: &CStr, - data: &'data T, - ) { + pub fn read_write_file(&self, name: &CStr, data: &'data T) { let vtable = &>::FILE_OPS; self.create_file(name, data, vtable) } @@ -612,7 +604,7 @@ pub fn read_write_file( /// /// This function does not produce an owning handle to the file. The created file is removed /// when the [`Scope`] that this directory belongs to is dropped. - pub fn read_write_binary_file( + pub fn read_write_binary_file( &self, name: &CStr, data: &'data T, @@ -655,7 +647,7 @@ pub fn read_write_callback_file( /// This function does not produce an owning handle to the file. The created /// file is removed when the [`Scope`] that this directory belongs /// to is dropped. - pub fn write_only_file(&self, name: &CStr, data: &'data T) { + pub fn write_only_file(&self, name: &CStr, data: &'data T) { let vtable = &>::FILE_OPS; self.create_file(name, data, vtable) } @@ -666,11 +658,7 @@ pub fn write_only_file(&self, name: &CStr, da /// /// This function does not produce an owning handle to the file. The created file is removed /// when the [`Scope`] that this directory belongs to is dropped. - pub fn write_binary_file( - &self, - name: &CStr, - data: &'data T, - ) { + pub fn write_binary_file(&self, name: &CStr, data: &'data T) { self.create_file(name, data, &T::FILE_OPS) } diff --git a/rust/kernel/debugfs/entry.rs b/rust/kernel/debugfs/entry.rs index 46aad64896ec..88a870d8c295 100644 --- a/rust/kernel/debugfs/entry.rs +++ b/rust/kernel/debugfs/entry.rs @@ -74,7 +74,7 @@ pub(crate) unsafe fn dynamic_file( parent.as_ptr(), core::ptr::from_ref(data) as *mut c_void, core::ptr::null(), - &**file_ops, + file_ops.fops(), ) }; @@ -127,7 +127,7 @@ pub(crate) fn file( parent.as_ptr(), core::ptr::from_ref(data) as *mut c_void, core::ptr::null(), - &**file_ops, + file_ops.fops(), ) }; diff --git a/rust/kernel/debugfs/file_ops.rs b/rust/kernel/debugfs/file_ops.rs index f15908f71c4a..47acc4851d47 100644 --- a/rust/kernel/debugfs/file_ops.rs +++ b/rust/kernel/debugfs/file_ops.rs @@ -20,9 +20,6 @@ use core::marker::PhantomData; -#[cfg(CONFIG_DEBUG_FS)] -use core::ops::Deref; - /// # Invariant /// /// `FileOps` will always contain an `operations` which is safe to use for a file backed @@ -30,7 +27,7 @@ /// into a reference. pub(super) struct FileOps { #[cfg(CONFIG_DEBUG_FS)] - operations: bindings::file_operations, + operations: &'static bindings::file_operations, #[cfg(CONFIG_DEBUG_FS)] mode: u16, _phantom: PhantomData, @@ -41,7 +38,7 @@ impl FileOps { /// /// The caller asserts that the provided `operations` is safe to use for a file whose /// inode has a pointer to `T` in its private data that is safe to convert into a reference. - const unsafe fn new(operations: bindings::file_operations, mode: u16) -> Self { + const unsafe fn new(operations: &'static bindings::file_operations, mode: u16) -> Self { Self { #[cfg(CONFIG_DEBUG_FS)] operations, @@ -65,11 +62,11 @@ pub(super) const fn adapt(&self) -> &FileOps { } #[cfg(CONFIG_DEBUG_FS)] -impl Deref for FileOps { - type Target = bindings::file_operations; - - fn deref(&self) -> &Self::Target { - &self.operations +impl FileOps { + /// Returns a `'static` reference to the inner `file_operations`. + #[inline] + pub(crate) fn fops(&self) -> &'static bindings::file_operations { + self.operations } } @@ -130,7 +127,7 @@ pub(crate) trait ReadFile { impl ReadFile for T { const FILE_OPS: FileOps = { - let operations = bindings::file_operations { + let operations = &bindings::file_operations { read: Some(bindings::seq_read), llseek: Some(bindings::seq_lseek), release: Some(bindings::single_release), @@ -181,7 +178,7 @@ pub(crate) trait ReadWriteFile { impl ReadWriteFile for T { const FILE_OPS: FileOps = { - let operations = bindings::file_operations { + let operations = &bindings::file_operations { open: Some(writer_open::), read: Some(bindings::seq_read), write: Some(write::), @@ -238,7 +235,7 @@ pub(crate) trait WriteFile { impl WriteFile for T { const FILE_OPS: FileOps = { - let operations = bindings::file_operations { + let operations = &bindings::file_operations { open: Some(write_only_open), write: Some(write_only_write::), llseek: Some(bindings::noop_llseek), @@ -290,7 +287,7 @@ pub(crate) trait BinaryReadFile { impl BinaryReadFile for T { const FILE_OPS: FileOps = { - let operations = bindings::file_operations { + let operations = &bindings::file_operations { read: Some(blob_read::), llseek: Some(bindings::default_llseek), open: Some(bindings::simple_open), @@ -344,7 +341,7 @@ pub(crate) trait BinaryWriteFile { impl BinaryWriteFile for T { const FILE_OPS: FileOps = { - let operations = bindings::file_operations { + let operations = &bindings::file_operations { write: Some(blob_write::), llseek: Some(bindings::default_llseek), open: Some(bindings::simple_open), @@ -368,7 +365,7 @@ pub(crate) trait BinaryReadWriteFile { impl BinaryReadWriteFile for T { const FILE_OPS: FileOps = { - let operations = bindings::file_operations { + let operations = &bindings::file_operations { read: Some(blob_read::), write: Some(blob_write::), llseek: Some(bindings::default_llseek), From f6b56da8e1b8217e5516290ef9b09728a8f96a10 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sat, 29 Aug 2026 14:54:14 +0200 Subject: [PATCH 008/110] rust: dma: tie CoherentHandle to the device's bound lifetime Add a lifetime parameter to CoherentHandle that ties the DMA allocation to the device's bound scope, ensuring it is freed before the device is unbound. DMA allocations carry device resources (e.g. IOMMU mappings) that must not outlive the device's bound lifetime. Without a lifetime parameter, there was no compile-time enforcement that a CoherentHandle is dropped before the device is unbound. Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260830193824.471089-3-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/fb.rs | 2 +- rust/kernel/dma.rs | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 1576399389b1..9ef232a73dee 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -49,7 +49,7 @@ pub(crate) struct SysmemFlush<'sys> { device: &'sys device::Device, bar: Bar0<'sys>, /// Keep the page alive as long as we need it. - page: CoherentHandle, + page: CoherentHandle<'sys>, } impl<'sys> SysmemFlush<'sys> { diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 2ce09f8e90c6..79f453e9ec0b 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -996,15 +996,15 @@ fn write_to_slice( /// - `size` is the allocation size in bytes as passed to `dma_alloc_attrs`. /// - `dma_attrs` contains the attributes used for the allocation, always including /// `DMA_ATTR_NO_KERNEL_MAPPING`. -pub struct CoherentHandle { - dev: ARef, +pub struct CoherentHandle<'a> { + dev: &'a device::Device, dma_addr: DmaAddress, cpu_handle: NonNull, size: usize, dma_attrs: Attrs, } -impl CoherentHandle { +impl<'a> CoherentHandle<'a> { /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping. /// /// Additional DMA attributes may be passed via `dma_attrs`; `DMA_ATTR_NO_KERNEL_MAPPING` is @@ -1012,7 +1012,7 @@ impl CoherentHandle { /// /// Returns `EINVAL` if `size` is zero, `ENOMEM` if the allocation fails. pub fn alloc_with_attrs( - dev: &device::Device, + dev: &'a device::Device, size: usize, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, @@ -1038,9 +1038,9 @@ pub fn alloc_with_attrs( // INVARIANT: `cpu_handle` is the opaque handle from a successful `dma_alloc_attrs` call // with `DMA_ATTR_NO_KERNEL_MAPPING`, `dma_addr` is the corresponding DMA address, - // and we hold a refcounted reference to the device. + // and `dev` is a valid reference to a bound device that outlives this allocation. Ok(Self { - dev: dev.into(), + dev, dma_addr, cpu_handle, size, @@ -1051,7 +1051,7 @@ pub fn alloc_with_attrs( /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping. #[inline] pub fn alloc( - dev: &device::Device, + dev: &'a device::Device, size: usize, gfp_flags: kernel::alloc::Flags, ) -> Result { @@ -1073,7 +1073,7 @@ pub fn size(&self) -> usize { } } -impl Drop for CoherentHandle { +impl Drop for CoherentHandle<'_> { fn drop(&mut self) { // SAFETY: All values are valid by the type invariants on `CoherentHandle`. // `cpu_handle` is the opaque handle from `dma_alloc_attrs` and is passed back unchanged. @@ -1091,12 +1091,12 @@ fn drop(&mut self) { // SAFETY: `CoherentHandle` only holds a device reference, a DMA address, an opaque CPU handle, // and a size. None of these are tied to a specific thread. -unsafe impl Send for CoherentHandle {} +unsafe impl Send for CoherentHandle<'_> {} // SAFETY: `CoherentHandle` provides no CPU access to the underlying allocation. The only // operations on `&CoherentHandle` are reading the DMA address and size, both of which are // plain `Copy` values. -unsafe impl Sync for CoherentHandle {} +unsafe impl Sync for CoherentHandle<'_> {} /// View type for `Coherent`. /// From c3d4fd0e9d27651d2daf223fa72128633a082b05 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sat, 29 Aug 2026 15:15:36 +0200 Subject: [PATCH 009/110] samples: rust_dma: separate driver type from driver data Split DmaSampleDriver into a driver type and a separate DmaSampleData struct for the driver's bus device private data, using DmaSampleData<'bound> as the Driver::Data<'bound> associated type. Store a &'bound pci::Device reference instead of an ARef, tying the data to the device's bound scope. This prepares for adding a lifetime parameter to dma::Coherent, which requires the data type to carry a lifetime. Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260830193824.471089-4-dakr@kernel.org Signed-off-by: Danilo Krummrich --- samples/rust/rust_dma.rs | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/samples/rust/rust_dma.rs b/samples/rust/rust_dma.rs index bd60034ded23..0fac9d4ae566 100644 --- a/samples/rust/rust_dma.rs +++ b/samples/rust/rust_dma.rs @@ -5,7 +5,10 @@ //! To make this driver probe, QEMU must be run with `-device pci-testdev`. use kernel::{ - device::Core, + device::{ + Bound, + Core, // + }, dma::{ Coherent, DataDirection, @@ -23,13 +26,14 @@ scatterlist::{ Owned, SGTable, // - }, - sync::aref::ARef, // + }, // }; +struct DmaSampleDriver; + #[pin_data(PinnedDrop)] -struct DmaSampleDriver { - pdev: ARef, +struct DmaSampleData<'bound> { + pdev: &'bound pci::Device, ca: Coherent<[MyStruct]>, #[pin] sgt: SGTable>>, @@ -67,13 +71,13 @@ unsafe impl kernel::transmute::FromBytes for MyStruct {} impl pci::Driver for DmaSampleDriver { type IdInfo = (); - type Data<'bound> = Self; + type Data<'bound> = DmaSampleData<'bound>; const ID_TABLE: pci::IdTable = &PCI_TABLE; fn probe<'bound>( pdev: &'bound pci::Device>, _info: Option<&'bound Self::IdInfo>, - ) -> impl PinInit + 'bound { + ) -> impl PinInit, Error> + 'bound { pin_init::pin_init_scope(move || { dev_info!(pdev, "Probe DMA test driver.\n"); @@ -94,8 +98,8 @@ fn probe<'bound>( let sgt = SGTable::new(pdev.as_ref(), pages, DataDirection::ToDevice, GFP_KERNEL); - Ok(try_pin_init!(Self { - pdev: pdev.into(), + Ok(try_pin_init!(Self::Data { + pdev, ca, sgt <- sgt, })) @@ -103,7 +107,7 @@ fn probe<'bound>( } } -impl DmaSampleDriver { +impl DmaSampleData<'_> { fn check_dma(&self) { for (i, value) in TEST_VALUES.into_iter().enumerate() { let val0 = io_read!(self.ca, [panic: i].h); @@ -116,7 +120,7 @@ fn check_dma(&self) { } #[pinned_drop] -impl PinnedDrop for DmaSampleDriver { +impl PinnedDrop for DmaSampleData<'_> { fn drop(self: Pin<&mut Self>) { dev_info!(self.pdev, "Unload DMA test driver.\n"); From bed87375f90f44cd2f9fe804a469274127d38593 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 30 Aug 2026 17:40:34 +0200 Subject: [PATCH 010/110] rust: dma: tie Coherent and CoherentBox to the device's bound lifetime Add a lifetime parameter to Coherent and CoherentBox that ties the DMA allocation to the device's bound scope, ensuring it is freed before the device is unbound. DMA allocations carry device resources (e.g. IOMMU mappings) that must not outlive the device's bound lifetime. Without a lifetime parameter, there was no compile-time enforcement that a Coherent or CoherentBox is dropped before the device is unbound. Propagate the new lifetime parameter through all users. Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260830193824.471089-5-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/falcon.rs | 2 +- drivers/gpu/nova-core/fb.rs | 2 +- drivers/gpu/nova-core/firmware/booter.rs | 2 +- drivers/gpu/nova-core/firmware/fsp.rs | 8 +- .../nova-core/firmware/fwsec/bootloader.rs | 12 +- drivers/gpu/nova-core/firmware/gsp.rs | 14 +- drivers/gpu/nova-core/firmware/riscv.rs | 8 +- drivers/gpu/nova-core/fsp.rs | 20 +-- drivers/gpu/nova-core/gpu.rs | 4 +- drivers/gpu/nova-core/gsp.rs | 30 ++--- drivers/gpu/nova-core/gsp/boot.rs | 10 +- drivers/gpu/nova-core/gsp/cmdq.rs | 35 +++-- drivers/gpu/nova-core/gsp/commands.rs | 2 +- drivers/gpu/nova-core/gsp/fw.rs | 12 +- drivers/gpu/nova-core/gsp/hal.rs | 14 +- drivers/gpu/nova-core/gsp/hal/gh100.rs | 16 +-- drivers/gpu/nova-core/gsp/hal/tu102.rs | 34 ++--- drivers/gpu/nova-core/gsp/sequencer.rs | 6 +- rust/kernel/dma.rs | 121 +++++++++--------- rust/kernel/uaccess.rs | 4 +- samples/rust/rust_dma.rs | 4 +- 21 files changed, 176 insertions(+), 184 deletions(-) diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index 65cb12d26e2b..15eba039cb69 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -506,7 +506,7 @@ pub(crate) fn pio_load + FalconPioLoadable>( /// `sec` is set if the loaded firmware is expected to run in secure mode. fn dma_wr( &self, - dma_obj: &Coherent<[u8]>, + dma_obj: &Coherent<'_, [u8]>, target_mem: FalconMem, load_offsets: FalconDmaLoadTarget, ) -> Result { diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 9ef232a73dee..b3a6ab8b57a6 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -177,7 +177,7 @@ impl FbRanges { pub(crate) fn new( chipset: Chipset, bar: Bar0<'_>, - gsp_fw: &GspFirmware, + gsp_fw: &GspFirmware<'_>, vgpu_state: VgpuState, ) -> Result { let hal = hal::fb_hal(chipset); diff --git a/drivers/gpu/nova-core/firmware/booter.rs b/drivers/gpu/nova-core/firmware/booter.rs index dc071edba331..aa4458bb3312 100644 --- a/drivers/gpu/nova-core/firmware/booter.rs +++ b/drivers/gpu/nova-core/firmware/booter.rs @@ -186,7 +186,7 @@ pub(crate) fn run( &self, dev: &device::Device, sec2_falcon: &Falcon<'_, Sec2>, - wpr_meta: &Coherent, + wpr_meta: &Coherent<'_, T>, ) -> Result { sec2_falcon.reset()?; sec2_falcon.load(self)?; diff --git a/drivers/gpu/nova-core/firmware/fsp.rs b/drivers/gpu/nova-core/firmware/fsp.rs index 5462e318410a..d47b1d2a1030 100644 --- a/drivers/gpu/nova-core/firmware/fsp.rs +++ b/drivers/gpu/nova-core/firmware/fsp.rs @@ -39,15 +39,15 @@ pub(crate) struct FmcSignatures { pub(crate) signature: [u8; FSP_SIG_SIZE], } -pub(crate) struct FspFirmware { +pub(crate) struct FspFirmware<'a> { /// FMC firmware image data - pub(crate) fmc_image: Coherent<[u8]>, + pub(crate) fmc_image: Coherent<'a, [u8]>, /// FMC firmware signatures. pub(crate) fmc_sigs: KBox, } -impl FspFirmware { - pub(crate) fn new(dev: &device::Device, chipset: Chipset) -> Result { +impl<'a> FspFirmware<'a> { + pub(crate) fn new(dev: &'a device::Device, chipset: Chipset) -> Result { let fw = request_tlv(dev, chipset, "fmc")?; let tlv = Tlv::new(fw.data())?; dev_dbg!(dev, "loaded fsp firmware v{}\n", tlv.get_string(b"VERS")?); diff --git a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs index ec4d92317a93..06a7936a8c5e 100644 --- a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs +++ b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs @@ -98,9 +98,9 @@ unsafe impl AsBytes for BootloaderDmemDescV2 {} /// Wrapper for [`FwsecFirmware`] that includes the bootloader performing the actual load /// operation. -pub(crate) struct FwsecFirmwareWithBl { +pub(crate) struct FwsecFirmwareWithBl<'a> { /// DMA object the bootloader will copy the firmware from. - _firmware_dma: Coherent<[u8]>, + _firmware_dma: Coherent<'a, [u8]>, /// Code of the bootloader to be loaded into non-secure IMEM. ucode: KVec, /// Descriptor to be loaded into DMEM for the bootloader to read. @@ -113,12 +113,12 @@ pub(crate) struct FwsecFirmwareWithBl { start_tag: u16, } -impl FwsecFirmwareWithBl { +impl<'a> FwsecFirmwareWithBl<'a> { /// Loads the bootloader firmware for `dev` and `chipset`, and wrap `firmware` so it can be /// loaded using it. pub(crate) fn new( firmware: FwsecFirmware, - dev: &Device, + dev: &'a Device, chipset: Chipset, ) -> Result { let fw = request_tlv(dev, chipset, "gen_bootloader")?; @@ -272,7 +272,7 @@ pub(crate) fn run( } } -impl FalconFirmware for FwsecFirmwareWithBl { +impl FalconFirmware for FwsecFirmwareWithBl<'_> { type Target = Gsp; fn brom_params(&self) -> FalconBromParams { @@ -286,7 +286,7 @@ fn boot_addr(&self) -> u32 { } } -impl FalconPioLoadable for FwsecFirmwareWithBl { +impl FalconPioLoadable for FwsecFirmwareWithBl<'_> { fn imem_sec_load_params(&self) -> Option> { None } diff --git a/drivers/gpu/nova-core/firmware/gsp.rs b/drivers/gpu/nova-core/firmware/gsp.rs index e8f9491e84cc..22d1f9329c9f 100644 --- a/drivers/gpu/nova-core/firmware/gsp.rs +++ b/drivers/gpu/nova-core/firmware/gsp.rs @@ -44,7 +44,7 @@ /// Each page is 4KB, each entry is 8 bytes (64-bit DMA address). /// Also known as "Radix3" firmware. #[pin_data] -pub(crate) struct GspFirmware { +pub(crate) struct GspFirmware<'a> { /// The GSP firmware inside a [`VVec`], device-mapped via a SG table. #[pin] fw: SGTable>>, @@ -55,19 +55,19 @@ pub(crate) struct GspFirmware { #[pin] level1: SGTable>>, /// Level 0 page table (single 4KB page) with one entry: DMA address of first level 1 page. - level0: Coherent<[u64]>, + level0: Coherent<'a, [u64]>, /// Size in bytes of the firmware contained in [`Self::fw`]. pub(crate) size: usize, /// Device-mapped GSP signatures matching the GPU's [`Chipset`]. - pub(crate) signatures: Coherent<[u8]>, + pub(crate) signatures: Coherent<'a, [u8]>, /// GSP bootloader, verifies the GSP firmware before loading and running it. - pub(crate) bootloader: RiscvFirmware, + pub(crate) bootloader: RiscvFirmware<'a>, } -impl GspFirmware { +impl<'a> GspFirmware<'a> { /// Loads the GSP firmware binaries, map them into `dev`'s address-space, and creates the page /// tables expected by the GSP bootloader to load it. - pub(crate) fn new<'a>( + pub(crate) fn new( dev: &'a device::Device, chipset: Chipset, ) -> impl PinInit + 'a { @@ -120,7 +120,7 @@ pub(crate) fn new<'a>( // Create level 0 page table data and fill its first entry with the level 1 // table. - let mut level0 = CoherentBox::<[u64]>::zeroed_slice( + let mut level0 = CoherentBox::<'_, [u64]>::zeroed_slice( dev, GSP_PAGE_SIZE / size_of::(), GFP_KERNEL diff --git a/drivers/gpu/nova-core/firmware/riscv.rs b/drivers/gpu/nova-core/firmware/riscv.rs index 1403f05a7305..f05cfb1c65da 100644 --- a/drivers/gpu/nova-core/firmware/riscv.rs +++ b/drivers/gpu/nova-core/firmware/riscv.rs @@ -13,7 +13,7 @@ use crate::firmware::tlv::Tlv; /// A parsed firmware for a RISC-V core, ready to be loaded and run. -pub(crate) struct RiscvFirmware { +pub(crate) struct RiscvFirmware<'a> { /// Offset at which the code starts in the firmware image. pub(crate) code_offset: u32, /// Offset at which the data starts in the firmware image. @@ -23,12 +23,12 @@ pub(crate) struct RiscvFirmware { /// Application version. pub(crate) app_version: u32, /// Device-mapped firmware image. - pub(crate) ucode: Coherent<[u8]>, + pub(crate) ucode: Coherent<'a, [u8]>, } -impl RiscvFirmware { +impl<'a> RiscvFirmware<'a> { /// Parses the RISC-V firmware image contained in `fw`. - pub(crate) fn new(dev: &device::Device, fw: &Firmware) -> Result { + pub(crate) fn new(dev: &'a device::Device, fw: &Firmware) -> Result { let tlv = Tlv::new(fw.data())?; dev_dbg!( dev, diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index ab685fb4168f..961e96fe9484 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -267,7 +267,7 @@ fn frts_vidmem_offset(hal: &dyn hal::FspHal, fb_info: &FbSizes) -> Result { /// Returns an in-place initializer for [`FspCotMessage`]. fn new<'a>( fb_info: &FbSizes, - fsp_fw: &'a FspFirmware, + fsp_fw: &'a FspFirmware<'_>, args: &'a FmcBootArgs<'_>, ) -> Result + 'a> { let hal = hal::fsp_hal(args.chipset).ok_or(ENOTSUPP)?; @@ -345,28 +345,28 @@ impl MessageToFsp for FspPrcMessage { /// Bundled arguments for FMC boot via FSP Chain of Trust. pub(crate) struct FmcBootArgs<'a> { chipset: Chipset, - fmc_boot_params: Coherent, + fmc_boot_params: Coherent<'a, GspFmcBootParams>, resume: bool, // Additional dependencies required to be kept alive for FMC boot. - _wpr_meta: Coherent, - _libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, + _wpr_meta: Coherent<'a, GspFwWprMeta>, + _libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>, } impl<'a> FmcBootArgs<'a> { /// Builds FMC boot arguments, allocating the DMA-coherent boot parameter /// structure that FSP will read. pub(crate) fn new( - dev: &device::Device, + dev: &'a device::Device, chipset: Chipset, - wpr_meta: Coherent, - libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, + wpr_meta: Coherent<'a, GspFwWprMeta>, + libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>, resume: bool, ) -> Result { let init = GspFmcBootParams::new(wpr_meta.dma_address(), libos.dma_address()); Ok(Self { chipset, - fmc_boot_params: Coherent::::init(dev, GFP_KERNEL, init)?, + fmc_boot_params: Coherent::init(dev, GFP_KERNEL, init)?, resume, _wpr_meta: wpr_meta, _libos: libos, @@ -374,7 +374,7 @@ pub(crate) fn new( } /// Returns the FMC boot parameters allocation. - pub(crate) fn boot_params(&self) -> &Coherent { + pub(crate) fn boot_params(&self) -> &Coherent<'_, GspFmcBootParams> { &self.fmc_boot_params } } @@ -386,7 +386,7 @@ pub(crate) fn boot_params(&self) -> &Coherent { /// Chain of Trust boot. pub(crate) struct Fsp<'a> { falcon: Falcon<'a, FspEngine>, - fsp_fw: FspFirmware, + fsp_fw: FspFirmware<'a>, } impl<'a> Fsp<'a> { diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index fd1414004dd0..6de75d16488d 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -272,9 +272,9 @@ struct GspResources<'gpu> { vgpu: VgpuManager, /// GSP runtime data. #[pin] - gsp: Gsp, + gsp: Gsp<'gpu>, /// GSP unload firmware bundle, if any. - unload_bundle: Option, + unload_bundle: Option>, } /// Structure holding the resources required to operate the GPU. diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index 13f361406a6c..25ea43f1cbe9 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -115,11 +115,11 @@ fn init(view: CoherentView<'_, Self>, start: DmaAddress) -> Result<()> { /// then pp points to index into the buffer where the next logging entry will /// be written. Therefore, the logging data is valid if: /// 1 <= pp < sizeof(buffer)/sizeof(u64) -struct LogBuffer(Coherent<[u8; LOG_BUFFER_SIZE]>); +struct LogBuffer<'a>(Coherent<'a, [u8; LOG_BUFFER_SIZE]>); -impl LogBuffer { +impl<'a> LogBuffer<'a> { /// Creates a new `LogBuffer` mapped on `dev`. - fn new(dev: &device::Device) -> Result { + fn new(dev: &'a device::Device) -> Result { let obj = Self(Coherent::zeroed(dev, GFP_KERNEL)?); let start_addr = obj.0.dma_address(); @@ -135,33 +135,33 @@ fn new(dev: &device::Device) -> Result { } } -struct LogBuffers { +struct LogBuffers<'a> { /// Init log buffer. - loginit: LogBuffer, + loginit: LogBuffer<'a>, /// Interrupts log buffer. - logintr: LogBuffer, + logintr: LogBuffer<'a>, /// RM log buffer. - logrm: LogBuffer, + logrm: LogBuffer<'a>, } /// GSP runtime data. #[pin_data] -pub(crate) struct Gsp { +pub(crate) struct Gsp<'gsp> { /// Libos arguments. - pub(crate) libos: Coherent<[LibosMemoryRegionInitArgument]>, + pub(crate) libos: Coherent<'gsp, [LibosMemoryRegionInitArgument]>, /// Log buffers, optionally exposed via debugfs. #[pin] - logs: debugfs::Scope, + logs: debugfs::Scope>, /// Command queue. #[pin] - pub(crate) cmdq: Cmdq, + pub(crate) cmdq: Cmdq<'gsp>, /// RM arguments. - rmargs: Coherent, + rmargs: Coherent<'gsp, GspArgumentsPadded>, } -impl Gsp { +impl<'gsp> Gsp<'gsp> { // Creates an in-place initializer for a `Gsp` manager for `pdev`. - pub(crate) fn new(pdev: &pci::Device) -> impl PinInit + '_ { + pub(crate) fn new(pdev: &'gsp pci::Device) -> impl PinInit + 'gsp { pin_init::pin_init_scope(move || { let dev = pdev.as_ref(); @@ -223,4 +223,4 @@ pub(crate) fn get_static_info(&self, bar: Bar0<'_>) -> Result); +pub(crate) struct UnloadBundle<'a>(KBox); diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index e03700ee7bea..e32c9e1f33ab 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -22,7 +22,7 @@ }, }; -impl super::Gsp { +impl<'gsp> super::Gsp<'gsp> { /// Attempt to boot the GSP. /// /// This is a GPU-dependent and complex procedure that involves loading firmware files from @@ -33,8 +33,8 @@ impl super::Gsp { /// [`Self::unload`]) returned. pub(crate) fn boot( self: Pin<&mut Self>, - mut ctx: super::GspBootContext<'_, '_>, - ) -> Result> { + mut ctx: super::GspBootContext<'_, 'gsp>, + ) -> Result>> { let pdev = ctx.pdev; let bar = ctx.bar; let chipset = ctx.chipset; @@ -88,7 +88,7 @@ pub(crate) fn boot( /// Shut down the GSP and wait until it is offline. fn shutdown_gsp( - cmdq: &Cmdq, + cmdq: &Cmdq<'_>, bar: Bar0<'_>, gsp_falcon: &Falcon<'_, Gsp>, mode: commands::PowerStateLevel, @@ -113,7 +113,7 @@ fn shutdown_gsp( pub(crate) fn unload( &self, mut ctx: super::GspBootContext<'_, '_>, - unload_bundle: Option, + unload_bundle: Option>, ) -> Result { let dev = ctx.dev(); diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index 6da728201281..fe087ce315b0 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -25,10 +25,7 @@ new_mutex, prelude::*, ptr, - sync::{ - aref::ARef, - Mutex, // - }, + sync::Mutex, time::Delta, transmute::{ AsBytes, @@ -230,19 +227,19 @@ unsafe impl FromBytes for GspMem {} /// pointer and the GSP read pointer. This region is returned by [`Self::driver_write_area`]. /// * The driver owns (i.e. can read from) the part of the GSP message queue between the CPU read /// pointer and the GSP write pointer. This region is returned by [`Self::driver_read_area`]. -struct DmaGspMem(Coherent); +struct DmaGspMem<'a>(Coherent<'a, GspMem>); -impl DmaGspMem { +impl<'a> DmaGspMem<'a> { /// Allocate a new instance and map it for `dev`. - fn new(dev: &device::Device) -> Result { + fn new(dev: &'a device::Device) -> Result { const MSGQ_SIZE: u32 = num::usize_into_u32::<{ size_of::() }>(); const RX_HDR_OFF: u32 = num::usize_into_u32::<{ mem::offset_of!(Msgq, rx) }>(); - let mut gsp_mem = CoherentBox::::zeroed(dev, GFP_KERNEL)?; + let mut gsp_mem = CoherentBox::<'_, GspMem>::zeroed(dev, GFP_KERNEL)?; gsp_mem.cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES); gsp_mem.cpuq.rx = MsgqRxHeader::new(); - let gsp_mem: Coherent<_> = gsp_mem.into(); + let gsp_mem: Coherent<'_, _> = gsp_mem.into(); PteArray::init(io_project!(gsp_mem, .ptes), gsp_mem.dma_address())?; Ok(Self(gsp_mem)) @@ -483,15 +480,15 @@ struct GspMessage<'a> { /// Provides the ability to send commands and receive messages from the GSP using a shared memory /// area. #[pin_data] -pub(crate) struct Cmdq { +pub(crate) struct Cmdq<'cmdq> { /// Inner mutex-protected state. #[pin] - inner: Mutex, + inner: Mutex>, /// DMA address of the command queue's shared memory region. pub(super) dma_addr: DmaAddress, } -impl Cmdq { +impl<'cmdq> Cmdq<'cmdq> { /// Offset of the data after the PTEs. const POST_PTE_OFFSET: usize = core::mem::offset_of!(GspMem, cpuq); @@ -512,14 +509,16 @@ impl Cmdq { pub(super) const RECEIVE_TIMEOUT: Delta = Delta::from_secs(5); /// Creates a new command queue for `dev`. - pub(crate) fn new(dev: &device::Device) -> impl PinInit + '_ { + pub(crate) fn new( + dev: &'cmdq device::Device, + ) -> impl PinInit + 'cmdq { pin_init_scope(move || { let gsp_mem = DmaGspMem::new(dev)?; Ok(try_pin_init!(Self { dma_addr: gsp_mem.0.dma_address(), inner <- new_mutex!(CmdqInner { - dev: dev.into(), + dev, gsp_mem, seq: 0, }), @@ -610,16 +609,16 @@ pub(crate) fn receive_msg(&self, timeout: Delta) -> Result } /// Inner mutex protected state of [`Cmdq`]. -struct CmdqInner { +struct CmdqInner<'a> { /// Device this command queue belongs to. - dev: ARef, + dev: &'a device::Device, /// Current command sequence number. seq: u32, /// Memory area shared with the GSP for communicating commands and messages. - gsp_mem: DmaGspMem, + gsp_mem: DmaGspMem<'a>, } -impl CmdqInner { +impl CmdqInner<'_> { /// Timeout for waiting for space on the command queue. const ALLOCATE_TIMEOUT: Delta = Delta::from_secs(1); diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index ffc25fd8c47b..69d7d41c1791 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -187,7 +187,7 @@ fn read( } /// Waits for GSP initialization to complete. -pub(crate) fn wait_gsp_init_done(cmdq: &Cmdq) -> Result { +pub(crate) fn wait_gsp_init_done(cmdq: &Cmdq<'_>) -> Result { loop { match cmdq.receive_msg::(Cmdq::RECEIVE_TIMEOUT) { Ok(_) => break Ok(()), diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 05f54fee6186..8778c4bf79c0 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -179,7 +179,7 @@ impl GspFwWprMeta { /// Returns an initializer for a `GspFwWprMeta` suitable for booting `gsp_firmware` using the /// framebuffer ranges `ranges`. pub(crate) fn from_ranges<'a>( - gsp_firmware: &'a GspFirmware, + gsp_firmware: &'a GspFirmware<'_>, ranges: &'a FbRanges, ) -> impl Init + 'a { let init_inner = init!(bindings::GspFwWprMeta { @@ -231,7 +231,7 @@ pub(crate) fn from_ranges<'a>( /// /// The region offsets are left at zero: the ACR ucode computes them when it sets up WPR2. pub(crate) fn from_sizes<'a>( - gsp_firmware: &'a GspFirmware, + gsp_firmware: &'a GspFirmware<'_>, sizes: &'a FbSizes, ) -> impl Init + 'a { /// VGA workspace size to reserve at the end of the framebuffer, in bytes. @@ -665,7 +665,7 @@ unsafe impl FromBytes for LibosMemoryRegionInitArgument {} impl LibosMemoryRegionInitArgument { pub(crate) fn new<'a, A: AsBytes + FromBytes + KnownSize + ?Sized>( name: &'static str, - obj: &'a Coherent, + obj: &'a Coherent<'_, A>, ) -> impl Init + 'a { /// Generates the `ID8` identifier required for some GSP objects. fn id8(name: &str) -> u64 { @@ -897,7 +897,7 @@ pub(crate) struct GspArgumentsCached { impl GspArgumentsCached { /// Creates the arguments for starting the GSP up using `cmdq` as its command queue. - pub(crate) fn new(cmdq: &Cmdq) -> impl Init + '_ { + pub(crate) fn new<'a, 'b>(cmdq: &'a Cmdq<'b>) -> impl Init + use<'a, 'b> { let init_inner = init!(bindings::GSP_ARGUMENTS_CACHED { messageQueueInitArguments <- MessageQueueInitArguments::new(cmdq), bDmemStack: 1, @@ -924,7 +924,7 @@ pub(crate) struct GspArgumentsPadded { } impl GspArgumentsPadded { - pub(crate) fn new(cmdq: &Cmdq) -> impl Init + '_ { + pub(crate) fn new<'a, 'b>(cmdq: &'a Cmdq<'b>) -> impl Init + use<'a, 'b> { init!(GspArgumentsPadded { inner <- GspArgumentsCached::new(cmdq), ..Zeroable::init_zeroed() @@ -944,7 +944,7 @@ unsafe impl FromBytes for GspArgumentsPadded {} impl MessageQueueInitArguments { /// Creates a new init arguments structure for `cmdq`. - fn new(cmdq: &Cmdq) -> impl Init + '_ { + fn new<'a, 'b>(cmdq: &'a Cmdq<'b>) -> impl Init + use<'a, 'b> { init!(MessageQueueInitArguments { sharedMemPhysAddr: cmdq.dma_addr, pageTableEntryCount: num::usize_into_u32::<{ Cmdq::NUM_PTES }>(), diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs index 5850fa0fe0e9..d8329f6fcc65 100644 --- a/drivers/gpu/nova-core/gsp/hal.rs +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -35,12 +35,12 @@ pub(super) trait GspHal: Send { /// /// Upon success, returns the [`crate::gsp::UnloadBundle`] to use with [`Gsp::unload`], if one /// could be created. - fn boot( + fn boot<'gpu>( &self, - gsp: &Gsp, - ctx: &mut GspBootContext<'_, '_>, - gsp_fw: &GspFirmware, - ) -> Result>; + gsp: &Gsp<'gpu>, + ctx: &mut GspBootContext<'_, 'gpu>, + gsp_fw: &GspFirmware<'gpu>, + ) -> Result>>; /// Performs HAL-specific post-GSP boot tasks. /// @@ -48,9 +48,9 @@ fn boot( /// after the initialization commands have been pushed onto its queue. fn post_boot( &self, - _gsp: &Gsp, + _gsp: &Gsp<'_>, _ctx: &mut GspBootContext<'_, '_>, - _gsp_fw: &GspFirmware, + _gsp_fw: &GspFirmware<'_>, ) -> Result { Ok(()) } diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index e283429a95dd..91201b51030e 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -58,7 +58,7 @@ fn combined_addr(&self) -> u64 { fn lockdown_released_or_error( &self, gsp_falcon: &Falcon<'_, GspEngine>, - fmc_boot_params: &Coherent, + fmc_boot_params: &Coherent<'_, GspFmcBootParams>, ) -> bool { // GSP-FMC normally clears the boot parameters address from the mailboxes early during // boot. If the address is still there, keep polling rather than treating it as an error. @@ -75,7 +75,7 @@ fn lockdown_released_or_error( fn wait_for_gsp_lockdown_release( dev: &device::Device, gsp_falcon: &Falcon<'_, GspEngine>, - fmc_boot_params: &Coherent, + fmc_boot_params: &Coherent<'_, GspFmcBootParams>, ) -> Result { dev_dbg!(dev, "Waiting for GSP lockdown release\n"); @@ -141,12 +141,12 @@ impl GspHal for Gh100 { /// /// This path uses FSP to establish a chain of trust and boot GSP-FMC. FSP handles /// the GSP boot internally - no manual GSP reset/boot is needed. - fn boot( + fn boot<'gpu>( &self, - gsp: &Gsp, - ctx: &mut GspBootContext<'_, '_>, - gsp_fw: &GspFirmware, - ) -> Result> { + gsp: &Gsp<'gpu>, + ctx: &mut GspBootContext<'_, 'gpu>, + gsp_fw: &GspFirmware<'gpu>, + ) -> Result>> { let dev = ctx.dev(); let chipset = ctx.chipset; let gsp_falcon = ctx.gsp_falcon; @@ -159,7 +159,7 @@ fn boot( let args = FmcBootArgs::new(dev, chipset, wpr_meta, &gsp.libos, false)?; let unload_bundle = crate::gsp::UnloadBundle( - KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox + KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox ); // Wait for the GSP RISC-V core to halt in case of error. We create this guard after `args` diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index a5c0ca355493..f315013cff86 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -52,12 +52,12 @@ // // Since there are two variants of the prepared firmware (with and without a bootloader), this type // abstracts the difference. -enum FwsecUnloadFirmware { +enum FwsecUnloadFirmware<'a> { WithoutBl(FwsecFirmware), - WithBl(FwsecFirmwareWithBl), + WithBl(FwsecFirmwareWithBl<'a>), } -impl FwsecUnloadFirmware { +impl FwsecUnloadFirmware<'_> { /// Runs the FWSEC SB firmware. fn run( &self, @@ -74,12 +74,12 @@ fn run( // Contains the firmware required to fully reset GSP on chipsets where the GSP is started using // FWSEC/Booter. -struct Sec2UnloadBundle { - fwsec_sb: FwsecUnloadFirmware, +struct Sec2UnloadBundle<'a> { + fwsec_sb: FwsecUnloadFirmware<'a>, booter_unloader: BooterFirmware, } -impl UnloadBundle for Sec2UnloadBundle { +impl UnloadBundle for Sec2UnloadBundle<'_> { fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result { let dev = ctx.dev(); let bar = ctx.bar; @@ -213,14 +213,14 @@ fn run_fwsec_frts( } /// Load and prepare the resources required to properly reset the GSP after it has been stopped. - fn build_unload_bundle( + fn build_unload_bundle<'gpu>( &self, - dev: &device::Device, + dev: &'gpu device::Device, chipset: Chipset, bios: &Vbios, gsp_falcon: &Falcon<'_, GspEngine>, sec2_falcon: &Falcon<'_, Sec2>, - ) -> Result { + ) -> Result> { // Load the FWSEC SB firmware, as well as its bootloader if required. let fwsec_sb = FwsecFirmware::new(dev, gsp_falcon, bios, FwsecCommand::Sb)?; let fwsec_sb = if self.needs_fwsec_bootloader { @@ -241,18 +241,18 @@ fn build_unload_bundle( }, GFP_KERNEL, ) - .map(|b| crate::gsp::UnloadBundle(b)) + .map(|b| crate::gsp::UnloadBundle(b as KBox)) .map_err(Into::into) } } impl GspHal for Tu102 { - fn boot( + fn boot<'gpu>( &self, - gsp: &Gsp, - ctx: &mut GspBootContext<'_, '_>, - gsp_fw: &GspFirmware, - ) -> Result> { + gsp: &Gsp<'gpu>, + ctx: &mut GspBootContext<'_, 'gpu>, + gsp_fw: &GspFirmware<'gpu>, + ) -> Result>> { let dev = ctx.dev(); let bar = ctx.bar; let chipset = ctx.chipset; @@ -317,9 +317,9 @@ fn boot( fn post_boot( &self, - gsp: &Gsp, + gsp: &Gsp<'_>, ctx: &mut GspBootContext<'_, '_>, - gsp_fw: &GspFirmware, + gsp_fw: &GspFirmware<'_>, ) -> Result { GspSequencer::run(&gsp.cmdq, ctx, &gsp.libos, gsp_fw.bootloader.app_version)?; diff --git a/drivers/gpu/nova-core/gsp/sequencer.rs b/drivers/gpu/nova-core/gsp/sequencer.rs index bcad1421953a..dae34c11eb05 100644 --- a/drivers/gpu/nova-core/gsp/sequencer.rs +++ b/drivers/gpu/nova-core/gsp/sequencer.rs @@ -138,7 +138,7 @@ pub(crate) struct GspSequencer<'a> { /// GSP falcon for core operations. gsp_falcon: &'a Falcon<'a, Gsp>, /// LibOS memory region init arguments. - libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, + libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>, /// Bootloader application version. bootloader_app_version: u32, /// Device for logging. @@ -338,9 +338,9 @@ fn next(&mut self) -> Option { impl<'a> GspSequencer<'a> { pub(crate) fn run( - cmdq: &Cmdq, + cmdq: &Cmdq<'_>, ctx: &'a GspBootContext<'_, '_>, - libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, + libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>, bootloader_app_version: u32, ) -> Result { let seq_info = loop { diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 79f453e9ec0b..4ce914b7d1da 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -24,7 +24,6 @@ }, prelude::*, ptr::KnownSize, - sync::aref::ARef, transmute::{ AsBytes, FromBytes, // @@ -223,7 +222,7 @@ pub const fn value(&self) -> u64 { /// /// # fn test(dev: &Device) -> Result { /// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN; -/// let c: Coherent<[u64]> = +/// let c: Coherent<'_, [u64]> = /// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, attribs)?; /// # Ok::<(), Error>(()) } /// ``` @@ -390,9 +389,9 @@ fn from(direction: DataDirection) -> Self { /// }; /// /// # fn test(dev: &Device) -> Result { -/// let mut dmem: CoherentBox = CoherentBox::zeroed(dev, GFP_KERNEL)?; +/// let mut dmem: CoherentBox<'_, u64> = CoherentBox::zeroed(dev, GFP_KERNEL)?; /// *dmem = 42; -/// let dmem: Coherent = dmem.into(); +/// let dmem: Coherent<'_, u64> = dmem.into(); /// # Ok::<(), Error>(()) } /// ``` /// @@ -410,18 +409,18 @@ fn from(direction: DataDirection) -> Self { /// }; /// /// # fn test(dev: &Device) -> Result { -/// let mut dmem: CoherentBox<[u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?; +/// let mut dmem: CoherentBox<'_, [u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?; /// dmem.fill(42); -/// let dmem: Coherent<[u64]> = dmem.into(); +/// let dmem: Coherent<'_, [u64]> = dmem.into(); /// # Ok::<(), Error>(()) } /// ``` -pub struct CoherentBox(Coherent); +pub struct CoherentBox<'a, T: KnownSize + ?Sized>(Coherent<'a, T>); -impl CoherentBox<[T]> { +impl<'a, T: AsBytes + FromBytes> CoherentBox<'a, [T]> { /// [`CoherentBox`] variant of [`Coherent::zeroed_slice_with_attrs`]. #[inline] pub fn zeroed_slice_with_attrs( - dev: &device::Device, + dev: &'a device::Device, count: usize, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, @@ -432,7 +431,7 @@ pub fn zeroed_slice_with_attrs( /// Same as [CoherentBox::zeroed_slice_with_attrs], but with `dma::Attrs(0)`. #[inline] pub fn zeroed_slice( - dev: &device::Device, + dev: &'a device::Device, count: usize, gfp_flags: kernel::alloc::Flags, ) -> Result { @@ -480,14 +479,14 @@ pub fn init_at(&mut self, i: usize, init: impl Init) -> Result /// /// # fn test(dev: &Device) -> Result { /// let data = [0u8, 1u8, 2u8, 3u8]; - /// let c: CoherentBox<[u8]> = + /// let c: CoherentBox<'_, [u8]> = /// CoherentBox::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?; /// /// assert_eq!(c.deref(), &data); /// # Ok::<(), Error>(()) } /// ``` pub fn from_slice_with_attrs( - dev: &device::Device, + dev: &'a device::Device, data: &[T], gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, @@ -512,7 +511,7 @@ pub fn from_slice_with_attrs( /// `dma_attrs` is 0 by default. #[inline] pub fn from_slice( - dev: &device::Device, + dev: &'a device::Device, data: &[T], gfp_flags: kernel::alloc::Flags, ) -> Result @@ -523,11 +522,11 @@ pub fn from_slice( } } -impl CoherentBox { +impl<'a, T: AsBytes + FromBytes> CoherentBox<'a, T> { /// Same as [`CoherentBox::zeroed_slice_with_attrs`], but for a single element. #[inline] pub fn zeroed_with_attrs( - dev: &device::Device, + dev: &'a device::Device, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, ) -> Result { @@ -536,12 +535,12 @@ pub fn zeroed_with_attrs( /// Same as [`CoherentBox::zeroed_slice`], but for a single element. #[inline] - pub fn zeroed(dev: &device::Device, gfp_flags: kernel::alloc::Flags) -> Result { + pub fn zeroed(dev: &'a device::Device, gfp_flags: kernel::alloc::Flags) -> Result { Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0)) } } -impl Deref for CoherentBox { +impl Deref for CoherentBox<'_, T> { type Target = T; #[inline] @@ -554,7 +553,7 @@ fn deref(&self) -> &Self::Target { } } -impl DerefMut for CoherentBox { +impl DerefMut for CoherentBox<'_, T> { #[inline] fn deref_mut(&mut self) -> &mut Self::Target { // SAFETY: @@ -565,9 +564,9 @@ fn deref_mut(&mut self) -> &mut Self::Target { } } -impl From> for Coherent { +impl<'a, T: AsBytes + FromBytes + KnownSize + ?Sized> From> for Coherent<'a, T> { #[inline] - fn from(value: CoherentBox) -> Self { + fn from(value: CoherentBox<'a, T>) -> Self { value.0 } } @@ -588,26 +587,20 @@ fn from(value: CoherentBox) -> Self { /// to an allocated region of coherent memory and `dma_addr` is the DMA address base of the /// region. /// - The size in bytes of the allocation is equal to size information via pointer. -// TODO // -// DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness -// reasons DMA allocation would need to be embedded in a `Devres` container, in order to ensure -// that device resources can never survive device unbind. -// -// However, it is neither desirable nor necessary to protect the allocated memory of the DMA -// allocation from surviving device unbind; it would require RCU read side critical sections to -// access the memory, which may require subsequent unnecessary copies. -// -// Hence, find a way to revoke the device resources of a `Coherent`, but not the -// entire `Coherent` including the allocated memory itself. -pub struct Coherent { - dev: ARef, +// The lifetime parameter ties DMA allocations to the device's bound scope, ensuring they are freed +// before the device is unbound under normal circumstances. However, if a `Coherent` is leaked (e.g. +// via `mem::forget`), device resources such as IOMMU mappings will not be released. Making all +// constructors `unsafe` to prevent this is considered too restrictive for the common case; this +// soundness hole is accepted for now. +pub struct Coherent<'a, T: KnownSize + ?Sized> { + dev: &'a device::Device, dma_addr: DmaAddress, cpu_addr: NonNull, dma_attrs: Attrs, } -impl Coherent { +impl Coherent<'_, T> { /// Returns the size in bytes of this allocation. #[inline] pub fn size(&self) -> usize { @@ -663,10 +656,10 @@ pub unsafe fn as_mut(&self) -> &mut T { } } -impl Coherent { +impl<'a, T: AsBytes + FromBytes> Coherent<'a, T> { /// Allocates a region of `T` of coherent memory. fn alloc_with_attrs( - dev: &device::Device, + dev: &'a device::Device, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, ) -> Result { @@ -692,9 +685,9 @@ fn alloc_with_attrs( // INVARIANT: // - We just successfully allocated a coherent region which is adequately sized for `T`, // hence the cpu address is valid. - // - We also hold a refcounted reference to the device. + // - `dev` is a valid reference to a bound device that outlives this allocation. Ok(Self { - dev: dev.into(), + dev, dma_addr, cpu_addr, dma_attrs, @@ -716,13 +709,13 @@ fn alloc_with_attrs( /// }; /// /// # fn test(dev: &Device) -> Result { - /// let c: Coherent<[u64; 4]> = + /// let c: Coherent<'_, [u64; 4]> = /// Coherent::zeroed_with_attrs(dev, GFP_KERNEL, DMA_ATTR_NO_WARN)?; /// # Ok::<(), Error>(()) } /// ``` #[inline] pub fn zeroed_with_attrs( - dev: &device::Device, + dev: &'a device::Device, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, ) -> Result { @@ -732,14 +725,14 @@ pub fn zeroed_with_attrs( /// Performs the same functionality as [`Coherent::zeroed_with_attrs`], except the /// `dma_attrs` is 0 by default. #[inline] - pub fn zeroed(dev: &device::Device, gfp_flags: kernel::alloc::Flags) -> Result { + pub fn zeroed(dev: &'a device::Device, gfp_flags: kernel::alloc::Flags) -> Result { Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0)) } /// Same as [`Coherent::zeroed_with_attrs`], but instead of a zero-initialization the memory is /// initialized with `init`. pub fn init_with_attrs( - dev: &device::Device, + dev: &'a device::Device, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, init: impl Init, @@ -764,7 +757,7 @@ pub fn init_with_attrs( /// with `init`. #[inline] pub fn init( - dev: &device::Device, + dev: &'a device::Device, gfp_flags: kernel::alloc::Flags, init: impl Init, ) -> Result @@ -776,11 +769,11 @@ pub fn init( /// Allocates a region of `[T; len]` of coherent memory. fn alloc_slice_with_attrs( - dev: &device::Device, + dev: &'a device::Device, len: usize, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, - ) -> Result> { + ) -> Result> { const { assert!( core::mem::size_of::() > 0, @@ -809,9 +802,9 @@ fn alloc_slice_with_attrs( // INVARIANT: // - We just successfully allocated a coherent region which is adequately sized for // `[T; len]`, hence the cpu address is valid. - // - We also hold a refcounted reference to the device. + // - `dev` is a valid reference to a bound device that outlives this allocation. Ok(Coherent { - dev: dev.into(), + dev, dma_addr, cpu_addr, dma_attrs, @@ -836,17 +829,17 @@ fn alloc_slice_with_attrs( /// }; /// /// # fn test(dev: &Device) -> Result { - /// let c: Coherent<[u64]> = + /// let c: Coherent<'_, [u64]> = /// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?; /// # Ok::<(), Error>(()) } /// ``` #[inline] pub fn zeroed_slice_with_attrs( - dev: &device::Device, + dev: &'a device::Device, len: usize, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, - ) -> Result> { + ) -> Result> { Coherent::alloc_slice_with_attrs(dev, len, gfp_flags | __GFP_ZERO, dma_attrs) } @@ -854,10 +847,10 @@ pub fn zeroed_slice_with_attrs( /// `dma_attrs` is 0 by default. #[inline] pub fn zeroed_slice( - dev: &device::Device, + dev: &'a device::Device, len: usize, gfp_flags: kernel::alloc::Flags, - ) -> Result> { + ) -> Result> { Self::zeroed_slice_with_attrs(dev, len, gfp_flags, Attrs(0)) } @@ -876,18 +869,18 @@ pub fn zeroed_slice( /// # fn test(dev: &Device) -> Result { /// let data = [0u8, 1u8, 2u8, 3u8]; /// // `c` has the same content as `data`. - /// let c: Coherent<[u8]> = + /// let c: Coherent<'_, [u8]> = /// Coherent::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?; /// /// # Ok::<(), Error>(()) } /// ``` #[inline] pub fn from_slice_with_attrs( - dev: &device::Device, + dev: &'a device::Device, data: &[T], gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, - ) -> Result> + ) -> Result> where T: Copy, { @@ -898,10 +891,10 @@ pub fn from_slice_with_attrs( /// `dma_attrs` is 0 by default. #[inline] pub fn from_slice( - dev: &device::Device, + dev: &'a device::Device, data: &[T], gfp_flags: kernel::alloc::Flags, - ) -> Result> + ) -> Result> where T: Copy, { @@ -909,7 +902,7 @@ pub fn from_slice( } } -impl Coherent<[T]> { +impl Coherent<'_, [T]> { /// Returns the number of elements `T` in this allocation. /// /// Note that this is not the size of the allocation in bytes, which is provided by @@ -922,10 +915,10 @@ pub fn len(&self) -> usize { } /// Note that the device configured to do DMA must be halted before this object is dropped. -impl Drop for Coherent { +impl Drop for Coherent<'_, T> { fn drop(&mut self) { let size = T::size(self.cpu_addr.as_ptr()); - // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. + // SAFETY: Device pointer is guaranteed as valid by the lifetime of this `Coherent`. // The cpu address, and the dma address are valid due to the type invariants on // `Coherent`. unsafe { @@ -942,15 +935,15 @@ fn drop(&mut self) { // SAFETY: It is safe to send a `Coherent` to another thread if `T` // can be sent to another thread. -unsafe impl Send for Coherent {} +unsafe impl Send for Coherent<'_, T> {} // SAFETY: Sharing `&Coherent` across threads is safe if `T` is `Sync`, because all // methods that access the buffer contents (`field_read`, `field_write`, `as_slice`, // `as_slice_mut`) are `unsafe`, and callers are responsible for ensuring no data races occur. // The safe methods only return metadata or raw pointers whose use requires `unsafe`. -unsafe impl Sync for Coherent {} +unsafe impl Sync for Coherent<'_, T> {} -impl debugfs::BinaryWriter for Coherent { +impl debugfs::BinaryWriter for Coherent<'_, T> { fn write_to_slice( &self, writer: &mut UserSliceWriter, @@ -1236,7 +1229,7 @@ fn as_view(self) -> CoherentView<'a, Self::Target> { } } -impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent { +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent<'_, T> { type Backend = CoherentIoBackend; type Target = T; diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs index 5f6c4d7a1a51..f09078228c53 100644 --- a/rust/kernel/uaccess.rs +++ b/rust/kernel/uaccess.rs @@ -520,14 +520,14 @@ pub fn write_slice(&mut self, data: &[u8]) -> Result { /// /// fn copy_dma_to_user( /// mut writer: UserSliceWriter, - /// alloc: &Coherent<[u8]>, + /// alloc: &Coherent<'_, [u8]>, /// ) -> Result { /// writer.write_dma(alloc, 0, 256) /// } /// ``` pub fn write_dma( &mut self, - alloc: &Coherent, + alloc: &Coherent<'_, T>, offset: usize, count: usize, ) -> Result { diff --git a/samples/rust/rust_dma.rs b/samples/rust/rust_dma.rs index 0fac9d4ae566..ffb693544673 100644 --- a/samples/rust/rust_dma.rs +++ b/samples/rust/rust_dma.rs @@ -34,7 +34,7 @@ #[pin_data(PinnedDrop)] struct DmaSampleData<'bound> { pdev: &'bound pci::Device, - ca: Coherent<[MyStruct]>, + ca: Coherent<'bound, [MyStruct]>, #[pin] sgt: SGTable>>, } @@ -86,7 +86,7 @@ fn probe<'bound>( // SAFETY: There are no concurrent calls to DMA allocation and mapping primitives. unsafe { pdev.dma_set_mask_and_coherent(mask)? }; - let ca: Coherent<[MyStruct]> = + let ca: Coherent<'_, [MyStruct]> = Coherent::zeroed_slice(pdev.as_ref(), TEST_VALUES.len(), GFP_KERNEL)?; for (i, value) in TEST_VALUES.into_iter().enumerate() { From e1df8a1560a97d2f5c1965d4844ab64769e92a91 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Fri, 4 Sep 2026 19:38:00 -0700 Subject: [PATCH 011/110] gpu: nova-core: gsp: fix a broken doc link on GspMessage The doc comment on the GSP message type names Cmdq as the type that defines wait_for_msg. That method belongs to CmdqInner, so the link has no target. Nothing catches the break, because rustdoc resolves a link only in an item it documents, and the kernel's rustdoc run skips private items. Point the link at CmdqInner. Assisted-by: LLM Signed-off-by: John Hubbard Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260905023800.1293631-1-jhubbard@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/gsp/cmdq.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index 14a711307654..28876c4aa42c 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -478,7 +478,7 @@ struct GspCommand<'a> { /// A message ready to be processed from the message queue. /// -/// This is the type returned by [`Cmdq::wait_for_msg`]. +/// This is the type returned by [`CmdqInner::wait_for_msg`]. struct GspMessage<'a> { // Reference to the header of the message. header: &'a GspMsgElement, From 1fc1f82756b1ff872c04950747925f7dcf5c1d8a Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:26 +0100 Subject: [PATCH 012/110] rust: mem: add `transmute` with deferred size check Implement a `transmute/safe_transmute` that checks size at monomorphization time instead of type-checking time. This allows more cases where we know that the size matches but this is not generically checkable. The signature is equivalent to the unstable `transmute_neo` function in the standard library. A safe variant is provided to use with types implementing `FromBytes` and `IntoBytes`. Existing users of `transmute_copy` to bypass size checks are converted. Signed-off-by: Gary Guo Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Acked-by: Miguel Ojeda Link: https://patch.msgid.link/20260901-typed_register-v4-2-5552b1d59525@garyguo.net [ Fix doc heading to use plural "Examples", add missing closing code block delimiter and fix a few other nits. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/device_id.rs | 3 +- rust/kernel/lib.rs | 1 + rust/kernel/mem.rs | 102 +++++++++++++++++++++++++++++++++++++ rust/kernel/sync/atomic.rs | 4 +- 4 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 rust/kernel/mem.rs diff --git a/rust/kernel/device_id.rs b/rust/kernel/device_id.rs index c81fca5b4986..f0b9cb84e58e 100644 --- a/rust/kernel/device_id.rs +++ b/rust/kernel/device_id.rs @@ -146,8 +146,7 @@ impl IdArray { /// If the device implements [`RawDeviceIdIndex`], consider using [`IdArray::new`] instead. pub const fn new_without_index(ids: [T; N]) -> Self { // SAFETY: `T` is layout-wise compatible with `T::RawType`, so is the array of them. - let raw_ids: [MaybeUninit; N] = unsafe { core::mem::transmute_copy(&ids) }; - core::mem::forget(ids); + let raw_ids: [MaybeUninit; N] = unsafe { crate::mem::transmute(ids) }; Self { ids: raw_ids, diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 4d5c96ddc49c..7225abc64084 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -98,6 +98,7 @@ pub mod kunit; pub mod list; pub mod maple_tree; +pub mod mem; pub mod miscdevice; pub mod mm; pub mod module; diff --git a/rust/kernel/mem.rs b/rust/kernel/mem.rs new file mode 100644 index 000000000000..35426f4d514e --- /dev/null +++ b/rust/kernel/mem.rs @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Basic utilities for dealing with memory, values, and types. + +use crate::prelude::*; + +/// Transmute between two types. +/// +/// Use this instead of [`core::mem::transmute`] when it is known that sizes are identical but this +/// cannot be proven by the compiler. +/// +/// This is equivalent to Rust's `transmute_unchecked` intrinsics. +/// +/// # Safety +/// +/// All safety requirements of [`core::mem::transmute`] apply, plus that the size `Src` and `Dst` +/// must match. +/// +/// # Examples +/// +/// This can be used when types are known to have the same size, but only at runtime. +/// +/// ```no_run +/// # use core::any::TypeId; +/// fn to_u32(v: T) -> Option { +/// if TypeId::of::() != TypeId::of::() { +/// return None; +/// } +/// +/// // `core::mem::transmute` won't work here. +/// // SAFETY: We've checked that `T` is `u32`! +/// Some(unsafe { kernel::mem::transmute_unchecked(v) }) +/// } +/// +/// to_u32(1u32); +/// ``` +#[inline(always)] +pub const unsafe fn transmute_unchecked(val: Src) -> Dst { + // SAFETY: This is identical to `transmute` except that we bypassed the size check; which is + // true per safety requirement. + unsafe { core::mem::transmute_copy(&core::mem::ManuallyDrop::new(val)) } +} + +/// Version of `transmute` that performs size check at monomorphization-time. +/// +/// Use this instead of [`core::mem::transmute`] when it is known that sizes are identical but this +/// cannot be proven by the compiler during type checking and can be proven during monomorphization. +/// +/// The signature is equivalent to Rust standard library's unstable `transmute_neo` and that of +/// [RFC 3844](https://github.com/rust-lang/rfcs/pull/3844). +/// +/// # Safety +/// +/// Same as [`core::mem::transmute`]. +/// +/// # Examples +/// +/// This is typically used in generic code where it's known that type will have the same size, but +/// the compiler cannot prove it generically. +/// +/// ```no_run +/// trait IsU32 {} +/// impl IsU32 for u32 {} +/// +/// fn to_u32(v: T) -> u32 { +/// // `core::mem::transmute` won't work here. +/// // SAFETY: We know that `v` is u32! +/// unsafe { kernel::mem::transmute(v) } +/// } +/// +/// to_u32(1u32); +/// ``` +#[inline(always)] +pub const unsafe fn transmute(val: Src) -> Dst { + const_assert!(size_of::() == size_of::()); + + // SAFETY: Size is checked above. Other safety requirements follow those of the function. + unsafe { transmute_unchecked(val) } +} + +/// Safely transmutes a value of one type to a value of another type of the same size. +/// +/// The sizes are checked during monomorphization. +/// +/// This can be considered as generic version of [`zerocopy::transmute!`] macro that defers the size +/// check and thus can be used in more cases. +/// +/// # Examples +/// +/// ```no_run +/// fn to_u32(v: T) -> u32 { +/// // `zerocopy::transmute!` won't work here. +/// kernel::mem::safe_transmute(v) +/// } +/// +/// to_u32(1i32); +/// ``` +#[inline(always)] +pub const fn safe_transmute(val: Src) -> Dst { + // SAFETY: `transmute` is safe with `IntoBytes` and `FromBytes` bounds. + unsafe { transmute(val) } +} diff --git a/rust/kernel/sync/atomic.rs b/rust/kernel/sync/atomic.rs index 9cd009d57e35..6d27898add42 100644 --- a/rust/kernel/sync/atomic.rs +++ b/rust/kernel/sync/atomic.rs @@ -140,7 +140,7 @@ pub unsafe trait AtomicAdd: AtomicType { const fn into_repr(v: T) -> T::Repr { // SAFETY: Per the safety requirement of `AtomicType`, `T` is round-trip transmutable to // `T::Repr`, therefore the transmute operation is sound. - unsafe { core::mem::transmute_copy(&v) } + unsafe { crate::mem::transmute(v) } } /// # Safety @@ -149,7 +149,7 @@ const fn into_repr(v: T) -> T::Repr { #[inline(always)] const unsafe fn from_repr(r: T::Repr) -> T { // SAFETY: Per the safety requirement of the function, the transmute operation is sound. - unsafe { core::mem::transmute_copy(&r) } + unsafe { crate::mem::transmute(r) } } impl Atomic { From ef605a433d80987e97fcf62a41bdf437abc61e2c Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:27 +0100 Subject: [PATCH 013/110] rust: mem: add `AsRepr` and `AsReprMut` Some API like atomics and I/O operate on primitives only; therefore other types would need to converted to these primitive first. Add two traits `AsRepr` and `AsReprMut` to indicate that the type can be turned into a primitive for these operations. `T: AsRepr` means that `&T` can be viewed as `&T::Repr` and thus it needs to support transmutability in one direction. `T: AsReprMut` means that `&mut T` can be viewed as `&mut T::Repr` and thus it needs to support bi-directional transmutability. To avoid duplicating implementations, all repr types are normalized to unsigned integers. Signed-off-by: Gary Guo Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Acked-by: Miguel Ojeda Link: https://patch.msgid.link/20260901-typed_register-v4-3-5552b1d59525@garyguo.net [ Fix grammar in from_repr_unchecked() documentation, rephrase confusing safety comment, use rustdoc links for Self, fix backtick formatting in comments and use consistent 'must' wording in safety sections. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/mem.rs | 132 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/rust/kernel/mem.rs b/rust/kernel/mem.rs index 35426f4d514e..f2d4cdf87d00 100644 --- a/rust/kernel/mem.rs +++ b/rust/kernel/mem.rs @@ -100,3 +100,135 @@ pub const fn safe_transmute(val: Src) -> Dst { // SAFETY: `transmute` is safe with `IntoBytes` and `FromBytes` bounds. unsafe { transmute(val) } } + +/// Type that is layout-compatible with a primitive representation. +/// +/// # Safety +/// +/// - [`Self`] must have the same size and alignment as [`Self::Repr`]. +/// - [`Self`] must be [transmutable] to [`Self::Repr`]. +/// - Neither [`Self`] nor [`Self::Repr`] contains interior mutability. +/// +/// The above basically says that `&Self` can be transmuted to `&Self::Repr`. +/// +/// [transmutable]: core::mem::transmute +pub unsafe trait AsRepr: Sized { + /// Primitive representation of this type. + type Repr; + + /// Convert from [`&Self`](Self) to [`&Self::Repr`](AsRepr::Repr). + #[inline(always)] + fn as_repr(this: &Self) -> &Self::Repr { + // SAFETY: Per safety requirement of the trait. + unsafe { core::mem::transmute(this) } + } + + /// Convert from [`Self`] to [`Self::Repr`]. + #[inline(always)] + fn into_repr(this: Self) -> Self::Repr { + // SAFETY: Per safety requirement of the trait. + unsafe { transmute(this) } + } + + /// Convert from [`Self::Repr`] to [`Self`]. + /// + /// # Safety + /// + /// `repr` must be a valid bit pattern of [`Self`] and satisfy type-specific invariants of it. + /// + /// Alternatively, if `repr` is previously obtained using [`Self::into_repr`], and each + /// `from_repr_unchecked` should correspond to a unique `into_repr` call, then it is safe to + /// call as well (this means that we're undoing a `into_repr` call getting the exact bytes + /// back). + /// + /// No guarantee is made if the result of a `into_repr` is passed to multiple + /// `from_repr_unchecked` (i.e. copies are made), to allow for cases where `Repr` is a pointer + /// and the user of the API wants ownership transfer. Users that want the ability to call + /// `from_repr_unchecked` after copying can require `Copy` bound explicitly. + #[inline(always)] + unsafe fn from_repr_unchecked(repr: Self::Repr) -> Self { + // SAFETY: Per safety requirement, `repr` is valid repr of `Self`, or it is previously from + // `into_repr`, in which case we're undoing the transmute so it is also safe. + unsafe { transmute(repr) } + } +} + +/// Type that is bi-directionally transmutable with a primitive representation. +/// +/// # Safety +/// +/// - [`Self`] must be [transmutable] from [`Self::Repr`]. +/// +/// [transmutable]: core::mem::transmute +/// [`Self::Repr`]: AsRepr::Repr +pub unsafe trait AsReprMut: AsRepr { + /// Convert from `&mut Self` to [`&mut Self::Repr`](AsRepr::Repr). + #[inline(always)] + fn as_repr_mut(this: &mut Self) -> &mut Self::Repr { + // SAFETY: Per safety requirement of the trait. + unsafe { core::mem::transmute(this) } + } + + /// Convert from [`Self::Repr`](AsRepr::Repr) to `Self`. + #[inline(always)] + fn from_repr(repr: Self::Repr) -> Self { + // SAFETY: Per safety requirement of the trait. + unsafe { transmute(repr) } + } +} + +// SAFETY: `bool` has the same size and alignment as `u8`, and Rust guarantees that `bool` has +// only two valid bit patterns: 0 (`false`) and 1 (`true`). Thus `bool` can be transmuted to `u8`. +// Neither types contain interior mutability. +unsafe impl AsRepr for bool { + type Repr = u8; +} + +// SAFETY: `*mut T` has the same size and alignment with `*const c_void`, and thus `*mut T` is +// transmutable to `*const c_void`. Neither types contain interior mutability. +unsafe impl AsRepr for *mut T { + type Repr = *const c_void; +} + +// SAFETY: `*mut T` is transmutable from `*const c_void`. +unsafe impl AsReprMut for *mut T {} + +// SAFETY: `*const T` has the same size and alignment with `*const c_void`, and is transmutable to +// `*const c_void`. Neither types contain interior mutability. +unsafe impl AsRepr for *const T { + type Repr = *const c_void; +} + +// SAFETY: `*const T` is transmutable from `*const c_void`. +unsafe impl AsReprMut for *const T {} + +macro_rules! int_impl { + ($($unsigned:ident $signed:ident ,)*) => {$( + // SAFETY: `$unsigned` has the same size and alignment with itself, and is transmutable to + // itself. It does not contain interior mutability. + unsafe impl AsRepr for $unsigned { + type Repr = $unsigned; + } + + // SAFETY: `$unsigned` is transmutable from itself. + unsafe impl AsReprMut for $unsigned {} + + // SAFETY: `$signed` has the same size and alignment with `$unsigned`, and is transmutable + // to it Neither types contain interior mutability. + unsafe impl AsRepr for $signed { + type Repr = $unsigned; + } + + // SAFETY: `$signed` is transmutable from `$unsigned`. + unsafe impl AsReprMut for $signed {} + )*}; +} + +int_impl! { + u8 i8, + u16 i16, + u32 i32, + u64 i64, + // `usize` is not normalized to particular integer for portability. + usize isize, +} From ff067ace5fbeada7e629d4aaccfa1d04da3eeebc Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:28 +0100 Subject: [PATCH 014/110] rust: io: perform conversions using `AsRepr` For types that are layout-compatible with an I/O capable type, we would want the ability to use them directly for I/O operations. E.g. bitfield! { pub struct Foo(u32) { ... } } #[repr(C)] struct Bar { foo: Foo, } let mmio: Mmio<'_, Bar> = ...; io_read!(mmio, .foo) Currently this feature is available from `register!()` macro but not otherwise available with `io_read!`, `io_write!`. Support this by performing conversions to I/O primitives via the `AsRepr`/`AsReprMut` trait. This makes the `IoLoc::IoType` and `Register::Storage` redundant; thus remove them; also convert register methods to use the `read_val` and `write_val` instead. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-4-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/bitfield.rs | 9 +++ rust/kernel/io.rs | 128 ++++++++++++++++++++----------------- rust/kernel/io/register.rs | 15 ----- rust/macros/io/register.rs | 2 - 4 files changed, 78 insertions(+), 76 deletions(-) diff --git a/rust/kernel/bitfield.rs b/rust/kernel/bitfield.rs index a0d089423f21..15c78790e151 100644 --- a/rust/kernel/bitfield.rs +++ b/rust/kernel/bitfield.rs @@ -346,6 +346,15 @@ fn from(val: $storage) -> $name { Self::from_raw(val) } } + + // SAFETY: `$name` is transparent over `$storage` and `$storage` has no interior mutability. + unsafe impl $crate::mem::AsRepr for $name { + // Normalize `$storage` to the canonical repr type in case it is signed. + type Repr = <$storage as $crate::mem::AsRepr>::Repr; + } + + // SAFETY: `$name` is transparent over `$storage`. + unsafe impl $crate::mem::AsReprMut for $name {} }; // Definitions requiring knowledge of individual fields: private and public field accessors, diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 5ce9fd129068..18c71bd765b2 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -11,6 +11,10 @@ use crate::{ bindings, + mem::{ + AsRepr, + AsReprMut, // + }, prelude::*, ptr::{ Alignment, @@ -226,6 +230,17 @@ fn io_view<'a, IO: Io<'a>, U>( Ok(unsafe { IO::Backend::project_view(view, projected_ptr) }) } +/// Returns the primitive view of a I/O view. +#[inline] +fn io_view_as_repr<'a, IO: Io<'a, Target = T>, T: AsRepr>( + this: IO, +) -> ::View<'a, T::Repr> { + let view = this.as_view(); + + // SAFETY: `AsRepr` guarantees layout compatibility. + unsafe { IO::Backend::project_view(view, IO::Backend::as_ptr(view).cast::()) } +} + /// I/O backends. /// /// This is an abstract representation to be implemented by arbitrary I/O @@ -353,15 +368,12 @@ fn copy_write(view: Self::View<'_, T>, value: T) { /// /// - The valid `Base` to operate on. For most registers, this should be [`Region`]. /// - The offset to access (returned by [`IoLoc::offset`]), -/// - The width of the access (determined by [`IoLoc::IoType`]), -/// - The type `T` in which the raw data is returned or provided. +/// - The type `T` in which the data is returned or provided. /// -/// `T` and `IoLoc::IoType` may differ: for instance, a typed register has `T` = the register type -/// with its bitfields, and `IoType` = its backing primitive (e.g. `u32`). +/// `T` is not necessarily the type for underlying I/O operation. Methods that take `IoLoc` have `T: +/// AsRepr` bound and the `::Repr` type would be used to perform I/O and converted to +/// `T` instead. pub trait IoLoc { - /// Size ([`u8`], [`u16`], etc) of the I/O performed on the returned [`offset`](IoLoc::offset). - type IoType: Into + From; - /// Consumes `self` and returns the offset of this location. fn offset(self) -> usize; } @@ -372,8 +384,6 @@ macro_rules! impl_usize_ioloc { ($($ty:ty),*) => { $( impl IoLoc, $ty> for usize { - type IoType = $ty; - #[inline(always)] fn offset(self) -> usize { self @@ -498,10 +508,10 @@ fn try_cast(self) -> Result<::View<'a, U>> #[inline] fn read_val(self) -> Self::Target where - Self::Backend: IoCapable, - Self::Target: Sized, + Self::Target: AsReprMut, + Self::Backend: IoCapable<::Repr>, { - Self::Backend::io_read(self.as_view()) + Self::Target::from_repr(Self::Backend::io_read(io_view_as_repr(self))) } /// Write a value to I/O. @@ -520,10 +530,10 @@ fn read_val(self) -> Self::Target #[inline] fn write_val(self, value: Self::Target) where - Self::Backend: IoCapable, - Self::Target: Sized, + Self::Target: AsRepr, + Self::Backend: IoCapable<::Repr>, { - Self::Backend::io_write(self.as_view(), value) + Self::Backend::io_write(io_view_as_repr(self), Self::Target::into_repr(value)) } /// Copy-read from I/O memory. @@ -645,7 +655,7 @@ fn copy_to_slice(self, data: &mut [u8]) #[inline(always)] fn try_read8(self, offset: usize) -> Result where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.try_read(offset) @@ -655,7 +665,7 @@ fn try_read8(self, offset: usize) -> Result #[inline(always)] fn try_read16(self, offset: usize) -> Result where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.try_read(offset) @@ -665,7 +675,7 @@ fn try_read16(self, offset: usize) -> Result #[inline(always)] fn try_read32(self, offset: usize) -> Result where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.try_read(offset) @@ -675,7 +685,7 @@ fn try_read32(self, offset: usize) -> Result #[inline(always)] fn try_read64(self, offset: usize) -> Result where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.try_read(offset) @@ -685,7 +695,7 @@ fn try_read64(self, offset: usize) -> Result #[inline(always)] fn try_write8(self, value: u8, offset: usize) -> Result where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.try_write(offset, value) @@ -695,7 +705,7 @@ fn try_write8(self, value: u8, offset: usize) -> Result #[inline(always)] fn try_write16(self, value: u16, offset: usize) -> Result where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.try_write(offset, value) @@ -705,7 +715,7 @@ fn try_write16(self, value: u16, offset: usize) -> Result #[inline(always)] fn try_write32(self, value: u32, offset: usize) -> Result where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.try_write(offset, value) @@ -715,7 +725,7 @@ fn try_write32(self, value: u32, offset: usize) -> Result #[inline(always)] fn try_write64(self, value: u64, offset: usize) -> Result where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.try_write(offset, value) @@ -727,7 +737,7 @@ fn try_write64(self, value: u64, offset: usize) -> Result #[inline(always)] fn read8(self, offset: usize) -> u8 where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.read(offset) @@ -739,7 +749,7 @@ fn read8(self, offset: usize) -> u8 #[inline(always)] fn read16(self, offset: usize) -> u16 where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.read(offset) @@ -751,7 +761,7 @@ fn read16(self, offset: usize) -> u16 #[inline(always)] fn read32(self, offset: usize) -> u32 where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.read(offset) @@ -763,7 +773,7 @@ fn read32(self, offset: usize) -> u32 #[inline(always)] fn read64(self, offset: usize) -> u64 where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.read(offset) @@ -775,7 +785,7 @@ fn read64(self, offset: usize) -> u64 #[inline(always)] fn write8(self, value: u8, offset: usize) where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.write(offset, value) @@ -787,7 +797,7 @@ fn write8(self, value: u8, offset: usize) #[inline(always)] fn write16(self, value: u16, offset: usize) where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.write(offset, value) @@ -799,7 +809,7 @@ fn write16(self, value: u16, offset: usize) #[inline(always)] fn write32(self, value: u32, offset: usize) where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.write(offset, value) @@ -811,7 +821,7 @@ fn write32(self, value: u32, offset: usize) #[inline(always)] fn write64(self, value: u64, offset: usize) where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.write(offset, value) @@ -843,11 +853,12 @@ fn write64(self, value: u64, offset: usize) #[inline(always)] fn try_read(self, location: L) -> Result where + T: AsReprMut, L: IoLoc, - Self::Backend: IoCapable, + Self::Backend: IoCapable<::Repr>, { - let view = io_view::(self, location.offset())?; - Ok(Self::Backend::io_read(view).into()) + let view = io_view::(self, location.offset())?; + Ok(view.read_val()) } /// Generic fallible write with runtime bounds check. @@ -876,12 +887,12 @@ fn try_read(self, location: L) -> Result #[inline(always)] fn try_write(self, location: L, value: T) -> Result where + T: AsRepr, L: IoLoc, - Self::Backend: IoCapable, + Self::Backend: IoCapable<::Repr>, { - let view = io_view::(self, location.offset())?; - let io_value = value.into(); - Self::Backend::io_write(view, io_value); + let view = io_view::(self, location.offset())?; + view.write_val(value); Ok(()) } @@ -920,9 +931,10 @@ fn try_write(self, location: L, value: T) -> Result #[inline(always)] fn try_write_reg(self, value: V) -> Result where + T: AsRepr, L: IoLoc, V: LocatedRegister, - Self::Backend: IoCapable, + Self::Backend: IoCapable<::Repr>, { let (location, value) = value.into_io_op(); @@ -954,16 +966,13 @@ fn try_write_reg(self, value: V) -> Result #[inline(always)] fn try_update(self, location: L, f: F) -> Result where + T: AsReprMut, L: IoLoc, - Self::Backend: IoCapable, + Self::Backend: IoCapable<::Repr>, F: FnOnce(T) -> T, { - let view = io_view::(self, location.offset())?; - - let value: T = Self::Backend::io_read(view).into(); - let io_value = f(value).into(); - Self::Backend::io_write(view, io_value); - + let view = io_view::(self, location.offset())?; + view.write_val(f(view.read_val())); Ok(()) } @@ -991,11 +1000,12 @@ fn try_update(self, location: L, f: F) -> Result #[inline(always)] fn read(self, location: L) -> T where + T: AsReprMut, L: IoLoc, - Self::Backend: IoCapable, + Self::Backend: IoCapable<::Repr>, { - let view = io_view_assert::(self, location.offset()); - Self::Backend::io_read(view).into() + let view = io_view_assert::(self, location.offset()); + view.read_val() } /// Generic infallible write with compile-time bounds check. @@ -1022,12 +1032,12 @@ fn read(self, location: L) -> T #[inline(always)] fn write(self, location: L, value: T) where + T: AsRepr, L: IoLoc, - Self::Backend: IoCapable, + Self::Backend: IoCapable<::Repr>, { - let view = io_view_assert::(self, location.offset()); - let io_value = value.into(); - Self::Backend::io_write(view, io_value); + let view = io_view_assert::(self, location.offset()); + view.write_val(value) } /// Generic infallible write of a fully-located register value. @@ -1064,9 +1074,10 @@ fn write(self, location: L, value: T) #[inline(always)] fn write_reg(self, value: V) where + T: AsRepr, L: IoLoc, V: LocatedRegister, - Self::Backend: IoCapable, + Self::Backend: IoCapable<::Repr>, { let (location, value) = value.into_io_op(); @@ -1098,14 +1109,13 @@ fn write_reg(self, value: V) #[inline(always)] fn update(self, location: L, f: F) where + T: AsReprMut, L: IoLoc, - Self::Backend: IoCapable, + Self::Backend: IoCapable<::Repr>, F: FnOnce(T) -> T, { - let view = io_view_assert::(self, location.offset()); - let value: T = Self::Backend::io_read(view).into(); - let io_value = f(value).into(); - Self::Backend::io_write(view, io_value); + let view = io_view_assert::(self, location.offset()); + view.write_val(f(view.read_val())); } } diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index 6a19552ffb95..45d2e73a92e7 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -117,9 +117,6 @@ /// Trait implemented by all registers. pub trait Register: Sized { - /// Backing primitive type of the register. - type Storage: Into + From; - /// Start offset of the register. /// /// The interpretation of this offset depends on the type of the register. @@ -135,8 +132,6 @@ impl IoLoc, T> for () where T: FixedRegister, { - type IoType = T::Storage; - #[inline(always)] fn offset(self) -> usize { T::OFFSET @@ -149,8 +144,6 @@ impl IoLoc, T> for T where T: FixedRegister, { - type IoType = T::Storage; - #[inline(always)] fn offset(self) -> usize { T::OFFSET @@ -174,8 +167,6 @@ impl IoLoc, T> for FixedRegisterLoc where T: FixedRegister, { - type IoType = T::Storage; - #[inline(always)] fn offset(self) -> usize { T::OFFSET @@ -246,8 +237,6 @@ impl IoLoc, T> for RelativeRegisterLoc + ?Sized, { - type IoType = T::Storage; - #[inline(always)] fn offset(self) -> usize { RelativeRegisterLoc::offset(self) @@ -289,8 +278,6 @@ impl IoLoc, T> for RegisterArrayLoc where T: RegisterArray, { - type IoType = T::Storage; - #[inline(always)] fn offset(self) -> usize { T::OFFSET + self.0 * T::STRIDE @@ -377,8 +364,6 @@ impl IoLoc, T> for RelativeRegisterArrayLo T: RelativeRegisterArray, B: RegisterBase + ?Sized, { - type IoType = T::Storage; - #[inline(always)] fn offset(self) -> usize { self.0.offset() + self.1 * T::STRIDE diff --git a/rust/macros/io/register.rs b/rust/macros/io/register.rs index cc5bcad31ae8..2e4cdabb2795 100644 --- a/rust/macros/io/register.rs +++ b/rust/macros/io/register.rs @@ -213,8 +213,6 @@ pub(crate) fn register(def: RegDef) -> Result { ); impl ::kernel::io::register::Register for #name { - type Storage = #storage; - const OFFSET: usize = #offset; } )); From 2a750c56179b416f3501c2644f787265ad7a6785 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:29 +0100 Subject: [PATCH 015/110] rust: io: support register projections `IoLoc`s themselves just describe a projection from a region to a concrete register. Thus, support it in `io_project` macro too. Also, update methods that operate on `IoLoc` to use I/O projection. Documentation of `io_project!` is not expanded yet as the example works better when `register!` type can specify base type. `io_read!` and `io_write!` gains the ability to operate on registers as corollary of the capability of `io_project!`. Examples are not added because `read` and `write` is still preferably used instead. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-5-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 56 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 18c71bd765b2..8df268969566 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -857,8 +857,7 @@ fn try_read(self, location: L) -> Result L: IoLoc, Self::Backend: IoCapable<::Repr>, { - let view = io_view::(self, location.offset())?; - Ok(view.read_val()) + Ok(io_read!(self, try: location)) } /// Generic fallible write with runtime bounds check. @@ -891,8 +890,7 @@ fn try_write(self, location: L, value: T) -> Result L: IoLoc, Self::Backend: IoCapable<::Repr>, { - let view = io_view::(self, location.offset())?; - view.write_val(value); + io_write!(self, try: location, value); Ok(()) } @@ -971,7 +969,7 @@ fn try_update(self, location: L, f: F) -> Result Self::Backend: IoCapable<::Repr>, F: FnOnce(T) -> T, { - let view = io_view::(self, location.offset())?; + let view = io_project!(self, try: location); view.write_val(f(view.read_val())); Ok(()) } @@ -1004,8 +1002,7 @@ fn read(self, location: L) -> T L: IoLoc, Self::Backend: IoCapable<::Repr>, { - let view = io_view_assert::(self, location.offset()); - view.read_val() + io_read!(self, build: location) } /// Generic infallible write with compile-time bounds check. @@ -1036,8 +1033,7 @@ fn write(self, location: L, value: T) L: IoLoc, Self::Backend: IoCapable<::Repr>, { - let view = io_view_assert::(self, location.offset()); - view.write_val(value) + io_write!(self, build: location, value); } /// Generic infallible write of a fully-located register value. @@ -1114,7 +1110,7 @@ fn update(self, location: L, f: F) Self::Backend: IoCapable<::Repr>, F: FnOnce(T) -> T, { - let view = io_view_assert::(self, location.offset()); + let view = io_project!(self, build: location); view.write_val(f(view.read_val())); } } @@ -1659,6 +1655,25 @@ pub unsafe fn project_view( // SAFETY: Per safety requirement. unsafe { T::Backend::project_view::(self.0, ptr) } } + + #[inline(always)] + pub fn try_project_loc( + self, + location: L, + ) -> Result<::View<'a, U>> + where + L: IoLoc, + { + io_view::<_, U>(self.0, location.offset()) + } + + #[inline(always)] + pub fn project_loc(self, location: L) -> ::View<'a, U> + where + L: IoLoc, + { + io_view_assert::<_, U>(self.0, location.offset()) + } } /// Project an I/O type to a subview of it. @@ -1686,6 +1701,21 @@ pub unsafe fn project_view( #[macro_export] #[doc(hidden)] macro_rules! io_project { + // Register projection + ($io:expr, try: $ioloc:expr) => {{ + #[allow(unused)] + use $crate::io::IoBase as _; + let view = $crate::io::ProjectHelper($io.as_view()); + view.try_project_loc($ioloc)? + }}; + ($io:expr, build: $ioloc:expr) => {{ + #[allow(unused)] + use $crate::io::IoBase as _; + let view = $crate::io::ProjectHelper($io.as_view()); + view.project_loc($ioloc) + }}; + + // Field or index projection ($io:expr, $($proj:tt)*) => {{ #[allow(unused)] use $crate::io::IoBase as _; @@ -1756,6 +1786,12 @@ macro_rules! io_write { (@parse [$io:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => { $crate::io_write!(@parse [$io] [$($proj)* [$flavor: $index]] [$($rest)*]) }; + (@parse [$io:expr] [] [try: $ioloc:expr, $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [try: $ioloc] [, $($rest)*]) + }; + (@parse [$io:expr] [] [build: $ioloc:expr, $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [build: $ioloc] [, $($rest)*]) + }; ($io:expr, $($rest:tt)*) => { $crate::io_write!(@parse [$io] [] [$($rest)*]) }; From 12c78173d8b21fd53cf98e456c14b005e6a3675e Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:30 +0100 Subject: [PATCH 016/110] rust: io: register: allow explicit base type specification Currently registers work for all untyped I/O regions, which is not ideal. It allows registers defined for device A to work for another device B and there is no safeguarding at all. All users of the `register!` macro know what type it will be operating on, and that type is consistent across the driver. Therefore, add a `base` parameter to `register!`. Currently this parameter is unused in the generated code; it will be used when all users of `register!` is converted to gain the parameter. Signed-off-by: Gary Guo Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-6-5552b1d59525@garyguo.net [ Remove unnecessary #[allow(unused)] from the 'base' field. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 4 +++ rust/kernel/io/register.rs | 51 ++++++++++++++++++++++++++++++++------ rust/macros/io/register.rs | 22 +++++++++++++++- 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 8df268969566..05c7fa12fdbe 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -909,6 +909,8 @@ fn try_write(self, location: L, value: T) -> Result /// }; /// /// register! { + /// base: Region; + /// /// VERSION(u32) @ 0x100 { /// 15:8 major; /// 7:0 minor; @@ -1051,6 +1053,8 @@ fn write(self, location: L, value: T) /// }; /// /// register! { + /// base: Region<0x1000>; + /// /// VERSION(u32) @ 0x100 { /// 15:8 major; /// 7:0 minor; diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index 45d2e73a92e7..49a5fe069e47 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -13,9 +13,14 @@ //! # Simple example //! //! ```no_run -//! use kernel::io::register; +//! use kernel::io::{ +//! register, +//! Region, +//! }; //! //! register! { +//! base: Region<0x1000>; +//! //! /// Basic information about the chip. //! pub BOOT_0(u32) @ 0x00000100 { //! /// Vendor ID. @@ -55,11 +60,14 @@ //! register, //! Io, //! IoLoc, +//! Region, //! }, //! num::Bounded, //! }; -//! # use kernel::io::{Mmio, Region}; +//! # use kernel::io::Mmio; //! # register! { +//! # base: Region<0x1000>; +//! # //! # pub BOOT_0(u32) @ 0x00000100 { //! # 15:8 vendor_id; //! # 7:4 major_revision; @@ -429,11 +437,14 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// io::{ /// register, /// Io, +/// Region, /// }, /// }; -/// # use kernel::io::{Mmio, Region}; +/// # use kernel::io::Mmio; /// /// register! { +/// base: Region<0x1000>; +/// /// FIXED_REG(u32) @ 0x100 { /// 15:8 high_byte; /// 7:0 low_byte; @@ -464,9 +475,14 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// the context: /// /// ```no_run -/// use kernel::io::register; +/// use kernel::io::{ +/// register, +/// Region, +/// }; /// /// register! { +/// base: Region<0x1000>; +/// /// /// Scratch register. /// pub SCRATCH(u32) @ 0x00000200 { /// 31:0 value; @@ -516,6 +532,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// /// ```ignore /// register! { +/// ... /// pub RELATIVE_REG(u32) @ Base + 0x80 { /// ... /// } @@ -542,9 +559,10 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// WithBase, /// }, /// Io, +/// Region, /// }, /// }; -/// # use kernel::io::{Mmio, Region}; +/// # use kernel::io::Mmio; /// /// // Type used to identify the base. /// pub struct CpuCtlBase; @@ -563,6 +581,8 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// /// // This makes `CPU_CTL` accessible from all implementors of `RegisterBase`. /// register! { +/// base: Region<0x1000>; +/// /// /// CPU core control. /// pub CPU_CTL(u32) @ CpuCtlBase + 0x10 { /// 0:0 start; @@ -579,6 +599,8 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// /// // Aliases can also be defined for relative register. /// register! { +/// base: Region<0x1000>; +/// /// /// Alias to CPU core control. /// pub CPU_CTL_ALIAS(u32) => CpuCtlBase + CPU_CTL { /// /// Start the aliased CPU core. @@ -621,15 +643,18 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// register, /// register::Array, /// Io, +/// Region, /// }, /// }; -/// # use kernel::io::{Mmio, Region}; +/// # use kernel::io::Mmio; /// # fn get_scratch_idx() -> usize { /// # 0x15 /// # } /// /// // Array of 64 consecutive registers with the same layout starting at offset `0x80`. /// register! { +/// base: Region<0x1000>; +/// /// /// Scratch registers. /// pub SCRATCH(u32)[64] @ 0x00000080 { /// 31:0 value; @@ -655,6 +680,8 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// // Alias to a specific register in an array. /// // Here `SCRATCH[8]` is used to convey the firmware exit code. /// register! { +/// base: Region<0x1000>; +/// /// /// Firmware exit status code. /// pub FIRMWARE_STATUS(u32) => SCRATCH[8] { /// 7:0 status; @@ -667,6 +694,8 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the /// // registers of the two declarations below are interleaved. /// register! { +/// base: Region<0x1000>; +/// /// /// Scratch registers bank 0. /// pub SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ 0x000000c0 { /// 31:0 value; @@ -688,6 +717,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// /// ```ignore /// register! { +/// ... /// pub RELATIVE_REGISTER_ARRAY(u8)[10, stride = 4] @ Base + 0x100 { /// ... /// } @@ -707,9 +737,10 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// WithBase, /// }, /// Io, +/// Region, /// }, /// }; -/// # use kernel::io::{Mmio, Region}; +/// # use kernel::io::Mmio; /// # fn get_scratch_idx() -> usize { /// # 0x15 /// # } @@ -731,6 +762,8 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// /// // 64 per-cpu scratch registers, arranged as a contiguous array. /// register! { +/// base: Region<0x1000>; +/// /// /// Per-CPU scratch registers. /// pub CPU_SCRATCH(u32)[64] @ CpuCtlBase + 0x00000080 { /// 31:0 value; @@ -758,6 +791,8 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// /// // Alias to `SCRATCH[8]` used to convey the firmware exit code. /// register! { +/// base: Region<0x1000>; +/// /// /// Per-CPU firmware exit status code. /// pub CPU_FIRMWARE_STATUS(u32) => CpuCtlBase + CPU_SCRATCH[8] { /// 7:0 status; @@ -768,6 +803,8 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the /// // registers of the two declarations below are interleaved. /// register! { +/// base: Region<0x1000>; +/// /// /// Scratch registers bank 0. /// pub CPU_SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d00 { /// 31:0 value; diff --git a/rust/macros/io/register.rs b/rust/macros/io/register.rs index 2e4cdabb2795..403f56692f6b 100644 --- a/rust/macros/io/register.rs +++ b/rust/macros/io/register.rs @@ -29,6 +29,7 @@ }; mod kw { + syn::custom_keyword!(base); syn::custom_keyword!(stride); } @@ -152,22 +153,41 @@ fn parse(input: syn::parse::ParseStream<'_>) -> Result { } pub(crate) struct RegDef { + base: Option, regs: Vec, } impl Parse for RegDef { fn parse(input: syn::parse::ParseStream<'_>) -> Result { + let base = if input.peek(kw::base) { + let _: kw::base = input.parse()?; + let _: Token![:] = input.parse()?; + let base = input.parse()?; + let _: Token![;] = input.parse()?; + Some(base) + } else { + None + }; let mut regs = Vec::new(); while !input.is_empty() { regs.push(input.parse()?); } - Ok(RegDef { regs }) + Ok(RegDef { base, regs }) } } pub(crate) fn register(def: RegDef) -> Result { let mut outputs = TokenStream::new(); + if let Some(base) = &def.base { + outputs.extend(quote_spanned!(base.span() => + const _: () = { + #[allow(unused)] + type Base = #base; + }; + )); + } + for reg in def.regs { let Reg { attrs, From f7477c8324041757d2969d230be67e77a9334437 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:31 +0100 Subject: [PATCH 017/110] gpu: nova-core: specify base type for registers All registers use the same base type, which is `::Target`. Thus add the base parameter to `register!` invocation. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-7-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/driver.rs | 1 + drivers/gpu/nova-core/fb/regs.rs | 12 ++++++++++ drivers/gpu/nova-core/gsp/regs.rs | 9 +++++++- drivers/gpu/nova-core/regs.rs | 37 ++++++++++++++++++++++++++++++- drivers/gpu/nova-core/vbios.rs | 11 ++++++++- 5 files changed, 67 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index bbd93959e0b2..cf3534dd47d4 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -37,6 +37,7 @@ pub(crate) struct NovaCore<'bound> { const BAR0_SIZE: usize = SZ_16M; pub(crate) type Bar0<'a> = &'a pci::Bar<'a, BAR0_SIZE>; +pub(crate) type NovaRegisters = kernel::io::Region; kernel::pci_device_table!( PCI_TABLE, diff --git a/drivers/gpu/nova-core/fb/regs.rs b/drivers/gpu/nova-core/fb/regs.rs index 95adbe124a30..c27582e376e2 100644 --- a/drivers/gpu/nova-core/fb/regs.rs +++ b/drivers/gpu/nova-core/fb/regs.rs @@ -5,9 +5,13 @@ sizes::SizeConstants, // }; +use crate::driver::NovaRegisters; + // PDISP register! { + base: NovaRegisters; + pub(super) NV_PDISP_VGA_WORKSPACE_BASE(u32) @ 0x00625f04 { /// VGA workspace base address divided by 0x10000. 31:8 addr; @@ -30,6 +34,8 @@ pub(super) fn vga_workspace_addr(self) -> Option { // PFB register! { + base: NovaRegisters; + /// Low bits of the physical system memory address used by the GPU to perform sysmembar /// operations (see [`crate::fb::SysmemFlush`]). pub(super) NV_PFB_NISO_FLUSH_SYSMEM_ADDR(u32) @ 0x00100c10 { @@ -65,6 +71,8 @@ pub(super) fn vga_workspace_addr(self) -> Option { pub(super) struct Hshub0Base(()); register! { + base: NovaRegisters; + // GB10x sysmem flush registers, relative to the HSHUB0 base. GB10x routes sysmembar // through a primary and an EG (egress) pair that must both be programmed to the same // address. Hardware ignores bits 7:0 of each LO register. The boot path uses a fixed @@ -87,6 +95,8 @@ pub(super) fn vga_workspace_addr(self) -> Option { } register! { + base: NovaRegisters; + // GB20x FBHUB0 sysmem flush registers. Unlike the older // NV_PFB_NISO_FLUSH_SYSMEM_ADDR registers, which encode the address with an // 8-bit right-shift, these take the raw address split into lower and upper @@ -101,6 +111,8 @@ pub(super) fn vga_workspace_addr(self) -> Option { } register! { + base: NovaRegisters; + /// Low bits of the physical system memory address used by the GPU to perform /// sysmembar operations on Hopper. /// diff --git a/drivers/gpu/nova-core/gsp/regs.rs b/drivers/gpu/nova-core/gsp/regs.rs index 9a48aa87e7fb..3c410d65e8e4 100644 --- a/drivers/gpu/nova-core/gsp/regs.rs +++ b/drivers/gpu/nova-core/gsp/regs.rs @@ -2,11 +2,16 @@ use kernel::io::register; -use crate::regs::NV_PBUS_SW_SCRATCH; +use crate::{ + driver::NovaRegisters, + regs::NV_PBUS_SW_SCRATCH, // +}; // PGSP register! { + base: NovaRegisters; + pub(super) NV_PGSP_QUEUE_HEAD(u32) @ 0x00110c00 { 31:0 address; } @@ -15,6 +20,8 @@ // PBUS register! { + base: NovaRegisters; + /// Scratch register 0xe used as FRTS firmware error code. pub(super) NV_PBUS_SW_SCRATCH_0E_FRTS_ERR(u32) => NV_PBUS_SW_SCRATCH[0xe] { 31:16 frts_err_code; diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index caeef4d85874..1af073f3861f 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -13,7 +13,10 @@ }; use crate::{ - driver::Bar0, + driver::{ + Bar0, + NovaRegisters, // + }, falcon::{ DmaTrfCmdSize, FalconCoreRev, @@ -37,6 +40,8 @@ // PMC register! { + base: NovaRegisters; + /// Basic revision information about the GPU. pub(crate) NV_PMC_BOOT_0(u32) @ 0x00000000 { /// Lower bits of the architecture. @@ -108,6 +113,8 @@ fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { // PBUS register! { + base: NovaRegisters; + pub(crate) NV_PBUS_SW_SCRATCH(u32)[64] @ 0x00001400 {} } @@ -121,6 +128,8 @@ fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { // number. register! { + base: NovaRegisters; + /// Boot Sequence Interface (BSI) register used to determine /// if GSP reload/resume has completed during the boot process. pub(crate) NV_PGC6_BSI_SECURE_SCRATCH_14(u32) @ 0x001180f8 { @@ -175,6 +184,8 @@ pub(crate) fn usable_fb_size(self) -> u64 { pub(crate) const NV_FUSE_OPT_FPF_SIZE: usize = 16; register! { + base: NovaRegisters; + pub(crate) NV_FUSE_OPT_FPF_NVDEC_UCODE1_VERSION(u32)[NV_FUSE_OPT_FPF_SIZE] @ 0x00824100 { 15:0 data => u16; } @@ -191,6 +202,8 @@ pub(crate) fn usable_fb_size(self) -> u64 { // PFALCON register! { + base: NovaRegisters; + pub(crate) NV_PFALCON_FALCON_IRQSCLR(u32) @ PFalconBase + 0x00000004 { 6:6 swgen0 => bool; 4:4 halt => bool; @@ -392,6 +405,8 @@ pub(crate) fn mem_scrubbing_done(self) -> bool { /* PFALCON2 */ register! { + base: NovaRegisters; + pub(crate) NV_PFALCON2_FALCON_MOD_SEL(u32) @ PFalcon2Base + 0x00000180 { 7:0 algo ?=> FalconModSelAlgo; } @@ -414,6 +429,8 @@ pub(crate) fn mem_scrubbing_done(self) -> bool { // PRISCV register! { + base: NovaRegisters; + /// RISC-V status register for debug (Turing and GA100 only). /// Reflects current RISC-V core status. pub(crate) NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS(u32) @ PFalcon2Base + 0x00000240 { @@ -439,6 +456,8 @@ pub(crate) fn mem_scrubbing_done(self) -> bool { // These registers manage falcon EMEM communication queues. register! { + base: NovaRegisters; + pub(crate) NV_PFSP_QUEUE_HEAD(u32)[8] @ 0x008f2c00 { 31:0 address => u32; } @@ -462,9 +481,13 @@ pub(crate) fn mem_scrubbing_done(self) -> bool { pub(crate) mod gm107 { use kernel::io::register; + use crate::driver::NovaRegisters; + // FUSE register! { + base: NovaRegisters; + pub(crate) NV_FUSE_STATUS_OPT_DISPLAY(u32) @ 0x00021c04 { 0:0 display_disabled => bool; } @@ -474,9 +497,13 @@ pub(crate) mod gm107 { pub(crate) mod ga100 { use kernel::io::register; + use crate::driver::NovaRegisters; + // FUSE register! { + base: NovaRegisters; + pub(crate) NV_FUSE_STATUS_OPT_DISPLAY(u32) @ 0x00820c04 { 0:0 display_disabled => bool; } @@ -488,9 +515,13 @@ pub(crate) mod ga100 { pub(crate) mod gh100 { use kernel::io::register; + use crate::driver::NovaRegisters; + // PTHERM register! { + base: NovaRegisters; + pub(crate) NV_THERM_I2CS_SCRATCH(u32) @ 0x000200bc { 31:0 data; } @@ -505,9 +536,13 @@ pub(crate) mod gh100 { pub(crate) mod gb202 { use kernel::io::register; + use crate::driver::NovaRegisters; + // PTHERM register! { + base: NovaRegisters; + pub(crate) NV_THERM_I2CS_SCRATCH(u32) @ 0x00ad00bc { 31:0 data; } diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index c03650ee5226..9c214b9f4dd9 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -16,7 +16,10 @@ }; use crate::{ - driver::Bar0, + driver::{ + Bar0, + NovaRegisters, // + }, firmware::{ fwsec::Bcrt30Rsa3kSignature, FalconUCodeDesc, @@ -92,12 +95,16 @@ impl<'a> VbiosIterator<'a> { fn rom_offset(dev: &device::Device, bar0: Bar0<'_>) -> Result { // IFR Header in VBIOS. register! { + base: NovaRegisters; + NV_PBUS_IFR_FMT_FIXED0(u32) @ 0x300000 { 31:0 signature; } } register! { + base: NovaRegisters; + NV_PBUS_IFR_FMT_FIXED1(u32) @ 0x300004 { 30:16 fixed_data_size; 15:8 version => u8; @@ -105,6 +112,8 @@ fn rom_offset(dev: &device::Device, bar0: Bar0<'_>) -> Result { } register! { + base: NovaRegisters; + NV_PBUS_IFR_FMT_FIXED2(u32) @ 0x300008 { 19:0 total_data_size; } From be5ffd2696e4e0359ba78fde75eef739befa4fab Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:32 +0100 Subject: [PATCH 018/110] drm/tyr: specify base type for registers All registers use the same base type, which is `::Target`. Thus add the base parameter to `register!` invocation. Signed-off-by: Gary Guo Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-8-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/tyr/driver.rs | 1 + drivers/gpu/drm/tyr/fw.rs | 5 +++- drivers/gpu/drm/tyr/regs.rs | 45 ++++++++++++++++++++++++++++++++++- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index bfb0ba19caff..730b84e37a54 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -46,6 +46,7 @@ }; pub(crate) type IoMem<'a> = kernel::io::mem::IoMem<'a, SZ_2M>; +pub(crate) type TyrRegisters = kernel::io::Region; pub(crate) struct TyrDrmDriver; diff --git a/drivers/gpu/drm/tyr/fw.rs b/drivers/gpu/drm/tyr/fw.rs index 47d25c901bd0..7edb5eff1707 100644 --- a/drivers/gpu/drm/tyr/fw.rs +++ b/drivers/gpu/drm/tyr/fw.rs @@ -39,7 +39,8 @@ use crate::{ driver::{ IoMem, - TyrDrmDevice, // + TyrDrmDevice, + TyrRegisters, // }, fw::parser::{ FwParser, @@ -101,6 +102,8 @@ fn from(value: CacheMode) -> Self { } register! { + base: TyrRegisters; + #[allow(non_upper_case_globals)] pub(super) SectionFlags(u32) @ 0x0 { 0:0 read => bool; diff --git a/drivers/gpu/drm/tyr/regs.rs b/drivers/gpu/drm/tyr/regs.rs index a62724378ced..0c419c4e1186 100644 --- a/drivers/gpu/drm/tyr/regs.rs +++ b/drivers/gpu/drm/tyr/regs.rs @@ -57,7 +57,11 @@ pub(crate) mod gpu_control { uapi, // }; + use crate::driver::TyrRegisters; + register! { + base: TyrRegisters; + /// GPU identification register. pub(crate) GPU_ID(u32) @ 0x0 { /// Status of the GPU release. @@ -315,6 +319,8 @@ fn from(mode: FlushMode) -> Self { } register! { + base: TyrRegisters; + /// GPU command register. /// /// Use the constructor methods to create commands: @@ -380,6 +386,8 @@ pub(crate) fn clear_fault() -> Self { } register! { + base: TyrRegisters; + /// GPU status register. Read only. pub(crate) GPU_STATUS(u32) @ 0x34 { /// GPU active, a 1-bit boolean flag. @@ -463,6 +471,8 @@ fn from(access: AccessType) -> Self { } register! { + base: TyrRegisters; + /// GPU fault status register. Read only. pub(crate) GPU_FAULTSTATUS(u32) @ 0x3c { /// Exception type. @@ -768,6 +778,8 @@ fn from(mode: CoherencyMode) -> Self { } register! { + base: TyrRegisters; + /// Coherency enable. An index of which coherency protocols should be used. /// This register only selects the protocol for coherency messages on the /// interconnect. This is not to enable or disable coherency controlled by MMU. @@ -808,6 +820,8 @@ fn from(mode: McuControlMode) -> Self { } register! { + base: TyrRegisters; + /// MCU control. pub(crate) MCU_CONTROL(u32) @ 0x700 { /// Request MCU state change. @@ -849,6 +863,8 @@ fn from(status: McuStatus) -> Self { } register! { + base: TyrRegisters; + /// MCU status. Read only. pub(crate) MCU_STATUS(u32) @ 0x704 { /// Read current state of MCU. @@ -862,7 +878,11 @@ fn from(status: McuStatus) -> Self { pub(crate) mod job_control { use kernel::register; + use crate::driver::TyrRegisters; + register! { + base: TyrRegisters; + /// Raw status of job interrupts. /// /// Write to this register to trigger these interrupts. @@ -912,7 +932,11 @@ pub(crate) mod job_control { pub(crate) mod mmu_control { use kernel::register; + use crate::driver::TyrRegisters; + register! { + base: TyrRegisters; + /// IRQ sources raw status. /// /// This register contains the raw unmasked interrupt sources for MMU status and exception @@ -966,9 +990,10 @@ pub(crate) mod mmu_as_control { prelude::*, register, // }; - use pin_init::Zeroable; + use crate::driver::TyrRegisters; + /// Maximum number of hardware address space slots. /// The actual number of slots available is usually lower. pub(crate) const MAX_AS: usize = 16; @@ -977,6 +1002,8 @@ pub(crate) mod mmu_as_control { const STRIDE: usize = 0x40; register! { + base: TyrRegisters; + /// Translation table base address. A 64-bit pointer. /// /// This field contains the address of the top level of a translation table structure. @@ -1104,6 +1131,8 @@ fn from(val: MemoryType) -> Self { } register! { + base: TyrRegisters; + /// Stage 1 memory attributes (8-bit bitfield). /// /// This is not an actual register, but a bitfield definition used by the MEMATTR @@ -1137,6 +1166,8 @@ fn from(val: MMU_MEMATTR_STAGE1) -> Self { } register! { + base: TyrRegisters; + /// Memory attributes. /// /// Each address space can configure up to 8 different memory attribute profiles. @@ -1292,6 +1323,8 @@ pub(crate) fn from_mair(mair: u64) -> Self { } register! { + base: TyrRegisters; + /// Lock region address for each address space. pub(crate) LOCKADDR(u64)[MAX_AS, stride = STRIDE] @ 0x2410 { /// Lock region size. @@ -1353,6 +1386,8 @@ fn from(cmd: MmuCommand) -> Self { } register! { + base: TyrRegisters; + /// MMU command register for each address space. Write only. pub(crate) COMMAND(u32)[MAX_AS, stride = STRIDE] @ 0x2418 { 7:0 command ?=> MmuCommand; @@ -1480,6 +1515,8 @@ fn from(access: MmuAccessType) -> Self { } register! { + base: TyrRegisters; + /// Fault status register for each address space. Read only. pub(crate) FAULTSTATUS(u32)[MAX_AS, stride = STRIDE] @ 0x241c { /// Exception type. @@ -1705,6 +1742,8 @@ fn from(sh: PtwShareability) -> Self { } register! { + base: TyrRegisters; + /// Translation configuration and control. pub(crate) TRANSCFG(u64)[MAX_AS, stride = STRIDE] @ 0x2430 { /// Address space mode. @@ -1760,6 +1799,8 @@ fn from(sh: PtwShareability) -> Self { pub(crate) mod doorbell_block { use kernel::register; + use crate::driver::TyrRegisters; + /// Number of doorbells available. pub(crate) const NUM_DOORBELLS: usize = 64; @@ -1770,6 +1811,8 @@ pub(crate) mod doorbell_block { const STRIDE: usize = 0x10000; register! { + base: TyrRegisters; + /// Doorbell request register. Write-only. pub(crate) DOORBELL(u32)[NUM_DOORBELLS, stride = STRIDE] @ 0x80000 { /// Doorbell set. Writing 1 triggers the doorbell. From 9b27311fa940839b0839e5a96c27ffa24d8374cf Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:33 +0100 Subject: [PATCH 019/110] samples: rust: pci: specify base type for registers The `register!` macro is going to require explicit base type, specify it for both `register!` usages in PCI sample driver. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-9-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- samples/rust/rust_driver_pci.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/samples/rust/rust_driver_pci.rs b/samples/rust/rust_driver_pci.rs index 2282191e6292..13b035a95756 100644 --- a/samples/rust/rust_driver_pci.rs +++ b/samples/rust/rust_driver_pci.rs @@ -23,6 +23,8 @@ mod regs { use super::*; register! { + base: kernel::io::Region; + pub(super) TEST(u8) @ 0x0 { 7:0 index => TestIndex; } @@ -102,6 +104,8 @@ fn config_space(pdev: &pci::Device) { // Some PCI configuration space registers. register! { + base: pci::Normal; + VENDOR_ID(u16) @ 0x0 { 15:0 vendor_id; } From 4f620881808e9d3d951343e2edc9c8c41286391a Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:34 +0100 Subject: [PATCH 020/110] rust: io: register: make register have a typed base Previously `register!` defined registers can be used on any untyped I/O regions. With all users specifying their desired register type now, propagate the specified type and restrict I/O access only when type matches. Also, add an `io_project!` example which is enabled by this change. Signed-off-by: Gary Guo Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-10-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 13 +++++++++ rust/kernel/io/register.rs | 54 +++++++++++++++++++++++++++----------- rust/macros/io/register.rs | 44 ++++++++++++------------------- 3 files changed, 68 insertions(+), 43 deletions(-) diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 05c7fa12fdbe..7a0e0ebc6c85 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -1685,21 +1685,34 @@ pub fn project_loc(self, location: L) -> ::View<' /// The syntax is of form `io_project!(io, proj)` where `io` is an expression to a type that /// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!). /// +/// `io_project!` can also project to a subview of registers defined with [`register!`] macro. +/// Register projection has syntax `io_project!(io, try: REGISTER)` for fallible projection and +/// `io_project!(io, build: REGISTER)` for infallible projection. +/// /// # Examples /// /// ``` /// use kernel::io::{ /// io_project, +/// register, /// Mmio, /// }; /// #[repr(C)] /// struct MyStruct { field: u32, } /// +/// register! { +/// base: MyStruct; +/// FIELD(u32) @ 0 { +/// 31:0 val; +/// } +/// } +/// /// # fn test(mmio: Mmio<'_, [MyStruct]>) -> Result { /// // let mmio: Mmio<[MyStruct]>; /// let field: Mmio<'_, u32> = io_project!(mmio, [try: 1].field); /// let whole: Mmio<'_, MyStruct> = io_project!(mmio, [try: 2]); /// let nested: Mmio<'_, u32> = io_project!(whole, .field); +/// let reg: Mmio<'_, FIELD> = io_project!(whole, build: FIELD); /// # Ok::<(), Error>(()) } /// ``` #[macro_export] diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index 49a5fe069e47..58dcf651dfd0 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -121,10 +121,11 @@ io::IoLoc, // }; -use super::Region; - /// Trait implemented by all registers. pub trait Register: Sized { + /// Base type for this register. + type Base: ?Sized; + /// Start offset of the register. /// /// The interpretation of this offset depends on the type of the register. @@ -136,9 +137,9 @@ pub trait FixedRegister: Register {} /// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when /// passing a [`FixedRegister`] value. -impl IoLoc, T> for () +impl IoLoc for () where - T: FixedRegister, + T: FixedRegister, { #[inline(always)] fn offset(self) -> usize { @@ -148,9 +149,9 @@ fn offset(self) -> usize { /// A [`FixedRegister`] carries its location in its type. Thus `FixedRegister` values can be used /// as an [`IoLoc`]. -impl IoLoc, T> for T +impl IoLoc for T where - T: FixedRegister, + T: FixedRegister, { #[inline(always)] fn offset(self) -> usize { @@ -171,9 +172,9 @@ pub const fn new() -> Self { } } -impl IoLoc, T> for FixedRegisterLoc +impl IoLoc for FixedRegisterLoc where - T: FixedRegister, + T: FixedRegister, { #[inline(always)] fn offset(self) -> usize { @@ -240,9 +241,9 @@ const fn offset(self) -> usize { } } -impl IoLoc, T> for RelativeRegisterLoc +impl IoLoc for RelativeRegisterLoc where - T: RelativeRegister, + T: RelativeRegister, B: RegisterBase + ?Sized, { #[inline(always)] @@ -282,9 +283,9 @@ pub fn try_new(idx: usize) -> Option { } } -impl IoLoc, T> for RegisterArrayLoc +impl IoLoc for RegisterArrayLoc where - T: RegisterArray, + T: RegisterArray, { #[inline(always)] fn offset(self) -> usize { @@ -367,9 +368,9 @@ pub fn try_at(self, idx: usize) -> Option> { } } -impl IoLoc, T> for RelativeRegisterArrayLoc +impl IoLoc for RelativeRegisterArrayLoc where - T: RelativeRegisterArray, + T: RelativeRegisterArray, B: RegisterBase + ?Sized, { #[inline(always)] @@ -393,9 +394,9 @@ pub trait LocatedRegister { fn into_io_op(self) -> (Self::Location, Self::Value); } -impl LocatedRegister> for T +impl LocatedRegister for T where - T: FixedRegister, + T: FixedRegister, { type Location = FixedRegisterLoc; type Value = T; @@ -406,6 +407,27 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { } } +/// Helper function for register alias implementation. +/// +/// This is used to enforce base matching. +#[doc(hidden)] +#[inline(always)] // for const eval only +pub const fn alias_offset>() -> usize { + Alias::OFFSET +} + +/// Helper function for register element alias implementation. +/// +/// This is used to enforce base matching and provide bounds checking. +#[doc(hidden)] +#[inline(always)] // for const eval only +pub const fn element_alias_offset>( + idx: usize, +) -> usize { + assert!(idx < Alias::SIZE); + Alias::OFFSET + idx * Alias::STRIDE +} + /// Defines a dedicated type for a register, including getter and setter methods for its fields and /// methods to read and write it from an [`Io`](kernel::io::Io) region. /// diff --git a/rust/macros/io/register.rs b/rust/macros/io/register.rs index 403f56692f6b..a05671cba2dc 100644 --- a/rust/macros/io/register.rs +++ b/rust/macros/io/register.rs @@ -19,6 +19,7 @@ spanned::Spanned, token, Attribute, + Error, Expr, Ident, Path, @@ -153,21 +154,23 @@ fn parse(input: syn::parse::ParseStream<'_>) -> Result { } pub(crate) struct RegDef { - base: Option, + base: Type, regs: Vec, } impl Parse for RegDef { fn parse(input: syn::parse::ParseStream<'_>) -> Result { - let base = if input.peek(kw::base) { - let _: kw::base = input.parse()?; - let _: Token![:] = input.parse()?; - let base = input.parse()?; - let _: Token![;] = input.parse()?; - Some(base) - } else { - None - }; + let _: kw::base = input.parse().map_err(|e| { + Error::new( + e.span(), + "a base type needs to be specified for `register!` invocation with `base: ty;`", + ) + })?; + + let _: Token![:] = input.parse()?; + let base = input.parse()?; + let _: Token![;] = input.parse()?; + let mut regs = Vec::new(); while !input.is_empty() { regs.push(input.parse()?); @@ -179,15 +182,7 @@ fn parse(input: syn::parse::ParseStream<'_>) -> Result { pub(crate) fn register(def: RegDef) -> Result { let mut outputs = TokenStream::new(); - if let Some(base) = &def.base { - outputs.extend(quote_spanned!(base.span() => - const _: () = { - #[allow(unused)] - type Base = #base; - }; - )); - } - + let base = &def.base; for reg in def.regs { let Reg { attrs, @@ -208,18 +203,12 @@ pub(crate) fn register(def: RegDef) -> Result { RegOffset::Fixed { offset } => quote!(#offset), RegOffset::Alias { alias } => { quote_spanned!(alias.span().resolved_at(span) => - <#alias as ::kernel::io::register::Register>::OFFSET + ::kernel::io::register::alias_offset::<#base, #alias>() ) } RegOffset::ElementAlias { alias, idx } => { - outputs.extend(quote_spanned!(idx.span().resolved_at(span) => - ::kernel::build_assert::static_assert!( - (#idx) < <#alias as ::kernel::io::register::RegisterArray>::SIZE - ); - )); quote_spanned!(alias.span().resolved_at(span) => - <#alias as ::kernel::io::register::Register>::OFFSET - + (#idx) * <#alias as ::kernel::io::register::RegisterArray>::STRIDE + ::kernel::io::register::element_alias_offset::<#base, #alias>(#idx) ) } }; @@ -233,6 +222,7 @@ pub(crate) fn register(def: RegDef) -> Result { ); impl ::kernel::io::register::Register for #name { + type Base = #base; const OFFSET: usize = #offset; } )); From c09d3a26e1b810d43ee8a15c4f09ee752fee55a6 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:35 +0100 Subject: [PATCH 021/110] rust: io: register: support fixed offset register without bitfield Add a rule to allow creating `IoLoc` in `register!()` using an existing type and not create a bitfield. Add an example to demonstrate this for FIFO registers. This rule is also going to be used to create subregions for registers; the example of doing so will be added later when relative registers are removed. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-11-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io/register.rs | 31 ++++++++++++++ rust/macros/io/register.rs | 82 ++++++++++++++++++++++++++------------ 2 files changed, 87 insertions(+), 26 deletions(-) diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index 58dcf651dfd0..b9cb4fce8628 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -182,6 +182,24 @@ fn offset(self) -> usize { } } +// Provides a `IoLoc` impl that for a fixed offset. +#[doc(hidden)] +pub struct OffsetLoc(usize, PhantomData<(T, Base)>); + +impl OffsetLoc { + #[inline] + pub const fn new(offset: usize) -> Self { + Self(offset, PhantomData) + } +} + +impl IoLoc for OffsetLoc { + #[inline(always)] + fn offset(self) -> usize { + self.0 + } +} + /// Trait providing a base address to be added to the offset of a relative register to obtain /// its actual offset. /// @@ -520,6 +538,19 @@ pub const fn element_alias_offset; +/// +/// /// UART RX register. +/// pub UART_RX: u8 @ 0x100; +/// } +/// ``` +/// /// ## Relative registers /// /// Relative registers can be instantiated several times at a relative offset of a group of bases. diff --git a/rust/macros/io/register.rs b/rust/macros/io/register.rs index a05671cba2dc..e948b0959154 100644 --- a/rust/macros/io/register.rs +++ b/rust/macros/io/register.rs @@ -16,6 +16,7 @@ bracketed, parenthesized, parse::Parse, + parse_quote, spanned::Spanned, token, Attribute, @@ -62,11 +63,11 @@ struct Reg { attrs: Vec, vis: Visibility, name: Ident, - storage: Type, + ty: Type, array: Option, relative_base: Option, offset: RegOffset, - bitfield_args: Group, + bitfield: Option<(Type, Group)>, } impl Parse for Reg { @@ -74,11 +75,22 @@ fn parse(input: syn::parse::ParseStream<'_>) -> Result { let attrs = input.call(Attribute::parse_outer)?; let vis = input.parse()?; let name = input.parse()?; - let storage = { + + let lh = input.lookahead1(); + let (ty, bitfield_storage) = if lh.peek(Token![:]) { + let _: Token![:] = input.parse()?; + (input.parse()?, None) + } else if lh.peek(token::Paren) { let content; parenthesized!(content in input); - content.parse()? + let bitfield_storage = Some(content.parse()?); + + // For bitfields, bitfield macro will generate a type with the same name as `name`. + (parse_quote!(#name), bitfield_storage) + } else { + Err(lh.error())? }; + let array = if input.peek(token::Bracket) { let content; bracketed!(content in input); @@ -133,22 +145,28 @@ fn parse(input: syn::parse::ParseStream<'_>) -> Result { Err(lh.error())? }; - let lh = input.lookahead1(); - let bitfield_args = if lh.peek(token::Brace) { - input.parse()? + let bitfield = if let Some(storage) = bitfield_storage { + let lh = input.lookahead1(); + let args = if lh.peek(token::Brace) { + input.parse()? + } else { + Err(lh.error())? + }; + Some((storage, args)) } else { - Err(lh.error())? + let _: Token![;] = input.parse()?; + None }; Ok(Self { attrs, vis, name, - storage, + ty, array, relative_base, offset, - bitfield_args, + bitfield, }) } } @@ -188,11 +206,11 @@ pub(crate) fn register(def: RegDef) -> Result { attrs, vis, name, - storage, + ty, array, relative_base, offset, - bitfield_args, + bitfield, } = reg; // Use register name's span for generated code, so error messages (if any) can point to it @@ -213,21 +231,33 @@ pub(crate) fn register(def: RegDef) -> Result { } }; - outputs.extend(quote_spanned!(span => - ::kernel::bitfield!( - // `#[allow(non_camel_case_types)]` is added since register names typically use - // `SCREAMING_CASE`. - #[allow(non_camel_case_types)] - #(#attrs)* #vis struct #name(#storage) #bitfield_args - ); + if let Some((storage, args)) = &bitfield { + outputs.extend(quote_spanned!(span => + ::kernel::bitfield!( + // `#[allow(non_camel_case_types)]` is added since register names typically use + // `SCREAMING_CASE`. + #[allow(non_camel_case_types)] + #(#attrs)* #vis struct #name(#storage) #args + ); - impl ::kernel::io::register::Register for #name { - type Base = #base; - const OFFSET: usize = #offset; - } - )); + impl ::kernel::io::register::Register for #name { + type Base = #base; + const OFFSET: usize = #offset; + } + )); + } match array { + None if bitfield.is_none() && relative_base.is_none() => outputs.extend(quote!( + #(#attrs)* #vis const #name: ::kernel::io::register::OffsetLoc<#base, #ty> = + ::kernel::io::register::OffsetLoc::new(#offset); + )), + + _ if bitfield.is_none() => Err(Error::new_spanned( + ty, + "defining without bitfield is not yet supported for this type of register", + ))?, + None => match relative_base { None => outputs.extend(quote_spanned!(span => impl ::kernel::io::register::FixedRegister for #name {} @@ -249,12 +279,12 @@ impl ::kernel::io::register::RelativeRegister for #name {} let stride = if let Some(stride) = &def.stride { outputs.extend(quote_spanned!(stride.span().resolved_at(span) => ::kernel::build_assert::static_assert!( - ::core::mem::size_of::<#storage>() <= #stride + ::core::mem::size_of::<#ty>() <= #stride ); )); quote!(#stride) } else { - quote_spanned!(span => ::core::mem::size_of::<#storage>()) + quote_spanned!(span => ::core::mem::size_of::<#ty>()) }; outputs.extend(quote_spanned!(span => From ea79a50124315589da68cbcaf518455a142d6d23 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:36 +0100 Subject: [PATCH 022/110] gpu: nova-core: use projection for PFALCON and PFALCON2 registers Add fixed size region types `PFalconRegisters` and `PFalcon2Registers` and update PFALCON and PFALCON2 registers to be fixed register on them and not relative registers on `NovaRegisters`. Update `Falcon` struct to store projected views when constructing and access with `self.pfalcon` and `self.pfalcon2`. Signed-off-by: Gary Guo Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-12-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/falcon.rs | 161 ++++++++---------- drivers/gpu/nova-core/falcon/fsp.rs | 63 +++---- drivers/gpu/nova-core/falcon/gsp.rs | 51 +++--- drivers/gpu/nova-core/falcon/hal/ga102.rs | 59 +++---- drivers/gpu/nova-core/falcon/hal/tu102.rs | 9 +- drivers/gpu/nova-core/falcon/sec2.rs | 37 ++-- .../nova-core/firmware/fwsec/bootloader.rs | 18 +- drivers/gpu/nova-core/gsp/hal/tu102.rs | 7 +- drivers/gpu/nova-core/regs.rs | 92 +++++----- 9 files changed, 239 insertions(+), 258 deletions(-) diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index 65cb12d26e2b..e3af59fd57e3 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -14,13 +14,12 @@ io::{ io_project, poll::read_poll_timeout, - register::{ - RegisterBase, - WithBase, // - }, + register::Array, Io, + Mmio, // }, prelude::*, + sizes::SZ_4K, time::Delta, }; @@ -165,18 +164,25 @@ pub(crate) enum FalconFbifMemType with From> { } } -/// Type used to represent the `PFALCON` registers address base for a given falcon engine. -pub(crate) struct PFalconBase(()); +const PFALCON_REGION_SIZE: usize = SZ_4K; +const PFALCON2_REGION_SIZE: usize = SZ_4K; -/// Type used to represent the `PFALCON2` registers address base for a given falcon engine. -pub(crate) struct PFalcon2Base(()); +/// Type used to represent the `PFALCON` registers. +#[repr(align(4))] +#[derive(FromBytes, IntoBytes)] +pub(crate) struct PFalconRegisters([u8; PFALCON_REGION_SIZE]); + +/// Type used to represent the `PFALCON2` registers. +#[repr(align(4))] +#[derive(FromBytes, IntoBytes)] +pub(crate) struct PFalcon2Registers([u8; PFALCON2_REGION_SIZE]); /// Trait defining the parameters of a given Falcon engine. /// /// Each engine provides one base for `PFALCON` and `PFALCON2` registers. -pub(crate) trait FalconEngine: - Send + Sync + RegisterBase + RegisterBase + Sized -{ +pub(crate) trait FalconEngine: Send + Sync + Sized { + fn pfalcon(io: Bar0<'_>) -> Mmio<'_, PFalconRegisters>; + fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, PFalcon2Registers>; } /// Represents a portion of the firmware to be loaded into a particular memory (e.g. IMEM or DMEM) @@ -358,6 +364,9 @@ pub(crate) struct Falcon<'a, E: FalconEngine> { hal: KBox>, dev: &'a device::Device, bar: Bar0<'a>, + // TODO: make private + pub(crate) pfalcon: Mmio<'a, PFalconRegisters>, + pfalcon2: Mmio<'a, PFalcon2Registers>, } impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { @@ -371,19 +380,19 @@ pub(crate) fn new( hal: hal::falcon_hal(chipset)?, dev, bar, + pfalcon: E::pfalcon(bar), + pfalcon2: E::pfalcon2(bar), }) } /// Resets DMA-related registers. pub(crate) fn dma_reset(&self) { - self.bar.update(regs::NV_PFALCON_FBIF_CTL::of::(), |v| { + self.pfalcon.update(regs::NV_PFALCON_FBIF_CTL, |v| { v.with_allow_phys_no_ctx(true) }); - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_DMACTL::zeroed(), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_DMACTL::zeroed()); } /// Reset the controller, select the falcon core, and wait for memory scrubbing to complete. @@ -392,10 +401,9 @@ pub(crate) fn reset(&self) -> Result { self.hal.select_core(self)?; self.hal.reset_wait_mem_scrubbing(self)?; - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_RM::from(self.bar.read(regs::NV_PMC_BOOT_0).into_raw()), - ); + self.pfalcon.write_reg(regs::NV_PFALCON_FALCON_RM::from( + self.bar.read(regs::NV_PMC_BOOT_0).into_raw(), + )); Ok(()) } @@ -413,8 +421,8 @@ fn pio_wr_imem_slice(&self, load_offsets: FalconPioImemLoadTarget<'_>) -> Result return Err(EINVAL); } - self.bar.write( - WithBase::of::().at(Self::PIO_PORT), + self.pfalcon.write( + Array::at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_IMEMC::zeroed() .with_secure(load_offsets.secure) .with_aincw(true) @@ -424,14 +432,14 @@ fn pio_wr_imem_slice(&self, load_offsets: FalconPioImemLoadTarget<'_>) -> Result for (n, block) in load_offsets.data.chunks(MEM_BLOCK_ALIGNMENT).enumerate() { let n = u16::try_from(n)?; let tag: u16 = load_offsets.start_tag.checked_add(n).ok_or(ERANGE)?; - self.bar.write( - WithBase::of::().at(Self::PIO_PORT), + self.pfalcon.write( + Array::at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_IMEMT::zeroed().with_tag(tag), ); for word in block.chunks_exact(4) { let w = [word[0], word[1], word[2], word[3]]; - self.bar.write( - WithBase::of::().at(Self::PIO_PORT), + self.pfalcon.write( + Array::at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_IMEMD::zeroed().with_data(u32::from_le_bytes(w)), ); } @@ -450,8 +458,8 @@ fn pio_wr_dmem_slice(&self, load_offsets: FalconPioDmemLoadTarget<'_>) -> Result return Err(EINVAL); } - self.bar.write( - WithBase::of::().at(Self::PIO_PORT), + self.pfalcon.write( + Array::at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_DMEMC::zeroed() .with_aincw(true) .with_offs(load_offsets.dst_start), @@ -459,8 +467,8 @@ fn pio_wr_dmem_slice(&self, load_offsets: FalconPioDmemLoadTarget<'_>) -> Result for word in load_offsets.data.chunks_exact(4) { let w = [word[0], word[1], word[2], word[3]]; - self.bar.write( - WithBase::of::().at(Self::PIO_PORT), + self.pfalcon.write( + Array::at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_DMEMD::zeroed().with_data(u32::from_le_bytes(w)), ); } @@ -473,14 +481,12 @@ pub(crate) fn pio_load + FalconPioLoadable>( &self, fw: &F, ) -> Result { - self.bar.update(regs::NV_PFALCON_FBIF_CTL::of::(), |v| { + self.pfalcon.update(regs::NV_PFALCON_FBIF_CTL, |v| { v.with_allow_phys_no_ctx(true) }); - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_DMACTL::zeroed(), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_DMACTL::zeroed()); if let Some(imem_ns) = fw.imem_ns_load_params() { self.pio_wr_imem_slice(imem_ns)?; @@ -492,10 +498,8 @@ pub(crate) fn pio_load + FalconPioLoadable>( self.hal.program_brom(self, &fw.brom_params()); - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr())); Ok(()) } @@ -547,16 +551,13 @@ fn dma_wr( // Set up the base source DMA address. - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_DMATRFBASE::zeroed().with_base( + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_DMATRFBASE::zeroed().with_base( // CAST: `as u32` is used on purpose since we do want to strip the upper bits, // which will be written to `NV_PFALCON_FALCON_DMATRFBASE1`. (dma_address >> 8) as u32, - ), - ); - self.bar.write( - WithBase::of::(), + )); + self.pfalcon.write_reg( regs::NV_PFALCON_FALCON_DMATRFBASE1::zeroed().try_with_base(dma_address >> 40)?, ); @@ -566,23 +567,21 @@ fn dma_wr( for pos in (0..num_transfers).map(|i| i * DMA_LEN) { // Perform a transfer of size `DMA_LEN`. - self.bar.write( - WithBase::of::(), + self.pfalcon.write_reg( regs::NV_PFALCON_FALCON_DMATRFMOFFS::zeroed() .try_with_offs(load_offsets.dst_start + pos)?, ); - self.bar.write( - WithBase::of::(), + self.pfalcon.write_reg( regs::NV_PFALCON_FALCON_DMATRFFBOFFS::zeroed().with_offs(src_start + pos), ); - self.bar.write(WithBase::of::(), cmd); + self.pfalcon.write_reg(cmd); // Wait for the transfer to complete. // TIMEOUT: arbitrarily large value, no DMA transfer to the falcon's small memories // should ever take that long. read_poll_timeout( - || Ok(self.bar.read(regs::NV_PFALCON_FALCON_DMATRFCMD::of::())), + || Ok(self.pfalcon.read(regs::NV_PFALCON_FALCON_DMATRFCMD)), |r| r.idle(), Delta::ZERO, Delta::from_secs(2), @@ -614,8 +613,8 @@ fn dma_load + FalconDmaLoadable>(&self, fw: &F) -> }; self.dma_reset(); - self.bar - .update(regs::NV_PFALCON_FBIF_TRANSCFG::of::().at(0), |v| { + self.pfalcon + .update(regs::NV_PFALCON_FBIF_TRANSCFG::at(0), |v| { v.with_target(FalconFbifTarget::CoherentSysmem) .with_mem_type(FalconFbifMemType::Physical) }); @@ -626,10 +625,8 @@ fn dma_load + FalconDmaLoadable>(&self, fw: &F) -> self.hal.program_brom(self, &fw.brom_params()); // Set `BootVec` to start of non-secure code. - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr())); Ok(()) } @@ -638,7 +635,7 @@ fn dma_load + FalconDmaLoadable>(&self, fw: &F) -> pub(crate) fn wait_till_halted(&self) -> Result<()> { // TIMEOUT: arbitrarily large value, firmwares should complete in less than 2 seconds. read_poll_timeout( - || Ok(self.bar.read(regs::NV_PFALCON_FALCON_CPUCTL::of::())), + || Ok(self.pfalcon.read(regs::NV_PFALCON_FALCON_CPUCTL)), |r| r.halted(), Delta::ZERO, Delta::from_secs(2), @@ -649,19 +646,13 @@ pub(crate) fn wait_till_halted(&self) -> Result<()> { /// Start the falcon CPU. pub(crate) fn start(&self) -> Result<()> { - match self - .bar - .read(regs::NV_PFALCON_FALCON_CPUCTL::of::()) - .alias_en() - { - true => self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_CPUCTL_ALIAS::zeroed().with_startcpu(true), - ), - false => self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_CPUCTL::zeroed().with_startcpu(true), - ), + match self.pfalcon.read(regs::NV_PFALCON_FALCON_CPUCTL).alias_en() { + true => self + .pfalcon + .write_reg(regs::NV_PFALCON_FALCON_CPUCTL_ALIAS::zeroed().with_startcpu(true)), + false => self + .pfalcon + .write_reg(regs::NV_PFALCON_FALCON_CPUCTL::zeroed().with_startcpu(true)), } Ok(()) @@ -670,32 +661,24 @@ pub(crate) fn start(&self) -> Result<()> { /// Writes values to the mailbox registers if provided. pub(crate) fn write_mailboxes(&self, mbox0: Option, mbox1: Option) { if let Some(mbox0) = mbox0 { - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_MAILBOX0::zeroed().with_value(mbox0), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_MAILBOX0::zeroed().with_value(mbox0)); } if let Some(mbox1) = mbox1 { - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_MAILBOX1::zeroed().with_value(mbox1), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_MAILBOX1::zeroed().with_value(mbox1)); } } /// Reads the value from `mbox0` register. pub(crate) fn read_mailbox0(&self) -> u32 { - self.bar - .read(regs::NV_PFALCON_FALCON_MAILBOX0::of::()) - .value() + self.pfalcon.read(regs::NV_PFALCON_FALCON_MAILBOX0).value() } /// Reads the value from `mbox1` register. pub(crate) fn read_mailbox1(&self) -> u32 { - self.bar - .read(regs::NV_PFALCON_FALCON_MAILBOX1::of::()) - .value() + self.pfalcon.read(regs::NV_PFALCON_FALCON_MAILBOX1).value() } /// Reads values from both mailbox registers. @@ -760,9 +743,7 @@ pub(crate) fn load + FalconDmaLoadable>(&self, fw: /// Write the application version to the OS register. pub(crate) fn write_os_version(&self, app_version: u32) { - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_OS::zeroed().with_value(app_version), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_OS::zeroed().with_value(app_version)); } } diff --git a/drivers/gpu/nova-core/falcon/fsp.rs b/drivers/gpu/nova-core/falcon/fsp.rs index 0437180b8829..85f9c8c5d60e 100644 --- a/drivers/gpu/nova-core/falcon/fsp.rs +++ b/drivers/gpu/nova-core/falcon/fsp.rs @@ -8,13 +8,12 @@ use kernel::{ io::{ + io_project, poll::read_poll_timeout, - register::{ - Array, - RegisterBase, - WithBase, // - }, - Io, // + register, + register::Array, + Io, + Mmio, // }, prelude::*, sizes::SZ_1K, @@ -22,11 +21,13 @@ }; use crate::{ + driver::{ + Bar0, + NovaRegisters, // + }, falcon::{ Falcon, - FalconEngine, - PFalcon2Base, - PFalconBase, // + FalconEngine, // }, num, regs, // @@ -41,15 +42,24 @@ /// Type specifying the `Fsp` falcon engine. Cannot be instantiated. pub(crate) struct Fsp(()); -impl RegisterBase for Fsp { - const BASE: usize = 0x8f2000; +register! { + base: NovaRegisters; + + PFALCON: super::PFalconRegisters @ 0x8f2000; + PFALCON2: super::PFalcon2Registers @ 0x8f3000; } -impl RegisterBase for Fsp { - const BASE: usize = 0x8f3000; -} +impl FalconEngine for Fsp { + #[inline] + fn pfalcon(io: Bar0<'_>) -> Mmio<'_, super::PFalconRegisters> { + io_project!(io, build: PFALCON) + } -impl FalconEngine for Fsp {} + #[inline] + fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, super::PFalcon2Registers> { + io_project!(io, build: PFALCON2) + } +} impl<'a> Falcon<'a, Fsp> { /// Writes `data` to FSP external memory at offset `0`. @@ -62,19 +72,15 @@ fn write_emem(&mut self, data: &[u8]) -> Result { } // Begin a write burst at offset `0`, auto-incrementing on each write. - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincw(true), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincw(true)); for chunk in data.chunks_exact(4) { let value = u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); // Write the next 32-bit `value`; hardware advances the offset. - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_EMEMD::zeroed().with_data(value), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_EMEMD::zeroed().with_data(value)); } Ok(()) @@ -90,17 +96,12 @@ fn read_emem(&mut self, data: &mut [u8]) -> Result { } // Begin a read burst at offset `0`, auto-incrementing on each read. - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincr(true), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincr(true)); for chunk in data.chunks_exact_mut(4) { // Read the next 32-bit word; hardware advances the offset. - let value = self - .bar - .read(regs::NV_PFALCON_FALCON_EMEMD::of::()) - .data(); + let value = self.pfalcon.read(regs::NV_PFALCON_FALCON_EMEMD).data(); chunk.copy_from_slice(&value.to_le_bytes()); } diff --git a/drivers/gpu/nova-core/falcon/gsp.rs b/drivers/gpu/nova-core/falcon/gsp.rs index ae32f401aeb0..4c96ae325fda 100644 --- a/drivers/gpu/nova-core/falcon/gsp.rs +++ b/drivers/gpu/nova-core/falcon/gsp.rs @@ -2,23 +2,24 @@ use kernel::{ io::{ + io_project, poll::read_poll_timeout, - register::{ - RegisterBase, - WithBase, // - }, + register, Io, + Mmio, // }, prelude::*, time::Delta, // }; use crate::{ + driver::{ + Bar0, + NovaRegisters, // + }, falcon::{ Falcon, - FalconEngine, - PFalcon2Base, - PFalconBase, // + FalconEngine, // }, regs, }; @@ -26,24 +27,31 @@ /// Type specifying the `Gsp` falcon engine. Cannot be instantiated. pub(crate) struct Gsp(()); -impl RegisterBase for Gsp { - const BASE: usize = 0x00110000; +register! { + base: NovaRegisters; + + PFALCON: super::PFalconRegisters @ 0x00110000; + PFALCON2: super::PFalcon2Registers @ 0x00111000; } -impl RegisterBase for Gsp { - const BASE: usize = 0x00111000; -} +impl FalconEngine for Gsp { + #[inline] + fn pfalcon(io: Bar0<'_>) -> Mmio<'_, super::PFalconRegisters> { + io_project!(io, build: PFALCON) + } -impl FalconEngine for Gsp {} + #[inline] + fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, super::PFalcon2Registers> { + io_project!(io, build: PFALCON2) + } +} impl<'a> Falcon<'a, Gsp> { /// Clears the SWGEN0 bit in the Falcon's IRQ status clear register to /// allow GSP to signal CPU for processing new messages in message queue. pub(crate) fn clear_swgen0_intr(&self) { - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_IRQSCLR::zeroed().with_swgen0(true), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_IRQSCLR::zeroed().with_swgen0(true)); } /// Checks if GSP reload/resume has completed during the boot process. @@ -59,8 +67,8 @@ pub(crate) fn check_reload_completed(&self, timeout: Delta) -> Result { /// Returns whether the RISC-V branch privilege lockdown bit is set. pub(crate) fn riscv_branch_privilege_lockdown(&self) -> bool { - self.bar - .read(regs::NV_PFALCON_FALCON_HWCFG2::of::()) + self.pfalcon + .read(regs::NV_PFALCON_FALCON_HWCFG2) .riscv_br_priv_lockdown() } @@ -71,10 +79,7 @@ pub(crate) fn priv_target_mask_released(&self) -> bool { const LOCKED_PATTERN: u32 = 0xbadf_4100; const LOCKED_MASK: u32 = 0xffff_ff00; - let hwcfg2 = self - .bar - .read(regs::NV_PFALCON_FALCON_HWCFG2::of::()) - .into_raw(); + let hwcfg2 = self.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2).into_raw(); hwcfg2 != 0 && (hwcfg2 & LOCKED_MASK) != LOCKED_PATTERN } diff --git a/drivers/gpu/nova-core/falcon/hal/ga102.rs b/drivers/gpu/nova-core/falcon/hal/ga102.rs index 7600ee07ca2e..f9a8444cf840 100644 --- a/drivers/gpu/nova-core/falcon/hal/ga102.rs +++ b/drivers/gpu/nova-core/falcon/hal/ga102.rs @@ -6,11 +6,9 @@ device, io::{ poll::read_poll_timeout, - register::{ - Array, - WithBase, // - }, - Io, // + register::Array, + Io, + Mmio, // }, prelude::*, time::Delta, // @@ -24,6 +22,7 @@ FalconBromParams, FalconEngine, FalconModSelAlgo, + PFalcon2Registers, PeregrineCoreSelect, // }, regs, @@ -31,17 +30,16 @@ use super::FalconHal; -fn select_core_ga102(bar: Bar0<'_>) -> Result { - let bcr_ctrl = bar.read(regs::NV_PRISCV_RISCV_BCR_CTRL::of::()); +fn select_core_ga102(pfalcon2: Mmio<'_, PFalcon2Registers>) -> Result { + let bcr_ctrl = pfalcon2.read(regs::NV_PRISCV_RISCV_BCR_CTRL); if bcr_ctrl.core_select() != PeregrineCoreSelect::Falcon { - bar.write( - WithBase::of::(), + pfalcon2.write_reg( regs::NV_PRISCV_RISCV_BCR_CTRL::zeroed().with_core_select(PeregrineCoreSelect::Falcon), ); // TIMEOUT: falcon core should take less than 10ms to report being enabled. read_poll_timeout( - || Ok(bar.read(regs::NV_PRISCV_RISCV_BCR_CTRL::of::())), + || Ok(pfalcon2.read(regs::NV_PRISCV_RISCV_BCR_CTRL)), |r| r.valid(), Delta::ZERO, Delta::from_millis(10), @@ -86,24 +84,20 @@ fn signature_reg_fuse_version_ga102( Ok(u16::BITS - reg_fuse_version.leading_zeros()) } -fn program_brom_ga102(bar: Bar0<'_>, params: &FalconBromParams) { - bar.write( - WithBase::of::().at(0), +fn program_brom_ga102(pfalcon2: Mmio<'_, PFalcon2Registers>, params: &FalconBromParams) { + pfalcon2.write( + Array::at(0), regs::NV_PFALCON2_FALCON_BROM_PARAADDR::zeroed().with_value(params.pkc_data_offset), ); - bar.write( - WithBase::of::(), + pfalcon2.write_reg( regs::NV_PFALCON2_FALCON_BROM_ENGIDMASK::zeroed() .with_value(u32::from(params.engine_id_mask)), ); - bar.write( - WithBase::of::(), + pfalcon2.write_reg( regs::NV_PFALCON2_FALCON_BROM_CURR_UCODE_ID::zeroed().with_ucode_id(params.ucode_id), ); - bar.write( - WithBase::of::(), - regs::NV_PFALCON2_FALCON_MOD_SEL::zeroed().with_algo(FalconModSelAlgo::Rsa3k), - ); + pfalcon2 + .write_reg(regs::NV_PFALCON2_FALCON_MOD_SEL::zeroed().with_algo(FalconModSelAlgo::Rsa3k)); } pub(super) struct Ga102(PhantomData); @@ -116,7 +110,7 @@ pub(super) fn new() -> Self { impl FalconHal for Ga102 { fn select_core(&self, falcon: &Falcon<'_, E>) -> Result { - select_core_ga102::(falcon.bar) + select_core_ga102(falcon.pfalcon2) } fn signature_reg_fuse_version( @@ -129,27 +123,24 @@ fn signature_reg_fuse_version( } fn program_brom(&self, falcon: &Falcon<'_, E>, params: &FalconBromParams) { - program_brom_ga102::(falcon.bar, params); + program_brom_ga102(falcon.pfalcon2, params); } fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool { falcon - .bar - .read(regs::NV_PRISCV_RISCV_CPUCTL::of::()) + .pfalcon2 + .read(regs::NV_PRISCV_RISCV_CPUCTL) .active_stat() } fn is_riscv_halted(&self, falcon: &Falcon<'_, E>) -> Result { - Ok(falcon - .bar - .read(regs::NV_PRISCV_RISCV_CPUCTL::of::()) - .halted()) + Ok(falcon.pfalcon2.read(regs::NV_PRISCV_RISCV_CPUCTL).halted()) } fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result { // TIMEOUT: memory scrubbing should complete in less than 20ms. read_poll_timeout( - || Ok(falcon.bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::())), + || Ok(falcon.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2)), |r| r.mem_scrubbing_done(), Delta::ZERO, Delta::from_millis(20), @@ -158,20 +149,18 @@ fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result { } fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result { - let bar = falcon.bar; - - let _ = bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::()); + let _ = falcon.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2); // According to OpenRM's `kflcnPreResetWait_GA102` documentation, HW sometimes does not set // RESET_READY so a non-failing timeout is used. let _ = read_poll_timeout( - || Ok(bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::())), + || Ok(falcon.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2)), |r| r.reset_ready(), Delta::ZERO, Delta::from_micros(150), ); - regs::NV_PFALCON_FALCON_ENGINE::reset_engine::(bar); + regs::NV_PFALCON_FALCON_ENGINE::reset_engine(falcon.pfalcon); self.reset_wait_mem_scrubbing(falcon)?; Ok(()) diff --git a/drivers/gpu/nova-core/falcon/hal/tu102.rs b/drivers/gpu/nova-core/falcon/hal/tu102.rs index 5291598fedf7..7fc6e83c2566 100644 --- a/drivers/gpu/nova-core/falcon/hal/tu102.rs +++ b/drivers/gpu/nova-core/falcon/hal/tu102.rs @@ -5,7 +5,6 @@ use kernel::{ io::{ poll::read_poll_timeout, - register::WithBase, Io, // }, prelude::*, @@ -50,8 +49,8 @@ fn program_brom(&self, _falcon: &Falcon<'_, E>, _params: &FalconBromParams) {} fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool { falcon - .bar - .read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS::of::()) + .pfalcon2 + .read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS) .active_stat() } @@ -62,7 +61,7 @@ fn is_riscv_halted(&self, _falcon: &Falcon<'_, E>) -> Result { fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result { // TIMEOUT: memory scrubbing should complete in less than 10ms. read_poll_timeout( - || Ok(falcon.bar.read(regs::NV_PFALCON_FALCON_DMACTL::of::())), + || Ok(falcon.pfalcon.read(regs::NV_PFALCON_FALCON_DMACTL)), |r| r.mem_scrubbing_done(), Delta::ZERO, Delta::from_millis(10), @@ -71,7 +70,7 @@ fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result { } fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result { - regs::NV_PFALCON_FALCON_ENGINE::reset_engine::(falcon.bar); + regs::NV_PFALCON_FALCON_ENGINE::reset_engine(falcon.pfalcon); self.reset_wait_mem_scrubbing(falcon)?; Ok(()) diff --git a/drivers/gpu/nova-core/falcon/sec2.rs b/drivers/gpu/nova-core/falcon/sec2.rs index 91ec7d49c1f5..6648a397d38a 100644 --- a/drivers/gpu/nova-core/falcon/sec2.rs +++ b/drivers/gpu/nova-core/falcon/sec2.rs @@ -1,22 +1,37 @@ // SPDX-License-Identifier: GPL-2.0 -use kernel::io::register::RegisterBase; +use kernel::io::{ + io_project, + register, + Mmio, // +}; -use crate::falcon::{ - FalconEngine, - PFalcon2Base, - PFalconBase, // +use crate::{ + driver::{ + Bar0, + NovaRegisters, // + }, + falcon::FalconEngine, // }; /// Type specifying the `Sec2` falcon engine. Cannot be instantiated. pub(crate) struct Sec2(()); -impl RegisterBase for Sec2 { - const BASE: usize = 0x00840000; +register! { + base: NovaRegisters; + + PFALCON: super::PFalconRegisters @ 0x00840000; + PFALCON2: super::PFalcon2Registers @ 0x00841000; } -impl RegisterBase for Sec2 { - const BASE: usize = 0x00841000; -} +impl FalconEngine for Sec2 { + #[inline] + fn pfalcon(io: Bar0<'_>) -> Mmio<'_, super::PFalconRegisters> { + io_project!(io, build: PFALCON) + } -impl FalconEngine for Sec2 {} + #[inline] + fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, super::PFalcon2Registers> { + io_project!(io, build: PFALCON2) + } +} diff --git a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs index ec4d92317a93..02a36fd59902 100644 --- a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs +++ b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs @@ -12,7 +12,10 @@ Device, // }, dma::Coherent, - io::{register::WithBase, Io}, + io::{ + register::Array, + Io, // + }, prelude::*, ptr::{ Alignable, @@ -23,7 +26,6 @@ }; use crate::{ - driver::Bar0, falcon::{ self, gsp::Gsp, @@ -235,12 +237,7 @@ pub(crate) fn new( /// /// The bootloader will load the FWSEC firmware and then execute it. This function returns /// after FWSEC has reached completion. - pub(crate) fn run( - &self, - dev: &Device, - falcon: &Falcon<'_, Gsp>, - bar: Bar0<'_>, - ) -> Result<()> { + pub(crate) fn run(&self, dev: &Device, falcon: &Falcon<'_, Gsp>) -> Result<()> { // Reset falcon, load the firmware, and run it. falcon .reset() @@ -250,9 +247,8 @@ pub(crate) fn run( .inspect_err(|e| dev_err!(dev, "Failed to load FWSEC firmware: {:?}\n", e))?; // Configure DMA index for the bootloader to fetch the FWSEC firmware from system memory. - bar.update( - regs::NV_PFALCON_FBIF_TRANSCFG::of::() - .try_at(usize::from_safe_cast(self.dmem_desc.ctx_dma)) + falcon.pfalcon.update( + regs::NV_PFALCON_FBIF_TRANSCFG::try_at(usize::from_safe_cast(self.dmem_desc.ctx_dma)) .ok_or(EINVAL)?, |v| { v.with_target(FalconFbifTarget::CoherentSysmem) diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index a5c0ca355493..d2ef35080689 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -62,12 +62,11 @@ impl FwsecUnloadFirmware { fn run( &self, dev: &device::Device, - bar: Bar0<'_>, gsp_falcon: &Falcon<'_, GspEngine>, ) -> Result { match self { Self::WithoutBl(fw) => fw.run(dev, gsp_falcon), - Self::WithBl(fw) => fw.run(dev, gsp_falcon, bar), + Self::WithBl(fw) => fw.run(dev, gsp_falcon), } } } @@ -88,7 +87,7 @@ fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result { // Log errors but keep going if it fails. let fwsec_sb_res = self .fwsec_sb - .run(dev, bar, ctx.gsp_falcon) + .run(dev, ctx.gsp_falcon) .inspect_err(|e| dev_err!(dev, "FWSEC-SB failed to run: {:?}\n", e)); // Remove WPR2 region if set. @@ -168,7 +167,7 @@ fn run_fwsec_frts( if self.needs_fwsec_bootloader { let fwsec_frts_bl = FwsecFirmwareWithBl::new(fwsec_frts, dev, chipset)?; // Load and run the bootloader, which will load FWSEC-FRTS and run it. - fwsec_frts_bl.run(dev, falcon, bar)?; + fwsec_frts_bl.run(dev, falcon)?; } else { // Load and run FWSEC-FRTS directly. fwsec_frts.run(dev, falcon)?; diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index 1af073f3861f..90ac8dac5349 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -4,8 +4,8 @@ use kernel::{ io::{ register, - register::WithBase, - Io, // + Io, + Mmio, // }, prelude::*, sizes::SizeConstants, @@ -13,22 +13,18 @@ }; use crate::{ - driver::{ - Bar0, - NovaRegisters, // - }, + driver::NovaRegisters, falcon::{ DmaTrfCmdSize, FalconCoreRev, FalconCoreRevSubversion, - FalconEngine, FalconFbifMemType, FalconFbifTarget, FalconMem, FalconModSelAlgo, FalconSecurityModel, - PFalcon2Base, - PFalconBase, + PFalcon2Registers, + PFalconRegisters, PeregrineCoreSelect, // }, gpu::{ @@ -202,32 +198,32 @@ pub(crate) fn usable_fb_size(self) -> u64 { // PFALCON register! { - base: NovaRegisters; + base: PFalconRegisters; - pub(crate) NV_PFALCON_FALCON_IRQSCLR(u32) @ PFalconBase + 0x00000004 { + pub(crate) NV_PFALCON_FALCON_IRQSCLR(u32) @ 0x00000004 { 6:6 swgen0 => bool; 4:4 halt => bool; } - pub(crate) NV_PFALCON_FALCON_MAILBOX0(u32) @ PFalconBase + 0x00000040 { + pub(crate) NV_PFALCON_FALCON_MAILBOX0(u32) @ 0x00000040 { 31:0 value => u32; } - pub(crate) NV_PFALCON_FALCON_MAILBOX1(u32) @ PFalconBase + 0x00000044 { + pub(crate) NV_PFALCON_FALCON_MAILBOX1(u32) @ 0x00000044 { 31:0 value => u32; } /// Used to store version information about the firmware running /// on the Falcon processor. - pub(crate) NV_PFALCON_FALCON_OS(u32) @ PFalconBase + 0x00000080 { + pub(crate) NV_PFALCON_FALCON_OS(u32) @ 0x00000080 { 31:0 value => u32; } - pub(crate) NV_PFALCON_FALCON_RM(u32) @ PFalconBase + 0x00000084 { + pub(crate) NV_PFALCON_FALCON_RM(u32) @ 0x00000084 { 31:0 value => u32; } - pub(crate) NV_PFALCON_FALCON_HWCFG2(u32) @ PFalconBase + 0x000000f4 { + pub(crate) NV_PFALCON_FALCON_HWCFG2(u32) @ 0x000000f4 { /// Signal indicating that reset is completed (GA102+). 31:31 reset_ready => bool; /// RISC-V branch privilege lockdown bit. @@ -237,17 +233,17 @@ pub(crate) fn usable_fb_size(self) -> u64 { 10:10 riscv => bool; } - pub(crate) NV_PFALCON_FALCON_CPUCTL(u32) @ PFalconBase + 0x00000100 { + pub(crate) NV_PFALCON_FALCON_CPUCTL(u32) @ 0x00000100 { 6:6 alias_en => bool; 4:4 halted => bool; 1:1 startcpu => bool; } - pub(crate) NV_PFALCON_FALCON_BOOTVEC(u32) @ PFalconBase + 0x00000104 { + pub(crate) NV_PFALCON_FALCON_BOOTVEC(u32) @ 0x00000104 { 31:0 value => u32; } - pub(crate) NV_PFALCON_FALCON_DMACTL(u32) @ PFalconBase + 0x0000010c { + pub(crate) NV_PFALCON_FALCON_DMACTL(u32) @ 0x0000010c { 7:7 secure_stat => bool; 6:3 dmaq_num; 2:2 imem_scrubbing => bool; @@ -255,15 +251,15 @@ pub(crate) fn usable_fb_size(self) -> u64 { 0:0 require_ctx => bool; } - pub(crate) NV_PFALCON_FALCON_DMATRFBASE(u32) @ PFalconBase + 0x00000110 { + pub(crate) NV_PFALCON_FALCON_DMATRFBASE(u32) @ 0x00000110 { 31:0 base => u32; } - pub(crate) NV_PFALCON_FALCON_DMATRFMOFFS(u32) @ PFalconBase + 0x00000114 { + pub(crate) NV_PFALCON_FALCON_DMATRFMOFFS(u32) @ 0x00000114 { 23:0 offs; } - pub(crate) NV_PFALCON_FALCON_DMATRFCMD(u32) @ PFalconBase + 0x00000118 { + pub(crate) NV_PFALCON_FALCON_DMATRFCMD(u32) @ 0x00000118 { 16:16 set_dmtag; 14:12 ctxdma; 10:8 size ?=> DmaTrfCmdSize; @@ -274,15 +270,15 @@ pub(crate) fn usable_fb_size(self) -> u64 { 0:0 full => bool; } - pub(crate) NV_PFALCON_FALCON_DMATRFFBOFFS(u32) @ PFalconBase + 0x0000011c { + pub(crate) NV_PFALCON_FALCON_DMATRFFBOFFS(u32) @ 0x0000011c { 31:0 offs => u32; } - pub(crate) NV_PFALCON_FALCON_DMATRFBASE1(u32) @ PFalconBase + 0x00000128 { + pub(crate) NV_PFALCON_FALCON_DMATRFBASE1(u32) @ 0x00000128 { 8:0 base; } - pub(crate) NV_PFALCON_FALCON_HWCFG1(u32) @ PFalconBase + 0x0000012c { + pub(crate) NV_PFALCON_FALCON_HWCFG1(u32) @ 0x0000012c { /// Core revision subversion. 7:6 core_rev_subversion => FalconCoreRevSubversion; /// Security model. @@ -291,12 +287,12 @@ pub(crate) fn usable_fb_size(self) -> u64 { 3:0 core_rev ?=> FalconCoreRev; } - pub(crate) NV_PFALCON_FALCON_CPUCTL_ALIAS(u32) @ PFalconBase + 0x00000130 { + pub(crate) NV_PFALCON_FALCON_CPUCTL_ALIAS(u32) @ 0x00000130 { 1:1 startcpu => bool; } /// IMEM access control register. Up to 4 ports are available for IMEM access. - pub(crate) NV_PFALCON_FALCON_IMEMC(u32)[4, stride = 16] @ PFalconBase + 0x00000180 { + pub(crate) NV_PFALCON_FALCON_IMEMC(u32)[4, stride = 16] @ 0x00000180 { /// Access secure IMEM. 28:28 secure => bool; /// Auto-increment on write. @@ -307,17 +303,17 @@ pub(crate) fn usable_fb_size(self) -> u64 { /// IMEM data register. Reading/writing this register accesses IMEM at the address /// specified by the corresponding IMEMC register. - pub(crate) NV_PFALCON_FALCON_IMEMD(u32)[4, stride = 16] @ PFalconBase + 0x00000184 { + pub(crate) NV_PFALCON_FALCON_IMEMD(u32)[4, stride = 16] @ 0x00000184 { 31:0 data; } /// IMEM tag register. Used to set the tag for the current IMEM block. - pub(crate) NV_PFALCON_FALCON_IMEMT(u32)[4, stride = 16] @ PFalconBase + 0x00000188 { + pub(crate) NV_PFALCON_FALCON_IMEMT(u32)[4, stride = 16] @ 0x00000188 { 15:0 tag; } /// DMEM access control register. Up to 8 ports are available for DMEM access. - pub(crate) NV_PFALCON_FALCON_DMEMC(u32)[8, stride = 8] @ PFalconBase + 0x000001c0 { + pub(crate) NV_PFALCON_FALCON_DMEMC(u32)[8, stride = 8] @ 0x000001c0 { /// Auto-increment on write. 24:24 aincw => bool; /// DMEM block and word offset. @@ -326,29 +322,29 @@ pub(crate) fn usable_fb_size(self) -> u64 { /// DMEM data register. Reading/writing this register accesses DMEM at the address /// specified by the corresponding DMEMC register. - pub(crate) NV_PFALCON_FALCON_DMEMD(u32)[8, stride = 8] @ PFalconBase + 0x000001c4 { + pub(crate) NV_PFALCON_FALCON_DMEMD(u32)[8, stride = 8] @ 0x000001c4 { 31:0 data; } /// Actually known as `NV_PSEC_FALCON_ENGINE` and `NV_PGSP_FALCON_ENGINE` depending on the /// falcon instance. - pub(crate) NV_PFALCON_FALCON_ENGINE(u32) @ PFalconBase + 0x000003c0 { + pub(crate) NV_PFALCON_FALCON_ENGINE(u32) @ 0x000003c0 { 0:0 reset => bool; } - pub(crate) NV_PFALCON_FBIF_TRANSCFG(u32)[8] @ PFalconBase + 0x00000600 { + pub(crate) NV_PFALCON_FBIF_TRANSCFG(u32)[8] @ 0x00000600 { 2:2 mem_type => FalconFbifMemType; 1:0 target ?=> FalconFbifTarget; } - pub(crate) NV_PFALCON_FBIF_CTL(u32) @ PFalconBase + 0x00000624 { + pub(crate) NV_PFALCON_FBIF_CTL(u32) @ 0x00000624 { 7:7 allow_phys_no_ctx => bool; } // Falcon EMEM PIO registers (used by FSP on Hopper/Blackwell). // These provide the falcon external memory communication interface. - pub(crate) NV_PFALCON_FALCON_EMEMC(u32) @ PFalconBase + 0x00000ac0 { + pub(crate) NV_PFALCON_FALCON_EMEMC(u32) @ 0x00000ac0 { /// EMEM byte offset (4-byte aligned) within the block. 7:2 offs; /// EMEM block to access. @@ -359,7 +355,7 @@ pub(crate) fn usable_fb_size(self) -> u64 { 25:25 aincr => bool; } - pub(crate) NV_PFALCON_FALCON_EMEMD(u32) @ PFalconBase + 0x00000ac4 { + pub(crate) NV_PFALCON_FALCON_EMEMD(u32) @ 0x00000ac4 { 31:0 data => u32; } } @@ -385,13 +381,13 @@ pub(crate) fn with_falcon_mem(self, mem: FalconMem) -> Self { impl NV_PFALCON_FALCON_ENGINE { /// Resets the falcon - pub(crate) fn reset_engine(bar: Bar0<'_>) { - bar.update(Self::of::(), |r| r.with_reset(true)); + pub(crate) fn reset_engine(pfalcon: Mmio<'_, PFalconRegisters>) { + pfalcon.update(NV_PFALCON_FALCON_ENGINE, |r| r.with_reset(true)); // TIMEOUT: falcon engine should not take more than 10us to reset. time::delay::fsleep(time::Delta::from_micros(10)); - bar.update(Self::of::(), |r| r.with_reset(false)); + pfalcon.update(NV_PFALCON_FALCON_ENGINE, |r| r.with_reset(false)); } } @@ -405,23 +401,23 @@ pub(crate) fn mem_scrubbing_done(self) -> bool { /* PFALCON2 */ register! { - base: NovaRegisters; + base: PFalcon2Registers; - pub(crate) NV_PFALCON2_FALCON_MOD_SEL(u32) @ PFalcon2Base + 0x00000180 { + pub(crate) NV_PFALCON2_FALCON_MOD_SEL(u32) @ 0x00000180 { 7:0 algo ?=> FalconModSelAlgo; } - pub(crate) NV_PFALCON2_FALCON_BROM_CURR_UCODE_ID(u32) @ PFalcon2Base + 0x00000198 { + pub(crate) NV_PFALCON2_FALCON_BROM_CURR_UCODE_ID(u32) @ 0x00000198 { 7:0 ucode_id => u8; } - pub(crate) NV_PFALCON2_FALCON_BROM_ENGIDMASK(u32) @ PFalcon2Base + 0x0000019c { + pub(crate) NV_PFALCON2_FALCON_BROM_ENGIDMASK(u32) @ 0x0000019c { 31:0 value => u32; } /// OpenRM defines this as a register array, but doesn't specify its size and only uses its /// first element. Be conservative until we know the actual size or need to use more registers. - pub(crate) NV_PFALCON2_FALCON_BROM_PARAADDR(u32)[1] @ PFalcon2Base + 0x00000210 { + pub(crate) NV_PFALCON2_FALCON_BROM_PARAADDR(u32)[1] @ 0x00000210 { 31:0 value => u32; } } @@ -429,23 +425,23 @@ pub(crate) fn mem_scrubbing_done(self) -> bool { // PRISCV register! { - base: NovaRegisters; + base: PFalcon2Registers; /// RISC-V status register for debug (Turing and GA100 only). /// Reflects current RISC-V core status. - pub(crate) NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS(u32) @ PFalcon2Base + 0x00000240 { + pub(crate) NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS(u32) @ 0x00000240 { /// RISC-V core active/inactive status. 0:0 active_stat => bool; } /// GA102 and later. - pub(crate) NV_PRISCV_RISCV_CPUCTL(u32) @ PFalcon2Base + 0x00000388 { + pub(crate) NV_PRISCV_RISCV_CPUCTL(u32) @ 0x00000388 { 7:7 active_stat => bool; 4:4 halted => bool; } /// GA102 and later. - pub(crate) NV_PRISCV_RISCV_BCR_CTRL(u32) @ PFalcon2Base + 0x00000668 { + pub(crate) NV_PRISCV_RISCV_BCR_CTRL(u32) @ 0x00000668 { 8:8 br_fetch => bool; 4:4 core_select => PeregrineCoreSelect; 0:0 valid => bool; From a6355a2ef819ae569f9b14ac808ed5c1f4b62ddd Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:37 +0100 Subject: [PATCH 023/110] gpu: nova-core: convert hshub0 from relative register to projection Similar to the PFALCON and PFALCON2 conversion, the hshub0 relative access can also be achieved cleanly with projection and a new base. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-13-5552b1d59525@garyguo.net [ Split nested import into kernel vertical style. - Danilo ] Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/fb/hal/gb100.rs | 59 +++++++++++++-------------- drivers/gpu/nova-core/fb/regs.rs | 24 +++++++---- 2 files changed, 45 insertions(+), 38 deletions(-) diff --git a/drivers/gpu/nova-core/fb/hal/gb100.rs b/drivers/gpu/nova-core/fb/hal/gb100.rs index d9e4d62ae632..9fa094939600 100644 --- a/drivers/gpu/nova-core/fb/hal/gb100.rs +++ b/drivers/gpu/nova-core/fb/hal/gb100.rs @@ -5,11 +5,10 @@ use kernel::{ io::{ - register::{ - RegisterBase, - WithBase, // - }, - Io, // + io_project, + register, + Io, + Mmio, // }, num::Bounded, prelude::*, @@ -21,7 +20,10 @@ }; use crate::{ - driver::Bar0, + driver::{ + Bar0, + NovaRegisters, // + }, fb::{ hal::FbHal, regs, // @@ -31,17 +33,26 @@ struct Gb100; -impl RegisterBase for Gb100 { - const BASE: usize = 0x0087_0000; +register! { + base: NovaRegisters; + + HSHUB0: regs::Hshub0Registers @ 0x0087_0000; } -fn read_sysmem_flush_page_gb100(bar: Bar0<'_>) -> u64 { +#[inline] +fn hshub0(bar: Bar0<'_>) -> Mmio<'_, regs::Hshub0Registers> { + io_project!(bar, build: HSHUB0) +} + +fn read_sysmem_flush_page_gb100(hshub0: Mmio<'_, regs::Hshub0Registers>) -> u64 { let lo = u64::from( - bar.read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::()) + hshub0 + .read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO) .adr(), ); let hi = u64::from( - bar.read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::()) + hshub0 + .read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI) .adr(), ); @@ -52,7 +63,7 @@ fn read_sysmem_flush_page_gb100(bar: Bar0<'_>) -> u64 { /// /// Both the primary and EG (egress) register pairs must be programmed to the same address, /// as required by hardware. -fn write_sysmem_flush_page_gb100(bar: Bar0<'_>, addr: Bounded) { +fn write_sysmem_flush_page_gb100(hshub0: Mmio<'_, regs::Hshub0Registers>, addr: Bounded) { // CAST: lower 32 bits. Hardware ignores bits 7:0. let addr_lo = *addr as u32; let addr_hi = addr.shr::<32, 20>().cast::(); @@ -60,24 +71,12 @@ fn write_sysmem_flush_page_gb100(bar: Bar0<'_>, addr: Bounded) { // Write HI first. The hardware will trigger the flush on the LO write. // Primary HSHUB pair. - bar.write( - regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::(), - regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi), - ); - bar.write( - regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::(), - regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo), - ); + hshub0.write_reg(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi)); + hshub0.write_reg(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo)); // EG (egress) pair -- must match the primary pair. - bar.write( - regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI::of::(), - regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi), - ); - bar.write( - regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO::of::(), - regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo), - ); + hshub0.write_reg(regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi)); + hshub0.write_reg(regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo)); } // This PMU reservation size is r570-specific. @@ -88,13 +87,13 @@ pub(super) const fn pmu_reserved_size_gb100() -> u32 { impl FbHal for Gb100 { fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 { - read_sysmem_flush_page_gb100(bar) + read_sysmem_flush_page_gb100(hshub0(bar)) } fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result { let addr = Bounded::::try_new(addr).ok_or(EINVAL)?; - write_sysmem_flush_page_gb100(bar, addr); + write_sysmem_flush_page_gb100(hshub0(bar), addr); Ok(()) } diff --git a/drivers/gpu/nova-core/fb/regs.rs b/drivers/gpu/nova-core/fb/regs.rs index c27582e376e2..131787996a24 100644 --- a/drivers/gpu/nova-core/fb/regs.rs +++ b/drivers/gpu/nova-core/fb/regs.rs @@ -2,7 +2,11 @@ use kernel::{ io::register, - sizes::SizeConstants, // + prelude::*, + sizes::{ + SizeConstants, + SZ_4K, // + }, // }; use crate::driver::NovaRegisters; @@ -65,31 +69,35 @@ pub(super) fn vga_workspace_addr(self) -> Option { } } -/// Base of the GB10x HSHUB0 register window (`NV_HSHUB0_PRIV_BASE` in Open RM). +const HSHUB0_REGION_SIZE: usize = SZ_4K; + +/// The GB10x HSHUB0 register window (Base defined as `NV_HSHUB0_PRIV_BASE` in Open RM). /// /// The base is provided by the GB10x framebuffer HAL. -pub(super) struct Hshub0Base(()); +#[repr(align(4))] +#[derive(FromBytes, IntoBytes)] +pub(super) struct Hshub0Registers([u8; HSHUB0_REGION_SIZE]); register! { - base: NovaRegisters; + base: Hshub0Registers; // GB10x sysmem flush registers, relative to the HSHUB0 base. GB10x routes sysmembar // through a primary and an EG (egress) pair that must both be programmed to the same // address. Hardware ignores bits 7:0 of each LO register. The boot path uses a fixed // HSHUB0 base, so the multiple runtime-discovered HSHUB bases are not needed here. - pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x00000e50 { + pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x00000e50 { 31:0 adr => u32; } - pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x00000e54 { + pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x00000e54 { 19:0 adr; } - pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x000006c0 { + pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x000006c0 { 31:0 adr => u32; } - pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x000006c4 { + pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x000006c4 { 19:0 adr; } } From 631adefcac893728c278f4bd2c9634cfe3f35482 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:38 +0100 Subject: [PATCH 024/110] rust: io: register: remove relative registers Relative registers can be better served by projection to subregion instead of ad-hoc handling in register macro. Projection composes better (e.g. it natively allows relative registers of relative registers without needing additional support). Remove relative register support, and update the documentation to demonstrate how projection and subregions can be used to achieve this instead. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-14-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io/register.rs | 388 +++++-------------------------------- rust/macros/io/register.rs | 52 +---- 2 files changed, 61 insertions(+), 379 deletions(-) diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index b9cb4fce8628..187f28f16614 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -8,7 +8,7 @@ //! //! Note: most of the items in this module are public so they can be referenced by the macro, but //! most are not to be used directly by users. Outside of the `register!` macro itself, the only -//! items you might want to import from this module are [`WithBase`] and [`Array`]. +//! item you might want to import from this module is [`Array`]. //! //! # Simple example //! @@ -200,76 +200,6 @@ fn offset(self) -> usize { } } -/// Trait providing a base address to be added to the offset of a relative register to obtain -/// its actual offset. -/// -/// The `T` generic argument is used to distinguish which base to use, in case a type provides -/// several bases. It is given to the `register!` macro to restrict the use of the register to -/// implementors of this particular variant. -pub trait RegisterBase { - /// Base address to which register offsets are added. - const BASE: usize; -} - -/// Trait implemented by all registers that are relative to a base. -pub trait WithBase { - /// Family of bases applicable to this register. - type BaseFamily; - - /// Returns the absolute location of this type when using `B` as its base. - #[inline(always)] - fn of>() -> RelativeRegisterLoc - where - Self: Register, - { - RelativeRegisterLoc::new() - } -} - -/// Trait implemented by relative registers. -pub trait RelativeRegister: Register + WithBase {} - -/// Location of a relative register. -/// -/// This can either be an immediately accessible regular [`RelativeRegister`], or a -/// [`RelativeRegisterArray`] that needs one additional resolution through -/// [`RelativeRegisterLoc::at`]. -pub struct RelativeRegisterLoc(PhantomData, PhantomData); - -impl RelativeRegisterLoc -where - T: Register + WithBase, - B: RegisterBase + ?Sized, -{ - /// Returns the location of a relative register or register array. - #[inline(always)] - // We do not implement `Default` so we can be const. - #[expect(clippy::new_without_default)] - pub const fn new() -> Self { - Self(PhantomData, PhantomData) - } - - // Returns the absolute offset of the relative register using base `B`. - // - // This is implemented as a private const method so it can be reused by the [`IoLoc`] - // implementations of both [`RelativeRegisterLoc`] and [`RelativeRegisterArrayLoc`]. - #[inline] - const fn offset(self) -> usize { - B::BASE + T::OFFSET - } -} - -impl IoLoc for RelativeRegisterLoc -where - T: RelativeRegister, - B: RegisterBase + ?Sized, -{ - #[inline(always)] - fn offset(self) -> usize { - RelativeRegisterLoc::offset(self) - } -} - /// Trait implemented by arrays of registers. pub trait RegisterArray: Register { /// Number of elements in the registers array. @@ -332,71 +262,6 @@ fn try_at(idx: usize) -> Option> } } -/// Trait implemented by arrays of relative registers. -pub trait RelativeRegisterArray: RegisterArray + WithBase {} - -/// Location of a relative array register. -pub struct RelativeRegisterArrayLoc< - T: RelativeRegisterArray, - B: RegisterBase + ?Sized, ->(RelativeRegisterLoc, usize); - -impl RelativeRegisterArrayLoc -where - T: RelativeRegisterArray, - B: RegisterBase + ?Sized, -{ - /// Returns the location of register `T` from the base `B` at index `idx`, with build-time - /// validation. - #[inline(always)] - pub fn new(idx: usize) -> Self { - build_assert!(idx < T::SIZE); - - Self(RelativeRegisterLoc::new(), idx) - } - - /// Attempts to return the location of register `T` from the base `B` at index `idx`, with - /// runtime validation. - #[inline(always)] - pub fn try_new(idx: usize) -> Option { - if idx < T::SIZE { - Some(Self(RelativeRegisterLoc::new(), idx)) - } else { - None - } - } -} - -/// Methods exclusive to [`RelativeRegisterLoc`]s created with a [`RelativeRegisterArray`]. -impl RelativeRegisterLoc -where - T: RelativeRegisterArray, - B: RegisterBase + ?Sized, -{ - /// Returns the location of the register at position `idx`, with build-time validation. - #[inline(always)] - pub fn at(self, idx: usize) -> RelativeRegisterArrayLoc { - RelativeRegisterArrayLoc::new(idx) - } - - /// Returns the location of the register at position `idx`, with runtime validation. - #[inline(always)] - pub fn try_at(self, idx: usize) -> Option> { - RelativeRegisterArrayLoc::try_new(idx) - } -} - -impl IoLoc for RelativeRegisterArrayLoc -where - T: RelativeRegisterArray, - B: RegisterBase + ?Sized, -{ - #[inline(always)] - fn offset(self) -> usize { - self.0.offset() + self.1 * T::STRIDE - } -} - /// Trait implemented by items that contain both a register value and the absolute I/O location at /// which to write it. /// @@ -452,8 +317,7 @@ pub const fn element_alias_offset+------------CPU0-------------+ -/// | | -/// 0x110--->+-----------------------------+ -/// | CPU_CTL | -/// +-----------------------------+ -/// | ... | -/// | | -/// | | -/// 0x200--->+------------CPU1-------------+ -/// | | -/// 0x210--->+-----------------------------+ -/// | CPU_CTL | -/// +-----------------------------+ -/// | ... | -/// +-----------------------------+ -/// ``` -/// -/// `CPU0` and `CPU1` both have a `CPU_CTL` register that starts at offset `0x10` of their I/O -/// space segment. Since both instances of `CPU_CTL` share the same layout, we don't want to define -/// them twice and would prefer a way to select which one to use from a single definition. -/// -/// This can be done using the `Base + Offset` syntax when specifying the register's address: -/// -/// ```ignore -/// register! { -/// ... -/// pub RELATIVE_REG(u32) @ Base + 0x80 { -/// ... -/// } -/// } -/// ``` -/// -/// This creates a register with an offset of `0x80` from a given base. -/// -/// `Base` is an arbitrary type (typically a ZST) to be used as a generic parameter of the -/// [`RegisterBase`] trait to provide the base as a constant, i.e. each type providing a base for -/// this register needs to implement `RegisterBase`. -/// -/// The location of relative registers can be built using the [`WithBase::of`] method to specify -/// its base. All relative registers implement [`WithBase`]. -/// -/// Here is the above layout translated into code: -/// -/// ```no_run -/// use kernel::{ -/// io::{ -/// register, -/// register::{ -/// RegisterBase, -/// WithBase, -/// }, -/// Io, -/// Region, -/// }, -/// }; -/// # use kernel::io::Mmio; -/// -/// // Type used to identify the base. -/// pub struct CpuCtlBase; -/// -/// // ZST describing `CPU0`. -/// struct Cpu0; -/// impl RegisterBase for Cpu0 { -/// const BASE: usize = 0x100; -/// } -/// -/// // ZST describing `CPU1`. -/// struct Cpu1; -/// impl RegisterBase for Cpu1 { -/// const BASE: usize = 0x200; -/// } -/// -/// // This makes `CPU_CTL` accessible from all implementors of `RegisterBase`. -/// register! { -/// base: Region<0x1000>; -/// -/// /// CPU core control. -/// pub CPU_CTL(u32) @ CpuCtlBase + 0x10 { -/// 0:0 start; -/// } -/// } -/// -/// # fn test(io: Mmio<'_, Region<0x1000>>) { -/// // Read the status of `Cpu0`. -/// let cpu0_started = io.read(CPU_CTL::of::()); -/// -/// // Stop `Cpu0`. -/// io.write(WithBase::of::(), CPU_CTL::zeroed()); -/// # } -/// -/// // Aliases can also be defined for relative register. -/// register! { -/// base: Region<0x1000>; -/// -/// /// Alias to CPU core control. -/// pub CPU_CTL_ALIAS(u32) => CpuCtlBase + CPU_CTL { -/// /// Start the aliased CPU core. -/// 1:1 alias_start; -/// } -/// } -/// -/// # fn test2(io: Mmio<'_, Region<0x1000>>) { -/// // Start the aliased `CPU0`, leaving its other fields untouched. -/// io.update(CPU_CTL_ALIAS::of::(), |r| r.with_alias_start(true)); -/// # } -/// ``` -/// /// ## Arrays of registers /// /// Some I/O areas contain consecutive registers that share the same field layout. These areas can @@ -763,115 +511,83 @@ pub const fn element_alias_offset+------------CPU0-------------+ +/// | | +/// 0x110--->+-----------------------------+ +/// | CPU_CTL | +/// +-----------------------------+ +/// | ... | +/// | | +/// | | +/// 0x200--->+------------CPU1-------------+ +/// | | +/// 0x210--->+-----------------------------+ +/// | CPU_CTL | +/// +-----------------------------+ +/// | ... | +/// +-----------------------------+ /// ``` /// -/// Like relative registers, they implement the [`WithBase`] trait. However the return value of -/// [`WithBase::of`] cannot be used directly as a location and must be further specified using the -/// [`at`](RelativeRegisterLoc::at) method. +/// `CPU0` and `CPU1` both have a `CPU_CTL` register that starts at offset `0x10` of their I/O +/// space segment. Since both instances of `CPU_CTL` share the same layout, we don't want to define +/// them twice and would prefer a way to select which one to use from a single definition. +/// +/// This can be done by defining a new type for the subregion, and then defining registers that use +/// the new type as the base: /// /// ```no_run /// use kernel::{ /// io::{ +/// io_project, /// register, -/// register::{ -/// RegisterBase, -/// WithBase, -/// }, /// Io, /// Region, /// }, /// }; /// # use kernel::io::Mmio; -/// # fn get_scratch_idx() -> usize { -/// # 0x15 -/// # } /// -/// // Type used as parameter of `RegisterBase` to specify the base. -/// pub struct CpuCtlBase; +/// // Subregion type. Make sure it has adequate size and alignment. +/// #[repr(align(4))] +/// #[derive(FromBytes, IntoBytes)] +/// pub struct CpuCtl([u8; 0x100]); /// -/// // ZST describing `CPU0`. -/// struct Cpu0; -/// impl RegisterBase for Cpu0 { -/// const BASE: usize = 0x100; -/// } -/// -/// // ZST describing `CPU1`. -/// struct Cpu1; -/// impl RegisterBase for Cpu1 { -/// const BASE: usize = 0x200; -/// } -/// -/// // 64 per-cpu scratch registers, arranged as a contiguous array. /// register! { /// base: Region<0x1000>; /// -/// /// Per-CPU scratch registers. -/// pub CPU_SCRATCH(u32)[64] @ CpuCtlBase + 0x00000080 { -/// 31:0 value; -/// } +/// // Subregions can just be defined like normal registers. +/// CPU0: CpuCtl @ 0x100; +/// CPU1: CpuCtl @ 0x200; /// } /// -/// # fn test(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { -/// // Read scratch register 0 of CPU0. -/// let scratch = io.read(CPU_SCRATCH::of::().at(0)); -/// -/// // Write the retrieved value into scratch register 15 of CPU1. -/// io.write(WithBase::of::().at(15), scratch); -/// -/// // This won't build. -/// // let cpu0_scratch_128 = io.read(CPU_SCRATCH::of::().at(128)).value(); -/// -/// // Runtime-obtained array index. -/// let scratch_idx = get_scratch_idx(); -/// // Access on a runtime index returns an error if it is out-of-bounds. -/// let cpu0_scratch = io.read( -/// CPU_SCRATCH::of::().try_at(scratch_idx).ok_or(EINVAL)? -/// ).value(); -/// # Ok(()) -/// # } -/// -/// // Alias to `SCRATCH[8]` used to convey the firmware exit code. +/// // Then you can define new registers on the subregion. /// register! { -/// base: Region<0x1000>; +/// base: CpuCtl; /// -/// /// Per-CPU firmware exit status code. -/// pub CPU_FIRMWARE_STATUS(u32) => CpuCtlBase + CPU_SCRATCH[8] { -/// 7:0 status; +/// /// CPU core control. +/// pub CPU_CTL(u32) @ 0x10 { +/// 0:0 start; /// } /// } /// -/// // Non-contiguous relative register arrays can be defined by adding a stride parameter. -/// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the -/// // registers of the two declarations below are interleaved. -/// register! { -/// base: Region<0x1000>; +/// # fn test(io: Mmio<'_, Region<0x1000>>) { +/// // Read the status of `Cpu0`. +/// let cpu0_started = io_project!(io, build: CPU0).read(CPU_CTL); /// -/// /// Scratch registers bank 0. -/// pub CPU_SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d00 { -/// 31:0 value; -/// } -/// -/// /// Scratch registers bank 1. -/// pub CPU_SCRATCH_INTERLEAVED_1(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d04 { -/// 31:0 value; -/// } -/// } -/// -/// # fn test2(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { -/// let cpu0_status = io.read(CPU_FIRMWARE_STATUS::of::()).status(); -/// # Ok(()) +/// // Stop `Cpu0`. +/// io_project!(io, build: CPU0).write_reg(CPU_CTL::zeroed()); /// # } /// ``` #[macro_export] diff --git a/rust/macros/io/register.rs b/rust/macros/io/register.rs index e948b0959154..500805aa6b23 100644 --- a/rust/macros/io/register.rs +++ b/rust/macros/io/register.rs @@ -65,7 +65,6 @@ struct Reg { name: Ident, ty: Type, array: Option, - relative_base: Option, offset: RegOffset, bitfield: Option<(Type, Group)>, } @@ -110,26 +109,15 @@ fn parse(input: syn::parse::ParseStream<'_>) -> Result { // Parse offset and the base it's relative to. let lh = input.lookahead1(); - let mut relative_base = None; let offset = if lh.peek(Token![@]) { let _: Token![@] = input.parse()?; - if input.peek(Ident) { - relative_base = Some(input.parse()?); - let _: Token![+] = input.parse()?; - } - RegOffset::Fixed { offset: input.parse()?, } } else if lh.peek(Token![=>]) { let _: Token![=>] = input.parse()?; - let mut alias: Path = input.parse()?; - if input.peek(Token![+]) { - let _: Token![+] = input.parse()?; - relative_base = Some(alias); - alias = input.parse()?; - } + let alias: Path = input.parse()?; if input.peek(token::Bracket) { let content; @@ -164,7 +152,6 @@ fn parse(input: syn::parse::ParseStream<'_>) -> Result { name, ty, array, - relative_base, offset, bitfield, }) @@ -208,7 +195,6 @@ pub(crate) fn register(def: RegDef) -> Result { name, ty, array, - relative_base, offset, bitfield, } = reg; @@ -248,7 +234,7 @@ impl ::kernel::io::register::Register for #name { } match array { - None if bitfield.is_none() && relative_base.is_none() => outputs.extend(quote!( + None if bitfield.is_none() => outputs.extend(quote!( #(#attrs)* #vis const #name: ::kernel::io::register::OffsetLoc<#base, #ty> = ::kernel::io::register::OffsetLoc::new(#offset); )), @@ -258,21 +244,12 @@ impl ::kernel::io::register::Register for #name { "defining without bitfield is not yet supported for this type of register", ))?, - None => match relative_base { - None => outputs.extend(quote_spanned!(span => - impl ::kernel::io::register::FixedRegister for #name {} + None => outputs.extend(quote_spanned!(span => + impl ::kernel::io::register::FixedRegister for #name {} - #(#attrs)* #vis const #name: ::kernel::io::register::FixedRegisterLoc<#name> = - ::kernel::io::register::FixedRegisterLoc::<#name>::new(); - )), - Some(relative_base) => outputs.extend(quote_spanned!(span => - impl ::kernel::io::register::WithBase for #name { - type BaseFamily = #relative_base; - } - - impl ::kernel::io::register::RelativeRegister for #name {} - )), - }, + #(#attrs)* #vis const #name: ::kernel::io::register::FixedRegisterLoc<#name> = + ::kernel::io::register::FixedRegisterLoc::<#name>::new(); + )), Some(def) => { let size = &def.size; @@ -288,24 +265,13 @@ impl ::kernel::io::register::RelativeRegister for #name {} }; outputs.extend(quote_spanned!(span => + impl ::kernel::io::register::Array for #name {} + impl ::kernel::io::register::RegisterArray for #name { const SIZE: usize = #size; const STRIDE: usize = #stride; } )); - - match relative_base { - None => outputs.extend(quote_spanned!(span => - impl ::kernel::io::register::Array for #name {} - )), - Some(relative_base) => outputs.extend(quote_spanned!(span => - impl ::kernel::io::register::WithBase for #name { - type BaseFamily = #relative_base; - } - - impl ::kernel::io::register::RelativeRegisterArray for #name {} - )), - } } }; } From a8ab43a1089e289b58d303a1c10d0f81284087e7 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:39 +0100 Subject: [PATCH 025/110] rust: io: register: remove `Register` trait and cleanup macro With the removal of relative registers, there are only two type of registers left, fixed register and register arrays. There is not much benefit in having a common super trait for them anymore, thus remove it, and cleanup the macro rules associated with it. Signed-off-by: Gary Guo Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-15-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io/register.rs | 22 +++++++++++++--------- rust/macros/io/register.rs | 12 ++++++------ 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index 187f28f16614..9df7572aee9d 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -121,8 +121,8 @@ io::IoLoc, // }; -/// Trait implemented by all registers. -pub trait Register: Sized { +/// Trait implemented by registers with a fixed offset. +pub trait FixedRegister: Sized { /// Base type for this register. type Base: ?Sized; @@ -132,9 +132,6 @@ pub trait Register: Sized { const OFFSET: usize; } -/// Trait implemented by registers with a fixed offset. -pub trait FixedRegister: Register {} - /// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when /// passing a [`FixedRegister`] value. impl IoLoc for () @@ -201,7 +198,14 @@ fn offset(self) -> usize { } /// Trait implemented by arrays of registers. -pub trait RegisterArray: Register { +pub trait RegisterArray: Sized { + /// Base type for this register. + type Base: ?Sized; + + /// Start offset of the register. + /// + /// The interpretation of this offset depends on the type of the register. + const OFFSET: usize; /// Number of elements in the registers array. const SIZE: usize; /// Number of bytes between the start of elements in the registers array. @@ -267,8 +271,8 @@ fn try_at(idx: usize) -> Option> /// /// Implementors can be used with [`Io::write_reg`](super::Io::write_reg). pub trait LocatedRegister { - /// Register value to write. - type Value: Register; + /// Value to write. + type Value; /// Full location information at which to write the value. type Location: IoLoc; @@ -295,7 +299,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// This is used to enforce base matching. #[doc(hidden)] #[inline(always)] // for const eval only -pub const fn alias_offset>() -> usize { +pub const fn alias_offset>() -> usize { Alias::OFFSET } diff --git a/rust/macros/io/register.rs b/rust/macros/io/register.rs index 500805aa6b23..5511b8e5b875 100644 --- a/rust/macros/io/register.rs +++ b/rust/macros/io/register.rs @@ -225,11 +225,6 @@ pub(crate) fn register(def: RegDef) -> Result { #[allow(non_camel_case_types)] #(#attrs)* #vis struct #name(#storage) #args ); - - impl ::kernel::io::register::Register for #name { - type Base = #base; - const OFFSET: usize = #offset; - } )); } @@ -245,7 +240,10 @@ impl ::kernel::io::register::Register for #name { ))?, None => outputs.extend(quote_spanned!(span => - impl ::kernel::io::register::FixedRegister for #name {} + impl ::kernel::io::register::FixedRegister for #name { + type Base = #base; + const OFFSET: usize = #offset; + } #(#attrs)* #vis const #name: ::kernel::io::register::FixedRegisterLoc<#name> = ::kernel::io::register::FixedRegisterLoc::<#name>::new(); @@ -268,6 +266,8 @@ impl ::kernel::io::register::FixedRegister for #name {} impl ::kernel::io::register::Array for #name {} impl ::kernel::io::register::RegisterArray for #name { + type Base = #base; + const OFFSET: usize = #offset; const SIZE: usize = #size; const STRIDE: usize = #stride; } From a12f711dc53c1ca1d7ff4184e1609182174de87b Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:40 +0100 Subject: [PATCH 026/110] rust: io: register: unify handling of register with/without bitfields Move the `FixedRegister` from a property of register to become a property of type. Name the new trait `FixedIoLoc` indicating if I/O location of a type is unique for a specific base. Thus, bitfields become just a special case of this (where type is unique because we're generating it in the register macro), and expose feature to registers without inline bitfield definition with the `#[unique]` attribute. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-16-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io/register.rs | 114 +++++++++++++++++-------------------- rust/macros/io/register.rs | 58 ++++++++++++------- 2 files changed, 89 insertions(+), 83 deletions(-) diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index 9df7572aee9d..b6513fa0f412 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -121,61 +121,15 @@ io::IoLoc, // }; -/// Trait implemented by registers with a fixed offset. -pub trait FixedRegister: Sized { - /// Base type for this register. - type Base: ?Sized; - - /// Start offset of the register. - /// - /// The interpretation of this offset depends on the type of the register. - const OFFSET: usize; -} - /// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when -/// passing a [`FixedRegister`] value. +/// passing a [`FixedIoLoc`] value. impl IoLoc for () where - T: FixedRegister, + T: FixedIoLoc, { #[inline(always)] fn offset(self) -> usize { - T::OFFSET - } -} - -/// A [`FixedRegister`] carries its location in its type. Thus `FixedRegister` values can be used -/// as an [`IoLoc`]. -impl IoLoc for T -where - T: FixedRegister, -{ - #[inline(always)] - fn offset(self) -> usize { - T::OFFSET - } -} - -/// Location of a fixed register. -pub struct FixedRegisterLoc(PhantomData); - -impl FixedRegisterLoc { - /// Returns the location of `T`. - #[inline(always)] - // We do not implement `Default` so we can be const. - #[expect(clippy::new_without_default)] - pub const fn new() -> Self { - Self(PhantomData) - } -} - -impl IoLoc for FixedRegisterLoc -where - T: FixedRegister, -{ - #[inline(always)] - fn offset(self) -> usize { - T::OFFSET + T::LOCATION.offset() } } @@ -188,6 +142,11 @@ impl OffsetLoc { pub const fn new(offset: usize) -> Self { Self(offset, PhantomData) } + + #[inline] + pub const fn const_offset(self) -> usize { + self.0 + } } impl IoLoc for OffsetLoc { @@ -266,6 +225,17 @@ fn try_at(idx: usize) -> Option> } } +/// Trait implemented by types that indicate there is a fixed I/O location for this given type. +/// +/// Implementors can be used with [`Io::write_reg`](super::Io::write_reg). +pub trait FixedIoLoc: Sized { + /// Type of [`FixedIoLoc::LOCATION`]. + type Location: IoLoc; + + /// Location of this type within given base. + const LOCATION: Self::Location; +} + /// Trait implemented by items that contain both a register value and the absolute I/O location at /// which to write it. /// @@ -283,26 +253,17 @@ pub trait LocatedRegister { impl LocatedRegister for T where - T: FixedRegister, + T: FixedIoLoc, { - type Location = FixedRegisterLoc; + type Location = T::Location; type Value = T; #[inline(always)] - fn into_io_op(self) -> (FixedRegisterLoc, T) { - (FixedRegisterLoc::new(), self) + fn into_io_op(self) -> (T::Location, T) { + (T::LOCATION, self) } } -/// Helper function for register alias implementation. -/// -/// This is used to enforce base matching. -#[doc(hidden)] -#[inline(always)] // for const eval only -pub const fn alias_offset>() -> usize { - Alias::OFFSET -} - /// Helper function for register element alias implementation. /// /// This is used to enforce base matching and provide bounds checking. @@ -419,6 +380,35 @@ pub const fn element_alias_offset; +/// +/// pub RESET: #[unique] Reset @ 0x100; +/// } +/// +/// # fn test(mmio: Mmio<'_, Region<0x1000>>) { +/// // let mmio: Mmio<'_, Region<0x1000>>; +/// mmio.write_reg(Reset::zeroed().with_const_reset::<1>()); +/// # } +/// ``` +/// /// ## Arrays of registers /// /// Some I/O areas contain consecutive registers that share the same field layout. These areas can diff --git a/rust/macros/io/register.rs b/rust/macros/io/register.rs index 5511b8e5b875..420c3ad052b8 100644 --- a/rust/macros/io/register.rs +++ b/rust/macros/io/register.rs @@ -63,6 +63,7 @@ struct Reg { attrs: Vec, vis: Visibility, name: Ident, + unique: bool, ty: Type, array: Option, offset: RegOffset, @@ -76,16 +77,26 @@ fn parse(input: syn::parse::ParseStream<'_>) -> Result { let name = input.parse()?; let lh = input.lookahead1(); - let (ty, bitfield_storage) = if lh.peek(Token![:]) { + let (unique, ty, bitfield_storage) = if lh.peek(Token![:]) { let _: Token![:] = input.parse()?; - (input.parse()?, None) + + let mut attrs = input.call(Attribute::parse_outer)?; + let unique = attrs + .extract_if(.., |attr| attr.path().is_ident("unique")) + .count() + != 0; + if !attrs.is_empty() { + Err(Error::new_spanned(&attrs[0], "unexpected attributes"))? + } + + (unique, input.parse()?, None) } else if lh.peek(token::Paren) { let content; parenthesized!(content in input); let bitfield_storage = Some(content.parse()?); // For bitfields, bitfield macro will generate a type with the same name as `name`. - (parse_quote!(#name), bitfield_storage) + (true, parse_quote!(#name), bitfield_storage) } else { Err(lh.error())? }; @@ -150,6 +161,7 @@ fn parse(input: syn::parse::ParseStream<'_>) -> Result { attrs, vis, name, + unique, ty, array, offset, @@ -193,6 +205,7 @@ pub(crate) fn register(def: RegDef) -> Result { attrs, vis, name, + unique, ty, array, offset, @@ -207,7 +220,7 @@ pub(crate) fn register(def: RegDef) -> Result { RegOffset::Fixed { offset } => quote!(#offset), RegOffset::Alias { alias } => { quote_spanned!(alias.span().resolved_at(span) => - ::kernel::io::register::alias_offset::<#base, #alias>() + ::kernel::io::register::OffsetLoc::<#base, _>::const_offset(#alias) ) } RegOffset::ElementAlias { alias, idx } => { @@ -229,27 +242,30 @@ pub(crate) fn register(def: RegDef) -> Result { } match array { - None if bitfield.is_none() => outputs.extend(quote!( - #(#attrs)* #vis const #name: ::kernel::io::register::OffsetLoc<#base, #ty> = - ::kernel::io::register::OffsetLoc::new(#offset); - )), - - _ if bitfield.is_none() => Err(Error::new_spanned( - ty, - "defining without bitfield is not yet supported for this type of register", - ))?, - - None => outputs.extend(quote_spanned!(span => - impl ::kernel::io::register::FixedRegister for #name { - type Base = #base; - const OFFSET: usize = #offset; + None => { + if unique { + outputs.extend(quote!( + impl ::kernel::io::register::FixedIoLoc<#base> for #ty { + type Location = ::kernel::io::register::OffsetLoc<#base, #ty>; + const LOCATION: Self::Location = #name; + } + )) } - #(#attrs)* #vis const #name: ::kernel::io::register::FixedRegisterLoc<#name> = - ::kernel::io::register::FixedRegisterLoc::<#name>::new(); - )), + outputs.extend(quote_spanned!(span => + #(#attrs)* #vis const #name: ::kernel::io::register::OffsetLoc<#base, #ty> = + ::kernel::io::register::OffsetLoc::new(#offset); + )); + } Some(def) => { + if bitfield.is_none() { + Err(Error::new_spanned( + &ty, + "defining without bitfield is not yet supported for this type of register", + ))? + } + let size = &def.size; let stride = if let Some(stride) = &def.stride { outputs.extend(quote_spanned!(stride.span().resolved_at(span) => From 1473ea0e38b3e3bc76d5e9f89ea4378550ed6328 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 27 Aug 2026 17:51:42 +0900 Subject: [PATCH 027/110] gpu: nova-core: mm: Add VramAddress type Add the `VramAddress` type representing a physical address in VRAM. Also add an arithmetic helper, comparison, and operator overloads which are required in later patches for address arithmetic. Signed-off-by: Joel Fernandes [ecourtney: create mm.rs here, squashing in the arithmetic patch] [ecourtney: splice the two commit bodies] [ecourtney: drop the Pfn fields, open-coding what bitfield! generated] [ecourtney: drop align_down and the IntoVramOffset/IntoVramRange traits] [ecourtney: make checked_add() const over a plain u64, derive the ordering] [ecourtney: doc wording, header, import, and signature cleanups] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-1-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm.rs | 60 ++++++++++++++++++++++++++++++ drivers/gpu/nova-core/nova_core.rs | 1 + 2 files changed, 61 insertions(+) create mode 100644 drivers/gpu/nova-core/mm.rs diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs new file mode 100644 index 000000000000..dcd5e5e919bf --- /dev/null +++ b/drivers/gpu/nova-core/mm.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Memory management subsystems. + +#![expect(dead_code)] + +use core::{ + fmt::LowerHex, + ops, // +}; + +use kernel::fmt; + +/// Physical VRAM address in GPU video memory. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[repr(transparent)] +pub(crate) struct VramAddress(u64); + +impl VramAddress { + /// Creates an address from a raw value. + pub(crate) const fn from_raw(addr: u64) -> Self { + Self(addr) + } + + /// Returns the address as a raw value. + pub(crate) const fn into_raw(self) -> u64 { + self.0 + } + + /// Adds `rhs` to this address, returning [`None`] on overflow. + pub(crate) const fn checked_add(self, rhs: u64) -> Option { + match self.into_raw().checked_add(rhs) { + Some(addr) => Some(Self::from_raw(addr)), + None => None, + } + } +} + +impl LowerHex for VramAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + LowerHex::fmt(&self.into_raw(), f) + } +} + +impl ops::Add for VramAddress { + type Output = Self; + + fn add(self, rhs: u64) -> Self::Output { + Self::from_raw(self.into_raw() + rhs) + } +} + +impl ops::Sub for VramAddress { + type Output = u64; + + fn sub(self, rhs: Self) -> Self::Output { + self.into_raw() - rhs.into_raw() + } +} diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 35a8b1214b0e..8f59cfa97017 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -18,6 +18,7 @@ mod gpu; mod gsp; mod mctp; +mod mm; #[macro_use] mod num; mod regs; From 8e43629439b7bbdbe4336300aa33727bae9792ca Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Thu, 27 Aug 2026 17:51:43 +0900 Subject: [PATCH 028/110] gpu: nova-core: mm: Implement Alignable and Debug for VramAddress Later patches align VRAM addresses down to the PRAMIN window. Implement `Alignable` trait for `VramAddress` and plus add a `ZERO` constant. Also print the address in hex under `{:?}`, so it reads well in debug output. Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-2-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm.rs | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index dcd5e5e919bf..f9d80ffbe01d 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -10,7 +10,14 @@ ops, // }; -use kernel::fmt; +use kernel::{ + fmt, + prelude::*, + ptr::{ + Alignable, + Alignment, // + }, +}; /// Physical VRAM address in GPU video memory. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -18,6 +25,9 @@ pub(crate) struct VramAddress(u64); impl VramAddress { + /// The zero address. + pub(crate) const ZERO: Self = Self::from_raw(0); + /// Creates an address from a raw value. pub(crate) const fn from_raw(addr: u64) -> Self { Self(addr) @@ -37,12 +47,28 @@ pub(crate) const fn checked_add(self, rhs: u64) -> Option { } } +impl Alignable for VramAddress { + fn align_down(self, alignment: Alignment) -> Self { + Self::from_raw(self.into_raw().align_down(alignment)) + } + + fn align_up(self, alignment: Alignment) -> Option { + self.into_raw().align_up(alignment).map(Self::from_raw) + } +} + impl LowerHex for VramAddress { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { LowerHex::fmt(&self.into_raw(), f) } } +impl fmt::Debug for VramAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_fmt(fmt!("{:#x}", self)) + } +} + impl ops::Add for VramAddress { type Output = Self; From d3bd73dcf51d88ae116a574ce2f49209c7b48944 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 27 Aug 2026 17:51:44 +0900 Subject: [PATCH 029/110] gpu: nova-core: mm: Add PRAMIN window registers PRAMIN apertures are a crucial mechanism for direct CPU read/write to VRAM. Add the BAR0 window registers that position the PRAMIN aperture on all supported GPU architectures: Turing, Ampere, Ada (via `NV_PBUS_BAR0_WINDOW`), Hopper (via `gh100::NV_XAL_EP_BAR0_WINDOW`), and Blackwell (via `gb100::NV_XAL_EP_BAR0_WINDOW`). Hopper/Blackwell window-base registers are based on Eliot Courtney's offlist reference patch. Signed-off-by: Joel Fernandes [ecourtney: split the registers out of the PRAMIN patch into mm/regs.rs] [ecourtney: drop the register read path] [ecourtney: reword the message for the split, narrow visibility to mm] [ecourtney: plain base fields, as the bitfield cast+shift patch is dropped] [ecourtney: rename the target to VidMem, fix derives, redo the target docs] [ecourtney: annotate the register base types] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-3-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm.rs | 2 + drivers/gpu/nova-core/mm/regs.rs | 70 ++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 drivers/gpu/nova-core/mm/regs.rs diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index f9d80ffbe01d..7d24ad790310 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -19,6 +19,8 @@ }, }; +mod regs; + /// Physical VRAM address in GPU video memory. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] #[repr(transparent)] diff --git a/drivers/gpu/nova-core/mm/regs.rs b/drivers/gpu/nova-core/mm/regs.rs new file mode 100644 index 000000000000..82de6dfa4e8b --- /dev/null +++ b/drivers/gpu/nova-core/mm/regs.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Registers used by the memory management subsystems: the BAR0 PRAMIN window. + +use kernel::io::register; + +use crate::{ + bounded_enum, + driver::NovaRegisters, // +}; + +// PRAMIN window + +bounded_enum! { + /// Target memory type for the BAR0 window register. + /// + /// Only VRAM is needed by the driver. Pre-Hopper window registers also define + /// system-memory targets that are unused here; Hopper+ uses a separate register + /// without a target field. + #[derive(Debug, Copy, Clone)] + pub(super) enum Bar0WindowTarget with TryFrom> { + /// Video memory (GPU framebuffer memory). + VidMem = 0, + } +} + +register! { + base: NovaRegisters; + + /// BAR0 window control for PRAMIN access. + pub(super) NV_PBUS_BAR0_WINDOW(u32) @ 0x00001700 { + /// Target memory aperture for the window. + 25:24 target ?=> Bar0WindowTarget; + /// PRAMIN window base bits 39:16. + 23:0 base; + } +} + +pub(super) mod gh100 { + use kernel::io::register; + + use crate::driver::NovaRegisters; + + register! { + base: NovaRegisters; + + /// Hopper register for PRAMIN window. + pub(crate) NV_XAL_EP_BAR0_WINDOW(u32) @ 0x0010fd40 { + /// PRAMIN window base bits 37:16. + 21:0 base; + } + } +} + +pub(super) mod gb100 { + use kernel::io::register; + + use crate::driver::NovaRegisters; + + register! { + base: NovaRegisters; + + /// Blackwell GB10x/GB20x register for PRAMIN window. + pub(crate) NV_XAL_EP_BAR0_WINDOW(u32) @ 0x0010fd40 { + /// PRAMIN window base bits 38:16. + 22:0 base; + } + } +} From b05ff82081b7b90fdd70cc7d07b3da9d85184f88 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Thu, 27 Aug 2026 17:51:45 +0900 Subject: [PATCH 030/110] gpu: nova-core: mm: Add the memory management HAL Positioning the PRAMIN window requires writing an architecture-specific register: `NV_PBUS_BAR0_WINDOW` on Turing, Ampere and Ada, and `NV_XAL_EP_BAR0_WINDOW` with a different field width on Hopper and on Blackwell. A `MmHal` trait with one implementation per hardware family hides the register choice from the rest of the mm code, matching the layout of the driver's other HALs. Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-4-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm.rs | 1 + drivers/gpu/nova-core/mm/hal.rs | 56 +++++++++++++++++++++++++++ drivers/gpu/nova-core/mm/hal/gb100.rs | 35 +++++++++++++++++ drivers/gpu/nova-core/mm/hal/gh100.rs | 35 +++++++++++++++++ drivers/gpu/nova-core/mm/hal/tu102.rs | 37 ++++++++++++++++++ 5 files changed, 164 insertions(+) create mode 100644 drivers/gpu/nova-core/mm/hal.rs create mode 100644 drivers/gpu/nova-core/mm/hal/gb100.rs create mode 100644 drivers/gpu/nova-core/mm/hal/gh100.rs create mode 100644 drivers/gpu/nova-core/mm/hal/tu102.rs diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index 7d24ad790310..07dce4ce2473 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -19,6 +19,7 @@ }, }; +mod hal; mod regs; /// Physical VRAM address in GPU video memory. diff --git a/drivers/gpu/nova-core/mm/hal.rs b/drivers/gpu/nova-core/mm/hal.rs new file mode 100644 index 000000000000..e7fd1e38bd38 --- /dev/null +++ b/drivers/gpu/nova-core/mm/hal.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Memory management HAL. + +use kernel::{ + num::Bounded, + prelude::*, // +}; + +use crate::{ + driver::Bar0, + gpu::{ + Architecture, + Chipset, // + }, + mm::VramAddress, // +}; + +mod gb100; +mod gh100; +mod tu102; + +/// Trait implemented by per-architecture MM HALs. +/// +/// `Sync` is required so that the `&'static dyn MmHal` references can be stored in `Send` +/// structures. +pub(super) trait MmHal: Sync { + /// Positions the PRAMIN window at `base`. + /// + /// This fails if `base` is not aligned to the 64 KiB window alignment or is too large for + /// the receiving register. + fn write_pramin_window_base(&self, bar: Bar0<'_>, base: VramAddress) -> Result; +} + +/// Returns the HAL corresponding to `chipset`. +pub(super) fn mm_hal(chipset: Chipset) -> &'static dyn MmHal { + match chipset.arch() { + Architecture::Turing | Architecture::Ampere | Architecture::Ada => tu102::TU102_HAL, + Architecture::Hopper => gh100::GH100_HAL, + Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => gb100::GB100_HAL, + } +} + +/// Converts `base` into the value of the window-base register field. +/// +/// Fails with [`EINVAL`] if `base` is not aligned to the window alignment required by the register +/// field's shift, or if the shifted value does not fit within `RES` bits. +fn window_base(base: VramAddress) -> Result> { + const WINDOW_BASE_SHIFT: u32 = 16; + + Bounded::::from(base.into_raw()) + .shr_exact::() + .and_then(Bounded::try_shrink) + .ok_or(EINVAL) +} diff --git a/drivers/gpu/nova-core/mm/hal/gb100.rs b/drivers/gpu/nova-core/mm/hal/gb100.rs new file mode 100644 index 000000000000..3781e143dea7 --- /dev/null +++ b/drivers/gpu/nova-core/mm/hal/gb100.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Blackwell GB10x/GB20x memory management HAL. + +use kernel::{ + io::Io, + prelude::*, // +}; + +use crate::{ + driver::Bar0, + mm::{ + hal::{ + window_base, + MmHal, // + }, + regs, + VramAddress, // + }, +}; + +struct Gb100; + +impl MmHal for Gb100 { + fn write_pramin_window_base(&self, bar: Bar0<'_>, base: VramAddress) -> Result { + bar.write_reg( + regs::gb100::NV_XAL_EP_BAR0_WINDOW::zeroed().with_base(window_base(base)?.cast()), + ); + Ok(()) + } +} + +const GB100: Gb100 = Gb100; +pub(super) const GB100_HAL: &dyn MmHal = &GB100; diff --git a/drivers/gpu/nova-core/mm/hal/gh100.rs b/drivers/gpu/nova-core/mm/hal/gh100.rs new file mode 100644 index 000000000000..8af384db2921 --- /dev/null +++ b/drivers/gpu/nova-core/mm/hal/gh100.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Hopper memory management HAL. + +use kernel::{ + io::Io, + prelude::*, // +}; + +use crate::{ + driver::Bar0, + mm::{ + hal::{ + window_base, + MmHal, // + }, + regs, + VramAddress, // + }, +}; + +struct Gh100; + +impl MmHal for Gh100 { + fn write_pramin_window_base(&self, bar: Bar0<'_>, base: VramAddress) -> Result { + bar.write_reg( + regs::gh100::NV_XAL_EP_BAR0_WINDOW::zeroed().with_base(window_base(base)?.cast()), + ); + Ok(()) + } +} + +const GH100: Gh100 = Gh100; +pub(super) const GH100_HAL: &dyn MmHal = &GH100; diff --git a/drivers/gpu/nova-core/mm/hal/tu102.rs b/drivers/gpu/nova-core/mm/hal/tu102.rs new file mode 100644 index 000000000000..e4fe7561223c --- /dev/null +++ b/drivers/gpu/nova-core/mm/hal/tu102.rs @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Turing, Ampere and Ada memory management HAL. + +use kernel::{ + io::Io, + prelude::*, // +}; + +use crate::{ + driver::Bar0, + mm::{ + hal::{ + window_base, + MmHal, // + }, + regs, + VramAddress, // + }, +}; + +struct Tu102; + +impl MmHal for Tu102 { + fn write_pramin_window_base(&self, bar: Bar0<'_>, base: VramAddress) -> Result { + bar.write_reg( + regs::NV_PBUS_BAR0_WINDOW::zeroed() + .with_target(regs::Bar0WindowTarget::VidMem) + .with_base(window_base(base)?.cast()), + ); + Ok(()) + } +} + +const TU102: Tu102 = Tu102; +pub(super) const TU102_HAL: &dyn MmHal = &TU102; From 9aa6069d624ea877a17b7d0952c993d2eb07dc22 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Thu, 27 Aug 2026 17:51:46 +0900 Subject: [PATCH 031/110] rust: io: add static `cast()` method for views Add a compile-time checked variant of `try_cast()` using the minimum size and alignment information. Signed-off-by: Gary Guo Link: https://patch.msgid.link/20260805-typed_register-v2-1-c3ca142220a0@garyguo.net [ecourtney: fix the doc example type and doc grammar per v2 review] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-5-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- rust/kernel/io.rs | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 7a0e0ebc6c85..de8ef8e2aec4 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -447,6 +447,45 @@ fn is_empty(self) -> bool self.len() == 0 } + /// Convert into a different typed I/O view. + /// + /// The target type must be known (statically) to be of the same or smaller size to current + /// type, and the current view must be properly aligned for the target type. + /// + /// # Examples + /// + /// ```no_run + /// use kernel::io::{ + /// io_project, + /// Mmio, + /// Io, + /// Region, + /// }; + /// #[derive(FromBytes, IntoBytes)] + /// #[repr(C)] + /// struct MyStruct { field: u32, } + /// + /// # fn test(mmio: &Mmio<'_, Region<0x1000>>) { + /// // let mmio: Mmio<'_, Region<0x1000>>; + /// let whole: Mmio<'_, MyStruct> = mmio.cast(); + /// # } + /// ``` + #[inline] + fn cast(self) -> ::View<'a, U> + where + Self::Target: FromBytes + IntoBytes, + U: FromBytes + IntoBytes, + { + let view = self.as_view(); + let ptr = Self::Backend::as_ptr(view); + + const_assert!(size_of::() <= Self::Target::MIN_SIZE); + const_assert!(align_of::() <= Self::Target::MIN_ALIGN.as_usize()); + + // SAFETY: We have checked bounds and alignment, so this is a valid projection. + unsafe { Self::Backend::project_view(view, ptr.cast()) } + } + /// Try to convert into a different typed I/O view. /// /// A runtime check is performed to ensure that the target type is of same or smaller size to From a14a957c0e619437fd3a67bc435120b0b6270914 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 27 Aug 2026 17:51:47 +0900 Subject: [PATCH 032/110] gpu: nova-core: mm: Add support to use PRAMIN windows to write to VRAM PRAMIN apertures are a crucial mechanism for direct CPU read/write to VRAM. Add a `Pramin` manager whose `window_at()` returns a typed MMIO view of VRAM through the 1 MiB PRAMIN aperture in BAR0, validating the view against the VRAM region and repositioning the window as needed for the accessed address. A view borrows `Pramin` mutably, so the window cannot move while the view is in use, and it inserts an ordering point on Drop. Signed-off-by: Joel Fernandes [ecourtney: split the registers and HAL into the two preceding patches] [ecourtney: rebase w.r.t. Bar0 lifetime changes and register projections] [ecourtney: drop the window guard and mutex, use &mut self] [ecourtney: position at init to avoid reads, reposition in window_offset] [ecourtney: return typed MMIO views instead of read/write accessors] [ecourtney: insert an ordering read when a view drops] [ecourtney: declare the window location, drop the doc examples] [ecourtney: add the copyright header, doc and naming cleanups] [ecourtney: the pramin module is mm-internal] Co-developed-by: Eliot Courtney Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-6-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm.rs | 1 + drivers/gpu/nova-core/mm/pramin.rs | 178 +++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 drivers/gpu/nova-core/mm/pramin.rs diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index 07dce4ce2473..ef5b1cad56c3 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -20,6 +20,7 @@ }; mod hal; +mod pramin; mod regs; /// Physical VRAM address in GPU video memory. diff --git a/drivers/gpu/nova-core/mm/pramin.rs b/drivers/gpu/nova-core/mm/pramin.rs new file mode 100644 index 000000000000..20be3fc471ba --- /dev/null +++ b/drivers/gpu/nova-core/mm/pramin.rs @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Utilities for accessing VRAM through the PRAMIN window. + +use core::ops::Range; + +use kernel::{ + io::{ + io_project, + register, + register::OffsetLoc, + Io, + Mmio, // + }, + prelude::*, + ptr::{ + Alignable, + Alignment, // + }, + sizes::{ + SZ_1M, + SZ_64K, // + }, +}; + +use crate::{ + driver::{ + Bar0, + NovaRegisters, // + }, + gpu::Chipset, + mm::{ + hal::{ + self, + MmHal, // + }, + VramAddress, // + }, + num::IntoSafeCast, // +}; + +/// Size of the PRAMIN window (1 MiB). +const WINDOW_SIZE: usize = SZ_1M; + +/// The PRAMIN window, which is a 1 MiB window into VRAM at a fixed BAR0 offset. +#[derive(FromBytes, IntoBytes)] +struct PraminWindow([u8; WINDOW_SIZE]); + +register! { + base: NovaRegisters; + + /// Location of the window inside BAR0. + PRAMIN: PraminWindow @ 0x700000; +} + +/// Owner of the PRAMIN window state. +/// +/// [`Pramin::window_at()`] repositions the window as needed and returns a typed MMIO view into +/// it, holding the manager borrowed for the lifetime of the view. +pub(super) struct Pramin<'gpu> { + bar: Bar0<'gpu>, + hal: &'static dyn MmHal, + /// MMIO view of the PRAMIN window in BAR0. + window: Mmio<'gpu, PraminWindow>, + /// VRAM range to keep the PRAMIN window inside. + vram_range: Range, + /// Cached window position. + window_range: Range, +} + +/// Typed view of VRAM through the PRAMIN window. +/// +/// Inserts an ordering point after previous writes through the window on drop. Views returned +/// by [`PraminAccess::view()`] cannot outlive this access, so the ordering point covers every +/// write made through them. +pub(super) struct PraminAccess<'a, T> +where + T: FromBytes + IntoBytes, +{ + view: Mmio<'a, T>, +} + +impl PraminAccess<'_, T> +where + T: FromBytes + IntoBytes, +{ + /// Returns the MMIO view of the accessed location. + pub(super) fn view(&self) -> Mmio<'_, T> { + self.view + } +} + +impl Drop for PraminAccess<'_, T> +where + T: FromBytes + IntoBytes, +{ + fn drop(&mut self) { + // Insert an ordering point after previous writes through this window. + self.view.cast::().read_val(); + } +} + +impl<'gpu> Pramin<'gpu> { + /// Alignment required by the PRAMIN window. + const BASE_ALIGN: Alignment = Alignment::new::(); + + /// Creates the window manager for the given VRAM region. + pub(super) fn new( + bar: Bar0<'gpu>, + chipset: Chipset, + vram_range: Range, + ) -> Result { + let hal = hal::mm_hal(chipset); + let window = io_project!(bar, build: PRAMIN); + let base = vram_range.start.align_down(Self::BASE_ALIGN); + let window_range = Self::window_range(base)?; + hal.write_pramin_window_base(bar, base)?; + + Ok(Self { + bar, + hal, + window, + vram_range, + window_range, + }) + } + + /// Returns the VRAM range a window based at `base` exposes. + fn window_range(base: VramAddress) -> Result> { + let end = base + .checked_add(WINDOW_SIZE.into_safe_cast()) + .ok_or(EINVAL)?; + Ok(base..end) + } + + /// Check the window covers `len` bytes at `addr`, moving it if needed. + /// + /// Returns the window offset at which to perform the access. + fn window_offset(&mut self, addr: VramAddress, len: usize) -> Result { + let end = addr.checked_add(len.into_safe_cast()).ok_or(EINVAL)?; + + let inside = |r: &Range| r.contains(&addr) && end <= r.end; + if !inside(&self.vram_range) { + return Err(EINVAL); + } + + // Reposition the window if the access falls outside it. + if !inside(&self.window_range) { + let base = addr.align_down(Self::BASE_ALIGN); + let window_range = Self::window_range(base)?; + if !inside(&window_range) { + return Err(EINVAL); + } + self.hal.write_pramin_window_base(self.bar, base)?; + self.window_range = window_range; + } + + Ok((addr - self.window_range.start).into_safe_cast()) + } + + /// Return a typed MMIO view of a `T` at `vram_addr`. + /// + /// Returns an error if `vram_addr` is not aligned to `T`'s alignment, or if + /// a `T` at `vram_addr` does not fit within the VRAM region. + pub(super) fn window_at<'a, T>( + &'a mut self, + vram_addr: VramAddress, + ) -> Result> + where + T: FromBytes + IntoBytes, + { + let offset = self.window_offset(vram_addr, size_of::())?; + let view = io_project!(self.window, try: OffsetLoc::new(offset)); + + Ok(PraminAccess { view }) + } +} From b4481b9e3b5a3e1b45b5a70880cc2b730bc3700a Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 27 Aug 2026 17:51:48 +0900 Subject: [PATCH 033/110] docs: gpu: nova-core: Document the PRAMIN aperture mechanism Add documentation for the PRAMIN aperture mechanism used by nova-core for direct VRAM access. Nova only uses TARGET=VRAM for VRAM access. The SYS_MEM target values are documented for completeness but not used by the driver. Signed-off-by: Joel Fernandes [ecourtney: fix the TARGET encodings, KiB/MiB units, wording nits] [ecourtney: wrap at 80 columns] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-7-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- Documentation/gpu/nova/core/pramin.rst | 128 +++++++++++++++++++++++++ Documentation/gpu/nova/index.rst | 1 + 2 files changed, 129 insertions(+) create mode 100644 Documentation/gpu/nova/core/pramin.rst diff --git a/Documentation/gpu/nova/core/pramin.rst b/Documentation/gpu/nova/core/pramin.rst new file mode 100644 index 000000000000..f50b052d73ba --- /dev/null +++ b/Documentation/gpu/nova/core/pramin.rst @@ -0,0 +1,128 @@ +.. SPDX-License-Identifier: GPL-2.0 + +========================= +PRAMIN aperture mechanism +========================= + +.. note:: + The following description is approximate and current as of the Ampere + family. It may change for future generations and is intended to assist in + understanding the driver code. + +Introduction +============ + +PRAMIN is a hardware aperture mechanism that provides CPU access to GPU Video +RAM (VRAM) before the GPU's Memory Management Unit (MMU) and page tables are +initialized. This 1 MiB sliding window, located at a fixed offset within BAR0, +is essential for setting up page tables and other critical GPU data structures +without relying on the GPU's MMU. + +Architecture Overview +===================== + +The PRAMIN aperture mechanism is logically implemented by the GPU's PBUS (PCIe +Bus Controller Unit) and provides a CPU-accessible window into VRAM through the +PCIe interface:: + + +-----------------+ PCIe +------------------------------+ + | CPU |<----------->| GPU | + +-----------------+ | | + | +----------------------+ | + | | PBUS | | + | | (Bus Controller) | | + | | | | + | | +--------------+ <------------ [1] + | | | PRAMIN | | | + | | | Window | | | + | | | (1 MiB) | | | + | | +--------------+ | | + | | | | | + | +---------|------------+ | + | | | + | v | + | +----------------------+ <------- [2] + | | VRAM | | + | | (Several GiB) | | + | | | | + | | FB[0x0000000000] | | + | | ... | | + | | FB[0xFFFFFFFFFF] | | + | +----------------------+ | + +------------------------------+ + + [1] Window starts at BAR0 + 0x700000. + [2] Program PRAMIN to any 64 KiB-aligned VRAM boundary. + +PBUS is responsible for, among other things, handling MMIO +accesses to the BAR registers. + +PRAMIN Window Operation +======================= + +The PRAMIN window provides a 1 MiB sliding aperture that can be repositioned +over the entire VRAM address space using the ``NV_PBUS_BAR0_WINDOW`` register. + +Window Control Mechanism +------------------------- + +:: + + NV_PBUS_BAR0_WINDOW Register (0x1700): + +-------+--------+--------------------------------------+ + | 31:26 | 25:24 | 23:0 | + | RSVD | TARGET | BASE_ADDR | + | | | (bits 39:16 of VRAM address) | + +-------+--------+--------------------------------------+ + + The 24-bit BASE_ADDR field encodes bits [39:16] of the target VRAM address, + providing 40-bit (1 TiB) address space coverage with 64 KiB alignment. + + TARGET field (bits 25:24): + - 0x0: VRAM (Video Memory) + - 0x1: Reserved (unused) + - 0x2: SYS_MEM_COH (Coherent System Memory) + - 0x3: SYS_MEM_NONCOH (Non-coherent System Memory) + +.. note:: + Nova only uses TARGET=VRAM (0x0) for video memory access. The SYS_MEM + target values are documented here for hardware completeness but are + not used by the driver. + +64 KiB Alignment Requirement +---------------------------- + +The PRAMIN window must be aligned to 64 KiB boundaries in VRAM. This is enforced +by the ``BASE_ADDR`` field representing bits [39:16] of the target address:: + + VRAM Address Calculation: + actual_vram_addr = (BASE_ADDR << 16) + pramin_offset + Where: + - BASE_ADDR: 24-bit value from NV_PBUS_BAR0_WINDOW[23:0] + - pramin_offset: 20-bit offset within the PRAMIN window [0x00000-0xFFFFF] + + Example Window Positioning: + +---------------------------------------------------------+ + | VRAM Space | + | | + | 0x0000000000 +-----------------+ <-- 64 KiB aligned | + | | PRAMIN Window | | + | | (1 MiB) | | + | 0x00000FFFFF +-----------------+ | + | | + | | ^ | + | | | Window can slide | + | v | to any 64 KiB-aligned boundary | + | | + | 0x0123400000 +-----------------+ <-- 64 KiB aligned | + | | PRAMIN Window | | + | | (1 MiB) | | + | 0x01234FFFFF +-----------------+ | + | | + | ... | + | | + | 0xFFFFF00000 +-----------------+ <-- 64 KiB aligned | + | | PRAMIN Window | | + | | (1 MiB) | | + | 0xFFFFFFFFFF +-----------------+ | + +---------------------------------------------------------+ diff --git a/Documentation/gpu/nova/index.rst b/Documentation/gpu/nova/index.rst index 2afa58e8f08d..59b206238498 100644 --- a/Documentation/gpu/nova/index.rst +++ b/Documentation/gpu/nova/index.rst @@ -34,3 +34,4 @@ vGPU manager VFIO driver and the nova-drm driver. core/fwsec core/falcon core/tlv + core/pramin From 5958450e98e52adb61c4967d67b76734ff9052c0 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 27 Aug 2026 17:51:49 +0900 Subject: [PATCH 034/110] gpu: nova-core: mm: Add GpuMm centralized memory manager Introduce GpuMm as the centralized GPU memory manager. At this point in the series, GpuMm only owns the PRAMIN window for direct VRAM access; the buddy allocator and TLB manager are added later when those backing types become available. This provides a clean ownership model where GpuMm provides accessor methods for its components that can be used for memory management operations, and lets follow-on patches (such as the PRAMIN aperture self-tests) reference `pramin_mut()` cleanly. Signed-off-by: Joel Fernandes [ecourtney: squash the total VRAM end patch, drop its dev_info print] [ecourtney: take the maximum FB limit before adding one, fail with EINVAL] [ecourtney: rebase for the Bar0 lifetime and ownership changes, drop Arcs] [ecourtney: source the chipset through gsp_resources] [ecourtney: build the VRAM region in new(), pramin() becomes pramin_mut()] [ecourtney: declare mm before gsp_resources, doc wording cleanups] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-8-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gpu.rs | 18 +++++++++++- drivers/gpu/nova-core/gsp/commands.rs | 4 +++ drivers/gpu/nova-core/gsp/fw/commands.rs | 5 ++++ drivers/gpu/nova-core/mm.rs | 35 ++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index e1ac8ee9ba4d..0e270fe1dd87 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -29,6 +29,10 @@ Gsp, GspBootContext, // }, + mm::{ + GpuMm, + VramAddress, // + }, vgpu::VgpuManager, // }; @@ -283,6 +287,11 @@ pub(crate) struct Gpu<'gpu> { spec: Spec, /// Static GPU information as provided by the GSP. gsp_static_info: GetGspStaticInfoReply, + /// GPU memory manager owning memory management resources. + /// + /// Must be kept declared *before* `gsp_resources`, so that its components are dropped while + /// the GSP is still operational. + mm: GpuMm<'gpu>, /// GSP and its resources. #[pin] gsp_resources: GspResources<'gpu>, @@ -410,7 +419,14 @@ pub(crate) fn new<'a>( } info - } + }, + + // Create GPU memory manager owning memory management resources. + mm: GpuMm::new( + bar, + gsp_resources.spec.chipset, + VramAddress::from_raw(gsp_static_info.total_fb_end), + )?, }) } } diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index ffc25fd8c47b..eea1c9ed4684 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -216,6 +216,8 @@ pub(crate) struct GetGspStaticInfoReply { gpu_name: [u8; 64], /// Usable FB (VRAM) regions for driver memory allocation. pub(crate) usable_fb_regions: KVec>, + /// Exclusive end of the FB physical address space. + pub(crate) total_fb_end: u64, } impl MessageFromGsp for GetGspStaticInfoReply { @@ -231,10 +233,12 @@ fn read( for region in msg.usable_fb_regions() { usable_fb_regions.push(region, GFP_KERNEL)?; } + let total_fb_end = msg.total_fb_end().ok_or(EINVAL)?; Ok(GetGspStaticInfoReply { gpu_name: msg.gpu_name_str(), usable_fb_regions, + total_fb_end, }) } } diff --git a/drivers/gpu/nova-core/gsp/fw/commands.rs b/drivers/gpu/nova-core/gsp/fw/commands.rs index 6dc31d1bf5ae..6e85442a4b13 100644 --- a/drivers/gpu/nova-core/gsp/fw/commands.rs +++ b/drivers/gpu/nova-core/gsp/fw/commands.rs @@ -165,6 +165,11 @@ pub(crate) fn usable_fb_regions(&self) -> impl Iterator> + '_ } }) } + + /// Computes the exclusive end of the FB physical address space. + pub(crate) fn total_fb_end(&self) -> Option { + self.fb_regions().map(|reg| reg.limit).max()?.checked_add(1) + } } // SAFETY: Padding is explicit and will not contain uninitialized data. diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index ef5b1cad56c3..58dce211a337 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -19,10 +19,45 @@ }, }; +use crate::{ + driver::Bar0, + gpu::Chipset, // +}; + mod hal; mod pramin; mod regs; +/// GPU Memory Manager - owns all core MM components. +/// +/// Provides centralized ownership of memory management resources: +/// - [`pramin::Pramin`] for direct VRAM access. +pub(crate) struct GpuMm<'gpu> { + pramin: pramin::Pramin<'gpu>, +} + +impl<'gpu> GpuMm<'gpu> { + /// Creates the GPU memory manager. + pub(crate) fn new( + bar: Bar0<'gpu>, + chipset: Chipset, + total_fb_end: VramAddress, + ) -> Result { + // PRAMIN covers all physical VRAM (including GSP-reserved areas + // above the usable region, e.g. the BAR1 page directory). + let vram_region = VramAddress::ZERO..total_fb_end; + + Ok(Self { + pramin: pramin::Pramin::new(bar, chipset, vram_region)?, + }) + } + + /// Access the [`pramin::Pramin`]. + fn pramin_mut(&mut self) -> &mut pramin::Pramin<'gpu> { + &mut self.pramin + } +} + /// Physical VRAM address in GPU video memory. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] #[repr(transparent)] From 2c76c4ec9eb0c91f7b37795f83df8a34d4172285 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Thu, 27 Aug 2026 17:51:50 +0900 Subject: [PATCH 035/110] gpu: nova-core: Add self-test assertion macros and config option The existing assert! and assert_eq! macros cause a panic. For self tests in nova-core, it's inconvenient to cause a panic since these need to be run on actual hardware. Instead, define similar macros that log an error then return an Err. Also add the NOVA_CORE_SELFTESTS Kconfig option that gates the driver self-tests. Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-9-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/Kconfig | 9 +++++ drivers/gpu/nova-core/nova_core.rs | 2 + drivers/gpu/nova-core/selftest.rs | 64 ++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 drivers/gpu/nova-core/selftest.rs diff --git a/drivers/gpu/nova-core/Kconfig b/drivers/gpu/nova-core/Kconfig index f918f69e0599..cb7f0b00f796 100644 --- a/drivers/gpu/nova-core/Kconfig +++ b/drivers/gpu/nova-core/Kconfig @@ -15,3 +15,12 @@ config NOVA_CORE This driver is work in progress and may not be functional. If M is selected, the module will be called nova-core. + +config NOVA_CORE_SELFTESTS + bool "Nova Core driver self-tests" + depends on NOVA_CORE + default n + help + Build the driver self-tests and run them when the GPU is probed. + + If unsure, say N. diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 8f59cfa97017..1133c6ce5c55 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -23,6 +23,8 @@ mod num; mod regs; mod sbuffer; +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +mod selftest; mod vbios; mod vgpu; diff --git a/drivers/gpu/nova-core/selftest.rs b/drivers/gpu/nova-core/selftest.rs new file mode 100644 index 000000000000..f5b5965b7e6a --- /dev/null +++ b/drivers/gpu/nova-core/selftest.rs @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Assertion macros for driver self-tests. +//! +//! Self-tests run against live hardware during probe, so a failed assertion should not panic. These +//! macros log the failure on the device and fail the enclosing test by returning +//! [`EIO`](kernel::error::code::EIO) instead. + +/// Like [`assert!`], but logs the failure via `dev` and fails the enclosing test instead of +/// panicking. +/// +/// As with [`assert!`], a custom message with format arguments can follow the condition. +#[macro_export] +macro_rules! selftest_assert { + ($dev:expr, $cond:expr $(,)?) => { + $crate::selftest_assert!($dev, $cond, "assertion failed: {}", ::core::stringify!($cond)) + }; + ($dev:expr, $cond:expr, $($arg:tt)+) => {{ + if !$cond { + ::kernel::dev_err!( + $dev, + "Selftest: {}:{}: {}\n", + ::core::file!(), + ::core::line!(), + ::kernel::prelude::fmt!($($arg)+) + ); + return Err(::kernel::error::code::EIO); + } + }}; +} + +/// Like [`assert_eq!`], but logs the failure via `dev` and fails the enclosing test instead of +/// panicking. +/// +/// As with [`assert_eq!`], a custom message with format arguments can follow the compared values. +#[macro_export] +macro_rules! selftest_assert_eq { + ($dev:expr, $left:expr, $right:expr $(,)?) => { + match (&$left, &$right) { + (left, right) => $crate::selftest_assert!( + $dev, + left == right, + "assertion `{} == {}` failed: left {:?}, right {:?}", + ::core::stringify!($left), + ::core::stringify!($right), + left, + right + ), + } + }; + ($dev:expr, $left:expr, $right:expr, $($arg:tt)+) => { + match (&$left, &$right) { + (left, right) => $crate::selftest_assert!( + $dev, + left == right, + "assertion `left == right` failed: {}: left {:?}, right {:?}", + ::kernel::prelude::fmt!($($arg)+), + left, + right + ), + } + }; +} From e6a2c988ed96a5a3af51ed97ecba980521bf2fc0 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 27 Aug 2026 17:51:51 +0900 Subject: [PATCH 036/110] gpu: nova-core: mm: Add PRAMIN aperture self-tests Add self-tests for the PRAMIN aperture mechanism to verify correct operation during GPU probe. The tests validate various alignment requirements and corner cases. The tests are default disabled and behind CONFIG_NOVA_CORE_SELFTESTS. When enabled, tests run after GSP boot during probe. Signed-off-by: Joel Fernandes [ecourtney: convert the tests to window_at(), macros, and the new types] [ecourtney: cfg-gate the tests and expect(dead_code), not a runtime no-op] [ecourtney: run the self-tests on all architectures, drop the chipset arg] [ecourtney: test within a usable FB region, skip when none is large enough] [ecourtney: report failures without failing probe, start banner at dev_dbg] [ecourtney: removed the mm-specific Kconfig option] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-10-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/driver.rs | 3 + drivers/gpu/nova-core/gpu.rs | 12 +++ drivers/gpu/nova-core/mm.rs | 40 ++++++++- drivers/gpu/nova-core/mm/pramin.rs | 134 +++++++++++++++++++++++++++++ 4 files changed, 188 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index cf3534dd47d4..8cef4c284277 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -86,6 +86,9 @@ fn probe<'bound>( // (`try_pin_init!()` initializes fields in declaration order), lives at a pinned // stable address, and is dropped after `gpu` (struct field drop order). gpu <- Gpu::new(pdev, unsafe { &*core::ptr::from_ref(bar) }), + // Run optional GPU selftests. + #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] + _: { gpu.run_selftests(pdev) }, _reg: auxiliary::Registration::new( pdev.as_ref(), c"nova-drm", diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 0e270fe1dd87..a52a3d4d86af 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -429,6 +429,18 @@ pub(crate) fn new<'a>( )?, }) } + + /// Runs self-tests on the constructed [`Gpu`], logging failures without failing probe. + #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] + pub(crate) fn run_selftests(self: Pin<&mut Self>, pdev: &pci::Device) { + let this = self.project(); + let dev = pdev.as_ref(); + let regions = &this.gsp_static_info.usable_fb_regions; + + if let Err(err) = crate::mm::selftest::run(dev, this.mm, regions) { + dev_err!(dev, "self-tests failed: {:?}\n", err); + } + } } /// Reads the boot0 register and returns its raw value. diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index 58dce211a337..9e4338c7c393 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -3,7 +3,7 @@ //! Memory management subsystems. -#![expect(dead_code)] +#![cfg_attr(not(CONFIG_NOVA_CORE_SELFTESTS), expect(dead_code))] use core::{ fmt::LowerHex, @@ -123,3 +123,41 @@ fn sub(self, rhs: Self) -> Self::Output { self.into_raw() - rhs.into_raw() } } + +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +pub(crate) mod selftest { + use core::ops::Range; + + use kernel::{ + device, + sizes::SizeConstants, // + }; + + use super::*; + + /// Run MM subsystem self-tests during probe. + pub(crate) fn run( + dev: &device::Device, + mm: &mut GpuMm<'_>, + usable_fb_regions: &[Range], + ) -> Result { + // VRAM span the self-tests are free to overwrite, from the chosen test base. + const SELFTEST_SPAN: u64 = u64::SZ_64M; + + let base = usable_fb_regions.iter().find_map(|region| { + // Tests rely on this being 8 byte aligned for checking misalignment handling. + let base = region.start.align_up(Alignment::new::<8>())?; + (base.checked_add(SELFTEST_SPAN)? <= region.end).then_some(base) + }); + let Some(base) = base else { + dev_warn!( + dev, + "PRAMIN: skipping self-tests, no usable VRAM region of {:#x} bytes\n", + SELFTEST_SPAN + ); + return Ok(()); + }; + + pramin::selftest::run(dev, mm.pramin_mut(), VramAddress::from_raw(base)) + } +} diff --git a/drivers/gpu/nova-core/mm/pramin.rs b/drivers/gpu/nova-core/mm/pramin.rs index 20be3fc471ba..7f89c093d591 100644 --- a/drivers/gpu/nova-core/mm/pramin.rs +++ b/drivers/gpu/nova-core/mm/pramin.rs @@ -176,3 +176,137 @@ pub(super) fn window_at<'a, T>( Ok(PraminAccess { view }) } } + +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +pub(super) mod selftest { + use kernel::{ + device, + io::io_read, + sizes::SizeConstants, // + }; + + use super::*; + use crate::{ + selftest_assert, + selftest_assert_eq, // + }; + + /// Test read/write at byte granularity, at unaligned addresses. + fn test_byte_readwrite( + dev: &device::Device, + pramin: &mut Pramin<'_>, + base: VramAddress, + ) -> Result { + for i in 0u8..4 { + let addr = base + 1 + u64::from(i); + pramin.window_at::(addr)?.view().write_val(0xA0 + i); + } + + for i in 0u8..4 { + let addr = base + 1 + u64::from(i); + selftest_assert_eq!( + dev, + pramin.window_at::(addr)?.view().read_val(), + 0xA0 + i + ); + } + Ok(()) + } + + /// Test writing a `u32` and reading back as individual `u8`s. + fn test_u32_as_bytes( + dev: &device::Device, + pramin: &mut Pramin<'_>, + base: VramAddress, + ) -> Result { + let addr = base + 0x10; + let val: u32 = 0xDEADBEEF; + pramin.window_at::(addr)?.view().write_val(val); + + let window = pramin.window_at::<[u8; 4]>(addr)?; + for (i, &expected) in val.to_le_bytes().iter().enumerate() { + selftest_assert_eq!(dev, io_read!(window.view(), [build: i]), expected); + } + Ok(()) + } + + /// Test window repositioning across 1 MiB boundaries. + fn test_window_reposition( + dev: &device::Device, + pramin: &mut Pramin<'_>, + base: VramAddress, + ) -> Result { + let addr_a = base; + let addr_b = base + u64::SZ_2M; // base + 2 MiB (different 1 MiB region). + let val_a: u32 = 0x11111111; + let val_b: u32 = 0x22222222; + + pramin.window_at::(addr_a)?.view().write_val(val_a); + pramin.window_at::(addr_b)?.view().write_val(val_b); + + selftest_assert_eq!( + dev, + pramin.window_at::(addr_a)?.view().read_val(), + val_a + ); + selftest_assert_eq!( + dev, + pramin.window_at::(addr_b)?.view().read_val(), + val_b + ); + Ok(()) + } + + /// Test that offsets outside the VRAM region are rejected. + fn test_invalid_offset( + dev: &device::Device, + pramin: &mut Pramin<'_>, + vram_end: VramAddress, + ) -> Result { + selftest_assert!(dev, pramin.window_at::(vram_end).is_err()); + Ok(()) + } + + /// Test that misaligned accesses are rejected. + fn test_misaligned_access( + dev: &device::Device, + pramin: &mut Pramin<'_>, + base: VramAddress, + ) -> Result { + // `u16` at odd offset (not 2-byte aligned). + selftest_assert!(dev, pramin.window_at::(base + 0x21).is_err()); + + // `u32` at 2-byte-aligned (not 4-byte-aligned) offset. + selftest_assert!(dev, pramin.window_at::(base + 2).is_err()); + + // `u64` at a 4-byte-aligned (not 8-byte-aligned) address. + selftest_assert!(dev, pramin.window_at::(base + 0x44).is_err()); + + // A `u16` view at an even address is allowed. + pramin.window_at::(base + 0x22)?; + Ok(()) + } + + /// Run PRAMIN self-tests during probe. + /// + /// `base` is the start of a driver-usable VRAM span that the tests are free to + /// overwrite. + pub(crate) fn run( + dev: &device::Device, + pramin: &mut Pramin<'_>, + base: VramAddress, + ) -> Result { + dev_dbg!(dev, "PRAMIN: starting self-tests\n"); + + let vram_end = pramin.vram_range.end; + + test_byte_readwrite(dev, pramin, base)?; + test_u32_as_bytes(dev, pramin, base)?; + test_window_reposition(dev, pramin, base)?; + test_invalid_offset(dev, pramin, vram_end)?; + test_misaligned_access(dev, pramin, base)?; + + dev_info!(dev, "PRAMIN: self-tests passed\n"); + Ok(()) + } +} From 3282d591601949e2970b73024a00e54f0f6fdb05 Mon Sep 17 00:00:00 2001 From: Philipp Stanner Date: Sat, 5 Sep 2026 10:53:42 +0200 Subject: [PATCH 037/110] rust: Add dma_fence abstractions DMA fences are synchronisation primitives that will be needed by all Rust GPU drivers. The dma_fence framework sets a number of rules, notably: - fences must only be signaled once - all fences must be signaled at some point - fence error codes must only be set before signaling - every pointer to a fence must be backed by a reference All those rules are being addressed by these abstractions. To cleanly decouple fence issuers and consumers, two types are provided: - DriverFence: the only fence type that can be signaled and that carries driver-specific data. - Fence: the fence type to be shared with other drivers and / or userspace. The only type callbacks can be registered on. Cannot be signaled. Hereby, a Fence lives in the same chunk of memory as a DriverFence. Both share the refcount of the underlying C dma_fence. Since this implementation does not provide a custom dma_fence_backend_ops.release() function, the memory is freed by the dma_fence backend once the refcount drops to 0. To create a DriverFence, the user must first allocate a DriverFenceAllocation, so that the creation of the DriverFence later on can always succeed. Otherwise, deadlocks could occur if fences need to be created in a GPU job submission path. Synchronization is ensured by the dma_fence backend. All DriverFence's created through this abstraction must be signaled by the creator with an error code. In case a DriverFence drops without being signaled beforehand, it is signaled with -ECANCELLED as its error and a warning is printed. This allows the Rust abstraction to very cleanly decouple fence issuer and consumer by relying on the decoupling mechanisms in the C backend, which ensures through RCU and the 'signaled' fence-flag that dma_fence_backend_ops functions cannot access the potentially unloaded driver code anymore. Signalling fences on drop thus grants many advantages. Not signaling fences on drop would risk deadlock and does not grant real advantages: By definition only the drivers can ensure that a fence always represents the hardware's state correctly. This implementation models a DmaFenceContext object on which fences are to be created, thereby ensuring correct sequence numbering according to the timeline. dma_fence supports a variety of callbacks. The mandatory callbacks (get_timeline_name() and get_driver_name()) are implemented in this patch. For convenience, they store those name parameters in the fence context, saving the driver from implementing these two callbacks. Support for other callbacks (like for hardware signaling) is prepared for through the fact that both DriverFence and Fence live in the same allocation, allowing for usage of container_of from the callback to access the driver-specific data. It is expected that other callbacks, added in the future, also mostly operate on the generic data in the FenceContext. To make this safe, the implementation ensures through a lifetime that a DriverFence cannot outlive its FenceContext. Synchronization for dma_fence_ops callbacks is ensured by only running the Rust deconstructor delayed with call_rcu(), which prevents UAF-bugs should a DriverFence drop while a Fence callback is currently operating on the associated driver data. Since they can also operate on the FenceContext's data, its drop implementation also performs the necessary delay with rcu_barrier(). An additional issue discovered during the review process of this code is that there is (currently) no mechanism in Rust to prevent someone from circumventing the DriverFence's FenceContext-reference's lifetime by "forgetting" the fence, e.g. with core::mem::forget(). This would enable UAF bugs on the FenceContext. Throw a panic if this happens and document a path towards a more robust solution. Add abstractions for dma_fence in Rust. Signed-off-by: Philipp Stanner Tested-by: Daniel Almeida Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260905085343.1827305-3-phasta@kernel.org [ In dma_fence_callback(), split combined unsafe block into separate blocks for container_of!() and pointer dereference. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/bindings/bindings_helper.h | 1 + rust/helpers/dma_fence.c | 49 ++ rust/helpers/helpers.c | 1 + rust/kernel/dma_buf/dma_fence.rs | 1022 ++++++++++++++++++++++++++++++ rust/kernel/dma_buf/mod.rs | 14 + rust/kernel/lib.rs | 1 + 6 files changed, 1088 insertions(+) create mode 100644 rust/helpers/dma_fence.c create mode 100644 rust/kernel/dma_buf/dma_fence.rs create mode 100644 rust/kernel/dma_buf/mod.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 4b31aa7f432f..930e63290cdd 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -53,6 +53,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/helpers/dma_fence.c b/rust/helpers/dma_fence.c new file mode 100644 index 000000000000..549f6b6a7171 --- /dev/null +++ b/rust/helpers/dma_fence.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_dma_fence_get(struct dma_fence *f) +{ + dma_fence_get(f); +} + +__rust_helper void rust_helper_dma_fence_put(struct dma_fence *f) +{ + dma_fence_put(f); +} + +__rust_helper bool rust_helper_dma_fence_begin_signalling(void) +{ + return dma_fence_begin_signalling(); +} + +__rust_helper void rust_helper_dma_fence_end_signalling(bool cookie) +{ + dma_fence_end_signalling(cookie); +} + +__rust_helper bool rust_helper_dma_fence_is_signaled(struct dma_fence *f) +{ + return dma_fence_is_signaled(f); +} + +__rust_helper bool rust_helper_dma_fence_test_signaled_flag(struct dma_fence *f) +{ + return dma_fence_test_signaled_flag(f); +} + +__rust_helper void rust_helper_dma_fence_lock_irqsave(struct dma_fence *f, unsigned long *flags) +{ + dma_fence_lock_irqsave(f, *flags); +} + +__rust_helper void rust_helper_dma_fence_unlock_irqrestore(struct dma_fence *f, + unsigned long *flags) +{ + dma_fence_unlock_irqrestore(f, *flags); +} + +__rust_helper void rust_helper_dma_fence_set_error(struct dma_fence *f, int error) +{ + dma_fence_set_error(f, error); +} diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 440fb7638e3c..609689fba3dd 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -57,6 +57,7 @@ #include "cred.c" #include "device.c" #include "dma.c" +#include "dma_fence.c" #ifdef CONFIG_DMA_SHARED_BUFFER #include "dma-resv.c" #endif diff --git a/rust/kernel/dma_buf/dma_fence.rs b/rust/kernel/dma_buf/dma_fence.rs new file mode 100644 index 000000000000..18a43e1bb442 --- /dev/null +++ b/rust/kernel/dma_buf/dma_fence.rs @@ -0,0 +1,1022 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2025-2026 Red Hat Inc. + * Author: Philipp Stanner + */ + +//! DMA Fence support. +//! +//! Reference: +//! +//! header: [`include/linux/dma-fence.h`](srctree/include/linux/dma-fence.h) + +use crate::{ + alloc::AllocError, + bindings, + container_of, + error::to_result, + prelude::*, + types::ForeignOwnable, + types::Opaque, // +}; + +use core::{ + marker::PhantomData, + mem::ManuallyDrop, + ops::Deref, + ptr, + ptr::{ + drop_in_place, + NonNull, // + }, // +}; + +use kernel::{ + str::CString, + sync::{ + aref::{ + ARef, + AlwaysRefCounted, // + }, + atomic::{ + Atomic, + Relaxed, // + }, + rcu::rcu_barrier, // + }, // +}; + +/// VTable for dma_fence backend_ops callbacks. +// +// Mandatory dma_fence backend_ops are implemented implicitly through +// [`FenceContext`]. Additional ones shall get implemented on this trait. +pub trait FenceContextOps { + /// The generic payload data for [`DriverFence`]s created on this fctx. + type FenceDataType: Send + Sync; +} + +/// A dma-fence context. A fence context takes care of associating related fences +/// with each other, providing each with raising sequence numbers and a common +/// identifier. +#[pin_data(PinnedDrop)] +pub struct FenceContext { + /// The fence context number. + nr: u64, + /// The sequence number for the next fence created. + seqno: Atomic, + // The name parameters can be accessed by the dma_fence backend_ops. UAF + // errors are prevented by the `call_rcu()` in `drop_driver_fence_data()`. + /// The name of the driver this FenceContext's fences belong to. + driver_name: CString, + /// The name of the timeline this FenceContext's fences belong to. + timeline_name: CString, + /// The number of all unsignaled fences on this context. + // Used to prevent bugs due to forgotten fences. + // + // The lifetime on `DriverFence`s should typically prevent this from + // happening. + // + // However, we cannot fully guarantee in Rust that `DriverFence`s will not + // be forgotten, e.g., through `core::mem::forget()`. This could circumvent + // the lifetime which intends to enforce that all fences disappear before + // their context. + nr_of_unsignaled_fences: Atomic, + /// The user's data. + #[pin] + data: T, +} + +impl<'a, T: Send + Sync + FenceContextOps> FenceContext { + // This can later be extended as a vtable in case other parties need support + // for the more "exotic" callbacks. + const OPS: bindings::dma_fence_ops = bindings::dma_fence_ops { + get_driver_name: Some(Self::get_driver_name), + get_timeline_name: Some(Self::get_timeline_name), + enable_signaling: None, + signaled: None, + // Deprecated. + wait: None, + // Must never be implemented for these abstractions. + release: None, + set_deadline: None, + }; + + /// Create a new `FenceContext`. + pub fn new( + initial_seqno: u64, + driver_name: &CStr, + timeline_name: &CStr, + data: impl PinInit, + ) -> impl PinInit + where + Error: From, + { + let driver_name = CString::try_from(driver_name); + let timeline_name = CString::try_from(timeline_name); + try_pin_init!(Self { + // SAFETY: `dma_fence_context_alloc()` merely works on a global + // atomic. Parameter `1` is the number of contexts we want to + // allocate. + nr: unsafe { bindings::dma_fence_context_alloc(1) }, + seqno: Atomic::new(initial_seqno), + driver_name: driver_name?, + timeline_name: timeline_name?, + nr_of_unsignaled_fences: Atomic::new(0), + data <- data, + }) + } + + fn next_seqno(&self) -> u64 { + self.seqno.fetch_add(1, Relaxed) + } + + /// Allocate the memory for a [`DriverFence`] and already store `data` inside. + /// + /// This is needed because many times, creation of a [`DriverFence`] must not + /// fail, and allocating might deadlock in some situations. + /// + /// The `data` you pass here must not perform any operations that are illegal + /// in atomic context in its [`Drop`] implementation. + pub fn new_fence_allocation( + &self, + data: T::FenceDataType, + ) -> Result> { + let fence_data = DriverFenceData { + rcu_head: Default::default(), + // `inner` remains uninitialized until a `DriverFence` takes over. + inner: Fence { + inner: Opaque::uninit(), + }, + fctx: self, + data, + }; + + // In order to support the C dma_fence callbacks, it is necessary for + // a `Fence` and a `DriverFence` to live in the same allocation, + // because the C backend passes a dma_fence, from which the driver most + // likely wants to be able to access its `data` in `DriverFence`. + // + // Hence, we need the manage the memory manually. It will be freed by the + // C backend automatically once the refcount within `Fence` drops to 0. + let data = KBox::new(fence_data, GFP_KERNEL | __GFP_ZERO)?; + + Ok(DriverFenceAllocation { + data, + ops: &Self::OPS, + }) + } + + extern "C" fn get_driver_name(ptr: *mut bindings::dma_fence) -> *const c_char { + // SAFETY: The C backend only invokes this callback with `ptr` pointing + // to a valid, unsignaled `bindings::dma_fence`. All fences created in + // this module always reside within `Fence` which always resides in a + // `DriverFenceData`, thus satisfying the function's safety + // requirements. + let fctx = unsafe { Self::from_raw_fence(ptr) }; + + fctx.driver_name.as_char_ptr() + } + + extern "C" fn get_timeline_name(ptr: *mut bindings::dma_fence) -> *const c_char { + // SAFETY: The C backend only invokes this callback with `ptr` pointing + // to a valid, unsignaled `bindings::dma_fence`. All fences created in + // this module always reside within `Fence` which always resides in a + // `DriverFenceData`, thus satisfying the function's safety + // requirements. + let fctx = unsafe { Self::from_raw_fence(ptr) }; + + fctx.timeline_name.as_char_ptr() + } + + /// Create a [`FenceContext`] from an associated [`bindings::dma_fence`]. + /// + /// # Safety + /// + /// `ptr` must be a valid pointer to a [`bindings::dma_fence`] which resides + /// within a [`Fence`], which in turn resides in a [`DriverFenceData`]. + unsafe fn from_raw_fence(ptr: *mut bindings::dma_fence) -> &'a Self { + let opaque_fence = Opaque::cast_from(ptr); + + // SAFETY: Safe due to the function's overall safety requirements. + let fence_ptr = unsafe { container_of!(opaque_fence, Fence, inner) }; + + // CAST: `DriverFenceData` is `repr(C)` and a `Fence` is its first member. + let fence_data_ptr: *const DriverFenceData<'a, T> = fence_ptr.cast(); + + // SAFETY: Safe because of the comments directly above. + let fence_data = unsafe { &*fence_data_ptr }; + + fence_data.fctx + } +} + +#[pinned_drop] +impl PinnedDrop for FenceContext { + fn drop(self: Pin<&mut Self>) { + // Fence ops callbacks can be called on unsignaled fences. Since these + // callbacks can access the fence context and its data, it needs to be + // guaranteed that a context only drops after all associated + // `DriverFence`s have been dropped. This is unlikely to occur, but + // would result in silent UAF. Throw a panic to prevent that. + // + // TODO: + // It would be better if the fence context signals all forgotten fences + // itself. To do so, it would keep a list of unsignaled fences. That + // list's members would have to be pre-allocated (see + // `FenceCallback::new_fence_allocation()`). + if self.nr_of_unsignaled_fences.load(Relaxed) != 0 { + panic!("Forgotten fences in FenceContext."); + } + + // Ensure that the driver cannot unload while there are still dma_fence + // callbacks running. At the same time, the RCU barrier addresses the + // problem inherited by the C backend, in which backend ops callbacks + // might be accessing the fence while it is being signaled (or shortly + // after). This could cause UAF access on the fence context's + // `fctx.driver_name` and `fctx.timeline_name`. + // + // Wait for the RCU callbacks in `DriverFence::drop`. + rcu_barrier(); + } +} + +/// Error type for fence callback registration. +/// +/// Generic over `T` so that `AlreadySignaled` can return the callback to the +/// caller, allowing it to reclaim any resources owned by the callback (e.g., +/// a fence handle that needs to be signaled). +#[derive(Debug)] +pub enum CallbackError { + /// The fence was already signaled. The callback is returned so the caller + /// can extract owned resources without losing them. + AlreadySignaled(T), + /// Some other error occurred during registration. + Other(Error), +} + +impl From> for Error { + #[inline] + fn from(err: CallbackError) -> Self { + match err { + CallbackError::AlreadySignaled(_) => ENOENT, + CallbackError::Other(e) => e, + } + } +} + +impl From for CallbackError { + #[inline] + fn from(e: AllocError) -> Self { + CallbackError::Other(Error::from(e)) + } +} + +/// Trait for callbacks that can be registered on fences. +/// +/// When the fence signals, the callback will be invoked. +/// +/// # Example +/// +/// ```rust +/// use kernel::dma_buf::FenceCallback; +/// +/// struct MyCallback { +/// // Your callback state here +/// } +/// +/// impl FenceCallback for MyCallback { +/// fn on_signal(&mut self) { +/// pr_info!("Fence signaled!\n"); +/// // Handle fence completion +/// } +/// } +/// ``` +pub trait FenceCallback: Send + 'static { + /// Called when the fence is signaled. + /// + /// This is called from the fence signaling path, which may be in interrupt + /// context or with locks held, which is why `self` is only borrowed, so that + /// it cannot drop. Implementations must not sleep or perform + /// long-running operations. + /// + /// An implementation likely wants to inform itself (e.g., through a work item) + /// within this callback that the associated [`FenceCallbackRegistration`] + /// can now be dropped. + fn on_signal(&mut self); +} + +/// A callback registration on a fence. +/// +/// When this object is dropped, the callback is automatically removed if it +/// hasn't been called yet. +#[pin_data(PinnedDrop)] +pub struct FenceCallbackRegistration { + #[pin] + callback_foreign: Opaque, + callback: ManuallyDrop, + fence: ARef, +} + +impl FenceCallbackRegistration { + /// Create a [`PinInit`] closure for registering a callback on a fence. + /// + /// The actual attempt at registering the callback will take place once you + /// call an allocator's `pin_init()` function. + /// + /// On success the callback is pinned in place and will fire when the fence + /// signals. On `AlreadySignaled` the callback is returned to the caller so + /// that owned resources can be reclaimed. + pub fn new<'a>(fence: &'a Fence, callback: T) -> impl PinInit> + 'a + where + T: 'a, + { + try_pin_init!(Self { + // We need to fully initialize the fence because after + // `dma_fence_add_callback()` ran, the callback might immediately + // get invoked. + callback: ManuallyDrop::new(callback), + fence: ARef::from(fence), + callback_foreign <- Opaque::try_ffi_init(|ptr| { + // SAFETY: `fence.inner.get()` is a valid, initialized `struct + // dma_fence`. `ptr` points to the `struct dma_fence_cb` field + // within the pinned allocation, so it remains valid until + // `dma_fence_remove_callback()` in `PinnedDrop` or until the + // callback fires. + let ret = unsafe { + to_result(bindings::dma_fence_add_callback( + fence.inner.get(), + ptr, + Some(Self::dma_fence_callback), + )) + }; + match ret { + Ok(()) => Ok(()), + Err(e) => { + // SAFETY: We could not register the callback. Thus, + // C will not use it. So we can just take it back + // and pass it to the user again. + let cb_back = unsafe { ManuallyDrop::take(callback) }; + if e == ENOENT { + Err(CallbackError::AlreadySignaled(cb_back)) + } else { + Err(CallbackError::Other(e)) + } + }, + } + }), + }? CallbackError) + } + + /// Raw dma fence callback that is called by the C code. + /// + /// # Safety + /// + /// This is only called by the dma_fence subsystem with valid pointers. + unsafe extern "C" fn dma_fence_callback( + _fence: *mut bindings::dma_fence, + callback_foreign: *mut bindings::dma_fence_cb, + ) { + let ptr = Opaque::cast_from(callback_foreign).cast_mut(); + + // SAFETY: All callbacks we can receive here have been created in such a way that they are + // embedded into a `FenceCallbackRegistration`. + let reg: *mut Self = unsafe { container_of!(ptr, Self, callback_foreign) }; + + // SAFETY: `reg` is a valid `Self` pointer. + // + // The backend ensures synchronisation so whoever holds the registration object cannot drop + // it while this code is running. See `FenceCallbackRegistration::drop`. + unsafe { (*reg).callback.on_signal() }; + } + + /// Returns a reference to the fence this callback is registered on. + #[inline] + pub fn fence(&self) -> &Fence { + &self.fence + } +} + +#[pinned_drop] +impl PinnedDrop for FenceCallbackRegistration { + fn drop(self: Pin<&mut Self>) { + // Always call `dma_fence_remove_callback()`, even if the callback + // already ran. This is necessary for synchronization: + // `dma_fence_remove_callback()` acquires `fence->lock`, which ensures + // that any in-flight `dma_fence_signal()` (which calls our callback + // while holding the same lock) has completed before we free the struct. + // + // Without this, Drop can race with a concurrent signal: + // CPU0 (signal, lock held): take() -> on_signal(fence_ref) (in progress) + // CPU1 (drop): skips lock -> frees struct + // CPU0: accesses fence_ref -> use-after-free + // + // When the callback has already fired, the signal path detached the + // list node via `INIT_LIST_HEAD()`, so dma_fence_remove_callback just + // sees an empty node and returns false — the lock acquisition is the + // only thing that matters. + // + // SAFETY: The fence pointer is valid and the cb was initialized by + // `dma_fence_add_callback()` during construction. + unsafe { + bindings::dma_fence_remove_callback(self.fence.as_raw(), self.callback_foreign.get()) + }; + + // SAFETY: This is literally the drop implementation, so no one has + // dropped this so far; so we can do it now. + unsafe { ManuallyDrop::::drop(self.project().callback) }; + } +} + +// SAFETY: FenceCallbackRegistration can be sent between threads. +unsafe impl Send for FenceCallbackRegistration {} + +// SAFETY: &FenceCallbackRegistration can be shared between threads if &T can. +unsafe impl Sync for FenceCallbackRegistration where T: Sync {} + +/// The receiving counterpart of a [`DriverFence`]. +/// +/// The Rust DMA fence implementation has a dualistic design: [`DriverFence`]s +/// are the producer-side, intended to be always owned by only one party. That +/// party has the monopoly on signaling the fence. +/// +/// A [`Fence`] is the counterpart for consumers. Thus, [`Fence`]s are always +/// refcounted and can shared with an arbitrary number of parties, including +/// userspace. A [`Fence`] can only be used for actions such as checking the +/// fence's status or for registering callbacks on it. +/// +/// Once the associated [`DriverFence`] signals, all +/// [`FenceCallbackRegistration`]s registered on the [`Fence`] will be executed. +/// +/// A [`Fence`] can arbitrarily outlive its [`DriverFence`] and the +/// [`FenceContext`]. Signaling a [`DriverFence`] decouples it from its +/// [`Fence`]s. +#[repr(transparent)] +pub struct Fence { + /// The actual dma_fence passed to C. + inner: Opaque, +} + +/// Guard helper for locking within this module. +/// +/// Its only purpose for now is to avoid a number of unsafe lock-unlock cycles. +/// It is never used outside of this module. +// TODO: This should be made more canonical, probably by basing it on a +// SpinLockIrqGuard once available. +struct FenceGuard<'a> { + inner: &'a Fence, + flags: usize, +} + +impl<'a> Deref for FenceGuard<'a> { + type Target = &'a Fence; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl Drop for FenceGuard<'_> { + fn drop(&mut self) { + // SAFETY: `fence` is valid because `self` is valid. `flag_ptr` is + // merely a pointer to an integer, which lives as long as this function. + // When a `FenceGuard` exists, the lock has been taken by definition. + unsafe { bindings::dma_fence_unlock_irqrestore(self.as_raw(), &raw mut self.flags) }; + } +} + +// SAFETY: Fences are literally designed to be shared between threads. +unsafe impl Send for Fence {} +// SAFETY: Fences are literally designed to be shared between threads. +unsafe impl Sync for Fence {} + +impl Fence { + /// Check whether the fence was signaled at the moment of the function call. + /// + /// Note that this can return `true` for a [`Fence`] whose [`DriverFence`] + /// has not yet been dropped. The reason is that the fence ops callbacks can + /// cause the fence to get signaled by the C backend. + #[inline] + pub fn is_signaled(&self) -> bool { + // We should not use `dma_fence_is_signaled_locked()` here, because + // according to the C backend's recommendations, that function is + // problematic and we should avoid calling that function with a lock + // held. + + // SAFETY: Inner `fence` is valid because `self` is valid. + let ret = unsafe { bindings::dma_fence_is_signaled(self.as_raw()) }; + + // To be as robust as possible for the future we guarantee that an API + // caller can 100% rely on the signaling being completed (i.e., all + // fence callbacks ran), so we have to take the lock. + // + // The reason is that the C dma_fence backend currently does not + // carefully synchronize the `dma_fence_is_signaled()` function with the + // proper spinlock. This can lead to the function returning `true` while + // fence callbacks are still being executed. This can be mitigated by + // guarding the entire function with the spinlock. + // + // The fundamental reason is that the C backend currently does guard + // setting of the fence's signaled-bit with the fence's spinlock, but + // reading is done locklessly. + // + // See commit c8a5d5ea3ba6a. + let _ = self.lock(); + + ret + } + + /// Lock the fence. A helper only to be used internally in this module. + fn lock(&self) -> FenceGuard<'_> { + let mut guard = FenceGuard { + inner: self, + flags: 0, + }; + + // SAFETY: `fence` is valid because `self` is valid. `flag_ptr` is + // merely a pointer to an integer, whose lifetime is tied to the guard + // object. + unsafe { bindings::dma_fence_lock_irqsave(self.as_raw(), &raw mut guard.flags) }; + + guard + } + + /// Get the fence's sequence number. + #[inline] + pub fn seqno(&self) -> u64 { + // SAFETY: Valid because `self` is valid. + unsafe { (*self.as_raw()).seqno } + } + + fn as_raw(&self) -> *mut bindings::dma_fence { + self.inner.get() + } + + /// Create a [`Fence`] from a raw C [`bindings::dma_fence`]. + /// + /// # Safety + /// + /// `ptr` must point to an initialized fence that is embedded into a [`Fence`]. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::dma_fence) -> &'a Self { + // SAFETY: Safe as per the function's overall safety requirements. + unsafe { &*ptr.cast() } + } +} + +// SAFETY: These implement the C backends refcounting methods which are proven +// to work correctly. +unsafe impl AlwaysRefCounted for Fence { + fn inc_ref(&self) { + // SAFETY: `self.as_raw()` is a pointer to a valid `struct dma_fence`. + unsafe { bindings::dma_fence_get(self.as_raw()) } + } + + unsafe fn dec_ref(ptr: NonNull) { + // SAFETY: `ptr` is never a NULL pointer; and when `dec_ref()` is called + // the fence is by definition still valid. + let fence = unsafe { (*ptr.as_ptr()).inner.get() }; + + // SAFETY: `fence` was created validly above. When `dec_ref()` is called, + // there is by definition still a reference alive that can be put. + unsafe { bindings::dma_fence_put(fence) } + } +} + +// Necessary to guarantee that `inner` always comes first and can be freed by C. +// Also useful for using casts instead of container_of(). +#[repr(C)] +#[pin_data] +struct DriverFenceData<'a, T: Send + Sync + FenceContextOps> { + #[pin] + /// The inner fence. + // Must always be the first member so that unsafe casting works; but also + // necessary so that the C backend can free the allocation (coming from our + // Rust code) with kfree_rcu(). + inner: Fence, + /// Callback head for dropping this in a deferred manner through RCU. + rcu_head: bindings::callback_head, + /// Reference to access the FenceContext. + fctx: &'a FenceContext, + /// The API user's data. It is essential that the data only performs + /// operations legal in atomic context in its [`Drop`] implementation. + #[pin] + data: T::FenceDataType, +} + +/// A synchronization primitive mainly for GPU drivers. +/// +/// The Rust DMA fence implementation has a dualistic design: [`DriverFence`]s +/// are the producer-side, intended to be always owned by only one party. That +/// party has the monopoly on signaling the fence. +/// +/// A [`Fence`] is the counterpart for consumers. Thus, [`Fence`]s are always +/// refcounted and can be shared with an arbitrary number of parties, including +/// userspace. A [`Fence`] can only be used for actions such as checking the +/// fence's status or for registering callbacks on it. +/// +/// Once the associated [`DriverFence`] signals, all +/// [`FenceCallbackRegistration`]s registered on a [`Fence`] will be executed. +/// +/// A [`Fence`] can arbitrarily outlive its [`DriverFence`] and the +/// [`FenceContext`]. Signaling a [`DriverFence`] decouples it from its +/// [`Fence`]s. +/// +/// It is crucial that a [`DriverFence`] always correctly represents the state +/// of the associated job on the hardware. Especially, it is strictly necessary +/// that the owner ensures that all [`DriverFence`]s eventually get signaled. +/// As a last resort, a [`DriverFence`] will signal itself if it drops +/// unsignaled and print a warning. +/// +/// This design intends to implement the [`bindings::dma_fence_ops`] in such a +/// way that the driver-data necessary to implement the callback's functionality +/// resides in the [`FenceContext`]. Thus, a [`DriverFence`] contains a +/// reference to the context, which can be accessed in the callbacks. The +/// implementation, therefore, ensures that a [`DriverFence`] cannot outlive its +/// [`FenceContext`]. Unfortunately, this can be circumvented under certain +/// circumstances in Rust (e.g., usage of [`core::mem::forget`]). +/// +/// In the unlikely case of such violations, a panic is thrown. +/// +/// # Examples +/// +/// ``` +/// use kernel::{ +/// dma_buf::{ +/// DriverFence, +/// FenceContext, +/// FenceContextOps, +/// FenceCallback, +/// FenceCallbackRegistration, +/// }, +/// str::CString, +/// sync::aref::ARef, // +/// }; +/// use core::fmt::Display; +/// +/// struct CallbackData { } +/// +/// impl FenceCallback for CallbackData { +/// fn on_signal(&mut self) { +/// pr_info!("DmaFence callback executed.\n"); +/// } +/// } +/// +/// #[pin_data] +/// struct FenceContextData {} +/// +/// impl FenceContextData { +/// fn new() -> impl PinInit { +/// pin_init!(Self {}) +/// } +/// } +/// +/// impl FenceContextOps for FenceContextData { +/// type FenceDataType = FenceData; +/// } +/// +/// let fctx_data = FenceContextData::new(); +/// +/// +/// let mut fctx = KBox::pin_init( +/// FenceContext::new(0, c"dummy_driver", c"dummy_timeline", fctx_data), +/// GFP_KERNEL +/// )?; +/// +/// struct FenceData { +/// data: CString, +/// } +/// +/// let fence_data = FenceData { data: c"dummy_data".try_into()? }; +/// +/// let fence_alloc = fctx.new_fence_allocation(fence_data)?; +/// let mut fence = fence_alloc.new_fence(); +/// +/// let cb_data = CallbackData { }; +/// let waiting_fence = ARef::from(fence.as_fence()); +/// let cb_reg = FenceCallbackRegistration::new(&waiting_fence, cb_data); +/// let cb_reg = KBox::pin_init(cb_reg, GFP_KERNEL)?; +/// +/// // TODO signalling guards +/// assert_eq!(waiting_fence.is_signaled(), false); +/// fence.signal(Ok(())); +/// assert_eq!(waiting_fence.is_signaled(), true); +/// +/// Ok::<(), Error>(()) +/// ``` +pub struct DriverFence<'a, T: Send + Sync + FenceContextOps> { + /// The actual content of the fence. Lives in a [`NonNull`] so that its + /// memory can be managed independently. Valid until both the [`DriverFence`] + /// and all associated [`Fence`]s have disappeared. + data: NonNull>, +} + +/// A pre-prepared DMA fence, carrying the user's data and the memory it and the +/// fence reside in. Only useful for creating a [`DriverFence`]. Splitting +/// allocation and full initialization is necessary because fences cannot be +/// allocated dynamically in some circumstances (deadlock). +pub struct DriverFenceAllocation<'a, T: Send + Sync + FenceContextOps> { + /// The memory for the actual content of the fence. + /// Handed over to a [`DriverFence`], or deallocated once the + /// [`DriverFenceAllocation`] drops. + data: KBox>, + /// Reference for the ops for the associated [`FenceContext`] + ops: &'static bindings::dma_fence_ops, +} + +impl<'a, T: Send + Sync + FenceContextOps> DriverFenceAllocation<'a, T> { + /// Create a new [`DriverFence`], the signalable counterpart of a [`Fence`]. + /// + /// This increments the sequence number in the associated [`FenceContext`]. + pub fn new_fence(self) -> DriverFence<'a, T> { + // We feed the C dma_fence backend a NULL for the spinlock so that it + // uses per-fence locks automatically. + let null_ptr: *mut bindings::spinlock = ptr::null_mut(); + let seqno = self.data.fctx.next_seqno(); + let fence_ptr = self.as_raw(); + // SAFETY: `fence_ptr` has been created directly above. It will live + // at least as long as `Self`. The same applies to `&Self::OPS`. + unsafe { + bindings::dma_fence_init(fence_ptr, self.ops, null_ptr, self.data.fctx.nr, seqno) + }; + + self.data.fctx.nr_of_unsignaled_fences.fetch_add(1, Relaxed); + + // A `DriverFenceAllocation`'s purpose is to carry allocated memory, so + // that `DriverFence`s can always be created without allocating. In this + // method, ownership over that memory is transferred to the new + // `DriverFence` and managed through refcounting. The C dma_fence + // backend will ultimately free the memory once the refcount reaches 0. + let ptr = KBox::into_raw(self.data); + // SAFETY: `ptr` was just created validly directly above. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; + + DriverFence { data: ptr } + } + + fn as_raw(&self) -> *mut bindings::dma_fence { + self.data.inner.inner.get() + } +} + +impl<'a, T: Send + Sync + FenceContextOps> DriverFence<'a, T> { + fn as_raw(&self) -> *mut bindings::dma_fence { + // SAFETY: Valid because `self` is valid. + let fence_data = unsafe { &*self.data.as_ptr() }; + + fence_data.inner.inner.get() + } + + /// Create a [`DriverFence`] from a raw pointer to a [`bindings::dma_fence`]. + /// + /// # Safety + /// + /// `ptr` must be a valid pointer to a `dma_fence` that was obtained through + /// a [`DriverFence`] with matching generic data for both fence and associated + /// [`FenceContext`]. + unsafe fn from_raw(ptr: *mut bindings::dma_fence) -> Self { + let opaque_fence = Opaque::cast_from(ptr); + + // SAFETY: Safe due to the function's overall safety requirements. + let fence_ptr = unsafe { container_of!(opaque_fence, Fence, inner) }; + + // DriverFenceData is `repr(C)` and a Fence is its first member. + let fence_data_ptr = fence_ptr as *mut DriverFenceData<'a, T>; + + // SAFETY: `fence_data_ptr` was created validly above. + let data = unsafe { NonNull::new_unchecked(fence_data_ptr) }; + + Self { data } + } + + /// Return the underlying [`Fence`]. + #[inline] + pub fn as_fence(&self) -> &Fence { + // SAFETY: `self` is by definition still valid, and it cannot drop until + // this new reference is gone. + unsafe { Fence::from_raw(self.as_raw()) } + } + + /// Signal the fence. This will invoke all registered callbacks. + pub fn signal(self, res: Result) { + let fence = self.as_fence().lock(); + + // SAFETY: `fence` is valid because `self` is valid. The lock must be + // held, which we acquired directly above. + if !unsafe { bindings::dma_fence_test_signaled_flag(fence.as_raw()) } { + if let Err(err) = res { + // SAFETY: `fence` is valid because `self` is valid. The fence + // must not have been signaled yet, which we check directly above. + unsafe { bindings::dma_fence_set_error(fence.as_raw(), err.to_errno()) }; + } + // SAFETY: `fence` is valid because `self` is valid. The lock must + // be held, which we acquired above. + unsafe { bindings::dma_fence_signal_locked(fence.as_raw()) }; + } + + // SAFETY: `self.data` is valid because `self` is valid. + let fctx = unsafe { self.data.as_ref().fctx }; + let _ = fctx.nr_of_unsignaled_fences.fetch_sub(1, Relaxed); + } +} + +// SAFETY: Fences are literally designed to be shared between threads. +unsafe impl<'a, T: Send + Sync + FenceContextOps> Send for DriverFence<'a, T> {} +// SAFETY: Fences are literally designed to be shared between threads. +unsafe impl<'a, T: Send + Sync + FenceContextOps> Sync for DriverFence<'a, T> {} + +impl<'a, T: Send + Sync + FenceContextOps> Deref for DriverFence<'a, T> { + type Target = T::FenceDataType; + + fn deref(&self) -> &Self::Target { + // SAFETY: Thanks to refcounting, `data` is always valid as long as `self` is. + let data = unsafe { &*self.data.as_ptr() }; + + &data.data + } +} + +/// A borrow wrapper for [`DriverFence`]. Implements [`Deref`]. +pub struct DriverFenceBorrow<'a, T: Send + Sync + FenceContextOps> { + driver_fence: ManuallyDrop>, + _lifetime: PhantomData<&'a T>, +} + +impl<'a, T: Send + Sync + FenceContextOps> Deref for DriverFenceBorrow<'a, T> { + type Target = DriverFence<'a, T>; + + fn deref(&self) -> &Self::Target { + self.driver_fence.deref() + } +} + +// SAFETY: The Rust dma_fence abstractions are already designed around the inner +// C `dma_fence`, which can serve safely as the identification point when being +// owned by C. Moreover, safety is ensured by not dropping `DriverFence` and by +// only allowing operations without side effects on the Borrowed type. +unsafe impl ForeignOwnable for DriverFence<'_, T> { + type Borrowed<'a> + = DriverFenceBorrow<'a, T> + where + Self: 'a; + type BorrowedMut<'a> + = DriverFenceBorrow<'a, T> + where + Self: 'a; + + const FOREIGN_ALIGN: usize = core::mem::align_of::(); + + fn into_foreign(self) -> *mut c_void { + let fence = self; + + let ptr = fence.as_raw(); + + // DriverFence must not drop. + let _ = ManuallyDrop::new(fence); + + ptr.cast() + } + + unsafe fn from_foreign(ptr: *mut c_void) -> Self { + // SAFETY: Safe because the trait implementation only invokes this with + // a valid `ptr`, associated to a `DriverFence` with matching generic data. + unsafe { Self::from_raw(ptr.cast()) } + } + + unsafe fn borrow<'a>(ptr: *mut c_void) -> Self::Borrowed<'a> + where + Self: 'a, + { + // SAFETY: The trait implementation ensures that `ptr` always resides + // within a [`Fence`] within a [`DriverFenceData`]. + let driver_fence = unsafe { Self::from_raw(ptr.cast()) }; + + let driver_fence = ManuallyDrop::new(driver_fence); + + DriverFenceBorrow { + driver_fence, + _lifetime: PhantomData, + } + } + + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> Self::BorrowedMut<'a> + // FIXME: The bound below and the one above in `borrow` should actually be + // unnecessary since the compiler should be able to completely derive all + // necessary information automatically. There is currently a compiler bug + // preventing that, though: + // + // https://github.com/rust-lang/rust/issues/155430. + // + // (Help to) fix the compiler bug and remove the bounds afterwards. + where + Self: 'a, + { + // SAFETY: The trait implementation ensures that `ptr` always resides + // within a [`Fence`] within a [`DriverFenceData`]. + let driver_fence = unsafe { Self::from_raw(ptr.cast()) }; + + let driver_fence = ManuallyDrop::new(driver_fence); + + DriverFenceBorrow { + driver_fence, + _lifetime: PhantomData, + } + } +} + +impl<'a, T: Send + Sync + FenceContextOps> Drop for DriverFence<'a, T> { + fn drop(&mut self) { + let guard = self.as_fence().lock(); + + // Use dma_fence_test_signaled_flag() instead of + // dma_fence_is_signaled_locked() because the C backend wants to get rid + // of the latter. + + // SAFETY: `guard` is valid until the `call_rcu()` below. + let signaled: bool = unsafe { bindings::dma_fence_test_signaled_flag(guard.as_raw()) }; + if !signaled { + pr_err!("DriverFence drops unsignaled. Danger of memory corruption!\n"); + // SAFETY: `guard` is valid until the `call_rcu()` below. The fence + // must not have been signaled yet, which we check directly above. + unsafe { bindings::dma_fence_set_error(guard.as_raw(), ECANCELED.to_errno()) }; + // SAFETY: `guard` is valid until the `call_rcu()` below. The lock + // must be held, which we acquired above. + unsafe { bindings::dma_fence_signal_locked(guard.as_raw()) }; + + // SAFETY: `self.data` is valid because `self` is valid. + let fctx = unsafe { self.data.as_ref().fctx }; + let _ = fctx.nr_of_unsignaled_fences.fetch_sub(1, Relaxed); + } + drop(guard); + + // `DriverFenceData` could be accessed through some dma_fence + // callbacks right now. Access is being revoked in principle above by + // signaling the fence, but since the C backend does not guarantee + // perfect full synchronization, we have to wait for one grace period to + // ensure that all accessors of `DriverFenceData` (through the + // dma_fence_ops accessible through a `Fence`) are gone. + + if !core::mem::needs_drop::() { + // SAFETY: Once a `DriverFence` is initialized, the inner `fence` is + // valid and initialized. It is valid until the refcount drops to 0, + // which can earliest happen once we drop the `DriverFence`'s + // reference here. + unsafe { bindings::dma_fence_put(self.as_raw()) }; + return; + } + + // SAFETY: Valid because `self` is valid. + let rcu_head_ptr = unsafe { &raw mut (*self.data.as_ptr()).rcu_head }; + + // SAFETY: `call_rcu()` is always safe to be called. `rcu_head_ptr` was + // created validly above. The module must perform a `synchronize_rcu()` + // or `rcu_barrier()` call to guard against module unload. + unsafe { bindings::call_rcu(rcu_head_ptr, Some(drop_driver_fence_data::)) }; + } +} + +// TODO: +// The entire call_rcu() mechanism in the drop above and the code below would be +// unnecessary if C's dma_fence_signal() could be reworked in a way that after it +// ran, the caller knows that no fence_ops callbacks can be running anymore. +// In other words, if the dma_fence backend would use its spinlock for full +// synchronization. +// +// Then we could move the drop_in_place() and dma_fence_put() upwards into the +// drop() implementation and call it a day. + +/// Finally really drop this `DriverFence` +/// +/// # Safety +/// +/// `head` references the `rcu_head` field of an `DriverFenceData`. All +/// accessors to that `DriverFenceData` must be gone by now. This must be +/// ensured by signalling the associated `DriverFence` and then waiting +/// for a grace period until calling this function here. +unsafe extern "C" fn drop_driver_fence_data( + head: *mut bindings::callback_head, +) { + // SAFETY: Caller provides a pointer to the `rcu_head` field of a `DriverFenceData`. + let fence_data = unsafe { container_of!(head, DriverFenceData<'_, T>, rcu_head) }; + + // SAFETY: `fence_data` was created validly above. All the fence's data will + // only drop below, but the raw pointer to the raw C `dma_fence` remains + // valid because the reference count is only decremented at the end of the + // function. + let fence = unsafe { (*fence_data).inner.inner.get() }; + + // SAFETY: `fence_data` was created validly above. The user has already + // dropped the only conventional accessor to the user data, the `DriverFence`, + // one grace period ago. All accessors are gone now. + unsafe { drop_in_place(&raw mut (*fence_data).data) }; + + // The inner `Fence` explicitly does not get dropped because there may be + // many more users / consumers, each holding their own reference. + + // SAFETY: Once a `DriverFence` is initialized, the inner `fence` is valid + // and initialized. It is valid until the refcount drops to 0, which can + // earliest happen once we drop the `DriverFence`'s reference here. + unsafe { bindings::dma_fence_put(fence) }; + + // The actual memory the data associated with a `DriverFence` lives in + // gets freed by the C dma_fence backend once the fence's refcount reaches 0. +} diff --git a/rust/kernel/dma_buf/mod.rs b/rust/kernel/dma_buf/mod.rs new file mode 100644 index 000000000000..4764a828642e --- /dev/null +++ b/rust/kernel/dma_buf/mod.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DMA-buf subsystem abstractions. + +pub mod dma_fence; + +pub use self::dma_fence::{ + DriverFence, + Fence, + FenceCallback, + FenceCallbackRegistration, + FenceContext, + FenceContextOps, // +}; diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 7225abc64084..72f5527ba66c 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -67,6 +67,7 @@ pub mod device_id; pub mod devres; pub mod dma; +pub mod dma_buf; pub mod driver; #[cfg(CONFIG_DRM = "y")] pub mod drm; From 7aa432f5121bea8fe96fb3cfedfbd7f4e448133a Mon Sep 17 00:00:00 2001 From: Philipp Stanner Date: Sat, 5 Sep 2026 10:53:43 +0200 Subject: [PATCH 038/110] MAINTAINERS: Add entry for Rust dma-buf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rust does now have abstractions for dma_fence. These abstractions are quite complicated and require expertise with both the C and the Rust side. Therefore, using the existing entry also for maintenance of the Rust code appears reasonable. Philipp volunteers to help maintain the dma_fence abstractions. Add a corresponding MAINTAINERS entry. Signed-off-by: Philipp Stanner Acked-by: Christian König Acked-by: Sumit Semwal Tested-by: Daniel Almeida Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260905085343.1827305-4-phasta@kernel.org Signed-off-by: Danilo Krummrich --- MAINTAINERS | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index d92c7d834693..2389df80ba50 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7681,6 +7681,7 @@ F: fs/dlm/ DMA BUFFER SHARING FRAMEWORK M: Sumit Semwal M: Christian König +M: Philipp Stanner L: linux-media@vger.kernel.org L: dri-devel@lists.freedesktop.org L: linaro-mm-sig@lists.linaro.org (moderated for non-subscribers) @@ -7694,6 +7695,8 @@ F: include/linux/dma-buf.h F: include/linux/dma-buf/ F: include/linux/dma-resv.h F: rust/helpers/dma-resv.c +F: rust/helpers/dma_fence.c +F: rust/kernel/dma_buf/ K: \bdma_(?:buf|fence|resv)\b DMA GENERIC OFFLOAD ENGINE SUBSYSTEM @@ -8698,7 +8701,9 @@ T: git https://gitlab.freedesktop.org/drm/rust/kernel.git F: drivers/gpu/drm/nova/ F: drivers/gpu/drm/tyr/ F: drivers/gpu/nova-core/ +F: rust/helpers/dma_fence.c F: rust/helpers/gpu.c +F: rust/kernel/dma_buf/ F: rust/kernel/drm/ F: rust/kernel/gpu.rs F: rust/kernel/gpu/ From 73e5616f3d197c1af5a04a481fe0f13aa3913bd1 Mon Sep 17 00:00:00 2001 From: Edwin Peer Date: Wed, 9 Sep 2026 13:23:27 -0700 Subject: [PATCH 039/110] rust: dma_fence: gate abstractions on CONFIG_DMA_SHARED_BUFFER The dma_fence abstractions added by commit 3282d5916019 ("rust: Add dma_fence abstractions") are compiled unconditionally, but the C `dma_fence_*` symbols they reference live in drivers/dma-buf/dma-fence.c, which is only built when CONFIG_DMA_SHARED_BUFFER=y. Kernels with CONFIG_RUST=y and CONFIG_RUST_KERNEL_DOCTESTS=y but CONFIG_DMA_SHARED_BUFFER=n fail to link: ld.lld: error: undefined symbol: dma_fence_release >>> referenced by usercopy_64.c >>> vmlinux.o:(rust_helper_dma_fence_put) ... dma_fence_signal{,_locked}, dma_fence_context_alloc, dma_fence_init, dma_fence_{add,remove}_callback ... Wrap the dma_fence.c helper include in the same `pub mod dma_buf;` in rust/kernel/lib.rs with a matching #[cfg]. Fixes: 3282d5916019 ("rust: Add dma_fence abstractions") Signed-off-by: Edwin Peer Reviewed-by: John Hubbard Reviewed-by: Philipp Stanner Link: https://patch.msgid.link/20260909202327.33285-1-epeer@nvidia.com Signed-off-by: Danilo Krummrich --- rust/helpers/helpers.c | 2 +- rust/kernel/lib.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 609689fba3dd..2c5eecc091c4 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -57,8 +57,8 @@ #include "cred.c" #include "device.c" #include "dma.c" -#include "dma_fence.c" #ifdef CONFIG_DMA_SHARED_BUFFER +#include "dma_fence.c" #include "dma-resv.c" #endif #include "drm.c" diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 72f5527ba66c..f9ef36217bb5 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -67,6 +67,7 @@ pub mod device_id; pub mod devres; pub mod dma; +#[cfg(CONFIG_DMA_SHARED_BUFFER)] pub mod dma_buf; pub mod driver; #[cfg(CONFIG_DRM = "y")] From 9cf1bc8bbc71a6b727ed6def13f125b07cf35210 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Tue, 8 Sep 2026 18:54:12 +0900 Subject: [PATCH 040/110] rust: maple_tree: implement Send and Sync for MapleTree The C maple_tree struct contains a *mut c_void, which prevents Rust from auto-deriving Send/Sync. Following is an example error message when using MapleTree in nova-core's Vmm. This propagates up through MapleTreeAlloc to Vmm, BarUser, Gpu, and NovaCore, causing NovaCore to fail the Send bound required by pci::Driver: error[E0277]: `*mut c_void` cannot be sent between threads safely --> drivers/gpu/nova-core/driver.rs:77:22 | 77 | impl pci::Driver for NovaCore { | ^^^^^^^^ `*mut c_void` cannot be sent between threads safely | = help: within `MapleTreeAlloc<()>`, the trait `Send` is not implemented for `*mut c_void` note: required because it appears within the type `kernel::bindings::maple_tree` note: required because it appears within the type `Opaque` note: required because it appears within the type `MapleTree<()>` note: required because it appears within the type `MapleTreeAlloc<()>` = note: required for `Box, Kmalloc>` to implement `Send` note: required because it appears within the type `core::pin::Pin, Kmalloc>>` note: required because it appears within the type `Vmm` note: required because it appears within the type `BarUser` note: required because it appears within the type `Gpu` note: required because it appears within the type `NovaCore` note: required by a bound in `kernel::pci::Driver` --> rust/kernel/pci.rs:294:19 Implement Send and Sync for MapleTree. The tree contains no thread-local state, and all shared access goes through the internal ma_lock spinlock. Signed-off-by: Joel Fernandes Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng [ecourtney: also require T: Sync for the Sync impl] Acked-by: Danilo Krummrich Acked-by: Liam R. Howlett (Oracle) Link: https://patch.msgid.link/20260908-maple-tree-send-sync-v2-1-24aae5b3a990@nvidia.com Signed-off-by: Eliot Courtney --- rust/kernel/maple_tree.rs | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/rust/kernel/maple_tree.rs b/rust/kernel/maple_tree.rs index 265d6396a78a..7abe41228cb7 100644 --- a/rust/kernel/maple_tree.rs +++ b/rust/kernel/maple_tree.rs @@ -16,7 +16,11 @@ alloc::Flags, error::to_result, prelude::*, - types::{ForeignOwnable, Opaque}, + types::{ + ForeignOwnable, + NotThreadSafe, + Opaque, // + }, }; /// A maple tree optimized for storing non-overlapping ranges. @@ -240,7 +244,10 @@ pub fn lock(&self) -> MapleGuard<'_, T> { unsafe { bindings::spin_lock(self.ma_lock()) }; // INVARIANT: We just took the spinlock. - MapleGuard(self) + MapleGuard { + tree: self, + _not_send: NotThreadSafe, + } } #[inline] @@ -302,19 +309,30 @@ fn drop(mut self: Pin<&mut Self>) { } } +// SAFETY: `MapleTree` is `Send` if `T` is `Send` because `MapleTree` owns its elements. +unsafe impl Send for MapleTree {} + +// SAFETY: `&MapleTree` allows inserting and erasing entries from any thread, so `T: Send` is +// required, and shared borrows of entries require `T: Sync`. +unsafe impl Sync for MapleTree {} + /// A reference to a [`MapleTree`] that owns the inner lock. /// /// # Invariants /// /// This guard owns the inner spinlock. #[must_use = "if unused, the lock will be immediately unlocked"] -pub struct MapleGuard<'tree, T: ForeignOwnable>(&'tree MapleTree); +pub struct MapleGuard<'tree, T: ForeignOwnable> { + tree: &'tree MapleTree, + // A held spinlock must be released on the same CPU that acquired it. + _not_send: NotThreadSafe, +} impl<'tree, T: ForeignOwnable> Drop for MapleGuard<'tree, T> { #[inline] fn drop(&mut self) { // SAFETY: By the type invariants, we hold this spinlock. - unsafe { bindings::spin_unlock(self.0.ma_lock()) }; + unsafe { bindings::spin_unlock(self.tree.ma_lock()) }; } } @@ -323,7 +341,7 @@ impl<'tree, T: ForeignOwnable> MapleGuard<'tree, T> { pub fn ma_state(&mut self, first: usize, end: usize) -> MaState<'_, T> { // SAFETY: The `MaState` borrows this `MapleGuard`, so it can also borrow the `MapleGuard`s // read/write permissions to the maple tree. - unsafe { MaState::new_raw(self.0, first, end) } + unsafe { MaState::new_raw(self.tree, first, end) } } /// Load the value at the given index. @@ -375,7 +393,7 @@ pub fn ma_state(&mut self, first: usize, end: usize) -> MaState<'_, T> { #[inline] pub fn load(&mut self, index: usize) -> Option> { // SAFETY: `self.tree` contains a valid maple tree. - let ret = unsafe { bindings::mtree_load(self.0.tree.get(), index) }; + let ret = unsafe { bindings::mtree_load(self.tree.tree.get(), index) }; if ret.is_null() { return None; } From 3f1e12fce16f6e88fbaab3404df33f65c8f4df87 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Tue, 8 Sep 2026 16:51:48 +0900 Subject: [PATCH 041/110] rust: pci: add resource_flags accessor Add a `Device::resource_flags()` method to the PCI Rust abstraction, wrapping the C-side static inline `pci_resource_flags()`. The flags returned correspond to the `IORESOURCE` bitmask carried by a PCI BAR's `struct resource`. The immediate motivation is BAR layout discovery on NVIDIA GPUs: a 64-bit BAR consumes two consecutive Linux PCI resource slots (the lower 32 bits at index N and the upper 32 bits at index N+1, with the latter having no flags or size of its own). Signed-off-by: Joel Fernandes Reviewed-by: Alexandre Courbot Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260908-pci-resource-flags-v1-1-269f29f81de6@nvidia.com Signed-off-by: Eliot Courtney --- rust/helpers/pci.c | 6 ++++++ rust/kernel/io/resource.rs | 8 ++++++++ rust/kernel/pci.rs | 14 ++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/rust/helpers/pci.c b/rust/helpers/pci.c index a714cc2bfb7a..3686e405160d 100644 --- a/rust/helpers/pci.c +++ b/rust/helpers/pci.c @@ -19,6 +19,12 @@ __rust_helper resource_size_t rust_helper_pci_resource_len(struct pci_dev *pdev, return pci_resource_len(pdev, bar); } +__rust_helper unsigned long rust_helper_pci_resource_flags(const struct pci_dev *pdev, + int bar) +{ + return pci_resource_flags(pdev, bar); +} + __rust_helper bool rust_helper_dev_is_pci(const struct device *dev) { return dev_is_pci(dev); diff --git a/rust/kernel/io/resource.rs b/rust/kernel/io/resource.rs index 17b0c174cfc5..0d3b34f83334 100644 --- a/rust/kernel/io/resource.rs +++ b/rust/kernel/io/resource.rs @@ -226,10 +226,18 @@ impl Flags { /// Resource represents a memory region that must be ioremaped using `ioremap_np`. pub const IORESOURCE_MEM_NONPOSTED: Flags = Flags::new(bindings::IORESOURCE_MEM_NONPOSTED); + /// Memory region uses a 64-bit address (consumes two consecutive PCI resource slots). + pub const IORESOURCE_MEM_64: Flags = Flags::new(bindings::IORESOURCE_MEM_64); + // Always inline to optimize out error path of `build_assert`. #[inline(always)] const fn new(value: u32) -> Self { build_assert!(value as u64 <= c_ulong::MAX as u64); Flags(value as c_ulong) } + + /// Wrap a raw `c_ulong` value returned by a C API into [`Flags`]. + pub(crate) const fn from_raw(value: c_ulong) -> Self { + Flags(value) + } } diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index 3ec897709e89..19a219847c17 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -17,6 +17,7 @@ from_result, to_result, // }, + io::resource, prelude::*, str::CStr, types::Opaque, @@ -439,6 +440,19 @@ pub fn resource_len(&self, bar: u32) -> Result { Ok(unsafe { bindings::pci_resource_len(self.as_raw(), bar.try_into()?) }) } + /// Returns the resource flags (`IORESOURCE_*`) of the given PCI BAR. + pub fn resource_flags(&self, bar: u32) -> Result { + if !Bar::index_is_valid(bar) { + return Err(EINVAL); + } + + // SAFETY: + // - `bar` is a valid bar number, as guaranteed by the above call to `Bar::index_is_valid`, + // - by its type invariant `self.as_raw` is always a valid pointer to a `struct pci_dev`. + let raw = unsafe { bindings::pci_resource_flags(self.as_raw(), bar.try_into()?) }; + Ok(resource::Flags::from_raw(raw)) + } + /// Returns the PCI class as a `Class` struct. #[inline] pub fn pci_class(&self) -> Class { From b8ca2d0b7fcf5b8ff0ec174b092f732a9d3d27a3 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:39 +0900 Subject: [PATCH 042/110] gpu: nova-core: mm: Add common types for virtual memory management Add common virtual memory management types: `PAGE_SIZE` constant, `VirtualAddress` bitfield type, `Vfn` (Virtual Frame Number) type, `Pfn` (Physical Frame Number) type. Signed-off-by: Joel Fernandes [ecourtney: restore Pfn and its VramAddress conversions on the raw API] [ecourtney: make dead_code expect unconditional until the self-tests patch] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-1-8dd5d4225d2e@nvidia.com [acourbot: drop repeated "memory" in commit message.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm.rs | 142 +++++++++++++++++++++++++++++++++++- 1 file changed, 141 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index 9e4338c7c393..4234ba4d596a 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -3,7 +3,34 @@ //! Memory management subsystems. -#![cfg_attr(not(CONFIG_NOVA_CORE_SELFTESTS), expect(dead_code))] +#![expect(dead_code)] + +/// Implements `From` conversions between a frame-number type and `Bounded`. +/// +/// Each MMU version module should invoke this for the specific bit widths used by that version's +/// PTE/PDE bitfield definitions. +macro_rules! impl_frame_number_bounded { + ($type:ty, $bits:literal) => { + impl From> for $type { + fn from(val: Bounded) -> Self { + Self::new(val.get()) + } + } + + impl From<$type> for Bounded { + fn from(v: $type) -> Self { + Bounded::from_expr(v.raw() & ::kernel::bits::genmask_u64(0..=($bits - 1))) + } + } + }; +} + +/// Implements `From` conversions between [`Pfn`] and `Bounded` for bitfield interop. +macro_rules! impl_pfn_bounded { + ($bits:literal) => { + impl_frame_number_bounded!(Pfn, $bits); + }; +} use core::{ fmt::LowerHex, @@ -11,12 +38,15 @@ }; use kernel::{ + bitfield, fmt, + num::Bounded, prelude::*, ptr::{ Alignable, Alignment, // }, + sizes::SZ_4K, // }; use crate::{ @@ -58,6 +88,9 @@ fn pramin_mut(&mut self) -> &mut pramin::Pramin<'gpu> { } } +/// Page size in bytes (4 KiB). +pub(crate) const PAGE_SIZE: usize = SZ_4K; + /// Physical VRAM address in GPU video memory. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] #[repr(transparent)] @@ -124,6 +157,113 @@ fn sub(self, rhs: Self) -> Self::Output { } } +impl From for VramAddress { + fn from(pfn: Pfn) -> Self { + Self::from_raw(pfn.raw() << 12) + } +} + +bitfield! { + /// Virtual address in GPU address space. + pub(crate) struct VirtualAddress(u64) { + /// Offset within 4KB page. + 11:0 offset; + /// Virtual frame number. + 63:12 frame_number => Vfn; + } +} + +impl VirtualAddress { + /// Create a new virtual address from a raw value. + pub(crate) const fn new(addr: u64) -> Self { + Self::from_raw(addr) + } +} + +impl From for VirtualAddress { + fn from(vfn: Vfn) -> Self { + Self::zeroed().with_frame_number(vfn) + } +} + +/// Physical Frame Number. +/// +/// Represents a physical page in VRAM. +#[repr(transparent)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct Pfn(u64); + +impl Pfn { + /// Create a new PFN from a frame number. + pub(crate) const fn new(frame_number: u64) -> Self { + Self(frame_number) + } + + /// Get the raw frame number. + pub(crate) const fn raw(self) -> u64 { + self.0 + } +} + +impl From for Pfn { + fn from(addr: VramAddress) -> Self { + Self::new(addr.into_raw() >> 12) + } +} + +impl From for Pfn { + fn from(val: u64) -> Self { + Self(val) + } +} + +impl From for u64 { + fn from(pfn: Pfn) -> Self { + pfn.0 + } +} + +impl_pfn_bounded!(52); + +/// Virtual Frame Number. +/// +/// Represents a virtual page in GPU address space. +#[repr(transparent)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct Vfn(u64); + +impl Vfn { + /// Create a new VFN from a frame number. + pub(crate) const fn new(frame_number: u64) -> Self { + Self(frame_number) + } + + /// Get the raw frame number. + pub(crate) const fn raw(self) -> u64 { + self.0 + } +} + +impl From for Vfn { + fn from(addr: VirtualAddress) -> Self { + addr.frame_number() + } +} + +impl From for Vfn { + fn from(val: u64) -> Self { + Self(val) + } +} + +impl From for u64 { + fn from(vfn: Vfn) -> Self { + vfn.0 + } +} + +impl_frame_number_bounded!(Vfn, 52); + #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] pub(crate) mod selftest { use core::ops::Range; From 33be8f687bd8204965c137a5600822a1959033b8 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:40 +0900 Subject: [PATCH 043/110] gpu: nova-core: mm: Add buddy allocator and TLB to GpuMm Extend GpuMm with the remaining two memory-management components: - Buddy allocator for VRAM allocation. - TLB manager for translation buffer operations. PRAMIN was added in an earlier commit; this completes the centralized ownership model with accessor methods for each component. Signed-off-by: Joel Fernandes [ecourtney: rebase GpuMm for borrowed BAR0, pin Tlb, write regs directly] [ecourtney: size the buddy from the first usable FB region, drop dev_info] [ecourtney: update for the VramAddress raw API and typed register base] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-2-8dd5d4225d2e@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/Kconfig | 1 + drivers/gpu/nova-core/gpu.rs | 27 +++++-- drivers/gpu/nova-core/mm.rs | 24 +++++++ drivers/gpu/nova-core/mm/tlb.rs | 120 ++++++++++++++++++++++++++++++++ drivers/gpu/nova-core/regs.rs | 68 ++++++++++++++++++ 5 files changed, 234 insertions(+), 6 deletions(-) create mode 100644 drivers/gpu/nova-core/mm/tlb.rs diff --git a/drivers/gpu/nova-core/Kconfig b/drivers/gpu/nova-core/Kconfig index cb7f0b00f796..1934f17baa8b 100644 --- a/drivers/gpu/nova-core/Kconfig +++ b/drivers/gpu/nova-core/Kconfig @@ -5,6 +5,7 @@ config NOVA_CORE depends on RUST depends on !CPU_BIG_ENDIAN select AUXILIARY_BUS + select GPU_BUDDY select RUST_FW_LOADER_ABSTRACTIONS default n help diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 98834c60f8e1..e9d3c10708f1 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -6,11 +6,16 @@ device, dma::Device, fmt, + gpu::buddy::GpuBuddyParams, io::Io, num::Bounded, pci, prelude::*, - sizes::SizeConstants, // + ptr::Alignment, + sizes::{ + SizeConstants, + SZ_4K, // + }, }; use crate::{ @@ -422,11 +427,21 @@ pub(crate) fn new<'a>( }, // Create GPU memory manager owning memory management resources. - mm: GpuMm::new( - bar, - gsp_resources.spec.chipset, - VramAddress::from_raw(gsp_static_info.total_fb_end), - )?, + mm: { + let usable_vram = gsp_static_info.usable_fb_regions.first().ok_or(ENODEV)?; + let buddy_params = GpuBuddyParams { + base_offset: usable_vram.start, + size: usable_vram.end - usable_vram.start, + chunk_size: Alignment::new::(), + }; + + GpuMm::new( + bar, + gsp_resources.spec.chipset, + buddy_params, + VramAddress::from_raw(gsp_static_info.total_fb_end), + )? + }, }) } diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index 4234ba4d596a..202bb8f8ebed 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -40,6 +40,10 @@ macro_rules! impl_pfn_bounded { use kernel::{ bitfield, fmt, + gpu::buddy::{ + GpuBuddy, + GpuBuddyParams, // + }, num::Bounded, prelude::*, ptr::{ @@ -54,16 +58,23 @@ macro_rules! impl_pfn_bounded { gpu::Chipset, // }; +pub(crate) use tlb::Tlb; + mod hal; mod pramin; mod regs; +pub(super) mod tlb; /// GPU Memory Manager - owns all core MM components. /// /// Provides centralized ownership of memory management resources: +/// - [`GpuBuddy`] allocator for VRAM page table allocation. /// - [`pramin::Pramin`] for direct VRAM access. +/// - [`Tlb`] manager for translation buffer flush operations. pub(crate) struct GpuMm<'gpu> { + buddy: GpuBuddy, pramin: pramin::Pramin<'gpu>, + tlb: Pin>>, } impl<'gpu> GpuMm<'gpu> { @@ -71,6 +82,7 @@ impl<'gpu> GpuMm<'gpu> { pub(crate) fn new( bar: Bar0<'gpu>, chipset: Chipset, + buddy_params: GpuBuddyParams, total_fb_end: VramAddress, ) -> Result { // PRAMIN covers all physical VRAM (including GSP-reserved areas @@ -78,14 +90,26 @@ pub(crate) fn new( let vram_region = VramAddress::ZERO..total_fb_end; Ok(Self { + buddy: GpuBuddy::new(buddy_params)?, pramin: pramin::Pramin::new(bar, chipset, vram_region)?, + tlb: KBox::pin_init(Tlb::new(bar), GFP_KERNEL)?, }) } + /// Access the [`GpuBuddy`] allocator. + pub(crate) fn buddy(&self) -> &GpuBuddy { + &self.buddy + } + /// Access the [`pramin::Pramin`]. fn pramin_mut(&mut self) -> &mut pramin::Pramin<'gpu> { &mut self.pramin } + + /// Access the [`Tlb`] manager. + pub(crate) fn tlb(&self) -> &Tlb<'gpu> { + self.tlb.as_ref().get_ref() + } } /// Page size in bytes (4 KiB). diff --git a/drivers/gpu/nova-core/mm/tlb.rs b/drivers/gpu/nova-core/mm/tlb.rs new file mode 100644 index 000000000000..cc862e8159a1 --- /dev/null +++ b/drivers/gpu/nova-core/mm/tlb.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! TLB (Translation Lookaside Buffer) flush support for GPU MMU. +//! +//! After modifying page table entries, the GPU's TLB must be flushed to +//! ensure the new mappings take effect. This module provides TLB flush +//! functionality for virtual memory managers. +//! +//! # Examples +//! +//! ```ignore +//! use crate::mm::tlb::Tlb; +//! +//! fn page_table_update(tlb: &Tlb, pdb_addr: VramAddress) -> Result<()> { +//! // ... modify page tables ... +//! +//! // Flush TLB to make changes visible (polls for completion). +//! tlb.flush(pdb_addr)?; +//! +//! Ok(()) +//! } +//! ``` + +use kernel::{ + io::poll::read_poll_timeout, + io::Io, + new_mutex, + prelude::*, + sync::Mutex, + time::Delta, // +}; + +use crate::{ + bounded_enum, + driver::Bar0, + mm::VramAddress, + regs, // +}; + +bounded_enum! { + /// TLB invalidation acknowledgment scope. + /// + /// Controls how far the hardware waits for the invalidation to propagate + /// before clearing the `trigger` bit of `NV_TLB_FLUSH_CTRL`. + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + pub(crate) enum TlbAckMode with TryFrom> { + /// Fire-and-forget: no acknowledgment required. + None = 0, + /// Wait for acknowledgment from all consumers, including remote GPUs + /// reachable over NVLink. + /// + /// Globally is strictly required only during unmap or permission + /// tightening, because the backing memory may be reassigned after the + /// flush returns and a stale TLB entry could let the GPU access freed + /// memory. For new mapping or relaxing permissions, a stale entry would + /// merely cause a redundant fault and retry, so [`TlbAckMode::None`] + /// would suffice. + Globally = 1, + /// Wait for acknowledgment from consumers within the local NVLink + /// fabric node only; skip cross-node ack. + Intranode = 2, + } +} + +/// TLB manager for GPU translation buffer operations. +#[pin_data] +pub(crate) struct Tlb<'gpu> { + bar: Bar0<'gpu>, + /// TLB flush serialization lock: This lock is designed to be acquired during + /// the DMA fence signalling critical path. It should NEVER be held across any + /// reclaimable CPU memory allocations because the memory reclaim path can + /// call `dma_fence_wait()` (when implemented), which would deadlock if lock held. + #[pin] + lock: Mutex<()>, +} + +impl<'gpu> Tlb<'gpu> { + /// Create a new TLB manager. + pub(super) fn new(bar: Bar0<'gpu>) -> impl PinInit { + pin_init!(Self { + bar, + lock <- new_mutex!((), "tlb_flush"), + }) + } + + /// Flush the GPU TLB for a specific page directory base. + /// + /// This invalidates all TLB entries associated with the given PDB address. + /// Must be called after modifying page table entries to ensure the GPU sees + /// the updated mappings. + pub(super) fn flush(&self, pdb_addr: VramAddress) -> Result { + let _guard = self.lock.lock(); + + // Write PDB address. + self.bar.write_reg(regs::NV_TLB_FLUSH_PDB_LO::from_pdb_addr( + pdb_addr.into_raw(), + )); + self.bar.write_reg(regs::NV_TLB_FLUSH_PDB_HI::from_pdb_addr( + pdb_addr.into_raw(), + )); + + // Trigger flush. + self.bar.write_reg( + regs::NV_TLB_FLUSH_CTRL::zeroed() + .with_all_va(true) + .with_ack(TlbAckMode::None) + .with_trigger(true), + ); + + // Poll for completion. + read_poll_timeout( + || Ok(self.bar.read(regs::NV_TLB_FLUSH_CTRL)), + |ctrl: ®s::NV_TLB_FLUSH_CTRL| !ctrl.trigger(), + Delta::ZERO, + Delta::from_secs(2), + )?; + + Ok(()) + } +} diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index bf1f3a97c632..9978fb2803b0 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -10,6 +10,7 @@ sizes::SizeConstants, time, // }; +use pin_init::Zeroable; use crate::{ driver::NovaRegisters, @@ -26,6 +27,7 @@ PFalconRegisters, PeregrineCoreSelect, // }, + mm::tlb::TlbAckMode, // }; // PBUS @@ -471,3 +473,69 @@ pub(crate) mod gb202 { } } } + +// MMU TLB + +register! { + base: NovaRegisters; + + /// TLB flush register: PDB address lower bits. + pub(crate) NV_TLB_FLUSH_PDB_LO(u32) @ 0x00b830a0 { + /// PDB address bits [39:8]. + 31:0 pdb_lo => u32; + } + + /// TLB flush register: PDB address higher bits. + pub(crate) NV_TLB_FLUSH_PDB_HI(u32) @ 0x00b830a4 { + /// PDB address bits [47:40]. + 7:0 pdb_hi => u8; + } + + /// TLB flush control register. + pub(crate) NV_TLB_FLUSH_CTRL(u32) @ 0x00b830b0 { + /// Invalidate every VA in the PDB selected by `NV_TLB_FLUSH_PDB_LO/HI`. + 0:0 all_va => bool; + /// Invalidate TLBs for all PDBs (ignores `NV_TLB_FLUSH_PDB_LO/HI`). + 1:1 all_pdb => bool; + /// Restrict the flush to the HUB MMU's TLBs; skip broadcasting to the + /// per-GPC L2 TLBs. + /// + /// The GPU MMU has a two-level TLB hierarchy: + /// 1. The *HUB MMU* sits at the top and serves memory requests from + /// "host-side" engines: the host/channel interface, copy engines, + /// display, and BAR1/BAR2 accesses. + /// 2. Each GPC (Graphics Processing Cluster — the block that houses + /// shader cores / SMs) has its own L2 TLB that serves requests from + /// the compute and graphics engines inside the cluster. + /// + /// When set, only the HUB TLBs are invalidated. This is a performance + /// optimization for flushes that only affect HUB-side mappings (e.g. + /// BAR1/BAR2 windows), where fanning the invalidation out to every + /// GPC's L2 TLB would be wasted work. Must be false when flushing + /// mappings that may be cached by compute/graphics engines. + 2:2 hubtlb_only => bool; + /// Invalidation acknowledgment scope. See [`TlbAckMode`] for details. + 8:7 ack ?=> TlbAckMode; + /// Write 1 to kick off the flush. Hardware clears this bit when the + /// flush completes; reads as 1 while the flush is in progress. + 31:31 trigger => bool; + } +} + +impl NV_TLB_FLUSH_PDB_LO { + /// Create a register value from a PDB address. + /// + /// Extracts bits [39:8] of the address and shifts it right by 8 bits. + pub(crate) fn from_pdb_addr(addr: u64) -> Self { + Self::zeroed().with_pdb_lo(((addr >> 8) & 0xFFFF_FFFF) as u32) + } +} + +impl NV_TLB_FLUSH_PDB_HI { + /// Create a register value from a PDB address. + /// + /// Extracts bits [47:40] of the address and shifts it right by 40 bits. + pub(crate) fn from_pdb_addr(addr: u64) -> Self { + Self::zeroed().with_pdb_hi(((addr >> 40) & 0xFF) as u8) + } +} From a5dfe5955c6beff8403ce966efde6822700a6634 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:41 +0900 Subject: [PATCH 044/110] gpu: nova-core: mm: Add common types for all page table formats Add common page table types shared between MMU v2 and v3. These types are hardware-agnostic and used by both MMU versions. Signed-off-by: Joel Fernandes Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-3-8dd5d4225d2e@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm.rs | 1 + drivers/gpu/nova-core/mm/pagetable.rs | 158 ++++++++++++++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 drivers/gpu/nova-core/mm/pagetable.rs diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index 202bb8f8ebed..377cc9d44702 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -61,6 +61,7 @@ macro_rules! impl_pfn_bounded { pub(crate) use tlb::Tlb; mod hal; +pub(super) mod pagetable; mod pramin; mod regs; pub(super) mod tlb; diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs new file mode 100644 index 000000000000..ed0f3d731c63 --- /dev/null +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Common page table types shared between MMU v2 and v3. +//! +//! This module provides foundational types used by both MMU versions: +//! - Page table level hierarchy +//! - Memory aperture types for PDEs and PTEs + +#![expect(dead_code)] + +use kernel::num::Bounded; + +use crate::gpu::Architecture; + +/// Extracts the page table index at a given level from a virtual address. +pub(super) trait VaLevelIndex { + /// Return the page table index at `level` for this virtual address. + fn level_index(&self, level: u64) -> u64; +} + +/// MMU version enumeration. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum MmuVersion { + /// MMU v2 for Turing/Ampere/Ada. + V2, + /// MMU v3 for Hopper and later. + V3, +} + +impl From for MmuVersion { + fn from(arch: Architecture) -> Self { + match arch { + Architecture::Turing | Architecture::Ampere | Architecture::Ada => Self::V2, + Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { + Self::V3 + } + } + } +} + +/// Page Table Level hierarchy for MMU v2/v3. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum PageTableLevel { + /// Level 0 - Page Directory Base (root). + Pdb, + /// Level 1 - Intermediate page directory. + L1, + /// Level 2 - Intermediate page directory. + L2, + /// Level 3 - Intermediate page directory or dual PDE (version-dependent). + L3, + /// Level 4 - PTE level for v2, intermediate page directory for v3. + L4, + /// Level 5 - PTE level used for MMU v3 only. + L5, +} + +impl PageTableLevel { + /// Number of entries per page table (512 for 4KB pages). + pub(super) const ENTRIES_PER_TABLE: usize = 512; + + /// Get the next level in the hierarchy. + pub(super) const fn next(&self) -> Option { + match self { + Self::Pdb => Some(Self::L1), + Self::L1 => Some(Self::L2), + Self::L2 => Some(Self::L3), + Self::L3 => Some(Self::L4), + Self::L4 => Some(Self::L5), + Self::L5 => None, + } + } + + /// Convert level to index. + pub(super) const fn as_index(&self) -> u64 { + match self { + Self::Pdb => 0, + Self::L1 => 1, + Self::L2 => 2, + Self::L3 => 3, + Self::L4 => 4, + Self::L5 => 5, + } + } +} + +/// Memory aperture for Page Table Entries (`PTE`s). +/// +/// Determines which memory region the `PTE` points to. +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(super) enum AperturePte { + /// Local video memory (VRAM). + #[default] + VideoMemory = 0, + /// Peer GPU's video memory. + PeerMemory = 1, + /// System memory with cache coherence. + SystemCoherent = 2, + /// System memory without cache coherence. + SystemNonCoherent = 3, +} + +// TODO[FPRI]: Replace with `#[derive(FromPrimitive)]` when available. +impl From> for AperturePte { + fn from(val: Bounded) -> Self { + match *val { + 0 => Self::VideoMemory, + 1 => Self::PeerMemory, + 2 => Self::SystemCoherent, + 3 => Self::SystemNonCoherent, + _ => Self::VideoMemory, + } + } +} + +// TODO[FPRI]: Replace with `#[derive(ToPrimitive)]` when available. +impl From for Bounded { + fn from(val: AperturePte) -> Self { + Bounded::from_expr(val as u64 & 0x3) + } +} + +/// Memory aperture for Page Directory Entries (`PDE`s). +/// +/// Note: For `PDE`s, `Invalid` (0) means the entry is not valid. +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(super) enum AperturePde { + /// Invalid/unused entry. + #[default] + Invalid = 0, + /// Page table is in video memory. + VideoMemory = 1, + /// Page table is in system memory with coherence. + SystemCoherent = 2, + /// Page table is in system memory without coherence. + SystemNonCoherent = 3, +} + +// TODO[FPRI]: Replace with `#[derive(FromPrimitive)]` when available. +impl From> for AperturePde { + fn from(val: Bounded) -> Self { + match *val { + 1 => Self::VideoMemory, + 2 => Self::SystemCoherent, + 3 => Self::SystemNonCoherent, + _ => Self::Invalid, + } + } +} + +// TODO[FPRI]: Replace with `#[derive(ToPrimitive)]` when available. +impl From for Bounded { + fn from(val: AperturePde) -> Self { + Bounded::from_expr(val as u64 & 0x3) + } +} From 239bb057a81dca1623857102cfd963e1305aadc1 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:42 +0900 Subject: [PATCH 045/110] gpu: nova-core: mm: pagetable: Add PteOps trait Introduce a trait for GPU Page Table Entries (PTEs). New `read()`/`write()` helpers are provided that go through a `Pramin`). The forthcoming MMU v2, v3 PTE structs will each implement `PteOps`, allowing the later page-table walker and mapper to call PTE operations. Signed-off-by: Joel Fernandes [ecourtney: replace PraminWindow accesses with typed MMIO views] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-4-8dd5d4225d2e@nvidia.com [acourbot: use vertical import style.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm/pagetable.rs | 50 +++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs index ed0f3d731c63..d5c02a0ea4fb 100644 --- a/drivers/gpu/nova-core/mm/pagetable.rs +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -8,9 +8,20 @@ #![expect(dead_code)] -use kernel::num::Bounded; +use kernel::{ + io::Io, + num::Bounded, + prelude::*, // +}; -use crate::gpu::Architecture; +use crate::{ + gpu::Architecture, + mm::{ + pramin, + Pfn, + VramAddress, // + }, +}; /// Extracts the page table index at a given level from a virtual address. pub(super) trait VaLevelIndex { @@ -84,6 +95,41 @@ pub(super) const fn as_index(&self) -> u64 { } } +// Trait abstractions for page table operations. + +/// Operations on Page Table Entries (`PTE`s). +pub(super) trait PteOps: Copy + core::fmt::Debug + Into { + /// Create a `PTE` from a raw `u64` value. + fn from_raw(val: u64) -> Self; + + /// Create an invalid `PTE`. + fn invalid() -> Self; + + /// Create a valid `PTE` for the given memory aperture. + fn new(aperture: AperturePte, pfn: Pfn, writable: bool) -> Self; + + /// Check if this `PTE` is valid. + fn is_valid(&self) -> bool; + + /// Get the physical frame number. + fn frame_number(&self) -> Pfn; + + /// Read a `PTE` from VRAM. + fn read(pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { + let val = pramin.window_at::(addr)?.view().read_val(); + Ok(Self::from_raw(val)) + } + + /// Write this `PTE` to VRAM. + fn write(&self, pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { + pramin + .window_at::(addr)? + .view() + .write_val((*self).into()); + Ok(()) + } +} + /// Memory aperture for Page Table Entries (`PTE`s). /// /// Determines which memory region the `PTE` points to. From 85a7542510cdff8bbeef9fb016e98a2f8bcd1171 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:43 +0900 Subject: [PATCH 046/110] gpu: nova-core: mm: pagetable: Add PdeOps trait Introduce a trait for GPU Page Directory Entries (PDEs). Default `read()`/`write()` helpers via a `Pramin` are provided. The forthcoming MMU v2, v3 PDE structs will each implement `PdeOps`, allowing the later page-table walker and mapper to call PDE operations. Signed-off-by: Joel Fernandes [ecourtney: replace PraminWindow accesses with typed MMIO views] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-5-8dd5d4225d2e@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm/pagetable.rs | 41 +++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs index d5c02a0ea4fb..03804da784cc 100644 --- a/drivers/gpu/nova-core/mm/pagetable.rs +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -130,6 +130,47 @@ fn write(&self, pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { } } +/// Operations on Page Directory Entries (`PDE`s). +pub(super) trait PdeOps: Copy + core::fmt::Debug + Into { + /// Create a `PDE` from a raw `u64` value. + fn from_raw(val: u64) -> Self; + + /// Create a valid `PDE` pointing to a page table in the given aperture. + fn new(aperture: AperturePde, table_pfn: Pfn) -> Self; + + /// Create an invalid `PDE`. + fn invalid() -> Self; + + /// Check if this `PDE` is valid. + fn is_valid(&self) -> bool; + + /// Get the memory aperture of this `PDE`. + fn aperture(&self) -> AperturePde; + + /// Get the VRAM address of the page table. + fn table_vram_address(&self) -> VramAddress; + + /// Read a `PDE` from VRAM. + fn read(pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { + let val = pramin.window_at::(addr)?.view().read_val(); + Ok(Self::from_raw(val)) + } + + /// Write this `PDE` to VRAM. + fn write(&self, pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { + pramin + .window_at::(addr)? + .view() + .write_val((*self).into()); + Ok(()) + } + + /// Check if this `PDE` is valid and points to video memory. + fn is_valid_vram(&self) -> bool { + self.is_valid() && self.aperture() == AperturePde::VideoMemory + } +} + /// Memory aperture for Page Table Entries (`PTE`s). /// /// Determines which memory region the `PTE` points to. From 71e99998a6559d71e5d6eff85afcc0f7f99ece25 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:44 +0900 Subject: [PATCH 047/110] gpu: nova-core: mm: pagetable: Add DualPdeOps trait Introduce a trait for 128-bit Dual Page Directory Entries. The `read()`/`write()` helpers issue two 64-bit accesses through a `Pramin` to load/store the 128-bit value. Signed-off-by: Joel Fernandes [ecourtney: replace PraminWindow accesses with typed MMIO views] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-6-8dd5d4225d2e@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm/pagetable.rs | 41 +++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs index 03804da784cc..4e63f79b95e5 100644 --- a/drivers/gpu/nova-core/mm/pagetable.rs +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -171,6 +171,47 @@ fn is_valid_vram(&self) -> bool { } } +/// Operations on Dual Page Directory Entries (128-bit `DualPde`s). +pub(super) trait DualPdeOps: Copy + core::fmt::Debug { + /// Create a `DualPde` from raw 128-bit value (two `u64`s). + fn from_raw(big: u64, small: u64) -> Self; + + /// Create a `DualPde` with only the small page table pointer set. + fn new_small(table_pfn: Pfn) -> Self; + + /// Check if the small page table pointer is valid. + fn has_small(&self) -> bool; + + /// Get the small page table VRAM address. + fn small_vram_address(&self) -> VramAddress; + + /// Get the raw `u64` value of the big PDE. + fn big_raw_u64(&self) -> u64; + + /// Get the raw `u64` value of the small PDE. + fn small_raw_u64(&self) -> u64; + + /// Read a dual PDE (128-bit) from VRAM. + fn read(pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { + let lo = pramin.window_at::(addr)?.view().read_val(); + let hi = pramin.window_at::(addr + 8)?.view().read_val(); + Ok(Self::from_raw(lo, hi)) + } + + /// Write this dual PDE (128-bit) to VRAM. + fn write(&self, pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { + pramin + .window_at::(addr)? + .view() + .write_val(self.big_raw_u64()); + pramin + .window_at::(addr + 8)? + .view() + .write_val(self.small_raw_u64()); + Ok(()) + } +} + /// Memory aperture for Page Table Entries (`PTE`s). /// /// Determines which memory region the `PTE` points to. From 98b7273f5de149b2c6ff0d98bc2b3fa55b749b74 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:45 +0900 Subject: [PATCH 048/110] gpu: nova-core: mm: Add MMU v2 page table types Add page table entry and directory structures for MMU version 2 used by Hopper and later GPUs. The `Pte`, `Pde`, and `DualPde` types each implement the `PteOps`, `PdeOps`, and `DualPdeOps` traits introduced earlier in the series, providing the version-agnostic API used by the forthcoming page-table walker and mapper. Signed-off-by: Joel Fernandes Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-7-8dd5d4225d2e@nvidia.com [acourbot: use vertical import style.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm/pagetable.rs | 2 + drivers/gpu/nova-core/mm/pagetable/ver2.rs | 275 +++++++++++++++++++++ 2 files changed, 277 insertions(+) create mode 100644 drivers/gpu/nova-core/mm/pagetable/ver2.rs diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs index 4e63f79b95e5..6f3f39ebde13 100644 --- a/drivers/gpu/nova-core/mm/pagetable.rs +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -8,6 +8,8 @@ #![expect(dead_code)] +pub(super) mod ver2; + use kernel::{ io::Io, num::Bounded, diff --git a/drivers/gpu/nova-core/mm/pagetable/ver2.rs b/drivers/gpu/nova-core/mm/pagetable/ver2.rs new file mode 100644 index 000000000000..d7169a0fcff9 --- /dev/null +++ b/drivers/gpu/nova-core/mm/pagetable/ver2.rs @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! MMU v2 page table types for Turing, Ampere and Ada GPUs. +//! +//! This module defines MMU version 2 specific types (Turing, Ampere and Ada GPUs). +//! +//! Bit field layouts derived from the NVIDIA OpenRM documentation: +//! `open-gpu-kernel-modules/src/common/inc/swref/published/turing/tu102/dev_mmu.h` + +#![allow(dead_code)] + +use kernel::{ + bitfield, + num::Bounded, // +}; + +use pin_init::Zeroable; + +use super::{ + AperturePde, + AperturePte, + DualPdeOps, + PageTableLevel, + PdeOps, + PteOps, + VaLevelIndex, // +}; + +use crate::mm::{ + Pfn, + VirtualAddress, + VramAddress, // +}; + +// Bounded to version 2 Pfn bitfield conversions: +// 25 bits for video memory frame numbers (bits 32:8). +impl_pfn_bounded!(25); +// 46 bits for system memory frame numbers (bits 53:8). +impl_pfn_bounded!(46); + +bitfield! { + /// MMU v2 49-bit virtual address layout. + pub(super) struct VirtualAddressV2(u64) { + /// Page offset [11:0]. + 11:0 offset; + /// PT index [20:12]. + 20:12 pt_idx; + /// PDE0 index [28:21]. + 28:21 pde0_idx; + /// PDE1 index [37:29]. + 37:29 pde1_idx; + /// PDE2 index [46:38]. + 46:38 pde2_idx; + /// PDE3 index [48:47]. + 48:47 pde3_idx; + } +} + +impl VirtualAddressV2 { + /// Create a [`VirtualAddressV2`] from a [`VirtualAddress`]. + pub(super) fn new(va: VirtualAddress) -> Self { + Self::from_raw(va.into_raw()) + } +} + +impl VaLevelIndex for VirtualAddressV2 { + fn level_index(&self, level: u64) -> u64 { + match level { + 0 => *self.pde3_idx(), + 1 => *self.pde2_idx(), + 2 => *self.pde1_idx(), + 3 => *self.pde0_idx(), + 4 => *self.pt_idx(), + _ => 0, + } + } +} + +/// `PDE` levels for MMU v2 (5-level hierarchy: `PDB` -> `L1` -> `L2` -> `L3` -> `L4`). +pub(super) const PDE_LEVELS: &[PageTableLevel] = &[ + PageTableLevel::Pdb, + PageTableLevel::L1, + PageTableLevel::L2, + PageTableLevel::L3, +]; + +/// `PTE` level for MMU v2. +pub(super) const PTE_LEVEL: PageTableLevel = PageTableLevel::L4; + +/// Dual `PDE` level for MMU v2 (128-bit entries). +pub(super) const DUAL_PDE_LEVEL: PageTableLevel = PageTableLevel::L3; + +// Page Table Entry (PTE) for MMU v2 - 64-bit entry at level 4. +bitfield! { + /// Page Table Entry for MMU v2. + pub(in crate::mm) struct Pte(u64) { + /// Entry is valid. + 0:0 valid; + /// Memory aperture type. + 2:1 aperture => AperturePte; + /// Volatile (bypass L2 cache). + 3:3 volatile; + /// Encryption enabled (Confidential Computing). + 4:4 encrypted; + /// Privileged access only. + 5:5 privilege; + /// Write protection. + 6:6 read_only; + /// Atomic operations disabled. + 7:7 atomic_disable; + /// Frame number for system memory. + 53:8 frame_number_sys => Pfn; + /// Frame number for video memory. + 32:8 frame_number_vid => Pfn; + /// Peer GPU ID for peer memory (0-7). + 35:33 peer_id; + /// Compression tag line bits. + 53:36 comptagline; + /// Surface kind/format. + 63:56 kind; + } +} + +impl PteOps for Pte { + fn from_raw(val: u64) -> Self { + Self::from_raw(val) + } + + fn invalid() -> Self { + Self::zeroed() + } + + fn new(aperture: AperturePte, pfn: Pfn, writable: bool) -> Self { + let base = Self::zeroed() + .with_valid(true) + .with_aperture(aperture) + .with_read_only(!writable); + match aperture { + AperturePte::VideoMemory => base.with_frame_number_vid(pfn), + // Sysmem PTEs use VOL=1 to bypass L2 for cache coherency. + AperturePte::SystemCoherent => base.with_frame_number_sys(pfn).with_volatile(true), + AperturePte::PeerMemory | AperturePte::SystemNonCoherent => { + kernel::pr_warn!("MMU v2 PTE aperture {:?} not supported\n", aperture); + Self::invalid() + } + } + } + + fn is_valid(&self) -> bool { + self.valid().into_bool() + } + + fn frame_number(&self) -> Pfn { + match self.aperture() { + AperturePte::VideoMemory => self.frame_number_vid(), + _ => self.frame_number_sys(), + } + } +} + +// Page Directory Entry (PDE) for MMU v2 - 64-bit entry at levels 0-2. +bitfield! { + /// Page Directory Entry for MMU v2. + pub(in crate::mm) struct Pde(u64) { + /// Valid bit (inverted logic). + 0:0 valid_inverted; + /// Memory aperture type. + 2:1 aperture => AperturePde; + /// Volatile (bypass L2 cache). + 3:3 volatile; + /// Disable Address Translation Services. + 5:5 no_ats; + /// Table frame number for system memory. + 53:8 table_frame_sys => Pfn; + /// Table frame number for video memory. + 32:8 table_frame_vid => Pfn; + /// Peer GPU ID (0-7). + 35:33 peer_id; + } +} + +impl PdeOps for Pde { + fn from_raw(val: u64) -> Self { + Self::from_raw(val) + } + + fn new(aperture: AperturePde, table_pfn: Pfn) -> Self { + let base = Self::zeroed() + .with_valid_inverted(false) // 0 = valid + .with_aperture(aperture); + match aperture { + AperturePde::VideoMemory => base.with_table_frame_vid(table_pfn), + // Sysmem PTEs use VOL=1 to bypass L2 for cache coherency. + AperturePde::SystemCoherent => base.with_table_frame_sys(table_pfn).with_volatile(true), + AperturePde::Invalid | AperturePde::SystemNonCoherent => { + kernel::pr_warn!("MMU v2 PDE aperture {:?} not supported\n", aperture); + Self::invalid() + } + } + } + + fn invalid() -> Self { + Self::zeroed() + .with_valid_inverted(true) + .with_aperture(AperturePde::Invalid) + } + + fn is_valid(&self) -> bool { + !self.valid_inverted().into_bool() && self.aperture() != AperturePde::Invalid + } + + fn aperture(&self) -> AperturePde { + Pde::aperture(*self) + } + + fn table_vram_address(&self) -> VramAddress { + debug_assert!( + Pde::aperture(*self) == AperturePde::VideoMemory, + "table_vram_address called on non-VRAM PDE (aperture: {:?})", + Pde::aperture(*self) + ); + VramAddress::from(self.table_frame_vid()) + } +} + +/// Dual `PDE` at Level 3 - 128-bit entry of Large/Small Page Table pointers. +/// +/// The dual `PDE` supports both large (64KB) and small (4KB) page tables. +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub(in crate::mm) struct DualPde { + /// Large/Big Page Table pointer (lower 64 bits). + pub(super) big: Pde, + /// Small Page Table pointer (upper 64 bits). + pub(super) small: Pde, +} + +impl DualPde { + /// Check if the big page table pointer is valid. + fn has_big(&self) -> bool { + PdeOps::is_valid(&self.big) + } +} + +impl DualPdeOps for DualPde { + fn from_raw(big: u64, small: u64) -> Self { + Self { + big: PdeOps::from_raw(big), + small: PdeOps::from_raw(small), + } + } + + fn new_small(table_pfn: Pfn) -> Self { + Self { + big: PdeOps::from_raw(0), + small: PdeOps::new(AperturePde::VideoMemory, table_pfn), + } + } + + fn has_small(&self) -> bool { + PdeOps::is_valid(&self.small) + } + + fn small_vram_address(&self) -> VramAddress { + PdeOps::table_vram_address(&self.small) + } + + fn big_raw_u64(&self) -> u64 { + self.big.into_raw() + } + + fn small_raw_u64(&self) -> u64 { + self.small.into_raw() + } +} From 103fb3a4b2b85ebf30f11a433df17df708f85689 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:46 +0900 Subject: [PATCH 049/110] gpu: nova-core: mm: Add MMU v3 page table types Add page table entry and directory structures for MMU version 3 used by Hopper and later GPUs. The `Pte`, `Pde`, and `DualPde` types each implement the `PteOps`, `PdeOps`, and `DualPdeOps` traits introduced earlier in the series, providing the version-agnostic API used by the forthcoming page-table walker and mapper. Signed-off-by: Joel Fernandes [ecourtney: update for the VramAddress raw API, apply rustfmt] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-8-8dd5d4225d2e@nvidia.com [acourbot: use vertical import style.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm/pagetable.rs | 1 + drivers/gpu/nova-core/mm/pagetable/ver3.rs | 421 +++++++++++++++++++++ 2 files changed, 422 insertions(+) create mode 100644 drivers/gpu/nova-core/mm/pagetable/ver3.rs diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs index 6f3f39ebde13..f8ec62decae3 100644 --- a/drivers/gpu/nova-core/mm/pagetable.rs +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -9,6 +9,7 @@ #![expect(dead_code)] pub(super) mod ver2; +pub(super) mod ver3; use kernel::{ io::Io, diff --git a/drivers/gpu/nova-core/mm/pagetable/ver3.rs b/drivers/gpu/nova-core/mm/pagetable/ver3.rs new file mode 100644 index 000000000000..47ed3339026b --- /dev/null +++ b/drivers/gpu/nova-core/mm/pagetable/ver3.rs @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! MMU v3 page table types for Hopper and later GPUs. +//! +//! This module defines MMU version 3 specific types (Hopper and later GPUs). +//! +//! Key differences from MMU v2: +//! - Unified 40-bit address field for all apertures (v2 had separate sys/vid fields). +//! - PCF (Page Classification Field) replaces separate privilege/RO/atomic/cache bits. +//! - KIND field is 4 bits (not 8). +//! - IS_PTE bit in PDE to support large pages directly. +//! - No COMPTAGLINE field (compression handled differently in v3). +//! - No separate ENCRYPTED bit. +//! +//! Bit field layouts derived from the NVIDIA OpenRM documentation: +//! `open-gpu-kernel-modules/src/common/inc/swref/published/hopper/gh100/dev_mmu.h` + +#![allow(dead_code)] + +use kernel::{ + bitfield, + num::Bounded, + prelude::*, // +}; + +use pin_init::Zeroable; + +use super::{ + AperturePde, + AperturePte, + DualPdeOps, + PageTableLevel, + PdeOps, + PteOps, + VaLevelIndex, // +}; + +use crate::mm::{ + Pfn, + VirtualAddress, + VramAddress, // +}; + +// Bounded to version 3 Pfn conversion. +impl_pfn_bounded!(40); + +bitfield! { + /// MMU v3 57-bit virtual address layout. + pub(super) struct VirtualAddressV3(u64) { + /// Page offset [11:0]. + 11:0 offset; + /// PT index [20:12]. + 20:12 pt_idx; + /// PDE0 index [28:21]. + 28:21 pde0_idx; + /// PDE1 index [37:29]. + 37:29 pde1_idx; + /// PDE2 index [46:38]. + 46:38 pde2_idx; + /// PDE3 index [55:47]. + 55:47 pde3_idx; + /// PDE4 index [56]. + 56:56 pde4_idx; + } +} + +impl VirtualAddressV3 { + /// Create a [`VirtualAddressV3`] from a [`VirtualAddress`]. + pub(super) fn new(va: VirtualAddress) -> Self { + Self::from_raw(va.into_raw()) + } +} + +impl VaLevelIndex for VirtualAddressV3 { + fn level_index(&self, level: u64) -> u64 { + match level { + 0 => *self.pde4_idx(), + 1 => *self.pde3_idx(), + 2 => *self.pde2_idx(), + 3 => *self.pde1_idx(), + 4 => *self.pde0_idx(), + 5 => *self.pt_idx(), + _ => 0, + } + } +} + +/// PDE levels for MMU v3 (6-level hierarchy). +pub(super) const PDE_LEVELS: &[PageTableLevel] = &[ + PageTableLevel::Pdb, + PageTableLevel::L1, + PageTableLevel::L2, + PageTableLevel::L3, + PageTableLevel::L4, +]; + +/// PTE level for MMU v3. +pub(super) const PTE_LEVEL: PageTableLevel = PageTableLevel::L5; + +/// Dual PDE level for MMU v3 (128-bit entries). +pub(super) const DUAL_PDE_LEVEL: PageTableLevel = PageTableLevel::L4; + +bitfield! { + /// Page Classification Field for PTEs (5 bits) in MMU v3. + pub(in crate::mm) struct PtePcf(u8) { + /// Bypass L2 cache (0=cached, 1=bypass). + 0:0 uncached; + /// Access counting disabled (0=enabled, 1=disabled). + 1:1 acd; + /// Read-only access (0=read-write, 1=read-only). + 2:2 read_only; + /// Atomics disabled (0=enabled, 1=disabled). + 3:3 no_atomic; + /// Privileged access only (0=regular, 1=privileged). + 4:4 privileged; + } +} + +impl PtePcf { + /// Create PCF for read-write mapping (cached, no atomics, regular mode). + fn rw() -> Self { + Self::zeroed().with_no_atomic(true) + } + + /// Create PCF for read-only mapping (cached, no atomics, regular mode). + fn ro() -> Self { + Self::zeroed().with_read_only(true).with_no_atomic(true) + } + + /// Get the raw `u8` value. + fn raw_u8(&self) -> u8 { + self.into_raw() + } +} + +impl From> for PtePcf { + fn from(val: Bounded) -> Self { + Self::from_raw(u8::from(val)) + } +} + +impl From for Bounded { + fn from(pcf: PtePcf) -> Self { + Bounded::from_expr(u64::from(pcf.into_raw()) & 0x1F) + } +} + +bitfield! { + /// Page Classification Field for PDEs (3 bits) in MMU v3. + /// + /// Controls Address Translation Services (ATS) and caching. + pub(in crate::mm) struct PdePcf(u8) { + /// Bypass L2 cache (0=cached, 1=bypass). + 0:0 uncached; + /// ATS disabled (0=enabled, 1=disabled). + 1:1 no_ats; + } +} + +impl PdePcf { + /// Create PCF for cached mapping with ATS enabled (default). + fn cached() -> Self { + Self::zeroed() + } + + /// Get the raw `u8` value. + fn raw_u8(&self) -> u8 { + self.into_raw() + } +} + +impl From> for PdePcf { + fn from(val: Bounded) -> Self { + Self::from_raw(u8::from(val)) + } +} + +impl From for Bounded { + fn from(pcf: PdePcf) -> Self { + Bounded::from_expr(u64::from(pcf.into_raw()) & 0x7) + } +} + +bitfield! { + /// Page Table Entry for MMU v3. + pub(in crate::mm) struct Pte(u64) { + /// Entry is valid. + 0:0 valid; + /// Memory aperture type. + 2:1 aperture => AperturePte; + /// Page Classification Field. + 7:3 pcf => PtePcf; + /// Surface kind (4 bits, 0x0=pitch, 0xF=invalid). + 11:8 kind; + /// Physical frame number (for all apertures). + 51:12 frame_number => Pfn; + /// Peer GPU ID for peer memory (0-7). + 63:61 peer_id; + } +} + +impl PteOps for Pte { + fn from_raw(val: u64) -> Self { + Self::from_raw(val) + } + + fn invalid() -> Self { + Self::zeroed() + } + + fn new(aperture: AperturePte, pfn: Pfn, writable: bool) -> Self { + let pcf = match (aperture, writable) { + (AperturePte::VideoMemory, true) => PtePcf::rw(), + (AperturePte::VideoMemory, false) => PtePcf::ro(), + // Sysmem PTEs use uncached+no_atomic PCF for cache coherency. + (AperturePte::SystemCoherent, true) => { + PtePcf::zeroed().with_uncached(true).with_no_atomic(true) + } + (AperturePte::SystemCoherent, false) => PtePcf::zeroed() + .with_uncached(true) + .with_no_atomic(true) + .with_read_only(true), + (AperturePte::PeerMemory | AperturePte::SystemNonCoherent, _) => { + kernel::pr_warn!("MMU v3 PTE aperture {:?} not supported\n", aperture); + return Self::invalid(); + } + }; + Self::zeroed() + .with_valid(true) + .with_aperture(aperture) + .with_pcf(pcf) + .with_frame_number(pfn) + } + + fn is_valid(&self) -> bool { + self.valid().into_bool() + } + + fn frame_number(&self) -> Pfn { + Pte::frame_number(*self) + } +} + +bitfield! { + /// Page Directory Entry for MMU v3 (Hopper+). + /// + /// ## Note + /// + /// v3 uses a unified 40-bit address field (v2 had separate sys/vid address fields). + pub(in crate::mm) struct Pde(u64) { + /// Entry is a PTE (0=PDE, 1=large page PTE). + 0:0 is_pte; + /// Memory aperture type. + 2:1 aperture => AperturePde; + /// Page Classification Field (3 bits for PDE). + 5:3 pcf => PdePcf; + /// Table frame number (40-bit unified address). + 51:12 table_frame => Pfn; + } +} + +impl PdeOps for Pde { + fn from_raw(val: u64) -> Self { + Self::from_raw(val) + } + + fn new(aperture: AperturePde, table_pfn: Pfn) -> Self { + match aperture { + AperturePde::VideoMemory => Self::zeroed() + .with_is_pte(false) + .with_aperture(aperture) + .with_table_frame(table_pfn), + AperturePde::Invalid | AperturePde::SystemCoherent | AperturePde::SystemNonCoherent => { + kernel::pr_warn!("MMU v3 PDE aperture {:?} not supported\n", aperture); + Self::invalid() + } + } + } + + fn invalid() -> Self { + Self::zeroed().with_aperture(AperturePde::Invalid) + } + + fn is_valid(&self) -> bool { + Pde::aperture(*self) != AperturePde::Invalid + } + + fn aperture(&self) -> AperturePde { + Pde::aperture(*self) + } + + fn table_vram_address(&self) -> VramAddress { + debug_assert!( + Pde::aperture(*self) == AperturePde::VideoMemory, + "table_vram_address called on non-VRAM PDE (aperture: {:?})", + Pde::aperture(*self) + ); + VramAddress::from(self.table_frame()) + } +} + +bitfield! { + /// Big Page Table pointer in Dual PDE (MMU v3). + /// + /// 64-bit lower word of the 128-bit Dual PDE. + pub(super) struct DualPdeBig(u64) { + /// Entry is a PTE (for large pages). + 0:0 is_pte; + /// Memory aperture type. + 2:1 aperture => AperturePde; + /// Page Classification Field. + 5:3 pcf => PdePcf; + /// Table frame (table address 256-byte aligned). + 51:8 table_frame; + } +} + +impl DualPdeBig { + /// Create an invalid big page table pointer. + fn invalid() -> Self { + Self::zeroed().with_aperture(AperturePde::Invalid) + } + + /// Create a valid big PDE pointing to a page table in the given aperture. + fn new(aperture: AperturePde, table_addr: VramAddress) -> Result { + // Big page table addresses must be 256-byte aligned (shift 8). + if table_addr.into_raw() & 0xFF != 0 { + return Err(EINVAL); + } + let table_frame = Bounded::from_expr(table_addr.into_raw() >> 8); + match aperture { + AperturePde::VideoMemory => Ok(Self::zeroed() + .with_is_pte(false) + .with_aperture(aperture) + .with_table_frame(table_frame)), + AperturePde::Invalid | AperturePde::SystemCoherent | AperturePde::SystemNonCoherent => { + kernel::pr_warn!("MMU v3 DualPdeBig aperture {:?} not supported\n", aperture); + Ok(Self::invalid()) + } + } + } + + /// Check if this big PDE is valid. + fn is_valid(&self) -> bool { + self.aperture() != AperturePde::Invalid + } + + /// Get the VRAM address of the big page table. + fn table_vram_address(&self) -> VramAddress { + debug_assert!( + self.aperture() == AperturePde::VideoMemory, + "table_vram_address called on non-VRAM DualPdeBig (aperture: {:?})", + self.aperture() + ); + VramAddress::from_raw(*self.table_frame() << 8) + } +} + +/// Dual PDE at Level 4 for MMU v3 - 128-bit entry. +/// +/// Contains both big (64KB) and small (4KB) page table pointers: +/// - Lower 64 bits: Big Page Table pointer. +/// - Upper 64 bits: Small Page Table pointer. +/// +/// ## Note +/// +/// The big and small page table pointers have different address layouts: +/// - Big address = field value << 8 (256-byte alignment). +/// - Small address = field value << 12 (4KB alignment). +/// +/// This is why `DualPdeBig` is a separate type from `Pde`. +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub(in crate::mm) struct DualPde { + /// Big Page Table pointer. + pub(super) big: DualPdeBig, + /// Small Page Table pointer. + pub(super) small: Pde, +} + +// SAFETY: Both `DualPdeBig` and `Pde` fields are `Zeroable` (bitfield types are Zeroable). +unsafe impl Zeroable for DualPde {} + +impl DualPde { + /// Check if the big page table pointer is valid. + fn has_big(&self) -> bool { + self.big.is_valid() + } +} + +impl DualPdeOps for DualPde { + fn from_raw(big: u64, small: u64) -> Self { + Self { + big: DualPdeBig::from_raw(big), + small: PdeOps::from_raw(small), + } + } + + fn new_small(table_pfn: Pfn) -> Self { + Self { + big: DualPdeBig::invalid(), + small: PdeOps::new(AperturePde::VideoMemory, table_pfn), + } + } + + fn has_small(&self) -> bool { + PdeOps::is_valid(&self.small) + } + + fn small_vram_address(&self) -> VramAddress { + PdeOps::table_vram_address(&self.small) + } + + fn big_raw_u64(&self) -> u64 { + self.big.into_raw() + } + + fn small_raw_u64(&self) -> u64 { + self.small.into_raw() + } +} From a9eabe320b60e5af72ff10551be1b2d67ca614af Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:47 +0900 Subject: [PATCH 050/110] gpu: nova-core: mm: pagetable: Add MmuConfig trait Introduce `MmuConfig`, the trait that ties the entry-operation traits (`PteOps`, `PdeOps`, `DualPdeOps`) together with the version-specific constants and helpers. `MmuV2` and `MmuV3` are zero-sized marker structs that implement `MmuConfig` for Turing/Ampere/Ada and Hopper/Blackwell respectively. Dispatch is fully resolved at compile time through these markers, so version-specific code is selected without runtime overhead and without wrapper enums. This enables version-agnostic page-table operations while keeping version-specific implementation details encapsulated in the `ver2` and `ver3` modules. Signed-off-by: Joel Fernandes Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-9-8dd5d4225d2e@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm/pagetable.rs | 109 ++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs index f8ec62decae3..89f828194c59 100644 --- a/drivers/gpu/nova-core/mm/pagetable.rs +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -22,6 +22,7 @@ mm::{ pramin, Pfn, + VirtualAddress, VramAddress, // }, }; @@ -215,6 +216,114 @@ fn write(&self, pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { } } +/// MMU configuration trait -- encodes version-specific constants and types. +pub(super) trait MmuConfig: 'static { + /// Page Table Entry type. + type Pte: PteOps; + /// Page Directory Entry type. + type Pde: PdeOps; + /// Dual Page Directory Entry type (128-bit). + type DualPde: DualPdeOps; + + /// PDE levels (excluding PTE level) for page table walking. + const PDE_LEVELS: &'static [PageTableLevel]; + /// PTE level for this MMU version. + const PTE_LEVEL: PageTableLevel; + /// Dual PDE level (128-bit entries) for this MMU version. + const DUAL_PDE_LEVEL: PageTableLevel; + + /// Get the number of entries per page table page for a given level. + fn entries_per_page(level: PageTableLevel) -> usize; + + /// Extract the page table index at `level` from `va`. + fn level_index(va: VirtualAddress, level: u64) -> u64; + + /// Get the entry size in bytes for a given level. + fn entry_size(level: PageTableLevel) -> usize { + if level == Self::DUAL_PDE_LEVEL { + 16 // 128-bit dual PDE + } else { + 8 // 64-bit PDE/PTE + } + } + + /// Compute upper bound on page table pages needed for `num_virt_pages`. + /// + /// Walks from PTE level up through PDE levels, accumulating the tree. + fn pt_pages_upper_bound(num_virt_pages: usize) -> usize { + let mut total = 0; + + // PTE pages at the leaf level. + let pte_epp = Self::entries_per_page(Self::PTE_LEVEL); + let mut pages_at_level = num_virt_pages.div_ceil(pte_epp); + total += pages_at_level; + + // Walk PDE levels bottom-up (reverse of PDE_LEVELS). + for &level in Self::PDE_LEVELS.iter().rev() { + let epp = Self::entries_per_page(level); + + // How many pages at this level do we need to point to + // the previous pages_at_level? + pages_at_level = pages_at_level.div_ceil(epp); + total += pages_at_level; + } + + total + } +} + +/// Marker struct for MMU v2 (Turing/Ampere/Ada). +pub(super) struct MmuV2; + +impl MmuConfig for MmuV2 { + type Pte = ver2::Pte; + type Pde = ver2::Pde; + type DualPde = ver2::DualPde; + + const PDE_LEVELS: &'static [PageTableLevel] = ver2::PDE_LEVELS; + const PTE_LEVEL: PageTableLevel = ver2::PTE_LEVEL; + const DUAL_PDE_LEVEL: PageTableLevel = ver2::DUAL_PDE_LEVEL; + + fn entries_per_page(level: PageTableLevel) -> usize { + // TODO: Calculate these values from the bitfield dynamically + // instead of hardcoding them. + match level { + PageTableLevel::Pdb => 4, // PD3 root: bits [48:47] = 2 bits + PageTableLevel::L3 => 256, // PD0 dual: bits [28:21] = 8 bits + _ => 512, // PD2, PD1, PT: 9 bits each + } + } + + fn level_index(va: VirtualAddress, level: u64) -> u64 { + ver2::VirtualAddressV2::new(va).level_index(level) + } +} + +/// Marker struct for MMU v3 (Hopper and later). +pub(super) struct MmuV3; + +impl MmuConfig for MmuV3 { + type Pte = ver3::Pte; + type Pde = ver3::Pde; + type DualPde = ver3::DualPde; + + const PDE_LEVELS: &'static [PageTableLevel] = ver3::PDE_LEVELS; + const PTE_LEVEL: PageTableLevel = ver3::PTE_LEVEL; + const DUAL_PDE_LEVEL: PageTableLevel = ver3::DUAL_PDE_LEVEL; + + fn entries_per_page(level: PageTableLevel) -> usize { + match level { + PageTableLevel::Pdb => 2, // PDE4 root: bit [56] = 1 bit, 2 entries + PageTableLevel::L4 => 256, // PDE0 dual: bits [28:21] = 8 bits + _ => 512, // PDE3, PDE2, PDE1, PT: 9 bits each + } + } + + fn level_index(va: VirtualAddress, level: u64) -> u64 { + ver3::VirtualAddressV3::new(va).level_index(level) + } +} + /// Memory aperture for Page Table Entries (`PTE`s). /// /// Determines which memory region the `PTE` points to. From 1123774f3610c45a7acf055b8c9841883ed4bf75 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:48 +0900 Subject: [PATCH 051/110] gpu: nova-core: mm: Add page table walker for MMU v2/v3 Add the page table walker implementation that traverses the page table hierarchy for both MMU v2 (5-level) and MMU v3 (6-level) to resolve virtual addresses to physical addresses or find PTE locations. Currently only v2 has been tested (nova-core currently boots pre-hopper) with some initial preparatory work done for v3. Signed-off-by: Joel Fernandes [ecourtney: take mutable GpuMm and Pramin, not a bound device and window] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-10-8dd5d4225d2e@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm/pagetable.rs | 1 + drivers/gpu/nova-core/mm/pagetable/walk.rs | 244 +++++++++++++++++++++ 2 files changed, 245 insertions(+) create mode 100644 drivers/gpu/nova-core/mm/pagetable/walk.rs diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs index 89f828194c59..0e9ffad34c90 100644 --- a/drivers/gpu/nova-core/mm/pagetable.rs +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -10,6 +10,7 @@ pub(super) mod ver2; pub(super) mod ver3; +pub(super) mod walk; use kernel::{ io::Io, diff --git a/drivers/gpu/nova-core/mm/pagetable/walk.rs b/drivers/gpu/nova-core/mm/pagetable/walk.rs new file mode 100644 index 000000000000..76c1729971f5 --- /dev/null +++ b/drivers/gpu/nova-core/mm/pagetable/walk.rs @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Page table walker implementation for NVIDIA GPUs. +//! +//! This module provides page table walking functionality for MMU v2 and v3. +//! The walker traverses the page table hierarchy to resolve virtual addresses +//! to physical addresses or to find PTE locations. +//! +//! # Page Table Hierarchy +//! +//! ## MMU v2 (Turing/Ampere/Ada) - 5 levels +//! +//! ```text +//! +-------+ +-------+ +-------+ +---------+ +-------+ +//! | PDB |---->| L1 |---->| L2 |---->| L3 Dual |---->| L4 | +//! | (L0) | | | | | | PDE | | (PTE) | +//! +-------+ +-------+ +-------+ +---------+ +-------+ +//! 64-bit 64-bit 64-bit 128-bit 64-bit +//! PDE PDE PDE (big+small) PTE +//! ``` +//! +//! ## MMU v3 (Hopper+) - 6 levels +//! +//! ```text +//! +-------+ +-------+ +-------+ +-------+ +---------+ +-------+ +//! | PDB |---->| L1 |---->| L2 |---->| L3 |---->| L4 Dual |---->| L5 | +//! | (L0) | | | | | | | | PDE | | (PTE) | +//! +-------+ +-------+ +-------+ +-------+ +---------+ +-------+ +//! 64-bit 64-bit 64-bit 64-bit 128-bit 64-bit +//! PDE PDE PDE PDE (big+small) PTE +//! ``` +//! +//! # Result of a page table walk +//! +//! The walker returns a [`WalkResult`] indicating the outcome. + +use core::marker::PhantomData; + +use kernel::prelude::*; + +use super::{ + DualPdeOps, + MmuConfig, + MmuV2, + MmuV3, + MmuVersion, + PageTableLevel, + PdeOps, + PteOps, // +}; +use crate::{ + mm::{ + pramin, + GpuMm, + Pfn, + Vfn, + VirtualAddress, + VramAddress, // + }, + num::{ + IntoSafeCast, // + }, +}; + +/// Result of walking to a PTE. +#[derive(Debug, Clone, Copy)] +pub(in crate::mm) enum WalkResult { + /// Intermediate page tables are missing (only returned in lookup mode). + PageTableMissing, + /// PTE exists but is invalid (page not mapped). + Unmapped { pte_addr: VramAddress }, + /// PTE exists and is valid (page is mapped). + Mapped { pte_addr: VramAddress, pfn: Pfn }, +} + +/// Result of walking PDE levels only. +/// +/// Returned by [`PtWalkInner::walk_pde_levels()`] to indicate whether all PDE +/// levels resolved or a PDE is missing. +#[derive(Debug, Clone, Copy)] +pub(in crate::mm) enum WalkPdeResult { + /// All PDE levels resolved -- returns PTE page table address. + Complete { + /// VRAM address of the PTE-level page table. + pte_table: VramAddress, + }, + /// A PDE is missing and no prepared page was provided by the closure. + Missing { + /// PDE slot address in the parent page table (where to install). + install_addr: VramAddress, + /// The page table level that is missing. + level: PageTableLevel, + }, +} + +/// Page table walker. +pub(in crate::mm) struct PtWalkInner { + pdb_addr: VramAddress, + _phantom: PhantomData, +} + +impl PtWalkInner { + /// Calculate the VRAM address of an entry within a page table. + fn entry_addr(table: VramAddress, level: PageTableLevel, index: u64) -> VramAddress { + let entry_size: u64 = M::entry_size(level).into_safe_cast(); + table + index * entry_size + } + + /// Create a new page table walker. + pub(super) fn new(pdb_addr: VramAddress) -> Self { + Self { + pdb_addr, + _phantom: PhantomData, + } + } + + /// Walk PDE levels with closure-based resolution for missing PDEs. + /// + /// Traverses all PDE levels for the MMU version. At each level, reads the PDE. + /// If valid, extracts the child table address and continues. If missing, calls + /// `resolve_prepared(install_addr)` to resolve the missing PDE. + pub(super) fn walk_pde_levels( + &self, + pramin: &mut pramin::Pramin<'_>, + vfn: Vfn, + resolve_prepared: impl Fn(VramAddress) -> Option, + ) -> Result { + let va = VirtualAddress::from(vfn); + let mut cur_table = self.pdb_addr; + + for &level in M::PDE_LEVELS { + let idx = M::level_index(va, level.as_index()); + let install_addr = Self::entry_addr(cur_table, level, idx); + + if level == M::DUAL_PDE_LEVEL { + // 128-bit dual PDE with big+small page table pointers. + let dpde = M::DualPde::read(pramin, install_addr)?; + if dpde.has_small() { + cur_table = dpde.small_vram_address(); + continue; + } + } else { + // Regular 64-bit PDE. Use `is_valid_vram()` because + // `table_vram_address()` only reads the VRAM frame-number + // bitfield; system-memory PDEs store the address in a + // different (wider) field and would be silently truncated. + let pde = M::Pde::read(pramin, install_addr)?; + if pde.is_valid_vram() { + cur_table = pde.table_vram_address(); + continue; + } + } + + // PDE missing in HW. Ask caller for resolution. + if let Some(prepared_addr) = resolve_prepared(install_addr) { + cur_table = prepared_addr; + continue; + } + + return Ok(WalkPdeResult::Missing { + install_addr, + level, + }); + } + + Ok(WalkPdeResult::Complete { + pte_table: cur_table, + }) + } + + /// Walk to PTE for lookup only (no allocation). + /// + /// Returns [`WalkResult::PageTableMissing`] if intermediate tables don't exist. + pub(super) fn walk_to_pte_lookup(&self, mm: &mut GpuMm<'_>, vfn: Vfn) -> Result { + self.walk_to_pte_lookup_with_window(mm.pramin_mut(), vfn) + } + + /// Walk to PTE using a caller-provided PRAMIN manager (lookup only). + pub(super) fn walk_to_pte_lookup_with_window( + &self, + pramin: &mut pramin::Pramin<'_>, + vfn: Vfn, + ) -> Result { + match self.walk_pde_levels(pramin, vfn, |_| None)? { + WalkPdeResult::Complete { pte_table } => { + Self::read_pte_at_level(pramin, vfn, pte_table) + } + WalkPdeResult::Missing { .. } => Ok(WalkResult::PageTableMissing), + } + } + + /// Read the PTE at the PTE level given the PTE table address. + fn read_pte_at_level( + pramin: &mut pramin::Pramin<'_>, + vfn: Vfn, + pte_table: VramAddress, + ) -> Result { + let va = VirtualAddress::from(vfn); + let pte_level = M::PTE_LEVEL; + let pte_idx = M::level_index(va, pte_level.as_index()); + let pte_addr = Self::entry_addr(pte_table, pte_level, pte_idx); + let pte = M::Pte::read(pramin, pte_addr)?; + + if pte.is_valid() { + return Ok(WalkResult::Mapped { + pte_addr, + pfn: pte.frame_number(), + }); + } + Ok(WalkResult::Unmapped { pte_addr }) + } +} + +macro_rules! pt_walk_dispatch { + ($self:expr, $method:ident ( $($arg:expr),* $(,)? )) => { + match $self { + PtWalk::V2(inner) => inner.$method($($arg),*), + PtWalk::V3(inner) => inner.$method($($arg),*), + } + }; +} + +/// Page table walker dispatch. +pub(in crate::mm) enum PtWalk { + /// MMU v2 (Turing/Ampere/Ada). + V2(PtWalkInner), + /// MMU v3 (Hopper+). + V3(PtWalkInner), +} + +impl PtWalk { + /// Create a new page table walker for the given MMU version. + pub(in crate::mm) fn new(pdb_addr: VramAddress, version: MmuVersion) -> Self { + match version { + MmuVersion::V2 => Self::V2(PtWalkInner::::new(pdb_addr)), + MmuVersion::V3 => Self::V3(PtWalkInner::::new(pdb_addr)), + } + } + + /// Walk to PTE for lookup. + pub(in crate::mm) fn walk_to_pte(&self, mm: &mut GpuMm<'_>, vfn: Vfn) -> Result { + pt_walk_dispatch!(self, walk_to_pte_lookup(mm, vfn)) + } +} From 3c7561e194d573a3926702d1e051a8c8fbc187ea Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:49 +0900 Subject: [PATCH 052/110] gpu: nova-core: mm: Add Virtual Memory Manager Add the Virtual Memory Manager (VMM) infrastructure for GPU address space management. Each Vmm instance manages a single address space identified by its Page Directory Base (PDB) address, used for Channel, BAR1 and BAR2 mappings. Mapping APIs and virtual address range tracking are added in later commits. Signed-off-by: Joel Fernandes [ecourtney: pass mutable GpuMm to mapping lookup] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-11-8dd5d4225d2e@nvidia.com [acourbot: use vertical import style.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm.rs | 1 + drivers/gpu/nova-core/mm/vmm.rs | 61 +++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 drivers/gpu/nova-core/mm/vmm.rs diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index 377cc9d44702..2cf37254fbb9 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -65,6 +65,7 @@ macro_rules! impl_pfn_bounded { mod pramin; mod regs; pub(super) mod tlb; +pub(super) mod vmm; /// GPU Memory Manager - owns all core MM components. /// diff --git a/drivers/gpu/nova-core/mm/vmm.rs b/drivers/gpu/nova-core/mm/vmm.rs new file mode 100644 index 000000000000..3738262442d1 --- /dev/null +++ b/drivers/gpu/nova-core/mm/vmm.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Virtual Memory Manager for NVIDIA GPU page table management. +//! +//! The [`Vmm`] provides high-level page mapping and unmapping operations for GPU +//! virtual address spaces (Channels, BAR1, BAR2). It wraps the page table walker +//! and handles TLB flushing after modifications. + +use kernel::{ + gpu::buddy::AllocatedBlocks, + prelude::*, // +}; + +use crate::mm::{ + pagetable::{ + walk::{ + PtWalk, + WalkResult, // + }, + MmuVersion, // + }, + GpuMm, + Pfn, + Vfn, + VramAddress, // +}; + +/// Virtual Memory Manager for a GPU address space. +/// +/// Each [`Vmm`] instance manages a single address space identified by its Page +/// Directory Base (`PDB`) address. The [`Vmm`] is used for Channel, BAR1 and +/// BAR2 mappings. +pub(crate) struct Vmm { + /// Page Directory Base address for this address space. + pdb_addr: VramAddress, + /// MMU version used for page table layout. + mmu_version: MmuVersion, + /// Page table allocations required for mappings. + page_table_allocs: KVec>>, +} + +impl Vmm { + /// Create a new [`Vmm`] for the given Page Directory Base address. + pub(crate) fn new(pdb_addr: VramAddress, mmu_version: MmuVersion) -> Result { + Ok(Self { + pdb_addr, + mmu_version, + page_table_allocs: KVec::new(), + }) + } + + /// Read the [`Pfn`] for a mapped [`Vfn`] if one is mapped. + pub(super) fn read_mapping(&self, mm: &mut GpuMm<'_>, vfn: Vfn) -> Result> { + let walker = PtWalk::new(self.pdb_addr, self.mmu_version); + + match walker.walk_to_pte(mm, vfn)? { + WalkResult::Mapped { pfn, .. } => Ok(Some(pfn)), + WalkResult::Unmapped { .. } | WalkResult::PageTableMissing => Ok(None), + } + } +} From 16f573374fd6ff8ff4ab54abceb5d093869de9d2 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:50 +0900 Subject: [PATCH 053/110] gpu: nova-core: mm: Add virtual address range tracking to VMM Add virtual address range tracking to the VMM using a maple tree allocator. This enables contiguous virtual address range allocation for mappings. Signed-off-by: Joel Fernandes Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-12-8dd5d4225d2e@nvidia.com [acourbot: use vertical import style.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm/vmm.rs | 87 ++++++++++++++++++++++++++++----- 1 file changed, 76 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/nova-core/mm/vmm.rs b/drivers/gpu/nova-core/mm/vmm.rs index 3738262442d1..9f4f40109227 100644 --- a/drivers/gpu/nova-core/mm/vmm.rs +++ b/drivers/gpu/nova-core/mm/vmm.rs @@ -8,21 +8,30 @@ use kernel::{ gpu::buddy::AllocatedBlocks, + maple_tree::MapleTreeAlloc, prelude::*, // }; -use crate::mm::{ - pagetable::{ - walk::{ - PtWalk, - WalkResult, // +use core::ops::Range; + +use crate::{ + mm::{ + pagetable::{ + walk::{ + PtWalk, + WalkResult, // + }, + MmuVersion, // }, - MmuVersion, // + GpuMm, + Pfn, + Vfn, + VramAddress, + PAGE_SIZE, // + }, + num::{ + IntoSafeCast, // }, - GpuMm, - Pfn, - Vfn, - VramAddress, // }; /// Virtual Memory Manager for a GPU address space. @@ -37,18 +46,74 @@ pub(crate) struct Vmm { mmu_version: MmuVersion, /// Page table allocations required for mappings. page_table_allocs: KVec>>, + /// Maple tree allocator for virtual address range tracking. + virt_alloc: Pin>>, + /// Total number of pages in the virtual address space. + va_pages: usize, } impl Vmm { /// Create a new [`Vmm`] for the given Page Directory Base address. - pub(crate) fn new(pdb_addr: VramAddress, mmu_version: MmuVersion) -> Result { + /// + /// The [`Vmm`] will manage a virtual address space of `va_size` bytes. + pub(crate) fn new( + pdb_addr: VramAddress, + mmu_version: MmuVersion, + va_size: u64, + ) -> Result { + let page_size: u64 = PAGE_SIZE.into_safe_cast(); + let va_pages: usize = (va_size / page_size).into_safe_cast(); + let virt_alloc = KBox::pin_init(MapleTreeAlloc::<()>::new(), GFP_KERNEL)?; + Ok(Self { pdb_addr, mmu_version, page_table_allocs: KVec::new(), + virt_alloc, + va_pages, }) } + /// Allocate a contiguous virtual frame number range. + /// + /// # Arguments + /// + /// - `num_pages`: Number of pages to allocate. + /// - `va_range`: `None` = allocate anywhere, `Some(range)` = constrain allocation to the given + /// range. + fn alloc_vfn_range(&self, num_pages: usize, va_range: Option>) -> Result { + let page_size: u64 = PAGE_SIZE.into_safe_cast(); + + let start_vfn = match va_range { + Some(r) => { + let num_pages_u64: u64 = num_pages.into_safe_cast(); + let size = num_pages_u64.checked_mul(page_size).ok_or(EOVERFLOW)?; + let range_size = r.end.checked_sub(r.start).ok_or(EOVERFLOW)?; + if range_size != size { + return Err(EINVAL); + } + let start_vfn: usize = (r.start / page_size).into_safe_cast(); + let end_vfn: usize = (r.end / page_size).into_safe_cast(); + self.virt_alloc + .insert_range(start_vfn..end_vfn, (), GFP_KERNEL)?; + start_vfn + } + None => self + .virt_alloc + .alloc_range(num_pages, (), ..self.va_pages, GFP_KERNEL)?, + }; + + Ok(Vfn::new(start_vfn.into_safe_cast())) + } + + /// Free a virtual frame number range back to the maple tree. + fn free_vfn(&self, vfn: Vfn) { + let vfn_index: usize = vfn.raw().into_safe_cast(); + if self.virt_alloc.erase(vfn_index).is_none() { + kernel::pr_warn!("free_vfn: VFN {} not found in maple tree\n", vfn_index); + } + } + /// Read the [`Pfn`] for a mapped [`Vfn`] if one is mapped. pub(super) fn read_mapping(&self, mm: &mut GpuMm<'_>, vfn: Vfn) -> Result> { let walker = PtWalk::new(self.pdb_addr, self.mmu_version); From 4e4182282d4e8d81bfe62ed8ff634526fdd5e238 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:51 +0900 Subject: [PATCH 054/110] gpu: nova-core: mm: Add multi-page mapping API to VMM Add the page table mapping and unmapping API to the Virtual Memory Manager, implementing a two-phase prepare/execute model suitable for use both inside and outside the DMA fence signalling critical path. Signed-off-by: Joel Fernandes [ecourtney: pass mutable GpuMm, replace window guards with scoped borrows] [ecourtney: use current raw address helpers, drop stale expects] [ecourtney: zero page table pages through one PRAMIN view] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-13-8dd5d4225d2e@nvidia.com [acourbot: use vertical import style.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm/pagetable.rs | 1 + drivers/gpu/nova-core/mm/pagetable/map.rs | 345 ++++++++++++++++++++++ drivers/gpu/nova-core/mm/vmm.rs | 255 ++++++++++++++-- 3 files changed, 583 insertions(+), 18 deletions(-) create mode 100644 drivers/gpu/nova-core/mm/pagetable/map.rs diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs index 0e9ffad34c90..63111ff95780 100644 --- a/drivers/gpu/nova-core/mm/pagetable.rs +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -8,6 +8,7 @@ #![expect(dead_code)] +pub(super) mod map; pub(super) mod ver2; pub(super) mod ver3; pub(super) mod walk; diff --git a/drivers/gpu/nova-core/mm/pagetable/map.rs b/drivers/gpu/nova-core/mm/pagetable/map.rs new file mode 100644 index 000000000000..77431c509a89 --- /dev/null +++ b/drivers/gpu/nova-core/mm/pagetable/map.rs @@ -0,0 +1,345 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Page table mapping operations for NVIDIA GPUs. + +use core::marker::PhantomData; + +use kernel::{ + gpu::buddy::{ + AllocatedBlocks, + GpuBuddyAllocFlags, + GpuBuddyAllocMode, // + }, + io::io_write, + prelude::*, + ptr::Alignment, + rbtree::{ + RBTree, + RBTreeNode, // + }, + sizes::SZ_4K, // +}; + +use super::{ + walk::{ + PtWalkInner, + WalkPdeResult, + WalkResult, // + }, + AperturePde, + AperturePte, + DualPdeOps, + MmuConfig, + MmuV2, + MmuV3, + MmuVersion, + PageTableLevel, + PdeOps, + PteOps, // +}; +use crate::{ + mm::{ + GpuMm, + Pfn, + Vfn, + VramAddress, + PAGE_SIZE, // + }, + num::{ + IntoSafeCast, // + }, +}; + +/// A pre-allocated and zeroed page table page. +/// +/// Created during the mapping prepare phase and consumed during the execute phase. +/// Stored in an [`RBTree`] keyed by the PDE slot address (`install_addr`). +pub(in crate::mm) struct PreparedPtPage { + /// The allocated and zeroed page table page. + pub(in crate::mm) alloc: Pin>, + /// Page table level -- needed to determine if this PT page is for a dual PDE. + pub(in crate::mm) level: PageTableLevel, +} + +/// Page table mapper. +pub(in crate::mm) struct PtMapInner { + walker: PtWalkInner, + pdb_addr: VramAddress, + _phantom: PhantomData, +} + +impl PtMapInner { + /// Create a new [`PtMapInner`]. + pub(super) fn new(pdb_addr: VramAddress) -> Self { + Self { + walker: PtWalkInner::::new(pdb_addr), + pdb_addr, + _phantom: PhantomData, + } + } + + /// Allocate and zero a physical page table page. + fn alloc_and_zero_page(mm: &mut GpuMm<'_>, level: PageTableLevel) -> Result { + let blocks = KBox::pin_init( + mm.buddy().alloc_blocks( + GpuBuddyAllocMode::Simple, + SZ_4K.into_safe_cast(), + Alignment::new::(), + GpuBuddyAllocFlags::default(), + ), + GFP_KERNEL, + )?; + + let page_vram = VramAddress::from_raw(blocks.iter().next().ok_or(ENOMEM)?.offset()); + + // Zero via PRAMIN. + let window = mm + .pramin_mut() + .window_at::<[u64; PAGE_SIZE / 8]>(page_vram)?; + for i in 0..PAGE_SIZE / 8 { + io_write!(window.view(), [build: i], 0); + } + + Ok(PreparedPtPage { + alloc: blocks, + level, + }) + } + + /// Ensure all intermediate page table pages exist for a single VFN. + /// + /// The mutable PRAMIN borrow ends before each allocation. + fn ensure_single_pte_path( + &self, + mm: &mut GpuMm<'_>, + vfn: Vfn, + pt_pages: &mut RBTree, + ) -> Result { + let max_iter = 2 * M::PDE_LEVELS.len(); + + for _ in 0..max_iter { + let result = self + .walker + .walk_pde_levels(mm.pramin_mut(), vfn, |install_addr| { + pt_pages.get(&install_addr).and_then(|p| { + p.alloc + .iter() + .next() + .map(|b| VramAddress::from_raw(b.offset())) + }) + })?; + + match result { + WalkPdeResult::Complete { .. } => { + return Ok(()); + } + WalkPdeResult::Missing { + install_addr, + level, + } => { + let page = Self::alloc_and_zero_page(mm, level)?; + let node = RBTreeNode::new(install_addr, page, GFP_KERNEL)?; + let old = pt_pages.insert(node); + if old.is_some() { + kernel::pr_warn_once!( + "VMM: duplicate install_addr in pt_pages (internal consistency error)\n" + ); + return Err(EIO); + } + } + } + } + + kernel::pr_warn!( + "VMM: ensure_pte_path: loop exhausted after {} iters (VFN {:?})\n", + max_iter, + vfn + ); + Err(EIO) + } + + /// Prepare page table resources for mapping `num_pages` pages starting at `vfn_start`. + /// + /// Reserves capacity in `page_table_allocs`, then walks the hierarchy + /// per-VFN to prepare pages for all missing PDEs. + pub(super) fn prepare_map( + &self, + mm: &mut GpuMm<'_>, + vfn_start: Vfn, + num_pages: usize, + page_table_allocs: &mut KVec>>, + pt_pages: &mut RBTree, + ) -> Result { + // Pre-reserve so install_mappings() can use push_within_capacity (no alloc + // in fence signalling critical path). + let pt_upper_bound = M::pt_pages_upper_bound(num_pages); + page_table_allocs.reserve(pt_upper_bound, GFP_KERNEL)?; + + // Walk the hierarchy per-VFN to prepare pages for all missing PDEs. + for i in 0..num_pages { + let i_u64: u64 = i.into_safe_cast(); + let vfn = Vfn::new(vfn_start.raw() + i_u64); + self.ensure_single_pte_path(mm, vfn, pt_pages)?; + } + Ok(()) + } + + /// Install prepared PDEs and write PTEs, then flush TLB. + /// + /// Drains `pt_pages` and moves allocations into `page_table_allocs`. + pub(super) fn install_mappings( + &self, + mm: &mut GpuMm<'_>, + pt_pages: &mut RBTree, + page_table_allocs: &mut KVec>>, + vfn_start: Vfn, + pfns: &[Pfn], + writable: bool, + ) -> Result { + { + let pramin = mm.pramin_mut(); + + // Drain prepared PT pages, install all pending PDEs. + let mut cursor = pt_pages.cursor_front_mut(); + while let Some(c) = cursor { + let (next, node) = c.remove_current(); + let (install_addr, page) = node.to_key_value(); + let page_vram = + VramAddress::from_raw(page.alloc.iter().next().ok_or(ENOMEM)?.offset()); + + if page.level == M::DUAL_PDE_LEVEL { + let new_dpde = M::DualPde::new_small(Pfn::from(page_vram)); + new_dpde.write(pramin, install_addr)?; + } else { + let new_pde = M::Pde::new(AperturePde::VideoMemory, Pfn::from(page_vram)); + new_pde.write(pramin, install_addr)?; + } + + page_table_allocs + .push_within_capacity(page.alloc) + .map_err(|_| ENOMEM)?; + + cursor = next; + } + + // Write PTEs (all PDEs now installed in HW). + for (i, &pfn) in pfns.iter().enumerate() { + let i_u64: u64 = i.into_safe_cast(); + let vfn = Vfn::new(vfn_start.raw() + i_u64); + let result = self.walker.walk_to_pte_lookup_with_window(pramin, vfn)?; + + match result { + WalkResult::Unmapped { pte_addr } | WalkResult::Mapped { pte_addr, .. } => { + let pte = M::Pte::new(AperturePte::VideoMemory, pfn, writable); + pte.write(pramin, pte_addr)?; + } + WalkResult::PageTableMissing => { + kernel::pr_warn_once!("VMM: page table missing for VFN {vfn:?}\n"); + return Err(EIO); + } + } + } + } + + // Flush TLB. + mm.tlb().flush(self.pdb_addr) + } + + /// Invalidate PTEs for a range and flush TLB. + pub(super) fn invalidate_ptes( + &self, + mm: &mut GpuMm<'_>, + vfn_start: Vfn, + num_pages: usize, + ) -> Result { + let invalid_pte = M::Pte::invalid(); + + { + let pramin = mm.pramin_mut(); + for i in 0..num_pages { + let i_u64: u64 = i.into_safe_cast(); + let vfn = Vfn::new(vfn_start.raw() + i_u64); + let result = self.walker.walk_to_pte_lookup_with_window(pramin, vfn)?; + + match result { + WalkResult::Mapped { pte_addr, .. } | WalkResult::Unmapped { pte_addr } => { + invalid_pte.write(pramin, pte_addr)?; + } + WalkResult::PageTableMissing => { + continue; + } + } + } + } + + mm.tlb().flush(self.pdb_addr) + } +} + +macro_rules! pt_map_dispatch { + ($self:expr, $method:ident ( $($arg:expr),* $(,)? )) => { + match $self { + PtMap::V2(inner) => inner.$method($($arg),*), + PtMap::V3(inner) => inner.$method($($arg),*), + } + }; +} + +/// Page table mapper dispatch. +pub(in crate::mm) enum PtMap { + /// MMU v2 (Turing/Ampere/Ada). + V2(PtMapInner), + /// MMU v3 (Hopper+). + V3(PtMapInner), +} + +impl PtMap { + /// Create a new page table mapper for the given MMU version. + pub(in crate::mm) fn new(pdb_addr: VramAddress, version: MmuVersion) -> Self { + match version { + MmuVersion::V2 => Self::V2(PtMapInner::::new(pdb_addr)), + MmuVersion::V3 => Self::V3(PtMapInner::::new(pdb_addr)), + } + } + + /// Prepare page table resources for a mapping. + pub(in crate::mm) fn prepare_map( + &self, + mm: &mut GpuMm<'_>, + vfn_start: Vfn, + num_pages: usize, + page_table_allocs: &mut KVec>>, + pt_pages: &mut RBTree, + ) -> Result { + pt_map_dispatch!( + self, + prepare_map(mm, vfn_start, num_pages, page_table_allocs, pt_pages) + ) + } + + /// Install prepared PDEs and write PTEs, then flush TLB. + pub(in crate::mm) fn install_mappings( + &self, + mm: &mut GpuMm<'_>, + pt_pages: &mut RBTree, + page_table_allocs: &mut KVec>>, + vfn_start: Vfn, + pfns: &[Pfn], + writable: bool, + ) -> Result { + pt_map_dispatch!( + self, + install_mappings(mm, pt_pages, page_table_allocs, vfn_start, pfns, writable) + ) + } + + /// Invalidate PTEs for a range and flush TLB. + pub(in crate::mm) fn invalidate_ptes( + &self, + mm: &mut GpuMm<'_>, + vfn_start: Vfn, + num_pages: usize, + ) -> Result { + pt_map_dispatch!(self, invalidate_ptes(mm, vfn_start, num_pages)) + } +} diff --git a/drivers/gpu/nova-core/mm/vmm.rs b/drivers/gpu/nova-core/mm/vmm.rs index 9f4f40109227..411710d03f7a 100644 --- a/drivers/gpu/nova-core/mm/vmm.rs +++ b/drivers/gpu/nova-core/mm/vmm.rs @@ -3,20 +3,26 @@ //! Virtual Memory Manager for NVIDIA GPU page table management. //! //! The [`Vmm`] provides high-level page mapping and unmapping operations for GPU -//! virtual address spaces (Channels, BAR1, BAR2). It wraps the page table walker -//! and handles TLB flushing after modifications. +//! virtual address spaces (Channels, BAR1, BAR2). use kernel::{ gpu::buddy::AllocatedBlocks, maple_tree::MapleTreeAlloc, - prelude::*, // + prelude::*, + rbtree::RBTree, // }; -use core::ops::Range; +use core::{ + cell::Cell, + ops::Range, // +}; use crate::{ mm::{ pagetable::{ + map::{ + PtMap, // + }, walk::{ PtWalk, WalkResult, // @@ -34,22 +40,108 @@ }, }; +/// Multi-page prepared mapping -- VA range allocated, ready for execute. +/// +/// Produced by [`Vmm::prepare_map()`], consumed by [`Vmm::execute_map()`]. +/// The VA space allocation is tracked in the [`Vmm`]'s maple tree and freed +/// on error or via [`Vmm::unmap_pages()`]. +/// +/// Dropping without calling [`Vmm::execute_map()`] logs a warning and leaks +/// the VA range in the maple tree. +pub(crate) struct PreparedMapping { + vfn_start: Vfn, + num_pages: usize, + /// Logs a warning if dropped without executing. + _drop_guard: MustExecuteGuard, +} + +/// Result of a mapping operation -- tracks the active mapped range. +/// +/// Returned by [`Vmm::execute_map()`] and [`Vmm::map_pages()`]. +/// Callers must call [`Vmm::unmap_pages()`] before dropping to invalidate +/// PTEs and free the VA range. Dropping without unmapping logs a warning +/// and leaks the VA range in the maple tree. +pub(crate) struct MappedRange { + pub(super) vfn_start: Vfn, + pub(super) num_pages: usize, + /// Logs a warning if dropped without unmapping. + _drop_guard: MustUnmapGuard, +} + +/// Guard that logs a warning if a [`PreparedMapping`] is dropped without +/// being consumed by [`Vmm::execute_map()`]. +struct MustExecuteGuard { + armed: Cell, +} + +impl MustExecuteGuard { + const fn new() -> Self { + Self { + armed: Cell::new(true), + } + } + + fn disarm(&self) { + self.armed.set(false); + } +} + +impl Drop for MustExecuteGuard { + fn drop(&mut self) { + if self.armed.get() { + kernel::pr_warn!("PreparedMapping dropped without calling execute_map()\n"); + } + } +} + +/// Guard that logs a warning if a [`MappedRange`] is dropped without +/// calling [`Vmm::unmap_pages()`]. +struct MustUnmapGuard { + armed: Cell, +} + +impl MustUnmapGuard { + const fn new() -> Self { + Self { + armed: Cell::new(true), + } + } + + fn disarm(&self) { + self.armed.set(false); + } +} + +impl Drop for MustUnmapGuard { + fn drop(&mut self) { + if self.armed.get() { + kernel::pr_warn!("MappedRange dropped without calling unmap_pages()\n"); + } + } +} + /// Virtual Memory Manager for a GPU address space. /// /// Each [`Vmm`] instance manages a single address space identified by its Page -/// Directory Base (`PDB`) address. The [`Vmm`] is used for Channel, BAR1 and -/// BAR2 mappings. +/// Directory Base (`PDB`) address. Used for Channel, BAR1 and BAR2 mappings. pub(crate) struct Vmm { /// Page Directory Base address for this address space. pdb_addr: VramAddress, - /// MMU version used for page table layout. - mmu_version: MmuVersion, + /// Page table walker for reading existing mappings. + pt_walk: PtWalk, + /// Page table mapper for prepare/execute operations. + pt_map: PtMap, /// Page table allocations required for mappings. page_table_allocs: KVec>>, /// Maple tree allocator for virtual address range tracking. virt_alloc: Pin>>, /// Total number of pages in the virtual address space. va_pages: usize, + /// Prepared PT pages pending PDE installation, keyed by `install_addr`. + /// + /// Populated during prepare phase and drained in execute phase. Shared by all + /// pending maps, preventing races on the same PDE slot. + pt_pages: RBTree, } impl Vmm { @@ -67,20 +159,16 @@ pub(crate) fn new( Ok(Self { pdb_addr, - mmu_version, + pt_walk: PtWalk::new(pdb_addr, mmu_version), + pt_map: PtMap::new(pdb_addr, mmu_version), page_table_allocs: KVec::new(), virt_alloc, va_pages, + pt_pages: RBTree::new(), }) } /// Allocate a contiguous virtual frame number range. - /// - /// # Arguments - /// - /// - `num_pages`: Number of pages to allocate. - /// - `va_range`: `None` = allocate anywhere, `Some(range)` = constrain allocation to the given - /// range. fn alloc_vfn_range(&self, num_pages: usize, va_range: Option>) -> Result { let page_size: u64 = PAGE_SIZE.into_safe_cast(); @@ -116,11 +204,142 @@ fn free_vfn(&self, vfn: Vfn) { /// Read the [`Pfn`] for a mapped [`Vfn`] if one is mapped. pub(super) fn read_mapping(&self, mm: &mut GpuMm<'_>, vfn: Vfn) -> Result> { - let walker = PtWalk::new(self.pdb_addr, self.mmu_version); - - match walker.walk_to_pte(mm, vfn)? { + match self.pt_walk.walk_to_pte(mm, vfn)? { WalkResult::Mapped { pfn, .. } => Ok(Some(pfn)), WalkResult::Unmapped { .. } | WalkResult::PageTableMissing => Ok(None), } } + + /// Prepare resources for mapping `num_pages` pages. + /// + /// Allocates a contiguous VA range, then walks the hierarchy per-VFN to prepare pages + /// for all missing PDEs. Returns a [`PreparedMapping`] with the VA allocation. + /// + /// If `va_range` is not `None`, the VA range is constrained to the given range. Safe + /// to call outside the fence signalling critical path. + pub(crate) fn prepare_map( + &mut self, + mm: &mut GpuMm<'_>, + num_pages: usize, + va_range: Option>, + ) -> Result { + if num_pages == 0 { + return Err(EINVAL); + } + + // Allocate contiguous VA range. + let vfn_start = self.alloc_vfn_range(num_pages, va_range)?; + + if let Err(e) = self.pt_map.prepare_map( + mm, + vfn_start, + num_pages, + &mut self.page_table_allocs, + &mut self.pt_pages, + ) { + self.free_vfn(vfn_start); + return Err(e); + } + + Ok(PreparedMapping { + vfn_start, + num_pages, + _drop_guard: MustExecuteGuard::new(), + }) + } + + /// Execute a prepared multi-page mapping. + /// + /// Installs all prepared PDEs and writes PTEs into the page table, then flushes TLB. + pub(crate) fn execute_map( + &mut self, + mm: &mut GpuMm<'_>, + prepared: PreparedMapping, + pfns: &[Pfn], + writable: bool, + ) -> Result { + if pfns.len() != prepared.num_pages { + self.free_vfn(prepared.vfn_start); + return Err(EINVAL); + } + + let PreparedMapping { + vfn_start, + num_pages, + _drop_guard, + } = prepared; + _drop_guard.disarm(); + + if let Err(e) = self.pt_map.install_mappings( + mm, + &mut self.pt_pages, + &mut self.page_table_allocs, + vfn_start, + pfns, + writable, + ) { + self.free_vfn(vfn_start); + return Err(e); + } + + Ok(MappedRange { + vfn_start, + num_pages, + _drop_guard: MustUnmapGuard::new(), + }) + } + + /// Map pages doing prepare and execute in the same call. + /// + /// This is a convenience wrapper for callers outside the fence signalling critical + /// path (e.g., BAR mappings). For DRM usecases, [`Vmm::prepare_map()`] and + /// [`Vmm::execute_map()`] will be called separately. + pub(crate) fn map_pages( + &mut self, + mm: &mut GpuMm<'_>, + pfns: &[Pfn], + va_range: Option>, + writable: bool, + ) -> Result { + if pfns.is_empty() { + return Err(EINVAL); + } + + // Check if provided VA range is sufficient (if provided). + if let Some(ref range) = va_range { + let required: u64 = pfns + .len() + .checked_mul(PAGE_SIZE) + .ok_or(EOVERFLOW)? + .into_safe_cast(); + let available = range.end.checked_sub(range.start).ok_or(EINVAL)?; + if available < required { + return Err(EINVAL); + } + } + + let prepared = self.prepare_map(mm, pfns.len(), va_range)?; + self.execute_map(mm, prepared, pfns, writable) + } + + /// Unmap all pages in a [`MappedRange`] with a single TLB flush. + pub(crate) fn unmap_pages(&mut self, mm: &mut GpuMm<'_>, range: MappedRange) -> Result { + let result = self + .pt_map + .invalidate_ptes(mm, range.vfn_start, range.num_pages); + + // TODO: Internal page table pages (PDE, PTE pages) are still kept around. + // This is by design as repeated maps/unmaps will be fast. As a future TODO, + // we can add a reclaimer here to reclaim if VRAM is short. For now, the PT + // pages are dropped once the `Vmm` is dropped. + + // Free the VA range regardless of PTE invalidation success, so that the VA + // range is recovered even on failure (PTEs may be stale, but that is better + // than leaking both PTEs and VA range). + self.free_vfn(range.vfn_start); + + // Unmap complete, safe to drop `MappedRange`. + range._drop_guard.disarm(); + result + } } From 75ad43886a20e17aa5fad9ae71ebb413cadd8d64 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:52 +0900 Subject: [PATCH 055/110] gpu: nova-core: Add BAR1 aperture type and size constant Add BAR1_SIZE constant and Bar1 type alias for the 256MB BAR1 aperture. These are prerequisites for BAR1 memory access functionality. Co-developed-by: Zhi Wang Signed-off-by: Zhi Wang Signed-off-by: Joel Fernandes [ecourtney: add the Bar1 lifetime for current PCI BAR ownership] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-14-8dd5d4225d2e@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/driver.rs | 2 ++ drivers/gpu/nova-core/gsp/commands.rs | 4 ++++ drivers/gpu/nova-core/gsp/fw/commands.rs | 8 ++++++++ 3 files changed, 14 insertions(+) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index 8cef4c284277..5723ff8f71ea 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -38,6 +38,8 @@ pub(crate) struct NovaCore<'bound> { pub(crate) type Bar0<'a> = &'a pci::Bar<'a, BAR0_SIZE>; pub(crate) type NovaRegisters = kernel::io::Region; +#[expect(dead_code)] +pub(crate) type Bar1<'a> = pci::Bar<'a>; kernel::pci_device_table!( PCI_TABLE, diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index 91f7037edefc..0b399448ee1c 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -214,6 +214,9 @@ fn init(&self) -> impl Init { /// The reply from the GSP to the [`GetGspStaticInfo`] command. pub(crate) struct GetGspStaticInfoReply { gpu_name: [u8; 64], + /// BAR1 Page Directory Entry base address. + #[expect(dead_code)] + pub(crate) bar1_pde_base: u64, /// Usable FB (VRAM) regions for driver memory allocation. pub(crate) usable_fb_regions: KVec>, /// Exclusive end of the FB physical address space. @@ -237,6 +240,7 @@ fn read( Ok(GetGspStaticInfoReply { gpu_name: msg.gpu_name_str(), + bar1_pde_base: msg.bar1_pde_base(), usable_fb_regions, total_fb_end, }) diff --git a/drivers/gpu/nova-core/gsp/fw/commands.rs b/drivers/gpu/nova-core/gsp/fw/commands.rs index 6e85442a4b13..32856ff74183 100644 --- a/drivers/gpu/nova-core/gsp/fw/commands.rs +++ b/drivers/gpu/nova-core/gsp/fw/commands.rs @@ -131,6 +131,14 @@ impl GspStaticConfigInfo { self.0.gpuNameString } + /// Returns the BAR1 Page Directory Entry base address. + /// + /// This is the root page table address for BAR1 virtual memory, + /// set up by GSP-RM firmware. + pub(crate) fn bar1_pde_base(&self) -> u64 { + self.0.bar1PdeBase + } + /// Returns an iterator over valid FB regions from GSP firmware data. fn fb_regions( &self, From 159427b213524a90853ef24d0a0c2ed1aec1dbf9 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:53 +0900 Subject: [PATCH 056/110] gpu: nova-core: mm: Add BAR1 user interface Add the BAR1 user interface for CPU access to GPU virtual memory through the BAR1 aperture. Signed-off-by: Joel Fernandes [ecourtney: map BAR1 in NovaCore, borrow it in Gpu and BarUser, no Devres] [ecourtney: update for the VramAddress raw API and gsp_resources chipset] [ecourtney: drop the owned GpuMm, pass it mutably through map and release] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-15-8dd5d4225d2e@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/driver.rs | 45 +++++-- drivers/gpu/nova-core/gpu.rs | 32 ++++- drivers/gpu/nova-core/gsp/commands.rs | 1 - drivers/gpu/nova-core/mm.rs | 1 + drivers/gpu/nova-core/mm/bar_user.rs | 171 ++++++++++++++++++++++++++ 5 files changed, 240 insertions(+), 10 deletions(-) create mode 100644 drivers/gpu/nova-core/mm/bar_user.rs diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index 5723ff8f71ea..0672a0707a71 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -2,7 +2,11 @@ use kernel::{ auxiliary, - device::Core, + device::{ + Bound, + Core, // + }, + io::resource, pci, pci::{ Class, @@ -28,6 +32,7 @@ pub(crate) struct NovaCore<'bound> { #[pin] pub(crate) gpu: Gpu<'bound>, bar: pci::Bar<'bound, BAR0_SIZE>, + bar1: Bar1<'bound>, #[allow(clippy::type_complexity)] _reg: auxiliary::Registration<'bound, CovariantForLt!(())>, } @@ -38,9 +43,27 @@ pub(crate) struct NovaCore<'bound> { pub(crate) type Bar0<'a> = &'a pci::Bar<'a, BAR0_SIZE>; pub(crate) type NovaRegisters = kernel::io::Region; -#[expect(dead_code)] pub(crate) type Bar1<'a> = pci::Bar<'a>; +/// Returns the Linux PCI resource index that holds BAR1 for an NVIDIA GPU. +/// +/// On Maxwell through Ada, BAR0 is a 32-bit memory BAR occupying a single +/// Linux PCI resource slot, so BAR1 lives at index 1. Starting with Blackwell +/// (and on some Ampere GA100 / Hopper SKUs) BAR0 is a 64-bit memory BAR that +/// consumes two consecutive resource slots: index 0 holds the low 32 bits and +/// index 1 holds the high 32 bits (with no `flags` / or size of its own), +/// shifting BAR1 to index 2. +pub(crate) fn bar1_resource_index(pdev: &pci::Device) -> Result { + // Probe the `IORESOURCE_MEM_64` flag of BAR0 as a robust way of exposing + // if BAR0 and hence BAR1 is 64-bit. + let flags0 = pdev.resource_flags(0)?; + if flags0.contains(resource::Flags::IORESOURCE_MEM_64) { + Ok(2) + } else { + Ok(1) + } +} + kernel::pci_device_table!( PCI_TABLE, ::IdInfo, @@ -82,12 +105,18 @@ fn probe<'bound>( Ok(try_pin_init!(NovaCore { bar: pdev.iomap_region_sized::(0, c"nova-core/bar0")?, - // TODO: Use `&bar` self-referential pin-init syntax once available. - // - // SAFETY: `bar` is initialized before this expression is evaluated - // (`try_pin_init!()` initializes fields in declaration order), lives at a pinned - // stable address, and is dropped after `gpu` (struct field drop order). - gpu <- Gpu::new(pdev, unsafe { &*core::ptr::from_ref(bar) }), + bar1: { + let bar1_idx = bar1_resource_index(pdev)?; + pdev.iomap_region(bar1_idx, c"nova-core/bar1")? + }, + // TODO: Use self-referential pin-init syntax once available. + gpu <- Gpu::new( + pdev, + // SAFETY: `bar` is initialized above, pinned, and outlives `gpu`. + unsafe { &*core::ptr::from_ref(bar) }, + // SAFETY: `bar1` is initialized above, pinned, and outlives `gpu`. + unsafe { &*core::ptr::from_ref(bar1) }, + ), // Run optional GPU selftests. #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] _: { gpu.run_selftests(pdev) }, diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index e9d3c10708f1..9848e474c9bf 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -16,11 +16,15 @@ SizeConstants, SZ_4K, // }, + sync::Arc, }; use crate::{ bounded_enum, - driver::Bar0, + driver::{ + Bar0, + Bar1, // + }, falcon::{ gsp::Gsp as GspFalcon, sec2::Sec2 as Sec2Falcon, @@ -35,6 +39,8 @@ GspBootContext, // }, mm::{ + bar_user::BarUser, + pagetable::MmuVersion, GpuMm, VramAddress, // }, @@ -148,6 +154,11 @@ pub(crate) const fn arch(self) -> Architecture { pub(crate) fn pci_config_mirror_range(self) -> Range { hal::gpu_hal(self).pci_config_mirror_range() } + + /// Returns the MMU version for this chipset. + pub(crate) fn mmu_version(self) -> MmuVersion { + MmuVersion::from(self.arch()) + } } // TODO @@ -297,6 +308,8 @@ pub(crate) struct Gpu<'gpu> { /// Must be kept declared *before* `gsp_resources`, so that its components are dropped while /// the GSP is still operational. mm: GpuMm<'gpu>, + /// BAR1 user interface for CPU access to GPU virtual memory. + bar_user: Arc>, /// GSP and its resources. #[pin] gsp_resources: GspResources<'gpu>, @@ -340,6 +353,7 @@ impl<'gpu> Gpu<'gpu> { pub(crate) fn new<'a>( pdev: &'gpu pci::Device>, bar: Bar0<'gpu>, + bar1: &'gpu Bar1<'gpu>, ) -> impl PinInit + use<'gpu, 'a> { let dev = pdev.as_ref(); @@ -442,6 +456,22 @@ pub(crate) fn new<'a>( VramAddress::from_raw(gsp_static_info.total_fb_end), )? }, + + // Create BAR1 user interface for CPU access to GPU virtual memory. + bar_user: { + let pdb_addr = VramAddress::from_raw(gsp_static_info.bar1_pde_base); + let bar1_idx = crate::driver::bar1_resource_index(pdev)?; + let bar1_size = pdev.resource_len(bar1_idx)?; + Arc::pin_init( + BarUser::new( + pdb_addr, + gsp_resources.spec.chipset, + bar1_size, + bar1, + )?, + GFP_KERNEL, + )? + }, }) } diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index 0b399448ee1c..e087c9e8c35c 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -215,7 +215,6 @@ fn init(&self) -> impl Init { pub(crate) struct GetGspStaticInfoReply { gpu_name: [u8; 64], /// BAR1 Page Directory Entry base address. - #[expect(dead_code)] pub(crate) bar1_pde_base: u64, /// Usable FB (VRAM) regions for driver memory allocation. pub(crate) usable_fb_regions: KVec>, diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index 2cf37254fbb9..e04497256519 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -60,6 +60,7 @@ macro_rules! impl_pfn_bounded { pub(crate) use tlb::Tlb; +pub(crate) mod bar_user; mod hal; pub(super) mod pagetable; mod pramin; diff --git a/drivers/gpu/nova-core/mm/bar_user.rs b/drivers/gpu/nova-core/mm/bar_user.rs new file mode 100644 index 000000000000..ef1d8e6f8c9c --- /dev/null +++ b/drivers/gpu/nova-core/mm/bar_user.rs @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! BAR1 user interface for CPU access to GPU virtual memory. Used for USERD +//! for GPU work submission, and applications to access GPU buffers via mmap(). + +use kernel::{ + io::Io, + new_mutex, + prelude::*, + sync::{ + Arc, + Mutex, // + }, +}; + +use crate::{ + driver::Bar1, + gpu::Chipset, + mm::{ + vmm::{ + MappedRange, + Vmm, // + }, + GpuMm, + Pfn, + Vfn, + VirtualAddress, + VramAddress, + PAGE_SIZE, // + }, + num::IntoSafeCast, +}; + +/// BAR1 user interface for virtual memory mappings. +/// +/// Owns the [`Vmm`] for the BAR1 address space. +#[pin_data] +pub(crate) struct BarUser<'gpu> { + #[pin] + vmm: Mutex, + bar1: &'gpu Bar1<'gpu>, +} + +impl<'gpu> BarUser<'gpu> { + /// Create a pin-initializer for [`BarUser`]. + pub(crate) fn new( + pdb_addr: VramAddress, + chipset: Chipset, + va_size: u64, + bar1: &'gpu Bar1<'gpu>, + ) -> Result + 'gpu> { + let vmm = Vmm::new(pdb_addr, chipset.mmu_version(), va_size)?; + Ok(pin_init!(Self { + vmm <- new_mutex!(vmm, "bar_user_vmm"), + bar1, + })) + } + + /// Map physical pages to a contiguous BAR1 virtual range. + pub(crate) fn map( + self: &Arc, + mm: &mut GpuMm<'_>, + pfns: &[Pfn], + writable: bool, + ) -> Result> { + if pfns.is_empty() { + return Err(EINVAL); + } + let mut vmm = self.vmm.lock(); + let mapped = vmm.map_pages(mm, pfns, None, writable)?; + + Ok(BarUserAccess { + bar_user: self.clone(), + mapped: Some(mapped), + }) + } +} + +/// Access object for a mapped BAR1 region. +pub(crate) struct BarUserAccess<'gpu> { + bar_user: Arc>, + /// [`BarUserAccess::release`] [`Option::take`]s this; `Some` at + /// drop time means `release()` was never called. + mapped: Option, +} + +impl BarUserAccess<'_> { + /// Tear down the BAR1 mapping. + pub(crate) fn release(mut self, mm: &mut GpuMm<'_>) -> Result { + let mapped = self.mapped.take().ok_or(EINVAL)?; + let mut vmm = self.bar_user.vmm.lock(); + vmm.unmap_pages(mm, mapped)?; + Ok(()) + } + + /// Returns the active mapping. + fn mapped(&self) -> &MappedRange { + // `mapped` is only `None` after `take()` in `release`; hence unwrap() + // cannot panic here. + self.mapped.as_ref().unwrap() + } + + /// Get the base virtual address of this mapping. + pub(crate) fn base(&self) -> VirtualAddress { + VirtualAddress::from(self.mapped().vfn_start) + } + + /// Get the total size of the mapped region in bytes. + pub(crate) fn size(&self) -> usize { + self.mapped().num_pages * PAGE_SIZE + } + + /// Get the starting virtual frame number. + pub(crate) fn vfn_start(&self) -> Vfn { + self.mapped().vfn_start + } + + /// Get the number of pages in this mapping. + pub(crate) fn num_pages(&self) -> usize { + self.mapped().num_pages + } + + /// Translate an offset within this mapping to a BAR1 aperture offset. + fn bar_offset(&self, offset: usize) -> Result { + if offset >= self.size() { + return Err(EINVAL); + } + + let base_vfn: usize = self.mapped().vfn_start.raw().into_safe_cast(); + let base = base_vfn.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?; + base.checked_add(offset).ok_or(EOVERFLOW) + } + + // Fallible accessors with runtime bounds checking. + + /// Read a 32-bit value at the given offset. + pub(crate) fn try_read32(&self, offset: usize) -> Result { + let off = self.bar_offset(offset)?; + self.bar_user.bar1.try_read32(off) + } + + /// Write a 32-bit value at the given offset. + pub(crate) fn try_write32(&self, value: u32, offset: usize) -> Result { + let off = self.bar_offset(offset)?; + self.bar_user.bar1.try_write32(value, off) + } + + /// Read a 64-bit value at the given offset. + pub(crate) fn try_read64(&self, offset: usize) -> Result { + let off = self.bar_offset(offset)?; + self.bar_user.bar1.try_read64(off) + } + + /// Write a 64-bit value at the given offset. + pub(crate) fn try_write64(&self, value: u64, offset: usize) -> Result { + let off = self.bar_offset(offset)?; + self.bar_user.bar1.try_write64(value, off) + } +} + +impl Drop for BarUserAccess<'_> { + fn drop(&mut self) { + if self.mapped.is_some() { + kernel::pr_warn!( + "BarUserAccess dropped without calling release(). BarUser address space will leak.\n" + ); + } + // The inner `MappedRange`'s own `MustUnmapGuard` will also fire, + // identifying the leaked VA range. + } +} From d669686f8170c234edf12212fac9180ea18b1448 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:54 +0900 Subject: [PATCH 057/110] gpu: nova-core: mm: Add BAR1 memory management self-tests Add self-tests for BAR1 access during driver probe when CONFIG_NOVA_CORE_SELFTESTS is enabled (default disabled). This results in testing the Vmm, GPU buddy allocator and BAR1 region all of which should function correctly for the tests to pass. Signed-off-by: Joel Fernandes [ecourtney: use existing self-test runner and CONFIG_NOVA_CORE_SELFTESTS] [ecourtney: update for mutable GpuMm, PRAMIN views and VramAddress raw API] [ecourtney: restore the conditional dead_code expect] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-16-8dd5d4225d2e@nvidia.com [acourbot: use vertical import style.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gpu.rs | 9 +- drivers/gpu/nova-core/mm.rs | 12 +- drivers/gpu/nova-core/mm/bar_user.rs | 253 ++++++++++++++++++++++++++ drivers/gpu/nova-core/mm/pagetable.rs | 24 +++ drivers/gpu/nova-core/mm/vmm.rs | 1 + 5 files changed, 295 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 9848e474c9bf..d763bc8d3827 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -482,7 +482,14 @@ pub(crate) fn run_selftests(self: Pin<&mut Self>, pdev: &pci::Device`. /// @@ -202,6 +202,7 @@ pub(crate) struct VirtualAddress(u64) { impl VirtualAddress { /// Create a new virtual address from a raw value. + #[expect(dead_code)] pub(crate) const fn new(addr: u64) -> Self { Self::from_raw(addr) } @@ -297,7 +298,8 @@ pub(crate) mod selftest { use kernel::{ device, - sizes::SizeConstants, // + sizes::SizeConstants, + sync::Arc, // }; use super::*; @@ -307,6 +309,9 @@ pub(crate) fn run( dev: &device::Device, mm: &mut GpuMm<'_>, usable_fb_regions: &[Range], + bar_user: &Arc>, + bar1_pdb: u64, + chipset: Chipset, ) -> Result { // VRAM span the self-tests are free to overwrite, from the chosen test base. const SELFTEST_SPAN: u64 = u64::SZ_64M; @@ -325,6 +330,7 @@ pub(crate) fn run( return Ok(()); }; - pramin::selftest::run(dev, mm.pramin_mut(), VramAddress::from_raw(base)) + pramin::selftest::run(dev, mm.pramin_mut(), VramAddress::from_raw(base))?; + bar_user::run_self_test(dev, mm, bar_user, bar1_pdb, chipset) } } diff --git a/drivers/gpu/nova-core/mm/bar_user.rs b/drivers/gpu/nova-core/mm/bar_user.rs index ef1d8e6f8c9c..8f4a27c1fd14 100644 --- a/drivers/gpu/nova-core/mm/bar_user.rs +++ b/drivers/gpu/nova-core/mm/bar_user.rs @@ -31,6 +31,9 @@ num::IntoSafeCast, }; +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +use kernel::device; + /// BAR1 user interface for virtual memory mappings. /// /// Owns the [`Vmm`] for the BAR1 address space. @@ -84,6 +87,7 @@ pub(crate) struct BarUserAccess<'gpu> { mapped: Option, } +#[expect(dead_code)] impl BarUserAccess<'_> { /// Tear down the BAR1 mapping. pub(crate) fn release(mut self, mm: &mut GpuMm<'_>) -> Result { @@ -169,3 +173,252 @@ fn drop(&mut self) { // identifying the leaked VA range. } } + +/// Run MM subsystem self-tests during probe. +/// +/// Tests page table infrastructure and `BAR1` MMIO access using the `BAR1` +/// address space. Uses the `GpuMm`'s buddy allocator to allocate page tables +/// and test pages as needed. +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +pub(crate) fn run_self_test( + dev: &device::Device, + mm: &mut GpuMm<'_>, + bar_user: &Arc>, + bar1_pdb: u64, + chipset: Chipset, +) -> Result { + use kernel::{ + gpu::buddy::{ + GpuBuddyAllocFlags, + GpuBuddyAllocMode, // + }, + ptr::Alignment, + sizes::{ + SZ_16K, + SZ_32K, + SZ_4K, + SZ_64K, // + }, + }; + + // Test patterns. + const PATTERN_PRAMIN: u32 = 0xDEAD_BEEF; + const PATTERN_BAR1: u32 = 0xCAFE_BABE; + + let bar1 = bar_user.bar1; + dev_info!(dev, "MM: Starting self-test...\n"); + + let pdb_addr = VramAddress::from_raw(bar1_pdb); + + // Check if initial page tables are in VRAM. + if crate::mm::pagetable::check_pdb_valid(mm.pramin_mut(), pdb_addr, chipset).is_err() { + dev_info!(dev, "MM: Self-test SKIPPED - no valid VRAM page tables\n"); + return Ok(()); + } + + // Set up a test page from the buddy allocator. + let test_page_blocks = KBox::pin_init( + mm.buddy().alloc_blocks( + GpuBuddyAllocMode::Simple, + SZ_4K.into_safe_cast(), + Alignment::new::(), + GpuBuddyAllocFlags::default(), + ), + GFP_KERNEL, + )?; + let test_vram_offset = test_page_blocks.iter().next().ok_or(ENOMEM)?.offset(); + let test_vram = VramAddress::from_raw(test_vram_offset); + let test_pfn = Pfn::from(test_vram); + + // Create a VMM of size 64K to track virtual memory mappings. + let mut vmm = Vmm::new(pdb_addr, chipset.mmu_version(), SZ_64K.into_safe_cast())?; + + // Create a test mapping. + let mapped = vmm.map_pages(mm, &[test_pfn], None, true)?; + let test_vfn = mapped.vfn_start; + + // Pre-compute test addresses for the PRAMIN to BAR1 read test. + let vfn_offset: usize = test_vfn.raw().into_safe_cast(); + let bar1_base_offset = vfn_offset.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?; + let bar1_read_offset: usize = bar1_base_offset + 0x100; + let vram_read_addr = test_vram + 0x100; + + // Test 1: Write via PRAMIN, read via BAR1. + mm.pramin_mut() + .window_at::(vram_read_addr)? + .view() + .write_val(PATTERN_PRAMIN); + + // Read back via BAR1 aperture. + let bar1_value = bar1.try_read32(bar1_read_offset)?; + + let test1_passed = if bar1_value == PATTERN_PRAMIN { + true + } else { + dev_err!( + dev, + "MM: Test 1 FAILED - Expected {:#010x}, got {:#010x}\n", + PATTERN_PRAMIN, + bar1_value + ); + false + }; + + // Cleanup - invalidate PTE. + vmm.unmap_pages(mm, mapped)?; + + // Test 2: Two-phase prepare/execute API. + let prepared = vmm.prepare_map(mm, 1, None)?; + let mapped2 = vmm.execute_map(mm, prepared, &[test_pfn], true)?; + let readback = vmm.read_mapping(mm, mapped2.vfn_start)?; + let test2_passed = if readback == Some(test_pfn) { + true + } else { + dev_err!(dev, "MM: Test 2 FAILED - Two-phase map readback mismatch\n"); + false + }; + vmm.unmap_pages(mm, mapped2)?; + + // Test 3: Range-constrained allocation with a hole — exercises block.size()-driven + // BAR1 mapping. A 4K hole is punched at base+16K, then a single 32K allocation + // is requested within [base, base+36K). The buddy allocator must split around the + // hole, returning multiple blocks (expected: {16K, 4K, 8K, 4K} = 32K total). + // Each block is mapped into BAR1 and verified via PRAMIN read-back. + // + // Address layout (base = 0x10000): + // [ 16K ] [HOLE 4K] [4K] [ 8K ] [4K] + // 0x10000 0x14000 0x15000 0x16000 0x18000 0x19000 + let range_base: u64 = SZ_64K.into_safe_cast(); + let sz_4k: u64 = SZ_4K.into_safe_cast(); + let sz_16k: u64 = SZ_16K.into_safe_cast(); + let sz_32k_4k: u64 = (SZ_32K + SZ_4K).into_safe_cast(); + + // Punch a 4K hole at base+16K so the subsequent 32K allocation must split. + let _hole = KBox::pin_init( + mm.buddy().alloc_blocks( + GpuBuddyAllocMode::Range(range_base + sz_16k..range_base + sz_16k + sz_4k), + SZ_4K.into_safe_cast(), + Alignment::new::(), + GpuBuddyAllocFlags::default(), + ), + GFP_KERNEL, + )?; + + // Allocate 32K within [base, base+36K). The hole forces the allocator to return + // split blocks whose sizes are determined by buddy alignment. + let blocks = KBox::pin_init( + mm.buddy().alloc_blocks( + GpuBuddyAllocMode::Range(range_base..range_base + sz_32k_4k), + SZ_32K.into_safe_cast(), + Alignment::new::(), + GpuBuddyAllocFlags::default(), + ), + GFP_KERNEL, + )?; + + let mut test3_passed = true; + let mut total_size = 0usize; + + for block in blocks.iter() { + total_size += IntoSafeCast::::into_safe_cast(block.size()); + + // Map all pages of this block. + let page_size: u64 = PAGE_SIZE.into_safe_cast(); + let num_pages: usize = (block.size() / page_size).into_safe_cast(); + + let mut pfns = KVec::new(); + for j in 0..num_pages { + let j_u64: u64 = j.into_safe_cast(); + pfns.push( + Pfn::from(VramAddress::from_raw( + block.offset() + j_u64.checked_mul(page_size).ok_or(EOVERFLOW)?, + )), + GFP_KERNEL, + )?; + } + + let mapped = vmm.map_pages(mm, &pfns, None, true)?; + let bar1_base_vfn: usize = mapped.vfn_start.raw().into_safe_cast(); + let bar1_base = bar1_base_vfn.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?; + + for j in 0..num_pages { + let page_bar1_off = bar1_base + j * PAGE_SIZE; + let j_u64: u64 = j.into_safe_cast(); + let page_phys = block.offset() + + j_u64 + .checked_mul(PAGE_SIZE.into_safe_cast()) + .ok_or(EOVERFLOW)?; + + bar1.try_write32(PATTERN_BAR1, page_bar1_off)?; + + let pramin_val = mm + .pramin_mut() + .window_at::(VramAddress::from_raw(page_phys))? + .view() + .read_val(); + + if pramin_val != PATTERN_BAR1 { + dev_err!( + dev, + "MM: Test 3 FAILED block offset {:#x} page {} (val={:#x})\n", + block.offset(), + j, + pramin_val + ); + test3_passed = false; + } + } + + vmm.unmap_pages(mm, mapped)?; + } + + // Verify aggregate: all returned block sizes must sum to allocation size. + if total_size != SZ_32K { + dev_err!( + dev, + "MM: Test 3 FAILED - total size {} != expected {}\n", + total_size, + SZ_32K + ); + test3_passed = false; + } + + // Release Tests 1-3's Vmm before Test 4 constructs a fresh BarUser on + // the same PDB. + drop(vmm); + + // Test 4: Exercise `BarUser::map()` end-to-end. + let bar_user = Arc::pin_init( + BarUser::new(pdb_addr, chipset, SZ_64K.into_safe_cast(), bar1)?, + GFP_KERNEL, + )?; + let access = bar_user.map(mm, &[test_pfn], true)?; + + // Write pattern via PRAMIN, read via BarUserAccess. + mm.pramin_mut() + .window_at::(test_vram)? + .view() + .write_val(PATTERN_BAR1); + + let readback = access.try_read32(0)?; + let test4_passed = if readback == PATTERN_BAR1 { + true + } else { + dev_err!( + dev, + "MM: Test 4 FAILED - Expected {:#010x}, got {:#010x}\n", + PATTERN_BAR1, + readback + ); + false + }; + access.release(mm)?; + + if test1_passed && test2_passed && test3_passed && test4_passed { + dev_info!(dev, "MM: All self-tests PASSED\n"); + Ok(()) + } else { + dev_err!(dev, "MM: Self-tests FAILED\n"); + Err(EIO) + } +} diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs index 63111ff95780..63a7e1855caa 100644 --- a/drivers/gpu/nova-core/mm/pagetable.rs +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -398,3 +398,27 @@ fn from(val: AperturePde) -> Self { Bounded::from_expr(val as u64 & 0x3) } } + +/// Check if the PDB has valid, VRAM-backed page tables. +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +fn check_pdb_inner(pramin: &mut pramin::Pramin<'_>, pdb_addr: VramAddress) -> Result { + let raw = pramin.window_at::(pdb_addr)?.view().read_val(); + + if !M::Pde::from_raw(raw).is_valid_vram() { + return Err(ENOENT); + } + Ok(()) +} + +/// Check if the PDB has valid, VRAM-backed page tables, dispatching by MMU version. +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +pub(super) fn check_pdb_valid( + pramin: &mut pramin::Pramin<'_>, + pdb_addr: VramAddress, + chipset: crate::gpu::Chipset, +) -> Result { + match MmuVersion::from(chipset.arch()) { + MmuVersion::V2 => check_pdb_inner::(pramin, pdb_addr), + MmuVersion::V3 => check_pdb_inner::(pramin, pdb_addr), + } +} diff --git a/drivers/gpu/nova-core/mm/vmm.rs b/drivers/gpu/nova-core/mm/vmm.rs index 411710d03f7a..51b500a27233 100644 --- a/drivers/gpu/nova-core/mm/vmm.rs +++ b/drivers/gpu/nova-core/mm/vmm.rs @@ -126,6 +126,7 @@ fn drop(&mut self) { /// Directory Base (`PDB`) address. Used for Channel, BAR1 and BAR2 mappings. pub(crate) struct Vmm { /// Page Directory Base address for this address space. + #[expect(dead_code)] pdb_addr: VramAddress, /// Page table walker for reading existing mappings. pt_walk: PtWalk, From 66a2c223b620d844fe26c6bd4844d2a6a8c9dffc Mon Sep 17 00:00:00 2001 From: Manuel Ebner Date: Wed, 9 Sep 2026 15:53:19 +0200 Subject: [PATCH 058/110] Documentation: nova: Fix bracket Add missing ')'. Signed-off-by: Manuel Ebner Acked-by: Randy Dunlap Link: https://patch.msgid.link/20260909135319.1364776-3-manuelebnerli@mailbox.org [acourbot: add missing `:` in commit title] Signed-off-by: Alexandre Courbot --- Documentation/gpu/nova/core/todo.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/gpu/nova/core/todo.rst b/Documentation/gpu/nova/core/todo.rst index d5130b2b08fb..a01c362b1be0 100644 --- a/Documentation/gpu/nova/core/todo.rst +++ b/Documentation/gpu/nova/core/todo.rst @@ -33,7 +33,7 @@ A good example from nova-core would be the ``Chipset`` enum type, which defines the value ``AD102``. When probing the GPU the value ``0x192`` can be read from a certain register indication the chipset AD102. Hence, the enum value ``AD102`` should be derived from the number ``0x192``. Currently, nova-core uses a custom -implementation (``Chipset::from_u32`` for this. +implementation (``Chipset::from_u32``) for this. Instead, it would be desirable to have something like the ``FromPrimitive`` trait [1] from the num crate. From 6bd5eaeb9827ca7ba953b68fb27948fdb3e40630 Mon Sep 17 00:00:00 2001 From: Matteo Kloiber Date: Sun, 13 Sep 2026 23:08:27 +0200 Subject: [PATCH 059/110] gpu: nova-core: declare unlimited DMA max segment size The PCI default max_seg_size is 64 KiB. nova-core never raises it, so mapping the GSP firmware image (~60 MiB) as a scatter-gather table triggers a DMA-API debug warning when contiguous pages coalesce into segments that exceed the default: DMA-API: nova-core 0000:00:03.0: mapping sg segment longer than device claims to support [len=62914560] [max=65536] CONFIG_DMA_API_DEBUG=y is required to see this warning. Assisted-by: LLM Signed-off-by: Matteo Kloiber Link: https://patch.msgid.link/20260913210828.125655-2-kernel@matt3o12.de Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gpu.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index d763bc8d3827..3217e23caa4a 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -371,6 +371,13 @@ pub(crate) fn new<'a>( // still constructing it, so no concurrent DMA allocations can exist. unsafe { pdev.dma_set_mask_and_coherent(dma_mask)? }; + // Nova walks SG segments to build page tables, so their length is + // irrelevant to the device. + // + // SAFETY: `Gpu` owns all DMA allocations for this device, and we are + // still constructing it, so no concurrent DMA allocations can exist. + unsafe { pdev.dma_set_max_seg_size(u32::MAX) }; + hal.wait_gfw_boot_completion(bar) .inspect_err(|_| dev_err!(dev, "GFW boot did not complete\n"))?; }, From 38d4302c2fd0231e0720e85ec4e718bb743fe9d2 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Sun, 13 Sep 2026 12:54:11 -0700 Subject: [PATCH 060/110] gpu: nova-core: extract radix3 page table into its own module In order to boot the GSP, nova-core loads the GSP firmware image into system memory, and the GPU copies it into the framebuffer, the GPU's own memory. The GPU finds the GSP firmware image's pages via "radix3", a three-level page table of 4 KiB pages that the driver builds. Today the table exists as fields within the GSP firmware wrapper, which is fine because there is only one radix3 consumer. The upcoming r000 series[1], however, introduces a second radix3 caller: the ucodes firmware image. That is because the GSP has been changed, so that it automatically fetches the subset of ucodes that it requires, thus reducing the size of the GSP image. This is a helpful change for the firmware situation overall, but it will require nova-core to load and provide ucodes, in much the same way that it was loading and providing the GSP firmware image. Therefore, split out radix3 into its own module. [1] https://lore.kernel.org/all/20260822015448.238214-1-jhubbard@nvidia.com/ [acourbot: keep only the mechanical move, split other changes into distinct commits] Assisted-by: LLM Signed-off-by: John Hubbard Reviewed-by: Timur Tabi Link: https://patch.msgid.link/20260913195413.742143-2-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fb.rs | 2 +- drivers/gpu/nova-core/firmware.rs | 1 + drivers/gpu/nova-core/firmware/gsp.rs | 107 ++--------------- drivers/gpu/nova-core/firmware/radix3.rs | 140 +++++++++++++++++++++++ drivers/gpu/nova-core/gsp/fw.rs | 4 +- 5 files changed, 155 insertions(+), 99 deletions(-) create mode 100644 drivers/gpu/nova-core/firmware/radix3.rs diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index b3a6ab8b57a6..b18dc5447a73 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -236,7 +236,7 @@ pub(crate) fn new( let elf = { const ELF_DOWN_ALIGN: Alignment = Alignment::new::(); - let elf_size = u64::from_safe_cast(gsp_fw.size); + let elf_size = u64::from_safe_cast(gsp_fw.size()); let elf_addr = (boot.start - elf_size).align_down(ELF_DOWN_ALIGN); FbRange(elf_addr..elf_addr + elf_size) diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index c16fee6e2b2a..d8f6509a35d1 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -26,6 +26,7 @@ pub(crate) mod fwsec; pub(crate) mod gsp; pub(crate) mod gsp_fmc; +pub(crate) mod radix3; pub(crate) mod riscv; pub(crate) mod tlv; diff --git a/drivers/gpu/nova-core/firmware/gsp.rs b/drivers/gpu/nova-core/firmware/gsp.rs index 22d1f9329c9f..49a9626166bb 100644 --- a/drivers/gpu/nova-core/firmware/gsp.rs +++ b/drivers/gpu/nova-core/firmware/gsp.rs @@ -4,21 +4,16 @@ device, dma::{ Coherent, - CoherentBox, - DataDirection, DmaAddress, // }, firmware, prelude::*, - scatterlist::{ - Owned, - SGTable, // - }, str::CString, }; use crate::{ firmware::{ + radix3::Radix3, riscv::RiscvFirmware, // tlv::{ request_tlv, // @@ -26,38 +21,15 @@ }, }, gpu::Chipset, - gsp::GSP_PAGE_SIZE, num::FromSafeCast, }; -/// GSP firmware with 3-level radix page tables for the GSP bootloader. -/// -/// The bootloader expects firmware to be mapped starting at address 0 in GSP's virtual address -/// space: -/// -/// ```text -/// Level 0: 1 page, 1 entry -> points to first level 1 page -/// Level 1: Multiple pages/entries -> each entry points to a level 2 page -/// Level 2: Multiple pages/entries -> each entry points to a firmware page -/// ``` -/// -/// Each page is 4KB, each entry is 8 bytes (64-bit DMA address). -/// Also known as "Radix3" firmware. +/// The GSP firmware image, its signatures, and the GSP bootloader. #[pin_data] pub(crate) struct GspFirmware<'a> { - /// The GSP firmware inside a [`VVec`], device-mapped via a SG table. + /// The firmware image and the radix3 table that maps it. #[pin] - fw: SGTable>>, - /// Level 2 page table whose entries contain DMA addresses of firmware pages. - #[pin] - level2: SGTable>>, - /// Level 1 page table whose entries contain DMA addresses of level 2 pages. - #[pin] - level1: SGTable>>, - /// Level 0 page table (single 4KB page) with one entry: DMA address of first level 1 page. - level0: Coherent<'a, [u64]>, - /// Size in bytes of the firmware contained in [`Self::fw`]. - pub(crate) size: usize, + radix3: Radix3<'a>, /// Device-mapped GSP signatures matching the GPU's [`Chipset`]. pub(crate) signatures: Coherent<'a, [u8]>, /// GSP bootloader, verifies the GSP firmware before loading and running it. @@ -87,49 +59,7 @@ pub(crate) fn new( let signatures = Coherent::from_slice(dev, tlv.get_bytes(b"SIGN")?, GFP_KERNEL)?; Ok(try_pin_init!(Self { - fw <- SGTable::new(dev, fw_vvec, DataDirection::ToDevice, GFP_KERNEL), - level2 <- { - // Allocate the level 2 page table, map the firmware onto it, and map it into - // the device address space. - VVec::::with_capacity( - fw.iter().count() * core::mem::size_of::(), - GFP_KERNEL, - ) - .map_err(|_| ENOMEM) - .and_then(|level2| map_into_lvl(&fw, level2)) - .map(|level2| SGTable::new(dev, level2, DataDirection::ToDevice, GFP_KERNEL))? - }, - level1 <- { - // Allocate the level 1 page table, map the level 2 page table onto it, and map - // it into the device address space. - VVec::::with_capacity( - level2.iter().count() * core::mem::size_of::(), - GFP_KERNEL, - ) - .map_err(|_| ENOMEM) - .and_then(|level1| map_into_lvl(&level2, level1)) - .map(|level1| SGTable::new(dev, level1, DataDirection::ToDevice, GFP_KERNEL))? - }, - level0: { - // Allocate the level 0 page table as a device-visible DMA object, and map the - // level 1 page table onto it. - - // Fill level 1 page entry. - let level1_entry = level1.iter().next().ok_or(EINVAL)?; - let level1_entry_addr = level1_entry.dma_address(); - - // Create level 0 page table data and fill its first entry with the level 1 - // table. - let mut level0 = CoherentBox::<'_, [u64]>::zeroed_slice( - dev, - GSP_PAGE_SIZE / size_of::(), - GFP_KERNEL - )?; - level0[0] = level1_entry_addr.to_le(); - - level0.into() - }, - size, + radix3 <- Radix3::new(dev, fw_vvec), signatures, bootloader: { let bl = request_tlv(dev, chipset, "gsp_bootloader")?; @@ -140,28 +70,13 @@ pub(crate) fn new( }) } + /// Returns the size of the firmware image, in bytes. + pub(crate) fn size(&self) -> usize { + self.radix3.size() + } + /// Returns the DMA address of the radix3 level 0 page table. pub(crate) fn radix3_dma_address(&self) -> DmaAddress { - self.level0.dma_address() + self.radix3.dma_address() } } - -/// Build a page table from a scatter-gather list. -/// -/// Takes each DMA-mapped region from `sg_table` and writes page table entries -/// for all 4KB pages within that region. For example, a 16KB SG entry becomes -/// 4 consecutive page table entries. -fn map_into_lvl(sg_table: &SGTable>>, mut dst: VVec) -> Result> { - for sg_entry in sg_table.iter() { - // Number of pages we need to map. - let num_pages = usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE); - - for i in 0..num_pages { - let entry = sg_entry.dma_address() - + (u64::from_safe_cast(i) * u64::from_safe_cast(GSP_PAGE_SIZE)); - dst.extend_from_slice(&entry.to_le_bytes(), GFP_KERNEL)?; - } - } - - Ok(dst) -} diff --git a/drivers/gpu/nova-core/firmware/radix3.rs b/drivers/gpu/nova-core/firmware/radix3.rs new file mode 100644 index 000000000000..225a3dd1b4a9 --- /dev/null +++ b/drivers/gpu/nova-core/firmware/radix3.rs @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use kernel::{ + device, + dma::{ + Coherent, + CoherentBox, + DataDirection, + DmaAddress, // + }, + prelude::*, + scatterlist::{ + Owned, + SGTable, // + }, +}; + +use crate::{ + gsp::GSP_PAGE_SIZE, + num::FromSafeCast, // +}; + +/// GSP firmware with 3-level radix page tables for the GSP bootloader. +/// +/// The bootloader expects firmware to be mapped starting at address 0 in GSP's virtual address +/// space: +/// +/// ```text +/// Level 0: 1 page, 1 entry -> points to first level 1 page +/// Level 1: Multiple pages/entries -> each entry points to a level 2 page +/// Level 2: Multiple pages/entries -> each entry points to a firmware page +/// ``` +/// +/// Each page is 4KB, each entry is 8 bytes (64-bit DMA address). +/// Also known as "Radix3" firmware. +#[pin_data] +pub(crate) struct Radix3<'a> { + /// The GSP firmware inside a [`VVec`], device-mapped via a SG table. + #[pin] + fw: SGTable>>, + /// Level 2 page table whose entries contain DMA addresses of firmware pages. + #[pin] + level2: SGTable>>, + /// Level 1 page table whose entries contain DMA addresses of level 2 pages. + #[pin] + level1: SGTable>>, + /// Level 0 page table (single 4KB page) with one entry: DMA address of first level 1 page. + level0: Coherent<'a, [u64]>, + /// Size in bytes of the firmware contained in [`Self::fw`]. + size: usize, +} + +impl<'a> Radix3<'a> { + /// Builds a radix3 page table over `fw_vvec`, mapped for `dev` to read. May sleep. + pub(crate) fn new( + dev: &'a device::Device, + fw_vvec: VVec, + ) -> impl PinInit + 'a { + let size = fw_vvec.len(); + + pin_init::pin_init_scope(move || { + Ok(try_pin_init!(Self { + fw <- SGTable::new(dev, fw_vvec, DataDirection::ToDevice, GFP_KERNEL), + level2 <- { + // Allocate the level 2 page table, map the firmware onto it, and map it into + // the device address space. + VVec::::with_capacity( + fw.iter().count() * core::mem::size_of::(), + GFP_KERNEL, + ) + .map_err(|_| ENOMEM) + .and_then(|level2| map_into_lvl(&fw, level2)) + .map(|level2| SGTable::new(dev, level2, DataDirection::ToDevice, GFP_KERNEL))? + }, + level1 <- { + // Allocate the level 1 page table, map the level 2 page table onto it, and map + // it into the device address space. + VVec::::with_capacity( + level2.iter().count() * core::mem::size_of::(), + GFP_KERNEL, + ) + .map_err(|_| ENOMEM) + .and_then(|level1| map_into_lvl(&level2, level1)) + .map(|level1| SGTable::new(dev, level1, DataDirection::ToDevice, GFP_KERNEL))? + }, + level0: { + // Allocate the level 0 page table as a device-visible DMA object, and map the + // level 1 page table onto it. + + // Fill level 1 page entry. + let level1_entry = level1.iter().next().ok_or(EINVAL)?; + let level1_entry_addr = level1_entry.dma_address(); + + // Create level 0 page table data and fill its first entry with the level 1 + // table. + let mut level0 = CoherentBox::<'_, [u64]>::zeroed_slice( + dev, + GSP_PAGE_SIZE / size_of::(), + GFP_KERNEL + )?; + level0[0] = level1_entry_addr.to_le(); + + level0.into() + }, + size, + })) + }) + } + + /// Returns the DMA address of the radix3 level 0 page table. + pub(crate) fn dma_address(&self) -> DmaAddress { + self.level0.dma_address() + } + + /// Returns the length of the mapped data, in bytes. + pub(crate) fn size(&self) -> usize { + self.size + } +} + +/// Build a page table from a scatter-gather list. +/// +/// Takes each DMA-mapped region from `sg_table` and writes page table entries +/// for all 4KB pages within that region. For example, a 16KB SG entry becomes +/// 4 consecutive page table entries. +fn map_into_lvl(sg_table: &SGTable>>, mut dst: VVec) -> Result> { + for sg_entry in sg_table.iter() { + // Number of pages we need to map. + let num_pages = usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE); + + for i in 0..num_pages { + let entry = sg_entry.dma_address() + + (u64::from_safe_cast(i) * u64::from_safe_cast(GSP_PAGE_SIZE)); + dst.extend_from_slice(&entry.to_le_bytes(), GFP_KERNEL)?; + } + } + + Ok(dst) +} diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 8778c4bf79c0..ae98da42410b 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -187,7 +187,7 @@ pub(crate) fn from_ranges<'a>( magic: bindings::GSP_FW_WPR_META_MAGIC as u64, revision: u64::from(bindings::GSP_FW_WPR_META_REVISION), sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_address(), - sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size), + sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size()), sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_address(), sizeOfBootloader: u64::from_safe_cast(gsp_firmware.bootloader.ucode.size()), bootloaderCodeOffset: u64::from(gsp_firmware.bootloader.code_offset), @@ -242,7 +242,7 @@ pub(crate) fn from_sizes<'a>( magic: bindings::GSP_FW_WPR_META_MAGIC as u64, revision: u64::from(bindings::GSP_FW_WPR_META_REVISION), sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_address(), - sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size), + sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size()), sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_address(), sizeOfBootloader: u64::from_safe_cast(gsp_firmware.bootloader.ucode.size()), bootloaderCodeOffset: u64::from(gsp_firmware.bootloader.code_offset), From 4a19823f6c033f69d56f73d27fce020bbd4bf262 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Sun, 13 Sep 2026 12:54:11 -0700 Subject: [PATCH 061/110] gpu: nova-core: generalize radix3 buffer names The extracted table builder can map buffers other than the GSP firmware image. Rename its `fw` field and `fw_vvec` parameter to `data`, updating the existing documentation references to match. [acourbot: split the original patch into logical changes.] Assisted-by: LLM Signed-off-by: John Hubbard Reviewed-by: Timur Tabi Link: https://patch.msgid.link/20260913195413.742143-2-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/firmware/radix3.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/nova-core/firmware/radix3.rs b/drivers/gpu/nova-core/firmware/radix3.rs index 225a3dd1b4a9..9051509cabaa 100644 --- a/drivers/gpu/nova-core/firmware/radix3.rs +++ b/drivers/gpu/nova-core/firmware/radix3.rs @@ -38,7 +38,7 @@ pub(crate) struct Radix3<'a> { /// The GSP firmware inside a [`VVec`], device-mapped via a SG table. #[pin] - fw: SGTable>>, + data: SGTable>>, /// Level 2 page table whose entries contain DMA addresses of firmware pages. #[pin] level2: SGTable>>, @@ -47,30 +47,30 @@ pub(crate) struct Radix3<'a> { level1: SGTable>>, /// Level 0 page table (single 4KB page) with one entry: DMA address of first level 1 page. level0: Coherent<'a, [u64]>, - /// Size in bytes of the firmware contained in [`Self::fw`]. + /// Size in bytes of the firmware contained in [`Self::data`]. size: usize, } impl<'a> Radix3<'a> { - /// Builds a radix3 page table over `fw_vvec`, mapped for `dev` to read. May sleep. + /// Builds a radix3 page table over `data`, mapped for `dev` to read. May sleep. pub(crate) fn new( dev: &'a device::Device, - fw_vvec: VVec, + data: VVec, ) -> impl PinInit + 'a { - let size = fw_vvec.len(); + let size = data.len(); pin_init::pin_init_scope(move || { Ok(try_pin_init!(Self { - fw <- SGTable::new(dev, fw_vvec, DataDirection::ToDevice, GFP_KERNEL), + data <- SGTable::new(dev, data, DataDirection::ToDevice, GFP_KERNEL), level2 <- { // Allocate the level 2 page table, map the firmware onto it, and map it into // the device address space. VVec::::with_capacity( - fw.iter().count() * core::mem::size_of::(), + data.iter().count() * core::mem::size_of::(), GFP_KERNEL, ) .map_err(|_| ENOMEM) - .and_then(|level2| map_into_lvl(&fw, level2)) + .and_then(|level2| map_into_lvl(&data, level2)) .map(|level2| SGTable::new(dev, level2, DataDirection::ToDevice, GFP_KERNEL))? }, level1 <- { From 12eb69dafe6ede8e10dbda03ba1ed0c4470f30df Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Sun, 13 Sep 2026 12:54:11 -0700 Subject: [PATCH 062/110] gpu: nova-core: revise radix3 documentation and comments Describe the reusable buffer mapping in module documentation and update the relevant descriptions. [acourbot: split the original patch into logical changes.] Assisted-by: LLM Signed-off-by: John Hubbard Reviewed-by: Timur Tabi Link: https://patch.msgid.link/20260913195413.742143-2-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/firmware/gsp.rs | 2 +- drivers/gpu/nova-core/firmware/radix3.rs | 44 ++++++++++-------------- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/nova-core/firmware/gsp.rs b/drivers/gpu/nova-core/firmware/gsp.rs index 49a9626166bb..341a8b19aa38 100644 --- a/drivers/gpu/nova-core/firmware/gsp.rs +++ b/drivers/gpu/nova-core/firmware/gsp.rs @@ -75,7 +75,7 @@ pub(crate) fn size(&self) -> usize { self.radix3.size() } - /// Returns the DMA address of the radix3 level 0 page table. + /// Returns the DMA address of the radix3 table that maps the firmware image. pub(crate) fn radix3_dma_address(&self) -> DmaAddress { self.radix3.dma_address() } diff --git a/drivers/gpu/nova-core/firmware/radix3.rs b/drivers/gpu/nova-core/firmware/radix3.rs index 9051509cabaa..ca47c5d66978 100644 --- a/drivers/gpu/nova-core/firmware/radix3.rs +++ b/drivers/gpu/nova-core/firmware/radix3.rs @@ -1,6 +1,16 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +//! The radix3 page table, through which firmware running on the GPU reads a buffer in system +//! memory. LibOS, the operating system of the GSP, defines the format: three levels of +//! [`GSP_PAGE_SIZE`] pages, each entry the little-endian DMA address of one page. +//! +//! ```text +//! Level 0: one page, one entry -> the first level 1 page +//! Level 1: pages of entries -> each entry a level 2 page +//! Level 2: pages of entries -> each entry a page of the buffer +//! ``` + use kernel::{ device, dma::{ @@ -21,33 +31,21 @@ num::FromSafeCast, // }; -/// GSP firmware with 3-level radix page tables for the GSP bootloader. -/// -/// The bootloader expects firmware to be mapped starting at address 0 in GSP's virtual address -/// space: -/// -/// ```text -/// Level 0: 1 page, 1 entry -> points to first level 1 page -/// Level 1: Multiple pages/entries -> each entry points to a level 2 page -/// Level 2: Multiple pages/entries -> each entry points to a firmware page -/// ``` -/// -/// Each page is 4KB, each entry is 8 bytes (64-bit DMA address). -/// Also known as "Radix3" firmware. +/// A radix3 page table and the data it maps. #[pin_data] pub(crate) struct Radix3<'a> { - /// The GSP firmware inside a [`VVec`], device-mapped via a SG table. + /// The mapped data. #[pin] data: SGTable>>, - /// Level 2 page table whose entries contain DMA addresses of firmware pages. + /// Level 2: one entry per page of `data`. #[pin] level2: SGTable>>, - /// Level 1 page table whose entries contain DMA addresses of level 2 pages. + /// Level 1: one entry per page of `level2`. #[pin] level1: SGTable>>, - /// Level 0 page table (single 4KB page) with one entry: DMA address of first level 1 page. + /// Level 0: one page, whose single entry is the DMA address of the first `level1` page. level0: Coherent<'a, [u64]>, - /// Size in bytes of the firmware contained in [`Self::data`]. + /// Length of `data`, in bytes. size: usize, } @@ -108,7 +106,7 @@ pub(crate) fn new( }) } - /// Returns the DMA address of the radix3 level 0 page table. + /// Returns the DMA address of the level 0 page, which is the address of the table. pub(crate) fn dma_address(&self) -> DmaAddress { self.level0.dma_address() } @@ -119,14 +117,10 @@ pub(crate) fn size(&self) -> usize { } } -/// Build a page table from a scatter-gather list. -/// -/// Takes each DMA-mapped region from `sg_table` and writes page table entries -/// for all 4KB pages within that region. For example, a 16KB SG entry becomes -/// 4 consecutive page table entries. +/// Appends one level of the table to `dst`: one entry per [`GSP_PAGE_SIZE`] page of each +/// DMA-mapped region of `sg_table`, in region order. fn map_into_lvl(sg_table: &SGTable>>, mut dst: VVec) -> Result> { for sg_entry in sg_table.iter() { - // Number of pages we need to map. let num_pages = usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE); for i in 0..num_pages { From 12c70c819eeab04122b9df42a3fbdb3cbbc3177a Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Tue, 22 Sep 2026 20:12:21 +0900 Subject: [PATCH 063/110] gpu: nova-core: zero-pad radix3 page table levels to page boundary On Turing through Ada, the booter, a firmware stage that runs on the GPU's SEC2 falcon, copies the GSP firmware image from system memory into the framebuffer. It walks the radix3 table to find each page of the image, and it reads each level of the table by DMA, a whole 4 KiB page at a time. The last page of a level is only partly filled with entries, so the bytes past the last entry reach the booter too. Nova-core allocated each level at the size of its entries and wrote only the entries, so the rest of the level's last page held whatever the allocator left there, and the booter read that stale kernel memory. Allocate each level zeroed and sized to a whole number of 4 KiB pages, as Open RM does, and write the entries into it. Assisted-by: LLM Signed-off-by: John Hubbard Reviewed-by: Timur Tabi Link: https://patch.msgid.link/20260913195413.742143-3-jhubbard@nvidia.com [acourbot: move chunks from previous commit into this one for consistency] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/firmware/radix3.rs | 54 ++++++++++++++---------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/nova-core/firmware/radix3.rs b/drivers/gpu/nova-core/firmware/radix3.rs index ca47c5d66978..94235bd1209d 100644 --- a/drivers/gpu/nova-core/firmware/radix3.rs +++ b/drivers/gpu/nova-core/firmware/radix3.rs @@ -61,26 +61,12 @@ pub(crate) fn new( Ok(try_pin_init!(Self { data <- SGTable::new(dev, data, DataDirection::ToDevice, GFP_KERNEL), level2 <- { - // Allocate the level 2 page table, map the firmware onto it, and map it into - // the device address space. - VVec::::with_capacity( - data.iter().count() * core::mem::size_of::(), - GFP_KERNEL, - ) - .map_err(|_| ENOMEM) - .and_then(|level2| map_into_lvl(&data, level2)) - .map(|level2| SGTable::new(dev, level2, DataDirection::ToDevice, GFP_KERNEL))? + build_lvl(&data) + .map(|l2| SGTable::new(dev, l2, DataDirection::ToDevice, GFP_KERNEL))? }, level1 <- { - // Allocate the level 1 page table, map the level 2 page table onto it, and map - // it into the device address space. - VVec::::with_capacity( - level2.iter().count() * core::mem::size_of::(), - GFP_KERNEL, - ) - .map_err(|_| ENOMEM) - .and_then(|level1| map_into_lvl(&level2, level1)) - .map(|level1| SGTable::new(dev, level1, DataDirection::ToDevice, GFP_KERNEL))? + build_lvl(&level2) + .map(|l1| SGTable::new(dev, l1, DataDirection::ToDevice, GFP_KERNEL))? }, level0: { // Allocate the level 0 page table as a device-visible DMA object, and map the @@ -117,16 +103,40 @@ pub(crate) fn size(&self) -> usize { } } -/// Appends one level of the table to `dst`: one entry per [`GSP_PAGE_SIZE`] page of each -/// DMA-mapped region of `sg_table`, in region order. -fn map_into_lvl(sg_table: &SGTable>>, mut dst: VVec) -> Result> { +/// Returns the size of the level that maps `sg_table`: one `u64` entry per [`GSP_PAGE_SIZE`] +/// page of each DMA-mapped region, rounded up to whole pages. +fn lvl_size(sg_table: &SGTable>>) -> usize { + let entries: usize = sg_table + .iter() + .map(|sg_entry| usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE)) + .sum(); + + (entries * size_of::()).next_multiple_of(GSP_PAGE_SIZE) +} + +/// Builds one level of the table over `sg_table`: one entry per [`GSP_PAGE_SIZE`] page of each +/// DMA-mapped region, in region order. The level is a whole number of pages and every byte past +/// the last entry is zero, because the booter reads a level a whole page at a time. +/// +/// # Errors +/// +/// - `ENOMEM` if the level cannot be allocated. +/// - `EINVAL` if `sg_table` yields more entries than [`lvl_size`] sized the level for. +fn build_lvl(sg_table: &SGTable>>) -> Result> { + let mut dst = VVec::::zeroed(lvl_size(sg_table), GFP_KERNEL).map_err(|_| ENOMEM)?; + let mut entries = dst.chunks_exact_mut(size_of::()); + for sg_entry in sg_table.iter() { let num_pages = usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE); for i in 0..num_pages { let entry = sg_entry.dma_address() + (u64::from_safe_cast(i) * u64::from_safe_cast(GSP_PAGE_SIZE)); - dst.extend_from_slice(&entry.to_le_bytes(), GFP_KERNEL)?; + + entries + .next() + .ok_or(EINVAL)? + .copy_from_slice(&entry.to_le_bytes()); } } From f9836eb2cf53fb6d8d707f40f873b9e4b1513b40 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Sun, 13 Sep 2026 12:54:13 -0700 Subject: [PATCH 064/110] gpu: nova-core: rename the FbRanges elf field to fw_image A TLV firmware file carries the GSP firmware image as a section extracted from the original ELF, and the section is not itself an ELF. Before the driver moved to TLV firmware files, it parsed that ELF itself, and it named the framebuffer range that holds the image for the ELF. The range kept the name through the move. Rename the field for the framebuffer range that holds the GSP firmware image, from elf to fw_image. After this, the following: $ cd drivers/gpu/nova-core $ grep -iw ELF comes up empty. :) Assisted-by: LLM Signed-off-by: John Hubbard Reviewed-by: Timur Tabi Link: https://patch.msgid.link/20260913195413.742143-4-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fb.rs | 18 +++++++++--------- drivers/gpu/nova-core/gsp/fw.rs | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index b18dc5447a73..b0c0b8fe6008 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -159,7 +159,7 @@ pub(crate) struct FbRanges { /// Memory area containing the GSP bootloader image. pub(crate) boot: FbRange, /// Memory area containing the GSP firmware image. - pub(crate) elf: FbRange, + pub(crate) fw_image: FbRange, /// WPR2 heap. pub(crate) wpr2_heap: FbRange, /// WPR2 region range, starting with an instance of `GspFwWprMeta`. @@ -234,25 +234,25 @@ pub(crate) fn new( FbRange(bootloader_base..bootloader_base + bootloader_size) }; - let elf = { - const ELF_DOWN_ALIGN: Alignment = Alignment::new::(); - let elf_size = u64::from_safe_cast(gsp_fw.size()); - let elf_addr = (boot.start - elf_size).align_down(ELF_DOWN_ALIGN); + let fw_image = { + const FW_IMAGE_DOWN_ALIGN: Alignment = Alignment::new::(); + let fw_image_size = u64::from_safe_cast(gsp_fw.size()); + let fw_image_addr = (boot.start - fw_image_size).align_down(FW_IMAGE_DOWN_ALIGN); - FbRange(elf_addr..elf_addr + elf_size) + FbRange(fw_image_addr..fw_image_addr + fw_image_size) }; let (vf_partition_count, wpr2_heap_size) = wpr2_heap_params(chipset, vgpu_state, fb.end)?; let wpr2_heap = { const WPR2_HEAP_DOWN_ALIGN: Alignment = Alignment::new::(); - let wpr2_heap_addr = elf + let wpr2_heap_addr = fw_image .start .checked_sub(wpr2_heap_size) .ok_or(EOVERFLOW)? .align_down(WPR2_HEAP_DOWN_ALIGN); - FbRange(wpr2_heap_addr..(elf.start).align_down(WPR2_HEAP_DOWN_ALIGN)) + FbRange(wpr2_heap_addr..(fw_image.start).align_down(WPR2_HEAP_DOWN_ALIGN)) }; let wpr2 = { @@ -273,7 +273,7 @@ pub(crate) fn new( vga_workspace, frts, boot, - elf, + fw_image, wpr2_heap, wpr2, non_wpr_heap, diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index ae98da42410b..918a7ae809eb 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -205,7 +205,7 @@ pub(crate) fn from_ranges<'a>( gspFwWprStart: ranges.wpr2.start, gspFwHeapOffset: ranges.wpr2_heap.start, gspFwHeapSize: ranges.wpr2_heap.len(), - gspFwOffset: ranges.elf.start, + gspFwOffset: ranges.fw_image.start, bootBinOffset: ranges.boot.start, frtsOffset: ranges.frts.start, frtsSize: ranges.frts.len(), From 6f0bdccfff6c84bd33f0385517c6f1f331a5ea04 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Thu, 17 Sep 2026 18:06:48 -0700 Subject: [PATCH 065/110] gpu: nova-core: set MCTP transport header version to 1 FSP, the GPU's Foundation Security Processor, receives the driver's messages as packets of MCTP, the Management Component Transport Protocol. It requires header version 1 in bits 3:0 of the MCTP transport header. Nova-core left those bits at zero, so every Chain of Trust request and every Product Reconfiguration Control request that it sent carried an invalid transport header. Declare the version field and set it. Declare the destination endpoint ID field as well. The driver leaves that field at zero. Assisted-by: LLM Signed-off-by: John Hubbard Reviewed-by: Timur Tabi Reviewed-by: Zhi Wang Link: https://patch.msgid.link/20260918010719.1176945-3-jhubbard@nvidia.com [acourbot: explicitly set deid field] Signed-off-by: Alexandre Courbot --- Documentation/gpu/nova/core/fsp.rst | 2 ++ drivers/gpu/nova-core/mctp.rs | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Documentation/gpu/nova/core/fsp.rst b/Documentation/gpu/nova/core/fsp.rst index 52d618d22bb8..60c647181da0 100644 --- a/Documentation/gpu/nova/core/fsp.rst +++ b/Documentation/gpu/nova/core/fsp.rst @@ -106,6 +106,8 @@ All FSP messages share a common header format consisting of two 32-bit words: - Bit 30: EOM (End of Message) - Bits 29:28: Packet sequence number - Bits 23:16: Source Endpoint ID +- Bits 15:8: Destination Endpoint ID +- Bits 3:0: MCTP header version (1) **NVDM header** (NVIDIA Vendor Defined Message): diff --git a/drivers/gpu/nova-core/mctp.rs b/drivers/gpu/nova-core/mctp.rs index 90c642c91a72..f62e79f56b05 100644 --- a/drivers/gpu/nova-core/mctp.rs +++ b/drivers/gpu/nova-core/mctp.rs @@ -42,13 +42,25 @@ pub(crate) struct MctpHeader(u32) { 29:28 seq; /// Source endpoint ID. 23:16 seid; + /// Destination endpoint ID. + 15:8 deid; + /// MCTP header version. + 3:0 version; } } impl MctpHeader { - /// Builds a single-packet MCTP header (`SOM=1`, `EOM=1`, `SEQ=0`, `SEID=0`). + /// The MCTP header version that this driver uses. + const VERSION: u32 = 1; + + /// Builds the MCTP header of a message that fits in one packet: `SOM` and `EOM` set, the + /// version set, and every other field zero. pub(crate) fn single_packet() -> Self { - Self::zeroed().with_som(true).with_eom(true) + Self::zeroed() + .with_const_version::<{ Self::VERSION }>() + .with_const_deid::<0>() + .with_som(true) + .with_eom(true) } /// Returns whether this is a complete single-packet message (`SOM=1` and `EOM=1`). From e1a9f41b896f0d57eb1165dd8a4c7446be6af638 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Thu, 17 Sep 2026 18:06:49 -0700 Subject: [PATCH 066/110] gpu: nova-core: gsp: give the command queue its own BAR0 mapping Sending a command to the GSP ends by writing the GSP doorbell, a BAR0 register, so every send path through the command queue needs the BAR0 mapping. A command too large for one queue element goes out as a first element followed by continuation records, so the send path has four functions. Nova-core passed the mapping down from the caller as an argument, which was threaded through all four functions, and every caller of a send had to carry it as well. Store the mapping in the command queue and drop the argument. The queue lives exactly as long as the mapping does, since both belong to the bound device. Suggested-by: Alexandre Courbot Assisted-by: LLM Signed-off-by: John Hubbard Reviewed-by: Timur Tabi Link: https://patch.msgid.link/20260918010719.1176945-4-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gpu.rs | 4 ++-- drivers/gpu/nova-core/gsp.rs | 11 +++++++---- drivers/gpu/nova-core/gsp/boot.rs | 10 +++------- drivers/gpu/nova-core/gsp/cmdq.rs | 24 ++++++++++++++---------- 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 3217e23caa4a..8f30186f10ba 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -405,7 +405,7 @@ pub(crate) fn new<'a>( vgpu: VgpuManager::new(pdev, spec.chipset, fsp.as_mut()), - gsp <- Gsp::new(pdev), + gsp <- Gsp::new(pdev, bar), // This member must be initialized last, so the `UnloadBundle` can never be dropped // from outside of the constructed `GspResources`, ensuring that the unload sequence @@ -423,7 +423,7 @@ pub(crate) fn new<'a>( gsp_static_info: { // Obtain and display basic GPU information. - let info = gsp_resources.gsp.get_static_info(bar)?; + let info = gsp_resources.gsp.get_static_info()?; match info.gpu_name() { Ok(name) => dev_info!(dev, "GPU name: {}\n", name), Err(e) => dev_warn!(dev, "GPU name unavailable: {:?}\n", e), diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index 25ea43f1cbe9..dda58095f40b 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -161,7 +161,10 @@ pub(crate) struct Gsp<'gsp> { impl<'gsp> Gsp<'gsp> { // Creates an in-place initializer for a `Gsp` manager for `pdev`. - pub(crate) fn new(pdev: &'gsp pci::Device) -> impl PinInit + 'gsp { + pub(crate) fn new( + pdev: &'gsp pci::Device, + bar: Bar0<'gsp>, + ) -> impl PinInit + 'gsp { pin_init::pin_init_scope(move || { let dev = pdev.as_ref(); @@ -173,7 +176,7 @@ pub(crate) fn new(pdev: &'gsp pci::Device) -> impl PinInit) -> impl PinInit) -> Result { - self.cmdq.send_command(bar, commands::GetGspStaticInfo) + pub(crate) fn get_static_info(&self) -> Result { + self.cmdq.send_command(commands::GetGspStaticInfo) } } diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 60bed3dc2f5a..4fb1b69ac9d5 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -10,7 +10,6 @@ }; use crate::{ - driver::Bar0, falcon::{ gsp::Gsp, Falcon, // @@ -36,7 +35,6 @@ pub(crate) fn boot( mut ctx: super::GspBootContext<'_, 'gsp>, ) -> Result>> { let pdev = ctx.pdev; - let bar = ctx.bar; let chipset = ctx.chipset; let gsp_falcon = ctx.gsp_falcon; let dev = pdev.as_ref(); @@ -45,9 +43,9 @@ pub(crate) fn boot( let gsp_fw = KBox::pin_init(GspFirmware::new(dev, chipset), GFP_KERNEL)?; self.cmdq - .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev, chipset))?; + .send_command_no_wait(commands::SetSystemInfo::new(pdev, chipset))?; self.cmdq - .send_command_no_wait(bar, commands::SetRegistry::new(ctx.vgpu.state())?)?; + .send_command_no_wait(commands::SetRegistry::new(ctx.vgpu.state())?)?; // Perform the chipset-specific boot sequence, and retrieve the unload bundle. let unload_bundle = hal.boot(&self, &mut ctx, &gsp_fw)?.or_else(|| { @@ -89,12 +87,11 @@ pub(crate) fn boot( /// Shut down the GSP and wait until it is offline. fn shutdown_gsp( cmdq: &Cmdq<'_>, - bar: Bar0<'_>, gsp_falcon: &Falcon<'_, Gsp>, mode: commands::PowerStateLevel, ) -> Result { // Command to shut the GSP down. - cmdq.send_command(bar, commands::UnloadingGuestDriver::new(mode))?; + cmdq.send_command(commands::UnloadingGuestDriver::new(mode))?; // Wait until GSP signals it is suspended. const LIBOS_INTERRUPT_PROCESSOR_SUSPENDED: u32 = bits::bit_u32(31); @@ -120,7 +117,6 @@ pub(crate) fn unload( // Shut down the GSP. Keep going even in case of error. let mut res = Self::shutdown_gsp( &self.cmdq, - ctx.bar, ctx.gsp_falcon, commands::PowerStateLevel::Level0, ) diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index 9f99e6bbb4fa..a5595da23407 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -522,6 +522,7 @@ impl<'cmdq> Cmdq<'cmdq> { /// Creates a new command queue for `dev`. pub(crate) fn new( dev: &'cmdq device::Device, + bar: Bar0<'cmdq>, ) -> impl PinInit + 'cmdq { pin_init_scope(move || { let gsp_mem = DmaGspMem::new(dev)?; @@ -530,6 +531,7 @@ pub(crate) fn new( dma_addr: gsp_mem.0.dma_address(), inner <- new_mutex!(CmdqInner { dev, + bar, gsp_mem, seq: 0, }), @@ -571,7 +573,7 @@ fn notify_gsp(bar: Bar0<'_>) { /// written to by its [`CommandToGsp::init_variable_payload`] method. /// /// Error codes returned by the command and reply initializers are propagated as-is. - pub(crate) fn send_command(&self, bar: Bar0<'_>, command: M) -> Result + pub(crate) fn send_command(&self, command: M) -> Result where M: CommandToGsp, M::Reply: MessageFromGsp, @@ -579,7 +581,7 @@ pub(crate) fn send_command(&self, bar: Bar0<'_>, command: M) -> Result::InitError>, { let mut inner = self.inner.lock(); - inner.send_command(bar, command)?; + inner.send_command(command)?; loop { match inner.receive_msg::(Self::RECEIVE_TIMEOUT) { @@ -599,12 +601,12 @@ pub(crate) fn send_command(&self, bar: Bar0<'_>, command: M) -> Result(&self, bar: Bar0<'_>, command: M) -> Result + pub(crate) fn send_command_no_wait(&self, command: M) -> Result where M: CommandToGsp, Error: From, { - self.inner.lock().send_command(bar, command) + self.inner.lock().send_command(command) } /// Receive a message from the GSP. @@ -623,6 +625,8 @@ pub(crate) fn receive_msg(&self, timeout: Delta) -> Result struct CmdqInner<'a> { /// Device this command queue belongs to. dev: &'a device::Device, + /// MMIO mapping of PCI BAR0, for writing the GSP doorbell. + bar: Bar0<'a>, /// Current command sequence number. seq: u32, /// Memory area shared with the GSP for communicating commands and messages. @@ -643,7 +647,7 @@ impl CmdqInner<'_> { /// written to by its [`CommandToGsp::init_variable_payload`] method. /// /// Error codes returned by the command initializers are propagated as-is. - fn send_single_command(&mut self, bar: Bar0<'_>, command: M) -> Result + fn send_single_command(&mut self, command: M) -> Result where M: CommandToGsp, // This allows all error types, including `Infallible`, to be used for `M::InitError`. @@ -697,7 +701,7 @@ fn send_single_command(&mut self, bar: Bar0<'_>, command: M) -> Result let elem_count = dst.header.element_count(); self.seq += 1; self.gsp_mem.advance_cpu_write_ptr(elem_count); - Cmdq::notify_gsp(bar); + Cmdq::notify_gsp(self.bar); Ok(()) } @@ -713,19 +717,19 @@ fn send_single_command(&mut self, bar: Bar0<'_>, command: M) -> Result /// written to by its [`CommandToGsp::init_variable_payload`] method. /// /// Error codes returned by the command initializers are propagated as-is. - fn send_command(&mut self, bar: Bar0<'_>, command: M) -> Result + fn send_command(&mut self, command: M) -> Result where M: CommandToGsp, Error: From, { match SplitState::new(command)? { - SplitState::Single(command) => self.send_single_command(bar, command), + SplitState::Single(command) => self.send_single_command(command), SplitState::Split(command, mut continuations) => { - self.send_single_command(bar, command)?; + self.send_single_command(command)?; while let Some(continuation) = continuations.next() { // Turbofish needed because the compiler cannot infer M here. - self.send_single_command::>(bar, continuation)?; + self.send_single_command::>(continuation)?; } Ok(()) From 4515a693faf557b9d48e9d2b7dbd28b114a06869 Mon Sep 17 00:00:00 2001 From: Deborah Brouwer Date: Tue, 22 Sep 2026 16:16:08 -0700 Subject: [PATCH 067/110] drm/tyr: gpu_info: zero out new/unused fields If a new field is added to the struct drm_panthor_gpu_info in the panthor uapi, Tyr must initialize this field or it will cause a compile error. To pre-emptively avoid compile errors, use ..pin_init::zeroed() to zero out all new fields added to this struct. Tyr can subsequently change the initialization as necessary. Also use it instead of manually zeroing out existing fields at the end of the struct. Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260922-b4-gpu_info_zero-v1-1-7259840069c6@collabora.com Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/gpu.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/tyr/gpu.rs b/drivers/gpu/drm/tyr/gpu.rs index 592b8bb16eba..e1b04a0c5159 100644 --- a/drivers/gpu/drm/tyr/gpu.rs +++ b/drivers/gpu/drm/tyr/gpu.rs @@ -74,10 +74,8 @@ pub(crate) fn new(io: &IoMem<'_>) -> Self { io.read(TILER_PRESENT_HI).into_raw(), ), core_features: io.read(CORE_FEATURES).into_raw(), - // Padding must be zero. - pad: 0, - //GPU_FEATURES register is not available; it was introduced in arch 11.x. - gpu_features: 0, + // Zero unsupported and newly added UAPI fields. + ..pin_init::zeroed() }) } From 370153694442fee19ed43726e698acd65df334dc Mon Sep 17 00:00:00 2001 From: Deborah Brouwer Date: Thu, 10 Sep 2026 16:20:04 -0700 Subject: [PATCH 068/110] MAINTAINERS: add rust-for-linux list for Tyr Tyr is a Rust DRM driver and Tyr patches are reviewed on the rust-for-linux mailing list. Add the list to the Tyr MAINTAINERS entry so that scripts/get_maintainer.pl and b4 prep --auto-to-cc include it. Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260910-b4-add-list-for-tyr-v1-1-cd74ff2c1f8a@collabora.com Signed-off-by: Alice Ryhl --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 2389df80ba50..04709f381d04 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2333,6 +2333,7 @@ ARM MALI TYR DRM DRIVER M: Daniel Almeida M: Alice Ryhl L: dri-devel@lists.freedesktop.org +L: rust-for-linux@vger.kernel.org S: Supported W: https://rust-for-linux.com/tyr-gpu-driver W: https://drm.pages.freedesktop.org/maintainer-tools/drm-rust.html From 896ed083362758b33c49a1b5e5a3423c5814d87e Mon Sep 17 00:00:00 2001 From: Deborah Brouwer Date: Thu, 10 Sep 2026 16:01:36 -0700 Subject: [PATCH 069/110] drm/tyr: reduce size of doorbell array MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently the number of doorbells is set to 64 which is an architectural maximum for the Mali GPU major v10 supported by Tyr. Since each doorbell needs a 64 KiB page, the doorbell array alone would need 4 MiB of register memory. Since the doorbell array starts at offset 0x80000, its exclusive end would be 0x480000, but the rk3588 device that Tyr is compatible with only provides a 2 MiB iomem region in total. Reduce the number of doorbells so that they fit into the rk3588 iomem region. This is consistent with Panthor’s doorbell id assignment. Doorbell 0 is the global doorbell. Panthor assigns the remaining doorbells to the active command stream group (CSG) slots. The number of slots is capped at 16 (MAX_CSG_PRIO + 1). Therefore Panthor uses a maximum of 17 doorbells. The 17 doorbell pages span 0x110000 bytes. Since the doorbell array starts at offset 0x80000 its exclusive end is 0x190000 and so fits within the 2 MiB register region. Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260910-b4-doorbell-v1-1-f3ca2f501600@collabora.com Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/regs.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/tyr/regs.rs b/drivers/gpu/drm/tyr/regs.rs index 0c419c4e1186..68701c39aee2 100644 --- a/drivers/gpu/drm/tyr/regs.rs +++ b/drivers/gpu/drm/tyr/regs.rs @@ -1802,7 +1802,10 @@ pub(crate) mod doorbell_block { use crate::driver::TyrRegisters; /// Number of doorbells available. - pub(crate) const NUM_DOORBELLS: usize = 64; + /// The architectural maximum is 64 but this full array does not + /// fit in the 2 MiB iomem region. Panthor currently uses a maximum of + /// 17 doorbells so do the same here. + pub(crate) const NUM_DOORBELLS: usize = 17; /// Doorbell block stride (64KiB). /// From c0c5d21b466c57d7f3dc28269c2c13a1c965d54c Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sat, 26 Sep 2026 14:43:24 +0200 Subject: [PATCH 070/110] gpu: nova-core: align Bar1 alias with Bar0 Make Bar1 an alias for a borrowed pci::Bar in order to match the Bar0 type alias. Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260926124405.604707-2-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/driver.rs | 4 ++-- drivers/gpu/nova-core/gpu.rs | 2 +- drivers/gpu/nova-core/mm/bar_user.rs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index 0672a0707a71..9ce70703b2ce 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -32,7 +32,7 @@ pub(crate) struct NovaCore<'bound> { #[pin] pub(crate) gpu: Gpu<'bound>, bar: pci::Bar<'bound, BAR0_SIZE>, - bar1: Bar1<'bound>, + bar1: pci::Bar<'bound>, #[allow(clippy::type_complexity)] _reg: auxiliary::Registration<'bound, CovariantForLt!(())>, } @@ -43,7 +43,7 @@ pub(crate) struct NovaCore<'bound> { pub(crate) type Bar0<'a> = &'a pci::Bar<'a, BAR0_SIZE>; pub(crate) type NovaRegisters = kernel::io::Region; -pub(crate) type Bar1<'a> = pci::Bar<'a>; +pub(crate) type Bar1<'a> = &'a pci::Bar<'a>; /// Returns the Linux PCI resource index that holds BAR1 for an NVIDIA GPU. /// diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 8f30186f10ba..1ae287c9e776 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -353,7 +353,7 @@ impl<'gpu> Gpu<'gpu> { pub(crate) fn new<'a>( pdev: &'gpu pci::Device>, bar: Bar0<'gpu>, - bar1: &'gpu Bar1<'gpu>, + bar1: Bar1<'gpu>, ) -> impl PinInit + use<'gpu, 'a> { let dev = pdev.as_ref(); diff --git a/drivers/gpu/nova-core/mm/bar_user.rs b/drivers/gpu/nova-core/mm/bar_user.rs index 8f4a27c1fd14..86b5d302188e 100644 --- a/drivers/gpu/nova-core/mm/bar_user.rs +++ b/drivers/gpu/nova-core/mm/bar_user.rs @@ -41,7 +41,7 @@ pub(crate) struct BarUser<'gpu> { #[pin] vmm: Mutex, - bar1: &'gpu Bar1<'gpu>, + bar1: Bar1<'gpu>, } impl<'gpu> BarUser<'gpu> { @@ -50,7 +50,7 @@ pub(crate) fn new( pdb_addr: VramAddress, chipset: Chipset, va_size: u64, - bar1: &'gpu Bar1<'gpu>, + bar1: Bar1<'gpu>, ) -> Result + 'gpu> { let vmm = Vmm::new(pdb_addr, chipset.mmu_version(), va_size)?; Ok(pin_init!(Self { From 2246188075757345be21291d2e130814b9e51768 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sat, 26 Sep 2026 14:43:25 +0200 Subject: [PATCH 071/110] gpu: nova-core: remove unnecessary Arc around BarUser Remove the unnecessary Arc around BarUser and get rid of Result> in favor of impl PinInit. Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260926124405.604707-3-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/gpu.rs | 24 +++++++++++------------- drivers/gpu/nova-core/mm.rs | 5 ++--- drivers/gpu/nova-core/mm/bar_user.rs | 16 +++++++++------- 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 1ae287c9e776..123e605fa2c2 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -16,7 +16,6 @@ SizeConstants, SZ_4K, // }, - sync::Arc, }; use crate::{ @@ -309,7 +308,8 @@ pub(crate) struct Gpu<'gpu> { /// the GSP is still operational. mm: GpuMm<'gpu>, /// BAR1 user interface for CPU access to GPU virtual memory. - bar_user: Arc>, + #[pin] + bar_user: BarUser<'gpu>, /// GSP and its resources. #[pin] gsp_resources: GspResources<'gpu>, @@ -465,19 +465,17 @@ pub(crate) fn new<'a>( }, // Create BAR1 user interface for CPU access to GPU virtual memory. - bar_user: { + bar_user <- { let pdb_addr = VramAddress::from_raw(gsp_static_info.bar1_pde_base); let bar1_idx = crate::driver::bar1_resource_index(pdev)?; let bar1_size = pdev.resource_len(bar1_idx)?; - Arc::pin_init( - BarUser::new( - pdb_addr, - gsp_resources.spec.chipset, - bar1_size, - bar1, - )?, - GFP_KERNEL, - )? + + BarUser::new( + pdb_addr, + gsp_resources.spec.chipset, + bar1_size, + bar1, + ) }, }) } @@ -493,7 +491,7 @@ pub(crate) fn run_selftests(self: Pin<&mut Self>, pdev: &pci::Device, mm: &mut GpuMm<'_>, usable_fb_regions: &[Range], - bar_user: &Arc>, + bar_user: &bar_user::BarUser<'_>, bar1_pdb: u64, chipset: Chipset, ) -> Result { diff --git a/drivers/gpu/nova-core/mm/bar_user.rs b/drivers/gpu/nova-core/mm/bar_user.rs index 86b5d302188e..6c8fa06d9f4b 100644 --- a/drivers/gpu/nova-core/mm/bar_user.rs +++ b/drivers/gpu/nova-core/mm/bar_user.rs @@ -51,12 +51,14 @@ pub(crate) fn new( chipset: Chipset, va_size: u64, bar1: Bar1<'gpu>, - ) -> Result + 'gpu> { - let vmm = Vmm::new(pdb_addr, chipset.mmu_version(), va_size)?; - Ok(pin_init!(Self { - vmm <- new_mutex!(vmm, "bar_user_vmm"), + ) -> impl PinInit + 'gpu { + try_pin_init!(Self { + vmm <- new_mutex!( + Vmm::new(pdb_addr, chipset.mmu_version(), va_size)?, + "bar_user_vmm", + ), bar1, - })) + }) } /// Map physical pages to a contiguous BAR1 virtual range. @@ -183,7 +185,7 @@ fn drop(&mut self) { pub(crate) fn run_self_test( dev: &device::Device, mm: &mut GpuMm<'_>, - bar_user: &Arc>, + bar_user: &BarUser<'_>, bar1_pdb: u64, chipset: Chipset, ) -> Result { @@ -389,7 +391,7 @@ pub(crate) fn run_self_test( // Test 4: Exercise `BarUser::map()` end-to-end. let bar_user = Arc::pin_init( - BarUser::new(pdb_addr, chipset, SZ_64K.into_safe_cast(), bar1)?, + BarUser::new(pdb_addr, chipset, SZ_64K.into_safe_cast(), bar1), GFP_KERNEL, )?; let access = bar_user.map(mm, &[test_pfn], true)?; From 05e647ea406967ee18ebc075cde67f0ba368917c Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sat, 26 Sep 2026 14:43:26 +0200 Subject: [PATCH 072/110] gpu: nova-core: mm: get rid of Arc in BarUserAccess There is no need for a reference count here. BarUserAccess can just borrow from BarUser. Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260926124405.604707-4-dakr@kernel.org [ Update stale comment in run_self_test(). - Danilo ] Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/mm/bar_user.rs | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/nova-core/mm/bar_user.rs b/drivers/gpu/nova-core/mm/bar_user.rs index 6c8fa06d9f4b..56caa6b26c41 100644 --- a/drivers/gpu/nova-core/mm/bar_user.rs +++ b/drivers/gpu/nova-core/mm/bar_user.rs @@ -7,10 +7,7 @@ io::Io, new_mutex, prelude::*, - sync::{ - Arc, - Mutex, // - }, + sync::Mutex, // }; use crate::{ @@ -63,11 +60,11 @@ pub(crate) fn new( /// Map physical pages to a contiguous BAR1 virtual range. pub(crate) fn map( - self: &Arc, + &self, mm: &mut GpuMm<'_>, pfns: &[Pfn], writable: bool, - ) -> Result> { + ) -> Result> { if pfns.is_empty() { return Err(EINVAL); } @@ -75,15 +72,17 @@ pub(crate) fn map( let mapped = vmm.map_pages(mm, pfns, None, writable)?; Ok(BarUserAccess { - bar_user: self.clone(), + bar_user: self, mapped: Some(mapped), }) } } /// Access object for a mapped BAR1 region. -pub(crate) struct BarUserAccess<'gpu> { - bar_user: Arc>, +/// +/// Borrows the [`BarUser`] managing the mapping for the lifetime of the access object. +pub(crate) struct BarUserAccess<'a> { + bar_user: &'a BarUser<'a>, /// [`BarUserAccess::release`] [`Option::take`]s this; `Some` at /// drop time means `release()` was never called. mapped: Option, @@ -385,15 +384,10 @@ pub(crate) fn run_self_test( test3_passed = false; } - // Release Tests 1-3's Vmm before Test 4 constructs a fresh BarUser on - // the same PDB. + // Release Tests 1-3's Vmm before Test 4 uses the GPU's BarUser on the same PDB. drop(vmm); // Test 4: Exercise `BarUser::map()` end-to-end. - let bar_user = Arc::pin_init( - BarUser::new(pdb_addr, chipset, SZ_64K.into_safe_cast(), bar1), - GFP_KERNEL, - )?; let access = bar_user.map(mm, &[test_pfn], true)?; // Write pattern via PRAMIN, read via BarUserAccess. From bf91bd56fc46b5a656f1c6be629787e05965ef82 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sat, 26 Sep 2026 14:43:27 +0200 Subject: [PATCH 073/110] gpu: nova-core: mm: fix lifetime name in BarUser BarUser should not use 'gpu as a name for its lifetime; stick to a generic 'a until we need pin-init selfref for this struct. Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260926124405.604707-5-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/mm/bar_user.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/nova-core/mm/bar_user.rs b/drivers/gpu/nova-core/mm/bar_user.rs index 56caa6b26c41..6e9384f06f43 100644 --- a/drivers/gpu/nova-core/mm/bar_user.rs +++ b/drivers/gpu/nova-core/mm/bar_user.rs @@ -35,20 +35,20 @@ /// /// Owns the [`Vmm`] for the BAR1 address space. #[pin_data] -pub(crate) struct BarUser<'gpu> { +pub(crate) struct BarUser<'a> { #[pin] vmm: Mutex, - bar1: Bar1<'gpu>, + bar1: Bar1<'a>, } -impl<'gpu> BarUser<'gpu> { +impl<'a> BarUser<'a> { /// Create a pin-initializer for [`BarUser`]. pub(crate) fn new( pdb_addr: VramAddress, chipset: Chipset, va_size: u64, - bar1: Bar1<'gpu>, - ) -> impl PinInit + 'gpu { + bar1: Bar1<'a>, + ) -> impl PinInit + 'a { try_pin_init!(Self { vmm <- new_mutex!( Vmm::new(pdb_addr, chipset.mmu_version(), va_size)?, From 4075acc9d489cc6aaa37e3f4930943f32a60e7e4 Mon Sep 17 00:00:00 2001 From: Alistair Popple Date: Wed, 9 Sep 2026 16:44:54 +1000 Subject: [PATCH 074/110] rust: auxiliary: let registration_data_with() closures return covariant sub-fields The closure passed to registration_data_with() currently receives `Pin<&'a F::Of<'a>>` with `'a` universally quantified. This prevents the closure from returning references derived from the registration data, even for sub-fields that are covariant in their lifetime, because the compiler cannot relate `'a` to any lifetime the caller knows about. Tie the outer reference to the `&self` lifetime instead, i.e. pass `Pin<&'this F::Of<'a>>`. This gives the closure the implied bound `'a: 'this`, so covariant sub-fields such as `&'a T` can be coerced to `'this` and returned directly, while invariant fields still cannot be coerced and therefore cannot escape with an incorrect lifetime. This allows auxiliary child drivers to project covariant data out of invariant registration data without having to wrap every use in a closure. Link: https://lore.kernel.org/nova-gpu/DL3WPTVM033J.33RWYCZOC67Z1@kernel.org/ Suggested-by: Gary Guo Signed-off-by: Alistair Popple Link: https://patch.msgid.link/20260909064506.910162-2-apopple@nvidia.com Co-developed-by: Danilo Krummrich Signed-off-by: Danilo Krummrich --- rust/kernel/auxiliary.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs index 60dfbec8f330..06f816420790 100644 --- a/rust/kernel/auxiliary.rs +++ b/rust/kernel/auxiliary.rs @@ -305,6 +305,10 @@ unsafe fn registration_data_pinned(&self) -> Result(&self) -> Result( - &self, - f: impl for<'a> FnOnce(Pin<&'a F::Of<'a>>) -> R, + pub fn registration_data_with<'this, F: ForLt + 'static, R>( + &'this self, + f: impl for<'a> FnOnce(Pin<&'this F::Of<'a>>) -> R, ) -> Result { - // SAFETY: The HRTB closure prevents the caller from smuggling in references with a - // concrete short lifetime, making the round-trip from `'static` sound regardless of - // variance. + // SAFETY: The HRTB on the inner type prevents the caller from exploiting a specific + // choice of `'a`. Covariant sub-fields can be safely coerced to `'this`, while + // invariant fields cannot be coerced and thus cannot escape with an incorrect + // lifetime. let pinned = unsafe { self.registration_data_pinned::()? }; Ok(f(pinned)) From 6e70b1719991fb5b51306dec3bdba1e7dbfe838c Mon Sep 17 00:00:00 2001 From: Alistair Popple Date: Wed, 9 Sep 2026 16:44:55 +1000 Subject: [PATCH 075/110] gpu: nova-core: Add public driver API to nova-core Nova core will be used to export core functionality to other drivers which will bind to it via auxiliary bus devices. Add a NovaCoreApi type which drivers can use to call nova-core methods. Auxiliary bus drivers obtain a NovaCoreApiHandle for a particular GPU using NovaCoreApi::of(). This takes a reference to a bound auxiliary bus device and checks that its registration data is a NovaCoreApi. The NovaCoreApi itself is then accessed by passing a closure to NovaCoreApiHandle::with(). Closure based access is used because NovaCoreApi cannot be assumed to be covariant over its lifetime. It references the Gpu, which will carry locks around types holding device resources, making it invariant. Covariant sub-fields can still be returned directly from the closure thanks to the lifetime bound on registration_data_with(). Signed-off-by: Alistair Popple Link: https://patch.msgid.link/20260909064506.910162-3-apopple@nvidia.com [ Rebase and use pin_chain() for Gpu::run_selftests(). - Danilo ] Co-developed-by: Danilo Krummrich Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/api.rs | 48 ++++++++++++++++++++++++++ drivers/gpu/nova-core/driver.rs | 55 +++++++++++++++++++++--------- drivers/gpu/nova-core/gsp/hal.rs | 2 +- drivers/gpu/nova-core/nova_core.rs | 1 + 4 files changed, 88 insertions(+), 18 deletions(-) create mode 100644 drivers/gpu/nova-core/api.rs diff --git a/drivers/gpu/nova-core/api.rs b/drivers/gpu/nova-core/api.rs new file mode 100644 index 000000000000..ae023242594e --- /dev/null +++ b/drivers/gpu/nova-core/api.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Nova-core auxbus data. Contains all the methods used by the auxbus drivers +//! to interact with nova-core. + +use core::pin::Pin; + +use kernel::{ + auxiliary, + device::Bound, + prelude::*, + types::ForLt, // +}; + +use crate::gpu::Gpu; + +/// API handle for the auxiliary bus child drivers to interact with nova-core. +pub struct NovaCoreApi<'bound> { + #[expect(unused)] + pub(crate) gpu: Pin<&'bound Gpu<'bound>>, +} + +impl NovaCoreApi<'_> { + /// Obtain a [`NovaCoreApi`] handle from an auxiliary device registered + /// by nova-core. + pub fn of(adev: &auxiliary::Device) -> Result> { + NovaCoreApiHandle::of(adev) + } +} + +/// Expose a handle to nova-core API +pub struct NovaCoreApiHandle<'a> { + adev: &'a auxiliary::Device, +} + +impl<'a> NovaCoreApiHandle<'a> { + fn of(adev: &'a auxiliary::Device) -> Result { + adev.registration_data_with::), ()>(|_| ())?; + Ok(Self { adev }) + } + + /// Access the [`NovaCoreApi`] through a closure. + pub fn with(&self, f: impl for<'b> FnOnce(Pin<&'a NovaCoreApi<'b>>) -> R) -> R { + self.adev + .registration_data_with::), R>(f) + .expect("TypeId was validated in NovaCoreApiHandle::of()") + } +} diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index 9ce70703b2ce..d2f20943142d 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -19,22 +19,25 @@ Atomic, Relaxed, // }, - types::CovariantForLt, + types::ForLt, }; -use crate::gpu::Gpu; +use crate::{ + api::NovaCoreApi, + gpu::Gpu, // +}; /// Counter for generating unique auxiliary device IDs. static AUXILIARY_ID_COUNTER: Atomic = Atomic::new(0); #[pin_data] pub(crate) struct NovaCore<'bound> { + #[allow(clippy::type_complexity)] + _reg: auxiliary::Registration<'bound, ForLt!(NovaCoreApi<'_>)>, #[pin] pub(crate) gpu: Gpu<'bound>, bar: pci::Bar<'bound, BAR0_SIZE>, bar1: pci::Bar<'bound>, - #[allow(clippy::type_complexity)] - _reg: auxiliary::Registration<'bound, CovariantForLt!(())>, } pub(crate) struct NovaCoreDriver; @@ -116,19 +119,37 @@ fn probe<'bound>( unsafe { &*core::ptr::from_ref(bar) }, // SAFETY: `bar1` is initialized above, pinned, and outlives `gpu`. unsafe { &*core::ptr::from_ref(bar1) }, - ), - // Run optional GPU selftests. - #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] - _: { gpu.run_selftests(pdev) }, - _reg: auxiliary::Registration::new( - pdev.as_ref(), - c"nova-drm", - // TODO[XARR]: Use XArray or perhaps IDA for proper ID allocation/recycling. For - // now, use a simple atomic counter that never recycles IDs. - AUXILIARY_ID_COUNTER.fetch_add(1, Relaxed), - crate::MODULE_NAME, - (), - )?, + ).pin_chain(|_gpu| { + #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] + _gpu.run_selftests(pdev); + Ok(()) + }), + _reg: { + // TODO: Use `&gpu` self-referential pin-init syntax once available. + // + // SAFETY: `gpu` is initialized before this expression is evaluated + // (`try_pin_init!()` initializes fields in initializer order), lives at + // a pinned stable address, and is dropped after `_reg` (struct field + // drop order). + let gpu = unsafe { + Pin::new_unchecked(&*core::ptr::from_ref(gpu.as_ref().get_ref())) + }; + + // SAFETY: `NovaCore` is dropped when the device is unbound; + // i.e. `mem::forget()` is never called on it. + unsafe { + auxiliary::Registration::new_with_lt( + pdev.as_ref(), + c"nova-drm", + // TODO[XARR]: Use XArray or perhaps IDA for proper ID + // allocation/recycling. For now, use a simple atomic counter that + // never recycles IDs. + AUXILIARY_ID_COUNTER.fetch_add(1, Relaxed), + crate::MODULE_NAME, + NovaCoreApi { gpu }, + )? + } + }, })) }) } diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs index d8329f6fcc65..a0117d6d16a2 100644 --- a/drivers/gpu/nova-core/gsp/hal.rs +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -24,7 +24,7 @@ /// The GSP unload code might run in a situation where we cannot load firmware dynamically (e.g. /// because we are in shutdown and the file system is not accessible anymore). Thus, the firmware /// required for unloading is prepared at load time, and stored here until it needs to be run. -pub(super) trait UnloadBundle: Send { +pub(super) trait UnloadBundle: Send + Sync { /// Performs the steps required to properly reset the GSP after it has been stopped. fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result; } diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 1133c6ce5c55..4f8ce4e4c187 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -10,6 +10,7 @@ InPlaceModule, // }; +pub mod api; mod driver; mod falcon; mod fb; From ecedc604e0aa84838f67ee309a8db1076940e3c2 Mon Sep 17 00:00:00 2001 From: Alistair Popple Date: Wed, 9 Sep 2026 16:44:56 +1000 Subject: [PATCH 076/110] drm: nova: Add DRM registration data Currently the nova-drm stub driver doesn't really interact with a real device, so it doesn't have any registration data and has no way to call into nova-core. To allow for this add a DrmRegData type which will hold a NovaCoreApiHandle for the NovaCoreApi associated with the auxbus device. Signed-off-by: Alistair Popple Suggested-by: Danilo Krummrich Link: https://patch.msgid.link/20260909064506.910162-4-apopple@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/driver.rs | 18 ++++++++++++++++-- drivers/gpu/drm/nova/file.rs | 12 ++++++++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index 60752bccddc0..52550e694729 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -18,6 +18,11 @@ use crate::file::File; use crate::gem::NovaObject; +use nova_core::api::{ + NovaCoreApi, + NovaCoreApiHandle, // +}; + pub(crate) struct NovaDriver; pub(crate) struct Nova<'bound> { @@ -26,6 +31,12 @@ pub(crate) struct Nova<'bound> { _reg: drm::Registration<'bound, NovaDriver>, } +/// DRM registration data, accessible from ioctl handlers via the registration guard. +pub(crate) struct DrmRegData<'bound> { + #[expect(unused)] + pub(crate) api: NovaCoreApiHandle<'bound>, +} + /// Convienence type alias for the DRM device type for this driver pub(crate) type NovaDevice = drm::Device; @@ -59,9 +70,12 @@ fn probe<'bound>( _info: &'bound Self::IdInfo, ) -> impl PinInit, Error> + 'bound { let drm = drm::UnregisteredDevice::::new(adev, Ok(()))?; + let reg_data = DrmRegData { + api: NovaCoreApi::of(adev)?, + }; // SAFETY: `reg` is stored in `Nova` and dropped when the driver is unbound; it is // never forgotten. - let reg = unsafe { drm::Registration::new(adev.as_ref(), drm, (), 0)? }; + let reg = unsafe { drm::Registration::new(adev.as_ref(), drm, reg_data, 0)? }; Ok(Nova { drm: reg.device().into(), @@ -73,7 +87,7 @@ fn probe<'bound>( #[vtable] impl drm::Driver for NovaDriver { type Data = (); - type RegistrationData<'a> = (); + type RegistrationData<'a> = DrmRegData<'a>; type File = File; type Object = gem::Object; type ParentDevice = auxiliary::Device; diff --git a/drivers/gpu/drm/nova/file.rs b/drivers/gpu/drm/nova/file.rs index 298c02bacb4b..1156df51c533 100644 --- a/drivers/gpu/drm/nova/file.rs +++ b/drivers/gpu/drm/nova/file.rs @@ -1,6 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 -use crate::driver::{NovaDevice, NovaDriver}; +use crate::driver::{ + DrmRegData, + NovaDevice, + NovaDriver, // +}; use crate::gem::NovaObject; use kernel::{ alloc::flags::*, @@ -30,7 +34,7 @@ impl File { /// IOCTL: get_param: Query GPU / driver metadata. pub(crate) fn get_param( dev: &NovaDevice, - _reg_data: &(), + _reg_data: &DrmRegData<'_>, getparam: &mut uapi::drm_nova_getparam, _file: &drm::File, ) -> Result { @@ -50,7 +54,7 @@ pub(crate) fn get_param( /// IOCTL: gem_create: Create a new DRM GEM object. pub(crate) fn gem_create( dev: &NovaDevice, - _reg_data: &(), + _reg_data: &DrmRegData<'_>, req: &mut uapi::drm_nova_gem_create, file: &drm::File, ) -> Result { @@ -64,7 +68,7 @@ pub(crate) fn gem_create( /// IOCTL: gem_info: Query GEM metadata. pub(crate) fn gem_info( _dev: &NovaDevice, - _reg_data: &(), + _reg_data: &DrmRegData<'_>, req: &mut uapi::drm_nova_gem_info, file: &drm::File, ) -> Result { From 342aaa7bb3cc1066ae9055c936b667ecd66de0ec Mon Sep 17 00:00:00 2001 From: Alistair Popple Date: Wed, 9 Sep 2026 16:44:57 +1000 Subject: [PATCH 077/110] drm: nova: Add GPU architecture enum to nova-drm UAPI The GPU architecture to be exposed to user-space. This adds a public enum to the userspace headers for each chip architecture. Nova-core can then use this enum to define its architectures. This does create a coupling between nova-drm and nova-core whereby nova-core depends on the values defined by the user-space API for nova-drm. However this is entirely appropriate as nova-core must be bound by the UAPI headers as the enum values are read by nova-core and passed through to user-space. It also requires a minor change to the bounded_enum! macro to match the architecture values in an expression context. Signed-off-by: Alistair Popple Link: https://patch.msgid.link/20260909064506.910162-5-apopple@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/gpu.rs | 26 ++++++++++++++++---------- drivers/gpu/nova-core/num.rs | 2 +- include/uapi/drm/nova_drm.h | 12 ++++++++++++ 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 123e605fa2c2..e17fc6cc3440 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -16,6 +16,7 @@ SizeConstants, SZ_4K, // }, + uapi, // }; use crate::{ @@ -50,8 +51,9 @@ mod regs; macro_rules! define_chipset { - ({ $($variant:ident = $value:expr),* $(,)* }) => + ({ $($variant:ident = $value:literal),* $(,)* }) => { + ::kernel::macros::paste!( /// Enum representation of the GPU chipset. #[derive(fmt::Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq)] pub(crate) enum Chipset { @@ -63,7 +65,6 @@ impl Chipset { $( Chipset::$variant, )* ]; - ::kernel::macros::paste!( /// Returns the name of this chipset, in lowercase. /// /// # Examples @@ -79,7 +80,6 @@ pub(crate) const fn name(&self) -> &'static str { )* } } - ); } // TODO[FPRI]: replace with something like derive(FromPrimitive) @@ -88,11 +88,14 @@ impl TryFrom for Chipset { fn try_from(value: u32) -> Result { match value { - $( $value => Ok(Chipset::$variant), )* + $( + $value => Ok(Chipset::$variant), + )* _ => Err(ENODEV), } } } + ); } } @@ -177,13 +180,16 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { bounded_enum! { /// Enum representation of the GPU generation. #[derive(fmt::Debug, Copy, Clone)] + #[repr(u32)] pub(crate) enum Architecture with TryFrom> { - Turing = 0x16, - Ampere = 0x17, - Hopper = 0x18, - Ada = 0x19, - BlackwellGB10x = 0x1a, - BlackwellGB20x = 0x1b, + Turing = uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_TURING, + Ampere = uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_AMPERE, + Hopper = uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_HOPPER, + Ada = uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_ADA, + BlackwellGB10x = + uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_BLACKWELL_GB10X, + BlackwellGB20x = + uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_BLACKWELL_GB20X, } } diff --git a/drivers/gpu/nova-core/num.rs b/drivers/gpu/nova-core/num.rs index 6eb174d136ab..f4169235bc24 100644 --- a/drivers/gpu/nova-core/num.rs +++ b/drivers/gpu/nova-core/num.rs @@ -263,7 +263,7 @@ fn try_from( ) -> kernel::error::Result { match value.get() { $( - $value => Ok($enum_type::$variant), + value if value == $value => Ok($enum_type::$variant), )* _ => Err(kernel::error::code::EINVAL), } diff --git a/include/uapi/drm/nova_drm.h b/include/uapi/drm/nova_drm.h index 3ca90ed9d2bb..f0dcbca1908d 100644 --- a/include/uapi/drm/nova_drm.h +++ b/include/uapi/drm/nova_drm.h @@ -25,6 +25,18 @@ extern "C" { */ #define NOVA_GETPARAM_VRAM_BAR_SIZE 0x1 +/** + * enum drm_nova_architecture - GPU architecture identifier + */ +enum drm_nova_architecture { + NOVA_DRM_ARCHITECTURE_TURING = 0x16, + NOVA_DRM_ARCHITECTURE_AMPERE = 0x17, + NOVA_DRM_ARCHITECTURE_HOPPER = 0x18, + NOVA_DRM_ARCHITECTURE_ADA = 0x19, + NOVA_DRM_ARCHITECTURE_BLACKWELL_GB10X = 0x1a, + NOVA_DRM_ARCHITECTURE_BLACKWELL_GB20X = 0x1b, +}; + /** * struct drm_nova_getparam - query GPU and driver metadata */ From a1678ed1f8a344f1d2e4fc9362226555372d76d4 Mon Sep 17 00:00:00 2001 From: Alistair Popple Date: Wed, 9 Sep 2026 16:44:58 +1000 Subject: [PATCH 078/110] drm: nova: Add chipid enum to nova-drm UAPI The chip identifier uniquely identifies a GPU chip and needs to be exposed to user-space so it can look up chip specific properties that cannot be derived from the architecture alone. This adds a public enum to the userspace headers for each chip ID. Nova-core can then use this enum to define its chipsets. The values are opaque to user-space. They may be compared against the enum to identify a chip but user-space must not assume the values carry any particular meaning or encoding. This does create a coupling between nova-drm and nova-core whereby nova-core depends on the values defined by the user-space API for nova-drm. However this is entirely appropriate as nova-core must be bound by the UAPI headers as the enum values are read by nova-core and passed through to user-space. Note that nova-core currently refers to the chipid as a chipset. This isn't ideal given that chipid is the preferred and more accurate term and is what should be exposed via the uAPI. Therefore a future patch series will be posted that aligns nova-core internals to using chipid rather than chipset. Assisted-by: LLM Signed-off-by: Alistair Popple Link: https://patch.msgid.link/20260909064506.910162-6-apopple@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/gpu.rs | 57 +++++++++++++++++++----------------- include/uapi/drm/nova_drm.h | 40 +++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 27 deletions(-) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index e17fc6cc3440..a291d2d63f43 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -51,13 +51,14 @@ mod regs; macro_rules! define_chipset { - ({ $($variant:ident = $value:literal),* $(,)* }) => + ({ $($variant:ident),* $(,)* }) => { ::kernel::macros::paste!( /// Enum representation of the GPU chipset. #[derive(fmt::Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq)] + #[repr(u32)] pub(crate) enum Chipset { - $($variant = $value),*, + $($variant = uapi::[]),*, } impl Chipset { @@ -89,7 +90,9 @@ impl TryFrom for Chipset { fn try_from(value: u32) -> Result { match value { $( - $value => Ok(Chipset::$variant), + uapi::[] => { + Ok(Chipset::$variant) + } )* _ => Err(ENODEV), } @@ -101,35 +104,35 @@ fn try_from(value: u32) -> Result { define_chipset!({ // Turing - TU102 = 0x162, - TU104 = 0x164, - TU106 = 0x166, - TU117 = 0x167, - TU116 = 0x168, + TU102, + TU104, + TU106, + TU117, + TU116, // Ampere - GA100 = 0x170, - GA102 = 0x172, - GA103 = 0x173, - GA104 = 0x174, - GA106 = 0x176, - GA107 = 0x177, + GA100, + GA102, + GA103, + GA104, + GA106, + GA107, // Hopper - GH100 = 0x180, + GH100, // Ada - AD102 = 0x192, - AD103 = 0x193, - AD104 = 0x194, - AD106 = 0x196, - AD107 = 0x197, + AD102, + AD103, + AD104, + AD106, + AD107, // Blackwell GB10x - GB100 = 0x1a0, - GB102 = 0x1a2, + GB100, + GB102, // Blackwell GB20x - GB202 = 0x1b2, - GB203 = 0x1b3, - GB205 = 0x1b5, - GB206 = 0x1b6, - GB207 = 0x1b7, + GB202, + GB203, + GB205, + GB206, + GB207, }); impl Chipset { diff --git a/include/uapi/drm/nova_drm.h b/include/uapi/drm/nova_drm.h index f0dcbca1908d..e35f09a1ebe8 100644 --- a/include/uapi/drm/nova_drm.h +++ b/include/uapi/drm/nova_drm.h @@ -37,6 +37,46 @@ enum drm_nova_architecture { NOVA_DRM_ARCHITECTURE_BLACKWELL_GB20X = 0x1b, }; +/** + * enum drm_nova_chipid - opaque GPU chip identifier + * + * These values identify the chip a GPU is based on. They are otherwise + * opaque: userspace must not assume the values carry any particular meaning + * or encoding, only that they may be compared against this enum. + */ +enum drm_nova_chipid { + /* Turing */ + NOVA_DRM_CHIPID_TU102 = 0x162, + NOVA_DRM_CHIPID_TU104 = 0x164, + NOVA_DRM_CHIPID_TU106 = 0x166, + NOVA_DRM_CHIPID_TU117 = 0x167, + NOVA_DRM_CHIPID_TU116 = 0x168, + /* Ampere */ + NOVA_DRM_CHIPID_GA100 = 0x170, + NOVA_DRM_CHIPID_GA102 = 0x172, + NOVA_DRM_CHIPID_GA103 = 0x173, + NOVA_DRM_CHIPID_GA104 = 0x174, + NOVA_DRM_CHIPID_GA106 = 0x176, + NOVA_DRM_CHIPID_GA107 = 0x177, + /* Hopper */ + NOVA_DRM_CHIPID_GH100 = 0x180, + /* Ada */ + NOVA_DRM_CHIPID_AD102 = 0x192, + NOVA_DRM_CHIPID_AD103 = 0x193, + NOVA_DRM_CHIPID_AD104 = 0x194, + NOVA_DRM_CHIPID_AD106 = 0x196, + NOVA_DRM_CHIPID_AD107 = 0x197, + /* Blackwell GB10x */ + NOVA_DRM_CHIPID_GB100 = 0x1a0, + NOVA_DRM_CHIPID_GB102 = 0x1a2, + /* Blackwell GB20x */ + NOVA_DRM_CHIPID_GB202 = 0x1b2, + NOVA_DRM_CHIPID_GB203 = 0x1b3, + NOVA_DRM_CHIPID_GB205 = 0x1b5, + NOVA_DRM_CHIPID_GB206 = 0x1b6, + NOVA_DRM_CHIPID_GB207 = 0x1b7, +}; + /** * struct drm_nova_getparam - query GPU and driver metadata */ From b40f5ed85c5f162dfb61fe12d3c58a1d5420e3dc Mon Sep 17 00:00:00 2001 From: Alistair Popple Date: Wed, 9 Sep 2026 16:44:59 +1000 Subject: [PATCH 079/110] rust: uaccess: add UserSliceWriter::write_truncated() Add a helper that writes as much of an AsBytes value as fits in the remaining userspace buffer and returns the number of bytes copied. This avoids requiring callers of versioned UAPIs to convert values to byte slices and truncate them manually. Suggested-by: Danilo Krummrich Link: https://lore.kernel.org/nova-gpu/DKC6T1DQX2L3.HTHPB2L167TC@kernel.org/ Signed-off-by: Alistair Popple Acked-by: Miguel Ojeda Link: https://patch.msgid.link/20260909064506.910162-7-apopple@nvidia.com [ Inline write_slice_partial() and use it to simplify write_truncated(). - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/uaccess.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs index f09078228c53..f1aecda7a90c 100644 --- a/rust/kernel/uaccess.rs +++ b/rust/kernel/uaccess.rs @@ -558,6 +558,7 @@ pub fn write_dma( /// truncates the write to the boundaries of `self` and `data`. /// /// On success, returns the number of bytes written. + #[inline] pub fn write_slice_partial(&mut self, data: &[u8], offset: usize) -> Result { let end = offset.saturating_add(self.len()).min(data.len()); @@ -624,6 +625,19 @@ pub fn write(&mut self, value: &T) -> Result { self.length -= len; Ok(()) } + + /// Writes as much of the provided value as fits in the remaining buffer. + /// + /// Copies `min(size_of::(), self.len())` bytes to userspace. Returns the number of bytes + /// actually written. This is useful for versioned structs where an older userspace may provide + /// a smaller buffer than the current kernel struct. + /// + /// Fails with [`EFAULT`] if the write happens on a bad address. This call may modify the + /// associated userspace slice even if it returns an error. + #[inline] + pub fn write_truncated(&mut self, value: &T) -> Result { + self.write_slice_partial(value.as_bytes(), 0) + } } /// Reads a nul-terminated string into `dst` and returns the length. From ce6e59897ba550a63a4552e5a887e6eb121c2507 Mon Sep 17 00:00:00 2001 From: Alistair Popple Date: Wed, 9 Sep 2026 16:45:00 +1000 Subject: [PATCH 080/110] drm: nova: Add an info ioctl Add an extensible info ioctl and use it to report basic GPU information. One of the first things userspace needs to know about a GPU is its architecture and which chip it is, so add those as the first fields in the GPU info result. The chip identifier is opaque to userspace and is only meaningful when compared against the values of enum drm_nova_chipid. The ioctl selects an information type by ID and writes the result through a sized userspace buffer. The kernel truncates results to the supplied size, allowing information structures to grow and new logical information groups to be added without introducing new ioctls. Passing a NULL data pointer returns the size of the requested information structure instead. Suggested-by: Danilo Krummrich Signed-off-by: Alistair Popple Link: https://patch.msgid.link/20260909064506.910162-8-apopple@nvidia.com [ Avoid 'as' casts (add From for u32, From for u32 and use IntoSafeCast, IntoSafeCastArch); improve / simplify a few comments. - Danilo ] Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/driver.rs | 2 +- drivers/gpu/drm/nova/file.rs | 67 +++++++++++++++++++++++++++++++++- drivers/gpu/nova-core/api.rs | 8 +++- drivers/gpu/nova-core/gpu.rs | 40 ++++++++++++++++---- include/uapi/drm/nova_drm.h | 55 ++++++++++++++++++++++++++++ 5 files changed, 162 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index 52550e694729..0acdd2eb45ca 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -33,7 +33,6 @@ pub(crate) struct Nova<'bound> { /// DRM registration data, accessible from ioctl handlers via the registration guard. pub(crate) struct DrmRegData<'bound> { - #[expect(unused)] pub(crate) api: NovaCoreApiHandle<'bound>, } @@ -98,5 +97,6 @@ impl drm::Driver for NovaDriver { (NOVA_GETPARAM, drm_nova_getparam, ioctl::RENDER_ALLOW, File::get_param), (NOVA_GEM_CREATE, drm_nova_gem_create, ioctl::AUTH | ioctl::RENDER_ALLOW, File::gem_create), (NOVA_GEM_INFO, drm_nova_gem_info, ioctl::AUTH | ioctl::RENDER_ALLOW, File::gem_info), + (NOVA_INFO, drm_nova_info, ioctl::RENDER_ALLOW, File::info), } } diff --git a/drivers/gpu/drm/nova/file.rs b/drivers/gpu/drm/nova/file.rs index 1156df51c533..1c026f6c2f87 100644 --- a/drivers/gpu/drm/nova/file.rs +++ b/drivers/gpu/drm/nova/file.rs @@ -15,13 +15,63 @@ gem::BaseObject, Registered, // }, + num::casts::{ + arch::IntoSafeCastArch, + IntoSafeCast, // + }, pci, prelude::*, + transmute::AsBytes, + uaccess::UserSlice, uapi, }; pub(crate) struct File; +/// GPU information returned to userspace. +#[repr(transparent)] +struct GpuInfo(uapi::drm_nova_info_gpu); + +impl GpuInfo { + /// Collects the GPU information reported to userspace. + /// + /// This is fallible so that unexpected GSP behaviour, such as a malformed name string, is + /// reported to userspace instead of being silently replaced with a made up value. When adding + /// a field, take care that a value the GSP simply does not provide, for example because the + /// firmware predates it, does not cause a failure. Such fields must fall back to zero or + /// another documented default instead. + fn new(reg_data: &DrmRegData<'_>) -> Result { + let spec = reg_data.api.with(|api| api.get_ref().spec()); + + let info = uapi::drm_nova_info_gpu { + architecture: spec.chipset.arch().into(), + chipid: spec.chipset.into(), + }; + Ok(Self(info)) + } +} + +// SAFETY: `GpuInfo` has no interior mutability, and there are no uninitialized bytes. +unsafe impl AsBytes for GpuInfo {} + +fn write_info(info: &mut uapi::drm_nova_info, value: T) -> Result { + // A NULL data pointer requests the size of the information structure. + if info.data == 0 { + info.size = size_of::().try_into()?; + return Ok(()); + } + + let mut writer = UserSlice::new( + UserPtr::from_addr(info.data.into_safe_cast_arch()), + info.size.into_safe_cast(), + ) + .writer(); + + info.size = writer.write_truncated(&value)?.try_into()?; + + Ok(()) +} + impl drm::file::DriverFile for File { type Driver = NovaDriver; @@ -41,7 +91,7 @@ pub(crate) fn get_param( let adev: &auxiliary::Device = dev.as_ref(); let pdev: &pci::Device = adev.parent().try_into()?; - let value = match getparam.param as u32 { + let value = match getparam.param.try_into()? { uapi::NOVA_GETPARAM_VRAM_BAR_SIZE => pdev.resource_len(1)?, _ => return Err(EINVAL), }; @@ -78,4 +128,19 @@ pub(crate) fn gem_info( Ok(0) } + + /// IOCTL: info: Query device information. + pub(crate) fn info( + _dev: &NovaDevice, + reg_data: &DrmRegData<'_>, + info: &mut uapi::drm_nova_info, + _file: &drm::File, + ) -> Result { + match info.id { + uapi::DRM_NOVA_INFO_GPU => write_info(info, GpuInfo::new(reg_data)?)?, + _ => return Err(EINVAL), + } + + Ok(0) + } } diff --git a/drivers/gpu/nova-core/api.rs b/drivers/gpu/nova-core/api.rs index ae023242594e..876451dcc051 100644 --- a/drivers/gpu/nova-core/api.rs +++ b/drivers/gpu/nova-core/api.rs @@ -12,11 +12,12 @@ types::ForLt, // }; +pub use crate::gpu::Spec; + use crate::gpu::Gpu; /// API handle for the auxiliary bus child drivers to interact with nova-core. pub struct NovaCoreApi<'bound> { - #[expect(unused)] pub(crate) gpu: Pin<&'bound Gpu<'bound>>, } @@ -26,6 +27,11 @@ impl NovaCoreApi<'_> { pub fn of(adev: &auxiliary::Device) -> Result> { NovaCoreApiHandle::of(adev) } + + /// Returns the GPU [`Spec`]. + pub fn spec(&self) -> &Spec { + &self.gpu.spec + } } /// Expose a handle to nova-core API diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index a291d2d63f43..190a14ea8189 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -57,12 +57,14 @@ macro_rules! define_chipset { /// Enum representation of the GPU chipset. #[derive(fmt::Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq)] #[repr(u32)] - pub(crate) enum Chipset { + #[allow(missing_docs)] + pub enum Chipset { $($variant = uapi::[]),*, } impl Chipset { - pub(crate) const ALL: &'static [Chipset] = &[ + /// All chipsets known to the driver. + pub const ALL: &'static [Chipset] = &[ $( Chipset::$variant, )* ]; @@ -136,7 +138,8 @@ fn try_from(value: u32) -> Result { }); impl Chipset { - pub(crate) const fn arch(self) -> Architecture { + /// Returns the [`Architecture`] generation of this chipset. + pub const fn arch(self) -> Architecture { match self { Self::TU102 | Self::TU104 | Self::TU106 | Self::TU117 | Self::TU116 => { Architecture::Turing @@ -166,6 +169,14 @@ pub(crate) fn mmu_version(self) -> MmuVersion { } } +impl From for u32 { + #[inline] + fn from(value: Chipset) -> Self { + // CAST: `Chipset` is `repr(u32)` and can thus be cast losslessly. + value as u32 + } +} + // TODO // // The resulting strings are used to generate firmware paths, hence the @@ -184,18 +195,32 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { /// Enum representation of the GPU generation. #[derive(fmt::Debug, Copy, Clone)] #[repr(u32)] - pub(crate) enum Architecture with TryFrom> { + pub enum Architecture with TryFrom> { + /// Turing (TU1xx). Turing = uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_TURING, + /// Ampere (GA10x). Ampere = uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_AMPERE, + /// Hopper (GH100). Hopper = uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_HOPPER, + /// Ada Lovelace (AD10x). Ada = uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_ADA, + /// Blackwell (GB10x). BlackwellGB10x = uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_BLACKWELL_GB10X, + /// Blackwell (GB20x). BlackwellGB20x = uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_BLACKWELL_GB20X, } } +impl From for u32 { + #[inline] + fn from(value: Architecture) -> Self { + // CAST: `Architecture` is `repr(u32)` and can thus be cast losslessly. + value as u32 + } +} + #[derive(Clone, Copy)] pub(crate) struct Revision { major: Bounded, @@ -219,8 +244,9 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { /// Structure holding a basic description of the GPU: `Chipset` and `Revision`. #[derive(Clone, Copy)] -pub(crate) struct Spec { - chipset: Chipset, +pub struct Spec { + /// The GPU chipset. + pub chipset: Chipset, revision: Revision, } @@ -308,7 +334,7 @@ struct GspResources<'gpu> { /// Structure holding the resources required to operate the GPU. #[pin_data] pub(crate) struct Gpu<'gpu> { - spec: Spec, + pub(crate) spec: Spec, /// Static GPU information as provided by the GSP. gsp_static_info: GetGspStaticInfoReply, /// GPU memory manager owning memory management resources. diff --git a/include/uapi/drm/nova_drm.h b/include/uapi/drm/nova_drm.h index e35f09a1ebe8..079da428164d 100644 --- a/include/uapi/drm/nova_drm.h +++ b/include/uapi/drm/nova_drm.h @@ -132,9 +132,62 @@ struct drm_nova_gem_info { __u64 size; }; +/** + * struct drm_nova_info - query device information + */ +struct drm_nova_info { + /** + * @id: The identifier of the information to query. + */ + __u32 id; + + /** + * @size: The amount of space allocated by userspace at @data. The kernel + * will return the number of bytes it wrote, or the size of the queried + * information structure if @data is NULL. + */ + __u32 size; + + /** + * @data: Pointer to the userspace buffer into which the queried + * information will be written. May be NULL, in which case nothing is + * written and @size is set to the size of the information structure so + * userspace can allocate a suitably sized buffer. + */ + __u64 data; +}; + +/** + * DRM_NOVA_INFO_GPU + * + * Query GPU information. The result is returned in a + * &struct drm_nova_info_gpu. + */ +#define DRM_NOVA_INFO_GPU 0x00 + +/** + * struct drm_nova_info_gpu - GPU information + */ +struct drm_nova_info_gpu { + /** + * @architecture: GPU architecture identifier. + * + * See &enum drm_nova_architecture for currently known architectures. + */ + __u32 architecture; + + /** + * @chipid: Opaque GPU chip identifier. + * + * See &enum drm_nova_chipid for currently known chips. + */ + __u32 chipid; +}; + #define DRM_NOVA_GETPARAM 0x00 #define DRM_NOVA_GEM_CREATE 0x01 #define DRM_NOVA_GEM_INFO 0x02 +#define DRM_NOVA_INFO 0x03 /* Note: this is an enum so that it can be resolved by Rust bindgen. */ enum { @@ -144,6 +197,8 @@ enum { struct drm_nova_gem_create), DRM_IOCTL_NOVA_GEM_INFO = DRM_IOWR(DRM_COMMAND_BASE + DRM_NOVA_GEM_INFO, struct drm_nova_gem_info), + DRM_IOCTL_NOVA_INFO = DRM_IOWR(DRM_COMMAND_BASE + DRM_NOVA_INFO, + struct drm_nova_info), }; #if defined(__cplusplus) From d9886a1a06cd355fc292987d22fb088a716691e0 Mon Sep 17 00:00:00 2001 From: Alistair Popple Date: Wed, 9 Sep 2026 16:45:01 +1000 Subject: [PATCH 081/110] drm: nova: Add usable VRAM size to GPU info Add a field to the GPU info structure containing the total usable framebuffer size. The usable framebuffer excludes GSP carveouts and other protected regions, so it may differ from the BAR size and total physical VRAM. Signed-off-by: Alistair Popple Link: https://patch.msgid.link/20260909064506.910162-9-apopple@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/file.rs | 2 ++ drivers/gpu/nova-core/api.rs | 6 ++++++ drivers/gpu/nova-core/gpu.rs | 6 ++---- drivers/gpu/nova-core/gsp/commands.rs | 10 +++++++++- include/uapi/drm/nova_drm.h | 6 ++++++ 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/nova/file.rs b/drivers/gpu/drm/nova/file.rs index 1c026f6c2f87..988f639af116 100644 --- a/drivers/gpu/drm/nova/file.rs +++ b/drivers/gpu/drm/nova/file.rs @@ -42,10 +42,12 @@ impl GpuInfo { /// another documented default instead. fn new(reg_data: &DrmRegData<'_>) -> Result { let spec = reg_data.api.with(|api| api.get_ref().spec()); + let gsp_static_info = reg_data.api.with(|api| api.get_ref().gsp_static_info()); let info = uapi::drm_nova_info_gpu { architecture: spec.chipset.arch().into(), chipid: spec.chipset.into(), + vram_size: gsp_static_info.vram_size(), }; Ok(Self(info)) } diff --git a/drivers/gpu/nova-core/api.rs b/drivers/gpu/nova-core/api.rs index 876451dcc051..64f9ea442278 100644 --- a/drivers/gpu/nova-core/api.rs +++ b/drivers/gpu/nova-core/api.rs @@ -15,6 +15,7 @@ pub use crate::gpu::Spec; use crate::gpu::Gpu; +use crate::gsp::commands::GetGspStaticInfoReply; /// API handle for the auxiliary bus child drivers to interact with nova-core. pub struct NovaCoreApi<'bound> { @@ -28,6 +29,11 @@ pub fn of(adev: &auxiliary::Device) -> Result> { NovaCoreApiHandle::of(adev) } + /// Returns the GPU [`GetGspStaticInfoReply`]. + pub fn gsp_static_info(&self) -> &GetGspStaticInfoReply { + &self.gpu.gsp_static_info + } + /// Returns the GPU [`Spec`]. pub fn spec(&self) -> &Spec { &self.gpu.spec diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 190a14ea8189..fb6f8a86a503 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -336,7 +336,7 @@ struct GspResources<'gpu> { pub(crate) struct Gpu<'gpu> { pub(crate) spec: Spec, /// Static GPU information as provided by the GSP. - gsp_static_info: GetGspStaticInfoReply, + pub(crate) gsp_static_info: GetGspStaticInfoReply, /// GPU memory manager owning memory management resources. /// /// Must be kept declared *before* `gsp_resources`, so that its components are dropped while @@ -473,9 +473,7 @@ pub(crate) fn new<'a>( dev_dbg!( dev, "Total usable VRAM: {} MiB\n", - info.usable_fb_regions.iter().fold(0u64, |res, region| res - .saturating_add(region.end - region.start)) - / u64::SZ_1M + info.vram_size() / u64::SZ_1M ); } diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index e087c9e8c35c..c193149bc20e 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -212,7 +212,7 @@ fn init(&self) -> impl Init { } /// The reply from the GSP to the [`GetGspStaticInfo`] command. -pub(crate) struct GetGspStaticInfoReply { +pub struct GetGspStaticInfoReply { gpu_name: [u8; 64], /// BAR1 Page Directory Entry base address. pub(crate) bar1_pde_base: u64, @@ -268,6 +268,14 @@ pub(crate) fn gpu_name(&self) -> core::result::Result<&str, GpuNameError> { .to_str() .map_err(GpuNameError::InvalidUtf8) } + + /// Returns the total usable VRAM size in bytes, i.e. the summed lengths of all usable FB + /// regions. + pub fn vram_size(&self) -> u64 { + self.usable_fb_regions.iter().fold(0, |size, region| { + size.saturating_add(region.end - region.start) + }) + } } pub(crate) use fw::commands::PowerStateLevel; diff --git a/include/uapi/drm/nova_drm.h b/include/uapi/drm/nova_drm.h index 079da428164d..97be2b54ce9b 100644 --- a/include/uapi/drm/nova_drm.h +++ b/include/uapi/drm/nova_drm.h @@ -182,6 +182,12 @@ struct drm_nova_info_gpu { * See &enum drm_nova_chipid for currently known chips. */ __u32 chipid; + + /** + * @vram_size: Amount of usable FB, excluding GSP carveouts and protected + * regions. + */ + __u64 vram_size; }; #define DRM_NOVA_GETPARAM 0x00 From 7edc491f6fca8e22f51e37a16d74914cd959274d Mon Sep 17 00:00:00 2001 From: Alistair Popple Date: Wed, 9 Sep 2026 16:45:02 +1000 Subject: [PATCH 082/110] drm: nova: Use nova-core to read VRAM_BAR_SIZE parameter Currently nova-drm reads the VRAM BAR size directly from the PCIe device which requires trying to cast the parent device into a PCIe device. This obviously requires the parent device to actually be a PCIe bus device. Whilst that is true today it may not always be the case, and there is no reason to make this assumption now that NovaCoreApi can hold a reference to the bound PCIe device. So convert nova-drm to using nova-core to obtain the VRAM_BAR_SIZE parameter. Suggested-by: Danilo Krummrich Signed-off-by: Alistair Popple Link: https://patch.msgid.link/20260909064506.910162-10-apopple@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/file.rs | 12 +++--------- drivers/gpu/nova-core/api.rs | 8 ++++++++ drivers/gpu/nova-core/driver.rs | 2 +- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/nova/file.rs b/drivers/gpu/drm/nova/file.rs index 988f639af116..38aca1a1e3a4 100644 --- a/drivers/gpu/drm/nova/file.rs +++ b/drivers/gpu/drm/nova/file.rs @@ -8,8 +8,6 @@ use crate::gem::NovaObject; use kernel::{ alloc::flags::*, - auxiliary, - device::Bound, drm::{ self, gem::BaseObject, @@ -19,7 +17,6 @@ arch::IntoSafeCastArch, IntoSafeCast, // }, - pci, prelude::*, transmute::AsBytes, uaccess::UserSlice, @@ -85,16 +82,13 @@ fn open(_dev: &NovaDevice) -> Result>> { impl File { /// IOCTL: get_param: Query GPU / driver metadata. pub(crate) fn get_param( - dev: &NovaDevice, - _reg_data: &DrmRegData<'_>, + _dev: &NovaDevice, + reg_data: &DrmRegData<'_>, getparam: &mut uapi::drm_nova_getparam, _file: &drm::File, ) -> Result { - let adev: &auxiliary::Device = dev.as_ref(); - let pdev: &pci::Device = adev.parent().try_into()?; - let value = match getparam.param.try_into()? { - uapi::NOVA_GETPARAM_VRAM_BAR_SIZE => pdev.resource_len(1)?, + uapi::NOVA_GETPARAM_VRAM_BAR_SIZE => reg_data.api.with(|api| api.bar1_size())?, _ => return Err(EINVAL), }; diff --git a/drivers/gpu/nova-core/api.rs b/drivers/gpu/nova-core/api.rs index 64f9ea442278..f02c6c7c7e51 100644 --- a/drivers/gpu/nova-core/api.rs +++ b/drivers/gpu/nova-core/api.rs @@ -8,6 +8,7 @@ use kernel::{ auxiliary, device::Bound, + pci, prelude::*, types::ForLt, // }; @@ -20,6 +21,7 @@ /// API handle for the auxiliary bus child drivers to interact with nova-core. pub struct NovaCoreApi<'bound> { pub(crate) gpu: Pin<&'bound Gpu<'bound>>, + pub(crate) pdev: &'bound pci::Device, } impl NovaCoreApi<'_> { @@ -38,6 +40,12 @@ pub fn gsp_static_info(&self) -> &GetGspStaticInfoReply { pub fn spec(&self) -> &Spec { &self.gpu.spec } + + /// Returns the size of the PCIe BAR used for accessing VRAM, typically + /// BAR1. + pub fn bar1_size(&self) -> Result { + self.pdev.resource_len(1) + } } /// Expose a handle to nova-core API diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index d2f20943142d..bdaed5408a00 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -146,7 +146,7 @@ fn probe<'bound>( // never recycles IDs. AUXILIARY_ID_COUNTER.fetch_add(1, Relaxed), crate::MODULE_NAME, - NovaCoreApi { gpu }, + NovaCoreApi { gpu, pdev }, )? } }, From 843bb3ccde8e16f55e33a10a6c3da3e0a76db6ef Mon Sep 17 00:00:00 2001 From: Alistair Popple Date: Wed, 9 Sep 2026 16:45:03 +1000 Subject: [PATCH 083/110] drm: nova: Expose a render node nova-drm currently only exposes a primary node even though all of its ioctls are already marked DRM_RENDER_ALLOW. Set the DRIVER_RENDER feature so that a render node (/dev/dri/renderDXX) is created as well. This is required to allow render and compute clients to interact with nova-drm. Signed-off-by: Alistair Popple Link: https://patch.msgid.link/20260909064506.910162-11-apopple@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/driver.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index 0acdd2eb45ca..46e13fe795eb 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -92,6 +92,7 @@ impl drm::Driver for NovaDriver { type ParentDevice = auxiliary::Device; const INFO: drm::DriverInfo = INFO; + const FEAT_RENDER: bool = true; kernel::declare_drm_ioctls! { (NOVA_GETPARAM, drm_nova_getparam, ioctl::RENDER_ALLOW, File::get_param), From 391cc215d22c5137d71b93a846266489ba7d98bd Mon Sep 17 00:00:00 2001 From: Alistair Popple Date: Wed, 9 Sep 2026 16:45:04 +1000 Subject: [PATCH 084/110] drm: nova: Report GPU name in GPU info Add the full GPU name to the GPU info structure. The name is copied from the validated, NUL-terminated string nova-core already extracts from the GSP static info rather than from the raw firmware bytes. GpuInfo::new() is deliberately fallible here. A malformed name string from the GSP is unexpected and is reported to userspace as an error rather than being silently replaced with an empty name. Signed-off-by: Alistair Popple Link: https://patch.msgid.link/20260909064506.910162-12-apopple@nvidia.com [ Use ..pin_init::zeroed() instead of ..default::Default(). - Danilo ] Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/file.rs | 24 +++++++++++++++++++++++- drivers/gpu/nova-core/gsp/commands.rs | 5 ++--- include/uapi/drm/nova_drm.h | 5 +++++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/nova/file.rs b/drivers/gpu/drm/nova/file.rs index 38aca1a1e3a4..9aa6d6660fc8 100644 --- a/drivers/gpu/drm/nova/file.rs +++ b/drivers/gpu/drm/nova/file.rs @@ -29,6 +29,21 @@ #[repr(transparent)] struct GpuInfo(uapi::drm_nova_info_gpu); +/// Copies `name` into the zero initialised, fixed size uAPI buffer `dst`, keeping it +/// NUL-terminated. +/// +/// Fails with [`ENAMETOOLONG`] if `name` does not fit in `dst` with room for the terminator. +fn copy_name(dst: &mut [u8], name: &str) -> Result { + let bytes = name.as_bytes(); + + if bytes.len() >= dst.len() { + return Err(ENAMETOOLONG); + } + dst[..bytes.len()].copy_from_slice(bytes); + + Ok(()) +} + impl GpuInfo { /// Collects the GPU information reported to userspace. /// @@ -41,11 +56,18 @@ fn new(reg_data: &DrmRegData<'_>) -> Result { let spec = reg_data.api.with(|api| api.get_ref().spec()); let gsp_static_info = reg_data.api.with(|api| api.get_ref().gsp_static_info()); - let info = uapi::drm_nova_info_gpu { + let mut info = uapi::drm_nova_info_gpu { architecture: spec.chipset.arch().into(), chipid: spec.chipset.into(), vram_size: gsp_static_info.vram_size(), + ..pin_init::zeroed() }; + + copy_name( + &mut info.gpu_name, + gsp_static_info.gpu_name().map_err(|_| EINVAL)?, + )?; + Ok(Self(info)) } } diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index c193149bc20e..727c95f7781f 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -248,12 +248,11 @@ fn read( /// Error type for [`GetGspStaticInfoReply::gpu_name`]. #[derive(Debug)] -pub(crate) enum GpuNameError { +pub enum GpuNameError { /// The GPU name string does not contain a null terminator. NoNullTerminator(FromBytesUntilNulError), /// The GPU name string contains invalid UTF-8. - #[expect(dead_code)] InvalidUtf8(Utf8Error), } @@ -262,7 +261,7 @@ impl GetGspStaticInfoReply { /// /// Returns an error if the string given by the GSP does not contain a null terminator or /// contains invalid UTF-8. - pub(crate) fn gpu_name(&self) -> core::result::Result<&str, GpuNameError> { + pub fn gpu_name(&self) -> Result<&str, GpuNameError> { CStr::from_bytes_until_nul(&self.gpu_name) .map_err(GpuNameError::NoNullTerminator)? .to_str() diff --git a/include/uapi/drm/nova_drm.h b/include/uapi/drm/nova_drm.h index 97be2b54ce9b..09dfa2af3ed8 100644 --- a/include/uapi/drm/nova_drm.h +++ b/include/uapi/drm/nova_drm.h @@ -188,6 +188,11 @@ struct drm_nova_info_gpu { * regions. */ __u64 vram_size; + + /** + * @gpu_name: NUL-terminated full GPU name. + */ + __u8 gpu_name[64]; }; #define DRM_NOVA_GETPARAM 0x00 From e6d9c5b08c23520ae261ba4282f0d588c3d661a3 Mon Sep 17 00:00:00 2001 From: Alistair Popple Date: Wed, 9 Sep 2026 16:45:05 +1000 Subject: [PATCH 085/110] drm: nova: Report GPU short name in GPU info Add the short GPU name to the GPU info structure. As with the full name it is copied from a validated, NUL-terminated string extracted by nova-core rather than from the raw firmware bytes. Signed-off-by: Alistair Popple Link: https://patch.msgid.link/20260909064506.910162-13-apopple@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/file.rs | 5 +++++ drivers/gpu/nova-core/gsp/commands.rs | 13 +++++++++++++ drivers/gpu/nova-core/gsp/fw/commands.rs | 5 +++++ include/uapi/drm/nova_drm.h | 5 +++++ 4 files changed, 28 insertions(+) diff --git a/drivers/gpu/drm/nova/file.rs b/drivers/gpu/drm/nova/file.rs index 9aa6d6660fc8..134f011d16c6 100644 --- a/drivers/gpu/drm/nova/file.rs +++ b/drivers/gpu/drm/nova/file.rs @@ -68,6 +68,11 @@ fn new(reg_data: &DrmRegData<'_>) -> Result { gsp_static_info.gpu_name().map_err(|_| EINVAL)?, )?; + copy_name( + &mut info.gpu_short_name, + gsp_static_info.gpu_short_name().map_err(|_| EINVAL)?, + )?; + Ok(Self(info)) } } diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index 727c95f7781f..e865b92047fa 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -214,6 +214,7 @@ fn init(&self) -> impl Init { /// The reply from the GSP to the [`GetGspStaticInfo`] command. pub struct GetGspStaticInfoReply { gpu_name: [u8; 64], + gpu_short_name: [u8; 64], /// BAR1 Page Directory Entry base address. pub(crate) bar1_pde_base: u64, /// Usable FB (VRAM) regions for driver memory allocation. @@ -239,6 +240,7 @@ fn read( Ok(GetGspStaticInfoReply { gpu_name: msg.gpu_name_str(), + gpu_short_name: msg.gpu_short_name_str(), bar1_pde_base: msg.bar1_pde_base(), usable_fb_regions, total_fb_end, @@ -268,6 +270,17 @@ pub fn gpu_name(&self) -> Result<&str, GpuNameError> { .map_err(GpuNameError::InvalidUtf8) } + /// Returns the short name of the GPU as a string. + /// + /// Returns an error if the string given by the GSP does not contain a null terminator or + /// contains invalid UTF-8. + pub fn gpu_short_name(&self) -> core::result::Result<&str, GpuNameError> { + CStr::from_bytes_until_nul(&self.gpu_short_name) + .map_err(GpuNameError::NoNullTerminator)? + .to_str() + .map_err(GpuNameError::InvalidUtf8) + } + /// Returns the total usable VRAM size in bytes, i.e. the summed lengths of all usable FB /// regions. pub fn vram_size(&self) -> u64 { diff --git a/drivers/gpu/nova-core/gsp/fw/commands.rs b/drivers/gpu/nova-core/gsp/fw/commands.rs index 32856ff74183..5891922ff678 100644 --- a/drivers/gpu/nova-core/gsp/fw/commands.rs +++ b/drivers/gpu/nova-core/gsp/fw/commands.rs @@ -131,6 +131,11 @@ impl GspStaticConfigInfo { self.0.gpuNameString } + /// Returns a bytes array containing the (hopefully) zero-terminated short name of this GPU. + pub(crate) fn gpu_short_name_str(&self) -> [u8; 64] { + self.0.gpuShortNameString + } + /// Returns the BAR1 Page Directory Entry base address. /// /// This is the root page table address for BAR1 virtual memory, diff --git a/include/uapi/drm/nova_drm.h b/include/uapi/drm/nova_drm.h index 09dfa2af3ed8..0e6a44c2ea5a 100644 --- a/include/uapi/drm/nova_drm.h +++ b/include/uapi/drm/nova_drm.h @@ -193,6 +193,11 @@ struct drm_nova_info_gpu { * @gpu_name: NUL-terminated full GPU name. */ __u8 gpu_name[64]; + + /** + * @gpu_short_name: NUL-terminated short GPU name. + */ + __u8 gpu_short_name[64]; }; #define DRM_NOVA_GETPARAM 0x00 From 10a6623a24a85708650efad7be15182289403cd7 Mon Sep 17 00:00:00 2001 From: Alistair Popple Date: Wed, 9 Sep 2026 16:45:06 +1000 Subject: [PATCH 086/110] drm: nova: Report GPU GID in GPU info Add the GPU GID to the reported GPU info. This is the 16-byte SHA-1 based identifier GSP-RM reports for the GPU. Signed-off-by: Alistair Popple Link: https://patch.msgid.link/20260909064506.910162-14-apopple@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/file.rs | 1 + drivers/gpu/nova-core/gsp/commands.rs | 3 +++ drivers/gpu/nova-core/gsp/fw/commands.rs | 10 ++++++++++ include/uapi/drm/nova_drm.h | 5 +++++ 4 files changed, 19 insertions(+) diff --git a/drivers/gpu/drm/nova/file.rs b/drivers/gpu/drm/nova/file.rs index 134f011d16c6..766a6ddb7e10 100644 --- a/drivers/gpu/drm/nova/file.rs +++ b/drivers/gpu/drm/nova/file.rs @@ -60,6 +60,7 @@ fn new(reg_data: &DrmRegData<'_>) -> Result { architecture: spec.chipset.arch().into(), chipid: spec.chipset.into(), vram_size: gsp_static_info.vram_size(), + gpu_gid: gsp_static_info.gpu_gid, ..pin_init::zeroed() }; diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index e865b92047fa..59d7d7fb15e8 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -215,6 +215,8 @@ fn init(&self) -> impl Init { pub struct GetGspStaticInfoReply { gpu_name: [u8; 64], gpu_short_name: [u8; 64], + /// The 16-byte SHA-1 based GPU identifier (GID) reported by GSP-RM. + pub gpu_gid: [u8; 16], /// BAR1 Page Directory Entry base address. pub(crate) bar1_pde_base: u64, /// Usable FB (VRAM) regions for driver memory allocation. @@ -241,6 +243,7 @@ fn read( Ok(GetGspStaticInfoReply { gpu_name: msg.gpu_name_str(), gpu_short_name: msg.gpu_short_name_str(), + gpu_gid: msg.gpu_gid(), bar1_pde_base: msg.bar1_pde_base(), usable_fb_regions, total_fb_end, diff --git a/drivers/gpu/nova-core/gsp/fw/commands.rs b/drivers/gpu/nova-core/gsp/fw/commands.rs index 5891922ff678..567c56fbbfe3 100644 --- a/drivers/gpu/nova-core/gsp/fw/commands.rs +++ b/drivers/gpu/nova-core/gsp/fw/commands.rs @@ -136,6 +136,16 @@ impl GspStaticConfigInfo { self.0.gpuShortNameString } + /// Returns the 16-byte SHA-1 GPU identifier supplied by GSP-RM. + /// + /// GSP-RM reports the GID in binary SHA-1 form, which occupies the first 16 bytes of the + /// GID info payload. + pub(crate) fn gpu_gid(&self) -> [u8; 16] { + let mut gid = [0u8; 16]; + gid.copy_from_slice(&self.0.gidInfo.data[..16]); + gid + } + /// Returns the BAR1 Page Directory Entry base address. /// /// This is the root page table address for BAR1 virtual memory, diff --git a/include/uapi/drm/nova_drm.h b/include/uapi/drm/nova_drm.h index 0e6a44c2ea5a..a74daba6ea94 100644 --- a/include/uapi/drm/nova_drm.h +++ b/include/uapi/drm/nova_drm.h @@ -198,6 +198,11 @@ struct drm_nova_info_gpu { * @gpu_short_name: NUL-terminated short GPU name. */ __u8 gpu_short_name[64]; + + /** + * @gpu_gid: 16-byte SHA-1 GPU identifier supplied by GSP-RM. + */ + __u8 gpu_gid[16]; }; #define DRM_NOVA_GETPARAM 0x00 From 658c5f0042e6e897338dd04330a363aa7022c85d Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 27 Sep 2026 16:44:03 +0200 Subject: [PATCH 087/110] gpu: nova-core: use a single try_pin_init!() block in probe() Just like in Gpu::new() a single try_pin_init!() block makes the code more readable. Besides that, it prepares the code for a proper PCI device enable guard that we will get soon. Reviewed-by: Eliot Courtney Acked-by: Alexandre Courbot Link: https://patch.msgid.link/20260927144417.1280385-1-dakr@kernel.org [ Drop useless dev_dbg!() in probe(). - Danilo ] Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/driver.rs | 97 +++++++++++++++++---------------- 1 file changed, 49 insertions(+), 48 deletions(-) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index bdaed5408a00..291a047e4d86 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -100,57 +100,58 @@ fn probe<'bound>( pdev: &'bound pci::Device>, _info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { - pin_init::pin_init_scope(move || { - dev_dbg!(pdev, "Probe Nova Core GPU driver.\n"); + try_pin_init!(NovaCore { + _: { + pdev.enable_device_mem()?; + pdev.set_master(); + }, - pdev.enable_device_mem()?; - pdev.set_master(); + bar: pdev.iomap_region_sized::(0, c"nova-core/bar0")?, - Ok(try_pin_init!(NovaCore { - bar: pdev.iomap_region_sized::(0, c"nova-core/bar0")?, - bar1: { - let bar1_idx = bar1_resource_index(pdev)?; - pdev.iomap_region(bar1_idx, c"nova-core/bar1")? - }, - // TODO: Use self-referential pin-init syntax once available. - gpu <- Gpu::new( - pdev, - // SAFETY: `bar` is initialized above, pinned, and outlives `gpu`. - unsafe { &*core::ptr::from_ref(bar) }, - // SAFETY: `bar1` is initialized above, pinned, and outlives `gpu`. - unsafe { &*core::ptr::from_ref(bar1) }, - ).pin_chain(|_gpu| { - #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] - _gpu.run_selftests(pdev); - Ok(()) - }), - _reg: { - // TODO: Use `&gpu` self-referential pin-init syntax once available. - // - // SAFETY: `gpu` is initialized before this expression is evaluated - // (`try_pin_init!()` initializes fields in initializer order), lives at - // a pinned stable address, and is dropped after `_reg` (struct field - // drop order). - let gpu = unsafe { - Pin::new_unchecked(&*core::ptr::from_ref(gpu.as_ref().get_ref())) - }; + bar1: { + let bar1_idx = bar1_resource_index(pdev)?; + pdev.iomap_region(bar1_idx, c"nova-core/bar1")? + }, - // SAFETY: `NovaCore` is dropped when the device is unbound; - // i.e. `mem::forget()` is never called on it. - unsafe { - auxiliary::Registration::new_with_lt( - pdev.as_ref(), - c"nova-drm", - // TODO[XARR]: Use XArray or perhaps IDA for proper ID - // allocation/recycling. For now, use a simple atomic counter that - // never recycles IDs. - AUXILIARY_ID_COUNTER.fetch_add(1, Relaxed), - crate::MODULE_NAME, - NovaCoreApi { gpu, pdev }, - )? - } - }, - })) + // TODO: Use self-referential pin-init syntax once available. + gpu <- Gpu::new( + pdev, + // SAFETY: `bar` is initialized above, pinned, and outlives `gpu`. + unsafe { &*core::ptr::from_ref(bar) }, + // SAFETY: `bar1` is initialized above, pinned, and outlives `gpu`. + unsafe { &*core::ptr::from_ref(bar1) }, + ).pin_chain(|_gpu| { + #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] + _gpu.run_selftests(pdev); + Ok(()) + }), + + _reg: { + // TODO: Use `&gpu` self-referential pin-init syntax once available. + // + // SAFETY: `gpu` is initialized before this expression is evaluated + // (`try_pin_init!()` initializes fields in initializer order), lives at + // a pinned stable address, and is dropped after `_reg` (struct field + // drop order). + let gpu = unsafe { + Pin::new_unchecked(&*core::ptr::from_ref(gpu.as_ref().get_ref())) + }; + + // SAFETY: `NovaCore` is dropped when the device is unbound; + // i.e. `mem::forget()` is never called on it. + unsafe { + auxiliary::Registration::new_with_lt( + pdev.as_ref(), + c"nova-drm", + // TODO[XARR]: Use XArray or perhaps IDA for proper ID + // allocation/recycling. For now, use a simple atomic counter that + // never recycles IDs. + AUXILIARY_ID_COUNTER.fetch_add(1, Relaxed), + crate::MODULE_NAME, + NovaCoreApi { gpu, pdev }, + )? + } + }, }) } } From 3fac3b6aabaee62478bc7957487d412c9f1f94ff Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Thu, 10 Sep 2026 17:57:33 +0100 Subject: [PATCH 088/110] drm/tyr: use `bitfield!` instead of `register!` for non-registers `SectionFlags` and `MMU_MEMATTR_STAGE1` are not registers themselves. Use `bitfield!` directly. Signed-off-by: Gary Guo Reviewed-by: Deborah Brouwer Link: https://patch.msgid.link/20260910165734.80378-1-gary@kernel.org Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/fw.rs | 12 ++++-------- drivers/gpu/drm/tyr/regs.rs | 11 +++++------ 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/tyr/fw.rs b/drivers/gpu/drm/tyr/fw.rs index 7edb5eff1707..aea151ff1f93 100644 --- a/drivers/gpu/drm/tyr/fw.rs +++ b/drivers/gpu/drm/tyr/fw.rs @@ -14,6 +14,7 @@ //! [`Section`]: crate::fw::Section use kernel::{ + bitfield, device::{ Bound, Device, // @@ -27,7 +28,6 @@ }, num::Bounded, prelude::*, - register, str::CString, sync::{ Arc, @@ -39,8 +39,7 @@ use crate::{ driver::{ IoMem, - TyrDrmDevice, - TyrRegisters, // + TyrDrmDevice, // }, fw::parser::{ FwParser, @@ -101,11 +100,8 @@ fn from(value: CacheMode) -> Self { } } -register! { - base: TyrRegisters; - - #[allow(non_upper_case_globals)] - pub(super) SectionFlags(u32) @ 0x0 { +bitfield! { + pub(super) struct SectionFlags(u32) { 0:0 read => bool; 1:1 write => bool; 2:2 exec => bool; diff --git a/drivers/gpu/drm/tyr/regs.rs b/drivers/gpu/drm/tyr/regs.rs index 68701c39aee2..69c901afb28f 100644 --- a/drivers/gpu/drm/tyr/regs.rs +++ b/drivers/gpu/drm/tyr/regs.rs @@ -986,6 +986,7 @@ pub(crate) mod mmu_control { /// This array contains 16 instances of the MMU_AS_CONTROL register page. pub(crate) mod mmu_as_control { use kernel::{ + bitfield, num::Bounded, prelude::*, register, // @@ -1130,14 +1131,12 @@ fn from(val: MemoryType) -> Self { } } - register! { - base: TyrRegisters; - + bitfield! { /// Stage 1 memory attributes (8-bit bitfield). /// - /// This is not an actual register, but a bitfield definition used by the MEMATTR - /// register. Each of the 8 bytes in MEMATTR follows this layout. - MMU_MEMATTR_STAGE1(u8) @ 0x0 { + /// Each of the 8 bytes in MEMATTR follows this layout. + #[allow(non_camel_case_types)] + struct MMU_MEMATTR_STAGE1(u8) { /// Inner cache write allocation policy. 0:0 alloc_w => bool; /// Inner cache read allocation policy. From c9ca2e379463751e267cb2bf1f7cad0b8be1cfe6 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Tue, 29 Sep 2026 14:07:49 -0700 Subject: [PATCH 089/110] gpu: nova-core: restore the "Probe Nova Core GPU driver" debug print This is a high value thing to print, because it acts as kind of a bookend, showing the beginning of trying to boot up. Cc: Danilo Krummrich Signed-off-by: John Hubbard Link: https://patch.msgid.link/20260929210749.409380-1-jhubbard@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/driver.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index 291a047e4d86..623d59dcde6e 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -100,6 +100,8 @@ fn probe<'bound>( pdev: &'bound pci::Device>, _info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { + dev_dbg!(pdev, "Probe Nova Core GPU driver.\n"); + try_pin_init!(NovaCore { _: { pdev.enable_device_mem()?; From 38c9764d330dfe1d59009179904e313b0923f7b4 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Tue, 29 Sep 2026 20:41:34 -0700 Subject: [PATCH 090/110] rust: pci: declare IrqType and IrqTypes with impl_flags The kernel provides impl_flags! for declaring a bitmask type alongside the enum of its individual flags, generating the bit operators and the containment queries. IrqTypes open-coded that pattern with a with() builder, so a caller naming two interrupt types chained two calls onto IrqTypes::default(). Declare both types through impl_flags!, so the same set reads as IrqType::Msi | IrqType::MsiX. Suggested-by: Gary Guo Signed-off-by: John Hubbard Reviewed-by: Alexandre Courbot Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260930034148.590687-2-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- rust/kernel/pci/irq.rs | 68 +++++++++++++----------------------------- 1 file changed, 21 insertions(+), 47 deletions(-) diff --git a/rust/kernel/pci/irq.rs b/rust/kernel/pci/irq.rs index 6741046ec1c0..f074aad7f1d8 100644 --- a/rust/kernel/pci/irq.rs +++ b/rust/kernel/pci/irq.rs @@ -13,27 +13,26 @@ }; use core::num::NonZero; -/// IRQ type flags for PCI interrupt allocation. -#[derive(Debug, Clone, Copy)] -pub enum IrqType { - /// INTx interrupts. - Intx, - /// Message Signaled Interrupts (MSI). - Msi, - /// Extended Message Signaled Interrupts (MSI-X). - MsiX, -} +crate::impl_flags!( + /// Set of IRQ types that can be used for PCI interrupt allocation. + #[derive(Debug, Clone, Copy, Default)] + pub struct IrqTypes(u32); + + /// IRQ type flags for PCI interrupt allocation. + #[derive(Debug, Clone, Copy)] + pub enum IrqType { + /// INTx interrupts. + Intx = bindings::PCI_IRQ_INTX, + + /// Message Signaled Interrupts (MSI). + Msi = bindings::PCI_IRQ_MSI, + + /// Extended Message Signaled Interrupts (MSI-X). + MsiX = bindings::PCI_IRQ_MSIX, + } +); impl IrqType { - /// Convert to the corresponding kernel flags. - const fn as_raw(self) -> u32 { - match self { - IrqType::Intx => bindings::PCI_IRQ_INTX, - IrqType::Msi => bindings::PCI_IRQ_MSI, - IrqType::MsiX => bindings::PCI_IRQ_MSIX, - } - } - /// Construct from raw value. #[inline] const fn from_raw(raw: u32) -> Self { @@ -45,33 +44,10 @@ const fn from_raw(raw: u32) -> Self { } } -/// Set of IRQ types that can be used for PCI interrupt allocation. -#[derive(Debug, Clone, Copy, Default)] -pub struct IrqTypes(u32); - impl IrqTypes { /// Create a set containing all IRQ types (MSI-X, MSI, and INTx). pub const fn all() -> Self { - Self(bindings::PCI_IRQ_ALL_TYPES) - } - - /// Build a set of IRQ types. - /// - /// # Examples - /// - /// ```ignore - /// // Create a set with only MSI and MSI-X (no INTx interrupts). - /// let msi_only = IrqTypes::default() - /// .with(IrqType::Msi) - /// .with(IrqType::MsiX); - /// ``` - pub const fn with(self, irq_type: IrqType) -> Self { - Self(self.0 | irq_type.as_raw()) - } - - /// Get the raw flags value. - const fn as_raw(self) -> u32 { - self.0 + Self(Self::all_bits()) } } @@ -203,9 +179,7 @@ impl Device { /// let vectors = dev.alloc_irq_vectors(1, 32, pci::IrqTypes::all())?; /// /// // Allocate MSI or MSI-X only (no INTx interrupts). - /// let msi_only = pci::IrqTypes::default() - /// .with(pci::IrqType::Msi) - /// .with(pci::IrqType::MsiX); + /// let msi_only = pci::IrqType::Msi | pci::IrqType::MsiX; /// let vectors = dev.alloc_irq_vectors(4, 16, msi_only)?; /// # Ok(()) /// # } @@ -222,7 +196,7 @@ pub fn alloc_irq_vectors( // - `pci_alloc_irq_vectors` internally validates all other parameters // and returns error codes. let ret = unsafe { - bindings::pci_alloc_irq_vectors(self.as_raw(), min_vecs, max_vecs, irq_types.as_raw()) + bindings::pci_alloc_irq_vectors(self.as_raw(), min_vecs, max_vecs, u32::from(irq_types)) }; to_result(ret)?; From e6c579b496f2740c447d215ef868dc5f9cc7dc93 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Tue, 29 Sep 2026 20:41:36 -0700 Subject: [PATCH 091/110] gpu: nova-core: add the GIN vector, leaf and subtree types GIN, the GPU Interrupt and Notification unit, is the GPU's interrupt controller. Each interrupt source has a GIN vector number, and the controller latches a pending vector in a two-level tree: one bit of a LEAF register, summarized two leaves at a time by one bit of the TOP register. A vector's number fixes its position in that tree: leaf = vector / 32 bit = vector % 32 subtree = leaf / 2 A tree implements either 8 or 16 leaves, depending on the GPU family. The leaf count sets both the number of subtrees and the highest vector that the tree carries. Without distinct types, a vector, a leaf index, a set of vectors within one leaf, a subtree and a set of subtrees are all plain integers. A caller can pass one where another belongs, and a register field can accept the wrong one. Add a type for each of those, and for the leaf count. A vector converts to its own leaf, bit and subtree. Its constructor rejects, at build time, a number beyond the widest supported tree, and a validation method rejects, at run time, a number beyond the leaves that the current tree implements. A leaf count yields the set of subtrees that it implements. The module has no user yet. Suggested-by: Danilo Krummrich Signed-off-by: John Hubbard Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260930034148.590687-4-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/irq.rs | 12 + drivers/gpu/nova-core/irq/interrupt_tree.rs | 238 ++++++++++++++++++++ drivers/gpu/nova-core/nova_core.rs | 2 + 3 files changed, 252 insertions(+) create mode 100644 drivers/gpu/nova-core/irq.rs create mode 100644 drivers/gpu/nova-core/irq/interrupt_tree.rs diff --git a/drivers/gpu/nova-core/irq.rs b/drivers/gpu/nova-core/irq.rs new file mode 100644 index 000000000000..f1323f633a03 --- /dev/null +++ b/drivers/gpu/nova-core/irq.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! GPU interrupt support. +//! +//! GIN, the GPU Interrupt and Notification unit, is the GPU's interrupt controller. It latches +//! every interrupt source in a two-level register tree and delivers the tree to the CPU as a +//! message-signaled PCI interrupt. +//! +//! See `Documentation/gpu/nova/core/interrupts.rst`. + +mod interrupt_tree; diff --git a/drivers/gpu/nova-core/irq/interrupt_tree.rs b/drivers/gpu/nova-core/irq/interrupt_tree.rs new file mode 100644 index 000000000000..206e6940ce77 --- /dev/null +++ b/drivers/gpu/nova-core/irq/interrupt_tree.rs @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Vector addressing in the GIN CPU interrupt tree. +//! +//! A [`GinVector`] names an interrupt source, a [`LeafIndex`] the leaf register that latches it, +//! a [`LeafMask`] a set of vectors within one leaf, and a [`Subtree`] one `TOP` bit. The types +//! keep the four from being confused with one another. +//! +//! See `Documentation/gpu/nova/core/interrupts.rst`. + +use kernel::{ + num::Bounded, + prelude::*, // +}; + +use crate::num; + +/// Number of vectors one leaf register carries, one per bit. +const VECTORS_PER_LEAF: u32 = u32::BITS; + +/// Number of leaves one subtree covers. +const LEAVES_PER_SUBTREE: u32 = 2; + +/// Number of subtrees the widest supported tree implements. +const MAX_NUM_SUBTREES: u32 = 8; + +/// Number of leaves the widest supported tree implements. +const MAX_NUM_LEAVES: u32 = MAX_NUM_SUBTREES * LEAVES_PER_SUBTREE; + +/// Number of bits needed to address every vector in the widest supported tree. +const VECTOR_BITS: u32 = (MAX_NUM_LEAVES * VECTORS_PER_LEAF).ilog2(); + +/// Index of a leaf register within the widest supported tree. An 8-leaf tree implements only the +/// lower half of the range. +pub(super) type LeafIndex = Bounded; + +/// Number of leaves a tree implements. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u32)] +pub(super) enum LeafCount { + /// Turing through Ada. + Eight = 8, + + /// Hopper and later. + Sixteen = 16, +} + +impl LeafCount { + pub(super) const fn into_u32(self) -> u32 { + // CAST: `LeafCount` is `repr(u32)`, so the cast is lossless. + self as u32 + } + + pub(super) const fn into_raw(self) -> usize { + num::u32_as_usize(self.into_u32()) + } + + /// Returns the number of subtrees a tree of this size implements. + pub(super) const fn subtree_count(self) -> u32 { + self.into_u32() / LEAVES_PER_SUBTREE + } + + /// Returns the set of every subtree a tree of this size implements. + pub(super) const fn subtree_set(self) -> SubtreeSet { + SubtreeSet((1u32 << self.subtree_count()) - 1) + } + + /// Returns the number of vectors a tree of this size carries. + pub(super) const fn vector_count(self) -> u32 { + self.into_u32() * VECTORS_PER_LEAF + } +} + +// `VECTOR_BITS` and `LeafCount::Sixteen` are written separately. This assert keeps them in +// agreement about the widest supported tree. +static_assert!(1 << VECTOR_BITS == LeafCount::Sixteen.vector_count()); + +/// Set of vectors within one leaf, one bit per vector. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct LeafMask(u32); + +impl LeafMask { + /// Returns the mask with every vector set. + pub(super) const fn all() -> Self { + Self(u32::MAX) + } + + pub(super) const fn from_raw(raw: u32) -> Self { + Self(raw) + } + + pub(super) const fn into_raw(self) -> u32 { + self.0 + } + + pub(super) const fn is_empty(self) -> bool { + self.0 == 0 + } + + /// Returns whether every vector in `other` is also in this mask. + pub(super) const fn contains(self, other: Self) -> bool { + self.0 & other.0 == other.0 + } +} + +impl From> for LeafMask { + fn from(vectors: Bounded) -> Self { + Self(vectors.get()) + } +} + +impl From for Bounded { + fn from(vectors: LeafMask) -> Self { + vectors.0.into() + } +} + +/// One subtree, held as the `TOP` bit that covers it. +/// +/// # Invariants +/// +/// Exactly one bit is set. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct Subtree(u32); + +impl Subtree { + /// Returns the subtree at index `idx`. + const fn new(idx: u32) -> Self { + // INVARIANT: shifting `1` left leaves exactly one bit set. + Self(1 << idx) + } + + /// Returns this subtree's index within the tree. + pub(super) const fn index(self) -> u32 { + self.0.trailing_zeros() + } + + pub(super) const fn into_raw(self) -> u32 { + self.0 + } +} + +/// Set of subtrees, one bit per subtree, in the layout of the `TOP` registers. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct SubtreeSet(u32); + +impl SubtreeSet { + pub(super) const fn contains(self, subtree: Subtree) -> bool { + self.0 & subtree.into_raw() != 0 + } + + pub(super) const fn is_empty(self) -> bool { + self.0 == 0 + } + + pub(super) const fn intersection(self, other: Self) -> Self { + Self(self.0 & other.0) + } + + /// Returns one more than the highest index in this set, or `0` for an empty set. An MSI-X + /// allocation that covers the set needs this many entries. + pub(super) const fn span(self) -> u32 { + u32::BITS - self.0.leading_zeros() + } + + /// Returns the subtrees of this set, lowest index first. + #[expect(dead_code)] + pub(super) fn iter(self) -> impl Iterator { + (0..u32::BITS) + .map(Subtree::new) + .filter(move |subtree| self.contains(*subtree)) + } +} + +impl From for SubtreeSet { + fn from(subtree: Subtree) -> Self { + Self(subtree.into_raw()) + } +} + +impl From> for SubtreeSet { + fn from(subtrees: Bounded) -> Self { + Self(subtrees.get()) + } +} + +impl From for Bounded { + fn from(subtrees: SubtreeSet) -> Self { + subtrees.0.into() + } +} + +/// A GIN interrupt vector, bounded to the widest tree that any supported chipset implements. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct GinVector(Bounded); + +impl GinVector { + /// Returns vector number `VECTOR`. + /// + /// Fails to compile if `VECTOR` is beyond the widest supported tree. + pub(super) const fn new() -> Self { + Self(Bounded::::new::()) + } + + pub(super) const fn into_raw(self) -> u32 { + self.0.get() + } + + /// Returns this vector's leaf. + pub(super) fn leaf_index(self) -> LeafIndex { + // CALC: `self.0 / VECTORS_PER_LEAF`. + self.0.shr::<{ VECTORS_PER_LEAF.ilog2() }, _>().cast() + } + + /// Returns this vector's bit within its leaf. + pub(super) const fn leaf_mask(self) -> LeafMask { + LeafMask(1 << (self.0.get() % VECTORS_PER_LEAF)) + } + + /// Returns this vector's subtree. + pub(super) const fn subtree(self) -> Subtree { + Subtree::new(self.0.get() / (VECTORS_PER_LEAF * LEAVES_PER_SUBTREE)) + } + + /// Checks that a tree with `leaves` leaves implements this vector. + /// + /// # Errors + /// + /// `EINVAL` if it does not. + pub(super) const fn validate(self, leaves: LeafCount) -> Result { + if self.0.get() >= leaves.vector_count() { + Err(EINVAL) + } else { + Ok(()) + } + } +} diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 4f8ce4e4c187..8202c4982efa 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -18,6 +18,8 @@ mod fsp; mod gpu; mod gsp; +#[expect(dead_code)] +mod irq; mod mctp; mod mm; #[macro_use] From 2ef55e142d29970641e9be04897457bb98a8eaca Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Tue, 29 Sep 2026 20:41:37 -0700 Subject: [PATCH 092/110] gpu: nova-core: add the GIN CPU interrupt tree and MSI EOI registers GIN is the GPU's interrupt controller. It latches each interrupt source in a two-level tree of LEAF registers summarized by TOP, and raises the PCI interrupt when an enabled vector in an enabled subtree becomes pending. A message-signaled interrupt is delivered once per edge, and pre-Hopper MSI rearms delivery by writing the end-of-interrupt register in the BAR0 mirror of PCI configuration space. Add the CPU tree registers that receiving GSP interrupts and running the software-triggered self-test need: the leaf pending and enable arrays, the TOP enables, and the leaf trigger. Add the end-of-interrupt register, NV_XVE_CYA_2, alongside them. Use the NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_* names on every chipset. The pre-Hopper headers call the same tree NV_CTRL, but each function reaches its own tree through this aperture on both families. Declare the leaf arrays at 16 entries, the widest tree that any supported chipset implements. Leave the read-only TOP summary undeclared. A vector that latched while disabled does not appear in TOP, so nova-core never descends from it and reads every implemented leaf instead. Declare each leaf field as a set of vectors within one leaf and each TOP field as a set of subtrees, so that a set of subtrees cannot be written to a leaf register, nor the reverse. The trigger register's vector field is 12 bits wide, wider than any GIN vector, so a vector converts into it infallibly. Assisted-by: LLM Signed-off-by: John Hubbard Reviewed-by: Will Pierce Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260930034148.590687-5-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/irq.rs | 1 + drivers/gpu/nova-core/irq/interrupt_tree.rs | 14 ++++ drivers/gpu/nova-core/irq/regs.rs | 88 +++++++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 drivers/gpu/nova-core/irq/regs.rs diff --git a/drivers/gpu/nova-core/irq.rs b/drivers/gpu/nova-core/irq.rs index f1323f633a03..1ec0bb055d3b 100644 --- a/drivers/gpu/nova-core/irq.rs +++ b/drivers/gpu/nova-core/irq.rs @@ -10,3 +10,4 @@ //! See `Documentation/gpu/nova/core/interrupts.rst`. mod interrupt_tree; +mod regs; diff --git a/drivers/gpu/nova-core/irq/interrupt_tree.rs b/drivers/gpu/nova-core/irq/interrupt_tree.rs index 206e6940ce77..279a43514b1f 100644 --- a/drivers/gpu/nova-core/irq/interrupt_tree.rs +++ b/drivers/gpu/nova-core/irq/interrupt_tree.rs @@ -16,6 +16,8 @@ use crate::num; +use super::regs::*; + /// Number of vectors one leaf register carries, one per bit. const VECTORS_PER_LEAF: u32 = u32::BITS; @@ -31,6 +33,12 @@ /// Number of bits needed to address every vector in the widest supported tree. const VECTOR_BITS: u32 = (MAX_NUM_LEAVES * VECTORS_PER_LEAF).ilog2(); +/// Width of the vector field in the leaf trigger register. +const TRIGGER_VECTOR_BITS: u32 = { + let range = NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF_TRIGGER::VECTOR_RANGE; + num::u8_as_u32(*range.end() - *range.start() + 1) +}; + /// Index of a leaf register within the widest supported tree. An 8-leaf tree implements only the /// lower half of the range. pub(super) type LeafIndex = Bounded; @@ -236,3 +244,9 @@ pub(super) const fn validate(self, leaves: LeafCount) -> Result { } } } + +impl From for Bounded { + fn from(vector: GinVector) -> Self { + vector.0.extend() + } +} diff --git a/drivers/gpu/nova-core/irq/regs.rs b/drivers/gpu/nova-core/irq/regs.rs new file mode 100644 index 000000000000..4ddd01bcbb23 --- /dev/null +++ b/drivers/gpu/nova-core/irq/regs.rs @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use kernel::io::register; + +use crate::driver::NovaRegisters; + +use super::interrupt_tree::{ + LeafMask, + SubtreeSet, // +}; + +// The GIN CPU interrupt tree, reached through the `NV_VIRTUAL_FUNCTION_PRIV` aperture. See +// "Register naming" in `Documentation/gpu/nova/core/interrupts.rst`. The leaf arrays are declared +// with 16 entries, the widest tree that any supported chipset implements. + +register! { + base: NovaRegisters; + + /// Pending bits of one leaf, one per vector. + /// + /// Vector `v` is bit `v % 32` of leaf `v / 32`. The bit is set when the vector's source + /// drives it, whether or not the vector is enabled. Writing a `1` clears the bit, and a `0` + /// leaves it as it was. + pub(super) NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF(u32)[16] @ 0x00b81000 { + /// The vectors pending in this leaf. + 31:0 vectors => LeafMask; + } + + /// Enables vectors of one leaf. + /// + /// A `1` enables the matching vector, and a `0` leaves it as it was. + pub(super) NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF_EN_SET(u32)[16] @ 0x00b81200 { + /// Vectors to enable. + 31:0 vectors => LeafMask; + } + + /// Disables vectors of one leaf. + /// + /// A `1` disables the matching vector, and a `0` leaves it as it was. A disabled vector still + /// latches in `LEAF`, and `TOP` does not show it. + pub(super) NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF_EN_CLEAR(u32)[16] @ 0x00b81400 { + /// Vectors to disable. + 31:0 vectors => LeafMask; + } + + /// Enables subtrees. + /// + /// Bit `N` covers subtree `N`, which is leaves `2N` and `2N + 1`. A `1` enables the matching + /// subtree, and a `0` leaves it as it was. + /// + /// The hardware headers declare a one-element array, so nova-core declares a scalar. + pub(super) NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_TOP_EN_SET(u32) @ 0x00b81608 { + /// Subtrees to enable. + 31:0 subtrees => SubtreeSet; + } + + /// Disables subtrees, with the bit layout of `TOP_EN_SET`. + /// + /// A `1` disables the matching subtree, and a `0` leaves it as it was. A disabled subtree + /// delivers nothing, and `TOP` still reports it. + pub(super) NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_TOP_EN_CLEAR(u32) @ 0x00b81610 { + /// Subtrees to disable. + 31:0 subtrees => SubtreeSet; + } + + /// Latches a vector from software. Write-only. + /// + /// The write sets the pending bit of the vector in its `LEAF` register, as if the vector's + /// source had raised it, and the vector reaches the CPU under the same enables. Implemented on + /// every supported chipset. + pub(super) NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF_TRIGGER(u32) @ 0x00b81640 { + /// Vector to latch. + 11:0 vector; + } +} + +// PCI configuration-space mirror in BAR0. + +register! { + base: NovaRegisters; + + /// MSI end-of-interrupt register. Writing any value rearms MSI delivery. + /// + /// Only pre-Hopper MSI rearms through this register. See "Rearming PCI interrupt delivery" + /// in `Documentation/gpu/nova/core/interrupts.rst`. + pub(super) NV_XVE_CYA_2(u32) @ 0x00088704 {} +} From 4022bcaa34e2115e175f6a64022ec19a186b0d72 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Tue, 29 Sep 2026 20:41:38 -0700 Subject: [PATCH 093/110] gpu: nova-core: add the per-architecture GIN CPU interrupt HAL The GIN CPU interrupt tree differs by GPU family in two ways: * The size of the tree. Turing, Ampere and Ada implement 8 leaves. Hopper and later families implement 16. * The write that rearms delivery. A message-signaled interrupt is delivered once per edge, and the PCI side delivers nothing more until the CPU rearms it. Before Hopper, MSI rearms by writing the end-of-interrupt register in the BAR0 mirror of PCI configuration space. On Hopper and later, MSI rearms by clearing and then setting the TOP enables of every serviced subtree, which produces a new edge. MSI-X rearms the same way on every family, but for the handler's own subtree only, since each subtree has its own table entry. Add an interrupt HAL that provides the leaf count and performs the rearm for each family. Name the interrupt type with two variants, MSI and MSI-X, since nova-core never allocates the level-triggered INTx that the kernel's PCI interrupt type also names. The HAL has no user yet. Assisted-by: LLM Signed-off-by: John Hubbard Reviewed-by: Will Pierce Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260930034148.590687-6-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/irq.rs | 13 +++++ drivers/gpu/nova-core/irq/hal.rs | 67 ++++++++++++++++++++++++++ drivers/gpu/nova-core/irq/hal/gh100.rs | 40 +++++++++++++++ drivers/gpu/nova-core/irq/hal/tu102.rs | 43 +++++++++++++++++ 4 files changed, 163 insertions(+) create mode 100644 drivers/gpu/nova-core/irq/hal.rs create mode 100644 drivers/gpu/nova-core/irq/hal/gh100.rs create mode 100644 drivers/gpu/nova-core/irq/hal/tu102.rs diff --git a/drivers/gpu/nova-core/irq.rs b/drivers/gpu/nova-core/irq.rs index 1ec0bb055d3b..62c242b71dc8 100644 --- a/drivers/gpu/nova-core/irq.rs +++ b/drivers/gpu/nova-core/irq.rs @@ -9,5 +9,18 @@ //! //! See `Documentation/gpu/nova/core/interrupts.rst`. +mod hal; mod interrupt_tree; mod regs; + +/// The message-signaled interrupt type that Linux granted. +/// +/// nova-core never requests INTx, so this has no variant for it, unlike [`kernel::pci::IrqType`]. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum MsiType { + /// A single message, which every subtree raises. + Msi, + + /// One table entry per subtree. + MsiX, +} diff --git a/drivers/gpu/nova-core/irq/hal.rs b/drivers/gpu/nova-core/irq/hal.rs new file mode 100644 index 000000000000..387695cca37b --- /dev/null +++ b/drivers/gpu/nova-core/irq/hal.rs @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Per-architecture differences in the GIN CPU interrupt tree. +//! +//! See "Per-architecture differences" in `Documentation/gpu/nova/core/interrupts.rst`. + +mod gh100; +mod tu102; + +use kernel::{ + io::Io, + prelude::*, // +}; + +use crate::{ + driver::Bar0, + gpu::{ + Architecture, + Chipset, // + }, // +}; + +use super::{ + interrupt_tree::{ + LeafCount, + Subtree, + SubtreeSet, // + }, + regs::*, + MsiType, // +}; + +/// Clears and then sets the `TOP` enables of `subtrees`, which produces a new delivery edge. +fn cycle_top_enables(bar: Bar0<'_>, subtrees: SubtreeSet) { + bar.write_reg(NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_TOP_EN_CLEAR::zeroed().with_subtrees(subtrees)); + bar.write_reg(NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_TOP_EN_SET::zeroed().with_subtrees(subtrees)); +} + +/// The leaf count and the rearm of the GIN CPU tree, which differ by GPU family. +pub(super) trait CpuInterruptHal: Send + Sync { + /// Returns the number of leaves the tree implements. + fn leaf_count(&self) -> LeafCount; + + /// Rearms PCI interrupt delivery after a handler serviced `subtree`. + /// + /// `serviced` is every subtree that nova-core services, for the Hopper-plus MSI form, which + /// cycles the `TOP` enables of them all. See "Rearming PCI interrupt delivery" in + /// `Documentation/gpu/nova/core/interrupts.rst`. + fn rearm_pci_irq( + &self, + bar: Bar0<'_>, + msi_type: MsiType, + serviced: SubtreeSet, + subtree: Subtree, + ); +} + +/// Returns the [`CpuInterruptHal`] for `chipset`'s architecture. +pub(super) fn cpu_interrupt_hal(chipset: Chipset) -> &'static dyn CpuInterruptHal { + match chipset.arch() { + Architecture::Turing | Architecture::Ampere | Architecture::Ada => tu102::TU102_HAL, + Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { + gh100::GH100_HAL + } + } +} diff --git a/drivers/gpu/nova-core/irq/hal/gh100.rs b/drivers/gpu/nova-core/irq/hal/gh100.rs new file mode 100644 index 000000000000..6e307c8d120b --- /dev/null +++ b/drivers/gpu/nova-core/irq/hal/gh100.rs @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use crate::driver::Bar0; + +use super::{ + cycle_top_enables, + CpuInterruptHal, + LeafCount, + MsiType, + Subtree, + SubtreeSet, // +}; + +/// The CPU interrupt tree properties of Hopper and Blackwell. +struct Gh100; + +impl CpuInterruptHal for Gh100 { + fn leaf_count(&self) -> LeafCount { + LeafCount::Sixteen + } + + fn rearm_pci_irq( + &self, + bar: Bar0<'_>, + msi_type: MsiType, + serviced: SubtreeSet, + subtree: Subtree, + ) { + let subtrees = match msi_type { + MsiType::Msi => serviced, + MsiType::MsiX => subtree.into(), + }; + + cycle_top_enables(bar, subtrees); + } +} + +const GH100: Gh100 = Gh100; +pub(super) const GH100_HAL: &dyn CpuInterruptHal = &GH100; diff --git a/drivers/gpu/nova-core/irq/hal/tu102.rs b/drivers/gpu/nova-core/irq/hal/tu102.rs new file mode 100644 index 000000000000..7e3aea946d05 --- /dev/null +++ b/drivers/gpu/nova-core/irq/hal/tu102.rs @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use kernel::io::Io; + +use crate::{ + driver::Bar0, + irq::regs::NV_XVE_CYA_2, // +}; + +use super::{ + cycle_top_enables, + CpuInterruptHal, + LeafCount, + MsiType, + Subtree, + SubtreeSet, // +}; + +/// The CPU interrupt tree properties of Turing, Ampere, and Ada. +struct Tu102; + +impl CpuInterruptHal for Tu102 { + fn leaf_count(&self) -> LeafCount { + LeafCount::Eight + } + + fn rearm_pci_irq( + &self, + bar: Bar0<'_>, + msi_type: MsiType, + _serviced: SubtreeSet, + subtree: Subtree, + ) { + match msi_type { + MsiType::Msi => bar.write(NV_XVE_CYA_2, 0u32.into()), + MsiType::MsiX => cycle_top_enables(bar, subtree.into()), + } + } +} + +const TU102: Tu102 = Tu102; +pub(super) const TU102_HAL: &dyn CpuInterruptHal = &TU102; From 939b40877cb883de1fa335e950ff1a3f3bf113ed Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Tue, 29 Sep 2026 20:41:39 -0700 Subject: [PATCH 094/110] gpu: nova-core: add the GIN interrupt tree and allocate its vectors Servicing a GIN leaf has a required order: read its pending bits, then clear them. Clearing a leaf first discards every vector latched in it, and the hardware keeps no record of what was discarded. The interrupt never arrives, and no register shows that it was ever pending. Every subtree enabled at TOP also needs an allocated PCI vector with a handler registered on it. Under MSI-X each subtree has its own table entry. Linux masks every entry until a driver requests its IRQ, and a masked entry sends no message. An enabled subtree whose entry was never requested raises interrupts that never reach a handler, while the leaf and TOP registers show them pending and enabled. Under MSI the whole tree raises a single message, so one entry serves every subtree. Add the CPU interrupt tree of one PCIe function. The tree allocates the PCI vectors for the subtrees that it services and hands out the request for each, so that the chipset is given once and the vectors live as long as the tree. Reading a leaf yields the handle that clears it, so that the wrong order does not compile. Building a tree fails if it names a subtree that the GPU does not implement. A reset disables every vector, clears every pending bit, rearms delivery, and leaves the serviced subtrees disabled at TOP, so that a handler registered afterwards receives no interrupt left over from boot. Request MSI-X entries 0 through the highest serviced subtree, since an MSI-X allocation cannot be sparse, and fall back to a single MSI message, never to INTx. [jhubbard: reworked on top of the GIN vector types: a leaf read yields the handle that clears it, enables are guarded, the leaf count and the rearm come from the interrupt HAL, the drain reads every implemented leaf rather than descending from TOP, the tree owns its PCI vectors, and a reset returns the tree to a known state before a handler is registered] Signed-off-by: Joel Fernandes Signed-off-by: John Hubbard Reviewed-by: Will Pierce Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260930034148.590687-7-jhubbard@nvidia.com [acourbot: keep item that doesn't need to be exported private.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/irq.rs | 2 +- drivers/gpu/nova-core/irq/interrupt_tree.rs | 296 +++++++++++++++++++- 2 files changed, 292 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/nova-core/irq.rs b/drivers/gpu/nova-core/irq.rs index 62c242b71dc8..b2cfe73af114 100644 --- a/drivers/gpu/nova-core/irq.rs +++ b/drivers/gpu/nova-core/irq.rs @@ -10,7 +10,7 @@ //! See `Documentation/gpu/nova/core/interrupts.rst`. mod hal; -mod interrupt_tree; +pub(crate) mod interrupt_tree; mod regs; /// The message-signaled interrupt type that Linux granted. diff --git a/drivers/gpu/nova-core/irq/interrupt_tree.rs b/drivers/gpu/nova-core/irq/interrupt_tree.rs index 279a43514b1f..da4203137bce 100644 --- a/drivers/gpu/nova-core/irq/interrupt_tree.rs +++ b/drivers/gpu/nova-core/irq/interrupt_tree.rs @@ -1,22 +1,47 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -//! Vector addressing in the GIN CPU interrupt tree. +//! The GIN CPU interrupt tree for one PCIe function. //! //! A [`GinVector`] names an interrupt source, a [`LeafIndex`] the leaf register that latches it, //! a [`LeafMask`] a set of vectors within one leaf, and a [`Subtree`] one `TOP` bit. The types //! keep the four from being confused with one another. //! +//! Servicing a leaf requires reading its pending bits before clearing them. Only +//! [`Tree::read_pending`] produces a [`LeafPending`], and only a [`LeafPending`] clears a leaf, +//! so that the wrong order does not compile. This module does not serialize access to the tree. +//! //! See `Documentation/gpu/nova/core/interrupts.rst`. use kernel::{ + device::Bound, + io::{ + register::Array, + Io, // + }, + irq, num::Bounded, + pci::{ + self, + IrqType, // + }, prelude::*, // }; -use crate::num; +use crate::{ + driver::Bar0, + gpu::Chipset, + num, // +}; -use super::regs::*; +use super::{ + hal::{ + cpu_interrupt_hal, + CpuInterruptHal, // + }, + regs::*, + MsiType, // +}; /// Number of vectors one leaf register carries, one per bit. const VECTORS_PER_LEAF: u32 = u32::BITS; @@ -78,6 +103,11 @@ pub(super) const fn subtree_set(self) -> SubtreeSet { pub(super) const fn vector_count(self) -> u32 { self.into_u32() * VECTORS_PER_LEAF } + + /// Returns every leaf a tree of this size implements. + pub(super) fn iter(self) -> impl Iterator { + (0..self.into_raw()).filter_map(LeafIndex::try_new) + } } // `VECTOR_BITS` and `LeafCount::Sixteen` are written separately. This assert keeps them in @@ -151,7 +181,7 @@ pub(super) const fn into_raw(self) -> u32 { /// Set of subtrees, one bit per subtree, in the layout of the `TOP` registers. #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) struct SubtreeSet(u32); +pub(crate) struct SubtreeSet(u32); impl SubtreeSet { pub(super) const fn contains(self, subtree: Subtree) -> bool { @@ -173,7 +203,6 @@ pub(super) const fn span(self) -> u32 { } /// Returns the subtrees of this set, lowest index first. - #[expect(dead_code)] pub(super) fn iter(self) -> impl Iterator { (0..u32::BITS) .map(Subtree::new) @@ -250,3 +279,260 @@ fn from(vector: GinVector) -> Self { vector.0.extend() } } + +/// Disables `vectors` in `leaf`. +fn clear_leaf_enables(bar: Bar0<'_>, leaf: LeafIndex, vectors: LeafMask) { + bar.write( + Array::at(*leaf), + NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF_EN_CLEAR::zeroed().with_vectors(vectors), + ); +} + +/// Disables the subtrees in `serviced` at `TOP`. +fn clear_top_enables(bar: Bar0<'_>, serviced: SubtreeSet) { + bar.write_reg(NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_TOP_EN_CLEAR::zeroed().with_subtrees(serviced)); +} + +/// The CPU tree of one PCIe function, the subtrees that nova-core services, and the PCI vectors +/// that deliver them. +/// +/// A subtree may be enabled at `TOP` only once a handler is registered on the PCI vector that +/// delivers it. See "The serviced-subtree invariant" in +/// `Documentation/gpu/nova/core/interrupts.rst`. +pub(crate) struct Tree<'a> { + bar: Bar0<'a>, + hal: &'static dyn CpuInterruptHal, + serviced: SubtreeSet, + /// The interrupt type that Linux granted for `vectors`. + pub(super) msi_type: MsiType, + vectors: pci::IrqVectorRegistration<'a>, +} + +impl<'a> Tree<'a> { + /// Creates the tree of `chipset` and allocates the PCI vectors for the subtrees in `serviced`. + /// + /// Requests MSI-X entries `0` through the highest subtree in `serviced`, since an allocation + /// cannot be sparse, and falls back to a single MSI message for the whole tree. + /// + /// # Errors + /// + /// `EINVAL` if `serviced` is empty or names a subtree that `chipset` does not implement. + /// Otherwise, when neither interrupt type could be allocated, the error from the MSI request. + pub(crate) fn new( + pdev: &'a pci::Device, + bar: Bar0<'a>, + chipset: Chipset, + serviced: SubtreeSet, + ) -> Result { + let hal = cpu_interrupt_hal(chipset); + let num_leaves = hal.leaf_count(); + + if serviced.is_empty() || serviced.intersection(num_leaves.subtree_set()) != serviced { + return Err(EINVAL); + } + + let entries = serviced.span(); + let (vectors, msi_type) = pdev + .alloc_irq_vectors(entries, entries, IrqType::MsiX.into()) + .map(|vectors| (vectors, MsiType::MsiX)) + // Every subtree raises the one MSI message, so one MSI vector serves the whole tree. + // See "Delivery over PCI" in `Documentation/gpu/nova/core/interrupts.rst`. + .or_else(|_| { + pdev.alloc_irq_vectors(1, 1, IrqType::Msi.into()) + .map(|vectors| (vectors, MsiType::Msi)) + })?; + + Ok(Self { + bar, + hal, + serviced, + msi_type, + vectors, + }) + } + + /// Returns the [`irq::IrqRequest`] for the PCI vector that delivers `subtree`. + /// + /// # Errors + /// + /// `EINVAL` if `subtree` is not one of the serviced subtrees. + pub(super) fn request_for(&self, subtree: Subtree) -> Result> { + if !self.serviced.contains(subtree) { + return Err(EINVAL); + } + + let entry = match self.msi_type { + MsiType::MsiX => num::u32_as_usize(subtree.index()), + MsiType::Msi => 0, + }; + + self.vectors.index(entry).map(Into::into) + } + + /// Rearms PCI interrupt delivery to the CPU after servicing `subtree`, the one subtree that + /// the calling handler serves. + /// + /// A handler must call this before returning, or it receives no further interrupts. + pub(super) fn rearm_pci_irq(&self, subtree: Subtree) { + self.hal + .rearm_pci_irq(self.bar, self.msi_type, self.serviced, subtree); + } + + /// Enables the serviced subtrees at `TOP`. + /// + /// Each of them must have a handler registered on its PCI vector. + pub(super) fn enable_top(&self) { + self.bar.write_reg( + NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_TOP_EN_SET::zeroed().with_subtrees(self.serviced), + ); + } + + /// Disables the serviced subtrees at `TOP`. + pub(super) fn disable_top(&self) { + clear_top_enables(self.bar, self.serviced); + } + + /// Enables the serviced subtrees at `TOP` until the returned guard drops. + pub(crate) fn enable_top_guarded(&self) -> TopEnableGuard<'a> { + self.enable_top(); + + TopEnableGuard { + bar: self.bar, + serviced: self.serviced, + } + } + + /// Enables `vectors` in `leaf`. + pub(super) fn enable_leaf(&self, leaf: LeafIndex, vectors: LeafMask) { + self.bar.write( + Array::at(*leaf), + NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF_EN_SET::zeroed().with_vectors(vectors), + ); + } + + /// Disables `vectors` in `leaf`. + pub(super) fn disable_leaf(&self, leaf: LeafIndex, vectors: LeafMask) { + clear_leaf_enables(self.bar, leaf, vectors); + } + + /// Enables `vectors` in `leaf` until the returned guard drops. + pub(super) fn enable_leaf_guarded( + &self, + leaf: LeafIndex, + vectors: LeafMask, + ) -> LeafEnableGuard<'a> { + self.enable_leaf(leaf, vectors); + + LeafEnableGuard { + bar: self.bar, + leaf, + vectors, + } + } + + /// Reads the pending bits of `leaf`, and returns the handle that clears them. + pub(super) fn read_pending(&self, leaf: LeafIndex) -> LeafPending<'a> { + let pending = self + .bar + .read(NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF::at(*leaf)) + .vectors(); + + LeafPending { + bar: self.bar, + leaf, + pending, + } + } + + /// Disables every vector in every implemented leaf, including the subtrees that nova-core does + /// not service. Call this only during probe. + pub(super) fn disable_all_leaves(&self) { + for leaf in self.hal.leaf_count().iter() { + self.disable_leaf(leaf, LeafMask::all()); + } + } + + /// Clears every pending bit in every implemented leaf, including the subtrees that nova-core + /// does not service. + /// + /// The serviced subtrees are disabled at `TOP` on return. Call this only during probe, with no + /// interrupt handler registered. + pub(super) fn drain(&self) { + self.disable_top(); + + // A vector that latched while disabled does not show in `TOP`, so read every leaf rather + // than descending from it. + for leaf in self.hal.leaf_count().iter() { + self.read_pending(leaf).clear(); + } + } + + /// Disables every vector, clears every pending bit, and rearms PCI interrupt delivery. + /// + /// The serviced subtrees are disabled at `TOP` on return. Call this only during probe, with + /// no interrupt handler registered. + pub(super) fn reset(&self) { + self.disable_all_leaves(); + self.drain(); + for subtree in self.serviced.iter() { + self.rearm_pci_irq(subtree); + } + + // A `TOP`-enable rearm leaves the subtrees that it cycles enabled, and a subtree may be + // enabled only once a handler is registered on its vector. + self.disable_top(); + } +} + +/// The pending bits of one leaf as they were read, and the handle that clears them. +pub(super) struct LeafPending<'a> { + bar: Bar0<'a>, + leaf: LeafIndex, + pending: LeafMask, +} + +impl LeafPending<'_> { + pub(super) fn vectors(&self) -> LeafMask { + self.pending + } + + /// Clears the vectors that were pending at the read. A vector that latched since stays pending. + pub(super) fn clear(&self) { + self.clear_vectors(self.pending); + } + + /// Clears `vectors` and no other bit. + pub(super) fn clear_vectors(&self, vectors: LeafMask) { + if !vectors.is_empty() { + self.bar.write( + Array::at(*self.leaf), + NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF::zeroed().with_vectors(vectors), + ); + } + } +} + +/// Disables a set of vectors in one leaf when dropped. +pub(super) struct LeafEnableGuard<'a> { + bar: Bar0<'a>, + leaf: LeafIndex, + vectors: LeafMask, +} + +impl Drop for LeafEnableGuard<'_> { + fn drop(&mut self) { + clear_leaf_enables(self.bar, self.leaf, self.vectors); + } +} + +/// Disables the serviced subtrees at `TOP` when dropped. +pub(crate) struct TopEnableGuard<'a> { + bar: Bar0<'a>, + serviced: SubtreeSet, +} + +impl Drop for TopEnableGuard<'_> { + fn drop(&mut self) { + clear_top_enables(self.bar, self.serviced); + } +} From 94ddc4b4843da8b1858535b13318346cb950800d Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Tue, 29 Sep 2026 20:41:40 -0700 Subject: [PATCH 095/110] gpu: nova-core: move wait for GFW boot code to associated function The GPU boots its own firmware, GFW, out of reset, and the driver must not program the GPU until GFW reports completion. nova-core waited for GFW inside the Gpu constructor, which also boots the GSP. Code that has to run after GFW and before GSP boot, such as a probe-time hardware self-test, had nowhere to go. Move the wait code into an associated function so it can also be called by test code that requires it before the Gpu instance is constructed. Suggested-by: Danilo Krummrich Link: https://lore.kernel.org/all/DLSNXUU1UXIH.JG4L0L5ZFV89@kernel.org/ Assisted-by: LLM Signed-off-by: John Hubbard Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260930034148.590687-8-jhubbard@nvidia.com [acourbot: move code to associated function, move relevant hunks to next patch.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gpu.rs | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index fb6f8a86a503..a7f6b1806f36 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -397,10 +397,8 @@ pub(crate) fn new<'a>( dev_info!(dev,"NVIDIA ({})\n", spec); })?, - // We must wait for GFW_BOOT completion before doing any significant setup on the GPU. _: { - let hal = hal::gpu_hal(spec.chipset); - let dma_mask = hal.dma_mask(); + let dma_mask = hal::gpu_hal(spec.chipset).dma_mask(); // SAFETY: `Gpu` owns all DMA allocations for this device, and we are // still constructing it, so no concurrent DMA allocations can exist. @@ -413,8 +411,8 @@ pub(crate) fn new<'a>( // still constructing it, so no concurrent DMA allocations can exist. unsafe { pdev.dma_set_max_seg_size(u32::MAX) }; - hal.wait_gfw_boot_completion(bar) - .inspect_err(|_| dev_err!(dev, "GFW boot did not complete\n"))?; + // We must wait for GFW_BOOT completion before doing any significant setup on the GPU. + Self::wait_gfw_boot_completion(dev, bar, spec.chipset)?; }, // Initialize this early because `gsp_resources` depends on it. @@ -513,6 +511,23 @@ pub(crate) fn new<'a>( }) } + /// Waits for GFW, the GPU's boot firmware, to report completion. + /// + /// The driver must not program the GPU before then. + /// + /// # Errors + /// + /// `ETIMEDOUT` if GFW does not report completion in time. + pub(crate) fn wait_gfw_boot_completion( + dev: &device::Device, + bar: Bar0<'_>, + chipset: Chipset, + ) -> Result { + hal::gpu_hal(chipset) + .wait_gfw_boot_completion(bar) + .inspect_err(|_| dev_err!(dev, "GFW boot did not complete\n")) + } + /// Runs self-tests on the constructed [`Gpu`], logging failures without failing probe. #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] pub(crate) fn run_selftests(self: Pin<&mut Self>, pdev: &pci::Device) { From 9cb3f5d56e81fd97a82ef361b210bc2322daaaa7 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Fri, 2 Oct 2026 16:11:35 +0900 Subject: [PATCH 096/110] gpu: nova-core: add an interrupt delivery self-test A GPU interrupt can be lost in the MSI or MSI-X allocation, in the GIN tree's enables, or in the rearm, and every one of those failures looks the same: no interrupt arrives, and no register or log line says which one broke. Add a probe-time self-test, built under NOVA_CORE_SELFTESTS, that latches the CPU doorbell vector through the GIN software trigger and waits for a registered handler to service it. One delivery would pass with a broken rearm, because the first message-signaled interrupt arrives whether or not the driver rearms, so the test triggers twice and waits for the first handler to finish before the second trigger. It runs after GFW boot and before GSP boot, on a quiesced tree, and fails probe unless both deliveries arrive, each finds only the doorbell pending, and the leaf ends clear. The doorbell has the same vector on every supported GPU, so the test names it without asking GSP-RM. It allocates the PCI vectors for the doorbell's subtree and releases them before returning, so under MSI-X the delivery also exercises that subtree's table entry. Assisted-by: LLM Co-developed-by: Joel Fernandes Signed-off-by: Joel Fernandes Signed-off-by: John Hubbard Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260930034148.590687-9-jhubbard@nvidia.com [acourbot: move relevant bits of previous patch into this one.] [acourbot: remove reference to self-test section of documentation.] [acourbot: replace completions with atomic polling code.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/Kconfig | 5 + drivers/gpu/nova-core/driver.rs | 19 +- drivers/gpu/nova-core/gpu.rs | 2 +- drivers/gpu/nova-core/irq.rs | 2 + drivers/gpu/nova-core/irq/doorbell_test.rs | 251 ++++++++++++++++++++ drivers/gpu/nova-core/irq/interrupt_tree.rs | 15 ++ drivers/gpu/nova-core/nova_core.rs | 2 +- 7 files changed, 293 insertions(+), 3 deletions(-) create mode 100644 drivers/gpu/nova-core/irq/doorbell_test.rs diff --git a/drivers/gpu/nova-core/Kconfig b/drivers/gpu/nova-core/Kconfig index 1934f17baa8b..2e11e46c99c7 100644 --- a/drivers/gpu/nova-core/Kconfig +++ b/drivers/gpu/nova-core/Kconfig @@ -24,4 +24,9 @@ config NOVA_CORE_SELFTESTS help Build the driver self-tests and run them when the GPU is probed. + If the interrupt delivery test fails, the probe fails and the driver + does not bind to the GPU. A broken interrupt path would otherwise + show up later as a hang, far from its cause. Every other self-test + logs its failure and lets the probe continue. + If unsure, say N. diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index 623d59dcde6e..d195fd2bcf8c 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -24,7 +24,10 @@ use crate::{ api::NovaCoreApi, - gpu::Gpu, // + gpu::{ + Gpu, + Spec, // + }, // }; /// Counter for generating unique auxiliary device IDs. @@ -115,6 +118,20 @@ fn probe<'bound>( pdev.iomap_region(bar1_idx, c"nova-core/bar1")? }, + // Run self-tests that do not depend on the `Gpu` instance. + #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] + _: { + let spec = Spec::new(pdev.as_ref(), bar)?; + + // We must wait for GFW_BOOT completion before doing any significant setup on + // the GPU. + Gpu::wait_gfw_boot_completion(pdev.as_ref(), bar, spec.chipset)?; + + // The self-test disables and drains the whole tree, so it has to run before + // `Gpu::new` boots the GSP. + crate::irq::doorbell_test::run_selftest(pdev, bar, spec.chipset)?; + }, + // TODO: Use self-referential pin-init syntax once available. gpu <- Gpu::new( pdev, diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index a7f6b1806f36..f91c7e22a303 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -251,7 +251,7 @@ pub struct Spec { } impl Spec { - fn new(dev: &device::Device, bar: Bar0<'_>) -> Result { + pub(crate) fn new(dev: &device::Device, bar: Bar0<'_>) -> Result { // Some brief notes about boot0 and boot42, in chronological order: // // NV04 through NV50: diff --git a/drivers/gpu/nova-core/irq.rs b/drivers/gpu/nova-core/irq.rs index b2cfe73af114..ff8b00a82442 100644 --- a/drivers/gpu/nova-core/irq.rs +++ b/drivers/gpu/nova-core/irq.rs @@ -9,6 +9,8 @@ //! //! See `Documentation/gpu/nova/core/interrupts.rst`. +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +pub(crate) mod doorbell_test; mod hal; pub(crate) mod interrupt_tree; mod regs; diff --git a/drivers/gpu/nova-core/irq/doorbell_test.rs b/drivers/gpu/nova-core/irq/doorbell_test.rs new file mode 100644 index 000000000000..db6338e4dbde --- /dev/null +++ b/drivers/gpu/nova-core/irq/doorbell_test.rs @@ -0,0 +1,251 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Interrupt delivery self-test. +//! +//! The test triggers the CPU doorbell vector from software, twice, and checks that each trigger +//! reaches a registered handler. It runs during probe under `CONFIG_NOVA_CORE_SELFTESTS`. + +use core::pin::Pin; + +use kernel::{ + device::Bound, + io::poll::read_poll_timeout, + irq, + pci, + prelude::*, + sync::atomic::{ + Acquire, + Atomic, + Relaxed, + Release, // + }, + time::Delta, // +}; + +use super::interrupt_tree::{ + GinVector, + LeafEnableGuard, + LeafMask, + Subtree, + TopEnableGuard, + Tree, // +}; + +use crate::{ + driver::Bar0, + gpu::Chipset, + selftest_assert, + selftest_assert_eq, // +}; + +/// The CPU doorbell vector. Every supported GPU uses this number, so the test needs nothing from +/// GSP-RM, which is not running yet. +const DOORBELL_VECTOR: GinVector = GinVector::new::<129>(); + +/// The only subtree that this test services. +const DOORBELL_SUBTREE: Subtree = DOORBELL_VECTOR.subtree(); + +/// Time allowed for each delivery to arrive. +const DELIVERY_TIMEOUT: Delta = Delta::from_millis(1000); + +/// The self-test's interrupt handler. +/// +/// It clears only the doorbell's bit, rearms delivery, and never walks the tree. A missing rearm +/// shows up as a timeout on the second delivery. +#[pin_data] +struct DoorbellTestHandler<'a> { + tree: &'a Tree<'a>, + /// Deliveries that found the doorbell bit set. + irq_count: Atomic, + /// The doorbell leaf's pending bits, as read by the first delivery. + first_pending: Atomic, + /// The doorbell leaf's pending bits, as read by the second delivery. + second_pending: Atomic, +} + +impl irq::Handler for DoorbellTestHandler<'_> { + fn handle(&self) -> irq::IrqReturn { + let leaf = self.tree.read_pending(DOORBELL_VECTOR.leaf_index()); + let pending = leaf.vectors(); + if !pending.contains(DOORBELL_VECTOR.leaf_mask()) { + self.tree.rearm_pci_irq(DOORBELL_SUBTREE); + return irq::IrqReturn::None; + } + leaf.clear_vectors(DOORBELL_VECTOR.leaf_mask()); + // Rearm before counting the delivery, since the waiting thread triggers the next doorbell + // as soon as it observes the count. + self.tree.rearm_pci_irq(DOORBELL_SUBTREE); + + match self.irq_count.load(Relaxed) { + 0 => self.first_pending.store(pending.into_raw(), Relaxed), + 1 => self.second_pending.store(pending.into_raw(), Relaxed), + _ => (), + } + self.irq_count.fetch_add(1, Release); + + irq::IrqReturn::Handled + } +} + +/// The self-test's handler registration and the enables that deliver to it. +/// +/// Drops in the order that "Enabling the GSP event" in +/// `Documentation/gpu/nova/core/interrupts.rst` requires: the vector is disabled, then the +/// handler is freed, then the subtree is disabled. +struct SelftestResources<'a, 'r> { + _leaf_guard: LeafEnableGuard<'a>, + reg: Pin>>>, + _top_guard: TopEnableGuard<'a>, +} + +impl<'a> SelftestResources<'a, '_> { + fn handler(&self) -> &DoorbellTestHandler<'a> { + self.reg.handler() + } + + /// Disables the doorbell vector and waits for a handler in flight on another CPU to finish. + /// + /// The handler's counters and the leaf's pending bits are final on return. + fn quiesce_source(&self) { + self.handler() + .tree + .disable_leaf(DOORBELL_VECTOR.leaf_index(), DOORBELL_VECTOR.leaf_mask()); + self.reg.synchronize(); + } +} + +/// Waits until `handler` has serviced `deliveries` interrupts, or [`DELIVERY_TIMEOUT`] elapses. +/// +/// Returns `false` on timeout. +fn wait_for_deliveries(handler: &DoorbellTestHandler<'_>, deliveries: u32) -> bool { + const DELIVERY_POLL_INTERVAL: Delta = Delta::from_micros(100); + + read_poll_timeout( + || Ok(handler.irq_count.load(Acquire)), + |count| *count >= deliveries, + DELIVERY_POLL_INTERVAL, + DELIVERY_TIMEOUT, + ) + .is_ok() +} + +/// Runs the interrupt delivery self-test. +/// +/// Call this only during probe, before GSP boot: it disables every vector in the tree and clears +/// every pending bit. On return, the doorbell's subtree is disabled at `TOP`, and the test's PCI +/// vectors and handler are released. +/// +/// # Errors +/// +/// `EINVAL` if `chipset` does not implement the doorbell's subtree. `ETIMEDOUT` if a delivery +/// does not arrive within [`DELIVERY_TIMEOUT`]. `EIO` if a self-test assertion fails. +/// Otherwise the error from allocating the PCI vectors or registering the handler. +pub(crate) fn run_selftest(pdev: &pci::Device, bar: Bar0<'_>, chipset: Chipset) -> Result { + let tree = Tree::new(pdev, bar, chipset, DOORBELL_SUBTREE.into())?; + let tree_ref = &tree; + let request = tree.request_for(DOORBELL_SUBTREE)?; + let doorbell = DOORBELL_VECTOR.leaf_index(); + let doorbell_mask = DOORBELL_VECTOR.leaf_mask(); + + dev_info!( + pdev, + "interrupt self-test: starting on vector {}, subtree {}, with {:?}\n", + DOORBELL_VECTOR.into_raw(), + DOORBELL_SUBTREE.index(), + tree.msi_type, + ); + + // GFW boot can leave vectors enabled and pending. Registering a handler unmasks the PCI + // interrupt, and they would be delivered to a handler that services only the doorbell. + tree.reset(); + + // A delivery proves nothing unless the doorbell bit starts out clear. + let pre_pending = tree.read_pending(doorbell).vectors(); + selftest_assert!( + pdev, + !pre_pending.contains(doorbell_mask), + "vector {} already pending, leaf[{}] is {:#x}", + DOORBELL_VECTOR.into_raw(), + doorbell.get(), + pre_pending.into_raw() + ); + + let handler_init = try_pin_init!(DoorbellTestHandler { + tree: tree_ref, + irq_count: Atomic::new(0), + first_pending: Atomic::new(0), + second_pending: Atomic::new(0), + }? Error); + + // Registration must precede any enable, or a delivery reaches no handler. + let reg = KBox::pin_init( + // SAFETY: this registration is dropped before the enclosing function returns, so its + // `Drop`, which calls `free_irq()`, always runs. + unsafe { + irq::Registration::new( + request, + irq::Flags::TRIGGER_NONE, + c"nova-core-selftest", + handler_init, + ) + }, + GFP_KERNEL, + )?; + + let resources = SelftestResources { + _leaf_guard: tree.enable_leaf_guarded(doorbell, doorbell_mask), + _top_guard: tree.enable_top_guarded(), + reg, + }; + let handler = resources.handler(); + + tree.trigger(DOORBELL_VECTOR)?; + let mut completed = wait_for_deliveries(handler, 1); + + // The second trigger waits for the first delivery, or the two could coalesce. + if completed { + tree.trigger(DOORBELL_VECTOR)?; + completed = wait_for_deliveries(handler, 2); + } + + resources.quiesce_source(); + + let count = handler.irq_count.load(Relaxed); + let first_pending = LeafMask::from_raw(handler.first_pending.load(Relaxed)); + let second_pending = LeafMask::from_raw(handler.second_pending.load(Relaxed)); + let residual = tree.read_pending(doorbell).vectors(); + + if !completed { + dev_err!( + pdev, + "interrupt self-test: only {} of 2 deliveries arrived within {} ms\n", + count, + DELIVERY_TIMEOUT.as_millis(), + ); + return Err(ETIMEDOUT); + } + + selftest_assert_eq!(pdev, count, 2, "delivery count"); + + // Every other vector in the leaf is disabled and was drained, so require the exact mask. + selftest_assert_eq!(pdev, first_pending, doorbell_mask, "first delivery"); + selftest_assert_eq!(pdev, second_pending, doorbell_mask, "second delivery"); + selftest_assert!( + pdev, + !residual.contains(doorbell_mask), + "vector {} still pending, leaf[{}] is {:#x}", + DOORBELL_VECTOR.into_raw(), + doorbell.get(), + residual.into_raw() + ); + + dev_info!( + pdev, + "interrupt self-test: passed, subtree {}, {} deliveries\n", + DOORBELL_SUBTREE.index(), + count, + ); + + Ok(()) +} diff --git a/drivers/gpu/nova-core/irq/interrupt_tree.rs b/drivers/gpu/nova-core/irq/interrupt_tree.rs index da4203137bce..c1576ea3e61f 100644 --- a/drivers/gpu/nova-core/irq/interrupt_tree.rs +++ b/drivers/gpu/nova-core/irq/interrupt_tree.rs @@ -444,6 +444,21 @@ pub(super) fn read_pending(&self, leaf: LeafIndex) -> LeafPending<'a> { } } + /// Sets the pending bit of `vector` in its leaf, as if the vector's source had raised it. + /// + /// # Errors + /// + /// `EINVAL` if this tree does not implement `vector`. + #[cfg_attr(not(CONFIG_NOVA_CORE_SELFTESTS), expect(dead_code))] + pub(super) fn trigger(&self, vector: GinVector) -> Result { + vector.validate(self.hal.leaf_count())?; + self.bar.write_reg( + NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF_TRIGGER::zeroed().with_vector(vector), + ); + + Ok(()) + } + /// Disables every vector in every implemented leaf, including the subtrees that nova-core does /// not service. Call this only during probe. pub(super) fn disable_all_leaves(&self) { diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 8202c4982efa..8c0761dedda4 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -18,7 +18,7 @@ mod fsp; mod gpu; mod gsp; -#[expect(dead_code)] +#[cfg_attr(not(CONFIG_NOVA_CORE_SELFTESTS), expect(dead_code))] mod irq; mod mctp; mod mm; From 0c551cdc465b9c99da33a21d6ee3b274cfda7006 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Tue, 29 Sep 2026 20:41:42 -0700 Subject: [PATCH 097/110] gpu: nova-core: log GSP events instead of discarding them The GSP posts unsolicited messages on the same queue that carries command replies: log records, OS error and robust-channel records, and lifecycle notices. nova-core discarded every message that was not the reply that a caller was waiting for, and an unrecognized function code aborted the in-flight command. The GSP's error reports never reached the kernel log. Log every non-reply message according to its function code, on the receive path that already reads it, and leave the in-flight command waiting for its reply. Event payloads, such as XID numbers and log contents, are not decoded. Assisted-by: LLM Signed-off-by: John Hubbard Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260930034148.590687-10-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/cmdq.rs | 52 ++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index a5595da23407..4591f6c61ce6 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -559,8 +559,7 @@ fn notify_gsp(bar: Bar0<'_>) { /// Sends `command` to the GSP and waits for the reply. /// - /// Messages with non-matching function codes are silently consumed until the expected reply - /// arrives. + /// Events that arrive before the reply are logged and consumed. /// /// The queue is locked for the entire send+receive cycle to ensure that no other command can /// be interleaved. @@ -819,8 +818,8 @@ fn wait_for_msg(&self, timeout: Delta) -> Result> { /// Receive a message from the GSP. /// - /// The expected message type is specified using the `M` generic parameter. If the pending - /// message has a different function code, `ERANGE` is returned and the message is consumed. + /// A message whose function code is `M::FUNCTION` is decoded and returned. Any other message + /// is logged as an event. /// /// The read pointer is always advanced past the message, regardless of whether it matched. /// @@ -829,8 +828,7 @@ fn wait_for_msg(&self, timeout: Delta) -> Result> { /// - `ETIMEDOUT` if `timeout` has elapsed before any message becomes available. /// - `EIO` if there was some inconsistency (e.g. message shorter than advertised) on the /// message queue. - /// - `EINVAL` if the function code of the message was not recognized. - /// - `ERANGE` if the message had a recognized but non-matching function code. + /// - `ERANGE` if the message was not the awaited reply. /// /// Error codes returned by [`MessageFromGsp::read`] are propagated as-is. fn receive_msg(&mut self, timeout: Delta) -> Result @@ -839,11 +837,11 @@ fn receive_msg(&mut self, timeout: Delta) -> Result Error: From, { let message = self.wait_for_msg(timeout)?; - let function = message.header.function().map_err(|_| EINVAL)?; + let function = message.header.function(); + let seq = message.header.sequence(); - // Extract the message. Store the result as we want to advance the read pointer even in - // case of failure. - let result = if function == M::FUNCTION { + // An early return here would leave the read pointer on this message. + let result = if matches!(function, Ok(f) if f == M::FUNCTION) { let (cmd, contents_1) = M::Message::from_bytes_prefix(message.contents.0).ok_or(EIO)?; let mut sbuffer = SBufferIter::new_reader([contents_1, message.contents.1]); @@ -854,11 +852,13 @@ fn receive_msg(&mut self, timeout: Delta) -> Result dev_warn!( &self.dev, "GSP message {:?} has unprocessed data\n", - function + M::FUNCTION ); } }) } else { + self.log_event(function, seq); + Err(ERANGE) }; @@ -869,4 +869,34 @@ fn receive_msg(&mut self, timeout: Delta) -> Result result } + + /// Logs an event, meaning a message that no caller was waiting for. + /// + /// An OS error or robust-channel record is logged at error level and an unknown function code + /// at warning level. Every other event is recorded only by the receive trace in + /// [`Self::wait_for_msg`]. + fn log_event(&self, function: Result, seq: u32) { + match function { + Ok(MsgFunction::OsErrorLog) => { + dev_err!(&self.dev, "GSP reported an OS error (seq {})\n", seq); + } + Ok(MsgFunction::RcTriggered) => { + dev_err!( + &self.dev, + "GSP triggered robust-channel recovery (seq {})\n", + seq + ); + } + // Nothing to do for the remaining known function codes. + Ok(_) => {} + Err(raw) => { + dev_warn!( + &self.dev, + "unknown GSP message function {:#x} (seq {})\n", + raw, + seq + ); + } + } + } } From d3cb3ed1a2ddd7cb77d9d270d82d3177ee744d1f Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Tue, 29 Sep 2026 20:41:43 -0700 Subject: [PATCH 098/110] gpu: nova-core: return ENOMSG for an unmatched GSP message The GSP posts unsolicited events on the same queue as command replies, so a receive that asks for one message type has to report that the message at the queue head was a different one. That case returned ERANGE, which means a value outside a valid range and says nothing about a message. Return ENOMSG, no message of the desired type, instead. Suggested-by: Gary Guo Suggested-by: Alexandre Courbot Assisted-by: LLM Signed-off-by: John Hubbard Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260930034148.590687-11-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/cmdq.rs | 6 +++--- drivers/gpu/nova-core/gsp/commands.rs | 2 +- drivers/gpu/nova-core/gsp/sequencer.rs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index 4591f6c61ce6..ad2a14d08cb0 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -585,7 +585,7 @@ pub(crate) fn send_command(&self, command: M) -> Result loop { match inner.receive_msg::(Self::RECEIVE_TIMEOUT) { Ok(reply) => break Ok(reply), - Err(ERANGE) => continue, + Err(ENOMSG) => continue, Err(e) => break Err(e), } } @@ -828,7 +828,7 @@ fn wait_for_msg(&self, timeout: Delta) -> Result> { /// - `ETIMEDOUT` if `timeout` has elapsed before any message becomes available. /// - `EIO` if there was some inconsistency (e.g. message shorter than advertised) on the /// message queue. - /// - `ERANGE` if the message was not the awaited reply. + /// - `ENOMSG` if the message was not the awaited reply. /// /// Error codes returned by [`MessageFromGsp::read`] are propagated as-is. fn receive_msg(&mut self, timeout: Delta) -> Result @@ -859,7 +859,7 @@ fn receive_msg(&mut self, timeout: Delta) -> Result } else { self.log_event(function, seq); - Err(ERANGE) + Err(ENOMSG) }; // Advance the read pointer past this message. diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index 59d7d7fb15e8..0c787fd689bf 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -191,7 +191,7 @@ pub(crate) fn wait_gsp_init_done(cmdq: &Cmdq<'_>) -> Result { loop { match cmdq.receive_msg::(Cmdq::RECEIVE_TIMEOUT) { Ok(_) => break Ok(()), - Err(ERANGE) => continue, + Err(ENOMSG) => continue, Err(e) => break Err(e), } } diff --git a/drivers/gpu/nova-core/gsp/sequencer.rs b/drivers/gpu/nova-core/gsp/sequencer.rs index dae34c11eb05..1782ed7d7ca6 100644 --- a/drivers/gpu/nova-core/gsp/sequencer.rs +++ b/drivers/gpu/nova-core/gsp/sequencer.rs @@ -346,7 +346,7 @@ pub(crate) fn run( let seq_info = loop { match cmdq.receive_msg::(Cmdq::RECEIVE_TIMEOUT) { Ok(seq_info) => break seq_info, - Err(ERANGE) => continue, + Err(ENOMSG) => continue, Err(e) => return Err(e), } }; From 621b455c56376f5469d896b58c2316236a3f939d Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Tue, 29 Sep 2026 20:41:44 -0700 Subject: [PATCH 099/110] gpu: nova-core: bound a GSP wait by a single deadline The GSP posts unsolicited events on the same queue as command replies, so a caller waiting for one message consumes whatever arrives first and reads again. Every read started a fresh five-second timeout, so a steady stream of events extended the wait without bound. The two boot-time waits for an unsolicited event also released the queue mutex between reads, so a command sent from another thread could consume the event and leave the waiter to time out. Compute one deadline when the wait begins and pass the time remaining to each read, and hold the queue mutex across the whole wait. Put the loop in one helper that the command reply wait and both boot-time event waits share. Assisted-by: LLM Signed-off-by: John Hubbard Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260930034148.590687-12-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/cmdq.rs | 72 ++++++++++++++++++++------ drivers/gpu/nova-core/gsp/commands.rs | 8 +-- drivers/gpu/nova-core/gsp/sequencer.rs | 8 +-- 3 files changed, 59 insertions(+), 29 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index ad2a14d08cb0..ae3808de44e2 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -28,7 +28,11 @@ }, Mutex, // }, - time::Delta, + time::{ + Delta, + Instant, + Monotonic, // + }, transmute::{ AsBytes, FromBytes, // @@ -130,7 +134,9 @@ fn size(&self) -> usize { /// Trait representing messages received from the GSP. /// -/// This trait tells [`Cmdq::receive_msg`] how it can receive a given type of message. +/// A reply that [`Cmdq::send_command`] waits for, or an event that [`Cmdq::await_msg`] waits for. +/// The receiver matches a message's function code against [`Self::FUNCTION`] and decodes the +/// message with [`Self::read`]. pub(crate) trait MessageFromGsp: Sized { /// Function identifying this message from the GSP. const FUNCTION: MsgFunction; @@ -566,8 +572,9 @@ fn notify_gsp(bar: Bar0<'_>) { /// /// # Errors /// - /// - `ETIMEDOUT` if space does not become available to send the command, or if the reply is - /// not received within the timeout. + /// - `ETIMEDOUT` if space does not become available to send the command, or if the reply does + /// not arrive within [`Self::RECEIVE_TIMEOUT`] of the send, however many events arrive + /// while waiting. /// - `EIO` if the variable payload requested by the command has not been entirely /// written to by its [`CommandToGsp::init_variable_payload`] method. /// @@ -582,13 +589,7 @@ pub(crate) fn send_command(&self, command: M) -> Result let mut inner = self.inner.lock(); inner.send_command(command)?; - loop { - match inner.receive_msg::(Self::RECEIVE_TIMEOUT) { - Ok(reply) => break Ok(reply), - Err(ENOMSG) => continue, - Err(e) => break Err(e), - } - } + inner.await_msg() } /// Sends `command` to the GSP without waiting for a reply. @@ -608,15 +609,25 @@ pub(crate) fn send_command_no_wait(&self, command: M) -> Result self.inner.lock().send_command(command) } - /// Receive a message from the GSP. + /// Waits for an unsolicited GSP event of type `M`. Events that arrive before it are logged and + /// consumed. /// - /// See [`CmdqInner::receive_msg`] for details. - pub(crate) fn receive_msg(&self, timeout: Delta) -> Result + /// The queue mutex is held for the whole wait, up to [`Self::RECEIVE_TIMEOUT`], so no other + /// caller can send a command or consume an event meanwhile. + /// + /// # Errors + /// + /// - `ETIMEDOUT` if the event does not arrive within [`Self::RECEIVE_TIMEOUT`] of the call, + /// however many other events arrive while waiting. + /// - `EIO` if a message fails framing or checksum validation. + /// + /// Error codes returned by [`MessageFromGsp::read`] are propagated as-is. + pub(crate) fn await_msg(&self) -> Result where // This allows all error types, including `Infallible`, to be used for `M::InitError`. Error: From, { - self.inner.lock().receive_msg(timeout) + self.inner.lock().await_msg() } } @@ -870,6 +881,37 @@ fn receive_msg(&mut self, timeout: Delta) -> Result result } + /// Receives a message of type `M`, waiting up to [`Cmdq::RECEIVE_TIMEOUT`] from the call. + /// + /// Any other message that arrives first is logged as an event and does not extend the + /// deadline. + /// + /// # Errors + /// + /// - `ETIMEDOUT` if no message of type `M` arrives before the deadline, however many other + /// messages arrive while waiting. + /// - `EIO` if a message fails framing or checksum validation (see [`Self::wait_for_msg`]). + /// + /// Error codes returned by [`MessageFromGsp::read`] are propagated as-is. + fn await_msg(&mut self) -> Result + where + // This allows all error types, including `Infallible`, to be used for `M::InitError`. + Error: From, + { + let deadline = Instant::::now() + Cmdq::RECEIVE_TIMEOUT; + loop { + let remaining = deadline - Instant::::now(); + if remaining.is_negative() { + break Err(ETIMEDOUT); + } + match self.receive_msg::(remaining) { + Ok(msg) => break Ok(msg), + Err(ENOMSG) => continue, + Err(e) => break Err(e), + } + } + } + /// Logs an event, meaning a message that no caller was waiting for. /// /// An OS error or robust-channel record is logged at error level and an unknown function code diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index 0c787fd689bf..cdbb13674f08 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -188,13 +188,7 @@ fn read( /// Waits for GSP initialization to complete. pub(crate) fn wait_gsp_init_done(cmdq: &Cmdq<'_>) -> Result { - loop { - match cmdq.receive_msg::(Cmdq::RECEIVE_TIMEOUT) { - Ok(_) => break Ok(()), - Err(ENOMSG) => continue, - Err(e) => break Err(e), - } - } + cmdq.await_msg::().map(|_| ()) } /// The `GetGspStaticInfo` command. diff --git a/drivers/gpu/nova-core/gsp/sequencer.rs b/drivers/gpu/nova-core/gsp/sequencer.rs index 1782ed7d7ca6..250adc9fe74f 100644 --- a/drivers/gpu/nova-core/gsp/sequencer.rs +++ b/drivers/gpu/nova-core/gsp/sequencer.rs @@ -343,13 +343,7 @@ pub(crate) fn run( libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>, bootloader_app_version: u32, ) -> Result { - let seq_info = loop { - match cmdq.receive_msg::(Cmdq::RECEIVE_TIMEOUT) { - Ok(seq_info) => break seq_info, - Err(ENOMSG) => continue, - Err(e) => return Err(e), - } - }; + let seq_info = cmdq.await_msg::()?; let sequencer = GspSequencer { bar: ctx.bar, From e727d1c382a0cb2e10fd7aabef9217d1a37c80c7 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Tue, 29 Sep 2026 20:41:45 -0700 Subject: [PATCH 100/110] gpu: nova-core: add the falcon interrupt registers and HAL methods A falcon latches each interrupt cause that is raised in its IRQSTAT register. On a RISC-V falcon, each cause is routed either to the host, meaning the CPU, or to the falcon's own core, and two routing registers hold that routing. The falcon signals the interrupt tree only when its set of host-routed causes goes from empty to non-empty, so a handler that clears the tree leaf while a cause is still latched has to write INTR_RETRIGGER, which makes the falcon re-emit its causes. Turing falcons have no INTR_RETRIGGER, and the routing registers have different offsets from GA102 on. Add IRQSTAT, INTR_RETRIGGER, and the two routing registers, the last at both sets of offsets, and add two falcon HAL methods: one intersects an IRQSTAT value with the routing registers, and one retriggers the falcon, or does nothing on Turing. GA100 keeps the Turing HAL, as it does for boot, and the retrigger register is a property of that HAL's instance. The two methods have no user yet. Assisted-by: LLM Signed-off-by: John Hubbard Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260930034148.590687-13-jhubbard@nvidia.com [acourbot: remove implementation-specific comments on interface doc.] [acourbot: make constructor take boolean instead of having two methods.] [acourbot: add NVIDIA copyright headers to modified files.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/falcon/hal.rs | 24 +++++++- drivers/gpu/nova-core/falcon/hal/ga102.rs | 17 ++++++ drivers/gpu/nova-core/falcon/hal/tu102.rs | 48 +++++++++++++++- drivers/gpu/nova-core/regs.rs | 69 +++++++++++++++++++++++ 4 files changed, 152 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/nova-core/falcon/hal.rs b/drivers/gpu/nova-core/falcon/hal.rs index 7e532889a1f4..cd2613a74595 100644 --- a/drivers/gpu/nova-core/falcon/hal.rs +++ b/drivers/gpu/nova-core/falcon/hal.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use kernel::prelude::*; @@ -12,6 +13,7 @@ Architecture, Chipset, // }, + regs, }; mod ga102; @@ -70,6 +72,22 @@ fn signature_reg_fuse_version( /// these. For anything above, the PIO registers appear to be masked to the CPU, so DMA is the /// only usable method. fn load_method(&self) -> LoadMethod; + + /// Returns the causes in `latched` that are routed to the host, meaning the CPU, rather than + /// to the falcon's own RISC-V core. + /// + /// The causes routed to the core belong to the firmware running on it, and the host does not + /// service them. + #[expect(dead_code)] + fn host_routed_causes( + &self, + falcon: &Falcon<'_, E>, + latched: regs::NV_PFALCON_FALCON_IRQSTAT, + ) -> regs::NV_PFALCON_FALCON_IRQSTAT; + + /// Retriggers the falcon, which then re-emits its host-routed causes into the interrupt tree. + #[expect(dead_code)] + fn retrigger(&self, falcon: &Falcon<'_, E>); } /// Returns a boxed falcon HAL adequate for `chipset`. @@ -82,11 +100,11 @@ pub(super) fn falcon_hal( ) -> Result>> { let hal = match chipset.arch() { Architecture::Turing => { - KBox::new(tu102::Tu102::::new(), GFP_KERNEL)? as KBox> + KBox::new(tu102::Tu102::::new(false), GFP_KERNEL)? as KBox> } - // GA100 boots like Turing so use Turing HAL + // GA100 boots like Turing, but features the retrigger register. Architecture::Ampere if chipset == Chipset::GA100 => { - KBox::new(tu102::Tu102::::new(), GFP_KERNEL)? as KBox> + KBox::new(tu102::Tu102::::new(true), GFP_KERNEL)? as KBox> } Architecture::Ampere | Architecture::Ada diff --git a/drivers/gpu/nova-core/falcon/hal/ga102.rs b/drivers/gpu/nova-core/falcon/hal/ga102.rs index f9a8444cf840..8e77c0bbaf61 100644 --- a/drivers/gpu/nova-core/falcon/hal/ga102.rs +++ b/drivers/gpu/nova-core/falcon/hal/ga102.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use core::marker::PhantomData; @@ -169,4 +170,20 @@ fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result { fn load_method(&self) -> LoadMethod { LoadMethod::Dma } + + fn host_routed_causes( + &self, + falcon: &Falcon<'_, E>, + latched: regs::NV_PFALCON_FALCON_IRQSTAT, + ) -> regs::NV_PFALCON_FALCON_IRQSTAT { + let pfalcon2 = falcon.pfalcon2; + let mask = pfalcon2.read(regs::ga102::NV_PRISCV_RISCV_IRQMASK).value(); + let dest = pfalcon2.read(regs::ga102::NV_PRISCV_RISCV_IRQDEST).value(); + + regs::NV_PFALCON_FALCON_IRQSTAT::from(latched.into_raw() & mask & dest) + } + + fn retrigger(&self, falcon: &Falcon<'_, E>) { + super::tu102::retrigger_ga100(falcon); + } } diff --git a/drivers/gpu/nova-core/falcon/hal/tu102.rs b/drivers/gpu/nova-core/falcon/hal/tu102.rs index 7fc6e83c2566..94c4438d3ebf 100644 --- a/drivers/gpu/nova-core/falcon/hal/tu102.rs +++ b/drivers/gpu/nova-core/falcon/hal/tu102.rs @@ -1,10 +1,12 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use core::marker::PhantomData; use kernel::{ io::{ poll::read_poll_timeout, + register::Array, Io, // }, prelude::*, @@ -23,11 +25,33 @@ use super::FalconHal; -pub(super) struct Tu102(PhantomData); +/// Writes the trigger register available on GA100 and more recent. +#[expect(dead_code)] +pub(super) fn retrigger_ga100(falcon: &Falcon<'_, E>) { + falcon.pfalcon.write( + Array::at(0), + regs::NV_PFALCON_FALCON_INTR_RETRIGGER::zeroed().with_trigger(true), + ); +} + +/// The falcon HAL for Turing and GA100. GA100 boots like a Turing, but unlike Turing, it has a +/// retrigger register. +pub(super) struct Tu102 { + /// If `true`, the falcons have `NV_PFALCON_FALCON_INTR_RETRIGGER`. + #[expect(dead_code)] + has_intr_retrigger: bool, + _engine: PhantomData, +} impl Tu102 { - pub(super) fn new() -> Self { - Self(PhantomData) + /// Creates a new HAL. + /// + /// `has_intr_retrigger` indicates whether the retrigger register can be used. + pub(super) fn new(has_intr_retrigger: bool) -> Self { + Self { + has_intr_retrigger, + _engine: PhantomData, + } } } @@ -79,4 +103,22 @@ fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result { fn load_method(&self) -> LoadMethod { LoadMethod::Pio } + + fn host_routed_causes( + &self, + falcon: &Falcon<'_, E>, + latched: regs::NV_PFALCON_FALCON_IRQSTAT, + ) -> regs::NV_PFALCON_FALCON_IRQSTAT { + let pfalcon2 = falcon.pfalcon2; + let mask = pfalcon2.read(regs::tu102::NV_PRISCV_RISCV_IRQMASK).value(); + let dest = pfalcon2.read(regs::tu102::NV_PRISCV_RISCV_IRQDEST).value(); + + regs::NV_PFALCON_FALCON_IRQSTAT::from(latched.into_raw() & mask & dest) + } + + fn retrigger(&self, falcon: &Falcon<'_, E>) { + if self.has_intr_retrigger { + retrigger_ga100(falcon); + } + } } diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index 9978fb2803b0..88662da634dc 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -124,11 +124,25 @@ pub(crate) fn usable_fb_size(self) -> u64 { register! { base: PFalconRegisters; + /// Clears the latch of every cause whose bit is written as `1`. Write-only. + /// + /// The write ends the latch and not the source, so a cause driven from outside the falcon + /// stays set. "Retriggering a falcon" in `Documentation/gpu/nova/core/interrupts.rst` names + /// those causes. pub(crate) NV_PFALCON_FALCON_IRQSCLR(u32) @ 0x00000004 { 6:6 swgen0 => bool; 4:4 halt => bool; } + /// Interrupt causes latched in the falcon, one bit per cause, whichever target each is routed + /// to. + /// + /// The causes routed to the host are the ones also set in `NV_PRISCV_RISCV_IRQMASK` and + /// `NV_PRISCV_RISCV_IRQDEST`. + pub(crate) NV_PFALCON_FALCON_IRQSTAT(u32) @ 0x00000008 { + 6:6 swgen0 => bool; + } + pub(crate) NV_PFALCON_FALCON_MAILBOX0(u32) @ 0x00000040 { 31:0 value => u32; } @@ -256,6 +270,16 @@ pub(crate) fn usable_fb_size(self) -> u64 { 0:0 reset => bool; } + /// Makes the falcon re-emit its host-routed causes into the interrupt tree. Write-only. + /// + /// Present from GA100 on. See "Retriggering a falcon" in + /// `Documentation/gpu/nova/core/interrupts.rst`. + /// + /// The hardware headers declare two elements, and OpenRM writes only the first. + pub(crate) NV_PFALCON_FALCON_INTR_RETRIGGER(u32)[2] @ 0x000003e8 { + 0:0 trigger => bool; + } + pub(crate) NV_PFALCON_FBIF_TRANSCFG(u32)[8] @ 0x00000600 { 2:2 mem_type => FalconFbifMemType; 1:0 target ?=> FalconFbifTarget; @@ -414,6 +438,29 @@ pub(crate) mod gm107 { } } +pub(crate) mod tu102 { + use kernel::io::register; + + use crate::falcon::PFalcon2Registers; + + // The RISC-V interrupt routing registers, at the offsets that Turing and GA100 use. + + register! { + base: PFalcon2Registers; + + /// Enabled causes, one bit per cause. Read-only to the host. + pub(crate) NV_PRISCV_RISCV_IRQMASK(u32) @ 0x000002b4 { + 31:0 value => u32; + } + + /// Causes routed to the host, one bit per cause. A clear bit routes the cause to the + /// RISC-V core. + pub(crate) NV_PRISCV_RISCV_IRQDEST(u32) @ 0x000002b8 { + 31:0 value => u32; + } + } +} + pub(crate) mod ga100 { use kernel::io::register; @@ -430,6 +477,28 @@ pub(crate) mod ga100 { } } +pub(crate) mod ga102 { + use kernel::io::register; + + use crate::falcon::PFalcon2Registers; + + // The RISC-V interrupt routing registers, at the offsets that GA102 and later use. + + register! { + base: PFalcon2Registers; + + /// Same as [`super::tu102::NV_PRISCV_RISCV_IRQMASK`], at the GA102 offset. + pub(crate) NV_PRISCV_RISCV_IRQMASK(u32) @ 0x00000528 { + 31:0 value => u32; + } + + /// Same as [`super::tu102::NV_PRISCV_RISCV_IRQDEST`], at the GA102 offset. + pub(crate) NV_PRISCV_RISCV_IRQDEST(u32) @ 0x0000052c { + 31:0 value => u32; + } + } +} + pub(crate) const NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE_STATUS_SUCCESS: u32 = 0xff; pub(crate) mod gh100 { From 476abf9f669d87f993429ebb42668169464c907c Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Tue, 29 Sep 2026 20:41:46 -0700 Subject: [PATCH 101/110] gpu: nova-core: service GSP events from the SWGEN0 interrupt The GSP posts each message for the CPU to the GSP-to-CPU queue and then raises SWGEN0, a software-generated interrupt cause of its falcon. GIN delivers SWGEN0 to the CPU on vector 155. A thread waiting for a command reply was the only reader of the queue, so an event posted between commands stayed unread until the next command was sent. Register a threaded handler on the GSP vector. Draining the queue takes the command-queue mutex, which can sleep, so the top half reads and writes only registers and wakes the IRQ thread to drain the queue. Every existing read of the queue asks for one message type and waits on a deadline, so add a drain that logs and consumes whatever the GSP has already posted and returns at once. When a falcon cause other than SWGEN0 stays set after its latch is cleared, as the Blackwell fault-containment and ECC causes do, disable the GSP vector rather than retrigger the falcon. A retrigger would re-emit the cause at once, and the CPU would take the same interrupt again and again. Assisted-by: LLM Signed-off-by: John Hubbard Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260930034148.590687-14-jhubbard@nvidia.com [acourbot: trim comment leaking unneeded implementation details.] [acourbot: make Subtree crate-visible here instead of in earlier patch.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/falcon/gsp.rs | 47 ++++- drivers/gpu/nova-core/falcon/hal.rs | 2 - drivers/gpu/nova-core/falcon/hal/tu102.rs | 2 - drivers/gpu/nova-core/gpu.rs | 96 +++++++++- drivers/gpu/nova-core/gsp/cmdq.rs | 40 ++++ drivers/gpu/nova-core/irq.rs | 1 + drivers/gpu/nova-core/irq/gsp.rs | 195 ++++++++++++++++++++ drivers/gpu/nova-core/irq/interrupt_tree.rs | 5 +- drivers/gpu/nova-core/nova_core.rs | 1 - 9 files changed, 375 insertions(+), 14 deletions(-) create mode 100644 drivers/gpu/nova-core/irq/gsp.rs diff --git a/drivers/gpu/nova-core/falcon/gsp.rs b/drivers/gpu/nova-core/falcon/gsp.rs index 4c96ae325fda..1faefaede9fe 100644 --- a/drivers/gpu/nova-core/falcon/gsp.rs +++ b/drivers/gpu/nova-core/falcon/gsp.rs @@ -47,13 +47,56 @@ fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, super::PFalcon2Registers> { } impl<'a> Falcon<'a, Gsp> { - /// Clears the SWGEN0 bit in the Falcon's IRQ status clear register to - /// allow GSP to signal CPU for processing new messages in message queue. + /// Clears the SWGEN0 latch in the GSP falcon. + /// + /// While the latch is set, no later message signals the tree, so a caller that consumed a + /// notification by polling must clear it. pub(crate) fn clear_swgen0_intr(&self) { self.pfalcon .write_reg(regs::NV_PFALCON_FALCON_IRQSCLR::zeroed().with_swgen0(true)); } + /// Reads the GSP falcon causes that are routed to the host, without clearing any latch. + /// + /// Every one of them other than SWGEN0 reports a GSP fault. + pub(crate) fn read_host_intr(&self) -> regs::NV_PFALCON_FALCON_IRQSTAT { + let latched = self.pfalcon.read(regs::NV_PFALCON_FALCON_IRQSTAT); + + self.hal.host_routed_causes(self, latched) + } + + /// Reads the host-routed causes and clears the SWGEN0 latch if it was set. + /// + /// Returns the causes as read, before the clear. No other latch changes. + pub(crate) fn take_host_intr(&self) -> regs::NV_PFALCON_FALCON_IRQSTAT { + let status = self.read_host_intr(); + + if status.swgen0() { + self.clear_swgen0_intr(); + } + + status + } + + /// Clears the latch of every interrupt cause set in `status`. + /// + /// A cause driven from outside the falcon is still set on return, and + /// [`Self::read_host_intr`] reports the causes that remain. + pub(crate) fn clear_intr(&self, status: regs::NV_PFALCON_FALCON_IRQSTAT) { + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_IRQSCLR::from(status.into_raw())); + } + + /// Retriggers the GSP falcon, which then re-emits its host-routed causes into the tree. + /// + /// Call this only once every host cause is clear. A cause still set is re-emitted at once, and + /// its vector arrives again as soon as delivery is rearmed. + /// + /// Does nothing on Turing, whose falcons have no retrigger register. + pub(crate) fn retrigger_intr(&self) { + self.hal.retrigger(self); + } + /// Checks if GSP reload/resume has completed during the boot process. pub(crate) fn check_reload_completed(&self, timeout: Delta) -> Result { read_poll_timeout( diff --git a/drivers/gpu/nova-core/falcon/hal.rs b/drivers/gpu/nova-core/falcon/hal.rs index cd2613a74595..de7825719ebc 100644 --- a/drivers/gpu/nova-core/falcon/hal.rs +++ b/drivers/gpu/nova-core/falcon/hal.rs @@ -78,7 +78,6 @@ fn signature_reg_fuse_version( /// /// The causes routed to the core belong to the firmware running on it, and the host does not /// service them. - #[expect(dead_code)] fn host_routed_causes( &self, falcon: &Falcon<'_, E>, @@ -86,7 +85,6 @@ fn host_routed_causes( ) -> regs::NV_PFALCON_FALCON_IRQSTAT; /// Retriggers the falcon, which then re-emits its host-routed causes into the interrupt tree. - #[expect(dead_code)] fn retrigger(&self, falcon: &Falcon<'_, E>); } diff --git a/drivers/gpu/nova-core/falcon/hal/tu102.rs b/drivers/gpu/nova-core/falcon/hal/tu102.rs index 94c4438d3ebf..010ca5c36fc6 100644 --- a/drivers/gpu/nova-core/falcon/hal/tu102.rs +++ b/drivers/gpu/nova-core/falcon/hal/tu102.rs @@ -26,7 +26,6 @@ use super::FalconHal; /// Writes the trigger register available on GA100 and more recent. -#[expect(dead_code)] pub(super) fn retrigger_ga100(falcon: &Falcon<'_, E>) { falcon.pfalcon.write( Array::at(0), @@ -38,7 +37,6 @@ pub(super) fn retrigger_ga100(falcon: &Falcon<'_, E>) { /// retrigger register. pub(super) struct Tu102 { /// If `true`, the falcons have `NV_PFALCON_FALCON_INTR_RETRIGGER`. - #[expect(dead_code)] has_intr_retrigger: bool, _engine: PhantomData, } diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index f91c7e22a303..4e59072cab50 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -34,10 +34,19 @@ fsp::Fsp, gsp::{ self, + cmdq::Cmdq, commands::GetGspStaticInfoReply, Gsp, GspBootContext, // }, + irq::{ + self, + gsp::GspIrq, + interrupt_tree::{ + TopEnableGuard, + Tree, // + }, // + }, mm::{ bar_user::BarUser, pagetable::MmuVersion, @@ -331,10 +340,53 @@ struct GspResources<'gpu> { unload_bundle: Option>, } +/// The GSP event handler's registration and the enable of its subtree at `TOP`. +/// +/// The two drop as a unit, the handler first, on the drop of [`Gpu`] and on the error path of +/// its constructor alike, so that the subtree is disabled only after the handler is freed. +#[pin_data] +struct GspSubtree<'a> { + #[pin] + irq: GspIrq<'a>, + /// Must be kept declared *after* `irq`. A handler still in flight enables the subtree again + /// through its rearm. + _top: TopEnableGuard<'a>, +} + +impl<'a> GspSubtree<'a> { + /// Returns an initializer that registers the GSP event handler and then enables its subtree at + /// `TOP`. + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the initialized `GspSubtree` or otherwise prevent its + /// [`Drop`] implementation, which runs `free_irq`, from running. + unsafe fn new( + pdev: &'a pci::Device, + tree: &'a Tree<'a>, + falcon: &'a Falcon<'a, GspFalcon>, + cmdq: &'a Cmdq<'a>, + ) -> impl PinInit + 'a { + try_pin_init!(Self { + // SAFETY: this function's caller must not leak the `GspSubtree` that owns this + // registration, so the registration's `Drop` runs. + irq <- unsafe { GspIrq::new(pdev, tree, falcon, cmdq) }, + _top: tree.enable_top_guarded(), + }) + } +} + /// Structure holding the resources required to operate the GPU. #[pin_data] pub(crate) struct Gpu<'gpu> { pub(crate) spec: Spec, + /// GSP event interrupt registration, and the enable of its subtree. + /// + /// Must be kept declared *before* `gsp_resources`, so that the handler is unregistered, and + /// any in-flight run of it has finished, before the command queue that it drains and the + /// falcon that it reads are freed, and before the GSP is unloaded. + #[pin] + _gsp_subtree: GspSubtree<'gpu>, /// Static GPU information as provided by the GSP. pub(crate) gsp_static_info: GetGspStaticInfoReply, /// GPU memory manager owning memory management resources. @@ -354,6 +406,14 @@ pub(crate) struct Gpu<'gpu> { /// Must be kept declared *after* `gsp_resources`, as the latter's `PinnedDrop` implementation /// requires the sysmem flush page to be in place. sysmem_flush: SysmemFlush<'gpu>, + /// Borrow of `tree` that `_gsp_subtree` holds. A field that borrows a sibling field is + /// self-referential, which `pin_init` cannot express, so the borrow is taken by hand. + tree_ref: &'gpu Tree<'gpu>, + /// The GIN CPU interrupt tree and the PCI vectors that deliver it. + /// + /// Must be kept declared *after* `_gsp_subtree`, which holds a borrow of it. + #[pin] + tree: Tree<'gpu>, } #[pinned_drop] @@ -397,6 +457,12 @@ pub(crate) fn new<'a>( dev_info!(dev,"NVIDIA ({})\n", spec); })?, + tree: Tree::new(pdev, bar, spec.chipset, irq::gsp::GSP_SUBTREE.into())?, + + // SAFETY: `tree` is initialized above, is pinned at a stable address, and is dropped + // after every field that uses `tree_ref` (struct field drop order). + tree_ref: unsafe { &*core::ptr::from_ref(tree.as_ref().get_ref()) }, + _: { let dma_mask = hal::gpu_hal(spec.chipset).dma_mask(); @@ -425,12 +491,7 @@ pub(crate) fn new<'a>( bar, - gsp_falcon: Falcon::new( - dev, - spec.chipset, - bar - ) - .inspect(|falcon| falcon.clear_swgen0_intr())?, + gsp_falcon: Falcon::new(dev, spec.chipset, bar)?, sec2_falcon: Falcon::new(dev, spec.chipset, bar)?, @@ -454,6 +515,29 @@ pub(crate) fn new<'a>( })?, }), + _: { + irq::gsp::quiesce(tree_ref, &gsp_resources.gsp_falcon); + }, + + // SAFETY: the GSP falcon and the command queue are fields of `gsp_resources`, which + // is initialized above and pinned, so both references outlive the registration. The + // registration is a field of `Gpu` and is never leaked, so its `Drop` runs, and field + // drop order runs it before either is freed. + _gsp_subtree <- unsafe { + GspSubtree::new( + pdev, + tree_ref, + &*core::ptr::from_ref(&gsp_resources.gsp_falcon), + &*core::ptr::from_ref(&gsp_resources.gsp.cmdq), + ) + }, + + // No interrupt announces the messages that the GSP posted during boot, before the + // SWGEN0 latch was cleared. + _: { + gsp_resources.gsp.cmdq.drain()?; + }, + gsp_static_info: { // Obtain and display basic GPU information. let info = gsp_resources.gsp.get_static_info()?; diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index ae3808de44e2..1b2f346a237f 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -629,6 +629,20 @@ pub(crate) fn await_msg(&self) -> Result { self.inner.lock().await_msg() } + + /// Logs and consumes every message the GSP has already posted, and returns without waiting for + /// more. + /// + /// No caller is waiting for a reply while this holds the queue mutex, so every message is + /// logged as an event. See "Draining the GSP-to-CPU queue" in + /// `Documentation/gpu/nova/core/interrupts.rst`. + /// + /// # Errors + /// + /// `EIO` if a message fails framing or checksum validation. + pub(crate) fn drain(&self) -> Result { + self.inner.lock().drain() + } } /// Inner mutex protected state of [`Cmdq`]. @@ -941,4 +955,30 @@ fn log_event(&self, function: Result, seq: u32) { } } } + + /// Logs and consumes every message the queue holds. + /// + /// # Errors + /// + /// `EIO` if a message fails framing or checksum validation, or if a message's page count + /// overflows a `u32`. + fn drain(&mut self) -> Result { + while !self.gsp_mem.driver_read_area().0.is_empty() { + // A message is available, so this returns without waiting. + let msg = self.wait_for_msg(Delta::ZERO)?; + + let pages = + u32::try_from(msg.header.length().div_ceil(GSP_PAGE_SIZE)).map_err(|_| { + dev_err!(&self.dev, "GSP drain: message length overflow\n"); + EIO + })?; + let function = msg.header.function(); + let seq = msg.header.sequence(); + + self.gsp_mem.advance_cpu_read_ptr(pages); + self.log_event(function, seq); + } + + Ok(()) + } } diff --git a/drivers/gpu/nova-core/irq.rs b/drivers/gpu/nova-core/irq.rs index ff8b00a82442..02f9bd84dd19 100644 --- a/drivers/gpu/nova-core/irq.rs +++ b/drivers/gpu/nova-core/irq.rs @@ -11,6 +11,7 @@ #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] pub(crate) mod doorbell_test; +pub(crate) mod gsp; mod hal; pub(crate) mod interrupt_tree; mod regs; diff --git a/drivers/gpu/nova-core/irq/gsp.rs b/drivers/gpu/nova-core/irq/gsp.rs new file mode 100644 index 000000000000..1abde8e0a405 --- /dev/null +++ b/drivers/gpu/nova-core/irq/gsp.rs @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! The GSP event interrupt. +//! +//! The GSP posts messages to the GSP-to-CPU queue and raises SWGEN0, a software-generated cause +//! of its falcon. A threaded handler services it: the top half clears the tree and falcon state, +//! and the IRQ thread drains the queue. +//! +//! See "The GSP event" in `Documentation/gpu/nova/core/interrupts.rst`. + +use kernel::{ + device, + irq, + pci, + prelude::*, // +}; + +use super::interrupt_tree::{ + GinVector, + LeafEnableGuard, + Subtree, + Tree, // +}; +use crate::{ + falcon::{ + gsp::Gsp as GspFalcon, + Falcon, // + }, + gsp::cmdq::Cmdq, + regs, // +}; + +/// The GSP event vector, which has the same number on every supported GPU. +const GSP_INTR_0_VECTOR: GinVector = GinVector::new::<155>(); + +/// The GSP event's subtree, the only one that nova-core services. +pub(crate) const GSP_SUBTREE: Subtree = GSP_INTR_0_VECTOR.subtree(); + +/// Clears the tree and falcon interrupt state that GSP boot leaves behind, and rearms PCI +/// interrupt delivery. +/// +/// On return, no vector is enabled at its leaf, the serviced subtrees are disabled at `TOP`, and +/// the SWGEN0 latch is clear, so the next message that the GSP posts signals the tree. +pub(crate) fn quiesce(tree: &Tree<'_>, falcon: &Falcon<'_, GspFalcon>) { + tree.reset(); + // The latch is cleared after the tree reset. The other order can leave the latch set with its + // leaf bit cleared. See "Enabling the GSP event" in interrupts.rst. + falcon.clear_swgen0_intr(); +} + +/// Threaded IRQ handler for the GSP event. +pub(crate) struct GspInterrupt<'a> { + falcon: &'a Falcon<'a, GspFalcon>, + cmdq: &'a Cmdq<'a>, + tree: &'a Tree<'a>, + /// For logging. + dev: &'a device::Device, +} + +impl GspInterrupt<'_> { + /// Clears the latch of every host-routed cause in `status` other than SWGEN0, and logs them. + /// + /// Returns the causes still set after the clear. A cause driven from outside the falcon stays + /// set, and only a device reset ends it. + fn clear_faults( + &self, + status: regs::NV_PFALCON_FALCON_IRQSTAT, + ) -> regs::NV_PFALCON_FALCON_IRQSTAT { + let faults = status.with_swgen0(false); + if faults.into_raw() == 0 { + return faults; + } + + dev_err!( + &self.dev, + "unserviceable GSP falcon interrupt, IRQSTAT {:#x}\n", + status.into_raw() + ); + self.falcon.clear_intr(faults); + + self.falcon.read_host_intr().with_swgen0(false) + } +} + +impl irq::ThreadedHandler for GspInterrupt<'_> { + /// Top half, in hard interrupt context. Services the GSP vector only, so another vector + /// pending in the same leaf stays pending. + fn handle(&self) -> irq::ThreadedIrqReturn { + let leaf = self.tree.read_pending(GSP_INTR_0_VECTOR.leaf_index()); + if !leaf.vectors().contains(GSP_INTR_0_VECTOR.leaf_mask()) { + self.tree.rearm_pci_irq(GSP_SUBTREE); + return irq::ThreadedIrqReturn::None; + } + leaf.clear_vectors(GSP_INTR_0_VECTOR.leaf_mask()); + + let status = self.falcon.take_host_intr(); + + let remaining_faults = self.clear_faults(status); + if remaining_faults.into_raw() == 0 { + self.falcon.retrigger_intr(); + } else { + // Disabling the vector loses no notification: the falcon signals nothing further + // while a cause stays set. See "Retriggering a falcon" in interrupts.rst. + self.tree.disable_leaf( + GSP_INTR_0_VECTOR.leaf_index(), + GSP_INTR_0_VECTOR.leaf_mask(), + ); + dev_err!( + &self.dev, + "GSP falcon cause {:#x} needs a device reset, GSP events are no longer serviced\n", + remaining_faults.into_raw() + ); + } + + self.tree.rearm_pci_irq(GSP_SUBTREE); + + if status.swgen0() { + irq::ThreadedIrqReturn::WakeThread + } else { + irq::ThreadedIrqReturn::Handled + } + } + + /// IRQ thread. Drains the GSP-to-CPU queue, which may sleep. + fn handle_threaded(&self) -> irq::IrqReturn { + if let Err(e) = self.cmdq.drain() { + // The failed message stays at the queue head, so every later drain fails the same way. + self.tree.disable_leaf( + GSP_INTR_0_VECTOR.leaf_index(), + GSP_INTR_0_VECTOR.leaf_mask(), + ); + dev_err!( + &self.dev, + "GSP event drain failed ({:?}), the message queue is no longer serviced\n", + e + ); + } + irq::IrqReturn::Handled + } +} + +/// The registered GSP event handler and the enable of its vector. +/// +/// The declaration order is the drop order, and it is required: the vector is disabled before +/// `free_irq` runs. See "Enabling the GSP event" in `Documentation/gpu/nova/core/interrupts.rst`. +#[pin_data] +pub(crate) struct GspIrq<'a> { + _leaf_guard: LeafEnableGuard<'a>, + #[pin] + reg: irq::ThreadedRegistration<'a, GspInterrupt<'a>>, +} + +impl<'a> GspIrq<'a> { + /// Returns an initializer that registers the threaded handler and then enables the GSP + /// vector at its leaf. + /// + /// An event that latched while the vector was disabled is delivered as soon as the GSP + /// subtree is enabled at `TOP`. + /// + /// # Errors + /// + /// `EINVAL` if `tree` does not service the GSP subtree. Otherwise the error from + /// `request_threaded_irq`. + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the initialized `GspIrq` or otherwise prevent its [`Drop`] + /// implementation, which runs `free_irq`, from running. + pub(crate) unsafe fn new( + pdev: &'a pci::Device, + tree: &'a Tree<'a>, + falcon: &'a Falcon<'a, GspFalcon>, + cmdq: &'a Cmdq<'a>, + ) -> impl PinInit + 'a { + let dev = pdev.as_ref(); + + try_pin_init!(Self { + // SAFETY: this function's caller must not leak the `GspIrq` that owns this + // registration, so the registration's `Drop` runs. + reg <- unsafe { + irq::ThreadedRegistration::new( + tree.request_for(GSP_SUBTREE)?, + irq::Flags::TRIGGER_NONE, + c"nova-core", + Ok(GspInterrupt { falcon, cmdq, tree, dev }), + ) + }, + _leaf_guard: tree.enable_leaf_guarded( + GSP_INTR_0_VECTOR.leaf_index(), + GSP_INTR_0_VECTOR.leaf_mask(), + ), + }) + } +} diff --git a/drivers/gpu/nova-core/irq/interrupt_tree.rs b/drivers/gpu/nova-core/irq/interrupt_tree.rs index c1576ea3e61f..90d4d11e8ba3 100644 --- a/drivers/gpu/nova-core/irq/interrupt_tree.rs +++ b/drivers/gpu/nova-core/irq/interrupt_tree.rs @@ -124,10 +124,12 @@ pub(super) const fn all() -> Self { Self(u32::MAX) } + #[cfg_attr(not(CONFIG_NOVA_CORE_SELFTESTS), expect(dead_code))] pub(super) const fn from_raw(raw: u32) -> Self { Self(raw) } + #[cfg_attr(not(CONFIG_NOVA_CORE_SELFTESTS), expect(dead_code))] pub(super) const fn into_raw(self) -> u32 { self.0 } @@ -160,7 +162,7 @@ fn from(vectors: LeafMask) -> Self { /// /// Exactly one bit is set. #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) struct Subtree(u32); +pub(crate) struct Subtree(u32); impl Subtree { /// Returns the subtree at index `idx`. @@ -240,6 +242,7 @@ pub(super) const fn new() -> Self { Self(Bounded::::new::()) } + #[cfg_attr(not(CONFIG_NOVA_CORE_SELFTESTS), expect(dead_code))] pub(super) const fn into_raw(self) -> u32 { self.0.get() } diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 8c0761dedda4..a8f6dc22057b 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -18,7 +18,6 @@ mod fsp; mod gpu; mod gsp; -#[cfg_attr(not(CONFIG_NOVA_CORE_SELFTESTS), expect(dead_code))] mod irq; mod mctp; mod mm; From 306279ce974d396136544aa88027ee921ac09540 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Tue, 29 Sep 2026 20:41:47 -0700 Subject: [PATCH 102/110] gpu: nova-core: add KUnit tests for the interrupt tree The vector arithmetic and the leaf and subtree sets touch no hardware, so KUnit can cover them without a GPU. A wrong leaf, bit or subtree for a vector would otherwise show up only as a lost interrupt on hardware. Add the nova_core_gin_tree suite. It checks that a leaf index stops at the widest supported tree, that a leaf count implies the right number of subtrees and vectors, that a leaf count enumerates every leaf in order, that a vector maps to the right leaf, bit and subtree, and that a vector beyond an 8-leaf tree is rejected there and accepted in a 16-leaf tree. It also exercises the subtree set operations and checks that every supported chipset implements the subtree that carries the GSP event. Assisted-by: LLM Signed-off-by: John Hubbard Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260930034148.590687-15-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/irq/interrupt_tree.rs | 128 +++++++++++++++++++- 1 file changed, 127 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/irq/interrupt_tree.rs b/drivers/gpu/nova-core/irq/interrupt_tree.rs index 90d4d11e8ba3..590f921894b6 100644 --- a/drivers/gpu/nova-core/irq/interrupt_tree.rs +++ b/drivers/gpu/nova-core/irq/interrupt_tree.rs @@ -129,7 +129,10 @@ pub(super) const fn from_raw(raw: u32) -> Self { Self(raw) } - #[cfg_attr(not(CONFIG_NOVA_CORE_SELFTESTS), expect(dead_code))] + #[cfg_attr( + not(any(CONFIG_NOVA_CORE_SELFTESTS, CONFIG_KUNIT = "y")), + expect(dead_code) + )] pub(super) const fn into_raw(self) -> u32 { self.0 } @@ -554,3 +557,126 @@ fn drop(&mut self) { clear_top_enables(self.bar, self.serviced); } } + +#[kunit_tests(nova_core_gin_tree)] +mod tests { + use super::*; + + /// A leaf index cannot name a leaf beyond the widest supported tree. + #[test] + fn leaf_index_bounds() { + assert!(LeafIndex::try_new(0).is_some()); + assert!(LeafIndex::try_new(15).is_some()); + assert!(LeafIndex::try_new(16).is_none()); + } + + /// The subtree count, the implemented-subtree set, and the vector count follow the leaf count. + #[test] + fn leaf_count_derives_subtrees_and_vectors() { + assert_eq!(LeafCount::Eight.subtree_count(), 4); + assert_eq!( + Bounded::::from(LeafCount::Eight.subtree_set()).get(), + 0x0f + ); + assert_eq!(LeafCount::Eight.vector_count(), 256); + + assert_eq!(LeafCount::Sixteen.subtree_count(), 8); + assert_eq!( + Bounded::::from(LeafCount::Sixteen.subtree_set()).get(), + 0xff + ); + assert_eq!(LeafCount::Sixteen.vector_count(), 512); + } + + /// A tree enumerates every leaf that it implements, in order, and no more. + #[test] + fn leaf_count_iter_covers_the_tree() { + for (count, expected) in [(LeafCount::Eight, 8usize), (LeafCount::Sixteen, 16)] { + let mut seen = 0; + + for (index, leaf) in count.iter().enumerate() { + assert_eq!(leaf.get(), index); + seen += 1; + } + + assert_eq!(seen, expected); + } + } + + /// A vector maps to its leaf, its bit within that leaf, and its subtree. The doorbell (129) + /// and the GSP event (155) share a subtree. + #[test] + fn vector_maps_to_leaf_bit_and_subtree() { + let doorbell = GinVector::new::<129>(); + let gsp = GinVector::new::<155>(); + + assert_eq!(doorbell.leaf_index().get(), 4); + assert_eq!(doorbell.leaf_mask().into_raw(), 1 << 1); + assert_eq!(doorbell.subtree().index(), 2); + + assert_eq!(gsp.leaf_index().get(), 4); + assert_eq!(gsp.leaf_mask().into_raw(), 1 << 27); + assert_eq!(gsp.subtree().index(), 2); + + assert_eq!(doorbell.subtree(), gsp.subtree()); + } + + /// Both fixed vectors are within the 8-leaf tree, so every supported chipset implements them. + #[test] + fn fixed_vectors_fit_the_narrowest_tree() { + assert!(GinVector::new::<129>().validate(LeafCount::Eight).is_ok()); + assert!(GinVector::new::<155>().validate(LeafCount::Eight).is_ok()); + + // The first vector beyond an 8-leaf tree. + assert!(GinVector::new::<256>().validate(LeafCount::Eight).is_err()); + assert!(GinVector::new::<256>().validate(LeafCount::Sixteen).is_ok()); + } + + /// A subtree set reports membership, intersection, and its span from subtree 0. + #[test] + fn subtree_set_operations() { + let gsp = GinVector::new::<155>().subtree(); + + assert!(LeafCount::Eight.subtree_set().contains(gsp)); + assert!(!LeafCount::Eight.subtree_set().is_empty()); + + // The GSP needs no subtree above 2, so an MSI-X request covers entries 0 through 2. + assert_eq!(SubtreeSet::from(gsp).span(), 3); + + // A 16-leaf tree implements every subtree that an 8-leaf tree does. + assert_eq!( + LeafCount::Sixteen + .subtree_set() + .intersection(LeafCount::Eight.subtree_set()), + LeafCount::Eight.subtree_set() + ); + } + + /// Iterating a subtree set yields each subtree once, lowest index first, and nothing for an + /// empty set. + #[test] + fn subtree_set_iterates_its_members() { + assert!(LeafCount::Eight + .subtree_set() + .iter() + .map(Subtree::index) + .eq([0u32, 1, 2, 3])); + + let gsp = SubtreeSet::from(GinVector::new::<155>().subtree()); + assert!(gsp.iter().map(Subtree::index).eq([2u32])); + + let empty = SubtreeSet::from(Bounded::::new::<0>()); + assert_eq!(empty.iter().count(), 0); + } + + /// Every supported chipset implements the subtree that carries the GSP event. + #[test] + fn gsp_subtree_is_implemented_everywhere() { + for &chipset in Chipset::ALL { + assert!(cpu_interrupt_hal(chipset) + .leaf_count() + .subtree_set() + .contains(crate::irq::gsp::GSP_SUBTREE)); + } + } +} From df718311a84188e26d499301c3921e4c392d557e Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Tue, 29 Sep 2026 20:41:48 -0700 Subject: [PATCH 103/110] gpu: nova-core: document the GIN interrupt controller and GSP events The interrupt code rests on hardware behavior that the code cannot show on its own: how GIN, the GPU's interrupt controller, records and delivers interrupts, what edge-triggered delivery requires of a handler, and how the GSP signals the CPU. Some of those requirements come from OpenRM rather than from the hardware manuals. Add a design document that records that behavior, the rules that nova-core follows because of it, and the terms that the code uses for it. The code comments cite the document by section rather than repeating it. Assisted-by: LLM Signed-off-by: John Hubbard Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260930034148.590687-16-jhubbard@nvidia.com [acourbot: cut the last 3 sections.] Signed-off-by: Alexandre Courbot --- Documentation/gpu/nova/core/interrupts.rst | 603 +++++++++++++++++++++ Documentation/gpu/nova/index.rst | 1 + 2 files changed, 604 insertions(+) create mode 100644 Documentation/gpu/nova/core/interrupts.rst diff --git a/Documentation/gpu/nova/core/interrupts.rst b/Documentation/gpu/nova/core/interrupts.rst new file mode 100644 index 000000000000..e42286915711 --- /dev/null +++ b/Documentation/gpu/nova/core/interrupts.rst @@ -0,0 +1,603 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +============================================= +GPU interrupt handling: GIN and the GSP event +============================================= + +This document describes how nova-core receives interrupts from the GPU on Turing +and later chipsets. It covers the GPU Interrupt and Notification unit (GIN), +which is the GPU's interrupt controller, and the GSP event, the interrupt that +nova-core services in normal operation. + +Throughout, *CPU* means the CPU and the nova-core driver running on it. The GPU +also has on-chip processors that run their own firmware and receive their own +interrupts. The GSP (GPU System Processor) is one of them. + +Register names are the names from the GPU hardware reference headers. The +pre-Hopper headers call the controller ``NV_CTRL`` and the Hopper-plus headers +call it ``NV_GIN``. This document calls it GIN throughout, because the tree that +nova-core services is the same on every supported chipset. OpenRM, NVIDIA's +open-source GPU kernel driver, is cited wherever nova-core follows it. + +Terminology +=========== + +The three levels of the controller, innermost first: + +leaf + One ``LEAF`` register. Each of its 32 bits is the pending bit of one + interrupt source. A Turing, Ampere, or Ada tree has 8 leaves. A Hopper or + Blackwell tree has 16. + +subtree + Two consecutive leaves, summarized by one bit of ``TOP``. A subtree is the + unit of enabling at ``TOP``, and under MSI-X it is the unit of delivery: + every interrupt from one subtree arrives on one MSI-X entry. + +tree + One ``TOP`` register and the leaves under it. Every PCIe function has its + own tree, and nova-core services the CPU tree of one function. + +The hardware headers, OpenRM, and the Linux PCI API all use the word "vector", +each for a different number. This document gives each one its own name, and a +bare "vector" always means a GIN vector. + +GIN vector + The GPU-internal interrupt source number. It addresses one bit of one leaf. + A 16-leaf tree holds vectors 0 through 511, and an 8-leaf tree holds 0 + through 255. The CPU doorbell is vector 129 and the GSP event is vector + 155. + +MSI-X entry + An index into the device's MSI-X table. One entry serves one subtree. + +PCI vector + One of the interrupts that ``pci_alloc_irq_vectors()`` allocates: an MSI-X + entry, or the single MSI message. + +Linux IRQ number + What ``request_irq()`` takes, obtained from ``pci_irq_vector()`` for a PCI + vector. Linux's ``struct msix_entry`` calls this number ``.vector`` as + well. + +The remaining terms, each named for the register or the specification that +defines it: + +enable, disable a vector + Writes to ``LEAF_EN_SET`` and ``LEAF_EN_CLEAR``. + +enable, disable a subtree + Writes to ``TOP_EN_SET`` and ``TOP_EN_CLEAR``. + +serviced subtree + A subtree that nova-core enables and has a handler for. + +rearm + Restoring PCI interrupt delivery after servicing an interrupt. See + "Rearming PCI interrupt delivery". + +mask + Reserved for the two places where hardware and the PCI specification use + the word: the MSI-X per-entry Vector Control mask bit, which Linux + controls, and the falcon interrupt masks. It never names a GIN enable. + +latched, pending + Two names for one state, a set ``LEAF`` bit. The vector's source sets the + bit whether or not the vector is enabled. + +clear a vector + Write a 1 to the vector's bit in ``LEAF``. OpenRM calls the same operation + ``intrClearLeafVector_HAL``. + +pending bits + The plain 32-bit value read from a ``LEAF`` register. + +notification + An interrupt whose only content is that something happened, such as a + posted message. Servicing a notification means reading what it announces. + The unit that raised it needs no attention. The GSP event is one. + +unit + Any block that raises an interrupt. "Engine" is reserved for the blocks + that do user work: GR, CE, NVDEC, and the like. + +falcon + One of the GPU's microcontrollers (see + Documentation/gpu/nova/core/falcon.rst). The GSP runs on the RISC-V core + inside its falcon. A falcon latches each of its interrupt causes and routes + it either to the host, meaning the CPU, or to its own core. + +The GIN controller +================== + +A GPU has many interrupt sources: the GSP, the copy engines, the graphics +engine, video decode and encode, the MMU fault path, timers, and others. GIN +records which of them are pending and raises the PCI interrupt to the CPU. + +Trees +----- + +GIN keeps one tree for each destination it can deliver an interrupt to. The CPU +has one tree per PCIe function, so the physical function and each virtual +function have their own. The GSP has a tree, and so do the other on-chip +processors that receive interrupts. Every tree has the same two-level layout, +and a function reaches its own tree through the per-function register aperture. + +nova-core services the CPU tree of one function. A virtual function's tree +belongs to that function's driver, and a processor's tree belongs to the +firmware running on that processor. + +The two-level tree +------------------ + +A tree is a set of ``LEAF`` registers and one ``TOP`` register. + +* ``LEAF(i)`` is a 32-bit register that holds the pending bits of vectors + ``32i`` through ``32i + 31``. A set bit is a pending vector. +* ``TOP`` is a 32-bit read-only register. Bit ``N`` summarizes subtree ``N``, + which is ``LEAF(2N)`` and ``LEAF(2N + 1)``. The bit is set when an enabled + vector is pending in either leaf. + +A tree with L leaves has L / 2 subtrees and uses TOP bits 0 through L / 2 - 1. +The other TOP bits read 0. The leaves and subtrees that a chipset has are its +implemented leaves and subtrees, and "Per-architecture differences" gives the +counts. For an 8-leaf tree:: + + TOP bit 0 -> subtree 0 -> LEAF(0), LEAF(1) vectors 0..63 + TOP bit 1 -> subtree 1 -> LEAF(2), LEAF(3) vectors 64..127 + TOP bit 2 -> subtree 2 -> LEAF(4), LEAF(5) vectors 128..191 + TOP bit 3 -> subtree 3 -> LEAF(6), LEAF(7) vectors 192..255 + + LEAF(4), one bit per vector, holds vectors 128..159: + + bit 1 = vector 129 (CPU doorbell) + bit 27 = vector 155 (GSP event) + +A vector's number fixes its place in the tree:: + + leaf = vector / 32 + bit = vector % 32 + subtree = leaf / 2 + +Registers +--------- + +nova-core defines the tree's registers in the ``irq`` module's ``regs.rs``. The +leaf registers are arrays indexed by leaf number. + +* ``LEAF(i)`` reads as the pending bits of leaf ``i``. Writing a 1 to a bit + clears that vector, and a 0 leaves the bit as it was. +* ``LEAF_EN_SET(i)`` and ``LEAF_EN_CLEAR(i)`` enable and disable the vectors + of leaf ``i``, one bit per vector. +* ``TOP_EN_SET`` and ``TOP_EN_CLEAR`` enable and disable subtrees, one bit per + subtree. +* ``LEAF_TRIGGER`` takes a vector number and latches that vector, exactly as + the vector's own source would. It is write-only. The self-test uses it. + +nova-core does not read ``TOP``. "Servicing the tree" says why. + +Every set and clear register acts per bit: a 1 performs the action for that +bit, and a 0 leaves the bit alone. No register needs a read-modify-write. + +GIN delivers a vector to the CPU only when its leaf enable bit and its +subtree's TOP enable bit are both set. The enables do not affect the latch. The +source of a disabled vector still sets its ``LEAF`` bit. ``TOP`` does not show +that bit, so reading the leaf is the only way to see it. + +How a unit interrupt reaches the CPU +------------------------------------ + +A unit does not write a ``LEAF`` register. Each unit has an interrupt control +register that GSP firmware programs. The control register holds the unit's +vector, the GFID that identifies the PCIe function whose tree receives the +interrupt, and one enable bit per destination: the CPU, the GSP, and the other +on-chip processors. When the unit has an event:: + + 1. The unit sends GIN an interrupt message carrying the vector, the GFID, + and the destination enables from its control register. + 2. In the tree of each destination that the message selects, GIN sets bit + (vector % 32) of LEAF(vector / 32). + 3. If the vector and its subtree are enabled in the CPU tree, GIN raises + the PCI interrupt. + +Because firmware assigns the vectors, nova-core does not hardcode which vector +belongs to which unit, with two exceptions. The hardware headers of every +supported chipset define the GSP event as vector 155, and GSP firmware's own +interrupt table uses that definition. The CPU doorbell is vector 129. Pre-Hopper +hardware fixes that number, and GSP firmware keeps it on Hopper and later. +nova-core names both by number. A driver can fetch the full unit-to-vector +table from the GSP by RPC, and nouveau does. nova-core does not, because a +fixed vector needs no lookup. + +Edge-triggered delivery +----------------------- + +A ``LEAF`` bit is a latch. Its source sets it on a rising edge, and it stays set +until the CPU clears it. A source that stays high does not set the bit again. + +GIN raises the PCI interrupt for a subtree when the subtree's enabled pending +state goes from low to high:: + + Per vector, in leaf i at bit b: + LEAF(i)[b] AND LEAF_EN(i)[b] + + Per subtree N, across leaves 2N and 2N + 1: + OR of every enabled pending bit -> TOP[N] + + Delivery for subtree N: + TOP[N] AND TOP_EN[N] -> rising edge -> PCI interrupt + +``TOP_EN`` applies after the summary, so disabling a subtree stops delivery +without changing what ``TOP`` reports. + +Three consequences: + +* Code that must find every pending vector reads the leaves. A vector that + latched while disabled is not in ``TOP``. +* Writing ``TOP_EN_SET`` for a subtree with an enabled pending bit produces a + new edge. GIN delivers an interrupt for a pending bit left uncleared as soon + as its subtree is enabled again. +* A source that holds its signal high produces no new edge after the CPU + clears the leaf bit. Such a source has to re-emit its interrupt. The falcons + do that through ``INTR_RETRIGGER`` (see "Retriggering a falcon"). + +Delivery over PCI +================= + +GIN delivers the tree's interrupts to the CPU as MSI or MSI-X, whichever Linux +grants. nova-core requests MSI-X first and falls back to MSI, and never uses +INTx. + +MSI has a single message, and every subtree raises that one message, so one +Linux IRQ serves the whole tree. + +MSI-X gives each subtree its own table entry, at the index equal to the subtree +number. Linux masks every entry until a driver requests its Linux IRQ number, +and a masked entry sends no message: the GPU records the interrupt in the MSI-X +pending bit array, where it stays until Linux unmasks the entry. An entry that +the driver never requests is never unmasked. A driver that enables a subtree +without requesting that subtree's entry loses every interrupt from that +subtree, with nothing reported: the leaf and TOP registers show the vector +pending and enabled while no handler runs. + +The serviced-subtree invariant +------------------------------ + +Every subtree enabled at ``TOP`` has an allocated PCI vector with a registered +handler. + +MSI satisfies this with its single message. MSI-X needs one allocated entry per +serviced subtree, and a PCI allocation cannot be sparse, so nova-core requests +entries 0 through the highest serviced subtree:: + + MSI-X, with subtree 2 serviced: + + subtree 0 -> entry 0 allocated, no handler, stays masked + subtree 1 -> entry 1 allocated, no handler, stays masked + subtree 2 -> entry 2 handler here, and its rearm covers subtree 2 + + MSI, with any serviced set: + + every serviced subtree -> the one allocated PCI vector, whose + handler's rearm covers the whole serviced set + +An allocated entry whose subtree nova-core does not service costs nothing. The +entry stays masked, and a disabled subtree raises no interrupt. + +nova-core services one subtree. The GSP event, vector 155, is in leaf 4, which +is in subtree 2. OpenRM's headers place its UVM_SHARED interrupt category in +subtree 2 on every chipset that nova-core supports. The self-test doorbell, +vector 129, is in the same leaf, and the test allocates its own vectors for it. + +Rearming PCI interrupt delivery +------------------------------- + +A message-signaled interrupt is delivered once per edge, and the PCI side +delivers no further interrupt until the CPU rearms it. The rearm operation +depends on the GPU family and on the interrupt type that Linux granted: + +================== ===== =========================================== +Architecture Type Rearm operation +================== ===== =========================================== +Turing through Ada MSI write the configuration-mirror EOI register +Hopper and later MSI clear then set the serviced TOP enables +Any MSI-X clear then set the handler's own TOP enable +================== ===== =========================================== + +The end-of-interrupt register is ``NV_XVE_CYA_2`` in the BAR0 mirror of PCI +configuration space, and the value written does not matter. The ``TOP_EN`` +cycle produces a new delivery edge. The MSI forms cover every serviced subtree, +because one message serves all of them. The MSI-X form covers one subtree, +because each serviced subtree has its own entry and its own handler. + +A handler rearms once per delivered interrupt, on every path, including the +path where it finds its vector not pending. A handler that skips the rearm +receives no further interrupts. + +OpenRM makes the same split. It writes the configuration-space EOI for MSI on +pre-Hopper chipsets, and cycles the TOP enables of the subtrees that it +services for Hopper-plus MSI and for MSI-X. + +Servicing the tree +================== + +Servicing a leaf has a required order: read its pending bits, then clear them. +Clearing a leaf before reading it discards every vector latched in it, and no +register reports the loss. In nova-core, reading a leaf produces the handle that +clears it, so that the wrong order does not compile. The handle clears exactly +the bits that it read, so a vector that latched after the read stays pending. + +A handler clears its bit before it services the vector. Clearing afterwards +would discard an interrupt that the source raised while the handler ran. + +nova-core services the tree in two ways. + +The notification path services one vector. It reads the vector's leaf, clears +only the vector's bit, and rearms. The subtree stays enabled, and a vector +pending beside it in the same leaf keeps its bit set for the code that services +that vector. The GSP event handler takes this path, and so does the self-test +handler. + +The startup drain walks the whole tree, because it must clear whatever is +pending across every subtree rather than one known vector. It disables the +serviced subtrees at ``TOP``, reads and clears every implemented leaf, and +leaves the subtrees disabled for its caller to enable once the caller is ready +for deliveries. The drain reads every leaf rather than descending from ``TOP``, +because sources latch vectors during boot while those vectors are disabled, and +``TOP`` does not show them. OpenRM's stall-interrupt path reads every leaf for +the same reason. + +The two paths as register operations:: + + Startup drain, run once during probe: + write TOP_EN_CLEAR = serviced stop new deliveries + for each implemented leaf i: + pending = read LEAF(i) + write LEAF(i) = pending clear what was read + (returns with TOP_EN still clear) + + Notification, the subtree stays enabled: + pending = read LEAF(leaf) is the handler's bit set? + write LEAF(leaf) = bit clear that one bit + rearm PCI interrupt delivery + +The drain clears every pending bit, including bits that nova-core never +services. An uncleared bit holds its subtree in the pending state, and enabling +that subtree again would deliver an interrupt for a vector that no handler +services. + +The drain's ``TOP_EN_CLEAR`` is not a rearm, and pre-Hopper MSI rearms through +the configuration mirror, which the drain never writes. An interrupt delivered +before probe had no handler to rearm it, so the tree reset rearms explicitly +after the drain. A TOP-enable rearm leaves the serviced subtrees enabled, so +the reset then disables them again. + +nova-core does not serialize access to the tree. The GSP event handler +touches only its own leaf, and the drain runs during probe, before that handler +is registered. + +Per-architecture differences +============================ + +The tree is the same on every supported GPU except for its size, which changes +at Hopper: + +=================== ====== ======== ==================== +GPUs Leaves Subtrees Implemented subtrees +=================== ====== ======== ==================== +Turing, Ampere, Ada 8 4 ``0x0f`` +Hopper, Blackwell 16 8 ``0xff`` +=================== ====== ======== ==================== + +The interrupt HAL provides the leaf count, and the subtree count and the +implemented-subtree set derive from it. A subtree that the chipset does not +implement has no TOP bit, so building a tree that services one fails with +``EINVAL``. Vectors 129 and 155 are in the 8-leaf tree, so every supported +chipset has them. + +OpenRM's headers assign every interrupt category of a 16-leaf tree to leaves 0 +through 11. The drain reads all 16, because a vector can be latched in any +implemented leaf. + +The HAL performs the rearm as well (see "Rearming PCI interrupt delivery"). Two +falcon properties also differ by family, and the falcon HAL carries them: +Turing falcons have no ``INTR_RETRIGGER``, and the RISC-V routing registers +moved at GA102 (see "Retriggering a falcon"). + +The GSP event +============= + +When the GSP has output for the CPU, it writes messages into the GSP-to-CPU +queue in shared memory and raises SWGEN0, one of the software-generated +interrupt causes of the GSP falcon. SWGEN0 is routed to the host, at vector +155, leaf 4 bit 27, in subtree 2. + +The queue carries notifications (log records, error records, lifecycle events) +and command replies. A thread waiting for a reply reads the queue itself, so +the interrupt is only the trigger to drain the queue (see "Draining the +GSP-to-CPU queue"). + +The falcon latches every cause that it raises, SWGEN0 among them, in its +``IRQSTAT`` register. The handler services the host-routed causes and clears +their latches, and then it writes ``INTR_RETRIGGER`` so that the falcon re-emits +any cause that latched in the meantime. "Retriggering a falcon" has the details. + +Draining the queue takes the command-queue mutex and walks shared memory, so it +cannot run in hard interrupt context. nova-core registers a threaded handler, +under the name ``nova-core`` in ``/proc/interrupts``. The top half runs in hard +interrupt context and reads and writes only registers, and it wakes the IRQ +thread to drain the queue:: + + GSP writes messages into the GSP-to-CPU queue + GSP raises SWGEN0 + GIN sets bit 27 of LEAF(4), and subtree 2 becomes pending + PCI interrupt -> Linux IRQ -> top half, in hard interrupt context: + read LEAF(4), and if bit 27 is clear, rearm and return + clear bit 27 (the subtree stays enabled) + read the falcon causes routed to the host, clearing SWGEN0 if set + for every other host cause: log it, clear its latch, and read the + host causes back + if the clear ended all of them: retrigger the falcon + otherwise: disable vector 155 at its leaf and skip the retrigger + rearm PCI interrupt delivery + wake the IRQ thread if SWGEN0 was set + IRQ thread, which may sleep: + take the command-queue mutex and drain the GSP-to-CPU queue + +A halt and a posted message can be pending together, so the top half services +every cause that the status reports. + +A drain fails when a message's framing or checksum is bad (see "Draining the +GSP-to-CPU queue"). The message stays at the queue head, so every later drain +would fail the same way, and the IRQ thread disables vector 155 and logs the +failure, which leaves the queue unserviced until the device is reset. + +The handler, the self-test, and the rest of the driver read BAR0 through one +shared mapping. nova-core unregisters an interrupt handler when the device +unbinds, so a handler runs only while the mapping exists. + +Retriggering a falcon +--------------------- + +A falcon signals the tree when its set of host-routed causes goes from empty to +non-empty. A cause left latched keeps the set non-empty, so no later cause +signals the tree, and the vector is lost. For a cause that stays latched, the +handler can clear the tree leaf first or the falcon latch first, and the loss +is the same. + +``IRQSTAT`` latches every cause in the falcon, including the causes routed to +the falcon's own RISC-V core and owned by the firmware running on it. A host +handler owns only the causes that ``PRISCV_RISCV_IRQMASK`` and +``PRISCV_RISCV_IRQDEST`` both select, so it intersects ``IRQSTAT`` with both +before it reads or clears a cause. OpenRM computes the same intersection in +``kflcnRiscvReadIntrStatus``. GA100 keeps the Turing offsets of the two routing +registers and GA102 moves them, so the offsets change at GA102 rather than at +the Ampere boundary. The handler masks no cause: ``PRISCV_RISCV_IRQMASK`` is +read-only to the host, and ``FALCON_IRQMASK`` has no effect on host routing on +a RISC-V falcon. + +``INTR_RETRIGGER`` makes the falcon re-emit its host-routed causes into the +tree, which supplies the transition that clearing the leaf lost. The handler +writes ``INTR_RETRIGGER`` only on a path where it ended every cause that it +read, because a re-emitted cause that stays set arrives again at once and +on every pass after that. + +``IRQSCLR`` ends a latch and does not end the source behind it, so a cause +driven from outside the falcon stays set after the write. On Blackwell the +fault-containment and ECC causes are driven that way: they appear in +``IRQSTAT`` but come from ``PRISCV_RISCV_FAULT_CONTAINMENT_SRCSTAT`` and +``PGSP_ECC_INTR_STATUS``, and only a device reset ends them. So the handler +clears the latch of every host cause other than SWGEN0, reads the host causes +back, and retriggers only when the read-back is empty. When a cause is still +set, the handler disables vector 155 instead and reports that the device needs +a reset. Disabling loses no notification: the cause that is still set holds the +host-routed set non-empty, so the falcon would signal nothing further either +way. OpenRM makes the same choice, and ``kgspService_TU102`` skips +``kflcnIntrRetrigger`` once it has recorded a fatal error. + +The handler cannot distinguish a fault cause that arrives after the clear from +one that the clear failed to end, so it disables the vector in that case too. +Both mean that the GSP has faulted. + +Turing falcons have no ``INTR_RETRIGGER``, so a Turing handler cannot re-create +a transition that it has lost. It must leave no host cause latched: it reads the +host-routed status once and takes every cause that the status reports, rather +than stopping at the first one that it recognizes. One window stays open. A +cause that arrives after the handler has read the status is not in the value +that the handler clears, so it stays latched after the leaf has been cleared, +and no later cause from that falcon signals the tree. OpenRM has the same window +on Turing, where ``kflcnIntrRetrigger`` does nothing. + +Enabling the GSP event +---------------------- + +SWGEN0 is a latch, and the GSP drives no new edge into the tree while it stays +set. nova-core's GSP boot code consumes the GSP's notifications by polling the +queue, which leaves the latch set and leaves pending bits in the tree. The +handoff from polling to interrupts has a required order:: + + disable every implemented vector drop enables left by boot, or by a + driver that ran before this one + drain the tree clear stale pending bits + rearm PCI interrupt delivery required under pre-Hopper MSI, which + no later step rearms + disable subtree 2 at TOP a TOP-enable rearm enabled it again + clear the SWGEN0 latch so the next message makes an edge + register the threaded handler no delivery can reach it yet + enable vector 155 at LEAF(4) the subtree is still disabled + enable subtree 2 at TOP deliveries become possible here + drain the GSP-to-CPU queue messages posted before the clear + +The first four steps are the tree reset. nova-core runs them, and clears the +latch, before it registers the handler. Registering unmasks the PCI interrupt, +and GIN would then deliver a vector that boot left enabled to a handler that +services one vector and has no way to service any other. OpenRM clears every +leaf enable at the same point for the same reason. + +nova-core clears the latch after the drain. If nova-core cleared the latch +first, a message posted before the drain could set it again, along with bit 27 +of LEAF(4). The drain would then clear the leaf bit while the latch stays set, +and no later message would signal the tree. Clearing after the drain can +instead leave the leaf bit pending with the latch already clear. Enabling the +subtree then delivers one interrupt whose ``IRQSTAT`` reads zero. The top half +clears the leaf bit, rearms, and does not wake the IRQ thread, and the queue +drain that follows reads the message. + +Clearing the latch makes the first interrupt possible. A message that the GSP +posted before that clear produces no interrupt, so the sequence ends by +draining the queue. + +The subtree is enabled at ``TOP`` only once the handler is registered. A +TOP-enable rearm sets the enables that it cycles, so the reset disables the +serviced subtrees again after its rearm, and the tree reaches the registration +with subtree 2 disabled under every rearm method. On teardown nova-core disables +the vector at its leaf, so that no vector in the subtree can be delivered, then +calls ``free_irq()``, and disables the subtree last. Disabling the subtree +earlier would let a handler still in flight enable it again through the +``TOP_EN`` cycle of its rearm, which would leave the subtree enabled with no +handler registered. The driver's ``Gpu`` object, which owns the tree, holds the +subtree enable, and a handler's registration holds only the enable of its own +vector, so that tearing down one handler does not disable a subtree that another +handler shares. nova-core tears down the registration before it frees the queue +that the handler drains and the falcon that the handler reads, and before it +unloads the GSP. + +Draining the GSP-to-CPU queue +----------------------------- + +The queue carries command replies and unsolicited events, and a message's +function code says which it is. + +* A function code that matches the awaited reply: the message is decoded and + returned to the caller that sent the command. +* Anything else is an event. An OS error record and a robust-channel record + are logged at error level, and an unrecognized function code at warning + level. The other known events (GSP logs, libos prints, assertion records, + lifecycle notices) need no action and get no line of their own, because the + receive trace at debug level already records every message's arrival with + its sequence number, function code, and length. + +The sequence number takes no part in the match, because the GSP does not echo +the command's sequence number on every reply. On r570 the reply to +``UnloadingGuestDriver`` carries sequence 0. + +The read pointer advances past every message that passes framing and checksum +validation, whether it matched or was an event. A matched message that is too +short to decode is the exception, and it stays at the queue head. + +A message's length is inside the region that the checksum covers, so once the +framing or the checksum fails there is no trustworthy length with which to skip +the message. Such a message also stays at the queue head, and every later +receive fails on it with ``EIO`` until the device is reset. + +The polling path and the IRQ thread both read the queue under the command-queue +mutex. Replies and events share one queue and one read pointer, so one lock is +held across the whole drain. A thread waiting for a reply logs each event that +arrives before the reply and keeps waiting. One deadline of 5 seconds applies to +the whole wait, rather than a fresh timeout after each message, and the thread +holds the mutex for the whole wait, so that no other caller consumes the message +that it waits for. + +With one lock, a drain waits for an in-flight command's receive to finish or +time out. For log and error records that delay does not matter. diff --git a/Documentation/gpu/nova/index.rst b/Documentation/gpu/nova/index.rst index 59b206238498..224caef9ea42 100644 --- a/Documentation/gpu/nova/index.rst +++ b/Documentation/gpu/nova/index.rst @@ -35,3 +35,4 @@ vGPU manager VFIO driver and the nova-drm driver. core/falcon core/tlv core/pramin + core/interrupts From dc90f13368ac509b05189089bed178858234c050 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Sun, 4 Oct 2026 10:00:49 +0900 Subject: [PATCH 104/110] gpu: nova-core: fix warning with NOVA_CORE_SELFTESTS disabled A module-wide new import becomes unused if the `NOVA_CORE_SELFTESTS` config option is disabled: warning: unused import: `Spec` --> ../drivers/gpu/nova-core/driver.rs:29:9 | 29 | Spec, // | ^^^^ | Fix this by fully qualifying the unique use of `Spec` when the tests are enabled. Fixes: 9cb3f5d56e81 ("gpu: nova-core: add an interrupt delivery self-test") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202610031630.JOlyp48X-lkp@intel.com/ Reviewed-by: John Hubbard Link: https://patch.msgid.link/20261004-nova-int-fixup-v1-1-9e4bce097f94@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/driver.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index d195fd2bcf8c..560a70c4f5a7 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -24,10 +24,7 @@ use crate::{ api::NovaCoreApi, - gpu::{ - Gpu, - Spec, // - }, // + gpu::Gpu, // }; /// Counter for generating unique auxiliary device IDs. @@ -121,7 +118,7 @@ fn probe<'bound>( // Run self-tests that do not depend on the `Gpu` instance. #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] _: { - let spec = Spec::new(pdev.as_ref(), bar)?; + let spec = crate::gpu::Spec::new(pdev.as_ref(), bar)?; // We must wait for GFW_BOOT completion before doing any significant setup on // the GPU. From 04476103b7bc58d6919ecec1b52e0166a426c187 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 27 Sep 2026 16:22:09 +0200 Subject: [PATCH 105/110] drm: nova: Move VRAM BAR size query to GPU info Nova exposes static information via its DRM_IOCTL_NOVA_INFO ioctl, thus add vram_bar_size to struct drm_nova_info_gpu and remove DRM_IOCTL_NOVA_GETPARAM and its uAPI definitions. This breaks the uAPI, which is still fine at this point. It requires the corresponding drm-test change in [1]. Link: https://gitlab.freedesktop.org/dakr/drm-test/-/merge_requests/2 [1] Reviewed-by: Alexandre Courbot Reviewed-by: Eliot Courtney Reviewed-by: Alistair Popple Link: https://patch.msgid.link/20260927142258.1260711-1-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/driver.rs | 3 +-- drivers/gpu/drm/nova/file.rs | 18 +---------------- include/uapi/drm/nova_drm.h | 36 ++++++++-------------------------- 3 files changed, 10 insertions(+), 47 deletions(-) diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index 46e13fe795eb..851d50b4dd15 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -95,9 +95,8 @@ impl drm::Driver for NovaDriver { const FEAT_RENDER: bool = true; kernel::declare_drm_ioctls! { - (NOVA_GETPARAM, drm_nova_getparam, ioctl::RENDER_ALLOW, File::get_param), + (NOVA_INFO, drm_nova_info, ioctl::RENDER_ALLOW, File::info), (NOVA_GEM_CREATE, drm_nova_gem_create, ioctl::AUTH | ioctl::RENDER_ALLOW, File::gem_create), (NOVA_GEM_INFO, drm_nova_gem_info, ioctl::AUTH | ioctl::RENDER_ALLOW, File::gem_info), - (NOVA_INFO, drm_nova_info, ioctl::RENDER_ALLOW, File::info), } } diff --git a/drivers/gpu/drm/nova/file.rs b/drivers/gpu/drm/nova/file.rs index 766a6ddb7e10..a05264917a0b 100644 --- a/drivers/gpu/drm/nova/file.rs +++ b/drivers/gpu/drm/nova/file.rs @@ -60,6 +60,7 @@ fn new(reg_data: &DrmRegData<'_>) -> Result { architecture: spec.chipset.arch().into(), chipid: spec.chipset.into(), vram_size: gsp_static_info.vram_size(), + vram_bar_size: reg_data.api.with(|api| api.bar1_size())?, gpu_gid: gsp_static_info.gpu_gid, ..pin_init::zeroed() }; @@ -108,23 +109,6 @@ fn open(_dev: &NovaDevice) -> Result>> { } impl File { - /// IOCTL: get_param: Query GPU / driver metadata. - pub(crate) fn get_param( - _dev: &NovaDevice, - reg_data: &DrmRegData<'_>, - getparam: &mut uapi::drm_nova_getparam, - _file: &drm::File, - ) -> Result { - let value = match getparam.param.try_into()? { - uapi::NOVA_GETPARAM_VRAM_BAR_SIZE => reg_data.api.with(|api| api.bar1_size())?, - _ => return Err(EINVAL), - }; - - getparam.value = Into::::into(value); - - Ok(0) - } - /// IOCTL: gem_create: Create a new DRM GEM object. pub(crate) fn gem_create( dev: &NovaDevice, diff --git a/include/uapi/drm/nova_drm.h b/include/uapi/drm/nova_drm.h index a74daba6ea94..d7017ed17143 100644 --- a/include/uapi/drm/nova_drm.h +++ b/include/uapi/drm/nova_drm.h @@ -18,13 +18,6 @@ extern "C" { #endif -/* - * NOVA_GETPARAM_VRAM_BAR_SIZE - * - * Query the VRAM BAR size in bytes. - */ -#define NOVA_GETPARAM_VRAM_BAR_SIZE 0x1 - /** * enum drm_nova_architecture - GPU architecture identifier */ @@ -77,21 +70,6 @@ enum drm_nova_chipid { NOVA_DRM_CHIPID_GB207 = 0x1b7, }; -/** - * struct drm_nova_getparam - query GPU and driver metadata - */ -struct drm_nova_getparam { - /** - * @param: The identifier of the parameter to query. - */ - __u64 param; - - /** - * @value: The value for the specified parameter. - */ - __u64 value; -}; - /** * struct drm_nova_gem_create - create a new DRM GEM object */ @@ -189,6 +167,11 @@ struct drm_nova_info_gpu { */ __u64 vram_size; + /** + * @vram_bar_size: Size of the PCI BAR aperture used to access VRAM, in bytes. + */ + __u64 vram_bar_size; + /** * @gpu_name: NUL-terminated full GPU name. */ @@ -205,21 +188,18 @@ struct drm_nova_info_gpu { __u8 gpu_gid[16]; }; -#define DRM_NOVA_GETPARAM 0x00 +#define DRM_NOVA_INFO 0x00 #define DRM_NOVA_GEM_CREATE 0x01 #define DRM_NOVA_GEM_INFO 0x02 -#define DRM_NOVA_INFO 0x03 /* Note: this is an enum so that it can be resolved by Rust bindgen. */ enum { - DRM_IOCTL_NOVA_GETPARAM = DRM_IOWR(DRM_COMMAND_BASE + DRM_NOVA_GETPARAM, - struct drm_nova_getparam), + DRM_IOCTL_NOVA_INFO = DRM_IOWR(DRM_COMMAND_BASE + DRM_NOVA_INFO, + struct drm_nova_info), DRM_IOCTL_NOVA_GEM_CREATE = DRM_IOWR(DRM_COMMAND_BASE + DRM_NOVA_GEM_CREATE, struct drm_nova_gem_create), DRM_IOCTL_NOVA_GEM_INFO = DRM_IOWR(DRM_COMMAND_BASE + DRM_NOVA_GEM_INFO, struct drm_nova_gem_info), - DRM_IOCTL_NOVA_INFO = DRM_IOWR(DRM_COMMAND_BASE + DRM_NOVA_INFO, - struct drm_nova_info), }; #if defined(__cplusplus) From e56059016a8753f26dcd6c9d826ccfdd45e8b80c Mon Sep 17 00:00:00 2001 From: Alistair Popple Date: Thu, 1 Oct 2026 10:56:15 +1000 Subject: [PATCH 106/110] gpu: nova-core: Use bar1_resource_index() in NovaCoreApi::bar1_size() bar1_size() reports the length of PCI resource 1 as the size of BAR1. That is only correct when BAR0 is a 32-bit BAR. On Blackwell, and on some Ampere GA100 and Hopper SKUs, BAR0 is 64-bit and takes resource slots 0 and 1, pushing BAR1 to slot 2. Slot 1 is then the upper half of BAR0 rather than a BAR of its own, so NOVA_GETPARAM_VRAM_BAR_SIZE reports the wrong size on those GPUs. Probe handles this with the new bar1_resource_index(), which it uses to map BAR1, so use the same lookup here. Fixes: 7edc491f6fca ("drm: nova: Use nova-core to read VRAM_BAR_SIZE parameter") Signed-off-by: Alistair Popple Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20261001005615.77130-1-apopple@nvidia.com [ Import the driver module. - Danilo ] Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/api.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/nova-core/api.rs b/drivers/gpu/nova-core/api.rs index f02c6c7c7e51..29207c4bb743 100644 --- a/drivers/gpu/nova-core/api.rs +++ b/drivers/gpu/nova-core/api.rs @@ -15,8 +15,11 @@ pub use crate::gpu::Spec; -use crate::gpu::Gpu; -use crate::gsp::commands::GetGspStaticInfoReply; +use crate::{ + driver, + gpu::Gpu, + gsp::commands::GetGspStaticInfoReply, // +}; /// API handle for the auxiliary bus child drivers to interact with nova-core. pub struct NovaCoreApi<'bound> { @@ -44,7 +47,9 @@ pub fn spec(&self) -> &Spec { /// Returns the size of the PCIe BAR used for accessing VRAM, typically /// BAR1. pub fn bar1_size(&self) -> Result { - self.pdev.resource_len(1) + let bar1 = driver::bar1_resource_index(self.pdev)?; + + self.pdev.resource_len(bar1) } } From 6148fb2fc4d3cce734b0fc1b26baceff0da90094 Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Tue, 6 Oct 2026 13:53:06 +0200 Subject: [PATCH 107/110] gpu: nova-core: gsp: use KVVec::zeroed() for payload buffers Use KVVec::zeroed() instead of KVVec::from_elem() to allocate and zero-initialize the command and continuation payload buffers. Also use KVVec::zeroed() in the test helper read_payload(). Signed-off-by: Thorsten Blum Link: https://patch.msgid.link/20261006115311.92873-3-blum@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/gsp/cmdq/continuation.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/cmdq/continuation.rs b/drivers/gpu/nova-core/gsp/cmdq/continuation.rs index 05e904f18097..9ef19f28de97 100644 --- a/drivers/gpu/nova-core/gsp/cmdq/continuation.rs +++ b/drivers/gpu/nova-core/gsp/cmdq/continuation.rs @@ -106,9 +106,9 @@ pub(super) fn new(command: C) -> Result { if command.size() > MAX_CMD_SIZE { let mut command_payload = - KVVec::::from_elem(0u8, payload_len.min(Self::MAX_FIRST_PAYLOAD), GFP_KERNEL)?; + KVVec::zeroed(payload_len.min(Self::MAX_FIRST_PAYLOAD), GFP_KERNEL)?; let mut continuation_payload = - KVVec::::from_elem(0u8, payload_len - command_payload.len(), GFP_KERNEL)?; + KVVec::zeroed(payload_len - command_payload.len(), GFP_KERNEL)?; let mut sbuffer = SBufferIter::new_writer([ command_payload.as_mut_slice(), continuation_payload.as_mut_slice(), @@ -235,7 +235,7 @@ fn init_variable_payload( fn read_payload(cmd: impl CommandToGsp) -> Result> { let len = cmd.variable_payload_len(); - let mut buf = KVVec::from_elem(0u8, len, GFP_KERNEL)?; + let mut buf = KVVec::zeroed(len, GFP_KERNEL)?; let mut sbuf = SBufferIter::new_writer([buf.as_mut_slice(), &mut []]); cmd.init_variable_payload(&mut sbuf)?; drop(sbuf); From 3e3404c52370595e37eb5b942df169884f38a124 Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Tue, 6 Oct 2026 08:54:35 +0200 Subject: [PATCH 108/110] gpu: nova-core: fsp: use KVec::zeroed() in recv_msg() Use KVec::zeroed() to create the message buffer in recv_msg() instead of creating an empty vector and resizing it. Signed-off-by: Thorsten Blum Link: https://patch.msgid.link/20261006065437.58408-3-blum@kernel.org [ Avoid the unnecessary turbofish syntax. - Danilo ] Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/falcon/fsp.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/gpu/nova-core/falcon/fsp.rs b/drivers/gpu/nova-core/falcon/fsp.rs index 85f9c8c5d60e..d7c213848005 100644 --- a/drivers/gpu/nova-core/falcon/fsp.rs +++ b/drivers/gpu/nova-core/falcon/fsp.rs @@ -173,9 +173,7 @@ pub(crate) fn recv_msg(&mut self) -> Result> { return Err(EMSGSIZE); } - let mut buffer = KVec::::new(); - buffer.resize(msg_size, 0, GFP_KERNEL)?; - + let mut buffer = KVec::zeroed(msg_size, GFP_KERNEL)?; self.read_emem(&mut buffer)?; // Reset message queue pointers after reading. From 700eed1364e8a5ec4318d457cf700611decdb175 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Tue, 29 Sep 2026 20:59:30 +0200 Subject: [PATCH 109/110] rust: drm: Return impl PinInit from drm::Registration::new() Convert drm::Registration to return an impl PinInit rather than creating a separate allocation. This removes an otherwise unnecessary allocation. Update both in-tree callers (nova, tyr) to construct their bus device private data using try_pin_init!(). Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20260929190024.2248099-1-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/driver.rs | 25 ++++---- drivers/gpu/drm/tyr/driver.rs | 114 +++++++++++++++++---------------- rust/kernel/drm/driver.rs | 74 ++++++++++++--------- 3 files changed, 116 insertions(+), 97 deletions(-) diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index 851d50b4dd15..7a51d49feb1f 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -25,10 +25,11 @@ pub(crate) struct NovaDriver; +#[pin_data] pub(crate) struct Nova<'bound> { - #[expect(unused)] drm: ARef>, - _reg: drm::Registration<'bound, NovaDriver>, + #[pin] + reg: drm::Registration<'bound, NovaDriver>, } /// DRM registration data, accessible from ioctl handlers via the registration guard. @@ -68,17 +69,19 @@ fn probe<'bound>( adev: &'bound auxiliary::Device>, _info: &'bound Self::IdInfo, ) -> impl PinInit, Error> + 'bound { - let drm = drm::UnregisteredDevice::::new(adev, Ok(()))?; - let reg_data = DrmRegData { - api: NovaCoreApi::of(adev)?, - }; - // SAFETY: `reg` is stored in `Nova` and dropped when the driver is unbound; it is - // never forgotten. - let reg = unsafe { drm::Registration::new(adev.as_ref(), drm, reg_data, 0)? }; + try_pin_init!(Self::Data { + reg <- { + let drm = drm::UnregisteredDevice::::new(adev, Ok(()))?; - Ok(Nova { + let reg_data = DrmRegData { + api: NovaCoreApi::of(adev)?, + }; + + // SAFETY: `reg` is stored in `Self::Data` and dropped when the driver is unbound; + // it is never forgotten. + unsafe { drm::Registration::new(adev.as_ref(), drm, reg_data, 0) } + }, drm: reg.device().into(), - _reg: reg, }) } } diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index 730b84e37a54..6e1008290b5f 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -57,6 +57,7 @@ #[pin_data(PinnedDrop)] pub(crate) struct TyrPlatformDriverData<'bound> { + #[pin] _reg: drm::Registration<'bound, TyrDrmDriver>, } @@ -117,73 +118,76 @@ fn probe<'bound>( pdev: &'bound platform::Device>, _info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { - let core_clk = Clk::get(pdev.as_ref(), Some(c"core"))?; - let stacks_clk = OptionalClk::get(pdev.as_ref(), Some(c"stacks"))?; - let coregroup_clk = OptionalClk::get(pdev.as_ref(), Some(c"coregroup"))?; + pin_init::pin_init_scope(move || { + let core_clk = Clk::get(pdev.as_ref(), Some(c"core"))?; + let stacks_clk = OptionalClk::get(pdev.as_ref(), Some(c"stacks"))?; + let coregroup_clk = OptionalClk::get(pdev.as_ref(), Some(c"coregroup"))?; - core_clk.prepare_enable()?; - stacks_clk.prepare_enable()?; - coregroup_clk.prepare_enable()?; + core_clk.prepare_enable()?; + stacks_clk.prepare_enable()?; + coregroup_clk.prepare_enable()?; - let mali_regulator = Regulator::::get(pdev.as_ref(), c"mali")?; - let sram_regulator = Regulator::::get(pdev.as_ref(), c"sram")?; + let mali_regulator = Regulator::::get(pdev.as_ref(), c"mali")?; + let sram_regulator = Regulator::::get(pdev.as_ref(), c"sram")?; - let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; + let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; - let iomem = Arc::new(request.iomap_sized::()?, GFP_KERNEL)?; + let iomem = Arc::new(request.iomap_sized::()?, GFP_KERNEL)?; - issue_soft_reset(pdev.as_ref(), &iomem)?; - gpu::l2_power_on(pdev.as_ref(), &iomem)?; + issue_soft_reset(pdev.as_ref(), &iomem)?; + gpu::l2_power_on(pdev.as_ref(), &iomem)?; - let gpu_info = GpuInfo::new(&iomem); - gpu_info.log(pdev.as_ref()); + let gpu_info = GpuInfo::new(&iomem); + gpu_info.log(pdev.as_ref()); - let pa_bits = MMU_FEATURES::from_raw(gpu_info.mmu_features) - .pa_bits() - .get(); - // SAFETY: No concurrent DMA allocations or mappings can be made because - // the device is still being probed and therefore isn't being used by - // other threads of execution. - unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(pa_bits)?)? }; + let pa_bits = MMU_FEATURES::from_raw(gpu_info.mmu_features) + .pa_bits() + .get(); + // SAFETY: No concurrent DMA allocations or mappings can be made because + // the device is still being probed and therefore isn't being used by + // other threads of execution. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(pa_bits)?)? }; - let unreg_dev = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + let unreg_dev = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - let mmu = Mmu::new(pdev.as_ref(), iomem.as_arc_borrow(), &gpu_info)?; + let mmu = Mmu::new(pdev.as_ref(), iomem.as_arc_borrow(), &gpu_info)?; - let firmware = Firmware::new( - pdev.as_ref(), - iomem.clone(), - &unreg_dev, - mmu.as_arc_borrow(), - &gpu_info, - )?; + let firmware = Firmware::new( + pdev.as_ref(), + iomem.clone(), + &unreg_dev, + mmu.as_arc_borrow(), + &gpu_info, + )?; - firmware.boot()?; + firmware.boot()?; - let reg_data = pin_init!(TyrDrmRegistrationData { - pdev, - fw: firmware, - clks <- new_mutex!(Clocks { - core: core_clk, - stacks: stacks_clk, - coregroup: coregroup_clk, - }), - regulators <- new_mutex!(Regulators { - _mali: mali_regulator, - _sram: sram_regulator, - }), - iomem, - gpu_info, - }); - - // SAFETY: `reg` is stored in `TyrPlatformDriverData` and dropped when the driver is - // unbound; it is never forgotten. - let reg = unsafe { drm::Registration::new(pdev.as_ref(), unreg_dev, reg_data, 0)? }; - - let driver = TyrPlatformDriverData { _reg: reg }; - - dev_dbg!(pdev, "Tyr initialized correctly."); - Ok(driver) + Ok(try_pin_init!(TyrPlatformDriverData { + // SAFETY: `_reg` is stored in `TyrPlatformDriverData` and dropped when the + // driver is unbound; it is never forgotten. + _reg <- unsafe { drm::Registration::new( + pdev.as_ref(), + unreg_dev, + pin_init!(TyrDrmRegistrationData { + pdev, + fw: firmware, + clks <- new_mutex!(Clocks { + core: core_clk, + stacks: stacks_clk, + coregroup: coregroup_clk, + }), + regulators <- new_mutex!(Regulators { + _mali: mali_regulator, + _sram: sram_regulator, + }), + iomem, + gpu_info, + }), + 0, + )}, + _: { dev_dbg!(pdev, "Tyr initialized correctly.") }, + })) + }) } } diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 74f6ed690d8b..13aa609350b1 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -12,7 +12,10 @@ prelude::*, sync::aref::ARef, // }; -use core::ptr::NonNull; +use core::{ + marker::PhantomPinned, + ptr::NonNull, // +}; /// Driver use the GEM memory manager. This should be set for all modern drivers. pub(crate) const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM; @@ -143,9 +146,15 @@ pub trait Driver { /// The registration type of a `drm::Device`. /// /// Once the `Registration` structure is dropped, the device is unregistered. +#[pin_data(PinnedDrop)] pub struct Registration<'a, T: Driver> { drm: ARef>, - _reg_data: Pin>>, + #[pin] + data: T::RegistrationData<'a>, + /// Even if `T::RegistrationData` is `Unpin`, the registration must not be moved because a + /// pointer to `data` is stored in the DRM device. + #[pin] + _pin: PhantomPinned, } impl<'a, T: Driver> Registration<'a, T> { @@ -159,39 +168,41 @@ impl<'a, T: Driver> Registration<'a, T> { pub unsafe fn new( dev: &'a device::Device, drm: drm::UnregisteredDevice, - reg_data: impl PinInit, E>, + data: impl PinInit, E>, flags: usize, - ) -> Result + ) -> impl PinInit where Error: From, { - let parent = drm.as_ref(); - if parent.as_ref().as_raw() != dev.as_raw() { - return Err(EINVAL); - } + try_pin_init!(Self { + _: { + let parent = drm.as_ref(); + if parent.as_ref().as_raw() != dev.as_raw() { + return Err(EINVAL); + } + }, - let reg_data: Pin>> = KBox::pin_init(reg_data, GFP_KERNEL)?; - - // Store the registration data pointer in the device before registration, so that it is - // visible once ioctls can be called. - let ptr: NonNull> = - NonNull::from(Pin::get_ref(reg_data.as_ref())).cast(); - - // SAFETY: No concurrent access; the device is not yet registered. - unsafe { *drm.registration_data.get() = ptr }; - - // SAFETY: `drm` is a valid, initialized but not yet registered DRM device. - let ret = unsafe { bindings::drm_dev_register(drm.as_raw(), flags) }; - if let Err(e) = to_result(ret) { - // SAFETY: `drm_dev_register()` synchronizes SRCU on failure, so no concurrent - // access to `registration_data` is possible at this point. - unsafe { *drm.registration_data.get() = NonNull::dangling() }; - return Err(e); - } - - Ok(Self { + data <- data, drm: (&*drm).into(), - _reg_data: reg_data, + _pin: PhantomPinned, + + _: { + // Store the registration data pointer in the device before registration, so + // that it is visible once ioctls can be called. + let ptr: NonNull> = + NonNull::from(data.as_ref().get_ref()).cast(); + + // SAFETY: No concurrent access; the device is not yet registered. + unsafe { *drm.registration_data.get() = ptr }; + + // SAFETY: `drm` is a valid, initialized but not yet registered DRM device. + let ret = unsafe { bindings::drm_dev_register(drm.as_raw(), flags) }; + to_result(ret).inspect_err(|_| { + // SAFETY: `drm_dev_register()` synchronizes SRCU on failure, so no + // concurrent access to `registration_data` is possible at this point. + unsafe { *drm.registration_data.get() = NonNull::dangling() }; + })?; + }, }) } @@ -208,8 +219,9 @@ unsafe impl Sync for Registration<'_, T> {} // SAFETY: Registration with and unregistration from the DRM subsystem can happen from any thread. unsafe impl Send for Registration<'_, T> {} -impl Drop for Registration<'_, T> { - fn drop(&mut self) { +#[pinned_drop] +impl PinnedDrop for Registration<'_, T> { + fn drop(self: Pin<&mut Self>) { // Use `drm_dev_unplug` rather than `drm_dev_unregister` to ensure that existing // `drm_dev_enter()` critical sections complete before unregistration proceeds. This // is required for the safety of `RegistrationGuard`, which relies on the SRCU barrier in From d511064b2e732b7152a542d589d07a4814c1b674 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Tue, 29 Sep 2026 20:59:31 +0200 Subject: [PATCH 110/110] rust: drm: Remove unused flags argument from Registration::new() The flags argument to drm_dev_register() is only passed to the deprecated .load() callback. Rust DRM drivers always set this callback to NULL, so the argument has no effect. Remove the argument from drm::Registration::new(), pass zero to drm_dev_register(), and update the Nova and Tyr callers. Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20260929190024.2248099-2-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/driver.rs | 2 +- drivers/gpu/drm/tyr/driver.rs | 1 - rust/kernel/drm/driver.rs | 3 +-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index 7a51d49feb1f..44a52e60b3c7 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -79,7 +79,7 @@ fn probe<'bound>( // SAFETY: `reg` is stored in `Self::Data` and dropped when the driver is unbound; // it is never forgotten. - unsafe { drm::Registration::new(adev.as_ref(), drm, reg_data, 0) } + unsafe { drm::Registration::new(adev.as_ref(), drm, reg_data) } }, drm: reg.device().into(), }) diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index 6e1008290b5f..c2b99d67369f 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -183,7 +183,6 @@ fn probe<'bound>( iomem, gpu_info, }), - 0, )}, _: { dev_dbg!(pdev, "Tyr initialized correctly.") }, })) diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 13aa609350b1..f6db91e81e79 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -169,7 +169,6 @@ pub unsafe fn new( dev: &'a device::Device, drm: drm::UnregisteredDevice, data: impl PinInit, E>, - flags: usize, ) -> impl PinInit where Error: From, @@ -196,7 +195,7 @@ pub unsafe fn new( unsafe { *drm.registration_data.get() = ptr }; // SAFETY: `drm` is a valid, initialized but not yet registered DRM device. - let ret = unsafe { bindings::drm_dev_register(drm.as_raw(), flags) }; + let ret = unsafe { bindings::drm_dev_register(drm.as_raw(), 0) }; to_result(ret).inspect_err(|_| { // SAFETY: `drm_dev_register()` synchronizes SRCU on failure, so no // concurrent access to `registration_data` is possible at this point.