diff --git a/drivers/gpu/nova-core/falcon/gsp.rs b/drivers/gpu/nova-core/falcon/gsp.rs index 4c96ae325fda..1faefaede9fe 100644 --- a/drivers/gpu/nova-core/falcon/gsp.rs +++ b/drivers/gpu/nova-core/falcon/gsp.rs @@ -47,13 +47,56 @@ fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, super::PFalcon2Registers> { } impl<'a> Falcon<'a, Gsp> { - /// Clears the SWGEN0 bit in the Falcon's IRQ status clear register to - /// allow GSP to signal CPU for processing new messages in message queue. + /// Clears the SWGEN0 latch in the GSP falcon. + /// + /// While the latch is set, no later message signals the tree, so a caller that consumed a + /// notification by polling must clear it. pub(crate) fn clear_swgen0_intr(&self) { self.pfalcon .write_reg(regs::NV_PFALCON_FALCON_IRQSCLR::zeroed().with_swgen0(true)); } + /// Reads the GSP falcon causes that are routed to the host, without clearing any latch. + /// + /// Every one of them other than SWGEN0 reports a GSP fault. + pub(crate) fn read_host_intr(&self) -> regs::NV_PFALCON_FALCON_IRQSTAT { + let latched = self.pfalcon.read(regs::NV_PFALCON_FALCON_IRQSTAT); + + self.hal.host_routed_causes(self, latched) + } + + /// Reads the host-routed causes and clears the SWGEN0 latch if it was set. + /// + /// Returns the causes as read, before the clear. No other latch changes. + pub(crate) fn take_host_intr(&self) -> regs::NV_PFALCON_FALCON_IRQSTAT { + let status = self.read_host_intr(); + + if status.swgen0() { + self.clear_swgen0_intr(); + } + + status + } + + /// Clears the latch of every interrupt cause set in `status`. + /// + /// A cause driven from outside the falcon is still set on return, and + /// [`Self::read_host_intr`] reports the causes that remain. + pub(crate) fn clear_intr(&self, status: regs::NV_PFALCON_FALCON_IRQSTAT) { + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_IRQSCLR::from(status.into_raw())); + } + + /// Retriggers the GSP falcon, which then re-emits its host-routed causes into the tree. + /// + /// Call this only once every host cause is clear. A cause still set is re-emitted at once, and + /// its vector arrives again as soon as delivery is rearmed. + /// + /// Does nothing on Turing, whose falcons have no retrigger register. + pub(crate) fn retrigger_intr(&self) { + self.hal.retrigger(self); + } + /// Checks if GSP reload/resume has completed during the boot process. pub(crate) fn check_reload_completed(&self, timeout: Delta) -> Result { read_poll_timeout( diff --git a/drivers/gpu/nova-core/falcon/hal.rs b/drivers/gpu/nova-core/falcon/hal.rs index cd2613a74595..de7825719ebc 100644 --- a/drivers/gpu/nova-core/falcon/hal.rs +++ b/drivers/gpu/nova-core/falcon/hal.rs @@ -78,7 +78,6 @@ fn signature_reg_fuse_version( /// /// The causes routed to the core belong to the firmware running on it, and the host does not /// service them. - #[expect(dead_code)] fn host_routed_causes( &self, falcon: &Falcon<'_, E>, @@ -86,7 +85,6 @@ fn host_routed_causes( ) -> regs::NV_PFALCON_FALCON_IRQSTAT; /// Retriggers the falcon, which then re-emits its host-routed causes into the interrupt tree. - #[expect(dead_code)] fn retrigger(&self, falcon: &Falcon<'_, E>); } diff --git a/drivers/gpu/nova-core/falcon/hal/tu102.rs b/drivers/gpu/nova-core/falcon/hal/tu102.rs index 94c4438d3ebf..010ca5c36fc6 100644 --- a/drivers/gpu/nova-core/falcon/hal/tu102.rs +++ b/drivers/gpu/nova-core/falcon/hal/tu102.rs @@ -26,7 +26,6 @@ use super::FalconHal; /// Writes the trigger register available on GA100 and more recent. -#[expect(dead_code)] pub(super) fn retrigger_ga100(falcon: &Falcon<'_, E>) { falcon.pfalcon.write( Array::at(0), @@ -38,7 +37,6 @@ pub(super) fn retrigger_ga100(falcon: &Falcon<'_, E>) { /// retrigger register. pub(super) struct Tu102 { /// If `true`, the falcons have `NV_PFALCON_FALCON_INTR_RETRIGGER`. - #[expect(dead_code)] has_intr_retrigger: bool, _engine: PhantomData, } diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index f91c7e22a303..4e59072cab50 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -34,10 +34,19 @@ fsp::Fsp, gsp::{ self, + cmdq::Cmdq, commands::GetGspStaticInfoReply, Gsp, GspBootContext, // }, + irq::{ + self, + gsp::GspIrq, + interrupt_tree::{ + TopEnableGuard, + Tree, // + }, // + }, mm::{ bar_user::BarUser, pagetable::MmuVersion, @@ -331,10 +340,53 @@ struct GspResources<'gpu> { unload_bundle: Option>, } +/// The GSP event handler's registration and the enable of its subtree at `TOP`. +/// +/// The two drop as a unit, the handler first, on the drop of [`Gpu`] and on the error path of +/// its constructor alike, so that the subtree is disabled only after the handler is freed. +#[pin_data] +struct GspSubtree<'a> { + #[pin] + irq: GspIrq<'a>, + /// Must be kept declared *after* `irq`. A handler still in flight enables the subtree again + /// through its rearm. + _top: TopEnableGuard<'a>, +} + +impl<'a> GspSubtree<'a> { + /// Returns an initializer that registers the GSP event handler and then enables its subtree at + /// `TOP`. + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the initialized `GspSubtree` or otherwise prevent its + /// [`Drop`] implementation, which runs `free_irq`, from running. + unsafe fn new( + pdev: &'a pci::Device, + tree: &'a Tree<'a>, + falcon: &'a Falcon<'a, GspFalcon>, + cmdq: &'a Cmdq<'a>, + ) -> impl PinInit + 'a { + try_pin_init!(Self { + // SAFETY: this function's caller must not leak the `GspSubtree` that owns this + // registration, so the registration's `Drop` runs. + irq <- unsafe { GspIrq::new(pdev, tree, falcon, cmdq) }, + _top: tree.enable_top_guarded(), + }) + } +} + /// Structure holding the resources required to operate the GPU. #[pin_data] pub(crate) struct Gpu<'gpu> { pub(crate) spec: Spec, + /// GSP event interrupt registration, and the enable of its subtree. + /// + /// Must be kept declared *before* `gsp_resources`, so that the handler is unregistered, and + /// any in-flight run of it has finished, before the command queue that it drains and the + /// falcon that it reads are freed, and before the GSP is unloaded. + #[pin] + _gsp_subtree: GspSubtree<'gpu>, /// Static GPU information as provided by the GSP. pub(crate) gsp_static_info: GetGspStaticInfoReply, /// GPU memory manager owning memory management resources. @@ -354,6 +406,14 @@ pub(crate) struct Gpu<'gpu> { /// Must be kept declared *after* `gsp_resources`, as the latter's `PinnedDrop` implementation /// requires the sysmem flush page to be in place. sysmem_flush: SysmemFlush<'gpu>, + /// Borrow of `tree` that `_gsp_subtree` holds. A field that borrows a sibling field is + /// self-referential, which `pin_init` cannot express, so the borrow is taken by hand. + tree_ref: &'gpu Tree<'gpu>, + /// The GIN CPU interrupt tree and the PCI vectors that deliver it. + /// + /// Must be kept declared *after* `_gsp_subtree`, which holds a borrow of it. + #[pin] + tree: Tree<'gpu>, } #[pinned_drop] @@ -397,6 +457,12 @@ pub(crate) fn new<'a>( dev_info!(dev,"NVIDIA ({})\n", spec); })?, + tree: Tree::new(pdev, bar, spec.chipset, irq::gsp::GSP_SUBTREE.into())?, + + // SAFETY: `tree` is initialized above, is pinned at a stable address, and is dropped + // after every field that uses `tree_ref` (struct field drop order). + tree_ref: unsafe { &*core::ptr::from_ref(tree.as_ref().get_ref()) }, + _: { let dma_mask = hal::gpu_hal(spec.chipset).dma_mask(); @@ -425,12 +491,7 @@ pub(crate) fn new<'a>( bar, - gsp_falcon: Falcon::new( - dev, - spec.chipset, - bar - ) - .inspect(|falcon| falcon.clear_swgen0_intr())?, + gsp_falcon: Falcon::new(dev, spec.chipset, bar)?, sec2_falcon: Falcon::new(dev, spec.chipset, bar)?, @@ -454,6 +515,29 @@ pub(crate) fn new<'a>( })?, }), + _: { + irq::gsp::quiesce(tree_ref, &gsp_resources.gsp_falcon); + }, + + // SAFETY: the GSP falcon and the command queue are fields of `gsp_resources`, which + // is initialized above and pinned, so both references outlive the registration. The + // registration is a field of `Gpu` and is never leaked, so its `Drop` runs, and field + // drop order runs it before either is freed. + _gsp_subtree <- unsafe { + GspSubtree::new( + pdev, + tree_ref, + &*core::ptr::from_ref(&gsp_resources.gsp_falcon), + &*core::ptr::from_ref(&gsp_resources.gsp.cmdq), + ) + }, + + // No interrupt announces the messages that the GSP posted during boot, before the + // SWGEN0 latch was cleared. + _: { + gsp_resources.gsp.cmdq.drain()?; + }, + gsp_static_info: { // Obtain and display basic GPU information. let info = gsp_resources.gsp.get_static_info()?; diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index ae3808de44e2..1b2f346a237f 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -629,6 +629,20 @@ pub(crate) fn await_msg(&self) -> Result { self.inner.lock().await_msg() } + + /// Logs and consumes every message the GSP has already posted, and returns without waiting for + /// more. + /// + /// No caller is waiting for a reply while this holds the queue mutex, so every message is + /// logged as an event. See "Draining the GSP-to-CPU queue" in + /// `Documentation/gpu/nova/core/interrupts.rst`. + /// + /// # Errors + /// + /// `EIO` if a message fails framing or checksum validation. + pub(crate) fn drain(&self) -> Result { + self.inner.lock().drain() + } } /// Inner mutex protected state of [`Cmdq`]. @@ -941,4 +955,30 @@ fn log_event(&self, function: Result, seq: u32) { } } } + + /// Logs and consumes every message the queue holds. + /// + /// # Errors + /// + /// `EIO` if a message fails framing or checksum validation, or if a message's page count + /// overflows a `u32`. + fn drain(&mut self) -> Result { + while !self.gsp_mem.driver_read_area().0.is_empty() { + // A message is available, so this returns without waiting. + let msg = self.wait_for_msg(Delta::ZERO)?; + + let pages = + u32::try_from(msg.header.length().div_ceil(GSP_PAGE_SIZE)).map_err(|_| { + dev_err!(&self.dev, "GSP drain: message length overflow\n"); + EIO + })?; + let function = msg.header.function(); + let seq = msg.header.sequence(); + + self.gsp_mem.advance_cpu_read_ptr(pages); + self.log_event(function, seq); + } + + Ok(()) + } } diff --git a/drivers/gpu/nova-core/irq.rs b/drivers/gpu/nova-core/irq.rs index ff8b00a82442..02f9bd84dd19 100644 --- a/drivers/gpu/nova-core/irq.rs +++ b/drivers/gpu/nova-core/irq.rs @@ -11,6 +11,7 @@ #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] pub(crate) mod doorbell_test; +pub(crate) mod gsp; mod hal; pub(crate) mod interrupt_tree; mod regs; diff --git a/drivers/gpu/nova-core/irq/gsp.rs b/drivers/gpu/nova-core/irq/gsp.rs new file mode 100644 index 000000000000..1abde8e0a405 --- /dev/null +++ b/drivers/gpu/nova-core/irq/gsp.rs @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! The GSP event interrupt. +//! +//! The GSP posts messages to the GSP-to-CPU queue and raises SWGEN0, a software-generated cause +//! of its falcon. A threaded handler services it: the top half clears the tree and falcon state, +//! and the IRQ thread drains the queue. +//! +//! See "The GSP event" in `Documentation/gpu/nova/core/interrupts.rst`. + +use kernel::{ + device, + irq, + pci, + prelude::*, // +}; + +use super::interrupt_tree::{ + GinVector, + LeafEnableGuard, + Subtree, + Tree, // +}; +use crate::{ + falcon::{ + gsp::Gsp as GspFalcon, + Falcon, // + }, + gsp::cmdq::Cmdq, + regs, // +}; + +/// The GSP event vector, which has the same number on every supported GPU. +const GSP_INTR_0_VECTOR: GinVector = GinVector::new::<155>(); + +/// The GSP event's subtree, the only one that nova-core services. +pub(crate) const GSP_SUBTREE: Subtree = GSP_INTR_0_VECTOR.subtree(); + +/// Clears the tree and falcon interrupt state that GSP boot leaves behind, and rearms PCI +/// interrupt delivery. +/// +/// On return, no vector is enabled at its leaf, the serviced subtrees are disabled at `TOP`, and +/// the SWGEN0 latch is clear, so the next message that the GSP posts signals the tree. +pub(crate) fn quiesce(tree: &Tree<'_>, falcon: &Falcon<'_, GspFalcon>) { + tree.reset(); + // The latch is cleared after the tree reset. The other order can leave the latch set with its + // leaf bit cleared. See "Enabling the GSP event" in interrupts.rst. + falcon.clear_swgen0_intr(); +} + +/// Threaded IRQ handler for the GSP event. +pub(crate) struct GspInterrupt<'a> { + falcon: &'a Falcon<'a, GspFalcon>, + cmdq: &'a Cmdq<'a>, + tree: &'a Tree<'a>, + /// For logging. + dev: &'a device::Device, +} + +impl GspInterrupt<'_> { + /// Clears the latch of every host-routed cause in `status` other than SWGEN0, and logs them. + /// + /// Returns the causes still set after the clear. A cause driven from outside the falcon stays + /// set, and only a device reset ends it. + fn clear_faults( + &self, + status: regs::NV_PFALCON_FALCON_IRQSTAT, + ) -> regs::NV_PFALCON_FALCON_IRQSTAT { + let faults = status.with_swgen0(false); + if faults.into_raw() == 0 { + return faults; + } + + dev_err!( + &self.dev, + "unserviceable GSP falcon interrupt, IRQSTAT {:#x}\n", + status.into_raw() + ); + self.falcon.clear_intr(faults); + + self.falcon.read_host_intr().with_swgen0(false) + } +} + +impl irq::ThreadedHandler for GspInterrupt<'_> { + /// Top half, in hard interrupt context. Services the GSP vector only, so another vector + /// pending in the same leaf stays pending. + fn handle(&self) -> irq::ThreadedIrqReturn { + let leaf = self.tree.read_pending(GSP_INTR_0_VECTOR.leaf_index()); + if !leaf.vectors().contains(GSP_INTR_0_VECTOR.leaf_mask()) { + self.tree.rearm_pci_irq(GSP_SUBTREE); + return irq::ThreadedIrqReturn::None; + } + leaf.clear_vectors(GSP_INTR_0_VECTOR.leaf_mask()); + + let status = self.falcon.take_host_intr(); + + let remaining_faults = self.clear_faults(status); + if remaining_faults.into_raw() == 0 { + self.falcon.retrigger_intr(); + } else { + // Disabling the vector loses no notification: the falcon signals nothing further + // while a cause stays set. See "Retriggering a falcon" in interrupts.rst. + self.tree.disable_leaf( + GSP_INTR_0_VECTOR.leaf_index(), + GSP_INTR_0_VECTOR.leaf_mask(), + ); + dev_err!( + &self.dev, + "GSP falcon cause {:#x} needs a device reset, GSP events are no longer serviced\n", + remaining_faults.into_raw() + ); + } + + self.tree.rearm_pci_irq(GSP_SUBTREE); + + if status.swgen0() { + irq::ThreadedIrqReturn::WakeThread + } else { + irq::ThreadedIrqReturn::Handled + } + } + + /// IRQ thread. Drains the GSP-to-CPU queue, which may sleep. + fn handle_threaded(&self) -> irq::IrqReturn { + if let Err(e) = self.cmdq.drain() { + // The failed message stays at the queue head, so every later drain fails the same way. + self.tree.disable_leaf( + GSP_INTR_0_VECTOR.leaf_index(), + GSP_INTR_0_VECTOR.leaf_mask(), + ); + dev_err!( + &self.dev, + "GSP event drain failed ({:?}), the message queue is no longer serviced\n", + e + ); + } + irq::IrqReturn::Handled + } +} + +/// The registered GSP event handler and the enable of its vector. +/// +/// The declaration order is the drop order, and it is required: the vector is disabled before +/// `free_irq` runs. See "Enabling the GSP event" in `Documentation/gpu/nova/core/interrupts.rst`. +#[pin_data] +pub(crate) struct GspIrq<'a> { + _leaf_guard: LeafEnableGuard<'a>, + #[pin] + reg: irq::ThreadedRegistration<'a, GspInterrupt<'a>>, +} + +impl<'a> GspIrq<'a> { + /// Returns an initializer that registers the threaded handler and then enables the GSP + /// vector at its leaf. + /// + /// An event that latched while the vector was disabled is delivered as soon as the GSP + /// subtree is enabled at `TOP`. + /// + /// # Errors + /// + /// `EINVAL` if `tree` does not service the GSP subtree. Otherwise the error from + /// `request_threaded_irq`. + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the initialized `GspIrq` or otherwise prevent its [`Drop`] + /// implementation, which runs `free_irq`, from running. + pub(crate) unsafe fn new( + pdev: &'a pci::Device, + tree: &'a Tree<'a>, + falcon: &'a Falcon<'a, GspFalcon>, + cmdq: &'a Cmdq<'a>, + ) -> impl PinInit + 'a { + let dev = pdev.as_ref(); + + try_pin_init!(Self { + // SAFETY: this function's caller must not leak the `GspIrq` that owns this + // registration, so the registration's `Drop` runs. + reg <- unsafe { + irq::ThreadedRegistration::new( + tree.request_for(GSP_SUBTREE)?, + irq::Flags::TRIGGER_NONE, + c"nova-core", + Ok(GspInterrupt { falcon, cmdq, tree, dev }), + ) + }, + _leaf_guard: tree.enable_leaf_guarded( + GSP_INTR_0_VECTOR.leaf_index(), + GSP_INTR_0_VECTOR.leaf_mask(), + ), + }) + } +} diff --git a/drivers/gpu/nova-core/irq/interrupt_tree.rs b/drivers/gpu/nova-core/irq/interrupt_tree.rs index c1576ea3e61f..90d4d11e8ba3 100644 --- a/drivers/gpu/nova-core/irq/interrupt_tree.rs +++ b/drivers/gpu/nova-core/irq/interrupt_tree.rs @@ -124,10 +124,12 @@ pub(super) const fn all() -> Self { Self(u32::MAX) } + #[cfg_attr(not(CONFIG_NOVA_CORE_SELFTESTS), expect(dead_code))] pub(super) const fn from_raw(raw: u32) -> Self { Self(raw) } + #[cfg_attr(not(CONFIG_NOVA_CORE_SELFTESTS), expect(dead_code))] pub(super) const fn into_raw(self) -> u32 { self.0 } @@ -160,7 +162,7 @@ fn from(vectors: LeafMask) -> Self { /// /// Exactly one bit is set. #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) struct Subtree(u32); +pub(crate) struct Subtree(u32); impl Subtree { /// Returns the subtree at index `idx`. @@ -240,6 +242,7 @@ pub(super) const fn new() -> Self { Self(Bounded::::new::()) } + #[cfg_attr(not(CONFIG_NOVA_CORE_SELFTESTS), expect(dead_code))] pub(super) const fn into_raw(self) -> u32 { self.0.get() } diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 8c0761dedda4..a8f6dc22057b 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -18,7 +18,6 @@ mod fsp; mod gpu; mod gsp; -#[cfg_attr(not(CONFIG_NOVA_CORE_SELFTESTS), expect(dead_code))] mod irq; mod mctp; mod mm;