gpu/nova-core/gpu.rs

// SPDX-License-Identifier: GPL-2.0

use core::ops::Range;

use kernel::{
    device,
    dma::Device,
    fmt,
    gpu::buddy::GpuBuddyParams,
    io::Io,
    num::Bounded,
    pci,
    prelude::*,
    ptr::Alignment,
    sizes::{
        SizeConstants,
        SZ_4K, //
    },
    uapi, //
};

use crate::{
    bounded_enum,
    driver::{
        Bar0,
        Bar1, //
    },
    falcon::{
        gsp::Gsp as GspFalcon,
        sec2::Sec2 as Sec2Falcon,
        Falcon, //
    },
    fb::SysmemFlush,
    fsp::Fsp,
    gsp::{
        self,
        cmdq::Cmdq,
        commands::GetGspStaticInfoReply,
        Gsp,
        GspBootContext, //
    },
    irq::{
        self,
        gsp::GspIrq,
        interrupt_tree::{
            TopEnableGuard,
            Tree, //
        }, //
    },
    mm::{
        bar_user::BarUser,
        pagetable::MmuVersion,
        GpuMm,
        VramAddress, //
    },
    vgpu::VgpuManager, //
};

mod hal;
mod regs;

macro_rules! define_chipset {
    ({ $($variant:ident),* $(,)* }) =>
    {
        ::kernel::macros::paste!(
        /// Enum representation of the GPU chipset.
        #[derive(fmt::Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq)]
        #[repr(u32)]
        #[allow(missing_docs)]
        pub enum Chipset {
            $($variant = uapi::[<drm_nova_chipid_NOVA_DRM_CHIPID_ $variant:upper>]),*,
        }

        impl Chipset {
            /// All chipsets known to the driver.
            pub const ALL: &'static [Chipset] = &[
                $( Chipset::$variant, )*
            ];

            /// Returns the name of this chipset, in lowercase.
            ///
            /// # Examples
            ///
            /// ```
            /// let chipset = Chipset::GA102;
            /// assert_eq!(chipset.name(), "ga102");
            /// ```
            pub(crate) const fn name(&self) -> &'static str {
                match *self {
                $(
                    Chipset::$variant => stringify!([<$variant:lower>]),
                )*
                }
            }
        }

        // TODO[FPRI]: replace with something like derive(FromPrimitive)
        impl TryFrom<u32> for Chipset {
            type Error = kernel::error::Error;

            fn try_from(value: u32) -> Result<Self, Self::Error> {
                match value {
                    $(
                        uapi::[<drm_nova_chipid_NOVA_DRM_CHIPID_ $variant:upper>] => {
                            Ok(Chipset::$variant)
                        }
                    )*
                    _ => Err(ENODEV),
                }
            }
        }
    );
    }
}

define_chipset!({
    // Turing
    TU102,
    TU104,
    TU106,
    TU117,
    TU116,
    // Ampere
    GA100,
    GA102,
    GA103,
    GA104,
    GA106,
    GA107,
    // Hopper
    GH100,
    // Ada
    AD102,
    AD103,
    AD104,
    AD106,
    AD107,
    // Blackwell GB10x
    GB100,
    GB102,
    // Blackwell GB20x
    GB202,
    GB203,
    GB205,
    GB206,
    GB207,
});

impl Chipset {
    /// Returns the [`Architecture`] generation of this chipset.
    pub const fn arch(self) -> Architecture {
        match self {
            Self::TU102 | Self::TU104 | Self::TU106 | Self::TU117 | Self::TU116 => {
                Architecture::Turing
            }
            Self::GA100 | Self::GA102 | Self::GA103 | Self::GA104 | Self::GA106 | Self::GA107 => {
                Architecture::Ampere
            }
            Self::GH100 => Architecture::Hopper,
            Self::AD102 | Self::AD103 | Self::AD104 | Self::AD106 | Self::AD107 => {
                Architecture::Ada
            }
            Self::GB100 | Self::GB102 => Architecture::BlackwellGB10x,
            Self::GB202 | Self::GB203 | Self::GB205 | Self::GB206 | Self::GB207 => {
                Architecture::BlackwellGB20x
            }
        }
    }

    /// Returns the address range of the PCI config mirror space.
    pub(crate) fn pci_config_mirror_range(self) -> Range<u32> {
        hal::gpu_hal(self).pci_config_mirror_range()
    }

    /// Returns the MMU version for this chipset.
    pub(crate) fn mmu_version(self) -> MmuVersion {
        MmuVersion::from(self.arch())
    }
}

impl From<Chipset> for u32 {
    #[inline]
    fn from(value: Chipset) -> Self {
        // CAST: `Chipset` is `repr(u32)` and can thus be cast losslessly.
        value as u32
    }
}

// TODO
//
// The resulting strings are used to generate firmware paths, hence the
// generated strings have to be stable.
//
// Hence, replace with something like strum_macros derive(Display).
//
// For now, redirect to fmt::Debug for convenience.
impl fmt::Display for Chipset {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{self:?}")
    }
}

bounded_enum! {
    /// Enum representation of the GPU generation.
    #[derive(fmt::Debug, Copy, Clone)]
    #[repr(u32)]
    pub enum Architecture with TryFrom<Bounded<u32, 6>> {
        /// Turing (TU1xx).
        Turing = uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_TURING,
        /// Ampere (GA10x).
        Ampere = uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_AMPERE,
        /// Hopper (GH100).
        Hopper = uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_HOPPER,
        /// Ada Lovelace (AD10x).
        Ada = uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_ADA,
        /// Blackwell (GB10x).
        BlackwellGB10x =
            uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_BLACKWELL_GB10X,
        /// Blackwell (GB20x).
        BlackwellGB20x =
            uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_BLACKWELL_GB20X,
    }
}

impl From<Architecture> for u32 {
    #[inline]
    fn from(value: Architecture) -> Self {
        // CAST: `Architecture` is `repr(u32)` and can thus be cast losslessly.
        value as u32
    }
}

#[derive(Clone, Copy)]
pub(crate) struct Revision {
    major: Bounded<u8, 4>,
    minor: Bounded<u8, 4>,
}

impl From<regs::NV_PMC_BOOT_42> for Revision {
    fn from(boot0: regs::NV_PMC_BOOT_42) -> Self {
        Self {
            major: boot0.major_revision().cast(),
            minor: boot0.minor_revision().cast(),
        }
    }
}

impl fmt::Display for Revision {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{:x}.{:x}", self.major, self.minor)
    }
}

/// Structure holding a basic description of the GPU: `Chipset` and `Revision`.
#[derive(Clone, Copy)]
pub struct Spec {
    /// The GPU chipset.
    pub chipset: Chipset,
    revision: Revision,
}

impl Spec {
    pub(crate) fn new(dev: &device::Device, bar: Bar0<'_>) -> Result<Spec> {
        // Some brief notes about boot0 and boot42, in chronological order:
        //
        // NV04 through NV50:
        //
        //    Not supported by Nova. boot0 is necessary and sufficient to identify these GPUs.
        //    boot42 may not even exist on some of these GPUs.
        //
        // Fermi through Volta:
        //
        //     Not supported by Nova. boot0 is still sufficient to identify these GPUs, but boot42
        //     is also guaranteed to be both present and accurate.
        //
        // Turing and later:
        //
        //     Supported by Nova. Identified by first checking boot0 to ensure that the GPU is not
        //     from an earlier (pre-Fermi) era, and then using boot42 to precisely identify the GPU.
        //     Somewhere in the Rubin timeframe, boot0 will no longer have space to add new GPU IDs.

        let boot0 = bar.read(regs::NV_PMC_BOOT_0);

        if boot0.is_older_than_fermi() {
            return Err(ENODEV);
        }

        let boot42 = bar.read(regs::NV_PMC_BOOT_42);
        Spec::try_from(boot42).inspect_err(|_| {
            dev_err!(dev, "Unsupported chipset: {}\n", boot42);
        })
    }
}

impl TryFrom<regs::NV_PMC_BOOT_42> for Spec {
    type Error = Error;

    fn try_from(boot42: regs::NV_PMC_BOOT_42) -> Result<Self> {
        Ok(Self {
            chipset: boot42.chipset()?,
            revision: boot42.into(),
        })
    }
}

impl fmt::Display for Spec {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_fmt(fmt!(
            "Chipset: {}, Architecture: {:?}, Revision: {}",
            self.chipset,
            self.chipset.arch(),
            self.revision
        ))
    }
}

/// Self-contained resources to operate and drop the GSP.
#[pin_data(PinnedDrop)]
struct GspResources<'gpu> {
    /// Device owning the GPU.
    device: &'gpu pci::Device<device::Bound>,
    /// Details about the chipset.
    spec: Spec,
    /// MMIO mapping of PCI BAR 0.
    bar: Bar0<'gpu>,
    /// GSP falcon instance, used for GSP boot up and cleanup.
    gsp_falcon: Falcon<'gpu, GspFalcon>,
    /// SEC2 falcon instance, used for GSP boot up and cleanup.
    sec2_falcon: Falcon<'gpu, Sec2Falcon>,
    /// FSP instance, if on an arch that supports it.
    // TODO: use different resource types for each boot method, and make the relevant Gsp methods
    // generic against them.
    fsp: Option<Fsp<'gpu>>,
    /// vGPU state detected before GSP boot.
    vgpu: VgpuManager,
    /// GSP runtime data.
    #[pin]
    gsp: Gsp<'gpu>,
    /// GSP unload firmware bundle, if any.
    unload_bundle: Option<gsp::UnloadBundle<'gpu>>,
}

/// The GSP event handler's registration and the enable of its subtree at `TOP`.
///
/// The two drop as a unit, the handler first, on the drop of [`Gpu`] and on the error path of
/// its constructor alike, so that the subtree is disabled only after the handler is freed.
#[pin_data]
struct GspSubtree<'a> {
    #[pin]
    irq: GspIrq<'a>,
    /// Must be kept declared *after* `irq`. A handler still in flight enables the subtree again
    /// through its rearm.
    _top: TopEnableGuard<'a>,
}

impl<'a> GspSubtree<'a> {
    /// Returns an initializer that registers the GSP event handler and then enables its subtree at
    /// `TOP`.
    ///
    /// # Safety
    ///
    /// Callers must not `mem::forget()` the initialized `GspSubtree` or otherwise prevent its
    /// [`Drop`] implementation, which runs `free_irq`, from running.
    unsafe fn new(
        pdev: &'a pci::Device<device::Bound>,
        tree: &'a Tree<'a>,
        falcon: &'a Falcon<'a, GspFalcon>,
        cmdq: &'a Cmdq<'a>,
    ) -> impl PinInit<Self, Error> + 'a {
        try_pin_init!(Self {
            // SAFETY: this function's caller must not leak the `GspSubtree` that owns this
            // registration, so the registration's `Drop` runs.
            irq <- unsafe { GspIrq::new(pdev, tree, falcon, cmdq) },
            _top: tree.enable_top_guarded(),
        })
    }
}

/// Structure holding the resources required to operate the GPU.
#[pin_data]
pub(crate) struct Gpu<'gpu> {
    pub(crate) spec: Spec,
    /// GSP event interrupt registration, and the enable of its subtree.
    ///
    /// Must be kept declared *before* `gsp_resources`, so that the handler is unregistered, and
    /// any in-flight run of it has finished, before the command queue that it drains and the
    /// falcon that it reads are freed, and before the GSP is unloaded.
    #[pin]
    _gsp_subtree: GspSubtree<'gpu>,
    /// Static GPU information as provided by the GSP.
    pub(crate) gsp_static_info: GetGspStaticInfoReply,
    /// GPU memory manager owning memory management resources.
    ///
    /// Must be kept declared *before* `gsp_resources`, so that its components are dropped while
    /// the GSP is still operational.
    mm: GpuMm<'gpu>,
    /// BAR1 user interface for CPU access to GPU virtual memory.
    #[pin]
    bar_user: BarUser<'gpu>,
    /// GSP and its resources.
    #[pin]
    gsp_resources: GspResources<'gpu>,
    /// System memory page required for flushing all pending GPU-side memory writes done through
    /// PCIE into system memory, via sysmembar (A GPU-initiated HW memory-barrier operation).
    ///
    /// Must be kept declared *after* `gsp_resources`, as the latter's `PinnedDrop` implementation
    /// requires the sysmem flush page to be in place.
    sysmem_flush: SysmemFlush<'gpu>,
    /// Borrow of `tree` that `_gsp_subtree` holds. A field that borrows a sibling field is
    /// self-referential, which `pin_init` cannot express, so the borrow is taken by hand.
    tree_ref: &'gpu Tree<'gpu>,
    /// The GIN CPU interrupt tree and the PCI vectors that deliver it.
    ///
    /// Must be kept declared *after* `_gsp_subtree`, which holds a borrow of it.
    #[pin]
    tree: Tree<'gpu>,
}

#[pinned_drop]
impl PinnedDrop for GspResources<'_> {
    fn drop(self: Pin<&mut Self>) {
        let this = self.project();
        let device = *this.device;
        let bar = *this.bar;
        let bundle = this.unload_bundle.take();

        let _ = this
            .gsp
            .as_ref()
            .get_ref()
            .unload(
                GspBootContext {
                    pdev: device,
                    bar,
                    chipset: this.spec.chipset,
                    gsp_falcon: &*this.gsp_falcon,
                    sec2_falcon: &*this.sec2_falcon,
                    fsp: this.fsp.as_mut(),
                    vgpu: &*this.vgpu,
                },
                bundle,
            )
            .inspect_err(|e| dev_err!(device, "failed to unload GSP: {:?}\n", e));
    }
}

impl<'gpu> Gpu<'gpu> {
    pub(crate) fn new<'a>(
        pdev: &'gpu pci::Device<device::Core<'a>>,
        bar: Bar0<'gpu>,
        bar1: Bar1<'gpu>,
    ) -> impl PinInit<Self, Error> + use<'gpu, 'a> {
        let dev = pdev.as_ref();

        try_pin_init!(Self {
            spec: Spec::new(dev, bar).inspect(|spec| {
                dev_info!(dev,"NVIDIA ({})\n", spec);
            })?,

            tree: Tree::new(pdev, bar, spec.chipset, irq::gsp::GSP_SUBTREE.into())?,

            // SAFETY: `tree` is initialized above, is pinned at a stable address, and is dropped
            // after every field that uses `tree_ref` (struct field drop order).
            tree_ref: unsafe { &*core::ptr::from_ref(tree.as_ref().get_ref()) },

            _: {
                let dma_mask = hal::gpu_hal(spec.chipset).dma_mask();

                // SAFETY: `Gpu` owns all DMA allocations for this device, and we are
                // still constructing it, so no concurrent DMA allocations can exist.
                unsafe { pdev.dma_set_mask_and_coherent(dma_mask)? };

                // Nova walks SG segments to build page tables, so their length is
                // irrelevant to the device.
                //
                // SAFETY: `Gpu` owns all DMA allocations for this device, and we are
                // still constructing it, so no concurrent DMA allocations can exist.
                unsafe { pdev.dma_set_max_seg_size(u32::MAX) };

                // We must wait for GFW_BOOT completion before doing any significant setup on the GPU.
                Self::wait_gfw_boot_completion(dev, bar, spec.chipset)?;
            },

            // Initialize this early because `gsp_resources` depends on it.
            sysmem_flush: SysmemFlush::register(dev, bar, spec.chipset)?,

            gsp_resources <- try_pin_init!(GspResources {
                device: pdev,

                spec: *spec,

                bar,

                gsp_falcon: Falcon::new(dev, spec.chipset, bar)?,

                sec2_falcon: Falcon::new(dev, spec.chipset, bar)?,

                fsp: Fsp::try_new(dev, bar, spec.chipset)?,

                vgpu: VgpuManager::new(pdev, spec.chipset, fsp.as_mut()),

                gsp <- Gsp::new(pdev, bar),

                // This member must be initialized last, so the `UnloadBundle` can never be dropped
                // from outside of the constructed `GspResources`, ensuring that the unload sequence
                // is properly run in case of failure.
                unload_bundle: gsp.boot(GspBootContext {
                    pdev,
                    bar,
                    chipset: spec.chipset,
                    gsp_falcon,
                    sec2_falcon,
                    fsp: fsp.as_mut(),
                    vgpu,
                })?,
            }),

            _: {
                irq::gsp::quiesce(tree_ref, &gsp_resources.gsp_falcon);
            },

            // SAFETY: the GSP falcon and the command queue are fields of `gsp_resources`, which
            // is initialized above and pinned, so both references outlive the registration. The
            // registration is a field of `Gpu` and is never leaked, so its `Drop` runs, and field
            // drop order runs it before either is freed.
            _gsp_subtree <- unsafe {
                GspSubtree::new(
                    pdev,
                    tree_ref,
                    &*core::ptr::from_ref(&gsp_resources.gsp_falcon),
                    &*core::ptr::from_ref(&gsp_resources.gsp.cmdq),
                )
            },

            // No interrupt announces the messages that the GSP posted during boot, before the
            // SWGEN0 latch was cleared.
            _: {
                gsp_resources.gsp.cmdq.drain()?;
            },

            gsp_static_info: {
                // Obtain and display basic GPU information.
                let info = gsp_resources.gsp.get_static_info()?;
                match info.gpu_name() {
                    Ok(name) => dev_info!(dev, "GPU name: {}\n", name),
                    Err(e) => dev_warn!(dev, "GPU name unavailable: {:?}\n", e),
                }

                if !info.usable_fb_regions.is_empty() {
                    dev_dbg!(dev, "Usable FB regions:\n");
                    for region in &info.usable_fb_regions {
                        dev_dbg!(dev, "  - {:#x?}\n", region);
                    }

                    dev_dbg!(
                        dev,
                        "Total usable VRAM: {} MiB\n",
                        info.vram_size() / u64::SZ_1M
                    );
                }

                info
            },

            // Create GPU memory manager owning memory management resources.
            mm: {
                let usable_vram = gsp_static_info.usable_fb_regions.first().ok_or(ENODEV)?;
                let buddy_params = GpuBuddyParams {
                    base_offset: usable_vram.start,
                    size: usable_vram.end - usable_vram.start,
                    chunk_size: Alignment::new::<SZ_4K>(),
                };

                GpuMm::new(
                    bar,
                    gsp_resources.spec.chipset,
                    buddy_params,
                    VramAddress::from_raw(gsp_static_info.total_fb_end),
                )?
            },

            // Create BAR1 user interface for CPU access to GPU virtual memory.
            bar_user <- {
                let pdb_addr = VramAddress::from_raw(gsp_static_info.bar1_pde_base);
                let bar1_idx = crate::driver::bar1_resource_index(pdev)?;
                let bar1_size = pdev.resource_len(bar1_idx)?;

                BarUser::new(
                    pdb_addr,
                    gsp_resources.spec.chipset,
                    bar1_size,
                    bar1,
                )
            },
        })
    }

    /// Waits for GFW, the GPU's boot firmware, to report completion.
    ///
    /// The driver must not program the GPU before then.
    ///
    /// # Errors
    ///
    /// `ETIMEDOUT` if GFW does not report completion in time.
    pub(crate) fn wait_gfw_boot_completion(
        dev: &device::Device<device::Bound>,
        bar: Bar0<'_>,
        chipset: Chipset,
    ) -> Result {
        hal::gpu_hal(chipset)
            .wait_gfw_boot_completion(bar)
            .inspect_err(|_| dev_err!(dev, "GFW boot did not complete\n"))
    }

    /// Runs self-tests on the constructed [`Gpu`], logging failures without failing probe.
    #[cfg(CONFIG_NOVA_CORE_SELFTESTS)]
    pub(crate) fn run_selftests(self: Pin<&mut Self>, pdev: &pci::Device<device::Bound>) {
        let this = self.project();
        let dev = pdev.as_ref();
        let regions = &this.gsp_static_info.usable_fb_regions;

        if let Err(err) = crate::mm::selftest::run(
            dev,
            this.mm,
            regions,
            &this.bar_user,
            this.gsp_static_info.bar1_pde_base,
            this.spec.chipset,
        ) {
            dev_err!(dev, "self-tests failed: {:?}\n", err);
        }
    }
}

/// Reads the boot0 register and returns its raw value.
pub(crate) fn boot_0_raw(bar: Bar0<'_>) -> u32 {
    bar.read(regs::NV_PMC_BOOT_0).into_raw()
}