1 // SPDX-License-Identifier: GPL-2.0 2 3 use kernel::{device, devres::Devres, error::code::*, pci, prelude::*, sync::Arc}; 4 5 use crate::driver::Bar0; 6 use crate::falcon::{gsp::Gsp, sec2::Sec2, Falcon}; 7 use crate::fb::FbLayout; 8 use crate::fb::SysmemFlush; 9 use crate::firmware::fwsec::{FwsecCommand, FwsecFirmware}; 10 use crate::firmware::{Firmware, FIRMWARE_VERSION}; 11 use crate::gfw; 12 use crate::regs; 13 use crate::util; 14 use crate::vbios::Vbios; 15 use core::fmt; 16 17 macro_rules! define_chipset { 18 ({ $($variant:ident = $value:expr),* $(,)* }) => 19 { 20 /// Enum representation of the GPU chipset. 21 #[derive(fmt::Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq)] 22 pub(crate) enum Chipset { 23 $($variant = $value),*, 24 } 25 26 impl Chipset { 27 pub(crate) const ALL: &'static [Chipset] = &[ 28 $( Chipset::$variant, )* 29 ]; 30 31 pub(crate) const NAMES: [&'static str; Self::ALL.len()] = [ 32 $( util::const_bytes_to_str( 33 util::to_lowercase_bytes::<{ stringify!($variant).len() }>( 34 stringify!($variant) 35 ).as_slice() 36 ), )* 37 ]; 38 } 39 40 // TODO[FPRI]: replace with something like derive(FromPrimitive) 41 impl TryFrom<u32> for Chipset { 42 type Error = kernel::error::Error; 43 44 fn try_from(value: u32) -> Result<Self, Self::Error> { 45 match value { 46 $( $value => Ok(Chipset::$variant), )* 47 _ => Err(ENODEV), 48 } 49 } 50 } 51 } 52 } 53 54 define_chipset!({ 55 // Turing 56 TU102 = 0x162, 57 TU104 = 0x164, 58 TU106 = 0x166, 59 TU117 = 0x167, 60 TU116 = 0x168, 61 // Ampere 62 GA100 = 0x170, 63 GA102 = 0x172, 64 GA103 = 0x173, 65 GA104 = 0x174, 66 GA106 = 0x176, 67 GA107 = 0x177, 68 // Ada 69 AD102 = 0x192, 70 AD103 = 0x193, 71 AD104 = 0x194, 72 AD106 = 0x196, 73 AD107 = 0x197, 74 }); 75 76 impl Chipset { 77 pub(crate) fn arch(&self) -> Architecture { 78 match self { 79 Self::TU102 | Self::TU104 | Self::TU106 | Self::TU117 | Self::TU116 => { 80 Architecture::Turing 81 } 82 Self::GA100 | Self::GA102 | Self::GA103 | Self::GA104 | Self::GA106 | Self::GA107 => { 83 Architecture::Ampere 84 } 85 Self::AD102 | Self::AD103 | Self::AD104 | Self::AD106 | Self::AD107 => { 86 Architecture::Ada 87 } 88 } 89 } 90 } 91 92 // TODO 93 // 94 // The resulting strings are used to generate firmware paths, hence the 95 // generated strings have to be stable. 96 // 97 // Hence, replace with something like strum_macros derive(Display). 98 // 99 // For now, redirect to fmt::Debug for convenience. 100 impl fmt::Display for Chipset { 101 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 102 write!(f, "{self:?}") 103 } 104 } 105 106 /// Enum representation of the GPU generation. 107 #[derive(fmt::Debug)] 108 pub(crate) enum Architecture { 109 Turing = 0x16, 110 Ampere = 0x17, 111 Ada = 0x19, 112 } 113 114 impl TryFrom<u8> for Architecture { 115 type Error = Error; 116 117 fn try_from(value: u8) -> Result<Self> { 118 match value { 119 0x16 => Ok(Self::Turing), 120 0x17 => Ok(Self::Ampere), 121 0x19 => Ok(Self::Ada), 122 _ => Err(ENODEV), 123 } 124 } 125 } 126 127 pub(crate) struct Revision { 128 major: u8, 129 minor: u8, 130 } 131 132 impl Revision { 133 fn from_boot0(boot0: regs::NV_PMC_BOOT_0) -> Self { 134 Self { 135 major: boot0.major_revision(), 136 minor: boot0.minor_revision(), 137 } 138 } 139 } 140 141 impl fmt::Display for Revision { 142 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 143 write!(f, "{:x}.{:x}", self.major, self.minor) 144 } 145 } 146 147 /// Structure holding the metadata of the GPU. 148 pub(crate) struct Spec { 149 chipset: Chipset, 150 /// The revision of the chipset. 151 revision: Revision, 152 } 153 154 impl Spec { 155 fn new(bar: &Bar0) -> Result<Spec> { 156 let boot0 = regs::NV_PMC_BOOT_0::read(bar); 157 158 Ok(Self { 159 chipset: boot0.chipset()?, 160 revision: Revision::from_boot0(boot0), 161 }) 162 } 163 } 164 165 /// Structure holding the resources required to operate the GPU. 166 #[pin_data] 167 pub(crate) struct Gpu { 168 spec: Spec, 169 /// MMIO mapping of PCI BAR 0 170 bar: Arc<Devres<Bar0>>, 171 fw: Firmware, 172 /// System memory page required for flushing all pending GPU-side memory writes done through 173 /// PCIE into system memory, via sysmembar (A GPU-initiated HW memory-barrier operation). 174 sysmem_flush: SysmemFlush, 175 } 176 177 impl Gpu { 178 /// Helper function to load and run the FWSEC-FRTS firmware and confirm that it has properly 179 /// created the WPR2 region. 180 /// 181 /// TODO: this needs to be moved into a larger type responsible for booting the whole GSP 182 /// (`GspBooter`?). 183 fn run_fwsec_frts( 184 dev: &device::Device<device::Bound>, 185 falcon: &Falcon<Gsp>, 186 bar: &Bar0, 187 bios: &Vbios, 188 fb_layout: &FbLayout, 189 ) -> Result<()> { 190 // Check that the WPR2 region does not already exists - if it does, we cannot run 191 // FWSEC-FRTS until the GPU is reset. 192 if regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI::read(bar).higher_bound() != 0 { 193 dev_err!( 194 dev, 195 "WPR2 region already exists - GPU needs to be reset to proceed\n" 196 ); 197 return Err(EBUSY); 198 } 199 200 let fwsec_frts = FwsecFirmware::new( 201 dev, 202 falcon, 203 bar, 204 bios, 205 FwsecCommand::Frts { 206 frts_addr: fb_layout.frts.start, 207 frts_size: fb_layout.frts.end - fb_layout.frts.start, 208 }, 209 )?; 210 211 // Run FWSEC-FRTS to create the WPR2 region. 212 fwsec_frts.run(dev, falcon, bar)?; 213 214 // SCRATCH_E contains the error code for FWSEC-FRTS. 215 let frts_status = regs::NV_PBUS_SW_SCRATCH_0E_FRTS_ERR::read(bar).frts_err_code(); 216 if frts_status != 0 { 217 dev_err!( 218 dev, 219 "FWSEC-FRTS returned with error code {:#x}", 220 frts_status 221 ); 222 223 return Err(EIO); 224 } 225 226 // Check that the WPR2 region has been created as we requested. 227 let (wpr2_lo, wpr2_hi) = ( 228 regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO::read(bar).lower_bound(), 229 regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI::read(bar).higher_bound(), 230 ); 231 232 match (wpr2_lo, wpr2_hi) { 233 (_, 0) => { 234 dev_err!(dev, "WPR2 region not created after running FWSEC-FRTS\n"); 235 236 Err(EIO) 237 } 238 (wpr2_lo, _) if wpr2_lo != fb_layout.frts.start => { 239 dev_err!( 240 dev, 241 "WPR2 region created at unexpected address {:#x}; expected {:#x}\n", 242 wpr2_lo, 243 fb_layout.frts.start, 244 ); 245 246 Err(EIO) 247 } 248 (wpr2_lo, wpr2_hi) => { 249 dev_dbg!(dev, "WPR2: {:#x}-{:#x}\n", wpr2_lo, wpr2_hi); 250 dev_dbg!(dev, "GPU instance built\n"); 251 252 Ok(()) 253 } 254 } 255 } 256 257 pub(crate) fn new( 258 pdev: &pci::Device<device::Bound>, 259 devres_bar: Arc<Devres<Bar0>>, 260 ) -> Result<impl PinInit<Self>> { 261 let bar = devres_bar.access(pdev.as_ref())?; 262 let spec = Spec::new(bar)?; 263 let fw = Firmware::new(pdev.as_ref(), spec.chipset, FIRMWARE_VERSION)?; 264 265 dev_info!( 266 pdev.as_ref(), 267 "NVIDIA (Chipset: {}, Architecture: {:?}, Revision: {})\n", 268 spec.chipset, 269 spec.chipset.arch(), 270 spec.revision 271 ); 272 273 // We must wait for GFW_BOOT completion before doing any significant setup on the GPU. 274 gfw::wait_gfw_boot_completion(bar) 275 .inspect_err(|_| dev_err!(pdev.as_ref(), "GFW boot did not complete"))?; 276 277 let sysmem_flush = SysmemFlush::register(pdev.as_ref(), bar, spec.chipset)?; 278 279 let gsp_falcon = Falcon::<Gsp>::new( 280 pdev.as_ref(), 281 spec.chipset, 282 bar, 283 spec.chipset > Chipset::GA100, 284 )?; 285 gsp_falcon.clear_swgen0_intr(bar); 286 287 let _sec2_falcon = Falcon::<Sec2>::new(pdev.as_ref(), spec.chipset, bar, true)?; 288 289 let fb_layout = FbLayout::new(spec.chipset, bar)?; 290 dev_dbg!(pdev.as_ref(), "{:#x?}\n", fb_layout); 291 292 let bios = Vbios::new(pdev.as_ref(), bar)?; 293 294 Self::run_fwsec_frts(pdev.as_ref(), &gsp_falcon, bar, &bios, &fb_layout)?; 295 296 Ok(pin_init!(Self { 297 spec, 298 bar: devres_bar, 299 fw, 300 sysmem_flush, 301 })) 302 } 303 304 /// Called when the corresponding [`Device`](device::Device) is unbound. 305 /// 306 /// Note: This method must only be called from `Driver::unbind`. 307 pub(crate) fn unbind(&self, dev: &device::Device<device::Core>) { 308 kernel::warn_on!(self 309 .bar 310 .access(dev) 311 .inspect(|bar| self.sysmem_flush.unregister(bar)) 312 .is_err()); 313 } 314 } 315