1 // SPDX-License-Identifier: GPL-2.0 2 3 use kernel::{device, devres::Devres, error::code::*, fmt, pci, prelude::*, sync::Arc}; 4 5 use crate::driver::Bar0; 6 use crate::falcon::{gsp::Gsp, sec2::Sec2, Falcon}; 7 use crate::fb::FbLayout; 8 use crate::fb::SysmemFlush; 9 use crate::firmware::fwsec::{FwsecCommand, FwsecFirmware}; 10 use crate::firmware::{Firmware, FIRMWARE_VERSION}; 11 use crate::gfw; 12 use crate::regs; 13 use crate::util; 14 use crate::vbios::Vbios; 15 16 macro_rules! define_chipset { 17 ({ $($variant:ident = $value:expr),* $(,)* }) => 18 { 19 /// Enum representation of the GPU chipset. 20 #[derive(fmt::Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq)] 21 pub(crate) enum Chipset { 22 $($variant = $value),*, 23 } 24 25 impl Chipset { 26 pub(crate) const ALL: &'static [Chipset] = &[ 27 $( Chipset::$variant, )* 28 ]; 29 30 pub(crate) const NAMES: [&'static str; Self::ALL.len()] = [ 31 $( util::const_bytes_to_str( 32 util::to_lowercase_bytes::<{ stringify!($variant).len() }>( 33 stringify!($variant) 34 ).as_slice() 35 ), )* 36 ]; 37 } 38 39 // TODO[FPRI]: replace with something like derive(FromPrimitive) 40 impl TryFrom<u32> for Chipset { 41 type Error = kernel::error::Error; 42 43 fn try_from(value: u32) -> Result<Self, Self::Error> { 44 match value { 45 $( $value => Ok(Chipset::$variant), )* 46 _ => Err(ENODEV), 47 } 48 } 49 } 50 } 51 } 52 53 define_chipset!({ 54 // Turing 55 TU102 = 0x162, 56 TU104 = 0x164, 57 TU106 = 0x166, 58 TU117 = 0x167, 59 TU116 = 0x168, 60 // Ampere 61 GA100 = 0x170, 62 GA102 = 0x172, 63 GA103 = 0x173, 64 GA104 = 0x174, 65 GA106 = 0x176, 66 GA107 = 0x177, 67 // Ada 68 AD102 = 0x192, 69 AD103 = 0x193, 70 AD104 = 0x194, 71 AD106 = 0x196, 72 AD107 = 0x197, 73 }); 74 75 impl Chipset { 76 pub(crate) fn arch(&self) -> Architecture { 77 match self { 78 Self::TU102 | Self::TU104 | Self::TU106 | Self::TU117 | Self::TU116 => { 79 Architecture::Turing 80 } 81 Self::GA100 | Self::GA102 | Self::GA103 | Self::GA104 | Self::GA106 | Self::GA107 => { 82 Architecture::Ampere 83 } 84 Self::AD102 | Self::AD103 | Self::AD104 | Self::AD106 | Self::AD107 => { 85 Architecture::Ada 86 } 87 } 88 } 89 } 90 91 // TODO 92 // 93 // The resulting strings are used to generate firmware paths, hence the 94 // generated strings have to be stable. 95 // 96 // Hence, replace with something like strum_macros derive(Display). 97 // 98 // For now, redirect to fmt::Debug for convenience. 99 impl fmt::Display for Chipset { 100 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 101 write!(f, "{self:?}") 102 } 103 } 104 105 /// Enum representation of the GPU generation. 106 #[derive(fmt::Debug)] 107 pub(crate) enum Architecture { 108 Turing = 0x16, 109 Ampere = 0x17, 110 Ada = 0x19, 111 } 112 113 impl TryFrom<u8> for Architecture { 114 type Error = Error; 115 116 fn try_from(value: u8) -> Result<Self> { 117 match value { 118 0x16 => Ok(Self::Turing), 119 0x17 => Ok(Self::Ampere), 120 0x19 => Ok(Self::Ada), 121 _ => Err(ENODEV), 122 } 123 } 124 } 125 126 pub(crate) struct Revision { 127 major: u8, 128 minor: u8, 129 } 130 131 impl Revision { 132 fn from_boot0(boot0: regs::NV_PMC_BOOT_0) -> Self { 133 Self { 134 major: boot0.major_revision(), 135 minor: boot0.minor_revision(), 136 } 137 } 138 } 139 140 impl fmt::Display for Revision { 141 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 142 write!(f, "{:x}.{:x}", self.major, self.minor) 143 } 144 } 145 146 /// Structure holding the metadata of the GPU. 147 pub(crate) struct Spec { 148 chipset: Chipset, 149 /// The revision of the chipset. 150 revision: Revision, 151 } 152 153 impl Spec { 154 fn new(bar: &Bar0) -> Result<Spec> { 155 let boot0 = regs::NV_PMC_BOOT_0::read(bar); 156 157 Ok(Self { 158 chipset: boot0.chipset()?, 159 revision: Revision::from_boot0(boot0), 160 }) 161 } 162 } 163 164 /// Structure holding the resources required to operate the GPU. 165 #[pin_data(PinnedDrop)] 166 pub(crate) struct Gpu { 167 spec: Spec, 168 /// MMIO mapping of PCI BAR 0 169 bar: Arc<Devres<Bar0>>, 170 fw: Firmware, 171 /// System memory page required for flushing all pending GPU-side memory writes done through 172 /// PCIE into system memory, via sysmembar (A GPU-initiated HW memory-barrier operation). 173 sysmem_flush: SysmemFlush, 174 } 175 176 #[pinned_drop] 177 impl PinnedDrop for Gpu { 178 fn drop(self: Pin<&mut Self>) { 179 // Unregister the sysmem flush page before we release it. 180 self.bar 181 .try_access_with(|b| self.sysmem_flush.unregister(b)); 182 } 183 } 184 185 impl Gpu { 186 /// Helper function to load and run the FWSEC-FRTS firmware and confirm that it has properly 187 /// created the WPR2 region. 188 /// 189 /// TODO: this needs to be moved into a larger type responsible for booting the whole GSP 190 /// (`GspBooter`?). 191 fn run_fwsec_frts( 192 dev: &device::Device<device::Bound>, 193 falcon: &Falcon<Gsp>, 194 bar: &Bar0, 195 bios: &Vbios, 196 fb_layout: &FbLayout, 197 ) -> Result<()> { 198 // Check that the WPR2 region does not already exists - if it does, we cannot run 199 // FWSEC-FRTS until the GPU is reset. 200 if regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI::read(bar).higher_bound() != 0 { 201 dev_err!( 202 dev, 203 "WPR2 region already exists - GPU needs to be reset to proceed\n" 204 ); 205 return Err(EBUSY); 206 } 207 208 let fwsec_frts = FwsecFirmware::new( 209 dev, 210 falcon, 211 bar, 212 bios, 213 FwsecCommand::Frts { 214 frts_addr: fb_layout.frts.start, 215 frts_size: fb_layout.frts.end - fb_layout.frts.start, 216 }, 217 )?; 218 219 // Run FWSEC-FRTS to create the WPR2 region. 220 fwsec_frts.run(dev, falcon, bar)?; 221 222 // SCRATCH_E contains the error code for FWSEC-FRTS. 223 let frts_status = regs::NV_PBUS_SW_SCRATCH_0E::read(bar).frts_err_code(); 224 if frts_status != 0 { 225 dev_err!( 226 dev, 227 "FWSEC-FRTS returned with error code {:#x}", 228 frts_status 229 ); 230 231 return Err(EIO); 232 } 233 234 // Check that the WPR2 region has been created as we requested. 235 let (wpr2_lo, wpr2_hi) = ( 236 regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO::read(bar).lower_bound(), 237 regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI::read(bar).higher_bound(), 238 ); 239 240 match (wpr2_lo, wpr2_hi) { 241 (_, 0) => { 242 dev_err!(dev, "WPR2 region not created after running FWSEC-FRTS\n"); 243 244 Err(EIO) 245 } 246 (wpr2_lo, _) if wpr2_lo != fb_layout.frts.start => { 247 dev_err!( 248 dev, 249 "WPR2 region created at unexpected address {:#x}; expected {:#x}\n", 250 wpr2_lo, 251 fb_layout.frts.start, 252 ); 253 254 Err(EIO) 255 } 256 (wpr2_lo, wpr2_hi) => { 257 dev_dbg!(dev, "WPR2: {:#x}-{:#x}\n", wpr2_lo, wpr2_hi); 258 dev_dbg!(dev, "GPU instance built\n"); 259 260 Ok(()) 261 } 262 } 263 } 264 265 pub(crate) fn new( 266 pdev: &pci::Device<device::Bound>, 267 devres_bar: Arc<Devres<Bar0>>, 268 ) -> Result<impl PinInit<Self>> { 269 let bar = devres_bar.access(pdev.as_ref())?; 270 let spec = Spec::new(bar)?; 271 let fw = Firmware::new(pdev.as_ref(), spec.chipset, FIRMWARE_VERSION)?; 272 273 dev_info!( 274 pdev.as_ref(), 275 "NVIDIA (Chipset: {}, Architecture: {:?}, Revision: {})\n", 276 spec.chipset, 277 spec.chipset.arch(), 278 spec.revision 279 ); 280 281 // We must wait for GFW_BOOT completion before doing any significant setup on the GPU. 282 gfw::wait_gfw_boot_completion(bar) 283 .inspect_err(|_| dev_err!(pdev.as_ref(), "GFW boot did not complete"))?; 284 285 let sysmem_flush = SysmemFlush::register(pdev.as_ref(), bar, spec.chipset)?; 286 287 let gsp_falcon = Falcon::<Gsp>::new( 288 pdev.as_ref(), 289 spec.chipset, 290 bar, 291 spec.chipset > Chipset::GA100, 292 )?; 293 gsp_falcon.clear_swgen0_intr(bar); 294 295 let _sec2_falcon = Falcon::<Sec2>::new(pdev.as_ref(), spec.chipset, bar, true)?; 296 297 let fb_layout = FbLayout::new(spec.chipset, bar)?; 298 dev_dbg!(pdev.as_ref(), "{:#x?}\n", fb_layout); 299 300 let bios = Vbios::new(pdev, bar)?; 301 302 Self::run_fwsec_frts(pdev.as_ref(), &gsp_falcon, bar, &bios, &fb_layout)?; 303 304 Ok(pin_init!(Self { 305 spec, 306 bar: devres_bar, 307 fw, 308 sysmem_flush, 309 })) 310 } 311 } 312