1 // SPDX-License-Identifier: GPL-2.0 2 3 use kernel::{ 4 device, 5 dma::{ 6 Coherent, 7 CoherentBox, 8 DataDirection, 9 DmaAddress, // 10 }, 11 prelude::*, 12 scatterlist::{ 13 Owned, 14 SGTable, // 15 }, 16 }; 17 18 use crate::{ 19 firmware::riscv::RiscvFirmware, 20 gpu::{ 21 Architecture, 22 Chipset, // 23 }, 24 gsp::GSP_PAGE_SIZE, 25 num::FromSafeCast, 26 }; 27 28 /// Ad-hoc and temporary module to extract sections from ELF images. 29 /// 30 /// Some firmware images are currently packaged as ELF files, where sections names are used as keys 31 /// to specific and related bits of data. Future firmware versions are scheduled to move away from 32 /// that scheme before nova-core becomes stable, which means this module will eventually be 33 /// removed. 34 mod elf { 35 use kernel::{ 36 bindings, 37 prelude::*, 38 transmute::FromBytes, // 39 }; 40 41 /// Newtype to provide a [`FromBytes`] implementation. 42 #[repr(transparent)] 43 struct Elf64Hdr(bindings::elf64_hdr); 44 // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. 45 unsafe impl FromBytes for Elf64Hdr {} 46 47 #[repr(transparent)] 48 struct Elf64SHdr(bindings::elf64_shdr); 49 // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. 50 unsafe impl FromBytes for Elf64SHdr {} 51 52 /// Tries to extract section with name `name` from the ELF64 image `elf`, and returns it. 53 pub(super) fn elf64_section<'a, 'b>(elf: &'a [u8], name: &'b str) -> Option<&'a [u8]> { 54 let hdr = &elf 55 .get(0..size_of::<bindings::elf64_hdr>()) 56 .and_then(Elf64Hdr::from_bytes)? 57 .0; 58 59 // Get all the section headers. 60 let mut shdr = { 61 let shdr_num = usize::from(hdr.e_shnum); 62 let shdr_start = usize::try_from(hdr.e_shoff).ok()?; 63 let shdr_end = shdr_num 64 .checked_mul(size_of::<Elf64SHdr>()) 65 .and_then(|v| v.checked_add(shdr_start))?; 66 67 elf.get(shdr_start..shdr_end) 68 .map(|slice| slice.chunks_exact(size_of::<Elf64SHdr>()))? 69 }; 70 71 // Get the strings table. 72 let strhdr = shdr 73 .clone() 74 .nth(usize::from(hdr.e_shstrndx)) 75 .and_then(Elf64SHdr::from_bytes)?; 76 77 // Find the section which name matches `name` and return it. 78 shdr.find(|&sh| { 79 let Some(hdr) = Elf64SHdr::from_bytes(sh) else { 80 return false; 81 }; 82 83 let Some(name_idx) = strhdr 84 .0 85 .sh_offset 86 .checked_add(u64::from(hdr.0.sh_name)) 87 .and_then(|idx| usize::try_from(idx).ok()) 88 else { 89 return false; 90 }; 91 92 // Get the start of the name. 93 elf.get(name_idx..) 94 .and_then(|nstr| CStr::from_bytes_until_nul(nstr).ok()) 95 // Convert into str. 96 .and_then(|c_str| c_str.to_str().ok()) 97 // Check that the name matches. 98 .map(|str| str == name) 99 .unwrap_or(false) 100 }) 101 // Return the slice containing the section. 102 .and_then(|sh| { 103 let hdr = Elf64SHdr::from_bytes(sh)?; 104 let start = usize::try_from(hdr.0.sh_offset).ok()?; 105 let end = usize::try_from(hdr.0.sh_size) 106 .ok() 107 .and_then(|sh_size| start.checked_add(sh_size))?; 108 109 elf.get(start..end) 110 }) 111 } 112 } 113 114 /// GSP firmware with 3-level radix page tables for the GSP bootloader. 115 /// 116 /// The bootloader expects firmware to be mapped starting at address 0 in GSP's virtual address 117 /// space: 118 /// 119 /// ```text 120 /// Level 0: 1 page, 1 entry -> points to first level 1 page 121 /// Level 1: Multiple pages/entries -> each entry points to a level 2 page 122 /// Level 2: Multiple pages/entries -> each entry points to a firmware page 123 /// ``` 124 /// 125 /// Each page is 4KB, each entry is 8 bytes (64-bit DMA address). 126 /// Also known as "Radix3" firmware. 127 #[pin_data] 128 pub(crate) struct GspFirmware { 129 /// The GSP firmware inside a [`VVec`], device-mapped via a SG table. 130 #[pin] 131 fw: SGTable<Owned<VVec<u8>>>, 132 /// Level 2 page table whose entries contain DMA addresses of firmware pages. 133 #[pin] 134 level2: SGTable<Owned<VVec<u8>>>, 135 /// Level 1 page table whose entries contain DMA addresses of level 2 pages. 136 #[pin] 137 level1: SGTable<Owned<VVec<u8>>>, 138 /// Level 0 page table (single 4KB page) with one entry: DMA address of first level 1 page. 139 level0: Coherent<[u64]>, 140 /// Size in bytes of the firmware contained in [`Self::fw`]. 141 pub(crate) size: usize, 142 /// Device-mapped GSP signatures matching the GPU's [`Chipset`]. 143 pub(crate) signatures: Coherent<[u8]>, 144 /// GSP bootloader, verifies the GSP firmware before loading and running it. 145 pub(crate) bootloader: RiscvFirmware, 146 } 147 148 impl GspFirmware { 149 /// Loads the GSP firmware binaries, map them into `dev`'s address-space, and creates the page 150 /// tables expected by the GSP bootloader to load it. 151 pub(crate) fn new<'a>( 152 dev: &'a device::Device<device::Bound>, 153 chipset: Chipset, 154 ver: &'a str, 155 ) -> impl PinInit<Self, Error> + 'a { 156 pin_init::pin_init_scope(move || { 157 let firmware = super::request_firmware(dev, chipset, "gsp", ver)?; 158 159 let fw_section = elf::elf64_section(firmware.data(), ".fwimage").ok_or(EINVAL)?; 160 161 let size = fw_section.len(); 162 163 // Move the firmware into a vmalloc'd vector and map it into the device address 164 // space. 165 let fw_vvec = VVec::with_capacity(fw_section.len(), GFP_KERNEL) 166 .and_then(|mut v| { 167 v.extend_from_slice(fw_section, GFP_KERNEL)?; 168 Ok(v) 169 }) 170 .map_err(|_| ENOMEM)?; 171 172 Ok(try_pin_init!(Self { 173 fw <- SGTable::new(dev, fw_vvec, DataDirection::ToDevice, GFP_KERNEL), 174 level2 <- { 175 // Allocate the level 2 page table, map the firmware onto it, and map it into 176 // the device address space. 177 VVec::<u8>::with_capacity( 178 fw.iter().count() * core::mem::size_of::<u64>(), 179 GFP_KERNEL, 180 ) 181 .map_err(|_| ENOMEM) 182 .and_then(|level2| map_into_lvl(&fw, level2)) 183 .map(|level2| SGTable::new(dev, level2, DataDirection::ToDevice, GFP_KERNEL))? 184 }, 185 level1 <- { 186 // Allocate the level 1 page table, map the level 2 page table onto it, and map 187 // it into the device address space. 188 VVec::<u8>::with_capacity( 189 level2.iter().count() * core::mem::size_of::<u64>(), 190 GFP_KERNEL, 191 ) 192 .map_err(|_| ENOMEM) 193 .and_then(|level1| map_into_lvl(&level2, level1)) 194 .map(|level1| SGTable::new(dev, level1, DataDirection::ToDevice, GFP_KERNEL))? 195 }, 196 level0: { 197 // Allocate the level 0 page table as a device-visible DMA object, and map the 198 // level 1 page table onto it. 199 200 // Fill level 1 page entry. 201 let level1_entry = level1.iter().next().ok_or(EINVAL)?; 202 let level1_entry_addr = level1_entry.dma_address(); 203 204 // Create level 0 page table data and fill its first entry with the level 1 205 // table. 206 let mut level0 = CoherentBox::<[u64]>::zeroed_slice( 207 dev, 208 GSP_PAGE_SIZE / size_of::<u64>(), 209 GFP_KERNEL 210 )?; 211 level0[0] = level1_entry_addr.to_le(); 212 213 level0.into() 214 }, 215 size, 216 signatures: { 217 let sigs_section = match chipset.arch() { 218 Architecture::Turing 219 if matches!(chipset, Chipset::TU116 | Chipset::TU117) => 220 { 221 ".fwsignature_tu11x" 222 } 223 Architecture::Turing => ".fwsignature_tu10x", 224 // GA100 uses the same firmware as Turing 225 Architecture::Ampere if chipset == Chipset::GA100 => ".fwsignature_tu10x", 226 Architecture::Ampere => ".fwsignature_ga10x", 227 Architecture::Ada => ".fwsignature_ad10x", 228 }; 229 230 elf::elf64_section(firmware.data(), sigs_section) 231 .ok_or(EINVAL) 232 .and_then(|data| Coherent::from_slice(dev, data, GFP_KERNEL))? 233 }, 234 bootloader: { 235 let bl = super::request_firmware(dev, chipset, "bootloader", ver)?; 236 237 RiscvFirmware::new(dev, &bl)? 238 }, 239 })) 240 }) 241 } 242 243 /// Returns the DMA handle of the radix3 level 0 page table. 244 pub(crate) fn radix3_dma_handle(&self) -> DmaAddress { 245 self.level0.dma_handle() 246 } 247 } 248 249 /// Build a page table from a scatter-gather list. 250 /// 251 /// Takes each DMA-mapped region from `sg_table` and writes page table entries 252 /// for all 4KB pages within that region. For example, a 16KB SG entry becomes 253 /// 4 consecutive page table entries. 254 fn map_into_lvl(sg_table: &SGTable<Owned<VVec<u8>>>, mut dst: VVec<u8>) -> Result<VVec<u8>> { 255 for sg_entry in sg_table.iter() { 256 // Number of pages we need to map. 257 let num_pages = usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE); 258 259 for i in 0..num_pages { 260 let entry = sg_entry.dma_address() 261 + (u64::from_safe_cast(i) * u64::from_safe_cast(GSP_PAGE_SIZE)); 262 dst.extend_from_slice(&entry.to_le_bytes(), GFP_KERNEL)?; 263 } 264 } 265 266 Ok(dst) 267 } 268