1 // SPDX-License-Identifier: GPL-2.0 2 3 use core::mem::size_of_val; 4 5 use kernel::{ 6 device, 7 dma::{ 8 DataDirection, 9 DmaAddress, // 10 }, 11 kvec, 12 prelude::*, 13 scatterlist::{ 14 Owned, 15 SGTable, // 16 }, 17 }; 18 19 use crate::{ 20 dma::DmaObject, 21 firmware::riscv::RiscvFirmware, 22 gpu::{ 23 Architecture, 24 Chipset, // 25 }, 26 gsp::GSP_PAGE_SIZE, 27 num::FromSafeCast, 28 }; 29 30 /// Ad-hoc and temporary module to extract sections from ELF images. 31 /// 32 /// Some firmware images are currently packaged as ELF files, where sections names are used as keys 33 /// to specific and related bits of data. Future firmware versions are scheduled to move away from 34 /// that scheme before nova-core becomes stable, which means this module will eventually be 35 /// removed. 36 mod elf { 37 use core::mem::size_of; 38 39 use kernel::bindings; 40 use kernel::str::CStr; 41 use kernel::transmute::FromBytes; 42 43 /// Newtype to provide a [`FromBytes`] implementation. 44 #[repr(transparent)] 45 struct Elf64Hdr(bindings::elf64_hdr); 46 // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. 47 unsafe impl FromBytes for Elf64Hdr {} 48 49 #[repr(transparent)] 50 struct Elf64SHdr(bindings::elf64_shdr); 51 // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. 52 unsafe impl FromBytes for Elf64SHdr {} 53 54 /// Tries to extract section with name `name` from the ELF64 image `elf`, and returns it. 55 pub(super) fn elf64_section<'a, 'b>(elf: &'a [u8], name: &'b str) -> Option<&'a [u8]> { 56 let hdr = &elf 57 .get(0..size_of::<bindings::elf64_hdr>()) 58 .and_then(Elf64Hdr::from_bytes)? 59 .0; 60 61 // Get all the section headers. 62 let mut shdr = { 63 let shdr_num = usize::from(hdr.e_shnum); 64 let shdr_start = usize::try_from(hdr.e_shoff).ok()?; 65 let shdr_end = shdr_num 66 .checked_mul(size_of::<Elf64SHdr>()) 67 .and_then(|v| v.checked_add(shdr_start))?; 68 69 elf.get(shdr_start..shdr_end) 70 .map(|slice| slice.chunks_exact(size_of::<Elf64SHdr>()))? 71 }; 72 73 // Get the strings table. 74 let strhdr = shdr 75 .clone() 76 .nth(usize::from(hdr.e_shstrndx)) 77 .and_then(Elf64SHdr::from_bytes)?; 78 79 // Find the section which name matches `name` and return it. 80 shdr.find(|&sh| { 81 let Some(hdr) = Elf64SHdr::from_bytes(sh) else { 82 return false; 83 }; 84 85 let Some(name_idx) = strhdr 86 .0 87 .sh_offset 88 .checked_add(u64::from(hdr.0.sh_name)) 89 .and_then(|idx| usize::try_from(idx).ok()) 90 else { 91 return false; 92 }; 93 94 // Get the start of the name. 95 elf.get(name_idx..) 96 // Stop at the first `0`. 97 .and_then(|nstr| nstr.get(0..=nstr.iter().position(|b| *b == 0)?)) 98 // Convert into CStr. This should never fail because of the line above. 99 .and_then(|nstr| CStr::from_bytes_with_nul(nstr).ok()) 100 // Convert into str. 101 .and_then(|c_str| c_str.to_str().ok()) 102 // Check that the name matches. 103 .map(|str| str == name) 104 .unwrap_or(false) 105 }) 106 // Return the slice containing the section. 107 .and_then(|sh| { 108 let hdr = Elf64SHdr::from_bytes(sh)?; 109 let start = usize::try_from(hdr.0.sh_offset).ok()?; 110 let end = usize::try_from(hdr.0.sh_size) 111 .ok() 112 .and_then(|sh_size| start.checked_add(sh_size))?; 113 114 elf.get(start..end) 115 }) 116 } 117 } 118 119 /// GSP firmware with 3-level radix page tables for the GSP bootloader. 120 /// 121 /// The bootloader expects firmware to be mapped starting at address 0 in GSP's virtual address 122 /// space: 123 /// 124 /// ```text 125 /// Level 0: 1 page, 1 entry -> points to first level 1 page 126 /// Level 1: Multiple pages/entries -> each entry points to a level 2 page 127 /// Level 2: Multiple pages/entries -> each entry points to a firmware page 128 /// ``` 129 /// 130 /// Each page is 4KB, each entry is 8 bytes (64-bit DMA address). 131 /// Also known as "Radix3" firmware. 132 #[pin_data] 133 pub(crate) struct GspFirmware { 134 /// The GSP firmware inside a [`VVec`], device-mapped via a SG table. 135 #[pin] 136 fw: SGTable<Owned<VVec<u8>>>, 137 /// Level 2 page table whose entries contain DMA addresses of firmware pages. 138 #[pin] 139 level2: SGTable<Owned<VVec<u8>>>, 140 /// Level 1 page table whose entries contain DMA addresses of level 2 pages. 141 #[pin] 142 level1: SGTable<Owned<VVec<u8>>>, 143 /// Level 0 page table (single 4KB page) with one entry: DMA address of first level 1 page. 144 level0: DmaObject, 145 /// Size in bytes of the firmware contained in [`Self::fw`]. 146 pub(crate) size: usize, 147 /// Device-mapped GSP signatures matching the GPU's [`Chipset`]. 148 pub(crate) signatures: DmaObject, 149 /// GSP bootloader, verifies the GSP firmware before loading and running it. 150 pub(crate) bootloader: RiscvFirmware, 151 } 152 153 impl GspFirmware { 154 /// Loads the GSP firmware binaries, map them into `dev`'s address-space, and creates the page 155 /// tables expected by the GSP bootloader to load it. 156 pub(crate) fn new<'a>( 157 dev: &'a device::Device<device::Bound>, 158 chipset: Chipset, 159 ver: &'a str, 160 ) -> impl PinInit<Self, Error> + 'a { 161 pin_init::pin_init_scope(move || { 162 let fw = super::request_firmware(dev, chipset, "gsp", ver)?; 163 164 let fw_section = elf::elf64_section(fw.data(), ".fwimage").ok_or(EINVAL)?; 165 166 let sigs_section = match chipset.arch() { 167 Architecture::Ampere => ".fwsignature_ga10x", 168 Architecture::Ada => ".fwsignature_ad10x", 169 _ => return Err(ENOTSUPP), 170 }; 171 let signatures = elf::elf64_section(fw.data(), sigs_section) 172 .ok_or(EINVAL) 173 .and_then(|data| DmaObject::from_data(dev, data))?; 174 175 let size = fw_section.len(); 176 177 // Move the firmware into a vmalloc'd vector and map it into the device address 178 // space. 179 let fw_vvec = VVec::with_capacity(fw_section.len(), GFP_KERNEL) 180 .and_then(|mut v| { 181 v.extend_from_slice(fw_section, GFP_KERNEL)?; 182 Ok(v) 183 }) 184 .map_err(|_| ENOMEM)?; 185 186 let bl = super::request_firmware(dev, chipset, "bootloader", ver)?; 187 let bootloader = RiscvFirmware::new(dev, &bl)?; 188 189 Ok(try_pin_init!(Self { 190 fw <- SGTable::new(dev, fw_vvec, DataDirection::ToDevice, GFP_KERNEL), 191 level2 <- { 192 // Allocate the level 2 page table, map the firmware onto it, and map it into 193 // the device address space. 194 VVec::<u8>::with_capacity( 195 fw.iter().count() * core::mem::size_of::<u64>(), 196 GFP_KERNEL, 197 ) 198 .map_err(|_| ENOMEM) 199 .and_then(|level2| map_into_lvl(&fw, level2)) 200 .map(|level2| SGTable::new(dev, level2, DataDirection::ToDevice, GFP_KERNEL))? 201 }, 202 level1 <- { 203 // Allocate the level 1 page table, map the level 2 page table onto it, and map 204 // it into the device address space. 205 VVec::<u8>::with_capacity( 206 level2.iter().count() * core::mem::size_of::<u64>(), 207 GFP_KERNEL, 208 ) 209 .map_err(|_| ENOMEM) 210 .and_then(|level1| map_into_lvl(&level2, level1)) 211 .map(|level1| SGTable::new(dev, level1, DataDirection::ToDevice, GFP_KERNEL))? 212 }, 213 level0: { 214 // Allocate the level 0 page table as a device-visible DMA object, and map the 215 // level 1 page table onto it. 216 217 // Level 0 page table data. 218 let mut level0_data = kvec![0u8; GSP_PAGE_SIZE]?; 219 220 // Fill level 1 page entry. 221 let level1_entry = level1.iter().next().ok_or(EINVAL)?; 222 let level1_entry_addr = level1_entry.dma_address(); 223 let dst = &mut level0_data[..size_of_val(&level1_entry_addr)]; 224 dst.copy_from_slice(&level1_entry_addr.to_le_bytes()); 225 226 // Turn the level0 page table into a [`DmaObject`]. 227 DmaObject::from_data(dev, &level0_data)? 228 }, 229 size, 230 signatures, 231 bootloader, 232 })) 233 }) 234 } 235 236 /// Returns the DMA handle of the radix3 level 0 page table. 237 pub(crate) fn radix3_dma_handle(&self) -> DmaAddress { 238 self.level0.dma_handle() 239 } 240 } 241 242 /// Build a page table from a scatter-gather list. 243 /// 244 /// Takes each DMA-mapped region from `sg_table` and writes page table entries 245 /// for all 4KB pages within that region. For example, a 16KB SG entry becomes 246 /// 4 consecutive page table entries. 247 fn map_into_lvl(sg_table: &SGTable<Owned<VVec<u8>>>, mut dst: VVec<u8>) -> Result<VVec<u8>> { 248 for sg_entry in sg_table.iter() { 249 // Number of pages we need to map. 250 let num_pages = usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE); 251 252 for i in 0..num_pages { 253 let entry = sg_entry.dma_address() 254 + (u64::from_safe_cast(i) * u64::from_safe_cast(GSP_PAGE_SIZE)); 255 dst.extend_from_slice(&entry.to_le_bytes(), GFP_KERNEL)?; 256 } 257 } 258 259 Ok(dst) 260 } 261