1 // SPDX-License-Identifier: GPL-2.0 2 3 use core::mem::size_of_val; 4 5 use kernel::device; 6 use kernel::dma::{DataDirection, DmaAddress}; 7 use kernel::kvec; 8 use kernel::prelude::*; 9 use kernel::scatterlist::{Owned, SGTable}; 10 11 use crate::dma::DmaObject; 12 use crate::firmware::riscv::RiscvFirmware; 13 use crate::gpu::{Architecture, Chipset}; 14 use crate::gsp::GSP_PAGE_SIZE; 15 16 /// Ad-hoc and temporary module to extract sections from ELF images. 17 /// 18 /// Some firmware images are currently packaged as ELF files, where sections names are used as keys 19 /// to specific and related bits of data. Future firmware versions are scheduled to move away from 20 /// that scheme before nova-core becomes stable, which means this module will eventually be 21 /// removed. 22 mod elf { 23 use core::mem::size_of; 24 25 use kernel::bindings; 26 use kernel::str::CStr; 27 use kernel::transmute::FromBytes; 28 29 /// Newtype to provide a [`FromBytes`] implementation. 30 #[repr(transparent)] 31 struct Elf64Hdr(bindings::elf64_hdr); 32 // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. 33 unsafe impl FromBytes for Elf64Hdr {} 34 35 #[repr(transparent)] 36 struct Elf64SHdr(bindings::elf64_shdr); 37 // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. 38 unsafe impl FromBytes for Elf64SHdr {} 39 40 /// Tries to extract section with name `name` from the ELF64 image `elf`, and returns it. 41 pub(super) fn elf64_section<'a, 'b>(elf: &'a [u8], name: &'b str) -> Option<&'a [u8]> { 42 let hdr = &elf 43 .get(0..size_of::<bindings::elf64_hdr>()) 44 .and_then(Elf64Hdr::from_bytes)? 45 .0; 46 47 // Get all the section headers. 48 let mut shdr = { 49 let shdr_num = usize::from(hdr.e_shnum); 50 let shdr_start = usize::try_from(hdr.e_shoff).ok()?; 51 let shdr_end = shdr_num 52 .checked_mul(size_of::<Elf64SHdr>()) 53 .and_then(|v| v.checked_add(shdr_start))?; 54 55 elf.get(shdr_start..shdr_end) 56 .map(|slice| slice.chunks_exact(size_of::<Elf64SHdr>()))? 57 }; 58 59 // Get the strings table. 60 let strhdr = shdr 61 .clone() 62 .nth(usize::from(hdr.e_shstrndx)) 63 .and_then(Elf64SHdr::from_bytes)?; 64 65 // Find the section which name matches `name` and return it. 66 shdr.find(|&sh| { 67 let Some(hdr) = Elf64SHdr::from_bytes(sh) else { 68 return false; 69 }; 70 71 let Some(name_idx) = strhdr 72 .0 73 .sh_offset 74 .checked_add(u64::from(hdr.0.sh_name)) 75 .and_then(|idx| usize::try_from(idx).ok()) 76 else { 77 return false; 78 }; 79 80 // Get the start of the name. 81 elf.get(name_idx..) 82 // Stop at the first `0`. 83 .and_then(|nstr| nstr.get(0..=nstr.iter().position(|b| *b == 0)?)) 84 // Convert into CStr. This should never fail because of the line above. 85 .and_then(|nstr| CStr::from_bytes_with_nul(nstr).ok()) 86 // Convert into str. 87 .and_then(|c_str| c_str.to_str().ok()) 88 // Check that the name matches. 89 .map(|str| str == name) 90 .unwrap_or(false) 91 }) 92 // Return the slice containing the section. 93 .and_then(|sh| { 94 let hdr = Elf64SHdr::from_bytes(sh)?; 95 let start = usize::try_from(hdr.0.sh_offset).ok()?; 96 let end = usize::try_from(hdr.0.sh_size) 97 .ok() 98 .and_then(|sh_size| start.checked_add(sh_size))?; 99 100 elf.get(start..end) 101 }) 102 } 103 } 104 105 /// GSP firmware with 3-level radix page tables for the GSP bootloader. 106 /// 107 /// The bootloader expects firmware to be mapped starting at address 0 in GSP's virtual address 108 /// space: 109 /// 110 /// ```text 111 /// Level 0: 1 page, 1 entry -> points to first level 1 page 112 /// Level 1: Multiple pages/entries -> each entry points to a level 2 page 113 /// Level 2: Multiple pages/entries -> each entry points to a firmware page 114 /// ``` 115 /// 116 /// Each page is 4KB, each entry is 8 bytes (64-bit DMA address). 117 /// Also known as "Radix3" firmware. 118 #[pin_data] 119 pub(crate) struct GspFirmware { 120 /// The GSP firmware inside a [`VVec`], device-mapped via a SG table. 121 #[pin] 122 fw: SGTable<Owned<VVec<u8>>>, 123 /// Level 2 page table whose entries contain DMA addresses of firmware pages. 124 #[pin] 125 level2: SGTable<Owned<VVec<u8>>>, 126 /// Level 1 page table whose entries contain DMA addresses of level 2 pages. 127 #[pin] 128 level1: SGTable<Owned<VVec<u8>>>, 129 /// Level 0 page table (single 4KB page) with one entry: DMA address of first level 1 page. 130 level0: DmaObject, 131 /// Size in bytes of the firmware contained in [`Self::fw`]. 132 size: usize, 133 /// Device-mapped GSP signatures matching the GPU's [`Chipset`]. 134 signatures: DmaObject, 135 /// GSP bootloader, verifies the GSP firmware before loading and running it. 136 bootloader: RiscvFirmware, 137 } 138 139 impl GspFirmware { 140 /// Loads the GSP firmware binaries, map them into `dev`'s address-space, and creates the page 141 /// tables expected by the GSP bootloader to load it. 142 pub(crate) fn new<'a, 'b>( 143 dev: &'a device::Device<device::Bound>, 144 chipset: Chipset, 145 ver: &'b str, 146 ) -> Result<impl PinInit<Self, Error> + 'a> { 147 let fw = super::request_firmware(dev, chipset, "gsp", ver)?; 148 149 let fw_section = elf::elf64_section(fw.data(), ".fwimage").ok_or(EINVAL)?; 150 151 let sigs_section = match chipset.arch() { 152 Architecture::Ampere => ".fwsignature_ga10x", 153 Architecture::Ada => ".fwsignature_ad10x", 154 _ => return Err(ENOTSUPP), 155 }; 156 let signatures = elf::elf64_section(fw.data(), sigs_section) 157 .ok_or(EINVAL) 158 .and_then(|data| DmaObject::from_data(dev, data))?; 159 160 let size = fw_section.len(); 161 162 // Move the firmware into a vmalloc'd vector and map it into the device address 163 // space. 164 let fw_vvec = VVec::with_capacity(fw_section.len(), GFP_KERNEL) 165 .and_then(|mut v| { 166 v.extend_from_slice(fw_section, GFP_KERNEL)?; 167 Ok(v) 168 }) 169 .map_err(|_| ENOMEM)?; 170 171 let bl = super::request_firmware(dev, chipset, "bootloader", ver)?; 172 let bootloader = RiscvFirmware::new(dev, &bl)?; 173 174 Ok(try_pin_init!(Self { 175 fw <- SGTable::new(dev, fw_vvec, DataDirection::ToDevice, GFP_KERNEL), 176 level2 <- { 177 // Allocate the level 2 page table, map the firmware onto it, and map it into the 178 // device address space. 179 VVec::<u8>::with_capacity( 180 fw.iter().count() * core::mem::size_of::<u64>(), 181 GFP_KERNEL, 182 ) 183 .map_err(|_| ENOMEM) 184 .and_then(|level2| map_into_lvl(&fw, level2)) 185 .map(|level2| SGTable::new(dev, level2, DataDirection::ToDevice, GFP_KERNEL))? 186 }, 187 level1 <- { 188 // Allocate the level 1 page table, map the level 2 page table onto it, and map it 189 // into the device address space. 190 VVec::<u8>::with_capacity( 191 level2.iter().count() * core::mem::size_of::<u64>(), 192 GFP_KERNEL, 193 ) 194 .map_err(|_| ENOMEM) 195 .and_then(|level1| map_into_lvl(&level2, level1)) 196 .map(|level1| SGTable::new(dev, level1, DataDirection::ToDevice, GFP_KERNEL))? 197 }, 198 level0: { 199 // Allocate the level 0 page table as a device-visible DMA object, and map the 200 // level 1 page table onto it. 201 202 // Level 0 page table data. 203 let mut level0_data = kvec![0u8; GSP_PAGE_SIZE]?; 204 205 // Fill level 1 page entry. 206 let level1_entry = level1.iter().next().ok_or(EINVAL)?; 207 let level1_entry_addr = level1_entry.dma_address(); 208 let dst = &mut level0_data[..size_of_val(&level1_entry_addr)]; 209 dst.copy_from_slice(&level1_entry_addr.to_le_bytes()); 210 211 // Turn the level0 page table into a [`DmaObject`]. 212 DmaObject::from_data(dev, &level0_data)? 213 }, 214 size, 215 signatures, 216 bootloader, 217 })) 218 } 219 220 #[expect(unused)] 221 /// Returns the DMA handle of the radix3 level 0 page table. 222 pub(crate) fn radix3_dma_handle(&self) -> DmaAddress { 223 self.level0.dma_handle() 224 } 225 } 226 227 /// Build a page table from a scatter-gather list. 228 /// 229 /// Takes each DMA-mapped region from `sg_table` and writes page table entries 230 /// for all 4KB pages within that region. For example, a 16KB SG entry becomes 231 /// 4 consecutive page table entries. 232 fn map_into_lvl(sg_table: &SGTable<Owned<VVec<u8>>>, mut dst: VVec<u8>) -> Result<VVec<u8>> { 233 for sg_entry in sg_table.iter() { 234 // Number of pages we need to map. 235 let num_pages = (sg_entry.dma_len() as usize).div_ceil(GSP_PAGE_SIZE); 236 237 for i in 0..num_pages { 238 let entry = sg_entry.dma_address() + (i as u64 * GSP_PAGE_SIZE as u64); 239 dst.extend_from_slice(&entry.to_le_bytes(), GFP_KERNEL)?; 240 } 241 } 242 243 Ok(dst) 244 } 245