1 // SPDX-License-Identifier: GPL-2.0 2 3 use kernel::{ 4 device, 5 dma::{ 6 Coherent, 7 CoherentBox, 8 DataDirection, 9 DmaAddress, // 10 }, 11 prelude::*, 12 scatterlist::{ 13 Owned, 14 SGTable, // 15 }, 16 }; 17 18 use crate::{ 19 firmware::{ 20 elf, 21 riscv::RiscvFirmware, // 22 }, 23 gpu::{ 24 Architecture, 25 Chipset, // 26 }, 27 gsp::GSP_PAGE_SIZE, 28 num::FromSafeCast, 29 }; 30 31 /// GSP firmware with 3-level radix page tables for the GSP bootloader. 32 /// 33 /// The bootloader expects firmware to be mapped starting at address 0 in GSP's virtual address 34 /// space: 35 /// 36 /// ```text 37 /// Level 0: 1 page, 1 entry -> points to first level 1 page 38 /// Level 1: Multiple pages/entries -> each entry points to a level 2 page 39 /// Level 2: Multiple pages/entries -> each entry points to a firmware page 40 /// ``` 41 /// 42 /// Each page is 4KB, each entry is 8 bytes (64-bit DMA address). 43 /// Also known as "Radix3" firmware. 44 #[pin_data] 45 pub(crate) struct GspFirmware { 46 /// The GSP firmware inside a [`VVec`], device-mapped via a SG table. 47 #[pin] 48 fw: SGTable<Owned<VVec<u8>>>, 49 /// Level 2 page table whose entries contain DMA addresses of firmware pages. 50 #[pin] 51 level2: SGTable<Owned<VVec<u8>>>, 52 /// Level 1 page table whose entries contain DMA addresses of level 2 pages. 53 #[pin] 54 level1: SGTable<Owned<VVec<u8>>>, 55 /// Level 0 page table (single 4KB page) with one entry: DMA address of first level 1 page. 56 level0: Coherent<[u64]>, 57 /// Size in bytes of the firmware contained in [`Self::fw`]. 58 pub(crate) size: usize, 59 /// Device-mapped GSP signatures matching the GPU's [`Chipset`]. 60 pub(crate) signatures: Coherent<[u8]>, 61 /// GSP bootloader, verifies the GSP firmware before loading and running it. 62 pub(crate) bootloader: RiscvFirmware, 63 } 64 65 impl GspFirmware { 66 fn find_gsp_sigs_section(chipset: Chipset) -> &'static str { 67 match chipset.arch() { 68 Architecture::Turing if matches!(chipset, Chipset::TU116 | Chipset::TU117) => { 69 ".fwsignature_tu11x" 70 } 71 Architecture::Turing => ".fwsignature_tu10x", 72 Architecture::Ampere if chipset == Chipset::GA100 => ".fwsignature_ga100", 73 Architecture::Ampere => ".fwsignature_ga10x", 74 Architecture::Ada => ".fwsignature_ad10x", 75 Architecture::Hopper => ".fwsignature_gh10x", 76 Architecture::BlackwellGB10x => ".fwsignature_gb10x", 77 Architecture::BlackwellGB20x => ".fwsignature_gb20x", 78 } 79 } 80 81 /// Loads the GSP firmware binaries, map them into `dev`'s address-space, and creates the page 82 /// tables expected by the GSP bootloader to load it. 83 pub(crate) fn new<'a>( 84 dev: &'a device::Device<device::Bound>, 85 chipset: Chipset, 86 ver: &'a str, 87 ) -> impl PinInit<Self, Error> + 'a { 88 pin_init::pin_init_scope(move || { 89 let firmware = super::request_firmware(dev, chipset, "gsp", ver)?; 90 91 let fw_section = elf::elf_section(firmware.data(), ".fwimage").ok_or(EINVAL)?; 92 93 let size = fw_section.len(); 94 95 // Move the firmware into a vmalloc'd vector and map it into the device address 96 // space. 97 let fw_vvec = VVec::with_capacity(fw_section.len(), GFP_KERNEL) 98 .and_then(|mut v| { 99 v.extend_from_slice(fw_section, GFP_KERNEL)?; 100 Ok(v) 101 }) 102 .map_err(|_| ENOMEM)?; 103 104 Ok(try_pin_init!(Self { 105 fw <- SGTable::new(dev, fw_vvec, DataDirection::ToDevice, GFP_KERNEL), 106 level2 <- { 107 // Allocate the level 2 page table, map the firmware onto it, and map it into 108 // the device address space. 109 VVec::<u8>::with_capacity( 110 fw.iter().count() * core::mem::size_of::<u64>(), 111 GFP_KERNEL, 112 ) 113 .map_err(|_| ENOMEM) 114 .and_then(|level2| map_into_lvl(&fw, level2)) 115 .map(|level2| SGTable::new(dev, level2, DataDirection::ToDevice, GFP_KERNEL))? 116 }, 117 level1 <- { 118 // Allocate the level 1 page table, map the level 2 page table onto it, and map 119 // it into the device address space. 120 VVec::<u8>::with_capacity( 121 level2.iter().count() * core::mem::size_of::<u64>(), 122 GFP_KERNEL, 123 ) 124 .map_err(|_| ENOMEM) 125 .and_then(|level1| map_into_lvl(&level2, level1)) 126 .map(|level1| SGTable::new(dev, level1, DataDirection::ToDevice, GFP_KERNEL))? 127 }, 128 level0: { 129 // Allocate the level 0 page table as a device-visible DMA object, and map the 130 // level 1 page table onto it. 131 132 // Fill level 1 page entry. 133 let level1_entry = level1.iter().next().ok_or(EINVAL)?; 134 let level1_entry_addr = level1_entry.dma_address(); 135 136 // Create level 0 page table data and fill its first entry with the level 1 137 // table. 138 let mut level0 = CoherentBox::<[u64]>::zeroed_slice( 139 dev, 140 GSP_PAGE_SIZE / size_of::<u64>(), 141 GFP_KERNEL 142 )?; 143 level0[0] = level1_entry_addr.to_le(); 144 145 level0.into() 146 }, 147 size, 148 signatures: { 149 let sigs_section = Self::find_gsp_sigs_section(chipset); 150 151 elf::elf_section(firmware.data(), sigs_section) 152 .ok_or(EINVAL) 153 .and_then(|data| Coherent::from_slice(dev, data, GFP_KERNEL))? 154 }, 155 bootloader: { 156 let bl = super::request_firmware(dev, chipset, "bootloader", ver)?; 157 158 RiscvFirmware::new(dev, &bl)? 159 }, 160 })) 161 }) 162 } 163 164 /// Returns the DMA handle of the radix3 level 0 page table. 165 pub(crate) fn radix3_dma_handle(&self) -> DmaAddress { 166 self.level0.dma_handle() 167 } 168 } 169 170 /// Build a page table from a scatter-gather list. 171 /// 172 /// Takes each DMA-mapped region from `sg_table` and writes page table entries 173 /// for all 4KB pages within that region. For example, a 16KB SG entry becomes 174 /// 4 consecutive page table entries. 175 fn map_into_lvl(sg_table: &SGTable<Owned<VVec<u8>>>, mut dst: VVec<u8>) -> Result<VVec<u8>> { 176 for sg_entry in sg_table.iter() { 177 // Number of pages we need to map. 178 let num_pages = usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE); 179 180 for i in 0..num_pages { 181 let entry = sg_entry.dma_address() 182 + (u64::from_safe_cast(i) * u64::from_safe_cast(GSP_PAGE_SIZE)); 183 dst.extend_from_slice(&entry.to_le_bytes(), GFP_KERNEL)?; 184 } 185 } 186 187 Ok(dst) 188 } 189