1 // SPDX-License-Identifier: GPL-2.0 2 3 use kernel::{ 4 device, 5 dma::{ 6 Coherent, 7 CoherentBox, 8 DataDirection, 9 DmaAddress, // 10 }, 11 prelude::*, 12 scatterlist::{ 13 Owned, 14 SGTable, // 15 }, 16 }; 17 18 use crate::{ 19 firmware::{ 20 elf, 21 riscv::RiscvFirmware, // 22 }, 23 gpu::{ 24 Architecture, 25 Chipset, // 26 }, 27 gsp::GSP_PAGE_SIZE, 28 num::FromSafeCast, 29 }; 30 31 /// GSP firmware with 3-level radix page tables for the GSP bootloader. 32 /// 33 /// The bootloader expects firmware to be mapped starting at address 0 in GSP's virtual address 34 /// space: 35 /// 36 /// ```text 37 /// Level 0: 1 page, 1 entry -> points to first level 1 page 38 /// Level 1: Multiple pages/entries -> each entry points to a level 2 page 39 /// Level 2: Multiple pages/entries -> each entry points to a firmware page 40 /// ``` 41 /// 42 /// Each page is 4KB, each entry is 8 bytes (64-bit DMA address). 43 /// Also known as "Radix3" firmware. 44 #[pin_data] 45 pub(crate) struct GspFirmware { 46 /// The GSP firmware inside a [`VVec`], device-mapped via a SG table. 47 #[pin] 48 fw: SGTable<Owned<VVec<u8>>>, 49 /// Level 2 page table whose entries contain DMA addresses of firmware pages. 50 #[pin] 51 level2: SGTable<Owned<VVec<u8>>>, 52 /// Level 1 page table whose entries contain DMA addresses of level 2 pages. 53 #[pin] 54 level1: SGTable<Owned<VVec<u8>>>, 55 /// Level 0 page table (single 4KB page) with one entry: DMA address of first level 1 page. 56 level0: Coherent<[u64]>, 57 /// Size in bytes of the firmware contained in [`Self::fw`]. 58 pub(crate) size: usize, 59 /// Device-mapped GSP signatures matching the GPU's [`Chipset`]. 60 pub(crate) signatures: Coherent<[u8]>, 61 /// GSP bootloader, verifies the GSP firmware before loading and running it. 62 pub(crate) bootloader: RiscvFirmware, 63 } 64 65 impl GspFirmware { 66 /// Loads the GSP firmware binaries, map them into `dev`'s address-space, and creates the page 67 /// tables expected by the GSP bootloader to load it. 68 pub(crate) fn new<'a>( 69 dev: &'a device::Device<device::Bound>, 70 chipset: Chipset, 71 ver: &'a str, 72 ) -> impl PinInit<Self, Error> + 'a { 73 pin_init::pin_init_scope(move || { 74 let firmware = super::request_firmware(dev, chipset, "gsp", ver)?; 75 76 let fw_section = elf::elf64_section(firmware.data(), ".fwimage").ok_or(EINVAL)?; 77 78 let size = fw_section.len(); 79 80 // Move the firmware into a vmalloc'd vector and map it into the device address 81 // space. 82 let fw_vvec = VVec::with_capacity(fw_section.len(), GFP_KERNEL) 83 .and_then(|mut v| { 84 v.extend_from_slice(fw_section, GFP_KERNEL)?; 85 Ok(v) 86 }) 87 .map_err(|_| ENOMEM)?; 88 89 Ok(try_pin_init!(Self { 90 fw <- SGTable::new(dev, fw_vvec, DataDirection::ToDevice, GFP_KERNEL), 91 level2 <- { 92 // Allocate the level 2 page table, map the firmware onto it, and map it into 93 // the device address space. 94 VVec::<u8>::with_capacity( 95 fw.iter().count() * core::mem::size_of::<u64>(), 96 GFP_KERNEL, 97 ) 98 .map_err(|_| ENOMEM) 99 .and_then(|level2| map_into_lvl(&fw, level2)) 100 .map(|level2| SGTable::new(dev, level2, DataDirection::ToDevice, GFP_KERNEL))? 101 }, 102 level1 <- { 103 // Allocate the level 1 page table, map the level 2 page table onto it, and map 104 // it into the device address space. 105 VVec::<u8>::with_capacity( 106 level2.iter().count() * core::mem::size_of::<u64>(), 107 GFP_KERNEL, 108 ) 109 .map_err(|_| ENOMEM) 110 .and_then(|level1| map_into_lvl(&level2, level1)) 111 .map(|level1| SGTable::new(dev, level1, DataDirection::ToDevice, GFP_KERNEL))? 112 }, 113 level0: { 114 // Allocate the level 0 page table as a device-visible DMA object, and map the 115 // level 1 page table onto it. 116 117 // Fill level 1 page entry. 118 let level1_entry = level1.iter().next().ok_or(EINVAL)?; 119 let level1_entry_addr = level1_entry.dma_address(); 120 121 // Create level 0 page table data and fill its first entry with the level 1 122 // table. 123 let mut level0 = CoherentBox::<[u64]>::zeroed_slice( 124 dev, 125 GSP_PAGE_SIZE / size_of::<u64>(), 126 GFP_KERNEL 127 )?; 128 level0[0] = level1_entry_addr.to_le(); 129 130 level0.into() 131 }, 132 size, 133 signatures: { 134 let sigs_section = match chipset.arch() { 135 Architecture::Turing 136 if matches!(chipset, Chipset::TU116 | Chipset::TU117) => 137 { 138 ".fwsignature_tu11x" 139 } 140 Architecture::Turing => ".fwsignature_tu10x", 141 // GA100 uses the same firmware as Turing 142 Architecture::Ampere if chipset == Chipset::GA100 => ".fwsignature_tu10x", 143 Architecture::Ampere => ".fwsignature_ga10x", 144 Architecture::Ada => ".fwsignature_ad10x", 145 }; 146 147 elf::elf64_section(firmware.data(), sigs_section) 148 .ok_or(EINVAL) 149 .and_then(|data| Coherent::from_slice(dev, data, GFP_KERNEL))? 150 }, 151 bootloader: { 152 let bl = super::request_firmware(dev, chipset, "bootloader", ver)?; 153 154 RiscvFirmware::new(dev, &bl)? 155 }, 156 })) 157 }) 158 } 159 160 /// Returns the DMA handle of the radix3 level 0 page table. 161 pub(crate) fn radix3_dma_handle(&self) -> DmaAddress { 162 self.level0.dma_handle() 163 } 164 } 165 166 /// Build a page table from a scatter-gather list. 167 /// 168 /// Takes each DMA-mapped region from `sg_table` and writes page table entries 169 /// for all 4KB pages within that region. For example, a 16KB SG entry becomes 170 /// 4 consecutive page table entries. 171 fn map_into_lvl(sg_table: &SGTable<Owned<VVec<u8>>>, mut dst: VVec<u8>) -> Result<VVec<u8>> { 172 for sg_entry in sg_table.iter() { 173 // Number of pages we need to map. 174 let num_pages = usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE); 175 176 for i in 0..num_pages { 177 let entry = sg_entry.dma_address() 178 + (u64::from_safe_cast(i) * u64::from_safe_cast(GSP_PAGE_SIZE)); 179 dst.extend_from_slice(&entry.to_le_bytes(), GFP_KERNEL)?; 180 } 181 } 182 183 Ok(dst) 184 } 185