1 // SPDX-License-Identifier: GPL-2.0 2 // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 4 //! The radix3 page table, through which firmware running on the GPU reads a buffer in system 5 //! memory. LibOS, the operating system of the GSP, defines the format: three levels of 6 //! [`GSP_PAGE_SIZE`] pages, each entry the little-endian DMA address of one page. 7 //! 8 //! ```text 9 //! Level 0: one page, one entry -> the first level 1 page 10 //! Level 1: pages of entries -> each entry a level 2 page 11 //! Level 2: pages of entries -> each entry a page of the buffer 12 //! ``` 13 14 use kernel::{ 15 device, 16 dma::{ 17 Coherent, 18 CoherentBox, 19 DataDirection, 20 DmaAddress, // 21 }, 22 prelude::*, 23 scatterlist::{ 24 Owned, 25 SGTable, // 26 }, 27 }; 28 29 use crate::{ 30 gsp::GSP_PAGE_SIZE, 31 num::FromSafeCast, // 32 }; 33 34 /// A radix3 page table and the data it maps. 35 #[pin_data] 36 pub(crate) struct Radix3<'a> { 37 /// The mapped data. 38 #[pin] 39 data: SGTable<Owned<VVec<u8>>>, 40 /// Level 2: one entry per page of `data`. 41 #[pin] 42 level2: SGTable<Owned<VVec<u8>>>, 43 /// Level 1: one entry per page of `level2`. 44 #[pin] 45 level1: SGTable<Owned<VVec<u8>>>, 46 /// Level 0: one page, whose single entry is the DMA address of the first `level1` page. 47 level0: Coherent<'a, [u64]>, 48 /// Length of `data`, in bytes. 49 size: usize, 50 } 51 52 impl<'a> Radix3<'a> { 53 /// Builds a radix3 page table over `data`, mapped for `dev` to read. May sleep. 54 pub(crate) fn new( 55 dev: &'a device::Device<device::Bound>, 56 data: VVec<u8>, 57 ) -> impl PinInit<Self, Error> + 'a { 58 let size = data.len(); 59 60 pin_init::pin_init_scope(move || { 61 Ok(try_pin_init!(Self { 62 data <- SGTable::new(dev, data, DataDirection::ToDevice, GFP_KERNEL), 63 level2 <- { 64 build_lvl(&data) 65 .map(|l2| SGTable::new(dev, l2, DataDirection::ToDevice, GFP_KERNEL))? 66 }, 67 level1 <- { 68 build_lvl(&level2) 69 .map(|l1| SGTable::new(dev, l1, DataDirection::ToDevice, GFP_KERNEL))? 70 }, 71 level0: { 72 // Allocate the level 0 page table as a device-visible DMA object, and map the 73 // level 1 page table onto it. 74 75 // Fill level 1 page entry. 76 let level1_entry = level1.iter().next().ok_or(EINVAL)?; 77 let level1_entry_addr = level1_entry.dma_address(); 78 79 // Create level 0 page table data and fill its first entry with the level 1 80 // table. 81 let mut level0 = CoherentBox::<'_, [u64]>::zeroed_slice( 82 dev, 83 GSP_PAGE_SIZE / size_of::<u64>(), 84 GFP_KERNEL 85 )?; 86 level0[0] = level1_entry_addr.to_le(); 87 88 level0.into() 89 }, 90 size, 91 })) 92 }) 93 } 94 95 /// Returns the DMA address of the level 0 page, which is the address of the table. 96 pub(crate) fn dma_address(&self) -> DmaAddress { 97 self.level0.dma_address() 98 } 99 100 /// Returns the length of the mapped data, in bytes. 101 pub(crate) fn size(&self) -> usize { 102 self.size 103 } 104 } 105 106 /// Returns the size of the level that maps `sg_table`: one `u64` entry per [`GSP_PAGE_SIZE`] 107 /// page of each DMA-mapped region, rounded up to whole pages. 108 fn lvl_size(sg_table: &SGTable<Owned<VVec<u8>>>) -> usize { 109 let entries: usize = sg_table 110 .iter() 111 .map(|sg_entry| usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE)) 112 .sum(); 113 114 (entries * size_of::<u64>()).next_multiple_of(GSP_PAGE_SIZE) 115 } 116 117 /// Builds one level of the table over `sg_table`: one entry per [`GSP_PAGE_SIZE`] page of each 118 /// DMA-mapped region, in region order. The level is a whole number of pages and every byte past 119 /// the last entry is zero, because the booter reads a level a whole page at a time. 120 /// 121 /// # Errors 122 /// 123 /// - `ENOMEM` if the level cannot be allocated. 124 /// - `EINVAL` if `sg_table` yields more entries than [`lvl_size`] sized the level for. 125 fn build_lvl(sg_table: &SGTable<Owned<VVec<u8>>>) -> Result<VVec<u8>> { 126 let mut dst = VVec::<u8>::zeroed(lvl_size(sg_table), GFP_KERNEL).map_err(|_| ENOMEM)?; 127 let mut entries = dst.chunks_exact_mut(size_of::<u64>()); 128 129 for sg_entry in sg_table.iter() { 130 let num_pages = usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE); 131 132 for i in 0..num_pages { 133 let entry = sg_entry.dma_address() 134 + (u64::from_safe_cast(i) * u64::from_safe_cast(GSP_PAGE_SIZE)); 135 136 entries 137 .next() 138 .ok_or(EINVAL)? 139 .copy_from_slice(&entry.to_le_bytes()); 140 } 141 } 142 143 Ok(dst) 144 } 145