xref: /linux/drivers/gpu/nova-core/firmware/radix3.rs (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 // SPDX-License-Identifier: GPL-2.0
2 // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 
4 //! The radix3 page table, through which firmware running on the GPU reads a buffer in system
5 //! memory. LibOS, the operating system of the GSP, defines the format: three levels of
6 //! [`GSP_PAGE_SIZE`] pages, each entry the little-endian DMA address of one page.
7 //!
8 //! ```text
9 //! Level 0:  one page, one entry  ->  the first level 1 page
10 //! Level 1:  pages of entries     ->  each entry a level 2 page
11 //! Level 2:  pages of entries     ->  each entry a page of the buffer
12 //! ```
13 
14 use kernel::{
15     device,
16     dma::{
17         Coherent,
18         CoherentBox,
19         DataDirection,
20         DmaAddress, //
21     },
22     prelude::*,
23     scatterlist::{
24         Owned,
25         SGTable, //
26     },
27 };
28 
29 use crate::{
30     gsp::GSP_PAGE_SIZE,
31     num::FromSafeCast, //
32 };
33 
34 /// A radix3 page table and the data it maps.
35 #[pin_data]
36 pub(crate) struct Radix3<'a> {
37     /// The mapped data.
38     #[pin]
39     data: SGTable<Owned<VVec<u8>>>,
40     /// Level 2: one entry per page of `data`.
41     #[pin]
42     level2: SGTable<Owned<VVec<u8>>>,
43     /// Level 1: one entry per page of `level2`.
44     #[pin]
45     level1: SGTable<Owned<VVec<u8>>>,
46     /// Level 0: one page, whose single entry is the DMA address of the first `level1` page.
47     level0: Coherent<'a, [u64]>,
48     /// Length of `data`, in bytes.
49     size: usize,
50 }
51 
52 impl<'a> Radix3<'a> {
53     /// Builds a radix3 page table over `data`, mapped for `dev` to read. May sleep.
54     pub(crate) fn new(
55         dev: &'a device::Device<device::Bound>,
56         data: VVec<u8>,
57     ) -> impl PinInit<Self, Error> + 'a {
58         let size = data.len();
59 
60         pin_init::pin_init_scope(move || {
61             Ok(try_pin_init!(Self {
62                 data <- SGTable::new(dev, data, DataDirection::ToDevice, GFP_KERNEL),
63                 level2 <- {
64                     build_lvl(&data)
65                         .map(|l2| SGTable::new(dev, l2, DataDirection::ToDevice, GFP_KERNEL))?
66                 },
67                 level1 <- {
68                     build_lvl(&level2)
69                         .map(|l1| SGTable::new(dev, l1, DataDirection::ToDevice, GFP_KERNEL))?
70                 },
71                 level0: {
72                     // Allocate the level 0 page table as a device-visible DMA object, and map the
73                     // level 1 page table onto it.
74 
75                     // Fill level 1 page entry.
76                     let level1_entry = level1.iter().next().ok_or(EINVAL)?;
77                     let level1_entry_addr = level1_entry.dma_address();
78 
79                     // Create level 0 page table data and fill its first entry with the level 1
80                     // table.
81                     let mut level0 = CoherentBox::<'_, [u64]>::zeroed_slice(
82                         dev,
83                         GSP_PAGE_SIZE / size_of::<u64>(),
84                         GFP_KERNEL
85                     )?;
86                     level0[0] = level1_entry_addr.to_le();
87 
88                     level0.into()
89                 },
90                 size,
91             }))
92         })
93     }
94 
95     /// Returns the DMA address of the level 0 page, which is the address of the table.
96     pub(crate) fn dma_address(&self) -> DmaAddress {
97         self.level0.dma_address()
98     }
99 
100     /// Returns the length of the mapped data, in bytes.
101     pub(crate) fn size(&self) -> usize {
102         self.size
103     }
104 }
105 
106 /// Returns the size of the level that maps `sg_table`: one `u64` entry per [`GSP_PAGE_SIZE`]
107 /// page of each DMA-mapped region, rounded up to whole pages.
108 fn lvl_size(sg_table: &SGTable<Owned<VVec<u8>>>) -> usize {
109     let entries: usize = sg_table
110         .iter()
111         .map(|sg_entry| usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE))
112         .sum();
113 
114     (entries * size_of::<u64>()).next_multiple_of(GSP_PAGE_SIZE)
115 }
116 
117 /// Builds one level of the table over `sg_table`: one entry per [`GSP_PAGE_SIZE`] page of each
118 /// DMA-mapped region, in region order. The level is a whole number of pages and every byte past
119 /// the last entry is zero, because the booter reads a level a whole page at a time.
120 ///
121 /// # Errors
122 ///
123 /// - `ENOMEM` if the level cannot be allocated.
124 /// - `EINVAL` if `sg_table` yields more entries than [`lvl_size`] sized the level for.
125 fn build_lvl(sg_table: &SGTable<Owned<VVec<u8>>>) -> Result<VVec<u8>> {
126     let mut dst = VVec::<u8>::zeroed(lvl_size(sg_table), GFP_KERNEL).map_err(|_| ENOMEM)?;
127     let mut entries = dst.chunks_exact_mut(size_of::<u64>());
128 
129     for sg_entry in sg_table.iter() {
130         let num_pages = usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE);
131 
132         for i in 0..num_pages {
133             let entry = sg_entry.dma_address()
134                 + (u64::from_safe_cast(i) * u64::from_safe_cast(GSP_PAGE_SIZE));
135 
136             entries
137                 .next()
138                 .ok_or(EINVAL)?
139                 .copy_from_slice(&entry.to_le_bytes());
140         }
141     }
142 
143     Ok(dst)
144 }
145