1 // SPDX-License-Identifier: GPL-2.0 2 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 4 //! Memory management subsystems. 5 6 #![expect(dead_code)] 7 8 /// Implements `From` conversions between a frame-number type and `Bounded<u64, N>`. 9 /// 10 /// Each MMU version module should invoke this for the specific bit widths used by that version's 11 /// PTE/PDE bitfield definitions. 12 macro_rules! impl_frame_number_bounded { 13 ($type:ty, $bits:literal) => { 14 impl From<Bounded<u64, $bits>> for $type { 15 fn from(val: Bounded<u64, $bits>) -> Self { 16 Self::new(val.get()) 17 } 18 } 19 20 impl From<$type> for Bounded<u64, $bits> { 21 fn from(v: $type) -> Self { 22 Bounded::from_expr(v.raw() & ::kernel::bits::genmask_u64(0..=($bits - 1))) 23 } 24 } 25 }; 26 } 27 28 /// Implements `From` conversions between [`Pfn`] and `Bounded<u64, N>` for bitfield interop. 29 macro_rules! impl_pfn_bounded { 30 ($bits:literal) => { 31 impl_frame_number_bounded!(Pfn, $bits); 32 }; 33 } 34 35 use core::{ 36 fmt::LowerHex, 37 ops, // 38 }; 39 40 use kernel::{ 41 bitfield, 42 fmt, 43 gpu::buddy::{ 44 GpuBuddy, 45 GpuBuddyParams, // 46 }, 47 num::Bounded, 48 prelude::*, 49 ptr::{ 50 Alignable, 51 Alignment, // 52 }, 53 sizes::SZ_4K, // 54 }; 55 56 use crate::{ 57 driver::Bar0, 58 gpu::Chipset, // 59 }; 60 61 pub(crate) use tlb::Tlb; 62 63 mod hal; 64 pub(super) mod pagetable; 65 mod pramin; 66 mod regs; 67 pub(super) mod tlb; 68 pub(super) mod vmm; 69 70 /// GPU Memory Manager - owns all core MM components. 71 /// 72 /// Provides centralized ownership of memory management resources: 73 /// - [`GpuBuddy`] allocator for VRAM page table allocation. 74 /// - [`pramin::Pramin`] for direct VRAM access. 75 /// - [`Tlb`] manager for translation buffer flush operations. 76 pub(crate) struct GpuMm<'gpu> { 77 buddy: GpuBuddy, 78 pramin: pramin::Pramin<'gpu>, 79 tlb: Pin<KBox<Tlb<'gpu>>>, 80 } 81 82 impl<'gpu> GpuMm<'gpu> { 83 /// Creates the GPU memory manager. 84 pub(crate) fn new( 85 bar: Bar0<'gpu>, 86 chipset: Chipset, 87 buddy_params: GpuBuddyParams, 88 total_fb_end: VramAddress, 89 ) -> Result<Self> { 90 // PRAMIN covers all physical VRAM (including GSP-reserved areas 91 // above the usable region, e.g. the BAR1 page directory). 92 let vram_region = VramAddress::ZERO..total_fb_end; 93 94 Ok(Self { 95 buddy: GpuBuddy::new(buddy_params)?, 96 pramin: pramin::Pramin::new(bar, chipset, vram_region)?, 97 tlb: KBox::pin_init(Tlb::new(bar), GFP_KERNEL)?, 98 }) 99 } 100 101 /// Access the [`GpuBuddy`] allocator. 102 pub(crate) fn buddy(&self) -> &GpuBuddy { 103 &self.buddy 104 } 105 106 /// Access the [`pramin::Pramin`]. 107 fn pramin_mut(&mut self) -> &mut pramin::Pramin<'gpu> { 108 &mut self.pramin 109 } 110 111 /// Access the [`Tlb`] manager. 112 pub(crate) fn tlb(&self) -> &Tlb<'gpu> { 113 self.tlb.as_ref().get_ref() 114 } 115 } 116 117 /// Page size in bytes (4 KiB). 118 pub(crate) const PAGE_SIZE: usize = SZ_4K; 119 120 /// Physical VRAM address in GPU video memory. 121 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] 122 #[repr(transparent)] 123 pub(crate) struct VramAddress(u64); 124 125 impl VramAddress { 126 /// The zero address. 127 pub(crate) const ZERO: Self = Self::from_raw(0); 128 129 /// Creates an address from a raw value. 130 pub(crate) const fn from_raw(addr: u64) -> Self { 131 Self(addr) 132 } 133 134 /// Returns the address as a raw value. 135 pub(crate) const fn into_raw(self) -> u64 { 136 self.0 137 } 138 139 /// Adds `rhs` to this address, returning [`None`] on overflow. 140 pub(crate) const fn checked_add(self, rhs: u64) -> Option<Self> { 141 match self.into_raw().checked_add(rhs) { 142 Some(addr) => Some(Self::from_raw(addr)), 143 None => None, 144 } 145 } 146 } 147 148 impl Alignable for VramAddress { 149 fn align_down(self, alignment: Alignment) -> Self { 150 Self::from_raw(self.into_raw().align_down(alignment)) 151 } 152 153 fn align_up(self, alignment: Alignment) -> Option<Self> { 154 self.into_raw().align_up(alignment).map(Self::from_raw) 155 } 156 } 157 158 impl LowerHex for VramAddress { 159 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 160 LowerHex::fmt(&self.into_raw(), f) 161 } 162 } 163 164 impl fmt::Debug for VramAddress { 165 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 166 f.write_fmt(fmt!("{:#x}", self)) 167 } 168 } 169 170 impl ops::Add<u64> for VramAddress { 171 type Output = Self; 172 173 fn add(self, rhs: u64) -> Self::Output { 174 Self::from_raw(self.into_raw() + rhs) 175 } 176 } 177 178 impl ops::Sub for VramAddress { 179 type Output = u64; 180 181 fn sub(self, rhs: Self) -> Self::Output { 182 self.into_raw() - rhs.into_raw() 183 } 184 } 185 186 impl From<Pfn> for VramAddress { 187 fn from(pfn: Pfn) -> Self { 188 Self::from_raw(pfn.raw() << 12) 189 } 190 } 191 192 bitfield! { 193 /// Virtual address in GPU address space. 194 pub(crate) struct VirtualAddress(u64) { 195 /// Offset within 4KB page. 196 11:0 offset; 197 /// Virtual frame number. 198 63:12 frame_number => Vfn; 199 } 200 } 201 202 impl VirtualAddress { 203 /// Create a new virtual address from a raw value. 204 pub(crate) const fn new(addr: u64) -> Self { 205 Self::from_raw(addr) 206 } 207 } 208 209 impl From<Vfn> for VirtualAddress { 210 fn from(vfn: Vfn) -> Self { 211 Self::zeroed().with_frame_number(vfn) 212 } 213 } 214 215 /// Physical Frame Number. 216 /// 217 /// Represents a physical page in VRAM. 218 #[repr(transparent)] 219 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] 220 pub(crate) struct Pfn(u64); 221 222 impl Pfn { 223 /// Create a new PFN from a frame number. 224 pub(crate) const fn new(frame_number: u64) -> Self { 225 Self(frame_number) 226 } 227 228 /// Get the raw frame number. 229 pub(crate) const fn raw(self) -> u64 { 230 self.0 231 } 232 } 233 234 impl From<VramAddress> for Pfn { 235 fn from(addr: VramAddress) -> Self { 236 Self::new(addr.into_raw() >> 12) 237 } 238 } 239 240 impl From<u64> for Pfn { 241 fn from(val: u64) -> Self { 242 Self(val) 243 } 244 } 245 246 impl From<Pfn> for u64 { 247 fn from(pfn: Pfn) -> Self { 248 pfn.0 249 } 250 } 251 252 impl_pfn_bounded!(52); 253 254 /// Virtual Frame Number. 255 /// 256 /// Represents a virtual page in GPU address space. 257 #[repr(transparent)] 258 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] 259 pub(crate) struct Vfn(u64); 260 261 impl Vfn { 262 /// Create a new VFN from a frame number. 263 pub(crate) const fn new(frame_number: u64) -> Self { 264 Self(frame_number) 265 } 266 267 /// Get the raw frame number. 268 pub(crate) const fn raw(self) -> u64 { 269 self.0 270 } 271 } 272 273 impl From<VirtualAddress> for Vfn { 274 fn from(addr: VirtualAddress) -> Self { 275 addr.frame_number() 276 } 277 } 278 279 impl From<u64> for Vfn { 280 fn from(val: u64) -> Self { 281 Self(val) 282 } 283 } 284 285 impl From<Vfn> for u64 { 286 fn from(vfn: Vfn) -> Self { 287 vfn.0 288 } 289 } 290 291 impl_frame_number_bounded!(Vfn, 52); 292 293 #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] 294 pub(crate) mod selftest { 295 use core::ops::Range; 296 297 use kernel::{ 298 device, 299 sizes::SizeConstants, // 300 }; 301 302 use super::*; 303 304 /// Run MM subsystem self-tests during probe. 305 pub(crate) fn run( 306 dev: &device::Device<device::Bound>, 307 mm: &mut GpuMm<'_>, 308 usable_fb_regions: &[Range<u64>], 309 ) -> Result { 310 // VRAM span the self-tests are free to overwrite, from the chosen test base. 311 const SELFTEST_SPAN: u64 = u64::SZ_64M; 312 313 let base = usable_fb_regions.iter().find_map(|region| { 314 // Tests rely on this being 8 byte aligned for checking misalignment handling. 315 let base = region.start.align_up(Alignment::new::<8>())?; 316 (base.checked_add(SELFTEST_SPAN)? <= region.end).then_some(base) 317 }); 318 let Some(base) = base else { 319 dev_warn!( 320 dev, 321 "PRAMIN: skipping self-tests, no usable VRAM region of {:#x} bytes\n", 322 SELFTEST_SPAN 323 ); 324 return Ok(()); 325 }; 326 327 pramin::selftest::run(dev, mm.pramin_mut(), VramAddress::from_raw(base)) 328 } 329 } 330