1 // SPDX-License-Identifier: GPL-2.0 2 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 4 //! Memory management subsystems. 5 6 #![expect(dead_code)] 7 8 /// Implements `From` conversions between a frame-number type and `Bounded<u64, N>`. 9 /// 10 /// Each MMU version module should invoke this for the specific bit widths used by that version's 11 /// PTE/PDE bitfield definitions. 12 macro_rules! impl_frame_number_bounded { 13 ($type:ty, $bits:literal) => { 14 impl From<Bounded<u64, $bits>> for $type { 15 fn from(val: Bounded<u64, $bits>) -> Self { 16 Self::new(val.get()) 17 } 18 } 19 20 impl From<$type> for Bounded<u64, $bits> { 21 fn from(v: $type) -> Self { 22 Bounded::from_expr(v.raw() & ::kernel::bits::genmask_u64(0..=($bits - 1))) 23 } 24 } 25 }; 26 } 27 28 /// Implements `From` conversions between [`Pfn`] and `Bounded<u64, N>` for bitfield interop. 29 macro_rules! impl_pfn_bounded { 30 ($bits:literal) => { 31 impl_frame_number_bounded!(Pfn, $bits); 32 }; 33 } 34 35 use core::{ 36 fmt::LowerHex, 37 ops, // 38 }; 39 40 use kernel::{ 41 bitfield, 42 fmt, 43 gpu::buddy::{ 44 GpuBuddy, 45 GpuBuddyParams, // 46 }, 47 num::Bounded, 48 prelude::*, 49 ptr::{ 50 Alignable, 51 Alignment, // 52 }, 53 sizes::SZ_4K, // 54 }; 55 56 use crate::{ 57 driver::Bar0, 58 gpu::Chipset, // 59 }; 60 61 pub(crate) use tlb::Tlb; 62 63 pub(crate) mod bar_user; 64 mod hal; 65 pub(super) mod pagetable; 66 mod pramin; 67 mod regs; 68 pub(super) mod tlb; 69 pub(super) mod vmm; 70 71 /// GPU Memory Manager - owns all core MM components. 72 /// 73 /// Provides centralized ownership of memory management resources: 74 /// - [`GpuBuddy`] allocator for VRAM page table allocation. 75 /// - [`pramin::Pramin`] for direct VRAM access. 76 /// - [`Tlb`] manager for translation buffer flush operations. 77 pub(crate) struct GpuMm<'gpu> { 78 buddy: GpuBuddy, 79 pramin: pramin::Pramin<'gpu>, 80 tlb: Pin<KBox<Tlb<'gpu>>>, 81 } 82 83 impl<'gpu> GpuMm<'gpu> { 84 /// Creates the GPU memory manager. 85 pub(crate) fn new( 86 bar: Bar0<'gpu>, 87 chipset: Chipset, 88 buddy_params: GpuBuddyParams, 89 total_fb_end: VramAddress, 90 ) -> Result<Self> { 91 // PRAMIN covers all physical VRAM (including GSP-reserved areas 92 // above the usable region, e.g. the BAR1 page directory). 93 let vram_region = VramAddress::ZERO..total_fb_end; 94 95 Ok(Self { 96 buddy: GpuBuddy::new(buddy_params)?, 97 pramin: pramin::Pramin::new(bar, chipset, vram_region)?, 98 tlb: KBox::pin_init(Tlb::new(bar), GFP_KERNEL)?, 99 }) 100 } 101 102 /// Access the [`GpuBuddy`] allocator. 103 pub(crate) fn buddy(&self) -> &GpuBuddy { 104 &self.buddy 105 } 106 107 /// Access the [`pramin::Pramin`]. 108 fn pramin_mut(&mut self) -> &mut pramin::Pramin<'gpu> { 109 &mut self.pramin 110 } 111 112 /// Access the [`Tlb`] manager. 113 pub(crate) fn tlb(&self) -> &Tlb<'gpu> { 114 self.tlb.as_ref().get_ref() 115 } 116 } 117 118 /// Page size in bytes (4 KiB). 119 pub(crate) const PAGE_SIZE: usize = SZ_4K; 120 121 /// Physical VRAM address in GPU video memory. 122 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] 123 #[repr(transparent)] 124 pub(crate) struct VramAddress(u64); 125 126 impl VramAddress { 127 /// The zero address. 128 pub(crate) const ZERO: Self = Self::from_raw(0); 129 130 /// Creates an address from a raw value. 131 pub(crate) const fn from_raw(addr: u64) -> Self { 132 Self(addr) 133 } 134 135 /// Returns the address as a raw value. 136 pub(crate) const fn into_raw(self) -> u64 { 137 self.0 138 } 139 140 /// Adds `rhs` to this address, returning [`None`] on overflow. 141 pub(crate) const fn checked_add(self, rhs: u64) -> Option<Self> { 142 match self.into_raw().checked_add(rhs) { 143 Some(addr) => Some(Self::from_raw(addr)), 144 None => None, 145 } 146 } 147 } 148 149 impl Alignable for VramAddress { 150 fn align_down(self, alignment: Alignment) -> Self { 151 Self::from_raw(self.into_raw().align_down(alignment)) 152 } 153 154 fn align_up(self, alignment: Alignment) -> Option<Self> { 155 self.into_raw().align_up(alignment).map(Self::from_raw) 156 } 157 } 158 159 impl LowerHex for VramAddress { 160 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 161 LowerHex::fmt(&self.into_raw(), f) 162 } 163 } 164 165 impl fmt::Debug for VramAddress { 166 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 167 f.write_fmt(fmt!("{:#x}", self)) 168 } 169 } 170 171 impl ops::Add<u64> for VramAddress { 172 type Output = Self; 173 174 fn add(self, rhs: u64) -> Self::Output { 175 Self::from_raw(self.into_raw() + rhs) 176 } 177 } 178 179 impl ops::Sub for VramAddress { 180 type Output = u64; 181 182 fn sub(self, rhs: Self) -> Self::Output { 183 self.into_raw() - rhs.into_raw() 184 } 185 } 186 187 impl From<Pfn> for VramAddress { 188 fn from(pfn: Pfn) -> Self { 189 Self::from_raw(pfn.raw() << 12) 190 } 191 } 192 193 bitfield! { 194 /// Virtual address in GPU address space. 195 pub(crate) struct VirtualAddress(u64) { 196 /// Offset within 4KB page. 197 11:0 offset; 198 /// Virtual frame number. 199 63:12 frame_number => Vfn; 200 } 201 } 202 203 impl VirtualAddress { 204 /// Create a new virtual address from a raw value. 205 pub(crate) const fn new(addr: u64) -> Self { 206 Self::from_raw(addr) 207 } 208 } 209 210 impl From<Vfn> for VirtualAddress { 211 fn from(vfn: Vfn) -> Self { 212 Self::zeroed().with_frame_number(vfn) 213 } 214 } 215 216 /// Physical Frame Number. 217 /// 218 /// Represents a physical page in VRAM. 219 #[repr(transparent)] 220 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] 221 pub(crate) struct Pfn(u64); 222 223 impl Pfn { 224 /// Create a new PFN from a frame number. 225 pub(crate) const fn new(frame_number: u64) -> Self { 226 Self(frame_number) 227 } 228 229 /// Get the raw frame number. 230 pub(crate) const fn raw(self) -> u64 { 231 self.0 232 } 233 } 234 235 impl From<VramAddress> for Pfn { 236 fn from(addr: VramAddress) -> Self { 237 Self::new(addr.into_raw() >> 12) 238 } 239 } 240 241 impl From<u64> for Pfn { 242 fn from(val: u64) -> Self { 243 Self(val) 244 } 245 } 246 247 impl From<Pfn> for u64 { 248 fn from(pfn: Pfn) -> Self { 249 pfn.0 250 } 251 } 252 253 impl_pfn_bounded!(52); 254 255 /// Virtual Frame Number. 256 /// 257 /// Represents a virtual page in GPU address space. 258 #[repr(transparent)] 259 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] 260 pub(crate) struct Vfn(u64); 261 262 impl Vfn { 263 /// Create a new VFN from a frame number. 264 pub(crate) const fn new(frame_number: u64) -> Self { 265 Self(frame_number) 266 } 267 268 /// Get the raw frame number. 269 pub(crate) const fn raw(self) -> u64 { 270 self.0 271 } 272 } 273 274 impl From<VirtualAddress> for Vfn { 275 fn from(addr: VirtualAddress) -> Self { 276 addr.frame_number() 277 } 278 } 279 280 impl From<u64> for Vfn { 281 fn from(val: u64) -> Self { 282 Self(val) 283 } 284 } 285 286 impl From<Vfn> for u64 { 287 fn from(vfn: Vfn) -> Self { 288 vfn.0 289 } 290 } 291 292 impl_frame_number_bounded!(Vfn, 52); 293 294 #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] 295 pub(crate) mod selftest { 296 use core::ops::Range; 297 298 use kernel::{ 299 device, 300 sizes::SizeConstants, // 301 }; 302 303 use super::*; 304 305 /// Run MM subsystem self-tests during probe. 306 pub(crate) fn run( 307 dev: &device::Device<device::Bound>, 308 mm: &mut GpuMm<'_>, 309 usable_fb_regions: &[Range<u64>], 310 ) -> Result { 311 // VRAM span the self-tests are free to overwrite, from the chosen test base. 312 const SELFTEST_SPAN: u64 = u64::SZ_64M; 313 314 let base = usable_fb_regions.iter().find_map(|region| { 315 // Tests rely on this being 8 byte aligned for checking misalignment handling. 316 let base = region.start.align_up(Alignment::new::<8>())?; 317 (base.checked_add(SELFTEST_SPAN)? <= region.end).then_some(base) 318 }); 319 let Some(base) = base else { 320 dev_warn!( 321 dev, 322 "PRAMIN: skipping self-tests, no usable VRAM region of {:#x} bytes\n", 323 SELFTEST_SPAN 324 ); 325 return Ok(()); 326 }; 327 328 pramin::selftest::run(dev, mm.pramin_mut(), VramAddress::from_raw(base)) 329 } 330 } 331