1 // SPDX-License-Identifier: GPL-2.0 2 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 4 //! Memory management subsystems. 5 6 #![cfg_attr(not(CONFIG_NOVA_CORE_SELFTESTS), expect(dead_code))] 7 8 /// Implements `From` conversions between a frame-number type and `Bounded<u64, N>`. 9 /// 10 /// Each MMU version module should invoke this for the specific bit widths used by that version's 11 /// PTE/PDE bitfield definitions. 12 macro_rules! impl_frame_number_bounded { 13 ($type:ty, $bits:literal) => { 14 impl From<Bounded<u64, $bits>> for $type { 15 fn from(val: Bounded<u64, $bits>) -> Self { 16 Self::new(val.get()) 17 } 18 } 19 20 impl From<$type> for Bounded<u64, $bits> { 21 fn from(v: $type) -> Self { 22 Bounded::from_expr(v.raw() & ::kernel::bits::genmask_u64(0..=($bits - 1))) 23 } 24 } 25 }; 26 } 27 28 /// Implements `From` conversions between [`Pfn`] and `Bounded<u64, N>` for bitfield interop. 29 macro_rules! impl_pfn_bounded { 30 ($bits:literal) => { 31 impl_frame_number_bounded!(Pfn, $bits); 32 }; 33 } 34 35 use core::{ 36 fmt::LowerHex, 37 ops, // 38 }; 39 40 use kernel::{ 41 bitfield, 42 fmt, 43 gpu::buddy::{ 44 GpuBuddy, 45 GpuBuddyParams, // 46 }, 47 num::Bounded, 48 prelude::*, 49 ptr::{ 50 Alignable, 51 Alignment, // 52 }, 53 sizes::SZ_4K, // 54 }; 55 56 use crate::{ 57 driver::Bar0, 58 gpu::Chipset, // 59 }; 60 61 pub(crate) use tlb::Tlb; 62 63 pub(crate) mod bar_user; 64 mod hal; 65 pub(super) mod pagetable; 66 mod pramin; 67 mod regs; 68 pub(super) mod tlb; 69 pub(super) mod vmm; 70 71 /// GPU Memory Manager - owns all core MM components. 72 /// 73 /// Provides centralized ownership of memory management resources: 74 /// - [`GpuBuddy`] allocator for VRAM page table allocation. 75 /// - [`pramin::Pramin`] for direct VRAM access. 76 /// - [`Tlb`] manager for translation buffer flush operations. 77 pub(crate) struct GpuMm<'gpu> { 78 buddy: GpuBuddy, 79 pramin: pramin::Pramin<'gpu>, 80 tlb: Pin<KBox<Tlb<'gpu>>>, 81 } 82 83 impl<'gpu> GpuMm<'gpu> { 84 /// Creates the GPU memory manager. 85 pub(crate) fn new( 86 bar: Bar0<'gpu>, 87 chipset: Chipset, 88 buddy_params: GpuBuddyParams, 89 total_fb_end: VramAddress, 90 ) -> Result<Self> { 91 // PRAMIN covers all physical VRAM (including GSP-reserved areas 92 // above the usable region, e.g. the BAR1 page directory). 93 let vram_region = VramAddress::ZERO..total_fb_end; 94 95 Ok(Self { 96 buddy: GpuBuddy::new(buddy_params)?, 97 pramin: pramin::Pramin::new(bar, chipset, vram_region)?, 98 tlb: KBox::pin_init(Tlb::new(bar), GFP_KERNEL)?, 99 }) 100 } 101 102 /// Access the [`GpuBuddy`] allocator. 103 pub(crate) fn buddy(&self) -> &GpuBuddy { 104 &self.buddy 105 } 106 107 /// Access the [`pramin::Pramin`]. 108 fn pramin_mut(&mut self) -> &mut pramin::Pramin<'gpu> { 109 &mut self.pramin 110 } 111 112 /// Access the [`Tlb`] manager. 113 pub(crate) fn tlb(&self) -> &Tlb<'gpu> { 114 self.tlb.as_ref().get_ref() 115 } 116 } 117 118 /// Page size in bytes (4 KiB). 119 pub(crate) const PAGE_SIZE: usize = SZ_4K; 120 121 /// Physical VRAM address in GPU video memory. 122 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] 123 #[repr(transparent)] 124 pub(crate) struct VramAddress(u64); 125 126 impl VramAddress { 127 /// The zero address. 128 pub(crate) const ZERO: Self = Self::from_raw(0); 129 130 /// Creates an address from a raw value. 131 pub(crate) const fn from_raw(addr: u64) -> Self { 132 Self(addr) 133 } 134 135 /// Returns the address as a raw value. 136 pub(crate) const fn into_raw(self) -> u64 { 137 self.0 138 } 139 140 /// Adds `rhs` to this address, returning [`None`] on overflow. 141 pub(crate) const fn checked_add(self, rhs: u64) -> Option<Self> { 142 match self.into_raw().checked_add(rhs) { 143 Some(addr) => Some(Self::from_raw(addr)), 144 None => None, 145 } 146 } 147 } 148 149 impl Alignable for VramAddress { 150 fn align_down(self, alignment: Alignment) -> Self { 151 Self::from_raw(self.into_raw().align_down(alignment)) 152 } 153 154 fn align_up(self, alignment: Alignment) -> Option<Self> { 155 self.into_raw().align_up(alignment).map(Self::from_raw) 156 } 157 } 158 159 impl LowerHex for VramAddress { 160 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 161 LowerHex::fmt(&self.into_raw(), f) 162 } 163 } 164 165 impl fmt::Debug for VramAddress { 166 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 167 f.write_fmt(fmt!("{:#x}", self)) 168 } 169 } 170 171 impl ops::Add<u64> for VramAddress { 172 type Output = Self; 173 174 fn add(self, rhs: u64) -> Self::Output { 175 Self::from_raw(self.into_raw() + rhs) 176 } 177 } 178 179 impl ops::Sub for VramAddress { 180 type Output = u64; 181 182 fn sub(self, rhs: Self) -> Self::Output { 183 self.into_raw() - rhs.into_raw() 184 } 185 } 186 187 impl From<Pfn> for VramAddress { 188 fn from(pfn: Pfn) -> Self { 189 Self::from_raw(pfn.raw() << 12) 190 } 191 } 192 193 bitfield! { 194 /// Virtual address in GPU address space. 195 pub(crate) struct VirtualAddress(u64) { 196 /// Offset within 4KB page. 197 11:0 offset; 198 /// Virtual frame number. 199 63:12 frame_number => Vfn; 200 } 201 } 202 203 impl VirtualAddress { 204 /// Create a new virtual address from a raw value. 205 #[expect(dead_code)] 206 pub(crate) const fn new(addr: u64) -> Self { 207 Self::from_raw(addr) 208 } 209 } 210 211 impl From<Vfn> for VirtualAddress { 212 fn from(vfn: Vfn) -> Self { 213 Self::zeroed().with_frame_number(vfn) 214 } 215 } 216 217 /// Physical Frame Number. 218 /// 219 /// Represents a physical page in VRAM. 220 #[repr(transparent)] 221 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] 222 pub(crate) struct Pfn(u64); 223 224 impl Pfn { 225 /// Create a new PFN from a frame number. 226 pub(crate) const fn new(frame_number: u64) -> Self { 227 Self(frame_number) 228 } 229 230 /// Get the raw frame number. 231 pub(crate) const fn raw(self) -> u64 { 232 self.0 233 } 234 } 235 236 impl From<VramAddress> for Pfn { 237 fn from(addr: VramAddress) -> Self { 238 Self::new(addr.into_raw() >> 12) 239 } 240 } 241 242 impl From<u64> for Pfn { 243 fn from(val: u64) -> Self { 244 Self(val) 245 } 246 } 247 248 impl From<Pfn> for u64 { 249 fn from(pfn: Pfn) -> Self { 250 pfn.0 251 } 252 } 253 254 impl_pfn_bounded!(52); 255 256 /// Virtual Frame Number. 257 /// 258 /// Represents a virtual page in GPU address space. 259 #[repr(transparent)] 260 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] 261 pub(crate) struct Vfn(u64); 262 263 impl Vfn { 264 /// Create a new VFN from a frame number. 265 pub(crate) const fn new(frame_number: u64) -> Self { 266 Self(frame_number) 267 } 268 269 /// Get the raw frame number. 270 pub(crate) const fn raw(self) -> u64 { 271 self.0 272 } 273 } 274 275 impl From<VirtualAddress> for Vfn { 276 fn from(addr: VirtualAddress) -> Self { 277 addr.frame_number() 278 } 279 } 280 281 impl From<u64> for Vfn { 282 fn from(val: u64) -> Self { 283 Self(val) 284 } 285 } 286 287 impl From<Vfn> for u64 { 288 fn from(vfn: Vfn) -> Self { 289 vfn.0 290 } 291 } 292 293 impl_frame_number_bounded!(Vfn, 52); 294 295 #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] 296 pub(crate) mod selftest { 297 use core::ops::Range; 298 299 use kernel::{ 300 device, 301 sizes::SizeConstants, // 302 }; 303 304 use super::*; 305 306 /// Run MM subsystem self-tests during probe. 307 pub(crate) fn run( 308 dev: &device::Device<device::Bound>, 309 mm: &mut GpuMm<'_>, 310 usable_fb_regions: &[Range<u64>], 311 bar_user: &bar_user::BarUser<'_>, 312 bar1_pdb: u64, 313 chipset: Chipset, 314 ) -> Result { 315 // VRAM span the self-tests are free to overwrite, from the chosen test base. 316 const SELFTEST_SPAN: u64 = u64::SZ_64M; 317 318 let base = usable_fb_regions.iter().find_map(|region| { 319 // Tests rely on this being 8 byte aligned for checking misalignment handling. 320 let base = region.start.align_up(Alignment::new::<8>())?; 321 (base.checked_add(SELFTEST_SPAN)? <= region.end).then_some(base) 322 }); 323 let Some(base) = base else { 324 dev_warn!( 325 dev, 326 "PRAMIN: skipping self-tests, no usable VRAM region of {:#x} bytes\n", 327 SELFTEST_SPAN 328 ); 329 return Ok(()); 330 }; 331 332 pramin::selftest::run(dev, mm.pramin_mut(), VramAddress::from_raw(base))?; 333 bar_user::run_self_test(dev, mm, bar_user, bar1_pdb, chipset) 334 } 335 } 336