1 // SPDX-License-Identifier: GPL-2.0 2 3 //! BAR1 user interface for CPU access to GPU virtual memory. Used for USERD 4 //! for GPU work submission, and applications to access GPU buffers via mmap(). 5 6 use kernel::{ 7 io::Io, 8 new_mutex, 9 prelude::*, 10 sync::Mutex, // 11 }; 12 13 use crate::{ 14 driver::Bar1, 15 gpu::Chipset, 16 mm::{ 17 vmm::{ 18 MappedRange, 19 Vmm, // 20 }, 21 GpuMm, 22 Pfn, 23 Vfn, 24 VirtualAddress, 25 VramAddress, 26 PAGE_SIZE, // 27 }, 28 num::IntoSafeCast, 29 }; 30 31 #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] 32 use kernel::device; 33 34 /// BAR1 user interface for virtual memory mappings. 35 /// 36 /// Owns the [`Vmm`] for the BAR1 address space. 37 #[pin_data] 38 pub(crate) struct BarUser<'a> { 39 #[pin] 40 vmm: Mutex<Vmm>, 41 bar1: Bar1<'a>, 42 } 43 44 impl<'a> BarUser<'a> { 45 /// Create a pin-initializer for [`BarUser`]. 46 pub(crate) fn new( 47 pdb_addr: VramAddress, 48 chipset: Chipset, 49 va_size: u64, 50 bar1: Bar1<'a>, 51 ) -> impl PinInit<Self, Error> + 'a { 52 try_pin_init!(Self { 53 vmm <- new_mutex!( 54 Vmm::new(pdb_addr, chipset.mmu_version(), va_size)?, 55 "bar_user_vmm", 56 ), 57 bar1, 58 }) 59 } 60 61 /// Map physical pages to a contiguous BAR1 virtual range. 62 pub(crate) fn map( 63 &self, 64 mm: &mut GpuMm<'_>, 65 pfns: &[Pfn], 66 writable: bool, 67 ) -> Result<BarUserAccess<'_>> { 68 if pfns.is_empty() { 69 return Err(EINVAL); 70 } 71 let mut vmm = self.vmm.lock(); 72 let mapped = vmm.map_pages(mm, pfns, None, writable)?; 73 74 Ok(BarUserAccess { 75 bar_user: self, 76 mapped: Some(mapped), 77 }) 78 } 79 } 80 81 /// Access object for a mapped BAR1 region. 82 /// 83 /// Borrows the [`BarUser`] managing the mapping for the lifetime of the access object. 84 pub(crate) struct BarUserAccess<'a> { 85 bar_user: &'a BarUser<'a>, 86 /// [`BarUserAccess::release`] [`Option::take`]s this; `Some` at 87 /// drop time means `release()` was never called. 88 mapped: Option<MappedRange>, 89 } 90 91 #[expect(dead_code)] 92 impl BarUserAccess<'_> { 93 /// Tear down the BAR1 mapping. 94 pub(crate) fn release(mut self, mm: &mut GpuMm<'_>) -> Result { 95 let mapped = self.mapped.take().ok_or(EINVAL)?; 96 let mut vmm = self.bar_user.vmm.lock(); 97 vmm.unmap_pages(mm, mapped)?; 98 Ok(()) 99 } 100 101 /// Returns the active mapping. 102 fn mapped(&self) -> &MappedRange { 103 // `mapped` is only `None` after `take()` in `release`; hence unwrap() 104 // cannot panic here. 105 self.mapped.as_ref().unwrap() 106 } 107 108 /// Get the base virtual address of this mapping. 109 pub(crate) fn base(&self) -> VirtualAddress { 110 VirtualAddress::from(self.mapped().vfn_start) 111 } 112 113 /// Get the total size of the mapped region in bytes. 114 pub(crate) fn size(&self) -> usize { 115 self.mapped().num_pages * PAGE_SIZE 116 } 117 118 /// Get the starting virtual frame number. 119 pub(crate) fn vfn_start(&self) -> Vfn { 120 self.mapped().vfn_start 121 } 122 123 /// Get the number of pages in this mapping. 124 pub(crate) fn num_pages(&self) -> usize { 125 self.mapped().num_pages 126 } 127 128 /// Translate an offset within this mapping to a BAR1 aperture offset. 129 fn bar_offset(&self, offset: usize) -> Result<usize> { 130 if offset >= self.size() { 131 return Err(EINVAL); 132 } 133 134 let base_vfn: usize = self.mapped().vfn_start.raw().into_safe_cast(); 135 let base = base_vfn.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?; 136 base.checked_add(offset).ok_or(EOVERFLOW) 137 } 138 139 // Fallible accessors with runtime bounds checking. 140 141 /// Read a 32-bit value at the given offset. 142 pub(crate) fn try_read32(&self, offset: usize) -> Result<u32> { 143 let off = self.bar_offset(offset)?; 144 self.bar_user.bar1.try_read32(off) 145 } 146 147 /// Write a 32-bit value at the given offset. 148 pub(crate) fn try_write32(&self, value: u32, offset: usize) -> Result { 149 let off = self.bar_offset(offset)?; 150 self.bar_user.bar1.try_write32(value, off) 151 } 152 153 /// Read a 64-bit value at the given offset. 154 pub(crate) fn try_read64(&self, offset: usize) -> Result<u64> { 155 let off = self.bar_offset(offset)?; 156 self.bar_user.bar1.try_read64(off) 157 } 158 159 /// Write a 64-bit value at the given offset. 160 pub(crate) fn try_write64(&self, value: u64, offset: usize) -> Result { 161 let off = self.bar_offset(offset)?; 162 self.bar_user.bar1.try_write64(value, off) 163 } 164 } 165 166 impl Drop for BarUserAccess<'_> { 167 fn drop(&mut self) { 168 if self.mapped.is_some() { 169 kernel::pr_warn!( 170 "BarUserAccess dropped without calling release(). BarUser address space will leak.\n" 171 ); 172 } 173 // The inner `MappedRange`'s own `MustUnmapGuard` will also fire, 174 // identifying the leaked VA range. 175 } 176 } 177 178 /// Run MM subsystem self-tests during probe. 179 /// 180 /// Tests page table infrastructure and `BAR1` MMIO access using the `BAR1` 181 /// address space. Uses the `GpuMm`'s buddy allocator to allocate page tables 182 /// and test pages as needed. 183 #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] 184 pub(crate) fn run_self_test( 185 dev: &device::Device<device::Bound>, 186 mm: &mut GpuMm<'_>, 187 bar_user: &BarUser<'_>, 188 bar1_pdb: u64, 189 chipset: Chipset, 190 ) -> Result { 191 use kernel::{ 192 gpu::buddy::{ 193 GpuBuddyAllocFlags, 194 GpuBuddyAllocMode, // 195 }, 196 ptr::Alignment, 197 sizes::{ 198 SZ_16K, 199 SZ_32K, 200 SZ_4K, 201 SZ_64K, // 202 }, 203 }; 204 205 // Test patterns. 206 const PATTERN_PRAMIN: u32 = 0xDEAD_BEEF; 207 const PATTERN_BAR1: u32 = 0xCAFE_BABE; 208 209 let bar1 = bar_user.bar1; 210 dev_info!(dev, "MM: Starting self-test...\n"); 211 212 let pdb_addr = VramAddress::from_raw(bar1_pdb); 213 214 // Check if initial page tables are in VRAM. 215 if crate::mm::pagetable::check_pdb_valid(mm.pramin_mut(), pdb_addr, chipset).is_err() { 216 dev_info!(dev, "MM: Self-test SKIPPED - no valid VRAM page tables\n"); 217 return Ok(()); 218 } 219 220 // Set up a test page from the buddy allocator. 221 let test_page_blocks = KBox::pin_init( 222 mm.buddy().alloc_blocks( 223 GpuBuddyAllocMode::Simple, 224 SZ_4K.into_safe_cast(), 225 Alignment::new::<SZ_4K>(), 226 GpuBuddyAllocFlags::default(), 227 ), 228 GFP_KERNEL, 229 )?; 230 let test_vram_offset = test_page_blocks.iter().next().ok_or(ENOMEM)?.offset(); 231 let test_vram = VramAddress::from_raw(test_vram_offset); 232 let test_pfn = Pfn::from(test_vram); 233 234 // Create a VMM of size 64K to track virtual memory mappings. 235 let mut vmm = Vmm::new(pdb_addr, chipset.mmu_version(), SZ_64K.into_safe_cast())?; 236 237 // Create a test mapping. 238 let mapped = vmm.map_pages(mm, &[test_pfn], None, true)?; 239 let test_vfn = mapped.vfn_start; 240 241 // Pre-compute test addresses for the PRAMIN to BAR1 read test. 242 let vfn_offset: usize = test_vfn.raw().into_safe_cast(); 243 let bar1_base_offset = vfn_offset.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?; 244 let bar1_read_offset: usize = bar1_base_offset + 0x100; 245 let vram_read_addr = test_vram + 0x100; 246 247 // Test 1: Write via PRAMIN, read via BAR1. 248 mm.pramin_mut() 249 .window_at::<u32>(vram_read_addr)? 250 .view() 251 .write_val(PATTERN_PRAMIN); 252 253 // Read back via BAR1 aperture. 254 let bar1_value = bar1.try_read32(bar1_read_offset)?; 255 256 let test1_passed = if bar1_value == PATTERN_PRAMIN { 257 true 258 } else { 259 dev_err!( 260 dev, 261 "MM: Test 1 FAILED - Expected {:#010x}, got {:#010x}\n", 262 PATTERN_PRAMIN, 263 bar1_value 264 ); 265 false 266 }; 267 268 // Cleanup - invalidate PTE. 269 vmm.unmap_pages(mm, mapped)?; 270 271 // Test 2: Two-phase prepare/execute API. 272 let prepared = vmm.prepare_map(mm, 1, None)?; 273 let mapped2 = vmm.execute_map(mm, prepared, &[test_pfn], true)?; 274 let readback = vmm.read_mapping(mm, mapped2.vfn_start)?; 275 let test2_passed = if readback == Some(test_pfn) { 276 true 277 } else { 278 dev_err!(dev, "MM: Test 2 FAILED - Two-phase map readback mismatch\n"); 279 false 280 }; 281 vmm.unmap_pages(mm, mapped2)?; 282 283 // Test 3: Range-constrained allocation with a hole — exercises block.size()-driven 284 // BAR1 mapping. A 4K hole is punched at base+16K, then a single 32K allocation 285 // is requested within [base, base+36K). The buddy allocator must split around the 286 // hole, returning multiple blocks (expected: {16K, 4K, 8K, 4K} = 32K total). 287 // Each block is mapped into BAR1 and verified via PRAMIN read-back. 288 // 289 // Address layout (base = 0x10000): 290 // [ 16K ] [HOLE 4K] [4K] [ 8K ] [4K] 291 // 0x10000 0x14000 0x15000 0x16000 0x18000 0x19000 292 let range_base: u64 = SZ_64K.into_safe_cast(); 293 let sz_4k: u64 = SZ_4K.into_safe_cast(); 294 let sz_16k: u64 = SZ_16K.into_safe_cast(); 295 let sz_32k_4k: u64 = (SZ_32K + SZ_4K).into_safe_cast(); 296 297 // Punch a 4K hole at base+16K so the subsequent 32K allocation must split. 298 let _hole = KBox::pin_init( 299 mm.buddy().alloc_blocks( 300 GpuBuddyAllocMode::Range(range_base + sz_16k..range_base + sz_16k + sz_4k), 301 SZ_4K.into_safe_cast(), 302 Alignment::new::<SZ_4K>(), 303 GpuBuddyAllocFlags::default(), 304 ), 305 GFP_KERNEL, 306 )?; 307 308 // Allocate 32K within [base, base+36K). The hole forces the allocator to return 309 // split blocks whose sizes are determined by buddy alignment. 310 let blocks = KBox::pin_init( 311 mm.buddy().alloc_blocks( 312 GpuBuddyAllocMode::Range(range_base..range_base + sz_32k_4k), 313 SZ_32K.into_safe_cast(), 314 Alignment::new::<SZ_4K>(), 315 GpuBuddyAllocFlags::default(), 316 ), 317 GFP_KERNEL, 318 )?; 319 320 let mut test3_passed = true; 321 let mut total_size = 0usize; 322 323 for block in blocks.iter() { 324 total_size += IntoSafeCast::<usize>::into_safe_cast(block.size()); 325 326 // Map all pages of this block. 327 let page_size: u64 = PAGE_SIZE.into_safe_cast(); 328 let num_pages: usize = (block.size() / page_size).into_safe_cast(); 329 330 let mut pfns = KVec::new(); 331 for j in 0..num_pages { 332 let j_u64: u64 = j.into_safe_cast(); 333 pfns.push( 334 Pfn::from(VramAddress::from_raw( 335 block.offset() + j_u64.checked_mul(page_size).ok_or(EOVERFLOW)?, 336 )), 337 GFP_KERNEL, 338 )?; 339 } 340 341 let mapped = vmm.map_pages(mm, &pfns, None, true)?; 342 let bar1_base_vfn: usize = mapped.vfn_start.raw().into_safe_cast(); 343 let bar1_base = bar1_base_vfn.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?; 344 345 for j in 0..num_pages { 346 let page_bar1_off = bar1_base + j * PAGE_SIZE; 347 let j_u64: u64 = j.into_safe_cast(); 348 let page_phys = block.offset() 349 + j_u64 350 .checked_mul(PAGE_SIZE.into_safe_cast()) 351 .ok_or(EOVERFLOW)?; 352 353 bar1.try_write32(PATTERN_BAR1, page_bar1_off)?; 354 355 let pramin_val = mm 356 .pramin_mut() 357 .window_at::<u32>(VramAddress::from_raw(page_phys))? 358 .view() 359 .read_val(); 360 361 if pramin_val != PATTERN_BAR1 { 362 dev_err!( 363 dev, 364 "MM: Test 3 FAILED block offset {:#x} page {} (val={:#x})\n", 365 block.offset(), 366 j, 367 pramin_val 368 ); 369 test3_passed = false; 370 } 371 } 372 373 vmm.unmap_pages(mm, mapped)?; 374 } 375 376 // Verify aggregate: all returned block sizes must sum to allocation size. 377 if total_size != SZ_32K { 378 dev_err!( 379 dev, 380 "MM: Test 3 FAILED - total size {} != expected {}\n", 381 total_size, 382 SZ_32K 383 ); 384 test3_passed = false; 385 } 386 387 // Release Tests 1-3's Vmm before Test 4 uses the GPU's BarUser on the same PDB. 388 drop(vmm); 389 390 // Test 4: Exercise `BarUser::map()` end-to-end. 391 let access = bar_user.map(mm, &[test_pfn], true)?; 392 393 // Write pattern via PRAMIN, read via BarUserAccess. 394 mm.pramin_mut() 395 .window_at::<u32>(test_vram)? 396 .view() 397 .write_val(PATTERN_BAR1); 398 399 let readback = access.try_read32(0)?; 400 let test4_passed = if readback == PATTERN_BAR1 { 401 true 402 } else { 403 dev_err!( 404 dev, 405 "MM: Test 4 FAILED - Expected {:#010x}, got {:#010x}\n", 406 PATTERN_BAR1, 407 readback 408 ); 409 false 410 }; 411 access.release(mm)?; 412 413 if test1_passed && test2_passed && test3_passed && test4_passed { 414 dev_info!(dev, "MM: All self-tests PASSED\n"); 415 Ok(()) 416 } else { 417 dev_err!(dev, "MM: Self-tests FAILED\n"); 418 Err(EIO) 419 } 420 } 421