xref: /linux/drivers/gpu/nova-core/mm/bar_user.rs (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 // SPDX-License-Identifier: GPL-2.0
2 
3 //! BAR1 user interface for CPU access to GPU virtual memory. Used for USERD
4 //! for GPU work submission, and applications to access GPU buffers via mmap().
5 
6 use kernel::{
7     io::Io,
8     new_mutex,
9     prelude::*,
10     sync::Mutex, //
11 };
12 
13 use crate::{
14     driver::Bar1,
15     gpu::Chipset,
16     mm::{
17         vmm::{
18             MappedRange,
19             Vmm, //
20         },
21         GpuMm,
22         Pfn,
23         Vfn,
24         VirtualAddress,
25         VramAddress,
26         PAGE_SIZE, //
27     },
28     num::IntoSafeCast,
29 };
30 
31 #[cfg(CONFIG_NOVA_CORE_SELFTESTS)]
32 use kernel::device;
33 
34 /// BAR1 user interface for virtual memory mappings.
35 ///
36 /// Owns the [`Vmm`] for the BAR1 address space.
37 #[pin_data]
38 pub(crate) struct BarUser<'a> {
39     #[pin]
40     vmm: Mutex<Vmm>,
41     bar1: Bar1<'a>,
42 }
43 
44 impl<'a> BarUser<'a> {
45     /// Create a pin-initializer for [`BarUser`].
46     pub(crate) fn new(
47         pdb_addr: VramAddress,
48         chipset: Chipset,
49         va_size: u64,
50         bar1: Bar1<'a>,
51     ) -> impl PinInit<Self, Error> + 'a {
52         try_pin_init!(Self {
53             vmm <- new_mutex!(
54                 Vmm::new(pdb_addr, chipset.mmu_version(), va_size)?,
55                 "bar_user_vmm",
56             ),
57             bar1,
58         })
59     }
60 
61     /// Map physical pages to a contiguous BAR1 virtual range.
62     pub(crate) fn map(
63         &self,
64         mm: &mut GpuMm<'_>,
65         pfns: &[Pfn],
66         writable: bool,
67     ) -> Result<BarUserAccess<'_>> {
68         if pfns.is_empty() {
69             return Err(EINVAL);
70         }
71         let mut vmm = self.vmm.lock();
72         let mapped = vmm.map_pages(mm, pfns, None, writable)?;
73 
74         Ok(BarUserAccess {
75             bar_user: self,
76             mapped: Some(mapped),
77         })
78     }
79 }
80 
81 /// Access object for a mapped BAR1 region.
82 ///
83 /// Borrows the [`BarUser`] managing the mapping for the lifetime of the access object.
84 pub(crate) struct BarUserAccess<'a> {
85     bar_user: &'a BarUser<'a>,
86     /// [`BarUserAccess::release`] [`Option::take`]s this; `Some` at
87     /// drop time means `release()` was never called.
88     mapped: Option<MappedRange>,
89 }
90 
91 #[expect(dead_code)]
92 impl BarUserAccess<'_> {
93     /// Tear down the BAR1 mapping.
94     pub(crate) fn release(mut self, mm: &mut GpuMm<'_>) -> Result {
95         let mapped = self.mapped.take().ok_or(EINVAL)?;
96         let mut vmm = self.bar_user.vmm.lock();
97         vmm.unmap_pages(mm, mapped)?;
98         Ok(())
99     }
100 
101     /// Returns the active mapping.
102     fn mapped(&self) -> &MappedRange {
103         // `mapped` is only `None` after `take()` in `release`; hence unwrap()
104         // cannot panic here.
105         self.mapped.as_ref().unwrap()
106     }
107 
108     /// Get the base virtual address of this mapping.
109     pub(crate) fn base(&self) -> VirtualAddress {
110         VirtualAddress::from(self.mapped().vfn_start)
111     }
112 
113     /// Get the total size of the mapped region in bytes.
114     pub(crate) fn size(&self) -> usize {
115         self.mapped().num_pages * PAGE_SIZE
116     }
117 
118     /// Get the starting virtual frame number.
119     pub(crate) fn vfn_start(&self) -> Vfn {
120         self.mapped().vfn_start
121     }
122 
123     /// Get the number of pages in this mapping.
124     pub(crate) fn num_pages(&self) -> usize {
125         self.mapped().num_pages
126     }
127 
128     /// Translate an offset within this mapping to a BAR1 aperture offset.
129     fn bar_offset(&self, offset: usize) -> Result<usize> {
130         if offset >= self.size() {
131             return Err(EINVAL);
132         }
133 
134         let base_vfn: usize = self.mapped().vfn_start.raw().into_safe_cast();
135         let base = base_vfn.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?;
136         base.checked_add(offset).ok_or(EOVERFLOW)
137     }
138 
139     // Fallible accessors with runtime bounds checking.
140 
141     /// Read a 32-bit value at the given offset.
142     pub(crate) fn try_read32(&self, offset: usize) -> Result<u32> {
143         let off = self.bar_offset(offset)?;
144         self.bar_user.bar1.try_read32(off)
145     }
146 
147     /// Write a 32-bit value at the given offset.
148     pub(crate) fn try_write32(&self, value: u32, offset: usize) -> Result {
149         let off = self.bar_offset(offset)?;
150         self.bar_user.bar1.try_write32(value, off)
151     }
152 
153     /// Read a 64-bit value at the given offset.
154     pub(crate) fn try_read64(&self, offset: usize) -> Result<u64> {
155         let off = self.bar_offset(offset)?;
156         self.bar_user.bar1.try_read64(off)
157     }
158 
159     /// Write a 64-bit value at the given offset.
160     pub(crate) fn try_write64(&self, value: u64, offset: usize) -> Result {
161         let off = self.bar_offset(offset)?;
162         self.bar_user.bar1.try_write64(value, off)
163     }
164 }
165 
166 impl Drop for BarUserAccess<'_> {
167     fn drop(&mut self) {
168         if self.mapped.is_some() {
169             kernel::pr_warn!(
170                 "BarUserAccess dropped without calling release(). BarUser address space will leak.\n"
171             );
172         }
173         // The inner `MappedRange`'s own `MustUnmapGuard` will also fire,
174         // identifying the leaked VA range.
175     }
176 }
177 
178 /// Run MM subsystem self-tests during probe.
179 ///
180 /// Tests page table infrastructure and `BAR1` MMIO access using the `BAR1`
181 /// address space. Uses the `GpuMm`'s buddy allocator to allocate page tables
182 /// and test pages as needed.
183 #[cfg(CONFIG_NOVA_CORE_SELFTESTS)]
184 pub(crate) fn run_self_test(
185     dev: &device::Device<device::Bound>,
186     mm: &mut GpuMm<'_>,
187     bar_user: &BarUser<'_>,
188     bar1_pdb: u64,
189     chipset: Chipset,
190 ) -> Result {
191     use kernel::{
192         gpu::buddy::{
193             GpuBuddyAllocFlags,
194             GpuBuddyAllocMode, //
195         },
196         ptr::Alignment,
197         sizes::{
198             SZ_16K,
199             SZ_32K,
200             SZ_4K,
201             SZ_64K, //
202         },
203     };
204 
205     // Test patterns.
206     const PATTERN_PRAMIN: u32 = 0xDEAD_BEEF;
207     const PATTERN_BAR1: u32 = 0xCAFE_BABE;
208 
209     let bar1 = bar_user.bar1;
210     dev_info!(dev, "MM: Starting self-test...\n");
211 
212     let pdb_addr = VramAddress::from_raw(bar1_pdb);
213 
214     // Check if initial page tables are in VRAM.
215     if crate::mm::pagetable::check_pdb_valid(mm.pramin_mut(), pdb_addr, chipset).is_err() {
216         dev_info!(dev, "MM: Self-test SKIPPED - no valid VRAM page tables\n");
217         return Ok(());
218     }
219 
220     // Set up a test page from the buddy allocator.
221     let test_page_blocks = KBox::pin_init(
222         mm.buddy().alloc_blocks(
223             GpuBuddyAllocMode::Simple,
224             SZ_4K.into_safe_cast(),
225             Alignment::new::<SZ_4K>(),
226             GpuBuddyAllocFlags::default(),
227         ),
228         GFP_KERNEL,
229     )?;
230     let test_vram_offset = test_page_blocks.iter().next().ok_or(ENOMEM)?.offset();
231     let test_vram = VramAddress::from_raw(test_vram_offset);
232     let test_pfn = Pfn::from(test_vram);
233 
234     // Create a VMM of size 64K to track virtual memory mappings.
235     let mut vmm = Vmm::new(pdb_addr, chipset.mmu_version(), SZ_64K.into_safe_cast())?;
236 
237     // Create a test mapping.
238     let mapped = vmm.map_pages(mm, &[test_pfn], None, true)?;
239     let test_vfn = mapped.vfn_start;
240 
241     // Pre-compute test addresses for the PRAMIN to BAR1 read test.
242     let vfn_offset: usize = test_vfn.raw().into_safe_cast();
243     let bar1_base_offset = vfn_offset.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?;
244     let bar1_read_offset: usize = bar1_base_offset + 0x100;
245     let vram_read_addr = test_vram + 0x100;
246 
247     // Test 1: Write via PRAMIN, read via BAR1.
248     mm.pramin_mut()
249         .window_at::<u32>(vram_read_addr)?
250         .view()
251         .write_val(PATTERN_PRAMIN);
252 
253     // Read back via BAR1 aperture.
254     let bar1_value = bar1.try_read32(bar1_read_offset)?;
255 
256     let test1_passed = if bar1_value == PATTERN_PRAMIN {
257         true
258     } else {
259         dev_err!(
260             dev,
261             "MM: Test 1 FAILED - Expected {:#010x}, got {:#010x}\n",
262             PATTERN_PRAMIN,
263             bar1_value
264         );
265         false
266     };
267 
268     // Cleanup - invalidate PTE.
269     vmm.unmap_pages(mm, mapped)?;
270 
271     // Test 2: Two-phase prepare/execute API.
272     let prepared = vmm.prepare_map(mm, 1, None)?;
273     let mapped2 = vmm.execute_map(mm, prepared, &[test_pfn], true)?;
274     let readback = vmm.read_mapping(mm, mapped2.vfn_start)?;
275     let test2_passed = if readback == Some(test_pfn) {
276         true
277     } else {
278         dev_err!(dev, "MM: Test 2 FAILED - Two-phase map readback mismatch\n");
279         false
280     };
281     vmm.unmap_pages(mm, mapped2)?;
282 
283     // Test 3: Range-constrained allocation with a hole — exercises block.size()-driven
284     // BAR1 mapping. A 4K hole is punched at base+16K, then a single 32K allocation
285     // is requested within [base, base+36K). The buddy allocator must split around the
286     // hole, returning multiple blocks (expected: {16K, 4K, 8K, 4K} = 32K total).
287     // Each block is mapped into BAR1 and verified via PRAMIN read-back.
288     //
289     // Address layout (base = 0x10000):
290     //   [    16K    ] [HOLE 4K] [4K] [ 8K ] [4K]
291     //   0x10000       0x14000  0x15000 0x16000 0x18000 0x19000
292     let range_base: u64 = SZ_64K.into_safe_cast();
293     let sz_4k: u64 = SZ_4K.into_safe_cast();
294     let sz_16k: u64 = SZ_16K.into_safe_cast();
295     let sz_32k_4k: u64 = (SZ_32K + SZ_4K).into_safe_cast();
296 
297     // Punch a 4K hole at base+16K so the subsequent 32K allocation must split.
298     let _hole = KBox::pin_init(
299         mm.buddy().alloc_blocks(
300             GpuBuddyAllocMode::Range(range_base + sz_16k..range_base + sz_16k + sz_4k),
301             SZ_4K.into_safe_cast(),
302             Alignment::new::<SZ_4K>(),
303             GpuBuddyAllocFlags::default(),
304         ),
305         GFP_KERNEL,
306     )?;
307 
308     // Allocate 32K within [base, base+36K). The hole forces the allocator to return
309     // split blocks whose sizes are determined by buddy alignment.
310     let blocks = KBox::pin_init(
311         mm.buddy().alloc_blocks(
312             GpuBuddyAllocMode::Range(range_base..range_base + sz_32k_4k),
313             SZ_32K.into_safe_cast(),
314             Alignment::new::<SZ_4K>(),
315             GpuBuddyAllocFlags::default(),
316         ),
317         GFP_KERNEL,
318     )?;
319 
320     let mut test3_passed = true;
321     let mut total_size = 0usize;
322 
323     for block in blocks.iter() {
324         total_size += IntoSafeCast::<usize>::into_safe_cast(block.size());
325 
326         // Map all pages of this block.
327         let page_size: u64 = PAGE_SIZE.into_safe_cast();
328         let num_pages: usize = (block.size() / page_size).into_safe_cast();
329 
330         let mut pfns = KVec::new();
331         for j in 0..num_pages {
332             let j_u64: u64 = j.into_safe_cast();
333             pfns.push(
334                 Pfn::from(VramAddress::from_raw(
335                     block.offset() + j_u64.checked_mul(page_size).ok_or(EOVERFLOW)?,
336                 )),
337                 GFP_KERNEL,
338             )?;
339         }
340 
341         let mapped = vmm.map_pages(mm, &pfns, None, true)?;
342         let bar1_base_vfn: usize = mapped.vfn_start.raw().into_safe_cast();
343         let bar1_base = bar1_base_vfn.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?;
344 
345         for j in 0..num_pages {
346             let page_bar1_off = bar1_base + j * PAGE_SIZE;
347             let j_u64: u64 = j.into_safe_cast();
348             let page_phys = block.offset()
349                 + j_u64
350                     .checked_mul(PAGE_SIZE.into_safe_cast())
351                     .ok_or(EOVERFLOW)?;
352 
353             bar1.try_write32(PATTERN_BAR1, page_bar1_off)?;
354 
355             let pramin_val = mm
356                 .pramin_mut()
357                 .window_at::<u32>(VramAddress::from_raw(page_phys))?
358                 .view()
359                 .read_val();
360 
361             if pramin_val != PATTERN_BAR1 {
362                 dev_err!(
363                     dev,
364                     "MM: Test 3 FAILED block offset {:#x} page {} (val={:#x})\n",
365                     block.offset(),
366                     j,
367                     pramin_val
368                 );
369                 test3_passed = false;
370             }
371         }
372 
373         vmm.unmap_pages(mm, mapped)?;
374     }
375 
376     // Verify aggregate: all returned block sizes must sum to allocation size.
377     if total_size != SZ_32K {
378         dev_err!(
379             dev,
380             "MM: Test 3 FAILED - total size {} != expected {}\n",
381             total_size,
382             SZ_32K
383         );
384         test3_passed = false;
385     }
386 
387     // Release Tests 1-3's Vmm before Test 4 uses the GPU's BarUser on the same PDB.
388     drop(vmm);
389 
390     // Test 4: Exercise `BarUser::map()` end-to-end.
391     let access = bar_user.map(mm, &[test_pfn], true)?;
392 
393     // Write pattern via PRAMIN, read via BarUserAccess.
394     mm.pramin_mut()
395         .window_at::<u32>(test_vram)?
396         .view()
397         .write_val(PATTERN_BAR1);
398 
399     let readback = access.try_read32(0)?;
400     let test4_passed = if readback == PATTERN_BAR1 {
401         true
402     } else {
403         dev_err!(
404             dev,
405             "MM: Test 4 FAILED - Expected {:#010x}, got {:#010x}\n",
406             PATTERN_BAR1,
407             readback
408         );
409         false
410     };
411     access.release(mm)?;
412 
413     if test1_passed && test2_passed && test3_passed && test4_passed {
414         dev_info!(dev, "MM: All self-tests PASSED\n");
415         Ok(())
416     } else {
417         dev_err!(dev, "MM: Self-tests FAILED\n");
418         Err(EIO)
419     }
420 }
421