xref: /linux/rust/kernel/scatterlist.rs (revision 486b257357c23743d9729ea715136d105fff461e)
1 // SPDX-License-Identifier: GPL-2.0
2 
3 //! Abstractions for scatter-gather lists.
4 //!
5 //! C header: [`include/linux/scatterlist.h`](srctree/include/linux/scatterlist.h)
6 //!
7 //! Scatter-gather (SG) I/O is a memory access technique that allows devices to perform DMA
8 //! operations on data buffers that are not physically contiguous in memory. It works by creating a
9 //! "scatter-gather list", an array where each entry specifies the address and length of a
10 //! physically contiguous memory segment.
11 //!
12 //! The device's DMA controller can then read this list and process the segments sequentially as
13 //! part of one logical I/O request. This avoids the need for a single, large, physically contiguous
14 //! memory buffer, which can be difficult or impossible to allocate.
15 //!
16 //! This module provides safe Rust abstractions over the kernel's `struct scatterlist` and
17 //! `struct sg_table` types.
18 //!
19 //! The main entry point is the [`SGTable`] type, which represents a complete scatter-gather table.
20 //! It can be either:
21 //!
22 //! - An owned table ([`SGTable<Owned<P>>`]), created from a Rust memory buffer (e.g., [`VVec`]).
23 //!   This type manages the allocation of the `struct sg_table`, the DMA mapping of the buffer, and
24 //!   the automatic cleanup of all resources.
25 //! - A borrowed reference (&[`SGTable`]), which provides safe, read-only access to a table that was
26 //!   allocated by other (e.g., C) code.
27 //!
28 //! Individual entries in the table are represented by [`SGEntry`], which can be accessed by
29 //! iterating over an [`SGTable`].
30 
31 use crate::{
32     alloc,
33     alloc::allocator::VmallocPageIter,
34     bindings,
35     device::{Bound, Device},
36     devres::Devres,
37     dma,
38     error,
39     page,
40     prelude::*,
41     sync::aref::ARef,
42     types::Opaque, //
43 };
44 use core::{ops::Deref, ptr::NonNull};
45 
46 /// A single entry in a scatter-gather list.
47 ///
48 /// An `SGEntry` represents a single, physically contiguous segment of memory that has been mapped
49 /// for DMA.
50 ///
51 /// Instances of this struct are obtained by iterating over an [`SGTable`]. Drivers do not create
52 /// or own [`SGEntry`] objects directly.
53 #[repr(transparent)]
54 pub struct SGEntry(Opaque<bindings::scatterlist>);
55 
56 // SAFETY: `SGEntry` can be sent to any task.
57 unsafe impl Send for SGEntry {}
58 
59 // SAFETY: `SGEntry` has no interior mutability and can be accessed concurrently.
60 unsafe impl Sync for SGEntry {}
61 
62 impl SGEntry {
63     /// Convert a raw `struct scatterlist *` to a `&'a SGEntry`.
64     ///
65     /// # Safety
66     ///
67     /// Callers must ensure that the `struct scatterlist` pointed to by `ptr` is valid for the
68     /// lifetime `'a`.
69     #[inline]
70     unsafe fn from_raw<'a>(ptr: *mut bindings::scatterlist) -> &'a Self {
71         // SAFETY: The safety requirements of this function guarantee that `ptr` is a valid pointer
72         // to a `struct scatterlist` for the duration of `'a`.
73         unsafe { &*ptr.cast() }
74     }
75 
76     /// Obtain the raw `struct scatterlist *`.
77     #[inline]
78     fn as_raw(&self) -> *mut bindings::scatterlist {
79         self.0.get()
80     }
81 
82     /// Returns the DMA address of this SG entry.
83     ///
84     /// This is the address that the device should use to access the memory segment.
85     #[inline]
86     pub fn dma_address(&self) -> dma::DmaAddress {
87         // SAFETY: `self.as_raw()` is a valid pointer to a `struct scatterlist`.
88         unsafe { bindings::sg_dma_address(self.as_raw()) }
89     }
90 
91     /// Returns the length of this SG entry in bytes.
92     #[inline]
93     pub fn dma_len(&self) -> u32 {
94         // SAFETY: `self.as_raw()` is a valid pointer to a `struct scatterlist`.
95         unsafe { bindings::sg_dma_len(self.as_raw()) }
96     }
97 }
98 
99 /// The borrowed generic type of an [`SGTable`], representing a borrowed or externally managed
100 /// table.
101 #[repr(transparent)]
102 pub struct Borrowed(Opaque<bindings::sg_table>);
103 
104 // SAFETY: `Borrowed` can be sent to any task.
105 unsafe impl Send for Borrowed {}
106 
107 // SAFETY: `Borrowed` has no interior mutability and can be accessed concurrently.
108 unsafe impl Sync for Borrowed {}
109 
110 /// A scatter-gather table.
111 ///
112 /// This struct is a wrapper around the kernel's `struct sg_table`. It manages a list of DMA-mapped
113 /// memory segments that can be passed to a device for I/O operations.
114 ///
115 /// The generic parameter `T` is used as a generic type to distinguish between owned and borrowed
116 /// tables.
117 ///
118 ///  - [`SGTable<Owned>`]: An owned table created and managed entirely by Rust code. It handles
119 ///    allocation, DMA mapping, and cleanup of all associated resources. See [`SGTable::new`].
120 ///  - [`SGTable<Borrowed>`} (or simply [`SGTable`]): Represents a table whose lifetime is managed
121 ///    externally. It can be used safely via a borrowed reference `&'a SGTable`, where `'a` is the
122 ///    external lifetime.
123 ///
124 /// All [`SGTable`] variants can be iterated over the individual [`SGEntry`]s.
125 #[repr(transparent)]
126 #[pin_data]
127 pub struct SGTable<T: private::Sealed = Borrowed> {
128     #[pin]
129     inner: T,
130 }
131 
132 impl SGTable {
133     /// Creates a borrowed `&'a SGTable` from a raw `struct sg_table` pointer.
134     ///
135     /// This allows safe access to an `sg_table` that is managed elsewhere (for example, in C code).
136     ///
137     /// # Safety
138     ///
139     /// Callers must ensure that:
140     ///
141     /// - the `struct sg_table` pointed to by `ptr` is valid for the entire lifetime of `'a`,
142     /// - the data behind `ptr` is not modified concurrently for the duration of `'a`.
143     #[inline]
144     pub unsafe fn from_raw<'a>(ptr: *mut bindings::sg_table) -> &'a Self {
145         // SAFETY: The safety requirements of this function guarantee that `ptr` is a valid pointer
146         // to a `struct sg_table` for the duration of `'a`.
147         unsafe { &*ptr.cast() }
148     }
149 
150     #[inline]
151     fn as_raw(&self) -> *mut bindings::sg_table {
152         self.inner.0.get()
153     }
154 
155     /// Returns an [`SGTableIter`] bound to the lifetime of `self`.
156     pub fn iter(&self) -> SGTableIter<'_> {
157         // SAFETY: `self.as_raw()` is a valid pointer to a `struct sg_table`.
158         let nents = unsafe { (*self.as_raw()).nents };
159 
160         let pos = if nents > 0 {
161             // SAFETY: `self.as_raw()` is a valid pointer to a `struct sg_table`.
162             let ptr = unsafe { (*self.as_raw()).sgl };
163 
164             // SAFETY: `ptr` is guaranteed to be a valid pointer to a `struct scatterlist`.
165             Some(unsafe { SGEntry::from_raw(ptr) })
166         } else {
167             None
168         };
169 
170         SGTableIter { pos, nents }
171     }
172 }
173 
174 /// Represents the DMA mapping state of a `struct sg_table`.
175 ///
176 /// This is used as an inner type of [`Owned`] to manage the DMA mapping lifecycle.
177 ///
178 /// # Invariants
179 ///
180 /// - `sgt` is a valid pointer to a `struct sg_table` for the entire lifetime of the
181 ///   [`DmaMappedSgt`].
182 /// - `sgt` is always DMA mapped.
183 struct DmaMappedSgt {
184     sgt: NonNull<bindings::sg_table>,
185     dev: ARef<Device>,
186     dir: dma::DataDirection,
187 }
188 
189 // SAFETY: `DmaMappedSgt` can be sent to any task.
190 unsafe impl Send for DmaMappedSgt {}
191 
192 // SAFETY: `DmaMappedSgt` has no interior mutability and can be accessed concurrently.
193 unsafe impl Sync for DmaMappedSgt {}
194 
195 impl DmaMappedSgt {
196     /// # Safety
197     ///
198     /// - `sgt` must be a valid pointer to a `struct sg_table` for the entire lifetime of the
199     ///   returned [`DmaMappedSgt`].
200     /// - The caller must guarantee that `sgt` remains DMA mapped for the entire lifetime of
201     ///   [`DmaMappedSgt`].
202     unsafe fn new(
203         sgt: NonNull<bindings::sg_table>,
204         dev: &Device<Bound>,
205         dir: dma::DataDirection,
206     ) -> Result<Self> {
207         // SAFETY:
208         // - `dev.as_raw()` is a valid pointer to a `struct device`, which is guaranteed to be
209         //   bound to a driver for the duration of this call.
210         // - `sgt` is a valid pointer to a `struct sg_table`.
211         error::to_result(unsafe {
212             bindings::dma_map_sgtable(dev.as_raw(), sgt.as_ptr(), dir.into(), 0)
213         })?;
214 
215         // INVARIANT: By the safety requirements of this function it is guaranteed that `sgt` is
216         // valid for the entire lifetime of this object instance.
217         Ok(Self {
218             sgt,
219             dev: dev.into(),
220             dir,
221         })
222     }
223 }
224 
225 impl Drop for DmaMappedSgt {
226     #[inline]
227     fn drop(&mut self) {
228         // SAFETY:
229         // - `self.dev.as_raw()` is a pointer to a valid `struct device`.
230         // - `self.dev` is the same device the mapping has been created for in `Self::new()`.
231         // - `self.sgt.as_ptr()` is a valid pointer to a `struct sg_table` by the type invariants
232         //   of `Self`.
233         // - `self.dir` is the same `dma::DataDirection` the mapping has been created with in
234         //   `Self::new()`.
235         unsafe {
236             bindings::dma_unmap_sgtable(self.dev.as_raw(), self.sgt.as_ptr(), self.dir.into(), 0)
237         };
238     }
239 }
240 
241 /// A transparent wrapper around a `struct sg_table`.
242 ///
243 /// While we could also create the `struct sg_table` in the constructor of [`Owned`], we can't tear
244 /// down the `struct sg_table` in [`Owned::drop`]; the drop order in [`Owned`] matters.
245 #[repr(transparent)]
246 struct RawSGTable(Opaque<bindings::sg_table>);
247 
248 // SAFETY: `RawSGTable` can be sent to any task.
249 unsafe impl Send for RawSGTable {}
250 
251 // SAFETY: `RawSGTable` has no interior mutability and can be accessed concurrently.
252 unsafe impl Sync for RawSGTable {}
253 
254 impl RawSGTable {
255     /// # Safety
256     ///
257     /// - `pages` must be a slice of valid `struct page *`.
258     /// - The pages pointed to by `pages` must remain valid for the entire lifetime of the returned
259     ///   [`RawSGTable`].
260     unsafe fn new(
261         pages: &mut [*mut bindings::page],
262         size: usize,
263         max_segment: u32,
264         flags: alloc::Flags,
265     ) -> Result<Self> {
266         // `sg_alloc_table_from_pages_segment()` expects at least one page, otherwise it
267         // produces a NPE.
268         if pages.is_empty() {
269             return Err(EINVAL);
270         }
271 
272         let sgt = Opaque::zeroed();
273         // SAFETY:
274         // - `sgt.get()` is a valid pointer to uninitialized memory.
275         // - As by the check above, `pages` is not empty.
276         error::to_result(unsafe {
277             bindings::sg_alloc_table_from_pages_segment(
278                 sgt.get(),
279                 pages.as_mut_ptr(),
280                 pages.len().try_into()?,
281                 0,
282                 size,
283                 max_segment,
284                 flags.as_raw(),
285             )
286         })?;
287 
288         Ok(Self(sgt))
289     }
290 
291     #[inline]
292     fn as_raw(&self) -> *mut bindings::sg_table {
293         self.0.get()
294     }
295 }
296 
297 impl Drop for RawSGTable {
298     #[inline]
299     fn drop(&mut self) {
300         // SAFETY: `sgt` is a valid and initialized `struct sg_table`.
301         unsafe { bindings::sg_free_table(self.0.get()) };
302     }
303 }
304 
305 /// The [`Owned`] generic type of an [`SGTable`].
306 ///
307 /// A [`SGTable<Owned>`] signifies that the [`SGTable`] owns all associated resources:
308 ///
309 /// - The backing memory pages.
310 /// - The `struct sg_table` allocation (`sgt`).
311 /// - The DMA mapping, managed through a [`Devres`]-managed `DmaMappedSgt`.
312 ///
313 /// Users interact with this type through the [`SGTable`] handle and do not need to manage
314 /// [`Owned`] directly.
315 #[pin_data]
316 pub struct Owned<P> {
317     // Note: The drop order is relevant; we first have to unmap the `struct sg_table`, then free the
318     // `struct sg_table` and finally free the backing pages.
319     #[pin]
320     dma: Devres<DmaMappedSgt>,
321     sgt: RawSGTable,
322     _pages: P,
323 }
324 
325 // SAFETY: `Owned` can be sent to any task if `P` can be send to any task.
326 unsafe impl<P: Send> Send for Owned<P> {}
327 
328 // SAFETY: `Owned` has no interior mutability and can be accessed concurrently if `P` can be
329 // accessed concurrently.
330 unsafe impl<P: Sync> Sync for Owned<P> {}
331 
332 impl<P> Owned<P>
333 where
334     for<'a> P: page::AsPageIter<Iter<'a> = VmallocPageIter<'a>> + 'static,
335 {
336     fn new(
337         dev: &Device<Bound>,
338         mut pages: P,
339         dir: dma::DataDirection,
340         flags: alloc::Flags,
341     ) -> Result<impl PinInit<Self, Error> + '_> {
342         let page_iter = pages.page_iter();
343         let size = page_iter.size();
344 
345         let mut page_vec: KVec<*mut bindings::page> =
346             KVec::with_capacity(page_iter.page_count(), flags)?;
347 
348         for page in page_iter {
349             page_vec.push(page.as_ptr(), flags)?;
350         }
351 
352         // Cap segments at both the DMA mapping-path limit and the device's declared
353         // max segment size.
354         //
355         // `dma_max_mapping_size` returns `size_t`, but `sg_alloc_table_from_pages_segment()` takes
356         // an `unsigned int`.
357         //
358         // SAFETY: `dev.as_raw()` is a valid pointer to a `struct device`.
359         let max_mapping_size = match unsafe { bindings::dma_max_mapping_size(dev.as_raw()) } {
360             0 => u32::MAX,
361             max_mapping_size => u32::try_from(max_mapping_size).unwrap_or(u32::MAX),
362         };
363 
364         // SAFETY: `dev.as_raw()` is a valid pointer to a `struct device`.
365         let max_seg_size = unsafe { bindings::dma_get_max_seg_size(dev.as_raw()) };
366 
367         let max_segment = max_mapping_size.min(max_seg_size);
368 
369         Ok(try_pin_init!(&this in Self {
370             // SAFETY:
371             // - `page_vec` is a `KVec` of valid `struct page *` obtained from `pages`.
372             // - The pages contained in `pages` remain valid for the entire lifetime of the
373             //   `RawSGTable`.
374             sgt: unsafe { RawSGTable::new(&mut page_vec, size, max_segment, flags) }?,
375             dma <- {
376                 // SAFETY: `this` is a valid pointer to uninitialized memory.
377                 let sgt = unsafe { &raw mut (*this.as_ptr()).sgt }.cast();
378 
379                 // SAFETY: `sgt` is guaranteed to be non-null.
380                 let sgt = unsafe { NonNull::new_unchecked(sgt) };
381 
382                 // SAFETY:
383                 // - It is guaranteed that the object returned by `DmaMappedSgt::new` won't out-live
384                 //   `sgt`.
385                 // - `sgt` is never DMA unmapped manually.
386                 Devres::new(dev, unsafe { DmaMappedSgt::new(sgt, dev, dir) })
387             },
388             _pages: pages,
389         }))
390     }
391 }
392 
393 impl<P> SGTable<Owned<P>>
394 where
395     for<'a> P: page::AsPageIter<Iter<'a> = VmallocPageIter<'a>> + 'static,
396 {
397     /// Allocates a new scatter-gather table from the given pages and maps it for DMA.
398     ///
399     /// This constructor creates a new [`SGTable<Owned>`] that takes ownership of `P`.
400     /// It allocates a `struct sg_table`, populates it with entries corresponding to the physical
401     /// pages of `P`, and maps the table for DMA with the specified [`Device`] and
402     /// [`dma::DataDirection`].
403     ///
404     /// The DMA mapping is managed through [`Devres`], ensuring that the DMA mapping is unmapped
405     /// once the associated [`Device`] is unbound, or when the [`SGTable<Owned>`] is dropped.
406     ///
407     /// # Parameters
408     ///
409     /// * `dev`: The [`Device`] that will be performing the DMA.
410     /// * `pages`: The entity providing the backing pages. It must implement [`page::AsPageIter`].
411     ///   The ownership of this entity is moved into the new [`SGTable<Owned>`].
412     /// * `dir`: The [`dma::DataDirection`] of the DMA transfer.
413     /// * `flags`: Allocation flags for internal allocations (e.g., [`GFP_KERNEL`]).
414     ///
415     /// # Examples
416     ///
417     /// ```
418     /// use kernel::{
419     ///     device::{Bound, Device},
420     ///     dma, page,
421     ///     prelude::*,
422     ///     scatterlist::{SGTable, Owned},
423     /// };
424     ///
425     /// fn test(dev: &Device<Bound>) -> Result {
426     ///     let size = 4 * page::PAGE_SIZE;
427     ///     let pages = VVec::<u8>::with_capacity(size, GFP_KERNEL)?;
428     ///
429     ///     let sgt = KBox::pin_init(SGTable::new(
430     ///         dev,
431     ///         pages,
432     ///         dma::DataDirection::ToDevice,
433     ///         GFP_KERNEL,
434     ///     ), GFP_KERNEL)?;
435     ///
436     ///     Ok(())
437     /// }
438     /// ```
439     pub fn new(
440         dev: &Device<Bound>,
441         pages: P,
442         dir: dma::DataDirection,
443         flags: alloc::Flags,
444     ) -> impl PinInit<Self, Error> + '_ {
445         try_pin_init!(Self {
446             inner <- Owned::new(dev, pages, dir, flags)?
447         })
448     }
449 }
450 
451 impl<P> Deref for SGTable<Owned<P>> {
452     type Target = SGTable;
453 
454     #[inline]
455     fn deref(&self) -> &Self::Target {
456         // SAFETY:
457         // - `self.inner.sgt.as_raw()` is a valid pointer to a `struct sg_table` for the entire
458         //   lifetime of `self`.
459         // - The backing `struct sg_table` is not modified for the entire lifetime of `self`.
460         unsafe { SGTable::from_raw(self.inner.sgt.as_raw()) }
461     }
462 }
463 
464 mod private {
465     pub trait Sealed {}
466 
467     impl Sealed for super::Borrowed {}
468     impl<P> Sealed for super::Owned<P> {}
469 }
470 
471 /// An [`Iterator`] over the DMA mapped [`SGEntry`] items of an [`SGTable`].
472 ///
473 /// Note that the existence of an [`SGTableIter`] does not guarantee that the [`SGEntry`] items
474 /// actually remain DMA mapped; they are prone to be unmapped on device unbind.
475 pub struct SGTableIter<'a> {
476     pos: Option<&'a SGEntry>,
477     /// The number of DMA mapped entries in a `struct sg_table`.
478     nents: c_uint,
479 }
480 
481 impl<'a> Iterator for SGTableIter<'a> {
482     type Item = &'a SGEntry;
483 
484     fn next(&mut self) -> Option<Self::Item> {
485         let entry = self.pos?;
486         self.nents = self.nents.saturating_sub(1);
487 
488         // SAFETY: `entry.as_raw()` is a valid pointer to a `struct scatterlist`.
489         let next = unsafe { bindings::sg_next(entry.as_raw()) };
490 
491         self.pos = (!next.is_null() && self.nents > 0).then(|| {
492             // SAFETY: If `next` is not NULL, `sg_next()` guarantees to return a valid pointer to
493             // the next `struct scatterlist`.
494             unsafe { SGEntry::from_raw(next) }
495         });
496 
497         Some(entry)
498     }
499 }
500