1 // SPDX-License-Identifier: GPL-2.0
3 //! Direct memory access (DMA).
5 //! C header: [`include/linux/dma-mapping.h`](srctree/include/linux/dma-mapping.h)
17 /// DMA address type.
19 /// Represents a bus address used for Direct Memory Access (DMA) operations.
24 /// Note that this may be `u64` even on 32-bit architectures.
27 /// Trait to be implemented by DMA capable bus devices.
29 /// The [`dma::Device`](Device) trait should be implemented by bus specific device representations,
30 /// where the underlying bus is DMA capable, such as [`pci::Device`](::kernel::pci::Device) or
33     /// Set up the device's DMA streaming addressing capabilities.
40     /// This method must not be called concurrently with any DMA allocation or mapping primitives,
42     unsafe fn dma_set_mask(&self, mask: DmaMask) -> Result {  in dma_set_mask()
44         // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.  in dma_set_mask()
45         // - The safety requirement of this function guarantees that there are no concurrent calls  in dma_set_mask()
46         //   to DMA allocation and mapping primitives using this mask.  in dma_set_mask()
50     /// Set up the device's DMA coherent addressing capabilities.
57     /// This method must not be called concurrently with any DMA allocation or mapping primitives,
59     unsafe fn dma_set_coherent_mask(&self, mask: DmaMask) -> Result {  in dma_set_coherent_mask()
61         // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.  in dma_set_coherent_mask()
62         // - The safety requirement of this function guarantees that there are no concurrent calls  in dma_set_coherent_mask()
63         //   to DMA allocation and mapping primitives using this mask.  in dma_set_coherent_mask()
67     /// Set up the device's DMA addressing capabilities.
76     /// This method must not be called concurrently with any DMA allocation or mapping primitives,
78     unsafe fn dma_set_mask_and_coherent(&self, mask: DmaMask) -> Result {  in dma_set_mask_and_coherent()
80         // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.  in dma_set_mask_and_coherent()
81         // - The safety requirement of this function guarantees that there are no concurrent calls  in dma_set_mask_and_coherent()
82         //   to DMA allocation and mapping primitives using this mask.  in dma_set_mask_and_coherent()
89 /// A DMA mask that holds a bitmask with the lowest `n` bits set.
107     /// use kernel::dma::DmaMask;
122     pub const fn new<const N: u32>() -> Self {  in new()
124             build_error!("Invalid DMA Mask.");  in new()
138     /// use kernel::dma::DmaMask;
154     pub const fn try_new(n: u32) -> Result<Self> {  in try_new()
157             1..=64 => u64::MAX >> (64 - n),  in try_new()
164     pub const fn value(&self) -> u64 {  in value()
169 /// Possible attributes associated with a DMA mapping.
179 /// use kernel::dma::{attrs::*, CoherentAllocation};
181 /// # fn test(dev: &Device<Bound>) -> Result {
193     pub(crate) fn as_raw(self) -> crate::ffi::c_ulong {  in as_raw()
198     pub fn contains(self, flags: Attrs) -> bool {  in contains()
205     fn bitor(self, rhs: Self) -> Self::Output {  in bitor()
212     fn bitand(self, rhs: Self) -> Self::Output {  in bitand()
219     fn not(self) -> Self::Output {  in not()
224 /// DMA mapping attributes.
245     /// Hints DMA-mapping subsystem that it's probably not worth the time to try
249     /// This tells the DMA-mapping subsystem to suppress allocation failure reports (similarly to
254     /// ideally inaccessible or at least read-only at lesser-privileged levels).
261 /// DMA data direction.
265 /// [`enum dma_data_direction`]: srctree/include/linux/dma-direction.h
269     /// The DMA mapping is for bidirectional data transfer.
275     /// The DMA mapping is for data transfer from memory to the device (write).
281     /// The DMA mapping is for data transfer from the device to memory (read).
283     /// The device will write data into the buffer for the CPU to read.
287     /// The DMA mapping is not for data transfer.
289     /// This is primarily for debugging purposes. With this direction, the DMA mapping API
295     /// Casts the bindgen-generated enum type to a `u32` at compile time.
297     /// This function will cause a compile-time error if the underlying value of the
299     const fn const_cast(val: bindings::dma_data_direction) -> u32 {  in const_cast()
308             // Trigger a compile-time error in a const context.  in const_cast()
320     fn from(direction: DataDirection) -> Self {  in from()
324         // compile-time checks.  in from()
332 /// large coherent DMA regions.
336 /// as the DMA address base of the region. The region is released once [`CoherentAllocation`]
341 /// - For the lifetime of an instance of [`CoherentAllocation`], the `cpu_addr` is a valid pointer
342 ///   to an allocated region of coherent memory and `dma_handle` is the DMA address base of the
344 /// - The size in bytes of the allocation is equal to `size_of::<T> * count`.
345 /// - `size_of::<T> * count` fits into a `usize`.
348 // DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness
349 // reasons DMA allocation would need to be embedded in a `Devres` container, in order to ensure
352 // However, it is neither desirable nor necessary to protect the allocated memory of the DMA
373     /// use kernel::dma::{attrs::*, CoherentAllocation};
375     /// # fn test(dev: &Device<Bound>) -> Result {
385     ) -> Result<CoherentAllocation<T>> {  in alloc_attrs()
407         // - We just successfully allocated a coherent region which is accessible for  in alloc_attrs()
410         // - The allocated `size` is equal to `size_of::<T> * count`.  in alloc_attrs()
411         // - The allocated `size` fits into a `usize`.  in alloc_attrs()
427     ) -> Result<CoherentAllocation<T>> {  in alloc_coherent()
435     pub fn count(&self) -> usize {  in count()
440     pub fn size(&self) -> usize {  in size()
447     pub fn start_ptr(&self) -> *const T {  in start_ptr()
453     pub fn start_ptr_mut(&mut self) -> *mut T {  in start_ptr_mut()
457     /// Returns a DMA handle which may be given to the device as the DMA address base of
459     pub fn dma_handle(&self) -> DmaAddress {  in dma_handle()
463     /// Returns a DMA handle starting at `offset` (in units of `T`) which may be given to the
464     /// device as the DMA address base of the region.
467     pub fn dma_handle_with_offset(&self, offset: usize) -> Result<DmaAddress> {  in dma_handle_with_offset()
479     fn validate_range(&self, offset: usize, count: usize) -> Result {  in validate_range()
489     /// For ringbuffer type of r/w access or use-cases where the pointer to the live data is needed,
495     /// * Callers must ensure that the device does not read/write to/from memory while the returned
497     /// * Callers must ensure that this call does not race with a write to the same region while
499     pub unsafe fn as_slice(&self, offset: usize, count: usize) -> Result<&[T]> {  in as_slice()
502         // - The pointer is valid due to type invariant on `CoherentAllocation`,  in as_slice()
505         // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked  in as_slice()
515     /// * Callers must ensure that the device does not read/write to/from memory while the returned
517     /// * Callers must ensure that this call does not race with a read or write to the same region
519     pub unsafe fn as_slice_mut(&mut self, offset: usize, count: usize) -> Result<&mut [T]> {  in as_slice_mut()
522         // - The pointer is valid due to type invariant on `CoherentAllocation`,  in as_slice_mut()
525         // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked  in as_slice_mut()
535     /// * Callers must ensure that the device does not read/write to/from memory while the returned
537     /// * Callers must ensure that this call does not race with a read or write to the same region
538     ///   that overlaps with this write.
543     /// # fn test(alloc: &mut kernel::dma::CoherentAllocation<u8>) -> Result {
548     /// unsafe { alloc.write(buf, 0)?; }
551     pub unsafe fn write(&mut self, src: &[T], offset: usize) -> Result {  in write()  method
554         // - The pointer is valid due to type invariant on `CoherentAllocation`  in write()
556         // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked  in write()
573     pub fn item_from_index(&self, offset: usize) -> Result<*mut T> {  in item_from_index()
578         // - The pointer is valid due to type invariant on `CoherentAllocation`  in item_from_index()
580         // - `offset` can't overflow since it is smaller than `self.count` and we've checked  in item_from_index()
594     pub unsafe fn field_read<F: FromBytes>(&self, field: *const F) -> F {  in field_read()
596         // - By the safety requirements field is valid.  in field_read()
597         // - Using read_volatile() here is not sound as per the usual rules, the usage here is  in field_read()
599         // race from a hardware or code outside kernel (e.g. user-space program), we need that  in field_read()
619         // - By the safety requirements field is valid.  in field_write()
620         // - Using write_volatile() here is not sound as per the usual rules, the usage here is  in field_write()
622         // race from a hardware or code outside kernel (e.g. user-space program), we need that  in field_write()
623         // write on a valid memory is not UB. Currently write_volatile() is used for this, and the  in field_write()
632 /// Note that the device configured to do DMA must be halted before this object is dropped.
637         // The cpu address, and the dma handle are valid due to the type invariants on  in drop()
661 /// use kernel::dma::{attrs::*, CoherentAllocation};
670 /// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result {
677     ($dma:expr, $idx: expr, $($field:tt)*) => {{
678         (|| -> ::core::result::Result<_, $crate::error::Error> {
679             let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
686                     $crate::dma::CoherentAllocation::field_read(&$dma, ptr_field)
691     ($dma:ident [ $idx:expr ] $($field:tt)* ) => {
692         $crate::dma_read!($dma, $idx, $($field)*)
694     ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {
695         $crate::dma_read!($($dma).*, $idx, $($field)*)
705 /// use kernel::dma::{attrs::*, CoherentAllocation};
714 /// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result {
721     ($dma:ident [ $idx:expr ] $($field:tt)*) => {{
722         $crate::dma_write!($dma, $idx, $($field)*)
724     ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {{
725         $crate::dma_write!($($dma).*, $idx, $($field)*)
727     ($dma:expr, $idx: expr, = $val:expr) => {
728         (|| -> ::core::result::Result<_, $crate::error::Error> {
729             let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
731             unsafe { $crate::dma::CoherentAllocation::field_write(&$dma, item, $val) }
735     ($dma:expr, $idx: expr, $(.$field:ident)* = $val:expr) => {
736         (|| -> ::core::result::Result<_, $crate::error::Error> {
737             let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
743                 $crate::dma::CoherentAllocation::field_write(&$dma, ptr_field, $val)