1 // SPDX-License-Identifier: GPL-2.0 2 3 //! Direct memory access (DMA). 4 //! 5 //! C header: [`include/linux/dma-mapping.h`](srctree/include/linux/dma-mapping.h) 6 7 use crate::{ 8 bindings, build_assert, device, 9 device::{Bound, Core}, 10 error::{to_result, Result}, 11 prelude::*, 12 sync::aref::ARef, 13 transmute::{AsBytes, FromBytes}, 14 }; 15 use core::ptr::NonNull; 16 17 /// DMA address type. 18 /// 19 /// Represents a bus address used for Direct Memory Access (DMA) operations. 20 /// 21 /// This is an alias of the kernel's `dma_addr_t`, which may be `u32` or `u64` depending on 22 /// `CONFIG_ARCH_DMA_ADDR_T_64BIT`. 23 /// 24 /// Note that this may be `u64` even on 32-bit architectures. 25 pub type DmaAddress = bindings::dma_addr_t; 26 27 /// Trait to be implemented by DMA capable bus devices. 28 /// 29 /// The [`dma::Device`](Device) trait should be implemented by bus specific device representations, 30 /// where the underlying bus is DMA capable, such as: 31 #[cfg_attr(CONFIG_PCI, doc = "* [`pci::Device`](kernel::pci::Device)")] 32 /// * [`platform::Device`](::kernel::platform::Device) 33 pub trait Device: AsRef<device::Device<Core>> { 34 /// Set up the device's DMA streaming addressing capabilities. 35 /// 36 /// This method is usually called once from `probe()` as soon as the device capabilities are 37 /// known. 38 /// 39 /// # Safety 40 /// 41 /// This method must not be called concurrently with any DMA allocation or mapping primitives, 42 /// such as [`CoherentAllocation::alloc_attrs`]. 43 unsafe fn dma_set_mask(&self, mask: DmaMask) -> Result { 44 // SAFETY: 45 // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. 46 // - The safety requirement of this function guarantees that there are no concurrent calls 47 // to DMA allocation and mapping primitives using this mask. 48 to_result(unsafe { bindings::dma_set_mask(self.as_ref().as_raw(), mask.value()) }) 49 } 50 51 /// Set up the device's DMA coherent addressing capabilities. 52 /// 53 /// This method is usually called once from `probe()` as soon as the device capabilities are 54 /// known. 55 /// 56 /// # Safety 57 /// 58 /// This method must not be called concurrently with any DMA allocation or mapping primitives, 59 /// such as [`CoherentAllocation::alloc_attrs`]. 60 unsafe fn dma_set_coherent_mask(&self, mask: DmaMask) -> Result { 61 // SAFETY: 62 // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. 63 // - The safety requirement of this function guarantees that there are no concurrent calls 64 // to DMA allocation and mapping primitives using this mask. 65 to_result(unsafe { bindings::dma_set_coherent_mask(self.as_ref().as_raw(), mask.value()) }) 66 } 67 68 /// Set up the device's DMA addressing capabilities. 69 /// 70 /// This is a combination of [`Device::dma_set_mask`] and [`Device::dma_set_coherent_mask`]. 71 /// 72 /// This method is usually called once from `probe()` as soon as the device capabilities are 73 /// known. 74 /// 75 /// # Safety 76 /// 77 /// This method must not be called concurrently with any DMA allocation or mapping primitives, 78 /// such as [`CoherentAllocation::alloc_attrs`]. 79 unsafe fn dma_set_mask_and_coherent(&self, mask: DmaMask) -> Result { 80 // SAFETY: 81 // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. 82 // - The safety requirement of this function guarantees that there are no concurrent calls 83 // to DMA allocation and mapping primitives using this mask. 84 to_result(unsafe { 85 bindings::dma_set_mask_and_coherent(self.as_ref().as_raw(), mask.value()) 86 }) 87 } 88 89 /// Set the maximum size of a single DMA segment the device may request. 90 /// 91 /// This method is usually called once from `probe()` as soon as the device capabilities are 92 /// known. 93 /// 94 /// # Safety 95 /// 96 /// This method must not be called concurrently with any DMA allocation or mapping primitives, 97 /// such as [`CoherentAllocation::alloc_attrs`]. 98 unsafe fn dma_set_max_seg_size(&self, size: u32) { 99 // SAFETY: 100 // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. 101 // - The safety requirement of this function guarantees that there are no concurrent calls 102 // to DMA allocation and mapping primitives using this parameter. 103 unsafe { bindings::dma_set_max_seg_size(self.as_ref().as_raw(), size) } 104 } 105 } 106 107 /// A DMA mask that holds a bitmask with the lowest `n` bits set. 108 /// 109 /// Use [`DmaMask::new`] or [`DmaMask::try_new`] to construct a value. Values 110 /// are guaranteed to never exceed the bit width of `u64`. 111 /// 112 /// This is the Rust equivalent of the C macro `DMA_BIT_MASK()`. 113 #[derive(Debug, Clone, Copy, PartialEq, Eq)] 114 pub struct DmaMask(u64); 115 116 impl DmaMask { 117 /// Constructs a `DmaMask` with the lowest `n` bits set to `1`. 118 /// 119 /// For `n <= 64`, sets exactly the lowest `n` bits. 120 /// For `n > 64`, results in a build error. 121 /// 122 /// # Examples 123 /// 124 /// ``` 125 /// use kernel::dma::DmaMask; 126 /// 127 /// let mask0 = DmaMask::new::<0>(); 128 /// assert_eq!(mask0.value(), 0); 129 /// 130 /// let mask1 = DmaMask::new::<1>(); 131 /// assert_eq!(mask1.value(), 0b1); 132 /// 133 /// let mask64 = DmaMask::new::<64>(); 134 /// assert_eq!(mask64.value(), u64::MAX); 135 /// 136 /// // Build failure. 137 /// // let mask_overflow = DmaMask::new::<100>(); 138 /// ``` 139 #[inline] 140 pub const fn new<const N: u32>() -> Self { 141 let Ok(mask) = Self::try_new(N) else { 142 build_error!("Invalid DMA Mask."); 143 }; 144 145 mask 146 } 147 148 /// Constructs a `DmaMask` with the lowest `n` bits set to `1`. 149 /// 150 /// For `n <= 64`, sets exactly the lowest `n` bits. 151 /// For `n > 64`, returns [`EINVAL`]. 152 /// 153 /// # Examples 154 /// 155 /// ``` 156 /// use kernel::dma::DmaMask; 157 /// 158 /// let mask0 = DmaMask::try_new(0)?; 159 /// assert_eq!(mask0.value(), 0); 160 /// 161 /// let mask1 = DmaMask::try_new(1)?; 162 /// assert_eq!(mask1.value(), 0b1); 163 /// 164 /// let mask64 = DmaMask::try_new(64)?; 165 /// assert_eq!(mask64.value(), u64::MAX); 166 /// 167 /// let mask_overflow = DmaMask::try_new(100); 168 /// assert!(mask_overflow.is_err()); 169 /// # Ok::<(), Error>(()) 170 /// ``` 171 #[inline] 172 pub const fn try_new(n: u32) -> Result<Self> { 173 Ok(Self(match n { 174 0 => 0, 175 1..=64 => u64::MAX >> (64 - n), 176 _ => return Err(EINVAL), 177 })) 178 } 179 180 /// Returns the underlying `u64` bitmask value. 181 #[inline] 182 pub const fn value(&self) -> u64 { 183 self.0 184 } 185 } 186 187 /// Possible attributes associated with a DMA mapping. 188 /// 189 /// They can be combined with the operators `|`, `&`, and `!`. 190 /// 191 /// Values can be used from the [`attrs`] module. 192 /// 193 /// # Examples 194 /// 195 /// ``` 196 /// # use kernel::device::{Bound, Device}; 197 /// use kernel::dma::{attrs::*, CoherentAllocation}; 198 /// 199 /// # fn test(dev: &Device<Bound>) -> Result { 200 /// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN; 201 /// let c: CoherentAllocation<u64> = 202 /// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, attribs)?; 203 /// # Ok::<(), Error>(()) } 204 /// ``` 205 #[derive(Clone, Copy, PartialEq)] 206 #[repr(transparent)] 207 pub struct Attrs(u32); 208 209 impl Attrs { 210 /// Get the raw representation of this attribute. 211 pub(crate) fn as_raw(self) -> crate::ffi::c_ulong { 212 self.0 as crate::ffi::c_ulong 213 } 214 215 /// Check whether `flags` is contained in `self`. 216 pub fn contains(self, flags: Attrs) -> bool { 217 (self & flags) == flags 218 } 219 } 220 221 impl core::ops::BitOr for Attrs { 222 type Output = Self; 223 fn bitor(self, rhs: Self) -> Self::Output { 224 Self(self.0 | rhs.0) 225 } 226 } 227 228 impl core::ops::BitAnd for Attrs { 229 type Output = Self; 230 fn bitand(self, rhs: Self) -> Self::Output { 231 Self(self.0 & rhs.0) 232 } 233 } 234 235 impl core::ops::Not for Attrs { 236 type Output = Self; 237 fn not(self) -> Self::Output { 238 Self(!self.0) 239 } 240 } 241 242 /// DMA mapping attributes. 243 pub mod attrs { 244 use super::Attrs; 245 246 /// Specifies that reads and writes to the mapping may be weakly ordered, that is that reads 247 /// and writes may pass each other. 248 pub const DMA_ATTR_WEAK_ORDERING: Attrs = Attrs(bindings::DMA_ATTR_WEAK_ORDERING); 249 250 /// Specifies that writes to the mapping may be buffered to improve performance. 251 pub const DMA_ATTR_WRITE_COMBINE: Attrs = Attrs(bindings::DMA_ATTR_WRITE_COMBINE); 252 253 /// Lets the platform to avoid creating a kernel virtual mapping for the allocated buffer. 254 pub const DMA_ATTR_NO_KERNEL_MAPPING: Attrs = Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING); 255 256 /// Allows platform code to skip synchronization of the CPU cache for the given buffer assuming 257 /// that it has been already transferred to 'device' domain. 258 pub const DMA_ATTR_SKIP_CPU_SYNC: Attrs = Attrs(bindings::DMA_ATTR_SKIP_CPU_SYNC); 259 260 /// Forces contiguous allocation of the buffer in physical memory. 261 pub const DMA_ATTR_FORCE_CONTIGUOUS: Attrs = Attrs(bindings::DMA_ATTR_FORCE_CONTIGUOUS); 262 263 /// Hints DMA-mapping subsystem that it's probably not worth the time to try 264 /// to allocate memory to in a way that gives better TLB efficiency. 265 pub const DMA_ATTR_ALLOC_SINGLE_PAGES: Attrs = Attrs(bindings::DMA_ATTR_ALLOC_SINGLE_PAGES); 266 267 /// This tells the DMA-mapping subsystem to suppress allocation failure reports (similarly to 268 /// `__GFP_NOWARN`). 269 pub const DMA_ATTR_NO_WARN: Attrs = Attrs(bindings::DMA_ATTR_NO_WARN); 270 271 /// Indicates that the buffer is fully accessible at an elevated privilege level (and 272 /// ideally inaccessible or at least read-only at lesser-privileged levels). 273 pub const DMA_ATTR_PRIVILEGED: Attrs = Attrs(bindings::DMA_ATTR_PRIVILEGED); 274 275 /// Indicates that the buffer is MMIO memory. 276 pub const DMA_ATTR_MMIO: Attrs = Attrs(bindings::DMA_ATTR_MMIO); 277 } 278 279 /// DMA data direction. 280 /// 281 /// Corresponds to the C [`enum dma_data_direction`]. 282 /// 283 /// [`enum dma_data_direction`]: srctree/include/linux/dma-direction.h 284 #[derive(Copy, Clone, PartialEq, Eq, Debug)] 285 #[repr(u32)] 286 pub enum DataDirection { 287 /// The DMA mapping is for bidirectional data transfer. 288 /// 289 /// This is used when the buffer can be both read from and written to by the device. 290 /// The cache for the corresponding memory region is both flushed and invalidated. 291 Bidirectional = Self::const_cast(bindings::dma_data_direction_DMA_BIDIRECTIONAL), 292 293 /// The DMA mapping is for data transfer from memory to the device (write). 294 /// 295 /// The CPU has prepared data in the buffer, and the device will read it. 296 /// The cache for the corresponding memory region is flushed before device access. 297 ToDevice = Self::const_cast(bindings::dma_data_direction_DMA_TO_DEVICE), 298 299 /// The DMA mapping is for data transfer from the device to memory (read). 300 /// 301 /// The device will write data into the buffer for the CPU to read. 302 /// The cache for the corresponding memory region is invalidated before CPU access. 303 FromDevice = Self::const_cast(bindings::dma_data_direction_DMA_FROM_DEVICE), 304 305 /// The DMA mapping is not for data transfer. 306 /// 307 /// This is primarily for debugging purposes. With this direction, the DMA mapping API 308 /// will not perform any cache coherency operations. 309 None = Self::const_cast(bindings::dma_data_direction_DMA_NONE), 310 } 311 312 impl DataDirection { 313 /// Casts the bindgen-generated enum type to a `u32` at compile time. 314 /// 315 /// This function will cause a compile-time error if the underlying value of the 316 /// C enum is out of bounds for `u32`. 317 const fn const_cast(val: bindings::dma_data_direction) -> u32 { 318 // CAST: The C standard allows compilers to choose different integer types for enums. 319 // To safely check the value, we cast it to a wide signed integer type (`i128`) 320 // which can hold any standard C integer enum type without truncation. 321 let wide_val = val as i128; 322 323 // Check if the value is outside the valid range for the target type `u32`. 324 // CAST: `u32::MAX` is cast to `i128` to match the type of `wide_val` for the comparison. 325 if wide_val < 0 || wide_val > u32::MAX as i128 { 326 // Trigger a compile-time error in a const context. 327 build_error!("C enum value is out of bounds for the target type `u32`."); 328 } 329 330 // CAST: This cast is valid because the check above guarantees that `wide_val` 331 // is within the representable range of `u32`. 332 wide_val as u32 333 } 334 } 335 336 impl From<DataDirection> for bindings::dma_data_direction { 337 /// Returns the raw representation of [`enum dma_data_direction`]. 338 fn from(direction: DataDirection) -> Self { 339 // CAST: `direction as u32` gets the underlying representation of our `#[repr(u32)]` enum. 340 // The subsequent cast to `Self` (the bindgen type) assumes the C enum is compatible 341 // with the enum variants of `DataDirection`, which is a valid assumption given our 342 // compile-time checks. 343 direction as u32 as Self 344 } 345 } 346 347 /// An abstraction of the `dma_alloc_coherent` API. 348 /// 349 /// This is an abstraction around the `dma_alloc_coherent` API which is used to allocate and map 350 /// large coherent DMA regions. 351 /// 352 /// A [`CoherentAllocation`] instance contains a pointer to the allocated region (in the 353 /// processor's virtual address space) and the device address which can be given to the device 354 /// as the DMA address base of the region. The region is released once [`CoherentAllocation`] 355 /// is dropped. 356 /// 357 /// # Invariants 358 /// 359 /// - For the lifetime of an instance of [`CoherentAllocation`], the `cpu_addr` is a valid pointer 360 /// to an allocated region of coherent memory and `dma_handle` is the DMA address base of the 361 /// region. 362 /// - The size in bytes of the allocation is equal to `size_of::<T> * count`. 363 /// - `size_of::<T> * count` fits into a `usize`. 364 // TODO 365 // 366 // DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness 367 // reasons DMA allocation would need to be embedded in a `Devres` container, in order to ensure 368 // that device resources can never survive device unbind. 369 // 370 // However, it is neither desirable nor necessary to protect the allocated memory of the DMA 371 // allocation from surviving device unbind; it would require RCU read side critical sections to 372 // access the memory, which may require subsequent unnecessary copies. 373 // 374 // Hence, find a way to revoke the device resources of a `CoherentAllocation`, but not the 375 // entire `CoherentAllocation` including the allocated memory itself. 376 pub struct CoherentAllocation<T: AsBytes + FromBytes> { 377 dev: ARef<device::Device>, 378 dma_handle: DmaAddress, 379 count: usize, 380 cpu_addr: NonNull<T>, 381 dma_attrs: Attrs, 382 } 383 384 impl<T: AsBytes + FromBytes> CoherentAllocation<T> { 385 /// Allocates a region of `size_of::<T> * count` of coherent memory. 386 /// 387 /// # Examples 388 /// 389 /// ``` 390 /// # use kernel::device::{Bound, Device}; 391 /// use kernel::dma::{attrs::*, CoherentAllocation}; 392 /// 393 /// # fn test(dev: &Device<Bound>) -> Result { 394 /// let c: CoherentAllocation<u64> = 395 /// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?; 396 /// # Ok::<(), Error>(()) } 397 /// ``` 398 pub fn alloc_attrs( 399 dev: &device::Device<Bound>, 400 count: usize, 401 gfp_flags: kernel::alloc::Flags, 402 dma_attrs: Attrs, 403 ) -> Result<CoherentAllocation<T>> { 404 build_assert!( 405 core::mem::size_of::<T>() > 0, 406 "It doesn't make sense for the allocated type to be a ZST" 407 ); 408 409 let size = count 410 .checked_mul(core::mem::size_of::<T>()) 411 .ok_or(EOVERFLOW)?; 412 let mut dma_handle = 0; 413 // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. 414 let addr = unsafe { 415 bindings::dma_alloc_attrs( 416 dev.as_raw(), 417 size, 418 &mut dma_handle, 419 gfp_flags.as_raw(), 420 dma_attrs.as_raw(), 421 ) 422 }; 423 let addr = NonNull::new(addr).ok_or(ENOMEM)?; 424 // INVARIANT: 425 // - We just successfully allocated a coherent region which is accessible for 426 // `count` elements, hence the cpu address is valid. We also hold a refcounted reference 427 // to the device. 428 // - The allocated `size` is equal to `size_of::<T> * count`. 429 // - The allocated `size` fits into a `usize`. 430 Ok(Self { 431 dev: dev.into(), 432 dma_handle, 433 count, 434 cpu_addr: addr.cast(), 435 dma_attrs, 436 }) 437 } 438 439 /// Performs the same functionality as [`CoherentAllocation::alloc_attrs`], except the 440 /// `dma_attrs` is 0 by default. 441 pub fn alloc_coherent( 442 dev: &device::Device<Bound>, 443 count: usize, 444 gfp_flags: kernel::alloc::Flags, 445 ) -> Result<CoherentAllocation<T>> { 446 CoherentAllocation::alloc_attrs(dev, count, gfp_flags, Attrs(0)) 447 } 448 449 /// Returns the number of elements `T` in this allocation. 450 /// 451 /// Note that this is not the size of the allocation in bytes, which is provided by 452 /// [`Self::size`]. 453 pub fn count(&self) -> usize { 454 self.count 455 } 456 457 /// Returns the size in bytes of this allocation. 458 pub fn size(&self) -> usize { 459 // INVARIANT: The type invariant of `Self` guarantees that `size_of::<T> * count` fits into 460 // a `usize`. 461 self.count * core::mem::size_of::<T>() 462 } 463 464 /// Returns the base address to the allocated region in the CPU's virtual address space. 465 pub fn start_ptr(&self) -> *const T { 466 self.cpu_addr.as_ptr() 467 } 468 469 /// Returns the base address to the allocated region in the CPU's virtual address space as 470 /// a mutable pointer. 471 pub fn start_ptr_mut(&mut self) -> *mut T { 472 self.cpu_addr.as_ptr() 473 } 474 475 /// Returns a DMA handle which may be given to the device as the DMA address base of 476 /// the region. 477 pub fn dma_handle(&self) -> DmaAddress { 478 self.dma_handle 479 } 480 481 /// Returns a DMA handle starting at `offset` (in units of `T`) which may be given to the 482 /// device as the DMA address base of the region. 483 /// 484 /// Returns `EINVAL` if `offset` is not within the bounds of the allocation. 485 pub fn dma_handle_with_offset(&self, offset: usize) -> Result<DmaAddress> { 486 if offset >= self.count { 487 Err(EINVAL) 488 } else { 489 // INVARIANT: The type invariant of `Self` guarantees that `size_of::<T> * count` fits 490 // into a `usize`, and `offset` is inferior to `count`. 491 Ok(self.dma_handle + (offset * core::mem::size_of::<T>()) as DmaAddress) 492 } 493 } 494 495 /// Common helper to validate a range applied from the allocated region in the CPU's virtual 496 /// address space. 497 fn validate_range(&self, offset: usize, count: usize) -> Result { 498 if offset.checked_add(count).ok_or(EOVERFLOW)? > self.count { 499 return Err(EINVAL); 500 } 501 Ok(()) 502 } 503 504 /// Returns the data from the region starting from `offset` as a slice. 505 /// `offset` and `count` are in units of `T`, not the number of bytes. 506 /// 507 /// For ringbuffer type of r/w access or use-cases where the pointer to the live data is needed, 508 /// [`CoherentAllocation::start_ptr`] or [`CoherentAllocation::start_ptr_mut`] could be used 509 /// instead. 510 /// 511 /// # Safety 512 /// 513 /// * Callers must ensure that the device does not read/write to/from memory while the returned 514 /// slice is live. 515 /// * Callers must ensure that this call does not race with a write to the same region while 516 /// the returned slice is live. 517 pub unsafe fn as_slice(&self, offset: usize, count: usize) -> Result<&[T]> { 518 self.validate_range(offset, count)?; 519 // SAFETY: 520 // - The pointer is valid due to type invariant on `CoherentAllocation`, 521 // we've just checked that the range and index is within bounds. The immutability of the 522 // data is also guaranteed by the safety requirements of the function. 523 // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked 524 // that `self.count` won't overflow early in the constructor. 525 Ok(unsafe { core::slice::from_raw_parts(self.start_ptr().add(offset), count) }) 526 } 527 528 /// Performs the same functionality as [`CoherentAllocation::as_slice`], except that a mutable 529 /// slice is returned. 530 /// 531 /// # Safety 532 /// 533 /// * Callers must ensure that the device does not read/write to/from memory while the returned 534 /// slice is live. 535 /// * Callers must ensure that this call does not race with a read or write to the same region 536 /// while the returned slice is live. 537 pub unsafe fn as_slice_mut(&mut self, offset: usize, count: usize) -> Result<&mut [T]> { 538 self.validate_range(offset, count)?; 539 // SAFETY: 540 // - The pointer is valid due to type invariant on `CoherentAllocation`, 541 // we've just checked that the range and index is within bounds. The immutability of the 542 // data is also guaranteed by the safety requirements of the function. 543 // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked 544 // that `self.count` won't overflow early in the constructor. 545 Ok(unsafe { core::slice::from_raw_parts_mut(self.start_ptr_mut().add(offset), count) }) 546 } 547 548 /// Writes data to the region starting from `offset`. `offset` is in units of `T`, not the 549 /// number of bytes. 550 /// 551 /// # Safety 552 /// 553 /// * Callers must ensure that this call does not race with a read or write to the same region 554 /// that overlaps with this write. 555 /// 556 /// # Examples 557 /// 558 /// ``` 559 /// # fn test(alloc: &mut kernel::dma::CoherentAllocation<u8>) -> Result { 560 /// let somedata: [u8; 4] = [0xf; 4]; 561 /// let buf: &[u8] = &somedata; 562 /// // SAFETY: There is no concurrent HW operation on the device and no other R/W access to the 563 /// // region. 564 /// unsafe { alloc.write(buf, 0)?; } 565 /// # Ok::<(), Error>(()) } 566 /// ``` 567 pub unsafe fn write(&mut self, src: &[T], offset: usize) -> Result { 568 self.validate_range(offset, src.len())?; 569 // SAFETY: 570 // - The pointer is valid due to type invariant on `CoherentAllocation` 571 // and we've just checked that the range and index is within bounds. 572 // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked 573 // that `self.count` won't overflow early in the constructor. 574 unsafe { 575 core::ptr::copy_nonoverlapping( 576 src.as_ptr(), 577 self.start_ptr_mut().add(offset), 578 src.len(), 579 ) 580 }; 581 Ok(()) 582 } 583 584 /// Returns a pointer to an element from the region with bounds checking. `offset` is in 585 /// units of `T`, not the number of bytes. 586 /// 587 /// Public but hidden since it should only be used from [`dma_read`] and [`dma_write`] macros. 588 #[doc(hidden)] 589 pub fn item_from_index(&self, offset: usize) -> Result<*mut T> { 590 if offset >= self.count { 591 return Err(EINVAL); 592 } 593 // SAFETY: 594 // - The pointer is valid due to type invariant on `CoherentAllocation` 595 // and we've just checked that the range and index is within bounds. 596 // - `offset` can't overflow since it is smaller than `self.count` and we've checked 597 // that `self.count` won't overflow early in the constructor. 598 Ok(unsafe { self.cpu_addr.as_ptr().add(offset) }) 599 } 600 601 /// Reads the value of `field` and ensures that its type is [`FromBytes`]. 602 /// 603 /// # Safety 604 /// 605 /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is 606 /// validated beforehand. 607 /// 608 /// Public but hidden since it should only be used from [`dma_read`] macro. 609 #[doc(hidden)] 610 pub unsafe fn field_read<F: FromBytes>(&self, field: *const F) -> F { 611 // SAFETY: 612 // - By the safety requirements field is valid. 613 // - Using read_volatile() here is not sound as per the usual rules, the usage here is 614 // a special exception with the following notes in place. When dealing with a potential 615 // race from a hardware or code outside kernel (e.g. user-space program), we need that 616 // read on a valid memory is not UB. Currently read_volatile() is used for this, and the 617 // rationale behind is that it should generate the same code as READ_ONCE() which the 618 // kernel already relies on to avoid UB on data races. Note that the usage of 619 // read_volatile() is limited to this particular case, it cannot be used to prevent 620 // the UB caused by racing between two kernel functions nor do they provide atomicity. 621 unsafe { field.read_volatile() } 622 } 623 624 /// Writes a value to `field` and ensures that its type is [`AsBytes`]. 625 /// 626 /// # Safety 627 /// 628 /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is 629 /// validated beforehand. 630 /// 631 /// Public but hidden since it should only be used from [`dma_write`] macro. 632 #[doc(hidden)] 633 pub unsafe fn field_write<F: AsBytes>(&self, field: *mut F, val: F) { 634 // SAFETY: 635 // - By the safety requirements field is valid. 636 // - Using write_volatile() here is not sound as per the usual rules, the usage here is 637 // a special exception with the following notes in place. When dealing with a potential 638 // race from a hardware or code outside kernel (e.g. user-space program), we need that 639 // write on a valid memory is not UB. Currently write_volatile() is used for this, and the 640 // rationale behind is that it should generate the same code as WRITE_ONCE() which the 641 // kernel already relies on to avoid UB on data races. Note that the usage of 642 // write_volatile() is limited to this particular case, it cannot be used to prevent 643 // the UB caused by racing between two kernel functions nor do they provide atomicity. 644 unsafe { field.write_volatile(val) } 645 } 646 } 647 648 /// Note that the device configured to do DMA must be halted before this object is dropped. 649 impl<T: AsBytes + FromBytes> Drop for CoherentAllocation<T> { 650 fn drop(&mut self) { 651 let size = self.count * core::mem::size_of::<T>(); 652 // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. 653 // The cpu address, and the dma handle are valid due to the type invariants on 654 // `CoherentAllocation`. 655 unsafe { 656 bindings::dma_free_attrs( 657 self.dev.as_raw(), 658 size, 659 self.start_ptr_mut().cast(), 660 self.dma_handle, 661 self.dma_attrs.as_raw(), 662 ) 663 } 664 } 665 } 666 667 // SAFETY: It is safe to send a `CoherentAllocation` to another thread if `T` 668 // can be sent to another thread. 669 unsafe impl<T: AsBytes + FromBytes + Send> Send for CoherentAllocation<T> {} 670 671 /// Reads a field of an item from an allocated region of structs. 672 /// 673 /// # Examples 674 /// 675 /// ``` 676 /// use kernel::device::Device; 677 /// use kernel::dma::{attrs::*, CoherentAllocation}; 678 /// 679 /// struct MyStruct { field: u32, } 680 /// 681 /// // SAFETY: All bit patterns are acceptable values for `MyStruct`. 682 /// unsafe impl kernel::transmute::FromBytes for MyStruct{}; 683 /// // SAFETY: Instances of `MyStruct` have no uninitialized portions. 684 /// unsafe impl kernel::transmute::AsBytes for MyStruct{}; 685 /// 686 /// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result { 687 /// let whole = kernel::dma_read!(alloc[2]); 688 /// let field = kernel::dma_read!(alloc[1].field); 689 /// # Ok::<(), Error>(()) } 690 /// ``` 691 #[macro_export] 692 macro_rules! dma_read { 693 ($dma:expr, $idx: expr, $($field:tt)*) => {{ 694 (|| -> ::core::result::Result<_, $crate::error::Error> { 695 let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; 696 // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be 697 // dereferenced. The compiler also further validates the expression on whether `field` 698 // is a member of `item` when expanded by the macro. 699 unsafe { 700 let ptr_field = ::core::ptr::addr_of!((*item) $($field)*); 701 ::core::result::Result::Ok( 702 $crate::dma::CoherentAllocation::field_read(&$dma, ptr_field) 703 ) 704 } 705 })() 706 }}; 707 ($dma:ident [ $idx:expr ] $($field:tt)* ) => { 708 $crate::dma_read!($dma, $idx, $($field)*) 709 }; 710 ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => { 711 $crate::dma_read!($($dma).*, $idx, $($field)*) 712 }; 713 } 714 715 /// Writes to a field of an item from an allocated region of structs. 716 /// 717 /// # Examples 718 /// 719 /// ``` 720 /// use kernel::device::Device; 721 /// use kernel::dma::{attrs::*, CoherentAllocation}; 722 /// 723 /// struct MyStruct { member: u32, } 724 /// 725 /// // SAFETY: All bit patterns are acceptable values for `MyStruct`. 726 /// unsafe impl kernel::transmute::FromBytes for MyStruct{}; 727 /// // SAFETY: Instances of `MyStruct` have no uninitialized portions. 728 /// unsafe impl kernel::transmute::AsBytes for MyStruct{}; 729 /// 730 /// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result { 731 /// kernel::dma_write!(alloc[2].member = 0xf); 732 /// kernel::dma_write!(alloc[1] = MyStruct { member: 0xf }); 733 /// # Ok::<(), Error>(()) } 734 /// ``` 735 #[macro_export] 736 macro_rules! dma_write { 737 ($dma:ident [ $idx:expr ] $($field:tt)*) => {{ 738 $crate::dma_write!($dma, $idx, $($field)*) 739 }}; 740 ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {{ 741 $crate::dma_write!($($dma).*, $idx, $($field)*) 742 }}; 743 ($dma:expr, $idx: expr, = $val:expr) => { 744 (|| -> ::core::result::Result<_, $crate::error::Error> { 745 let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; 746 // SAFETY: `item_from_index` ensures that `item` is always a valid item. 747 unsafe { $crate::dma::CoherentAllocation::field_write(&$dma, item, $val) } 748 ::core::result::Result::Ok(()) 749 })() 750 }; 751 ($dma:expr, $idx: expr, $(.$field:ident)* = $val:expr) => { 752 (|| -> ::core::result::Result<_, $crate::error::Error> { 753 let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; 754 // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be 755 // dereferenced. The compiler also further validates the expression on whether `field` 756 // is a member of `item` when expanded by the macro. 757 unsafe { 758 let ptr_field = ::core::ptr::addr_of_mut!((*item) $(.$field)*); 759 $crate::dma::CoherentAllocation::field_write(&$dma, ptr_field, $val) 760 } 761 ::core::result::Result::Ok(()) 762 })() 763 }; 764 } 765