1 // SPDX-License-Identifier: GPL-2.0 2 3 //! Direct memory access (DMA). 4 //! 5 //! C header: [`include/linux/dma-mapping.h`](srctree/include/linux/dma-mapping.h) 6 7 use crate::{ 8 bindings, build_assert, device, 9 device::{Bound, Core}, 10 error::{to_result, Result}, 11 prelude::*, 12 sync::aref::ARef, 13 transmute::{AsBytes, FromBytes}, 14 }; 15 16 /// DMA address type. 17 /// 18 /// Represents a bus address used for Direct Memory Access (DMA) operations. 19 /// 20 /// This is an alias of the kernel's `dma_addr_t`, which may be `u32` or `u64` depending on 21 /// `CONFIG_ARCH_DMA_ADDR_T_64BIT`. 22 /// 23 /// Note that this may be `u64` even on 32-bit architectures. 24 pub type DmaAddress = bindings::dma_addr_t; 25 26 /// Trait to be implemented by DMA capable bus devices. 27 /// 28 /// The [`dma::Device`](Device) trait should be implemented by bus specific device representations, 29 /// where the underlying bus is DMA capable, such as [`pci::Device`](::kernel::pci::Device) or 30 /// [`platform::Device`](::kernel::platform::Device). 31 pub trait Device: AsRef<device::Device<Core>> { 32 /// Set up the device's DMA streaming addressing capabilities. 33 /// 34 /// This method is usually called once from `probe()` as soon as the device capabilities are 35 /// known. 36 /// 37 /// # Safety 38 /// 39 /// This method must not be called concurrently with any DMA allocation or mapping primitives, 40 /// such as [`CoherentAllocation::alloc_attrs`]. 41 unsafe fn dma_set_mask(&self, mask: DmaMask) -> Result { 42 // SAFETY: 43 // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. 44 // - The safety requirement of this function guarantees that there are no concurrent calls 45 // to DMA allocation and mapping primitives using this mask. 46 to_result(unsafe { bindings::dma_set_mask(self.as_ref().as_raw(), mask.value()) }) 47 } 48 49 /// Set up the device's DMA coherent addressing capabilities. 50 /// 51 /// This method is usually called once from `probe()` as soon as the device capabilities are 52 /// known. 53 /// 54 /// # Safety 55 /// 56 /// This method must not be called concurrently with any DMA allocation or mapping primitives, 57 /// such as [`CoherentAllocation::alloc_attrs`]. 58 unsafe fn dma_set_coherent_mask(&self, mask: DmaMask) -> Result { 59 // SAFETY: 60 // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. 61 // - The safety requirement of this function guarantees that there are no concurrent calls 62 // to DMA allocation and mapping primitives using this mask. 63 to_result(unsafe { bindings::dma_set_coherent_mask(self.as_ref().as_raw(), mask.value()) }) 64 } 65 66 /// Set up the device's DMA addressing capabilities. 67 /// 68 /// This is a combination of [`Device::dma_set_mask`] and [`Device::dma_set_coherent_mask`]. 69 /// 70 /// This method is usually called once from `probe()` as soon as the device capabilities are 71 /// known. 72 /// 73 /// # Safety 74 /// 75 /// This method must not be called concurrently with any DMA allocation or mapping primitives, 76 /// such as [`CoherentAllocation::alloc_attrs`]. 77 unsafe fn dma_set_mask_and_coherent(&self, mask: DmaMask) -> Result { 78 // SAFETY: 79 // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. 80 // - The safety requirement of this function guarantees that there are no concurrent calls 81 // to DMA allocation and mapping primitives using this mask. 82 to_result(unsafe { 83 bindings::dma_set_mask_and_coherent(self.as_ref().as_raw(), mask.value()) 84 }) 85 } 86 } 87 88 /// A DMA mask that holds a bitmask with the lowest `n` bits set. 89 /// 90 /// Use [`DmaMask::new`] or [`DmaMask::try_new`] to construct a value. Values 91 /// are guaranteed to never exceed the bit width of `u64`. 92 /// 93 /// This is the Rust equivalent of the C macro `DMA_BIT_MASK()`. 94 #[derive(Debug, Clone, Copy, PartialEq, Eq)] 95 pub struct DmaMask(u64); 96 97 impl DmaMask { 98 /// Constructs a `DmaMask` with the lowest `n` bits set to `1`. 99 /// 100 /// For `n <= 64`, sets exactly the lowest `n` bits. 101 /// For `n > 64`, results in a build error. 102 /// 103 /// # Examples 104 /// 105 /// ``` 106 /// use kernel::dma::DmaMask; 107 /// 108 /// let mask0 = DmaMask::new::<0>(); 109 /// assert_eq!(mask0.value(), 0); 110 /// 111 /// let mask1 = DmaMask::new::<1>(); 112 /// assert_eq!(mask1.value(), 0b1); 113 /// 114 /// let mask64 = DmaMask::new::<64>(); 115 /// assert_eq!(mask64.value(), u64::MAX); 116 /// 117 /// // Build failure. 118 /// // let mask_overflow = DmaMask::new::<100>(); 119 /// ``` 120 #[inline] 121 pub const fn new<const N: u32>() -> Self { 122 let Ok(mask) = Self::try_new(N) else { 123 build_error!("Invalid DMA Mask."); 124 }; 125 126 mask 127 } 128 129 /// Constructs a `DmaMask` with the lowest `n` bits set to `1`. 130 /// 131 /// For `n <= 64`, sets exactly the lowest `n` bits. 132 /// For `n > 64`, returns [`EINVAL`]. 133 /// 134 /// # Examples 135 /// 136 /// ``` 137 /// use kernel::dma::DmaMask; 138 /// 139 /// let mask0 = DmaMask::try_new(0)?; 140 /// assert_eq!(mask0.value(), 0); 141 /// 142 /// let mask1 = DmaMask::try_new(1)?; 143 /// assert_eq!(mask1.value(), 0b1); 144 /// 145 /// let mask64 = DmaMask::try_new(64)?; 146 /// assert_eq!(mask64.value(), u64::MAX); 147 /// 148 /// let mask_overflow = DmaMask::try_new(100); 149 /// assert!(mask_overflow.is_err()); 150 /// # Ok::<(), Error>(()) 151 /// ``` 152 #[inline] 153 pub const fn try_new(n: u32) -> Result<Self> { 154 Ok(Self(match n { 155 0 => 0, 156 1..=64 => u64::MAX >> (64 - n), 157 _ => return Err(EINVAL), 158 })) 159 } 160 161 /// Returns the underlying `u64` bitmask value. 162 #[inline] 163 pub const fn value(&self) -> u64 { 164 self.0 165 } 166 } 167 168 /// Possible attributes associated with a DMA mapping. 169 /// 170 /// They can be combined with the operators `|`, `&`, and `!`. 171 /// 172 /// Values can be used from the [`attrs`] module. 173 /// 174 /// # Examples 175 /// 176 /// ``` 177 /// # use kernel::device::{Bound, Device}; 178 /// use kernel::dma::{attrs::*, CoherentAllocation}; 179 /// 180 /// # fn test(dev: &Device<Bound>) -> Result { 181 /// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN; 182 /// let c: CoherentAllocation<u64> = 183 /// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, attribs)?; 184 /// # Ok::<(), Error>(()) } 185 /// ``` 186 #[derive(Clone, Copy, PartialEq)] 187 #[repr(transparent)] 188 pub struct Attrs(u32); 189 190 impl Attrs { 191 /// Get the raw representation of this attribute. 192 pub(crate) fn as_raw(self) -> crate::ffi::c_ulong { 193 self.0 as crate::ffi::c_ulong 194 } 195 196 /// Check whether `flags` is contained in `self`. 197 pub fn contains(self, flags: Attrs) -> bool { 198 (self & flags) == flags 199 } 200 } 201 202 impl core::ops::BitOr for Attrs { 203 type Output = Self; 204 fn bitor(self, rhs: Self) -> Self::Output { 205 Self(self.0 | rhs.0) 206 } 207 } 208 209 impl core::ops::BitAnd for Attrs { 210 type Output = Self; 211 fn bitand(self, rhs: Self) -> Self::Output { 212 Self(self.0 & rhs.0) 213 } 214 } 215 216 impl core::ops::Not for Attrs { 217 type Output = Self; 218 fn not(self) -> Self::Output { 219 Self(!self.0) 220 } 221 } 222 223 /// DMA mapping attributes. 224 pub mod attrs { 225 use super::Attrs; 226 227 /// Specifies that reads and writes to the mapping may be weakly ordered, that is that reads 228 /// and writes may pass each other. 229 pub const DMA_ATTR_WEAK_ORDERING: Attrs = Attrs(bindings::DMA_ATTR_WEAK_ORDERING); 230 231 /// Specifies that writes to the mapping may be buffered to improve performance. 232 pub const DMA_ATTR_WRITE_COMBINE: Attrs = Attrs(bindings::DMA_ATTR_WRITE_COMBINE); 233 234 /// Lets the platform to avoid creating a kernel virtual mapping for the allocated buffer. 235 pub const DMA_ATTR_NO_KERNEL_MAPPING: Attrs = Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING); 236 237 /// Allows platform code to skip synchronization of the CPU cache for the given buffer assuming 238 /// that it has been already transferred to 'device' domain. 239 pub const DMA_ATTR_SKIP_CPU_SYNC: Attrs = Attrs(bindings::DMA_ATTR_SKIP_CPU_SYNC); 240 241 /// Forces contiguous allocation of the buffer in physical memory. 242 pub const DMA_ATTR_FORCE_CONTIGUOUS: Attrs = Attrs(bindings::DMA_ATTR_FORCE_CONTIGUOUS); 243 244 /// Hints DMA-mapping subsystem that it's probably not worth the time to try 245 /// to allocate memory to in a way that gives better TLB efficiency. 246 pub const DMA_ATTR_ALLOC_SINGLE_PAGES: Attrs = Attrs(bindings::DMA_ATTR_ALLOC_SINGLE_PAGES); 247 248 /// This tells the DMA-mapping subsystem to suppress allocation failure reports (similarly to 249 /// `__GFP_NOWARN`). 250 pub const DMA_ATTR_NO_WARN: Attrs = Attrs(bindings::DMA_ATTR_NO_WARN); 251 252 /// Indicates that the buffer is fully accessible at an elevated privilege level (and 253 /// ideally inaccessible or at least read-only at lesser-privileged levels). 254 pub const DMA_ATTR_PRIVILEGED: Attrs = Attrs(bindings::DMA_ATTR_PRIVILEGED); 255 } 256 257 /// DMA data direction. 258 /// 259 /// Corresponds to the C [`enum dma_data_direction`]. 260 /// 261 /// [`enum dma_data_direction`]: srctree/include/linux/dma-direction.h 262 #[derive(Copy, Clone, PartialEq, Eq, Debug)] 263 #[repr(u32)] 264 pub enum DataDirection { 265 /// The DMA mapping is for bidirectional data transfer. 266 /// 267 /// This is used when the buffer can be both read from and written to by the device. 268 /// The cache for the corresponding memory region is both flushed and invalidated. 269 Bidirectional = Self::const_cast(bindings::dma_data_direction_DMA_BIDIRECTIONAL), 270 271 /// The DMA mapping is for data transfer from memory to the device (write). 272 /// 273 /// The CPU has prepared data in the buffer, and the device will read it. 274 /// The cache for the corresponding memory region is flushed before device access. 275 ToDevice = Self::const_cast(bindings::dma_data_direction_DMA_TO_DEVICE), 276 277 /// The DMA mapping is for data transfer from the device to memory (read). 278 /// 279 /// The device will write data into the buffer for the CPU to read. 280 /// The cache for the corresponding memory region is invalidated before CPU access. 281 FromDevice = Self::const_cast(bindings::dma_data_direction_DMA_FROM_DEVICE), 282 283 /// The DMA mapping is not for data transfer. 284 /// 285 /// This is primarily for debugging purposes. With this direction, the DMA mapping API 286 /// will not perform any cache coherency operations. 287 None = Self::const_cast(bindings::dma_data_direction_DMA_NONE), 288 } 289 290 impl DataDirection { 291 /// Casts the bindgen-generated enum type to a `u32` at compile time. 292 /// 293 /// This function will cause a compile-time error if the underlying value of the 294 /// C enum is out of bounds for `u32`. 295 const fn const_cast(val: bindings::dma_data_direction) -> u32 { 296 // CAST: The C standard allows compilers to choose different integer types for enums. 297 // To safely check the value, we cast it to a wide signed integer type (`i128`) 298 // which can hold any standard C integer enum type without truncation. 299 let wide_val = val as i128; 300 301 // Check if the value is outside the valid range for the target type `u32`. 302 // CAST: `u32::MAX` is cast to `i128` to match the type of `wide_val` for the comparison. 303 if wide_val < 0 || wide_val > u32::MAX as i128 { 304 // Trigger a compile-time error in a const context. 305 build_error!("C enum value is out of bounds for the target type `u32`."); 306 } 307 308 // CAST: This cast is valid because the check above guarantees that `wide_val` 309 // is within the representable range of `u32`. 310 wide_val as u32 311 } 312 } 313 314 impl From<DataDirection> for bindings::dma_data_direction { 315 /// Returns the raw representation of [`enum dma_data_direction`]. 316 fn from(direction: DataDirection) -> Self { 317 // CAST: `direction as u32` gets the underlying representation of our `#[repr(u32)]` enum. 318 // The subsequent cast to `Self` (the bindgen type) assumes the C enum is compatible 319 // with the enum variants of `DataDirection`, which is a valid assumption given our 320 // compile-time checks. 321 direction as u32 as Self 322 } 323 } 324 325 /// An abstraction of the `dma_alloc_coherent` API. 326 /// 327 /// This is an abstraction around the `dma_alloc_coherent` API which is used to allocate and map 328 /// large coherent DMA regions. 329 /// 330 /// A [`CoherentAllocation`] instance contains a pointer to the allocated region (in the 331 /// processor's virtual address space) and the device address which can be given to the device 332 /// as the DMA address base of the region. The region is released once [`CoherentAllocation`] 333 /// is dropped. 334 /// 335 /// # Invariants 336 /// 337 /// - For the lifetime of an instance of [`CoherentAllocation`], the `cpu_addr` is a valid pointer 338 /// to an allocated region of coherent memory and `dma_handle` is the DMA address base of the 339 /// region. 340 /// - The size in bytes of the allocation is equal to `size_of::<T> * count`. 341 /// - `size_of::<T> * count` fits into a `usize`. 342 // TODO 343 // 344 // DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness 345 // reasons DMA allocation would need to be embedded in a `Devres` container, in order to ensure 346 // that device resources can never survive device unbind. 347 // 348 // However, it is neither desirable nor necessary to protect the allocated memory of the DMA 349 // allocation from surviving device unbind; it would require RCU read side critical sections to 350 // access the memory, which may require subsequent unnecessary copies. 351 // 352 // Hence, find a way to revoke the device resources of a `CoherentAllocation`, but not the 353 // entire `CoherentAllocation` including the allocated memory itself. 354 pub struct CoherentAllocation<T: AsBytes + FromBytes> { 355 dev: ARef<device::Device>, 356 dma_handle: DmaAddress, 357 count: usize, 358 cpu_addr: *mut T, 359 dma_attrs: Attrs, 360 } 361 362 impl<T: AsBytes + FromBytes> CoherentAllocation<T> { 363 /// Allocates a region of `size_of::<T> * count` of coherent memory. 364 /// 365 /// # Examples 366 /// 367 /// ``` 368 /// # use kernel::device::{Bound, Device}; 369 /// use kernel::dma::{attrs::*, CoherentAllocation}; 370 /// 371 /// # fn test(dev: &Device<Bound>) -> Result { 372 /// let c: CoherentAllocation<u64> = 373 /// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?; 374 /// # Ok::<(), Error>(()) } 375 /// ``` 376 pub fn alloc_attrs( 377 dev: &device::Device<Bound>, 378 count: usize, 379 gfp_flags: kernel::alloc::Flags, 380 dma_attrs: Attrs, 381 ) -> Result<CoherentAllocation<T>> { 382 build_assert!( 383 core::mem::size_of::<T>() > 0, 384 "It doesn't make sense for the allocated type to be a ZST" 385 ); 386 387 let size = count 388 .checked_mul(core::mem::size_of::<T>()) 389 .ok_or(EOVERFLOW)?; 390 let mut dma_handle = 0; 391 // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. 392 let ret = unsafe { 393 bindings::dma_alloc_attrs( 394 dev.as_raw(), 395 size, 396 &mut dma_handle, 397 gfp_flags.as_raw(), 398 dma_attrs.as_raw(), 399 ) 400 }; 401 if ret.is_null() { 402 return Err(ENOMEM); 403 } 404 // INVARIANT: 405 // - We just successfully allocated a coherent region which is accessible for 406 // `count` elements, hence the cpu address is valid. We also hold a refcounted reference 407 // to the device. 408 // - The allocated `size` is equal to `size_of::<T> * count`. 409 // - The allocated `size` fits into a `usize`. 410 Ok(Self { 411 dev: dev.into(), 412 dma_handle, 413 count, 414 cpu_addr: ret.cast::<T>(), 415 dma_attrs, 416 }) 417 } 418 419 /// Performs the same functionality as [`CoherentAllocation::alloc_attrs`], except the 420 /// `dma_attrs` is 0 by default. 421 pub fn alloc_coherent( 422 dev: &device::Device<Bound>, 423 count: usize, 424 gfp_flags: kernel::alloc::Flags, 425 ) -> Result<CoherentAllocation<T>> { 426 CoherentAllocation::alloc_attrs(dev, count, gfp_flags, Attrs(0)) 427 } 428 429 /// Returns the number of elements `T` in this allocation. 430 /// 431 /// Note that this is not the size of the allocation in bytes, which is provided by 432 /// [`Self::size`]. 433 pub fn count(&self) -> usize { 434 self.count 435 } 436 437 /// Returns the size in bytes of this allocation. 438 pub fn size(&self) -> usize { 439 // INVARIANT: The type invariant of `Self` guarantees that `size_of::<T> * count` fits into 440 // a `usize`. 441 self.count * core::mem::size_of::<T>() 442 } 443 444 /// Returns the base address to the allocated region in the CPU's virtual address space. 445 pub fn start_ptr(&self) -> *const T { 446 self.cpu_addr 447 } 448 449 /// Returns the base address to the allocated region in the CPU's virtual address space as 450 /// a mutable pointer. 451 pub fn start_ptr_mut(&mut self) -> *mut T { 452 self.cpu_addr 453 } 454 455 /// Returns a DMA handle which may be given to the device as the DMA address base of 456 /// the region. 457 pub fn dma_handle(&self) -> DmaAddress { 458 self.dma_handle 459 } 460 461 /// Returns a DMA handle starting at `offset` (in units of `T`) which may be given to the 462 /// device as the DMA address base of the region. 463 /// 464 /// Returns `EINVAL` if `offset` is not within the bounds of the allocation. 465 pub fn dma_handle_with_offset(&self, offset: usize) -> Result<DmaAddress> { 466 if offset >= self.count { 467 Err(EINVAL) 468 } else { 469 // INVARIANT: The type invariant of `Self` guarantees that `size_of::<T> * count` fits 470 // into a `usize`, and `offset` is inferior to `count`. 471 Ok(self.dma_handle + (offset * core::mem::size_of::<T>()) as DmaAddress) 472 } 473 } 474 475 /// Common helper to validate a range applied from the allocated region in the CPU's virtual 476 /// address space. 477 fn validate_range(&self, offset: usize, count: usize) -> Result { 478 if offset.checked_add(count).ok_or(EOVERFLOW)? > self.count { 479 return Err(EINVAL); 480 } 481 Ok(()) 482 } 483 484 /// Returns the data from the region starting from `offset` as a slice. 485 /// `offset` and `count` are in units of `T`, not the number of bytes. 486 /// 487 /// For ringbuffer type of r/w access or use-cases where the pointer to the live data is needed, 488 /// [`CoherentAllocation::start_ptr`] or [`CoherentAllocation::start_ptr_mut`] could be used 489 /// instead. 490 /// 491 /// # Safety 492 /// 493 /// * Callers must ensure that the device does not read/write to/from memory while the returned 494 /// slice is live. 495 /// * Callers must ensure that this call does not race with a write to the same region while 496 /// the returned slice is live. 497 pub unsafe fn as_slice(&self, offset: usize, count: usize) -> Result<&[T]> { 498 self.validate_range(offset, count)?; 499 // SAFETY: 500 // - The pointer is valid due to type invariant on `CoherentAllocation`, 501 // we've just checked that the range and index is within bounds. The immutability of the 502 // data is also guaranteed by the safety requirements of the function. 503 // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked 504 // that `self.count` won't overflow early in the constructor. 505 Ok(unsafe { core::slice::from_raw_parts(self.cpu_addr.add(offset), count) }) 506 } 507 508 /// Performs the same functionality as [`CoherentAllocation::as_slice`], except that a mutable 509 /// slice is returned. 510 /// 511 /// # Safety 512 /// 513 /// * Callers must ensure that the device does not read/write to/from memory while the returned 514 /// slice is live. 515 /// * Callers must ensure that this call does not race with a read or write to the same region 516 /// while the returned slice is live. 517 pub unsafe fn as_slice_mut(&mut self, offset: usize, count: usize) -> Result<&mut [T]> { 518 self.validate_range(offset, count)?; 519 // SAFETY: 520 // - The pointer is valid due to type invariant on `CoherentAllocation`, 521 // we've just checked that the range and index is within bounds. The immutability of the 522 // data is also guaranteed by the safety requirements of the function. 523 // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked 524 // that `self.count` won't overflow early in the constructor. 525 Ok(unsafe { core::slice::from_raw_parts_mut(self.cpu_addr.add(offset), count) }) 526 } 527 528 /// Writes data to the region starting from `offset`. `offset` is in units of `T`, not the 529 /// number of bytes. 530 /// 531 /// # Safety 532 /// 533 /// * Callers must ensure that the device does not read/write to/from memory while the returned 534 /// slice is live. 535 /// * Callers must ensure that this call does not race with a read or write to the same region 536 /// that overlaps with this write. 537 /// 538 /// # Examples 539 /// 540 /// ``` 541 /// # fn test(alloc: &mut kernel::dma::CoherentAllocation<u8>) -> Result { 542 /// let somedata: [u8; 4] = [0xf; 4]; 543 /// let buf: &[u8] = &somedata; 544 /// // SAFETY: There is no concurrent HW operation on the device and no other R/W access to the 545 /// // region. 546 /// unsafe { alloc.write(buf, 0)?; } 547 /// # Ok::<(), Error>(()) } 548 /// ``` 549 pub unsafe fn write(&mut self, src: &[T], offset: usize) -> Result { 550 self.validate_range(offset, src.len())?; 551 // SAFETY: 552 // - The pointer is valid due to type invariant on `CoherentAllocation` 553 // and we've just checked that the range and index is within bounds. 554 // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked 555 // that `self.count` won't overflow early in the constructor. 556 unsafe { 557 core::ptr::copy_nonoverlapping(src.as_ptr(), self.cpu_addr.add(offset), src.len()) 558 }; 559 Ok(()) 560 } 561 562 /// Returns a pointer to an element from the region with bounds checking. `offset` is in 563 /// units of `T`, not the number of bytes. 564 /// 565 /// Public but hidden since it should only be used from [`dma_read`] and [`dma_write`] macros. 566 #[doc(hidden)] 567 pub fn item_from_index(&self, offset: usize) -> Result<*mut T> { 568 if offset >= self.count { 569 return Err(EINVAL); 570 } 571 // SAFETY: 572 // - The pointer is valid due to type invariant on `CoherentAllocation` 573 // and we've just checked that the range and index is within bounds. 574 // - `offset` can't overflow since it is smaller than `self.count` and we've checked 575 // that `self.count` won't overflow early in the constructor. 576 Ok(unsafe { self.cpu_addr.add(offset) }) 577 } 578 579 /// Reads the value of `field` and ensures that its type is [`FromBytes`]. 580 /// 581 /// # Safety 582 /// 583 /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is 584 /// validated beforehand. 585 /// 586 /// Public but hidden since it should only be used from [`dma_read`] macro. 587 #[doc(hidden)] 588 pub unsafe fn field_read<F: FromBytes>(&self, field: *const F) -> F { 589 // SAFETY: 590 // - By the safety requirements field is valid. 591 // - Using read_volatile() here is not sound as per the usual rules, the usage here is 592 // a special exception with the following notes in place. When dealing with a potential 593 // race from a hardware or code outside kernel (e.g. user-space program), we need that 594 // read on a valid memory is not UB. Currently read_volatile() is used for this, and the 595 // rationale behind is that it should generate the same code as READ_ONCE() which the 596 // kernel already relies on to avoid UB on data races. Note that the usage of 597 // read_volatile() is limited to this particular case, it cannot be used to prevent 598 // the UB caused by racing between two kernel functions nor do they provide atomicity. 599 unsafe { field.read_volatile() } 600 } 601 602 /// Writes a value to `field` and ensures that its type is [`AsBytes`]. 603 /// 604 /// # Safety 605 /// 606 /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is 607 /// validated beforehand. 608 /// 609 /// Public but hidden since it should only be used from [`dma_write`] macro. 610 #[doc(hidden)] 611 pub unsafe fn field_write<F: AsBytes>(&self, field: *mut F, val: F) { 612 // SAFETY: 613 // - By the safety requirements field is valid. 614 // - Using write_volatile() here is not sound as per the usual rules, the usage here is 615 // a special exception with the following notes in place. When dealing with a potential 616 // race from a hardware or code outside kernel (e.g. user-space program), we need that 617 // write on a valid memory is not UB. Currently write_volatile() is used for this, and the 618 // rationale behind is that it should generate the same code as WRITE_ONCE() which the 619 // kernel already relies on to avoid UB on data races. Note that the usage of 620 // write_volatile() is limited to this particular case, it cannot be used to prevent 621 // the UB caused by racing between two kernel functions nor do they provide atomicity. 622 unsafe { field.write_volatile(val) } 623 } 624 } 625 626 /// Note that the device configured to do DMA must be halted before this object is dropped. 627 impl<T: AsBytes + FromBytes> Drop for CoherentAllocation<T> { 628 fn drop(&mut self) { 629 let size = self.count * core::mem::size_of::<T>(); 630 // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. 631 // The cpu address, and the dma handle are valid due to the type invariants on 632 // `CoherentAllocation`. 633 unsafe { 634 bindings::dma_free_attrs( 635 self.dev.as_raw(), 636 size, 637 self.cpu_addr.cast(), 638 self.dma_handle, 639 self.dma_attrs.as_raw(), 640 ) 641 } 642 } 643 } 644 645 // SAFETY: It is safe to send a `CoherentAllocation` to another thread if `T` 646 // can be sent to another thread. 647 unsafe impl<T: AsBytes + FromBytes + Send> Send for CoherentAllocation<T> {} 648 649 /// Reads a field of an item from an allocated region of structs. 650 /// 651 /// # Examples 652 /// 653 /// ``` 654 /// use kernel::device::Device; 655 /// use kernel::dma::{attrs::*, CoherentAllocation}; 656 /// 657 /// struct MyStruct { field: u32, } 658 /// 659 /// // SAFETY: All bit patterns are acceptable values for `MyStruct`. 660 /// unsafe impl kernel::transmute::FromBytes for MyStruct{}; 661 /// // SAFETY: Instances of `MyStruct` have no uninitialized portions. 662 /// unsafe impl kernel::transmute::AsBytes for MyStruct{}; 663 /// 664 /// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result { 665 /// let whole = kernel::dma_read!(alloc[2]); 666 /// let field = kernel::dma_read!(alloc[1].field); 667 /// # Ok::<(), Error>(()) } 668 /// ``` 669 #[macro_export] 670 macro_rules! dma_read { 671 ($dma:expr, $idx: expr, $($field:tt)*) => {{ 672 (|| -> ::core::result::Result<_, $crate::error::Error> { 673 let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; 674 // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be 675 // dereferenced. The compiler also further validates the expression on whether `field` 676 // is a member of `item` when expanded by the macro. 677 unsafe { 678 let ptr_field = ::core::ptr::addr_of!((*item) $($field)*); 679 ::core::result::Result::Ok( 680 $crate::dma::CoherentAllocation::field_read(&$dma, ptr_field) 681 ) 682 } 683 })() 684 }}; 685 ($dma:ident [ $idx:expr ] $($field:tt)* ) => { 686 $crate::dma_read!($dma, $idx, $($field)*) 687 }; 688 ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => { 689 $crate::dma_read!($($dma).*, $idx, $($field)*) 690 }; 691 } 692 693 /// Writes to a field of an item from an allocated region of structs. 694 /// 695 /// # Examples 696 /// 697 /// ``` 698 /// use kernel::device::Device; 699 /// use kernel::dma::{attrs::*, CoherentAllocation}; 700 /// 701 /// struct MyStruct { member: u32, } 702 /// 703 /// // SAFETY: All bit patterns are acceptable values for `MyStruct`. 704 /// unsafe impl kernel::transmute::FromBytes for MyStruct{}; 705 /// // SAFETY: Instances of `MyStruct` have no uninitialized portions. 706 /// unsafe impl kernel::transmute::AsBytes for MyStruct{}; 707 /// 708 /// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result { 709 /// kernel::dma_write!(alloc[2].member = 0xf); 710 /// kernel::dma_write!(alloc[1] = MyStruct { member: 0xf }); 711 /// # Ok::<(), Error>(()) } 712 /// ``` 713 #[macro_export] 714 macro_rules! dma_write { 715 ($dma:ident [ $idx:expr ] $($field:tt)*) => {{ 716 $crate::dma_write!($dma, $idx, $($field)*) 717 }}; 718 ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {{ 719 $crate::dma_write!($($dma).*, $idx, $($field)*) 720 }}; 721 ($dma:expr, $idx: expr, = $val:expr) => { 722 (|| -> ::core::result::Result<_, $crate::error::Error> { 723 let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; 724 // SAFETY: `item_from_index` ensures that `item` is always a valid item. 725 unsafe { $crate::dma::CoherentAllocation::field_write(&$dma, item, $val) } 726 ::core::result::Result::Ok(()) 727 })() 728 }; 729 ($dma:expr, $idx: expr, $(.$field:ident)* = $val:expr) => { 730 (|| -> ::core::result::Result<_, $crate::error::Error> { 731 let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; 732 // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be 733 // dereferenced. The compiler also further validates the expression on whether `field` 734 // is a member of `item` when expanded by the macro. 735 unsafe { 736 let ptr_field = ::core::ptr::addr_of_mut!((*item) $(.$field)*); 737 $crate::dma::CoherentAllocation::field_write(&$dma, ptr_field, $val) 738 } 739 ::core::result::Result::Ok(()) 740 })() 741 }; 742 } 743