1 // SPDX-License-Identifier: GPL-2.0 2 3 //! Direct memory access (DMA). 4 //! 5 //! C header: [`include/linux/dma-mapping.h`](srctree/include/linux/dma-mapping.h) 6 7 use crate::{ 8 bindings, build_assert, 9 device::{Bound, Device}, 10 error::code::*, 11 error::Result, 12 transmute::{AsBytes, FromBytes}, 13 types::ARef, 14 }; 15 16 /// Possible attributes associated with a DMA mapping. 17 /// 18 /// They can be combined with the operators `|`, `&`, and `!`. 19 /// 20 /// Values can be used from the [`attrs`] module. 21 /// 22 /// # Examples 23 /// 24 /// ``` 25 /// # use kernel::device::{Bound, Device}; 26 /// use kernel::dma::{attrs::*, CoherentAllocation}; 27 /// 28 /// # fn test(dev: &Device<Bound>) -> Result { 29 /// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN; 30 /// let c: CoherentAllocation<u64> = 31 /// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, attribs)?; 32 /// # Ok::<(), Error>(()) } 33 /// ``` 34 #[derive(Clone, Copy, PartialEq)] 35 #[repr(transparent)] 36 pub struct Attrs(u32); 37 38 impl Attrs { 39 /// Get the raw representation of this attribute. 40 pub(crate) fn as_raw(self) -> crate::ffi::c_ulong { 41 self.0 as _ 42 } 43 44 /// Check whether `flags` is contained in `self`. 45 pub fn contains(self, flags: Attrs) -> bool { 46 (self & flags) == flags 47 } 48 } 49 50 impl core::ops::BitOr for Attrs { 51 type Output = Self; 52 fn bitor(self, rhs: Self) -> Self::Output { 53 Self(self.0 | rhs.0) 54 } 55 } 56 57 impl core::ops::BitAnd for Attrs { 58 type Output = Self; 59 fn bitand(self, rhs: Self) -> Self::Output { 60 Self(self.0 & rhs.0) 61 } 62 } 63 64 impl core::ops::Not for Attrs { 65 type Output = Self; 66 fn not(self) -> Self::Output { 67 Self(!self.0) 68 } 69 } 70 71 /// DMA mapping attributes. 72 pub mod attrs { 73 use super::Attrs; 74 75 /// Specifies that reads and writes to the mapping may be weakly ordered, that is that reads 76 /// and writes may pass each other. 77 pub const DMA_ATTR_WEAK_ORDERING: Attrs = Attrs(bindings::DMA_ATTR_WEAK_ORDERING); 78 79 /// Specifies that writes to the mapping may be buffered to improve performance. 80 pub const DMA_ATTR_WRITE_COMBINE: Attrs = Attrs(bindings::DMA_ATTR_WRITE_COMBINE); 81 82 /// Lets the platform to avoid creating a kernel virtual mapping for the allocated buffer. 83 pub const DMA_ATTR_NO_KERNEL_MAPPING: Attrs = Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING); 84 85 /// Allows platform code to skip synchronization of the CPU cache for the given buffer assuming 86 /// that it has been already transferred to 'device' domain. 87 pub const DMA_ATTR_SKIP_CPU_SYNC: Attrs = Attrs(bindings::DMA_ATTR_SKIP_CPU_SYNC); 88 89 /// Forces contiguous allocation of the buffer in physical memory. 90 pub const DMA_ATTR_FORCE_CONTIGUOUS: Attrs = Attrs(bindings::DMA_ATTR_FORCE_CONTIGUOUS); 91 92 /// Hints DMA-mapping subsystem that it's probably not worth the time to try 93 /// to allocate memory to in a way that gives better TLB efficiency. 94 pub const DMA_ATTR_ALLOC_SINGLE_PAGES: Attrs = Attrs(bindings::DMA_ATTR_ALLOC_SINGLE_PAGES); 95 96 /// This tells the DMA-mapping subsystem to suppress allocation failure reports (similarly to 97 /// `__GFP_NOWARN`). 98 pub const DMA_ATTR_NO_WARN: Attrs = Attrs(bindings::DMA_ATTR_NO_WARN); 99 100 /// Indicates that the buffer is fully accessible at an elevated privilege level (and 101 /// ideally inaccessible or at least read-only at lesser-privileged levels). 102 pub const DMA_ATTR_PRIVILEGED: Attrs = Attrs(bindings::DMA_ATTR_PRIVILEGED); 103 } 104 105 /// An abstraction of the `dma_alloc_coherent` API. 106 /// 107 /// This is an abstraction around the `dma_alloc_coherent` API which is used to allocate and map 108 /// large coherent DMA regions. 109 /// 110 /// A [`CoherentAllocation`] instance contains a pointer to the allocated region (in the 111 /// processor's virtual address space) and the device address which can be given to the device 112 /// as the DMA address base of the region. The region is released once [`CoherentAllocation`] 113 /// is dropped. 114 /// 115 /// # Invariants 116 /// 117 /// - For the lifetime of an instance of [`CoherentAllocation`], the `cpu_addr` is a valid pointer 118 /// to an allocated region of coherent memory and `dma_handle` is the DMA address base of the 119 /// region. 120 /// - The size in bytes of the allocation is equal to `size_of::<T> * count`. 121 /// - `size_of::<T> * count` fits into a `usize`. 122 // TODO 123 // 124 // DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness 125 // reasons DMA allocation would need to be embedded in a `Devres` container, in order to ensure 126 // that device resources can never survive device unbind. 127 // 128 // However, it is neither desirable nor necessary to protect the allocated memory of the DMA 129 // allocation from surviving device unbind; it would require RCU read side critical sections to 130 // access the memory, which may require subsequent unnecessary copies. 131 // 132 // Hence, find a way to revoke the device resources of a `CoherentAllocation`, but not the 133 // entire `CoherentAllocation` including the allocated memory itself. 134 pub struct CoherentAllocation<T: AsBytes + FromBytes> { 135 dev: ARef<Device>, 136 dma_handle: bindings::dma_addr_t, 137 count: usize, 138 cpu_addr: *mut T, 139 dma_attrs: Attrs, 140 } 141 142 impl<T: AsBytes + FromBytes> CoherentAllocation<T> { 143 /// Allocates a region of `size_of::<T> * count` of coherent memory. 144 /// 145 /// # Examples 146 /// 147 /// ``` 148 /// # use kernel::device::{Bound, Device}; 149 /// use kernel::dma::{attrs::*, CoherentAllocation}; 150 /// 151 /// # fn test(dev: &Device<Bound>) -> Result { 152 /// let c: CoherentAllocation<u64> = 153 /// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?; 154 /// # Ok::<(), Error>(()) } 155 /// ``` 156 pub fn alloc_attrs( 157 dev: &Device<Bound>, 158 count: usize, 159 gfp_flags: kernel::alloc::Flags, 160 dma_attrs: Attrs, 161 ) -> Result<CoherentAllocation<T>> { 162 build_assert!( 163 core::mem::size_of::<T>() > 0, 164 "It doesn't make sense for the allocated type to be a ZST" 165 ); 166 167 let size = count 168 .checked_mul(core::mem::size_of::<T>()) 169 .ok_or(EOVERFLOW)?; 170 let mut dma_handle = 0; 171 // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. 172 let ret = unsafe { 173 bindings::dma_alloc_attrs( 174 dev.as_raw(), 175 size, 176 &mut dma_handle, 177 gfp_flags.as_raw(), 178 dma_attrs.as_raw(), 179 ) 180 }; 181 if ret.is_null() { 182 return Err(ENOMEM); 183 } 184 // INVARIANT: 185 // - We just successfully allocated a coherent region which is accessible for 186 // `count` elements, hence the cpu address is valid. We also hold a refcounted reference 187 // to the device. 188 // - The allocated `size` is equal to `size_of::<T> * count`. 189 // - The allocated `size` fits into a `usize`. 190 Ok(Self { 191 dev: dev.into(), 192 dma_handle, 193 count, 194 cpu_addr: ret as *mut T, 195 dma_attrs, 196 }) 197 } 198 199 /// Performs the same functionality as [`CoherentAllocation::alloc_attrs`], except the 200 /// `dma_attrs` is 0 by default. 201 pub fn alloc_coherent( 202 dev: &Device<Bound>, 203 count: usize, 204 gfp_flags: kernel::alloc::Flags, 205 ) -> Result<CoherentAllocation<T>> { 206 CoherentAllocation::alloc_attrs(dev, count, gfp_flags, Attrs(0)) 207 } 208 209 /// Returns the number of elements `T` in this allocation. 210 /// 211 /// Note that this is not the size of the allocation in bytes, which is provided by 212 /// [`Self::size`]. 213 pub fn count(&self) -> usize { 214 self.count 215 } 216 217 /// Returns the size in bytes of this allocation. 218 pub fn size(&self) -> usize { 219 // INVARIANT: The type invariant of `Self` guarantees that `size_of::<T> * count` fits into 220 // a `usize`. 221 self.count * core::mem::size_of::<T>() 222 } 223 224 /// Returns the base address to the allocated region in the CPU's virtual address space. 225 pub fn start_ptr(&self) -> *const T { 226 self.cpu_addr 227 } 228 229 /// Returns the base address to the allocated region in the CPU's virtual address space as 230 /// a mutable pointer. 231 pub fn start_ptr_mut(&mut self) -> *mut T { 232 self.cpu_addr 233 } 234 235 /// Returns a DMA handle which may be given to the device as the DMA address base of 236 /// the region. 237 pub fn dma_handle(&self) -> bindings::dma_addr_t { 238 self.dma_handle 239 } 240 241 /// Returns a DMA handle starting at `offset` (in units of `T`) which may be given to the 242 /// device as the DMA address base of the region. 243 /// 244 /// Returns `EINVAL` if `offset` is not within the bounds of the allocation. 245 pub fn dma_handle_with_offset(&self, offset: usize) -> Result<bindings::dma_addr_t> { 246 if offset >= self.count { 247 Err(EINVAL) 248 } else { 249 // INVARIANT: The type invariant of `Self` guarantees that `size_of::<T> * count` fits 250 // into a `usize`, and `offset` is inferior to `count`. 251 Ok(self.dma_handle + (offset * core::mem::size_of::<T>()) as bindings::dma_addr_t) 252 } 253 } 254 255 /// Common helper to validate a range applied from the allocated region in the CPU's virtual 256 /// address space. 257 fn validate_range(&self, offset: usize, count: usize) -> Result { 258 if offset.checked_add(count).ok_or(EOVERFLOW)? > self.count { 259 return Err(EINVAL); 260 } 261 Ok(()) 262 } 263 264 /// Returns the data from the region starting from `offset` as a slice. 265 /// `offset` and `count` are in units of `T`, not the number of bytes. 266 /// 267 /// For ringbuffer type of r/w access or use-cases where the pointer to the live data is needed, 268 /// [`CoherentAllocation::start_ptr`] or [`CoherentAllocation::start_ptr_mut`] could be used 269 /// instead. 270 /// 271 /// # Safety 272 /// 273 /// * Callers must ensure that the device does not read/write to/from memory while the returned 274 /// slice is live. 275 /// * Callers must ensure that this call does not race with a write to the same region while 276 /// the returned slice is live. 277 pub unsafe fn as_slice(&self, offset: usize, count: usize) -> Result<&[T]> { 278 self.validate_range(offset, count)?; 279 // SAFETY: 280 // - The pointer is valid due to type invariant on `CoherentAllocation`, 281 // we've just checked that the range and index is within bounds. The immutability of the 282 // data is also guaranteed by the safety requirements of the function. 283 // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked 284 // that `self.count` won't overflow early in the constructor. 285 Ok(unsafe { core::slice::from_raw_parts(self.cpu_addr.add(offset), count) }) 286 } 287 288 /// Performs the same functionality as [`CoherentAllocation::as_slice`], except that a mutable 289 /// slice is returned. 290 /// 291 /// # Safety 292 /// 293 /// * Callers must ensure that the device does not read/write to/from memory while the returned 294 /// slice is live. 295 /// * Callers must ensure that this call does not race with a read or write to the same region 296 /// while the returned slice is live. 297 pub unsafe fn as_slice_mut(&self, offset: usize, count: usize) -> Result<&mut [T]> { 298 self.validate_range(offset, count)?; 299 // SAFETY: 300 // - The pointer is valid due to type invariant on `CoherentAllocation`, 301 // we've just checked that the range and index is within bounds. The immutability of the 302 // data is also guaranteed by the safety requirements of the function. 303 // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked 304 // that `self.count` won't overflow early in the constructor. 305 Ok(unsafe { core::slice::from_raw_parts_mut(self.cpu_addr.add(offset), count) }) 306 } 307 308 /// Writes data to the region starting from `offset`. `offset` is in units of `T`, not the 309 /// number of bytes. 310 /// 311 /// # Safety 312 /// 313 /// * Callers must ensure that the device does not read/write to/from memory while the returned 314 /// slice is live. 315 /// * Callers must ensure that this call does not race with a read or write to the same region 316 /// that overlaps with this write. 317 /// 318 /// # Examples 319 /// 320 /// ``` 321 /// # fn test(alloc: &mut kernel::dma::CoherentAllocation<u8>) -> Result { 322 /// let somedata: [u8; 4] = [0xf; 4]; 323 /// let buf: &[u8] = &somedata; 324 /// // SAFETY: There is no concurrent HW operation on the device and no other R/W access to the 325 /// // region. 326 /// unsafe { alloc.write(buf, 0)?; } 327 /// # Ok::<(), Error>(()) } 328 /// ``` 329 pub unsafe fn write(&self, src: &[T], offset: usize) -> Result { 330 self.validate_range(offset, src.len())?; 331 // SAFETY: 332 // - The pointer is valid due to type invariant on `CoherentAllocation` 333 // and we've just checked that the range and index is within bounds. 334 // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked 335 // that `self.count` won't overflow early in the constructor. 336 unsafe { 337 core::ptr::copy_nonoverlapping(src.as_ptr(), self.cpu_addr.add(offset), src.len()) 338 }; 339 Ok(()) 340 } 341 342 /// Returns a pointer to an element from the region with bounds checking. `offset` is in 343 /// units of `T`, not the number of bytes. 344 /// 345 /// Public but hidden since it should only be used from [`dma_read`] and [`dma_write`] macros. 346 #[doc(hidden)] 347 pub fn item_from_index(&self, offset: usize) -> Result<*mut T> { 348 if offset >= self.count { 349 return Err(EINVAL); 350 } 351 // SAFETY: 352 // - The pointer is valid due to type invariant on `CoherentAllocation` 353 // and we've just checked that the range and index is within bounds. 354 // - `offset` can't overflow since it is smaller than `self.count` and we've checked 355 // that `self.count` won't overflow early in the constructor. 356 Ok(unsafe { self.cpu_addr.add(offset) }) 357 } 358 359 /// Reads the value of `field` and ensures that its type is [`FromBytes`]. 360 /// 361 /// # Safety 362 /// 363 /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is 364 /// validated beforehand. 365 /// 366 /// Public but hidden since it should only be used from [`dma_read`] macro. 367 #[doc(hidden)] 368 pub unsafe fn field_read<F: FromBytes>(&self, field: *const F) -> F { 369 // SAFETY: 370 // - By the safety requirements field is valid. 371 // - Using read_volatile() here is not sound as per the usual rules, the usage here is 372 // a special exception with the following notes in place. When dealing with a potential 373 // race from a hardware or code outside kernel (e.g. user-space program), we need that 374 // read on a valid memory is not UB. Currently read_volatile() is used for this, and the 375 // rationale behind is that it should generate the same code as READ_ONCE() which the 376 // kernel already relies on to avoid UB on data races. Note that the usage of 377 // read_volatile() is limited to this particular case, it cannot be used to prevent 378 // the UB caused by racing between two kernel functions nor do they provide atomicity. 379 unsafe { field.read_volatile() } 380 } 381 382 /// Writes a value to `field` and ensures that its type is [`AsBytes`]. 383 /// 384 /// # Safety 385 /// 386 /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is 387 /// validated beforehand. 388 /// 389 /// Public but hidden since it should only be used from [`dma_write`] macro. 390 #[doc(hidden)] 391 pub unsafe fn field_write<F: AsBytes>(&self, field: *mut F, val: F) { 392 // SAFETY: 393 // - By the safety requirements field is valid. 394 // - Using write_volatile() here is not sound as per the usual rules, the usage here is 395 // a special exception with the following notes in place. When dealing with a potential 396 // race from a hardware or code outside kernel (e.g. user-space program), we need that 397 // write on a valid memory is not UB. Currently write_volatile() is used for this, and the 398 // rationale behind is that it should generate the same code as WRITE_ONCE() which the 399 // kernel already relies on to avoid UB on data races. Note that the usage of 400 // write_volatile() is limited to this particular case, it cannot be used to prevent 401 // the UB caused by racing between two kernel functions nor do they provide atomicity. 402 unsafe { field.write_volatile(val) } 403 } 404 } 405 406 /// Note that the device configured to do DMA must be halted before this object is dropped. 407 impl<T: AsBytes + FromBytes> Drop for CoherentAllocation<T> { 408 fn drop(&mut self) { 409 let size = self.count * core::mem::size_of::<T>(); 410 // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. 411 // The cpu address, and the dma handle are valid due to the type invariants on 412 // `CoherentAllocation`. 413 unsafe { 414 bindings::dma_free_attrs( 415 self.dev.as_raw(), 416 size, 417 self.cpu_addr as _, 418 self.dma_handle, 419 self.dma_attrs.as_raw(), 420 ) 421 } 422 } 423 } 424 425 // SAFETY: It is safe to send a `CoherentAllocation` to another thread if `T` 426 // can be sent to another thread. 427 unsafe impl<T: AsBytes + FromBytes + Send> Send for CoherentAllocation<T> {} 428 429 /// Reads a field of an item from an allocated region of structs. 430 /// 431 /// # Examples 432 /// 433 /// ``` 434 /// use kernel::device::Device; 435 /// use kernel::dma::{attrs::*, CoherentAllocation}; 436 /// 437 /// struct MyStruct { field: u32, } 438 /// 439 /// // SAFETY: All bit patterns are acceptable values for `MyStruct`. 440 /// unsafe impl kernel::transmute::FromBytes for MyStruct{}; 441 /// // SAFETY: Instances of `MyStruct` have no uninitialized portions. 442 /// unsafe impl kernel::transmute::AsBytes for MyStruct{}; 443 /// 444 /// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result { 445 /// let whole = kernel::dma_read!(alloc[2]); 446 /// let field = kernel::dma_read!(alloc[1].field); 447 /// # Ok::<(), Error>(()) } 448 /// ``` 449 #[macro_export] 450 macro_rules! dma_read { 451 ($dma:expr, $idx: expr, $($field:tt)*) => {{ 452 (|| -> ::core::result::Result<_, $crate::error::Error> { 453 let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; 454 // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be 455 // dereferenced. The compiler also further validates the expression on whether `field` 456 // is a member of `item` when expanded by the macro. 457 unsafe { 458 let ptr_field = ::core::ptr::addr_of!((*item) $($field)*); 459 ::core::result::Result::Ok( 460 $crate::dma::CoherentAllocation::field_read(&$dma, ptr_field) 461 ) 462 } 463 })() 464 }}; 465 ($dma:ident [ $idx:expr ] $($field:tt)* ) => { 466 $crate::dma_read!($dma, $idx, $($field)*) 467 }; 468 ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => { 469 $crate::dma_read!($($dma).*, $idx, $($field)*) 470 }; 471 } 472 473 /// Writes to a field of an item from an allocated region of structs. 474 /// 475 /// # Examples 476 /// 477 /// ``` 478 /// use kernel::device::Device; 479 /// use kernel::dma::{attrs::*, CoherentAllocation}; 480 /// 481 /// struct MyStruct { member: u32, } 482 /// 483 /// // SAFETY: All bit patterns are acceptable values for `MyStruct`. 484 /// unsafe impl kernel::transmute::FromBytes for MyStruct{}; 485 /// // SAFETY: Instances of `MyStruct` have no uninitialized portions. 486 /// unsafe impl kernel::transmute::AsBytes for MyStruct{}; 487 /// 488 /// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result { 489 /// kernel::dma_write!(alloc[2].member = 0xf); 490 /// kernel::dma_write!(alloc[1] = MyStruct { member: 0xf }); 491 /// # Ok::<(), Error>(()) } 492 /// ``` 493 #[macro_export] 494 macro_rules! dma_write { 495 ($dma:ident [ $idx:expr ] $($field:tt)*) => {{ 496 $crate::dma_write!($dma, $idx, $($field)*) 497 }}; 498 ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {{ 499 $crate::dma_write!($($dma).*, $idx, $($field)*) 500 }}; 501 ($dma:expr, $idx: expr, = $val:expr) => { 502 (|| -> ::core::result::Result<_, $crate::error::Error> { 503 let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; 504 // SAFETY: `item_from_index` ensures that `item` is always a valid item. 505 unsafe { $crate::dma::CoherentAllocation::field_write(&$dma, item, $val) } 506 ::core::result::Result::Ok(()) 507 })() 508 }; 509 ($dma:expr, $idx: expr, $(.$field:ident)* = $val:expr) => { 510 (|| -> ::core::result::Result<_, $crate::error::Error> { 511 let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; 512 // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be 513 // dereferenced. The compiler also further validates the expression on whether `field` 514 // is a member of `item` when expanded by the macro. 515 unsafe { 516 let ptr_field = ::core::ptr::addr_of_mut!((*item) $(.$field)*); 517 $crate::dma::CoherentAllocation::field_write(&$dma, ptr_field, $val) 518 } 519 ::core::result::Result::Ok(()) 520 })() 521 }; 522 } 523