1 // SPDX-License-Identifier: GPL-2.0 2 3 //! Memory-mapped IO. 4 //! 5 //! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h) 6 7 use core::{ 8 marker::PhantomData, 9 mem::MaybeUninit, 10 num::TryFromIntError, // 11 }; 12 13 use crate::{ 14 bindings, 15 fmt, 16 mem::{ 17 AsRepr, 18 AsReprMut, // 19 }, 20 prelude::*, 21 ptr::{ 22 Alignment, 23 KnownSize, // 24 }, // 25 }; 26 27 #[cfg(CONFIG_HAS_IOMEM)] 28 pub mod mem; 29 pub mod poll; 30 pub mod register; 31 pub mod resource; 32 33 pub use crate::register; 34 pub use resource::Resource; 35 36 use register::LocatedRegister; 37 38 /// Physical address type. 39 /// 40 /// This is a type alias to either `u32` or `u64` depending on the config option 41 /// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. 42 pub type PhysAddr = bindings::phys_addr_t; 43 44 /// Resource size type. 45 /// 46 /// This wraps either `u32` or `u64` depending on the config option 47 /// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a `u64` even on 32-bit architectures. 48 /// 49 /// # Examples 50 /// 51 /// ``` 52 /// use kernel::io::ResourceSize; 53 /// 54 /// let size = ResourceSize::from_raw(0x1000); 55 /// assert_eq!(size.into_raw(), 0x1000); 56 /// 57 /// // Round-trips through the raw C type. 58 /// let raw: kernel::bindings::resource_size_t = size.into(); 59 /// assert_eq!(ResourceSize::from(raw), size); 60 /// 61 /// // Fallible conversion to `usize` (can truncate on 32-bit). 62 /// assert_eq!(usize::try_from(size)?, 0x1000); 63 /// # Ok::<(), core::num::TryFromIntError>(()) 64 /// ``` 65 #[repr(transparent)] 66 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] 67 pub struct ResourceSize(bindings::resource_size_t); 68 69 impl ResourceSize { 70 /// Creates a resource size from the raw C type. 71 #[inline] 72 pub const fn from_raw(value: bindings::resource_size_t) -> Self { 73 Self(value) 74 } 75 76 /// Turns this resource size into the raw C type. 77 #[inline] 78 pub const fn into_raw(self) -> bindings::resource_size_t { 79 self.0 80 } 81 } 82 83 impl fmt::Debug for ResourceSize { 84 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 85 write!(f, "{:#x}", self.0) 86 } 87 } 88 89 impl From<bindings::resource_size_t> for ResourceSize { 90 #[inline] 91 fn from(value: bindings::resource_size_t) -> Self { 92 Self::from_raw(value) 93 } 94 } 95 96 impl From<ResourceSize> for bindings::resource_size_t { 97 #[inline] 98 fn from(value: ResourceSize) -> Self { 99 value.into_raw() 100 } 101 } 102 103 impl TryFrom<ResourceSize> for usize { 104 type Error = TryFromIntError; 105 106 #[inline] 107 fn try_from(value: ResourceSize) -> Result<Self, Self::Error> { 108 Self::try_from(value.into_raw()) 109 } 110 } 111 112 /// Untyped I/O region. 113 /// 114 /// This type can be used when an I/O region without known type information has a compile-time known 115 /// minimum size (and a runtime known actual size). 116 /// 117 /// # Invariants 118 /// 119 /// - Size of the region is at least as large as the `SIZE` generic parameter. 120 /// - Size of the region is multiple of 4. 121 #[repr(C, align(4))] 122 #[derive(FromBytes)] 123 pub struct Region<const SIZE: usize = 0> { 124 inner: [u8], 125 } 126 127 impl<const SIZE: usize> Region<SIZE> { 128 /// Create a raw mutable pointer from given base address and size. 129 /// 130 /// `size` should be at least as large as the minimum size `SIZE`, and `base` and `size` should 131 /// be 4-byte aligned to uphold the type invariant. 132 /// 133 /// Just like other methods on raw pointers, it is not unsafe to create a raw pointer 134 /// that does not uphold the type invariants. However such pointers are not valid. 135 #[inline] 136 pub fn ptr_from_raw_parts_mut(base: *mut u8, size: usize) -> *mut Self { 137 core::ptr::slice_from_raw_parts_mut(base, size) as *mut Region<SIZE> 138 } 139 140 /// Create a raw mutable pointer from given base address and size. 141 /// 142 /// The alignment of `base` is checked, and `size` is checked against the minimum size specified 143 /// via const generics. 144 #[inline] 145 pub fn ptr_try_from_raw_parts_mut(base: *mut u8, size: usize) -> Result<*mut Self> { 146 if size < SIZE || base.align_offset(4) != 0 || !size.is_multiple_of(4) { 147 return Err(EINVAL); 148 } 149 150 Ok(Self::ptr_from_raw_parts_mut(base, size)) 151 } 152 } 153 154 impl<const SIZE: usize> KnownSize for Region<SIZE> { 155 const MIN_SIZE: usize = SIZE; 156 // Alignment of 4 is the most common; different base types can be added once required. 157 const MIN_ALIGN: Alignment = Alignment::new::<4>(); 158 159 #[inline(always)] 160 fn size(p: *const Self) -> usize { 161 (p as *const [u8]).len() 162 } 163 } 164 165 // SAFETY: 166 // - Values read from I/O are always treated as initialized. 167 // - Per type invariant the size is multiple of 4 and the type is 4-byte aligned, so it is padding 168 // free. 169 // 170 // This cannot be derived as `derive(IntoBytes)` as the padding free property comes from type 171 // invariant which the macro does not know. 172 unsafe impl<const SIZE: usize> IntoBytes for Region<SIZE> { 173 #[inline] 174 #[allow(unused)] // Rust 1.87+ stops requiring this and will emit unused warnings. 175 fn only_derive_is_allowed_to_implement_this_trait() {} 176 } 177 178 /// Raw representation of an MMIO region. 179 /// 180 /// `MmioRaw<T>` is equivalent to `T __iomem *` in C. 181 /// 182 /// By itself, the existence of an instance of this structure does not provide any guarantees that 183 /// the represented MMIO region does exist or is properly mapped. 184 /// 185 /// Instead, the bus specific MMIO implementation must convert this raw representation into an 186 /// `Mmio` instance providing the actual memory accessors. Only by the conversion into an `Mmio` 187 /// structure any guarantees are given. 188 pub struct MmioRaw<T: ?Sized> { 189 /// Pointer is in I/O address space. 190 /// 191 /// The provenance does not matter, only the address and metadata do. 192 ptr: *mut T, 193 } 194 195 impl<T: ?Sized> Copy for MmioRaw<T> {} 196 impl<T: ?Sized> Clone for MmioRaw<T> { 197 #[inline] 198 fn clone(&self) -> Self { 199 *self 200 } 201 } 202 203 // SAFETY: `MmioRaw` is just an address, so is thread-safe. 204 unsafe impl<T: ?Sized> Send for MmioRaw<T> {} 205 // SAFETY: `MmioRaw` is just an address, so is thread-safe. 206 unsafe impl<T: ?Sized> Sync for MmioRaw<T> {} 207 208 impl<T> MmioRaw<T> { 209 /// Create a `MmioRaw` from address. 210 #[inline] 211 pub fn new(addr: usize) -> Self { 212 Self { 213 ptr: core::ptr::without_provenance_mut(addr), 214 } 215 } 216 } 217 218 impl<const SIZE: usize> MmioRaw<Region<SIZE>> { 219 /// Create a `MmioRaw` representing a I/O region with given size. 220 /// 221 /// The size is checked against the minimum size specified via const generics. 222 #[inline] 223 pub fn new_region(addr: usize, size: usize) -> Result<Self> { 224 Ok(Self { 225 ptr: Region::ptr_try_from_raw_parts_mut(core::ptr::without_provenance_mut(addr), size)?, 226 }) 227 } 228 } 229 230 impl<T: ?Sized + KnownSize> MmioRaw<T> { 231 /// Returns the base address of the MMIO region. 232 #[inline] 233 pub fn addr(&self) -> usize { 234 self.ptr.addr() 235 } 236 237 /// Returns the size of the MMIO region. 238 #[inline] 239 pub fn size(&self) -> usize { 240 KnownSize::size(self.ptr) 241 } 242 } 243 244 /// Checks whether an access of type `U` at the given `base` and the given `offset` 245 /// is valid within this region. 246 /// 247 /// The `base` is used for alignment checking only. This can be set to 0 to skip the check. 248 #[inline] 249 const fn offset_valid<U>(base: usize, offset: usize, size: usize) -> bool { 250 if let Some(end) = offset.checked_add(size_of::<U>()) { 251 end <= size && (base.wrapping_add(offset) % align_of::<U>() == 0) 252 } else { 253 false 254 } 255 } 256 257 /// Returns a view for a given `offset`, performing compile-time bound checks. 258 // Always inline to optimize out error path of `build_assert`. 259 #[inline(always)] 260 fn io_view_assert<'a, IO: Io<'a>, U>( 261 this: IO, 262 offset: usize, 263 ) -> <IO::Backend as IoBackend>::View<'a, U> { 264 // We cannot check alignment with `offset_valid` using `ptr.addr()`. So set 0 for it and 265 // ensure alignment by checking that the alignment of `U` is smaller or equal to the 266 // alignment of `IO::Target`. 267 const_assert!(Alignment::of::<U>().as_usize() <= IO::Target::MIN_ALIGN.as_usize()); 268 build_assert!(offset_valid::<U>(0, offset, IO::Target::MIN_SIZE)); 269 270 let view = this.as_view(); 271 let ptr = IO::Backend::as_ptr(view); 272 let projected_ptr = ptr.cast::<U>().wrapping_byte_add(offset); 273 // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a 274 // valid projection. 275 unsafe { IO::Backend::project_view(view, projected_ptr) } 276 } 277 278 /// Returns a view for a given `offset`, performing runtime bound checks. 279 #[inline] 280 fn io_view<'a, IO: Io<'a>, U>( 281 this: IO, 282 offset: usize, 283 ) -> Result<<IO::Backend as IoBackend>::View<'a, U>> { 284 let view = this.as_view(); 285 let ptr = IO::Backend::as_ptr(view); 286 287 if !offset_valid::<U>(ptr.addr(), offset, KnownSize::size(ptr)) { 288 return Err(EINVAL); 289 } 290 291 let projected_ptr = ptr.cast::<U>().wrapping_byte_add(offset); 292 // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a 293 // valid projection. 294 Ok(unsafe { IO::Backend::project_view(view, projected_ptr) }) 295 } 296 297 /// Returns the primitive view of a I/O view. 298 #[inline] 299 fn io_view_as_repr<'a, IO: Io<'a, Target = T>, T: AsRepr>( 300 this: IO, 301 ) -> <IO::Backend as IoBackend>::View<'a, T::Repr> { 302 let view = this.as_view(); 303 304 // SAFETY: `AsRepr` guarantees layout compatibility. 305 unsafe { IO::Backend::project_view(view, IO::Backend::as_ptr(view).cast::<T::Repr>()) } 306 } 307 308 /// I/O backends. 309 /// 310 /// This is an abstract representation to be implemented by arbitrary I/O 311 /// backends (e.g. MMIO, PCI config space, etc.). 312 /// 313 /// The base trait only defines the projection operations; which I/O methods are available depends 314 /// on which [`IoCapable<T>`] traits are implemented for the type. For example, for MMIO regions, 315 /// all widths (u8, u16, u32, and u64 on 64-bit systems) are typically supported. For PCI 316 /// configuration space, u8, u16, and u32 are supported but u64 is not. 317 /// 318 /// This trait is separate from the `Io` trait as multiple different I/O types may share the same 319 /// operation. 320 pub trait IoBackend { 321 /// View type for this I/O backend. 322 type View<'a, T: ?Sized + KnownSize>: IoBase<'a, Backend = Self, Target = T>; 323 324 /// Convert a `view` to a raw pointer for projection. 325 /// 326 /// The returned pointer is private implementation detail of the backend; it is likely not 327 /// valid. It should not be dereferenced. 328 fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T; 329 330 /// Project `view` to its subregion indicated by `ptr`. 331 /// 332 /// If input `view` is valid, returned view must also be valid. 333 /// 334 /// # Safety 335 /// 336 /// `ptr` must be a projection of `Self::as_ptr(view)`. 337 unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( 338 view: Self::View<'a, T>, 339 ptr: *mut U, 340 ) -> Self::View<'a, U>; 341 } 342 343 /// Trait indicating that an I/O backend supports operations of a certain type and providing an 344 /// implementation for these operations. 345 /// 346 /// Different I/O backends can implement this trait to expose only the operations they support. 347 /// 348 /// For example, a PCI configuration space may implement `IoCapable<u8>`, `IoCapable<u16>`, 349 /// and `IoCapable<u32>`, but not `IoCapable<u64>`, while an MMIO region on a 64-bit 350 /// system might implement all four. 351 pub trait IoCapable<T>: IoBackend { 352 /// Performs an I/O read of type `T` at `view` and returns the result. 353 fn io_read<'a>(view: Self::View<'a, T>) -> T; 354 355 /// Performs an I/O write of `value` at `view`. 356 fn io_write<'a>(view: Self::View<'a, T>, value: T); 357 } 358 359 /// Trait indicating that an I/O backend supports memory copy operations. 360 pub trait IoCopyable: IoBackend { 361 /// Copy contents of `view` to `buffer`. 362 /// 363 /// # Safety 364 /// 365 /// - `buffer` is valid for volatile write for `view.size()` bytes. 366 /// - `buffer` should not overlap with `view`. 367 unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8); 368 369 /// Copy contents from `buffer` to `view`. 370 /// 371 /// # Safety 372 /// 373 /// - `buffer` is valid for volatile read for `view.size()` bytes. 374 /// - `buffer` should not overlap with `view`. 375 unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8); 376 377 /// Copy from `view` and return the value. 378 #[inline] 379 fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T { 380 // Project `self` to `[u8]`. 381 let ptr = Self::as_ptr(view); 382 // SAFETY: This is a identity projection. 383 let slice_view = unsafe { 384 Self::project_view( 385 view, 386 core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(), size_of::<T>()), 387 ) 388 }; 389 390 let mut buf = MaybeUninit::<T>::uninit(); 391 // SAFETY: 392 // - `buf.as_mut_ptr()` is valid for write for `size_of::<T>()` bytes. 393 // - `buf` is local so `buf.as_mut_ptr()` cannot overlap with `slice_view`. 394 unsafe { Self::copy_from_io(slice_view, buf.as_mut_ptr().cast()) }; 395 // SAFETY: `T: FromBytes` guarantee that all bit patterns are valid. 396 unsafe { buf.assume_init() } 397 } 398 399 /// Copy `value` to `view`. 400 /// 401 /// Destructor of `value` will not be executed, consistent with [`zerocopy::transmute`]. 402 #[inline] 403 fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) { 404 // Project `self` to `[u8]`. 405 let ptr = Self::as_ptr(view); 406 // SAFETY: This is a identity projection. 407 let slice_view = unsafe { 408 Self::project_view( 409 view, 410 core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(), size_of::<T>()), 411 ) 412 }; 413 414 // SAFETY: 415 // - `&raw const value` is valid for read for `size_of::<T>()` bytes. 416 // - `value` is local so `&raw const value` cannot overlap with `slice_view`. 417 unsafe { Self::copy_to_io(slice_view, (&raw const value).cast()) }; 418 core::mem::forget(value); 419 } 420 } 421 422 /// Describes a given I/O location: its offset, width, and type to convert the raw value from and 423 /// into. 424 /// 425 /// This trait is the key abstraction allowing [`Io::read`], [`Io::write`], and [`Io::update`] (and 426 /// their fallible [`try_read`](Io::try_read), [`try_write`](Io::try_write) and 427 /// [`try_update`](Io::try_update) counterparts) to work uniformly with both raw [`usize`] offsets 428 /// (for primitive types like [`u32`]) and typed ones (like those generated by the [`register!`] 429 /// macro). 430 /// 431 /// An `IoLoc<Base, T>` carries the following pieces of information: 432 /// 433 /// - The valid `Base` to operate on. For most registers, this should be [`Region`]. 434 /// - The offset to access (returned by [`IoLoc::offset`]), 435 /// - The type `T` in which the data is returned or provided. 436 /// 437 /// `T` is not necessarily the type for underlying I/O operation. Methods that take `IoLoc` have `T: 438 /// AsRepr` bound and the `<T as AsRepr>::Repr` type would be used to perform I/O and converted to 439 /// `T` instead. 440 pub trait IoLoc<Base: ?Sized, T> { 441 /// Consumes `self` and returns the offset of this location. 442 fn offset(self) -> usize; 443 } 444 445 /// Implements [`IoLoc<Region<SIZE>, $ty>`] for [`usize`], allowing [`usize`] to be used as a 446 /// parameter of [`Io::read`] and [`Io::write`]. 447 macro_rules! impl_usize_ioloc { 448 ($($ty:ty),*) => { 449 $( 450 impl<const SIZE: usize> IoLoc<Region<SIZE>, $ty> for usize { 451 #[inline(always)] 452 fn offset(self) -> usize { 453 self 454 } 455 } 456 )* 457 } 458 } 459 460 // Provide the ability to read any primitive type from a [`usize`]. 461 impl_usize_ioloc!(u8, u16, u32, u64); 462 463 /// Types implementing this trait (e.g. MMIO BARs or PCI config regions) 464 /// can perform I/O operations on regions of memory. 465 /// 466 /// This trait defines which backend shall be used for I/O operations and provides a method to 467 /// convert into [`IoBackend::View`]. Users should use the [`Io`] trait which provides the actual 468 /// methods to perform I/O operations. 469 /// 470 /// This should be implemented on cheaply copyable handles, such as references or view types. 471 pub trait IoBase<'a>: Copy { 472 /// Type that defines all I/O operations. 473 type Backend: IoBackend; 474 475 /// Type of this I/O region. For untyped regions, [`Region`] can be used. 476 type Target: ?Sized + KnownSize; 477 478 /// Return a view that covers the full region. 479 fn as_view(self) -> <Self::Backend as IoBackend>::View<'a, Self::Target>; 480 } 481 482 /// Extension trait to provide I/O operation methods to types that implement [`IoBase`]. 483 /// 484 /// This trait provides: 485 /// - Helper methods for offset validation and address calculation 486 /// - Fallible (runtime checked) accessors for different data widths 487 /// 488 /// Which I/O methods are available depends on the associated [`IoBackend`] implementation. 489 pub trait Io<'a>: IoBase<'a> { 490 /// Returns the size of this I/O region. 491 #[inline] 492 fn size(self) -> usize { 493 KnownSize::size(Self::Backend::as_ptr(self.as_view())) 494 } 495 496 /// Returns the length of the slice in number of elements. 497 #[inline] 498 fn len<T>(self) -> usize 499 where 500 Self: Io<'a, Target = [T]>, 501 { 502 Self::Backend::as_ptr(self.as_view()).len() 503 } 504 505 /// Returns `true` if the slice has a length of 0. 506 #[inline] 507 fn is_empty<T>(self) -> bool 508 where 509 Self: Io<'a, Target = [T]>, 510 { 511 self.len() == 0 512 } 513 514 /// Try to convert into a different typed I/O view. 515 /// 516 /// A runtime check is performed to ensure that the target type is of same or smaller size to 517 /// current type, and the current view is properly aligned for the target type. Returns 518 /// `Err(EINVAL)` if the runtime check fails. 519 /// 520 /// # Examples 521 /// 522 /// ```no_run 523 /// use kernel::io::{ 524 /// io_project, 525 /// Mmio, 526 /// Io, 527 /// Region, 528 /// }; 529 /// #[derive(FromBytes, IntoBytes)] 530 /// #[repr(C)] 531 /// struct MyStruct { field: u32, } 532 /// 533 /// # fn test(mmio: &Mmio<'_, Region>) -> Result { 534 /// // let mmio: Mmio<'_, Region>; 535 /// let whole: Mmio<'_, MyStruct> = mmio.try_cast()?; 536 /// # Ok::<(), Error>(()) } 537 /// ``` 538 #[inline] 539 fn try_cast<U>(self) -> Result<<Self::Backend as IoBackend>::View<'a, U>> 540 where 541 Self::Target: FromBytes + IntoBytes, 542 U: FromBytes + IntoBytes, 543 { 544 let view = self.as_view(); 545 let ptr = Self::Backend::as_ptr(view); 546 547 if size_of::<U>() > KnownSize::size(ptr) { 548 return Err(EINVAL); 549 } 550 551 if ptr.addr() % align_of::<U>() != 0 { 552 return Err(EINVAL); 553 } 554 555 // SAFETY: We have checked bounds and alignment, so this is a valid projection. 556 Ok(unsafe { Self::Backend::project_view(view, ptr.cast()) }) 557 } 558 559 /// Read a value from I/O. 560 /// 561 /// This only works for primitives supported by the I/O backend. 562 /// 563 /// # Examples 564 /// 565 /// ```no_run 566 /// # use kernel::io::*; 567 /// # fn test_read_val(mmio: Mmio<'_, u32>) { 568 /// // let mmio: Mmio<'_, u32>; 569 /// let val: u32 = mmio.read_val(); 570 /// # } 571 /// ``` 572 #[inline] 573 fn read_val(self) -> Self::Target 574 where 575 Self::Target: AsReprMut, 576 Self::Backend: IoCapable<<Self::Target as AsRepr>::Repr>, 577 { 578 Self::Target::from_repr(Self::Backend::io_read(io_view_as_repr(self))) 579 } 580 581 /// Write a value to I/O. 582 /// 583 /// This only works for primitives supported by the I/O backend. 584 /// 585 /// # Examples 586 /// 587 /// ```no_run 588 /// # use kernel::io::*; 589 /// # fn test_write_val(mmio: Mmio<'_, u32>) { 590 /// // let mmio: Mmio<'_, u32>; 591 /// mmio.write_val(1u32); 592 /// # } 593 /// ``` 594 #[inline] 595 fn write_val(self, value: Self::Target) 596 where 597 Self::Target: AsRepr, 598 Self::Backend: IoCapable<<Self::Target as AsRepr>::Repr>, 599 { 600 Self::Backend::io_write(io_view_as_repr(self), Self::Target::into_repr(value)) 601 } 602 603 /// Copy-read from I/O memory. 604 /// 605 /// This is equivalent to reading from the I/O memory with byte-wise copy, although the actual 606 /// implementation might be more efficient. There is no atomicity guarantee. Note that for some 607 /// backends (e.g. `Mmio`), this can read different value compared to [`read_val`] as 608 /// byte-swapping is not performed. 609 /// 610 /// [`read_val`]: Io::read_val 611 /// 612 /// # Examples 613 /// 614 /// ```no_run 615 /// # use kernel::io::*; 616 /// # fn test_copy_read(mmio: Mmio<'_, [u8; 6]>) { 617 /// // let mmio: Mmio<'_, [u8; 6]>; 618 /// let val: [u8; 6] = mmio.copy_read(); 619 /// # } 620 /// ``` 621 #[inline] 622 fn copy_read(self) -> Self::Target 623 where 624 Self::Backend: IoCopyable, 625 Self::Target: Sized + FromBytes, 626 { 627 Self::Backend::copy_read(self.as_view()) 628 } 629 630 /// Copy-write to I/O memory. 631 /// 632 /// This is equivalent to writing to the I/O memory with byte-wise copy, although the actual 633 /// implementation might be more efficient. There is no atomicity guarantee. Note that for some 634 /// backends (e.g. `Mmio`), this can write different value compared to [`write_val`] as 635 /// byte-swapping is not performed. 636 /// 637 /// [`write_val`]: Io::write_val 638 /// 639 /// # Examples 640 /// 641 /// ```no_run 642 /// # use kernel::io::*; 643 /// # fn test_copy_write(mmio: Mmio<'_, [u8; 6]>) { 644 /// // let mmio: Mmio<'_, [u8; 6]>; 645 /// mmio.copy_write([0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]); 646 /// # } 647 /// ``` 648 #[inline] 649 fn copy_write(self, value: Self::Target) 650 where 651 Self::Backend: IoCopyable, 652 Self::Target: Sized + IntoBytes, 653 { 654 Self::Backend::copy_write(self.as_view(), value); 655 } 656 657 /// Copy bytes from `data` to I/O memory. 658 /// 659 /// # Panics 660 /// 661 /// This function will panic if the length of `self` differs from the length of `data`, similar 662 /// to [`[u8]::copy_from_slice`]. 663 /// 664 /// # Examples 665 /// 666 /// ```no_run 667 /// # use kernel::io::*; 668 /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) { 669 /// // let mmio: Mmio<'_, [u8]>; 670 /// mmio.copy_from_slice(&[0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]); 671 /// # } 672 /// ``` 673 #[inline] 674 fn copy_from_slice(self, data: &[u8]) 675 where 676 Self::Backend: IoCopyable, 677 Self: Io<'a, Target = [u8]>, 678 { 679 assert_eq!(self.len(), data.len()); 680 681 // SAFETY: `data.as_ptr()` is valid for read for `self.size()` bytes. 682 unsafe { 683 Self::Backend::copy_to_io(self.as_view(), data.as_ptr()); 684 } 685 } 686 687 /// Copy bytes from I/O memory to `data`. 688 /// 689 /// # Panics 690 /// 691 /// This function will panic if the length of `self` differs from the length of `data`, similar 692 /// to [`[u8]::copy_from_slice`]. 693 /// 694 /// # Examples 695 /// 696 /// ```no_run 697 /// # use kernel::io::*; 698 /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) { 699 /// // let mmio: Mmio<'_, [u8]>; 700 /// let mut buf = [0; 6]; 701 /// mmio.copy_to_slice(&mut buf); 702 /// # } 703 /// ``` 704 #[inline] 705 fn copy_to_slice(self, data: &mut [u8]) 706 where 707 Self::Backend: IoCopyable, 708 Self: Io<'a, Target = [u8]>, 709 { 710 assert_eq!(self.len(), data.len()); 711 712 // SAFETY: `data.as_mut_ptr()` is valid for write for `self.size()` bytes. 713 unsafe { 714 Self::Backend::copy_from_io(self.as_view(), data.as_mut_ptr()); 715 } 716 } 717 718 /// Fallible 8-bit read with runtime bounds check. 719 #[inline(always)] 720 fn try_read8(self, offset: usize) -> Result<u8> 721 where 722 usize: IoLoc<Self::Target, u8>, 723 Self::Backend: IoCapable<u8>, 724 { 725 self.try_read(offset) 726 } 727 728 /// Fallible 16-bit read with runtime bounds check. 729 #[inline(always)] 730 fn try_read16(self, offset: usize) -> Result<u16> 731 where 732 usize: IoLoc<Self::Target, u16>, 733 Self::Backend: IoCapable<u16>, 734 { 735 self.try_read(offset) 736 } 737 738 /// Fallible 32-bit read with runtime bounds check. 739 #[inline(always)] 740 fn try_read32(self, offset: usize) -> Result<u32> 741 where 742 usize: IoLoc<Self::Target, u32>, 743 Self::Backend: IoCapable<u32>, 744 { 745 self.try_read(offset) 746 } 747 748 /// Fallible 64-bit read with runtime bounds check. 749 #[inline(always)] 750 fn try_read64(self, offset: usize) -> Result<u64> 751 where 752 usize: IoLoc<Self::Target, u64>, 753 Self::Backend: IoCapable<u64>, 754 { 755 self.try_read(offset) 756 } 757 758 /// Fallible 8-bit write with runtime bounds check. 759 #[inline(always)] 760 fn try_write8(self, value: u8, offset: usize) -> Result 761 where 762 usize: IoLoc<Self::Target, u8>, 763 Self::Backend: IoCapable<u8>, 764 { 765 self.try_write(offset, value) 766 } 767 768 /// Fallible 16-bit write with runtime bounds check. 769 #[inline(always)] 770 fn try_write16(self, value: u16, offset: usize) -> Result 771 where 772 usize: IoLoc<Self::Target, u16>, 773 Self::Backend: IoCapable<u16>, 774 { 775 self.try_write(offset, value) 776 } 777 778 /// Fallible 32-bit write with runtime bounds check. 779 #[inline(always)] 780 fn try_write32(self, value: u32, offset: usize) -> Result 781 where 782 usize: IoLoc<Self::Target, u32>, 783 Self::Backend: IoCapable<u32>, 784 { 785 self.try_write(offset, value) 786 } 787 788 /// Fallible 64-bit write with runtime bounds check. 789 #[inline(always)] 790 fn try_write64(self, value: u64, offset: usize) -> Result 791 where 792 usize: IoLoc<Self::Target, u64>, 793 Self::Backend: IoCapable<u64>, 794 { 795 self.try_write(offset, value) 796 } 797 798 /// Infallible 8-bit read with compile-time bounds check. 799 /// 800 /// `offset` should be constant. 801 #[inline(always)] 802 fn read8(self, offset: usize) -> u8 803 where 804 usize: IoLoc<Self::Target, u8>, 805 Self::Backend: IoCapable<u8>, 806 { 807 self.read(offset) 808 } 809 810 /// Infallible 16-bit read with compile-time bounds check. 811 /// 812 /// `offset` should be constant. 813 #[inline(always)] 814 fn read16(self, offset: usize) -> u16 815 where 816 usize: IoLoc<Self::Target, u16>, 817 Self::Backend: IoCapable<u16>, 818 { 819 self.read(offset) 820 } 821 822 /// Infallible 32-bit read with compile-time bounds check. 823 /// 824 /// `offset` should be constant. 825 #[inline(always)] 826 fn read32(self, offset: usize) -> u32 827 where 828 usize: IoLoc<Self::Target, u32>, 829 Self::Backend: IoCapable<u32>, 830 { 831 self.read(offset) 832 } 833 834 /// Infallible 64-bit read with compile-time bounds check. 835 /// 836 /// `offset` should be constant. 837 #[inline(always)] 838 fn read64(self, offset: usize) -> u64 839 where 840 usize: IoLoc<Self::Target, u64>, 841 Self::Backend: IoCapable<u64>, 842 { 843 self.read(offset) 844 } 845 846 /// Infallible 8-bit write with compile-time bounds check. 847 /// 848 /// `offset` should be constant. 849 #[inline(always)] 850 fn write8(self, value: u8, offset: usize) 851 where 852 usize: IoLoc<Self::Target, u8>, 853 Self::Backend: IoCapable<u8>, 854 { 855 self.write(offset, value) 856 } 857 858 /// Infallible 16-bit write with compile-time bounds check. 859 /// 860 /// `offset` should be constant. 861 #[inline(always)] 862 fn write16(self, value: u16, offset: usize) 863 where 864 usize: IoLoc<Self::Target, u16>, 865 Self::Backend: IoCapable<u16>, 866 { 867 self.write(offset, value) 868 } 869 870 /// Infallible 32-bit write with compile-time bounds check. 871 /// 872 /// `offset` should be constant. 873 #[inline(always)] 874 fn write32(self, value: u32, offset: usize) 875 where 876 usize: IoLoc<Self::Target, u32>, 877 Self::Backend: IoCapable<u32>, 878 { 879 self.write(offset, value) 880 } 881 882 /// Infallible 64-bit write with compile-time bounds check. 883 /// 884 /// `offset` should be constant. 885 #[inline(always)] 886 fn write64(self, value: u64, offset: usize) 887 where 888 usize: IoLoc<Self::Target, u64>, 889 Self::Backend: IoCapable<u64>, 890 { 891 self.write(offset, value) 892 } 893 894 /// Generic fallible read with runtime bounds check. 895 /// 896 /// # Examples 897 /// 898 /// Read a primitive type from an I/O address: 899 /// 900 /// ```no_run 901 /// use kernel::io::{ 902 /// Io, 903 /// Mmio, 904 /// Region, 905 /// }; 906 /// 907 /// fn do_reads(io: Mmio<'_, Region>) -> Result { 908 /// // 32-bit read from address `0x10`. 909 /// let v: u32 = io.try_read(0x10)?; 910 /// 911 /// // 8-bit read from address `0xfff`. 912 /// let v: u8 = io.try_read(0xfff)?; 913 /// 914 /// Ok(()) 915 /// } 916 /// ``` 917 #[inline(always)] 918 fn try_read<T, L>(self, location: L) -> Result<T> 919 where 920 T: AsReprMut, 921 L: IoLoc<Self::Target, T>, 922 Self::Backend: IoCapable<<T as AsRepr>::Repr>, 923 { 924 Ok(io_read!(self, try: location)) 925 } 926 927 /// Generic fallible write with runtime bounds check. 928 /// 929 /// # Examples 930 /// 931 /// Write a primitive type to an I/O address: 932 /// 933 /// ```no_run 934 /// use kernel::io::{ 935 /// Io, 936 /// Mmio, 937 /// Region, 938 /// }; 939 /// 940 /// fn do_writes(io: Mmio<'_, Region>) -> Result { 941 /// // 32-bit write of value `1` at address `0x10`. 942 /// io.try_write(0x10, 1u32)?; 943 /// 944 /// // 8-bit write of value `0xff` at address `0xfff`. 945 /// io.try_write(0xfff, 0xffu8)?; 946 /// 947 /// Ok(()) 948 /// } 949 /// ``` 950 #[inline(always)] 951 fn try_write<T, L>(self, location: L, value: T) -> Result 952 where 953 T: AsRepr, 954 L: IoLoc<Self::Target, T>, 955 Self::Backend: IoCapable<<T as AsRepr>::Repr>, 956 { 957 io_write!(self, try: location, value); 958 Ok(()) 959 } 960 961 /// Generic fallible write of a fully-located register value. 962 /// 963 /// # Examples 964 /// 965 /// Tuples carrying a location and a value can be used with this method: 966 /// 967 /// ```no_run 968 /// use kernel::io::{ 969 /// register, 970 /// Io, 971 /// Mmio, 972 /// Region, 973 /// }; 974 /// 975 /// register! { 976 /// base: Region; 977 /// 978 /// VERSION(u32) @ 0x100 { 979 /// 15:8 major; 980 /// 7:0 minor; 981 /// } 982 /// } 983 /// 984 /// impl VERSION { 985 /// fn new(major: u8, minor: u8) -> Self { 986 /// VERSION::zeroed().with_major(major).with_minor(minor) 987 /// } 988 /// } 989 /// 990 /// fn do_write_reg(io: Mmio<'_, Region>) -> Result { 991 /// 992 /// io.try_write_reg(VERSION::new(1, 0)) 993 /// } 994 /// ``` 995 #[inline(always)] 996 fn try_write_reg<T, L, V>(self, value: V) -> Result 997 where 998 T: AsRepr, 999 L: IoLoc<Self::Target, T>, 1000 V: LocatedRegister<Self::Target, Location = L, Value = T>, 1001 Self::Backend: IoCapable<<T as AsRepr>::Repr>, 1002 { 1003 let (location, value) = value.into_io_op(); 1004 1005 self.try_write(location, value) 1006 } 1007 1008 /// Generic fallible update with runtime bounds check. 1009 /// 1010 /// Note: this does not perform any synchronization. The caller is responsible for ensuring 1011 /// exclusive access if required. 1012 /// 1013 /// # Examples 1014 /// 1015 /// Read the u32 value at address `0x10`, increment it, and store the updated value back: 1016 /// 1017 /// ```no_run 1018 /// use kernel::io::{ 1019 /// Io, 1020 /// Mmio, 1021 /// Region, 1022 /// }; 1023 /// 1024 /// fn do_update(io: Mmio<'_, Region<0x1000>>) -> Result { 1025 /// io.try_update(0x10, |v: u32| { 1026 /// v + 1 1027 /// }) 1028 /// } 1029 /// ``` 1030 #[inline(always)] 1031 fn try_update<T, L, F>(self, location: L, f: F) -> Result 1032 where 1033 T: AsReprMut, 1034 L: IoLoc<Self::Target, T>, 1035 Self::Backend: IoCapable<<T as AsRepr>::Repr>, 1036 F: FnOnce(T) -> T, 1037 { 1038 let view = io_project!(self, try: location); 1039 view.write_val(f(view.read_val())); 1040 Ok(()) 1041 } 1042 1043 /// Generic infallible read with compile-time bounds check. 1044 /// 1045 /// # Examples 1046 /// 1047 /// Read a primitive type from an I/O address: 1048 /// 1049 /// ```no_run 1050 /// use kernel::io::{ 1051 /// Io, 1052 /// Mmio, 1053 /// Region, 1054 /// }; 1055 /// 1056 /// fn do_reads(io: Mmio<'_, Region<0x1000>>) { 1057 /// // 32-bit read from address `0x10`. 1058 /// let v: u32 = io.read(0x10); 1059 /// 1060 /// // 8-bit read from the top of the I/O space. 1061 /// let v: u8 = io.read(0xfff); 1062 /// } 1063 /// ``` 1064 #[inline(always)] 1065 fn read<T, L>(self, location: L) -> T 1066 where 1067 T: AsReprMut, 1068 L: IoLoc<Self::Target, T>, 1069 Self::Backend: IoCapable<<T as AsRepr>::Repr>, 1070 { 1071 io_read!(self, build: location) 1072 } 1073 1074 /// Generic infallible write with compile-time bounds check. 1075 /// 1076 /// # Examples 1077 /// 1078 /// Write a primitive type to an I/O address: 1079 /// 1080 /// ```no_run 1081 /// use kernel::io::{ 1082 /// Io, 1083 /// Mmio, 1084 /// Region, 1085 /// }; 1086 /// 1087 /// fn do_writes(io: Mmio<'_, Region<0x1000>>) { 1088 /// // 32-bit write of value `1` at address `0x10`. 1089 /// io.write(0x10, 1u32); 1090 /// 1091 /// // 8-bit write of value `0xff` at the top of the I/O space. 1092 /// io.write(0xfff, 0xffu8); 1093 /// } 1094 /// ``` 1095 #[inline(always)] 1096 fn write<T, L>(self, location: L, value: T) 1097 where 1098 T: AsRepr, 1099 L: IoLoc<Self::Target, T>, 1100 Self::Backend: IoCapable<<T as AsRepr>::Repr>, 1101 { 1102 io_write!(self, build: location, value); 1103 } 1104 1105 /// Generic infallible write of a fully-located register value. 1106 /// 1107 /// # Examples 1108 /// 1109 /// Tuples carrying a location and a value can be used with this method: 1110 /// 1111 /// ```no_run 1112 /// use kernel::io::{ 1113 /// register, 1114 /// Io, 1115 /// Mmio, 1116 /// Region, 1117 /// }; 1118 /// 1119 /// register! { 1120 /// base: Region<0x1000>; 1121 /// 1122 /// VERSION(u32) @ 0x100 { 1123 /// 15:8 major; 1124 /// 7:0 minor; 1125 /// } 1126 /// } 1127 /// 1128 /// impl VERSION { 1129 /// fn new(major: u8, minor: u8) -> Self { 1130 /// VERSION::zeroed().with_major(major).with_minor(minor) 1131 /// } 1132 /// } 1133 /// 1134 /// fn do_write_reg(io: Mmio<'_, Region<0x1000>>) { 1135 /// io.write_reg(VERSION::new(1, 0)); 1136 /// } 1137 /// ``` 1138 #[inline(always)] 1139 fn write_reg<T, L, V>(self, value: V) 1140 where 1141 T: AsRepr, 1142 L: IoLoc<Self::Target, T>, 1143 V: LocatedRegister<Self::Target, Location = L, Value = T>, 1144 Self::Backend: IoCapable<<T as AsRepr>::Repr>, 1145 { 1146 let (location, value) = value.into_io_op(); 1147 1148 self.write(location, value) 1149 } 1150 1151 /// Generic infallible update with compile-time bounds check. 1152 /// 1153 /// Note: this does not perform any synchronization. The caller is responsible for ensuring 1154 /// exclusive access if required. 1155 /// 1156 /// # Examples 1157 /// 1158 /// Read the u32 value at address `0x10`, increment it, and store the updated value back: 1159 /// 1160 /// ```no_run 1161 /// use kernel::io::{ 1162 /// Io, 1163 /// Mmio, 1164 /// Region, 1165 /// }; 1166 /// 1167 /// fn do_update(io: Mmio<'_, Region<0x1000>>) { 1168 /// io.update(0x10, |v: u32| { 1169 /// v + 1 1170 /// }) 1171 /// } 1172 /// ``` 1173 #[inline(always)] 1174 fn update<T, L, F>(self, location: L, f: F) 1175 where 1176 T: AsReprMut, 1177 L: IoLoc<Self::Target, T>, 1178 Self::Backend: IoCapable<<T as AsRepr>::Repr>, 1179 F: FnOnce(T) -> T, 1180 { 1181 let view = io_project!(self, build: location); 1182 view.write_val(f(view.read_val())); 1183 } 1184 } 1185 1186 // Blanket implementation ensures that provided methods cannot be arbitrarily overridden by 1187 // implementers, which is relied upon for correctness and soundness. 1188 impl<'a, T: IoBase<'a>> Io<'a> for T {} 1189 1190 /// A view of memory-mapped I/O region. 1191 /// 1192 /// # Invariant 1193 /// 1194 /// `ptr` points to a valid and aligned memory-mapped I/O region for the duration lifetime `'a`. 1195 pub struct Mmio<'a, T: ?Sized> { 1196 ptr: *mut T, 1197 phantom: PhantomData<&'a ()>, 1198 } 1199 1200 impl<T: ?Sized> Copy for Mmio<'_, T> {} 1201 impl<T: ?Sized> Clone for Mmio<'_, T> { 1202 #[inline] 1203 fn clone(&self) -> Self { 1204 *self 1205 } 1206 } 1207 1208 impl<'a, T: ?Sized> Mmio<'a, T> { 1209 /// Create a `Mmio`, providing the accessors to the MMIO mapping. 1210 /// 1211 /// # Safety 1212 /// 1213 /// `raw` represents a valid and aligned memory-mapped I/O region while `'a` is alive. 1214 #[inline] 1215 pub unsafe fn from_raw(raw: MmioRaw<T>) -> Self { 1216 // INVARIANT: Per safety requirement. 1217 Self { 1218 ptr: raw.ptr, 1219 phantom: PhantomData, 1220 } 1221 } 1222 } 1223 1224 // SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory. 1225 unsafe impl<T: ?Sized + Sync> Send for Mmio<'_, T> {} 1226 1227 // SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory. 1228 unsafe impl<T: ?Sized + Sync> Sync for Mmio<'_, T> {} 1229 1230 impl<'a, T: ?Sized + KnownSize> IoBase<'a> for Mmio<'a, T> { 1231 type Backend = MmioBackend; 1232 type Target = T; 1233 1234 #[inline] 1235 fn as_view(self) -> Mmio<'a, T> { 1236 self 1237 } 1238 } 1239 1240 /// I/O Backend for memory-mapped I/O. 1241 pub struct MmioBackend; 1242 1243 impl IoBackend for MmioBackend { 1244 type View<'a, T: ?Sized + KnownSize> = Mmio<'a, T>; 1245 1246 #[inline] 1247 fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { 1248 view.ptr 1249 } 1250 1251 #[inline] 1252 unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( 1253 _view: Self::View<'a, T>, 1254 ptr: *mut U, 1255 ) -> Self::View<'a, U> { 1256 // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid 1257 // memory-mapped I/O region. 1258 Mmio { 1259 ptr, 1260 phantom: PhantomData, 1261 } 1262 } 1263 } 1264 1265 /// Implements [`IoCapable`] on `$backend` for `$ty` using `$read_fn` and `$write_fn`. 1266 macro_rules! impl_mmio_io_capable { 1267 ($backend: ident, $ty:ty, $read_fn:ident, $write_fn:ident) => { 1268 impl IoCapable<$ty> for $backend { 1269 #[inline] 1270 fn io_read(view: <$backend as IoBackend>::View<'_, $ty>) -> $ty { 1271 // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both 1272 // `MmioBackend` and `RelaxedMmioBackend`. 1273 unsafe { bindings::$read_fn($backend::as_ptr(view).cast_const().cast()) } 1274 } 1275 1276 #[inline] 1277 fn io_write(view: <$backend as IoBackend>::View<'_, $ty>, value: $ty) { 1278 // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both 1279 // `MmioBackend` and `RelaxedMmioBackend`. 1280 unsafe { bindings::$write_fn(value, $backend::as_ptr(view).cast()) } 1281 } 1282 } 1283 }; 1284 } 1285 1286 // MMIO regions support 8, 16, and 32-bit accesses. 1287 impl_mmio_io_capable!(MmioBackend, u8, readb, writeb); 1288 impl_mmio_io_capable!(MmioBackend, u16, readw, writew); 1289 impl_mmio_io_capable!(MmioBackend, u32, readl, writel); 1290 // MMIO regions on 64-bit systems also support 64-bit accesses. 1291 #[cfg(CONFIG_64BIT)] 1292 impl_mmio_io_capable!(MmioBackend, u64, readq, writeq); 1293 1294 impl IoCopyable for MmioBackend { 1295 #[inline] 1296 unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { 1297 // SAFETY: 1298 // - `view.ptr` is valid MMIO memory for `view.size()` bytes. 1299 // - `buffer` is valid for write for `view.size()` bytes. 1300 unsafe { 1301 bindings::memcpy_fromio(buffer.cast(), view.ptr.cast(), view.size()); 1302 } 1303 } 1304 1305 #[inline] 1306 unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { 1307 // SAFETY: 1308 // - `view.ptr` is valid MMIO memory for `view.size()` bytes. 1309 // - `buffer` is valid for read for `view.size()` bytes. 1310 unsafe { 1311 bindings::memcpy_toio(view.ptr.cast(), buffer.cast(), view.size()); 1312 } 1313 } 1314 } 1315 1316 /// [`Mmio`] but using relaxed accessors. 1317 /// 1318 /// This type provides an implementation of [`Io`] that uses relaxed I/O MMIO operands instead of 1319 /// the regular ones. 1320 /// 1321 /// See [`Mmio::relaxed`] for a usage example. 1322 pub struct RelaxedMmio<'a, T: ?Sized>(Mmio<'a, T>); 1323 1324 impl<T: ?Sized> Copy for RelaxedMmio<'_, T> {} 1325 impl<T: ?Sized> Clone for RelaxedMmio<'_, T> { 1326 #[inline] 1327 fn clone(&self) -> Self { 1328 *self 1329 } 1330 } 1331 1332 /// I/O Backend for memory-mapped I/O, with relaxed access semantics. 1333 pub struct RelaxedMmioBackend; 1334 1335 impl IoBackend for RelaxedMmioBackend { 1336 type View<'a, T: ?Sized + KnownSize> = RelaxedMmio<'a, T>; 1337 1338 #[inline] 1339 fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { 1340 MmioBackend::as_ptr(view.0) 1341 } 1342 1343 #[inline] 1344 unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( 1345 view: Self::View<'a, T>, 1346 ptr: *mut U, 1347 ) -> Self::View<'a, U> { 1348 // SAFETY: Per safety requirement. 1349 RelaxedMmio(unsafe { MmioBackend::project_view(view.0, ptr) }) 1350 } 1351 } 1352 1353 impl<'a, T: ?Sized + KnownSize> IoBase<'a> for RelaxedMmio<'a, T> { 1354 type Backend = RelaxedMmioBackend; 1355 type Target = T; 1356 1357 #[inline] 1358 fn as_view(self) -> RelaxedMmio<'a, T> { 1359 self 1360 } 1361 } 1362 1363 impl<'a, T: ?Sized> Mmio<'a, T> { 1364 /// Returns a [`RelaxedMmio`] that performs relaxed I/O operations. 1365 /// 1366 /// Relaxed accessors do not provide ordering guarantees with respect to DMA or memory accesses 1367 /// and can be used when such ordering is not required. 1368 /// 1369 /// # Examples 1370 /// 1371 /// ```no_run 1372 /// use kernel::io::{ 1373 /// Io, 1374 /// Mmio, 1375 /// Region, 1376 /// RelaxedMmio, 1377 /// }; 1378 /// 1379 /// fn do_io(io: Mmio<'_, Region<0x100>>) { 1380 /// // The access is performed using `readl_relaxed` instead of `readl`. 1381 /// let v = io.relaxed().read32(0x10); 1382 /// } 1383 /// 1384 /// ``` 1385 #[inline] 1386 pub fn relaxed(self) -> RelaxedMmio<'a, T> { 1387 RelaxedMmio(self) 1388 } 1389 } 1390 1391 // MMIO regions support 8, 16, and 32-bit accesses. 1392 impl_mmio_io_capable!(RelaxedMmioBackend, u8, readb_relaxed, writeb_relaxed); 1393 impl_mmio_io_capable!(RelaxedMmioBackend, u16, readw_relaxed, writew_relaxed); 1394 impl_mmio_io_capable!(RelaxedMmioBackend, u32, readl_relaxed, writel_relaxed); 1395 // MMIO regions on 64-bit systems also support 64-bit accesses. 1396 #[cfg(CONFIG_64BIT)] 1397 impl_mmio_io_capable!(RelaxedMmioBackend, u64, readq_relaxed, writeq_relaxed); 1398 1399 /// I/O Backend for system memory. 1400 pub struct SysMemBackend; 1401 1402 impl IoBackend for SysMemBackend { 1403 type View<'a, T: ?Sized + KnownSize> = SysMem<'a, T>; 1404 1405 #[inline] 1406 fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { 1407 view.ptr 1408 } 1409 1410 #[inline] 1411 unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( 1412 _view: Self::View<'a, T>, 1413 ptr: *mut U, 1414 ) -> Self::View<'a, U> { 1415 // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid 1416 // kernel accessible memory region. 1417 SysMem { 1418 ptr, 1419 phantom: PhantomData, 1420 } 1421 } 1422 } 1423 1424 /// Implements [`IoCapable`] on `SysMemBackend` for `$ty` using `read_volatile` and 1425 /// `write_volatile`. 1426 macro_rules! impl_sysmem_io_capable { 1427 ($ty:ty) => { 1428 impl IoCapable<$ty> for SysMemBackend { 1429 #[inline] 1430 fn io_read(view: SysMem<'_, $ty>) -> $ty { 1431 // SAFETY: 1432 // - Per type invariant, `ptr` is valid and aligned. 1433 // - Using read_volatile() here so that race with hardware is well-defined. 1434 // - Using read_volatile() here is not sound if it races with other CPU per Rust 1435 // rules, but this is allowed per LKMM. 1436 // - The macro is only used on primitives so all bit patterns are valid. 1437 unsafe { view.ptr.read_volatile() } 1438 } 1439 1440 #[inline] 1441 fn io_write(view: SysMem<'_, $ty>, value: $ty) { 1442 // SAFETY: 1443 // - Per type invariant, `ptr` is valid and aligned. 1444 // - Using write_volatile() here so that race with hardware is well-defined. 1445 // - Using write_volatile() here is not sound if it races with other CPU per Rust 1446 // rules, but this is allowed per LKMM. 1447 unsafe { view.ptr.write_volatile(value) } 1448 } 1449 } 1450 }; 1451 } 1452 1453 impl_sysmem_io_capable!(u8); 1454 impl_sysmem_io_capable!(u16); 1455 impl_sysmem_io_capable!(u32); 1456 #[cfg(CONFIG_64BIT)] 1457 impl_sysmem_io_capable!(u64); 1458 1459 impl IoCopyable for SysMemBackend { 1460 #[inline] 1461 unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { 1462 // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile. 1463 // SAFETY: 1464 // - `view.ptr` is in CPU address space and valid for read. 1465 // - `buffer` is valid for write for `view.size()` bytes which is equal to `view.ptr.len()`. 1466 unsafe { bindings::memcpy(buffer.cast(), view.ptr.cast(), view.ptr.len()) }; 1467 } 1468 1469 #[inline] 1470 unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { 1471 // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile. 1472 // SAFETY: 1473 // - `view.ptr` is in CPU address space and valid for write. 1474 // - `buffer` is valid for read for `view.size()` bytes which is equal to `view.ptr.len()`. 1475 unsafe { bindings::memcpy(view.ptr.cast(), buffer.cast(), view.ptr.len()) }; 1476 } 1477 1478 #[inline] 1479 fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T { 1480 // SAFETY: 1481 // - Per type invariant, `ptr` is valid and aligned. 1482 // - Using read_volatile() here so that race with hardware is well-defined. 1483 // - Using read_volatile() here is not sound if it races with other CPU per Rust 1484 // rules, but this is allowed per LKMM. 1485 // - `T: FromBytes` so all bit patterns are valid. 1486 unsafe { view.ptr.read_volatile() } 1487 } 1488 1489 #[inline] 1490 fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) { 1491 // SAFETY: 1492 // - Per type invariant, `ptr` is valid and aligned. 1493 // - Using write_volatile() here so that race with hardware is well-defined. 1494 // - Using write_volatile() here is not sound if it races with other CPU per Rust 1495 // rules, but this is allowed per LKMM. 1496 unsafe { view.ptr.write_volatile(value) } 1497 } 1498 } 1499 1500 /// A view of a system memory region. 1501 /// 1502 /// Provides `Io` trait implementation for kernel virtual address ranges, 1503 /// using volatile read/write to safely access shared memory that may be 1504 /// concurrently accessed by external hardware. 1505 /// 1506 /// # Invariants 1507 /// 1508 /// `self.ptr.addr() .. self.ptr.addr() + KnownSize::size(self.ptr)` is valid and aligned kernel 1509 /// accessible memory region for the lifetime `'a`. 1510 pub struct SysMem<'a, T: ?Sized> { 1511 ptr: *mut T, 1512 phantom: PhantomData<&'a ()>, 1513 } 1514 1515 impl<T: ?Sized> Copy for SysMem<'_, T> {} 1516 impl<T: ?Sized> Clone for SysMem<'_, T> { 1517 #[inline] 1518 fn clone(&self) -> Self { 1519 *self 1520 } 1521 } 1522 1523 // SAFETY: `SysMem<'_, T>` is conceptually `&T`. 1524 unsafe impl<T: ?Sized + Sync> Send for SysMem<'_, T> {} 1525 1526 // SAFETY: `SysMem<'_, T>` is conceptually `&T`. 1527 unsafe impl<T: ?Sized + Sync> Sync for SysMem<'_, T> {} 1528 1529 impl<'a, T: ?Sized> SysMem<'a, T> { 1530 /// Create a `SysMem` from a raw pointer. 1531 /// 1532 /// # Safety 1533 /// 1534 /// `ptr.addr() .. ptr.addr() + KnownSize::size(ptr)` must be valid and aligned kernel 1535 /// accessible memory region for the lifetime `'a`. 1536 #[inline] 1537 pub unsafe fn new(ptr: *mut T) -> Self { 1538 // INVARIANT: Per safety requirement. 1539 Self { 1540 ptr, 1541 phantom: PhantomData, 1542 } 1543 } 1544 1545 /// Obtain the raw pointer to the memory. 1546 #[inline] 1547 pub fn as_ptr(self) -> *mut T { 1548 self.ptr 1549 } 1550 } 1551 1552 impl<'a, T: ?Sized + KnownSize> IoBase<'a> for SysMem<'a, T> { 1553 type Backend = SysMemBackend; 1554 type Target = T; 1555 1556 #[inline] 1557 fn as_view(self) -> <Self::Backend as IoBackend>::View<'a, Self::Target> { 1558 self 1559 } 1560 } 1561 1562 /// I/O Backend for [`IoSysMap`]. 1563 pub struct IoSysMapBackend; 1564 1565 /// Either [`Mmio`] or [`SysMem`]. 1566 /// 1567 /// This can be used when a piece of logic may wish to handle both MMIO or system memory but does 1568 /// not want or cannot be generic over I/O backends. This serves a similar purpose to 1569 /// [`include/linux/iosys-map.h`] in C. 1570 /// 1571 /// This type can be used like any other types that implements [`Io`]; this also include 1572 /// [`io_project!`], [`io_read!`], [`io_write!`]. 1573 /// 1574 /// [`include/linux/iosys-map.h`]: srctree/include/linux/iosys-map.h 1575 pub enum IoSysMap<'a, T: ?Sized> { 1576 /// The view is I/O memory. 1577 Io(Mmio<'a, T>), 1578 /// The view is system memory. 1579 Sys(SysMem<'a, T>), 1580 } 1581 1582 impl<T: ?Sized> Copy for IoSysMap<'_, T> {} 1583 impl<T: ?Sized> Clone for IoSysMap<'_, T> { 1584 #[inline] 1585 fn clone(&self) -> Self { 1586 *self 1587 } 1588 } 1589 1590 impl<'a, T: ?Sized> From<Mmio<'a, T>> for IoSysMap<'a, T> { 1591 #[inline] 1592 fn from(value: Mmio<'a, T>) -> Self { 1593 IoSysMap::Io(value) 1594 } 1595 } 1596 1597 impl<'a, T: ?Sized> From<SysMem<'a, T>> for IoSysMap<'a, T> { 1598 #[inline] 1599 fn from(value: SysMem<'a, T>) -> Self { 1600 IoSysMap::Sys(value) 1601 } 1602 } 1603 1604 impl IoBackend for IoSysMapBackend { 1605 type View<'a, T: ?Sized + KnownSize> = IoSysMap<'a, T>; 1606 1607 #[inline] 1608 fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { 1609 match view { 1610 IoSysMap::Io(l) => MmioBackend::as_ptr(l), 1611 IoSysMap::Sys(r) => SysMemBackend::as_ptr(r), 1612 } 1613 } 1614 1615 #[inline] 1616 unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( 1617 view: Self::View<'a, T>, 1618 ptr: *mut U, 1619 ) -> Self::View<'a, U> { 1620 match view { 1621 // SAFETY: Per safety requirement. 1622 IoSysMap::Io(l) => IoSysMap::Io(unsafe { MmioBackend::project_view(l, ptr) }), 1623 // SAFETY: Per safety requirement. 1624 IoSysMap::Sys(r) => IoSysMap::Sys(unsafe { SysMemBackend::project_view(r, ptr) }), 1625 } 1626 } 1627 } 1628 1629 impl<T> IoCapable<T> for IoSysMapBackend 1630 where 1631 MmioBackend: IoCapable<T>, 1632 SysMemBackend: IoCapable<T>, 1633 { 1634 #[inline] 1635 fn io_read(view: Self::View<'_, T>) -> T { 1636 match view { 1637 IoSysMap::Io(l) => MmioBackend::io_read(l), 1638 IoSysMap::Sys(r) => SysMemBackend::io_read(r), 1639 } 1640 } 1641 1642 #[inline] 1643 fn io_write<'a>(view: Self::View<'a, T>, value: T) { 1644 match view { 1645 IoSysMap::Io(l) => MmioBackend::io_write(l, value), 1646 IoSysMap::Sys(r) => SysMemBackend::io_write(r, value), 1647 } 1648 } 1649 } 1650 1651 impl IoCopyable for IoSysMapBackend { 1652 #[inline] 1653 unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { 1654 match view { 1655 // SAFETY: Per safety requirement. 1656 IoSysMap::Io(l) => unsafe { MmioBackend::copy_from_io(l, buffer) }, 1657 // SAFETY: Per safety requirement. 1658 IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_from_io(r, buffer) }, 1659 } 1660 } 1661 1662 #[inline] 1663 unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { 1664 match view { 1665 // SAFETY: Per safety requirement. 1666 IoSysMap::Io(l) => unsafe { MmioBackend::copy_to_io(l, buffer) }, 1667 // SAFETY: Per safety requirement. 1668 IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_to_io(r, buffer) }, 1669 } 1670 } 1671 1672 #[inline] 1673 fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T { 1674 match view { 1675 IoSysMap::Io(l) => MmioBackend::copy_read(l), 1676 IoSysMap::Sys(r) => SysMemBackend::copy_read(r), 1677 } 1678 } 1679 1680 #[inline] 1681 fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) { 1682 match view { 1683 IoSysMap::Io(l) => MmioBackend::copy_write(l, value), 1684 IoSysMap::Sys(r) => SysMemBackend::copy_write(r, value), 1685 } 1686 } 1687 } 1688 1689 impl<'a, T: ?Sized + KnownSize> IoBase<'a> for IoSysMap<'a, T> { 1690 type Backend = IoSysMapBackend; 1691 type Target = T; 1692 1693 #[inline] 1694 fn as_view(self) -> IoSysMap<'a, T> { 1695 self 1696 } 1697 } 1698 1699 // This helper turns associated functions to methods so it can be invoked in macro. 1700 // Used by `io_project!()` only. 1701 #[doc(hidden)] 1702 #[derive(Clone, Copy)] 1703 pub struct ProjectHelper<T>(pub T); 1704 1705 impl<'a, T> ProjectHelper<T> 1706 where 1707 T: Io<'a, Backend: IoBackend<View<'a, T::Target> = T>>, 1708 { 1709 // These helper methods must not have symbols present in the binary to avoid confusion. 1710 #[inline(always)] 1711 pub fn as_ptr(self) -> *mut T::Target { 1712 T::Backend::as_ptr(self.0) 1713 } 1714 1715 /// # Safety 1716 /// 1717 /// Same as `IoBackend::project_view` 1718 #[inline(always)] 1719 pub unsafe fn project_view<U: ?Sized + KnownSize>( 1720 self, 1721 ptr: *mut U, 1722 ) -> <T::Backend as IoBackend>::View<'a, U> { 1723 // SAFETY: Per safety requirement. 1724 unsafe { T::Backend::project_view::<T::Target, _>(self.0, ptr) } 1725 } 1726 1727 #[inline(always)] 1728 pub fn try_project_loc<U, L>( 1729 self, 1730 location: L, 1731 ) -> Result<<T::Backend as IoBackend>::View<'a, U>> 1732 where 1733 L: IoLoc<T::Target, U>, 1734 { 1735 io_view::<_, U>(self.0, location.offset()) 1736 } 1737 1738 #[inline(always)] 1739 pub fn project_loc<U, L>(self, location: L) -> <T::Backend as IoBackend>::View<'a, U> 1740 where 1741 L: IoLoc<T::Target, U>, 1742 { 1743 io_view_assert::<_, U>(self.0, location.offset()) 1744 } 1745 } 1746 1747 /// Project an I/O type to a subview of it. 1748 /// 1749 /// The syntax is of form `io_project!(io, proj)` where `io` is an expression to a type that 1750 /// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!). 1751 /// 1752 /// `io_project!` can also project to a subview of registers defined with [`register!`] macro. 1753 /// Register projection has syntax `io_project!(io, try: REGISTER)` for fallible projection and 1754 /// `io_project!(io, build: REGISTER)` for infallible projection. 1755 /// 1756 /// # Examples 1757 /// 1758 /// ``` 1759 /// use kernel::io::{ 1760 /// io_project, 1761 /// register, 1762 /// Mmio, 1763 /// }; 1764 /// #[repr(C)] 1765 /// struct MyStruct { field: u32, } 1766 /// 1767 /// register! { 1768 /// base: MyStruct; 1769 /// FIELD(u32) @ 0 { 1770 /// 31:0 val; 1771 /// } 1772 /// } 1773 /// 1774 /// # fn test(mmio: Mmio<'_, [MyStruct]>) -> Result { 1775 /// // let mmio: Mmio<[MyStruct]>; 1776 /// let field: Mmio<'_, u32> = io_project!(mmio, [try: 1].field); 1777 /// let whole: Mmio<'_, MyStruct> = io_project!(mmio, [try: 2]); 1778 /// let nested: Mmio<'_, u32> = io_project!(whole, .field); 1779 /// let reg: Mmio<'_, FIELD> = io_project!(whole, build: FIELD); 1780 /// # Ok::<(), Error>(()) } 1781 /// ``` 1782 #[macro_export] 1783 #[doc(hidden)] 1784 macro_rules! io_project { 1785 // Register projection 1786 ($io:expr, try: $ioloc:expr) => {{ 1787 #[allow(unused)] 1788 use $crate::io::IoBase as _; 1789 let view = $crate::io::ProjectHelper($io.as_view()); 1790 view.try_project_loc($ioloc)? 1791 }}; 1792 ($io:expr, build: $ioloc:expr) => {{ 1793 #[allow(unused)] 1794 use $crate::io::IoBase as _; 1795 let view = $crate::io::ProjectHelper($io.as_view()); 1796 view.project_loc($ioloc) 1797 }}; 1798 1799 // Field or index projection 1800 ($io:expr, $($proj:tt)*) => {{ 1801 #[allow(unused)] 1802 use $crate::io::IoBase as _; 1803 let view = $crate::io::ProjectHelper($io.as_view()); 1804 let ptr = $crate::ptr::project!( 1805 mut view.as_ptr(), $($proj)* 1806 ); 1807 #[allow(unused_unsafe)] 1808 // SAFETY: `ptr` is a projection. 1809 unsafe { view.project_view(ptr) } 1810 }}; 1811 } 1812 #[doc(inline)] 1813 pub use crate::io_project; 1814 1815 /// Read from I/O memory. 1816 /// 1817 /// The syntax is of form `io_read!(io, proj)` where `io` is an expression to a type that 1818 /// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!). 1819 /// 1820 /// # Examples 1821 /// 1822 /// ``` 1823 /// #[repr(C)] 1824 /// struct MyStruct { field: u32, } 1825 /// 1826 /// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result { 1827 /// // let mmio: Mmio<'_, [MyStruct]>; 1828 /// let field: u32 = kernel::io::io_read!(mmio, [try: 2].field); 1829 /// # Ok::<(), Error>(()) } 1830 /// ``` 1831 #[macro_export] 1832 #[doc(hidden)] 1833 macro_rules! io_read { 1834 ($io:expr, $($proj:tt)*) => { 1835 $crate::io::Io::read_val($crate::io_project!($io, $($proj)*)) 1836 }; 1837 } 1838 #[doc(inline)] 1839 pub use crate::io_read; 1840 1841 /// Writes to I/O memory. 1842 /// 1843 /// The syntax is of form `io_write!(io, proj, val)` where `io` is an expression to a type that 1844 /// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!), 1845 /// and `val` is the value to be written to the projected location. 1846 /// 1847 /// # Examples 1848 /// 1849 /// ``` 1850 /// #[repr(C)] 1851 /// struct MyStruct { field: u32, } 1852 /// 1853 /// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result { 1854 /// // let mmio: Mmio<'_, [MyStruct]>; 1855 /// kernel::io::io_write!(mmio, [try: 2].field, 10); 1856 /// # Ok::<(), Error>(()) } 1857 /// ``` 1858 #[macro_export] 1859 #[doc(hidden)] 1860 macro_rules! io_write { 1861 (@parse [$io:expr] [$($proj:tt)*] [, $val:expr]) => { 1862 $crate::io::Io::write_val($crate::io_project!($io, $($proj)*), $val) 1863 }; 1864 (@parse [$io:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => { 1865 $crate::io_write!(@parse [$io] [$($proj)* .$field] [$($rest)*]) 1866 }; 1867 (@parse [$io:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => { 1868 $crate::io_write!(@parse [$io] [$($proj)* [$flavor: $index]] [$($rest)*]) 1869 }; 1870 (@parse [$io:expr] [] [try: $ioloc:expr, $($rest:tt)*]) => { 1871 $crate::io_write!(@parse [$io] [try: $ioloc] [, $($rest)*]) 1872 }; 1873 (@parse [$io:expr] [] [build: $ioloc:expr, $($rest:tt)*]) => { 1874 $crate::io_write!(@parse [$io] [build: $ioloc] [, $($rest)*]) 1875 }; 1876 ($io:expr, $($rest:tt)*) => { 1877 $crate::io_write!(@parse [$io] [] [$($rest)*]) 1878 }; 1879 } 1880 #[doc(inline)] 1881 pub use crate::io_write; 1882