xref: /linux/rust/kernel/io.rs (revision 486b257357c23743d9729ea715136d105fff461e)
1 // SPDX-License-Identifier: GPL-2.0
2 
3 //! Memory-mapped IO.
4 //!
5 //! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h)
6 
7 use core::{
8     marker::PhantomData,
9     mem::MaybeUninit,
10     num::TryFromIntError, //
11 };
12 
13 use crate::{
14     bindings,
15     fmt,
16     mem::{
17         AsRepr,
18         AsReprMut, //
19     },
20     prelude::*,
21     ptr::{
22         Alignment,
23         KnownSize, //
24     }, //
25 };
26 
27 #[cfg(CONFIG_HAS_IOMEM)]
28 pub mod mem;
29 pub mod poll;
30 pub mod register;
31 pub mod resource;
32 
33 pub use crate::register;
34 pub use resource::Resource;
35 
36 use register::LocatedRegister;
37 
38 /// Physical address type.
39 ///
40 /// This is a type alias to either `u32` or `u64` depending on the config option
41 /// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures.
42 pub type PhysAddr = bindings::phys_addr_t;
43 
44 /// Resource size type.
45 ///
46 /// This wraps either `u32` or `u64` depending on the config option
47 /// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a `u64` even on 32-bit architectures.
48 ///
49 /// # Examples
50 ///
51 /// ```
52 /// use kernel::io::ResourceSize;
53 ///
54 /// let size = ResourceSize::from_raw(0x1000);
55 /// assert_eq!(size.into_raw(), 0x1000);
56 ///
57 /// // Round-trips through the raw C type.
58 /// let raw: kernel::bindings::resource_size_t = size.into();
59 /// assert_eq!(ResourceSize::from(raw), size);
60 ///
61 /// // Fallible conversion to `usize` (can truncate on 32-bit).
62 /// assert_eq!(usize::try_from(size)?, 0x1000);
63 /// # Ok::<(), core::num::TryFromIntError>(())
64 /// ```
65 #[repr(transparent)]
66 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
67 pub struct ResourceSize(bindings::resource_size_t);
68 
69 impl ResourceSize {
70     /// Creates a resource size from the raw C type.
71     #[inline]
72     pub const fn from_raw(value: bindings::resource_size_t) -> Self {
73         Self(value)
74     }
75 
76     /// Turns this resource size into the raw C type.
77     #[inline]
78     pub const fn into_raw(self) -> bindings::resource_size_t {
79         self.0
80     }
81 }
82 
83 impl fmt::Debug for ResourceSize {
84     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
85         write!(f, "{:#x}", self.0)
86     }
87 }
88 
89 impl From<bindings::resource_size_t> for ResourceSize {
90     #[inline]
91     fn from(value: bindings::resource_size_t) -> Self {
92         Self::from_raw(value)
93     }
94 }
95 
96 impl From<ResourceSize> for bindings::resource_size_t {
97     #[inline]
98     fn from(value: ResourceSize) -> Self {
99         value.into_raw()
100     }
101 }
102 
103 impl TryFrom<ResourceSize> for usize {
104     type Error = TryFromIntError;
105 
106     #[inline]
107     fn try_from(value: ResourceSize) -> Result<Self, Self::Error> {
108         Self::try_from(value.into_raw())
109     }
110 }
111 
112 /// Untyped I/O region.
113 ///
114 /// This type can be used when an I/O region without known type information has a compile-time known
115 /// minimum size (and a runtime known actual size).
116 ///
117 /// # Invariants
118 ///
119 /// - Size of the region is at least as large as the `SIZE` generic parameter.
120 /// - Size of the region is multiple of 4.
121 #[repr(C, align(4))]
122 #[derive(FromBytes)]
123 pub struct Region<const SIZE: usize = 0> {
124     inner: [u8],
125 }
126 
127 impl<const SIZE: usize> Region<SIZE> {
128     /// Create a raw mutable pointer from given base address and size.
129     ///
130     /// `size` should be at least as large as the minimum size `SIZE`, and `base` and `size` should
131     /// be 4-byte aligned to uphold the type invariant.
132     ///
133     /// Just like other methods on raw pointers, it is not unsafe to create a raw pointer
134     /// that does not uphold the type invariants. However such pointers are not valid.
135     #[inline]
136     pub fn ptr_from_raw_parts_mut(base: *mut u8, size: usize) -> *mut Self {
137         core::ptr::slice_from_raw_parts_mut(base, size) as *mut Region<SIZE>
138     }
139 
140     /// Create a raw mutable pointer from given base address and size.
141     ///
142     /// The alignment of `base` is checked, and `size` is checked against the minimum size specified
143     /// via const generics.
144     #[inline]
145     pub fn ptr_try_from_raw_parts_mut(base: *mut u8, size: usize) -> Result<*mut Self> {
146         if size < SIZE || base.align_offset(4) != 0 || !size.is_multiple_of(4) {
147             return Err(EINVAL);
148         }
149 
150         Ok(Self::ptr_from_raw_parts_mut(base, size))
151     }
152 }
153 
154 impl<const SIZE: usize> KnownSize for Region<SIZE> {
155     const MIN_SIZE: usize = SIZE;
156     // Alignment of 4 is the most common; different base types can be added once required.
157     const MIN_ALIGN: Alignment = Alignment::new::<4>();
158 
159     #[inline(always)]
160     fn size(p: *const Self) -> usize {
161         (p as *const [u8]).len()
162     }
163 }
164 
165 // SAFETY:
166 // - Values read from I/O are always treated as initialized.
167 // - Per type invariant the size is multiple of 4 and the type is 4-byte aligned, so it is padding
168 //   free.
169 //
170 // This cannot be derived as `derive(IntoBytes)` as the padding free property comes from type
171 // invariant which the macro does not know.
172 unsafe impl<const SIZE: usize> IntoBytes for Region<SIZE> {
173     #[inline]
174     #[allow(unused)] // Rust 1.87+ stops requiring this and will emit unused warnings.
175     fn only_derive_is_allowed_to_implement_this_trait() {}
176 }
177 
178 /// Raw representation of an MMIO region.
179 ///
180 /// `MmioRaw<T>` is equivalent to `T __iomem *` in C.
181 ///
182 /// By itself, the existence of an instance of this structure does not provide any guarantees that
183 /// the represented MMIO region does exist or is properly mapped.
184 ///
185 /// Instead, the bus specific MMIO implementation must convert this raw representation into an
186 /// `Mmio` instance providing the actual memory accessors. Only by the conversion into an `Mmio`
187 /// structure any guarantees are given.
188 pub struct MmioRaw<T: ?Sized> {
189     /// Pointer is in I/O address space.
190     ///
191     /// The provenance does not matter, only the address and metadata do.
192     ptr: *mut T,
193 }
194 
195 impl<T: ?Sized> Copy for MmioRaw<T> {}
196 impl<T: ?Sized> Clone for MmioRaw<T> {
197     #[inline]
198     fn clone(&self) -> Self {
199         *self
200     }
201 }
202 
203 // SAFETY: `MmioRaw` is just an address, so is thread-safe.
204 unsafe impl<T: ?Sized> Send for MmioRaw<T> {}
205 // SAFETY: `MmioRaw` is just an address, so is thread-safe.
206 unsafe impl<T: ?Sized> Sync for MmioRaw<T> {}
207 
208 impl<T> MmioRaw<T> {
209     /// Create a `MmioRaw` from address.
210     #[inline]
211     pub fn new(addr: usize) -> Self {
212         Self {
213             ptr: core::ptr::without_provenance_mut(addr),
214         }
215     }
216 }
217 
218 impl<const SIZE: usize> MmioRaw<Region<SIZE>> {
219     /// Create a `MmioRaw` representing a I/O region with given size.
220     ///
221     /// The size is checked against the minimum size specified via const generics.
222     #[inline]
223     pub fn new_region(addr: usize, size: usize) -> Result<Self> {
224         Ok(Self {
225             ptr: Region::ptr_try_from_raw_parts_mut(core::ptr::without_provenance_mut(addr), size)?,
226         })
227     }
228 }
229 
230 impl<T: ?Sized + KnownSize> MmioRaw<T> {
231     /// Returns the base address of the MMIO region.
232     #[inline]
233     pub fn addr(&self) -> usize {
234         self.ptr.addr()
235     }
236 
237     /// Returns the size of the MMIO region.
238     #[inline]
239     pub fn size(&self) -> usize {
240         KnownSize::size(self.ptr)
241     }
242 }
243 
244 /// Checks whether an access of type `U` at the given `base` and the given `offset`
245 /// is valid within this region.
246 ///
247 /// The `base` is used for alignment checking only. This can be set to 0 to skip the check.
248 #[inline]
249 const fn offset_valid<U>(base: usize, offset: usize, size: usize) -> bool {
250     if let Some(end) = offset.checked_add(size_of::<U>()) {
251         end <= size && (base.wrapping_add(offset) % align_of::<U>() == 0)
252     } else {
253         false
254     }
255 }
256 
257 /// Returns a view for a given `offset`, performing compile-time bound checks.
258 // Always inline to optimize out error path of `build_assert`.
259 #[inline(always)]
260 fn io_view_assert<'a, IO: Io<'a>, U>(
261     this: IO,
262     offset: usize,
263 ) -> <IO::Backend as IoBackend>::View<'a, U> {
264     // We cannot check alignment with `offset_valid` using `ptr.addr()`. So set 0 for it and
265     // ensure alignment by checking that the alignment of `U` is smaller or equal to the
266     // alignment of `IO::Target`.
267     const_assert!(Alignment::of::<U>().as_usize() <= IO::Target::MIN_ALIGN.as_usize());
268     build_assert!(offset_valid::<U>(0, offset, IO::Target::MIN_SIZE));
269 
270     let view = this.as_view();
271     let ptr = IO::Backend::as_ptr(view);
272     let projected_ptr = ptr.cast::<U>().wrapping_byte_add(offset);
273     // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a
274     // valid projection.
275     unsafe { IO::Backend::project_view(view, projected_ptr) }
276 }
277 
278 /// Returns a view for a given `offset`, performing runtime bound checks.
279 #[inline]
280 fn io_view<'a, IO: Io<'a>, U>(
281     this: IO,
282     offset: usize,
283 ) -> Result<<IO::Backend as IoBackend>::View<'a, U>> {
284     let view = this.as_view();
285     let ptr = IO::Backend::as_ptr(view);
286 
287     if !offset_valid::<U>(ptr.addr(), offset, KnownSize::size(ptr)) {
288         return Err(EINVAL);
289     }
290 
291     let projected_ptr = ptr.cast::<U>().wrapping_byte_add(offset);
292     // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a
293     // valid projection.
294     Ok(unsafe { IO::Backend::project_view(view, projected_ptr) })
295 }
296 
297 /// Returns the primitive view of a I/O view.
298 #[inline]
299 fn io_view_as_repr<'a, IO: Io<'a, Target = T>, T: AsRepr>(
300     this: IO,
301 ) -> <IO::Backend as IoBackend>::View<'a, T::Repr> {
302     let view = this.as_view();
303 
304     // SAFETY: `AsRepr` guarantees layout compatibility.
305     unsafe { IO::Backend::project_view(view, IO::Backend::as_ptr(view).cast::<T::Repr>()) }
306 }
307 
308 /// I/O backends.
309 ///
310 /// This is an abstract representation to be implemented by arbitrary I/O
311 /// backends (e.g. MMIO, PCI config space, etc.).
312 ///
313 /// The base trait only defines the projection operations; which I/O methods are available depends
314 /// on which [`IoCapable<T>`] traits are implemented for the type. For example, for MMIO regions,
315 /// all widths (u8, u16, u32, and u64 on 64-bit systems) are typically supported. For PCI
316 /// configuration space, u8, u16, and u32 are supported but u64 is not.
317 ///
318 /// This trait is separate from the `Io` trait as multiple different I/O types may share the same
319 /// operation.
320 pub trait IoBackend {
321     /// View type for this I/O backend.
322     type View<'a, T: ?Sized + KnownSize>: IoBase<'a, Backend = Self, Target = T>;
323 
324     /// Convert a `view` to a raw pointer for projection.
325     ///
326     /// The returned pointer is private implementation detail of the backend; it is likely not
327     /// valid. It should not be dereferenced.
328     fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T;
329 
330     /// Project `view` to its subregion indicated by `ptr`.
331     ///
332     /// If input `view` is valid, returned view must also be valid.
333     ///
334     /// # Safety
335     ///
336     /// `ptr` must be a projection of `Self::as_ptr(view)`.
337     unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
338         view: Self::View<'a, T>,
339         ptr: *mut U,
340     ) -> Self::View<'a, U>;
341 }
342 
343 /// Trait indicating that an I/O backend supports operations of a certain type and providing an
344 /// implementation for these operations.
345 ///
346 /// Different I/O backends can implement this trait to expose only the operations they support.
347 ///
348 /// For example, a PCI configuration space may implement `IoCapable<u8>`, `IoCapable<u16>`,
349 /// and `IoCapable<u32>`, but not `IoCapable<u64>`, while an MMIO region on a 64-bit
350 /// system might implement all four.
351 pub trait IoCapable<T>: IoBackend {
352     /// Performs an I/O read of type `T` at `view` and returns the result.
353     fn io_read<'a>(view: Self::View<'a, T>) -> T;
354 
355     /// Performs an I/O write of `value` at `view`.
356     fn io_write<'a>(view: Self::View<'a, T>, value: T);
357 }
358 
359 /// Trait indicating that an I/O backend supports memory copy operations.
360 pub trait IoCopyable: IoBackend {
361     /// Copy contents of `view` to `buffer`.
362     ///
363     /// # Safety
364     ///
365     /// - `buffer` is valid for volatile write for `view.size()` bytes.
366     /// - `buffer` should not overlap with `view`.
367     unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8);
368 
369     /// Copy contents from `buffer` to `view`.
370     ///
371     /// # Safety
372     ///
373     /// - `buffer` is valid for volatile read for `view.size()` bytes.
374     /// - `buffer` should not overlap with `view`.
375     unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8);
376 
377     /// Copy from `view` and return the value.
378     #[inline]
379     fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T {
380         // Project `self` to `[u8]`.
381         let ptr = Self::as_ptr(view);
382         // SAFETY: This is a identity projection.
383         let slice_view = unsafe {
384             Self::project_view(
385                 view,
386                 core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(), size_of::<T>()),
387             )
388         };
389 
390         let mut buf = MaybeUninit::<T>::uninit();
391         // SAFETY:
392         // - `buf.as_mut_ptr()` is valid for write for `size_of::<T>()` bytes.
393         // - `buf` is local so `buf.as_mut_ptr()` cannot overlap with `slice_view`.
394         unsafe { Self::copy_from_io(slice_view, buf.as_mut_ptr().cast()) };
395         // SAFETY: `T: FromBytes` guarantee that all bit patterns are valid.
396         unsafe { buf.assume_init() }
397     }
398 
399     /// Copy `value` to `view`.
400     ///
401     /// Destructor of `value` will not be executed, consistent with [`zerocopy::transmute`].
402     #[inline]
403     fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) {
404         // Project `self` to `[u8]`.
405         let ptr = Self::as_ptr(view);
406         // SAFETY: This is a identity projection.
407         let slice_view = unsafe {
408             Self::project_view(
409                 view,
410                 core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(), size_of::<T>()),
411             )
412         };
413 
414         // SAFETY:
415         // - `&raw const value` is valid for read for `size_of::<T>()` bytes.
416         // - `value` is local so `&raw const value` cannot overlap with `slice_view`.
417         unsafe { Self::copy_to_io(slice_view, (&raw const value).cast()) };
418         core::mem::forget(value);
419     }
420 }
421 
422 /// Describes a given I/O location: its offset, width, and type to convert the raw value from and
423 /// into.
424 ///
425 /// This trait is the key abstraction allowing [`Io::read`], [`Io::write`], and [`Io::update`] (and
426 /// their fallible [`try_read`](Io::try_read), [`try_write`](Io::try_write) and
427 /// [`try_update`](Io::try_update) counterparts) to work uniformly with both raw [`usize`] offsets
428 /// (for primitive types like [`u32`]) and typed ones (like those generated by the [`register!`]
429 /// macro).
430 ///
431 /// An `IoLoc<Base, T>` carries the following pieces of information:
432 ///
433 /// - The valid `Base` to operate on. For most registers, this should be [`Region`].
434 /// - The offset to access (returned by [`IoLoc::offset`]),
435 /// - The type `T` in which the data is returned or provided.
436 ///
437 /// `T` is not necessarily the type for underlying I/O operation. Methods that take `IoLoc` have `T:
438 /// AsRepr` bound and the `<T as AsRepr>::Repr` type would be used to perform I/O and converted to
439 /// `T` instead.
440 pub trait IoLoc<Base: ?Sized, T> {
441     /// Consumes `self` and returns the offset of this location.
442     fn offset(self) -> usize;
443 }
444 
445 /// Implements [`IoLoc<Region<SIZE>, $ty>`] for [`usize`], allowing [`usize`] to be used as a
446 /// parameter of [`Io::read`] and [`Io::write`].
447 macro_rules! impl_usize_ioloc {
448     ($($ty:ty),*) => {
449         $(
450             impl<const SIZE: usize> IoLoc<Region<SIZE>, $ty> for usize {
451                 #[inline(always)]
452                 fn offset(self) -> usize {
453                     self
454                 }
455             }
456         )*
457     }
458 }
459 
460 // Provide the ability to read any primitive type from a [`usize`].
461 impl_usize_ioloc!(u8, u16, u32, u64);
462 
463 /// Types implementing this trait (e.g. MMIO BARs or PCI config regions)
464 /// can perform I/O operations on regions of memory.
465 ///
466 /// This trait defines which backend shall be used for I/O operations and provides a method to
467 /// convert into [`IoBackend::View`]. Users should use the [`Io`] trait which provides the actual
468 /// methods to perform I/O operations.
469 ///
470 /// This should be implemented on cheaply copyable handles, such as references or view types.
471 pub trait IoBase<'a>: Copy {
472     /// Type that defines all I/O operations.
473     type Backend: IoBackend;
474 
475     /// Type of this I/O region. For untyped regions, [`Region`] can be used.
476     type Target: ?Sized + KnownSize;
477 
478     /// Return a view that covers the full region.
479     fn as_view(self) -> <Self::Backend as IoBackend>::View<'a, Self::Target>;
480 }
481 
482 /// Extension trait to provide I/O operation methods to types that implement [`IoBase`].
483 ///
484 /// This trait provides:
485 /// - Helper methods for offset validation and address calculation
486 /// - Fallible (runtime checked) accessors for different data widths
487 ///
488 /// Which I/O methods are available depends on the associated [`IoBackend`] implementation.
489 pub trait Io<'a>: IoBase<'a> {
490     /// Returns the size of this I/O region.
491     #[inline]
492     fn size(self) -> usize {
493         KnownSize::size(Self::Backend::as_ptr(self.as_view()))
494     }
495 
496     /// Returns the length of the slice in number of elements.
497     #[inline]
498     fn len<T>(self) -> usize
499     where
500         Self: Io<'a, Target = [T]>,
501     {
502         Self::Backend::as_ptr(self.as_view()).len()
503     }
504 
505     /// Returns `true` if the slice has a length of 0.
506     #[inline]
507     fn is_empty<T>(self) -> bool
508     where
509         Self: Io<'a, Target = [T]>,
510     {
511         self.len() == 0
512     }
513 
514     /// Try to convert into a different typed I/O view.
515     ///
516     /// A runtime check is performed to ensure that the target type is of same or smaller size to
517     /// current type, and the current view is properly aligned for the target type. Returns
518     /// `Err(EINVAL)` if the runtime check fails.
519     ///
520     /// # Examples
521     ///
522     /// ```no_run
523     /// use kernel::io::{
524     ///     io_project,
525     ///     Mmio,
526     ///     Io,
527     ///     Region,
528     /// };
529     /// #[derive(FromBytes, IntoBytes)]
530     /// #[repr(C)]
531     /// struct MyStruct { field: u32, }
532     ///
533     /// # fn test(mmio: &Mmio<'_, Region>) -> Result {
534     /// // let mmio: Mmio<'_, Region>;
535     /// let whole: Mmio<'_, MyStruct> = mmio.try_cast()?;
536     /// # Ok::<(), Error>(()) }
537     /// ```
538     #[inline]
539     fn try_cast<U>(self) -> Result<<Self::Backend as IoBackend>::View<'a, U>>
540     where
541         Self::Target: FromBytes + IntoBytes,
542         U: FromBytes + IntoBytes,
543     {
544         let view = self.as_view();
545         let ptr = Self::Backend::as_ptr(view);
546 
547         if size_of::<U>() > KnownSize::size(ptr) {
548             return Err(EINVAL);
549         }
550 
551         if ptr.addr() % align_of::<U>() != 0 {
552             return Err(EINVAL);
553         }
554 
555         // SAFETY: We have checked bounds and alignment, so this is a valid projection.
556         Ok(unsafe { Self::Backend::project_view(view, ptr.cast()) })
557     }
558 
559     /// Read a value from I/O.
560     ///
561     /// This only works for primitives supported by the I/O backend.
562     ///
563     /// # Examples
564     ///
565     /// ```no_run
566     /// # use kernel::io::*;
567     /// # fn test_read_val(mmio: Mmio<'_, u32>) {
568     /// // let mmio: Mmio<'_, u32>;
569     /// let val: u32 = mmio.read_val();
570     /// # }
571     /// ```
572     #[inline]
573     fn read_val(self) -> Self::Target
574     where
575         Self::Target: AsReprMut,
576         Self::Backend: IoCapable<<Self::Target as AsRepr>::Repr>,
577     {
578         Self::Target::from_repr(Self::Backend::io_read(io_view_as_repr(self)))
579     }
580 
581     /// Write a value to I/O.
582     ///
583     /// This only works for primitives supported by the I/O backend.
584     ///
585     /// # Examples
586     ///
587     /// ```no_run
588     /// # use kernel::io::*;
589     /// # fn test_write_val(mmio: Mmio<'_, u32>) {
590     /// // let mmio: Mmio<'_, u32>;
591     /// mmio.write_val(1u32);
592     /// # }
593     /// ```
594     #[inline]
595     fn write_val(self, value: Self::Target)
596     where
597         Self::Target: AsRepr,
598         Self::Backend: IoCapable<<Self::Target as AsRepr>::Repr>,
599     {
600         Self::Backend::io_write(io_view_as_repr(self), Self::Target::into_repr(value))
601     }
602 
603     /// Copy-read from I/O memory.
604     ///
605     /// This is equivalent to reading from the I/O memory with byte-wise copy, although the actual
606     /// implementation might be more efficient. There is no atomicity guarantee. Note that for some
607     /// backends (e.g. `Mmio`), this can read different value compared to [`read_val`] as
608     /// byte-swapping is not performed.
609     ///
610     /// [`read_val`]: Io::read_val
611     ///
612     /// # Examples
613     ///
614     /// ```no_run
615     /// # use kernel::io::*;
616     /// # fn test_copy_read(mmio: Mmio<'_, [u8; 6]>) {
617     /// // let mmio: Mmio<'_, [u8; 6]>;
618     /// let val: [u8; 6] = mmio.copy_read();
619     /// # }
620     /// ```
621     #[inline]
622     fn copy_read(self) -> Self::Target
623     where
624         Self::Backend: IoCopyable,
625         Self::Target: Sized + FromBytes,
626     {
627         Self::Backend::copy_read(self.as_view())
628     }
629 
630     /// Copy-write to I/O memory.
631     ///
632     /// This is equivalent to writing to the I/O memory with byte-wise copy, although the actual
633     /// implementation might be more efficient. There is no atomicity guarantee. Note that for some
634     /// backends (e.g. `Mmio`), this can write different value compared to [`write_val`] as
635     /// byte-swapping is not performed.
636     ///
637     /// [`write_val`]: Io::write_val
638     ///
639     /// # Examples
640     ///
641     /// ```no_run
642     /// # use kernel::io::*;
643     /// # fn test_copy_write(mmio: Mmio<'_, [u8; 6]>) {
644     /// // let mmio: Mmio<'_, [u8; 6]>;
645     /// mmio.copy_write([0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]);
646     /// # }
647     /// ```
648     #[inline]
649     fn copy_write(self, value: Self::Target)
650     where
651         Self::Backend: IoCopyable,
652         Self::Target: Sized + IntoBytes,
653     {
654         Self::Backend::copy_write(self.as_view(), value);
655     }
656 
657     /// Copy bytes from `data` to I/O memory.
658     ///
659     /// # Panics
660     ///
661     /// This function will panic if the length of `self` differs from the length of `data`, similar
662     /// to [`[u8]::copy_from_slice`].
663     ///
664     /// # Examples
665     ///
666     /// ```no_run
667     /// # use kernel::io::*;
668     /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) {
669     /// // let mmio: Mmio<'_, [u8]>;
670     /// mmio.copy_from_slice(&[0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]);
671     /// # }
672     /// ```
673     #[inline]
674     fn copy_from_slice(self, data: &[u8])
675     where
676         Self::Backend: IoCopyable,
677         Self: Io<'a, Target = [u8]>,
678     {
679         assert_eq!(self.len(), data.len());
680 
681         // SAFETY: `data.as_ptr()` is valid for read for `self.size()` bytes.
682         unsafe {
683             Self::Backend::copy_to_io(self.as_view(), data.as_ptr());
684         }
685     }
686 
687     /// Copy bytes from I/O memory to `data`.
688     ///
689     /// # Panics
690     ///
691     /// This function will panic if the length of `self` differs from the length of `data`, similar
692     /// to [`[u8]::copy_from_slice`].
693     ///
694     /// # Examples
695     ///
696     /// ```no_run
697     /// # use kernel::io::*;
698     /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) {
699     /// // let mmio: Mmio<'_, [u8]>;
700     /// let mut buf = [0; 6];
701     /// mmio.copy_to_slice(&mut buf);
702     /// # }
703     /// ```
704     #[inline]
705     fn copy_to_slice(self, data: &mut [u8])
706     where
707         Self::Backend: IoCopyable,
708         Self: Io<'a, Target = [u8]>,
709     {
710         assert_eq!(self.len(), data.len());
711 
712         // SAFETY: `data.as_mut_ptr()` is valid for write for `self.size()` bytes.
713         unsafe {
714             Self::Backend::copy_from_io(self.as_view(), data.as_mut_ptr());
715         }
716     }
717 
718     /// Fallible 8-bit read with runtime bounds check.
719     #[inline(always)]
720     fn try_read8(self, offset: usize) -> Result<u8>
721     where
722         usize: IoLoc<Self::Target, u8>,
723         Self::Backend: IoCapable<u8>,
724     {
725         self.try_read(offset)
726     }
727 
728     /// Fallible 16-bit read with runtime bounds check.
729     #[inline(always)]
730     fn try_read16(self, offset: usize) -> Result<u16>
731     where
732         usize: IoLoc<Self::Target, u16>,
733         Self::Backend: IoCapable<u16>,
734     {
735         self.try_read(offset)
736     }
737 
738     /// Fallible 32-bit read with runtime bounds check.
739     #[inline(always)]
740     fn try_read32(self, offset: usize) -> Result<u32>
741     where
742         usize: IoLoc<Self::Target, u32>,
743         Self::Backend: IoCapable<u32>,
744     {
745         self.try_read(offset)
746     }
747 
748     /// Fallible 64-bit read with runtime bounds check.
749     #[inline(always)]
750     fn try_read64(self, offset: usize) -> Result<u64>
751     where
752         usize: IoLoc<Self::Target, u64>,
753         Self::Backend: IoCapable<u64>,
754     {
755         self.try_read(offset)
756     }
757 
758     /// Fallible 8-bit write with runtime bounds check.
759     #[inline(always)]
760     fn try_write8(self, value: u8, offset: usize) -> Result
761     where
762         usize: IoLoc<Self::Target, u8>,
763         Self::Backend: IoCapable<u8>,
764     {
765         self.try_write(offset, value)
766     }
767 
768     /// Fallible 16-bit write with runtime bounds check.
769     #[inline(always)]
770     fn try_write16(self, value: u16, offset: usize) -> Result
771     where
772         usize: IoLoc<Self::Target, u16>,
773         Self::Backend: IoCapable<u16>,
774     {
775         self.try_write(offset, value)
776     }
777 
778     /// Fallible 32-bit write with runtime bounds check.
779     #[inline(always)]
780     fn try_write32(self, value: u32, offset: usize) -> Result
781     where
782         usize: IoLoc<Self::Target, u32>,
783         Self::Backend: IoCapable<u32>,
784     {
785         self.try_write(offset, value)
786     }
787 
788     /// Fallible 64-bit write with runtime bounds check.
789     #[inline(always)]
790     fn try_write64(self, value: u64, offset: usize) -> Result
791     where
792         usize: IoLoc<Self::Target, u64>,
793         Self::Backend: IoCapable<u64>,
794     {
795         self.try_write(offset, value)
796     }
797 
798     /// Infallible 8-bit read with compile-time bounds check.
799     ///
800     /// `offset` should be constant.
801     #[inline(always)]
802     fn read8(self, offset: usize) -> u8
803     where
804         usize: IoLoc<Self::Target, u8>,
805         Self::Backend: IoCapable<u8>,
806     {
807         self.read(offset)
808     }
809 
810     /// Infallible 16-bit read with compile-time bounds check.
811     ///
812     /// `offset` should be constant.
813     #[inline(always)]
814     fn read16(self, offset: usize) -> u16
815     where
816         usize: IoLoc<Self::Target, u16>,
817         Self::Backend: IoCapable<u16>,
818     {
819         self.read(offset)
820     }
821 
822     /// Infallible 32-bit read with compile-time bounds check.
823     ///
824     /// `offset` should be constant.
825     #[inline(always)]
826     fn read32(self, offset: usize) -> u32
827     where
828         usize: IoLoc<Self::Target, u32>,
829         Self::Backend: IoCapable<u32>,
830     {
831         self.read(offset)
832     }
833 
834     /// Infallible 64-bit read with compile-time bounds check.
835     ///
836     /// `offset` should be constant.
837     #[inline(always)]
838     fn read64(self, offset: usize) -> u64
839     where
840         usize: IoLoc<Self::Target, u64>,
841         Self::Backend: IoCapable<u64>,
842     {
843         self.read(offset)
844     }
845 
846     /// Infallible 8-bit write with compile-time bounds check.
847     ///
848     /// `offset` should be constant.
849     #[inline(always)]
850     fn write8(self, value: u8, offset: usize)
851     where
852         usize: IoLoc<Self::Target, u8>,
853         Self::Backend: IoCapable<u8>,
854     {
855         self.write(offset, value)
856     }
857 
858     /// Infallible 16-bit write with compile-time bounds check.
859     ///
860     /// `offset` should be constant.
861     #[inline(always)]
862     fn write16(self, value: u16, offset: usize)
863     where
864         usize: IoLoc<Self::Target, u16>,
865         Self::Backend: IoCapable<u16>,
866     {
867         self.write(offset, value)
868     }
869 
870     /// Infallible 32-bit write with compile-time bounds check.
871     ///
872     /// `offset` should be constant.
873     #[inline(always)]
874     fn write32(self, value: u32, offset: usize)
875     where
876         usize: IoLoc<Self::Target, u32>,
877         Self::Backend: IoCapable<u32>,
878     {
879         self.write(offset, value)
880     }
881 
882     /// Infallible 64-bit write with compile-time bounds check.
883     ///
884     /// `offset` should be constant.
885     #[inline(always)]
886     fn write64(self, value: u64, offset: usize)
887     where
888         usize: IoLoc<Self::Target, u64>,
889         Self::Backend: IoCapable<u64>,
890     {
891         self.write(offset, value)
892     }
893 
894     /// Generic fallible read with runtime bounds check.
895     ///
896     /// # Examples
897     ///
898     /// Read a primitive type from an I/O address:
899     ///
900     /// ```no_run
901     /// use kernel::io::{
902     ///     Io,
903     ///     Mmio,
904     ///     Region,
905     /// };
906     ///
907     /// fn do_reads(io: Mmio<'_, Region>) -> Result {
908     ///     // 32-bit read from address `0x10`.
909     ///     let v: u32 = io.try_read(0x10)?;
910     ///
911     ///     // 8-bit read from address `0xfff`.
912     ///     let v: u8 = io.try_read(0xfff)?;
913     ///
914     ///     Ok(())
915     /// }
916     /// ```
917     #[inline(always)]
918     fn try_read<T, L>(self, location: L) -> Result<T>
919     where
920         T: AsReprMut,
921         L: IoLoc<Self::Target, T>,
922         Self::Backend: IoCapable<<T as AsRepr>::Repr>,
923     {
924         Ok(io_read!(self, try: location))
925     }
926 
927     /// Generic fallible write with runtime bounds check.
928     ///
929     /// # Examples
930     ///
931     /// Write a primitive type to an I/O address:
932     ///
933     /// ```no_run
934     /// use kernel::io::{
935     ///     Io,
936     ///     Mmio,
937     ///     Region,
938     /// };
939     ///
940     /// fn do_writes(io: Mmio<'_, Region>) -> Result {
941     ///     // 32-bit write of value `1` at address `0x10`.
942     ///     io.try_write(0x10, 1u32)?;
943     ///
944     ///     // 8-bit write of value `0xff` at address `0xfff`.
945     ///     io.try_write(0xfff, 0xffu8)?;
946     ///
947     ///     Ok(())
948     /// }
949     /// ```
950     #[inline(always)]
951     fn try_write<T, L>(self, location: L, value: T) -> Result
952     where
953         T: AsRepr,
954         L: IoLoc<Self::Target, T>,
955         Self::Backend: IoCapable<<T as AsRepr>::Repr>,
956     {
957         io_write!(self, try: location, value);
958         Ok(())
959     }
960 
961     /// Generic fallible write of a fully-located register value.
962     ///
963     /// # Examples
964     ///
965     /// Tuples carrying a location and a value can be used with this method:
966     ///
967     /// ```no_run
968     /// use kernel::io::{
969     ///     register,
970     ///     Io,
971     ///     Mmio,
972     ///     Region,
973     /// };
974     ///
975     /// register! {
976     ///     base: Region;
977     ///
978     ///     VERSION(u32) @ 0x100 {
979     ///         15:8 major;
980     ///         7:0  minor;
981     ///     }
982     /// }
983     ///
984     /// impl VERSION {
985     ///     fn new(major: u8, minor: u8) -> Self {
986     ///         VERSION::zeroed().with_major(major).with_minor(minor)
987     ///     }
988     /// }
989     ///
990     /// fn do_write_reg(io: Mmio<'_, Region>) -> Result {
991     ///
992     ///     io.try_write_reg(VERSION::new(1, 0))
993     /// }
994     /// ```
995     #[inline(always)]
996     fn try_write_reg<T, L, V>(self, value: V) -> Result
997     where
998         T: AsRepr,
999         L: IoLoc<Self::Target, T>,
1000         V: LocatedRegister<Self::Target, Location = L, Value = T>,
1001         Self::Backend: IoCapable<<T as AsRepr>::Repr>,
1002     {
1003         let (location, value) = value.into_io_op();
1004 
1005         self.try_write(location, value)
1006     }
1007 
1008     /// Generic fallible update with runtime bounds check.
1009     ///
1010     /// Note: this does not perform any synchronization. The caller is responsible for ensuring
1011     /// exclusive access if required.
1012     ///
1013     /// # Examples
1014     ///
1015     /// Read the u32 value at address `0x10`, increment it, and store the updated value back:
1016     ///
1017     /// ```no_run
1018     /// use kernel::io::{
1019     ///     Io,
1020     ///     Mmio,
1021     ///     Region,
1022     /// };
1023     ///
1024     /// fn do_update(io: Mmio<'_, Region<0x1000>>) -> Result {
1025     ///     io.try_update(0x10, |v: u32| {
1026     ///         v + 1
1027     ///     })
1028     /// }
1029     /// ```
1030     #[inline(always)]
1031     fn try_update<T, L, F>(self, location: L, f: F) -> Result
1032     where
1033         T: AsReprMut,
1034         L: IoLoc<Self::Target, T>,
1035         Self::Backend: IoCapable<<T as AsRepr>::Repr>,
1036         F: FnOnce(T) -> T,
1037     {
1038         let view = io_project!(self, try: location);
1039         view.write_val(f(view.read_val()));
1040         Ok(())
1041     }
1042 
1043     /// Generic infallible read with compile-time bounds check.
1044     ///
1045     /// # Examples
1046     ///
1047     /// Read a primitive type from an I/O address:
1048     ///
1049     /// ```no_run
1050     /// use kernel::io::{
1051     ///     Io,
1052     ///     Mmio,
1053     ///     Region,
1054     /// };
1055     ///
1056     /// fn do_reads(io: Mmio<'_, Region<0x1000>>) {
1057     ///     // 32-bit read from address `0x10`.
1058     ///     let v: u32 = io.read(0x10);
1059     ///
1060     ///     // 8-bit read from the top of the I/O space.
1061     ///     let v: u8 = io.read(0xfff);
1062     /// }
1063     /// ```
1064     #[inline(always)]
1065     fn read<T, L>(self, location: L) -> T
1066     where
1067         T: AsReprMut,
1068         L: IoLoc<Self::Target, T>,
1069         Self::Backend: IoCapable<<T as AsRepr>::Repr>,
1070     {
1071         io_read!(self, build: location)
1072     }
1073 
1074     /// Generic infallible write with compile-time bounds check.
1075     ///
1076     /// # Examples
1077     ///
1078     /// Write a primitive type to an I/O address:
1079     ///
1080     /// ```no_run
1081     /// use kernel::io::{
1082     ///     Io,
1083     ///     Mmio,
1084     ///     Region,
1085     /// };
1086     ///
1087     /// fn do_writes(io: Mmio<'_, Region<0x1000>>) {
1088     ///     // 32-bit write of value `1` at address `0x10`.
1089     ///     io.write(0x10, 1u32);
1090     ///
1091     ///     // 8-bit write of value `0xff` at the top of the I/O space.
1092     ///     io.write(0xfff, 0xffu8);
1093     /// }
1094     /// ```
1095     #[inline(always)]
1096     fn write<T, L>(self, location: L, value: T)
1097     where
1098         T: AsRepr,
1099         L: IoLoc<Self::Target, T>,
1100         Self::Backend: IoCapable<<T as AsRepr>::Repr>,
1101     {
1102         io_write!(self, build: location, value);
1103     }
1104 
1105     /// Generic infallible write of a fully-located register value.
1106     ///
1107     /// # Examples
1108     ///
1109     /// Tuples carrying a location and a value can be used with this method:
1110     ///
1111     /// ```no_run
1112     /// use kernel::io::{
1113     ///     register,
1114     ///     Io,
1115     ///     Mmio,
1116     ///     Region,
1117     /// };
1118     ///
1119     /// register! {
1120     ///     base: Region<0x1000>;
1121     ///
1122     ///     VERSION(u32) @ 0x100 {
1123     ///         15:8 major;
1124     ///         7:0  minor;
1125     ///     }
1126     /// }
1127     ///
1128     /// impl VERSION {
1129     ///     fn new(major: u8, minor: u8) -> Self {
1130     ///         VERSION::zeroed().with_major(major).with_minor(minor)
1131     ///     }
1132     /// }
1133     ///
1134     /// fn do_write_reg(io: Mmio<'_, Region<0x1000>>) {
1135     ///     io.write_reg(VERSION::new(1, 0));
1136     /// }
1137     /// ```
1138     #[inline(always)]
1139     fn write_reg<T, L, V>(self, value: V)
1140     where
1141         T: AsRepr,
1142         L: IoLoc<Self::Target, T>,
1143         V: LocatedRegister<Self::Target, Location = L, Value = T>,
1144         Self::Backend: IoCapable<<T as AsRepr>::Repr>,
1145     {
1146         let (location, value) = value.into_io_op();
1147 
1148         self.write(location, value)
1149     }
1150 
1151     /// Generic infallible update with compile-time bounds check.
1152     ///
1153     /// Note: this does not perform any synchronization. The caller is responsible for ensuring
1154     /// exclusive access if required.
1155     ///
1156     /// # Examples
1157     ///
1158     /// Read the u32 value at address `0x10`, increment it, and store the updated value back:
1159     ///
1160     /// ```no_run
1161     /// use kernel::io::{
1162     ///     Io,
1163     ///     Mmio,
1164     ///     Region,
1165     /// };
1166     ///
1167     /// fn do_update(io: Mmio<'_, Region<0x1000>>) {
1168     ///     io.update(0x10, |v: u32| {
1169     ///         v + 1
1170     ///     })
1171     /// }
1172     /// ```
1173     #[inline(always)]
1174     fn update<T, L, F>(self, location: L, f: F)
1175     where
1176         T: AsReprMut,
1177         L: IoLoc<Self::Target, T>,
1178         Self::Backend: IoCapable<<T as AsRepr>::Repr>,
1179         F: FnOnce(T) -> T,
1180     {
1181         let view = io_project!(self, build: location);
1182         view.write_val(f(view.read_val()));
1183     }
1184 }
1185 
1186 // Blanket implementation ensures that provided methods cannot be arbitrarily overridden by
1187 // implementers, which is relied upon for correctness and soundness.
1188 impl<'a, T: IoBase<'a>> Io<'a> for T {}
1189 
1190 /// A view of memory-mapped I/O region.
1191 ///
1192 /// # Invariant
1193 ///
1194 /// `ptr` points to a valid and aligned memory-mapped I/O region for the duration lifetime `'a`.
1195 pub struct Mmio<'a, T: ?Sized> {
1196     ptr: *mut T,
1197     phantom: PhantomData<&'a ()>,
1198 }
1199 
1200 impl<T: ?Sized> Copy for Mmio<'_, T> {}
1201 impl<T: ?Sized> Clone for Mmio<'_, T> {
1202     #[inline]
1203     fn clone(&self) -> Self {
1204         *self
1205     }
1206 }
1207 
1208 impl<'a, T: ?Sized> Mmio<'a, T> {
1209     /// Create a `Mmio`, providing the accessors to the MMIO mapping.
1210     ///
1211     /// # Safety
1212     ///
1213     /// `raw` represents a valid and aligned memory-mapped I/O region while `'a` is alive.
1214     #[inline]
1215     pub unsafe fn from_raw(raw: MmioRaw<T>) -> Self {
1216         // INVARIANT: Per safety requirement.
1217         Self {
1218             ptr: raw.ptr,
1219             phantom: PhantomData,
1220         }
1221     }
1222 }
1223 
1224 // SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory.
1225 unsafe impl<T: ?Sized + Sync> Send for Mmio<'_, T> {}
1226 
1227 // SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory.
1228 unsafe impl<T: ?Sized + Sync> Sync for Mmio<'_, T> {}
1229 
1230 impl<'a, T: ?Sized + KnownSize> IoBase<'a> for Mmio<'a, T> {
1231     type Backend = MmioBackend;
1232     type Target = T;
1233 
1234     #[inline]
1235     fn as_view(self) -> Mmio<'a, T> {
1236         self
1237     }
1238 }
1239 
1240 /// I/O Backend for memory-mapped I/O.
1241 pub struct MmioBackend;
1242 
1243 impl IoBackend for MmioBackend {
1244     type View<'a, T: ?Sized + KnownSize> = Mmio<'a, T>;
1245 
1246     #[inline]
1247     fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
1248         view.ptr
1249     }
1250 
1251     #[inline]
1252     unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
1253         _view: Self::View<'a, T>,
1254         ptr: *mut U,
1255     ) -> Self::View<'a, U> {
1256         // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid
1257         // memory-mapped I/O region.
1258         Mmio {
1259             ptr,
1260             phantom: PhantomData,
1261         }
1262     }
1263 }
1264 
1265 /// Implements [`IoCapable`] on `$backend` for `$ty` using `$read_fn` and `$write_fn`.
1266 macro_rules! impl_mmio_io_capable {
1267     ($backend: ident, $ty:ty, $read_fn:ident, $write_fn:ident) => {
1268         impl IoCapable<$ty> for $backend {
1269             #[inline]
1270             fn io_read(view: <$backend as IoBackend>::View<'_, $ty>) -> $ty {
1271                 // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both
1272                 // `MmioBackend` and `RelaxedMmioBackend`.
1273                 unsafe { bindings::$read_fn($backend::as_ptr(view).cast_const().cast()) }
1274             }
1275 
1276             #[inline]
1277             fn io_write(view: <$backend as IoBackend>::View<'_, $ty>, value: $ty) {
1278                 // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both
1279                 // `MmioBackend` and `RelaxedMmioBackend`.
1280                 unsafe { bindings::$write_fn(value, $backend::as_ptr(view).cast()) }
1281             }
1282         }
1283     };
1284 }
1285 
1286 // MMIO regions support 8, 16, and 32-bit accesses.
1287 impl_mmio_io_capable!(MmioBackend, u8, readb, writeb);
1288 impl_mmio_io_capable!(MmioBackend, u16, readw, writew);
1289 impl_mmio_io_capable!(MmioBackend, u32, readl, writel);
1290 // MMIO regions on 64-bit systems also support 64-bit accesses.
1291 #[cfg(CONFIG_64BIT)]
1292 impl_mmio_io_capable!(MmioBackend, u64, readq, writeq);
1293 
1294 impl IoCopyable for MmioBackend {
1295     #[inline]
1296     unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) {
1297         // SAFETY:
1298         // - `view.ptr` is valid MMIO memory for `view.size()` bytes.
1299         // - `buffer` is valid for write for `view.size()` bytes.
1300         unsafe {
1301             bindings::memcpy_fromio(buffer.cast(), view.ptr.cast(), view.size());
1302         }
1303     }
1304 
1305     #[inline]
1306     unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) {
1307         // SAFETY:
1308         // - `view.ptr` is valid MMIO memory for `view.size()` bytes.
1309         // - `buffer` is valid for read for `view.size()` bytes.
1310         unsafe {
1311             bindings::memcpy_toio(view.ptr.cast(), buffer.cast(), view.size());
1312         }
1313     }
1314 }
1315 
1316 /// [`Mmio`] but using relaxed accessors.
1317 ///
1318 /// This type provides an implementation of [`Io`] that uses relaxed I/O MMIO operands instead of
1319 /// the regular ones.
1320 ///
1321 /// See [`Mmio::relaxed`] for a usage example.
1322 pub struct RelaxedMmio<'a, T: ?Sized>(Mmio<'a, T>);
1323 
1324 impl<T: ?Sized> Copy for RelaxedMmio<'_, T> {}
1325 impl<T: ?Sized> Clone for RelaxedMmio<'_, T> {
1326     #[inline]
1327     fn clone(&self) -> Self {
1328         *self
1329     }
1330 }
1331 
1332 /// I/O Backend for memory-mapped I/O, with relaxed access semantics.
1333 pub struct RelaxedMmioBackend;
1334 
1335 impl IoBackend for RelaxedMmioBackend {
1336     type View<'a, T: ?Sized + KnownSize> = RelaxedMmio<'a, T>;
1337 
1338     #[inline]
1339     fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
1340         MmioBackend::as_ptr(view.0)
1341     }
1342 
1343     #[inline]
1344     unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
1345         view: Self::View<'a, T>,
1346         ptr: *mut U,
1347     ) -> Self::View<'a, U> {
1348         // SAFETY: Per safety requirement.
1349         RelaxedMmio(unsafe { MmioBackend::project_view(view.0, ptr) })
1350     }
1351 }
1352 
1353 impl<'a, T: ?Sized + KnownSize> IoBase<'a> for RelaxedMmio<'a, T> {
1354     type Backend = RelaxedMmioBackend;
1355     type Target = T;
1356 
1357     #[inline]
1358     fn as_view(self) -> RelaxedMmio<'a, T> {
1359         self
1360     }
1361 }
1362 
1363 impl<'a, T: ?Sized> Mmio<'a, T> {
1364     /// Returns a [`RelaxedMmio`] that performs relaxed I/O operations.
1365     ///
1366     /// Relaxed accessors do not provide ordering guarantees with respect to DMA or memory accesses
1367     /// and can be used when such ordering is not required.
1368     ///
1369     /// # Examples
1370     ///
1371     /// ```no_run
1372     /// use kernel::io::{
1373     ///     Io,
1374     ///     Mmio,
1375     ///     Region,
1376     ///     RelaxedMmio,
1377     /// };
1378     ///
1379     /// fn do_io(io: Mmio<'_, Region<0x100>>) {
1380     ///     // The access is performed using `readl_relaxed` instead of `readl`.
1381     ///     let v = io.relaxed().read32(0x10);
1382     /// }
1383     ///
1384     /// ```
1385     #[inline]
1386     pub fn relaxed(self) -> RelaxedMmio<'a, T> {
1387         RelaxedMmio(self)
1388     }
1389 }
1390 
1391 // MMIO regions support 8, 16, and 32-bit accesses.
1392 impl_mmio_io_capable!(RelaxedMmioBackend, u8, readb_relaxed, writeb_relaxed);
1393 impl_mmio_io_capable!(RelaxedMmioBackend, u16, readw_relaxed, writew_relaxed);
1394 impl_mmio_io_capable!(RelaxedMmioBackend, u32, readl_relaxed, writel_relaxed);
1395 // MMIO regions on 64-bit systems also support 64-bit accesses.
1396 #[cfg(CONFIG_64BIT)]
1397 impl_mmio_io_capable!(RelaxedMmioBackend, u64, readq_relaxed, writeq_relaxed);
1398 
1399 /// I/O Backend for system memory.
1400 pub struct SysMemBackend;
1401 
1402 impl IoBackend for SysMemBackend {
1403     type View<'a, T: ?Sized + KnownSize> = SysMem<'a, T>;
1404 
1405     #[inline]
1406     fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
1407         view.ptr
1408     }
1409 
1410     #[inline]
1411     unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
1412         _view: Self::View<'a, T>,
1413         ptr: *mut U,
1414     ) -> Self::View<'a, U> {
1415         // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid
1416         // kernel accessible memory region.
1417         SysMem {
1418             ptr,
1419             phantom: PhantomData,
1420         }
1421     }
1422 }
1423 
1424 /// Implements [`IoCapable`] on `SysMemBackend` for `$ty` using `read_volatile` and
1425 /// `write_volatile`.
1426 macro_rules! impl_sysmem_io_capable {
1427     ($ty:ty) => {
1428         impl IoCapable<$ty> for SysMemBackend {
1429             #[inline]
1430             fn io_read(view: SysMem<'_, $ty>) -> $ty {
1431                 // SAFETY:
1432                 // - Per type invariant, `ptr` is valid and aligned.
1433                 // - Using read_volatile() here so that race with hardware is well-defined.
1434                 // - Using read_volatile() here is not sound if it races with other CPU per Rust
1435                 //   rules, but this is allowed per LKMM.
1436                 // - The macro is only used on primitives so all bit patterns are valid.
1437                 unsafe { view.ptr.read_volatile() }
1438             }
1439 
1440             #[inline]
1441             fn io_write(view: SysMem<'_, $ty>, value: $ty) {
1442                 // SAFETY:
1443                 // - Per type invariant, `ptr` is valid and aligned.
1444                 // - Using write_volatile() here so that race with hardware is well-defined.
1445                 // - Using write_volatile() here is not sound if it races with other CPU per Rust
1446                 //   rules, but this is allowed per LKMM.
1447                 unsafe { view.ptr.write_volatile(value) }
1448             }
1449         }
1450     };
1451 }
1452 
1453 impl_sysmem_io_capable!(u8);
1454 impl_sysmem_io_capable!(u16);
1455 impl_sysmem_io_capable!(u32);
1456 #[cfg(CONFIG_64BIT)]
1457 impl_sysmem_io_capable!(u64);
1458 
1459 impl IoCopyable for SysMemBackend {
1460     #[inline]
1461     unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) {
1462         // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile.
1463         // SAFETY:
1464         // - `view.ptr` is in CPU address space and valid for read.
1465         // - `buffer` is valid for write for `view.size()` bytes which is equal to `view.ptr.len()`.
1466         unsafe { bindings::memcpy(buffer.cast(), view.ptr.cast(), view.ptr.len()) };
1467     }
1468 
1469     #[inline]
1470     unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) {
1471         // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile.
1472         // SAFETY:
1473         // - `view.ptr` is in CPU address space and valid for write.
1474         // - `buffer` is valid for read for `view.size()` bytes which is equal to `view.ptr.len()`.
1475         unsafe { bindings::memcpy(view.ptr.cast(), buffer.cast(), view.ptr.len()) };
1476     }
1477 
1478     #[inline]
1479     fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T {
1480         // SAFETY:
1481         // - Per type invariant, `ptr` is valid and aligned.
1482         // - Using read_volatile() here so that race with hardware is well-defined.
1483         // - Using read_volatile() here is not sound if it races with other CPU per Rust
1484         //   rules, but this is allowed per LKMM.
1485         // - `T: FromBytes` so all bit patterns are valid.
1486         unsafe { view.ptr.read_volatile() }
1487     }
1488 
1489     #[inline]
1490     fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) {
1491         // SAFETY:
1492         // - Per type invariant, `ptr` is valid and aligned.
1493         // - Using write_volatile() here so that race with hardware is well-defined.
1494         // - Using write_volatile() here is not sound if it races with other CPU per Rust
1495         //   rules, but this is allowed per LKMM.
1496         unsafe { view.ptr.write_volatile(value) }
1497     }
1498 }
1499 
1500 /// A view of a system memory region.
1501 ///
1502 /// Provides `Io` trait implementation for kernel virtual address ranges,
1503 /// using volatile read/write to safely access shared memory that may be
1504 /// concurrently accessed by external hardware.
1505 ///
1506 /// # Invariants
1507 ///
1508 /// `self.ptr.addr() .. self.ptr.addr() + KnownSize::size(self.ptr)` is valid and aligned kernel
1509 /// accessible memory region for the lifetime `'a`.
1510 pub struct SysMem<'a, T: ?Sized> {
1511     ptr: *mut T,
1512     phantom: PhantomData<&'a ()>,
1513 }
1514 
1515 impl<T: ?Sized> Copy for SysMem<'_, T> {}
1516 impl<T: ?Sized> Clone for SysMem<'_, T> {
1517     #[inline]
1518     fn clone(&self) -> Self {
1519         *self
1520     }
1521 }
1522 
1523 // SAFETY: `SysMem<'_, T>` is conceptually `&T`.
1524 unsafe impl<T: ?Sized + Sync> Send for SysMem<'_, T> {}
1525 
1526 // SAFETY: `SysMem<'_, T>` is conceptually `&T`.
1527 unsafe impl<T: ?Sized + Sync> Sync for SysMem<'_, T> {}
1528 
1529 impl<'a, T: ?Sized> SysMem<'a, T> {
1530     /// Create a `SysMem` from a raw pointer.
1531     ///
1532     /// # Safety
1533     ///
1534     /// `ptr.addr() .. ptr.addr() + KnownSize::size(ptr)` must be valid and aligned kernel
1535     /// accessible memory region for the lifetime `'a`.
1536     #[inline]
1537     pub unsafe fn new(ptr: *mut T) -> Self {
1538         // INVARIANT: Per safety requirement.
1539         Self {
1540             ptr,
1541             phantom: PhantomData,
1542         }
1543     }
1544 
1545     /// Obtain the raw pointer to the memory.
1546     #[inline]
1547     pub fn as_ptr(self) -> *mut T {
1548         self.ptr
1549     }
1550 }
1551 
1552 impl<'a, T: ?Sized + KnownSize> IoBase<'a> for SysMem<'a, T> {
1553     type Backend = SysMemBackend;
1554     type Target = T;
1555 
1556     #[inline]
1557     fn as_view(self) -> <Self::Backend as IoBackend>::View<'a, Self::Target> {
1558         self
1559     }
1560 }
1561 
1562 /// I/O Backend for [`IoSysMap`].
1563 pub struct IoSysMapBackend;
1564 
1565 /// Either [`Mmio`] or [`SysMem`].
1566 ///
1567 /// This can be used when a piece of logic may wish to handle both MMIO or system memory but does
1568 /// not want or cannot be generic over I/O backends. This serves a similar purpose to
1569 /// [`include/linux/iosys-map.h`] in C.
1570 ///
1571 /// This type can be used like any other types that implements [`Io`]; this also include
1572 /// [`io_project!`], [`io_read!`], [`io_write!`].
1573 ///
1574 /// [`include/linux/iosys-map.h`]: srctree/include/linux/iosys-map.h
1575 pub enum IoSysMap<'a, T: ?Sized> {
1576     /// The view is I/O memory.
1577     Io(Mmio<'a, T>),
1578     /// The view is system memory.
1579     Sys(SysMem<'a, T>),
1580 }
1581 
1582 impl<T: ?Sized> Copy for IoSysMap<'_, T> {}
1583 impl<T: ?Sized> Clone for IoSysMap<'_, T> {
1584     #[inline]
1585     fn clone(&self) -> Self {
1586         *self
1587     }
1588 }
1589 
1590 impl<'a, T: ?Sized> From<Mmio<'a, T>> for IoSysMap<'a, T> {
1591     #[inline]
1592     fn from(value: Mmio<'a, T>) -> Self {
1593         IoSysMap::Io(value)
1594     }
1595 }
1596 
1597 impl<'a, T: ?Sized> From<SysMem<'a, T>> for IoSysMap<'a, T> {
1598     #[inline]
1599     fn from(value: SysMem<'a, T>) -> Self {
1600         IoSysMap::Sys(value)
1601     }
1602 }
1603 
1604 impl IoBackend for IoSysMapBackend {
1605     type View<'a, T: ?Sized + KnownSize> = IoSysMap<'a, T>;
1606 
1607     #[inline]
1608     fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
1609         match view {
1610             IoSysMap::Io(l) => MmioBackend::as_ptr(l),
1611             IoSysMap::Sys(r) => SysMemBackend::as_ptr(r),
1612         }
1613     }
1614 
1615     #[inline]
1616     unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
1617         view: Self::View<'a, T>,
1618         ptr: *mut U,
1619     ) -> Self::View<'a, U> {
1620         match view {
1621             // SAFETY: Per safety requirement.
1622             IoSysMap::Io(l) => IoSysMap::Io(unsafe { MmioBackend::project_view(l, ptr) }),
1623             // SAFETY: Per safety requirement.
1624             IoSysMap::Sys(r) => IoSysMap::Sys(unsafe { SysMemBackend::project_view(r, ptr) }),
1625         }
1626     }
1627 }
1628 
1629 impl<T> IoCapable<T> for IoSysMapBackend
1630 where
1631     MmioBackend: IoCapable<T>,
1632     SysMemBackend: IoCapable<T>,
1633 {
1634     #[inline]
1635     fn io_read(view: Self::View<'_, T>) -> T {
1636         match view {
1637             IoSysMap::Io(l) => MmioBackend::io_read(l),
1638             IoSysMap::Sys(r) => SysMemBackend::io_read(r),
1639         }
1640     }
1641 
1642     #[inline]
1643     fn io_write<'a>(view: Self::View<'a, T>, value: T) {
1644         match view {
1645             IoSysMap::Io(l) => MmioBackend::io_write(l, value),
1646             IoSysMap::Sys(r) => SysMemBackend::io_write(r, value),
1647         }
1648     }
1649 }
1650 
1651 impl IoCopyable for IoSysMapBackend {
1652     #[inline]
1653     unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) {
1654         match view {
1655             // SAFETY: Per safety requirement.
1656             IoSysMap::Io(l) => unsafe { MmioBackend::copy_from_io(l, buffer) },
1657             // SAFETY: Per safety requirement.
1658             IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_from_io(r, buffer) },
1659         }
1660     }
1661 
1662     #[inline]
1663     unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) {
1664         match view {
1665             // SAFETY: Per safety requirement.
1666             IoSysMap::Io(l) => unsafe { MmioBackend::copy_to_io(l, buffer) },
1667             // SAFETY: Per safety requirement.
1668             IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_to_io(r, buffer) },
1669         }
1670     }
1671 
1672     #[inline]
1673     fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T {
1674         match view {
1675             IoSysMap::Io(l) => MmioBackend::copy_read(l),
1676             IoSysMap::Sys(r) => SysMemBackend::copy_read(r),
1677         }
1678     }
1679 
1680     #[inline]
1681     fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) {
1682         match view {
1683             IoSysMap::Io(l) => MmioBackend::copy_write(l, value),
1684             IoSysMap::Sys(r) => SysMemBackend::copy_write(r, value),
1685         }
1686     }
1687 }
1688 
1689 impl<'a, T: ?Sized + KnownSize> IoBase<'a> for IoSysMap<'a, T> {
1690     type Backend = IoSysMapBackend;
1691     type Target = T;
1692 
1693     #[inline]
1694     fn as_view(self) -> IoSysMap<'a, T> {
1695         self
1696     }
1697 }
1698 
1699 // This helper turns associated functions to methods so it can be invoked in macro.
1700 // Used by `io_project!()` only.
1701 #[doc(hidden)]
1702 #[derive(Clone, Copy)]
1703 pub struct ProjectHelper<T>(pub T);
1704 
1705 impl<'a, T> ProjectHelper<T>
1706 where
1707     T: Io<'a, Backend: IoBackend<View<'a, T::Target> = T>>,
1708 {
1709     // These helper methods must not have symbols present in the binary to avoid confusion.
1710     #[inline(always)]
1711     pub fn as_ptr(self) -> *mut T::Target {
1712         T::Backend::as_ptr(self.0)
1713     }
1714 
1715     /// # Safety
1716     ///
1717     /// Same as `IoBackend::project_view`
1718     #[inline(always)]
1719     pub unsafe fn project_view<U: ?Sized + KnownSize>(
1720         self,
1721         ptr: *mut U,
1722     ) -> <T::Backend as IoBackend>::View<'a, U> {
1723         // SAFETY: Per safety requirement.
1724         unsafe { T::Backend::project_view::<T::Target, _>(self.0, ptr) }
1725     }
1726 
1727     #[inline(always)]
1728     pub fn try_project_loc<U, L>(
1729         self,
1730         location: L,
1731     ) -> Result<<T::Backend as IoBackend>::View<'a, U>>
1732     where
1733         L: IoLoc<T::Target, U>,
1734     {
1735         io_view::<_, U>(self.0, location.offset())
1736     }
1737 
1738     #[inline(always)]
1739     pub fn project_loc<U, L>(self, location: L) -> <T::Backend as IoBackend>::View<'a, U>
1740     where
1741         L: IoLoc<T::Target, U>,
1742     {
1743         io_view_assert::<_, U>(self.0, location.offset())
1744     }
1745 }
1746 
1747 /// Project an I/O type to a subview of it.
1748 ///
1749 /// The syntax is of form `io_project!(io, proj)` where `io` is an expression to a type that
1750 /// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!).
1751 ///
1752 /// `io_project!` can also project to a subview of registers defined with [`register!`] macro.
1753 /// Register projection has syntax `io_project!(io, try: REGISTER)` for fallible projection and
1754 /// `io_project!(io, build: REGISTER)` for infallible projection.
1755 ///
1756 /// # Examples
1757 ///
1758 /// ```
1759 /// use kernel::io::{
1760 ///     io_project,
1761 ///     register,
1762 ///     Mmio,
1763 /// };
1764 /// #[repr(C)]
1765 /// struct MyStruct { field: u32, }
1766 ///
1767 /// register! {
1768 ///     base: MyStruct;
1769 ///     FIELD(u32) @ 0 {
1770 ///         31:0 val;
1771 ///     }
1772 /// }
1773 ///
1774 /// # fn test(mmio: Mmio<'_, [MyStruct]>) -> Result {
1775 /// // let mmio: Mmio<[MyStruct]>;
1776 /// let field: Mmio<'_, u32> = io_project!(mmio, [try: 1].field);
1777 /// let whole: Mmio<'_, MyStruct> = io_project!(mmio, [try: 2]);
1778 /// let nested: Mmio<'_, u32> = io_project!(whole, .field);
1779 /// let reg: Mmio<'_, FIELD> = io_project!(whole, build: FIELD);
1780 /// # Ok::<(), Error>(()) }
1781 /// ```
1782 #[macro_export]
1783 #[doc(hidden)]
1784 macro_rules! io_project {
1785     // Register projection
1786     ($io:expr, try: $ioloc:expr) => {{
1787         #[allow(unused)]
1788         use $crate::io::IoBase as _;
1789         let view = $crate::io::ProjectHelper($io.as_view());
1790         view.try_project_loc($ioloc)?
1791     }};
1792     ($io:expr, build: $ioloc:expr) => {{
1793         #[allow(unused)]
1794         use $crate::io::IoBase as _;
1795         let view = $crate::io::ProjectHelper($io.as_view());
1796         view.project_loc($ioloc)
1797     }};
1798 
1799     // Field or index projection
1800     ($io:expr, $($proj:tt)*) => {{
1801         #[allow(unused)]
1802         use $crate::io::IoBase as _;
1803         let view = $crate::io::ProjectHelper($io.as_view());
1804         let ptr = $crate::ptr::project!(
1805             mut view.as_ptr(), $($proj)*
1806         );
1807         #[allow(unused_unsafe)]
1808         // SAFETY: `ptr` is a projection.
1809         unsafe { view.project_view(ptr) }
1810     }};
1811 }
1812 #[doc(inline)]
1813 pub use crate::io_project;
1814 
1815 /// Read from I/O memory.
1816 ///
1817 /// The syntax is of form `io_read!(io, proj)` where `io` is an expression to a type that
1818 /// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!).
1819 ///
1820 /// # Examples
1821 ///
1822 /// ```
1823 /// #[repr(C)]
1824 /// struct MyStruct { field: u32, }
1825 ///
1826 /// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result {
1827 /// // let mmio: Mmio<'_, [MyStruct]>;
1828 /// let field: u32 = kernel::io::io_read!(mmio, [try: 2].field);
1829 /// # Ok::<(), Error>(()) }
1830 /// ```
1831 #[macro_export]
1832 #[doc(hidden)]
1833 macro_rules! io_read {
1834     ($io:expr, $($proj:tt)*) => {
1835         $crate::io::Io::read_val($crate::io_project!($io, $($proj)*))
1836     };
1837 }
1838 #[doc(inline)]
1839 pub use crate::io_read;
1840 
1841 /// Writes to I/O memory.
1842 ///
1843 /// The syntax is of form `io_write!(io, proj, val)` where `io` is an expression to a type that
1844 /// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!),
1845 /// and `val` is the value to be written to the projected location.
1846 ///
1847 /// # Examples
1848 ///
1849 /// ```
1850 /// #[repr(C)]
1851 /// struct MyStruct { field: u32, }
1852 ///
1853 /// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result {
1854 /// // let mmio: Mmio<'_, [MyStruct]>;
1855 /// kernel::io::io_write!(mmio, [try: 2].field, 10);
1856 /// # Ok::<(), Error>(()) }
1857 /// ```
1858 #[macro_export]
1859 #[doc(hidden)]
1860 macro_rules! io_write {
1861     (@parse [$io:expr] [$($proj:tt)*] [, $val:expr]) => {
1862         $crate::io::Io::write_val($crate::io_project!($io, $($proj)*), $val)
1863     };
1864     (@parse [$io:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => {
1865         $crate::io_write!(@parse [$io] [$($proj)* .$field] [$($rest)*])
1866     };
1867     (@parse [$io:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => {
1868         $crate::io_write!(@parse [$io] [$($proj)* [$flavor: $index]] [$($rest)*])
1869     };
1870     (@parse [$io:expr] [] [try: $ioloc:expr, $($rest:tt)*]) => {
1871         $crate::io_write!(@parse [$io] [try: $ioloc] [, $($rest)*])
1872     };
1873     (@parse [$io:expr] [] [build: $ioloc:expr, $($rest:tt)*]) => {
1874         $crate::io_write!(@parse [$io] [build: $ioloc] [, $($rest)*])
1875     };
1876     ($io:expr, $($rest:tt)*) => {
1877         $crate::io_write!(@parse [$io] [] [$($rest)*])
1878     };
1879 }
1880 #[doc(inline)]
1881 pub use crate::io_write;
1882