xref: /linux/rust/kernel/init.rs (revision a6021aa24f6417416d93318bbfa022ab229c33c8)
1 // SPDX-License-Identifier: Apache-2.0 OR MIT
2 
3 //! API to safely and fallibly initialize pinned `struct`s using in-place constructors.
4 //!
5 //! It also allows in-place initialization of big `struct`s that would otherwise produce a stack
6 //! overflow.
7 //!
8 //! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential
9 //! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move.
10 //!
11 //! # Overview
12 //!
13 //! To initialize a `struct` with an in-place constructor you will need two things:
14 //! - an in-place constructor,
15 //! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`],
16 //!   [`UniqueArc<T>`], [`Box<T>`] or any other smart pointer that implements [`InPlaceInit`]).
17 //!
18 //! To get an in-place constructor there are generally three options:
19 //! - directly creating an in-place constructor using the [`pin_init!`] macro,
20 //! - a custom function/macro returning an in-place constructor provided by someone else,
21 //! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer.
22 //!
23 //! Aside from pinned initialization, this API also supports in-place construction without pinning,
24 //! the macros/types/functions are generally named like the pinned variants without the `pin`
25 //! prefix.
26 //!
27 //! # Examples
28 //!
29 //! ## Using the [`pin_init!`] macro
30 //!
31 //! If you want to use [`PinInit`], then you will have to annotate your `struct` with
32 //! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for
33 //! [structurally pinned fields]. After doing this, you can then create an in-place constructor via
34 //! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is
35 //! that you need to write `<-` instead of `:` for fields that you want to initialize in-place.
36 //!
37 //! ```rust
38 //! # #![allow(clippy::disallowed_names)]
39 //! use kernel::sync::{new_mutex, Mutex};
40 //! # use core::pin::Pin;
41 //! #[pin_data]
42 //! struct Foo {
43 //!     #[pin]
44 //!     a: Mutex<usize>,
45 //!     b: u32,
46 //! }
47 //!
48 //! let foo = pin_init!(Foo {
49 //!     a <- new_mutex!(42, "Foo::a"),
50 //!     b: 24,
51 //! });
52 //! ```
53 //!
54 //! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like
55 //! (or just the stack) to actually initialize a `Foo`:
56 //!
57 //! ```rust
58 //! # #![allow(clippy::disallowed_names)]
59 //! # use kernel::sync::{new_mutex, Mutex};
60 //! # use core::pin::Pin;
61 //! # #[pin_data]
62 //! # struct Foo {
63 //! #     #[pin]
64 //! #     a: Mutex<usize>,
65 //! #     b: u32,
66 //! # }
67 //! # let foo = pin_init!(Foo {
68 //! #     a <- new_mutex!(42, "Foo::a"),
69 //! #     b: 24,
70 //! # });
71 //! let foo: Result<Pin<Box<Foo>>> = Box::pin_init(foo, GFP_KERNEL);
72 //! ```
73 //!
74 //! For more information see the [`pin_init!`] macro.
75 //!
76 //! ## Using a custom function/macro that returns an initializer
77 //!
78 //! Many types from the kernel supply a function/macro that returns an initializer, because the
79 //! above method only works for types where you can access the fields.
80 //!
81 //! ```rust
82 //! # use kernel::sync::{new_mutex, Arc, Mutex};
83 //! let mtx: Result<Arc<Mutex<usize>>> =
84 //!     Arc::pin_init(new_mutex!(42, "example::mtx"), GFP_KERNEL);
85 //! ```
86 //!
87 //! To declare an init macro/function you just return an [`impl PinInit<T, E>`]:
88 //!
89 //! ```rust
90 //! # #![allow(clippy::disallowed_names)]
91 //! # use kernel::{sync::Mutex, new_mutex, init::PinInit, try_pin_init};
92 //! #[pin_data]
93 //! struct DriverData {
94 //!     #[pin]
95 //!     status: Mutex<i32>,
96 //!     buffer: Box<[u8; 1_000_000]>,
97 //! }
98 //!
99 //! impl DriverData {
100 //!     fn new() -> impl PinInit<Self, Error> {
101 //!         try_pin_init!(Self {
102 //!             status <- new_mutex!(0, "DriverData::status"),
103 //!             buffer: Box::init(kernel::init::zeroed(), GFP_KERNEL)?,
104 //!         })
105 //!     }
106 //! }
107 //! ```
108 //!
109 //! ## Manual creation of an initializer
110 //!
111 //! Often when working with primitives the previous approaches are not sufficient. That is where
112 //! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a
113 //! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure
114 //! actually does the initialization in the correct way. Here are the things to look out for
115 //! (we are calling the parameter to the closure `slot`):
116 //! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so
117 //!   `slot` now contains a valid bit pattern for the type `T`,
118 //! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so
119 //!   you need to take care to clean up anything if your initialization fails mid-way,
120 //! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of
121 //!   `slot` gets called.
122 //!
123 //! ```rust
124 //! # #![allow(unreachable_pub, clippy::disallowed_names)]
125 //! use kernel::{init, types::Opaque};
126 //! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin};
127 //! # mod bindings {
128 //! #     #![allow(non_camel_case_types)]
129 //! #     pub struct foo;
130 //! #     pub unsafe fn init_foo(_ptr: *mut foo) {}
131 //! #     pub unsafe fn destroy_foo(_ptr: *mut foo) {}
132 //! #     pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 }
133 //! # }
134 //! # // `Error::from_errno` is `pub(crate)` in the `kernel` crate, thus provide a workaround.
135 //! # trait FromErrno {
136 //! #     fn from_errno(errno: core::ffi::c_int) -> Error {
137 //! #         // Dummy error that can be constructed outside the `kernel` crate.
138 //! #         Error::from(core::fmt::Error)
139 //! #     }
140 //! # }
141 //! # impl FromErrno for Error {}
142 //! /// # Invariants
143 //! ///
144 //! /// `foo` is always initialized
145 //! #[pin_data(PinnedDrop)]
146 //! pub struct RawFoo {
147 //!     #[pin]
148 //!     foo: Opaque<bindings::foo>,
149 //!     #[pin]
150 //!     _p: PhantomPinned,
151 //! }
152 //!
153 //! impl RawFoo {
154 //!     pub fn new(flags: u32) -> impl PinInit<Self, Error> {
155 //!         // SAFETY:
156 //!         // - when the closure returns `Ok(())`, then it has successfully initialized and
157 //!         //   enabled `foo`,
158 //!         // - when it returns `Err(e)`, then it has cleaned up before
159 //!         unsafe {
160 //!             init::pin_init_from_closure(move |slot: *mut Self| {
161 //!                 // `slot` contains uninit memory, avoid creating a reference.
162 //!                 let foo = addr_of_mut!((*slot).foo);
163 //!
164 //!                 // Initialize the `foo`
165 //!                 bindings::init_foo(Opaque::raw_get(foo));
166 //!
167 //!                 // Try to enable it.
168 //!                 let err = bindings::enable_foo(Opaque::raw_get(foo), flags);
169 //!                 if err != 0 {
170 //!                     // Enabling has failed, first clean up the foo and then return the error.
171 //!                     bindings::destroy_foo(Opaque::raw_get(foo));
172 //!                     return Err(Error::from_errno(err));
173 //!                 }
174 //!
175 //!                 // All fields of `RawFoo` have been initialized, since `_p` is a ZST.
176 //!                 Ok(())
177 //!             })
178 //!         }
179 //!     }
180 //! }
181 //!
182 //! #[pinned_drop]
183 //! impl PinnedDrop for RawFoo {
184 //!     fn drop(self: Pin<&mut Self>) {
185 //!         // SAFETY: Since `foo` is initialized, destroying is safe.
186 //!         unsafe { bindings::destroy_foo(self.foo.get()) };
187 //!     }
188 //! }
189 //! ```
190 //!
191 //! For the special case where initializing a field is a single FFI-function call that cannot fail,
192 //! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single
193 //! [`Opaque`] field by just delegating to the supplied closure. You can use these in combination
194 //! with [`pin_init!`].
195 //!
196 //! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside
197 //! the `kernel` crate. The [`sync`] module is a good starting point.
198 //!
199 //! [`sync`]: kernel::sync
200 //! [pinning]: https://doc.rust-lang.org/std/pin/index.html
201 //! [structurally pinned fields]:
202 //!     https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field
203 //! [stack]: crate::stack_pin_init
204 //! [`Arc<T>`]: crate::sync::Arc
205 //! [`impl PinInit<Foo>`]: PinInit
206 //! [`impl PinInit<T, E>`]: PinInit
207 //! [`impl Init<T, E>`]: Init
208 //! [`Opaque`]: kernel::types::Opaque
209 //! [`Opaque::ffi_init`]: kernel::types::Opaque::ffi_init
210 //! [`pin_data`]: ::macros::pin_data
211 //! [`pin_init!`]: crate::pin_init!
212 
213 use crate::{
214     alloc::{box_ext::BoxExt, AllocError, Flags},
215     error::{self, Error},
216     sync::Arc,
217     sync::UniqueArc,
218     types::{Opaque, ScopeGuard},
219 };
220 use alloc::boxed::Box;
221 use core::{
222     cell::UnsafeCell,
223     convert::Infallible,
224     marker::PhantomData,
225     mem::MaybeUninit,
226     num::*,
227     pin::Pin,
228     ptr::{self, NonNull},
229 };
230 
231 #[doc(hidden)]
232 pub mod __internal;
233 #[doc(hidden)]
234 pub mod macros;
235 
236 /// Initialize and pin a type directly on the stack.
237 ///
238 /// # Examples
239 ///
240 /// ```rust
241 /// # #![allow(clippy::disallowed_names)]
242 /// # use kernel::{init, macros::pin_data, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex};
243 /// # use core::pin::Pin;
244 /// #[pin_data]
245 /// struct Foo {
246 ///     #[pin]
247 ///     a: Mutex<usize>,
248 ///     b: Bar,
249 /// }
250 ///
251 /// #[pin_data]
252 /// struct Bar {
253 ///     x: u32,
254 /// }
255 ///
256 /// stack_pin_init!(let foo = pin_init!(Foo {
257 ///     a <- new_mutex!(42),
258 ///     b: Bar {
259 ///         x: 64,
260 ///     },
261 /// }));
262 /// let foo: Pin<&mut Foo> = foo;
263 /// pr_info!("a: {}", &*foo.a.lock());
264 /// ```
265 ///
266 /// # Syntax
267 ///
268 /// A normal `let` binding with optional type annotation. The expression is expected to implement
269 /// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error
270 /// type, then use [`stack_try_pin_init!`].
271 ///
272 /// [`stack_try_pin_init!`]: crate::stack_try_pin_init!
273 #[macro_export]
274 macro_rules! stack_pin_init {
275     (let $var:ident $(: $t:ty)? = $val:expr) => {
276         let val = $val;
277         let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
278         let mut $var = match $crate::init::__internal::StackInit::init($var, val) {
279             Ok(res) => res,
280             Err(x) => {
281                 let x: ::core::convert::Infallible = x;
282                 match x {}
283             }
284         };
285     };
286 }
287 
288 /// Initialize and pin a type directly on the stack.
289 ///
290 /// # Examples
291 ///
292 /// ```rust,ignore
293 /// # #![allow(clippy::disallowed_names)]
294 /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex};
295 /// # use macros::pin_data;
296 /// # use core::{alloc::AllocError, pin::Pin};
297 /// #[pin_data]
298 /// struct Foo {
299 ///     #[pin]
300 ///     a: Mutex<usize>,
301 ///     b: Box<Bar>,
302 /// }
303 ///
304 /// struct Bar {
305 ///     x: u32,
306 /// }
307 ///
308 /// stack_try_pin_init!(let foo: Result<Pin<&mut Foo>, AllocError> = pin_init!(Foo {
309 ///     a <- new_mutex!(42),
310 ///     b: Box::new(Bar {
311 ///         x: 64,
312 ///     }, GFP_KERNEL)?,
313 /// }));
314 /// let foo = foo.unwrap();
315 /// pr_info!("a: {}", &*foo.a.lock());
316 /// ```
317 ///
318 /// ```rust,ignore
319 /// # #![allow(clippy::disallowed_names)]
320 /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex};
321 /// # use macros::pin_data;
322 /// # use core::{alloc::AllocError, pin::Pin};
323 /// #[pin_data]
324 /// struct Foo {
325 ///     #[pin]
326 ///     a: Mutex<usize>,
327 ///     b: Box<Bar>,
328 /// }
329 ///
330 /// struct Bar {
331 ///     x: u32,
332 /// }
333 ///
334 /// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo {
335 ///     a <- new_mutex!(42),
336 ///     b: Box::new(Bar {
337 ///         x: 64,
338 ///     }, GFP_KERNEL)?,
339 /// }));
340 /// pr_info!("a: {}", &*foo.a.lock());
341 /// # Ok::<_, AllocError>(())
342 /// ```
343 ///
344 /// # Syntax
345 ///
346 /// A normal `let` binding with optional type annotation. The expression is expected to implement
347 /// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the
348 /// `=` will propagate this error.
349 #[macro_export]
350 macro_rules! stack_try_pin_init {
351     (let $var:ident $(: $t:ty)? = $val:expr) => {
352         let val = $val;
353         let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
354         let mut $var = $crate::init::__internal::StackInit::init($var, val);
355     };
356     (let $var:ident $(: $t:ty)? =? $val:expr) => {
357         let val = $val;
358         let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
359         let mut $var = $crate::init::__internal::StackInit::init($var, val)?;
360     };
361 }
362 
363 /// Construct an in-place, pinned initializer for `struct`s.
364 ///
365 /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
366 /// [`try_pin_init!`].
367 ///
368 /// The syntax is almost identical to that of a normal `struct` initializer:
369 ///
370 /// ```rust
371 /// # #![allow(clippy::disallowed_names)]
372 /// # use kernel::{init, pin_init, macros::pin_data, init::*};
373 /// # use core::pin::Pin;
374 /// #[pin_data]
375 /// struct Foo {
376 ///     a: usize,
377 ///     b: Bar,
378 /// }
379 ///
380 /// #[pin_data]
381 /// struct Bar {
382 ///     x: u32,
383 /// }
384 ///
385 /// # fn demo() -> impl PinInit<Foo> {
386 /// let a = 42;
387 ///
388 /// let initializer = pin_init!(Foo {
389 ///     a,
390 ///     b: Bar {
391 ///         x: 64,
392 ///     },
393 /// });
394 /// # initializer }
395 /// # Box::pin_init(demo(), GFP_KERNEL).unwrap();
396 /// ```
397 ///
398 /// Arbitrary Rust expressions can be used to set the value of a variable.
399 ///
400 /// The fields are initialized in the order that they appear in the initializer. So it is possible
401 /// to read already initialized fields using raw pointers.
402 ///
403 /// IMPORTANT: You are not allowed to create references to fields of the struct inside of the
404 /// initializer.
405 ///
406 /// # Init-functions
407 ///
408 /// When working with this API it is often desired to let others construct your types without
409 /// giving access to all fields. This is where you would normally write a plain function `new`
410 /// that would return a new instance of your type. With this API that is also possible.
411 /// However, there are a few extra things to keep in mind.
412 ///
413 /// To create an initializer function, simply declare it like this:
414 ///
415 /// ```rust
416 /// # #![allow(clippy::disallowed_names)]
417 /// # use kernel::{init, pin_init, init::*};
418 /// # use core::pin::Pin;
419 /// # #[pin_data]
420 /// # struct Foo {
421 /// #     a: usize,
422 /// #     b: Bar,
423 /// # }
424 /// # #[pin_data]
425 /// # struct Bar {
426 /// #     x: u32,
427 /// # }
428 /// impl Foo {
429 ///     fn new() -> impl PinInit<Self> {
430 ///         pin_init!(Self {
431 ///             a: 42,
432 ///             b: Bar {
433 ///                 x: 64,
434 ///             },
435 ///         })
436 ///     }
437 /// }
438 /// ```
439 ///
440 /// Users of `Foo` can now create it like this:
441 ///
442 /// ```rust
443 /// # #![allow(clippy::disallowed_names)]
444 /// # use kernel::{init, pin_init, macros::pin_data, init::*};
445 /// # use core::pin::Pin;
446 /// # #[pin_data]
447 /// # struct Foo {
448 /// #     a: usize,
449 /// #     b: Bar,
450 /// # }
451 /// # #[pin_data]
452 /// # struct Bar {
453 /// #     x: u32,
454 /// # }
455 /// # impl Foo {
456 /// #     fn new() -> impl PinInit<Self> {
457 /// #         pin_init!(Self {
458 /// #             a: 42,
459 /// #             b: Bar {
460 /// #                 x: 64,
461 /// #             },
462 /// #         })
463 /// #     }
464 /// # }
465 /// let foo = Box::pin_init(Foo::new(), GFP_KERNEL);
466 /// ```
467 ///
468 /// They can also easily embed it into their own `struct`s:
469 ///
470 /// ```rust
471 /// # #![allow(clippy::disallowed_names)]
472 /// # use kernel::{init, pin_init, macros::pin_data, init::*};
473 /// # use core::pin::Pin;
474 /// # #[pin_data]
475 /// # struct Foo {
476 /// #     a: usize,
477 /// #     b: Bar,
478 /// # }
479 /// # #[pin_data]
480 /// # struct Bar {
481 /// #     x: u32,
482 /// # }
483 /// # impl Foo {
484 /// #     fn new() -> impl PinInit<Self> {
485 /// #         pin_init!(Self {
486 /// #             a: 42,
487 /// #             b: Bar {
488 /// #                 x: 64,
489 /// #             },
490 /// #         })
491 /// #     }
492 /// # }
493 /// #[pin_data]
494 /// struct FooContainer {
495 ///     #[pin]
496 ///     foo1: Foo,
497 ///     #[pin]
498 ///     foo2: Foo,
499 ///     other: u32,
500 /// }
501 ///
502 /// impl FooContainer {
503 ///     fn new(other: u32) -> impl PinInit<Self> {
504 ///         pin_init!(Self {
505 ///             foo1 <- Foo::new(),
506 ///             foo2 <- Foo::new(),
507 ///             other,
508 ///         })
509 ///     }
510 /// }
511 /// ```
512 ///
513 /// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`.
514 /// This signifies that the given field is initialized in-place. As with `struct` initializers, just
515 /// writing the field (in this case `other`) without `:` or `<-` means `other: other,`.
516 ///
517 /// # Syntax
518 ///
519 /// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with
520 /// the following modifications is expected:
521 /// - Fields that you want to initialize in-place have to use `<-` instead of `:`.
522 /// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`]
523 ///   pointer named `this` inside of the initializer.
524 /// - Using struct update syntax one can place `..Zeroable::zeroed()` at the very end of the
525 ///   struct, this initializes every field with 0 and then runs all initializers specified in the
526 ///   body. This can only be done if [`Zeroable`] is implemented for the struct.
527 ///
528 /// For instance:
529 ///
530 /// ```rust
531 /// # use kernel::{macros::{Zeroable, pin_data}, pin_init};
532 /// # use core::{ptr::addr_of_mut, marker::PhantomPinned};
533 /// #[pin_data]
534 /// #[derive(Zeroable)]
535 /// struct Buf {
536 ///     // `ptr` points into `buf`.
537 ///     ptr: *mut u8,
538 ///     buf: [u8; 64],
539 ///     #[pin]
540 ///     pin: PhantomPinned,
541 /// }
542 /// pin_init!(&this in Buf {
543 ///     buf: [0; 64],
544 ///     ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() },
545 ///     pin: PhantomPinned,
546 /// });
547 /// pin_init!(Buf {
548 ///     buf: [1; 64],
549 ///     ..Zeroable::zeroed()
550 /// });
551 /// ```
552 ///
553 /// [`try_pin_init!`]: kernel::try_pin_init
554 /// [`NonNull<Self>`]: core::ptr::NonNull
555 // For a detailed example of how this macro works, see the module documentation of the hidden
556 // module `__internal` inside of `init/__internal.rs`.
557 #[macro_export]
558 macro_rules! pin_init {
559     ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
560         $($fields:tt)*
561     }) => {
562         $crate::__init_internal!(
563             @this($($this)?),
564             @typ($t $(::<$($generics),*>)?),
565             @fields($($fields)*),
566             @error(::core::convert::Infallible),
567             @data(PinData, use_data),
568             @has_data(HasPinData, __pin_data),
569             @construct_closure(pin_init_from_closure),
570             @munch_fields($($fields)*),
571         )
572     };
573 }
574 
575 /// Construct an in-place, fallible pinned initializer for `struct`s.
576 ///
577 /// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`].
578 ///
579 /// You can use the `?` operator or use `return Err(err)` inside the initializer to stop
580 /// initialization and return the error.
581 ///
582 /// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when
583 /// initialization fails, the memory can be safely deallocated without any further modifications.
584 ///
585 /// This macro defaults the error to [`Error`].
586 ///
587 /// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type`
588 /// after the `struct` initializer to specify the error type you want to use.
589 ///
590 /// # Examples
591 ///
592 /// ```rust
593 /// # #![feature(new_uninit)]
594 /// use kernel::{init::{self, PinInit}, error::Error};
595 /// #[pin_data]
596 /// struct BigBuf {
597 ///     big: Box<[u8; 1024 * 1024 * 1024]>,
598 ///     small: [u8; 1024 * 1024],
599 ///     ptr: *mut u8,
600 /// }
601 ///
602 /// impl BigBuf {
603 ///     fn new() -> impl PinInit<Self, Error> {
604 ///         try_pin_init!(Self {
605 ///             big: Box::init(init::zeroed(), GFP_KERNEL)?,
606 ///             small: [0; 1024 * 1024],
607 ///             ptr: core::ptr::null_mut(),
608 ///         }? Error)
609 ///     }
610 /// }
611 /// ```
612 // For a detailed example of how this macro works, see the module documentation of the hidden
613 // module `__internal` inside of `init/__internal.rs`.
614 #[macro_export]
615 macro_rules! try_pin_init {
616     ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
617         $($fields:tt)*
618     }) => {
619         $crate::__init_internal!(
620             @this($($this)?),
621             @typ($t $(::<$($generics),*>)? ),
622             @fields($($fields)*),
623             @error($crate::error::Error),
624             @data(PinData, use_data),
625             @has_data(HasPinData, __pin_data),
626             @construct_closure(pin_init_from_closure),
627             @munch_fields($($fields)*),
628         )
629     };
630     ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
631         $($fields:tt)*
632     }? $err:ty) => {
633         $crate::__init_internal!(
634             @this($($this)?),
635             @typ($t $(::<$($generics),*>)? ),
636             @fields($($fields)*),
637             @error($err),
638             @data(PinData, use_data),
639             @has_data(HasPinData, __pin_data),
640             @construct_closure(pin_init_from_closure),
641             @munch_fields($($fields)*),
642         )
643     };
644 }
645 
646 /// Construct an in-place initializer for `struct`s.
647 ///
648 /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
649 /// [`try_init!`].
650 ///
651 /// The syntax is identical to [`pin_init!`] and its safety caveats also apply:
652 /// - `unsafe` code must guarantee either full initialization or return an error and allow
653 ///   deallocation of the memory.
654 /// - the fields are initialized in the order given in the initializer.
655 /// - no references to fields are allowed to be created inside of the initializer.
656 ///
657 /// This initializer is for initializing data in-place that might later be moved. If you want to
658 /// pin-initialize, use [`pin_init!`].
659 ///
660 /// [`try_init!`]: crate::try_init!
661 // For a detailed example of how this macro works, see the module documentation of the hidden
662 // module `__internal` inside of `init/__internal.rs`.
663 #[macro_export]
664 macro_rules! init {
665     ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
666         $($fields:tt)*
667     }) => {
668         $crate::__init_internal!(
669             @this($($this)?),
670             @typ($t $(::<$($generics),*>)?),
671             @fields($($fields)*),
672             @error(::core::convert::Infallible),
673             @data(InitData, /*no use_data*/),
674             @has_data(HasInitData, __init_data),
675             @construct_closure(init_from_closure),
676             @munch_fields($($fields)*),
677         )
678     }
679 }
680 
681 /// Construct an in-place fallible initializer for `struct`s.
682 ///
683 /// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use
684 /// [`init!`].
685 ///
686 /// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error,
687 /// append `? $type` after the `struct` initializer.
688 /// The safety caveats from [`try_pin_init!`] also apply:
689 /// - `unsafe` code must guarantee either full initialization or return an error and allow
690 ///   deallocation of the memory.
691 /// - the fields are initialized in the order given in the initializer.
692 /// - no references to fields are allowed to be created inside of the initializer.
693 ///
694 /// # Examples
695 ///
696 /// ```rust
697 /// use kernel::{init::{PinInit, zeroed}, error::Error};
698 /// struct BigBuf {
699 ///     big: Box<[u8; 1024 * 1024 * 1024]>,
700 ///     small: [u8; 1024 * 1024],
701 /// }
702 ///
703 /// impl BigBuf {
704 ///     fn new() -> impl Init<Self, Error> {
705 ///         try_init!(Self {
706 ///             big: Box::init(zeroed(), GFP_KERNEL)?,
707 ///             small: [0; 1024 * 1024],
708 ///         }? Error)
709 ///     }
710 /// }
711 /// ```
712 // For a detailed example of how this macro works, see the module documentation of the hidden
713 // module `__internal` inside of `init/__internal.rs`.
714 #[macro_export]
715 macro_rules! try_init {
716     ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
717         $($fields:tt)*
718     }) => {
719         $crate::__init_internal!(
720             @this($($this)?),
721             @typ($t $(::<$($generics),*>)?),
722             @fields($($fields)*),
723             @error($crate::error::Error),
724             @data(InitData, /*no use_data*/),
725             @has_data(HasInitData, __init_data),
726             @construct_closure(init_from_closure),
727             @munch_fields($($fields)*),
728         )
729     };
730     ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
731         $($fields:tt)*
732     }? $err:ty) => {
733         $crate::__init_internal!(
734             @this($($this)?),
735             @typ($t $(::<$($generics),*>)?),
736             @fields($($fields)*),
737             @error($err),
738             @data(InitData, /*no use_data*/),
739             @has_data(HasInitData, __init_data),
740             @construct_closure(init_from_closure),
741             @munch_fields($($fields)*),
742         )
743     };
744 }
745 
746 /// Asserts that a field on a struct using `#[pin_data]` is marked with `#[pin]` ie. that it is
747 /// structurally pinned.
748 ///
749 /// # Example
750 ///
751 /// This will succeed:
752 /// ```
753 /// use kernel::assert_pinned;
754 /// #[pin_data]
755 /// struct MyStruct {
756 ///     #[pin]
757 ///     some_field: u64,
758 /// }
759 ///
760 /// assert_pinned!(MyStruct, some_field, u64);
761 /// ```
762 ///
763 /// This will fail:
764 // TODO: replace with `compile_fail` when supported.
765 /// ```ignore
766 /// use kernel::assert_pinned;
767 /// #[pin_data]
768 /// struct MyStruct {
769 ///     some_field: u64,
770 /// }
771 ///
772 /// assert_pinned!(MyStruct, some_field, u64);
773 /// ```
774 ///
775 /// Some uses of the macro may trigger the `can't use generic parameters from outer item` error. To
776 /// work around this, you may pass the `inline` parameter to the macro. The `inline` parameter can
777 /// only be used when the macro is invoked from a function body.
778 /// ```
779 /// use kernel::assert_pinned;
780 /// #[pin_data]
781 /// struct Foo<T> {
782 ///     #[pin]
783 ///     elem: T,
784 /// }
785 ///
786 /// impl<T> Foo<T> {
787 ///     fn project(self: Pin<&mut Self>) -> Pin<&mut T> {
788 ///         assert_pinned!(Foo<T>, elem, T, inline);
789 ///
790 ///         // SAFETY: The field is structurally pinned.
791 ///         unsafe { self.map_unchecked_mut(|me| &mut me.elem) }
792 ///     }
793 /// }
794 /// ```
795 #[macro_export]
796 macro_rules! assert_pinned {
797     ($ty:ty, $field:ident, $field_ty:ty, inline) => {
798         let _ = move |ptr: *mut $field_ty| {
799             // SAFETY: This code is unreachable.
800             let data = unsafe { <$ty as $crate::init::__internal::HasPinData>::__pin_data() };
801             let init = $crate::init::__internal::AlwaysFail::<$field_ty>::new();
802             // SAFETY: This code is unreachable.
803             unsafe { data.$field(ptr, init) }.ok();
804         };
805     };
806 
807     ($ty:ty, $field:ident, $field_ty:ty) => {
808         const _: () = {
809             $crate::assert_pinned!($ty, $field, $field_ty, inline);
810         };
811     };
812 }
813 
814 /// A pin-initializer for the type `T`.
815 ///
816 /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
817 /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the
818 /// [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc<T>`] on this.
819 ///
820 /// Also see the [module description](self).
821 ///
822 /// # Safety
823 ///
824 /// When implementing this trait you will need to take great care. Also there are probably very few
825 /// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible.
826 ///
827 /// The [`PinInit::__pinned_init`] function:
828 /// - returns `Ok(())` if it initialized every field of `slot`,
829 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
830 ///     - `slot` can be deallocated without UB occurring,
831 ///     - `slot` does not need to be dropped,
832 ///     - `slot` is not partially initialized.
833 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
834 ///
835 /// [`Arc<T>`]: crate::sync::Arc
836 /// [`Arc::pin_init`]: crate::sync::Arc::pin_init
837 #[must_use = "An initializer must be used in order to create its value."]
838 pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
839     /// Initializes `slot`.
840     ///
841     /// # Safety
842     ///
843     /// - `slot` is a valid pointer to uninitialized memory.
844     /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
845     ///   deallocate.
846     /// - `slot` will not move until it is dropped, i.e. it will be pinned.
847     unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>;
848 
849     /// First initializes the value using `self` then calls the function `f` with the initialized
850     /// value.
851     ///
852     /// If `f` returns an error the value is dropped and the initializer will forward the error.
853     ///
854     /// # Examples
855     ///
856     /// ```rust
857     /// # #![allow(clippy::disallowed_names)]
858     /// use kernel::{types::Opaque, init::pin_init_from_closure};
859     /// #[repr(C)]
860     /// struct RawFoo([u8; 16]);
861     /// extern {
862     ///     fn init_foo(_: *mut RawFoo);
863     /// }
864     ///
865     /// #[pin_data]
866     /// struct Foo {
867     ///     #[pin]
868     ///     raw: Opaque<RawFoo>,
869     /// }
870     ///
871     /// impl Foo {
872     ///     fn setup(self: Pin<&mut Self>) {
873     ///         pr_info!("Setting up foo");
874     ///     }
875     /// }
876     ///
877     /// let foo = pin_init!(Foo {
878     ///     raw <- unsafe {
879     ///         Opaque::ffi_init(|s| {
880     ///             init_foo(s);
881     ///         })
882     ///     },
883     /// }).pin_chain(|foo| {
884     ///     foo.setup();
885     ///     Ok(())
886     /// });
887     /// ```
888     fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E>
889     where
890         F: FnOnce(Pin<&mut T>) -> Result<(), E>,
891     {
892         ChainPinInit(self, f, PhantomData)
893     }
894 }
895 
896 /// An initializer returned by [`PinInit::pin_chain`].
897 pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, Box<T>)>);
898 
899 // SAFETY: The `__pinned_init` function is implemented such that it
900 // - returns `Ok(())` on successful initialization,
901 // - returns `Err(err)` on error and in this case `slot` will be dropped.
902 // - considers `slot` pinned.
903 unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainPinInit<I, F, T, E>
904 where
905     I: PinInit<T, E>,
906     F: FnOnce(Pin<&mut T>) -> Result<(), E>,
907 {
908     unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
909         // SAFETY: All requirements fulfilled since this function is `__pinned_init`.
910         unsafe { self.0.__pinned_init(slot)? };
911         // SAFETY: The above call initialized `slot` and we still have unique access.
912         let val = unsafe { &mut *slot };
913         // SAFETY: `slot` is considered pinned.
914         let val = unsafe { Pin::new_unchecked(val) };
915         // SAFETY: `slot` was initialized above.
916         (self.1)(val).inspect_err(|_| unsafe { core::ptr::drop_in_place(slot) })
917     }
918 }
919 
920 /// An initializer for `T`.
921 ///
922 /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
923 /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the
924 /// [`InPlaceInit::init`] function of a smart pointer like [`Arc<T>`] on this. Because
925 /// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well.
926 ///
927 /// Also see the [module description](self).
928 ///
929 /// # Safety
930 ///
931 /// When implementing this trait you will need to take great care. Also there are probably very few
932 /// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible.
933 ///
934 /// The [`Init::__init`] function:
935 /// - returns `Ok(())` if it initialized every field of `slot`,
936 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
937 ///     - `slot` can be deallocated without UB occurring,
938 ///     - `slot` does not need to be dropped,
939 ///     - `slot` is not partially initialized.
940 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
941 ///
942 /// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same
943 /// code as `__init`.
944 ///
945 /// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to
946 /// move the pointee after initialization.
947 ///
948 /// [`Arc<T>`]: crate::sync::Arc
949 #[must_use = "An initializer must be used in order to create its value."]
950 pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
951     /// Initializes `slot`.
952     ///
953     /// # Safety
954     ///
955     /// - `slot` is a valid pointer to uninitialized memory.
956     /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
957     ///   deallocate.
958     unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
959 
960     /// First initializes the value using `self` then calls the function `f` with the initialized
961     /// value.
962     ///
963     /// If `f` returns an error the value is dropped and the initializer will forward the error.
964     ///
965     /// # Examples
966     ///
967     /// ```rust
968     /// # #![allow(clippy::disallowed_names)]
969     /// use kernel::{types::Opaque, init::{self, init_from_closure}};
970     /// struct Foo {
971     ///     buf: [u8; 1_000_000],
972     /// }
973     ///
974     /// impl Foo {
975     ///     fn setup(&mut self) {
976     ///         pr_info!("Setting up foo");
977     ///     }
978     /// }
979     ///
980     /// let foo = init!(Foo {
981     ///     buf <- init::zeroed()
982     /// }).chain(|foo| {
983     ///     foo.setup();
984     ///     Ok(())
985     /// });
986     /// ```
987     fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E>
988     where
989         F: FnOnce(&mut T) -> Result<(), E>,
990     {
991         ChainInit(self, f, PhantomData)
992     }
993 }
994 
995 /// An initializer returned by [`Init::chain`].
996 pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, Box<T>)>);
997 
998 // SAFETY: The `__init` function is implemented such that it
999 // - returns `Ok(())` on successful initialization,
1000 // - returns `Err(err)` on error and in this case `slot` will be dropped.
1001 unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E>
1002 where
1003     I: Init<T, E>,
1004     F: FnOnce(&mut T) -> Result<(), E>,
1005 {
1006     unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
1007         // SAFETY: All requirements fulfilled since this function is `__init`.
1008         unsafe { self.0.__pinned_init(slot)? };
1009         // SAFETY: The above call initialized `slot` and we still have unique access.
1010         (self.1)(unsafe { &mut *slot }).inspect_err(|_|
1011             // SAFETY: `slot` was initialized above.
1012             unsafe { core::ptr::drop_in_place(slot) })
1013     }
1014 }
1015 
1016 // SAFETY: `__pinned_init` behaves exactly the same as `__init`.
1017 unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E>
1018 where
1019     I: Init<T, E>,
1020     F: FnOnce(&mut T) -> Result<(), E>,
1021 {
1022     unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
1023         // SAFETY: `__init` has less strict requirements compared to `__pinned_init`.
1024         unsafe { self.__init(slot) }
1025     }
1026 }
1027 
1028 /// Creates a new [`PinInit<T, E>`] from the given closure.
1029 ///
1030 /// # Safety
1031 ///
1032 /// The closure:
1033 /// - returns `Ok(())` if it initialized every field of `slot`,
1034 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1035 ///     - `slot` can be deallocated without UB occurring,
1036 ///     - `slot` does not need to be dropped,
1037 ///     - `slot` is not partially initialized.
1038 /// - may assume that the `slot` does not move if `T: !Unpin`,
1039 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1040 #[inline]
1041 pub const unsafe fn pin_init_from_closure<T: ?Sized, E>(
1042     f: impl FnOnce(*mut T) -> Result<(), E>,
1043 ) -> impl PinInit<T, E> {
1044     __internal::InitClosure(f, PhantomData)
1045 }
1046 
1047 /// Creates a new [`Init<T, E>`] from the given closure.
1048 ///
1049 /// # Safety
1050 ///
1051 /// The closure:
1052 /// - returns `Ok(())` if it initialized every field of `slot`,
1053 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1054 ///     - `slot` can be deallocated without UB occurring,
1055 ///     - `slot` does not need to be dropped,
1056 ///     - `slot` is not partially initialized.
1057 /// - the `slot` may move after initialization.
1058 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1059 #[inline]
1060 pub const unsafe fn init_from_closure<T: ?Sized, E>(
1061     f: impl FnOnce(*mut T) -> Result<(), E>,
1062 ) -> impl Init<T, E> {
1063     __internal::InitClosure(f, PhantomData)
1064 }
1065 
1066 /// An initializer that leaves the memory uninitialized.
1067 ///
1068 /// The initializer is a no-op. The `slot` memory is not changed.
1069 #[inline]
1070 pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
1071     // SAFETY: The memory is allowed to be uninitialized.
1072     unsafe { init_from_closure(|_| Ok(())) }
1073 }
1074 
1075 /// Initializes an array by initializing each element via the provided initializer.
1076 ///
1077 /// # Examples
1078 ///
1079 /// ```rust
1080 /// use kernel::{error::Error, init::init_array_from_fn};
1081 /// let array: Box<[usize; 1_000]> = Box::init::<Error>(init_array_from_fn(|i| i), GFP_KERNEL).unwrap();
1082 /// assert_eq!(array.len(), 1_000);
1083 /// ```
1084 pub fn init_array_from_fn<I, const N: usize, T, E>(
1085     mut make_init: impl FnMut(usize) -> I,
1086 ) -> impl Init<[T; N], E>
1087 where
1088     I: Init<T, E>,
1089 {
1090     let init = move |slot: *mut [T; N]| {
1091         let slot = slot.cast::<T>();
1092         // Counts the number of initialized elements and when dropped drops that many elements from
1093         // `slot`.
1094         let mut init_count = ScopeGuard::new_with_data(0, |i| {
1095             // We now free every element that has been initialized before.
1096             // SAFETY: The loop initialized exactly the values from 0..i and since we
1097             // return `Err` below, the caller will consider the memory at `slot` as
1098             // uninitialized.
1099             unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
1100         });
1101         for i in 0..N {
1102             let init = make_init(i);
1103             // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
1104             let ptr = unsafe { slot.add(i) };
1105             // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
1106             // requirements.
1107             unsafe { init.__init(ptr) }?;
1108             *init_count += 1;
1109         }
1110         init_count.dismiss();
1111         Ok(())
1112     };
1113     // SAFETY: The initializer above initializes every element of the array. On failure it drops
1114     // any initialized elements and returns `Err`.
1115     unsafe { init_from_closure(init) }
1116 }
1117 
1118 /// Initializes an array by initializing each element via the provided initializer.
1119 ///
1120 /// # Examples
1121 ///
1122 /// ```rust
1123 /// use kernel::{sync::{Arc, Mutex}, init::pin_init_array_from_fn, new_mutex};
1124 /// let array: Arc<[Mutex<usize>; 1_000]> =
1125 ///     Arc::pin_init(pin_init_array_from_fn(|i| new_mutex!(i)), GFP_KERNEL).unwrap();
1126 /// assert_eq!(array.len(), 1_000);
1127 /// ```
1128 pub fn pin_init_array_from_fn<I, const N: usize, T, E>(
1129     mut make_init: impl FnMut(usize) -> I,
1130 ) -> impl PinInit<[T; N], E>
1131 where
1132     I: PinInit<T, E>,
1133 {
1134     let init = move |slot: *mut [T; N]| {
1135         let slot = slot.cast::<T>();
1136         // Counts the number of initialized elements and when dropped drops that many elements from
1137         // `slot`.
1138         let mut init_count = ScopeGuard::new_with_data(0, |i| {
1139             // We now free every element that has been initialized before.
1140             // SAFETY: The loop initialized exactly the values from 0..i and since we
1141             // return `Err` below, the caller will consider the memory at `slot` as
1142             // uninitialized.
1143             unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
1144         });
1145         for i in 0..N {
1146             let init = make_init(i);
1147             // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
1148             let ptr = unsafe { slot.add(i) };
1149             // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
1150             // requirements.
1151             unsafe { init.__pinned_init(ptr) }?;
1152             *init_count += 1;
1153         }
1154         init_count.dismiss();
1155         Ok(())
1156     };
1157     // SAFETY: The initializer above initializes every element of the array. On failure it drops
1158     // any initialized elements and returns `Err`.
1159     unsafe { pin_init_from_closure(init) }
1160 }
1161 
1162 // SAFETY: Every type can be initialized by-value.
1163 unsafe impl<T, E> Init<T, E> for T {
1164     unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
1165         unsafe { slot.write(self) };
1166         Ok(())
1167     }
1168 }
1169 
1170 // SAFETY: Every type can be initialized by-value. `__pinned_init` calls `__init`.
1171 unsafe impl<T, E> PinInit<T, E> for T {
1172     unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
1173         unsafe { self.__init(slot) }
1174     }
1175 }
1176 
1177 /// Smart pointer that can initialize memory in-place.
1178 pub trait InPlaceInit<T>: Sized {
1179     /// Pinned version of `Self`.
1180     ///
1181     /// If a type already implicitly pins its pointee, `Pin<Self>` is unnecessary. In this case use
1182     /// `Self`, otherwise just use `Pin<Self>`.
1183     type PinnedSelf;
1184 
1185     /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
1186     /// type.
1187     ///
1188     /// If `T: !Unpin` it will not be able to move afterwards.
1189     fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
1190     where
1191         E: From<AllocError>;
1192 
1193     /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
1194     /// type.
1195     ///
1196     /// If `T: !Unpin` it will not be able to move afterwards.
1197     fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Self::PinnedSelf>
1198     where
1199         Error: From<E>,
1200     {
1201         // SAFETY: We delegate to `init` and only change the error type.
1202         let init = unsafe {
1203             pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
1204         };
1205         Self::try_pin_init(init, flags)
1206     }
1207 
1208     /// Use the given initializer to in-place initialize a `T`.
1209     fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
1210     where
1211         E: From<AllocError>;
1212 
1213     /// Use the given initializer to in-place initialize a `T`.
1214     fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self>
1215     where
1216         Error: From<E>,
1217     {
1218         // SAFETY: We delegate to `init` and only change the error type.
1219         let init = unsafe {
1220             init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
1221         };
1222         Self::try_init(init, flags)
1223     }
1224 }
1225 
1226 impl<T> InPlaceInit<T> for Arc<T> {
1227     type PinnedSelf = Self;
1228 
1229     #[inline]
1230     fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
1231     where
1232         E: From<AllocError>,
1233     {
1234         UniqueArc::try_pin_init(init, flags).map(|u| u.into())
1235     }
1236 
1237     #[inline]
1238     fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
1239     where
1240         E: From<AllocError>,
1241     {
1242         UniqueArc::try_init(init, flags).map(|u| u.into())
1243     }
1244 }
1245 
1246 impl<T> InPlaceInit<T> for Box<T> {
1247     type PinnedSelf = Pin<Self>;
1248 
1249     #[inline]
1250     fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
1251     where
1252         E: From<AllocError>,
1253     {
1254         <Box<_> as BoxExt<_>>::new_uninit(flags)?.write_pin_init(init)
1255     }
1256 
1257     #[inline]
1258     fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
1259     where
1260         E: From<AllocError>,
1261     {
1262         <Box<_> as BoxExt<_>>::new_uninit(flags)?.write_init(init)
1263     }
1264 }
1265 
1266 impl<T> InPlaceInit<T> for UniqueArc<T> {
1267     type PinnedSelf = Pin<Self>;
1268 
1269     #[inline]
1270     fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
1271     where
1272         E: From<AllocError>,
1273     {
1274         UniqueArc::new_uninit(flags)?.write_pin_init(init)
1275     }
1276 
1277     #[inline]
1278     fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
1279     where
1280         E: From<AllocError>,
1281     {
1282         UniqueArc::new_uninit(flags)?.write_init(init)
1283     }
1284 }
1285 
1286 /// Smart pointer containing uninitialized memory and that can write a value.
1287 pub trait InPlaceWrite<T> {
1288     /// The type `Self` turns into when the contents are initialized.
1289     type Initialized;
1290 
1291     /// Use the given initializer to write a value into `self`.
1292     ///
1293     /// Does not drop the current value and considers it as uninitialized memory.
1294     fn write_init<E>(self, init: impl Init<T, E>) -> Result<Self::Initialized, E>;
1295 
1296     /// Use the given pin-initializer to write a value into `self`.
1297     ///
1298     /// Does not drop the current value and considers it as uninitialized memory.
1299     fn write_pin_init<E>(self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E>;
1300 }
1301 
1302 impl<T> InPlaceWrite<T> for Box<MaybeUninit<T>> {
1303     type Initialized = Box<T>;
1304 
1305     fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
1306         let slot = self.as_mut_ptr();
1307         // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1308         // slot is valid.
1309         unsafe { init.__init(slot)? };
1310         // SAFETY: All fields have been initialized.
1311         Ok(unsafe { self.assume_init() })
1312     }
1313 
1314     fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
1315         let slot = self.as_mut_ptr();
1316         // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1317         // slot is valid and will not be moved, because we pin it later.
1318         unsafe { init.__pinned_init(slot)? };
1319         // SAFETY: All fields have been initialized.
1320         Ok(unsafe { self.assume_init() }.into())
1321     }
1322 }
1323 
1324 impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> {
1325     type Initialized = UniqueArc<T>;
1326 
1327     fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
1328         let slot = self.as_mut_ptr();
1329         // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1330         // slot is valid.
1331         unsafe { init.__init(slot)? };
1332         // SAFETY: All fields have been initialized.
1333         Ok(unsafe { self.assume_init() })
1334     }
1335 
1336     fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
1337         let slot = self.as_mut_ptr();
1338         // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1339         // slot is valid and will not be moved, because we pin it later.
1340         unsafe { init.__pinned_init(slot)? };
1341         // SAFETY: All fields have been initialized.
1342         Ok(unsafe { self.assume_init() }.into())
1343     }
1344 }
1345 
1346 /// Trait facilitating pinned destruction.
1347 ///
1348 /// Use [`pinned_drop`] to implement this trait safely:
1349 ///
1350 /// ```rust
1351 /// # use kernel::sync::Mutex;
1352 /// use kernel::macros::pinned_drop;
1353 /// use core::pin::Pin;
1354 /// #[pin_data(PinnedDrop)]
1355 /// struct Foo {
1356 ///     #[pin]
1357 ///     mtx: Mutex<usize>,
1358 /// }
1359 ///
1360 /// #[pinned_drop]
1361 /// impl PinnedDrop for Foo {
1362 ///     fn drop(self: Pin<&mut Self>) {
1363 ///         pr_info!("Foo is being dropped!");
1364 ///     }
1365 /// }
1366 /// ```
1367 ///
1368 /// # Safety
1369 ///
1370 /// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl.
1371 ///
1372 /// [`pinned_drop`]: kernel::macros::pinned_drop
1373 pub unsafe trait PinnedDrop: __internal::HasPinData {
1374     /// Executes the pinned destructor of this type.
1375     ///
1376     /// While this function is marked safe, it is actually unsafe to call it manually. For this
1377     /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code
1378     /// and thus prevents this function from being called where it should not.
1379     ///
1380     /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute
1381     /// automatically.
1382     fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop);
1383 }
1384 
1385 /// Marker trait for types that can be initialized by writing just zeroes.
1386 ///
1387 /// # Safety
1388 ///
1389 /// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words,
1390 /// this is not UB:
1391 ///
1392 /// ```rust,ignore
1393 /// let val: Self = unsafe { core::mem::zeroed() };
1394 /// ```
1395 pub unsafe trait Zeroable {}
1396 
1397 /// Create a new zeroed T.
1398 ///
1399 /// The returned initializer will write `0x00` to every byte of the given `slot`.
1400 #[inline]
1401 pub fn zeroed<T: Zeroable>() -> impl Init<T> {
1402     // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T`
1403     // and because we write all zeroes, the memory is initialized.
1404     unsafe {
1405         init_from_closure(|slot: *mut T| {
1406             slot.write_bytes(0, 1);
1407             Ok(())
1408         })
1409     }
1410 }
1411 
1412 macro_rules! impl_zeroable {
1413     ($($({$($generics:tt)*})? $t:ty, )*) => {
1414         $(unsafe impl$($($generics)*)? Zeroable for $t {})*
1415     };
1416 }
1417 
1418 impl_zeroable! {
1419     // SAFETY: All primitives that are allowed to be zero.
1420     bool,
1421     char,
1422     u8, u16, u32, u64, u128, usize,
1423     i8, i16, i32, i64, i128, isize,
1424     f32, f64,
1425 
1426     // Note: do not add uninhabited types (such as `!` or `core::convert::Infallible`) to this list;
1427     // creating an instance of an uninhabited type is immediate undefined behavior. For more on
1428     // uninhabited/empty types, consult The Rustonomicon:
1429     // <https://doc.rust-lang.org/stable/nomicon/exotic-sizes.html#empty-types>. The Rust Reference
1430     // also has information on undefined behavior:
1431     // <https://doc.rust-lang.org/stable/reference/behavior-considered-undefined.html>.
1432     //
1433     // SAFETY: These are inhabited ZSTs; there is nothing to zero and a valid value exists.
1434     {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, (),
1435 
1436     // SAFETY: Type is allowed to take any value, including all zeros.
1437     {<T>} MaybeUninit<T>,
1438     // SAFETY: Type is allowed to take any value, including all zeros.
1439     {<T>} Opaque<T>,
1440 
1441     // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`.
1442     {<T: ?Sized + Zeroable>} UnsafeCell<T>,
1443 
1444     // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
1445     Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>,
1446     Option<NonZeroU128>, Option<NonZeroUsize>,
1447     Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>,
1448     Option<NonZeroI128>, Option<NonZeroIsize>,
1449 
1450     // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
1451     //
1452     // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant.
1453     {<T: ?Sized>} Option<NonNull<T>>,
1454     {<T: ?Sized>} Option<Box<T>>,
1455 
1456     // SAFETY: `null` pointer is valid.
1457     //
1458     // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be
1459     // null.
1460     //
1461     // When `Pointee` gets stabilized, we could use
1462     // `T: ?Sized where <T as Pointee>::Metadata: Zeroable`
1463     {<T>} *mut T, {<T>} *const T,
1464 
1465     // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be
1466     // zero.
1467     {<T>} *mut [T], {<T>} *const [T], *mut str, *const str,
1468 
1469     // SAFETY: `T` is `Zeroable`.
1470     {<const N: usize, T: Zeroable>} [T; N], {<T: Zeroable>} Wrapping<T>,
1471 }
1472 
1473 macro_rules! impl_tuple_zeroable {
1474     ($(,)?) => {};
1475     ($first:ident, $($t:ident),* $(,)?) => {
1476         // SAFETY: All elements are zeroable and padding can be zero.
1477         unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {}
1478         impl_tuple_zeroable!($($t),* ,);
1479     }
1480 }
1481 
1482 impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J);
1483