1 // SPDX-License-Identifier: Apache-2.0 OR MIT 2 3 //! This module provides the macros that actually implement the proc-macros `pin_data` and 4 //! `pinned_drop`. It also contains `__init_internal` the implementation of the `{try_}{pin_}init!` 5 //! macros. 6 //! 7 //! These macros should never be called directly, since they expect their input to be 8 //! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in 9 //! safe code! Use the public facing macros instead. 10 //! 11 //! This architecture has been chosen because the kernel does not yet have access to `syn` which 12 //! would make matters a lot easier for implementing these as proc-macros. 13 //! 14 //! # Macro expansion example 15 //! 16 //! This section is intended for readers trying to understand the macros in this module and the 17 //! `pin_init!` macros from `init.rs`. 18 //! 19 //! We will look at the following example: 20 //! 21 //! ```rust,ignore 22 //! # use kernel::init::*; 23 //! # use core::pin::Pin; 24 //! #[pin_data] 25 //! #[repr(C)] 26 //! struct Bar<T> { 27 //! #[pin] 28 //! t: T, 29 //! pub x: usize, 30 //! } 31 //! 32 //! impl<T> Bar<T> { 33 //! fn new(t: T) -> impl PinInit<Self> { 34 //! pin_init!(Self { t, x: 0 }) 35 //! } 36 //! } 37 //! 38 //! #[pin_data(PinnedDrop)] 39 //! struct Foo { 40 //! a: usize, 41 //! #[pin] 42 //! b: Bar<u32>, 43 //! } 44 //! 45 //! #[pinned_drop] 46 //! impl PinnedDrop for Foo { 47 //! fn drop(self: Pin<&mut Self>) { 48 //! pr_info!("{self:p} is getting dropped."); 49 //! } 50 //! } 51 //! 52 //! let a = 42; 53 //! let initializer = pin_init!(Foo { 54 //! a, 55 //! b <- Bar::new(36), 56 //! }); 57 //! ``` 58 //! 59 //! This example includes the most common and important features of the pin-init API. 60 //! 61 //! Below you can find individual section about the different macro invocations. Here are some 62 //! general things we need to take into account when designing macros: 63 //! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()` 64 //! this ensures that the correct item is used, since users could define their own `mod core {}` 65 //! and then their own `panic!` inside to execute arbitrary code inside of our macro. 66 //! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied 67 //! expressions inside of an `unsafe` block in the macro, because this would allow users to do 68 //! `unsafe` operations without an associated `unsafe` block. 69 //! 70 //! ## `#[pin_data]` on `Bar` 71 //! 72 //! This macro is used to specify which fields are structurally pinned and which fields are not. It 73 //! is placed on the struct definition and allows `#[pin]` to be placed on the fields. 74 //! 75 //! Here is the definition of `Bar` from our example: 76 //! 77 //! ```rust,ignore 78 //! # use kernel::init::*; 79 //! #[pin_data] 80 //! #[repr(C)] 81 //! struct Bar<T> { 82 //! #[pin] 83 //! t: T, 84 //! pub x: usize, 85 //! } 86 //! ``` 87 //! 88 //! This expands to the following code: 89 //! 90 //! ```rust,ignore 91 //! // Firstly the normal definition of the struct, attributes are preserved: 92 //! #[repr(C)] 93 //! struct Bar<T> { 94 //! t: T, 95 //! pub x: usize, 96 //! } 97 //! // Then an anonymous constant is defined, this is because we do not want any code to access the 98 //! // types that we define inside: 99 //! const _: () = { 100 //! // We define the pin-data carrying struct, it is a ZST and needs to have the same generics, 101 //! // since we need to implement access functions for each field and thus need to know its 102 //! // type. 103 //! struct __ThePinData<T> { 104 //! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>, 105 //! } 106 //! // We implement `Copy` for the pin-data struct, since all functions it defines will take 107 //! // `self` by value. 108 //! impl<T> ::core::clone::Clone for __ThePinData<T> { 109 //! fn clone(&self) -> Self { 110 //! *self 111 //! } 112 //! } 113 //! impl<T> ::core::marker::Copy for __ThePinData<T> {} 114 //! // For every field of `Bar`, the pin-data struct will define a function with the same name 115 //! // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the 116 //! // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field 117 //! // (if pinning is structural for the field, then `PinInit` otherwise `Init`). 118 //! #[allow(dead_code)] 119 //! impl<T> __ThePinData<T> { 120 //! unsafe fn t<E>( 121 //! self, 122 //! slot: *mut T, 123 //! // Since `t` is `#[pin]`, this is `PinInit`. 124 //! init: impl ::kernel::init::PinInit<T, E>, 125 //! ) -> ::core::result::Result<(), E> { 126 //! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) } 127 //! } 128 //! pub unsafe fn x<E>( 129 //! self, 130 //! slot: *mut usize, 131 //! // Since `x` is not `#[pin]`, this is `Init`. 132 //! init: impl ::kernel::init::Init<usize, E>, 133 //! ) -> ::core::result::Result<(), E> { 134 //! unsafe { ::kernel::init::Init::__init(init, slot) } 135 //! } 136 //! } 137 //! // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct 138 //! // that we constructed above. 139 //! unsafe impl<T> ::kernel::init::__internal::HasPinData for Bar<T> { 140 //! type PinData = __ThePinData<T>; 141 //! unsafe fn __pin_data() -> Self::PinData { 142 //! __ThePinData { 143 //! __phantom: ::core::marker::PhantomData, 144 //! } 145 //! } 146 //! } 147 //! // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data 148 //! // struct. This is important to ensure that no user can implement a rogue `__pin_data` 149 //! // function without using `unsafe`. 150 //! unsafe impl<T> ::kernel::init::__internal::PinData for __ThePinData<T> { 151 //! type Datee = Bar<T>; 152 //! } 153 //! // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is 154 //! // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned 155 //! // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our 156 //! // case no such fields exist, hence this is almost empty. The two phantomdata fields exist 157 //! // for two reasons: 158 //! // - `__phantom`: every generic must be used, since we cannot really know which generics 159 //! // are used, we declare all and then use everything here once. 160 //! // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant 161 //! // over it. The lifetime is needed to work around the limitation that trait bounds must 162 //! // not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is 163 //! // unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler 164 //! // into accepting these bounds regardless. 165 //! #[allow(dead_code)] 166 //! struct __Unpin<'__pin, T> { 167 //! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>, 168 //! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>, 169 //! // Our only `#[pin]` field is `t`. 170 //! t: T, 171 //! } 172 //! #[doc(hidden)] 173 //! impl<'__pin, T> ::core::marker::Unpin for Bar<T> 174 //! where 175 //! __Unpin<'__pin, T>: ::core::marker::Unpin, 176 //! {} 177 //! // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users 178 //! // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to 179 //! // UB with only safe code, so we disallow this by giving a trait implementation error using 180 //! // a direct impl and a blanket implementation. 181 //! trait MustNotImplDrop {} 182 //! // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do 183 //! // (normally people want to know if a type has any kind of drop glue at all, here we want 184 //! // to know if it has any kind of custom drop glue, which is exactly what this bound does). 185 //! #[expect(drop_bounds)] 186 //! impl<T: ::core::ops::Drop> MustNotImplDrop for T {} 187 //! impl<T> MustNotImplDrop for Bar<T> {} 188 //! // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to 189 //! // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed 190 //! // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`. 191 //! #[expect(non_camel_case_types)] 192 //! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} 193 //! impl< 194 //! T: ::kernel::init::PinnedDrop, 195 //! > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} 196 //! impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {} 197 //! }; 198 //! ``` 199 //! 200 //! ## `pin_init!` in `impl Bar` 201 //! 202 //! This macro creates an pin-initializer for the given struct. It requires that the struct is 203 //! annotated by `#[pin_data]`. 204 //! 205 //! Here is the impl on `Bar` defining the new function: 206 //! 207 //! ```rust,ignore 208 //! impl<T> Bar<T> { 209 //! fn new(t: T) -> impl PinInit<Self> { 210 //! pin_init!(Self { t, x: 0 }) 211 //! } 212 //! } 213 //! ``` 214 //! 215 //! This expands to the following code: 216 //! 217 //! ```rust,ignore 218 //! impl<T> Bar<T> { 219 //! fn new(t: T) -> impl PinInit<Self> { 220 //! { 221 //! // We do not want to allow arbitrary returns, so we declare this type as the `Ok` 222 //! // return type and shadow it later when we insert the arbitrary user code. That way 223 //! // there will be no possibility of returning without `unsafe`. 224 //! struct __InitOk; 225 //! // Get the data about fields from the supplied type. 226 //! // - the function is unsafe, hence the unsafe block 227 //! // - we `use` the `HasPinData` trait in the block, it is only available in that 228 //! // scope. 229 //! let data = unsafe { 230 //! use ::kernel::init::__internal::HasPinData; 231 //! Self::__pin_data() 232 //! }; 233 //! // Ensure that `data` really is of type `PinData` and help with type inference: 234 //! let init = ::kernel::init::__internal::PinData::make_closure::< 235 //! _, 236 //! __InitOk, 237 //! ::core::convert::Infallible, 238 //! >(data, move |slot| { 239 //! { 240 //! // Shadow the structure so it cannot be used to return early. If a user 241 //! // tries to write `return Ok(__InitOk)`, then they get a type error, 242 //! // since that will refer to this struct instead of the one defined 243 //! // above. 244 //! struct __InitOk; 245 //! // This is the expansion of `t,`, which is syntactic sugar for `t: t,`. 246 //! { 247 //! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) }; 248 //! } 249 //! // Since initialization could fail later (not in this case, since the 250 //! // error type is `Infallible`) we will need to drop this field if there 251 //! // is an error later. This `DropGuard` will drop the field when it gets 252 //! // dropped and has not yet been forgotten. 253 //! let __t_guard = unsafe { 254 //! ::pinned_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t)) 255 //! }; 256 //! // Expansion of `x: 0,`: 257 //! // Since this can be an arbitrary expression we cannot place it inside 258 //! // of the `unsafe` block, so we bind it here. 259 //! { 260 //! let x = 0; 261 //! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) }; 262 //! } 263 //! // We again create a `DropGuard`. 264 //! let __x_guard = unsafe { 265 //! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x)) 266 //! }; 267 //! // Since initialization has successfully completed, we can now forget 268 //! // the guards. This is not `mem::forget`, since we only have 269 //! // `&DropGuard`. 270 //! ::core::mem::forget(__x_guard); 271 //! ::core::mem::forget(__t_guard); 272 //! // Here we use the type checker to ensure that every field has been 273 //! // initialized exactly once, since this is `if false` it will never get 274 //! // executed, but still type-checked. 275 //! // Additionally we abuse `slot` to automatically infer the correct type 276 //! // for the struct. This is also another check that every field is 277 //! // accessible from this scope. 278 //! #[allow(unreachable_code, clippy::diverging_sub_expression)] 279 //! let _ = || { 280 //! unsafe { 281 //! ::core::ptr::write( 282 //! slot, 283 //! Self { 284 //! // We only care about typecheck finding every field 285 //! // here, the expression does not matter, just conjure 286 //! // one using `panic!()`: 287 //! t: ::core::panic!(), 288 //! x: ::core::panic!(), 289 //! }, 290 //! ); 291 //! }; 292 //! }; 293 //! } 294 //! // We leave the scope above and gain access to the previously shadowed 295 //! // `__InitOk` that we need to return. 296 //! Ok(__InitOk) 297 //! }); 298 //! // Change the return type from `__InitOk` to `()`. 299 //! let init = move | 300 //! slot, 301 //! | -> ::core::result::Result<(), ::core::convert::Infallible> { 302 //! init(slot).map(|__InitOk| ()) 303 //! }; 304 //! // Construct the initializer. 305 //! let init = unsafe { 306 //! ::kernel::init::pin_init_from_closure::< 307 //! _, 308 //! ::core::convert::Infallible, 309 //! >(init) 310 //! }; 311 //! init 312 //! } 313 //! } 314 //! } 315 //! ``` 316 //! 317 //! ## `#[pin_data]` on `Foo` 318 //! 319 //! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the 320 //! differences/new things in the expansion of the `Foo` definition: 321 //! 322 //! ```rust,ignore 323 //! #[pin_data(PinnedDrop)] 324 //! struct Foo { 325 //! a: usize, 326 //! #[pin] 327 //! b: Bar<u32>, 328 //! } 329 //! ``` 330 //! 331 //! This expands to the following code: 332 //! 333 //! ```rust,ignore 334 //! struct Foo { 335 //! a: usize, 336 //! b: Bar<u32>, 337 //! } 338 //! const _: () = { 339 //! struct __ThePinData { 340 //! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>, 341 //! } 342 //! impl ::core::clone::Clone for __ThePinData { 343 //! fn clone(&self) -> Self { 344 //! *self 345 //! } 346 //! } 347 //! impl ::core::marker::Copy for __ThePinData {} 348 //! #[allow(dead_code)] 349 //! impl __ThePinData { 350 //! unsafe fn b<E>( 351 //! self, 352 //! slot: *mut Bar<u32>, 353 //! init: impl ::kernel::init::PinInit<Bar<u32>, E>, 354 //! ) -> ::core::result::Result<(), E> { 355 //! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) } 356 //! } 357 //! unsafe fn a<E>( 358 //! self, 359 //! slot: *mut usize, 360 //! init: impl ::kernel::init::Init<usize, E>, 361 //! ) -> ::core::result::Result<(), E> { 362 //! unsafe { ::kernel::init::Init::__init(init, slot) } 363 //! } 364 //! } 365 //! unsafe impl ::kernel::init::__internal::HasPinData for Foo { 366 //! type PinData = __ThePinData; 367 //! unsafe fn __pin_data() -> Self::PinData { 368 //! __ThePinData { 369 //! __phantom: ::core::marker::PhantomData, 370 //! } 371 //! } 372 //! } 373 //! unsafe impl ::kernel::init::__internal::PinData for __ThePinData { 374 //! type Datee = Foo; 375 //! } 376 //! #[allow(dead_code)] 377 //! struct __Unpin<'__pin> { 378 //! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>, 379 //! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>, 380 //! b: Bar<u32>, 381 //! } 382 //! #[doc(hidden)] 383 //! impl<'__pin> ::core::marker::Unpin for Foo 384 //! where 385 //! __Unpin<'__pin>: ::core::marker::Unpin, 386 //! {} 387 //! // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to 388 //! // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like 389 //! // before, instead we implement `Drop` here and delegate to `PinnedDrop`. 390 //! impl ::core::ops::Drop for Foo { 391 //! fn drop(&mut self) { 392 //! // Since we are getting dropped, no one else has a reference to `self` and thus we 393 //! // can assume that we never move. 394 //! let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) }; 395 //! // Create the unsafe token that proves that we are inside of a destructor, this 396 //! // type is only allowed to be created in a destructor. 397 //! let token = unsafe { ::kernel::init::__internal::OnlyCallFromDrop::new() }; 398 //! ::kernel::init::PinnedDrop::drop(pinned, token); 399 //! } 400 //! } 401 //! }; 402 //! ``` 403 //! 404 //! ## `#[pinned_drop]` on `impl PinnedDrop for Foo` 405 //! 406 //! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an 407 //! extra parameter that should not be used at all. The macro hides that parameter. 408 //! 409 //! Here is the `PinnedDrop` impl for `Foo`: 410 //! 411 //! ```rust,ignore 412 //! #[pinned_drop] 413 //! impl PinnedDrop for Foo { 414 //! fn drop(self: Pin<&mut Self>) { 415 //! pr_info!("{self:p} is getting dropped."); 416 //! } 417 //! } 418 //! ``` 419 //! 420 //! This expands to the following code: 421 //! 422 //! ```rust,ignore 423 //! // `unsafe`, full path and the token parameter are added, everything else stays the same. 424 //! unsafe impl ::kernel::init::PinnedDrop for Foo { 425 //! fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) { 426 //! pr_info!("{self:p} is getting dropped."); 427 //! } 428 //! } 429 //! ``` 430 //! 431 //! ## `pin_init!` on `Foo` 432 //! 433 //! Since we already took a look at `pin_init!` on `Bar`, this section will only show the expansion 434 //! of `pin_init!` on `Foo`: 435 //! 436 //! ```rust,ignore 437 //! let a = 42; 438 //! let initializer = pin_init!(Foo { 439 //! a, 440 //! b <- Bar::new(36), 441 //! }); 442 //! ``` 443 //! 444 //! This expands to the following code: 445 //! 446 //! ```rust,ignore 447 //! let a = 42; 448 //! let initializer = { 449 //! struct __InitOk; 450 //! let data = unsafe { 451 //! use ::kernel::init::__internal::HasPinData; 452 //! Foo::__pin_data() 453 //! }; 454 //! let init = ::kernel::init::__internal::PinData::make_closure::< 455 //! _, 456 //! __InitOk, 457 //! ::core::convert::Infallible, 458 //! >(data, move |slot| { 459 //! { 460 //! struct __InitOk; 461 //! { 462 //! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) }; 463 //! } 464 //! let __a_guard = unsafe { 465 //! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a)) 466 //! }; 467 //! let init = Bar::new(36); 468 //! unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? }; 469 //! let __b_guard = unsafe { 470 //! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b)) 471 //! }; 472 //! ::core::mem::forget(__b_guard); 473 //! ::core::mem::forget(__a_guard); 474 //! #[allow(unreachable_code, clippy::diverging_sub_expression)] 475 //! let _ = || { 476 //! unsafe { 477 //! ::core::ptr::write( 478 //! slot, 479 //! Foo { 480 //! a: ::core::panic!(), 481 //! b: ::core::panic!(), 482 //! }, 483 //! ); 484 //! }; 485 //! }; 486 //! } 487 //! Ok(__InitOk) 488 //! }); 489 //! let init = move | 490 //! slot, 491 //! | -> ::core::result::Result<(), ::core::convert::Infallible> { 492 //! init(slot).map(|__InitOk| ()) 493 //! }; 494 //! let init = unsafe { 495 //! ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init) 496 //! }; 497 //! init 498 //! }; 499 //! ``` 500 501 /// Creates a `unsafe impl<...> PinnedDrop for $type` block. 502 /// 503 /// See [`PinnedDrop`] for more information. 504 #[doc(hidden)] 505 #[macro_export] 506 macro_rules! __pinned_drop { 507 ( 508 @impl_sig($($impl_sig:tt)*), 509 @impl_body( 510 $(#[$($attr:tt)*])* 511 fn drop($($sig:tt)*) { 512 $($inner:tt)* 513 } 514 ), 515 ) => { 516 // SAFETY: TODO. 517 unsafe $($impl_sig)* { 518 // Inherit all attributes and the type/ident tokens for the signature. 519 $(#[$($attr)*])* 520 fn drop($($sig)*, _: $crate::init::__internal::OnlyCallFromDrop) { 521 $($inner)* 522 } 523 } 524 } 525 } 526 527 /// This macro first parses the struct definition such that it separates pinned and not pinned 528 /// fields. Afterwards it declares the struct and implement the `PinData` trait safely. 529 #[doc(hidden)] 530 #[macro_export] 531 macro_rules! __pin_data { 532 // Proc-macro entry point, this is supplied by the proc-macro pre-parsing. 533 (parse_input: 534 @args($($pinned_drop:ident)?), 535 @sig( 536 $(#[$($struct_attr:tt)*])* 537 $vis:vis struct $name:ident 538 $(where $($whr:tt)*)? 539 ), 540 @impl_generics($($impl_generics:tt)*), 541 @ty_generics($($ty_generics:tt)*), 542 @decl_generics($($decl_generics:tt)*), 543 @body({ $($fields:tt)* }), 544 ) => { 545 // We now use token munching to iterate through all of the fields. While doing this we 546 // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user 547 // wants these to be structurally pinned. The rest of the fields are the 548 // 'not pinned fields'. Additionally we collect all fields, since we need them in the right 549 // order to declare the struct. 550 // 551 // In this call we also put some explaining comments for the parameters. 552 $crate::__pin_data!(find_pinned_fields: 553 // Attributes on the struct itself, these will just be propagated to be put onto the 554 // struct definition. 555 @struct_attrs($(#[$($struct_attr)*])*), 556 // The visibility of the struct. 557 @vis($vis), 558 // The name of the struct. 559 @name($name), 560 // The 'impl generics', the generics that will need to be specified on the struct inside 561 // of an `impl<$ty_generics>` block. 562 @impl_generics($($impl_generics)*), 563 // The 'ty generics', the generics that will need to be specified on the impl blocks. 564 @ty_generics($($ty_generics)*), 565 // The 'decl generics', the generics that need to be specified on the struct 566 // definition. 567 @decl_generics($($decl_generics)*), 568 // The where clause of any impl block and the declaration. 569 @where($($($whr)*)?), 570 // The remaining fields tokens that need to be processed. 571 // We add a `,` at the end to ensure correct parsing. 572 @fields_munch($($fields)* ,), 573 // The pinned fields. 574 @pinned(), 575 // The not pinned fields. 576 @not_pinned(), 577 // All fields. 578 @fields(), 579 // The accumulator containing all attributes already parsed. 580 @accum(), 581 // Contains `yes` or `` to indicate if `#[pin]` was found on the current field. 582 @is_pinned(), 583 // The proc-macro argument, this should be `PinnedDrop` or ``. 584 @pinned_drop($($pinned_drop)?), 585 ); 586 }; 587 (find_pinned_fields: 588 @struct_attrs($($struct_attrs:tt)*), 589 @vis($vis:vis), 590 @name($name:ident), 591 @impl_generics($($impl_generics:tt)*), 592 @ty_generics($($ty_generics:tt)*), 593 @decl_generics($($decl_generics:tt)*), 594 @where($($whr:tt)*), 595 // We found a PhantomPinned field, this should generally be pinned! 596 @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*), 597 @pinned($($pinned:tt)*), 598 @not_pinned($($not_pinned:tt)*), 599 @fields($($fields:tt)*), 600 @accum($($accum:tt)*), 601 // This field is not pinned. 602 @is_pinned(), 603 @pinned_drop($($pinned_drop:ident)?), 604 ) => { 605 ::core::compile_error!(concat!( 606 "The field `", 607 stringify!($field), 608 "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.", 609 )); 610 $crate::__pin_data!(find_pinned_fields: 611 @struct_attrs($($struct_attrs)*), 612 @vis($vis), 613 @name($name), 614 @impl_generics($($impl_generics)*), 615 @ty_generics($($ty_generics)*), 616 @decl_generics($($decl_generics)*), 617 @where($($whr)*), 618 @fields_munch($($rest)*), 619 @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,), 620 @not_pinned($($not_pinned)*), 621 @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,), 622 @accum(), 623 @is_pinned(), 624 @pinned_drop($($pinned_drop)?), 625 ); 626 }; 627 (find_pinned_fields: 628 @struct_attrs($($struct_attrs:tt)*), 629 @vis($vis:vis), 630 @name($name:ident), 631 @impl_generics($($impl_generics:tt)*), 632 @ty_generics($($ty_generics:tt)*), 633 @decl_generics($($decl_generics:tt)*), 634 @where($($whr:tt)*), 635 // We reached the field declaration. 636 @fields_munch($field:ident : $type:ty, $($rest:tt)*), 637 @pinned($($pinned:tt)*), 638 @not_pinned($($not_pinned:tt)*), 639 @fields($($fields:tt)*), 640 @accum($($accum:tt)*), 641 // This field is pinned. 642 @is_pinned(yes), 643 @pinned_drop($($pinned_drop:ident)?), 644 ) => { 645 $crate::__pin_data!(find_pinned_fields: 646 @struct_attrs($($struct_attrs)*), 647 @vis($vis), 648 @name($name), 649 @impl_generics($($impl_generics)*), 650 @ty_generics($($ty_generics)*), 651 @decl_generics($($decl_generics)*), 652 @where($($whr)*), 653 @fields_munch($($rest)*), 654 @pinned($($pinned)* $($accum)* $field: $type,), 655 @not_pinned($($not_pinned)*), 656 @fields($($fields)* $($accum)* $field: $type,), 657 @accum(), 658 @is_pinned(), 659 @pinned_drop($($pinned_drop)?), 660 ); 661 }; 662 (find_pinned_fields: 663 @struct_attrs($($struct_attrs:tt)*), 664 @vis($vis:vis), 665 @name($name:ident), 666 @impl_generics($($impl_generics:tt)*), 667 @ty_generics($($ty_generics:tt)*), 668 @decl_generics($($decl_generics:tt)*), 669 @where($($whr:tt)*), 670 // We reached the field declaration. 671 @fields_munch($field:ident : $type:ty, $($rest:tt)*), 672 @pinned($($pinned:tt)*), 673 @not_pinned($($not_pinned:tt)*), 674 @fields($($fields:tt)*), 675 @accum($($accum:tt)*), 676 // This field is not pinned. 677 @is_pinned(), 678 @pinned_drop($($pinned_drop:ident)?), 679 ) => { 680 $crate::__pin_data!(find_pinned_fields: 681 @struct_attrs($($struct_attrs)*), 682 @vis($vis), 683 @name($name), 684 @impl_generics($($impl_generics)*), 685 @ty_generics($($ty_generics)*), 686 @decl_generics($($decl_generics)*), 687 @where($($whr)*), 688 @fields_munch($($rest)*), 689 @pinned($($pinned)*), 690 @not_pinned($($not_pinned)* $($accum)* $field: $type,), 691 @fields($($fields)* $($accum)* $field: $type,), 692 @accum(), 693 @is_pinned(), 694 @pinned_drop($($pinned_drop)?), 695 ); 696 }; 697 (find_pinned_fields: 698 @struct_attrs($($struct_attrs:tt)*), 699 @vis($vis:vis), 700 @name($name:ident), 701 @impl_generics($($impl_generics:tt)*), 702 @ty_generics($($ty_generics:tt)*), 703 @decl_generics($($decl_generics:tt)*), 704 @where($($whr:tt)*), 705 // We found the `#[pin]` attr. 706 @fields_munch(#[pin] $($rest:tt)*), 707 @pinned($($pinned:tt)*), 708 @not_pinned($($not_pinned:tt)*), 709 @fields($($fields:tt)*), 710 @accum($($accum:tt)*), 711 @is_pinned($($is_pinned:ident)?), 712 @pinned_drop($($pinned_drop:ident)?), 713 ) => { 714 $crate::__pin_data!(find_pinned_fields: 715 @struct_attrs($($struct_attrs)*), 716 @vis($vis), 717 @name($name), 718 @impl_generics($($impl_generics)*), 719 @ty_generics($($ty_generics)*), 720 @decl_generics($($decl_generics)*), 721 @where($($whr)*), 722 @fields_munch($($rest)*), 723 // We do not include `#[pin]` in the list of attributes, since it is not actually an 724 // attribute that is defined somewhere. 725 @pinned($($pinned)*), 726 @not_pinned($($not_pinned)*), 727 @fields($($fields)*), 728 @accum($($accum)*), 729 // Set this to `yes`. 730 @is_pinned(yes), 731 @pinned_drop($($pinned_drop)?), 732 ); 733 }; 734 (find_pinned_fields: 735 @struct_attrs($($struct_attrs:tt)*), 736 @vis($vis:vis), 737 @name($name:ident), 738 @impl_generics($($impl_generics:tt)*), 739 @ty_generics($($ty_generics:tt)*), 740 @decl_generics($($decl_generics:tt)*), 741 @where($($whr:tt)*), 742 // We reached the field declaration with visibility, for simplicity we only munch the 743 // visibility and put it into `$accum`. 744 @fields_munch($fvis:vis $field:ident $($rest:tt)*), 745 @pinned($($pinned:tt)*), 746 @not_pinned($($not_pinned:tt)*), 747 @fields($($fields:tt)*), 748 @accum($($accum:tt)*), 749 @is_pinned($($is_pinned:ident)?), 750 @pinned_drop($($pinned_drop:ident)?), 751 ) => { 752 $crate::__pin_data!(find_pinned_fields: 753 @struct_attrs($($struct_attrs)*), 754 @vis($vis), 755 @name($name), 756 @impl_generics($($impl_generics)*), 757 @ty_generics($($ty_generics)*), 758 @decl_generics($($decl_generics)*), 759 @where($($whr)*), 760 @fields_munch($field $($rest)*), 761 @pinned($($pinned)*), 762 @not_pinned($($not_pinned)*), 763 @fields($($fields)*), 764 @accum($($accum)* $fvis), 765 @is_pinned($($is_pinned)?), 766 @pinned_drop($($pinned_drop)?), 767 ); 768 }; 769 (find_pinned_fields: 770 @struct_attrs($($struct_attrs:tt)*), 771 @vis($vis:vis), 772 @name($name:ident), 773 @impl_generics($($impl_generics:tt)*), 774 @ty_generics($($ty_generics:tt)*), 775 @decl_generics($($decl_generics:tt)*), 776 @where($($whr:tt)*), 777 // Some other attribute, just put it into `$accum`. 778 @fields_munch(#[$($attr:tt)*] $($rest:tt)*), 779 @pinned($($pinned:tt)*), 780 @not_pinned($($not_pinned:tt)*), 781 @fields($($fields:tt)*), 782 @accum($($accum:tt)*), 783 @is_pinned($($is_pinned:ident)?), 784 @pinned_drop($($pinned_drop:ident)?), 785 ) => { 786 $crate::__pin_data!(find_pinned_fields: 787 @struct_attrs($($struct_attrs)*), 788 @vis($vis), 789 @name($name), 790 @impl_generics($($impl_generics)*), 791 @ty_generics($($ty_generics)*), 792 @decl_generics($($decl_generics)*), 793 @where($($whr)*), 794 @fields_munch($($rest)*), 795 @pinned($($pinned)*), 796 @not_pinned($($not_pinned)*), 797 @fields($($fields)*), 798 @accum($($accum)* #[$($attr)*]), 799 @is_pinned($($is_pinned)?), 800 @pinned_drop($($pinned_drop)?), 801 ); 802 }; 803 (find_pinned_fields: 804 @struct_attrs($($struct_attrs:tt)*), 805 @vis($vis:vis), 806 @name($name:ident), 807 @impl_generics($($impl_generics:tt)*), 808 @ty_generics($($ty_generics:tt)*), 809 @decl_generics($($decl_generics:tt)*), 810 @where($($whr:tt)*), 811 // We reached the end of the fields, plus an optional additional comma, since we added one 812 // before and the user is also allowed to put a trailing comma. 813 @fields_munch($(,)?), 814 @pinned($($pinned:tt)*), 815 @not_pinned($($not_pinned:tt)*), 816 @fields($($fields:tt)*), 817 @accum(), 818 @is_pinned(), 819 @pinned_drop($($pinned_drop:ident)?), 820 ) => { 821 // Declare the struct with all fields in the correct order. 822 $($struct_attrs)* 823 $vis struct $name <$($decl_generics)*> 824 where $($whr)* 825 { 826 $($fields)* 827 } 828 829 // We put the rest into this const item, because it then will not be accessible to anything 830 // outside. 831 const _: () = { 832 // We declare this struct which will host all of the projection function for our type. 833 // it will be invariant over all generic parameters which are inherited from the 834 // struct. 835 $vis struct __ThePinData<$($impl_generics)*> 836 where $($whr)* 837 { 838 __phantom: ::core::marker::PhantomData< 839 fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*> 840 >, 841 } 842 843 impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*> 844 where $($whr)* 845 { 846 fn clone(&self) -> Self { *self } 847 } 848 849 impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*> 850 where $($whr)* 851 {} 852 853 // Make all projection functions. 854 $crate::__pin_data!(make_pin_data: 855 @pin_data(__ThePinData), 856 @impl_generics($($impl_generics)*), 857 @ty_generics($($ty_generics)*), 858 @where($($whr)*), 859 @pinned($($pinned)*), 860 @not_pinned($($not_pinned)*), 861 ); 862 863 // SAFETY: We have added the correct projection functions above to `__ThePinData` and 864 // we also use the least restrictive generics possible. 865 unsafe impl<$($impl_generics)*> 866 $crate::init::__internal::HasPinData for $name<$($ty_generics)*> 867 where $($whr)* 868 { 869 type PinData = __ThePinData<$($ty_generics)*>; 870 871 unsafe fn __pin_data() -> Self::PinData { 872 __ThePinData { __phantom: ::core::marker::PhantomData } 873 } 874 } 875 876 // SAFETY: TODO. 877 unsafe impl<$($impl_generics)*> 878 $crate::init::__internal::PinData for __ThePinData<$($ty_generics)*> 879 where $($whr)* 880 { 881 type Datee = $name<$($ty_generics)*>; 882 } 883 884 // This struct will be used for the unpin analysis. Since only structurally pinned 885 // fields are relevant whether the struct should implement `Unpin`. 886 #[allow(dead_code)] 887 struct __Unpin <'__pin, $($impl_generics)*> 888 where $($whr)* 889 { 890 __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>, 891 __phantom: ::core::marker::PhantomData< 892 fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*> 893 >, 894 // Only the pinned fields. 895 $($pinned)* 896 } 897 898 #[doc(hidden)] 899 impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*> 900 where 901 __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin, 902 $($whr)* 903 {} 904 905 // We need to disallow normal `Drop` implementation, the exact behavior depends on 906 // whether `PinnedDrop` was specified as the parameter. 907 $crate::__pin_data!(drop_prevention: 908 @name($name), 909 @impl_generics($($impl_generics)*), 910 @ty_generics($($ty_generics)*), 911 @where($($whr)*), 912 @pinned_drop($($pinned_drop)?), 913 ); 914 }; 915 }; 916 // When no `PinnedDrop` was specified, then we have to prevent implementing drop. 917 (drop_prevention: 918 @name($name:ident), 919 @impl_generics($($impl_generics:tt)*), 920 @ty_generics($($ty_generics:tt)*), 921 @where($($whr:tt)*), 922 @pinned_drop(), 923 ) => { 924 // We prevent this by creating a trait that will be implemented for all types implementing 925 // `Drop`. Additionally we will implement this trait for the struct leading to a conflict, 926 // if it also implements `Drop` 927 trait MustNotImplDrop {} 928 #[expect(drop_bounds)] 929 impl<T: ::core::ops::Drop> MustNotImplDrop for T {} 930 impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*> 931 where $($whr)* {} 932 // We also take care to prevent users from writing a useless `PinnedDrop` implementation. 933 // They might implement `PinnedDrop` correctly for the struct, but forget to give 934 // `PinnedDrop` as the parameter to `#[pin_data]`. 935 #[expect(non_camel_case_types)] 936 trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} 937 impl<T: $crate::init::PinnedDrop> 938 UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} 939 impl<$($impl_generics)*> 940 UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*> 941 where $($whr)* {} 942 }; 943 // When `PinnedDrop` was specified we just implement `Drop` and delegate. 944 (drop_prevention: 945 @name($name:ident), 946 @impl_generics($($impl_generics:tt)*), 947 @ty_generics($($ty_generics:tt)*), 948 @where($($whr:tt)*), 949 @pinned_drop(PinnedDrop), 950 ) => { 951 impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*> 952 where $($whr)* 953 { 954 fn drop(&mut self) { 955 // SAFETY: Since this is a destructor, `self` will not move after this function 956 // terminates, since it is inaccessible. 957 let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) }; 958 // SAFETY: Since this is a drop function, we can create this token to call the 959 // pinned destructor of this type. 960 let token = unsafe { $crate::init::__internal::OnlyCallFromDrop::new() }; 961 $crate::init::PinnedDrop::drop(pinned, token); 962 } 963 } 964 }; 965 // If some other parameter was specified, we emit a readable error. 966 (drop_prevention: 967 @name($name:ident), 968 @impl_generics($($impl_generics:tt)*), 969 @ty_generics($($ty_generics:tt)*), 970 @where($($whr:tt)*), 971 @pinned_drop($($rest:tt)*), 972 ) => { 973 compile_error!( 974 "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.", 975 stringify!($($rest)*), 976 ); 977 }; 978 (make_pin_data: 979 @pin_data($pin_data:ident), 980 @impl_generics($($impl_generics:tt)*), 981 @ty_generics($($ty_generics:tt)*), 982 @where($($whr:tt)*), 983 @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?), 984 @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?), 985 ) => { 986 // For every field, we create a projection function according to its projection type. If a 987 // field is structurally pinned, then it must be initialized via `PinInit`, if it is not 988 // structurally pinned, then it can be initialized via `Init`. 989 // 990 // The functions are `unsafe` to prevent accidentally calling them. 991 #[allow(dead_code)] 992 #[expect(clippy::missing_safety_doc)] 993 impl<$($impl_generics)*> $pin_data<$($ty_generics)*> 994 where $($whr)* 995 { 996 $( 997 $(#[$($p_attr)*])* 998 $pvis unsafe fn $p_field<E>( 999 self, 1000 slot: *mut $p_type, 1001 init: impl $crate::init::PinInit<$p_type, E>, 1002 ) -> ::core::result::Result<(), E> { 1003 // SAFETY: TODO. 1004 unsafe { $crate::init::PinInit::__pinned_init(init, slot) } 1005 } 1006 )* 1007 $( 1008 $(#[$($attr)*])* 1009 $fvis unsafe fn $field<E>( 1010 self, 1011 slot: *mut $type, 1012 init: impl $crate::init::Init<$type, E>, 1013 ) -> ::core::result::Result<(), E> { 1014 // SAFETY: TODO. 1015 unsafe { $crate::init::Init::__init(init, slot) } 1016 } 1017 )* 1018 } 1019 }; 1020 } 1021 1022 /// The internal init macro. Do not call manually! 1023 /// 1024 /// This is called by the `{try_}{pin_}init!` macros with various inputs. 1025 /// 1026 /// This macro has multiple internal call configurations, these are always the very first ident: 1027 /// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros. 1028 /// - `with_update_parsed`: when the `..Zeroable::zeroed()` syntax has been handled. 1029 /// - `init_slot`: recursively creates the code that initializes all fields in `slot`. 1030 /// - `make_initializer`: recursively create the struct initializer that guarantees that every 1031 /// field has been initialized exactly once. 1032 #[doc(hidden)] 1033 #[macro_export] 1034 macro_rules! __init_internal { 1035 ( 1036 @this($($this:ident)?), 1037 @typ($t:path), 1038 @fields($($fields:tt)*), 1039 @error($err:ty), 1040 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` 1041 // case. 1042 @data($data:ident, $($use_data:ident)?), 1043 // `HasPinData` or `HasInitData`. 1044 @has_data($has_data:ident, $get_data:ident), 1045 // `pin_init_from_closure` or `init_from_closure`. 1046 @construct_closure($construct_closure:ident), 1047 @munch_fields(), 1048 ) => { 1049 $crate::__init_internal!(with_update_parsed: 1050 @this($($this)?), 1051 @typ($t), 1052 @fields($($fields)*), 1053 @error($err), 1054 @data($data, $($use_data)?), 1055 @has_data($has_data, $get_data), 1056 @construct_closure($construct_closure), 1057 @zeroed(), // Nothing means default behavior. 1058 ) 1059 }; 1060 ( 1061 @this($($this:ident)?), 1062 @typ($t:path), 1063 @fields($($fields:tt)*), 1064 @error($err:ty), 1065 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` 1066 // case. 1067 @data($data:ident, $($use_data:ident)?), 1068 // `HasPinData` or `HasInitData`. 1069 @has_data($has_data:ident, $get_data:ident), 1070 // `pin_init_from_closure` or `init_from_closure`. 1071 @construct_closure($construct_closure:ident), 1072 @munch_fields(..Zeroable::zeroed()), 1073 ) => { 1074 $crate::__init_internal!(with_update_parsed: 1075 @this($($this)?), 1076 @typ($t), 1077 @fields($($fields)*), 1078 @error($err), 1079 @data($data, $($use_data)?), 1080 @has_data($has_data, $get_data), 1081 @construct_closure($construct_closure), 1082 @zeroed(()), // `()` means zero all fields not mentioned. 1083 ) 1084 }; 1085 ( 1086 @this($($this:ident)?), 1087 @typ($t:path), 1088 @fields($($fields:tt)*), 1089 @error($err:ty), 1090 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` 1091 // case. 1092 @data($data:ident, $($use_data:ident)?), 1093 // `HasPinData` or `HasInitData`. 1094 @has_data($has_data:ident, $get_data:ident), 1095 // `pin_init_from_closure` or `init_from_closure`. 1096 @construct_closure($construct_closure:ident), 1097 @munch_fields($ignore:tt $($rest:tt)*), 1098 ) => { 1099 $crate::__init_internal!( 1100 @this($($this)?), 1101 @typ($t), 1102 @fields($($fields)*), 1103 @error($err), 1104 @data($data, $($use_data)?), 1105 @has_data($has_data, $get_data), 1106 @construct_closure($construct_closure), 1107 @munch_fields($($rest)*), 1108 ) 1109 }; 1110 (with_update_parsed: 1111 @this($($this:ident)?), 1112 @typ($t:path), 1113 @fields($($fields:tt)*), 1114 @error($err:ty), 1115 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` 1116 // case. 1117 @data($data:ident, $($use_data:ident)?), 1118 // `HasPinData` or `HasInitData`. 1119 @has_data($has_data:ident, $get_data:ident), 1120 // `pin_init_from_closure` or `init_from_closure`. 1121 @construct_closure($construct_closure:ident), 1122 @zeroed($($init_zeroed:expr)?), 1123 ) => {{ 1124 // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return 1125 // type and shadow it later when we insert the arbitrary user code. That way there will be 1126 // no possibility of returning without `unsafe`. 1127 struct __InitOk; 1128 // Get the data about fields from the supplied type. 1129 // 1130 // SAFETY: TODO. 1131 let data = unsafe { 1132 use $crate::init::__internal::$has_data; 1133 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal 1134 // information that is associated to already parsed fragments, so a path fragment 1135 // cannot be used in this position. Doing the retokenization results in valid rust 1136 // code. 1137 ::kernel::macros::paste!($t::$get_data()) 1138 }; 1139 // Ensure that `data` really is of type `$data` and help with type inference: 1140 let init = $crate::init::__internal::$data::make_closure::<_, __InitOk, $err>( 1141 data, 1142 move |slot| { 1143 { 1144 // Shadow the structure so it cannot be used to return early. 1145 struct __InitOk; 1146 // If `$init_zeroed` is present we should zero the slot now and not emit an 1147 // error when fields are missing (since they will be zeroed). We also have to 1148 // check that the type actually implements `Zeroable`. 1149 $({ 1150 fn assert_zeroable<T: $crate::init::Zeroable>(_: *mut T) {} 1151 // Ensure that the struct is indeed `Zeroable`. 1152 assert_zeroable(slot); 1153 // SAFETY: The type implements `Zeroable` by the check above. 1154 unsafe { ::core::ptr::write_bytes(slot, 0, 1) }; 1155 $init_zeroed // This will be `()` if set. 1156 })? 1157 // Create the `this` so it can be referenced by the user inside of the 1158 // expressions creating the individual fields. 1159 $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? 1160 // Initialize every field. 1161 $crate::__init_internal!(init_slot($($use_data)?): 1162 @data(data), 1163 @slot(slot), 1164 @guards(), 1165 @munch_fields($($fields)*,), 1166 ); 1167 // We use unreachable code to ensure that all fields have been mentioned exactly 1168 // once, this struct initializer will still be type-checked and complain with a 1169 // very natural error message if a field is forgotten/mentioned more than once. 1170 #[allow(unreachable_code, clippy::diverging_sub_expression)] 1171 let _ = || { 1172 $crate::__init_internal!(make_initializer: 1173 @slot(slot), 1174 @type_name($t), 1175 @munch_fields($($fields)*,), 1176 @acc(), 1177 ); 1178 }; 1179 } 1180 Ok(__InitOk) 1181 } 1182 ); 1183 let init = move |slot| -> ::core::result::Result<(), $err> { 1184 init(slot).map(|__InitOk| ()) 1185 }; 1186 // SAFETY: TODO. 1187 let init = unsafe { $crate::init::$construct_closure::<_, $err>(init) }; 1188 init 1189 }}; 1190 (init_slot($($use_data:ident)?): 1191 @data($data:ident), 1192 @slot($slot:ident), 1193 @guards($($guards:ident,)*), 1194 @munch_fields($(..Zeroable::zeroed())? $(,)?), 1195 ) => { 1196 // Endpoint of munching, no fields are left. If execution reaches this point, all fields 1197 // have been initialized. Therefore we can now dismiss the guards by forgetting them. 1198 $(::core::mem::forget($guards);)* 1199 }; 1200 (init_slot($use_data:ident): // `use_data` is present, so we use the `data` to init fields. 1201 @data($data:ident), 1202 @slot($slot:ident), 1203 @guards($($guards:ident,)*), 1204 // In-place initialization syntax. 1205 @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 1206 ) => { 1207 let init = $val; 1208 // Call the initializer. 1209 // 1210 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we 1211 // return when an error/panic occurs. 1212 // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`. 1213 unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? }; 1214 // Create the drop guard: 1215 // 1216 // We rely on macro hygiene to make it impossible for users to access this local variable. 1217 // We use `paste!` to create new hygiene for `$field`. 1218 ::kernel::macros::paste! { 1219 // SAFETY: We forget the guard later when initialization has succeeded. 1220 let [< __ $field _guard >] = unsafe { 1221 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) 1222 }; 1223 1224 $crate::__init_internal!(init_slot($use_data): 1225 @data($data), 1226 @slot($slot), 1227 @guards([< __ $field _guard >], $($guards,)*), 1228 @munch_fields($($rest)*), 1229 ); 1230 } 1231 }; 1232 (init_slot(): // No `use_data`, so we use `Init::__init` directly. 1233 @data($data:ident), 1234 @slot($slot:ident), 1235 @guards($($guards:ident,)*), 1236 // In-place initialization syntax. 1237 @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 1238 ) => { 1239 let init = $val; 1240 // Call the initializer. 1241 // 1242 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we 1243 // return when an error/panic occurs. 1244 unsafe { $crate::init::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? }; 1245 // Create the drop guard: 1246 // 1247 // We rely on macro hygiene to make it impossible for users to access this local variable. 1248 // We use `paste!` to create new hygiene for `$field`. 1249 ::kernel::macros::paste! { 1250 // SAFETY: We forget the guard later when initialization has succeeded. 1251 let [< __ $field _guard >] = unsafe { 1252 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) 1253 }; 1254 1255 $crate::__init_internal!(init_slot(): 1256 @data($data), 1257 @slot($slot), 1258 @guards([< __ $field _guard >], $($guards,)*), 1259 @munch_fields($($rest)*), 1260 ); 1261 } 1262 }; 1263 (init_slot($($use_data:ident)?): 1264 @data($data:ident), 1265 @slot($slot:ident), 1266 @guards($($guards:ident,)*), 1267 // Init by-value. 1268 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), 1269 ) => { 1270 { 1271 $(let $field = $val;)? 1272 // Initialize the field. 1273 // 1274 // SAFETY: The memory at `slot` is uninitialized. 1275 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; 1276 } 1277 // Create the drop guard: 1278 // 1279 // We rely on macro hygiene to make it impossible for users to access this local variable. 1280 // We use `paste!` to create new hygiene for `$field`. 1281 ::kernel::macros::paste! { 1282 // SAFETY: We forget the guard later when initialization has succeeded. 1283 let [< __ $field _guard >] = unsafe { 1284 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) 1285 }; 1286 1287 $crate::__init_internal!(init_slot($($use_data)?): 1288 @data($data), 1289 @slot($slot), 1290 @guards([< __ $field _guard >], $($guards,)*), 1291 @munch_fields($($rest)*), 1292 ); 1293 } 1294 }; 1295 (make_initializer: 1296 @slot($slot:ident), 1297 @type_name($t:path), 1298 @munch_fields(..Zeroable::zeroed() $(,)?), 1299 @acc($($acc:tt)*), 1300 ) => { 1301 // Endpoint, nothing more to munch, create the initializer. Since the users specified 1302 // `..Zeroable::zeroed()`, the slot will already have been zeroed and all field that have 1303 // not been overwritten are thus zero and initialized. We still check that all fields are 1304 // actually accessible by using the struct update syntax ourselves. 1305 // We are inside of a closure that is never executed and thus we can abuse `slot` to 1306 // get the correct type inference here: 1307 #[allow(unused_assignments)] 1308 unsafe { 1309 let mut zeroed = ::core::mem::zeroed(); 1310 // We have to use type inference here to make zeroed have the correct type. This does 1311 // not get executed, so it has no effect. 1312 ::core::ptr::write($slot, zeroed); 1313 zeroed = ::core::mem::zeroed(); 1314 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal 1315 // information that is associated to already parsed fragments, so a path fragment 1316 // cannot be used in this position. Doing the retokenization results in valid rust 1317 // code. 1318 ::kernel::macros::paste!( 1319 ::core::ptr::write($slot, $t { 1320 $($acc)* 1321 ..zeroed 1322 }); 1323 ); 1324 } 1325 }; 1326 (make_initializer: 1327 @slot($slot:ident), 1328 @type_name($t:path), 1329 @munch_fields($(,)?), 1330 @acc($($acc:tt)*), 1331 ) => { 1332 // Endpoint, nothing more to munch, create the initializer. 1333 // Since we are in the closure that is never called, this will never get executed. 1334 // We abuse `slot` to get the correct type inference here: 1335 // 1336 // SAFETY: TODO. 1337 unsafe { 1338 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal 1339 // information that is associated to already parsed fragments, so a path fragment 1340 // cannot be used in this position. Doing the retokenization results in valid rust 1341 // code. 1342 ::kernel::macros::paste!( 1343 ::core::ptr::write($slot, $t { 1344 $($acc)* 1345 }); 1346 ); 1347 } 1348 }; 1349 (make_initializer: 1350 @slot($slot:ident), 1351 @type_name($t:path), 1352 @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 1353 @acc($($acc:tt)*), 1354 ) => { 1355 $crate::__init_internal!(make_initializer: 1356 @slot($slot), 1357 @type_name($t), 1358 @munch_fields($($rest)*), 1359 @acc($($acc)* $field: ::core::panic!(),), 1360 ); 1361 }; 1362 (make_initializer: 1363 @slot($slot:ident), 1364 @type_name($t:path), 1365 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), 1366 @acc($($acc:tt)*), 1367 ) => { 1368 $crate::__init_internal!(make_initializer: 1369 @slot($slot), 1370 @type_name($t), 1371 @munch_fields($($rest)*), 1372 @acc($($acc)* $field: ::core::panic!(),), 1373 ); 1374 }; 1375 } 1376 1377 #[doc(hidden)] 1378 #[macro_export] 1379 macro_rules! __derive_zeroable { 1380 (parse_input: 1381 @sig( 1382 $(#[$($struct_attr:tt)*])* 1383 $vis:vis struct $name:ident 1384 $(where $($whr:tt)*)? 1385 ), 1386 @impl_generics($($impl_generics:tt)*), 1387 @ty_generics($($ty_generics:tt)*), 1388 @body({ 1389 $( 1390 $(#[$($field_attr:tt)*])* 1391 $field:ident : $field_ty:ty 1392 ),* $(,)? 1393 }), 1394 ) => { 1395 // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero. 1396 #[automatically_derived] 1397 unsafe impl<$($impl_generics)*> $crate::init::Zeroable for $name<$($ty_generics)*> 1398 where 1399 $($($whr)*)? 1400 {} 1401 const _: () = { 1402 fn assert_zeroable<T: ?::core::marker::Sized + $crate::init::Zeroable>() {} 1403 fn ensure_zeroable<$($impl_generics)*>() 1404 where $($($whr)*)? 1405 { 1406 $(assert_zeroable::<$field_ty>();)* 1407 } 1408 }; 1409 }; 1410 } 1411