1 // SPDX-License-Identifier: Apache-2.0 OR MIT 2 3 //! This module provides the macros that actually implement the proc-macros `pin_data` and 4 //! `pinned_drop`. It also contains `__init_internal`, the implementation of the 5 //! `{try_}{pin_}init!` macros. 6 //! 7 //! These macros should never be called directly, since they expect their input to be 8 //! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in 9 //! safe code! Use the public facing macros instead. 10 //! 11 //! This architecture has been chosen because the kernel does not yet have access to `syn` which 12 //! would make matters a lot easier for implementing these as proc-macros. 13 //! 14 //! Since this library and the kernel implementation should diverge as little as possible, the same 15 //! approach has been taken here. 16 //! 17 //! # Macro expansion example 18 //! 19 //! This section is intended for readers trying to understand the macros in this module and the 20 //! `[try_][pin_]init!` macros from `lib.rs`. 21 //! 22 //! We will look at the following example: 23 //! 24 //! ```rust,ignore 25 //! #[pin_data] 26 //! #[repr(C)] 27 //! struct Bar<T> { 28 //! #[pin] 29 //! t: T, 30 //! pub x: usize, 31 //! } 32 //! 33 //! impl<T> Bar<T> { 34 //! fn new(t: T) -> impl PinInit<Self> { 35 //! pin_init!(Self { t, x: 0 }) 36 //! } 37 //! } 38 //! 39 //! #[pin_data(PinnedDrop)] 40 //! struct Foo { 41 //! a: usize, 42 //! #[pin] 43 //! b: Bar<u32>, 44 //! } 45 //! 46 //! #[pinned_drop] 47 //! impl PinnedDrop for Foo { 48 //! fn drop(self: Pin<&mut Self>) { 49 //! println!("{self:p} is getting dropped."); 50 //! } 51 //! } 52 //! 53 //! let a = 42; 54 //! let initializer = pin_init!(Foo { 55 //! a, 56 //! b <- Bar::new(36), 57 //! }); 58 //! ``` 59 //! 60 //! This example includes the most common and important features of the pin-init API. 61 //! 62 //! Below you can find individual section about the different macro invocations. Here are some 63 //! general things we need to take into account when designing macros: 64 //! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()` 65 //! this ensures that the correct item is used, since users could define their own `mod core {}` 66 //! and then their own `panic!` inside to execute arbitrary code inside of our macro. 67 //! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied 68 //! expressions inside of an `unsafe` block in the macro, because this would allow users to do 69 //! `unsafe` operations without an associated `unsafe` block. 70 //! 71 //! ## `#[pin_data]` on `Bar` 72 //! 73 //! This macro is used to specify which fields are structurally pinned and which fields are not. It 74 //! is placed on the struct definition and allows `#[pin]` to be placed on the fields. 75 //! 76 //! Here is the definition of `Bar` from our example: 77 //! 78 //! ```rust,ignore 79 //! #[pin_data] 80 //! #[repr(C)] 81 //! struct Bar<T> { 82 //! #[pin] 83 //! t: T, 84 //! pub x: usize, 85 //! } 86 //! ``` 87 //! 88 //! This expands to the following code: 89 //! 90 //! ```rust,ignore 91 //! // Firstly the normal definition of the struct, attributes are preserved: 92 //! #[repr(C)] 93 //! struct Bar<T> { 94 //! t: T, 95 //! pub x: usize, 96 //! } 97 //! // Then an anonymous constant is defined, this is because we do not want any code to access the 98 //! // types that we define inside: 99 //! const _: () = { 100 //! // We define the pin-data carrying struct, it is a ZST and needs to have the same generics, 101 //! // since we need to implement access functions for each field and thus need to know its 102 //! // type. 103 //! struct __ThePinData<T> { 104 //! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>, 105 //! } 106 //! // We implement `Copy` for the pin-data struct, since all functions it defines will take 107 //! // `self` by value. 108 //! impl<T> ::core::clone::Clone for __ThePinData<T> { 109 //! fn clone(&self) -> Self { 110 //! *self 111 //! } 112 //! } 113 //! impl<T> ::core::marker::Copy for __ThePinData<T> {} 114 //! // For every field of `Bar`, the pin-data struct will define a function with the same name 115 //! // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the 116 //! // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field 117 //! // (if pinning is structural for the field, then `PinInit` otherwise `Init`). 118 //! #[allow(dead_code)] 119 //! impl<T> __ThePinData<T> { 120 //! unsafe fn t<E>( 121 //! self, 122 //! slot: *mut T, 123 //! // Since `t` is `#[pin]`, this is `PinInit`. 124 //! init: impl ::pin_init::PinInit<T, E>, 125 //! ) -> ::core::result::Result<(), E> { 126 //! unsafe { ::pin_init::PinInit::__pinned_init(init, slot) } 127 //! } 128 //! pub unsafe fn x<E>( 129 //! self, 130 //! slot: *mut usize, 131 //! // Since `x` is not `#[pin]`, this is `Init`. 132 //! init: impl ::pin_init::Init<usize, E>, 133 //! ) -> ::core::result::Result<(), E> { 134 //! unsafe { ::pin_init::Init::__init(init, slot) } 135 //! } 136 //! } 137 //! // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct 138 //! // that we constructed above. 139 //! unsafe impl<T> ::pin_init::__internal::HasPinData for Bar<T> { 140 //! type PinData = __ThePinData<T>; 141 //! unsafe fn __pin_data() -> Self::PinData { 142 //! __ThePinData { 143 //! __phantom: ::core::marker::PhantomData, 144 //! } 145 //! } 146 //! } 147 //! // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data 148 //! // struct. This is important to ensure that no user can implement a rogue `__pin_data` 149 //! // function without using `unsafe`. 150 //! unsafe impl<T> ::pin_init::__internal::PinData for __ThePinData<T> { 151 //! type Datee = Bar<T>; 152 //! } 153 //! // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is 154 //! // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned 155 //! // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our 156 //! // case no such fields exist, hence this is almost empty. The two phantomdata fields exist 157 //! // for two reasons: 158 //! // - `__phantom`: every generic must be used, since we cannot really know which generics 159 //! // are used, we declare all and then use everything here once. 160 //! // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant 161 //! // over it. The lifetime is needed to work around the limitation that trait bounds must 162 //! // not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is 163 //! // unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler 164 //! // into accepting these bounds regardless. 165 //! #[allow(dead_code)] 166 //! struct __Unpin<'__pin, T> { 167 //! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>, 168 //! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>, 169 //! // Our only `#[pin]` field is `t`. 170 //! t: T, 171 //! } 172 //! #[doc(hidden)] 173 //! impl<'__pin, T> ::core::marker::Unpin for Bar<T> 174 //! where 175 //! __Unpin<'__pin, T>: ::core::marker::Unpin, 176 //! {} 177 //! // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users 178 //! // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to 179 //! // UB with only safe code, so we disallow this by giving a trait implementation error using 180 //! // a direct impl and a blanket implementation. 181 //! trait MustNotImplDrop {} 182 //! // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do 183 //! // (normally people want to know if a type has any kind of drop glue at all, here we want 184 //! // to know if it has any kind of custom drop glue, which is exactly what this bound does). 185 //! #[expect(drop_bounds)] 186 //! impl<T: ::core::ops::Drop> MustNotImplDrop for T {} 187 //! impl<T> MustNotImplDrop for Bar<T> {} 188 //! // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to 189 //! // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed 190 //! // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`. 191 //! #[expect(non_camel_case_types)] 192 //! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} 193 //! impl< 194 //! T: ::pin_init::PinnedDrop, 195 //! > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} 196 //! impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {} 197 //! }; 198 //! ``` 199 //! 200 //! ## `pin_init!` in `impl Bar` 201 //! 202 //! This macro creates an pin-initializer for the given struct. It requires that the struct is 203 //! annotated by `#[pin_data]`. 204 //! 205 //! Here is the impl on `Bar` defining the new function: 206 //! 207 //! ```rust,ignore 208 //! impl<T> Bar<T> { 209 //! fn new(t: T) -> impl PinInit<Self> { 210 //! pin_init!(Self { t, x: 0 }) 211 //! } 212 //! } 213 //! ``` 214 //! 215 //! This expands to the following code: 216 //! 217 //! ```rust,ignore 218 //! impl<T> Bar<T> { 219 //! fn new(t: T) -> impl PinInit<Self> { 220 //! { 221 //! // We do not want to allow arbitrary returns, so we declare this type as the `Ok` 222 //! // return type and shadow it later when we insert the arbitrary user code. That way 223 //! // there will be no possibility of returning without `unsafe`. 224 //! struct __InitOk; 225 //! // Get the data about fields from the supplied type. 226 //! // - the function is unsafe, hence the unsafe block 227 //! // - we `use` the `HasPinData` trait in the block, it is only available in that 228 //! // scope. 229 //! let data = unsafe { 230 //! use ::pin_init::__internal::HasPinData; 231 //! Self::__pin_data() 232 //! }; 233 //! // Ensure that `data` really is of type `PinData` and help with type inference: 234 //! let init = ::pin_init::__internal::PinData::make_closure::< 235 //! _, 236 //! __InitOk, 237 //! ::core::convert::Infallible, 238 //! >(data, move |slot| { 239 //! { 240 //! // Shadow the structure so it cannot be used to return early. If a user 241 //! // tries to write `return Ok(__InitOk)`, then they get a type error, 242 //! // since that will refer to this struct instead of the one defined 243 //! // above. 244 //! struct __InitOk; 245 //! // This is the expansion of `t,`, which is syntactic sugar for `t: t,`. 246 //! { 247 //! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) }; 248 //! } 249 //! // Since initialization could fail later (not in this case, since the 250 //! // error type is `Infallible`) we will need to drop this field if there 251 //! // is an error later. This `DropGuard` will drop the field when it gets 252 //! // dropped and has not yet been forgotten. 253 //! let __t_guard = unsafe { 254 //! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t)) 255 //! }; 256 //! // Expansion of `x: 0,`: 257 //! // Since this can be an arbitrary expression we cannot place it inside 258 //! // of the `unsafe` block, so we bind it here. 259 //! { 260 //! let x = 0; 261 //! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) }; 262 //! } 263 //! // We again create a `DropGuard`. 264 //! let __x_guard = unsafe { 265 //! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x)) 266 //! }; 267 //! // Since initialization has successfully completed, we can now forget 268 //! // the guards. This is not `mem::forget`, since we only have 269 //! // `&DropGuard`. 270 //! ::core::mem::forget(__x_guard); 271 //! ::core::mem::forget(__t_guard); 272 //! // Here we use the type checker to ensure that every field has been 273 //! // initialized exactly once, since this is `if false` it will never get 274 //! // executed, but still type-checked. 275 //! // Additionally we abuse `slot` to automatically infer the correct type 276 //! // for the struct. This is also another check that every field is 277 //! // accessible from this scope. 278 //! #[allow(unreachable_code, clippy::diverging_sub_expression)] 279 //! let _ = || { 280 //! unsafe { 281 //! ::core::ptr::write( 282 //! slot, 283 //! Self { 284 //! // We only care about typecheck finding every field 285 //! // here, the expression does not matter, just conjure 286 //! // one using `panic!()`: 287 //! t: ::core::panic!(), 288 //! x: ::core::panic!(), 289 //! }, 290 //! ); 291 //! }; 292 //! }; 293 //! } 294 //! // We leave the scope above and gain access to the previously shadowed 295 //! // `__InitOk` that we need to return. 296 //! Ok(__InitOk) 297 //! }); 298 //! // Change the return type from `__InitOk` to `()`. 299 //! let init = move | 300 //! slot, 301 //! | -> ::core::result::Result<(), ::core::convert::Infallible> { 302 //! init(slot).map(|__InitOk| ()) 303 //! }; 304 //! // Construct the initializer. 305 //! let init = unsafe { 306 //! ::pin_init::pin_init_from_closure::< 307 //! _, 308 //! ::core::convert::Infallible, 309 //! >(init) 310 //! }; 311 //! init 312 //! } 313 //! } 314 //! } 315 //! ``` 316 //! 317 //! ## `#[pin_data]` on `Foo` 318 //! 319 //! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the 320 //! differences/new things in the expansion of the `Foo` definition: 321 //! 322 //! ```rust,ignore 323 //! #[pin_data(PinnedDrop)] 324 //! struct Foo { 325 //! a: usize, 326 //! #[pin] 327 //! b: Bar<u32>, 328 //! } 329 //! ``` 330 //! 331 //! This expands to the following code: 332 //! 333 //! ```rust,ignore 334 //! struct Foo { 335 //! a: usize, 336 //! b: Bar<u32>, 337 //! } 338 //! const _: () = { 339 //! struct __ThePinData { 340 //! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>, 341 //! } 342 //! impl ::core::clone::Clone for __ThePinData { 343 //! fn clone(&self) -> Self { 344 //! *self 345 //! } 346 //! } 347 //! impl ::core::marker::Copy for __ThePinData {} 348 //! #[allow(dead_code)] 349 //! impl __ThePinData { 350 //! unsafe fn b<E>( 351 //! self, 352 //! slot: *mut Bar<u32>, 353 //! init: impl ::pin_init::PinInit<Bar<u32>, E>, 354 //! ) -> ::core::result::Result<(), E> { 355 //! unsafe { ::pin_init::PinInit::__pinned_init(init, slot) } 356 //! } 357 //! unsafe fn a<E>( 358 //! self, 359 //! slot: *mut usize, 360 //! init: impl ::pin_init::Init<usize, E>, 361 //! ) -> ::core::result::Result<(), E> { 362 //! unsafe { ::pin_init::Init::__init(init, slot) } 363 //! } 364 //! } 365 //! unsafe impl ::pin_init::__internal::HasPinData for Foo { 366 //! type PinData = __ThePinData; 367 //! unsafe fn __pin_data() -> Self::PinData { 368 //! __ThePinData { 369 //! __phantom: ::core::marker::PhantomData, 370 //! } 371 //! } 372 //! } 373 //! unsafe impl ::pin_init::__internal::PinData for __ThePinData { 374 //! type Datee = Foo; 375 //! } 376 //! #[allow(dead_code)] 377 //! struct __Unpin<'__pin> { 378 //! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>, 379 //! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>, 380 //! b: Bar<u32>, 381 //! } 382 //! #[doc(hidden)] 383 //! impl<'__pin> ::core::marker::Unpin for Foo 384 //! where 385 //! __Unpin<'__pin>: ::core::marker::Unpin, 386 //! {} 387 //! // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to 388 //! // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like 389 //! // before, instead we implement `Drop` here and delegate to `PinnedDrop`. 390 //! impl ::core::ops::Drop for Foo { 391 //! fn drop(&mut self) { 392 //! // Since we are getting dropped, no one else has a reference to `self` and thus we 393 //! // can assume that we never move. 394 //! let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) }; 395 //! // Create the unsafe token that proves that we are inside of a destructor, this 396 //! // type is only allowed to be created in a destructor. 397 //! let token = unsafe { ::pin_init::__internal::OnlyCallFromDrop::new() }; 398 //! ::pin_init::PinnedDrop::drop(pinned, token); 399 //! } 400 //! } 401 //! }; 402 //! ``` 403 //! 404 //! ## `#[pinned_drop]` on `impl PinnedDrop for Foo` 405 //! 406 //! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an 407 //! extra parameter that should not be used at all. The macro hides that parameter. 408 //! 409 //! Here is the `PinnedDrop` impl for `Foo`: 410 //! 411 //! ```rust,ignore 412 //! #[pinned_drop] 413 //! impl PinnedDrop for Foo { 414 //! fn drop(self: Pin<&mut Self>) { 415 //! println!("{self:p} is getting dropped."); 416 //! } 417 //! } 418 //! ``` 419 //! 420 //! This expands to the following code: 421 //! 422 //! ```rust,ignore 423 //! // `unsafe`, full path and the token parameter are added, everything else stays the same. 424 //! unsafe impl ::pin_init::PinnedDrop for Foo { 425 //! fn drop(self: Pin<&mut Self>, _: ::pin_init::__internal::OnlyCallFromDrop) { 426 //! println!("{self:p} is getting dropped."); 427 //! } 428 //! } 429 //! ``` 430 //! 431 //! ## `pin_init!` on `Foo` 432 //! 433 //! Since we already took a look at `pin_init!` on `Bar`, this section will only show the expansion 434 //! of `pin_init!` on `Foo`: 435 //! 436 //! ```rust,ignore 437 //! let a = 42; 438 //! let initializer = pin_init!(Foo { 439 //! a, 440 //! b <- Bar::new(36), 441 //! }); 442 //! ``` 443 //! 444 //! This expands to the following code: 445 //! 446 //! ```rust,ignore 447 //! let a = 42; 448 //! let initializer = { 449 //! struct __InitOk; 450 //! let data = unsafe { 451 //! use ::pin_init::__internal::HasPinData; 452 //! Foo::__pin_data() 453 //! }; 454 //! let init = ::pin_init::__internal::PinData::make_closure::< 455 //! _, 456 //! __InitOk, 457 //! ::core::convert::Infallible, 458 //! >(data, move |slot| { 459 //! { 460 //! struct __InitOk; 461 //! { 462 //! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) }; 463 //! } 464 //! let __a_guard = unsafe { 465 //! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a)) 466 //! }; 467 //! let init = Bar::new(36); 468 //! unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? }; 469 //! let __b_guard = unsafe { 470 //! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b)) 471 //! }; 472 //! ::core::mem::forget(__b_guard); 473 //! ::core::mem::forget(__a_guard); 474 //! #[allow(unreachable_code, clippy::diverging_sub_expression)] 475 //! let _ = || { 476 //! unsafe { 477 //! ::core::ptr::write( 478 //! slot, 479 //! Foo { 480 //! a: ::core::panic!(), 481 //! b: ::core::panic!(), 482 //! }, 483 //! ); 484 //! }; 485 //! }; 486 //! } 487 //! Ok(__InitOk) 488 //! }); 489 //! let init = move | 490 //! slot, 491 //! | -> ::core::result::Result<(), ::core::convert::Infallible> { 492 //! init(slot).map(|__InitOk| ()) 493 //! }; 494 //! let init = unsafe { 495 //! ::pin_init::pin_init_from_closure::<_, ::core::convert::Infallible>(init) 496 //! }; 497 //! init 498 //! }; 499 //! ``` 500 501 #[cfg(kernel)] 502 pub use ::macros::paste; 503 #[cfg(not(kernel))] 504 pub use ::paste::paste; 505 506 /// Creates a `unsafe impl<...> PinnedDrop for $type` block. 507 /// 508 /// See [`PinnedDrop`] for more information. 509 #[doc(hidden)] 510 #[macro_export] 511 macro_rules! __pinned_drop { 512 ( 513 @impl_sig($($impl_sig:tt)*), 514 @impl_body( 515 $(#[$($attr:tt)*])* 516 fn drop($($sig:tt)*) { 517 $($inner:tt)* 518 } 519 ), 520 ) => { 521 // SAFETY: TODO. 522 unsafe $($impl_sig)* { 523 // Inherit all attributes and the type/ident tokens for the signature. 524 $(#[$($attr)*])* 525 fn drop($($sig)*, _: $crate::__internal::OnlyCallFromDrop) { 526 $($inner)* 527 } 528 } 529 } 530 } 531 532 /// This macro first parses the struct definition such that it separates pinned and not pinned 533 /// fields. Afterwards it declares the struct and implement the `PinData` trait safely. 534 #[doc(hidden)] 535 #[macro_export] 536 macro_rules! __pin_data { 537 // Proc-macro entry point, this is supplied by the proc-macro pre-parsing. 538 (parse_input: 539 @args($($pinned_drop:ident)?), 540 @sig( 541 $(#[$($struct_attr:tt)*])* 542 $vis:vis struct $name:ident 543 $(where $($whr:tt)*)? 544 ), 545 @impl_generics($($impl_generics:tt)*), 546 @ty_generics($($ty_generics:tt)*), 547 @decl_generics($($decl_generics:tt)*), 548 @body({ $($fields:tt)* }), 549 ) => { 550 // We now use token munching to iterate through all of the fields. While doing this we 551 // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user 552 // wants these to be structurally pinned. The rest of the fields are the 553 // 'not pinned fields'. Additionally we collect all fields, since we need them in the right 554 // order to declare the struct. 555 // 556 // In this call we also put some explaining comments for the parameters. 557 $crate::__pin_data!(find_pinned_fields: 558 // Attributes on the struct itself, these will just be propagated to be put onto the 559 // struct definition. 560 @struct_attrs($(#[$($struct_attr)*])*), 561 // The visibility of the struct. 562 @vis($vis), 563 // The name of the struct. 564 @name($name), 565 // The 'impl generics', the generics that will need to be specified on the struct inside 566 // of an `impl<$ty_generics>` block. 567 @impl_generics($($impl_generics)*), 568 // The 'ty generics', the generics that will need to be specified on the impl blocks. 569 @ty_generics($($ty_generics)*), 570 // The 'decl generics', the generics that need to be specified on the struct 571 // definition. 572 @decl_generics($($decl_generics)*), 573 // The where clause of any impl block and the declaration. 574 @where($($($whr)*)?), 575 // The remaining fields tokens that need to be processed. 576 // We add a `,` at the end to ensure correct parsing. 577 @fields_munch($($fields)* ,), 578 // The pinned fields. 579 @pinned(), 580 // The not pinned fields. 581 @not_pinned(), 582 // All fields. 583 @fields(), 584 // The accumulator containing all attributes already parsed. 585 @accum(), 586 // Contains `yes` or `` to indicate if `#[pin]` was found on the current field. 587 @is_pinned(), 588 // The proc-macro argument, this should be `PinnedDrop` or ``. 589 @pinned_drop($($pinned_drop)?), 590 ); 591 }; 592 (find_pinned_fields: 593 @struct_attrs($($struct_attrs:tt)*), 594 @vis($vis:vis), 595 @name($name:ident), 596 @impl_generics($($impl_generics:tt)*), 597 @ty_generics($($ty_generics:tt)*), 598 @decl_generics($($decl_generics:tt)*), 599 @where($($whr:tt)*), 600 // We found a PhantomPinned field, this should generally be pinned! 601 @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*), 602 @pinned($($pinned:tt)*), 603 @not_pinned($($not_pinned:tt)*), 604 @fields($($fields:tt)*), 605 @accum($($accum:tt)*), 606 // This field is not pinned. 607 @is_pinned(), 608 @pinned_drop($($pinned_drop:ident)?), 609 ) => { 610 ::core::compile_error!(concat!( 611 "The field `", 612 stringify!($field), 613 "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.", 614 )); 615 $crate::__pin_data!(find_pinned_fields: 616 @struct_attrs($($struct_attrs)*), 617 @vis($vis), 618 @name($name), 619 @impl_generics($($impl_generics)*), 620 @ty_generics($($ty_generics)*), 621 @decl_generics($($decl_generics)*), 622 @where($($whr)*), 623 @fields_munch($($rest)*), 624 @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,), 625 @not_pinned($($not_pinned)*), 626 @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,), 627 @accum(), 628 @is_pinned(), 629 @pinned_drop($($pinned_drop)?), 630 ); 631 }; 632 (find_pinned_fields: 633 @struct_attrs($($struct_attrs:tt)*), 634 @vis($vis:vis), 635 @name($name:ident), 636 @impl_generics($($impl_generics:tt)*), 637 @ty_generics($($ty_generics:tt)*), 638 @decl_generics($($decl_generics:tt)*), 639 @where($($whr:tt)*), 640 // We reached the field declaration. 641 @fields_munch($field:ident : $type:ty, $($rest:tt)*), 642 @pinned($($pinned:tt)*), 643 @not_pinned($($not_pinned:tt)*), 644 @fields($($fields:tt)*), 645 @accum($($accum:tt)*), 646 // This field is pinned. 647 @is_pinned(yes), 648 @pinned_drop($($pinned_drop:ident)?), 649 ) => { 650 $crate::__pin_data!(find_pinned_fields: 651 @struct_attrs($($struct_attrs)*), 652 @vis($vis), 653 @name($name), 654 @impl_generics($($impl_generics)*), 655 @ty_generics($($ty_generics)*), 656 @decl_generics($($decl_generics)*), 657 @where($($whr)*), 658 @fields_munch($($rest)*), 659 @pinned($($pinned)* $($accum)* $field: $type,), 660 @not_pinned($($not_pinned)*), 661 @fields($($fields)* $($accum)* $field: $type,), 662 @accum(), 663 @is_pinned(), 664 @pinned_drop($($pinned_drop)?), 665 ); 666 }; 667 (find_pinned_fields: 668 @struct_attrs($($struct_attrs:tt)*), 669 @vis($vis:vis), 670 @name($name:ident), 671 @impl_generics($($impl_generics:tt)*), 672 @ty_generics($($ty_generics:tt)*), 673 @decl_generics($($decl_generics:tt)*), 674 @where($($whr:tt)*), 675 // We reached the field declaration. 676 @fields_munch($field:ident : $type:ty, $($rest:tt)*), 677 @pinned($($pinned:tt)*), 678 @not_pinned($($not_pinned:tt)*), 679 @fields($($fields:tt)*), 680 @accum($($accum:tt)*), 681 // This field is not pinned. 682 @is_pinned(), 683 @pinned_drop($($pinned_drop:ident)?), 684 ) => { 685 $crate::__pin_data!(find_pinned_fields: 686 @struct_attrs($($struct_attrs)*), 687 @vis($vis), 688 @name($name), 689 @impl_generics($($impl_generics)*), 690 @ty_generics($($ty_generics)*), 691 @decl_generics($($decl_generics)*), 692 @where($($whr)*), 693 @fields_munch($($rest)*), 694 @pinned($($pinned)*), 695 @not_pinned($($not_pinned)* $($accum)* $field: $type,), 696 @fields($($fields)* $($accum)* $field: $type,), 697 @accum(), 698 @is_pinned(), 699 @pinned_drop($($pinned_drop)?), 700 ); 701 }; 702 (find_pinned_fields: 703 @struct_attrs($($struct_attrs:tt)*), 704 @vis($vis:vis), 705 @name($name:ident), 706 @impl_generics($($impl_generics:tt)*), 707 @ty_generics($($ty_generics:tt)*), 708 @decl_generics($($decl_generics:tt)*), 709 @where($($whr:tt)*), 710 // We found the `#[pin]` attr. 711 @fields_munch(#[pin] $($rest:tt)*), 712 @pinned($($pinned:tt)*), 713 @not_pinned($($not_pinned:tt)*), 714 @fields($($fields:tt)*), 715 @accum($($accum:tt)*), 716 @is_pinned($($is_pinned:ident)?), 717 @pinned_drop($($pinned_drop:ident)?), 718 ) => { 719 $crate::__pin_data!(find_pinned_fields: 720 @struct_attrs($($struct_attrs)*), 721 @vis($vis), 722 @name($name), 723 @impl_generics($($impl_generics)*), 724 @ty_generics($($ty_generics)*), 725 @decl_generics($($decl_generics)*), 726 @where($($whr)*), 727 @fields_munch($($rest)*), 728 // We do not include `#[pin]` in the list of attributes, since it is not actually an 729 // attribute that is defined somewhere. 730 @pinned($($pinned)*), 731 @not_pinned($($not_pinned)*), 732 @fields($($fields)*), 733 @accum($($accum)*), 734 // Set this to `yes`. 735 @is_pinned(yes), 736 @pinned_drop($($pinned_drop)?), 737 ); 738 }; 739 (find_pinned_fields: 740 @struct_attrs($($struct_attrs:tt)*), 741 @vis($vis:vis), 742 @name($name:ident), 743 @impl_generics($($impl_generics:tt)*), 744 @ty_generics($($ty_generics:tt)*), 745 @decl_generics($($decl_generics:tt)*), 746 @where($($whr:tt)*), 747 // We reached the field declaration with visibility, for simplicity we only munch the 748 // visibility and put it into `$accum`. 749 @fields_munch($fvis:vis $field:ident $($rest:tt)*), 750 @pinned($($pinned:tt)*), 751 @not_pinned($($not_pinned:tt)*), 752 @fields($($fields:tt)*), 753 @accum($($accum:tt)*), 754 @is_pinned($($is_pinned:ident)?), 755 @pinned_drop($($pinned_drop:ident)?), 756 ) => { 757 $crate::__pin_data!(find_pinned_fields: 758 @struct_attrs($($struct_attrs)*), 759 @vis($vis), 760 @name($name), 761 @impl_generics($($impl_generics)*), 762 @ty_generics($($ty_generics)*), 763 @decl_generics($($decl_generics)*), 764 @where($($whr)*), 765 @fields_munch($field $($rest)*), 766 @pinned($($pinned)*), 767 @not_pinned($($not_pinned)*), 768 @fields($($fields)*), 769 @accum($($accum)* $fvis), 770 @is_pinned($($is_pinned)?), 771 @pinned_drop($($pinned_drop)?), 772 ); 773 }; 774 (find_pinned_fields: 775 @struct_attrs($($struct_attrs:tt)*), 776 @vis($vis:vis), 777 @name($name:ident), 778 @impl_generics($($impl_generics:tt)*), 779 @ty_generics($($ty_generics:tt)*), 780 @decl_generics($($decl_generics:tt)*), 781 @where($($whr:tt)*), 782 // Some other attribute, just put it into `$accum`. 783 @fields_munch(#[$($attr:tt)*] $($rest:tt)*), 784 @pinned($($pinned:tt)*), 785 @not_pinned($($not_pinned:tt)*), 786 @fields($($fields:tt)*), 787 @accum($($accum:tt)*), 788 @is_pinned($($is_pinned:ident)?), 789 @pinned_drop($($pinned_drop:ident)?), 790 ) => { 791 $crate::__pin_data!(find_pinned_fields: 792 @struct_attrs($($struct_attrs)*), 793 @vis($vis), 794 @name($name), 795 @impl_generics($($impl_generics)*), 796 @ty_generics($($ty_generics)*), 797 @decl_generics($($decl_generics)*), 798 @where($($whr)*), 799 @fields_munch($($rest)*), 800 @pinned($($pinned)*), 801 @not_pinned($($not_pinned)*), 802 @fields($($fields)*), 803 @accum($($accum)* #[$($attr)*]), 804 @is_pinned($($is_pinned)?), 805 @pinned_drop($($pinned_drop)?), 806 ); 807 }; 808 (find_pinned_fields: 809 @struct_attrs($($struct_attrs:tt)*), 810 @vis($vis:vis), 811 @name($name:ident), 812 @impl_generics($($impl_generics:tt)*), 813 @ty_generics($($ty_generics:tt)*), 814 @decl_generics($($decl_generics:tt)*), 815 @where($($whr:tt)*), 816 // We reached the end of the fields, plus an optional additional comma, since we added one 817 // before and the user is also allowed to put a trailing comma. 818 @fields_munch($(,)?), 819 @pinned($($pinned:tt)*), 820 @not_pinned($($not_pinned:tt)*), 821 @fields($($fields:tt)*), 822 @accum(), 823 @is_pinned(), 824 @pinned_drop($($pinned_drop:ident)?), 825 ) => { 826 // Declare the struct with all fields in the correct order. 827 $($struct_attrs)* 828 $vis struct $name <$($decl_generics)*> 829 where $($whr)* 830 { 831 $($fields)* 832 } 833 834 $crate::__pin_data!(make_pin_projections: 835 @vis($vis), 836 @name($name), 837 @impl_generics($($impl_generics)*), 838 @ty_generics($($ty_generics)*), 839 @decl_generics($($decl_generics)*), 840 @where($($whr)*), 841 @pinned($($pinned)*), 842 @not_pinned($($not_pinned)*), 843 ); 844 845 // We put the rest into this const item, because it then will not be accessible to anything 846 // outside. 847 const _: () = { 848 // We declare this struct which will host all of the projection function for our type. 849 // it will be invariant over all generic parameters which are inherited from the 850 // struct. 851 $vis struct __ThePinData<$($impl_generics)*> 852 where $($whr)* 853 { 854 __phantom: ::core::marker::PhantomData< 855 fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*> 856 >, 857 } 858 859 impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*> 860 where $($whr)* 861 { 862 fn clone(&self) -> Self { *self } 863 } 864 865 impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*> 866 where $($whr)* 867 {} 868 869 // Make all projection functions. 870 $crate::__pin_data!(make_pin_data: 871 @pin_data(__ThePinData), 872 @impl_generics($($impl_generics)*), 873 @ty_generics($($ty_generics)*), 874 @where($($whr)*), 875 @pinned($($pinned)*), 876 @not_pinned($($not_pinned)*), 877 ); 878 879 // SAFETY: We have added the correct projection functions above to `__ThePinData` and 880 // we also use the least restrictive generics possible. 881 unsafe impl<$($impl_generics)*> 882 $crate::__internal::HasPinData for $name<$($ty_generics)*> 883 where $($whr)* 884 { 885 type PinData = __ThePinData<$($ty_generics)*>; 886 887 unsafe fn __pin_data() -> Self::PinData { 888 __ThePinData { __phantom: ::core::marker::PhantomData } 889 } 890 } 891 892 // SAFETY: TODO. 893 unsafe impl<$($impl_generics)*> 894 $crate::__internal::PinData for __ThePinData<$($ty_generics)*> 895 where $($whr)* 896 { 897 type Datee = $name<$($ty_generics)*>; 898 } 899 900 // This struct will be used for the unpin analysis. Since only structurally pinned 901 // fields are relevant whether the struct should implement `Unpin`. 902 #[allow(dead_code)] 903 struct __Unpin <'__pin, $($impl_generics)*> 904 where $($whr)* 905 { 906 __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>, 907 __phantom: ::core::marker::PhantomData< 908 fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*> 909 >, 910 // Only the pinned fields. 911 $($pinned)* 912 } 913 914 #[doc(hidden)] 915 impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*> 916 where 917 __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin, 918 $($whr)* 919 {} 920 921 // We need to disallow normal `Drop` implementation, the exact behavior depends on 922 // whether `PinnedDrop` was specified as the parameter. 923 $crate::__pin_data!(drop_prevention: 924 @name($name), 925 @impl_generics($($impl_generics)*), 926 @ty_generics($($ty_generics)*), 927 @where($($whr)*), 928 @pinned_drop($($pinned_drop)?), 929 ); 930 }; 931 }; 932 // When no `PinnedDrop` was specified, then we have to prevent implementing drop. 933 (drop_prevention: 934 @name($name:ident), 935 @impl_generics($($impl_generics:tt)*), 936 @ty_generics($($ty_generics:tt)*), 937 @where($($whr:tt)*), 938 @pinned_drop(), 939 ) => { 940 // We prevent this by creating a trait that will be implemented for all types implementing 941 // `Drop`. Additionally we will implement this trait for the struct leading to a conflict, 942 // if it also implements `Drop` 943 trait MustNotImplDrop {} 944 #[expect(drop_bounds)] 945 impl<T: ::core::ops::Drop> MustNotImplDrop for T {} 946 impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*> 947 where $($whr)* {} 948 // We also take care to prevent users from writing a useless `PinnedDrop` implementation. 949 // They might implement `PinnedDrop` correctly for the struct, but forget to give 950 // `PinnedDrop` as the parameter to `#[pin_data]`. 951 #[expect(non_camel_case_types)] 952 trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} 953 impl<T: $crate::PinnedDrop> 954 UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} 955 impl<$($impl_generics)*> 956 UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*> 957 where $($whr)* {} 958 }; 959 // When `PinnedDrop` was specified we just implement `Drop` and delegate. 960 (drop_prevention: 961 @name($name:ident), 962 @impl_generics($($impl_generics:tt)*), 963 @ty_generics($($ty_generics:tt)*), 964 @where($($whr:tt)*), 965 @pinned_drop(PinnedDrop), 966 ) => { 967 impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*> 968 where $($whr)* 969 { 970 fn drop(&mut self) { 971 // SAFETY: Since this is a destructor, `self` will not move after this function 972 // terminates, since it is inaccessible. 973 let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) }; 974 // SAFETY: Since this is a drop function, we can create this token to call the 975 // pinned destructor of this type. 976 let token = unsafe { $crate::__internal::OnlyCallFromDrop::new() }; 977 $crate::PinnedDrop::drop(pinned, token); 978 } 979 } 980 }; 981 // If some other parameter was specified, we emit a readable error. 982 (drop_prevention: 983 @name($name:ident), 984 @impl_generics($($impl_generics:tt)*), 985 @ty_generics($($ty_generics:tt)*), 986 @where($($whr:tt)*), 987 @pinned_drop($($rest:tt)*), 988 ) => { 989 compile_error!( 990 "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.", 991 stringify!($($rest)*), 992 ); 993 }; 994 (make_pin_projections: 995 @vis($vis:vis), 996 @name($name:ident), 997 @impl_generics($($impl_generics:tt)*), 998 @ty_generics($($ty_generics:tt)*), 999 @decl_generics($($decl_generics:tt)*), 1000 @where($($whr:tt)*), 1001 @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?), 1002 @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?), 1003 ) => { 1004 $crate::macros::paste! { 1005 #[doc(hidden)] 1006 $vis struct [< $name Projection >] <'__pin, $($decl_generics)*> { 1007 $($(#[$($p_attr)*])* $pvis $p_field : ::core::pin::Pin<&'__pin mut $p_type>,)* 1008 $($(#[$($attr)*])* $fvis $field : &'__pin mut $type,)* 1009 ___pin_phantom_data: ::core::marker::PhantomData<&'__pin mut ()>, 1010 } 1011 1012 impl<$($impl_generics)*> $name<$($ty_generics)*> 1013 where $($whr)* 1014 { 1015 /// Pin-projects all fields of `Self`. 1016 /// 1017 /// These fields are structurally pinned: 1018 $(#[doc = ::core::concat!(" - `", ::core::stringify!($p_field), "`")])* 1019 /// 1020 /// These fields are **not** structurally pinned: 1021 $(#[doc = ::core::concat!(" - `", ::core::stringify!($field), "`")])* 1022 #[inline] 1023 $vis fn project<'__pin>( 1024 self: ::core::pin::Pin<&'__pin mut Self>, 1025 ) -> [< $name Projection >] <'__pin, $($ty_generics)*> { 1026 // SAFETY: we only give access to `&mut` for fields not structurally pinned. 1027 let this = unsafe { ::core::pin::Pin::get_unchecked_mut(self) }; 1028 [< $name Projection >] { 1029 $( 1030 // SAFETY: `$p_field` is structurally pinned. 1031 $(#[$($p_attr)*])* 1032 $p_field : unsafe { ::core::pin::Pin::new_unchecked(&mut this.$p_field) }, 1033 )* 1034 $( 1035 $(#[$($attr)*])* 1036 $field : &mut this.$field, 1037 )* 1038 ___pin_phantom_data: ::core::marker::PhantomData, 1039 } 1040 } 1041 } 1042 } 1043 }; 1044 (make_pin_data: 1045 @pin_data($pin_data:ident), 1046 @impl_generics($($impl_generics:tt)*), 1047 @ty_generics($($ty_generics:tt)*), 1048 @where($($whr:tt)*), 1049 @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?), 1050 @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?), 1051 ) => { 1052 $crate::macros::paste! { 1053 // For every field, we create a projection function according to its projection type. If a 1054 // field is structurally pinned, then it must be initialized via `PinInit`, if it is not 1055 // structurally pinned, then it can be initialized via `Init`. 1056 // 1057 // The functions are `unsafe` to prevent accidentally calling them. 1058 #[allow(dead_code)] 1059 #[expect(clippy::missing_safety_doc)] 1060 impl<$($impl_generics)*> $pin_data<$($ty_generics)*> 1061 where $($whr)* 1062 { 1063 $( 1064 $(#[$($p_attr)*])* 1065 $pvis unsafe fn $p_field<E>( 1066 self, 1067 slot: *mut $p_type, 1068 init: impl $crate::PinInit<$p_type, E>, 1069 ) -> ::core::result::Result<(), E> { 1070 // SAFETY: TODO. 1071 unsafe { $crate::PinInit::__pinned_init(init, slot) } 1072 } 1073 1074 $(#[$($p_attr)*])* 1075 $pvis unsafe fn [<__project_ $p_field>]<'__slot>( 1076 self, 1077 slot: &'__slot mut $p_type, 1078 ) -> ::core::pin::Pin<&'__slot mut $p_type> { 1079 ::core::pin::Pin::new_unchecked(slot) 1080 } 1081 )* 1082 $( 1083 $(#[$($attr)*])* 1084 $fvis unsafe fn $field<E>( 1085 self, 1086 slot: *mut $type, 1087 init: impl $crate::Init<$type, E>, 1088 ) -> ::core::result::Result<(), E> { 1089 // SAFETY: TODO. 1090 unsafe { $crate::Init::__init(init, slot) } 1091 } 1092 1093 $(#[$($attr)*])* 1094 $fvis unsafe fn [<__project_ $field>]<'__slot>( 1095 self, 1096 slot: &'__slot mut $type, 1097 ) -> &'__slot mut $type { 1098 slot 1099 } 1100 )* 1101 } 1102 } 1103 }; 1104 } 1105 1106 /// The internal init macro. Do not call manually! 1107 /// 1108 /// This is called by the `{try_}{pin_}init!` macros with various inputs. 1109 /// 1110 /// This macro has multiple internal call configurations, these are always the very first ident: 1111 /// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros. 1112 /// - `with_update_parsed`: when the `..Zeroable::init_zeroed()` syntax has been handled. 1113 /// - `init_slot`: recursively creates the code that initializes all fields in `slot`. 1114 /// - `make_initializer`: recursively create the struct initializer that guarantees that every 1115 /// field has been initialized exactly once. 1116 #[doc(hidden)] 1117 #[macro_export] 1118 macro_rules! __init_internal { 1119 ( 1120 @this($($this:ident)?), 1121 @typ($t:path), 1122 @fields($($fields:tt)*), 1123 @error($err:ty), 1124 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` 1125 // case. 1126 @data($data:ident, $($use_data:ident)?), 1127 // `HasPinData` or `HasInitData`. 1128 @has_data($has_data:ident, $get_data:ident), 1129 // `pin_init_from_closure` or `init_from_closure`. 1130 @construct_closure($construct_closure:ident), 1131 @munch_fields(), 1132 ) => { 1133 $crate::__init_internal!(with_update_parsed: 1134 @this($($this)?), 1135 @typ($t), 1136 @fields($($fields)*), 1137 @error($err), 1138 @data($data, $($use_data)?), 1139 @has_data($has_data, $get_data), 1140 @construct_closure($construct_closure), 1141 @init_zeroed(), // Nothing means default behavior. 1142 ) 1143 }; 1144 ( 1145 @this($($this:ident)?), 1146 @typ($t:path), 1147 @fields($($fields:tt)*), 1148 @error($err:ty), 1149 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` 1150 // case. 1151 @data($data:ident, $($use_data:ident)?), 1152 // `HasPinData` or `HasInitData`. 1153 @has_data($has_data:ident, $get_data:ident), 1154 // `pin_init_from_closure` or `init_from_closure`. 1155 @construct_closure($construct_closure:ident), 1156 @munch_fields(..Zeroable::init_zeroed()), 1157 ) => { 1158 $crate::__init_internal!(with_update_parsed: 1159 @this($($this)?), 1160 @typ($t), 1161 @fields($($fields)*), 1162 @error($err), 1163 @data($data, $($use_data)?), 1164 @has_data($has_data, $get_data), 1165 @construct_closure($construct_closure), 1166 @init_zeroed(()), // `()` means zero all fields not mentioned. 1167 ) 1168 }; 1169 ( 1170 @this($($this:ident)?), 1171 @typ($t:path), 1172 @fields($($fields:tt)*), 1173 @error($err:ty), 1174 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` 1175 // case. 1176 @data($data:ident, $($use_data:ident)?), 1177 // `HasPinData` or `HasInitData`. 1178 @has_data($has_data:ident, $get_data:ident), 1179 // `pin_init_from_closure` or `init_from_closure`. 1180 @construct_closure($construct_closure:ident), 1181 @munch_fields($ignore:tt $($rest:tt)*), 1182 ) => { 1183 $crate::__init_internal!( 1184 @this($($this)?), 1185 @typ($t), 1186 @fields($($fields)*), 1187 @error($err), 1188 @data($data, $($use_data)?), 1189 @has_data($has_data, $get_data), 1190 @construct_closure($construct_closure), 1191 @munch_fields($($rest)*), 1192 ) 1193 }; 1194 (with_update_parsed: 1195 @this($($this:ident)?), 1196 @typ($t:path), 1197 @fields($($fields:tt)*), 1198 @error($err:ty), 1199 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` 1200 // case. 1201 @data($data:ident, $($use_data:ident)?), 1202 // `HasPinData` or `HasInitData`. 1203 @has_data($has_data:ident, $get_data:ident), 1204 // `pin_init_from_closure` or `init_from_closure`. 1205 @construct_closure($construct_closure:ident), 1206 @init_zeroed($($init_zeroed:expr)?), 1207 ) => {{ 1208 // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return 1209 // type and shadow it later when we insert the arbitrary user code. That way there will be 1210 // no possibility of returning without `unsafe`. 1211 struct __InitOk; 1212 // Get the data about fields from the supplied type. 1213 // 1214 // SAFETY: TODO. 1215 let data = unsafe { 1216 use $crate::__internal::$has_data; 1217 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal 1218 // information that is associated to already parsed fragments, so a path fragment 1219 // cannot be used in this position. Doing the retokenization results in valid rust 1220 // code. 1221 $crate::macros::paste!($t::$get_data()) 1222 }; 1223 // Ensure that `data` really is of type `$data` and help with type inference: 1224 let init = $crate::__internal::$data::make_closure::<_, __InitOk, $err>( 1225 data, 1226 move |slot| { 1227 { 1228 // Shadow the structure so it cannot be used to return early. 1229 struct __InitOk; 1230 // If `$init_zeroed` is present we should zero the slot now and not emit an 1231 // error when fields are missing (since they will be zeroed). We also have to 1232 // check that the type actually implements `Zeroable`. 1233 $({ 1234 fn assert_zeroable<T: $crate::Zeroable>(_: *mut T) {} 1235 // Ensure that the struct is indeed `Zeroable`. 1236 assert_zeroable(slot); 1237 // SAFETY: The type implements `Zeroable` by the check above. 1238 unsafe { ::core::ptr::write_bytes(slot, 0, 1) }; 1239 $init_zeroed // This will be `()` if set. 1240 })? 1241 // Create the `this` so it can be referenced by the user inside of the 1242 // expressions creating the individual fields. 1243 $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? 1244 // Initialize every field. 1245 $crate::__init_internal!(init_slot($($use_data)?): 1246 @data(data), 1247 @slot(slot), 1248 @guards(), 1249 @munch_fields($($fields)*,), 1250 ); 1251 // We use unreachable code to ensure that all fields have been mentioned exactly 1252 // once, this struct initializer will still be type-checked and complain with a 1253 // very natural error message if a field is forgotten/mentioned more than once. 1254 #[allow(unreachable_code, clippy::diverging_sub_expression)] 1255 let _ = || { 1256 $crate::__init_internal!(make_initializer: 1257 @slot(slot), 1258 @type_name($t), 1259 @munch_fields($($fields)*,), 1260 @acc(), 1261 ); 1262 }; 1263 } 1264 Ok(__InitOk) 1265 } 1266 ); 1267 let init = move |slot| -> ::core::result::Result<(), $err> { 1268 init(slot).map(|__InitOk| ()) 1269 }; 1270 // SAFETY: TODO. 1271 let init = unsafe { $crate::$construct_closure::<_, $err>(init) }; 1272 init 1273 }}; 1274 (init_slot($($use_data:ident)?): 1275 @data($data:ident), 1276 @slot($slot:ident), 1277 @guards($($guards:ident,)*), 1278 @munch_fields($(..Zeroable::init_zeroed())? $(,)?), 1279 ) => { 1280 // Endpoint of munching, no fields are left. If execution reaches this point, all fields 1281 // have been initialized. Therefore we can now dismiss the guards by forgetting them. 1282 $(::core::mem::forget($guards);)* 1283 }; 1284 (init_slot($($use_data:ident)?): 1285 @data($data:ident), 1286 @slot($slot:ident), 1287 @guards($($guards:ident,)*), 1288 // arbitrary code block 1289 @munch_fields(_: { $($code:tt)* }, $($rest:tt)*), 1290 ) => { 1291 { $($code)* } 1292 $crate::__init_internal!(init_slot($($use_data)?): 1293 @data($data), 1294 @slot($slot), 1295 @guards($($guards,)*), 1296 @munch_fields($($rest)*), 1297 ); 1298 }; 1299 (init_slot($use_data:ident): // `use_data` is present, so we use the `data` to init fields. 1300 @data($data:ident), 1301 @slot($slot:ident), 1302 @guards($($guards:ident,)*), 1303 // In-place initialization syntax. 1304 @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 1305 ) => { 1306 let init = $val; 1307 // Call the initializer. 1308 // 1309 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we 1310 // return when an error/panic occurs. 1311 // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`. 1312 unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? }; 1313 // SAFETY: 1314 // - the project function does the correct field projection, 1315 // - the field has been initialized, 1316 // - the reference is only valid until the end of the initializer. 1317 #[allow(unused_variables)] 1318 let $field = $crate::macros::paste!(unsafe { $data.[< __project_ $field >](&mut (*$slot).$field) }); 1319 1320 // Create the drop guard: 1321 // 1322 // We rely on macro hygiene to make it impossible for users to access this local variable. 1323 // We use `paste!` to create new hygiene for `$field`. 1324 $crate::macros::paste! { 1325 // SAFETY: We forget the guard later when initialization has succeeded. 1326 let [< __ $field _guard >] = unsafe { 1327 $crate::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) 1328 }; 1329 1330 $crate::__init_internal!(init_slot($use_data): 1331 @data($data), 1332 @slot($slot), 1333 @guards([< __ $field _guard >], $($guards,)*), 1334 @munch_fields($($rest)*), 1335 ); 1336 } 1337 }; 1338 (init_slot(): // No `use_data`, so we use `Init::__init` directly. 1339 @data($data:ident), 1340 @slot($slot:ident), 1341 @guards($($guards:ident,)*), 1342 // In-place initialization syntax. 1343 @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 1344 ) => { 1345 let init = $val; 1346 // Call the initializer. 1347 // 1348 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we 1349 // return when an error/panic occurs. 1350 unsafe { $crate::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? }; 1351 1352 // SAFETY: 1353 // - the field is not structurally pinned, since the line above must compile, 1354 // - the field has been initialized, 1355 // - the reference is only valid until the end of the initializer. 1356 #[allow(unused_variables)] 1357 let $field = unsafe { &mut (*$slot).$field }; 1358 1359 // Create the drop guard: 1360 // 1361 // We rely on macro hygiene to make it impossible for users to access this local variable. 1362 // We use `paste!` to create new hygiene for `$field`. 1363 $crate::macros::paste! { 1364 // SAFETY: We forget the guard later when initialization has succeeded. 1365 let [< __ $field _guard >] = unsafe { 1366 $crate::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) 1367 }; 1368 1369 $crate::__init_internal!(init_slot(): 1370 @data($data), 1371 @slot($slot), 1372 @guards([< __ $field _guard >], $($guards,)*), 1373 @munch_fields($($rest)*), 1374 ); 1375 } 1376 }; 1377 (init_slot(): // No `use_data`, so all fields are not structurally pinned 1378 @data($data:ident), 1379 @slot($slot:ident), 1380 @guards($($guards:ident,)*), 1381 // Init by-value. 1382 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), 1383 ) => { 1384 { 1385 $(let $field = $val;)? 1386 // Initialize the field. 1387 // 1388 // SAFETY: The memory at `slot` is uninitialized. 1389 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; 1390 } 1391 1392 #[allow(unused_variables)] 1393 // SAFETY: 1394 // - the field is not structurally pinned, since no `use_data` was required to create this 1395 // initializer, 1396 // - the field has been initialized, 1397 // - the reference is only valid until the end of the initializer. 1398 let $field = unsafe { &mut (*$slot).$field }; 1399 1400 // Create the drop guard: 1401 // 1402 // We rely on macro hygiene to make it impossible for users to access this local variable. 1403 // We use `paste!` to create new hygiene for `$field`. 1404 $crate::macros::paste! { 1405 // SAFETY: We forget the guard later when initialization has succeeded. 1406 let [< __ $field _guard >] = unsafe { 1407 $crate::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) 1408 }; 1409 1410 $crate::__init_internal!(init_slot(): 1411 @data($data), 1412 @slot($slot), 1413 @guards([< __ $field _guard >], $($guards,)*), 1414 @munch_fields($($rest)*), 1415 ); 1416 } 1417 }; 1418 (init_slot($use_data:ident): 1419 @data($data:ident), 1420 @slot($slot:ident), 1421 @guards($($guards:ident,)*), 1422 // Init by-value. 1423 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), 1424 ) => { 1425 { 1426 $(let $field = $val;)? 1427 // Initialize the field. 1428 // 1429 // SAFETY: The memory at `slot` is uninitialized. 1430 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; 1431 } 1432 // SAFETY: 1433 // - the project function does the correct field projection, 1434 // - the field has been initialized, 1435 // - the reference is only valid until the end of the initializer. 1436 #[allow(unused_variables)] 1437 let $field = $crate::macros::paste!(unsafe { $data.[< __project_ $field >](&mut (*$slot).$field) }); 1438 1439 // Create the drop guard: 1440 // 1441 // We rely on macro hygiene to make it impossible for users to access this local variable. 1442 // We use `paste!` to create new hygiene for `$field`. 1443 $crate::macros::paste! { 1444 // SAFETY: We forget the guard later when initialization has succeeded. 1445 let [< __ $field _guard >] = unsafe { 1446 $crate::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) 1447 }; 1448 1449 $crate::__init_internal!(init_slot($use_data): 1450 @data($data), 1451 @slot($slot), 1452 @guards([< __ $field _guard >], $($guards,)*), 1453 @munch_fields($($rest)*), 1454 ); 1455 } 1456 }; 1457 (make_initializer: 1458 @slot($slot:ident), 1459 @type_name($t:path), 1460 @munch_fields(_: { $($code:tt)* }, $($rest:tt)*), 1461 @acc($($acc:tt)*), 1462 ) => { 1463 // code blocks are ignored for the initializer check 1464 $crate::__init_internal!(make_initializer: 1465 @slot($slot), 1466 @type_name($t), 1467 @munch_fields($($rest)*), 1468 @acc($($acc)*), 1469 ); 1470 }; 1471 (make_initializer: 1472 @slot($slot:ident), 1473 @type_name($t:path), 1474 @munch_fields(..Zeroable::init_zeroed() $(,)?), 1475 @acc($($acc:tt)*), 1476 ) => { 1477 // Endpoint, nothing more to munch, create the initializer. Since the users specified 1478 // `..Zeroable::init_zeroed()`, the slot will already have been zeroed and all field that have 1479 // not been overwritten are thus zero and initialized. We still check that all fields are 1480 // actually accessible by using the struct update syntax ourselves. 1481 // We are inside of a closure that is never executed and thus we can abuse `slot` to 1482 // get the correct type inference here: 1483 #[allow(unused_assignments)] 1484 unsafe { 1485 let mut zeroed = ::core::mem::zeroed(); 1486 // We have to use type inference here to make zeroed have the correct type. This does 1487 // not get executed, so it has no effect. 1488 ::core::ptr::write($slot, zeroed); 1489 zeroed = ::core::mem::zeroed(); 1490 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal 1491 // information that is associated to already parsed fragments, so a path fragment 1492 // cannot be used in this position. Doing the retokenization results in valid rust 1493 // code. 1494 $crate::macros::paste!( 1495 ::core::ptr::write($slot, $t { 1496 $($acc)* 1497 ..zeroed 1498 }); 1499 ); 1500 } 1501 }; 1502 (make_initializer: 1503 @slot($slot:ident), 1504 @type_name($t:path), 1505 @munch_fields($(,)?), 1506 @acc($($acc:tt)*), 1507 ) => { 1508 // Endpoint, nothing more to munch, create the initializer. 1509 // Since we are in the closure that is never called, this will never get executed. 1510 // We abuse `slot` to get the correct type inference here: 1511 // 1512 // SAFETY: TODO. 1513 unsafe { 1514 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal 1515 // information that is associated to already parsed fragments, so a path fragment 1516 // cannot be used in this position. Doing the retokenization results in valid rust 1517 // code. 1518 $crate::macros::paste!( 1519 ::core::ptr::write($slot, $t { 1520 $($acc)* 1521 }); 1522 ); 1523 } 1524 }; 1525 (make_initializer: 1526 @slot($slot:ident), 1527 @type_name($t:path), 1528 @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 1529 @acc($($acc:tt)*), 1530 ) => { 1531 $crate::__init_internal!(make_initializer: 1532 @slot($slot), 1533 @type_name($t), 1534 @munch_fields($($rest)*), 1535 @acc($($acc)* $field: ::core::panic!(),), 1536 ); 1537 }; 1538 (make_initializer: 1539 @slot($slot:ident), 1540 @type_name($t:path), 1541 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), 1542 @acc($($acc:tt)*), 1543 ) => { 1544 $crate::__init_internal!(make_initializer: 1545 @slot($slot), 1546 @type_name($t), 1547 @munch_fields($($rest)*), 1548 @acc($($acc)* $field: ::core::panic!(),), 1549 ); 1550 }; 1551 } 1552 1553 #[doc(hidden)] 1554 #[macro_export] 1555 macro_rules! __derive_zeroable { 1556 (parse_input: 1557 @sig( 1558 $(#[$($struct_attr:tt)*])* 1559 $vis:vis struct $name:ident 1560 $(where $($whr:tt)*)? 1561 ), 1562 @impl_generics($($impl_generics:tt)*), 1563 @ty_generics($($ty_generics:tt)*), 1564 @body({ 1565 $( 1566 $(#[$($field_attr:tt)*])* 1567 $field_vis:vis $field:ident : $field_ty:ty 1568 ),* $(,)? 1569 }), 1570 ) => { 1571 // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero. 1572 #[automatically_derived] 1573 unsafe impl<$($impl_generics)*> $crate::Zeroable for $name<$($ty_generics)*> 1574 where 1575 $($($whr)*)? 1576 {} 1577 const _: () = { 1578 fn assert_zeroable<T: ?::core::marker::Sized + $crate::Zeroable>() {} 1579 fn ensure_zeroable<$($impl_generics)*>() 1580 where $($($whr)*)? 1581 { 1582 $(assert_zeroable::<$field_ty>();)* 1583 } 1584 }; 1585 }; 1586 (parse_input: 1587 @sig( 1588 $(#[$($struct_attr:tt)*])* 1589 $vis:vis union $name:ident 1590 $(where $($whr:tt)*)? 1591 ), 1592 @impl_generics($($impl_generics:tt)*), 1593 @ty_generics($($ty_generics:tt)*), 1594 @body({ 1595 $( 1596 $(#[$($field_attr:tt)*])* 1597 $field_vis:vis $field:ident : $field_ty:ty 1598 ),* $(,)? 1599 }), 1600 ) => { 1601 // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero. 1602 #[automatically_derived] 1603 unsafe impl<$($impl_generics)*> $crate::Zeroable for $name<$($ty_generics)*> 1604 where 1605 $($($whr)*)? 1606 {} 1607 const _: () = { 1608 fn assert_zeroable<T: ?::core::marker::Sized + $crate::Zeroable>() {} 1609 fn ensure_zeroable<$($impl_generics)*>() 1610 where $($($whr)*)? 1611 { 1612 $(assert_zeroable::<$field_ty>();)* 1613 } 1614 }; 1615 }; 1616 } 1617 1618 #[doc(hidden)] 1619 #[macro_export] 1620 macro_rules! __maybe_derive_zeroable { 1621 (parse_input: 1622 @sig( 1623 $(#[$($struct_attr:tt)*])* 1624 $vis:vis struct $name:ident 1625 $(where $($whr:tt)*)? 1626 ), 1627 @impl_generics($($impl_generics:tt)*), 1628 @ty_generics($($ty_generics:tt)*), 1629 @body({ 1630 $( 1631 $(#[$($field_attr:tt)*])* 1632 $field_vis:vis $field:ident : $field_ty:ty 1633 ),* $(,)? 1634 }), 1635 ) => { 1636 // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero. 1637 #[automatically_derived] 1638 unsafe impl<$($impl_generics)*> $crate::Zeroable for $name<$($ty_generics)*> 1639 where 1640 $( 1641 // the `for<'__dummy>` HRTB makes this not error without the `trivial_bounds` 1642 // feature <https://github.com/rust-lang/rust/issues/48214#issuecomment-2557829956>. 1643 $field_ty: for<'__dummy> $crate::Zeroable, 1644 )* 1645 $($($whr)*)? 1646 {} 1647 }; 1648 (parse_input: 1649 @sig( 1650 $(#[$($struct_attr:tt)*])* 1651 $vis:vis union $name:ident 1652 $(where $($whr:tt)*)? 1653 ), 1654 @impl_generics($($impl_generics:tt)*), 1655 @ty_generics($($ty_generics:tt)*), 1656 @body({ 1657 $( 1658 $(#[$($field_attr:tt)*])* 1659 $field_vis:vis $field:ident : $field_ty:ty 1660 ),* $(,)? 1661 }), 1662 ) => { 1663 // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero. 1664 #[automatically_derived] 1665 unsafe impl<$($impl_generics)*> $crate::Zeroable for $name<$($ty_generics)*> 1666 where 1667 $( 1668 // the `for<'__dummy>` HRTB makes this not error without the `trivial_bounds` 1669 // feature <https://github.com/rust-lang/rust/issues/48214#issuecomment-2557829956>. 1670 $field_ty: for<'__dummy> $crate::Zeroable, 1671 )* 1672 $($($whr)*)? 1673 {} 1674 }; 1675 } 1676