xref: /linux/rust/kernel/sync/arc.rs (revision be239684b18e1cdcafcf8c7face4a2f562c745ad)
1 // SPDX-License-Identifier: GPL-2.0
2 
3 //! A reference-counted pointer.
4 //!
5 //! This module implements a way for users to create reference-counted objects and pointers to
6 //! them. Such a pointer automatically increments and decrements the count, and drops the
7 //! underlying object when it reaches zero. It is also safe to use concurrently from multiple
8 //! threads.
9 //!
10 //! It is different from the standard library's [`Arc`] in a few ways:
11 //! 1. It is backed by the kernel's `refcount_t` type.
12 //! 2. It does not support weak references, which allows it to be half the size.
13 //! 3. It saturates the reference count instead of aborting when it goes over a threshold.
14 //! 4. It does not provide a `get_mut` method, so the ref counted object is pinned.
15 //!
16 //! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html
17 
18 use crate::{
19     bindings,
20     error::{self, Error},
21     init::{self, InPlaceInit, Init, PinInit},
22     try_init,
23     types::{ForeignOwnable, Opaque},
24 };
25 use alloc::boxed::Box;
26 use core::{
27     alloc::{AllocError, Layout},
28     fmt,
29     marker::{PhantomData, Unsize},
30     mem::{ManuallyDrop, MaybeUninit},
31     ops::{Deref, DerefMut},
32     pin::Pin,
33     ptr::{NonNull, Pointee},
34 };
35 use macros::pin_data;
36 
37 mod std_vendor;
38 
39 /// A reference-counted pointer to an instance of `T`.
40 ///
41 /// The reference count is incremented when new instances of [`Arc`] are created, and decremented
42 /// when they are dropped. When the count reaches zero, the underlying `T` is also dropped.
43 ///
44 /// # Invariants
45 ///
46 /// The reference count on an instance of [`Arc`] is always non-zero.
47 /// The object pointed to by [`Arc`] is always pinned.
48 ///
49 /// # Examples
50 ///
51 /// ```
52 /// use kernel::sync::Arc;
53 ///
54 /// struct Example {
55 ///     a: u32,
56 ///     b: u32,
57 /// }
58 ///
59 /// // Create a ref-counted instance of `Example`.
60 /// let obj = Arc::try_new(Example { a: 10, b: 20 })?;
61 ///
62 /// // Get a new pointer to `obj` and increment the refcount.
63 /// let cloned = obj.clone();
64 ///
65 /// // Assert that both `obj` and `cloned` point to the same underlying object.
66 /// assert!(core::ptr::eq(&*obj, &*cloned));
67 ///
68 /// // Destroy `obj` and decrement its refcount.
69 /// drop(obj);
70 ///
71 /// // Check that the values are still accessible through `cloned`.
72 /// assert_eq!(cloned.a, 10);
73 /// assert_eq!(cloned.b, 20);
74 ///
75 /// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed.
76 /// # Ok::<(), Error>(())
77 /// ```
78 ///
79 /// Using `Arc<T>` as the type of `self`:
80 ///
81 /// ```
82 /// use kernel::sync::Arc;
83 ///
84 /// struct Example {
85 ///     a: u32,
86 ///     b: u32,
87 /// }
88 ///
89 /// impl Example {
90 ///     fn take_over(self: Arc<Self>) {
91 ///         // ...
92 ///     }
93 ///
94 ///     fn use_reference(self: &Arc<Self>) {
95 ///         // ...
96 ///     }
97 /// }
98 ///
99 /// let obj = Arc::try_new(Example { a: 10, b: 20 })?;
100 /// obj.use_reference();
101 /// obj.take_over();
102 /// # Ok::<(), Error>(())
103 /// ```
104 ///
105 /// Coercion from `Arc<Example>` to `Arc<dyn MyTrait>`:
106 ///
107 /// ```
108 /// use kernel::sync::{Arc, ArcBorrow};
109 ///
110 /// trait MyTrait {
111 ///     // Trait has a function whose `self` type is `Arc<Self>`.
112 ///     fn example1(self: Arc<Self>) {}
113 ///
114 ///     // Trait has a function whose `self` type is `ArcBorrow<'_, Self>`.
115 ///     fn example2(self: ArcBorrow<'_, Self>) {}
116 /// }
117 ///
118 /// struct Example;
119 /// impl MyTrait for Example {}
120 ///
121 /// // `obj` has type `Arc<Example>`.
122 /// let obj: Arc<Example> = Arc::try_new(Example)?;
123 ///
124 /// // `coerced` has type `Arc<dyn MyTrait>`.
125 /// let coerced: Arc<dyn MyTrait> = obj;
126 /// # Ok::<(), Error>(())
127 /// ```
128 pub struct Arc<T: ?Sized> {
129     ptr: NonNull<ArcInner<T>>,
130     _p: PhantomData<ArcInner<T>>,
131 }
132 
133 #[pin_data]
134 #[repr(C)]
135 struct ArcInner<T: ?Sized> {
136     refcount: Opaque<bindings::refcount_t>,
137     data: T,
138 }
139 
140 // This is to allow [`Arc`] (and variants) to be used as the type of `self`.
141 impl<T: ?Sized> core::ops::Receiver for Arc<T> {}
142 
143 // This is to allow coercion from `Arc<T>` to `Arc<U>` if `T` can be converted to the
144 // dynamically-sized type (DST) `U`.
145 impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {}
146 
147 // This is to allow `Arc<U>` to be dispatched on when `Arc<T>` can be coerced into `Arc<U>`.
148 impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<Arc<U>> for Arc<T> {}
149 
150 // SAFETY: It is safe to send `Arc<T>` to another thread when the underlying `T` is `Sync` because
151 // it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs
152 // `T` to be `Send` because any thread that has an `Arc<T>` may ultimately access `T` using a
153 // mutable reference when the reference count reaches zero and `T` is dropped.
154 unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {}
155 
156 // SAFETY: It is safe to send `&Arc<T>` to another thread when the underlying `T` is `Sync`
157 // because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally,
158 // it needs `T` to be `Send` because any thread that has a `&Arc<T>` may clone it and get an
159 // `Arc<T>` on that thread, so the thread may ultimately access `T` using a mutable reference when
160 // the reference count reaches zero and `T` is dropped.
161 unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {}
162 
163 impl<T> Arc<T> {
164     /// Constructs a new reference counted instance of `T`.
165     pub fn try_new(contents: T) -> Result<Self, AllocError> {
166         // INVARIANT: The refcount is initialised to a non-zero value.
167         let value = ArcInner {
168             // SAFETY: There are no safety requirements for this FFI call.
169             refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }),
170             data: contents,
171         };
172 
173         let inner = Box::try_new(value)?;
174 
175         // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new
176         // `Arc` object.
177         Ok(unsafe { Self::from_inner(Box::leak(inner).into()) })
178     }
179 
180     /// Use the given initializer to in-place initialize a `T`.
181     ///
182     /// If `T: !Unpin` it will not be able to move afterwards.
183     #[inline]
184     pub fn pin_init<E>(init: impl PinInit<T, E>) -> error::Result<Self>
185     where
186         Error: From<E>,
187     {
188         UniqueArc::pin_init(init).map(|u| u.into())
189     }
190 
191     /// Use the given initializer to in-place initialize a `T`.
192     ///
193     /// This is equivalent to [`Arc<T>::pin_init`], since an [`Arc`] is always pinned.
194     #[inline]
195     pub fn init<E>(init: impl Init<T, E>) -> error::Result<Self>
196     where
197         Error: From<E>,
198     {
199         UniqueArc::init(init).map(|u| u.into())
200     }
201 }
202 
203 impl<T: ?Sized> Arc<T> {
204     /// Constructs a new [`Arc`] from an existing [`ArcInner`].
205     ///
206     /// # Safety
207     ///
208     /// The caller must ensure that `inner` points to a valid location and has a non-zero reference
209     /// count, one of which will be owned by the new [`Arc`] instance.
210     unsafe fn from_inner(inner: NonNull<ArcInner<T>>) -> Self {
211         // INVARIANT: By the safety requirements, the invariants hold.
212         Arc {
213             ptr: inner,
214             _p: PhantomData,
215         }
216     }
217 
218     /// Convert the [`Arc`] into a raw pointer.
219     ///
220     /// The raw pointer has ownership of the refcount that this Arc object owned.
221     pub fn into_raw(self) -> *const T {
222         let ptr = self.ptr.as_ptr();
223         core::mem::forget(self);
224         // SAFETY: The pointer is valid.
225         unsafe { core::ptr::addr_of!((*ptr).data) }
226     }
227 
228     /// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`].
229     ///
230     /// # Safety
231     ///
232     /// `ptr` must have been returned by a previous call to [`Arc::into_raw`]. Additionally, it
233     /// must not be called more than once for each previous call to [`Arc::into_raw`].
234     pub unsafe fn from_raw(ptr: *const T) -> Self {
235         let refcount_layout = Layout::new::<bindings::refcount_t>();
236         // SAFETY: The caller guarantees that the pointer is valid.
237         let val_layout = Layout::for_value(unsafe { &*ptr });
238         // SAFETY: We're computing the layout of a real struct that existed when compiling this
239         // binary, so its layout is not so large that it can trigger arithmetic overflow.
240         let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 };
241 
242         let metadata: <T as Pointee>::Metadata = core::ptr::metadata(ptr);
243         // SAFETY: The metadata of `T` and `ArcInner<T>` is the same because `ArcInner` is a struct
244         // with `T` as its last field.
245         //
246         // This is documented at:
247         // <https://doc.rust-lang.org/std/ptr/trait.Pointee.html>.
248         let metadata: <ArcInner<T> as Pointee>::Metadata =
249             unsafe { core::mem::transmute_copy(&metadata) };
250         // SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the
251         // pointer, since it originates from a previous call to `Arc::into_raw` and is still valid.
252         let ptr = unsafe { (ptr as *mut u8).sub(val_offset) as *mut () };
253         let ptr = core::ptr::from_raw_parts_mut(ptr, metadata);
254 
255         // SAFETY: By the safety requirements we know that `ptr` came from `Arc::into_raw`, so the
256         // reference count held then will be owned by the new `Arc` object.
257         unsafe { Self::from_inner(NonNull::new_unchecked(ptr)) }
258     }
259 
260     /// Returns an [`ArcBorrow`] from the given [`Arc`].
261     ///
262     /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method
263     /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised.
264     #[inline]
265     pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> {
266         // SAFETY: The constraint that the lifetime of the shared reference must outlive that of
267         // the returned `ArcBorrow` ensures that the object remains alive and that no mutable
268         // reference can be created.
269         unsafe { ArcBorrow::new(self.ptr) }
270     }
271 
272     /// Compare whether two [`Arc`] pointers reference the same underlying object.
273     pub fn ptr_eq(this: &Self, other: &Self) -> bool {
274         core::ptr::eq(this.ptr.as_ptr(), other.ptr.as_ptr())
275     }
276 }
277 
278 impl<T: 'static> ForeignOwnable for Arc<T> {
279     type Borrowed<'a> = ArcBorrow<'a, T>;
280 
281     fn into_foreign(self) -> *const core::ffi::c_void {
282         ManuallyDrop::new(self).ptr.as_ptr() as _
283     }
284 
285     unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> ArcBorrow<'a, T> {
286         // SAFETY: By the safety requirement of this function, we know that `ptr` came from
287         // a previous call to `Arc::into_foreign`.
288         let inner = NonNull::new(ptr as *mut ArcInner<T>).unwrap();
289 
290         // SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive
291         // for the lifetime of the returned value.
292         unsafe { ArcBorrow::new(inner) }
293     }
294 
295     unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self {
296         // SAFETY: By the safety requirement of this function, we know that `ptr` came from
297         // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and
298         // holds a reference count increment that is transferrable to us.
299         unsafe { Self::from_inner(NonNull::new(ptr as _).unwrap()) }
300     }
301 }
302 
303 impl<T: ?Sized> Deref for Arc<T> {
304     type Target = T;
305 
306     fn deref(&self) -> &Self::Target {
307         // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
308         // safe to dereference it.
309         unsafe { &self.ptr.as_ref().data }
310     }
311 }
312 
313 impl<T: ?Sized> AsRef<T> for Arc<T> {
314     fn as_ref(&self) -> &T {
315         self.deref()
316     }
317 }
318 
319 impl<T: ?Sized> Clone for Arc<T> {
320     fn clone(&self) -> Self {
321         // INVARIANT: C `refcount_inc` saturates the refcount, so it cannot overflow to zero.
322         // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
323         // safe to increment the refcount.
324         unsafe { bindings::refcount_inc(self.ptr.as_ref().refcount.get()) };
325 
326         // SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`.
327         unsafe { Self::from_inner(self.ptr) }
328     }
329 }
330 
331 impl<T: ?Sized> Drop for Arc<T> {
332     fn drop(&mut self) {
333         // SAFETY: By the type invariant, there is necessarily a reference to the object. We cannot
334         // touch `refcount` after it's decremented to a non-zero value because another thread/CPU
335         // may concurrently decrement it to zero and free it. It is ok to have a raw pointer to
336         // freed/invalid memory as long as it is never dereferenced.
337         let refcount = unsafe { self.ptr.as_ref() }.refcount.get();
338 
339         // INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and
340         // this instance is being dropped, so the broken invariant is not observable.
341         // SAFETY: Also by the type invariant, we are allowed to decrement the refcount.
342         let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) };
343         if is_zero {
344             // The count reached zero, we must free the memory.
345             //
346             // SAFETY: The pointer was initialised from the result of `Box::leak`.
347             unsafe { drop(Box::from_raw(self.ptr.as_ptr())) };
348         }
349     }
350 }
351 
352 impl<T: ?Sized> From<UniqueArc<T>> for Arc<T> {
353     fn from(item: UniqueArc<T>) -> Self {
354         item.inner
355     }
356 }
357 
358 impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> {
359     fn from(item: Pin<UniqueArc<T>>) -> Self {
360         // SAFETY: The type invariants of `Arc` guarantee that the data is pinned.
361         unsafe { Pin::into_inner_unchecked(item).inner }
362     }
363 }
364 
365 /// A borrowed reference to an [`Arc`] instance.
366 ///
367 /// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler
368 /// to use just `&T`, which we can trivially get from an `Arc<T>` instance.
369 ///
370 /// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow<T>`
371 /// over `&Arc<T>` because the latter results in a double-indirection: a pointer (shared reference)
372 /// to a pointer (`Arc<T>`) to the object (`T`). An [`ArcBorrow`] eliminates this double
373 /// indirection while still allowing one to increment the refcount and getting an `Arc<T>` when/if
374 /// needed.
375 ///
376 /// # Invariants
377 ///
378 /// There are no mutable references to the underlying [`Arc`], and it remains valid for the
379 /// lifetime of the [`ArcBorrow`] instance.
380 ///
381 /// # Example
382 ///
383 /// ```
384 /// use kernel::sync::{Arc, ArcBorrow};
385 ///
386 /// struct Example;
387 ///
388 /// fn do_something(e: ArcBorrow<'_, Example>) -> Arc<Example> {
389 ///     e.into()
390 /// }
391 ///
392 /// let obj = Arc::try_new(Example)?;
393 /// let cloned = do_something(obj.as_arc_borrow());
394 ///
395 /// // Assert that both `obj` and `cloned` point to the same underlying object.
396 /// assert!(core::ptr::eq(&*obj, &*cloned));
397 /// # Ok::<(), Error>(())
398 /// ```
399 ///
400 /// Using `ArcBorrow<T>` as the type of `self`:
401 ///
402 /// ```
403 /// use kernel::sync::{Arc, ArcBorrow};
404 ///
405 /// struct Example {
406 ///     a: u32,
407 ///     b: u32,
408 /// }
409 ///
410 /// impl Example {
411 ///     fn use_reference(self: ArcBorrow<'_, Self>) {
412 ///         // ...
413 ///     }
414 /// }
415 ///
416 /// let obj = Arc::try_new(Example { a: 10, b: 20 })?;
417 /// obj.as_arc_borrow().use_reference();
418 /// # Ok::<(), Error>(())
419 /// ```
420 pub struct ArcBorrow<'a, T: ?Sized + 'a> {
421     inner: NonNull<ArcInner<T>>,
422     _p: PhantomData<&'a ()>,
423 }
424 
425 // This is to allow [`ArcBorrow`] (and variants) to be used as the type of `self`.
426 impl<T: ?Sized> core::ops::Receiver for ArcBorrow<'_, T> {}
427 
428 // This is to allow `ArcBorrow<U>` to be dispatched on when `ArcBorrow<T>` can be coerced into
429 // `ArcBorrow<U>`.
430 impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<ArcBorrow<'_, U>>
431     for ArcBorrow<'_, T>
432 {
433 }
434 
435 impl<T: ?Sized> Clone for ArcBorrow<'_, T> {
436     fn clone(&self) -> Self {
437         *self
438     }
439 }
440 
441 impl<T: ?Sized> Copy for ArcBorrow<'_, T> {}
442 
443 impl<T: ?Sized> ArcBorrow<'_, T> {
444     /// Creates a new [`ArcBorrow`] instance.
445     ///
446     /// # Safety
447     ///
448     /// Callers must ensure the following for the lifetime of the returned [`ArcBorrow`] instance:
449     /// 1. That `inner` remains valid;
450     /// 2. That no mutable references to `inner` are created.
451     unsafe fn new(inner: NonNull<ArcInner<T>>) -> Self {
452         // INVARIANT: The safety requirements guarantee the invariants.
453         Self {
454             inner,
455             _p: PhantomData,
456         }
457     }
458 }
459 
460 impl<T: ?Sized> From<ArcBorrow<'_, T>> for Arc<T> {
461     fn from(b: ArcBorrow<'_, T>) -> Self {
462         // SAFETY: The existence of `b` guarantees that the refcount is non-zero. `ManuallyDrop`
463         // guarantees that `drop` isn't called, so it's ok that the temporary `Arc` doesn't own the
464         // increment.
465         ManuallyDrop::new(unsafe { Arc::from_inner(b.inner) })
466             .deref()
467             .clone()
468     }
469 }
470 
471 impl<T: ?Sized> Deref for ArcBorrow<'_, T> {
472     type Target = T;
473 
474     fn deref(&self) -> &Self::Target {
475         // SAFETY: By the type invariant, the underlying object is still alive with no mutable
476         // references to it, so it is safe to create a shared reference.
477         unsafe { &self.inner.as_ref().data }
478     }
479 }
480 
481 /// A refcounted object that is known to have a refcount of 1.
482 ///
483 /// It is mutable and can be converted to an [`Arc`] so that it can be shared.
484 ///
485 /// # Invariants
486 ///
487 /// `inner` always has a reference count of 1.
488 ///
489 /// # Examples
490 ///
491 /// In the following example, we make changes to the inner object before turning it into an
492 /// `Arc<Test>` object (after which point, it cannot be mutated directly). Note that `x.into()`
493 /// cannot fail.
494 ///
495 /// ```
496 /// use kernel::sync::{Arc, UniqueArc};
497 ///
498 /// struct Example {
499 ///     a: u32,
500 ///     b: u32,
501 /// }
502 ///
503 /// fn test() -> Result<Arc<Example>> {
504 ///     let mut x = UniqueArc::try_new(Example { a: 10, b: 20 })?;
505 ///     x.a += 1;
506 ///     x.b += 1;
507 ///     Ok(x.into())
508 /// }
509 ///
510 /// # test().unwrap();
511 /// ```
512 ///
513 /// In the following example we first allocate memory for a ref-counted `Example` but we don't
514 /// initialise it on allocation. We do initialise it later with a call to [`UniqueArc::write`],
515 /// followed by a conversion to `Arc<Example>`. This is particularly useful when allocation happens
516 /// in one context (e.g., sleepable) and initialisation in another (e.g., atomic):
517 ///
518 /// ```
519 /// use kernel::sync::{Arc, UniqueArc};
520 ///
521 /// struct Example {
522 ///     a: u32,
523 ///     b: u32,
524 /// }
525 ///
526 /// fn test() -> Result<Arc<Example>> {
527 ///     let x = UniqueArc::try_new_uninit()?;
528 ///     Ok(x.write(Example { a: 10, b: 20 }).into())
529 /// }
530 ///
531 /// # test().unwrap();
532 /// ```
533 ///
534 /// In the last example below, the caller gets a pinned instance of `Example` while converting to
535 /// `Arc<Example>`; this is useful in scenarios where one needs a pinned reference during
536 /// initialisation, for example, when initialising fields that are wrapped in locks.
537 ///
538 /// ```
539 /// use kernel::sync::{Arc, UniqueArc};
540 ///
541 /// struct Example {
542 ///     a: u32,
543 ///     b: u32,
544 /// }
545 ///
546 /// fn test() -> Result<Arc<Example>> {
547 ///     let mut pinned = Pin::from(UniqueArc::try_new(Example { a: 10, b: 20 })?);
548 ///     // We can modify `pinned` because it is `Unpin`.
549 ///     pinned.as_mut().a += 1;
550 ///     Ok(pinned.into())
551 /// }
552 ///
553 /// # test().unwrap();
554 /// ```
555 pub struct UniqueArc<T: ?Sized> {
556     inner: Arc<T>,
557 }
558 
559 impl<T> UniqueArc<T> {
560     /// Tries to allocate a new [`UniqueArc`] instance.
561     pub fn try_new(value: T) -> Result<Self, AllocError> {
562         Ok(Self {
563             // INVARIANT: The newly-created object has a ref-count of 1.
564             inner: Arc::try_new(value)?,
565         })
566     }
567 
568     /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet.
569     pub fn try_new_uninit() -> Result<UniqueArc<MaybeUninit<T>>, AllocError> {
570         // INVARIANT: The refcount is initialised to a non-zero value.
571         let inner = Box::try_init::<AllocError>(try_init!(ArcInner {
572             // SAFETY: There are no safety requirements for this FFI call.
573             refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }),
574             data <- init::uninit::<T, AllocError>(),
575         }? AllocError))?;
576         Ok(UniqueArc {
577             // INVARIANT: The newly-created object has a ref-count of 1.
578             // SAFETY: The pointer from the `Box` is valid.
579             inner: unsafe { Arc::from_inner(Box::leak(inner).into()) },
580         })
581     }
582 }
583 
584 impl<T> UniqueArc<MaybeUninit<T>> {
585     /// Converts a `UniqueArc<MaybeUninit<T>>` into a `UniqueArc<T>` by writing a value into it.
586     pub fn write(mut self, value: T) -> UniqueArc<T> {
587         self.deref_mut().write(value);
588         // SAFETY: We just wrote the value to be initialized.
589         unsafe { self.assume_init() }
590     }
591 
592     /// Unsafely assume that `self` is initialized.
593     ///
594     /// # Safety
595     ///
596     /// The caller guarantees that the value behind this pointer has been initialized. It is
597     /// *immediate* UB to call this when the value is not initialized.
598     pub unsafe fn assume_init(self) -> UniqueArc<T> {
599         let inner = ManuallyDrop::new(self).inner.ptr;
600         UniqueArc {
601             // SAFETY: The new `Arc` is taking over `ptr` from `self.inner` (which won't be
602             // dropped). The types are compatible because `MaybeUninit<T>` is compatible with `T`.
603             inner: unsafe { Arc::from_inner(inner.cast()) },
604         }
605     }
606 
607     /// Initialize `self` using the given initializer.
608     pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> {
609         // SAFETY: The supplied pointer is valid for initialization.
610         match unsafe { init.__init(self.as_mut_ptr()) } {
611             // SAFETY: Initialization completed successfully.
612             Ok(()) => Ok(unsafe { self.assume_init() }),
613             Err(err) => Err(err),
614         }
615     }
616 
617     /// Pin-initialize `self` using the given pin-initializer.
618     pub fn pin_init_with<E>(
619         mut self,
620         init: impl PinInit<T, E>,
621     ) -> core::result::Result<Pin<UniqueArc<T>>, E> {
622         // SAFETY: The supplied pointer is valid for initialization and we will later pin the value
623         // to ensure it does not move.
624         match unsafe { init.__pinned_init(self.as_mut_ptr()) } {
625             // SAFETY: Initialization completed successfully.
626             Ok(()) => Ok(unsafe { self.assume_init() }.into()),
627             Err(err) => Err(err),
628         }
629     }
630 }
631 
632 impl<T: ?Sized> From<UniqueArc<T>> for Pin<UniqueArc<T>> {
633     fn from(obj: UniqueArc<T>) -> Self {
634         // SAFETY: It is not possible to move/replace `T` inside a `Pin<UniqueArc<T>>` (unless `T`
635         // is `Unpin`), so it is ok to convert it to `Pin<UniqueArc<T>>`.
636         unsafe { Pin::new_unchecked(obj) }
637     }
638 }
639 
640 impl<T: ?Sized> Deref for UniqueArc<T> {
641     type Target = T;
642 
643     fn deref(&self) -> &Self::Target {
644         self.inner.deref()
645     }
646 }
647 
648 impl<T: ?Sized> DerefMut for UniqueArc<T> {
649     fn deref_mut(&mut self) -> &mut Self::Target {
650         // SAFETY: By the `Arc` type invariant, there is necessarily a reference to the object, so
651         // it is safe to dereference it. Additionally, we know there is only one reference when
652         // it's inside a `UniqueArc`, so it is safe to get a mutable reference.
653         unsafe { &mut self.inner.ptr.as_mut().data }
654     }
655 }
656 
657 impl<T: fmt::Display + ?Sized> fmt::Display for UniqueArc<T> {
658     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
659         fmt::Display::fmt(self.deref(), f)
660     }
661 }
662 
663 impl<T: fmt::Display + ?Sized> fmt::Display for Arc<T> {
664     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
665         fmt::Display::fmt(self.deref(), f)
666     }
667 }
668 
669 impl<T: fmt::Debug + ?Sized> fmt::Debug for UniqueArc<T> {
670     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
671         fmt::Debug::fmt(self.deref(), f)
672     }
673 }
674 
675 impl<T: fmt::Debug + ?Sized> fmt::Debug for Arc<T> {
676     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
677         fmt::Debug::fmt(self.deref(), f)
678     }
679 }
680