1 // SPDX-License-Identifier: GPL-2.0 2 3 // Copyright (C) 2024 Google LLC. 4 5 //! Utilities for working with `struct poll_table`. 6 7 use crate::{ 8 alloc::AllocError, 9 bindings, 10 fs::File, 11 prelude::*, 12 sync::{ 13 rcu::synchronize_rcu, 14 CondVar, 15 LockClassKey, // 16 }, // 17 types::Opaque, // 18 }; 19 use core::{ 20 marker::PhantomData, 21 mem::ManuallyDrop, 22 ops::Deref, // 23 }; 24 25 /// Creates a [`PollCondVar`] initialiser with the given name and a newly-created lock class. 26 #[macro_export] 27 macro_rules! new_poll_condvar { 28 ($($name:literal)?) => { 29 $crate::sync::poll::PollCondVar::new( 30 $crate::optional_name!($($name)?), $crate::static_lock_class!() 31 ) 32 }; 33 } 34 35 /// Wraps the kernel's `poll_table`. 36 /// 37 /// # Invariants 38 /// 39 /// The pointer must be null or reference a valid `poll_table`. 40 #[repr(transparent)] 41 pub struct PollTable<'a> { 42 table: *mut bindings::poll_table, 43 _lifetime: PhantomData<&'a bindings::poll_table>, 44 } 45 46 impl<'a> PollTable<'a> { 47 /// Creates a [`PollTable`] from a valid pointer. 48 /// 49 /// # Safety 50 /// 51 /// The pointer must be null or reference a valid `poll_table` for the duration of `'a`. 52 pub unsafe fn from_raw(table: *mut bindings::poll_table) -> Self { 53 // INVARIANTS: The safety requirements are the same as the struct invariants. 54 PollTable { 55 table, 56 _lifetime: PhantomData, 57 } 58 } 59 60 /// Register this [`PollTable`] with the provided [`PollCondVar`], so that it can be notified 61 /// using the condition variable. 62 pub fn register_wait(&self, file: &File, cv: &PollCondVar) { 63 // SAFETY: 64 // * `file.as_ptr()` references a valid file for the duration of this call. 65 // * `self.table` is null or references a valid poll_table for the duration of this call. 66 // * Since `PollCondVar` is pinned, its destructor is guaranteed to run before the memory 67 // containing `cv.wait_queue_head` is invalidated. Since the destructor clears all 68 // waiters and then waits for an rcu grace period, it's guaranteed that 69 // `cv.wait_queue_head` remains valid for at least an rcu grace period after the removal 70 // of the last waiter. 71 unsafe { bindings::poll_wait(file.as_ptr(), cv.wait_queue_head.get(), self.table) } 72 } 73 } 74 75 /// A wrapper around [`CondVar`] that makes it usable with [`PollTable`]. 76 /// 77 /// [`CondVar`]: crate::sync::CondVar 78 #[pin_data(PinnedDrop)] 79 #[repr(transparent)] 80 pub struct PollCondVar { 81 #[pin] 82 inner: CondVar, 83 } 84 85 impl PollCondVar { 86 /// Constructs a new condvar initialiser. 87 pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self> { 88 pin_init!(Self { 89 inner <- CondVar::new(name, key), 90 }) 91 } 92 } 93 94 // Make the `CondVar` methods callable on `PollCondVar`. 95 impl Deref for PollCondVar { 96 type Target = CondVar; 97 98 fn deref(&self) -> &CondVar { 99 &self.inner 100 } 101 } 102 103 #[pinned_drop] 104 impl PinnedDrop for PollCondVar { 105 #[inline] 106 fn drop(self: Pin<&mut Self>) { 107 // Clear anything registered using `register_wait`. 108 // 109 // SAFETY: The pointer points at a valid `wait_queue_head`. 110 unsafe { bindings::__wake_up_pollfree(self.inner.wait_queue_head.get()) }; 111 112 // Wait for epoll items to be properly removed. 113 synchronize_rcu(); 114 } 115 } 116 117 /// A [`KBox<PollCondVar>`] that uses `kfree_rcu`. 118 /// 119 /// [`KBox<PollCondVar>`]: PollCondVar 120 pub struct PollCondVarBox { 121 inner: ManuallyDrop<Pin<KBox<PollCondVarBoxInner>>>, 122 } 123 124 #[pin_data] 125 #[repr(C)] 126 struct PollCondVarBoxInner { 127 #[pin] 128 inner: PollCondVar, 129 rcu: Opaque<bindings::kvfree_rcu_head>, 130 } 131 132 // SAFETY: PollCondVar is Send 133 unsafe impl Send for PollCondVarBoxInner {} 134 // SAFETY: PollCondVar is Sync 135 unsafe impl Sync for PollCondVarBoxInner {} 136 137 impl PollCondVarBox { 138 /// Constructs a new boxed [`PollCondVar`]. 139 pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> Result<Self, AllocError> { 140 let b = KBox::pin_init( 141 pin_init!(PollCondVarBoxInner { 142 inner <- PollCondVar::new(name, key), 143 rcu: Opaque::uninit(), 144 }), 145 GFP_KERNEL, 146 ) 147 .map_err(|_| AllocError)?; 148 149 Ok(PollCondVarBox { 150 inner: ManuallyDrop::new(b), 151 }) 152 } 153 } 154 155 impl Deref for PollCondVarBox { 156 type Target = PollCondVar; 157 fn deref(&self) -> &PollCondVar { 158 &self.inner.inner 159 } 160 } 161 162 impl Drop for PollCondVarBox { 163 #[inline] 164 fn drop(&mut self) { 165 // SAFETY: ManuallyDrop::take ok because not already taken. 166 let boxed = unsafe { ManuallyDrop::take(&mut self.inner) }; 167 168 // SAFETY: The code below frees the box without calling the actual destructor of the type, 169 // but it's okay because it re-implements the destructor using `kfree_rcu()` in place of 170 // `synchronize_rcu()`. 171 let ptr = KBox::into_raw(unsafe { Pin::into_inner_unchecked(boxed) }); 172 173 // SAFETY: The pointer points at a valid `wait_queue_head`. 174 unsafe { bindings::__wake_up_pollfree((*ptr).inner.inner.wait_queue_head.get()) }; 175 176 // SAFETY: This was allocated using `KBox::pin_init`, so it can be freed with `kvfree`. 177 unsafe { bindings::kvfree_call_rcu((*ptr).rcu.get(), ptr.cast::<ffi::c_void>()) }; 178 } 179 } 180