1 // SPDX-License-Identifier: GPL-2.0 2 3 //! This module provides an interface for blk-mq drivers to implement. 4 //! 5 //! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h) 6 7 use crate::{ 8 bindings, 9 block::mq::{request::RequestDataWrapper, Request}, 10 error::{from_result, Result}, 11 prelude::*, 12 sync::Refcount, 13 types::{ARef, ForeignOwnable}, 14 }; 15 use core::marker::PhantomData; 16 17 type ForeignBorrowed<'a, T> = <T as ForeignOwnable>::Borrowed<'a>; 18 19 /// Implement this trait to interface blk-mq as block devices. 20 /// 21 /// To implement a block device driver, implement this trait as described in the 22 /// [module level documentation]. The kernel will use the implementation of the 23 /// functions defined in this trait to interface a block device driver. Note: 24 /// There is no need for an exit_request() implementation, because the `drop` 25 /// implementation of the [`Request`] type will be invoked by automatically by 26 /// the C/Rust glue logic. 27 /// 28 /// [module level documentation]: kernel::block::mq 29 #[macros::vtable] 30 pub trait Operations: Sized { 31 /// Data associated with the `struct request_queue` that is allocated for 32 /// the `GenDisk` associated with this `Operations` implementation. 33 type QueueData: ForeignOwnable; 34 35 /// Called by the kernel to queue a request with the driver. If `is_last` is 36 /// `false`, the driver is allowed to defer committing the request. 37 fn queue_rq( 38 queue_data: ForeignBorrowed<'_, Self::QueueData>, 39 rq: ARef<Request<Self>>, 40 is_last: bool, 41 ) -> Result; 42 43 /// Called by the kernel to indicate that queued requests should be submitted. 44 fn commit_rqs(queue_data: ForeignBorrowed<'_, Self::QueueData>); 45 46 /// Called by the kernel when the request is completed. 47 fn complete(rq: ARef<Request<Self>>); 48 49 /// Called by the kernel to poll the device for completed requests. Only 50 /// used for poll queues. 51 fn poll() -> bool { 52 build_error!(crate::error::VTABLE_DEFAULT_ERROR) 53 } 54 } 55 56 /// A vtable for blk-mq to interact with a block device driver. 57 /// 58 /// A `bindings::blk_mq_ops` vtable is constructed from pointers to the `extern 59 /// "C"` functions of this struct, exposed through the `OperationsVTable::VTABLE`. 60 /// 61 /// For general documentation of these methods, see the kernel source 62 /// documentation related to `struct blk_mq_operations` in 63 /// [`include/linux/blk-mq.h`]. 64 /// 65 /// [`include/linux/blk-mq.h`]: srctree/include/linux/blk-mq.h 66 pub(crate) struct OperationsVTable<T: Operations>(PhantomData<T>); 67 68 impl<T: Operations> OperationsVTable<T> { 69 /// This function is called by the C kernel. A pointer to this function is 70 /// installed in the `blk_mq_ops` vtable for the driver. 71 /// 72 /// # Safety 73 /// 74 /// - The caller of this function must ensure that the pointee of `bd` is 75 /// valid for reads for the duration of this function. 76 /// - This function must be called for an initialized and live `hctx`. That 77 /// is, `Self::init_hctx_callback` was called and 78 /// `Self::exit_hctx_callback()` was not yet called. 79 /// - `(*bd).rq` must point to an initialized and live `bindings:request`. 80 /// That is, `Self::init_request_callback` was called but 81 /// `Self::exit_request_callback` was not yet called for the request. 82 /// - `(*bd).rq` must be owned by the driver. That is, the block layer must 83 /// promise to not access the request until the driver calls 84 /// `bindings::blk_mq_end_request` for the request. 85 unsafe extern "C" fn queue_rq_callback( 86 hctx: *mut bindings::blk_mq_hw_ctx, 87 bd: *const bindings::blk_mq_queue_data, 88 ) -> bindings::blk_status_t { 89 // SAFETY: `bd.rq` is valid as required by the safety requirement for 90 // this function. 91 let request = unsafe { &*(*bd).rq.cast::<Request<T>>() }; 92 93 // One refcount for the ARef, one for being in flight 94 request.wrapper_ref().refcount().set(2); 95 96 // SAFETY: 97 // - We own a refcount that we took above. We pass that to `ARef`. 98 // - By the safety requirements of this function, `request` is a valid 99 // `struct request` and the private data is properly initialized. 100 // - `rq` will be alive until `blk_mq_end_request` is called and is 101 // reference counted by `ARef` until then. 102 let rq = unsafe { Request::aref_from_raw((*bd).rq) }; 103 104 // SAFETY: `hctx` is valid as required by this function. 105 let queue_data = unsafe { (*(*hctx).queue).queuedata }; 106 107 // SAFETY: `queue.queuedata` was created by `GenDiskBuilder::build` with 108 // a call to `ForeignOwnable::into_foreign` to create `queuedata`. 109 // `ForeignOwnable::from_foreign` is only called when the tagset is 110 // dropped, which happens after we are dropped. 111 let queue_data = unsafe { T::QueueData::borrow(queue_data) }; 112 113 // SAFETY: We have exclusive access and we just set the refcount above. 114 unsafe { Request::start_unchecked(&rq) }; 115 116 let ret = T::queue_rq( 117 queue_data, 118 rq, 119 // SAFETY: `bd` is valid as required by the safety requirement for 120 // this function. 121 unsafe { (*bd).last }, 122 ); 123 124 if let Err(e) = ret { 125 e.to_blk_status() 126 } else { 127 bindings::BLK_STS_OK as bindings::blk_status_t 128 } 129 } 130 131 /// This function is called by the C kernel. A pointer to this function is 132 /// installed in the `blk_mq_ops` vtable for the driver. 133 /// 134 /// # Safety 135 /// 136 /// This function may only be called by blk-mq C infrastructure. The caller 137 /// must ensure that `hctx` is valid. 138 unsafe extern "C" fn commit_rqs_callback(hctx: *mut bindings::blk_mq_hw_ctx) { 139 // SAFETY: `hctx` is valid as required by this function. 140 let queue_data = unsafe { (*(*hctx).queue).queuedata }; 141 142 // SAFETY: `queue.queuedata` was created by `GenDisk::try_new()` with a 143 // call to `ForeignOwnable::into_foreign()` to create `queuedata`. 144 // `ForeignOwnable::from_foreign()` is only called when the tagset is 145 // dropped, which happens after we are dropped. 146 let queue_data = unsafe { T::QueueData::borrow(queue_data) }; 147 T::commit_rqs(queue_data) 148 } 149 150 /// This function is called by the C kernel. A pointer to this function is 151 /// installed in the `blk_mq_ops` vtable for the driver. 152 /// 153 /// # Safety 154 /// 155 /// This function may only be called by blk-mq C infrastructure. `rq` must 156 /// point to a valid request that has been marked as completed. The pointee 157 /// of `rq` must be valid for write for the duration of this function. 158 unsafe extern "C" fn complete_callback(rq: *mut bindings::request) { 159 // SAFETY: This function can only be dispatched through 160 // `Request::complete`. We leaked a refcount then which we pick back up 161 // now. 162 let aref = unsafe { Request::aref_from_raw(rq) }; 163 T::complete(aref); 164 } 165 166 /// This function is called by the C kernel. A pointer to this function is 167 /// installed in the `blk_mq_ops` vtable for the driver. 168 /// 169 /// # Safety 170 /// 171 /// This function may only be called by blk-mq C infrastructure. 172 unsafe extern "C" fn poll_callback( 173 _hctx: *mut bindings::blk_mq_hw_ctx, 174 _iob: *mut bindings::io_comp_batch, 175 ) -> crate::ffi::c_int { 176 T::poll().into() 177 } 178 179 /// This function is called by the C kernel. A pointer to this function is 180 /// installed in the `blk_mq_ops` vtable for the driver. 181 /// 182 /// # Safety 183 /// 184 /// This function may only be called by blk-mq C infrastructure. This 185 /// function may only be called once before `exit_hctx_callback` is called 186 /// for the same context. 187 unsafe extern "C" fn init_hctx_callback( 188 _hctx: *mut bindings::blk_mq_hw_ctx, 189 _tagset_data: *mut crate::ffi::c_void, 190 _hctx_idx: crate::ffi::c_uint, 191 ) -> crate::ffi::c_int { 192 from_result(|| Ok(0)) 193 } 194 195 /// This function is called by the C kernel. A pointer to this function is 196 /// installed in the `blk_mq_ops` vtable for the driver. 197 /// 198 /// # Safety 199 /// 200 /// This function may only be called by blk-mq C infrastructure. 201 unsafe extern "C" fn exit_hctx_callback( 202 _hctx: *mut bindings::blk_mq_hw_ctx, 203 _hctx_idx: crate::ffi::c_uint, 204 ) { 205 } 206 207 /// This function is called by the C kernel. A pointer to this function is 208 /// installed in the `blk_mq_ops` vtable for the driver. 209 /// 210 /// # Safety 211 /// 212 /// - This function may only be called by blk-mq C infrastructure. 213 /// - `_set` must point to an initialized `TagSet<T>`. 214 /// - `rq` must point to an initialized `bindings::request`. 215 /// - The allocation pointed to by `rq` must be at the size of `Request` 216 /// plus the size of `RequestDataWrapper`. 217 unsafe extern "C" fn init_request_callback( 218 _set: *mut bindings::blk_mq_tag_set, 219 rq: *mut bindings::request, 220 _hctx_idx: crate::ffi::c_uint, 221 _numa_node: crate::ffi::c_uint, 222 ) -> crate::ffi::c_int { 223 from_result(|| { 224 // SAFETY: By the safety requirements of this function, `rq` points 225 // to a valid allocation. 226 let pdu = unsafe { Request::wrapper_ptr(rq.cast::<Request<T>>()) }; 227 228 // SAFETY: The refcount field is allocated but not initialized, so 229 // it is valid for writes. 230 unsafe { RequestDataWrapper::refcount_ptr(pdu.as_ptr()).write(Refcount::new(0)) }; 231 232 Ok(0) 233 }) 234 } 235 236 /// This function is called by the C kernel. A pointer to this function is 237 /// installed in the `blk_mq_ops` vtable for the driver. 238 /// 239 /// # Safety 240 /// 241 /// - This function may only be called by blk-mq C infrastructure. 242 /// - `_set` must point to an initialized `TagSet<T>`. 243 /// - `rq` must point to an initialized and valid `Request`. 244 unsafe extern "C" fn exit_request_callback( 245 _set: *mut bindings::blk_mq_tag_set, 246 rq: *mut bindings::request, 247 _hctx_idx: crate::ffi::c_uint, 248 ) { 249 // SAFETY: The tagset invariants guarantee that all requests are allocated with extra memory 250 // for the request data. 251 let pdu = unsafe { bindings::blk_mq_rq_to_pdu(rq) }.cast::<RequestDataWrapper>(); 252 253 // SAFETY: `pdu` is valid for read and write and is properly initialised. 254 unsafe { core::ptr::drop_in_place(pdu) }; 255 } 256 257 const VTABLE: bindings::blk_mq_ops = bindings::blk_mq_ops { 258 queue_rq: Some(Self::queue_rq_callback), 259 queue_rqs: None, 260 commit_rqs: Some(Self::commit_rqs_callback), 261 get_budget: None, 262 put_budget: None, 263 set_rq_budget_token: None, 264 get_rq_budget_token: None, 265 timeout: None, 266 poll: if T::HAS_POLL { 267 Some(Self::poll_callback) 268 } else { 269 None 270 }, 271 complete: Some(Self::complete_callback), 272 init_hctx: Some(Self::init_hctx_callback), 273 exit_hctx: Some(Self::exit_hctx_callback), 274 init_request: Some(Self::init_request_callback), 275 exit_request: Some(Self::exit_request_callback), 276 cleanup_rq: None, 277 busy: None, 278 map_queues: None, 279 #[cfg(CONFIG_BLK_DEBUG_FS)] 280 show_rq: None, 281 }; 282 283 pub(crate) const fn build() -> &'static bindings::blk_mq_ops { 284 &Self::VTABLE 285 } 286 } 287