1 // SPDX-License-Identifier: GPL-2.0
2
3 //! The `kernel` crate.
4 //!
5 //! This crate contains the kernel APIs that have been ported or wrapped for
6 //! usage by Rust code in the kernel and is shared by all of them.
7 //!
8 //! In other words, all the rest of the Rust code in the kernel (e.g. kernel
9 //! modules written in Rust) depends on [`core`] and this crate.
10 //!
11 //! If you need a kernel C API that is not ported or wrapped yet here, then
12 //! do so first instead of bypassing this crate.
13
14 #![no_std]
15 //
16 // Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on
17 // the unstable features in use.
18 //
19 // Stable since Rust 1.79.0.
20 #![feature(generic_nonzero)]
21 #![feature(inline_const)]
22 #![feature(pointer_is_aligned)]
23 //
24 // Stable since Rust 1.80.0.
25 #![feature(slice_flatten)]
26 //
27 // Stable since Rust 1.81.0.
28 #![feature(lint_reasons)]
29 //
30 // Stable since Rust 1.82.0.
31 #![feature(raw_ref_op)]
32 //
33 // Stable since Rust 1.83.0.
34 #![feature(const_maybe_uninit_as_mut_ptr)]
35 #![feature(const_mut_refs)]
36 #![feature(const_option)]
37 #![feature(const_ptr_write)]
38 #![feature(const_refs_to_cell)]
39 //
40 // Expected to become stable.
41 #![feature(arbitrary_self_types)]
42 //
43 // To be determined.
44 #![feature(used_with_arg)]
45 //
46 // `feature(derive_coerce_pointee)` is expected to become stable. Before Rust
47 // 1.84.0, it did not exist, so enable the predecessor features.
48 #![cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, feature(derive_coerce_pointee))]
49 #![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(coerce_unsized))]
50 #![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(dispatch_from_dyn))]
51 #![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(unsize))]
52 //
53 // `feature(file_with_nul)` is expected to become stable. Before Rust 1.89.0, it did not exist, so
54 // enable it conditionally.
55 #![cfg_attr(CONFIG_RUSTC_HAS_FILE_WITH_NUL, feature(file_with_nul))]
56
57 // Ensure conditional compilation based on the kernel configuration works;
58 // otherwise we may silently break things like initcall handling.
59 #[cfg(not(CONFIG_RUST))]
60 compile_error!("Missing kernel configuration for conditional compilation");
61
62 // Allow proc-macros to refer to `::kernel` inside the `kernel` crate (this crate).
63 extern crate self as kernel;
64
65 pub use ffi;
66
67 pub mod acpi;
68 pub mod alloc;
69 #[cfg(CONFIG_AUXILIARY_BUS)]
70 pub mod auxiliary;
71 pub mod bitmap;
72 pub mod bits;
73 #[cfg(CONFIG_BLOCK)]
74 pub mod block;
75 pub mod bug;
76 #[doc(hidden)]
77 pub mod build_assert;
78 pub mod clk;
79 #[cfg(CONFIG_CONFIGFS_FS)]
80 pub mod configfs;
81 pub mod cpu;
82 #[cfg(CONFIG_CPU_FREQ)]
83 pub mod cpufreq;
84 pub mod cpumask;
85 pub mod cred;
86 pub mod debugfs;
87 pub mod device;
88 pub mod device_id;
89 pub mod devres;
90 pub mod dma;
91 pub mod driver;
92 #[cfg(CONFIG_DRM = "y")]
93 pub mod drm;
94 pub mod error;
95 pub mod faux;
96 #[cfg(CONFIG_RUST_FW_LOADER_ABSTRACTIONS)]
97 pub mod firmware;
98 pub mod fmt;
99 pub mod fs;
100 #[cfg(CONFIG_I2C = "y")]
101 pub mod i2c;
102 pub mod id_pool;
103 pub mod init;
104 pub mod io;
105 pub mod ioctl;
106 pub mod iov;
107 pub mod irq;
108 pub mod jump_label;
109 #[cfg(CONFIG_KUNIT)]
110 pub mod kunit;
111 pub mod list;
112 pub mod maple_tree;
113 pub mod miscdevice;
114 pub mod mm;
115 pub mod module_param;
116 #[cfg(CONFIG_NET)]
117 pub mod net;
118 pub mod num;
119 pub mod of;
120 #[cfg(CONFIG_PM_OPP)]
121 pub mod opp;
122 pub mod page;
123 #[cfg(CONFIG_PCI)]
124 pub mod pci;
125 pub mod pid_namespace;
126 pub mod platform;
127 pub mod prelude;
128 pub mod print;
129 pub mod processor;
130 pub mod ptr;
131 #[cfg(CONFIG_RUST_PWM_ABSTRACTIONS)]
132 pub mod pwm;
133 pub mod rbtree;
134 pub mod regulator;
135 pub mod revocable;
136 pub mod scatterlist;
137 pub mod security;
138 pub mod seq_file;
139 pub mod sizes;
140 pub mod slice;
141 mod static_assert;
142 #[doc(hidden)]
143 pub mod std_vendor;
144 pub mod str;
145 pub mod sync;
146 pub mod task;
147 pub mod time;
148 pub mod tracepoint;
149 pub mod transmute;
150 pub mod types;
151 pub mod uaccess;
152 #[cfg(CONFIG_USB = "y")]
153 pub mod usb;
154 pub mod workqueue;
155 pub mod xarray;
156
157 #[doc(hidden)]
158 pub use bindings;
159 pub use macros;
160 pub use uapi;
161
162 /// Prefix to appear before log messages printed from within the `kernel` crate.
163 const __LOG_PREFIX: &[u8] = b"rust_kernel\0";
164
165 /// The top level entrypoint to implementing a kernel module.
166 ///
167 /// For any teardown or cleanup operations, your type may implement [`Drop`].
168 pub trait Module: Sized + Sync + Send {
169 /// Called at module initialization time.
170 ///
171 /// Use this method to perform whatever setup or registration your module
172 /// should do.
173 ///
174 /// Equivalent to the `module_init` macro in the C API.
init(module: &'static ThisModule) -> error::Result<Self>175 fn init(module: &'static ThisModule) -> error::Result<Self>;
176 }
177
178 /// A module that is pinned and initialised in-place.
179 pub trait InPlaceModule: Sync + Send {
180 /// Creates an initialiser for the module.
181 ///
182 /// It is called when the module is loaded.
init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error>183 fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error>;
184 }
185
186 impl<T: Module> InPlaceModule for T {
init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error>187 fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error> {
188 let initer = move |slot: *mut Self| {
189 let m = <Self as Module>::init(module)?;
190
191 // SAFETY: `slot` is valid for write per the contract with `pin_init_from_closure`.
192 unsafe { slot.write(m) };
193 Ok(())
194 };
195
196 // SAFETY: On success, `initer` always fully initialises an instance of `Self`.
197 unsafe { pin_init::pin_init_from_closure(initer) }
198 }
199 }
200
201 /// Metadata attached to a [`Module`] or [`InPlaceModule`].
202 pub trait ModuleMetadata {
203 /// The name of the module as specified in the `module!` macro.
204 const NAME: &'static crate::str::CStr;
205 }
206
207 /// Equivalent to `THIS_MODULE` in the C API.
208 ///
209 /// C header: [`include/linux/init.h`](srctree/include/linux/init.h)
210 pub struct ThisModule(*mut bindings::module);
211
212 // SAFETY: `THIS_MODULE` may be used from all threads within a module.
213 unsafe impl Sync for ThisModule {}
214
215 impl ThisModule {
216 /// Creates a [`ThisModule`] given the `THIS_MODULE` pointer.
217 ///
218 /// # Safety
219 ///
220 /// The pointer must be equal to the right `THIS_MODULE`.
from_ptr(ptr: *mut bindings::module) -> ThisModule221 pub const unsafe fn from_ptr(ptr: *mut bindings::module) -> ThisModule {
222 ThisModule(ptr)
223 }
224
225 /// Access the raw pointer for this module.
226 ///
227 /// It is up to the user to use it correctly.
as_ptr(&self) -> *mut bindings::module228 pub const fn as_ptr(&self) -> *mut bindings::module {
229 self.0
230 }
231 }
232
233 #[cfg(not(testlib))]
234 #[panic_handler]
panic(info: &core::panic::PanicInfo<'_>) -> !235 fn panic(info: &core::panic::PanicInfo<'_>) -> ! {
236 pr_emerg!("{}\n", info);
237 // SAFETY: FFI call.
238 unsafe { bindings::BUG() };
239 }
240
241 /// Produces a pointer to an object from a pointer to one of its fields.
242 ///
243 /// If you encounter a type mismatch due to the [`Opaque`] type, then use [`Opaque::cast_into`] or
244 /// [`Opaque::cast_from`] to resolve the mismatch.
245 ///
246 /// [`Opaque`]: crate::types::Opaque
247 /// [`Opaque::cast_into`]: crate::types::Opaque::cast_into
248 /// [`Opaque::cast_from`]: crate::types::Opaque::cast_from
249 ///
250 /// # Safety
251 ///
252 /// The pointer passed to this macro, and the pointer returned by this macro, must both be in
253 /// bounds of the same allocation.
254 ///
255 /// # Examples
256 ///
257 /// ```
258 /// # use kernel::container_of;
259 /// struct Test {
260 /// a: u64,
261 /// b: u32,
262 /// }
263 ///
264 /// let test = Test { a: 10, b: 20 };
265 /// let b_ptr: *const _ = &test.b;
266 /// // SAFETY: The pointer points at the `b` field of a `Test`, so the resulting pointer will be
267 /// // in-bounds of the same allocation as `b_ptr`.
268 /// let test_alias = unsafe { container_of!(b_ptr, Test, b) };
269 /// assert!(core::ptr::eq(&test, test_alias));
270 /// ```
271 #[macro_export]
272 macro_rules! container_of {
273 ($field_ptr:expr, $Container:ty, $($fields:tt)*) => {{
274 let offset: usize = ::core::mem::offset_of!($Container, $($fields)*);
275 let field_ptr = $field_ptr;
276 let container_ptr = field_ptr.byte_sub(offset).cast::<$Container>();
277 $crate::assert_same_type(field_ptr, (&raw const (*container_ptr).$($fields)*).cast_mut());
278 container_ptr
279 }}
280 }
281
282 /// Helper for [`container_of!`].
283 #[doc(hidden)]
assert_same_type<T>(_: T, _: T)284 pub fn assert_same_type<T>(_: T, _: T) {}
285
286 /// Helper for `.rs.S` files.
287 #[doc(hidden)]
288 #[macro_export]
289 macro_rules! concat_literals {
290 ($( $asm:literal )* ) => {
291 ::core::concat!($($asm),*)
292 };
293 }
294
295 /// Wrapper around `asm!` configured for use in the kernel.
296 ///
297 /// Uses a semicolon to avoid parsing ambiguities, even though this does not match native `asm!`
298 /// syntax.
299 // For x86, `asm!` uses intel syntax by default, but we want to use at&t syntax in the kernel.
300 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
301 #[macro_export]
302 macro_rules! asm {
303 ($($asm:expr),* ; $($rest:tt)*) => {
304 ::core::arch::asm!( $($asm)*, options(att_syntax), $($rest)* )
305 };
306 }
307
308 /// Wrapper around `asm!` configured for use in the kernel.
309 ///
310 /// Uses a semicolon to avoid parsing ambiguities, even though this does not match native `asm!`
311 /// syntax.
312 // For non-x86 arches we just pass through to `asm!`.
313 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
314 #[macro_export]
315 macro_rules! asm {
316 ($($asm:expr),* ; $($rest:tt)*) => {
317 ::core::arch::asm!( $($asm)*, $($rest)* )
318 };
319 }
320
321 /// Gets the C string file name of a [`Location`].
322 ///
323 /// If `Location::file_as_c_str()` is not available, returns a string that warns about it.
324 ///
325 /// [`Location`]: core::panic::Location
326 ///
327 /// # Examples
328 ///
329 /// ```
330 /// # use kernel::file_from_location;
331 ///
332 /// #[track_caller]
333 /// fn foo() {
334 /// let caller = core::panic::Location::caller();
335 ///
336 /// // Output:
337 /// // - A path like "rust/kernel/example.rs" if `file_as_c_str()` is available.
338 /// // - "<Location::file_as_c_str() not supported>" otherwise.
339 /// let caller_file = file_from_location(caller);
340 ///
341 /// // Prints out the message with caller's file name.
342 /// pr_info!("foo() called in file {caller_file:?}\n");
343 ///
344 /// # if cfg!(CONFIG_RUSTC_HAS_FILE_WITH_NUL) {
345 /// # assert_eq!(Ok(caller.file()), caller_file.to_str());
346 /// # }
347 /// }
348 ///
349 /// # foo();
350 /// ```
351 #[inline]
file_from_location<'a>(loc: &'a core::panic::Location<'a>) -> &'a core::ffi::CStr352 pub fn file_from_location<'a>(loc: &'a core::panic::Location<'a>) -> &'a core::ffi::CStr {
353 #[cfg(CONFIG_RUSTC_HAS_FILE_AS_C_STR)]
354 {
355 loc.file_as_c_str()
356 }
357
358 #[cfg(all(CONFIG_RUSTC_HAS_FILE_WITH_NUL, not(CONFIG_RUSTC_HAS_FILE_AS_C_STR)))]
359 {
360 loc.file_with_nul()
361 }
362
363 #[cfg(not(CONFIG_RUSTC_HAS_FILE_WITH_NUL))]
364 {
365 let _ = loc;
366 c"<Location::file_as_c_str() not supported>"
367 }
368 }
369