1 // SPDX-License-Identifier: GPL-2.0
2
3 //! The `kernel` crate.
4 //!
5 //! This crate contains the kernel APIs that have been ported or wrapped for
6 //! usage by Rust code in the kernel and is shared by all of them.
7 //!
8 //! In other words, all the rest of the Rust code in the kernel (e.g. kernel
9 //! modules written in Rust) depends on [`core`] and this crate.
10 //!
11 //! If you need a kernel C API that is not ported or wrapped yet here, then
12 //! do so first instead of bypassing this crate.
13
14 #![no_std]
15 //
16 // Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on
17 // the unstable features in use.
18 //
19 // Stable since Rust 1.79.0.
20 #![feature(generic_nonzero)]
21 #![feature(inline_const)]
22 #![feature(pointer_is_aligned)]
23 //
24 // Stable since Rust 1.81.0.
25 #![feature(lint_reasons)]
26 //
27 // Stable since Rust 1.82.0.
28 #![feature(raw_ref_op)]
29 //
30 // Stable since Rust 1.83.0.
31 #![feature(const_maybe_uninit_as_mut_ptr)]
32 #![feature(const_mut_refs)]
33 #![feature(const_option)]
34 #![feature(const_ptr_write)]
35 #![feature(const_refs_to_cell)]
36 //
37 // Expected to become stable.
38 #![feature(arbitrary_self_types)]
39 //
40 // To be determined.
41 #![feature(used_with_arg)]
42 //
43 // `feature(derive_coerce_pointee)` is expected to become stable. Before Rust
44 // 1.84.0, it did not exist, so enable the predecessor features.
45 #![cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, feature(derive_coerce_pointee))]
46 #![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(coerce_unsized))]
47 #![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(dispatch_from_dyn))]
48 #![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(unsize))]
49 //
50 // `feature(file_with_nul)` is expected to become stable. Before Rust 1.89.0, it did not exist, so
51 // enable it conditionally.
52 #![cfg_attr(CONFIG_RUSTC_HAS_FILE_WITH_NUL, feature(file_with_nul))]
53
54 // Ensure conditional compilation based on the kernel configuration works;
55 // otherwise we may silently break things like initcall handling.
56 #[cfg(not(CONFIG_RUST))]
57 compile_error!("Missing kernel configuration for conditional compilation");
58
59 // Allow proc-macros to refer to `::kernel` inside the `kernel` crate (this crate).
60 extern crate self as kernel;
61
62 pub use ffi;
63
64 pub mod acpi;
65 pub mod alloc;
66 #[cfg(CONFIG_AUXILIARY_BUS)]
67 pub mod auxiliary;
68 pub mod bitmap;
69 pub mod bits;
70 #[cfg(CONFIG_BLOCK)]
71 pub mod block;
72 pub mod bug;
73 #[doc(hidden)]
74 pub mod build_assert;
75 pub mod clk;
76 #[cfg(CONFIG_CONFIGFS_FS)]
77 pub mod configfs;
78 pub mod cpu;
79 #[cfg(CONFIG_CPU_FREQ)]
80 pub mod cpufreq;
81 pub mod cpumask;
82 pub mod cred;
83 pub mod debugfs;
84 pub mod device;
85 pub mod device_id;
86 pub mod devres;
87 pub mod dma;
88 pub mod driver;
89 #[cfg(CONFIG_DRM = "y")]
90 pub mod drm;
91 pub mod error;
92 pub mod faux;
93 #[cfg(CONFIG_RUST_FW_LOADER_ABSTRACTIONS)]
94 pub mod firmware;
95 pub mod fmt;
96 pub mod fs;
97 pub mod id_pool;
98 pub mod init;
99 pub mod io;
100 pub mod ioctl;
101 pub mod irq;
102 pub mod jump_label;
103 #[cfg(CONFIG_KUNIT)]
104 pub mod kunit;
105 pub mod list;
106 pub mod maple_tree;
107 pub mod miscdevice;
108 pub mod mm;
109 #[cfg(CONFIG_NET)]
110 pub mod net;
111 pub mod of;
112 #[cfg(CONFIG_PM_OPP)]
113 pub mod opp;
114 pub mod page;
115 #[cfg(CONFIG_PCI)]
116 pub mod pci;
117 pub mod pid_namespace;
118 pub mod platform;
119 pub mod prelude;
120 pub mod print;
121 pub mod processor;
122 pub mod ptr;
123 pub mod rbtree;
124 pub mod regulator;
125 pub mod revocable;
126 pub mod scatterlist;
127 pub mod security;
128 pub mod seq_file;
129 pub mod sizes;
130 mod static_assert;
131 #[doc(hidden)]
132 pub mod std_vendor;
133 pub mod str;
134 pub mod sync;
135 pub mod task;
136 pub mod time;
137 pub mod tracepoint;
138 pub mod transmute;
139 pub mod types;
140 pub mod uaccess;
141 pub mod workqueue;
142 pub mod xarray;
143
144 #[doc(hidden)]
145 pub use bindings;
146 pub use macros;
147 pub use uapi;
148
149 /// Prefix to appear before log messages printed from within the `kernel` crate.
150 const __LOG_PREFIX: &[u8] = b"rust_kernel\0";
151
152 /// The top level entrypoint to implementing a kernel module.
153 ///
154 /// For any teardown or cleanup operations, your type may implement [`Drop`].
155 pub trait Module: Sized + Sync + Send {
156 /// Called at module initialization time.
157 ///
158 /// Use this method to perform whatever setup or registration your module
159 /// should do.
160 ///
161 /// Equivalent to the `module_init` macro in the C API.
init(module: &'static ThisModule) -> error::Result<Self>162 fn init(module: &'static ThisModule) -> error::Result<Self>;
163 }
164
165 /// A module that is pinned and initialised in-place.
166 pub trait InPlaceModule: Sync + Send {
167 /// Creates an initialiser for the module.
168 ///
169 /// It is called when the module is loaded.
init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error>170 fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error>;
171 }
172
173 impl<T: Module> InPlaceModule for T {
init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error>174 fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error> {
175 let initer = move |slot: *mut Self| {
176 let m = <Self as Module>::init(module)?;
177
178 // SAFETY: `slot` is valid for write per the contract with `pin_init_from_closure`.
179 unsafe { slot.write(m) };
180 Ok(())
181 };
182
183 // SAFETY: On success, `initer` always fully initialises an instance of `Self`.
184 unsafe { pin_init::pin_init_from_closure(initer) }
185 }
186 }
187
188 /// Metadata attached to a [`Module`] or [`InPlaceModule`].
189 pub trait ModuleMetadata {
190 /// The name of the module as specified in the `module!` macro.
191 const NAME: &'static crate::str::CStr;
192 }
193
194 /// Equivalent to `THIS_MODULE` in the C API.
195 ///
196 /// C header: [`include/linux/init.h`](srctree/include/linux/init.h)
197 pub struct ThisModule(*mut bindings::module);
198
199 // SAFETY: `THIS_MODULE` may be used from all threads within a module.
200 unsafe impl Sync for ThisModule {}
201
202 impl ThisModule {
203 /// Creates a [`ThisModule`] given the `THIS_MODULE` pointer.
204 ///
205 /// # Safety
206 ///
207 /// The pointer must be equal to the right `THIS_MODULE`.
from_ptr(ptr: *mut bindings::module) -> ThisModule208 pub const unsafe fn from_ptr(ptr: *mut bindings::module) -> ThisModule {
209 ThisModule(ptr)
210 }
211
212 /// Access the raw pointer for this module.
213 ///
214 /// It is up to the user to use it correctly.
as_ptr(&self) -> *mut bindings::module215 pub const fn as_ptr(&self) -> *mut bindings::module {
216 self.0
217 }
218 }
219
220 #[cfg(not(testlib))]
221 #[panic_handler]
panic(info: &core::panic::PanicInfo<'_>) -> !222 fn panic(info: &core::panic::PanicInfo<'_>) -> ! {
223 pr_emerg!("{}\n", info);
224 // SAFETY: FFI call.
225 unsafe { bindings::BUG() };
226 }
227
228 /// Produces a pointer to an object from a pointer to one of its fields.
229 ///
230 /// If you encounter a type mismatch due to the [`Opaque`] type, then use [`Opaque::cast_into`] or
231 /// [`Opaque::cast_from`] to resolve the mismatch.
232 ///
233 /// [`Opaque`]: crate::types::Opaque
234 /// [`Opaque::cast_into`]: crate::types::Opaque::cast_into
235 /// [`Opaque::cast_from`]: crate::types::Opaque::cast_from
236 ///
237 /// # Safety
238 ///
239 /// The pointer passed to this macro, and the pointer returned by this macro, must both be in
240 /// bounds of the same allocation.
241 ///
242 /// # Examples
243 ///
244 /// ```
245 /// # use kernel::container_of;
246 /// struct Test {
247 /// a: u64,
248 /// b: u32,
249 /// }
250 ///
251 /// let test = Test { a: 10, b: 20 };
252 /// let b_ptr: *const _ = &test.b;
253 /// // SAFETY: The pointer points at the `b` field of a `Test`, so the resulting pointer will be
254 /// // in-bounds of the same allocation as `b_ptr`.
255 /// let test_alias = unsafe { container_of!(b_ptr, Test, b) };
256 /// assert!(core::ptr::eq(&test, test_alias));
257 /// ```
258 #[macro_export]
259 macro_rules! container_of {
260 ($field_ptr:expr, $Container:ty, $($fields:tt)*) => {{
261 let offset: usize = ::core::mem::offset_of!($Container, $($fields)*);
262 let field_ptr = $field_ptr;
263 let container_ptr = field_ptr.byte_sub(offset).cast::<$Container>();
264 $crate::assert_same_type(field_ptr, (&raw const (*container_ptr).$($fields)*).cast_mut());
265 container_ptr
266 }}
267 }
268
269 /// Helper for [`container_of!`].
270 #[doc(hidden)]
assert_same_type<T>(_: T, _: T)271 pub fn assert_same_type<T>(_: T, _: T) {}
272
273 /// Helper for `.rs.S` files.
274 #[doc(hidden)]
275 #[macro_export]
276 macro_rules! concat_literals {
277 ($( $asm:literal )* ) => {
278 ::core::concat!($($asm),*)
279 };
280 }
281
282 /// Wrapper around `asm!` configured for use in the kernel.
283 ///
284 /// Uses a semicolon to avoid parsing ambiguities, even though this does not match native `asm!`
285 /// syntax.
286 // For x86, `asm!` uses intel syntax by default, but we want to use at&t syntax in the kernel.
287 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
288 #[macro_export]
289 macro_rules! asm {
290 ($($asm:expr),* ; $($rest:tt)*) => {
291 ::core::arch::asm!( $($asm)*, options(att_syntax), $($rest)* )
292 };
293 }
294
295 /// Wrapper around `asm!` configured for use in the kernel.
296 ///
297 /// Uses a semicolon to avoid parsing ambiguities, even though this does not match native `asm!`
298 /// syntax.
299 // For non-x86 arches we just pass through to `asm!`.
300 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
301 #[macro_export]
302 macro_rules! asm {
303 ($($asm:expr),* ; $($rest:tt)*) => {
304 ::core::arch::asm!( $($asm)*, $($rest)* )
305 };
306 }
307
308 /// Gets the C string file name of a [`Location`].
309 ///
310 /// If `Location::file_as_c_str()` is not available, returns a string that warns about it.
311 ///
312 /// [`Location`]: core::panic::Location
313 ///
314 /// # Examples
315 ///
316 /// ```
317 /// # use kernel::file_from_location;
318 ///
319 /// #[track_caller]
320 /// fn foo() {
321 /// let caller = core::panic::Location::caller();
322 ///
323 /// // Output:
324 /// // - A path like "rust/kernel/example.rs" if `file_as_c_str()` is available.
325 /// // - "<Location::file_as_c_str() not supported>" otherwise.
326 /// let caller_file = file_from_location(caller);
327 ///
328 /// // Prints out the message with caller's file name.
329 /// pr_info!("foo() called in file {caller_file:?}\n");
330 ///
331 /// # if cfg!(CONFIG_RUSTC_HAS_FILE_WITH_NUL) {
332 /// # assert_eq!(Ok(caller.file()), caller_file.to_str());
333 /// # }
334 /// }
335 ///
336 /// # foo();
337 /// ```
338 #[inline]
file_from_location<'a>(loc: &'a core::panic::Location<'a>) -> &'a core::ffi::CStr339 pub fn file_from_location<'a>(loc: &'a core::panic::Location<'a>) -> &'a core::ffi::CStr {
340 #[cfg(CONFIG_RUSTC_HAS_FILE_AS_C_STR)]
341 {
342 loc.file_as_c_str()
343 }
344
345 #[cfg(all(CONFIG_RUSTC_HAS_FILE_WITH_NUL, not(CONFIG_RUSTC_HAS_FILE_AS_C_STR)))]
346 {
347 loc.file_with_nul()
348 }
349
350 #[cfg(not(CONFIG_RUSTC_HAS_FILE_WITH_NUL))]
351 {
352 let _ = loc;
353 c"<Location::file_as_c_str() not supported>"
354 }
355 }
356