1
// SPDX-License-Identifier: GPL-2.0
2

3
//! The `kernel` crate.
4
//!
5
//! This crate contains the kernel APIs that have been ported or wrapped for
6
//! usage by Rust code in the kernel and is shared by all of them.
7
//!
8
//! In other words, all the rest of the Rust code in the kernel (e.g. kernel
9
//! modules written in Rust) depends on [`core`] and this crate.
10
//!
11
//! If you need a kernel C API that is not ported or wrapped yet here, then
12
//! do so first instead of bypassing this crate.
13

14
#![no_std]
15
//
16
// Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on
17
// the unstable features in use.
18
//
19
// Stable since Rust 1.79.0.
20
#![feature(generic_nonzero)]
21
#![feature(inline_const)]
22
#![feature(pointer_is_aligned)]
23
//
24
// Stable since Rust 1.81.0.
25
#![feature(lint_reasons)]
26
//
27
// Stable since Rust 1.82.0.
28
#![feature(raw_ref_op)]
29
//
30
// Stable since Rust 1.83.0.
31
#![feature(const_maybe_uninit_as_mut_ptr)]
32
#![feature(const_mut_refs)]
33
#![feature(const_option)]
34
#![feature(const_ptr_write)]
35
#![feature(const_refs_to_cell)]
36
//
37
// Expected to become stable.
38
#![feature(arbitrary_self_types)]
39
//
40
// To be determined.
41
#![feature(used_with_arg)]
42
//
43
// `feature(derive_coerce_pointee)` is expected to become stable. Before Rust
44
// 1.84.0, it did not exist, so enable the predecessor features.
45
#![cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, feature(derive_coerce_pointee))]
46
#![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(coerce_unsized))]
47
#![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(dispatch_from_dyn))]
48
#![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(unsize))]
49
//
50
// `feature(file_with_nul)` is expected to become stable. Before Rust 1.89.0, it did not exist, so
51
// enable it conditionally.
52
#![cfg_attr(CONFIG_RUSTC_HAS_FILE_WITH_NUL, feature(file_with_nul))]
53

54
// Ensure conditional compilation based on the kernel configuration works;
55
// otherwise we may silently break things like initcall handling.
56
#[cfg(not(CONFIG_RUST))]
57
compile_error!("Missing kernel configuration for conditional compilation");
58

59
// Allow proc-macros to refer to `::kernel` inside the `kernel` crate (this crate).
60
extern crate self as kernel;
61

62
pub use ffi;
63

64
pub mod acpi;
65
pub mod alloc;
66
#[cfg(CONFIG_AUXILIARY_BUS)]
67
pub mod auxiliary;
68
pub mod bitmap;
69
pub mod bits;
70
#[cfg(CONFIG_BLOCK)]
71
pub mod block;
72
pub mod bug;
73
#[doc(hidden)]
74
pub mod build_assert;
75
pub mod clk;
76
#[cfg(CONFIG_CONFIGFS_FS)]
77
pub mod configfs;
78
pub mod cpu;
79
#[cfg(CONFIG_CPU_FREQ)]
80
pub mod cpufreq;
81
pub mod cpumask;
82
pub mod cred;
83
pub mod debugfs;
84
pub mod device;
85
pub mod device_id;
86
pub mod devres;
87
pub mod dma;
88
pub mod driver;
89
#[cfg(CONFIG_DRM = "y")]
90
pub mod drm;
91
pub mod error;
92
pub mod faux;
93
#[cfg(CONFIG_RUST_FW_LOADER_ABSTRACTIONS)]
94
pub mod firmware;
95
pub mod fmt;
96
pub mod fs;
97
pub mod id_pool;
98
pub mod init;
99
pub mod io;
100
pub mod ioctl;
101
pub mod iov;
102
pub mod irq;
103
pub mod jump_label;
104
#[cfg(CONFIG_KUNIT)]
105
pub mod kunit;
106
pub mod list;
107
pub mod maple_tree;
108
pub mod miscdevice;
109
pub mod mm;
110
#[cfg(CONFIG_NET)]
111
pub mod net;
112
pub mod of;
113
#[cfg(CONFIG_PM_OPP)]
114
pub mod opp;
115
pub mod page;
116
#[cfg(CONFIG_PCI)]
117
pub mod pci;
118
pub mod pid_namespace;
119
pub mod platform;
120
pub mod prelude;
121
pub mod print;
122
pub mod processor;
123
pub mod ptr;
124
pub mod rbtree;
125
pub mod regulator;
126
pub mod revocable;
127
pub mod scatterlist;
128
pub mod security;
129
pub mod seq_file;
130
pub mod sizes;
131
mod static_assert;
132
#[doc(hidden)]
133
pub mod std_vendor;
134
pub mod str;
135
pub mod sync;
136
pub mod task;
137
pub mod time;
138
pub mod tracepoint;
139
pub mod transmute;
140
pub mod types;
141
pub mod uaccess;
142
pub mod workqueue;
143
pub mod xarray;
144

145
#[doc(hidden)]
146
pub use bindings;
147
pub use macros;
148
pub use uapi;
149

150
/// Prefix to appear before log messages printed from within the `kernel` crate.
151
const __LOG_PREFIX: &[u8] = b"rust_kernel\0";
152

153
/// The top level entrypoint to implementing a kernel module.
154
///
155
/// For any teardown or cleanup operations, your type may implement [`Drop`].
156
pub trait Module: Sized + Sync + Send {
157
    /// Called at module initialization time.
158
    ///
159
    /// Use this method to perform whatever setup or registration your module
160
    /// should do.
161
    ///
162
    /// Equivalent to the `module_init` macro in the C API.
163
    fn init(module: &'static ThisModule) -> error::Result<Self>;
164
}
165

166
/// A module that is pinned and initialised in-place.
167
pub trait InPlaceModule: Sync + Send {
168
    /// Creates an initialiser for the module.
169
    ///
170
    /// It is called when the module is loaded.
171
    fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error>;
172
}
173

174
impl<T: Module> InPlaceModule for T {
175
    fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error> {
176
        let initer = move |slot: *mut Self| {
177
            let m = <Self as Module>::init(module)?;
178

179
            // SAFETY: `slot` is valid for write per the contract with `pin_init_from_closure`.
180
            unsafe { slot.write(m) };
181
            Ok(())
182
        };
183

184
        // SAFETY: On success, `initer` always fully initialises an instance of `Self`.
185
        unsafe { pin_init::pin_init_from_closure(initer) }
186
    }
187
}
188

189
/// Metadata attached to a [`Module`] or [`InPlaceModule`].
190
pub trait ModuleMetadata {
191
    /// The name of the module as specified in the `module!` macro.
192
    const NAME: &'static crate::str::CStr;
193
}
194

195
/// Equivalent to `THIS_MODULE` in the C API.
196
///
197
/// C header: [`include/linux/init.h`](srctree/include/linux/init.h)
198
pub struct ThisModule(*mut bindings::module);
199

200
// SAFETY: `THIS_MODULE` may be used from all threads within a module.
201
unsafe impl Sync for ThisModule {}
202

203
impl ThisModule {
204
    /// Creates a [`ThisModule`] given the `THIS_MODULE` pointer.
205
    ///
206
    /// # Safety
207
    ///
208
    /// The pointer must be equal to the right `THIS_MODULE`.
209
    pub const unsafe fn from_ptr(ptr: *mut bindings::module) -> ThisModule {
210
        ThisModule(ptr)
211
    }
212

213
    /// Access the raw pointer for this module.
214
    ///
215
    /// It is up to the user to use it correctly.
216
    pub const fn as_ptr(&self) -> *mut bindings::module {
217
        self.0
218
    }
219
}
220

221
#[cfg(not(testlib))]
222
#[panic_handler]
223
fn panic(info: &core::panic::PanicInfo<'_>) -> ! {
224
    pr_emerg!("{}\n", info);
225
    // SAFETY: FFI call.
226
    unsafe { bindings::BUG() };
227
}
228

229
/// Produces a pointer to an object from a pointer to one of its fields.
230
///
231
/// If you encounter a type mismatch due to the [`Opaque`] type, then use [`Opaque::cast_into`] or
232
/// [`Opaque::cast_from`] to resolve the mismatch.
233
///
234
/// [`Opaque`]: crate::types::Opaque
235
/// [`Opaque::cast_into`]: crate::types::Opaque::cast_into
236
/// [`Opaque::cast_from`]: crate::types::Opaque::cast_from
237
///
238
/// # Safety
239
///
240
/// The pointer passed to this macro, and the pointer returned by this macro, must both be in
241
/// bounds of the same allocation.
242
///
243
/// # Examples
244
///
245
/// ```
246
/// # use kernel::container_of;
247
/// struct Test {
248
///     a: u64,
249
///     b: u32,
250
/// }
251
///
252
/// let test = Test { a: 10, b: 20 };
253
/// let b_ptr: *const _ = &test.b;
254
/// // SAFETY: The pointer points at the `b` field of a `Test`, so the resulting pointer will be
255
/// // in-bounds of the same allocation as `b_ptr`.
256
/// let test_alias = unsafe { container_of!(b_ptr, Test, b) };
257
/// assert!(core::ptr::eq(&test, test_alias));
258
/// ```
259
#[macro_export]
260
macro_rules! container_of {
261
    ($field_ptr:expr, $Container:ty, $($fields:tt)*) => {{
262
        let offset: usize = ::core::mem::offset_of!($Container, $($fields)*);
263
        let field_ptr = $field_ptr;
264
        let container_ptr = field_ptr.byte_sub(offset).cast::<$Container>();
265
        $crate::assert_same_type(field_ptr, (&raw const (*container_ptr).$($fields)*).cast_mut());
266
        container_ptr
267
    }}
268
}
269

270
/// Helper for [`container_of!`].
271
#[doc(hidden)]
272
pub fn assert_same_type<T>(_: T, _: T) {}
273

274
/// Helper for `.rs.S` files.
275
#[doc(hidden)]
276
#[macro_export]
277
macro_rules! concat_literals {
278
    ($( $asm:literal )* ) => {
279
        ::core::concat!($($asm),*)
280
    };
281
}
282

283
/// Wrapper around `asm!` configured for use in the kernel.
284
///
285
/// Uses a semicolon to avoid parsing ambiguities, even though this does not match native `asm!`
286
/// syntax.
287
// For x86, `asm!` uses intel syntax by default, but we want to use at&t syntax in the kernel.
288
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
289
#[macro_export]
290
macro_rules! asm {
291
    ($($asm:expr),* ; $($rest:tt)*) => {
292
        ::core::arch::asm!( $($asm)*, options(att_syntax), $($rest)* )
293
    };
294
}
295

296
/// Wrapper around `asm!` configured for use in the kernel.
297
///
298
/// Uses a semicolon to avoid parsing ambiguities, even though this does not match native `asm!`
299
/// syntax.
300
// For non-x86 arches we just pass through to `asm!`.
301
#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
302
#[macro_export]
303
macro_rules! asm {
304
    ($($asm:expr),* ; $($rest:tt)*) => {
305
        ::core::arch::asm!( $($asm)*, $($rest)* )
306
    };
307
}
308

309
/// Gets the C string file name of a [`Location`].
310
///
311
/// If `Location::file_as_c_str()` is not available, returns a string that warns about it.
312
///
313
/// [`Location`]: core::panic::Location
314
///
315
/// # Examples
316
///
317
/// ```
318
/// # use kernel::file_from_location;
319
///
320
/// #[track_caller]
321
/// fn foo() {
322
///     let caller = core::panic::Location::caller();
323
///
324
///     // Output:
325
///     // - A path like "rust/kernel/example.rs" if `file_as_c_str()` is available.
326
///     // - "<Location::file_as_c_str() not supported>" otherwise.
327
///     let caller_file = file_from_location(caller);
328
///
329
///     // Prints out the message with caller's file name.
330
///     pr_info!("foo() called in file {caller_file:?}\n");
331
///
332
///     # if cfg!(CONFIG_RUSTC_HAS_FILE_WITH_NUL) {
333
///     #     assert_eq!(Ok(caller.file()), caller_file.to_str());
334
///     # }
335
/// }
336
///
337
/// # foo();
338
/// ```
339
#[inline]
340
pub fn file_from_location<'a>(loc: &'a core::panic::Location<'a>) -> &'a core::ffi::CStr {
341
    #[cfg(CONFIG_RUSTC_HAS_FILE_AS_C_STR)]
342
    {
343
        loc.file_as_c_str()
344
    }
345

346
    #[cfg(all(CONFIG_RUSTC_HAS_FILE_WITH_NUL, not(CONFIG_RUSTC_HAS_FILE_AS_C_STR)))]
347
    {
348
        loc.file_with_nul()
349
    }
350

351
    #[cfg(not(CONFIG_RUSTC_HAS_FILE_WITH_NUL))]
352
    {
353
        let _ = loc;
354
        c"<Location::file_as_c_str() not supported>"
355
    }
356
}
357

358