summaryrefslogtreecommitdiff
path: root/rust/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'rust/kernel')
-rw-r--r--rust/kernel/alloc.rs4
-rw-r--r--rust/kernel/alloc/allocator.rs1
-rw-r--r--rust/kernel/alloc/allocator_test.rs20
-rw-r--r--rust/kernel/alloc/kbox.rs91
-rw-r--r--rust/kernel/alloc/kvec.rs430
-rw-r--r--rust/kernel/alloc/kvec/errors.rs61
-rw-r--r--rust/kernel/auxiliary.rs362
-rw-r--r--rust/kernel/block/mq/gen_disk.rs2
-rw-r--r--rust/kernel/block/mq/request.rs4
-rw-r--r--rust/kernel/block/mq/tag_set.rs5
-rw-r--r--rust/kernel/clk.rs334
-rw-r--r--rust/kernel/configfs.rs1049
-rw-r--r--rust/kernel/cpu.rs151
-rw-r--r--rust/kernel/cpufreq.rs1404
-rw-r--r--rust/kernel/cpumask.rs351
-rw-r--r--rust/kernel/cred.rs5
-rw-r--r--rust/kernel/device.rs161
-rw-r--r--rust/kernel/device_id.rs4
-rw-r--r--rust/kernel/devres.rs116
-rw-r--r--rust/kernel/dma.rs391
-rw-r--r--rust/kernel/driver.rs6
-rw-r--r--rust/kernel/drm/device.rs202
-rw-r--r--rust/kernel/drm/driver.rs166
-rw-r--r--rust/kernel/drm/file.rs99
-rw-r--r--rust/kernel/drm/gem/mod.rs332
-rw-r--r--rust/kernel/drm/ioctl.rs162
-rw-r--r--rust/kernel/drm/mod.rs19
-rw-r--r--rust/kernel/error.rs126
-rw-r--r--rust/kernel/faux.rs16
-rw-r--r--rust/kernel/firmware.rs224
-rw-r--r--rust/kernel/fs/file.rs18
-rw-r--r--rust/kernel/init.rs1450
-rw-r--r--rust/kernel/init/__internal.rs264
-rw-r--r--rust/kernel/init/macros.rs1410
-rw-r--r--rust/kernel/io.rs66
-rw-r--r--rust/kernel/kunit.rs202
-rw-r--r--rust/kernel/lib.rs68
-rw-r--r--rust/kernel/list.rs585
-rw-r--r--rust/kernel/list/arc.rs6
-rw-r--r--rust/kernel/miscdevice.rs342
-rw-r--r--rust/kernel/mm.rs296
-rw-r--r--rust/kernel/mm/mmput_async.rs68
-rw-r--r--rust/kernel/mm/virt.rs471
-rw-r--r--rust/kernel/net/phy.rs5
-rw-r--r--rust/kernel/opp.rs1146
-rw-r--r--rust/kernel/page.rs2
-rw-r--r--rust/kernel/pci.rs178
-rw-r--r--rust/kernel/platform.rs127
-rw-r--r--rust/kernel/prelude.rs13
-rw-r--r--rust/kernel/print.rs37
-rw-r--r--rust/kernel/rbtree.rs35
-rw-r--r--rust/kernel/revocable.rs46
-rw-r--r--rust/kernel/security.rs12
-rw-r--r--rust/kernel/seq_file.rs3
-rw-r--r--rust/kernel/static_assert.rs9
-rw-r--r--rust/kernel/std_vendor.rs2
-rw-r--r--rust/kernel/str.rs174
-rw-r--r--rust/kernel/sync.rs66
-rw-r--r--rust/kernel/sync/arc.rs106
-rw-r--r--rust/kernel/sync/completion.rs112
-rw-r--r--rust/kernel/sync/condvar.rs34
-rw-r--r--rust/kernel/sync/lock.rs40
-rw-r--r--rust/kernel/sync/lock/global.rs5
-rw-r--r--rust/kernel/sync/lock/mutex.rs2
-rw-r--r--rust/kernel/sync/lock/spinlock.rs2
-rw-r--r--rust/kernel/sync/locked_by.rs2
-rw-r--r--rust/kernel/sync/poll.rs6
-rw-r--r--rust/kernel/sync/rcu.rs5
-rw-r--r--rust/kernel/task.rs253
-rw-r--r--rust/kernel/time.rs219
-rw-r--r--rust/kernel/time/hrtimer.rs534
-rw-r--r--rust/kernel/time/hrtimer/arc.rs100
-rw-r--r--rust/kernel/time/hrtimer/pin.rs104
-rw-r--r--rust/kernel/time/hrtimer/pin_mut.rs108
-rw-r--r--rust/kernel/time/hrtimer/tbox.rs120
-rw-r--r--rust/kernel/types.rs77
-rw-r--r--rust/kernel/uaccess.rs9
-rw-r--r--rust/kernel/workqueue.rs76
-rw-r--r--rust/kernel/xarray.rs275
79 files changed, 11709 insertions, 3879 deletions
diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs
index fc9c9c41cd79..a2c49e5494d3 100644
--- a/rust/kernel/alloc.rs
+++ b/rust/kernel/alloc.rs
@@ -94,10 +94,10 @@ pub mod flags {
///
/// A lower watermark is applied to allow access to "atomic reserves". The current
/// implementation doesn't support NMI and few other strict non-preemptive contexts (e.g.
- /// raw_spin_lock). The same applies to [`GFP_NOWAIT`].
+ /// `raw_spin_lock`). The same applies to [`GFP_NOWAIT`].
pub const GFP_ATOMIC: Flags = Flags(bindings::GFP_ATOMIC);
- /// Typical for kernel-internal allocations. The caller requires ZONE_NORMAL or a lower zone
+ /// Typical for kernel-internal allocations. The caller requires `ZONE_NORMAL` or a lower zone
/// for direct access but can direct reclaim.
pub const GFP_KERNEL: Flags = Flags(bindings::GFP_KERNEL);
diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs
index 439985e29fbc..aa2dfa9dca4c 100644
--- a/rust/kernel/alloc/allocator.rs
+++ b/rust/kernel/alloc/allocator.rs
@@ -80,6 +80,7 @@ impl ReallocFunc {
/// This method has the same guarantees as `Allocator::realloc`. Additionally
/// - it accepts any pointer to a valid memory allocation allocated by this function.
/// - memory allocated by this function remains valid until it is passed to this function.
+ #[inline]
unsafe fn call(
&self,
ptr: Option<NonNull<u8>>,
diff --git a/rust/kernel/alloc/allocator_test.rs b/rust/kernel/alloc/allocator_test.rs
index e3240d16040b..d19c06ef0498 100644
--- a/rust/kernel/alloc/allocator_test.rs
+++ b/rust/kernel/alloc/allocator_test.rs
@@ -4,7 +4,7 @@
//! of those types (e.g. `CString`) use kernel allocators for instantiation.
//!
//! In order to allow userspace test cases to make use of such types as well, implement the
-//! `Cmalloc` allocator within the allocator_test module and type alias all kernel allocators to
+//! `Cmalloc` allocator within the `allocator_test` module and type alias all kernel allocators to
//! `Cmalloc`. The `Cmalloc` allocator uses libc's `realloc()` function as allocator backend.
#![allow(missing_docs)]
@@ -62,6 +62,24 @@ unsafe impl Allocator for Cmalloc {
));
}
+ // ISO C (ISO/IEC 9899:2011) defines `aligned_alloc`:
+ //
+ // > The value of alignment shall be a valid alignment supported by the implementation
+ // [...].
+ //
+ // As an example of the "supported by the implementation" requirement, POSIX.1-2001 (IEEE
+ // 1003.1-2001) defines `posix_memalign`:
+ //
+ // > The value of alignment shall be a power of two multiple of sizeof (void *).
+ //
+ // and POSIX-based implementations of `aligned_alloc` inherit this requirement. At the time
+ // of writing, this is known to be the case on macOS (but not in glibc).
+ //
+ // Satisfy the stricter requirement to avoid spurious test failures on some platforms.
+ let min_align = core::mem::size_of::<*const crate::ffi::c_void>();
+ let layout = layout.align_to(min_align).map_err(|_| AllocError)?;
+ let layout = layout.pad_to_align();
+
// SAFETY: Returns either NULL or a pointer to a memory allocation that satisfies or
// exceeds the given size and alignment requirements.
let dst = unsafe { libc_aligned_alloc(layout.align(), layout.size()) } as *mut u8;
diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs
index cb4ebea3b074..c386ff771d50 100644
--- a/rust/kernel/alloc/kbox.rs
+++ b/rust/kernel/alloc/kbox.rs
@@ -15,8 +15,9 @@ use core::pin::Pin;
use core::ptr::NonNull;
use core::result::Result;
-use crate::init::{InPlaceInit, InPlaceWrite, Init, PinInit};
+use crate::init::InPlaceInit;
use crate::types::ForeignOwnable;
+use pin_init::{InPlaceWrite, Init, PinInit, ZeroableOption};
/// The kernel's [`Box`] type -- a heap allocation for a single value of type `T`.
///
@@ -56,12 +57,50 @@ use crate::types::ForeignOwnable;
/// assert!(KVBox::<Huge>::new_uninit(GFP_KERNEL).is_ok());
/// ```
///
+/// [`Box`]es can also be used to store trait objects by coercing their type:
+///
+/// ```
+/// trait FooTrait {}
+///
+/// struct FooStruct;
+/// impl FooTrait for FooStruct {}
+///
+/// let _ = KBox::new(FooStruct, GFP_KERNEL)? as KBox<dyn FooTrait>;
+/// # Ok::<(), Error>(())
+/// ```
+///
/// # Invariants
///
/// `self.0` is always properly aligned and either points to memory allocated with `A` or, for
/// zero-sized types, is a dangling, well aligned pointer.
#[repr(transparent)]
-pub struct Box<T: ?Sized, A: Allocator>(NonNull<T>, PhantomData<A>);
+#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))]
+pub struct Box<#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, pointee)] T: ?Sized, A: Allocator>(
+ NonNull<T>,
+ PhantomData<A>,
+);
+
+// This is to allow coercion from `Box<T, A>` to `Box<U, A>` if `T` can be converted to the
+// dynamically-sized type (DST) `U`.
+#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))]
+impl<T, U, A> core::ops::CoerceUnsized<Box<U, A>> for Box<T, A>
+where
+ T: ?Sized + core::marker::Unsize<U>,
+ U: ?Sized,
+ A: Allocator,
+{
+}
+
+// This is to allow `Box<U, A>` to be dispatched on when `Box<T, A>` can be coerced into `Box<U,
+// A>`.
+#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))]
+impl<T, U, A> core::ops::DispatchFromDyn<Box<U, A>> for Box<T, A>
+where
+ T: ?Sized + core::marker::Unsize<U>,
+ U: ?Sized,
+ A: Allocator,
+{
+}
/// Type alias for [`Box`] with a [`Kmalloc`] allocator.
///
@@ -99,6 +138,10 @@ pub type VBox<T> = Box<T, super::allocator::Vmalloc>;
/// ```
pub type KVBox<T> = Box<T, super::allocator::KVmalloc>;
+// SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee:
+// <https://doc.rust-lang.org/stable/std/option/index.html#representation>).
+unsafe impl<T, A: Allocator> ZeroableOption for Box<T, A> {}
+
// SAFETY: `Box` is `Send` if `T` is `Send` because the `Box` owns a `T`.
unsafe impl<T, A> Send for Box<T, A>
where
@@ -245,6 +288,12 @@ where
Ok(Self::new(x, flags)?.into())
}
+ /// Convert a [`Box<T,A>`] to a [`Pin<Box<T,A>>`]. If `T` does not implement
+ /// [`Unpin`], then `x` will be pinned in memory and can't be moved.
+ pub fn into_pin(this: Self) -> Pin<Self> {
+ this.into()
+ }
+
/// Forgets the contents (does not run the destructor), but keeps the allocation.
fn forget_contents(this: Self) -> Box<MaybeUninit<T>, A> {
let ptr = Self::into_raw(this);
@@ -349,68 +398,70 @@ where
}
}
-impl<T: 'static, A> ForeignOwnable for Box<T, A>
+// SAFETY: The `into_foreign` function returns a pointer that is well-aligned.
+unsafe impl<T: 'static, A> ForeignOwnable for Box<T, A>
where
A: Allocator,
{
+ type PointedTo = T;
type Borrowed<'a> = &'a T;
type BorrowedMut<'a> = &'a mut T;
- fn into_foreign(self) -> *mut crate::ffi::c_void {
- Box::into_raw(self).cast()
+ fn into_foreign(self) -> *mut Self::PointedTo {
+ Box::into_raw(self)
}
- unsafe fn from_foreign(ptr: *mut crate::ffi::c_void) -> Self {
+ unsafe fn from_foreign(ptr: *mut Self::PointedTo) -> Self {
// SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous
// call to `Self::into_foreign`.
- unsafe { Box::from_raw(ptr.cast()) }
+ unsafe { Box::from_raw(ptr) }
}
- unsafe fn borrow<'a>(ptr: *mut crate::ffi::c_void) -> &'a T {
+ unsafe fn borrow<'a>(ptr: *mut Self::PointedTo) -> &'a T {
// SAFETY: The safety requirements of this method ensure that the object remains alive and
// immutable for the duration of 'a.
- unsafe { &*ptr.cast() }
+ unsafe { &*ptr }
}
- unsafe fn borrow_mut<'a>(ptr: *mut crate::ffi::c_void) -> &'a mut T {
- let ptr = ptr.cast();
+ unsafe fn borrow_mut<'a>(ptr: *mut Self::PointedTo) -> &'a mut T {
// SAFETY: The safety requirements of this method ensure that the pointer is valid and that
// nothing else will access the value for the duration of 'a.
unsafe { &mut *ptr }
}
}
-impl<T: 'static, A> ForeignOwnable for Pin<Box<T, A>>
+// SAFETY: The `into_foreign` function returns a pointer that is well-aligned.
+unsafe impl<T: 'static, A> ForeignOwnable for Pin<Box<T, A>>
where
A: Allocator,
{
+ type PointedTo = T;
type Borrowed<'a> = Pin<&'a T>;
type BorrowedMut<'a> = Pin<&'a mut T>;
- fn into_foreign(self) -> *mut crate::ffi::c_void {
+ fn into_foreign(self) -> *mut Self::PointedTo {
// SAFETY: We are still treating the box as pinned.
- Box::into_raw(unsafe { Pin::into_inner_unchecked(self) }).cast()
+ Box::into_raw(unsafe { Pin::into_inner_unchecked(self) })
}
- unsafe fn from_foreign(ptr: *mut crate::ffi::c_void) -> Self {
+ unsafe fn from_foreign(ptr: *mut Self::PointedTo) -> Self {
// SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous
// call to `Self::into_foreign`.
- unsafe { Pin::new_unchecked(Box::from_raw(ptr.cast())) }
+ unsafe { Pin::new_unchecked(Box::from_raw(ptr)) }
}
- unsafe fn borrow<'a>(ptr: *mut crate::ffi::c_void) -> Pin<&'a T> {
+ unsafe fn borrow<'a>(ptr: *mut Self::PointedTo) -> Pin<&'a T> {
// SAFETY: The safety requirements for this function ensure that the object is still alive,
// so it is safe to dereference the raw pointer.
// The safety requirements of `from_foreign` also ensure that the object remains alive for
// the lifetime of the returned value.
- let r = unsafe { &*ptr.cast() };
+ let r = unsafe { &*ptr };
// SAFETY: This pointer originates from a `Pin<Box<T>>`.
unsafe { Pin::new_unchecked(r) }
}
- unsafe fn borrow_mut<'a>(ptr: *mut crate::ffi::c_void) -> Pin<&'a mut T> {
- let ptr = ptr.cast();
+ unsafe fn borrow_mut<'a>(ptr: *mut Self::PointedTo) -> Pin<&'a mut T> {
// SAFETY: The safety requirements for this function ensure that the object is still alive,
// so it is safe to dereference the raw pointer.
// The safety requirements of `from_foreign` also ensure that the object remains alive for
diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs
index ae9d072741ce..1a0dd852a468 100644
--- a/rust/kernel/alloc/kvec.rs
+++ b/rust/kernel/alloc/kvec.rs
@@ -21,6 +21,9 @@ use core::{
slice::SliceIndex,
};
+mod errors;
+pub use self::errors::{InsertError, PushError, RemoveError};
+
/// Create a [`KVec`] containing the arguments.
///
/// New memory is allocated with `GFP_KERNEL`.
@@ -90,6 +93,8 @@ macro_rules! kvec {
/// without re-allocation. For ZSTs `self.layout`'s capacity is zero. However, it is legal for the
/// backing buffer to be larger than `layout`.
///
+/// - `self.len()` is always less than or equal to `self.capacity()`.
+///
/// - The `Allocator` type `A` of the vector is the exact same `Allocator` type the backing buffer
/// was allocated with (and must be freed with).
pub struct Vec<T, A: Allocator> {
@@ -183,17 +188,38 @@ where
self.len
}
- /// Forcefully sets `self.len` to `new_len`.
+ /// Increments `self.len` by `additional`.
///
/// # Safety
///
- /// - `new_len` must be less than or equal to [`Self::capacity`].
- /// - If `new_len` is greater than `self.len`, all elements within the interval
- /// [`self.len`,`new_len`) must be initialized.
+ /// - `additional` must be less than or equal to `self.capacity - self.len`.
+ /// - All elements within the interval [`self.len`,`self.len + additional`) must be initialized.
#[inline]
- pub unsafe fn set_len(&mut self, new_len: usize) {
- debug_assert!(new_len <= self.capacity());
- self.len = new_len;
+ pub unsafe fn inc_len(&mut self, additional: usize) {
+ // Guaranteed by the type invariant to never underflow.
+ debug_assert!(additional <= self.capacity() - self.len());
+ // INVARIANT: By the safety requirements of this method this represents the exact number of
+ // elements stored within `self`.
+ self.len += additional;
+ }
+
+ /// Decreases `self.len` by `count`.
+ ///
+ /// Returns a mutable slice to the elements forgotten by the vector. It is the caller's
+ /// responsibility to drop these elements if necessary.
+ ///
+ /// # Safety
+ ///
+ /// - `count` must be less than or equal to `self.len`.
+ unsafe fn dec_len(&mut self, count: usize) -> &mut [T] {
+ debug_assert!(count <= self.len());
+ // INVARIANT: We relinquish ownership of the elements within the range `[self.len - count,
+ // self.len)`, hence the updated value of `set.len` represents the exact number of elements
+ // stored within `self`.
+ self.len -= count;
+ // SAFETY: The memory after `self.len()` is guaranteed to contain `count` initialized
+ // elements of type `T`.
+ unsafe { slice::from_raw_parts_mut(self.as_mut_ptr().add(self.len), count) }
}
/// Returns a slice of the entire vector.
@@ -259,8 +285,8 @@ where
/// Returns a slice of `MaybeUninit<T>` for the remaining spare capacity of the vector.
pub fn spare_capacity_mut(&mut self) -> &mut [MaybeUninit<T>] {
// SAFETY:
- // - `self.len` is smaller than `self.capacity` and hence, the resulting pointer is
- // guaranteed to be part of the same allocated object.
+ // - `self.len` is smaller than `self.capacity` by the type invariant and hence, the
+ // resulting pointer is guaranteed to be part of the same allocated object.
// - `self.len` can not overflow `isize`.
let ptr = unsafe { self.as_mut_ptr().add(self.len) } as *mut MaybeUninit<T>;
@@ -284,24 +310,170 @@ where
/// ```
pub fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError> {
self.reserve(1, flags)?;
+ // SAFETY: The call to `reserve` was successful, so the capacity is at least one greater
+ // than the length.
+ unsafe { self.push_within_capacity_unchecked(v) };
+ Ok(())
+ }
- // SAFETY:
- // - `self.len` is smaller than `self.capacity` and hence, the resulting pointer is
- // guaranteed to be part of the same allocated object.
- // - `self.len` can not overflow `isize`.
- let ptr = unsafe { self.as_mut_ptr().add(self.len) };
+ /// Appends an element to the back of the [`Vec`] instance without reallocating.
+ ///
+ /// Fails if the vector does not have capacity for the new element.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// let mut v = KVec::with_capacity(10, GFP_KERNEL)?;
+ /// for i in 0..10 {
+ /// v.push_within_capacity(i)?;
+ /// }
+ ///
+ /// assert!(v.push_within_capacity(10).is_err());
+ /// # Ok::<(), Error>(())
+ /// ```
+ pub fn push_within_capacity(&mut self, v: T) -> Result<(), PushError<T>> {
+ if self.len() < self.capacity() {
+ // SAFETY: The length is less than the capacity.
+ unsafe { self.push_within_capacity_unchecked(v) };
+ Ok(())
+ } else {
+ Err(PushError(v))
+ }
+ }
- // SAFETY:
- // - `ptr` is properly aligned and valid for writes.
- unsafe { core::ptr::write(ptr, v) };
+ /// Appends an element to the back of the [`Vec`] instance without reallocating.
+ ///
+ /// # Safety
+ ///
+ /// The length must be less than the capacity.
+ unsafe fn push_within_capacity_unchecked(&mut self, v: T) {
+ let spare = self.spare_capacity_mut();
+
+ // SAFETY: By the safety requirements, `spare` is non-empty.
+ unsafe { spare.get_unchecked_mut(0) }.write(v);
// SAFETY: We just initialised the first spare entry, so it is safe to increase the length
- // by 1. We also know that the new length is <= capacity because of the previous call to
- // `reserve` above.
- unsafe { self.set_len(self.len() + 1) };
+ // by 1. We also know that the new length is <= capacity because the caller guarantees that
+ // the length is less than the capacity at the beginning of this function.
+ unsafe { self.inc_len(1) };
+ }
+
+ /// Inserts an element at the given index in the [`Vec`] instance.
+ ///
+ /// Fails if the vector does not have capacity for the new element. Panics if the index is out
+ /// of bounds.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use kernel::alloc::kvec::InsertError;
+ ///
+ /// let mut v = KVec::with_capacity(5, GFP_KERNEL)?;
+ /// for i in 0..5 {
+ /// v.insert_within_capacity(0, i)?;
+ /// }
+ ///
+ /// assert!(matches!(v.insert_within_capacity(0, 5), Err(InsertError::OutOfCapacity(_))));
+ /// assert!(matches!(v.insert_within_capacity(1000, 5), Err(InsertError::IndexOutOfBounds(_))));
+ /// assert_eq!(v, [4, 3, 2, 1, 0]);
+ /// # Ok::<(), Error>(())
+ /// ```
+ pub fn insert_within_capacity(
+ &mut self,
+ index: usize,
+ element: T,
+ ) -> Result<(), InsertError<T>> {
+ let len = self.len();
+ if index > len {
+ return Err(InsertError::IndexOutOfBounds(element));
+ }
+
+ if len >= self.capacity() {
+ return Err(InsertError::OutOfCapacity(element));
+ }
+
+ // SAFETY: This is in bounds since `index <= len < capacity`.
+ let p = unsafe { self.as_mut_ptr().add(index) };
+ // INVARIANT: This breaks the Vec invariants by making `index` contain an invalid element,
+ // but we restore the invariants below.
+ // SAFETY: Both the src and dst ranges end no later than one element after the length.
+ // Since the length is less than the capacity, both ranges are in bounds of the allocation.
+ unsafe { ptr::copy(p, p.add(1), len - index) };
+ // INVARIANT: This restores the Vec invariants.
+ // SAFETY: The pointer is in-bounds of the allocation.
+ unsafe { ptr::write(p, element) };
+ // SAFETY: Index `len` contains a valid element due to the above copy and write.
+ unsafe { self.inc_len(1) };
Ok(())
}
+ /// Removes the last element from a vector and returns it, or `None` if it is empty.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// let mut v = KVec::new();
+ /// v.push(1, GFP_KERNEL)?;
+ /// v.push(2, GFP_KERNEL)?;
+ /// assert_eq!(&v, &[1, 2]);
+ ///
+ /// assert_eq!(v.pop(), Some(2));
+ /// assert_eq!(v.pop(), Some(1));
+ /// assert_eq!(v.pop(), None);
+ /// # Ok::<(), Error>(())
+ /// ```
+ pub fn pop(&mut self) -> Option<T> {
+ if self.is_empty() {
+ return None;
+ }
+
+ let removed: *mut T = {
+ // SAFETY: We just checked that the length is at least one.
+ let slice = unsafe { self.dec_len(1) };
+ // SAFETY: The argument to `dec_len` was 1 so this returns a slice of length 1.
+ unsafe { slice.get_unchecked_mut(0) }
+ };
+
+ // SAFETY: The guarantees of `dec_len` allow us to take ownership of this value.
+ Some(unsafe { removed.read() })
+ }
+
+ /// Removes the element at the given index.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// let mut v = kernel::kvec![1, 2, 3]?;
+ /// assert_eq!(v.remove(1)?, 2);
+ /// assert_eq!(v, [1, 3]);
+ /// # Ok::<(), Error>(())
+ /// ```
+ pub fn remove(&mut self, i: usize) -> Result<T, RemoveError> {
+ let value = {
+ let value_ref = self.get(i).ok_or(RemoveError)?;
+ // INVARIANT: This breaks the invariants by invalidating the value at index `i`, but we
+ // restore the invariants below.
+ // SAFETY: The value at index `i` is valid, because otherwise we would have already
+ // failed with `RemoveError`.
+ unsafe { ptr::read(value_ref) }
+ };
+
+ // SAFETY: We checked that `i` is in-bounds.
+ let p = unsafe { self.as_mut_ptr().add(i) };
+
+ // INVARIANT: After this call, the invalid value is at the last slot, so the Vec invariants
+ // are restored after the below call to `dec_len(1)`.
+ // SAFETY: `p.add(1).add(self.len - i - 1)` is `i+1+len-i-1 == len` elements after the
+ // beginning of the vector, so this is in-bounds of the vector's allocation.
+ unsafe { ptr::copy(p.add(1), p, self.len - i - 1) };
+
+ // SAFETY: Since the check at the beginning of this call did not fail with `RemoveError`,
+ // the length is at least one.
+ unsafe { self.dec_len(1) };
+
+ Ok(value)
+ }
+
/// Creates a new [`Vec`] instance with at least the given capacity.
///
/// # Examples
@@ -395,6 +567,26 @@ where
(ptr, len, capacity)
}
+ /// Clears the vector, removing all values.
+ ///
+ /// Note that this method has no effect on the allocated capacity
+ /// of the vector.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// let mut v = kernel::kvec![1, 2, 3]?;
+ ///
+ /// v.clear();
+ ///
+ /// assert!(v.is_empty());
+ /// # Ok::<(), Error>(())
+ /// ```
+ #[inline]
+ pub fn clear(&mut self) {
+ self.truncate(0);
+ }
+
/// Ensures that the capacity exceeds the length by at least `additional` elements.
///
/// # Examples
@@ -452,6 +644,80 @@ where
Ok(())
}
+
+ /// Shortens the vector, setting the length to `len` and drops the removed values.
+ /// If `len` is greater than or equal to the current length, this does nothing.
+ ///
+ /// This has no effect on the capacity and will not allocate.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// let mut v = kernel::kvec![1, 2, 3]?;
+ /// v.truncate(1);
+ /// assert_eq!(v.len(), 1);
+ /// assert_eq!(&v, &[1]);
+ ///
+ /// # Ok::<(), Error>(())
+ /// ```
+ pub fn truncate(&mut self, len: usize) {
+ if let Some(count) = self.len().checked_sub(len) {
+ // SAFETY: `count` is `self.len() - len` so it is guaranteed to be less than or
+ // equal to `self.len()`.
+ let ptr: *mut [T] = unsafe { self.dec_len(count) };
+
+ // SAFETY: the contract of `dec_len` guarantees that the elements in `ptr` are
+ // valid elements whose ownership has been transferred to the caller.
+ unsafe { ptr::drop_in_place(ptr) };
+ }
+ }
+
+ /// Takes ownership of all items in this vector without consuming the allocation.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// let mut v = kernel::kvec![0, 1, 2, 3]?;
+ ///
+ /// for (i, j) in v.drain_all().enumerate() {
+ /// assert_eq!(i, j);
+ /// }
+ ///
+ /// assert!(v.capacity() >= 4);
+ /// # Ok::<(), Error>(())
+ /// ```
+ pub fn drain_all(&mut self) -> DrainAll<'_, T> {
+ // SAFETY: This does not underflow the length.
+ let elems = unsafe { self.dec_len(self.len()) };
+ // INVARIANT: The first `len` elements of the spare capacity are valid values, and as we
+ // just set the length to zero, we may transfer ownership to the `DrainAll` object.
+ DrainAll {
+ elements: elems.iter_mut(),
+ }
+ }
+
+ /// Removes all elements that don't match the provided closure.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// let mut v = kernel::kvec![1, 2, 3, 4]?;
+ /// v.retain(|i| *i % 2 == 0);
+ /// assert_eq!(v, [2, 4]);
+ /// # Ok::<(), Error>(())
+ /// ```
+ pub fn retain(&mut self, mut f: impl FnMut(&mut T) -> bool) {
+ let mut num_kept = 0;
+ let mut next_to_check = 0;
+ while let Some(to_check) = self.get_mut(next_to_check) {
+ if f(to_check) {
+ self.swap(num_kept, next_to_check);
+ num_kept += 1;
+ }
+ next_to_check += 1;
+ }
+ self.truncate(num_kept);
+ }
}
impl<T: Clone, A: Allocator> Vec<T, A> {
@@ -475,7 +741,7 @@ impl<T: Clone, A: Allocator> Vec<T, A> {
// SAFETY:
// - `self.len() + n < self.capacity()` due to the call to reserve above,
// - the loop and the line above initialized the next `n` elements.
- unsafe { self.set_len(self.len() + n) };
+ unsafe { self.inc_len(n) };
Ok(())
}
@@ -506,7 +772,7 @@ impl<T: Clone, A: Allocator> Vec<T, A> {
// the length by the same number.
// - `self.len() + other.len() <= self.capacity()` is guaranteed by the preceding `reserve`
// call.
- unsafe { self.set_len(self.len() + other.len()) };
+ unsafe { self.inc_len(other.len()) };
Ok(())
}
@@ -518,6 +784,33 @@ impl<T: Clone, A: Allocator> Vec<T, A> {
Ok(v)
}
+
+ /// Resizes the [`Vec`] so that `len` is equal to `new_len`.
+ ///
+ /// If `new_len` is smaller than `len`, the `Vec` is [`Vec::truncate`]d.
+ /// If `new_len` is larger, each new slot is filled with clones of `value`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// let mut v = kernel::kvec![1, 2, 3]?;
+ /// v.resize(1, 42, GFP_KERNEL)?;
+ /// assert_eq!(&v, &[1]);
+ ///
+ /// v.resize(3, 42, GFP_KERNEL)?;
+ /// assert_eq!(&v, &[1, 42, 42]);
+ ///
+ /// # Ok::<(), Error>(())
+ /// ```
+ pub fn resize(&mut self, new_len: usize, value: T, flags: Flags) -> Result<(), AllocError> {
+ match new_len.checked_sub(self.len()) {
+ Some(n) => self.extend_with(n, value, flags),
+ None => {
+ self.truncate(new_len);
+ Ok(())
+ }
+ }
+ }
}
impl<T, A> Drop for Vec<T, A>
@@ -757,12 +1050,13 @@ where
unsafe { ptr::copy(ptr, buf.as_ptr(), len) };
ptr = buf.as_ptr();
- // SAFETY: `len` is guaranteed to be smaller than `self.layout.len()`.
+ // SAFETY: `len` is guaranteed to be smaller than `self.layout.len()` by the type
+ // invariant.
let layout = unsafe { ArrayLayout::<T>::new_unchecked(len) };
- // SAFETY: `buf` points to the start of the backing buffer and `len` is guaranteed to be
- // smaller than `cap`. Depending on `alloc` this operation may shrink the buffer or leaves
- // it as it is.
+ // SAFETY: `buf` points to the start of the backing buffer and `len` is guaranteed by
+ // the type invariant to be smaller than `cap`. Depending on `realloc` this operation
+ // may shrink the buffer or leave it as it is.
ptr = match unsafe {
A::realloc(Some(buf.cast()), layout.into(), old_layout.into(), flags)
} {
@@ -911,3 +1205,87 @@ where
}
}
}
+
+/// An iterator that owns all items in a vector, but does not own its allocation.
+///
+/// # Invariants
+///
+/// Every `&mut T` returned by the iterator references a `T` that the iterator may take ownership
+/// of.
+pub struct DrainAll<'vec, T> {
+ elements: slice::IterMut<'vec, T>,
+}
+
+impl<'vec, T> Iterator for DrainAll<'vec, T> {
+ type Item = T;
+
+ fn next(&mut self) -> Option<T> {
+ let elem: *mut T = self.elements.next()?;
+ // SAFETY: By the type invariants, we may take ownership of this value.
+ Some(unsafe { elem.read() })
+ }
+
+ fn size_hint(&self) -> (usize, Option<usize>) {
+ self.elements.size_hint()
+ }
+}
+
+impl<'vec, T> Drop for DrainAll<'vec, T> {
+ fn drop(&mut self) {
+ if core::mem::needs_drop::<T>() {
+ let iter = core::mem::take(&mut self.elements);
+ let ptr: *mut [T] = iter.into_slice();
+ // SAFETY: By the type invariants, we own these values so we may destroy them.
+ unsafe { ptr::drop_in_place(ptr) };
+ }
+ }
+}
+
+#[macros::kunit_tests(rust_kvec_kunit)]
+mod tests {
+ use super::*;
+ use crate::prelude::*;
+
+ #[test]
+ fn test_kvec_retain() {
+ /// Verify correctness for one specific function.
+ #[expect(clippy::needless_range_loop)]
+ fn verify(c: &[bool]) {
+ let mut vec1: KVec<usize> = KVec::with_capacity(c.len(), GFP_KERNEL).unwrap();
+ let mut vec2: KVec<usize> = KVec::with_capacity(c.len(), GFP_KERNEL).unwrap();
+
+ for i in 0..c.len() {
+ vec1.push_within_capacity(i).unwrap();
+ if c[i] {
+ vec2.push_within_capacity(i).unwrap();
+ }
+ }
+
+ vec1.retain(|i| c[*i]);
+
+ assert_eq!(vec1, vec2);
+ }
+
+ /// Add one to a binary integer represented as a boolean array.
+ fn add(value: &mut [bool]) {
+ let mut carry = true;
+ for v in value {
+ let new_v = carry != *v;
+ carry = carry && *v;
+ *v = new_v;
+ }
+ }
+
+ // This boolean array represents a function from index to boolean. We check that `retain`
+ // behaves correctly for all possible boolean arrays of every possible length less than
+ // ten.
+ let mut func = KVec::with_capacity(10, GFP_KERNEL).unwrap();
+ for len in 0..10 {
+ for _ in 0u32..1u32 << len {
+ verify(&func);
+ add(&mut func);
+ }
+ func.push_within_capacity(false).unwrap();
+ }
+ }
+}
diff --git a/rust/kernel/alloc/kvec/errors.rs b/rust/kernel/alloc/kvec/errors.rs
new file mode 100644
index 000000000000..348b8d27e102
--- /dev/null
+++ b/rust/kernel/alloc/kvec/errors.rs
@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Errors for the [`Vec`] type.
+
+use core::fmt::{self, Debug, Formatter};
+use kernel::prelude::*;
+
+/// Error type for [`Vec::push_within_capacity`].
+pub struct PushError<T>(pub T);
+
+impl<T> Debug for PushError<T> {
+ fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+ write!(f, "Not enough capacity")
+ }
+}
+
+impl<T> From<PushError<T>> for Error {
+ fn from(_: PushError<T>) -> Error {
+ // Returning ENOMEM isn't appropriate because the system is not out of memory. The vector
+ // is just full and we are refusing to resize it.
+ EINVAL
+ }
+}
+
+/// Error type for [`Vec::remove`].
+pub struct RemoveError;
+
+impl Debug for RemoveError {
+ fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+ write!(f, "Index out of bounds")
+ }
+}
+
+impl From<RemoveError> for Error {
+ fn from(_: RemoveError) -> Error {
+ EINVAL
+ }
+}
+
+/// Error type for [`Vec::insert_within_capacity`].
+pub enum InsertError<T> {
+ /// The value could not be inserted because the index is out of bounds.
+ IndexOutOfBounds(T),
+ /// The value could not be inserted because the vector is out of capacity.
+ OutOfCapacity(T),
+}
+
+impl<T> Debug for InsertError<T> {
+ fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+ match self {
+ InsertError::IndexOutOfBounds(_) => write!(f, "Index out of bounds"),
+ InsertError::OutOfCapacity(_) => write!(f, "Not enough capacity"),
+ }
+ }
+}
+
+impl<T> From<InsertError<T>> for Error {
+ fn from(_: InsertError<T>) -> Error {
+ EINVAL
+ }
+}
diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs
new file mode 100644
index 000000000000..d2cfe1eeefb6
--- /dev/null
+++ b/rust/kernel/auxiliary.rs
@@ -0,0 +1,362 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Abstractions for the auxiliary bus.
+//!
+//! C header: [`include/linux/auxiliary_bus.h`](srctree/include/linux/auxiliary_bus.h)
+
+use crate::{
+ bindings, container_of, device,
+ device_id::RawDeviceId,
+ driver,
+ error::{to_result, Result},
+ prelude::*,
+ str::CStr,
+ types::{ForeignOwnable, Opaque},
+ ThisModule,
+};
+use core::{
+ marker::PhantomData,
+ ptr::{addr_of_mut, NonNull},
+};
+
+/// An adapter for the registration of auxiliary drivers.
+pub struct Adapter<T: Driver>(T);
+
+// SAFETY: A call to `unregister` for a given instance of `RegType` is guaranteed to be valid if
+// a preceding call to `register` has been successful.
+unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> {
+ type RegType = bindings::auxiliary_driver;
+
+ unsafe fn register(
+ adrv: &Opaque<Self::RegType>,
+ name: &'static CStr,
+ module: &'static ThisModule,
+ ) -> Result {
+ // SAFETY: It's safe to set the fields of `struct auxiliary_driver` on initialization.
+ unsafe {
+ (*adrv.get()).name = name.as_char_ptr();
+ (*adrv.get()).probe = Some(Self::probe_callback);
+ (*adrv.get()).remove = Some(Self::remove_callback);
+ (*adrv.get()).id_table = T::ID_TABLE.as_ptr();
+ }
+
+ // SAFETY: `adrv` is guaranteed to be a valid `RegType`.
+ to_result(unsafe {
+ bindings::__auxiliary_driver_register(adrv.get(), module.0, name.as_char_ptr())
+ })
+ }
+
+ unsafe fn unregister(adrv: &Opaque<Self::RegType>) {
+ // SAFETY: `adrv` is guaranteed to be a valid `RegType`.
+ unsafe { bindings::auxiliary_driver_unregister(adrv.get()) }
+ }
+}
+
+impl<T: Driver + 'static> Adapter<T> {
+ extern "C" fn probe_callback(
+ adev: *mut bindings::auxiliary_device,
+ id: *const bindings::auxiliary_device_id,
+ ) -> kernel::ffi::c_int {
+ // SAFETY: The auxiliary bus only ever calls the probe callback with a valid pointer to a
+ // `struct auxiliary_device`.
+ //
+ // INVARIANT: `adev` is valid for the duration of `probe_callback()`.
+ let adev = unsafe { &*adev.cast::<Device<device::Core>>() };
+
+ // SAFETY: `DeviceId` is a `#[repr(transparent)`] wrapper of `struct auxiliary_device_id`
+ // and does not add additional invariants, so it's safe to transmute.
+ let id = unsafe { &*id.cast::<DeviceId>() };
+ let info = T::ID_TABLE.info(id.index());
+
+ match T::probe(adev, info) {
+ Ok(data) => {
+ // Let the `struct auxiliary_device` own a reference of the driver's private data.
+ // SAFETY: By the type invariant `adev.as_raw` returns a valid pointer to a
+ // `struct auxiliary_device`.
+ unsafe {
+ bindings::auxiliary_set_drvdata(adev.as_raw(), data.into_foreign().cast())
+ };
+ }
+ Err(err) => return Error::to_errno(err),
+ }
+
+ 0
+ }
+
+ extern "C" fn remove_callback(adev: *mut bindings::auxiliary_device) {
+ // SAFETY: The auxiliary bus only ever calls the remove callback with a valid pointer to a
+ // `struct auxiliary_device`.
+ let ptr = unsafe { bindings::auxiliary_get_drvdata(adev) };
+
+ // SAFETY: `remove_callback` is only ever called after a successful call to
+ // `probe_callback`, hence it's guaranteed that `ptr` points to a valid and initialized
+ // `KBox<T>` pointer created through `KBox::into_foreign`.
+ drop(unsafe { KBox::<T>::from_foreign(ptr.cast()) });
+ }
+}
+
+/// Declares a kernel module that exposes a single auxiliary driver.
+#[macro_export]
+macro_rules! module_auxiliary_driver {
+ ($($f:tt)*) => {
+ $crate::module_driver!(<T>, $crate::auxiliary::Adapter<T>, { $($f)* });
+ };
+}
+
+/// Abstraction for `bindings::auxiliary_device_id`.
+#[repr(transparent)]
+#[derive(Clone, Copy)]
+pub struct DeviceId(bindings::auxiliary_device_id);
+
+impl DeviceId {
+ /// Create a new [`DeviceId`] from name.
+ pub const fn new(modname: &'static CStr, name: &'static CStr) -> Self {
+ let name = name.as_bytes_with_nul();
+ let modname = modname.as_bytes_with_nul();
+
+ // TODO: Replace with `bindings::auxiliary_device_id::default()` once stabilized for
+ // `const`.
+ //
+ // SAFETY: FFI type is valid to be zero-initialized.
+ let mut id: bindings::auxiliary_device_id = unsafe { core::mem::zeroed() };
+
+ let mut i = 0;
+ while i < modname.len() {
+ id.name[i] = modname[i];
+ i += 1;
+ }
+
+ // Reuse the space of the NULL terminator.
+ id.name[i - 1] = b'.';
+
+ let mut j = 0;
+ while j < name.len() {
+ id.name[i] = name[j];
+ i += 1;
+ j += 1;
+ }
+
+ Self(id)
+ }
+}
+
+// SAFETY:
+// * `DeviceId` is a `#[repr(transparent)`] wrapper of `auxiliary_device_id` and does not add
+// additional invariants, so it's safe to transmute to `RawType`.
+// * `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field.
+unsafe impl RawDeviceId for DeviceId {
+ type RawType = bindings::auxiliary_device_id;
+
+ const DRIVER_DATA_OFFSET: usize =
+ core::mem::offset_of!(bindings::auxiliary_device_id, driver_data);
+
+ fn index(&self) -> usize {
+ self.0.driver_data
+ }
+}
+
+/// IdTable type for auxiliary drivers.
+pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>;
+
+/// Create a auxiliary `IdTable` with its alias for modpost.
+#[macro_export]
+macro_rules! auxiliary_device_table {
+ ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => {
+ const $table_name: $crate::device_id::IdArray<
+ $crate::auxiliary::DeviceId,
+ $id_info_type,
+ { $table_data.len() },
+ > = $crate::device_id::IdArray::new($table_data);
+
+ $crate::module_device_table!("auxiliary", $module_table_name, $table_name);
+ };
+}
+
+/// The auxiliary driver trait.
+///
+/// Drivers must implement this trait in order to get an auxiliary driver registered.
+pub trait Driver {
+ /// The type holding information about each device id supported by the driver.
+ ///
+ /// TODO: Use associated_type_defaults once stabilized:
+ ///
+ /// type IdInfo: 'static = ();
+ type IdInfo: 'static;
+
+ /// The table of device ids supported by the driver.
+ const ID_TABLE: IdTable<Self::IdInfo>;
+
+ /// Auxiliary driver probe.
+ ///
+ /// Called when an auxiliary device is matches a corresponding driver.
+ fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> Result<Pin<KBox<Self>>>;
+}
+
+/// The auxiliary device representation.
+///
+/// This structure represents the Rust abstraction for a C `struct auxiliary_device`. The
+/// implementation abstracts the usage of an already existing C `struct auxiliary_device` within
+/// Rust code that we get passed from the C side.
+///
+/// # Invariants
+///
+/// A [`Device`] instance represents a valid `struct auxiliary_device` created by the C portion of
+/// the kernel.
+#[repr(transparent)]
+pub struct Device<Ctx: device::DeviceContext = device::Normal>(
+ Opaque<bindings::auxiliary_device>,
+ PhantomData<Ctx>,
+);
+
+impl<Ctx: device::DeviceContext> Device<Ctx> {
+ fn as_raw(&self) -> *mut bindings::auxiliary_device {
+ self.0.get()
+ }
+
+ /// Returns the auxiliary device' id.
+ pub fn id(&self) -> u32 {
+ // SAFETY: By the type invariant `self.as_raw()` is a valid pointer to a
+ // `struct auxiliary_device`.
+ unsafe { (*self.as_raw()).id }
+ }
+
+ /// Returns a reference to the parent [`device::Device`], if any.
+ pub fn parent(&self) -> Option<&device::Device> {
+ let ptr: *const Self = self;
+ // CAST: `Device<Ctx: DeviceContext>` types are transparent to each other.
+ let ptr: *const Device = ptr.cast();
+ // SAFETY: `ptr` was derived from `&self`.
+ let this = unsafe { &*ptr };
+
+ this.as_ref().parent()
+ }
+}
+
+impl Device {
+ extern "C" fn release(dev: *mut bindings::device) {
+ // SAFETY: By the type invariant `self.0.as_raw` is a pointer to the `struct device`
+ // embedded in `struct auxiliary_device`.
+ let adev = unsafe { container_of!(dev, bindings::auxiliary_device, dev) };
+
+ // SAFETY: `adev` points to the memory that has been allocated in `Registration::new`, via
+ // `KBox::new(Opaque::<bindings::auxiliary_device>::zeroed(), GFP_KERNEL)`.
+ let _ = unsafe { KBox::<Opaque<bindings::auxiliary_device>>::from_raw(adev.cast()) };
+ }
+}
+
+// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic
+// argument.
+kernel::impl_device_context_deref!(unsafe { Device });
+kernel::impl_device_context_into_aref!(Device);
+
+// SAFETY: Instances of `Device` are always reference-counted.
+unsafe impl crate::types::AlwaysRefCounted for Device {
+ fn inc_ref(&self) {
+ // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero.
+ unsafe { bindings::get_device(self.as_ref().as_raw()) };
+ }
+
+ unsafe fn dec_ref(obj: NonNull<Self>) {
+ // CAST: `Self` a transparent wrapper of `bindings::auxiliary_device`.
+ let adev: *mut bindings::auxiliary_device = obj.cast().as_ptr();
+
+ // SAFETY: By the type invariant of `Self`, `adev` is a pointer to a valid
+ // `struct auxiliary_device`.
+ let dev = unsafe { addr_of_mut!((*adev).dev) };
+
+ // SAFETY: The safety requirements guarantee that the refcount is non-zero.
+ unsafe { bindings::put_device(dev) }
+ }
+}
+
+impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for Device<Ctx> {
+ fn as_ref(&self) -> &device::Device<Ctx> {
+ // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid
+ // `struct auxiliary_device`.
+ let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) };
+
+ // SAFETY: `dev` points to a valid `struct device`.
+ unsafe { device::Device::as_ref(dev) }
+ }
+}
+
+// SAFETY: A `Device` is always reference-counted and can be released from any thread.
+unsafe impl Send for Device {}
+
+// SAFETY: `Device` can be shared among threads because all methods of `Device`
+// (i.e. `Device<Normal>) are thread safe.
+unsafe impl Sync for Device {}
+
+/// The registration of an auxiliary device.
+///
+/// This type represents the registration of a [`struct auxiliary_device`]. When an instance of this
+/// type is dropped, its respective auxiliary device will be unregistered from the system.
+///
+/// # Invariants
+///
+/// `self.0` always holds a valid pointer to an initialized and registered
+/// [`struct auxiliary_device`].
+pub struct Registration(NonNull<bindings::auxiliary_device>);
+
+impl Registration {
+ /// Create and register a new auxiliary device.
+ pub fn new(parent: &device::Device, name: &CStr, id: u32, modname: &CStr) -> Result<Self> {
+ let boxed = KBox::new(Opaque::<bindings::auxiliary_device>::zeroed(), GFP_KERNEL)?;
+ let adev = boxed.get();
+
+ // SAFETY: It's safe to set the fields of `struct auxiliary_device` on initialization.
+ unsafe {
+ (*adev).dev.parent = parent.as_raw();
+ (*adev).dev.release = Some(Device::release);
+ (*adev).name = name.as_char_ptr();
+ (*adev).id = id;
+ }
+
+ // SAFETY: `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`,
+ // which has not been initialized yet.
+ unsafe { bindings::auxiliary_device_init(adev) };
+
+ // Now that `adev` is initialized, leak the `Box`; the corresponding memory will be freed
+ // by `Device::release` when the last reference to the `struct auxiliary_device` is dropped.
+ let _ = KBox::into_raw(boxed);
+
+ // SAFETY:
+ // - `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, which has
+ // been initialialized,
+ // - `modname.as_char_ptr()` is a NULL terminated string.
+ let ret = unsafe { bindings::__auxiliary_device_add(adev, modname.as_char_ptr()) };
+ if ret != 0 {
+ // SAFETY: `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`,
+ // which has been initialialized.
+ unsafe { bindings::auxiliary_device_uninit(adev) };
+
+ return Err(Error::from_errno(ret));
+ }
+
+ // SAFETY: `adev` is guaranteed to be non-null, since the `KBox` was allocated successfully.
+ //
+ // INVARIANT: The device will remain registered until `auxiliary_device_delete()` is called,
+ // which happens in `Self::drop()`.
+ Ok(Self(unsafe { NonNull::new_unchecked(adev) }))
+ }
+}
+
+impl Drop for Registration {
+ fn drop(&mut self) {
+ // SAFETY: By the type invariant of `Self`, `self.0.as_ptr()` is a valid registered
+ // `struct auxiliary_device`.
+ unsafe { bindings::auxiliary_device_delete(self.0.as_ptr()) };
+
+ // This drops the reference we acquired through `auxiliary_device_init()`.
+ //
+ // SAFETY: By the type invariant of `Self`, `self.0.as_ptr()` is a valid registered
+ // `struct auxiliary_device`.
+ unsafe { bindings::auxiliary_device_uninit(self.0.as_ptr()) };
+ }
+}
+
+// SAFETY: A `Registration` of a `struct auxiliary_device` can be released from any thread.
+unsafe impl Send for Registration {}
+
+// SAFETY: `Registration` does not expose any methods or fields that need synchronization.
+unsafe impl Sync for Registration {}
diff --git a/rust/kernel/block/mq/gen_disk.rs b/rust/kernel/block/mq/gen_disk.rs
index 14806e1997fd..cd54cd64ea88 100644
--- a/rust/kernel/block/mq/gen_disk.rs
+++ b/rust/kernel/block/mq/gen_disk.rs
@@ -129,7 +129,7 @@ impl GenDiskBuilder {
get_unique_id: None,
// TODO: Set to THIS_MODULE. Waiting for const_refs_to_static feature to
// be merged (unstable in rustc 1.78 which is staged for linux 6.10)
- // https://github.com/rust-lang/rust/issues/119618
+ // <https://github.com/rust-lang/rust/issues/119618>
owner: core::ptr::null_mut(),
pr_ops: core::ptr::null_mut(),
free_disk: None,
diff --git a/rust/kernel/block/mq/request.rs b/rust/kernel/block/mq/request.rs
index 7943f43b9575..4a5b7ec914ef 100644
--- a/rust/kernel/block/mq/request.rs
+++ b/rust/kernel/block/mq/request.rs
@@ -12,7 +12,7 @@ use crate::{
};
use core::{
marker::PhantomData,
- ptr::{addr_of_mut, NonNull},
+ ptr::NonNull,
sync::atomic::{AtomicU64, Ordering},
};
@@ -187,7 +187,7 @@ impl RequestDataWrapper {
pub(crate) unsafe fn refcount_ptr(this: *mut Self) -> *mut AtomicU64 {
// SAFETY: Because of the safety requirements of this function, the
// field projection is safe.
- unsafe { addr_of_mut!((*this).refcount) }
+ unsafe { &raw mut (*this).refcount }
}
}
diff --git a/rust/kernel/block/mq/tag_set.rs b/rust/kernel/block/mq/tag_set.rs
index 00ddcc71dfa2..bcf4214ad149 100644
--- a/rust/kernel/block/mq/tag_set.rs
+++ b/rust/kernel/block/mq/tag_set.rs
@@ -10,12 +10,11 @@ use crate::{
bindings,
block::mq::{operations::OperationsVTable, request::RequestDataWrapper, Operations},
error,
- prelude::PinInit,
- try_pin_init,
+ prelude::try_pin_init,
types::Opaque,
};
use core::{convert::TryInto, marker::PhantomData};
-use macros::{pin_data, pinned_drop};
+use pin_init::{pin_data, pinned_drop, PinInit};
/// A wrapper for the C `struct blk_mq_tag_set`.
///
diff --git a/rust/kernel/clk.rs b/rust/kernel/clk.rs
new file mode 100644
index 000000000000..6041c6d07527
--- /dev/null
+++ b/rust/kernel/clk.rs
@@ -0,0 +1,334 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Clock abstractions.
+//!
+//! C header: [`include/linux/clk.h`](srctree/include/linux/clk.h)
+//!
+//! Reference: <https://docs.kernel.org/driver-api/clk.html>
+
+use crate::ffi::c_ulong;
+
+/// The frequency unit.
+///
+/// Represents a frequency in hertz, wrapping a [`c_ulong`] value.
+///
+/// ## Examples
+///
+/// ```
+/// use kernel::clk::Hertz;
+///
+/// let hz = 1_000_000_000;
+/// let rate = Hertz(hz);
+///
+/// assert_eq!(rate.as_hz(), hz);
+/// assert_eq!(rate, Hertz(hz));
+/// assert_eq!(rate, Hertz::from_khz(hz / 1_000));
+/// assert_eq!(rate, Hertz::from_mhz(hz / 1_000_000));
+/// assert_eq!(rate, Hertz::from_ghz(hz / 1_000_000_000));
+/// ```
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub struct Hertz(pub c_ulong);
+
+impl Hertz {
+ /// Create a new instance from kilohertz (kHz)
+ pub fn from_khz(khz: c_ulong) -> Self {
+ Self(khz * 1_000)
+ }
+
+ /// Create a new instance from megahertz (MHz)
+ pub fn from_mhz(mhz: c_ulong) -> Self {
+ Self(mhz * 1_000_000)
+ }
+
+ /// Create a new instance from gigahertz (GHz)
+ pub fn from_ghz(ghz: c_ulong) -> Self {
+ Self(ghz * 1_000_000_000)
+ }
+
+ /// Get the frequency in hertz
+ pub fn as_hz(&self) -> c_ulong {
+ self.0
+ }
+
+ /// Get the frequency in kilohertz
+ pub fn as_khz(&self) -> c_ulong {
+ self.0 / 1_000
+ }
+
+ /// Get the frequency in megahertz
+ pub fn as_mhz(&self) -> c_ulong {
+ self.0 / 1_000_000
+ }
+
+ /// Get the frequency in gigahertz
+ pub fn as_ghz(&self) -> c_ulong {
+ self.0 / 1_000_000_000
+ }
+}
+
+impl From<Hertz> for c_ulong {
+ fn from(freq: Hertz) -> Self {
+ freq.0
+ }
+}
+
+#[cfg(CONFIG_COMMON_CLK)]
+mod common_clk {
+ use super::Hertz;
+ use crate::{
+ device::Device,
+ error::{from_err_ptr, to_result, Result},
+ prelude::*,
+ };
+
+ use core::{ops::Deref, ptr};
+
+ /// A reference-counted clock.
+ ///
+ /// Rust abstraction for the C [`struct clk`].
+ ///
+ /// # Invariants
+ ///
+ /// A [`Clk`] instance holds either a pointer to a valid [`struct clk`] created by the C
+ /// portion of the kernel or a NULL pointer.
+ ///
+ /// Instances of this type are reference-counted. Calling [`Clk::get`] ensures that the
+ /// allocation remains valid for the lifetime of the [`Clk`].
+ ///
+ /// ## Examples
+ ///
+ /// The following example demonstrates how to obtain and configure a clock for a device.
+ ///
+ /// ```
+ /// use kernel::c_str;
+ /// use kernel::clk::{Clk, Hertz};
+ /// use kernel::device::Device;
+ /// use kernel::error::Result;
+ ///
+ /// fn configure_clk(dev: &Device) -> Result {
+ /// let clk = Clk::get(dev, Some(c_str!("apb_clk")))?;
+ ///
+ /// clk.prepare_enable()?;
+ ///
+ /// let expected_rate = Hertz::from_ghz(1);
+ ///
+ /// if clk.rate() != expected_rate {
+ /// clk.set_rate(expected_rate)?;
+ /// }
+ ///
+ /// clk.disable_unprepare();
+ /// Ok(())
+ /// }
+ /// ```
+ ///
+ /// [`struct clk`]: https://docs.kernel.org/driver-api/clk.html
+ #[repr(transparent)]
+ pub struct Clk(*mut bindings::clk);
+
+ impl Clk {
+ /// Gets [`Clk`] corresponding to a [`Device`] and a connection id.
+ ///
+ /// Equivalent to the kernel's [`clk_get`] API.
+ ///
+ /// [`clk_get`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_get
+ pub fn get(dev: &Device, name: Option<&CStr>) -> Result<Self> {
+ let con_id = if let Some(name) = name {
+ name.as_ptr()
+ } else {
+ ptr::null()
+ };
+
+ // SAFETY: It is safe to call [`clk_get`] for a valid device pointer.
+ //
+ // INVARIANT: The reference-count is decremented when [`Clk`] goes out of scope.
+ Ok(Self(from_err_ptr(unsafe {
+ bindings::clk_get(dev.as_raw(), con_id)
+ })?))
+ }
+
+ /// Obtain the raw [`struct clk`] pointer.
+ #[inline]
+ pub fn as_raw(&self) -> *mut bindings::clk {
+ self.0
+ }
+
+ /// Enable the clock.
+ ///
+ /// Equivalent to the kernel's [`clk_enable`] API.
+ ///
+ /// [`clk_enable`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_enable
+ #[inline]
+ pub fn enable(&self) -> Result {
+ // SAFETY: By the type invariants, self.as_raw() is a valid argument for
+ // [`clk_enable`].
+ to_result(unsafe { bindings::clk_enable(self.as_raw()) })
+ }
+
+ /// Disable the clock.
+ ///
+ /// Equivalent to the kernel's [`clk_disable`] API.
+ ///
+ /// [`clk_disable`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_disable
+ #[inline]
+ pub fn disable(&self) {
+ // SAFETY: By the type invariants, self.as_raw() is a valid argument for
+ // [`clk_disable`].
+ unsafe { bindings::clk_disable(self.as_raw()) };
+ }
+
+ /// Prepare the clock.
+ ///
+ /// Equivalent to the kernel's [`clk_prepare`] API.
+ ///
+ /// [`clk_prepare`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_prepare
+ #[inline]
+ pub fn prepare(&self) -> Result {
+ // SAFETY: By the type invariants, self.as_raw() is a valid argument for
+ // [`clk_prepare`].
+ to_result(unsafe { bindings::clk_prepare(self.as_raw()) })
+ }
+
+ /// Unprepare the clock.
+ ///
+ /// Equivalent to the kernel's [`clk_unprepare`] API.
+ ///
+ /// [`clk_unprepare`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_unprepare
+ #[inline]
+ pub fn unprepare(&self) {
+ // SAFETY: By the type invariants, self.as_raw() is a valid argument for
+ // [`clk_unprepare`].
+ unsafe { bindings::clk_unprepare(self.as_raw()) };
+ }
+
+ /// Prepare and enable the clock.
+ ///
+ /// Equivalent to calling [`Clk::prepare`] followed by [`Clk::enable`].
+ #[inline]
+ pub fn prepare_enable(&self) -> Result {
+ // SAFETY: By the type invariants, self.as_raw() is a valid argument for
+ // [`clk_prepare_enable`].
+ to_result(unsafe { bindings::clk_prepare_enable(self.as_raw()) })
+ }
+
+ /// Disable and unprepare the clock.
+ ///
+ /// Equivalent to calling [`Clk::disable`] followed by [`Clk::unprepare`].
+ #[inline]
+ pub fn disable_unprepare(&self) {
+ // SAFETY: By the type invariants, self.as_raw() is a valid argument for
+ // [`clk_disable_unprepare`].
+ unsafe { bindings::clk_disable_unprepare(self.as_raw()) };
+ }
+
+ /// Get clock's rate.
+ ///
+ /// Equivalent to the kernel's [`clk_get_rate`] API.
+ ///
+ /// [`clk_get_rate`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_get_rate
+ #[inline]
+ pub fn rate(&self) -> Hertz {
+ // SAFETY: By the type invariants, self.as_raw() is a valid argument for
+ // [`clk_get_rate`].
+ Hertz(unsafe { bindings::clk_get_rate(self.as_raw()) })
+ }
+
+ /// Set clock's rate.
+ ///
+ /// Equivalent to the kernel's [`clk_set_rate`] API.
+ ///
+ /// [`clk_set_rate`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_set_rate
+ #[inline]
+ pub fn set_rate(&self, rate: Hertz) -> Result {
+ // SAFETY: By the type invariants, self.as_raw() is a valid argument for
+ // [`clk_set_rate`].
+ to_result(unsafe { bindings::clk_set_rate(self.as_raw(), rate.as_hz()) })
+ }
+ }
+
+ impl Drop for Clk {
+ fn drop(&mut self) {
+ // SAFETY: By the type invariants, self.as_raw() is a valid argument for [`clk_put`].
+ unsafe { bindings::clk_put(self.as_raw()) };
+ }
+ }
+
+ /// A reference-counted optional clock.
+ ///
+ /// A lightweight wrapper around an optional [`Clk`]. An [`OptionalClk`] represents a [`Clk`]
+ /// that a driver can function without but may improve performance or enable additional
+ /// features when available.
+ ///
+ /// # Invariants
+ ///
+ /// An [`OptionalClk`] instance encapsulates a [`Clk`] with either a valid [`struct clk`] or
+ /// `NULL` pointer.
+ ///
+ /// Instances of this type are reference-counted. Calling [`OptionalClk::get`] ensures that the
+ /// allocation remains valid for the lifetime of the [`OptionalClk`].
+ ///
+ /// ## Examples
+ ///
+ /// The following example demonstrates how to obtain and configure an optional clock for a
+ /// device. The code functions correctly whether or not the clock is available.
+ ///
+ /// ```
+ /// use kernel::c_str;
+ /// use kernel::clk::{OptionalClk, Hertz};
+ /// use kernel::device::Device;
+ /// use kernel::error::Result;
+ ///
+ /// fn configure_clk(dev: &Device) -> Result {
+ /// let clk = OptionalClk::get(dev, Some(c_str!("apb_clk")))?;
+ ///
+ /// clk.prepare_enable()?;
+ ///
+ /// let expected_rate = Hertz::from_ghz(1);
+ ///
+ /// if clk.rate() != expected_rate {
+ /// clk.set_rate(expected_rate)?;
+ /// }
+ ///
+ /// clk.disable_unprepare();
+ /// Ok(())
+ /// }
+ /// ```
+ ///
+ /// [`struct clk`]: https://docs.kernel.org/driver-api/clk.html
+ pub struct OptionalClk(Clk);
+
+ impl OptionalClk {
+ /// Gets [`OptionalClk`] corresponding to a [`Device`] and a connection id.
+ ///
+ /// Equivalent to the kernel's [`clk_get_optional`] API.
+ ///
+ /// [`clk_get_optional`]:
+ /// https://docs.kernel.org/core-api/kernel-api.html#c.clk_get_optional
+ pub fn get(dev: &Device, name: Option<&CStr>) -> Result<Self> {
+ let con_id = if let Some(name) = name {
+ name.as_ptr()
+ } else {
+ ptr::null()
+ };
+
+ // SAFETY: It is safe to call [`clk_get_optional`] for a valid device pointer.
+ //
+ // INVARIANT: The reference-count is decremented when [`OptionalClk`] goes out of
+ // scope.
+ Ok(Self(Clk(from_err_ptr(unsafe {
+ bindings::clk_get_optional(dev.as_raw(), con_id)
+ })?)))
+ }
+ }
+
+ // Make [`OptionalClk`] behave like [`Clk`].
+ impl Deref for OptionalClk {
+ type Target = Clk;
+
+ fn deref(&self) -> &Clk {
+ &self.0
+ }
+ }
+}
+
+#[cfg(CONFIG_COMMON_CLK)]
+pub use common_clk::*;
diff --git a/rust/kernel/configfs.rs b/rust/kernel/configfs.rs
new file mode 100644
index 000000000000..34d0bea4f9a5
--- /dev/null
+++ b/rust/kernel/configfs.rs
@@ -0,0 +1,1049 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! configfs interface: Userspace-driven Kernel Object Configuration
+//!
+//! configfs is an in-memory pseudo file system for configuration of kernel
+//! modules. Please see the [C documentation] for details and intended use of
+//! configfs.
+//!
+//! This module does not support the following configfs features:
+//!
+//! - Items. All group children are groups.
+//! - Symlink support.
+//! - `disconnect_notify` hook.
+//! - Default groups.
+//!
+//! See the [`rust_configfs.rs`] sample for a full example use of this module.
+//!
+//! C header: [`include/linux/configfs.h`](srctree/include/linux/configfs.h)
+//!
+//! # Example
+//!
+//! ```ignore
+//! use kernel::alloc::flags;
+//! use kernel::c_str;
+//! use kernel::configfs_attrs;
+//! use kernel::configfs;
+//! use kernel::new_mutex;
+//! use kernel::page::PAGE_SIZE;
+//! use kernel::sync::Mutex;
+//! use kernel::ThisModule;
+//!
+//! #[pin_data]
+//! struct RustConfigfs {
+//! #[pin]
+//! config: configfs::Subsystem<Configuration>,
+//! }
+//!
+//! impl kernel::InPlaceModule for RustConfigfs {
+//! fn init(_module: &'static ThisModule) -> impl PinInit<Self, Error> {
+//! pr_info!("Rust configfs sample (init)\n");
+//!
+//! let item_type = configfs_attrs! {
+//! container: configfs::Subsystem<Configuration>,
+//! data: Configuration,
+//! attributes: [
+//! message: 0,
+//! bar: 1,
+//! ],
+//! };
+//!
+//! try_pin_init!(Self {
+//! config <- configfs::Subsystem::new(
+//! c_str!("rust_configfs"), item_type, Configuration::new()
+//! ),
+//! })
+//! }
+//! }
+//!
+//! #[pin_data]
+//! struct Configuration {
+//! message: &'static CStr,
+//! #[pin]
+//! bar: Mutex<(KBox<[u8; PAGE_SIZE]>, usize)>,
+//! }
+//!
+//! impl Configuration {
+//! fn new() -> impl PinInit<Self, Error> {
+//! try_pin_init!(Self {
+//! message: c_str!("Hello World\n"),
+//! bar <- new_mutex!((KBox::new([0; PAGE_SIZE], flags::GFP_KERNEL)?, 0)),
+//! })
+//! }
+//! }
+//!
+//! #[vtable]
+//! impl configfs::AttributeOperations<0> for Configuration {
+//! type Data = Configuration;
+//!
+//! fn show(container: &Configuration, page: &mut [u8; PAGE_SIZE]) -> Result<usize> {
+//! pr_info!("Show message\n");
+//! let data = container.message;
+//! page[0..data.len()].copy_from_slice(data);
+//! Ok(data.len())
+//! }
+//! }
+//!
+//! #[vtable]
+//! impl configfs::AttributeOperations<1> for Configuration {
+//! type Data = Configuration;
+//!
+//! fn show(container: &Configuration, page: &mut [u8; PAGE_SIZE]) -> Result<usize> {
+//! pr_info!("Show bar\n");
+//! let guard = container.bar.lock();
+//! let data = guard.0.as_slice();
+//! let len = guard.1;
+//! page[0..len].copy_from_slice(&data[0..len]);
+//! Ok(len)
+//! }
+//!
+//! fn store(container: &Configuration, page: &[u8]) -> Result {
+//! pr_info!("Store bar\n");
+//! let mut guard = container.bar.lock();
+//! guard.0[0..page.len()].copy_from_slice(page);
+//! guard.1 = page.len();
+//! Ok(())
+//! }
+//! }
+//! ```
+//!
+//! [C documentation]: srctree/Documentation/filesystems/configfs.rst
+//! [`rust_configfs.rs`]: srctree/samples/rust/rust_configfs.rs
+
+use crate::alloc::flags;
+use crate::container_of;
+use crate::page::PAGE_SIZE;
+use crate::prelude::*;
+use crate::str::CString;
+use crate::sync::Arc;
+use crate::sync::ArcBorrow;
+use crate::types::Opaque;
+use core::cell::UnsafeCell;
+use core::marker::PhantomData;
+
+/// A configfs subsystem.
+///
+/// This is the top level entrypoint for a configfs hierarchy. To register
+/// with configfs, embed a field of this type into your kernel module struct.
+#[pin_data(PinnedDrop)]
+pub struct Subsystem<Data> {
+ #[pin]
+ subsystem: Opaque<bindings::configfs_subsystem>,
+ #[pin]
+ data: Data,
+}
+
+// SAFETY: We do not provide any operations on `Subsystem`.
+unsafe impl<Data> Sync for Subsystem<Data> {}
+
+// SAFETY: Ownership of `Subsystem` can safely be transferred to other threads.
+unsafe impl<Data> Send for Subsystem<Data> {}
+
+impl<Data> Subsystem<Data> {
+ /// Create an initializer for a [`Subsystem`].
+ ///
+ /// The subsystem will appear in configfs as a directory name given by
+ /// `name`. The attributes available in directory are specified by
+ /// `item_type`.
+ pub fn new(
+ name: &'static CStr,
+ item_type: &'static ItemType<Subsystem<Data>, Data>,
+ data: impl PinInit<Data, Error>,
+ ) -> impl PinInit<Self, Error> {
+ try_pin_init!(Self {
+ subsystem <- pin_init::zeroed().chain(
+ |place: &mut Opaque<bindings::configfs_subsystem>| {
+ // SAFETY: We initialized the required fields of `place.group` above.
+ unsafe {
+ bindings::config_group_init_type_name(
+ &mut (*place.get()).su_group,
+ name.as_ptr(),
+ item_type.as_ptr(),
+ )
+ };
+
+ // SAFETY: `place.su_mutex` is valid for use as a mutex.
+ unsafe {
+ bindings::__mutex_init(
+ &mut (*place.get()).su_mutex,
+ kernel::optional_name!().as_char_ptr(),
+ kernel::static_lock_class!().as_ptr(),
+ )
+ }
+ Ok(())
+ }
+ ),
+ data <- data,
+ })
+ .pin_chain(|this| {
+ crate::error::to_result(
+ // SAFETY: We initialized `this.subsystem` according to C API contract above.
+ unsafe { bindings::configfs_register_subsystem(this.subsystem.get()) },
+ )
+ })
+ }
+}
+
+#[pinned_drop]
+impl<Data> PinnedDrop for Subsystem<Data> {
+ fn drop(self: Pin<&mut Self>) {
+ // SAFETY: We registered `self.subsystem` in the initializer returned by `Self::new`.
+ unsafe { bindings::configfs_unregister_subsystem(self.subsystem.get()) };
+ // SAFETY: We initialized the mutex in `Subsystem::new`.
+ unsafe { bindings::mutex_destroy(&raw mut (*self.subsystem.get()).su_mutex) };
+ }
+}
+
+/// Trait that allows offset calculations for structs that embed a
+/// `bindings::config_group`.
+///
+/// Users of the configfs API should not need to implement this trait.
+///
+/// # Safety
+///
+/// - Implementers of this trait must embed a `bindings::config_group`.
+/// - Methods must be implemented according to method documentation.
+pub unsafe trait HasGroup<Data> {
+ /// Return the address of the `bindings::config_group` embedded in [`Self`].
+ ///
+ /// # Safety
+ ///
+ /// - `this` must be a valid allocation of at least the size of [`Self`].
+ unsafe fn group(this: *const Self) -> *const bindings::config_group;
+
+ /// Return the address of the [`Self`] that `group` is embedded in.
+ ///
+ /// # Safety
+ ///
+ /// - `group` must point to the `bindings::config_group` that is embedded in
+ /// [`Self`].
+ unsafe fn container_of(group: *const bindings::config_group) -> *const Self;
+}
+
+// SAFETY: `Subsystem<Data>` embeds a field of type `bindings::config_group`
+// within the `subsystem` field.
+unsafe impl<Data> HasGroup<Data> for Subsystem<Data> {
+ unsafe fn group(this: *const Self) -> *const bindings::config_group {
+ // SAFETY: By impl and function safety requirement this projection is in bounds.
+ unsafe { &raw const (*(*this).subsystem.get()).su_group }
+ }
+
+ unsafe fn container_of(group: *const bindings::config_group) -> *const Self {
+ // SAFETY: By impl and function safety requirement this projection is in bounds.
+ let c_subsys_ptr = unsafe { container_of!(group, bindings::configfs_subsystem, su_group) };
+ let opaque_ptr = c_subsys_ptr.cast::<Opaque<bindings::configfs_subsystem>>();
+ // SAFETY: By impl and function safety requirement, `opaque_ptr` and the
+ // pointer it returns, are within the same allocation.
+ unsafe { container_of!(opaque_ptr, Subsystem<Data>, subsystem) }
+ }
+}
+
+/// A configfs group.
+///
+/// To add a subgroup to configfs, pass this type as `ctype` to
+/// [`crate::configfs_attrs`] when creating a group in [`GroupOperations::make_group`].
+#[pin_data]
+pub struct Group<Data> {
+ #[pin]
+ group: Opaque<bindings::config_group>,
+ #[pin]
+ data: Data,
+}
+
+impl<Data> Group<Data> {
+ /// Create an initializer for a new group.
+ ///
+ /// When instantiated, the group will appear as a directory with the name
+ /// given by `name` and it will contain attributes specified by `item_type`.
+ pub fn new(
+ name: CString,
+ item_type: &'static ItemType<Group<Data>, Data>,
+ data: impl PinInit<Data, Error>,
+ ) -> impl PinInit<Self, Error> {
+ try_pin_init!(Self {
+ group <- pin_init::zeroed().chain(|v: &mut Opaque<bindings::config_group>| {
+ let place = v.get();
+ let name = name.as_bytes_with_nul().as_ptr();
+ // SAFETY: It is safe to initialize a group once it has been zeroed.
+ unsafe {
+ bindings::config_group_init_type_name(place, name.cast(), item_type.as_ptr())
+ };
+ Ok(())
+ }),
+ data <- data,
+ })
+ }
+}
+
+// SAFETY: `Group<Data>` embeds a field of type `bindings::config_group`
+// within the `group` field.
+unsafe impl<Data> HasGroup<Data> for Group<Data> {
+ unsafe fn group(this: *const Self) -> *const bindings::config_group {
+ Opaque::raw_get(
+ // SAFETY: By impl and function safety requirements this field
+ // projection is within bounds of the allocation.
+ unsafe { &raw const (*this).group },
+ )
+ }
+
+ unsafe fn container_of(group: *const bindings::config_group) -> *const Self {
+ let opaque_ptr = group.cast::<Opaque<bindings::config_group>>();
+ // SAFETY: By impl and function safety requirement, `opaque_ptr` and
+ // pointer it returns will be in the same allocation.
+ unsafe { container_of!(opaque_ptr, Self, group) }
+ }
+}
+
+/// # Safety
+///
+/// `this` must be a valid pointer.
+///
+/// If `this` does not represent the root group of a configfs subsystem,
+/// `this` must be a pointer to a `bindings::config_group` embedded in a
+/// `Group<Parent>`.
+///
+/// Otherwise, `this` must be a pointer to a `bindings::config_group` that
+/// is embedded in a `bindings::configfs_subsystem` that is embedded in a
+/// `Subsystem<Parent>`.
+unsafe fn get_group_data<'a, Parent>(this: *mut bindings::config_group) -> &'a Parent {
+ // SAFETY: `this` is a valid pointer.
+ let is_root = unsafe { (*this).cg_subsys.is_null() };
+
+ if !is_root {
+ // SAFETY: By C API contact,`this` was returned from a call to
+ // `make_group`. The pointer is known to be embedded within a
+ // `Group<Parent>`.
+ unsafe { &(*Group::<Parent>::container_of(this)).data }
+ } else {
+ // SAFETY: By C API contract, `this` is a pointer to the
+ // `bindings::config_group` field within a `Subsystem<Parent>`.
+ unsafe { &(*Subsystem::container_of(this)).data }
+ }
+}
+
+struct GroupOperationsVTable<Parent, Child>(PhantomData<(Parent, Child)>);
+
+impl<Parent, Child> GroupOperationsVTable<Parent, Child>
+where
+ Parent: GroupOperations<Child = Child>,
+ Child: 'static,
+{
+ /// # Safety
+ ///
+ /// `this` must be a valid pointer.
+ ///
+ /// If `this` does not represent the root group of a configfs subsystem,
+ /// `this` must be a pointer to a `bindings::config_group` embedded in a
+ /// `Group<Parent>`.
+ ///
+ /// Otherwise, `this` must be a pointer to a `bindings::config_group` that
+ /// is embedded in a `bindings::configfs_subsystem` that is embedded in a
+ /// `Subsystem<Parent>`.
+ ///
+ /// `name` must point to a null terminated string.
+ unsafe extern "C" fn make_group(
+ this: *mut bindings::config_group,
+ name: *const kernel::ffi::c_char,
+ ) -> *mut bindings::config_group {
+ // SAFETY: By function safety requirements of this function, this call
+ // is safe.
+ let parent_data = unsafe { get_group_data(this) };
+
+ let group_init = match Parent::make_group(
+ parent_data,
+ // SAFETY: By function safety requirements, name points to a null
+ // terminated string.
+ unsafe { CStr::from_char_ptr(name) },
+ ) {
+ Ok(init) => init,
+ Err(e) => return e.to_ptr(),
+ };
+
+ let child_group = <Arc<Group<Child>> as InPlaceInit<Group<Child>>>::try_pin_init(
+ group_init,
+ flags::GFP_KERNEL,
+ );
+
+ match child_group {
+ Ok(child_group) => {
+ let child_group_ptr = child_group.into_raw();
+ // SAFETY: We allocated the pointee of `child_ptr` above as a
+ // `Group<Child>`.
+ unsafe { Group::<Child>::group(child_group_ptr) }.cast_mut()
+ }
+ Err(e) => e.to_ptr(),
+ }
+ }
+
+ /// # Safety
+ ///
+ /// If `this` does not represent the root group of a configfs subsystem,
+ /// `this` must be a pointer to a `bindings::config_group` embedded in a
+ /// `Group<Parent>`.
+ ///
+ /// Otherwise, `this` must be a pointer to a `bindings::config_group` that
+ /// is embedded in a `bindings::configfs_subsystem` that is embedded in a
+ /// `Subsystem<Parent>`.
+ ///
+ /// `item` must point to a `bindings::config_item` within a
+ /// `bindings::config_group` within a `Group<Child>`.
+ unsafe extern "C" fn drop_item(
+ this: *mut bindings::config_group,
+ item: *mut bindings::config_item,
+ ) {
+ // SAFETY: By function safety requirements of this function, this call
+ // is safe.
+ let parent_data = unsafe { get_group_data(this) };
+
+ // SAFETY: By function safety requirements, `item` is embedded in a
+ // `config_group`.
+ let c_child_group_ptr = unsafe { container_of!(item, bindings::config_group, cg_item) };
+ // SAFETY: By function safety requirements, `c_child_group_ptr` is
+ // embedded within a `Group<Child>`.
+ let r_child_group_ptr = unsafe { Group::<Child>::container_of(c_child_group_ptr) };
+
+ if Parent::HAS_DROP_ITEM {
+ // SAFETY: We called `into_raw` to produce `r_child_group_ptr` in
+ // `make_group`.
+ let arc: Arc<Group<Child>> = unsafe { Arc::from_raw(r_child_group_ptr.cast_mut()) };
+
+ Parent::drop_item(parent_data, arc.as_arc_borrow());
+ arc.into_raw();
+ }
+
+ // SAFETY: By C API contract, we are required to drop a refcount on
+ // `item`.
+ unsafe { bindings::config_item_put(item) };
+ }
+
+ const VTABLE: bindings::configfs_group_operations = bindings::configfs_group_operations {
+ make_item: None,
+ make_group: Some(Self::make_group),
+ disconnect_notify: None,
+ drop_item: Some(Self::drop_item),
+ is_visible: None,
+ is_bin_visible: None,
+ };
+
+ const fn vtable_ptr() -> *const bindings::configfs_group_operations {
+ &Self::VTABLE as *const bindings::configfs_group_operations
+ }
+}
+
+struct ItemOperationsVTable<Container, Data>(PhantomData<(Container, Data)>);
+
+impl<Data> ItemOperationsVTable<Group<Data>, Data>
+where
+ Data: 'static,
+{
+ /// # Safety
+ ///
+ /// `this` must be a pointer to a `bindings::config_group` embedded in a
+ /// `Group<Parent>`.
+ ///
+ /// This function will destroy the pointee of `this`. The pointee of `this`
+ /// must not be accessed after the function returns.
+ unsafe extern "C" fn release(this: *mut bindings::config_item) {
+ // SAFETY: By function safety requirements, `this` is embedded in a
+ // `config_group`.
+ let c_group_ptr = unsafe { kernel::container_of!(this, bindings::config_group, cg_item) };
+ // SAFETY: By function safety requirements, `c_group_ptr` is
+ // embedded within a `Group<Data>`.
+ let r_group_ptr = unsafe { Group::<Data>::container_of(c_group_ptr) };
+
+ // SAFETY: We called `into_raw` on `r_group_ptr` in
+ // `make_group`.
+ let pin_self: Arc<Group<Data>> = unsafe { Arc::from_raw(r_group_ptr.cast_mut()) };
+ drop(pin_self);
+ }
+
+ const VTABLE: bindings::configfs_item_operations = bindings::configfs_item_operations {
+ release: Some(Self::release),
+ allow_link: None,
+ drop_link: None,
+ };
+
+ const fn vtable_ptr() -> *const bindings::configfs_item_operations {
+ &Self::VTABLE as *const bindings::configfs_item_operations
+ }
+}
+
+impl<Data> ItemOperationsVTable<Subsystem<Data>, Data> {
+ const VTABLE: bindings::configfs_item_operations = bindings::configfs_item_operations {
+ release: None,
+ allow_link: None,
+ drop_link: None,
+ };
+
+ const fn vtable_ptr() -> *const bindings::configfs_item_operations {
+ &Self::VTABLE as *const bindings::configfs_item_operations
+ }
+}
+
+/// Operations implemented by configfs groups that can create subgroups.
+///
+/// Implement this trait on structs that embed a [`Subsystem`] or a [`Group`].
+#[vtable]
+pub trait GroupOperations {
+ /// The child data object type.
+ ///
+ /// This group will create subgroups (subdirectories) backed by this kind of
+ /// object.
+ type Child: 'static;
+
+ /// Creates a new subgroup.
+ ///
+ /// The kernel will call this method in response to `mkdir(2)` in the
+ /// directory representing `this`.
+ ///
+ /// To accept the request to create a group, implementations should
+ /// return an initializer of a `Group<Self::Child>`. To prevent creation,
+ /// return a suitable error.
+ fn make_group(&self, name: &CStr) -> Result<impl PinInit<Group<Self::Child>, Error>>;
+
+ /// Prepares the group for removal from configfs.
+ ///
+ /// The kernel will call this method before the directory representing `_child` is removed from
+ /// configfs.
+ ///
+ /// Implementations can use this method to do house keeping before configfs drops its
+ /// reference to `Child`.
+ ///
+ /// NOTE: "drop" in the name of this function is not related to the Rust drop term. Rather, the
+ /// name is inherited from the callback name in the underlying C code.
+ fn drop_item(&self, _child: ArcBorrow<'_, Group<Self::Child>>) {
+ kernel::build_error!(kernel::error::VTABLE_DEFAULT_ERROR)
+ }
+}
+
+/// A configfs attribute.
+///
+/// An attribute appears as a file in configfs, inside a folder that represent
+/// the group that the attribute belongs to.
+#[repr(transparent)]
+pub struct Attribute<const ID: u64, O, Data> {
+ attribute: Opaque<bindings::configfs_attribute>,
+ _p: PhantomData<(O, Data)>,
+}
+
+// SAFETY: We do not provide any operations on `Attribute`.
+unsafe impl<const ID: u64, O, Data> Sync for Attribute<ID, O, Data> {}
+
+// SAFETY: Ownership of `Attribute` can safely be transferred to other threads.
+unsafe impl<const ID: u64, O, Data> Send for Attribute<ID, O, Data> {}
+
+impl<const ID: u64, O, Data> Attribute<ID, O, Data>
+where
+ O: AttributeOperations<ID, Data = Data>,
+{
+ /// # Safety
+ ///
+ /// `item` must be embedded in a `bindings::config_group`.
+ ///
+ /// If `item` does not represent the root group of a configfs subsystem,
+ /// the group must be embedded in a `Group<Data>`.
+ ///
+ /// Otherwise, the group must be a embedded in a
+ /// `bindings::configfs_subsystem` that is embedded in a `Subsystem<Data>`.
+ ///
+ /// `page` must point to a writable buffer of size at least [`PAGE_SIZE`].
+ unsafe extern "C" fn show(
+ item: *mut bindings::config_item,
+ page: *mut kernel::ffi::c_char,
+ ) -> isize {
+ let c_group: *mut bindings::config_group =
+ // SAFETY: By function safety requirements, `item` is embedded in a
+ // `config_group`.
+ unsafe { container_of!(item, bindings::config_group, cg_item) };
+
+ // SAFETY: The function safety requirements for this function satisfy
+ // the conditions for this call.
+ let data: &Data = unsafe { get_group_data(c_group) };
+
+ // SAFETY: By function safety requirements, `page` is writable for `PAGE_SIZE`.
+ let ret = O::show(data, unsafe { &mut *(page as *mut [u8; PAGE_SIZE]) });
+
+ match ret {
+ Ok(size) => size as isize,
+ Err(err) => err.to_errno() as isize,
+ }
+ }
+
+ /// # Safety
+ ///
+ /// `item` must be embedded in a `bindings::config_group`.
+ ///
+ /// If `item` does not represent the root group of a configfs subsystem,
+ /// the group must be embedded in a `Group<Data>`.
+ ///
+ /// Otherwise, the group must be a embedded in a
+ /// `bindings::configfs_subsystem` that is embedded in a `Subsystem<Data>`.
+ ///
+ /// `page` must point to a readable buffer of size at least `size`.
+ unsafe extern "C" fn store(
+ item: *mut bindings::config_item,
+ page: *const kernel::ffi::c_char,
+ size: usize,
+ ) -> isize {
+ let c_group: *mut bindings::config_group =
+ // SAFETY: By function safety requirements, `item` is embedded in a
+ // `config_group`.
+ unsafe { container_of!(item, bindings::config_group, cg_item) };
+
+ // SAFETY: The function safety requirements for this function satisfy
+ // the conditions for this call.
+ let data: &Data = unsafe { get_group_data(c_group) };
+
+ let ret = O::store(
+ data,
+ // SAFETY: By function safety requirements, `page` is readable
+ // for at least `size`.
+ unsafe { core::slice::from_raw_parts(page.cast(), size) },
+ );
+
+ match ret {
+ Ok(()) => size as isize,
+ Err(err) => err.to_errno() as isize,
+ }
+ }
+
+ /// Create a new attribute.
+ ///
+ /// The attribute will appear as a file with name given by `name`.
+ pub const fn new(name: &'static CStr) -> Self {
+ Self {
+ attribute: Opaque::new(bindings::configfs_attribute {
+ ca_name: name.as_char_ptr(),
+ ca_owner: core::ptr::null_mut(),
+ ca_mode: 0o660,
+ show: Some(Self::show),
+ store: if O::HAS_STORE {
+ Some(Self::store)
+ } else {
+ None
+ },
+ }),
+ _p: PhantomData,
+ }
+ }
+}
+
+/// Operations supported by an attribute.
+///
+/// Implement this trait on type and pass that type as generic parameter when
+/// creating an [`Attribute`]. The type carrying the implementation serve no
+/// purpose other than specifying the attribute operations.
+///
+/// This trait must be implemented on the `Data` type of for types that
+/// implement `HasGroup<Data>`. The trait must be implemented once for each
+/// attribute of the group. The constant type parameter `ID` maps the
+/// implementation to a specific `Attribute`. `ID` must be passed when declaring
+/// attributes via the [`kernel::configfs_attrs`] macro, to tie
+/// `AttributeOperations` implementations to concrete named attributes.
+#[vtable]
+pub trait AttributeOperations<const ID: u64 = 0> {
+ /// The type of the object that contains the field that is backing the
+ /// attribute for this operation.
+ type Data;
+
+ /// Renders the value of an attribute.
+ ///
+ /// This function is called by the kernel to read the value of an attribute.
+ ///
+ /// Implementations should write the rendering of the attribute to `page`
+ /// and return the number of bytes written.
+ fn show(data: &Self::Data, page: &mut [u8; PAGE_SIZE]) -> Result<usize>;
+
+ /// Stores the value of an attribute.
+ ///
+ /// This function is called by the kernel to update the value of an attribute.
+ ///
+ /// Implementations should parse the value from `page` and update internal
+ /// state to reflect the parsed value.
+ fn store(_data: &Self::Data, _page: &[u8]) -> Result {
+ kernel::build_error!(kernel::error::VTABLE_DEFAULT_ERROR)
+ }
+}
+
+/// A list of attributes.
+///
+/// This type is used to construct a new [`ItemType`]. It represents a list of
+/// [`Attribute`] that will appear in the directory representing a [`Group`].
+/// Users should not directly instantiate this type, rather they should use the
+/// [`kernel::configfs_attrs`] macro to declare a static set of attributes for a
+/// group.
+///
+/// # Note
+///
+/// Instances of this type are constructed statically at compile by the
+/// [`kernel::configfs_attrs`] macro.
+#[repr(transparent)]
+pub struct AttributeList<const N: usize, Data>(
+ /// Null terminated Array of pointers to [`Attribute`]. The type is [`c_void`]
+ /// to conform to the C API.
+ UnsafeCell<[*mut kernel::ffi::c_void; N]>,
+ PhantomData<Data>,
+);
+
+// SAFETY: Ownership of `AttributeList` can safely be transferred to other threads.
+unsafe impl<const N: usize, Data> Send for AttributeList<N, Data> {}
+
+// SAFETY: We do not provide any operations on `AttributeList` that need synchronization.
+unsafe impl<const N: usize, Data> Sync for AttributeList<N, Data> {}
+
+impl<const N: usize, Data> AttributeList<N, Data> {
+ /// # Safety
+ ///
+ /// This function must only be called by the [`kernel::configfs_attrs`]
+ /// macro.
+ #[doc(hidden)]
+ pub const unsafe fn new() -> Self {
+ Self(UnsafeCell::new([core::ptr::null_mut(); N]), PhantomData)
+ }
+
+ /// # Safety
+ ///
+ /// The caller must ensure that there are no other concurrent accesses to
+ /// `self`. That is, the caller has exclusive access to `self.`
+ #[doc(hidden)]
+ pub const unsafe fn add<const I: usize, const ID: u64, O>(
+ &'static self,
+ attribute: &'static Attribute<ID, O, Data>,
+ ) where
+ O: AttributeOperations<ID, Data = Data>,
+ {
+ // We need a space at the end of our list for a null terminator.
+ const { assert!(I < N - 1, "Invalid attribute index") };
+
+ // SAFETY: By function safety requirements, we have exclusive access to
+ // `self` and the reference created below will be exclusive.
+ unsafe {
+ (&mut *self.0.get())[I] = (attribute as *const Attribute<ID, O, Data>)
+ .cast_mut()
+ .cast()
+ };
+ }
+}
+
+/// A representation of the attributes that will appear in a [`Group`] or
+/// [`Subsystem`].
+///
+/// Users should not directly instantiate objects of this type. Rather, they
+/// should use the [`kernel::configfs_attrs`] macro to statically declare the
+/// shape of a [`Group`] or [`Subsystem`].
+#[pin_data]
+pub struct ItemType<Container, Data> {
+ #[pin]
+ item_type: Opaque<bindings::config_item_type>,
+ _p: PhantomData<(Container, Data)>,
+}
+
+// SAFETY: We do not provide any operations on `ItemType` that need synchronization.
+unsafe impl<Container, Data> Sync for ItemType<Container, Data> {}
+
+// SAFETY: Ownership of `ItemType` can safely be transferred to other threads.
+unsafe impl<Container, Data> Send for ItemType<Container, Data> {}
+
+macro_rules! impl_item_type {
+ ($tpe:ty) => {
+ impl<Data> ItemType<$tpe, Data> {
+ #[doc(hidden)]
+ pub const fn new_with_child_ctor<const N: usize, Child>(
+ owner: &'static ThisModule,
+ attributes: &'static AttributeList<N, Data>,
+ ) -> Self
+ where
+ Data: GroupOperations<Child = Child>,
+ Child: 'static,
+ {
+ Self {
+ item_type: Opaque::new(bindings::config_item_type {
+ ct_owner: owner.as_ptr(),
+ ct_group_ops: GroupOperationsVTable::<Data, Child>::vtable_ptr().cast_mut(),
+ ct_item_ops: ItemOperationsVTable::<$tpe, Data>::vtable_ptr().cast_mut(),
+ ct_attrs: (attributes as *const AttributeList<N, Data>)
+ .cast_mut()
+ .cast(),
+ ct_bin_attrs: core::ptr::null_mut(),
+ }),
+ _p: PhantomData,
+ }
+ }
+
+ #[doc(hidden)]
+ pub const fn new<const N: usize>(
+ owner: &'static ThisModule,
+ attributes: &'static AttributeList<N, Data>,
+ ) -> Self {
+ Self {
+ item_type: Opaque::new(bindings::config_item_type {
+ ct_owner: owner.as_ptr(),
+ ct_group_ops: core::ptr::null_mut(),
+ ct_item_ops: ItemOperationsVTable::<$tpe, Data>::vtable_ptr().cast_mut(),
+ ct_attrs: (attributes as *const AttributeList<N, Data>)
+ .cast_mut()
+ .cast(),
+ ct_bin_attrs: core::ptr::null_mut(),
+ }),
+ _p: PhantomData,
+ }
+ }
+ }
+ };
+}
+
+impl_item_type!(Subsystem<Data>);
+impl_item_type!(Group<Data>);
+
+impl<Container, Data> ItemType<Container, Data> {
+ fn as_ptr(&self) -> *const bindings::config_item_type {
+ self.item_type.get()
+ }
+}
+
+/// Define a list of configfs attributes statically.
+///
+/// Invoking the macro in the following manner:
+///
+/// ```ignore
+/// let item_type = configfs_attrs! {
+/// container: configfs::Subsystem<Configuration>,
+/// data: Configuration,
+/// child: Child,
+/// attributes: [
+/// message: 0,
+/// bar: 1,
+/// ],
+/// };
+/// ```
+///
+/// Expands the following output:
+///
+/// ```ignore
+/// let item_type = {
+/// static CONFIGURATION_MESSAGE_ATTR: kernel::configfs::Attribute<
+/// 0,
+/// Configuration,
+/// Configuration,
+/// > = unsafe {
+/// kernel::configfs::Attribute::new({
+/// const S: &str = "message\u{0}";
+/// const C: &kernel::str::CStr = match kernel::str::CStr::from_bytes_with_nul(
+/// S.as_bytes()
+/// ) {
+/// Ok(v) => v,
+/// Err(_) => {
+/// core::panicking::panic_fmt(core::const_format_args!(
+/// "string contains interior NUL"
+/// ));
+/// }
+/// };
+/// C
+/// })
+/// };
+///
+/// static CONFIGURATION_BAR_ATTR: kernel::configfs::Attribute<
+/// 1,
+/// Configuration,
+/// Configuration
+/// > = unsafe {
+/// kernel::configfs::Attribute::new({
+/// const S: &str = "bar\u{0}";
+/// const C: &kernel::str::CStr = match kernel::str::CStr::from_bytes_with_nul(
+/// S.as_bytes()
+/// ) {
+/// Ok(v) => v,
+/// Err(_) => {
+/// core::panicking::panic_fmt(core::const_format_args!(
+/// "string contains interior NUL"
+/// ));
+/// }
+/// };
+/// C
+/// })
+/// };
+///
+/// const N: usize = (1usize + (1usize + 0usize)) + 1usize;
+///
+/// static CONFIGURATION_ATTRS: kernel::configfs::AttributeList<N, Configuration> =
+/// unsafe { kernel::configfs::AttributeList::new() };
+///
+/// {
+/// const N: usize = 0usize;
+/// unsafe { CONFIGURATION_ATTRS.add::<N, 0, _>(&CONFIGURATION_MESSAGE_ATTR) };
+/// }
+///
+/// {
+/// const N: usize = (1usize + 0usize);
+/// unsafe { CONFIGURATION_ATTRS.add::<N, 1, _>(&CONFIGURATION_BAR_ATTR) };
+/// }
+///
+/// static CONFIGURATION_TPE:
+/// kernel::configfs::ItemType<configfs::Subsystem<Configuration> ,Configuration>
+/// = kernel::configfs::ItemType::<
+/// configfs::Subsystem<Configuration>,
+/// Configuration
+/// >::new_with_child_ctor::<N,Child>(
+/// &THIS_MODULE,
+/// &CONFIGURATION_ATTRS
+/// );
+///
+/// &CONFIGURATION_TPE
+/// }
+/// ```
+#[macro_export]
+macro_rules! configfs_attrs {
+ (
+ container: $container:ty,
+ data: $data:ty,
+ attributes: [
+ $($name:ident: $attr:literal),* $(,)?
+ ] $(,)?
+ ) => {
+ $crate::configfs_attrs!(
+ count:
+ @container($container),
+ @data($data),
+ @child(),
+ @no_child(x),
+ @attrs($($name $attr)*),
+ @eat($($name $attr,)*),
+ @assign(),
+ @cnt(0usize),
+ )
+ };
+ (
+ container: $container:ty,
+ data: $data:ty,
+ child: $child:ty,
+ attributes: [
+ $($name:ident: $attr:literal),* $(,)?
+ ] $(,)?
+ ) => {
+ $crate::configfs_attrs!(
+ count:
+ @container($container),
+ @data($data),
+ @child($child),
+ @no_child(),
+ @attrs($($name $attr)*),
+ @eat($($name $attr,)*),
+ @assign(),
+ @cnt(0usize),
+ )
+ };
+ (count:
+ @container($container:ty),
+ @data($data:ty),
+ @child($($child:ty)?),
+ @no_child($($no_child:ident)?),
+ @attrs($($aname:ident $aattr:literal)*),
+ @eat($name:ident $attr:literal, $($rname:ident $rattr:literal,)*),
+ @assign($($assign:block)*),
+ @cnt($cnt:expr),
+ ) => {
+ $crate::configfs_attrs!(
+ count:
+ @container($container),
+ @data($data),
+ @child($($child)?),
+ @no_child($($no_child)?),
+ @attrs($($aname $aattr)*),
+ @eat($($rname $rattr,)*),
+ @assign($($assign)* {
+ const N: usize = $cnt;
+ // The following macro text expands to a call to `Attribute::add`.
+
+ // SAFETY: By design of this macro, the name of the variable we
+ // invoke the `add` method on below, is not visible outside of
+ // the macro expansion. The macro does not operate concurrently
+ // on this variable, and thus we have exclusive access to the
+ // variable.
+ unsafe {
+ $crate::macros::paste!(
+ [< $data:upper _ATTRS >]
+ .add::<N, $attr, _>(&[< $data:upper _ $name:upper _ATTR >])
+ )
+ };
+ }),
+ @cnt(1usize + $cnt),
+ )
+ };
+ (count:
+ @container($container:ty),
+ @data($data:ty),
+ @child($($child:ty)?),
+ @no_child($($no_child:ident)?),
+ @attrs($($aname:ident $aattr:literal)*),
+ @eat(),
+ @assign($($assign:block)*),
+ @cnt($cnt:expr),
+ ) =>
+ {
+ $crate::configfs_attrs!(
+ final:
+ @container($container),
+ @data($data),
+ @child($($child)?),
+ @no_child($($no_child)?),
+ @attrs($($aname $aattr)*),
+ @assign($($assign)*),
+ @cnt($cnt),
+ )
+ };
+ (final:
+ @container($container:ty),
+ @data($data:ty),
+ @child($($child:ty)?),
+ @no_child($($no_child:ident)?),
+ @attrs($($name:ident $attr:literal)*),
+ @assign($($assign:block)*),
+ @cnt($cnt:expr),
+ ) =>
+ {
+ $crate::macros::paste!{
+ {
+ $(
+ // SAFETY: We are expanding `configfs_attrs`.
+ static [< $data:upper _ $name:upper _ATTR >]:
+ $crate::configfs::Attribute<$attr, $data, $data> =
+ unsafe {
+ $crate::configfs::Attribute::new(c_str!(::core::stringify!($name)))
+ };
+ )*
+
+
+ // We need space for a null terminator.
+ const N: usize = $cnt + 1usize;
+
+ // SAFETY: We are expanding `configfs_attrs`.
+ static [< $data:upper _ATTRS >]:
+ $crate::configfs::AttributeList<N, $data> =
+ unsafe { $crate::configfs::AttributeList::new() };
+
+ $($assign)*
+
+ $(
+ const [<$no_child:upper>]: bool = true;
+
+ static [< $data:upper _TPE >] : $crate::configfs::ItemType<$container, $data> =
+ $crate::configfs::ItemType::<$container, $data>::new::<N>(
+ &THIS_MODULE, &[<$ data:upper _ATTRS >]
+ );
+ )?
+
+ $(
+ static [< $data:upper _TPE >]:
+ $crate::configfs::ItemType<$container, $data> =
+ $crate::configfs::ItemType::<$container, $data>::
+ new_with_child_ctor::<N, $child>(
+ &THIS_MODULE, &[<$ data:upper _ATTRS >]
+ );
+ )?
+
+ & [< $data:upper _TPE >]
+ }
+ }
+ };
+
+}
diff --git a/rust/kernel/cpu.rs b/rust/kernel/cpu.rs
new file mode 100644
index 000000000000..b75403b0eb56
--- /dev/null
+++ b/rust/kernel/cpu.rs
@@ -0,0 +1,151 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Generic CPU definitions.
+//!
+//! C header: [`include/linux/cpu.h`](srctree/include/linux/cpu.h)
+
+use crate::{bindings, device::Device, error::Result, prelude::ENODEV};
+
+/// Returns the maximum number of possible CPUs in the current system configuration.
+#[inline]
+pub fn nr_cpu_ids() -> u32 {
+ #[cfg(any(NR_CPUS_1, CONFIG_FORCE_NR_CPUS))]
+ {
+ bindings::NR_CPUS
+ }
+
+ #[cfg(not(any(NR_CPUS_1, CONFIG_FORCE_NR_CPUS)))]
+ // SAFETY: `nr_cpu_ids` is a valid global provided by the kernel.
+ unsafe {
+ bindings::nr_cpu_ids
+ }
+}
+
+/// The CPU ID.
+///
+/// Represents a CPU identifier as a wrapper around an [`u32`].
+///
+/// # Invariants
+///
+/// The CPU ID lies within the range `[0, nr_cpu_ids())`.
+///
+/// # Examples
+///
+/// ```
+/// use kernel::cpu::CpuId;
+///
+/// let cpu = 0;
+///
+/// // SAFETY: 0 is always a valid CPU number.
+/// let id = unsafe { CpuId::from_u32_unchecked(cpu) };
+///
+/// assert_eq!(id.as_u32(), cpu);
+/// assert!(CpuId::from_i32(0).is_some());
+/// assert!(CpuId::from_i32(-1).is_none());
+/// ```
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub struct CpuId(u32);
+
+impl CpuId {
+ /// Creates a new [`CpuId`] from the given `id` without checking bounds.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `id` is a valid CPU ID (i.e., `0 <= id < nr_cpu_ids()`).
+ #[inline]
+ pub unsafe fn from_i32_unchecked(id: i32) -> Self {
+ debug_assert!(id >= 0);
+ debug_assert!((id as u32) < nr_cpu_ids());
+
+ // INVARIANT: The function safety guarantees `id` is a valid CPU id.
+ Self(id as u32)
+ }
+
+ /// Creates a new [`CpuId`] from the given `id`, checking that it is valid.
+ pub fn from_i32(id: i32) -> Option<Self> {
+ if id < 0 || id as u32 >= nr_cpu_ids() {
+ None
+ } else {
+ // INVARIANT: `id` has just been checked as a valid CPU ID.
+ Some(Self(id as u32))
+ }
+ }
+
+ /// Creates a new [`CpuId`] from the given `id` without checking bounds.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `id` is a valid CPU ID (i.e., `0 <= id < nr_cpu_ids()`).
+ #[inline]
+ pub unsafe fn from_u32_unchecked(id: u32) -> Self {
+ debug_assert!(id < nr_cpu_ids());
+
+ // Ensure the `id` fits in an [`i32`] as it's also representable that way.
+ debug_assert!(id <= i32::MAX as u32);
+
+ // INVARIANT: The function safety guarantees `id` is a valid CPU id.
+ Self(id)
+ }
+
+ /// Creates a new [`CpuId`] from the given `id`, checking that it is valid.
+ pub fn from_u32(id: u32) -> Option<Self> {
+ if id >= nr_cpu_ids() {
+ None
+ } else {
+ // INVARIANT: `id` has just been checked as a valid CPU ID.
+ Some(Self(id))
+ }
+ }
+
+ /// Returns CPU number.
+ #[inline]
+ pub fn as_u32(&self) -> u32 {
+ self.0
+ }
+
+ /// Returns the ID of the CPU the code is currently running on.
+ ///
+ /// The returned value is considered unstable because it may change
+ /// unexpectedly due to preemption or CPU migration. It should only be
+ /// used when the context ensures that the task remains on the same CPU
+ /// or the users could use a stale (yet valid) CPU ID.
+ pub fn current() -> Self {
+ // SAFETY: raw_smp_processor_id() always returns a valid CPU ID.
+ unsafe { Self::from_u32_unchecked(bindings::raw_smp_processor_id()) }
+ }
+}
+
+impl From<CpuId> for u32 {
+ fn from(id: CpuId) -> Self {
+ id.as_u32()
+ }
+}
+
+impl From<CpuId> for i32 {
+ fn from(id: CpuId) -> Self {
+ id.as_u32() as i32
+ }
+}
+
+/// Creates a new instance of CPU's device.
+///
+/// # Safety
+///
+/// Reference counting is not implemented for the CPU device in the C code. When a CPU is
+/// hot-unplugged, the corresponding CPU device is unregistered, but its associated memory
+/// is not freed.
+///
+/// Callers must ensure that the CPU device is not used after it has been unregistered.
+/// This can be achieved, for example, by registering a CPU hotplug notifier and removing
+/// any references to the CPU device within the notifier's callback.
+pub unsafe fn from_cpu(cpu: CpuId) -> Result<&'static Device> {
+ // SAFETY: It is safe to call `get_cpu_device()` for any CPU.
+ let ptr = unsafe { bindings::get_cpu_device(u32::from(cpu)) };
+ if ptr.is_null() {
+ return Err(ENODEV);
+ }
+
+ // SAFETY: The pointer returned by `get_cpu_device()`, if not `NULL`, is a valid pointer to
+ // a `struct device` and is never freed by the C code.
+ Ok(unsafe { Device::as_ref(ptr) })
+}
diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs
new file mode 100644
index 000000000000..11b03e9d7e89
--- /dev/null
+++ b/rust/kernel/cpufreq.rs
@@ -0,0 +1,1404 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! CPU frequency scaling.
+//!
+//! This module provides rust abstractions for interacting with the cpufreq subsystem.
+//!
+//! C header: [`include/linux/cpufreq.h`](srctree/include/linux/cpufreq.h)
+//!
+//! Reference: <https://docs.kernel.org/admin-guide/pm/cpufreq.html>
+
+use crate::{
+ clk::Hertz,
+ cpu::CpuId,
+ cpumask,
+ device::{Bound, Device},
+ devres::Devres,
+ error::{code::*, from_err_ptr, from_result, to_result, Result, VTABLE_DEFAULT_ERROR},
+ ffi::{c_char, c_ulong},
+ prelude::*,
+ types::ForeignOwnable,
+ types::Opaque,
+};
+
+#[cfg(CONFIG_COMMON_CLK)]
+use crate::clk::Clk;
+
+use core::{
+ cell::UnsafeCell,
+ marker::PhantomData,
+ mem::MaybeUninit,
+ ops::{Deref, DerefMut},
+ pin::Pin,
+ ptr,
+};
+
+use macros::vtable;
+
+/// Maximum length of CPU frequency driver's name.
+const CPUFREQ_NAME_LEN: usize = bindings::CPUFREQ_NAME_LEN as usize;
+
+/// Default transition latency value in nanoseconds.
+pub const ETERNAL_LATENCY_NS: u32 = bindings::CPUFREQ_ETERNAL as u32;
+
+/// CPU frequency driver flags.
+pub mod flags {
+ /// Driver needs to update internal limits even if frequency remains unchanged.
+ pub const NEED_UPDATE_LIMITS: u16 = 1 << 0;
+
+ /// Platform where constants like `loops_per_jiffy` are unaffected by frequency changes.
+ pub const CONST_LOOPS: u16 = 1 << 1;
+
+ /// Register driver as a thermal cooling device automatically.
+ pub const IS_COOLING_DEV: u16 = 1 << 2;
+
+ /// Supports multiple clock domains with per-policy governors in `cpu/cpuN/cpufreq/`.
+ pub const HAVE_GOVERNOR_PER_POLICY: u16 = 1 << 3;
+
+ /// Allows post-change notifications outside of the `target()` routine.
+ pub const ASYNC_NOTIFICATION: u16 = 1 << 4;
+
+ /// Ensure CPU starts at a valid frequency from the driver's freq-table.
+ pub const NEED_INITIAL_FREQ_CHECK: u16 = 1 << 5;
+
+ /// Disallow governors with `dynamic_switching` capability.
+ pub const NO_AUTO_DYNAMIC_SWITCHING: u16 = 1 << 6;
+}
+
+/// Relations from the C code.
+const CPUFREQ_RELATION_L: u32 = 0;
+const CPUFREQ_RELATION_H: u32 = 1;
+const CPUFREQ_RELATION_C: u32 = 2;
+
+/// Can be used with any of the above values.
+const CPUFREQ_RELATION_E: u32 = 1 << 2;
+
+/// CPU frequency selection relations.
+///
+/// CPU frequency selection relations, each optionally marked as "efficient".
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+pub enum Relation {
+ /// Select the lowest frequency at or above target.
+ Low(bool),
+ /// Select the highest frequency below or at target.
+ High(bool),
+ /// Select the closest frequency to the target.
+ Close(bool),
+}
+
+impl Relation {
+ // Construct from a C-compatible `u32` value.
+ fn new(val: u32) -> Result<Self> {
+ let efficient = val & CPUFREQ_RELATION_E != 0;
+
+ Ok(match val & !CPUFREQ_RELATION_E {
+ CPUFREQ_RELATION_L => Self::Low(efficient),
+ CPUFREQ_RELATION_H => Self::High(efficient),
+ CPUFREQ_RELATION_C => Self::Close(efficient),
+ _ => return Err(EINVAL),
+ })
+ }
+}
+
+impl From<Relation> for u32 {
+ // Convert to a C-compatible `u32` value.
+ fn from(rel: Relation) -> Self {
+ let (mut val, efficient) = match rel {
+ Relation::Low(e) => (CPUFREQ_RELATION_L, e),
+ Relation::High(e) => (CPUFREQ_RELATION_H, e),
+ Relation::Close(e) => (CPUFREQ_RELATION_C, e),
+ };
+
+ if efficient {
+ val |= CPUFREQ_RELATION_E;
+ }
+
+ val
+ }
+}
+
+/// Policy data.
+///
+/// Rust abstraction for the C `struct cpufreq_policy_data`.
+///
+/// # Invariants
+///
+/// A [`PolicyData`] instance always corresponds to a valid C `struct cpufreq_policy_data`.
+///
+/// The callers must ensure that the `struct cpufreq_policy_data` is valid for access and remains
+/// valid for the lifetime of the returned reference.
+#[repr(transparent)]
+pub struct PolicyData(Opaque<bindings::cpufreq_policy_data>);
+
+impl PolicyData {
+ /// Creates a mutable reference to an existing `struct cpufreq_policy_data` pointer.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime
+ /// of the returned reference.
+ #[inline]
+ pub unsafe fn from_raw_mut<'a>(ptr: *mut bindings::cpufreq_policy_data) -> &'a mut Self {
+ // SAFETY: Guaranteed by the safety requirements of the function.
+ //
+ // INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the
+ // lifetime of the returned reference.
+ unsafe { &mut *ptr.cast() }
+ }
+
+ /// Returns a raw pointer to the underlying C `cpufreq_policy_data`.
+ #[inline]
+ pub fn as_raw(&self) -> *mut bindings::cpufreq_policy_data {
+ let this: *const Self = self;
+ this.cast_mut().cast()
+ }
+
+ /// Wrapper for `cpufreq_generic_frequency_table_verify`.
+ #[inline]
+ pub fn generic_verify(&self) -> Result {
+ // SAFETY: By the type invariant, the pointer stored in `self` is valid.
+ to_result(unsafe { bindings::cpufreq_generic_frequency_table_verify(self.as_raw()) })
+ }
+}
+
+/// The frequency table index.
+///
+/// Represents index with a frequency table.
+///
+/// # Invariants
+///
+/// The index must correspond to a valid entry in the [`Table`] it is used for.
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub struct TableIndex(usize);
+
+impl TableIndex {
+ /// Creates an instance of [`TableIndex`].
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `index` correspond to a valid entry in the [`Table`] it is used
+ /// for.
+ pub unsafe fn new(index: usize) -> Self {
+ // INVARIANT: The caller ensures that `index` correspond to a valid entry in the [`Table`].
+ Self(index)
+ }
+}
+
+impl From<TableIndex> for usize {
+ #[inline]
+ fn from(index: TableIndex) -> Self {
+ index.0
+ }
+}
+
+/// CPU frequency table.
+///
+/// Rust abstraction for the C `struct cpufreq_frequency_table`.
+///
+/// # Invariants
+///
+/// A [`Table`] instance always corresponds to a valid C `struct cpufreq_frequency_table`.
+///
+/// The callers must ensure that the `struct cpufreq_frequency_table` is valid for access and
+/// remains valid for the lifetime of the returned reference.
+///
+/// ## Examples
+///
+/// The following example demonstrates how to read a frequency value from [`Table`].
+///
+/// ```
+/// use kernel::cpufreq::{Policy, TableIndex};
+///
+/// fn show_freq(policy: &Policy) -> Result {
+/// let table = policy.freq_table()?;
+///
+/// // SAFETY: Index is a valid entry in the table.
+/// let index = unsafe { TableIndex::new(0) };
+///
+/// pr_info!("The frequency at index 0 is: {:?}\n", table.freq(index)?);
+/// pr_info!("The flags at index 0 is: {}\n", table.flags(index));
+/// pr_info!("The data at index 0 is: {}\n", table.data(index));
+/// Ok(())
+/// }
+/// ```
+#[repr(transparent)]
+pub struct Table(Opaque<bindings::cpufreq_frequency_table>);
+
+impl Table {
+ /// Creates a reference to an existing C `struct cpufreq_frequency_table` pointer.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime
+ /// of the returned reference.
+ #[inline]
+ pub unsafe fn from_raw<'a>(ptr: *const bindings::cpufreq_frequency_table) -> &'a Self {
+ // SAFETY: Guaranteed by the safety requirements of the function.
+ //
+ // INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the
+ // lifetime of the returned reference.
+ unsafe { &*ptr.cast() }
+ }
+
+ /// Returns the raw mutable pointer to the C `struct cpufreq_frequency_table`.
+ #[inline]
+ pub fn as_raw(&self) -> *mut bindings::cpufreq_frequency_table {
+ let this: *const Self = self;
+ this.cast_mut().cast()
+ }
+
+ /// Returns frequency at `index` in the [`Table`].
+ #[inline]
+ pub fn freq(&self, index: TableIndex) -> Result<Hertz> {
+ // SAFETY: By the type invariant, the pointer stored in `self` is valid and `index` is
+ // guaranteed to be valid by its safety requirements.
+ Ok(Hertz::from_khz(unsafe {
+ (*self.as_raw().add(index.into())).frequency.try_into()?
+ }))
+ }
+
+ /// Returns flags at `index` in the [`Table`].
+ #[inline]
+ pub fn flags(&self, index: TableIndex) -> u32 {
+ // SAFETY: By the type invariant, the pointer stored in `self` is valid and `index` is
+ // guaranteed to be valid by its safety requirements.
+ unsafe { (*self.as_raw().add(index.into())).flags }
+ }
+
+ /// Returns data at `index` in the [`Table`].
+ #[inline]
+ pub fn data(&self, index: TableIndex) -> u32 {
+ // SAFETY: By the type invariant, the pointer stored in `self` is valid and `index` is
+ // guaranteed to be valid by its safety requirements.
+ unsafe { (*self.as_raw().add(index.into())).driver_data }
+ }
+}
+
+/// CPU frequency table owned and pinned in memory, created from a [`TableBuilder`].
+pub struct TableBox {
+ entries: Pin<KVec<bindings::cpufreq_frequency_table>>,
+}
+
+impl TableBox {
+ /// Constructs a new [`TableBox`] from a [`KVec`] of entries.
+ ///
+ /// # Errors
+ ///
+ /// Returns `EINVAL` if the entries list is empty.
+ #[inline]
+ fn new(entries: KVec<bindings::cpufreq_frequency_table>) -> Result<Self> {
+ if entries.is_empty() {
+ return Err(EINVAL);
+ }
+
+ Ok(Self {
+ // Pin the entries to memory, since we are passing its pointer to the C code.
+ entries: Pin::new(entries),
+ })
+ }
+
+ /// Returns a raw pointer to the underlying C `cpufreq_frequency_table`.
+ #[inline]
+ fn as_raw(&self) -> *const bindings::cpufreq_frequency_table {
+ // The pointer is valid until the table gets dropped.
+ self.entries.as_ptr()
+ }
+}
+
+impl Deref for TableBox {
+ type Target = Table;
+
+ fn deref(&self) -> &Self::Target {
+ // SAFETY: The caller owns TableBox, it is safe to deref.
+ unsafe { Self::Target::from_raw(self.as_raw()) }
+ }
+}
+
+/// CPU frequency table builder.
+///
+/// This is used by the CPU frequency drivers to build a frequency table dynamically.
+///
+/// ## Examples
+///
+/// The following example demonstrates how to create a CPU frequency table.
+///
+/// ```
+/// use kernel::cpufreq::{TableBuilder, TableIndex};
+/// use kernel::clk::Hertz;
+///
+/// let mut builder = TableBuilder::new();
+///
+/// // Adds few entries to the table.
+/// builder.add(Hertz::from_mhz(700), 0, 1).unwrap();
+/// builder.add(Hertz::from_mhz(800), 2, 3).unwrap();
+/// builder.add(Hertz::from_mhz(900), 4, 5).unwrap();
+/// builder.add(Hertz::from_ghz(1), 6, 7).unwrap();
+///
+/// let table = builder.to_table().unwrap();
+///
+/// // SAFETY: Index values correspond to valid entries in the table.
+/// let (index0, index2) = unsafe { (TableIndex::new(0), TableIndex::new(2)) };
+///
+/// assert_eq!(table.freq(index0), Ok(Hertz::from_mhz(700)));
+/// assert_eq!(table.flags(index0), 0);
+/// assert_eq!(table.data(index0), 1);
+///
+/// assert_eq!(table.freq(index2), Ok(Hertz::from_mhz(900)));
+/// assert_eq!(table.flags(index2), 4);
+/// assert_eq!(table.data(index2), 5);
+/// ```
+#[derive(Default)]
+#[repr(transparent)]
+pub struct TableBuilder {
+ entries: KVec<bindings::cpufreq_frequency_table>,
+}
+
+impl TableBuilder {
+ /// Creates a new instance of [`TableBuilder`].
+ #[inline]
+ pub fn new() -> Self {
+ Self {
+ entries: KVec::new(),
+ }
+ }
+
+ /// Adds a new entry to the table.
+ pub fn add(&mut self, freq: Hertz, flags: u32, driver_data: u32) -> Result {
+ // Adds the new entry at the end of the vector.
+ Ok(self.entries.push(
+ bindings::cpufreq_frequency_table {
+ flags,
+ driver_data,
+ frequency: freq.as_khz() as u32,
+ },
+ GFP_KERNEL,
+ )?)
+ }
+
+ /// Consumes the [`TableBuilder`] and returns [`TableBox`].
+ pub fn to_table(mut self) -> Result<TableBox> {
+ // Add last entry to the table.
+ self.add(Hertz(c_ulong::MAX), 0, 0)?;
+
+ TableBox::new(self.entries)
+ }
+}
+
+/// CPU frequency policy.
+///
+/// Rust abstraction for the C `struct cpufreq_policy`.
+///
+/// # Invariants
+///
+/// A [`Policy`] instance always corresponds to a valid C `struct cpufreq_policy`.
+///
+/// The callers must ensure that the `struct cpufreq_policy` is valid for access and remains valid
+/// for the lifetime of the returned reference.
+///
+/// ## Examples
+///
+/// The following example demonstrates how to create a CPU frequency table.
+///
+/// ```
+/// use kernel::cpufreq::{ETERNAL_LATENCY_NS, Policy};
+///
+/// fn update_policy(policy: &mut Policy) {
+/// policy
+/// .set_dvfs_possible_from_any_cpu(true)
+/// .set_fast_switch_possible(true)
+/// .set_transition_latency_ns(ETERNAL_LATENCY_NS);
+///
+/// pr_info!("The policy details are: {:?}\n", (policy.cpu(), policy.cur()));
+/// }
+/// ```
+#[repr(transparent)]
+pub struct Policy(Opaque<bindings::cpufreq_policy>);
+
+impl Policy {
+ /// Creates a reference to an existing `struct cpufreq_policy` pointer.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime
+ /// of the returned reference.
+ #[inline]
+ pub unsafe fn from_raw<'a>(ptr: *const bindings::cpufreq_policy) -> &'a Self {
+ // SAFETY: Guaranteed by the safety requirements of the function.
+ //
+ // INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the
+ // lifetime of the returned reference.
+ unsafe { &*ptr.cast() }
+ }
+
+ /// Creates a mutable reference to an existing `struct cpufreq_policy` pointer.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime
+ /// of the returned reference.
+ #[inline]
+ pub unsafe fn from_raw_mut<'a>(ptr: *mut bindings::cpufreq_policy) -> &'a mut Self {
+ // SAFETY: Guaranteed by the safety requirements of the function.
+ //
+ // INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the
+ // lifetime of the returned reference.
+ unsafe { &mut *ptr.cast() }
+ }
+
+ /// Returns a raw mutable pointer to the C `struct cpufreq_policy`.
+ #[inline]
+ fn as_raw(&self) -> *mut bindings::cpufreq_policy {
+ let this: *const Self = self;
+ this.cast_mut().cast()
+ }
+
+ #[inline]
+ fn as_ref(&self) -> &bindings::cpufreq_policy {
+ // SAFETY: By the type invariant, the pointer stored in `self` is valid.
+ unsafe { &*self.as_raw() }
+ }
+
+ #[inline]
+ fn as_mut_ref(&mut self) -> &mut bindings::cpufreq_policy {
+ // SAFETY: By the type invariant, the pointer stored in `self` is valid.
+ unsafe { &mut *self.as_raw() }
+ }
+
+ /// Returns the primary CPU for the [`Policy`].
+ #[inline]
+ pub fn cpu(&self) -> CpuId {
+ // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number.
+ unsafe { CpuId::from_u32_unchecked(self.as_ref().cpu) }
+ }
+
+ /// Returns the minimum frequency for the [`Policy`].
+ #[inline]
+ pub fn min(&self) -> Hertz {
+ Hertz::from_khz(self.as_ref().min as usize)
+ }
+
+ /// Set the minimum frequency for the [`Policy`].
+ #[inline]
+ pub fn set_min(&mut self, min: Hertz) -> &mut Self {
+ self.as_mut_ref().min = min.as_khz() as u32;
+ self
+ }
+
+ /// Returns the maximum frequency for the [`Policy`].
+ #[inline]
+ pub fn max(&self) -> Hertz {
+ Hertz::from_khz(self.as_ref().max as usize)
+ }
+
+ /// Set the maximum frequency for the [`Policy`].
+ #[inline]
+ pub fn set_max(&mut self, max: Hertz) -> &mut Self {
+ self.as_mut_ref().max = max.as_khz() as u32;
+ self
+ }
+
+ /// Returns the current frequency for the [`Policy`].
+ #[inline]
+ pub fn cur(&self) -> Hertz {
+ Hertz::from_khz(self.as_ref().cur as usize)
+ }
+
+ /// Returns the suspend frequency for the [`Policy`].
+ #[inline]
+ pub fn suspend_freq(&self) -> Hertz {
+ Hertz::from_khz(self.as_ref().suspend_freq as usize)
+ }
+
+ /// Sets the suspend frequency for the [`Policy`].
+ #[inline]
+ pub fn set_suspend_freq(&mut self, freq: Hertz) -> &mut Self {
+ self.as_mut_ref().suspend_freq = freq.as_khz() as u32;
+ self
+ }
+
+ /// Provides a wrapper to the generic suspend routine.
+ #[inline]
+ pub fn generic_suspend(&mut self) -> Result {
+ // SAFETY: By the type invariant, the pointer stored in `self` is valid.
+ to_result(unsafe { bindings::cpufreq_generic_suspend(self.as_mut_ref()) })
+ }
+
+ /// Provides a wrapper to the generic get routine.
+ #[inline]
+ pub fn generic_get(&self) -> Result<u32> {
+ // SAFETY: By the type invariant, the pointer stored in `self` is valid.
+ Ok(unsafe { bindings::cpufreq_generic_get(u32::from(self.cpu())) })
+ }
+
+ /// Provides a wrapper to the register with energy model using the OPP core.
+ #[cfg(CONFIG_PM_OPP)]
+ #[inline]
+ pub fn register_em_opp(&mut self) {
+ // SAFETY: By the type invariant, the pointer stored in `self` is valid.
+ unsafe { bindings::cpufreq_register_em_with_opp(self.as_mut_ref()) };
+ }
+
+ /// Gets [`cpumask::Cpumask`] for a cpufreq [`Policy`].
+ #[inline]
+ pub fn cpus(&mut self) -> &mut cpumask::Cpumask {
+ // SAFETY: The pointer to `cpus` is valid for writing and remains valid for the lifetime of
+ // the returned reference.
+ unsafe { cpumask::CpumaskVar::as_mut_ref(&mut self.as_mut_ref().cpus) }
+ }
+
+ /// Sets clock for the [`Policy`].
+ ///
+ /// # Safety
+ ///
+ /// The caller must guarantee that the returned [`Clk`] is not dropped while it is getting used
+ /// by the C code.
+ #[cfg(CONFIG_COMMON_CLK)]
+ pub unsafe fn set_clk(&mut self, dev: &Device, name: Option<&CStr>) -> Result<Clk> {
+ let clk = Clk::get(dev, name)?;
+ self.as_mut_ref().clk = clk.as_raw();
+ Ok(clk)
+ }
+
+ /// Allows / disallows frequency switching code to run on any CPU.
+ #[inline]
+ pub fn set_dvfs_possible_from_any_cpu(&mut self, val: bool) -> &mut Self {
+ self.as_mut_ref().dvfs_possible_from_any_cpu = val;
+ self
+ }
+
+ /// Returns if fast switching of frequencies is possible or not.
+ #[inline]
+ pub fn fast_switch_possible(&self) -> bool {
+ self.as_ref().fast_switch_possible
+ }
+
+ /// Enables / disables fast frequency switching.
+ #[inline]
+ pub fn set_fast_switch_possible(&mut self, val: bool) -> &mut Self {
+ self.as_mut_ref().fast_switch_possible = val;
+ self
+ }
+
+ /// Sets transition latency (in nanoseconds) for the [`Policy`].
+ #[inline]
+ pub fn set_transition_latency_ns(&mut self, latency_ns: u32) -> &mut Self {
+ self.as_mut_ref().cpuinfo.transition_latency = latency_ns;
+ self
+ }
+
+ /// Sets cpuinfo `min_freq`.
+ #[inline]
+ pub fn set_cpuinfo_min_freq(&mut self, min_freq: Hertz) -> &mut Self {
+ self.as_mut_ref().cpuinfo.min_freq = min_freq.as_khz() as u32;
+ self
+ }
+
+ /// Sets cpuinfo `max_freq`.
+ #[inline]
+ pub fn set_cpuinfo_max_freq(&mut self, max_freq: Hertz) -> &mut Self {
+ self.as_mut_ref().cpuinfo.max_freq = max_freq.as_khz() as u32;
+ self
+ }
+
+ /// Set `transition_delay_us`, i.e. the minimum time between successive frequency change
+ /// requests.
+ #[inline]
+ pub fn set_transition_delay_us(&mut self, transition_delay_us: u32) -> &mut Self {
+ self.as_mut_ref().transition_delay_us = transition_delay_us;
+ self
+ }
+
+ /// Returns reference to the CPU frequency [`Table`] for the [`Policy`].
+ pub fn freq_table(&self) -> Result<&Table> {
+ if self.as_ref().freq_table.is_null() {
+ return Err(EINVAL);
+ }
+
+ // SAFETY: The `freq_table` is guaranteed to be valid for reading and remains valid for the
+ // lifetime of the returned reference.
+ Ok(unsafe { Table::from_raw(self.as_ref().freq_table) })
+ }
+
+ /// Sets the CPU frequency [`Table`] for the [`Policy`].
+ ///
+ /// # Safety
+ ///
+ /// The caller must guarantee that the [`Table`] is not dropped while it is getting used by the
+ /// C code.
+ #[inline]
+ pub unsafe fn set_freq_table(&mut self, table: &Table) -> &mut Self {
+ self.as_mut_ref().freq_table = table.as_raw();
+ self
+ }
+
+ /// Returns the [`Policy`]'s private data.
+ pub fn data<T: ForeignOwnable>(&mut self) -> Option<<T>::Borrowed<'_>> {
+ if self.as_ref().driver_data.is_null() {
+ None
+ } else {
+ // SAFETY: The data is earlier set from [`set_data`].
+ Some(unsafe { T::borrow(self.as_ref().driver_data.cast()) })
+ }
+ }
+
+ /// Sets the private data of the [`Policy`] using a foreign-ownable wrapper.
+ ///
+ /// # Errors
+ ///
+ /// Returns `EBUSY` if private data is already set.
+ fn set_data<T: ForeignOwnable>(&mut self, data: T) -> Result {
+ if self.as_ref().driver_data.is_null() {
+ // Transfer the ownership of the data to the foreign interface.
+ self.as_mut_ref().driver_data = <T as ForeignOwnable>::into_foreign(data) as _;
+ Ok(())
+ } else {
+ Err(EBUSY)
+ }
+ }
+
+ /// Clears and returns ownership of the private data.
+ fn clear_data<T: ForeignOwnable>(&mut self) -> Option<T> {
+ if self.as_ref().driver_data.is_null() {
+ None
+ } else {
+ let data = Some(
+ // SAFETY: The data is earlier set by us from [`set_data`]. It is safe to take
+ // back the ownership of the data from the foreign interface.
+ unsafe { <T as ForeignOwnable>::from_foreign(self.as_ref().driver_data.cast()) },
+ );
+ self.as_mut_ref().driver_data = ptr::null_mut();
+ data
+ }
+ }
+}
+
+/// CPU frequency policy created from a CPU number.
+///
+/// This struct represents the CPU frequency policy obtained for a specific CPU, providing safe
+/// access to the underlying `cpufreq_policy` and ensuring proper cleanup when the `PolicyCpu` is
+/// dropped.
+struct PolicyCpu<'a>(&'a mut Policy);
+
+impl<'a> PolicyCpu<'a> {
+ fn from_cpu(cpu: CpuId) -> Result<Self> {
+ // SAFETY: It is safe to call `cpufreq_cpu_get` for any valid CPU.
+ let ptr = from_err_ptr(unsafe { bindings::cpufreq_cpu_get(u32::from(cpu)) })?;
+
+ Ok(Self(
+ // SAFETY: The `ptr` is guaranteed to be valid and remains valid for the lifetime of
+ // the returned reference.
+ unsafe { Policy::from_raw_mut(ptr) },
+ ))
+ }
+}
+
+impl<'a> Deref for PolicyCpu<'a> {
+ type Target = Policy;
+
+ fn deref(&self) -> &Self::Target {
+ self.0
+ }
+}
+
+impl<'a> DerefMut for PolicyCpu<'a> {
+ fn deref_mut(&mut self) -> &mut Policy {
+ self.0
+ }
+}
+
+impl<'a> Drop for PolicyCpu<'a> {
+ fn drop(&mut self) {
+ // SAFETY: The underlying pointer is guaranteed to be valid for the lifetime of `self`.
+ unsafe { bindings::cpufreq_cpu_put(self.0.as_raw()) };
+ }
+}
+
+/// CPU frequency driver.
+///
+/// Implement this trait to provide a CPU frequency driver and its callbacks.
+///
+/// Reference: <https://docs.kernel.org/cpu-freq/cpu-drivers.html>
+#[vtable]
+pub trait Driver {
+ /// Driver's name.
+ const NAME: &'static CStr;
+
+ /// Driver's flags.
+ const FLAGS: u16;
+
+ /// Boost support.
+ const BOOST_ENABLED: bool;
+
+ /// Policy specific data.
+ ///
+ /// Require that `PData` implements `ForeignOwnable`. We guarantee to never move the underlying
+ /// wrapped data structure.
+ type PData: ForeignOwnable;
+
+ /// Driver's `init` callback.
+ fn init(policy: &mut Policy) -> Result<Self::PData>;
+
+ /// Driver's `exit` callback.
+ fn exit(_policy: &mut Policy, _data: Option<Self::PData>) -> Result {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `online` callback.
+ fn online(_policy: &mut Policy) -> Result {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `offline` callback.
+ fn offline(_policy: &mut Policy) -> Result {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `suspend` callback.
+ fn suspend(_policy: &mut Policy) -> Result {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `resume` callback.
+ fn resume(_policy: &mut Policy) -> Result {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `ready` callback.
+ fn ready(_policy: &mut Policy) {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `verify` callback.
+ fn verify(data: &mut PolicyData) -> Result;
+
+ /// Driver's `setpolicy` callback.
+ fn setpolicy(_policy: &mut Policy) -> Result {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `target` callback.
+ fn target(_policy: &mut Policy, _target_freq: u32, _relation: Relation) -> Result {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `target_index` callback.
+ fn target_index(_policy: &mut Policy, _index: TableIndex) -> Result {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `fast_switch` callback.
+ fn fast_switch(_policy: &mut Policy, _target_freq: u32) -> u32 {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `adjust_perf` callback.
+ fn adjust_perf(_policy: &mut Policy, _min_perf: usize, _target_perf: usize, _capacity: usize) {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `get_intermediate` callback.
+ fn get_intermediate(_policy: &mut Policy, _index: TableIndex) -> u32 {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `target_intermediate` callback.
+ fn target_intermediate(_policy: &mut Policy, _index: TableIndex) -> Result {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `get` callback.
+ fn get(_policy: &mut Policy) -> Result<u32> {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `update_limits` callback.
+ fn update_limits(_policy: &mut Policy) {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `bios_limit` callback.
+ fn bios_limit(_policy: &mut Policy, _limit: &mut u32) -> Result {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `set_boost` callback.
+ fn set_boost(_policy: &mut Policy, _state: i32) -> Result {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// Driver's `register_em` callback.
+ fn register_em(_policy: &mut Policy) {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+}
+
+/// CPU frequency driver Registration.
+///
+/// ## Examples
+///
+/// The following example demonstrates how to register a cpufreq driver.
+///
+/// ```
+/// use kernel::{
+/// cpufreq,
+/// c_str,
+/// device::{Core, Device},
+/// macros::vtable,
+/// of, platform,
+/// sync::Arc,
+/// };
+/// struct SampleDevice;
+///
+/// #[derive(Default)]
+/// struct SampleDriver;
+///
+/// #[vtable]
+/// impl cpufreq::Driver for SampleDriver {
+/// const NAME: &'static CStr = c_str!("cpufreq-sample");
+/// const FLAGS: u16 = cpufreq::flags::NEED_INITIAL_FREQ_CHECK | cpufreq::flags::IS_COOLING_DEV;
+/// const BOOST_ENABLED: bool = true;
+///
+/// type PData = Arc<SampleDevice>;
+///
+/// fn init(policy: &mut cpufreq::Policy) -> Result<Self::PData> {
+/// // Initialize here
+/// Ok(Arc::new(SampleDevice, GFP_KERNEL)?)
+/// }
+///
+/// fn exit(_policy: &mut cpufreq::Policy, _data: Option<Self::PData>) -> Result {
+/// Ok(())
+/// }
+///
+/// fn suspend(policy: &mut cpufreq::Policy) -> Result {
+/// policy.generic_suspend()
+/// }
+///
+/// fn verify(data: &mut cpufreq::PolicyData) -> Result {
+/// data.generic_verify()
+/// }
+///
+/// fn target_index(policy: &mut cpufreq::Policy, index: cpufreq::TableIndex) -> Result {
+/// // Update CPU frequency
+/// Ok(())
+/// }
+///
+/// fn get(policy: &mut cpufreq::Policy) -> Result<u32> {
+/// policy.generic_get()
+/// }
+/// }
+///
+/// impl platform::Driver for SampleDriver {
+/// type IdInfo = ();
+/// const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = None;
+///
+/// fn probe(
+/// pdev: &platform::Device<Core>,
+/// _id_info: Option<&Self::IdInfo>,
+/// ) -> Result<Pin<KBox<Self>>> {
+/// cpufreq::Registration::<SampleDriver>::new_foreign_owned(pdev.as_ref())?;
+/// Ok(KBox::new(Self {}, GFP_KERNEL)?.into())
+/// }
+/// }
+/// ```
+#[repr(transparent)]
+pub struct Registration<T: Driver>(KBox<UnsafeCell<bindings::cpufreq_driver>>, PhantomData<T>);
+
+/// SAFETY: `Registration` doesn't offer any methods or access to fields when shared between threads
+/// or CPUs, so it is safe to share it.
+unsafe impl<T: Driver> Sync for Registration<T> {}
+
+#[allow(clippy::non_send_fields_in_send_ty)]
+/// SAFETY: Registration with and unregistration from the cpufreq subsystem can happen from any
+/// thread.
+unsafe impl<T: Driver> Send for Registration<T> {}
+
+impl<T: Driver> Registration<T> {
+ const VTABLE: bindings::cpufreq_driver = bindings::cpufreq_driver {
+ name: Self::copy_name(T::NAME),
+ boost_enabled: T::BOOST_ENABLED,
+ flags: T::FLAGS,
+
+ // Initialize mandatory callbacks.
+ init: Some(Self::init_callback),
+ verify: Some(Self::verify_callback),
+
+ // Initialize optional callbacks based on the traits of `T`.
+ setpolicy: if T::HAS_SETPOLICY {
+ Some(Self::setpolicy_callback)
+ } else {
+ None
+ },
+ target: if T::HAS_TARGET {
+ Some(Self::target_callback)
+ } else {
+ None
+ },
+ target_index: if T::HAS_TARGET_INDEX {
+ Some(Self::target_index_callback)
+ } else {
+ None
+ },
+ fast_switch: if T::HAS_FAST_SWITCH {
+ Some(Self::fast_switch_callback)
+ } else {
+ None
+ },
+ adjust_perf: if T::HAS_ADJUST_PERF {
+ Some(Self::adjust_perf_callback)
+ } else {
+ None
+ },
+ get_intermediate: if T::HAS_GET_INTERMEDIATE {
+ Some(Self::get_intermediate_callback)
+ } else {
+ None
+ },
+ target_intermediate: if T::HAS_TARGET_INTERMEDIATE {
+ Some(Self::target_intermediate_callback)
+ } else {
+ None
+ },
+ get: if T::HAS_GET {
+ Some(Self::get_callback)
+ } else {
+ None
+ },
+ update_limits: if T::HAS_UPDATE_LIMITS {
+ Some(Self::update_limits_callback)
+ } else {
+ None
+ },
+ bios_limit: if T::HAS_BIOS_LIMIT {
+ Some(Self::bios_limit_callback)
+ } else {
+ None
+ },
+ online: if T::HAS_ONLINE {
+ Some(Self::online_callback)
+ } else {
+ None
+ },
+ offline: if T::HAS_OFFLINE {
+ Some(Self::offline_callback)
+ } else {
+ None
+ },
+ exit: if T::HAS_EXIT {
+ Some(Self::exit_callback)
+ } else {
+ None
+ },
+ suspend: if T::HAS_SUSPEND {
+ Some(Self::suspend_callback)
+ } else {
+ None
+ },
+ resume: if T::HAS_RESUME {
+ Some(Self::resume_callback)
+ } else {
+ None
+ },
+ ready: if T::HAS_READY {
+ Some(Self::ready_callback)
+ } else {
+ None
+ },
+ set_boost: if T::HAS_SET_BOOST {
+ Some(Self::set_boost_callback)
+ } else {
+ None
+ },
+ register_em: if T::HAS_REGISTER_EM {
+ Some(Self::register_em_callback)
+ } else {
+ None
+ },
+ // SAFETY: All zeros is a valid value for `bindings::cpufreq_driver`.
+ ..unsafe { MaybeUninit::zeroed().assume_init() }
+ };
+
+ const fn copy_name(name: &'static CStr) -> [c_char; CPUFREQ_NAME_LEN] {
+ let src = name.as_bytes_with_nul();
+ let mut dst = [0; CPUFREQ_NAME_LEN];
+
+ build_assert!(src.len() <= CPUFREQ_NAME_LEN);
+
+ let mut i = 0;
+ while i < src.len() {
+ dst[i] = src[i];
+ i += 1;
+ }
+
+ dst
+ }
+
+ /// Registers a CPU frequency driver with the cpufreq core.
+ pub fn new() -> Result<Self> {
+ // We can't use `&Self::VTABLE` directly because the cpufreq core modifies some fields in
+ // the C `struct cpufreq_driver`, which requires a mutable reference.
+ let mut drv = KBox::new(UnsafeCell::new(Self::VTABLE), GFP_KERNEL)?;
+
+ // SAFETY: `drv` is guaranteed to be valid for the lifetime of `Registration`.
+ to_result(unsafe { bindings::cpufreq_register_driver(drv.get_mut()) })?;
+
+ Ok(Self(drv, PhantomData))
+ }
+
+ /// Same as [`Registration::new`], but does not return a [`Registration`] instance.
+ ///
+ /// Instead the [`Registration`] is owned by [`Devres`] and will be revoked / dropped, once the
+ /// device is detached.
+ pub fn new_foreign_owned(dev: &Device<Bound>) -> Result {
+ Devres::new_foreign_owned(dev, Self::new()?, GFP_KERNEL)
+ }
+}
+
+/// CPU frequency driver callbacks.
+impl<T: Driver> Registration<T> {
+ /// Driver's `init` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ /// - The pointer arguments must be valid pointers.
+ unsafe extern "C" fn init_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+
+ let data = T::init(policy)?;
+ policy.set_data(data)?;
+ Ok(0)
+ })
+ }
+
+ /// Driver's `exit` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ /// - The pointer arguments must be valid pointers.
+ unsafe extern "C" fn exit_callback(ptr: *mut bindings::cpufreq_policy) {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+
+ let data = policy.clear_data();
+ let _ = T::exit(policy, data);
+ }
+
+ /// Driver's `online` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ /// - The pointer arguments must be valid pointers.
+ unsafe extern "C" fn online_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::online(policy).map(|()| 0)
+ })
+ }
+
+ /// Driver's `offline` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ /// - The pointer arguments must be valid pointers.
+ unsafe extern "C" fn offline_callback(
+ ptr: *mut bindings::cpufreq_policy,
+ ) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::offline(policy).map(|()| 0)
+ })
+ }
+
+ /// Driver's `suspend` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ /// - The pointer arguments must be valid pointers.
+ unsafe extern "C" fn suspend_callback(
+ ptr: *mut bindings::cpufreq_policy,
+ ) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::suspend(policy).map(|()| 0)
+ })
+ }
+
+ /// Driver's `resume` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ /// - The pointer arguments must be valid pointers.
+ unsafe extern "C" fn resume_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::resume(policy).map(|()| 0)
+ })
+ }
+
+ /// Driver's `ready` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ /// - The pointer arguments must be valid pointers.
+ unsafe extern "C" fn ready_callback(ptr: *mut bindings::cpufreq_policy) {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::ready(policy);
+ }
+
+ /// Driver's `verify` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ /// - The pointer arguments must be valid pointers.
+ unsafe extern "C" fn verify_callback(
+ ptr: *mut bindings::cpufreq_policy_data,
+ ) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let data = unsafe { PolicyData::from_raw_mut(ptr) };
+ T::verify(data).map(|()| 0)
+ })
+ }
+
+ /// Driver's `setpolicy` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ /// - The pointer arguments must be valid pointers.
+ unsafe extern "C" fn setpolicy_callback(
+ ptr: *mut bindings::cpufreq_policy,
+ ) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::setpolicy(policy).map(|()| 0)
+ })
+ }
+
+ /// Driver's `target` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ /// - The pointer arguments must be valid pointers.
+ unsafe extern "C" fn target_callback(
+ ptr: *mut bindings::cpufreq_policy,
+ target_freq: u32,
+ relation: u32,
+ ) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::target(policy, target_freq, Relation::new(relation)?).map(|()| 0)
+ })
+ }
+
+ /// Driver's `target_index` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ /// - The pointer arguments must be valid pointers.
+ unsafe extern "C" fn target_index_callback(
+ ptr: *mut bindings::cpufreq_policy,
+ index: u32,
+ ) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+
+ // SAFETY: The C code guarantees that `index` corresponds to a valid entry in the
+ // frequency table.
+ let index = unsafe { TableIndex::new(index as usize) };
+
+ T::target_index(policy, index).map(|()| 0)
+ })
+ }
+
+ /// Driver's `fast_switch` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ /// - The pointer arguments must be valid pointers.
+ unsafe extern "C" fn fast_switch_callback(
+ ptr: *mut bindings::cpufreq_policy,
+ target_freq: u32,
+ ) -> kernel::ffi::c_uint {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::fast_switch(policy, target_freq)
+ }
+
+ /// Driver's `adjust_perf` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ unsafe extern "C" fn adjust_perf_callback(
+ cpu: u32,
+ min_perf: usize,
+ target_perf: usize,
+ capacity: usize,
+ ) {
+ // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number.
+ let cpu_id = unsafe { CpuId::from_u32_unchecked(cpu) };
+
+ if let Ok(mut policy) = PolicyCpu::from_cpu(cpu_id) {
+ T::adjust_perf(&mut policy, min_perf, target_perf, capacity);
+ }
+ }
+
+ /// Driver's `get_intermediate` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ /// - The pointer arguments must be valid pointers.
+ unsafe extern "C" fn get_intermediate_callback(
+ ptr: *mut bindings::cpufreq_policy,
+ index: u32,
+ ) -> kernel::ffi::c_uint {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+
+ // SAFETY: The C code guarantees that `index` corresponds to a valid entry in the
+ // frequency table.
+ let index = unsafe { TableIndex::new(index as usize) };
+
+ T::get_intermediate(policy, index)
+ }
+
+ /// Driver's `target_intermediate` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ /// - The pointer arguments must be valid pointers.
+ unsafe extern "C" fn target_intermediate_callback(
+ ptr: *mut bindings::cpufreq_policy,
+ index: u32,
+ ) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+
+ // SAFETY: The C code guarantees that `index` corresponds to a valid entry in the
+ // frequency table.
+ let index = unsafe { TableIndex::new(index as usize) };
+
+ T::target_intermediate(policy, index).map(|()| 0)
+ })
+ }
+
+ /// Driver's `get` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ unsafe extern "C" fn get_callback(cpu: u32) -> kernel::ffi::c_uint {
+ // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number.
+ let cpu_id = unsafe { CpuId::from_u32_unchecked(cpu) };
+
+ PolicyCpu::from_cpu(cpu_id).map_or(0, |mut policy| T::get(&mut policy).map_or(0, |f| f))
+ }
+
+ /// Driver's `update_limit` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ /// - The pointer arguments must be valid pointers.
+ unsafe extern "C" fn update_limits_callback(ptr: *mut bindings::cpufreq_policy) {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::update_limits(policy);
+ }
+
+ /// Driver's `bios_limit` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ /// - The pointer arguments must be valid pointers.
+ unsafe extern "C" fn bios_limit_callback(cpu: i32, limit: *mut u32) -> kernel::ffi::c_int {
+ // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number.
+ let cpu_id = unsafe { CpuId::from_i32_unchecked(cpu) };
+
+ from_result(|| {
+ let mut policy = PolicyCpu::from_cpu(cpu_id)?;
+
+ // SAFETY: `limit` is guaranteed by the C code to be valid.
+ T::bios_limit(&mut policy, &mut (unsafe { *limit })).map(|()| 0)
+ })
+ }
+
+ /// Driver's `set_boost` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ /// - The pointer arguments must be valid pointers.
+ unsafe extern "C" fn set_boost_callback(
+ ptr: *mut bindings::cpufreq_policy,
+ state: i32,
+ ) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::set_boost(policy, state).map(|()| 0)
+ })
+ }
+
+ /// Driver's `register_em` callback.
+ ///
+ /// # Safety
+ ///
+ /// - This function may only be called from the cpufreq C infrastructure.
+ /// - The pointer arguments must be valid pointers.
+ unsafe extern "C" fn register_em_callback(ptr: *mut bindings::cpufreq_policy) {
+ // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
+ // lifetime of `policy`.
+ let policy = unsafe { Policy::from_raw_mut(ptr) };
+ T::register_em(policy);
+ }
+}
+
+impl<T: Driver> Drop for Registration<T> {
+ /// Unregisters with the cpufreq core.
+ fn drop(&mut self) {
+ // SAFETY: `self.0` is guaranteed to be valid for the lifetime of `Registration`.
+ unsafe { bindings::cpufreq_unregister_driver(self.0.get_mut()) };
+ }
+}
diff --git a/rust/kernel/cpumask.rs b/rust/kernel/cpumask.rs
new file mode 100644
index 000000000000..19c607709b5f
--- /dev/null
+++ b/rust/kernel/cpumask.rs
@@ -0,0 +1,351 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! CPU Mask abstractions.
+//!
+//! C header: [`include/linux/cpumask.h`](srctree/include/linux/cpumask.h)
+
+use crate::{
+ alloc::{AllocError, Flags},
+ cpu::CpuId,
+ prelude::*,
+ types::Opaque,
+};
+
+#[cfg(CONFIG_CPUMASK_OFFSTACK)]
+use core::ptr::{self, NonNull};
+
+#[cfg(not(CONFIG_CPUMASK_OFFSTACK))]
+use core::mem::MaybeUninit;
+
+use core::ops::{Deref, DerefMut};
+
+/// A CPU Mask.
+///
+/// Rust abstraction for the C `struct cpumask`.
+///
+/// # Invariants
+///
+/// A [`Cpumask`] instance always corresponds to a valid C `struct cpumask`.
+///
+/// The callers must ensure that the `struct cpumask` is valid for access and
+/// remains valid for the lifetime of the returned reference.
+///
+/// ## Examples
+///
+/// The following example demonstrates how to update a [`Cpumask`].
+///
+/// ```
+/// use kernel::bindings;
+/// use kernel::cpu::CpuId;
+/// use kernel::cpumask::Cpumask;
+///
+/// fn set_clear_cpu(ptr: *mut bindings::cpumask, set_cpu: CpuId, clear_cpu: CpuId) {
+/// // SAFETY: The `ptr` is valid for writing and remains valid for the lifetime of the
+/// // returned reference.
+/// let mask = unsafe { Cpumask::as_mut_ref(ptr) };
+///
+/// mask.set(set_cpu);
+/// mask.clear(clear_cpu);
+/// }
+/// ```
+#[repr(transparent)]
+pub struct Cpumask(Opaque<bindings::cpumask>);
+
+impl Cpumask {
+ /// Creates a mutable reference to an existing `struct cpumask` pointer.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime
+ /// of the returned reference.
+ pub unsafe fn as_mut_ref<'a>(ptr: *mut bindings::cpumask) -> &'a mut Self {
+ // SAFETY: Guaranteed by the safety requirements of the function.
+ //
+ // INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the
+ // lifetime of the returned reference.
+ unsafe { &mut *ptr.cast() }
+ }
+
+ /// Creates a reference to an existing `struct cpumask` pointer.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime
+ /// of the returned reference.
+ pub unsafe fn as_ref<'a>(ptr: *const bindings::cpumask) -> &'a Self {
+ // SAFETY: Guaranteed by the safety requirements of the function.
+ //
+ // INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the
+ // lifetime of the returned reference.
+ unsafe { &*ptr.cast() }
+ }
+
+ /// Obtain the raw `struct cpumask` pointer.
+ pub fn as_raw(&self) -> *mut bindings::cpumask {
+ let this: *const Self = self;
+ this.cast_mut().cast()
+ }
+
+ /// Set `cpu` in the cpumask.
+ ///
+ /// ATTENTION: Contrary to C, this Rust `set()` method is non-atomic.
+ /// This mismatches kernel naming convention and corresponds to the C
+ /// function `__cpumask_set_cpu()`.
+ #[inline]
+ pub fn set(&mut self, cpu: CpuId) {
+ // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `__cpumask_set_cpu`.
+ unsafe { bindings::__cpumask_set_cpu(u32::from(cpu), self.as_raw()) };
+ }
+
+ /// Clear `cpu` in the cpumask.
+ ///
+ /// ATTENTION: Contrary to C, this Rust `clear()` method is non-atomic.
+ /// This mismatches kernel naming convention and corresponds to the C
+ /// function `__cpumask_clear_cpu()`.
+ #[inline]
+ pub fn clear(&mut self, cpu: CpuId) {
+ // SAFETY: By the type invariant, `self.as_raw` is a valid argument to
+ // `__cpumask_clear_cpu`.
+ unsafe { bindings::__cpumask_clear_cpu(i32::from(cpu), self.as_raw()) };
+ }
+
+ /// Test `cpu` in the cpumask.
+ ///
+ /// Equivalent to the kernel's `cpumask_test_cpu` API.
+ #[inline]
+ pub fn test(&self, cpu: CpuId) -> bool {
+ // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_test_cpu`.
+ unsafe { bindings::cpumask_test_cpu(i32::from(cpu), self.as_raw()) }
+ }
+
+ /// Set all CPUs in the cpumask.
+ ///
+ /// Equivalent to the kernel's `cpumask_setall` API.
+ #[inline]
+ pub fn setall(&mut self) {
+ // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_setall`.
+ unsafe { bindings::cpumask_setall(self.as_raw()) };
+ }
+
+ /// Checks if cpumask is empty.
+ ///
+ /// Equivalent to the kernel's `cpumask_empty` API.
+ #[inline]
+ pub fn empty(&self) -> bool {
+ // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_empty`.
+ unsafe { bindings::cpumask_empty(self.as_raw()) }
+ }
+
+ /// Checks if cpumask is full.
+ ///
+ /// Equivalent to the kernel's `cpumask_full` API.
+ #[inline]
+ pub fn full(&self) -> bool {
+ // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_full`.
+ unsafe { bindings::cpumask_full(self.as_raw()) }
+ }
+
+ /// Get weight of the cpumask.
+ ///
+ /// Equivalent to the kernel's `cpumask_weight` API.
+ #[inline]
+ pub fn weight(&self) -> u32 {
+ // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_weight`.
+ unsafe { bindings::cpumask_weight(self.as_raw()) }
+ }
+
+ /// Copy cpumask.
+ ///
+ /// Equivalent to the kernel's `cpumask_copy` API.
+ #[inline]
+ pub fn copy(&self, dstp: &mut Self) {
+ // SAFETY: By the type invariant, `Self::as_raw` is a valid argument to `cpumask_copy`.
+ unsafe { bindings::cpumask_copy(dstp.as_raw(), self.as_raw()) };
+ }
+}
+
+/// A CPU Mask pointer.
+///
+/// Rust abstraction for the C `struct cpumask_var_t`.
+///
+/// # Invariants
+///
+/// A [`CpumaskVar`] instance always corresponds to a valid C `struct cpumask_var_t`.
+///
+/// The callers must ensure that the `struct cpumask_var_t` is valid for access and remains valid
+/// for the lifetime of [`CpumaskVar`].
+///
+/// ## Examples
+///
+/// The following example demonstrates how to create and update a [`CpumaskVar`].
+///
+/// ```
+/// use kernel::cpu::CpuId;
+/// use kernel::cpumask::CpumaskVar;
+///
+/// let mut mask = CpumaskVar::new_zero(GFP_KERNEL).unwrap();
+///
+/// assert!(mask.empty());
+/// let mut count = 0;
+///
+/// let cpu2 = CpuId::from_u32(2);
+/// if let Some(cpu) = cpu2 {
+/// mask.set(cpu);
+/// assert!(mask.test(cpu));
+/// count += 1;
+/// }
+///
+/// let cpu3 = CpuId::from_u32(3);
+/// if let Some(cpu) = cpu3 {
+/// mask.set(cpu);
+/// assert!(mask.test(cpu));
+/// count += 1;
+/// }
+///
+/// assert_eq!(mask.weight(), count);
+///
+/// let mask2 = CpumaskVar::try_clone(&mask).unwrap();
+///
+/// if let Some(cpu) = cpu2 {
+/// assert!(mask2.test(cpu));
+/// }
+///
+/// if let Some(cpu) = cpu3 {
+/// assert!(mask2.test(cpu));
+/// }
+/// assert_eq!(mask2.weight(), count);
+/// ```
+pub struct CpumaskVar {
+ #[cfg(CONFIG_CPUMASK_OFFSTACK)]
+ ptr: NonNull<Cpumask>,
+ #[cfg(not(CONFIG_CPUMASK_OFFSTACK))]
+ mask: Cpumask,
+}
+
+impl CpumaskVar {
+ /// Creates a zero-initialized instance of the [`CpumaskVar`].
+ pub fn new_zero(_flags: Flags) -> Result<Self, AllocError> {
+ Ok(Self {
+ #[cfg(CONFIG_CPUMASK_OFFSTACK)]
+ ptr: {
+ let mut ptr: *mut bindings::cpumask = ptr::null_mut();
+
+ // SAFETY: It is safe to call this method as the reference to `ptr` is valid.
+ //
+ // INVARIANT: The associated memory is freed when the `CpumaskVar` goes out of
+ // scope.
+ unsafe { bindings::zalloc_cpumask_var(&mut ptr, _flags.as_raw()) };
+ NonNull::new(ptr.cast()).ok_or(AllocError)?
+ },
+
+ #[cfg(not(CONFIG_CPUMASK_OFFSTACK))]
+ // SAFETY: FFI type is valid to be zero-initialized.
+ //
+ // INVARIANT: The associated memory is freed when the `CpumaskVar` goes out of scope.
+ mask: unsafe { core::mem::zeroed() },
+ })
+ }
+
+ /// Creates an instance of the [`CpumaskVar`].
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that the returned [`CpumaskVar`] is properly initialized before
+ /// getting used.
+ pub unsafe fn new(_flags: Flags) -> Result<Self, AllocError> {
+ Ok(Self {
+ #[cfg(CONFIG_CPUMASK_OFFSTACK)]
+ ptr: {
+ let mut ptr: *mut bindings::cpumask = ptr::null_mut();
+
+ // SAFETY: It is safe to call this method as the reference to `ptr` is valid.
+ //
+ // INVARIANT: The associated memory is freed when the `CpumaskVar` goes out of
+ // scope.
+ unsafe { bindings::alloc_cpumask_var(&mut ptr, _flags.as_raw()) };
+ NonNull::new(ptr.cast()).ok_or(AllocError)?
+ },
+ #[cfg(not(CONFIG_CPUMASK_OFFSTACK))]
+ // SAFETY: Guaranteed by the safety requirements of the function.
+ //
+ // INVARIANT: The associated memory is freed when the `CpumaskVar` goes out of scope.
+ mask: unsafe { MaybeUninit::uninit().assume_init() },
+ })
+ }
+
+ /// Creates a mutable reference to an existing `struct cpumask_var_t` pointer.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `ptr` is valid for writing and remains valid for the lifetime
+ /// of the returned reference.
+ pub unsafe fn as_mut_ref<'a>(ptr: *mut bindings::cpumask_var_t) -> &'a mut Self {
+ // SAFETY: Guaranteed by the safety requirements of the function.
+ //
+ // INVARIANT: The caller ensures that `ptr` is valid for writing and remains valid for the
+ // lifetime of the returned reference.
+ unsafe { &mut *ptr.cast() }
+ }
+
+ /// Creates a reference to an existing `struct cpumask_var_t` pointer.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `ptr` is valid for reading and remains valid for the lifetime
+ /// of the returned reference.
+ pub unsafe fn as_ref<'a>(ptr: *const bindings::cpumask_var_t) -> &'a Self {
+ // SAFETY: Guaranteed by the safety requirements of the function.
+ //
+ // INVARIANT: The caller ensures that `ptr` is valid for reading and remains valid for the
+ // lifetime of the returned reference.
+ unsafe { &*ptr.cast() }
+ }
+
+ /// Clones cpumask.
+ pub fn try_clone(cpumask: &Cpumask) -> Result<Self> {
+ // SAFETY: The returned cpumask_var is initialized right after this call.
+ let mut cpumask_var = unsafe { Self::new(GFP_KERNEL) }?;
+
+ cpumask.copy(&mut cpumask_var);
+ Ok(cpumask_var)
+ }
+}
+
+// Make [`CpumaskVar`] behave like a pointer to [`Cpumask`].
+impl Deref for CpumaskVar {
+ type Target = Cpumask;
+
+ #[cfg(CONFIG_CPUMASK_OFFSTACK)]
+ fn deref(&self) -> &Self::Target {
+ // SAFETY: The caller owns CpumaskVar, so it is safe to deref the cpumask.
+ unsafe { &*self.ptr.as_ptr() }
+ }
+
+ #[cfg(not(CONFIG_CPUMASK_OFFSTACK))]
+ fn deref(&self) -> &Self::Target {
+ &self.mask
+ }
+}
+
+impl DerefMut for CpumaskVar {
+ #[cfg(CONFIG_CPUMASK_OFFSTACK)]
+ fn deref_mut(&mut self) -> &mut Cpumask {
+ // SAFETY: The caller owns CpumaskVar, so it is safe to deref the cpumask.
+ unsafe { self.ptr.as_mut() }
+ }
+
+ #[cfg(not(CONFIG_CPUMASK_OFFSTACK))]
+ fn deref_mut(&mut self) -> &mut Cpumask {
+ &mut self.mask
+ }
+}
+
+impl Drop for CpumaskVar {
+ fn drop(&mut self) {
+ #[cfg(CONFIG_CPUMASK_OFFSTACK)]
+ // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `free_cpumask_var`.
+ unsafe {
+ bindings::free_cpumask_var(self.as_raw())
+ };
+ }
+}
diff --git a/rust/kernel/cred.rs b/rust/kernel/cred.rs
index 81d67789b16f..2599f01e8b28 100644
--- a/rust/kernel/cred.rs
+++ b/rust/kernel/cred.rs
@@ -47,6 +47,7 @@ impl Credential {
///
/// The caller must ensure that `ptr` is valid and remains valid for the lifetime of the
/// returned [`Credential`] reference.
+ #[inline]
pub unsafe fn from_ptr<'a>(ptr: *const bindings::cred) -> &'a Credential {
// SAFETY: The safety requirements guarantee the validity of the dereference, while the
// `Credential` type being transparent makes the cast ok.
@@ -54,6 +55,7 @@ impl Credential {
}
/// Get the id for this security context.
+ #[inline]
pub fn get_secid(&self) -> u32 {
let mut secid = 0;
// SAFETY: The invariants of this type ensures that the pointer is valid.
@@ -62,6 +64,7 @@ impl Credential {
}
/// Returns the effective UID of the given credential.
+ #[inline]
pub fn euid(&self) -> Kuid {
// SAFETY: By the type invariant, we know that `self.0` is valid. Furthermore, the `euid`
// field of a credential is never changed after initialization, so there is no potential
@@ -72,11 +75,13 @@ impl Credential {
// SAFETY: The type invariants guarantee that `Credential` is always ref-counted.
unsafe impl AlwaysRefCounted for Credential {
+ #[inline]
fn inc_ref(&self) {
// SAFETY: The existence of a shared reference means that the refcount is nonzero.
unsafe { bindings::get_cred(self.0.get()) };
}
+ #[inline]
unsafe fn dec_ref(obj: core::ptr::NonNull<Credential>) {
// SAFETY: The safety requirements guarantee that the refcount is nonzero. The cast is okay
// because `Credential` has the same representation as `struct cred`.
diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs
index db2d9658ba47..dea06b79ecb5 100644
--- a/rust/kernel/device.rs
+++ b/rust/kernel/device.rs
@@ -9,7 +9,7 @@ use crate::{
str::CStr,
types::{ARef, Opaque},
};
-use core::{fmt, ptr};
+use core::{fmt, marker::PhantomData, ptr};
#[cfg(CONFIG_PRINTK)]
use crate::c_str;
@@ -42,7 +42,7 @@ use crate::c_str;
/// `bindings::device::release` is valid to be called from any thread, hence `ARef<Device>` can be
/// dropped from any thread.
#[repr(transparent)]
-pub struct Device(Opaque<bindings::device>);
+pub struct Device<Ctx: DeviceContext = Normal>(Opaque<bindings::device>, PhantomData<Ctx>);
impl Device {
/// Creates a new reference-counted abstraction instance of an existing `struct device` pointer.
@@ -59,12 +59,33 @@ impl Device {
// SAFETY: By the safety requirements ptr is valid
unsafe { Self::as_ref(ptr) }.into()
}
+}
+impl<Ctx: DeviceContext> Device<Ctx> {
/// Obtain the raw `struct device *`.
pub(crate) fn as_raw(&self) -> *mut bindings::device {
self.0.get()
}
+ /// Returns a reference to the parent device, if any.
+ #[cfg_attr(not(CONFIG_AUXILIARY_BUS), expect(dead_code))]
+ pub(crate) fn parent(&self) -> Option<&Self> {
+ // SAFETY:
+ // - By the type invariant `self.as_raw()` is always valid.
+ // - The parent device is only ever set at device creation.
+ let parent = unsafe { (*self.as_raw()).parent };
+
+ if parent.is_null() {
+ None
+ } else {
+ // SAFETY:
+ // - Since `parent` is not NULL, it must be a valid pointer to a `struct device`.
+ // - `parent` is valid for the lifetime of `self`, since a `struct device` holds a
+ // reference count of its parent.
+ Some(unsafe { Self::as_ref(parent) })
+ }
+ }
+
/// Convert a raw C `struct device` pointer to a `&'a Device`.
///
/// # Safety
@@ -189,6 +210,11 @@ impl Device {
}
}
+// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic
+// argument.
+kernel::impl_device_context_deref!(unsafe { Device });
+kernel::impl_device_context_into_aref!(Device);
+
// SAFETY: Instances of `Device` are always reference-counted.
unsafe impl crate::types::AlwaysRefCounted for Device {
fn inc_ref(&self) {
@@ -209,12 +235,117 @@ unsafe impl Send for Device {}
// synchronization in `struct device`.
unsafe impl Sync for Device {}
+/// Marker trait for the context of a bus specific device.
+///
+/// Some functions of a bus specific device should only be called from a certain context, i.e. bus
+/// callbacks, such as `probe()`.
+///
+/// This is the marker trait for structures representing the context of a bus specific device.
+pub trait DeviceContext: private::Sealed {}
+
+/// The [`Normal`] context is the context of a bus specific device when it is not an argument of
+/// any bus callback.
+pub struct Normal;
+
+/// The [`Core`] context is the context of a bus specific device when it is supplied as argument of
+/// any of the bus callbacks, such as `probe()`.
+pub struct Core;
+
+/// The [`Bound`] context is the context of a bus specific device reference when it is guaranteed to
+/// be bound for the duration of its lifetime.
+pub struct Bound;
+
+mod private {
+ pub trait Sealed {}
+
+ impl Sealed for super::Bound {}
+ impl Sealed for super::Core {}
+ impl Sealed for super::Normal {}
+}
+
+impl DeviceContext for Bound {}
+impl DeviceContext for Core {}
+impl DeviceContext for Normal {}
+
+/// # Safety
+///
+/// The type given as `$device` must be a transparent wrapper of a type that doesn't depend on the
+/// generic argument of `$device`.
+#[doc(hidden)]
+#[macro_export]
+macro_rules! __impl_device_context_deref {
+ (unsafe { $device:ident, $src:ty => $dst:ty }) => {
+ impl ::core::ops::Deref for $device<$src> {
+ type Target = $device<$dst>;
+
+ fn deref(&self) -> &Self::Target {
+ let ptr: *const Self = self;
+
+ // CAST: `$device<$src>` and `$device<$dst>` transparently wrap the same type by the
+ // safety requirement of the macro.
+ let ptr = ptr.cast::<Self::Target>();
+
+ // SAFETY: `ptr` was derived from `&self`.
+ unsafe { &*ptr }
+ }
+ }
+ };
+}
+
+/// Implement [`core::ops::Deref`] traits for allowed [`DeviceContext`] conversions of a (bus
+/// specific) device.
+///
+/// # Safety
+///
+/// The type given as `$device` must be a transparent wrapper of a type that doesn't depend on the
+/// generic argument of `$device`.
+#[macro_export]
+macro_rules! impl_device_context_deref {
+ (unsafe { $device:ident }) => {
+ // SAFETY: This macro has the exact same safety requirement as
+ // `__impl_device_context_deref!`.
+ ::kernel::__impl_device_context_deref!(unsafe {
+ $device,
+ $crate::device::Core => $crate::device::Bound
+ });
+
+ // SAFETY: This macro has the exact same safety requirement as
+ // `__impl_device_context_deref!`.
+ ::kernel::__impl_device_context_deref!(unsafe {
+ $device,
+ $crate::device::Bound => $crate::device::Normal
+ });
+ };
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! __impl_device_context_into_aref {
+ ($src:ty, $device:tt) => {
+ impl ::core::convert::From<&$device<$src>> for $crate::types::ARef<$device> {
+ fn from(dev: &$device<$src>) -> Self {
+ (&**dev).into()
+ }
+ }
+ };
+}
+
+/// Implement [`core::convert::From`], such that all `&Device<Ctx>` can be converted to an
+/// `ARef<Device>`.
+#[macro_export]
+macro_rules! impl_device_context_into_aref {
+ ($device:tt) => {
+ ::kernel::__impl_device_context_into_aref!($crate::device::Core, $device);
+ ::kernel::__impl_device_context_into_aref!($crate::device::Bound, $device);
+ };
+}
+
#[doc(hidden)]
#[macro_export]
macro_rules! dev_printk {
($method:ident, $dev:expr, $($f:tt)*) => {
{
- ($dev).$method(core::format_args!($($f)*));
+ ($dev).$method(::core::format_args!($($f)*));
}
}
}
@@ -226,9 +357,10 @@ macro_rules! dev_printk {
/// Equivalent to the kernel's `dev_emerg` macro.
///
/// Mimics the interface of [`std::print!`]. More information about the syntax is available from
-/// [`core::fmt`] and `alloc::format!`.
+/// [`core::fmt`] and [`std::format!`].
///
/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html
///
/// # Examples
///
@@ -251,9 +383,10 @@ macro_rules! dev_emerg {
/// Equivalent to the kernel's `dev_alert` macro.
///
/// Mimics the interface of [`std::print!`]. More information about the syntax is available from
-/// [`core::fmt`] and `alloc::format!`.
+/// [`core::fmt`] and [`std::format!`].
///
/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html
///
/// # Examples
///
@@ -276,9 +409,10 @@ macro_rules! dev_alert {
/// Equivalent to the kernel's `dev_crit` macro.
///
/// Mimics the interface of [`std::print!`]. More information about the syntax is available from
-/// [`core::fmt`] and `alloc::format!`.
+/// [`core::fmt`] and [`std::format!`].
///
/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html
///
/// # Examples
///
@@ -301,9 +435,10 @@ macro_rules! dev_crit {
/// Equivalent to the kernel's `dev_err` macro.
///
/// Mimics the interface of [`std::print!`]. More information about the syntax is available from
-/// [`core::fmt`] and `alloc::format!`.
+/// [`core::fmt`] and [`std::format!`].
///
/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html
///
/// # Examples
///
@@ -326,9 +461,10 @@ macro_rules! dev_err {
/// Equivalent to the kernel's `dev_warn` macro.
///
/// Mimics the interface of [`std::print!`]. More information about the syntax is available from
-/// [`core::fmt`] and `alloc::format!`.
+/// [`core::fmt`] and [`std::format!`].
///
/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html
///
/// # Examples
///
@@ -351,9 +487,10 @@ macro_rules! dev_warn {
/// Equivalent to the kernel's `dev_notice` macro.
///
/// Mimics the interface of [`std::print!`]. More information about the syntax is available from
-/// [`core::fmt`] and `alloc::format!`.
+/// [`core::fmt`] and [`std::format!`].
///
/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html
///
/// # Examples
///
@@ -376,9 +513,10 @@ macro_rules! dev_notice {
/// Equivalent to the kernel's `dev_info` macro.
///
/// Mimics the interface of [`std::print!`]. More information about the syntax is available from
-/// [`core::fmt`] and `alloc::format!`.
+/// [`core::fmt`] and [`std::format!`].
///
/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html
///
/// # Examples
///
@@ -401,9 +539,10 @@ macro_rules! dev_info {
/// Equivalent to the kernel's `dev_dbg` macro, except that it doesn't support dynamic debug yet.
///
/// Mimics the interface of [`std::print!`]. More information about the syntax is available from
-/// [`core::fmt`] and `alloc::format!`.
+/// [`core::fmt`] and [`std::format!`].
///
/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html
///
/// # Examples
///
diff --git a/rust/kernel/device_id.rs b/rust/kernel/device_id.rs
index e5859217a579..0a4eb56d98f2 100644
--- a/rust/kernel/device_id.rs
+++ b/rust/kernel/device_id.rs
@@ -159,7 +159,7 @@ macro_rules! module_device_table {
"_", line!(),
"_", stringify!($table_name))
]
- static $module_table_name: [core::mem::MaybeUninit<u8>; $table_name.raw_ids().size()] =
- unsafe { core::mem::transmute_copy($table_name.raw_ids()) };
+ static $module_table_name: [::core::mem::MaybeUninit<u8>; $table_name.raw_ids().size()] =
+ unsafe { ::core::mem::transmute_copy($table_name.raw_ids()) };
};
}
diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs
index 942376f6f3af..57502534d985 100644
--- a/rust/kernel/devres.rs
+++ b/rust/kernel/devres.rs
@@ -8,30 +8,32 @@
use crate::{
alloc::Flags,
bindings,
- device::Device,
+ device::{Bound, Device},
error::{Error, Result},
ffi::c_void,
prelude::*,
- revocable::Revocable,
- sync::Arc,
+ revocable::{Revocable, RevocableGuard},
+ sync::{rcu, Arc, Completion},
types::ARef,
};
-use core::ops::Deref;
-
#[pin_data]
struct DevresInner<T> {
dev: ARef<Device>,
callback: unsafe extern "C" fn(*mut c_void),
#[pin]
data: Revocable<T>,
+ #[pin]
+ revoke: Completion,
}
/// This abstraction is meant to be used by subsystems to containerize [`Device`] bound resources to
/// manage their lifetime.
///
/// [`Device`] bound resources should be freed when either the resource goes out of scope or the
-/// [`Device`] is unbound respectively, depending on what happens first.
+/// [`Device`] is unbound respectively, depending on what happens first. In any case, it is always
+/// guaranteed that revoking the device resource is completed before the corresponding [`Device`]
+/// is unbound.
///
/// To achieve that [`Devres`] registers a devres callback on creation, which is called once the
/// [`Device`] is unbound, revoking access to the encapsulated resource (see also [`Revocable`]).
@@ -45,7 +47,7 @@ struct DevresInner<T> {
/// # Example
///
/// ```no_run
-/// # use kernel::{bindings, c_str, device::Device, devres::Devres, io::{Io, IoRaw}};
+/// # use kernel::{bindings, c_str, device::{Bound, Device}, devres::Devres, io::{Io, IoRaw}};
/// # use core::ops::Deref;
///
/// // See also [`pci::Bar`] for a real example.
@@ -83,28 +85,26 @@ struct DevresInner<T> {
/// unsafe { Io::from_raw(&self.0) }
/// }
/// }
-/// # fn no_run() -> Result<(), Error> {
-/// # // SAFETY: Invalid usage; just for the example to get an `ARef<Device>` instance.
-/// # let dev = unsafe { Device::get_device(core::ptr::null_mut()) };
-///
+/// # fn no_run(dev: &Device<Bound>) -> Result<(), Error> {
/// // SAFETY: Invalid usage for example purposes.
/// let iomem = unsafe { IoMem::<{ core::mem::size_of::<u32>() }>::new(0xBAAAAAAD)? };
-/// let devres = Devres::new(&dev, iomem, GFP_KERNEL)?;
+/// let devres = Devres::new(dev, iomem, GFP_KERNEL)?;
///
/// let res = devres.try_access().ok_or(ENXIO)?;
-/// res.writel(0x42, 0x0);
+/// res.write8(0x42, 0x0);
/// # Ok(())
/// # }
/// ```
pub struct Devres<T>(Arc<DevresInner<T>>);
impl<T> DevresInner<T> {
- fn new(dev: &Device, data: T, flags: Flags) -> Result<Arc<DevresInner<T>>> {
+ fn new(dev: &Device<Bound>, data: T, flags: Flags) -> Result<Arc<DevresInner<T>>> {
let inner = Arc::pin_init(
pin_init!( DevresInner {
dev: dev.into(),
callback: Self::devres_callback,
data <- Revocable::new(data),
+ revoke <- Completion::new(),
}),
flags,
)?;
@@ -133,26 +133,28 @@ impl<T> DevresInner<T> {
self as _
}
- fn remove_action(this: &Arc<Self>) {
+ fn remove_action(this: &Arc<Self>) -> bool {
// SAFETY:
// - `self.inner.dev` is a valid `Device`,
// - the `action` and `data` pointers are the exact same ones as given to devm_add_action()
// previously,
// - `self` is always valid, even if the action has been released already.
- let ret = unsafe {
+ let success = unsafe {
bindings::devm_remove_action_nowarn(
this.dev.as_raw(),
Some(this.callback),
this.as_ptr() as _,
)
- };
+ } == 0;
- if ret == 0 {
+ if success {
// SAFETY: We leaked an `Arc` reference to devm_add_action() in `DevresInner::new`; if
// devm_remove_action_nowarn() was successful we can (and have to) claim back ownership
// of this reference.
let _ = unsafe { Arc::from_raw(this.as_ptr()) };
}
+
+ success
}
#[allow(clippy::missing_safety_doc)]
@@ -164,14 +166,19 @@ impl<T> DevresInner<T> {
// `DevresInner::new`.
let inner = unsafe { Arc::from_raw(ptr) };
- inner.data.revoke();
+ if !inner.data.revoke() {
+ // If `revoke()` returns false, it means that `Devres::drop` already started revoking
+ // `inner.data` for us. Hence we have to wait until `Devres::drop()` signals that it
+ // completed revoking `inner.data`.
+ inner.revoke.wait_for_completion();
+ }
}
}
impl<T> Devres<T> {
/// Creates a new [`Devres`] instance of the given `data`. The `data` encapsulated within the
/// returned `Devres` instance' `data` will be revoked once the device is detached.
- pub fn new(dev: &Device, data: T, flags: Flags) -> Result<Self> {
+ pub fn new(dev: &Device<Bound>, data: T, flags: Flags) -> Result<Self> {
let inner = DevresInner::new(dev, data, flags)?;
Ok(Devres(inner))
@@ -179,23 +186,78 @@ impl<T> Devres<T> {
/// Same as [`Devres::new`], but does not return a `Devres` instance. Instead the given `data`
/// is owned by devres and will be revoked / dropped, once the device is detached.
- pub fn new_foreign_owned(dev: &Device, data: T, flags: Flags) -> Result {
+ pub fn new_foreign_owned(dev: &Device<Bound>, data: T, flags: Flags) -> Result {
let _ = DevresInner::new(dev, data, flags)?;
Ok(())
}
-}
-impl<T> Deref for Devres<T> {
- type Target = Revocable<T>;
+ /// Obtain `&'a T`, bypassing the [`Revocable`].
+ ///
+ /// This method allows to directly obtain a `&'a T`, bypassing the [`Revocable`], by presenting
+ /// a `&'a Device<Bound>` of the same [`Device`] this [`Devres`] instance has been created with.
+ ///
+ /// # Errors
+ ///
+ /// An error is returned if `dev` does not match the same [`Device`] this [`Devres`] instance
+ /// has been created with.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// # #![cfg(CONFIG_PCI)]
+ /// # use kernel::{device::Core, devres::Devres, pci};
+ ///
+ /// fn from_core(dev: &pci::Device<Core>, devres: Devres<pci::Bar<0x4>>) -> Result {
+ /// let bar = devres.access(dev.as_ref())?;
+ ///
+ /// let _ = bar.read32(0x0);
+ ///
+ /// // might_sleep()
+ ///
+ /// bar.write32(0x42, 0x0);
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub fn access<'a>(&'a self, dev: &'a Device<Bound>) -> Result<&'a T> {
+ if self.0.dev.as_raw() != dev.as_raw() {
+ return Err(EINVAL);
+ }
+
+ // SAFETY: `dev` being the same device as the device this `Devres` has been created for
+ // proves that `self.0.data` hasn't been revoked and is guaranteed to not be revoked as
+ // long as `dev` lives; `dev` lives at least as long as `self`.
+ Ok(unsafe { self.0.data.access() })
+ }
+
+ /// [`Devres`] accessor for [`Revocable::try_access`].
+ pub fn try_access(&self) -> Option<RevocableGuard<'_, T>> {
+ self.0.data.try_access()
+ }
- fn deref(&self) -> &Self::Target {
- &self.0.data
+ /// [`Devres`] accessor for [`Revocable::try_access_with`].
+ pub fn try_access_with<R, F: FnOnce(&T) -> R>(&self, f: F) -> Option<R> {
+ self.0.data.try_access_with(f)
+ }
+
+ /// [`Devres`] accessor for [`Revocable::try_access_with_guard`].
+ pub fn try_access_with_guard<'a>(&'a self, guard: &'a rcu::Guard) -> Option<&'a T> {
+ self.0.data.try_access_with_guard(guard)
}
}
impl<T> Drop for Devres<T> {
fn drop(&mut self) {
- DevresInner::remove_action(&self.0);
+ // SAFETY: When `drop` runs, it is guaranteed that nobody is accessing the revocable data
+ // anymore, hence it is safe not to wait for the grace period to finish.
+ if unsafe { self.0.data.revoke_nosync() } {
+ // We revoked `self.0.data` before the devres action did, hence try to remove it.
+ if !DevresInner::remove_action(&self.0) {
+ // We could not remove the devres action, which means that it now runs concurrently,
+ // hence signal that `self.0.data` has been revoked successfully.
+ self.0.revoke.complete_all();
+ }
+ }
}
}
diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs
new file mode 100644
index 000000000000..a33261c62e0c
--- /dev/null
+++ b/rust/kernel/dma.rs
@@ -0,0 +1,391 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Direct memory access (DMA).
+//!
+//! C header: [`include/linux/dma-mapping.h`](srctree/include/linux/dma-mapping.h)
+
+use crate::{
+ bindings, build_assert,
+ device::{Bound, Device},
+ error::code::*,
+ error::Result,
+ transmute::{AsBytes, FromBytes},
+ types::ARef,
+};
+
+/// Possible attributes associated with a DMA mapping.
+///
+/// They can be combined with the operators `|`, `&`, and `!`.
+///
+/// Values can be used from the [`attrs`] module.
+///
+/// # Examples
+///
+/// ```
+/// # use kernel::device::{Bound, Device};
+/// use kernel::dma::{attrs::*, CoherentAllocation};
+///
+/// # fn test(dev: &Device<Bound>) -> Result {
+/// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN;
+/// let c: CoherentAllocation<u64> =
+/// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, attribs)?;
+/// # Ok::<(), Error>(()) }
+/// ```
+#[derive(Clone, Copy, PartialEq)]
+#[repr(transparent)]
+pub struct Attrs(u32);
+
+impl Attrs {
+ /// Get the raw representation of this attribute.
+ pub(crate) fn as_raw(self) -> crate::ffi::c_ulong {
+ self.0 as _
+ }
+
+ /// Check whether `flags` is contained in `self`.
+ pub fn contains(self, flags: Attrs) -> bool {
+ (self & flags) == flags
+ }
+}
+
+impl core::ops::BitOr for Attrs {
+ type Output = Self;
+ fn bitor(self, rhs: Self) -> Self::Output {
+ Self(self.0 | rhs.0)
+ }
+}
+
+impl core::ops::BitAnd for Attrs {
+ type Output = Self;
+ fn bitand(self, rhs: Self) -> Self::Output {
+ Self(self.0 & rhs.0)
+ }
+}
+
+impl core::ops::Not for Attrs {
+ type Output = Self;
+ fn not(self) -> Self::Output {
+ Self(!self.0)
+ }
+}
+
+/// DMA mapping attributes.
+pub mod attrs {
+ use super::Attrs;
+
+ /// Specifies that reads and writes to the mapping may be weakly ordered, that is that reads
+ /// and writes may pass each other.
+ pub const DMA_ATTR_WEAK_ORDERING: Attrs = Attrs(bindings::DMA_ATTR_WEAK_ORDERING);
+
+ /// Specifies that writes to the mapping may be buffered to improve performance.
+ pub const DMA_ATTR_WRITE_COMBINE: Attrs = Attrs(bindings::DMA_ATTR_WRITE_COMBINE);
+
+ /// Lets the platform to avoid creating a kernel virtual mapping for the allocated buffer.
+ pub const DMA_ATTR_NO_KERNEL_MAPPING: Attrs = Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING);
+
+ /// Allows platform code to skip synchronization of the CPU cache for the given buffer assuming
+ /// that it has been already transferred to 'device' domain.
+ pub const DMA_ATTR_SKIP_CPU_SYNC: Attrs = Attrs(bindings::DMA_ATTR_SKIP_CPU_SYNC);
+
+ /// Forces contiguous allocation of the buffer in physical memory.
+ pub const DMA_ATTR_FORCE_CONTIGUOUS: Attrs = Attrs(bindings::DMA_ATTR_FORCE_CONTIGUOUS);
+
+ /// This is a hint to the DMA-mapping subsystem that it's probably not worth the time to try
+ /// to allocate memory to in a way that gives better TLB efficiency.
+ pub const DMA_ATTR_ALLOC_SINGLE_PAGES: Attrs = Attrs(bindings::DMA_ATTR_ALLOC_SINGLE_PAGES);
+
+ /// This tells the DMA-mapping subsystem to suppress allocation failure reports (similarly to
+ /// `__GFP_NOWARN`).
+ pub const DMA_ATTR_NO_WARN: Attrs = Attrs(bindings::DMA_ATTR_NO_WARN);
+
+ /// Used to indicate that the buffer is fully accessible at an elevated privilege level (and
+ /// ideally inaccessible or at least read-only at lesser-privileged levels).
+ pub const DMA_ATTR_PRIVILEGED: Attrs = Attrs(bindings::DMA_ATTR_PRIVILEGED);
+}
+
+/// An abstraction of the `dma_alloc_coherent` API.
+///
+/// This is an abstraction around the `dma_alloc_coherent` API which is used to allocate and map
+/// large consistent DMA regions.
+///
+/// A [`CoherentAllocation`] instance contains a pointer to the allocated region (in the
+/// processor's virtual address space) and the device address which can be given to the device
+/// as the DMA address base of the region. The region is released once [`CoherentAllocation`]
+/// is dropped.
+///
+/// # Invariants
+///
+/// For the lifetime of an instance of [`CoherentAllocation`], the `cpu_addr` is a valid pointer
+/// to an allocated region of consistent memory and `dma_handle` is the DMA address base of
+/// the region.
+// TODO
+//
+// DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness
+// reasons DMA allocation would need to be embedded in a `Devres` container, in order to ensure
+// that device resources can never survive device unbind.
+//
+// However, it is neither desirable nor necessary to protect the allocated memory of the DMA
+// allocation from surviving device unbind; it would require RCU read side critical sections to
+// access the memory, which may require subsequent unnecessary copies.
+//
+// Hence, find a way to revoke the device resources of a `CoherentAllocation`, but not the
+// entire `CoherentAllocation` including the allocated memory itself.
+pub struct CoherentAllocation<T: AsBytes + FromBytes> {
+ dev: ARef<Device>,
+ dma_handle: bindings::dma_addr_t,
+ count: usize,
+ cpu_addr: *mut T,
+ dma_attrs: Attrs,
+}
+
+impl<T: AsBytes + FromBytes> CoherentAllocation<T> {
+ /// Allocates a region of `size_of::<T> * count` of consistent memory.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// # use kernel::device::{Bound, Device};
+ /// use kernel::dma::{attrs::*, CoherentAllocation};
+ ///
+ /// # fn test(dev: &Device<Bound>) -> Result {
+ /// let c: CoherentAllocation<u64> =
+ /// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
+ /// # Ok::<(), Error>(()) }
+ /// ```
+ pub fn alloc_attrs(
+ dev: &Device<Bound>,
+ count: usize,
+ gfp_flags: kernel::alloc::Flags,
+ dma_attrs: Attrs,
+ ) -> Result<CoherentAllocation<T>> {
+ build_assert!(
+ core::mem::size_of::<T>() > 0,
+ "It doesn't make sense for the allocated type to be a ZST"
+ );
+
+ let size = count
+ .checked_mul(core::mem::size_of::<T>())
+ .ok_or(EOVERFLOW)?;
+ let mut dma_handle = 0;
+ // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`.
+ let ret = unsafe {
+ bindings::dma_alloc_attrs(
+ dev.as_raw(),
+ size,
+ &mut dma_handle,
+ gfp_flags.as_raw(),
+ dma_attrs.as_raw(),
+ )
+ };
+ if ret.is_null() {
+ return Err(ENOMEM);
+ }
+ // INVARIANT: We just successfully allocated a coherent region which is accessible for
+ // `count` elements, hence the cpu address is valid. We also hold a refcounted reference
+ // to the device.
+ Ok(Self {
+ dev: dev.into(),
+ dma_handle,
+ count,
+ cpu_addr: ret as *mut T,
+ dma_attrs,
+ })
+ }
+
+ /// Performs the same functionality as [`CoherentAllocation::alloc_attrs`], except the
+ /// `dma_attrs` is 0 by default.
+ pub fn alloc_coherent(
+ dev: &Device<Bound>,
+ count: usize,
+ gfp_flags: kernel::alloc::Flags,
+ ) -> Result<CoherentAllocation<T>> {
+ CoherentAllocation::alloc_attrs(dev, count, gfp_flags, Attrs(0))
+ }
+
+ /// Returns the base address to the allocated region in the CPU's virtual address space.
+ pub fn start_ptr(&self) -> *const T {
+ self.cpu_addr
+ }
+
+ /// Returns the base address to the allocated region in the CPU's virtual address space as
+ /// a mutable pointer.
+ pub fn start_ptr_mut(&mut self) -> *mut T {
+ self.cpu_addr
+ }
+
+ /// Returns a DMA handle which may given to the device as the DMA address base of
+ /// the region.
+ pub fn dma_handle(&self) -> bindings::dma_addr_t {
+ self.dma_handle
+ }
+
+ /// Returns a pointer to an element from the region with bounds checking. `offset` is in
+ /// units of `T`, not the number of bytes.
+ ///
+ /// Public but hidden since it should only be used from [`dma_read`] and [`dma_write`] macros.
+ #[doc(hidden)]
+ pub fn item_from_index(&self, offset: usize) -> Result<*mut T> {
+ if offset >= self.count {
+ return Err(EINVAL);
+ }
+ // SAFETY:
+ // - The pointer is valid due to type invariant on `CoherentAllocation`
+ // and we've just checked that the range and index is within bounds.
+ // - `offset` can't overflow since it is smaller than `self.count` and we've checked
+ // that `self.count` won't overflow early in the constructor.
+ Ok(unsafe { self.cpu_addr.add(offset) })
+ }
+
+ /// Reads the value of `field` and ensures that its type is [`FromBytes`].
+ ///
+ /// # Safety
+ ///
+ /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is
+ /// validated beforehand.
+ ///
+ /// Public but hidden since it should only be used from [`dma_read`] macro.
+ #[doc(hidden)]
+ pub unsafe fn field_read<F: FromBytes>(&self, field: *const F) -> F {
+ // SAFETY:
+ // - By the safety requirements field is valid.
+ // - Using read_volatile() here is not sound as per the usual rules, the usage here is
+ // a special exception with the following notes in place. When dealing with a potential
+ // race from a hardware or code outside kernel (e.g. user-space program), we need that
+ // read on a valid memory is not UB. Currently read_volatile() is used for this, and the
+ // rationale behind is that it should generate the same code as READ_ONCE() which the
+ // kernel already relies on to avoid UB on data races. Note that the usage of
+ // read_volatile() is limited to this particular case, it cannot be used to prevent
+ // the UB caused by racing between two kernel functions nor do they provide atomicity.
+ unsafe { field.read_volatile() }
+ }
+
+ /// Writes a value to `field` and ensures that its type is [`AsBytes`].
+ ///
+ /// # Safety
+ ///
+ /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is
+ /// validated beforehand.
+ ///
+ /// Public but hidden since it should only be used from [`dma_write`] macro.
+ #[doc(hidden)]
+ pub unsafe fn field_write<F: AsBytes>(&self, field: *mut F, val: F) {
+ // SAFETY:
+ // - By the safety requirements field is valid.
+ // - Using write_volatile() here is not sound as per the usual rules, the usage here is
+ // a special exception with the following notes in place. When dealing with a potential
+ // race from a hardware or code outside kernel (e.g. user-space program), we need that
+ // write on a valid memory is not UB. Currently write_volatile() is used for this, and the
+ // rationale behind is that it should generate the same code as WRITE_ONCE() which the
+ // kernel already relies on to avoid UB on data races. Note that the usage of
+ // write_volatile() is limited to this particular case, it cannot be used to prevent
+ // the UB caused by racing between two kernel functions nor do they provide atomicity.
+ unsafe { field.write_volatile(val) }
+ }
+}
+
+/// Note that the device configured to do DMA must be halted before this object is dropped.
+impl<T: AsBytes + FromBytes> Drop for CoherentAllocation<T> {
+ fn drop(&mut self) {
+ let size = self.count * core::mem::size_of::<T>();
+ // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`.
+ // The cpu address, and the dma handle are valid due to the type invariants on
+ // `CoherentAllocation`.
+ unsafe {
+ bindings::dma_free_attrs(
+ self.dev.as_raw(),
+ size,
+ self.cpu_addr as _,
+ self.dma_handle,
+ self.dma_attrs.as_raw(),
+ )
+ }
+ }
+}
+
+// SAFETY: It is safe to send a `CoherentAllocation` to another thread if `T`
+// can be sent to another thread.
+unsafe impl<T: AsBytes + FromBytes + Send> Send for CoherentAllocation<T> {}
+
+/// Reads a field of an item from an allocated region of structs.
+///
+/// # Examples
+///
+/// ```
+/// use kernel::device::Device;
+/// use kernel::dma::{attrs::*, CoherentAllocation};
+///
+/// struct MyStruct { field: u32, }
+///
+/// // SAFETY: All bit patterns are acceptable values for `MyStruct`.
+/// unsafe impl kernel::transmute::FromBytes for MyStruct{};
+/// // SAFETY: Instances of `MyStruct` have no uninitialized portions.
+/// unsafe impl kernel::transmute::AsBytes for MyStruct{};
+///
+/// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result {
+/// let whole = kernel::dma_read!(alloc[2]);
+/// let field = kernel::dma_read!(alloc[1].field);
+/// # Ok::<(), Error>(()) }
+/// ```
+#[macro_export]
+macro_rules! dma_read {
+ ($dma:expr, $idx: expr, $($field:tt)*) => {{
+ let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
+ // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be
+ // dereferenced. The compiler also further validates the expression on whether `field`
+ // is a member of `item` when expanded by the macro.
+ unsafe {
+ let ptr_field = ::core::ptr::addr_of!((*item) $($field)*);
+ $crate::dma::CoherentAllocation::field_read(&$dma, ptr_field)
+ }
+ }};
+ ($dma:ident [ $idx:expr ] $($field:tt)* ) => {
+ $crate::dma_read!($dma, $idx, $($field)*);
+ };
+ ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {
+ $crate::dma_read!($($dma).*, $idx, $($field)*);
+ };
+}
+
+/// Writes to a field of an item from an allocated region of structs.
+///
+/// # Examples
+///
+/// ```
+/// use kernel::device::Device;
+/// use kernel::dma::{attrs::*, CoherentAllocation};
+///
+/// struct MyStruct { member: u32, }
+///
+/// // SAFETY: All bit patterns are acceptable values for `MyStruct`.
+/// unsafe impl kernel::transmute::FromBytes for MyStruct{};
+/// // SAFETY: Instances of `MyStruct` have no uninitialized portions.
+/// unsafe impl kernel::transmute::AsBytes for MyStruct{};
+///
+/// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result {
+/// kernel::dma_write!(alloc[2].member = 0xf);
+/// kernel::dma_write!(alloc[1] = MyStruct { member: 0xf });
+/// # Ok::<(), Error>(()) }
+/// ```
+#[macro_export]
+macro_rules! dma_write {
+ ($dma:ident [ $idx:expr ] $($field:tt)*) => {{
+ $crate::dma_write!($dma, $idx, $($field)*);
+ }};
+ ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {{
+ $crate::dma_write!($($dma).*, $idx, $($field)*);
+ }};
+ ($dma:expr, $idx: expr, = $val:expr) => {
+ let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
+ // SAFETY: `item_from_index` ensures that `item` is always a valid item.
+ unsafe { $crate::dma::CoherentAllocation::field_write(&$dma, item, $val) }
+ };
+ ($dma:expr, $idx: expr, $(.$field:ident)* = $val:expr) => {
+ let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
+ // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be
+ // dereferenced. The compiler also further validates the expression on whether `field`
+ // is a member of `item` when expanded by the macro.
+ unsafe {
+ let ptr_field = ::core::ptr::addr_of_mut!((*item) $(.$field)*);
+ $crate::dma::CoherentAllocation::field_write(&$dma, ptr_field, $val)
+ }
+ };
+}
diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs
index 2a16d5e64e6c..ec9166cedfa7 100644
--- a/rust/kernel/driver.rs
+++ b/rust/kernel/driver.rs
@@ -6,9 +6,9 @@
//! register using the [`Registration`] class.
use crate::error::{Error, Result};
-use crate::{device, init::PinInit, of, str::CStr, try_pin_init, types::Opaque, ThisModule};
+use crate::{device, of, str::CStr, try_pin_init, types::Opaque, ThisModule};
use core::pin::Pin;
-use macros::{pin_data, pinned_drop};
+use pin_init::{pin_data, pinned_drop, PinInit};
/// The [`RegistrationOps`] trait serves as generic interface for subsystems (e.g., PCI, Platform,
/// Amba, etc.) to provide the corresponding subsystem specific implementation to register /
@@ -114,7 +114,7 @@ macro_rules! module_driver {
impl $crate::InPlaceModule for DriverModule {
fn init(
module: &'static $crate::ThisModule
- ) -> impl $crate::init::PinInit<Self, $crate::error::Error> {
+ ) -> impl ::pin_init::PinInit<Self, $crate::error::Error> {
$crate::try_pin_init!(Self {
_driver <- $crate::driver::Registration::new(
<Self as $crate::ModuleMetadata>::NAME,
diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs
new file mode 100644
index 000000000000..624d7a4c83ea
--- /dev/null
+++ b/rust/kernel/drm/device.rs
@@ -0,0 +1,202 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+//! DRM device.
+//!
+//! C header: [`include/linux/drm/drm_device.h`](srctree/include/linux/drm/drm_device.h)
+
+use crate::{
+ bindings, device, drm,
+ drm::driver::AllocImpl,
+ error::from_err_ptr,
+ error::Result,
+ prelude::*,
+ types::{ARef, AlwaysRefCounted, Opaque},
+};
+use core::{mem, ops::Deref, ptr, ptr::NonNull};
+
+#[cfg(CONFIG_DRM_LEGACY)]
+macro_rules! drm_legacy_fields {
+ ( $($field:ident: $val:expr),* $(,)? ) => {
+ bindings::drm_driver {
+ $( $field: $val ),*,
+ firstopen: None,
+ preclose: None,
+ dma_ioctl: None,
+ dma_quiescent: None,
+ context_dtor: None,
+ irq_handler: None,
+ irq_preinstall: None,
+ irq_postinstall: None,
+ irq_uninstall: None,
+ get_vblank_counter: None,
+ enable_vblank: None,
+ disable_vblank: None,
+ dev_priv_size: 0,
+ }
+ }
+}
+
+#[cfg(not(CONFIG_DRM_LEGACY))]
+macro_rules! drm_legacy_fields {
+ ( $($field:ident: $val:expr),* $(,)? ) => {
+ bindings::drm_driver {
+ $( $field: $val ),*
+ }
+ }
+}
+
+/// A typed DRM device with a specific `drm::Driver` implementation.
+///
+/// The device is always reference-counted.
+///
+/// # Invariants
+///
+/// `self.dev` is a valid instance of a `struct device`.
+#[repr(C)]
+#[pin_data]
+pub struct Device<T: drm::Driver> {
+ dev: Opaque<bindings::drm_device>,
+ #[pin]
+ data: T::Data,
+}
+
+impl<T: drm::Driver> Device<T> {
+ const VTABLE: bindings::drm_driver = drm_legacy_fields! {
+ load: None,
+ open: Some(drm::File::<T::File>::open_callback),
+ postclose: Some(drm::File::<T::File>::postclose_callback),
+ unload: None,
+ release: None,
+ master_set: None,
+ master_drop: None,
+ debugfs_init: None,
+ gem_create_object: T::Object::ALLOC_OPS.gem_create_object,
+ prime_handle_to_fd: T::Object::ALLOC_OPS.prime_handle_to_fd,
+ prime_fd_to_handle: T::Object::ALLOC_OPS.prime_fd_to_handle,
+ gem_prime_import: T::Object::ALLOC_OPS.gem_prime_import,
+ gem_prime_import_sg_table: T::Object::ALLOC_OPS.gem_prime_import_sg_table,
+ dumb_create: T::Object::ALLOC_OPS.dumb_create,
+ dumb_map_offset: T::Object::ALLOC_OPS.dumb_map_offset,
+ show_fdinfo: None,
+ fbdev_probe: None,
+
+ major: T::INFO.major,
+ minor: T::INFO.minor,
+ patchlevel: T::INFO.patchlevel,
+ name: T::INFO.name.as_char_ptr() as *mut _,
+ desc: T::INFO.desc.as_char_ptr() as *mut _,
+
+ driver_features: drm::driver::FEAT_GEM,
+ ioctls: T::IOCTLS.as_ptr(),
+ num_ioctls: T::IOCTLS.len() as i32,
+ fops: &Self::GEM_FOPS as _,
+ };
+
+ const GEM_FOPS: bindings::file_operations = drm::gem::create_fops();
+
+ /// Create a new `drm::Device` for a `drm::Driver`.
+ pub fn new(dev: &device::Device, data: impl PinInit<T::Data, Error>) -> Result<ARef<Self>> {
+ // SAFETY:
+ // - `VTABLE`, as a `const` is pinned to the read-only section of the compilation,
+ // - `dev` is valid by its type invarants,
+ let raw_drm: *mut Self = unsafe {
+ bindings::__drm_dev_alloc(
+ dev.as_raw(),
+ &Self::VTABLE,
+ mem::size_of::<Self>(),
+ mem::offset_of!(Self, dev),
+ )
+ }
+ .cast();
+ let raw_drm = NonNull::new(from_err_ptr(raw_drm)?).ok_or(ENOMEM)?;
+
+ // SAFETY: `raw_drm` is a valid pointer to `Self`.
+ let raw_data = unsafe { ptr::addr_of_mut!((*raw_drm.as_ptr()).data) };
+
+ // SAFETY:
+ // - `raw_data` is a valid pointer to uninitialized memory.
+ // - `raw_data` will not move until it is dropped.
+ unsafe { data.__pinned_init(raw_data) }.inspect_err(|_| {
+ // SAFETY: `__drm_dev_alloc()` was successful, hence `raw_drm` must be valid and the
+ // refcount must be non-zero.
+ unsafe { bindings::drm_dev_put(ptr::addr_of_mut!((*raw_drm.as_ptr()).dev).cast()) };
+ })?;
+
+ // SAFETY: The reference count is one, and now we take ownership of that reference as a
+ // `drm::Device`.
+ Ok(unsafe { ARef::from_raw(raw_drm) })
+ }
+
+ pub(crate) fn as_raw(&self) -> *mut bindings::drm_device {
+ self.dev.get()
+ }
+
+ /// # Safety
+ ///
+ /// `ptr` must be a valid pointer to a `struct device` embedded in `Self`.
+ unsafe fn from_drm_device(ptr: *const bindings::drm_device) -> *mut Self {
+ let ptr: *const Opaque<bindings::drm_device> = ptr.cast();
+
+ // SAFETY: By the safety requirements of this function `ptr` is a valid pointer to a
+ // `struct drm_device` embedded in `Self`.
+ unsafe { crate::container_of!(ptr, Self, dev) }.cast_mut()
+ }
+
+ /// Not intended to be called externally, except via declare_drm_ioctls!()
+ ///
+ /// # Safety
+ ///
+ /// Callers must ensure that `ptr` is valid, non-null, and has a non-zero reference count,
+ /// i.e. it must be ensured that the reference count of the C `struct drm_device` `ptr` points
+ /// to can't drop to zero, for the duration of this function call and the entire duration when
+ /// the returned reference exists.
+ ///
+ /// Additionally, callers must ensure that the `struct device`, `ptr` is pointing to, is
+ /// embedded in `Self`.
+ #[doc(hidden)]
+ pub unsafe fn as_ref<'a>(ptr: *const bindings::drm_device) -> &'a Self {
+ // SAFETY: By the safety requirements of this function `ptr` is a valid pointer to a
+ // `struct drm_device` embedded in `Self`.
+ let ptr = unsafe { Self::from_drm_device(ptr) };
+
+ // SAFETY: `ptr` is valid by the safety requirements of this function.
+ unsafe { &*ptr.cast() }
+ }
+}
+
+impl<T: drm::Driver> Deref for Device<T> {
+ type Target = T::Data;
+
+ fn deref(&self) -> &Self::Target {
+ &self.data
+ }
+}
+
+// SAFETY: DRM device objects are always reference counted and the get/put functions
+// satisfy the requirements.
+unsafe impl<T: drm::Driver> AlwaysRefCounted for Device<T> {
+ fn inc_ref(&self) {
+ // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero.
+ unsafe { bindings::drm_dev_get(self.as_raw()) };
+ }
+
+ unsafe fn dec_ref(obj: NonNull<Self>) {
+ // SAFETY: The safety requirements guarantee that the refcount is non-zero.
+ unsafe { bindings::drm_dev_put(obj.cast().as_ptr()) };
+ }
+}
+
+impl<T: drm::Driver> AsRef<device::Device> for Device<T> {
+ fn as_ref(&self) -> &device::Device {
+ // SAFETY: `bindings::drm_device::dev` is valid as long as the DRM device itself is valid,
+ // which is guaranteed by the type invariant.
+ unsafe { device::Device::as_ref((*self.as_raw()).dev) }
+ }
+}
+
+// SAFETY: A `drm::Device` can be released from any thread.
+unsafe impl<T: drm::Driver> Send for Device<T> {}
+
+// SAFETY: A `drm::Device` can be shared among threads because all immutable methods are protected
+// by the synchronization in `struct drm_device`.
+unsafe impl<T: drm::Driver> Sync for Device<T> {}
diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs
new file mode 100644
index 000000000000..acb638086131
--- /dev/null
+++ b/rust/kernel/drm/driver.rs
@@ -0,0 +1,166 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+//! DRM driver core.
+//!
+//! C header: [`include/linux/drm/drm_drv.h`](srctree/include/linux/drm/drm_drv.h)
+
+use crate::{
+ bindings, device,
+ devres::Devres,
+ drm,
+ error::{to_result, Result},
+ prelude::*,
+ str::CStr,
+ types::ARef,
+};
+use macros::vtable;
+
+/// Driver use the GEM memory manager. This should be set for all modern drivers.
+pub(crate) const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM;
+
+/// Information data for a DRM Driver.
+pub struct DriverInfo {
+ /// Driver major version.
+ pub major: i32,
+ /// Driver minor version.
+ pub minor: i32,
+ /// Driver patchlevel version.
+ pub patchlevel: i32,
+ /// Driver name.
+ pub name: &'static CStr,
+ /// Driver description.
+ pub desc: &'static CStr,
+}
+
+/// Internal memory management operation set, normally created by memory managers (e.g. GEM).
+pub struct AllocOps {
+ pub(crate) gem_create_object: Option<
+ unsafe extern "C" fn(
+ dev: *mut bindings::drm_device,
+ size: usize,
+ ) -> *mut bindings::drm_gem_object,
+ >,
+ pub(crate) prime_handle_to_fd: Option<
+ unsafe extern "C" fn(
+ dev: *mut bindings::drm_device,
+ file_priv: *mut bindings::drm_file,
+ handle: u32,
+ flags: u32,
+ prime_fd: *mut core::ffi::c_int,
+ ) -> core::ffi::c_int,
+ >,
+ pub(crate) prime_fd_to_handle: Option<
+ unsafe extern "C" fn(
+ dev: *mut bindings::drm_device,
+ file_priv: *mut bindings::drm_file,
+ prime_fd: core::ffi::c_int,
+ handle: *mut u32,
+ ) -> core::ffi::c_int,
+ >,
+ pub(crate) gem_prime_import: Option<
+ unsafe extern "C" fn(
+ dev: *mut bindings::drm_device,
+ dma_buf: *mut bindings::dma_buf,
+ ) -> *mut bindings::drm_gem_object,
+ >,
+ pub(crate) gem_prime_import_sg_table: Option<
+ unsafe extern "C" fn(
+ dev: *mut bindings::drm_device,
+ attach: *mut bindings::dma_buf_attachment,
+ sgt: *mut bindings::sg_table,
+ ) -> *mut bindings::drm_gem_object,
+ >,
+ pub(crate) dumb_create: Option<
+ unsafe extern "C" fn(
+ file_priv: *mut bindings::drm_file,
+ dev: *mut bindings::drm_device,
+ args: *mut bindings::drm_mode_create_dumb,
+ ) -> core::ffi::c_int,
+ >,
+ pub(crate) dumb_map_offset: Option<
+ unsafe extern "C" fn(
+ file_priv: *mut bindings::drm_file,
+ dev: *mut bindings::drm_device,
+ handle: u32,
+ offset: *mut u64,
+ ) -> core::ffi::c_int,
+ >,
+}
+
+/// Trait for memory manager implementations. Implemented internally.
+pub trait AllocImpl: super::private::Sealed + drm::gem::IntoGEMObject {
+ /// The C callback operations for this memory manager.
+ const ALLOC_OPS: AllocOps;
+}
+
+/// The DRM `Driver` trait.
+///
+/// This trait must be implemented by drivers in order to create a `struct drm_device` and `struct
+/// drm_driver` to be registered in the DRM subsystem.
+#[vtable]
+pub trait Driver {
+ /// Context data associated with the DRM driver
+ type Data: Sync + Send;
+
+ /// The type used to manage memory for this driver.
+ type Object: AllocImpl;
+
+ /// The type used to represent a DRM File (client)
+ type File: drm::file::DriverFile;
+
+ /// Driver metadata
+ const INFO: DriverInfo;
+
+ /// IOCTL list. See `kernel::drm::ioctl::declare_drm_ioctls!{}`.
+ const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor];
+}
+
+/// The registration type of a `drm::Device`.
+///
+/// Once the `Registration` structure is dropped, the device is unregistered.
+pub struct Registration<T: Driver>(ARef<drm::Device<T>>);
+
+impl<T: Driver> Registration<T> {
+ /// Creates a new [`Registration`] and registers it.
+ fn new(drm: &drm::Device<T>, flags: usize) -> Result<Self> {
+ // SAFETY: `drm.as_raw()` is valid by the invariants of `drm::Device`.
+ to_result(unsafe { bindings::drm_dev_register(drm.as_raw(), flags) })?;
+
+ Ok(Self(drm.into()))
+ }
+
+ /// Same as [`Registration::new`}, but transfers ownership of the [`Registration`] to
+ /// [`Devres`].
+ pub fn new_foreign_owned(
+ drm: &drm::Device<T>,
+ dev: &device::Device<device::Bound>,
+ flags: usize,
+ ) -> Result {
+ if drm.as_ref().as_raw() != dev.as_raw() {
+ return Err(EINVAL);
+ }
+
+ let reg = Registration::<T>::new(drm, flags)?;
+ Devres::new_foreign_owned(dev, reg, GFP_KERNEL)
+ }
+
+ /// Returns a reference to the `Device` instance for this registration.
+ pub fn device(&self) -> &drm::Device<T> {
+ &self.0
+ }
+}
+
+// SAFETY: `Registration` doesn't offer any methods or access to fields when shared between
+// threads, hence it's safe to share it.
+unsafe impl<T: Driver> Sync for Registration<T> {}
+
+// SAFETY: Registration with and unregistration from the DRM subsystem can happen from any thread.
+unsafe impl<T: Driver> Send for Registration<T> {}
+
+impl<T: Driver> Drop for Registration<T> {
+ fn drop(&mut self) {
+ // SAFETY: Safe by the invariant of `ARef<drm::Device<T>>`. The existence of this
+ // `Registration` also guarantees the this `drm::Device` is actually registered.
+ unsafe { bindings::drm_dev_unregister(self.0.as_raw()) };
+ }
+}
diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs
new file mode 100644
index 000000000000..b9527705e551
--- /dev/null
+++ b/rust/kernel/drm/file.rs
@@ -0,0 +1,99 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+//! DRM File objects.
+//!
+//! C header: [`include/linux/drm/drm_file.h`](srctree/include/linux/drm/drm_file.h)
+
+use crate::{bindings, drm, error::Result, prelude::*, types::Opaque};
+use core::marker::PhantomData;
+use core::pin::Pin;
+
+/// Trait that must be implemented by DRM drivers to represent a DRM File (a client instance).
+pub trait DriverFile {
+ /// The parent `Driver` implementation for this `DriverFile`.
+ type Driver: drm::Driver;
+
+ /// Open a new file (called when a client opens the DRM device).
+ fn open(device: &drm::Device<Self::Driver>) -> Result<Pin<KBox<Self>>>;
+}
+
+/// An open DRM File.
+///
+/// # Invariants
+///
+/// `self.0` is a valid instance of a `struct drm_file`.
+#[repr(transparent)]
+pub struct File<T: DriverFile>(Opaque<bindings::drm_file>, PhantomData<T>);
+
+impl<T: DriverFile> File<T> {
+ #[doc(hidden)]
+ /// Not intended to be called externally, except via declare_drm_ioctls!()
+ ///
+ /// # Safety
+ ///
+ /// `raw_file` must be a valid pointer to an open `struct drm_file`, opened through `T::open`.
+ pub unsafe fn as_ref<'a>(ptr: *mut bindings::drm_file) -> &'a File<T> {
+ // SAFETY: `raw_file` is valid by the safety requirements of this function.
+ unsafe { &*ptr.cast() }
+ }
+
+ pub(super) fn as_raw(&self) -> *mut bindings::drm_file {
+ self.0.get()
+ }
+
+ fn driver_priv(&self) -> *mut T {
+ // SAFETY: By the type invariants of `Self`, `self.as_raw()` is always valid.
+ unsafe { (*self.as_raw()).driver_priv }.cast()
+ }
+
+ /// Return a pinned reference to the driver file structure.
+ pub fn inner(&self) -> Pin<&T> {
+ // SAFETY: By the type invariant the pointer `self.as_raw()` points to a valid and opened
+ // `struct drm_file`, hence `driver_priv` has been properly initialized by `open_callback`.
+ unsafe { Pin::new_unchecked(&*(self.driver_priv())) }
+ }
+
+ /// The open callback of a `struct drm_file`.
+ pub(crate) extern "C" fn open_callback(
+ raw_dev: *mut bindings::drm_device,
+ raw_file: *mut bindings::drm_file,
+ ) -> core::ffi::c_int {
+ // SAFETY: A callback from `struct drm_driver::open` guarantees that
+ // - `raw_dev` is valid pointer to a `struct drm_device`,
+ // - the corresponding `struct drm_device` has been registered.
+ let drm = unsafe { drm::Device::as_ref(raw_dev) };
+
+ // SAFETY: `raw_file` is a valid pointer to a `struct drm_file`.
+ let file = unsafe { File::<T>::as_ref(raw_file) };
+
+ let inner = match T::open(drm) {
+ Err(e) => {
+ return e.to_errno();
+ }
+ Ok(i) => i,
+ };
+
+ // SAFETY: This pointer is treated as pinned, and the Drop guarantee is upheld in
+ // `postclose_callback()`.
+ let driver_priv = KBox::into_raw(unsafe { Pin::into_inner_unchecked(inner) });
+
+ // SAFETY: By the type invariants of `Self`, `self.as_raw()` is always valid.
+ unsafe { (*file.as_raw()).driver_priv = driver_priv.cast() };
+
+ 0
+ }
+
+ /// The postclose callback of a `struct drm_file`.
+ pub(crate) extern "C" fn postclose_callback(
+ _raw_dev: *mut bindings::drm_device,
+ raw_file: *mut bindings::drm_file,
+ ) {
+ // SAFETY: This reference won't escape this function
+ let file = unsafe { File::<T>::as_ref(raw_file) };
+
+ // SAFETY: `file.driver_priv` has been created in `open_callback` through `KBox::into_raw`.
+ let _ = unsafe { KBox::from_raw(file.driver_priv()) };
+ }
+}
+
+impl<T: DriverFile> super::private::Sealed for File<T> {}
diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs
new file mode 100644
index 000000000000..4cd69fa84318
--- /dev/null
+++ b/rust/kernel/drm/gem/mod.rs
@@ -0,0 +1,332 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+//! DRM GEM API
+//!
+//! C header: [`include/linux/drm/drm_gem.h`](srctree/include/linux/drm/drm_gem.h)
+
+use crate::{
+ alloc::flags::*,
+ bindings, drm,
+ drm::driver::{AllocImpl, AllocOps},
+ error::{to_result, Result},
+ prelude::*,
+ types::{ARef, AlwaysRefCounted, Opaque},
+};
+use core::{mem, ops::Deref, ptr::NonNull};
+
+/// GEM object functions, which must be implemented by drivers.
+pub trait BaseDriverObject<T: BaseObject>: Sync + Send + Sized {
+ /// Create a new driver data object for a GEM object of a given size.
+ fn new(dev: &drm::Device<T::Driver>, size: usize) -> impl PinInit<Self, Error>;
+
+ /// Open a new handle to an existing object, associated with a File.
+ fn open(
+ _obj: &<<T as IntoGEMObject>::Driver as drm::Driver>::Object,
+ _file: &drm::File<<<T as IntoGEMObject>::Driver as drm::Driver>::File>,
+ ) -> Result {
+ Ok(())
+ }
+
+ /// Close a handle to an existing object, associated with a File.
+ fn close(
+ _obj: &<<T as IntoGEMObject>::Driver as drm::Driver>::Object,
+ _file: &drm::File<<<T as IntoGEMObject>::Driver as drm::Driver>::File>,
+ ) {
+ }
+}
+
+/// Trait that represents a GEM object subtype
+pub trait IntoGEMObject: Sized + super::private::Sealed + AlwaysRefCounted {
+ /// Owning driver for this type
+ type Driver: drm::Driver;
+
+ /// Returns a reference to the raw `drm_gem_object` structure, which must be valid as long as
+ /// this owning object is valid.
+ fn as_raw(&self) -> *mut bindings::drm_gem_object;
+
+ /// Converts a pointer to a `struct drm_gem_object` into a reference to `Self`.
+ ///
+ /// # Safety
+ ///
+ /// - `self_ptr` must be a valid pointer to `Self`.
+ /// - The caller promises that holding the immutable reference returned by this function does
+ /// not violate rust's data aliasing rules and remains valid throughout the lifetime of `'a`.
+ unsafe fn as_ref<'a>(self_ptr: *mut bindings::drm_gem_object) -> &'a Self;
+}
+
+// SAFETY: All gem objects are refcounted.
+unsafe impl<T: IntoGEMObject> AlwaysRefCounted for T {
+ fn inc_ref(&self) {
+ // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero.
+ unsafe { bindings::drm_gem_object_get(self.as_raw()) };
+ }
+
+ unsafe fn dec_ref(obj: NonNull<Self>) {
+ // SAFETY: We either hold the only refcount on `obj`, or one of many - meaning that no one
+ // else could possibly hold a mutable reference to `obj` and thus this immutable reference
+ // is safe.
+ let obj = unsafe { obj.as_ref() }.as_raw();
+
+ // SAFETY:
+ // - The safety requirements guarantee that the refcount is non-zero.
+ // - We hold no references to `obj` now, making it safe for us to potentially deallocate it.
+ unsafe { bindings::drm_gem_object_put(obj) };
+ }
+}
+
+/// Trait which must be implemented by drivers using base GEM objects.
+pub trait DriverObject: BaseDriverObject<Object<Self>> {
+ /// Parent `Driver` for this object.
+ type Driver: drm::Driver;
+}
+
+extern "C" fn open_callback<T: BaseDriverObject<U>, U: BaseObject>(
+ raw_obj: *mut bindings::drm_gem_object,
+ raw_file: *mut bindings::drm_file,
+) -> core::ffi::c_int {
+ // SAFETY: `open_callback` is only ever called with a valid pointer to a `struct drm_file`.
+ let file = unsafe {
+ drm::File::<<<U as IntoGEMObject>::Driver as drm::Driver>::File>::as_ref(raw_file)
+ };
+ // SAFETY: `open_callback` is specified in the AllocOps structure for `Object<T>`, ensuring that
+ // `raw_obj` is indeed contained within a `Object<T>`.
+ let obj = unsafe {
+ <<<U as IntoGEMObject>::Driver as drm::Driver>::Object as IntoGEMObject>::as_ref(raw_obj)
+ };
+
+ match T::open(obj, file) {
+ Err(e) => e.to_errno(),
+ Ok(()) => 0,
+ }
+}
+
+extern "C" fn close_callback<T: BaseDriverObject<U>, U: BaseObject>(
+ raw_obj: *mut bindings::drm_gem_object,
+ raw_file: *mut bindings::drm_file,
+) {
+ // SAFETY: `open_callback` is only ever called with a valid pointer to a `struct drm_file`.
+ let file = unsafe {
+ drm::File::<<<U as IntoGEMObject>::Driver as drm::Driver>::File>::as_ref(raw_file)
+ };
+ // SAFETY: `close_callback` is specified in the AllocOps structure for `Object<T>`, ensuring
+ // that `raw_obj` is indeed contained within a `Object<T>`.
+ let obj = unsafe {
+ <<<U as IntoGEMObject>::Driver as drm::Driver>::Object as IntoGEMObject>::as_ref(raw_obj)
+ };
+
+ T::close(obj, file);
+}
+
+impl<T: DriverObject> IntoGEMObject for Object<T> {
+ type Driver = T::Driver;
+
+ fn as_raw(&self) -> *mut bindings::drm_gem_object {
+ self.obj.get()
+ }
+
+ unsafe fn as_ref<'a>(self_ptr: *mut bindings::drm_gem_object) -> &'a Self {
+ let self_ptr: *mut Opaque<bindings::drm_gem_object> = self_ptr.cast();
+
+ // SAFETY: `obj` is guaranteed to be in an `Object<T>` via the safety contract of this
+ // function
+ unsafe { &*crate::container_of!(self_ptr, Object<T>, obj) }
+ }
+}
+
+/// Base operations shared by all GEM object classes
+pub trait BaseObject: IntoGEMObject {
+ /// Returns the size of the object in bytes.
+ fn size(&self) -> usize {
+ // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `struct drm_gem_object`.
+ unsafe { (*self.as_raw()).size }
+ }
+
+ /// Creates a new handle for the object associated with a given `File`
+ /// (or returns an existing one).
+ fn create_handle(
+ &self,
+ file: &drm::File<<<Self as IntoGEMObject>::Driver as drm::Driver>::File>,
+ ) -> Result<u32> {
+ let mut handle: u32 = 0;
+ // SAFETY: The arguments are all valid per the type invariants.
+ to_result(unsafe {
+ bindings::drm_gem_handle_create(file.as_raw().cast(), self.as_raw(), &mut handle)
+ })?;
+ Ok(handle)
+ }
+
+ /// Looks up an object by its handle for a given `File`.
+ fn lookup_handle(
+ file: &drm::File<<<Self as IntoGEMObject>::Driver as drm::Driver>::File>,
+ handle: u32,
+ ) -> Result<ARef<Self>> {
+ // SAFETY: The arguments are all valid per the type invariants.
+ let ptr = unsafe { bindings::drm_gem_object_lookup(file.as_raw().cast(), handle) };
+ if ptr.is_null() {
+ return Err(ENOENT);
+ }
+
+ // SAFETY:
+ // - A `drm::Driver` can only have a single `File` implementation.
+ // - `file` uses the same `drm::Driver` as `Self`.
+ // - Therefore, we're guaranteed that `ptr` must be a gem object embedded within `Self`.
+ // - And we check if the pointer is null befoe calling as_ref(), ensuring that `ptr` is a
+ // valid pointer to an initialized `Self`.
+ let obj = unsafe { Self::as_ref(ptr) };
+
+ // SAFETY:
+ // - We take ownership of the reference of `drm_gem_object_lookup()`.
+ // - Our `NonNull` comes from an immutable reference, thus ensuring it is a valid pointer to
+ // `Self`.
+ Ok(unsafe { ARef::from_raw(obj.into()) })
+ }
+
+ /// Creates an mmap offset to map the object from userspace.
+ fn create_mmap_offset(&self) -> Result<u64> {
+ // SAFETY: The arguments are valid per the type invariant.
+ to_result(unsafe { bindings::drm_gem_create_mmap_offset(self.as_raw()) })?;
+
+ // SAFETY: The arguments are valid per the type invariant.
+ Ok(unsafe { bindings::drm_vma_node_offset_addr(&raw mut (*self.as_raw()).vma_node) })
+ }
+}
+
+impl<T: IntoGEMObject> BaseObject for T {}
+
+/// A base GEM object.
+///
+/// Invariants
+///
+/// - `self.obj` is a valid instance of a `struct drm_gem_object`.
+/// - `self.dev` is always a valid pointer to a `struct drm_device`.
+#[repr(C)]
+#[pin_data]
+pub struct Object<T: DriverObject + Send + Sync> {
+ obj: Opaque<bindings::drm_gem_object>,
+ dev: NonNull<drm::Device<T::Driver>>,
+ #[pin]
+ data: T,
+}
+
+impl<T: DriverObject> Object<T> {
+ /// The size of this object's structure.
+ pub const SIZE: usize = mem::size_of::<Self>();
+
+ const OBJECT_FUNCS: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs {
+ free: Some(Self::free_callback),
+ open: Some(open_callback::<T, Object<T>>),
+ close: Some(close_callback::<T, Object<T>>),
+ print_info: None,
+ export: None,
+ pin: None,
+ unpin: None,
+ get_sg_table: None,
+ vmap: None,
+ vunmap: None,
+ mmap: None,
+ status: None,
+ vm_ops: core::ptr::null_mut(),
+ evict: None,
+ rss: None,
+ };
+
+ /// Create a new GEM object.
+ pub fn new(dev: &drm::Device<T::Driver>, size: usize) -> Result<ARef<Self>> {
+ let obj: Pin<KBox<Self>> = KBox::pin_init(
+ try_pin_init!(Self {
+ obj: Opaque::new(bindings::drm_gem_object::default()),
+ data <- T::new(dev, size),
+ // INVARIANT: The drm subsystem guarantees that the `struct drm_device` will live
+ // as long as the GEM object lives.
+ dev: dev.into(),
+ }),
+ GFP_KERNEL,
+ )?;
+
+ // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above.
+ unsafe { (*obj.as_raw()).funcs = &Self::OBJECT_FUNCS };
+
+ // SAFETY: The arguments are all valid per the type invariants.
+ to_result(unsafe { bindings::drm_gem_object_init(dev.as_raw(), obj.obj.get(), size) })?;
+
+ // SAFETY: We never move out of `Self`.
+ let ptr = KBox::into_raw(unsafe { Pin::into_inner_unchecked(obj) });
+
+ // SAFETY: `ptr` comes from `KBox::into_raw` and hence can't be NULL.
+ let ptr = unsafe { NonNull::new_unchecked(ptr) };
+
+ // SAFETY: We take over the initial reference count from `drm_gem_object_init()`.
+ Ok(unsafe { ARef::from_raw(ptr) })
+ }
+
+ /// Returns the `Device` that owns this GEM object.
+ pub fn dev(&self) -> &drm::Device<T::Driver> {
+ // SAFETY: The DRM subsystem guarantees that the `struct drm_device` will live as long as
+ // the GEM object lives, hence the pointer must be valid.
+ unsafe { self.dev.as_ref() }
+ }
+
+ fn as_raw(&self) -> *mut bindings::drm_gem_object {
+ self.obj.get()
+ }
+
+ extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) {
+ let ptr: *mut Opaque<bindings::drm_gem_object> = obj.cast();
+
+ // SAFETY: All of our objects are of type `Object<T>`.
+ let this = unsafe { crate::container_of!(ptr, Self, obj) };
+
+ // SAFETY: The C code only ever calls this callback with a valid pointer to a `struct
+ // drm_gem_object`.
+ unsafe { bindings::drm_gem_object_release(obj) };
+
+ // SAFETY: All of our objects are allocated via `KBox`, and we're in the
+ // free callback which guarantees this object has zero remaining references,
+ // so we can drop it.
+ let _ = unsafe { KBox::from_raw(this) };
+ }
+}
+
+impl<T: DriverObject> super::private::Sealed for Object<T> {}
+
+impl<T: DriverObject> Deref for Object<T> {
+ type Target = T;
+
+ fn deref(&self) -> &Self::Target {
+ &self.data
+ }
+}
+
+impl<T: DriverObject> AllocImpl for Object<T> {
+ const ALLOC_OPS: AllocOps = AllocOps {
+ gem_create_object: None,
+ prime_handle_to_fd: None,
+ prime_fd_to_handle: None,
+ gem_prime_import: None,
+ gem_prime_import_sg_table: None,
+ dumb_create: None,
+ dumb_map_offset: None,
+ };
+}
+
+pub(super) const fn create_fops() -> bindings::file_operations {
+ // SAFETY: As by the type invariant, it is safe to initialize `bindings::file_operations`
+ // zeroed.
+ let mut fops: bindings::file_operations = unsafe { core::mem::zeroed() };
+
+ fops.owner = core::ptr::null_mut();
+ fops.open = Some(bindings::drm_open);
+ fops.release = Some(bindings::drm_release);
+ fops.unlocked_ioctl = Some(bindings::drm_ioctl);
+ #[cfg(CONFIG_COMPAT)]
+ {
+ fops.compat_ioctl = Some(bindings::drm_compat_ioctl);
+ }
+ fops.poll = Some(bindings::drm_poll);
+ fops.read = Some(bindings::drm_read);
+ fops.llseek = Some(bindings::noop_llseek);
+ fops.mmap = Some(bindings::drm_gem_mmap);
+ fops.fop_flags = bindings::FOP_UNSIGNED_OFFSET;
+
+ fops
+}
diff --git a/rust/kernel/drm/ioctl.rs b/rust/kernel/drm/ioctl.rs
new file mode 100644
index 000000000000..445639404fb7
--- /dev/null
+++ b/rust/kernel/drm/ioctl.rs
@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+//! DRM IOCTL definitions.
+//!
+//! C header: [`include/linux/drm/drm_ioctl.h`](srctree/include/linux/drm/drm_ioctl.h)
+
+use crate::ioctl;
+
+const BASE: u32 = uapi::DRM_IOCTL_BASE as u32;
+
+/// Construct a DRM ioctl number with no argument.
+#[allow(non_snake_case)]
+#[inline(always)]
+pub const fn IO(nr: u32) -> u32 {
+ ioctl::_IO(BASE, nr)
+}
+
+/// Construct a DRM ioctl number with a read-only argument.
+#[allow(non_snake_case)]
+#[inline(always)]
+pub const fn IOR<T>(nr: u32) -> u32 {
+ ioctl::_IOR::<T>(BASE, nr)
+}
+
+/// Construct a DRM ioctl number with a write-only argument.
+#[allow(non_snake_case)]
+#[inline(always)]
+pub const fn IOW<T>(nr: u32) -> u32 {
+ ioctl::_IOW::<T>(BASE, nr)
+}
+
+/// Construct a DRM ioctl number with a read-write argument.
+#[allow(non_snake_case)]
+#[inline(always)]
+pub const fn IOWR<T>(nr: u32) -> u32 {
+ ioctl::_IOWR::<T>(BASE, nr)
+}
+
+/// Descriptor type for DRM ioctls. Use the `declare_drm_ioctls!{}` macro to construct them.
+pub type DrmIoctlDescriptor = bindings::drm_ioctl_desc;
+
+/// This is for ioctl which are used for rendering, and require that the file descriptor is either
+/// for a render node, or if it’s a legacy/primary node, then it must be authenticated.
+pub const AUTH: u32 = bindings::drm_ioctl_flags_DRM_AUTH;
+
+/// This must be set for any ioctl which can change the modeset or display state. Userspace must
+/// call the ioctl through a primary node, while it is the active master.
+///
+/// Note that read-only modeset ioctl can also be called by unauthenticated clients, or when a
+/// master is not the currently active one.
+pub const MASTER: u32 = bindings::drm_ioctl_flags_DRM_MASTER;
+
+/// Anything that could potentially wreak a master file descriptor needs to have this flag set.
+///
+/// Current that’s only for the SETMASTER and DROPMASTER ioctl, which e.g. logind can call to
+/// force a non-behaving master (display compositor) into compliance.
+///
+/// This is equivalent to callers with the SYSADMIN capability.
+pub const ROOT_ONLY: u32 = bindings::drm_ioctl_flags_DRM_ROOT_ONLY;
+
+/// This is used for all ioctl needed for rendering only, for drivers which support render nodes.
+/// This should be all new render drivers, and hence it should be always set for any ioctl with
+/// `AUTH` set. Note though that read-only query ioctl might have this set, but have not set
+/// DRM_AUTH because they do not require authentication.
+pub const RENDER_ALLOW: u32 = bindings::drm_ioctl_flags_DRM_RENDER_ALLOW;
+
+/// Internal structures used by the `declare_drm_ioctls!{}` macro. Do not use directly.
+#[doc(hidden)]
+pub mod internal {
+ pub use bindings::drm_device;
+ pub use bindings::drm_file;
+ pub use bindings::drm_ioctl_desc;
+}
+
+/// Declare the DRM ioctls for a driver.
+///
+/// Each entry in the list should have the form:
+///
+/// `(ioctl_number, argument_type, flags, user_callback),`
+///
+/// `argument_type` is the type name within the `bindings` crate.
+/// `user_callback` should have the following prototype:
+///
+/// ```ignore
+/// fn foo(device: &kernel::drm::Device<Self>,
+/// data: &Opaque<uapi::argument_type>,
+/// file: &kernel::drm::File<Self::File>,
+/// ) -> Result<u32>
+/// ```
+/// where `Self` is the drm::drv::Driver implementation these ioctls are being declared within.
+///
+/// # Examples
+///
+/// ```ignore
+/// kernel::declare_drm_ioctls! {
+/// (FOO_GET_PARAM, drm_foo_get_param, ioctl::RENDER_ALLOW, my_get_param_handler),
+/// }
+/// ```
+///
+#[macro_export]
+macro_rules! declare_drm_ioctls {
+ ( $(($cmd:ident, $struct:ident, $flags:expr, $func:expr)),* $(,)? ) => {
+ const IOCTLS: &'static [$crate::drm::ioctl::DrmIoctlDescriptor] = {
+ use $crate::uapi::*;
+ const _:() = {
+ let i: u32 = $crate::uapi::DRM_COMMAND_BASE;
+ // Assert that all the IOCTLs are in the right order and there are no gaps,
+ // and that the size of the specified type is correct.
+ $(
+ let cmd: u32 = $crate::macros::concat_idents!(DRM_IOCTL_, $cmd);
+ ::core::assert!(i == $crate::ioctl::_IOC_NR(cmd));
+ ::core::assert!(core::mem::size_of::<$crate::uapi::$struct>() ==
+ $crate::ioctl::_IOC_SIZE(cmd));
+ let i: u32 = i + 1;
+ )*
+ };
+
+ let ioctls = &[$(
+ $crate::drm::ioctl::internal::drm_ioctl_desc {
+ cmd: $crate::macros::concat_idents!(DRM_IOCTL_, $cmd) as u32,
+ func: {
+ #[allow(non_snake_case)]
+ unsafe extern "C" fn $cmd(
+ raw_dev: *mut $crate::drm::ioctl::internal::drm_device,
+ raw_data: *mut ::core::ffi::c_void,
+ raw_file: *mut $crate::drm::ioctl::internal::drm_file,
+ ) -> core::ffi::c_int {
+ // SAFETY:
+ // - The DRM core ensures the device lives while callbacks are being
+ // called.
+ // - The DRM device must have been registered when we're called through
+ // an IOCTL.
+ //
+ // FIXME: Currently there is nothing enforcing that the types of the
+ // dev/file match the current driver these ioctls are being declared
+ // for, and it's not clear how to enforce this within the type system.
+ let dev = $crate::drm::device::Device::as_ref(raw_dev);
+ // SAFETY: The ioctl argument has size `_IOC_SIZE(cmd)`, which we
+ // asserted above matches the size of this type, and all bit patterns of
+ // UAPI structs must be valid.
+ let data = unsafe {
+ &*(raw_data as *const $crate::types::Opaque<$crate::uapi::$struct>)
+ };
+ // SAFETY: This is just the DRM file structure
+ let file = unsafe { $crate::drm::File::as_ref(raw_file) };
+
+ match $func(dev, data, file) {
+ Err(e) => e.to_errno(),
+ Ok(i) => i.try_into()
+ .unwrap_or($crate::error::code::ERANGE.to_errno()),
+ }
+ }
+ Some($cmd)
+ },
+ flags: $flags,
+ name: $crate::c_str!(::core::stringify!($cmd)).as_char_ptr(),
+ }
+ ),*];
+ ioctls
+ };
+ };
+}
diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs
new file mode 100644
index 000000000000..1b82b6945edf
--- /dev/null
+++ b/rust/kernel/drm/mod.rs
@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+//! DRM subsystem abstractions.
+
+pub mod device;
+pub mod driver;
+pub mod file;
+pub mod gem;
+pub mod ioctl;
+
+pub use self::device::Device;
+pub use self::driver::Driver;
+pub use self::driver::DriverInfo;
+pub use self::driver::Registration;
+pub use self::file::File;
+
+pub(crate) mod private {
+ pub trait Sealed {}
+}
diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs
index f6ecf09cb65f..3dee3139fcd4 100644
--- a/rust/kernel/error.rs
+++ b/rust/kernel/error.rs
@@ -64,6 +64,7 @@ pub mod code {
declare_err!(EPIPE, "Broken pipe.");
declare_err!(EDOM, "Math argument out of domain of func.");
declare_err!(ERANGE, "Math result not representable.");
+ declare_err!(EOVERFLOW, "Value too large for defined data type.");
declare_err!(ERESTARTSYS, "Restart the system call.");
declare_err!(ERESTARTNOINTR, "System call was interrupted by a signal and will be restarted.");
declare_err!(ERESTARTNOHAND, "Restart if no handler.");
@@ -107,7 +108,7 @@ impl Error {
} else {
// TODO: Make it a `WARN_ONCE` once available.
crate::pr_warn!(
- "attempted to create `Error` with out of range `errno`: {}",
+ "attempted to create `Error` with out of range `errno`: {}\n",
errno
);
code::EINVAL
@@ -248,8 +249,129 @@ impl From<core::convert::Infallible> for Error {
/// [`Error`] as its error type.
///
/// Note that even if a function does not return anything when it succeeds,
-/// it should still be modeled as returning a `Result` rather than
+/// it should still be modeled as returning a [`Result`] rather than
/// just an [`Error`].
+///
+/// Calling a function that returns [`Result`] forces the caller to handle
+/// the returned [`Result`].
+///
+/// This can be done "manually" by using [`match`]. Using [`match`] to decode
+/// the [`Result`] is similar to C where all the return value decoding and the
+/// error handling is done explicitly by writing handling code for each
+/// error to cover. Using [`match`] the error and success handling can be
+/// implemented in all detail as required. For example (inspired by
+/// [`samples/rust/rust_minimal.rs`]):
+///
+/// ```
+/// # #[allow(clippy::single_match)]
+/// fn example() -> Result {
+/// let mut numbers = KVec::new();
+///
+/// match numbers.push(72, GFP_KERNEL) {
+/// Err(e) => {
+/// pr_err!("Error pushing 72: {e:?}");
+/// return Err(e.into());
+/// }
+/// // Do nothing, continue.
+/// Ok(()) => (),
+/// }
+///
+/// match numbers.push(108, GFP_KERNEL) {
+/// Err(e) => {
+/// pr_err!("Error pushing 108: {e:?}");
+/// return Err(e.into());
+/// }
+/// // Do nothing, continue.
+/// Ok(()) => (),
+/// }
+///
+/// match numbers.push(200, GFP_KERNEL) {
+/// Err(e) => {
+/// pr_err!("Error pushing 200: {e:?}");
+/// return Err(e.into());
+/// }
+/// // Do nothing, continue.
+/// Ok(()) => (),
+/// }
+///
+/// Ok(())
+/// }
+/// # example()?;
+/// # Ok::<(), Error>(())
+/// ```
+///
+/// An alternative to be more concise is the [`if let`] syntax:
+///
+/// ```
+/// fn example() -> Result {
+/// let mut numbers = KVec::new();
+///
+/// if let Err(e) = numbers.push(72, GFP_KERNEL) {
+/// pr_err!("Error pushing 72: {e:?}");
+/// return Err(e.into());
+/// }
+///
+/// if let Err(e) = numbers.push(108, GFP_KERNEL) {
+/// pr_err!("Error pushing 108: {e:?}");
+/// return Err(e.into());
+/// }
+///
+/// if let Err(e) = numbers.push(200, GFP_KERNEL) {
+/// pr_err!("Error pushing 200: {e:?}");
+/// return Err(e.into());
+/// }
+///
+/// Ok(())
+/// }
+/// # example()?;
+/// # Ok::<(), Error>(())
+/// ```
+///
+/// Instead of these verbose [`match`]/[`if let`], the [`?`] operator can
+/// be used to handle the [`Result`]. Using the [`?`] operator is often
+/// the best choice to handle [`Result`] in a non-verbose way as done in
+/// [`samples/rust/rust_minimal.rs`]:
+///
+/// ```
+/// fn example() -> Result {
+/// let mut numbers = KVec::new();
+///
+/// numbers.push(72, GFP_KERNEL)?;
+/// numbers.push(108, GFP_KERNEL)?;
+/// numbers.push(200, GFP_KERNEL)?;
+///
+/// Ok(())
+/// }
+/// # example()?;
+/// # Ok::<(), Error>(())
+/// ```
+///
+/// Another possibility is to call [`unwrap()`](Result::unwrap) or
+/// [`expect()`](Result::expect). However, use of these functions is
+/// *heavily discouraged* in the kernel because they trigger a Rust
+/// [`panic!`] if an error happens, which may destabilize the system or
+/// entirely break it as a result -- just like the C [`BUG()`] macro.
+/// Please see the documentation for the C macro [`BUG()`] for guidance
+/// on when to use these functions.
+///
+/// Alternatively, depending on the use case, using [`unwrap_or()`],
+/// [`unwrap_or_else()`], [`unwrap_or_default()`] or [`unwrap_unchecked()`]
+/// might be an option, as well.
+///
+/// For even more details, please see the [Rust documentation].
+///
+/// [`match`]: https://doc.rust-lang.org/reference/expressions/match-expr.html
+/// [`samples/rust/rust_minimal.rs`]: srctree/samples/rust/rust_minimal.rs
+/// [`if let`]: https://doc.rust-lang.org/reference/expressions/if-expr.html#if-let-expressions
+/// [`?`]: https://doc.rust-lang.org/reference/expressions/operator-expr.html#the-question-mark-operator
+/// [`unwrap()`]: Result::unwrap
+/// [`expect()`]: Result::expect
+/// [`BUG()`]: https://docs.kernel.org/process/deprecated.html#bug-and-bug-on
+/// [`unwrap_or()`]: Result::unwrap_or
+/// [`unwrap_or_else()`]: Result::unwrap_or_else
+/// [`unwrap_or_default()`]: Result::unwrap_or_default
+/// [`unwrap_unchecked()`]: Result::unwrap_unchecked
+/// [Rust documentation]: https://doc.rust-lang.org/book/ch09-02-recoverable-errors-with-result.html
pub type Result<T = (), E = Error> = core::result::Result<T, E>;
/// Converts an integer as returned by a C kernel function to an error if it's negative, and
diff --git a/rust/kernel/faux.rs b/rust/kernel/faux.rs
index 5acc0c02d451..8a50fcd4c9bb 100644
--- a/rust/kernel/faux.rs
+++ b/rust/kernel/faux.rs
@@ -19,16 +19,25 @@ use core::ptr::{addr_of_mut, null, null_mut, NonNull};
/// `self.0` always holds a valid pointer to an initialized and registered [`struct faux_device`].
///
/// [`struct faux_device`]: srctree/include/linux/device/faux.h
-#[repr(transparent)]
pub struct Registration(NonNull<bindings::faux_device>);
impl Registration {
/// Create and register a new faux device with the given name.
- pub fn new(name: &CStr) -> Result<Self> {
+ #[inline]
+ pub fn new(name: &CStr, parent: Option<&device::Device>) -> Result<Self> {
// SAFETY:
// - `name` is copied by this function into its own storage
// - `faux_ops` is safe to leave NULL according to the C API
- let dev = unsafe { bindings::faux_device_create(name.as_char_ptr(), null_mut(), null()) };
+ // - `parent` can be either NULL or a pointer to a `struct device`, and `faux_device_create`
+ // will take a reference to `parent` using `device_add` - ensuring that it remains valid
+ // for the lifetime of the faux device.
+ let dev = unsafe {
+ bindings::faux_device_create(
+ name.as_char_ptr(),
+ parent.map_or(null_mut(), |p| p.as_raw()),
+ null(),
+ )
+ };
// The above function will return either a valid device, or NULL on failure
// INVARIANT: The device will remain registered until faux_device_destroy() is called, which
@@ -50,6 +59,7 @@ impl AsRef<device::Device> for Registration {
}
impl Drop for Registration {
+ #[inline]
fn drop(&mut self) {
// SAFETY: `self.0` is a valid registered faux_device via our type invariants.
unsafe { bindings::faux_device_destroy(self.as_raw()) }
diff --git a/rust/kernel/firmware.rs b/rust/kernel/firmware.rs
index c5162fdc95ff..2494c96e105f 100644
--- a/rust/kernel/firmware.rs
+++ b/rust/kernel/firmware.rs
@@ -4,7 +4,7 @@
//!
//! C header: [`include/linux/firmware.h`](srctree/include/linux/firmware.h)
-use crate::{bindings, device::Device, error::Error, error::Result, str::CStr};
+use crate::{bindings, device::Device, error::Error, error::Result, ffi, str::CStr};
use core::ptr::NonNull;
/// # Invariants
@@ -12,7 +12,11 @@ use core::ptr::NonNull;
/// One of the following: `bindings::request_firmware`, `bindings::firmware_request_nowarn`,
/// `bindings::firmware_request_platform`, `bindings::request_firmware_direct`.
struct FwFunc(
- unsafe extern "C" fn(*mut *const bindings::firmware, *const u8, *mut bindings::device) -> i32,
+ unsafe extern "C" fn(
+ *mut *const bindings::firmware,
+ *const ffi::c_char,
+ *mut bindings::device,
+ ) -> i32,
);
impl FwFunc {
@@ -115,3 +119,219 @@ unsafe impl Send for Firmware {}
// SAFETY: `Firmware` only holds a pointer to a C `struct firmware`, references to which are safe to
// be used from any thread.
unsafe impl Sync for Firmware {}
+
+/// Create firmware .modinfo entries.
+///
+/// This macro is the counterpart of the C macro `MODULE_FIRMWARE()`, but instead of taking a
+/// simple string literals, which is already covered by the `firmware` field of
+/// [`crate::prelude::module!`], it allows the caller to pass a builder type, based on the
+/// [`ModInfoBuilder`], which can create the firmware modinfo strings in a more flexible way.
+///
+/// Drivers should extend the [`ModInfoBuilder`] with their own driver specific builder type.
+///
+/// The `builder` argument must be a type which implements the following function.
+///
+/// `const fn create(module_name: &'static CStr) -> ModInfoBuilder`
+///
+/// `create` should pass the `module_name` to the [`ModInfoBuilder`] and, with the help of
+/// it construct the corresponding firmware modinfo.
+///
+/// Typically, such contracts would be enforced by a trait, however traits do not (yet) support
+/// const functions.
+///
+/// # Example
+///
+/// ```
+/// # mod module_firmware_test {
+/// # use kernel::firmware;
+/// # use kernel::prelude::*;
+/// #
+/// # struct MyModule;
+/// #
+/// # impl kernel::Module for MyModule {
+/// # fn init(_module: &'static ThisModule) -> Result<Self> {
+/// # Ok(Self)
+/// # }
+/// # }
+/// #
+/// #
+/// struct Builder<const N: usize>;
+///
+/// impl<const N: usize> Builder<N> {
+/// const DIR: &'static str = "vendor/chip/";
+/// const FILES: [&'static str; 3] = [ "foo", "bar", "baz" ];
+///
+/// const fn create(module_name: &'static kernel::str::CStr) -> firmware::ModInfoBuilder<N> {
+/// let mut builder = firmware::ModInfoBuilder::new(module_name);
+///
+/// let mut i = 0;
+/// while i < Self::FILES.len() {
+/// builder = builder.new_entry()
+/// .push(Self::DIR)
+/// .push(Self::FILES[i])
+/// .push(".bin");
+///
+/// i += 1;
+/// }
+///
+/// builder
+/// }
+/// }
+///
+/// module! {
+/// type: MyModule,
+/// name: "module_firmware_test",
+/// author: "Rust for Linux",
+/// description: "module_firmware! test module",
+/// license: "GPL",
+/// }
+///
+/// kernel::module_firmware!(Builder);
+/// # }
+/// ```
+#[macro_export]
+macro_rules! module_firmware {
+ // The argument is the builder type without the const generic, since it's deferred from within
+ // this macro. Hence, we can neither use `expr` nor `ty`.
+ ($($builder:tt)*) => {
+ const _: () = {
+ const __MODULE_FIRMWARE_PREFIX: &'static $crate::str::CStr = if cfg!(MODULE) {
+ $crate::c_str!("")
+ } else {
+ <LocalModule as $crate::ModuleMetadata>::NAME
+ };
+
+ #[link_section = ".modinfo"]
+ #[used]
+ static __MODULE_FIRMWARE: [u8; $($builder)*::create(__MODULE_FIRMWARE_PREFIX)
+ .build_length()] = $($builder)*::create(__MODULE_FIRMWARE_PREFIX).build();
+ };
+ };
+}
+
+/// Builder for firmware module info.
+///
+/// [`ModInfoBuilder`] is a helper component to flexibly compose firmware paths strings for the
+/// .modinfo section in const context.
+///
+/// Therefore the [`ModInfoBuilder`] provides the methods [`ModInfoBuilder::new_entry`] and
+/// [`ModInfoBuilder::push`], where the latter is used to push path components and the former to
+/// mark the beginning of a new path string.
+///
+/// [`ModInfoBuilder`] is meant to be used in combination with [`kernel::module_firmware!`].
+///
+/// The const generic `N` as well as the `module_name` parameter of [`ModInfoBuilder::new`] is an
+/// internal implementation detail and supplied through the above macro.
+pub struct ModInfoBuilder<const N: usize> {
+ buf: [u8; N],
+ n: usize,
+ module_name: &'static CStr,
+}
+
+impl<const N: usize> ModInfoBuilder<N> {
+ /// Create an empty builder instance.
+ pub const fn new(module_name: &'static CStr) -> Self {
+ Self {
+ buf: [0; N],
+ n: 0,
+ module_name,
+ }
+ }
+
+ const fn push_internal(mut self, bytes: &[u8]) -> Self {
+ let mut j = 0;
+
+ if N == 0 {
+ self.n += bytes.len();
+ return self;
+ }
+
+ while j < bytes.len() {
+ if self.n < N {
+ self.buf[self.n] = bytes[j];
+ }
+ self.n += 1;
+ j += 1;
+ }
+ self
+ }
+
+ /// Push an additional path component.
+ ///
+ /// Append path components to the [`ModInfoBuilder`] instance. Paths need to be separated
+ /// with [`ModInfoBuilder::new_entry`].
+ ///
+ /// # Example
+ ///
+ /// ```
+ /// use kernel::firmware::ModInfoBuilder;
+ ///
+ /// # const DIR: &str = "vendor/chip/";
+ /// # const fn no_run<const N: usize>(builder: ModInfoBuilder<N>) {
+ /// let builder = builder.new_entry()
+ /// .push(DIR)
+ /// .push("foo.bin")
+ /// .new_entry()
+ /// .push(DIR)
+ /// .push("bar.bin");
+ /// # }
+ /// ```
+ pub const fn push(self, s: &str) -> Self {
+ // Check whether there has been an initial call to `next_entry()`.
+ if N != 0 && self.n == 0 {
+ crate::build_error!("Must call next_entry() before push().");
+ }
+
+ self.push_internal(s.as_bytes())
+ }
+
+ const fn push_module_name(self) -> Self {
+ let mut this = self;
+ let module_name = this.module_name;
+
+ if !this.module_name.is_empty() {
+ this = this.push_internal(module_name.as_bytes_with_nul());
+
+ if N != 0 {
+ // Re-use the space taken by the NULL terminator and swap it with the '.' separator.
+ this.buf[this.n - 1] = b'.';
+ }
+ }
+
+ this
+ }
+
+ /// Prepare the [`ModInfoBuilder`] for the next entry.
+ ///
+ /// This method acts as a separator between module firmware path entries.
+ ///
+ /// Must be called before constructing a new entry with subsequent calls to
+ /// [`ModInfoBuilder::push`].
+ ///
+ /// See [`ModInfoBuilder::push`] for an example.
+ pub const fn new_entry(self) -> Self {
+ self.push_internal(b"\0")
+ .push_module_name()
+ .push_internal(b"firmware=")
+ }
+
+ /// Build the byte array.
+ pub const fn build(self) -> [u8; N] {
+ // Add the final NULL terminator.
+ let this = self.push_internal(b"\0");
+
+ if this.n == N {
+ this.buf
+ } else {
+ crate::build_error!("Length mismatch.");
+ }
+ }
+}
+
+impl ModInfoBuilder<0> {
+ /// Return the length of the byte array to build.
+ pub const fn build_length(self) -> usize {
+ // Compensate for the NULL terminator added by `build`.
+ self.n + 1
+ }
+}
diff --git a/rust/kernel/fs/file.rs b/rust/kernel/fs/file.rs
index e03dbe14d62a..72d84fb0e266 100644
--- a/rust/kernel/fs/file.rs
+++ b/rust/kernel/fs/file.rs
@@ -219,12 +219,13 @@ unsafe impl AlwaysRefCounted for File {
/// must be on the same thread as this file.
///
/// [`assume_no_fdget_pos`]: LocalFile::assume_no_fdget_pos
+#[repr(transparent)]
pub struct LocalFile {
inner: Opaque<bindings::file>,
}
// SAFETY: The type invariants guarantee that `LocalFile` is always ref-counted. This implementation
-// makes `ARef<File>` own a normal refcount.
+// makes `ARef<LocalFile>` own a normal refcount.
unsafe impl AlwaysRefCounted for LocalFile {
#[inline]
fn inc_ref(&self) {
@@ -235,7 +236,8 @@ unsafe impl AlwaysRefCounted for LocalFile {
#[inline]
unsafe fn dec_ref(obj: ptr::NonNull<LocalFile>) {
// SAFETY: To call this method, the caller passes us ownership of a normal refcount, so we
- // may drop it. The cast is okay since `File` has the same representation as `struct file`.
+ // may drop it. The cast is okay since `LocalFile` has the same representation as
+ // `struct file`.
unsafe { bindings::fput(obj.cast().as_ptr()) }
}
}
@@ -267,13 +269,13 @@ impl LocalFile {
/// # Safety
///
/// * The caller must ensure that `ptr` points at a valid file and that the file's refcount is
- /// positive for the duration of 'a.
+ /// positive for the duration of `'a`.
/// * The caller must ensure that if there is an active call to `fdget_pos` that did not take
/// the `f_pos_lock` mutex, then that call is on the current thread.
#[inline]
pub unsafe fn from_raw_file<'a>(ptr: *const bindings::file) -> &'a LocalFile {
// SAFETY: The caller guarantees that the pointer is not dangling and stays valid for the
- // duration of 'a. The cast is okay because `File` is `repr(transparent)`.
+ // duration of `'a`. The cast is okay because `LocalFile` is `repr(transparent)`.
//
// INVARIANT: The caller guarantees that there are no problematic `fdget_pos` calls.
unsafe { &*ptr.cast() }
@@ -341,13 +343,13 @@ impl File {
/// # Safety
///
/// * The caller must ensure that `ptr` points at a valid file and that the file's refcount is
- /// positive for the duration of 'a.
+ /// positive for the duration of `'a`.
/// * The caller must ensure that if there are active `fdget_pos` calls on this file, then they
/// took the `f_pos_lock` mutex.
#[inline]
pub unsafe fn from_raw_file<'a>(ptr: *const bindings::file) -> &'a File {
// SAFETY: The caller guarantees that the pointer is not dangling and stays valid for the
- // duration of 'a. The cast is okay because `File` is `repr(transparent)`.
+ // duration of `'a`. The cast is okay because `File` is `repr(transparent)`.
//
// INVARIANT: The caller guarantees that there are no problematic `fdget_pos` calls.
unsafe { &*ptr.cast() }
@@ -392,6 +394,7 @@ pub struct FileDescriptorReservation {
impl FileDescriptorReservation {
/// Creates a new file descriptor reservation.
+ #[inline]
pub fn get_unused_fd_flags(flags: u32) -> Result<Self> {
// SAFETY: FFI call, there are no safety requirements on `flags`.
let fd: i32 = unsafe { bindings::get_unused_fd_flags(flags) };
@@ -405,6 +408,7 @@ impl FileDescriptorReservation {
}
/// Returns the file descriptor number that was reserved.
+ #[inline]
pub fn reserved_fd(&self) -> u32 {
self.fd
}
@@ -413,6 +417,7 @@ impl FileDescriptorReservation {
///
/// The previously reserved file descriptor is bound to `file`. This method consumes the
/// [`FileDescriptorReservation`], so it will not be usable after this call.
+ #[inline]
pub fn fd_install(self, file: ARef<File>) {
// SAFETY: `self.fd` was previously returned by `get_unused_fd_flags`. We have not yet used
// the fd, so it is still valid, and `current` still refers to the same task, as this type
@@ -433,6 +438,7 @@ impl FileDescriptorReservation {
}
impl Drop for FileDescriptorReservation {
+ #[inline]
fn drop(&mut self) {
// SAFETY: By the type invariants of this type, `self.fd` was previously returned by
// `get_unused_fd_flags`. We have not yet used the fd, so it is still valid, and `current`
diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs
index 7fd1ea8265a5..8d228c237954 100644
--- a/rust/kernel/init.rs
+++ b/rust/kernel/init.rs
@@ -1,131 +1,77 @@
-// SPDX-License-Identifier: Apache-2.0 OR MIT
+// SPDX-License-Identifier: GPL-2.0
-//! API to safely and fallibly initialize pinned `struct`s using in-place constructors.
-//!
-//! It also allows in-place initialization of big `struct`s that would otherwise produce a stack
-//! overflow.
+//! Extensions to the [`pin-init`] crate.
//!
//! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential
//! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move.
//!
-//! # Overview
+//! The [`pin-init`] crate is the way such structs are initialized on the Rust side. Please refer
+//! to its documentation to better understand how to use it. Additionally, there are many examples
+//! throughout the kernel, such as the types from the [`sync`] module. And the ones presented
+//! below.
+//!
+//! [`sync`]: crate::sync
+//! [pinning]: https://doc.rust-lang.org/std/pin/index.html
+//! [`pin-init`]: https://rust.docs.kernel.org/pin_init/
//!
-//! To initialize a `struct` with an in-place constructor you will need two things:
-//! - an in-place constructor,
-//! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`],
-//! [`UniqueArc<T>`], [`KBox<T>`] or any other smart pointer that implements [`InPlaceInit`]).
+//! # [`Opaque<T>`]
//!
-//! To get an in-place constructor there are generally three options:
-//! - directly creating an in-place constructor using the [`pin_init!`] macro,
-//! - a custom function/macro returning an in-place constructor provided by someone else,
-//! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer.
+//! For the special case where initializing a field is a single FFI-function call that cannot fail,
+//! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single
+//! [`Opaque<T>`] field by just delegating to the supplied closure. You can use these in
+//! combination with [`pin_init!`].
//!
-//! Aside from pinned initialization, this API also supports in-place construction without pinning,
-//! the macros/types/functions are generally named like the pinned variants without the `pin`
-//! prefix.
+//! [`Opaque<T>`]: crate::types::Opaque
+//! [`Opaque::ffi_init`]: crate::types::Opaque::ffi_init
+//! [`pin_init!`]: pin_init::pin_init
//!
//! # Examples
//!
-//! ## Using the [`pin_init!`] macro
+//! ## General Examples
//!
-//! If you want to use [`PinInit`], then you will have to annotate your `struct` with
-//! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for
-//! [structurally pinned fields]. After doing this, you can then create an in-place constructor via
-//! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is
-//! that you need to write `<-` instead of `:` for fields that you want to initialize in-place.
+//! ```rust,ignore
+//! # #![allow(clippy::disallowed_names)]
+//! use kernel::types::Opaque;
+//! use pin_init::pin_init_from_closure;
//!
-//! ```rust
-//! # #![expect(clippy::disallowed_names)]
-//! use kernel::sync::{new_mutex, Mutex};
-//! # use core::pin::Pin;
-//! #[pin_data]
-//! struct Foo {
-//! #[pin]
-//! a: Mutex<usize>,
-//! b: u32,
+//! // assume we have some `raw_foo` type in C:
+//! #[repr(C)]
+//! struct RawFoo([u8; 16]);
+//! extern {
+//! fn init_foo(_: *mut RawFoo);
//! }
//!
-//! let foo = pin_init!(Foo {
-//! a <- new_mutex!(42, "Foo::a"),
-//! b: 24,
-//! });
-//! ```
-//!
-//! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like
-//! (or just the stack) to actually initialize a `Foo`:
-//!
-//! ```rust
-//! # #![expect(clippy::disallowed_names)]
-//! # use kernel::sync::{new_mutex, Mutex};
-//! # use core::pin::Pin;
-//! # #[pin_data]
-//! # struct Foo {
-//! # #[pin]
-//! # a: Mutex<usize>,
-//! # b: u32,
-//! # }
-//! # let foo = pin_init!(Foo {
-//! # a <- new_mutex!(42, "Foo::a"),
-//! # b: 24,
-//! # });
-//! let foo: Result<Pin<KBox<Foo>>> = KBox::pin_init(foo, GFP_KERNEL);
-//! ```
-//!
-//! For more information see the [`pin_init!`] macro.
-//!
-//! ## Using a custom function/macro that returns an initializer
-//!
-//! Many types from the kernel supply a function/macro that returns an initializer, because the
-//! above method only works for types where you can access the fields.
-//!
-//! ```rust
-//! # use kernel::sync::{new_mutex, Arc, Mutex};
-//! let mtx: Result<Arc<Mutex<usize>>> =
-//! Arc::pin_init(new_mutex!(42, "example::mtx"), GFP_KERNEL);
-//! ```
-//!
-//! To declare an init macro/function you just return an [`impl PinInit<T, E>`]:
-//!
-//! ```rust
-//! # use kernel::{sync::Mutex, new_mutex, init::PinInit, try_pin_init};
//! #[pin_data]
-//! struct DriverData {
+//! struct Foo {
//! #[pin]
-//! status: Mutex<i32>,
-//! buffer: KBox<[u8; 1_000_000]>,
+//! raw: Opaque<RawFoo>,
//! }
//!
-//! impl DriverData {
-//! fn new() -> impl PinInit<Self, Error> {
-//! try_pin_init!(Self {
-//! status <- new_mutex!(0, "DriverData::status"),
-//! buffer: KBox::init(kernel::init::zeroed(), GFP_KERNEL)?,
-//! })
+//! impl Foo {
+//! fn setup(self: Pin<&mut Self>) {
+//! pr_info!("Setting up foo\n");
//! }
//! }
-//! ```
//!
-//! ## Manual creation of an initializer
-//!
-//! Often when working with primitives the previous approaches are not sufficient. That is where
-//! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a
-//! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure
-//! actually does the initialization in the correct way. Here are the things to look out for
-//! (we are calling the parameter to the closure `slot`):
-//! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so
-//! `slot` now contains a valid bit pattern for the type `T`,
-//! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so
-//! you need to take care to clean up anything if your initialization fails mid-way,
-//! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of
-//! `slot` gets called.
+//! let foo = pin_init!(Foo {
+//! raw <- unsafe {
+//! Opaque::ffi_init(|s| {
+//! // note that this cannot fail.
+//! init_foo(s);
+//! })
+//! },
+//! }).pin_chain(|foo| {
+//! foo.setup();
+//! Ok(())
+//! });
+//! ```
//!
-//! ```rust
-//! # #![expect(unreachable_pub, clippy::disallowed_names)]
-//! use kernel::{init, types::Opaque};
+//! ```rust,ignore
+//! # #![allow(unreachable_pub, clippy::disallowed_names)]
+//! use kernel::{prelude::*, types::Opaque};
//! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin};
//! # mod bindings {
-//! # #![expect(non_camel_case_types)]
-//! # #![expect(clippy::missing_safety_doc)]
+//! # #![allow(non_camel_case_types)]
//! # pub struct foo;
//! # pub unsafe fn init_foo(_ptr: *mut foo) {}
//! # pub unsafe fn destroy_foo(_ptr: *mut foo) {}
@@ -133,7 +79,7 @@
//! # }
//! # // `Error::from_errno` is `pub(crate)` in the `kernel` crate, thus provide a workaround.
//! # trait FromErrno {
-//! # fn from_errno(errno: kernel::ffi::c_int) -> Error {
+//! # fn from_errno(errno: core::ffi::c_int) -> Error {
//! # // Dummy error that can be constructed outside the `kernel` crate.
//! # Error::from(core::fmt::Error)
//! # }
@@ -157,7 +103,7 @@
//! // enabled `foo`,
//! // - when it returns `Err(e)`, then it has cleaned up before
//! unsafe {
-//! init::pin_init_from_closure(move |slot: *mut Self| {
+//! pin_init::pin_init_from_closure(move |slot: *mut Self| {
//! // `slot` contains uninit memory, avoid creating a reference.
//! let foo = addr_of_mut!((*slot).foo);
//!
@@ -187,401 +133,114 @@
//! }
//! }
//! ```
-//!
-//! For the special case where initializing a field is a single FFI-function call that cannot fail,
-//! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single
-//! [`Opaque`] field by just delegating to the supplied closure. You can use these in combination
-//! with [`pin_init!`].
-//!
-//! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside
-//! the `kernel` crate. The [`sync`] module is a good starting point.
-//!
-//! [`sync`]: kernel::sync
-//! [pinning]: https://doc.rust-lang.org/std/pin/index.html
-//! [structurally pinned fields]:
-//! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field
-//! [stack]: crate::stack_pin_init
-//! [`Arc<T>`]: crate::sync::Arc
-//! [`impl PinInit<Foo>`]: PinInit
-//! [`impl PinInit<T, E>`]: PinInit
-//! [`impl Init<T, E>`]: Init
-//! [`Opaque`]: kernel::types::Opaque
-//! [`Opaque::ffi_init`]: kernel::types::Opaque::ffi_init
-//! [`pin_data`]: ::macros::pin_data
-//! [`pin_init!`]: crate::pin_init!
use crate::{
- alloc::{AllocError, Flags, KBox},
+ alloc::{AllocError, Flags},
error::{self, Error},
- sync::Arc,
- sync::UniqueArc,
- types::{Opaque, ScopeGuard},
-};
-use core::{
- cell::UnsafeCell,
- convert::Infallible,
- marker::PhantomData,
- mem::MaybeUninit,
- num::*,
- pin::Pin,
- ptr::{self, NonNull},
};
+use pin_init::{init_from_closure, pin_init_from_closure, Init, PinInit};
-#[doc(hidden)]
-pub mod __internal;
-#[doc(hidden)]
-pub mod macros;
+/// Smart pointer that can initialize memory in-place.
+pub trait InPlaceInit<T>: Sized {
+ /// Pinned version of `Self`.
+ ///
+ /// If a type already implicitly pins its pointee, `Pin<Self>` is unnecessary. In this case use
+ /// `Self`, otherwise just use `Pin<Self>`.
+ type PinnedSelf;
-/// Initialize and pin a type directly on the stack.
-///
-/// # Examples
-///
-/// ```rust
-/// # #![expect(clippy::disallowed_names)]
-/// # use kernel::{init, macros::pin_data, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex};
-/// # use core::pin::Pin;
-/// #[pin_data]
-/// struct Foo {
-/// #[pin]
-/// a: Mutex<usize>,
-/// b: Bar,
-/// }
-///
-/// #[pin_data]
-/// struct Bar {
-/// x: u32,
-/// }
-///
-/// stack_pin_init!(let foo = pin_init!(Foo {
-/// a <- new_mutex!(42),
-/// b: Bar {
-/// x: 64,
-/// },
-/// }));
-/// let foo: Pin<&mut Foo> = foo;
-/// pr_info!("a: {}", &*foo.a.lock());
-/// ```
-///
-/// # Syntax
-///
-/// A normal `let` binding with optional type annotation. The expression is expected to implement
-/// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error
-/// type, then use [`stack_try_pin_init!`].
-///
-/// [`stack_try_pin_init!`]: crate::stack_try_pin_init!
-#[macro_export]
-macro_rules! stack_pin_init {
- (let $var:ident $(: $t:ty)? = $val:expr) => {
- let val = $val;
- let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
- let mut $var = match $crate::init::__internal::StackInit::init($var, val) {
- Ok(res) => res,
- Err(x) => {
- let x: ::core::convert::Infallible = x;
- match x {}
- }
+ /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
+ /// type.
+ ///
+ /// If `T: !Unpin` it will not be able to move afterwards.
+ fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
+ where
+ E: From<AllocError>;
+
+ /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
+ /// type.
+ ///
+ /// If `T: !Unpin` it will not be able to move afterwards.
+ fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Self::PinnedSelf>
+ where
+ Error: From<E>,
+ {
+ // SAFETY: We delegate to `init` and only change the error type.
+ let init = unsafe {
+ pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
};
- };
-}
+ Self::try_pin_init(init, flags)
+ }
-/// Initialize and pin a type directly on the stack.
-///
-/// # Examples
-///
-/// ```rust,ignore
-/// # #![expect(clippy::disallowed_names)]
-/// # use kernel::{
-/// # init,
-/// # pin_init,
-/// # stack_try_pin_init,
-/// # init::*,
-/// # sync::Mutex,
-/// # new_mutex,
-/// # alloc::AllocError,
-/// # };
-/// # use macros::pin_data;
-/// # use core::pin::Pin;
-/// #[pin_data]
-/// struct Foo {
-/// #[pin]
-/// a: Mutex<usize>,
-/// b: KBox<Bar>,
-/// }
-///
-/// struct Bar {
-/// x: u32,
-/// }
-///
-/// stack_try_pin_init!(let foo: Result<Pin<&mut Foo>, AllocError> = pin_init!(Foo {
-/// a <- new_mutex!(42),
-/// b: KBox::new(Bar {
-/// x: 64,
-/// }, GFP_KERNEL)?,
-/// }));
-/// let foo = foo.unwrap();
-/// pr_info!("a: {}", &*foo.a.lock());
-/// ```
-///
-/// ```rust,ignore
-/// # #![expect(clippy::disallowed_names)]
-/// # use kernel::{
-/// # init,
-/// # pin_init,
-/// # stack_try_pin_init,
-/// # init::*,
-/// # sync::Mutex,
-/// # new_mutex,
-/// # alloc::AllocError,
-/// # };
-/// # use macros::pin_data;
-/// # use core::pin::Pin;
-/// #[pin_data]
-/// struct Foo {
-/// #[pin]
-/// a: Mutex<usize>,
-/// b: KBox<Bar>,
-/// }
-///
-/// struct Bar {
-/// x: u32,
-/// }
-///
-/// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo {
-/// a <- new_mutex!(42),
-/// b: KBox::new(Bar {
-/// x: 64,
-/// }, GFP_KERNEL)?,
-/// }));
-/// pr_info!("a: {}", &*foo.a.lock());
-/// # Ok::<_, AllocError>(())
-/// ```
-///
-/// # Syntax
-///
-/// A normal `let` binding with optional type annotation. The expression is expected to implement
-/// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the
-/// `=` will propagate this error.
-#[macro_export]
-macro_rules! stack_try_pin_init {
- (let $var:ident $(: $t:ty)? = $val:expr) => {
- let val = $val;
- let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
- let mut $var = $crate::init::__internal::StackInit::init($var, val);
- };
- (let $var:ident $(: $t:ty)? =? $val:expr) => {
- let val = $val;
- let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
- let mut $var = $crate::init::__internal::StackInit::init($var, val)?;
- };
+ /// Use the given initializer to in-place initialize a `T`.
+ fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
+ where
+ E: From<AllocError>;
+
+ /// Use the given initializer to in-place initialize a `T`.
+ fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self>
+ where
+ Error: From<E>,
+ {
+ // SAFETY: We delegate to `init` and only change the error type.
+ let init = unsafe {
+ init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
+ };
+ Self::try_init(init, flags)
+ }
}
-/// Construct an in-place, pinned initializer for `struct`s.
-///
-/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
-/// [`try_pin_init!`].
-///
-/// The syntax is almost identical to that of a normal `struct` initializer:
-///
-/// ```rust
-/// # use kernel::{init, pin_init, macros::pin_data, init::*};
-/// # use core::pin::Pin;
-/// #[pin_data]
-/// struct Foo {
-/// a: usize,
-/// b: Bar,
-/// }
-///
-/// #[pin_data]
-/// struct Bar {
-/// x: u32,
-/// }
-///
-/// # fn demo() -> impl PinInit<Foo> {
-/// let a = 42;
-///
-/// let initializer = pin_init!(Foo {
-/// a,
-/// b: Bar {
-/// x: 64,
-/// },
-/// });
-/// # initializer }
-/// # KBox::pin_init(demo(), GFP_KERNEL).unwrap();
-/// ```
-///
-/// Arbitrary Rust expressions can be used to set the value of a variable.
-///
-/// The fields are initialized in the order that they appear in the initializer. So it is possible
-/// to read already initialized fields using raw pointers.
-///
-/// IMPORTANT: You are not allowed to create references to fields of the struct inside of the
-/// initializer.
-///
-/// # Init-functions
-///
-/// When working with this API it is often desired to let others construct your types without
-/// giving access to all fields. This is where you would normally write a plain function `new`
-/// that would return a new instance of your type. With this API that is also possible.
-/// However, there are a few extra things to keep in mind.
-///
-/// To create an initializer function, simply declare it like this:
-///
-/// ```rust
-/// # use kernel::{init, pin_init, init::*};
-/// # use core::pin::Pin;
-/// # #[pin_data]
-/// # struct Foo {
-/// # a: usize,
-/// # b: Bar,
-/// # }
-/// # #[pin_data]
-/// # struct Bar {
-/// # x: u32,
-/// # }
-/// impl Foo {
-/// fn new() -> impl PinInit<Self> {
-/// pin_init!(Self {
-/// a: 42,
-/// b: Bar {
-/// x: 64,
-/// },
-/// })
-/// }
-/// }
-/// ```
+/// Construct an in-place fallible initializer for `struct`s.
///
-/// Users of `Foo` can now create it like this:
+/// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use
+/// [`init!`].
///
-/// ```rust
-/// # #![expect(clippy::disallowed_names)]
-/// # use kernel::{init, pin_init, macros::pin_data, init::*};
-/// # use core::pin::Pin;
-/// # #[pin_data]
-/// # struct Foo {
-/// # a: usize,
-/// # b: Bar,
-/// # }
-/// # #[pin_data]
-/// # struct Bar {
-/// # x: u32,
-/// # }
-/// # impl Foo {
-/// # fn new() -> impl PinInit<Self> {
-/// # pin_init!(Self {
-/// # a: 42,
-/// # b: Bar {
-/// # x: 64,
-/// # },
-/// # })
-/// # }
-/// # }
-/// let foo = KBox::pin_init(Foo::new(), GFP_KERNEL);
-/// ```
+/// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error,
+/// append `? $type` after the `struct` initializer.
+/// The safety caveats from [`try_pin_init!`] also apply:
+/// - `unsafe` code must guarantee either full initialization or return an error and allow
+/// deallocation of the memory.
+/// - the fields are initialized in the order given in the initializer.
+/// - no references to fields are allowed to be created inside of the initializer.
///
-/// They can also easily embed it into their own `struct`s:
+/// # Examples
///
/// ```rust
-/// # use kernel::{init, pin_init, macros::pin_data, init::*};
-/// # use core::pin::Pin;
-/// # #[pin_data]
-/// # struct Foo {
-/// # a: usize,
-/// # b: Bar,
-/// # }
-/// # #[pin_data]
-/// # struct Bar {
-/// # x: u32,
-/// # }
-/// # impl Foo {
-/// # fn new() -> impl PinInit<Self> {
-/// # pin_init!(Self {
-/// # a: 42,
-/// # b: Bar {
-/// # x: 64,
-/// # },
-/// # })
-/// # }
-/// # }
-/// #[pin_data]
-/// struct FooContainer {
-/// #[pin]
-/// foo1: Foo,
-/// #[pin]
-/// foo2: Foo,
-/// other: u32,
+/// use kernel::error::Error;
+/// use pin_init::zeroed;
+/// struct BigBuf {
+/// big: KBox<[u8; 1024 * 1024 * 1024]>,
+/// small: [u8; 1024 * 1024],
/// }
///
-/// impl FooContainer {
-/// fn new(other: u32) -> impl PinInit<Self> {
-/// pin_init!(Self {
-/// foo1 <- Foo::new(),
-/// foo2 <- Foo::new(),
-/// other,
-/// })
+/// impl BigBuf {
+/// fn new() -> impl Init<Self, Error> {
+/// try_init!(Self {
+/// big: KBox::init(zeroed(), GFP_KERNEL)?,
+/// small: [0; 1024 * 1024],
+/// }? Error)
/// }
/// }
/// ```
///
-/// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`.
-/// This signifies that the given field is initialized in-place. As with `struct` initializers, just
-/// writing the field (in this case `other`) without `:` or `<-` means `other: other,`.
-///
-/// # Syntax
-///
-/// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with
-/// the following modifications is expected:
-/// - Fields that you want to initialize in-place have to use `<-` instead of `:`.
-/// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`]
-/// pointer named `this` inside of the initializer.
-/// - Using struct update syntax one can place `..Zeroable::zeroed()` at the very end of the
-/// struct, this initializes every field with 0 and then runs all initializers specified in the
-/// body. This can only be done if [`Zeroable`] is implemented for the struct.
-///
-/// For instance:
-///
-/// ```rust
-/// # use kernel::{macros::{Zeroable, pin_data}, pin_init};
-/// # use core::{ptr::addr_of_mut, marker::PhantomPinned};
-/// #[pin_data]
-/// #[derive(Zeroable)]
-/// struct Buf {
-/// // `ptr` points into `buf`.
-/// ptr: *mut u8,
-/// buf: [u8; 64],
-/// #[pin]
-/// pin: PhantomPinned,
-/// }
-/// pin_init!(&this in Buf {
-/// buf: [0; 64],
-/// // SAFETY: TODO.
-/// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() },
-/// pin: PhantomPinned,
-/// });
-/// pin_init!(Buf {
-/// buf: [1; 64],
-/// ..Zeroable::zeroed()
-/// });
-/// ```
-///
-/// [`try_pin_init!`]: kernel::try_pin_init
-/// [`NonNull<Self>`]: core::ptr::NonNull
-// For a detailed example of how this macro works, see the module documentation of the hidden
-// module `__internal` inside of `init/__internal.rs`.
+/// [`Infallible`]: core::convert::Infallible
+/// [`init!`]: pin_init::init
+/// [`try_pin_init!`]: crate::try_pin_init!
+/// [`Error`]: crate::error::Error
#[macro_export]
-macro_rules! pin_init {
+macro_rules! try_init {
($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
$($fields:tt)*
}) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t $(::<$($generics),*>)?),
- @fields($($fields)*),
- @error(::core::convert::Infallible),
- @data(PinData, use_data),
- @has_data(HasPinData, __pin_data),
- @construct_closure(pin_init_from_closure),
- @munch_fields($($fields)*),
- )
+ ::pin_init::try_init!($(&$this in)? $t $(::<$($generics),* $(,)?>)? {
+ $($fields)*
+ }? $crate::error::Error)
+ };
+ ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
+ $($fields:tt)*
+ }? $err:ty) => {
+ ::pin_init::try_init!($(&$this in)? $t $(::<$($generics),* $(,)?>)? {
+ $($fields)*
+ }? $err)
};
}
@@ -603,7 +262,9 @@ macro_rules! pin_init {
/// # Examples
///
/// ```rust
-/// use kernel::{init::{self, PinInit}, error::Error};
+/// # #![feature(new_uninit)]
+/// use kernel::error::Error;
+/// use pin_init::zeroed;
/// #[pin_data]
/// struct BigBuf {
/// big: KBox<[u8; 1024 * 1024 * 1024]>,
@@ -614,846 +275,31 @@ macro_rules! pin_init {
/// impl BigBuf {
/// fn new() -> impl PinInit<Self, Error> {
/// try_pin_init!(Self {
-/// big: KBox::init(init::zeroed(), GFP_KERNEL)?,
+/// big: KBox::init(zeroed(), GFP_KERNEL)?,
/// small: [0; 1024 * 1024],
/// ptr: core::ptr::null_mut(),
/// }? Error)
/// }
/// }
/// ```
-// For a detailed example of how this macro works, see the module documentation of the hidden
-// module `__internal` inside of `init/__internal.rs`.
-#[macro_export]
-macro_rules! try_pin_init {
- ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- $($fields:tt)*
- }) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t $(::<$($generics),*>)? ),
- @fields($($fields)*),
- @error($crate::error::Error),
- @data(PinData, use_data),
- @has_data(HasPinData, __pin_data),
- @construct_closure(pin_init_from_closure),
- @munch_fields($($fields)*),
- )
- };
- ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- $($fields:tt)*
- }? $err:ty) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t $(::<$($generics),*>)? ),
- @fields($($fields)*),
- @error($err),
- @data(PinData, use_data),
- @has_data(HasPinData, __pin_data),
- @construct_closure(pin_init_from_closure),
- @munch_fields($($fields)*),
- )
- };
-}
-
-/// Construct an in-place initializer for `struct`s.
-///
-/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
-/// [`try_init!`].
-///
-/// The syntax is identical to [`pin_init!`] and its safety caveats also apply:
-/// - `unsafe` code must guarantee either full initialization or return an error and allow
-/// deallocation of the memory.
-/// - the fields are initialized in the order given in the initializer.
-/// - no references to fields are allowed to be created inside of the initializer.
-///
-/// This initializer is for initializing data in-place that might later be moved. If you want to
-/// pin-initialize, use [`pin_init!`].
-///
-/// [`try_init!`]: crate::try_init!
-// For a detailed example of how this macro works, see the module documentation of the hidden
-// module `__internal` inside of `init/__internal.rs`.
-#[macro_export]
-macro_rules! init {
- ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- $($fields:tt)*
- }) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t $(::<$($generics),*>)?),
- @fields($($fields)*),
- @error(::core::convert::Infallible),
- @data(InitData, /*no use_data*/),
- @has_data(HasInitData, __init_data),
- @construct_closure(init_from_closure),
- @munch_fields($($fields)*),
- )
- }
-}
-
-/// Construct an in-place fallible initializer for `struct`s.
-///
-/// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use
-/// [`init!`].
-///
-/// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error,
-/// append `? $type` after the `struct` initializer.
-/// The safety caveats from [`try_pin_init!`] also apply:
-/// - `unsafe` code must guarantee either full initialization or return an error and allow
-/// deallocation of the memory.
-/// - the fields are initialized in the order given in the initializer.
-/// - no references to fields are allowed to be created inside of the initializer.
-///
-/// # Examples
-///
-/// ```rust
-/// use kernel::{alloc::KBox, init::{PinInit, zeroed}, error::Error};
-/// struct BigBuf {
-/// big: KBox<[u8; 1024 * 1024 * 1024]>,
-/// small: [u8; 1024 * 1024],
-/// }
///
-/// impl BigBuf {
-/// fn new() -> impl Init<Self, Error> {
-/// try_init!(Self {
-/// big: KBox::init(zeroed(), GFP_KERNEL)?,
-/// small: [0; 1024 * 1024],
-/// }? Error)
-/// }
-/// }
-/// ```
-// For a detailed example of how this macro works, see the module documentation of the hidden
-// module `__internal` inside of `init/__internal.rs`.
+/// [`Infallible`]: core::convert::Infallible
+/// [`pin_init!`]: pin_init::pin_init
+/// [`Error`]: crate::error::Error
#[macro_export]
-macro_rules! try_init {
+macro_rules! try_pin_init {
($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
$($fields:tt)*
}) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t $(::<$($generics),*>)?),
- @fields($($fields)*),
- @error($crate::error::Error),
- @data(InitData, /*no use_data*/),
- @has_data(HasInitData, __init_data),
- @construct_closure(init_from_closure),
- @munch_fields($($fields)*),
- )
+ ::pin_init::try_pin_init!($(&$this in)? $t $(::<$($generics),* $(,)?>)? {
+ $($fields)*
+ }? $crate::error::Error)
};
($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
$($fields:tt)*
}? $err:ty) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t $(::<$($generics),*>)?),
- @fields($($fields)*),
- @error($err),
- @data(InitData, /*no use_data*/),
- @has_data(HasInitData, __init_data),
- @construct_closure(init_from_closure),
- @munch_fields($($fields)*),
- )
+ ::pin_init::try_pin_init!($(&$this in)? $t $(::<$($generics),* $(,)?>)? {
+ $($fields)*
+ }? $err)
};
}
-
-/// Asserts that a field on a struct using `#[pin_data]` is marked with `#[pin]` ie. that it is
-/// structurally pinned.
-///
-/// # Example
-///
-/// This will succeed:
-/// ```
-/// use kernel::assert_pinned;
-/// #[pin_data]
-/// struct MyStruct {
-/// #[pin]
-/// some_field: u64,
-/// }
-///
-/// assert_pinned!(MyStruct, some_field, u64);
-/// ```
-///
-/// This will fail:
-// TODO: replace with `compile_fail` when supported.
-/// ```ignore
-/// use kernel::assert_pinned;
-/// #[pin_data]
-/// struct MyStruct {
-/// some_field: u64,
-/// }
-///
-/// assert_pinned!(MyStruct, some_field, u64);
-/// ```
-///
-/// Some uses of the macro may trigger the `can't use generic parameters from outer item` error. To
-/// work around this, you may pass the `inline` parameter to the macro. The `inline` parameter can
-/// only be used when the macro is invoked from a function body.
-/// ```
-/// use kernel::assert_pinned;
-/// #[pin_data]
-/// struct Foo<T> {
-/// #[pin]
-/// elem: T,
-/// }
-///
-/// impl<T> Foo<T> {
-/// fn project(self: Pin<&mut Self>) -> Pin<&mut T> {
-/// assert_pinned!(Foo<T>, elem, T, inline);
-///
-/// // SAFETY: The field is structurally pinned.
-/// unsafe { self.map_unchecked_mut(|me| &mut me.elem) }
-/// }
-/// }
-/// ```
-#[macro_export]
-macro_rules! assert_pinned {
- ($ty:ty, $field:ident, $field_ty:ty, inline) => {
- let _ = move |ptr: *mut $field_ty| {
- // SAFETY: This code is unreachable.
- let data = unsafe { <$ty as $crate::init::__internal::HasPinData>::__pin_data() };
- let init = $crate::init::__internal::AlwaysFail::<$field_ty>::new();
- // SAFETY: This code is unreachable.
- unsafe { data.$field(ptr, init) }.ok();
- };
- };
-
- ($ty:ty, $field:ident, $field_ty:ty) => {
- const _: () = {
- $crate::assert_pinned!($ty, $field, $field_ty, inline);
- };
- };
-}
-
-/// A pin-initializer for the type `T`.
-///
-/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
-/// be [`KBox<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use
-/// the [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc<T>`] on this.
-///
-/// Also see the [module description](self).
-///
-/// # Safety
-///
-/// When implementing this trait you will need to take great care. Also there are probably very few
-/// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible.
-///
-/// The [`PinInit::__pinned_init`] function:
-/// - returns `Ok(())` if it initialized every field of `slot`,
-/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
-/// - `slot` can be deallocated without UB occurring,
-/// - `slot` does not need to be dropped,
-/// - `slot` is not partially initialized.
-/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
-///
-/// [`Arc<T>`]: crate::sync::Arc
-/// [`Arc::pin_init`]: crate::sync::Arc::pin_init
-#[must_use = "An initializer must be used in order to create its value."]
-pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
- /// Initializes `slot`.
- ///
- /// # Safety
- ///
- /// - `slot` is a valid pointer to uninitialized memory.
- /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
- /// deallocate.
- /// - `slot` will not move until it is dropped, i.e. it will be pinned.
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>;
-
- /// First initializes the value using `self` then calls the function `f` with the initialized
- /// value.
- ///
- /// If `f` returns an error the value is dropped and the initializer will forward the error.
- ///
- /// # Examples
- ///
- /// ```rust
- /// # #![expect(clippy::disallowed_names)]
- /// use kernel::{types::Opaque, init::pin_init_from_closure};
- /// #[repr(C)]
- /// struct RawFoo([u8; 16]);
- /// extern "C" {
- /// fn init_foo(_: *mut RawFoo);
- /// }
- ///
- /// #[pin_data]
- /// struct Foo {
- /// #[pin]
- /// raw: Opaque<RawFoo>,
- /// }
- ///
- /// impl Foo {
- /// fn setup(self: Pin<&mut Self>) {
- /// pr_info!("Setting up foo");
- /// }
- /// }
- ///
- /// let foo = pin_init!(Foo {
- /// // SAFETY: TODO.
- /// raw <- unsafe {
- /// Opaque::ffi_init(|s| {
- /// init_foo(s);
- /// })
- /// },
- /// }).pin_chain(|foo| {
- /// foo.setup();
- /// Ok(())
- /// });
- /// ```
- fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E>
- where
- F: FnOnce(Pin<&mut T>) -> Result<(), E>,
- {
- ChainPinInit(self, f, PhantomData)
- }
-}
-
-/// An initializer returned by [`PinInit::pin_chain`].
-pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, KBox<T>)>);
-
-// SAFETY: The `__pinned_init` function is implemented such that it
-// - returns `Ok(())` on successful initialization,
-// - returns `Err(err)` on error and in this case `slot` will be dropped.
-// - considers `slot` pinned.
-unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainPinInit<I, F, T, E>
-where
- I: PinInit<T, E>,
- F: FnOnce(Pin<&mut T>) -> Result<(), E>,
-{
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: All requirements fulfilled since this function is `__pinned_init`.
- unsafe { self.0.__pinned_init(slot)? };
- // SAFETY: The above call initialized `slot` and we still have unique access.
- let val = unsafe { &mut *slot };
- // SAFETY: `slot` is considered pinned.
- let val = unsafe { Pin::new_unchecked(val) };
- // SAFETY: `slot` was initialized above.
- (self.1)(val).inspect_err(|_| unsafe { core::ptr::drop_in_place(slot) })
- }
-}
-
-/// An initializer for `T`.
-///
-/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
-/// be [`KBox<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use
-/// the [`InPlaceInit::init`] function of a smart pointer like [`Arc<T>`] on this. Because
-/// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well.
-///
-/// Also see the [module description](self).
-///
-/// # Safety
-///
-/// When implementing this trait you will need to take great care. Also there are probably very few
-/// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible.
-///
-/// The [`Init::__init`] function:
-/// - returns `Ok(())` if it initialized every field of `slot`,
-/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
-/// - `slot` can be deallocated without UB occurring,
-/// - `slot` does not need to be dropped,
-/// - `slot` is not partially initialized.
-/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
-///
-/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same
-/// code as `__init`.
-///
-/// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to
-/// move the pointee after initialization.
-///
-/// [`Arc<T>`]: crate::sync::Arc
-#[must_use = "An initializer must be used in order to create its value."]
-pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
- /// Initializes `slot`.
- ///
- /// # Safety
- ///
- /// - `slot` is a valid pointer to uninitialized memory.
- /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
- /// deallocate.
- unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
-
- /// First initializes the value using `self` then calls the function `f` with the initialized
- /// value.
- ///
- /// If `f` returns an error the value is dropped and the initializer will forward the error.
- ///
- /// # Examples
- ///
- /// ```rust
- /// # #![expect(clippy::disallowed_names)]
- /// use kernel::{types::Opaque, init::{self, init_from_closure}};
- /// struct Foo {
- /// buf: [u8; 1_000_000],
- /// }
- ///
- /// impl Foo {
- /// fn setup(&mut self) {
- /// pr_info!("Setting up foo");
- /// }
- /// }
- ///
- /// let foo = init!(Foo {
- /// buf <- init::zeroed()
- /// }).chain(|foo| {
- /// foo.setup();
- /// Ok(())
- /// });
- /// ```
- fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E>
- where
- F: FnOnce(&mut T) -> Result<(), E>,
- {
- ChainInit(self, f, PhantomData)
- }
-}
-
-/// An initializer returned by [`Init::chain`].
-pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, KBox<T>)>);
-
-// SAFETY: The `__init` function is implemented such that it
-// - returns `Ok(())` on successful initialization,
-// - returns `Err(err)` on error and in this case `slot` will be dropped.
-unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E>
-where
- I: Init<T, E>,
- F: FnOnce(&mut T) -> Result<(), E>,
-{
- unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: All requirements fulfilled since this function is `__init`.
- unsafe { self.0.__pinned_init(slot)? };
- // SAFETY: The above call initialized `slot` and we still have unique access.
- (self.1)(unsafe { &mut *slot }).inspect_err(|_|
- // SAFETY: `slot` was initialized above.
- unsafe { core::ptr::drop_in_place(slot) })
- }
-}
-
-// SAFETY: `__pinned_init` behaves exactly the same as `__init`.
-unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E>
-where
- I: Init<T, E>,
- F: FnOnce(&mut T) -> Result<(), E>,
-{
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: `__init` has less strict requirements compared to `__pinned_init`.
- unsafe { self.__init(slot) }
- }
-}
-
-/// Creates a new [`PinInit<T, E>`] from the given closure.
-///
-/// # Safety
-///
-/// The closure:
-/// - returns `Ok(())` if it initialized every field of `slot`,
-/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
-/// - `slot` can be deallocated without UB occurring,
-/// - `slot` does not need to be dropped,
-/// - `slot` is not partially initialized.
-/// - may assume that the `slot` does not move if `T: !Unpin`,
-/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
-#[inline]
-pub const unsafe fn pin_init_from_closure<T: ?Sized, E>(
- f: impl FnOnce(*mut T) -> Result<(), E>,
-) -> impl PinInit<T, E> {
- __internal::InitClosure(f, PhantomData)
-}
-
-/// Creates a new [`Init<T, E>`] from the given closure.
-///
-/// # Safety
-///
-/// The closure:
-/// - returns `Ok(())` if it initialized every field of `slot`,
-/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
-/// - `slot` can be deallocated without UB occurring,
-/// - `slot` does not need to be dropped,
-/// - `slot` is not partially initialized.
-/// - the `slot` may move after initialization.
-/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
-#[inline]
-pub const unsafe fn init_from_closure<T: ?Sized, E>(
- f: impl FnOnce(*mut T) -> Result<(), E>,
-) -> impl Init<T, E> {
- __internal::InitClosure(f, PhantomData)
-}
-
-/// An initializer that leaves the memory uninitialized.
-///
-/// The initializer is a no-op. The `slot` memory is not changed.
-#[inline]
-pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
- // SAFETY: The memory is allowed to be uninitialized.
- unsafe { init_from_closure(|_| Ok(())) }
-}
-
-/// Initializes an array by initializing each element via the provided initializer.
-///
-/// # Examples
-///
-/// ```rust
-/// use kernel::{alloc::KBox, error::Error, init::init_array_from_fn};
-/// let array: KBox<[usize; 1_000]> =
-/// KBox::init::<Error>(init_array_from_fn(|i| i), GFP_KERNEL)?;
-/// assert_eq!(array.len(), 1_000);
-/// # Ok::<(), Error>(())
-/// ```
-pub fn init_array_from_fn<I, const N: usize, T, E>(
- mut make_init: impl FnMut(usize) -> I,
-) -> impl Init<[T; N], E>
-where
- I: Init<T, E>,
-{
- let init = move |slot: *mut [T; N]| {
- let slot = slot.cast::<T>();
- // Counts the number of initialized elements and when dropped drops that many elements from
- // `slot`.
- let mut init_count = ScopeGuard::new_with_data(0, |i| {
- // We now free every element that has been initialized before.
- // SAFETY: The loop initialized exactly the values from 0..i and since we
- // return `Err` below, the caller will consider the memory at `slot` as
- // uninitialized.
- unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
- });
- for i in 0..N {
- let init = make_init(i);
- // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
- let ptr = unsafe { slot.add(i) };
- // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
- // requirements.
- unsafe { init.__init(ptr) }?;
- *init_count += 1;
- }
- init_count.dismiss();
- Ok(())
- };
- // SAFETY: The initializer above initializes every element of the array. On failure it drops
- // any initialized elements and returns `Err`.
- unsafe { init_from_closure(init) }
-}
-
-/// Initializes an array by initializing each element via the provided initializer.
-///
-/// # Examples
-///
-/// ```rust
-/// use kernel::{sync::{Arc, Mutex}, init::pin_init_array_from_fn, new_mutex};
-/// let array: Arc<[Mutex<usize>; 1_000]> =
-/// Arc::pin_init(pin_init_array_from_fn(|i| new_mutex!(i)), GFP_KERNEL)?;
-/// assert_eq!(array.len(), 1_000);
-/// # Ok::<(), Error>(())
-/// ```
-pub fn pin_init_array_from_fn<I, const N: usize, T, E>(
- mut make_init: impl FnMut(usize) -> I,
-) -> impl PinInit<[T; N], E>
-where
- I: PinInit<T, E>,
-{
- let init = move |slot: *mut [T; N]| {
- let slot = slot.cast::<T>();
- // Counts the number of initialized elements and when dropped drops that many elements from
- // `slot`.
- let mut init_count = ScopeGuard::new_with_data(0, |i| {
- // We now free every element that has been initialized before.
- // SAFETY: The loop initialized exactly the values from 0..i and since we
- // return `Err` below, the caller will consider the memory at `slot` as
- // uninitialized.
- unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
- });
- for i in 0..N {
- let init = make_init(i);
- // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
- let ptr = unsafe { slot.add(i) };
- // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
- // requirements.
- unsafe { init.__pinned_init(ptr) }?;
- *init_count += 1;
- }
- init_count.dismiss();
- Ok(())
- };
- // SAFETY: The initializer above initializes every element of the array. On failure it drops
- // any initialized elements and returns `Err`.
- unsafe { pin_init_from_closure(init) }
-}
-
-// SAFETY: Every type can be initialized by-value.
-unsafe impl<T, E> Init<T, E> for T {
- unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: TODO.
- unsafe { slot.write(self) };
- Ok(())
- }
-}
-
-// SAFETY: Every type can be initialized by-value. `__pinned_init` calls `__init`.
-unsafe impl<T, E> PinInit<T, E> for T {
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: TODO.
- unsafe { self.__init(slot) }
- }
-}
-
-/// Smart pointer that can initialize memory in-place.
-pub trait InPlaceInit<T>: Sized {
- /// Pinned version of `Self`.
- ///
- /// If a type already implicitly pins its pointee, `Pin<Self>` is unnecessary. In this case use
- /// `Self`, otherwise just use `Pin<Self>`.
- type PinnedSelf;
-
- /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
- /// type.
- ///
- /// If `T: !Unpin` it will not be able to move afterwards.
- fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
- where
- E: From<AllocError>;
-
- /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
- /// type.
- ///
- /// If `T: !Unpin` it will not be able to move afterwards.
- fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Self::PinnedSelf>
- where
- Error: From<E>,
- {
- // SAFETY: We delegate to `init` and only change the error type.
- let init = unsafe {
- pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
- };
- Self::try_pin_init(init, flags)
- }
-
- /// Use the given initializer to in-place initialize a `T`.
- fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
- where
- E: From<AllocError>;
-
- /// Use the given initializer to in-place initialize a `T`.
- fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self>
- where
- Error: From<E>,
- {
- // SAFETY: We delegate to `init` and only change the error type.
- let init = unsafe {
- init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
- };
- Self::try_init(init, flags)
- }
-}
-
-impl<T> InPlaceInit<T> for Arc<T> {
- type PinnedSelf = Self;
-
- #[inline]
- fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
- where
- E: From<AllocError>,
- {
- UniqueArc::try_pin_init(init, flags).map(|u| u.into())
- }
-
- #[inline]
- fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
- where
- E: From<AllocError>,
- {
- UniqueArc::try_init(init, flags).map(|u| u.into())
- }
-}
-
-impl<T> InPlaceInit<T> for UniqueArc<T> {
- type PinnedSelf = Pin<Self>;
-
- #[inline]
- fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
- where
- E: From<AllocError>,
- {
- UniqueArc::new_uninit(flags)?.write_pin_init(init)
- }
-
- #[inline]
- fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
- where
- E: From<AllocError>,
- {
- UniqueArc::new_uninit(flags)?.write_init(init)
- }
-}
-
-/// Smart pointer containing uninitialized memory and that can write a value.
-pub trait InPlaceWrite<T> {
- /// The type `Self` turns into when the contents are initialized.
- type Initialized;
-
- /// Use the given initializer to write a value into `self`.
- ///
- /// Does not drop the current value and considers it as uninitialized memory.
- fn write_init<E>(self, init: impl Init<T, E>) -> Result<Self::Initialized, E>;
-
- /// Use the given pin-initializer to write a value into `self`.
- ///
- /// Does not drop the current value and considers it as uninitialized memory.
- fn write_pin_init<E>(self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E>;
-}
-
-impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> {
- type Initialized = UniqueArc<T>;
-
- fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
- let slot = self.as_mut_ptr();
- // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
- // slot is valid.
- unsafe { init.__init(slot)? };
- // SAFETY: All fields have been initialized.
- Ok(unsafe { self.assume_init() })
- }
-
- fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
- let slot = self.as_mut_ptr();
- // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
- // slot is valid and will not be moved, because we pin it later.
- unsafe { init.__pinned_init(slot)? };
- // SAFETY: All fields have been initialized.
- Ok(unsafe { self.assume_init() }.into())
- }
-}
-
-/// Trait facilitating pinned destruction.
-///
-/// Use [`pinned_drop`] to implement this trait safely:
-///
-/// ```rust
-/// # use kernel::sync::Mutex;
-/// use kernel::macros::pinned_drop;
-/// use core::pin::Pin;
-/// #[pin_data(PinnedDrop)]
-/// struct Foo {
-/// #[pin]
-/// mtx: Mutex<usize>,
-/// }
-///
-/// #[pinned_drop]
-/// impl PinnedDrop for Foo {
-/// fn drop(self: Pin<&mut Self>) {
-/// pr_info!("Foo is being dropped!");
-/// }
-/// }
-/// ```
-///
-/// # Safety
-///
-/// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl.
-///
-/// [`pinned_drop`]: kernel::macros::pinned_drop
-pub unsafe trait PinnedDrop: __internal::HasPinData {
- /// Executes the pinned destructor of this type.
- ///
- /// While this function is marked safe, it is actually unsafe to call it manually. For this
- /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code
- /// and thus prevents this function from being called where it should not.
- ///
- /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute
- /// automatically.
- fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop);
-}
-
-/// Marker trait for types that can be initialized by writing just zeroes.
-///
-/// # Safety
-///
-/// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words,
-/// this is not UB:
-///
-/// ```rust,ignore
-/// let val: Self = unsafe { core::mem::zeroed() };
-/// ```
-pub unsafe trait Zeroable {}
-
-/// Create a new zeroed T.
-///
-/// The returned initializer will write `0x00` to every byte of the given `slot`.
-#[inline]
-pub fn zeroed<T: Zeroable>() -> impl Init<T> {
- // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T`
- // and because we write all zeroes, the memory is initialized.
- unsafe {
- init_from_closure(|slot: *mut T| {
- slot.write_bytes(0, 1);
- Ok(())
- })
- }
-}
-
-macro_rules! impl_zeroable {
- ($($({$($generics:tt)*})? $t:ty, )*) => {
- // SAFETY: Safety comments written in the macro invocation.
- $(unsafe impl$($($generics)*)? Zeroable for $t {})*
- };
-}
-
-impl_zeroable! {
- // SAFETY: All primitives that are allowed to be zero.
- bool,
- char,
- u8, u16, u32, u64, u128, usize,
- i8, i16, i32, i64, i128, isize,
- f32, f64,
-
- // Note: do not add uninhabited types (such as `!` or `core::convert::Infallible`) to this list;
- // creating an instance of an uninhabited type is immediate undefined behavior. For more on
- // uninhabited/empty types, consult The Rustonomicon:
- // <https://doc.rust-lang.org/stable/nomicon/exotic-sizes.html#empty-types>. The Rust Reference
- // also has information on undefined behavior:
- // <https://doc.rust-lang.org/stable/reference/behavior-considered-undefined.html>.
- //
- // SAFETY: These are inhabited ZSTs; there is nothing to zero and a valid value exists.
- {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, (),
-
- // SAFETY: Type is allowed to take any value, including all zeros.
- {<T>} MaybeUninit<T>,
- // SAFETY: Type is allowed to take any value, including all zeros.
- {<T>} Opaque<T>,
-
- // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`.
- {<T: ?Sized + Zeroable>} UnsafeCell<T>,
-
- // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
- Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>,
- Option<NonZeroU128>, Option<NonZeroUsize>,
- Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>,
- Option<NonZeroI128>, Option<NonZeroIsize>,
-
- // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
- //
- // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant.
- {<T: ?Sized>} Option<NonNull<T>>,
- {<T: ?Sized>} Option<KBox<T>>,
-
- // SAFETY: `null` pointer is valid.
- //
- // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be
- // null.
- //
- // When `Pointee` gets stabilized, we could use
- // `T: ?Sized where <T as Pointee>::Metadata: Zeroable`
- {<T>} *mut T, {<T>} *const T,
-
- // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be
- // zero.
- {<T>} *mut [T], {<T>} *const [T], *mut str, *const str,
-
- // SAFETY: `T` is `Zeroable`.
- {<const N: usize, T: Zeroable>} [T; N], {<T: Zeroable>} Wrapping<T>,
-}
-
-macro_rules! impl_tuple_zeroable {
- ($(,)?) => {};
- ($first:ident, $($t:ident),* $(,)?) => {
- // SAFETY: All elements are zeroable and padding can be zero.
- unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {}
- impl_tuple_zeroable!($($t),* ,);
- }
-}
-
-impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J);
diff --git a/rust/kernel/init/__internal.rs b/rust/kernel/init/__internal.rs
deleted file mode 100644
index 74329cc3262c..000000000000
--- a/rust/kernel/init/__internal.rs
+++ /dev/null
@@ -1,264 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0 OR MIT
-
-//! This module contains API-internal items for pin-init.
-//!
-//! These items must not be used outside of
-//! - `kernel/init.rs`
-//! - `macros/pin_data.rs`
-//! - `macros/pinned_drop.rs`
-
-use super::*;
-
-/// See the [nomicon] for what subtyping is. See also [this table].
-///
-/// [nomicon]: https://doc.rust-lang.org/nomicon/subtyping.html
-/// [this table]: https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns
-pub(super) type Invariant<T> = PhantomData<fn(*mut T) -> *mut T>;
-
-/// Module-internal type implementing `PinInit` and `Init`.
-///
-/// It is unsafe to create this type, since the closure needs to fulfill the same safety
-/// requirement as the `__pinned_init`/`__init` functions.
-pub(crate) struct InitClosure<F, T: ?Sized, E>(pub(crate) F, pub(crate) Invariant<(E, T)>);
-
-// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the
-// `__init` invariants.
-unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T, E>
-where
- F: FnOnce(*mut T) -> Result<(), E>,
-{
- #[inline]
- unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
- (self.0)(slot)
- }
-}
-
-// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the
-// `__pinned_init` invariants.
-unsafe impl<T: ?Sized, F, E> PinInit<T, E> for InitClosure<F, T, E>
-where
- F: FnOnce(*mut T) -> Result<(), E>,
-{
- #[inline]
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- (self.0)(slot)
- }
-}
-
-/// This trait is only implemented via the `#[pin_data]` proc-macro. It is used to facilitate
-/// the pin projections within the initializers.
-///
-/// # Safety
-///
-/// Only the `init` module is allowed to use this trait.
-pub unsafe trait HasPinData {
- type PinData: PinData;
-
- #[expect(clippy::missing_safety_doc)]
- unsafe fn __pin_data() -> Self::PinData;
-}
-
-/// Marker trait for pinning data of structs.
-///
-/// # Safety
-///
-/// Only the `init` module is allowed to use this trait.
-pub unsafe trait PinData: Copy {
- type Datee: ?Sized + HasPinData;
-
- /// Type inference helper function.
- fn make_closure<F, O, E>(self, f: F) -> F
- where
- F: FnOnce(*mut Self::Datee) -> Result<O, E>,
- {
- f
- }
-}
-
-/// This trait is automatically implemented for every type. It aims to provide the same type
-/// inference help as `HasPinData`.
-///
-/// # Safety
-///
-/// Only the `init` module is allowed to use this trait.
-pub unsafe trait HasInitData {
- type InitData: InitData;
-
- #[expect(clippy::missing_safety_doc)]
- unsafe fn __init_data() -> Self::InitData;
-}
-
-/// Same function as `PinData`, but for arbitrary data.
-///
-/// # Safety
-///
-/// Only the `init` module is allowed to use this trait.
-pub unsafe trait InitData: Copy {
- type Datee: ?Sized + HasInitData;
-
- /// Type inference helper function.
- fn make_closure<F, O, E>(self, f: F) -> F
- where
- F: FnOnce(*mut Self::Datee) -> Result<O, E>,
- {
- f
- }
-}
-
-pub struct AllData<T: ?Sized>(PhantomData<fn(KBox<T>) -> KBox<T>>);
-
-impl<T: ?Sized> Clone for AllData<T> {
- fn clone(&self) -> Self {
- *self
- }
-}
-
-impl<T: ?Sized> Copy for AllData<T> {}
-
-// SAFETY: TODO.
-unsafe impl<T: ?Sized> InitData for AllData<T> {
- type Datee = T;
-}
-
-// SAFETY: TODO.
-unsafe impl<T: ?Sized> HasInitData for T {
- type InitData = AllData<T>;
-
- unsafe fn __init_data() -> Self::InitData {
- AllData(PhantomData)
- }
-}
-
-/// Stack initializer helper type. Use [`stack_pin_init`] instead of this primitive.
-///
-/// # Invariants
-///
-/// If `self.is_init` is true, then `self.value` is initialized.
-///
-/// [`stack_pin_init`]: kernel::stack_pin_init
-pub struct StackInit<T> {
- value: MaybeUninit<T>,
- is_init: bool,
-}
-
-impl<T> Drop for StackInit<T> {
- #[inline]
- fn drop(&mut self) {
- if self.is_init {
- // SAFETY: As we are being dropped, we only call this once. And since `self.is_init` is
- // true, `self.value` is initialized.
- unsafe { self.value.assume_init_drop() };
- }
- }
-}
-
-impl<T> StackInit<T> {
- /// Creates a new [`StackInit<T>`] that is uninitialized. Use [`stack_pin_init`] instead of this
- /// primitive.
- ///
- /// [`stack_pin_init`]: kernel::stack_pin_init
- #[inline]
- pub fn uninit() -> Self {
- Self {
- value: MaybeUninit::uninit(),
- is_init: false,
- }
- }
-
- /// Initializes the contents and returns the result.
- #[inline]
- pub fn init<E>(self: Pin<&mut Self>, init: impl PinInit<T, E>) -> Result<Pin<&mut T>, E> {
- // SAFETY: We never move out of `this`.
- let this = unsafe { Pin::into_inner_unchecked(self) };
- // The value is currently initialized, so it needs to be dropped before we can reuse
- // the memory (this is a safety guarantee of `Pin`).
- if this.is_init {
- this.is_init = false;
- // SAFETY: `this.is_init` was true and therefore `this.value` is initialized.
- unsafe { this.value.assume_init_drop() };
- }
- // SAFETY: The memory slot is valid and this type ensures that it will stay pinned.
- unsafe { init.__pinned_init(this.value.as_mut_ptr())? };
- // INVARIANT: `this.value` is initialized above.
- this.is_init = true;
- // SAFETY: The slot is now pinned, since we will never give access to `&mut T`.
- Ok(unsafe { Pin::new_unchecked(this.value.assume_init_mut()) })
- }
-}
-
-/// When a value of this type is dropped, it drops a `T`.
-///
-/// Can be forgotten to prevent the drop.
-pub struct DropGuard<T: ?Sized> {
- ptr: *mut T,
-}
-
-impl<T: ?Sized> DropGuard<T> {
- /// Creates a new [`DropGuard<T>`]. It will [`ptr::drop_in_place`] `ptr` when it gets dropped.
- ///
- /// # Safety
- ///
- /// `ptr` must be a valid pointer.
- ///
- /// It is the callers responsibility that `self` will only get dropped if the pointee of `ptr`:
- /// - has not been dropped,
- /// - is not accessible by any other means,
- /// - will not be dropped by any other means.
- #[inline]
- pub unsafe fn new(ptr: *mut T) -> Self {
- Self { ptr }
- }
-}
-
-impl<T: ?Sized> Drop for DropGuard<T> {
- #[inline]
- fn drop(&mut self) {
- // SAFETY: A `DropGuard` can only be constructed using the unsafe `new` function
- // ensuring that this operation is safe.
- unsafe { ptr::drop_in_place(self.ptr) }
- }
-}
-
-/// Token used by `PinnedDrop` to prevent calling the function without creating this unsafely
-/// created struct. This is needed, because the `drop` function is safe, but should not be called
-/// manually.
-pub struct OnlyCallFromDrop(());
-
-impl OnlyCallFromDrop {
- /// # Safety
- ///
- /// This function should only be called from the [`Drop::drop`] function and only be used to
- /// delegate the destruction to the pinned destructor [`PinnedDrop::drop`] of the same type.
- pub unsafe fn new() -> Self {
- Self(())
- }
-}
-
-/// Initializer that always fails.
-///
-/// Used by [`assert_pinned!`].
-///
-/// [`assert_pinned!`]: crate::assert_pinned
-pub struct AlwaysFail<T: ?Sized> {
- _t: PhantomData<T>,
-}
-
-impl<T: ?Sized> AlwaysFail<T> {
- /// Creates a new initializer that always fails.
- pub fn new() -> Self {
- Self { _t: PhantomData }
- }
-}
-
-impl<T: ?Sized> Default for AlwaysFail<T> {
- fn default() -> Self {
- Self::new()
- }
-}
-
-// SAFETY: `__pinned_init` always fails, which is always okay.
-unsafe impl<T: ?Sized> PinInit<T, ()> for AlwaysFail<T> {
- unsafe fn __pinned_init(self, _slot: *mut T) -> Result<(), ()> {
- Err(())
- }
-}
diff --git a/rust/kernel/init/macros.rs b/rust/kernel/init/macros.rs
deleted file mode 100644
index 1fd146a83241..000000000000
--- a/rust/kernel/init/macros.rs
+++ /dev/null
@@ -1,1410 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0 OR MIT
-
-//! This module provides the macros that actually implement the proc-macros `pin_data` and
-//! `pinned_drop`. It also contains `__init_internal` the implementation of the `{try_}{pin_}init!`
-//! macros.
-//!
-//! These macros should never be called directly, since they expect their input to be
-//! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in
-//! safe code! Use the public facing macros instead.
-//!
-//! This architecture has been chosen because the kernel does not yet have access to `syn` which
-//! would make matters a lot easier for implementing these as proc-macros.
-//!
-//! # Macro expansion example
-//!
-//! This section is intended for readers trying to understand the macros in this module and the
-//! `pin_init!` macros from `init.rs`.
-//!
-//! We will look at the following example:
-//!
-//! ```rust,ignore
-//! # use kernel::init::*;
-//! # use core::pin::Pin;
-//! #[pin_data]
-//! #[repr(C)]
-//! struct Bar<T> {
-//! #[pin]
-//! t: T,
-//! pub x: usize,
-//! }
-//!
-//! impl<T> Bar<T> {
-//! fn new(t: T) -> impl PinInit<Self> {
-//! pin_init!(Self { t, x: 0 })
-//! }
-//! }
-//!
-//! #[pin_data(PinnedDrop)]
-//! struct Foo {
-//! a: usize,
-//! #[pin]
-//! b: Bar<u32>,
-//! }
-//!
-//! #[pinned_drop]
-//! impl PinnedDrop for Foo {
-//! fn drop(self: Pin<&mut Self>) {
-//! pr_info!("{self:p} is getting dropped.");
-//! }
-//! }
-//!
-//! let a = 42;
-//! let initializer = pin_init!(Foo {
-//! a,
-//! b <- Bar::new(36),
-//! });
-//! ```
-//!
-//! This example includes the most common and important features of the pin-init API.
-//!
-//! Below you can find individual section about the different macro invocations. Here are some
-//! general things we need to take into account when designing macros:
-//! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()`
-//! this ensures that the correct item is used, since users could define their own `mod core {}`
-//! and then their own `panic!` inside to execute arbitrary code inside of our macro.
-//! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied
-//! expressions inside of an `unsafe` block in the macro, because this would allow users to do
-//! `unsafe` operations without an associated `unsafe` block.
-//!
-//! ## `#[pin_data]` on `Bar`
-//!
-//! This macro is used to specify which fields are structurally pinned and which fields are not. It
-//! is placed on the struct definition and allows `#[pin]` to be placed on the fields.
-//!
-//! Here is the definition of `Bar` from our example:
-//!
-//! ```rust,ignore
-//! # use kernel::init::*;
-//! #[pin_data]
-//! #[repr(C)]
-//! struct Bar<T> {
-//! #[pin]
-//! t: T,
-//! pub x: usize,
-//! }
-//! ```
-//!
-//! This expands to the following code:
-//!
-//! ```rust,ignore
-//! // Firstly the normal definition of the struct, attributes are preserved:
-//! #[repr(C)]
-//! struct Bar<T> {
-//! t: T,
-//! pub x: usize,
-//! }
-//! // Then an anonymous constant is defined, this is because we do not want any code to access the
-//! // types that we define inside:
-//! const _: () = {
-//! // We define the pin-data carrying struct, it is a ZST and needs to have the same generics,
-//! // since we need to implement access functions for each field and thus need to know its
-//! // type.
-//! struct __ThePinData<T> {
-//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
-//! }
-//! // We implement `Copy` for the pin-data struct, since all functions it defines will take
-//! // `self` by value.
-//! impl<T> ::core::clone::Clone for __ThePinData<T> {
-//! fn clone(&self) -> Self {
-//! *self
-//! }
-//! }
-//! impl<T> ::core::marker::Copy for __ThePinData<T> {}
-//! // For every field of `Bar`, the pin-data struct will define a function with the same name
-//! // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the
-//! // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field
-//! // (if pinning is structural for the field, then `PinInit` otherwise `Init`).
-//! #[allow(dead_code)]
-//! impl<T> __ThePinData<T> {
-//! unsafe fn t<E>(
-//! self,
-//! slot: *mut T,
-//! // Since `t` is `#[pin]`, this is `PinInit`.
-//! init: impl ::kernel::init::PinInit<T, E>,
-//! ) -> ::core::result::Result<(), E> {
-//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
-//! }
-//! pub unsafe fn x<E>(
-//! self,
-//! slot: *mut usize,
-//! // Since `x` is not `#[pin]`, this is `Init`.
-//! init: impl ::kernel::init::Init<usize, E>,
-//! ) -> ::core::result::Result<(), E> {
-//! unsafe { ::kernel::init::Init::__init(init, slot) }
-//! }
-//! }
-//! // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct
-//! // that we constructed above.
-//! unsafe impl<T> ::kernel::init::__internal::HasPinData for Bar<T> {
-//! type PinData = __ThePinData<T>;
-//! unsafe fn __pin_data() -> Self::PinData {
-//! __ThePinData {
-//! __phantom: ::core::marker::PhantomData,
-//! }
-//! }
-//! }
-//! // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data
-//! // struct. This is important to ensure that no user can implement a rogue `__pin_data`
-//! // function without using `unsafe`.
-//! unsafe impl<T> ::kernel::init::__internal::PinData for __ThePinData<T> {
-//! type Datee = Bar<T>;
-//! }
-//! // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is
-//! // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned
-//! // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our
-//! // case no such fields exist, hence this is almost empty. The two phantomdata fields exist
-//! // for two reasons:
-//! // - `__phantom`: every generic must be used, since we cannot really know which generics
-//! // are used, we declare all and then use everything here once.
-//! // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant
-//! // over it. The lifetime is needed to work around the limitation that trait bounds must
-//! // not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is
-//! // unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler
-//! // into accepting these bounds regardless.
-//! #[allow(dead_code)]
-//! struct __Unpin<'__pin, T> {
-//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
-//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
-//! // Our only `#[pin]` field is `t`.
-//! t: T,
-//! }
-//! #[doc(hidden)]
-//! impl<'__pin, T> ::core::marker::Unpin for Bar<T>
-//! where
-//! __Unpin<'__pin, T>: ::core::marker::Unpin,
-//! {}
-//! // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users
-//! // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to
-//! // UB with only safe code, so we disallow this by giving a trait implementation error using
-//! // a direct impl and a blanket implementation.
-//! trait MustNotImplDrop {}
-//! // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do
-//! // (normally people want to know if a type has any kind of drop glue at all, here we want
-//! // to know if it has any kind of custom drop glue, which is exactly what this bound does).
-//! #[expect(drop_bounds)]
-//! impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
-//! impl<T> MustNotImplDrop for Bar<T> {}
-//! // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to
-//! // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed
-//! // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`.
-//! #[expect(non_camel_case_types)]
-//! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
-//! impl<
-//! T: ::kernel::init::PinnedDrop,
-//! > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
-//! impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {}
-//! };
-//! ```
-//!
-//! ## `pin_init!` in `impl Bar`
-//!
-//! This macro creates an pin-initializer for the given struct. It requires that the struct is
-//! annotated by `#[pin_data]`.
-//!
-//! Here is the impl on `Bar` defining the new function:
-//!
-//! ```rust,ignore
-//! impl<T> Bar<T> {
-//! fn new(t: T) -> impl PinInit<Self> {
-//! pin_init!(Self { t, x: 0 })
-//! }
-//! }
-//! ```
-//!
-//! This expands to the following code:
-//!
-//! ```rust,ignore
-//! impl<T> Bar<T> {
-//! fn new(t: T) -> impl PinInit<Self> {
-//! {
-//! // We do not want to allow arbitrary returns, so we declare this type as the `Ok`
-//! // return type and shadow it later when we insert the arbitrary user code. That way
-//! // there will be no possibility of returning without `unsafe`.
-//! struct __InitOk;
-//! // Get the data about fields from the supplied type.
-//! // - the function is unsafe, hence the unsafe block
-//! // - we `use` the `HasPinData` trait in the block, it is only available in that
-//! // scope.
-//! let data = unsafe {
-//! use ::kernel::init::__internal::HasPinData;
-//! Self::__pin_data()
-//! };
-//! // Ensure that `data` really is of type `PinData` and help with type inference:
-//! let init = ::kernel::init::__internal::PinData::make_closure::<
-//! _,
-//! __InitOk,
-//! ::core::convert::Infallible,
-//! >(data, move |slot| {
-//! {
-//! // Shadow the structure so it cannot be used to return early. If a user
-//! // tries to write `return Ok(__InitOk)`, then they get a type error,
-//! // since that will refer to this struct instead of the one defined
-//! // above.
-//! struct __InitOk;
-//! // This is the expansion of `t,`, which is syntactic sugar for `t: t,`.
-//! {
-//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) };
-//! }
-//! // Since initialization could fail later (not in this case, since the
-//! // error type is `Infallible`) we will need to drop this field if there
-//! // is an error later. This `DropGuard` will drop the field when it gets
-//! // dropped and has not yet been forgotten.
-//! let __t_guard = unsafe {
-//! ::pinned_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t))
-//! };
-//! // Expansion of `x: 0,`:
-//! // Since this can be an arbitrary expression we cannot place it inside
-//! // of the `unsafe` block, so we bind it here.
-//! {
-//! let x = 0;
-//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) };
-//! }
-//! // We again create a `DropGuard`.
-//! let __x_guard = unsafe {
-//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x))
-//! };
-//! // Since initialization has successfully completed, we can now forget
-//! // the guards. This is not `mem::forget`, since we only have
-//! // `&DropGuard`.
-//! ::core::mem::forget(__x_guard);
-//! ::core::mem::forget(__t_guard);
-//! // Here we use the type checker to ensure that every field has been
-//! // initialized exactly once, since this is `if false` it will never get
-//! // executed, but still type-checked.
-//! // Additionally we abuse `slot` to automatically infer the correct type
-//! // for the struct. This is also another check that every field is
-//! // accessible from this scope.
-//! #[allow(unreachable_code, clippy::diverging_sub_expression)]
-//! let _ = || {
-//! unsafe {
-//! ::core::ptr::write(
-//! slot,
-//! Self {
-//! // We only care about typecheck finding every field
-//! // here, the expression does not matter, just conjure
-//! // one using `panic!()`:
-//! t: ::core::panic!(),
-//! x: ::core::panic!(),
-//! },
-//! );
-//! };
-//! };
-//! }
-//! // We leave the scope above and gain access to the previously shadowed
-//! // `__InitOk` that we need to return.
-//! Ok(__InitOk)
-//! });
-//! // Change the return type from `__InitOk` to `()`.
-//! let init = move |
-//! slot,
-//! | -> ::core::result::Result<(), ::core::convert::Infallible> {
-//! init(slot).map(|__InitOk| ())
-//! };
-//! // Construct the initializer.
-//! let init = unsafe {
-//! ::kernel::init::pin_init_from_closure::<
-//! _,
-//! ::core::convert::Infallible,
-//! >(init)
-//! };
-//! init
-//! }
-//! }
-//! }
-//! ```
-//!
-//! ## `#[pin_data]` on `Foo`
-//!
-//! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the
-//! differences/new things in the expansion of the `Foo` definition:
-//!
-//! ```rust,ignore
-//! #[pin_data(PinnedDrop)]
-//! struct Foo {
-//! a: usize,
-//! #[pin]
-//! b: Bar<u32>,
-//! }
-//! ```
-//!
-//! This expands to the following code:
-//!
-//! ```rust,ignore
-//! struct Foo {
-//! a: usize,
-//! b: Bar<u32>,
-//! }
-//! const _: () = {
-//! struct __ThePinData {
-//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
-//! }
-//! impl ::core::clone::Clone for __ThePinData {
-//! fn clone(&self) -> Self {
-//! *self
-//! }
-//! }
-//! impl ::core::marker::Copy for __ThePinData {}
-//! #[allow(dead_code)]
-//! impl __ThePinData {
-//! unsafe fn b<E>(
-//! self,
-//! slot: *mut Bar<u32>,
-//! init: impl ::kernel::init::PinInit<Bar<u32>, E>,
-//! ) -> ::core::result::Result<(), E> {
-//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
-//! }
-//! unsafe fn a<E>(
-//! self,
-//! slot: *mut usize,
-//! init: impl ::kernel::init::Init<usize, E>,
-//! ) -> ::core::result::Result<(), E> {
-//! unsafe { ::kernel::init::Init::__init(init, slot) }
-//! }
-//! }
-//! unsafe impl ::kernel::init::__internal::HasPinData for Foo {
-//! type PinData = __ThePinData;
-//! unsafe fn __pin_data() -> Self::PinData {
-//! __ThePinData {
-//! __phantom: ::core::marker::PhantomData,
-//! }
-//! }
-//! }
-//! unsafe impl ::kernel::init::__internal::PinData for __ThePinData {
-//! type Datee = Foo;
-//! }
-//! #[allow(dead_code)]
-//! struct __Unpin<'__pin> {
-//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
-//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
-//! b: Bar<u32>,
-//! }
-//! #[doc(hidden)]
-//! impl<'__pin> ::core::marker::Unpin for Foo
-//! where
-//! __Unpin<'__pin>: ::core::marker::Unpin,
-//! {}
-//! // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to
-//! // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like
-//! // before, instead we implement `Drop` here and delegate to `PinnedDrop`.
-//! impl ::core::ops::Drop for Foo {
-//! fn drop(&mut self) {
-//! // Since we are getting dropped, no one else has a reference to `self` and thus we
-//! // can assume that we never move.
-//! let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
-//! // Create the unsafe token that proves that we are inside of a destructor, this
-//! // type is only allowed to be created in a destructor.
-//! let token = unsafe { ::kernel::init::__internal::OnlyCallFromDrop::new() };
-//! ::kernel::init::PinnedDrop::drop(pinned, token);
-//! }
-//! }
-//! };
-//! ```
-//!
-//! ## `#[pinned_drop]` on `impl PinnedDrop for Foo`
-//!
-//! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an
-//! extra parameter that should not be used at all. The macro hides that parameter.
-//!
-//! Here is the `PinnedDrop` impl for `Foo`:
-//!
-//! ```rust,ignore
-//! #[pinned_drop]
-//! impl PinnedDrop for Foo {
-//! fn drop(self: Pin<&mut Self>) {
-//! pr_info!("{self:p} is getting dropped.");
-//! }
-//! }
-//! ```
-//!
-//! This expands to the following code:
-//!
-//! ```rust,ignore
-//! // `unsafe`, full path and the token parameter are added, everything else stays the same.
-//! unsafe impl ::kernel::init::PinnedDrop for Foo {
-//! fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) {
-//! pr_info!("{self:p} is getting dropped.");
-//! }
-//! }
-//! ```
-//!
-//! ## `pin_init!` on `Foo`
-//!
-//! Since we already took a look at `pin_init!` on `Bar`, this section will only show the expansion
-//! of `pin_init!` on `Foo`:
-//!
-//! ```rust,ignore
-//! let a = 42;
-//! let initializer = pin_init!(Foo {
-//! a,
-//! b <- Bar::new(36),
-//! });
-//! ```
-//!
-//! This expands to the following code:
-//!
-//! ```rust,ignore
-//! let a = 42;
-//! let initializer = {
-//! struct __InitOk;
-//! let data = unsafe {
-//! use ::kernel::init::__internal::HasPinData;
-//! Foo::__pin_data()
-//! };
-//! let init = ::kernel::init::__internal::PinData::make_closure::<
-//! _,
-//! __InitOk,
-//! ::core::convert::Infallible,
-//! >(data, move |slot| {
-//! {
-//! struct __InitOk;
-//! {
-//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) };
-//! }
-//! let __a_guard = unsafe {
-//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a))
-//! };
-//! let init = Bar::new(36);
-//! unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? };
-//! let __b_guard = unsafe {
-//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b))
-//! };
-//! ::core::mem::forget(__b_guard);
-//! ::core::mem::forget(__a_guard);
-//! #[allow(unreachable_code, clippy::diverging_sub_expression)]
-//! let _ = || {
-//! unsafe {
-//! ::core::ptr::write(
-//! slot,
-//! Foo {
-//! a: ::core::panic!(),
-//! b: ::core::panic!(),
-//! },
-//! );
-//! };
-//! };
-//! }
-//! Ok(__InitOk)
-//! });
-//! let init = move |
-//! slot,
-//! | -> ::core::result::Result<(), ::core::convert::Infallible> {
-//! init(slot).map(|__InitOk| ())
-//! };
-//! let init = unsafe {
-//! ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init)
-//! };
-//! init
-//! };
-//! ```
-
-/// Creates a `unsafe impl<...> PinnedDrop for $type` block.
-///
-/// See [`PinnedDrop`] for more information.
-#[doc(hidden)]
-#[macro_export]
-macro_rules! __pinned_drop {
- (
- @impl_sig($($impl_sig:tt)*),
- @impl_body(
- $(#[$($attr:tt)*])*
- fn drop($($sig:tt)*) {
- $($inner:tt)*
- }
- ),
- ) => {
- // SAFETY: TODO.
- unsafe $($impl_sig)* {
- // Inherit all attributes and the type/ident tokens for the signature.
- $(#[$($attr)*])*
- fn drop($($sig)*, _: $crate::init::__internal::OnlyCallFromDrop) {
- $($inner)*
- }
- }
- }
-}
-
-/// This macro first parses the struct definition such that it separates pinned and not pinned
-/// fields. Afterwards it declares the struct and implement the `PinData` trait safely.
-#[doc(hidden)]
-#[macro_export]
-macro_rules! __pin_data {
- // Proc-macro entry point, this is supplied by the proc-macro pre-parsing.
- (parse_input:
- @args($($pinned_drop:ident)?),
- @sig(
- $(#[$($struct_attr:tt)*])*
- $vis:vis struct $name:ident
- $(where $($whr:tt)*)?
- ),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @decl_generics($($decl_generics:tt)*),
- @body({ $($fields:tt)* }),
- ) => {
- // We now use token munching to iterate through all of the fields. While doing this we
- // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user
- // wants these to be structurally pinned. The rest of the fields are the
- // 'not pinned fields'. Additionally we collect all fields, since we need them in the right
- // order to declare the struct.
- //
- // In this call we also put some explaining comments for the parameters.
- $crate::__pin_data!(find_pinned_fields:
- // Attributes on the struct itself, these will just be propagated to be put onto the
- // struct definition.
- @struct_attrs($(#[$($struct_attr)*])*),
- // The visibility of the struct.
- @vis($vis),
- // The name of the struct.
- @name($name),
- // The 'impl generics', the generics that will need to be specified on the struct inside
- // of an `impl<$ty_generics>` block.
- @impl_generics($($impl_generics)*),
- // The 'ty generics', the generics that will need to be specified on the impl blocks.
- @ty_generics($($ty_generics)*),
- // The 'decl generics', the generics that need to be specified on the struct
- // definition.
- @decl_generics($($decl_generics)*),
- // The where clause of any impl block and the declaration.
- @where($($($whr)*)?),
- // The remaining fields tokens that need to be processed.
- // We add a `,` at the end to ensure correct parsing.
- @fields_munch($($fields)* ,),
- // The pinned fields.
- @pinned(),
- // The not pinned fields.
- @not_pinned(),
- // All fields.
- @fields(),
- // The accumulator containing all attributes already parsed.
- @accum(),
- // Contains `yes` or `` to indicate if `#[pin]` was found on the current field.
- @is_pinned(),
- // The proc-macro argument, this should be `PinnedDrop` or ``.
- @pinned_drop($($pinned_drop)?),
- );
- };
- (find_pinned_fields:
- @struct_attrs($($struct_attrs:tt)*),
- @vis($vis:vis),
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @decl_generics($($decl_generics:tt)*),
- @where($($whr:tt)*),
- // We found a PhantomPinned field, this should generally be pinned!
- @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*),
- @pinned($($pinned:tt)*),
- @not_pinned($($not_pinned:tt)*),
- @fields($($fields:tt)*),
- @accum($($accum:tt)*),
- // This field is not pinned.
- @is_pinned(),
- @pinned_drop($($pinned_drop:ident)?),
- ) => {
- ::core::compile_error!(concat!(
- "The field `",
- stringify!($field),
- "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.",
- ));
- $crate::__pin_data!(find_pinned_fields:
- @struct_attrs($($struct_attrs)*),
- @vis($vis),
- @name($name),
- @impl_generics($($impl_generics)*),
- @ty_generics($($ty_generics)*),
- @decl_generics($($decl_generics)*),
- @where($($whr)*),
- @fields_munch($($rest)*),
- @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,),
- @not_pinned($($not_pinned)*),
- @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,),
- @accum(),
- @is_pinned(),
- @pinned_drop($($pinned_drop)?),
- );
- };
- (find_pinned_fields:
- @struct_attrs($($struct_attrs:tt)*),
- @vis($vis:vis),
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @decl_generics($($decl_generics:tt)*),
- @where($($whr:tt)*),
- // We reached the field declaration.
- @fields_munch($field:ident : $type:ty, $($rest:tt)*),
- @pinned($($pinned:tt)*),
- @not_pinned($($not_pinned:tt)*),
- @fields($($fields:tt)*),
- @accum($($accum:tt)*),
- // This field is pinned.
- @is_pinned(yes),
- @pinned_drop($($pinned_drop:ident)?),
- ) => {
- $crate::__pin_data!(find_pinned_fields:
- @struct_attrs($($struct_attrs)*),
- @vis($vis),
- @name($name),
- @impl_generics($($impl_generics)*),
- @ty_generics($($ty_generics)*),
- @decl_generics($($decl_generics)*),
- @where($($whr)*),
- @fields_munch($($rest)*),
- @pinned($($pinned)* $($accum)* $field: $type,),
- @not_pinned($($not_pinned)*),
- @fields($($fields)* $($accum)* $field: $type,),
- @accum(),
- @is_pinned(),
- @pinned_drop($($pinned_drop)?),
- );
- };
- (find_pinned_fields:
- @struct_attrs($($struct_attrs:tt)*),
- @vis($vis:vis),
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @decl_generics($($decl_generics:tt)*),
- @where($($whr:tt)*),
- // We reached the field declaration.
- @fields_munch($field:ident : $type:ty, $($rest:tt)*),
- @pinned($($pinned:tt)*),
- @not_pinned($($not_pinned:tt)*),
- @fields($($fields:tt)*),
- @accum($($accum:tt)*),
- // This field is not pinned.
- @is_pinned(),
- @pinned_drop($($pinned_drop:ident)?),
- ) => {
- $crate::__pin_data!(find_pinned_fields:
- @struct_attrs($($struct_attrs)*),
- @vis($vis),
- @name($name),
- @impl_generics($($impl_generics)*),
- @ty_generics($($ty_generics)*),
- @decl_generics($($decl_generics)*),
- @where($($whr)*),
- @fields_munch($($rest)*),
- @pinned($($pinned)*),
- @not_pinned($($not_pinned)* $($accum)* $field: $type,),
- @fields($($fields)* $($accum)* $field: $type,),
- @accum(),
- @is_pinned(),
- @pinned_drop($($pinned_drop)?),
- );
- };
- (find_pinned_fields:
- @struct_attrs($($struct_attrs:tt)*),
- @vis($vis:vis),
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @decl_generics($($decl_generics:tt)*),
- @where($($whr:tt)*),
- // We found the `#[pin]` attr.
- @fields_munch(#[pin] $($rest:tt)*),
- @pinned($($pinned:tt)*),
- @not_pinned($($not_pinned:tt)*),
- @fields($($fields:tt)*),
- @accum($($accum:tt)*),
- @is_pinned($($is_pinned:ident)?),
- @pinned_drop($($pinned_drop:ident)?),
- ) => {
- $crate::__pin_data!(find_pinned_fields:
- @struct_attrs($($struct_attrs)*),
- @vis($vis),
- @name($name),
- @impl_generics($($impl_generics)*),
- @ty_generics($($ty_generics)*),
- @decl_generics($($decl_generics)*),
- @where($($whr)*),
- @fields_munch($($rest)*),
- // We do not include `#[pin]` in the list of attributes, since it is not actually an
- // attribute that is defined somewhere.
- @pinned($($pinned)*),
- @not_pinned($($not_pinned)*),
- @fields($($fields)*),
- @accum($($accum)*),
- // Set this to `yes`.
- @is_pinned(yes),
- @pinned_drop($($pinned_drop)?),
- );
- };
- (find_pinned_fields:
- @struct_attrs($($struct_attrs:tt)*),
- @vis($vis:vis),
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @decl_generics($($decl_generics:tt)*),
- @where($($whr:tt)*),
- // We reached the field declaration with visibility, for simplicity we only munch the
- // visibility and put it into `$accum`.
- @fields_munch($fvis:vis $field:ident $($rest:tt)*),
- @pinned($($pinned:tt)*),
- @not_pinned($($not_pinned:tt)*),
- @fields($($fields:tt)*),
- @accum($($accum:tt)*),
- @is_pinned($($is_pinned:ident)?),
- @pinned_drop($($pinned_drop:ident)?),
- ) => {
- $crate::__pin_data!(find_pinned_fields:
- @struct_attrs($($struct_attrs)*),
- @vis($vis),
- @name($name),
- @impl_generics($($impl_generics)*),
- @ty_generics($($ty_generics)*),
- @decl_generics($($decl_generics)*),
- @where($($whr)*),
- @fields_munch($field $($rest)*),
- @pinned($($pinned)*),
- @not_pinned($($not_pinned)*),
- @fields($($fields)*),
- @accum($($accum)* $fvis),
- @is_pinned($($is_pinned)?),
- @pinned_drop($($pinned_drop)?),
- );
- };
- (find_pinned_fields:
- @struct_attrs($($struct_attrs:tt)*),
- @vis($vis:vis),
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @decl_generics($($decl_generics:tt)*),
- @where($($whr:tt)*),
- // Some other attribute, just put it into `$accum`.
- @fields_munch(#[$($attr:tt)*] $($rest:tt)*),
- @pinned($($pinned:tt)*),
- @not_pinned($($not_pinned:tt)*),
- @fields($($fields:tt)*),
- @accum($($accum:tt)*),
- @is_pinned($($is_pinned:ident)?),
- @pinned_drop($($pinned_drop:ident)?),
- ) => {
- $crate::__pin_data!(find_pinned_fields:
- @struct_attrs($($struct_attrs)*),
- @vis($vis),
- @name($name),
- @impl_generics($($impl_generics)*),
- @ty_generics($($ty_generics)*),
- @decl_generics($($decl_generics)*),
- @where($($whr)*),
- @fields_munch($($rest)*),
- @pinned($($pinned)*),
- @not_pinned($($not_pinned)*),
- @fields($($fields)*),
- @accum($($accum)* #[$($attr)*]),
- @is_pinned($($is_pinned)?),
- @pinned_drop($($pinned_drop)?),
- );
- };
- (find_pinned_fields:
- @struct_attrs($($struct_attrs:tt)*),
- @vis($vis:vis),
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @decl_generics($($decl_generics:tt)*),
- @where($($whr:tt)*),
- // We reached the end of the fields, plus an optional additional comma, since we added one
- // before and the user is also allowed to put a trailing comma.
- @fields_munch($(,)?),
- @pinned($($pinned:tt)*),
- @not_pinned($($not_pinned:tt)*),
- @fields($($fields:tt)*),
- @accum(),
- @is_pinned(),
- @pinned_drop($($pinned_drop:ident)?),
- ) => {
- // Declare the struct with all fields in the correct order.
- $($struct_attrs)*
- $vis struct $name <$($decl_generics)*>
- where $($whr)*
- {
- $($fields)*
- }
-
- // We put the rest into this const item, because it then will not be accessible to anything
- // outside.
- const _: () = {
- // We declare this struct which will host all of the projection function for our type.
- // it will be invariant over all generic parameters which are inherited from the
- // struct.
- $vis struct __ThePinData<$($impl_generics)*>
- where $($whr)*
- {
- __phantom: ::core::marker::PhantomData<
- fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
- >,
- }
-
- impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*>
- where $($whr)*
- {
- fn clone(&self) -> Self { *self }
- }
-
- impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*>
- where $($whr)*
- {}
-
- // Make all projection functions.
- $crate::__pin_data!(make_pin_data:
- @pin_data(__ThePinData),
- @impl_generics($($impl_generics)*),
- @ty_generics($($ty_generics)*),
- @where($($whr)*),
- @pinned($($pinned)*),
- @not_pinned($($not_pinned)*),
- );
-
- // SAFETY: We have added the correct projection functions above to `__ThePinData` and
- // we also use the least restrictive generics possible.
- unsafe impl<$($impl_generics)*>
- $crate::init::__internal::HasPinData for $name<$($ty_generics)*>
- where $($whr)*
- {
- type PinData = __ThePinData<$($ty_generics)*>;
-
- unsafe fn __pin_data() -> Self::PinData {
- __ThePinData { __phantom: ::core::marker::PhantomData }
- }
- }
-
- // SAFETY: TODO.
- unsafe impl<$($impl_generics)*>
- $crate::init::__internal::PinData for __ThePinData<$($ty_generics)*>
- where $($whr)*
- {
- type Datee = $name<$($ty_generics)*>;
- }
-
- // This struct will be used for the unpin analysis. Since only structurally pinned
- // fields are relevant whether the struct should implement `Unpin`.
- #[allow(dead_code)]
- struct __Unpin <'__pin, $($impl_generics)*>
- where $($whr)*
- {
- __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
- __phantom: ::core::marker::PhantomData<
- fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
- >,
- // Only the pinned fields.
- $($pinned)*
- }
-
- #[doc(hidden)]
- impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*>
- where
- __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin,
- $($whr)*
- {}
-
- // We need to disallow normal `Drop` implementation, the exact behavior depends on
- // whether `PinnedDrop` was specified as the parameter.
- $crate::__pin_data!(drop_prevention:
- @name($name),
- @impl_generics($($impl_generics)*),
- @ty_generics($($ty_generics)*),
- @where($($whr)*),
- @pinned_drop($($pinned_drop)?),
- );
- };
- };
- // When no `PinnedDrop` was specified, then we have to prevent implementing drop.
- (drop_prevention:
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @where($($whr:tt)*),
- @pinned_drop(),
- ) => {
- // We prevent this by creating a trait that will be implemented for all types implementing
- // `Drop`. Additionally we will implement this trait for the struct leading to a conflict,
- // if it also implements `Drop`
- trait MustNotImplDrop {}
- #[expect(drop_bounds)]
- impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
- impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*>
- where $($whr)* {}
- // We also take care to prevent users from writing a useless `PinnedDrop` implementation.
- // They might implement `PinnedDrop` correctly for the struct, but forget to give
- // `PinnedDrop` as the parameter to `#[pin_data]`.
- #[expect(non_camel_case_types)]
- trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
- impl<T: $crate::init::PinnedDrop>
- UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
- impl<$($impl_generics)*>
- UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*>
- where $($whr)* {}
- };
- // When `PinnedDrop` was specified we just implement `Drop` and delegate.
- (drop_prevention:
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @where($($whr:tt)*),
- @pinned_drop(PinnedDrop),
- ) => {
- impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*>
- where $($whr)*
- {
- fn drop(&mut self) {
- // SAFETY: Since this is a destructor, `self` will not move after this function
- // terminates, since it is inaccessible.
- let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
- // SAFETY: Since this is a drop function, we can create this token to call the
- // pinned destructor of this type.
- let token = unsafe { $crate::init::__internal::OnlyCallFromDrop::new() };
- $crate::init::PinnedDrop::drop(pinned, token);
- }
- }
- };
- // If some other parameter was specified, we emit a readable error.
- (drop_prevention:
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @where($($whr:tt)*),
- @pinned_drop($($rest:tt)*),
- ) => {
- compile_error!(
- "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.",
- stringify!($($rest)*),
- );
- };
- (make_pin_data:
- @pin_data($pin_data:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @where($($whr:tt)*),
- @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?),
- @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?),
- ) => {
- // For every field, we create a projection function according to its projection type. If a
- // field is structurally pinned, then it must be initialized via `PinInit`, if it is not
- // structurally pinned, then it can be initialized via `Init`.
- //
- // The functions are `unsafe` to prevent accidentally calling them.
- #[allow(dead_code)]
- #[expect(clippy::missing_safety_doc)]
- impl<$($impl_generics)*> $pin_data<$($ty_generics)*>
- where $($whr)*
- {
- $(
- $(#[$($p_attr)*])*
- $pvis unsafe fn $p_field<E>(
- self,
- slot: *mut $p_type,
- init: impl $crate::init::PinInit<$p_type, E>,
- ) -> ::core::result::Result<(), E> {
- // SAFETY: TODO.
- unsafe { $crate::init::PinInit::__pinned_init(init, slot) }
- }
- )*
- $(
- $(#[$($attr)*])*
- $fvis unsafe fn $field<E>(
- self,
- slot: *mut $type,
- init: impl $crate::init::Init<$type, E>,
- ) -> ::core::result::Result<(), E> {
- // SAFETY: TODO.
- unsafe { $crate::init::Init::__init(init, slot) }
- }
- )*
- }
- };
-}
-
-/// The internal init macro. Do not call manually!
-///
-/// This is called by the `{try_}{pin_}init!` macros with various inputs.
-///
-/// This macro has multiple internal call configurations, these are always the very first ident:
-/// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros.
-/// - `with_update_parsed`: when the `..Zeroable::zeroed()` syntax has been handled.
-/// - `init_slot`: recursively creates the code that initializes all fields in `slot`.
-/// - `make_initializer`: recursively create the struct initializer that guarantees that every
-/// field has been initialized exactly once.
-#[doc(hidden)]
-#[macro_export]
-macro_rules! __init_internal {
- (
- @this($($this:ident)?),
- @typ($t:path),
- @fields($($fields:tt)*),
- @error($err:ty),
- // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
- // case.
- @data($data:ident, $($use_data:ident)?),
- // `HasPinData` or `HasInitData`.
- @has_data($has_data:ident, $get_data:ident),
- // `pin_init_from_closure` or `init_from_closure`.
- @construct_closure($construct_closure:ident),
- @munch_fields(),
- ) => {
- $crate::__init_internal!(with_update_parsed:
- @this($($this)?),
- @typ($t),
- @fields($($fields)*),
- @error($err),
- @data($data, $($use_data)?),
- @has_data($has_data, $get_data),
- @construct_closure($construct_closure),
- @zeroed(), // Nothing means default behavior.
- )
- };
- (
- @this($($this:ident)?),
- @typ($t:path),
- @fields($($fields:tt)*),
- @error($err:ty),
- // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
- // case.
- @data($data:ident, $($use_data:ident)?),
- // `HasPinData` or `HasInitData`.
- @has_data($has_data:ident, $get_data:ident),
- // `pin_init_from_closure` or `init_from_closure`.
- @construct_closure($construct_closure:ident),
- @munch_fields(..Zeroable::zeroed()),
- ) => {
- $crate::__init_internal!(with_update_parsed:
- @this($($this)?),
- @typ($t),
- @fields($($fields)*),
- @error($err),
- @data($data, $($use_data)?),
- @has_data($has_data, $get_data),
- @construct_closure($construct_closure),
- @zeroed(()), // `()` means zero all fields not mentioned.
- )
- };
- (
- @this($($this:ident)?),
- @typ($t:path),
- @fields($($fields:tt)*),
- @error($err:ty),
- // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
- // case.
- @data($data:ident, $($use_data:ident)?),
- // `HasPinData` or `HasInitData`.
- @has_data($has_data:ident, $get_data:ident),
- // `pin_init_from_closure` or `init_from_closure`.
- @construct_closure($construct_closure:ident),
- @munch_fields($ignore:tt $($rest:tt)*),
- ) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t),
- @fields($($fields)*),
- @error($err),
- @data($data, $($use_data)?),
- @has_data($has_data, $get_data),
- @construct_closure($construct_closure),
- @munch_fields($($rest)*),
- )
- };
- (with_update_parsed:
- @this($($this:ident)?),
- @typ($t:path),
- @fields($($fields:tt)*),
- @error($err:ty),
- // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
- // case.
- @data($data:ident, $($use_data:ident)?),
- // `HasPinData` or `HasInitData`.
- @has_data($has_data:ident, $get_data:ident),
- // `pin_init_from_closure` or `init_from_closure`.
- @construct_closure($construct_closure:ident),
- @zeroed($($init_zeroed:expr)?),
- ) => {{
- // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
- // type and shadow it later when we insert the arbitrary user code. That way there will be
- // no possibility of returning without `unsafe`.
- struct __InitOk;
- // Get the data about fields from the supplied type.
- //
- // SAFETY: TODO.
- let data = unsafe {
- use $crate::init::__internal::$has_data;
- // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
- // information that is associated to already parsed fragments, so a path fragment
- // cannot be used in this position. Doing the retokenization results in valid rust
- // code.
- ::kernel::macros::paste!($t::$get_data())
- };
- // Ensure that `data` really is of type `$data` and help with type inference:
- let init = $crate::init::__internal::$data::make_closure::<_, __InitOk, $err>(
- data,
- move |slot| {
- {
- // Shadow the structure so it cannot be used to return early.
- struct __InitOk;
- // If `$init_zeroed` is present we should zero the slot now and not emit an
- // error when fields are missing (since they will be zeroed). We also have to
- // check that the type actually implements `Zeroable`.
- $({
- fn assert_zeroable<T: $crate::init::Zeroable>(_: *mut T) {}
- // Ensure that the struct is indeed `Zeroable`.
- assert_zeroable(slot);
- // SAFETY: The type implements `Zeroable` by the check above.
- unsafe { ::core::ptr::write_bytes(slot, 0, 1) };
- $init_zeroed // This will be `()` if set.
- })?
- // Create the `this` so it can be referenced by the user inside of the
- // expressions creating the individual fields.
- $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
- // Initialize every field.
- $crate::__init_internal!(init_slot($($use_data)?):
- @data(data),
- @slot(slot),
- @guards(),
- @munch_fields($($fields)*,),
- );
- // We use unreachable code to ensure that all fields have been mentioned exactly
- // once, this struct initializer will still be type-checked and complain with a
- // very natural error message if a field is forgotten/mentioned more than once.
- #[allow(unreachable_code, clippy::diverging_sub_expression)]
- let _ = || {
- $crate::__init_internal!(make_initializer:
- @slot(slot),
- @type_name($t),
- @munch_fields($($fields)*,),
- @acc(),
- );
- };
- }
- Ok(__InitOk)
- }
- );
- let init = move |slot| -> ::core::result::Result<(), $err> {
- init(slot).map(|__InitOk| ())
- };
- // SAFETY: TODO.
- let init = unsafe { $crate::init::$construct_closure::<_, $err>(init) };
- init
- }};
- (init_slot($($use_data:ident)?):
- @data($data:ident),
- @slot($slot:ident),
- @guards($($guards:ident,)*),
- @munch_fields($(..Zeroable::zeroed())? $(,)?),
- ) => {
- // Endpoint of munching, no fields are left. If execution reaches this point, all fields
- // have been initialized. Therefore we can now dismiss the guards by forgetting them.
- $(::core::mem::forget($guards);)*
- };
- (init_slot($use_data:ident): // `use_data` is present, so we use the `data` to init fields.
- @data($data:ident),
- @slot($slot:ident),
- @guards($($guards:ident,)*),
- // In-place initialization syntax.
- @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
- ) => {
- let init = $val;
- // Call the initializer.
- //
- // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
- // return when an error/panic occurs.
- // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`.
- unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? };
- // Create the drop guard:
- //
- // We rely on macro hygiene to make it impossible for users to access this local variable.
- // We use `paste!` to create new hygiene for `$field`.
- ::kernel::macros::paste! {
- // SAFETY: We forget the guard later when initialization has succeeded.
- let [< __ $field _guard >] = unsafe {
- $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
- };
-
- $crate::__init_internal!(init_slot($use_data):
- @data($data),
- @slot($slot),
- @guards([< __ $field _guard >], $($guards,)*),
- @munch_fields($($rest)*),
- );
- }
- };
- (init_slot(): // No `use_data`, so we use `Init::__init` directly.
- @data($data:ident),
- @slot($slot:ident),
- @guards($($guards:ident,)*),
- // In-place initialization syntax.
- @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
- ) => {
- let init = $val;
- // Call the initializer.
- //
- // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
- // return when an error/panic occurs.
- unsafe { $crate::init::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? };
- // Create the drop guard:
- //
- // We rely on macro hygiene to make it impossible for users to access this local variable.
- // We use `paste!` to create new hygiene for `$field`.
- ::kernel::macros::paste! {
- // SAFETY: We forget the guard later when initialization has succeeded.
- let [< __ $field _guard >] = unsafe {
- $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
- };
-
- $crate::__init_internal!(init_slot():
- @data($data),
- @slot($slot),
- @guards([< __ $field _guard >], $($guards,)*),
- @munch_fields($($rest)*),
- );
- }
- };
- (init_slot($($use_data:ident)?):
- @data($data:ident),
- @slot($slot:ident),
- @guards($($guards:ident,)*),
- // Init by-value.
- @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
- ) => {
- {
- $(let $field = $val;)?
- // Initialize the field.
- //
- // SAFETY: The memory at `slot` is uninitialized.
- unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
- }
- // Create the drop guard:
- //
- // We rely on macro hygiene to make it impossible for users to access this local variable.
- // We use `paste!` to create new hygiene for `$field`.
- ::kernel::macros::paste! {
- // SAFETY: We forget the guard later when initialization has succeeded.
- let [< __ $field _guard >] = unsafe {
- $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
- };
-
- $crate::__init_internal!(init_slot($($use_data)?):
- @data($data),
- @slot($slot),
- @guards([< __ $field _guard >], $($guards,)*),
- @munch_fields($($rest)*),
- );
- }
- };
- (make_initializer:
- @slot($slot:ident),
- @type_name($t:path),
- @munch_fields(..Zeroable::zeroed() $(,)?),
- @acc($($acc:tt)*),
- ) => {
- // Endpoint, nothing more to munch, create the initializer. Since the users specified
- // `..Zeroable::zeroed()`, the slot will already have been zeroed and all field that have
- // not been overwritten are thus zero and initialized. We still check that all fields are
- // actually accessible by using the struct update syntax ourselves.
- // We are inside of a closure that is never executed and thus we can abuse `slot` to
- // get the correct type inference here:
- #[allow(unused_assignments)]
- unsafe {
- let mut zeroed = ::core::mem::zeroed();
- // We have to use type inference here to make zeroed have the correct type. This does
- // not get executed, so it has no effect.
- ::core::ptr::write($slot, zeroed);
- zeroed = ::core::mem::zeroed();
- // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
- // information that is associated to already parsed fragments, so a path fragment
- // cannot be used in this position. Doing the retokenization results in valid rust
- // code.
- ::kernel::macros::paste!(
- ::core::ptr::write($slot, $t {
- $($acc)*
- ..zeroed
- });
- );
- }
- };
- (make_initializer:
- @slot($slot:ident),
- @type_name($t:path),
- @munch_fields($(,)?),
- @acc($($acc:tt)*),
- ) => {
- // Endpoint, nothing more to munch, create the initializer.
- // Since we are in the closure that is never called, this will never get executed.
- // We abuse `slot` to get the correct type inference here:
- //
- // SAFETY: TODO.
- unsafe {
- // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
- // information that is associated to already parsed fragments, so a path fragment
- // cannot be used in this position. Doing the retokenization results in valid rust
- // code.
- ::kernel::macros::paste!(
- ::core::ptr::write($slot, $t {
- $($acc)*
- });
- );
- }
- };
- (make_initializer:
- @slot($slot:ident),
- @type_name($t:path),
- @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
- @acc($($acc:tt)*),
- ) => {
- $crate::__init_internal!(make_initializer:
- @slot($slot),
- @type_name($t),
- @munch_fields($($rest)*),
- @acc($($acc)* $field: ::core::panic!(),),
- );
- };
- (make_initializer:
- @slot($slot:ident),
- @type_name($t:path),
- @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
- @acc($($acc:tt)*),
- ) => {
- $crate::__init_internal!(make_initializer:
- @slot($slot),
- @type_name($t),
- @munch_fields($($rest)*),
- @acc($($acc)* $field: ::core::panic!(),),
- );
- };
-}
-
-#[doc(hidden)]
-#[macro_export]
-macro_rules! __derive_zeroable {
- (parse_input:
- @sig(
- $(#[$($struct_attr:tt)*])*
- $vis:vis struct $name:ident
- $(where $($whr:tt)*)?
- ),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @body({
- $(
- $(#[$($field_attr:tt)*])*
- $field:ident : $field_ty:ty
- ),* $(,)?
- }),
- ) => {
- // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
- #[automatically_derived]
- unsafe impl<$($impl_generics)*> $crate::init::Zeroable for $name<$($ty_generics)*>
- where
- $($($whr)*)?
- {}
- const _: () = {
- fn assert_zeroable<T: ?::core::marker::Sized + $crate::init::Zeroable>() {}
- fn ensure_zeroable<$($impl_generics)*>()
- where $($($whr)*)?
- {
- $(assert_zeroable::<$field_ty>();)*
- }
- };
- };
-}
diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs
index d4a73e52e3ee..72d80a6f131e 100644
--- a/rust/kernel/io.rs
+++ b/rust/kernel/io.rs
@@ -98,9 +98,9 @@ impl<const SIZE: usize> IoRaw<SIZE> {
///# fn no_run() -> Result<(), Error> {
/// // SAFETY: Invalid usage for example purposes.
/// let iomem = unsafe { IoMem::<{ core::mem::size_of::<u32>() }>::new(0xBAAAAAAD)? };
-/// iomem.writel(0x42, 0x0);
-/// assert!(iomem.try_writel(0x42, 0x0).is_ok());
-/// assert!(iomem.try_writel(0x42, 0x4).is_err());
+/// iomem.write32(0x42, 0x0);
+/// assert!(iomem.try_write32(0x42, 0x0).is_ok());
+/// assert!(iomem.try_write32(0x42, 0x4).is_err());
/// # Ok(())
/// # }
/// ```
@@ -108,7 +108,7 @@ impl<const SIZE: usize> IoRaw<SIZE> {
pub struct Io<const SIZE: usize = 0>(IoRaw<SIZE>);
macro_rules! define_read {
- ($(#[$attr:meta])* $name:ident, $try_name:ident, $type_name:ty) => {
+ ($(#[$attr:meta])* $name:ident, $try_name:ident, $c_fn:ident -> $type_name:ty) => {
/// Read IO data from a given offset known at compile time.
///
/// Bound checks are performed on compile time, hence if the offset is not known at compile
@@ -119,7 +119,7 @@ macro_rules! define_read {
let addr = self.io_addr_assert::<$type_name>(offset);
// SAFETY: By the type invariant `addr` is a valid address for MMIO operations.
- unsafe { bindings::$name(addr as _) }
+ unsafe { bindings::$c_fn(addr as _) }
}
/// Read IO data from a given offset.
@@ -131,13 +131,13 @@ macro_rules! define_read {
let addr = self.io_addr::<$type_name>(offset)?;
// SAFETY: By the type invariant `addr` is a valid address for MMIO operations.
- Ok(unsafe { bindings::$name(addr as _) })
+ Ok(unsafe { bindings::$c_fn(addr as _) })
}
};
}
macro_rules! define_write {
- ($(#[$attr:meta])* $name:ident, $try_name:ident, $type_name:ty) => {
+ ($(#[$attr:meta])* $name:ident, $try_name:ident, $c_fn:ident <- $type_name:ty) => {
/// Write IO data from a given offset known at compile time.
///
/// Bound checks are performed on compile time, hence if the offset is not known at compile
@@ -148,7 +148,7 @@ macro_rules! define_write {
let addr = self.io_addr_assert::<$type_name>(offset);
// SAFETY: By the type invariant `addr` is a valid address for MMIO operations.
- unsafe { bindings::$name(value, addr as _, ) }
+ unsafe { bindings::$c_fn(value, addr as _, ) }
}
/// Write IO data from a given offset.
@@ -160,7 +160,7 @@ macro_rules! define_write {
let addr = self.io_addr::<$type_name>(offset)?;
// SAFETY: By the type invariant `addr` is a valid address for MMIO operations.
- unsafe { bindings::$name(value, addr as _) }
+ unsafe { bindings::$c_fn(value, addr as _) }
Ok(())
}
};
@@ -218,43 +218,43 @@ impl<const SIZE: usize> Io<SIZE> {
self.addr() + offset
}
- define_read!(readb, try_readb, u8);
- define_read!(readw, try_readw, u16);
- define_read!(readl, try_readl, u32);
+ define_read!(read8, try_read8, readb -> u8);
+ define_read!(read16, try_read16, readw -> u16);
+ define_read!(read32, try_read32, readl -> u32);
define_read!(
#[cfg(CONFIG_64BIT)]
- readq,
- try_readq,
- u64
+ read64,
+ try_read64,
+ readq -> u64
);
- define_read!(readb_relaxed, try_readb_relaxed, u8);
- define_read!(readw_relaxed, try_readw_relaxed, u16);
- define_read!(readl_relaxed, try_readl_relaxed, u32);
+ define_read!(read8_relaxed, try_read8_relaxed, readb_relaxed -> u8);
+ define_read!(read16_relaxed, try_read16_relaxed, readw_relaxed -> u16);
+ define_read!(read32_relaxed, try_read32_relaxed, readl_relaxed -> u32);
define_read!(
#[cfg(CONFIG_64BIT)]
- readq_relaxed,
- try_readq_relaxed,
- u64
+ read64_relaxed,
+ try_read64_relaxed,
+ readq_relaxed -> u64
);
- define_write!(writeb, try_writeb, u8);
- define_write!(writew, try_writew, u16);
- define_write!(writel, try_writel, u32);
+ define_write!(write8, try_write8, writeb <- u8);
+ define_write!(write16, try_write16, writew <- u16);
+ define_write!(write32, try_write32, writel <- u32);
define_write!(
#[cfg(CONFIG_64BIT)]
- writeq,
- try_writeq,
- u64
+ write64,
+ try_write64,
+ writeq <- u64
);
- define_write!(writeb_relaxed, try_writeb_relaxed, u8);
- define_write!(writew_relaxed, try_writew_relaxed, u16);
- define_write!(writel_relaxed, try_writel_relaxed, u32);
+ define_write!(write8_relaxed, try_write8_relaxed, writeb_relaxed <- u8);
+ define_write!(write16_relaxed, try_write16_relaxed, writew_relaxed <- u16);
+ define_write!(write32_relaxed, try_write32_relaxed, writel_relaxed <- u32);
define_write!(
#[cfg(CONFIG_64BIT)]
- writeq_relaxed,
- try_writeq_relaxed,
- u64
+ write64_relaxed,
+ try_write64_relaxed,
+ writeq_relaxed <- u64
);
}
diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs
index 824da0e9738a..4b8cdcb21e77 100644
--- a/rust/kernel/kunit.rs
+++ b/rust/kernel/kunit.rs
@@ -6,6 +6,7 @@
//!
//! Reference: <https://docs.kernel.org/dev-tools/kunit/index.html>
+use crate::prelude::*;
use core::{ffi::c_void, fmt};
/// Prints a KUnit error-level message.
@@ -57,7 +58,7 @@ macro_rules! kunit_assert {
}
static FILE: &'static $crate::str::CStr = $crate::c_str!($file);
- static LINE: i32 = core::line!() as i32 - $diff;
+ static LINE: i32 = ::core::line!() as i32 - $diff;
static CONDITION: &'static $crate::str::CStr = $crate::c_str!(stringify!($condition));
// SAFETY: FFI call without safety requirements.
@@ -128,11 +129,11 @@ macro_rules! kunit_assert {
unsafe {
$crate::bindings::__kunit_do_failed_assertion(
kunit_test,
- core::ptr::addr_of!(LOCATION.0),
+ ::core::ptr::addr_of!(LOCATION.0),
$crate::bindings::kunit_assert_type_KUNIT_ASSERTION,
- core::ptr::addr_of!(ASSERTION.0.assert),
+ ::core::ptr::addr_of!(ASSERTION.0.assert),
Some($crate::bindings::kunit_unary_assert_format),
- core::ptr::null(),
+ ::core::ptr::null(),
);
}
@@ -161,3 +162,196 @@ macro_rules! kunit_assert_eq {
$crate::kunit_assert!($name, $file, $diff, $left == $right);
}};
}
+
+trait TestResult {
+ fn is_test_result_ok(&self) -> bool;
+}
+
+impl TestResult for () {
+ fn is_test_result_ok(&self) -> bool {
+ true
+ }
+}
+
+impl<T, E> TestResult for Result<T, E> {
+ fn is_test_result_ok(&self) -> bool {
+ self.is_ok()
+ }
+}
+
+/// Returns whether a test result is to be considered OK.
+///
+/// This will be `assert!`ed from the generated tests.
+#[doc(hidden)]
+#[expect(private_bounds)]
+pub fn is_test_result_ok(t: impl TestResult) -> bool {
+ t.is_test_result_ok()
+}
+
+/// Represents an individual test case.
+///
+/// The [`kunit_unsafe_test_suite!`] macro expects a NULL-terminated list of valid test cases.
+/// Use [`kunit_case_null`] to generate such a delimiter.
+#[doc(hidden)]
+pub const fn kunit_case(
+ name: &'static kernel::str::CStr,
+ run_case: unsafe extern "C" fn(*mut kernel::bindings::kunit),
+) -> kernel::bindings::kunit_case {
+ kernel::bindings::kunit_case {
+ run_case: Some(run_case),
+ name: name.as_char_ptr(),
+ attr: kernel::bindings::kunit_attributes {
+ speed: kernel::bindings::kunit_speed_KUNIT_SPEED_NORMAL,
+ },
+ generate_params: None,
+ status: kernel::bindings::kunit_status_KUNIT_SUCCESS,
+ module_name: core::ptr::null_mut(),
+ log: core::ptr::null_mut(),
+ }
+}
+
+/// Represents the NULL test case delimiter.
+///
+/// The [`kunit_unsafe_test_suite!`] macro expects a NULL-terminated list of test cases. This
+/// function returns such a delimiter.
+#[doc(hidden)]
+pub const fn kunit_case_null() -> kernel::bindings::kunit_case {
+ kernel::bindings::kunit_case {
+ run_case: None,
+ name: core::ptr::null_mut(),
+ generate_params: None,
+ attr: kernel::bindings::kunit_attributes {
+ speed: kernel::bindings::kunit_speed_KUNIT_SPEED_NORMAL,
+ },
+ status: kernel::bindings::kunit_status_KUNIT_SUCCESS,
+ module_name: core::ptr::null_mut(),
+ log: core::ptr::null_mut(),
+ }
+}
+
+/// Registers a KUnit test suite.
+///
+/// # Safety
+///
+/// `test_cases` must be a NULL terminated array of valid test cases,
+/// whose lifetime is at least that of the test suite (i.e., static).
+///
+/// # Examples
+///
+/// ```ignore
+/// extern "C" fn test_fn(_test: *mut kernel::bindings::kunit) {
+/// let actual = 1 + 1;
+/// let expected = 2;
+/// assert_eq!(actual, expected);
+/// }
+///
+/// static mut KUNIT_TEST_CASES: [kernel::bindings::kunit_case; 2] = [
+/// kernel::kunit::kunit_case(kernel::c_str!("name"), test_fn),
+/// kernel::kunit::kunit_case_null(),
+/// ];
+/// kernel::kunit_unsafe_test_suite!(suite_name, KUNIT_TEST_CASES);
+/// ```
+#[doc(hidden)]
+#[macro_export]
+macro_rules! kunit_unsafe_test_suite {
+ ($name:ident, $test_cases:ident) => {
+ const _: () = {
+ const KUNIT_TEST_SUITE_NAME: [::kernel::ffi::c_char; 256] = {
+ let name_u8 = ::core::stringify!($name).as_bytes();
+ let mut ret = [0; 256];
+
+ if name_u8.len() > 255 {
+ panic!(concat!(
+ "The test suite name `",
+ ::core::stringify!($name),
+ "` exceeds the maximum length of 255 bytes."
+ ));
+ }
+
+ let mut i = 0;
+ while i < name_u8.len() {
+ ret[i] = name_u8[i] as ::kernel::ffi::c_char;
+ i += 1;
+ }
+
+ ret
+ };
+
+ static mut KUNIT_TEST_SUITE: ::kernel::bindings::kunit_suite =
+ ::kernel::bindings::kunit_suite {
+ name: KUNIT_TEST_SUITE_NAME,
+ #[allow(unused_unsafe)]
+ // SAFETY: `$test_cases` is passed in by the user, and
+ // (as documented) must be valid for the lifetime of
+ // the suite (i.e., static).
+ test_cases: unsafe {
+ ::core::ptr::addr_of_mut!($test_cases)
+ .cast::<::kernel::bindings::kunit_case>()
+ },
+ suite_init: None,
+ suite_exit: None,
+ init: None,
+ exit: None,
+ attr: ::kernel::bindings::kunit_attributes {
+ speed: ::kernel::bindings::kunit_speed_KUNIT_SPEED_NORMAL,
+ },
+ status_comment: [0; 256usize],
+ debugfs: ::core::ptr::null_mut(),
+ log: ::core::ptr::null_mut(),
+ suite_init_err: 0,
+ is_init: false,
+ };
+
+ #[used]
+ #[allow(unused_unsafe)]
+ #[cfg_attr(not(target_os = "macos"), link_section = ".kunit_test_suites")]
+ static mut KUNIT_TEST_SUITE_ENTRY: *const ::kernel::bindings::kunit_suite =
+ // SAFETY: `KUNIT_TEST_SUITE` is static.
+ unsafe { ::core::ptr::addr_of_mut!(KUNIT_TEST_SUITE) };
+ };
+ };
+}
+
+/// Returns whether we are currently running a KUnit test.
+///
+/// In some cases, you need to call test-only code from outside the test case, for example, to
+/// create a function mock. This function allows to change behavior depending on whether we are
+/// currently running a KUnit test or not.
+///
+/// # Examples
+///
+/// This example shows how a function can be mocked to return a well-known value while testing:
+///
+/// ```
+/// # use kernel::kunit::in_kunit_test;
+/// fn fn_mock_example(n: i32) -> i32 {
+/// if in_kunit_test() {
+/// return 100;
+/// }
+///
+/// n + 1
+/// }
+///
+/// let mock_res = fn_mock_example(5);
+/// assert_eq!(mock_res, 100);
+/// ```
+pub fn in_kunit_test() -> bool {
+ // SAFETY: `kunit_get_current_test()` is always safe to call (it has fallbacks for
+ // when KUnit is not enabled).
+ !unsafe { bindings::kunit_get_current_test() }.is_null()
+}
+
+#[kunit_tests(rust_kernel_kunit)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn rust_test_kunit_example_test() {
+ assert_eq!(1 + 1, 2);
+ }
+
+ #[test]
+ fn rust_test_kunit_in_kunit_test() {
+ assert!(in_kunit_test());
+ }
+}
diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs
index 398242f92a96..6b4774b2b1c3 100644
--- a/rust/kernel/lib.rs
+++ b/rust/kernel/lib.rs
@@ -6,24 +6,40 @@
//! usage by Rust code in the kernel and is shared by all of them.
//!
//! In other words, all the rest of the Rust code in the kernel (e.g. kernel
-//! modules written in Rust) depends on [`core`], [`alloc`] and this crate.
+//! modules written in Rust) depends on [`core`] and this crate.
//!
//! If you need a kernel C API that is not ported or wrapped yet here, then
//! do so first instead of bypassing this crate.
#![no_std]
-#![feature(arbitrary_self_types)]
-#![cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, feature(derive_coerce_pointee))]
-#![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(coerce_unsized))]
-#![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(dispatch_from_dyn))]
-#![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(unsize))]
+//
+// Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on
+// the unstable features in use.
+//
+// Stable since Rust 1.79.0.
#![feature(inline_const)]
+//
+// Stable since Rust 1.81.0.
#![feature(lint_reasons)]
-// Stable in Rust 1.83
+//
+// Stable since Rust 1.82.0.
+#![feature(raw_ref_op)]
+//
+// Stable since Rust 1.83.0.
#![feature(const_maybe_uninit_as_mut_ptr)]
#![feature(const_mut_refs)]
#![feature(const_ptr_write)]
#![feature(const_refs_to_cell)]
+//
+// Expected to become stable.
+#![feature(arbitrary_self_types)]
+//
+// `feature(derive_coerce_pointee)` is expected to become stable. Before Rust
+// 1.84.0, it did not exist, so enable the predecessor features.
+#![cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, feature(derive_coerce_pointee))]
+#![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(coerce_unsized))]
+#![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(dispatch_from_dyn))]
+#![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(unsize))]
// Ensure conditional compilation based on the kernel configuration works;
// otherwise we may silently break things like initcall handling.
@@ -36,15 +52,27 @@ extern crate self as kernel;
pub use ffi;
pub mod alloc;
+#[cfg(CONFIG_AUXILIARY_BUS)]
+pub mod auxiliary;
#[cfg(CONFIG_BLOCK)]
pub mod block;
#[doc(hidden)]
pub mod build_assert;
+pub mod clk;
+#[cfg(CONFIG_CONFIGFS_FS)]
+pub mod configfs;
+pub mod cpu;
+#[cfg(CONFIG_CPU_FREQ)]
+pub mod cpufreq;
+pub mod cpumask;
pub mod cred;
pub mod device;
pub mod device_id;
pub mod devres;
+pub mod dma;
pub mod driver;
+#[cfg(CONFIG_DRM = "y")]
+pub mod drm;
pub mod error;
pub mod faux;
#[cfg(CONFIG_RUST_FW_LOADER_ABSTRACTIONS)]
@@ -58,9 +86,12 @@ pub mod jump_label;
pub mod kunit;
pub mod list;
pub mod miscdevice;
+pub mod mm;
#[cfg(CONFIG_NET)]
pub mod net;
pub mod of;
+#[cfg(CONFIG_PM_OPP)]
+pub mod opp;
pub mod page;
#[cfg(CONFIG_PCI)]
pub mod pci;
@@ -85,6 +116,7 @@ pub mod transmute;
pub mod types;
pub mod uaccess;
pub mod workqueue;
+pub mod xarray;
#[doc(hidden)]
pub use bindings;
@@ -112,11 +144,11 @@ pub trait InPlaceModule: Sync + Send {
/// Creates an initialiser for the module.
///
/// It is called when the module is loaded.
- fn init(module: &'static ThisModule) -> impl init::PinInit<Self, error::Error>;
+ fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error>;
}
impl<T: Module> InPlaceModule for T {
- fn init(module: &'static ThisModule) -> impl init::PinInit<Self, error::Error> {
+ fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error> {
let initer = move |slot: *mut Self| {
let m = <Self as Module>::init(module)?;
@@ -126,7 +158,7 @@ impl<T: Module> InPlaceModule for T {
};
// SAFETY: On success, `initer` always fully initialises an instance of `Self`.
- unsafe { init::pin_init_from_closure(initer) }
+ unsafe { pin_init::pin_init_from_closure(initer) }
}
}
@@ -187,7 +219,7 @@ fn panic(info: &core::panic::PanicInfo<'_>) -> ! {
/// }
///
/// let test = Test { a: 10, b: 20 };
-/// let b_ptr = &test.b;
+/// let b_ptr: *const _ = &test.b;
/// // SAFETY: The pointer points at the `b` field of a `Test`, so the resulting pointer will be
/// // in-bounds of the same allocation as `b_ptr`.
/// let test_alias = unsafe { container_of!(b_ptr, Test, b) };
@@ -195,13 +227,19 @@ fn panic(info: &core::panic::PanicInfo<'_>) -> ! {
/// ```
#[macro_export]
macro_rules! container_of {
- ($ptr:expr, $type:ty, $($f:tt)*) => {{
- let ptr = $ptr as *const _ as *const u8;
- let offset: usize = ::core::mem::offset_of!($type, $($f)*);
- ptr.sub(offset) as *const $type
+ ($field_ptr:expr, $Container:ty, $($fields:tt)*) => {{
+ let offset: usize = ::core::mem::offset_of!($Container, $($fields)*);
+ let field_ptr = $field_ptr;
+ let container_ptr = field_ptr.byte_sub(offset).cast::<$Container>();
+ $crate::assert_same_type(field_ptr, (&raw const (*container_ptr).$($fields)*).cast_mut());
+ container_ptr
}}
}
+/// Helper for [`container_of!`].
+#[doc(hidden)]
+pub fn assert_same_type<T>(_: T, _: T) {}
+
/// Helper for `.rs.S` files.
#[doc(hidden)]
#[macro_export]
diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs
index fb93330f4af4..c391c30b80f8 100644
--- a/rust/kernel/list.rs
+++ b/rust/kernel/list.rs
@@ -4,12 +4,12 @@
//! A linked list implementation.
-use crate::init::PinInit;
use crate::sync::ArcBorrow;
use crate::types::Opaque;
use core::iter::{DoubleEndedIterator, FusedIterator};
use core::marker::PhantomData;
use core::ptr;
+use pin_init::PinInit;
mod impl_list_item_mod;
pub use self::impl_list_item_mod::{
@@ -35,6 +35,114 @@ pub use self::arc_field::{define_list_arc_field_getter, ListArcField};
/// * All prev/next pointers in `ListLinks` fields of items in the list are valid and form a cycle.
/// * For every item in the list, the list owns the associated [`ListArc`] reference and has
/// exclusive access to the `ListLinks` field.
+///
+/// # Examples
+///
+/// ```
+/// use kernel::list::*;
+///
+/// #[pin_data]
+/// struct BasicItem {
+/// value: i32,
+/// #[pin]
+/// links: ListLinks,
+/// }
+///
+/// impl BasicItem {
+/// fn new(value: i32) -> Result<ListArc<Self>> {
+/// ListArc::pin_init(try_pin_init!(Self {
+/// value,
+/// links <- ListLinks::new(),
+/// }), GFP_KERNEL)
+/// }
+/// }
+///
+/// impl_has_list_links! {
+/// impl HasListLinks<0> for BasicItem { self.links }
+/// }
+/// impl_list_arc_safe! {
+/// impl ListArcSafe<0> for BasicItem { untracked; }
+/// }
+/// impl_list_item! {
+/// impl ListItem<0> for BasicItem { using ListLinks; }
+/// }
+///
+/// // Create a new empty list.
+/// let mut list = List::new();
+/// {
+/// assert!(list.is_empty());
+/// }
+///
+/// // Insert 3 elements using `push_back()`.
+/// list.push_back(BasicItem::new(15)?);
+/// list.push_back(BasicItem::new(10)?);
+/// list.push_back(BasicItem::new(30)?);
+///
+/// // Iterate over the list to verify the nodes were inserted correctly.
+/// // [15, 10, 30]
+/// {
+/// let mut iter = list.iter();
+/// assert_eq!(iter.next().unwrap().value, 15);
+/// assert_eq!(iter.next().unwrap().value, 10);
+/// assert_eq!(iter.next().unwrap().value, 30);
+/// assert!(iter.next().is_none());
+///
+/// // Verify the length of the list.
+/// assert_eq!(list.iter().count(), 3);
+/// }
+///
+/// // Pop the items from the list using `pop_back()` and verify the content.
+/// {
+/// assert_eq!(list.pop_back().unwrap().value, 30);
+/// assert_eq!(list.pop_back().unwrap().value, 10);
+/// assert_eq!(list.pop_back().unwrap().value, 15);
+/// }
+///
+/// // Insert 3 elements using `push_front()`.
+/// list.push_front(BasicItem::new(15)?);
+/// list.push_front(BasicItem::new(10)?);
+/// list.push_front(BasicItem::new(30)?);
+///
+/// // Iterate over the list to verify the nodes were inserted correctly.
+/// // [30, 10, 15]
+/// {
+/// let mut iter = list.iter();
+/// assert_eq!(iter.next().unwrap().value, 30);
+/// assert_eq!(iter.next().unwrap().value, 10);
+/// assert_eq!(iter.next().unwrap().value, 15);
+/// assert!(iter.next().is_none());
+///
+/// // Verify the length of the list.
+/// assert_eq!(list.iter().count(), 3);
+/// }
+///
+/// // Pop the items from the list using `pop_front()` and verify the content.
+/// {
+/// assert_eq!(list.pop_front().unwrap().value, 30);
+/// assert_eq!(list.pop_front().unwrap().value, 10);
+/// }
+///
+/// // Push `list2` to `list` through `push_all_back()`.
+/// // list: [15]
+/// // list2: [25, 35]
+/// {
+/// let mut list2 = List::new();
+/// list2.push_back(BasicItem::new(25)?);
+/// list2.push_back(BasicItem::new(35)?);
+///
+/// list.push_all_back(&mut list2);
+///
+/// // list: [15, 25, 35]
+/// // list2: []
+/// let mut iter = list.iter();
+/// assert_eq!(iter.next().unwrap().value, 15);
+/// assert_eq!(iter.next().unwrap().value, 25);
+/// assert_eq!(iter.next().unwrap().value, 35);
+/// assert!(iter.next().is_none());
+/// assert!(list2.is_empty());
+/// }
+/// # Result::<(), Error>::Ok(())
+/// ```
pub struct List<T: ?Sized + ListItem<ID>, const ID: u64 = 0> {
first: *mut ListLinksFields,
_ty: PhantomData<ListArc<T, ID>>,
@@ -245,8 +353,20 @@ impl<T: ?Sized + ListItem<ID>, const ID: u64> List<T, ID> {
self.first.is_null()
}
- /// Add the provided item to the back of the list.
- pub fn push_back(&mut self, item: ListArc<T, ID>) {
+ /// Inserts `item` before `next` in the cycle.
+ ///
+ /// Returns a pointer to the newly inserted element. Never changes `self.first` unless the list
+ /// is empty.
+ ///
+ /// # Safety
+ ///
+ /// * `next` must be an element in this list or null.
+ /// * if `next` is null, then the list must be empty.
+ unsafe fn insert_inner(
+ &mut self,
+ item: ListArc<T, ID>,
+ next: *mut ListLinksFields,
+ ) -> *mut ListLinksFields {
let raw_item = ListArc::into_raw(item);
// SAFETY:
// * We just got `raw_item` from a `ListArc`, so it's in an `Arc`.
@@ -259,16 +379,16 @@ impl<T: ?Sized + ListItem<ID>, const ID: u64> List<T, ID> {
// SAFETY: We have not yet called `post_remove`, so `list_links` is still valid.
let item = unsafe { ListLinks::fields(list_links) };
- if self.first.is_null() {
- self.first = item;
+ // Check if the list is empty.
+ if next.is_null() {
// SAFETY: The caller just gave us ownership of these fields.
// INVARIANT: A linked list with one item should be cyclic.
unsafe {
(*item).next = item;
(*item).prev = item;
}
+ self.first = item;
} else {
- let next = self.first;
// SAFETY: By the type invariant, this pointer is valid or null. We just checked that
// it's not null, so it must be valid.
let prev = unsafe { (*next).prev };
@@ -282,50 +402,32 @@ impl<T: ?Sized + ListItem<ID>, const ID: u64> List<T, ID> {
(*next).prev = item;
}
}
+
+ item
+ }
+
+ /// Add the provided item to the back of the list.
+ pub fn push_back(&mut self, item: ListArc<T, ID>) {
+ // SAFETY:
+ // * `self.first` is null or in the list.
+ // * `self.first` is only null if the list is empty.
+ unsafe { self.insert_inner(item, self.first) };
}
/// Add the provided item to the front of the list.
pub fn push_front(&mut self, item: ListArc<T, ID>) {
- let raw_item = ListArc::into_raw(item);
// SAFETY:
- // * We just got `raw_item` from a `ListArc`, so it's in an `Arc`.
- // * If this requirement is violated, then the previous caller of `prepare_to_insert`
- // violated the safety requirement that they can't give up ownership of the `ListArc`
- // until they call `post_remove`.
- // * We own the `ListArc`.
- // * Removing items] from this list is always done using `remove_internal_inner`, which
- // calls `post_remove` before giving up ownership.
- let list_links = unsafe { T::prepare_to_insert(raw_item) };
- // SAFETY: We have not yet called `post_remove`, so `list_links` is still valid.
- let item = unsafe { ListLinks::fields(list_links) };
+ // * `self.first` is null or in the list.
+ // * `self.first` is only null if the list is empty.
+ let new_elem = unsafe { self.insert_inner(item, self.first) };
- if self.first.is_null() {
- // SAFETY: The caller just gave us ownership of these fields.
- // INVARIANT: A linked list with one item should be cyclic.
- unsafe {
- (*item).next = item;
- (*item).prev = item;
- }
- } else {
- let next = self.first;
- // SAFETY: We just checked that `next` is non-null.
- let prev = unsafe { (*next).prev };
- // SAFETY: Pointers in a linked list are never dangling, and the caller just gave us
- // ownership of the fields on `item`.
- // INVARIANT: This correctly inserts `item` between `prev` and `next`.
- unsafe {
- (*item).next = next;
- (*item).prev = prev;
- (*prev).next = item;
- (*next).prev = item;
- }
- }
- self.first = item;
+ // INVARIANT: `new_elem` is in the list because we just inserted it.
+ self.first = new_elem;
}
/// Removes the last item from this list.
pub fn pop_back(&mut self) -> Option<ListArc<T, ID>> {
- if self.first.is_null() {
+ if self.is_empty() {
return None;
}
@@ -337,7 +439,7 @@ impl<T: ?Sized + ListItem<ID>, const ID: u64> List<T, ID> {
/// Removes the first item from this list.
pub fn pop_front(&mut self) -> Option<ListArc<T, ID>> {
- if self.first.is_null() {
+ if self.is_empty() {
return None;
}
@@ -489,17 +591,21 @@ impl<T: ?Sized + ListItem<ID>, const ID: u64> List<T, ID> {
other.first = ptr::null_mut();
}
- /// Returns a cursor to the first element of the list.
- ///
- /// If the list is empty, this returns `None`.
- pub fn cursor_front(&mut self) -> Option<Cursor<'_, T, ID>> {
- if self.first.is_null() {
- None
- } else {
- Some(Cursor {
- current: self.first,
- list: self,
- })
+ /// Returns a cursor that points before the first element of the list.
+ pub fn cursor_front(&mut self) -> Cursor<'_, T, ID> {
+ // INVARIANT: `self.first` is in this list.
+ Cursor {
+ next: self.first,
+ list: self,
+ }
+ }
+
+ /// Returns a cursor that points after the last element in the list.
+ pub fn cursor_back(&mut self) -> Cursor<'_, T, ID> {
+ // INVARIANT: `next` is allowed to be null.
+ Cursor {
+ next: core::ptr::null_mut(),
+ list: self,
}
}
@@ -579,69 +685,358 @@ impl<'a, T: ?Sized + ListItem<ID>, const ID: u64> Iterator for Iter<'a, T, ID> {
/// A cursor into a [`List`].
///
+/// A cursor always rests between two elements in the list. This means that a cursor has a previous
+/// and next element, but no current element. It also means that it's possible to have a cursor
+/// into an empty list.
+///
+/// # Examples
+///
+/// ```
+/// use kernel::prelude::*;
+/// use kernel::list::{List, ListArc, ListLinks};
+///
+/// #[pin_data]
+/// struct ListItem {
+/// value: u32,
+/// #[pin]
+/// links: ListLinks,
+/// }
+///
+/// impl ListItem {
+/// fn new(value: u32) -> Result<ListArc<Self>> {
+/// ListArc::pin_init(try_pin_init!(Self {
+/// value,
+/// links <- ListLinks::new(),
+/// }), GFP_KERNEL)
+/// }
+/// }
+///
+/// kernel::list::impl_has_list_links! {
+/// impl HasListLinks<0> for ListItem { self.links }
+/// }
+/// kernel::list::impl_list_arc_safe! {
+/// impl ListArcSafe<0> for ListItem { untracked; }
+/// }
+/// kernel::list::impl_list_item! {
+/// impl ListItem<0> for ListItem { using ListLinks; }
+/// }
+///
+/// // Use a cursor to remove the first element with the given value.
+/// fn remove_first(list: &mut List<ListItem>, value: u32) -> Option<ListArc<ListItem>> {
+/// let mut cursor = list.cursor_front();
+/// while let Some(next) = cursor.peek_next() {
+/// if next.value == value {
+/// return Some(next.remove());
+/// }
+/// cursor.move_next();
+/// }
+/// None
+/// }
+///
+/// // Use a cursor to remove the last element with the given value.
+/// fn remove_last(list: &mut List<ListItem>, value: u32) -> Option<ListArc<ListItem>> {
+/// let mut cursor = list.cursor_back();
+/// while let Some(prev) = cursor.peek_prev() {
+/// if prev.value == value {
+/// return Some(prev.remove());
+/// }
+/// cursor.move_prev();
+/// }
+/// None
+/// }
+///
+/// // Use a cursor to remove all elements with the given value. The removed elements are moved to
+/// // a new list.
+/// fn remove_all(list: &mut List<ListItem>, value: u32) -> List<ListItem> {
+/// let mut out = List::new();
+/// let mut cursor = list.cursor_front();
+/// while let Some(next) = cursor.peek_next() {
+/// if next.value == value {
+/// out.push_back(next.remove());
+/// } else {
+/// cursor.move_next();
+/// }
+/// }
+/// out
+/// }
+///
+/// // Use a cursor to insert a value at a specific index. Returns an error if the index is out of
+/// // bounds.
+/// fn insert_at(list: &mut List<ListItem>, new: ListArc<ListItem>, idx: usize) -> Result {
+/// let mut cursor = list.cursor_front();
+/// for _ in 0..idx {
+/// if !cursor.move_next() {
+/// return Err(EINVAL);
+/// }
+/// }
+/// cursor.insert_next(new);
+/// Ok(())
+/// }
+///
+/// // Merge two sorted lists into a single sorted list.
+/// fn merge_sorted(list: &mut List<ListItem>, merge: List<ListItem>) {
+/// let mut cursor = list.cursor_front();
+/// for to_insert in merge {
+/// while let Some(next) = cursor.peek_next() {
+/// if to_insert.value < next.value {
+/// break;
+/// }
+/// cursor.move_next();
+/// }
+/// cursor.insert_prev(to_insert);
+/// }
+/// }
+///
+/// let mut list = List::new();
+/// list.push_back(ListItem::new(14)?);
+/// list.push_back(ListItem::new(12)?);
+/// list.push_back(ListItem::new(10)?);
+/// list.push_back(ListItem::new(12)?);
+/// list.push_back(ListItem::new(15)?);
+/// list.push_back(ListItem::new(14)?);
+/// assert_eq!(remove_all(&mut list, 12).iter().count(), 2);
+/// // [14, 10, 15, 14]
+/// assert!(remove_first(&mut list, 14).is_some());
+/// // [10, 15, 14]
+/// insert_at(&mut list, ListItem::new(12)?, 2)?;
+/// // [10, 15, 12, 14]
+/// assert!(remove_last(&mut list, 15).is_some());
+/// // [10, 12, 14]
+///
+/// let mut list2 = List::new();
+/// list2.push_back(ListItem::new(11)?);
+/// list2.push_back(ListItem::new(13)?);
+/// merge_sorted(&mut list, list2);
+///
+/// let mut items = list.into_iter();
+/// assert_eq!(items.next().unwrap().value, 10);
+/// assert_eq!(items.next().unwrap().value, 11);
+/// assert_eq!(items.next().unwrap().value, 12);
+/// assert_eq!(items.next().unwrap().value, 13);
+/// assert_eq!(items.next().unwrap().value, 14);
+/// assert!(items.next().is_none());
+/// # Result::<(), Error>::Ok(())
+/// ```
+///
/// # Invariants
///
-/// The `current` pointer points a value in `list`.
+/// The `next` pointer is null or points a value in `list`.
pub struct Cursor<'a, T: ?Sized + ListItem<ID>, const ID: u64 = 0> {
- current: *mut ListLinksFields,
list: &'a mut List<T, ID>,
+ /// Points at the element after this cursor, or null if the cursor is after the last element.
+ next: *mut ListLinksFields,
}
impl<'a, T: ?Sized + ListItem<ID>, const ID: u64> Cursor<'a, T, ID> {
- /// Access the current element of this cursor.
- pub fn current(&self) -> ArcBorrow<'_, T> {
- // SAFETY: The `current` pointer points a value in the list.
- let me = unsafe { T::view_value(ListLinks::from_fields(self.current)) };
- // SAFETY:
- // * All values in a list are stored in an `Arc`.
- // * The value cannot be removed from the list for the duration of the lifetime annotated
- // on the returned `ArcBorrow`, because removing it from the list would require mutable
- // access to the cursor or the list. However, the `ArcBorrow` holds an immutable borrow
- // on the cursor, which in turn holds a mutable borrow on the list, so any such
- // mutable access requires first releasing the immutable borrow on the cursor.
- // * Values in a list never have a `UniqueArc` reference, because the list has a `ListArc`
- // reference, and `UniqueArc` references must be unique.
- unsafe { ArcBorrow::from_raw(me) }
+ /// Returns a pointer to the element before the cursor.
+ ///
+ /// Returns null if there is no element before the cursor.
+ fn prev_ptr(&self) -> *mut ListLinksFields {
+ let mut next = self.next;
+ let first = self.list.first;
+ if next == first {
+ // We are before the first element.
+ return core::ptr::null_mut();
+ }
+
+ if next.is_null() {
+ // We are after the last element, so we need a pointer to the last element, which is
+ // the same as `(*first).prev`.
+ next = first;
+ }
+
+ // SAFETY: `next` can't be null, because then `first` must also be null, but in that case
+ // we would have exited at the `next == first` check. Thus, `next` is an element in the
+ // list, so we can access its `prev` pointer.
+ unsafe { (*next).prev }
+ }
+
+ /// Access the element after this cursor.
+ pub fn peek_next(&mut self) -> Option<CursorPeek<'_, 'a, T, true, ID>> {
+ if self.next.is_null() {
+ return None;
+ }
+
+ // INVARIANT:
+ // * We just checked that `self.next` is non-null, so it must be in `self.list`.
+ // * `ptr` is equal to `self.next`.
+ Some(CursorPeek {
+ ptr: self.next,
+ cursor: self,
+ })
}
- /// Move the cursor to the next element.
- pub fn next(self) -> Option<Cursor<'a, T, ID>> {
- // SAFETY: The `current` field is always in a list.
- let next = unsafe { (*self.current).next };
+ /// Access the element before this cursor.
+ pub fn peek_prev(&mut self) -> Option<CursorPeek<'_, 'a, T, false, ID>> {
+ let prev = self.prev_ptr();
+
+ if prev.is_null() {
+ return None;
+ }
+
+ // INVARIANT:
+ // * We just checked that `prev` is non-null, so it must be in `self.list`.
+ // * `self.prev_ptr()` never returns `self.next`.
+ Some(CursorPeek {
+ ptr: prev,
+ cursor: self,
+ })
+ }
+
+ /// Move the cursor one element forward.
+ ///
+ /// If the cursor is after the last element, then this call does nothing. This call returns
+ /// `true` if the cursor's position was changed.
+ pub fn move_next(&mut self) -> bool {
+ if self.next.is_null() {
+ return false;
+ }
+
+ // SAFETY: `self.next` is an element in the list and we borrow the list mutably, so we can
+ // access the `next` field.
+ let mut next = unsafe { (*self.next).next };
if next == self.list.first {
- None
- } else {
- // INVARIANT: Since `self.current` is in the `list`, its `next` pointer is also in the
- // `list`.
- Some(Cursor {
- current: next,
- list: self.list,
- })
+ next = core::ptr::null_mut();
}
+
+ // INVARIANT: `next` is either null or the next element after an element in the list.
+ self.next = next;
+ true
}
- /// Move the cursor to the previous element.
- pub fn prev(self) -> Option<Cursor<'a, T, ID>> {
- // SAFETY: The `current` field is always in a list.
- let prev = unsafe { (*self.current).prev };
+ /// Move the cursor one element backwards.
+ ///
+ /// If the cursor is before the first element, then this call does nothing. This call returns
+ /// `true` if the cursor's position was changed.
+ pub fn move_prev(&mut self) -> bool {
+ if self.next == self.list.first {
+ return false;
+ }
- if self.current == self.list.first {
- None
+ // INVARIANT: `prev_ptr()` always returns a pointer that is null or in the list.
+ self.next = self.prev_ptr();
+ true
+ }
+
+ /// Inserts an element where the cursor is pointing and get a pointer to the new element.
+ fn insert_inner(&mut self, item: ListArc<T, ID>) -> *mut ListLinksFields {
+ let ptr = if self.next.is_null() {
+ self.list.first
} else {
- // INVARIANT: Since `self.current` is in the `list`, its `prev` pointer is also in the
- // `list`.
- Some(Cursor {
- current: prev,
- list: self.list,
- })
+ self.next
+ };
+ // SAFETY:
+ // * `ptr` is an element in the list or null.
+ // * if `ptr` is null, then `self.list.first` is null so the list is empty.
+ let item = unsafe { self.list.insert_inner(item, ptr) };
+ if self.next == self.list.first {
+ // INVARIANT: We just inserted `item`, so it's a member of list.
+ self.list.first = item;
}
+ item
+ }
+
+ /// Insert an element at this cursor's location.
+ pub fn insert(mut self, item: ListArc<T, ID>) {
+ // This is identical to `insert_prev`, but consumes the cursor. This is helpful because it
+ // reduces confusion when the last operation on the cursor is an insertion; in that case,
+ // you just want to insert the element at the cursor, and it is confusing that the call
+ // involves the word prev or next.
+ self.insert_inner(item);
}
- /// Remove the current element from the list.
+ /// Inserts an element after this cursor.
+ ///
+ /// After insertion, the new element will be after the cursor.
+ pub fn insert_next(&mut self, item: ListArc<T, ID>) {
+ self.next = self.insert_inner(item);
+ }
+
+ /// Inserts an element before this cursor.
+ ///
+ /// After insertion, the new element will be before the cursor.
+ pub fn insert_prev(&mut self, item: ListArc<T, ID>) {
+ self.insert_inner(item);
+ }
+
+ /// Remove the next element from the list.
+ pub fn remove_next(&mut self) -> Option<ListArc<T, ID>> {
+ self.peek_next().map(|v| v.remove())
+ }
+
+ /// Remove the previous element from the list.
+ pub fn remove_prev(&mut self) -> Option<ListArc<T, ID>> {
+ self.peek_prev().map(|v| v.remove())
+ }
+}
+
+/// References the element in the list next to the cursor.
+///
+/// # Invariants
+///
+/// * `ptr` is an element in `self.cursor.list`.
+/// * `ISNEXT == (self.ptr == self.cursor.next)`.
+pub struct CursorPeek<'a, 'b, T: ?Sized + ListItem<ID>, const ISNEXT: bool, const ID: u64> {
+ cursor: &'a mut Cursor<'b, T, ID>,
+ ptr: *mut ListLinksFields,
+}
+
+impl<'a, 'b, T: ?Sized + ListItem<ID>, const ISNEXT: bool, const ID: u64>
+ CursorPeek<'a, 'b, T, ISNEXT, ID>
+{
+ /// Remove the element from the list.
pub fn remove(self) -> ListArc<T, ID> {
- // SAFETY: The `current` pointer always points at a member of the list.
- unsafe { self.list.remove_internal(self.current) }
+ if ISNEXT {
+ self.cursor.move_next();
+ }
+
+ // INVARIANT: `self.ptr` is not equal to `self.cursor.next` due to the above `move_next`
+ // call.
+ // SAFETY: By the type invariants of `Self`, `next` is not null, so `next` is an element of
+ // `self.cursor.list` by the type invariants of `Cursor`.
+ unsafe { self.cursor.list.remove_internal(self.ptr) }
+ }
+
+ /// Access this value as an [`ArcBorrow`].
+ pub fn arc(&self) -> ArcBorrow<'_, T> {
+ // SAFETY: `self.ptr` points at an element in `self.cursor.list`.
+ let me = unsafe { T::view_value(ListLinks::from_fields(self.ptr)) };
+ // SAFETY:
+ // * All values in a list are stored in an `Arc`.
+ // * The value cannot be removed from the list for the duration of the lifetime annotated
+ // on the returned `ArcBorrow`, because removing it from the list would require mutable
+ // access to the `CursorPeek`, the `Cursor` or the `List`. However, the `ArcBorrow` holds
+ // an immutable borrow on the `CursorPeek`, which in turn holds a mutable borrow on the
+ // `Cursor`, which in turn holds a mutable borrow on the `List`, so any such mutable
+ // access requires first releasing the immutable borrow on the `CursorPeek`.
+ // * Values in a list never have a `UniqueArc` reference, because the list has a `ListArc`
+ // reference, and `UniqueArc` references must be unique.
+ unsafe { ArcBorrow::from_raw(me) }
+ }
+}
+
+impl<'a, 'b, T: ?Sized + ListItem<ID>, const ISNEXT: bool, const ID: u64> core::ops::Deref
+ for CursorPeek<'a, 'b, T, ISNEXT, ID>
+{
+ // If you change the `ptr` field to have type `ArcBorrow<'a, T>`, it might seem like you could
+ // get rid of the `CursorPeek::arc` method and change the deref target to `ArcBorrow<'a, T>`.
+ // However, that doesn't work because 'a is too long. You could obtain an `ArcBorrow<'a, T>`
+ // and then call `CursorPeek::remove` without giving up the `ArcBorrow<'a, T>`, which would be
+ // unsound.
+ type Target = T;
+
+ fn deref(&self) -> &T {
+ // SAFETY: `self.ptr` points at an element in `self.cursor.list`.
+ let me = unsafe { T::view_value(ListLinks::from_fields(self.ptr)) };
+
+ // SAFETY: The value cannot be removed from the list for the duration of the lifetime
+ // annotated on the returned `&T`, because removing it from the list would require mutable
+ // access to the `CursorPeek`, the `Cursor` or the `List`. However, the `&T` holds an
+ // immutable borrow on the `CursorPeek`, which in turn holds a mutable borrow on the
+ // `Cursor`, which in turn holds a mutable borrow on the `List`, so any such mutable access
+ // requires first releasing the immutable borrow on the `CursorPeek`.
+ unsafe { &*me }
}
}
diff --git a/rust/kernel/list/arc.rs b/rust/kernel/list/arc.rs
index 13c50df37b89..d92bcf665c89 100644
--- a/rust/kernel/list/arc.rs
+++ b/rust/kernel/list/arc.rs
@@ -74,7 +74,7 @@ pub unsafe trait TryNewListArc<const ID: u64 = 0>: ListArcSafe<ID> {
///
/// * The `untracked` strategy does not actually keep track of whether a [`ListArc`] exists. When
/// using this strategy, the only way to create a [`ListArc`] is using a [`UniqueArc`].
-/// * The `tracked_by` strategy defers the tracking to a field of the struct. The user much specify
+/// * The `tracked_by` strategy defers the tracking to a field of the struct. The user must specify
/// which field to defer the tracking to. The field must implement [`ListArcSafe`]. If the field
/// implements [`TryNewListArc`], then the type will also implement [`TryNewListArc`].
///
@@ -96,7 +96,7 @@ macro_rules! impl_list_arc_safe {
} $($rest:tt)*) => {
impl$(<$($generics)*>)? $crate::list::ListArcSafe<$num> for $t {
unsafe fn on_create_list_arc_from_unique(self: ::core::pin::Pin<&mut Self>) {
- $crate::assert_pinned!($t, $field, $fty, inline);
+ ::pin_init::assert_pinned!($t, $field, $fty, inline);
// SAFETY: This field is structurally pinned as per the above assertion.
let field = unsafe {
@@ -464,7 +464,7 @@ where
/// A utility for tracking whether a [`ListArc`] exists using an atomic.
///
-/// # Invariant
+/// # Invariants
///
/// If the boolean is `false`, then there is no [`ListArc`] for this value.
#[repr(transparent)]
diff --git a/rust/kernel/miscdevice.rs b/rust/kernel/miscdevice.rs
index e14433b2ab9d..939278bc7b03 100644
--- a/rust/kernel/miscdevice.rs
+++ b/rust/kernel/miscdevice.rs
@@ -14,6 +14,7 @@ use crate::{
error::{to_result, Error, Result, VTABLE_DEFAULT_ERROR},
ffi::{c_int, c_long, c_uint, c_ulong},
fs::File,
+ mm::virt::VmaNew,
prelude::*,
seq_file::SeqFile,
str::CStr,
@@ -35,7 +36,7 @@ impl MiscDeviceOptions {
let mut result: bindings::miscdevice = unsafe { MaybeUninit::zeroed().assume_init() };
result.minor = bindings::MISC_DYNAMIC_MINOR as _;
result.name = self.name.as_char_ptr();
- result.fops = create_vtable::<T>();
+ result.fops = MiscdeviceVTable::<T>::build();
result
}
}
@@ -119,9 +120,25 @@ pub trait MiscDevice: Sized {
drop(device);
}
+ /// Handle for mmap.
+ ///
+ /// This function is invoked when a user space process invokes the `mmap` system call on
+ /// `file`. The function is a callback that is part of the VMA initializer. The kernel will do
+ /// initial setup of the VMA before calling this function. The function can then interact with
+ /// the VMA initialization by calling methods of `vma`. If the function does not return an
+ /// error, the kernel will complete initialization of the VMA according to the properties of
+ /// `vma`.
+ fn mmap(
+ _device: <Self::Ptr as ForeignOwnable>::Borrowed<'_>,
+ _file: &File,
+ _vma: &VmaNew,
+ ) -> Result {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
/// Handler for ioctls.
///
- /// The `cmd` argument is usually manipulated using the utilties in [`kernel::ioctl`].
+ /// The `cmd` argument is usually manipulated using the utilities in [`kernel::ioctl`].
///
/// [`kernel::ioctl`]: mod@crate::ioctl
fn ioctl(
@@ -160,171 +177,188 @@ pub trait MiscDevice: Sized {
}
}
-const fn create_vtable<T: MiscDevice>() -> &'static bindings::file_operations {
- const fn maybe_fn<T: Copy>(check: bool, func: T) -> Option<T> {
- if check {
- Some(func)
- } else {
- None
- }
- }
+/// A vtable for the file operations of a Rust miscdevice.
+struct MiscdeviceVTable<T: MiscDevice>(PhantomData<T>);
- struct VtableHelper<T: MiscDevice> {
- _t: PhantomData<T>,
- }
- impl<T: MiscDevice> VtableHelper<T> {
- const VTABLE: bindings::file_operations = bindings::file_operations {
- open: Some(fops_open::<T>),
- release: Some(fops_release::<T>),
- unlocked_ioctl: maybe_fn(T::HAS_IOCTL, fops_ioctl::<T>),
- #[cfg(CONFIG_COMPAT)]
- compat_ioctl: if T::HAS_COMPAT_IOCTL {
- Some(fops_compat_ioctl::<T>)
- } else if T::HAS_IOCTL {
- Some(bindings::compat_ptr_ioctl)
- } else {
- None
- },
- show_fdinfo: maybe_fn(T::HAS_SHOW_FDINFO, fops_show_fdinfo::<T>),
- // SAFETY: All zeros is a valid value for `bindings::file_operations`.
- ..unsafe { MaybeUninit::zeroed().assume_init() }
- };
- }
+impl<T: MiscDevice> MiscdeviceVTable<T> {
+ /// # Safety
+ ///
+ /// `file` and `inode` must be the file and inode for a file that is undergoing initialization.
+ /// The file must be associated with a `MiscDeviceRegistration<T>`.
+ unsafe extern "C" fn open(inode: *mut bindings::inode, raw_file: *mut bindings::file) -> c_int {
+ // SAFETY: The pointers are valid and for a file being opened.
+ let ret = unsafe { bindings::generic_file_open(inode, raw_file) };
+ if ret != 0 {
+ return ret;
+ }
- &VtableHelper::<T>::VTABLE
-}
+ // SAFETY: The open call of a file can access the private data.
+ let misc_ptr = unsafe { (*raw_file).private_data };
-/// # Safety
-///
-/// `file` and `inode` must be the file and inode for a file that is undergoing initialization.
-/// The file must be associated with a `MiscDeviceRegistration<T>`.
-unsafe extern "C" fn fops_open<T: MiscDevice>(
- inode: *mut bindings::inode,
- raw_file: *mut bindings::file,
-) -> c_int {
- // SAFETY: The pointers are valid and for a file being opened.
- let ret = unsafe { bindings::generic_file_open(inode, raw_file) };
- if ret != 0 {
- return ret;
- }
+ // SAFETY: This is a miscdevice, so `misc_open()` set the private data to a pointer to the
+ // associated `struct miscdevice` before calling into this method. Furthermore,
+ // `misc_open()` ensures that the miscdevice can't be unregistered and freed during this
+ // call to `fops_open`.
+ let misc = unsafe { &*misc_ptr.cast::<MiscDeviceRegistration<T>>() };
- // SAFETY: The open call of a file can access the private data.
- let misc_ptr = unsafe { (*raw_file).private_data };
+ // SAFETY:
+ // * This underlying file is valid for (much longer than) the duration of `T::open`.
+ // * There is no active fdget_pos region on the file on this thread.
+ let file = unsafe { File::from_raw_file(raw_file) };
- // SAFETY: This is a miscdevice, so `misc_open()` set the private data to a pointer to the
- // associated `struct miscdevice` before calling into this method. Furthermore, `misc_open()`
- // ensures that the miscdevice can't be unregistered and freed during this call to `fops_open`.
- let misc = unsafe { &*misc_ptr.cast::<MiscDeviceRegistration<T>>() };
+ let ptr = match T::open(file, misc) {
+ Ok(ptr) => ptr,
+ Err(err) => return err.to_errno(),
+ };
- // SAFETY:
- // * This underlying file is valid for (much longer than) the duration of `T::open`.
- // * There is no active fdget_pos region on the file on this thread.
- let file = unsafe { File::from_raw_file(raw_file) };
+ // This overwrites the private data with the value specified by the user, changing the type
+ // of this file's private data. All future accesses to the private data is performed by
+ // other fops_* methods in this file, which all correctly cast the private data to the new
+ // type.
+ //
+ // SAFETY: The open call of a file can access the private data.
+ unsafe { (*raw_file).private_data = ptr.into_foreign().cast() };
- let ptr = match T::open(file, misc) {
- Ok(ptr) => ptr,
- Err(err) => return err.to_errno(),
- };
+ 0
+ }
- // This overwrites the private data with the value specified by the user, changing the type of
- // this file's private data. All future accesses to the private data is performed by other
- // fops_* methods in this file, which all correctly cast the private data to the new type.
- //
- // SAFETY: The open call of a file can access the private data.
- unsafe { (*raw_file).private_data = ptr.into_foreign() };
+ /// # Safety
+ ///
+ /// `file` and `inode` must be the file and inode for a file that is being released. The file
+ /// must be associated with a `MiscDeviceRegistration<T>`.
+ unsafe extern "C" fn release(_inode: *mut bindings::inode, file: *mut bindings::file) -> c_int {
+ // SAFETY: The release call of a file owns the private data.
+ let private = unsafe { (*file).private_data }.cast();
+ // SAFETY: The release call of a file owns the private data.
+ let ptr = unsafe { <T::Ptr as ForeignOwnable>::from_foreign(private) };
+
+ // SAFETY:
+ // * The file is valid for the duration of this call.
+ // * There is no active fdget_pos region on the file on this thread.
+ T::release(ptr, unsafe { File::from_raw_file(file) });
+
+ 0
+ }
- 0
-}
+ /// # Safety
+ ///
+ /// `file` must be a valid file that is associated with a `MiscDeviceRegistration<T>`.
+ /// `vma` must be a vma that is currently being mmap'ed with this file.
+ unsafe extern "C" fn mmap(
+ file: *mut bindings::file,
+ vma: *mut bindings::vm_area_struct,
+ ) -> c_int {
+ // SAFETY: The mmap call of a file can access the private data.
+ let private = unsafe { (*file).private_data };
+ // SAFETY: This is a Rust Miscdevice, so we call `into_foreign` in `open` and
+ // `from_foreign` in `release`, and `fops_mmap` is guaranteed to be called between those
+ // two operations.
+ let device = unsafe { <T::Ptr as ForeignOwnable>::borrow(private.cast()) };
+ // SAFETY: The caller provides a vma that is undergoing initial VMA setup.
+ let area = unsafe { VmaNew::from_raw(vma) };
+ // SAFETY:
+ // * The file is valid for the duration of this call.
+ // * There is no active fdget_pos region on the file on this thread.
+ let file = unsafe { File::from_raw_file(file) };
+
+ match T::mmap(device, file, area) {
+ Ok(()) => 0,
+ Err(err) => err.to_errno(),
+ }
+ }
-/// # Safety
-///
-/// `file` and `inode` must be the file and inode for a file that is being released. The file must
-/// be associated with a `MiscDeviceRegistration<T>`.
-unsafe extern "C" fn fops_release<T: MiscDevice>(
- _inode: *mut bindings::inode,
- file: *mut bindings::file,
-) -> c_int {
- // SAFETY: The release call of a file owns the private data.
- let private = unsafe { (*file).private_data };
- // SAFETY: The release call of a file owns the private data.
- let ptr = unsafe { <T::Ptr as ForeignOwnable>::from_foreign(private) };
-
- // SAFETY:
- // * The file is valid for the duration of this call.
- // * There is no active fdget_pos region on the file on this thread.
- T::release(ptr, unsafe { File::from_raw_file(file) });
-
- 0
-}
+ /// # Safety
+ ///
+ /// `file` must be a valid file that is associated with a `MiscDeviceRegistration<T>`.
+ unsafe extern "C" fn ioctl(file: *mut bindings::file, cmd: c_uint, arg: c_ulong) -> c_long {
+ // SAFETY: The ioctl call of a file can access the private data.
+ let private = unsafe { (*file).private_data }.cast();
+ // SAFETY: Ioctl calls can borrow the private data of the file.
+ let device = unsafe { <T::Ptr as ForeignOwnable>::borrow(private) };
+
+ // SAFETY:
+ // * The file is valid for the duration of this call.
+ // * There is no active fdget_pos region on the file on this thread.
+ let file = unsafe { File::from_raw_file(file) };
+
+ match T::ioctl(device, file, cmd, arg) {
+ Ok(ret) => ret as c_long,
+ Err(err) => err.to_errno() as c_long,
+ }
+ }
-/// # Safety
-///
-/// `file` must be a valid file that is associated with a `MiscDeviceRegistration<T>`.
-unsafe extern "C" fn fops_ioctl<T: MiscDevice>(
- file: *mut bindings::file,
- cmd: c_uint,
- arg: c_ulong,
-) -> c_long {
- // SAFETY: The ioctl call of a file can access the private data.
- let private = unsafe { (*file).private_data };
- // SAFETY: Ioctl calls can borrow the private data of the file.
- let device = unsafe { <T::Ptr as ForeignOwnable>::borrow(private) };
-
- // SAFETY:
- // * The file is valid for the duration of this call.
- // * There is no active fdget_pos region on the file on this thread.
- let file = unsafe { File::from_raw_file(file) };
-
- match T::ioctl(device, file, cmd, arg) {
- Ok(ret) => ret as c_long,
- Err(err) => err.to_errno() as c_long,
+ /// # Safety
+ ///
+ /// `file` must be a valid file that is associated with a `MiscDeviceRegistration<T>`.
+ #[cfg(CONFIG_COMPAT)]
+ unsafe extern "C" fn compat_ioctl(
+ file: *mut bindings::file,
+ cmd: c_uint,
+ arg: c_ulong,
+ ) -> c_long {
+ // SAFETY: The compat ioctl call of a file can access the private data.
+ let private = unsafe { (*file).private_data }.cast();
+ // SAFETY: Ioctl calls can borrow the private data of the file.
+ let device = unsafe { <T::Ptr as ForeignOwnable>::borrow(private) };
+
+ // SAFETY:
+ // * The file is valid for the duration of this call.
+ // * There is no active fdget_pos region on the file on this thread.
+ let file = unsafe { File::from_raw_file(file) };
+
+ match T::compat_ioctl(device, file, cmd, arg) {
+ Ok(ret) => ret as c_long,
+ Err(err) => err.to_errno() as c_long,
+ }
}
-}
-/// # Safety
-///
-/// `file` must be a valid file that is associated with a `MiscDeviceRegistration<T>`.
-#[cfg(CONFIG_COMPAT)]
-unsafe extern "C" fn fops_compat_ioctl<T: MiscDevice>(
- file: *mut bindings::file,
- cmd: c_uint,
- arg: c_ulong,
-) -> c_long {
- // SAFETY: The compat ioctl call of a file can access the private data.
- let private = unsafe { (*file).private_data };
- // SAFETY: Ioctl calls can borrow the private data of the file.
- let device = unsafe { <T::Ptr as ForeignOwnable>::borrow(private) };
-
- // SAFETY:
- // * The file is valid for the duration of this call.
- // * There is no active fdget_pos region on the file on this thread.
- let file = unsafe { File::from_raw_file(file) };
-
- match T::compat_ioctl(device, file, cmd, arg) {
- Ok(ret) => ret as c_long,
- Err(err) => err.to_errno() as c_long,
+ /// # Safety
+ ///
+ /// - `file` must be a valid file that is associated with a `MiscDeviceRegistration<T>`.
+ /// - `seq_file` must be a valid `struct seq_file` that we can write to.
+ unsafe extern "C" fn show_fdinfo(seq_file: *mut bindings::seq_file, file: *mut bindings::file) {
+ // SAFETY: The release call of a file owns the private data.
+ let private = unsafe { (*file).private_data }.cast();
+ // SAFETY: Ioctl calls can borrow the private data of the file.
+ let device = unsafe { <T::Ptr as ForeignOwnable>::borrow(private) };
+ // SAFETY:
+ // * The file is valid for the duration of this call.
+ // * There is no active fdget_pos region on the file on this thread.
+ let file = unsafe { File::from_raw_file(file) };
+ // SAFETY: The caller ensures that the pointer is valid and exclusive for the duration in
+ // which this method is called.
+ let m = unsafe { SeqFile::from_raw(seq_file) };
+
+ T::show_fdinfo(device, m, file);
}
-}
-/// # Safety
-///
-/// - `file` must be a valid file that is associated with a `MiscDeviceRegistration<T>`.
-/// - `seq_file` must be a valid `struct seq_file` that we can write to.
-unsafe extern "C" fn fops_show_fdinfo<T: MiscDevice>(
- seq_file: *mut bindings::seq_file,
- file: *mut bindings::file,
-) {
- // SAFETY: The release call of a file owns the private data.
- let private = unsafe { (*file).private_data };
- // SAFETY: Ioctl calls can borrow the private data of the file.
- let device = unsafe { <T::Ptr as ForeignOwnable>::borrow(private) };
- // SAFETY:
- // * The file is valid for the duration of this call.
- // * There is no active fdget_pos region on the file on this thread.
- let file = unsafe { File::from_raw_file(file) };
- // SAFETY: The caller ensures that the pointer is valid and exclusive for the duration in which
- // this method is called.
- let m = unsafe { SeqFile::from_raw(seq_file) };
-
- T::show_fdinfo(device, m, file);
+ const VTABLE: bindings::file_operations = bindings::file_operations {
+ open: Some(Self::open),
+ release: Some(Self::release),
+ mmap: if T::HAS_MMAP { Some(Self::mmap) } else { None },
+ unlocked_ioctl: if T::HAS_IOCTL {
+ Some(Self::ioctl)
+ } else {
+ None
+ },
+ #[cfg(CONFIG_COMPAT)]
+ compat_ioctl: if T::HAS_COMPAT_IOCTL {
+ Some(Self::compat_ioctl)
+ } else if T::HAS_IOCTL {
+ Some(bindings::compat_ptr_ioctl)
+ } else {
+ None
+ },
+ show_fdinfo: if T::HAS_SHOW_FDINFO {
+ Some(Self::show_fdinfo)
+ } else {
+ None
+ },
+ // SAFETY: All zeros is a valid value for `bindings::file_operations`.
+ ..unsafe { MaybeUninit::zeroed().assume_init() }
+ };
+
+ const fn build() -> &'static bindings::file_operations {
+ &Self::VTABLE
+ }
}
diff --git a/rust/kernel/mm.rs b/rust/kernel/mm.rs
new file mode 100644
index 000000000000..43f525c0d16c
--- /dev/null
+++ b/rust/kernel/mm.rs
@@ -0,0 +1,296 @@
+// SPDX-License-Identifier: GPL-2.0
+
+// Copyright (C) 2024 Google LLC.
+
+//! Memory management.
+//!
+//! This module deals with managing the address space of userspace processes. Each process has an
+//! instance of [`Mm`], which keeps track of multiple VMAs (virtual memory areas). Each VMA
+//! corresponds to a region of memory that the userspace process can access, and the VMA lets you
+//! control what happens when userspace reads or writes to that region of memory.
+//!
+//! C header: [`include/linux/mm.h`](srctree/include/linux/mm.h)
+
+use crate::{
+ bindings,
+ types::{ARef, AlwaysRefCounted, NotThreadSafe, Opaque},
+};
+use core::{ops::Deref, ptr::NonNull};
+
+pub mod virt;
+use virt::VmaRef;
+
+#[cfg(CONFIG_MMU)]
+pub use mmput_async::MmWithUserAsync;
+mod mmput_async;
+
+/// A wrapper for the kernel's `struct mm_struct`.
+///
+/// This represents the address space of a userspace process, so each process has one `Mm`
+/// instance. It may hold many VMAs internally.
+///
+/// There is a counter called `mm_users` that counts the users of the address space; this includes
+/// the userspace process itself, but can also include kernel threads accessing the address space.
+/// Once `mm_users` reaches zero, this indicates that the address space can be destroyed. To access
+/// the address space, you must prevent `mm_users` from reaching zero while you are accessing it.
+/// The [`MmWithUser`] type represents an address space where this is guaranteed, and you can
+/// create one using [`mmget_not_zero`].
+///
+/// The `ARef<Mm>` smart pointer holds an `mmgrab` refcount. Its destructor may sleep.
+///
+/// # Invariants
+///
+/// Values of this type are always refcounted using `mmgrab`.
+///
+/// [`mmget_not_zero`]: Mm::mmget_not_zero
+#[repr(transparent)]
+pub struct Mm {
+ mm: Opaque<bindings::mm_struct>,
+}
+
+// SAFETY: It is safe to call `mmdrop` on another thread than where `mmgrab` was called.
+unsafe impl Send for Mm {}
+// SAFETY: All methods on `Mm` can be called in parallel from several threads.
+unsafe impl Sync for Mm {}
+
+// SAFETY: By the type invariants, this type is always refcounted.
+unsafe impl AlwaysRefCounted for Mm {
+ #[inline]
+ fn inc_ref(&self) {
+ // SAFETY: The pointer is valid since self is a reference.
+ unsafe { bindings::mmgrab(self.as_raw()) };
+ }
+
+ #[inline]
+ unsafe fn dec_ref(obj: NonNull<Self>) {
+ // SAFETY: The caller is giving up their refcount.
+ unsafe { bindings::mmdrop(obj.cast().as_ptr()) };
+ }
+}
+
+/// A wrapper for the kernel's `struct mm_struct`.
+///
+/// This type is like [`Mm`], but with non-zero `mm_users`. It can only be used when `mm_users` can
+/// be proven to be non-zero at compile-time, usually because the relevant code holds an `mmget`
+/// refcount. It can be used to access the associated address space.
+///
+/// The `ARef<MmWithUser>` smart pointer holds an `mmget` refcount. Its destructor may sleep.
+///
+/// # Invariants
+///
+/// Values of this type are always refcounted using `mmget`. The value of `mm_users` is non-zero.
+#[repr(transparent)]
+pub struct MmWithUser {
+ mm: Mm,
+}
+
+// SAFETY: It is safe to call `mmput` on another thread than where `mmget` was called.
+unsafe impl Send for MmWithUser {}
+// SAFETY: All methods on `MmWithUser` can be called in parallel from several threads.
+unsafe impl Sync for MmWithUser {}
+
+// SAFETY: By the type invariants, this type is always refcounted.
+unsafe impl AlwaysRefCounted for MmWithUser {
+ #[inline]
+ fn inc_ref(&self) {
+ // SAFETY: The pointer is valid since self is a reference.
+ unsafe { bindings::mmget(self.as_raw()) };
+ }
+
+ #[inline]
+ unsafe fn dec_ref(obj: NonNull<Self>) {
+ // SAFETY: The caller is giving up their refcount.
+ unsafe { bindings::mmput(obj.cast().as_ptr()) };
+ }
+}
+
+// Make all `Mm` methods available on `MmWithUser`.
+impl Deref for MmWithUser {
+ type Target = Mm;
+
+ #[inline]
+ fn deref(&self) -> &Mm {
+ &self.mm
+ }
+}
+
+// These methods are safe to call even if `mm_users` is zero.
+impl Mm {
+ /// Returns a raw pointer to the inner `mm_struct`.
+ #[inline]
+ pub fn as_raw(&self) -> *mut bindings::mm_struct {
+ self.mm.get()
+ }
+
+ /// Obtain a reference from a raw pointer.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `ptr` points at an `mm_struct`, and that it is not deallocated
+ /// during the lifetime 'a.
+ #[inline]
+ pub unsafe fn from_raw<'a>(ptr: *const bindings::mm_struct) -> &'a Mm {
+ // SAFETY: Caller promises that the pointer is valid for 'a. Layouts are compatible due to
+ // repr(transparent).
+ unsafe { &*ptr.cast() }
+ }
+
+ /// Calls `mmget_not_zero` and returns a handle if it succeeds.
+ #[inline]
+ pub fn mmget_not_zero(&self) -> Option<ARef<MmWithUser>> {
+ // SAFETY: The pointer is valid since self is a reference.
+ let success = unsafe { bindings::mmget_not_zero(self.as_raw()) };
+
+ if success {
+ // SAFETY: We just created an `mmget` refcount.
+ Some(unsafe { ARef::from_raw(NonNull::new_unchecked(self.as_raw().cast())) })
+ } else {
+ None
+ }
+ }
+}
+
+// These methods require `mm_users` to be non-zero.
+impl MmWithUser {
+ /// Obtain a reference from a raw pointer.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `ptr` points at an `mm_struct`, and that `mm_users` remains
+ /// non-zero for the duration of the lifetime 'a.
+ #[inline]
+ pub unsafe fn from_raw<'a>(ptr: *const bindings::mm_struct) -> &'a MmWithUser {
+ // SAFETY: Caller promises that the pointer is valid for 'a. The layout is compatible due
+ // to repr(transparent).
+ unsafe { &*ptr.cast() }
+ }
+
+ /// Attempt to access a vma using the vma read lock.
+ ///
+ /// This is an optimistic trylock operation, so it may fail if there is contention. In that
+ /// case, you should fall back to taking the mmap read lock.
+ ///
+ /// When per-vma locks are disabled, this always returns `None`.
+ #[inline]
+ pub fn lock_vma_under_rcu(&self, vma_addr: usize) -> Option<VmaReadGuard<'_>> {
+ #[cfg(CONFIG_PER_VMA_LOCK)]
+ {
+ // SAFETY: Calling `bindings::lock_vma_under_rcu` is always okay given an mm where
+ // `mm_users` is non-zero.
+ let vma = unsafe { bindings::lock_vma_under_rcu(self.as_raw(), vma_addr) };
+ if !vma.is_null() {
+ return Some(VmaReadGuard {
+ // SAFETY: If `lock_vma_under_rcu` returns a non-null ptr, then it points at a
+ // valid vma. The vma is stable for as long as the vma read lock is held.
+ vma: unsafe { VmaRef::from_raw(vma) },
+ _nts: NotThreadSafe,
+ });
+ }
+ }
+
+ // Silence warnings about unused variables.
+ #[cfg(not(CONFIG_PER_VMA_LOCK))]
+ let _ = vma_addr;
+
+ None
+ }
+
+ /// Lock the mmap read lock.
+ #[inline]
+ pub fn mmap_read_lock(&self) -> MmapReadGuard<'_> {
+ // SAFETY: The pointer is valid since self is a reference.
+ unsafe { bindings::mmap_read_lock(self.as_raw()) };
+
+ // INVARIANT: We just acquired the read lock.
+ MmapReadGuard {
+ mm: self,
+ _nts: NotThreadSafe,
+ }
+ }
+
+ /// Try to lock the mmap read lock.
+ #[inline]
+ pub fn mmap_read_trylock(&self) -> Option<MmapReadGuard<'_>> {
+ // SAFETY: The pointer is valid since self is a reference.
+ let success = unsafe { bindings::mmap_read_trylock(self.as_raw()) };
+
+ if success {
+ // INVARIANT: We just acquired the read lock.
+ Some(MmapReadGuard {
+ mm: self,
+ _nts: NotThreadSafe,
+ })
+ } else {
+ None
+ }
+ }
+}
+
+/// A guard for the mmap read lock.
+///
+/// # Invariants
+///
+/// This `MmapReadGuard` guard owns the mmap read lock.
+pub struct MmapReadGuard<'a> {
+ mm: &'a MmWithUser,
+ // `mmap_read_lock` and `mmap_read_unlock` must be called on the same thread
+ _nts: NotThreadSafe,
+}
+
+impl<'a> MmapReadGuard<'a> {
+ /// Look up a vma at the given address.
+ #[inline]
+ pub fn vma_lookup(&self, vma_addr: usize) -> Option<&virt::VmaRef> {
+ // SAFETY: By the type invariants we hold the mmap read guard, so we can safely call this
+ // method. Any value is okay for `vma_addr`.
+ let vma = unsafe { bindings::vma_lookup(self.mm.as_raw(), vma_addr) };
+
+ if vma.is_null() {
+ None
+ } else {
+ // SAFETY: We just checked that a vma was found, so the pointer references a valid vma.
+ //
+ // Furthermore, the returned vma is still under the protection of the read lock guard
+ // and can be used while the mmap read lock is still held. That the vma is not used
+ // after the MmapReadGuard gets dropped is enforced by the borrow-checker.
+ unsafe { Some(virt::VmaRef::from_raw(vma)) }
+ }
+ }
+}
+
+impl Drop for MmapReadGuard<'_> {
+ #[inline]
+ fn drop(&mut self) {
+ // SAFETY: We hold the read lock by the type invariants.
+ unsafe { bindings::mmap_read_unlock(self.mm.as_raw()) };
+ }
+}
+
+/// A guard for the vma read lock.
+///
+/// # Invariants
+///
+/// This `VmaReadGuard` guard owns the vma read lock.
+pub struct VmaReadGuard<'a> {
+ vma: &'a VmaRef,
+ // `vma_end_read` must be called on the same thread as where the lock was taken
+ _nts: NotThreadSafe,
+}
+
+// Make all `VmaRef` methods available on `VmaReadGuard`.
+impl Deref for VmaReadGuard<'_> {
+ type Target = VmaRef;
+
+ #[inline]
+ fn deref(&self) -> &VmaRef {
+ self.vma
+ }
+}
+
+impl Drop for VmaReadGuard<'_> {
+ #[inline]
+ fn drop(&mut self) {
+ // SAFETY: We hold the read lock by the type invariants.
+ unsafe { bindings::vma_end_read(self.vma.as_ptr()) };
+ }
+}
diff --git a/rust/kernel/mm/mmput_async.rs b/rust/kernel/mm/mmput_async.rs
new file mode 100644
index 000000000000..9289e05f7a67
--- /dev/null
+++ b/rust/kernel/mm/mmput_async.rs
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: GPL-2.0
+
+// Copyright (C) 2024 Google LLC.
+
+//! Version of `MmWithUser` using `mmput_async`.
+//!
+//! This is a separate file from `mm.rs` due to the dependency on `CONFIG_MMU=y`.
+#![cfg(CONFIG_MMU)]
+
+use crate::{
+ bindings,
+ mm::MmWithUser,
+ types::{ARef, AlwaysRefCounted},
+};
+use core::{ops::Deref, ptr::NonNull};
+
+/// A wrapper for the kernel's `struct mm_struct`.
+///
+/// This type is identical to `MmWithUser` except that it uses `mmput_async` when dropping a
+/// refcount. This means that the destructor of `ARef<MmWithUserAsync>` is safe to call in atomic
+/// context.
+///
+/// # Invariants
+///
+/// Values of this type are always refcounted using `mmget`. The value of `mm_users` is non-zero.
+#[repr(transparent)]
+pub struct MmWithUserAsync {
+ mm: MmWithUser,
+}
+
+// SAFETY: It is safe to call `mmput_async` on another thread than where `mmget` was called.
+unsafe impl Send for MmWithUserAsync {}
+// SAFETY: All methods on `MmWithUserAsync` can be called in parallel from several threads.
+unsafe impl Sync for MmWithUserAsync {}
+
+// SAFETY: By the type invariants, this type is always refcounted.
+unsafe impl AlwaysRefCounted for MmWithUserAsync {
+ #[inline]
+ fn inc_ref(&self) {
+ // SAFETY: The pointer is valid since self is a reference.
+ unsafe { bindings::mmget(self.as_raw()) };
+ }
+
+ #[inline]
+ unsafe fn dec_ref(obj: NonNull<Self>) {
+ // SAFETY: The caller is giving up their refcount.
+ unsafe { bindings::mmput_async(obj.cast().as_ptr()) };
+ }
+}
+
+// Make all `MmWithUser` methods available on `MmWithUserAsync`.
+impl Deref for MmWithUserAsync {
+ type Target = MmWithUser;
+
+ #[inline]
+ fn deref(&self) -> &MmWithUser {
+ &self.mm
+ }
+}
+
+impl MmWithUser {
+ /// Use `mmput_async` when dropping this refcount.
+ #[inline]
+ pub fn into_mmput_async(me: ARef<MmWithUser>) -> ARef<MmWithUserAsync> {
+ // SAFETY: The layouts and invariants are compatible.
+ unsafe { ARef::from_raw(ARef::into_raw(me).cast()) }
+ }
+}
diff --git a/rust/kernel/mm/virt.rs b/rust/kernel/mm/virt.rs
new file mode 100644
index 000000000000..31803674aecc
--- /dev/null
+++ b/rust/kernel/mm/virt.rs
@@ -0,0 +1,471 @@
+// SPDX-License-Identifier: GPL-2.0
+
+// Copyright (C) 2024 Google LLC.
+
+//! Virtual memory.
+//!
+//! This module deals with managing a single VMA in the address space of a userspace process. Each
+//! VMA corresponds to a region of memory that the userspace process can access, and the VMA lets
+//! you control what happens when userspace reads or writes to that region of memory.
+//!
+//! The module has several different Rust types that all correspond to the C type called
+//! `vm_area_struct`. The different structs represent what kind of access you have to the VMA, e.g.
+//! [`VmaRef`] is used when you hold the mmap or vma read lock. Using the appropriate struct
+//! ensures that you can't, for example, accidentally call a function that requires holding the
+//! write lock when you only hold the read lock.
+
+use crate::{
+ bindings,
+ error::{code::EINVAL, to_result, Result},
+ mm::MmWithUser,
+ page::Page,
+ types::Opaque,
+};
+
+use core::ops::Deref;
+
+/// A wrapper for the kernel's `struct vm_area_struct` with read access.
+///
+/// It represents an area of virtual memory.
+///
+/// # Invariants
+///
+/// The caller must hold the mmap read lock or the vma read lock.
+#[repr(transparent)]
+pub struct VmaRef {
+ vma: Opaque<bindings::vm_area_struct>,
+}
+
+// Methods you can call when holding the mmap or vma read lock (or stronger). They must be usable
+// no matter what the vma flags are.
+impl VmaRef {
+ /// Access a virtual memory area given a raw pointer.
+ ///
+ /// # Safety
+ ///
+ /// Callers must ensure that `vma` is valid for the duration of 'a, and that the mmap or vma
+ /// read lock (or stronger) is held for at least the duration of 'a.
+ #[inline]
+ pub unsafe fn from_raw<'a>(vma: *const bindings::vm_area_struct) -> &'a Self {
+ // SAFETY: The caller ensures that the invariants are satisfied for the duration of 'a.
+ unsafe { &*vma.cast() }
+ }
+
+ /// Returns a raw pointer to this area.
+ #[inline]
+ pub fn as_ptr(&self) -> *mut bindings::vm_area_struct {
+ self.vma.get()
+ }
+
+ /// Access the underlying `mm_struct`.
+ #[inline]
+ pub fn mm(&self) -> &MmWithUser {
+ // SAFETY: By the type invariants, this `vm_area_struct` is valid and we hold the mmap/vma
+ // read lock or stronger. This implies that the underlying mm has a non-zero value of
+ // `mm_users`.
+ unsafe { MmWithUser::from_raw((*self.as_ptr()).vm_mm) }
+ }
+
+ /// Returns the flags associated with the virtual memory area.
+ ///
+ /// The possible flags are a combination of the constants in [`flags`].
+ #[inline]
+ pub fn flags(&self) -> vm_flags_t {
+ // SAFETY: By the type invariants, the caller holds at least the mmap read lock, so this
+ // access is not a data race.
+ unsafe { (*self.as_ptr()).__bindgen_anon_2.vm_flags }
+ }
+
+ /// Returns the (inclusive) start address of the virtual memory area.
+ #[inline]
+ pub fn start(&self) -> usize {
+ // SAFETY: By the type invariants, the caller holds at least the mmap read lock, so this
+ // access is not a data race.
+ unsafe { (*self.as_ptr()).__bindgen_anon_1.__bindgen_anon_1.vm_start }
+ }
+
+ /// Returns the (exclusive) end address of the virtual memory area.
+ #[inline]
+ pub fn end(&self) -> usize {
+ // SAFETY: By the type invariants, the caller holds at least the mmap read lock, so this
+ // access is not a data race.
+ unsafe { (*self.as_ptr()).__bindgen_anon_1.__bindgen_anon_1.vm_end }
+ }
+
+ /// Zap pages in the given page range.
+ ///
+ /// This clears page table mappings for the range at the leaf level, leaving all other page
+ /// tables intact, and freeing any memory referenced by the VMA in this range. That is,
+ /// anonymous memory is completely freed, file-backed memory has its reference count on page
+ /// cache folio's dropped, any dirty data will still be written back to disk as usual.
+ ///
+ /// It may seem odd that we clear at the leaf level, this is however a product of the page
+ /// table structure used to map physical memory into a virtual address space - each virtual
+ /// address actually consists of a bitmap of array indices into page tables, which form a
+ /// hierarchical page table level structure.
+ ///
+ /// As a result, each page table level maps a multiple of page table levels below, and thus
+ /// span ever increasing ranges of pages. At the leaf or PTE level, we map the actual physical
+ /// memory.
+ ///
+ /// It is here where a zap operates, as it the only place we can be certain of clearing without
+ /// impacting any other virtual mappings. It is an implementation detail as to whether the
+ /// kernel goes further in freeing unused page tables, but for the purposes of this operation
+ /// we must only assume that the leaf level is cleared.
+ #[inline]
+ pub fn zap_page_range_single(&self, address: usize, size: usize) {
+ let (end, did_overflow) = address.overflowing_add(size);
+ if did_overflow || address < self.start() || self.end() < end {
+ // TODO: call WARN_ONCE once Rust version of it is added
+ return;
+ }
+
+ // SAFETY: By the type invariants, the caller has read access to this VMA, which is
+ // sufficient for this method call. This method has no requirements on the vma flags. The
+ // address range is checked to be within the vma.
+ unsafe {
+ bindings::zap_page_range_single(self.as_ptr(), address, size, core::ptr::null_mut())
+ };
+ }
+
+ /// If the [`VM_MIXEDMAP`] flag is set, returns a [`VmaMixedMap`] to this VMA, otherwise
+ /// returns `None`.
+ ///
+ /// This can be used to access methods that require [`VM_MIXEDMAP`] to be set.
+ ///
+ /// [`VM_MIXEDMAP`]: flags::MIXEDMAP
+ #[inline]
+ pub fn as_mixedmap_vma(&self) -> Option<&VmaMixedMap> {
+ if self.flags() & flags::MIXEDMAP != 0 {
+ // SAFETY: We just checked that `VM_MIXEDMAP` is set. All other requirements are
+ // satisfied by the type invariants of `VmaRef`.
+ Some(unsafe { VmaMixedMap::from_raw(self.as_ptr()) })
+ } else {
+ None
+ }
+ }
+}
+
+/// A wrapper for the kernel's `struct vm_area_struct` with read access and [`VM_MIXEDMAP`] set.
+///
+/// It represents an area of virtual memory.
+///
+/// This struct is identical to [`VmaRef`] except that it must only be used when the
+/// [`VM_MIXEDMAP`] flag is set on the vma.
+///
+/// # Invariants
+///
+/// The caller must hold the mmap read lock or the vma read lock. The `VM_MIXEDMAP` flag must be
+/// set.
+///
+/// [`VM_MIXEDMAP`]: flags::MIXEDMAP
+#[repr(transparent)]
+pub struct VmaMixedMap {
+ vma: VmaRef,
+}
+
+// Make all `VmaRef` methods available on `VmaMixedMap`.
+impl Deref for VmaMixedMap {
+ type Target = VmaRef;
+
+ #[inline]
+ fn deref(&self) -> &VmaRef {
+ &self.vma
+ }
+}
+
+impl VmaMixedMap {
+ /// Access a virtual memory area given a raw pointer.
+ ///
+ /// # Safety
+ ///
+ /// Callers must ensure that `vma` is valid for the duration of 'a, and that the mmap read lock
+ /// (or stronger) is held for at least the duration of 'a. The `VM_MIXEDMAP` flag must be set.
+ #[inline]
+ pub unsafe fn from_raw<'a>(vma: *const bindings::vm_area_struct) -> &'a Self {
+ // SAFETY: The caller ensures that the invariants are satisfied for the duration of 'a.
+ unsafe { &*vma.cast() }
+ }
+
+ /// Maps a single page at the given address within the virtual memory area.
+ ///
+ /// This operation does not take ownership of the page.
+ #[inline]
+ pub fn vm_insert_page(&self, address: usize, page: &Page) -> Result {
+ // SAFETY: By the type invariant of `Self` caller has read access and has verified that
+ // `VM_MIXEDMAP` is set. By invariant on `Page` the page has order 0.
+ to_result(unsafe { bindings::vm_insert_page(self.as_ptr(), address, page.as_ptr()) })
+ }
+}
+
+/// A configuration object for setting up a VMA in an `f_ops->mmap()` hook.
+///
+/// The `f_ops->mmap()` hook is called when a new VMA is being created, and the hook is able to
+/// configure the VMA in various ways to fit the driver that owns it. Using `VmaNew` indicates that
+/// you are allowed to perform operations on the VMA that can only be performed before the VMA is
+/// fully initialized.
+///
+/// # Invariants
+///
+/// For the duration of 'a, the referenced vma must be undergoing initialization in an
+/// `f_ops->mmap()` hook.
+pub struct VmaNew {
+ vma: VmaRef,
+}
+
+// Make all `VmaRef` methods available on `VmaNew`.
+impl Deref for VmaNew {
+ type Target = VmaRef;
+
+ #[inline]
+ fn deref(&self) -> &VmaRef {
+ &self.vma
+ }
+}
+
+impl VmaNew {
+ /// Access a virtual memory area given a raw pointer.
+ ///
+ /// # Safety
+ ///
+ /// Callers must ensure that `vma` is undergoing initial vma setup for the duration of 'a.
+ #[inline]
+ pub unsafe fn from_raw<'a>(vma: *mut bindings::vm_area_struct) -> &'a Self {
+ // SAFETY: The caller ensures that the invariants are satisfied for the duration of 'a.
+ unsafe { &*vma.cast() }
+ }
+
+ /// Internal method for updating the vma flags.
+ ///
+ /// # Safety
+ ///
+ /// This must not be used to set the flags to an invalid value.
+ #[inline]
+ unsafe fn update_flags(&self, set: vm_flags_t, unset: vm_flags_t) {
+ let mut flags = self.flags();
+ flags |= set;
+ flags &= !unset;
+
+ // SAFETY: This is not a data race: the vma is undergoing initial setup, so it's not yet
+ // shared. Additionally, `VmaNew` is `!Sync`, so it cannot be used to write in parallel.
+ // The caller promises that this does not set the flags to an invalid value.
+ unsafe { (*self.as_ptr()).__bindgen_anon_2.__vm_flags = flags };
+ }
+
+ /// Set the `VM_MIXEDMAP` flag on this vma.
+ ///
+ /// This enables the vma to contain both `struct page` and pure PFN pages. Returns a reference
+ /// that can be used to call `vm_insert_page` on the vma.
+ #[inline]
+ pub fn set_mixedmap(&self) -> &VmaMixedMap {
+ // SAFETY: We don't yet provide a way to set VM_PFNMAP, so this cannot put the flags in an
+ // invalid state.
+ unsafe { self.update_flags(flags::MIXEDMAP, 0) };
+
+ // SAFETY: We just set `VM_MIXEDMAP` on the vma.
+ unsafe { VmaMixedMap::from_raw(self.vma.as_ptr()) }
+ }
+
+ /// Set the `VM_IO` flag on this vma.
+ ///
+ /// This is used for memory mapped IO and similar. The flag tells other parts of the kernel to
+ /// avoid looking at the pages. For memory mapped IO this is useful as accesses to the pages
+ /// could have side effects.
+ #[inline]
+ pub fn set_io(&self) {
+ // SAFETY: Setting the VM_IO flag is always okay.
+ unsafe { self.update_flags(flags::IO, 0) };
+ }
+
+ /// Set the `VM_DONTEXPAND` flag on this vma.
+ ///
+ /// This prevents the vma from being expanded with `mremap()`.
+ #[inline]
+ pub fn set_dontexpand(&self) {
+ // SAFETY: Setting the VM_DONTEXPAND flag is always okay.
+ unsafe { self.update_flags(flags::DONTEXPAND, 0) };
+ }
+
+ /// Set the `VM_DONTCOPY` flag on this vma.
+ ///
+ /// This prevents the vma from being copied on fork. This option is only permanent if `VM_IO`
+ /// is set.
+ #[inline]
+ pub fn set_dontcopy(&self) {
+ // SAFETY: Setting the VM_DONTCOPY flag is always okay.
+ unsafe { self.update_flags(flags::DONTCOPY, 0) };
+ }
+
+ /// Set the `VM_DONTDUMP` flag on this vma.
+ ///
+ /// This prevents the vma from being included in core dumps. This option is only permanent if
+ /// `VM_IO` is set.
+ #[inline]
+ pub fn set_dontdump(&self) {
+ // SAFETY: Setting the VM_DONTDUMP flag is always okay.
+ unsafe { self.update_flags(flags::DONTDUMP, 0) };
+ }
+
+ /// Returns whether `VM_READ` is set.
+ ///
+ /// This flag indicates whether userspace is mapping this vma as readable.
+ #[inline]
+ pub fn readable(&self) -> bool {
+ (self.flags() & flags::READ) != 0
+ }
+
+ /// Try to clear the `VM_MAYREAD` flag, failing if `VM_READ` is set.
+ ///
+ /// This flag indicates whether userspace is allowed to make this vma readable with
+ /// `mprotect()`.
+ ///
+ /// Note that this operation is irreversible. Once `VM_MAYREAD` has been cleared, it can never
+ /// be set again.
+ #[inline]
+ pub fn try_clear_mayread(&self) -> Result {
+ if self.readable() {
+ return Err(EINVAL);
+ }
+ // SAFETY: Clearing `VM_MAYREAD` is okay when `VM_READ` is not set.
+ unsafe { self.update_flags(0, flags::MAYREAD) };
+ Ok(())
+ }
+
+ /// Returns whether `VM_WRITE` is set.
+ ///
+ /// This flag indicates whether userspace is mapping this vma as writable.
+ #[inline]
+ pub fn writable(&self) -> bool {
+ (self.flags() & flags::WRITE) != 0
+ }
+
+ /// Try to clear the `VM_MAYWRITE` flag, failing if `VM_WRITE` is set.
+ ///
+ /// This flag indicates whether userspace is allowed to make this vma writable with
+ /// `mprotect()`.
+ ///
+ /// Note that this operation is irreversible. Once `VM_MAYWRITE` has been cleared, it can never
+ /// be set again.
+ #[inline]
+ pub fn try_clear_maywrite(&self) -> Result {
+ if self.writable() {
+ return Err(EINVAL);
+ }
+ // SAFETY: Clearing `VM_MAYWRITE` is okay when `VM_WRITE` is not set.
+ unsafe { self.update_flags(0, flags::MAYWRITE) };
+ Ok(())
+ }
+
+ /// Returns whether `VM_EXEC` is set.
+ ///
+ /// This flag indicates whether userspace is mapping this vma as executable.
+ #[inline]
+ pub fn executable(&self) -> bool {
+ (self.flags() & flags::EXEC) != 0
+ }
+
+ /// Try to clear the `VM_MAYEXEC` flag, failing if `VM_EXEC` is set.
+ ///
+ /// This flag indicates whether userspace is allowed to make this vma executable with
+ /// `mprotect()`.
+ ///
+ /// Note that this operation is irreversible. Once `VM_MAYEXEC` has been cleared, it can never
+ /// be set again.
+ #[inline]
+ pub fn try_clear_mayexec(&self) -> Result {
+ if self.executable() {
+ return Err(EINVAL);
+ }
+ // SAFETY: Clearing `VM_MAYEXEC` is okay when `VM_EXEC` is not set.
+ unsafe { self.update_flags(0, flags::MAYEXEC) };
+ Ok(())
+ }
+}
+
+/// The integer type used for vma flags.
+#[doc(inline)]
+pub use bindings::vm_flags_t;
+
+/// All possible flags for [`VmaRef`].
+pub mod flags {
+ use super::vm_flags_t;
+ use crate::bindings;
+
+ /// No flags are set.
+ pub const NONE: vm_flags_t = bindings::VM_NONE as _;
+
+ /// Mapping allows reads.
+ pub const READ: vm_flags_t = bindings::VM_READ as _;
+
+ /// Mapping allows writes.
+ pub const WRITE: vm_flags_t = bindings::VM_WRITE as _;
+
+ /// Mapping allows execution.
+ pub const EXEC: vm_flags_t = bindings::VM_EXEC as _;
+
+ /// Mapping is shared.
+ pub const SHARED: vm_flags_t = bindings::VM_SHARED as _;
+
+ /// Mapping may be updated to allow reads.
+ pub const MAYREAD: vm_flags_t = bindings::VM_MAYREAD as _;
+
+ /// Mapping may be updated to allow writes.
+ pub const MAYWRITE: vm_flags_t = bindings::VM_MAYWRITE as _;
+
+ /// Mapping may be updated to allow execution.
+ pub const MAYEXEC: vm_flags_t = bindings::VM_MAYEXEC as _;
+
+ /// Mapping may be updated to be shared.
+ pub const MAYSHARE: vm_flags_t = bindings::VM_MAYSHARE as _;
+
+ /// Page-ranges managed without `struct page`, just pure PFN.
+ pub const PFNMAP: vm_flags_t = bindings::VM_PFNMAP as _;
+
+ /// Memory mapped I/O or similar.
+ pub const IO: vm_flags_t = bindings::VM_IO as _;
+
+ /// Do not copy this vma on fork.
+ pub const DONTCOPY: vm_flags_t = bindings::VM_DONTCOPY as _;
+
+ /// Cannot expand with mremap().
+ pub const DONTEXPAND: vm_flags_t = bindings::VM_DONTEXPAND as _;
+
+ /// Lock the pages covered when they are faulted in.
+ pub const LOCKONFAULT: vm_flags_t = bindings::VM_LOCKONFAULT as _;
+
+ /// Is a VM accounted object.
+ pub const ACCOUNT: vm_flags_t = bindings::VM_ACCOUNT as _;
+
+ /// Should the VM suppress accounting.
+ pub const NORESERVE: vm_flags_t = bindings::VM_NORESERVE as _;
+
+ /// Huge TLB Page VM.
+ pub const HUGETLB: vm_flags_t = bindings::VM_HUGETLB as _;
+
+ /// Synchronous page faults. (DAX-specific)
+ pub const SYNC: vm_flags_t = bindings::VM_SYNC as _;
+
+ /// Architecture-specific flag.
+ pub const ARCH_1: vm_flags_t = bindings::VM_ARCH_1 as _;
+
+ /// Wipe VMA contents in child on fork.
+ pub const WIPEONFORK: vm_flags_t = bindings::VM_WIPEONFORK as _;
+
+ /// Do not include in the core dump.
+ pub const DONTDUMP: vm_flags_t = bindings::VM_DONTDUMP as _;
+
+ /// Not soft dirty clean area.
+ pub const SOFTDIRTY: vm_flags_t = bindings::VM_SOFTDIRTY as _;
+
+ /// Can contain `struct page` and pure PFN pages.
+ pub const MIXEDMAP: vm_flags_t = bindings::VM_MIXEDMAP as _;
+
+ /// MADV_HUGEPAGE marked this vma.
+ pub const HUGEPAGE: vm_flags_t = bindings::VM_HUGEPAGE as _;
+
+ /// MADV_NOHUGEPAGE marked this vma.
+ pub const NOHUGEPAGE: vm_flags_t = bindings::VM_NOHUGEPAGE as _;
+
+ /// KSM may merge identical pages.
+ pub const MERGEABLE: vm_flags_t = bindings::VM_MERGEABLE as _;
+}
diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs
index bb654a28dab3..32ea43ece646 100644
--- a/rust/kernel/net/phy.rs
+++ b/rust/kernel/net/phy.rs
@@ -421,6 +421,7 @@ impl<T: Driver> Adapter<T> {
/// `phydev` must be passed by the corresponding callback in `phy_driver`.
unsafe extern "C" fn match_phy_device_callback(
phydev: *mut bindings::phy_device,
+ _phydrv: *const bindings::phy_driver,
) -> crate::ffi::c_int {
// SAFETY: This callback is called only in contexts
// where we hold `phy_device->lock`, so the accessors on
@@ -790,7 +791,7 @@ impl DeviceMask {
/// DeviceId::new_with_driver::<PhySample>()
/// ],
/// name: "rust_sample_phy",
-/// author: "Rust for Linux Contributors",
+/// authors: ["Rust for Linux Contributors"],
/// description: "Rust sample PHYs driver",
/// license: "GPL",
/// }
@@ -819,7 +820,7 @@ impl DeviceMask {
/// module! {
/// type: Module,
/// name: "rust_sample_phy",
-/// author: "Rust for Linux Contributors",
+/// authors: ["Rust for Linux Contributors"],
/// description: "Rust sample PHYs driver",
/// license: "GPL",
/// }
diff --git a/rust/kernel/opp.rs b/rust/kernel/opp.rs
new file mode 100644
index 000000000000..a566fc3e7dcb
--- /dev/null
+++ b/rust/kernel/opp.rs
@@ -0,0 +1,1146 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Operating performance points.
+//!
+//! This module provides rust abstractions for interacting with the OPP subsystem.
+//!
+//! C header: [`include/linux/pm_opp.h`](srctree/include/linux/pm_opp.h)
+//!
+//! Reference: <https://docs.kernel.org/power/opp.html>
+
+use crate::{
+ clk::Hertz,
+ cpumask::{Cpumask, CpumaskVar},
+ device::Device,
+ error::{code::*, from_err_ptr, from_result, to_result, Error, Result, VTABLE_DEFAULT_ERROR},
+ ffi::c_ulong,
+ prelude::*,
+ str::CString,
+ types::{ARef, AlwaysRefCounted, Opaque},
+};
+
+#[cfg(CONFIG_CPU_FREQ)]
+/// Frequency table implementation.
+mod freq {
+ use super::*;
+ use crate::cpufreq;
+ use core::ops::Deref;
+
+ /// OPP frequency table.
+ ///
+ /// A [`cpufreq::Table`] created from [`Table`].
+ pub struct FreqTable {
+ dev: ARef<Device>,
+ ptr: *mut bindings::cpufreq_frequency_table,
+ }
+
+ impl FreqTable {
+ /// Creates a new instance of [`FreqTable`] from [`Table`].
+ pub(crate) fn new(table: &Table) -> Result<Self> {
+ let mut ptr: *mut bindings::cpufreq_frequency_table = ptr::null_mut();
+
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements.
+ to_result(unsafe {
+ bindings::dev_pm_opp_init_cpufreq_table(table.dev.as_raw(), &mut ptr)
+ })?;
+
+ Ok(Self {
+ dev: table.dev.clone(),
+ ptr,
+ })
+ }
+
+ /// Returns a reference to the underlying [`cpufreq::Table`].
+ #[inline]
+ fn table(&self) -> &cpufreq::Table {
+ // SAFETY: The `ptr` is guaranteed by the C code to be valid.
+ unsafe { cpufreq::Table::from_raw(self.ptr) }
+ }
+ }
+
+ impl Deref for FreqTable {
+ type Target = cpufreq::Table;
+
+ #[inline]
+ fn deref(&self) -> &Self::Target {
+ self.table()
+ }
+ }
+
+ impl Drop for FreqTable {
+ fn drop(&mut self) {
+ // SAFETY: The pointer was created via `dev_pm_opp_init_cpufreq_table`, and is only
+ // freed here.
+ unsafe {
+ bindings::dev_pm_opp_free_cpufreq_table(self.dev.as_raw(), &mut self.as_raw())
+ };
+ }
+ }
+}
+
+#[cfg(CONFIG_CPU_FREQ)]
+pub use freq::FreqTable;
+
+use core::{marker::PhantomData, ptr};
+
+use macros::vtable;
+
+/// Creates a null-terminated slice of pointers to [`Cstring`]s.
+fn to_c_str_array(names: &[CString]) -> Result<KVec<*const u8>> {
+ // Allocated a null-terminated vector of pointers.
+ let mut list = KVec::with_capacity(names.len() + 1, GFP_KERNEL)?;
+
+ for name in names.iter() {
+ list.push(name.as_ptr() as _, GFP_KERNEL)?;
+ }
+
+ list.push(ptr::null(), GFP_KERNEL)?;
+ Ok(list)
+}
+
+/// The voltage unit.
+///
+/// Represents voltage in microvolts, wrapping a [`c_ulong`] value.
+///
+/// ## Examples
+///
+/// ```
+/// use kernel::opp::MicroVolt;
+///
+/// let raw = 90500;
+/// let volt = MicroVolt(raw);
+///
+/// assert_eq!(usize::from(volt), raw);
+/// assert_eq!(volt, MicroVolt(raw));
+/// ```
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub struct MicroVolt(pub c_ulong);
+
+impl From<MicroVolt> for c_ulong {
+ #[inline]
+ fn from(volt: MicroVolt) -> Self {
+ volt.0
+ }
+}
+
+/// The power unit.
+///
+/// Represents power in microwatts, wrapping a [`c_ulong`] value.
+///
+/// ## Examples
+///
+/// ```
+/// use kernel::opp::MicroWatt;
+///
+/// let raw = 1000000;
+/// let power = MicroWatt(raw);
+///
+/// assert_eq!(usize::from(power), raw);
+/// assert_eq!(power, MicroWatt(raw));
+/// ```
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub struct MicroWatt(pub c_ulong);
+
+impl From<MicroWatt> for c_ulong {
+ #[inline]
+ fn from(power: MicroWatt) -> Self {
+ power.0
+ }
+}
+
+/// Handle for a dynamically created [`OPP`].
+///
+/// The associated [`OPP`] is automatically removed when the [`Token`] is dropped.
+///
+/// ## Examples
+///
+/// The following example demonstrates how to create an [`OPP`] dynamically.
+///
+/// ```
+/// use kernel::clk::Hertz;
+/// use kernel::device::Device;
+/// use kernel::error::Result;
+/// use kernel::opp::{Data, MicroVolt, Token};
+/// use kernel::types::ARef;
+///
+/// fn create_opp(dev: &ARef<Device>, freq: Hertz, volt: MicroVolt, level: u32) -> Result<Token> {
+/// let data = Data::new(freq, volt, level, false);
+///
+/// // OPP is removed once token goes out of scope.
+/// data.add_opp(dev)
+/// }
+/// ```
+pub struct Token {
+ dev: ARef<Device>,
+ freq: Hertz,
+}
+
+impl Token {
+ /// Dynamically adds an [`OPP`] and returns a [`Token`] that removes it on drop.
+ fn new(dev: &ARef<Device>, mut data: Data) -> Result<Self> {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements.
+ to_result(unsafe { bindings::dev_pm_opp_add_dynamic(dev.as_raw(), &mut data.0) })?;
+ Ok(Self {
+ dev: dev.clone(),
+ freq: data.freq(),
+ })
+ }
+}
+
+impl Drop for Token {
+ fn drop(&mut self) {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements.
+ unsafe { bindings::dev_pm_opp_remove(self.dev.as_raw(), self.freq.into()) };
+ }
+}
+
+/// OPP data.
+///
+/// Rust abstraction for the C `struct dev_pm_opp_data`, used to define operating performance
+/// points (OPPs) dynamically.
+///
+/// ## Examples
+///
+/// The following example demonstrates how to create an [`OPP`] with [`Data`].
+///
+/// ```
+/// use kernel::clk::Hertz;
+/// use kernel::device::Device;
+/// use kernel::error::Result;
+/// use kernel::opp::{Data, MicroVolt, Token};
+/// use kernel::types::ARef;
+///
+/// fn create_opp(dev: &ARef<Device>, freq: Hertz, volt: MicroVolt, level: u32) -> Result<Token> {
+/// let data = Data::new(freq, volt, level, false);
+///
+/// // OPP is removed once token goes out of scope.
+/// data.add_opp(dev)
+/// }
+/// ```
+#[repr(transparent)]
+pub struct Data(bindings::dev_pm_opp_data);
+
+impl Data {
+ /// Creates a new instance of [`Data`].
+ ///
+ /// This can be used to define a dynamic OPP to be added to a device.
+ pub fn new(freq: Hertz, volt: MicroVolt, level: u32, turbo: bool) -> Self {
+ Self(bindings::dev_pm_opp_data {
+ turbo,
+ freq: freq.into(),
+ u_volt: volt.into(),
+ level,
+ })
+ }
+
+ /// Adds an [`OPP`] dynamically.
+ ///
+ /// Returns a [`Token`] that ensures the OPP is automatically removed
+ /// when it goes out of scope.
+ #[inline]
+ pub fn add_opp(self, dev: &ARef<Device>) -> Result<Token> {
+ Token::new(dev, self)
+ }
+
+ /// Returns the frequency associated with this OPP data.
+ #[inline]
+ fn freq(&self) -> Hertz {
+ Hertz(self.0.freq)
+ }
+}
+
+/// [`OPP`] search options.
+///
+/// ## Examples
+///
+/// Defines how to search for an [`OPP`] in a [`Table`] relative to a frequency.
+///
+/// ```
+/// use kernel::clk::Hertz;
+/// use kernel::error::Result;
+/// use kernel::opp::{OPP, SearchType, Table};
+/// use kernel::types::ARef;
+///
+/// fn find_opp(table: &Table, freq: Hertz) -> Result<ARef<OPP>> {
+/// let opp = table.opp_from_freq(freq, Some(true), None, SearchType::Exact)?;
+///
+/// pr_info!("OPP frequency is: {:?}\n", opp.freq(None));
+/// pr_info!("OPP voltage is: {:?}\n", opp.voltage());
+/// pr_info!("OPP level is: {}\n", opp.level());
+/// pr_info!("OPP power is: {:?}\n", opp.power());
+///
+/// Ok(opp)
+/// }
+/// ```
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+pub enum SearchType {
+ /// Match the exact frequency.
+ Exact,
+ /// Find the highest frequency less than or equal to the given value.
+ Floor,
+ /// Find the lowest frequency greater than or equal to the given value.
+ Ceil,
+}
+
+/// OPP configuration callbacks.
+///
+/// Implement this trait to customize OPP clock and regulator setup for your device.
+#[vtable]
+pub trait ConfigOps {
+ /// This is typically used to scale clocks when transitioning between OPPs.
+ #[inline]
+ fn config_clks(_dev: &Device, _table: &Table, _opp: &OPP, _scaling_down: bool) -> Result {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+
+ /// This provides access to the old and new OPPs, allowing for safe regulator adjustments.
+ #[inline]
+ fn config_regulators(
+ _dev: &Device,
+ _opp_old: &OPP,
+ _opp_new: &OPP,
+ _data: *mut *mut bindings::regulator,
+ _count: u32,
+ ) -> Result {
+ build_error!(VTABLE_DEFAULT_ERROR)
+ }
+}
+
+/// OPP configuration token.
+///
+/// Returned by the OPP core when configuration is applied to a [`Device`]. The associated
+/// configuration is automatically cleared when the token is dropped.
+pub struct ConfigToken(i32);
+
+impl Drop for ConfigToken {
+ fn drop(&mut self) {
+ // SAFETY: This is the same token value returned by the C code via `dev_pm_opp_set_config`.
+ unsafe { bindings::dev_pm_opp_clear_config(self.0) };
+ }
+}
+
+/// OPP configurations.
+///
+/// Rust abstraction for the C `struct dev_pm_opp_config`.
+///
+/// ## Examples
+///
+/// The following example demonstrates how to set OPP property-name configuration for a [`Device`].
+///
+/// ```
+/// use kernel::device::Device;
+/// use kernel::error::Result;
+/// use kernel::opp::{Config, ConfigOps, ConfigToken};
+/// use kernel::str::CString;
+/// use kernel::types::ARef;
+/// use kernel::macros::vtable;
+///
+/// #[derive(Default)]
+/// struct Driver;
+///
+/// #[vtable]
+/// impl ConfigOps for Driver {}
+///
+/// fn configure(dev: &ARef<Device>) -> Result<ConfigToken> {
+/// let name = CString::try_from_fmt(fmt!("{}", "slow"))?;
+///
+/// // The OPP configuration is cleared once the [`ConfigToken`] goes out of scope.
+/// Config::<Driver>::new()
+/// .set_prop_name(name)?
+/// .set(dev)
+/// }
+/// ```
+#[derive(Default)]
+pub struct Config<T: ConfigOps>
+where
+ T: Default,
+{
+ clk_names: Option<KVec<CString>>,
+ prop_name: Option<CString>,
+ regulator_names: Option<KVec<CString>>,
+ supported_hw: Option<KVec<u32>>,
+
+ // Tuple containing (required device, index)
+ required_dev: Option<(ARef<Device>, u32)>,
+ _data: PhantomData<T>,
+}
+
+impl<T: ConfigOps + Default> Config<T> {
+ /// Creates a new instance of [`Config`].
+ #[inline]
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ /// Initializes clock names.
+ pub fn set_clk_names(mut self, names: KVec<CString>) -> Result<Self> {
+ if self.clk_names.is_some() {
+ return Err(EBUSY);
+ }
+
+ if names.is_empty() {
+ return Err(EINVAL);
+ }
+
+ self.clk_names = Some(names);
+ Ok(self)
+ }
+
+ /// Initializes property name.
+ pub fn set_prop_name(mut self, name: CString) -> Result<Self> {
+ if self.prop_name.is_some() {
+ return Err(EBUSY);
+ }
+
+ self.prop_name = Some(name);
+ Ok(self)
+ }
+
+ /// Initializes regulator names.
+ pub fn set_regulator_names(mut self, names: KVec<CString>) -> Result<Self> {
+ if self.regulator_names.is_some() {
+ return Err(EBUSY);
+ }
+
+ if names.is_empty() {
+ return Err(EINVAL);
+ }
+
+ self.regulator_names = Some(names);
+
+ Ok(self)
+ }
+
+ /// Initializes required devices.
+ pub fn set_required_dev(mut self, dev: ARef<Device>, index: u32) -> Result<Self> {
+ if self.required_dev.is_some() {
+ return Err(EBUSY);
+ }
+
+ self.required_dev = Some((dev, index));
+ Ok(self)
+ }
+
+ /// Initializes supported hardware.
+ pub fn set_supported_hw(mut self, hw: KVec<u32>) -> Result<Self> {
+ if self.supported_hw.is_some() {
+ return Err(EBUSY);
+ }
+
+ if hw.is_empty() {
+ return Err(EINVAL);
+ }
+
+ self.supported_hw = Some(hw);
+ Ok(self)
+ }
+
+ /// Sets the configuration with the OPP core.
+ ///
+ /// The returned [`ConfigToken`] will remove the configuration when dropped.
+ pub fn set(self, dev: &Device) -> Result<ConfigToken> {
+ let (_clk_list, clk_names) = match &self.clk_names {
+ Some(x) => {
+ let list = to_c_str_array(x)?;
+ let ptr = list.as_ptr();
+ (Some(list), ptr)
+ }
+ None => (None, ptr::null()),
+ };
+
+ let (_regulator_list, regulator_names) = match &self.regulator_names {
+ Some(x) => {
+ let list = to_c_str_array(x)?;
+ let ptr = list.as_ptr();
+ (Some(list), ptr)
+ }
+ None => (None, ptr::null()),
+ };
+
+ let prop_name = self
+ .prop_name
+ .as_ref()
+ .map_or(ptr::null(), |p| p.as_char_ptr());
+
+ let (supported_hw, supported_hw_count) = self
+ .supported_hw
+ .as_ref()
+ .map_or((ptr::null(), 0), |hw| (hw.as_ptr(), hw.len() as u32));
+
+ let (required_dev, required_dev_index) = self
+ .required_dev
+ .as_ref()
+ .map_or((ptr::null_mut(), 0), |(dev, idx)| (dev.as_raw(), *idx));
+
+ let mut config = bindings::dev_pm_opp_config {
+ clk_names,
+ config_clks: if T::HAS_CONFIG_CLKS {
+ Some(Self::config_clks)
+ } else {
+ None
+ },
+ prop_name,
+ regulator_names,
+ config_regulators: if T::HAS_CONFIG_REGULATORS {
+ Some(Self::config_regulators)
+ } else {
+ None
+ },
+ supported_hw,
+ supported_hw_count,
+
+ required_dev,
+ required_dev_index,
+ };
+
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements. The OPP core guarantees not to access fields of [`Config`] after this call
+ // and so we don't need to save a copy of them for future use.
+ let ret = unsafe { bindings::dev_pm_opp_set_config(dev.as_raw(), &mut config) };
+ if ret < 0 {
+ Err(Error::from_errno(ret))
+ } else {
+ Ok(ConfigToken(ret))
+ }
+ }
+
+ /// Config's clk callback.
+ ///
+ /// SAFETY: Called from C. Inputs must be valid pointers.
+ extern "C" fn config_clks(
+ dev: *mut bindings::device,
+ opp_table: *mut bindings::opp_table,
+ opp: *mut bindings::dev_pm_opp,
+ _data: *mut kernel::ffi::c_void,
+ scaling_down: bool,
+ ) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: 'dev' is guaranteed by the C code to be valid.
+ let dev = unsafe { Device::get_device(dev) };
+ T::config_clks(
+ &dev,
+ // SAFETY: 'opp_table' is guaranteed by the C code to be valid.
+ &unsafe { Table::from_raw_table(opp_table, &dev) },
+ // SAFETY: 'opp' is guaranteed by the C code to be valid.
+ unsafe { OPP::from_raw_opp(opp)? },
+ scaling_down,
+ )
+ .map(|()| 0)
+ })
+ }
+
+ /// Config's regulator callback.
+ ///
+ /// SAFETY: Called from C. Inputs must be valid pointers.
+ extern "C" fn config_regulators(
+ dev: *mut bindings::device,
+ old_opp: *mut bindings::dev_pm_opp,
+ new_opp: *mut bindings::dev_pm_opp,
+ regulators: *mut *mut bindings::regulator,
+ count: kernel::ffi::c_uint,
+ ) -> kernel::ffi::c_int {
+ from_result(|| {
+ // SAFETY: 'dev' is guaranteed by the C code to be valid.
+ let dev = unsafe { Device::get_device(dev) };
+ T::config_regulators(
+ &dev,
+ // SAFETY: 'old_opp' is guaranteed by the C code to be valid.
+ unsafe { OPP::from_raw_opp(old_opp)? },
+ // SAFETY: 'new_opp' is guaranteed by the C code to be valid.
+ unsafe { OPP::from_raw_opp(new_opp)? },
+ regulators,
+ count,
+ )
+ .map(|()| 0)
+ })
+ }
+}
+
+/// A reference-counted OPP table.
+///
+/// Rust abstraction for the C `struct opp_table`.
+///
+/// # Invariants
+///
+/// The pointer stored in `Self` is non-null and valid for the lifetime of the [`Table`].
+///
+/// Instances of this type are reference-counted.
+///
+/// ## Examples
+///
+/// The following example demonstrates how to get OPP [`Table`] for a [`Cpumask`] and set its
+/// frequency.
+///
+/// ```
+/// # #![cfg(CONFIG_OF)]
+/// use kernel::clk::Hertz;
+/// use kernel::cpumask::Cpumask;
+/// use kernel::device::Device;
+/// use kernel::error::Result;
+/// use kernel::opp::Table;
+/// use kernel::types::ARef;
+///
+/// fn get_table(dev: &ARef<Device>, mask: &mut Cpumask, freq: Hertz) -> Result<Table> {
+/// let mut opp_table = Table::from_of_cpumask(dev, mask)?;
+///
+/// if opp_table.opp_count()? == 0 {
+/// return Err(EINVAL);
+/// }
+///
+/// pr_info!("Max transition latency is: {} ns\n", opp_table.max_transition_latency_ns());
+/// pr_info!("Suspend frequency is: {:?}\n", opp_table.suspend_freq());
+///
+/// opp_table.set_rate(freq)?;
+/// Ok(opp_table)
+/// }
+/// ```
+pub struct Table {
+ ptr: *mut bindings::opp_table,
+ dev: ARef<Device>,
+ #[allow(dead_code)]
+ em: bool,
+ #[allow(dead_code)]
+ of: bool,
+ cpus: Option<CpumaskVar>,
+}
+
+/// SAFETY: It is okay to send ownership of [`Table`] across thread boundaries.
+unsafe impl Send for Table {}
+
+/// SAFETY: It is okay to access [`Table`] through shared references from other threads because
+/// we're either accessing properties that don't change or that are properly synchronised by C code.
+unsafe impl Sync for Table {}
+
+impl Table {
+ /// Creates a new reference-counted [`Table`] from a raw pointer.
+ ///
+ /// # Safety
+ ///
+ /// Callers must ensure that `ptr` is valid and non-null.
+ unsafe fn from_raw_table(ptr: *mut bindings::opp_table, dev: &ARef<Device>) -> Self {
+ // SAFETY: By the safety requirements, ptr is valid and its refcount will be incremented.
+ //
+ // INVARIANT: The reference-count is decremented when [`Table`] goes out of scope.
+ unsafe { bindings::dev_pm_opp_get_opp_table_ref(ptr) };
+
+ Self {
+ ptr,
+ dev: dev.clone(),
+ em: false,
+ of: false,
+ cpus: None,
+ }
+ }
+
+ /// Creates a new reference-counted [`Table`] instance for a [`Device`].
+ pub fn from_dev(dev: &Device) -> Result<Self> {
+ // SAFETY: The requirements are satisfied by the existence of the [`Device`] and its safety
+ // requirements.
+ //
+ // INVARIANT: The reference-count is incremented by the C code and is decremented when
+ // [`Table`] goes out of scope.
+ let ptr = from_err_ptr(unsafe { bindings::dev_pm_opp_get_opp_table(dev.as_raw()) })?;
+
+ Ok(Self {
+ ptr,
+ dev: dev.into(),
+ em: false,
+ of: false,
+ cpus: None,
+ })
+ }
+
+ /// Creates a new reference-counted [`Table`] instance for a [`Device`] based on device tree
+ /// entries.
+ #[cfg(CONFIG_OF)]
+ pub fn from_of(dev: &ARef<Device>, index: i32) -> Result<Self> {
+ // SAFETY: The requirements are satisfied by the existence of the [`Device`] and its safety
+ // requirements.
+ //
+ // INVARIANT: The reference-count is incremented by the C code and is decremented when
+ // [`Table`] goes out of scope.
+ to_result(unsafe { bindings::dev_pm_opp_of_add_table_indexed(dev.as_raw(), index) })?;
+
+ // Get the newly created [`Table`].
+ let mut table = Self::from_dev(dev)?;
+ table.of = true;
+
+ Ok(table)
+ }
+
+ /// Remove device tree based [`Table`].
+ #[cfg(CONFIG_OF)]
+ #[inline]
+ fn remove_of(&self) {
+ // SAFETY: The requirements are satisfied by the existence of the [`Device`] and its safety
+ // requirements. We took the reference from [`from_of`] earlier, it is safe to drop the
+ // same now.
+ unsafe { bindings::dev_pm_opp_of_remove_table(self.dev.as_raw()) };
+ }
+
+ /// Creates a new reference-counted [`Table`] instance for a [`Cpumask`] based on device tree
+ /// entries.
+ #[cfg(CONFIG_OF)]
+ pub fn from_of_cpumask(dev: &Device, cpumask: &mut Cpumask) -> Result<Self> {
+ // SAFETY: The cpumask is valid and the returned pointer will be owned by the [`Table`]
+ // instance.
+ //
+ // INVARIANT: The reference-count is incremented by the C code and is decremented when
+ // [`Table`] goes out of scope.
+ to_result(unsafe { bindings::dev_pm_opp_of_cpumask_add_table(cpumask.as_raw()) })?;
+
+ // Fetch the newly created table.
+ let mut table = Self::from_dev(dev)?;
+ table.cpus = Some(CpumaskVar::try_clone(cpumask)?);
+
+ Ok(table)
+ }
+
+ /// Remove device tree based [`Table`] for a [`Cpumask`].
+ #[cfg(CONFIG_OF)]
+ #[inline]
+ fn remove_of_cpumask(&self, cpumask: &Cpumask) {
+ // SAFETY: The cpumask is valid and we took the reference from [`from_of_cpumask`] earlier,
+ // it is safe to drop the same now.
+ unsafe { bindings::dev_pm_opp_of_cpumask_remove_table(cpumask.as_raw()) };
+ }
+
+ /// Returns the number of [`OPP`]s in the [`Table`].
+ pub fn opp_count(&self) -> Result<u32> {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements.
+ let ret = unsafe { bindings::dev_pm_opp_get_opp_count(self.dev.as_raw()) };
+ if ret < 0 {
+ Err(Error::from_errno(ret))
+ } else {
+ Ok(ret as u32)
+ }
+ }
+
+ /// Returns max clock latency (in nanoseconds) of the [`OPP`]s in the [`Table`].
+ #[inline]
+ pub fn max_clock_latency_ns(&self) -> usize {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements.
+ unsafe { bindings::dev_pm_opp_get_max_clock_latency(self.dev.as_raw()) }
+ }
+
+ /// Returns max volt latency (in nanoseconds) of the [`OPP`]s in the [`Table`].
+ #[inline]
+ pub fn max_volt_latency_ns(&self) -> usize {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements.
+ unsafe { bindings::dev_pm_opp_get_max_volt_latency(self.dev.as_raw()) }
+ }
+
+ /// Returns max transition latency (in nanoseconds) of the [`OPP`]s in the [`Table`].
+ #[inline]
+ pub fn max_transition_latency_ns(&self) -> usize {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements.
+ unsafe { bindings::dev_pm_opp_get_max_transition_latency(self.dev.as_raw()) }
+ }
+
+ /// Returns the suspend [`OPP`]'s frequency.
+ #[inline]
+ pub fn suspend_freq(&self) -> Hertz {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements.
+ Hertz(unsafe { bindings::dev_pm_opp_get_suspend_opp_freq(self.dev.as_raw()) })
+ }
+
+ /// Synchronizes regulators used by the [`Table`].
+ #[inline]
+ pub fn sync_regulators(&self) -> Result {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements.
+ to_result(unsafe { bindings::dev_pm_opp_sync_regulators(self.dev.as_raw()) })
+ }
+
+ /// Gets sharing CPUs.
+ #[inline]
+ pub fn sharing_cpus(dev: &Device, cpumask: &mut Cpumask) -> Result {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements.
+ to_result(unsafe { bindings::dev_pm_opp_get_sharing_cpus(dev.as_raw(), cpumask.as_raw()) })
+ }
+
+ /// Sets sharing CPUs.
+ pub fn set_sharing_cpus(&mut self, cpumask: &mut Cpumask) -> Result {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements.
+ to_result(unsafe {
+ bindings::dev_pm_opp_set_sharing_cpus(self.dev.as_raw(), cpumask.as_raw())
+ })?;
+
+ if let Some(mask) = self.cpus.as_mut() {
+ // Update the cpumask as this will be used while removing the table.
+ cpumask.copy(mask);
+ }
+
+ Ok(())
+ }
+
+ /// Gets sharing CPUs from device tree.
+ #[cfg(CONFIG_OF)]
+ #[inline]
+ pub fn of_sharing_cpus(dev: &Device, cpumask: &mut Cpumask) -> Result {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements.
+ to_result(unsafe {
+ bindings::dev_pm_opp_of_get_sharing_cpus(dev.as_raw(), cpumask.as_raw())
+ })
+ }
+
+ /// Updates the voltage value for an [`OPP`].
+ #[inline]
+ pub fn adjust_voltage(
+ &self,
+ freq: Hertz,
+ volt: MicroVolt,
+ volt_min: MicroVolt,
+ volt_max: MicroVolt,
+ ) -> Result {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements.
+ to_result(unsafe {
+ bindings::dev_pm_opp_adjust_voltage(
+ self.dev.as_raw(),
+ freq.into(),
+ volt.into(),
+ volt_min.into(),
+ volt_max.into(),
+ )
+ })
+ }
+
+ /// Creates [`FreqTable`] from [`Table`].
+ #[cfg(CONFIG_CPU_FREQ)]
+ #[inline]
+ pub fn cpufreq_table(&mut self) -> Result<FreqTable> {
+ FreqTable::new(self)
+ }
+
+ /// Configures device with [`OPP`] matching the frequency value.
+ #[inline]
+ pub fn set_rate(&self, freq: Hertz) -> Result {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements.
+ to_result(unsafe { bindings::dev_pm_opp_set_rate(self.dev.as_raw(), freq.into()) })
+ }
+
+ /// Configures device with [`OPP`].
+ #[inline]
+ pub fn set_opp(&self, opp: &OPP) -> Result {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements.
+ to_result(unsafe { bindings::dev_pm_opp_set_opp(self.dev.as_raw(), opp.as_raw()) })
+ }
+
+ /// Finds [`OPP`] based on frequency.
+ pub fn opp_from_freq(
+ &self,
+ freq: Hertz,
+ available: Option<bool>,
+ index: Option<u32>,
+ stype: SearchType,
+ ) -> Result<ARef<OPP>> {
+ let raw_dev = self.dev.as_raw();
+ let index = index.unwrap_or(0);
+ let mut rate = freq.into();
+
+ let ptr = from_err_ptr(match stype {
+ SearchType::Exact => {
+ if let Some(available) = available {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and
+ // its safety requirements. The returned pointer will be owned by the new
+ // [`OPP`] instance.
+ unsafe {
+ bindings::dev_pm_opp_find_freq_exact_indexed(
+ raw_dev, rate, index, available,
+ )
+ }
+ } else {
+ return Err(EINVAL);
+ }
+ }
+
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements. The returned pointer will be owned by the new [`OPP`] instance.
+ SearchType::Ceil => unsafe {
+ bindings::dev_pm_opp_find_freq_ceil_indexed(raw_dev, &mut rate, index)
+ },
+
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements. The returned pointer will be owned by the new [`OPP`] instance.
+ SearchType::Floor => unsafe {
+ bindings::dev_pm_opp_find_freq_floor_indexed(raw_dev, &mut rate, index)
+ },
+ })?;
+
+ // SAFETY: The `ptr` is guaranteed by the C code to be valid.
+ unsafe { OPP::from_raw_opp_owned(ptr) }
+ }
+
+ /// Finds [`OPP`] based on level.
+ pub fn opp_from_level(&self, mut level: u32, stype: SearchType) -> Result<ARef<OPP>> {
+ let raw_dev = self.dev.as_raw();
+
+ let ptr = from_err_ptr(match stype {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements. The returned pointer will be owned by the new [`OPP`] instance.
+ SearchType::Exact => unsafe { bindings::dev_pm_opp_find_level_exact(raw_dev, level) },
+
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements. The returned pointer will be owned by the new [`OPP`] instance.
+ SearchType::Ceil => unsafe {
+ bindings::dev_pm_opp_find_level_ceil(raw_dev, &mut level)
+ },
+
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements. The returned pointer will be owned by the new [`OPP`] instance.
+ SearchType::Floor => unsafe {
+ bindings::dev_pm_opp_find_level_floor(raw_dev, &mut level)
+ },
+ })?;
+
+ // SAFETY: The `ptr` is guaranteed by the C code to be valid.
+ unsafe { OPP::from_raw_opp_owned(ptr) }
+ }
+
+ /// Finds [`OPP`] based on bandwidth.
+ pub fn opp_from_bw(&self, mut bw: u32, index: i32, stype: SearchType) -> Result<ARef<OPP>> {
+ let raw_dev = self.dev.as_raw();
+
+ let ptr = from_err_ptr(match stype {
+ // The OPP core doesn't support this yet.
+ SearchType::Exact => return Err(EINVAL),
+
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements. The returned pointer will be owned by the new [`OPP`] instance.
+ SearchType::Ceil => unsafe {
+ bindings::dev_pm_opp_find_bw_ceil(raw_dev, &mut bw, index)
+ },
+
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements. The returned pointer will be owned by the new [`OPP`] instance.
+ SearchType::Floor => unsafe {
+ bindings::dev_pm_opp_find_bw_floor(raw_dev, &mut bw, index)
+ },
+ })?;
+
+ // SAFETY: The `ptr` is guaranteed by the C code to be valid.
+ unsafe { OPP::from_raw_opp_owned(ptr) }
+ }
+
+ /// Enables the [`OPP`].
+ #[inline]
+ pub fn enable_opp(&self, freq: Hertz) -> Result {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements.
+ to_result(unsafe { bindings::dev_pm_opp_enable(self.dev.as_raw(), freq.into()) })
+ }
+
+ /// Disables the [`OPP`].
+ #[inline]
+ pub fn disable_opp(&self, freq: Hertz) -> Result {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements.
+ to_result(unsafe { bindings::dev_pm_opp_disable(self.dev.as_raw(), freq.into()) })
+ }
+
+ /// Registers with the Energy model.
+ #[cfg(CONFIG_OF)]
+ pub fn of_register_em(&mut self, cpumask: &mut Cpumask) -> Result {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements.
+ to_result(unsafe {
+ bindings::dev_pm_opp_of_register_em(self.dev.as_raw(), cpumask.as_raw())
+ })?;
+
+ self.em = true;
+ Ok(())
+ }
+
+ /// Unregisters with the Energy model.
+ #[cfg(all(CONFIG_OF, CONFIG_ENERGY_MODEL))]
+ #[inline]
+ fn of_unregister_em(&self) {
+ // SAFETY: The requirements are satisfied by the existence of [`Device`] and its safety
+ // requirements. We registered with the EM framework earlier, it is safe to unregister now.
+ unsafe { bindings::em_dev_unregister_perf_domain(self.dev.as_raw()) };
+ }
+}
+
+impl Drop for Table {
+ fn drop(&mut self) {
+ // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe
+ // to relinquish it now.
+ unsafe { bindings::dev_pm_opp_put_opp_table(self.ptr) };
+
+ #[cfg(CONFIG_OF)]
+ {
+ #[cfg(CONFIG_ENERGY_MODEL)]
+ if self.em {
+ self.of_unregister_em();
+ }
+
+ if self.of {
+ self.remove_of();
+ } else if let Some(cpumask) = self.cpus.take() {
+ self.remove_of_cpumask(&cpumask);
+ }
+ }
+ }
+}
+
+/// A reference-counted Operating performance point (OPP).
+///
+/// Rust abstraction for the C `struct dev_pm_opp`.
+///
+/// # Invariants
+///
+/// The pointer stored in `Self` is non-null and valid for the lifetime of the [`OPP`].
+///
+/// Instances of this type are reference-counted. The reference count is incremented by the
+/// `dev_pm_opp_get` function and decremented by `dev_pm_opp_put`. The Rust type `ARef<OPP>`
+/// represents a pointer that owns a reference count on the [`OPP`].
+///
+/// A reference to the [`OPP`], &[`OPP`], isn't refcounted by the Rust code.
+///
+/// ## Examples
+///
+/// The following example demonstrates how to get [`OPP`] corresponding to a frequency value and
+/// configure the device with it.
+///
+/// ```
+/// use kernel::clk::Hertz;
+/// use kernel::error::Result;
+/// use kernel::opp::{SearchType, Table};
+///
+/// fn configure_opp(table: &Table, freq: Hertz) -> Result {
+/// let opp = table.opp_from_freq(freq, Some(true), None, SearchType::Exact)?;
+///
+/// if opp.freq(None) != freq {
+/// return Err(EINVAL);
+/// }
+///
+/// table.set_opp(&opp)
+/// }
+/// ```
+#[repr(transparent)]
+pub struct OPP(Opaque<bindings::dev_pm_opp>);
+
+/// SAFETY: It is okay to send the ownership of [`OPP`] across thread boundaries.
+unsafe impl Send for OPP {}
+
+/// SAFETY: It is okay to access [`OPP`] through shared references from other threads because we're
+/// either accessing properties that don't change or that are properly synchronised by C code.
+unsafe impl Sync for OPP {}
+
+/// SAFETY: The type invariants guarantee that [`OPP`] is always refcounted.
+unsafe impl AlwaysRefCounted for OPP {
+ fn inc_ref(&self) {
+ // SAFETY: The existence of a shared reference means that the refcount is nonzero.
+ unsafe { bindings::dev_pm_opp_get(self.0.get()) };
+ }
+
+ unsafe fn dec_ref(obj: ptr::NonNull<Self>) {
+ // SAFETY: The safety requirements guarantee that the refcount is nonzero.
+ unsafe { bindings::dev_pm_opp_put(obj.cast().as_ptr()) }
+ }
+}
+
+impl OPP {
+ /// Creates an owned reference to a [`OPP`] from a valid pointer.
+ ///
+ /// The refcount is incremented by the C code and will be decremented by `dec_ref` when the
+ /// [`ARef`] object is dropped.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `ptr` is valid and the refcount of the [`OPP`] is incremented.
+ /// The caller must also ensure that it doesn't explicitly drop the refcount of the [`OPP`], as
+ /// the returned [`ARef`] object takes over the refcount increment on the underlying object and
+ /// the same will be dropped along with it.
+ pub unsafe fn from_raw_opp_owned(ptr: *mut bindings::dev_pm_opp) -> Result<ARef<Self>> {
+ let ptr = ptr::NonNull::new(ptr).ok_or(ENODEV)?;
+
+ // SAFETY: The safety requirements guarantee the validity of the pointer.
+ //
+ // INVARIANT: The reference-count is decremented when [`OPP`] goes out of scope.
+ Ok(unsafe { ARef::from_raw(ptr.cast()) })
+ }
+
+ /// Creates a reference to a [`OPP`] from a valid pointer.
+ ///
+ /// The refcount is not updated by the Rust API unless the returned reference is converted to
+ /// an [`ARef`] object.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that `ptr` is valid and remains valid for the duration of `'a`.
+ #[inline]
+ pub unsafe fn from_raw_opp<'a>(ptr: *mut bindings::dev_pm_opp) -> Result<&'a Self> {
+ // SAFETY: The caller guarantees that the pointer is not dangling and stays valid for the
+ // duration of 'a. The cast is okay because [`OPP`] is `repr(transparent)`.
+ Ok(unsafe { &*ptr.cast() })
+ }
+
+ #[inline]
+ fn as_raw(&self) -> *mut bindings::dev_pm_opp {
+ self.0.get()
+ }
+
+ /// Returns the frequency of an [`OPP`].
+ pub fn freq(&self, index: Option<u32>) -> Hertz {
+ let index = index.unwrap_or(0);
+
+ // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to
+ // use it.
+ Hertz(unsafe { bindings::dev_pm_opp_get_freq_indexed(self.as_raw(), index) })
+ }
+
+ /// Returns the voltage of an [`OPP`].
+ #[inline]
+ pub fn voltage(&self) -> MicroVolt {
+ // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to
+ // use it.
+ MicroVolt(unsafe { bindings::dev_pm_opp_get_voltage(self.as_raw()) })
+ }
+
+ /// Returns the level of an [`OPP`].
+ #[inline]
+ pub fn level(&self) -> u32 {
+ // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to
+ // use it.
+ unsafe { bindings::dev_pm_opp_get_level(self.as_raw()) }
+ }
+
+ /// Returns the power of an [`OPP`].
+ #[inline]
+ pub fn power(&self) -> MicroWatt {
+ // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to
+ // use it.
+ MicroWatt(unsafe { bindings::dev_pm_opp_get_power(self.as_raw()) })
+ }
+
+ /// Returns the required pstate of an [`OPP`].
+ #[inline]
+ pub fn required_pstate(&self, index: u32) -> u32 {
+ // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to
+ // use it.
+ unsafe { bindings::dev_pm_opp_get_required_pstate(self.as_raw(), index) }
+ }
+
+ /// Returns true if the [`OPP`] is turbo.
+ #[inline]
+ pub fn is_turbo(&self) -> bool {
+ // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to
+ // use it.
+ unsafe { bindings::dev_pm_opp_is_turbo(self.as_raw()) }
+ }
+}
diff --git a/rust/kernel/page.rs b/rust/kernel/page.rs
index f6126aca33a6..7c1b17246ed5 100644
--- a/rust/kernel/page.rs
+++ b/rust/kernel/page.rs
@@ -69,6 +69,7 @@ impl Page {
/// let page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?;
/// # Ok::<(), kernel::alloc::AllocError>(())
/// ```
+ #[inline]
pub fn alloc_page(flags: Flags) -> Result<Self, AllocError> {
// SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it
// is always safe to call this method.
@@ -251,6 +252,7 @@ impl Page {
}
impl Drop for Page {
+ #[inline]
fn drop(&mut self) {
// SAFETY: By the type invariants, we have ownership of the page and can free it.
unsafe { bindings::__free_pages(self.page.as_ptr(), 0) };
diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs
index 4c98b5b9aa1e..8435f8132e38 100644
--- a/rust/kernel/pci.rs
+++ b/rust/kernel/pci.rs
@@ -17,7 +17,11 @@ use crate::{
types::{ARef, ForeignOwnable, Opaque},
ThisModule,
};
-use core::{ops::Deref, ptr::addr_of_mut};
+use core::{
+ marker::PhantomData,
+ ops::Deref,
+ ptr::{addr_of_mut, NonNull},
+};
use kernel::prelude::*;
/// An adapter for the registration of PCI drivers.
@@ -60,17 +64,16 @@ impl<T: Driver + 'static> Adapter<T> {
) -> kernel::ffi::c_int {
// SAFETY: The PCI bus only ever calls the probe callback with a valid pointer to a
// `struct pci_dev`.
- let dev = unsafe { device::Device::get_device(addr_of_mut!((*pdev).dev)) };
- // SAFETY: `dev` is guaranteed to be embedded in a valid `struct pci_dev` by the call
- // above.
- let mut pdev = unsafe { Device::from_dev(dev) };
+ //
+ // INVARIANT: `pdev` is valid for the duration of `probe_callback()`.
+ let pdev = unsafe { &*pdev.cast::<Device<device::Core>>() };
// SAFETY: `DeviceId` is a `#[repr(transparent)` wrapper of `struct pci_device_id` and
// does not add additional invariants, so it's safe to transmute.
let id = unsafe { &*id.cast::<DeviceId>() };
let info = T::ID_TABLE.info(id.index());
- match T::probe(&mut pdev, info) {
+ match T::probe(pdev, info) {
Ok(data) => {
// Let the `struct pci_dev` own a reference of the driver's private data.
// SAFETY: By the type invariant `pdev.as_raw` returns a valid pointer to a
@@ -86,7 +89,7 @@ impl<T: Driver + 'static> Adapter<T> {
extern "C" fn remove_callback(pdev: *mut bindings::pci_dev) {
// SAFETY: The PCI bus only ever calls the remove callback with a valid pointer to a
// `struct pci_dev`.
- let ptr = unsafe { bindings::pci_get_drvdata(pdev) };
+ let ptr = unsafe { bindings::pci_get_drvdata(pdev) }.cast();
// SAFETY: `remove_callback` is only ever called after a successful call to
// `probe_callback`, hence it's guaranteed that `ptr` points to a valid and initialized
@@ -103,7 +106,7 @@ impl<T: Driver + 'static> Adapter<T> {
/// kernel::module_pci_driver! {
/// type: MyDriver,
/// name: "Module name",
-/// author: "Author name",
+/// authors: ["Author name"],
/// description: "Description",
/// license: "GPL v2",
/// }
@@ -115,7 +118,9 @@ macro_rules! module_pci_driver {
};
}
-/// Abstraction for bindings::pci_device_id.
+/// Abstraction for the PCI device ID structure ([`struct pci_device_id`]).
+///
+/// [`struct pci_device_id`]: https://docs.kernel.org/PCI/pci.html#c.pci_device_id
#[repr(transparent)]
#[derive(Clone, Copy)]
pub struct DeviceId(bindings::pci_device_id);
@@ -170,7 +175,7 @@ unsafe impl RawDeviceId for DeviceId {
}
}
-/// IdTable type for PCI
+/// `IdTable` type for PCI.
pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>;
/// Create a PCI `IdTable` with its alias for modpost.
@@ -192,7 +197,7 @@ macro_rules! pci_device_table {
/// # Example
///
///```
-/// # use kernel::{bindings, pci};
+/// # use kernel::{bindings, device::Core, pci};
///
/// struct MyDriver;
///
@@ -210,7 +215,7 @@ macro_rules! pci_device_table {
/// const ID_TABLE: pci::IdTable<Self::IdInfo> = &PCI_TABLE;
///
/// fn probe(
-/// _pdev: &mut pci::Device,
+/// _pdev: &pci::Device<Core>,
/// _id_info: &Self::IdInfo,
/// ) -> Result<Pin<KBox<Self>>> {
/// Err(ENODEV)
@@ -219,12 +224,13 @@ macro_rules! pci_device_table {
///```
/// Drivers must implement this trait in order to get a PCI driver registered. Please refer to the
/// `Adapter` documentation for an example.
-pub trait Driver {
+pub trait Driver: Send {
/// The type holding information about each device id supported by the driver.
- ///
- /// TODO: Use associated_type_defaults once stabilized:
- ///
- /// type IdInfo: 'static = ();
+ // TODO: Use `associated_type_defaults` once stabilized:
+ //
+ // ```
+ // type IdInfo: 'static = ();
+ // ```
type IdInfo: 'static;
/// The table of device ids supported by the driver.
@@ -234,20 +240,23 @@ pub trait Driver {
///
/// Called when a new platform device is added or discovered.
/// Implementers should attempt to initialize the device here.
- fn probe(dev: &mut Device, id_info: &Self::IdInfo) -> Result<Pin<KBox<Self>>>;
+ fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> Result<Pin<KBox<Self>>>;
}
/// The PCI device representation.
///
-/// A PCI device is based on an always reference counted `device:Device` instance. Cloning a PCI
-/// device, hence, also increments the base device' reference count.
+/// This structure represents the Rust abstraction for a C `struct pci_dev`. The implementation
+/// abstracts the usage of an already existing C `struct pci_dev` within Rust code that we get
+/// passed from the C side.
///
/// # Invariants
///
-/// `Device` hold a valid reference of `ARef<device::Device>` whose underlying `struct device` is a
-/// member of a `struct pci_dev`.
-#[derive(Clone)]
-pub struct Device(ARef<device::Device>);
+/// A [`Device`] instance represents a valid `struct device` created by the C portion of the kernel.
+#[repr(transparent)]
+pub struct Device<Ctx: device::DeviceContext = device::Normal>(
+ Opaque<bindings::pci_dev>,
+ PhantomData<Ctx>,
+);
/// A PCI BAR to perform I/O-Operations on.
///
@@ -256,13 +265,13 @@ pub struct Device(ARef<device::Device>);
/// `Bar` always holds an `IoRaw` inststance that holds a valid pointer to the start of the I/O
/// memory mapped PCI bar and its size.
pub struct Bar<const SIZE: usize = 0> {
- pdev: Device,
+ pdev: ARef<Device>,
io: IoRaw<SIZE>,
num: i32,
}
impl<const SIZE: usize> Bar<SIZE> {
- fn new(pdev: Device, num: u32, name: &CStr) -> Result<Self> {
+ fn new(pdev: &Device, num: u32, name: &CStr) -> Result<Self> {
let len = pdev.resource_len(num)?;
if len == 0 {
return Err(ENOMEM);
@@ -300,12 +309,16 @@ impl<const SIZE: usize> Bar<SIZE> {
// `pdev` is valid by the invariants of `Device`.
// `ioptr` is guaranteed to be the start of a valid I/O mapped memory region.
// `num` is checked for validity by a previous call to `Device::resource_len`.
- unsafe { Self::do_release(&pdev, ioptr, num) };
+ unsafe { Self::do_release(pdev, ioptr, num) };
return Err(err);
}
};
- Ok(Bar { pdev, io, num })
+ Ok(Bar {
+ pdev: pdev.into(),
+ io,
+ num,
+ })
}
/// # Safety
@@ -350,23 +363,13 @@ impl<const SIZE: usize> Deref for Bar<SIZE> {
}
}
-impl Device {
- /// Create a PCI Device instance from an existing `device::Device`.
- ///
- /// # Safety
- ///
- /// `dev` must be an `ARef<device::Device>` whose underlying `bindings::device` is a member of
- /// a `bindings::pci_dev`.
- pub unsafe fn from_dev(dev: ARef<device::Device>) -> Self {
- Self(dev)
- }
-
+impl<Ctx: device::DeviceContext> Device<Ctx> {
fn as_raw(&self) -> *mut bindings::pci_dev {
- // SAFETY: By the type invariant `self.0.as_raw` is a pointer to the `struct device`
- // embedded in `struct pci_dev`.
- unsafe { container_of!(self.0.as_raw(), bindings::pci_dev, dev) as _ }
+ self.0.get()
}
+}
+impl Device {
/// Returns the PCI vendor ID.
pub fn vendor_id(&self) -> u16 {
// SAFETY: `self.as_raw` is a valid pointer to a `struct pci_dev`.
@@ -379,23 +382,6 @@ impl Device {
unsafe { (*self.as_raw()).device }
}
- /// Enable memory resources for this device.
- pub fn enable_device_mem(&self) -> Result {
- // SAFETY: `self.as_raw` is guaranteed to be a pointer to a valid `struct pci_dev`.
- let ret = unsafe { bindings::pci_enable_device_mem(self.as_raw()) };
- if ret != 0 {
- Err(Error::from_errno(ret))
- } else {
- Ok(())
- }
- }
-
- /// Enable bus-mastering for this device.
- pub fn set_master(&self) {
- // SAFETY: `self.as_raw` is guaranteed to be a pointer to a valid `struct pci_dev`.
- unsafe { bindings::pci_set_master(self.as_raw()) };
- }
-
/// Returns the size of the given PCI bar resource.
pub fn resource_len(&self, bar: u32) -> Result<bindings::resource_size_t> {
if !Bar::index_is_valid(bar) {
@@ -407,7 +393,9 @@ impl Device {
// - by its type invariant `self.as_raw` is always a valid pointer to a `struct pci_dev`.
Ok(unsafe { bindings::pci_resource_len(self.as_raw(), bar.try_into()?) })
}
+}
+impl Device<device::Bound> {
/// Mapps an entire PCI-BAR after performing a region-request on it. I/O operation bound checks
/// can be performed on compile time for offsets (plus the requested type size) < SIZE.
pub fn iomap_region_sized<const SIZE: usize>(
@@ -415,7 +403,7 @@ impl Device {
bar: u32,
name: &CStr,
) -> Result<Devres<Bar<SIZE>>> {
- let bar = Bar::<SIZE>::new(self.clone(), bar, name)?;
+ let bar = Bar::<SIZE>::new(self, bar, name)?;
let devres = Devres::new(self.as_ref(), bar, GFP_KERNEL)?;
Ok(devres)
@@ -427,8 +415,72 @@ impl Device {
}
}
-impl AsRef<device::Device> for Device {
- fn as_ref(&self) -> &device::Device {
- &self.0
+impl Device<device::Core> {
+ /// Enable memory resources for this device.
+ pub fn enable_device_mem(&self) -> Result {
+ // SAFETY: `self.as_raw` is guaranteed to be a pointer to a valid `struct pci_dev`.
+ to_result(unsafe { bindings::pci_enable_device_mem(self.as_raw()) })
+ }
+
+ /// Enable bus-mastering for this device.
+ pub fn set_master(&self) {
+ // SAFETY: `self.as_raw` is guaranteed to be a pointer to a valid `struct pci_dev`.
+ unsafe { bindings::pci_set_master(self.as_raw()) };
+ }
+}
+
+// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic
+// argument.
+kernel::impl_device_context_deref!(unsafe { Device });
+kernel::impl_device_context_into_aref!(Device);
+
+// SAFETY: Instances of `Device` are always reference-counted.
+unsafe impl crate::types::AlwaysRefCounted for Device {
+ fn inc_ref(&self) {
+ // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero.
+ unsafe { bindings::pci_dev_get(self.as_raw()) };
+ }
+
+ unsafe fn dec_ref(obj: NonNull<Self>) {
+ // SAFETY: The safety requirements guarantee that the refcount is non-zero.
+ unsafe { bindings::pci_dev_put(obj.cast().as_ptr()) }
}
}
+
+impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for Device<Ctx> {
+ fn as_ref(&self) -> &device::Device<Ctx> {
+ // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid
+ // `struct pci_dev`.
+ let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) };
+
+ // SAFETY: `dev` points to a valid `struct device`.
+ unsafe { device::Device::as_ref(dev) }
+ }
+}
+
+impl<Ctx: device::DeviceContext> TryFrom<&device::Device<Ctx>> for &Device<Ctx> {
+ type Error = kernel::error::Error;
+
+ fn try_from(dev: &device::Device<Ctx>) -> Result<Self, Self::Error> {
+ // SAFETY: By the type invariant of `Device`, `dev.as_raw()` is a valid pointer to a
+ // `struct device`.
+ if !unsafe { bindings::dev_is_pci(dev.as_raw()) } {
+ return Err(EINVAL);
+ }
+
+ // SAFETY: We've just verified that the bus type of `dev` equals `bindings::pci_bus_type`,
+ // hence `dev` must be embedded in a valid `struct pci_dev` as guaranteed by the
+ // corresponding C code.
+ let pdev = unsafe { container_of!(dev.as_raw(), bindings::pci_dev, dev) };
+
+ // SAFETY: `pdev` is a valid pointer to a `struct pci_dev`.
+ Ok(unsafe { &*pdev.cast() })
+ }
+}
+
+// SAFETY: A `Device` is always reference-counted and can be released from any thread.
+unsafe impl Send for Device {}
+
+// SAFETY: `Device` can be shared among threads because all methods of `Device`
+// (i.e. `Device<Normal>) are thread safe.
+unsafe impl Sync for Device {}
diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs
index 50e6b0421813..5b21fa517e55 100644
--- a/rust/kernel/platform.rs
+++ b/rust/kernel/platform.rs
@@ -10,11 +10,14 @@ use crate::{
of,
prelude::*,
str::CStr,
- types::{ARef, ForeignOwnable, Opaque},
+ types::{ForeignOwnable, Opaque},
ThisModule,
};
-use core::ptr::addr_of_mut;
+use core::{
+ marker::PhantomData,
+ ptr::{addr_of_mut, NonNull},
+};
/// An adapter for the registration of platform drivers.
pub struct Adapter<T: Driver>(T);
@@ -54,14 +57,14 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> {
impl<T: Driver + 'static> Adapter<T> {
extern "C" fn probe_callback(pdev: *mut bindings::platform_device) -> kernel::ffi::c_int {
- // SAFETY: The platform bus only ever calls the probe callback with a valid `pdev`.
- let dev = unsafe { device::Device::get_device(addr_of_mut!((*pdev).dev)) };
- // SAFETY: `dev` is guaranteed to be embedded in a valid `struct platform_device` by the
- // call above.
- let mut pdev = unsafe { Device::from_dev(dev) };
+ // SAFETY: The platform bus only ever calls the probe callback with a valid pointer to a
+ // `struct platform_device`.
+ //
+ // INVARIANT: `pdev` is valid for the duration of `probe_callback()`.
+ let pdev = unsafe { &*pdev.cast::<Device<device::Core>>() };
let info = <Self as driver::Adapter>::id_info(pdev.as_ref());
- match T::probe(&mut pdev, info) {
+ match T::probe(pdev, info) {
Ok(data) => {
// Let the `struct platform_device` own a reference of the driver's private data.
// SAFETY: By the type invariant `pdev.as_raw` returns a valid pointer to a
@@ -76,7 +79,7 @@ impl<T: Driver + 'static> Adapter<T> {
extern "C" fn remove_callback(pdev: *mut bindings::platform_device) {
// SAFETY: `pdev` is a valid pointer to a `struct platform_device`.
- let ptr = unsafe { bindings::platform_get_drvdata(pdev) };
+ let ptr = unsafe { bindings::platform_get_drvdata(pdev) }.cast();
// SAFETY: `remove_callback` is only ever called after a successful call to
// `probe_callback`, hence it's guaranteed that `ptr` points to a valid and initialized
@@ -101,7 +104,7 @@ impl<T: Driver + 'static> driver::Adapter for Adapter<T> {
/// kernel::module_platform_driver! {
/// type: MyDriver,
/// name: "Module name",
-/// author: "Author name",
+/// authors: ["Author name"],
/// description: "Description",
/// license: "GPL v2",
/// }
@@ -120,7 +123,7 @@ macro_rules! module_platform_driver {
/// # Example
///
///```
-/// # use kernel::{bindings, c_str, of, platform};
+/// # use kernel::{bindings, c_str, device::Core, of, platform};
///
/// struct MyDriver;
///
@@ -138,19 +141,20 @@ macro_rules! module_platform_driver {
/// const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = Some(&OF_TABLE);
///
/// fn probe(
-/// _pdev: &mut platform::Device,
+/// _pdev: &platform::Device<Core>,
/// _id_info: Option<&Self::IdInfo>,
/// ) -> Result<Pin<KBox<Self>>> {
/// Err(ENODEV)
/// }
/// }
///```
-pub trait Driver {
+pub trait Driver: Send {
/// The type holding driver private data about each device id supported by the driver.
- ///
- /// TODO: Use associated_type_defaults once stabilized:
- ///
- /// type IdInfo: 'static = ();
+ // TODO: Use associated_type_defaults once stabilized:
+ //
+ // ```
+ // type IdInfo: 'static = ();
+ // ```
type IdInfo: 'static;
/// The table of OF device ids supported by the driver.
@@ -160,41 +164,84 @@ pub trait Driver {
///
/// Called when a new platform device is added or discovered.
/// Implementers should attempt to initialize the device here.
- fn probe(dev: &mut Device, id_info: Option<&Self::IdInfo>) -> Result<Pin<KBox<Self>>>;
+ fn probe(dev: &Device<device::Core>, id_info: Option<&Self::IdInfo>)
+ -> Result<Pin<KBox<Self>>>;
}
/// The platform device representation.
///
-/// A platform device is based on an always reference counted `device:Device` instance. Cloning a
-/// platform device, hence, also increments the base device' reference count.
+/// This structure represents the Rust abstraction for a C `struct platform_device`. The
+/// implementation abstracts the usage of an already existing C `struct platform_device` within Rust
+/// code that we get passed from the C side.
///
/// # Invariants
///
-/// `Device` holds a valid reference of `ARef<device::Device>` whose underlying `struct device` is a
-/// member of a `struct platform_device`.
-#[derive(Clone)]
-pub struct Device(ARef<device::Device>);
+/// A [`Device`] instance represents a valid `struct platform_device` created by the C portion of
+/// the kernel.
+#[repr(transparent)]
+pub struct Device<Ctx: device::DeviceContext = device::Normal>(
+ Opaque<bindings::platform_device>,
+ PhantomData<Ctx>,
+);
+
+impl<Ctx: device::DeviceContext> Device<Ctx> {
+ fn as_raw(&self) -> *mut bindings::platform_device {
+ self.0.get()
+ }
+}
-impl Device {
- /// Convert a raw kernel device into a `Device`
- ///
- /// # Safety
- ///
- /// `dev` must be an `Aref<device::Device>` whose underlying `bindings::device` is a member of a
- /// `bindings::platform_device`.
- unsafe fn from_dev(dev: ARef<device::Device>) -> Self {
- Self(dev)
+// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic
+// argument.
+kernel::impl_device_context_deref!(unsafe { Device });
+kernel::impl_device_context_into_aref!(Device);
+
+// SAFETY: Instances of `Device` are always reference-counted.
+unsafe impl crate::types::AlwaysRefCounted for Device {
+ fn inc_ref(&self) {
+ // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero.
+ unsafe { bindings::get_device(self.as_ref().as_raw()) };
}
- fn as_raw(&self) -> *mut bindings::platform_device {
- // SAFETY: By the type invariant `self.0.as_raw` is a pointer to the `struct device`
- // embedded in `struct platform_device`.
- unsafe { container_of!(self.0.as_raw(), bindings::platform_device, dev) }.cast_mut()
+ unsafe fn dec_ref(obj: NonNull<Self>) {
+ // SAFETY: The safety requirements guarantee that the refcount is non-zero.
+ unsafe { bindings::platform_device_put(obj.cast().as_ptr()) }
}
}
-impl AsRef<device::Device> for Device {
- fn as_ref(&self) -> &device::Device {
- &self.0
+impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for Device<Ctx> {
+ fn as_ref(&self) -> &device::Device<Ctx> {
+ // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid
+ // `struct platform_device`.
+ let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) };
+
+ // SAFETY: `dev` points to a valid `struct device`.
+ unsafe { device::Device::as_ref(dev) }
}
}
+
+impl<Ctx: device::DeviceContext> TryFrom<&device::Device<Ctx>> for &Device<Ctx> {
+ type Error = kernel::error::Error;
+
+ fn try_from(dev: &device::Device<Ctx>) -> Result<Self, Self::Error> {
+ // SAFETY: By the type invariant of `Device`, `dev.as_raw()` is a valid pointer to a
+ // `struct device`.
+ if !unsafe { bindings::dev_is_platform(dev.as_raw()) } {
+ return Err(EINVAL);
+ }
+
+ // SAFETY: We've just verified that the bus type of `dev` equals
+ // `bindings::platform_bus_type`, hence `dev` must be embedded in a valid
+ // `struct platform_device` as guaranteed by the corresponding C code.
+ let pdev = unsafe { container_of!(dev.as_raw(), bindings::platform_device, dev) };
+
+ // SAFETY: `pdev` is a valid pointer to a `struct platform_device`.
+ Ok(unsafe { &*pdev.cast() })
+ }
+}
+
+// SAFETY: A `Device` is always reference-counted and can be released from any thread.
+unsafe impl Send for Device {}
+
+// SAFETY: `Device` can be shared among threads because all methods of `Device`
+// (i.e. `Device<Normal>) are thread safe.
+unsafe impl Sync for Device {}
diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs
index dde2e0649790..2f30a398dddd 100644
--- a/rust/kernel/prelude.rs
+++ b/rust/kernel/prelude.rs
@@ -14,10 +14,17 @@
#[doc(no_inline)]
pub use core::pin::Pin;
+pub use ::ffi::{
+ c_char, c_int, c_long, c_longlong, c_schar, c_short, c_uchar, c_uint, c_ulong, c_ulonglong,
+ c_ushort, c_void,
+};
+
pub use crate::alloc::{flags::*, Box, KBox, KVBox, KVVec, KVec, VBox, VVec, Vec};
#[doc(no_inline)]
-pub use macros::{module, pin_data, pinned_drop, vtable, Zeroable};
+pub use macros::{export, kunit_tests, module, vtable};
+
+pub use pin_init::{init, pin_data, pin_init, pinned_drop, InPlaceWrite, Init, PinInit, Zeroable};
pub use super::{build_assert, build_error};
@@ -28,7 +35,7 @@ pub use super::fmt;
pub use super::{dev_alert, dev_crit, dev_dbg, dev_emerg, dev_err, dev_info, dev_notice, dev_warn};
pub use super::{pr_alert, pr_crit, pr_debug, pr_emerg, pr_err, pr_info, pr_notice, pr_warn};
-pub use super::{init, pin_init, try_init, try_pin_init};
+pub use super::{try_init, try_pin_init};
pub use super::static_assert;
@@ -36,6 +43,6 @@ pub use super::error::{code::*, Error, Result};
pub use super::{str::CStr, ThisModule};
-pub use super::init::{InPlaceInit, InPlaceWrite, Init, PinInit};
+pub use super::init::InPlaceInit;
pub use super::current;
diff --git a/rust/kernel/print.rs b/rust/kernel/print.rs
index b19ee490be58..9783d960a97a 100644
--- a/rust/kernel/print.rs
+++ b/rust/kernel/print.rs
@@ -6,16 +6,16 @@
//!
//! Reference: <https://docs.kernel.org/core-api/printk-basics.html>
-use core::{
+use crate::{
ffi::{c_char, c_void},
- fmt,
+ prelude::*,
+ str::RawFormatter,
};
-
-use crate::str::RawFormatter;
+use core::fmt;
// Called from `vsprintf` with format specifier `%pA`.
#[expect(clippy::missing_safety_doc)]
-#[no_mangle]
+#[export]
unsafe extern "C" fn rust_fmt_argument(
buf: *mut c_char,
end: *mut c_char,
@@ -198,10 +198,11 @@ macro_rules! print_macro (
/// Equivalent to the kernel's [`pr_emerg`] macro.
///
/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
-/// `alloc::format!` for information about the formatting syntax.
+/// [`std::format!`] for information about the formatting syntax.
///
/// [`pr_emerg`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_emerg
/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html
///
/// # Examples
///
@@ -222,10 +223,11 @@ macro_rules! pr_emerg (
/// Equivalent to the kernel's [`pr_alert`] macro.
///
/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
-/// `alloc::format!` for information about the formatting syntax.
+/// [`std::format!`] for information about the formatting syntax.
///
/// [`pr_alert`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_alert
/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html
///
/// # Examples
///
@@ -246,10 +248,11 @@ macro_rules! pr_alert (
/// Equivalent to the kernel's [`pr_crit`] macro.
///
/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
-/// `alloc::format!` for information about the formatting syntax.
+/// [`std::format!`] for information about the formatting syntax.
///
/// [`pr_crit`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_crit
/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html
///
/// # Examples
///
@@ -270,10 +273,11 @@ macro_rules! pr_crit (
/// Equivalent to the kernel's [`pr_err`] macro.
///
/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
-/// `alloc::format!` for information about the formatting syntax.
+/// [`std::format!`] for information about the formatting syntax.
///
/// [`pr_err`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_err
/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html
///
/// # Examples
///
@@ -294,10 +298,11 @@ macro_rules! pr_err (
/// Equivalent to the kernel's [`pr_warn`] macro.
///
/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
-/// `alloc::format!` for information about the formatting syntax.
+/// [`std::format!`] for information about the formatting syntax.
///
/// [`pr_warn`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_warn
/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html
///
/// # Examples
///
@@ -318,10 +323,11 @@ macro_rules! pr_warn (
/// Equivalent to the kernel's [`pr_notice`] macro.
///
/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
-/// `alloc::format!` for information about the formatting syntax.
+/// [`std::format!`] for information about the formatting syntax.
///
/// [`pr_notice`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_notice
/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html
///
/// # Examples
///
@@ -342,10 +348,11 @@ macro_rules! pr_notice (
/// Equivalent to the kernel's [`pr_info`] macro.
///
/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
-/// `alloc::format!` for information about the formatting syntax.
+/// [`std::format!`] for information about the formatting syntax.
///
/// [`pr_info`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_info
/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html
///
/// # Examples
///
@@ -368,10 +375,11 @@ macro_rules! pr_info (
/// yet.
///
/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
-/// `alloc::format!` for information about the formatting syntax.
+/// [`std::format!`] for information about the formatting syntax.
///
/// [`pr_debug`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_debug
/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html
///
/// # Examples
///
@@ -395,11 +403,12 @@ macro_rules! pr_debug (
/// Equivalent to the kernel's [`pr_cont`] macro.
///
/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
-/// `alloc::format!` for information about the formatting syntax.
+/// [`std::format!`] for information about the formatting syntax.
///
/// [`pr_info!`]: crate::pr_info!
/// [`pr_cont`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_cont
/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+/// [`std::format!`]: https://doc.rust-lang.org/std/macro.format.html
///
/// # Examples
///
diff --git a/rust/kernel/rbtree.rs b/rust/kernel/rbtree.rs
index 0d1e75810664..8d978c896747 100644
--- a/rust/kernel/rbtree.rs
+++ b/rust/kernel/rbtree.rs
@@ -424,7 +424,7 @@ where
while !node.is_null() {
// SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self`
// point to the links field of `Node<K, V>` objects.
- let this = unsafe { container_of!(node, Node<K, V>, links) }.cast_mut();
+ let this = unsafe { container_of!(node, Node<K, V>, links) };
// SAFETY: `this` is a non-null node so it is valid by the type invariants.
let this_key = unsafe { &(*this).key };
// SAFETY: `node` is a non-null node so it is valid by the type invariants.
@@ -496,7 +496,7 @@ impl<K, V> Drop for RBTree<K, V> {
// but it is not observable. The loop invariant is still maintained.
// SAFETY: `this` is valid per the loop invariant.
- unsafe { drop(KBox::from_raw(this.cast_mut())) };
+ unsafe { drop(KBox::from_raw(this)) };
}
}
}
@@ -761,7 +761,7 @@ impl<'a, K, V> Cursor<'a, K, V> {
let next = self.get_neighbor_raw(Direction::Next);
// SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self`
// point to the links field of `Node<K, V>` objects.
- let this = unsafe { container_of!(self.current.as_ptr(), Node<K, V>, links) }.cast_mut();
+ let this = unsafe { container_of!(self.current.as_ptr(), Node<K, V>, links) };
// SAFETY: `this` is valid by the type invariants as described above.
let node = unsafe { KBox::from_raw(this) };
let node = RBTreeNode { node };
@@ -806,7 +806,7 @@ impl<'a, K, V> Cursor<'a, K, V> {
unsafe { bindings::rb_erase(neighbor, addr_of_mut!(self.tree.root)) };
// SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self`
// point to the links field of `Node<K, V>` objects.
- let this = unsafe { container_of!(neighbor, Node<K, V>, links) }.cast_mut();
+ let this = unsafe { container_of!(neighbor, Node<K, V>, links) };
// SAFETY: `this` is valid by the type invariants as described above.
let node = unsafe { KBox::from_raw(this) };
return Some(RBTreeNode { node });
@@ -886,7 +886,7 @@ impl<'a, K, V> Cursor<'a, K, V> {
/// # Safety
///
/// - `node` must be a valid pointer to a node in an [`RBTree`].
- /// - The caller has immutable access to `node` for the duration of 'b.
+ /// - The caller has immutable access to `node` for the duration of `'b`.
unsafe fn to_key_value<'b>(node: NonNull<bindings::rb_node>) -> (&'b K, &'b V) {
// SAFETY: the caller guarantees that `node` is a valid pointer in an `RBTree`.
let (k, v) = unsafe { Self::to_key_value_raw(node) };
@@ -897,7 +897,7 @@ impl<'a, K, V> Cursor<'a, K, V> {
/// # Safety
///
/// - `node` must be a valid pointer to a node in an [`RBTree`].
- /// - The caller has mutable access to `node` for the duration of 'b.
+ /// - The caller has mutable access to `node` for the duration of `'b`.
unsafe fn to_key_value_mut<'b>(node: NonNull<bindings::rb_node>) -> (&'b K, &'b mut V) {
// SAFETY: the caller guarantees that `node` is a valid pointer in an `RBTree`.
let (k, v) = unsafe { Self::to_key_value_raw(node) };
@@ -908,11 +908,11 @@ impl<'a, K, V> Cursor<'a, K, V> {
/// # Safety
///
/// - `node` must be a valid pointer to a node in an [`RBTree`].
- /// - The caller has immutable access to the key for the duration of 'b.
+ /// - The caller has immutable access to the key for the duration of `'b`.
unsafe fn to_key_value_raw<'b>(node: NonNull<bindings::rb_node>) -> (&'b K, *mut V) {
// SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self`
// point to the links field of `Node<K, V>` objects.
- let this = unsafe { container_of!(node.as_ptr(), Node<K, V>, links) }.cast_mut();
+ let this = unsafe { container_of!(node.as_ptr(), Node<K, V>, links) };
// SAFETY: The passed `node` is the current node or a non-null neighbor,
// thus `this` is valid by the type invariants.
let k = unsafe { &(*this).key };
@@ -1021,7 +1021,7 @@ impl<K, V> Iterator for IterRaw<K, V> {
// SAFETY: By the type invariant of `IterRaw`, `self.next` is a valid node in an `RBTree`,
// and by the type invariant of `RBTree`, all nodes point to the links field of `Node<K, V>` objects.
- let cur = unsafe { container_of!(self.next, Node<K, V>, links) }.cast_mut();
+ let cur = unsafe { container_of!(self.next, Node<K, V>, links) };
// SAFETY: `self.next` is a valid tree node by the type invariants.
self.next = unsafe { bindings::rb_next(self.next) };
@@ -1168,12 +1168,12 @@ impl<'a, K, V> RawVacantEntry<'a, K, V> {
fn insert(self, node: RBTreeNode<K, V>) -> &'a mut V {
let node = KBox::into_raw(node.node);
- // SAFETY: `node` is valid at least until we call `Box::from_raw`, which only happens when
+ // SAFETY: `node` is valid at least until we call `KBox::from_raw`, which only happens when
// the node is removed or replaced.
let node_links = unsafe { addr_of_mut!((*node).links) };
// INVARIANT: We are linking in a new node, which is valid. It remains valid because we
- // "forgot" it with `Box::into_raw`.
+ // "forgot" it with `KBox::into_raw`.
// SAFETY: The type invariants of `RawVacantEntry` are exactly the safety requirements of `rb_link_node`.
unsafe { bindings::rb_link_node(node_links, self.parent, self.child_field_of_parent) };
@@ -1216,7 +1216,7 @@ impl<'a, K, V> OccupiedEntry<'a, K, V> {
// SAFETY:
// - `self.node_links` is a valid pointer to a node in the tree.
// - We have exclusive access to the underlying tree, and can thus give out a mutable reference.
- unsafe { &mut (*(container_of!(self.node_links, Node<K, V>, links).cast_mut())).value }
+ unsafe { &mut (*(container_of!(self.node_links, Node<K, V>, links))).value }
}
/// Converts the entry into a mutable reference to its value.
@@ -1226,7 +1226,7 @@ impl<'a, K, V> OccupiedEntry<'a, K, V> {
// SAFETY:
// - `self.node_links` is a valid pointer to a node in the tree.
// - This consumes the `&'a mut RBTree<K, V>`, therefore it can give out a mutable reference that lives for `'a`.
- unsafe { &mut (*(container_of!(self.node_links, Node<K, V>, links).cast_mut())).value }
+ unsafe { &mut (*(container_of!(self.node_links, Node<K, V>, links))).value }
}
/// Remove this entry from the [`RBTree`].
@@ -1239,9 +1239,7 @@ impl<'a, K, V> OccupiedEntry<'a, K, V> {
RBTreeNode {
// SAFETY: The node was a node in the tree, but we removed it, so we can convert it
// back into a box.
- node: unsafe {
- KBox::from_raw(container_of!(self.node_links, Node<K, V>, links).cast_mut())
- },
+ node: unsafe { KBox::from_raw(container_of!(self.node_links, Node<K, V>, links)) },
}
}
@@ -1259,7 +1257,7 @@ impl<'a, K, V> OccupiedEntry<'a, K, V> {
fn replace(self, node: RBTreeNode<K, V>) -> RBTreeNode<K, V> {
let node = KBox::into_raw(node.node);
- // SAFETY: `node` is valid at least until we call `Box::from_raw`, which only happens when
+ // SAFETY: `node` is valid at least until we call `KBox::from_raw`, which only happens when
// the node is removed or replaced.
let new_node_links = unsafe { addr_of_mut!((*node).links) };
@@ -1272,8 +1270,7 @@ impl<'a, K, V> OccupiedEntry<'a, K, V> {
// SAFETY:
// - `self.node_ptr` produces a valid pointer to a node in the tree.
// - Now that we removed this entry from the tree, we can convert the node to a box.
- let old_node =
- unsafe { KBox::from_raw(container_of!(self.node_links, Node<K, V>, links).cast_mut()) };
+ let old_node = unsafe { KBox::from_raw(container_of!(self.node_links, Node<K, V>, links)) };
RBTreeNode { node: old_node }
}
diff --git a/rust/kernel/revocable.rs b/rust/kernel/revocable.rs
index 1e5a9d25c21b..06a3cdfce344 100644
--- a/rust/kernel/revocable.rs
+++ b/rust/kernel/revocable.rs
@@ -123,11 +123,41 @@ impl<T> Revocable<T> {
}
}
+ /// Tries to access the wrapped object and run a closure on it while the guard is held.
+ ///
+ /// This is a convenience method to run short non-sleepable code blocks while ensuring the
+ /// guard is dropped afterwards. [`Self::try_access`] carries the risk that the caller will
+ /// forget to explicitly drop that returned guard before calling sleepable code; this method
+ /// adds an extra safety to make sure it doesn't happen.
+ ///
+ /// Returns [`None`] if the object has been revoked and is therefore no longer accessible, or
+ /// the result of the closure wrapped in [`Some`]. If the closure returns a [`Result`] then the
+ /// return type becomes `Option<Result<>>`, which can be inconvenient. Users are encouraged to
+ /// define their own macro that turns the [`Option`] into a proper error code and flattens the
+ /// inner result into it if it makes sense within their subsystem.
+ pub fn try_access_with<R, F: FnOnce(&T) -> R>(&self, f: F) -> Option<R> {
+ self.try_access().map(|t| f(&*t))
+ }
+
+ /// Directly access the revocable wrapped object.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure this [`Revocable`] instance hasn't been revoked and won't be revoked
+ /// as long as the returned `&T` lives.
+ pub unsafe fn access(&self) -> &T {
+ // SAFETY: By the safety requirement of this function it is guaranteed that
+ // `self.data.get()` is a valid pointer to an instance of `T`.
+ unsafe { &*self.data.get() }
+ }
+
/// # Safety
///
/// Callers must ensure that there are no more concurrent users of the revocable object.
- unsafe fn revoke_internal<const SYNC: bool>(&self) {
- if self.is_available.swap(false, Ordering::Relaxed) {
+ unsafe fn revoke_internal<const SYNC: bool>(&self) -> bool {
+ let revoke = self.is_available.swap(false, Ordering::Relaxed);
+
+ if revoke {
if SYNC {
// SAFETY: Just an FFI call, there are no further requirements.
unsafe { bindings::synchronize_rcu() };
@@ -137,6 +167,8 @@ impl<T> Revocable<T> {
// `compare_exchange` above that takes `is_available` from `true` to `false`.
unsafe { drop_in_place(self.data.get()) };
}
+
+ revoke
}
/// Revokes access to and drops the wrapped object.
@@ -144,10 +176,13 @@ impl<T> Revocable<T> {
/// Access to the object is revoked immediately to new callers of [`Revocable::try_access`],
/// expecting that there are no concurrent users of the object.
///
+ /// Returns `true` if `&self` has been revoked with this call, `false` if it was revoked
+ /// already.
+ ///
/// # Safety
///
/// Callers must ensure that there are no more concurrent users of the revocable object.
- pub unsafe fn revoke_nosync(&self) {
+ pub unsafe fn revoke_nosync(&self) -> bool {
// SAFETY: By the safety requirement of this function, the caller ensures that nobody is
// accessing the data anymore and hence we don't have to wait for the grace period to
// finish.
@@ -161,7 +196,10 @@ impl<T> Revocable<T> {
/// If there are concurrent users of the object (i.e., ones that called
/// [`Revocable::try_access`] beforehand and still haven't dropped the returned guard), this
/// function waits for the concurrent access to complete before dropping the wrapped object.
- pub fn revoke(&self) {
+ ///
+ /// Returns `true` if `&self` has been revoked with this call, `false` if it was revoked
+ /// already.
+ pub fn revoke(&self) -> bool {
// SAFETY: By passing `true` we ask `revoke_internal` to wait for the grace period to
// finish.
unsafe { self.revoke_internal::<true>() }
diff --git a/rust/kernel/security.rs b/rust/kernel/security.rs
index 25d2b1ac3833..0c63e9e7e564 100644
--- a/rust/kernel/security.rs
+++ b/rust/kernel/security.rs
@@ -16,13 +16,14 @@ use crate::{
/// # Invariants
///
/// The `ctx` field corresponds to a valid security context as returned by a successful call to
-/// `security_secid_to_secctx`, that has not yet been destroyed by `security_release_secctx`.
+/// `security_secid_to_secctx`, that has not yet been released by `security_release_secctx`.
pub struct SecurityCtx {
ctx: bindings::lsm_context,
}
impl SecurityCtx {
/// Get the security context given its id.
+ #[inline]
pub fn from_secid(secid: u32) -> Result<Self> {
// SAFETY: `struct lsm_context` can be initialized to all zeros.
let mut ctx: bindings::lsm_context = unsafe { core::mem::zeroed() };
@@ -35,16 +36,19 @@ impl SecurityCtx {
}
/// Returns whether the security context is empty.
+ #[inline]
pub fn is_empty(&self) -> bool {
self.ctx.len == 0
}
/// Returns the length of this security context.
+ #[inline]
pub fn len(&self) -> usize {
self.ctx.len as usize
}
/// Returns the bytes for this security context.
+ #[inline]
pub fn as_bytes(&self) -> &[u8] {
let ptr = self.ctx.context;
if ptr.is_null() {
@@ -61,10 +65,10 @@ impl SecurityCtx {
}
impl Drop for SecurityCtx {
+ #[inline]
fn drop(&mut self) {
- // SAFETY: By the invariant of `Self`, this frees a context that came from a successful
- // call to `security_secid_to_secctx` and has not yet been destroyed by
- // `security_release_secctx`.
+ // SAFETY: By the invariant of `Self`, this releases an lsm context that came from a
+ // successful call to `security_secid_to_secctx` and has not yet been released.
unsafe { bindings::security_release_secctx(&mut self.ctx) };
}
}
diff --git a/rust/kernel/seq_file.rs b/rust/kernel/seq_file.rs
index 04947c672979..7a9403eb6e5b 100644
--- a/rust/kernel/seq_file.rs
+++ b/rust/kernel/seq_file.rs
@@ -18,7 +18,7 @@ impl SeqFile {
///
/// # Safety
///
- /// The caller must ensure that for the duration of 'a the following is satisfied:
+ /// The caller must ensure that for the duration of `'a` the following is satisfied:
/// * The pointer points at a valid `struct seq_file`.
/// * The `struct seq_file` is not accessed from any other thread.
pub unsafe fn from_raw<'a>(ptr: *mut bindings::seq_file) -> &'a SeqFile {
@@ -30,6 +30,7 @@ impl SeqFile {
}
/// Used by the [`seq_print`] macro.
+ #[inline]
pub fn call_printf(&self, args: core::fmt::Arguments<'_>) {
// SAFETY: Passing a void pointer to `Arguments` is valid for `%pA`.
unsafe {
diff --git a/rust/kernel/static_assert.rs b/rust/kernel/static_assert.rs
index 3115ee0ba8e9..a57ba14315a0 100644
--- a/rust/kernel/static_assert.rs
+++ b/rust/kernel/static_assert.rs
@@ -6,6 +6,10 @@
///
/// Similar to C11 [`_Static_assert`] and C++11 [`static_assert`].
///
+/// An optional panic message can be supplied after the expression.
+/// Currently only a string literal without formatting is supported
+/// due to constness limitations of the [`assert!`] macro.
+///
/// The feature may be added to Rust in the future: see [RFC 2790].
///
/// [`_Static_assert`]: https://en.cppreference.com/w/c/language/_Static_assert
@@ -25,10 +29,11 @@
/// x + 2
/// }
/// static_assert!(f(40) == 42);
+/// static_assert!(f(40) == 42, "f(x) must add 2 to the given input.");
/// ```
#[macro_export]
macro_rules! static_assert {
- ($condition:expr) => {
- const _: () = core::assert!($condition);
+ ($condition:expr $(,$arg:literal)?) => {
+ const _: () = ::core::assert!($condition $(,$arg)?);
};
}
diff --git a/rust/kernel/std_vendor.rs b/rust/kernel/std_vendor.rs
index 279bd353687a..abbab5050cc5 100644
--- a/rust/kernel/std_vendor.rs
+++ b/rust/kernel/std_vendor.rs
@@ -148,7 +148,7 @@ macro_rules! dbg {
};
($val:expr $(,)?) => {
// Use of `match` here is intentional because it affects the lifetimes
- // of temporaries - https://stackoverflow.com/a/48732525/1063961
+ // of temporaries - <https://stackoverflow.com/a/48732525/1063961>
match $val {
tmp => {
$crate::pr_info!("[{}:{}:{}] {} = {:#?}\n",
diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs
index 28e2201604d6..a927db8e079c 100644
--- a/rust/kernel/str.rs
+++ b/rust/kernel/str.rs
@@ -6,7 +6,7 @@ use crate::alloc::{flags::*, AllocError, KVec};
use core::fmt::{self, Write};
use core::ops::{self, Deref, DerefMut, Index};
-use crate::error::{code::*, Error};
+use crate::prelude::*;
/// Byte string without UTF-8 validity guarantee.
#[repr(transparent)]
@@ -31,6 +31,23 @@ impl BStr {
// SAFETY: `BStr` is transparent to `[u8]`.
unsafe { &*(bytes as *const [u8] as *const BStr) }
}
+
+ /// Strip a prefix from `self`. Delegates to [`slice::strip_prefix`].
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// # use kernel::b_str;
+ /// assert_eq!(Some(b_str!("bar")), b_str!("foobar").strip_prefix(b_str!("foo")));
+ /// assert_eq!(None, b_str!("foobar").strip_prefix(b_str!("bar")));
+ /// assert_eq!(Some(b_str!("foobar")), b_str!("foobar").strip_prefix(b_str!("")));
+ /// assert_eq!(Some(b_str!("")), b_str!("foobar").strip_prefix(b_str!("foobar")));
+ /// ```
+ pub fn strip_prefix(&self, pattern: impl AsRef<Self>) -> Option<&BStr> {
+ self.deref()
+ .strip_prefix(pattern.as_ref().deref())
+ .map(Self::from_bytes)
+ }
}
impl fmt::Display for BStr {
@@ -56,7 +73,7 @@ impl fmt::Display for BStr {
b'\r' => f.write_str("\\r")?,
// Printable characters.
0x20..=0x7e => f.write_char(b as char)?,
- _ => write!(f, "\\x{:02x}", b)?,
+ _ => write!(f, "\\x{b:02x}")?,
}
}
Ok(())
@@ -92,7 +109,7 @@ impl fmt::Debug for BStr {
b'\\' => f.write_str("\\\\")?,
// Printable characters.
0x20..=0x7e => f.write_char(b as char)?,
- _ => write!(f, "\\x{:02x}", b)?,
+ _ => write!(f, "\\x{b:02x}")?,
}
}
f.write_char('"')
@@ -108,6 +125,35 @@ impl Deref for BStr {
}
}
+impl PartialEq for BStr {
+ fn eq(&self, other: &Self) -> bool {
+ self.deref().eq(other.deref())
+ }
+}
+
+impl<Idx> Index<Idx> for BStr
+where
+ [u8]: Index<Idx, Output = [u8]>,
+{
+ type Output = Self;
+
+ fn index(&self, index: Idx) -> &Self::Output {
+ BStr::from_bytes(&self.0[index])
+ }
+}
+
+impl AsRef<BStr> for [u8] {
+ fn as_ref(&self) -> &BStr {
+ BStr::from_bytes(self)
+ }
+}
+
+impl AsRef<BStr> for BStr {
+ fn as_ref(&self) -> &BStr {
+ self
+ }
+}
+
/// Creates a new [`BStr`] from a string literal.
///
/// `b_str!` converts the supplied string literal to byte string, so non-ASCII
@@ -401,7 +447,7 @@ impl fmt::Display for CStr {
// Printable character.
f.write_char(c as char)?;
} else {
- write!(f, "\\x{:02x}", c)?;
+ write!(f, "\\x{c:02x}")?;
}
}
Ok(())
@@ -433,7 +479,7 @@ impl fmt::Debug for CStr {
// Printable characters.
b'\"' => f.write_str("\\\"")?,
0x20..=0x7e => f.write_char(c as char)?,
- _ => write!(f, "\\x{:02x}", c)?,
+ _ => write!(f, "\\x{c:02x}")?,
}
}
f.write_str("\"")
@@ -526,30 +572,13 @@ macro_rules! c_str {
}};
}
-#[cfg(test)]
-#[expect(clippy::items_after_test_module)]
+#[kunit_tests(rust_kernel_str)]
mod tests {
use super::*;
- struct String(CString);
-
- impl String {
- fn from_fmt(args: fmt::Arguments<'_>) -> Self {
- String(CString::try_from_fmt(args).unwrap())
- }
- }
-
- impl Deref for String {
- type Target = str;
-
- fn deref(&self) -> &str {
- self.0.to_str().unwrap()
- }
- }
-
macro_rules! format {
($($f:tt)*) => ({
- &*String::from_fmt(kernel::fmt!($($f)*))
+ CString::try_from_fmt(::kernel::fmt!($($f)*))?.to_str()?
})
}
@@ -568,91 +597,98 @@ mod tests {
\\xf0\\xf1\\xf2\\xf3\\xf4\\xf5\\xf6\\xf7\\xf8\\xf9\\xfa\\xfb\\xfc\\xfd\\xfe\\xff";
#[test]
- fn test_cstr_to_str() {
+ fn test_cstr_to_str() -> Result {
let good_bytes = b"\xf0\x9f\xa6\x80\0";
- let checked_cstr = CStr::from_bytes_with_nul(good_bytes).unwrap();
- let checked_str = checked_cstr.to_str().unwrap();
+ let checked_cstr = CStr::from_bytes_with_nul(good_bytes)?;
+ let checked_str = checked_cstr.to_str()?;
assert_eq!(checked_str, "🦀");
+ Ok(())
}
#[test]
- #[should_panic]
- fn test_cstr_to_str_panic() {
+ fn test_cstr_to_str_invalid_utf8() -> Result {
let bad_bytes = b"\xc3\x28\0";
- let checked_cstr = CStr::from_bytes_with_nul(bad_bytes).unwrap();
- checked_cstr.to_str().unwrap();
+ let checked_cstr = CStr::from_bytes_with_nul(bad_bytes)?;
+ assert!(checked_cstr.to_str().is_err());
+ Ok(())
}
#[test]
- fn test_cstr_as_str_unchecked() {
+ fn test_cstr_as_str_unchecked() -> Result {
let good_bytes = b"\xf0\x9f\x90\xA7\0";
- let checked_cstr = CStr::from_bytes_with_nul(good_bytes).unwrap();
+ let checked_cstr = CStr::from_bytes_with_nul(good_bytes)?;
// SAFETY: The contents come from a string literal which contains valid UTF-8.
let unchecked_str = unsafe { checked_cstr.as_str_unchecked() };
assert_eq!(unchecked_str, "🐧");
+ Ok(())
}
#[test]
- fn test_cstr_display() {
- let hello_world = CStr::from_bytes_with_nul(b"hello, world!\0").unwrap();
- assert_eq!(format!("{}", hello_world), "hello, world!");
- let non_printables = CStr::from_bytes_with_nul(b"\x01\x09\x0a\0").unwrap();
- assert_eq!(format!("{}", non_printables), "\\x01\\x09\\x0a");
- let non_ascii = CStr::from_bytes_with_nul(b"d\xe9j\xe0 vu\0").unwrap();
- assert_eq!(format!("{}", non_ascii), "d\\xe9j\\xe0 vu");
- let good_bytes = CStr::from_bytes_with_nul(b"\xf0\x9f\xa6\x80\0").unwrap();
- assert_eq!(format!("{}", good_bytes), "\\xf0\\x9f\\xa6\\x80");
+ fn test_cstr_display() -> Result {
+ let hello_world = CStr::from_bytes_with_nul(b"hello, world!\0")?;
+ assert_eq!(format!("{hello_world}"), "hello, world!");
+ let non_printables = CStr::from_bytes_with_nul(b"\x01\x09\x0a\0")?;
+ assert_eq!(format!("{non_printables}"), "\\x01\\x09\\x0a");
+ let non_ascii = CStr::from_bytes_with_nul(b"d\xe9j\xe0 vu\0")?;
+ assert_eq!(format!("{non_ascii}"), "d\\xe9j\\xe0 vu");
+ let good_bytes = CStr::from_bytes_with_nul(b"\xf0\x9f\xa6\x80\0")?;
+ assert_eq!(format!("{good_bytes}"), "\\xf0\\x9f\\xa6\\x80");
+ Ok(())
}
#[test]
- fn test_cstr_display_all_bytes() {
+ fn test_cstr_display_all_bytes() -> Result {
let mut bytes: [u8; 256] = [0; 256];
// fill `bytes` with [1..=255] + [0]
for i in u8::MIN..=u8::MAX {
bytes[i as usize] = i.wrapping_add(1);
}
- let cstr = CStr::from_bytes_with_nul(&bytes).unwrap();
- assert_eq!(format!("{}", cstr), ALL_ASCII_CHARS);
+ let cstr = CStr::from_bytes_with_nul(&bytes)?;
+ assert_eq!(format!("{cstr}"), ALL_ASCII_CHARS);
+ Ok(())
}
#[test]
- fn test_cstr_debug() {
- let hello_world = CStr::from_bytes_with_nul(b"hello, world!\0").unwrap();
- assert_eq!(format!("{:?}", hello_world), "\"hello, world!\"");
- let non_printables = CStr::from_bytes_with_nul(b"\x01\x09\x0a\0").unwrap();
- assert_eq!(format!("{:?}", non_printables), "\"\\x01\\x09\\x0a\"");
- let non_ascii = CStr::from_bytes_with_nul(b"d\xe9j\xe0 vu\0").unwrap();
- assert_eq!(format!("{:?}", non_ascii), "\"d\\xe9j\\xe0 vu\"");
- let good_bytes = CStr::from_bytes_with_nul(b"\xf0\x9f\xa6\x80\0").unwrap();
- assert_eq!(format!("{:?}", good_bytes), "\"\\xf0\\x9f\\xa6\\x80\"");
+ fn test_cstr_debug() -> Result {
+ let hello_world = CStr::from_bytes_with_nul(b"hello, world!\0")?;
+ assert_eq!(format!("{hello_world:?}"), "\"hello, world!\"");
+ let non_printables = CStr::from_bytes_with_nul(b"\x01\x09\x0a\0")?;
+ assert_eq!(format!("{non_printables:?}"), "\"\\x01\\x09\\x0a\"");
+ let non_ascii = CStr::from_bytes_with_nul(b"d\xe9j\xe0 vu\0")?;
+ assert_eq!(format!("{non_ascii:?}"), "\"d\\xe9j\\xe0 vu\"");
+ let good_bytes = CStr::from_bytes_with_nul(b"\xf0\x9f\xa6\x80\0")?;
+ assert_eq!(format!("{good_bytes:?}"), "\"\\xf0\\x9f\\xa6\\x80\"");
+ Ok(())
}
#[test]
- fn test_bstr_display() {
+ fn test_bstr_display() -> Result {
let hello_world = BStr::from_bytes(b"hello, world!");
- assert_eq!(format!("{}", hello_world), "hello, world!");
+ assert_eq!(format!("{hello_world}"), "hello, world!");
let escapes = BStr::from_bytes(b"_\t_\n_\r_\\_\'_\"_");
- assert_eq!(format!("{}", escapes), "_\\t_\\n_\\r_\\_'_\"_");
+ assert_eq!(format!("{escapes}"), "_\\t_\\n_\\r_\\_'_\"_");
let others = BStr::from_bytes(b"\x01");
- assert_eq!(format!("{}", others), "\\x01");
+ assert_eq!(format!("{others}"), "\\x01");
let non_ascii = BStr::from_bytes(b"d\xe9j\xe0 vu");
- assert_eq!(format!("{}", non_ascii), "d\\xe9j\\xe0 vu");
+ assert_eq!(format!("{non_ascii}"), "d\\xe9j\\xe0 vu");
let good_bytes = BStr::from_bytes(b"\xf0\x9f\xa6\x80");
- assert_eq!(format!("{}", good_bytes), "\\xf0\\x9f\\xa6\\x80");
+ assert_eq!(format!("{good_bytes}"), "\\xf0\\x9f\\xa6\\x80");
+ Ok(())
}
#[test]
- fn test_bstr_debug() {
+ fn test_bstr_debug() -> Result {
let hello_world = BStr::from_bytes(b"hello, world!");
- assert_eq!(format!("{:?}", hello_world), "\"hello, world!\"");
+ assert_eq!(format!("{hello_world:?}"), "\"hello, world!\"");
let escapes = BStr::from_bytes(b"_\t_\n_\r_\\_\'_\"_");
- assert_eq!(format!("{:?}", escapes), "\"_\\t_\\n_\\r_\\\\_'_\\\"_\"");
+ assert_eq!(format!("{escapes:?}"), "\"_\\t_\\n_\\r_\\\\_'_\\\"_\"");
let others = BStr::from_bytes(b"\x01");
- assert_eq!(format!("{:?}", others), "\"\\x01\"");
+ assert_eq!(format!("{others:?}"), "\"\\x01\"");
let non_ascii = BStr::from_bytes(b"d\xe9j\xe0 vu");
- assert_eq!(format!("{:?}", non_ascii), "\"d\\xe9j\\xe0 vu\"");
+ assert_eq!(format!("{non_ascii:?}"), "\"d\\xe9j\\xe0 vu\"");
let good_bytes = BStr::from_bytes(b"\xf0\x9f\xa6\x80");
- assert_eq!(format!("{:?}", good_bytes), "\"\\xf0\\x9f\\xa6\\x80\"");
+ assert_eq!(format!("{good_bytes:?}"), "\"\\xf0\\x9f\\xa6\\x80\"");
+ Ok(())
}
}
@@ -706,7 +742,7 @@ impl RawFormatter {
/// for the lifetime of the returned [`RawFormatter`].
pub(crate) unsafe fn from_buffer(buf: *mut u8, len: usize) -> Self {
let pos = buf as usize;
- // INVARIANT: We ensure that `end` is never less then `buf`, and the safety requirements
+ // INVARIANT: We ensure that `end` is never less than `buf`, and the safety requirements
// guarantees that the memory region is valid for writes.
Self {
pos,
@@ -840,7 +876,7 @@ impl CString {
// SAFETY: The number of bytes that can be written to `f` is bounded by `size`, which is
// `buf`'s capacity. The contents of the buffer have been initialised by writes to `f`.
- unsafe { buf.set_len(f.bytes_written()) };
+ unsafe { buf.inc_len(f.bytes_written()) };
// Check that there are no `NUL` bytes before the end.
// SAFETY: The buffer is valid for read because `f.bytes_written()` is bounded by `size`
@@ -898,5 +934,5 @@ impl fmt::Debug for CString {
/// A convenience alias for [`core::format_args`].
#[macro_export]
macro_rules! fmt {
- ($($f:tt)*) => ( core::format_args!($($f)*) )
+ ($($f:tt)*) => ( ::core::format_args!($($f)*) )
}
diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs
index 3498fb344dc9..c23a12639924 100644
--- a/rust/kernel/sync.rs
+++ b/rust/kernel/sync.rs
@@ -5,9 +5,12 @@
//! This module contains the kernel APIs related to synchronisation that have been ported or
//! wrapped for usage by Rust code in the kernel.
+use crate::prelude::*;
use crate::types::Opaque;
+use pin_init;
mod arc;
+pub mod completion;
mod condvar;
pub mod lock;
mod locked_by;
@@ -15,6 +18,7 @@ pub mod poll;
pub mod rcu;
pub use arc::{Arc, ArcBorrow, UniqueArc};
+pub use completion::Completion;
pub use condvar::{new_condvar, CondVar, CondVarTimeoutResult};
pub use lock::global::{global_lock, GlobalGuard, GlobalLock, GlobalLockBackend, GlobalLockedBy};
pub use lock::mutex::{new_mutex, Mutex, MutexGuard};
@@ -23,26 +27,65 @@ pub use locked_by::LockedBy;
/// Represents a lockdep class. It's a wrapper around C's `lock_class_key`.
#[repr(transparent)]
-pub struct LockClassKey(Opaque<bindings::lock_class_key>);
+#[pin_data(PinnedDrop)]
+pub struct LockClassKey {
+ #[pin]
+ inner: Opaque<bindings::lock_class_key>,
+}
// SAFETY: `bindings::lock_class_key` is designed to be used concurrently from multiple threads and
// provides its own synchronization.
unsafe impl Sync for LockClassKey {}
impl LockClassKey {
- /// Creates a new lock class key.
- pub const fn new() -> Self {
- Self(Opaque::uninit())
+ /// Initializes a dynamically allocated lock class key. In the common case of using a
+ /// statically allocated lock class key, the static_lock_class! macro should be used instead.
+ ///
+ /// # Example
+ /// ```
+ /// # use kernel::c_str;
+ /// # use kernel::alloc::KBox;
+ /// # use kernel::types::ForeignOwnable;
+ /// # use kernel::sync::{LockClassKey, SpinLock};
+ /// # use pin_init::stack_pin_init;
+ ///
+ /// let key = KBox::pin_init(LockClassKey::new_dynamic(), GFP_KERNEL)?;
+ /// let key_ptr = key.into_foreign();
+ ///
+ /// {
+ /// stack_pin_init!(let num: SpinLock<u32> = SpinLock::new(
+ /// 0,
+ /// c_str!("my_spinlock"),
+ /// // SAFETY: `key_ptr` is returned by the above `into_foreign()`, whose
+ /// // `from_foreign()` has not yet been called.
+ /// unsafe { <Pin<KBox<LockClassKey>> as ForeignOwnable>::borrow(key_ptr) }
+ /// ));
+ /// }
+ ///
+ /// // SAFETY: We dropped `num`, the only use of the key, so the result of the previous
+ /// // `borrow` has also been dropped. Thus, it's safe to use from_foreign.
+ /// unsafe { drop(<Pin<KBox<LockClassKey>> as ForeignOwnable>::from_foreign(key_ptr)) };
+ ///
+ /// # Ok::<(), Error>(())
+ /// ```
+ pub fn new_dynamic() -> impl PinInit<Self> {
+ pin_init!(Self {
+ // SAFETY: lockdep_register_key expects an uninitialized block of memory
+ inner <- Opaque::ffi_init(|slot| unsafe { bindings::lockdep_register_key(slot) })
+ })
}
pub(crate) fn as_ptr(&self) -> *mut bindings::lock_class_key {
- self.0.get()
+ self.inner.get()
}
}
-impl Default for LockClassKey {
- fn default() -> Self {
- Self::new()
+#[pinned_drop]
+impl PinnedDrop for LockClassKey {
+ fn drop(self: Pin<&mut Self>) {
+ // SAFETY: self.as_ptr was registered with lockdep and self is pinned, so the address
+ // hasn't changed. Thus, it's safe to pass to unregister.
+ unsafe { bindings::lockdep_unregister_key(self.as_ptr()) }
}
}
@@ -51,8 +94,11 @@ impl Default for LockClassKey {
#[macro_export]
macro_rules! static_lock_class {
() => {{
- static CLASS: $crate::sync::LockClassKey = $crate::sync::LockClassKey::new();
- &CLASS
+ static CLASS: $crate::sync::LockClassKey =
+ // SAFETY: lockdep expects uninitialized memory when it's handed a statically allocated
+ // lock_class_key
+ unsafe { ::core::mem::MaybeUninit::uninit().assume_init() };
+ $crate::prelude::Pin::static_ref(&CLASS)
}};
}
diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs
index 3cefda7a4372..c7af0aa48a0a 100644
--- a/rust/kernel/sync/arc.rs
+++ b/rust/kernel/sync/arc.rs
@@ -19,7 +19,7 @@
use crate::{
alloc::{AllocError, Flags, KBox},
bindings,
- init::{self, InPlaceInit, Init, PinInit},
+ init::InPlaceInit,
try_init,
types::{ForeignOwnable, Opaque},
};
@@ -32,7 +32,7 @@ use core::{
pin::Pin,
ptr::NonNull,
};
-use macros::pin_data;
+use pin_init::{self, pin_data, InPlaceWrite, Init, PinInit};
mod std_vendor;
@@ -135,14 +135,15 @@ pub struct Arc<T: ?Sized> {
// meaningful with respect to dropck - but this may change in the future so this is left here
// out of an abundance of caution.
//
- // See https://doc.rust-lang.org/nomicon/phantom-data.html#generic-parameters-and-drop-checking
+ // See <https://doc.rust-lang.org/nomicon/phantom-data.html#generic-parameters-and-drop-checking>
// for more detail on the semantics of dropck in the presence of `PhantomData`.
_p: PhantomData<ArcInner<T>>,
}
+#[doc(hidden)]
#[pin_data]
#[repr(C)]
-struct ArcInner<T: ?Sized> {
+pub struct ArcInner<T: ?Sized> {
refcount: Opaque<bindings::refcount_t>,
data: T,
}
@@ -202,6 +203,26 @@ unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {}
// the reference count reaches zero and `T` is dropped.
unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {}
+impl<T> InPlaceInit<T> for Arc<T> {
+ type PinnedSelf = Self;
+
+ #[inline]
+ fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
+ where
+ E: From<AllocError>,
+ {
+ UniqueArc::try_pin_init(init, flags).map(|u| u.into())
+ }
+
+ #[inline]
+ fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
+ where
+ E: From<AllocError>,
+ {
+ UniqueArc::try_init(init, flags).map(|u| u.into())
+ }
+}
+
impl<T> Arc<T> {
/// Constructs a new reference counted instance of `T`.
pub fn new(contents: T, flags: Flags) -> Result<Self, AllocError> {
@@ -246,6 +267,15 @@ impl<T: ?Sized> Arc<T> {
unsafe { core::ptr::addr_of!((*ptr).data) }
}
+ /// Return a raw pointer to the data in this arc.
+ pub fn as_ptr(this: &Self) -> *const T {
+ let ptr = this.ptr.as_ptr();
+
+ // SAFETY: As `ptr` points to a valid allocation of type `ArcInner`,
+ // field projection to `data`is within bounds of the allocation.
+ unsafe { core::ptr::addr_of!((*ptr).data) }
+ }
+
/// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`].
///
/// # Safety
@@ -342,18 +372,20 @@ impl<T: ?Sized> Arc<T> {
}
}
-impl<T: 'static> ForeignOwnable for Arc<T> {
+// SAFETY: The `into_foreign` function returns a pointer that is well-aligned.
+unsafe impl<T: 'static> ForeignOwnable for Arc<T> {
+ type PointedTo = ArcInner<T>;
type Borrowed<'a> = ArcBorrow<'a, T>;
type BorrowedMut<'a> = Self::Borrowed<'a>;
- fn into_foreign(self) -> *mut crate::ffi::c_void {
- ManuallyDrop::new(self).ptr.as_ptr().cast()
+ fn into_foreign(self) -> *mut Self::PointedTo {
+ ManuallyDrop::new(self).ptr.as_ptr()
}
- unsafe fn from_foreign(ptr: *mut crate::ffi::c_void) -> Self {
+ unsafe fn from_foreign(ptr: *mut Self::PointedTo) -> Self {
// SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous
// call to `Self::into_foreign`.
- let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) };
+ let inner = unsafe { NonNull::new_unchecked(ptr) };
// SAFETY: By the safety requirement of this function, we know that `ptr` came from
// a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and
@@ -361,17 +393,17 @@ impl<T: 'static> ForeignOwnable for Arc<T> {
unsafe { Self::from_inner(inner) }
}
- unsafe fn borrow<'a>(ptr: *mut crate::ffi::c_void) -> ArcBorrow<'a, T> {
+ unsafe fn borrow<'a>(ptr: *mut Self::PointedTo) -> ArcBorrow<'a, T> {
// SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous
// call to `Self::into_foreign`.
- let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) };
+ let inner = unsafe { NonNull::new_unchecked(ptr) };
// SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive
// for the lifetime of the returned value.
unsafe { ArcBorrow::new(inner) }
}
- unsafe fn borrow_mut<'a>(ptr: *mut crate::ffi::c_void) -> ArcBorrow<'a, T> {
+ unsafe fn borrow_mut<'a>(ptr: *mut Self::PointedTo) -> ArcBorrow<'a, T> {
// SAFETY: The safety requirements for `borrow_mut` are a superset of the safety
// requirements for `borrow`.
unsafe { Self::borrow(ptr) }
@@ -460,7 +492,7 @@ impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> {
/// There are no mutable references to the underlying [`Arc`], and it remains valid for the
/// lifetime of the [`ArcBorrow`] instance.
///
-/// # Example
+/// # Examples
///
/// ```
/// use kernel::sync::{Arc, ArcBorrow};
@@ -539,11 +571,11 @@ impl<T: ?Sized> ArcBorrow<'_, T> {
}
/// Creates an [`ArcBorrow`] to an [`Arc`] that has previously been deconstructed with
- /// [`Arc::into_raw`].
+ /// [`Arc::into_raw`] or [`Arc::as_ptr`].
///
/// # Safety
///
- /// * The provided pointer must originate from a call to [`Arc::into_raw`].
+ /// * The provided pointer must originate from a call to [`Arc::into_raw`] or [`Arc::as_ptr`].
/// * For the duration of the lifetime annotated on this `ArcBorrow`, the reference count must
/// not hit zero.
/// * For the duration of the lifetime annotated on this `ArcBorrow`, there must not be a
@@ -659,6 +691,48 @@ pub struct UniqueArc<T: ?Sized> {
inner: Arc<T>,
}
+impl<T> InPlaceInit<T> for UniqueArc<T> {
+ type PinnedSelf = Pin<Self>;
+
+ #[inline]
+ fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
+ where
+ E: From<AllocError>,
+ {
+ UniqueArc::new_uninit(flags)?.write_pin_init(init)
+ }
+
+ #[inline]
+ fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
+ where
+ E: From<AllocError>,
+ {
+ UniqueArc::new_uninit(flags)?.write_init(init)
+ }
+}
+
+impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> {
+ type Initialized = UniqueArc<T>;
+
+ fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
+ let slot = self.as_mut_ptr();
+ // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
+ // slot is valid.
+ unsafe { init.__init(slot)? };
+ // SAFETY: All fields have been initialized.
+ Ok(unsafe { self.assume_init() })
+ }
+
+ fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
+ let slot = self.as_mut_ptr();
+ // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
+ // slot is valid and will not be moved, because we pin it later.
+ unsafe { init.__pinned_init(slot)? };
+ // SAFETY: All fields have been initialized.
+ Ok(unsafe { self.assume_init() }.into())
+ }
+}
+
impl<T> UniqueArc<T> {
/// Tries to allocate a new [`UniqueArc`] instance.
pub fn new(value: T, flags: Flags) -> Result<Self, AllocError> {
@@ -675,7 +749,7 @@ impl<T> UniqueArc<T> {
try_init!(ArcInner {
// SAFETY: There are no safety requirements for this FFI call.
refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }),
- data <- init::uninit::<T, AllocError>(),
+ data <- pin_init::uninit::<T, AllocError>(),
}? AllocError),
flags,
)?;
diff --git a/rust/kernel/sync/completion.rs b/rust/kernel/sync/completion.rs
new file mode 100644
index 000000000000..c50012a940a3
--- /dev/null
+++ b/rust/kernel/sync/completion.rs
@@ -0,0 +1,112 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Completion support.
+//!
+//! Reference: <https://docs.kernel.org/scheduler/completion.html>
+//!
+//! C header: [`include/linux/completion.h`](srctree/include/linux/completion.h)
+
+use crate::{bindings, prelude::*, types::Opaque};
+
+/// Synchronization primitive to signal when a certain task has been completed.
+///
+/// The [`Completion`] synchronization primitive signals when a certain task has been completed by
+/// waking up other tasks that have been queued up to wait for the [`Completion`] to be completed.
+///
+/// # Examples
+///
+/// ```
+/// use kernel::sync::{Arc, Completion};
+/// use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem};
+///
+/// #[pin_data]
+/// struct MyTask {
+/// #[pin]
+/// work: Work<MyTask>,
+/// #[pin]
+/// done: Completion,
+/// }
+///
+/// impl_has_work! {
+/// impl HasWork<Self> for MyTask { self.work }
+/// }
+///
+/// impl MyTask {
+/// fn new() -> Result<Arc<Self>> {
+/// let this = Arc::pin_init(pin_init!(MyTask {
+/// work <- new_work!("MyTask::work"),
+/// done <- Completion::new(),
+/// }), GFP_KERNEL)?;
+///
+/// let _ = workqueue::system().enqueue(this.clone());
+///
+/// Ok(this)
+/// }
+///
+/// fn wait_for_completion(&self) {
+/// self.done.wait_for_completion();
+///
+/// pr_info!("Completion: task complete\n");
+/// }
+/// }
+///
+/// impl WorkItem for MyTask {
+/// type Pointer = Arc<MyTask>;
+///
+/// fn run(this: Arc<MyTask>) {
+/// // process this task
+/// this.done.complete_all();
+/// }
+/// }
+///
+/// let task = MyTask::new()?;
+/// task.wait_for_completion();
+/// # Ok::<(), Error>(())
+/// ```
+#[pin_data]
+pub struct Completion {
+ #[pin]
+ inner: Opaque<bindings::completion>,
+}
+
+// SAFETY: `Completion` is safe to be send to any task.
+unsafe impl Send for Completion {}
+
+// SAFETY: `Completion` is safe to be accessed concurrently.
+unsafe impl Sync for Completion {}
+
+impl Completion {
+ /// Create an initializer for a new [`Completion`].
+ pub fn new() -> impl PinInit<Self> {
+ pin_init!(Self {
+ inner <- Opaque::ffi_init(|slot: *mut bindings::completion| {
+ // SAFETY: `slot` is a valid pointer to an uninitialized `struct completion`.
+ unsafe { bindings::init_completion(slot) };
+ }),
+ })
+ }
+
+ fn as_raw(&self) -> *mut bindings::completion {
+ self.inner.get()
+ }
+
+ /// Signal all tasks waiting on this completion.
+ ///
+ /// This method wakes up all tasks waiting on this completion; after this operation the
+ /// completion is permanently done, i.e. signals all current and future waiters.
+ pub fn complete_all(&self) {
+ // SAFETY: `self.as_raw()` is a pointer to a valid `struct completion`.
+ unsafe { bindings::complete_all(self.as_raw()) };
+ }
+
+ /// Wait for completion of a task.
+ ///
+ /// This method waits for the completion of a task; it is not interruptible and there is no
+ /// timeout.
+ ///
+ /// See also [`Completion::complete_all`].
+ pub fn wait_for_completion(&self) {
+ // SAFETY: `self.as_raw()` is a pointer to a valid `struct completion`.
+ unsafe { bindings::wait_for_completion(self.as_raw()) };
+ }
+}
diff --git a/rust/kernel/sync/condvar.rs b/rust/kernel/sync/condvar.rs
index 7df565038d7d..caebf03f553b 100644
--- a/rust/kernel/sync/condvar.rs
+++ b/rust/kernel/sync/condvar.rs
@@ -8,16 +8,15 @@
use super::{lock::Backend, lock::Guard, LockClassKey};
use crate::{
ffi::{c_int, c_long},
- init::PinInit,
- pin_init,
str::CStr,
- task::{MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE, TASK_NORMAL, TASK_UNINTERRUPTIBLE},
+ task::{
+ MAX_SCHEDULE_TIMEOUT, TASK_FREEZABLE, TASK_INTERRUPTIBLE, TASK_NORMAL, TASK_UNINTERRUPTIBLE,
+ },
time::Jiffies,
types::Opaque,
};
-use core::marker::PhantomPinned;
-use core::ptr;
-use macros::pin_data;
+use core::{marker::PhantomPinned, pin::Pin, ptr};
+use pin_init::{pin_data, pin_init, PinInit};
/// Creates a [`CondVar`] initialiser with the given name and a newly-created lock class.
#[macro_export]
@@ -37,7 +36,7 @@ pub use new_condvar;
/// spuriously.
///
/// Instances of [`CondVar`] need a lock class and to be pinned. The recommended way to create such
-/// instances is with the [`pin_init`](crate::pin_init) and [`new_condvar`] macros.
+/// instances is with the [`pin_init`](crate::pin_init!) and [`new_condvar`] macros.
///
/// # Examples
///
@@ -101,7 +100,7 @@ unsafe impl Sync for CondVar {}
impl CondVar {
/// Constructs a new condvar initialiser.
- pub fn new(name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> {
+ pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self> {
pin_init!(Self {
_pin: PhantomPinned,
// SAFETY: `slot` is valid while the closure is called and both `name` and `key` have
@@ -159,6 +158,25 @@ impl CondVar {
crate::current!().signal_pending()
}
+ /// Releases the lock and waits for a notification in interruptible and freezable mode.
+ ///
+ /// The process is allowed to be frozen during this sleep. No lock should be held when calling
+ /// this function, and there is a lockdep assertion for this. Freezing a task that holds a lock
+ /// can trivially deadlock vs another task that needs that lock to complete before it too can
+ /// hit freezable.
+ #[must_use = "wait_interruptible_freezable returns if a signal is pending, so the caller must check the return value"]
+ pub fn wait_interruptible_freezable<T: ?Sized, B: Backend>(
+ &self,
+ guard: &mut Guard<'_, T, B>,
+ ) -> bool {
+ self.wait_internal(
+ TASK_INTERRUPTIBLE | TASK_FREEZABLE,
+ guard,
+ MAX_SCHEDULE_TIMEOUT,
+ );
+ crate::current!().signal_pending()
+ }
+
/// Releases the lock and waits for a notification in interruptible mode.
///
/// Atomically releases the given lock (whose ownership is proven by the guard) and puts the
diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs
index eb80048e0110..e82fa5be289c 100644
--- a/rust/kernel/sync/lock.rs
+++ b/rust/kernel/sync/lock.rs
@@ -7,13 +7,11 @@
use super::LockClassKey;
use crate::{
- init::PinInit,
- pin_init,
str::CStr,
types::{NotThreadSafe, Opaque, ScopeGuard},
};
-use core::{cell::UnsafeCell, marker::PhantomPinned};
-use macros::pin_data;
+use core::{cell::UnsafeCell, marker::PhantomPinned, pin::Pin};
+use pin_init::{pin_data, pin_init, PinInit};
pub mod mutex;
pub mod spinlock;
@@ -129,7 +127,7 @@ unsafe impl<T: ?Sized + Send, B: Backend> Sync for Lock<T, B> {}
impl<T, B: Backend> Lock<T, B> {
/// Constructs a new lock initialiser.
- pub fn new(t: T, name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> {
+ pub fn new(t: T, name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self> {
pin_init!(Self {
data: UnsafeCell::new(t),
_pin: PhantomPinned,
@@ -199,7 +197,37 @@ pub struct Guard<'a, T: ?Sized, B: Backend> {
// SAFETY: `Guard` is sync when the data protected by the lock is also sync.
unsafe impl<T: Sync + ?Sized, B: Backend> Sync for Guard<'_, T, B> {}
-impl<T: ?Sized, B: Backend> Guard<'_, T, B> {
+impl<'a, T: ?Sized, B: Backend> Guard<'a, T, B> {
+ /// Returns the lock that this guard originates from.
+ ///
+ /// # Examples
+ ///
+ /// The following example shows how to use [`Guard::lock_ref()`] to assert the corresponding
+ /// lock is held.
+ ///
+ /// ```
+ /// # use kernel::{new_spinlock, sync::lock::{Backend, Guard, Lock}};
+ /// # use pin_init::stack_pin_init;
+ ///
+ /// fn assert_held<T, B: Backend>(guard: &Guard<'_, T, B>, lock: &Lock<T, B>) {
+ /// // Address-equal means the same lock.
+ /// assert!(core::ptr::eq(guard.lock_ref(), lock));
+ /// }
+ ///
+ /// // Creates a new lock on the stack.
+ /// stack_pin_init!{
+ /// let l = new_spinlock!(42)
+ /// }
+ ///
+ /// let g = l.lock();
+ ///
+ /// // `g` originates from `l`.
+ /// assert_held(&g, &l);
+ /// ```
+ pub fn lock_ref(&self) -> &'a Lock<T, B> {
+ self.lock
+ }
+
pub(crate) fn do_unlocked<U>(&mut self, cb: impl FnOnce() -> U) -> U {
// SAFETY: The caller owns the lock, so it is safe to unlock it.
unsafe { B::unlock(self.lock.state.get(), &self.state) };
diff --git a/rust/kernel/sync/lock/global.rs b/rust/kernel/sync/lock/global.rs
index 480ee724e3cc..d65f94b5caf2 100644
--- a/rust/kernel/sync/lock/global.rs
+++ b/rust/kernel/sync/lock/global.rs
@@ -13,6 +13,7 @@ use crate::{
use core::{
cell::UnsafeCell,
marker::{PhantomData, PhantomPinned},
+ pin::Pin,
};
/// Trait implemented for marker types for global locks.
@@ -26,7 +27,7 @@ pub trait GlobalLockBackend {
/// The backend used for this global lock.
type Backend: Backend + 'static;
/// The class for this global lock.
- fn get_lock_class() -> &'static LockClassKey;
+ fn get_lock_class() -> Pin<&'static LockClassKey>;
}
/// Type used for global locks.
@@ -270,7 +271,7 @@ macro_rules! global_lock {
type Item = $valuety;
type Backend = $crate::global_lock_inner!(backend $kind);
- fn get_lock_class() -> &'static $crate::sync::LockClassKey {
+ fn get_lock_class() -> Pin<&'static $crate::sync::LockClassKey> {
$crate::static_lock_class!()
}
}
diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs
index 70cadbc2e8e2..581cee7ab842 100644
--- a/rust/kernel/sync/lock/mutex.rs
+++ b/rust/kernel/sync/lock/mutex.rs
@@ -26,7 +26,7 @@ pub use new_mutex;
/// Since it may block, [`Mutex`] needs to be used with care in atomic contexts.
///
/// Instances of [`Mutex`] need a lock class and to be pinned. The recommended way to create such
-/// instances is with the [`pin_init`](crate::pin_init) and [`new_mutex`] macros.
+/// instances is with the [`pin_init`](pin_init::pin_init) and [`new_mutex`] macros.
///
/// # Examples
///
diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs
index ab2f8d075311..d7be38ccbdc7 100644
--- a/rust/kernel/sync/lock/spinlock.rs
+++ b/rust/kernel/sync/lock/spinlock.rs
@@ -24,7 +24,7 @@ pub use new_spinlock;
/// unlocked, at which point another CPU will be allowed to make progress.
///
/// Instances of [`SpinLock`] need a lock class and to be pinned. The recommended way to create such
-/// instances is with the [`pin_init`](crate::pin_init) and [`new_spinlock`] macros.
+/// instances is with the [`pin_init`](pin_init::pin_init) and [`new_spinlock`] macros.
///
/// # Examples
///
diff --git a/rust/kernel/sync/locked_by.rs b/rust/kernel/sync/locked_by.rs
index a7b244675c2b..61f100a45b35 100644
--- a/rust/kernel/sync/locked_by.rs
+++ b/rust/kernel/sync/locked_by.rs
@@ -55,7 +55,7 @@ use core::{cell::UnsafeCell, mem::size_of, ptr};
/// fn print_bytes_used(dir: &Directory, file: &File) {
/// let guard = dir.inner.lock();
/// let inner_file = file.inner.access(&guard);
-/// pr_info!("{} {}", guard.bytes_used, inner_file.bytes_used);
+/// pr_info!("{} {}\n", guard.bytes_used, inner_file.bytes_used);
/// }
///
/// /// Increments `bytes_used` for both the directory and file.
diff --git a/rust/kernel/sync/poll.rs b/rust/kernel/sync/poll.rs
index d5f17153b424..d7e6e59e124b 100644
--- a/rust/kernel/sync/poll.rs
+++ b/rust/kernel/sync/poll.rs
@@ -43,11 +43,11 @@ impl PollTable {
///
/// # Safety
///
- /// The caller must ensure that for the duration of 'a, the pointer will point at a valid poll
+ /// The caller must ensure that for the duration of `'a`, the pointer will point at a valid poll
/// table (as defined in the type invariants).
///
/// The caller must also ensure that the `poll_table` is only accessed via the returned
- /// reference for the duration of 'a.
+ /// reference for the duration of `'a`.
pub unsafe fn from_ptr<'a>(ptr: *mut bindings::poll_table) -> &'a mut PollTable {
// SAFETY: The safety requirements guarantee the validity of the dereference, while the
// `PollTable` type being transparent makes the cast ok.
@@ -89,7 +89,7 @@ pub struct PollCondVar {
impl PollCondVar {
/// Constructs a new condvar initialiser.
- pub fn new(name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> {
+ pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self> {
pin_init!(Self {
inner <- CondVar::new(name, key),
})
diff --git a/rust/kernel/sync/rcu.rs b/rust/kernel/sync/rcu.rs
index b51d9150ffe2..a32bef6e490b 100644
--- a/rust/kernel/sync/rcu.rs
+++ b/rust/kernel/sync/rcu.rs
@@ -17,6 +17,7 @@ pub struct Guard(NotThreadSafe);
impl Guard {
/// Acquires the RCU read side lock and returns a guard.
+ #[inline]
pub fn new() -> Self {
// SAFETY: An FFI call with no additional requirements.
unsafe { bindings::rcu_read_lock() };
@@ -25,16 +26,19 @@ impl Guard {
}
/// Explicitly releases the RCU read side lock.
+ #[inline]
pub fn unlock(self) {}
}
impl Default for Guard {
+ #[inline]
fn default() -> Self {
Self::new()
}
}
impl Drop for Guard {
+ #[inline]
fn drop(&mut self) {
// SAFETY: By the type invariants, the RCU read side is locked, so it is ok to unlock it.
unsafe { bindings::rcu_read_unlock() };
@@ -42,6 +46,7 @@ impl Drop for Guard {
}
/// Acquires the RCU read side lock.
+#[inline]
pub fn read_lock() -> Guard {
Guard::new()
}
diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs
index 07bc22a7645c..927413d85484 100644
--- a/rust/kernel/task.rs
+++ b/rust/kernel/task.rs
@@ -7,6 +7,7 @@
use crate::{
bindings,
ffi::{c_int, c_long, c_uint},
+ mm::MmWithUser,
pid_namespace::PidNamespace,
types::{ARef, NotThreadSafe, Opaque},
};
@@ -23,6 +24,8 @@ pub const MAX_SCHEDULE_TIMEOUT: c_long = c_long::MAX;
pub const TASK_INTERRUPTIBLE: c_int = bindings::TASK_INTERRUPTIBLE as c_int;
/// Bitmask for tasks that are sleeping in an uninterruptible state.
pub const TASK_UNINTERRUPTIBLE: c_int = bindings::TASK_UNINTERRUPTIBLE as c_int;
+/// Bitmask for tasks that are sleeping in a freezable state.
+pub const TASK_FREEZABLE: c_int = bindings::TASK_FREEZABLE as c_int;
/// Convenience constant for waking up tasks regardless of whether they are in interruptible or
/// uninterruptible sleep.
pub const TASK_NORMAL: c_uint = bindings::TASK_NORMAL as c_uint;
@@ -31,22 +34,20 @@ pub const TASK_NORMAL: c_uint = bindings::TASK_NORMAL as c_uint;
#[macro_export]
macro_rules! current {
() => {
- // SAFETY: Deref + addr-of below create a temporary `TaskRef` that cannot outlive the
- // caller.
+ // SAFETY: This expression creates a temporary value that is dropped at the end of the
+ // caller's scope. The following mechanisms ensure that the resulting `&CurrentTask` cannot
+ // leave current task context:
+ //
+ // * To return to userspace, the caller must leave the current scope.
+ // * Operations such as `begin_new_exec()` are necessarily unsafe and the caller of
+ // `begin_new_exec()` is responsible for safety.
+ // * Rust abstractions for things such as a `kthread_use_mm()` scope must require the
+ // closure to be `Send`, so the `NotThreadSafe` field of `CurrentTask` ensures that the
+ // `&CurrentTask` cannot cross the scope in either direction.
unsafe { &*$crate::task::Task::current() }
};
}
-/// Returns the currently running task's pid namespace.
-#[macro_export]
-macro_rules! current_pid_ns {
- () => {
- // SAFETY: Deref + addr-of below create a temporary `PidNamespaceRef` that cannot outlive
- // the caller.
- unsafe { &*$crate::task::Task::current_pid_ns() }
- };
-}
-
/// Wraps the kernel's `struct task_struct`.
///
/// # Invariants
@@ -85,7 +86,7 @@ macro_rules! current_pid_ns {
/// impl State {
/// fn new() -> Self {
/// Self {
-/// creator: current!().into(),
+/// creator: ARef::from(&**current!()),
/// index: 0,
/// }
/// }
@@ -105,8 +106,46 @@ unsafe impl Send for Task {}
// synchronised by C code (e.g., `signal_pending`).
unsafe impl Sync for Task {}
+/// Represents the [`Task`] in the `current` global.
+///
+/// This type exists to provide more efficient operations that are only valid on the current task.
+/// For example, to retrieve the pid-namespace of a task, you must use rcu protection unless it is
+/// the current task.
+///
+/// # Invariants
+///
+/// Each value of this type must only be accessed from the task context it was created within.
+///
+/// Of course, every thread is in a different task context, but for the purposes of this invariant,
+/// these operations also permanently leave the task context:
+///
+/// * Returning to userspace from system call context.
+/// * Calling `release_task()`.
+/// * Calling `begin_new_exec()` in a binary format loader.
+///
+/// Other operations temporarily create a new sub-context:
+///
+/// * Calling `kthread_use_mm()` creates a new context, and `kthread_unuse_mm()` returns to the
+/// old context.
+///
+/// This means that a `CurrentTask` obtained before a `kthread_use_mm()` call may be used again
+/// once `kthread_unuse_mm()` is called, but it must not be used between these two calls.
+/// Conversely, a `CurrentTask` obtained between a `kthread_use_mm()`/`kthread_unuse_mm()` pair
+/// must not be used after `kthread_unuse_mm()`.
+#[repr(transparent)]
+pub struct CurrentTask(Task, NotThreadSafe);
+
+// Make all `Task` methods available on `CurrentTask`.
+impl Deref for CurrentTask {
+ type Target = Task;
+ #[inline]
+ fn deref(&self) -> &Task {
+ &self.0
+ }
+}
+
/// The type of process identifiers (PIDs).
-type Pid = bindings::pid_t;
+pub type Pid = bindings::pid_t;
/// The type of user identifiers (UIDs).
#[derive(Copy, Clone)]
@@ -131,119 +170,29 @@ impl Task {
///
/// # Safety
///
- /// Callers must ensure that the returned object doesn't outlive the current task/thread.
- pub unsafe fn current() -> impl Deref<Target = Task> {
- struct TaskRef<'a> {
- task: &'a Task,
- _not_send: NotThreadSafe,
+ /// Callers must ensure that the returned object is only used to access a [`CurrentTask`]
+ /// within the task context that was active when this function was called. For more details,
+ /// see the invariants section for [`CurrentTask`].
+ pub unsafe fn current() -> impl Deref<Target = CurrentTask> {
+ struct TaskRef {
+ task: *const CurrentTask,
}
- impl Deref for TaskRef<'_> {
- type Target = Task;
+ impl Deref for TaskRef {
+ type Target = CurrentTask;
fn deref(&self) -> &Self::Target {
- self.task
+ // SAFETY: The returned reference borrows from this `TaskRef`, so it cannot outlive
+ // the `TaskRef`, which the caller of `Task::current()` has promised will not
+ // outlive the task/thread for which `self.task` is the `current` pointer. Thus, it
+ // is okay to return a `CurrentTask` reference here.
+ unsafe { &*self.task }
}
}
- let current = Task::current_raw();
TaskRef {
- // SAFETY: If the current thread is still running, the current task is valid. Given
- // that `TaskRef` is not `Send`, we know it cannot be transferred to another thread
- // (where it could potentially outlive the caller).
- task: unsafe { &*current.cast() },
- _not_send: NotThreadSafe,
- }
- }
-
- /// Returns a PidNamespace reference for the currently executing task's/thread's pid namespace.
- ///
- /// This function can be used to create an unbounded lifetime by e.g., storing the returned
- /// PidNamespace in a global variable which would be a bug. So the recommended way to get the
- /// current task's/thread's pid namespace is to use the [`current_pid_ns`] macro because it is
- /// safe.
- ///
- /// # Safety
- ///
- /// Callers must ensure that the returned object doesn't outlive the current task/thread.
- pub unsafe fn current_pid_ns() -> impl Deref<Target = PidNamespace> {
- struct PidNamespaceRef<'a> {
- task: &'a PidNamespace,
- _not_send: NotThreadSafe,
- }
-
- impl Deref for PidNamespaceRef<'_> {
- type Target = PidNamespace;
-
- fn deref(&self) -> &Self::Target {
- self.task
- }
- }
-
- // The lifetime of `PidNamespace` is bound to `Task` and `struct pid`.
- //
- // The `PidNamespace` of a `Task` doesn't ever change once the `Task` is alive. A
- // `unshare(CLONE_NEWPID)` or `setns(fd_pidns/pidfd, CLONE_NEWPID)` will not have an effect
- // on the calling `Task`'s pid namespace. It will only effect the pid namespace of children
- // created by the calling `Task`. This invariant guarantees that after having acquired a
- // reference to a `Task`'s pid namespace it will remain unchanged.
- //
- // When a task has exited and been reaped `release_task()` will be called. This will set
- // the `PidNamespace` of the task to `NULL`. So retrieving the `PidNamespace` of a task
- // that is dead will return `NULL`. Note, that neither holding the RCU lock nor holding a
- // referencing count to
- // the `Task` will prevent `release_task()` being called.
- //
- // In order to retrieve the `PidNamespace` of a `Task` the `task_active_pid_ns()` function
- // can be used. There are two cases to consider:
- //
- // (1) retrieving the `PidNamespace` of the `current` task
- // (2) retrieving the `PidNamespace` of a non-`current` task
- //
- // From system call context retrieving the `PidNamespace` for case (1) is always safe and
- // requires neither RCU locking nor a reference count to be held. Retrieving the
- // `PidNamespace` after `release_task()` for current will return `NULL` but no codepath
- // like that is exposed to Rust.
- //
- // Retrieving the `PidNamespace` from system call context for (2) requires RCU protection.
- // Accessing `PidNamespace` outside of RCU protection requires a reference count that
- // must've been acquired while holding the RCU lock. Note that accessing a non-`current`
- // task means `NULL` can be returned as the non-`current` task could have already passed
- // through `release_task()`.
- //
- // To retrieve (1) the `current_pid_ns!()` macro should be used which ensure that the
- // returned `PidNamespace` cannot outlive the calling scope. The associated
- // `current_pid_ns()` function should not be called directly as it could be abused to
- // created an unbounded lifetime for `PidNamespace`. The `current_pid_ns!()` macro allows
- // Rust to handle the common case of accessing `current`'s `PidNamespace` without RCU
- // protection and without having to acquire a reference count.
- //
- // For (2) the `task_get_pid_ns()` method must be used. This will always acquire a
- // reference on `PidNamespace` and will return an `Option` to force the caller to
- // explicitly handle the case where `PidNamespace` is `None`, something that tends to be
- // forgotten when doing the equivalent operation in `C`. Missing RCU primitives make it
- // difficult to perform operations that are otherwise safe without holding a reference
- // count as long as RCU protection is guaranteed. But it is not important currently. But we
- // do want it in the future.
- //
- // Note for (2) the required RCU protection around calling `task_active_pid_ns()`
- // synchronizes against putting the last reference of the associated `struct pid` of
- // `task->thread_pid`. The `struct pid` stored in that field is used to retrieve the
- // `PidNamespace` of the caller. When `release_task()` is called `task->thread_pid` will be
- // `NULL`ed and `put_pid()` on said `struct pid` will be delayed in `free_pid()` via
- // `call_rcu()` allowing everyone with an RCU protected access to the `struct pid` acquired
- // from `task->thread_pid` to finish.
- //
- // SAFETY: The current task's pid namespace is valid as long as the current task is running.
- let pidns = unsafe { bindings::task_active_pid_ns(Task::current_raw()) };
- PidNamespaceRef {
- // SAFETY: If the current thread is still running, the current task and its associated
- // pid namespace are valid. `PidNamespaceRef` is not `Send`, so we know it cannot be
- // transferred to another thread (where it could potentially outlive the current
- // `Task`). The caller needs to ensure that the PidNamespaceRef doesn't outlive the
- // current task/thread.
- task: unsafe { PidNamespace::from_ptr(pidns) },
- _not_send: NotThreadSafe,
+ // CAST: The layout of `struct task_struct` and `CurrentTask` is identical.
+ task: Task::current_raw().cast(),
}
}
@@ -320,12 +269,76 @@ impl Task {
/// Wakes up the task.
pub fn wake_up(&self) {
- // SAFETY: It's always safe to call `signal_pending` on a valid task, even if the task
+ // SAFETY: It's always safe to call `wake_up_process` on a valid task, even if the task
// running.
unsafe { bindings::wake_up_process(self.as_ptr()) };
}
}
+impl CurrentTask {
+ /// Access the address space of the current task.
+ ///
+ /// This function does not touch the refcount of the mm.
+ #[inline]
+ pub fn mm(&self) -> Option<&MmWithUser> {
+ // SAFETY: The `mm` field of `current` is not modified from other threads, so reading it is
+ // not a data race.
+ let mm = unsafe { (*self.as_ptr()).mm };
+
+ if mm.is_null() {
+ return None;
+ }
+
+ // SAFETY: If `current->mm` is non-null, then it references a valid mm with a non-zero
+ // value of `mm_users`. Furthermore, the returned `&MmWithUser` borrows from this
+ // `CurrentTask`, so it cannot escape the scope in which the current pointer was obtained.
+ //
+ // This is safe even if `kthread_use_mm()`/`kthread_unuse_mm()` are used. There are two
+ // relevant cases:
+ // * If the `&CurrentTask` was created before `kthread_use_mm()`, then it cannot be
+ // accessed during the `kthread_use_mm()`/`kthread_unuse_mm()` scope due to the
+ // `NotThreadSafe` field of `CurrentTask`.
+ // * If the `&CurrentTask` was created within a `kthread_use_mm()`/`kthread_unuse_mm()`
+ // scope, then the `&CurrentTask` cannot escape that scope, so the returned `&MmWithUser`
+ // also cannot escape that scope.
+ // In either case, it's not possible to read `current->mm` and keep using it after the
+ // scope is ended with `kthread_unuse_mm()`.
+ Some(unsafe { MmWithUser::from_raw(mm) })
+ }
+
+ /// Access the pid namespace of the current task.
+ ///
+ /// This function does not touch the refcount of the namespace or use RCU protection.
+ ///
+ /// To access the pid namespace of another task, see [`Task::get_pid_ns`].
+ #[doc(alias = "task_active_pid_ns")]
+ #[inline]
+ pub fn active_pid_ns(&self) -> Option<&PidNamespace> {
+ // SAFETY: It is safe to call `task_active_pid_ns` without RCU protection when calling it
+ // on the current task.
+ let active_ns = unsafe { bindings::task_active_pid_ns(self.as_ptr()) };
+
+ if active_ns.is_null() {
+ return None;
+ }
+
+ // The lifetime of `PidNamespace` is bound to `Task` and `struct pid`.
+ //
+ // The `PidNamespace` of a `Task` doesn't ever change once the `Task` is alive.
+ //
+ // From system call context retrieving the `PidNamespace` for the current task is always
+ // safe and requires neither RCU locking nor a reference count to be held. Retrieving the
+ // `PidNamespace` after `release_task()` for current will return `NULL` but no codepath
+ // like that is exposed to Rust.
+ //
+ // SAFETY: If `current`'s pid ns is non-null, then it references a valid pid ns.
+ // Furthermore, the returned `&PidNamespace` borrows from this `CurrentTask`, so it cannot
+ // escape the scope in which the current pointer was obtained, e.g. it cannot live past a
+ // `release_task()` call.
+ Some(unsafe { PidNamespace::from_ptr(active_ns) })
+ }
+}
+
// SAFETY: The type invariants guarantee that `Task` is always refcounted.
unsafe impl crate::types::AlwaysRefCounted for Task {
fn inc_ref(&self) {
diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs
index 379c0f5772e5..a8089a98da9e 100644
--- a/rust/kernel/time.rs
+++ b/rust/kernel/time.rs
@@ -5,12 +5,36 @@
//! This module contains the kernel APIs related to time and timers that
//! have been ported or wrapped for usage by Rust code in the kernel.
//!
+//! There are two types in this module:
+//!
+//! - The [`Instant`] type represents a specific point in time.
+//! - The [`Delta`] type represents a span of time.
+//!
+//! Note that the C side uses `ktime_t` type to represent both. However, timestamp
+//! and timedelta are different. To avoid confusion, we use two different types.
+//!
+//! A [`Instant`] object can be created by calling the [`Instant::now()`] function.
+//! It represents a point in time at which the object was created.
+//! By calling the [`Instant::elapsed()`] method, a [`Delta`] object representing
+//! the elapsed time can be created. The [`Delta`] object can also be created
+//! by subtracting two [`Instant`] objects.
+//!
+//! A [`Delta`] type supports methods to retrieve the duration in various units.
+//!
//! C header: [`include/linux/jiffies.h`](srctree/include/linux/jiffies.h).
//! C header: [`include/linux/ktime.h`](srctree/include/linux/ktime.h).
+pub mod hrtimer;
+
+/// The number of nanoseconds per microsecond.
+pub const NSEC_PER_USEC: i64 = bindings::NSEC_PER_USEC as i64;
+
/// The number of nanoseconds per millisecond.
pub const NSEC_PER_MSEC: i64 = bindings::NSEC_PER_MSEC as i64;
+/// The number of nanoseconds per second.
+pub const NSEC_PER_SEC: i64 = bindings::NSEC_PER_SEC as i64;
+
/// The time unit of Linux kernel. One jiffy equals (1/HZ) second.
pub type Jiffies = crate::ffi::c_ulong;
@@ -25,59 +49,192 @@ pub fn msecs_to_jiffies(msecs: Msecs) -> Jiffies {
unsafe { bindings::__msecs_to_jiffies(msecs) }
}
-/// A Rust wrapper around a `ktime_t`.
+/// A specific point in time.
+///
+/// # Invariants
+///
+/// The `inner` value is in the range from 0 to `KTIME_MAX`.
#[repr(transparent)]
-#[derive(Copy, Clone)]
-pub struct Ktime {
+#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord)]
+pub struct Instant {
inner: bindings::ktime_t,
}
-impl Ktime {
- /// Create a `Ktime` from a raw `ktime_t`.
- #[inline]
- pub fn from_raw(inner: bindings::ktime_t) -> Self {
- Self { inner }
- }
-
+impl Instant {
/// Get the current time using `CLOCK_MONOTONIC`.
#[inline]
- pub fn ktime_get() -> Self {
- // SAFETY: It is always safe to call `ktime_get` outside of NMI context.
- Self::from_raw(unsafe { bindings::ktime_get() })
+ pub fn now() -> Self {
+ // INVARIANT: The `ktime_get()` function returns a value in the range
+ // from 0 to `KTIME_MAX`.
+ Self {
+ // SAFETY: It is always safe to call `ktime_get()` outside of NMI context.
+ inner: unsafe { bindings::ktime_get() },
+ }
}
- /// Divide the number of nanoseconds by a compile-time constant.
+ /// Return the amount of time elapsed since the [`Instant`].
#[inline]
- fn divns_constant<const DIV: i64>(self) -> i64 {
- self.to_ns() / DIV
+ pub fn elapsed(&self) -> Delta {
+ Self::now() - *self
}
+}
+
+impl core::ops::Sub for Instant {
+ type Output = Delta;
- /// Returns the number of nanoseconds.
+ // By the type invariant, it never overflows.
#[inline]
- pub fn to_ns(self) -> i64 {
- self.inner
+ fn sub(self, other: Instant) -> Delta {
+ Delta {
+ nanos: self.inner - other.inner,
+ }
}
+}
- /// Returns the number of milliseconds.
- #[inline]
- pub fn to_ms(self) -> i64 {
- self.divns_constant::<NSEC_PER_MSEC>()
+/// An identifier for a clock. Used when specifying clock sources.
+///
+///
+/// Selection of the clock depends on the use case. In some cases the usage of a
+/// particular clock is mandatory, e.g. in network protocols, filesystems.In other
+/// cases the user of the clock has to decide which clock is best suited for the
+/// purpose. In most scenarios clock [`ClockId::Monotonic`] is the best choice as it
+/// provides a accurate monotonic notion of time (leap second smearing ignored).
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+#[repr(u32)]
+pub enum ClockId {
+ /// A settable system-wide clock that measures real (i.e., wall-clock) time.
+ ///
+ /// Setting this clock requires appropriate privileges. This clock is
+ /// affected by discontinuous jumps in the system time (e.g., if the system
+ /// administrator manually changes the clock), and by frequency adjustments
+ /// performed by NTP and similar applications via adjtime(3), adjtimex(2),
+ /// clock_adjtime(2), and ntp_adjtime(3). This clock normally counts the
+ /// number of seconds since 1970-01-01 00:00:00 Coordinated Universal Time
+ /// (UTC) except that it ignores leap seconds; near a leap second it may be
+ /// adjusted by leap second smearing to stay roughly in sync with UTC. Leap
+ /// second smearing applies frequency adjustments to the clock to speed up
+ /// or slow down the clock to account for the leap second without
+ /// discontinuities in the clock. If leap second smearing is not applied,
+ /// the clock will experience discontinuity around leap second adjustment.
+ RealTime = bindings::CLOCK_REALTIME,
+ /// A monotonically increasing clock.
+ ///
+ /// A nonsettable system-wide clock that represents monotonic time since—as
+ /// described by POSIX—"some unspecified point in the past". On Linux, that
+ /// point corresponds to the number of seconds that the system has been
+ /// running since it was booted.
+ ///
+ /// The CLOCK_MONOTONIC clock is not affected by discontinuous jumps in the
+ /// CLOCK_REAL (e.g., if the system administrator manually changes the
+ /// clock), but is affected by frequency adjustments. This clock does not
+ /// count time that the system is suspended.
+ Monotonic = bindings::CLOCK_MONOTONIC,
+ /// A monotonic that ticks while system is suspended.
+ ///
+ /// A nonsettable system-wide clock that is identical to CLOCK_MONOTONIC,
+ /// except that it also includes any time that the system is suspended. This
+ /// allows applications to get a suspend-aware monotonic clock without
+ /// having to deal with the complications of CLOCK_REALTIME, which may have
+ /// discontinuities if the time is changed using settimeofday(2) or similar.
+ BootTime = bindings::CLOCK_BOOTTIME,
+ /// International Atomic Time.
+ ///
+ /// A system-wide clock derived from wall-clock time but counting leap seconds.
+ ///
+ /// This clock is coupled to CLOCK_REALTIME and will be set when CLOCK_REALTIME is
+ /// set, or when the offset to CLOCK_REALTIME is changed via adjtimex(2). This
+ /// usually happens during boot and **should** not happen during normal operations.
+ /// However, if NTP or another application adjusts CLOCK_REALTIME by leap second
+ /// smearing, this clock will not be precise during leap second smearing.
+ ///
+ /// The acronym TAI refers to International Atomic Time.
+ TAI = bindings::CLOCK_TAI,
+}
+
+impl ClockId {
+ fn into_c(self) -> bindings::clockid_t {
+ self as bindings::clockid_t
}
}
-/// Returns the number of milliseconds between two ktimes.
-#[inline]
-pub fn ktime_ms_delta(later: Ktime, earlier: Ktime) -> i64 {
- (later - earlier).to_ms()
+/// A span of time.
+///
+/// This struct represents a span of time, with its value stored as nanoseconds.
+/// The value can represent any valid i64 value, including negative, zero, and
+/// positive numbers.
+#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)]
+pub struct Delta {
+ nanos: i64,
}
-impl core::ops::Sub for Ktime {
- type Output = Ktime;
+impl Delta {
+ /// A span of time equal to zero.
+ pub const ZERO: Self = Self { nanos: 0 };
+
+ /// Create a new [`Delta`] from a number of microseconds.
+ ///
+ /// The `micros` can range from -9_223_372_036_854_775 to 9_223_372_036_854_775.
+ /// If `micros` is outside this range, `i64::MIN` is used for negative values,
+ /// and `i64::MAX` is used for positive values due to saturation.
+ #[inline]
+ pub const fn from_micros(micros: i64) -> Self {
+ Self {
+ nanos: micros.saturating_mul(NSEC_PER_USEC),
+ }
+ }
+
+ /// Create a new [`Delta`] from a number of milliseconds.
+ ///
+ /// The `millis` can range from -9_223_372_036_854 to 9_223_372_036_854.
+ /// If `millis` is outside this range, `i64::MIN` is used for negative values,
+ /// and `i64::MAX` is used for positive values due to saturation.
+ #[inline]
+ pub const fn from_millis(millis: i64) -> Self {
+ Self {
+ nanos: millis.saturating_mul(NSEC_PER_MSEC),
+ }
+ }
+ /// Create a new [`Delta`] from a number of seconds.
+ ///
+ /// The `secs` can range from -9_223_372_036 to 9_223_372_036.
+ /// If `secs` is outside this range, `i64::MIN` is used for negative values,
+ /// and `i64::MAX` is used for positive values due to saturation.
#[inline]
- fn sub(self, other: Ktime) -> Ktime {
+ pub const fn from_secs(secs: i64) -> Self {
Self {
- inner: self.inner - other.inner,
+ nanos: secs.saturating_mul(NSEC_PER_SEC),
}
}
+
+ /// Return `true` if the [`Delta`] spans no time.
+ #[inline]
+ pub fn is_zero(self) -> bool {
+ self.as_nanos() == 0
+ }
+
+ /// Return `true` if the [`Delta`] spans a negative amount of time.
+ #[inline]
+ pub fn is_negative(self) -> bool {
+ self.as_nanos() < 0
+ }
+
+ /// Return the number of nanoseconds in the [`Delta`].
+ #[inline]
+ pub const fn as_nanos(self) -> i64 {
+ self.nanos
+ }
+
+ /// Return the smallest number of microseconds greater than or equal
+ /// to the value in the [`Delta`].
+ #[inline]
+ pub const fn as_micros_ceil(self) -> i64 {
+ self.as_nanos().saturating_add(NSEC_PER_USEC - 1) / NSEC_PER_USEC
+ }
+
+ /// Return the number of milliseconds in the [`Delta`].
+ #[inline]
+ pub const fn as_millis(self) -> i64 {
+ self.as_nanos() / NSEC_PER_MSEC
+ }
}
diff --git a/rust/kernel/time/hrtimer.rs b/rust/kernel/time/hrtimer.rs
new file mode 100644
index 000000000000..36e1290cd079
--- /dev/null
+++ b/rust/kernel/time/hrtimer.rs
@@ -0,0 +1,534 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Intrusive high resolution timers.
+//!
+//! Allows running timer callbacks without doing allocations at the time of
+//! starting the timer. For now, only one timer per type is allowed.
+//!
+//! # Vocabulary
+//!
+//! States:
+//!
+//! - Stopped: initialized but not started, or cancelled, or not restarted.
+//! - Started: initialized and started or restarted.
+//! - Running: executing the callback.
+//!
+//! Operations:
+//!
+//! * Start
+//! * Cancel
+//! * Restart
+//!
+//! Events:
+//!
+//! * Expire
+//!
+//! ## State Diagram
+//!
+//! ```text
+//! Return NoRestart
+//! +---------------------------------------------------------------------+
+//! | |
+//! | |
+//! | |
+//! | Return Restart |
+//! | +------------------------+ |
+//! | | | |
+//! | | | |
+//! v v | |
+//! +-----------------+ Start +------------------+ +--------+-----+--+
+//! | +---------------->| | | |
+//! Init | | | | Expire | |
+//! --------->| Stopped | | Started +---------->| Running |
+//! | | Cancel | | | |
+//! | |<----------------+ | | |
+//! +-----------------+ +---------------+--+ +-----------------+
+//! ^ |
+//! | |
+//! +---------+
+//! Restart
+//! ```
+//!
+//!
+//! A timer is initialized in the **stopped** state. A stopped timer can be
+//! **started** by the `start` operation, with an **expiry** time. After the
+//! `start` operation, the timer is in the **started** state. When the timer
+//! **expires**, the timer enters the **running** state and the handler is
+//! executed. After the handler has returned, the timer may enter the
+//! **started* or **stopped** state, depending on the return value of the
+//! handler. A timer in the **started** or **running** state may be **canceled**
+//! by the `cancel` operation. A timer that is cancelled enters the **stopped**
+//! state.
+//!
+//! A `cancel` or `restart` operation on a timer in the **running** state takes
+//! effect after the handler has returned and the timer has transitioned
+//! out of the **running** state.
+//!
+//! A `restart` operation on a timer in the **stopped** state is equivalent to a
+//! `start` operation.
+
+use super::ClockId;
+use crate::{prelude::*, types::Opaque};
+use core::marker::PhantomData;
+use pin_init::PinInit;
+
+/// A Rust wrapper around a `ktime_t`.
+// NOTE: Ktime is going to be removed when hrtimer is converted to Instant/Delta.
+#[repr(transparent)]
+#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord)]
+pub struct Ktime {
+ inner: bindings::ktime_t,
+}
+
+impl Ktime {
+ /// Returns the number of nanoseconds.
+ #[inline]
+ pub fn to_ns(self) -> i64 {
+ self.inner
+ }
+}
+
+/// A timer backed by a C `struct hrtimer`.
+///
+/// # Invariants
+///
+/// * `self.timer` is initialized by `bindings::hrtimer_setup`.
+#[pin_data]
+#[repr(C)]
+pub struct HrTimer<T> {
+ #[pin]
+ timer: Opaque<bindings::hrtimer>,
+ mode: HrTimerMode,
+ _t: PhantomData<T>,
+}
+
+// SAFETY: Ownership of an `HrTimer` can be moved to other threads and
+// used/dropped from there.
+unsafe impl<T> Send for HrTimer<T> {}
+
+// SAFETY: Timer operations are locked on the C side, so it is safe to operate
+// on a timer from multiple threads.
+unsafe impl<T> Sync for HrTimer<T> {}
+
+impl<T> HrTimer<T> {
+ /// Return an initializer for a new timer instance.
+ pub fn new(mode: HrTimerMode, clock: ClockId) -> impl PinInit<Self>
+ where
+ T: HrTimerCallback,
+ {
+ pin_init!(Self {
+ // INVARIANT: We initialize `timer` with `hrtimer_setup` below.
+ timer <- Opaque::ffi_init(move |place: *mut bindings::hrtimer| {
+ // SAFETY: By design of `pin_init!`, `place` is a pointer to a
+ // live allocation. hrtimer_setup will initialize `place` and
+ // does not require `place` to be initialized prior to the call.
+ unsafe {
+ bindings::hrtimer_setup(
+ place,
+ Some(T::Pointer::run),
+ clock.into_c(),
+ mode.into_c(),
+ );
+ }
+ }),
+ mode: mode,
+ _t: PhantomData,
+ })
+ }
+
+ /// Get a pointer to the contained `bindings::hrtimer`.
+ ///
+ /// This function is useful to get access to the value without creating
+ /// intermediate references.
+ ///
+ /// # Safety
+ ///
+ /// `this` must point to a live allocation of at least the size of `Self`.
+ unsafe fn raw_get(this: *const Self) -> *mut bindings::hrtimer {
+ // SAFETY: The field projection to `timer` does not go out of bounds,
+ // because the caller of this function promises that `this` points to an
+ // allocation of at least the size of `Self`.
+ unsafe { Opaque::raw_get(core::ptr::addr_of!((*this).timer)) }
+ }
+
+ /// Cancel an initialized and potentially running timer.
+ ///
+ /// If the timer handler is running, this function will block until the
+ /// handler returns.
+ ///
+ /// Note that the timer might be started by a concurrent start operation. If
+ /// so, the timer might not be in the **stopped** state when this function
+ /// returns.
+ ///
+ /// Users of the `HrTimer` API would not usually call this method directly.
+ /// Instead they would use the safe [`HrTimerHandle::cancel`] on the handle
+ /// returned when the timer was started.
+ ///
+ /// This function is useful to get access to the value without creating
+ /// intermediate references.
+ ///
+ /// # Safety
+ ///
+ /// `this` must point to a valid `Self`.
+ pub(crate) unsafe fn raw_cancel(this: *const Self) -> bool {
+ // SAFETY: `this` points to an allocation of at least `HrTimer` size.
+ let c_timer_ptr = unsafe { HrTimer::raw_get(this) };
+
+ // If the handler is running, this will wait for the handler to return
+ // before returning.
+ // SAFETY: `c_timer_ptr` is initialized and valid. Synchronization is
+ // handled on the C side.
+ unsafe { bindings::hrtimer_cancel(c_timer_ptr) != 0 }
+ }
+}
+
+/// Implemented by pointer types that point to structs that contain a [`HrTimer`].
+///
+/// `Self` must be [`Sync`] because it is passed to timer callbacks in another
+/// thread of execution (hard or soft interrupt context).
+///
+/// Starting a timer returns a [`HrTimerHandle`] that can be used to manipulate
+/// the timer. Note that it is OK to call the start function repeatedly, and
+/// that more than one [`HrTimerHandle`] associated with a [`HrTimerPointer`] may
+/// exist. A timer can be manipulated through any of the handles, and a handle
+/// may represent a cancelled timer.
+pub trait HrTimerPointer: Sync + Sized {
+ /// A handle representing a started or restarted timer.
+ ///
+ /// If the timer is running or if the timer callback is executing when the
+ /// handle is dropped, the drop method of [`HrTimerHandle`] should not return
+ /// until the timer is stopped and the callback has completed.
+ ///
+ /// Note: When implementing this trait, consider that it is not unsafe to
+ /// leak the handle.
+ type TimerHandle: HrTimerHandle;
+
+ /// Start the timer with expiry after `expires` time units. If the timer was
+ /// already running, it is restarted with the new expiry time.
+ fn start(self, expires: Ktime) -> Self::TimerHandle;
+}
+
+/// Unsafe version of [`HrTimerPointer`] for situations where leaking the
+/// [`HrTimerHandle`] returned by `start` would be unsound. This is the case for
+/// stack allocated timers.
+///
+/// Typical implementers are pinned references such as [`Pin<&T>`].
+///
+/// # Safety
+///
+/// Implementers of this trait must ensure that instances of types implementing
+/// [`UnsafeHrTimerPointer`] outlives any associated [`HrTimerPointer::TimerHandle`]
+/// instances.
+pub unsafe trait UnsafeHrTimerPointer: Sync + Sized {
+ /// A handle representing a running timer.
+ ///
+ /// # Safety
+ ///
+ /// If the timer is running, or if the timer callback is executing when the
+ /// handle is dropped, the drop method of [`Self::TimerHandle`] must not return
+ /// until the timer is stopped and the callback has completed.
+ type TimerHandle: HrTimerHandle;
+
+ /// Start the timer after `expires` time units. If the timer was already
+ /// running, it is restarted at the new expiry time.
+ ///
+ /// # Safety
+ ///
+ /// Caller promises keep the timer structure alive until the timer is dead.
+ /// Caller can ensure this by not leaking the returned [`Self::TimerHandle`].
+ unsafe fn start(self, expires: Ktime) -> Self::TimerHandle;
+}
+
+/// A trait for stack allocated timers.
+///
+/// # Safety
+///
+/// Implementers must ensure that `start_scoped` does not return until the
+/// timer is dead and the timer handler is not running.
+pub unsafe trait ScopedHrTimerPointer {
+ /// Start the timer to run after `expires` time units and immediately
+ /// after call `f`. When `f` returns, the timer is cancelled.
+ fn start_scoped<T, F>(self, expires: Ktime, f: F) -> T
+ where
+ F: FnOnce() -> T;
+}
+
+// SAFETY: By the safety requirement of [`UnsafeHrTimerPointer`], dropping the
+// handle returned by [`UnsafeHrTimerPointer::start`] ensures that the timer is
+// killed.
+unsafe impl<T> ScopedHrTimerPointer for T
+where
+ T: UnsafeHrTimerPointer,
+{
+ fn start_scoped<U, F>(self, expires: Ktime, f: F) -> U
+ where
+ F: FnOnce() -> U,
+ {
+ // SAFETY: We drop the timer handle below before returning.
+ let handle = unsafe { UnsafeHrTimerPointer::start(self, expires) };
+ let t = f();
+ drop(handle);
+ t
+ }
+}
+
+/// Implemented by [`HrTimerPointer`] implementers to give the C timer callback a
+/// function to call.
+// This is split from `HrTimerPointer` to make it easier to specify trait bounds.
+pub trait RawHrTimerCallback {
+ /// Type of the parameter passed to [`HrTimerCallback::run`]. It may be
+ /// [`Self`], or a pointer type derived from [`Self`].
+ type CallbackTarget<'a>;
+
+ /// Callback to be called from C when timer fires.
+ ///
+ /// # Safety
+ ///
+ /// Only to be called by C code in the `hrtimer` subsystem. `this` must point
+ /// to the `bindings::hrtimer` structure that was used to start the timer.
+ unsafe extern "C" fn run(this: *mut bindings::hrtimer) -> bindings::hrtimer_restart;
+}
+
+/// Implemented by structs that can be the target of a timer callback.
+pub trait HrTimerCallback {
+ /// The type whose [`RawHrTimerCallback::run`] method will be invoked when
+ /// the timer expires.
+ type Pointer<'a>: RawHrTimerCallback;
+
+ /// Called by the timer logic when the timer fires.
+ fn run(this: <Self::Pointer<'_> as RawHrTimerCallback>::CallbackTarget<'_>) -> HrTimerRestart
+ where
+ Self: Sized;
+}
+
+/// A handle representing a potentially running timer.
+///
+/// More than one handle representing the same timer might exist.
+///
+/// # Safety
+///
+/// When dropped, the timer represented by this handle must be cancelled, if it
+/// is running. If the timer handler is running when the handle is dropped, the
+/// drop method must wait for the handler to return before returning.
+///
+/// Note: One way to satisfy the safety requirement is to call `Self::cancel` in
+/// the drop implementation for `Self.`
+pub unsafe trait HrTimerHandle {
+ /// Cancel the timer. If the timer is in the running state, block till the
+ /// handler has returned.
+ ///
+ /// Note that the timer might be started by a concurrent start operation. If
+ /// so, the timer might not be in the **stopped** state when this function
+ /// returns.
+ fn cancel(&mut self) -> bool;
+}
+
+/// Implemented by structs that contain timer nodes.
+///
+/// Clients of the timer API would usually safely implement this trait by using
+/// the [`crate::impl_has_hr_timer`] macro.
+///
+/// # Safety
+///
+/// Implementers of this trait must ensure that the implementer has a
+/// [`HrTimer`] field and that all trait methods are implemented according to
+/// their documentation. All the methods of this trait must operate on the same
+/// field.
+pub unsafe trait HasHrTimer<T> {
+ /// Return a pointer to the [`HrTimer`] within `Self`.
+ ///
+ /// This function is useful to get access to the value without creating
+ /// intermediate references.
+ ///
+ /// # Safety
+ ///
+ /// `this` must be a valid pointer.
+ unsafe fn raw_get_timer(this: *const Self) -> *const HrTimer<T>;
+
+ /// Return a pointer to the struct that is containing the [`HrTimer`] pointed
+ /// to by `ptr`.
+ ///
+ /// This function is useful to get access to the value without creating
+ /// intermediate references.
+ ///
+ /// # Safety
+ ///
+ /// `ptr` must point to a [`HrTimer<T>`] field in a struct of type `Self`.
+ unsafe fn timer_container_of(ptr: *mut HrTimer<T>) -> *mut Self
+ where
+ Self: Sized;
+
+ /// Get pointer to the contained `bindings::hrtimer` struct.
+ ///
+ /// This function is useful to get access to the value without creating
+ /// intermediate references.
+ ///
+ /// # Safety
+ ///
+ /// `this` must be a valid pointer.
+ unsafe fn c_timer_ptr(this: *const Self) -> *const bindings::hrtimer {
+ // SAFETY: `this` is a valid pointer to a `Self`.
+ let timer_ptr = unsafe { Self::raw_get_timer(this) };
+
+ // SAFETY: timer_ptr points to an allocation of at least `HrTimer` size.
+ unsafe { HrTimer::raw_get(timer_ptr) }
+ }
+
+ /// Start the timer contained in the `Self` pointed to by `self_ptr`. If
+ /// it is already running it is removed and inserted.
+ ///
+ /// # Safety
+ ///
+ /// - `this` must point to a valid `Self`.
+ /// - Caller must ensure that the pointee of `this` lives until the timer
+ /// fires or is canceled.
+ unsafe fn start(this: *const Self, expires: Ktime) {
+ // SAFETY: By function safety requirement, `this` is a valid `Self`.
+ unsafe {
+ bindings::hrtimer_start_range_ns(
+ Self::c_timer_ptr(this).cast_mut(),
+ expires.to_ns(),
+ 0,
+ (*Self::raw_get_timer(this)).mode.into_c(),
+ );
+ }
+ }
+}
+
+/// Restart policy for timers.
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+#[repr(u32)]
+pub enum HrTimerRestart {
+ /// Timer should not be restarted.
+ NoRestart = bindings::hrtimer_restart_HRTIMER_NORESTART,
+ /// Timer should be restarted.
+ Restart = bindings::hrtimer_restart_HRTIMER_RESTART,
+}
+
+impl HrTimerRestart {
+ fn into_c(self) -> bindings::hrtimer_restart {
+ self as bindings::hrtimer_restart
+ }
+}
+
+/// Operational mode of [`HrTimer`].
+// NOTE: Some of these have the same encoding on the C side, so we keep
+// `repr(Rust)` and convert elsewhere.
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+pub enum HrTimerMode {
+ /// Timer expires at the given expiration time.
+ Absolute,
+ /// Timer expires after the given expiration time interpreted as a duration from now.
+ Relative,
+ /// Timer does not move between CPU cores.
+ Pinned,
+ /// Timer handler is executed in soft irq context.
+ Soft,
+ /// Timer handler is executed in hard irq context.
+ Hard,
+ /// Timer expires at the given expiration time.
+ /// Timer does not move between CPU cores.
+ AbsolutePinned,
+ /// Timer expires after the given expiration time interpreted as a duration from now.
+ /// Timer does not move between CPU cores.
+ RelativePinned,
+ /// Timer expires at the given expiration time.
+ /// Timer handler is executed in soft irq context.
+ AbsoluteSoft,
+ /// Timer expires after the given expiration time interpreted as a duration from now.
+ /// Timer handler is executed in soft irq context.
+ RelativeSoft,
+ /// Timer expires at the given expiration time.
+ /// Timer does not move between CPU cores.
+ /// Timer handler is executed in soft irq context.
+ AbsolutePinnedSoft,
+ /// Timer expires after the given expiration time interpreted as a duration from now.
+ /// Timer does not move between CPU cores.
+ /// Timer handler is executed in soft irq context.
+ RelativePinnedSoft,
+ /// Timer expires at the given expiration time.
+ /// Timer handler is executed in hard irq context.
+ AbsoluteHard,
+ /// Timer expires after the given expiration time interpreted as a duration from now.
+ /// Timer handler is executed in hard irq context.
+ RelativeHard,
+ /// Timer expires at the given expiration time.
+ /// Timer does not move between CPU cores.
+ /// Timer handler is executed in hard irq context.
+ AbsolutePinnedHard,
+ /// Timer expires after the given expiration time interpreted as a duration from now.
+ /// Timer does not move between CPU cores.
+ /// Timer handler is executed in hard irq context.
+ RelativePinnedHard,
+}
+
+impl HrTimerMode {
+ fn into_c(self) -> bindings::hrtimer_mode {
+ use bindings::*;
+ match self {
+ HrTimerMode::Absolute => hrtimer_mode_HRTIMER_MODE_ABS,
+ HrTimerMode::Relative => hrtimer_mode_HRTIMER_MODE_REL,
+ HrTimerMode::Pinned => hrtimer_mode_HRTIMER_MODE_PINNED,
+ HrTimerMode::Soft => hrtimer_mode_HRTIMER_MODE_SOFT,
+ HrTimerMode::Hard => hrtimer_mode_HRTIMER_MODE_HARD,
+ HrTimerMode::AbsolutePinned => hrtimer_mode_HRTIMER_MODE_ABS_PINNED,
+ HrTimerMode::RelativePinned => hrtimer_mode_HRTIMER_MODE_REL_PINNED,
+ HrTimerMode::AbsoluteSoft => hrtimer_mode_HRTIMER_MODE_ABS_SOFT,
+ HrTimerMode::RelativeSoft => hrtimer_mode_HRTIMER_MODE_REL_SOFT,
+ HrTimerMode::AbsolutePinnedSoft => hrtimer_mode_HRTIMER_MODE_ABS_PINNED_SOFT,
+ HrTimerMode::RelativePinnedSoft => hrtimer_mode_HRTIMER_MODE_REL_PINNED_SOFT,
+ HrTimerMode::AbsoluteHard => hrtimer_mode_HRTIMER_MODE_ABS_HARD,
+ HrTimerMode::RelativeHard => hrtimer_mode_HRTIMER_MODE_REL_HARD,
+ HrTimerMode::AbsolutePinnedHard => hrtimer_mode_HRTIMER_MODE_ABS_PINNED_HARD,
+ HrTimerMode::RelativePinnedHard => hrtimer_mode_HRTIMER_MODE_REL_PINNED_HARD,
+ }
+ }
+}
+
+/// Use to implement the [`HasHrTimer<T>`] trait.
+///
+/// See [`module`] documentation for an example.
+///
+/// [`module`]: crate::time::hrtimer
+#[macro_export]
+macro_rules! impl_has_hr_timer {
+ (
+ impl$({$($generics:tt)*})?
+ HasHrTimer<$timer_type:ty>
+ for $self:ty
+ { self.$field:ident }
+ $($rest:tt)*
+ ) => {
+ // SAFETY: This implementation of `raw_get_timer` only compiles if the
+ // field has the right type.
+ unsafe impl$(<$($generics)*>)? $crate::time::hrtimer::HasHrTimer<$timer_type> for $self {
+
+ #[inline]
+ unsafe fn raw_get_timer(
+ this: *const Self,
+ ) -> *const $crate::time::hrtimer::HrTimer<$timer_type> {
+ // SAFETY: The caller promises that the pointer is not dangling.
+ unsafe { ::core::ptr::addr_of!((*this).$field) }
+ }
+
+ #[inline]
+ unsafe fn timer_container_of(
+ ptr: *mut $crate::time::hrtimer::HrTimer<$timer_type>,
+ ) -> *mut Self {
+ // SAFETY: As per the safety requirement of this function, `ptr`
+ // is pointing inside a `$timer_type`.
+ unsafe { ::kernel::container_of!(ptr, $timer_type, $field) }
+ }
+ }
+ }
+}
+
+mod arc;
+pub use arc::ArcHrTimerHandle;
+mod pin;
+pub use pin::PinHrTimerHandle;
+mod pin_mut;
+pub use pin_mut::PinMutHrTimerHandle;
+// `box` is a reserved keyword, so prefix with `t` for timer
+mod tbox;
+pub use tbox::BoxHrTimerHandle;
diff --git a/rust/kernel/time/hrtimer/arc.rs b/rust/kernel/time/hrtimer/arc.rs
new file mode 100644
index 000000000000..ccf1e66e5b2d
--- /dev/null
+++ b/rust/kernel/time/hrtimer/arc.rs
@@ -0,0 +1,100 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use super::HasHrTimer;
+use super::HrTimer;
+use super::HrTimerCallback;
+use super::HrTimerHandle;
+use super::HrTimerPointer;
+use super::Ktime;
+use super::RawHrTimerCallback;
+use crate::sync::Arc;
+use crate::sync::ArcBorrow;
+
+/// A handle for an `Arc<HasHrTimer<T>>` returned by a call to
+/// [`HrTimerPointer::start`].
+pub struct ArcHrTimerHandle<T>
+where
+ T: HasHrTimer<T>,
+{
+ pub(crate) inner: Arc<T>,
+}
+
+// SAFETY: We implement drop below, and we cancel the timer in the drop
+// implementation.
+unsafe impl<T> HrTimerHandle for ArcHrTimerHandle<T>
+where
+ T: HasHrTimer<T>,
+{
+ fn cancel(&mut self) -> bool {
+ let self_ptr = Arc::as_ptr(&self.inner);
+
+ // SAFETY: As we obtained `self_ptr` from a valid reference above, it
+ // must point to a valid `T`.
+ let timer_ptr = unsafe { <T as HasHrTimer<T>>::raw_get_timer(self_ptr) };
+
+ // SAFETY: As `timer_ptr` points into `T` and `T` is valid, `timer_ptr`
+ // must point to a valid `HrTimer` instance.
+ unsafe { HrTimer::<T>::raw_cancel(timer_ptr) }
+ }
+}
+
+impl<T> Drop for ArcHrTimerHandle<T>
+where
+ T: HasHrTimer<T>,
+{
+ fn drop(&mut self) {
+ self.cancel();
+ }
+}
+
+impl<T> HrTimerPointer for Arc<T>
+where
+ T: 'static,
+ T: Send + Sync,
+ T: HasHrTimer<T>,
+ T: for<'a> HrTimerCallback<Pointer<'a> = Self>,
+{
+ type TimerHandle = ArcHrTimerHandle<T>;
+
+ fn start(self, expires: Ktime) -> ArcHrTimerHandle<T> {
+ // SAFETY:
+ // - We keep `self` alive by wrapping it in a handle below.
+ // - Since we generate the pointer passed to `start` from a valid
+ // reference, it is a valid pointer.
+ unsafe { T::start(Arc::as_ptr(&self), expires) };
+ ArcHrTimerHandle { inner: self }
+ }
+}
+
+impl<T> RawHrTimerCallback for Arc<T>
+where
+ T: 'static,
+ T: HasHrTimer<T>,
+ T: for<'a> HrTimerCallback<Pointer<'a> = Self>,
+{
+ type CallbackTarget<'a> = ArcBorrow<'a, T>;
+
+ unsafe extern "C" fn run(ptr: *mut bindings::hrtimer) -> bindings::hrtimer_restart {
+ // `HrTimer` is `repr(C)`
+ let timer_ptr = ptr.cast::<super::HrTimer<T>>();
+
+ // SAFETY: By C API contract `ptr` is the pointer we passed when
+ // queuing the timer, so it is a `HrTimer<T>` embedded in a `T`.
+ let data_ptr = unsafe { T::timer_container_of(timer_ptr) };
+
+ // SAFETY:
+ // - `data_ptr` is derived form the pointer to the `T` that was used to
+ // queue the timer.
+ // - As per the safety requirements of the trait `HrTimerHandle`, the
+ // `ArcHrTimerHandle` associated with this timer is guaranteed to
+ // be alive until this method returns. That handle borrows the `T`
+ // behind `data_ptr` thus guaranteeing the validity of
+ // the `ArcBorrow` created below.
+ // - We own one refcount in the `ArcTimerHandle` associated with this
+ // timer, so it is not possible to get a `UniqueArc` to this
+ // allocation from other `Arc` clones.
+ let receiver = unsafe { ArcBorrow::from_raw(data_ptr) };
+
+ T::run(receiver).into_c()
+ }
+}
diff --git a/rust/kernel/time/hrtimer/pin.rs b/rust/kernel/time/hrtimer/pin.rs
new file mode 100644
index 000000000000..293ca9cf058c
--- /dev/null
+++ b/rust/kernel/time/hrtimer/pin.rs
@@ -0,0 +1,104 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use super::HasHrTimer;
+use super::HrTimer;
+use super::HrTimerCallback;
+use super::HrTimerHandle;
+use super::Ktime;
+use super::RawHrTimerCallback;
+use super::UnsafeHrTimerPointer;
+use core::pin::Pin;
+
+/// A handle for a `Pin<&HasHrTimer>`. When the handle exists, the timer might be
+/// running.
+pub struct PinHrTimerHandle<'a, T>
+where
+ T: HasHrTimer<T>,
+{
+ pub(crate) inner: Pin<&'a T>,
+}
+
+// SAFETY: We cancel the timer when the handle is dropped. The implementation of
+// the `cancel` method will block if the timer handler is running.
+unsafe impl<'a, T> HrTimerHandle for PinHrTimerHandle<'a, T>
+where
+ T: HasHrTimer<T>,
+{
+ fn cancel(&mut self) -> bool {
+ let self_ptr: *const T = self.inner.get_ref();
+
+ // SAFETY: As we got `self_ptr` from a reference above, it must point to
+ // a valid `T`.
+ let timer_ptr = unsafe { <T as HasHrTimer<T>>::raw_get_timer(self_ptr) };
+
+ // SAFETY: As `timer_ptr` is derived from a reference, it must point to
+ // a valid and initialized `HrTimer`.
+ unsafe { HrTimer::<T>::raw_cancel(timer_ptr) }
+ }
+}
+
+impl<'a, T> Drop for PinHrTimerHandle<'a, T>
+where
+ T: HasHrTimer<T>,
+{
+ fn drop(&mut self) {
+ self.cancel();
+ }
+}
+
+// SAFETY: We capture the lifetime of `Self` when we create a `PinHrTimerHandle`,
+// so `Self` will outlive the handle.
+unsafe impl<'a, T> UnsafeHrTimerPointer for Pin<&'a T>
+where
+ T: Send + Sync,
+ T: HasHrTimer<T>,
+ T: HrTimerCallback<Pointer<'a> = Self>,
+{
+ type TimerHandle = PinHrTimerHandle<'a, T>;
+
+ unsafe fn start(self, expires: Ktime) -> Self::TimerHandle {
+ // Cast to pointer
+ let self_ptr: *const T = self.get_ref();
+
+ // SAFETY:
+ // - As we derive `self_ptr` from a reference above, it must point to a
+ // valid `T`.
+ // - We keep `self` alive by wrapping it in a handle below.
+ unsafe { T::start(self_ptr, expires) };
+
+ PinHrTimerHandle { inner: self }
+ }
+}
+
+impl<'a, T> RawHrTimerCallback for Pin<&'a T>
+where
+ T: HasHrTimer<T>,
+ T: HrTimerCallback<Pointer<'a> = Self>,
+{
+ type CallbackTarget<'b> = Self;
+
+ unsafe extern "C" fn run(ptr: *mut bindings::hrtimer) -> bindings::hrtimer_restart {
+ // `HrTimer` is `repr(C)`
+ let timer_ptr = ptr as *mut HrTimer<T>;
+
+ // SAFETY: By the safety requirement of this function, `timer_ptr`
+ // points to a `HrTimer<T>` contained in an `T`.
+ let receiver_ptr = unsafe { T::timer_container_of(timer_ptr) };
+
+ // SAFETY:
+ // - By the safety requirement of this function, `timer_ptr`
+ // points to a `HrTimer<T>` contained in an `T`.
+ // - As per the safety requirements of the trait `HrTimerHandle`, the
+ // `PinHrTimerHandle` associated with this timer is guaranteed to
+ // be alive until this method returns. That handle borrows the `T`
+ // behind `receiver_ptr`, thus guaranteeing the validity of
+ // the reference created below.
+ let receiver_ref = unsafe { &*receiver_ptr };
+
+ // SAFETY: `receiver_ref` only exists as pinned, so it is safe to pin it
+ // here.
+ let receiver_pin = unsafe { Pin::new_unchecked(receiver_ref) };
+
+ T::run(receiver_pin).into_c()
+ }
+}
diff --git a/rust/kernel/time/hrtimer/pin_mut.rs b/rust/kernel/time/hrtimer/pin_mut.rs
new file mode 100644
index 000000000000..6033572d35ad
--- /dev/null
+++ b/rust/kernel/time/hrtimer/pin_mut.rs
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use super::{
+ HasHrTimer, HrTimer, HrTimerCallback, HrTimerHandle, Ktime, RawHrTimerCallback,
+ UnsafeHrTimerPointer,
+};
+use core::{marker::PhantomData, pin::Pin, ptr::NonNull};
+
+/// A handle for a `Pin<&mut HasHrTimer>`. When the handle exists, the timer might
+/// be running.
+pub struct PinMutHrTimerHandle<'a, T>
+where
+ T: HasHrTimer<T>,
+{
+ pub(crate) inner: NonNull<T>,
+ _p: PhantomData<&'a mut T>,
+}
+
+// SAFETY: We cancel the timer when the handle is dropped. The implementation of
+// the `cancel` method will block if the timer handler is running.
+unsafe impl<'a, T> HrTimerHandle for PinMutHrTimerHandle<'a, T>
+where
+ T: HasHrTimer<T>,
+{
+ fn cancel(&mut self) -> bool {
+ let self_ptr = self.inner.as_ptr();
+
+ // SAFETY: As we got `self_ptr` from a reference above, it must point to
+ // a valid `T`.
+ let timer_ptr = unsafe { <T as HasHrTimer<T>>::raw_get_timer(self_ptr) };
+
+ // SAFETY: As `timer_ptr` is derived from a reference, it must point to
+ // a valid and initialized `HrTimer`.
+ unsafe { HrTimer::<T>::raw_cancel(timer_ptr) }
+ }
+}
+
+impl<'a, T> Drop for PinMutHrTimerHandle<'a, T>
+where
+ T: HasHrTimer<T>,
+{
+ fn drop(&mut self) {
+ self.cancel();
+ }
+}
+
+// SAFETY: We capture the lifetime of `Self` when we create a
+// `PinMutHrTimerHandle`, so `Self` will outlive the handle.
+unsafe impl<'a, T> UnsafeHrTimerPointer for Pin<&'a mut T>
+where
+ T: Send + Sync,
+ T: HasHrTimer<T>,
+ T: HrTimerCallback<Pointer<'a> = Self>,
+{
+ type TimerHandle = PinMutHrTimerHandle<'a, T>;
+
+ unsafe fn start(mut self, expires: Ktime) -> Self::TimerHandle {
+ // SAFETY:
+ // - We promise not to move out of `self`. We only pass `self`
+ // back to the caller as a `Pin<&mut self>`.
+ // - The return value of `get_unchecked_mut` is guaranteed not to be null.
+ let self_ptr = unsafe { NonNull::new_unchecked(self.as_mut().get_unchecked_mut()) };
+
+ // SAFETY:
+ // - As we derive `self_ptr` from a reference above, it must point to a
+ // valid `T`.
+ // - We keep `self` alive by wrapping it in a handle below.
+ unsafe { T::start(self_ptr.as_ptr(), expires) };
+
+ PinMutHrTimerHandle {
+ inner: self_ptr,
+ _p: PhantomData,
+ }
+ }
+}
+
+impl<'a, T> RawHrTimerCallback for Pin<&'a mut T>
+where
+ T: HasHrTimer<T>,
+ T: HrTimerCallback<Pointer<'a> = Self>,
+{
+ type CallbackTarget<'b> = Self;
+
+ unsafe extern "C" fn run(ptr: *mut bindings::hrtimer) -> bindings::hrtimer_restart {
+ // `HrTimer` is `repr(C)`
+ let timer_ptr = ptr as *mut HrTimer<T>;
+
+ // SAFETY: By the safety requirement of this function, `timer_ptr`
+ // points to a `HrTimer<T>` contained in an `T`.
+ let receiver_ptr = unsafe { T::timer_container_of(timer_ptr) };
+
+ // SAFETY:
+ // - By the safety requirement of this function, `timer_ptr`
+ // points to a `HrTimer<T>` contained in an `T`.
+ // - As per the safety requirements of the trait `HrTimerHandle`, the
+ // `PinMutHrTimerHandle` associated with this timer is guaranteed to
+ // be alive until this method returns. That handle borrows the `T`
+ // behind `receiver_ptr` mutably thus guaranteeing the validity of
+ // the reference created below.
+ let receiver_ref = unsafe { &mut *receiver_ptr };
+
+ // SAFETY: `receiver_ref` only exists as pinned, so it is safe to pin it
+ // here.
+ let receiver_pin = unsafe { Pin::new_unchecked(receiver_ref) };
+
+ T::run(receiver_pin).into_c()
+ }
+}
diff --git a/rust/kernel/time/hrtimer/tbox.rs b/rust/kernel/time/hrtimer/tbox.rs
new file mode 100644
index 000000000000..29526a5da203
--- /dev/null
+++ b/rust/kernel/time/hrtimer/tbox.rs
@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use super::HasHrTimer;
+use super::HrTimer;
+use super::HrTimerCallback;
+use super::HrTimerHandle;
+use super::HrTimerPointer;
+use super::Ktime;
+use super::RawHrTimerCallback;
+use crate::prelude::*;
+use core::ptr::NonNull;
+
+/// A handle for a [`Box<HasHrTimer<T>>`] returned by a call to
+/// [`HrTimerPointer::start`].
+///
+/// # Invariants
+///
+/// - `self.inner` comes from a `Box::into_raw` call.
+pub struct BoxHrTimerHandle<T, A>
+where
+ T: HasHrTimer<T>,
+ A: crate::alloc::Allocator,
+{
+ pub(crate) inner: NonNull<T>,
+ _p: core::marker::PhantomData<A>,
+}
+
+// SAFETY: We implement drop below, and we cancel the timer in the drop
+// implementation.
+unsafe impl<T, A> HrTimerHandle for BoxHrTimerHandle<T, A>
+where
+ T: HasHrTimer<T>,
+ A: crate::alloc::Allocator,
+{
+ fn cancel(&mut self) -> bool {
+ // SAFETY: As we obtained `self.inner` from a valid reference when we
+ // created `self`, it must point to a valid `T`.
+ let timer_ptr = unsafe { <T as HasHrTimer<T>>::raw_get_timer(self.inner.as_ptr()) };
+
+ // SAFETY: As `timer_ptr` points into `T` and `T` is valid, `timer_ptr`
+ // must point to a valid `HrTimer` instance.
+ unsafe { HrTimer::<T>::raw_cancel(timer_ptr) }
+ }
+}
+
+impl<T, A> Drop for BoxHrTimerHandle<T, A>
+where
+ T: HasHrTimer<T>,
+ A: crate::alloc::Allocator,
+{
+ fn drop(&mut self) {
+ self.cancel();
+ // SAFETY: By type invariant, `self.inner` came from a `Box::into_raw`
+ // call.
+ drop(unsafe { Box::<T, A>::from_raw(self.inner.as_ptr()) })
+ }
+}
+
+impl<T, A> HrTimerPointer for Pin<Box<T, A>>
+where
+ T: 'static,
+ T: Send + Sync,
+ T: HasHrTimer<T>,
+ T: for<'a> HrTimerCallback<Pointer<'a> = Pin<Box<T, A>>>,
+ A: crate::alloc::Allocator,
+{
+ type TimerHandle = BoxHrTimerHandle<T, A>;
+
+ fn start(self, expires: Ktime) -> Self::TimerHandle {
+ // SAFETY:
+ // - We will not move out of this box during timer callback (we pass an
+ // immutable reference to the callback).
+ // - `Box::into_raw` is guaranteed to return a valid pointer.
+ let inner =
+ unsafe { NonNull::new_unchecked(Box::into_raw(Pin::into_inner_unchecked(self))) };
+
+ // SAFETY:
+ // - We keep `self` alive by wrapping it in a handle below.
+ // - Since we generate the pointer passed to `start` from a valid
+ // reference, it is a valid pointer.
+ unsafe { T::start(inner.as_ptr(), expires) };
+
+ // INVARIANT: `inner` came from `Box::into_raw` above.
+ BoxHrTimerHandle {
+ inner,
+ _p: core::marker::PhantomData,
+ }
+ }
+}
+
+impl<T, A> RawHrTimerCallback for Pin<Box<T, A>>
+where
+ T: 'static,
+ T: HasHrTimer<T>,
+ T: for<'a> HrTimerCallback<Pointer<'a> = Pin<Box<T, A>>>,
+ A: crate::alloc::Allocator,
+{
+ type CallbackTarget<'a> = Pin<&'a mut T>;
+
+ unsafe extern "C" fn run(ptr: *mut bindings::hrtimer) -> bindings::hrtimer_restart {
+ // `HrTimer` is `repr(C)`
+ let timer_ptr = ptr.cast::<super::HrTimer<T>>();
+
+ // SAFETY: By C API contract `ptr` is the pointer we passed when
+ // queuing the timer, so it is a `HrTimer<T>` embedded in a `T`.
+ let data_ptr = unsafe { T::timer_container_of(timer_ptr) };
+
+ // SAFETY:
+ // - As per the safety requirements of the trait `HrTimerHandle`, the
+ // `BoxHrTimerHandle` associated with this timer is guaranteed to
+ // be alive until this method returns. That handle owns the `T`
+ // behind `data_ptr` thus guaranteeing the validity of
+ // the reference created below.
+ // - As `data_ptr` comes from a `Pin<Box<T>>`, only pinned references to
+ // `data_ptr` exist.
+ let data_mut_ref = unsafe { Pin::new_unchecked(&mut *data_ptr) };
+
+ T::run(data_mut_ref).into_c()
+ }
+}
diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs
index 2bbaab83b9d6..22985b6f6982 100644
--- a/rust/kernel/types.rs
+++ b/rust/kernel/types.rs
@@ -2,7 +2,6 @@
//! Kernel types.
-use crate::init::{self, PinInit};
use core::{
cell::UnsafeCell,
marker::{PhantomData, PhantomPinned},
@@ -10,6 +9,7 @@ use core::{
ops::{Deref, DerefMut},
ptr::NonNull,
};
+use pin_init::{PinInit, Zeroable};
/// Used to transfer ownership to and from foreign (non-Rust) languages.
///
@@ -18,7 +18,19 @@ use core::{
///
/// This trait is meant to be used in cases when Rust objects are stored in C objects and
/// eventually "freed" back to Rust.
-pub trait ForeignOwnable: Sized {
+///
+/// # Safety
+///
+/// Implementers must ensure that [`into_foreign`] returns a pointer which meets the alignment
+/// requirements of [`PointedTo`].
+///
+/// [`into_foreign`]: Self::into_foreign
+/// [`PointedTo`]: Self::PointedTo
+pub unsafe trait ForeignOwnable: Sized {
+ /// Type used when the value is foreign-owned. In practical terms only defines the alignment of
+ /// the pointer.
+ type PointedTo;
+
/// Type used to immutably borrow a value that is currently foreign-owned.
type Borrowed<'a>;
@@ -27,16 +39,18 @@ pub trait ForeignOwnable: Sized {
/// Converts a Rust-owned object to a foreign-owned one.
///
- /// The foreign representation is a pointer to void. There are no guarantees for this pointer.
- /// For example, it might be invalid, dangling or pointing to uninitialized memory. Using it in
- /// any way except for [`from_foreign`], [`try_from_foreign`], [`borrow`], or [`borrow_mut`] can
- /// result in undefined behavior.
+ /// # Guarantees
+ ///
+ /// The return value is guaranteed to be well-aligned, but there are no other guarantees for
+ /// this pointer. For example, it might be null, dangling, or point to uninitialized memory.
+ /// Using it in any way except for [`ForeignOwnable::from_foreign`], [`ForeignOwnable::borrow`],
+ /// [`ForeignOwnable::try_from_foreign`] can result in undefined behavior.
///
/// [`from_foreign`]: Self::from_foreign
/// [`try_from_foreign`]: Self::try_from_foreign
/// [`borrow`]: Self::borrow
/// [`borrow_mut`]: Self::borrow_mut
- fn into_foreign(self) -> *mut crate::ffi::c_void;
+ fn into_foreign(self) -> *mut Self::PointedTo;
/// Converts a foreign-owned object back to a Rust-owned one.
///
@@ -46,7 +60,7 @@ pub trait ForeignOwnable: Sized {
/// must not be passed to `from_foreign` more than once.
///
/// [`into_foreign`]: Self::into_foreign
- unsafe fn from_foreign(ptr: *mut crate::ffi::c_void) -> Self;
+ unsafe fn from_foreign(ptr: *mut Self::PointedTo) -> Self;
/// Tries to convert a foreign-owned object back to a Rust-owned one.
///
@@ -58,7 +72,7 @@ pub trait ForeignOwnable: Sized {
/// `ptr` must either be null or satisfy the safety requirements for [`from_foreign`].
///
/// [`from_foreign`]: Self::from_foreign
- unsafe fn try_from_foreign(ptr: *mut crate::ffi::c_void) -> Option<Self> {
+ unsafe fn try_from_foreign(ptr: *mut Self::PointedTo) -> Option<Self> {
if ptr.is_null() {
None
} else {
@@ -77,11 +91,11 @@ pub trait ForeignOwnable: Sized {
///
/// The provided pointer must have been returned by a previous call to [`into_foreign`], and if
/// the pointer is ever passed to [`from_foreign`], then that call must happen after the end of
- /// the lifetime 'a.
+ /// the lifetime `'a`.
///
/// [`into_foreign`]: Self::into_foreign
/// [`from_foreign`]: Self::from_foreign
- unsafe fn borrow<'a>(ptr: *mut crate::ffi::c_void) -> Self::Borrowed<'a>;
+ unsafe fn borrow<'a>(ptr: *mut Self::PointedTo) -> Self::Borrowed<'a>;
/// Borrows a foreign-owned object mutably.
///
@@ -100,30 +114,32 @@ pub trait ForeignOwnable: Sized {
///
/// The provided pointer must have been returned by a previous call to [`into_foreign`], and if
/// the pointer is ever passed to [`from_foreign`], then that call must happen after the end of
- /// the lifetime 'a.
+ /// the lifetime `'a`.
///
- /// The lifetime 'a must not overlap with the lifetime of any other call to [`borrow`] or
+ /// The lifetime `'a` must not overlap with the lifetime of any other call to [`borrow`] or
/// `borrow_mut` on the same object.
///
/// [`into_foreign`]: Self::into_foreign
/// [`from_foreign`]: Self::from_foreign
/// [`borrow`]: Self::borrow
/// [`Arc`]: crate::sync::Arc
- unsafe fn borrow_mut<'a>(ptr: *mut crate::ffi::c_void) -> Self::BorrowedMut<'a>;
+ unsafe fn borrow_mut<'a>(ptr: *mut Self::PointedTo) -> Self::BorrowedMut<'a>;
}
-impl ForeignOwnable for () {
+// SAFETY: The `into_foreign` function returns a pointer that is dangling, but well-aligned.
+unsafe impl ForeignOwnable for () {
+ type PointedTo = ();
type Borrowed<'a> = ();
type BorrowedMut<'a> = ();
- fn into_foreign(self) -> *mut crate::ffi::c_void {
+ fn into_foreign(self) -> *mut Self::PointedTo {
core::ptr::NonNull::dangling().as_ptr()
}
- unsafe fn from_foreign(_: *mut crate::ffi::c_void) -> Self {}
+ unsafe fn from_foreign(_: *mut Self::PointedTo) -> Self {}
- unsafe fn borrow<'a>(_: *mut crate::ffi::c_void) -> Self::Borrowed<'a> {}
- unsafe fn borrow_mut<'a>(_: *mut crate::ffi::c_void) -> Self::BorrowedMut<'a> {}
+ unsafe fn borrow<'a>(_: *mut Self::PointedTo) -> Self::Borrowed<'a> {}
+ unsafe fn borrow_mut<'a>(_: *mut Self::PointedTo) -> Self::BorrowedMut<'a> {}
}
/// Runs a cleanup function/closure when dropped.
@@ -251,7 +267,7 @@ impl<T, F: FnOnce(T)> Drop for ScopeGuard<T, F> {
/// Stores an opaque value.
///
-/// `Opaque<T>` is meant to be used with FFI objects that are never interpreted by Rust code.
+/// [`Opaque<T>`] is meant to be used with FFI objects that are never interpreted by Rust code.
///
/// It is used to wrap structs from the C side, like for example `Opaque<bindings::mutex>`.
/// It gets rid of all the usual assumptions that Rust has for a value:
@@ -266,7 +282,7 @@ impl<T, F: FnOnce(T)> Drop for ScopeGuard<T, F> {
/// This has to be used for all values that the C side has access to, because it can't be ensured
/// that the C side is adhering to the usual constraints that Rust needs.
///
-/// Using `Opaque<T>` allows to continue to use references on the Rust side even for values shared
+/// Using [`Opaque<T>`] allows to continue to use references on the Rust side even for values shared
/// with C.
///
/// # Examples
@@ -309,6 +325,9 @@ pub struct Opaque<T> {
_pin: PhantomPinned,
}
+// SAFETY: `Opaque<T>` allows the inner value to be any bit pattern, including all zeros.
+unsafe impl<T> Zeroable for Opaque<T> {}
+
impl<T> Opaque<T> {
/// Creates a new opaque value.
pub const fn new(value: T) -> Self {
@@ -326,6 +345,14 @@ impl<T> Opaque<T> {
}
}
+ /// Creates a new zeroed opaque value.
+ pub const fn zeroed() -> Self {
+ Self {
+ value: UnsafeCell::new(MaybeUninit::zeroed()),
+ _pin: PhantomPinned,
+ }
+ }
+
/// Create an opaque pin-initializer from the given pin-initializer.
pub fn pin_init(slot: impl PinInit<T>) -> impl PinInit<Self> {
Self::ffi_init(|ptr: *mut T| {
@@ -333,7 +360,7 @@ impl<T> Opaque<T> {
// - `ptr` is a valid pointer to uninitialized memory,
// - `slot` is not accessed on error; the call is infallible,
// - `slot` is pinned in memory.
- let _ = unsafe { init::PinInit::<T>::__pinned_init(slot, ptr) };
+ let _ = unsafe { PinInit::<T>::__pinned_init(slot, ptr) };
})
}
@@ -349,7 +376,7 @@ impl<T> Opaque<T> {
// SAFETY: We contain a `MaybeUninit`, so it is OK for the `init_func` to not fully
// initialize the `T`.
unsafe {
- init::pin_init_from_closure::<_, ::core::convert::Infallible>(move |slot| {
+ pin_init::pin_init_from_closure::<_, ::core::convert::Infallible>(move |slot| {
init_func(Self::raw_get(slot));
Ok(())
})
@@ -369,7 +396,9 @@ impl<T> Opaque<T> {
) -> impl PinInit<Self, E> {
// SAFETY: We contain a `MaybeUninit`, so it is OK for the `init_func` to not fully
// initialize the `T`.
- unsafe { init::pin_init_from_closure::<_, E>(move |slot| init_func(Self::raw_get(slot))) }
+ unsafe {
+ pin_init::pin_init_from_closure::<_, E>(move |slot| init_func(Self::raw_get(slot)))
+ }
}
/// Returns a raw pointer to the opaque data.
diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs
index 719b0a48ff55..6d70edd8086a 100644
--- a/rust/kernel/uaccess.rs
+++ b/rust/kernel/uaccess.rs
@@ -46,10 +46,9 @@ pub type UserPtr = usize;
///
/// ```no_run
/// use kernel::ffi::c_void;
-/// use kernel::error::Result;
/// use kernel::uaccess::{UserPtr, UserSlice};
///
-/// fn bytes_add_one(uptr: UserPtr, len: usize) -> Result<()> {
+/// fn bytes_add_one(uptr: UserPtr, len: usize) -> Result {
/// let (read, mut write) = UserSlice::new(uptr, len).reader_writer();
///
/// let mut buf = KVec::new();
@@ -68,7 +67,6 @@ pub type UserPtr = usize;
///
/// ```no_run
/// use kernel::ffi::c_void;
-/// use kernel::error::{code::EINVAL, Result};
/// use kernel::uaccess::{UserPtr, UserSlice};
///
/// /// Returns whether the data in this region is valid.
@@ -285,13 +283,12 @@ impl UserSliceReader {
let len = self.length;
buf.reserve(len, flags)?;
- // The call to `try_reserve` was successful, so the spare capacity is at least `len` bytes
- // long.
+ // The call to `reserve` was successful, so the spare capacity is at least `len` bytes long.
self.read_raw(&mut buf.spare_capacity_mut()[..len])?;
// SAFETY: Since the call to `read_raw` was successful, so the next `len` bytes of the
// vector have been initialized.
- unsafe { buf.set_len(buf.len() + len) };
+ unsafe { buf.inc_len(len) };
Ok(())
}
}
diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs
index 0cd100d2aefb..d092112d843f 100644
--- a/rust/kernel/workqueue.rs
+++ b/rust/kernel/workqueue.rs
@@ -60,7 +60,7 @@
//! type Pointer = Arc<MyStruct>;
//!
//! fn run(this: Arc<MyStruct>) {
-//! pr_info!("The value is: {}", this.value);
+//! pr_info!("The value is: {}\n", this.value);
//! }
//! }
//!
@@ -108,7 +108,7 @@
//! type Pointer = Arc<MyStruct>;
//!
//! fn run(this: Arc<MyStruct>) {
-//! pr_info!("The value is: {}", this.value_1);
+//! pr_info!("The value is: {}\n", this.value_1);
//! }
//! }
//!
@@ -116,7 +116,7 @@
//! type Pointer = Arc<MyStruct>;
//!
//! fn run(this: Arc<MyStruct>) {
-//! pr_info!("The second value is: {}", this.value_2);
+//! pr_info!("The second value is: {}\n", this.value_2);
//! }
//! }
//!
@@ -369,7 +369,7 @@ unsafe impl<T: ?Sized, const ID: u64> Sync for Work<T, ID> {}
impl<T: ?Sized, const ID: u64> Work<T, ID> {
/// Creates a new instance of [`Work`].
#[inline]
- pub fn new(name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self>
+ pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self>
where
T: WorkItem<ID>,
{
@@ -429,51 +429,28 @@ impl<T: ?Sized, const ID: u64> Work<T, ID> {
///
/// # Safety
///
-/// The [`OFFSET`] constant must be the offset of a field in `Self` of type [`Work<T, ID>`]. The
-/// methods on this trait must have exactly the behavior that the definitions given below have.
+/// The methods [`raw_get_work`] and [`work_container_of`] must return valid pointers and must be
+/// true inverses of each other; that is, they must satisfy the following invariants:
+/// - `work_container_of(raw_get_work(ptr)) == ptr` for any `ptr: *mut Self`.
+/// - `raw_get_work(work_container_of(ptr)) == ptr` for any `ptr: *mut Work<T, ID>`.
///
/// [`impl_has_work!`]: crate::impl_has_work
-/// [`OFFSET`]: HasWork::OFFSET
+/// [`raw_get_work`]: HasWork::raw_get_work
+/// [`work_container_of`]: HasWork::work_container_of
pub unsafe trait HasWork<T, const ID: u64 = 0> {
- /// The offset of the [`Work<T, ID>`] field.
- const OFFSET: usize;
-
- /// Returns the offset of the [`Work<T, ID>`] field.
- ///
- /// This method exists because the [`OFFSET`] constant cannot be accessed if the type is not
- /// [`Sized`].
- ///
- /// [`OFFSET`]: HasWork::OFFSET
- #[inline]
- fn get_work_offset(&self) -> usize {
- Self::OFFSET
- }
-
/// Returns a pointer to the [`Work<T, ID>`] field.
///
/// # Safety
///
/// The provided pointer must point at a valid struct of type `Self`.
- #[inline]
- unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work<T, ID> {
- // SAFETY: The caller promises that the pointer is valid.
- unsafe { (ptr as *mut u8).add(Self::OFFSET) as *mut Work<T, ID> }
- }
+ unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work<T, ID>;
/// Returns a pointer to the struct containing the [`Work<T, ID>`] field.
///
/// # Safety
///
/// The pointer must point at a [`Work<T, ID>`] field in a struct of type `Self`.
- #[inline]
- unsafe fn work_container_of(ptr: *mut Work<T, ID>) -> *mut Self
- where
- Self: Sized,
- {
- // SAFETY: The caller promises that the pointer points at a field of the right type in the
- // right kind of struct.
- unsafe { (ptr as *mut u8).sub(Self::OFFSET) as *mut Self }
- }
+ unsafe fn work_container_of(ptr: *mut Work<T, ID>) -> *mut Self;
}
/// Used to safely implement the [`HasWork<T, ID>`] trait.
@@ -504,8 +481,6 @@ macro_rules! impl_has_work {
// SAFETY: The implementation of `raw_get_work` only compiles if the field has the right
// type.
unsafe impl$(<$($generics)+>)? $crate::workqueue::HasWork<$work_type $(, $id)?> for $self {
- const OFFSET: usize = ::core::mem::offset_of!(Self, $field) as usize;
-
#[inline]
unsafe fn raw_get_work(ptr: *mut Self) -> *mut $crate::workqueue::Work<$work_type $(, $id)?> {
// SAFETY: The caller promises that the pointer is not dangling.
@@ -513,6 +488,15 @@ macro_rules! impl_has_work {
::core::ptr::addr_of_mut!((*ptr).$field)
}
}
+
+ #[inline]
+ unsafe fn work_container_of(
+ ptr: *mut $crate::workqueue::Work<$work_type $(, $id)?>,
+ ) -> *mut Self {
+ // SAFETY: The caller promises that the pointer points at a field of the right type
+ // in the right kind of struct.
+ unsafe { $crate::container_of!(ptr, Self, $field) }
+ }
}
)*};
}
@@ -703,3 +687,21 @@ pub fn system_freezable_power_efficient() -> &'static Queue {
// SAFETY: `system_freezable_power_efficient_wq` is a C global, always available.
unsafe { Queue::from_raw(bindings::system_freezable_power_efficient_wq) }
}
+
+/// Returns the system bottom halves work queue (`system_bh_wq`).
+///
+/// It is similar to the one returned by [`system`] but for work items which
+/// need to run from a softirq context.
+pub fn system_bh() -> &'static Queue {
+ // SAFETY: `system_bh_wq` is a C global, always available.
+ unsafe { Queue::from_raw(bindings::system_bh_wq) }
+}
+
+/// Returns the system bottom halves high-priority work queue (`system_bh_highpri_wq`).
+///
+/// It is similar to the one returned by [`system_bh`] but for work items which
+/// require higher scheduling priority.
+pub fn system_bh_highpri() -> &'static Queue {
+ // SAFETY: `system_bh_highpri_wq` is a C global, always available.
+ unsafe { Queue::from_raw(bindings::system_bh_highpri_wq) }
+}
diff --git a/rust/kernel/xarray.rs b/rust/kernel/xarray.rs
new file mode 100644
index 000000000000..75719e7bb491
--- /dev/null
+++ b/rust/kernel/xarray.rs
@@ -0,0 +1,275 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! XArray abstraction.
+//!
+//! C header: [`include/linux/xarray.h`](srctree/include/linux/xarray.h)
+
+use crate::{
+ alloc, bindings, build_assert,
+ error::{Error, Result},
+ types::{ForeignOwnable, NotThreadSafe, Opaque},
+};
+use core::{iter, marker::PhantomData, mem, pin::Pin, ptr::NonNull};
+use pin_init::{pin_data, pin_init, pinned_drop, PinInit};
+
+/// An array which efficiently maps sparse integer indices to owned objects.
+///
+/// This is similar to a [`crate::alloc::kvec::Vec<Option<T>>`], but more efficient when there are
+/// holes in the index space, and can be efficiently grown.
+///
+/// # Invariants
+///
+/// `self.xa` is always an initialized and valid [`bindings::xarray`] whose entries are either
+/// `XA_ZERO_ENTRY` or came from `T::into_foreign`.
+///
+/// # Examples
+///
+/// ```rust
+/// use kernel::alloc::KBox;
+/// use kernel::xarray::{AllocKind, XArray};
+///
+/// let xa = KBox::pin_init(XArray::new(AllocKind::Alloc1), GFP_KERNEL)?;
+///
+/// let dead = KBox::new(0xdead, GFP_KERNEL)?;
+/// let beef = KBox::new(0xbeef, GFP_KERNEL)?;
+///
+/// let mut guard = xa.lock();
+///
+/// assert_eq!(guard.get(0), None);
+///
+/// assert_eq!(guard.store(0, dead, GFP_KERNEL)?.as_deref(), None);
+/// assert_eq!(guard.get(0).copied(), Some(0xdead));
+///
+/// *guard.get_mut(0).unwrap() = 0xffff;
+/// assert_eq!(guard.get(0).copied(), Some(0xffff));
+///
+/// assert_eq!(guard.store(0, beef, GFP_KERNEL)?.as_deref().copied(), Some(0xffff));
+/// assert_eq!(guard.get(0).copied(), Some(0xbeef));
+///
+/// guard.remove(0);
+/// assert_eq!(guard.get(0), None);
+///
+/// # Ok::<(), Error>(())
+/// ```
+#[pin_data(PinnedDrop)]
+pub struct XArray<T: ForeignOwnable> {
+ #[pin]
+ xa: Opaque<bindings::xarray>,
+ _p: PhantomData<T>,
+}
+
+#[pinned_drop]
+impl<T: ForeignOwnable> PinnedDrop for XArray<T> {
+ fn drop(self: Pin<&mut Self>) {
+ self.iter().for_each(|ptr| {
+ let ptr = ptr.as_ptr();
+ // SAFETY: `ptr` came from `T::into_foreign`.
+ //
+ // INVARIANT: we own the only reference to the array which is being dropped so the
+ // broken invariant is not observable on function exit.
+ drop(unsafe { T::from_foreign(ptr) })
+ });
+
+ // SAFETY: `self.xa` is always valid by the type invariant.
+ unsafe { bindings::xa_destroy(self.xa.get()) };
+ }
+}
+
+/// Flags passed to [`XArray::new`] to configure the array's allocation tracking behavior.
+pub enum AllocKind {
+ /// Consider the first element to be at index 0.
+ Alloc,
+ /// Consider the first element to be at index 1.
+ Alloc1,
+}
+
+impl<T: ForeignOwnable> XArray<T> {
+ /// Creates a new initializer for this type.
+ pub fn new(kind: AllocKind) -> impl PinInit<Self> {
+ let flags = match kind {
+ AllocKind::Alloc => bindings::XA_FLAGS_ALLOC,
+ AllocKind::Alloc1 => bindings::XA_FLAGS_ALLOC1,
+ };
+ pin_init!(Self {
+ // SAFETY: `xa` is valid while the closure is called.
+ //
+ // INVARIANT: `xa` is initialized here to an empty, valid [`bindings::xarray`].
+ xa <- Opaque::ffi_init(|xa| unsafe {
+ bindings::xa_init_flags(xa, flags)
+ }),
+ _p: PhantomData,
+ })
+ }
+
+ fn iter(&self) -> impl Iterator<Item = NonNull<T::PointedTo>> + '_ {
+ let mut index = 0;
+
+ // SAFETY: `self.xa` is always valid by the type invariant.
+ iter::once(unsafe {
+ bindings::xa_find(self.xa.get(), &mut index, usize::MAX, bindings::XA_PRESENT)
+ })
+ .chain(iter::from_fn(move || {
+ // SAFETY: `self.xa` is always valid by the type invariant.
+ Some(unsafe {
+ bindings::xa_find_after(self.xa.get(), &mut index, usize::MAX, bindings::XA_PRESENT)
+ })
+ }))
+ .map_while(|ptr| NonNull::new(ptr.cast()))
+ }
+
+ /// Attempts to lock the [`XArray`] for exclusive access.
+ pub fn try_lock(&self) -> Option<Guard<'_, T>> {
+ // SAFETY: `self.xa` is always valid by the type invariant.
+ if (unsafe { bindings::xa_trylock(self.xa.get()) } != 0) {
+ Some(Guard {
+ xa: self,
+ _not_send: NotThreadSafe,
+ })
+ } else {
+ None
+ }
+ }
+
+ /// Locks the [`XArray`] for exclusive access.
+ pub fn lock(&self) -> Guard<'_, T> {
+ // SAFETY: `self.xa` is always valid by the type invariant.
+ unsafe { bindings::xa_lock(self.xa.get()) };
+
+ Guard {
+ xa: self,
+ _not_send: NotThreadSafe,
+ }
+ }
+}
+
+/// A lock guard.
+///
+/// The lock is unlocked when the guard goes out of scope.
+#[must_use = "the lock unlocks immediately when the guard is unused"]
+pub struct Guard<'a, T: ForeignOwnable> {
+ xa: &'a XArray<T>,
+ _not_send: NotThreadSafe,
+}
+
+impl<T: ForeignOwnable> Drop for Guard<'_, T> {
+ fn drop(&mut self) {
+ // SAFETY:
+ // - `self.xa.xa` is always valid by the type invariant.
+ // - The caller holds the lock, so it is safe to unlock it.
+ unsafe { bindings::xa_unlock(self.xa.xa.get()) };
+ }
+}
+
+/// The error returned by [`store`](Guard::store).
+///
+/// Contains the underlying error and the value that was not stored.
+pub struct StoreError<T> {
+ /// The error that occurred.
+ pub error: Error,
+ /// The value that was not stored.
+ pub value: T,
+}
+
+impl<T> From<StoreError<T>> for Error {
+ fn from(value: StoreError<T>) -> Self {
+ value.error
+ }
+}
+
+impl<'a, T: ForeignOwnable> Guard<'a, T> {
+ fn load<F, U>(&self, index: usize, f: F) -> Option<U>
+ where
+ F: FnOnce(NonNull<T::PointedTo>) -> U,
+ {
+ // SAFETY: `self.xa.xa` is always valid by the type invariant.
+ let ptr = unsafe { bindings::xa_load(self.xa.xa.get(), index) };
+ let ptr = NonNull::new(ptr.cast())?;
+ Some(f(ptr))
+ }
+
+ /// Provides a reference to the element at the given index.
+ pub fn get(&self, index: usize) -> Option<T::Borrowed<'_>> {
+ self.load(index, |ptr| {
+ // SAFETY: `ptr` came from `T::into_foreign`.
+ unsafe { T::borrow(ptr.as_ptr()) }
+ })
+ }
+
+ /// Provides a mutable reference to the element at the given index.
+ pub fn get_mut(&mut self, index: usize) -> Option<T::BorrowedMut<'_>> {
+ self.load(index, |ptr| {
+ // SAFETY: `ptr` came from `T::into_foreign`.
+ unsafe { T::borrow_mut(ptr.as_ptr()) }
+ })
+ }
+
+ /// Removes and returns the element at the given index.
+ pub fn remove(&mut self, index: usize) -> Option<T> {
+ // SAFETY:
+ // - `self.xa.xa` is always valid by the type invariant.
+ // - The caller holds the lock.
+ let ptr = unsafe { bindings::__xa_erase(self.xa.xa.get(), index) }.cast();
+ // SAFETY:
+ // - `ptr` is either NULL or came from `T::into_foreign`.
+ // - `&mut self` guarantees that the lifetimes of [`T::Borrowed`] and [`T::BorrowedMut`]
+ // borrowed from `self` have ended.
+ unsafe { T::try_from_foreign(ptr) }
+ }
+
+ /// Stores an element at the given index.
+ ///
+ /// May drop the lock if needed to allocate memory, and then reacquire it afterwards.
+ ///
+ /// On success, returns the element which was previously at the given index.
+ ///
+ /// On failure, returns the element which was attempted to be stored.
+ pub fn store(
+ &mut self,
+ index: usize,
+ value: T,
+ gfp: alloc::Flags,
+ ) -> Result<Option<T>, StoreError<T>> {
+ build_assert!(
+ mem::align_of::<T::PointedTo>() >= 4,
+ "pointers stored in XArray must be 4-byte aligned"
+ );
+ let new = value.into_foreign();
+
+ let old = {
+ let new = new.cast();
+ // SAFETY:
+ // - `self.xa.xa` is always valid by the type invariant.
+ // - The caller holds the lock.
+ //
+ // INVARIANT: `new` came from `T::into_foreign`.
+ unsafe { bindings::__xa_store(self.xa.xa.get(), index, new, gfp.as_raw()) }
+ };
+
+ // SAFETY: `__xa_store` returns the old entry at this index on success or `xa_err` if an
+ // error happened.
+ let errno = unsafe { bindings::xa_err(old) };
+ if errno != 0 {
+ // SAFETY: `new` came from `T::into_foreign` and `__xa_store` does not take
+ // ownership of the value on error.
+ let value = unsafe { T::from_foreign(new) };
+ Err(StoreError {
+ value,
+ error: Error::from_errno(errno),
+ })
+ } else {
+ let old = old.cast();
+ // SAFETY: `ptr` is either NULL or came from `T::into_foreign`.
+ //
+ // NB: `XA_ZERO_ENTRY` is never returned by functions belonging to the Normal XArray
+ // API; such entries present as `NULL`.
+ Ok(unsafe { T::try_from_foreign(old) })
+ }
+ }
+}
+
+// SAFETY: `XArray<T>` has no shared mutable state so it is `Send` iff `T` is `Send`.
+unsafe impl<T: ForeignOwnable + Send> Send for XArray<T> {}
+
+// SAFETY: `XArray<T>` serialises the interior mutability it provides so it is `Sync` iff `T` is
+// `Send`.
+unsafe impl<T: ForeignOwnable + Send> Sync for XArray<T> {}