diff options
| author | Miguel Ojeda <ojeda@kernel.org> | 2026-06-08 17:14:27 +0300 |
|---|---|---|
| committer | Miguel Ojeda <ojeda@kernel.org> | 2026-06-09 05:13:22 +0300 |
| commit | c37398010a05055e78cf0c75defb90df06c4e999 (patch) | |
| tree | 30b741b4e0d344508bef8058079ce1359143445a /rust/zerocopy/src/pointer | |
| parent | be43b5d9c26c1f46a89766ec59feb9dffee4c797 (diff) | |
| download | linux-c37398010a05055e78cf0c75defb90df06c4e999.tar.xz | |
rust: zerocopy: import crate
This is a subset of the Rust `zerocopy` crate, version v0.8.50 (released
2026-05-31), licensed under "BSD-2-Clause OR Apache-2.0 OR MIT", from:
https://github.com/google/zerocopy/tree/v0.8.50
The files are copied as-is, with no modifications whatsoever (not even
adding the SPDX identifiers).
The `benches` folder is added (i.e. not just `src` like in other cases)
since the files there are included in the rendered documentation,
as well as the `rustdoc` CSS style file that is needed to make those
visually more understandable.
For copyright details, please see:
https://github.com/google/zerocopy/blob/v0.8.50/README.md?plain=1
https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-BSD
https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-APACHE
https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-MIT
The next two patches modify these files as needed for use within the
kernel. This patch split allows reviewers to double-check the import
and to clearly see the differences introduced.
The following script may be used to verify the contents:
for path in $(cd rust/zerocopy/ && find . -type f); do
curl --silent --show-error --location \
https://github.com/google/zerocopy/raw/v0.8.50/$path \
| diff --unified rust/zerocopy/$path - && echo $path: OK
done
Cc: Joshua Liebow-Feeser <joshlf@google.com>
Cc: Jack Wrenn <jswrenn@google.com>
Link: https://patch.msgid.link/20260608141439.182634-9-ojeda@kernel.org
Signed-off-by: Miguel Ojeda <ojeda@kernel.org>
Diffstat (limited to 'rust/zerocopy/src/pointer')
| -rw-r--r-- | rust/zerocopy/src/pointer/inner.rs | 752 | ||||
| -rw-r--r-- | rust/zerocopy/src/pointer/invariant.rs | 296 | ||||
| -rw-r--r-- | rust/zerocopy/src/pointer/mod.rs | 408 | ||||
| -rw-r--r-- | rust/zerocopy/src/pointer/ptr.rs | 1584 | ||||
| -rw-r--r-- | rust/zerocopy/src/pointer/transmute.rs | 520 |
5 files changed, 3560 insertions, 0 deletions
diff --git a/rust/zerocopy/src/pointer/inner.rs b/rust/zerocopy/src/pointer/inner.rs new file mode 100644 index 000000000000..c8dd9f01f456 --- /dev/null +++ b/rust/zerocopy/src/pointer/inner.rs @@ -0,0 +1,752 @@ +// Copyright 2024 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use core::{marker::PhantomData, ops::Range, ptr::NonNull}; + +pub use _def::PtrInner; + +#[allow(unused_imports)] +use crate::util::polyfills::NumExt as _; +use crate::{ + layout::{CastType, MetadataCastError}, + pointer::cast, + util::AsAddress, + AlignmentError, CastError, KnownLayout, MetadataOf, SizeError, SplitAt, +}; + +mod _def { + use super::*; + /// The inner pointer stored inside a [`Ptr`][crate::Ptr]. + /// + /// `PtrInner<'a, T>` is [covariant] in `'a` and invariant in `T`. + /// + /// [covariant]: https://doc.rust-lang.org/reference/subtyping.html + #[allow(missing_debug_implementations)] + pub struct PtrInner<'a, T> + where + T: ?Sized, + { + /// # Invariants + /// + /// 0. If `ptr`'s referent is not zero sized, then `ptr` has valid + /// provenance for its referent, which is entirely contained in some + /// Rust allocation, `A`. + /// 1. If `ptr`'s referent is not zero sized, `A` is guaranteed to live + /// for at least `'a`. + /// + /// # Postconditions + /// + /// By virtue of these invariants, code may assume the following, which + /// are logical implications of the invariants: + /// - `ptr`'s referent is not larger than `isize::MAX` bytes \[1\] + /// - `ptr`'s referent does not wrap around the address space \[1\] + /// + /// \[1\] Per <https://doc.rust-lang.org/1.85.0/std/ptr/index.html#allocated-object>: + /// + /// For any allocated object with `base` address, `size`, and a set of + /// `addresses`, the following are guaranteed: + /// ... + /// - `size <= isize::MAX` + /// + /// As a consequence of these guarantees, given any address `a` within + /// the set of addresses of an allocated object: + /// ... + /// - It is guaranteed that, given `o = a - base` (i.e., the offset of + /// `a` within the allocated object), `base + o` will not wrap + /// around the address space (in other words, will not overflow + /// `usize`) + ptr: NonNull<T>, + // SAFETY: `&'a UnsafeCell<T>` is covariant in `'a` and invariant in `T` + // [1]. We use this construction rather than the equivalent `&mut T`, + // because our MSRV of 1.65 prohibits `&mut` types in const contexts. + // + // [1] https://doc.rust-lang.org/1.81.0/reference/subtyping.html#variance + _marker: PhantomData<&'a core::cell::UnsafeCell<T>>, + } + + impl<'a, T: 'a + ?Sized> Copy for PtrInner<'a, T> {} + impl<'a, T: 'a + ?Sized> Clone for PtrInner<'a, T> { + #[inline(always)] + fn clone(&self) -> PtrInner<'a, T> { + // SAFETY: None of the invariants on `ptr` are affected by having + // multiple copies of a `PtrInner`. + *self + } + } + + impl<'a, T: 'a + ?Sized> PtrInner<'a, T> { + /// Constructs a `Ptr` from a [`NonNull`]. + /// + /// # Safety + /// + /// The caller promises that: + /// + /// 0. If `ptr`'s referent is not zero sized, then `ptr` has valid + /// provenance for its referent, which is entirely contained in some + /// Rust allocation, `A`. + /// 1. If `ptr`'s referent is not zero sized, `A` is guaranteed to live + /// for at least `'a`. + #[inline(always)] + #[must_use] + pub const unsafe fn new(ptr: NonNull<T>) -> PtrInner<'a, T> { + // SAFETY: The caller has promised to satisfy all safety invariants + // of `PtrInner`. + Self { ptr, _marker: PhantomData } + } + + /// Converts this `PtrInner<T>` to a [`NonNull<T>`]. + /// + /// Note that this method does not consume `self`. The caller should + /// watch out for `unsafe` code which uses the returned `NonNull` in a + /// way that violates the safety invariants of `self`. + #[inline(always)] + #[must_use] + pub const fn as_non_null(&self) -> NonNull<T> { + self.ptr + } + + /// Converts this `PtrInner<T>` to a [`*mut T`]. + /// + /// Note that this method does not consume `self`. The caller should + /// watch out for `unsafe` code which uses the returned `*mut T` in a + /// way that violates the safety invariants of `self`. + #[inline(always)] + #[must_use] + pub const fn as_ptr(&self) -> *mut T { + self.ptr.as_ptr() + } + } +} + +impl<'a, T: ?Sized> PtrInner<'a, T> { + /// Constructs a `PtrInner` from a reference. + #[inline] + pub fn from_ref(ptr: &'a T) -> Self { + let ptr = NonNull::from(ptr); + // SAFETY: + // 0. If `ptr`'s referent is not zero sized, then `ptr`, by invariant on + // `&'a T` [1], has valid provenance for its referent, which is + // entirely contained in some Rust allocation, `A`. + // 1. If `ptr`'s referent is not zero sized, then `A`, by invariant on + // `&'a T`, is guaranteed to live for at least `'a`. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/primitive.reference.html#safety: + // + // For all types, `T: ?Sized`, and for all `t: &T` or `t: &mut T`, + // when such values cross an API boundary, the following invariants + // must generally be upheld: + // ... + // - if `size_of_val(t) > 0`, then `t` is dereferenceable for + // `size_of_val(t)` many bytes + // + // If `t` points at address `a`, being “dereferenceable” for N bytes + // means that the memory range `[a, a + N)` is all contained within a + // single allocated object. + unsafe { Self::new(ptr) } + } + + /// Constructs a `PtrInner` from a mutable reference. + #[inline] + pub fn from_mut(ptr: &'a mut T) -> Self { + let ptr = NonNull::from(ptr); + // SAFETY: + // 0. If `ptr`'s referent is not zero sized, then `ptr`, by invariant on + // `&'a mut T` [1], has valid provenance for its referent, which is + // entirely contained in some Rust allocation, `A`. + // 1. If `ptr`'s referent is not zero sized, then `A`, by invariant on + // `&'a mut T`, is guaranteed to live for at least `'a`. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/primitive.reference.html#safety: + // + // For all types, `T: ?Sized`, and for all `t: &T` or `t: &mut T`, + // when such values cross an API boundary, the following invariants + // must generally be upheld: + // ... + // - if `size_of_val(t) > 0`, then `t` is dereferenceable for + // `size_of_val(t)` many bytes + // + // If `t` points at address `a`, being “dereferenceable” for N bytes + // means that the memory range `[a, a + N)` is all contained within a + // single allocated object. + unsafe { Self::new(ptr) } + } + + /// # Safety + /// + /// The caller may assume that the resulting `PtrInner` addresses the subset + /// of the bytes of `self`'s referent addressed by `C::project(self)`. + #[must_use] + #[inline(always)] + pub fn project<U: ?Sized, C: cast::Project<T, U>>(self) -> PtrInner<'a, U> { + let projected_raw = C::project(self); + + // SAFETY: `self`'s referent lives at a `NonNull` address, and is either + // zero-sized or lives in an allocation. In either case, it does not + // wrap around the address space [1], and so none of the addresses + // contained in it or one-past-the-end of it are null. + // + // By invariant on `C: Project`, `C::project` is a provenance-preserving + // projection which preserves or shrinks the set of referent bytes, so + // `projected_raw` references a subset of `self`'s referent, and so it + // cannot be null. + // + // [1] https://doc.rust-lang.org/1.92.0/std/ptr/index.html#allocation + let projected_non_null = unsafe { NonNull::new_unchecked(projected_raw) }; + + // SAFETY: As described in the preceding safety comment, `projected_raw`, + // and thus `projected_non_null`, addresses a subset of `self`'s + // referent. Thus, `projected_non_null` either: + // - Addresses zero bytes or, + // - Addresses a subset of the referent of `self`. In this case, `self` + // has provenance for its referent, which lives in an allocation. + // Since `projected_non_null` was constructed using a sequence of + // provenance-preserving operations, it also has provenance for its + // referent and that referent lives in an allocation. By invariant on + // `self`, that allocation lives for `'a`. + unsafe { PtrInner::new(projected_non_null) } + } +} + +#[allow(clippy::needless_lifetimes)] +impl<'a, T> PtrInner<'a, T> +where + T: ?Sized + KnownLayout, +{ + /// Extracts the metadata of this `ptr`. + #[inline] + #[must_use] + pub fn meta(self) -> MetadataOf<T> { + let meta = T::pointer_to_metadata(self.as_ptr()); + // SAFETY: By invariant on `PtrInner`, `self.as_non_null()` addresses no + // more than `isize::MAX` bytes. + unsafe { MetadataOf::new_unchecked(meta) } + } + + /// Produces a `PtrInner` with the same address and provenance as `self` but + /// the given `meta`. + /// + /// # Safety + /// + /// The caller promises that if `self`'s referent is not zero sized, then + /// a pointer constructed from its address with the given `meta` metadata + /// will address a subset of the allocation pointed to by `self`. + #[inline] + #[must_use] + pub unsafe fn with_meta(self, meta: T::PointerMetadata) -> Self + where + T: KnownLayout, + { + let raw = T::raw_from_ptr_len(self.as_non_null().cast(), meta); + + // SAFETY: + // + // Lemma 0: `raw` either addresses zero bytes, or addresses a subset of + // the allocation pointed to by `self` and has the same + // provenance as `self`. Proof: `raw` is constructed using + // provenance-preserving operations, and the caller has + // promised that, if `self`'s referent is not zero-sized, the + // resulting pointer addresses a subset of the allocation + // pointed to by `self`. + // + // 0. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `ptr` is derived from some valid Rust allocation, + // `A`. + // 1. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `ptr` has valid provenance for `A`. + // 2. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `ptr` addresses a byte range which is entirely + // contained in `A`. + // 3. Per Lemma 0 and by invariant on `self`, `ptr` addresses a byte + // range whose length fits in an `isize`. + // 4. Per Lemma 0 and by invariant on `self`, `ptr` addresses a byte + // range which does not wrap around the address space. + // 5. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `A` is guaranteed to live for at least `'a`. + unsafe { PtrInner::new(raw) } + } +} + +#[allow(clippy::needless_lifetimes)] +impl<'a, T> PtrInner<'a, T> +where + T: ?Sized + KnownLayout<PointerMetadata = usize>, +{ + /// Splits `T` in two. + /// + /// # Safety + /// + /// The caller promises that: + /// - `l_len.get() <= self.meta()`. + /// + /// ## (Non-)Overlap + /// + /// Given `let (left, right) = ptr.split_at(l_len)`, it is guaranteed that + /// `left` and `right` are contiguous and non-overlapping if + /// `l_len.padding_needed_for() == 0`. This is true for all `[T]`. + /// + /// If `l_len.padding_needed_for() != 0`, then the left pointer will overlap + /// the right pointer to satisfy `T`'s padding requirements. + #[inline] + #[must_use] + pub unsafe fn split_at_unchecked( + self, + l_len: crate::util::MetadataOf<T>, + ) -> (Self, PtrInner<'a, [T::Elem]>) + where + T: SplitAt, + { + let l_len = l_len.get(); + + // SAFETY: The caller promises that `l_len.get() <= self.meta()`. + // Trivially, `0 <= l_len`. + let left = unsafe { self.with_meta(l_len) }; + + let right = self.trailing_slice(); + // SAFETY: The caller promises that `l_len <= self.meta() = slf.meta()`. + // Trivially, `slf.meta() <= slf.meta()`. + let right = unsafe { right.slice_unchecked(l_len..self.meta().get()) }; + + // SAFETY: If `l_len.padding_needed_for() == 0`, then `left` and `right` + // are non-overlapping. Proof: `left` is constructed `slf` with `l_len` + // as its (exclusive) upper bound. If `l_len.padding_needed_for() == 0`, + // then `left` requires no trailing padding following its final element. + // Since `right` is constructed from `slf`'s trailing slice with `l_len` + // as its (inclusive) lower bound, no byte is referred to by both + // pointers. + // + // Conversely, `l_len.padding_needed_for() == N`, where `N + // > 0`, `left` requires `N` bytes of trailing padding following its + // final element. Since `right` is constructed from the trailing slice + // of `slf` with `l_len` as its (inclusive) lower bound, the first `N` + // bytes of `right` are aliased by `left`. + (left, right) + } + + /// Produces the trailing slice of `self`. + #[inline] + #[must_use] + pub fn trailing_slice(self) -> PtrInner<'a, [T::Elem]> + where + T: SplitAt, + { + let offset = crate::trailing_slice_layout::<T>().offset; + + let bytes = self.as_non_null().cast::<u8>().as_ptr(); + + // SAFETY: + // - By invariant on `T: KnownLayout`, `T::LAYOUT` describes `T`'s + // layout. `offset` is the offset of the trailing slice within `T`, + // which is by definition in-bounds or one byte past the end of any + // `T`, regardless of metadata. By invariant on `PtrInner`, `self` + // (and thus `bytes`) points to a byte range of size `<= isize::MAX`, + // and so `offset <= isize::MAX`. Since `size_of::<u8>() == 1`, + // `offset * size_of::<u8>() <= isize::MAX`. + // - If `offset > 0`, then by invariant on `PtrInner`, `self` (and thus + // `bytes`) points to a byte range entirely contained within the same + // allocated object as `self`. As explained above, this offset results + // in a pointer to or one byte past the end of this allocated object. + let bytes = unsafe { bytes.add(offset) }; + + // SAFETY: By the preceding safety argument, `bytes` is within or one + // byte past the end of the same allocated object as `self`, which + // ensures that it is non-null. + let bytes = unsafe { NonNull::new_unchecked(bytes) }; + + let ptr = KnownLayout::raw_from_ptr_len(bytes, self.meta().get()); + + // SAFETY: + // 0. If `ptr`'s referent is not zero sized, then `ptr` is derived from + // some valid Rust allocation, `A`, because `ptr` is derived from + // the same allocated object as `self`. + // 1. If `ptr`'s referent is not zero sized, then `ptr` has valid + // provenance for `A` because `raw` is derived from the same + // allocated object as `self` via provenance-preserving operations. + // 2. If `ptr`'s referent is not zero sized, then `ptr` addresses a byte + // range which is entirely contained in `A`, by previous safety proof + // on `bytes`. + // 3. `ptr` addresses a byte range whose length fits in an `isize`, by + // consequence of #2. + // 4. `ptr` addresses a byte range which does not wrap around the + // address space, by consequence of #2. + // 5. If `ptr`'s referent is not zero sized, then `A` is guaranteed to + // live for at least `'a`, because `ptr` is derived from `self`. + unsafe { PtrInner::new(ptr) } + } +} + +#[allow(clippy::needless_lifetimes)] +impl<'a, T> PtrInner<'a, [T]> { + /// Creates a pointer which addresses the given `range` of self. + /// + /// # Safety + /// + /// `range` is a valid range (`start <= end`) and `end <= self.meta()`. + #[inline] + #[must_use] + pub unsafe fn slice_unchecked(self, range: Range<usize>) -> Self { + let base = self.as_non_null().cast::<T>().as_ptr(); + + // SAFETY: The caller promises that `start <= end <= self.meta()`. By + // invariant, if `self`'s referent is not zero-sized, then `self` refers + // to a byte range which is contained within a single allocation, which + // is no more than `isize::MAX` bytes long, and which does not wrap + // around the address space. Thus, this pointer arithmetic remains + // in-bounds of the same allocation, and does not wrap around the + // address space. The offset (in bytes) does not overflow `isize`. + // + // If `self`'s referent is zero-sized, then these conditions are + // trivially satisfied. + let base = unsafe { base.add(range.start) }; + + // SAFETY: The caller promises that `start <= end`, and so this will not + // underflow. + #[allow(unstable_name_collisions)] + let len = unsafe { range.end.unchecked_sub(range.start) }; + + let ptr = core::ptr::slice_from_raw_parts_mut(base, len); + + // SAFETY: By invariant, `self`'s referent is either a ZST or lives + // entirely in an allocation. `ptr` points inside of or one byte past + // the end of that referent. Thus, in either case, `ptr` is non-null. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; + + // SAFETY: + // + // Lemma 0: `ptr` addresses a subset of the bytes addressed by `self`, + // and has the same provenance. Proof: The caller guarantees + // that `start <= end <= self.meta()`. Thus, `base` is + // in-bounds of `self`, and `base + (end - start)` is also + // in-bounds of self. Finally, `ptr` is constructed using + // provenance-preserving operations. + // + // 0. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `ptr` has valid provenance for its referent, + // which is entirely contained in some Rust allocation, `A`. + // 1. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `A` is guaranteed to live for at least `'a`. + unsafe { PtrInner::new(ptr) } + } + + /// Iteratively projects the elements `PtrInner<T>` from `PtrInner<[T]>`. + #[inline] + pub fn iter(&self) -> impl Iterator<Item = PtrInner<'a, T>> { + // FIXME(#429): Once `NonNull::cast` documents that it preserves + // provenance, cite those docs. + let base = self.as_non_null().cast::<T>().as_ptr(); + (0..self.meta().get()).map(move |i| { + // FIXME(https://github.com/rust-lang/rust/issues/74265): Use + // `NonNull::get_unchecked_mut`. + + // SAFETY: If the following conditions are not satisfied + // `pointer::cast` may induce Undefined Behavior [1]: + // + // > - The computed offset, `count * size_of::<T>()` bytes, must not + // > overflow `isize``. + // > - If the computed offset is non-zero, then `self` must be + // > derived from a pointer to some allocated object, and the + // > entire memory range between `self` and the result must be in + // > bounds of that allocated object. In particular, this range + // > must not “wrap around” the edge of the address space. + // + // [1] https://doc.rust-lang.org/std/primitive.pointer.html#method.add + // + // We satisfy both of these conditions here: + // - By invariant on `Ptr`, `self` addresses a byte range whose + // length fits in an `isize`. Since `elem` is contained in `self`, + // the computed offset of `elem` must fit within `isize.` + // - If the computed offset is non-zero, then this means that the + // referent is not zero-sized. In this case, `base` points to an + // allocated object (by invariant on `self`). Thus: + // - By contract, `self.meta()` accurately reflects the number of + // elements in the slice. `i` is in bounds of `c.meta()` by + // construction, and so the result of this addition cannot + // overflow past the end of the allocation referred to by `c`. + // - By invariant on `Ptr`, `self` addresses a byte range which + // does not wrap around the address space. Since `elem` is + // contained in `self`, the computed offset of `elem` must wrap + // around the address space. + // + // FIXME(#429): Once `pointer::add` documents that it preserves + // provenance, cite those docs. + let elem = unsafe { base.add(i) }; + + // SAFETY: `elem` must not be null. `base` is constructed from a + // `NonNull` pointer, and the addition that produces `elem` must not + // overflow or wrap around, so `elem >= base > 0`. + // + // FIXME(#429): Once `NonNull::new_unchecked` documents that it + // preserves provenance, cite those docs. + let elem = unsafe { NonNull::new_unchecked(elem) }; + + // SAFETY: The safety invariants of `Ptr::new` (see definition) are + // satisfied: + // 0. If `elem`'s referent is not zero sized, then `elem` has valid + // provenance for its referent, because it derived from `self` + // using a series of provenance-preserving operations, and + // because `self` has valid provenance for its referent. By the + // same argument, `elem`'s referent is entirely contained within + // the same allocated object as `self`'s referent. + // 1. If `elem`'s referent is not zero sized, then the allocation of + // `elem` is guaranteed to live for at least `'a`, because `elem` + // is entirely contained in `self`, which lives for at least `'a` + // by invariant on `Ptr`. + unsafe { PtrInner::new(elem) } + }) + } +} + +impl<'a, T, const N: usize> PtrInner<'a, [T; N]> { + /// Casts this pointer-to-array into a slice. + /// + /// # Safety + /// + /// Callers may assume that the returned `PtrInner` references the same + /// address and length as `self`. + #[allow(clippy::wrong_self_convention)] + #[inline] + #[must_use] + pub fn as_slice(self) -> PtrInner<'a, [T]> { + let start = self.as_non_null().cast::<T>().as_ptr(); + let slice = core::ptr::slice_from_raw_parts_mut(start, N); + // SAFETY: `slice` is not null, because it is derived from `start` + // which is non-null. + let slice = unsafe { NonNull::new_unchecked(slice) }; + // SAFETY: Lemma: In the following safety arguments, note that `slice` + // is derived from `self` in two steps: first, by casting `self: [T; N]` + // to `start: T`, then by constructing a pointer to a slice starting at + // `start` of length `N`. As a result, `slice` references exactly the + // same allocation as `self`, if any. + // + // 0. By the above lemma, if `slice`'s referent is not zero sized, then + // `slice` has the same referent as `self`. By invariant on `self`, + // this referent is entirely contained within some allocation, `A`. + // Because `slice` was constructed using provenance-preserving + // operations, it has provenance for its entire referent. + // 1. By the above lemma, if `slice`'s referent is not zero sized, then + // `A` is guaranteed to live for at least `'a`, because it is derived + // from the same allocation as `self`, which, by invariant on + // `PtrInner`, lives for at least `'a`. + unsafe { PtrInner::new(slice) } + } +} + +impl<'a> PtrInner<'a, [u8]> { + /// Attempts to cast `self` to a `U` using the given cast type. + /// + /// If `U` is a slice DST and pointer metadata (`meta`) is provided, then + /// the cast will only succeed if it would produce an object with the given + /// metadata. + /// + /// Returns `None` if the resulting `U` would be invalidly-aligned, if no + /// `U` can fit in `self`, or if the provided pointer metadata describes an + /// invalid instance of `U`. On success, returns a pointer to the + /// largest-possible `U` which fits in `self`. + /// + /// # Safety + /// + /// The caller may assume that this implementation is correct, and may rely + /// on that assumption for the soundness of their code. In particular, the + /// caller may assume that, if `try_cast_into` returns `Some((ptr, + /// remainder))`, then `ptr` and `remainder` refer to non-overlapping byte + /// ranges within `self`, and that `ptr` and `remainder` entirely cover + /// `self`. Finally: + /// - If this is a prefix cast, `ptr` has the same address as `self`. + /// - If this is a suffix cast, `remainder` has the same address as `self`. + #[inline] + pub fn try_cast_into<U>( + self, + cast_type: CastType, + meta: Option<U::PointerMetadata>, + ) -> Result<(PtrInner<'a, U>, PtrInner<'a, [u8]>), CastError<Self, U>> + where + U: 'a + ?Sized + KnownLayout, + { + // PANICS: By invariant, the byte range addressed by + // `self.as_non_null()` does not wrap around the address space. This + // implies that the sum of the address (represented as a `usize`) and + // length do not overflow `usize`, as required by + // `validate_cast_and_convert_metadata`. Thus, this call to + // `validate_cast_and_convert_metadata` will only panic if `U` is a DST + // whose trailing slice element is zero-sized. + let maybe_metadata = MetadataOf::<U>::validate_cast_and_convert_metadata( + AsAddress::addr(self.as_ptr()), + self.meta(), + cast_type, + meta, + ); + + let (elems, split_at) = match maybe_metadata { + Ok((elems, split_at)) => (elems, split_at), + Err(MetadataCastError::Alignment) => { + // SAFETY: Since `validate_cast_and_convert_metadata` returned + // an alignment error, `U` must have an alignment requirement + // greater than one. + let err = unsafe { AlignmentError::<_, U>::new_unchecked(self) }; + return Err(CastError::Alignment(err)); + } + Err(MetadataCastError::Size) => return Err(CastError::Size(SizeError::new(self))), + }; + + // SAFETY: `validate_cast_and_convert_metadata` promises to return + // `split_at <= self.meta()`. + // + // Lemma 0: `l_slice` and `r_slice` are non-overlapping. Proof: By + // contract on `PtrInner::split_at_unchecked`, the produced `PtrInner`s + // are always non-overlapping if `self` is a `[T]`; here it is a `[u8]`. + let (l_slice, r_slice) = unsafe { self.split_at_unchecked(split_at) }; + + let (target, remainder) = match cast_type { + CastType::Prefix => (l_slice, r_slice), + CastType::Suffix => (r_slice, l_slice), + }; + + let base = target.as_non_null().cast::<u8>(); + + let ptr = U::raw_from_ptr_len(base, elems.get()); + + // SAFETY: + // 0. By invariant, if `target`'s referent is not zero sized, then + // `target` has provenance valid for some Rust allocation, `A`. + // Because `ptr` is derived from `target` via provenance-preserving + // operations, `ptr` will also have provenance valid for its entire + // referent. + // 1. `validate_cast_and_convert_metadata` promises that the object + // described by `elems` and `split_at` lives at a byte range which is + // a subset of the input byte range. Thus, by invariant, if + // `target`'s referent is not zero sized, then `target` refers to an + // allocation which is guaranteed to live for at least `'a`, and thus + // so does `ptr`. + Ok((unsafe { PtrInner::new(ptr) }, remainder)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::*; + + #[test] + fn test_meta() { + let arr = [1; 16]; + let dst = <[u8]>::ref_from_bytes(&arr[..]).unwrap(); + let ptr = PtrInner::from_ref(dst); + assert_eq!(ptr.meta().get(), 16); + + // SAFETY: 8 is less than 16 + let ptr = unsafe { ptr.with_meta(8) }; + + assert_eq!(ptr.meta().get(), 8); + } + + #[test] + fn test_split_at() { + fn test_split_at<const OFFSET: usize, const BUFFER_SIZE: usize>() { + #[derive(FromBytes, KnownLayout, SplitAt, Immutable)] + #[repr(C)] + struct SliceDst<const OFFSET: usize> { + prefix: [u8; OFFSET], + trailing: [u8], + } + + let n: usize = BUFFER_SIZE - OFFSET; + let arr = [1; BUFFER_SIZE]; + let dst = SliceDst::<OFFSET>::ref_from_bytes(&arr[..]).unwrap(); + let ptr = PtrInner::from_ref(dst); + for i in 0..=n { + assert_eq!(ptr.meta().get(), n); + // SAFETY: `i` is in bounds by construction. + let i = unsafe { MetadataOf::new_unchecked(i) }; + // SAFETY: `i` is in bounds by construction. + let (l, r) = unsafe { ptr.split_at_unchecked(i) }; + // SAFETY: Points to a valid value by construction. + #[allow(clippy::undocumented_unsafe_blocks, clippy::as_conversions)] + // Clippy false positive + let l_sum: usize = l + .trailing_slice() + .iter() + .map( + #[inline(always)] + |ptr| unsafe { core::ptr::read_unaligned(ptr.as_ptr()) } as usize, + ) + .sum(); + // SAFETY: Points to a valid value by construction. + #[allow(clippy::undocumented_unsafe_blocks, clippy::as_conversions)] + // Clippy false positive + let r_sum: usize = r + .iter() + .map( + #[inline(always)] + |ptr| unsafe { core::ptr::read_unaligned(ptr.as_ptr()) } as usize, + ) + .sum(); + assert_eq!(l_sum, i.get()); + assert_eq!(r_sum, n - i.get()); + assert_eq!(l_sum + r_sum, n); + } + } + + test_split_at::<0, 16>(); + test_split_at::<1, 17>(); + test_split_at::<2, 18>(); + } + + #[test] + fn test_trailing_slice() { + fn test_trailing_slice<const OFFSET: usize, const BUFFER_SIZE: usize>() { + #[derive(FromBytes, KnownLayout, SplitAt, Immutable)] + #[repr(C)] + struct SliceDst<const OFFSET: usize> { + prefix: [u8; OFFSET], + trailing: [u8], + } + + let n: usize = BUFFER_SIZE - OFFSET; + let arr = [1; BUFFER_SIZE]; + let dst = SliceDst::<OFFSET>::ref_from_bytes(&arr[..]).unwrap(); + let ptr = PtrInner::from_ref(dst); + + assert_eq!(ptr.meta().get(), n); + let trailing = ptr.trailing_slice(); + assert_eq!(trailing.meta().get(), n); + + assert_eq!( + // SAFETY: We assume this to be sound for the sake of this test, + // which will fail, here, in miri, if the safety precondition of + // `offset_of` is not satisfied. + unsafe { + #[allow(clippy::as_conversions)] + let offset = (trailing.as_ptr() as *mut u8).offset_from(ptr.as_ptr() as *mut _); + offset + }, + isize::try_from(OFFSET).unwrap(), + ); + + // SAFETY: Points to a valid value by construction. + #[allow(clippy::undocumented_unsafe_blocks, clippy::as_conversions)] + // Clippy false positive + let trailing: usize = trailing + .iter() + .map(|ptr| unsafe { core::ptr::read_unaligned(ptr.as_ptr()) } as usize) + .sum(); + + assert_eq!(trailing, n); + } + + test_trailing_slice::<0, 16>(); + test_trailing_slice::<1, 17>(); + test_trailing_slice::<2, 18>(); + } + #[test] + fn test_ptr_inner_clone() { + let mut x = 0u8; + let p = PtrInner::from_mut(&mut x); + #[allow(clippy::clone_on_copy)] + let p2 = p.clone(); + assert_eq!(p.as_non_null(), p2.as_non_null()); + } +} diff --git a/rust/zerocopy/src/pointer/invariant.rs b/rust/zerocopy/src/pointer/invariant.rs new file mode 100644 index 000000000000..2af211dd9fdd --- /dev/null +++ b/rust/zerocopy/src/pointer/invariant.rs @@ -0,0 +1,296 @@ +// Copyright 2024 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +#![allow(missing_copy_implementations, missing_debug_implementations, missing_docs)] + +//! The parameterized invariants of a [`Ptr`][super::Ptr]. +//! +//! Invariants are encoded as ([`Aliasing`], [`Alignment`], [`Validity`]) +//! triples implementing the [`Invariants`] trait. + +/// The invariants of a [`Ptr`][super::Ptr]. +pub trait Invariants: Sealed { + type Aliasing: Aliasing; + type Alignment: Alignment; + type Validity: Validity; +} + +impl<A: Aliasing, AA: Alignment, V: Validity> Invariants for (A, AA, V) { + type Aliasing = A; + type Alignment = AA; + type Validity = V; +} + +/// The aliasing invariant of a [`Ptr`][super::Ptr]. +/// +/// All aliasing invariants must permit reading from the bytes of a pointer's +/// referent which are not covered by [`UnsafeCell`]s. +/// +/// [`UnsafeCell`]: core::cell::UnsafeCell +pub trait Aliasing: Sealed { + /// Is `Self` [`Exclusive`]? + #[doc(hidden)] + const IS_EXCLUSIVE: bool; +} + +/// The alignment invariant of a [`Ptr`][super::Ptr]. +pub trait Alignment: Sealed { + #[doc(hidden)] + #[must_use] + fn read<T, I, R>(ptr: crate::Ptr<'_, T, I>) -> T + where + T: Copy + Read<I::Aliasing, R>, + I: Invariants<Alignment = Self, Validity = Valid>, + I::Aliasing: Reference; +} + +/// The validity invariant of a [`Ptr`][super::Ptr]. +/// +/// # Safety +/// +/// In this section, we will use `Ptr<T, V>` as a shorthand for `Ptr<T, I: +/// Invariants<Validity = V>>` for brevity. +/// +/// Each `V: Validity` defines a set of bit values which may appear in the +/// referent of a `Ptr<T, V>`, denoted `S(T, V)`. Each `V: Validity`, in its +/// documentation, provides a definition of `S(T, V)` which must be valid for +/// all `T: ?Sized`. Any `V: Validity` must guarantee that this set is only a +/// function of the *bit validity* of the referent type, `T`, and not of any +/// other property of `T`. As a consequence, given `V: Validity`, `T`, and `U` +/// where `T` and `U` have the same bit validity, `S(V, T) = S(V, U)`. +/// +/// It is guaranteed that the referent of any `ptr: Ptr<T, V>` is a member of +/// `S(T, V)`. Unsafe code must ensure that this guarantee will be upheld for +/// any existing `Ptr`s or any `Ptr`s that that code creates. +/// +/// An important implication of this guarantee is that it restricts what +/// transmutes are sound, where "transmute" is used in this context to refer to +/// changing the referent type or validity invariant of a `Ptr`, as either +/// change may change the set of bit values permitted to appear in the referent. +/// In particular, the following are necessary (but not sufficient) conditions +/// in order for a transmute from `src: Ptr<T, V>` to `dst: Ptr<U, W>` to be +/// sound: +/// - If `S(T, V) = S(U, W)`, then no restrictions apply; otherwise, +/// - If `dst` permits mutation of its referent (e.g. via `Exclusive` aliasing +/// or interior mutation under `Shared` aliasing), then it must hold that +/// `S(T, V) ⊇ S(U, W)` - in other words, the transmute must not expand the +/// set of allowed referent bit patterns. A violation of this requirement +/// would permit using `dst` to write `x` where `x ∈ S(U, W)` but `x ∉ S(T, +/// V)`, which would violate the guarantee that `src`'s referent may only +/// contain values in `S(T, V)`. +/// - If the referent may be mutated without going through `dst` while `dst` is +/// live (e.g. via interior mutation on a `Shared`-aliased `Ptr` or `&` +/// reference), then it must hold that `S(T, V) ⊆ S(U, W)` - in other words, +/// the transmute must not shrink the set of allowed referent bit patterns. A +/// violation of this requirement would permit using `src` or another +/// mechanism (e.g. a `&` reference used to derive `src`) to write `x` where +/// `x ∈ S(T, V)` but `x ∉ S(U, W)`, which would violate the guarantee that +/// `dst`'s referent may only contain values in `S(U, W)`. +pub unsafe trait Validity: Sealed { + const KIND: ValidityKind; +} + +pub enum ValidityKind { + Uninit, + AsInitialized, + Initialized, + Valid, +} + +/// An [`Aliasing`] invariant which is either [`Shared`] or [`Exclusive`]. +/// +/// # Safety +/// +/// Given `A: Reference`, callers may assume that either `A = Shared` or `A = +/// Exclusive`. +pub trait Reference: Aliasing + Sealed {} + +/// The `Ptr<'a, T>` adheres to the aliasing rules of a `&'a T`. +/// +/// The referent of a shared-aliased `Ptr` may be concurrently referenced by any +/// number of shared-aliased `Ptr` or `&T` references, or by any number of +/// `Ptr<U>` or `&U` references as permitted by `T`'s library safety invariants, +/// and may not be concurrently referenced by any exclusively-aliased `Ptr`s or +/// `&mut` references. The referent must not be mutated, except via +/// [`UnsafeCell`]s, and only when permitted by `T`'s library safety invariants. +/// +/// [`UnsafeCell`]: core::cell::UnsafeCell +pub enum Shared {} +impl Aliasing for Shared { + const IS_EXCLUSIVE: bool = false; +} +impl Reference for Shared {} + +/// The `Ptr<'a, T>` adheres to the aliasing rules of a `&'a mut T`. +/// +/// The referent of an exclusively-aliased `Ptr` may not be concurrently +/// referenced by any other `Ptr`s or references, and may not be accessed (read +/// or written) other than via this `Ptr`. +pub enum Exclusive {} +impl Aliasing for Exclusive { + const IS_EXCLUSIVE: bool = true; +} +impl Reference for Exclusive {} + +/// It is unknown whether the pointer is aligned. +pub enum Unaligned {} + +impl Alignment for Unaligned { + #[inline(always)] + fn read<T, I, R>(ptr: crate::Ptr<'_, T, I>) -> T + where + T: Copy + Read<I::Aliasing, R>, + I: Invariants<Alignment = Self, Validity = Valid>, + I::Aliasing: Reference, + { + (*ptr.into_unalign().as_ref()).into_inner() + } +} + +/// The referent is aligned: for `Ptr<T>`, the referent's address is a multiple +/// of the `T`'s alignment. +pub enum Aligned {} +impl Alignment for Aligned { + #[inline(always)] + fn read<T, I, R>(ptr: crate::Ptr<'_, T, I>) -> T + where + T: Copy + Read<I::Aliasing, R>, + I: Invariants<Alignment = Self, Validity = Valid>, + I::Aliasing: Reference, + { + *ptr.as_ref() + } +} + +/// Any bit pattern is allowed in the `Ptr`'s referent, including uninitialized +/// bytes. +pub enum Uninit {} +// SAFETY: `Uninit`'s validity is well-defined for all `T: ?Sized`, and is not a +// function of any property of `T` other than its bit validity (in fact, it's +// not even a property of `T`'s bit validity, but this is more than we are +// required to uphold). +unsafe impl Validity for Uninit { + const KIND: ValidityKind = ValidityKind::Uninit; +} + +/// The byte ranges initialized in `T` are also initialized in the referent of a +/// `Ptr<T>`. +/// +/// Formally: uninitialized bytes may only be present in `Ptr<T>`'s referent +/// where they are guaranteed to be present in `T`. This is a dynamic property: +/// if, at a particular byte offset, a valid enum discriminant is set, the +/// subsequent bytes may only have uninitialized bytes as specified by the +/// corresponding enum. +/// +/// Formally, given `len = size_of_val_raw(ptr)`, at every byte offset, `b`, in +/// the range `[0, len)`: +/// - If, in any instance `t: T` of length `len`, the byte at offset `b` in `t` +/// is initialized, then the byte at offset `b` within `*ptr` must be +/// initialized. +/// - Let `c` be the contents of the byte range `[0, b)` in `*ptr`. Let `S` be +/// the subset of valid instances of `T` of length `len` which contain `c` in +/// the offset range `[0, b)`. If, in any instance of `t: T` in `S`, the byte +/// at offset `b` in `t` is initialized, then the byte at offset `b` in `*ptr` +/// must be initialized. +/// +/// Pragmatically, this means that if `*ptr` is guaranteed to contain an enum +/// type at a particular offset, and the enum discriminant stored in `*ptr` +/// corresponds to a valid variant of that enum type, then it is guaranteed +/// that the appropriate bytes of `*ptr` are initialized as defined by that +/// variant's bit validity (although note that the variant may contain another +/// enum type, in which case the same rules apply depending on the state of +/// its discriminant, and so on recursively). +pub enum AsInitialized {} +// SAFETY: `AsInitialized`'s validity is well-defined for all `T: ?Sized`, and +// is not a function of any property of `T` other than its bit validity. +unsafe impl Validity for AsInitialized { + const KIND: ValidityKind = ValidityKind::AsInitialized; +} + +/// The byte ranges in the referent are fully initialized. In other words, if +/// the referent is `N` bytes long, then it contains a bit-valid `[u8; N]`. +pub enum Initialized {} +// SAFETY: `Initialized`'s validity is well-defined for all `T: ?Sized`, and is +// not a function of any property of `T` other than its bit validity (in fact, +// it's not even a property of `T`'s bit validity, but this is more than we are +// required to uphold). +unsafe impl Validity for Initialized { + const KIND: ValidityKind = ValidityKind::Initialized; +} + +/// The referent of a `Ptr<T>` is valid for `T`, upholding bit validity and any +/// library safety invariants. +pub enum Valid {} +// SAFETY: `Valid`'s validity is well-defined for all `T: ?Sized`, and is not a +// function of any property of `T` other than its bit validity. +unsafe impl Validity for Valid { + const KIND: ValidityKind = ValidityKind::Valid; +} + +/// # Safety +/// +/// `DT: CastableFrom<ST, SV, DV>` is sound if `SV = DV = Uninit` or `SV = DV = +/// Initialized`. +pub unsafe trait CastableFrom<ST: ?Sized, SV, DV> {} + +// SAFETY: `SV = DV = Uninit`. +unsafe impl<ST: ?Sized, DT: ?Sized> CastableFrom<ST, Uninit, Uninit> for DT {} +// SAFETY: `SV = DV = Initialized`. +unsafe impl<ST: ?Sized, DT: ?Sized> CastableFrom<ST, Initialized, Initialized> for DT {} + +/// [`Ptr`](crate::Ptr) referents that permit unsynchronized read operations. +/// +/// `T: Read<A, R>` implies that a pointer to `T` with aliasing `A` permits +/// unsynchronized read operations. This can be because `A` is [`Exclusive`] or +/// because `T` does not permit interior mutation. +/// +/// # Safety +/// +/// `T: Read<A, R>` if either of the following conditions holds: +/// - `A` is [`Exclusive`] +/// - `T` implements [`Immutable`](crate::Immutable) +/// +/// As a consequence, if `T: Read<A, R>`, then any `Ptr<T, (A, ...)>` is +/// permitted to perform unsynchronized reads from its referent. +pub trait Read<A: Aliasing, R> {} + +impl<A: Aliasing, T: ?Sized + crate::Immutable> Read<A, BecauseImmutable> for T {} +impl<T: ?Sized> Read<Exclusive, BecauseExclusive> for T {} + +/// Unsynchronized reads are permitted because only one live [`Ptr`](crate::Ptr) +/// or reference may exist to the referent bytes at a time. +#[derive(Copy, Clone, Debug)] +pub enum BecauseExclusive {} + +/// Unsynchronized reads are permitted because no live [`Ptr`](crate::Ptr)s or +/// references permit interior mutation. +#[derive(Copy, Clone, Debug)] +pub enum BecauseImmutable {} + +use sealed::Sealed; +mod sealed { + use super::*; + + pub trait Sealed {} + + impl Sealed for Shared {} + impl Sealed for Exclusive {} + + impl Sealed for Unaligned {} + impl Sealed for Aligned {} + + impl Sealed for Uninit {} + impl Sealed for AsInitialized {} + impl Sealed for Initialized {} + impl Sealed for Valid {} + + impl<A: Sealed, AA: Sealed, V: Sealed> Sealed for (A, AA, V) {} + + impl Sealed for BecauseImmutable {} + impl Sealed for BecauseExclusive {} +} diff --git a/rust/zerocopy/src/pointer/mod.rs b/rust/zerocopy/src/pointer/mod.rs new file mode 100644 index 000000000000..4c3658679d46 --- /dev/null +++ b/rust/zerocopy/src/pointer/mod.rs @@ -0,0 +1,408 @@ +// Copyright 2023 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Abstractions over raw pointers. + +#![allow(missing_docs)] + +mod inner; +pub mod invariant; +mod ptr; +pub mod transmute; + +pub use inner::PtrInner; +pub use invariant::{BecauseExclusive, BecauseImmutable, Read}; +pub use ptr::{Ptr, TryWithError}; +pub use transmute::*; + +use crate::wrappers::ReadOnly; + +/// A shorthand for a maybe-valid, maybe-aligned reference. Used as the argument +/// to [`TryFromBytes::is_bit_valid`]. +/// +/// [`TryFromBytes::is_bit_valid`]: crate::TryFromBytes::is_bit_valid +pub type Maybe<'a, T, Alignment = invariant::Unaligned> = + Ptr<'a, ReadOnly<T>, (invariant::Shared, Alignment, invariant::Initialized)>; + +/// Checks if the referent is zeroed. +pub(crate) fn is_zeroed<T, I>(ptr: Ptr<'_, T, I>) -> bool +where + T: crate::Immutable + crate::KnownLayout, + I: invariant::Invariants<Validity = invariant::Initialized>, + I::Aliasing: invariant::Reference, +{ + ptr.as_bytes().as_ref().iter().all( + #[inline(always)] + |&byte| byte == 0, + ) +} + +pub mod cast { + use core::{marker::PhantomData, mem}; + + use crate::{ + layout::{SizeInfo, TrailingSliceLayout}, + HasField, KnownLayout, PtrInner, + }; + + /// A pointer cast or projection. + /// + /// # Safety + /// + /// The implementation of `project` must satisfy its safety post-condition. + pub unsafe trait Project<Src: ?Sized, Dst: ?Sized> { + /// Projects a pointer from `Src` to `Dst`. + /// + /// Users should generally not call `project` directly, and instead + /// should use high-level APIs like [`PtrInner::project`] or + /// [`Ptr::project`]. + /// + /// [`Ptr::project`]: crate::pointer::Ptr::project + /// + /// # Safety + /// + /// The returned pointer refers to a non-strict subset of the bytes of + /// `src`'s referent, and has the same provenance as `src`. + fn project(src: PtrInner<'_, Src>) -> *mut Dst; + } + + /// A [`Project`] which preserves the address of the referent – a pointer + /// cast. + /// + /// # Safety + /// + /// A `Cast` projection must preserve the address of the referent. It may + /// shrink the set of referent bytes, and it may change the referent's type. + pub unsafe trait Cast<Src: ?Sized, Dst: ?Sized>: Project<Src, Dst> {} + + /// A [`Cast`] which does not shrink the set of referent bytes. + /// + /// # Safety + /// + /// A `CastExact` projection must preserve the set of referent bytes. + pub unsafe trait CastExact<Src: ?Sized, Dst: ?Sized>: Cast<Src, Dst> {} + + /// A no-op pointer cast. + #[derive(Default, Copy, Clone)] + #[allow(missing_debug_implementations)] + pub struct IdCast; + + // SAFETY: `project` returns its argument unchanged, and so it is a + // provenance-preserving projection which preserves the set of referent + // bytes. + unsafe impl<T: ?Sized> Project<T, T> for IdCast { + #[inline(always)] + fn project(src: PtrInner<'_, T>) -> *mut T { + src.as_ptr() + } + } + + // SAFETY: The `Project::project` impl preserves referent address. + unsafe impl<T: ?Sized> Cast<T, T> for IdCast {} + + // SAFETY: The `Project::project` impl preserves referent size. + unsafe impl<T: ?Sized> CastExact<T, T> for IdCast {} + + /// A pointer cast which preserves or shrinks the set of referent bytes of + /// a statically-sized referent. + /// + /// # Safety + /// + /// The implementation of [`Project`] uses a compile-time assertion to + /// guarantee that `Dst` is no larger than `Src`. Thus, `CastSized` has a + /// sound implementation of [`Project`] for all `Src` and `Dst` – the caller + /// may pass any `Src` and `Dst` without being responsible for soundness. + #[allow(missing_debug_implementations, missing_copy_implementations)] + pub enum CastSized {} + + // SAFETY: By the `static_assert!`, `Dst` is no larger than `Src`, + // and so all casts preserve or shrink the set of referent bytes. All + // operations preserve provenance. + unsafe impl<Src, Dst> Project<Src, Dst> for CastSized { + #[inline(always)] + fn project(src: PtrInner<'_, Src>) -> *mut Dst { + static_assert!(Src, Dst => mem::size_of::<Src>() >= mem::size_of::<Dst>()); + src.as_ptr().cast::<Dst>() + } + } + + // SAFETY: The `Project::project` impl preserves referent address. + unsafe impl<Src, Dst> Cast<Src, Dst> for CastSized {} + + /// A pointer cast which preserves the set of referent bytes of a + /// statically-sized referent. + /// + /// # Safety + /// + /// The implementation of [`Project`] uses a compile-time assertion to + /// guarantee that `Dst` has the same size as `Src`. Thus, `CastSizedExact` + /// has a sound implementation of [`Project`] for all `Src` and `Dst` – the + /// caller may pass any `Src` and `Dst` without being responsible for + /// soundness. + #[allow(missing_debug_implementations, missing_copy_implementations)] + pub enum CastSizedExact {} + + // SAFETY: By the `static_assert!`, `Dst` has the same size as `Src`, + // and so all casts preserve the set of referent bytes. All operations + // preserve provenance. + unsafe impl<Src, Dst> Project<Src, Dst> for CastSizedExact { + #[inline(always)] + fn project(src: PtrInner<'_, Src>) -> *mut Dst { + static_assert!(Src, Dst => mem::size_of::<Src>() == mem::size_of::<Dst>()); + src.as_ptr().cast::<Dst>() + } + } + + // SAFETY: The `Project::project_raw` impl preserves referent address. + unsafe impl<Src, Dst> Cast<Src, Dst> for CastSizedExact {} + + // SAFETY: By the `static_assert!`, `Project::project_raw` impl preserves + // referent size. + unsafe impl<Src, Dst> CastExact<Src, Dst> for CastSizedExact {} + + /// A pointer cast which preserves or shrinks the set of referent bytes of + /// a dynamically-sized referent. + /// + /// # Safety + /// + /// The implementation of [`Project`] uses a compile-time assertion to + /// guarantee that the cast preserves the set of referent bytes. Thus, + /// `CastUnsized` has a sound implementation of [`Project`] for all `Src` + /// and `Dst` – the caller may pass any `Src` and `Dst` without being + /// responsible for soundness. + #[allow(missing_debug_implementations, missing_copy_implementations)] + pub enum CastUnsized {} + + // SAFETY: By the `static_assert!`, `Src` and `Dst` are either: + // - Both sized and equal in size + // - Both slice DSTs with the same trailing slice offset and element size + // and with align_of::<Src>() == align_of::<Dst>(). These ensure that any + // given pointer metadata encodes the same size for both `Src` and `Dst` + // (note that the alignment is required as it affects the amount of + // trailing padding). Thus, `project` preserves the set of referent bytes. + unsafe impl<Src, Dst> Project<Src, Dst> for CastUnsized + where + Src: ?Sized + KnownLayout, + Dst: ?Sized + KnownLayout<PointerMetadata = Src::PointerMetadata>, + { + #[inline(always)] + fn project(src: PtrInner<'_, Src>) -> *mut Dst { + // FIXME: Do we want this to support shrinking casts as well? If so, + // we'll need to remove the `CastExact` impl. + static_assert!(Src: ?Sized + KnownLayout, Dst: ?Sized + KnownLayout => { + let src = <Src as KnownLayout>::LAYOUT; + let dst = <Dst as KnownLayout>::LAYOUT; + match (src.size_info, dst.size_info) { + (SizeInfo::Sized { size: src_size }, SizeInfo::Sized { size: dst_size }) => src_size == dst_size, + ( + SizeInfo::SliceDst(TrailingSliceLayout { offset: src_offset, elem_size: src_elem_size }), + SizeInfo::SliceDst(TrailingSliceLayout { offset: dst_offset, elem_size: dst_elem_size }) + ) => src.align.get() == dst.align.get() && src_offset == dst_offset && src_elem_size == dst_elem_size, + _ => false, + } + }); + + let metadata = Src::pointer_to_metadata(src.as_ptr()); + Dst::raw_from_ptr_len(src.as_non_null().cast::<u8>(), metadata).as_ptr() + } + } + + // SAFETY: The `Project::project` impl preserves referent address. + unsafe impl<Src, Dst> Cast<Src, Dst> for CastUnsized + where + Src: ?Sized + KnownLayout, + Dst: ?Sized + KnownLayout<PointerMetadata = Src::PointerMetadata>, + { + } + + // SAFETY: By the `static_assert!` in `Project::project`, `Src` and `Dst` + // are either: + // - Both sized and equal in size + // - Both slice DSTs with the same alignment, trailing slice offset, and + // element size. These ensure that any given pointer metadata encodes the + // same size for both `Src` and `Dst` (note that the alignment is required + // as it affects the amount of trailing padding). + unsafe impl<Src, Dst> CastExact<Src, Dst> for CastUnsized + where + Src: ?Sized + KnownLayout, + Dst: ?Sized + KnownLayout<PointerMetadata = Src::PointerMetadata>, + { + } + + /// A field projection + /// + /// A `Projection` is a [`Project`] which implements projection by + /// delegating to an implementation of [`HasField::project`]. + #[allow(missing_debug_implementations, missing_copy_implementations)] + pub struct Projection<F: ?Sized, const VARIANT_ID: i128, const FIELD_ID: i128> { + _never: core::convert::Infallible, + _phantom: PhantomData<F>, + } + + // SAFETY: `HasField::project` has the same safety post-conditions as + // `Project::project`. + unsafe impl<T: ?Sized, F, const VARIANT_ID: i128, const FIELD_ID: i128> Project<T, T::Type> + for Projection<F, VARIANT_ID, FIELD_ID> + where + T: HasField<F, VARIANT_ID, FIELD_ID>, + { + #[inline(always)] + fn project(src: PtrInner<'_, T>) -> *mut T::Type { + T::project(src) + } + } + + // SAFETY: All `repr(C)` union fields exist at offset 0 within the union [1], + // and so any union projection is actually a cast (ie, preserves address). + // + // [1] Per + // https://doc.rust-lang.org/1.92.0/reference/type-layout.html#reprc-unions, + // it's not *technically* guaranteed that non-maximally-sized fields + // are at offset 0, but it's clear that this is the intention of `repr(C)` + // unions. It says: + // + // > A union declared with `#[repr(C)]` will have the same size and + // > alignment as an equivalent C union declaration in the C language for + // > the target platform. + // + // Note that this only mentions size and alignment, not layout. However, + // C unions *do* guarantee that all fields start at offset 0. [2] + // + // This is also reinforced by + // https://doc.rust-lang.org/1.92.0/reference/items/unions.html#r-items.union.fields.offset: + // + // > Fields might have a non-zero offset (except when the C + // > representation is used); in that case the bits starting at the + // > offset of the fields are read + // + // [2] Per https://port70.net/~nsz/c/c11/n1570.html#6.7.2.1p16: + // + // > The size of a union is sufficient to contain the largest of its + // > members. The value of at most one of the members can be stored in a + // > union object at any time. A pointer to a union object, suitably + // > converted, points to each of its members (or if a member is a + // > bit-field, then to the unit in which it resides), and vice versa. + // + // FIXME(https://github.com/rust-lang/unsafe-code-guidelines/issues/595): + // Cite the documentation once it's updated. + unsafe impl<T: ?Sized, F, const FIELD_ID: i128> Cast<T, T::Type> + for Projection<F, { crate::REPR_C_UNION_VARIANT_ID }, FIELD_ID> + where + T: HasField<F, { crate::REPR_C_UNION_VARIANT_ID }, FIELD_ID>, + { + } + + /// A transitive sequence of projections. + /// + /// Given `TU: Project` and `UV: Project`, `TransitiveProject<_, TU, UV>` is + /// a [`Project`] which projects by applying `TU` followed by `UV`. + /// + /// If `TU: Cast` and `UV: Cast`, then `TransitiveProject<_, TU, UV>: Cast`. + #[allow(missing_debug_implementations)] + pub struct TransitiveProject<U: ?Sized, TU, UV> { + _never: core::convert::Infallible, + _projections: PhantomData<(TU, UV)>, + // On our MSRV (1.56), the debuginfo for a tuple containing both an + // uninhabited type and a DST causes an ICE. We split `U` from `TU` and + // `UV` to avoid this situation. + _u: PhantomData<U>, + } + + // SAFETY: Since `TU::project` and `UV::project` are each + // provenance-preserving operations which preserve or shrink the set of + // referent bytes, so is their composition. + unsafe impl<T, U, V, TU, UV> Project<T, V> for TransitiveProject<U, TU, UV> + where + T: ?Sized, + U: ?Sized, + V: ?Sized, + TU: Project<T, U>, + UV: Project<U, V>, + { + #[inline(always)] + fn project(t: PtrInner<'_, T>) -> *mut V { + t.project::<_, TU>().project::<_, UV>().as_ptr() + } + } + + // SAFETY: Since the `Project::project` impl delegates to `TU::project` and + // `UV::project`, and since `TU` and `UV` are `Cast`, the `Project::project` + // impl preserves the address of the referent. + unsafe impl<T, U, V, TU, UV> Cast<T, V> for TransitiveProject<U, TU, UV> + where + T: ?Sized, + U: ?Sized, + V: ?Sized, + TU: Cast<T, U>, + UV: Cast<U, V>, + { + } + + // SAFETY: Since the `Project::project` impl delegates to `TU::project` and + // `UV::project`, and since `TU` and `UV` are `CastExact`, the `Project::project` + // impl preserves the set of referent bytes. + unsafe impl<T, U, V, TU, UV> CastExact<T, V> for TransitiveProject<U, TU, UV> + where + T: ?Sized, + U: ?Sized, + V: ?Sized, + TU: CastExact<T, U>, + UV: CastExact<U, V>, + { + } + + /// A cast from `T` to `[u8]`. + #[allow(missing_copy_implementations, missing_debug_implementations)] + pub struct AsBytesCast; + + // SAFETY: `project` constructs a pointer with the same address as `src` + // and with a referent of the same size as `*src`. It does this using + // provenance-preserving operations. + // + // FIXME(https://github.com/rust-lang/unsafe-code-guidelines/issues/594): + // Technically, this proof assumes that `*src` is contiguous (the same is + // true of other proofs in this codebase). Is this guaranteed anywhere? + unsafe impl<T: ?Sized + KnownLayout> Project<T, [u8]> for AsBytesCast { + #[inline(always)] + fn project(src: PtrInner<'_, T>) -> *mut [u8] { + let bytes = match T::size_of_val_raw(src.as_non_null()) { + Some(bytes) => bytes, + // SAFETY: `KnownLayout::size_of_val_raw` promises to always + // return `Some` so long as the resulting size fits in a + // `usize`. By invariant on `PtrInner`, `src` refers to a range + // of bytes whose size fits in an `isize`, which implies that it + // also fits in a `usize`. + None => unsafe { core::hint::unreachable_unchecked() }, + }; + + core::ptr::slice_from_raw_parts_mut(src.as_ptr().cast::<u8>(), bytes) + } + } + + // SAFETY: The `Project::project` impl preserves referent address. + unsafe impl<T: ?Sized + KnownLayout> Cast<T, [u8]> for AsBytesCast {} + + // SAFETY: The `Project::project` impl preserves the set of referent bytes. + unsafe impl<T: ?Sized + KnownLayout> CastExact<T, [u8]> for AsBytesCast {} + + /// A cast from any type to `()`. + #[allow(missing_copy_implementations, missing_debug_implementations)] + pub struct CastToUnit; + + // SAFETY: The `project` implementation projects to a subset of its + // argument's referent using provenance-preserving operations. + unsafe impl<T: ?Sized> Project<T, ()> for CastToUnit { + #[inline(always)] + fn project(src: PtrInner<'_, T>) -> *mut () { + src.as_ptr().cast::<()>() + } + } + + // SAFETY: The `project` implementation preserves referent address. + unsafe impl<T: ?Sized> Cast<T, ()> for CastToUnit {} +} diff --git a/rust/zerocopy/src/pointer/ptr.rs b/rust/zerocopy/src/pointer/ptr.rs new file mode 100644 index 000000000000..307b2aee63ca --- /dev/null +++ b/rust/zerocopy/src/pointer/ptr.rs @@ -0,0 +1,1584 @@ +// Copyright 2023 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +#![allow(missing_docs)] + +use core::{ + fmt::{Debug, Formatter}, + marker::PhantomData, +}; + +use crate::{ + pointer::{ + inner::PtrInner, + invariant::*, + transmute::{MutationCompatible, SizeEq, TransmuteFromPtr}, + }, + AlignmentError, CastError, CastType, KnownLayout, SizeError, TryFromBytes, ValidityError, +}; + +/// Module used to gate access to [`Ptr`]'s fields. +mod def { + #[cfg(doc)] + use super::super::invariant; + use super::*; + + /// A raw pointer with more restrictions. + /// + /// `Ptr<T>` is similar to [`NonNull<T>`], but it is more restrictive in the + /// following ways (note that these requirements only hold of non-zero-sized + /// referents): + /// - It must derive from a valid allocation. + /// - It must reference a byte range which is contained inside the + /// allocation from which it derives. + /// - As a consequence, the byte range it references must have a size + /// which does not overflow `isize`. + /// + /// Depending on how `Ptr` is parameterized, it may have additional + /// invariants: + /// - `ptr` conforms to the aliasing invariant of + /// [`I::Aliasing`](invariant::Aliasing). + /// - `ptr` conforms to the alignment invariant of + /// [`I::Alignment`](invariant::Alignment). + /// - `ptr` conforms to the validity invariant of + /// [`I::Validity`](invariant::Validity). + /// + /// `Ptr<'a, T>` is [covariant] in `'a` and invariant in `T`. + /// + /// [`NonNull<T>`]: core::ptr::NonNull + /// [covariant]: https://doc.rust-lang.org/reference/subtyping.html + pub struct Ptr<'a, T, I> + where + T: ?Sized, + I: Invariants, + { + /// # Invariants + /// + /// 0. `ptr` conforms to the aliasing invariant of + /// [`I::Aliasing`](invariant::Aliasing). + /// 1. `ptr` conforms to the alignment invariant of + /// [`I::Alignment`](invariant::Alignment). + /// 2. `ptr` conforms to the validity invariant of + /// [`I::Validity`](invariant::Validity). + // SAFETY: `PtrInner<'a, T>` is covariant in `'a` and invariant in `T`. + ptr: PtrInner<'a, T>, + _invariants: PhantomData<I>, + } + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + { + /// Constructs a new `Ptr` from a [`PtrInner`]. + /// + /// # Safety + /// + /// The caller promises that: + /// + /// 0. `ptr` conforms to the aliasing invariant of + /// [`I::Aliasing`](invariant::Aliasing). + /// 1. `ptr` conforms to the alignment invariant of + /// [`I::Alignment`](invariant::Alignment). + /// 2. `ptr` conforms to the validity invariant of + /// [`I::Validity`](invariant::Validity). + pub(crate) unsafe fn from_inner(ptr: PtrInner<'a, T>) -> Ptr<'a, T, I> { + // SAFETY: The caller has promised to satisfy all safety invariants + // of `Ptr`. + Self { ptr, _invariants: PhantomData } + } + + /// Converts this `Ptr<T>` to a [`PtrInner<T>`]. + /// + /// Note that this method does not consume `self`. The caller should + /// watch out for `unsafe` code which uses the returned value in a way + /// that violates the safety invariants of `self`. + #[inline] + #[must_use] + pub fn as_inner(&self) -> PtrInner<'a, T> { + self.ptr + } + } +} + +#[allow(unreachable_pub)] // This is a false positive on our MSRV toolchain. +pub use def::Ptr; + +/// External trait implementations on [`Ptr`]. +mod _external { + use super::*; + + /// SAFETY: Shared pointers are safely `Copy`. `Ptr`'s other invariants + /// (besides aliasing) are unaffected by the number of references that exist + /// to `Ptr`'s referent. The notable cases are: + /// - Alignment is a property of the referent type (`T`) and the address, + /// both of which are unchanged + /// - Let `S(T, V)` be the set of bit values permitted to appear in the + /// referent of a `Ptr<T, I: Invariants<Validity = V>>`. Since this copy + /// does not change `I::Validity` or `T`, `S(T, I::Validity)` is also + /// unchanged. + /// + /// We are required to guarantee that the referents of the original `Ptr` + /// and of the copy (which, of course, are actually the same since they + /// live in the same byte address range) both remain in the set `S(T, + /// I::Validity)`. Since this invariant holds on the original `Ptr`, it + /// cannot be violated by the original `Ptr`, and thus the original `Ptr` + /// cannot be used to violate this invariant on the copy. The inverse + /// holds as well. + impl<'a, T, I> Copy for Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants<Aliasing = Shared>, + { + } + + /// SAFETY: See the safety comment on `Copy`. + impl<'a, T, I> Clone for Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants<Aliasing = Shared>, + { + #[inline] + fn clone(&self) -> Self { + *self + } + } + + impl<'a, T, I> Debug for Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + self.as_inner().as_non_null().fmt(f) + } + } +} + +/// Methods for converting to and from `Ptr` and Rust's safe reference types. +mod _conversions { + use super::*; + use crate::pointer::cast::{CastExact, CastSized, IdCast}; + + /// `&'a T` → `Ptr<'a, T>` + impl<'a, T> Ptr<'a, T, (Shared, Aligned, Valid)> + where + T: 'a + ?Sized, + { + /// Constructs a `Ptr` from a shared reference. + #[inline(always)] + pub fn from_ref(ptr: &'a T) -> Self { + let inner = PtrInner::from_ref(ptr); + // SAFETY: + // 0. `ptr`, by invariant on `&'a T`, conforms to the aliasing + // invariant of `Shared`. + // 1. `ptr`, by invariant on `&'a T`, conforms to the alignment + // invariant of `Aligned`. + // 2. `ptr`'s referent, by invariant on `&'a T`, is a bit-valid `T`. + // This satisfies the requirement that a `Ptr<T, (_, _, Valid)>` + // point to a bit-valid `T`. Even if `T` permits interior + // mutation, this invariant guarantees that the returned `Ptr` + // can only ever be used to modify the referent to store + // bit-valid `T`s, which ensures that the returned `Ptr` cannot + // be used to violate the soundness of the original `ptr: &'a T` + // or of any other references that may exist to the same + // referent. + unsafe { Self::from_inner(inner) } + } + } + + /// `&'a mut T` → `Ptr<'a, T>` + impl<'a, T> Ptr<'a, T, (Exclusive, Aligned, Valid)> + where + T: 'a + ?Sized, + { + /// Constructs a `Ptr` from an exclusive reference. + #[inline(always)] + pub fn from_mut(ptr: &'a mut T) -> Self { + let inner = PtrInner::from_mut(ptr); + // SAFETY: + // 0. `ptr`, by invariant on `&'a mut T`, conforms to the aliasing + // invariant of `Exclusive`. + // 1. `ptr`, by invariant on `&'a mut T`, conforms to the alignment + // invariant of `Aligned`. + // 2. `ptr`'s referent, by invariant on `&'a mut T`, is a bit-valid + // `T`. This satisfies the requirement that a `Ptr<T, (_, _, + // Valid)>` point to a bit-valid `T`. This invariant guarantees + // that the returned `Ptr` can only ever be used to modify the + // referent to store bit-valid `T`s, which ensures that the + // returned `Ptr` cannot be used to violate the soundness of the + // original `ptr: &'a mut T`. + unsafe { Self::from_inner(inner) } + } + } + + /// `Ptr<'a, T>` → `&'a T` + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants<Alignment = Aligned, Validity = Valid>, + I::Aliasing: Reference, + { + /// Converts `self` to a shared reference. + // This consumes `self`, not `&self`, because `self` is, logically, a + // pointer. For `I::Aliasing = invariant::Shared`, `Self: Copy`, and so + // this doesn't prevent the caller from still using the pointer after + // calling `as_ref`. + #[allow(clippy::wrong_self_convention)] + #[inline] + #[must_use] + pub fn as_ref(self) -> &'a T { + let raw = self.as_inner().as_non_null(); + // SAFETY: `self` satisfies the `Aligned` invariant, so we know that + // `raw` is validly-aligned for `T`. + #[cfg(miri)] + unsafe { + crate::util::miri_promise_symbolic_alignment( + raw.as_ptr().cast(), + core::mem::align_of_val_raw(raw.as_ptr()), + ); + } + // SAFETY: This invocation of `NonNull::as_ref` satisfies its + // documented safety preconditions: + // + // 1. The pointer is properly aligned. This is ensured by-contract + // on `Ptr`, because the `I::Alignment` is `Aligned`. + // + // 2. If the pointer's referent is not zero-sized, then the pointer + // must be “dereferenceable” in the sense defined in the module + // documentation; i.e.: + // + // > The memory range of the given size starting at the pointer + // > must all be within the bounds of a single allocated object. + // > [2] + // + // This is ensured by contract on all `PtrInner`s. + // + // 3. The pointer must point to a validly-initialized instance of + // `T`. This is ensured by-contract on `Ptr`, because the + // `I::Validity` is `Valid`. + // + // 4. You must enforce Rust’s aliasing rules. This is ensured by + // contract on `Ptr`, because `I::Aliasing: Reference`. Either it + // is `Shared` or `Exclusive`. If it is `Shared`, other + // references may not mutate the referent outside of + // `UnsafeCell`s. + // + // [1]: https://doc.rust-lang.org/std/ptr/struct.NonNull.html#method.as_ref + // [2]: https://doc.rust-lang.org/std/ptr/index.html#safety + unsafe { raw.as_ref() } + } + } + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + I::Aliasing: Reference, + { + /// Reborrows `self`, producing another `Ptr`. + /// + /// Since `self` is borrowed mutably, this prevents any methods from + /// being called on `self` as long as the returned `Ptr` exists. + #[inline] + #[must_use] + #[allow(clippy::needless_lifetimes)] // Allows us to name the lifetime in the safety comment below. + pub fn reborrow<'b>(&'b mut self) -> Ptr<'b, T, I> + where + 'a: 'b, + { + // SAFETY: The following all hold by invariant on `self`, and thus + // hold of `ptr = self.as_inner()`: + // 0. SEE BELOW. + // 1. `ptr` conforms to the alignment invariant of + // [`I::Alignment`](invariant::Alignment). + // 2. `ptr` conforms to the validity invariant of + // [`I::Validity`](invariant::Validity). `self` and the returned + // `Ptr` permit the same bit values in their referents since they + // have the same referent type (`T`) and the same validity + // (`I::Validity`). Thus, regardless of what mutation is + // permitted (`Exclusive` aliasing or `Shared`-aliased interior + // mutation), neither can be used to write a value to the + // referent which violates the other's validity invariant. + // + // For aliasing (0 above), since `I::Aliasing: Reference`, + // there are two cases for `I::Aliasing`: + // - For `invariant::Shared`: `'a` outlives `'b`, and so the + // returned `Ptr` does not permit accessing the referent any + // longer than is possible via `self`. For shared aliasing, it is + // sound for multiple `Ptr`s to exist simultaneously which + // reference the same memory, so creating a new one is not + // problematic. + // - For `invariant::Exclusive`: Since `self` is `&'b mut` and we + // return a `Ptr` with lifetime `'b`, `self` is inaccessible to + // the caller for the lifetime `'b` - in other words, `self` is + // inaccessible to the caller as long as the returned `Ptr` + // exists. Since `self` is an exclusive `Ptr`, no other live + // references or `Ptr`s may exist which refer to the same memory + // while `self` is live. Thus, as long as the returned `Ptr` + // exists, no other references or `Ptr`s which refer to the same + // memory may be live. + unsafe { Ptr::from_inner(self.as_inner()) } + } + + /// Reborrows `self` as shared, producing another `Ptr` with `Shared` + /// aliasing. + /// + /// Since `self` is borrowed mutably, this prevents any methods from + /// being called on `self` as long as the returned `Ptr` exists. + #[inline] + #[must_use] + #[allow(clippy::needless_lifetimes)] // Allows us to name the lifetime in the safety comment below. + pub fn reborrow_shared<'b>(&'b mut self) -> Ptr<'b, T, (Shared, I::Alignment, I::Validity)> + where + 'a: 'b, + { + // SAFETY: The following all hold by invariant on `self`, and thus + // hold of `ptr = self.as_inner()`: + // 0. SEE BELOW. + // 1. `ptr` conforms to the alignment invariant of + // [`I::Alignment`](invariant::Alignment). + // 2. `ptr` conforms to the validity invariant of + // [`I::Validity`](invariant::Validity). `self` and the returned + // `Ptr` permit the same bit values in their referents since they + // have the same referent type (`T`) and the same validity + // (`I::Validity`). Thus, regardless of what mutation is + // permitted (`Exclusive` aliasing or `Shared`-aliased interior + // mutation), neither can be used to write a value to the + // referent which violates the other's validity invariant. + // + // For aliasing (0 above), since `I::Aliasing: Reference`, + // there are two cases for `I::Aliasing`: + // - For `invariant::Shared`: `'a` outlives `'b`, and so the + // returned `Ptr` does not permit accessing the referent any + // longer than is possible via `self`. For shared aliasing, it is + // sound for multiple `Ptr`s to exist simultaneously which + // reference the same memory, so creating a new one is not + // problematic. + // - For `invariant::Exclusive`: Since `self` is `&'b mut` and we + // return a `Ptr` with lifetime `'b`, `self` is inaccessible to + // the caller for the lifetime `'b` - in other words, `self` is + // inaccessible to the caller as long as the returned `Ptr` + // exists. Since `self` is an exclusive `Ptr`, no other live + // references or `Ptr`s may exist which refer to the same memory + // while `self` is live. Thus, as long as the returned `Ptr` + // exists, no other references or `Ptr`s which refer to the same + // memory may be live. + unsafe { Ptr::from_inner(self.as_inner()) } + } + } + + /// `Ptr<'a, T>` → `&'a mut T` + impl<'a, T> Ptr<'a, T, (Exclusive, Aligned, Valid)> + where + T: 'a + ?Sized, + { + /// Converts `self` to a mutable reference. + #[allow(clippy::wrong_self_convention)] + #[inline] + #[must_use] + pub fn as_mut(self) -> &'a mut T { + let mut raw = self.as_inner().as_non_null(); + // SAFETY: `self` satisfies the `Aligned` invariant, so we know that + // `raw` is validly-aligned for `T`. + #[cfg(miri)] + unsafe { + crate::util::miri_promise_symbolic_alignment( + raw.as_ptr().cast(), + core::mem::align_of_val_raw(raw.as_ptr()), + ); + } + // SAFETY: This invocation of `NonNull::as_mut` satisfies its + // documented safety preconditions: + // + // 1. The pointer is properly aligned. This is ensured by-contract + // on `Ptr`, because the `ALIGNMENT_INVARIANT` is `Aligned`. + // + // 2. If the pointer's referent is not zero-sized, then the pointer + // must be “dereferenceable” in the sense defined in the module + // documentation; i.e.: + // + // > The memory range of the given size starting at the pointer + // > must all be within the bounds of a single allocated object. + // > [2] + // + // This is ensured by contract on all `PtrInner`s. + // + // 3. The pointer must point to a validly-initialized instance of + // `T`. This is ensured by-contract on `Ptr`, because the + // validity invariant is `Valid`. + // + // 4. You must enforce Rust’s aliasing rules. This is ensured by + // contract on `Ptr`, because the `ALIASING_INVARIANT` is + // `Exclusive`. + // + // [1]: https://doc.rust-lang.org/std/ptr/struct.NonNull.html#method.as_mut + // [2]: https://doc.rust-lang.org/std/ptr/index.html#safety + unsafe { raw.as_mut() } + } + } + + /// `Ptr<'a, T>` → `Ptr<'a, U>` + impl<'a, T: ?Sized, I> Ptr<'a, T, I> + where + I: Invariants, + { + #[must_use] + #[inline(always)] + pub fn transmute<U, V, R>(self) -> Ptr<'a, U, (I::Aliasing, Unaligned, V)> + where + V: Validity, + U: TransmuteFromPtr<T, I::Aliasing, I::Validity, V, <U as SizeEq<T>>::CastFrom, R> + + SizeEq<T> + + ?Sized, + { + self.transmute_with::<U, V, <U as SizeEq<T>>::CastFrom, R>() + } + + #[inline] + #[must_use] + pub fn transmute_with<U, V, C, R>(self) -> Ptr<'a, U, (I::Aliasing, Unaligned, V)> + where + V: Validity, + U: TransmuteFromPtr<T, I::Aliasing, I::Validity, V, C, R> + ?Sized, + C: CastExact<T, U>, + { + // SAFETY: + // - By `C: CastExact`, `C` preserves referent address, and so we + // don't need to consider projections in the following safety + // arguments. + // - If aliasing is `Shared`, then by `U: TransmuteFromPtr<T>`, at + // least one of the following holds: + // - `T: Immutable` and `U: Immutable`, in which case it is + // trivially sound for shared code to operate on a `&T` and `&U` + // at the same time, as neither can perform interior mutation + // - It is directly guaranteed that it is sound for shared code to + // operate on these references simultaneously + // - By `U: TransmuteFromPtr<T, I::Aliasing, I::Validity, C, V>`, it + // is sound to perform this transmute using `C`. + unsafe { self.project_transmute_unchecked::<_, _, C>() } + } + + #[inline] + #[must_use] + pub fn recall_validity<V, R>(self) -> Ptr<'a, T, (I::Aliasing, I::Alignment, V)> + where + V: Validity, + T: TransmuteFromPtr<T, I::Aliasing, I::Validity, V, IdCast, R>, + { + let ptr = self.transmute_with::<T, V, IdCast, R>(); + // SAFETY: `self` and `ptr` have the same address and referent type. + // Therefore, if `self` satisfies `I::Alignment`, then so does + // `ptr`. + unsafe { ptr.assume_alignment::<I::Alignment>() } + } + + /// Projects and/or transmutes to a different (unsized) referent type + /// without checking interior mutability. + /// + /// Callers should prefer [`cast`] or [`project`] where possible. + /// + /// [`cast`]: Ptr::cast + /// [`project`]: Ptr::project + /// + /// # Safety + /// + /// The caller promises that: + /// - If `I::Aliasing` is [`Shared`], it must not be possible for safe + /// code, operating on a `&T` and `&U`, with the referents of `self` + /// and `self.project_transmute_unchecked()`, respectively, to cause + /// undefined behavior. + /// - It is sound to project and/or transmute a pointer of type `T` with + /// aliasing `I::Aliasing` and validity `I::Validity` to a pointer of + /// type `U` with aliasing `I::Aliasing` and validity `V`. This is a + /// subtle soundness requirement that is a function of `T`, `U`, + /// `I::Aliasing`, `I::Validity`, and `V`, and may depend upon the + /// presence, absence, or specific location of `UnsafeCell`s in `T` + /// and/or `U`, and on whether interior mutation is ever permitted via + /// those `UnsafeCell`s. See [`Validity`] for more details. + #[inline] + #[must_use] + pub unsafe fn project_transmute_unchecked<U: ?Sized, V, P>( + self, + ) -> Ptr<'a, U, (I::Aliasing, Unaligned, V)> + where + V: Validity, + P: crate::pointer::cast::Project<T, U>, + { + let ptr = self.as_inner().project::<_, P>(); + + // SAFETY: + // + // The following safety arguments rely on the fact that `P: Project` + // guarantees that `P` is a referent-preserving or -shrinking + // projection. Thus, `ptr` addresses a subset of the bytes of + // `*self`, and so certain properties that hold of `*self` also hold + // of `*ptr`. + // + // 0. `ptr` conforms to the aliasing invariant of `I::Aliasing`: + // - `Exclusive`: `self` is the only `Ptr` or reference which is + // permitted to read or modify the referent for the lifetime + // `'a`. Since we consume `self` by value, the returned pointer + // remains the only `Ptr` or reference which is permitted to + // read or modify the referent for the lifetime `'a`. + // - `Shared`: Since `self` has aliasing `Shared`, we know that + // no other code may mutate the referent during the lifetime + // `'a`, except via `UnsafeCell`s, and except as permitted by + // `T`'s library safety invariants. The caller promises that + // any safe operations which can be permitted on a `&T` and a + // `&U` simultaneously must be sound. Thus, no operations on a + // `&U` could violate `&T`'s library safety invariants, and + // vice-versa. Since any mutation via shared references outside + // of `UnsafeCell`s is unsound, this must be impossible using + // `&T` and `&U`. + // - `Inaccessible`: There are no restrictions we need to uphold. + // 1. `ptr` trivially satisfies the alignment invariant `Unaligned`. + // 2. The caller promises that the returned pointer satisfies the + // validity invariant `V` with respect to its referent type, `U`. + unsafe { Ptr::from_inner(ptr) } + } + } + + /// `Ptr<'a, T, (_, _, _)>` → `Ptr<'a, Unalign<T>, (_, Aligned, _)>` + impl<'a, T, I> Ptr<'a, T, I> + where + I: Invariants, + { + /// Converts a `Ptr` an unaligned `T` into a `Ptr` to an aligned + /// `Unalign<T>`. + #[inline] + #[must_use] + pub fn into_unalign( + self, + ) -> Ptr<'a, crate::Unalign<T>, (I::Aliasing, Aligned, I::Validity)> { + // FIXME(#1359): This should be a `transmute_with` call. + // Unfortunately, to avoid blanket impl conflicts, we only implement + // `TransmuteFrom<T>` for `Unalign<T>` (and vice versa) specifically + // for `Valid` validity, not for all validity types. + + // SAFETY: + // - By `CastSized: Cast`, `CastSized` preserves referent address, + // and so we don't need to consider projections in the following + // safety arguments. + // - Since `Unalign<T>` has the same layout as `T`, the returned + // pointer refers to `UnsafeCell`s at the same locations as + // `self`. + // - `Unalign<T>` promises to have the same bit validity as `T`. By + // invariant on `Validity`, the set of bit patterns allowed in the + // referent of a `Ptr<X, (_, _, V)>` is only a function of the + // validity of `X` and of `V`. Thus, the set of bit patterns + // allowed in the referent of a `Ptr<T, (_, _, I::Validity)>` is + // the same as the set of bit patterns allowed in the referent of + // a `Ptr<Unalign<T>, (_, _, I::Validity)>`. As a result, `self` + // and the returned `Ptr` permit the same set of bit patterns in + // their referents, and so neither can be used to violate the + // validity of the other. + let ptr = unsafe { self.project_transmute_unchecked::<_, _, CastSized>() }; + ptr.bikeshed_recall_aligned() + } + } + + impl<'a, T, I> Ptr<'a, T, I> + where + T: ?Sized, + I: Invariants<Validity = Valid>, + I::Aliasing: Reference, + { + /// Reads the referent. + #[must_use] + #[inline(always)] + pub fn read<R>(self) -> T + where + T: Copy, + T: Read<I::Aliasing, R>, + { + <I::Alignment as Alignment>::read(self) + } + + /// Views the value as an aligned reference. + /// + /// This is only available if `T` is [`Unaligned`]. + #[must_use] + #[inline] + pub fn unaligned_as_ref(self) -> &'a T + where + T: crate::Unaligned, + { + self.bikeshed_recall_aligned().as_ref() + } + } +} + +/// State transitions between invariants. +mod _transitions { + use super::*; + use crate::{ + pointer::{cast::IdCast, transmute::TryTransmuteFromPtr}, + ReadOnly, + }; + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + { + /// Assumes that `self` satisfies the invariants `H`. + /// + /// # Safety + /// + /// The caller promises that `self` satisfies the invariants `H`. + unsafe fn assume_invariants<H: Invariants>(self) -> Ptr<'a, T, H> { + // SAFETY: The caller has promised to satisfy all parameterized + // invariants of `Ptr`. `Ptr`'s other invariants are satisfied + // by-contract by the source `Ptr`. + unsafe { Ptr::from_inner(self.as_inner()) } + } + + /// Helps the type system unify two distinct invariant types which are + /// actually the same. + #[inline] + #[must_use] + pub fn unify_invariants< + H: Invariants<Aliasing = I::Aliasing, Alignment = I::Alignment, Validity = I::Validity>, + >( + self, + ) -> Ptr<'a, T, H> { + // SAFETY: The associated type bounds on `H` ensure that the + // invariants are unchanged. + unsafe { self.assume_invariants::<H>() } + } + + /// Assumes that `self`'s referent is validly-aligned for `T` if + /// required by `A`. + /// + /// # Safety + /// + /// The caller promises that `self`'s referent conforms to the alignment + /// invariant of `T` if required by `A`. + #[inline] + pub(crate) unsafe fn assume_alignment<A: Alignment>( + self, + ) -> Ptr<'a, T, (I::Aliasing, A, I::Validity)> { + // SAFETY: The caller promises that `self`'s referent is + // well-aligned for `T` if required by `A` . + unsafe { self.assume_invariants() } + } + + /// Checks the `self`'s alignment at runtime, returning an aligned `Ptr` + /// on success. + #[inline] + pub fn try_into_aligned( + self, + ) -> Result<Ptr<'a, T, (I::Aliasing, Aligned, I::Validity)>, AlignmentError<Self, T>> + where + T: Sized, + { + if let Err(err) = + crate::util::validate_aligned_to::<_, T>(self.as_inner().as_non_null()) + { + return Err(err.with_src(self)); + } + + // SAFETY: We just checked the alignment. + Ok(unsafe { self.assume_alignment::<Aligned>() }) + } + + /// Recalls that `self`'s referent is validly-aligned for `T`. + #[inline] + // FIXME(#859): Reconsider the name of this method before making it + // public. + #[must_use] + pub fn bikeshed_recall_aligned(self) -> Ptr<'a, T, (I::Aliasing, Aligned, I::Validity)> + where + T: crate::Unaligned, + { + // SAFETY: The bound `T: Unaligned` ensures that `T` has no + // non-trivial alignment requirement. + unsafe { self.assume_alignment::<Aligned>() } + } + + /// Assumes that `self`'s referent conforms to the validity requirement + /// of `V`. + /// + /// # Safety + /// + /// The caller promises that `self`'s referent conforms to the validity + /// requirement of `V`. + #[must_use] + #[inline] + pub unsafe fn assume_validity<V: Validity>( + self, + ) -> Ptr<'a, T, (I::Aliasing, I::Alignment, V)> { + // SAFETY: The caller promises that `self`'s referent conforms to + // the validity requirement of `V`. + unsafe { self.assume_invariants() } + } + + /// A shorthand for `self.assume_validity<invariant::Initialized>()`. + /// + /// # Safety + /// + /// The caller promises to uphold the safety preconditions of + /// `self.assume_validity<invariant::Initialized>()`. + #[must_use] + #[inline] + pub unsafe fn assume_initialized( + self, + ) -> Ptr<'a, T, (I::Aliasing, I::Alignment, Initialized)> { + // SAFETY: The caller has promised to uphold the safety + // preconditions. + unsafe { self.assume_validity::<Initialized>() } + } + + /// A shorthand for `self.assume_validity<Valid>()`. + /// + /// # Safety + /// + /// The caller promises to uphold the safety preconditions of + /// `self.assume_validity<Valid>()`. + #[must_use] + #[inline] + pub unsafe fn assume_valid(self) -> Ptr<'a, T, (I::Aliasing, I::Alignment, Valid)> { + // SAFETY: The caller has promised to uphold the safety + // preconditions. + unsafe { self.assume_validity::<Valid>() } + } + + /// Checks that `self`'s referent is validly initialized for `T`, + /// returning a `Ptr` with `Valid` on success. + /// + /// # Panics + /// + /// This method will panic if + /// [`T::is_bit_valid`][TryFromBytes::is_bit_valid] panics. + /// + /// # Safety + /// + /// On error, unsafe code may rely on this method's returned + /// `ValidityError` containing `self`. + #[inline] + pub fn try_into_valid<R, S>( + mut self, + ) -> Result<Ptr<'a, T, (I::Aliasing, I::Alignment, Valid)>, ValidityError<Self, T>> + where + T: TryFromBytes + + Read<I::Aliasing, R> + + TryTransmuteFromPtr<T, I::Aliasing, I::Validity, Valid, IdCast, S>, + ReadOnly<T>: Read<I::Aliasing, R>, + I::Aliasing: Reference, + I: Invariants<Validity = Initialized>, + { + // This call may panic. If that happens, it doesn't cause any + // soundness issues, as we have not generated any invalid state + // which we need to fix before returning. + if T::is_bit_valid(self.reborrow().transmute::<_, _, _>().reborrow_shared()) { + // SAFETY: If `T::is_bit_valid`, code may assume that `self` + // contains a bit-valid instance of `T`. By `T: + // TryTransmuteFromPtr<T, I::Aliasing, I::Validity, Valid>`, so + // long as `self`'s referent conforms to the `Valid` validity + // for `T` (which we just confirmed), then this transmute is + // sound. + Ok(unsafe { self.assume_valid() }) + } else { + Err(ValidityError::new(self)) + } + } + + /// Forgets that `self`'s referent is validly-aligned for `T`. + #[inline] + #[must_use] + pub fn forget_aligned(self) -> Ptr<'a, T, (I::Aliasing, Unaligned, I::Validity)> { + // SAFETY: `Unaligned` is less restrictive than `Aligned`. + unsafe { self.assume_invariants() } + } + } +} + +/// Casts of the referent type. +#[cfg_attr(not(zerocopy_unstable_ptr), allow(unreachable_pub))] +pub use _casts::TryWithError; +mod _casts { + use core::cell::UnsafeCell; + + use super::*; + use crate::{ + pointer::cast::{AsBytesCast, Cast}, + HasTag, ProjectField, + }; + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + { + /// Casts to a different referent type without checking interior + /// mutability. + /// + /// Callers should prefer [`cast`][Ptr::cast] where possible. + /// + /// # Safety + /// + /// If `I::Aliasing` is [`Shared`], it must not be possible for safe + /// code, operating on a `&T` and `&U` with the same referent + /// simultaneously, to cause undefined behavior. + #[inline] + #[must_use] + pub unsafe fn cast_unchecked<U, C: Cast<T, U>>( + self, + ) -> Ptr<'a, U, (I::Aliasing, Unaligned, I::Validity)> + where + U: 'a + CastableFrom<T, I::Validity, I::Validity> + ?Sized, + { + // SAFETY: + // - By `C: Cast`, `C` preserves the address of the referent. + // - If `I::Aliasing` is [`Shared`], the caller promises that it + // is not possible for safe code, operating on a `&T` and `&U` + // with the same referent simultaneously, to cause undefined + // behavior. + // - By `U: CastableFrom<T, I::Validity, I::Validity>`, + // `I::Validity` is either `Uninit` or `Initialized`. In both + // cases, the bit validity `I::Validity` has the same semantics + // regardless of referent type. In other words, the set of allowed + // referent values for `Ptr<T, (_, _, I::Validity)>` and `Ptr<U, + // (_, _, I::Validity)>` are identical. As a consequence, neither + // `self` nor the returned `Ptr` can be used to write values which + // are invalid for the other. + unsafe { self.project_transmute_unchecked::<_, _, C>() } + } + + /// Casts to a different referent type. + #[inline] + #[must_use] + pub fn cast<U, C, R>(self) -> Ptr<'a, U, (I::Aliasing, Unaligned, I::Validity)> + where + T: MutationCompatible<U, I::Aliasing, I::Validity, I::Validity, R>, + U: 'a + ?Sized + CastableFrom<T, I::Validity, I::Validity>, + C: Cast<T, U>, + { + // SAFETY: Because `T: MutationCompatible<U, I::Aliasing, R>`, one + // of the following holds: + // - `T: Read<I::Aliasing>` and `U: Read<I::Aliasing>`, in which + // case one of the following holds: + // - `I::Aliasing` is `Exclusive` + // - `T` and `U` are both `Immutable` + // - It is sound for safe code to operate on `&T` and `&U` with the + // same referent simultaneously. + unsafe { self.cast_unchecked::<_, C>() } + } + + #[inline(always)] + pub fn project<F, const VARIANT_ID: i128, const FIELD_ID: i128>( + mut self, + ) -> Result<Ptr<'a, T::Type, T::Invariants>, T::Error> + where + T: ProjectField<F, I, VARIANT_ID, FIELD_ID>, + I::Aliasing: Reference, + { + use crate::pointer::cast::Projection; + match T::is_projectable(self.reborrow().project_tag()) { + Ok(()) => { + let inner = self.as_inner(); + let projected = inner.project::<_, Projection<F, VARIANT_ID, FIELD_ID>>(); + // SAFETY: By `T: ProjectField<F, I, VARIANT_ID, FIELD_ID>`, + // for `self: Ptr<'_, T, I>` such that `T::is_projectable` + // (which we've verified in this match arm), + // `T::project(self.as_inner())` conforms to + // `T::Invariants`. The `projected` pointer satisfies these + // invariants because it is produced by way of an + // abstraction that is equivalent to + // `T::project(ptr.as_inner())`: by invariant on + // `PtrInner::project`, `projected` is guaranteed to address + // the subset of the bytes of `inner`'s referent addressed + // by `Projection::project(inner)`, and by invariant on + // `Projection`, `Projection::project` is implemented by + // delegating to an implementation of `HasField::project`. + Ok(unsafe { Ptr::from_inner(projected) }) + } + Err(err) => Err(err), + } + } + + #[must_use] + #[inline(always)] + pub(crate) fn project_tag(self) -> Ptr<'a, T::Tag, I> + where + T: HasTag, + { + // SAFETY: By invariant on `Self::ProjectToTag`, this is a sound + // projection. + let tag = unsafe { self.project_transmute_unchecked::<_, _, T::ProjectToTag>() }; + // SAFETY: By invariant on `Self::ProjectToTag`, the projected + // pointer has the same alignment as `ptr`. + let tag = unsafe { tag.assume_alignment() }; + tag.unify_invariants() + } + + /// Attempts to transform the pointer, restoring the original on + /// failure. + /// + /// # Safety + /// + /// If `I::Aliasing != Shared`, then if `f` returns `Err(err)`, no copy + /// of `f`'s argument must exist outside of `err`. + #[inline(always)] + pub(crate) unsafe fn try_with_unchecked<U, J, E, F>( + self, + f: F, + ) -> Result<Ptr<'a, U, J>, E::Mapped> + where + U: 'a + ?Sized, + J: Invariants<Aliasing = I::Aliasing>, + E: TryWithError<Self>, + F: FnOnce(Ptr<'a, T, I>) -> Result<Ptr<'a, U, J>, E>, + { + let old_inner = self.as_inner(); + #[rustfmt::skip] + let res = f(self).map_err(#[inline(always)] move |err: E| { + err.map(#[inline(always)] |src| { + drop(src); + + // SAFETY: + // 0. Aliasing is either `Shared` or `Exclusive`: + // - If aliasing is `Shared`, then it cannot violate + // aliasing make another copy of this pointer (in fact, + // using `I::Aliasing = Shared`, we could have just + // cloned `self`). + // - If aliasing is `Exclusive`, then `f` is not allowed + // to make another copy of `self`. In `map_err`, we are + // consuming the only value in the returned `Result`. + // By invariant on `E: TryWithError<Self>`, that `err: + // E` only contains a single `Self` and no other + // non-ZST fields which could be `Ptr`s or references + // to `self`'s referent. By the same invariant, `map` + // consumes this single `Self` and passes it to this + // closure. Since `self` was, by invariant on + // `Exclusive`, the only `Ptr` or reference live for + // `'a` with this referent, and since we `drop(src)` + // above, there are no copies left, and so we are + // creating the only copy. + // 1. `self` conforms to `I::Aliasing` by invariant on + // `Ptr`, and `old_inner` has the same address, so it + // does too. + // 2. `f` could not have violated `self`'s validity without + // itself being unsound. Assuming that `f` is sound, the + // referent of `self` is still valid for `T`. + unsafe { Ptr::from_inner(old_inner) } + }) + }); + res + } + + /// Attempts to transform the pointer, restoring the original on + /// failure. + #[inline(always)] + pub fn try_with<U, J, E, F>(self, f: F) -> Result<Ptr<'a, U, J>, E::Mapped> + where + U: 'a + ?Sized, + J: Invariants<Aliasing = I::Aliasing>, + E: TryWithError<Self>, + F: FnOnce(Ptr<'a, T, I>) -> Result<Ptr<'a, U, J>, E>, + I: Invariants<Aliasing = Shared>, + { + // SAFETY: `I::Aliasing = Shared`, so the safety condition does not + // apply. + unsafe { self.try_with_unchecked(f) } + } + } + + /// # Safety + /// + /// `Self` only contains a single `Self::Inner`, and `Self::Mapped` only + /// contains a single `MappedInner`. Other than that, `Self` and + /// `Self::Mapped` contain no non-ZST fields. + /// + /// `map` must pass ownership of `self`'s sole `Self::Inner` to `f`. + pub unsafe trait TryWithError<MappedInner> { + type Inner; + type Mapped; + fn map<F: FnOnce(Self::Inner) -> MappedInner>(self, f: F) -> Self::Mapped; + } + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + KnownLayout + ?Sized, + I: Invariants, + { + /// Casts this pointer-to-initialized into a pointer-to-bytes. + #[allow(clippy::wrong_self_convention)] + #[must_use] + #[inline] + pub fn as_bytes<R>(self) -> Ptr<'a, [u8], (I::Aliasing, Aligned, Valid)> + where + [u8]: TransmuteFromPtr<T, I::Aliasing, I::Validity, Valid, AsBytesCast, R>, + { + self.transmute_with::<[u8], Valid, AsBytesCast, _>().bikeshed_recall_aligned() + } + } + + impl<'a, T, I, const N: usize> Ptr<'a, [T; N], I> + where + T: 'a, + I: Invariants, + { + /// Casts this pointer-to-array into a slice. + #[allow(clippy::wrong_self_convention)] + #[inline] + #[must_use] + pub fn as_slice(self) -> Ptr<'a, [T], I> { + let slice = self.as_inner().as_slice(); + // SAFETY: Note that, by post-condition on `PtrInner::as_slice`, + // `slice` refers to the same byte range as `self.as_inner()`. + // + // 0. Thus, `slice` conforms to the aliasing invariant of + // `I::Aliasing` because `self` does. + // 1. By the above lemma, `slice` conforms to the alignment + // invariant of `I::Alignment` because `self` does. + // 2. Since `[T; N]` and `[T]` have the same bit validity [1][2], + // and since `self` and the returned `Ptr` have the same validity + // invariant, neither `self` nor the returned `Ptr` can be used + // to write a value to the referent which violates the other's + // validity invariant. + // + // [1] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#array-layout: + // + // An array of `[T; N]` has a size of `size_of::<T>() * N` and the + // same alignment of `T`. Arrays are laid out so that the + // zero-based `nth` element of the array is offset from the start + // of the array by `n * size_of::<T>()` bytes. + // + // ... + // + // Slices have the same layout as the section of the array they + // slice. + // + // [2] Per https://doc.rust-lang.org/1.81.0/reference/types/array.html#array-types: + // + // All elements of arrays are always initialized + unsafe { Ptr::from_inner(slice) } + } + } + + /// For caller convenience, these methods are generic over alignment + /// invariant. In practice, the referent is always well-aligned, because the + /// alignment of `[u8]` is 1. + impl<'a, I> Ptr<'a, [u8], I> + where + I: Invariants<Validity = Valid>, + { + /// Attempts to cast `self` to a `U` using the given cast type. + /// + /// If `U` is a slice DST and pointer metadata (`meta`) is provided, + /// then the cast will only succeed if it would produce an object with + /// the given metadata. + /// + /// Returns `None` if the resulting `U` would be invalidly-aligned, if + /// no `U` can fit in `self`, or if the provided pointer metadata + /// describes an invalid instance of `U`. On success, returns a pointer + /// to the largest-possible `U` which fits in `self`. + /// + /// # Safety + /// + /// The caller may assume that this implementation is correct, and may + /// rely on that assumption for the soundness of their code. In + /// particular, the caller may assume that, if `try_cast_into` returns + /// `Some((ptr, remainder))`, then `ptr` and `remainder` refer to + /// non-overlapping byte ranges within `self`, and that `ptr` and + /// `remainder` entirely cover `self`. Finally: + /// - If this is a prefix cast, `ptr` has the same address as `self`. + /// - If this is a suffix cast, `remainder` has the same address as + /// `self`. + #[inline(always)] + pub fn try_cast_into<U, R>( + self, + cast_type: CastType, + meta: Option<U::PointerMetadata>, + ) -> Result< + (Ptr<'a, U, (I::Aliasing, Aligned, Initialized)>, Ptr<'a, [u8], I>), + CastError<Self, U>, + > + where + I::Aliasing: Reference, + U: 'a + ?Sized + KnownLayout + Read<I::Aliasing, R>, + { + let (inner, remainder) = self.as_inner().try_cast_into(cast_type, meta).map_err( + #[inline(always)] + |err| { + err.map_src( + #[inline(always)] + |inner| + // SAFETY: `PtrInner::try_cast_into` promises to return its + // original argument on error, which was originally produced + // by `self.as_inner()`, which is guaranteed to satisfy + // `Ptr`'s invariants. + unsafe { Ptr::from_inner(inner) }, + ) + }, + )?; + + // SAFETY: + // 0. Since `U: Read<I::Aliasing, _>`, either: + // - `I::Aliasing` is `Exclusive`, in which case both `src` and + // `ptr` conform to `Exclusive` + // - `I::Aliasing` is `Shared` and `U` is `Immutable` (we already + // know that `[u8]: Immutable`). In this case, neither `U` nor + // `[u8]` permit mutation, and so `Shared` aliasing is + // satisfied. + // 1. `ptr` conforms to the alignment invariant of `Aligned` because + // it is derived from `try_cast_into`, which promises that the + // object described by `target` is validly aligned for `U`. + // 2. By trait bound, `self` - and thus `target` - is a bit-valid + // `[u8]`. `Ptr<[u8], (_, _, Valid)>` and `Ptr<_, (_, _, + // Initialized)>` have the same bit validity, and so neither + // `self` nor `res` can be used to write a value to the referent + // which violates the other's validity invariant. + let res = unsafe { Ptr::from_inner(inner) }; + + // SAFETY: + // 0. `self` and `remainder` both have the type `[u8]`. Thus, they + // have `UnsafeCell`s at the same locations. Type casting does + // not affect aliasing. + // 1. `[u8]` has no alignment requirement. + // 2. `self` has validity `Valid` and has type `[u8]`. Since + // `remainder` references a subset of `self`'s referent, it is + // also a bit-valid `[u8]`. Thus, neither `self` nor `remainder` + // can be used to write a value to the referent which violates + // the other's validity invariant. + let remainder = unsafe { Ptr::from_inner(remainder) }; + + Ok((res, remainder)) + } + + /// Attempts to cast `self` into a `U`, failing if all of the bytes of + /// `self` cannot be treated as a `U`. + /// + /// In particular, this method fails if `self` is not validly-aligned + /// for `U` or if `self`'s size is not a valid size for `U`. + /// + /// # Safety + /// + /// On success, the caller may assume that the returned pointer + /// references the same byte range as `self`. + #[allow(unused)] + #[inline(always)] + pub fn try_cast_into_no_leftover<U, R>( + self, + meta: Option<U::PointerMetadata>, + ) -> Result<Ptr<'a, U, (I::Aliasing, Aligned, Initialized)>, CastError<Self, U>> + where + I::Aliasing: Reference, + U: 'a + ?Sized + KnownLayout + Read<I::Aliasing, R>, + [u8]: Read<I::Aliasing, R>, + { + // SAFETY: The provided closure returns the only copy of `slf`. + unsafe { + self.try_with_unchecked( + #[inline(always)] + |slf| match slf.try_cast_into(CastType::Prefix, meta) { + Ok((slf, remainder)) => { + if remainder.is_empty() { + Ok(slf) + } else { + Err(CastError::Size(SizeError::<_, U>::new(()))) + } + } + Err(err) => Err(err.map_src( + #[inline(always)] + |_slf| (), + )), + }, + ) + } + } + } + + impl<'a, T, I> Ptr<'a, UnsafeCell<T>, I> + where + T: 'a + ?Sized, + I: Invariants<Aliasing = Exclusive>, + { + /// Converts this `Ptr` into a pointer to the underlying data. + /// + /// This call borrows the `UnsafeCell` mutably (at compile-time) which + /// guarantees that we possess the only reference. + /// + /// This is like [`UnsafeCell::get_mut`], but for `Ptr`. + /// + /// [`UnsafeCell::get_mut`]: core::cell::UnsafeCell::get_mut + #[must_use] + #[inline(always)] + pub fn get_mut(self) -> Ptr<'a, T, I> { + // SAFETY: As described below, `UnsafeCell<T>` has the same size + // as `T: ?Sized` (same static size or same DST layout). Thus, + // `*const UnsafeCell<T> as *const T` is a size-preserving cast. + define_cast!(unsafe { Cast<T: ?Sized> = UnsafeCell<T> => T }); + + // SAFETY: + // - Aliasing is `Exclusive`, and so we are not required to promise + // anything about the locations of `UnsafeCell`s. + // - `UnsafeCell<T>` has the same bit validity as `T` [1]. + // Technically the term "representation" doesn't guarantee this, + // but the subsequent sentence in the documentation makes it clear + // that this is the intention. + // + // By invariant on `Validity`, since `T` and `UnsafeCell<T>` have + // the same bit validity, then the set of values which may appear + // in the referent of a `Ptr<T, (_, _, V)>` is the same as the set + // which may appear in the referent of a `Ptr<UnsafeCell<T>, (_, + // _, V)>`. Thus, neither `self` nor `ptr` may be used to write a + // value to the referent which would violate the other's validity + // invariant. + // + // [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout: + // + // `UnsafeCell<T>` has the same in-memory representation as its + // inner type `T`. A consequence of this guarantee is that it is + // possible to convert between `T` and `UnsafeCell<T>`. + let ptr = unsafe { self.project_transmute_unchecked::<_, _, Cast>() }; + + // SAFETY: `UnsafeCell<T>` has the same alignment as `T` [1], + // and so if `self` is guaranteed to be aligned, then so is the + // returned `Ptr`. + // + // [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout: + // + // `UnsafeCell<T>` has the same in-memory representation as + // its inner type `T`. A consequence of this guarantee is that + // it is possible to convert between `T` and `UnsafeCell<T>`. + let ptr = unsafe { ptr.assume_alignment::<I::Alignment>() }; + ptr.unify_invariants() + } + } +} + +/// Projections through the referent. +mod _project { + use super::*; + + impl<'a, T, I> Ptr<'a, [T], I> + where + T: 'a, + I: Invariants, + I::Aliasing: Reference, + { + /// Iteratively projects the elements `Ptr<T>` from `Ptr<[T]>`. + #[inline] + pub fn iter(self) -> impl Iterator<Item = Ptr<'a, T, I>> { + // SAFETY: + // 0. `elem` conforms to the aliasing invariant of `I::Aliasing`: + // - `Exclusive`: `self` is consumed by value, and therefore + // cannot be used to access the slice while any yielded + // element `Ptr` is live. Each non-zero-sized element is a + // disjoint byte range within the slice, and zero-sized + // elements address no bytes, so distinct yielded element + // `Ptr`s do not alias each other. + // - `Shared`: It is sound for multiple shared `Ptr`s to exist + // simultaneously which reference the same memory. + // 1. `elem`, conditionally, conforms to the validity invariant of + // `I::Alignment`. If `elem` is projected from data well-aligned + // for `[T]`, `elem` will be valid for `T`. + // 2. `elem` conforms to the validity invariant of `I::Validity`. + // Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#array-layout: + // + // Slices have the same layout as the section of the array they + // slice. + // + // Arrays are laid out so that the zero-based `nth` element of + // the array is offset from the start of the array by `n * + // size_of::<T>()` bytes. Thus, `elem` addresses a valid `T` + // within the slice. Since `self` satisfies `I::Validity`, `elem` + // also satisfies `I::Validity`. + self.as_inner().iter().map( + #[inline(always)] + |elem| unsafe { Ptr::from_inner(elem) }, + ) + } + } + + #[allow(clippy::needless_lifetimes)] + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized + KnownLayout<PointerMetadata = usize>, + I: Invariants, + { + /// The number of slice elements in the object referenced by `self`. + #[inline] + #[must_use] + pub fn len(&self) -> usize { + self.as_inner().meta().get() + } + + /// Returns `true` if the slice pointer has a length of 0. + #[inline] + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + } +} + +#[cfg(test)] +mod tests { + use core::mem::{self, MaybeUninit}; + + use super::*; + #[allow(unused)] // Needed on our MSRV, but considered unused on later toolchains. + use crate::util::AsAddress; + use crate::{pointer::BecauseImmutable, util::testutil::AU64, FromBytes, Immutable}; + + mod test_ptr_try_cast_into_soundness { + use super::*; + + // This test is designed so that if `Ptr::try_cast_into_xxx` are + // buggy, it will manifest as unsoundness that Miri can detect. + + // - If `size_of::<T>() == 0`, `N == 4` + // - Else, `N == 4 * size_of::<T>()` + // + // Each test will be run for each metadata in `metas`. + fn test<T, I, const N: usize>(metas: I) + where + T: ?Sized + KnownLayout + Immutable + FromBytes, + I: IntoIterator<Item = Option<T::PointerMetadata>> + Clone, + { + let mut bytes = [MaybeUninit::<u8>::uninit(); N]; + let initialized = [MaybeUninit::new(0u8); N]; + for start in 0..=bytes.len() { + for end in start..=bytes.len() { + // Set all bytes to uninitialized other than those in + // the range we're going to pass to `try_cast_from`. + // This allows Miri to detect out-of-bounds reads + // because they read uninitialized memory. Without this, + // some out-of-bounds reads would still be in-bounds of + // `bytes`, and so might spuriously be accepted. + bytes = [MaybeUninit::<u8>::uninit(); N]; + let bytes = &mut bytes[start..end]; + // Initialize only the byte range we're going to pass to + // `try_cast_from`. + bytes.copy_from_slice(&initialized[start..end]); + + let bytes = { + let bytes: *const [MaybeUninit<u8>] = bytes; + #[allow(clippy::as_conversions)] + let bytes = bytes as *const [u8]; + // SAFETY: We just initialized these bytes to valid + // `u8`s. + unsafe { &*bytes } + }; + + // SAFETY: The bytes in `slf` must be initialized. + unsafe fn validate_and_get_len< + T: ?Sized + KnownLayout + FromBytes + Immutable, + >( + slf: Ptr<'_, T, (Shared, Aligned, Initialized)>, + ) -> usize { + let t = slf.recall_validity().as_ref(); + + let bytes = { + let len = mem::size_of_val(t); + let t: *const T = t; + // SAFETY: + // - We know `t`'s bytes are all initialized + // because we just read it from `slf`, which + // points to an initialized range of bytes. If + // there's a bug and this doesn't hold, then + // that's exactly what we're hoping Miri will + // catch! + // - Since `T: FromBytes`, `T` doesn't contain + // any `UnsafeCell`s, so it's okay for `t: T` + // and a `&[u8]` to the same memory to be + // alive concurrently. + unsafe { core::slice::from_raw_parts(t.cast::<u8>(), len) } + }; + + // This assertion ensures that `t`'s bytes are read + // and compared to another value, which in turn + // ensures that Miri gets a chance to notice if any + // of `t`'s bytes are uninitialized, which they + // shouldn't be (see the comment above). + assert_eq!(bytes, vec![0u8; bytes.len()]); + + mem::size_of_val(t) + } + + for meta in metas.clone().into_iter() { + for cast_type in [CastType::Prefix, CastType::Suffix] { + if let Ok((slf, remaining)) = Ptr::from_ref(bytes) + .try_cast_into::<T, BecauseImmutable>(cast_type, meta) + { + // SAFETY: All bytes in `bytes` have been + // initialized. + let len = unsafe { validate_and_get_len(slf) }; + assert_eq!(remaining.len(), bytes.len() - len); + #[allow(unstable_name_collisions)] + let bytes_addr = bytes.as_ptr().addr(); + #[allow(unstable_name_collisions)] + let remaining_addr = remaining.as_inner().as_ptr().addr(); + match cast_type { + CastType::Prefix => { + assert_eq!(remaining_addr, bytes_addr + len) + } + CastType::Suffix => assert_eq!(remaining_addr, bytes_addr), + } + + if let Some(want) = meta { + let got = + KnownLayout::pointer_to_metadata(slf.as_inner().as_ptr()); + assert_eq!(got, want); + } + } + } + + if let Ok(slf) = Ptr::from_ref(bytes) + .try_cast_into_no_leftover::<T, BecauseImmutable>(meta) + { + // SAFETY: All bytes in `bytes` have been + // initialized. + let len = unsafe { validate_and_get_len(slf) }; + assert_eq!(len, bytes.len()); + + if let Some(want) = meta { + let got = KnownLayout::pointer_to_metadata(slf.as_inner().as_ptr()); + assert_eq!(got, want); + } + } + } + } + } + } + + #[derive(FromBytes, KnownLayout, Immutable)] + #[repr(C)] + struct SliceDst<T> { + a: u8, + trailing: [T], + } + + // Each test case becomes its own `#[test]` function. We do this because + // this test in particular takes far, far longer to execute under Miri + // than all of our other tests combined. Previously, we had these + // execute sequentially in a single test function. We run Miri tests in + // parallel in CI, but this test being sequential meant that most of + // that parallelism was wasted, as all other tests would finish in a + // fraction of the total execution time, leaving this test to execute on + // a single thread for the remainder of the test. By putting each test + // case in its own function, we permit better use of available + // parallelism. + macro_rules! test { + ($test_name:ident: $ty:ty) => { + #[test] + #[allow(non_snake_case)] + fn $test_name() { + const S: usize = core::mem::size_of::<$ty>(); + const N: usize = if S == 0 { 4 } else { S * 4 }; + test::<$ty, _, N>([None]); + + // If `$ty` is a ZST, then we can't pass `None` as the + // pointer metadata, or else computing the correct trailing + // slice length will panic. + if S == 0 { + test::<[$ty], _, N>([Some(0), Some(1), Some(2), Some(3)]); + test::<SliceDst<$ty>, _, N>([Some(0), Some(1), Some(2), Some(3)]); + } else { + test::<[$ty], _, N>([None, Some(0), Some(1), Some(2), Some(3)]); + test::<SliceDst<$ty>, _, N>([None, Some(0), Some(1), Some(2), Some(3)]); + } + } + }; + ($ty:ident) => { + test!($ty: $ty); + }; + ($($ty:ident),*) => { $(test!($ty);)* } + } + + test!(empty_tuple: ()); + test!(u8, u16, u32, u64, usize, AU64); + test!(i8, i16, i32, i64, isize); + test!(f32, f64); + } + + #[test] + fn test_try_cast_into_explicit_count() { + macro_rules! test { + ($ty:ty, $bytes:expr, $elems:expr, $expect:expr) => {{ + let bytes = [0u8; $bytes]; + let ptr = Ptr::from_ref(&bytes[..]); + let res = + ptr.try_cast_into::<$ty, BecauseImmutable>(CastType::Prefix, Some($elems)); + if let Some(expect) = $expect { + let (ptr, _) = res.unwrap(); + assert_eq!(KnownLayout::pointer_to_metadata(ptr.as_inner().as_ptr()), expect); + } else { + let _ = res.unwrap_err(); + } + }}; + } + + #[derive(KnownLayout, Immutable)] + #[repr(C)] + struct ZstDst { + u: [u8; 8], + slc: [()], + } + + test!(ZstDst, 8, 0, Some(0)); + test!(ZstDst, 7, 0, None); + + test!(ZstDst, 8, usize::MAX, Some(usize::MAX)); + test!(ZstDst, 7, usize::MAX, None); + + #[derive(KnownLayout, Immutable)] + #[repr(C)] + struct Dst { + u: [u8; 8], + slc: [u8], + } + + test!(Dst, 8, 0, Some(0)); + test!(Dst, 7, 0, None); + + test!(Dst, 9, 1, Some(1)); + test!(Dst, 8, 1, None); + + // If we didn't properly check for overflow, this would cause the + // metadata to overflow to 0, and thus the cast would spuriously + // succeed. + test!(Dst, 8, usize::MAX - 8 + 1, None); + } + + #[test] + fn test_try_cast_into_no_leftover_restores_original_slice() { + let bytes = [0u8; 4]; + let ptr = Ptr::from_ref(&bytes[..]); + let res = ptr.try_cast_into_no_leftover::<[u8; 2], BecauseImmutable>(None); + match res { + Ok(_) => panic!("should have failed due to leftover bytes"), + Err(CastError::Size(e)) => { + assert_eq!(e.into_src().len(), 4, "Should return original slice length"); + } + Err(e) => panic!("wrong error type: {:?}", e), + } + } + + #[test] + fn test_iter_exclusive_yields_disjoint_ptrs() { + let mut arr = [0u8, 1, 2, 3]; + + { + let mut iter = Ptr::from_mut(&mut arr[..]).iter(); + let first = iter.next().unwrap().as_mut(); + let second = iter.next().unwrap().as_mut(); + + *first = 10; + *second = 20; + *first = 30; + } + + assert_eq!(arr, [30, 20, 2, 3]); + } +} diff --git a/rust/zerocopy/src/pointer/transmute.rs b/rust/zerocopy/src/pointer/transmute.rs new file mode 100644 index 000000000000..45b4f727fd0d --- /dev/null +++ b/rust/zerocopy/src/pointer/transmute.rs @@ -0,0 +1,520 @@ +// Copyright 2025 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +#![allow(missing_docs)] + +use core::{ + cell::{Cell, UnsafeCell}, + mem::{ManuallyDrop, MaybeUninit}, + num::Wrapping, +}; + +use crate::{ + pointer::{ + cast::{self, CastExact, CastSizedExact}, + invariant::*, + }, + FromBytes, Immutable, IntoBytes, Unalign, +}; + +/// Transmutations which are sound to attempt, conditional on validating the bit +/// validity of the destination type. +/// +/// If a `Ptr` transmutation is `TryTransmuteFromPtr`, then it is sound to +/// perform that transmutation so long as some additional mechanism is used to +/// validate that the referent is bit-valid for the destination type. That +/// validation mechanism could be a type bound (such as `TransmuteFrom`) or a +/// runtime validity check. +/// +/// # Safety +/// +/// ## Post-conditions +/// +/// Given `Dst: TryTransmuteFromPtr<Src, A, SV, DV, C, _>`, callers may assume +/// the following: +/// +/// Given `src: Ptr<'a, Src, (A, _, SV)>`, if the referent of `src` is +/// `DV`-valid for `Dst`, then it is sound to transmute `src` into `dst: Ptr<'a, +/// Dst, (A, Unaligned, DV)>` using `C`. +/// +/// ## Pre-conditions +/// +/// Given `src: Ptr<Src, (A, _, SV)>` and `dst: Ptr<Dst, (A, Unaligned, DV)>`, +/// `Dst: TryTransmuteFromPtr<Src, A, SV, DV, C, _>` is sound if all of the +/// following hold: +/// - Forwards transmutation: Either of the following hold: +/// - So long as `dst` is active, no mutation of `dst`'s referent is allowed +/// except via `dst` itself +/// - The set of `DV`-valid referents of `dst` is a superset of the set of +/// `SV`-valid referents of `src` (NOTE: this condition effectively bans +/// shrinking or overwriting transmutes, which cannot satisfy this +/// condition) +/// - Reverse transmutation: Either of the following hold: +/// - `dst` does not permit mutation of its referent +/// - The set of `DV`-valid referents of `dst` is a subset of the set of +/// `SV`-valid referents of `src` (NOTE: this condition effectively bans +/// shrinking or overwriting transmutes, which cannot satisfy this +/// condition) +/// - No safe code, given access to `src` and `dst`, can cause undefined +/// behavior: Any of the following hold: +/// - `A` is `Exclusive` +/// - `Src: Immutable` and `Dst: Immutable` +/// - It is sound for shared code to operate on a `&Src` and `&Dst` which +/// reference the same byte range at the same time +/// +/// ## Proof +/// +/// Given: +/// - `src: Ptr<'a, Src, (A, _, SV)>` +/// - `src`'s referent is `DV`-valid for `Dst` +/// +/// We are trying to prove that it is sound to perform a cast from `src` to a +/// `dst: Ptr<'a, Dst, (A, Unaligned, DV)>` using `C`. We need to prove that +/// such a cast does not violate any of `src`'s invariants, and that it +/// satisfies all invariants of the destination `Ptr` type. +/// +/// First, by `C: CastExact`, `src`'s address is unchanged, so it still satisfies +/// its alignment. Since `dst`'s alignment is `Unaligned`, it trivially satisfies +/// its alignment. +/// +/// Second, aliasing is either `Exclusive` or `Shared`: +/// - If it is `Exclusive`, then both `src` and `dst` satisfy `Exclusive` +/// aliasing trivially: since `src` and `dst` have the same lifetime, `src` is +/// inaccessible so long as `dst` is alive, and no other live `Ptr`s or +/// references may reference the same referent. +/// - If it is `Shared`, then either: +/// - `Src: Immutable` and `Dst: Immutable`, and so neither `src` nor `dst` +/// permit interior mutation. +/// - It is explicitly sound for safe code to operate on a `&Src` and a `&Dst` +/// pointing to the same byte range at the same time. +/// +/// Third, `src`'s validity is satisfied. By invariant, `src`'s referent began +/// as an `SV`-valid `Src`. It is guaranteed to remain so, as either of the +/// following hold: +/// - `dst` does not permit mutation of its referent. +/// - The set of `DV`-valid referents of `dst` is a subset of the set of +/// `SV`-valid referents of `src`. Thus, any value written via `dst` is +/// guaranteed to be an `SV`-valid referent of `src`. +/// +/// Fourth, `dst`'s validity is satisfied. It is a given of this proof that the +/// referent is `DV`-valid for `Dst`. It is guaranteed to remain so, as either +/// of the following hold: +/// - So long as `dst` is active, no mutation of the referent is allowed except +/// via `dst` itself. +/// - The set of `DV`-valid referents of `dst` is a superset of the set of +/// `SV`-valid referents of `src`. Thus, any value written via `src` is +/// guaranteed to be a `DV`-valid referent of `dst`. +pub unsafe trait TryTransmuteFromPtr< + Src: ?Sized, + A: Aliasing, + SV: Validity, + DV: Validity, + C: CastExact<Src, Self>, + R, +> +{ +} + +#[allow(missing_copy_implementations, missing_debug_implementations)] +pub enum BecauseMutationCompatible {} + +// SAFETY: +// - Forwards transmutation: By `Dst: MutationCompatible<Src, A, SV, DV, _>`, we +// know that at least one of the following holds: +// - So long as `dst: Ptr<Dst>` is active, no mutation of its referent is +// allowed except via `dst` itself if either of the following hold: +// - Aliasing is `Exclusive`, in which case, so long as the `Dst` `Ptr` +// exists, no mutation is permitted except via that `Ptr` +// - Aliasing is `Shared`, `Src: Immutable`, and `Dst: Immutable`, in which +// case no mutation is possible via either `Ptr` +// - Since the underlying cast is size-preserving, `dst` addresses the same +// referent as `src`. By `Dst: TransmuteFrom<Src, SV, DV>`, the set of +// `DV`-valid referents of `dst` is a superset of the set of `SV`-valid +// referents of `src`. +// - Reverse transmutation: Since the underlying cast is size-preserving, `dst` +// addresses the same referent as `src`. By `Src: TransmuteFrom<Dst, DV, SV>`, +// the set of `DV`-valid referents of `src` is a subset of the set of +// `SV`-valid referents of `dst`. +// - No safe code, given access to `src` and `dst`, can cause undefined +// behavior: By `Dst: MutationCompatible<Src, A, SV, DV, _>`, at least one of +// the following holds: +// - `A` is `Exclusive` +// - `Src: Immutable` and `Dst: Immutable` +// - `Dst: InvariantsEq<Src>`, which guarantees that `Src` and `Dst` have the +// same invariants, and permit interior mutation on the same byte ranges +unsafe impl<Src, Dst, SV, DV, A, C, R> + TryTransmuteFromPtr<Src, A, SV, DV, C, (BecauseMutationCompatible, R)> for Dst +where + A: Aliasing, + SV: Validity, + DV: Validity, + Src: TransmuteFrom<Dst, DV, SV> + ?Sized, + Dst: MutationCompatible<Src, A, SV, DV, R> + ?Sized, + C: CastExact<Src, Dst>, +{ +} + +// SAFETY: +// - Forwards transmutation: Since aliasing is `Shared` and `Src: Immutable`, +// `src` does not permit mutation of its referent. +// - Reverse transmutation: Since aliasing is `Shared` and `Dst: Immutable`, +// `dst` does not permit mutation of its referent. +// - No safe code, given access to `src` and `dst`, can cause undefined +// behavior: `Src: Immutable` and `Dst: Immutable` +unsafe impl<Src, Dst, SV, DV, C> TryTransmuteFromPtr<Src, Shared, SV, DV, C, BecauseImmutable> + for Dst +where + SV: Validity, + DV: Validity, + Src: Immutable + ?Sized, + Dst: Immutable + ?Sized, + C: CastExact<Src, Dst>, +{ +} + +/// Denotes that `src: Ptr<Src, (A, _, SV)>` and `dst: Ptr<Self, (A, _, DV)>`, +/// referencing the same referent at the same time, cannot be used by safe code +/// to break library safety invariants of `Src` or `Self`. +/// +/// # Safety +/// +/// At least one of the following must hold: +/// - `Src: Read<A, _>` and `Self: Read<A, _>` +/// - `Self: InvariantsEq<Src>`, and, for some `V`: +/// - `Dst: TransmuteFrom<Src, V, V>` +/// - `Src: TransmuteFrom<Dst, V, V>` +pub unsafe trait MutationCompatible<Src: ?Sized, A: Aliasing, SV, DV, R> {} + +#[allow(missing_copy_implementations, missing_debug_implementations)] +pub enum BecauseRead {} + +// SAFETY: `Src: Read<A, _>` and `Dst: Read<A, _>`. +unsafe impl<Src: ?Sized, Dst: ?Sized, A: Aliasing, SV: Validity, DV: Validity, R> + MutationCompatible<Src, A, SV, DV, (BecauseRead, R)> for Dst +where + Src: Read<A, R>, + Dst: Read<A, R>, +{ +} + +/// Denotes that two types have the same invariants. +/// +/// # Safety +/// +/// It is sound for safe code to operate on a `&T` and a `&Self` pointing to the +/// same referent at the same time - no such safe code can cause undefined +/// behavior. +pub unsafe trait InvariantsEq<T: ?Sized> {} + +// SAFETY: Trivially sound to have multiple `&T` pointing to the same referent. +unsafe impl<T: ?Sized> InvariantsEq<T> for T {} + +// SAFETY: `Dst: InvariantsEq<Src> + TransmuteFrom<Src, SV, DV>`, and `Src: +// TransmuteFrom<Dst, DV, SV>`. +unsafe impl<Src: ?Sized, Dst: ?Sized, A: Aliasing, SV: Validity, DV: Validity> + MutationCompatible<Src, A, SV, DV, BecauseInvariantsEq> for Dst +where + Src: TransmuteFrom<Dst, DV, SV>, + Dst: TransmuteFrom<Src, SV, DV> + InvariantsEq<Src>, +{ +} + +#[allow(missing_debug_implementations, missing_copy_implementations)] +pub enum BecauseInvariantsEq {} + +macro_rules! unsafe_impl_invariants_eq { + ($tyvar:ident => $t:ty, $u:ty) => {{ + crate::util::macros::__unsafe(); + // SAFETY: The caller promises that this is sound. + unsafe impl<$tyvar> InvariantsEq<$t> for $u {} + // SAFETY: The caller promises that this is sound. + unsafe impl<$tyvar> InvariantsEq<$u> for $t {} + }}; +} + +impl_transitive_transmute_from!(T => MaybeUninit<T> => T => Wrapping<T>); +impl_transitive_transmute_from!(T => Wrapping<T> => T => MaybeUninit<T>); + +// SAFETY: `ManuallyDrop<T>` has the same size and bit validity as `T` [1], and +// implements `Deref<Target = T>` [2]. Thus, it is already possible for safe +// code to obtain a `&T` and a `&ManuallyDrop<T>` to the same referent at the +// same time. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/mem/struct.ManuallyDrop.html: +// +// `ManuallyDrop<T>` is guaranteed to have the same layout and bit +// validity as `T` +// +// [2] https://doc.rust-lang.org/1.81.0/std/mem/struct.ManuallyDrop.html#impl-Deref-for-ManuallyDrop%3CT%3E +unsafe impl<T: ?Sized> InvariantsEq<T> for ManuallyDrop<T> {} +// SAFETY: See previous safety comment. +unsafe impl<T: ?Sized> InvariantsEq<ManuallyDrop<T>> for T {} + +/// Transmutations which are always sound. +/// +/// `TransmuteFromPtr` is a shorthand for [`TryTransmuteFromPtr`] and +/// [`TransmuteFrom`]. +/// +/// # Safety +/// +/// `Dst: TransmuteFromPtr<Src, A, SV, DV, _>` is equivalent to `Dst: +/// TryTransmuteFromPtr<Src, A, SV, DV, _> + TransmuteFrom<Src, SV, DV>`. +pub unsafe trait TransmuteFromPtr< + Src: ?Sized, + A: Aliasing, + SV: Validity, + DV: Validity, + C: CastExact<Src, Self>, + R, +>: TryTransmuteFromPtr<Src, A, SV, DV, C, R> + TransmuteFrom<Src, SV, DV> +{ +} + +// SAFETY: The `where` bounds are equivalent to the safety invariant on +// `TransmuteFromPtr`. +unsafe impl< + Src: ?Sized, + Dst: ?Sized, + A: Aliasing, + SV: Validity, + DV: Validity, + C: CastExact<Src, Dst>, + R, + > TransmuteFromPtr<Src, A, SV, DV, C, R> for Dst +where + Dst: TransmuteFrom<Src, SV, DV> + TryTransmuteFromPtr<Src, A, SV, DV, C, R>, +{ +} + +/// Denotes that any `SV`-valid `Src` may soundly be transmuted into a +/// `DV`-valid `Self`. +/// +/// # Safety +/// +/// Given `src: Ptr<Src, (_, _, SV)>` and `dst: Ptr<Dst, (_, _, DV)>`, if the +/// referents of `src` and `dst` are the same size, then the set of bit patterns +/// allowed to appear in `src`'s referent must be a subset of the set allowed to +/// appear in `dst`'s referent. +/// +/// If the referents are not the same size, then `Dst: TransmuteFrom<Src, SV, +/// DV>` conveys no safety guarantee. +pub unsafe trait TransmuteFrom<Src: ?Sized, SV, DV> {} + +/// Carries the ability to perform a size-preserving cast or conversion from a +/// raw pointer to `Src` to a raw pointer to `Self`. +/// +/// The cast/conversion is carried by the associated [`CastFrom`] type, and +/// may be a no-op cast (without updating pointer metadata) or a conversion +/// which updates pointer metadata. +/// +/// # Safety +/// +/// `SizeEq` on its own conveys no safety guarantee. Any safety guarantees come +/// from the safety invariants on the associated [`CastFrom`] type, specifically +/// the [`CastExact`] bound. +/// +/// [`CastFrom`]: SizeEq::CastFrom +/// [`CastExact`]: CastExact +pub trait SizeEq<Src: ?Sized> { + type CastFrom: CastExact<Src, Self>; +} + +impl<T: ?Sized> SizeEq<T> for T { + type CastFrom = cast::IdCast; +} + +// SAFETY: Since `Src: IntoBytes`, the set of valid `Src`'s is the set of +// initialized bit patterns, which is exactly the set allowed in the referent of +// any `Initialized` `Ptr`. +unsafe impl<Src, Dst> TransmuteFrom<Src, Valid, Initialized> for Dst +where + Src: IntoBytes + ?Sized, + Dst: ?Sized, +{ +} + +// SAFETY: Since `Dst: FromBytes`, any initialized bit pattern may appear in the +// referent of a `Ptr<Dst, (_, _, Valid)>`. This is exactly equal to the set of +// bit patterns which may appear in the referent of any `Initialized` `Ptr`. +unsafe impl<Src, Dst> TransmuteFrom<Src, Initialized, Valid> for Dst +where + Src: ?Sized, + Dst: FromBytes + ?Sized, +{ +} + +// FIXME(#2354): This seems like a smell - the soundness of this bound has +// nothing to do with `Src` or `Dst` - we're basically just saying `[u8; N]` is +// transmutable into `[u8; N]`. + +// SAFETY: The set of allowed bit patterns in the referent of any `Initialized` +// `Ptr` is the same regardless of referent type. +unsafe impl<Src, Dst> TransmuteFrom<Src, Initialized, Initialized> for Dst +where + Src: ?Sized, + Dst: ?Sized, +{ +} + +// FIXME(#2354): This seems like a smell - the soundness of this bound has +// nothing to do with `Dst` - we're basically just saying that any type is +// transmutable into `MaybeUninit<[u8; N]>`. + +// SAFETY: A `Dst` with validity `Uninit` permits any byte sequence, and +// therefore can be transmuted from any value. +unsafe impl<Src, Dst, V> TransmuteFrom<Src, V, Uninit> for Dst +where + Src: ?Sized, + Dst: ?Sized, + V: Validity, +{ +} + +// SAFETY: +// - `ManuallyDrop<T>` has the same size as `T` [1] +// - `ManuallyDrop<T>` has the same validity as `T` [1] +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/mem/struct.ManuallyDrop.html: +// +// `ManuallyDrop<T>` is guaranteed to have the same layout and bit validity as +// `T` +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T: ?Sized => ManuallyDrop<T>) }; + +// SAFETY: +// - `Unalign<T>` promises to have the same size as `T`. +// - `Unalign<T>` promises to have the same validity as `T`. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T => Unalign<T>) }; +// SAFETY: `Unalign<T>` promises to have the same size and validity as `T`. +// Given `u: &Unalign<T>`, it is already possible to obtain `let t = +// u.try_deref().unwrap()`. Because `Unalign<T>` has the same size as `T`, the +// returned `&T` must point to the same referent as `u`, and thus it must be +// sound for these two references to exist at the same time since it's already +// possible for safe code to get into this state. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_invariants_eq!(T => T, Unalign<T>) }; + +// SAFETY: +// - `Wrapping<T>` has the same size as `T` [1]. +// - `Wrapping<T>` has only one field, which is `pub` [2]. We are also +// guaranteed per that `Wrapping<T>` has the same layout as `T` [1]. The only +// way for both of these to be true simultaneously is for `Wrapping<T>` to +// have the same bit validity as `T`. In particular, in order to change the +// bit validity, one of the following would need to happen: +// - `Wrapping` could change its `repr`, but this would violate the layout +// guarantee. +// - `Wrapping` could add or change its fields, but this would be a +// stability-breaking change. +// +// [1] Per https://doc.rust-lang.org/1.85.0/core/num/struct.Wrapping.html#layout-1: +// +// `Wrapping<T>` is guaranteed to have the same layout and ABI as `T`. +// +// [2] Definition from https://doc.rust-lang.org/1.85.0/core/num/struct.Wrapping.html: +// +// ``` +// #[repr(transparent)] +// pub struct Wrapping<T>(pub T); +// ``` +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T => Wrapping<T>) }; + +// SAFETY: By the preceding safety proof, `Wrapping<T>` and `T` have the same +// layout and bit validity. Since a `Wrapping<T>`'s `T` field is `pub`, given +// `w: &Wrapping<T>`, it's possible to do `let t = &w.t`, which means that it's +// already possible for safe code to obtain a `&Wrapping<T>` and a `&T` pointing +// to the same referent at the same time. Thus, this must be sound. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_invariants_eq!(T => T, Wrapping<T>) }; + +// SAFETY: +// - `UnsafeCell<T>` has the same size as `T` [1]. +// - Per [1], `UnsafeCell<T>` has the same bit validity as `T`. Technically the +// term "representation" doesn't guarantee this, but the subsequent sentence +// in the documentation makes it clear that this is the intention. +// +// [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout: +// +// `UnsafeCell<T>` has the same in-memory representation as its inner type +// `T`. A consequence of this guarantee is that it is possible to convert +// between `T` and `UnsafeCell<T>`. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T: ?Sized => UnsafeCell<T>) }; + +// SAFETY: +// - `Cell<T>` has the same size as `T` [1]. +// - Per [1], `Cell<T>` has the same bit validity as `T`. Technically the term +// "representation" doesn't guarantee this, but it does promise to have the +// "same memory layout and caveats as `UnsafeCell<T>`." The `UnsafeCell` docs +// [2] make it clear that bit validity is the intention even if that phrase +// isn't used. +// +// [1] Per https://doc.rust-lang.org/1.85.0/std/cell/struct.Cell.html#memory-layout: +// +// `Cell<T>` has the same memory layout and caveats as `UnsafeCell<T>`. In +// particular, this means that `Cell<T>` has the same in-memory representation +// as its inner type `T`. +// +// [2] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout: +// +// `UnsafeCell<T>` has the same in-memory representation as its inner type +// `T`. A consequence of this guarantee is that it is possible to convert +// between `T` and `UnsafeCell<T>`. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T: ?Sized => Cell<T>) }; + +impl_transitive_transmute_from!(T: ?Sized => Cell<T> => T => UnsafeCell<T>); +impl_transitive_transmute_from!(T: ?Sized => UnsafeCell<T> => T => Cell<T>); + +// SAFETY: `MaybeUninit<T>` has no validity requirements. Currently this is not +// explicitly guaranteed, but it's obvious from `MaybeUninit`'s documentation +// that this is the intention: +// https://doc.rust-lang.org/1.85.0/core/mem/union.MaybeUninit.html +unsafe impl<T> TransmuteFrom<T, Uninit, Valid> for MaybeUninit<T> {} + +impl<T> SizeEq<T> for MaybeUninit<T> { + type CastFrom = CastSizedExact; +} + +impl<T> SizeEq<MaybeUninit<T>> for T { + type CastFrom = CastSizedExact; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::pointer::cast::Project as _; + + fn test_size_eq<Src, Dst: SizeEq<Src>>(mut src: Src) { + let _: *mut Dst = + <Dst as SizeEq<Src>>::CastFrom::project(crate::pointer::PtrInner::from_mut(&mut src)); + } + + #[test] + fn test_transmute_coverage() { + // SizeEq<T> for MaybeUninit<T> + test_size_eq::<u8, MaybeUninit<u8>>(0u8); + + // SizeEq<MaybeUninit<T>> for T + test_size_eq::<MaybeUninit<u8>, u8>(MaybeUninit::<u8>::new(0)); + + // Transitive: MaybeUninit<T> -> Wrapping<T> + // T => MaybeUninit<T> => T => Wrapping<T> + test_size_eq::<u8, Wrapping<u8>>(0u8); + + // T => Wrapping<T> => T => MaybeUninit<T> + test_size_eq::<Wrapping<u8>, MaybeUninit<u8>>(Wrapping(0u8)); + + // T: ?Sized => Cell<T> => T => UnsafeCell<T> + test_size_eq::<Cell<u8>, UnsafeCell<u8>>(Cell::new(0u8)); + + // T: ?Sized => UnsafeCell<T> => T => Cell<T> + test_size_eq::<UnsafeCell<u8>, Cell<u8>>(UnsafeCell::new(0u8)); + } +} |
