diff options
251 files changed, 38724 insertions, 27 deletions
@@ -1956,6 +1956,8 @@ rustfmt: -path $(srctree)/rust/proc-macro2 \ -o -path $(srctree)/rust/quote \ -o -path $(srctree)/rust/syn \ + -o -path $(srctree)/rust/zerocopy \ + -o -path $(srctree)/rust/zerocopy-derive \ \) -prune -o \ -type f -a -name '*.rs' -a ! -name '*generated*' -print \ | xargs $(RUSTFMT) $(rustfmt_flags) diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index 6c2ab69cb605..ad37994ac15a 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -48,7 +48,7 @@ fn request_firmware( /// Structure used to describe some firmwares, notably FWSEC-FRTS. #[repr(C)] -#[derive(Debug, Clone)] +#[derive(Debug, Clone, FromBytes)] pub(crate) struct FalconUCodeDescV2 { /// Header defined by 'NV_BIT_FALCON_UCODE_DESC_HEADER_VDESC*' in OpenRM. hdr: u32, @@ -84,9 +84,6 @@ pub(crate) struct FalconUCodeDescV2 { pub(crate) alt_dmem_load_size: u32, } -// SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. -unsafe impl FromBytes for FalconUCodeDescV2 {} - /// Structure used to describe some firmwares, notably FWSEC-FRTS. #[repr(C)] #[derive(Debug, Clone)] diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index ebda28e596c5..8b7d17a24660 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -16,6 +16,8 @@ use kernel::{ transmute::FromBytes, }; +use zerocopy::FromBytes as _; + use crate::{ driver::Bar0, firmware::{ @@ -1011,8 +1013,8 @@ impl FwSecBiosImage { let data = self.base.data.get(falcon_ucode_offset..).ok_or(EINVAL)?; match ver { 2 => { - let v2 = FalconUCodeDescV2::from_bytes_copy_prefix(data) - .ok_or(EINVAL)? + let v2 = FalconUCodeDescV2::read_from_prefix(data) + .map_err(|_| EINVAL)? .0; Ok(FalconUCodeDesc::V2(v2)) } diff --git a/rust/Makefile b/rust/Makefile index b9e9f512cec3..2fbdebb93bf2 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -6,6 +6,8 @@ rustdoc_output := $(objtree)/Documentation/output/rust/rustdoc obj-$(CONFIG_RUST) += core.o compiler_builtins.o ffi.o always-$(CONFIG_RUST) += exports_core_generated.h +obj-$(CONFIG_RUST) += zerocopy.o + ifdef CONFIG_RUST_INLINE_HELPERS always-$(CONFIG_RUST) += helpers/helpers.bc helpers/helpers_module.bc else @@ -47,13 +49,14 @@ endif # Avoids running `$(RUSTC)` when it may not be available. ifdef CONFIG_RUST -libmacros_name := $(shell MAKEFLAGS= $(RUSTC) --print file-names --crate-name macros --crate-type proc-macro - </dev/null) -libmacros_extension := $(patsubst libmacros.%,%,$(libmacros_name)) +procmacro-name = $(shell MAKEFLAGS= $(RUSTC) --print file-names --crate-name $(1) --crate-type proc-macro - </dev/null) +procmacro-extension := $(patsubst libname.%,%,$(call procmacro-name,name)) -libpin_init_internal_name := $(shell MAKEFLAGS= $(RUSTC) --print file-names --crate-name pin_init_internal --crate-type proc-macro - </dev/null) -libpin_init_internal_extension := $(patsubst libpin_init_internal.%,%,$(libpin_init_internal_name)) +libzerocopy_derive_name := $(call procmacro-name,zerocopy_derive) +libmacros_name := $(call procmacro-name,macros) +libpin_init_internal_name := $(call procmacro-name,pin_init_internal) -always-$(CONFIG_RUST) += $(libmacros_name) $(libpin_init_internal_name) +always-$(CONFIG_RUST) += $(libzerocopy_derive_name) $(libmacros_name) $(libpin_init_internal_name) # `$(rust_flags)` is passed in case the user added `--sysroot`. rustc_sysroot := $(shell MAKEFLAGS= $(RUSTC) $(rust_flags) --print sysroot) @@ -81,6 +84,12 @@ core-flags := \ --edition=$(core-edition) \ $(call cfgs-to-flags,$(core-cfgs)) +zerocopy-flags := \ + --cap-lints=allow + +zerocopy-envs := \ + CARGO_PKG_VERSION=0.8.50 + proc_macro2-cfgs := \ feature="proc-macro" \ wrap_proc_macro \ @@ -118,6 +127,12 @@ syn-flags := \ --extern quote \ $(call cfgs-to-flags,$(syn-cfgs)) +zerocopy_derive-flags := \ + --cap-lints=allow \ + --extern proc_macro2 \ + --extern quote \ + --extern syn + pin_init_internal-cfgs := \ kernel USE_RUSTC_FEATURES @@ -144,6 +159,7 @@ doctests_modifiers_workaround := $(rustdoc_modifiers_workaround)$(if $(call rust quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< cmd_rustdoc = \ + $(rustc_target_envs) \ OBJTREE=$(abspath $(objtree)) \ $(RUSTDOC) $(filter-out $(skip_flags) --remap-path-scope=%,$(if $(rustdoc_host),$(rust_common_flags),$(rust_flags))) \ $(rustc_target_flags) -L$(objtree)/$(obj) \ @@ -166,7 +182,7 @@ quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< # command-like flags to solve the issue. Meanwhile, we use the non-custom case # and then retouch the generated files. rustdoc: rustdoc-core rustdoc-macros rustdoc-compiler_builtins \ - rustdoc-kernel rustdoc-pin_init + rustdoc-kernel rustdoc-pin_init rustdoc-zerocopy rustdoc-zerocopy_derive $(Q)grep -Ehro '<a href="srctree/([^"]+)"' $(rustdoc_output) | \ cut -d'"' -f2 | cut -d/ -f2- | while read f; do \ if [ ! -e "$(srctree)/$$f" ]; then \ @@ -199,6 +215,12 @@ rustdoc-syn: private rustc_target_flags = $(syn-flags) rustdoc-syn: $(src)/syn/lib.rs rustdoc-clean rustdoc-quote FORCE +$(call if_changed,rustdoc) +rustdoc-zerocopy_derive: private rustdoc_host = yes +rustdoc-zerocopy_derive: private rustc_target_flags = $(zerocopy_derive-flags) \ + --extern proc_macro --crate-type proc-macro +rustdoc-zerocopy_derive: $(src)/zerocopy-derive/lib.rs rustdoc-clean rustdoc-syn FORCE + +$(call if_changed,rustdoc) + rustdoc-macros: private rustdoc_host = yes rustdoc-macros: private rustc_target_flags = --crate-type proc-macro \ --extern proc_macro --extern proc_macro2 --extern quote --extern syn @@ -218,6 +240,13 @@ rustdoc-compiler_builtins: private is-kernel-object := y rustdoc-compiler_builtins: $(src)/compiler_builtins.rs rustdoc-core FORCE +$(call if_changed,rustdoc) +rustdoc-zerocopy: private rustc_target_envs := $(zerocopy-envs) +rustdoc-zerocopy: private is-kernel-object := y +rustdoc-zerocopy: private rustc_target_flags = $(zerocopy-flags) \ + --extend-css $(src)/zerocopy/rustdoc/style.css +rustdoc-zerocopy: $(src)/zerocopy/src/lib.rs rustdoc-clean rustdoc-core FORCE + +$(call if_changed,rustdoc) + rustdoc-ffi: private is-kernel-object := y rustdoc-ffi: $(src)/ffi.rs rustdoc-core FORCE +$(call if_changed,rustdoc) @@ -239,7 +268,8 @@ rustdoc-pin_init: $(src)/pin-init/src/lib.rs rustdoc-pin_init_internal \ rustdoc-kernel: private is-kernel-object := y rustdoc-kernel: private rustc_target_flags = --extern ffi --extern pin_init \ --extern build_error --extern macros \ - --extern bindings --extern uapi + --extern bindings --extern uapi \ + --extern zerocopy --extern zerocopy_derive rustdoc-kernel: $(src)/kernel/lib.rs rustdoc-core rustdoc-ffi rustdoc-macros \ rustdoc-pin_init rustdoc-compiler_builtins $(obj)/$(libmacros_name) \ $(obj)/bindings.o FORCE @@ -250,6 +280,7 @@ rustdoc-clean: FORCE quiet_cmd_rustc_test_library = $(RUSTC_OR_CLIPPY_QUIET) TL $< cmd_rustc_test_library = \ + $(rustc_target_envs) \ OBJTREE=$(abspath $(objtree)) \ $(RUSTC_OR_CLIPPY) $(filter-out $(skip_flags),$(rust_common_flags) $(rustc_target_flags)) \ @$(objtree)/include/generated/rustc_cfg \ @@ -258,6 +289,11 @@ quiet_cmd_rustc_test_library = $(RUSTC_OR_CLIPPY_QUIET) TL $< -L$(objtree)/$(obj)/test \ --crate-name $(subst rusttest-,,$(subst rusttestlib-,,$@)) $< +rusttestlib-zerocopy: private rustc_target_envs := $(zerocopy-envs) +rusttestlib-zerocopy: private rustc_target_flags = $(zerocopy-flags) +rusttestlib-zerocopy: $(src)/zerocopy/src/lib.rs FORCE + +$(call if_changed,rustc_test_library) + rusttestlib-build_error: $(src)/build_error.rs FORCE +$(call if_changed,rustc_test_library) @@ -277,6 +313,12 @@ rusttestlib-syn: private rustc_target_flags = $(syn-flags) rusttestlib-syn: $(src)/syn/lib.rs rusttestlib-quote FORCE +$(call if_changed,rustc_test_library) +rusttestlib-zerocopy_derive: private rustc_target_flags = $(zerocopy_derive-flags) \ + --extern proc_macro +rusttestlib-zerocopy_derive: private rustc_test_library_proc = yes +rusttestlib-zerocopy_derive: $(src)/zerocopy-derive/lib.rs rusttestlib-syn FORCE + +$(call if_changed,rustc_test_library) + rusttestlib-macros: private rustc_target_flags = --extern proc_macro \ --extern proc_macro2 --extern quote --extern syn rusttestlib-macros: private rustc_test_library_proc = yes @@ -298,10 +340,11 @@ rusttestlib-pin_init: $(src)/pin-init/src/lib.rs rusttestlib-macros \ rusttestlib-kernel: private rustc_target_flags = --extern ffi \ --extern build_error --extern macros --extern pin_init \ - --extern bindings --extern uapi + --extern bindings --extern uapi \ + --extern zerocopy --extern zerocopy_derive rusttestlib-kernel: $(src)/kernel/lib.rs rusttestlib-bindings rusttestlib-uapi \ rusttestlib-build_error rusttestlib-pin_init $(obj)/$(libmacros_name) \ - $(obj)/bindings.o FORCE + $(obj)/bindings.o rusttestlib-zerocopy rusttestlib-zerocopy_derive FORCE +$(call if_changed,rustc_test_library) rusttestlib-bindings: private rustc_target_flags = --extern ffi --extern pin_init @@ -314,6 +357,7 @@ rusttestlib-uapi: $(src)/uapi/lib.rs rusttestlib-ffi rusttestlib-pin_init FORCE quiet_cmd_rustdoc_test = RUSTDOC T $< cmd_rustdoc_test = \ + $(rustc_target_envs) \ RUST_MODFILE=test.rs \ OBJTREE=$(abspath $(objtree)) \ $(RUSTDOC) --test $(rust_common_flags) \ @@ -328,11 +372,13 @@ quiet_cmd_rustdoc_test_kernel = RUSTDOC TK $< cmd_rustdoc_test_kernel = \ rm -rf $(objtree)/$(obj)/test/doctests/kernel; \ mkdir -p $(objtree)/$(obj)/test/doctests/kernel; \ + $(rustc_target_envs) \ OBJTREE=$(abspath $(objtree)) \ $(RUSTDOC) --test $(filter-out --remap-path-scope=%,$(rust_flags)) \ -L$(objtree)/$(obj) --extern ffi --extern pin_init \ --extern kernel --extern build_error --extern macros \ --extern bindings --extern uapi \ + --extern zerocopy --extern zerocopy_derive \ --no-run --crate-name kernel -Zunstable-options \ --sysroot=/dev/null \ $(doctests_modifiers_workaround) \ @@ -350,6 +396,7 @@ quiet_cmd_rustdoc_test_kernel = RUSTDOC TK $< # so for the moment we skip `-Cpanic=abort`. quiet_cmd_rustc_test = $(RUSTC_OR_CLIPPY_QUIET) T $< cmd_rustc_test = \ + $(rustc_target_envs) \ OBJTREE=$(abspath $(objtree)) \ $(RUSTC_OR_CLIPPY) --test $(rust_common_flags) \ @$(objtree)/include/generated/rustc_cfg \ @@ -517,8 +564,9 @@ $(obj)/exports_bindings_generated.h: $(obj)/bindings.o FORCE $(obj)/exports_kernel_generated.h: $(obj)/kernel.o FORCE $(call if_changed,exports) -quiet_cmd_rustc_procmacrolibrary = $(RUSTC_OR_CLIPPY_QUIET) PL $@ +quiet_cmd_rustc_procmacrolibrary = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) PL $@ cmd_rustc_procmacrolibrary = \ + $(rustc_target_envs) \ $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \ $(filter-out $(skip_flags),$(rust_common_flags) $(rustc_target_flags)) \ --emit=dep-info=$(depfile) --emit=link=$@ --crate-type rlib -O \ @@ -541,17 +589,24 @@ $(obj)/libsyn.rlib: private rustc_target_flags = $(syn-flags) $(obj)/libsyn.rlib: $(src)/syn/lib.rs $(obj)/libquote.rlib FORCE +$(call if_changed_dep,rustc_procmacrolibrary) -quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@ +quiet_cmd_rustc_procmacro = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) P $@ cmd_rustc_procmacro = \ - $(RUSTC_OR_CLIPPY) $(rust_common_flags) $(rustc_target_flags) \ + $(rustc_target_envs) \ + $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) $(rust_common_flags) $(rustc_target_flags) \ -Clinker-flavor=gcc -Clinker=$(HOSTCC) \ -Clink-args='$(call escsq,$(KBUILD_PROCMACROLDFLAGS))' \ --emit=dep-info=$(depfile) --emit=link=$@ --extern proc_macro \ --crate-type proc-macro -L$(objtree)/$(obj) \ - --crate-name $(patsubst lib%.$(libmacros_extension),%,$(notdir $@)) \ + --crate-name $(patsubst lib%.$(procmacro-extension),%,$(notdir $@)) \ @$(objtree)/include/generated/rustc_cfg $< # Procedural macros can only be used with the `rustc` that compiled it. +$(obj)/$(libzerocopy_derive_name): private skip_clippy = 1 +$(obj)/$(libzerocopy_derive_name): private rustc_target_flags = $(zerocopy_derive-flags) +$(obj)/$(libzerocopy_derive_name): $(src)/zerocopy-derive/lib.rs $(obj)/libproc_macro2.rlib \ + $(obj)/libquote.rlib $(obj)/libsyn.rlib FORCE + +$(call if_changed_dep,rustc_procmacro) + $(obj)/$(libmacros_name): private rustc_target_flags = \ --extern proc_macro2 --extern quote --extern syn $(obj)/$(libmacros_name): $(src)/macros/lib.rs $(obj)/libproc_macro2.rlib \ @@ -567,6 +622,7 @@ $(obj)/$(libpin_init_internal_name): $(src)/pin-init/internal/src/lib.rs \ # since Rust 1.95.0 (https://github.com/rust-lang/rust/pull/151534). quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L $@ cmd_rustc_library = \ + $(rustc_target_envs) \ OBJTREE=$(abspath $(objtree)) \ $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \ $(filter-out $(skip_flags),$(rust_flags)) $(rustc_target_flags) \ @@ -591,6 +647,7 @@ rust-analyzer: --cfgs='syn=$(syn-cfgs)' \ --cfgs='pin_init_internal=$(pin_init_internal-cfgs)' \ --cfgs='pin_init=$(pin_init-cfgs)' \ + --envs='zerocopy=$(zerocopy-envs)' \ $(realpath $(srctree)) $(realpath $(objtree)) \ $(rustc_sysroot) $(RUST_LIB_SRC) $(if $(KBUILD_EXTMOD),$(srcroot)) \ > rust-project.json @@ -662,6 +719,13 @@ $(obj)/compiler_builtins.o: private rustc_objcopy = -w -W '__*' $(obj)/compiler_builtins.o: $(src)/compiler_builtins.rs $(obj)/core.o FORCE +$(call if_changed_rule,rustc_library) +$(obj)/zerocopy.o: private skip_clippy = 1 +$(obj)/zerocopy.o: private skip_gendwarfksyms = 1 +$(obj)/zerocopy.o: private rustc_target_envs := $(zerocopy-envs) +$(obj)/zerocopy.o: private rustc_target_flags = $(zerocopy-flags) +$(obj)/zerocopy.o: $(src)/zerocopy/src/lib.rs $(obj)/compiler_builtins.o FORCE + +$(call if_changed_rule,rustc_library) + $(obj)/pin_init.o: private skip_gendwarfksyms = 1 $(obj)/pin_init.o: private rustc_target_flags = $(pin_init-flags) $(obj)/pin_init.o: $(src)/pin-init/src/lib.rs $(obj)/compiler_builtins.o \ @@ -697,9 +761,11 @@ $(obj)/uapi.o: $(src)/uapi/lib.rs \ +$(call if_changed_rule,rustc_library) $(obj)/kernel.o: private rustc_target_flags = --extern ffi --extern pin_init \ - --extern build_error --extern macros --extern bindings --extern uapi + --extern build_error --extern macros --extern bindings --extern uapi \ + --extern zerocopy --extern zerocopy_derive $(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/build_error.o $(obj)/pin_init.o \ - $(obj)/$(libmacros_name) $(obj)/bindings.o $(obj)/uapi.o FORCE + $(obj)/$(libmacros_name) $(obj)/bindings.o $(obj)/uapi.o \ + $(obj)/zerocopy.o $(obj)/$(libzerocopy_derive_name) FORCE +$(call if_changed_rule,rustc_library) ifdef CONFIG_JUMP_LABEL diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index bcd4e7f90bc7..ca260cc3937a 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -61,6 +61,12 @@ pub use pin_init::{ }; #[doc(no_inline)] +pub use zerocopy::FromBytes; + +#[doc(no_inline)] +pub use zerocopy_derive::FromBytes; + +#[doc(no_inline)] pub use super::{ alloc::{ flags::*, diff --git a/rust/zerocopy-derive/README.md b/rust/zerocopy-derive/README.md new file mode 100644 index 000000000000..110f4a401778 --- /dev/null +++ b/rust/zerocopy-derive/README.md @@ -0,0 +1,14 @@ +# `zerocopy-derive` + +These source files come from the Rust `zerocopy-derive` crate, version v0.8.50 +(released 2026-05-31), hosted in the <https://github.com/google/zerocopy> +repository, licensed under "BSD-2-Clause OR Apache-2.0 OR MIT" and only +modified to add the SPDX license identifiers and to remove the generation of +non-ASCII identifiers. + +For copyright details, please see: + + https://github.com/google/zerocopy/blob/v0.8.50/README.md?plain=1 + https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-BSD + https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-APACHE + https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-MIT diff --git a/rust/zerocopy-derive/derive/from_bytes.rs b/rust/zerocopy-derive/derive/from_bytes.rs new file mode 100644 index 000000000000..d693a63b7645 --- /dev/null +++ b/rust/zerocopy-derive/derive/from_bytes.rs @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +use proc_macro2::{Span, TokenStream}; +use syn::{ + parse_quote, Data, DataEnum, DataStruct, DataUnion, Error, Expr, ExprLit, ExprUnary, Lit, UnOp, + WherePredicate, +}; + +use crate::{ + derive::try_from_bytes::derive_try_from_bytes, + repr::{CompoundRepr, EnumRepr, Repr, Spanned}, + util::{enum_size_from_repr, Ctx, FieldBounds, ImplBlockBuilder, Trait, TraitBound}, +}; +/// Returns `Ok(index)` if variant `index` of the enum has a discriminant of +/// zero. If `Err(bool)` is returned, the boolean is true if the enum has +/// unknown discriminants (e.g. discriminants set to const expressions which we +/// can't evaluate in a proc macro). If the enum has unknown discriminants, then +/// it might have a zero variant that we just can't detect. +pub(crate) fn find_zero_variant(enm: &DataEnum) -> Result<usize, bool> { + // Discriminants can be anywhere in the range [i128::MIN, u128::MAX] because + // the discriminant type may be signed or unsigned. Since we only care about + // tracking the discriminant when it's less than or equal to zero, we can + // avoid u128 -> i128 conversions and bounds checking by making the "next + // discriminant" value implicitly negative. + // Technically 64 bits is enough, but 128 is better for future compatibility + // with https://github.com/rust-lang/rust/issues/56071 + let mut next_negative_discriminant = Some(0); + + // Sometimes we encounter explicit discriminants that we can't know the + // value of (e.g. a constant expression that requires evaluation). These + // could evaluate to zero or a negative number, but we can't assume that + // they do (no false positives allowed!). So we treat them like strictly- + // positive values that can't result in any zero variants, and track whether + // we've encountered any unknown discriminants. + let mut has_unknown_discriminants = false; + + for (i, v) in enm.variants.iter().enumerate() { + match v.discriminant.as_ref() { + // Implicit discriminant + None => { + match next_negative_discriminant.as_mut() { + Some(0) => return Ok(i), + // n is nonzero so subtraction is always safe + Some(n) => *n -= 1, + None => (), + } + } + // Explicit positive discriminant + Some((_, Expr::Lit(ExprLit { lit: Lit::Int(int), .. }))) => { + match int.base10_parse::<u128>().ok() { + Some(0) => return Ok(i), + Some(_) => next_negative_discriminant = None, + None => { + // Numbers should never fail to parse, but just in case: + has_unknown_discriminants = true; + next_negative_discriminant = None; + } + } + } + // Explicit negative discriminant + Some((_, Expr::Unary(ExprUnary { op: UnOp::Neg(_), expr, .. }))) => match &**expr { + Expr::Lit(ExprLit { lit: Lit::Int(int), .. }) => { + match int.base10_parse::<u128>().ok() { + Some(0) => return Ok(i), + // x is nonzero so subtraction is always safe + Some(x) => next_negative_discriminant = Some(x - 1), + None => { + // Numbers should never fail to parse, but just in + // case: + has_unknown_discriminants = true; + next_negative_discriminant = None; + } + } + } + // Unknown negative discriminant (e.g. const repr) + _ => { + has_unknown_discriminants = true; + next_negative_discriminant = None; + } + }, + // Unknown discriminant (e.g. const expr) + _ => { + has_unknown_discriminants = true; + next_negative_discriminant = None; + } + } + } + + Err(has_unknown_discriminants) +} +pub(crate) fn derive_from_zeros(ctx: &Ctx, top_level: Trait) -> Result<TokenStream, Error> { + let try_from_bytes = derive_try_from_bytes(ctx, top_level)?; + let from_zeros = match &ctx.ast.data { + Data::Struct(strct) => derive_from_zeros_struct(ctx, strct), + Data::Enum(enm) => derive_from_zeros_enum(ctx, enm)?, + Data::Union(unn) => derive_from_zeros_union(ctx, unn), + }; + Ok(IntoIterator::into_iter([try_from_bytes, from_zeros]).collect()) +} +pub(crate) fn derive_from_bytes(ctx: &Ctx, top_level: Trait) -> Result<TokenStream, Error> { + let from_zeros = derive_from_zeros(ctx, top_level)?; + let from_bytes = match &ctx.ast.data { + Data::Struct(strct) => derive_from_bytes_struct(ctx, strct), + Data::Enum(enm) => derive_from_bytes_enum(ctx, enm)?, + Data::Union(unn) => derive_from_bytes_union(ctx, unn), + }; + + Ok(IntoIterator::into_iter([from_zeros, from_bytes]).collect()) +} +fn derive_from_zeros_struct(ctx: &Ctx, strct: &DataStruct) -> TokenStream { + ImplBlockBuilder::new(ctx, strct, Trait::FromZeros, FieldBounds::ALL_SELF).build() +} +fn derive_from_zeros_enum(ctx: &Ctx, enm: &DataEnum) -> Result<TokenStream, Error> { + let repr = EnumRepr::from_attrs(&ctx.ast.attrs)?; + + // We don't actually care what the repr is; we just care that it's one of + // the allowed ones. + match repr { + Repr::Compound(Spanned { t: CompoundRepr::C | CompoundRepr::Primitive(_), span: _ }, _) => { + } + Repr::Transparent(_) | Repr::Compound(Spanned { t: CompoundRepr::Rust, span: _ }, _) => { + return ctx.error_or_skip( + Error::new( + Span::call_site(), + "must have #[repr(C)] or #[repr(Int)] attribute in order to guarantee this type's memory layout", + ), + ); + } + } + + let zero_variant = match find_zero_variant(enm) { + Ok(index) => enm.variants.iter().nth(index).unwrap(), + // Has unknown variants + Err(true) => { + return ctx.error_or_skip(Error::new_spanned( + &ctx.ast, + "FromZeros only supported on enums with a variant that has a discriminant of `0`\n\ + help: This enum has discriminants which are not literal integers. One of those may \ + define or imply which variant has a discriminant of zero. Use a literal integer to \ + define or imply the variant with a discriminant of zero.", + )); + } + // Does not have unknown variants + Err(false) => { + return ctx.error_or_skip(Error::new_spanned( + &ctx.ast, + "FromZeros only supported on enums with a variant that has a discriminant of `0`", + )); + } + }; + + let zerocopy_crate = &ctx.zerocopy_crate; + let explicit_bounds = zero_variant + .fields + .iter() + .map(|field| { + let ty = &field.ty; + parse_quote! { #ty: #zerocopy_crate::FromZeros } + }) + .collect::<Vec<WherePredicate>>(); + + Ok(ImplBlockBuilder::new(ctx, enm, Trait::FromZeros, FieldBounds::Explicit(explicit_bounds)) + .build()) +} +fn derive_from_zeros_union(ctx: &Ctx, unn: &DataUnion) -> TokenStream { + let field_type_trait_bounds = FieldBounds::All(&[TraitBound::Slf]); + ImplBlockBuilder::new(ctx, unn, Trait::FromZeros, field_type_trait_bounds).build() +} +fn derive_from_bytes_struct(ctx: &Ctx, strct: &DataStruct) -> TokenStream { + ImplBlockBuilder::new(ctx, strct, Trait::FromBytes, FieldBounds::ALL_SELF).build() +} +fn derive_from_bytes_enum(ctx: &Ctx, enm: &DataEnum) -> Result<TokenStream, Error> { + let repr = EnumRepr::from_attrs(&ctx.ast.attrs)?; + + let variants_required = 1usize << enum_size_from_repr(&repr)?; + if enm.variants.len() != variants_required { + return ctx.error_or_skip(Error::new_spanned( + &ctx.ast, + format!( + "FromBytes only supported on {} enum with {} variants", + repr.repr_type_name(), + variants_required + ), + )); + } + + Ok(ImplBlockBuilder::new(ctx, enm, Trait::FromBytes, FieldBounds::ALL_SELF).build()) +} +fn derive_from_bytes_union(ctx: &Ctx, unn: &DataUnion) -> TokenStream { + let field_type_trait_bounds = FieldBounds::All(&[TraitBound::Slf]); + ImplBlockBuilder::new(ctx, unn, Trait::FromBytes, field_type_trait_bounds).build() +} diff --git a/rust/zerocopy-derive/derive/into_bytes.rs b/rust/zerocopy-derive/derive/into_bytes.rs new file mode 100644 index 000000000000..ad52a6b45d28 --- /dev/null +++ b/rust/zerocopy-derive/derive/into_bytes.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +use proc_macro2::{Span, TokenStream}; +use quote::quote; +use syn::{Data, DataEnum, DataStruct, DataUnion, Error, Type}; + +use crate::{ + repr::{EnumRepr, StructUnionRepr}, + util::{ + generate_tag_enum, Ctx, DataExt, FieldBounds, ImplBlockBuilder, PaddingCheck, Trait, + TraitBound, + }, +}; +pub(crate) fn derive_into_bytes(ctx: &Ctx, _top_level: Trait) -> Result<TokenStream, Error> { + match &ctx.ast.data { + Data::Struct(strct) => derive_into_bytes_struct(ctx, strct), + Data::Enum(enm) => derive_into_bytes_enum(ctx, enm), + Data::Union(unn) => derive_into_bytes_union(ctx, unn), + } +} +fn derive_into_bytes_struct(ctx: &Ctx, strct: &DataStruct) -> Result<TokenStream, Error> { + let repr = StructUnionRepr::from_attrs(&ctx.ast.attrs)?; + + let is_transparent = repr.is_transparent(); + let is_c = repr.is_c(); + let is_packed_1 = repr.is_packed_1(); + let num_fields = strct.fields().len(); + + let (padding_check, require_unaligned_fields) = if is_transparent || is_packed_1 { + // No padding check needed. + // - repr(transparent): The layout and ABI of the whole struct is the + // same as its only non-ZST field (meaning there's no padding outside + // of that field) and we require that field to be `IntoBytes` (meaning + // there's no padding in that field). + // - repr(packed): Any inter-field padding bytes are removed, meaning + // that any padding bytes would need to come from the fields, all of + // which we require to be `IntoBytes` (meaning they don't have any + // padding). Note that this holds regardless of other `repr` + // attributes, including `repr(Rust)`. [1] + // + // [1] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#the-alignment-modifiers: + // + // An important consequence of these rules is that a type with + // `#[repr(packed(1))]`` (or `#[repr(packed)]``) will have no + // inter-field padding. + (None, false) + } else if is_c && !repr.is_align_gt_1() && num_fields <= 1 { + // No padding check needed. A repr(C) struct with zero or one field has + // no padding unless #[repr(align)] explicitly adds padding, which we + // check for in this branch's condition. + (None, false) + } else if ctx.ast.generics.params.is_empty() { + // Is the last field a syntactic slice, i.e., `[SomeType]`. + let is_syntactic_dst = + strct.fields().last().map(|(_, _, ty)| matches!(ty, Type::Slice(_))).unwrap_or(false); + // Since there are no generics, we can emit a padding check. All reprs + // guarantee that fields won't overlap [1], so the padding check is + // sound. This is more permissive than the next case, which requires + // that all field types implement `Unaligned`. + // + // [1] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#the-rust-representation: + // + // The only data layout guarantees made by [`repr(Rust)`] are those + // required for soundness. They are: + // ... + // 2. The fields do not overlap. + // ... + if is_c && is_syntactic_dst { + (Some(PaddingCheck::ReprCStruct), false) + } else { + (Some(PaddingCheck::Struct), false) + } + } else if is_c && !repr.is_align_gt_1() { + // We can't use a padding check since there are generic type arguments. + // Instead, we require all field types to implement `Unaligned`. This + // ensures that the `repr(C)` layout algorithm will not insert any + // padding unless #[repr(align)] explicitly adds padding, which we check + // for in this branch's condition. + // + // FIXME(#10): Support type parameters for non-transparent, non-packed + // structs without requiring `Unaligned`. + (None, true) + } else { + return ctx.error_or_skip(Error::new( + Span::call_site(), + "must have a non-align #[repr(...)] attribute in order to guarantee this type's memory layout", + )); + }; + + let field_bounds = if require_unaligned_fields { + FieldBounds::All(&[TraitBound::Slf, TraitBound::Other(Trait::Unaligned)]) + } else { + FieldBounds::ALL_SELF + }; + + Ok(ImplBlockBuilder::new(ctx, strct, Trait::IntoBytes, field_bounds) + .padding_check(padding_check) + .build()) +} + +fn derive_into_bytes_enum(ctx: &Ctx, enm: &DataEnum) -> Result<TokenStream, Error> { + let repr = EnumRepr::from_attrs(&ctx.ast.attrs)?; + if !repr.is_c() && !repr.is_primitive() { + return ctx.error_or_skip(Error::new( + Span::call_site(), + "must have #[repr(C)] or #[repr(Int)] attribute in order to guarantee this type's memory layout", + )); + } + + let tag_type_definition = generate_tag_enum(ctx, &repr, enm); + Ok(ImplBlockBuilder::new(ctx, enm, Trait::IntoBytes, FieldBounds::ALL_SELF) + .padding_check(PaddingCheck::Enum { tag_type_definition }) + .build()) +} + +fn derive_into_bytes_union(ctx: &Ctx, unn: &DataUnion) -> Result<TokenStream, Error> { + // See #1792 for more context. + // + // By checking for `zerocopy_derive_union_into_bytes` both here and in the + // generated code, we ensure that `--cfg zerocopy_derive_union_into_bytes` + // need only be passed *either* when compiling this crate *or* when + // compiling the user's crate. The former is preferable, but in some + // situations (such as when cross-compiling using `cargo build --target`), + // it doesn't get propagated to this crate's build by default. + let cfg_compile_error = if cfg!(zerocopy_derive_union_into_bytes) { + quote!() + } else { + let core = ctx.core_path(); + let error_message = "requires --cfg zerocopy_derive_union_into_bytes; +please let us know you use this feature: https://github.com/google/zerocopy/discussions/1802"; + quote!( + #[allow(unused_attributes, unexpected_cfgs)] + const _: () = { + #[cfg(not(zerocopy_derive_union_into_bytes))] + #core::compile_error!(#error_message); + }; + ) + }; + + // FIXME(#10): Support type parameters. + if !ctx.ast.generics.params.is_empty() { + return ctx.error_or_skip(Error::new( + Span::call_site(), + "unsupported on types with type parameters", + )); + } + + // Because we don't support generics, we don't need to worry about + // special-casing different reprs. So long as there is *some* repr which + // guarantees the layout, our `PaddingCheck::Union` guarantees that there is + // no padding. + let repr = StructUnionRepr::from_attrs(&ctx.ast.attrs)?; + if !repr.is_c() && !repr.is_transparent() && !repr.is_packed_1() { + return ctx.error_or_skip(Error::new( + Span::call_site(), + "must be #[repr(C)], #[repr(packed)], or #[repr(transparent)]", + )); + } + + let impl_block = ImplBlockBuilder::new(ctx, unn, Trait::IntoBytes, FieldBounds::ALL_SELF) + .padding_check(PaddingCheck::Union) + .build(); + Ok(quote!(#cfg_compile_error #impl_block)) +} diff --git a/rust/zerocopy-derive/derive/known_layout.rs b/rust/zerocopy-derive/derive/known_layout.rs new file mode 100644 index 000000000000..fddffd167c82 --- /dev/null +++ b/rust/zerocopy-derive/derive/known_layout.rs @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +use proc_macro2::TokenStream; +use quote::quote; +use syn::{parse_quote, Data, Error, Type}; + +use crate::{ + repr::StructUnionRepr, + util::{Ctx, DataExt, FieldBounds, ImplBlockBuilder, SelfBounds, Trait}, +}; + +fn derive_known_layout_for_repr_c_struct<'a>( + ctx: &'a Ctx, + repr: &StructUnionRepr, + fields: &[(&'a syn::Visibility, TokenStream, &'a Type)], +) -> Option<(SelfBounds<'a>, TokenStream, Option<TokenStream>)> { + let (trailing_field, leading_fields) = fields.split_last()?; + + let (_vis, trailing_field_name, trailing_field_ty) = trailing_field; + let leading_fields_tys = leading_fields.iter().map(|(_vis, _name, ty)| ty); + + let core = ctx.core_path(); + let repr_align = repr + .get_align() + .map(|align| { + let align = align.t.get(); + quote!(#core::num::NonZeroUsize::new(#align as usize)) + }) + .unwrap_or_else(|| quote!(#core::option::Option::None)); + let repr_packed = repr + .get_packed() + .map(|packed| { + let packed = packed.get(); + quote!(#core::num::NonZeroUsize::new(#packed as usize)) + }) + .unwrap_or_else(|| quote!(#core::option::Option::None)); + + let zerocopy_crate = &ctx.zerocopy_crate; + let make_methods = |trailing_field_ty| { + quote! { + // SAFETY: + // - The returned pointer has the same address and provenance as + // `bytes`: + // - The recursive call to `raw_from_ptr_len` preserves both + // address and provenance. + // - The `as` cast preserves both address and provenance. + // - `NonNull::new_unchecked` preserves both address and + // provenance. + // - If `Self` is a slice DST, the returned pointer encodes + // `elems` elements in the trailing slice: + // - This is true of the recursive call to `raw_from_ptr_len`. + // - `trailing.as_ptr() as *mut Self` preserves trailing slice + // element count [1]. + // - `NonNull::new_unchecked` preserves trailing slice element + // count. + // + // [1] Per https://doc.rust-lang.org/reference/expressions/operator-expr.html#pointer-to-pointer-cast: + // + // `*const T`` / `*mut T` can be cast to `*const U` / `*mut U` + // with the following behavior: + // ... + // - If `T` and `U` are both unsized, the pointer is also + // returned unchanged. In particular, the metadata is + // preserved exactly. + // + // For instance, a cast from `*const [T]` to `*const [U]` + // preserves the number of elements. ... The same holds + // for str and any compound type whose unsized tail is a + // slice type, such as struct `Foo(i32, [u8])` or + // `(u64, Foo)`. + #[inline(always)] + fn raw_from_ptr_len( + bytes: #core::ptr::NonNull<u8>, + meta: <Self as #zerocopy_crate::KnownLayout>::PointerMetadata, + ) -> #core::ptr::NonNull<Self> { + let trailing = <#trailing_field_ty as #zerocopy_crate::KnownLayout>::raw_from_ptr_len(bytes, meta); + let slf = trailing.as_ptr() as *mut Self; + // SAFETY: Constructed from `trailing`, which is non-null. + unsafe { #core::ptr::NonNull::new_unchecked(slf) } + } + + #[inline(always)] + fn pointer_to_metadata(ptr: *mut Self) -> <Self as #zerocopy_crate::KnownLayout>::PointerMetadata { + <#trailing_field_ty>::pointer_to_metadata(ptr as *mut _) + } + } + }; + + let inner_extras = { + let leading_fields_tys = leading_fields_tys.clone(); + let methods = make_methods(*trailing_field_ty); + let (_, ty_generics, _) = ctx.ast.generics.split_for_impl(); + + quote!( + type PointerMetadata = <#trailing_field_ty as #zerocopy_crate::KnownLayout>::PointerMetadata; + + type MaybeUninit = __ZerocopyKnownLayoutMaybeUninit #ty_generics; + + // SAFETY: `LAYOUT` accurately describes the layout of `Self`. + // The documentation of `DstLayout::for_repr_c_struct` vows that + // invocations in this manner will accurately describe a type, + // so long as: + // + // - that type is `repr(C)`, + // - its fields are enumerated in the order they appear, + // - the presence of `repr_align` and `repr_packed` are + // correctly accounted for. + // + // We respect all three of these preconditions here. This + // expansion is only used if `is_repr_c_struct`, we enumerate + // the fields in order, and we extract the values of `align(N)` + // and `packed(N)`. + const LAYOUT: #zerocopy_crate::DstLayout = #zerocopy_crate::DstLayout::for_repr_c_struct( + #repr_align, + #repr_packed, + &[ + #(#zerocopy_crate::DstLayout::for_type::<#leading_fields_tys>(),)* + <#trailing_field_ty as #zerocopy_crate::KnownLayout>::LAYOUT + ], + ); + + #methods + ) + }; + + let outer_extras = { + let ident = &ctx.ast.ident; + let vis = &ctx.ast.vis; + let params = &ctx.ast.generics.params; + let (impl_generics, ty_generics, where_clause) = ctx.ast.generics.split_for_impl(); + + let predicates = if let Some(where_clause) = where_clause { + where_clause.predicates.clone() + } else { + Default::default() + }; + + // Generate a valid ident for a type-level handle to a field of a + // given `name`. + let field_index = |name: &TokenStream| ident!(("__Zerocopy_Field_{}", name), ident.span()); + + let field_indices: Vec<_> = + fields.iter().map(|(_vis, name, _ty)| field_index(name)).collect(); + + // Define the collection of type-level field handles. + let field_defs = field_indices.iter().zip(fields).map(|(idx, (vis, _, _))| { + quote! { + #vis struct #idx; + } + }); + + let field_impls = field_indices.iter().zip(fields).map(|(idx, (_, _, ty))| quote! { + // SAFETY: `#ty` is the type of `#ident`'s field at `#idx`. + // + // We implement `Field` for each field of the struct to create a + // projection from the field index to its type. This allows us + // to refer to the field's type in a way that respects `Self` + // hygiene. If we just copy-pasted the tokens of `#ty`, we + // would not respect `Self` hygiene, as `Self` would refer to + // the helper struct we are generating, not the derive target + // type. + unsafe impl #impl_generics #zerocopy_crate::util::macro_util::Field<#idx> for #ident #ty_generics + where + #predicates + { + type Type = #ty; + } + }); + + let trailing_field_index = field_index(trailing_field_name); + let leading_field_indices = + leading_fields.iter().map(|(_vis, name, _ty)| field_index(name)); + + // We use `Field` to project the type of the trailing field. This is + // required to ensure that if the field type uses `Self`, it + // resolves to the derive target type, not the helper struct we are + // generating. + let trailing_field_ty = quote! { + <#ident #ty_generics as + #zerocopy_crate::util::macro_util::Field<#trailing_field_index> + >::Type + }; + + let methods = make_methods(&parse_quote! { + <#trailing_field_ty as #zerocopy_crate::KnownLayout>::MaybeUninit + }); + + let core = ctx.core_path(); + + quote! { + #(#field_defs)* + + #(#field_impls)* + + // SAFETY: This has the same layout as the derive target type, + // except that it admits uninit bytes. This is ensured by using + // the same repr as the target type, and by using field types + // which have the same layout as the target type's fields, + // except that they admit uninit bytes. We indirect through + // `Field` to ensure that occurrences of `Self` resolve to + // `#ty`, not `__ZerocopyKnownLayoutMaybeUninit` (see #2116). + #repr + #[doc(hidden)] + #vis struct __ZerocopyKnownLayoutMaybeUninit<#params> ( + #(#core::mem::MaybeUninit< + <#ident #ty_generics as + #zerocopy_crate::util::macro_util::Field<#leading_field_indices> + >::Type + >,)* + // NOTE(#2302): We wrap in `ManuallyDrop` here in case the + // type we're operating on is both generic and + // `repr(packed)`. In that case, Rust needs to know that the + // type is *either* `Sized` or has a trivial `Drop`. + // `ManuallyDrop` has a trivial `Drop`, and so satisfies + // this requirement. + #core::mem::ManuallyDrop< + <#trailing_field_ty as #zerocopy_crate::KnownLayout>::MaybeUninit + > + ) + where + #trailing_field_ty: #zerocopy_crate::KnownLayout, + #predicates; + + // SAFETY: We largely defer to the `KnownLayout` implementation + // on the derive target type (both by using the same tokens, and + // by deferring to impl via type-level indirection). This is + // sound, since `__ZerocopyKnownLayoutMaybeUninit` is guaranteed + // to have the same layout as the derive target type, except + // that `__ZerocopyKnownLayoutMaybeUninit` admits uninit bytes. + unsafe impl #impl_generics #zerocopy_crate::KnownLayout for __ZerocopyKnownLayoutMaybeUninit #ty_generics + where + #trailing_field_ty: #zerocopy_crate::KnownLayout, + #predicates + { + fn only_derive_is_allowed_to_implement_this_trait() {} + + type PointerMetadata = <#ident #ty_generics as #zerocopy_crate::KnownLayout>::PointerMetadata; + + type MaybeUninit = Self; + + const LAYOUT: #zerocopy_crate::DstLayout = <#ident #ty_generics as #zerocopy_crate::KnownLayout>::LAYOUT; + + #methods + } + } + }; + + Some((SelfBounds::None, inner_extras, Some(outer_extras))) +} + +pub(crate) fn derive(ctx: &Ctx, _top_level: Trait) -> Result<TokenStream, Error> { + // If this is a `repr(C)` struct, then `c_struct_repr` contains the entire + // `repr` attribute. + let c_struct_repr = match &ctx.ast.data { + Data::Struct(..) => { + let repr = StructUnionRepr::from_attrs(&ctx.ast.attrs)?; + if repr.is_c() { + Some(repr) + } else { + None + } + } + Data::Enum(..) | Data::Union(..) => None, + }; + + let fields = ctx.ast.data.fields(); + + let (self_bounds, inner_extras, outer_extras) = c_struct_repr + .as_ref() + .and_then(|repr| { + derive_known_layout_for_repr_c_struct(ctx, repr, &fields) + }) + .unwrap_or_else(|| { + let zerocopy_crate = &ctx.zerocopy_crate; + let core = ctx.core_path(); + + // For enums, unions, and non-`repr(C)` structs, we require that + // `Self` is sized, and as a result don't need to reason about the + // internals of the type. + ( + SelfBounds::SIZED, + quote!( + type PointerMetadata = (); + type MaybeUninit = + #core::mem::MaybeUninit<Self>; + + // SAFETY: `LAYOUT` is guaranteed to accurately describe the + // layout of `Self`, because that is the documented safety + // contract of `DstLayout::for_type`. + const LAYOUT: #zerocopy_crate::DstLayout = #zerocopy_crate::DstLayout::for_type::<Self>(); + + // SAFETY: `.cast` preserves address and provenance. + // + // FIXME(#429): Add documentation to `.cast` that promises that + // it preserves provenance. + #[inline(always)] + fn raw_from_ptr_len(bytes: #core::ptr::NonNull<u8>, _meta: ()) -> #core::ptr::NonNull<Self> { + bytes.cast::<Self>() + } + + #[inline(always)] + fn pointer_to_metadata(_ptr: *mut Self) -> () {} + ), + None, + ) + }); + Ok(match &ctx.ast.data { + Data::Struct(strct) => { + let require_trait_bound_on_field_types = + if matches!(self_bounds, SelfBounds::All(&[Trait::Sized])) { + FieldBounds::None + } else { + FieldBounds::TRAILING_SELF + }; + + // A bound on the trailing field is required, since structs are + // unsized if their trailing field is unsized. Reflecting the layout + // of an usized trailing field requires that the field is + // `KnownLayout`. + ImplBlockBuilder::new( + ctx, + strct, + Trait::KnownLayout, + require_trait_bound_on_field_types, + ) + .self_type_trait_bounds(self_bounds) + .inner_extras(inner_extras) + .outer_extras(outer_extras) + .build() + } + Data::Enum(enm) => { + // A bound on the trailing field is not required, since enums cannot + // currently be unsized. + ImplBlockBuilder::new(ctx, enm, Trait::KnownLayout, FieldBounds::None) + .self_type_trait_bounds(SelfBounds::SIZED) + .inner_extras(inner_extras) + .outer_extras(outer_extras) + .build() + } + Data::Union(unn) => { + // A bound on the trailing field is not required, since unions + // cannot currently be unsized. + ImplBlockBuilder::new(ctx, unn, Trait::KnownLayout, FieldBounds::None) + .self_type_trait_bounds(SelfBounds::SIZED) + .inner_extras(inner_extras) + .outer_extras(outer_extras) + .build() + } + }) +} diff --git a/rust/zerocopy-derive/derive/mod.rs b/rust/zerocopy-derive/derive/mod.rs new file mode 100644 index 000000000000..665ba7da55a8 --- /dev/null +++ b/rust/zerocopy-derive/derive/mod.rs @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +pub mod from_bytes; +pub mod into_bytes; +pub mod known_layout; +pub mod try_from_bytes; +pub mod unaligned; + +use proc_macro2::{Span, TokenStream}; +use quote::quote; +use syn::{Data, Error}; + +use crate::{ + repr::StructUnionRepr, + util::{Ctx, DataExt, FieldBounds, ImplBlockBuilder, Trait}, +}; + +pub(crate) fn derive_immutable(ctx: &Ctx, _top_level: Trait) -> TokenStream { + match &ctx.ast.data { + Data::Struct(strct) => { + ImplBlockBuilder::new(ctx, strct, Trait::Immutable, FieldBounds::ALL_SELF).build() + } + Data::Enum(enm) => { + ImplBlockBuilder::new(ctx, enm, Trait::Immutable, FieldBounds::ALL_SELF).build() + } + Data::Union(unn) => { + ImplBlockBuilder::new(ctx, unn, Trait::Immutable, FieldBounds::ALL_SELF).build() + } + } +} + +pub(crate) fn derive_hash(ctx: &Ctx, _top_level: Trait) -> Result<TokenStream, Error> { + // This doesn't delegate to `impl_block` because `impl_block` assumes it is + // deriving a `zerocopy`-defined trait, and these trait impls share a common + // shape that `Hash` does not. In particular, `zerocopy` traits contain a + // method that only `zerocopy_derive` macros are supposed to implement, and + // `impl_block` generating this trait method is incompatible with `Hash`. + let type_ident = &ctx.ast.ident; + let (impl_generics, ty_generics, where_clause) = ctx.ast.generics.split_for_impl(); + let where_predicates = where_clause.map(|clause| &clause.predicates); + let zerocopy_crate = &ctx.zerocopy_crate; + let core = ctx.core_path(); + Ok(quote! { + impl #impl_generics #core::hash::Hash for #type_ident #ty_generics + where + Self: #zerocopy_crate::IntoBytes + #zerocopy_crate::Immutable, + #where_predicates + { + fn hash<H: #core::hash::Hasher>(&self, state: &mut H) { + #core::hash::Hasher::write(state, #zerocopy_crate::IntoBytes::as_bytes(self)) + } + + fn hash_slice<H: #core::hash::Hasher>(data: &[Self], state: &mut H) { + #core::hash::Hasher::write(state, #zerocopy_crate::IntoBytes::as_bytes(data)) + } + } + }) +} + +pub(crate) fn derive_eq(ctx: &Ctx, _top_level: Trait) -> Result<TokenStream, Error> { + // This doesn't delegate to `impl_block` because `impl_block` assumes it is + // deriving a `zerocopy`-defined trait, and these trait impls share a common + // shape that `Eq` does not. In particular, `zerocopy` traits contain a + // method that only `zerocopy_derive` macros are supposed to implement, and + // `impl_block` generating this trait method is incompatible with `Eq`. + let type_ident = &ctx.ast.ident; + let (impl_generics, ty_generics, where_clause) = ctx.ast.generics.split_for_impl(); + let where_predicates = where_clause.map(|clause| &clause.predicates); + let zerocopy_crate = &ctx.zerocopy_crate; + let core = ctx.core_path(); + Ok(quote! { + impl #impl_generics #core::cmp::PartialEq for #type_ident #ty_generics + where + Self: #zerocopy_crate::IntoBytes + #zerocopy_crate::Immutable, + #where_predicates + { + fn eq(&self, other: &Self) -> bool { + #core::cmp::PartialEq::eq( + #zerocopy_crate::IntoBytes::as_bytes(self), + #zerocopy_crate::IntoBytes::as_bytes(other), + ) + } + } + + impl #impl_generics #core::cmp::Eq for #type_ident #ty_generics + where + Self: #zerocopy_crate::IntoBytes + #zerocopy_crate::Immutable, + #where_predicates + { + } + }) +} + +pub(crate) fn derive_split_at(ctx: &Ctx, _top_level: Trait) -> Result<TokenStream, Error> { + let repr = StructUnionRepr::from_attrs(&ctx.ast.attrs)?; + + match &ctx.ast.data { + Data::Struct(_) => {} + Data::Enum(_) | Data::Union(_) => { + return Err(Error::new(Span::call_site(), "can only be applied to structs")); + } + }; + + if repr.get_packed().is_some() { + return Err(Error::new(Span::call_site(), "must not have #[repr(packed)] attribute")); + } + + if !(repr.is_c() || repr.is_transparent()) { + return Err(Error::new( + Span::call_site(), + "must have #[repr(C)] or #[repr(transparent)] in order to guarantee this type's layout is splitable", + )); + } + + let fields = ctx.ast.data.fields(); + let trailing_field = if let Some(((_, _, trailing_field), _)) = fields.split_last() { + trailing_field + } else { + return Err(Error::new(Span::call_site(), "must at least one field")); + }; + + let zerocopy_crate = &ctx.zerocopy_crate; + // SAFETY: `#ty`, per the above checks, is `repr(C)` or `repr(transparent)` + // and is not packed; its trailing field is guaranteed to be well-aligned + // for its type. By invariant on `FieldBounds::TRAILING_SELF`, the trailing + // slice of the trailing field is also well-aligned for its type. + Ok(ImplBlockBuilder::new(ctx, &ctx.ast.data, Trait::SplitAt, FieldBounds::TRAILING_SELF) + .inner_extras(quote! { + type Elem = <#trailing_field as #zerocopy_crate::SplitAt>::Elem; + }) + .build()) +} diff --git a/rust/zerocopy-derive/derive/try_from_bytes.rs b/rust/zerocopy-derive/derive/try_from_bytes.rs new file mode 100644 index 000000000000..a3e4a75631a5 --- /dev/null +++ b/rust/zerocopy-derive/derive/try_from_bytes.rs @@ -0,0 +1,765 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +use proc_macro2::TokenStream; +use quote::quote; +use syn::{ + parse_quote, spanned::Spanned as _, Data, DataEnum, DataStruct, DataUnion, DeriveInput, Error, + Expr, Fields, Ident, Index, Type, +}; + +use crate::{ + repr::{EnumRepr, StructUnionRepr}, + util::{ + const_block, enum_size_from_repr, generate_tag_enum, Ctx, DataExt, FieldBounds, + ImplBlockBuilder, Trait, TraitBound, + }, +}; +fn tag_ident(variant_ident: &Ident) -> Ident { + ident!(("___ZEROCOPY_TAG_{}", variant_ident), variant_ident.span()) +} + +/// Generates a constant for the tag associated with each variant of the enum. +/// When we match on the enum's tag, each arm matches one of these constants. We +/// have to use constants here because: +/// +/// - The type that we're matching on is not the type of the tag, it's an +/// integer of the same size as the tag type and with the same bit patterns. +/// - We can't read the enum tag as an enum because the bytes may not represent +/// a valid variant. +/// - Patterns do not currently support const expressions, so we have to assign +/// these constants to names rather than use them inline in the `match` +/// statement. +fn generate_tag_consts(data: &DataEnum) -> TokenStream { + let tags = data.variants.iter().map(|v| { + let variant_ident = &v.ident; + let tag_ident = tag_ident(variant_ident); + + quote! { + // This casts the enum variant to its discriminant, and then + // converts the discriminant to the target integral type via a + // numeric cast [1]. + // + // Because these are the same size, this is defined to be a no-op + // and therefore is a lossless conversion [2]. + // + // [1] Per https://doc.rust-lang.org/1.81.0/reference/expressions/operator-expr.html#enum-cast: + // + // Casts an enum to its discriminant. + // + // [2] Per https://doc.rust-lang.org/1.81.0/reference/expressions/operator-expr.html#numeric-cast: + // + // Casting between two integers of the same size (e.g. i32 -> u32) + // is a no-op. + const #tag_ident: ___ZerocopyTagPrimitive = + ___ZerocopyTag::#variant_ident as ___ZerocopyTagPrimitive; + } + }); + + quote! { + #(#tags)* + } +} + +fn variant_struct_ident(variant_ident: &Ident) -> Ident { + ident!(("___ZerocopyVariantStruct_{}", variant_ident), variant_ident.span()) +} + +/// Generates variant structs for the given enum variant. +/// +/// These are structs associated with each variant of an enum. They are +/// `repr(C)` tuple structs with the same fields as the variant after a +/// `MaybeUninit<___ZerocopyInnerTag>`. +/// +/// In order to unify the generated types for `repr(C)` and `repr(int)` enums, +/// we use a "fused" representation with fields for both an inner tag and an +/// outer tag. Depending on the repr, we will set one of these tags to the tag +/// type and the other to `()`. This lets us generate the same code but put the +/// tags in different locations. +fn generate_variant_structs(ctx: &Ctx, data: &DataEnum) -> TokenStream { + let (impl_generics, ty_generics, where_clause) = ctx.ast.generics.split_for_impl(); + + let enum_name = &ctx.ast.ident; + + // All variant structs have a `PhantomData<MyEnum<...>>` field because we + // don't know which generic parameters each variant will use, and unused + // generic parameters are a compile error. + let core = ctx.core_path(); + let phantom_ty = quote! { + #core::marker::PhantomData<#enum_name #ty_generics> + }; + + let variant_structs = data.variants.iter().filter_map(|variant| { + // We don't generate variant structs for unit variants because we only + // need to check the tag. This helps cut down our generated code a bit. + if matches!(variant.fields, Fields::Unit) { + return None; + } + + let variant_struct_ident = variant_struct_ident(&variant.ident); + let field_types = variant.fields.iter().map(|f| &f.ty); + + let variant_struct = parse_quote! { + #[repr(C)] + struct #variant_struct_ident #impl_generics ( + #core::mem::MaybeUninit<___ZerocopyInnerTag>, + #(#field_types,)* + #phantom_ty, + ) #where_clause; + }; + + // We do this rather than emitting `#[derive(::zerocopy::TryFromBytes)]` + // because that is not hygienic, and this is also more performant. + let try_from_bytes_impl = + derive_try_from_bytes(&ctx.with_input(&variant_struct), Trait::TryFromBytes) + .expect("derive_try_from_bytes should not fail on synthesized type"); + + Some(quote! { + #variant_struct + #try_from_bytes_impl + }) + }); + + quote! { + #(#variant_structs)* + } +} + +fn variants_union_field_ident(ident: &Ident) -> Ident { + // Field names are prefixed with `__field_` to prevent name collision + // with the `__nonempty` field. + ident!(("__field_{}", ident), ident.span()) +} + +fn generate_variants_union(ctx: &Ctx, data: &DataEnum) -> TokenStream { + let generics = &ctx.ast.generics; + let (_, ty_generics, _) = generics.split_for_impl(); + + let fields = data.variants.iter().filter_map(|variant| { + // We don't generate variant structs for unit variants because we only + // need to check the tag. This helps cut down our generated code a bit. + if matches!(variant.fields, Fields::Unit) { + return None; + } + + let field_name = variants_union_field_ident(&variant.ident); + let variant_struct_ident = variant_struct_ident(&variant.ident); + + let core = ctx.core_path(); + Some(quote! { + #field_name: #core::mem::ManuallyDrop<#variant_struct_ident #ty_generics>, + }) + }); + + let variants_union = parse_quote! { + #[repr(C)] + union ___ZerocopyVariants #generics { + #(#fields)* + // Enums can have variants with no fields, but unions must + // have at least one field. So we just add a trailing unit + // to ensure that this union always has at least one field. + // Because this union is `repr(C)`, this unit type does not + // affect the layout. + __nonempty: (), + } + }; + + let has_field = + derive_has_field_struct_union(&ctx.with_input(&variants_union), &variants_union.data); + + quote! { + #variants_union + #has_field + } +} + +/// Generates an implementation of `is_bit_valid` for an arbitrary enum. +/// +/// The general process is: +/// +/// 1. Generate a tag enum. This is an enum with the same repr, variants, and +/// corresponding discriminants as the original enum, but without any fields +/// on the variants. This gives us access to an enum where the variants have +/// the same discriminants as the one we're writing `is_bit_valid` for. +/// 2. Make constants from the variants of the tag enum. We need these because +/// we can't put const exprs in match arms. +/// 3. Generate variant structs. These are structs which have the same fields as +/// each variant of the enum, and are `#[repr(C)]` with an optional "inner +/// tag". +/// 4. Generate a variants union, with one field for each variant struct type. +/// 5. And finally, our raw enum is a `#[repr(C)]` struct of an "outer tag" and +/// the variants union. +/// +/// See these reference links for fully-worked example decompositions. +/// +/// - `repr(C)`: <https://doc.rust-lang.org/reference/type-layout.html#reprc-enums-with-fields> +/// - `repr(int)`: <https://doc.rust-lang.org/reference/type-layout.html#primitive-representation-of-enums-with-fields> +/// - `repr(C, int)`: <https://doc.rust-lang.org/reference/type-layout.html#combining-primitive-representations-of-enums-with-fields-and-reprc> +pub(crate) fn derive_is_bit_valid( + ctx: &Ctx, + data: &DataEnum, + repr: &EnumRepr, +) -> Result<TokenStream, Error> { + let trait_path = Trait::TryFromBytes.crate_path(ctx); + let tag_enum = generate_tag_enum(ctx, repr, data); + let tag_consts = generate_tag_consts(data); + + let (outer_tag_type, inner_tag_type) = if repr.is_c() { + (quote! { ___ZerocopyTag }, quote! { () }) + } else if repr.is_primitive() { + (quote! { () }, quote! { ___ZerocopyTag }) + } else { + return Err(Error::new( + ctx.ast.span(), + "must have #[repr(C)] or #[repr(Int)] attribute in order to guarantee this type's memory layout", + )); + }; + + let variant_structs = generate_variant_structs(ctx, data); + let variants_union = generate_variants_union(ctx, data); + + let (impl_generics, ty_generics, where_clause) = ctx.ast.generics.split_for_impl(); + + let zerocopy_crate = &ctx.zerocopy_crate; + let has_tag = ImplBlockBuilder::new(ctx, data, Trait::HasTag, FieldBounds::None) + .inner_extras(quote! { + type Tag = ___ZerocopyTag; + type ProjectToTag = #zerocopy_crate::pointer::cast::CastSized; + }) + .build(); + let has_fields = data.variants().into_iter().flat_map(|(variant, fields)| { + let variant_ident = &variant.unwrap().ident; + let variants_union_field_ident = variants_union_field_ident(variant_ident); + let field: Box<syn::Type> = parse_quote!(()); + fields.into_iter().enumerate().map(move |(idx, (vis, ident, ty))| { + // Rust does not presently support explicit visibility modifiers on + // enum fields, but we guard against the possibility to ensure this + // derive remains sound. + assert!(matches!(vis, syn::Visibility::Inherited)); + let variant_struct_field_index = Index::from(idx + 1); + let (_, ty_generics, _) = ctx.ast.generics.split_for_impl(); + let has_field_trait = Trait::HasField { + variant_id: parse_quote!({ #zerocopy_crate::ident_id!(#variant_ident) }), + // Since Rust does not presently support explicit visibility + // modifiers on enum fields, any public type is suitable here; + // we use `()`. + field: field.clone(), + field_id: parse_quote!({ #zerocopy_crate::ident_id!(#ident) }), + }; + let has_field_path = has_field_trait.crate_path(ctx); + let has_field = ImplBlockBuilder::new( + ctx, + data, + has_field_trait, + FieldBounds::None, + ) + .inner_extras(quote! { + type Type = #ty; + + #[inline(always)] + fn project(slf: #zerocopy_crate::pointer::PtrInner<'_, Self>) -> *mut <Self as #has_field_path>::Type { + use #zerocopy_crate::pointer::cast::{CastSized, Projection}; + + slf.project::<___ZerocopyRawEnum #ty_generics, CastSized>() + .project::<_, Projection<_, { #zerocopy_crate::STRUCT_VARIANT_ID }, { #zerocopy_crate::ident_id!(variants) }>>() + .project::<_, Projection<_, { #zerocopy_crate::REPR_C_UNION_VARIANT_ID }, { #zerocopy_crate::ident_id!(#variants_union_field_ident) }>>() + .project::<_, Projection<_, { #zerocopy_crate::STRUCT_VARIANT_ID }, { #zerocopy_crate::ident_id!(value) }>>() + .project::<_, Projection<_, { #zerocopy_crate::STRUCT_VARIANT_ID }, { #zerocopy_crate::ident_id!(#variant_struct_field_index) }>>() + .as_ptr() + } + }) + .build(); + + let project = ImplBlockBuilder::new( + ctx, + data, + Trait::ProjectField { + variant_id: parse_quote!({ #zerocopy_crate::ident_id!(#variant_ident) }), + // Since Rust does not presently support explicit visibility + // modifiers on enum fields, any public type is suitable + // here; we use `()`. + field: field.clone(), + field_id: parse_quote!({ #zerocopy_crate::ident_id!(#ident) }), + invariants: parse_quote!((Aliasing, Alignment, #zerocopy_crate::invariant::Initialized)), + }, + FieldBounds::None, + ) + .param_extras(vec![ + parse_quote!(Aliasing: #zerocopy_crate::invariant::Aliasing), + parse_quote!(Alignment: #zerocopy_crate::invariant::Alignment), + ]) + .inner_extras(quote! { + type Error = #zerocopy_crate::util::macro_util::core_reexport::convert::Infallible; + type Invariants = (Aliasing, Alignment, #zerocopy_crate::invariant::Initialized); + }) + .build(); + + quote! { + #has_field + #project + } + }) + }); + + let core = ctx.core_path(); + let match_arms = data.variants.iter().map(|variant| { + let tag_ident = tag_ident(&variant.ident); + let variant_struct_ident = variant_struct_ident(&variant.ident); + let variants_union_field_ident = variants_union_field_ident(&variant.ident); + + if matches!(variant.fields, Fields::Unit) { + // Unit variants don't need any further validation beyond checking + // the tag. + quote! { + #tag_ident => true + } + } else { + quote! { + #tag_ident => { + // SAFETY: Since we know that the tag is `#tag_ident`, we + // know that no other `&`s exist which refer to this enum + // as any other variant. + let variant_md = variants.cast::< + _, + #zerocopy_crate::pointer::cast::Projection< + // #zerocopy_crate::ReadOnly<_>, + _, + { #zerocopy_crate::REPR_C_UNION_VARIANT_ID }, + { #zerocopy_crate::ident_id!(#variants_union_field_ident) } + >, + _ + >(); + let variant = variant_md.cast::< + #zerocopy_crate::ReadOnly<#variant_struct_ident #ty_generics>, + #zerocopy_crate::pointer::cast::CastSized, + (#zerocopy_crate::pointer::BecauseRead, _) + >(); + < + #variant_struct_ident #ty_generics as #trait_path + >::is_bit_valid(variant) + } + } + } + }); + + let generics = &ctx.ast.generics; + let raw_enum: DeriveInput = parse_quote! { + #[repr(C)] + struct ___ZerocopyRawEnum #generics { + tag: ___ZerocopyOuterTag, + variants: ___ZerocopyVariants #ty_generics, + } + }; + + let self_ident = &ctx.ast.ident; + let invariants_eq_impl = quote! { + // SAFETY: `___ZerocopyRawEnum` is designed to have the same layout, + // validity, and invariants as `Self`. + unsafe impl #impl_generics #zerocopy_crate::pointer::InvariantsEq<___ZerocopyRawEnum #ty_generics> for #self_ident #ty_generics #where_clause {} + }; + + let raw_enum_projections = + derive_has_field_struct_union(&ctx.with_input(&raw_enum), &raw_enum.data); + + let raw_enum = quote! { + #raw_enum + #invariants_eq_impl + #raw_enum_projections + }; + + Ok(quote! { + // SAFETY: We use `is_bit_valid` to validate that the bit pattern of the + // enum's tag corresponds to one of the enum's discriminants. Then, we + // check the bit validity of each field of the corresponding variant. + // Thus, this is a sound implementation of `is_bit_valid`. + #[inline] + fn is_bit_valid<___ZcAlignment>( + mut candidate: #zerocopy_crate::Maybe<'_, Self, ___ZcAlignment>, + ) -> #core::primitive::bool + where + ___ZcAlignment: #zerocopy_crate::invariant::Alignment, + { + #tag_enum + + type ___ZerocopyTagPrimitive = #zerocopy_crate::util::macro_util::SizeToTag< + { #core::mem::size_of::<___ZerocopyTag>() }, + >; + + #tag_consts + + type ___ZerocopyOuterTag = #outer_tag_type; + type ___ZerocopyInnerTag = #inner_tag_type; + + #variant_structs + + #variants_union + + #raw_enum + + #has_tag + + #(#has_fields)* + + let tag = { + // SAFETY: + // - The provided cast addresses a subset of the bytes addressed + // by `candidate` because it addresses the starting tag of the + // enum. + // - Because the pointer is cast from `candidate`, it has the + // same provenance as it. + // - There are no `UnsafeCell`s in the tag because it is a + // primitive integer. + // - `tag_ptr` is casted from `candidate`, whose referent is + // `Initialized`. Since we have not written uninitialized + // bytes into the referent, `tag_ptr` is also `Initialized`. + // + // FIXME(#2874): Revise this to a `cast` once `candidate` + // references a `ReadOnly<Self>`. + let tag_ptr = unsafe { + candidate.reborrow().project_transmute_unchecked::< + _, + #zerocopy_crate::invariant::Initialized, + #zerocopy_crate::pointer::cast::CastSized + >() + }; + tag_ptr.recall_validity::<_, (_, (_, _))>().read::<#zerocopy_crate::BecauseImmutable>() + }; + + let mut raw_enum = candidate.cast::< + #zerocopy_crate::ReadOnly<___ZerocopyRawEnum #ty_generics>, + #zerocopy_crate::pointer::cast::CastSized, + (#zerocopy_crate::pointer::BecauseRead, _) + >(); + + let variants = #zerocopy_crate::into_inner!(raw_enum.project::< + _, + { #zerocopy_crate::STRUCT_VARIANT_ID }, + { #zerocopy_crate::ident_id!(variants) } + >()); + + match tag { + #(#match_arms,)* + _ => false, + } + } + }) +} +pub(crate) fn derive_try_from_bytes(ctx: &Ctx, top_level: Trait) -> Result<TokenStream, Error> { + match &ctx.ast.data { + Data::Struct(strct) => derive_try_from_bytes_struct(ctx, strct, top_level), + Data::Enum(enm) => derive_try_from_bytes_enum(ctx, enm, top_level), + Data::Union(unn) => Ok(derive_try_from_bytes_union(ctx, unn, top_level)), + } +} +fn derive_has_field_struct_union(ctx: &Ctx, data: &dyn DataExt) -> TokenStream { + let fields = ctx.ast.data.fields(); + if fields.is_empty() { + return quote! {}; + } + + let field_tokens = fields.iter().map(|(vis, ident, _)| { + let ident = ident!(("__z{}", ident), ident.span()); + quote!( + #vis enum #ident {} + ) + }); + + let zerocopy_crate = &ctx.zerocopy_crate; + let variant_id: Box<Expr> = match &ctx.ast.data { + Data::Struct(_) => parse_quote!({ #zerocopy_crate::STRUCT_VARIANT_ID }), + Data::Union(_) => { + let is_repr_c = StructUnionRepr::from_attrs(&ctx.ast.attrs) + .map(|repr| repr.is_c()) + .unwrap_or(false); + if is_repr_c { + parse_quote!({ #zerocopy_crate::REPR_C_UNION_VARIANT_ID }) + } else { + parse_quote!({ #zerocopy_crate::UNION_VARIANT_ID }) + } + } + _ => unreachable!(), + }; + + let core = ctx.core_path(); + let has_tag = ImplBlockBuilder::new(ctx, data, Trait::HasTag, FieldBounds::None) + .inner_extras(quote! { + type Tag = (); + type ProjectToTag = #zerocopy_crate::pointer::cast::CastToUnit; + }) + .build(); + let has_fields = fields.iter().map(move |(_, ident, ty)| { + let field_token = ident!(("__z{}", ident), ident.span()); + let field: Box<Type> = parse_quote!(#field_token); + let field_id: Box<Expr> = parse_quote!({ #zerocopy_crate::ident_id!(#ident) }); + let has_field_trait = Trait::HasField { + variant_id: variant_id.clone(), + field: field.clone(), + field_id: field_id.clone(), + }; + let has_field_path = has_field_trait.crate_path(ctx); + ImplBlockBuilder::new( + ctx, + data, + has_field_trait, + FieldBounds::None, + ) + .inner_extras(quote! { + type Type = #ty; + + #[inline(always)] + fn project(slf: #zerocopy_crate::pointer::PtrInner<'_, Self>) -> *mut <Self as #has_field_path>::Type { + let slf = slf.as_ptr(); + // SAFETY: By invariant on `PtrInner`, `slf` is a non-null + // pointer whose referent is zero-sized or lives in a valid + // allocation. Since `#ident` is a struct or union field of + // `Self`, this projection preserves or shrinks the referent + // size, and so the resulting referent also fits in the same + // allocation. + unsafe { #core::ptr::addr_of_mut!((*slf).#ident) } + } + }).outer_extras(if matches!(&ctx.ast.data, Data::Struct(..)) { + let fields_preserve_alignment = StructUnionRepr::from_attrs(&ctx.ast.attrs) + .map(|repr| repr.get_packed().is_none()) + .unwrap(); + let alignment = if fields_preserve_alignment { + quote! { Alignment } + } else { + quote! { #zerocopy_crate::invariant::Unaligned } + }; + // SAFETY: See comments on items. + ImplBlockBuilder::new( + ctx, + data, + Trait::ProjectField { + variant_id: variant_id.clone(), + field: field.clone(), + field_id: field_id.clone(), + invariants: parse_quote!((Aliasing, Alignment, #zerocopy_crate::invariant::Initialized)), + }, + FieldBounds::None, + ) + .param_extras(vec![ + parse_quote!(Aliasing: #zerocopy_crate::invariant::Aliasing), + parse_quote!(Alignment: #zerocopy_crate::invariant::Alignment), + ]) + .inner_extras(quote! { + // SAFETY: Projection into structs is always infallible. + type Error = #zerocopy_crate::util::macro_util::core_reexport::convert::Infallible; + // SAFETY: The alignment of the projected `Ptr` is `Unaligned` + // if the structure is packed; otherwise inherited from the + // outer `Ptr`. If the validity of the outer pointer is + // `Initialized`, so too is the validity of its fields. + type Invariants = (Aliasing, #alignment, #zerocopy_crate::invariant::Initialized); + }) + .build() + } else { + quote! {} + }) + .build() + }); + + const_block(field_tokens.into_iter().chain(Some(has_tag)).chain(has_fields).map(Some)) +} +fn derive_try_from_bytes_struct( + ctx: &Ctx, + strct: &DataStruct, + top_level: Trait, +) -> Result<TokenStream, Error> { + let extras = try_gen_trivial_is_bit_valid(ctx, top_level).unwrap_or_else(|| { + let zerocopy_crate = &ctx.zerocopy_crate; + let fields = strct.fields(); + let field_names = fields.iter().map(|(_vis, name, _ty)| name); + let field_tys = fields.iter().map(|(_vis, _name, ty)| ty); + let core = ctx.core_path(); + quote!( + // SAFETY: We use `is_bit_valid` to validate that each field is + // bit-valid, and only return `true` if all of them are. The bit + // validity of a struct is just the composition of the bit + // validities of its fields, so this is a sound implementation + // of `is_bit_valid`. + #[inline] + fn is_bit_valid<___ZcAlignment>( + mut candidate: #zerocopy_crate::Maybe<'_, Self, ___ZcAlignment>, + ) -> #core::primitive::bool + where + ___ZcAlignment: #zerocopy_crate::invariant::Alignment, + { + true #(&& { + let field_candidate = #zerocopy_crate::into_inner!(candidate.reborrow().project::< + _, + { #zerocopy_crate::STRUCT_VARIANT_ID }, + { #zerocopy_crate::ident_id!(#field_names) } + >()); + <#field_tys as #zerocopy_crate::TryFromBytes>::is_bit_valid(field_candidate) + })* + } + ) + }); + Ok(ImplBlockBuilder::new(ctx, strct, Trait::TryFromBytes, FieldBounds::ALL_SELF) + .inner_extras(extras) + .outer_extras(derive_has_field_struct_union(ctx, strct)) + .build()) +} +fn derive_try_from_bytes_union(ctx: &Ctx, unn: &DataUnion, top_level: Trait) -> TokenStream { + let field_type_trait_bounds = FieldBounds::All(&[TraitBound::Slf]); + + let zerocopy_crate = &ctx.zerocopy_crate; + let variant_id: Box<Expr> = { + let is_repr_c = + StructUnionRepr::from_attrs(&ctx.ast.attrs).map(|repr| repr.is_c()).unwrap_or(false); + if is_repr_c { + parse_quote!({ #zerocopy_crate::REPR_C_UNION_VARIANT_ID }) + } else { + parse_quote!({ #zerocopy_crate::UNION_VARIANT_ID }) + } + }; + + let extras = try_gen_trivial_is_bit_valid(ctx, top_level).unwrap_or_else(|| { + let fields = unn.fields(); + let field_names = fields.iter().map(|(_vis, name, _ty)| name); + let field_tys = fields.iter().map(|(_vis, _name, ty)| ty); + let core = ctx.core_path(); + quote!( + // SAFETY: We use `is_bit_valid` to validate that any field is + // bit-valid; we only return `true` if at least one of them is. + // The bit validity of a union is not yet well defined in Rust, + // but it is guaranteed to be no more strict than this + // definition. See #696 for a more in-depth discussion. + #[inline] + fn is_bit_valid<___ZcAlignment>( + mut candidate: #zerocopy_crate::Maybe<'_, Self, ___ZcAlignment>, + ) -> #core::primitive::bool + where + ___ZcAlignment: #zerocopy_crate::invariant::Alignment, + { + false #(|| { + // SAFETY: + // - Since `ReadOnly<Self>: Immutable` unconditionally, + // neither `*slf` nor the returned pointer's referent + // permit interior mutation. + // - Both source and destination validity are + // `Initialized`, which is always a sound + // transmutation. + let field_candidate = unsafe { + candidate.reborrow().project_transmute_unchecked::< + _, + _, + #zerocopy_crate::pointer::cast::Projection< + _, + #variant_id, + { #zerocopy_crate::ident_id!(#field_names) } + > + >() + }; + + <#field_tys as #zerocopy_crate::TryFromBytes>::is_bit_valid(field_candidate) + })* + } + ) + }); + ImplBlockBuilder::new(ctx, unn, Trait::TryFromBytes, field_type_trait_bounds) + .inner_extras(extras) + .outer_extras(derive_has_field_struct_union(ctx, unn)) + .build() +} +fn derive_try_from_bytes_enum( + ctx: &Ctx, + enm: &DataEnum, + top_level: Trait, +) -> Result<TokenStream, Error> { + let repr = EnumRepr::from_attrs(&ctx.ast.attrs)?; + + // If an enum has no fields, it has a well-defined integer representation, + // and every possible bit pattern corresponds to a valid discriminant tag, + // then it *could* be `FromBytes` (even if the user hasn't derived + // `FromBytes`). This holds if, for `repr(uN)` or `repr(iN)`, there are 2^N + // variants. + let could_be_from_bytes = enum_size_from_repr(&repr) + .map(|size| enm.fields().is_empty() && enm.variants.len() == 1usize << size) + .unwrap_or(false); + + let trivial_is_bit_valid = try_gen_trivial_is_bit_valid(ctx, top_level); + let extra = match (trivial_is_bit_valid, could_be_from_bytes) { + (Some(is_bit_valid), _) => is_bit_valid, + // SAFETY: It would be sound for the enum to implement `FromBytes`, as + // required by `gen_trivial_is_bit_valid_unchecked`. + (None, true) => unsafe { gen_trivial_is_bit_valid_unchecked(ctx) }, + (None, false) => match derive_is_bit_valid(ctx, enm, &repr) { + Ok(extra) => extra, + Err(_) if ctx.skip_on_error => return Ok(TokenStream::new()), + Err(e) => return Err(e), + }, + }; + + Ok(ImplBlockBuilder::new(ctx, enm, Trait::TryFromBytes, FieldBounds::ALL_SELF) + .inner_extras(extra) + .build()) +} +fn try_gen_trivial_is_bit_valid(ctx: &Ctx, top_level: Trait) -> Option<proc_macro2::TokenStream> { + // If the top-level trait is `FromBytes` and `Self` has no type parameters, + // then the `FromBytes` derive will fail compilation if `Self` is not + // actually soundly `FromBytes`, and so we can rely on that for our + // `is_bit_valid` impl. It's plausible that we could make changes - or Rust + // could make changes (such as the "trivial bounds" language feature) - that + // make this no longer true. To hedge against these, we include an explicit + // `Self: FromBytes` check in the generated `is_bit_valid`, which is + // bulletproof. + // + // If `ctx.skip_on_error` is true, we can't rely on the `FromBytes` derive + // to fail compilation if `Self` is not actually soundly `FromBytes`. + if matches!(top_level, Trait::FromBytes) + && ctx.ast.generics.params.is_empty() + && !ctx.skip_on_error + { + let zerocopy_crate = &ctx.zerocopy_crate; + let core = ctx.core_path(); + Some(quote!( + // SAFETY: See inline. + #[inline(always)] + fn is_bit_valid<___ZcAlignment>( + _candidate: #zerocopy_crate::Maybe<'_, Self, ___ZcAlignment>, + ) -> #core::primitive::bool + where + ___ZcAlignment: #zerocopy_crate::invariant::Alignment, + { + if false { + fn assert_is_from_bytes<T>() + where + T: #zerocopy_crate::FromBytes, + T: ?#core::marker::Sized, + { + } + + assert_is_from_bytes::<Self>(); + } + + // SAFETY: The preceding code only compiles if `Self: + // FromBytes`. Thus, this code only compiles if all initialized + // byte sequences represent valid instances of `Self`. + true + } + )) + } else { + None + } +} + +/// # Safety +/// +/// All initialized bit patterns must be valid for `Self`. +unsafe fn gen_trivial_is_bit_valid_unchecked(ctx: &Ctx) -> proc_macro2::TokenStream { + let zerocopy_crate = &ctx.zerocopy_crate; + let core = ctx.core_path(); + quote!( + // SAFETY: The caller of `gen_trivial_is_bit_valid_unchecked` has + // promised that all initialized bit patterns are valid for `Self`. + #[inline(always)] + fn is_bit_valid<___ZcAlignment>( + _candidate: #zerocopy_crate::Maybe<'_, Self, ___ZcAlignment>, + ) -> #core::primitive::bool + where + ___ZcAlignment: #zerocopy_crate::invariant::Alignment, + { + true + } + ) +} diff --git a/rust/zerocopy-derive/derive/unaligned.rs b/rust/zerocopy-derive/derive/unaligned.rs new file mode 100644 index 000000000000..d6dea0a11f1e --- /dev/null +++ b/rust/zerocopy-derive/derive/unaligned.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +use proc_macro2::{Span, TokenStream}; +use syn::{Data, DataEnum, DataStruct, DataUnion, Error}; + +use crate::{ + repr::{EnumRepr, StructUnionRepr}, + util::{Ctx, FieldBounds, ImplBlockBuilder, Trait}, +}; + +pub(crate) fn derive_unaligned(ctx: &Ctx, _top_level: Trait) -> Result<TokenStream, Error> { + match &ctx.ast.data { + Data::Struct(strct) => derive_unaligned_struct(ctx, strct), + Data::Enum(enm) => derive_unaligned_enum(ctx, enm), + Data::Union(unn) => derive_unaligned_union(ctx, unn), + } +} + +/// A struct is `Unaligned` if: +/// - `repr(align)` is no more than 1 and either +/// - `repr(C)` or `repr(transparent)` and +/// - all fields `Unaligned` +/// - `repr(packed)` +fn derive_unaligned_struct(ctx: &Ctx, strct: &DataStruct) -> Result<TokenStream, Error> { + let repr = StructUnionRepr::from_attrs(&ctx.ast.attrs)?; + repr.unaligned_validate_no_align_gt_1()?; + + let field_bounds = if repr.is_packed_1() { + FieldBounds::None + } else if repr.is_c() || repr.is_transparent() { + FieldBounds::ALL_SELF + } else { + return ctx.error_or_skip(Error::new( + Span::call_site(), + "must have #[repr(C)], #[repr(transparent)], or #[repr(packed)] attribute in order to guarantee this type's alignment", + )); + }; + + Ok(ImplBlockBuilder::new(ctx, strct, Trait::Unaligned, field_bounds).build()) +} + +/// An enum is `Unaligned` if: +/// - No `repr(align(N > 1))` +/// - `repr(u8)` or `repr(i8)` +fn derive_unaligned_enum(ctx: &Ctx, enm: &DataEnum) -> Result<TokenStream, Error> { + let repr = EnumRepr::from_attrs(&ctx.ast.attrs)?; + repr.unaligned_validate_no_align_gt_1()?; + + if !repr.is_u8() && !repr.is_i8() { + return ctx.error_or_skip(Error::new( + Span::call_site(), + "must have #[repr(u8)] or #[repr(i8)] attribute in order to guarantee this type's alignment", + )); + } + + Ok(ImplBlockBuilder::new(ctx, enm, Trait::Unaligned, FieldBounds::ALL_SELF).build()) +} + +/// Like structs, a union is `Unaligned` if: +/// - `repr(align)` is no more than 1 and either +/// - `repr(C)` or `repr(transparent)` and +/// - all fields `Unaligned` +/// - `repr(packed)` +fn derive_unaligned_union(ctx: &Ctx, unn: &DataUnion) -> Result<TokenStream, Error> { + let repr = StructUnionRepr::from_attrs(&ctx.ast.attrs)?; + repr.unaligned_validate_no_align_gt_1()?; + + let field_type_trait_bounds = if repr.is_packed_1() { + FieldBounds::None + } else if repr.is_c() || repr.is_transparent() { + FieldBounds::ALL_SELF + } else { + return ctx.error_or_skip(Error::new( + Span::call_site(), + "must have #[repr(C)], #[repr(transparent)], or #[repr(packed)] attribute in order to guarantee this type's alignment", + )); + }; + + Ok(ImplBlockBuilder::new(ctx, unn, Trait::Unaligned, field_type_trait_bounds).build()) +} diff --git a/rust/zerocopy-derive/lib.rs b/rust/zerocopy-derive/lib.rs new file mode 100644 index 000000000000..c517ea7db1eb --- /dev/null +++ b/rust/zerocopy-derive/lib.rs @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2019 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Derive macros for [zerocopy]'s traits. +//! +//! [zerocopy]: https://docs.rs/zerocopy + +// Sometimes we want to use lints which were added after our MSRV. +// `unknown_lints` is `warn` by default and we deny warnings in CI, so without +// this attribute, any unknown lint would cause a CI failure when testing with +// our MSRV. +#![allow(unknown_lints)] +#![deny(renamed_and_removed_lints)] +#![deny( + clippy::all, + clippy::missing_safety_doc, + clippy::multiple_unsafe_ops_per_block, + clippy::undocumented_unsafe_blocks +)] +// We defer to own discretion on type complexity. +#![allow(clippy::type_complexity)] +// Inlining format args isn't supported on our MSRV. +#![allow(clippy::uninlined_format_args)] +#![deny( + rustdoc::bare_urls, + rustdoc::broken_intra_doc_links, + rustdoc::invalid_codeblock_attributes, + rustdoc::invalid_html_tags, + rustdoc::invalid_rust_codeblocks, + rustdoc::missing_crate_level_docs, + rustdoc::private_intra_doc_links +)] +#![recursion_limit = "128"] + +macro_rules! ident { + (($fmt:literal $(, $arg:expr)*), $span:expr) => { + syn::Ident::new(&format!($fmt $(, crate::util::to_ident_str($arg))*), $span) + }; +} + +mod derive; +#[cfg(test)] +mod output_tests; +mod repr; +mod util; + +use syn::{DeriveInput, Error}; + +use crate::util::*; + +// FIXME(https://github.com/rust-lang/rust/issues/54140): Some errors could be +// made better if we could add multiple lines of error output like this: +// +// error: unsupported representation +// --> enum.rs:28:8 +// | +// 28 | #[repr(transparent)] +// | +// help: required by the derive of FromBytes +// +// Instead, we have more verbose error messages like "unsupported representation +// for deriving FromZeros, FromBytes, IntoBytes, or Unaligned on an enum" +// +// This will probably require Span::error +// (https://doc.rust-lang.org/nightly/proc_macro/struct.Span.html#method.error), +// which is currently unstable. Revisit this once it's stable. + +/// Defines a derive function named `$outer` which parses its input +/// `TokenStream` as a `DeriveInput` and then invokes the `$inner` function. +/// +/// Note that the separate `$outer` parameter is required - proc macro functions +/// are currently required to live at the crate root, and so the caller must +/// specify the name in order to avoid name collisions. +macro_rules! derive { + ($trait:ident => $outer:ident => $inner:path) => { + #[proc_macro_derive($trait, attributes(zerocopy))] + pub fn $outer(ts: proc_macro::TokenStream) -> proc_macro::TokenStream { + let ast = syn::parse_macro_input!(ts as DeriveInput); + let ctx = match Ctx::try_from_derive_input(ast) { + Ok(ctx) => ctx, + Err(e) => return e.into_compile_error().into(), + }; + let ts = $inner(&ctx, Trait::$trait).into_ts(); + // We wrap in `const_block` as a backstop in case any derive fails + // to wrap its output in `const_block` (and thus fails to annotate) + // with the full set of `#[allow(...)]` attributes). + let ts = const_block([Some(ts)]); + #[cfg(test)] + crate::util::testutil::check_hygiene(ts.clone()); + ts.into() + } + }; +} + +trait IntoTokenStream { + fn into_ts(self) -> proc_macro2::TokenStream; +} + +impl IntoTokenStream for proc_macro2::TokenStream { + fn into_ts(self) -> proc_macro2::TokenStream { + self + } +} + +impl IntoTokenStream for Result<proc_macro2::TokenStream, Error> { + fn into_ts(self) -> proc_macro2::TokenStream { + match self { + Ok(ts) => ts, + Err(err) => err.to_compile_error(), + } + } +} + +derive!(KnownLayout => derive_known_layout => crate::derive::known_layout::derive); +derive!(Immutable => derive_immutable => crate::derive::derive_immutable); +derive!(TryFromBytes => derive_try_from_bytes => crate::derive::try_from_bytes::derive_try_from_bytes); +derive!(FromZeros => derive_from_zeros => crate::derive::from_bytes::derive_from_zeros); +derive!(FromBytes => derive_from_bytes => crate::derive::from_bytes::derive_from_bytes); +derive!(IntoBytes => derive_into_bytes => crate::derive::into_bytes::derive_into_bytes); +derive!(Unaligned => derive_unaligned => crate::derive::unaligned::derive_unaligned); +derive!(ByteHash => derive_hash => crate::derive::derive_hash); +derive!(ByteEq => derive_eq => crate::derive::derive_eq); +derive!(SplitAt => derive_split_at => crate::derive::derive_split_at); + +/// Deprecated: prefer [`FromZeros`] instead. +#[deprecated(since = "0.8.0", note = "`FromZeroes` was renamed to `FromZeros`")] +#[doc(hidden)] +#[proc_macro_derive(FromZeroes)] +pub fn derive_from_zeroes(ts: proc_macro::TokenStream) -> proc_macro::TokenStream { + derive_from_zeros(ts) +} + +/// Deprecated: prefer [`IntoBytes`] instead. +#[deprecated(since = "0.8.0", note = "`AsBytes` was renamed to `IntoBytes`")] +#[doc(hidden)] +#[proc_macro_derive(AsBytes)] +pub fn derive_as_bytes(ts: proc_macro::TokenStream) -> proc_macro::TokenStream { + derive_into_bytes(ts) +} diff --git a/rust/zerocopy-derive/repr.rs b/rust/zerocopy-derive/repr.rs new file mode 100644 index 000000000000..74fd376d9fda --- /dev/null +++ b/rust/zerocopy-derive/repr.rs @@ -0,0 +1,851 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2019 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use core::{ + convert::{Infallible, TryFrom}, + num::NonZeroU32, +}; + +use proc_macro2::{Span, TokenStream}; +use quote::{quote_spanned, ToTokens, TokenStreamExt as _}; +use syn::{ + punctuated::Punctuated, spanned::Spanned as _, token::Comma, Attribute, Error, LitInt, Meta, + MetaList, +}; + +/// The computed representation of a type. +/// +/// This is the result of processing all `#[repr(...)]` attributes on a type, if +/// any. A `Repr` is only capable of representing legal combinations of +/// `#[repr(...)]` attributes. +#[cfg_attr(test, derive(Copy, Clone, Debug))] +pub(crate) enum Repr<Prim, Packed> { + /// `#[repr(transparent)]` + Transparent(Span), + /// A compound representation: `repr(C)`, `repr(Rust)`, or `repr(Int)` + /// optionally combined with `repr(packed(...))` or `repr(align(...))` + Compound(Spanned<CompoundRepr<Prim>>, Option<Spanned<AlignRepr<Packed>>>), +} + +/// A compound representation: `repr(C)`, `repr(Rust)`, or `repr(Int)`. +#[cfg_attr(test, derive(Copy, Clone, Debug, Eq, PartialEq))] +pub(crate) enum CompoundRepr<Prim> { + C, + Rust, + Primitive(Prim), +} + +/// `repr(Int)` +#[derive(Copy, Clone)] +#[cfg_attr(test, derive(Debug, Eq, PartialEq))] +pub(crate) enum PrimitiveRepr { + U8, + U16, + U32, + U64, + U128, + Usize, + I8, + I16, + I32, + I64, + I128, + Isize, +} + +/// `repr(packed(...))` or `repr(align(...))` +#[cfg_attr(test, derive(Copy, Clone, Debug, Eq, PartialEq))] +pub(crate) enum AlignRepr<Packed> { + Packed(Packed), + Align(NonZeroU32), +} + +/// The representations which can legally appear on a struct or union type. +pub(crate) type StructUnionRepr = Repr<Infallible, NonZeroU32>; + +/// The representations which can legally appear on an enum type. +pub(crate) type EnumRepr = Repr<PrimitiveRepr, Infallible>; + +impl<Prim, Packed> Repr<Prim, Packed> { + /// Gets the name of this "repr type" - the non-align `repr(X)` that is used + /// in prose to refer to this type. + /// + /// For example, we would refer to `#[repr(C, align(4))] struct Foo { ... }` + /// as a "`repr(C)` struct". + pub(crate) fn repr_type_name(&self) -> &str + where + Prim: Copy + With<PrimitiveRepr>, + { + use CompoundRepr::*; + use PrimitiveRepr::*; + use Repr::*; + match self { + Transparent(_span) => "repr(transparent)", + Compound(Spanned { t: repr, span: _ }, _align) => match repr { + C => "repr(C)", + Rust => "repr(Rust)", + Primitive(prim) => prim.with(|prim| match prim { + U8 => "repr(u8)", + U16 => "repr(u16)", + U32 => "repr(u32)", + U64 => "repr(u64)", + U128 => "repr(u128)", + Usize => "repr(usize)", + I8 => "repr(i8)", + I16 => "repr(i16)", + I32 => "repr(i32)", + I64 => "repr(i64)", + I128 => "repr(i128)", + Isize => "repr(isize)", + }), + }, + } + } + + pub(crate) fn is_transparent(&self) -> bool { + matches!(self, Repr::Transparent(_)) + } + + pub(crate) fn is_c(&self) -> bool { + use CompoundRepr::*; + matches!(self, Repr::Compound(Spanned { t: C, span: _ }, _align)) + } + + pub(crate) fn is_primitive(&self) -> bool { + use CompoundRepr::*; + matches!(self, Repr::Compound(Spanned { t: Primitive(_), span: _ }, _align)) + } + + pub(crate) fn get_packed(&self) -> Option<&Packed> { + use AlignRepr::*; + use Repr::*; + if let Compound(_, Some(Spanned { t: Packed(p), span: _ })) = self { + Some(p) + } else { + None + } + } + + pub(crate) fn get_align(&self) -> Option<Spanned<NonZeroU32>> { + use AlignRepr::*; + use Repr::*; + if let Compound(_, Some(Spanned { t: Align(n), span })) = self { + Some(Spanned::new(*n, *span)) + } else { + None + } + } + + pub(crate) fn is_align_gt_1(&self) -> bool { + self.get_align().map(|n| n.t.get() > 1).unwrap_or(false) + } + + /// When deriving `Unaligned`, validate that the decorated type has no + /// `#[repr(align(N))]` attribute where `N > 1`. If no such attribute exists + /// (including if `N == 1`), this returns `Ok(())`, and otherwise it returns + /// a descriptive error. + pub(crate) fn unaligned_validate_no_align_gt_1(&self) -> Result<(), Error> { + if let Some(n) = self.get_align().filter(|n| n.t.get() > 1) { + Err(Error::new( + n.span, + "cannot derive `Unaligned` on type with alignment greater than 1", + )) + } else { + Ok(()) + } + } +} + +impl<Prim> Repr<Prim, NonZeroU32> { + /// Does `self` describe a `#[repr(packed)]` or `#[repr(packed(1))]` type? + pub(crate) fn is_packed_1(&self) -> bool { + self.get_packed().map(|n| n.get() == 1).unwrap_or(false) + } +} + +impl<Packed> Repr<PrimitiveRepr, Packed> { + fn get_primitive(&self) -> Option<&PrimitiveRepr> { + use CompoundRepr::*; + use Repr::*; + if let Compound(Spanned { t: Primitive(p), span: _ }, _align) = self { + Some(p) + } else { + None + } + } + + /// Does `self` describe a `#[repr(u8)]` type? + pub(crate) fn is_u8(&self) -> bool { + matches!(self.get_primitive(), Some(PrimitiveRepr::U8)) + } + + /// Does `self` describe a `#[repr(i8)]` type? + pub(crate) fn is_i8(&self) -> bool { + matches!(self.get_primitive(), Some(PrimitiveRepr::I8)) + } +} + +impl<Prim, Packed> ToTokens for Repr<Prim, Packed> +where + Prim: With<PrimitiveRepr> + Copy, + Packed: With<NonZeroU32> + Copy, +{ + fn to_tokens(&self, ts: &mut TokenStream) { + use Repr::*; + match self { + Transparent(span) => ts.append_all(quote_spanned! { *span=> #[repr(transparent)] }), + Compound(repr, align) => { + repr.to_tokens(ts); + if let Some(align) = align { + align.to_tokens(ts); + } + } + } + } +} + +impl<Prim: With<PrimitiveRepr> + Copy> ToTokens for Spanned<CompoundRepr<Prim>> { + fn to_tokens(&self, ts: &mut TokenStream) { + use CompoundRepr::*; + match &self.t { + C => ts.append_all(quote_spanned! { self.span=> #[repr(C)] }), + Rust => ts.append_all(quote_spanned! { self.span=> #[repr(Rust)] }), + Primitive(prim) => prim.with(|prim| Spanned::new(prim, self.span).to_tokens(ts)), + } + } +} + +impl ToTokens for Spanned<PrimitiveRepr> { + fn to_tokens(&self, ts: &mut TokenStream) { + use PrimitiveRepr::*; + match self.t { + U8 => ts.append_all(quote_spanned! { self.span => #[repr(u8)] }), + U16 => ts.append_all(quote_spanned! { self.span => #[repr(u16)] }), + U32 => ts.append_all(quote_spanned! { self.span => #[repr(u32)] }), + U64 => ts.append_all(quote_spanned! { self.span => #[repr(u64)] }), + U128 => ts.append_all(quote_spanned! { self.span => #[repr(u128)] }), + Usize => ts.append_all(quote_spanned! { self.span => #[repr(usize)] }), + I8 => ts.append_all(quote_spanned! { self.span => #[repr(i8)] }), + I16 => ts.append_all(quote_spanned! { self.span => #[repr(i16)] }), + I32 => ts.append_all(quote_spanned! { self.span => #[repr(i32)] }), + I64 => ts.append_all(quote_spanned! { self.span => #[repr(i64)] }), + I128 => ts.append_all(quote_spanned! { self.span => #[repr(i128)] }), + Isize => ts.append_all(quote_spanned! { self.span => #[repr(isize)] }), + } + } +} + +impl<Packed: With<NonZeroU32> + Copy> ToTokens for Spanned<AlignRepr<Packed>> { + fn to_tokens(&self, ts: &mut TokenStream) { + use AlignRepr::*; + // We use `syn::Index` instead of `u32` because `quote_spanned!` + // serializes `u32` literals as `123u32`, not just `123`. Rust doesn't + // recognize that as a valid argument to `#[repr(align(...))]` or + // `#[repr(packed(...))]`. + let to_index = |n: NonZeroU32| syn::Index { index: n.get(), span: self.span }; + match self.t { + Packed(n) => n.with(|n| { + let n = to_index(n); + ts.append_all(quote_spanned! { self.span => #[repr(packed(#n))] }) + }), + Align(n) => { + let n = to_index(n); + ts.append_all(quote_spanned! { self.span => #[repr(align(#n))] }) + } + } + } +} + +/// The result of parsing a single `#[repr(...)]` attribute or a single +/// directive inside a compound `#[repr(..., ...)]` attribute. +#[derive(Copy, Clone, PartialEq, Eq)] +#[cfg_attr(test, derive(Debug))] +pub(crate) enum RawRepr { + Transparent, + C, + Rust, + U8, + U16, + U32, + U64, + U128, + Usize, + I8, + I16, + I32, + I64, + I128, + Isize, + Align(NonZeroU32), + PackedN(NonZeroU32), + Packed, +} + +/// The error from converting from a `RawRepr`. +#[cfg_attr(test, derive(Debug, Eq, PartialEq))] +pub(crate) enum FromRawReprError<E> { + /// The `RawRepr` doesn't affect the high-level repr we're parsing (e.g. + /// it's `align(...)` and we're parsing a `CompoundRepr`). + None, + /// The `RawRepr` is invalid for the high-level repr we're parsing (e.g. + /// it's `packed` repr and we're parsing an `AlignRepr` for an enum type). + Err(E), +} + +/// The representation hint is not supported for the decorated type. +#[cfg_attr(test, derive(Copy, Clone, Debug, Eq, PartialEq))] +pub(crate) struct UnsupportedReprError; + +impl<Prim: With<PrimitiveRepr>> TryFrom<RawRepr> for CompoundRepr<Prim> { + type Error = FromRawReprError<UnsupportedReprError>; + fn try_from( + raw: RawRepr, + ) -> Result<CompoundRepr<Prim>, FromRawReprError<UnsupportedReprError>> { + use RawRepr::*; + match raw { + C => Ok(CompoundRepr::C), + Rust => Ok(CompoundRepr::Rust), + raw @ (U8 | U16 | U32 | U64 | U128 | Usize | I8 | I16 | I32 | I64 | I128 | Isize) => { + Prim::try_with_or( + || match raw { + U8 => Ok(PrimitiveRepr::U8), + U16 => Ok(PrimitiveRepr::U16), + U32 => Ok(PrimitiveRepr::U32), + U64 => Ok(PrimitiveRepr::U64), + U128 => Ok(PrimitiveRepr::U128), + Usize => Ok(PrimitiveRepr::Usize), + I8 => Ok(PrimitiveRepr::I8), + I16 => Ok(PrimitiveRepr::I16), + I32 => Ok(PrimitiveRepr::I32), + I64 => Ok(PrimitiveRepr::I64), + I128 => Ok(PrimitiveRepr::I128), + Isize => Ok(PrimitiveRepr::Isize), + Transparent | C | Rust | Align(_) | PackedN(_) | Packed => { + Err(UnsupportedReprError) + } + }, + UnsupportedReprError, + ) + .map(CompoundRepr::Primitive) + .map_err(FromRawReprError::Err) + } + Transparent | Align(_) | PackedN(_) | Packed => Err(FromRawReprError::None), + } + } +} + +impl<Pcked: With<NonZeroU32>> TryFrom<RawRepr> for AlignRepr<Pcked> { + type Error = FromRawReprError<UnsupportedReprError>; + fn try_from(raw: RawRepr) -> Result<AlignRepr<Pcked>, FromRawReprError<UnsupportedReprError>> { + use RawRepr::*; + match raw { + Packed | PackedN(_) => Pcked::try_with_or( + || match raw { + Packed => Ok(NonZeroU32::new(1).unwrap()), + PackedN(n) => Ok(n), + U8 | U16 | U32 | U64 | U128 | Usize | I8 | I16 | I32 | I64 | I128 | Isize + | Transparent | C | Rust | Align(_) => Err(UnsupportedReprError), + }, + UnsupportedReprError, + ) + .map(AlignRepr::Packed) + .map_err(FromRawReprError::Err), + Align(n) => Ok(AlignRepr::Align(n)), + U8 | U16 | U32 | U64 | U128 | Usize | I8 | I16 | I32 | I64 | I128 | Isize + | Transparent | C | Rust => Err(FromRawReprError::None), + } + } +} + +/// The error from extracting a high-level repr type from a list of `RawRepr`s. +#[cfg_attr(test, derive(Copy, Clone, Debug, Eq, PartialEq))] +enum FromRawReprsError<E> { + /// One of the `RawRepr`s is invalid for the high-level repr we're parsing + /// (e.g. there's a `packed` repr and we're parsing an `AlignRepr` for an + /// enum type). + Single(E), + /// Two `RawRepr`s appear which both affect the high-level repr we're + /// parsing (e.g., the list is `#[repr(align(2), packed)]`). Note that we + /// conservatively treat redundant reprs as conflicting (e.g. + /// `#[repr(packed, packed)]`). + Conflict, +} + +/// Tries to extract a high-level repr from a list of `RawRepr`s. +fn try_from_raw_reprs<'a, E, R: TryFrom<RawRepr, Error = FromRawReprError<E>>>( + r: impl IntoIterator<Item = &'a Spanned<RawRepr>>, +) -> Result<Option<Spanned<R>>, Spanned<FromRawReprsError<E>>> { + // Walk the list of `RawRepr`s and attempt to convert each to an `R`. Bail + // if we find any errors. If we find more than one which converts to an `R`, + // bail with a `Conflict` error. + r.into_iter().try_fold(None, |found: Option<Spanned<R>>, raw| { + let new = match Spanned::<R>::try_from(*raw) { + Ok(r) => r, + // This `RawRepr` doesn't convert to an `R`, so keep the current + // found `R`, if any. + Err(FromRawReprError::None) => return Ok(found), + // This repr is unsupported for the decorated type (e.g. + // `repr(packed)` on an enum). + Err(FromRawReprError::Err(Spanned { t: err, span })) => { + return Err(Spanned::new(FromRawReprsError::Single(err), span)) + } + }; + + if let Some(found) = found { + // We already found an `R`, but this `RawRepr` also converts to an + // `R`, so that's a conflict. + // + // `Span::join` returns `None` if the two spans are from different + // files or if we're not on the nightly compiler. In that case, just + // use `new`'s span. + let span = found.span.join(new.span).unwrap_or(new.span); + Err(Spanned::new(FromRawReprsError::Conflict, span)) + } else { + Ok(Some(new)) + } + }) +} + +/// The error returned from [`Repr::from_attrs`]. +#[cfg_attr(test, derive(Copy, Clone, Debug, Eq, PartialEq))] +enum FromAttrsError { + FromRawReprs(FromRawReprsError<UnsupportedReprError>), + Unrecognized, +} + +impl From<FromRawReprsError<UnsupportedReprError>> for FromAttrsError { + fn from(err: FromRawReprsError<UnsupportedReprError>) -> FromAttrsError { + FromAttrsError::FromRawReprs(err) + } +} + +impl From<UnrecognizedReprError> for FromAttrsError { + fn from(_err: UnrecognizedReprError) -> FromAttrsError { + FromAttrsError::Unrecognized + } +} + +impl From<Spanned<FromAttrsError>> for Error { + fn from(err: Spanned<FromAttrsError>) -> Error { + let Spanned { t: err, span } = err; + match err { + FromAttrsError::FromRawReprs(FromRawReprsError::Single( + _err @ UnsupportedReprError, + )) => Error::new(span, "unsupported representation hint for the decorated type"), + FromAttrsError::FromRawReprs(FromRawReprsError::Conflict) => { + // NOTE: This says "another" rather than "a preceding" because + // when one of the reprs involved is `transparent`, we detect + // that condition in `Repr::from_attrs`, and at that point we + // can't tell which repr came first, so we might report this on + // the first involved repr rather than the second, third, etc. + Error::new(span, "this conflicts with another representation hint") + } + FromAttrsError::Unrecognized => Error::new(span, "unrecognized representation hint"), + } + } +} + +impl<Prim, Packed> Repr<Prim, Packed> { + fn from_attrs_inner(attrs: &[Attribute]) -> Result<Repr<Prim, Packed>, Spanned<FromAttrsError>> + where + Prim: With<PrimitiveRepr>, + Packed: With<NonZeroU32>, + { + let raw_reprs = RawRepr::from_attrs(attrs).map_err(Spanned::from)?; + + let transparent = { + let mut transparents = raw_reprs.iter().filter_map(|Spanned { t, span }| match t { + RawRepr::Transparent => Some(span), + _ => None, + }); + let first = transparents.next(); + let second = transparents.next(); + match (first, second) { + (None, None) => None, + (Some(span), None) => Some(*span), + (Some(_), Some(second)) => { + return Err(Spanned::new( + FromAttrsError::FromRawReprs(FromRawReprsError::Conflict), + *second, + )) + } + // An iterator can't produce a value only on the second call to + // `.next()`. + (None, Some(_)) => unreachable!(), + } + }; + + let compound: Option<Spanned<CompoundRepr<Prim>>> = + try_from_raw_reprs(raw_reprs.iter()).map_err(Spanned::from)?; + let align: Option<Spanned<AlignRepr<Packed>>> = + try_from_raw_reprs(raw_reprs.iter()).map_err(Spanned::from)?; + + if let Some(span) = transparent { + if compound.is_some() || align.is_some() { + // Arbitrarily report the problem on the `transparent` span. Any + // span will do. + return Err(Spanned::new(FromRawReprsError::Conflict.into(), span)); + } + + Ok(Repr::Transparent(span)) + } else { + Ok(Repr::Compound( + compound.unwrap_or(Spanned::new(CompoundRepr::Rust, Span::call_site())), + align, + )) + } + } +} + +impl<Prim, Packed> Repr<Prim, Packed> { + pub(crate) fn from_attrs(attrs: &[Attribute]) -> Result<Repr<Prim, Packed>, Error> + where + Prim: With<PrimitiveRepr>, + Packed: With<NonZeroU32>, + { + Repr::from_attrs_inner(attrs).map_err(Into::into) + } +} + +/// The representation hint could not be parsed or was unrecognized. +struct UnrecognizedReprError; + +impl RawRepr { + fn from_attrs( + attrs: &[Attribute], + ) -> Result<Vec<Spanned<RawRepr>>, Spanned<UnrecognizedReprError>> { + let mut reprs = Vec::new(); + for attr in attrs { + // Ignore documentation attributes. + if attr.path().is_ident("doc") { + continue; + } + if let Meta::List(ref meta_list) = attr.meta { + if meta_list.path.is_ident("repr") { + let parsed: Punctuated<Meta, Comma> = + match meta_list.parse_args_with(Punctuated::parse_terminated) { + Ok(parsed) => parsed, + Err(_) => { + return Err(Spanned::new( + UnrecognizedReprError, + meta_list.tokens.span(), + )) + } + }; + for meta in parsed { + let s = meta.span(); + reprs.push( + RawRepr::from_meta(&meta) + .map(|r| Spanned::new(r, s)) + .map_err(|e| Spanned::new(e, s))?, + ); + } + } + } + } + + Ok(reprs) + } + + fn from_meta(meta: &Meta) -> Result<RawRepr, UnrecognizedReprError> { + let (path, list) = match meta { + Meta::Path(path) => (path, None), + Meta::List(list) => (&list.path, Some(list)), + _ => return Err(UnrecognizedReprError), + }; + + let ident = path.get_ident().ok_or(UnrecognizedReprError)?; + + // Only returns `Ok` for non-zero power-of-two values. + let parse_nzu64 = |list: &MetaList| { + list.parse_args::<LitInt>() + .and_then(|int| int.base10_parse::<NonZeroU32>()) + .map_err(|_| UnrecognizedReprError) + .and_then(|nz| { + if nz.get().is_power_of_two() { + Ok(nz) + } else { + Err(UnrecognizedReprError) + } + }) + }; + + use RawRepr::*; + Ok(match (ident.to_string().as_str(), list) { + ("u8", None) => U8, + ("u16", None) => U16, + ("u32", None) => U32, + ("u64", None) => U64, + ("u128", None) => U128, + ("usize", None) => Usize, + ("i8", None) => I8, + ("i16", None) => I16, + ("i32", None) => I32, + ("i64", None) => I64, + ("i128", None) => I128, + ("isize", None) => Isize, + ("C", None) => C, + ("transparent", None) => Transparent, + ("Rust", None) => Rust, + ("packed", None) => Packed, + ("packed", Some(list)) => PackedN(parse_nzu64(list)?), + ("align", Some(list)) => Align(parse_nzu64(list)?), + _ => return Err(UnrecognizedReprError), + }) + } +} + +pub(crate) use util::*; +mod util { + use super::*; + /// A value with an associated span. + #[derive(Copy, Clone)] + #[cfg_attr(test, derive(Debug))] + pub(crate) struct Spanned<T> { + pub(crate) t: T, + pub(crate) span: Span, + } + + impl<T> Spanned<T> { + pub(super) fn new(t: T, span: Span) -> Spanned<T> { + Spanned { t, span } + } + + pub(super) fn from<U>(s: Spanned<U>) -> Spanned<T> + where + T: From<U>, + { + let Spanned { t: u, span } = s; + Spanned::new(u.into(), span) + } + + /// Delegates to `T: TryFrom`, preserving span information in both the + /// success and error cases. + pub(super) fn try_from<E, U>( + u: Spanned<U>, + ) -> Result<Spanned<T>, FromRawReprError<Spanned<E>>> + where + T: TryFrom<U, Error = FromRawReprError<E>>, + { + let Spanned { t: u, span } = u; + T::try_from(u).map(|t| Spanned { t, span }).map_err(|err| match err { + FromRawReprError::None => FromRawReprError::None, + FromRawReprError::Err(e) => FromRawReprError::Err(Spanned::new(e, span)), + }) + } + } + + // Used to permit implementing `With<T> for T: Inhabited` and for + // `Infallible` without a blanket impl conflict. + pub(crate) trait Inhabited {} + impl Inhabited for PrimitiveRepr {} + impl Inhabited for NonZeroU32 {} + + pub(crate) trait With<T> { + fn with<O, F: FnOnce(T) -> O>(self, f: F) -> O; + fn try_with_or<E, F: FnOnce() -> Result<T, E>>(f: F, err: E) -> Result<Self, E> + where + Self: Sized; + } + + impl<T: Inhabited> With<T> for T { + fn with<O, F: FnOnce(T) -> O>(self, f: F) -> O { + f(self) + } + + fn try_with_or<E, F: FnOnce() -> Result<T, E>>(f: F, _err: E) -> Result<Self, E> { + f() + } + } + + impl<T> With<T> for Infallible { + fn with<O, F: FnOnce(T) -> O>(self, _f: F) -> O { + match self {} + } + + fn try_with_or<E, F: FnOnce() -> Result<T, E>>(_f: F, err: E) -> Result<Self, E> { + Err(err) + } + } +} + +#[cfg(test)] +mod tests { + use syn::parse_quote; + + use super::*; + + impl<T> From<T> for Spanned<T> { + fn from(t: T) -> Spanned<T> { + Spanned::new(t, Span::call_site()) + } + } + + // We ignore spans for equality in testing since real spans are hard to + // synthesize and don't implement `PartialEq`. + impl<T: PartialEq> PartialEq for Spanned<T> { + fn eq(&self, other: &Spanned<T>) -> bool { + self.t.eq(&other.t) + } + } + + impl<T: Eq> Eq for Spanned<T> {} + + impl<Prim: PartialEq, Packed: PartialEq> PartialEq for Repr<Prim, Packed> { + fn eq(&self, other: &Repr<Prim, Packed>) -> bool { + match (self, other) { + (Repr::Transparent(_), Repr::Transparent(_)) => true, + (Repr::Compound(sc, sa), Repr::Compound(oc, oa)) => (sc, sa) == (oc, oa), + _ => false, + } + } + } + + fn s() -> Span { + Span::call_site() + } + + #[test] + fn test() { + // Test that a given `#[repr(...)]` attribute parses and returns the + // given `Repr` or error. + macro_rules! test { + ($(#[$attr:meta])* => $repr:expr) => { + test!(@inner $(#[$attr])* => Repr => Ok($repr)); + }; + // In the error case, the caller must explicitly provide the name of + // the `Repr` type to assist in type inference. + (@error $(#[$attr:meta])* => $typ:ident => $repr:expr) => { + test!(@inner $(#[$attr])* => $typ => Err($repr)); + }; + (@inner $(#[$attr:meta])* => $typ:ident => $repr:expr) => { + let attr: Attribute = parse_quote!($(#[$attr])*); + let mut got = $typ::from_attrs_inner(&[attr]); + let expect: Result<Repr<_, _>, _> = $repr; + if false { + // Force Rust to infer `got` as having the same type as + // `expect`. + got = expect; + } + assert_eq!(got, expect, stringify!($(#[$attr])*)); + }; + } + + use AlignRepr::*; + use CompoundRepr::*; + use PrimitiveRepr::*; + let nz = |n: u32| NonZeroU32::new(n).unwrap(); + + test!(#[repr(transparent)] => StructUnionRepr::Transparent(s())); + test!(#[repr()] => StructUnionRepr::Compound(Rust.into(), None)); + test!(#[repr(packed)] => StructUnionRepr::Compound(Rust.into(), Some(Packed(nz(1)).into()))); + test!(#[repr(packed(2))] => StructUnionRepr::Compound(Rust.into(), Some(Packed(nz(2)).into()))); + test!(#[repr(align(1))] => StructUnionRepr::Compound(Rust.into(), Some(Align(nz(1)).into()))); + test!(#[repr(align(2))] => StructUnionRepr::Compound(Rust.into(), Some(Align(nz(2)).into()))); + test!(#[repr(C)] => StructUnionRepr::Compound(C.into(), None)); + test!(#[repr(C, packed)] => StructUnionRepr::Compound(C.into(), Some(Packed(nz(1)).into()))); + test!(#[repr(C, packed(2))] => StructUnionRepr::Compound(C.into(), Some(Packed(nz(2)).into()))); + test!(#[repr(C, align(1))] => StructUnionRepr::Compound(C.into(), Some(Align(nz(1)).into()))); + test!(#[repr(C, align(2))] => StructUnionRepr::Compound(C.into(), Some(Align(nz(2)).into()))); + + test!(#[repr(transparent)] => EnumRepr::Transparent(s())); + test!(#[repr()] => EnumRepr::Compound(Rust.into(), None)); + test!(#[repr(align(1))] => EnumRepr::Compound(Rust.into(), Some(Align(nz(1)).into()))); + test!(#[repr(align(2))] => EnumRepr::Compound(Rust.into(), Some(Align(nz(2)).into()))); + + macro_rules! for_each_compound_repr { + ($($r:tt => $var:expr),*) => { + $( + test!(#[repr($r)] => EnumRepr::Compound($var.into(), None)); + test!(#[repr($r, align(1))] => EnumRepr::Compound($var.into(), Some(Align(nz(1)).into()))); + test!(#[repr($r, align(2))] => EnumRepr::Compound($var.into(), Some(Align(nz(2)).into()))); + )* + } + } + + for_each_compound_repr!( + C => C, + u8 => Primitive(U8), + u16 => Primitive(U16), + u32 => Primitive(U32), + u64 => Primitive(U64), + usize => Primitive(Usize), + i8 => Primitive(I8), + i16 => Primitive(I16), + i32 => Primitive(I32), + i64 => Primitive(I64), + isize => Primitive(Isize) + ); + + use FromAttrsError::*; + use FromRawReprsError::*; + + // Run failure tests which are valid for both `StructUnionRepr` and + // `EnumRepr`. + macro_rules! for_each_repr_type { + ($($repr:ident),*) => { + $( + // Invalid packed or align attributes + test!(@error #[repr(packed(0))] => $repr => Unrecognized.into()); + test!(@error #[repr(packed(3))] => $repr => Unrecognized.into()); + test!(@error #[repr(align(0))] => $repr => Unrecognized.into()); + test!(@error #[repr(align(3))] => $repr => Unrecognized.into()); + + // Conflicts + test!(@error #[repr(transparent, transparent)] => $repr => FromRawReprs(Conflict).into()); + test!(@error #[repr(transparent, C)] => $repr => FromRawReprs(Conflict).into()); + test!(@error #[repr(transparent, Rust)] => $repr => FromRawReprs(Conflict).into()); + + test!(@error #[repr(C, transparent)] => $repr => FromRawReprs(Conflict).into()); + test!(@error #[repr(C, C)] => $repr => FromRawReprs(Conflict).into()); + test!(@error #[repr(C, Rust)] => $repr => FromRawReprs(Conflict).into()); + + test!(@error #[repr(Rust, transparent)] => $repr => FromRawReprs(Conflict).into()); + test!(@error #[repr(Rust, C)] => $repr => FromRawReprs(Conflict).into()); + test!(@error #[repr(Rust, Rust)] => $repr => FromRawReprs(Conflict).into()); + )* + } + } + + for_each_repr_type!(StructUnionRepr, EnumRepr); + + // Enum-specific conflicts. + // + // We don't bother to test every combination since that would be a huge + // number (enums can have primitive reprs u8, u16, u32, u64, usize, i8, + // i16, i32, i64, and isize). Instead, since the conflict logic doesn't + // care what specific value of `PrimitiveRepr` is present, we assume + // that testing against u8 alone is fine. + test!(@error #[repr(transparent, u8)] => EnumRepr => FromRawReprs(Conflict).into()); + test!(@error #[repr(u8, transparent)] => EnumRepr => FromRawReprs(Conflict).into()); + test!(@error #[repr(C, u8)] => EnumRepr => FromRawReprs(Conflict).into()); + test!(@error #[repr(u8, C)] => EnumRepr => FromRawReprs(Conflict).into()); + test!(@error #[repr(Rust, u8)] => EnumRepr => FromRawReprs(Conflict).into()); + test!(@error #[repr(u8, Rust)] => EnumRepr => FromRawReprs(Conflict).into()); + test!(@error #[repr(u8, u8)] => EnumRepr => FromRawReprs(Conflict).into()); + + // Illegal struct/union reprs + test!(@error #[repr(u8)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(u16)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(u32)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(u64)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(usize)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(i8)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(i16)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(i32)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(i64)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(isize)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + + // Illegal enum reprs + test!(@error #[repr(packed)] => EnumRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(packed(1))] => EnumRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(packed(2))] => EnumRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + } +} diff --git a/rust/zerocopy-derive/util.rs b/rust/zerocopy-derive/util.rs new file mode 100644 index 000000000000..5ba5228e2a44 --- /dev/null +++ b/rust/zerocopy-derive/util.rs @@ -0,0 +1,845 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2019 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use std::num::NonZeroU32; + +use proc_macro2::{Span, TokenStream}; +use quote::{quote, quote_spanned, ToTokens}; +use syn::{ + parse_quote, spanned::Spanned as _, Data, DataEnum, DataStruct, DataUnion, DeriveInput, Error, + Expr, ExprLit, Field, GenericParam, Ident, Index, Lit, LitStr, Meta, Path, Type, Variant, + Visibility, WherePredicate, +}; + +use crate::repr::{CompoundRepr, EnumRepr, PrimitiveRepr, Repr, Spanned}; + +pub(crate) struct Ctx { + pub(crate) ast: DeriveInput, + pub(crate) zerocopy_crate: Path, + + // The value of the last `#[zerocopy(on_error = ...)]` attribute, or `false` + // if none is provided. + pub(crate) skip_on_error: bool, + + // The span of the last `#[zerocopy(on_error = ...)]` attribute, if any. + pub(crate) on_error_span: Option<proc_macro2::Span>, +} + +impl Ctx { + /// Attempt to extract a crate path from the provided attributes. Defaults to + /// `::zerocopy` if not found. + pub(crate) fn try_from_derive_input(ast: DeriveInput) -> Result<Self, Error> { + let mut path = parse_quote!(::zerocopy); + let mut skip_on_error = false; + let mut on_error_span = None; + + for attr in &ast.attrs { + if let Meta::List(ref meta_list) = attr.meta { + if meta_list.path.is_ident("zerocopy") { + attr.parse_nested_meta(|meta| { + if meta.path.is_ident("crate") { + let expr = meta.value().and_then(|value| value.parse()); + if let Ok(Expr::Lit(ExprLit { lit: Lit::Str(lit), .. })) = expr { + if let Ok(path_lit) = lit.parse::<Ident>() { + path = parse_quote!(::#path_lit); + return Ok(()); + } + } + + return Err(Error::new( + Span::call_site(), + "`crate` attribute requires a path as the value", + )); + } + + if meta.path.is_ident("on_error") { + on_error_span = Some(meta.path.span()); + let value = meta.value()?; + let s: LitStr = value.parse()?; + match s.value().as_str() { + "skip" => skip_on_error = true, + "fail" => skip_on_error = false, + _ => return Err(Error::new( + s.span(), + "unrecognized value for `on_error` attribute from `zerocopy`; expected `skip` or `fail`", + )), + } + return Ok(()); + } + + Err(Error::new( + Span::call_site(), + format!( + "unknown attribute encountered: {}", + meta.path.into_token_stream() + ), + )) + })?; + } + } + } + + Ok(Self { ast, zerocopy_crate: path, skip_on_error, on_error_span }) + } + + pub(crate) fn with_input(&self, input: &DeriveInput) -> Self { + Self { + ast: input.clone(), + zerocopy_crate: self.zerocopy_crate.clone(), + skip_on_error: self.skip_on_error, + on_error_span: self.on_error_span, + } + } + + pub(crate) fn core_path(&self) -> TokenStream { + let zerocopy_crate = &self.zerocopy_crate; + quote!(#zerocopy_crate::util::macro_util::core_reexport) + } + + pub(crate) fn cfg_compile_error(&self) -> TokenStream { + // By checking both during the compilation of the proc macro *and* in + // the generated code, we ensure that `--cfg + // zerocopy_unstable_derive_on_error` need only be passed *either* when + // compiling this crate *or* when compiling the user's crate. The former + // is preferable, but in some situations (such as when cross-compiling + // using `cargo build --target`), it doesn't get propagated to this + // crate's build by default. + if cfg!(zerocopy_unstable_derive_on_error) { + quote!() + } else if let Some(span) = self.on_error_span { + let core = self.core_path(); + let error_message = "`on_error` is experimental; pass '--cfg zerocopy_unstable_derive_on_error' to enable"; + quote::quote_spanned! {span=> + #[allow(unused_attributes, unexpected_cfgs)] + const _: () = { + #[cfg(not(zerocopy_unstable_derive_on_error))] + #core::compile_error!(#error_message); + }; + } + } else { + quote!() + } + } + + pub(crate) fn error_or_skip<E>(&self, error: E) -> Result<TokenStream, E> { + if self.skip_on_error { + Ok(self.cfg_compile_error()) + } else { + Err(error) + } + } +} + +pub(crate) trait DataExt { + /// Extracts the names and types of all fields. For enums, extracts the + /// names and types of fields from each variant. For tuple structs, the + /// names are the indices used to index into the struct (ie, `0`, `1`, etc). + /// + /// FIXME: Extracting field names for enums doesn't really make sense. Types + /// makes sense because we don't care about where they live - we just care + /// about transitive ownership. But for field names, we'd only use them when + /// generating is_bit_valid, which cares about where they live. + fn fields(&self) -> Vec<(&Visibility, TokenStream, &Type)>; + + fn variants(&self) -> Vec<(Option<&Variant>, Vec<(&Visibility, TokenStream, &Type)>)>; + + fn tag(&self) -> Option<Ident>; +} + +impl DataExt for Data { + fn fields(&self) -> Vec<(&Visibility, TokenStream, &Type)> { + match self { + Data::Struct(strc) => strc.fields(), + Data::Enum(enm) => enm.fields(), + Data::Union(un) => un.fields(), + } + } + + fn variants(&self) -> Vec<(Option<&Variant>, Vec<(&Visibility, TokenStream, &Type)>)> { + match self { + Data::Struct(strc) => strc.variants(), + Data::Enum(enm) => enm.variants(), + Data::Union(un) => un.variants(), + } + } + + fn tag(&self) -> Option<Ident> { + match self { + Data::Struct(strc) => strc.tag(), + Data::Enum(enm) => enm.tag(), + Data::Union(un) => un.tag(), + } + } +} + +impl DataExt for DataStruct { + fn fields(&self) -> Vec<(&Visibility, TokenStream, &Type)> { + map_fields(&self.fields) + } + + fn variants(&self) -> Vec<(Option<&Variant>, Vec<(&Visibility, TokenStream, &Type)>)> { + vec![(None, self.fields())] + } + + fn tag(&self) -> Option<Ident> { + None + } +} + +impl DataExt for DataEnum { + fn fields(&self) -> Vec<(&Visibility, TokenStream, &Type)> { + map_fields(self.variants.iter().flat_map(|var| &var.fields)) + } + + fn variants(&self) -> Vec<(Option<&Variant>, Vec<(&Visibility, TokenStream, &Type)>)> { + self.variants.iter().map(|var| (Some(var), map_fields(&var.fields))).collect() + } + + fn tag(&self) -> Option<Ident> { + Some(Ident::new("___ZerocopyTag", Span::call_site())) + } +} + +impl DataExt for DataUnion { + fn fields(&self) -> Vec<(&Visibility, TokenStream, &Type)> { + map_fields(&self.fields.named) + } + + fn variants(&self) -> Vec<(Option<&Variant>, Vec<(&Visibility, TokenStream, &Type)>)> { + vec![(None, self.fields())] + } + + fn tag(&self) -> Option<Ident> { + None + } +} + +fn map_fields<'a>( + fields: impl 'a + IntoIterator<Item = &'a Field>, +) -> Vec<(&'a Visibility, TokenStream, &'a Type)> { + fields + .into_iter() + .enumerate() + .map(|(idx, f)| { + ( + &f.vis, + f.ident + .as_ref() + .map(ToTokens::to_token_stream) + .unwrap_or_else(|| Index::from(idx).to_token_stream()), + &f.ty, + ) + }) + .collect() +} + +pub(crate) fn to_ident_str(t: &impl ToString) -> String { + let s = t.to_string(); + if let Some(stripped) = s.strip_prefix("r#") { + stripped.to_string() + } else { + s + } +} + +/// This enum describes what kind of padding check needs to be generated for the +/// associated impl. +pub(crate) enum PaddingCheck { + /// Check that the sum of the fields' sizes exactly equals the struct's + /// size. + Struct, + /// Check that a `repr(C)` struct has no padding. + ReprCStruct, + /// Check that the size of each field exactly equals the union's size. + Union, + /// Check that every variant of the enum contains no padding. + /// + /// Because doing so requires a tag enum, this padding check requires an + /// additional `TokenStream` which defines the tag enum as `___ZerocopyTag`. + Enum { tag_type_definition: TokenStream }, +} + +impl PaddingCheck { + /// Returns the idents of the trait to use and the macro to call in order to + /// validate that a type passes the relevant padding check. + pub(crate) fn validator_trait_and_macro_idents(&self) -> (Ident, Ident) { + let (trt, mcro) = match self { + PaddingCheck::Struct => ("PaddingFree", "struct_padding"), + PaddingCheck::ReprCStruct => ("DynamicPaddingFree", "repr_c_struct_has_padding"), + PaddingCheck::Union => ("PaddingFree", "union_padding"), + PaddingCheck::Enum { .. } => ("PaddingFree", "enum_padding"), + }; + + let trt = Ident::new(trt, Span::call_site()); + let mcro = Ident::new(mcro, Span::call_site()); + (trt, mcro) + } + + /// Sometimes performing the padding check requires some additional + /// "context" code. For enums, this is the definition of the tag enum. + pub(crate) fn validator_macro_context(&self) -> Option<&TokenStream> { + match self { + PaddingCheck::Struct | PaddingCheck::ReprCStruct | PaddingCheck::Union => None, + PaddingCheck::Enum { tag_type_definition } => Some(tag_type_definition), + } + } +} + +#[derive(Clone)] +pub(crate) enum Trait { + KnownLayout, + HasTag, + HasField { + variant_id: Box<Expr>, + field: Box<Type>, + field_id: Box<Expr>, + }, + ProjectField { + variant_id: Box<Expr>, + field: Box<Type>, + field_id: Box<Expr>, + invariants: Box<Type>, + }, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned, + Sized, + ByteHash, + ByteEq, + SplitAt, +} + +impl ToTokens for Trait { + fn to_tokens(&self, tokens: &mut TokenStream) { + // According to [1], the format of the derived `Debug`` output is not + // stable and therefore not guaranteed to represent the variant names. + // Indeed with the (unstable) `fmt-debug` compiler flag [2], it can + // return only a minimalized output or empty string. To make sure this + // code will work in the future and independent of the compiler flag, we + // translate the variants to their names manually here. + // + // [1] https://doc.rust-lang.org/1.81.0/std/fmt/trait.Debug.html#stability + // [2] https://doc.rust-lang.org/beta/unstable-book/compiler-flags/fmt-debug.html + let s = match self { + Trait::HasField { .. } => "HasField", + Trait::ProjectField { .. } => "ProjectField", + Trait::KnownLayout => "KnownLayout", + Trait::HasTag => "HasTag", + Trait::Immutable => "Immutable", + Trait::TryFromBytes => "TryFromBytes", + Trait::FromZeros => "FromZeros", + Trait::FromBytes => "FromBytes", + Trait::IntoBytes => "IntoBytes", + Trait::Unaligned => "Unaligned", + Trait::Sized => "Sized", + Trait::ByteHash => "ByteHash", + Trait::ByteEq => "ByteEq", + Trait::SplitAt => "SplitAt", + }; + let ident = Ident::new(s, Span::call_site()); + let arguments: Option<syn::AngleBracketedGenericArguments> = match self { + Trait::HasField { variant_id, field, field_id } => { + Some(parse_quote!(<#field, #variant_id, #field_id>)) + } + Trait::ProjectField { variant_id, field, field_id, invariants } => { + Some(parse_quote!(<#field, #invariants, #variant_id, #field_id>)) + } + Trait::KnownLayout + | Trait::HasTag + | Trait::Immutable + | Trait::TryFromBytes + | Trait::FromZeros + | Trait::FromBytes + | Trait::IntoBytes + | Trait::Unaligned + | Trait::Sized + | Trait::ByteHash + | Trait::ByteEq + | Trait::SplitAt => None, + }; + tokens.extend(quote!(#ident #arguments)); + } +} + +impl Trait { + pub(crate) fn crate_path(&self, ctx: &Ctx) -> Path { + let zerocopy_crate = &ctx.zerocopy_crate; + let core = ctx.core_path(); + match self { + Self::Sized => parse_quote!(#core::marker::#self), + _ => parse_quote!(#zerocopy_crate::#self), + } + } +} + +pub(crate) enum TraitBound { + Slf, + Other(Trait), +} + +pub(crate) enum FieldBounds<'a> { + None, + All(&'a [TraitBound]), + Trailing(&'a [TraitBound]), + Explicit(Vec<WherePredicate>), +} + +impl<'a> FieldBounds<'a> { + pub(crate) const ALL_SELF: FieldBounds<'a> = FieldBounds::All(&[TraitBound::Slf]); + pub(crate) const TRAILING_SELF: FieldBounds<'a> = FieldBounds::Trailing(&[TraitBound::Slf]); +} + +pub(crate) enum SelfBounds<'a> { + None, + All(&'a [Trait]), +} + +// FIXME(https://github.com/rust-lang/rust-clippy/issues/12908): This is a false +// positive. Explicit lifetimes are actually necessary here. +#[allow(clippy::needless_lifetimes)] +impl<'a> SelfBounds<'a> { + pub(crate) const SIZED: Self = Self::All(&[Trait::Sized]); +} + +/// Normalizes a slice of bounds by replacing [`TraitBound::Slf`] with `slf`. +pub(crate) fn normalize_bounds<'a>( + slf: &'a Trait, + bounds: &'a [TraitBound], +) -> impl 'a + Iterator<Item = Trait> { + bounds.iter().map(move |bound| match bound { + TraitBound::Slf => slf.clone(), + TraitBound::Other(trt) => trt.clone(), + }) +} + +pub(crate) struct ImplBlockBuilder<'a> { + ctx: &'a Ctx, + data: &'a dyn DataExt, + trt: Trait, + field_type_trait_bounds: FieldBounds<'a>, + self_type_trait_bounds: SelfBounds<'a>, + padding_check: Option<PaddingCheck>, + param_extras: Vec<GenericParam>, + inner_extras: Option<TokenStream>, + outer_extras: Option<TokenStream>, +} + +impl<'a> ImplBlockBuilder<'a> { + pub(crate) fn new( + ctx: &'a Ctx, + data: &'a dyn DataExt, + trt: Trait, + field_type_trait_bounds: FieldBounds<'a>, + ) -> Self { + Self { + ctx, + data, + trt, + field_type_trait_bounds, + self_type_trait_bounds: SelfBounds::None, + padding_check: None, + param_extras: Vec::new(), + inner_extras: None, + outer_extras: None, + } + } + + pub(crate) fn self_type_trait_bounds(mut self, self_type_trait_bounds: SelfBounds<'a>) -> Self { + self.self_type_trait_bounds = self_type_trait_bounds; + self + } + + pub(crate) fn padding_check<P: Into<Option<PaddingCheck>>>(mut self, padding_check: P) -> Self { + self.padding_check = padding_check.into(); + self + } + + pub(crate) fn param_extras(mut self, param_extras: Vec<GenericParam>) -> Self { + self.param_extras.extend(param_extras); + self + } + + pub(crate) fn inner_extras(mut self, inner_extras: TokenStream) -> Self { + self.inner_extras = Some(inner_extras); + self + } + + pub(crate) fn outer_extras<T: Into<Option<TokenStream>>>(mut self, outer_extras: T) -> Self { + self.outer_extras = outer_extras.into(); + self + } + + pub(crate) fn build(self) -> TokenStream { + // In this documentation, we will refer to this hypothetical struct: + // + // #[derive(FromBytes)] + // struct Foo<T, I: Iterator> + // where + // T: Copy, + // I: Clone, + // I::Item: Clone, + // { + // a: u8, + // b: T, + // c: I::Item, + // } + // + // We extract the field types, which in this case are `u8`, `T`, and + // `I::Item`. We re-use the existing parameters and where clauses. If + // `require_trait_bound == true` (as it is for `FromBytes), we add where + // bounds for each field's type: + // + // impl<T, I: Iterator> FromBytes for Foo<T, I> + // where + // T: Copy, + // I: Clone, + // I::Item: Clone, + // T: FromBytes, + // I::Item: FromBytes, + // { + // } + // + // NOTE: It is standard practice to only emit bounds for the type + // parameters themselves, not for field types based on those parameters + // (e.g., `T` vs `T::Foo`). For a discussion of why this is standard + // practice, see https://github.com/rust-lang/rust/issues/26925. + // + // The reason we diverge from this standard is that doing it that way + // for us would be unsound. E.g., consider a type, `T` where `T: + // FromBytes` but `T::Foo: !FromBytes`. It would not be sound for us to + // accept a type with a `T::Foo` field as `FromBytes` simply because `T: + // FromBytes`. + // + // While there's no getting around this requirement for us, it does have + // the pretty serious downside that, when lifetimes are involved, the + // trait solver ties itself in knots: + // + // #[derive(Unaligned)] + // #[repr(C)] + // struct Dup<'a, 'b> { + // a: PhantomData<&'a u8>, + // b: PhantomData<&'b u8>, + // } + // + // error[E0283]: type annotations required: cannot resolve `core::marker::PhantomData<&'a u8>: zerocopy::Unaligned` + // --> src/main.rs:6:10 + // | + // 6 | #[derive(Unaligned)] + // | ^^^^^^^^^ + // | + // = note: required by `zerocopy::Unaligned` + + let type_ident = &self.ctx.ast.ident; + let trait_path = self.trt.crate_path(self.ctx); + let fields = self.data.fields(); + let variants = self.data.variants(); + let tag = self.data.tag(); + let zerocopy_crate = &self.ctx.zerocopy_crate; + + fn bound_tt(ty: &Type, traits: impl Iterator<Item = Trait>, ctx: &Ctx) -> WherePredicate { + let traits = traits.map(|t| t.crate_path(ctx)); + parse_quote!(#ty: #(#traits)+*) + } + let field_type_bounds: Vec<_> = match (self.field_type_trait_bounds, &fields[..]) { + (FieldBounds::All(traits), _) => fields + .iter() + .map(|(_vis, _name, ty)| { + bound_tt(ty, normalize_bounds(&self.trt, traits), self.ctx) + }) + .collect(), + (FieldBounds::None, _) | (FieldBounds::Trailing(..), []) => vec![], + (FieldBounds::Trailing(traits), [.., last]) => { + vec![bound_tt(last.2, normalize_bounds(&self.trt, traits), self.ctx)] + } + (FieldBounds::Explicit(bounds), _) => bounds, + }; + + let padding_check_bound = self + .padding_check + .map(|check| { + // Parse the repr for `align` and `packed` modifiers. Note that + // `Repr::<PrimitiveRepr, NonZeroU32>` is more permissive than + // what Rust supports for structs, enums, or unions, and thus + // reliably extracts these modifiers for any kind of type. + let repr = + Repr::<PrimitiveRepr, NonZeroU32>::from_attrs(&self.ctx.ast.attrs).unwrap(); + let core = self.ctx.core_path(); + let option = quote! { #core::option::Option }; + let nonzero = quote! { #core::num::NonZeroUsize }; + let none = quote! { #option::None::<#nonzero> }; + let repr_align = + repr.get_align().map(|spanned| { + let n = spanned.t.get(); + quote_spanned! { spanned.span => (#nonzero::new(#n as usize)) } + }).unwrap_or(quote! { (#none) }); + let repr_packed = + repr.get_packed().map(|packed| { + let n = packed.get(); + quote! { (#nonzero::new(#n as usize)) } + }).unwrap_or(quote! { (#none) }); + let variant_types = variants.iter().map(|(_, fields)| { + let types = fields.iter().map(|(_vis, _name, ty)| ty); + quote!([#((#types)),*]) + }); + let validator_context = check.validator_macro_context(); + let (trt, validator_macro) = check.validator_trait_and_macro_idents(); + let t = tag.iter(); + parse_quote! { + (): #zerocopy_crate::util::macro_util::#trt< + Self, + { + #validator_context + #zerocopy_crate::#validator_macro!(Self, #repr_align, #repr_packed, #(#t,)* #(#variant_types),*) + } + > + } + }); + + let self_bounds: Option<WherePredicate> = match self.self_type_trait_bounds { + SelfBounds::None => None, + SelfBounds::All(traits) => { + Some(bound_tt(&parse_quote!(Self), traits.iter().cloned(), self.ctx)) + } + }; + + let bounds = self + .ctx + .ast + .generics + .where_clause + .as_ref() + .map(|where_clause| where_clause.predicates.iter()) + .into_iter() + .flatten() + .chain(field_type_bounds.iter()) + .chain(padding_check_bound.iter()) + .chain(self_bounds.iter()); + + // The parameters with trait bounds, but without type defaults. + let mut params: Vec<_> = self + .ctx + .ast + .generics + .params + .clone() + .into_iter() + .map(|mut param| { + match &mut param { + GenericParam::Type(ty) => ty.default = None, + GenericParam::Const(cnst) => cnst.default = None, + GenericParam::Lifetime(_) => {} + } + parse_quote!(#param) + }) + .chain(self.param_extras) + .collect(); + + // For MSRV purposes, ensure that lifetimes precede types precede const + // generics. + params.sort_by_cached_key(|param| match param { + GenericParam::Lifetime(_) => 0, + GenericParam::Type(_) => 1, + GenericParam::Const(_) => 2, + }); + + // The identifiers of the parameters without trait bounds or type + // defaults. + let param_idents = self.ctx.ast.generics.params.iter().map(|param| match param { + GenericParam::Type(ty) => { + let ident = &ty.ident; + quote!(#ident) + } + GenericParam::Lifetime(l) => { + let ident = &l.lifetime; + quote!(#ident) + } + GenericParam::Const(cnst) => { + let ident = &cnst.ident; + quote!({#ident}) + } + }); + + let inner_extras = self.inner_extras; + let allow_trivial_bounds = + if self.ctx.skip_on_error { quote!(#[allow(trivial_bounds)]) } else { quote!() }; + let impl_tokens = quote! { + #allow_trivial_bounds + unsafe impl < #(#params),* > #trait_path for #type_ident < #(#param_idents),* > + where + #(#bounds,)* + { + fn only_derive_is_allowed_to_implement_this_trait() {} + + #inner_extras + } + }; + + let outer_extras = self.outer_extras.filter(|e| !e.is_empty()); + let cfg_compile_error = self.ctx.cfg_compile_error(); + const_block([Some(cfg_compile_error), Some(impl_tokens), outer_extras]) + } +} + +// A polyfill for `Option::then_some`, which was added after our MSRV. +// +// The `#[allow(unused)]` is necessary because, on sufficiently recent toolchain +// versions, `b.then_some(...)` resolves to the inherent method rather than to +// this trait, and so this trait is considered unused. +// +// FIXME(#67): Remove this once our MSRV is >= 1.62. +#[allow(unused)] +trait BoolExt { + fn then_some<T>(self, t: T) -> Option<T>; +} + +impl BoolExt for bool { + fn then_some<T>(self, t: T) -> Option<T> { + if self { + Some(t) + } else { + None + } + } +} + +pub(crate) fn const_block(items: impl IntoIterator<Item = Option<TokenStream>>) -> TokenStream { + let items = items.into_iter().flatten(); + quote! { + #[allow( + // FIXME(#553): Add a test that generates a warning when + // `#[allow(deprecated)]` isn't present. + deprecated, + // Required on some rustc versions due to a lint that is only + // triggered when `derive(KnownLayout)` is applied to `repr(C)` + // structs that are generated by macros. See #2177 for details. + private_bounds, + non_local_definitions, + non_camel_case_types, + non_upper_case_globals, + non_snake_case, + non_ascii_idents, + clippy::missing_inline_in_public_items, + )] + #[deny(ambiguous_associated_items)] + // While there are not currently any warnings that this suppresses + // (that we're aware of), it's good future-proofing hygiene. + #[automatically_derived] + const _: () = { + #(#items)* + }; + } +} +pub(crate) fn generate_tag_enum(ctx: &Ctx, repr: &EnumRepr, data: &DataEnum) -> TokenStream { + let zerocopy_crate = &ctx.zerocopy_crate; + let variants = data.variants.iter().map(|v| { + let ident = &v.ident; + if let Some((eq, discriminant)) = &v.discriminant { + quote! { #ident #eq #discriminant } + } else { + quote! { #ident } + } + }); + + // Don't include any `repr(align)` when generating the tag enum, as that + // could add padding after the tag but before any variants, which is not the + // correct behavior. + let repr = match repr { + EnumRepr::Transparent(span) => quote::quote_spanned! { *span => #[repr(transparent)] }, + EnumRepr::Compound(c, _) => quote! { #c }, + }; + + quote! { + #repr + #[allow(dead_code)] + pub enum ___ZerocopyTag { + #(#variants,)* + } + + // SAFETY: `___ZerocopyTag` has no fields, and so it does not permit + // interior mutation. + unsafe impl #zerocopy_crate::Immutable for ___ZerocopyTag { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + } +} +pub(crate) fn enum_size_from_repr(repr: &EnumRepr) -> Result<usize, Error> { + use CompoundRepr::*; + use PrimitiveRepr::*; + use Repr::*; + match repr { + Transparent(span) + | Compound( + Spanned { + t: C | Rust | Primitive(U32 | I32 | U64 | I64 | U128 | I128 | Usize | Isize), + span, + }, + _, + ) => Err(Error::new( + *span, + "`FromBytes` only supported on enums with `#[repr(...)]` attributes `u8`, `i8`, `u16`, or `i16`", + )), + Compound(Spanned { t: Primitive(U8 | I8), span: _ }, _align) => Ok(8), + Compound(Spanned { t: Primitive(U16 | I16), span: _ }, _align) => Ok(16), + } +} + +#[cfg(test)] +pub(crate) mod testutil { + use proc_macro2::TokenStream; + use syn::visit::{self, Visit}; + + /// Checks for hygiene violations in the generated code. + /// + /// # Panics + /// + /// Panics if a hygiene violation is found. + pub(crate) fn check_hygiene(ts: TokenStream) { + struct AmbiguousItemVisitor; + + impl<'ast> Visit<'ast> for AmbiguousItemVisitor { + fn visit_path(&mut self, i: &'ast syn::Path) { + if i.segments.len() > 1 && i.segments.first().unwrap().ident == "Self" { + panic!( + "Found ambiguous path `{}` in generated output. \ + All associated item access must be fully qualified (e.g., `<Self as Trait>::Item`) \ + to prevent hygiene issues.", + quote::quote!(#i) + ); + } + visit::visit_path(self, i); + } + } + + let file = syn::parse2::<syn::File>(ts).expect("failed to parse generated output as File"); + AmbiguousItemVisitor.visit_file(&file); + } + + #[test] + fn test_check_hygiene_success() { + check_hygiene(quote::quote! { + fn foo() { + let _ = <Self as Trait>::Item; + } + }); + } + + #[test] + #[should_panic(expected = "Found ambiguous path `Self :: Ambiguous`")] + fn test_check_hygiene_failure() { + check_hygiene(quote::quote! { + fn foo() { + let _ = Self::Ambiguous; + } + }); + } +} diff --git a/rust/zerocopy/README.md b/rust/zerocopy/README.md new file mode 100644 index 000000000000..99e6cad0e26c --- /dev/null +++ b/rust/zerocopy/README.md @@ -0,0 +1,14 @@ +# `zerocopy` + +These source files come from the Rust `zerocopy` crate, version v0.8.50 +(released 2026-05-31), hosted in the <https://github.com/google/zerocopy> +repository, licensed under "BSD-2-Clause OR Apache-2.0 OR MIT" and only +modified to add the SPDX license identifiers and to remove `Display` +for `f32` and `f64`. + +For copyright details, please see: + + https://github.com/google/zerocopy/blob/v0.8.50/README.md?plain=1 + https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-BSD + https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-APACHE + https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-MIT diff --git a/rust/zerocopy/benches/as_bytes_dynamic_size.rs b/rust/zerocopy/benches/as_bytes_dynamic_size.rs new file mode 100644 index 000000000000..68cd1d6f4111 --- /dev/null +++ b/rust/zerocopy/benches/as_bytes_dynamic_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_as_bytes_dynamic_size(source: &format::CocoPacket) -> &[u8] { + source.as_bytes() +} diff --git a/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64 b/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64 new file mode 100644 index 000000000000..f68bad612695 --- /dev/null +++ b/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64 @@ -0,0 +1,5 @@ +bench_as_bytes_dynamic_size: + mov rax, rdi + lea rdx, [2*rsi + 5] + and rdx, -2 + ret diff --git a/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64.mca b/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..c3b92a9a95fe --- /dev/null +++ b/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64.mca @@ -0,0 +1,47 @@ +Iterations: 100 +Instructions: 400 +Total Cycles: 137 +Total uOps: 400 + +Dispatch Width: 4 +uOps Per Cycle: 2.92 +IPC: 2.92 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.50 lea rdx, [2*rsi + 5] + 1 1 0.33 and rdx, -2 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.33 1.33 - 1.34 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.66 - 0.34 - - mov rax, rdi + - - 0.33 0.67 - - - - lea rdx, [2*rsi + 5] + - - 1.00 - - - - - and rdx, -2 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/as_bytes_static_size.rs b/rust/zerocopy/benches/as_bytes_static_size.rs new file mode 100644 index 000000000000..2ad738e95480 --- /dev/null +++ b/rust/zerocopy/benches/as_bytes_static_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_as_bytes_static_size(source: &format::CocoPacket) -> &[u8] { + source.as_bytes() +} diff --git a/rust/zerocopy/benches/as_bytes_static_size.x86-64 b/rust/zerocopy/benches/as_bytes_static_size.x86-64 new file mode 100644 index 000000000000..213e74ab54ff --- /dev/null +++ b/rust/zerocopy/benches/as_bytes_static_size.x86-64 @@ -0,0 +1,4 @@ +bench_as_bytes_static_size: + mov rax, rdi + mov edx, 6 + ret diff --git a/rust/zerocopy/benches/as_bytes_static_size.x86-64.mca b/rust/zerocopy/benches/as_bytes_static_size.x86-64.mca new file mode 100644 index 000000000000..ae04a6ba9061 --- /dev/null +++ b/rust/zerocopy/benches/as_bytes_static_size.x86-64.mca @@ -0,0 +1,45 @@ +Iterations: 100 +Instructions: 300 +Total Cycles: 104 +Total uOps: 300 + +Dispatch Width: 4 +uOps Per Cycle: 2.88 +IPC: 2.88 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 mov edx, 6 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.99 1.00 - 1.01 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.99 - - 0.01 - - mov rax, rdi + - - - 1.00 - - - - mov edx, 6 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/extend_vec_zeroed.rs b/rust/zerocopy/benches/extend_vec_zeroed.rs new file mode 100644 index 000000000000..1fbf772d1ea7 --- /dev/null +++ b/rust/zerocopy/benches/extend_vec_zeroed.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_extend_vec_zeroed(v: &mut Vec<format::LocoPacket>, additional: usize) -> Option<()> { + FromZeros::extend_vec_zeroed(v, additional).ok() +} diff --git a/rust/zerocopy/benches/extend_vec_zeroed.x86-64 b/rust/zerocopy/benches/extend_vec_zeroed.x86-64 new file mode 100644 index 000000000000..831b2a075fec --- /dev/null +++ b/rust/zerocopy/benches/extend_vec_zeroed.x86-64 @@ -0,0 +1,60 @@ +bench_extend_vec_zeroed: + push r15 + push r14 + push r13 + push r12 + push rbx + sub rsp, 32 + mov rbx, rdi + mov rax, qword ptr [rdi] + mov r12, qword ptr [rdi + 16] + mov rcx, rax + sub rcx, r12 + cmp rsi, rcx + jbe .LBB6_3 + mov r15, r12 + add r15, rsi + jae .LBB6_6 +.LBB6_2: + xor eax, eax + jmp .LBB6_5 +.LBB6_3: + mov rax, qword ptr [rbx + 8] + lea r15, [r12 + rsi] +.LBB6_4: + lea rcx, [r12 + 2*r12] + lea rdi, [rax + 2*rcx] + add rsi, rsi + lea rdx, [rsi + 2*rsi] + xor esi, esi + call qword ptr [rip + memset@GOTPCREL] + mov qword ptr [rbx + 16], r15 + mov al, 1 +.LBB6_5: + add rsp, 32 + pop rbx + pop r12 + pop r13 + pop r14 + pop r15 + ret +.LBB6_6: + mov r13, rsi + lea rcx, [rax + rax] + cmp r15, rcx + cmova rcx, r15 + cmp rcx, 5 + mov r14d, 4 + cmovae r14, rcx + mov rdx, qword ptr [rbx + 8] + lea rdi, [rsp + 8] + mov rsi, rax + mov rcx, r14 + call <alloc::raw_vec::RawVecInner>::finish_grow + cmp dword ptr [rsp + 8], 1 + je .LBB6_2 + mov rax, qword ptr [rsp + 16] + mov qword ptr [rbx + 8], rax + mov qword ptr [rbx], r14 + mov rsi, r13 + jmp .LBB6_4 diff --git a/rust/zerocopy/benches/extend_vec_zeroed.x86-64.mca b/rust/zerocopy/benches/extend_vec_zeroed.x86-64.mca new file mode 100644 index 000000000000..cfab1eea8f56 --- /dev/null +++ b/rust/zerocopy/benches/extend_vec_zeroed.x86-64.mca @@ -0,0 +1,147 @@ +Iterations: 100 +Instructions: 5400 +Total Cycles: 6595 +Total uOps: 6800 + +Dispatch Width: 4 +uOps Per Cycle: 1.03 +IPC: 0.82 +Block RThroughput: 17.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push r15 + 2 5 1.00 * push r14 + 2 5 1.00 * push r13 + 2 5 1.00 * push r12 + 2 5 1.00 * push rbx + 1 1 0.33 sub rsp, 32 + 1 1 0.33 mov rbx, rdi + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 5 0.50 * mov r12, qword ptr [rdi + 16] + 1 1 0.33 mov rcx, rax + 1 1 0.33 sub rcx, r12 + 1 1 0.33 cmp rsi, rcx + 1 1 1.00 jbe .LBB6_3 + 1 1 0.33 mov r15, r12 + 1 1 0.33 add r15, rsi + 1 1 1.00 jae .LBB6_6 + 1 0 0.25 xor eax, eax + 1 1 1.00 jmp .LBB6_5 + 1 5 0.50 * mov rax, qword ptr [rbx + 8] + 1 1 0.50 lea r15, [r12 + rsi] + 1 1 0.50 lea rcx, [r12 + 2*r12] + 1 1 0.50 lea rdi, [rax + 2*rcx] + 1 1 0.33 add rsi, rsi + 1 1 0.50 lea rdx, [rsi + 2*rsi] + 1 0 0.25 xor esi, esi + 4 7 1.00 * call qword ptr [rip + memset@GOTPCREL] + 1 1 1.00 * mov qword ptr [rbx + 16], r15 + 1 1 0.33 mov al, 1 + 1 1 0.33 add rsp, 32 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r12 + 1 6 0.50 * pop r13 + 1 6 0.50 * pop r14 + 1 6 0.50 * pop r15 + 1 1 1.00 U ret + 1 1 0.33 mov r13, rsi + 1 1 0.50 lea rcx, [rax + rax] + 1 1 0.33 cmp r15, rcx + 3 3 1.00 cmova rcx, r15 + 1 1 0.33 cmp rcx, 5 + 1 1 0.33 mov r14d, 4 + 2 2 0.67 cmovae r14, rcx + 1 5 0.50 * mov rdx, qword ptr [rbx + 8] + 1 1 0.50 lea rdi, [rsp + 8] + 1 1 0.33 mov rsi, rax + 1 1 0.33 mov rcx, r14 + 3 5 1.00 call <alloc::raw_vec::RawVecInner>::finish_grow + 2 6 0.50 * cmp dword ptr [rsp + 8], 1 + 1 1 1.00 je .LBB6_2 + 1 5 0.50 * mov rax, qword ptr [rsp + 16] + 1 1 1.00 * mov qword ptr [rbx + 8], rax + 1 1 1.00 * mov qword ptr [rbx], r14 + 1 1 0.33 mov rsi, r13 + 1 1 1.00 jmp .LBB6_4 + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 12.00 12.00 10.00 13.00 11.00 11.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - 0.49 0.51 push r15 + - - - - 1.00 - 0.51 0.49 push r14 + - - - - 1.00 - 0.50 0.50 push r13 + - - - - 1.00 - 0.50 0.50 push r12 + - - - - 1.00 - 0.50 0.50 push rbx + - - 0.01 0.99 - - - - sub rsp, 32 + - - - - - 1.00 - - mov rbx, rdi + - - - - - - 0.50 0.50 mov rax, qword ptr [rdi] + - - - - - - 0.50 0.50 mov r12, qword ptr [rdi + 16] + - - - 1.00 - - - - mov rcx, rax + - - - 0.99 - 0.01 - - sub rcx, r12 + - - - - - 1.00 - - cmp rsi, rcx + - - - - - 1.00 - - jbe .LBB6_3 + - - 0.01 0.98 - 0.01 - - mov r15, r12 + - - 0.99 0.01 - - - - add r15, rsi + - - - - - 1.00 - - jae .LBB6_6 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - jmp .LBB6_5 + - - - - - - 0.50 0.50 mov rax, qword ptr [rbx + 8] + - - 1.00 - - - - - lea r15, [r12 + rsi] + - - 0.98 0.02 - - - - lea rcx, [r12 + 2*r12] + - - 0.99 0.01 - - - - lea rdi, [rax + 2*rcx] + - - - 1.00 - - - - add rsi, rsi + - - 0.99 0.01 - - - - lea rdx, [rsi + 2*rsi] + - - - - - - - - xor esi, esi + - - - - 1.00 1.00 1.00 1.00 call qword ptr [rip + memset@GOTPCREL] + - - - - 1.00 - 0.50 0.50 mov qword ptr [rbx + 16], r15 + - - 0.01 0.99 - - - - mov al, 1 + - - 1.00 - - - - - add rsp, 32 + - - - - - - 0.50 0.50 pop rbx + - - - - - - 0.50 0.50 pop r12 + - - - - - - 0.50 0.50 pop r13 + - - - - - - 0.50 0.50 pop r14 + - - - - - - 0.50 0.50 pop r15 + - - - - - 1.00 - - ret + - - 1.00 - - - - - mov r13, rsi + - - 0.01 0.99 - - - - lea rcx, [rax + rax] + - - 0.99 0.01 - - - - cmp r15, rcx + - - 2.00 0.01 - 0.99 - - cmova rcx, r15 + - - 0.01 0.99 - - - - cmp rcx, 5 + - - 0.01 0.99 - - - - mov r14d, 4 + - - 1.00 0.01 - 0.99 - - cmovae r14, rcx + - - - - - - 0.50 0.50 mov rdx, qword ptr [rbx + 8] + - - 0.01 0.99 - - - - lea rdi, [rsp + 8] + - - - 1.00 - - - - mov rsi, rax + - - - 0.01 - 0.99 - - mov rcx, r14 + - - - - 1.00 1.00 0.50 0.50 call <alloc::raw_vec::RawVecInner>::finish_grow + - - - 0.99 - 0.01 0.50 0.50 cmp dword ptr [rsp + 8], 1 + - - - - - 1.00 - - je .LBB6_2 + - - - - - - 0.50 0.50 mov rax, qword ptr [rsp + 16] + - - - - 1.00 - 0.49 0.51 mov qword ptr [rbx + 8], rax + - - - - 1.00 - 0.51 0.49 mov qword ptr [rbx], r14 + - - 0.99 0.01 - - - - mov rsi, r13 + - - - - - 1.00 - - jmp .LBB6_4 diff --git a/rust/zerocopy/benches/formats/coco_dynamic_padding.rs b/rust/zerocopy/benches/formats/coco_dynamic_padding.rs new file mode 100644 index 000000000000..e494bce67312 --- /dev/null +++ b/rust/zerocopy/benches/formats/coco_dynamic_padding.rs @@ -0,0 +1,24 @@ +use zerocopy_derive::*; + +// The only valid value of this type are the bytes `0xC0C0`. +#[derive(TryFromBytes, KnownLayout, Immutable)] +#[repr(u16)] +pub enum C0C0 { + _XC0C0 = 0xC0C0, +} + +#[derive(FromBytes, KnownLayout, Immutable, SplitAt)] +#[repr(C, align(4))] +pub struct Packet<Magic> { + magic_number: Magic, + milk: u8, + mug_size: u8, + temperature: [u8; 5], + marshmallows: [[u8; 3]], +} + +/// A packet begining with the magic number `0xC0C0`. +pub type CocoPacket = Packet<C0C0>; + +/// A packet beginning with any two initialized bytes. +pub type LocoPacket = Packet<[u8; 2]>; diff --git a/rust/zerocopy/benches/formats/coco_dynamic_size.rs b/rust/zerocopy/benches/formats/coco_dynamic_size.rs new file mode 100644 index 000000000000..59364638e66c --- /dev/null +++ b/rust/zerocopy/benches/formats/coco_dynamic_size.rs @@ -0,0 +1,27 @@ +use zerocopy_derive::*; + +// The only valid value of this type are the bytes `0xC0C0`. +#[derive(TryFromBytes, KnownLayout, Immutable, IntoBytes)] +#[repr(u16)] +pub enum C0C0 { + _XC0C0 = 0xC0C0, +} + +macro_rules! define_packet { + ($name: ident, $trait: ident, $leading_field: ty) => { + #[derive($trait, KnownLayout, Immutable, IntoBytes, SplitAt)] + #[repr(C, align(2))] + pub struct $name { + magic_number: $leading_field, + mug_size: u8, + temperature: u8, + marshmallows: [[u8; 2]], + } + }; +} + +/// Packet begins with bytes 0xC0C0. +define_packet!(CocoPacket, TryFromBytes, C0C0); + +/// Packet begins with any two bytes. +define_packet!(LocoPacket, FromBytes, [u8; 2]); diff --git a/rust/zerocopy/benches/formats/coco_static_size.rs b/rust/zerocopy/benches/formats/coco_static_size.rs new file mode 100644 index 000000000000..0839497e1748 --- /dev/null +++ b/rust/zerocopy/benches/formats/coco_static_size.rs @@ -0,0 +1,27 @@ +use zerocopy_derive::*; + +// The only valid value of this type are the bytes `0xC0C0`. +#[derive(TryFromBytes, KnownLayout, Immutable, IntoBytes)] +#[repr(u16)] +pub enum C0C0 { + _XC0C0 = 0xC0C0, +} + +macro_rules! define_packet { + ($name: ident, $trait: ident, $leading_field: ty) => { + #[derive($trait, KnownLayout, Immutable, IntoBytes)] + #[repr(C, align(2))] + pub struct $name { + magic_number: $leading_field, + mug_size: u8, + temperature: u8, + marshmallows: [u8; 2], + } + }; +} + +/// Packet begins with bytes 0xC0C0. +define_packet!(CocoPacket, TryFromBytes, C0C0); + +/// Packet begins with any two bytes. +define_packet!(LocoPacket, FromBytes, [u8; 2]); diff --git a/rust/zerocopy/benches/insert_vec_zeroed.rs b/rust/zerocopy/benches/insert_vec_zeroed.rs new file mode 100644 index 000000000000..a5d685c2b027 --- /dev/null +++ b/rust/zerocopy/benches/insert_vec_zeroed.rs @@ -0,0 +1,13 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_insert_vec_zeroed( + v: &mut Vec<format::LocoPacket>, + position: usize, + additional: usize, +) -> Option<()> { + FromZeros::insert_vec_zeroed(v, position, additional).ok() +} diff --git a/rust/zerocopy/benches/insert_vec_zeroed.x86-64 b/rust/zerocopy/benches/insert_vec_zeroed.x86-64 new file mode 100644 index 000000000000..9db87403cb55 --- /dev/null +++ b/rust/zerocopy/benches/insert_vec_zeroed.x86-64 @@ -0,0 +1,79 @@ +bench_insert_vec_zeroed: + push rbp + push r15 + push r14 + push r13 + push r12 + push rbx + sub rsp, 24 + mov r12, qword ptr [rdi + 16] + mov r13, r12 + sub r13, rsi + jb .LBB6_10 + mov rbx, rdi + mov rax, qword ptr [rdi] + mov rcx, rax + sub rcx, r12 + cmp rdx, rcx + jbe .LBB6_4 + add r12, rdx + jae .LBB6_7 +.LBB6_3: + xor eax, eax + jmp .LBB6_6 +.LBB6_4: + mov rax, qword ptr [rbx + 8] + add r12, rdx +.LBB6_5: + lea rcx, [rsi + 2*rsi] + lea r14, [rax + 2*rcx] + add rdx, rdx + lea r15, [rdx + 2*rdx] + lea rdi, [r14 + r15] + add r13, r13 + lea rdx, [2*r13] + add rdx, r13 + mov rsi, r14 + call qword ptr [rip + memmove@GOTPCREL] + mov rdi, r14 + xor esi, esi + mov rdx, r15 + call qword ptr [rip + memset@GOTPCREL] + mov qword ptr [rbx + 16], r12 + mov al, 1 +.LBB6_6: + add rsp, 24 + pop rbx + pop r12 + pop r13 + pop r14 + pop r15 + pop rbp + ret +.LBB6_7: + mov r15, rsi + mov rbp, rdx + lea rcx, [rax + rax] + cmp r12, rcx + cmova rcx, r12 + cmp rcx, 5 + mov r14d, 4 + cmovae r14, rcx + mov rdx, qword ptr [rbx + 8] + mov rdi, rsp + mov rsi, rax + mov rcx, r14 + call <alloc::raw_vec::RawVecInner>::finish_grow + cmp dword ptr [rsp], 1 + je .LBB6_3 + mov rax, qword ptr [rsp + 8] + mov qword ptr [rbx + 8], rax + mov qword ptr [rbx], r14 + mov rdx, rbp + mov rsi, r15 + jmp .LBB6_5 +.LBB6_10: + lea rdi, [rip + .Lanon.HASH.1] + lea rdx, [rip + .Lanon.HASH.3] + mov esi, 37 + call qword ptr [rip + core::panicking::panic@GOTPCREL] diff --git a/rust/zerocopy/benches/insert_vec_zeroed.x86-64.mca b/rust/zerocopy/benches/insert_vec_zeroed.x86-64.mca new file mode 100644 index 000000000000..665240667844 --- /dev/null +++ b/rust/zerocopy/benches/insert_vec_zeroed.x86-64.mca @@ -0,0 +1,183 @@ +Iterations: 100 +Instructions: 7200 +Total Cycles: 7648 +Total uOps: 9300 + +Dispatch Width: 4 +uOps Per Cycle: 1.22 +IPC: 0.94 +Block RThroughput: 23.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push rbp + 2 5 1.00 * push r15 + 2 5 1.00 * push r14 + 2 5 1.00 * push r13 + 2 5 1.00 * push r12 + 2 5 1.00 * push rbx + 1 1 0.33 sub rsp, 24 + 1 5 0.50 * mov r12, qword ptr [rdi + 16] + 1 1 0.33 mov r13, r12 + 1 1 0.33 sub r13, rsi + 1 1 1.00 jb .LBB6_10 + 1 1 0.33 mov rbx, rdi + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 1 0.33 mov rcx, rax + 1 1 0.33 sub rcx, r12 + 1 1 0.33 cmp rdx, rcx + 1 1 1.00 jbe .LBB6_4 + 1 1 0.33 add r12, rdx + 1 1 1.00 jae .LBB6_7 + 1 0 0.25 xor eax, eax + 1 1 1.00 jmp .LBB6_6 + 1 5 0.50 * mov rax, qword ptr [rbx + 8] + 1 1 0.33 add r12, rdx + 1 1 0.50 lea rcx, [rsi + 2*rsi] + 1 1 0.50 lea r14, [rax + 2*rcx] + 1 1 0.33 add rdx, rdx + 1 1 0.50 lea r15, [rdx + 2*rdx] + 1 1 0.50 lea rdi, [r14 + r15] + 1 1 0.33 add r13, r13 + 1 1 0.50 lea rdx, [2*r13] + 1 1 0.33 add rdx, r13 + 1 1 0.33 mov rsi, r14 + 4 7 1.00 * call qword ptr [rip + memmove@GOTPCREL] + 1 1 0.33 mov rdi, r14 + 1 0 0.25 xor esi, esi + 1 1 0.33 mov rdx, r15 + 4 7 1.00 * call qword ptr [rip + memset@GOTPCREL] + 1 1 1.00 * mov qword ptr [rbx + 16], r12 + 1 1 0.33 mov al, 1 + 1 1 0.33 add rsp, 24 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r12 + 1 6 0.50 * pop r13 + 1 6 0.50 * pop r14 + 1 6 0.50 * pop r15 + 1 6 0.50 * pop rbp + 1 1 1.00 U ret + 1 1 0.33 mov r15, rsi + 1 1 0.33 mov rbp, rdx + 1 1 0.50 lea rcx, [rax + rax] + 1 1 0.33 cmp r12, rcx + 3 3 1.00 cmova rcx, r12 + 1 1 0.33 cmp rcx, 5 + 1 1 0.33 mov r14d, 4 + 2 2 0.67 cmovae r14, rcx + 1 5 0.50 * mov rdx, qword ptr [rbx + 8] + 1 1 0.33 mov rdi, rsp + 1 1 0.33 mov rsi, rax + 1 1 0.33 mov rcx, r14 + 3 5 1.00 call <alloc::raw_vec::RawVecInner>::finish_grow + 2 6 0.50 * cmp dword ptr [rsp], 1 + 1 1 1.00 je .LBB6_3 + 1 5 0.50 * mov rax, qword ptr [rsp + 8] + 1 1 1.00 * mov qword ptr [rbx + 8], rax + 1 1 1.00 * mov qword ptr [rbx], r14 + 1 1 0.33 mov rdx, rbp + 1 1 0.33 mov rsi, r15 + 1 1 1.00 jmp .LBB6_5 + 1 1 0.50 lea rdi, [rip + .Lanon.HASH.1] + 1 1 0.50 lea rdx, [rip + .Lanon.HASH.3] + 1 1 0.33 mov esi, 37 + 4 7 1.00 * call qword ptr [rip + core::panicking::panic@GOTPCREL] + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 17.02 16.50 13.00 19.48 14.00 14.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - 0.98 0.02 push rbp + - - - - 1.00 - 0.02 0.98 push r15 + - - - - 1.00 - 0.99 0.01 push r14 + - - - - 1.00 - 0.01 0.99 push r13 + - - - - 1.00 - 0.99 0.01 push r12 + - - - - 1.00 - 0.01 0.99 push rbx + - - 0.49 0.51 - - - - sub rsp, 24 + - - - - - - 0.04 0.96 mov r12, qword ptr [rdi + 16] + - - 0.49 0.50 - 0.01 - - mov r13, r12 + - - 0.48 0.51 - 0.01 - - sub r13, rsi + - - - - - 1.00 - - jb .LBB6_10 + - - 0.49 0.49 - 0.02 - - mov rbx, rdi + - - - - - - 0.97 0.03 mov rax, qword ptr [rdi] + - - 0.51 0.49 - - - - mov rcx, rax + - - 0.49 0.02 - 0.49 - - sub rcx, r12 + - - 0.49 0.50 - 0.01 - - cmp rdx, rcx + - - - - - 1.00 - - jbe .LBB6_4 + - - 0.02 0.49 - 0.49 - - add r12, rdx + - - - - - 1.00 - - jae .LBB6_7 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - jmp .LBB6_6 + - - - - - - 0.97 0.03 mov rax, qword ptr [rbx + 8] + - - 0.51 0.49 - - - - add r12, rdx + - - 0.49 0.51 - - - - lea rcx, [rsi + 2*rsi] + - - 0.50 0.50 - - - - lea r14, [rax + 2*rcx] + - - 0.51 0.49 - - - - add rdx, rdx + - - 0.50 0.50 - - - - lea r15, [rdx + 2*rdx] + - - 0.49 0.51 - - - - lea rdi, [r14 + r15] + - - 0.50 0.49 - 0.01 - - add r13, r13 + - - 0.51 0.49 - - - - lea rdx, [2*r13] + - - 0.01 0.01 - 0.98 - - add rdx, r13 + - - 0.01 - - 0.99 - - mov rsi, r14 + - - - - 1.00 1.00 1.98 0.02 call qword ptr [rip + memmove@GOTPCREL] + - - 0.49 0.50 - 0.01 - - mov rdi, r14 + - - - - - - - - xor esi, esi + - - 0.50 0.50 - - - - mov rdx, r15 + - - - - 1.00 1.00 1.96 0.04 call qword ptr [rip + memset@GOTPCREL] + - - - - 1.00 - 0.01 0.99 mov qword ptr [rbx + 16], r12 + - - 0.50 - - 0.50 - - mov al, 1 + - - 0.51 0.49 - - - - add rsp, 24 + - - - - - - 0.02 0.98 pop rbx + - - - - - - 0.03 0.97 pop r12 + - - - - - - 0.03 0.97 pop r13 + - - - - - - 0.97 0.03 pop r14 + - - - - - - 0.03 0.97 pop r15 + - - - - - - 0.01 0.99 pop rbp + - - - - - 1.00 - - ret + - - 0.49 0.51 - - - - mov r15, rsi + - - 0.51 0.49 - - - - mov rbp, rdx + - - 0.49 0.51 - - - - lea rcx, [rax + rax] + - - 0.49 0.50 - 0.01 - - cmp r12, rcx + - - 1.04 0.50 - 1.46 - - cmova rcx, r12 + - - 0.49 0.49 - 0.02 - - cmp rcx, 5 + - - 0.50 - - 0.50 - - mov r14d, 4 + - - 0.50 0.51 - 0.99 - - cmovae r14, rcx + - - - - - - 0.97 0.03 mov rdx, qword ptr [rbx + 8] + - - - 0.51 - 0.49 - - mov rdi, rsp + - - 0.01 0.50 - 0.49 - - mov rsi, rax + - - 0.49 0.50 - 0.01 - - mov rcx, r14 + - - - - 1.00 1.00 0.99 0.01 call <alloc::raw_vec::RawVecInner>::finish_grow + - - 0.51 0.49 - - 0.50 0.50 cmp dword ptr [rsp], 1 + - - - - - 1.00 - - je .LBB6_3 + - - - - - - 0.50 0.50 mov rax, qword ptr [rsp + 8] + - - - - 1.00 - 0.99 0.01 mov qword ptr [rbx + 8], rax + - - - - 1.00 - 0.01 0.99 mov qword ptr [rbx], r14 + - - 0.49 0.50 - 0.01 - - mov rdx, rbp + - - 0.50 0.01 - 0.49 - - mov rsi, r15 + - - - - - 1.00 - - jmp .LBB6_5 + - - 0.01 0.99 - - - - lea rdi, [rip + .Lanon.HASH.1] + - - 0.99 0.01 - - - - lea rdx, [rip + .Lanon.HASH.3] + - - 0.02 0.49 - 0.49 - - mov esi, 37 + - - - - 1.00 1.00 0.02 1.98 call qword ptr [rip + core::panicking::panic@GOTPCREL] diff --git a/rust/zerocopy/benches/new_box_zeroed.rs b/rust/zerocopy/benches/new_box_zeroed.rs new file mode 100644 index 000000000000..aa9a66cce353 --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_new_box_zeroed() -> Option<Box<format::LocoPacket>> { + FromZeros::new_box_zeroed().ok() +} diff --git a/rust/zerocopy/benches/new_box_zeroed.x86-64 b/rust/zerocopy/benches/new_box_zeroed.x86-64 new file mode 100644 index 000000000000..ef74ea5388ac --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed.x86-64 @@ -0,0 +1,7 @@ +bench_new_box_zeroed: + push rax + call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + mov edi, 6 + mov esi, 2 + pop rax + jmp qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] diff --git a/rust/zerocopy/benches/new_box_zeroed.x86-64.mca b/rust/zerocopy/benches/new_box_zeroed.x86-64.mca new file mode 100644 index 000000000000..05afa7feb0b8 --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed.x86-64.mca @@ -0,0 +1,51 @@ +Iterations: 100 +Instructions: 600 +Total Cycles: 1197 +Total uOps: 1100 + +Dispatch Width: 4 +uOps Per Cycle: 0.92 +IPC: 0.50 +Block RThroughput: 2.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push rax + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + 1 1 0.33 mov edi, 6 + 1 1 0.33 mov esi, 2 + 1 6 0.50 * pop rax + 2 6 1.00 * jmp qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.99 1.00 2.00 2.01 2.07 2.93 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - 0.93 0.07 push rax + - - - - 1.00 1.00 0.12 1.88 call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + - - 0.99 - - 0.01 - - mov edi, 6 + - - - 1.00 - - - - mov esi, 2 + - - - - - - 0.94 0.06 pop rax + - - - - - 1.00 0.08 0.92 jmp qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] diff --git a/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..0afde999bff8 --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_new_box_zeroed_with_elems_dynamic_padding( + count: usize, +) -> Option<Box<format::LocoPacket>> { + FromZeros::new_box_zeroed_with_elems(count).ok() +} diff --git a/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..22a8d048ce0f --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,24 @@ +bench_new_box_zeroed_with_elems_dynamic_padding: + push r14 + push rbx + push rax + mov rbx, rdi + movabs rax, 3074457345618258598 + cmp rdi, rax + ja .LBB5_1 + lea r14, [rbx + 2*rbx] + or r14, 3 + add r14, 9 + call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + mov esi, 4 + mov rdi, r14 + call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + jmp .LBB5_3 +.LBB5_1: + xor eax, eax +.LBB5_3: + mov rdx, rbx + add rsp, 8 + pop rbx + pop r14 + ret diff --git a/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..e6efaeded476 --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,81 @@ +Iterations: 100 +Instructions: 2100 +Total Cycles: 2990 +Total uOps: 3000 + +Dispatch Width: 4 +uOps Per Cycle: 1.00 +IPC: 0.70 +Block RThroughput: 7.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push r14 + 2 5 1.00 * push rbx + 2 5 1.00 * push rax + 1 1 0.33 mov rbx, rdi + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdi, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.50 lea r14, [rbx + 2*rbx] + 1 1 0.33 or r14, 3 + 1 1 0.33 add r14, 9 + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + 1 1 0.33 mov esi, 4 + 1 1 0.33 mov rdi, r14 + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + 1 1 1.00 jmp .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov rdx, rbx + 1 1 0.33 add rsp, 8 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r14 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.49 4.50 5.00 6.01 4.50 4.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - 0.50 0.50 push r14 + - - - - 1.00 - 0.50 0.50 push rbx + - - - - 1.00 - 0.50 0.50 push rax + - - 0.49 0.50 - 0.01 - - mov rbx, rdi + - - 0.50 0.50 - - - - movabs rax, 3074457345618258598 + - - 0.50 0.50 - - - - cmp rdi, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.50 0.50 - - - - lea r14, [rbx + 2*rbx] + - - 0.50 0.50 - - - - or r14, 3 + - - 0.50 - - 0.50 - - add r14, 9 + - - - - 1.00 1.00 1.00 1.00 call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + - - - 0.50 - 0.50 - - mov esi, 4 + - - 0.50 0.50 - - - - mov rdi, r14 + - - - - 1.00 1.00 1.00 1.00 call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + - - - - - 1.00 - - jmp .LBB5_3 + - - - - - - - - xor eax, eax + - - 0.51 0.49 - - - - mov rdx, rbx + - - 0.49 0.51 - - - - add rsp, 8 + - - - - - - 0.50 0.50 pop rbx + - - - - - - 0.50 0.50 pop r14 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.rs b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..1b12ca220692 --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_new_box_zeroed_with_elems_dynamic_size(count: usize) -> Option<Box<format::LocoPacket>> { + FromZeros::new_box_zeroed_with_elems(count).ok() +} diff --git a/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..bff15e55ad9f --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64 @@ -0,0 +1,22 @@ +bench_new_box_zeroed_with_elems_dynamic_size: + push r14 + push rbx + push rax + mov rbx, rdi + movabs rax, 4611686018427387901 + cmp rdi, rax + ja .LBB5_1 + lea r14, [2*rbx + 4] + call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + mov esi, 2 + mov rdi, r14 + call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + jmp .LBB5_3 +.LBB5_1: + xor eax, eax +.LBB5_3: + mov rdx, rbx + add rsp, 8 + pop rbx + pop r14 + ret diff --git a/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..153d36c01ce0 --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 2990 +Total uOps: 2800 + +Dispatch Width: 4 +uOps Per Cycle: 0.94 +IPC: 0.64 +Block RThroughput: 7.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push r14 + 2 5 1.00 * push rbx + 2 5 1.00 * push rax + 1 1 0.33 mov rbx, rdi + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdi, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.50 lea r14, [2*rbx + 4] + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + 1 1 0.33 mov esi, 2 + 1 1 0.33 mov rdi, r14 + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + 1 1 1.00 jmp .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov rdx, rbx + 1 1 0.33 add rsp, 8 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r14 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.97 3.98 5.00 5.05 4.50 4.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - 0.50 0.50 push r14 + - - - - 1.00 - 0.50 0.50 push rbx + - - - - 1.00 - 0.50 0.50 push rax + - - 0.05 0.94 - 0.01 - - mov rbx, rdi + - - 0.94 0.06 - - - - movabs rax, 4611686018427387901 + - - 0.06 0.94 - - - - cmp rdi, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.94 0.06 - - - - lea r14, [2*rbx + 4] + - - - - 1.00 1.00 1.00 1.00 call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + - - 0.98 0.02 - - - - mov esi, 2 + - - 0.02 0.94 - 0.04 - - mov rdi, r14 + - - - - 1.00 1.00 1.00 1.00 call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + - - - - - 1.00 - - jmp .LBB5_3 + - - - - - - - - xor eax, eax + - - 0.94 0.06 - - - - mov rdx, rbx + - - 0.04 0.96 - - - - add rsp, 8 + - - - - - - 0.50 0.50 pop rbx + - - - - - - 0.50 0.50 pop r14 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/new_vec_zeroed.rs b/rust/zerocopy/benches/new_vec_zeroed.rs new file mode 100644 index 000000000000..3d95b2b24d8d --- /dev/null +++ b/rust/zerocopy/benches/new_vec_zeroed.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_new_vec_zeroed(len: usize) -> Option<Vec<format::LocoPacket>> { + FromZeros::new_vec_zeroed(len).ok() +} diff --git a/rust/zerocopy/benches/new_vec_zeroed.x86-64 b/rust/zerocopy/benches/new_vec_zeroed.x86-64 new file mode 100644 index 000000000000..b5c083aa0d36 --- /dev/null +++ b/rust/zerocopy/benches/new_vec_zeroed.x86-64 @@ -0,0 +1,40 @@ +bench_new_vec_zeroed: + mov rax, rdi + movabs rcx, 1537228672809129301 + cmp rsi, rcx + ja .LBB5_5 + test rsi, rsi + je .LBB5_2 + push r15 + push r14 + push rbx + lea rcx, [rsi + rsi] + lea rbx, [rcx + 2*rcx] + mov r14, rax + mov r15, rsi + call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + mov esi, 2 + mov rdi, rbx + call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + mov rsi, r15 + mov rcx, rax + mov rax, r14 + test rcx, rcx + pop rbx + pop r14 + pop r15 + je .LBB5_5 + mov qword ptr [rax], rsi + mov qword ptr [rax + 8], rcx + mov qword ptr [rax + 16], rsi + ret +.LBB5_5: + movabs rcx, -9223372036854775808 + mov qword ptr [rax], rcx + ret +.LBB5_2: + mov ecx, 2 + mov qword ptr [rax], rsi + mov qword ptr [rax + 8], rcx + mov qword ptr [rax + 16], rsi + ret diff --git a/rust/zerocopy/benches/new_vec_zeroed.x86-64.mca b/rust/zerocopy/benches/new_vec_zeroed.x86-64.mca new file mode 100644 index 000000000000..b4fb4544ec39 --- /dev/null +++ b/rust/zerocopy/benches/new_vec_zeroed.x86-64.mca @@ -0,0 +1,113 @@ +Iterations: 100 +Instructions: 3700 +Total Cycles: 3486 +Total uOps: 4600 + +Dispatch Width: 4 +uOps Per Cycle: 1.32 +IPC: 1.06 +Block RThroughput: 12.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 movabs rcx, 1537228672809129301 + 1 1 0.33 cmp rsi, rcx + 1 1 1.00 ja .LBB5_5 + 1 1 0.33 test rsi, rsi + 1 1 1.00 je .LBB5_2 + 2 5 1.00 * push r15 + 2 5 1.00 * push r14 + 2 5 1.00 * push rbx + 1 1 0.50 lea rcx, [rsi + rsi] + 1 1 0.50 lea rbx, [rcx + 2*rcx] + 1 1 0.33 mov r14, rax + 1 1 0.33 mov r15, rsi + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + 1 1 0.33 mov esi, 2 + 1 1 0.33 mov rdi, rbx + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + 1 1 0.33 mov rsi, r15 + 1 1 0.33 mov rcx, rax + 1 1 0.33 mov rax, r14 + 1 1 0.33 test rcx, rcx + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r14 + 1 6 0.50 * pop r15 + 1 1 1.00 je .LBB5_5 + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 * mov qword ptr [rax + 8], rcx + 1 1 1.00 * mov qword ptr [rax + 16], rsi + 1 1 1.00 U ret + 1 1 0.33 movabs rcx, -9223372036854775808 + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 U ret + 1 1 0.33 mov ecx, 2 + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 * mov qword ptr [rax + 8], rcx + 1 1 1.00 * mov qword ptr [rax + 16], rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.99 6.99 12.00 10.02 8.00 9.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.01 0.98 - 0.01 - - mov rax, rdi + - - 0.98 0.02 - - - - movabs rcx, 1537228672809129301 + - - 0.02 0.98 - - - - cmp rsi, rcx + - - - - - 1.00 - - ja .LBB5_5 + - - 0.98 - - 0.02 - - test rsi, rsi + - - - - - 1.00 - - je .LBB5_2 + - - - - 1.00 - - 1.00 push r15 + - - - - 1.00 - 1.00 - push r14 + - - - - 1.00 - - 1.00 push rbx + - - - 1.00 - - - - lea rcx, [rsi + rsi] + - - - 1.00 - - - - lea rbx, [rcx + 2*rcx] + - - 1.00 - - - - - mov r14, rax + - - 1.00 - - - - - mov r15, rsi + - - - - 1.00 1.00 2.00 - call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + - - - 0.01 - 0.99 - - mov esi, 2 + - - 0.01 0.99 - - - - mov rdi, rbx + - - - - 1.00 1.00 - 2.00 call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + - - 0.01 - - 0.99 - - mov rsi, r15 + - - 0.99 0.01 - - - - mov rcx, rax + - - - 0.99 - 0.01 - - mov rax, r14 + - - 0.99 0.01 - - - - test rcx, rcx + - - - - - - - 1.00 pop rbx + - - - - - - 1.00 - pop r14 + - - - - - - - 1.00 pop r15 + - - - - - 1.00 - - je .LBB5_5 + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - 1.00 - - 1.00 mov qword ptr [rax + 8], rcx + - - - - 1.00 - 1.00 - mov qword ptr [rax + 16], rsi + - - - - - 1.00 - - ret + - - 0.01 0.99 - - - - movabs rcx, -9223372036854775808 + - - - - 1.00 - - 1.00 mov qword ptr [rax], rcx + - - - - - 1.00 - - ret + - - 0.99 0.01 - - - - mov ecx, 2 + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - 1.00 - - 1.00 mov qword ptr [rax + 8], rcx + - - - - 1.00 - 1.00 - mov qword ptr [rax + 16], rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/new_zeroed.rs b/rust/zerocopy/benches/new_zeroed.rs new file mode 100644 index 000000000000..b49f62edb146 --- /dev/null +++ b/rust/zerocopy/benches/new_zeroed.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_new_zeroed() -> format::LocoPacket { + FromZeros::new_zeroed() +} diff --git a/rust/zerocopy/benches/new_zeroed.x86-64 b/rust/zerocopy/benches/new_zeroed.x86-64 new file mode 100644 index 000000000000..b4d305e41fff --- /dev/null +++ b/rust/zerocopy/benches/new_zeroed.x86-64 @@ -0,0 +1,3 @@ +bench_new_zeroed: + xor eax, eax + ret diff --git a/rust/zerocopy/benches/new_zeroed.x86-64.mca b/rust/zerocopy/benches/new_zeroed.x86-64.mca new file mode 100644 index 000000000000..44583ca3089f --- /dev/null +++ b/rust/zerocopy/benches/new_zeroed.x86-64.mca @@ -0,0 +1,43 @@ +Iterations: 100 +Instructions: 200 +Total Cycles: 103 +Total uOps: 200 + +Dispatch Width: 4 +uOps Per Cycle: 1.94 +IPC: 1.94 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - - - - 1.00 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/read_from_bytes.rs b/rust/zerocopy/benches/read_from_bytes.rs new file mode 100644 index 000000000000..8a3baddad9cb --- /dev/null +++ b/rust/zerocopy/benches/read_from_bytes.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_read_from_bytes_static_size(source: &[u8]) -> Option<format::LocoPacket> { + zerocopy::FromBytes::read_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/read_from_bytes.x86-64 b/rust/zerocopy/benches/read_from_bytes.x86-64 new file mode 100644 index 000000000000..9082d79f1fd5 --- /dev/null +++ b/rust/zerocopy/benches/read_from_bytes.x86-64 @@ -0,0 +1,15 @@ +bench_read_from_bytes_static_size: + mov rcx, rsi + cmp rsi, 6 + jne .LBB5_2 + mov eax, dword ptr [rdi] + movzx ecx, word ptr [rdi + 4] + shl rcx, 32 + or rcx, rax +.LBB5_2: + shl rcx, 16 + inc rcx + xor eax, eax + cmp rsi, 6 + cmove rax, rcx + ret diff --git a/rust/zerocopy/benches/read_from_bytes.x86-64.mca b/rust/zerocopy/benches/read_from_bytes.x86-64.mca new file mode 100644 index 000000000000..77e787c19032 --- /dev/null +++ b/rust/zerocopy/benches/read_from_bytes.x86-64.mca @@ -0,0 +1,65 @@ +Iterations: 100 +Instructions: 1300 +Total Cycles: 377 +Total uOps: 1400 + +Dispatch Width: 4 +uOps Per Cycle: 3.71 +IPC: 3.45 +Block RThroughput: 3.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rcx, rsi + 1 1 0.33 cmp rsi, 6 + 1 1 1.00 jne .LBB5_2 + 1 5 0.50 * mov eax, dword ptr [rdi] + 1 5 0.50 * movzx ecx, word ptr [rdi + 4] + 1 1 0.50 shl rcx, 32 + 1 1 0.33 or rcx, rax + 1 1 0.50 shl rcx, 16 + 1 1 0.33 inc rcx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp rsi, 6 + 2 2 0.67 cmove rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.66 3.67 - 3.67 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.63 0.36 - 0.01 - - mov rcx, rsi + - - 0.05 0.05 - 0.90 - - cmp rsi, 6 + - - - - - 1.00 - - jne .LBB5_2 + - - - - - - - 1.00 mov eax, dword ptr [rdi] + - - - - - - 1.00 - movzx ecx, word ptr [rdi + 4] + - - 0.97 - - 0.03 - - shl rcx, 32 + - - 0.02 0.35 - 0.63 - - or rcx, rax + - - 0.98 - - 0.02 - - shl rcx, 16 + - - - 0.98 - 0.02 - - inc rcx + - - - - - - - - xor eax, eax + - - 0.03 0.93 - 0.04 - - cmp rsi, 6 + - - 0.98 1.00 - 0.02 - - cmove rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/read_from_prefix.rs b/rust/zerocopy/benches/read_from_prefix.rs new file mode 100644 index 000000000000..d49bf80ab785 --- /dev/null +++ b/rust/zerocopy/benches/read_from_prefix.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_read_from_prefix_static_size(source: &[u8]) -> Option<format::LocoPacket> { + match zerocopy::FromBytes::read_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/read_from_prefix.x86-64 b/rust/zerocopy/benches/read_from_prefix.x86-64 new file mode 100644 index 000000000000..c75b06c0c22a --- /dev/null +++ b/rust/zerocopy/benches/read_from_prefix.x86-64 @@ -0,0 +1,14 @@ +bench_read_from_prefix_static_size: + cmp rsi, 5 + jbe .LBB5_2 + mov eax, dword ptr [rdi] + movzx edi, word ptr [rdi + 4] + shl rdi, 32 + or rdi, rax +.LBB5_2: + shl rdi, 16 + inc rdi + xor eax, eax + cmp rsi, 6 + cmovae rax, rdi + ret diff --git a/rust/zerocopy/benches/read_from_prefix.x86-64.mca b/rust/zerocopy/benches/read_from_prefix.x86-64.mca new file mode 100644 index 000000000000..04e76cdd07ee --- /dev/null +++ b/rust/zerocopy/benches/read_from_prefix.x86-64.mca @@ -0,0 +1,63 @@ +Iterations: 100 +Instructions: 1200 +Total Cycles: 905 +Total uOps: 1300 + +Dispatch Width: 4 +uOps Per Cycle: 1.44 +IPC: 1.33 +Block RThroughput: 3.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 cmp rsi, 5 + 1 1 1.00 jbe .LBB5_2 + 1 5 0.50 * mov eax, dword ptr [rdi] + 1 5 0.50 * movzx edi, word ptr [rdi + 4] + 1 1 0.50 shl rdi, 32 + 1 1 0.33 or rdi, rax + 1 1 0.50 shl rdi, 16 + 1 1 0.33 inc rdi + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp rsi, 6 + 2 2 0.67 cmovae rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.32 3.32 - 3.36 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.05 0.94 - 0.01 - - cmp rsi, 5 + - - - - - 1.00 - - jbe .LBB5_2 + - - - - - - - 1.00 mov eax, dword ptr [rdi] + - - - - - - 1.00 - movzx edi, word ptr [rdi + 4] + - - 0.71 - - 0.29 - - shl rdi, 32 + - - - 0.64 - 0.36 - - or rdi, rax + - - 1.00 - - - - - shl rdi, 16 + - - 0.31 0.40 - 0.29 - - inc rdi + - - - - - - - - xor eax, eax + - - 0.34 0.35 - 0.31 - - cmp rsi, 6 + - - 0.91 0.99 - 0.10 - - cmovae rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/read_from_suffix.rs b/rust/zerocopy/benches/read_from_suffix.rs new file mode 100644 index 000000000000..4eaadb0d4dff --- /dev/null +++ b/rust/zerocopy/benches/read_from_suffix.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_read_from_suffix_static_size(source: &[u8]) -> Option<format::LocoPacket> { + match zerocopy::FromBytes::read_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/read_from_suffix.x86-64 b/rust/zerocopy/benches/read_from_suffix.x86-64 new file mode 100644 index 000000000000..5cff2a0e2f36 --- /dev/null +++ b/rust/zerocopy/benches/read_from_suffix.x86-64 @@ -0,0 +1,15 @@ +bench_read_from_suffix_static_size: + mov rcx, rsi + cmp rsi, 6 + jb .LBB5_2 + mov eax, dword ptr [rdi + rsi - 6] + movzx ecx, word ptr [rdi + rsi - 2] + shl rcx, 32 + or rcx, rax +.LBB5_2: + shl rcx, 16 + inc rcx + xor eax, eax + cmp rsi, 6 + cmovae rax, rcx + ret diff --git a/rust/zerocopy/benches/read_from_suffix.x86-64.mca b/rust/zerocopy/benches/read_from_suffix.x86-64.mca new file mode 100644 index 000000000000..0107de89562a --- /dev/null +++ b/rust/zerocopy/benches/read_from_suffix.x86-64.mca @@ -0,0 +1,65 @@ +Iterations: 100 +Instructions: 1300 +Total Cycles: 377 +Total uOps: 1400 + +Dispatch Width: 4 +uOps Per Cycle: 3.71 +IPC: 3.45 +Block RThroughput: 3.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rcx, rsi + 1 1 0.33 cmp rsi, 6 + 1 1 1.00 jb .LBB5_2 + 1 5 0.50 * mov eax, dword ptr [rdi + rsi - 6] + 1 5 0.50 * movzx ecx, word ptr [rdi + rsi - 2] + 1 1 0.50 shl rcx, 32 + 1 1 0.33 or rcx, rax + 1 1 0.50 shl rcx, 16 + 1 1 0.33 inc rcx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp rsi, 6 + 2 2 0.67 cmovae rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.66 3.67 - 3.67 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.63 0.36 - 0.01 - - mov rcx, rsi + - - 0.05 0.05 - 0.90 - - cmp rsi, 6 + - - - - - 1.00 - - jb .LBB5_2 + - - - - - - - 1.00 mov eax, dword ptr [rdi + rsi - 6] + - - - - - - 1.00 - movzx ecx, word ptr [rdi + rsi - 2] + - - 0.97 - - 0.03 - - shl rcx, 32 + - - 0.02 0.35 - 0.63 - - or rcx, rax + - - 0.98 - - 0.02 - - shl rcx, 16 + - - - 0.98 - 0.02 - - inc rcx + - - - - - - - - xor eax, eax + - - 0.03 0.93 - 0.04 - - cmp rsi, 6 + - - 0.98 1.00 - 0.02 - - cmovae rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.rs b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.rs new file mode 100644 index 000000000000..29708df55b45 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_bytes_dynamic_padding(source: &[u8]) -> Option<&format::LocoPacket> { + zerocopy::FromBytes::ref_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 new file mode 100644 index 000000000000..e844a4608fac --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 @@ -0,0 +1,22 @@ +bench_ref_from_bytes_dynamic_padding: + test dil, 3 + jne .LBB5_3 + movabs rax, 9223372036854775804 + and rax, rsi + cmp rax, 9 + jb .LBB5_3 + add rax, -9 + movabs rcx, -6148914691236517205 + mul rcx + shr rdx + lea rax, [rdx + 2*rdx] + or rax, 3 + add rax, 9 + cmp rsi, rax + je .LBB5_4 +.LBB5_3: + xor edi, edi + mov rdx, rsi +.LBB5_4: + mov rax, rdi + ret diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..423ed38ba28d --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 645 +Total uOps: 2000 + +Dispatch Width: 4 +uOps Per Cycle: 3.10 +IPC: 2.95 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 test dil, 3 + 1 1 1.00 jne .LBB5_3 + 1 1 0.33 movabs rax, 9223372036854775804 + 1 1 0.33 and rax, rsi + 1 1 0.33 cmp rax, 9 + 1 1 1.00 jb .LBB5_3 + 1 1 0.33 add rax, -9 + 1 1 0.33 movabs rcx, -6148914691236517205 + 2 4 1.00 mul rcx + 1 1 0.50 shr rdx + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, 9 + 1 1 0.33 cmp rsi, rax + 1 1 1.00 je .LBB5_4 + 1 0 0.25 xor edi, edi + 1 1 0.33 mov rdx, rsi + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.32 6.33 - 6.35 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.64 0.35 - 0.01 - - test dil, 3 + - - - - - 1.00 - - jne .LBB5_3 + - - 0.34 0.65 - 0.01 - - movabs rax, 9223372036854775804 + - - 0.35 0.65 - - - - and rax, rsi + - - 0.33 0.34 - 0.33 - - cmp rax, 9 + - - - - - 1.00 - - jb .LBB5_3 + - - 0.35 - - 0.65 - - add rax, -9 + - - 0.97 0.01 - 0.02 - - movabs rcx, -6148914691236517205 + - - 1.00 1.00 - - - - mul rcx + - - 0.99 - - 0.01 - - shr rdx + - - 0.33 0.67 - - - - lea rax, [rdx + 2*rdx] + - - 0.34 0.66 - - - - or rax, 3 + - - 0.33 0.66 - 0.01 - - add rax, 9 + - - 0.01 0.99 - - - - cmp rsi, rax + - - - - - 1.00 - - je .LBB5_4 + - - - - - - - - xor edi, edi + - - 0.32 0.01 - 0.67 - - mov rdx, rsi + - - 0.02 0.34 - 0.64 - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_size.rs b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.rs new file mode 100644 index 000000000000..4eb4f970e365 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_bytes_dynamic_size(source: &[u8]) -> Option<&format::LocoPacket> { + zerocopy::FromBytes::ref_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 new file mode 100644 index 000000000000..cc905b76c06f --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 @@ -0,0 +1,20 @@ +bench_ref_from_bytes_dynamic_size: + mov rdx, rsi + cmp rsi, 4 + setb al + or al, dil + test al, 1 + je .LBB5_2 + xor eax, eax + ret +.LBB5_2: + lea rcx, [rdx - 4] + mov rsi, rcx + and rsi, -2 + add rsi, 4 + shr rcx + xor eax, eax + cmp rdx, rsi + cmove rdx, rcx + cmove rax, rdi + ret diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..68aea583e401 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca @@ -0,0 +1,75 @@ +Iterations: 100 +Instructions: 1800 +Total Cycles: 704 +Total uOps: 2000 + +Dispatch Width: 4 +uOps Per Cycle: 2.84 +IPC: 2.56 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rdx, rsi + 1 1 0.33 cmp rsi, 4 + 1 1 0.50 setb al + 1 1 0.33 or al, dil + 1 1 0.33 test al, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.50 lea rcx, [rdx - 4] + 1 1 0.33 mov rsi, rcx + 1 1 0.33 and rsi, -2 + 1 1 0.33 add rsi, 4 + 1 1 0.50 shr rcx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp rdx, rsi + 2 2 0.67 cmove rdx, rcx + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.97 5.98 - 6.05 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.97 0.01 - 0.02 - - mov rdx, rsi + - - 0.01 0.02 - 0.97 - - cmp rsi, 4 + - - 0.03 - - 0.97 - - setb al + - - 0.01 0.02 - 0.97 - - or al, dil + - - - 0.98 - 0.02 - - test al, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.98 0.02 - - - - lea rcx, [rdx - 4] + - - 0.01 0.99 - - - - mov rsi, rcx + - - - 0.98 - 0.02 - - and rsi, -2 + - - 0.98 0.01 - 0.01 - - add rsi, 4 + - - 0.99 - - 0.01 - - shr rcx + - - - - - - - - xor eax, eax + - - 0.02 0.97 - 0.01 - - cmp rdx, rsi + - - 0.99 0.99 - 0.02 - - cmove rdx, rcx + - - 0.98 0.99 - 0.03 - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_bytes_static_size.rs b/rust/zerocopy/benches/ref_from_bytes_static_size.rs new file mode 100644 index 000000000000..3742bba0780f --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_static_size.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_bytes_static_size(source: &[u8]) -> Option<&format::LocoPacket> { + zerocopy::FromBytes::ref_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64 b/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64 new file mode 100644 index 000000000000..2c8da68c8b5b --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64 @@ -0,0 +1,8 @@ +bench_ref_from_bytes_static_size: + mov ecx, edi + and ecx, 1 + xor rsi, 6 + xor eax, eax + or rsi, rcx + cmove rax, rdi + ret diff --git a/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64.mca b/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64.mca new file mode 100644 index 000000000000..832697801ee2 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64.mca @@ -0,0 +1,53 @@ +Iterations: 100 +Instructions: 700 +Total Cycles: 240 +Total uOps: 800 + +Dispatch Width: 4 +uOps Per Cycle: 3.33 +IPC: 2.92 +Block RThroughput: 2.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov ecx, edi + 1 1 0.33 and ecx, 1 + 1 1 0.33 xor rsi, 6 + 1 0 0.25 xor eax, eax + 1 1 0.33 or rsi, rcx + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.33 2.33 - 2.34 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.01 0.98 - 0.01 - - mov ecx, edi + - - 0.02 0.66 - 0.32 - - and ecx, 1 + - - 0.33 0.66 - 0.01 - - xor rsi, 6 + - - - - - - - - xor eax, eax + - - 0.98 0.02 - - - - or rsi, rcx + - - 0.99 0.01 - 1.00 - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..b4fea4aee51d --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_bytes_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option<&format::LocoPacket> { + zerocopy::FromBytes::ref_from_bytes_with_elems(source, count).ok() +} diff --git a/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..d579b3faefe7 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,19 @@ +bench_ref_from_bytes_with_elems_dynamic_padding: + movabs rax, 3074457345618258598 + cmp rdx, rax + seta cl + mov rax, rdi + test al, 3 + setne dil + or dil, cl + jne .LBB5_2 + lea rcx, [rdx + 2*rdx] + or rcx, 3 + add rcx, 9 + cmp rsi, rcx + je .LBB5_3 +.LBB5_2: + xor eax, eax + mov rdx, rsi +.LBB5_3: + ret diff --git a/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..ea2d83dbd17a --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,71 @@ +Iterations: 100 +Instructions: 1600 +Total Cycles: 539 +Total uOps: 1700 + +Dispatch Width: 4 +uOps Per Cycle: 3.15 +IPC: 2.97 +Block RThroughput: 4.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdx, rax + 2 2 1.00 seta cl + 1 1 0.33 mov rax, rdi + 1 1 0.33 test al, 3 + 1 1 0.50 setne dil + 1 1 0.33 or dil, cl + 1 1 1.00 jne .LBB5_2 + 1 1 0.50 lea rcx, [rdx + 2*rdx] + 1 1 0.33 or rcx, 3 + 1 1 0.33 add rcx, 9 + 1 1 0.33 cmp rsi, rcx + 1 1 1.00 je .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov rdx, rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.33 5.32 - 5.35 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.01 0.98 - 0.01 - - movabs rax, 3074457345618258598 + - - - 1.00 - - - - cmp rdx, rax + - - 1.98 - - 0.02 - - seta cl + - - 0.02 0.98 - - - - mov rax, rdi + - - - 0.67 - 0.33 - - test al, 3 + - - 0.67 - - 0.33 - - setne dil + - - 0.99 - - 0.01 - - or dil, cl + - - - - - 1.00 - - jne .LBB5_2 + - - 0.01 0.99 - - - - lea rcx, [rdx + 2*rdx] + - - - 0.01 - 0.99 - - or rcx, 3 + - - 0.65 0.02 - 0.33 - - add rcx, 9 + - - 0.99 0.01 - - - - cmp rsi, rcx + - - - - - 1.00 - - je .LBB5_3 + - - - - - - - - xor eax, eax + - - 0.01 0.66 - 0.33 - - mov rdx, rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.rs b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..9d33a7c31bc3 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_bytes_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option<&format::LocoPacket> { + zerocopy::FromBytes::ref_from_bytes_with_elems(source, count).ok() +} diff --git a/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..3d8d15b7f6c1 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64 @@ -0,0 +1,16 @@ +bench_ref_from_bytes_with_elems_dynamic_size: + movabs rax, 4611686018427387901 + cmp rdx, rax + seta cl + mov rax, rdi + or dil, cl + test dil, 1 + jne .LBB5_2 + lea rcx, [2*rdx + 4] + cmp rsi, rcx + je .LBB5_3 +.LBB5_2: + xor eax, eax + mov rdx, rsi +.LBB5_3: + ret diff --git a/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..602179f3c903 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,65 @@ +Iterations: 100 +Instructions: 1300 +Total Cycles: 439 +Total uOps: 1400 + +Dispatch Width: 4 +uOps Per Cycle: 3.19 +IPC: 2.96 +Block RThroughput: 3.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdx, rax + 2 2 1.00 seta cl + 1 1 0.33 mov rax, rdi + 1 1 0.33 or dil, cl + 1 1 0.33 test dil, 1 + 1 1 1.00 jne .LBB5_2 + 1 1 0.50 lea rcx, [2*rdx + 4] + 1 1 0.33 cmp rsi, rcx + 1 1 1.00 je .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov rdx, rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.32 4.33 - 4.35 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - movabs rax, 4611686018427387901 + - - 0.33 0.67 - - - - cmp rdx, rax + - - 1.98 - - 0.02 - - seta cl + - - 0.01 0.99 - - - - mov rax, rdi + - - 1.00 - - - - - or dil, cl + - - 0.99 0.01 - - - - test dil, 1 + - - - - - 1.00 - - jne .LBB5_2 + - - - 1.00 - - - - lea rcx, [2*rdx + 4] + - - 0.01 - - 0.99 - - cmp rsi, rcx + - - - - - 1.00 - - je .LBB5_3 + - - - - - - - - xor eax, eax + - - - 0.67 - 0.33 - - mov rdx, rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.rs b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.rs new file mode 100644 index 000000000000..53c707b88256 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_prefix_dynamic_padding(source: &[u8]) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64 new file mode 100644 index 000000000000..a58592a24503 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64 @@ -0,0 +1,22 @@ +bench_ref_from_prefix_dynamic_padding: + xor edx, edx + mov eax, 0 + test dil, 3 + je .LBB5_1 + ret +.LBB5_1: + movabs rax, 9223372036854775804 + and rsi, rax + cmp rsi, 9 + jae .LBB5_3 + mov edx, 1 + xor eax, eax + ret +.LBB5_3: + add rsi, -9 + movabs rcx, -6148914691236517205 + mov rax, rsi + mul rcx + shr rdx + mov rax, rdi + ret diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..62ea4babaf28 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 608 +Total uOps: 2000 + +Dispatch Width: 4 +uOps Per Cycle: 3.29 +IPC: 3.13 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor edx, edx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 3 + 1 1 1.00 je .LBB5_1 + 1 1 1.00 U ret + 1 1 0.33 movabs rax, 9223372036854775804 + 1 1 0.33 and rsi, rax + 1 1 0.33 cmp rsi, 9 + 1 1 1.00 jae .LBB5_3 + 1 1 0.33 mov edx, 1 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rsi, -9 + 1 1 0.33 movabs rcx, -6148914691236517205 + 1 1 0.33 mov rax, rsi + 2 4 1.00 mul rcx + 1 1 0.50 shr rdx + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.00 6.00 - 6.00 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor edx, edx + - - 0.01 0.98 - 0.01 - - mov eax, 0 + - - 0.98 0.01 - 0.01 - - test dil, 3 + - - - - - 1.00 - - je .LBB5_1 + - - - - - 1.00 - - ret + - - 0.01 0.99 - - - - movabs rax, 9223372036854775804 + - - - 1.00 - - - - and rsi, rax + - - - 1.00 - - - - cmp rsi, 9 + - - - - - 1.00 - - jae .LBB5_3 + - - 1.00 - - - - - mov edx, 1 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.02 0.02 - 0.96 - - add rsi, -9 + - - 0.99 0.01 - - - - movabs rcx, -6148914691236517205 + - - 0.01 0.99 - - - - mov rax, rsi + - - 1.00 1.00 - - - - mul rcx + - - 1.00 - - - - - shr rdx + - - 0.98 - - 0.02 - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_size.rs b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.rs new file mode 100644 index 000000000000..a3f26f6b4e6c --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_prefix_dynamic_size(source: &[u8]) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64 new file mode 100644 index 000000000000..fe6332c9100c --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64 @@ -0,0 +1,17 @@ +bench_ref_from_prefix_dynamic_size: + xor edx, edx + mov eax, 0 + test dil, 1 + jne .LBB5_4 + cmp rsi, 4 + jae .LBB5_3 + mov edx, 1 + xor eax, eax + ret +.LBB5_3: + add rsi, -4 + shr rsi + mov rdx, rsi + mov rax, rdi +.LBB5_4: + ret diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..3900a5946138 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64.mca @@ -0,0 +1,67 @@ +Iterations: 100 +Instructions: 1400 +Total Cycles: 405 +Total uOps: 1400 + +Dispatch Width: 4 +uOps Per Cycle: 3.46 +IPC: 3.46 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor edx, edx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 1 + 1 1 1.00 jne .LBB5_4 + 1 1 0.33 cmp rsi, 4 + 1 1 1.00 jae .LBB5_3 + 1 1 0.33 mov edx, 1 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rsi, -4 + 1 1 0.50 shr rsi + 1 1 0.33 mov rdx, rsi + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.99 3.99 - 4.02 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor edx, edx + - - 0.01 0.98 - 0.01 - - mov eax, 0 + - - 0.98 0.02 - - - - test dil, 1 + - - - - - 1.00 - - jne .LBB5_4 + - - 0.02 0.98 - - - - cmp rsi, 4 + - - - - - 1.00 - - jae .LBB5_3 + - - 0.98 0.01 - 0.01 - - mov edx, 1 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.01 0.99 - - - - add rsi, -4 + - - 1.00 - - - - - shr rsi + - - - 1.00 - - - - mov rdx, rsi + - - 0.99 0.01 - - - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_prefix_static_size.rs b/rust/zerocopy/benches/ref_from_prefix_static_size.rs new file mode 100644 index 000000000000..834fa3928611 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_static_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_prefix_static_size(source: &[u8]) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_prefix_static_size.x86-64 b/rust/zerocopy/benches/ref_from_prefix_static_size.x86-64 new file mode 100644 index 000000000000..7c1bf45bb6c2 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_static_size.x86-64 @@ -0,0 +1,8 @@ +bench_ref_from_prefix_static_size: + xor eax, eax + cmp rsi, 6 + mov rcx, rdi + cmovb rcx, rax + test dil, 1 + cmove rax, rcx + ret diff --git a/rust/zerocopy/benches/ref_from_prefix_static_size.x86-64.mca b/rust/zerocopy/benches/ref_from_prefix_static_size.x86-64.mca new file mode 100644 index 000000000000..9691b88fe03a --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_static_size.x86-64.mca @@ -0,0 +1,53 @@ +Iterations: 100 +Instructions: 700 +Total Cycles: 274 +Total uOps: 900 + +Dispatch Width: 4 +uOps Per Cycle: 3.28 +IPC: 2.55 +Block RThroughput: 2.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp rsi, 6 + 1 1 0.33 mov rcx, rdi + 2 2 0.67 cmovb rcx, rax + 1 1 0.33 test dil, 1 + 2 2 0.67 cmove rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.66 2.67 - 2.67 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor eax, eax + - - - 0.01 - 0.99 - - cmp rsi, 6 + - - 0.01 0.67 - 0.32 - - mov rcx, rdi + - - 1.00 0.99 - 0.01 - - cmovb rcx, rax + - - 0.66 0.01 - 0.33 - - test dil, 1 + - - 0.99 0.99 - 0.02 - - cmove rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..55d495e00c59 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_prefix_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_prefix_with_elems(source, count) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..5b31277bdebe --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,26 @@ +bench_ref_from_prefix_with_elems_dynamic_padding: + movabs rax, 3074457345618258598 + cmp rdx, rax + ja .LBB5_1 + xor ecx, ecx + mov eax, 0 + test dil, 3 + je .LBB5_3 + mov rdx, rcx + ret +.LBB5_1: + mov edx, 1 + xor eax, eax + ret +.LBB5_3: + lea rax, [rdx + 2*rdx] + or rax, 3 + add rax, 9 + xor r8d, r8d + cmp rax, rsi + mov ecx, 1 + cmovbe rcx, rdx + cmova rdi, r8 + mov rax, rdi + mov rdx, rcx + ret diff --git a/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..2f212ec6d03b --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,85 @@ +Iterations: 100 +Instructions: 2300 +Total Cycles: 807 +Total uOps: 2700 + +Dispatch Width: 4 +uOps Per Cycle: 3.35 +IPC: 2.85 +Block RThroughput: 6.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 0 0.25 xor ecx, ecx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 3 + 1 1 1.00 je .LBB5_3 + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + 1 1 0.33 mov edx, 1 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, 9 + 1 0 0.25 xor r8d, r8d + 1 1 0.33 cmp rax, rsi + 1 1 0.33 mov ecx, 1 + 3 3 1.00 cmovbe rcx, rdx + 3 3 1.00 cmova rdi, r8 + 1 1 0.33 mov rax, rdi + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 7.99 7.99 - 8.02 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.47 0.52 - 0.01 - - movabs rax, 3074457345618258598 + - - 0.94 0.01 - 0.05 - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - - - - - - - xor ecx, ecx + - - 0.03 0.97 - - - - mov eax, 0 + - - 0.01 0.52 - 0.47 - - test dil, 3 + - - - - - 1.00 - - je .LBB5_3 + - - 0.03 0.51 - 0.46 - - mov rdx, rcx + - - - - - 1.00 - - ret + - - 0.04 0.96 - - - - mov edx, 1 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.01 0.99 - - - - lea rax, [rdx + 2*rdx] + - - 0.52 0.48 - - - - or rax, 3 + - - 0.51 0.49 - - - - add rax, 9 + - - - - - - - - xor r8d, r8d + - - 0.97 0.03 - - - - cmp rax, rsi + - - 0.01 0.99 - - - - mov ecx, 1 + - - 1.04 0.97 - 0.99 - - cmovbe rcx, rdx + - - 1.44 0.54 - 1.02 - - cmova rdi, r8 + - - 0.97 0.01 - 0.02 - - mov rax, rdi + - - 1.00 - - - - - mov rdx, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.rs b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..e9663c721ece --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_prefix_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_prefix_with_elems(source, count) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..069fd4859c74 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64 @@ -0,0 +1,22 @@ +bench_ref_from_prefix_with_elems_dynamic_size: + movabs rax, 4611686018427387901 + cmp rdx, rax + ja .LBB5_1 + mov rcx, rdx + xor edx, edx + mov eax, 0 + test dil, 1 + jne .LBB5_4 + lea rax, [2*rcx + 4] + xor r8d, r8d + cmp rax, rsi + mov edx, 1 + cmovbe rdx, rcx + cmova rdi, r8 + mov rax, rdi +.LBB5_4: + ret +.LBB5_1: + mov edx, 1 + xor eax, eax + ret diff --git a/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..6f227264066d --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 672 +Total uOps: 2300 + +Dispatch Width: 4 +uOps Per Cycle: 3.42 +IPC: 2.83 +Block RThroughput: 5.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.33 mov rcx, rdx + 1 0 0.25 xor edx, edx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 1 + 1 1 1.00 jne .LBB5_4 + 1 1 0.50 lea rax, [2*rcx + 4] + 1 0 0.25 xor r8d, r8d + 1 1 0.33 cmp rax, rsi + 1 1 0.33 mov edx, 1 + 3 3 1.00 cmovbe rdx, rcx + 3 3 1.00 cmova rdi, r8 + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + 1 1 0.33 mov edx, 1 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.66 6.66 - 6.68 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - movabs rax, 4611686018427387901 + - - 0.37 0.63 - - - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.63 0.37 - - - - mov rcx, rdx + - - - - - - - - xor edx, edx + - - 0.01 0.98 - 0.01 - - mov eax, 0 + - - 0.98 0.02 - - - - test dil, 1 + - - - - - 1.00 - - jne .LBB5_4 + - - 0.01 0.99 - - - - lea rax, [2*rcx + 4] + - - - - - - - - xor r8d, r8d + - - 1.00 - - - - - cmp rax, rsi + - - - 0.67 - 0.33 - - mov edx, 1 + - - 0.73 0.98 - 1.29 - - cmovbe rdx, rcx + - - 1.60 0.36 - 1.04 - - cmova rdi, r8 + - - 0.99 0.01 - - - - mov rax, rdi + - - - - - 1.00 - - ret + - - 0.34 0.66 - - - - mov edx, 1 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.rs b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.rs new file mode 100644 index 000000000000..5a6ea3a33dde --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_suffix_dynamic_padding(source: &[u8]) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64 new file mode 100644 index 000000000000..3e05f6023f38 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64 @@ -0,0 +1,23 @@ +bench_ref_from_suffix_dynamic_padding: + lea eax, [rsi + rdi] + test al, 3 + jne .LBB5_1 + movabs rax, 9223372036854775804 + and rax, rsi + cmp rax, 9 + jae .LBB5_3 +.LBB5_1: + xor eax, eax + ret +.LBB5_3: + add rax, -9 + movabs rcx, -6148914691236517205 + mul rcx + shr rdx + lea rax, [rdx + 2*rdx] + sub rsi, rax + or rax, -4 + add rsi, rdi + add rax, rsi + add rax, -8 + ret diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..73599d5b6aab --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64.mca @@ -0,0 +1,79 @@ +Iterations: 100 +Instructions: 2000 +Total Cycles: 682 +Total uOps: 2100 + +Dispatch Width: 4 +uOps Per Cycle: 3.08 +IPC: 2.93 +Block RThroughput: 5.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea eax, [rsi + rdi] + 1 1 0.33 test al, 3 + 1 1 1.00 jne .LBB5_1 + 1 1 0.33 movabs rax, 9223372036854775804 + 1 1 0.33 and rax, rsi + 1 1 0.33 cmp rax, 9 + 1 1 1.00 jae .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rax, -9 + 1 1 0.33 movabs rcx, -6148914691236517205 + 2 4 1.00 mul rcx + 1 1 0.50 shr rdx + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 sub rsi, rax + 1 1 0.33 or rax, -4 + 1 1 0.33 add rsi, rdi + 1 1 0.33 add rax, rsi + 1 1 0.33 add rax, -8 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.65 6.67 - 6.68 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.90 0.10 - - - - lea eax, [rsi + rdi] + - - 0.93 - - 0.07 - - test al, 3 + - - - - - 1.00 - - jne .LBB5_1 + - - 0.51 0.47 - 0.02 - - movabs rax, 9223372036854775804 + - - - - - 1.00 - - and rax, rsi + - - - 0.09 - 0.91 - - cmp rax, 9 + - - - - - 1.00 - - jae .LBB5_3 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.43 0.47 - 0.10 - - add rax, -9 + - - 0.42 0.39 - 0.19 - - movabs rcx, -6148914691236517205 + - - 1.00 1.00 - - - - mul rcx + - - 0.69 - - 0.31 - - shr rdx + - - 0.54 0.46 - - - - lea rax, [rdx + 2*rdx] + - - 0.07 0.91 - 0.02 - - sub rsi, rax + - - 0.91 0.05 - 0.04 - - or rax, -4 + - - 0.08 0.90 - 0.02 - - add rsi, rdi + - - 0.09 0.91 - - - - add rax, rsi + - - 0.08 0.92 - - - - add rax, -8 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_size.rs b/rust/zerocopy/benches/ref_from_suffix_dynamic_size.rs new file mode 100644 index 000000000000..3437b14f404a --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_suffix_dynamic_size(source: &[u8]) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64 new file mode 100644 index 000000000000..bd4ace89836a --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64 @@ -0,0 +1,13 @@ +bench_ref_from_suffix_dynamic_size: + mov rdx, rsi + lea ecx, [rsi + rdi] + mov eax, edx + and eax, 1 + add rax, rdi + xor esi, esi + sub rdx, 4 + cmovb rax, rsi + shr rdx + test cl, 1 + cmovne rax, rsi + ret diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..1398bcfe27ae --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64.mca @@ -0,0 +1,63 @@ +Iterations: 100 +Instructions: 1200 +Total Cycles: 439 +Total uOps: 1400 + +Dispatch Width: 4 +uOps Per Cycle: 3.19 +IPC: 2.73 +Block RThroughput: 3.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rdx, rsi + 1 1 0.50 lea ecx, [rsi + rdi] + 1 1 0.33 mov eax, edx + 1 1 0.33 and eax, 1 + 1 1 0.33 add rax, rdi + 1 0 0.25 xor esi, esi + 1 1 0.33 sub rdx, 4 + 2 2 0.67 cmovb rax, rsi + 1 1 0.50 shr rdx + 1 1 0.33 test cl, 1 + 2 2 0.67 cmovne rax, rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.33 4.33 - 4.34 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.02 0.32 - 0.66 - - mov rdx, rsi + - - 0.32 0.68 - - - - lea ecx, [rsi + rdi] + - - 0.66 - - 0.34 - - mov eax, edx + - - 0.02 0.33 - 0.65 - - and eax, 1 + - - - 0.99 - 0.01 - - add rax, rdi + - - - - - - - - xor esi, esi + - - 0.65 - - 0.35 - - sub rdx, 4 + - - 1.00 1.00 - - - - cmovb rax, rsi + - - 0.66 - - 0.34 - - shr rdx + - - - 0.01 - 0.99 - - test cl, 1 + - - 1.00 1.00 - - - - cmovne rax, rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_suffix_static_size.rs b/rust/zerocopy/benches/ref_from_suffix_static_size.rs new file mode 100644 index 000000000000..c8435d10d38c --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_static_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_suffix_static_size(source: &[u8]) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_suffix_static_size.x86-64 b/rust/zerocopy/benches/ref_from_suffix_static_size.x86-64 new file mode 100644 index 000000000000..9e90b9e2543f --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_static_size.x86-64 @@ -0,0 +1,13 @@ +bench_ref_from_suffix_static_size: + lea eax, [rsi + rdi] + cmp rsi, 6 + setb cl + or cl, al + test cl, 1 + je .LBB5_2 + xor eax, eax + ret +.LBB5_2: + lea rax, [rdi + rsi] + add rax, -6 + ret diff --git a/rust/zerocopy/benches/ref_from_suffix_static_size.x86-64.mca b/rust/zerocopy/benches/ref_from_suffix_static_size.x86-64.mca new file mode 100644 index 000000000000..ef5892647b81 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_static_size.x86-64.mca @@ -0,0 +1,61 @@ +Iterations: 100 +Instructions: 1100 +Total Cycles: 338 +Total uOps: 1100 + +Dispatch Width: 4 +uOps Per Cycle: 3.25 +IPC: 3.25 +Block RThroughput: 3.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea eax, [rsi + rdi] + 1 1 0.33 cmp rsi, 6 + 1 1 0.50 setb cl + 1 1 0.33 or cl, al + 1 1 0.33 test cl, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.50 lea rax, [rdi + rsi] + 1 1 0.33 add rax, -6 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.32 3.33 - 3.35 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.97 0.03 - - - - lea eax, [rsi + rdi] + - - 0.33 0.32 - 0.35 - - cmp rsi, 6 + - - 1.00 - - - - - setb cl + - - - 1.00 - - - - or cl, al + - - - 1.00 - - - - test cl, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.34 0.66 - - - - lea rax, [rdi + rsi] + - - 0.68 0.32 - - - - add rax, -6 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..73d91cee5992 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_suffix_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_suffix_with_elems(source, count) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..c3d10b5fc685 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,27 @@ +bench_ref_from_suffix_with_elems_dynamic_padding: + movabs rax, 3074457345618258598 + cmp rdx, rax + ja .LBB5_1 + lea r8d, [rsi + rdi] + xor ecx, ecx + mov eax, 0 + test r8b, 3 + je .LBB5_3 + mov rdx, rcx + ret +.LBB5_3: + lea rax, [rdx + 2*rdx] + or rax, 3 + add rax, 9 + sub rsi, rax + jae .LBB5_4 +.LBB5_1: + xor eax, eax + mov edx, 1 + ret +.LBB5_4: + add rdi, rsi + mov rcx, rdx + mov rax, rdi + mov rdx, rcx + ret diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..92e6280bb4cc --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,85 @@ +Iterations: 100 +Instructions: 2300 +Total Cycles: 706 +Total uOps: 2300 + +Dispatch Width: 4 +uOps Per Cycle: 3.26 +IPC: 3.26 +Block RThroughput: 6.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.50 lea r8d, [rsi + rdi] + 1 0 0.25 xor ecx, ecx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test r8b, 3 + 1 1 1.00 je .LBB5_3 + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, 9 + 1 1 0.33 sub rsi, rax + 1 1 1.00 jae .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov edx, 1 + 1 1 1.00 U ret + 1 1 0.33 add rdi, rsi + 1 1 0.33 mov rcx, rdx + 1 1 0.33 mov rax, rdi + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.99 7.00 - 7.01 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - movabs rax, 3074457345618258598 + - - 0.01 0.50 - 0.49 - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - - 1.00 - - - - lea r8d, [rsi + rdi] + - - - - - - - - xor ecx, ecx + - - 0.50 0.49 - 0.01 - - mov eax, 0 + - - 0.49 0.51 - - - - test r8b, 3 + - - - - - 1.00 - - je .LBB5_3 + - - 0.51 0.49 - - - - mov rdx, rcx + - - - - - 1.00 - - ret + - - 0.50 0.50 - - - - lea rax, [rdx + 2*rdx] + - - 1.00 - - - - - or rax, 3 + - - 1.00 - - - - - add rax, 9 + - - 0.99 0.01 - - - - sub rsi, rax + - - - - - 1.00 - - jae .LBB5_4 + - - - - - - - - xor eax, eax + - - - 1.00 - - - - mov edx, 1 + - - - - - 1.00 - - ret + - - 1.00 - - - - - add rdi, rsi + - - - 1.00 - - - - mov rcx, rdx + - - 0.99 0.01 - - - - mov rax, rdi + - - - 0.50 - 0.50 - - mov rdx, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.rs b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..68a28baf55e6 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_suffix_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_suffix_with_elems(source, count) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..bdca57192455 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64 @@ -0,0 +1,23 @@ +bench_ref_from_suffix_with_elems_dynamic_size: + movabs rax, 4611686018427387901 + cmp rdx, rax + ja .LBB5_1 + lea r8d, [rsi + rdi] + xor ecx, ecx + mov eax, 0 + test r8b, 1 + jne .LBB5_5 + lea rax, [2*rdx + 4] + sub rsi, rax + jae .LBB5_4 +.LBB5_1: + xor eax, eax + mov edx, 1 + ret +.LBB5_4: + add rdi, rsi + mov rcx, rdx + mov rax, rdi +.LBB5_5: + mov rdx, rcx + ret diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..6d9de0b3eb5c --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 571 +Total uOps: 1900 + +Dispatch Width: 4 +uOps Per Cycle: 3.33 +IPC: 3.33 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.50 lea r8d, [rsi + rdi] + 1 0 0.25 xor ecx, ecx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test r8b, 1 + 1 1 1.00 jne .LBB5_5 + 1 1 0.50 lea rax, [2*rdx + 4] + 1 1 0.33 sub rsi, rax + 1 1 1.00 jae .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov edx, 1 + 1 1 1.00 U ret + 1 1 0.33 add rdi, rsi + 1 1 0.33 mov rcx, rdx + 1 1 0.33 mov rax, rdi + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.66 5.66 - 5.68 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.66 0.33 - 0.01 - - movabs rax, 4611686018427387901 + - - 0.01 0.99 - - - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.99 0.01 - - - - lea r8d, [rsi + rdi] + - - - - - - - - xor ecx, ecx + - - 0.33 0.33 - 0.34 - - mov eax, 0 + - - 0.33 0.34 - 0.33 - - test r8b, 1 + - - - - - 1.00 - - jne .LBB5_5 + - - 0.34 0.66 - - - - lea rax, [2*rdx + 4] + - - - 1.00 - - - - sub rsi, rax + - - - - - 1.00 - - jae .LBB5_4 + - - - - - - - - xor eax, eax + - - 1.00 - - - - - mov edx, 1 + - - - - - 1.00 - - ret + - - - 1.00 - - - - add rdi, rsi + - - 1.00 - - - - - mov rcx, rdx + - - 0.32 0.68 - - - - mov rax, rdi + - - 0.68 0.32 - - - - mov rdx, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_at_dynamic_padding.rs b/rust/zerocopy/benches/split_at_dynamic_padding.rs new file mode 100644 index 000000000000..bed90f60165e --- /dev/null +++ b/rust/zerocopy/benches/split_at_dynamic_padding.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_split_at_dynamic_padding( + source: &format::CocoPacket, + len: usize, +) -> Option<Split<&format::CocoPacket>> { + source.split_at(len) +} diff --git a/rust/zerocopy/benches/split_at_dynamic_padding.x86-64 b/rust/zerocopy/benches/split_at_dynamic_padding.x86-64 new file mode 100644 index 000000000000..6eaf5a004612 --- /dev/null +++ b/rust/zerocopy/benches/split_at_dynamic_padding.x86-64 @@ -0,0 +1,12 @@ +bench_split_at_dynamic_padding: + mov rax, rdi + cmp rcx, rdx + jbe .LBB5_2 + xor esi, esi + mov qword ptr [rax], rsi + ret +.LBB5_2: + mov qword ptr [rax + 8], rdx + mov qword ptr [rax + 16], rcx + mov qword ptr [rax], rsi + ret diff --git a/rust/zerocopy/benches/split_at_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/split_at_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..19ab3414d77a --- /dev/null +++ b/rust/zerocopy/benches/split_at_dynamic_padding.x86-64.mca @@ -0,0 +1,59 @@ +Iterations: 100 +Instructions: 1000 +Total Cycles: 404 +Total uOps: 1000 + +Dispatch Width: 4 +uOps Per Cycle: 2.48 +IPC: 2.48 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 cmp rcx, rdx + 1 1 1.00 jbe .LBB5_2 + 1 0 0.25 xor esi, esi + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + 1 1 1.00 * mov qword ptr [rax + 8], rdx + 1 1 1.00 * mov qword ptr [rax + 16], rcx + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.99 1.00 4.00 3.01 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.99 - - 0.01 - - mov rax, rdi + - - - 1.00 - - - - cmp rcx, rdx + - - - - - 1.00 - - jbe .LBB5_2 + - - - - - - - - xor esi, esi + - - - - 1.00 - - 1.00 mov qword ptr [rax], rsi + - - - - - 1.00 - - ret + - - - - 1.00 - 1.00 - mov qword ptr [rax + 8], rdx + - - - - 1.00 - - 1.00 mov qword ptr [rax + 16], rcx + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_at_dynamic_size.rs b/rust/zerocopy/benches/split_at_dynamic_size.rs new file mode 100644 index 000000000000..07a22ba2e5b5 --- /dev/null +++ b/rust/zerocopy/benches/split_at_dynamic_size.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_split_at_dynamic_size( + source: &format::CocoPacket, + len: usize, +) -> Option<Split<&format::CocoPacket>> { + source.split_at(len) +} diff --git a/rust/zerocopy/benches/split_at_dynamic_size.x86-64 b/rust/zerocopy/benches/split_at_dynamic_size.x86-64 new file mode 100644 index 000000000000..8d81b98bdab1 --- /dev/null +++ b/rust/zerocopy/benches/split_at_dynamic_size.x86-64 @@ -0,0 +1,12 @@ +bench_split_at_dynamic_size: + mov rax, rdi + cmp rcx, rdx + jbe .LBB5_2 + xor esi, esi + mov qword ptr [rax], rsi + ret +.LBB5_2: + mov qword ptr [rax + 8], rdx + mov qword ptr [rax + 16], rcx + mov qword ptr [rax], rsi + ret diff --git a/rust/zerocopy/benches/split_at_dynamic_size.x86-64.mca b/rust/zerocopy/benches/split_at_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..19ab3414d77a --- /dev/null +++ b/rust/zerocopy/benches/split_at_dynamic_size.x86-64.mca @@ -0,0 +1,59 @@ +Iterations: 100 +Instructions: 1000 +Total Cycles: 404 +Total uOps: 1000 + +Dispatch Width: 4 +uOps Per Cycle: 2.48 +IPC: 2.48 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 cmp rcx, rdx + 1 1 1.00 jbe .LBB5_2 + 1 0 0.25 xor esi, esi + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + 1 1 1.00 * mov qword ptr [rax + 8], rdx + 1 1 1.00 * mov qword ptr [rax + 16], rcx + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.99 1.00 4.00 3.01 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.99 - - 0.01 - - mov rax, rdi + - - - 1.00 - - - - cmp rcx, rdx + - - - - - 1.00 - - jbe .LBB5_2 + - - - - - - - - xor esi, esi + - - - - 1.00 - - 1.00 mov qword ptr [rax], rsi + - - - - - 1.00 - - ret + - - - - 1.00 - 1.00 - mov qword ptr [rax + 8], rdx + - - - - 1.00 - - 1.00 mov qword ptr [rax + 16], rcx + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.rs b/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.rs new file mode 100644 index 000000000000..3c147d3bdd39 --- /dev/null +++ b/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +unsafe fn bench_split_at_unchecked_dynamic_padding( + source: &format::CocoPacket, + len: usize, +) -> Split<&format::CocoPacket> { + unsafe { source.split_at_unchecked(len) } +} diff --git a/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64 b/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64 new file mode 100644 index 000000000000..74c3b52f63fa --- /dev/null +++ b/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64 @@ -0,0 +1,6 @@ +bench_split_at_unchecked_dynamic_padding: + mov rax, rdi + mov qword ptr [rdi], rsi + mov qword ptr [rdi + 8], rdx + mov qword ptr [rdi + 16], rcx + ret diff --git a/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..e8c61591c086 --- /dev/null +++ b/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64.mca @@ -0,0 +1,49 @@ +Iterations: 100 +Instructions: 500 +Total Cycles: 303 +Total uOps: 500 + +Dispatch Width: 4 +uOps Per Cycle: 1.65 +IPC: 1.65 +Block RThroughput: 3.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 1.00 * mov qword ptr [rdi], rsi + 1 1 1.00 * mov qword ptr [rdi + 8], rdx + 1 1 1.00 * mov qword ptr [rdi + 16], rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.49 0.50 3.00 1.01 1.50 1.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.49 0.50 - 0.01 - - mov rax, rdi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rdi], rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rdi + 8], rdx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rdi + 16], rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_at_unchecked_dynamic_size.rs b/rust/zerocopy/benches/split_at_unchecked_dynamic_size.rs new file mode 100644 index 000000000000..b1aa1dfb35be --- /dev/null +++ b/rust/zerocopy/benches/split_at_unchecked_dynamic_size.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +unsafe fn bench_split_at_unchecked_dynamic_size( + source: &format::CocoPacket, + len: usize, +) -> Split<&format::CocoPacket> { + unsafe { source.split_at_unchecked(len) } +} diff --git a/rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64 b/rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64 new file mode 100644 index 000000000000..56671d1ee8dc --- /dev/null +++ b/rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64 @@ -0,0 +1,6 @@ +bench_split_at_unchecked_dynamic_size: + mov rax, rdi + mov qword ptr [rdi], rsi + mov qword ptr [rdi + 8], rdx + mov qword ptr [rdi + 16], rcx + ret diff --git a/rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64.mca b/rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..e8c61591c086 --- /dev/null +++ b/rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64.mca @@ -0,0 +1,49 @@ +Iterations: 100 +Instructions: 500 +Total Cycles: 303 +Total uOps: 500 + +Dispatch Width: 4 +uOps Per Cycle: 1.65 +IPC: 1.65 +Block RThroughput: 3.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 1.00 * mov qword ptr [rdi], rsi + 1 1 1.00 * mov qword ptr [rdi + 8], rdx + 1 1 1.00 * mov qword ptr [rdi + 16], rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.49 0.50 3.00 1.01 1.50 1.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.49 0.50 - 0.01 - - mov rax, rdi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rdi], rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rdi + 8], rdx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rdi + 16], rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_via_immutable_dynamic_padding.rs b/rust/zerocopy/benches/split_via_immutable_dynamic_padding.rs new file mode 100644 index 000000000000..b86ad3614bf7 --- /dev/null +++ b/rust/zerocopy/benches/split_via_immutable_dynamic_padding.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_split_via_immutable_dynamic_padding( + split: Split<&format::CocoPacket>, +) -> (&format::CocoPacket, &[[u8; 3]]) { + split.via_immutable() +} diff --git a/rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64 b/rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64 new file mode 100644 index 000000000000..dac183428b31 --- /dev/null +++ b/rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64 @@ -0,0 +1,14 @@ +bench_split_via_immutable_dynamic_padding: + mov rax, rdi + mov rcx, qword ptr [rsi] + mov rdx, qword ptr [rsi + 8] + mov rsi, qword ptr [rsi + 16] + lea rdi, [rsi + 2*rsi] + add rdi, rcx + add rdi, 9 + sub rdx, rsi + mov qword ptr [rax], rcx + mov qword ptr [rax + 8], rsi + mov qword ptr [rax + 16], rdi + mov qword ptr [rax + 24], rdx + ret diff --git a/rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..6ab4e838767e --- /dev/null +++ b/rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64.mca @@ -0,0 +1,65 @@ +Iterations: 100 +Instructions: 1300 +Total Cycles: 510 +Total uOps: 1300 + +Dispatch Width: 4 +uOps Per Cycle: 2.55 +IPC: 2.55 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 5 0.50 * mov rcx, qword ptr [rsi] + 1 5 0.50 * mov rdx, qword ptr [rsi + 8] + 1 5 0.50 * mov rsi, qword ptr [rsi + 16] + 1 1 0.50 lea rdi, [rsi + 2*rsi] + 1 1 0.33 add rdi, rcx + 1 1 0.33 add rdi, 9 + 1 1 0.33 sub rdx, rsi + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 * mov qword ptr [rax + 8], rsi + 1 1 1.00 * mov qword ptr [rax + 16], rdi + 1 1 1.00 * mov qword ptr [rax + 24], rdx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.00 2.00 4.00 2.00 3.50 3.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.03 0.93 - 0.04 - - mov rax, rdi + - - - - - - 0.49 0.51 mov rcx, qword ptr [rsi] + - - - - - - 1.00 - mov rdx, qword ptr [rsi + 8] + - - - - - - 0.01 0.99 mov rsi, qword ptr [rsi + 16] + - - 0.93 0.07 - - - - lea rdi, [rsi + 2*rsi] + - - 0.05 0.02 - 0.93 - - add rdi, rcx + - - 0.49 0.49 - 0.02 - - add rdi, 9 + - - 0.50 0.49 - 0.01 - - sub rdx, rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax], rcx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax + 8], rsi + - - - - 1.00 - 0.49 0.51 mov qword ptr [rax + 16], rdi + - - - - 1.00 - 0.51 0.49 mov qword ptr [rax + 24], rdx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_via_immutable_dynamic_size.rs b/rust/zerocopy/benches/split_via_immutable_dynamic_size.rs new file mode 100644 index 000000000000..7d115caa3c00 --- /dev/null +++ b/rust/zerocopy/benches/split_via_immutable_dynamic_size.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_split_via_immutable_dynamic_size( + split: Split<&format::CocoPacket>, +) -> (&format::CocoPacket, &[[u8; 2]]) { + split.via_immutable() +} diff --git a/rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64 b/rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64 new file mode 100644 index 000000000000..58f6b09fc9d2 --- /dev/null +++ b/rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64 @@ -0,0 +1,13 @@ +bench_split_via_immutable_dynamic_size: + mov rax, rdi + mov rcx, qword ptr [rsi] + mov rdx, qword ptr [rsi + 8] + mov rsi, qword ptr [rsi + 16] + lea rdi, [rcx + 2*rsi] + add rdi, 4 + sub rdx, rsi + mov qword ptr [rax], rcx + mov qword ptr [rax + 8], rsi + mov qword ptr [rax + 16], rdi + mov qword ptr [rax + 24], rdx + ret diff --git a/rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64.mca b/rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..4549f20de53c --- /dev/null +++ b/rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64.mca @@ -0,0 +1,63 @@ +Iterations: 100 +Instructions: 1200 +Total Cycles: 509 +Total uOps: 1200 + +Dispatch Width: 4 +uOps Per Cycle: 2.36 +IPC: 2.36 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 5 0.50 * mov rcx, qword ptr [rsi] + 1 5 0.50 * mov rdx, qword ptr [rsi + 8] + 1 5 0.50 * mov rsi, qword ptr [rsi + 16] + 1 1 0.50 lea rdi, [rcx + 2*rsi] + 1 1 0.33 add rdi, 4 + 1 1 0.33 sub rdx, rsi + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 * mov qword ptr [rax + 8], rsi + 1 1 1.00 * mov qword ptr [rax + 16], rdi + 1 1 1.00 * mov qword ptr [rax + 24], rdx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.66 1.66 4.00 1.68 3.50 3.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.34 0.33 - 0.33 - - mov rax, rdi + - - - - - - 0.49 0.51 mov rcx, qword ptr [rsi] + - - - - - - 0.51 0.49 mov rdx, qword ptr [rsi + 8] + - - - - - - 0.01 0.99 mov rsi, qword ptr [rsi + 16] + - - 0.33 0.67 - - - - lea rdi, [rcx + 2*rsi] + - - 0.63 0.34 - 0.03 - - add rdi, 4 + - - 0.36 0.32 - 0.32 - - sub rdx, rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax], rcx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax + 8], rsi + - - - - 1.00 - 0.98 0.02 mov qword ptr [rax + 16], rdi + - - - - 1.00 - 0.51 0.49 mov qword ptr [rax + 24], rdx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.rs b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.rs new file mode 100644 index 000000000000..edba7bf34ba8 --- /dev/null +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_split_via_runtime_check_dynamic_padding( + split: Split<&format::CocoPacket>, +) -> Option<(&format::CocoPacket, &[[u8; 3]])> { + split.via_runtime_check().ok() +} diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64 b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64 new file mode 100644 index 000000000000..03684cb09f8f --- /dev/null +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64 @@ -0,0 +1,22 @@ +bench_split_via_runtime_check_dynamic_padding: + mov rax, rdi + mov rdx, qword ptr [rsi + 16] + mov ecx, edx + and ecx, 3 + cmp ecx, 1 + jne .LBB5_1 + mov rcx, qword ptr [rsi] + mov rsi, qword ptr [rsi + 8] + lea rdi, [rdx + 2*rdx] + add rdi, rcx + add rdi, 9 + sub rsi, rdx + mov qword ptr [rax + 8], rdx + mov qword ptr [rax + 16], rdi + mov qword ptr [rax + 24], rsi + mov qword ptr [rax], rcx + ret +.LBB5_1: + xor ecx, ecx + mov qword ptr [rax], rcx + ret diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..5034ab0583bd --- /dev/null +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64.mca @@ -0,0 +1,79 @@ +Iterations: 100 +Instructions: 2000 +Total Cycles: 708 +Total uOps: 2000 + +Dispatch Width: 4 +uOps Per Cycle: 2.82 +IPC: 2.82 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 5 0.50 * mov rdx, qword ptr [rsi + 16] + 1 1 0.33 mov ecx, edx + 1 1 0.33 and ecx, 3 + 1 1 0.33 cmp ecx, 1 + 1 1 1.00 jne .LBB5_1 + 1 5 0.50 * mov rcx, qword ptr [rsi] + 1 5 0.50 * mov rsi, qword ptr [rsi + 8] + 1 1 0.50 lea rdi, [rdx + 2*rdx] + 1 1 0.33 add rdi, rcx + 1 1 0.33 add rdi, 9 + 1 1 0.33 sub rsi, rdx + 1 1 1.00 * mov qword ptr [rax + 8], rdx + 1 1 1.00 * mov qword ptr [rax + 16], rdi + 1 1 1.00 * mov qword ptr [rax + 24], rsi + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 U ret + 1 0 0.25 xor ecx, ecx + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.00 3.02 5.00 4.98 4.00 4.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - mov rax, rdi + - - - - - - - 1.00 mov rdx, qword ptr [rsi + 16] + - - 0.99 0.01 - - - - mov ecx, edx + - - 0.99 0.01 - - - - and ecx, 3 + - - 0.97 0.03 - - - - cmp ecx, 1 + - - - - - 1.00 - - jne .LBB5_1 + - - - - - - 1.00 - mov rcx, qword ptr [rsi] + - - - - - - 0.99 0.01 mov rsi, qword ptr [rsi + 8] + - - 0.01 0.99 - - - - lea rdi, [rdx + 2*rdx] + - - - 0.96 - 0.04 - - add rdi, rcx + - - 0.03 - - 0.97 - - add rdi, 9 + - - 0.01 0.03 - 0.96 - - sub rsi, rdx + - - - - 1.00 - 1.00 - mov qword ptr [rax + 8], rdx + - - - - 1.00 - - 1.00 mov qword ptr [rax + 16], rdi + - - - - 1.00 - 0.01 0.99 mov qword ptr [rax + 24], rsi + - - - - 1.00 - 0.99 0.01 mov qword ptr [rax], rcx + - - - - - 1.00 - - ret + - - - - - - - - xor ecx, ecx + - - - - 1.00 - 0.01 0.99 mov qword ptr [rax], rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.rs b/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.rs new file mode 100644 index 000000000000..ce22de909aca --- /dev/null +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_split_via_runtime_check_dynamic_size( + split: Split<&format::CocoPacket>, +) -> Option<(&format::CocoPacket, &[[u8; 2]])> { + split.via_runtime_check().ok() +} diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64 b/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64 new file mode 100644 index 000000000000..e54276bd85e8 --- /dev/null +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64 @@ -0,0 +1,13 @@ +bench_split_via_runtime_check_dynamic_size: + mov rax, rdi + mov rcx, qword ptr [rsi] + mov rdx, qword ptr [rsi + 8] + mov rsi, qword ptr [rsi + 16] + lea rdi, [rcx + 2*rsi] + add rdi, 4 + sub rdx, rsi + mov qword ptr [rax], rcx + mov qword ptr [rax + 8], rsi + mov qword ptr [rax + 16], rdi + mov qword ptr [rax + 24], rdx + ret diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64.mca b/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..4549f20de53c --- /dev/null +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64.mca @@ -0,0 +1,63 @@ +Iterations: 100 +Instructions: 1200 +Total Cycles: 509 +Total uOps: 1200 + +Dispatch Width: 4 +uOps Per Cycle: 2.36 +IPC: 2.36 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 5 0.50 * mov rcx, qword ptr [rsi] + 1 5 0.50 * mov rdx, qword ptr [rsi + 8] + 1 5 0.50 * mov rsi, qword ptr [rsi + 16] + 1 1 0.50 lea rdi, [rcx + 2*rsi] + 1 1 0.33 add rdi, 4 + 1 1 0.33 sub rdx, rsi + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 * mov qword ptr [rax + 8], rsi + 1 1 1.00 * mov qword ptr [rax + 16], rdi + 1 1 1.00 * mov qword ptr [rax + 24], rdx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.66 1.66 4.00 1.68 3.50 3.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.34 0.33 - 0.33 - - mov rax, rdi + - - - - - - 0.49 0.51 mov rcx, qword ptr [rsi] + - - - - - - 0.51 0.49 mov rdx, qword ptr [rsi + 8] + - - - - - - 0.01 0.99 mov rsi, qword ptr [rsi + 16] + - - 0.33 0.67 - - - - lea rdi, [rcx + 2*rsi] + - - 0.63 0.34 - 0.03 - - add rdi, 4 + - - 0.36 0.32 - 0.32 - - sub rdx, rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax], rcx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax + 8], rsi + - - - - 1.00 - 0.98 0.02 mov qword ptr [rax + 16], rdi + - - - - 1.00 - 0.51 0.49 mov qword ptr [rax + 24], rdx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.rs b/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.rs new file mode 100644 index 000000000000..21d74dba176f --- /dev/null +++ b/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +unsafe fn bench_split_via_unchecked_dynamic_padding( + split: Split<&format::CocoPacket>, +) -> (&format::CocoPacket, &[[u8; 3]]) { + unsafe { split.via_unchecked() } +} diff --git a/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64 b/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64 new file mode 100644 index 000000000000..3c2c4ec9f6ed --- /dev/null +++ b/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64 @@ -0,0 +1,14 @@ +bench_split_via_unchecked_dynamic_padding: + mov rax, rdi + mov rcx, qword ptr [rsi] + mov rdx, qword ptr [rsi + 8] + mov rsi, qword ptr [rsi + 16] + lea rdi, [rsi + 2*rsi] + add rdi, rcx + add rdi, 9 + sub rdx, rsi + mov qword ptr [rax], rcx + mov qword ptr [rax + 8], rsi + mov qword ptr [rax + 16], rdi + mov qword ptr [rax + 24], rdx + ret diff --git a/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..6ab4e838767e --- /dev/null +++ b/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64.mca @@ -0,0 +1,65 @@ +Iterations: 100 +Instructions: 1300 +Total Cycles: 510 +Total uOps: 1300 + +Dispatch Width: 4 +uOps Per Cycle: 2.55 +IPC: 2.55 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 5 0.50 * mov rcx, qword ptr [rsi] + 1 5 0.50 * mov rdx, qword ptr [rsi + 8] + 1 5 0.50 * mov rsi, qword ptr [rsi + 16] + 1 1 0.50 lea rdi, [rsi + 2*rsi] + 1 1 0.33 add rdi, rcx + 1 1 0.33 add rdi, 9 + 1 1 0.33 sub rdx, rsi + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 * mov qword ptr [rax + 8], rsi + 1 1 1.00 * mov qword ptr [rax + 16], rdi + 1 1 1.00 * mov qword ptr [rax + 24], rdx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.00 2.00 4.00 2.00 3.50 3.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.03 0.93 - 0.04 - - mov rax, rdi + - - - - - - 0.49 0.51 mov rcx, qword ptr [rsi] + - - - - - - 1.00 - mov rdx, qword ptr [rsi + 8] + - - - - - - 0.01 0.99 mov rsi, qword ptr [rsi + 16] + - - 0.93 0.07 - - - - lea rdi, [rsi + 2*rsi] + - - 0.05 0.02 - 0.93 - - add rdi, rcx + - - 0.49 0.49 - 0.02 - - add rdi, 9 + - - 0.50 0.49 - 0.01 - - sub rdx, rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax], rcx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax + 8], rsi + - - - - 1.00 - 0.49 0.51 mov qword ptr [rax + 16], rdi + - - - - 1.00 - 0.51 0.49 mov qword ptr [rax + 24], rdx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_via_unchecked_dynamic_size.rs b/rust/zerocopy/benches/split_via_unchecked_dynamic_size.rs new file mode 100644 index 000000000000..824e22d67a7b --- /dev/null +++ b/rust/zerocopy/benches/split_via_unchecked_dynamic_size.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +unsafe fn bench_split_via_unchecked_dynamic_size( + split: Split<&format::CocoPacket>, +) -> (&format::CocoPacket, &[[u8; 2]]) { + unsafe { split.via_unchecked() } +} diff --git a/rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64 b/rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64 new file mode 100644 index 000000000000..1e31268edc7f --- /dev/null +++ b/rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64 @@ -0,0 +1,13 @@ +bench_split_via_unchecked_dynamic_size: + mov rax, rdi + mov rcx, qword ptr [rsi] + mov rdx, qword ptr [rsi + 8] + mov rsi, qword ptr [rsi + 16] + lea rdi, [rcx + 2*rsi] + add rdi, 4 + sub rdx, rsi + mov qword ptr [rax], rcx + mov qword ptr [rax + 8], rsi + mov qword ptr [rax + 16], rdi + mov qword ptr [rax + 24], rdx + ret diff --git a/rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64.mca b/rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..4549f20de53c --- /dev/null +++ b/rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64.mca @@ -0,0 +1,63 @@ +Iterations: 100 +Instructions: 1200 +Total Cycles: 509 +Total uOps: 1200 + +Dispatch Width: 4 +uOps Per Cycle: 2.36 +IPC: 2.36 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 5 0.50 * mov rcx, qword ptr [rsi] + 1 5 0.50 * mov rdx, qword ptr [rsi + 8] + 1 5 0.50 * mov rsi, qword ptr [rsi + 16] + 1 1 0.50 lea rdi, [rcx + 2*rsi] + 1 1 0.33 add rdi, 4 + 1 1 0.33 sub rdx, rsi + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 * mov qword ptr [rax + 8], rsi + 1 1 1.00 * mov qword ptr [rax + 16], rdi + 1 1 1.00 * mov qword ptr [rax + 24], rdx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.66 1.66 4.00 1.68 3.50 3.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.34 0.33 - 0.33 - - mov rax, rdi + - - - - - - 0.49 0.51 mov rcx, qword ptr [rsi] + - - - - - - 0.51 0.49 mov rdx, qword ptr [rsi + 8] + - - - - - - 0.01 0.99 mov rsi, qword ptr [rsi + 16] + - - 0.33 0.67 - - - - lea rdi, [rcx + 2*rsi] + - - 0.63 0.34 - 0.03 - - add rdi, 4 + - - 0.36 0.32 - 0.32 - - sub rdx, rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax], rcx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax + 8], rsi + - - - - 1.00 - 0.98 0.02 mov qword ptr [rax + 16], rdi + - - - - 1.00 - 0.51 0.49 mov qword ptr [rax + 24], rdx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/transmute.rs b/rust/zerocopy/benches/transmute.rs new file mode 100644 index 000000000000..e60bfb252f5c --- /dev/null +++ b/rust/zerocopy/benches/transmute.rs @@ -0,0 +1,16 @@ +use zerocopy::Unalign; +use zerocopy_derive::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[derive(IntoBytes, KnownLayout, Immutable)] +#[repr(C)] +struct MinimalViableSource { + bytes: [u8; 6], +} + +#[unsafe(no_mangle)] +fn bench_transmute(source: MinimalViableSource) -> Unalign<format::LocoPacket> { + zerocopy::transmute!(source) +} diff --git a/rust/zerocopy/benches/transmute.x86-64 b/rust/zerocopy/benches/transmute.x86-64 new file mode 100644 index 000000000000..a4c6299d1112 --- /dev/null +++ b/rust/zerocopy/benches/transmute.x86-64 @@ -0,0 +1,3 @@ +bench_transmute: + mov rax, rdi + ret diff --git a/rust/zerocopy/benches/transmute.x86-64.mca b/rust/zerocopy/benches/transmute.x86-64.mca new file mode 100644 index 000000000000..f297729c6523 --- /dev/null +++ b/rust/zerocopy/benches/transmute.x86-64.mca @@ -0,0 +1,43 @@ +Iterations: 100 +Instructions: 200 +Total Cycles: 104 +Total uOps: 200 + +Dispatch Width: 4 +uOps Per Cycle: 1.92 +IPC: 1.92 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.49 0.50 - 1.01 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.49 0.50 - 0.01 - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/transmute_ref_dynamic_size.rs b/rust/zerocopy/benches/transmute_ref_dynamic_size.rs new file mode 100644 index 000000000000..825f0f2bed24 --- /dev/null +++ b/rust/zerocopy/benches/transmute_ref_dynamic_size.rs @@ -0,0 +1,16 @@ +use zerocopy_derive::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[derive(IntoBytes, KnownLayout, Immutable)] +#[repr(C, align(2))] +struct MinimalViableSource { + header: [u8; 6], + trailer: [[u8; 2]], +} + +#[unsafe(no_mangle)] +fn bench_transmute_ref_dynamic_size(source: &MinimalViableSource) -> &format::LocoPacket { + zerocopy::transmute_ref!(source) +} diff --git a/rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64 b/rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64 new file mode 100644 index 000000000000..80a0f5906610 --- /dev/null +++ b/rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64 @@ -0,0 +1,4 @@ +bench_transmute_ref_dynamic_size: + mov rax, rdi + lea rdx, [rsi + 1] + ret diff --git a/rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64.mca b/rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..ef1bcdca5ea3 --- /dev/null +++ b/rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64.mca @@ -0,0 +1,45 @@ +Iterations: 100 +Instructions: 300 +Total Cycles: 104 +Total uOps: 300 + +Dispatch Width: 4 +uOps Per Cycle: 2.88 +IPC: 2.88 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.50 lea rdx, [rsi + 1] + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.99 1.00 - 1.01 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.99 - - 0.01 - - mov rax, rdi + - - - 1.00 - - - - lea rdx, [rsi + 1] + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/transmute_ref_static_size.rs b/rust/zerocopy/benches/transmute_ref_static_size.rs new file mode 100644 index 000000000000..a6db611fde0d --- /dev/null +++ b/rust/zerocopy/benches/transmute_ref_static_size.rs @@ -0,0 +1,15 @@ +use zerocopy_derive::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[derive(IntoBytes, KnownLayout, Immutable)] +#[repr(C, align(2))] +struct MinimalViableSource { + bytes: [u8; 6], +} + +#[unsafe(no_mangle)] +fn bench_transmute_ref_static_size(source: &MinimalViableSource) -> &format::LocoPacket { + zerocopy::transmute_ref!(source) +} diff --git a/rust/zerocopy/benches/transmute_ref_static_size.x86-64 b/rust/zerocopy/benches/transmute_ref_static_size.x86-64 new file mode 100644 index 000000000000..7a9229c129b0 --- /dev/null +++ b/rust/zerocopy/benches/transmute_ref_static_size.x86-64 @@ -0,0 +1,3 @@ +bench_transmute_ref_static_size: + mov rax, rdi + ret diff --git a/rust/zerocopy/benches/transmute_ref_static_size.x86-64.mca b/rust/zerocopy/benches/transmute_ref_static_size.x86-64.mca new file mode 100644 index 000000000000..f297729c6523 --- /dev/null +++ b/rust/zerocopy/benches/transmute_ref_static_size.x86-64.mca @@ -0,0 +1,43 @@ +Iterations: 100 +Instructions: 200 +Total Cycles: 104 +Total uOps: 200 + +Dispatch Width: 4 +uOps Per Cycle: 1.92 +IPC: 1.92 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.49 0.50 - 1.01 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.49 0.50 - 0.01 - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_read_from_bytes.rs b/rust/zerocopy/benches/try_read_from_bytes.rs new file mode 100644 index 000000000000..f8384b32f518 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_bytes.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_read_from_bytes_static_size(source: &[u8]) -> Option<format::CocoPacket> { + zerocopy::TryFromBytes::try_read_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/try_read_from_bytes.x86-64 b/rust/zerocopy/benches/try_read_from_bytes.x86-64 new file mode 100644 index 000000000000..08088a08fd85 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_bytes.x86-64 @@ -0,0 +1,23 @@ +bench_try_read_from_bytes_static_size: + mov ax, -16191 + cmp rsi, 6 + jne .LBB5_1 + mov ecx, dword ptr [rdi] + movzx edx, cx + cmp edx, 49344 + jne .LBB5_4 + movzx eax, word ptr [rdi + 4] + shl rax, 32 + or rcx, rax + shr rcx, 16 + mov ax, -16192 +.LBB5_4: + shl rcx, 16 + movzx eax, ax + or rax, rcx + ret +.LBB5_1: + shl rcx, 16 + movzx eax, ax + or rax, rcx + ret diff --git a/rust/zerocopy/benches/try_read_from_bytes.x86-64.mca b/rust/zerocopy/benches/try_read_from_bytes.x86-64.mca new file mode 100644 index 000000000000..385e6a480253 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_bytes.x86-64.mca @@ -0,0 +1,79 @@ +Iterations: 100 +Instructions: 2000 +Total Cycles: 608 +Total uOps: 2000 + +Dispatch Width: 4 +uOps Per Cycle: 3.29 +IPC: 3.29 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov ax, -16191 + 1 1 0.33 cmp rsi, 6 + 1 1 1.00 jne .LBB5_1 + 1 5 0.50 * mov ecx, dword ptr [rdi] + 1 1 0.33 movzx edx, cx + 1 1 0.33 cmp edx, 49344 + 1 1 1.00 jne .LBB5_4 + 1 5 0.50 * movzx eax, word ptr [rdi + 4] + 1 1 0.50 shl rax, 32 + 1 1 0.33 or rcx, rax + 1 1 0.50 shr rcx, 16 + 1 1 0.33 mov ax, -16192 + 1 1 0.50 shl rcx, 16 + 1 1 0.33 movzx eax, ax + 1 1 0.33 or rax, rcx + 1 1 1.00 U ret + 1 1 0.50 shl rcx, 16 + 1 1 0.33 movzx eax, ax + 1 1 0.33 or rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.99 5.99 - 6.02 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - mov ax, -16191 + - - - 0.01 - 0.99 - - cmp rsi, 6 + - - - - - 1.00 - - jne .LBB5_1 + - - - - - - - 1.00 mov ecx, dword ptr [rdi] + - - 0.98 - - 0.02 - - movzx edx, cx + - - 0.99 0.01 - - - - cmp edx, 49344 + - - - - - 1.00 - - jne .LBB5_4 + - - - - - - 1.00 - movzx eax, word ptr [rdi + 4] + - - 0.01 - - 0.99 - - shl rax, 32 + - - 0.02 0.98 - - - - or rcx, rax + - - 1.00 - - - - - shr rcx, 16 + - - 0.99 0.01 - - - - mov ax, -16192 + - - 1.00 - - - - - shl rcx, 16 + - - - 1.00 - - - - movzx eax, ax + - - - 1.00 - - - - or rax, rcx + - - - - - 1.00 - - ret + - - 1.00 - - - - - shl rcx, 16 + - - - 1.00 - - - - movzx eax, ax + - - - 0.99 - 0.01 - - or rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_read_from_prefix.rs b/rust/zerocopy/benches/try_read_from_prefix.rs new file mode 100644 index 000000000000..fabbac3bd7d9 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_prefix.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_read_from_prefix_static_size(source: &[u8]) -> Option<format::CocoPacket> { + match zerocopy::TryFromBytes::try_read_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_read_from_prefix.x86-64 b/rust/zerocopy/benches/try_read_from_prefix.x86-64 new file mode 100644 index 000000000000..d3e1edc3ea73 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_prefix.x86-64 @@ -0,0 +1,16 @@ +bench_try_read_from_prefix_static_size: + mov eax, 49345 + cmp rsi, 6 + jb .LBB5_2 + mov eax, dword ptr [rdi] + movzx ecx, word ptr [rdi + 4] + shl rcx, 32 + or rcx, rax + movzx eax, cx + and rcx, -65536 + or rcx, 49344 + cmp eax, 49344 + mov eax, 49345 + cmove rax, rcx +.LBB5_2: + ret diff --git a/rust/zerocopy/benches/try_read_from_prefix.x86-64.mca b/rust/zerocopy/benches/try_read_from_prefix.x86-64.mca new file mode 100644 index 000000000000..40401d89e83d --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_prefix.x86-64.mca @@ -0,0 +1,67 @@ +Iterations: 100 +Instructions: 1400 +Total Cycles: 442 +Total uOps: 1500 + +Dispatch Width: 4 +uOps Per Cycle: 3.39 +IPC: 3.17 +Block RThroughput: 3.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov eax, 49345 + 1 1 0.33 cmp rsi, 6 + 1 1 1.00 jb .LBB5_2 + 1 5 0.50 * mov eax, dword ptr [rdi] + 1 5 0.50 * movzx ecx, word ptr [rdi + 4] + 1 1 0.50 shl rcx, 32 + 1 1 0.33 or rcx, rax + 1 1 0.33 movzx eax, cx + 1 1 0.33 and rcx, -65536 + 1 1 0.33 or rcx, 49344 + 1 1 0.33 cmp eax, 49344 + 1 1 0.33 mov eax, 49345 + 2 2 0.67 cmove rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.33 4.33 - 4.34 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.65 0.01 - 0.34 - - mov eax, 49345 + - - 0.01 0.33 - 0.66 - - cmp rsi, 6 + - - - - - 1.00 - - jb .LBB5_2 + - - - - - - - 1.00 mov eax, dword ptr [rdi] + - - - - - - 1.00 - movzx ecx, word ptr [rdi + 4] + - - 0.65 - - 0.35 - - shl rcx, 32 + - - - 0.67 - 0.33 - - or rcx, rax + - - 0.01 0.99 - - - - movzx eax, cx + - - 0.99 0.01 - - - - and rcx, -65536 + - - 0.01 0.99 - - - - or rcx, 49344 + - - 0.99 0.01 - - - - cmp eax, 49344 + - - 0.02 0.33 - 0.65 - - mov eax, 49345 + - - 1.00 0.99 - 0.01 - - cmove rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_read_from_suffix.rs b/rust/zerocopy/benches/try_read_from_suffix.rs new file mode 100644 index 000000000000..1e961647b3f3 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_suffix.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_read_from_suffix_static_size(source: &[u8]) -> Option<format::CocoPacket> { + match zerocopy::TryFromBytes::try_read_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_read_from_suffix.x86-64 b/rust/zerocopy/benches/try_read_from_suffix.x86-64 new file mode 100644 index 000000000000..095e326f0467 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_suffix.x86-64 @@ -0,0 +1,18 @@ +bench_try_read_from_suffix_static_size: + mov eax, 49345 + cmp rsi, 6 + jb .LBB5_2 + mov eax, dword ptr [rdi + rsi - 6] + movzx ecx, word ptr [rdi + rsi - 2] + shl rcx, 32 + or rcx, rax + movzx edx, cx + xor eax, eax + cmp edx, 49344 + cmovne rcx, rsi + sete al + and rcx, -65536 + xor rax, 49345 + or rax, rcx +.LBB5_2: + ret diff --git a/rust/zerocopy/benches/try_read_from_suffix.x86-64.mca b/rust/zerocopy/benches/try_read_from_suffix.x86-64.mca new file mode 100644 index 000000000000..d3eaadbb8a81 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_suffix.x86-64.mca @@ -0,0 +1,71 @@ +Iterations: 100 +Instructions: 1600 +Total Cycles: 478 +Total uOps: 1700 + +Dispatch Width: 4 +uOps Per Cycle: 3.56 +IPC: 3.35 +Block RThroughput: 4.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov eax, 49345 + 1 1 0.33 cmp rsi, 6 + 1 1 1.00 jb .LBB5_2 + 1 5 0.50 * mov eax, dword ptr [rdi + rsi - 6] + 1 5 0.50 * movzx ecx, word ptr [rdi + rsi - 2] + 1 1 0.50 shl rcx, 32 + 1 1 0.33 or rcx, rax + 1 1 0.33 movzx edx, cx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp edx, 49344 + 2 2 0.67 cmovne rcx, rsi + 1 1 0.50 sete al + 1 1 0.33 and rcx, -65536 + 1 1 0.33 xor rax, 49345 + 1 1 0.33 or rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.66 4.66 - 4.68 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.32 0.01 - 0.67 - - mov eax, 49345 + - - 0.62 0.02 - 0.36 - - cmp rsi, 6 + - - - - - 1.00 - - jb .LBB5_2 + - - - - - - - 1.00 mov eax, dword ptr [rdi + rsi - 6] + - - - - - - 1.00 - movzx ecx, word ptr [rdi + rsi - 2] + - - 0.37 - - 0.63 - - shl rcx, 32 + - - 0.99 0.01 - - - - or rcx, rax + - - 1.00 - - - - - movzx edx, cx + - - - - - - - - xor eax, eax + - - 0.35 0.64 - 0.01 - - cmp edx, 49344 + - - 1.00 1.00 - - - - cmovne rcx, rsi + - - - - - 1.00 - - sete al + - - 0.01 0.99 - - - - and rcx, -65536 + - - - 1.00 - - - - xor rax, 49345 + - - - 0.99 - 0.01 - - or rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.rs b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.rs new file mode 100644 index 000000000000..126009cd71d5 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_bytes_dynamic_padding(source: &[u8]) -> Option<&format::CocoPacket> { + zerocopy::TryFromBytes::try_ref_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 new file mode 100644 index 000000000000..217c5fc61796 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 @@ -0,0 +1,24 @@ +bench_try_ref_from_bytes_dynamic_padding: + test dil, 3 + jne .LBB5_4 + movabs rax, 9223372036854775804 + and rax, rsi + cmp rax, 9 + jb .LBB5_4 + add rax, -9 + movabs rcx, -6148914691236517205 + mul rcx + shr rdx + lea rax, [rdx + 2*rdx] + or rax, 3 + add rax, 9 + cmp rsi, rax + jne .LBB5_4 + cmp word ptr [rdi], -16192 + je .LBB5_5 +.LBB5_4: + xor edi, edi + mov rdx, rsi +.LBB5_5: + mov rax, rdi + ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..95b993c7e0a8 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca @@ -0,0 +1,81 @@ +Iterations: 100 +Instructions: 2100 +Total Cycles: 709 +Total uOps: 2300 + +Dispatch Width: 4 +uOps Per Cycle: 3.24 +IPC: 2.96 +Block RThroughput: 5.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 test dil, 3 + 1 1 1.00 jne .LBB5_4 + 1 1 0.33 movabs rax, 9223372036854775804 + 1 1 0.33 and rax, rsi + 1 1 0.33 cmp rax, 9 + 1 1 1.00 jb .LBB5_4 + 1 1 0.33 add rax, -9 + 1 1 0.33 movabs rcx, -6148914691236517205 + 2 4 1.00 mul rcx + 1 1 0.50 shr rdx + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, 9 + 1 1 0.33 cmp rsi, rax + 1 1 1.00 jne .LBB5_4 + 2 6 0.50 * cmp word ptr [rdi], -16192 + 1 1 1.00 je .LBB5_5 + 1 0 0.25 xor edi, edi + 1 1 0.33 mov rdx, rsi + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.98 6.99 - 7.03 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.48 0.51 - 0.01 - - test dil, 3 + - - - - - 1.00 - - jne .LBB5_4 + - - 0.51 0.49 - - - - movabs rax, 9223372036854775804 + - - 0.01 0.99 - - - - and rax, rsi + - - 0.51 0.49 - - - - cmp rax, 9 + - - - - - 1.00 - - jb .LBB5_4 + - - 0.98 - - 0.02 - - add rax, -9 + - - 0.98 0.02 - - - - movabs rcx, -6148914691236517205 + - - 1.00 1.00 - - - - mul rcx + - - 0.99 - - 0.01 - - shr rdx + - - - 1.00 - - - - lea rax, [rdx + 2*rdx] + - - - 0.51 - 0.49 - - or rax, 3 + - - 0.01 0.49 - 0.50 - - add rax, 9 + - - - 0.02 - 0.98 - - cmp rsi, rax + - - - - - 1.00 - - jne .LBB5_4 + - - 0.51 0.49 - - 0.50 0.50 cmp word ptr [rdi], -16192 + - - - - - 1.00 - - je .LBB5_5 + - - - - - - - - xor edi, edi + - - 0.50 0.50 - - - - mov rdx, rsi + - - 0.50 0.48 - 0.02 - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.rs b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.rs new file mode 100644 index 000000000000..fc3cfbae27a2 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_bytes_dynamic_size(source: &[u8]) -> Option<&format::CocoPacket> { + zerocopy::TryFromBytes::try_ref_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 new file mode 100644 index 000000000000..cf67afd31ce0 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 @@ -0,0 +1,22 @@ +bench_try_ref_from_bytes_dynamic_size: + mov rdx, rsi + mov rax, rdi + cmp rsi, 4 + setb cl + or cl, al + test cl, 1 + jne .LBB5_4 + lea rcx, [rdx - 4] + mov rsi, rcx + and rsi, -2 + add rsi, 4 + cmp rdx, rsi + jne .LBB5_4 + cmp word ptr [rax], -16192 + jne .LBB5_4 + shr rcx + mov rdx, rcx + ret +.LBB5_4: + xor eax, eax + ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..ecd7a18f6d6d --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca @@ -0,0 +1,79 @@ +Iterations: 100 +Instructions: 2000 +Total Cycles: 639 +Total uOps: 2100 + +Dispatch Width: 4 +uOps Per Cycle: 3.29 +IPC: 3.13 +Block RThroughput: 5.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rdx, rsi + 1 1 0.33 mov rax, rdi + 1 1 0.33 cmp rsi, 4 + 1 1 0.50 setb cl + 1 1 0.33 or cl, al + 1 1 0.33 test cl, 1 + 1 1 1.00 jne .LBB5_4 + 1 1 0.50 lea rcx, [rdx - 4] + 1 1 0.33 mov rsi, rcx + 1 1 0.33 and rsi, -2 + 1 1 0.33 add rsi, 4 + 1 1 0.33 cmp rdx, rsi + 1 1 1.00 jne .LBB5_4 + 2 6 0.50 * cmp word ptr [rax], -16192 + 1 1 1.00 jne .LBB5_4 + 1 1 0.50 shr rcx + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.32 6.32 - 6.36 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.33 0.66 - 0.01 - - mov rdx, rsi + - - 0.66 0.34 - - - - mov rax, rdi + - - 0.34 0.66 - - - - cmp rsi, 4 + - - 0.99 - - 0.01 - - setb cl + - - 0.01 0.99 - - - - or cl, al + - - - 1.00 - - - - test cl, 1 + - - - - - 1.00 - - jne .LBB5_4 + - - 0.66 0.34 - - - - lea rcx, [rdx - 4] + - - 0.33 0.66 - 0.01 - - mov rsi, rcx + - - 1.00 - - - - - and rsi, -2 + - - 0.66 0.34 - - - - add rsi, 4 + - - - 1.00 - - - - cmp rdx, rsi + - - - - - 1.00 - - jne .LBB5_4 + - - - - - 1.00 0.50 0.50 cmp word ptr [rax], -16192 + - - - - - 1.00 - - jne .LBB5_4 + - - 0.67 - - 0.33 - - shr rcx + - - 0.67 0.33 - - - - mov rdx, rcx + - - - - - 1.00 - - ret + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_static_size.rs b/rust/zerocopy/benches/try_ref_from_bytes_static_size.rs new file mode 100644 index 000000000000..521557146324 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_static_size.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_bytes_static_size(source: &[u8]) -> Option<&format::CocoPacket> { + zerocopy::TryFromBytes::try_ref_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64 b/rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64 new file mode 100644 index 000000000000..a11f27189e90 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64 @@ -0,0 +1,13 @@ +bench_try_ref_from_bytes_static_size: + mov rax, rdi + cmp rsi, 6 + setne cl + or cl, al + test cl, 1 + jne .LBB5_2 + cmp word ptr [rax], -16192 + je .LBB5_3 +.LBB5_2: + xor eax, eax +.LBB5_3: + ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64.mca new file mode 100644 index 000000000000..e6bd20533a83 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64.mca @@ -0,0 +1,59 @@ +Iterations: 100 +Instructions: 1000 +Total Cycles: 308 +Total uOps: 1100 + +Dispatch Width: 4 +uOps Per Cycle: 3.57 +IPC: 3.25 +Block RThroughput: 3.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 cmp rsi, 6 + 1 1 0.50 setne cl + 1 1 0.33 or cl, al + 1 1 0.33 test cl, 1 + 1 1 1.00 jne .LBB5_2 + 2 6 0.50 * cmp word ptr [rax], -16192 + 1 1 1.00 je .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.98 2.98 - 3.04 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.02 0.97 - 0.01 - - mov rax, rdi + - - 0.02 0.98 - - - - cmp rsi, 6 + - - 1.00 - - - - - setne cl + - - 0.97 0.02 - 0.01 - - or cl, al + - - 0.96 0.03 - 0.01 - - test cl, 1 + - - - - - 1.00 - - jne .LBB5_2 + - - 0.01 0.98 - 0.01 0.50 0.50 cmp word ptr [rax], -16192 + - - - - - 1.00 - - je .LBB5_3 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..8b9e7355e367 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_bytes_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option<&format::CocoPacket> { + zerocopy::TryFromBytes::try_ref_from_bytes_with_elems(source, count).ok() +} diff --git a/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..3ef8d1448a50 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,21 @@ +bench_try_ref_from_bytes_with_elems_dynamic_padding: + movabs rax, 3074457345618258598 + cmp rdx, rax + seta cl + mov rax, rdi + test al, 3 + setne dil + or dil, cl + jne .LBB5_3 + lea rcx, [rdx + 2*rdx] + or rcx, 3 + add rcx, 9 + cmp rsi, rcx + jne .LBB5_3 + cmp word ptr [rax], -16192 + je .LBB5_4 +.LBB5_3: + xor eax, eax + mov rdx, rsi +.LBB5_4: + ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..8131f3bd549e --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,75 @@ +Iterations: 100 +Instructions: 1800 +Total Cycles: 607 +Total uOps: 2000 + +Dispatch Width: 4 +uOps Per Cycle: 3.29 +IPC: 2.97 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdx, rax + 2 2 1.00 seta cl + 1 1 0.33 mov rax, rdi + 1 1 0.33 test al, 3 + 1 1 0.50 setne dil + 1 1 0.33 or dil, cl + 1 1 1.00 jne .LBB5_3 + 1 1 0.50 lea rcx, [rdx + 2*rdx] + 1 1 0.33 or rcx, 3 + 1 1 0.33 add rcx, 9 + 1 1 0.33 cmp rsi, rcx + 1 1 1.00 jne .LBB5_3 + 2 6 0.50 * cmp word ptr [rax], -16192 + 1 1 1.00 je .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov rdx, rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.99 5.99 - 6.02 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - movabs rax, 3074457345618258598 + - - - 1.00 - - - - cmp rdx, rax + - - - - - 2.00 - - seta cl + - - 1.00 - - - - - mov rax, rdi + - - 0.99 0.01 - - - - test al, 3 + - - 1.00 - - - - - setne dil + - - - 0.99 - 0.01 - - or dil, cl + - - - - - 1.00 - - jne .LBB5_3 + - - 0.01 0.99 - - - - lea rcx, [rdx + 2*rdx] + - - - 1.00 - - - - or rcx, 3 + - - 0.99 0.01 - - - - add rcx, 9 + - - - 1.00 - - - - cmp rsi, rcx + - - - - - 1.00 - - jne .LBB5_3 + - - 1.00 - - - 0.50 0.50 cmp word ptr [rax], -16192 + - - - - - 1.00 - - je .LBB5_4 + - - - - - - - - xor eax, eax + - - 1.00 - - - - - mov rdx, rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.rs b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..9ccd6fef558a --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_bytes_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option<&format::CocoPacket> { + zerocopy::TryFromBytes::try_ref_from_bytes_with_elems(source, count).ok() +} diff --git a/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..ba34b1855bf1 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64 @@ -0,0 +1,18 @@ +bench_try_ref_from_bytes_with_elems_dynamic_size: + movabs rax, 4611686018427387901 + cmp rdx, rax + seta cl + mov rax, rdi + or dil, cl + test dil, 1 + jne .LBB5_3 + lea rcx, [2*rdx + 4] + cmp rsi, rcx + jne .LBB5_3 + cmp word ptr [rax], -16192 + je .LBB5_4 +.LBB5_3: + xor eax, eax + mov rdx, rsi +.LBB5_4: + ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..ae049c03dfde --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,69 @@ +Iterations: 100 +Instructions: 1500 +Total Cycles: 507 +Total uOps: 1700 + +Dispatch Width: 4 +uOps Per Cycle: 3.35 +IPC: 2.96 +Block RThroughput: 4.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdx, rax + 2 2 1.00 seta cl + 1 1 0.33 mov rax, rdi + 1 1 0.33 or dil, cl + 1 1 0.33 test dil, 1 + 1 1 1.00 jne .LBB5_3 + 1 1 0.50 lea rcx, [2*rdx + 4] + 1 1 0.33 cmp rsi, rcx + 1 1 1.00 jne .LBB5_3 + 2 6 0.50 * cmp word ptr [rax], -16192 + 1 1 1.00 je .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov rdx, rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.98 4.99 - 5.03 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - movabs rax, 4611686018427387901 + - - 0.50 0.50 - - - - cmp rdx, rax + - - 1.96 - - 0.04 - - seta cl + - - 0.01 0.99 - - - - mov rax, rdi + - - 1.00 - - - - - or dil, cl + - - 0.99 0.01 - - - - test dil, 1 + - - - - - 1.00 - - jne .LBB5_3 + - - 0.01 0.99 - - - - lea rcx, [2*rdx + 4] + - - 0.02 0.49 - 0.49 - - cmp rsi, rcx + - - - - - 1.00 - - jne .LBB5_3 + - - - 0.51 - 0.49 0.50 0.50 cmp word ptr [rax], -16192 + - - - - - 1.00 - - je .LBB5_4 + - - - - - - - - xor eax, eax + - - 0.49 0.51 - - - - mov rdx, rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.rs b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.rs new file mode 100644 index 000000000000..23b346f9c98d --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_prefix_dynamic_padding(source: &[u8]) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64 new file mode 100644 index 000000000000..d832cb7ecf7f --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64 @@ -0,0 +1,29 @@ +bench_try_ref_from_prefix_dynamic_padding: + xor edx, edx + mov eax, 0 + test dil, 3 + je .LBB5_1 + ret +.LBB5_1: + movabs rax, 9223372036854775804 + and rsi, rax + cmp rsi, 9 + jae .LBB5_3 + mov edx, 1 + xor eax, eax + ret +.LBB5_3: + add rsi, -9 + movabs rcx, -6148914691236517205 + mov rax, rsi + mul rcx + mov rax, rdx + shr rax + movzx ecx, word ptr [rdi] + cmp cx, -16192 + mov edx, 2 + cmove rdx, rax + xor eax, eax + cmp ecx, 49344 + cmove rax, rdi + ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..482112a39b33 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64.mca @@ -0,0 +1,91 @@ +Iterations: 100 +Instructions: 2600 +Total Cycles: 843 +Total uOps: 2900 + +Dispatch Width: 4 +uOps Per Cycle: 3.44 +IPC: 3.08 +Block RThroughput: 7.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor edx, edx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 3 + 1 1 1.00 je .LBB5_1 + 1 1 1.00 U ret + 1 1 0.33 movabs rax, 9223372036854775804 + 1 1 0.33 and rsi, rax + 1 1 0.33 cmp rsi, 9 + 1 1 1.00 jae .LBB5_3 + 1 1 0.33 mov edx, 1 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rsi, -9 + 1 1 0.33 movabs rcx, -6148914691236517205 + 1 1 0.33 mov rax, rsi + 2 4 1.00 mul rcx + 1 1 0.33 mov rax, rdx + 1 1 0.50 shr rax + 1 5 0.50 * movzx ecx, word ptr [rdi] + 1 1 0.33 cmp cx, -16192 + 1 1 0.33 mov edx, 2 + 2 2 0.67 cmove rdx, rax + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp ecx, 49344 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 8.33 8.33 - 8.34 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor edx, edx + - - 0.32 0.34 - 0.34 - - mov eax, 0 + - - 0.34 0.33 - 0.33 - - test dil, 3 + - - - - - 1.00 - - je .LBB5_1 + - - - - - 1.00 - - ret + - - 0.35 0.65 - - - - movabs rax, 9223372036854775804 + - - 0.96 0.03 - 0.01 - - and rsi, rax + - - 0.01 0.97 - 0.02 - - cmp rsi, 9 + - - - - - 1.00 - - jae .LBB5_3 + - - 0.67 0.01 - 0.32 - - mov edx, 1 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.02 0.34 - 0.64 - - add rsi, -9 + - - 0.33 0.66 - 0.01 - - movabs rcx, -6148914691236517205 + - - 0.66 0.34 - - - - mov rax, rsi + - - 1.00 1.00 - - - - mul rcx + - - 0.01 0.99 - - - - mov rax, rdx + - - 0.99 - - 0.01 - - shr rax + - - - - - - 0.50 0.50 movzx ecx, word ptr [rdi] + - - 0.33 0.03 - 0.64 - - cmp cx, -16192 + - - 0.01 0.31 - 0.68 - - mov edx, 2 + - - 1.00 1.00 - - - - cmove rdx, rax + - - - - - - - - xor eax, eax + - - 0.33 0.33 - 0.34 - - cmp ecx, 49344 + - - 1.00 1.00 - - - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.rs b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.rs new file mode 100644 index 000000000000..41a466ec8e6a --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_prefix_dynamic_size(source: &[u8]) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64 new file mode 100644 index 000000000000..be7f34b9f8f5 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64 @@ -0,0 +1,22 @@ +bench_try_ref_from_prefix_dynamic_size: + xor edx, edx + mov eax, 0 + test dil, 1 + jne .LBB5_4 + cmp rsi, 4 + jae .LBB5_3 + mov edx, 1 + xor eax, eax + ret +.LBB5_3: + add rsi, -4 + shr rsi + movzx ecx, word ptr [rdi] + cmp ecx, 49344 + mov edx, 2 + cmove rdx, rsi + xor eax, eax + cmp cx, -16192 + cmove rax, rdi +.LBB5_4: + ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..11706defe11e --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 573 +Total uOps: 2100 + +Dispatch Width: 4 +uOps Per Cycle: 3.66 +IPC: 3.32 +Block RThroughput: 5.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor edx, edx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 1 + 1 1 1.00 jne .LBB5_4 + 1 1 0.33 cmp rsi, 4 + 1 1 1.00 jae .LBB5_3 + 1 1 0.33 mov edx, 1 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rsi, -4 + 1 1 0.50 shr rsi + 1 5 0.50 * movzx ecx, word ptr [rdi] + 1 1 0.33 cmp ecx, 49344 + 1 1 0.33 mov edx, 2 + 2 2 0.67 cmove rdx, rsi + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp cx, -16192 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.66 5.67 - 5.67 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor edx, edx + - - 0.30 0.37 - 0.33 - - mov eax, 0 + - - 0.35 0.32 - 0.33 - - test dil, 1 + - - - - - 1.00 - - jne .LBB5_4 + - - 0.32 0.33 - 0.35 - - cmp rsi, 4 + - - - - - 1.00 - - jae .LBB5_3 + - - 0.33 0.35 - 0.32 - - mov edx, 1 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.34 0.64 - 0.02 - - add rsi, -4 + - - 1.00 - - - - - shr rsi + - - - - - - 0.50 0.50 movzx ecx, word ptr [rdi] + - - 0.60 0.40 - - - - cmp ecx, 49344 + - - 0.05 0.95 - - - - mov edx, 2 + - - 1.00 1.00 - - - - cmove rdx, rsi + - - - - - - - - xor eax, eax + - - 0.37 0.31 - 0.32 - - cmp cx, -16192 + - - 1.00 1.00 - - - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_static_size.rs b/rust/zerocopy/benches/try_ref_from_prefix_static_size.rs new file mode 100644 index 000000000000..5f13d482b505 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_static_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_prefix_static_size(source: &[u8]) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64 b/rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64 new file mode 100644 index 000000000000..83212f776ea6 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64 @@ -0,0 +1,15 @@ +bench_try_ref_from_prefix_static_size: + cmp rsi, 6 + setb al + or al, dil + test al, 1 + jne .LBB5_2 + movzx eax, word ptr [rdi] + cmp eax, 49344 + mov eax, 2 + cmove rax, rdi + je .LBB5_3 +.LBB5_2: + xor eax, eax +.LBB5_3: + ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64.mca new file mode 100644 index 000000000000..5d02b863a741 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64.mca @@ -0,0 +1,63 @@ +Iterations: 100 +Instructions: 1200 +Total Cycles: 374 +Total uOps: 1300 + +Dispatch Width: 4 +uOps Per Cycle: 3.48 +IPC: 3.21 +Block RThroughput: 3.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 cmp rsi, 6 + 1 1 0.50 setb al + 1 1 0.33 or al, dil + 1 1 0.33 test al, 1 + 1 1 1.00 jne .LBB5_2 + 1 5 0.50 * movzx eax, word ptr [rdi] + 1 1 0.33 cmp eax, 49344 + 1 1 0.33 mov eax, 2 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 je .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.66 3.65 - 3.69 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.35 0.64 - 0.01 - - cmp rsi, 6 + - - 1.00 - - - - - setb al + - - 0.02 0.66 - 0.32 - - or al, dil + - - 0.03 0.65 - 0.32 - - test al, 1 + - - - - - 1.00 - - jne .LBB5_2 + - - - - - - 0.50 0.50 movzx eax, word ptr [rdi] + - - 0.92 0.07 - 0.01 - - cmp eax, 49344 + - - 0.37 0.63 - - - - mov eax, 2 + - - 0.97 1.00 - 0.03 - - cmove rax, rdi + - - - - - 1.00 - - je .LBB5_3 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..1744a40759b1 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_prefix_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_prefix_with_elems(source, count) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..80e66ba1601c --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,30 @@ +bench_try_ref_from_prefix_with_elems_dynamic_padding: + movabs rax, 3074457345618258598 + cmp rdx, rax + ja .LBB5_1 + xor ecx, ecx + mov eax, 0 + test dil, 3 + je .LBB5_3 + mov rdx, rcx + ret +.LBB5_3: + lea rax, [rdx + 2*rdx] + or rax, 3 + add rax, 9 + cmp rax, rsi + jbe .LBB5_4 +.LBB5_1: + xor eax, eax + mov edx, 1 + ret +.LBB5_4: + movzx esi, word ptr [rdi] + cmp si, -16192 + mov ecx, 2 + cmove rcx, rdx + xor eax, eax + cmp esi, 49344 + cmove rax, rdi + mov rdx, rcx + ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..512e8ce64393 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,91 @@ +Iterations: 100 +Instructions: 2600 +Total Cycles: 806 +Total uOps: 2800 + +Dispatch Width: 4 +uOps Per Cycle: 3.47 +IPC: 3.23 +Block RThroughput: 7.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 0 0.25 xor ecx, ecx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 3 + 1 1 1.00 je .LBB5_3 + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, 9 + 1 1 0.33 cmp rax, rsi + 1 1 1.00 jbe .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov edx, 1 + 1 1 1.00 U ret + 1 5 0.50 * movzx esi, word ptr [rdi] + 1 1 0.33 cmp si, -16192 + 1 1 0.33 mov ecx, 2 + 2 2 0.67 cmove rcx, rdx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp esi, 49344 + 2 2 0.67 cmove rax, rdi + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 7.98 7.99 - 8.03 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.98 - - 0.02 - - movabs rax, 3074457345618258598 + - - - 1.00 - - - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - - - - - - - xor ecx, ecx + - - 0.99 0.01 - - - - mov eax, 0 + - - 0.01 0.96 - 0.03 - - test dil, 3 + - - - - - 1.00 - - je .LBB5_3 + - - 0.97 0.01 - 0.02 - - mov rdx, rcx + - - - - - 1.00 - - ret + - - 0.03 0.97 - - - - lea rax, [rdx + 2*rdx] + - - 0.03 0.97 - - - - or rax, 3 + - - 0.01 0.99 - - - - add rax, 9 + - - - 1.00 - - - - cmp rax, rsi + - - - - - 1.00 - - jbe .LBB5_4 + - - - - - - - - xor eax, eax + - - 0.98 0.01 - 0.01 - - mov edx, 1 + - - - - - 1.00 - - ret + - - - - - - 0.50 0.50 movzx esi, word ptr [rdi] + - - 0.97 0.03 - - - - cmp si, -16192 + - - 0.98 0.01 - 0.01 - - mov ecx, 2 + - - 1.00 0.03 - 0.97 - - cmove rcx, rdx + - - - - - - - - xor eax, eax + - - 0.03 0.97 - - - - cmp esi, 49344 + - - 1.00 1.00 - - - - cmove rax, rdi + - - - 0.03 - 0.97 - - mov rdx, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.rs b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..ed0f50941194 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_prefix_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_prefix_with_elems(source, count) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..c12e87c137c5 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64 @@ -0,0 +1,26 @@ +bench_try_ref_from_prefix_with_elems_dynamic_size: + movabs rax, 4611686018427387901 + cmp rdx, rax + ja .LBB5_1 + mov rcx, rdx + xor edx, edx + mov eax, 0 + test dil, 1 + jne .LBB5_5 + lea rax, [2*rcx + 4] + cmp rax, rsi + jbe .LBB5_4 +.LBB5_1: + xor eax, eax + mov edx, 1 + ret +.LBB5_4: + movzx esi, word ptr [rdi] + cmp si, -16192 + mov edx, 2 + cmove rdx, rcx + xor eax, eax + cmp esi, 49344 + cmove rax, rdi +.LBB5_5: + ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..6c3f1a1ec97a --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,83 @@ +Iterations: 100 +Instructions: 2200 +Total Cycles: 674 +Total uOps: 2400 + +Dispatch Width: 4 +uOps Per Cycle: 3.56 +IPC: 3.26 +Block RThroughput: 6.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.33 mov rcx, rdx + 1 0 0.25 xor edx, edx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 1 + 1 1 1.00 jne .LBB5_5 + 1 1 0.50 lea rax, [2*rcx + 4] + 1 1 0.33 cmp rax, rsi + 1 1 1.00 jbe .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov edx, 1 + 1 1 1.00 U ret + 1 5 0.50 * movzx esi, word ptr [rdi] + 1 1 0.33 cmp si, -16192 + 1 1 0.33 mov edx, 2 + 2 2 0.67 cmove rdx, rcx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp esi, 49344 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.65 6.66 - 6.69 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.66 0.33 - 0.01 - - movabs rax, 4611686018427387901 + - - 0.02 0.66 - 0.32 - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.66 0.33 - 0.01 - - mov rcx, rdx + - - - - - - - - xor edx, edx + - - 0.33 0.01 - 0.66 - - mov eax, 0 + - - 0.34 0.65 - 0.01 - - test dil, 1 + - - - - - 1.00 - - jne .LBB5_5 + - - 0.65 0.35 - - - - lea rax, [2*rcx + 4] + - - - 1.00 - - - - cmp rax, rsi + - - - - - 1.00 - - jbe .LBB5_4 + - - - - - - - - xor eax, eax + - - 0.34 0.01 - 0.65 - - mov edx, 1 + - - - - - 1.00 - - ret + - - - - - - 0.50 0.50 movzx esi, word ptr [rdi] + - - 0.65 0.34 - 0.01 - - cmp si, -16192 + - - 0.66 0.34 - - - - mov edx, 2 + - - 1.00 0.99 - 0.01 - - cmove rdx, rcx + - - - - - - - - xor eax, eax + - - 0.34 0.66 - - - - cmp esi, 49344 + - - 1.00 0.99 - 0.01 - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.rs b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.rs new file mode 100644 index 000000000000..981feca3ca24 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_suffix_dynamic_padding(source: &[u8]) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64 new file mode 100644 index 000000000000..b3e924442865 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64 @@ -0,0 +1,26 @@ +bench_try_ref_from_suffix_dynamic_padding: + lea eax, [rsi + rdi] + test al, 3 + jne .LBB5_1 + movabs rax, 9223372036854775804 + and rax, rsi + cmp rax, 9 + jae .LBB5_3 +.LBB5_1: + xor eax, eax + ret +.LBB5_3: + add rax, -9 + movabs rcx, -6148914691236517205 + mul rcx + shr rdx + lea rcx, [rdx + 2*rdx] + sub rsi, rcx + or rcx, -4 + add rsi, rdi + lea rdi, [rcx + rsi] + add rdi, -8 + xor eax, eax + cmp word ptr [rcx + rsi - 8], -16192 + cmove rax, rdi + ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..d56ae56d854a --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64.mca @@ -0,0 +1,85 @@ +Iterations: 100 +Instructions: 2300 +Total Cycles: 791 +Total uOps: 2600 + +Dispatch Width: 4 +uOps Per Cycle: 3.29 +IPC: 2.91 +Block RThroughput: 6.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea eax, [rsi + rdi] + 1 1 0.33 test al, 3 + 1 1 1.00 jne .LBB5_1 + 1 1 0.33 movabs rax, 9223372036854775804 + 1 1 0.33 and rax, rsi + 1 1 0.33 cmp rax, 9 + 1 1 1.00 jae .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rax, -9 + 1 1 0.33 movabs rcx, -6148914691236517205 + 2 4 1.00 mul rcx + 1 1 0.50 shr rdx + 1 1 0.50 lea rcx, [rdx + 2*rdx] + 1 1 0.33 sub rsi, rcx + 1 1 0.33 or rcx, -4 + 1 1 0.33 add rsi, rdi + 1 1 0.50 lea rdi, [rcx + rsi] + 1 1 0.33 add rdi, -8 + 1 0 0.25 xor eax, eax + 2 6 0.50 * cmp word ptr [rcx + rsi - 8], -16192 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 7.70 7.58 - 7.72 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.26 0.74 - - - - lea eax, [rsi + rdi] + - - 0.19 0.28 - 0.53 - - test al, 3 + - - - - - 1.00 - - jne .LBB5_1 + - - 0.93 0.06 - 0.01 - - movabs rax, 9223372036854775804 + - - 0.81 0.14 - 0.05 - - and rax, rsi + - - 0.55 0.43 - 0.02 - - cmp rax, 9 + - - - - - 1.00 - - jae .LBB5_3 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.42 0.56 - 0.02 - - add rax, -9 + - - 0.67 0.30 - 0.03 - - movabs rcx, -6148914691236517205 + - - 1.00 1.00 - - - - mul rcx + - - 0.71 - - 0.29 - - shr rdx + - - 0.32 0.68 - - - - lea rcx, [rdx + 2*rdx] + - - 0.57 0.04 - 0.39 - - sub rsi, rcx + - - 0.28 0.67 - 0.05 - - or rcx, -4 + - - 0.29 0.29 - 0.42 - - add rsi, rdi + - - 0.02 0.98 - - - - lea rdi, [rcx + rsi] + - - 0.02 0.41 - 0.57 - - add rdi, -8 + - - - - - - - - xor eax, eax + - - 0.57 0.01 - 0.42 0.50 0.50 cmp word ptr [rcx + rsi - 8], -16192 + - - 0.09 0.99 - 0.92 - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.rs b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.rs new file mode 100644 index 000000000000..c3d75ac9b3e1 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_suffix_dynamic_size(source: &[u8]) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64 new file mode 100644 index 000000000000..d51f7817e599 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64 @@ -0,0 +1,18 @@ +bench_try_ref_from_suffix_dynamic_size: + lea eax, [rsi + rdi] + cmp rsi, 4 + setb cl + or cl, al + test cl, 1 + je .LBB5_2 + xor eax, eax + ret +.LBB5_2: + lea rdx, [rsi - 4] + shr rdx + and esi, 1 + lea rcx, [rdi + rsi] + xor eax, eax + cmp word ptr [rdi + rsi], -16192 + cmove rax, rcx + ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..6cf7f8e493f5 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64.mca @@ -0,0 +1,71 @@ +Iterations: 100 +Instructions: 1600 +Total Cycles: 510 +Total uOps: 1800 + +Dispatch Width: 4 +uOps Per Cycle: 3.53 +IPC: 3.14 +Block RThroughput: 4.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea eax, [rsi + rdi] + 1 1 0.33 cmp rsi, 4 + 1 1 0.50 setb cl + 1 1 0.33 or cl, al + 1 1 0.33 test cl, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.50 lea rdx, [rsi - 4] + 1 1 0.50 shr rdx + 1 1 0.33 and esi, 1 + 1 1 0.50 lea rcx, [rdi + rsi] + 1 0 0.25 xor eax, eax + 2 6 0.50 * cmp word ptr [rdi + rsi], -16192 + 2 2 0.67 cmove rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.99 5.00 - 5.01 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.98 0.02 - - - - lea eax, [rsi + rdi] + - - - 0.98 - 0.02 - - cmp rsi, 4 + - - 1.00 - - - - - setb cl + - - 0.01 0.99 - - - - or cl, al + - - 0.01 0.07 - 0.92 - - test cl, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.93 0.07 - - - - lea rdx, [rsi - 4] + - - 0.93 - - 0.07 - - shr rdx + - - 0.06 0.93 - 0.01 - - and esi, 1 + - - 0.07 0.93 - - - - lea rcx, [rdi + rsi] + - - - - - - - - xor eax, eax + - - - 0.01 - 0.99 0.50 0.50 cmp word ptr [rdi + rsi], -16192 + - - 1.00 1.00 - - - - cmove rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_static_size.rs b/rust/zerocopy/benches/try_ref_from_suffix_static_size.rs new file mode 100644 index 000000000000..d4b92f639a32 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_static_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_suffix_static_size(source: &[u8]) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64 b/rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64 new file mode 100644 index 000000000000..cd39f70931bc --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64 @@ -0,0 +1,16 @@ +bench_try_ref_from_suffix_static_size: + lea eax, [rsi + rdi] + cmp rsi, 6 + setb cl + or cl, al + test cl, 1 + je .LBB5_2 + xor eax, eax + ret +.LBB5_2: + lea rcx, [rdi + rsi] + add rcx, -6 + xor eax, eax + cmp word ptr [rdi + rsi - 6], -16192 + cmove rax, rcx + ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64.mca new file mode 100644 index 000000000000..087d1e7ed971 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64.mca @@ -0,0 +1,67 @@ +Iterations: 100 +Instructions: 1400 +Total Cycles: 443 +Total uOps: 1600 + +Dispatch Width: 4 +uOps Per Cycle: 3.61 +IPC: 3.16 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea eax, [rsi + rdi] + 1 1 0.33 cmp rsi, 6 + 1 1 0.50 setb cl + 1 1 0.33 or cl, al + 1 1 0.33 test cl, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.50 lea rcx, [rdi + rsi] + 1 1 0.33 add rcx, -6 + 1 0 0.25 xor eax, eax + 2 6 0.50 * cmp word ptr [rdi + rsi - 6], -16192 + 2 2 0.67 cmove rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.33 4.33 - 4.34 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.32 0.68 - - - - lea eax, [rsi + rdi] + - - 0.05 0.94 - 0.01 - - cmp rsi, 6 + - - 1.00 - - - - - setb cl + - - 0.95 0.05 - - - - or cl, al + - - 0.95 0.02 - 0.03 - - test cl, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.04 0.96 - - - - lea rcx, [rdi + rsi] + - - 0.02 0.97 - 0.01 - - add rcx, -6 + - - - - - - - - xor eax, eax + - - 0.03 0.66 - 0.31 0.50 0.50 cmp word ptr [rdi + rsi - 6], -16192 + - - 0.97 0.05 - 0.98 - - cmove rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..1da455c9a238 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_suffix_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_suffix_with_elems(source, count) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..c7530d8b6815 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,32 @@ +bench_try_ref_from_suffix_with_elems_dynamic_padding: + movabs rax, 3074457345618258598 + cmp rdx, rax + ja .LBB5_1 + lea r8d, [rsi + rdi] + xor ecx, ecx + mov eax, 0 + test r8b, 3 + je .LBB5_3 + mov rdx, rcx + ret +.LBB5_3: + lea rax, [rdx + 2*rdx] + or rax, 3 + add rax, 9 + sub rsi, rax + jae .LBB5_4 +.LBB5_1: + xor eax, eax + mov edx, 1 + ret +.LBB5_4: + lea r8, [rdi + rsi] + movzx esi, word ptr [rdi + rsi] + cmp si, -16192 + mov ecx, 2 + cmove rcx, rdx + xor eax, eax + cmp esi, 49344 + cmove rax, r8 + mov rdx, rcx + ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..be736c00c250 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,95 @@ +Iterations: 100 +Instructions: 2800 +Total Cycles: 878 +Total uOps: 3000 + +Dispatch Width: 4 +uOps Per Cycle: 3.42 +IPC: 3.19 +Block RThroughput: 7.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.50 lea r8d, [rsi + rdi] + 1 0 0.25 xor ecx, ecx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test r8b, 3 + 1 1 1.00 je .LBB5_3 + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, 9 + 1 1 0.33 sub rsi, rax + 1 1 1.00 jae .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov edx, 1 + 1 1 1.00 U ret + 1 1 0.50 lea r8, [rdi + rsi] + 1 5 0.50 * movzx esi, word ptr [rdi + rsi] + 1 1 0.33 cmp si, -16192 + 1 1 0.33 mov ecx, 2 + 2 2 0.67 cmove rcx, rdx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp esi, 49344 + 2 2 0.67 cmove rax, r8 + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 8.65 8.65 - 8.70 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.67 0.30 - 0.03 - - movabs rax, 3074457345618258598 + - - 0.01 0.99 - - - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.99 0.01 - - - - lea r8d, [rsi + rdi] + - - - - - - - - xor ecx, ecx + - - 0.35 0.62 - 0.03 - - mov eax, 0 + - - 0.99 0.01 - - - - test r8b, 3 + - - - - - 1.00 - - je .LBB5_3 + - - 0.68 0.30 - 0.02 - - mov rdx, rcx + - - - - - 1.00 - - ret + - - 0.07 0.93 - - - - lea rax, [rdx + 2*rdx] + - - 0.06 0.35 - 0.59 - - or rax, 3 + - - 0.02 0.07 - 0.91 - - add rax, 9 + - - 0.01 0.04 - 0.95 - - sub rsi, rax + - - - - - 1.00 - - jae .LBB5_4 + - - - - - - - - xor eax, eax + - - 0.92 0.01 - 0.07 - - mov edx, 1 + - - - - - 1.00 - - ret + - - - 1.00 - - - - lea r8, [rdi + rsi] + - - - - - - 0.50 0.50 movzx esi, word ptr [rdi + rsi] + - - 0.01 0.99 - - - - cmp si, -16192 + - - 0.88 0.04 - 0.08 - - mov ecx, 2 + - - 1.00 0.99 - 0.01 - - cmove rcx, rdx + - - - - - - - - xor eax, eax + - - 0.99 0.01 - - - - cmp esi, 49344 + - - 1.00 1.00 - - - - cmove rax, r8 + - - - 0.99 - 0.01 - - mov rdx, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.rs b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..8c2b80f8762f --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_suffix_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_suffix_with_elems(source, count) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..952eb12de8d6 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64 @@ -0,0 +1,28 @@ +bench_try_ref_from_suffix_with_elems_dynamic_size: + movabs rax, 4611686018427387901 + cmp rdx, rax + ja .LBB5_1 + lea r8d, [rsi + rdi] + xor ecx, ecx + mov eax, 0 + test r8b, 1 + jne .LBB5_5 + lea rax, [2*rdx + 4] + sub rsi, rax + jae .LBB5_4 +.LBB5_1: + xor eax, eax + mov edx, 1 + ret +.LBB5_4: + lea r8, [rdi + rsi] + movzx esi, word ptr [rdi + rsi] + cmp si, -16192 + mov ecx, 2 + cmove rcx, rdx + xor eax, eax + cmp esi, 49344 + cmove rax, r8 +.LBB5_5: + mov rdx, rcx + ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..d4f78f67a252 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,87 @@ +Iterations: 100 +Instructions: 2400 +Total Cycles: 1107 +Total uOps: 2600 + +Dispatch Width: 4 +uOps Per Cycle: 2.35 +IPC: 2.17 +Block RThroughput: 6.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.50 lea r8d, [rsi + rdi] + 1 0 0.25 xor ecx, ecx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test r8b, 1 + 1 1 1.00 jne .LBB5_5 + 1 1 0.50 lea rax, [2*rdx + 4] + 1 1 0.33 sub rsi, rax + 1 1 1.00 jae .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov edx, 1 + 1 1 1.00 U ret + 1 1 0.50 lea r8, [rdi + rsi] + 1 5 0.50 * movzx esi, word ptr [rdi + rsi] + 1 1 0.33 cmp si, -16192 + 1 1 0.33 mov ecx, 2 + 2 2 0.67 cmove rcx, rdx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp esi, 49344 + 2 2 0.67 cmove rax, r8 + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.99 7.00 - 8.01 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.02 0.95 - 0.03 - - movabs rax, 4611686018427387901 + - - 0.93 0.04 - 0.03 - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.96 0.04 - - - - lea r8d, [rsi + rdi] + - - - - - - - - xor ecx, ecx + - - 0.95 0.02 - 0.03 - - mov eax, 0 + - - 0.95 0.05 - - - - test r8b, 1 + - - - - - 1.00 - - jne .LBB5_5 + - - 0.06 0.94 - - - - lea rax, [2*rdx + 4] + - - 0.93 0.07 - - - - sub rsi, rax + - - - - - 1.00 - - jae .LBB5_4 + - - - - - - - - xor eax, eax + - - 0.03 0.95 - 0.02 - - mov edx, 1 + - - - - - 1.00 - - ret + - - 0.97 0.03 - - - - lea r8, [rdi + rsi] + - - - - - - 0.50 0.50 movzx esi, word ptr [rdi + rsi] + - - 0.03 0.97 - - - - cmp si, -16192 + - - 0.05 0.94 - 0.01 - - mov ecx, 2 + - - 0.06 0.98 - 0.96 - - cmove rcx, rdx + - - - - - - - - xor eax, eax + - - 0.97 0.03 - - - - cmp esi, 49344 + - - 0.06 0.96 - 0.98 - - cmove rax, r8 + - - 0.02 0.03 - 0.95 - - mov rdx, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_transmute.rs b/rust/zerocopy/benches/try_transmute.rs new file mode 100644 index 000000000000..c0de07a8d094 --- /dev/null +++ b/rust/zerocopy/benches/try_transmute.rs @@ -0,0 +1,16 @@ +use zerocopy::Unalign; +use zerocopy_derive::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[derive(IntoBytes, KnownLayout, Immutable)] +#[repr(C)] +struct MinimalViableSource { + bytes: [u8; 6], +} + +#[unsafe(no_mangle)] +fn bench_try_transmute(source: MinimalViableSource) -> Option<Unalign<format::CocoPacket>> { + zerocopy::try_transmute!(source).ok() +} diff --git a/rust/zerocopy/benches/try_transmute.x86-64 b/rust/zerocopy/benches/try_transmute.x86-64 new file mode 100644 index 000000000000..9e16a663257c --- /dev/null +++ b/rust/zerocopy/benches/try_transmute.x86-64 @@ -0,0 +1,9 @@ +bench_try_transmute: + movzx ecx, di + xor eax, eax + cmp ecx, 49344 + sete al + and rdi, -65536 + xor rax, 49345 + or rax, rdi + ret diff --git a/rust/zerocopy/benches/try_transmute.x86-64.mca b/rust/zerocopy/benches/try_transmute.x86-64.mca new file mode 100644 index 000000000000..33abc3bf341e --- /dev/null +++ b/rust/zerocopy/benches/try_transmute.x86-64.mca @@ -0,0 +1,55 @@ +Iterations: 100 +Instructions: 800 +Total Cycles: 238 +Total uOps: 800 + +Dispatch Width: 4 +uOps Per Cycle: 3.36 +IPC: 3.36 +Block RThroughput: 2.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movzx ecx, di + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp ecx, 49344 + 1 1 0.50 sete al + 1 1 0.33 and rdi, -65536 + 1 1 0.33 xor rax, 49345 + 1 1 0.33 or rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.33 2.33 - 2.34 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.32 0.67 - 0.01 - - movzx ecx, di + - - - - - - - - xor eax, eax + - - 0.33 0.67 - - - - cmp ecx, 49344 + - - 1.00 - - - - - sete al + - - 0.67 0.33 - - - - and rdi, -65536 + - - - 0.66 - 0.34 - - xor rax, 49345 + - - 0.01 - - 0.99 - - or rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_transmute_ref_dynamic_size.rs b/rust/zerocopy/benches/try_transmute_ref_dynamic_size.rs new file mode 100644 index 000000000000..c9236e13d23c --- /dev/null +++ b/rust/zerocopy/benches/try_transmute_ref_dynamic_size.rs @@ -0,0 +1,18 @@ +use zerocopy_derive::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[derive(IntoBytes, KnownLayout, Immutable)] +#[repr(C, align(2))] +struct MinimalViableSource { + header: [u8; 6], + trailer: [[u8; 2]], +} + +#[unsafe(no_mangle)] +fn bench_try_transmute_ref_dynamic_size( + source: &MinimalViableSource, +) -> Option<&format::CocoPacket> { + zerocopy::try_transmute_ref!(source).ok() +} diff --git a/rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64 b/rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64 new file mode 100644 index 000000000000..d34d2b3eb3df --- /dev/null +++ b/rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64 @@ -0,0 +1,6 @@ +bench_try_transmute_ref_dynamic_size: + lea rdx, [rsi + 1] + xor eax, eax + cmp word ptr [rdi], -16192 + cmove rax, rdi + ret diff --git a/rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..bc771b504713 --- /dev/null +++ b/rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64.mca @@ -0,0 +1,49 @@ +Iterations: 100 +Instructions: 500 +Total Cycles: 209 +Total uOps: 700 + +Dispatch Width: 4 +uOps Per Cycle: 3.35 +IPC: 2.39 +Block RThroughput: 1.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea rdx, [rsi + 1] + 1 0 0.25 xor eax, eax + 2 6 0.50 * cmp word ptr [rdi], -16192 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.50 1.51 - 1.99 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.51 0.49 - - - - lea rdx, [rsi + 1] + - - - - - - - - xor eax, eax + - - - 0.02 - 0.98 0.50 0.50 cmp word ptr [rdi], -16192 + - - 0.99 1.00 - 0.01 - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_transmute_ref_static_size.rs b/rust/zerocopy/benches/try_transmute_ref_static_size.rs new file mode 100644 index 000000000000..631cce2b0bb9 --- /dev/null +++ b/rust/zerocopy/benches/try_transmute_ref_static_size.rs @@ -0,0 +1,17 @@ +use zerocopy_derive::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[derive(IntoBytes, KnownLayout, Immutable)] +#[repr(C, align(2))] +struct MinimalViableSource { + bytes: [u8; 6], +} + +#[unsafe(no_mangle)] +fn bench_try_transmute_ref_static_size( + source: &MinimalViableSource, +) -> Option<&format::CocoPacket> { + zerocopy::try_transmute_ref!(source).ok() +} diff --git a/rust/zerocopy/benches/try_transmute_ref_static_size.x86-64 b/rust/zerocopy/benches/try_transmute_ref_static_size.x86-64 new file mode 100644 index 000000000000..8c16face9875 --- /dev/null +++ b/rust/zerocopy/benches/try_transmute_ref_static_size.x86-64 @@ -0,0 +1,5 @@ +bench_try_transmute_ref_static_size: + xor eax, eax + cmp word ptr [rdi], -16192 + cmove rax, rdi + ret diff --git a/rust/zerocopy/benches/try_transmute_ref_static_size.x86-64.mca b/rust/zerocopy/benches/try_transmute_ref_static_size.x86-64.mca new file mode 100644 index 000000000000..cf7384989536 --- /dev/null +++ b/rust/zerocopy/benches/try_transmute_ref_static_size.x86-64.mca @@ -0,0 +1,47 @@ +Iterations: 100 +Instructions: 400 +Total Cycles: 160 +Total uOps: 600 + +Dispatch Width: 4 +uOps Per Cycle: 3.75 +IPC: 2.50 +Block RThroughput: 1.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor eax, eax + 2 6 0.50 * cmp word ptr [rdi], -16192 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.02 1.48 - 1.50 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor eax, eax + - - 0.02 0.49 - 0.49 0.50 0.50 cmp word ptr [rdi], -16192 + - - 1.00 0.99 - 0.01 - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_dynamic_size.rs b/rust/zerocopy/benches/write_to_dynamic_size.rs new file mode 100644 index 000000000000..c126a1468c9b --- /dev/null +++ b/rust/zerocopy/benches/write_to_dynamic_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_write_to_dynamic_size(source: &format::CocoPacket, destination: &mut [u8]) -> Option<()> { + source.write_to(destination).ok() +} diff --git a/rust/zerocopy/benches/write_to_dynamic_size.x86-64 b/rust/zerocopy/benches/write_to_dynamic_size.x86-64 new file mode 100644 index 000000000000..c5abb17f7eba --- /dev/null +++ b/rust/zerocopy/benches/write_to_dynamic_size.x86-64 @@ -0,0 +1,21 @@ +bench_write_to_dynamic_size: + push r14 + push rbx + push rax + mov rbx, rcx + lea r14, [2*rsi + 5] + and r14, -2 + cmp rcx, r14 + jne .LBB5_2 + mov rax, rdi + mov rdi, rdx + mov rsi, rax + mov rdx, rbx + call qword ptr [rip + memcpy@GOTPCREL] +.LBB5_2: + cmp rbx, r14 + sete al + add rsp, 8 + pop rbx + pop r14 + ret diff --git a/rust/zerocopy/benches/write_to_dynamic_size.x86-64.mca b/rust/zerocopy/benches/write_to_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..5b2c08a31a29 --- /dev/null +++ b/rust/zerocopy/benches/write_to_dynamic_size.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 2890 +Total uOps: 2500 + +Dispatch Width: 4 +uOps Per Cycle: 0.87 +IPC: 0.66 +Block RThroughput: 6.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push r14 + 2 5 1.00 * push rbx + 2 5 1.00 * push rax + 1 1 0.33 mov rbx, rcx + 1 1 0.50 lea r14, [2*rsi + 5] + 1 1 0.33 and r14, -2 + 1 1 0.33 cmp rcx, r14 + 1 1 1.00 jne .LBB5_2 + 1 1 0.33 mov rax, rdi + 1 1 0.33 mov rdi, rdx + 1 1 0.33 mov rsi, rax + 1 1 0.33 mov rdx, rbx + 4 7 1.00 * call qword ptr [rip + memcpy@GOTPCREL] + 1 1 0.33 cmp rbx, r14 + 1 1 0.50 sete al + 1 1 0.33 add rsp, 8 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r14 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.66 4.64 4.00 4.70 4.00 3.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - - 1.00 push r14 + - - - - 1.00 - 1.00 - push rbx + - - - - 1.00 - - 1.00 push rax + - - 0.02 0.97 - 0.01 - - mov rbx, rcx + - - 0.97 0.03 - - - - lea r14, [2*rsi + 5] + - - 0.63 0.35 - 0.02 - - and r14, -2 + - - 0.31 0.34 - 0.35 - - cmp rcx, r14 + - - - - - 1.00 - - jne .LBB5_2 + - - 0.33 0.33 - 0.34 - - mov rax, rdi + - - 0.36 0.31 - 0.33 - - mov rdi, rdx + - - 0.33 0.35 - 0.32 - - mov rsi, rax + - - 0.35 0.63 - 0.02 - - mov rdx, rbx + - - - - 1.00 1.00 2.00 - call qword ptr [rip + memcpy@GOTPCREL] + - - 0.65 0.35 - - - - cmp rbx, r14 + - - 0.69 - - 0.31 - - sete al + - - 0.02 0.98 - - - - add rsp, 8 + - - - - - - - 1.00 pop rbx + - - - - - - 1.00 - pop r14 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_prefix_dynamic_size.rs b/rust/zerocopy/benches/write_to_prefix_dynamic_size.rs new file mode 100644 index 000000000000..a54d32773113 --- /dev/null +++ b/rust/zerocopy/benches/write_to_prefix_dynamic_size.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_write_to_prefix_dynamic_size( + source: &format::CocoPacket, + destination: &mut [u8], +) -> Option<()> { + source.write_to_prefix(destination).ok() +} diff --git a/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64 b/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64 new file mode 100644 index 000000000000..d7779c6c9178 --- /dev/null +++ b/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64 @@ -0,0 +1,21 @@ +bench_write_to_prefix_dynamic_size: + push r14 + push rbx + push rax + mov rbx, rcx + lea r14, [2*rsi + 5] + and r14, -2 + cmp r14, rcx + ja .LBB5_2 + mov rax, rdi + mov rdi, rdx + mov rsi, rax + mov rdx, r14 + call qword ptr [rip + memcpy@GOTPCREL] +.LBB5_2: + cmp r14, rbx + setbe al + add rsp, 8 + pop rbx + pop r14 + ret diff --git a/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..4cebe24d4f3b --- /dev/null +++ b/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 2890 +Total uOps: 2600 + +Dispatch Width: 4 +uOps Per Cycle: 0.90 +IPC: 0.66 +Block RThroughput: 6.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push r14 + 2 5 1.00 * push rbx + 2 5 1.00 * push rax + 1 1 0.33 mov rbx, rcx + 1 1 0.50 lea r14, [2*rsi + 5] + 1 1 0.33 and r14, -2 + 1 1 0.33 cmp r14, rcx + 1 1 1.00 ja .LBB5_2 + 1 1 0.33 mov rax, rdi + 1 1 0.33 mov rdi, rdx + 1 1 0.33 mov rsi, rax + 1 1 0.33 mov rdx, r14 + 4 7 1.00 * call qword ptr [rip + memcpy@GOTPCREL] + 1 1 0.33 cmp r14, rbx + 2 2 1.00 setbe al + 1 1 0.33 add rsp, 8 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r14 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.47 4.49 4.00 5.04 4.00 3.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - - 1.00 push r14 + - - - - 1.00 - 1.00 - push rbx + - - - - 1.00 - - 1.00 push rax + - - 0.48 0.51 - 0.01 - - mov rbx, rcx + - - 0.51 0.49 - - - - lea r14, [2*rsi + 5] + - - 0.48 0.05 - 0.47 - - and r14, -2 + - - 0.48 0.49 - 0.03 - - cmp r14, rcx + - - - - - 1.00 - - ja .LBB5_2 + - - 0.04 0.47 - 0.49 - - mov rax, rdi + - - 0.49 0.03 - 0.48 - - mov rdi, rdx + - - 0.03 0.48 - 0.49 - - mov rsi, rax + - - 0.48 0.51 - 0.01 - - mov rdx, r14 + - - - - 1.00 1.00 2.00 - call qword ptr [rip + memcpy@GOTPCREL] + - - 0.51 0.49 - - - - cmp r14, rbx + - - 1.94 - - 0.06 - - setbe al + - - 0.03 0.97 - - - - add rsp, 8 + - - - - - - - 1.00 pop rbx + - - - - - - 1.00 - pop r14 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_prefix_static_size.rs b/rust/zerocopy/benches/write_to_prefix_static_size.rs new file mode 100644 index 000000000000..826222c129fe --- /dev/null +++ b/rust/zerocopy/benches/write_to_prefix_static_size.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_write_to_prefix_static_size( + source: &format::CocoPacket, + destination: &mut [u8], +) -> Option<()> { + source.write_to_prefix(destination).ok() +} diff --git a/rust/zerocopy/benches/write_to_prefix_static_size.x86-64 b/rust/zerocopy/benches/write_to_prefix_static_size.x86-64 new file mode 100644 index 000000000000..9cf066295304 --- /dev/null +++ b/rust/zerocopy/benches/write_to_prefix_static_size.x86-64 @@ -0,0 +1,11 @@ +bench_write_to_prefix_static_size: + cmp rdx, 6 + jb .LBB5_2 + movzx eax, word ptr [rdi + 4] + mov word ptr [rsi + 4], ax + mov eax, dword ptr [rdi] + mov dword ptr [rsi], eax +.LBB5_2: + cmp rdx, 6 + setae al + ret diff --git a/rust/zerocopy/benches/write_to_prefix_static_size.x86-64.mca b/rust/zerocopy/benches/write_to_prefix_static_size.x86-64.mca new file mode 100644 index 000000000000..5d17200abd2d --- /dev/null +++ b/rust/zerocopy/benches/write_to_prefix_static_size.x86-64.mca @@ -0,0 +1,57 @@ +Iterations: 100 +Instructions: 900 +Total Cycles: 233 +Total uOps: 900 + +Dispatch Width: 4 +uOps Per Cycle: 3.86 +IPC: 3.86 +Block RThroughput: 2.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 cmp rdx, 6 + 1 1 1.00 jb .LBB5_2 + 1 5 0.50 * movzx eax, word ptr [rdi + 4] + 1 1 1.00 * mov word ptr [rsi + 4], ax + 1 5 0.50 * mov eax, dword ptr [rdi] + 1 1 1.00 * mov dword ptr [rsi], eax + 1 1 0.33 cmp rdx, 6 + 1 1 0.50 setae al + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.50 1.49 2.00 2.01 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.25 0.74 - 0.01 - - cmp rdx, 6 + - - - - - 1.00 - - jb .LBB5_2 + - - - - - - 0.50 0.50 movzx eax, word ptr [rdi + 4] + - - - - 1.00 - 0.48 0.52 mov word ptr [rsi + 4], ax + - - - - - - 0.52 0.48 mov eax, dword ptr [rdi] + - - - - 1.00 - 0.50 0.50 mov dword ptr [rsi], eax + - - 0.25 0.75 - - - - cmp rdx, 6 + - - 1.00 - - - - - setae al + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_static_size.rs b/rust/zerocopy/benches/write_to_static_size.rs new file mode 100644 index 000000000000..3bb9435c5ade --- /dev/null +++ b/rust/zerocopy/benches/write_to_static_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_write_to_static_size(source: &format::CocoPacket, destination: &mut [u8]) -> Option<()> { + source.write_to(destination).ok() +} diff --git a/rust/zerocopy/benches/write_to_static_size.x86-64 b/rust/zerocopy/benches/write_to_static_size.x86-64 new file mode 100644 index 000000000000..d6413e0fd614 --- /dev/null +++ b/rust/zerocopy/benches/write_to_static_size.x86-64 @@ -0,0 +1,11 @@ +bench_write_to_static_size: + cmp rdx, 6 + jne .LBB5_2 + movzx eax, word ptr [rdi + 4] + mov word ptr [rsi + 4], ax + mov eax, dword ptr [rdi] + mov dword ptr [rsi], eax +.LBB5_2: + cmp rdx, 6 + sete al + ret diff --git a/rust/zerocopy/benches/write_to_static_size.x86-64.mca b/rust/zerocopy/benches/write_to_static_size.x86-64.mca new file mode 100644 index 000000000000..cc5bb1d26fc0 --- /dev/null +++ b/rust/zerocopy/benches/write_to_static_size.x86-64.mca @@ -0,0 +1,57 @@ +Iterations: 100 +Instructions: 900 +Total Cycles: 233 +Total uOps: 900 + +Dispatch Width: 4 +uOps Per Cycle: 3.86 +IPC: 3.86 +Block RThroughput: 2.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 cmp rdx, 6 + 1 1 1.00 jne .LBB5_2 + 1 5 0.50 * movzx eax, word ptr [rdi + 4] + 1 1 1.00 * mov word ptr [rsi + 4], ax + 1 5 0.50 * mov eax, dword ptr [rdi] + 1 1 1.00 * mov dword ptr [rsi], eax + 1 1 0.33 cmp rdx, 6 + 1 1 0.50 sete al + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.50 1.49 2.00 2.01 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.25 0.74 - 0.01 - - cmp rdx, 6 + - - - - - 1.00 - - jne .LBB5_2 + - - - - - - 0.50 0.50 movzx eax, word ptr [rdi + 4] + - - - - 1.00 - 0.48 0.52 mov word ptr [rsi + 4], ax + - - - - - - 0.52 0.48 mov eax, dword ptr [rdi] + - - - - 1.00 - 0.50 0.50 mov dword ptr [rsi], eax + - - 0.25 0.75 - - - - cmp rdx, 6 + - - 1.00 - - - - - sete al + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_suffix_dynamic_size.rs b/rust/zerocopy/benches/write_to_suffix_dynamic_size.rs new file mode 100644 index 000000000000..9fa6b91cda41 --- /dev/null +++ b/rust/zerocopy/benches/write_to_suffix_dynamic_size.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_write_to_suffix_dynamic_size( + source: &format::CocoPacket, + destination: &mut [u8], +) -> Option<()> { + source.write_to_suffix(destination).ok() +} diff --git a/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64 b/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64 new file mode 100644 index 000000000000..75f349562db6 --- /dev/null +++ b/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64 @@ -0,0 +1,22 @@ +bench_write_to_suffix_dynamic_size: + push r14 + push rbx + push rax + mov rbx, rcx + lea r14, [2*rsi + 5] + and r14, -2 + sub rcx, r14 + jb .LBB5_2 + mov rax, rdi + add rdx, rcx + mov rdi, rdx + mov rsi, rax + mov rdx, r14 + call qword ptr [rip + memcpy@GOTPCREL] +.LBB5_2: + cmp rbx, r14 + setae al + add rsp, 8 + pop rbx + pop r14 + ret diff --git a/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..95cb9dfe2eb7 --- /dev/null +++ b/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64.mca @@ -0,0 +1,79 @@ +Iterations: 100 +Instructions: 2000 +Total Cycles: 2890 +Total uOps: 2600 + +Dispatch Width: 4 +uOps Per Cycle: 0.90 +IPC: 0.69 +Block RThroughput: 6.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push r14 + 2 5 1.00 * push rbx + 2 5 1.00 * push rax + 1 1 0.33 mov rbx, rcx + 1 1 0.50 lea r14, [2*rsi + 5] + 1 1 0.33 and r14, -2 + 1 1 0.33 sub rcx, r14 + 1 1 1.00 jb .LBB5_2 + 1 1 0.33 mov rax, rdi + 1 1 0.33 add rdx, rcx + 1 1 0.33 mov rdi, rdx + 1 1 0.33 mov rsi, rax + 1 1 0.33 mov rdx, r14 + 4 7 1.00 * call qword ptr [rip + memcpy@GOTPCREL] + 1 1 0.33 cmp rbx, r14 + 1 1 0.50 setae al + 1 1 0.33 add rsp, 8 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r14 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.98 4.98 4.00 5.04 4.00 3.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - - 1.00 push r14 + - - - - 1.00 - 1.00 - push rbx + - - - - 1.00 - - 1.00 push rax + - - 0.94 0.05 - 0.01 - - mov rbx, rcx + - - 0.06 0.94 - - - - lea r14, [2*rsi + 5] + - - 0.93 0.02 - 0.05 - - and r14, -2 + - - 0.05 0.94 - 0.01 - - sub rcx, r14 + - - - - - 1.00 - - jb .LBB5_2 + - - 0.02 0.04 - 0.94 - - mov rax, rdi + - - 0.03 0.97 - - - - add rdx, rcx + - - 0.95 0.05 - - - - mov rdi, rdx + - - 0.94 0.03 - 0.03 - - mov rsi, rax + - - 0.01 0.03 - 0.96 - - mov rdx, r14 + - - - - 1.00 1.00 2.00 - call qword ptr [rip + memcpy@GOTPCREL] + - - 0.05 0.94 - 0.01 - - cmp rbx, r14 + - - 0.97 - - 0.03 - - setae al + - - 0.03 0.97 - - - - add rsp, 8 + - - - - - - - 1.00 pop rbx + - - - - - - 1.00 - pop r14 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_suffix_static_size.rs b/rust/zerocopy/benches/write_to_suffix_static_size.rs new file mode 100644 index 000000000000..1c95aba4b16c --- /dev/null +++ b/rust/zerocopy/benches/write_to_suffix_static_size.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_write_to_suffix_static_size( + source: &format::CocoPacket, + destination: &mut [u8], +) -> Option<()> { + source.write_to_suffix(destination).ok() +} diff --git a/rust/zerocopy/benches/write_to_suffix_static_size.x86-64 b/rust/zerocopy/benches/write_to_suffix_static_size.x86-64 new file mode 100644 index 000000000000..934aa370d4d6 --- /dev/null +++ b/rust/zerocopy/benches/write_to_suffix_static_size.x86-64 @@ -0,0 +1,11 @@ +bench_write_to_suffix_static_size: + cmp rdx, 6 + jb .LBB5_2 + movzx eax, word ptr [rdi + 4] + mov word ptr [rsi + rdx - 2], ax + mov eax, dword ptr [rdi] + mov dword ptr [rsi + rdx - 6], eax +.LBB5_2: + cmp rdx, 6 + setae al + ret diff --git a/rust/zerocopy/benches/write_to_suffix_static_size.x86-64.mca b/rust/zerocopy/benches/write_to_suffix_static_size.x86-64.mca new file mode 100644 index 000000000000..6b18e4a44585 --- /dev/null +++ b/rust/zerocopy/benches/write_to_suffix_static_size.x86-64.mca @@ -0,0 +1,57 @@ +Iterations: 100 +Instructions: 900 +Total Cycles: 233 +Total uOps: 900 + +Dispatch Width: 4 +uOps Per Cycle: 3.86 +IPC: 3.86 +Block RThroughput: 2.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 cmp rdx, 6 + 1 1 1.00 jb .LBB5_2 + 1 5 0.50 * movzx eax, word ptr [rdi + 4] + 1 1 1.00 * mov word ptr [rsi + rdx - 2], ax + 1 5 0.50 * mov eax, dword ptr [rdi] + 1 1 1.00 * mov dword ptr [rsi + rdx - 6], eax + 1 1 0.33 cmp rdx, 6 + 1 1 0.50 setae al + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.50 1.49 2.00 2.01 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.25 0.74 - 0.01 - - cmp rdx, 6 + - - - - - 1.00 - - jb .LBB5_2 + - - - - - - 0.50 0.50 movzx eax, word ptr [rdi + 4] + - - - - 1.00 - 0.48 0.52 mov word ptr [rsi + rdx - 2], ax + - - - - - - 0.52 0.48 mov eax, dword ptr [rdi] + - - - - 1.00 - 0.50 0.50 mov dword ptr [rsi + rdx - 6], eax + - - 0.25 0.75 - - - - cmp rdx, 6 + - - 1.00 - - - - - setae al + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/zero_dynamic_padding.rs b/rust/zerocopy/benches/zero_dynamic_padding.rs new file mode 100644 index 000000000000..8eda0953d1e6 --- /dev/null +++ b/rust/zerocopy/benches/zero_dynamic_padding.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_zero_dynamic_padding(source: &mut format::LocoPacket) { + source.zero() +} diff --git a/rust/zerocopy/benches/zero_dynamic_padding.x86-64 b/rust/zerocopy/benches/zero_dynamic_padding.x86-64 new file mode 100644 index 000000000000..7dccf1745f81 --- /dev/null +++ b/rust/zerocopy/benches/zero_dynamic_padding.x86-64 @@ -0,0 +1,7 @@ +bench_zero_dynamic_padding: + lea rax, [rsi + 2*rsi] + movabs rdx, 9223372036854775804 + and rdx, rax + add rdx, 12 + xor esi, esi + jmp qword ptr [rip + memset@GOTPCREL] diff --git a/rust/zerocopy/benches/zero_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/zero_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..098fc107875f --- /dev/null +++ b/rust/zerocopy/benches/zero_dynamic_padding.x86-64.mca @@ -0,0 +1,51 @@ +Iterations: 100 +Instructions: 600 +Total Cycles: 209 +Total uOps: 700 + +Dispatch Width: 4 +uOps Per Cycle: 3.35 +IPC: 2.87 +Block RThroughput: 1.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea rax, [rsi + 2*rsi] + 1 1 0.33 movabs rdx, 9223372036854775804 + 1 1 0.33 and rdx, rax + 1 1 0.33 add rdx, 12 + 1 0 0.25 xor esi, esi + 2 6 1.00 * jmp qword ptr [rip + memset@GOTPCREL] + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.66 1.66 - 1.68 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.33 0.67 - - - - lea rax, [rsi + 2*rsi] + - - 0.98 - - 0.02 - - movabs rdx, 9223372036854775804 + - - 0.01 0.66 - 0.33 - - and rdx, rax + - - 0.34 0.33 - 0.33 - - add rdx, 12 + - - - - - - - - xor esi, esi + - - - - - 1.00 0.50 0.50 jmp qword ptr [rip + memset@GOTPCREL] diff --git a/rust/zerocopy/benches/zero_dynamic_size.rs b/rust/zerocopy/benches/zero_dynamic_size.rs new file mode 100644 index 000000000000..536d800ebc76 --- /dev/null +++ b/rust/zerocopy/benches/zero_dynamic_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_zero_dynamic_size(source: &mut format::LocoPacket) { + source.zero() +} diff --git a/rust/zerocopy/benches/zero_dynamic_size.x86-64 b/rust/zerocopy/benches/zero_dynamic_size.x86-64 new file mode 100644 index 000000000000..2b31ed644eaf --- /dev/null +++ b/rust/zerocopy/benches/zero_dynamic_size.x86-64 @@ -0,0 +1,5 @@ +bench_zero_dynamic_size: + lea rdx, [2*rsi + 5] + and rdx, -2 + xor esi, esi + jmp qword ptr [rip + memset@GOTPCREL] diff --git a/rust/zerocopy/benches/zero_dynamic_size.x86-64.mca b/rust/zerocopy/benches/zero_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..0b086a29b0ed --- /dev/null +++ b/rust/zerocopy/benches/zero_dynamic_size.x86-64.mca @@ -0,0 +1,47 @@ +Iterations: 100 +Instructions: 400 +Total Cycles: 142 +Total uOps: 500 + +Dispatch Width: 4 +uOps Per Cycle: 3.52 +IPC: 2.82 +Block RThroughput: 1.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea rdx, [2*rsi + 5] + 1 1 0.33 and rdx, -2 + 1 0 0.25 xor esi, esi + 2 6 1.00 * jmp qword ptr [rip + memset@GOTPCREL] + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.99 1.00 - 1.01 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.99 0.01 - - - - lea rdx, [2*rsi + 5] + - - - 0.99 - 0.01 - - and rdx, -2 + - - - - - - - - xor esi, esi + - - - - - 1.00 0.50 0.50 jmp qword ptr [rip + memset@GOTPCREL] diff --git a/rust/zerocopy/benches/zero_static_size.rs b/rust/zerocopy/benches/zero_static_size.rs new file mode 100644 index 000000000000..fa7fa0839c15 --- /dev/null +++ b/rust/zerocopy/benches/zero_static_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_zero_static_size(source: &mut format::LocoPacket) { + source.zero() +} diff --git a/rust/zerocopy/benches/zero_static_size.x86-64 b/rust/zerocopy/benches/zero_static_size.x86-64 new file mode 100644 index 000000000000..ced8e18949b2 --- /dev/null +++ b/rust/zerocopy/benches/zero_static_size.x86-64 @@ -0,0 +1,4 @@ +bench_zero_static_size: + mov word ptr [rdi + 4], 0 + mov dword ptr [rdi], 0 + ret diff --git a/rust/zerocopy/benches/zero_static_size.x86-64.mca b/rust/zerocopy/benches/zero_static_size.x86-64.mca new file mode 100644 index 000000000000..042897ef2dfe --- /dev/null +++ b/rust/zerocopy/benches/zero_static_size.x86-64.mca @@ -0,0 +1,45 @@ +Iterations: 100 +Instructions: 300 +Total Cycles: 203 +Total uOps: 300 + +Dispatch Width: 4 +uOps Per Cycle: 1.48 +IPC: 1.48 +Block RThroughput: 2.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 1.00 * mov word ptr [rdi + 4], 0 + 1 1 1.00 * mov dword ptr [rdi], 0 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - - - 2.00 1.00 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - - 1.00 mov word ptr [rdi + 4], 0 + - - - - 1.00 - 1.00 - mov dword ptr [rdi], 0 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/rustdoc/style.css b/rust/zerocopy/rustdoc/style.css new file mode 100644 index 000000000000..414348964af2 --- /dev/null +++ b/rust/zerocopy/rustdoc/style.css @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT */ + +/* +Copyright 2026 The Fuchsia Authors + +Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +<LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +This file may not be copied, modified, or distributed except according to +those terms. +*/ + +.codegen-tabs { + display: grid; + grid-template-columns: repeat(var(--arity), minmax(200px, 1fr)); + grid-template-rows: auto 1fr; + column-gap: 1rem; +} + +.codegen-tabs:not(:has(> details[open]))::after { + grid-column: 1/-1; + content: 'Click one of the above headers to expand its contents.'; + font-style: italic; + font-size: small; + text-align: center; +} + +.codegen-tabs details { + display: grid; + grid-column: 1 / -1; + grid-row: 1 / span 2; + grid-template-columns: subgrid; + grid-template-rows: subgrid; +} + +.codegen-tabs summary { + display: grid; + grid-column: var(--n) / span 1; + grid-row: 1; + z-index: 1; + border-bottom: 2px solid var(--headings-border-bottom-color); + cursor: pointer; +} + +.codegen-tabs details[open] > summary { + background-color: var(--code-block-background-color); + border-bottom-color: var(--target-border-color); +} + +.codegen-tabs details::details-content { + grid-column: 1 / -1; + grid-row: 2; +} + +.codegen-tabs details:not([open])::details-content { + display: none; +} diff --git a/rust/zerocopy/src/byte_slice.rs b/rust/zerocopy/src/byte_slice.rs new file mode 100644 index 000000000000..a5ded4a18b39 --- /dev/null +++ b/rust/zerocopy/src/byte_slice.rs @@ -0,0 +1,434 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2024 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Traits for types that encapsulate a `[u8]`. +//! +//! These traits are used to bound the `B` parameter of [`Ref`]. + +use core::{ + cell, + ops::{Deref, DerefMut}, +}; + +// For each trait polyfill, as soon as the corresponding feature is stable, the +// polyfill import will be unused because method/function resolution will prefer +// the inherent method/function over a trait method/function. Thus, we suppress +// the `unused_imports` warning. +// +// See the documentation on `util::polyfills` for more information. +#[allow(unused_imports)] +use crate::util::polyfills::{self, NonNullExt as _, NumExt as _}; +#[cfg(doc)] +use crate::Ref; + +/// A mutable or immutable reference to a byte slice. +/// +/// `ByteSlice` abstracts over the mutability of a byte slice reference, and is +/// implemented for various special reference types such as +/// [`Ref<[u8]>`](core::cell::Ref) and [`RefMut<[u8]>`](core::cell::RefMut). +/// +/// # Safety +/// +/// Implementations of `ByteSlice` must promise that their implementations of +/// [`Deref`] and [`DerefMut`] are "stable". In particular, given `B: ByteSlice` +/// and `b: B`, two calls, each to either `b.deref()` or `b.deref_mut()`, must +/// return a byte slice with the same address and length. This must hold even if +/// the two calls are separated by an arbitrary sequence of calls to methods on +/// `ByteSlice`, [`ByteSliceMut`], [`IntoByteSlice`], or [`IntoByteSliceMut`], +/// or on their super-traits. This does *not* need to hold if the two calls are +/// separated by any method calls, field accesses, or field modifications *other +/// than* those from these traits. +/// +/// Note that this also implies that, given `b: B`, the address and length +/// cannot be modified via objects other than `b`, either on the same thread or +/// on another thread. +pub unsafe trait ByteSlice: Deref<Target = [u8]> + Sized {} + +/// A mutable reference to a byte slice. +/// +/// `ByteSliceMut` abstracts over various ways of storing a mutable reference to +/// a byte slice, and is implemented for various special reference types such as +/// `RefMut<[u8]>`. +/// +/// `ByteSliceMut` is a shorthand for [`ByteSlice`] and [`DerefMut`]. +pub trait ByteSliceMut: ByteSlice + DerefMut {} +impl<B: ByteSlice + DerefMut> ByteSliceMut for B {} + +/// A [`ByteSlice`] which can be copied without violating dereference stability. +/// +/// # Safety +/// +/// If `B: CopyableByteSlice`, then the dereference stability properties +/// required by [`ByteSlice`] (see that trait's safety documentation) do not +/// only hold regarding two calls to `b.deref()` or `b.deref_mut()`, but also +/// hold regarding `c.deref()` or `c.deref_mut()`, where `c` is produced by +/// copying `b`. +pub unsafe trait CopyableByteSlice: ByteSlice + Copy + CloneableByteSlice {} + +/// A [`ByteSlice`] which can be cloned without violating dereference stability. +/// +/// # Safety +/// +/// If `B: CloneableByteSlice`, then the dereference stability properties +/// required by [`ByteSlice`] (see that trait's safety documentation) do not +/// only hold regarding two calls to `b.deref()` or `b.deref_mut()`, but also +/// hold regarding `c.deref()` or `c.deref_mut()`, where `c` is produced by +/// `b.clone()`, `b.clone().clone()`, etc. +pub unsafe trait CloneableByteSlice: ByteSlice + Clone {} + +/// A [`ByteSlice`] that can be split in two. +/// +/// # Safety +/// +/// Unsafe code may depend for its soundness on the assumption that `split_at` +/// and `split_at_unchecked` are implemented correctly. In particular, given `B: +/// SplitByteSlice` and `b: B`, if `b.deref()` returns a byte slice with address +/// `addr` and length `len`, then if `split <= len`, both of these +/// invocations: +/// - `b.split_at(split)` +/// - `b.split_at_unchecked(split)` +/// +/// ...will return `(first, second)` such that: +/// - `first`'s address is `addr` and its length is `split` +/// - `second`'s address is `addr + split` and its length is `len - split` +pub unsafe trait SplitByteSlice: ByteSlice { + /// Attempts to split `self` at the midpoint. + /// + /// `s.split_at(mid)` returns `Ok((s[..mid], s[mid..]))` if `mid <= + /// s.deref().len()` and otherwise returns `Err(s)`. + /// + /// # Safety + /// + /// Unsafe code may rely on this function correctly implementing the above + /// functionality. + #[inline] + fn split_at(self, mid: usize) -> Result<(Self, Self), Self> { + if mid <= self.deref().len() { + // SAFETY: Above, we ensure that `mid <= self.deref().len()`. By + // invariant on `ByteSlice`, a supertrait of `SplitByteSlice`, + // `.deref()` is guaranteed to be "stable"; i.e., it will always + // dereference to a byte slice of the same address and length. Thus, + // we can be sure that the above precondition remains satisfied + // through the call to `split_at_unchecked`. + unsafe { Ok(self.split_at_unchecked(mid)) } + } else { + Err(self) + } + } + + /// Splits the slice at the midpoint, possibly omitting bounds checks. + /// + /// `s.split_at_unchecked(mid)` returns `s[..mid]` and `s[mid..]`. + /// + /// # Safety + /// + /// `mid` must not be greater than `self.deref().len()`. + /// + /// # Panics + /// + /// Implementations of this method may choose to perform a bounds check and + /// panic if `mid > self.deref().len()`. They may also panic for any other + /// reason. Since it is optional, callers must not rely on this behavior for + /// soundness. + #[must_use] + unsafe fn split_at_unchecked(self, mid: usize) -> (Self, Self); +} + +/// A shorthand for [`SplitByteSlice`] and [`ByteSliceMut`]. +pub trait SplitByteSliceMut: SplitByteSlice + ByteSliceMut {} +impl<B: SplitByteSlice + ByteSliceMut> SplitByteSliceMut for B {} + +#[allow(clippy::missing_safety_doc)] // There's a `Safety` section on `into_byte_slice`. +/// A [`ByteSlice`] that conveys no ownership, and so can be converted into a +/// byte slice. +/// +/// Some `ByteSlice` types (notably, the standard library's [`Ref`] type) convey +/// ownership, and so they cannot soundly be moved by-value into a byte slice +/// type (`&[u8]`). Some methods in this crate's API (such as [`Ref::into_ref`]) +/// are only compatible with `ByteSlice` types without these ownership +/// semantics. +/// +/// [`Ref`]: core::cell::Ref +pub unsafe trait IntoByteSlice<'a>: ByteSlice { + /// Coverts `self` into a `&[u8]`. + /// + /// # Safety + /// + /// The returned reference has the same address and length as `self.deref()` + /// and `self.deref_mut()`. + /// + /// Note that, combined with the safety invariant on [`ByteSlice`], this + /// safety invariant implies that the returned reference is "stable" in the + /// sense described in the `ByteSlice` docs. + fn into_byte_slice(self) -> &'a [u8]; +} + +#[allow(clippy::missing_safety_doc)] // There's a `Safety` section on `into_byte_slice_mut`. +/// A [`ByteSliceMut`] that conveys no ownership, and so can be converted into a +/// mutable byte slice. +/// +/// Some `ByteSliceMut` types (notably, the standard library's [`RefMut`] type) +/// convey ownership, and so they cannot soundly be moved by-value into a byte +/// slice type (`&mut [u8]`). Some methods in this crate's API (such as +/// [`Ref::into_mut`]) are only compatible with `ByteSliceMut` types without +/// these ownership semantics. +/// +/// [`RefMut`]: core::cell::RefMut +pub unsafe trait IntoByteSliceMut<'a>: IntoByteSlice<'a> + ByteSliceMut { + /// Coverts `self` into a `&mut [u8]`. + /// + /// # Safety + /// + /// The returned reference has the same address and length as `self.deref()` + /// and `self.deref_mut()`. + /// + /// Note that, combined with the safety invariant on [`ByteSlice`], this + /// safety invariant implies that the returned reference is "stable" in the + /// sense described in the `ByteSlice` docs. + fn into_byte_slice_mut(self) -> &'a mut [u8]; +} + +// FIXME(#429): Add a "SAFETY" comment and remove this `allow`. +#[allow(clippy::undocumented_unsafe_blocks)] +unsafe impl ByteSlice for &[u8] {} + +// FIXME(#429): Add a "SAFETY" comment and remove this `allow`. +#[allow(clippy::undocumented_unsafe_blocks)] +unsafe impl CopyableByteSlice for &[u8] {} + +// FIXME(#429): Add a "SAFETY" comment and remove this `allow`. +#[allow(clippy::undocumented_unsafe_blocks)] +unsafe impl CloneableByteSlice for &[u8] {} + +// SAFETY: This delegates to `polyfills:split_at_unchecked`, which is documented +// to correctly split `self` into two slices at the given `mid` point. +unsafe impl SplitByteSlice for &[u8] { + #[inline] + unsafe fn split_at_unchecked(self, mid: usize) -> (Self, Self) { + // SAFETY: By contract on caller, `mid` is not greater than + // `self.len()`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + unsafe { + (<[u8]>::get_unchecked(self, ..mid), <[u8]>::get_unchecked(self, mid..)) + } + } +} + +// SAFETY: See inline. +unsafe impl<'a> IntoByteSlice<'a> for &'a [u8] { + #[inline(always)] + fn into_byte_slice(self) -> &'a [u8] { + // SAFETY: It would be patently insane to implement `<Deref for + // &[u8]>::deref` as anything other than `fn deref(&self) -> &[u8] { + // *self }`. Assuming this holds, then `self` is stable as required by + // `into_byte_slice`. + self + } +} + +// FIXME(#429): Add a "SAFETY" comment and remove this `allow`. +#[allow(clippy::undocumented_unsafe_blocks)] +unsafe impl ByteSlice for &mut [u8] {} + +// SAFETY: This delegates to `polyfills:split_at_mut_unchecked`, which is +// documented to correctly split `self` into two slices at the given `mid` +// point. +unsafe impl SplitByteSlice for &mut [u8] { + #[inline] + unsafe fn split_at_unchecked(self, mid: usize) -> (Self, Self) { + use core::slice::from_raw_parts_mut; + + // `l_ptr` is non-null, because `self` is non-null, by invariant on + // `&mut [u8]`. + let l_ptr = self.as_mut_ptr(); + + // SAFETY: By contract on caller, `mid` is not greater than + // `self.len()`. + let r_ptr = unsafe { l_ptr.add(mid) }; + + let l_len = mid; + + // SAFETY: By contract on caller, `mid` is not greater than + // `self.len()`. + // + // FIXME(#67): Remove this allow. See NumExt for more details. + #[allow(unstable_name_collisions)] + let r_len = unsafe { self.len().unchecked_sub(mid) }; + + // SAFETY: These invocations of `from_raw_parts_mut` satisfy its + // documented safety preconditions [1]: + // - The data `l_ptr` and `r_ptr` are valid for both reads and writes of + // `l_len` and `r_len` bytes, respectively, and they are trivially + // aligned. In particular: + // - The entire memory range of each slice is contained within a + // single allocated object, since `l_ptr` and `r_ptr` are both + // derived from within the address range of `self`. + // - Both `l_ptr` and `r_ptr` are non-null and trivially aligned. + // `self` is non-null by invariant on `&mut [u8]`, and the + // operations that derive `l_ptr` and `r_ptr` from `self` do not + // nullify either pointer. + // - The data `l_ptr` and `r_ptr` point to `l_len` and `r_len`, + // respectively, consecutive properly initialized values of type `u8`. + // This is true for `self` by invariant on `&mut [u8]`, and remains + // true for these two sub-slices of `self`. + // - The memory referenced by the returned slice cannot be accessed + // through any other pointer (not derived from the return value) for + // the duration of lifetime `'a``, because: + // - `split_at_unchecked` consumes `self` (which is not `Copy`), + // - `split_at_unchecked` does not exfiltrate any references to this + // memory, besides those references returned below, + // - the returned slices are non-overlapping. + // - The individual sizes of the sub-slices of `self` are no larger than + // `isize::MAX`, because their combined sizes are no larger than + // `isize::MAX`, by invariant on `self`. + // + // [1] https://doc.rust-lang.org/std/slice/fn.from_raw_parts_mut.html#safety + #[allow(clippy::multiple_unsafe_ops_per_block)] + unsafe { + (from_raw_parts_mut(l_ptr, l_len), from_raw_parts_mut(r_ptr, r_len)) + } + } +} + +// SAFETY: See inline. +unsafe impl<'a> IntoByteSlice<'a> for &'a mut [u8] { + #[inline(always)] + fn into_byte_slice(self) -> &'a [u8] { + // SAFETY: It would be patently insane to implement `<Deref for &mut + // [u8]>::deref` as anything other than `fn deref(&self) -> &[u8] { + // *self }`. Assuming this holds, then `self` is stable as required by + // `into_byte_slice`. + self + } +} + +// SAFETY: See inline. +unsafe impl<'a> IntoByteSliceMut<'a> for &'a mut [u8] { + #[inline(always)] + fn into_byte_slice_mut(self) -> &'a mut [u8] { + // SAFETY: It would be patently insane to implement `<DerefMut for &mut + // [u8]>::deref` as anything other than `fn deref_mut(&mut self) -> &mut + // [u8] { *self }`. Assuming this holds, then `self` is stable as + // required by `into_byte_slice_mut`. + self + } +} + +// FIXME(#429): Add a "SAFETY" comment and remove this `allow`. +#[allow(clippy::undocumented_unsafe_blocks)] +unsafe impl ByteSlice for cell::Ref<'_, [u8]> {} + +// SAFETY: This delegates to stdlib implementation of `Ref::map_split`, which is +// assumed to be correct, and `SplitByteSlice::split_at_unchecked`, which is +// documented to correctly split `self` into two slices at the given `mid` +// point. +unsafe impl SplitByteSlice for cell::Ref<'_, [u8]> { + #[inline] + unsafe fn split_at_unchecked(self, mid: usize) -> (Self, Self) { + cell::Ref::map_split(self, |slice| + // SAFETY: By precondition on caller, `mid` is not greater than + // `slice.len()`. + unsafe { + SplitByteSlice::split_at_unchecked(slice, mid) + }) + } +} + +// FIXME(#429): Add a "SAFETY" comment and remove this `allow`. +#[allow(clippy::undocumented_unsafe_blocks)] +unsafe impl ByteSlice for cell::RefMut<'_, [u8]> {} + +// SAFETY: This delegates to stdlib implementation of `RefMut::map_split`, which +// is assumed to be correct, and `SplitByteSlice::split_at_unchecked`, which is +// documented to correctly split `self` into two slices at the given `mid` +// point. +unsafe impl SplitByteSlice for cell::RefMut<'_, [u8]> { + #[inline] + unsafe fn split_at_unchecked(self, mid: usize) -> (Self, Self) { + cell::RefMut::map_split(self, |slice| + // SAFETY: By precondition on caller, `mid` is not greater than + // `slice.len()` + unsafe { + SplitByteSlice::split_at_unchecked(slice, mid) + }) + } +} + +#[cfg(kani)] +mod proofs { + use super::*; + + fn any_vec() -> Vec<u8> { + let len = kani::any(); + kani::assume(len <= crate::DstLayout::MAX_SIZE); + vec![0u8; len] + } + + #[kani::proof] + fn prove_split_at_unchecked() { + let v = any_vec(); + let slc = v.as_slice(); + let mid = kani::any(); + kani::assume(mid <= slc.len()); + let (l, r) = unsafe { slc.split_at_unchecked(mid) }; + assert_eq!(l.len() + r.len(), slc.len()); + + let slc: *const _ = slc; + let l: *const _ = l; + let r: *const _ = r; + + assert_eq!(slc.cast::<u8>(), l.cast::<u8>()); + assert_eq!(unsafe { slc.cast::<u8>().add(mid) }, r.cast::<u8>()); + + let mut v = any_vec(); + let slc = v.as_mut_slice(); + let len = slc.len(); + let mid = kani::any(); + kani::assume(mid <= slc.len()); + let (l, r) = unsafe { slc.split_at_unchecked(mid) }; + assert_eq!(l.len() + r.len(), len); + + let l: *mut _ = l; + let r: *mut _ = r; + let slc: *mut _ = slc; + + assert_eq!(slc.cast::<u8>(), l.cast::<u8>()); + assert_eq!(unsafe { slc.cast::<u8>().add(mid) }, r.cast::<u8>()); + } +} + +#[cfg(test)] +mod tests { + use core::cell::RefCell; + + use super::*; + + #[test] + fn test_ref_split_at_unchecked() { + let cell = RefCell::new([1, 2, 3, 4]); + let borrow = cell.borrow(); + let slice_ref: cell::Ref<'_, [u8]> = cell::Ref::map(borrow, |a| &a[..]); + // SAFETY: 2 is within bounds of [1, 2, 3, 4] + let (l, r) = unsafe { slice_ref.split_at_unchecked(2) }; + assert_eq!(*l, [1, 2]); + assert_eq!(*r, [3, 4]); + } + + #[test] + fn test_ref_mut_split_at_unchecked() { + let cell = RefCell::new([1, 2, 3, 4]); + let borrow_mut = cell.borrow_mut(); + let slice_ref_mut: cell::RefMut<'_, [u8]> = cell::RefMut::map(borrow_mut, |a| &mut a[..]); + // SAFETY: 2 is within bounds of [1, 2, 3, 4] + let (l, r) = unsafe { slice_ref_mut.split_at_unchecked(2) }; + assert_eq!(*l, [1, 2]); + assert_eq!(*r, [3, 4]); + } +} diff --git a/rust/zerocopy/src/byteorder.rs b/rust/zerocopy/src/byteorder.rs new file mode 100644 index 000000000000..8f70048f1eb0 --- /dev/null +++ b/rust/zerocopy/src/byteorder.rs @@ -0,0 +1,1564 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2019 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Byte order-aware numeric primitives. +//! +//! This module contains equivalents of the native multi-byte integer types with +//! no alignment requirement and supporting byte order conversions. +//! +//! For each native multi-byte integer type - `u16`, `i16`, `u32`, etc - and +//! floating point type - `f32` and `f64` - an equivalent type is defined by +//! this module - [`U16`], [`I16`], [`U32`], [`F32`], [`F64`], etc. Unlike their +//! native counterparts, these types have alignment 1, and take a type parameter +//! specifying the byte order in which the bytes are stored in memory. Each type +//! implements this crate's relevant conversion and marker traits. +//! +//! These two properties, taken together, make these types useful for defining +//! data structures whose memory layout matches a wire format such as that of a +//! network protocol or a file format. Such formats often have multi-byte values +//! at offsets that do not respect the alignment requirements of the equivalent +//! native types, and stored in a byte order not necessarily the same as that of +//! the target platform. +//! +//! Type aliases are provided for common byte orders in the [`big_endian`], +//! [`little_endian`], [`network_endian`], and [`native_endian`] submodules. +//! Note that network-endian is a synonym for big-endian. +//! +//! # Example +//! +//! One use of these types is for representing network packet formats, such as +//! UDP: +//! +//! ```rust +//! use zerocopy::{*, byteorder::network_endian::U16}; +//! # use zerocopy_derive::*; +//! +//! #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] +//! #[repr(C)] +//! struct UdpHeader { +//! src_port: U16, +//! dst_port: U16, +//! length: U16, +//! checksum: U16, +//! } +//! +//! #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] +//! #[repr(C, packed)] +//! struct UdpPacket { +//! header: UdpHeader, +//! body: [u8], +//! } +//! +//! impl UdpPacket { +//! fn parse(bytes: &[u8]) -> Option<&UdpPacket> { +//! UdpPacket::ref_from_bytes(bytes).ok() +//! } +//! } +//! ``` + +use core::{ + convert::{TryFrom, TryInto}, + fmt::{Binary, Debug, LowerHex, Octal, UpperHex}, + hash::Hash, + num::TryFromIntError, +}; + +use super::*; + +/// A type-level representation of byte order. +/// +/// This type is implemented by [`BigEndian`] and [`LittleEndian`], which +/// represent big-endian and little-endian byte order respectively. This module +/// also provides a number of useful aliases for those types: [`NativeEndian`], +/// [`NetworkEndian`], [`BE`], and [`LE`]. +/// +/// `ByteOrder` types can be used to specify the byte order of the types in this +/// module - for example, [`U32<BigEndian>`] is a 32-bit integer stored in +/// big-endian byte order. +/// +/// [`U32<BigEndian>`]: U32 +pub trait ByteOrder: + Copy + Clone + Debug + Display + Eq + PartialEq + Ord + PartialOrd + Hash + private::Sealed +{ + #[doc(hidden)] + const ORDER: Order; +} + +mod private { + pub trait Sealed {} + + impl Sealed for super::BigEndian {} + impl Sealed for super::LittleEndian {} +} + +#[allow(missing_copy_implementations, missing_debug_implementations)] +#[doc(hidden)] +pub enum Order { + BigEndian, + LittleEndian, +} + +/// Big-endian byte order. +/// +/// See [`ByteOrder`] for more details. +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum BigEndian {} + +impl ByteOrder for BigEndian { + const ORDER: Order = Order::BigEndian; +} + +impl Display for BigEndian { + #[inline] + fn fmt(&self, _: &mut Formatter<'_>) -> fmt::Result { + match *self {} + } +} + +/// Little-endian byte order. +/// +/// See [`ByteOrder`] for more details. +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum LittleEndian {} + +impl ByteOrder for LittleEndian { + const ORDER: Order = Order::LittleEndian; +} + +impl Display for LittleEndian { + #[inline] + fn fmt(&self, _: &mut Formatter<'_>) -> fmt::Result { + match *self {} + } +} + +/// The endianness used by this platform. +/// +/// This is a type alias for [`BigEndian`] or [`LittleEndian`] depending on the +/// endianness of the target platform. +#[cfg(target_endian = "big")] +pub type NativeEndian = BigEndian; + +/// The endianness used by this platform. +/// +/// This is a type alias for [`BigEndian`] or [`LittleEndian`] depending on the +/// endianness of the target platform. +#[cfg(target_endian = "little")] +pub type NativeEndian = LittleEndian; + +/// The endianness used in many network protocols. +/// +/// This is a type alias for [`BigEndian`]. +pub type NetworkEndian = BigEndian; + +/// A type alias for [`BigEndian`]. +pub type BE = BigEndian; + +/// A type alias for [`LittleEndian`]. +pub type LE = LittleEndian; + +macro_rules! impl_fmt_trait { + ($name:ident, $native:ident, $trait:ident) => { + impl<O: ByteOrder> $trait for $name<O> { + #[inline(always)] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + $trait::fmt(&self.get(), f) + } + } + }; +} + +macro_rules! impl_fmt_traits { + ($name:ident, $native:ident, "floating point number") => { + }; + ($name:ident, $native:ident, "unsigned integer") => { + impl_fmt_traits!($name, $native, @all_types); + }; + ($name:ident, $native:ident, "signed integer") => { + impl_fmt_traits!($name, $native, @all_types); + }; + ($name:ident, $native:ident, @all_types) => { + impl_fmt_trait!($name, $native, Display); + impl_fmt_trait!($name, $native, Octal); + impl_fmt_trait!($name, $native, LowerHex); + impl_fmt_trait!($name, $native, UpperHex); + impl_fmt_trait!($name, $native, Binary); + }; +} + +macro_rules! impl_ops_traits { + ($name:ident, $native:ident, "floating point number") => { + impl_ops_traits!($name, $native, @all_types); + impl_ops_traits!($name, $native, @signed_integer_floating_point); + + impl<O: ByteOrder> PartialOrd for $name<O> { + #[inline(always)] + fn partial_cmp(&self, other: &Self) -> Option<Ordering> { + self.get().partial_cmp(&other.get()) + } + } + }; + ($name:ident, $native:ident, "unsigned integer") => { + impl_ops_traits!($name, $native, @signed_unsigned_integer); + impl_ops_traits!($name, $native, @all_types); + }; + ($name:ident, $native:ident, "signed integer") => { + impl_ops_traits!($name, $native, @signed_unsigned_integer); + impl_ops_traits!($name, $native, @signed_integer_floating_point); + impl_ops_traits!($name, $native, @all_types); + }; + ($name:ident, $native:ident, @signed_unsigned_integer) => { + impl_ops_traits!(@without_byteorder_swap $name, $native, BitAnd, bitand, BitAndAssign, bitand_assign); + impl_ops_traits!(@without_byteorder_swap $name, $native, BitOr, bitor, BitOrAssign, bitor_assign); + impl_ops_traits!(@without_byteorder_swap $name, $native, BitXor, bitxor, BitXorAssign, bitxor_assign); + impl_ops_traits!(@with_byteorder_swap $name, $native, Shl, shl, ShlAssign, shl_assign); + impl_ops_traits!(@with_byteorder_swap $name, $native, Shr, shr, ShrAssign, shr_assign); + + impl<O> core::ops::Not for $name<O> { + type Output = $name<O>; + + #[inline(always)] + fn not(self) -> $name<O> { + let self_native = $native::from_ne_bytes(self.0); + $name((!self_native).to_ne_bytes(), PhantomData) + } + } + + impl<O: ByteOrder> PartialOrd for $name<O> { + #[inline(always)] + fn partial_cmp(&self, other: &Self) -> Option<Ordering> { + Some(self.cmp(other)) + } + } + + impl<O: ByteOrder> Ord for $name<O> { + #[inline(always)] + fn cmp(&self, other: &Self) -> Ordering { + self.get().cmp(&other.get()) + } + } + + impl<O: ByteOrder> PartialOrd<$native> for $name<O> { + #[inline(always)] + fn partial_cmp(&self, other: &$native) -> Option<Ordering> { + self.get().partial_cmp(other) + } + } + }; + ($name:ident, $native:ident, @signed_integer_floating_point) => { + impl<O: ByteOrder> core::ops::Neg for $name<O> { + type Output = $name<O>; + + #[inline(always)] + fn neg(self) -> $name<O> { + let self_native: $native = self.get(); + #[allow(clippy::arithmetic_side_effects)] + $name::<O>::new(-self_native) + } + } + }; + ($name:ident, $native:ident, @all_types) => { + impl_ops_traits!(@with_byteorder_swap $name, $native, Add, add, AddAssign, add_assign); + impl_ops_traits!(@with_byteorder_swap $name, $native, Div, div, DivAssign, div_assign); + impl_ops_traits!(@with_byteorder_swap $name, $native, Mul, mul, MulAssign, mul_assign); + impl_ops_traits!(@with_byteorder_swap $name, $native, Rem, rem, RemAssign, rem_assign); + impl_ops_traits!(@with_byteorder_swap $name, $native, Sub, sub, SubAssign, sub_assign); + }; + (@with_byteorder_swap $name:ident, $native:ident, $trait:ident, $method:ident, $trait_assign:ident, $method_assign:ident) => { + impl<O: ByteOrder> core::ops::$trait<$name<O>> for $name<O> { + type Output = $name<O>; + + #[inline(always)] + fn $method(self, rhs: $name<O>) -> $name<O> { + let self_native: $native = self.get(); + let rhs_native: $native = rhs.get(); + let result_native = core::ops::$trait::$method(self_native, rhs_native); + $name::<O>::new(result_native) + } + } + + impl<O: ByteOrder> core::ops::$trait<$name<O>> for $native { + type Output = $name<O>; + + #[inline(always)] + fn $method(self, rhs: $name<O>) -> $name<O> { + let rhs_native: $native = rhs.get(); + let result_native = core::ops::$trait::$method(self, rhs_native); + $name::<O>::new(result_native) + } + } + + impl<O: ByteOrder> core::ops::$trait<$native> for $name<O> { + type Output = $name<O>; + + #[inline(always)] + fn $method(self, rhs: $native) -> $name<O> { + let self_native: $native = self.get(); + let result_native = core::ops::$trait::$method(self_native, rhs); + $name::<O>::new(result_native) + } + } + + impl<O: ByteOrder> core::ops::$trait_assign<$name<O>> for $name<O> { + #[inline(always)] + fn $method_assign(&mut self, rhs: $name<O>) { + *self = core::ops::$trait::$method(*self, rhs); + } + } + + impl<O: ByteOrder> core::ops::$trait_assign<$name<O>> for $native { + #[inline(always)] + fn $method_assign(&mut self, rhs: $name<O>) { + let rhs_native: $native = rhs.get(); + *self = core::ops::$trait::$method(*self, rhs_native); + } + } + + impl<O: ByteOrder> core::ops::$trait_assign<$native> for $name<O> { + #[inline(always)] + fn $method_assign(&mut self, rhs: $native) { + *self = core::ops::$trait::$method(*self, rhs); + } + } + }; + // Implement traits in terms of the same trait on the native type, but + // without performing a byte order swap when both operands are byteorder + // types. This only works for bitwise operations like `&`, `|`, etc. + // + // When only one operand is a byteorder type, we still need to perform a + // byteorder swap. + (@without_byteorder_swap $name:ident, $native:ident, $trait:ident, $method:ident, $trait_assign:ident, $method_assign:ident) => { + impl<O: ByteOrder> core::ops::$trait<$name<O>> for $name<O> { + type Output = $name<O>; + + #[inline(always)] + fn $method(self, rhs: $name<O>) -> $name<O> { + let self_native = $native::from_ne_bytes(self.0); + let rhs_native = $native::from_ne_bytes(rhs.0); + let result_native = core::ops::$trait::$method(self_native, rhs_native); + $name(result_native.to_ne_bytes(), PhantomData) + } + } + + impl<O: ByteOrder> core::ops::$trait<$name<O>> for $native { + type Output = $name<O>; + + #[inline(always)] + fn $method(self, rhs: $name<O>) -> $name<O> { + // No runtime cost - just byte packing + let rhs_native = $native::from_ne_bytes(rhs.0); + // (Maybe) runtime cost - byte order swap + let slf_byteorder = $name::<O>::new(self); + // No runtime cost - just byte packing + let slf_native = $native::from_ne_bytes(slf_byteorder.0); + // Runtime cost - perform the operation + let result_native = core::ops::$trait::$method(slf_native, rhs_native); + // No runtime cost - just byte unpacking + $name(result_native.to_ne_bytes(), PhantomData) + } + } + + impl<O: ByteOrder> core::ops::$trait<$native> for $name<O> { + type Output = $name<O>; + + #[inline(always)] + fn $method(self, rhs: $native) -> $name<O> { + // (Maybe) runtime cost - byte order swap + let rhs_byteorder = $name::<O>::new(rhs); + // No runtime cost - just byte packing + let rhs_native = $native::from_ne_bytes(rhs_byteorder.0); + // No runtime cost - just byte packing + let slf_native = $native::from_ne_bytes(self.0); + // Runtime cost - perform the operation + let result_native = core::ops::$trait::$method(slf_native, rhs_native); + // No runtime cost - just byte unpacking + $name(result_native.to_ne_bytes(), PhantomData) + } + } + + impl<O: ByteOrder> core::ops::$trait_assign<$name<O>> for $name<O> { + #[inline(always)] + fn $method_assign(&mut self, rhs: $name<O>) { + *self = core::ops::$trait::$method(*self, rhs); + } + } + + impl<O: ByteOrder> core::ops::$trait_assign<$name<O>> for $native { + #[inline(always)] + fn $method_assign(&mut self, rhs: $name<O>) { + // (Maybe) runtime cost - byte order swap + let rhs_native = rhs.get(); + // Runtime cost - perform the operation + *self = core::ops::$trait::$method(*self, rhs_native); + } + } + + impl<O: ByteOrder> core::ops::$trait_assign<$native> for $name<O> { + #[inline(always)] + fn $method_assign(&mut self, rhs: $native) { + *self = core::ops::$trait::$method(*self, rhs); + } + } + }; +} + +macro_rules! doc_comment { + ($x:expr, $($tt:tt)*) => { + #[doc = $x] + $($tt)* + }; +} + +macro_rules! define_max_value_constant { + ($name:ident, $bytes:expr, "unsigned integer") => { + /// The maximum value. + /// + /// This constant should be preferred to constructing a new value using + /// `new`, as `new` may perform an endianness swap depending on the + /// endianness `O` and the endianness of the platform. + pub const MAX_VALUE: $name<O> = $name([0xFFu8; $bytes], PhantomData); + }; + // We don't provide maximum and minimum value constants for signed values + // and floats because there's no way to do it generically - it would require + // a different value depending on the value of the `ByteOrder` type + // parameter. Currently, one workaround would be to provide implementations + // for concrete implementations of that trait. In the long term, if we are + // ever able to make the `new` constructor a const fn, we could use that + // instead. + ($name:ident, $bytes:expr, "signed integer") => {}; + ($name:ident, $bytes:expr, "floating point number") => {}; +} + +macro_rules! define_type { + ( + $article:ident, + $description:expr, + $name:ident, + $native:ident, + $bits:expr, + $bytes:expr, + $from_be_fn:path, + $to_be_fn:path, + $from_le_fn:path, + $to_le_fn:path, + $number_kind:tt, + [$($larger_native:ty),*], + [$($larger_native_try:ty),*], + [$($larger_byteorder:ident),*], + [$($larger_byteorder_try:ident),*] + ) => { + doc_comment! { + concat!($description, " stored in a given byte order. + +`", stringify!($name), "` is like the native `", stringify!($native), "` type with +two major differences: First, it has no alignment requirement (its alignment is 1). +Second, the endianness of its memory layout is given by the type parameter `O`, +which can be any type which implements [`ByteOrder`]. In particular, this refers +to [`BigEndian`], [`LittleEndian`], [`NativeEndian`], and [`NetworkEndian`]. + +", stringify!($article), " `", stringify!($name), "` can be constructed using +the [`new`] method, and its contained value can be obtained as a native +`",stringify!($native), "` using the [`get`] method, or updated in place with +the [`set`] method. In all cases, if the endianness `O` is not the same as the +endianness of the current platform, an endianness swap will be performed in +order to uphold the invariants that a) the layout of `", stringify!($name), "` +has endianness `O` and that, b) the layout of `", stringify!($native), "` has +the platform's native endianness. + +`", stringify!($name), "` implements [`FromBytes`], [`IntoBytes`], and [`Unaligned`], +making it useful for parsing and serialization. See the module documentation for an +example of how it can be used for parsing UDP packets. + +[`new`]: crate::byteorder::", stringify!($name), "::new +[`get`]: crate::byteorder::", stringify!($name), "::get +[`set`]: crate::byteorder::", stringify!($name), "::set +[`FromBytes`]: crate::FromBytes +[`IntoBytes`]: crate::IntoBytes +[`Unaligned`]: crate::Unaligned"), + #[derive(Copy, Clone, Eq, PartialEq, Hash)] + #[cfg_attr(any(feature = "derive", test), derive(KnownLayout, Immutable, FromBytes, IntoBytes, Unaligned))] + #[repr(transparent)] + pub struct $name<O>([u8; $bytes], PhantomData<O>); + } + + #[cfg(not(any(feature = "derive", test)))] + impl_known_layout!(O => $name<O>); + + #[allow(unused_unsafe)] // Unused when `feature = "derive"`. + // SAFETY: `$name<O>` is `repr(transparent)`, and so it has the same + // layout as its only non-zero field, which is a `u8` array. `u8` arrays + // are `Immutable`, `TryFromBytes`, `FromZeros`, `FromBytes`, + // `IntoBytes`, and `Unaligned`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + impl_or_verify!(O => Immutable for $name<O>); + impl_or_verify!(O => TryFromBytes for $name<O>); + impl_or_verify!(O => FromZeros for $name<O>); + impl_or_verify!(O => FromBytes for $name<O>); + impl_or_verify!(O => IntoBytes for $name<O>); + impl_or_verify!(O => Unaligned for $name<O>); + }; + + impl<O> Default for $name<O> { + #[inline(always)] + fn default() -> $name<O> { + $name::ZERO + } + } + + impl<O> $name<O> { + /// The value zero. + /// + /// This constant should be preferred to constructing a new value + /// using `new`, as `new` may perform an endianness swap depending + /// on the endianness and platform. + pub const ZERO: $name<O> = $name([0u8; $bytes], PhantomData); + + define_max_value_constant!($name, $bytes, $number_kind); + + /// Constructs a new value from bytes which are already in `O` byte + /// order. + #[must_use = "has no side effects"] + #[inline(always)] + pub const fn from_bytes(bytes: [u8; $bytes]) -> $name<O> { + $name(bytes, PhantomData) + } + + /// Extracts the bytes of `self` without swapping the byte order. + /// + /// The returned bytes will be in `O` byte order. + #[must_use = "has no side effects"] + #[inline(always)] + pub const fn to_bytes(self) -> [u8; $bytes] { + self.0 + } + } + + impl<O: ByteOrder> $name<O> { + maybe_const_trait_bounded_fn! { + /// Constructs a new value, possibly performing an endianness + /// swap to guarantee that the returned value has endianness + /// `O`. + #[must_use = "has no side effects"] + #[inline(always)] + pub const fn new(n: $native) -> $name<O> { + let bytes = match O::ORDER { + Order::BigEndian => $to_be_fn(n), + Order::LittleEndian => $to_le_fn(n), + }; + + $name(bytes, PhantomData) + } + } + + maybe_const_trait_bounded_fn! { + /// Returns the value as a primitive type, possibly performing + /// an endianness swap to guarantee that the return value has + /// the endianness of the native platform. + #[must_use = "has no side effects"] + #[inline(always)] + pub const fn get(self) -> $native { + match O::ORDER { + Order::BigEndian => $from_be_fn(self.0), + Order::LittleEndian => $from_le_fn(self.0), + } + } + } + + /// Updates the value in place as a primitive type, possibly + /// performing an endianness swap to guarantee that the stored value + /// has the endianness `O`. + #[inline(always)] + pub fn set(&mut self, n: $native) { + *self = Self::new(n); + } + } + + // The reasoning behind which traits to implement here is to only + // implement traits which won't cause inference issues. Notably, + // comparison traits like PartialEq and PartialOrd tend to cause + // inference issues. + + impl<O: ByteOrder> From<$name<O>> for [u8; $bytes] { + #[inline(always)] + fn from(x: $name<O>) -> [u8; $bytes] { + x.0 + } + } + + impl<O: ByteOrder> From<[u8; $bytes]> for $name<O> { + #[inline(always)] + fn from(bytes: [u8; $bytes]) -> $name<O> { + $name(bytes, PhantomData) + } + } + + impl<O: ByteOrder> From<$name<O>> for $native { + #[inline(always)] + fn from(x: $name<O>) -> $native { + x.get() + } + } + + impl<O: ByteOrder> From<$native> for $name<O> { + #[inline(always)] + fn from(x: $native) -> $name<O> { + $name::new(x) + } + } + + $( + impl<O: ByteOrder> From<$name<O>> for $larger_native { + #[inline(always)] + fn from(x: $name<O>) -> $larger_native { + x.get().into() + } + } + )* + + $( + impl<O: ByteOrder> TryFrom<$larger_native_try> for $name<O> { + type Error = TryFromIntError; + #[inline(always)] + fn try_from(x: $larger_native_try) -> Result<$name<O>, TryFromIntError> { + $native::try_from(x).map($name::new) + } + } + )* + + $( + impl<O: ByteOrder, P: ByteOrder> From<$name<O>> for $larger_byteorder<P> { + #[inline(always)] + fn from(x: $name<O>) -> $larger_byteorder<P> { + $larger_byteorder::new(x.get().into()) + } + } + )* + + $( + impl<O: ByteOrder, P: ByteOrder> TryFrom<$larger_byteorder_try<P>> for $name<O> { + type Error = TryFromIntError; + #[inline(always)] + fn try_from(x: $larger_byteorder_try<P>) -> Result<$name<O>, TryFromIntError> { + x.get().try_into().map($name::new) + } + } + )* + + impl<O> AsRef<[u8; $bytes]> for $name<O> { + #[inline(always)] + fn as_ref(&self) -> &[u8; $bytes] { + &self.0 + } + } + + impl<O> AsMut<[u8; $bytes]> for $name<O> { + #[inline(always)] + fn as_mut(&mut self) -> &mut [u8; $bytes] { + &mut self.0 + } + } + + impl<O> PartialEq<$name<O>> for [u8; $bytes] { + #[inline(always)] + fn eq(&self, other: &$name<O>) -> bool { + self.eq(&other.0) + } + } + + impl<O> PartialEq<[u8; $bytes]> for $name<O> { + #[inline(always)] + fn eq(&self, other: &[u8; $bytes]) -> bool { + self.0.eq(other) + } + } + + impl<O: ByteOrder> PartialEq<$native> for $name<O> { + #[inline(always)] + fn eq(&self, other: &$native) -> bool { + self.get().eq(other) + } + } + + impl_fmt_traits!($name, $native, $number_kind); + impl_ops_traits!($name, $native, $number_kind); + + impl<O: ByteOrder> Debug for $name<O> { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + // This results in a format like "U16(42)". + f.debug_tuple(stringify!($name)).field(&self.get()).finish() + } + } + }; +} + +define_type!( + A, + "A 16-bit unsigned integer", + U16, + u16, + 16, + 2, + u16::from_be_bytes, + u16::to_be_bytes, + u16::from_le_bytes, + u16::to_le_bytes, + "unsigned integer", + [u32, u64, u128, usize], + [u32, u64, u128, usize], + [U32, U64, U128, Usize], + [U32, U64, U128, Usize] +); +define_type!( + A, + "A 32-bit unsigned integer", + U32, + u32, + 32, + 4, + u32::from_be_bytes, + u32::to_be_bytes, + u32::from_le_bytes, + u32::to_le_bytes, + "unsigned integer", + [u64, u128], + [u64, u128], + [U64, U128], + [U64, U128] +); +define_type!( + A, + "A 64-bit unsigned integer", + U64, + u64, + 64, + 8, + u64::from_be_bytes, + u64::to_be_bytes, + u64::from_le_bytes, + u64::to_le_bytes, + "unsigned integer", + [u128], + [u128], + [U128], + [U128] +); +define_type!( + A, + "A 128-bit unsigned integer", + U128, + u128, + 128, + 16, + u128::from_be_bytes, + u128::to_be_bytes, + u128::from_le_bytes, + u128::to_le_bytes, + "unsigned integer", + [], + [], + [], + [] +); +define_type!( + A, + "A word-sized unsigned integer", + Usize, + usize, + mem::size_of::<usize>() * 8, + mem::size_of::<usize>(), + usize::from_be_bytes, + usize::to_be_bytes, + usize::from_le_bytes, + usize::to_le_bytes, + "unsigned integer", + [], + [], + [], + [] +); +define_type!( + An, + "A 16-bit signed integer", + I16, + i16, + 16, + 2, + i16::from_be_bytes, + i16::to_be_bytes, + i16::from_le_bytes, + i16::to_le_bytes, + "signed integer", + [i32, i64, i128, isize], + [i32, i64, i128, isize], + [I32, I64, I128, Isize], + [I32, I64, I128, Isize] +); +define_type!( + An, + "A 32-bit signed integer", + I32, + i32, + 32, + 4, + i32::from_be_bytes, + i32::to_be_bytes, + i32::from_le_bytes, + i32::to_le_bytes, + "signed integer", + [i64, i128], + [i64, i128], + [I64, I128], + [I64, I128] +); +define_type!( + An, + "A 64-bit signed integer", + I64, + i64, + 64, + 8, + i64::from_be_bytes, + i64::to_be_bytes, + i64::from_le_bytes, + i64::to_le_bytes, + "signed integer", + [i128], + [i128], + [I128], + [I128] +); +define_type!( + An, + "A 128-bit signed integer", + I128, + i128, + 128, + 16, + i128::from_be_bytes, + i128::to_be_bytes, + i128::from_le_bytes, + i128::to_le_bytes, + "signed integer", + [], + [], + [], + [] +); +define_type!( + An, + "A word-sized signed integer", + Isize, + isize, + mem::size_of::<isize>() * 8, + mem::size_of::<isize>(), + isize::from_be_bytes, + isize::to_be_bytes, + isize::from_le_bytes, + isize::to_le_bytes, + "signed integer", + [], + [], + [], + [] +); + +// FIXME(https://github.com/rust-lang/rust/issues/72447): Use the endianness +// conversion methods directly once those are const-stable. +macro_rules! define_float_conversion { + ($ty:ty, $bits:ident, $bytes:expr, $mod:ident) => { + mod $mod { + use super::*; + + define_float_conversion!($ty, $bits, $bytes, from_be_bytes, to_be_bytes); + define_float_conversion!($ty, $bits, $bytes, from_le_bytes, to_le_bytes); + } + }; + ($ty:ty, $bits:ident, $bytes:expr, $from:ident, $to:ident) => { + // Clippy: The suggestion of using `from_bits()` instead doesn't work + // because `from_bits` is not const-stable on our MSRV. + #[allow(clippy::unnecessary_transmutes)] + pub(crate) const fn $from(bytes: [u8; $bytes]) -> $ty { + transmute!($bits::$from(bytes)) + } + + pub(crate) const fn $to(f: $ty) -> [u8; $bytes] { + // Clippy: The suggestion of using `f.to_bits()` instead doesn't + // work because `to_bits` is not const-stable on our MSRV. + #[allow(clippy::unnecessary_transmutes)] + let bits: $bits = transmute!(f); + bits.$to() + } + }; +} + +define_float_conversion!(f32, u32, 4, f32_ext); +define_float_conversion!(f64, u64, 8, f64_ext); + +define_type!( + An, + "A 32-bit floating point number", + F32, + f32, + 32, + 4, + f32_ext::from_be_bytes, + f32_ext::to_be_bytes, + f32_ext::from_le_bytes, + f32_ext::to_le_bytes, + "floating point number", + [f64], + [], + [F64], + [] +); +define_type!( + An, + "A 64-bit floating point number", + F64, + f64, + 64, + 8, + f64_ext::from_be_bytes, + f64_ext::to_be_bytes, + f64_ext::from_le_bytes, + f64_ext::to_le_bytes, + "floating point number", + [], + [], + [], + [] +); + +macro_rules! module { + ($name:ident, $trait:ident, $endianness_str:expr) => { + /// Numeric primitives stored in + #[doc = $endianness_str] + /// byte order. + pub mod $name { + use super::$trait; + + module!(@ty U16, $trait, "16-bit unsigned integer", $endianness_str); + module!(@ty U32, $trait, "32-bit unsigned integer", $endianness_str); + module!(@ty U64, $trait, "64-bit unsigned integer", $endianness_str); + module!(@ty U128, $trait, "128-bit unsigned integer", $endianness_str); + module!(@ty I16, $trait, "16-bit signed integer", $endianness_str); + module!(@ty I32, $trait, "32-bit signed integer", $endianness_str); + module!(@ty I64, $trait, "64-bit signed integer", $endianness_str); + module!(@ty I128, $trait, "128-bit signed integer", $endianness_str); + module!(@ty F32, $trait, "32-bit floating point number", $endianness_str); + module!(@ty F64, $trait, "64-bit floating point number", $endianness_str); + } + }; + (@ty $ty:ident, $trait:ident, $desc_str:expr, $endianness_str:expr) => { + /// A + #[doc = $desc_str] + /// stored in + #[doc = $endianness_str] + /// byte order. + pub type $ty = crate::byteorder::$ty<$trait>; + }; +} + +module!(big_endian, BigEndian, "big-endian"); +module!(little_endian, LittleEndian, "little-endian"); +module!(network_endian, NetworkEndian, "network-endian"); +module!(native_endian, NativeEndian, "native-endian"); + +#[cfg(any(test, kani))] +mod tests { + use super::*; + + #[cfg(not(kani))] + mod compatibility { + pub(super) use rand::{ + distributions::{Distribution, Standard}, + rngs::SmallRng, + Rng, SeedableRng, + }; + + pub(crate) trait Arbitrary {} + + impl<T> Arbitrary for T {} + } + + #[cfg(kani)] + mod compatibility { + pub(crate) use kani::Arbitrary; + + pub(crate) struct SmallRng; + + impl SmallRng { + pub(crate) fn seed_from_u64(_state: u64) -> Self { + Self + } + } + + pub(crate) trait Rng { + fn sample<T, D: Distribution<T>>(&mut self, _distr: D) -> T + where + T: Arbitrary, + { + kani::any() + } + } + + impl Rng for SmallRng {} + + pub(crate) trait Distribution<T> {} + impl<T, U> Distribution<T> for U {} + + pub(crate) struct Standard; + } + + use compatibility::*; + + // A native integer type (u16, i32, etc). + trait Native: Arbitrary + FromBytes + IntoBytes + Immutable + Copy + PartialEq + Debug { + const ZERO: Self; + const MAX_VALUE: Self; + + type Distribution: Distribution<Self>; + const DIST: Self::Distribution; + + fn rand<R: Rng>(rng: &mut R) -> Self { + rng.sample(Self::DIST) + } + + #[cfg_attr(kani, allow(unused))] + fn checked_add(self, rhs: Self) -> Option<Self>; + + #[cfg_attr(kani, allow(unused))] + fn checked_div(self, rhs: Self) -> Option<Self>; + + #[cfg_attr(kani, allow(unused))] + fn checked_mul(self, rhs: Self) -> Option<Self>; + + #[cfg_attr(kani, allow(unused))] + fn checked_rem(self, rhs: Self) -> Option<Self>; + + #[cfg_attr(kani, allow(unused))] + fn checked_sub(self, rhs: Self) -> Option<Self>; + + #[cfg_attr(kani, allow(unused))] + fn checked_shl(self, rhs: Self) -> Option<Self>; + + #[cfg_attr(kani, allow(unused))] + fn checked_shr(self, rhs: Self) -> Option<Self>; + + fn is_nan(self) -> bool; + + /// For `f32` and `f64`, NaN values are not considered equal to + /// themselves. This method is like `assert_eq!`, but it treats NaN + /// values as equal. + fn assert_eq_or_nan(self, other: Self) { + let slf = (!self.is_nan()).then(|| self); + let other = (!other.is_nan()).then(|| other); + assert_eq!(slf, other); + } + } + + trait ByteArray: + FromBytes + IntoBytes + Immutable + Copy + AsRef<[u8]> + AsMut<[u8]> + Debug + Default + Eq + { + /// Invert the order of the bytes in the array. + fn invert(self) -> Self; + } + + trait ByteOrderType: + FromBytes + IntoBytes + Unaligned + Copy + Eq + Debug + Hash + From<Self::Native> + { + type Native: Native; + type ByteArray: ByteArray; + + const ZERO: Self; + + fn new(native: Self::Native) -> Self; + fn get(self) -> Self::Native; + fn set(&mut self, native: Self::Native); + fn from_bytes(bytes: Self::ByteArray) -> Self; + fn into_bytes(self) -> Self::ByteArray; + + /// For `f32` and `f64`, NaN values are not considered equal to + /// themselves. This method is like `assert_eq!`, but it treats NaN + /// values as equal. + fn assert_eq_or_nan(self, other: Self) { + let slf = (!self.get().is_nan()).then(|| self); + let other = (!other.get().is_nan()).then(|| other); + assert_eq!(slf, other); + } + } + + trait ByteOrderTypeUnsigned: ByteOrderType { + const MAX_VALUE: Self; + } + + macro_rules! impl_byte_array { + ($bytes:expr) => { + impl ByteArray for [u8; $bytes] { + fn invert(mut self) -> [u8; $bytes] { + self.reverse(); + self + } + } + }; + } + + impl_byte_array!(2); + impl_byte_array!(4); + impl_byte_array!(8); + impl_byte_array!(16); + + macro_rules! impl_byte_order_type_unsigned { + ($name:ident, unsigned) => { + impl<O: ByteOrder> ByteOrderTypeUnsigned for $name<O> { + const MAX_VALUE: $name<O> = $name::MAX_VALUE; + } + }; + ($name:ident, signed) => {}; + } + + macro_rules! impl_traits { + ($name:ident, $native:ident, $sign:ident $(, @$float:ident)?) => { + impl Native for $native { + // For some types, `0 as $native` is required (for example, when + // `$native` is a floating-point type; `0` is an integer), but + // for other types, it's a trivial cast. In all cases, Clippy + // thinks it's dangerous. + #[allow(trivial_numeric_casts, clippy::as_conversions)] + const ZERO: $native = 0 as $native; + const MAX_VALUE: $native = $native::MAX; + + type Distribution = Standard; + const DIST: Standard = Standard; + + impl_traits!(@float_dependent_methods $(@$float)?); + } + + impl<O: ByteOrder> ByteOrderType for $name<O> { + type Native = $native; + type ByteArray = [u8; mem::size_of::<$native>()]; + + const ZERO: $name<O> = $name::ZERO; + + fn new(native: $native) -> $name<O> { + $name::new(native) + } + + fn get(self) -> $native { + $name::get(self) + } + + fn set(&mut self, native: $native) { + $name::set(self, native) + } + + fn from_bytes(bytes: [u8; mem::size_of::<$native>()]) -> $name<O> { + $name::from(bytes) + } + + fn into_bytes(self) -> [u8; mem::size_of::<$native>()] { + <[u8; mem::size_of::<$native>()]>::from(self) + } + } + + impl_byte_order_type_unsigned!($name, $sign); + }; + (@float_dependent_methods) => { + fn checked_add(self, rhs: Self) -> Option<Self> { self.checked_add(rhs) } + fn checked_div(self, rhs: Self) -> Option<Self> { self.checked_div(rhs) } + fn checked_mul(self, rhs: Self) -> Option<Self> { self.checked_mul(rhs) } + fn checked_rem(self, rhs: Self) -> Option<Self> { self.checked_rem(rhs) } + fn checked_sub(self, rhs: Self) -> Option<Self> { self.checked_sub(rhs) } + fn checked_shl(self, rhs: Self) -> Option<Self> { self.checked_shl(rhs.try_into().unwrap_or(u32::MAX)) } + fn checked_shr(self, rhs: Self) -> Option<Self> { self.checked_shr(rhs.try_into().unwrap_or(u32::MAX)) } + fn is_nan(self) -> bool { false } + }; + (@float_dependent_methods @float) => { + fn checked_add(self, rhs: Self) -> Option<Self> { Some(self + rhs) } + fn checked_div(self, rhs: Self) -> Option<Self> { Some(self / rhs) } + fn checked_mul(self, rhs: Self) -> Option<Self> { Some(self * rhs) } + fn checked_rem(self, rhs: Self) -> Option<Self> { Some(self % rhs) } + fn checked_sub(self, rhs: Self) -> Option<Self> { Some(self - rhs) } + fn checked_shl(self, _rhs: Self) -> Option<Self> { unimplemented!() } + fn checked_shr(self, _rhs: Self) -> Option<Self> { unimplemented!() } + fn is_nan(self) -> bool { self.is_nan() } + }; + } + + impl_traits!(U16, u16, unsigned); + impl_traits!(U32, u32, unsigned); + impl_traits!(U64, u64, unsigned); + impl_traits!(U128, u128, unsigned); + impl_traits!(Usize, usize, unsigned); + impl_traits!(I16, i16, signed); + impl_traits!(I32, i32, signed); + impl_traits!(I64, i64, signed); + impl_traits!(I128, i128, signed); + impl_traits!(Isize, isize, unsigned); + impl_traits!(F32, f32, signed, @float); + impl_traits!(F64, f64, signed, @float); + + macro_rules! call_for_unsigned_types { + ($fn:ident, $byteorder:ident) => { + $fn::<U16<$byteorder>>(); + $fn::<U32<$byteorder>>(); + $fn::<U64<$byteorder>>(); + $fn::<U128<$byteorder>>(); + $fn::<Usize<$byteorder>>(); + }; + } + + macro_rules! call_for_signed_types { + ($fn:ident, $byteorder:ident) => { + $fn::<I16<$byteorder>>(); + $fn::<I32<$byteorder>>(); + $fn::<I64<$byteorder>>(); + $fn::<I128<$byteorder>>(); + $fn::<Isize<$byteorder>>(); + }; + } + + macro_rules! call_for_float_types { + ($fn:ident, $byteorder:ident) => { + $fn::<F32<$byteorder>>(); + $fn::<F64<$byteorder>>(); + }; + } + + macro_rules! call_for_all_types { + ($fn:ident, $byteorder:ident) => { + call_for_unsigned_types!($fn, $byteorder); + call_for_signed_types!($fn, $byteorder); + call_for_float_types!($fn, $byteorder); + }; + } + + #[cfg(target_endian = "big")] + type NonNativeEndian = LittleEndian; + #[cfg(target_endian = "little")] + type NonNativeEndian = BigEndian; + + // We use a `u64` seed so that we can use `SeedableRng::seed_from_u64`. + // `SmallRng`'s `SeedableRng::Seed` differs by platform, so if we wanted to + // call `SeedableRng::from_seed`, which takes a `Seed`, we would need + // conditional compilation by `target_pointer_width`. + const RNG_SEED: u64 = 0x7A03CAE2F32B5B8F; + + const RAND_ITERS: usize = if cfg!(any(miri, kani)) { + // The tests below which use this constant used to take a very long time + // on Miri, which slows down local development and CI jobs. We're not + // using Miri to check for the correctness of our code, but rather its + // soundness, and at least in the context of these particular tests, a + // single loop iteration is just as good for surfacing UB as multiple + // iterations are. + // + // As of the writing of this comment, here's one set of measurements: + // + // $ # RAND_ITERS == 1 + // $ cargo miri test -- -Z unstable-options --report-time endian + // test byteorder::tests::test_native_endian ... ok <0.049s> + // test byteorder::tests::test_non_native_endian ... ok <0.061s> + // + // $ # RAND_ITERS == 1024 + // $ cargo miri test -- -Z unstable-options --report-time endian + // test byteorder::tests::test_native_endian ... ok <25.716s> + // test byteorder::tests::test_non_native_endian ... ok <38.127s> + 1 + } else { + 1024 + }; + + #[test] + fn test_const_methods() { + use big_endian::*; + + #[rustversion::since(1.61.0)] + const _U: U16 = U16::new(0); + #[rustversion::since(1.61.0)] + const _NATIVE: u16 = _U.get(); + const _FROM_BYTES: U16 = U16::from_bytes([0, 1]); + const _BYTES: [u8; 2] = _FROM_BYTES.to_bytes(); + } + + #[cfg_attr(test, test)] + #[cfg_attr(kani, kani::proof)] + fn test_zero() { + fn test_zero<T: ByteOrderType>() { + assert_eq!(T::ZERO.get(), T::Native::ZERO); + } + + call_for_all_types!(test_zero, NativeEndian); + call_for_all_types!(test_zero, NonNativeEndian); + } + + #[cfg_attr(test, test)] + #[cfg_attr(kani, kani::proof)] + fn test_max_value() { + fn test_max_value<T: ByteOrderTypeUnsigned>() { + assert_eq!(T::MAX_VALUE.get(), T::Native::MAX_VALUE); + } + + call_for_unsigned_types!(test_max_value, NativeEndian); + call_for_unsigned_types!(test_max_value, NonNativeEndian); + } + + #[cfg_attr(test, test)] + #[cfg_attr(kani, kani::proof)] + fn test_endian() { + fn test<T: ByteOrderType>(invert: bool) { + let mut r = SmallRng::seed_from_u64(RNG_SEED); + for _ in 0..RAND_ITERS { + let native = T::Native::rand(&mut r); + let mut bytes = T::ByteArray::default(); + bytes.as_mut_bytes().copy_from_slice(native.as_bytes()); + if invert { + bytes = bytes.invert(); + } + let mut from_native = T::new(native); + let from_bytes = T::from_bytes(bytes); + + from_native.assert_eq_or_nan(from_bytes); + from_native.get().assert_eq_or_nan(native); + from_bytes.get().assert_eq_or_nan(native); + + assert_eq!(from_native.into_bytes(), bytes); + assert_eq!(from_bytes.into_bytes(), bytes); + + let updated = T::Native::rand(&mut r); + from_native.set(updated); + from_native.get().assert_eq_or_nan(updated); + } + } + + fn test_native<T: ByteOrderType>() { + test::<T>(false); + } + + fn test_non_native<T: ByteOrderType>() { + test::<T>(true); + } + + call_for_all_types!(test_native, NativeEndian); + call_for_all_types!(test_non_native, NonNativeEndian); + } + + #[test] + fn test_ops_impls() { + // Test implementations of traits in `core::ops`. Some of these are + // fairly banal, but some are optimized to perform the operation without + // swapping byte order (namely, bit-wise operations which are identical + // regardless of byte order). These are important to test, and while + // we're testing those anyway, it's trivial to test all of the impls. + + fn test<T, FTT, FTN, FNT, FNN, FNNChecked, FATT, FATN, FANT>( + op_t_t: FTT, + op_t_n: FTN, + op_n_t: FNT, + op_n_n: FNN, + op_n_n_checked: Option<FNNChecked>, + op_assign: Option<(FATT, FATN, FANT)>, + ) where + T: ByteOrderType, + FTT: Fn(T, T) -> T, + FTN: Fn(T, T::Native) -> T, + FNT: Fn(T::Native, T) -> T, + FNN: Fn(T::Native, T::Native) -> T::Native, + FNNChecked: Fn(T::Native, T::Native) -> Option<T::Native>, + FATT: Fn(&mut T, T), + FATN: Fn(&mut T, T::Native), + FANT: Fn(&mut T::Native, T), + { + let mut r = SmallRng::seed_from_u64(RNG_SEED); + for _ in 0..RAND_ITERS { + let n0 = T::Native::rand(&mut r); + let n1 = T::Native::rand(&mut r); + let t0 = T::new(n0); + let t1 = T::new(n1); + + // If this operation would overflow/underflow, skip it rather + // than attempt to catch and recover from panics. + if matches!(&op_n_n_checked, Some(checked) if checked(n0, n1).is_none()) { + continue; + } + + let t_t_res = op_t_t(t0, t1); + let t_n_res = op_t_n(t0, n1); + let n_t_res = op_n_t(n0, t1); + let n_n_res = op_n_n(n0, n1); + + // For `f32` and `f64`, NaN values are not considered equal to + // themselves. We store `Option<f32>`/`Option<f64>` and store + // NaN as `None` so they can still be compared. + let val_or_none = |t: T| (!T::Native::is_nan(t.get())).then(|| t.get()); + let t_t_res = val_or_none(t_t_res); + let t_n_res = val_or_none(t_n_res); + let n_t_res = val_or_none(n_t_res); + let n_n_res = (!T::Native::is_nan(n_n_res)).then(|| n_n_res); + assert_eq!(t_t_res, n_n_res); + assert_eq!(t_n_res, n_n_res); + assert_eq!(n_t_res, n_n_res); + + if let Some((op_assign_t_t, op_assign_t_n, op_assign_n_t)) = &op_assign { + let mut t_t_res = t0; + op_assign_t_t(&mut t_t_res, t1); + let mut t_n_res = t0; + op_assign_t_n(&mut t_n_res, n1); + let mut n_t_res = n0; + op_assign_n_t(&mut n_t_res, t1); + + // For `f32` and `f64`, NaN values are not considered equal to + // themselves. We store `Option<f32>`/`Option<f64>` and store + // NaN as `None` so they can still be compared. + let t_t_res = val_or_none(t_t_res); + let t_n_res = val_or_none(t_n_res); + let n_t_res = (!T::Native::is_nan(n_t_res)).then(|| n_t_res); + assert_eq!(t_t_res, n_n_res); + assert_eq!(t_n_res, n_n_res); + assert_eq!(n_t_res, n_n_res); + } + } + } + + macro_rules! test { + ( + @binary + $trait:ident, + $method:ident $([$checked_method:ident])?, + $trait_assign:ident, + $method_assign:ident, + $($call_for_macros:ident),* + ) => {{ + fn t<T>() + where + T: ByteOrderType, + T: core::ops::$trait<T, Output = T>, + T: core::ops::$trait<T::Native, Output = T>, + T::Native: core::ops::$trait<T, Output = T>, + T::Native: core::ops::$trait<T::Native, Output = T::Native>, + + T: core::ops::$trait_assign<T>, + T: core::ops::$trait_assign<T::Native>, + T::Native: core::ops::$trait_assign<T>, + T::Native: core::ops::$trait_assign<T::Native>, + { + test::<T, _, _, _, _, _, _, _, _>( + core::ops::$trait::$method, + core::ops::$trait::$method, + core::ops::$trait::$method, + core::ops::$trait::$method, + { + #[allow(unused_mut, unused_assignments)] + let mut op_native_checked = None::<fn(T::Native, T::Native) -> Option<T::Native>>; + $( + op_native_checked = Some(T::Native::$checked_method); + )? + op_native_checked + }, + Some(( + <T as core::ops::$trait_assign<T>>::$method_assign, + <T as core::ops::$trait_assign::<T::Native>>::$method_assign, + <T::Native as core::ops::$trait_assign::<T>>::$method_assign + )), + ); + } + + $( + $call_for_macros!(t, NativeEndian); + $call_for_macros!(t, NonNativeEndian); + )* + }}; + ( + @unary + $trait:ident, + $method:ident, + $($call_for_macros:ident),* + ) => {{ + fn t<T>() + where + T: ByteOrderType, + T: core::ops::$trait<Output = T>, + T::Native: core::ops::$trait<Output = T::Native>, + { + test::<T, _, _, _, _, _, _, _, _>( + |slf, _rhs| core::ops::$trait::$method(slf), + |slf, _rhs| core::ops::$trait::$method(slf), + |slf, _rhs| core::ops::$trait::$method(slf).into(), + |slf, _rhs| core::ops::$trait::$method(slf), + None::<fn(T::Native, T::Native) -> Option<T::Native>>, + None::<(fn(&mut T, T), fn(&mut T, T::Native), fn(&mut T::Native, T))>, + ); + } + + $( + $call_for_macros!(t, NativeEndian); + $call_for_macros!(t, NonNativeEndian); + )* + }}; + } + + test!(@binary Add, add[checked_add], AddAssign, add_assign, call_for_all_types); + test!(@binary Div, div[checked_div], DivAssign, div_assign, call_for_all_types); + test!(@binary Mul, mul[checked_mul], MulAssign, mul_assign, call_for_all_types); + test!(@binary Rem, rem[checked_rem], RemAssign, rem_assign, call_for_all_types); + test!(@binary Sub, sub[checked_sub], SubAssign, sub_assign, call_for_all_types); + + test!(@binary BitAnd, bitand, BitAndAssign, bitand_assign, call_for_unsigned_types, call_for_signed_types); + test!(@binary BitOr, bitor, BitOrAssign, bitor_assign, call_for_unsigned_types, call_for_signed_types); + test!(@binary BitXor, bitxor, BitXorAssign, bitxor_assign, call_for_unsigned_types, call_for_signed_types); + test!(@binary Shl, shl[checked_shl], ShlAssign, shl_assign, call_for_unsigned_types, call_for_signed_types); + test!(@binary Shr, shr[checked_shr], ShrAssign, shr_assign, call_for_unsigned_types, call_for_signed_types); + + test!(@unary Not, not, call_for_signed_types, call_for_unsigned_types); + test!(@unary Neg, neg, call_for_signed_types, call_for_float_types); + } + + #[test] + fn test_debug_impl() { + // Ensure that Debug applies format options to the inner value. + let val = U16::<LE>::new(10); + assert_eq!(format!("{:?}", val), "U16(10)"); + assert_eq!(format!("{:03?}", val), "U16(010)"); + assert_eq!(format!("{:x?}", val), "U16(a)"); + } + + #[test] + fn test_byteorder_traits_coverage() { + let val_be = U16::<BigEndian>::from_bytes([0, 1]); + let val_le = U16::<LittleEndian>::from_bytes([1, 0]); + + assert_eq!(val_be.get(), 1); + assert_eq!(val_le.get(), 1); + + // Debug + assert_eq!(format!("{:?}", val_be), "U16(1)"); + assert_eq!(format!("{:?}", val_le), "U16(1)"); + + // PartialOrd, Ord with same type + assert!(val_be >= val_be); + assert!(val_be <= val_be); + assert_eq!(val_be.cmp(&val_be), core::cmp::Ordering::Equal); + + // PartialOrd with native + assert!(val_be == 1u16); + assert!(val_be >= 1u16); + + // Default + let default_be: U16<BigEndian> = Default::default(); + assert_eq!(default_be.get(), 0); + + // I16 + let val_be_i16 = I16::<BigEndian>::from_bytes([0, 1]); + assert_eq!(val_be_i16.get(), 1); + assert_eq!(format!("{:?}", val_be_i16), "I16(1)"); + assert_eq!(val_be_i16.cmp(&val_be_i16), core::cmp::Ordering::Equal); + } +} diff --git a/rust/zerocopy/src/deprecated.rs b/rust/zerocopy/src/deprecated.rs new file mode 100644 index 000000000000..24bafbf9adeb --- /dev/null +++ b/rust/zerocopy/src/deprecated.rs @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2024 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License <LICENSE-BSD or +// https://opensource.org/license/bsd-2-clause>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Deprecated items. These are kept separate so that they don't clutter up +//! other modules. + +use super::*; + +impl<B, T> Ref<B, T> +where + B: ByteSlice, + T: KnownLayout + Immutable + ?Sized, +{ + #[deprecated(since = "0.8.0", note = "renamed to `Ref::from_bytes`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new(bytes: B) -> Option<Ref<B, T>> { + Self::from_bytes(bytes).ok() + } +} + +impl<B, T> Ref<B, T> +where + B: SplitByteSlice, + T: KnownLayout + Immutable + ?Sized, +{ + #[deprecated(since = "0.8.0", note = "renamed to `Ref::from_prefix`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_from_prefix(bytes: B) -> Option<(Ref<B, T>, B)> { + Self::from_prefix(bytes).ok() + } +} + +impl<B, T> Ref<B, T> +where + B: SplitByteSlice, + T: KnownLayout + Immutable + ?Sized, +{ + #[deprecated(since = "0.8.0", note = "renamed to `Ref::from_suffix`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_from_suffix(bytes: B) -> Option<(B, Ref<B, T>)> { + Self::from_suffix(bytes).ok() + } +} + +impl<B, T> Ref<B, T> +where + B: ByteSlice, + T: Unaligned + KnownLayout + Immutable + ?Sized, +{ + #[deprecated( + since = "0.8.0", + note = "use `Ref::from_bytes`; for `T: Unaligned`, the returned `CastError` implements `Into<SizeError>`" + )] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_unaligned(bytes: B) -> Option<Ref<B, T>> { + Self::from_bytes(bytes).ok() + } +} + +impl<B, T> Ref<B, T> +where + B: SplitByteSlice, + T: Unaligned + KnownLayout + Immutable + ?Sized, +{ + #[deprecated( + since = "0.8.0", + note = "use `Ref::from_prefix`; for `T: Unaligned`, the returned `CastError` implements `Into<SizeError>`" + )] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_unaligned_from_prefix(bytes: B) -> Option<(Ref<B, T>, B)> { + Self::from_prefix(bytes).ok() + } +} + +impl<B, T> Ref<B, T> +where + B: SplitByteSlice, + T: Unaligned + KnownLayout + Immutable + ?Sized, +{ + #[deprecated( + since = "0.8.0", + note = "use `Ref::from_suffix`; for `T: Unaligned`, the returned `CastError` implements `Into<SizeError>`" + )] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_unaligned_from_suffix(bytes: B) -> Option<(B, Ref<B, T>)> { + Self::from_suffix(bytes).ok() + } +} + +impl<B, T> Ref<B, [T]> +where + B: ByteSlice, + T: Immutable, +{ + #[deprecated(since = "0.8.0", note = "`Ref::from_bytes` now supports slices")] + #[doc(hidden)] + #[inline(always)] + pub fn new_slice(bytes: B) -> Option<Ref<B, [T]>> { + Self::from_bytes(bytes).ok() + } +} + +impl<B, T> Ref<B, [T]> +where + B: ByteSlice, + T: Unaligned + Immutable, +{ + #[deprecated( + since = "0.8.0", + note = "`Ref::from_bytes` now supports slices; for `T: Unaligned`, the returned `CastError` implements `Into<SizeError>`" + )] + #[doc(hidden)] + #[inline(always)] + pub fn new_slice_unaligned(bytes: B) -> Option<Ref<B, [T]>> { + Ref::from_bytes(bytes).ok() + } +} + +impl<'a, B, T> Ref<B, [T]> +where + B: 'a + IntoByteSlice<'a>, + T: FromBytes + Immutable, +{ + #[deprecated(since = "0.8.0", note = "`Ref::into_ref` now supports slices")] + #[doc(hidden)] + #[inline(always)] + pub fn into_slice(self) -> &'a [T] { + Ref::into_ref(self) + } +} + +impl<'a, B, T> Ref<B, [T]> +where + B: 'a + IntoByteSliceMut<'a>, + T: FromBytes + IntoBytes + Immutable, +{ + #[deprecated(since = "0.8.0", note = "`Ref::into_mut` now supports slices")] + #[doc(hidden)] + #[inline(always)] + pub fn into_mut_slice(self) -> &'a mut [T] { + Ref::into_mut(self) + } +} + +impl<B, T> Ref<B, [T]> +where + B: SplitByteSlice, + T: Immutable, +{ + #[deprecated(since = "0.8.0", note = "replaced by `Ref::from_prefix_with_elems`")] + #[must_use = "has no side effects"] + #[doc(hidden)] + #[inline(always)] + pub fn new_slice_from_prefix(bytes: B, count: usize) -> Option<(Ref<B, [T]>, B)> { + Ref::from_prefix_with_elems(bytes, count).ok() + } + + #[deprecated(since = "0.8.0", note = "replaced by `Ref::from_suffix_with_elems`")] + #[must_use = "has no side effects"] + #[doc(hidden)] + #[inline(always)] + pub fn new_slice_from_suffix(bytes: B, count: usize) -> Option<(B, Ref<B, [T]>)> { + Ref::from_suffix_with_elems(bytes, count).ok() + } +} + +impl<B, T> Ref<B, [T]> +where + B: SplitByteSlice, + T: Unaligned + Immutable, +{ + #[deprecated( + since = "0.8.0", + note = "use `Ref::from_prefix_with_elems`; for `T: Unaligned`, the returned `CastError` implements `Into<SizeError>`" + )] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_slice_unaligned_from_prefix(bytes: B, count: usize) -> Option<(Ref<B, [T]>, B)> { + Ref::from_prefix_with_elems(bytes, count).ok() + } + + #[deprecated( + since = "0.8.0", + note = "use `Ref::from_suffix_with_elems`; for `T: Unaligned`, the returned `CastError` implements `Into<SizeError>`" + )] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_slice_unaligned_from_suffix(bytes: B, count: usize) -> Option<(B, Ref<B, [T]>)> { + Ref::from_suffix_with_elems(bytes, count).ok() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[allow(deprecated)] + fn test_deprecated_ref_methods() { + let bytes = &[0u8; 1][..]; + let bytes_slice = &[0u8; 4][..]; + + let r: Option<Ref<&[u8], u8>> = Ref::new(bytes); + assert!(r.is_some()); + + let r: Option<(Ref<&[u8], u8>, &[u8])> = Ref::new_from_prefix(bytes); + assert!(r.is_some()); + + let r: Option<(&[u8], Ref<&[u8], u8>)> = Ref::new_from_suffix(bytes); + assert!(r.is_some()); + + let r: Option<Ref<&[u8], u8>> = Ref::new_unaligned(bytes); + assert!(r.is_some()); + + let r: Option<(Ref<&[u8], u8>, &[u8])> = Ref::new_unaligned_from_prefix(bytes); + assert!(r.is_some()); + + let r: Option<(&[u8], Ref<&[u8], u8>)> = Ref::new_unaligned_from_suffix(bytes); + assert!(r.is_some()); + + let r: Option<Ref<&[u8], [u8]>> = Ref::new_slice(bytes_slice); + assert!(r.is_some()); + + let r: Option<Ref<&[u8], [u8]>> = Ref::new_slice_unaligned(bytes_slice); + assert!(r.is_some()); + + let r: Option<(Ref<&[u8], [u8]>, &[u8])> = Ref::new_slice_from_prefix(bytes_slice, 1); + assert!(r.is_some()); + + let r: Option<(&[u8], Ref<&[u8], [u8]>)> = Ref::new_slice_from_suffix(bytes_slice, 1); + assert!(r.is_some()); + + let r: Option<(Ref<&[u8], [u8]>, &[u8])> = + Ref::new_slice_unaligned_from_prefix(bytes_slice, 1); + assert!(r.is_some()); + + let r: Option<(&[u8], Ref<&[u8], [u8]>)> = + Ref::new_slice_unaligned_from_suffix(bytes_slice, 1); + assert!(r.is_some()); + } + + #[test] + #[allow(deprecated)] + fn test_deprecated_into_slice() { + let bytes = &[0u8; 4][..]; + let r: Ref<&[u8], [u8]> = Ref::from_bytes(bytes).unwrap(); + let slice: &[u8] = r.into_slice(); + assert_eq!(slice.len(), 4); + } + + #[test] + #[allow(deprecated)] + fn test_deprecated_into_mut_slice() { + let mut bytes = [0u8; 4]; + let r: Ref<&mut [u8], [u8]> = Ref::from_bytes(&mut bytes[..]).unwrap(); + let slice: &mut [u8] = r.into_mut_slice(); + assert_eq!(slice.len(), 4); + } +} diff --git a/rust/zerocopy/src/error.rs b/rust/zerocopy/src/error.rs new file mode 100644 index 000000000000..7cb08c31d452 --- /dev/null +++ b/rust/zerocopy/src/error.rs @@ -0,0 +1,1350 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2024 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License <LICENSE-BSD or +// https://opensource.org/license/bsd-2-clause>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Types related to error reporting. +//! +//! ## Single failure mode errors +//! +//! Generally speaking, zerocopy's conversions may fail for one of up to three +//! reasons: +//! - [`AlignmentError`]: the conversion source was improperly aligned +//! - [`SizeError`]: the conversion source was of incorrect size +//! - [`ValidityError`]: the conversion source contained invalid data +//! +//! Methods that only have one failure mode, like +//! [`FromBytes::read_from_bytes`], return that mode's corresponding error type +//! directly. +//! +//! ## Compound errors +//! +//! Conversion methods that have either two or three possible failure modes +//! return one of these error types: +//! - [`CastError`]: the error type of reference conversions +//! - [`TryCastError`]: the error type of fallible reference conversions +//! - [`TryReadError`]: the error type of fallible read conversions +//! +//! ## [`Unaligned`] destination types +//! +//! For [`Unaligned`] destination types, alignment errors are impossible. All +//! compound error types support infallibly discarding the alignment error via +//! [`From`] so long as `Dst: Unaligned`. For example, see [`<SizeError as +//! From<ConvertError>>::from`][size-error-from]. +//! +//! [size-error-from]: struct.SizeError.html#method.from-1 +//! +//! ## Accessing the conversion source +//! +//! All error types provide an `into_src` method that converts the error into +//! the source value underlying the failed conversion. +//! +//! ## Display formatting +//! +//! All error types provide a `Display` implementation that produces a +//! human-readable error message. When `debug_assertions` are enabled, these +//! error messages are verbose and may include potentially sensitive +//! information, including: +//! +//! - the names of the involved types +//! - the sizes of the involved types +//! - the addresses of the involved types +//! - the contents of the involved types +//! +//! When `debug_assertions` are disabled (as is default for `release` builds), +//! such potentially sensitive information is excluded. +//! +//! In the future, we may support manually configuring this behavior. If you are +//! interested in this feature, [let us know on GitHub][issue-1457] so we know +//! to prioritize it. +//! +//! [issue-1457]: https://github.com/google/zerocopy/issues/1457 +//! +//! ## Validation order +//! +//! Our conversion methods typically check alignment, then size, then bit +//! validity. However, we do not guarantee that this is always the case, and +//! this behavior may change between releases. +//! +//! ## `Send`, `Sync`, and `'static` +//! +//! Our error types are `Send`, `Sync`, and `'static` when their `Src` parameter +//! is `Send`, `Sync`, or `'static`, respectively. This can cause issues when an +//! error is sent or synchronized across threads; e.g.: +//! +//! ```compile_fail,E0515 +//! use zerocopy::*; +//! +//! let result: SizeError<&[u8], u32> = std::thread::spawn(|| { +//! let source = &mut [0u8, 1, 2][..]; +//! // Try (and fail) to read a `u32` from `source`. +//! u32::read_from_bytes(source).unwrap_err() +//! }).join().unwrap(); +//! ``` +//! +//! To work around this, use [`map_src`][CastError::map_src] to convert the +//! source parameter to an unproblematic type; e.g.: +//! +//! ``` +//! use zerocopy::*; +//! +//! let result: SizeError<(), u32> = std::thread::spawn(|| { +//! let source = &mut [0u8, 1, 2][..]; +//! // Try (and fail) to read a `u32` from `source`. +//! u32::read_from_bytes(source).unwrap_err() +//! // Erase the error source. +//! .map_src(drop) +//! }).join().unwrap(); +//! ``` +//! +//! Alternatively, use `.to_string()` to eagerly convert the error into a +//! human-readable message; e.g.: +//! +//! ``` +//! use zerocopy::*; +//! +//! let result: Result<u32, String> = std::thread::spawn(|| { +//! let source = &mut [0u8, 1, 2][..]; +//! // Try (and fail) to read a `u32` from `source`. +//! u32::read_from_bytes(source) +//! // Eagerly render the error message. +//! .map_err(|err| err.to_string()) +//! }).join().unwrap(); +//! ``` +#[cfg(not(no_zerocopy_core_error_1_81_0))] +use core::error::Error; +use core::{ + convert::Infallible, + fmt::{self, Debug, Write}, + ops::Deref, +}; +#[cfg(all(no_zerocopy_core_error_1_81_0, any(feature = "std", test)))] +use std::error::Error; + +use crate::{util::SendSyncPhantomData, KnownLayout, TryFromBytes, Unaligned}; +#[cfg(doc)] +use crate::{FromBytes, Ref}; + +/// Zerocopy's generic error type. +/// +/// Generally speaking, zerocopy's conversions may fail for one of up to three +/// reasons: +/// - [`AlignmentError`]: the conversion source was improperly aligned +/// - [`SizeError`]: the conversion source was of incorrect size +/// - [`ValidityError`]: the conversion source contained invalid data +/// +/// However, not all conversions produce all errors. For instance, +/// [`FromBytes::ref_from_bytes`] may fail due to alignment or size issues, but +/// not validity issues. This generic error type captures these +/// (im)possibilities via parameterization: `A` is parameterized with +/// [`AlignmentError`], `S` is parameterized with [`SizeError`], and `V` is +/// parameterized with [`Infallible`]. +/// +/// Zerocopy never uses this type directly in its API. Rather, we provide three +/// pre-parameterized aliases: +/// - [`CastError`]: the error type of reference conversions +/// - [`TryCastError`]: the error type of fallible reference conversions +/// - [`TryReadError`]: the error type of fallible read conversions +#[derive(PartialEq, Eq, Clone)] +pub enum ConvertError<A, S, V> { + /// The conversion source was improperly aligned. + Alignment(A), + /// The conversion source was of incorrect size. + Size(S), + /// The conversion source contained invalid data. + Validity(V), +} + +impl<Src, Dst: ?Sized + Unaligned, S, V> From<ConvertError<AlignmentError<Src, Dst>, S, V>> + for ConvertError<Infallible, S, V> +{ + /// Infallibly discards the alignment error from this `ConvertError` since + /// `Dst` is unaligned. + /// + /// Since [`Dst: Unaligned`], it is impossible to encounter an alignment + /// error. This method permits discarding that alignment error infallibly + /// and replacing it with [`Infallible`]. + /// + /// [`Dst: Unaligned`]: crate::Unaligned + /// + /// # Examples + /// + /// ``` + /// use core::convert::Infallible; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, KnownLayout, Unaligned, Immutable)] + /// #[repr(C, packed)] + /// struct Bools { + /// one: bool, + /// two: bool, + /// many: [bool], + /// } + /// + /// impl Bools { + /// fn parse(bytes: &[u8]) -> Result<&Bools, AlignedTryCastError<&[u8], Bools>> { + /// // Since `Bools: Unaligned`, we can infallibly discard + /// // the alignment error. + /// Bools::try_ref_from_bytes(bytes).map_err(Into::into) + /// } + /// } + /// ``` + #[inline] + fn from(err: ConvertError<AlignmentError<Src, Dst>, S, V>) -> ConvertError<Infallible, S, V> { + match err { + ConvertError::Alignment(e) => { + #[allow(unreachable_code)] + return ConvertError::Alignment(Infallible::from(e)); + } + ConvertError::Size(e) => ConvertError::Size(e), + ConvertError::Validity(e) => ConvertError::Validity(e), + } + } +} + +impl<A: fmt::Debug, S: fmt::Debug, V: fmt::Debug> fmt::Debug for ConvertError<A, S, V> { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Alignment(e) => f.debug_tuple("Alignment").field(e).finish(), + Self::Size(e) => f.debug_tuple("Size").field(e).finish(), + Self::Validity(e) => f.debug_tuple("Validity").field(e).finish(), + } + } +} + +/// Produces a human-readable error message. +/// +/// The message differs between debug and release builds. When +/// `debug_assertions` are enabled, this message is verbose and includes +/// potentially sensitive information. +impl<A: fmt::Display, S: fmt::Display, V: fmt::Display> fmt::Display for ConvertError<A, S, V> { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Alignment(e) => e.fmt(f), + Self::Size(e) => e.fmt(f), + Self::Validity(e) => e.fmt(f), + } + } +} + +#[cfg(any(not(no_zerocopy_core_error_1_81_0), feature = "std", test))] +#[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.81.0", feature = "std"))))] +impl<A, S, V> Error for ConvertError<A, S, V> +where + A: fmt::Display + fmt::Debug, + S: fmt::Display + fmt::Debug, + V: fmt::Display + fmt::Debug, +{ +} + +/// The error emitted if the conversion source is improperly aligned. +pub struct AlignmentError<Src, Dst: ?Sized> { + /// The source value involved in the conversion. + src: Src, + /// The inner destination type involved in the conversion. + /// + /// INVARIANT: An `AlignmentError` may only be constructed if `Dst`'s + /// alignment requirement is greater than one. + _dst: SendSyncPhantomData<Dst>, +} + +impl<Src, Dst: ?Sized> AlignmentError<Src, Dst> { + /// # Safety + /// + /// The caller must ensure that `Dst`'s alignment requirement is greater + /// than one. + pub(crate) unsafe fn new_unchecked(src: Src) -> Self { + // INVARIANT: The caller guarantees that `Dst`'s alignment requirement + // is greater than one. + Self { src, _dst: SendSyncPhantomData::default() } + } + + /// Produces the source underlying the failed conversion. + #[inline] + pub fn into_src(self) -> Src { + self.src + } + + pub(crate) fn with_src<NewSrc>(self, new_src: NewSrc) -> AlignmentError<NewSrc, Dst> { + // INVARIANT: `with_src` doesn't change the type of `Dst`, so the + // invariant that `Dst`'s alignment requirement is greater than one is + // preserved. + AlignmentError { src: new_src, _dst: SendSyncPhantomData::default() } + } + + /// Maps the source value associated with the conversion error. + /// + /// This can help mitigate [issues with `Send`, `Sync` and `'static` + /// bounds][self#send-sync-and-static]. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::*; + /// + /// let unaligned = Unalign::new(0u16); + /// + /// // Attempt to deref `unaligned`. This might fail with an alignment error. + /// let maybe_n: Result<&u16, AlignmentError<&Unalign<u16>, u16>> = unaligned.try_deref(); + /// + /// // Map the error's source to its address as a usize. + /// let maybe_n: Result<&u16, AlignmentError<usize, u16>> = maybe_n.map_err(|err| { + /// err.map_src(|src| src as *const _ as usize) + /// }); + /// ``` + #[inline] + pub fn map_src<NewSrc>(self, f: impl FnOnce(Src) -> NewSrc) -> AlignmentError<NewSrc, Dst> { + AlignmentError { src: f(self.src), _dst: SendSyncPhantomData::default() } + } + + pub(crate) fn into<S, V>(self) -> ConvertError<Self, S, V> { + ConvertError::Alignment(self) + } + + /// Format extra details for a verbose, human-readable error message. + /// + /// This formatting may include potentially sensitive information. + fn display_verbose_extras(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + where + Src: Deref, + Dst: KnownLayout, + { + #[allow(clippy::as_conversions)] + let addr = self.src.deref() as *const _ as *const (); + let addr_align = 2usize.pow((crate::util::AsAddress::addr(addr)).trailing_zeros()); + + f.write_str("\n\nSource type: ")?; + f.write_str(core::any::type_name::<Src>())?; + + f.write_str("\nSource address: ")?; + addr.fmt(f)?; + f.write_str(" (a multiple of ")?; + addr_align.fmt(f)?; + f.write_str(")")?; + + f.write_str("\nDestination type: ")?; + f.write_str(core::any::type_name::<Dst>())?; + + f.write_str("\nDestination alignment: ")?; + <Dst as KnownLayout>::LAYOUT.align.get().fmt(f)?; + + Ok(()) + } +} + +impl<Src: Clone, Dst: ?Sized> Clone for AlignmentError<Src, Dst> { + #[inline] + fn clone(&self) -> Self { + Self { src: self.src.clone(), _dst: SendSyncPhantomData::default() } + } +} + +impl<Src: PartialEq, Dst: ?Sized> PartialEq for AlignmentError<Src, Dst> { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.src == other.src + } +} + +impl<Src: Eq, Dst: ?Sized> Eq for AlignmentError<Src, Dst> {} + +impl<Src, Dst: ?Sized + Unaligned> From<AlignmentError<Src, Dst>> for Infallible { + #[inline(always)] + fn from(_: AlignmentError<Src, Dst>) -> Infallible { + // SAFETY: `AlignmentError`s can only be constructed when `Dst`'s + // alignment requirement is greater than one. In this block, `Dst: + // Unaligned`, which means that its alignment requirement is equal to + // one. Thus, it's not possible to reach here at runtime. + unsafe { core::hint::unreachable_unchecked() } + } +} + +#[cfg(test)] +impl<Src, Dst> AlignmentError<Src, Dst> { + // A convenience constructor so that test code doesn't need to write + // `unsafe`. + fn new_checked(src: Src) -> AlignmentError<Src, Dst> { + assert_ne!(core::mem::align_of::<Dst>(), 1); + // SAFETY: The preceding assertion guarantees that `Dst`'s alignment + // requirement is greater than one. + unsafe { AlignmentError::new_unchecked(src) } + } +} + +impl<Src, Dst: ?Sized> fmt::Debug for AlignmentError<Src, Dst> { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AlignmentError").finish() + } +} + +/// Produces a human-readable error message. +/// +/// The message differs between debug and release builds. When +/// `debug_assertions` are enabled, this message is verbose and includes +/// potentially sensitive information. +impl<Src, Dst: ?Sized> fmt::Display for AlignmentError<Src, Dst> +where + Src: Deref, + Dst: KnownLayout, +{ + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("The conversion failed because the address of the source is not a multiple of the alignment of the destination type.")?; + + if cfg!(debug_assertions) { + self.display_verbose_extras(f) + } else { + Ok(()) + } + } +} + +#[cfg(any(not(no_zerocopy_core_error_1_81_0), feature = "std", test))] +#[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.81.0", feature = "std"))))] +impl<Src, Dst: ?Sized> Error for AlignmentError<Src, Dst> +where + Src: Deref, + Dst: KnownLayout, +{ +} + +impl<Src, Dst: ?Sized, S, V> From<AlignmentError<Src, Dst>> + for ConvertError<AlignmentError<Src, Dst>, S, V> +{ + #[inline(always)] + fn from(err: AlignmentError<Src, Dst>) -> Self { + Self::Alignment(err) + } +} + +/// The error emitted if the conversion source is of incorrect size. +pub struct SizeError<Src, Dst: ?Sized> { + /// The source value involved in the conversion. + src: Src, + /// The inner destination type involved in the conversion. + _dst: SendSyncPhantomData<Dst>, +} + +impl<Src, Dst: ?Sized> SizeError<Src, Dst> { + pub(crate) fn new(src: Src) -> Self { + Self { src, _dst: SendSyncPhantomData::default() } + } + + /// Produces the source underlying the failed conversion. + #[inline] + pub fn into_src(self) -> Src { + self.src + } + + /// Sets the source value associated with the conversion error. + pub(crate) fn with_src<NewSrc>(self, new_src: NewSrc) -> SizeError<NewSrc, Dst> { + SizeError { src: new_src, _dst: SendSyncPhantomData::default() } + } + + /// Maps the source value associated with the conversion error. + /// + /// This can help mitigate [issues with `Send`, `Sync` and `'static` + /// bounds][self#send-sync-and-static]. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::*; + /// + /// let source: [u8; 3] = [0, 1, 2]; + /// + /// // Try to read a `u32` from `source`. This will fail because there are insufficient + /// // bytes in `source`. + /// let maybe_u32: Result<u32, SizeError<&[u8], u32>> = u32::read_from_bytes(&source[..]); + /// + /// // Map the error's source to its size. + /// let maybe_u32: Result<u32, SizeError<usize, u32>> = maybe_u32.map_err(|err| { + /// err.map_src(|src| src.len()) + /// }); + /// ``` + #[inline] + pub fn map_src<NewSrc>(self, f: impl FnOnce(Src) -> NewSrc) -> SizeError<NewSrc, Dst> { + SizeError { src: f(self.src), _dst: SendSyncPhantomData::default() } + } + + /// Sets the destination type associated with the conversion error. + pub(crate) fn with_dst<NewDst: ?Sized>(self) -> SizeError<Src, NewDst> { + SizeError { src: self.src, _dst: SendSyncPhantomData::default() } + } + + /// Converts the error into a general [`ConvertError`]. + pub(crate) fn into<A, V>(self) -> ConvertError<A, Self, V> { + ConvertError::Size(self) + } + + /// Format extra details for a verbose, human-readable error message. + /// + /// This formatting may include potentially sensitive information. + fn display_verbose_extras(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + where + Src: Deref, + Dst: KnownLayout, + { + // include the source type + f.write_str("\nSource type: ")?; + f.write_str(core::any::type_name::<Src>())?; + + // include the source.deref() size + let src_size = core::mem::size_of_val(&*self.src); + f.write_str("\nSource size: ")?; + src_size.fmt(f)?; + f.write_str(" byte")?; + if src_size != 1 { + f.write_char('s')?; + } + + // if `Dst` is `Sized`, include the `Dst` size + if let crate::SizeInfo::Sized { size } = Dst::LAYOUT.size_info { + f.write_str("\nDestination size: ")?; + size.fmt(f)?; + f.write_str(" byte")?; + if size != 1 { + f.write_char('s')?; + } + } + + // include the destination type + f.write_str("\nDestination type: ")?; + f.write_str(core::any::type_name::<Dst>())?; + + Ok(()) + } +} + +impl<Src: Clone, Dst: ?Sized> Clone for SizeError<Src, Dst> { + #[inline] + fn clone(&self) -> Self { + Self { src: self.src.clone(), _dst: SendSyncPhantomData::default() } + } +} + +impl<Src: PartialEq, Dst: ?Sized> PartialEq for SizeError<Src, Dst> { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.src == other.src + } +} + +impl<Src: Eq, Dst: ?Sized> Eq for SizeError<Src, Dst> {} + +impl<Src, Dst: ?Sized> fmt::Debug for SizeError<Src, Dst> { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SizeError").finish() + } +} + +/// Produces a human-readable error message. +/// +/// The message differs between debug and release builds. When +/// `debug_assertions` are enabled, this message is verbose and includes +/// potentially sensitive information. +impl<Src, Dst: ?Sized> fmt::Display for SizeError<Src, Dst> +where + Src: Deref, + Dst: KnownLayout, +{ + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("The conversion failed because the source was incorrectly sized to complete the conversion into the destination type.")?; + if cfg!(debug_assertions) { + f.write_str("\n")?; + self.display_verbose_extras(f)?; + } + Ok(()) + } +} + +#[cfg(any(not(no_zerocopy_core_error_1_81_0), feature = "std", test))] +#[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.81.0", feature = "std"))))] +impl<Src, Dst: ?Sized> Error for SizeError<Src, Dst> +where + Src: Deref, + Dst: KnownLayout, +{ +} + +impl<Src, Dst: ?Sized, A, V> From<SizeError<Src, Dst>> for ConvertError<A, SizeError<Src, Dst>, V> { + #[inline(always)] + fn from(err: SizeError<Src, Dst>) -> Self { + Self::Size(err) + } +} + +/// The error emitted if the conversion source contains invalid data. +pub struct ValidityError<Src, Dst: ?Sized + TryFromBytes> { + /// The source value involved in the conversion. + pub(crate) src: Src, + /// The inner destination type involved in the conversion. + _dst: SendSyncPhantomData<Dst>, +} + +impl<Src, Dst: ?Sized + TryFromBytes> ValidityError<Src, Dst> { + pub(crate) fn new(src: Src) -> Self { + Self { src, _dst: SendSyncPhantomData::default() } + } + + /// Produces the source underlying the failed conversion. + #[inline] + pub fn into_src(self) -> Src { + self.src + } + + /// Maps the source value associated with the conversion error. + /// + /// This can help mitigate [issues with `Send`, `Sync` and `'static` + /// bounds][self#send-sync-and-static]. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::*; + /// + /// let source: u8 = 42; + /// + /// // Try to transmute the `source` to a `bool`. This will fail. + /// let maybe_bool: Result<bool, ValidityError<u8, bool>> = try_transmute!(source); + /// + /// // Drop the error's source. + /// let maybe_bool: Result<bool, ValidityError<(), bool>> = maybe_bool.map_err(|err| { + /// err.map_src(drop) + /// }); + /// ``` + #[inline] + pub fn map_src<NewSrc>(self, f: impl FnOnce(Src) -> NewSrc) -> ValidityError<NewSrc, Dst> { + ValidityError { src: f(self.src), _dst: SendSyncPhantomData::default() } + } + + /// Converts the error into a general [`ConvertError`]. + pub(crate) fn into<A, S>(self) -> ConvertError<A, S, Self> { + ConvertError::Validity(self) + } + + /// Format extra details for a verbose, human-readable error message. + /// + /// This formatting may include potentially sensitive information. + fn display_verbose_extras(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + where + Dst: KnownLayout, + { + f.write_str("Destination type: ")?; + f.write_str(core::any::type_name::<Dst>())?; + Ok(()) + } +} + +impl<Src: Clone, Dst: ?Sized + TryFromBytes> Clone for ValidityError<Src, Dst> { + #[inline] + fn clone(&self) -> Self { + Self { src: self.src.clone(), _dst: SendSyncPhantomData::default() } + } +} + +// SAFETY: `ValidityError` contains a single `Self::Inner = Src`, and no other +// non-ZST fields. `map` passes ownership of `self`'s sole `Self::Inner` to `f`. +unsafe impl<Src, NewSrc, Dst> crate::pointer::TryWithError<NewSrc> + for crate::ValidityError<Src, Dst> +where + Dst: TryFromBytes + ?Sized, +{ + type Inner = Src; + type Mapped = crate::ValidityError<NewSrc, Dst>; + #[inline] + fn map<F: FnOnce(Src) -> NewSrc>(self, f: F) -> Self::Mapped { + self.map_src(f) + } +} + +impl<Src: PartialEq, Dst: ?Sized + TryFromBytes> PartialEq for ValidityError<Src, Dst> { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.src == other.src + } +} + +impl<Src: Eq, Dst: ?Sized + TryFromBytes> Eq for ValidityError<Src, Dst> {} + +impl<Src, Dst: ?Sized + TryFromBytes> fmt::Debug for ValidityError<Src, Dst> { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ValidityError").finish() + } +} + +/// Produces a human-readable error message. +/// +/// The message differs between debug and release builds. When +/// `debug_assertions` are enabled, this message is verbose and includes +/// potentially sensitive information. +impl<Src, Dst: ?Sized> fmt::Display for ValidityError<Src, Dst> +where + Dst: KnownLayout + TryFromBytes, +{ + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("The conversion failed because the source bytes are not a valid value of the destination type.")?; + if cfg!(debug_assertions) { + f.write_str("\n\n")?; + self.display_verbose_extras(f)?; + } + Ok(()) + } +} + +#[cfg(any(not(no_zerocopy_core_error_1_81_0), feature = "std", test))] +#[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.81.0", feature = "std"))))] +impl<Src, Dst: ?Sized> Error for ValidityError<Src, Dst> where Dst: KnownLayout + TryFromBytes {} + +impl<Src, Dst: ?Sized + TryFromBytes, A, S> From<ValidityError<Src, Dst>> + for ConvertError<A, S, ValidityError<Src, Dst>> +{ + #[inline(always)] + fn from(err: ValidityError<Src, Dst>) -> Self { + Self::Validity(err) + } +} + +/// The error type of reference conversions. +/// +/// Reference conversions, like [`FromBytes::ref_from_bytes`] may emit +/// [alignment](AlignmentError) and [size](SizeError) errors. +// Bounds on generic parameters are not enforced in type aliases, but they do +// appear in rustdoc. +#[allow(type_alias_bounds)] +pub type CastError<Src, Dst: ?Sized> = + ConvertError<AlignmentError<Src, Dst>, SizeError<Src, Dst>, Infallible>; + +impl<Src, Dst: ?Sized> CastError<Src, Dst> { + /// Produces the source underlying the failed conversion. + #[inline] + pub fn into_src(self) -> Src { + match self { + Self::Alignment(e) => e.src, + Self::Size(e) => e.src, + Self::Validity(i) => match i {}, + } + } + + /// Sets the source value associated with the conversion error. + pub(crate) fn with_src<NewSrc>(self, new_src: NewSrc) -> CastError<NewSrc, Dst> { + match self { + Self::Alignment(e) => CastError::Alignment(e.with_src(new_src)), + Self::Size(e) => CastError::Size(e.with_src(new_src)), + Self::Validity(i) => match i {}, + } + } + + /// Maps the source value associated with the conversion error. + /// + /// This can help mitigate [issues with `Send`, `Sync` and `'static` + /// bounds][self#send-sync-and-static]. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::*; + /// + /// let source: [u8; 3] = [0, 1, 2]; + /// + /// // Try to read a `u32` from `source`. This will fail because there are insufficient + /// // bytes in `source`. + /// let maybe_u32: Result<&u32, CastError<&[u8], u32>> = u32::ref_from_bytes(&source[..]); + /// + /// // Map the error's source to its size and address. + /// let maybe_u32: Result<&u32, CastError<(usize, usize), u32>> = maybe_u32.map_err(|err| { + /// err.map_src(|src| (src.len(), src.as_ptr() as usize)) + /// }); + /// ``` + #[inline] + pub fn map_src<NewSrc>(self, f: impl FnOnce(Src) -> NewSrc) -> CastError<NewSrc, Dst> { + match self { + Self::Alignment(e) => CastError::Alignment(e.map_src(f)), + Self::Size(e) => CastError::Size(e.map_src(f)), + Self::Validity(i) => match i {}, + } + } + + /// Converts the error into a general [`ConvertError`]. + pub(crate) fn into(self) -> TryCastError<Src, Dst> + where + Dst: TryFromBytes, + { + match self { + Self::Alignment(e) => TryCastError::Alignment(e), + Self::Size(e) => TryCastError::Size(e), + Self::Validity(i) => match i {}, + } + } +} + +// SAFETY: `CastError` is either a single `AlignmentError` or a single +// `SizeError`. In either case, it contains a single `Self::Inner = Src`, and no +// other non-ZST fields. `map` passes ownership of `self`'s sole `Self::Inner` +// to `f`. +unsafe impl<Src, NewSrc, Dst> crate::pointer::TryWithError<NewSrc> for crate::CastError<Src, Dst> +where + Dst: ?Sized, +{ + type Inner = Src; + type Mapped = crate::CastError<NewSrc, Dst>; + + #[inline] + fn map<F: FnOnce(Src) -> NewSrc>(self, f: F) -> Self::Mapped { + self.map_src(f) + } +} + +impl<Src, Dst: ?Sized + Unaligned> From<CastError<Src, Dst>> for SizeError<Src, Dst> { + /// Infallibly extracts the [`SizeError`] from this `CastError` since `Dst` + /// is unaligned. + /// + /// Since [`Dst: Unaligned`], it is impossible to encounter an alignment + /// error, and so the only error that can be encountered at runtime is a + /// [`SizeError`]. This method permits extracting that `SizeError` + /// infallibly. + /// + /// [`Dst: Unaligned`]: crate::Unaligned + /// + /// # Examples + /// + /// ```rust + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] + /// #[repr(C)] + /// struct UdpHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] + /// #[repr(C, packed)] + /// struct UdpPacket { + /// header: UdpHeader, + /// body: [u8], + /// } + /// + /// impl UdpPacket { + /// pub fn parse(bytes: &[u8]) -> Result<&UdpPacket, SizeError<&[u8], UdpPacket>> { + /// // Since `UdpPacket: Unaligned`, we can map the `CastError` to a `SizeError`. + /// UdpPacket::ref_from_bytes(bytes).map_err(Into::into) + /// } + /// } + /// ``` + #[inline(always)] + fn from(err: CastError<Src, Dst>) -> SizeError<Src, Dst> { + match err { + #[allow(unreachable_code)] + CastError::Alignment(e) => match Infallible::from(e) {}, + CastError::Size(e) => e, + CastError::Validity(i) => match i {}, + } + } +} + +/// The error type of fallible reference conversions. +/// +/// Fallible reference conversions, like [`TryFromBytes::try_ref_from_bytes`] +/// may emit [alignment](AlignmentError), [size](SizeError), and +/// [validity](ValidityError) errors. +// Bounds on generic parameters are not enforced in type aliases, but they do +// appear in rustdoc. +#[allow(type_alias_bounds)] +pub type TryCastError<Src, Dst: ?Sized + TryFromBytes> = + ConvertError<AlignmentError<Src, Dst>, SizeError<Src, Dst>, ValidityError<Src, Dst>>; + +// FIXME(#1139): Remove the `TryFromBytes` here and in other downstream +// locations (all the way to `ValidityError`) if we determine it's not necessary +// for rich validity errors. +impl<Src, Dst: ?Sized + TryFromBytes> TryCastError<Src, Dst> { + /// Produces the source underlying the failed conversion. + #[inline] + pub fn into_src(self) -> Src { + match self { + Self::Alignment(e) => e.src, + Self::Size(e) => e.src, + Self::Validity(e) => e.src, + } + } + + /// Maps the source value associated with the conversion error. + /// + /// This can help mitigate [issues with `Send`, `Sync` and `'static` + /// bounds][self#send-sync-and-static]. + /// + /// # Examples + /// + /// ``` + /// use core::num::NonZeroU32; + /// use zerocopy::*; + /// + /// let source: [u8; 3] = [0, 0, 0]; + /// + /// // Try to read a `NonZeroU32` from `source`. + /// let maybe_u32: Result<&NonZeroU32, TryCastError<&[u8], NonZeroU32>> + /// = NonZeroU32::try_ref_from_bytes(&source[..]); + /// + /// // Map the error's source to its size and address. + /// let maybe_u32: Result<&NonZeroU32, TryCastError<(usize, usize), NonZeroU32>> = + /// maybe_u32.map_err(|err| { + /// err.map_src(|src| (src.len(), src.as_ptr() as usize)) + /// }); + /// ``` + #[inline] + pub fn map_src<NewSrc>(self, f: impl FnOnce(Src) -> NewSrc) -> TryCastError<NewSrc, Dst> { + match self { + Self::Alignment(e) => TryCastError::Alignment(e.map_src(f)), + Self::Size(e) => TryCastError::Size(e.map_src(f)), + Self::Validity(e) => TryCastError::Validity(e.map_src(f)), + } + } +} + +impl<Src, Dst: ?Sized + TryFromBytes> From<CastError<Src, Dst>> for TryCastError<Src, Dst> { + #[inline] + fn from(value: CastError<Src, Dst>) -> Self { + match value { + CastError::Alignment(e) => Self::Alignment(e), + CastError::Size(e) => Self::Size(e), + CastError::Validity(i) => match i {}, + } + } +} + +/// The error type of fallible read-conversions. +/// +/// Fallible read-conversions, like [`TryFromBytes::try_read_from_bytes`] may +/// emit [size](SizeError) and [validity](ValidityError) errors, but not +/// alignment errors. +// Bounds on generic parameters are not enforced in type aliases, but they do +// appear in rustdoc. +#[allow(type_alias_bounds)] +pub type TryReadError<Src, Dst: ?Sized + TryFromBytes> = + ConvertError<Infallible, SizeError<Src, Dst>, ValidityError<Src, Dst>>; + +impl<Src, Dst: ?Sized + TryFromBytes> TryReadError<Src, Dst> { + /// Produces the source underlying the failed conversion. + #[inline] + pub fn into_src(self) -> Src { + match self { + Self::Alignment(i) => match i {}, + Self::Size(e) => e.src, + Self::Validity(e) => e.src, + } + } + + /// Maps the source value associated with the conversion error. + /// + /// This can help mitigate [issues with `Send`, `Sync` and `'static` + /// bounds][self#send-sync-and-static]. + /// + /// # Examples + /// + /// ``` + /// use core::num::NonZeroU32; + /// use zerocopy::*; + /// + /// let source: [u8; 3] = [0, 0, 0]; + /// + /// // Try to read a `NonZeroU32` from `source`. + /// let maybe_u32: Result<NonZeroU32, TryReadError<&[u8], NonZeroU32>> + /// = NonZeroU32::try_read_from_bytes(&source[..]); + /// + /// // Map the error's source to its size. + /// let maybe_u32: Result<NonZeroU32, TryReadError<usize, NonZeroU32>> = + /// maybe_u32.map_err(|err| { + /// err.map_src(|src| src.len()) + /// }); + /// ``` + #[inline] + pub fn map_src<NewSrc>(self, f: impl FnOnce(Src) -> NewSrc) -> TryReadError<NewSrc, Dst> { + match self { + Self::Alignment(i) => match i {}, + Self::Size(e) => TryReadError::Size(e.map_src(f)), + Self::Validity(e) => TryReadError::Validity(e.map_src(f)), + } + } +} + +/// The error type of well-aligned, fallible casts. +/// +/// This is like [`TryCastError`], but for casts that are always well-aligned. +/// It is identical to `TryCastError`, except that its alignment error is +/// [`Infallible`]. +/// +/// As of this writing, none of zerocopy's API produces this error directly. +/// However, it is useful since it permits users to infallibly discard alignment +/// errors when they can prove statically that alignment errors are impossible. +/// +/// # Examples +/// +/// ``` +/// use core::convert::Infallible; +/// use zerocopy::*; +/// # use zerocopy_derive::*; +/// +/// #[derive(TryFromBytes, KnownLayout, Unaligned, Immutable)] +/// #[repr(C, packed)] +/// struct Bools { +/// one: bool, +/// two: bool, +/// many: [bool], +/// } +/// +/// impl Bools { +/// fn parse(bytes: &[u8]) -> Result<&Bools, AlignedTryCastError<&[u8], Bools>> { +/// // Since `Bools: Unaligned`, we can infallibly discard +/// // the alignment error. +/// Bools::try_ref_from_bytes(bytes).map_err(Into::into) +/// } +/// } +/// ``` +#[allow(type_alias_bounds)] +pub type AlignedTryCastError<Src, Dst: ?Sized + TryFromBytes> = + ConvertError<Infallible, SizeError<Src, Dst>, ValidityError<Src, Dst>>; + +/// The error type of a failed allocation. +/// +/// This type is intended to be deprecated in favor of the standard library's +/// [`AllocError`] type once it is stabilized. When that happens, this type will +/// be replaced by a type alias to the standard library type. We do not intend +/// to treat this as a breaking change; users who wish to avoid breakage should +/// avoid writing code which assumes that this is *not* such an alias. For +/// example, implementing the same trait for both types will result in an impl +/// conflict once this type is an alias. +/// +/// [`AllocError`]: https://doc.rust-lang.org/alloc/alloc/struct.AllocError.html +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct AllocError; + +#[cfg(test)] +mod tests { + use core::convert::Infallible; + + use super::*; + + #[test] + fn test_send_sync() { + // Test that all error types are `Send + Sync` even if `Dst: !Send + + // !Sync`. + + #[allow(dead_code)] + fn is_send_sync<T: Send + Sync>(_t: T) {} + + #[allow(dead_code)] + fn alignment_err_is_send_sync<Src: Send + Sync, Dst>(err: AlignmentError<Src, Dst>) { + is_send_sync(err) + } + + #[allow(dead_code)] + fn size_err_is_send_sync<Src: Send + Sync, Dst>(err: SizeError<Src, Dst>) { + is_send_sync(err) + } + + #[allow(dead_code)] + fn validity_err_is_send_sync<Src: Send + Sync, Dst: TryFromBytes>( + err: ValidityError<Src, Dst>, + ) { + is_send_sync(err) + } + + #[allow(dead_code)] + fn convert_error_is_send_sync<Src: Send + Sync, Dst: TryFromBytes>( + err: ConvertError< + AlignmentError<Src, Dst>, + SizeError<Src, Dst>, + ValidityError<Src, Dst>, + >, + ) { + is_send_sync(err) + } + } + + #[test] + fn test_eq_partial_eq_clone() { + // Test that all error types implement `Eq`, `PartialEq` + // and `Clone` if src does + // even if `Dst: !Eq`, `!PartialEq`, `!Clone`. + + #[allow(dead_code)] + fn is_eq_partial_eq_clone<T: Eq + PartialEq + Clone>(_t: T) {} + + #[allow(dead_code)] + fn alignment_err_is_eq_partial_eq_clone<Src: Eq + PartialEq + Clone, Dst>( + err: AlignmentError<Src, Dst>, + ) { + is_eq_partial_eq_clone(err) + } + + #[allow(dead_code)] + fn size_err_is_eq_partial_eq_clone<Src: Eq + PartialEq + Clone, Dst>( + err: SizeError<Src, Dst>, + ) { + is_eq_partial_eq_clone(err) + } + + #[allow(dead_code)] + fn validity_err_is_eq_partial_eq_clone<Src: Eq + PartialEq + Clone, Dst: TryFromBytes>( + err: ValidityError<Src, Dst>, + ) { + is_eq_partial_eq_clone(err) + } + + #[allow(dead_code)] + fn convert_error_is_eq_partial_eq_clone<Src: Eq + PartialEq + Clone, Dst: TryFromBytes>( + err: ConvertError< + AlignmentError<Src, Dst>, + SizeError<Src, Dst>, + ValidityError<Src, Dst>, + >, + ) { + is_eq_partial_eq_clone(err) + } + } + + #[test] + fn alignment_display() { + #[repr(C, align(128))] + struct Aligned { + bytes: [u8; 128], + } + + impl_known_layout!(elain::Align::<8>); + + let aligned = Aligned { bytes: [0; 128] }; + + let bytes = &aligned.bytes[1..]; + let addr = crate::util::AsAddress::addr(bytes); + assert_eq!( + AlignmentError::<_, elain::Align::<8>>::new_checked(bytes).to_string(), + format!("The conversion failed because the address of the source is not a multiple of the alignment of the destination type.\n\ + \nSource type: &[u8]\ + \nSource address: 0x{:x} (a multiple of 1)\ + \nDestination type: elain::Align<8>\ + \nDestination alignment: 8", addr) + ); + + let bytes = &aligned.bytes[2..]; + let addr = crate::util::AsAddress::addr(bytes); + assert_eq!( + AlignmentError::<_, elain::Align::<8>>::new_checked(bytes).to_string(), + format!("The conversion failed because the address of the source is not a multiple of the alignment of the destination type.\n\ + \nSource type: &[u8]\ + \nSource address: 0x{:x} (a multiple of 2)\ + \nDestination type: elain::Align<8>\ + \nDestination alignment: 8", addr) + ); + + let bytes = &aligned.bytes[3..]; + let addr = crate::util::AsAddress::addr(bytes); + assert_eq!( + AlignmentError::<_, elain::Align::<8>>::new_checked(bytes).to_string(), + format!("The conversion failed because the address of the source is not a multiple of the alignment of the destination type.\n\ + \nSource type: &[u8]\ + \nSource address: 0x{:x} (a multiple of 1)\ + \nDestination type: elain::Align<8>\ + \nDestination alignment: 8", addr) + ); + + let bytes = &aligned.bytes[4..]; + let addr = crate::util::AsAddress::addr(bytes); + assert_eq!( + AlignmentError::<_, elain::Align::<8>>::new_checked(bytes).to_string(), + format!("The conversion failed because the address of the source is not a multiple of the alignment of the destination type.\n\ + \nSource type: &[u8]\ + \nSource address: 0x{:x} (a multiple of 4)\ + \nDestination type: elain::Align<8>\ + \nDestination alignment: 8", addr) + ); + } + + #[test] + fn size_display() { + assert_eq!( + SizeError::<_, [u8]>::new(&[0u8; 2][..]).to_string(), + "The conversion failed because the source was incorrectly sized to complete the conversion into the destination type.\n\ + \nSource type: &[u8]\ + \nSource size: 2 bytes\ + \nDestination type: [u8]" + ); + + assert_eq!( + SizeError::<_, [u8; 2]>::new(&[0u8; 1][..]).to_string(), + "The conversion failed because the source was incorrectly sized to complete the conversion into the destination type.\n\ + \nSource type: &[u8]\ + \nSource size: 1 byte\ + \nDestination size: 2 bytes\ + \nDestination type: [u8; 2]" + ); + } + + #[test] + fn validity_display() { + assert_eq!( + ValidityError::<_, bool>::new(&[2u8; 1][..]).to_string(), + "The conversion failed because the source bytes are not a valid value of the destination type.\n\ + \n\ + Destination type: bool" + ); + } + + #[test] + fn test_convert_error_debug() { + let err: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = ConvertError::Alignment(AlignmentError::new_checked(&[0u8])); + assert_eq!(format!("{:?}", err), "Alignment(AlignmentError)"); + + let err: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = ConvertError::Size(SizeError::new(&[0u8])); + assert_eq!(format!("{:?}", err), "Size(SizeError)"); + + let err: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = ConvertError::Validity(ValidityError::new(&[0u8])); + assert_eq!(format!("{:?}", err), "Validity(ValidityError)"); + } + + #[test] + fn test_convert_error_from_unaligned() { + // u8 is Unaligned + let err: ConvertError< + AlignmentError<&[u8], u8>, + SizeError<&[u8], u8>, + ValidityError<&[u8], bool>, + > = ConvertError::Size(SizeError::new(&[0u8])); + let converted: ConvertError<Infallible, SizeError<&[u8], u8>, ValidityError<&[u8], bool>> = + ConvertError::from(err); + match converted { + ConvertError::Size(_) => {} + _ => panic!("Expected Size error"), + } + } + + #[test] + fn test_alignment_error_display_debug() { + let err: AlignmentError<&[u8], u16> = AlignmentError::new_checked(&[0u8]); + assert!(format!("{:?}", err).contains("AlignmentError")); + assert!(format!("{}", err).contains("address of the source is not a multiple")); + } + + #[test] + fn test_size_error_display_debug() { + let err: SizeError<&[u8], u16> = SizeError::new(&[0u8]); + assert!(format!("{:?}", err).contains("SizeError")); + assert!(format!("{}", err).contains("source was incorrectly sized")); + } + + #[test] + fn test_validity_error_display_debug() { + let err: ValidityError<&[u8], bool> = ValidityError::new(&[0u8]); + assert!(format!("{:?}", err).contains("ValidityError")); + assert!(format!("{}", err).contains("source bytes are not a valid value")); + } + + #[test] + fn test_convert_error_display_debug_more() { + let err: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = ConvertError::Alignment(AlignmentError::new_checked(&[0u8])); + assert!(format!("{}", err).contains("address of the source is not a multiple")); + + let err: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = ConvertError::Size(SizeError::new(&[0u8])); + assert!(format!("{}", err).contains("source was incorrectly sized")); + + let err: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = ConvertError::Validity(ValidityError::new(&[0u8])); + assert!(format!("{}", err).contains("source bytes are not a valid value")); + } + + #[test] + fn test_alignment_error_methods() { + let err: AlignmentError<&[u8], u16> = AlignmentError::new_checked(&[0u8]); + + // into_src + let src = err.clone().into_src(); + assert_eq!(src, &[0u8]); + + // into + let converted: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = err.clone().into(); + match converted { + ConvertError::Alignment(_) => {} + _ => panic!("Expected Alignment error"), + } + + // clone + let cloned = err.clone(); + assert_eq!(err, cloned); + + // eq + assert_eq!(err, cloned); + let err2: AlignmentError<&[u8], u16> = AlignmentError::new_checked(&[1u8]); + assert_ne!(err, err2); + } + + #[test] + fn test_convert_error_from_unaligned_variants() { + // u8 is Unaligned + let err: ConvertError< + AlignmentError<&[u8], u8>, + SizeError<&[u8], u8>, + ValidityError<&[u8], bool>, + > = ConvertError::Validity(ValidityError::new(&[0u8])); + let converted: ConvertError<Infallible, SizeError<&[u8], u8>, ValidityError<&[u8], bool>> = + ConvertError::from(err); + match converted { + ConvertError::Validity(_) => {} + _ => panic!("Expected Validity error"), + } + + let err: ConvertError< + AlignmentError<&[u8], u8>, + SizeError<&[u8], u8>, + ValidityError<&[u8], bool>, + > = ConvertError::Size(SizeError::new(&[0u8])); + let converted: ConvertError<Infallible, SizeError<&[u8], u8>, ValidityError<&[u8], bool>> = + ConvertError::from(err); + match converted { + ConvertError::Size(_) => {} + _ => panic!("Expected Size error"), + } + } +} diff --git a/rust/zerocopy/src/impls.rs b/rust/zerocopy/src/impls.rs new file mode 100644 index 000000000000..22fd6c3d5d94 --- /dev/null +++ b/rust/zerocopy/src/impls.rs @@ -0,0 +1,2389 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2024 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License <LICENSE-BSD or +// https://opensource.org/license/bsd-2-clause>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use core::{ + cell::{Cell, UnsafeCell}, + mem::MaybeUninit as CoreMaybeUninit, + ptr::NonNull, +}; + +use super::*; +use crate::pointer::cast::{CastSizedExact, CastUnsized}; + +// SAFETY: Per the reference [1], "the unit tuple (`()`) ... is guaranteed as a +// zero-sized type to have a size of 0 and an alignment of 1." +// - `Immutable`: `()` self-evidently does not contain any `UnsafeCell`s. +// - `TryFromBytes` (with no validator), `FromZeros`, `FromBytes`: There is only +// one possible sequence of 0 bytes, and `()` is inhabited. +// - `IntoBytes`: Since `()` has size 0, it contains no padding bytes. +// - `Unaligned`: `()` has alignment 1. +// +// [1] https://doc.rust-lang.org/1.81.0/reference/type-layout.html#tuple-layout +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!((): Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_unaligned!(()); +}; + +// SAFETY: +// - `Immutable`: These types self-evidently do not contain any `UnsafeCell`s. +// - `TryFromBytes` (with no validator), `FromZeros`, `FromBytes`: all bit +// patterns are valid for numeric types [1] +// - `IntoBytes`: numeric types have no padding bytes [1] +// - `Unaligned` (`u8` and `i8` only): The reference [2] specifies the size of +// `u8` and `i8` as 1 byte. We also know that: +// - Alignment is >= 1 [3] +// - Size is an integer multiple of alignment [4] +// - The only value >= 1 for which 1 is an integer multiple is 1 Therefore, +// the only possible alignment for `u8` and `i8` is 1. +// +// [1] Per https://doc.rust-lang.org/1.81.0/reference/types/numeric.html#bit-validity: +// +// For every numeric type, `T`, the bit validity of `T` is equivalent to +// the bit validity of `[u8; size_of::<T>()]`. An uninitialized byte is +// not a valid `u8`. +// +// [2] https://doc.rust-lang.org/1.81.0/reference/type-layout.html#primitive-data-layout +// +// [3] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#size-and-alignment: +// +// Alignment is measured in bytes, and must be at least 1. +// +// [4] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#size-and-alignment: +// +// The size of a value is always a multiple of its alignment. +// +// FIXME(#278): Once we've updated the trait docs to refer to `u8`s rather than +// bits or bytes, update this comment, especially the reference to [1]. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(u8: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + unsafe_impl!(i8: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_unaligned!(u8, i8); + unsafe_impl!(u16: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(i16: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(u32: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(i32: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(u64: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(i64: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(u128: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(i128: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(usize: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(isize: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(f32: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(f64: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + #[cfg(feature = "float-nightly")] + unsafe_impl!(#[cfg_attr(doc_cfg, doc(cfg(feature = "float-nightly")))] f16: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + #[cfg(feature = "float-nightly")] + unsafe_impl!(#[cfg_attr(doc_cfg, doc(cfg(feature = "float-nightly")))] f128: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); +}; + +// SAFETY: +// - `Immutable`: `bool` self-evidently does not contain any `UnsafeCell`s. +// - `FromZeros`: Valid since "[t]he value false has the bit pattern 0x00" [1]. +// - `IntoBytes`: Since "the boolean type has a size and alignment of 1 each" +// and "The value false has the bit pattern 0x00 and the value true has the +// bit pattern 0x01" [1]. Thus, the only byte of the bool is always +// initialized. +// - `Unaligned`: Per the reference [1], "[a]n object with the boolean type has +// a size and alignment of 1 each." +// +// [1] https://doc.rust-lang.org/1.81.0/reference/types/boolean.html +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl!(bool: Immutable, FromZeros, IntoBytes, Unaligned) }; +assert_unaligned!(bool); + +// SAFETY: The impl must only return `true` for its argument if the original +// `Maybe<bool>` refers to a valid `bool`. We only return true if the `u8` value +// is 0 or 1, and both of these are valid values for `bool` [1]. +// +// [1] Per https://doc.rust-lang.org/1.81.0/reference/types/boolean.html: +// +// The value false has the bit pattern 0x00 and the value true has the bit +// pattern 0x01. +const _: () = unsafe { + unsafe_impl!(=> TryFromBytes for bool; |byte| { + let byte = byte.transmute_with::<u8, invariant::Valid, CastSizedExact, BecauseImmutable>(); + *byte.unaligned_as_ref() < 2 + }) +}; + +// SAFETY: +// - `Immutable`: `char` self-evidently does not contain any `UnsafeCell`s. +// - `FromZeros`: Per reference [1], "[a] value of type char is a Unicode scalar +// value (i.e. a code point that is not a surrogate), represented as a 32-bit +// unsigned word in the 0x0000 to 0xD7FF or 0xE000 to 0x10FFFF range" which +// contains 0x0000. +// - `IntoBytes`: `char` is per reference [1] "represented as a 32-bit unsigned +// word" (`u32`) which is `IntoBytes`. Note that unlike `u32`, not all bit +// patterns are valid for `char`. +// +// [1] https://doc.rust-lang.org/1.81.0/reference/types/textual.html +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl!(char: Immutable, FromZeros, IntoBytes) }; + +// SAFETY: The impl must only return `true` for its argument if the original +// `Maybe<char>` refers to a valid `char`. `char::from_u32` guarantees that it +// returns `None` if its input is not a valid `char` [1]. +// +// [1] Per https://doc.rust-lang.org/core/primitive.char.html#method.from_u32: +// +// `from_u32()` will return `None` if the input is not a valid value for a +// `char`. +const _: () = unsafe { + unsafe_impl!(=> TryFromBytes for char; |c| { + let c = c.transmute_with::<Unalign<u32>, invariant::Valid, CastSizedExact, BecauseImmutable>(); + let c = c.read().into_inner(); + char::from_u32(c).is_some() + }); +}; + +// SAFETY: Per the Reference [1], `str` has the same layout as `[u8]`. +// - `Immutable`: `[u8]` does not contain any `UnsafeCell`s. +// - `FromZeros`, `IntoBytes`, `Unaligned`: `[u8]` is `FromZeros`, `IntoBytes`, +// and `Unaligned`. +// +// Note that we don't `assert_unaligned!(str)` because `assert_unaligned!` uses +// `align_of`, which only works for `Sized` types. +// +// FIXME(#429): Improve safety proof for `FromZeros` and `IntoBytes`; having the same +// layout as `[u8]` isn't sufficient. +// +// [1] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#str-layout: +// +// String slices are a UTF-8 representation of characters that have the same +// layout as slices of type `[u8]`. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl!(str: Immutable, FromZeros, IntoBytes, Unaligned) }; + +// SAFETY: The impl must only return `true` for its argument if the original +// `Maybe<str>` refers to a valid `str`. `str::from_utf8` guarantees that it +// returns `Err` if its input is not a valid `str` [1]. +// +// [1] Per https://doc.rust-lang.org/core/str/fn.from_utf8.html#errors: +// +// Returns `Err` if the slice is not UTF-8. +const _: () = unsafe { + unsafe_impl!(=> TryFromBytes for str; |c| { + let c = c.transmute_with::<[u8], invariant::Valid, CastUnsized, BecauseImmutable>(); + let c = c.unaligned_as_ref(); + core::str::from_utf8(c).is_ok() + }) +}; + +macro_rules! unsafe_impl_try_from_bytes_for_nonzero { + ($($nonzero:ident[$prim:ty]),*) => { + $( + unsafe_impl!(=> TryFromBytes for $nonzero; |n| { + let n = n.transmute_with::<Unalign<$prim>, invariant::Valid, CastSizedExact, BecauseImmutable>(); + $nonzero::new(n.read().into_inner()).is_some() + }); + )* + } +} + +// `NonZeroXxx` is `IntoBytes`, but not `FromZeros` or `FromBytes`. +// +// SAFETY: +// - `IntoBytes`: `NonZeroXxx` has the same layout as its associated primitive. +// Since it is the same size, this guarantees it has no padding - integers +// have no padding, and there's no room for padding if it can represent all +// of the same values except 0. +// - `Unaligned`: `NonZeroU8` and `NonZeroI8` document that `Option<NonZeroU8>` +// and `Option<NonZeroI8>` both have size 1. [1] [2] This is worded in a way +// that makes it unclear whether it's meant as a guarantee, but given the +// purpose of those types, it's virtually unthinkable that that would ever +// change. `Option` cannot be smaller than its contained type, which implies +// that, and `NonZeroX8` are of size 1 or 0. `NonZeroX8` can represent +// multiple states, so they cannot be 0 bytes, which means that they must be 1 +// byte. The only valid alignment for a 1-byte type is 1. +// +// FIXME(#429): +// - Add quotes from documentation. +// - Add safety comment for `Immutable`. How can we prove that `NonZeroXxx` +// doesn't contain any `UnsafeCell`s? It's obviously true, but it's not clear +// how we'd prove it short of adding text to the stdlib docs that says so +// explicitly, which likely wouldn't be accepted. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/num/type.NonZeroU8.html: +// +// `NonZeroU8` is guaranteed to have the same layout and bit validity as `u8` with +// the exception that 0 is not a valid instance. +// +// [2] Per https://doc.rust-lang.org/1.81.0/std/num/type.NonZeroI8.html: +// +// `NonZeroI8` is guaranteed to have the same layout and bit validity as `i8` with +// the exception that 0 is not a valid instance. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(NonZeroU8: Immutable, IntoBytes, Unaligned); + unsafe_impl!(NonZeroI8: Immutable, IntoBytes, Unaligned); + assert_unaligned!(NonZeroU8, NonZeroI8); + unsafe_impl!(NonZeroU16: Immutable, IntoBytes); + unsafe_impl!(NonZeroI16: Immutable, IntoBytes); + unsafe_impl!(NonZeroU32: Immutable, IntoBytes); + unsafe_impl!(NonZeroI32: Immutable, IntoBytes); + unsafe_impl!(NonZeroU64: Immutable, IntoBytes); + unsafe_impl!(NonZeroI64: Immutable, IntoBytes); + unsafe_impl!(NonZeroU128: Immutable, IntoBytes); + unsafe_impl!(NonZeroI128: Immutable, IntoBytes); + unsafe_impl!(NonZeroUsize: Immutable, IntoBytes); + unsafe_impl!(NonZeroIsize: Immutable, IntoBytes); + unsafe_impl_try_from_bytes_for_nonzero!( + NonZeroU8[u8], + NonZeroI8[i8], + NonZeroU16[u16], + NonZeroI16[i16], + NonZeroU32[u32], + NonZeroI32[i32], + NonZeroU64[u64], + NonZeroI64[i64], + NonZeroU128[u128], + NonZeroI128[i128], + NonZeroUsize[usize], + NonZeroIsize[isize] + ); +}; + +// SAFETY: +// - `TryFromBytes` (with no validator), `FromZeros`, `FromBytes`, `IntoBytes`: +// The Rust compiler reuses `0` value to represent `None`, so +// `size_of::<Option<NonZeroXxx>>() == size_of::<xxx>()`; see `NonZeroXxx` +// documentation. +// - `Unaligned`: `NonZeroU8` and `NonZeroI8` document that `Option<NonZeroU8>` +// and `Option<NonZeroI8>` both have size 1. [1] [2] This is worded in a way +// that makes it unclear whether it's meant as a guarantee, but given the +// purpose of those types, it's virtually unthinkable that that would ever +// change. The only valid alignment for a 1-byte type is 1. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/num/type.NonZeroU8.html: +// +// `Option<NonZeroU8>` is guaranteed to be compatible with `u8`, including in FFI. +// +// Thanks to the null pointer optimization, `NonZeroU8` and `Option<NonZeroU8>` +// are guaranteed to have the same size and alignment: +// +// [2] Per https://doc.rust-lang.org/1.81.0/std/num/type.NonZeroI8.html: +// +// `Option<NonZeroI8>` is guaranteed to be compatible with `i8`, including in FFI. +// +// Thanks to the null pointer optimization, `NonZeroI8` and `Option<NonZeroI8>` +// are guaranteed to have the same size and alignment: +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(Option<NonZeroU8>: TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + unsafe_impl!(Option<NonZeroI8>: TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_unaligned!(Option<NonZeroU8>, Option<NonZeroI8>); + unsafe_impl!(Option<NonZeroU16>: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option<NonZeroI16>: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option<NonZeroU32>: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option<NonZeroI32>: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option<NonZeroU64>: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option<NonZeroI64>: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option<NonZeroU128>: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option<NonZeroI128>: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option<NonZeroUsize>: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option<NonZeroIsize>: TryFromBytes, FromZeros, FromBytes, IntoBytes); +}; + +// SAFETY: While it's not fully documented, the consensus is that `Box<T>` does +// not contain any `UnsafeCell`s for `T: Sized` [1]. This is not a complete +// proof, but we are accepting this as a known risk per #1358. +// +// [1] https://github.com/rust-lang/unsafe-code-guidelines/issues/492 +#[cfg(feature = "alloc")] +const _: () = unsafe { + unsafe_impl!( + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + T: Sized => Immutable for Box<T> + ) +}; + +// SAFETY: The following types can be transmuted from `[0u8; size_of::<T>()]`. [1] +// +// [1] Per https://doc.rust-lang.org/1.89.0/core/option/index.html#representation: +// +// Rust guarantees to optimize the following types `T` such that [`Option<T>`] +// has the same size and alignment as `T`. In some of these cases, Rust +// further guarantees that `transmute::<_, Option<T>>([0u8; size_of::<T>()])` +// is sound and produces `Option::<T>::None`. These cases are identified by +// the second column: +// +// | `T` | `transmute::<_, Option<T>>([0u8; size_of::<T>()])` sound? | +// |-----------------------------------|-----------------------------------------------------------| +// | [`Box<U>`] | when `U: Sized` | +// | `&U` | when `U: Sized` | +// | `&mut U` | when `U: Sized` | +// | [`ptr::NonNull<U>`] | when `U: Sized` | +// | `fn`, `extern "C" fn`[^extern_fn] | always | +// +// [^extern_fn]: this remains true for `unsafe` variants, any argument/return +// types, and any other ABI: `[unsafe] extern "abi" fn` (_e.g._, `extern +// "system" fn`) +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + #[cfg(feature = "alloc")] + unsafe_impl!( + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + T => TryFromBytes for Option<Box<T>>; |c| pointer::is_zeroed(c) + ); + #[cfg(feature = "alloc")] + unsafe_impl!( + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + T => FromZeros for Option<Box<T>> + ); + unsafe_impl!( + T => TryFromBytes for Option<&'_ T>; |c| pointer::is_zeroed(c) + ); + unsafe_impl!(T => FromZeros for Option<&'_ T>); + unsafe_impl!( + T => TryFromBytes for Option<&'_ mut T>; |c| pointer::is_zeroed(c) + ); + unsafe_impl!(T => FromZeros for Option<&'_ mut T>); + unsafe_impl!( + T => TryFromBytes for Option<NonNull<T>>; |c| pointer::is_zeroed(c) + ); + unsafe_impl!(T => FromZeros for Option<NonNull<T>>); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => FromZeros for opt_fn!(...)); + unsafe_impl_for_power_set!( + A, B, C, D, E, F, G, H, I, J, K, L -> M => TryFromBytes for opt_fn!(...); + |c| pointer::is_zeroed(c) + ); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => FromZeros for opt_unsafe_fn!(...)); + unsafe_impl_for_power_set!( + A, B, C, D, E, F, G, H, I, J, K, L -> M => TryFromBytes for opt_unsafe_fn!(...); + |c| pointer::is_zeroed(c) + ); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => FromZeros for opt_extern_c_fn!(...)); + unsafe_impl_for_power_set!( + A, B, C, D, E, F, G, H, I, J, K, L -> M => TryFromBytes for opt_extern_c_fn!(...); + |c| pointer::is_zeroed(c) + ); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => FromZeros for opt_unsafe_extern_c_fn!(...)); + unsafe_impl_for_power_set!( + A, B, C, D, E, F, G, H, I, J, K, L -> M => TryFromBytes for opt_unsafe_extern_c_fn!(...); + |c| pointer::is_zeroed(c) + ); +}; + +// SAFETY: `[unsafe] [extern "C"] fn()` self-evidently do not contain +// `UnsafeCell`s. This is not a proof, but we are accepting this as a known risk +// per #1358. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => Immutable for opt_fn!(...)); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => Immutable for opt_unsafe_fn!(...)); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => Immutable for opt_extern_c_fn!(...)); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => Immutable for opt_unsafe_extern_c_fn!(...)); +}; + +#[cfg(all( + not(no_zerocopy_target_has_atomics_1_60_0), + any( + target_has_atomic = "8", + target_has_atomic = "16", + target_has_atomic = "32", + target_has_atomic = "64", + target_has_atomic = "ptr" + ) +))] +#[cfg_attr(doc_cfg, doc(cfg(rust = "1.60.0")))] +mod atomics { + use super::*; + + macro_rules! impl_traits_for_atomics { + ($($atomics:tt [$primitives:ty]),* $(,)?) => { + $( + impl_known_layout!($atomics); + impl_for_transmute_from!(=> FromZeros for $atomics [$primitives]); + impl_for_transmute_from!(=> FromBytes for $atomics [$primitives]); + impl_for_transmute_from!(=> TryFromBytes for $atomics [$primitives]); + impl_for_transmute_from!(=> IntoBytes for $atomics [$primitives]); + )* + }; + } + + /// Implements `TransmuteFrom` for `$atomic`, `$prim`, and + /// `UnsafeCell<$prim>`. + /// + /// # Safety + /// + /// `$atomic` must have the same size and bit validity as `$prim`. + macro_rules! unsafe_impl_transmute_from_for_atomic { + ($($($tyvar:ident)? => $atomic:ty [$prim:ty]),*) => {{ + crate::util::macros::__unsafe(); + + use crate::pointer::{SizeEq, TransmuteFrom, invariant::Valid}; + + $( + // SAFETY: The caller promised that `$atomic` and `$prim` have + // the same size and bit validity. + unsafe impl<$($tyvar)?> TransmuteFrom<$atomic, Valid, Valid> for $prim {} + // SAFETY: The caller promised that `$atomic` and `$prim` have + // the same size and bit validity. + unsafe impl<$($tyvar)?> TransmuteFrom<$prim, Valid, Valid> for $atomic {} + + impl<$($tyvar)?> SizeEq<ReadOnly<$atomic>> for ReadOnly<$prim> { + type CastFrom = $crate::pointer::cast::CastSizedExact; + } + + // SAFETY: The caller promised that `$atomic` and `$prim` have + // the same bit validity. `UnsafeCell<T>` has the same bit + // validity as `T` [1]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/cell/struct.UnsafeCell.html#memory-layout: + // + // `UnsafeCell<T>` has the same in-memory representation as + // its inner type `T`. A consequence of this guarantee is that + // it is possible to convert between `T` and `UnsafeCell<T>`. + unsafe impl<$($tyvar)?> TransmuteFrom<$atomic, Valid, Valid> for core::cell::UnsafeCell<$prim> {} + // SAFETY: See previous safety comment. + unsafe impl<$($tyvar)?> TransmuteFrom<core::cell::UnsafeCell<$prim>, Valid, Valid> for $atomic {} + )* + }}; + } + + #[cfg(target_has_atomic = "8")] + #[cfg_attr(doc_cfg, doc(cfg(target_has_atomic = "8")))] + mod atomic_8 { + use core::sync::atomic::{AtomicBool, AtomicI8, AtomicU8}; + + use super::*; + + impl_traits_for_atomics!(AtomicU8[u8], AtomicI8[i8]); + + impl_known_layout!(AtomicBool); + impl_for_transmute_from!(=> FromZeros for AtomicBool [bool]); + impl_for_transmute_from!(=> TryFromBytes for AtomicBool [bool]); + impl_for_transmute_from!(=> IntoBytes for AtomicBool [bool]); + + // SAFETY: Per [1], `AtomicBool`, `AtomicU8`, and `AtomicI8` have the + // same size as `bool`, `u8`, and `i8` respectively. Since a type's + // alignment cannot be smaller than 1 [2], and since its alignment + // cannot be greater than its size [3], the only possible value for the + // alignment is 1. Thus, it is sound to implement `Unaligned`. + // + // [1] Per (for example) https://doc.rust-lang.org/1.81.0/std/sync/atomic/struct.AtomicU8.html: + // + // This type has the same size, alignment, and bit validity as the + // underlying integer type + // + // [2] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#size-and-alignment: + // + // Alignment is measured in bytes, and must be at least 1. + // + // [3] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#size-and-alignment: + // + // The size of a value is always a multiple of its alignment. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + unsafe_impl!(AtomicBool: Unaligned); + unsafe_impl!(AtomicU8: Unaligned); + unsafe_impl!(AtomicI8: Unaligned); + assert_unaligned!(AtomicBool, AtomicU8, AtomicI8); + }; + + // SAFETY: `AtomicU8`, `AtomicI8`, and `AtomicBool` have the same size + // and bit validity as `u8`, `i8`, and `bool` respectively [1][2][3]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicU8.html: + // + // This type has the same size, alignment, and bit validity as the + // underlying integer type, `u8`. + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicI8.html: + // + // This type has the same size, alignment, and bit validity as the + // underlying integer type, `i8`. + // + // [3] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicBool.html: + // + // This type has the same size, alignment, and bit validity a `bool`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + unsafe_impl_transmute_from_for_atomic!( + => AtomicU8 [u8], + => AtomicI8 [i8], + => AtomicBool [bool] + ) + }; + } + + #[cfg(target_has_atomic = "16")] + #[cfg_attr(doc_cfg, doc(cfg(target_has_atomic = "16")))] + mod atomic_16 { + use core::sync::atomic::{AtomicI16, AtomicU16}; + + use super::*; + + impl_traits_for_atomics!(AtomicU16[u16], AtomicI16[i16]); + + // SAFETY: `AtomicU16` and `AtomicI16` have the same size and bit + // validity as `u16` and `i16` respectively [1][2]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicU16.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `u16`. + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicI16.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `i16`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + unsafe_impl_transmute_from_for_atomic!(=> AtomicU16 [u16], => AtomicI16 [i16]) + }; + } + + #[cfg(target_has_atomic = "32")] + #[cfg_attr(doc_cfg, doc(cfg(target_has_atomic = "32")))] + mod atomic_32 { + use core::sync::atomic::{AtomicI32, AtomicU32}; + + use super::*; + + impl_traits_for_atomics!(AtomicU32[u32], AtomicI32[i32]); + + // SAFETY: `AtomicU32` and `AtomicI32` have the same size and bit + // validity as `u32` and `i32` respectively [1][2]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicU32.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `u32`. + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicI32.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `i32`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + unsafe_impl_transmute_from_for_atomic!(=> AtomicU32 [u32], => AtomicI32 [i32]) + }; + } + + #[cfg(target_has_atomic = "64")] + #[cfg_attr(doc_cfg, doc(cfg(target_has_atomic = "64")))] + mod atomic_64 { + use core::sync::atomic::{AtomicI64, AtomicU64}; + + use super::*; + + impl_traits_for_atomics!(AtomicU64[u64], AtomicI64[i64]); + + // SAFETY: `AtomicU64` and `AtomicI64` have the same size and bit + // validity as `u64` and `i64` respectively [1][2]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicU64.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `u64`. + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicI64.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `i64`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + unsafe_impl_transmute_from_for_atomic!(=> AtomicU64 [u64], => AtomicI64 [i64]) + }; + } + + #[cfg(target_has_atomic = "ptr")] + #[cfg_attr(doc_cfg, doc(cfg(target_has_atomic = "ptr")))] + mod atomic_ptr { + use core::sync::atomic::{AtomicIsize, AtomicPtr, AtomicUsize}; + + use super::*; + + impl_traits_for_atomics!(AtomicUsize[usize], AtomicIsize[isize]); + + // FIXME(#170): Implement `FromBytes` and `IntoBytes` once we implement + // those traits for `*mut T`. + impl_known_layout!(T => AtomicPtr<T>); + impl_for_transmute_from!(T => TryFromBytes for AtomicPtr<T> [*mut T]); + impl_for_transmute_from!(T => FromZeros for AtomicPtr<T> [*mut T]); + + // SAFETY: `AtomicUsize` and `AtomicIsize` have the same size and bit + // validity as `usize` and `isize` respectively [1][2]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicUsize.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `usize`. + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicIsize.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `isize`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + unsafe_impl_transmute_from_for_atomic!(=> AtomicUsize [usize], => AtomicIsize [isize]) + }; + + // SAFETY: Per + // https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicPtr.html: + // + // This type has the same size and bit validity as a `*mut T`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { unsafe_impl_transmute_from_for_atomic!(T => AtomicPtr<T> [*mut T]) }; + } +} + +// SAFETY: Per reference [1]: "For all T, the following are guaranteed: +// size_of::<PhantomData<T>>() == 0 align_of::<PhantomData<T>>() == 1". This +// gives: +// - `Immutable`: `PhantomData` has no fields. +// - `TryFromBytes` (with no validator), `FromZeros`, `FromBytes`: There is only +// one possible sequence of 0 bytes, and `PhantomData` is inhabited. +// - `IntoBytes`: Since `PhantomData` has size 0, it contains no padding bytes. +// - `Unaligned`: Per the preceding reference, `PhantomData` has alignment 1. +// +// [1] https://doc.rust-lang.org/1.81.0/std/marker/struct.PhantomData.html#layout-1 +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(T: ?Sized => Immutable for PhantomData<T>); + unsafe_impl!(T: ?Sized => TryFromBytes for PhantomData<T>); + unsafe_impl!(T: ?Sized => FromZeros for PhantomData<T>); + unsafe_impl!(T: ?Sized => FromBytes for PhantomData<T>); + unsafe_impl!(T: ?Sized => IntoBytes for PhantomData<T>); + unsafe_impl!(T: ?Sized => Unaligned for PhantomData<T>); + assert_unaligned!(PhantomData<()>, PhantomData<u8>, PhantomData<u64>); +}; + +impl_for_transmute_from!(T: TryFromBytes => TryFromBytes for Wrapping<T>[T]); +impl_for_transmute_from!(T: FromZeros => FromZeros for Wrapping<T>[T]); +impl_for_transmute_from!(T: FromBytes => FromBytes for Wrapping<T>[T]); +impl_for_transmute_from!(T: IntoBytes => IntoBytes for Wrapping<T>[T]); +assert_unaligned!(Wrapping<()>, Wrapping<u8>); + +// SAFETY: Per [1], `Wrapping<T>` has the same layout as `T`. Since its single +// field (of type `T`) is public, it would be a breaking change to add or remove +// fields. Thus, we know that `Wrapping<T>` contains a `T` (as opposed to just +// having the same size and alignment as `T`) with no pre- or post-padding. +// Thus, `Wrapping<T>` must have `UnsafeCell`s covering the same byte ranges as +// `Inner = T`. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/num/struct.Wrapping.html#layout-1: +// +// `Wrapping<T>` is guaranteed to have the same layout and ABI as `T` +const _: () = unsafe { unsafe_impl!(T: Immutable => Immutable for Wrapping<T>) }; + +// SAFETY: Per [1] in the preceding safety comment, `Wrapping<T>` has the same +// alignment as `T`. +const _: () = unsafe { unsafe_impl!(T: Unaligned => Unaligned for Wrapping<T>) }; + +// SAFETY: `TryFromBytes` (with no validator), `FromZeros`, `FromBytes`: +// `MaybeUninit<T>` has no restrictions on its contents. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(T => TryFromBytes for CoreMaybeUninit<T>); + unsafe_impl!(T => FromZeros for CoreMaybeUninit<T>); + unsafe_impl!(T => FromBytes for CoreMaybeUninit<T>); +}; + +// SAFETY: `MaybeUninit<T>` has `UnsafeCell`s covering the same byte ranges as +// `Inner = T`. This is not explicitly documented, but it can be inferred. Per +// [1], `MaybeUninit<T>` has the same size as `T`. Further, note the signature +// of `MaybeUninit::assume_init_ref` [2]: +// +// pub unsafe fn assume_init_ref(&self) -> &T +// +// If the argument `&MaybeUninit<T>` and the returned `&T` had `UnsafeCell`s at +// different offsets, this would be unsound. Its existence is proof that this is +// not the case. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/mem/union.MaybeUninit.html#layout-1: +// +// `MaybeUninit<T>` is guaranteed to have the same size, alignment, and ABI as +// `T`. +// +// [2] https://doc.rust-lang.org/1.81.0/std/mem/union.MaybeUninit.html#method.assume_init_ref +const _: () = unsafe { unsafe_impl!(T: Immutable => Immutable for CoreMaybeUninit<T>) }; + +// SAFETY: Per [1] in the preceding safety comment, `MaybeUninit<T>` has the +// same alignment as `T`. +const _: () = unsafe { unsafe_impl!(T: Unaligned => Unaligned for CoreMaybeUninit<T>) }; +assert_unaligned!(CoreMaybeUninit<()>, CoreMaybeUninit<u8>); + +// SAFETY: `ManuallyDrop<T>` has the same layout as `T` [1]. This strongly +// implies, but does not guarantee, that it contains `UnsafeCell`s covering the +// same byte ranges as in `T`. However, it also implements `Defer<Target = T>` +// [2], which provides the ability to convert `&ManuallyDrop<T> -> &T`. This, +// combined with having the same size as `T`, implies that `ManuallyDrop<T>` +// exactly contains a `T` with the same fields and `UnsafeCell`s covering the +// same byte ranges, or else the `Deref` impl would permit safe code to obtain +// different shared references to the same region of memory with different +// `UnsafeCell` coverage, which would in turn permit interior mutation that +// would violate the invariants of a shared reference. +// +// [1] Per https://doc.rust-lang.org/1.85.0/std/mem/struct.ManuallyDrop.html: +// +// `ManuallyDrop<T>` is guaranteed to have the same layout and bit validity as +// `T` +// +// [2] https://doc.rust-lang.org/1.85.0/std/mem/struct.ManuallyDrop.html#impl-Deref-for-ManuallyDrop%3CT%3E +const _: () = unsafe { unsafe_impl!(T: ?Sized + Immutable => Immutable for ManuallyDrop<T>) }; + +impl_for_transmute_from!(T: ?Sized + TryFromBytes => TryFromBytes for ManuallyDrop<T>[T]); +impl_for_transmute_from!(T: ?Sized + FromZeros => FromZeros for ManuallyDrop<T>[T]); +impl_for_transmute_from!(T: ?Sized + FromBytes => FromBytes for ManuallyDrop<T>[T]); +impl_for_transmute_from!(T: ?Sized + IntoBytes => IntoBytes for ManuallyDrop<T>[T]); +// SAFETY: `ManuallyDrop<T>` has the same layout as `T` [1], and thus has the +// same alignment as `T`. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/mem/struct.ManuallyDrop.html: +// +// `ManuallyDrop<T>` is guaranteed to have the same layout and bit validity as +// `T` +const _: () = unsafe { unsafe_impl!(T: ?Sized + Unaligned => Unaligned for ManuallyDrop<T>) }; +assert_unaligned!(ManuallyDrop<()>, ManuallyDrop<u8>); + +const _: () = { + #[allow( + non_camel_case_types, + missing_copy_implementations, + missing_debug_implementations, + missing_docs + )] + pub enum value {} + + // SAFETY: See safety comment on `ProjectToTag`. + unsafe impl<T: ?Sized> HasTag for ManuallyDrop<T> { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + type Tag = (); + + // SAFETY: It is trivially sound to project any pointer to a pointer to + // a type of size zero and alignment 1 (which `()` is [1]). Such a + // pointer will trivially satisfy its aliasing and validity requirements + // (since it has a zero-sized referent), and its alignment requirement + // (since it is aligned to 1). + // + // [1] Per https://doc.rust-lang.org/1.92.0/reference/type-layout.html#r-layout.tuple.unit: + // + // [T]he unit tuple (`()`)... is guaranteed as a zero-sized type to + // have a size of 0 and an alignment of 1. + type ProjectToTag = crate::pointer::cast::CastToUnit; + } + + // SAFETY: `ManuallyDrop<T>` has a field of type `T` at offset `0` without + // any safety invariants beyond those of `T`. Its existence is not + // explicitly documented, but it can be inferred; per [1] `ManuallyDrop<T>` + // has the same size and bit validity as `T`. This field is not literally + // public, but is effectively so; the field can be transparently: + // + // - initialized via `ManuallyDrop::new` + // - moved via `ManuallyDrop::into_inner` + // - referenced via `ManuallyDrop::deref` + // - exclusively referenced via `ManuallyDrop::deref_mut` + // + // We call this field `value`, both because that is both the name of this + // private field, and because it is the name it is referred to in the public + // documentation of `ManuallyDrop::new`, `ManuallyDrop::into_inner`, + // `ManuallyDrop::take` and `ManuallyDrop::drop`. + unsafe impl<T: ?Sized> + HasField<value, { crate::STRUCT_VARIANT_ID }, { crate::ident_id!(value) }> + for ManuallyDrop<T> + { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + type Type = T; + + #[inline(always)] + fn project(slf: PtrInner<'_, Self>) -> *mut T { + // SAFETY: `ManuallyDrop<T>` has the same layout and bit validity as + // `T` [1]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/mem/struct.ManuallyDrop.html: + // + // `ManuallyDrop<T>` is guaranteed to have the same layout and bit + // validity as `T` + #[allow(clippy::as_conversions)] + return slf.as_ptr() as *mut T; + } + } +}; + +impl_for_transmute_from!(T: ?Sized + TryFromBytes => TryFromBytes for Cell<T>[T]); +impl_for_transmute_from!(T: ?Sized + FromZeros => FromZeros for Cell<T>[T]); +impl_for_transmute_from!(T: ?Sized + FromBytes => FromBytes for Cell<T>[T]); +impl_for_transmute_from!(T: ?Sized + IntoBytes => IntoBytes for Cell<T>[T]); +// SAFETY: `Cell<T>` has the same in-memory representation as `T` [1], and thus +// has the same alignment as `T`. +// +// [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.Cell.html#memory-layout: +// +// `Cell<T>` has the same in-memory representation as its inner type `T`. +const _: () = unsafe { unsafe_impl!(T: ?Sized + Unaligned => Unaligned for Cell<T>) }; + +impl_for_transmute_from!(T: ?Sized + FromZeros => FromZeros for UnsafeCell<T>[T]); +impl_for_transmute_from!(T: ?Sized + FromBytes => FromBytes for UnsafeCell<T>[T]); +impl_for_transmute_from!(T: ?Sized + IntoBytes => IntoBytes for UnsafeCell<T>[T]); +// SAFETY: `UnsafeCell<T>` has the same in-memory representation as `T` [1], and +// thus has the same alignment as `T`. +// +// [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout: +// +// `UnsafeCell<T>` has the same in-memory representation as its inner type +// `T`. +const _: () = unsafe { unsafe_impl!(T: ?Sized + Unaligned => Unaligned for UnsafeCell<T>) }; +assert_unaligned!(UnsafeCell<()>, UnsafeCell<u8>); + +// SAFETY: See safety comment in `is_bit_valid` impl. +unsafe impl<T: TryFromBytes + ?Sized> TryFromBytes for UnsafeCell<T> { + #[allow(clippy::missing_inline_in_public_items)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + #[inline(always)] + fn is_bit_valid<A>(candidate: Maybe<'_, Self, A>) -> bool + where + A: invariant::Alignment, + { + T::is_bit_valid(candidate.transmute::<_, _, BecauseImmutable>()) + } +} + +// SAFETY: Per the reference [1]: +// +// An array of `[T; N]` has a size of `size_of::<T>() * N` and the same +// alignment of `T`. Arrays are laid out so that the zero-based `nth` element +// of the array is offset from the start of the array by `n * size_of::<T>()` +// bytes. +// +// ... +// +// Slices have the same layout as the section of the array they slice. +// +// In other words, the layout of a `[T]` or `[T; N]` is a sequence of `T`s laid +// out back-to-back with no bytes in between. Therefore, `[T]` or `[T; N]` are +// `Immutable`, `TryFromBytes`, `FromZeros`, `FromBytes`, and `IntoBytes` if `T` +// is (respectively). Furthermore, since an array/slice has "the same alignment +// of `T`", `[T]` and `[T; N]` are `Unaligned` if `T` is. +// +// Note that we don't `assert_unaligned!` for slice types because +// `assert_unaligned!` uses `align_of`, which only works for `Sized` types. +// +// [1] https://doc.rust-lang.org/1.81.0/reference/type-layout.html#array-layout +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(const N: usize, T: Immutable => Immutable for [T; N]); + unsafe_impl!(const N: usize, T: TryFromBytes => TryFromBytes for [T; N]; |c| { + let c: Ptr<'_, [ReadOnly<T>; N], _> = c.cast::<_, crate::pointer::cast::CastSized, _>(); + let c: Ptr<'_, [ReadOnly<T>], _> = c.as_slice(); + let c: Ptr<'_, ReadOnly<[T]>, _> = c.cast::<_, crate::pointer::cast::CastUnsized, _>(); + + // Note that this call may panic, but it would still be sound even if it + // did. `is_bit_valid` does not promise that it will not panic (in fact, + // it explicitly warns that it's a possibility), and we have not + // violated any safety invariants that we must fix before returning. + <[T] as TryFromBytes>::is_bit_valid(c) + }); + unsafe_impl!(const N: usize, T: FromZeros => FromZeros for [T; N]); + unsafe_impl!(const N: usize, T: FromBytes => FromBytes for [T; N]); + unsafe_impl!(const N: usize, T: IntoBytes => IntoBytes for [T; N]); + unsafe_impl!(const N: usize, T: Unaligned => Unaligned for [T; N]); + assert_unaligned!([(); 0], [(); 1], [u8; 0], [u8; 1]); + unsafe_impl!(T: Immutable => Immutable for [T]); + unsafe_impl!(T: TryFromBytes => TryFromBytes for [T]; |c| { + let c: Ptr<'_, [ReadOnly<T>], _> = c.cast::<_, crate::pointer::cast::CastUnsized, _>(); + + // SAFETY: Per the reference [1]: + // + // An array of `[T; N]` has a size of `size_of::<T>() * N` and the + // same alignment of `T`. Arrays are laid out so that the zero-based + // `nth` element of the array is offset from the start of the array by + // `n * size_of::<T>()` bytes. + // + // ... + // + // Slices have the same layout as the section of the array they slice. + // + // In other words, the layout of a `[T] is a sequence of `T`s laid out + // back-to-back with no bytes in between. If all elements in `candidate` + // are `is_bit_valid`, so too is `candidate`. + // + // Note that any of the below calls may panic, but it would still be + // sound even if it did. `is_bit_valid` does not promise that it will + // not panic (in fact, it explicitly warns that it's a possibility), and + // we have not violated any safety invariants that we must fix before + // returning. + c.iter().all(<T as TryFromBytes>::is_bit_valid) + }); + unsafe_impl!(T: FromZeros => FromZeros for [T]); + unsafe_impl!(T: FromBytes => FromBytes for [T]); + unsafe_impl!(T: IntoBytes => IntoBytes for [T]); + unsafe_impl!(T: Unaligned => Unaligned for [T]); +}; + +// SAFETY: +// - `Immutable`: Raw pointers do not contain any `UnsafeCell`s. +// - `FromZeros`: For thin pointers (note that `T: Sized`), the zero pointer is +// considered "null". [1] No operations which require provenance are legal on +// null pointers, so this is not a footgun. +// - `TryFromBytes`: By the same reasoning as for `FromZeroes`, we can implement +// `TryFromBytes` for thin pointers provided that +// [`TryFromByte::is_bit_valid`] only produces `true` for zeroed bytes. +// +// NOTE(#170): Implementing `FromBytes` and `IntoBytes` for raw pointers would +// be sound, but carries provenance footguns. We want to support `FromBytes` and +// `IntoBytes` for raw pointers eventually, but we are holding off until we can +// figure out how to address those footguns. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/ptr/fn.null.html: +// +// Creates a null raw pointer. +// +// This function is equivalent to zero-initializing the pointer: +// `MaybeUninit::<*const T>::zeroed().assume_init()`. +// +// The resulting pointer has the address 0. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(T: ?Sized => Immutable for *const T); + unsafe_impl!(T: ?Sized => Immutable for *mut T); + unsafe_impl!(T => TryFromBytes for *const T; |c| pointer::is_zeroed(c)); + unsafe_impl!(T => FromZeros for *const T); + unsafe_impl!(T => TryFromBytes for *mut T; |c| pointer::is_zeroed(c)); + unsafe_impl!(T => FromZeros for *mut T); +}; + +// SAFETY: `NonNull<T>` self-evidently does not contain `UnsafeCell`s. This is +// not a proof, but we are accepting this as a known risk per #1358. +const _: () = unsafe { unsafe_impl!(T: ?Sized => Immutable for NonNull<T>) }; + +// SAFETY: Reference types do not contain any `UnsafeCell`s. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(T: ?Sized => Immutable for &'_ T); + unsafe_impl!(T: ?Sized => Immutable for &'_ mut T); +}; + +// SAFETY: `Option` is not `#[non_exhaustive]` [1], which means that the types +// in its variants cannot change, and no new variants can be added. `Option<T>` +// does not contain any `UnsafeCell`s outside of `T`. [1] +// +// [1] https://doc.rust-lang.org/core/option/enum.Option.html +const _: () = unsafe { unsafe_impl!(T: Immutable => Immutable for Option<T>) }; + +mod tuples { + use super::*; + + /// Generates various trait implementations for tuples. + /// + /// # Safety + /// + /// `impl_tuple!` should be provided name-number pairs, where each number is + /// the ordinal of the preceding type name. + macro_rules! impl_tuple { + // Entry point. + ($($T:ident $I:tt),+ $(,)?) => { + crate::util::macros::__unsafe(); + impl_tuple!(@all [] [$($T $I)+]); + }; + + // Build up the set of tuple types (i.e., `(A,)`, `(A, B)`, `(A, B, C)`, + // etc.) Trait implementations that do not depend on field index may be + // added to this branch. + (@all [$($head_T:ident $head_I:tt)*] [$next_T:ident $next_I:tt $($tail:tt)*]) => { + // SAFETY: If all fields of the tuple `Self` are `Immutable`, so too is `Self`. + unsafe_impl!($($head_T: Immutable,)* $next_T: Immutable => Immutable for ($($head_T,)* $next_T,)); + + // SAFETY: If all fields in `c` are `is_bit_valid`, so too is `c`. + unsafe_impl!($($head_T: TryFromBytes,)* $next_T: TryFromBytes => TryFromBytes for ($($head_T,)* $next_T,); |c| { + let mut c = c; + $(TryFromBytes::is_bit_valid(into_inner!(c.reborrow().project::<_, { crate::STRUCT_VARIANT_ID }, { crate::ident_id!($head_I) }>())) &&)* + TryFromBytes::is_bit_valid(into_inner!(c.reborrow().project::<_, { crate::STRUCT_VARIANT_ID }, { crate::ident_id!($next_I) }>())) + }); + + // SAFETY: If all fields in `Self` are `FromZeros`, so too is `Self`. + unsafe_impl!($($head_T: FromZeros,)* $next_T: FromZeros => FromZeros for ($($head_T,)* $next_T,)); + + // SAFETY: If all fields in `Self` are `FromBytes`, so too is `Self`. + unsafe_impl!($($head_T: FromBytes,)* $next_T: FromBytes => FromBytes for ($($head_T,)* $next_T,)); + + // SAFETY: See safety comment on `ProjectToTag`. + unsafe impl<$($head_T,)* $next_T> crate::HasTag for ($($head_T,)* $next_T,) { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized + {} + + type Tag = (); + + // SAFETY: It is trivially sound to project any pointer to a + // pointer to a type of size zero and alignment 1 (which `()` is + // [1]). Such a pointer will trivially satisfy its aliasing and + // validity requirements (since it has a zero-sized referent), + // and its alignment requirement (since it is aligned to 1). + // + // [1] Per https://doc.rust-lang.org/1.92.0/reference/type-layout.html#r-layout.tuple.unit: + // + // [T]he unit tuple (`()`)... is guaranteed as a zero-sized + // type to have a size of 0 and an alignment of 1. + type ProjectToTag = crate::pointer::cast::CastToUnit; + } + + // Generate impls that depend on tuple index. + impl_tuple!(@variants + [$($head_T $head_I)* $next_T $next_I] + [] + [$($head_T $head_I)* $next_T $next_I] + ); + + // Recurse to next tuple size + impl_tuple!(@all [$($head_T $head_I)* $next_T $next_I] [$($tail)*]); + }; + (@all [$($head_T:ident $head_I:tt)*] []) => {}; + + // Emit trait implementations that depend on field index. + (@variants + // The full tuple definition in type–index pairs. + [$($AllT:ident $AllI:tt)+] + // Types before the current index. + [$($BeforeT:ident)*] + // The types and indices at and after the current index. + [$CurrT:ident $CurrI:tt $($AfterT:ident $AfterI:tt)*] + ) => { + // SAFETY: + // - `Self` is a struct (albeit anonymous), so `VARIANT_ID` is + // `STRUCT_VARIANT_ID`. + // - `$CurrI` is the field at index `$CurrI`, so `FIELD_ID` is + // `zerocopy::ident_id!($CurrI)` + // - `()` has the same visibility as the `.$CurrI` field (ie, `.0`, + // `.1`, etc) + // - `Type` has the same type as `$CurrI`; i.e., `$CurrT`. + unsafe impl<$($AllT),+> crate::HasField< + (), + { crate::STRUCT_VARIANT_ID }, + { crate::ident_id!($CurrI)} + > for ($($AllT,)+) { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized + {} + + type Type = $CurrT; + + #[inline(always)] + fn project(slf: crate::PtrInner<'_, Self>) -> *mut Self::Type { + let slf = slf.as_non_null().as_ptr(); + // SAFETY: `PtrInner` promises it references either a zero-sized + // byte range, or else will reference a byte range that is + // entirely contained within an allocated object. In either + // case, this guarantees that `(*slf).$CurrI` is in-bounds of + // `slf`. + unsafe { core::ptr::addr_of_mut!((*slf).$CurrI) } + } + } + + // SAFETY: See comments on items. + unsafe impl<Aliasing, Alignment, $($AllT),+> crate::ProjectField< + (), + (Aliasing, Alignment, crate::invariant::Uninit), + { crate::STRUCT_VARIANT_ID }, + { crate::ident_id!($CurrI)} + > for ($($AllT,)+) + where + Aliasing: crate::invariant::Aliasing, + Alignment: crate::invariant::Alignment, + { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized + {} + + // SAFETY: Tuples are product types whose fields are + // well-aligned, so projection preserves both the alignment and + // validity invariants of the outer pointer. + type Invariants = (Aliasing, Alignment, crate::invariant::Uninit); + + // SAFETY: Tuples are product types and so projection is infallible; + type Error = core::convert::Infallible; + } + + // SAFETY: See comments on items. + unsafe impl<Aliasing, Alignment, $($AllT),+> crate::ProjectField< + (), + (Aliasing, Alignment, crate::invariant::Initialized), + { crate::STRUCT_VARIANT_ID }, + { crate::ident_id!($CurrI)} + > for ($($AllT,)+) + where + Aliasing: crate::invariant::Aliasing, + Alignment: crate::invariant::Alignment, + { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized + {} + + // SAFETY: Tuples are product types whose fields are + // well-aligned, so projection preserves both the alignment and + // validity invariants of the outer pointer. + type Invariants = (Aliasing, Alignment, crate::invariant::Initialized); + + // SAFETY: Tuples are product types and so projection is infallible; + type Error = core::convert::Infallible; + } + + // SAFETY: See comments on items. + unsafe impl<Aliasing, Alignment, $($AllT),+> crate::ProjectField< + (), + (Aliasing, Alignment, crate::invariant::Valid), + { crate::STRUCT_VARIANT_ID }, + { crate::ident_id!($CurrI)} + > for ($($AllT,)+) + where + Aliasing: crate::invariant::Aliasing, + Alignment: crate::invariant::Alignment, + { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized + {} + + // SAFETY: Tuples are product types whose fields are + // well-aligned, so projection preserves both the alignment and + // validity invariants of the outer pointer. + type Invariants = (Aliasing, Alignment, crate::invariant::Valid); + + // SAFETY: Tuples are product types and so projection is infallible; + type Error = core::convert::Infallible; + } + + // Recurse to the next index. + impl_tuple!(@variants [$($AllT $AllI)+] [$($BeforeT)* $CurrT] [$($AfterT $AfterI)*]); + }; + (@variants [$($AllT:ident $AllI:tt)+] [$($BeforeT:ident)*] []) => {}; + } + + // SAFETY: `impl_tuple` is provided name-number pairs, where number is the + // ordinal of the name. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + impl_tuple! { + A 0, + B 1, + C 2, + D 3, + E 4, + F 5, + G 6, + H 7, + I 8, + J 9, + K 10, + L 11, + M 12, + N 13, + O 14, + P 15, + Q 16, + R 17, + S 18, + T 19, + U 20, + V 21, + W 22, + X 23, + Y 24, + Z 25, + }; + }; +} + +// SIMD support +// +// Per the Unsafe Code Guidelines Reference [1]: +// +// Packed SIMD vector types are `repr(simd)` homogeneous tuple-structs +// containing `N` elements of type `T` where `N` is a power-of-two and the +// size and alignment requirements of `T` are equal: +// +// ```rust +// #[repr(simd)] +// struct Vector<T, N>(T_0, ..., T_(N - 1)); +// ``` +// +// ... +// +// The size of `Vector` is `N * size_of::<T>()` and its alignment is an +// implementation-defined function of `T` and `N` greater than or equal to +// `align_of::<T>()`. +// +// ... +// +// Vector elements are laid out in source field order, enabling random access +// to vector elements by reinterpreting the vector as an array: +// +// ```rust +// union U { +// vec: Vector<T, N>, +// arr: [T; N] +// } +// +// assert_eq!(size_of::<Vector<T, N>>(), size_of::<[T; N]>()); +// assert!(align_of::<Vector<T, N>>() >= align_of::<[T; N]>()); +// +// unsafe { +// let u = U { vec: Vector<T, N>(t_0, ..., t_(N - 1)) }; +// +// assert_eq!(u.vec.0, u.arr[0]); +// // ... +// assert_eq!(u.vec.(N - 1), u.arr[N - 1]); +// } +// ``` +// +// Given this background, we can observe that: +// - The size and bit pattern requirements of a SIMD type are equivalent to the +// equivalent array type. Thus, for any SIMD type whose primitive `T` is +// `Immutable`, `TryFromBytes`, `FromZeros`, `FromBytes`, or `IntoBytes`, that +// SIMD type is also `Immutable`, `TryFromBytes`, `FromZeros`, `FromBytes`, or +// `IntoBytes` respectively. +// - Since no upper bound is placed on the alignment, no SIMD type can be +// guaranteed to be `Unaligned`. +// +// Also per [1]: +// +// This chapter represents the consensus from issue #38. The statements in +// here are not (yet) "guaranteed" not to change until an RFC ratifies them. +// +// See issue #38 [2]. While this behavior is not technically guaranteed, the +// likelihood that the behavior will change such that SIMD types are no longer +// `TryFromBytes`, `FromZeros`, `FromBytes`, or `IntoBytes` is next to zero, as +// that would defeat the entire purpose of SIMD types. Nonetheless, we put this +// behavior behind the `simd` Cargo feature, which requires consumers to opt +// into this stability hazard. +// +// [1] https://rust-lang.github.io/unsafe-code-guidelines/layout/packed-simd-vectors.html +// [2] https://github.com/rust-lang/unsafe-code-guidelines/issues/38 +#[cfg(feature = "simd")] +#[cfg_attr(doc_cfg, doc(cfg(feature = "simd")))] +mod simd { + /// Defines a module which implements `TryFromBytes`, `FromZeros`, + /// `FromBytes`, and `IntoBytes` for a set of types from a module in + /// `core::arch`. + /// + /// `$arch` is both the name of the defined module and the name of the + /// module in `core::arch`, and `$typ` is the list of items from that module + /// to implement `FromZeros`, `FromBytes`, and `IntoBytes` for. + #[allow(unused_macros)] // `allow(unused_macros)` is needed because some + // target/feature combinations don't emit any impls + // and thus don't use this macro. + macro_rules! simd_arch_mod { + ($(#[cfg $cfg:tt])* $(#[cfg_attr $cfg_attr:tt])? $arch:ident, $mod:ident, $($typ:ident),*) => { + $(#[cfg $cfg])* + #[cfg_attr(doc_cfg, doc(cfg $($cfg)*))] + $(#[cfg_attr $cfg_attr])? + mod $mod { + use core::arch::$arch::{$($typ),*}; + + use crate::*; + impl_known_layout!($($typ),*); + // SAFETY: See comment on module definition for justification. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + $( unsafe_impl!($typ: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); )* + }; + } + }; + } + + #[rustfmt::skip] + const _: () = { + simd_arch_mod!( + #[cfg(target_arch = "x86")] + x86, x86, __m128, __m128d, __m128i, __m256, __m256d, __m256i + ); + #[cfg(not(no_zerocopy_simd_x86_avx12_1_89_0))] + simd_arch_mod!( + #[cfg(target_arch = "x86")] + #[cfg_attr(doc_cfg, doc(cfg(rust = "1.89.0")))] + x86, x86_nightly, __m512bh, __m512, __m512d, __m512i + ); + simd_arch_mod!( + #[cfg(target_arch = "x86_64")] + x86_64, x86_64, __m128, __m128d, __m128i, __m256, __m256d, __m256i + ); + #[cfg(not(no_zerocopy_simd_x86_avx12_1_89_0))] + simd_arch_mod!( + #[cfg(target_arch = "x86_64")] + #[cfg_attr(doc_cfg, doc(cfg(rust = "1.89.0")))] + x86_64, x86_64_nightly, __m512bh, __m512, __m512d, __m512i + ); + simd_arch_mod!( + #[cfg(target_arch = "wasm32")] + wasm32, wasm32, v128 + ); + simd_arch_mod!( + #[cfg(all(feature = "simd-nightly", target_arch = "powerpc"))] + powerpc, powerpc, vector_bool_long, vector_double, vector_signed_long, vector_unsigned_long + ); + simd_arch_mod!( + #[cfg(all(feature = "simd-nightly", target_arch = "powerpc64"))] + powerpc64, powerpc64, vector_bool_long, vector_double, vector_signed_long, vector_unsigned_long + ); + // NOTE: NEON intrinsics were broken on big-endian platforms from their stabilization up to + // Rust 1.87. (Context in https://github.com/rust-lang/stdarch/issues/1484). Support is + // split in two different version ranges on top of the base configuration, requiring either + // little endian or the more recent version to be detected as well. + #[cfg(not(no_zerocopy_aarch64_simd_1_59_0))] + simd_arch_mod!( + #[cfg(all( + target_arch = "aarch64", + any( + target_endian = "little", + not(no_zerocopy_aarch64_simd_be_1_87_0) + ) + ))] + #[cfg_attr( + doc_cfg, + doc(cfg(all(target_arch = "aarch64", any( + all(rust = "1.59.0", target_endian = "little"), + rust = "1.87.0", + )))) + )] + aarch64, aarch64, float32x2_t, float32x4_t, float64x1_t, float64x2_t, int8x8_t, int8x8x2_t, + int8x8x3_t, int8x8x4_t, int8x16_t, int8x16x2_t, int8x16x3_t, int8x16x4_t, int16x4_t, + int16x8_t, int32x2_t, int32x4_t, int64x1_t, int64x2_t, poly8x8_t, poly8x8x2_t, poly8x8x3_t, + poly8x8x4_t, poly8x16_t, poly8x16x2_t, poly8x16x3_t, poly8x16x4_t, poly16x4_t, poly16x8_t, + poly64x1_t, poly64x2_t, uint8x8_t, uint8x8x2_t, uint8x8x3_t, uint8x8x4_t, uint8x16_t, + uint8x16x2_t, uint8x16x3_t, uint8x16x4_t, uint16x4_t, uint16x4x2_t, uint16x4x3_t, + uint16x4x4_t, uint16x8_t, uint32x2_t, uint32x4_t, uint64x1_t, uint64x2_t + ); + }; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_impls() { + // A type that can supply test cases for testing + // `TryFromBytes::is_bit_valid`. All types passed to `assert_impls!` + // must implement this trait; that macro uses it to generate runtime + // tests for `TryFromBytes` impls. + // + // All `T: FromBytes` types are provided with a blanket impl. Other + // types must implement `TryFromBytesTestable` directly (ie using + // `impl_try_from_bytes_testable!`). + trait TryFromBytesTestable { + fn with_passing_test_cases<F: Fn(Box<ReadOnly<Self>>)>(f: F); + fn with_failing_test_cases<F: Fn(&mut [u8])>(f: F); + } + + impl<T: FromBytes> TryFromBytesTestable for T { + fn with_passing_test_cases<F: Fn(Box<ReadOnly<Self>>)>(f: F) { + // Test with a zeroed value. + f(ReadOnly::<Self>::new_box_zeroed().unwrap()); + + let ffs = { + let mut t = ReadOnly::new(Self::new_zeroed()); + let ptr: *mut T = ReadOnly::as_mut(&mut t); + // SAFETY: `T: FromBytes` + unsafe { ptr::write_bytes(ptr.cast::<u8>(), 0xFF, mem::size_of::<T>()) }; + t + }; + + // Test with a value initialized with 0xFF. + f(Box::new(ffs)); + } + + fn with_failing_test_cases<F: Fn(&mut [u8])>(_f: F) {} + } + + macro_rules! impl_try_from_bytes_testable_for_null_pointer_optimization { + ($($tys:ty),*) => { + $( + impl TryFromBytesTestable for Option<$tys> { + fn with_passing_test_cases<F: Fn(Box<ReadOnly<Self>>)>(f: F) { + // Test with a zeroed value. + f(Box::new(ReadOnly::new(None))); + } + + fn with_failing_test_cases<F: Fn(&mut [u8])>(f: F) { + for pos in 0..mem::size_of::<Self>() { + let mut bytes = [0u8; mem::size_of::<Self>()]; + bytes[pos] = 0x01; + f(&mut bytes[..]); + } + } + } + )* + }; + } + + // Implements `TryFromBytesTestable`. + macro_rules! impl_try_from_bytes_testable { + // Base case for recursion (when the list of types has run out). + (=> @success $($success_case:expr),* $(, @failure $($failure_case:expr),*)?) => {}; + // Implements for type(s) with no type parameters. + ($ty:ty $(,$tys:ty)* => @success $($success_case:expr),* $(, @failure $($failure_case:expr),*)?) => { + impl TryFromBytesTestable for $ty { + impl_try_from_bytes_testable!( + @methods @success $($success_case),* + $(, @failure $($failure_case),*)? + ); + } + impl_try_from_bytes_testable!($($tys),* => @success $($success_case),* $(, @failure $($failure_case),*)?); + }; + // Implements for multiple types with no type parameters. + ($($($ty:ty),* => @success $($success_case:expr), * $(, @failure $($failure_case:expr),*)?;)*) => { + $( + impl_try_from_bytes_testable!($($ty),* => @success $($success_case),* $(, @failure $($failure_case),*)*); + )* + }; + // Implements only the methods; caller must invoke this from inside + // an impl block. + (@methods @success $($success_case:expr),* $(, @failure $($failure_case:expr),*)?) => { + fn with_passing_test_cases<F: Fn(Box<ReadOnly<Self>>)>(_f: F) { + $( + let bx = Box::<Self>::from($success_case); + let ro: Box<ReadOnly<_>> = { + let raw = Box::into_raw(bx); + // SAFETY: `ReadOnly<T>` has the same layout and bit + // validity as `T`. + #[allow(clippy::as_conversions)] + unsafe { Box::from_raw(raw as *mut _) } + }; + _f(ro); + )* + } + + fn with_failing_test_cases<F: Fn(&mut [u8])>(_f: F) { + $($( + let mut case = $failure_case; + _f(case.as_mut_bytes()); + )*)? + } + }; + } + + impl_try_from_bytes_testable_for_null_pointer_optimization!( + Box<UnsafeCell<NotZerocopy>>, + &'static UnsafeCell<NotZerocopy>, + &'static mut UnsafeCell<NotZerocopy>, + NonNull<UnsafeCell<NotZerocopy>>, + fn(), + FnManyArgs, + extern "C" fn(), + ECFnManyArgs + ); + + macro_rules! bx { + ($e:expr) => { + Box::new($e) + }; + } + + // Note that these impls are only for types which are not `FromBytes`. + // `FromBytes` types are covered by a preceding blanket impl. + impl_try_from_bytes_testable!( + bool => @success true, false, + @failure 2u8, 3u8, 0xFFu8; + char => @success '\u{0}', '\u{D7FF}', '\u{E000}', '\u{10FFFF}', + @failure 0xD800u32, 0xDFFFu32, 0x110000u32; + str => @success "", "hello", "❤️🧡💛💚💙💜", + @failure [0, 159, 146, 150]; + [u8] => @success vec![].into_boxed_slice(), vec![0, 1, 2].into_boxed_slice(); + NonZeroU8, NonZeroI8, NonZeroU16, NonZeroI16, NonZeroU32, + NonZeroI32, NonZeroU64, NonZeroI64, NonZeroU128, NonZeroI128, + NonZeroUsize, NonZeroIsize + => @success Self::new(1).unwrap(), + // Doing this instead of `0` ensures that we always satisfy + // the size and alignment requirements of `Self` (whereas `0` + // may be any integer type with a different size or alignment + // than some `NonZeroXxx` types). + @failure Option::<Self>::None; + [bool; 0] => @success []; + [bool; 1] + => @success [true], [false], + @failure [2u8], [3u8], [0xFFu8]; + [bool] + => @success vec![true, false].into_boxed_slice(), vec![false, true].into_boxed_slice(), + @failure [2u8], [3u8], [0xFFu8], [0u8, 1u8, 2u8]; + Unalign<bool> + => @success Unalign::new(false), Unalign::new(true), + @failure 2u8, 0xFFu8; + ManuallyDrop<bool> + => @success ManuallyDrop::new(false), ManuallyDrop::new(true), + @failure 2u8, 0xFFu8; + ManuallyDrop<[u8]> + => @success bx!(ManuallyDrop::new([])), bx!(ManuallyDrop::new([0u8])), bx!(ManuallyDrop::new([0u8, 1u8])); + ManuallyDrop<[bool]> + => @success bx!(ManuallyDrop::new([])), bx!(ManuallyDrop::new([false])), bx!(ManuallyDrop::new([false, true])), + @failure [2u8], [3u8], [0xFFu8], [0u8, 1u8, 2u8]; + ManuallyDrop<[UnsafeCell<u8>]> + => @success bx!(ManuallyDrop::new([UnsafeCell::new(0)])), bx!(ManuallyDrop::new([UnsafeCell::new(0), UnsafeCell::new(1)])); + ManuallyDrop<[UnsafeCell<bool>]> + => @success bx!(ManuallyDrop::new([UnsafeCell::new(false)])), bx!(ManuallyDrop::new([UnsafeCell::new(false), UnsafeCell::new(true)])), + @failure [2u8], [3u8], [0xFFu8], [0u8, 1u8, 2u8]; + Wrapping<bool> + => @success Wrapping(false), Wrapping(true), + @failure 2u8, 0xFFu8; + *const NotZerocopy + => @success ptr::null::<NotZerocopy>(), + @failure [0x01; mem::size_of::<*const NotZerocopy>()]; + *mut NotZerocopy + => @success ptr::null_mut::<NotZerocopy>(), + @failure [0x01; mem::size_of::<*mut NotZerocopy>()]; + ); + + // Use the trick described in [1] to allow us to call methods + // conditional on certain trait bounds. + // + // In all of these cases, methods return `Option<R>`, where `R` is the + // return type of the method we're conditionally calling. The "real" + // implementations (the ones defined in traits using `&self`) return + // `Some`, and the default implementations (the ones defined as inherent + // methods using `&mut self`) return `None`. + // + // [1] https://github.com/dtolnay/case-studies/blob/master/autoref-specialization/README.md + mod autoref_trick { + use super::*; + + pub(super) struct AutorefWrapper<T: ?Sized>(pub(super) PhantomData<T>); + + pub(super) trait TestIsBitValidShared<T: ?Sized> { + #[allow(clippy::needless_lifetimes)] + fn test_is_bit_valid_shared<'ptr>(&self, candidate: Maybe<'ptr, T>) + -> Option<bool>; + } + + impl<T: TryFromBytes + Immutable + ?Sized> TestIsBitValidShared<T> for AutorefWrapper<T> { + #[allow(clippy::needless_lifetimes)] + fn test_is_bit_valid_shared<'ptr>( + &self, + candidate: Maybe<'ptr, T>, + ) -> Option<bool> { + Some(T::is_bit_valid(candidate)) + } + } + + pub(super) trait TestTryFromRef<T: ?Sized> { + #[allow(clippy::needless_lifetimes)] + fn test_try_from_ref<'bytes>( + &self, + bytes: &'bytes [u8], + ) -> Option<Option<&'bytes T>>; + } + + impl<T: TryFromBytes + Immutable + KnownLayout + ?Sized> TestTryFromRef<T> for AutorefWrapper<T> { + #[allow(clippy::needless_lifetimes)] + fn test_try_from_ref<'bytes>( + &self, + bytes: &'bytes [u8], + ) -> Option<Option<&'bytes T>> { + Some(T::try_ref_from_bytes(bytes).ok()) + } + } + + pub(super) trait TestTryFromMut<T: ?Sized> { + #[allow(clippy::needless_lifetimes)] + fn test_try_from_mut<'bytes>( + &self, + bytes: &'bytes mut [u8], + ) -> Option<Option<&'bytes mut T>>; + } + + impl<T: TryFromBytes + IntoBytes + KnownLayout + ?Sized> TestTryFromMut<T> for AutorefWrapper<T> { + #[allow(clippy::needless_lifetimes)] + fn test_try_from_mut<'bytes>( + &self, + bytes: &'bytes mut [u8], + ) -> Option<Option<&'bytes mut T>> { + Some(T::try_mut_from_bytes(bytes).ok()) + } + } + + pub(super) trait TestTryReadFrom<T> { + fn test_try_read_from(&self, bytes: &[u8]) -> Option<Option<T>>; + } + + impl<T: TryFromBytes> TestTryReadFrom<T> for AutorefWrapper<T> { + fn test_try_read_from(&self, bytes: &[u8]) -> Option<Option<T>> { + Some(T::try_read_from_bytes(bytes).ok()) + } + } + + pub(super) trait TestAsBytes<T: ?Sized> { + #[allow(clippy::needless_lifetimes)] + fn test_as_bytes<'slf, 't>(&'slf self, t: &'t ReadOnly<T>) -> Option<&'t [u8]>; + } + + impl<T: IntoBytes + Immutable + ?Sized> TestAsBytes<T> for AutorefWrapper<T> { + #[allow(clippy::needless_lifetimes)] + fn test_as_bytes<'slf, 't>(&'slf self, t: &'t ReadOnly<T>) -> Option<&'t [u8]> { + Some(t.as_bytes()) + } + } + } + + use autoref_trick::*; + + // Asserts that `$ty` is one of a list of types which are allowed to not + // provide a "real" implementation for `$fn_name`. Since the + // `autoref_trick` machinery fails silently, this allows us to ensure + // that the "default" impls are only being used for types which we + // expect. + // + // Note that, since this is a runtime test, it is possible to have an + // allowlist which is too restrictive if the function in question is + // never called for a particular type. For example, if `as_bytes` is not + // supported for a particular type, and so `test_as_bytes` returns + // `None`, methods such as `test_try_from_ref` may never be called for + // that type. As a result, it's possible that, for example, adding + // `as_bytes` support for a type would cause other allowlist assertions + // to fail. This means that allowlist assertion failures should not + // automatically be taken as a sign of a bug. + macro_rules! assert_on_allowlist { + ($fn_name:ident($ty:ty) $(: $($tys:ty),*)?) => {{ + use core::any::TypeId; + + let allowlist: &[TypeId] = &[ $($(TypeId::of::<$tys>()),*)? ]; + let allowlist_names: &[&str] = &[ $($(stringify!($tys)),*)? ]; + + let id = TypeId::of::<$ty>(); + assert!(allowlist.contains(&id), "{} is not on allowlist for {}: {:?}", stringify!($ty), stringify!($fn_name), allowlist_names); + }}; + } + + // Asserts that `$ty` implements any `$trait` and doesn't implement any + // `!$trait`. Note that all `$trait`s must come before any `!$trait`s. + // + // For `T: TryFromBytes`, uses `TryFromBytesTestable` to test success + // and failure cases. + macro_rules! assert_impls { + ($ty:ty: TryFromBytes) => { + // "Default" implementations that match the "real" + // implementations defined in the `autoref_trick` module above. + #[allow(unused, non_local_definitions)] + impl AutorefWrapper<$ty> { + #[allow(clippy::needless_lifetimes)] + fn test_is_bit_valid_shared<'ptr>( + &mut self, + candidate: Maybe<'ptr, $ty>, + ) -> Option<bool> { + assert_on_allowlist!( + test_is_bit_valid_shared($ty): + ManuallyDrop<UnsafeCell<()>>, + ManuallyDrop<[UnsafeCell<u8>]>, + ManuallyDrop<[UnsafeCell<bool>]>, + CoreMaybeUninit<NotZerocopy>, + CoreMaybeUninit<UnsafeCell<()>>, + Wrapping<UnsafeCell<()>> + ); + + None + } + + #[allow(clippy::needless_lifetimes)] + fn test_try_from_ref<'bytes>(&mut self, _bytes: &'bytes [u8]) -> Option<Option<&'bytes $ty>> { + assert_on_allowlist!( + test_try_from_ref($ty): + ManuallyDrop<[UnsafeCell<bool>]> + ); + + None + } + + #[allow(clippy::needless_lifetimes)] + fn test_try_from_mut<'bytes>(&mut self, _bytes: &'bytes mut [u8]) -> Option<Option<&'bytes mut $ty>> { + assert_on_allowlist!( + test_try_from_mut($ty): + Option<Box<UnsafeCell<NotZerocopy>>>, + Option<&'static UnsafeCell<NotZerocopy>>, + Option<&'static mut UnsafeCell<NotZerocopy>>, + Option<NonNull<UnsafeCell<NotZerocopy>>>, + Option<fn()>, + Option<FnManyArgs>, + Option<extern "C" fn()>, + Option<ECFnManyArgs>, + *const NotZerocopy, + *mut NotZerocopy + ); + + None + } + + fn test_try_read_from(&mut self, _bytes: &[u8]) -> Option<Option<&$ty>> { + assert_on_allowlist!( + test_try_read_from($ty): + str, + ManuallyDrop<[u8]>, + ManuallyDrop<[bool]>, + ManuallyDrop<[UnsafeCell<bool>]>, + [u8], + [bool] + ); + + None + } + + fn test_as_bytes(&mut self, _t: &ReadOnly<$ty>) -> Option<&[u8]> { + assert_on_allowlist!( + test_as_bytes($ty): + Option<&'static UnsafeCell<NotZerocopy>>, + Option<&'static mut UnsafeCell<NotZerocopy>>, + Option<NonNull<UnsafeCell<NotZerocopy>>>, + Option<Box<UnsafeCell<NotZerocopy>>>, + Option<fn()>, + Option<FnManyArgs>, + Option<extern "C" fn()>, + Option<ECFnManyArgs>, + CoreMaybeUninit<u8>, + CoreMaybeUninit<NotZerocopy>, + CoreMaybeUninit<UnsafeCell<()>>, + ManuallyDrop<UnsafeCell<()>>, + ManuallyDrop<[UnsafeCell<u8>]>, + ManuallyDrop<[UnsafeCell<bool>]>, + Wrapping<UnsafeCell<()>>, + *const NotZerocopy, + *mut NotZerocopy + ); + + None + } + } + + <$ty as TryFromBytesTestable>::with_passing_test_cases(|mut val| { + // FIXME(#494): These tests only get exercised for types + // which are `IntoBytes`. Once we implement #494, we should + // be able to support non-`IntoBytes` types by zeroing + // padding. + + // We define `w` and `ww` since, in the case of the inherent + // methods, Rust thinks they're both borrowed mutably at the + // same time (given how we use them below). If we just + // defined a single `w` and used it for multiple operations, + // this would conflict. + // + // We `#[allow(unused_mut]` for the cases where the "real" + // impls are used, which take `&self`. + #[allow(unused_mut)] + let (mut w, mut ww) = (AutorefWrapper::<$ty>(PhantomData), AutorefWrapper::<$ty>(PhantomData)); + + let c = Ptr::from_ref(&*val); + let c = c.forget_aligned(); + // SAFETY: FIXME(#899): This is unsound. `$ty` is not + // necessarily `IntoBytes`, but that's the corner we've + // backed ourselves into by using `Ptr::from_ref`. + let c = unsafe { c.assume_initialized() }; + let res = w.test_is_bit_valid_shared(c); + if let Some(res) = res { + assert!(res, "{}::is_bit_valid (shared `Ptr`): got false, expected true", stringify!($ty)); + } + + let c = Ptr::from_mut(&mut *val); + let c = c.forget_aligned(); + // SAFETY: FIXME(#899): This is unsound. `$ty` is not + // necessarily `IntoBytes`, but that's the corner we've + // backed ourselves into by using `Ptr::from_ref`. + let mut c = unsafe { c.assume_initialized() }; + let res = <$ty as TryFromBytes>::is_bit_valid(c.reborrow_shared()); + assert!(res, "{}::is_bit_valid (exclusive `Ptr`): got false, expected true", stringify!($ty)); + + // `bytes` is `Some(val.as_bytes())` if `$ty: IntoBytes + + // Immutable` and `None` otherwise. + let bytes = w.test_as_bytes(&*val); + + // The inner closure returns + // `Some($ty::try_ref_from_bytes(bytes))` if `$ty: + // Immutable` and `None` otherwise. + let res = bytes.and_then(|bytes| ww.test_try_from_ref(bytes)); + if let Some(res) = res { + assert!(res.is_some(), "{}::try_ref_from_bytes: got `None`, expected `Some`", stringify!($ty)); + } + + if let Some(bytes) = bytes { + // We need to get a mutable byte slice, and so we clone + // into a `Vec`. However, we also need these bytes to + // satisfy `$ty`'s alignment requirement, which isn't + // guaranteed for `Vec<u8>`. In order to get around + // this, we create a `Vec` which is twice as long as we + // need. There is guaranteed to be an aligned byte range + // of size `size_of_val(val)` within that range. + let val = &*val; + let size = mem::size_of_val(val); + let align = mem::align_of_val(val); + + let mut vec = bytes.to_vec(); + vec.extend(bytes); + let slc = vec.as_slice(); + let offset = slc.as_ptr().align_offset(align); + let bytes_mut = &mut vec.as_mut_slice()[offset..offset+size]; + bytes_mut.copy_from_slice(bytes); + + let res = ww.test_try_from_mut(bytes_mut); + if let Some(res) = res { + assert!(res.is_some(), "{}::try_mut_from_bytes: got `None`, expected `Some`", stringify!($ty)); + } + } + + let res = bytes.and_then(|bytes| ww.test_try_read_from(bytes)); + if let Some(res) = res { + assert!(res.is_some(), "{}::try_read_from_bytes: got `None`, expected `Some`", stringify!($ty)); + } + }); + #[allow(clippy::as_conversions)] + <$ty as TryFromBytesTestable>::with_failing_test_cases(|c| { + #[allow(unused_mut)] // For cases where the "real" impls are used, which take `&self`. + let mut w = AutorefWrapper::<$ty>(PhantomData); + + // This is `Some($ty::try_ref_from_bytes(c))` if `$ty: + // Immutable` and `None` otherwise. + let res = w.test_try_from_ref(c); + if let Some(res) = res { + assert!(res.is_none(), "{}::try_ref_from_bytes({:?}): got Some, expected None", stringify!($ty), c); + } + + let res = w.test_try_from_mut(c); + if let Some(res) = res { + assert!(res.is_none(), "{}::try_mut_from_bytes({:?}): got Some, expected None", stringify!($ty), c); + } + + + let res = w.test_try_read_from(c); + if let Some(res) = res { + assert!(res.is_none(), "{}::try_read_from_bytes({:?}): got Some, expected None", stringify!($ty), c); + } + }); + + #[allow(dead_code)] + const _: () = { static_assertions::assert_impl_all!($ty: TryFromBytes); }; + }; + ($ty:ty: $trait:ident) => { + #[allow(dead_code)] + const _: () = { static_assertions::assert_impl_all!($ty: $trait); }; + }; + ($ty:ty: !$trait:ident) => { + #[allow(dead_code)] + const _: () = { static_assertions::assert_not_impl_any!($ty: $trait); }; + }; + ($ty:ty: $($trait:ident),* $(,)? $(!$negative_trait:ident),*) => { + $( + assert_impls!($ty: $trait); + )* + + $( + assert_impls!($ty: !$negative_trait); + )* + }; + } + + // NOTE: The negative impl assertions here are not necessarily + // prescriptive. They merely serve as change detectors to make sure + // we're aware of what trait impls are getting added with a given + // change. Of course, some impls would be invalid (e.g., `bool: + // FromBytes`), and so this change detection is very important. + + assert_impls!( + (): KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned + ); + assert_impls!( + u8: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned + ); + assert_impls!( + i8: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned + ); + assert_impls!( + u16: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + i16: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + u32: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + i32: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + u64: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + i64: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + u128: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + i128: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + usize: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + isize: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + #[cfg(feature = "float-nightly")] + assert_impls!( + f16: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + f32: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + f64: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + #[cfg(feature = "float-nightly")] + assert_impls!( + f128: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + bool: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + IntoBytes, + Unaligned, + !FromBytes + ); + assert_impls!( + char: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + str: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + IntoBytes, + Unaligned, + !FromBytes + ); + + assert_impls!( + NonZeroU8: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + Unaligned, + !FromZeros, + !FromBytes + ); + assert_impls!( + NonZeroI8: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + Unaligned, + !FromZeros, + !FromBytes + ); + assert_impls!( + NonZeroU16: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroI16: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroU32: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroI32: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroU64: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroI64: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroU128: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroI128: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroUsize: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroIsize: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + + assert_impls!(Option<NonZeroU8>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_impls!(Option<NonZeroI8>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_impls!(Option<NonZeroU16>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option<NonZeroI16>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option<NonZeroU32>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option<NonZeroI32>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option<NonZeroU64>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option<NonZeroI64>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option<NonZeroU128>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option<NonZeroI128>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option<NonZeroUsize>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option<NonZeroIsize>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + + // Implements none of the ZC traits. + struct NotZerocopy; + + #[rustfmt::skip] + type FnManyArgs = fn( + NotZerocopy, u8, u8, u8, u8, u8, u8, u8, u8, u8, u8, u8, + ) -> (NotZerocopy, NotZerocopy); + + // Allowed, because we're not actually using this type for FFI. + #[allow(improper_ctypes_definitions)] + #[rustfmt::skip] + type ECFnManyArgs = extern "C" fn( + NotZerocopy, u8, u8, u8, u8, u8, u8, u8, u8, u8, u8, u8, + ) -> (NotZerocopy, NotZerocopy); + + #[cfg(feature = "alloc")] + assert_impls!(Option<Box<UnsafeCell<NotZerocopy>>>: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<Box<[UnsafeCell<NotZerocopy>]>>: KnownLayout, !Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<&'static UnsafeCell<NotZerocopy>>: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<&'static [UnsafeCell<NotZerocopy>]>: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<&'static mut UnsafeCell<NotZerocopy>>: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<&'static mut [UnsafeCell<NotZerocopy>]>: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<NonNull<UnsafeCell<NotZerocopy>>>: KnownLayout, TryFromBytes, FromZeros, Immutable, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<NonNull<[UnsafeCell<NotZerocopy>]>>: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<fn()>: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<FnManyArgs>: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<extern "C" fn()>: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<ECFnManyArgs>: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + + assert_impls!(PhantomData<NotZerocopy>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_impls!(PhantomData<UnsafeCell<()>>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_impls!(PhantomData<[u8]>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + + assert_impls!(ManuallyDrop<u8>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + // This test is important because it allows us to test our hand-rolled + // implementation of `<ManuallyDrop<T> as TryFromBytes>::is_bit_valid`. + assert_impls!(ManuallyDrop<bool>: KnownLayout, Immutable, TryFromBytes, FromZeros, IntoBytes, Unaligned, !FromBytes); + assert_impls!(ManuallyDrop<[u8]>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + // This test is important because it allows us to test our hand-rolled + // implementation of `<ManuallyDrop<T> as TryFromBytes>::is_bit_valid`. + assert_impls!(ManuallyDrop<[bool]>: KnownLayout, Immutable, TryFromBytes, FromZeros, IntoBytes, Unaligned, !FromBytes); + assert_impls!(ManuallyDrop<NotZerocopy>: !Immutable, !TryFromBytes, !KnownLayout, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(ManuallyDrop<[NotZerocopy]>: KnownLayout, !Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(ManuallyDrop<UnsafeCell<()>>: KnownLayout, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned, !Immutable); + assert_impls!(ManuallyDrop<[UnsafeCell<u8>]>: KnownLayout, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned, !Immutable); + assert_impls!(ManuallyDrop<[UnsafeCell<bool>]>: KnownLayout, TryFromBytes, FromZeros, IntoBytes, Unaligned, !Immutable, !FromBytes); + + assert_impls!(CoreMaybeUninit<u8>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, Unaligned, !IntoBytes); + assert_impls!(CoreMaybeUninit<NotZerocopy>: KnownLayout, TryFromBytes, FromZeros, FromBytes, !Immutable, !IntoBytes, !Unaligned); + assert_impls!(CoreMaybeUninit<UnsafeCell<()>>: KnownLayout, TryFromBytes, FromZeros, FromBytes, Unaligned, !Immutable, !IntoBytes); + + assert_impls!(Wrapping<u8>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + // This test is important because it allows us to test our hand-rolled + // implementation of `<Wrapping<T> as TryFromBytes>::is_bit_valid`. + assert_impls!(Wrapping<bool>: KnownLayout, Immutable, TryFromBytes, FromZeros, IntoBytes, Unaligned, !FromBytes); + assert_impls!(Wrapping<NotZerocopy>: KnownLayout, !Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Wrapping<UnsafeCell<()>>: KnownLayout, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned, !Immutable); + + assert_impls!(Unalign<u8>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + // This test is important because it allows us to test our hand-rolled + // implementation of `<Unalign<T> as TryFromBytes>::is_bit_valid`. + assert_impls!(Unalign<bool>: KnownLayout, Immutable, TryFromBytes, FromZeros, IntoBytes, Unaligned, !FromBytes); + assert_impls!(Unalign<NotZerocopy>: KnownLayout, Unaligned, !Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes); + + assert_impls!( + [u8]: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned + ); + assert_impls!( + [bool]: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + IntoBytes, + Unaligned, + !FromBytes + ); + assert_impls!([NotZerocopy]: KnownLayout, !Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!( + [u8; 0]: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned, + ); + assert_impls!( + [NotZerocopy; 0]: KnownLayout, + !Immutable, + !TryFromBytes, + !FromZeros, + !FromBytes, + !IntoBytes, + !Unaligned + ); + assert_impls!( + [u8; 1]: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned, + ); + assert_impls!( + [NotZerocopy; 1]: KnownLayout, + !Immutable, + !TryFromBytes, + !FromZeros, + !FromBytes, + !IntoBytes, + !Unaligned + ); + + assert_impls!(*const NotZerocopy: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(*mut NotZerocopy: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(*const [NotZerocopy]: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(*mut [NotZerocopy]: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(*const dyn Debug: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(*mut dyn Debug: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + + #[cfg(feature = "simd")] + { + #[allow(unused_macros)] + macro_rules! test_simd_arch_mod { + ($arch:ident, $($typ:ident),*) => { + { + use core::arch::$arch::{$($typ),*}; + use crate::*; + $( assert_impls!($typ: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); )* + } + }; + } + #[cfg(target_arch = "x86")] + test_simd_arch_mod!(x86, __m128, __m128d, __m128i, __m256, __m256d, __m256i); + + #[cfg(all(not(no_zerocopy_simd_x86_avx12_1_89_0), target_arch = "x86"))] + test_simd_arch_mod!(x86, __m512bh, __m512, __m512d, __m512i); + + #[cfg(target_arch = "x86_64")] + test_simd_arch_mod!(x86_64, __m128, __m128d, __m128i, __m256, __m256d, __m256i); + + #[cfg(all(not(no_zerocopy_simd_x86_avx12_1_89_0), target_arch = "x86_64"))] + test_simd_arch_mod!(x86_64, __m512bh, __m512, __m512d, __m512i); + + #[cfg(target_arch = "wasm32")] + test_simd_arch_mod!(wasm32, v128); + + #[cfg(all(feature = "simd-nightly", target_arch = "powerpc"))] + test_simd_arch_mod!( + powerpc, + vector_bool_long, + vector_double, + vector_signed_long, + vector_unsigned_long + ); + + #[cfg(all(feature = "simd-nightly", target_arch = "powerpc64"))] + test_simd_arch_mod!( + powerpc64, + vector_bool_long, + vector_double, + vector_signed_long, + vector_unsigned_long + ); + #[cfg(all(target_arch = "aarch64", not(no_zerocopy_aarch64_simd_1_59_0)))] + #[rustfmt::skip] + test_simd_arch_mod!( + aarch64, float32x2_t, float32x4_t, float64x1_t, float64x2_t, int8x8_t, int8x8x2_t, + int8x8x3_t, int8x8x4_t, int8x16_t, int8x16x2_t, int8x16x3_t, int8x16x4_t, int16x4_t, + int16x8_t, int32x2_t, int32x4_t, int64x1_t, int64x2_t, poly8x8_t, poly8x8x2_t, poly8x8x3_t, + poly8x8x4_t, poly8x16_t, poly8x16x2_t, poly8x16x3_t, poly8x16x4_t, poly16x4_t, poly16x8_t, + poly64x1_t, poly64x2_t, uint8x8_t, uint8x8x2_t, uint8x8x3_t, uint8x8x4_t, uint8x16_t, + uint8x16x2_t, uint8x16x3_t, uint8x16x4_t, uint16x4_t, uint16x4x2_t, uint16x4x3_t, + uint16x4x4_t, uint16x8_t, uint32x2_t, uint32x4_t, uint64x1_t, uint64x2_t + ); + } + } +} diff --git a/rust/zerocopy/src/layout.rs b/rust/zerocopy/src/layout.rs new file mode 100644 index 000000000000..6015d0f2de52 --- /dev/null +++ b/rust/zerocopy/src/layout.rs @@ -0,0 +1,2225 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2024 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License <LICENSE-BSD or +// https://opensource.org/license/bsd-2-clause>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use core::{mem, num::NonZeroUsize}; + +use crate::util; + +/// The target pointer width, counted in bits. +const POINTER_WIDTH_BITS: usize = mem::size_of::<usize>() * 8; + +/// The layout of a type which might be dynamically-sized. +/// +/// `DstLayout` describes the layout of sized types, slice types, and "slice +/// DSTs" - ie, those that are known by the type system to have a trailing slice +/// (as distinguished from `dyn Trait` types - such types *might* have a +/// trailing slice type, but the type system isn't aware of it). +/// +/// Note that `DstLayout` does not have any internal invariants, so no guarantee +/// is made that a `DstLayout` conforms to any of Rust's requirements regarding +/// the layout of real Rust types or instances of types. +#[doc(hidden)] +#[allow(missing_debug_implementations, missing_copy_implementations)] +#[cfg_attr(any(kani, test), derive(Debug, PartialEq, Eq))] +#[derive(Copy, Clone)] +pub struct DstLayout { + pub(crate) align: NonZeroUsize, + pub(crate) size_info: SizeInfo, + // Is it guaranteed statically (without knowing a value's runtime metadata) + // that the top-level type contains no padding? This does *not* apply + // recursively - for example, `[(u8, u16)]` has `statically_shallow_unpadded + // = true` even though this type likely has padding inside each `(u8, u16)`. + pub(crate) statically_shallow_unpadded: bool, +} + +#[cfg_attr(any(kani, test), derive(Debug, PartialEq, Eq))] +#[derive(Copy, Clone)] +pub(crate) enum SizeInfo<E = usize> { + Sized { size: usize }, + SliceDst(TrailingSliceLayout<E>), +} + +#[cfg_attr(any(kani, test), derive(Debug, PartialEq, Eq))] +#[derive(Copy, Clone)] +pub(crate) struct TrailingSliceLayout<E = usize> { + // The offset of the first byte of the trailing slice field. Note that this + // is NOT the same as the minimum size of the type. For example, consider + // the following type: + // + // struct Foo { + // a: u16, + // b: u8, + // c: [u8], + // } + // + // In `Foo`, `c` is at byte offset 3. When `c.len() == 0`, `c` is followed + // by a padding byte. + pub(crate) offset: usize, + // The size of the element type of the trailing slice field. + pub(crate) elem_size: E, +} + +impl SizeInfo { + /// Attempts to create a `SizeInfo` from `Self` in which `elem_size` is a + /// `NonZeroUsize`. If `elem_size` is 0, returns `None`. + #[allow(unused)] + const fn try_to_nonzero_elem_size(&self) -> Option<SizeInfo<NonZeroUsize>> { + Some(match *self { + SizeInfo::Sized { size } => SizeInfo::Sized { size }, + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { + if let Some(elem_size) = NonZeroUsize::new(elem_size) { + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) + } else { + return None; + } + } + }) + } +} + +#[doc(hidden)] +#[derive(Copy, Clone)] +#[cfg_attr(test, derive(Debug))] +#[allow(missing_debug_implementations)] +pub enum CastType { + Prefix, + Suffix, +} + +#[cfg_attr(test, derive(Debug))] +pub(crate) enum MetadataCastError { + Alignment, + Size, +} + +impl DstLayout { + /// The minimum possible alignment of a type. + const MIN_ALIGN: NonZeroUsize = match NonZeroUsize::new(1) { + Some(min_align) => min_align, + None => const_unreachable!(), + }; + + /// The maximum theoretic possible alignment of a type. + /// + /// For compatibility with future Rust versions, this is defined as the + /// maximum power-of-two that fits into a `usize`. See also + /// [`DstLayout::CURRENT_MAX_ALIGN`]. + pub(crate) const THEORETICAL_MAX_ALIGN: NonZeroUsize = + match NonZeroUsize::new(1 << (POINTER_WIDTH_BITS - 1)) { + Some(max_align) => max_align, + None => const_unreachable!(), + }; + + /// The current, documented max alignment of a type \[1\]. + /// + /// \[1\] Per <https://doc.rust-lang.org/reference/type-layout.html#the-alignment-modifiers>: + /// + /// The alignment value must be a power of two from 1 up to + /// 2<sup>29</sup>. + #[cfg(not(kani))] + #[cfg(not(target_pointer_width = "16"))] + pub(crate) const CURRENT_MAX_ALIGN: NonZeroUsize = match NonZeroUsize::new(1 << 28) { + Some(max_align) => max_align, + None => const_unreachable!(), + }; + + #[cfg(not(kani))] + #[cfg(target_pointer_width = "16")] + pub(crate) const CURRENT_MAX_ALIGN: NonZeroUsize = match NonZeroUsize::new(1 << 15) { + Some(max_align) => max_align, + None => const_unreachable!(), + }; + + /// The maximum size of an allocation \[1\]. + /// + /// \[1\] Per <https://doc.rust-lang.org/1.91.1/std/ptr/index.html#allocation>: + /// + /// For any allocation with base `address`, `size`, and a set of `addresses`, + /// the following are guaranteed: [..] + /// + /// - `size <= isize::MAX` + /// + #[allow(clippy::as_conversions)] + pub(crate) const MAX_SIZE: usize = isize::MAX as usize; + + /// Assumes that this layout lacks static shallow padding. + /// + /// # Panics + /// + /// This method does not panic. + /// + /// # Safety + /// + /// If `self` describes the size and alignment of type that lacks static + /// shallow padding, unsafe code may assume that the result of this method + /// accurately reflects the size, alignment, and lack of static shallow + /// padding of that type. + const fn assume_shallow_unpadded(self) -> Self { + Self { statically_shallow_unpadded: true, ..self } + } + + /// Constructs a `DstLayout` for a zero-sized type with `repr_align` + /// alignment (or 1). If `repr_align` is provided, then it must be a power + /// of two. + /// + /// # Panics + /// + /// This function panics if the supplied `repr_align` is not a power of two. + /// + /// # Safety + /// + /// Unsafe code may assume that the contract of this function is satisfied. + #[doc(hidden)] + #[must_use] + #[inline] + pub const fn new_zst(repr_align: Option<NonZeroUsize>) -> DstLayout { + let align = match repr_align { + Some(align) => align, + None => Self::MIN_ALIGN, + }; + + const_assert!(align.get().is_power_of_two()); + + DstLayout { + align, + size_info: SizeInfo::Sized { size: 0 }, + statically_shallow_unpadded: true, + } + } + + /// Constructs a `DstLayout` which describes `T` and assumes `T` may contain + /// padding. + /// + /// # Safety + /// + /// Unsafe code may assume that `DstLayout` is the correct layout for `T`. + #[doc(hidden)] + #[must_use] + #[inline] + pub const fn for_type<T>() -> DstLayout { + // SAFETY: `align` is correct by construction. `T: Sized`, and so it is + // sound to initialize `size_info` to `SizeInfo::Sized { size }`; the + // `size` field is also correct by construction. `unpadded` can safely + // default to `false`. + DstLayout { + align: match NonZeroUsize::new(mem::align_of::<T>()) { + Some(align) => align, + None => const_unreachable!(), + }, + size_info: SizeInfo::Sized { size: mem::size_of::<T>() }, + statically_shallow_unpadded: false, + } + } + + /// Constructs a `DstLayout` which describes a `T` that does not contain + /// padding. + /// + /// # Safety + /// + /// Unsafe code may assume that `DstLayout` is the correct layout for `T`. + #[doc(hidden)] + #[must_use] + #[inline] + pub const fn for_unpadded_type<T>() -> DstLayout { + Self::for_type::<T>().assume_shallow_unpadded() + } + + /// Constructs a `DstLayout` which describes `[T]`. + /// + /// # Safety + /// + /// Unsafe code may assume that `DstLayout` is the correct layout for `[T]`. + pub(crate) const fn for_slice<T>() -> DstLayout { + // SAFETY: The alignment of a slice is equal to the alignment of its + // element type, and so `align` is initialized correctly. + // + // Since this is just a slice type, there is no offset between the + // beginning of the type and the beginning of the slice, so it is + // correct to set `offset: 0`. The `elem_size` is correct by + // construction. Since `[T]` is a (degenerate case of a) slice DST, it + // is correct to initialize `size_info` to `SizeInfo::SliceDst`. + DstLayout { + align: match NonZeroUsize::new(mem::align_of::<T>()) { + Some(align) => align, + None => const_unreachable!(), + }, + size_info: SizeInfo::SliceDst(TrailingSliceLayout { + offset: 0, + elem_size: mem::size_of::<T>(), + }), + statically_shallow_unpadded: true, + } + } + + /// Constructs a complete `DstLayout` reflecting a `repr(C)` struct with the + /// given alignment modifiers and fields. + /// + /// This method cannot be used to match the layout of a record with the + /// default representation, as that representation is mostly unspecified. + /// + /// # Safety + /// + /// For any definition of a `repr(C)` struct, if this method is invoked with + /// alignment modifiers and fields corresponding to that definition, the + /// resulting `DstLayout` will correctly encode the layout of that struct. + /// + /// We make no guarantees to the behavior of this method when it is invoked + /// with arguments that cannot correspond to a valid `repr(C)` struct. + #[must_use] + #[inline] + pub const fn for_repr_c_struct( + repr_align: Option<NonZeroUsize>, + repr_packed: Option<NonZeroUsize>, + fields: &[DstLayout], + ) -> DstLayout { + let mut layout = DstLayout::new_zst(repr_align); + + let mut i = 0; + #[allow(clippy::arithmetic_side_effects)] + while i < fields.len() { + #[allow(clippy::indexing_slicing)] + let field = fields[i]; + layout = layout.extend(field, repr_packed); + i += 1; + } + + layout = layout.pad_to_align(); + + // SAFETY: `layout` accurately describes the layout of a `repr(C)` + // struct with `repr_align` or `repr_packed` alignment modifications and + // the given `fields`. The `layout` is constructed using a sequence of + // invocations of `DstLayout::{new_zst,extend,pad_to_align}`. The + // documentation of these items vows that invocations in this manner + // will accurately describe a type, so long as: + // + // - that type is `repr(C)`, + // - its fields are enumerated in the order they appear, + // - the presence of `repr_align` and `repr_packed` are correctly accounted for. + // + // We respect all three of these preconditions above. + layout + } + + /// Like `Layout::extend`, this creates a layout that describes a record + /// whose layout consists of `self` followed by `next` that includes the + /// necessary inter-field padding, but not any trailing padding. + /// + /// In order to match the layout of a `#[repr(C)]` struct, this method + /// should be invoked for each field in declaration order. To add trailing + /// padding, call `DstLayout::pad_to_align` after extending the layout for + /// all fields. If `self` corresponds to a type marked with + /// `repr(packed(N))`, then `repr_packed` should be set to `Some(N)`, + /// otherwise `None`. + /// + /// This method cannot be used to match the layout of a record with the + /// default representation, as that representation is mostly unspecified. + /// + /// # Safety + /// + /// If a (potentially hypothetical) valid `repr(C)` Rust type begins with + /// fields whose layout are `self`, and those fields are immediately + /// followed by a field whose layout is `field`, then unsafe code may rely + /// on `self.extend(field, repr_packed)` producing a layout that correctly + /// encompasses those two components. + /// + /// We make no guarantees to the behavior of this method if these fragments + /// cannot appear in a valid Rust type (e.g., the concatenation of the + /// layouts would lead to a size larger than `isize::MAX`). + #[doc(hidden)] + #[must_use] + #[inline] + pub const fn extend(self, field: DstLayout, repr_packed: Option<NonZeroUsize>) -> Self { + use util::{max, min, padding_needed_for}; + + // If `repr_packed` is `None`, there are no alignment constraints, and + // the value can be defaulted to `THEORETICAL_MAX_ALIGN`. + let max_align = match repr_packed { + Some(max_align) => max_align, + None => Self::THEORETICAL_MAX_ALIGN, + }; + + const_assert!(max_align.get().is_power_of_two()); + + // We use Kani to prove that this method is robust to future increases + // in Rust's maximum allowed alignment. However, if such a change ever + // actually occurs, we'd like to be notified via assertion failures. + #[cfg(not(kani))] + { + const_debug_assert!(self.align.get() <= DstLayout::CURRENT_MAX_ALIGN.get()); + const_debug_assert!(field.align.get() <= DstLayout::CURRENT_MAX_ALIGN.get()); + if let Some(repr_packed) = repr_packed { + const_debug_assert!(repr_packed.get() <= DstLayout::CURRENT_MAX_ALIGN.get()); + } + } + + // The field's alignment is clamped by `repr_packed` (i.e., the + // `repr(packed(N))` attribute, if any) [1]. + // + // [1] Per https://doc.rust-lang.org/reference/type-layout.html#the-alignment-modifiers: + // + // The alignments of each field, for the purpose of positioning + // fields, is the smaller of the specified alignment and the alignment + // of the field's type. + let field_align = min(field.align, max_align); + + // The struct's alignment is the maximum of its previous alignment and + // `field_align`. + let align = max(self.align, field_align); + + let (interfield_padding, size_info) = match self.size_info { + // If the layout is already a DST, we panic; DSTs cannot be extended + // with additional fields. + SizeInfo::SliceDst(..) => const_panic!("Cannot extend a DST with additional fields."), + + SizeInfo::Sized { size: preceding_size } => { + // Compute the minimum amount of inter-field padding needed to + // satisfy the field's alignment, and offset of the trailing + // field. [1] + // + // [1] Per https://doc.rust-lang.org/reference/type-layout.html#the-alignment-modifiers: + // + // Inter-field padding is guaranteed to be the minimum + // required in order to satisfy each field's (possibly + // altered) alignment. + let padding = padding_needed_for(preceding_size, field_align); + + // This will not panic (and is proven to not panic, with Kani) + // if the layout components can correspond to a leading layout + // fragment of a valid Rust type, but may panic otherwise (e.g., + // combining or aligning the components would create a size + // exceeding `isize::MAX`). + let offset = match preceding_size.checked_add(padding) { + Some(offset) => offset, + None => const_panic!("Adding padding to `self`'s size overflows `usize`."), + }; + + ( + padding, + match field.size_info { + SizeInfo::Sized { size: field_size } => { + // If the trailing field is sized, the resulting layout + // will be sized. Its size will be the sum of the + // preceding layout, the size of the new field, and the + // size of inter-field padding between the two. + // + // This will not panic (and is proven with Kani to not + // panic) if the layout components can correspond to a + // leading layout fragment of a valid Rust type, but may + // panic otherwise (e.g., combining or aligning the + // components would create a size exceeding + // `usize::MAX`). + let size = match offset.checked_add(field_size) { + Some(size) => size, + None => const_panic!("`field` cannot be appended without the total size overflowing `usize`"), + }; + SizeInfo::Sized { size } + } + SizeInfo::SliceDst(TrailingSliceLayout { + offset: trailing_offset, + elem_size, + }) => { + // If the trailing field is dynamically sized, so too + // will the resulting layout. The offset of the trailing + // slice component is the sum of the offset of the + // trailing field and the trailing slice offset within + // that field. + // + // This will not panic (and is proven with Kani to not + // panic) if the layout components can correspond to a + // leading layout fragment of a valid Rust type, but may + // panic otherwise (e.g., combining or aligning the + // components would create a size exceeding + // `usize::MAX`). + let offset = match offset.checked_add(trailing_offset) { + Some(offset) => offset, + None => const_panic!("`field` cannot be appended without the total size overflowing `usize`"), + }; + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) + } + }, + ) + } + }; + + let statically_shallow_unpadded = self.statically_shallow_unpadded + && field.statically_shallow_unpadded + && interfield_padding == 0; + + DstLayout { align, size_info, statically_shallow_unpadded } + } + + /// Like `Layout::pad_to_align`, this routine rounds the size of this layout + /// up to the nearest multiple of this type's alignment or `repr_packed` + /// (whichever is less). This method leaves DST layouts unchanged, since the + /// trailing padding of DSTs is computed at runtime. + /// + /// The accompanying boolean is `true` if the resulting composition of + /// fields necessitated static (as opposed to dynamic) padding; otherwise + /// `false`. + /// + /// In order to match the layout of a `#[repr(C)]` struct, this method + /// should be invoked after the invocations of [`DstLayout::extend`]. If + /// `self` corresponds to a type marked with `repr(packed(N))`, then + /// `repr_packed` should be set to `Some(N)`, otherwise `None`. + /// + /// This method cannot be used to match the layout of a record with the + /// default representation, as that representation is mostly unspecified. + /// + /// # Safety + /// + /// If a (potentially hypothetical) valid `repr(C)` type begins with fields + /// whose layout are `self` followed only by zero or more bytes of trailing + /// padding (not included in `self`), then unsafe code may rely on + /// `self.pad_to_align(repr_packed)` producing a layout that correctly + /// encapsulates the layout of that type. + /// + /// We make no guarantees to the behavior of this method if `self` cannot + /// appear in a valid Rust type (e.g., because the addition of trailing + /// padding would lead to a size larger than `isize::MAX`). + #[doc(hidden)] + #[must_use] + #[inline] + pub const fn pad_to_align(self) -> Self { + use util::padding_needed_for; + + let (static_padding, size_info) = match self.size_info { + // For sized layouts, we add the minimum amount of trailing padding + // needed to satisfy alignment. + SizeInfo::Sized { size: unpadded_size } => { + let padding = padding_needed_for(unpadded_size, self.align); + let size = match unpadded_size.checked_add(padding) { + Some(size) => size, + None => const_panic!("Adding padding caused size to overflow `usize`."), + }; + (padding, SizeInfo::Sized { size }) + } + // For DST layouts, trailing padding depends on the length of the + // trailing DST and is computed at runtime. This does not alter the + // offset or element size of the layout, so we leave `size_info` + // unchanged. + size_info @ SizeInfo::SliceDst(_) => (0, size_info), + }; + + let statically_shallow_unpadded = self.statically_shallow_unpadded && static_padding == 0; + + DstLayout { align: self.align, size_info, statically_shallow_unpadded } + } + + /// Produces `true` if `self` requires static padding; otherwise `false`. + #[must_use] + #[inline(always)] + pub const fn requires_static_padding(self) -> bool { + !self.statically_shallow_unpadded + } + + /// Produces `true` if there exists any metadata for which a type of layout + /// `self` would require dynamic trailing padding; otherwise `false`. + #[must_use] + #[inline(always)] + pub const fn requires_dynamic_padding(self) -> bool { + // A `% self.align.get()` cannot panic, since `align` is non-zero. + #[allow(clippy::arithmetic_side_effects)] + match self.size_info { + SizeInfo::Sized { .. } => false, + SizeInfo::SliceDst(trailing_slice_layout) => { + // SAFETY: This predicate is formally proved sound by + // `proofs::prove_requires_dynamic_padding`. + trailing_slice_layout.offset % self.align.get() != 0 + || trailing_slice_layout.elem_size % self.align.get() != 0 + } + } + } + + /// Validates that a cast is sound from a layout perspective. + /// + /// Validates that the size and alignment requirements of a type with the + /// layout described in `self` would not be violated by performing a + /// `cast_type` cast from a pointer with address `addr` which refers to a + /// memory region of size `bytes_len`. + /// + /// If the cast is valid, `validate_cast_and_convert_metadata` returns + /// `(elems, split_at)`. If `self` describes a dynamically-sized type, then + /// `elems` is the maximum number of trailing slice elements for which a + /// cast would be valid (for sized types, `elem` is meaningless and should + /// be ignored). `split_at` is the index at which to split the memory region + /// in order for the prefix (suffix) to contain the result of the cast, and + /// in order for the remaining suffix (prefix) to contain the leftover + /// bytes. + /// + /// There are three conditions under which a cast can fail: + /// - The smallest possible value for the type is larger than the provided + /// memory region + /// - A prefix cast is requested, and `addr` does not satisfy `self`'s + /// alignment requirement + /// - A suffix cast is requested, and `addr + bytes_len` does not satisfy + /// `self`'s alignment requirement (as a consequence, since all instances + /// of the type are a multiple of its alignment, no size for the type will + /// result in a starting address which is properly aligned) + /// + /// # Safety + /// + /// The caller may assume that this implementation is correct, and may rely + /// on that assumption for the soundness of their code. In particular, the + /// caller may assume that, if `validate_cast_and_convert_metadata` returns + /// `Some((elems, split_at))`, then: + /// - A pointer to the type (for dynamically sized types, this includes + /// `elems` as its pointer metadata) describes an object of size `size <= + /// bytes_len` + /// - If this is a prefix cast: + /// - `addr` satisfies `self`'s alignment + /// - `size == split_at` + /// - If this is a suffix cast: + /// - `split_at == bytes_len - size` + /// - `addr + split_at` satisfies `self`'s alignment + /// + /// Note that this method does *not* ensure that a pointer constructed from + /// its return values will be a valid pointer. In particular, this method + /// does not reason about `isize` overflow, which is a requirement of many + /// Rust pointer APIs, and may at some point be determined to be a validity + /// invariant of pointer types themselves. This should never be a problem so + /// long as the arguments to this method are derived from a known-valid + /// pointer (e.g., one derived from a safe Rust reference), but it is + /// nonetheless the caller's responsibility to justify that pointer + /// arithmetic will not overflow based on a safety argument *other than* the + /// mere fact that this method returned successfully. + /// + /// # Panics + /// + /// `validate_cast_and_convert_metadata` will panic if `self` describes a + /// DST whose trailing slice element is zero-sized. + /// + /// If `addr + bytes_len` overflows `usize`, + /// `validate_cast_and_convert_metadata` may panic, or it may return + /// incorrect results. No guarantees are made about when + /// `validate_cast_and_convert_metadata` will panic. The caller should not + /// rely on `validate_cast_and_convert_metadata` panicking in any particular + /// condition, even if `debug_assertions` are enabled. + #[allow(unused)] + #[inline(always)] + pub(crate) const fn validate_cast_and_convert_metadata( + &self, + addr: usize, + bytes_len: usize, + cast_type: CastType, + ) -> Result<(usize, usize), MetadataCastError> { + // `debug_assert!`, but with `#[allow(clippy::arithmetic_side_effects)]`. + macro_rules! __const_debug_assert { + ($e:expr $(, $msg:expr)?) => { + const_debug_assert!({ + #[allow(clippy::arithmetic_side_effects)] + let e = $e; + e + } $(, $msg)?); + }; + } + + // Note that, in practice, `self` is always a compile-time constant. We + // do this check earlier than needed to ensure that we always panic as a + // result of bugs in the program (such as calling this function on an + // invalid type) instead of allowing this panic to be hidden if the cast + // would have failed anyway for runtime reasons (such as a too-small + // memory region). + // + // FIXME(#67): Once our MSRV is 1.65, use let-else: + // https://blog.rust-lang.org/2022/11/03/Rust-1.65.0.html#let-else-statements + let size_info = match self.size_info.try_to_nonzero_elem_size() { + Some(size_info) => size_info, + None => const_panic!("attempted to cast to slice type with zero-sized element"), + }; + + // Precondition + __const_debug_assert!( + addr.checked_add(bytes_len).is_some(), + "`addr` + `bytes_len` > usize::MAX" + ); + + // Alignment checks go in their own block to avoid introducing variables + // into the top-level scope. + { + // We check alignment for `addr` (for prefix casts) or `addr + + // bytes_len` (for suffix casts). For a prefix cast, the correctness + // of this check is trivial - `addr` is the address the object will + // live at. + // + // For a suffix cast, we know that all valid sizes for the type are + // a multiple of the alignment (and by safety precondition, we know + // `DstLayout` may only describe valid Rust types). Thus, a + // validly-sized instance which lives at a validly-aligned address + // must also end at a validly-aligned address. Thus, if the end + // address for a suffix cast (`addr + bytes_len`) is not aligned, + // then no valid start address will be aligned either. + let offset = match cast_type { + CastType::Prefix => 0, + CastType::Suffix => bytes_len, + }; + + // Addition is guaranteed not to overflow because `offset <= + // bytes_len`, and `addr + bytes_len <= usize::MAX` is a + // precondition of this method. Modulus is guaranteed not to divide + // by 0 because `align` is non-zero. + #[allow(clippy::arithmetic_side_effects)] + if (addr + offset) % self.align.get() != 0 { + return Err(MetadataCastError::Alignment); + } + } + + let (elems, self_bytes) = match size_info { + SizeInfo::Sized { size } => { + if size > bytes_len { + return Err(MetadataCastError::Size); + } + (0, size) + } + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { + // Calculate the maximum number of bytes that could be consumed + // - any number of bytes larger than this will either not be a + // multiple of the alignment, or will be larger than + // `bytes_len`. + let max_total_bytes = + util::round_down_to_next_multiple_of_alignment(bytes_len, self.align); + // Calculate the maximum number of bytes that could be consumed + // by the trailing slice. + // + // FIXME(#67): Once our MSRV is 1.65, use let-else: + // https://blog.rust-lang.org/2022/11/03/Rust-1.65.0.html#let-else-statements + let max_slice_and_padding_bytes = match max_total_bytes.checked_sub(offset) { + Some(max) => max, + // `bytes_len` too small even for 0 trailing slice elements. + None => return Err(MetadataCastError::Size), + }; + + // Calculate the number of elements that fit in + // `max_slice_and_padding_bytes`; any remaining bytes will be + // considered padding. + // + // Guaranteed not to divide by zero: `elem_size` is non-zero. + #[allow(clippy::arithmetic_side_effects)] + let elems = max_slice_and_padding_bytes / elem_size.get(); + // Guaranteed not to overflow on multiplication: `usize::MAX >= + // max_slice_and_padding_bytes >= (max_slice_and_padding_bytes / + // elem_size) * elem_size`. + // + // Guaranteed not to overflow on addition: + // - max_slice_and_padding_bytes == max_total_bytes - offset + // - elems * elem_size <= max_slice_and_padding_bytes == max_total_bytes - offset + // - elems * elem_size + offset <= max_total_bytes <= usize::MAX + #[allow(clippy::arithmetic_side_effects)] + let without_padding = offset + elems * elem_size.get(); + // `self_bytes` is equal to the offset bytes plus the bytes + // consumed by the trailing slice plus any padding bytes + // required to satisfy the alignment. Note that we have computed + // the maximum number of trailing slice elements that could fit + // in `self_bytes`, so any padding is guaranteed to be less than + // the size of an extra element. + // + // Guaranteed not to overflow: + // - By previous comment: without_padding == elems * elem_size + + // offset <= max_total_bytes + // - By construction, `max_total_bytes` is a multiple of + // `self.align`. + // - At most, adding padding needed to round `without_padding` + // up to the next multiple of the alignment will bring + // `self_bytes` up to `max_total_bytes`. + #[allow(clippy::arithmetic_side_effects)] + let self_bytes = + without_padding + util::padding_needed_for(without_padding, self.align); + (elems, self_bytes) + } + }; + + __const_debug_assert!(self_bytes <= bytes_len); + + let split_at = match cast_type { + CastType::Prefix => self_bytes, + // Guaranteed not to underflow: + // - In the `Sized` branch, only returns `size` if `size <= + // bytes_len`. + // - In the `SliceDst` branch, calculates `self_bytes <= + // max_toatl_bytes`, which is upper-bounded by `bytes_len`. + #[allow(clippy::arithmetic_side_effects)] + CastType::Suffix => bytes_len - self_bytes, + }; + + Ok((elems, split_at)) + } +} + +pub(crate) use cast_from::CastFrom; +mod cast_from { + use crate::*; + + pub(crate) struct CastFrom<Dst: ?Sized> { + _never: core::convert::Infallible, + _marker: PhantomData<Dst>, + } + + // SAFETY: The implementation of `Project::project` preserves the address + // of the referent – it only modifies pointer metadata. + unsafe impl<Src, Dst> crate::pointer::cast::Cast<Src, Dst> for CastFrom<Dst> + where + Src: KnownLayout + ?Sized, + Dst: KnownLayout + ?Sized, + { + } + + // SAFETY: The implementation of `Project::project` preserves the size of + // the referent (see inline comments for a more detailed proof of this). + unsafe impl<Src, Dst> crate::pointer::cast::CastExact<Src, Dst> for CastFrom<Dst> + where + Src: KnownLayout + ?Sized, + Dst: KnownLayout + ?Sized, + { + } + + // SAFETY: `project` produces a pointer which refers to the same referent + // bytes as its input, or to a subset of them (see inline comments for a + // more detailed proof of this). It does this using provenance-preserving + // operations. + unsafe impl<Src, Dst> crate::pointer::cast::Project<Src, Dst> for CastFrom<Dst> + where + Src: KnownLayout + ?Sized, + Dst: KnownLayout + ?Sized, + { + /// # PME + /// + /// Generates a post-monomorphization error if it is not possible to + /// implement soundly. + // + // FIXME(#1817): Support Sized->Unsized and Unsized->Sized casts + fn project(src: PtrInner<'_, Src>) -> *mut Dst { + /// The parameters required in order to perform a pointer cast from + /// `Src` to `Dst`. + /// + /// These are a compile-time function of the layouts of `Src` + /// and `Dst`. + /// + /// # Safety + /// + /// `Src`'s alignment must not be smaller than `Dst`'s alignment. + struct CastParams<Src: ?Sized, Dst: ?Sized> { + inner: CastParamsInner, + _src: PhantomData<Src>, + _dst: PhantomData<Dst>, + } + + #[derive(Copy, Clone)] + enum CastParamsInner { + // At compile time (specifically, post-monomorphization time), + // we need to compute two things: + // - Whether, given *any* `*Src`, it is possible to construct a + // `*Dst` which addresses the same number of bytes (ie, + // whether, for any `Src` pointer metadata, there exists `Dst` + // pointer metadata that addresses the same number of bytes) + // - If this is possible, any information necessary to perform + // the `Src`->`Dst` metadata conversion at runtime. + // + // Assume that `Src` and `Dst` are slice DSTs, and define: + // - `S_OFF = Src::LAYOUT.size_info.offset` + // - `S_ELEM = Src::LAYOUT.size_info.elem_size` + // - `D_OFF = Dst::LAYOUT.size_info.offset` + // - `D_ELEM = Dst::LAYOUT.size_info.elem_size` + // + // We are trying to solve the following equation: + // + // D_OFF + d_meta * D_ELEM = S_OFF + s_meta * S_ELEM + // + // At runtime, we will be attempting to compute `d_meta`, given + // `s_meta` (a runtime value) and all other parameters (which + // are compile-time values). We can solve like so: + // + // D_OFF + d_meta * D_ELEM = S_OFF + s_meta * S_ELEM + // + // d_meta * D_ELEM = S_OFF - D_OFF + s_meta * S_ELEM + // + // d_meta = (S_OFF - D_OFF + s_meta * S_ELEM)/D_ELEM + // + // Since `d_meta` will be a `usize`, we need the right-hand side + // to be an integer, and this needs to hold for *any* value of + // `s_meta` (in order for our conversion to be infallible - ie, + // to not have to reject certain values of `s_meta` at runtime). + // This means that: + // + // - `s_meta * S_ELEM` must be a multiple of `D_ELEM` + // - Since this must hold for any value of `s_meta`, `S_ELEM` + // must be a multiple of `D_ELEM` + // - `S_OFF - D_OFF` must be a multiple of `D_ELEM` + // + // Thus, let `OFFSET_DELTA_ELEMS = (S_OFF - D_OFF)/D_ELEM` and + // `ELEM_MULTIPLE = S_ELEM/D_ELEM`. We can rewrite the above + // expression as: + // + // d_meta = (S_OFF - D_OFF + s_meta * S_ELEM)/D_ELEM + // + // d_meta = OFFSET_DELTA_ELEMS + s_meta * ELEM_MULTIPLE + // + // Thus, we just need to compute the following and confirm that + // they have integer solutions in order to both a) determine + // whether infallible `Src` -> `Dst` casts are possible and, b) + // pre-compute the parameters necessary to perform those casts + // at runtime. These parameters are encapsulated in + // `CastParams`, which acts as a witness that such infallible + // casts are possible. + /// The parameters required in order to perform an + /// unsized-to-unsized pointer cast from `Src` to `Dst` as + /// described above. + /// + /// # Safety + /// + /// `Src` and `Dst` must both be slice DSTs. + /// + /// `offset_delta_elems` and `elem_multiple` must be valid as + /// described above. + UnsizedToUnsized { offset_delta_elems: usize, elem_multiple: usize }, + + /// The metadata of a `Dst` which has the same size as `Src: + /// Sized`. + /// + /// # Safety + /// + /// `Src: Sized` and `Dst` must be a slice DST. + /// + /// A raw `Dst` pointer with metadata `dst_meta` must address + /// `size_of::<Src>()` bytes. + SizedToUnsized { dst_meta: usize }, + + /// The metadata of a `Dst` which has the same size as `Src: + /// Sized`. + /// + /// # Safety + /// + /// `Src` and `Dst` must both be `Sized` and `size_of::<Src>() + /// == size_of::<Dst>()`. + SizedToSized, + } + + impl<Src: ?Sized, Dst: ?Sized> Copy for CastParams<Src, Dst> {} + impl<Src: ?Sized, Dst: ?Sized> Clone for CastParams<Src, Dst> { + fn clone(&self) -> Self { + *self + } + } + + impl<Src: ?Sized, Dst: ?Sized> CastParams<Src, Dst> { + const fn try_compute( + src: &DstLayout, + dst: &DstLayout, + ) -> Option<CastParams<Src, Dst>> { + if src.align.get() < dst.align.get() { + return None; + } + + let inner = match (src.size_info, dst.size_info) { + ( + SizeInfo::Sized { size: src_size }, + SizeInfo::Sized { size: dst_size }, + ) => { + if src_size != dst_size { + return None; + } + + // SAFETY: We checked above that `src_size == + // dst_size`. + CastParamsInner::SizedToSized + } + (SizeInfo::Sized { size: src_size }, SizeInfo::SliceDst(dst)) => { + let offset_delta = if let Some(od) = src_size.checked_sub(dst.offset) { + od + } else { + return None; + }; + + let dst_elem_size = if let Some(e) = NonZeroUsize::new(dst.elem_size) { + e + } else { + return None; + }; + + // PANICS: `dst_elem_size: NonZeroUsize`, so this won't + // divide by zero. + #[allow(clippy::arithmetic_side_effects)] + let delta_mod_other_elem = offset_delta % dst_elem_size.get(); + + if delta_mod_other_elem != 0 { + return None; + } + + // PANICS: `dst_elem_size: NonZeroUsize`, so this won't + // divide by zero. + #[allow(clippy::arithmetic_side_effects)] + let dst_meta = offset_delta / dst_elem_size.get(); + + // SAFETY: The preceding math ensures that a `Dst` + // with `dst_meta` addresses `src_size` bytes. + CastParamsInner::SizedToUnsized { dst_meta } + } + (SizeInfo::SliceDst(src), SizeInfo::SliceDst(dst)) => { + let offset_delta = if let Some(od) = src.offset.checked_sub(dst.offset) + { + od + } else { + return None; + }; + + let dst_elem_size = if let Some(e) = NonZeroUsize::new(dst.elem_size) { + e + } else { + return None; + }; + + // PANICS: `dst_elem_size: NonZeroUsize`, so this won't + // divide by zero. + #[allow(clippy::arithmetic_side_effects)] + let delta_mod_other_elem = offset_delta % dst_elem_size.get(); + + // PANICS: `dst_elem_size: NonZeroUsize`, so this won't + // divide by zero. + #[allow(clippy::arithmetic_side_effects)] + let elem_remainder = src.elem_size % dst_elem_size.get(); + + if delta_mod_other_elem != 0 + || src.elem_size < dst.elem_size + || elem_remainder != 0 + { + return None; + } + + // PANICS: `dst_elem_size: NonZeroUsize`, so this won't + // divide by zero. + #[allow(clippy::arithmetic_side_effects)] + let offset_delta_elems = offset_delta / dst_elem_size.get(); + + // PANICS: `dst_elem_size: NonZeroUsize`, so this won't + // divide by zero. + #[allow(clippy::arithmetic_side_effects)] + let elem_multiple = src.elem_size / dst_elem_size.get(); + + CastParamsInner::UnsizedToUnsized { + // SAFETY: We checked above that this is an exact ratio. + offset_delta_elems, + // SAFETY: We checked above that this is an exact ratio. + elem_multiple, + } + } + _ => return None, + }; + + // SAFETY: We checked above that `src.align >= dst.align`. + Some(CastParams { inner, _src: PhantomData, _dst: PhantomData }) + } + } + + impl<Src: KnownLayout + ?Sized, Dst: KnownLayout + ?Sized> CastParams<Src, Dst> { + /// # Safety + /// + /// `src_meta` describes a `Src` whose size is no larger than + /// `isize::MAX`. + /// + /// The returned metadata describes a `Dst` of the same size as + /// the original `Src`. + #[inline(always)] + unsafe fn cast_metadata( + self, + src_meta: Src::PointerMetadata, + ) -> Dst::PointerMetadata { + #[allow(unused)] + use crate::util::polyfills::*; + + let dst_meta = match self.inner { + CastParamsInner::UnsizedToUnsized { offset_delta_elems, elem_multiple } => { + let src_meta = src_meta.to_elem_count(); + #[allow( + unstable_name_collisions, + clippy::multiple_unsafe_ops_per_block + )] + // SAFETY: `self` is a witness that the following + // equation holds: + // + // D_OFF + d_meta * D_ELEM = S_OFF + s_meta * S_ELEM + // + // Since the caller promises that `src_meta` is + // valid `Src` metadata, this math will not + // overflow, and the returned value will describe a + // `Dst` of the same size. + unsafe { + offset_delta_elems + .unchecked_add(src_meta.unchecked_mul(elem_multiple)) + } + } + CastParamsInner::SizedToUnsized { dst_meta } => dst_meta, + CastParamsInner::SizedToSized => 0, + }; + Dst::PointerMetadata::from_elem_count(dst_meta) + } + } + + trait Params<Src: ?Sized> { + const CAST_PARAMS: CastParams<Src, Self>; + } + + impl<Src, Dst> Params<Src> for Dst + where + Src: KnownLayout + ?Sized, + Dst: KnownLayout + ?Sized, + { + const CAST_PARAMS: CastParams<Src, Dst> = + match CastParams::try_compute(&Src::LAYOUT, &Dst::LAYOUT) { + Some(params) => params, + None => const_panic!( + "cannot `transmute_ref!` or `transmute_mut!` between incompatible types" + ), + }; + } + + let src_meta = <Src as KnownLayout>::pointer_to_metadata(src.as_ptr()); + let params = <Dst as Params<Src>>::CAST_PARAMS; + + // SAFETY: `src: PtrInner` guarantees that `src`'s referent is zero + // bytes or lives in a single allocation, which means that it is no + // larger than `isize::MAX` bytes [1]. + // + // [1] https://doc.rust-lang.org/1.92.0/std/ptr/index.html#allocation + let dst_meta = unsafe { params.cast_metadata(src_meta) }; + + <Dst as KnownLayout>::raw_from_ptr_len(src.as_non_null().cast(), dst_meta).as_ptr() + } + } +} + +// FIXME(#67): For some reason, on our MSRV toolchain, this `allow` isn't +// enforced despite having `#![allow(unknown_lints)]` at the crate root, but +// putting it here works. Once our MSRV is high enough that this bug has been +// fixed, remove this `allow`. +#[allow(unknown_lints)] +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_dst_layout_for_slice() { + let layout = DstLayout::for_slice::<u32>(); + match layout.size_info { + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { + assert_eq!(offset, 0); + assert_eq!(elem_size, 4); + } + _ => panic!("Expected SliceDst"), + } + assert_eq!(layout.align.get(), 4); + } + + /// Tests of when a sized `DstLayout` is extended with a sized field. + #[allow(clippy::decimal_literal_representation)] + #[test] + fn test_dst_layout_extend_sized_with_sized() { + // This macro constructs a layout corresponding to a `u8` and extends it + // with a zero-sized trailing field of given alignment `n`. The macro + // tests that the resulting layout has both size and alignment `min(n, + // P)` for all valid values of `repr(packed(P))`. + macro_rules! test_align_is_size { + ($n:expr) => { + let base = DstLayout::for_type::<u8>(); + let trailing_field = DstLayout::for_type::<elain::Align<$n>>(); + + let packs = + core::iter::once(None).chain((0..29).map(|p| NonZeroUsize::new(2usize.pow(p)))); + + for pack in packs { + let composite = base.extend(trailing_field, pack); + let max_align = pack.unwrap_or(DstLayout::CURRENT_MAX_ALIGN); + let align = $n.min(max_align.get()); + assert_eq!( + composite, + DstLayout { + align: NonZeroUsize::new(align).unwrap(), + size_info: SizeInfo::Sized { size: align }, + statically_shallow_unpadded: false, + } + ) + } + }; + } + + test_align_is_size!(1); + test_align_is_size!(2); + test_align_is_size!(4); + test_align_is_size!(8); + test_align_is_size!(16); + test_align_is_size!(32); + test_align_is_size!(64); + test_align_is_size!(128); + test_align_is_size!(256); + test_align_is_size!(512); + test_align_is_size!(1024); + test_align_is_size!(2048); + test_align_is_size!(4096); + test_align_is_size!(8192); + test_align_is_size!(16384); + test_align_is_size!(32768); + test_align_is_size!(65536); + test_align_is_size!(131072); + test_align_is_size!(262144); + test_align_is_size!(524288); + test_align_is_size!(1048576); + test_align_is_size!(2097152); + test_align_is_size!(4194304); + test_align_is_size!(8388608); + test_align_is_size!(16777216); + test_align_is_size!(33554432); + test_align_is_size!(67108864); + test_align_is_size!(33554432); + test_align_is_size!(134217728); + test_align_is_size!(268435456); + } + + /// Tests of when a sized `DstLayout` is extended with a DST field. + #[test] + fn test_dst_layout_extend_sized_with_dst() { + // Test that for all combinations of real-world alignments and + // `repr_packed` values, that the extension of a sized `DstLayout`` with + // a DST field correctly computes the trailing offset in the composite + // layout. + + let aligns = (0..29).map(|p| NonZeroUsize::new(2usize.pow(p)).unwrap()); + let packs = core::iter::once(None).chain(aligns.clone().map(Some)); + + for align in aligns { + for pack in packs.clone() { + let base = DstLayout::for_type::<u8>(); + let elem_size = 42; + let trailing_field_offset = 11; + + let trailing_field = DstLayout { + align, + size_info: SizeInfo::SliceDst(TrailingSliceLayout { elem_size, offset: 11 }), + statically_shallow_unpadded: false, + }; + + let composite = base.extend(trailing_field, pack); + + let max_align = pack.unwrap_or(DstLayout::CURRENT_MAX_ALIGN).get(); + + let align = align.get().min(max_align); + + assert_eq!( + composite, + DstLayout { + align: NonZeroUsize::new(align).unwrap(), + size_info: SizeInfo::SliceDst(TrailingSliceLayout { + elem_size, + offset: align + trailing_field_offset, + }), + statically_shallow_unpadded: false, + } + ) + } + } + } + + /// Tests that calling `pad_to_align` on a sized `DstLayout` adds the + /// expected amount of trailing padding. + #[test] + fn test_dst_layout_pad_to_align_with_sized() { + // For all valid alignments `align`, construct a one-byte layout aligned + // to `align`, call `pad_to_align`, and assert that the size of the + // resulting layout is equal to `align`. + for align in (0..29).map(|p| NonZeroUsize::new(2usize.pow(p)).unwrap()) { + let layout = DstLayout { + align, + size_info: SizeInfo::Sized { size: 1 }, + statically_shallow_unpadded: true, + }; + + assert_eq!( + layout.pad_to_align(), + DstLayout { + align, + size_info: SizeInfo::Sized { size: align.get() }, + statically_shallow_unpadded: align.get() == 1 + } + ); + } + + // Test explicitly-provided combinations of unpadded and padded + // counterparts. + + macro_rules! test { + (unpadded { size: $unpadded_size:expr, align: $unpadded_align:expr } + => padded { size: $padded_size:expr, align: $padded_align:expr }) => { + let unpadded = DstLayout { + align: NonZeroUsize::new($unpadded_align).unwrap(), + size_info: SizeInfo::Sized { size: $unpadded_size }, + statically_shallow_unpadded: false, + }; + let padded = unpadded.pad_to_align(); + + assert_eq!( + padded, + DstLayout { + align: NonZeroUsize::new($padded_align).unwrap(), + size_info: SizeInfo::Sized { size: $padded_size }, + statically_shallow_unpadded: false, + } + ); + }; + } + + test!(unpadded { size: 0, align: 4 } => padded { size: 0, align: 4 }); + test!(unpadded { size: 1, align: 4 } => padded { size: 4, align: 4 }); + test!(unpadded { size: 2, align: 4 } => padded { size: 4, align: 4 }); + test!(unpadded { size: 3, align: 4 } => padded { size: 4, align: 4 }); + test!(unpadded { size: 4, align: 4 } => padded { size: 4, align: 4 }); + test!(unpadded { size: 5, align: 4 } => padded { size: 8, align: 4 }); + test!(unpadded { size: 6, align: 4 } => padded { size: 8, align: 4 }); + test!(unpadded { size: 7, align: 4 } => padded { size: 8, align: 4 }); + test!(unpadded { size: 8, align: 4 } => padded { size: 8, align: 4 }); + + let current_max_align = DstLayout::CURRENT_MAX_ALIGN.get(); + + test!(unpadded { size: 1, align: current_max_align } + => padded { size: current_max_align, align: current_max_align }); + + test!(unpadded { size: current_max_align + 1, align: current_max_align } + => padded { size: current_max_align * 2, align: current_max_align }); + } + + /// Tests that calling `pad_to_align` on a DST `DstLayout` is a no-op. + #[test] + fn test_dst_layout_pad_to_align_with_dst() { + for align in (0..29).map(|p| NonZeroUsize::new(2usize.pow(p)).unwrap()) { + for offset in 0..10 { + for elem_size in 0..10 { + let layout = DstLayout { + align, + size_info: SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }), + statically_shallow_unpadded: false, + }; + assert_eq!(layout.pad_to_align(), layout); + } + } + } + } + + // This test takes a long time when running under Miri, so we skip it in + // that case. This is acceptable because this is a logic test that doesn't + // attempt to expose UB. + #[test] + #[cfg_attr(miri, ignore)] + fn test_validate_cast_and_convert_metadata() { + #[allow(non_local_definitions)] + impl From<usize> for SizeInfo { + fn from(size: usize) -> SizeInfo { + SizeInfo::Sized { size } + } + } + + #[allow(non_local_definitions)] + impl From<(usize, usize)> for SizeInfo { + fn from((offset, elem_size): (usize, usize)) -> SizeInfo { + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) + } + } + + fn layout<S: Into<SizeInfo>>(s: S, align: usize) -> DstLayout { + DstLayout { + size_info: s.into(), + align: NonZeroUsize::new(align).unwrap(), + statically_shallow_unpadded: false, + } + } + + /// This macro accepts arguments in the form of: + /// + /// layout(_, _).validate(_, _, _), Ok(Some((_, _))) + /// | | | | | | | + /// size ---------+ | | | | | | + /// align -----------+ | | | | | + /// addr ------------------------+ | | | | + /// bytes_len ----------------------+ | | | + /// cast_type -------------------------+ | | + /// elems ------------------------------------------+ | + /// split_at ------------------------------------------+ + /// + /// `.validate` is shorthand for `.validate_cast_and_convert_metadata` + /// for brevity. + /// + /// Each argument can either be an iterator or a wildcard. Each + /// wildcarded variable is implicitly replaced by an iterator over a + /// representative sample of values for that variable. Each `test!` + /// invocation iterates over every combination of values provided by + /// each variable's iterator (ie, the cartesian product) and validates + /// that the results are expected. + /// + /// The final argument uses the same syntax, but it has a different + /// meaning: + /// - If it is `Ok(pat)`, then the pattern `pat` is supplied to + /// a matching assert to validate the computed result for each + /// combination of input values. + /// - If it is `Err(Some(msg) | None)`, then `test!` validates that the + /// call to `validate_cast_and_convert_metadata` panics with the given + /// panic message or, if the current Rust toolchain version is too + /// early to support panicking in `const fn`s, panics with *some* + /// message. In the latter case, the `const_panic!` macro is used, + /// which emits code which causes a non-panicking error at const eval + /// time, but which does panic when invoked at runtime. Thus, it is + /// merely difficult to predict the *value* of this panic. We deem + /// that testing against the real panic strings on stable and nightly + /// toolchains is enough to ensure correctness. + /// + /// Note that the meta-variables that match these variables have the + /// `tt` type, and some valid expressions are not valid `tt`s (such as + /// `a..b`). In this case, wrap the expression in parentheses, and it + /// will become valid `tt`. + macro_rules! test { + ( + layout($size:tt, $align:tt) + .validate($addr:tt, $bytes_len:tt, $cast_type:tt), $expect:pat $(,)? + ) => { + itertools::iproduct!( + test!(@generate_size $size), + test!(@generate_align $align), + test!(@generate_usize $addr), + test!(@generate_usize $bytes_len), + test!(@generate_cast_type $cast_type) + ).for_each(|(size_info, align, addr, bytes_len, cast_type)| { + // Temporarily disable the panic hook installed by the test + // harness. If we don't do this, all panic messages will be + // kept in an internal log. On its own, this isn't a + // problem, but if a non-caught panic ever happens (ie, in + // code later in this test not in this macro), all of the + // previously-buffered messages will be dumped, hiding the + // real culprit. + let previous_hook = std::panic::take_hook(); + // I don't understand why, but this seems to be required in + // addition to the previous line. + std::panic::set_hook(Box::new(|_| {})); + let actual = std::panic::catch_unwind(|| { + layout(size_info, align).validate_cast_and_convert_metadata(addr, bytes_len, cast_type) + }).map_err(|d| { + let msg = d.downcast::<&'static str>().ok().map(|s| *s.as_ref()); + assert!(msg.is_some() || cfg!(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0), "non-string panic messages are not permitted when usage of panic in const fn is enabled"); + msg + }); + std::panic::set_hook(previous_hook); + + assert!( + matches!(actual, $expect), + "layout({:?}, {}).validate_cast_and_convert_metadata({}, {}, {:?})" ,size_info, align, addr, bytes_len, cast_type + ); + }); + }; + (@generate_usize _) => { 0..8 }; + // Generate sizes for both Sized and !Sized types. + (@generate_size _) => { + test!(@generate_size (_)).chain(test!(@generate_size (_, _))) + }; + // Generate sizes for both Sized and !Sized types by chaining + // specified iterators for each. + (@generate_size ($sized_sizes:tt | $unsized_sizes:tt)) => { + test!(@generate_size ($sized_sizes)).chain(test!(@generate_size $unsized_sizes)) + }; + // Generate sizes for Sized types. + (@generate_size (_)) => { test!(@generate_size (0..8)) }; + (@generate_size ($sizes:expr)) => { $sizes.into_iter().map(Into::<SizeInfo>::into) }; + // Generate sizes for !Sized types. + (@generate_size ($min_sizes:tt, $elem_sizes:tt)) => { + itertools::iproduct!( + test!(@generate_min_size $min_sizes), + test!(@generate_elem_size $elem_sizes) + ).map(Into::<SizeInfo>::into) + }; + (@generate_fixed_size _) => { (0..8).into_iter().map(Into::<SizeInfo>::into) }; + (@generate_min_size _) => { 0..8 }; + (@generate_elem_size _) => { 1..8 }; + (@generate_align _) => { [1, 2, 4, 8, 16] }; + (@generate_opt_usize _) => { [None].into_iter().chain((0..8).map(Some).into_iter()) }; + (@generate_cast_type _) => { [CastType::Prefix, CastType::Suffix] }; + (@generate_cast_type $variant:ident) => { [CastType::$variant] }; + // Some expressions need to be wrapped in parentheses in order to be + // valid `tt`s (required by the top match pattern). See the comment + // below for more details. This arm removes these parentheses to + // avoid generating an `unused_parens` warning. + (@$_:ident ($vals:expr)) => { $vals }; + (@$_:ident $vals:expr) => { $vals }; + } + + const EVENS: [usize; 8] = [0, 2, 4, 6, 8, 10, 12, 14]; + const ODDS: [usize; 8] = [1, 3, 5, 7, 9, 11, 13, 15]; + + // base_size is too big for the memory region. + test!( + layout(((1..8) | ((1..8), (1..8))), _).validate([0], [0], _), + Ok(Err(MetadataCastError::Size)) + ); + test!( + layout(((2..8) | ((2..8), (2..8))), _).validate([0], [1], Prefix), + Ok(Err(MetadataCastError::Size)) + ); + test!( + layout(((2..8) | ((2..8), (2..8))), _).validate([0x1000_0000 - 1], [1], Suffix), + Ok(Err(MetadataCastError::Size)) + ); + + // addr is unaligned for prefix cast + test!(layout(_, [2]).validate(ODDS, _, Prefix), Ok(Err(MetadataCastError::Alignment))); + test!(layout(_, [2]).validate(ODDS, _, Prefix), Ok(Err(MetadataCastError::Alignment))); + + // addr is aligned, but end of buffer is unaligned for suffix cast + test!(layout(_, [2]).validate(EVENS, ODDS, Suffix), Ok(Err(MetadataCastError::Alignment))); + test!(layout(_, [2]).validate(EVENS, ODDS, Suffix), Ok(Err(MetadataCastError::Alignment))); + + // Unfortunately, these constants cannot easily be used in the + // implementation of `validate_cast_and_convert_metadata`, since + // `panic!` consumes a string literal, not an expression. + // + // It's important that these messages be in a separate module. If they + // were at the function's top level, we'd pass them to `test!` as, e.g., + // `Err(TRAILING)`, which would run into a subtle Rust footgun - the + // `TRAILING` identifier would be treated as a pattern to match rather + // than a value to check for equality. + mod msgs { + pub(super) const TRAILING: &str = + "attempted to cast to slice type with zero-sized element"; + pub(super) const OVERFLOW: &str = "`addr` + `bytes_len` > usize::MAX"; + } + + // casts with ZST trailing element types are unsupported + test!(layout((_, [0]), _).validate(_, _, _), Err(Some(msgs::TRAILING) | None),); + + // addr + bytes_len must not overflow usize + test!(layout(_, _).validate([usize::MAX], (1..100), _), Err(Some(msgs::OVERFLOW) | None)); + test!(layout(_, _).validate((1..100), [usize::MAX], _), Err(Some(msgs::OVERFLOW) | None)); + test!( + layout(_, _).validate( + [usize::MAX / 2 + 1, usize::MAX], + [usize::MAX / 2 + 1, usize::MAX], + _ + ), + Err(Some(msgs::OVERFLOW) | None) + ); + + // Validates that `validate_cast_and_convert_metadata` satisfies its own + // documented safety postconditions, and also a few other properties + // that aren't documented but we want to guarantee anyway. + fn validate_behavior( + (layout, addr, bytes_len, cast_type): (DstLayout, usize, usize, CastType), + ) { + if let Ok((elems, split_at)) = + layout.validate_cast_and_convert_metadata(addr, bytes_len, cast_type) + { + let (size_info, align) = (layout.size_info, layout.align); + let debug_str = format!( + "layout({:?}, {}).validate_cast_and_convert_metadata({}, {}, {:?}) => ({}, {})", + size_info, align, addr, bytes_len, cast_type, elems, split_at + ); + + // If this is a sized type (no trailing slice), then `elems` is + // meaningless, but in practice we set it to 0. Callers are not + // allowed to rely on this, but a lot of math is nicer if + // they're able to, and some callers might accidentally do that. + let sized = matches!(layout.size_info, SizeInfo::Sized { .. }); + assert!(!(sized && elems != 0), "{}", debug_str); + + let resulting_size = match layout.size_info { + SizeInfo::Sized { size } => size, + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { + let padded_size = |elems| { + let without_padding = offset + elems * elem_size; + without_padding + util::padding_needed_for(without_padding, align) + }; + + let resulting_size = padded_size(elems); + // Test that `validate_cast_and_convert_metadata` + // computed the largest possible value that fits in the + // given range. + assert!(padded_size(elems + 1) > bytes_len, "{}", debug_str); + resulting_size + } + }; + + // Test safety postconditions guaranteed by + // `validate_cast_and_convert_metadata`. + assert!(resulting_size <= bytes_len, "{}", debug_str); + match cast_type { + CastType::Prefix => { + assert_eq!(addr % align, 0, "{}", debug_str); + assert_eq!(resulting_size, split_at, "{}", debug_str); + } + CastType::Suffix => { + assert_eq!(split_at, bytes_len - resulting_size, "{}", debug_str); + assert_eq!((addr + split_at) % align, 0, "{}", debug_str); + } + } + } else { + let min_size = match layout.size_info { + SizeInfo::Sized { size } => size, + SizeInfo::SliceDst(TrailingSliceLayout { offset, .. }) => { + offset + util::padding_needed_for(offset, layout.align) + } + }; + + // If a cast is invalid, it is either because... + // 1. there are insufficient bytes at the given region for type: + let insufficient_bytes = bytes_len < min_size; + // 2. performing the cast would misalign type: + let base = match cast_type { + CastType::Prefix => 0, + CastType::Suffix => bytes_len, + }; + let misaligned = (base + addr) % layout.align != 0; + + assert!(insufficient_bytes || misaligned); + } + } + + let sizes = 0..8; + let elem_sizes = 1..8; + let size_infos = sizes + .clone() + .map(Into::<SizeInfo>::into) + .chain(itertools::iproduct!(sizes, elem_sizes).map(Into::<SizeInfo>::into)); + let layouts = itertools::iproduct!(size_infos, [1, 2, 4, 8, 16, 32]) + .filter(|(size_info, align)| !matches!(size_info, SizeInfo::Sized { size } if size % align != 0)) + .map(|(size_info, align)| layout(size_info, align)); + itertools::iproduct!(layouts, 0..8, 0..8, [CastType::Prefix, CastType::Suffix]) + .for_each(validate_behavior); + } + + #[test] + #[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] + fn test_validate_rust_layout() { + use core::{ + convert::TryInto as _, + ptr::{self, NonNull}, + }; + + use crate::util::testutil::*; + + // This test synthesizes pointers with various metadata and uses Rust's + // built-in APIs to confirm that Rust makes decisions about type layout + // which are consistent with what we believe is guaranteed by the + // language. If this test fails, it doesn't just mean our code is wrong + // - it means we're misunderstanding the language's guarantees. + + #[derive(Debug)] + struct MacroArgs { + offset: usize, + align: NonZeroUsize, + elem_size: Option<usize>, + } + + /// # Safety + /// + /// `test` promises to only call `addr_of_slice_field` on a `NonNull<T>` + /// which points to a valid `T`. + /// + /// `with_elems` must produce a pointer which points to a valid `T`. + fn test<T: ?Sized, W: Fn(usize) -> NonNull<T>>( + args: MacroArgs, + with_elems: W, + addr_of_slice_field: Option<fn(NonNull<T>) -> NonNull<u8>>, + ) { + let dst = args.elem_size.is_some(); + let layout = { + let size_info = match args.elem_size { + Some(elem_size) => { + SizeInfo::SliceDst(TrailingSliceLayout { offset: args.offset, elem_size }) + } + None => SizeInfo::Sized { + // Rust only supports types whose sizes are a multiple + // of their alignment. If the macro created a type like + // this: + // + // #[repr(C, align(2))] + // struct Foo([u8; 1]); + // + // ...then Rust will automatically round the type's size + // up to 2. + size: args.offset + util::padding_needed_for(args.offset, args.align), + }, + }; + DstLayout { size_info, align: args.align, statically_shallow_unpadded: false } + }; + + for elems in 0..128 { + let ptr = with_elems(elems); + + if let Some(addr_of_slice_field) = addr_of_slice_field { + let slc_field_ptr = addr_of_slice_field(ptr).as_ptr(); + // SAFETY: Both `slc_field_ptr` and `ptr` are pointers to + // the same valid Rust object. + // Work around https://github.com/rust-lang/rust-clippy/issues/12280 + let offset: usize = + unsafe { slc_field_ptr.byte_offset_from(ptr.as_ptr()).try_into().unwrap() }; + assert_eq!(offset, args.offset); + } + + // SAFETY: `ptr` points to a valid `T`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + let (size, align) = unsafe { + (mem::size_of_val_raw(ptr.as_ptr()), mem::align_of_val_raw(ptr.as_ptr())) + }; + + // Avoid expensive allocation when running under Miri. + let assert_msg = if !cfg!(miri) { + format!("\n{:?}\nsize:{}, align:{}", args, size, align) + } else { + String::new() + }; + + let without_padding = + args.offset + args.elem_size.map(|elem_size| elems * elem_size).unwrap_or(0); + assert!(size >= without_padding, "{}", assert_msg); + assert_eq!(align, args.align.get(), "{}", assert_msg); + + // This encodes the most important part of the test: our + // understanding of how Rust determines the layout of repr(C) + // types. Sized repr(C) types are trivial, but DST types have + // some subtlety. Note that: + // - For sized types, `without_padding` is just the size of the + // type that we constructed for `Foo`. Since we may have + // requested a larger alignment, `Foo` may actually be larger + // than this, hence `padding_needed_for`. + // - For unsized types, `without_padding` is dynamically + // computed from the offset, the element size, and element + // count. We expect that the size of the object should be + // `offset + elem_size * elems` rounded up to the next + // alignment. + let expected_size = + without_padding + util::padding_needed_for(without_padding, args.align); + assert_eq!(expected_size, size, "{}", assert_msg); + + // For zero-sized element types, + // `validate_cast_and_convert_metadata` just panics, so we skip + // testing those types. + if args.elem_size.map(|elem_size| elem_size > 0).unwrap_or(true) { + let addr = ptr.addr().get(); + let (got_elems, got_split_at) = layout + .validate_cast_and_convert_metadata(addr, size, CastType::Prefix) + .unwrap(); + // Avoid expensive allocation when running under Miri. + let assert_msg = if !cfg!(miri) { + format!( + "{}\nvalidate_cast_and_convert_metadata({}, {})", + assert_msg, addr, size, + ) + } else { + String::new() + }; + assert_eq!(got_split_at, size, "{}", assert_msg); + if dst { + assert!(got_elems >= elems, "{}", assert_msg); + if got_elems != elems { + // If `validate_cast_and_convert_metadata` + // returned more elements than `elems`, that + // means that `elems` is not the maximum number + // of elements that can fit in `size` - in other + // words, there is enough padding at the end of + // the value to fit at least one more element. + // If we use this metadata to synthesize a + // pointer, despite having a different element + // count, we still expect it to have the same + // size. + let got_ptr = with_elems(got_elems); + // SAFETY: `got_ptr` is a pointer to a valid `T`. + let size_of_got_ptr = unsafe { mem::size_of_val_raw(got_ptr.as_ptr()) }; + assert_eq!(size_of_got_ptr, size, "{}", assert_msg); + } + } else { + // For sized casts, the returned element value is + // technically meaningless, and we don't guarantee any + // particular value. In practice, it's always zero. + assert_eq!(got_elems, 0, "{}", assert_msg) + } + } + } + } + + macro_rules! validate_against_rust { + ($offset:literal, $align:literal $(, $elem_size:literal)?) => {{ + #[repr(C, align($align))] + struct Foo([u8; $offset]$(, [[u8; $elem_size]])?); + + let args = MacroArgs { + offset: $offset, + align: $align.try_into().unwrap(), + elem_size: { + #[allow(unused)] + let ret = None::<usize>; + $(let ret = Some($elem_size);)? + ret + } + }; + + #[repr(C, align($align))] + struct FooAlign; + // Create an aligned buffer to use in order to synthesize + // pointers to `Foo`. We don't ever load values from these + // pointers - we just do arithmetic on them - so having a "real" + // block of memory as opposed to a validly-aligned-but-dangling + // pointer is only necessary to make Miri happy since we run it + // with "strict provenance" checking enabled. + let aligned_buf = Align::<_, FooAlign>::new([0u8; 1024]); + let with_elems = |elems| { + let slc = NonNull::slice_from_raw_parts(NonNull::from(&aligned_buf.t), elems); + #[allow(clippy::as_conversions)] + NonNull::new(slc.as_ptr() as *mut Foo).unwrap() + }; + let addr_of_slice_field = { + #[allow(unused)] + let f = None::<fn(NonNull<Foo>) -> NonNull<u8>>; + $( + // SAFETY: `test` promises to only call `f` with a `ptr` + // to a valid `Foo`. + let f: Option<fn(NonNull<Foo>) -> NonNull<u8>> = Some(|ptr: NonNull<Foo>| unsafe { + NonNull::new(ptr::addr_of_mut!((*ptr.as_ptr()).1)).unwrap().cast::<u8>() + }); + let _ = $elem_size; + )? + f + }; + + test::<Foo, _>(args, with_elems, addr_of_slice_field); + }}; + } + + // Every permutation of: + // - offset in [0, 4] + // - align in [1, 16] + // - elem_size in [0, 4] (plus no elem_size) + validate_against_rust!(0, 1); + validate_against_rust!(0, 1, 0); + validate_against_rust!(0, 1, 1); + validate_against_rust!(0, 1, 2); + validate_against_rust!(0, 1, 3); + validate_against_rust!(0, 1, 4); + validate_against_rust!(0, 2); + validate_against_rust!(0, 2, 0); + validate_against_rust!(0, 2, 1); + validate_against_rust!(0, 2, 2); + validate_against_rust!(0, 2, 3); + validate_against_rust!(0, 2, 4); + validate_against_rust!(0, 4); + validate_against_rust!(0, 4, 0); + validate_against_rust!(0, 4, 1); + validate_against_rust!(0, 4, 2); + validate_against_rust!(0, 4, 3); + validate_against_rust!(0, 4, 4); + validate_against_rust!(0, 8); + validate_against_rust!(0, 8, 0); + validate_against_rust!(0, 8, 1); + validate_against_rust!(0, 8, 2); + validate_against_rust!(0, 8, 3); + validate_against_rust!(0, 8, 4); + validate_against_rust!(0, 16); + validate_against_rust!(0, 16, 0); + validate_against_rust!(0, 16, 1); + validate_against_rust!(0, 16, 2); + validate_against_rust!(0, 16, 3); + validate_against_rust!(0, 16, 4); + validate_against_rust!(1, 1); + validate_against_rust!(1, 1, 0); + validate_against_rust!(1, 1, 1); + validate_against_rust!(1, 1, 2); + validate_against_rust!(1, 1, 3); + validate_against_rust!(1, 1, 4); + validate_against_rust!(1, 2); + validate_against_rust!(1, 2, 0); + validate_against_rust!(1, 2, 1); + validate_against_rust!(1, 2, 2); + validate_against_rust!(1, 2, 3); + validate_against_rust!(1, 2, 4); + validate_against_rust!(1, 4); + validate_against_rust!(1, 4, 0); + validate_against_rust!(1, 4, 1); + validate_against_rust!(1, 4, 2); + validate_against_rust!(1, 4, 3); + validate_against_rust!(1, 4, 4); + validate_against_rust!(1, 8); + validate_against_rust!(1, 8, 0); + validate_against_rust!(1, 8, 1); + validate_against_rust!(1, 8, 2); + validate_against_rust!(1, 8, 3); + validate_against_rust!(1, 8, 4); + validate_against_rust!(1, 16); + validate_against_rust!(1, 16, 0); + validate_against_rust!(1, 16, 1); + validate_against_rust!(1, 16, 2); + validate_against_rust!(1, 16, 3); + validate_against_rust!(1, 16, 4); + validate_against_rust!(2, 1); + validate_against_rust!(2, 1, 0); + validate_against_rust!(2, 1, 1); + validate_against_rust!(2, 1, 2); + validate_against_rust!(2, 1, 3); + validate_against_rust!(2, 1, 4); + validate_against_rust!(2, 2); + validate_against_rust!(2, 2, 0); + validate_against_rust!(2, 2, 1); + validate_against_rust!(2, 2, 2); + validate_against_rust!(2, 2, 3); + validate_against_rust!(2, 2, 4); + validate_against_rust!(2, 4); + validate_against_rust!(2, 4, 0); + validate_against_rust!(2, 4, 1); + validate_against_rust!(2, 4, 2); + validate_against_rust!(2, 4, 3); + validate_against_rust!(2, 4, 4); + validate_against_rust!(2, 8); + validate_against_rust!(2, 8, 0); + validate_against_rust!(2, 8, 1); + validate_against_rust!(2, 8, 2); + validate_against_rust!(2, 8, 3); + validate_against_rust!(2, 8, 4); + validate_against_rust!(2, 16); + validate_against_rust!(2, 16, 0); + validate_against_rust!(2, 16, 1); + validate_against_rust!(2, 16, 2); + validate_against_rust!(2, 16, 3); + validate_against_rust!(2, 16, 4); + validate_against_rust!(3, 1); + validate_against_rust!(3, 1, 0); + validate_against_rust!(3, 1, 1); + validate_against_rust!(3, 1, 2); + validate_against_rust!(3, 1, 3); + validate_against_rust!(3, 1, 4); + validate_against_rust!(3, 2); + validate_against_rust!(3, 2, 0); + validate_against_rust!(3, 2, 1); + validate_against_rust!(3, 2, 2); + validate_against_rust!(3, 2, 3); + validate_against_rust!(3, 2, 4); + validate_against_rust!(3, 4); + validate_against_rust!(3, 4, 0); + validate_against_rust!(3, 4, 1); + validate_against_rust!(3, 4, 2); + validate_against_rust!(3, 4, 3); + validate_against_rust!(3, 4, 4); + validate_against_rust!(3, 8); + validate_against_rust!(3, 8, 0); + validate_against_rust!(3, 8, 1); + validate_against_rust!(3, 8, 2); + validate_against_rust!(3, 8, 3); + validate_against_rust!(3, 8, 4); + validate_against_rust!(3, 16); + validate_against_rust!(3, 16, 0); + validate_against_rust!(3, 16, 1); + validate_against_rust!(3, 16, 2); + validate_against_rust!(3, 16, 3); + validate_against_rust!(3, 16, 4); + validate_against_rust!(4, 1); + validate_against_rust!(4, 1, 0); + validate_against_rust!(4, 1, 1); + validate_against_rust!(4, 1, 2); + validate_against_rust!(4, 1, 3); + validate_against_rust!(4, 1, 4); + validate_against_rust!(4, 2); + validate_against_rust!(4, 2, 0); + validate_against_rust!(4, 2, 1); + validate_against_rust!(4, 2, 2); + validate_against_rust!(4, 2, 3); + validate_against_rust!(4, 2, 4); + validate_against_rust!(4, 4); + validate_against_rust!(4, 4, 0); + validate_against_rust!(4, 4, 1); + validate_against_rust!(4, 4, 2); + validate_against_rust!(4, 4, 3); + validate_against_rust!(4, 4, 4); + validate_against_rust!(4, 8); + validate_against_rust!(4, 8, 0); + validate_against_rust!(4, 8, 1); + validate_against_rust!(4, 8, 2); + validate_against_rust!(4, 8, 3); + validate_against_rust!(4, 8, 4); + validate_against_rust!(4, 16); + validate_against_rust!(4, 16, 0); + validate_against_rust!(4, 16, 1); + validate_against_rust!(4, 16, 2); + validate_against_rust!(4, 16, 3); + validate_against_rust!(4, 16, 4); + } +} + +#[cfg(kani)] +mod proofs { + use core::alloc::Layout; + + use super::*; + + impl kani::Arbitrary for DstLayout { + fn any() -> Self { + let align: NonZeroUsize = kani::any(); + let size_info: SizeInfo = kani::any(); + + kani::assume(align.is_power_of_two()); + kani::assume(align < DstLayout::THEORETICAL_MAX_ALIGN); + + // For testing purposes, we most care about instantiations of + // `DstLayout` that can correspond to actual Rust types. We use + // `Layout` to verify that our `DstLayout` satisfies the validity + // conditions of Rust layouts. + kani::assume( + match size_info { + SizeInfo::Sized { size } => Layout::from_size_align(size, align.get()), + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size: _ }) => { + // `SliceDst` cannot encode an exact size, but we know + // it is at least `offset` bytes. + Layout::from_size_align(offset, align.get()) + } + } + .is_ok(), + ); + + Self { align: align, size_info: size_info, statically_shallow_unpadded: kani::any() } + } + } + + impl kani::Arbitrary for SizeInfo { + fn any() -> Self { + let is_sized: bool = kani::any(); + + match is_sized { + true => { + let size: usize = kani::any(); + + kani::assume(size <= DstLayout::MAX_SIZE); + + SizeInfo::Sized { size } + } + false => SizeInfo::SliceDst(kani::any()), + } + } + } + + impl kani::Arbitrary for TrailingSliceLayout { + fn any() -> Self { + let elem_size: usize = kani::any(); + let offset: usize = kani::any(); + + kani::assume(elem_size < DstLayout::MAX_SIZE); + kani::assume(offset < DstLayout::MAX_SIZE); + + TrailingSliceLayout { elem_size, offset } + } + } + + #[kani::proof] + fn prove_requires_dynamic_padding() { + let layout: DstLayout = kani::any(); + + let SizeInfo::SliceDst(size_info) = layout.size_info else { + kani::assume(false); + loop {} + }; + + let meta: usize = kani::any(); + + let Some(trailing_slice_size) = size_info.elem_size.checked_mul(meta) else { + // The `trailing_slice_size` exceeds `usize::MAX`; `meta` is invalid. + kani::assume(false); + loop {} + }; + + let Some(unpadded_size) = size_info.offset.checked_add(trailing_slice_size) else { + // The `unpadded_size` exceeds `usize::MAX`; `meta`` is invalid. + kani::assume(false); + loop {} + }; + + if unpadded_size >= DstLayout::MAX_SIZE { + // The `unpadded_size` exceeds `isize::MAX`; `meta` is invalid. + kani::assume(false); + loop {} + } + + let trailing_padding = util::padding_needed_for(unpadded_size, layout.align); + + if !layout.requires_dynamic_padding() { + assert!(trailing_padding == 0); + } + } + + #[kani::proof] + fn prove_dst_layout_extend() { + use crate::util::{max, min, padding_needed_for}; + + let base: DstLayout = kani::any(); + let field: DstLayout = kani::any(); + let packed: Option<NonZeroUsize> = kani::any(); + + if let Some(max_align) = packed { + kani::assume(max_align.is_power_of_two()); + kani::assume(base.align <= max_align); + } + + // The base can only be extended if it's sized. + kani::assume(matches!(base.size_info, SizeInfo::Sized { .. })); + let base_size = if let SizeInfo::Sized { size } = base.size_info { + size + } else { + unreachable!(); + }; + + // Under the above conditions, `DstLayout::extend` will not panic. + let composite = base.extend(field, packed); + + // The field's alignment is clamped by `max_align` (i.e., the + // `packed` attribute, if any) [1]. + // + // [1] Per https://doc.rust-lang.org/reference/type-layout.html#the-alignment-modifiers: + // + // The alignments of each field, for the purpose of positioning + // fields, is the smaller of the specified alignment and the + // alignment of the field's type. + let field_align = min(field.align, packed.unwrap_or(DstLayout::THEORETICAL_MAX_ALIGN)); + + // The struct's alignment is the maximum of its previous alignment and + // `field_align`. + assert_eq!(composite.align, max(base.align, field_align)); + + // Compute the minimum amount of inter-field padding needed to + // satisfy the field's alignment, and offset of the trailing field. + // [1] + // + // [1] Per https://doc.rust-lang.org/reference/type-layout.html#the-alignment-modifiers: + // + // Inter-field padding is guaranteed to be the minimum required in + // order to satisfy each field's (possibly altered) alignment. + let padding = padding_needed_for(base_size, field_align); + let offset = base_size + padding; + + // For testing purposes, we'll also construct `alloc::Layout` + // stand-ins for `DstLayout`, and show that `extend` behaves + // comparably on both types. + let base_analog = Layout::from_size_align(base_size, base.align.get()).unwrap(); + + match field.size_info { + SizeInfo::Sized { size: field_size } => { + if let SizeInfo::Sized { size: composite_size } = composite.size_info { + // If the trailing field is sized, the resulting layout will + // be sized. Its size will be the sum of the preceding + // layout, the size of the new field, and the size of + // inter-field padding between the two. + assert_eq!(composite_size, offset + field_size); + + let field_analog = + Layout::from_size_align(field_size, field_align.get()).unwrap(); + + if let Ok((actual_composite, actual_offset)) = base_analog.extend(field_analog) + { + assert_eq!(actual_offset, offset); + assert_eq!(actual_composite.size(), composite_size); + assert_eq!(actual_composite.align(), composite.align.get()); + } else { + // An error here reflects that composite of `base` + // and `field` cannot correspond to a real Rust type + // fragment, because such a fragment would violate + // the basic invariants of a valid Rust layout. At + // the time of writing, `DstLayout` is a little more + // permissive than `Layout`, so we don't assert + // anything in this branch (e.g., unreachability). + } + } else { + panic!("The composite of two sized layouts must be sized.") + } + } + SizeInfo::SliceDst(TrailingSliceLayout { + offset: field_offset, + elem_size: field_elem_size, + }) => { + if let SizeInfo::SliceDst(TrailingSliceLayout { + offset: composite_offset, + elem_size: composite_elem_size, + }) = composite.size_info + { + // The offset of the trailing slice component is the sum + // of the offset of the trailing field and the trailing + // slice offset within that field. + assert_eq!(composite_offset, offset + field_offset); + // The elem size is unchanged. + assert_eq!(composite_elem_size, field_elem_size); + + let field_analog = + Layout::from_size_align(field_offset, field_align.get()).unwrap(); + + if let Ok((actual_composite, actual_offset)) = base_analog.extend(field_analog) + { + assert_eq!(actual_offset, offset); + assert_eq!(actual_composite.size(), composite_offset); + assert_eq!(actual_composite.align(), composite.align.get()); + } else { + // An error here reflects that composite of `base` + // and `field` cannot correspond to a real Rust type + // fragment, because such a fragment would violate + // the basic invariants of a valid Rust layout. At + // the time of writing, `DstLayout` is a little more + // permissive than `Layout`, so we don't assert + // anything in this branch (e.g., unreachability). + } + } else { + panic!("The extension of a layout with a DST must result in a DST.") + } + } + } + } + + #[kani::proof] + #[kani::should_panic] + fn prove_dst_layout_extend_dst_panics() { + let base: DstLayout = kani::any(); + let field: DstLayout = kani::any(); + let packed: Option<NonZeroUsize> = kani::any(); + + if let Some(max_align) = packed { + kani::assume(max_align.is_power_of_two()); + kani::assume(base.align <= max_align); + } + + kani::assume(matches!(base.size_info, SizeInfo::SliceDst(..))); + + let _ = base.extend(field, packed); + } + + #[kani::proof] + fn prove_dst_layout_pad_to_align() { + use crate::util::padding_needed_for; + + let layout: DstLayout = kani::any(); + + let padded = layout.pad_to_align(); + + // Calling `pad_to_align` does not alter the `DstLayout`'s alignment. + assert_eq!(padded.align, layout.align); + + if let SizeInfo::Sized { size: unpadded_size } = layout.size_info { + if let SizeInfo::Sized { size: padded_size } = padded.size_info { + // If the layout is sized, it will remain sized after padding is + // added. Its sum will be its unpadded size and the size of the + // trailing padding needed to satisfy its alignment + // requirements. + let padding = padding_needed_for(unpadded_size, layout.align); + assert_eq!(padded_size, unpadded_size + padding); + + // Prove that calling `DstLayout::pad_to_align` behaves + // identically to `Layout::pad_to_align`. + let layout_analog = + Layout::from_size_align(unpadded_size, layout.align.get()).unwrap(); + let padded_analog = layout_analog.pad_to_align(); + assert_eq!(padded_analog.align(), layout.align.get()); + assert_eq!(padded_analog.size(), padded_size); + } else { + panic!("The padding of a sized layout must result in a sized layout.") + } + } else { + // If the layout is a DST, padding cannot be statically added. + assert_eq!(padded.size_info, layout.size_info); + } + } +} diff --git a/rust/zerocopy/src/lib.rs b/rust/zerocopy/src/lib.rs new file mode 100644 index 000000000000..3302d67602ab --- /dev/null +++ b/rust/zerocopy/src/lib.rs @@ -0,0 +1,7612 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2018 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License <LICENSE-BSD or +// https://opensource.org/license/bsd-2-clause>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +// After updating the following doc comment, make sure to run the following +// command to update `README.md` based on its contents: +// +// cargo -q run --manifest-path tools/Cargo.toml -p generate-readme > README.md + +//! ***<span style="font-size: 140%">Fast, safe, <span +//! style="color:red;">compile error</span>. Pick two.</span>*** +//! +//! Zerocopy makes zero-cost memory manipulation effortless. We write `unsafe` +//! so you don't have to. +//! +//! *For an overview of what's changed from zerocopy 0.7, check out our [release +//! notes][release-notes], which include a step-by-step upgrading guide.* +//! +//! *Have questions? Need more out of zerocopy? Submit a [customer request +//! issue][customer-request-issue] or ask the maintainers on +//! [GitHub][github-q-a] or [Discord][discord]!* +//! +//! [customer-request-issue]: https://github.com/google/zerocopy/issues/new/choose +//! [release-notes]: https://github.com/google/zerocopy/discussions/1680 +//! [github-q-a]: https://github.com/google/zerocopy/discussions/categories/q-a +//! [discord]: https://discord.gg/MAvWH2R6zk +//! +//! # Overview +//! +//! ##### Conversion Traits +//! +//! Zerocopy provides four derivable traits for zero-cost conversions: +//! - [`TryFromBytes`] indicates that a type may safely be converted from +//! certain byte sequences (conditional on runtime checks) +//! - [`FromZeros`] indicates that a sequence of zero bytes represents a valid +//! instance of a type +//! - [`FromBytes`] indicates that a type may safely be converted from an +//! arbitrary byte sequence +//! - [`IntoBytes`] indicates that a type may safely be converted *to* a byte +//! sequence +//! +//! These traits support sized types, slices, and [slice DSTs][slice-dsts]. +//! +//! [slice-dsts]: KnownLayout#dynamically-sized-types +//! +//! ##### Marker Traits +//! +//! Zerocopy provides three derivable marker traits that do not provide any +//! functionality themselves, but are required to call certain methods provided +//! by the conversion traits: +//! - [`KnownLayout`] indicates that zerocopy can reason about certain layout +//! qualities of a type +//! - [`Immutable`] indicates that a type is free from interior mutability, +//! except by ownership or an exclusive (`&mut`) borrow +//! - [`Unaligned`] indicates that a type's alignment requirement is 1 +//! +//! You should generally derive these marker traits whenever possible. +//! +//! ##### Conversion Macros +//! +//! Zerocopy provides six macros for safe casting between types: +//! +//! - ([`try_`][try_transmute])[`transmute`] (conditionally) converts a value of +//! one type to a value of another type of the same size +//! - ([`try_`][try_transmute_mut])[`transmute_mut`] (conditionally) converts a +//! mutable reference of one type to a mutable reference of another type of +//! the same size +//! - ([`try_`][try_transmute_ref])[`transmute_ref`] (conditionally) converts a +//! mutable or immutable reference of one type to an immutable reference of +//! another type of the same size +//! +//! These macros perform *compile-time* size and alignment checks, meaning that +//! unconditional casts have zero cost at runtime. Conditional casts do not need +//! to validate size or alignment runtime, but do need to validate contents. +//! +//! These macros cannot be used in generic contexts. For generic conversions, +//! use the methods defined by the [conversion traits](#conversion-traits). +//! +//! ##### Byteorder-Aware Numerics +//! +//! Zerocopy provides byte-order aware integer types that support these +//! conversions; see the [`byteorder`] module. These types are especially useful +//! for network parsing. +//! +//! # Cargo Features +//! +//! - **`alloc`** +//! By default, `zerocopy` is `no_std`. When the `alloc` feature is enabled, +//! the `alloc` crate is added as a dependency, and some allocation-related +//! functionality is added. +//! +//! - **`std`** +//! By default, `zerocopy` is `no_std`. When the `std` feature is enabled, the +//! `std` crate is added as a dependency (ie, `no_std` is disabled), and +//! support for some `std` types is added. `std` implies `alloc`. +//! +//! - **`derive`** +//! Provides derives for the core marker traits via the `zerocopy-derive` +//! crate. These derives are re-exported from `zerocopy`, so it is not +//! necessary to depend on `zerocopy-derive` directly. +//! +//! However, you may experience better compile times if you instead directly +//! depend on both `zerocopy` and `zerocopy-derive` in your `Cargo.toml`, +//! since doing so will allow Rust to compile these crates in parallel. To do +//! so, do *not* enable the `derive` feature, and list both dependencies in +//! your `Cargo.toml` with the same leading non-zero version number; e.g: +//! +//! ```toml +//! [dependencies] +//! zerocopy = "0.X" +//! zerocopy-derive = "0.X" +//! ``` +//! +//! To avoid the risk of [duplicate import errors][duplicate-import-errors] if +//! one of your dependencies enables zerocopy's `derive` feature, import +//! derives as `use zerocopy_derive::*` rather than by name (e.g., `use +//! zerocopy_derive::FromBytes`). +//! +//! - **`simd`** +//! When the `simd` feature is enabled, `FromZeros`, `FromBytes`, and +//! `IntoBytes` impls are emitted for all stable SIMD types which exist on the +//! target platform. Note that the layout of SIMD types is not yet stabilized, +//! so these impls may be removed in the future if layout changes make them +//! invalid. For more information, see the Unsafe Code Guidelines Reference +//! page on the [layout of packed SIMD vectors][simd-layout]. +//! +//! - **`simd-nightly`** +//! Enables the `simd` feature and adds support for SIMD types which are only +//! available on nightly. Since these types are unstable, support for any type +//! may be removed at any point in the future. +//! +//! - **`float-nightly`** +//! Adds support for the unstable `f16` and `f128` types. These types are +//! not yet fully implemented and may not be supported on all platforms. +//! +//! [duplicate-import-errors]: https://github.com/google/zerocopy/issues/1587 +//! [simd-layout]: https://rust-lang.github.io/unsafe-code-guidelines/layout/packed-simd-vectors.html +//! +//! # Build Tuning +//! +//! ## `--cfg zerocopy_inline_always` +//! +//! Upgrades `#[inline]` to `#[inline(always)]` on many of zerocopy's public +//! functions and methods. This provides a narrowly-scoped alternative that +//! *may* improve the optimization of hot paths using zerocopy without the broad +//! compile-time penalties of configuring `codegen-units=1`. +//! +//! # Security Ethos +//! +//! Zerocopy is expressly designed for use in security-critical contexts. We +//! strive to ensure that that zerocopy code is sound under Rust's current +//! memory model, and *any future memory model*. We ensure this by: +//! - **...not 'guessing' about Rust's semantics.** +//! We annotate `unsafe` code with a precise rationale for its soundness that +//! cites a relevant section of Rust's official documentation. When Rust's +//! documented semantics are unclear, we work with the Rust Operational +//! Semantics Team to clarify Rust's documentation. +//! - **...rigorously testing our implementation.** +//! We run tests using [Miri], ensuring that zerocopy is sound across a wide +//! array of supported target platforms of varying endianness and pointer +//! width, and across both current and experimental memory models of Rust. +//! - **...formally proving the correctness of our implementation.** +//! We apply formal verification tools like [Kani][kani] to prove zerocopy's +//! correctness. +//! +//! For more information, see our full [soundness policy]. +//! +//! [Miri]: https://github.com/rust-lang/miri +//! [Kani]: https://github.com/model-checking/kani +//! [soundness policy]: https://github.com/google/zerocopy/blob/main/POLICIES.md#soundness +//! +//! # Relationship to Project Safe Transmute +//! +//! [Project Safe Transmute] is an official initiative of the Rust Project to +//! develop language-level support for safer transmutation. The Project consults +//! with crates like zerocopy to identify aspects of safer transmutation that +//! would benefit from compiler support, and has developed an [experimental, +//! compiler-supported analysis][mcp-transmutability] which determines whether, +//! for a given type, any value of that type may be soundly transmuted into +//! another type. Once this functionality is sufficiently mature, zerocopy +//! intends to replace its internal transmutability analysis (implemented by our +//! custom derives) with the compiler-supported one. This change will likely be +//! an implementation detail that is invisible to zerocopy's users. +//! +//! Project Safe Transmute will not replace the need for most of zerocopy's +//! higher-level abstractions. The experimental compiler analysis is a tool for +//! checking the soundness of `unsafe` code, not a tool to avoid writing +//! `unsafe` code altogether. For the foreseeable future, crates like zerocopy +//! will still be required in order to provide higher-level abstractions on top +//! of the building block provided by Project Safe Transmute. +//! +//! [Project Safe Transmute]: https://rust-lang.github.io/rfcs/2835-project-safe-transmute.html +//! [mcp-transmutability]: https://github.com/rust-lang/compiler-team/issues/411 +//! +//! # MSRV +//! +//! See our [MSRV policy]. +//! +//! [MSRV policy]: https://github.com/google/zerocopy/blob/main/POLICIES.md#msrv +//! +//! # Changelog +//! +//! Zerocopy uses [GitHub Releases]. +//! +//! [GitHub Releases]: https://github.com/google/zerocopy/releases +//! +//! # Thanks +//! +//! Zerocopy is maintained by engineers at Google with help from [many wonderful +//! contributors][contributors]. Thank you to everyone who has lent a hand in +//! making Rust a little more secure! +//! +//! [contributors]: https://github.com/google/zerocopy/graphs/contributors + +// Sometimes we want to use lints which were added after our MSRV. +// `unknown_lints` is `warn` by default and we deny warnings in CI, so without +// this attribute, any unknown lint would cause a CI failure when testing with +// our MSRV. +#![allow(unknown_lints, non_local_definitions, unreachable_patterns)] +#![deny(renamed_and_removed_lints)] +#![deny( + anonymous_parameters, + deprecated_in_future, + late_bound_lifetime_arguments, + missing_copy_implementations, + missing_debug_implementations, + missing_docs, + path_statements, + patterns_in_fns_without_body, + rust_2018_idioms, + trivial_numeric_casts, + unreachable_pub, + unsafe_op_in_unsafe_fn, + unused_extern_crates, + // We intentionally choose not to deny `unused_qualifications`. When items + // are added to the prelude (e.g., `core::mem::size_of`), this has the + // consequence of making some uses trigger this lint on the latest toolchain + // (e.g., `mem::size_of`), but fixing it (e.g. by replacing with `size_of`) + // does not work on older toolchains. + // + // We tested a more complicated fix in #1413, but ultimately decided that, + // since this lint is just a minor style lint, the complexity isn't worth it + // - it's fine to occasionally have unused qualifications slip through, + // especially since these do not affect our user-facing API in any way. + variant_size_differences +)] +#![cfg_attr( + __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, + deny(fuzzy_provenance_casts, lossy_provenance_casts) +)] +#![deny( + clippy::all, + clippy::alloc_instead_of_core, + clippy::arithmetic_side_effects, + clippy::as_underscore, + clippy::assertions_on_result_states, + clippy::as_conversions, + clippy::correctness, + clippy::dbg_macro, + clippy::decimal_literal_representation, + clippy::double_must_use, + clippy::get_unwrap, + clippy::indexing_slicing, + clippy::missing_inline_in_public_items, + clippy::missing_safety_doc, + clippy::multiple_unsafe_ops_per_block, + clippy::must_use_candidate, + clippy::must_use_unit, + clippy::obfuscated_if_else, + clippy::perf, + clippy::print_stdout, + clippy::return_self_not_must_use, + clippy::std_instead_of_core, + clippy::style, + clippy::suspicious, + clippy::todo, + clippy::undocumented_unsafe_blocks, + clippy::unimplemented, + clippy::unnested_or_patterns, + clippy::unwrap_used, + clippy::use_debug +)] +// `clippy::incompatible_msrv` (implied by `clippy::suspicious`): This sometimes +// has false positives, and we test on our MSRV in CI, so it doesn't help us +// anyway. +#![allow(clippy::needless_lifetimes, clippy::type_complexity, clippy::incompatible_msrv)] +#![deny( + rustdoc::bare_urls, + rustdoc::broken_intra_doc_links, + rustdoc::invalid_codeblock_attributes, + rustdoc::invalid_html_tags, + rustdoc::invalid_rust_codeblocks, + rustdoc::missing_crate_level_docs, + rustdoc::private_intra_doc_links +)] +// In test code, it makes sense to weight more heavily towards concise, readable +// code over correct or debuggable code. +#![cfg_attr(any(test, kani), allow( + // In tests, you get line numbers and have access to source code, so panic + // messages are less important. You also often unwrap a lot, which would + // make expect'ing instead very verbose. + clippy::unwrap_used, + // In tests, there's no harm to "panic risks" - the worst that can happen is + // that your test will fail, and you'll fix it. By contrast, panic risks in + // production code introduce the possibly of code panicking unexpectedly "in + // the field". + clippy::arithmetic_side_effects, + clippy::indexing_slicing, +))] +#![cfg_attr(not(any(test, kani, feature = "std")), no_std)] +#![cfg_attr( + all(feature = "simd-nightly", target_arch = "arm"), + feature(stdarch_arm_neon_intrinsics) +)] +#![cfg_attr( + all(feature = "simd-nightly", any(target_arch = "powerpc", target_arch = "powerpc64")), + feature(stdarch_powerpc) +)] +#![cfg_attr(feature = "float-nightly", feature(f16, f128))] +#![cfg_attr(doc_cfg, feature(doc_cfg))] +#![cfg_attr(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, feature(coverage_attribute))] +#![cfg_attr( + any(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, miri), + feature(layout_for_ptr) +)] +#![cfg_attr(all(test, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), feature(test))] + +// This is a hack to allow zerocopy-derive derives to work in this crate. They +// assume that zerocopy is linked as an extern crate, so they access items from +// it as `zerocopy::Xxx`. This makes that still work. +#[cfg(any(feature = "derive", test))] +extern crate self as zerocopy; + +#[cfg(all(test, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS))] +extern crate test; + +#[doc(hidden)] +#[macro_use] +pub mod util; + +pub mod byte_slice; +pub mod byteorder; +mod deprecated; + +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_DEV_MODE)] +pub mod doctests; + +// This module is `pub` so that zerocopy's error types and error handling +// documentation is grouped together in a cohesive module. In practice, we +// expect most users to use the re-export of `error`'s items to avoid identifier +// stuttering. +pub mod error; +mod impls; +#[doc(hidden)] +pub mod layout; +mod macros; +#[cfg_attr(not(zerocopy_unstable_ptr), doc(hidden))] +#[cfg_attr(doc_cfg, doc(cfg(zerocopy_unstable_ptr)))] +pub mod pointer; +mod r#ref; +mod split_at; +// FIXME(#252): If we make this pub, come up with a better name. +mod wrappers; + +use core::{ + cell::{Cell, UnsafeCell}, + cmp::Ordering, + fmt::{self, Debug, Display, Formatter}, + hash::Hasher, + marker::PhantomData, + mem::{self, ManuallyDrop, MaybeUninit as CoreMaybeUninit}, + num::{ + NonZeroI128, NonZeroI16, NonZeroI32, NonZeroI64, NonZeroI8, NonZeroIsize, NonZeroU128, + NonZeroU16, NonZeroU32, NonZeroU64, NonZeroU8, NonZeroUsize, Wrapping, + }, + ops::{Deref, DerefMut}, + ptr::{self, NonNull}, + slice, +}; +#[cfg(feature = "std")] +use std::io; + +#[doc(hidden)] +pub use crate::pointer::{ + invariant::{self, BecauseExclusive}, + PtrInner, +}; +pub use crate::{ + byte_slice::*, + byteorder::*, + error::*, + r#ref::*, + split_at::{Split, SplitAt}, + wrappers::*, +}; + +#[cfg(any(feature = "alloc", test, kani))] +extern crate alloc; +#[cfg(any(feature = "alloc", test))] +use alloc::{boxed::Box, vec::Vec}; +#[cfg(any(feature = "alloc", test))] +use core::alloc::Layout; + +// Used by `KnownLayout`. +#[doc(hidden)] +pub use crate::layout::*; +// Used by `TryFromBytes::is_bit_valid`. +#[doc(hidden)] +pub use crate::pointer::{invariant::BecauseImmutable, Maybe, Ptr}; +// For each trait polyfill, as soon as the corresponding feature is stable, the +// polyfill import will be unused because method/function resolution will prefer +// the inherent method/function over a trait method/function. Thus, we suppress +// the `unused_imports` warning. +// +// See the documentation on `util::polyfills` for more information. +#[allow(unused_imports)] +use crate::util::polyfills::{self, NonNullExt as _, NumExt as _}; +#[cfg_attr(not(zerocopy_unstable_ptr), doc(hidden))] +#[cfg_attr(doc_cfg, doc(cfg(zerocopy_unstable_ptr)))] +pub use crate::util::MetadataOf; + +#[cfg(all(test, not(__ZEROCOPY_INTERNAL_USE_ONLY_DEV_MODE)))] +const _: () = { + #[deprecated = "Development of zerocopy using cargo is not supported. Please use `cargo.sh` or `win-cargo.bat` instead."] + #[allow(unused)] + const WARNING: () = (); + #[warn(deprecated)] + WARNING +}; + +/// Implements [`KnownLayout`]. +/// +/// This derive analyzes various aspects of a type's layout that are needed for +/// some of zerocopy's APIs. It can be applied to structs, enums, and unions; +/// e.g.: +/// +/// ``` +/// # use zerocopy_derive::KnownLayout; +/// #[derive(KnownLayout)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(KnownLayout)] +/// enum MyEnum { +/// # V00, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(KnownLayout)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// # Limitations +/// +/// This derive cannot currently be applied to unsized structs without an +/// explicit `repr` attribute. +/// +/// Some invocations of this derive run afoul of a [known bug] in Rust's type +/// privacy checker. For example, this code: +/// +/// ```compile_fail,E0446 +/// use zerocopy::*; +/// # use zerocopy_derive::*; +/// +/// #[derive(KnownLayout)] +/// #[repr(C)] +/// pub struct PublicType { +/// leading: Foo, +/// trailing: Bar, +/// } +/// +/// #[derive(KnownLayout)] +/// struct Foo; +/// +/// #[derive(KnownLayout)] +/// struct Bar; +/// ``` +/// +/// ...results in a compilation error: +/// +/// ```text +/// error[E0446]: private type `Bar` in public interface +/// --> examples/bug.rs:3:10 +/// | +/// 3 | #[derive(KnownLayout)] +/// | ^^^^^^^^^^^ can't leak private type +/// ... +/// 14 | struct Bar; +/// | ---------- `Bar` declared as private +/// | +/// = note: this error originates in the derive macro `KnownLayout` (in Nightly builds, run with -Z macro-backtrace for more info) +/// ``` +/// +/// This issue arises when `#[derive(KnownLayout)]` is applied to `repr(C)` +/// structs whose trailing field type is less public than the enclosing struct. +/// +/// To work around this, mark the trailing field type `pub` and annotate it with +/// `#[doc(hidden)]`; e.g.: +/// +/// ```no_run +/// use zerocopy::*; +/// # use zerocopy_derive::*; +/// +/// #[derive(KnownLayout)] +/// #[repr(C)] +/// pub struct PublicType { +/// leading: Foo, +/// trailing: Bar, +/// } +/// +/// #[derive(KnownLayout)] +/// struct Foo; +/// +/// #[doc(hidden)] +/// #[derive(KnownLayout)] +/// pub struct Bar; // <- `Bar` is now also `pub` +/// ``` +/// +/// [known bug]: https://github.com/rust-lang/rust/issues/45713 +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::KnownLayout; +// These exist so that code which was written against the old names will get +// less confusing error messages when they upgrade to a more recent version of +// zerocopy. On our MSRV toolchain, the error messages read, for example: +// +// error[E0603]: trait `FromZeroes` is private +// --> examples/deprecated.rs:1:15 +// | +// 1 | use zerocopy::FromZeroes; +// | ^^^^^^^^^^ private trait +// | +// note: the trait `FromZeroes` is defined here +// --> /Users/josh/workspace/zerocopy/src/lib.rs:1845:5 +// | +// 1845 | use FromZeros as FromZeroes; +// | ^^^^^^^^^^^^^^^^^^^^^^^ +// +// The "note" provides enough context to make it easy to figure out how to fix +// the error. +#[allow(unused)] +use {FromZeros as FromZeroes, IntoBytes as AsBytes, Ref as LayoutVerified}; + +/// Indicates that zerocopy can reason about certain aspects of a type's layout. +/// +/// This trait is required by many of zerocopy's APIs. It supports sized types, +/// slices, and [slice DSTs](#dynamically-sized-types). +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(KnownLayout)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::KnownLayout; +/// #[derive(KnownLayout)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(KnownLayout)] +/// enum MyEnum { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(KnownLayout)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive performs a sophisticated analysis to deduce the layout +/// characteristics of types. You **must** implement this trait via the derive. +/// +/// # Dynamically-sized types +/// +/// `KnownLayout` supports slice-based dynamically sized types ("slice DSTs"). +/// +/// A slice DST is a type whose trailing field is either a slice or another +/// slice DST, rather than a type with fixed size. For example: +/// +/// ``` +/// #[repr(C)] +/// struct PacketHeader { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[repr(C)] +/// struct Packet { +/// header: PacketHeader, +/// body: [u8], +/// } +/// ``` +/// +/// It can be useful to think of slice DSTs as a generalization of slices - in +/// other words, a normal slice is just the special case of a slice DST with +/// zero leading fields. In particular: +/// - Like slices, slice DSTs can have different lengths at runtime +/// - Like slices, slice DSTs cannot be passed by-value, but only by reference +/// or via other indirection such as `Box` +/// - Like slices, a reference (or `Box`, or other pointer type) to a slice DST +/// encodes the number of elements in the trailing slice field +/// +/// ## Slice DST layout +/// +/// Just like other composite Rust types, the layout of a slice DST is not +/// well-defined unless it is specified using an explicit `#[repr(...)]` +/// attribute such as `#[repr(C)]`. [Other representations are +/// supported][reprs], but in this section, we'll use `#[repr(C)]` as our +/// example. +/// +/// A `#[repr(C)]` slice DST is laid out [just like sized `#[repr(C)]` +/// types][repr-c-structs], but the presence of a variable-length field +/// introduces the possibility of *dynamic padding*. In particular, it may be +/// necessary to add trailing padding *after* the trailing slice field in order +/// to satisfy the outer type's alignment, and the amount of padding required +/// may be a function of the length of the trailing slice field. This is just a +/// natural consequence of the normal `#[repr(C)]` rules applied to slice DSTs, +/// but it can result in surprising behavior. For example, consider the +/// following type: +/// +/// ``` +/// #[repr(C)] +/// struct Foo { +/// a: u32, +/// b: u8, +/// z: [u16], +/// } +/// ``` +/// +/// Assuming that `u32` has alignment 4 (this is not true on all platforms), +/// then `Foo` has alignment 4 as well. Here is the smallest possible value for +/// `Foo`: +/// +/// ```text +/// byte offset | 01234567 +/// field | aaaab--- +/// >< +/// ``` +/// +/// In this value, `z` has length 0. Abiding by `#[repr(C)]`, the lowest offset +/// that we can place `z` at is 5, but since `z` has alignment 2, we need to +/// round up to offset 6. This means that there is one byte of padding between +/// `b` and `z`, then 0 bytes of `z` itself (denoted `><` in this diagram), and +/// then two bytes of padding after `z` in order to satisfy the overall +/// alignment of `Foo`. The size of this instance is 8 bytes. +/// +/// What about if `z` has length 1? +/// +/// ```text +/// byte offset | 01234567 +/// field | aaaab-zz +/// ``` +/// +/// In this instance, `z` has length 1, and thus takes up 2 bytes. That means +/// that we no longer need padding after `z` in order to satisfy `Foo`'s +/// alignment. We've now seen two different values of `Foo` with two different +/// lengths of `z`, but they both have the same size - 8 bytes. +/// +/// What about if `z` has length 2? +/// +/// ```text +/// byte offset | 012345678901 +/// field | aaaab-zzzz-- +/// ``` +/// +/// Now `z` has length 2, and thus takes up 4 bytes. This brings our un-padded +/// size to 10, and so we now need another 2 bytes of padding after `z` to +/// satisfy `Foo`'s alignment. +/// +/// Again, all of this is just a logical consequence of the `#[repr(C)]` rules +/// applied to slice DSTs, but it can be surprising that the amount of trailing +/// padding becomes a function of the trailing slice field's length, and thus +/// can only be computed at runtime. +/// +/// [reprs]: https://doc.rust-lang.org/reference/type-layout.html#representations +/// [repr-c-structs]: https://doc.rust-lang.org/reference/type-layout.html#reprc-structs +/// +/// ## What is a valid size? +/// +/// There are two places in zerocopy's API that we refer to "a valid size" of a +/// type. In normal casts or conversions, where the source is a byte slice, we +/// need to know whether the source byte slice is a valid size of the +/// destination type. In prefix or suffix casts, we need to know whether *there +/// exists* a valid size of the destination type which fits in the source byte +/// slice and, if so, what the largest such size is. +/// +/// As outlined above, a slice DST's size is defined by the number of elements +/// in its trailing slice field. However, there is not necessarily a 1-to-1 +/// mapping between trailing slice field length and overall size. As we saw in +/// the previous section with the type `Foo`, instances with both 0 and 1 +/// elements in the trailing `z` field result in a `Foo` whose size is 8 bytes. +/// +/// When we say "x is a valid size of `T`", we mean one of two things: +/// - If `T: Sized`, then we mean that `x == size_of::<T>()` +/// - If `T` is a slice DST, then we mean that there exists a `len` such that the instance of +/// `T` with `len` trailing slice elements has size `x` +/// +/// When we say "largest possible size of `T` that fits in a byte slice", we +/// mean one of two things: +/// - If `T: Sized`, then we mean `size_of::<T>()` if the byte slice is at least +/// `size_of::<T>()` bytes long +/// - If `T` is a slice DST, then we mean to consider all values, `len`, such +/// that the instance of `T` with `len` trailing slice elements fits in the +/// byte slice, and to choose the largest such `len`, if any +/// +/// +/// # Safety +/// +/// This trait does not convey any safety guarantees to code outside this crate. +/// +/// You must not rely on the `#[doc(hidden)]` internals of `KnownLayout`. Future +/// releases of zerocopy may make backwards-breaking changes to these items, +/// including changes that only affect soundness, which may cause code which +/// uses those items to silently become unsound. +/// +#[cfg_attr(feature = "derive", doc = "[derive]: zerocopy_derive::KnownLayout")] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.KnownLayout.html"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(KnownLayout)]` to `{Self}`") +)] +pub unsafe trait KnownLayout { + // The `Self: Sized` bound makes it so that `KnownLayout` can still be + // object safe. It's not currently object safe thanks to `const LAYOUT`, and + // it likely won't be in the future, but there's no reason not to be + // forwards-compatible with object safety. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// The type of metadata stored in a pointer to `Self`. + /// + /// This is `()` for sized types and [`usize`] for slice DSTs. + type PointerMetadata: PointerMetadata; + + /// A maybe-uninitialized analog of `Self` + /// + /// # Safety + /// + /// `Self::LAYOUT` and `Self::MaybeUninit::LAYOUT` are identical. + /// `Self::MaybeUninit` admits uninitialized bytes in all positions. + #[doc(hidden)] + type MaybeUninit: ?Sized + KnownLayout<PointerMetadata = Self::PointerMetadata>; + + /// The layout of `Self`. + /// + /// # Safety + /// + /// Callers may assume that `LAYOUT` accurately reflects the layout of + /// `Self`. In particular: + /// - `LAYOUT.align` is equal to `Self`'s alignment + /// - If `Self: Sized`, then `LAYOUT.size_info == SizeInfo::Sized { size }` + /// where `size == size_of::<Self>()` + /// - If `Self` is a slice DST, then `LAYOUT.size_info == + /// SizeInfo::SliceDst(slice_layout)` where: + /// - The size, `size`, of an instance of `Self` with `elems` trailing + /// slice elements is equal to `slice_layout.offset + + /// slice_layout.elem_size * elems` rounded up to the nearest multiple + /// of `LAYOUT.align` + /// - For such an instance, any bytes in the range `[slice_layout.offset + + /// slice_layout.elem_size * elems, size)` are padding and must not be + /// assumed to be initialized + #[doc(hidden)] + const LAYOUT: DstLayout; + + /// SAFETY: The returned pointer has the same address and provenance as + /// `bytes`. If `Self` is a DST, the returned pointer's referent has `elems` + /// elements in its trailing slice. + #[doc(hidden)] + fn raw_from_ptr_len(bytes: NonNull<u8>, meta: Self::PointerMetadata) -> NonNull<Self>; + + /// Extracts the metadata from a pointer to `Self`. + /// + /// # Safety + /// + /// `pointer_to_metadata` always returns the correct metadata stored in + /// `ptr`. + #[doc(hidden)] + fn pointer_to_metadata(ptr: *mut Self) -> Self::PointerMetadata; + + /// Computes the length of the byte range addressed by `ptr`. + /// + /// Returns `None` if the resulting length would not fit in an `usize`. + /// + /// # Safety + /// + /// Callers may assume that `size_of_val_raw` always returns the correct + /// size. + /// + /// Callers may assume that, if `ptr` addresses a byte range whose length + /// fits in an `usize`, this will return `Some`. + #[doc(hidden)] + #[must_use] + #[inline(always)] + fn size_of_val_raw(ptr: NonNull<Self>) -> Option<usize> { + let meta = Self::pointer_to_metadata(ptr.as_ptr()); + // SAFETY: `size_for_metadata` promises to only return `None` if the + // resulting size would not fit in a `usize`. + Self::size_for_metadata(meta) + } + + #[doc(hidden)] + #[must_use] + #[inline(always)] + fn raw_dangling() -> NonNull<Self> { + let meta = Self::PointerMetadata::from_elem_count(0); + Self::raw_from_ptr_len(NonNull::dangling(), meta) + } + + /// Computes the size of an object of type `Self` with the given pointer + /// metadata. + /// + /// # Safety + /// + /// `size_for_metadata` promises to return `None` if and only if the + /// resulting size would not fit in a [`usize`]. Note that the returned size + /// could exceed the actual maximum valid size of an allocated object, + /// [`isize::MAX`]. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::KnownLayout; + /// + /// assert_eq!(u8::size_for_metadata(()), Some(1)); + /// assert_eq!(u16::size_for_metadata(()), Some(2)); + /// assert_eq!(<[u8]>::size_for_metadata(42), Some(42)); + /// assert_eq!(<[u16]>::size_for_metadata(42), Some(84)); + /// + /// // This size exceeds the maximum valid object size (`isize::MAX`): + /// assert_eq!(<[u8]>::size_for_metadata(usize::MAX), Some(usize::MAX)); + /// + /// // This size, if computed, would exceed `usize::MAX`: + /// assert_eq!(<[u16]>::size_for_metadata(usize::MAX), None); + /// ``` + #[inline(always)] + fn size_for_metadata(meta: Self::PointerMetadata) -> Option<usize> { + meta.size_for_metadata(Self::LAYOUT) + } + + /// Computes whether `meta` can describe a valid allocation of `Self`. + /// + /// # Safety + /// + /// `is_valid_metadata` promises to return `true` if and only if the size of + /// an allocation of `Self` with `meta` would not overflow an + /// [`isize::MAX`]. + #[doc(hidden)] + #[inline(always)] + fn is_valid_metadata(meta: Self::PointerMetadata) -> bool { + meta.to_elem_count() <= maximum_trailing_slice_len::<Self>().to_elem_count() + } +} + +/// Efficiently produces the [`TrailingSliceLayout`] of `T`. +#[inline(always)] +pub(crate) fn trailing_slice_layout<T>() -> TrailingSliceLayout +where + T: ?Sized + KnownLayout<PointerMetadata = usize>, +{ + trait LayoutFacts { + const SIZE_INFO: TrailingSliceLayout; + } + + impl<T: ?Sized> LayoutFacts for T + where + T: KnownLayout<PointerMetadata = usize>, + { + const SIZE_INFO: TrailingSliceLayout = match T::LAYOUT.size_info { + crate::SizeInfo::Sized { .. } => const_panic!("unreachable"), + crate::SizeInfo::SliceDst(info) => info, + }; + } + + T::SIZE_INFO +} + +/// Efficiently produces the maximum trailing slice length `T`. +#[inline(always)] +pub(crate) fn maximum_trailing_slice_len<T>() -> usize +where + T: ?Sized + KnownLayout, +{ + trait LayoutFacts { + const MAX_LEN: usize; + } + + impl<T: ?Sized> LayoutFacts for T + where + T: KnownLayout, + { + const MAX_LEN: usize = match T::LAYOUT.size_info { + SizeInfo::SliceDst(TrailingSliceLayout { elem_size: 0, .. }) => usize::MAX, + _ => match T::LAYOUT.validate_cast_and_convert_metadata( + T::LAYOUT.align.get(), + DstLayout::MAX_SIZE, + CastType::Prefix, + ) { + Ok((elems, _)) => elems, + Err(_) => const_panic!("unreachable"), + }, + }; + } + + T::MAX_LEN +} + +/// The metadata associated with a [`KnownLayout`] type. +#[doc(hidden)] +pub trait PointerMetadata: Copy + Eq + Debug + Ord { + /// Constructs a `Self` from an element count. + /// + /// If `Self = ()`, this returns `()`. If `Self = usize`, this returns + /// `elems`. No other types are currently supported. + fn from_elem_count(elems: usize) -> Self; + + /// Converts `self` to an element count. + /// + /// If `Self = ()`, this returns `0`. If `Self = usize`, this returns + /// `self`. No other types are currently supported. + fn to_elem_count(self) -> usize; + + /// Computes the size of the object with the given layout and pointer + /// metadata. + /// + /// # Panics + /// + /// If `Self = ()`, `layout` must describe a sized type. If `Self = usize`, + /// `layout` must describe a slice DST. Otherwise, `size_for_metadata` may + /// panic. + /// + /// # Safety + /// + /// `size_for_metadata` promises to only return `None` if the resulting size + /// would not fit in a `usize`. + fn size_for_metadata(self, layout: DstLayout) -> Option<usize>; +} + +impl PointerMetadata for () { + #[inline] + #[allow(clippy::unused_unit)] + fn from_elem_count(_elems: usize) -> () {} + + #[inline] + fn to_elem_count(self) -> usize { + 0 + } + + #[inline] + fn size_for_metadata(self, layout: DstLayout) -> Option<usize> { + match layout.size_info { + SizeInfo::Sized { size } => Some(size), + // NOTE: This branch is unreachable, but we return `None` rather + // than `unreachable!()` to avoid generating panic paths. + SizeInfo::SliceDst(_) => None, + } + } +} + +impl PointerMetadata for usize { + #[inline] + fn from_elem_count(elems: usize) -> usize { + elems + } + + #[inline] + fn to_elem_count(self) -> usize { + self + } + + #[inline] + fn size_for_metadata(self, layout: DstLayout) -> Option<usize> { + match layout.size_info { + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { + let slice_len = elem_size.checked_mul(self)?; + let without_padding = offset.checked_add(slice_len)?; + without_padding.checked_add(util::padding_needed_for(without_padding, layout.align)) + } + // NOTE: This branch is unreachable, but we return `None` rather + // than `unreachable!()` to avoid generating panic paths. + SizeInfo::Sized { .. } => None, + } + } +} + +// SAFETY: Delegates safety to `DstLayout::for_slice`. +unsafe impl<T> KnownLayout for [T] { + #[allow(clippy::missing_inline_in_public_items, dead_code)] + #[cfg_attr( + all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + coverage(off) + )] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + type PointerMetadata = usize; + + // SAFETY: `CoreMaybeUninit<T>::LAYOUT` and `T::LAYOUT` are identical + // because `CoreMaybeUninit<T>` has the same size and alignment as `T` [1]. + // Consequently, `[CoreMaybeUninit<T>]::LAYOUT` and `[T]::LAYOUT` are + // identical, because they both lack a fixed-sized prefix and because they + // inherit the alignments of their inner element type (which are identical) + // [2][3]. + // + // `[CoreMaybeUninit<T>]` admits uninitialized bytes at all positions + // because `CoreMaybeUninit<T>` admits uninitialized bytes at all positions + // and because the inner elements of `[CoreMaybeUninit<T>]` are laid out + // back-to-back [2][3]. + // + // [1] Per https://doc.rust-lang.org/1.81.0/std/mem/union.MaybeUninit.html#layout-1: + // + // `MaybeUninit<T>` is guaranteed to have the same size, alignment, and ABI as + // `T` + // + // [2] Per https://doc.rust-lang.org/1.82.0/reference/type-layout.html#slice-layout: + // + // Slices have the same layout as the section of the array they slice. + // + // [3] Per https://doc.rust-lang.org/1.82.0/reference/type-layout.html#array-layout: + // + // An array of `[T; N]` has a size of `size_of::<T>() * N` and the same + // alignment of `T`. Arrays are laid out so that the zero-based `nth` + // element of the array is offset from the start of the array by `n * + // size_of::<T>()` bytes. + type MaybeUninit = [CoreMaybeUninit<T>]; + + const LAYOUT: DstLayout = DstLayout::for_slice::<T>(); + + // SAFETY: `.cast` preserves address and provenance. The returned pointer + // refers to an object with `elems` elements by construction. + #[inline(always)] + fn raw_from_ptr_len(data: NonNull<u8>, elems: usize) -> NonNull<Self> { + // FIXME(#67): Remove this allow. See NonNullExt for more details. + #[allow(unstable_name_collisions)] + NonNull::slice_from_raw_parts(data.cast::<T>(), elems) + } + + #[inline(always)] + fn pointer_to_metadata(ptr: *mut [T]) -> usize { + #[allow(clippy::as_conversions)] + let slc = ptr as *const [()]; + + // SAFETY: + // - `()` has alignment 1, so `slc` is trivially aligned. + // - `slc` was derived from a non-null pointer. + // - The size is 0 regardless of the length, so it is sound to + // materialize a reference regardless of location. + // - By invariant, `self.ptr` has valid provenance. + let slc = unsafe { &*slc }; + + // This is correct because the preceding `as` cast preserves the number + // of slice elements. [1] + // + // [1] Per https://doc.rust-lang.org/reference/expressions/operator-expr.html#pointer-to-pointer-cast: + // + // For slice types like `[T]` and `[U]`, the raw pointer types `*const + // [T]`, `*mut [T]`, `*const [U]`, and `*mut [U]` encode the number of + // elements in this slice. Casts between these raw pointer types + // preserve the number of elements. ... The same holds for `str` and + // any compound type whose unsized tail is a slice type, such as + // struct `Foo(i32, [u8])` or `(u64, Foo)`. + slc.len() + } +} + +#[rustfmt::skip] +impl_known_layout!( + (), + u8, i8, u16, i16, u32, i32, u64, i64, u128, i128, usize, isize, f32, f64, + bool, char, + NonZeroU8, NonZeroI8, NonZeroU16, NonZeroI16, NonZeroU32, NonZeroI32, + NonZeroU64, NonZeroI64, NonZeroU128, NonZeroI128, NonZeroUsize, NonZeroIsize +); +#[rustfmt::skip] +#[cfg(feature = "float-nightly")] +impl_known_layout!( + #[cfg_attr(doc_cfg, doc(cfg(feature = "float-nightly")))] + f16, + #[cfg_attr(doc_cfg, doc(cfg(feature = "float-nightly")))] + f128 +); +#[rustfmt::skip] +impl_known_layout!( + T => Option<T>, + T: ?Sized => PhantomData<T>, + T => Wrapping<T>, + T => CoreMaybeUninit<T>, + T: ?Sized => *const T, + T: ?Sized => *mut T, + T: ?Sized => &'_ T, + T: ?Sized => &'_ mut T, +); +impl_known_layout!(const N: usize, T => [T; N]); + +// SAFETY: `str` has the same representation as `[u8]`. `ManuallyDrop<T>` [1], +// `UnsafeCell<T>` [2], and `Cell<T>` [3] have the same representation as `T`. +// +// [1] Per https://doc.rust-lang.org/1.85.0/std/mem/struct.ManuallyDrop.html: +// +// `ManuallyDrop<T>` is guaranteed to have the same layout and bit validity as +// `T` +// +// [2] Per https://doc.rust-lang.org/1.85.0/core/cell/struct.UnsafeCell.html#memory-layout: +// +// `UnsafeCell<T>` has the same in-memory representation as its inner type +// `T`. +// +// [3] Per https://doc.rust-lang.org/1.85.0/core/cell/struct.Cell.html#memory-layout: +// +// `Cell<T>` has the same in-memory representation as `T`. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl_known_layout!( + #[repr([u8])] + str + ); + unsafe_impl_known_layout!(T: ?Sized + KnownLayout => #[repr(T)] ManuallyDrop<T>); + unsafe_impl_known_layout!(T: ?Sized + KnownLayout => #[repr(T)] UnsafeCell<T>); + unsafe_impl_known_layout!(T: ?Sized + KnownLayout => #[repr(T)] Cell<T>); +}; + +// SAFETY: +// - By consequence of the invariant on `T::MaybeUninit` that `T::LAYOUT` and +// `T::MaybeUninit::LAYOUT` are equal, `T` and `T::MaybeUninit` have the same: +// - Fixed prefix size +// - Alignment +// - (For DSTs) trailing slice element size +// - By consequence of the above, referents `T::MaybeUninit` and `T` have the +// require the same kind of pointer metadata, and thus it is valid to perform +// an `as` cast from `*mut T` and `*mut T::MaybeUninit`, and this operation +// preserves referent size (ie, `size_of_val_raw`). +const _: () = unsafe { + unsafe_impl_known_layout!(T: ?Sized + KnownLayout => #[repr(T::MaybeUninit)] MaybeUninit<T>) +}; + +// FIXME(#196, #2856): Eventually, we'll want to support enums variants and +// union fields being treated uniformly since they behave similarly to each +// other in terms of projecting validity – specifically, for a type `T` with +// validity `V`, if `T` is a struct type, then its fields straightforwardly also +// have validity `V`. By contrast, if `T` is an enum or union type, then +// validity is not straightforwardly recursive in this way. +#[doc(hidden)] +pub const STRUCT_VARIANT_ID: i128 = -1; +#[doc(hidden)] +pub const UNION_VARIANT_ID: i128 = -2; +#[doc(hidden)] +pub const REPR_C_UNION_VARIANT_ID: i128 = -3; + +/// # Safety +/// +/// `Self::ProjectToTag` must satisfy its safety invariant. +#[doc(hidden)] +pub unsafe trait HasTag { + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// The type's enum tag, or `()` for non-enum types. + type Tag: Immutable; + + /// A pointer projection from `Self` to its tag. + /// + /// # Safety + /// + /// It must be the case that, for all `slf: Ptr<'_, Self, I>`, it is sound + /// to project from `slf` to `Ptr<'_, Self::Tag, I>` using this projection. + type ProjectToTag: pointer::cast::Project<Self, Self::Tag>; +} + +/// Projects a given field from `Self`. +/// +/// All implementations of `HasField` for a particular field `f` in `Self` +/// should use the same `Field` type; this ensures that `Field` is inferable +/// given an explicit `VARIANT_ID` and `FIELD_ID`. +/// +/// # Safety +/// +/// A field `f` is `HasField` for `Self` if and only if: +/// +/// - If `Self` has the layout of a struct or union type, then `VARIANT_ID` is +/// `STRUCT_VARIANT_ID` or `UNION_VARIANT_ID` respectively; otherwise, if +/// `Self` has the layout of an enum type, `VARIANT_ID` is the numerical index +/// of the enum variant in which `f` appears. Note that `Self` does not need +/// to actually *be* such a type – it just needs to have the same layout as +/// such a type. For example, a `#[repr(transparent)]` wrapper around an enum +/// has the same layout as that enum. +/// - If `f` has name `n`, `FIELD_ID` is `zerocopy::ident_id!(n)`; otherwise, +/// if `f` is at index `i`, `FIELD_ID` is `zerocopy::ident_id!(i)`. +/// - `Field` is a type with the same visibility as `f`. +/// - `Type` has the same type as `f`. +/// +/// The caller must **not** assume that a pointer's referent being aligned +/// implies that calling `project` on that pointer will result in a pointer to +/// an aligned referent. For example, `HasField` may be implemented for +/// `#[repr(packed)]` structs. +/// +/// The implementation of `project` must satisfy its safety post-condition. +#[doc(hidden)] +pub unsafe trait HasField<Field, const VARIANT_ID: i128, const FIELD_ID: i128>: + HasTag +{ + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// The type of the field. + type Type: ?Sized; + + /// Projects from `slf` to the field. + /// + /// Users should generally not call `project` directly, and instead should + /// use high-level APIs like [`PtrInner::project`] or [`Ptr::project`]. + /// + /// # Safety + /// + /// The returned pointer refers to a non-strict subset of the bytes of + /// `slf`'s referent, and has the same provenance as `slf`. + #[must_use] + fn project(slf: PtrInner<'_, Self>) -> *mut Self::Type; +} + +/// Projects a given field from `Self`. +/// +/// Implementations of this trait encode the conditions under which a field can +/// be projected from a `Ptr<'_, Self, I>`, and how the invariants of that +/// [`Ptr`] (`I`) determine the invariants of pointers projected from it. In +/// other words, it is a type-level function over invariants; `I` goes in, +/// `Self::Invariants` comes out. +/// +/// # Safety +/// +/// `T: ProjectField<Field, I, VARIANT_ID, FIELD_ID>` if, for a +/// `ptr: Ptr<'_, T, I>` such that `T::is_projectable(ptr).is_ok()`, +/// `<T as HasField<Field, VARIANT_ID, FIELD_ID>>::project(ptr.as_inner())` +/// conforms to `T::Invariants`. +#[doc(hidden)] +pub unsafe trait ProjectField<Field, I, const VARIANT_ID: i128, const FIELD_ID: i128>: + HasField<Field, VARIANT_ID, FIELD_ID> +where + I: invariant::Invariants, +{ + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// The invariants of the projected field pointer, with respect to the + /// invariants, `I`, of the containing pointer. The aliasing dimension of + /// the invariants is guaranteed to remain unchanged. + type Invariants: invariant::Invariants<Aliasing = I::Aliasing>; + + /// The failure mode of projection. `()` if the projection is fallible, + /// otherwise [`core::convert::Infallible`]. + type Error; + + /// Is the given field projectable from `ptr`? + /// + /// If a field with [`Self::Invariants`] is projectable from the referent, + /// this function produces an `Ok(ptr)` from which the projection can be + /// made; otherwise `Err`. + /// + /// This method must be overriden if the field's projectability depends on + /// the value of the bytes in `ptr`. + #[inline(always)] + fn is_projectable<'a>(_ptr: Ptr<'a, Self::Tag, I>) -> Result<(), Self::Error> { + trait IsInfallible { + const IS_INFALLIBLE: bool; + } + + struct Projection<T, Field, I, const VARIANT_ID: i128, const FIELD_ID: i128>( + PhantomData<(Field, I, T)>, + ) + where + T: ?Sized + HasField<Field, VARIANT_ID, FIELD_ID>, + I: invariant::Invariants; + + impl<T, Field, I, const VARIANT_ID: i128, const FIELD_ID: i128> IsInfallible + for Projection<T, Field, I, VARIANT_ID, FIELD_ID> + where + T: ?Sized + HasField<Field, VARIANT_ID, FIELD_ID>, + I: invariant::Invariants, + { + const IS_INFALLIBLE: bool = { + let is_infallible = match VARIANT_ID { + // For nondestructive projections of struct and union + // fields, the projected field's satisfaction of + // `Invariants` does not depend on the value of the + // referent. This default implementation of `is_projectable` + // is non-destructive, as it does not overwrite any part of + // the referent. + crate::STRUCT_VARIANT_ID | crate::UNION_VARIANT_ID => true, + _enum_variant => { + use crate::invariant::{Validity, ValidityKind}; + match I::Validity::KIND { + // The `Uninit` and `Initialized` validity + // invariants do not depend on the enum's tag. In + // particular, we don't actually care about what + // variant is present – we can treat *any* range of + // uninitialized or initialized memory as containing + // an uninitialized or initialized instance of *any* + // type – the type itself is irrelevant. + ValidityKind::Uninit | ValidityKind::Initialized => true, + // The projectability of an enum field from an + // `AsInitialized` or `Valid` state is a dynamic + // property of its tag. + ValidityKind::AsInitialized | ValidityKind::Valid => false, + } + } + }; + const_assert!(is_infallible); + is_infallible + }; + } + + const_assert!( + <Projection<Self, Field, I, VARIANT_ID, FIELD_ID> as IsInfallible>::IS_INFALLIBLE + ); + + Ok(()) + } +} + +/// Analyzes whether a type is [`FromZeros`]. +/// +/// This derive analyzes, at compile time, whether the annotated type satisfies +/// the [safety conditions] of `FromZeros` and implements `FromZeros` and its +/// supertraits if it is sound to do so. This derive can be applied to structs, +/// enums, and unions; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{FromZeros, Immutable}; +/// #[derive(FromZeros)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromZeros)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant0, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromZeros, Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// [safety conditions]: trait@FromZeros#safety +/// +/// # Analysis +/// +/// *This section describes, roughly, the analysis performed by this derive to +/// determine whether it is sound to implement `FromZeros` for a given type. +/// Unless you are modifying the implementation of this derive, or attempting to +/// manually implement `FromZeros` for a type yourself, you don't need to read +/// this section.* +/// +/// If a type has the following properties, then this derive can implement +/// `FromZeros` for that type: +/// +/// - If the type is a struct, all of its fields must be `FromZeros`. +/// - If the type is an enum: +/// - It must have a defined representation (`repr`s `C`, `u8`, `u16`, `u32`, +/// `u64`, `usize`, `i8`, `i16`, `i32`, `i64`, or `isize`). +/// - It must have a variant with a discriminant/tag of `0`, and its fields +/// must be `FromZeros`. See [the reference] for a description of +/// discriminant values are specified. +/// - The fields of that variant must be `FromZeros`. +/// +/// This analysis is subject to change. Unsafe code may *only* rely on the +/// documented [safety conditions] of `FromZeros`, and must *not* rely on the +/// implementation details of this derive. +/// +/// [the reference]: https://doc.rust-lang.org/reference/items/enumerations.html#custom-discriminant-values-for-fieldless-enumerations +/// +/// ## Why isn't an explicit representation required for structs? +/// +/// Neither this derive, nor the [safety conditions] of `FromZeros`, requires +/// that structs are marked with `#[repr(C)]`. +/// +/// Per the [Rust reference](reference), +/// +/// > The representation of a type can change the padding between fields, but +/// > does not change the layout of the fields themselves. +/// +/// [reference]: https://doc.rust-lang.org/reference/type-layout.html#representations +/// +/// Since the layout of structs only consists of padding bytes and field bytes, +/// a struct is soundly `FromZeros` if: +/// 1. its padding is soundly `FromZeros`, and +/// 2. its fields are soundly `FromZeros`. +/// +/// The answer to the first question is always yes: padding bytes do not have +/// any validity constraints. A [discussion] of this question in the Unsafe Code +/// Guidelines Working Group concluded that it would be virtually unimaginable +/// for future versions of rustc to add validity constraints to padding bytes. +/// +/// [discussion]: https://github.com/rust-lang/unsafe-code-guidelines/issues/174 +/// +/// Whether a struct is soundly `FromZeros` therefore solely depends on whether +/// its fields are `FromZeros`. +// FIXME(#146): Document why we don't require an enum to have an explicit `repr` +// attribute. +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::FromZeros; +/// Analyzes whether a type is [`Immutable`]. +/// +/// This derive analyzes, at compile time, whether the annotated type satisfies +/// the [safety conditions] of `Immutable` and implements `Immutable` if it is +/// sound to do so. This derive can be applied to structs, enums, and unions; +/// e.g.: +/// +/// ``` +/// # use zerocopy_derive::Immutable; +/// #[derive(Immutable)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Immutable)] +/// enum MyEnum { +/// # Variant0, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// # Analysis +/// +/// *This section describes, roughly, the analysis performed by this derive to +/// determine whether it is sound to implement `Immutable` for a given type. +/// Unless you are modifying the implementation of this derive, you don't need +/// to read this section.* +/// +/// If a type has the following properties, then this derive can implement +/// `Immutable` for that type: +/// +/// - All fields must be `Immutable`. +/// +/// This analysis is subject to change. Unsafe code may *only* rely on the +/// documented [safety conditions] of `Immutable`, and must *not* rely on the +/// implementation details of this derive. +/// +/// [safety conditions]: trait@Immutable#safety +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::Immutable; + +/// Types which are free from interior mutability. +/// +/// `T: Immutable` indicates that `T` does not permit interior mutation, except +/// by ownership or an exclusive (`&mut`) borrow. +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(Immutable)]`][derive] (requires the `derive` Cargo feature); +/// e.g.: +/// +/// ``` +/// # use zerocopy_derive::Immutable; +/// #[derive(Immutable)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Immutable)] +/// enum MyEnum { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive performs a sophisticated, compile-time safety analysis to +/// determine whether a type is `Immutable`. +/// +/// # Safety +/// +/// Unsafe code outside of this crate must not make any assumptions about `T` +/// based on `T: Immutable`. We reserve the right to relax the requirements for +/// `Immutable` in the future, and if unsafe code outside of this crate makes +/// assumptions based on `T: Immutable`, future relaxations may cause that code +/// to become unsound. +/// +// # Safety (Internal) +// +// If `T: Immutable`, unsafe code *inside of this crate* may assume that, given +// `t: &T`, `t` does not permit interior mutation of its referent. Because +// [`UnsafeCell`] is the only type which permits interior mutation, it is +// sufficient (though not necessary) to guarantee that `T` contains no +// `UnsafeCell`s. +// +// [`UnsafeCell`]: core::cell::UnsafeCell +#[cfg_attr( + feature = "derive", + doc = "[derive]: zerocopy_derive::Immutable", + doc = "[derive-analysis]: zerocopy_derive::Immutable#analysis" +)] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.Immutable.html"), + doc = concat!("[derive-analysis]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.Immutable.html#analysis"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(Immutable)]` to `{Self}`") +)] +pub unsafe trait Immutable { + // The `Self: Sized` bound makes it so that `Immutable` is still object + // safe. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; +} + +/// Implements [`TryFromBytes`]. +/// +/// This derive synthesizes the runtime checks required to check whether a +/// sequence of initialized bytes corresponds to a valid instance of a type. +/// This derive can be applied to structs, enums, and unions; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{TryFromBytes, Immutable}; +/// #[derive(TryFromBytes)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(TryFromBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # V00, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(TryFromBytes, Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// # Portability +/// +/// To ensure consistent endianness for enums with multi-byte representations, +/// explicitly specify and convert each discriminant using `.to_le()` or +/// `.to_be()`; e.g.: +/// +/// ``` +/// # use zerocopy_derive::TryFromBytes; +/// // `DataStoreVersion` is encoded in little-endian. +/// #[derive(TryFromBytes)] +/// #[repr(u32)] +/// pub enum DataStoreVersion { +/// /// Version 1 of the data store. +/// V1 = 9u32.to_le(), +/// +/// /// Version 2 of the data store. +/// V2 = 10u32.to_le(), +/// } +/// ``` +/// +/// [safety conditions]: trait@TryFromBytes#safety +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::TryFromBytes; + +/// Types for which some bit patterns are valid. +/// +/// A memory region of the appropriate length which contains initialized bytes +/// can be viewed as a `TryFromBytes` type so long as the runtime value of those +/// bytes corresponds to a [*valid instance*] of that type. For example, +/// [`bool`] is `TryFromBytes`, so zerocopy can transmute a [`u8`] into a +/// [`bool`] so long as it first checks that the value of the [`u8`] is `0` or +/// `1`. +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(TryFromBytes)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{TryFromBytes, Immutable}; +/// #[derive(TryFromBytes)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(TryFromBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # V00, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(TryFromBytes, Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive ensures that the runtime check of whether bytes correspond to a +/// valid instance is sound. You **must** implement this trait via the derive. +/// +/// # What is a "valid instance"? +/// +/// In Rust, each type has *bit validity*, which refers to the set of bit +/// patterns which may appear in an instance of that type. It is impossible for +/// safe Rust code to produce values which violate bit validity (ie, values +/// outside of the "valid" set of bit patterns). If `unsafe` code produces an +/// invalid value, this is considered [undefined behavior]. +/// +/// Rust's bit validity rules are currently being decided, which means that some +/// types have three classes of bit patterns: those which are definitely valid, +/// and whose validity is documented in the language; those which may or may not +/// be considered valid at some point in the future; and those which are +/// definitely invalid. +/// +/// Zerocopy takes a conservative approach, and only considers a bit pattern to +/// be valid if its validity is a documented guarantee provided by the +/// language. +/// +/// For most use cases, Rust's current guarantees align with programmers' +/// intuitions about what ought to be valid. As a result, zerocopy's +/// conservatism should not affect most users. +/// +/// If you are negatively affected by lack of support for a particular type, +/// we encourage you to let us know by [filing an issue][github-repo]. +/// +/// # `TryFromBytes` is not symmetrical with [`IntoBytes`] +/// +/// There are some types which implement both `TryFromBytes` and [`IntoBytes`], +/// but for which `TryFromBytes` is not guaranteed to accept all byte sequences +/// produced by `IntoBytes`. In other words, for some `T: TryFromBytes + +/// IntoBytes`, there exist values of `t: T` such that +/// `TryFromBytes::try_ref_from_bytes(t.as_bytes()) == None`. Code should not +/// generally assume that values produced by `IntoBytes` will necessarily be +/// accepted as valid by `TryFromBytes`. +/// +/// # Safety +/// +/// On its own, `T: TryFromBytes` does not make any guarantees about the layout +/// or representation of `T`. It merely provides the ability to perform a +/// validity check at runtime via methods like [`try_ref_from_bytes`]. +/// +/// You must not rely on the `#[doc(hidden)]` internals of `TryFromBytes`. +/// Future releases of zerocopy may make backwards-breaking changes to these +/// items, including changes that only affect soundness, which may cause code +/// which uses those items to silently become unsound. +/// +/// [undefined behavior]: https://raphlinus.github.io/programming/rust/2018/08/17/undefined-behavior.html +/// [github-repo]: https://github.com/google/zerocopy +/// [`try_ref_from_bytes`]: TryFromBytes::try_ref_from_bytes +/// [*valid instance*]: #what-is-a-valid-instance +#[cfg_attr(feature = "derive", doc = "[derive]: zerocopy_derive::TryFromBytes")] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.TryFromBytes.html"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(TryFromBytes)]` to `{Self}`") +)] +pub unsafe trait TryFromBytes { + // The `Self: Sized` bound makes it so that `TryFromBytes` is still object + // safe. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// Does a given memory range contain a valid instance of `Self`? + /// + /// # Safety + /// + /// Unsafe code may assume that, if `is_bit_valid(candidate)` returns true, + /// `*candidate` contains a valid `Self`. + /// + /// # Panics + /// + /// `is_bit_valid` may panic. Callers are responsible for ensuring that any + /// `unsafe` code remains sound even in the face of `is_bit_valid` + /// panicking. (We support user-defined validation routines; so long as + /// these routines are not required to be `unsafe`, there is no way to + /// ensure that these do not generate panics.) + /// + /// Besides user-defined validation routines panicking, `is_bit_valid` will + /// either panic or fail to compile if called on a pointer with [`Shared`] + /// aliasing when `Self: !Immutable`. + /// + /// [`UnsafeCell`]: core::cell::UnsafeCell + /// [`Shared`]: invariant::Shared + #[doc(hidden)] + fn is_bit_valid<A>(candidate: Maybe<'_, Self, A>) -> bool + where + A: invariant::Alignment; + + /// Attempts to interpret the given `source` as a `&Self`. + /// + /// If the bytes of `source` are a valid instance of `Self`, this method + /// returns a reference to those bytes interpreted as a `Self`. If the + /// length of `source` is not a [valid size of `Self`][valid-size], or if + /// `source` is not appropriately aligned, or if `source` is not a valid + /// instance of `Self`, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][ConvertError::from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = ZSTy::try_ref_from_bytes(0u16.as_bytes()); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the byte sequence `0xC0C0`. + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &[0xC0, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5][..]; + /// + /// let packet = Packet::try_ref_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[0, 1], [2, 3], [4, 5]]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0x10, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5][..]; + /// assert!(Packet::try_ref_from_bytes(bytes).is_err()); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_ref_from_bytes", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_ref_from_bytes(source: &[u8]) -> Result<&Self, TryCastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + static_assert_dst_is_not_zst!(Self); + match Ptr::from_ref(source).try_cast_into_no_leftover::<Self, BecauseImmutable>(None) { + Ok(source) => { + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to + // fix before returning. + match source.try_into_valid() { + Ok(valid) => Ok(valid.as_ref()), + Err(e) => { + Err(e.map_src(|src| src.as_bytes::<BecauseImmutable>().as_ref()).into()) + } + } + } + Err(e) => Err(e.map_src(Ptr::as_ref).into()), + } + } + + /// Attempts to interpret the prefix of the given `source` as a `&Self`. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the leading bytes of `source`. If that prefix is a valid + /// instance of `Self`, this method returns a reference to those bytes + /// interpreted as `Self`, and a reference to the remaining bytes. If there + /// are insufficient bytes, or if `source` is not appropriately aligned, or + /// if those bytes are not a valid instance of `Self`, this returns `Err`. + /// If [`Self: Unaligned`][self-unaligned], you can [infallibly discard the + /// alignment error][ConvertError::from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = ZSTy::try_ref_from_prefix(0u16.as_bytes()); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &[0xC0, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// + /// let (packet, suffix) = Packet::try_ref_from_prefix(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[0, 1], [2, 3], [4, 5]]); + /// assert_eq!(suffix, &[6u8][..]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0x10, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// assert!(Packet::try_ref_from_prefix(bytes).is_err()); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_ref_from_prefix", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_ref_from_prefix(source: &[u8]) -> Result<(&Self, &[u8]), TryCastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + static_assert_dst_is_not_zst!(Self); + try_ref_from_prefix_suffix(source, CastType::Prefix, None) + } + + /// Attempts to interpret the suffix of the given `source` as a `&Self`. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the trailing bytes of `source`. If that suffix is a + /// valid instance of `Self`, this method returns a reference to those bytes + /// interpreted as `Self`, and a reference to the preceding bytes. If there + /// are insufficient bytes, or if the suffix of `source` would not be + /// appropriately aligned, or if the suffix is not a valid instance of + /// `Self`, this returns `Err`. If [`Self: Unaligned`][self-unaligned], you + /// can [infallibly discard the alignment error][ConvertError::from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = ZSTy::try_ref_from_suffix(0u16.as_bytes()); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &[0, 0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7][..]; + /// + /// let (prefix, packet) = Packet::try_ref_from_suffix(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// assert_eq!(prefix, &[0u8][..]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 77, 240, 0xC0, 0x10][..]; + /// assert!(Packet::try_ref_from_suffix(bytes).is_err()); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_ref_from_suffix", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_ref_from_suffix(source: &[u8]) -> Result<(&[u8], &Self), TryCastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + static_assert_dst_is_not_zst!(Self); + try_ref_from_prefix_suffix(source, CastType::Suffix, None).map(swap) + } + + /// Attempts to interpret the given `source` as a `&mut Self` without + /// copying. + /// + /// If the bytes of `source` are a valid instance of `Self`, this method + /// returns a reference to those bytes interpreted as a `Self`. If the + /// length of `source` is not a [valid size of `Self`][valid-size], or if + /// `source` is not appropriately aligned, or if `source` is not a valid + /// instance of `Self`, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][ConvertError::from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let mut source = [85, 85]; + /// let _ = ZSTy::try_mut_from_bytes(&mut source[..]); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &mut [0xC0, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5][..]; + /// + /// let packet = Packet::try_mut_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[0, 1], [2, 3], [4, 5]]); + /// + /// packet.temperature = 111; + /// + /// assert_eq!(bytes, [0xC0, 0xC0, 240, 111, 0, 1, 2, 3, 4, 5]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0x10, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// assert!(Packet::try_mut_from_bytes(bytes).is_err()); + /// ``` + /// + #[doc = codegen_header!("h5", "try_mut_from_bytes")] + /// + /// See [`TryFromBytes::try_ref_from_bytes`](#method.try_ref_from_bytes.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_mut_from_bytes(bytes: &mut [u8]) -> Result<&mut Self, TryCastError<&mut [u8], Self>> + where + Self: KnownLayout + IntoBytes, + { + static_assert_dst_is_not_zst!(Self); + match Ptr::from_mut(bytes).try_cast_into_no_leftover::<Self, BecauseExclusive>(None) { + Ok(source) => { + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to + // fix before returning. + match source.try_into_valid() { + Ok(source) => Ok(source.as_mut()), + Err(e) => Err(e.map_src(|src| src.as_bytes().as_mut()).into()), + } + } + Err(e) => Err(e.map_src(Ptr::as_mut).into()), + } + } + + /// Attempts to interpret the prefix of the given `source` as a `&mut + /// Self`. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the leading bytes of `source`. If that prefix is a valid + /// instance of `Self`, this method returns a reference to those bytes + /// interpreted as `Self`, and a reference to the remaining bytes. If there + /// are insufficient bytes, or if `source` is not appropriately aligned, or + /// if the bytes are not a valid instance of `Self`, this returns `Err`. If + /// [`Self: Unaligned`][self-unaligned], you can [infallibly discard the + /// alignment error][ConvertError::from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let mut source = [85, 85]; + /// let _ = ZSTy::try_mut_from_prefix(&mut source[..]); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &mut [0xC0, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// + /// let (packet, suffix) = Packet::try_mut_from_prefix(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[0, 1], [2, 3], [4, 5]]); + /// assert_eq!(suffix, &[6u8][..]); + /// + /// packet.temperature = 111; + /// suffix[0] = 222; + /// + /// assert_eq!(bytes, [0xC0, 0xC0, 240, 111, 0, 1, 2, 3, 4, 5, 222]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0x10, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// assert!(Packet::try_mut_from_prefix(bytes).is_err()); + /// ``` + /// + #[doc = codegen_header!("h5", "try_mut_from_prefix")] + /// + /// See [`TryFromBytes::try_ref_from_prefix`](#method.try_ref_from_prefix.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_mut_from_prefix( + source: &mut [u8], + ) -> Result<(&mut Self, &mut [u8]), TryCastError<&mut [u8], Self>> + where + Self: KnownLayout + IntoBytes, + { + static_assert_dst_is_not_zst!(Self); + try_mut_from_prefix_suffix(source, CastType::Prefix, None) + } + + /// Attempts to interpret the suffix of the given `source` as a `&mut + /// Self`. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the trailing bytes of `source`. If that suffix is a + /// valid instance of `Self`, this method returns a reference to those bytes + /// interpreted as `Self`, and a reference to the preceding bytes. If there + /// are insufficient bytes, or if the suffix of `source` would not be + /// appropriately aligned, or if the suffix is not a valid instance of + /// `Self`, this returns `Err`. If [`Self: Unaligned`][self-unaligned], you + /// can [infallibly discard the alignment error][ConvertError::from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let mut source = [85, 85]; + /// let _ = ZSTy::try_mut_from_suffix(&mut source[..]); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &mut [0, 0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7][..]; + /// + /// let (prefix, packet) = Packet::try_mut_from_suffix(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// assert_eq!(prefix, &[0u8][..]); + /// + /// prefix[0] = 111; + /// packet.temperature = 222; + /// + /// assert_eq!(bytes, [111, 0xC0, 0xC0, 240, 222, 2, 3, 4, 5, 6, 7]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 77, 240, 0xC0, 0x10][..]; + /// assert!(Packet::try_mut_from_suffix(bytes).is_err()); + /// ``` + /// + #[doc = codegen_header!("h5", "try_mut_from_suffix")] + /// + /// See [`TryFromBytes::try_ref_from_suffix`](#method.try_ref_from_suffix.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_mut_from_suffix( + source: &mut [u8], + ) -> Result<(&mut [u8], &mut Self), TryCastError<&mut [u8], Self>> + where + Self: KnownLayout + IntoBytes, + { + static_assert_dst_is_not_zst!(Self); + try_mut_from_prefix_suffix(source, CastType::Suffix, None).map(swap) + } + + /// Attempts to interpret the given `source` as a `&Self` with a DST length + /// equal to `count`. + /// + /// This method attempts to return a reference to `source` interpreted as a + /// `Self` with `count` trailing elements. If the length of `source` is not + /// equal to the size of `Self` with `count` elements, if `source` is not + /// appropriately aligned, or if `source` does not contain a valid instance + /// of `Self`, this returns `Err`. If [`Self: Unaligned`][self-unaligned], + /// you can [infallibly discard the alignment error][ConvertError::from]. + /// + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &[0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7][..]; + /// + /// let packet = Packet::try_ref_from_bytes_with_elems(bytes, 3).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 77, 240, 0xC0, 0xC0][..]; + /// assert!(Packet::try_ref_from_bytes_with_elems(bytes, 3).is_err()); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`try_ref_from_bytes`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use core::num::NonZeroU16; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: NonZeroU16, + /// trailing_dst: [()], + /// } + /// + /// let src = 0xCAFEu16.as_bytes(); + /// let zsty = ZSTy::try_ref_from_bytes_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`try_ref_from_bytes`]: TryFromBytes::try_ref_from_bytes + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_ref_from_bytes_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_ref_from_bytes_with_elems( + source: &[u8], + count: usize, + ) -> Result<&Self, TryCastError<&[u8], Self>> + where + Self: KnownLayout<PointerMetadata = usize> + Immutable, + { + match Ptr::from_ref(source).try_cast_into_no_leftover::<Self, BecauseImmutable>(Some(count)) + { + Ok(source) => { + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to + // fix before returning. + match source.try_into_valid() { + Ok(source) => Ok(source.as_ref()), + Err(e) => { + Err(e.map_src(|src| src.as_bytes::<BecauseImmutable>().as_ref()).into()) + } + } + } + Err(e) => Err(e.map_src(Ptr::as_ref).into()), + } + } + + /// Attempts to interpret the prefix of the given `source` as a `&Self` with + /// a DST length equal to `count`. + /// + /// This method attempts to return a reference to the prefix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the remaining bytes. If the length of `source` is less than the size + /// of `Self` with `count` elements, if `source` is not appropriately + /// aligned, or if the prefix of `source` does not contain a valid instance + /// of `Self`, this returns `Err`. If [`Self: Unaligned`][self-unaligned], + /// you can [infallibly discard the alignment error][ConvertError::from]. + /// + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &[0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7, 8][..]; + /// + /// let (packet, suffix) = Packet::try_ref_from_prefix_with_elems(bytes, 3).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// assert_eq!(suffix, &[8u8][..]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 77, 240, 0xC0, 0xC0][..]; + /// assert!(Packet::try_ref_from_prefix_with_elems(bytes, 3).is_err()); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`try_ref_from_prefix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use core::num::NonZeroU16; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: NonZeroU16, + /// trailing_dst: [()], + /// } + /// + /// let src = 0xCAFEu16.as_bytes(); + /// let (zsty, _) = ZSTy::try_ref_from_prefix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`try_ref_from_prefix`]: TryFromBytes::try_ref_from_prefix + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_ref_from_prefix_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_ref_from_prefix_with_elems( + source: &[u8], + count: usize, + ) -> Result<(&Self, &[u8]), TryCastError<&[u8], Self>> + where + Self: KnownLayout<PointerMetadata = usize> + Immutable, + { + try_ref_from_prefix_suffix(source, CastType::Prefix, Some(count)) + } + + /// Attempts to interpret the suffix of the given `source` as a `&Self` with + /// a DST length equal to `count`. + /// + /// This method attempts to return a reference to the suffix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the preceding bytes. If the length of `source` is less than the size + /// of `Self` with `count` elements, if the suffix of `source` is not + /// appropriately aligned, or if the suffix of `source` does not contain a + /// valid instance of `Self`, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][ConvertError::from]. + /// + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &[123, 0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7][..]; + /// + /// let (prefix, packet) = Packet::try_ref_from_suffix_with_elems(bytes, 3).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// assert_eq!(prefix, &[123u8][..]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 77, 240, 0xC0, 0xC0][..]; + /// assert!(Packet::try_ref_from_suffix_with_elems(bytes, 3).is_err()); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`try_ref_from_prefix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use core::num::NonZeroU16; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: NonZeroU16, + /// trailing_dst: [()], + /// } + /// + /// let src = 0xCAFEu16.as_bytes(); + /// let (_, zsty) = ZSTy::try_ref_from_suffix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`try_ref_from_prefix`]: TryFromBytes::try_ref_from_prefix + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_ref_from_suffix_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_ref_from_suffix_with_elems( + source: &[u8], + count: usize, + ) -> Result<(&[u8], &Self), TryCastError<&[u8], Self>> + where + Self: KnownLayout<PointerMetadata = usize> + Immutable, + { + try_ref_from_prefix_suffix(source, CastType::Suffix, Some(count)).map(swap) + } + + /// Attempts to interpret the given `source` as a `&mut Self` with a DST + /// length equal to `count`. + /// + /// This method attempts to return a reference to `source` interpreted as a + /// `Self` with `count` trailing elements. If the length of `source` is not + /// equal to the size of `Self` with `count` elements, if `source` is not + /// appropriately aligned, or if `source` does not contain a valid instance + /// of `Self`, this returns `Err`. If [`Self: Unaligned`][self-unaligned], + /// you can [infallibly discard the alignment error][ConvertError::from]. + /// + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &mut [0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7][..]; + /// + /// let packet = Packet::try_mut_from_bytes_with_elems(bytes, 3).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// + /// packet.temperature = 111; + /// + /// assert_eq!(bytes, [0xC0, 0xC0, 240, 111, 2, 3, 4, 5, 6, 7]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 77, 240, 0xC0, 0xC0][..]; + /// assert!(Packet::try_mut_from_bytes_with_elems(bytes, 3).is_err()); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`try_mut_from_bytes`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use core::num::NonZeroU16; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: NonZeroU16, + /// trailing_dst: [()], + /// } + /// + /// let mut src = 0xCAFEu16; + /// let src = src.as_mut_bytes(); + /// let zsty = ZSTy::try_mut_from_bytes_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`try_mut_from_bytes`]: TryFromBytes::try_mut_from_bytes + /// + #[doc = codegen_header!("h5", "try_mut_from_bytes_with_elems")] + /// + /// See [`TryFromBytes::try_ref_from_bytes_with_elems`](#method.try_ref_from_bytes_with_elems.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_mut_from_bytes_with_elems( + source: &mut [u8], + count: usize, + ) -> Result<&mut Self, TryCastError<&mut [u8], Self>> + where + Self: KnownLayout<PointerMetadata = usize> + IntoBytes, + { + match Ptr::from_mut(source).try_cast_into_no_leftover::<Self, BecauseExclusive>(Some(count)) + { + Ok(source) => { + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to + // fix before returning. + match source.try_into_valid() { + Ok(source) => Ok(source.as_mut()), + Err(e) => Err(e.map_src(|src| src.as_bytes().as_mut()).into()), + } + } + Err(e) => Err(e.map_src(Ptr::as_mut).into()), + } + } + + /// Attempts to interpret the prefix of the given `source` as a `&mut Self` + /// with a DST length equal to `count`. + /// + /// This method attempts to return a reference to the prefix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the remaining bytes. If the length of `source` is less than the size + /// of `Self` with `count` elements, if `source` is not appropriately + /// aligned, or if the prefix of `source` does not contain a valid instance + /// of `Self`, this returns `Err`. If [`Self: Unaligned`][self-unaligned], + /// you can [infallibly discard the alignment error][ConvertError::from]. + /// + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &mut [0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7, 8][..]; + /// + /// let (packet, suffix) = Packet::try_mut_from_prefix_with_elems(bytes, 3).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// assert_eq!(suffix, &[8u8][..]); + /// + /// packet.temperature = 111; + /// suffix[0] = 222; + /// + /// assert_eq!(bytes, [0xC0, 0xC0, 240, 111, 2, 3, 4, 5, 6, 7, 222]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 77, 240, 0xC0, 0xC0][..]; + /// assert!(Packet::try_mut_from_prefix_with_elems(bytes, 3).is_err()); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`try_mut_from_prefix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use core::num::NonZeroU16; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: NonZeroU16, + /// trailing_dst: [()], + /// } + /// + /// let mut src = 0xCAFEu16; + /// let src = src.as_mut_bytes(); + /// let (zsty, _) = ZSTy::try_mut_from_prefix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`try_mut_from_prefix`]: TryFromBytes::try_mut_from_prefix + /// + #[doc = codegen_header!("h5", "try_mut_from_prefix_with_elems")] + /// + /// See [`TryFromBytes::try_ref_from_prefix_with_elems`](#method.try_ref_from_prefix_with_elems.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_mut_from_prefix_with_elems( + source: &mut [u8], + count: usize, + ) -> Result<(&mut Self, &mut [u8]), TryCastError<&mut [u8], Self>> + where + Self: KnownLayout<PointerMetadata = usize> + IntoBytes, + { + try_mut_from_prefix_suffix(source, CastType::Prefix, Some(count)) + } + + /// Attempts to interpret the suffix of the given `source` as a `&mut Self` + /// with a DST length equal to `count`. + /// + /// This method attempts to return a reference to the suffix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the preceding bytes. If the length of `source` is less than the size + /// of `Self` with `count` elements, if the suffix of `source` is not + /// appropriately aligned, or if the suffix of `source` does not contain a + /// valid instance of `Self`, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][ConvertError::from]. + /// + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &mut [123, 0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7][..]; + /// + /// let (prefix, packet) = Packet::try_mut_from_suffix_with_elems(bytes, 3).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// assert_eq!(prefix, &[123u8][..]); + /// + /// prefix[0] = 111; + /// packet.temperature = 222; + /// + /// assert_eq!(bytes, [111, 0xC0, 0xC0, 240, 222, 2, 3, 4, 5, 6, 7]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 77, 240, 0xC0, 0xC0][..]; + /// assert!(Packet::try_mut_from_suffix_with_elems(bytes, 3).is_err()); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`try_mut_from_prefix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use core::num::NonZeroU16; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: NonZeroU16, + /// trailing_dst: [()], + /// } + /// + /// let mut src = 0xCAFEu16; + /// let src = src.as_mut_bytes(); + /// let (_, zsty) = ZSTy::try_mut_from_suffix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`try_mut_from_prefix`]: TryFromBytes::try_mut_from_prefix + /// + #[doc = codegen_header!("h5", "try_mut_from_suffix_with_elems")] + /// + /// See [`TryFromBytes::try_ref_from_suffix_with_elems`](#method.try_ref_from_suffix_with_elems.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_mut_from_suffix_with_elems( + source: &mut [u8], + count: usize, + ) -> Result<(&mut [u8], &mut Self), TryCastError<&mut [u8], Self>> + where + Self: KnownLayout<PointerMetadata = usize> + IntoBytes, + { + try_mut_from_prefix_suffix(source, CastType::Suffix, Some(count)).map(swap) + } + + /// Attempts to read the given `source` as a `Self`. + /// + /// If `source.len() != size_of::<Self>()` or the bytes are not a valid + /// instance of `Self`, this returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// } + /// + /// let bytes = &[0xC0, 0xC0, 240, 77][..]; + /// + /// let packet = Packet::try_read_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0x10, 0xC0, 240, 77][..]; + /// assert!(Packet::try_read_from_bytes(bytes).is_err()); + /// ``` + /// + /// # Performance Considerations + /// + /// In this version of zerocopy, this method reads the `source` into a + /// well-aligned stack allocation and *then* validates that the allocation + /// is a valid `Self`. This ensures that validation can be performed using + /// aligned reads (which carry a performance advantage over unaligned reads + /// on many platforms) at the cost of an unconditional copy. + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_read_from_bytes", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_read_from_bytes(source: &[u8]) -> Result<Self, TryReadError<&[u8], Self>> + where + Self: Sized, + { + // FIXME(#2981): If `align_of::<Self>() == 1`, validate `source` in-place. + + let candidate = match CoreMaybeUninit::<Self>::read_from_bytes(source) { + Ok(candidate) => candidate, + Err(e) => { + return Err(TryReadError::Size(e.with_dst())); + } + }; + // SAFETY: `candidate` was copied from from `source: &[u8]`, so all of + // its bytes are initialized. + unsafe { try_read_from(source, candidate) } + } + + /// Attempts to read a `Self` from the prefix of the given `source`. + /// + /// This attempts to read a `Self` from the first `size_of::<Self>()` bytes + /// of `source`, returning that `Self` and any remaining bytes. If + /// `source.len() < size_of::<Self>()` or the bytes are not a valid instance + /// of `Self`, it returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &[0xC0, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// + /// let (packet, suffix) = Packet::try_read_from_prefix(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(suffix, &[0u8, 1, 2, 3, 4, 5, 6][..]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0x10, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// assert!(Packet::try_read_from_prefix(bytes).is_err()); + /// ``` + /// + /// # Performance Considerations + /// + /// In this version of zerocopy, this method reads the `source` into a + /// well-aligned stack allocation and *then* validates that the allocation + /// is a valid `Self`. This ensures that validation can be performed using + /// aligned reads (which carry a performance advantage over unaligned reads + /// on many platforms) at the cost of an unconditional copy. + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_read_from_prefix", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_read_from_prefix(source: &[u8]) -> Result<(Self, &[u8]), TryReadError<&[u8], Self>> + where + Self: Sized, + { + // FIXME(#2981): If `align_of::<Self>() == 1`, validate `source` in-place. + + let (candidate, suffix) = match CoreMaybeUninit::<Self>::read_from_prefix(source) { + Ok(candidate) => candidate, + Err(e) => { + return Err(TryReadError::Size(e.with_dst())); + } + }; + // SAFETY: `candidate` was copied from from `source: &[u8]`, so all of + // its bytes are initialized. + unsafe { try_read_from(source, candidate).map(|slf| (slf, suffix)) } + } + + /// Attempts to read a `Self` from the suffix of the given `source`. + /// + /// This attempts to read a `Self` from the last `size_of::<Self>()` bytes + /// of `source`, returning that `Self` and any preceding bytes. If + /// `source.len() < size_of::<Self>()` or the bytes are not a valid instance + /// of `Self`, it returns `Err`. + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 0xC0, 0xC0, 240, 77][..]; + /// + /// let (prefix, packet) = Packet::try_read_from_suffix(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(prefix, &[0u8, 1, 2, 3, 4, 5][..]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 0x10, 0xC0, 240, 77][..]; + /// assert!(Packet::try_read_from_suffix(bytes).is_err()); + /// ``` + /// + /// # Performance Considerations + /// + /// In this version of zerocopy, this method reads the `source` into a + /// well-aligned stack allocation and *then* validates that the allocation + /// is a valid `Self`. This ensures that validation can be performed using + /// aligned reads (which carry a performance advantage over unaligned reads + /// on many platforms) at the cost of an unconditional copy. + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_read_from_suffix", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_read_from_suffix(source: &[u8]) -> Result<(&[u8], Self), TryReadError<&[u8], Self>> + where + Self: Sized, + { + // FIXME(#2981): If `align_of::<Self>() == 1`, validate `source` in-place. + + let (prefix, candidate) = match CoreMaybeUninit::<Self>::read_from_suffix(source) { + Ok(candidate) => candidate, + Err(e) => { + return Err(TryReadError::Size(e.with_dst())); + } + }; + // SAFETY: `candidate` was copied from from `source: &[u8]`, so all of + // its bytes are initialized. + unsafe { try_read_from(source, candidate).map(|slf| (prefix, slf)) } + } +} + +#[inline(always)] +fn try_ref_from_prefix_suffix<T: TryFromBytes + KnownLayout + Immutable + ?Sized>( + source: &[u8], + cast_type: CastType, + meta: Option<T::PointerMetadata>, +) -> Result<(&T, &[u8]), TryCastError<&[u8], T>> { + match Ptr::from_ref(source).try_cast_into::<T, BecauseImmutable>(cast_type, meta) { + Ok((source, prefix_suffix)) => { + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to + // fix before returning. + match source.try_into_valid() { + Ok(valid) => Ok((valid.as_ref(), prefix_suffix.as_ref())), + Err(e) => Err(e.map_src(|src| src.as_bytes::<BecauseImmutable>().as_ref()).into()), + } + } + Err(e) => Err(e.map_src(Ptr::as_ref).into()), + } +} + +#[inline(always)] +fn try_mut_from_prefix_suffix<T: IntoBytes + TryFromBytes + KnownLayout + ?Sized>( + candidate: &mut [u8], + cast_type: CastType, + meta: Option<T::PointerMetadata>, +) -> Result<(&mut T, &mut [u8]), TryCastError<&mut [u8], T>> { + match Ptr::from_mut(candidate).try_cast_into::<T, BecauseExclusive>(cast_type, meta) { + Ok((candidate, prefix_suffix)) => { + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to + // fix before returning. + match candidate.try_into_valid() { + Ok(valid) => Ok((valid.as_mut(), prefix_suffix.as_mut())), + Err(e) => Err(e.map_src(|src| src.as_bytes().as_mut()).into()), + } + } + Err(e) => Err(e.map_src(Ptr::as_mut).into()), + } +} + +#[inline(always)] +fn swap<T, U>((t, u): (T, U)) -> (U, T) { + (u, t) +} + +/// # Safety +/// +/// All bytes of `candidate` must be initialized. +#[inline(always)] +unsafe fn try_read_from<S, T: TryFromBytes>( + source: S, + mut candidate: CoreMaybeUninit<T>, +) -> Result<T, TryReadError<S, T>> { + // We use `from_mut` despite not mutating via `c_ptr` so that we don't need + // to add a `T: Immutable` bound. + let c_ptr = Ptr::from_mut(&mut candidate); + // SAFETY: `c_ptr` has no uninitialized sub-ranges because it derived from + // `candidate`, which the caller promises is entirely initialized. Since + // `candidate` is a `MaybeUninit`, it has no validity requirements, and so + // no values written to an `Initialized` `c_ptr` can violate its validity. + // Since `c_ptr` has `Exclusive` aliasing, no mutations may happen except + // via `c_ptr` so long as it is live, so we don't need to worry about the + // fact that `c_ptr` may have more restricted validity than `candidate`. + let c_ptr = unsafe { c_ptr.assume_validity::<invariant::Initialized>() }; + let mut c_ptr = c_ptr.cast::<_, crate::pointer::cast::CastSized, _>(); + + // Since we don't have `T: KnownLayout`, we hack around that by using + // `Wrapping<T>`, which implements `KnownLayout` even if `T` doesn't. + // + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to fix + // before returning. + if !Wrapping::<T>::is_bit_valid(c_ptr.reborrow_shared().forget_aligned()) { + return Err(ValidityError::new(source).into()); + } + + fn _assert_same_size_and_validity<T>() + where + Wrapping<T>: pointer::TransmuteFrom<T, invariant::Valid, invariant::Valid>, + T: pointer::TransmuteFrom<Wrapping<T>, invariant::Valid, invariant::Valid>, + { + } + + _assert_same_size_and_validity::<T>(); + + // SAFETY: We just validated that `candidate` contains a valid + // `Wrapping<T>`, which has the same size and bit validity as `T`, as + // guaranteed by the preceding type assertion. + Ok(unsafe { candidate.assume_init() }) +} + +/// Types for which a sequence of `0` bytes is a valid instance. +/// +/// Any memory region of the appropriate length which is guaranteed to contain +/// only zero bytes can be viewed as any `FromZeros` type with no runtime +/// overhead. This is useful whenever memory is known to be in a zeroed state, +/// such memory returned from some allocation routines. +/// +/// # Warning: Padding bytes +/// +/// Note that, when a value is moved or copied, only the non-padding bytes of +/// that value are guaranteed to be preserved. It is unsound to assume that +/// values written to padding bytes are preserved after a move or copy. For more +/// details, see the [`FromBytes` docs][frombytes-warning-padding-bytes]. +/// +/// [frombytes-warning-padding-bytes]: FromBytes#warning-padding-bytes +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(FromZeros)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{FromZeros, Immutable}; +/// #[derive(FromZeros)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromZeros)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant0, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromZeros, Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive performs a sophisticated, compile-time safety analysis to +/// determine whether a type is `FromZeros`. +/// +/// # Safety +/// +/// *This section describes what is required in order for `T: FromZeros`, and +/// what unsafe code may assume of such types. If you don't plan on implementing +/// `FromZeros` manually, and you don't plan on writing unsafe code that +/// operates on `FromZeros` types, then you don't need to read this section.* +/// +/// If `T: FromZeros`, then unsafe code may assume that it is sound to produce a +/// `T` whose bytes are all initialized to zero. If a type is marked as +/// `FromZeros` which violates this contract, it may cause undefined behavior. +/// +/// `#[derive(FromZeros)]` only permits [types which satisfy these +/// requirements][derive-analysis]. +/// +#[cfg_attr( + feature = "derive", + doc = "[derive]: zerocopy_derive::FromZeros", + doc = "[derive-analysis]: zerocopy_derive::FromZeros#analysis" +)] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.FromZeros.html"), + doc = concat!("[derive-analysis]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.FromZeros.html#analysis"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(FromZeros)]` to `{Self}`") +)] +pub unsafe trait FromZeros: TryFromBytes { + // The `Self: Sized` bound makes it so that `FromZeros` is still object + // safe. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// Overwrites `self` with zeros. + /// + /// Sets every byte in `self` to 0. While this is similar to doing `*self = + /// Self::new_zeroed()`, it differs in that `zero` does not semantically + /// drop the current value and replace it with a new one — it simply + /// modifies the bytes of the existing value. + /// + /// # Examples + /// + /// ``` + /// # use zerocopy::FromZeros; + /// # use zerocopy_derive::*; + /// # + /// #[derive(FromZeros)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let mut header = PacketHeader { + /// src_port: 100u16.to_be_bytes(), + /// dst_port: 200u16.to_be_bytes(), + /// length: 300u16.to_be_bytes(), + /// checksum: 400u16.to_be_bytes(), + /// }; + /// + /// header.zero(); + /// + /// assert_eq!(header.src_port, [0, 0]); + /// assert_eq!(header.dst_port, [0, 0]); + /// assert_eq!(header.length, [0, 0]); + /// assert_eq!(header.checksum, [0, 0]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "zero", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[inline(always)] + fn zero(&mut self) { + let slf: *mut Self = self; + let len = mem::size_of_val(self); + // SAFETY: + // - `self` is guaranteed by the type system to be valid for writes of + // size `size_of_val(self)`. + // - `u8`'s alignment is 1, and thus `self` is guaranteed to be aligned + // as required by `u8`. + // - Since `Self: FromZeros`, the all-zeros instance is a valid instance + // of `Self.` + // + // FIXME(#429): Add references to docs and quotes. + unsafe { ptr::write_bytes(slf.cast::<u8>(), 0, len) }; + } + + /// Creates an instance of `Self` from zeroed bytes. + /// + /// # Examples + /// + /// ``` + /// # use zerocopy::FromZeros; + /// # use zerocopy_derive::*; + /// # + /// #[derive(FromZeros)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let header: PacketHeader = FromZeros::new_zeroed(); + /// + /// assert_eq!(header.src_port, [0, 0]); + /// assert_eq!(header.dst_port, [0, 0]); + /// assert_eq!(header.length, [0, 0]); + /// assert_eq!(header.checksum, [0, 0]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "new_zeroed", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[inline(always)] + fn new_zeroed() -> Self + where + Self: Sized, + { + // SAFETY: `FromZeros` says that the all-zeros bit pattern is legal. + unsafe { mem::zeroed() } + } + + /// Creates a `Box<Self>` from zeroed bytes. + /// + /// This function is useful for allocating large values on the heap and + /// zero-initializing them, without ever creating a temporary instance of + /// `Self` on the stack. For example, `<[u8; 1048576]>::new_box_zeroed()` + /// will allocate `[u8; 1048576]` directly on the heap; it does not require + /// storing `[u8; 1048576]` in a temporary variable on the stack. + /// + /// On systems that use a heap implementation that supports allocating from + /// pre-zeroed memory, using `new_box_zeroed` (or related functions) may + /// have performance benefits. + /// + /// # Errors + /// + /// Returns an error on allocation failure. Allocation failure is guaranteed + /// never to cause a panic or an abort. + /// + #[doc = codegen_section!( + header = "h5", + bench = "new_box_zeroed", + format = "coco_static_size", + )] + #[must_use = "has no side effects (other than allocation)"] + #[cfg(any(feature = "alloc", test))] + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + #[inline] + fn new_box_zeroed() -> Result<Box<Self>, AllocError> + where + Self: Sized, + { + // If `T` is a ZST, then return a proper boxed instance of it. There is + // no allocation, but `Box` does require a correct dangling pointer. + let layout = Layout::new::<Self>(); + if layout.size() == 0 { + // Construct the `Box` from a dangling pointer to avoid calling + // `Self::new_zeroed`. This ensures that stack space is never + // allocated for `Self` even on lower opt-levels where this branch + // might not get optimized out. + + // SAFETY: Per [1], when `T` is a ZST, `Box<T>`'s only validity + // requirements are that the pointer is non-null and sufficiently + // aligned. Per [2], `NonNull::dangling` produces a pointer which + // is sufficiently aligned. Since the produced pointer is a + // `NonNull`, it is non-null. + // + // [1] Per https://doc.rust-lang.org/1.81.0/std/boxed/index.html#memory-layout: + // + // For zero-sized values, the `Box` pointer has to be non-null and sufficiently aligned. + // + // [2] Per https://doc.rust-lang.org/std/ptr/struct.NonNull.html#method.dangling: + // + // Creates a new `NonNull` that is dangling, but well-aligned. + return Ok(unsafe { Box::from_raw(NonNull::dangling().as_ptr()) }); + } + + // FIXME(#429): Add a "SAFETY" comment and remove this `allow`. + #[allow(clippy::undocumented_unsafe_blocks)] + let ptr = unsafe { alloc::alloc::alloc_zeroed(layout).cast::<Self>() }; + if ptr.is_null() { + return Err(AllocError); + } + // FIXME(#429): Add a "SAFETY" comment and remove this `allow`. + #[allow(clippy::undocumented_unsafe_blocks)] + Ok(unsafe { Box::from_raw(ptr) }) + } + + /// Creates a `Box<[Self]>` (a boxed slice) from zeroed bytes. + /// + /// This function is useful for allocating large values of `[Self]` on the + /// heap and zero-initializing them, without ever creating a temporary + /// instance of `[Self; _]` on the stack. For example, + /// `u8::new_box_slice_zeroed(1048576)` will allocate the slice directly on + /// the heap; it does not require storing the slice on the stack. + /// + /// On systems that use a heap implementation that supports allocating from + /// pre-zeroed memory, using `new_box_slice_zeroed` may have performance + /// benefits. + /// + /// If `Self` is a zero-sized type, then this function will return a + /// `Box<[Self]>` that has the correct `len`. Such a box cannot contain any + /// actual information, but its `len()` property will report the correct + /// value. + /// + /// # Errors + /// + /// Returns an error on allocation failure. Allocation failure is + /// guaranteed never to cause a panic or an abort. + /// + #[doc = codegen_section!( + header = "h5", + bench = "new_box_zeroed_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects (other than allocation)"] + #[cfg(feature = "alloc")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + #[inline] + fn new_box_zeroed_with_elems(count: usize) -> Result<Box<Self>, AllocError> + where + Self: KnownLayout<PointerMetadata = usize>, + { + // SAFETY: `alloc::alloc::alloc_zeroed` is a valid argument of + // `new_box`. The referent of the pointer returned by `alloc_zeroed` + // (and, consequently, the `Box` derived from it) is a valid instance of + // `Self`, because `Self` is `FromZeros`. + unsafe { crate::util::new_box(count, alloc::alloc::alloc_zeroed) } + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromZeros::new_box_zeroed_with_elems`")] + #[doc(hidden)] + #[cfg(feature = "alloc")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + #[must_use = "has no side effects (other than allocation)"] + #[inline(always)] + fn new_box_slice_zeroed(len: usize) -> Result<Box<[Self]>, AllocError> + where + Self: Sized, + { + <[Self]>::new_box_zeroed_with_elems(len) + } + + /// Creates a `Vec<Self>` from zeroed bytes. + /// + /// This function is useful for allocating large values of `Vec`s and + /// zero-initializing them, without ever creating a temporary instance of + /// `[Self; _]` (or many temporary instances of `Self`) on the stack. For + /// example, `u8::new_vec_zeroed(1048576)` will allocate directly on the + /// heap; it does not require storing intermediate values on the stack. + /// + /// On systems that use a heap implementation that supports allocating from + /// pre-zeroed memory, using `new_vec_zeroed` may have performance benefits. + /// + /// If `Self` is a zero-sized type, then this function will return a + /// `Vec<Self>` that has the correct `len`. Such a `Vec` cannot contain any + /// actual information, but its `len()` property will report the correct + /// value. + /// + /// # Errors + /// + /// Returns an error on allocation failure. Allocation failure is + /// guaranteed never to cause a panic or an abort. + /// + #[doc = codegen_section!( + header = "h5", + bench = "new_vec_zeroed", + format = "coco_static_size", + )] + #[must_use = "has no side effects (other than allocation)"] + #[cfg(feature = "alloc")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + #[inline(always)] + fn new_vec_zeroed(len: usize) -> Result<Vec<Self>, AllocError> + where + Self: Sized, + { + <[Self]>::new_box_zeroed_with_elems(len).map(Into::into) + } + + /// Extends a `Vec<Self>` by pushing `additional` new items onto the end of + /// the vector. The new items are initialized with zeros. + /// + #[doc = codegen_section!( + header = "h5", + bench = "extend_vec_zeroed", + format = "coco_static_size", + )] + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[cfg(feature = "alloc")] + #[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.57.0", feature = "alloc"))))] + #[inline(always)] + fn extend_vec_zeroed(v: &mut Vec<Self>, additional: usize) -> Result<(), AllocError> + where + Self: Sized, + { + // PANICS: We pass `v.len()` for `position`, so the `position > v.len()` + // panic condition is not satisfied. + <Self as FromZeros>::insert_vec_zeroed(v, v.len(), additional) + } + + /// Inserts `additional` new items into `Vec<Self>` at `position`. The new + /// items are initialized with zeros. + /// + /// # Panics + /// + /// Panics if `position > v.len()`. + /// + #[doc = codegen_section!( + header = "h5", + bench = "insert_vec_zeroed", + format = "coco_static_size", + )] + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[cfg(feature = "alloc")] + #[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.57.0", feature = "alloc"))))] + #[inline] + fn insert_vec_zeroed( + v: &mut Vec<Self>, + position: usize, + additional: usize, + ) -> Result<(), AllocError> + where + Self: Sized, + { + assert!(position <= v.len()); + // We only conditionally compile on versions on which `try_reserve` is + // stable; the Clippy lint is a false positive. + v.try_reserve(additional).map_err(|_| AllocError)?; + // SAFETY: The `try_reserve` call guarantees that these cannot overflow: + // * `ptr.add(position)` + // * `position + additional` + // * `v.len() + additional` + // + // `v.len() - position` cannot overflow because we asserted that + // `position <= v.len()`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + unsafe { + // This is a potentially overlapping copy. + let ptr = v.as_mut_ptr(); + #[allow(clippy::arithmetic_side_effects)] + ptr.add(position).copy_to(ptr.add(position + additional), v.len() - position); + ptr.add(position).write_bytes(0, additional); + #[allow(clippy::arithmetic_side_effects)] + v.set_len(v.len() + additional); + } + + Ok(()) + } +} + +/// Analyzes whether a type is [`FromBytes`]. +/// +/// This derive analyzes, at compile time, whether the annotated type satisfies +/// the [safety conditions] of `FromBytes` and implements `FromBytes` and its +/// supertraits if it is sound to do so. This derive can be applied to structs, +/// enums, and unions; +/// e.g.: +/// +/// ``` +/// # use zerocopy_derive::{FromBytes, FromZeros, Immutable}; +/// #[derive(FromBytes)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # V00, V01, V02, V03, V04, V05, V06, V07, V08, V09, V0A, V0B, V0C, V0D, V0E, +/// # V0F, V10, V11, V12, V13, V14, V15, V16, V17, V18, V19, V1A, V1B, V1C, V1D, +/// # V1E, V1F, V20, V21, V22, V23, V24, V25, V26, V27, V28, V29, V2A, V2B, V2C, +/// # V2D, V2E, V2F, V30, V31, V32, V33, V34, V35, V36, V37, V38, V39, V3A, V3B, +/// # V3C, V3D, V3E, V3F, V40, V41, V42, V43, V44, V45, V46, V47, V48, V49, V4A, +/// # V4B, V4C, V4D, V4E, V4F, V50, V51, V52, V53, V54, V55, V56, V57, V58, V59, +/// # V5A, V5B, V5C, V5D, V5E, V5F, V60, V61, V62, V63, V64, V65, V66, V67, V68, +/// # V69, V6A, V6B, V6C, V6D, V6E, V6F, V70, V71, V72, V73, V74, V75, V76, V77, +/// # V78, V79, V7A, V7B, V7C, V7D, V7E, V7F, V80, V81, V82, V83, V84, V85, V86, +/// # V87, V88, V89, V8A, V8B, V8C, V8D, V8E, V8F, V90, V91, V92, V93, V94, V95, +/// # V96, V97, V98, V99, V9A, V9B, V9C, V9D, V9E, V9F, VA0, VA1, VA2, VA3, VA4, +/// # VA5, VA6, VA7, VA8, VA9, VAA, VAB, VAC, VAD, VAE, VAF, VB0, VB1, VB2, VB3, +/// # VB4, VB5, VB6, VB7, VB8, VB9, VBA, VBB, VBC, VBD, VBE, VBF, VC0, VC1, VC2, +/// # VC3, VC4, VC5, VC6, VC7, VC8, VC9, VCA, VCB, VCC, VCD, VCE, VCF, VD0, VD1, +/// # VD2, VD3, VD4, VD5, VD6, VD7, VD8, VD9, VDA, VDB, VDC, VDD, VDE, VDF, VE0, +/// # VE1, VE2, VE3, VE4, VE5, VE6, VE7, VE8, VE9, VEA, VEB, VEC, VED, VEE, VEF, +/// # VF0, VF1, VF2, VF3, VF4, VF5, VF6, VF7, VF8, VF9, VFA, VFB, VFC, VFD, VFE, +/// # VFF, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromBytes, Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// [safety conditions]: trait@FromBytes#safety +/// +/// # Analysis +/// +/// *This section describes, roughly, the analysis performed by this derive to +/// determine whether it is sound to implement `FromBytes` for a given type. +/// Unless you are modifying the implementation of this derive, or attempting to +/// manually implement `FromBytes` for a type yourself, you don't need to read +/// this section.* +/// +/// If a type has the following properties, then this derive can implement +/// `FromBytes` for that type: +/// +/// - If the type is a struct, all of its fields must be `FromBytes`. +/// - If the type is an enum: +/// - It must have a defined representation which is one of `u8`, `u16`, `i8`, +/// or `i16`. +/// - The maximum number of discriminants must be used (so that every possible +/// bit pattern is a valid one). +/// - Its fields must be `FromBytes`. +/// +/// This analysis is subject to change. Unsafe code may *only* rely on the +/// documented [safety conditions] of `FromBytes`, and must *not* rely on the +/// implementation details of this derive. +/// +/// ## Why isn't an explicit representation required for structs? +/// +/// Neither this derive, nor the [safety conditions] of `FromBytes`, requires +/// that structs are marked with `#[repr(C)]`. +/// +/// Per the [Rust reference](reference), +/// +/// > The representation of a type can change the padding between fields, but +/// > does not change the layout of the fields themselves. +/// +/// [reference]: https://doc.rust-lang.org/reference/type-layout.html#representations +/// +/// Since the layout of structs only consists of padding bytes and field bytes, +/// a struct is soundly `FromBytes` if: +/// 1. its padding is soundly `FromBytes`, and +/// 2. its fields are soundly `FromBytes`. +/// +/// The answer to the first question is always yes: padding bytes do not have +/// any validity constraints. A [discussion] of this question in the Unsafe Code +/// Guidelines Working Group concluded that it would be virtually unimaginable +/// for future versions of rustc to add validity constraints to padding bytes. +/// +/// [discussion]: https://github.com/rust-lang/unsafe-code-guidelines/issues/174 +/// +/// Whether a struct is soundly `FromBytes` therefore solely depends on whether +/// its fields are `FromBytes`. +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::FromBytes; + +/// Types for which any bit pattern is valid. +/// +/// Any memory region of the appropriate length which contains initialized bytes +/// can be viewed as any `FromBytes` type with no runtime overhead. This is +/// useful for efficiently parsing bytes as structured data. +/// +/// # Warning: Padding bytes +/// +/// Note that, when a value is moved or copied, only the non-padding bytes of +/// that value are guaranteed to be preserved. It is unsound to assume that +/// values written to padding bytes are preserved after a move or copy. For +/// example, the following is unsound: +/// +/// ```rust,no_run +/// use core::mem::{size_of, transmute}; +/// use zerocopy::FromZeros; +/// # use zerocopy_derive::*; +/// +/// // Assume `Foo` is a type with padding bytes. +/// #[derive(FromZeros, Default)] +/// struct Foo { +/// # /* +/// ... +/// # */ +/// } +/// +/// let mut foo: Foo = Foo::default(); +/// FromZeros::zero(&mut foo); +/// // UNSOUND: Although `FromZeros::zero` writes zeros to all bytes of `foo`, +/// // those writes are not guaranteed to be preserved in padding bytes when +/// // `foo` is moved, so this may expose padding bytes as `u8`s. +/// let foo_bytes: [u8; size_of::<Foo>()] = unsafe { transmute(foo) }; +/// ``` +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(FromBytes)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{FromBytes, Immutable}; +/// #[derive(FromBytes)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # V00, V01, V02, V03, V04, V05, V06, V07, V08, V09, V0A, V0B, V0C, V0D, V0E, +/// # V0F, V10, V11, V12, V13, V14, V15, V16, V17, V18, V19, V1A, V1B, V1C, V1D, +/// # V1E, V1F, V20, V21, V22, V23, V24, V25, V26, V27, V28, V29, V2A, V2B, V2C, +/// # V2D, V2E, V2F, V30, V31, V32, V33, V34, V35, V36, V37, V38, V39, V3A, V3B, +/// # V3C, V3D, V3E, V3F, V40, V41, V42, V43, V44, V45, V46, V47, V48, V49, V4A, +/// # V4B, V4C, V4D, V4E, V4F, V50, V51, V52, V53, V54, V55, V56, V57, V58, V59, +/// # V5A, V5B, V5C, V5D, V5E, V5F, V60, V61, V62, V63, V64, V65, V66, V67, V68, +/// # V69, V6A, V6B, V6C, V6D, V6E, V6F, V70, V71, V72, V73, V74, V75, V76, V77, +/// # V78, V79, V7A, V7B, V7C, V7D, V7E, V7F, V80, V81, V82, V83, V84, V85, V86, +/// # V87, V88, V89, V8A, V8B, V8C, V8D, V8E, V8F, V90, V91, V92, V93, V94, V95, +/// # V96, V97, V98, V99, V9A, V9B, V9C, V9D, V9E, V9F, VA0, VA1, VA2, VA3, VA4, +/// # VA5, VA6, VA7, VA8, VA9, VAA, VAB, VAC, VAD, VAE, VAF, VB0, VB1, VB2, VB3, +/// # VB4, VB5, VB6, VB7, VB8, VB9, VBA, VBB, VBC, VBD, VBE, VBF, VC0, VC1, VC2, +/// # VC3, VC4, VC5, VC6, VC7, VC8, VC9, VCA, VCB, VCC, VCD, VCE, VCF, VD0, VD1, +/// # VD2, VD3, VD4, VD5, VD6, VD7, VD8, VD9, VDA, VDB, VDC, VDD, VDE, VDF, VE0, +/// # VE1, VE2, VE3, VE4, VE5, VE6, VE7, VE8, VE9, VEA, VEB, VEC, VED, VEE, VEF, +/// # VF0, VF1, VF2, VF3, VF4, VF5, VF6, VF7, VF8, VF9, VFA, VFB, VFC, VFD, VFE, +/// # VFF, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromBytes, Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive performs a sophisticated, compile-time safety analysis to +/// determine whether a type is `FromBytes`. +/// +/// # Safety +/// +/// *This section describes what is required in order for `T: FromBytes`, and +/// what unsafe code may assume of such types. If you don't plan on implementing +/// `FromBytes` manually, and you don't plan on writing unsafe code that +/// operates on `FromBytes` types, then you don't need to read this section.* +/// +/// If `T: FromBytes`, then unsafe code may assume that it is sound to produce a +/// `T` whose bytes are initialized to any sequence of valid `u8`s (in other +/// words, any byte value which is not uninitialized). If a type is marked as +/// `FromBytes` which violates this contract, it may cause undefined behavior. +/// +/// `#[derive(FromBytes)]` only permits [types which satisfy these +/// requirements][derive-analysis]. +/// +#[cfg_attr( + feature = "derive", + doc = "[derive]: zerocopy_derive::FromBytes", + doc = "[derive-analysis]: zerocopy_derive::FromBytes#analysis" +)] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.FromBytes.html"), + doc = concat!("[derive-analysis]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.FromBytes.html#analysis"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(FromBytes)]` to `{Self}`") +)] +pub unsafe trait FromBytes: FromZeros { + // The `Self: Sized` bound makes it so that `FromBytes` is still object + // safe. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// Interprets the given `source` as a `&Self`. + /// + /// This method attempts to return a reference to `source` interpreted as a + /// `Self`. If the length of `source` is not a [valid size of + /// `Self`][valid-size], or if `source` is not appropriately aligned, this + /// returns `Err`. If [`Self: Unaligned`][self-unaligned], you can + /// [infallibly discard the alignment error][size-error-from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = ZSTy::ref_from_bytes(0u16.as_bytes()); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// #[derive(FromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// header: PacketHeader, + /// body: [u8], + /// } + /// + /// // These bytes encode a `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11][..]; + /// + /// let packet = Packet::ref_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.header.src_port, [0, 1]); + /// assert_eq!(packet.header.dst_port, [2, 3]); + /// assert_eq!(packet.header.length, [4, 5]); + /// assert_eq!(packet.header.checksum, [6, 7]); + /// assert_eq!(packet.body, [8, 9, 10, 11]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_from_bytes", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn ref_from_bytes(source: &[u8]) -> Result<&Self, CastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + static_assert_dst_is_not_zst!(Self); + match Ptr::from_ref(source).try_cast_into_no_leftover::<_, BecauseImmutable>(None) { + Ok(ptr) => Ok(ptr.recall_validity().as_ref()), + Err(err) => Err(err.map_src(|src| src.as_ref())), + } + } + + /// Interprets the prefix of the given `source` as a `&Self` without + /// copying. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the leading bytes of `source`, then attempts to return + /// both a reference to those bytes interpreted as a `Self`, and a reference + /// to the remaining bytes. If there are insufficient bytes, or if `source` + /// is not appropriately aligned, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. See [`ref_from_prefix_with_elems`], which does + /// support such types. Attempting to use this method on such types results + /// in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = ZSTy::ref_from_prefix(0u16.as_bytes()); // ⚠ Compile Error! + /// ``` + /// + /// [`ref_from_prefix_with_elems`]: FromBytes::ref_from_prefix_with_elems + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// #[derive(FromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// header: PacketHeader, + /// body: [[u8; 2]], + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14][..]; + /// + /// let (packet, suffix) = Packet::ref_from_prefix(bytes).unwrap(); + /// + /// assert_eq!(packet.header.src_port, [0, 1]); + /// assert_eq!(packet.header.dst_port, [2, 3]); + /// assert_eq!(packet.header.length, [4, 5]); + /// assert_eq!(packet.header.checksum, [6, 7]); + /// assert_eq!(packet.body, [[8, 9], [10, 11], [12, 13]]); + /// assert_eq!(suffix, &[14u8][..]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_from_prefix", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn ref_from_prefix(source: &[u8]) -> Result<(&Self, &[u8]), CastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + static_assert_dst_is_not_zst!(Self); + ref_from_prefix_suffix(source, None, CastType::Prefix) + } + + /// Interprets the suffix of the given bytes as a `&Self`. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the trailing bytes of `source`, then attempts to return + /// both a reference to those bytes interpreted as a `Self`, and a reference + /// to the preceding bytes. If there are insufficient bytes, or if that + /// suffix of `source` is not appropriately aligned, this returns `Err`. If + /// [`Self: Unaligned`][self-unaligned], you can [infallibly discard the + /// alignment error][size-error-from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. See [`ref_from_suffix_with_elems`], which does + /// support such types. Attempting to use this method on such types results + /// in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = ZSTy::ref_from_suffix(0u16.as_bytes()); // ⚠ Compile Error! + /// ``` + /// + /// [`ref_from_suffix_with_elems`]: FromBytes::ref_from_suffix_with_elems + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct PacketTrailer { + /// frame_check_sequence: [u8; 4], + /// } + /// + /// // These are more bytes than are needed to encode a `PacketTrailer`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (prefix, trailer) = PacketTrailer::ref_from_suffix(bytes).unwrap(); + /// + /// assert_eq!(prefix, &[0, 1, 2, 3, 4, 5][..]); + /// assert_eq!(trailer.frame_check_sequence, [6, 7, 8, 9]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_from_suffix", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn ref_from_suffix(source: &[u8]) -> Result<(&[u8], &Self), CastError<&[u8], Self>> + where + Self: Immutable + KnownLayout, + { + static_assert_dst_is_not_zst!(Self); + ref_from_prefix_suffix(source, None, CastType::Suffix).map(swap) + } + + /// Interprets the given `source` as a `&mut Self`. + /// + /// This method attempts to return a reference to `source` interpreted as a + /// `Self`. If the length of `source` is not a [valid size of + /// `Self`][valid-size], or if `source` is not appropriately aligned, this + /// returns `Err`. If [`Self: Unaligned`][self-unaligned], you can + /// [infallibly discard the alignment error][size-error-from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. See [`mut_from_prefix_with_elems`], which does + /// support such types. Attempting to use this method on such types results + /// in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let mut source = [85, 85]; + /// let _ = ZSTy::mut_from_bytes(&mut source[..]); // ⚠ Compile Error! + /// ``` + /// + /// [`mut_from_prefix_with_elems`]: FromBytes::mut_from_prefix_with_elems + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// // These bytes encode a `PacketHeader`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7][..]; + /// + /// let header = PacketHeader::mut_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(header.src_port, [0, 1]); + /// assert_eq!(header.dst_port, [2, 3]); + /// assert_eq!(header.length, [4, 5]); + /// assert_eq!(header.checksum, [6, 7]); + /// + /// header.checksum = [0, 0]; + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 4, 5, 0, 0]); + /// + /// ``` + /// + #[doc = codegen_header!("h5", "mut_from_bytes")] + /// + /// See [`FromBytes::ref_from_bytes`](#method.ref_from_bytes.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn mut_from_bytes(source: &mut [u8]) -> Result<&mut Self, CastError<&mut [u8], Self>> + where + Self: IntoBytes + KnownLayout, + { + static_assert_dst_is_not_zst!(Self); + match Ptr::from_mut(source).try_cast_into_no_leftover::<_, BecauseExclusive>(None) { + Ok(ptr) => Ok(ptr.recall_validity::<_, (_, (_, _))>().as_mut()), + Err(err) => Err(err.map_src(|src| src.as_mut())), + } + } + + /// Interprets the prefix of the given `source` as a `&mut Self` without + /// copying. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the leading bytes of `source`, then attempts to return + /// both a reference to those bytes interpreted as a `Self`, and a reference + /// to the remaining bytes. If there are insufficient bytes, or if `source` + /// is not appropriately aligned, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. See [`mut_from_suffix_with_elems`], which does + /// support such types. Attempting to use this method on such types results + /// in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let mut source = [85, 85]; + /// let _ = ZSTy::mut_from_prefix(&mut source[..]); // ⚠ Compile Error! + /// ``` + /// + /// [`mut_from_suffix_with_elems`]: FromBytes::mut_from_suffix_with_elems + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// // These are more bytes than are needed to encode a `PacketHeader`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (header, body) = PacketHeader::mut_from_prefix(bytes).unwrap(); + /// + /// assert_eq!(header.src_port, [0, 1]); + /// assert_eq!(header.dst_port, [2, 3]); + /// assert_eq!(header.length, [4, 5]); + /// assert_eq!(header.checksum, [6, 7]); + /// assert_eq!(body, &[8, 9][..]); + /// + /// header.checksum = [0, 0]; + /// body.fill(1); + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 4, 5, 0, 0, 1, 1]); + /// ``` + /// + #[doc = codegen_header!("h5", "mut_from_prefix")] + /// + /// See [`FromBytes::ref_from_prefix`](#method.ref_from_prefix.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn mut_from_prefix( + source: &mut [u8], + ) -> Result<(&mut Self, &mut [u8]), CastError<&mut [u8], Self>> + where + Self: IntoBytes + KnownLayout, + { + static_assert_dst_is_not_zst!(Self); + mut_from_prefix_suffix(source, None, CastType::Prefix) + } + + /// Interprets the suffix of the given `source` as a `&mut Self` without + /// copying. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the trailing bytes of `source`, then attempts to return + /// both a reference to those bytes interpreted as a `Self`, and a reference + /// to the preceding bytes. If there are insufficient bytes, or if that + /// suffix of `source` is not appropriately aligned, this returns `Err`. If + /// [`Self: Unaligned`][self-unaligned], you can [infallibly discard the + /// alignment error][size-error-from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let mut source = [85, 85]; + /// let _ = ZSTy::mut_from_suffix(&mut source[..]); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct PacketTrailer { + /// frame_check_sequence: [u8; 4], + /// } + /// + /// // These are more bytes than are needed to encode a `PacketTrailer`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (prefix, trailer) = PacketTrailer::mut_from_suffix(bytes).unwrap(); + /// + /// assert_eq!(prefix, &[0u8, 1, 2, 3, 4, 5][..]); + /// assert_eq!(trailer.frame_check_sequence, [6, 7, 8, 9]); + /// + /// prefix.fill(0); + /// trailer.frame_check_sequence.fill(1); + /// + /// assert_eq!(bytes, [0, 0, 0, 0, 0, 0, 1, 1, 1, 1]); + /// ``` + /// + #[doc = codegen_header!("h5", "mut_from_suffix")] + /// + /// See [`FromBytes::ref_from_suffix`](#method.ref_from_suffix.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn mut_from_suffix( + source: &mut [u8], + ) -> Result<(&mut [u8], &mut Self), CastError<&mut [u8], Self>> + where + Self: IntoBytes + KnownLayout, + { + static_assert_dst_is_not_zst!(Self); + mut_from_prefix_suffix(source, None, CastType::Suffix).map(swap) + } + + /// Interprets the given `source` as a `&Self` with a DST length equal to + /// `count`. + /// + /// This method attempts to return a reference to `source` interpreted as a + /// `Self` with `count` trailing elements. If the length of `source` is not + /// equal to the size of `Self` with `count` elements, or if `source` is not + /// appropriately aligned, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Debug, PartialEq, Eq)] + /// #[derive(FromBytes, Immutable)] + /// #[repr(C)] + /// struct Pixel { + /// r: u8, + /// g: u8, + /// b: u8, + /// a: u8, + /// } + /// + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7][..]; + /// + /// let pixels = <[Pixel]>::ref_from_bytes_with_elems(bytes, 2).unwrap(); + /// + /// assert_eq!(pixels, &[ + /// Pixel { r: 0, g: 1, b: 2, a: 3 }, + /// Pixel { r: 4, g: 5, b: 6, a: 7 }, + /// ]); + /// + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`ref_from_bytes`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let src = &[85, 85][..]; + /// let zsty = ZSTy::ref_from_bytes_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`ref_from_bytes`]: FromBytes::ref_from_bytes + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_from_bytes_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn ref_from_bytes_with_elems( + source: &[u8], + count: usize, + ) -> Result<&Self, CastError<&[u8], Self>> + where + Self: KnownLayout<PointerMetadata = usize> + Immutable, + { + let source = Ptr::from_ref(source); + let maybe_slf = source.try_cast_into_no_leftover::<_, BecauseImmutable>(Some(count)); + match maybe_slf { + Ok(slf) => Ok(slf.recall_validity().as_ref()), + Err(err) => Err(err.map_src(|s| s.as_ref())), + } + } + + /// Interprets the prefix of the given `source` as a DST `&Self` with length + /// equal to `count`. + /// + /// This method attempts to return a reference to the prefix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the remaining bytes. If there are insufficient bytes, or if `source` + /// is not appropriately aligned, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Debug, PartialEq, Eq)] + /// #[derive(FromBytes, Immutable)] + /// #[repr(C)] + /// struct Pixel { + /// r: u8, + /// g: u8, + /// b: u8, + /// a: u8, + /// } + /// + /// // These are more bytes than are needed to encode two `Pixel`s. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (pixels, suffix) = <[Pixel]>::ref_from_prefix_with_elems(bytes, 2).unwrap(); + /// + /// assert_eq!(pixels, &[ + /// Pixel { r: 0, g: 1, b: 2, a: 3 }, + /// Pixel { r: 4, g: 5, b: 6, a: 7 }, + /// ]); + /// + /// assert_eq!(suffix, &[8, 9]); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`ref_from_prefix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let src = &[85, 85][..]; + /// let (zsty, _) = ZSTy::ref_from_prefix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`ref_from_prefix`]: FromBytes::ref_from_prefix + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_from_prefix_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn ref_from_prefix_with_elems( + source: &[u8], + count: usize, + ) -> Result<(&Self, &[u8]), CastError<&[u8], Self>> + where + Self: KnownLayout<PointerMetadata = usize> + Immutable, + { + ref_from_prefix_suffix(source, Some(count), CastType::Prefix) + } + + /// Interprets the suffix of the given `source` as a DST `&Self` with length + /// equal to `count`. + /// + /// This method attempts to return a reference to the suffix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the preceding bytes. If there are insufficient bytes, or if that + /// suffix of `source` is not appropriately aligned, this returns `Err`. If + /// [`Self: Unaligned`][self-unaligned], you can [infallibly discard the + /// alignment error][size-error-from]. + /// + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Debug, PartialEq, Eq)] + /// #[derive(FromBytes, Immutable)] + /// #[repr(C)] + /// struct Pixel { + /// r: u8, + /// g: u8, + /// b: u8, + /// a: u8, + /// } + /// + /// // These are more bytes than are needed to encode two `Pixel`s. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (prefix, pixels) = <[Pixel]>::ref_from_suffix_with_elems(bytes, 2).unwrap(); + /// + /// assert_eq!(prefix, &[0, 1]); + /// + /// assert_eq!(pixels, &[ + /// Pixel { r: 2, g: 3, b: 4, a: 5 }, + /// Pixel { r: 6, g: 7, b: 8, a: 9 }, + /// ]); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`ref_from_suffix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let src = &[85, 85][..]; + /// let (_, zsty) = ZSTy::ref_from_suffix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`ref_from_suffix`]: FromBytes::ref_from_suffix + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_from_suffix_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn ref_from_suffix_with_elems( + source: &[u8], + count: usize, + ) -> Result<(&[u8], &Self), CastError<&[u8], Self>> + where + Self: KnownLayout<PointerMetadata = usize> + Immutable, + { + ref_from_prefix_suffix(source, Some(count), CastType::Suffix).map(swap) + } + + /// Interprets the given `source` as a `&mut Self` with a DST length equal + /// to `count`. + /// + /// This method attempts to return a reference to `source` interpreted as a + /// `Self` with `count` trailing elements. If the length of `source` is not + /// equal to the size of `Self` with `count` elements, or if `source` is not + /// appropriately aligned, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Debug, PartialEq, Eq)] + /// #[derive(KnownLayout, FromBytes, IntoBytes, Immutable)] + /// #[repr(C)] + /// struct Pixel { + /// r: u8, + /// g: u8, + /// b: u8, + /// a: u8, + /// } + /// + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7][..]; + /// + /// let pixels = <[Pixel]>::mut_from_bytes_with_elems(bytes, 2).unwrap(); + /// + /// assert_eq!(pixels, &[ + /// Pixel { r: 0, g: 1, b: 2, a: 3 }, + /// Pixel { r: 4, g: 5, b: 6, a: 7 }, + /// ]); + /// + /// pixels[1] = Pixel { r: 0, g: 0, b: 0, a: 0 }; + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 0, 0, 0, 0]); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`mut_from_bytes`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, Immutable, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let src = &mut [85, 85][..]; + /// let zsty = ZSTy::mut_from_bytes_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`mut_from_bytes`]: FromBytes::mut_from_bytes + /// + #[doc = codegen_header!("h5", "mut_from_bytes_with_elems")] + /// + /// See [`TryFromBytes::ref_from_bytes_with_elems`](#method.ref_from_bytes_with_elems.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn mut_from_bytes_with_elems( + source: &mut [u8], + count: usize, + ) -> Result<&mut Self, CastError<&mut [u8], Self>> + where + Self: IntoBytes + KnownLayout<PointerMetadata = usize> + Immutable, + { + let source = Ptr::from_mut(source); + let maybe_slf = source.try_cast_into_no_leftover::<_, BecauseImmutable>(Some(count)); + match maybe_slf { + Ok(slf) => Ok(slf.recall_validity::<_, (_, (_, BecauseExclusive))>().as_mut()), + Err(err) => Err(err.map_src(|s| s.as_mut())), + } + } + + /// Interprets the prefix of the given `source` as a `&mut Self` with DST + /// length equal to `count`. + /// + /// This method attempts to return a reference to the prefix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the preceding bytes. If there are insufficient bytes, or if `source` + /// is not appropriately aligned, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Debug, PartialEq, Eq)] + /// #[derive(KnownLayout, FromBytes, IntoBytes, Immutable)] + /// #[repr(C)] + /// struct Pixel { + /// r: u8, + /// g: u8, + /// b: u8, + /// a: u8, + /// } + /// + /// // These are more bytes than are needed to encode two `Pixel`s. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (pixels, suffix) = <[Pixel]>::mut_from_prefix_with_elems(bytes, 2).unwrap(); + /// + /// assert_eq!(pixels, &[ + /// Pixel { r: 0, g: 1, b: 2, a: 3 }, + /// Pixel { r: 4, g: 5, b: 6, a: 7 }, + /// ]); + /// + /// assert_eq!(suffix, &[8, 9]); + /// + /// pixels[1] = Pixel { r: 0, g: 0, b: 0, a: 0 }; + /// suffix.fill(1); + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 0, 0, 0, 0, 1, 1]); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`mut_from_prefix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, Immutable, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let src = &mut [85, 85][..]; + /// let (zsty, _) = ZSTy::mut_from_prefix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`mut_from_prefix`]: FromBytes::mut_from_prefix + /// + #[doc = codegen_header!("h5", "mut_from_prefix_with_elems")] + /// + /// See [`TryFromBytes::ref_from_prefix_with_elems`](#method.ref_from_prefix_with_elems.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn mut_from_prefix_with_elems( + source: &mut [u8], + count: usize, + ) -> Result<(&mut Self, &mut [u8]), CastError<&mut [u8], Self>> + where + Self: IntoBytes + KnownLayout<PointerMetadata = usize>, + { + mut_from_prefix_suffix(source, Some(count), CastType::Prefix) + } + + /// Interprets the suffix of the given `source` as a `&mut Self` with DST + /// length equal to `count`. + /// + /// This method attempts to return a reference to the suffix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the remaining bytes. If there are insufficient bytes, or if that + /// suffix of `source` is not appropriately aligned, this returns `Err`. If + /// [`Self: Unaligned`][self-unaligned], you can [infallibly discard the + /// alignment error][size-error-from]. + /// + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Debug, PartialEq, Eq)] + /// #[derive(FromBytes, IntoBytes, Immutable)] + /// #[repr(C)] + /// struct Pixel { + /// r: u8, + /// g: u8, + /// b: u8, + /// a: u8, + /// } + /// + /// // These are more bytes than are needed to encode two `Pixel`s. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (prefix, pixels) = <[Pixel]>::mut_from_suffix_with_elems(bytes, 2).unwrap(); + /// + /// assert_eq!(prefix, &[0, 1]); + /// + /// assert_eq!(pixels, &[ + /// Pixel { r: 2, g: 3, b: 4, a: 5 }, + /// Pixel { r: 6, g: 7, b: 8, a: 9 }, + /// ]); + /// + /// prefix.fill(9); + /// pixels[1] = Pixel { r: 0, g: 0, b: 0, a: 0 }; + /// + /// assert_eq!(bytes, [9, 9, 2, 3, 4, 5, 0, 0, 0, 0]); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`mut_from_suffix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, Immutable, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let src = &mut [85, 85][..]; + /// let (_, zsty) = ZSTy::mut_from_suffix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`mut_from_suffix`]: FromBytes::mut_from_suffix + /// + #[doc = codegen_header!("h5", "mut_from_suffix_with_elems")] + /// + /// See [`TryFromBytes::ref_from_suffix_with_elems`](#method.ref_from_suffix_with_elems.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn mut_from_suffix_with_elems( + source: &mut [u8], + count: usize, + ) -> Result<(&mut [u8], &mut Self), CastError<&mut [u8], Self>> + where + Self: IntoBytes + KnownLayout<PointerMetadata = usize>, + { + mut_from_prefix_suffix(source, Some(count), CastType::Suffix).map(swap) + } + + /// Reads a copy of `Self` from the given `source`. + /// + /// If `source.len() != size_of::<Self>()`, `read_from_bytes` returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// // These bytes encode a `PacketHeader`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7][..]; + /// + /// let header = PacketHeader::read_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(header.src_port, [0, 1]); + /// assert_eq!(header.dst_port, [2, 3]); + /// assert_eq!(header.length, [4, 5]); + /// assert_eq!(header.checksum, [6, 7]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "read_from_bytes", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn read_from_bytes(source: &[u8]) -> Result<Self, SizeError<&[u8], Self>> + where + Self: Sized, + { + match Ref::<_, Unalign<Self>>::sized_from(source) { + Ok(r) => Ok(Ref::read(&r).into_inner()), + Err(CastError::Size(e)) => Err(e.with_dst()), + Err(CastError::Alignment(_)) => { + // SAFETY: `Unalign<Self>` is trivially aligned, so + // `Ref::sized_from` cannot fail due to unmet alignment + // requirements. + unsafe { core::hint::unreachable_unchecked() } + } + Err(CastError::Validity(i)) => match i {}, + } + } + + /// Reads a copy of `Self` from the prefix of the given `source`. + /// + /// This attempts to read a `Self` from the first `size_of::<Self>()` bytes + /// of `source`, returning that `Self` and any remaining bytes. If + /// `source.len() < size_of::<Self>()`, it returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// // These are more bytes than are needed to encode a `PacketHeader`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (header, body) = PacketHeader::read_from_prefix(bytes).unwrap(); + /// + /// assert_eq!(header.src_port, [0, 1]); + /// assert_eq!(header.dst_port, [2, 3]); + /// assert_eq!(header.length, [4, 5]); + /// assert_eq!(header.checksum, [6, 7]); + /// assert_eq!(body, [8, 9]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "read_from_prefix", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn read_from_prefix(source: &[u8]) -> Result<(Self, &[u8]), SizeError<&[u8], Self>> + where + Self: Sized, + { + match Ref::<_, Unalign<Self>>::sized_from_prefix(source) { + Ok((r, suffix)) => Ok((Ref::read(&r).into_inner(), suffix)), + Err(CastError::Size(e)) => Err(e.with_dst()), + Err(CastError::Alignment(_)) => { + // SAFETY: `Unalign<Self>` is trivially aligned, so + // `Ref::sized_from_prefix` cannot fail due to unmet alignment + // requirements. + unsafe { core::hint::unreachable_unchecked() } + } + Err(CastError::Validity(i)) => match i {}, + } + } + + /// Reads a copy of `Self` from the suffix of the given `source`. + /// + /// This attempts to read a `Self` from the last `size_of::<Self>()` bytes + /// of `source`, returning that `Self` and any preceding bytes. If + /// `source.len() < size_of::<Self>()`, it returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes)] + /// #[repr(C)] + /// struct PacketTrailer { + /// frame_check_sequence: [u8; 4], + /// } + /// + /// // These are more bytes than are needed to encode a `PacketTrailer`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (prefix, trailer) = PacketTrailer::read_from_suffix(bytes).unwrap(); + /// + /// assert_eq!(prefix, [0, 1, 2, 3, 4, 5]); + /// assert_eq!(trailer.frame_check_sequence, [6, 7, 8, 9]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "read_from_suffix", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn read_from_suffix(source: &[u8]) -> Result<(&[u8], Self), SizeError<&[u8], Self>> + where + Self: Sized, + { + match Ref::<_, Unalign<Self>>::sized_from_suffix(source) { + Ok((prefix, r)) => Ok((prefix, Ref::read(&r).into_inner())), + Err(CastError::Size(e)) => Err(e.with_dst()), + Err(CastError::Alignment(_)) => { + // SAFETY: `Unalign<Self>` is trivially aligned, so + // `Ref::sized_from_suffix` cannot fail due to unmet alignment + // requirements. + unsafe { core::hint::unreachable_unchecked() } + } + Err(CastError::Validity(i)) => match i {}, + } + } + + /// Reads a copy of `self` from an `io::Read`. + /// + /// This is useful for interfacing with operating system byte sinks (files, + /// sockets, etc.). + /// + /// # Examples + /// + /// ```no_run + /// use zerocopy::{byteorder::big_endian::*, FromBytes}; + /// use std::fs::File; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes)] + /// #[repr(C)] + /// struct BitmapFileHeader { + /// signature: [u8; 2], + /// size: U32, + /// reserved: U64, + /// offset: U64, + /// } + /// + /// let mut file = File::open("image.bin").unwrap(); + /// let header = BitmapFileHeader::read_from_io(&mut file).unwrap(); + /// ``` + #[cfg(feature = "std")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "std")))] + #[inline(always)] + fn read_from_io<R>(mut src: R) -> io::Result<Self> + where + Self: Sized, + R: io::Read, + { + // NOTE(#2319, #2320): We do `buf.zero()` separately rather than + // constructing `let buf = CoreMaybeUninit::zeroed()` because, if `Self` + // contains padding bytes, then a typed copy of `CoreMaybeUninit<Self>` + // will not necessarily preserve zeros written to those padding byte + // locations, and so `buf` could contain uninitialized bytes. + let mut buf = CoreMaybeUninit::<Self>::uninit(); + buf.zero(); + + let ptr = Ptr::from_mut(&mut buf); + // SAFETY: After `buf.zero()`, `buf` consists entirely of initialized, + // zeroed bytes. Since `MaybeUninit` has no validity requirements, `ptr` + // cannot be used to write values which will violate `buf`'s bit + // validity. Since `ptr` has `Exclusive` aliasing, nothing other than + // `ptr` may be used to mutate `ptr`'s referent, and so its bit validity + // cannot be violated even though `buf` may have more permissive bit + // validity than `ptr`. + let ptr = unsafe { ptr.assume_validity::<invariant::Initialized>() }; + let ptr = ptr.as_bytes(); + src.read_exact(ptr.as_mut())?; + // SAFETY: `buf` entirely consists of initialized bytes, and `Self` is + // `FromBytes`. + Ok(unsafe { buf.assume_init() }) + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::ref_from_bytes`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn ref_from(source: &[u8]) -> Option<&Self> + where + Self: KnownLayout + Immutable, + { + Self::ref_from_bytes(source).ok() + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::mut_from_bytes`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn mut_from(source: &mut [u8]) -> Option<&mut Self> + where + Self: KnownLayout + IntoBytes, + { + Self::mut_from_bytes(source).ok() + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::ref_from_prefix_with_elems`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn slice_from_prefix(source: &[u8], count: usize) -> Option<(&[Self], &[u8])> + where + Self: Sized + Immutable, + { + <[Self]>::ref_from_prefix_with_elems(source, count).ok() + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::ref_from_suffix_with_elems`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn slice_from_suffix(source: &[u8], count: usize) -> Option<(&[u8], &[Self])> + where + Self: Sized + Immutable, + { + <[Self]>::ref_from_suffix_with_elems(source, count).ok() + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::mut_from_prefix_with_elems`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn mut_slice_from_prefix(source: &mut [u8], count: usize) -> Option<(&mut [Self], &mut [u8])> + where + Self: Sized + IntoBytes, + { + <[Self]>::mut_from_prefix_with_elems(source, count).ok() + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::mut_from_suffix_with_elems`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn mut_slice_from_suffix(source: &mut [u8], count: usize) -> Option<(&mut [u8], &mut [Self])> + where + Self: Sized + IntoBytes, + { + <[Self]>::mut_from_suffix_with_elems(source, count).ok() + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::read_from_bytes`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn read_from(source: &[u8]) -> Option<Self> + where + Self: Sized, + { + Self::read_from_bytes(source).ok() + } +} + +/// Interprets the given affix of the given bytes as a `&Self`. +/// +/// This method computes the largest possible size of `Self` that can fit in the +/// prefix or suffix bytes of `source`, then attempts to return both a reference +/// to those bytes interpreted as a `Self`, and a reference to the excess bytes. +/// If there are insufficient bytes, or if that affix of `source` is not +/// appropriately aligned, this returns `Err`. +#[inline(always)] +fn ref_from_prefix_suffix<T: FromBytes + KnownLayout + Immutable + ?Sized>( + source: &[u8], + meta: Option<T::PointerMetadata>, + cast_type: CastType, +) -> Result<(&T, &[u8]), CastError<&[u8], T>> { + let (slf, prefix_suffix) = Ptr::from_ref(source) + .try_cast_into::<_, BecauseImmutable>(cast_type, meta) + .map_err(|err| err.map_src(|s| s.as_ref()))?; + Ok((slf.recall_validity().as_ref(), prefix_suffix.as_ref())) +} + +/// Interprets the given affix of the given bytes as a `&mut Self` without +/// copying. +/// +/// This method computes the largest possible size of `Self` that can fit in the +/// prefix or suffix bytes of `source`, then attempts to return both a reference +/// to those bytes interpreted as a `Self`, and a reference to the excess bytes. +/// If there are insufficient bytes, or if that affix of `source` is not +/// appropriately aligned, this returns `Err`. +#[inline(always)] +fn mut_from_prefix_suffix<T: FromBytes + IntoBytes + KnownLayout + ?Sized>( + source: &mut [u8], + meta: Option<T::PointerMetadata>, + cast_type: CastType, +) -> Result<(&mut T, &mut [u8]), CastError<&mut [u8], T>> { + let (slf, prefix_suffix) = Ptr::from_mut(source) + .try_cast_into::<_, BecauseExclusive>(cast_type, meta) + .map_err(|err| err.map_src(|s| s.as_mut()))?; + Ok((slf.recall_validity::<_, (_, (_, _))>().as_mut(), prefix_suffix.as_mut())) +} + +/// Analyzes whether a type is [`IntoBytes`]. +/// +/// This derive analyzes, at compile time, whether the annotated type satisfies +/// the [safety conditions] of `IntoBytes` and implements `IntoBytes` if it is +/// sound to do so. This derive can be applied to structs and enums (see below +/// for union support); e.g.: +/// +/// ``` +/// # use zerocopy_derive::{IntoBytes}; +/// #[derive(IntoBytes)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(IntoBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// [safety conditions]: trait@IntoBytes#safety +/// +/// # Error Messages +/// +/// On Rust toolchains prior to 1.78.0, due to the way that the custom derive +/// for `IntoBytes` is implemented, you may get an error like this: +/// +/// ```text +/// error[E0277]: the trait bound `(): PaddingFree<Foo, true>` is not satisfied +/// --> lib.rs:23:10 +/// | +/// 1 | #[derive(IntoBytes)] +/// | ^^^^^^^^^ the trait `PaddingFree<Foo, true>` is not implemented for `()` +/// | +/// = help: the following implementations were found: +/// <() as PaddingFree<T, false>> +/// ``` +/// +/// This error indicates that the type being annotated has padding bytes, which +/// is illegal for `IntoBytes` types. Consider reducing the alignment of some +/// fields by using types in the [`byteorder`] module, wrapping field types in +/// [`Unalign`], adding explicit struct fields where those padding bytes would +/// be, or using `#[repr(packed)]`. See the Rust Reference's page on [type +/// layout] for more information about type layout and padding. +/// +/// [type layout]: https://doc.rust-lang.org/reference/type-layout.html +/// +/// # Unions +/// +/// Currently, union bit validity is [up in the air][union-validity], and so +/// zerocopy does not support `#[derive(IntoBytes)]` on unions by default. +/// However, implementing `IntoBytes` on a union type is likely sound on all +/// existing Rust toolchains - it's just that it may become unsound in the +/// future. You can opt-in to `#[derive(IntoBytes)]` support on unions by +/// passing the unstable `zerocopy_derive_union_into_bytes` cfg: +/// +/// ```shell +/// $ RUSTFLAGS='--cfg zerocopy_derive_union_into_bytes' cargo build +/// ``` +/// +/// However, it is your responsibility to ensure that this derive is sound on +/// the specific versions of the Rust toolchain you are using! We make no +/// stability or soundness guarantees regarding this cfg, and may remove it at +/// any point. +/// +/// We are actively working with Rust to stabilize the necessary language +/// guarantees to support this in a forwards-compatible way, which will enable +/// us to remove the cfg gate. As part of this effort, we need to know how much +/// demand there is for this feature. If you would like to use `IntoBytes` on +/// unions, [please let us know][discussion]. +/// +/// [union-validity]: https://github.com/rust-lang/unsafe-code-guidelines/issues/438 +/// [discussion]: https://github.com/google/zerocopy/discussions/1802 +/// +/// # Analysis +/// +/// *This section describes, roughly, the analysis performed by this derive to +/// determine whether it is sound to implement `IntoBytes` for a given type. +/// Unless you are modifying the implementation of this derive, or attempting to +/// manually implement `IntoBytes` for a type yourself, you don't need to read +/// this section.* +/// +/// If a type has the following properties, then this derive can implement +/// `IntoBytes` for that type: +/// +/// - If the type is a struct, its fields must be [`IntoBytes`]. Additionally: +/// - if the type is `repr(transparent)` or `repr(packed)`, it is +/// [`IntoBytes`] if its fields are [`IntoBytes`]; else, +/// - if the type is `repr(C)` with at most one field, it is [`IntoBytes`] +/// if its field is [`IntoBytes`]; else, +/// - if the type has no generic parameters, it is [`IntoBytes`] if the type +/// is sized and has no padding bytes; else, +/// - if the type is `repr(C)`, its fields must be [`Unaligned`]. +/// - If the type is an enum: +/// - It must have a defined representation (`repr`s `C`, `u8`, `u16`, `u32`, +/// `u64`, `usize`, `i8`, `i16`, `i32`, `i64`, or `isize`). +/// - It must have no padding bytes. +/// - Its fields must be [`IntoBytes`]. +/// +/// This analysis is subject to change. Unsafe code may *only* rely on the +/// documented [safety conditions] of `FromBytes`, and must *not* rely on the +/// implementation details of this derive. +/// +/// [Rust Reference]: https://doc.rust-lang.org/reference/type-layout.html +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::IntoBytes; + +/// Types that can be converted to an immutable slice of initialized bytes. +/// +/// Any `IntoBytes` type can be converted to a slice of initialized bytes of the +/// same size. This is useful for efficiently serializing structured data as raw +/// bytes. +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(IntoBytes)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::IntoBytes; +/// #[derive(IntoBytes)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(IntoBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant0, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive performs a sophisticated, compile-time safety analysis to +/// determine whether a type is `IntoBytes`. See the [derive +/// documentation][derive] for guidance on how to interpret error messages +/// produced by the derive's analysis. +/// +/// # Safety +/// +/// *This section describes what is required in order for `T: IntoBytes`, and +/// what unsafe code may assume of such types. If you don't plan on implementing +/// `IntoBytes` manually, and you don't plan on writing unsafe code that +/// operates on `IntoBytes` types, then you don't need to read this section.* +/// +/// If `T: IntoBytes`, then unsafe code may assume that it is sound to treat any +/// `t: T` as an immutable `[u8]` of length `size_of_val(t)`. If a type is +/// marked as `IntoBytes` which violates this contract, it may cause undefined +/// behavior. +/// +/// `#[derive(IntoBytes)]` only permits [types which satisfy these +/// requirements][derive-analysis]. +/// +#[cfg_attr( + feature = "derive", + doc = "[derive]: zerocopy_derive::IntoBytes", + doc = "[derive-analysis]: zerocopy_derive::IntoBytes#analysis" +)] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.IntoBytes.html"), + doc = concat!("[derive-analysis]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.IntoBytes.html#analysis"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(IntoBytes)]` to `{Self}`") +)] +pub unsafe trait IntoBytes { + // The `Self: Sized` bound makes it so that this function doesn't prevent + // `IntoBytes` from being object safe. Note that other `IntoBytes` methods + // prevent object safety, but those provide a benefit in exchange for object + // safety. If at some point we remove those methods, change their type + // signatures, or move them out of this trait so that `IntoBytes` is object + // safe again, it's important that this function not prevent object safety. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// Gets the bytes of this value. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::IntoBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(IntoBytes, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let header = PacketHeader { + /// src_port: [0, 1], + /// dst_port: [2, 3], + /// length: [4, 5], + /// checksum: [6, 7], + /// }; + /// + /// let bytes = header.as_bytes(); + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 4, 5, 6, 7]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "as_bytes", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] + )] + #[must_use = "has no side effects"] + #[inline(always)] + fn as_bytes(&self) -> &[u8] + where + Self: Immutable, + { + // Note that this method does not have a `Self: Sized` bound; + // `size_of_val` works for unsized values too. + let len = mem::size_of_val(self); + let slf: *const Self = self; + + // SAFETY: + // - `slf.cast::<u8>()` is valid for reads for `len * size_of::<u8>()` + // many bytes because... + // - `slf` is the same pointer as `self`, and `self` is a reference + // which points to an object whose size is `len`. Thus... + // - The entire region of `len` bytes starting at `slf` is contained + // within a single allocation. + // - `slf` is non-null. + // - `slf` is trivially aligned to `align_of::<u8>() == 1`. + // - `Self: IntoBytes` ensures that all of the bytes of `slf` are + // initialized. + // - Since `slf` is derived from `self`, and `self` is an immutable + // reference, the only other references to this memory region that + // could exist are other immutable references, which by `Self: + // Immutable` don't permit mutation. + // - The total size of the resulting slice is no larger than + // `isize::MAX` because no allocation produced by safe code can be + // larger than `isize::MAX`. + // + // FIXME(#429): Add references to docs and quotes. + unsafe { slice::from_raw_parts(slf.cast::<u8>(), len) } + } + + /// Gets the bytes of this value mutably. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::IntoBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Eq, PartialEq, Debug)] + /// #[derive(FromBytes, IntoBytes, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let mut header = PacketHeader { + /// src_port: [0, 1], + /// dst_port: [2, 3], + /// length: [4, 5], + /// checksum: [6, 7], + /// }; + /// + /// let bytes = header.as_mut_bytes(); + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 4, 5, 6, 7]); + /// + /// bytes.reverse(); + /// + /// assert_eq!(header, PacketHeader { + /// src_port: [7, 6], + /// dst_port: [5, 4], + /// length: [3, 2], + /// checksum: [1, 0], + /// }); + /// ``` + /// + #[doc = codegen_header!("h5", "as_mut_bytes")] + /// + /// See [`IntoBytes::as_bytes`](#method.as_bytes.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + fn as_mut_bytes(&mut self) -> &mut [u8] + where + Self: FromBytes, + { + // Note that this method does not have a `Self: Sized` bound; + // `size_of_val` works for unsized values too. + let len = mem::size_of_val(self); + let slf: *mut Self = self; + + // SAFETY: + // - `slf.cast::<u8>()` is valid for reads and writes for `len * + // size_of::<u8>()` many bytes because... + // - `slf` is the same pointer as `self`, and `self` is a reference + // which points to an object whose size is `len`. Thus... + // - The entire region of `len` bytes starting at `slf` is contained + // within a single allocation. + // - `slf` is non-null. + // - `slf` is trivially aligned to `align_of::<u8>() == 1`. + // - `Self: IntoBytes` ensures that all of the bytes of `slf` are + // initialized. + // - `Self: FromBytes` ensures that no write to this memory region + // could result in it containing an invalid `Self`. + // - Since `slf` is derived from `self`, and `self` is a mutable + // reference, no other references to this memory region can exist. + // - The total size of the resulting slice is no larger than + // `isize::MAX` because no allocation produced by safe code can be + // larger than `isize::MAX`. + // + // FIXME(#429): Add references to docs and quotes. + unsafe { slice::from_raw_parts_mut(slf.cast::<u8>(), len) } + } + + /// Writes a copy of `self` to `dst`. + /// + /// If `dst.len() != size_of_val(self)`, `write_to` returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::IntoBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(IntoBytes, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let header = PacketHeader { + /// src_port: [0, 1], + /// dst_port: [2, 3], + /// length: [4, 5], + /// checksum: [6, 7], + /// }; + /// + /// let mut bytes = [0, 0, 0, 0, 0, 0, 0, 0]; + /// + /// header.write_to(&mut bytes[..]); + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 4, 5, 6, 7]); + /// ``` + /// + /// If too many or too few target bytes are provided, `write_to` returns + /// `Err` and leaves the target bytes unmodified: + /// + /// ``` + /// # use zerocopy::IntoBytes; + /// # let header = u128::MAX; + /// let mut excessive_bytes = &mut [0u8; 128][..]; + /// + /// let write_result = header.write_to(excessive_bytes); + /// + /// assert!(write_result.is_err()); + /// assert_eq!(excessive_bytes, [0u8; 128]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "write_to", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] + )] + #[must_use = "callers should check the return value to see if the operation succeeded"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + #[allow(clippy::mut_from_ref)] // False positive: `&self -> &mut [u8]` + fn write_to(&self, dst: &mut [u8]) -> Result<(), SizeError<&Self, &mut [u8]>> + where + Self: Immutable, + { + let src = self.as_bytes(); + if dst.len() == src.len() { + // SAFETY: Within this branch of the conditional, we have ensured + // that `dst.len()` is equal to `src.len()`. Neither the size of the + // source nor the size of the destination change between the above + // size check and the invocation of `copy_unchecked`. + unsafe { util::copy_unchecked(src, dst) } + Ok(()) + } else { + Err(SizeError::new(self)) + } + } + + /// Writes a copy of `self` to the prefix of `dst`. + /// + /// `write_to_prefix` writes `self` to the first `size_of_val(self)` bytes + /// of `dst`. If `dst.len() < size_of_val(self)`, it returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::IntoBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(IntoBytes, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let header = PacketHeader { + /// src_port: [0, 1], + /// dst_port: [2, 3], + /// length: [4, 5], + /// checksum: [6, 7], + /// }; + /// + /// let mut bytes = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + /// + /// header.write_to_prefix(&mut bytes[..]); + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 4, 5, 6, 7, 0, 0]); + /// ``` + /// + /// If insufficient target bytes are provided, `write_to_prefix` returns + /// `Err` and leaves the target bytes unmodified: + /// + /// ``` + /// # use zerocopy::IntoBytes; + /// # let header = u128::MAX; + /// let mut insufficient_bytes = &mut [0, 0][..]; + /// + /// let write_result = header.write_to_suffix(insufficient_bytes); + /// + /// assert!(write_result.is_err()); + /// assert_eq!(insufficient_bytes, [0, 0]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "write_to_prefix", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] + )] + #[must_use = "callers should check the return value to see if the operation succeeded"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + #[allow(clippy::mut_from_ref)] // False positive: `&self -> &mut [u8]` + fn write_to_prefix(&self, dst: &mut [u8]) -> Result<(), SizeError<&Self, &mut [u8]>> + where + Self: Immutable, + { + let src = self.as_bytes(); + match dst.get_mut(..src.len()) { + Some(dst) => { + // SAFETY: Within this branch of the `match`, we have ensured + // through fallible subslicing that `dst.len()` is equal to + // `src.len()`. Neither the size of the source nor the size of + // the destination change between the above subslicing operation + // and the invocation of `copy_unchecked`. + unsafe { util::copy_unchecked(src, dst) } + Ok(()) + } + None => Err(SizeError::new(self)), + } + } + + /// Writes a copy of `self` to the suffix of `dst`. + /// + /// `write_to_suffix` writes `self` to the last `size_of_val(self)` bytes of + /// `dst`. If `dst.len() < size_of_val(self)`, it returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::IntoBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(IntoBytes, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let header = PacketHeader { + /// src_port: [0, 1], + /// dst_port: [2, 3], + /// length: [4, 5], + /// checksum: [6, 7], + /// }; + /// + /// let mut bytes = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + /// + /// header.write_to_suffix(&mut bytes[..]); + /// + /// assert_eq!(bytes, [0, 0, 0, 1, 2, 3, 4, 5, 6, 7]); + /// + /// let mut insufficient_bytes = &mut [0, 0][..]; + /// + /// let write_result = header.write_to_suffix(insufficient_bytes); + /// + /// assert!(write_result.is_err()); + /// assert_eq!(insufficient_bytes, [0, 0]); + /// ``` + /// + /// If insufficient target bytes are provided, `write_to_suffix` returns + /// `Err` and leaves the target bytes unmodified: + /// + /// ``` + /// # use zerocopy::IntoBytes; + /// # let header = u128::MAX; + /// let mut insufficient_bytes = &mut [0, 0][..]; + /// + /// let write_result = header.write_to_suffix(insufficient_bytes); + /// + /// assert!(write_result.is_err()); + /// assert_eq!(insufficient_bytes, [0, 0]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "write_to_suffix", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] + )] + #[must_use = "callers should check the return value to see if the operation succeeded"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + #[allow(clippy::mut_from_ref)] // False positive: `&self -> &mut [u8]` + fn write_to_suffix(&self, dst: &mut [u8]) -> Result<(), SizeError<&Self, &mut [u8]>> + where + Self: Immutable, + { + let src = self.as_bytes(); + let start = if let Some(start) = dst.len().checked_sub(src.len()) { + start + } else { + return Err(SizeError::new(self)); + }; + let dst = if let Some(dst) = dst.get_mut(start..) { + dst + } else { + // get_mut() should never return None here. We return a `SizeError` + // rather than .unwrap() because in the event the branch is not + // optimized away, returning a value is generally lighter-weight + // than panicking. + return Err(SizeError::new(self)); + }; + // SAFETY: Through fallible subslicing of `dst`, we have ensured that + // `dst.len()` is equal to `src.len()`. Neither the size of the source + // nor the size of the destination change between the above subslicing + // operation and the invocation of `copy_unchecked`. + unsafe { + util::copy_unchecked(src, dst); + } + Ok(()) + } + + /// Writes a copy of `self` to an `io::Write`. + /// + /// This is a shorthand for `dst.write_all(self.as_bytes())`, and is useful + /// for interfacing with operating system byte sinks (files, sockets, etc.). + /// + /// # Examples + /// + /// ```no_run + /// use zerocopy::{byteorder::big_endian::U16, FromBytes, IntoBytes}; + /// use std::fs::File; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, Immutable, KnownLayout)] + /// #[repr(C, packed)] + /// struct GrayscaleImage { + /// height: U16, + /// width: U16, + /// pixels: [U16], + /// } + /// + /// let image = GrayscaleImage::ref_from_bytes(&[0, 0, 0, 0][..]).unwrap(); + /// let mut file = File::create("image.bin").unwrap(); + /// image.write_to_io(&mut file).unwrap(); + /// ``` + /// + /// If the write fails, `write_to_io` returns `Err` and a partial write may + /// have occurred; e.g.: + /// + /// ``` + /// # use zerocopy::IntoBytes; + /// + /// let src = u128::MAX; + /// let mut dst = [0u8; 2]; + /// + /// let write_result = src.write_to_io(&mut dst[..]); + /// + /// assert!(write_result.is_err()); + /// assert_eq!(dst, [255, 255]); + /// ``` + #[cfg(feature = "std")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "std")))] + #[inline(always)] + fn write_to_io<W>(&self, mut dst: W) -> io::Result<()> + where + Self: Immutable, + W: io::Write, + { + dst.write_all(self.as_bytes()) + } + + #[deprecated(since = "0.8.0", note = "`IntoBytes::as_bytes_mut` was renamed to `as_mut_bytes`")] + #[doc(hidden)] + #[inline] + fn as_bytes_mut(&mut self) -> &mut [u8] + where + Self: FromBytes, + { + self.as_mut_bytes() + } +} + +/// Analyzes whether a type is [`Unaligned`]. +/// +/// This derive analyzes, at compile time, whether the annotated type satisfies +/// the [safety conditions] of `Unaligned` and implements `Unaligned` if it is +/// sound to do so. This derive can be applied to structs, enums, and unions; +/// e.g.: +/// +/// ``` +/// # use zerocopy_derive::Unaligned; +/// #[derive(Unaligned)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Unaligned)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant0, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Unaligned)] +/// #[repr(packed)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// # Analysis +/// +/// *This section describes, roughly, the analysis performed by this derive to +/// determine whether it is sound to implement `Unaligned` for a given type. +/// Unless you are modifying the implementation of this derive, or attempting to +/// manually implement `Unaligned` for a type yourself, you don't need to read +/// this section.* +/// +/// If a type has the following properties, then this derive can implement +/// `Unaligned` for that type: +/// +/// - If the type is a struct or union: +/// - If `repr(align(N))` is provided, `N` must equal 1. +/// - If the type is `repr(C)` or `repr(transparent)`, all fields must be +/// [`Unaligned`]. +/// - If the type is not `repr(C)` or `repr(transparent)`, it must be +/// `repr(packed)` or `repr(packed(1))`. +/// - If the type is an enum: +/// - If `repr(align(N))` is provided, `N` must equal 1. +/// - It must be a field-less enum (meaning that all variants have no fields). +/// - It must be `repr(i8)` or `repr(u8)`. +/// +/// [safety conditions]: trait@Unaligned#safety +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::Unaligned; + +/// Types with no alignment requirement. +/// +/// If `T: Unaligned`, then `align_of::<T>() == 1`. +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(Unaligned)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::Unaligned; +/// #[derive(Unaligned)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Unaligned)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant0, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Unaligned)] +/// #[repr(packed)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive performs a sophisticated, compile-time safety analysis to +/// determine whether a type is `Unaligned`. +/// +/// # Safety +/// +/// *This section describes what is required in order for `T: Unaligned`, and +/// what unsafe code may assume of such types. If you don't plan on implementing +/// `Unaligned` manually, and you don't plan on writing unsafe code that +/// operates on `Unaligned` types, then you don't need to read this section.* +/// +/// If `T: Unaligned`, then unsafe code may assume that it is sound to produce a +/// reference to `T` at any memory location regardless of alignment. If a type +/// is marked as `Unaligned` which violates this contract, it may cause +/// undefined behavior. +/// +/// `#[derive(Unaligned)]` only permits [types which satisfy these +/// requirements][derive-analysis]. +/// +#[cfg_attr( + feature = "derive", + doc = "[derive]: zerocopy_derive::Unaligned", + doc = "[derive-analysis]: zerocopy_derive::Unaligned#analysis" +)] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.Unaligned.html"), + doc = concat!("[derive-analysis]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.Unaligned.html#analysis"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(Unaligned)]` to `{Self}`") +)] +pub unsafe trait Unaligned { + // The `Self: Sized` bound makes it so that `Unaligned` is still object + // safe. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; +} + +/// Derives optimized [`PartialEq`] and [`Eq`] implementations. +/// +/// This derive can be applied to structs and enums implementing both +/// [`Immutable`] and [`IntoBytes`]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{ByteEq, Immutable, IntoBytes}; +/// #[derive(ByteEq, Immutable, IntoBytes)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(ByteEq, Immutable, IntoBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// The standard library's [`derive(Eq, PartialEq)`][derive@PartialEq] computes +/// equality by individually comparing each field. Instead, the implementation +/// of [`PartialEq::eq`] emitted by `derive(ByteHash)` converts the entirety of +/// `self` and `other` to byte slices and compares those slices for equality. +/// This may have performance advantages. +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::ByteEq; +/// Derives an optimized [`Hash`] implementation. +/// +/// This derive can be applied to structs and enums implementing both +/// [`Immutable`] and [`IntoBytes`]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{ByteHash, Immutable, IntoBytes}; +/// #[derive(ByteHash, Immutable, IntoBytes)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(ByteHash, Immutable, IntoBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// The standard library's [`derive(Hash)`][derive@Hash] produces hashes by +/// individually hashing each field and combining the results. Instead, the +/// implementations of [`Hash::hash()`] and [`Hash::hash_slice()`] generated by +/// `derive(ByteHash)` convert the entirety of `self` to a byte slice and hashes +/// it in a single call to [`Hasher::write()`]. This may have performance +/// advantages. +/// +/// [`Hash`]: core::hash::Hash +/// [`Hash::hash()`]: core::hash::Hash::hash() +/// [`Hash::hash_slice()`]: core::hash::Hash::hash_slice() +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::ByteHash; +/// Implements [`SplitAt`]. +/// +/// This derive can be applied to structs; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{ByteEq, Immutable, IntoBytes}; +/// #[derive(ByteEq, Immutable, IntoBytes)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// ``` +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::SplitAt; + +#[cfg(feature = "alloc")] +#[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] +#[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] +mod alloc_support { + use super::*; + + /// Extends a `Vec<T>` by pushing `additional` new items onto the end of the + /// vector. The new items are initialized with zeros. + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[doc(hidden)] + #[deprecated(since = "0.8.0", note = "moved to `FromZeros`")] + #[inline(always)] + pub fn extend_vec_zeroed<T: FromZeros>( + v: &mut Vec<T>, + additional: usize, + ) -> Result<(), AllocError> { + <T as FromZeros>::extend_vec_zeroed(v, additional) + } + + /// Inserts `additional` new items into `Vec<T>` at `position`. The new + /// items are initialized with zeros. + /// + /// # Panics + /// + /// Panics if `position > v.len()`. + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[doc(hidden)] + #[deprecated(since = "0.8.0", note = "moved to `FromZeros`")] + #[inline(always)] + pub fn insert_vec_zeroed<T: FromZeros>( + v: &mut Vec<T>, + position: usize, + additional: usize, + ) -> Result<(), AllocError> { + <T as FromZeros>::insert_vec_zeroed(v, position, additional) + } +} + +#[cfg(feature = "alloc")] +#[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] +#[doc(hidden)] +pub use alloc_support::*; + +#[cfg(test)] +#[allow(clippy::assertions_on_result_states, clippy::unreadable_literal)] +mod tests { + use static_assertions::assert_impl_all; + + use super::*; + use crate::util::testutil::*; + + // An unsized type. + // + // This is used to test the custom derives of our traits. The `[u8]` type + // gets a hand-rolled impl, so it doesn't exercise our custom derives. + #[derive(Debug, Eq, PartialEq, FromBytes, IntoBytes, Unaligned, Immutable)] + #[repr(transparent)] + struct Unsized([u8]); + + impl Unsized { + fn from_mut_slice(slc: &mut [u8]) -> &mut Unsized { + // SAFETY: This *probably* sound - since the layouts of `[u8]` and + // `Unsized` are the same, so are the layouts of `&mut [u8]` and + // `&mut Unsized`. [1] Even if it turns out that this isn't actually + // guaranteed by the language spec, we can just change this since + // it's in test code. + // + // [1] https://github.com/rust-lang/unsafe-code-guidelines/issues/375 + unsafe { mem::transmute(slc) } + } + } + + #[test] + fn test_known_layout() { + // Test that `$ty` and `ManuallyDrop<$ty>` have the expected layout. + // Test that `PhantomData<$ty>` has the same layout as `()` regardless + // of `$ty`. + macro_rules! test { + ($ty:ty, $expect:expr) => { + let expect = $expect; + assert_eq!(<$ty as KnownLayout>::LAYOUT, expect); + assert_eq!(<ManuallyDrop<$ty> as KnownLayout>::LAYOUT, expect); + assert_eq!(<PhantomData<$ty> as KnownLayout>::LAYOUT, <() as KnownLayout>::LAYOUT); + }; + } + + let layout = + |offset, align, trailing_slice_elem_size, statically_shallow_unpadded| DstLayout { + align: NonZeroUsize::new(align).unwrap(), + size_info: match trailing_slice_elem_size { + None => SizeInfo::Sized { size: offset }, + Some(elem_size) => { + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) + } + }, + statically_shallow_unpadded, + }; + + test!((), layout(0, 1, None, false)); + test!(u8, layout(1, 1, None, false)); + // Use `align_of` because `u64` alignment may be smaller than 8 on some + // platforms. + test!(u64, layout(8, mem::align_of::<u64>(), None, false)); + test!(AU64, layout(8, 8, None, false)); + + test!(Option<&'static ()>, usize::LAYOUT); + + test!([()], layout(0, 1, Some(0), true)); + test!([u8], layout(0, 1, Some(1), true)); + test!(str, layout(0, 1, Some(1), true)); + } + + #[cfg(feature = "derive")] + #[test] + fn test_known_layout_derive() { + // In this and other files (`late_compile_pass.rs`, + // `mid_compile_pass.rs`, and `struct.rs`), we test success and failure + // modes of `derive(KnownLayout)` for the following combination of + // properties: + // + // +------------+--------------------------------------+-----------+ + // | | trailing field properties | | + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // |------------+----------+----------------+----------+-----------| + // | N | N | N | N | KL00 | + // | N | N | N | Y | KL01 | + // | N | N | Y | N | KL02 | + // | N | N | Y | Y | KL03 | + // | N | Y | N | N | KL04 | + // | N | Y | N | Y | KL05 | + // | N | Y | Y | N | KL06 | + // | N | Y | Y | Y | KL07 | + // | Y | N | N | N | KL08 | + // | Y | N | N | Y | KL09 | + // | Y | N | Y | N | KL10 | + // | Y | N | Y | Y | KL11 | + // | Y | Y | N | N | KL12 | + // | Y | Y | N | Y | KL13 | + // | Y | Y | Y | N | KL14 | + // | Y | Y | Y | Y | KL15 | + // +------------+----------+----------------+----------+-----------+ + + struct NotKnownLayout<T = ()> { + _t: T, + } + + #[derive(KnownLayout)] + #[repr(C)] + struct AlignSize<const ALIGN: usize, const SIZE: usize> + where + elain::Align<ALIGN>: elain::Alignment, + { + _align: elain::Align<ALIGN>, + size: [u8; SIZE], + } + + type AU16 = AlignSize<2, 2>; + type AU32 = AlignSize<4, 4>; + + fn _assert_kl<T: ?Sized + KnownLayout>(_: &T) {} + + let sized_layout = |align, size| DstLayout { + align: NonZeroUsize::new(align).unwrap(), + size_info: SizeInfo::Sized { size }, + statically_shallow_unpadded: false, + }; + + let unsized_layout = |align, elem_size, offset, statically_shallow_unpadded| DstLayout { + align: NonZeroUsize::new(align).unwrap(), + size_info: SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }), + statically_shallow_unpadded, + }; + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | N | N | N | Y | KL01 | + #[allow(dead_code)] + #[derive(KnownLayout)] + struct KL01(NotKnownLayout<AU32>, NotKnownLayout<AU16>); + + let expected = DstLayout::for_type::<KL01>(); + + assert_eq!(<KL01 as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL01 as KnownLayout>::LAYOUT, sized_layout(4, 8)); + + // ...with `align(N)`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(align(64))] + struct KL01Align(NotKnownLayout<AU32>, NotKnownLayout<AU16>); + + let expected = DstLayout::for_type::<KL01Align>(); + + assert_eq!(<KL01Align as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL01Align as KnownLayout>::LAYOUT, sized_layout(64, 64)); + + // ...with `packed`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(packed)] + struct KL01Packed(NotKnownLayout<AU32>, NotKnownLayout<AU16>); + + let expected = DstLayout::for_type::<KL01Packed>(); + + assert_eq!(<KL01Packed as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL01Packed as KnownLayout>::LAYOUT, sized_layout(1, 6)); + + // ...with `packed(N)`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(packed(2))] + struct KL01PackedN(NotKnownLayout<AU32>, NotKnownLayout<AU16>); + + assert_impl_all!(KL01PackedN: KnownLayout); + + let expected = DstLayout::for_type::<KL01PackedN>(); + + assert_eq!(<KL01PackedN as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL01PackedN as KnownLayout>::LAYOUT, sized_layout(2, 6)); + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | N | N | Y | Y | KL03 | + #[allow(dead_code)] + #[derive(KnownLayout)] + struct KL03(NotKnownLayout, u8); + + let expected = DstLayout::for_type::<KL03>(); + + assert_eq!(<KL03 as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL03 as KnownLayout>::LAYOUT, sized_layout(1, 1)); + + // ... with `align(N)` + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(align(64))] + struct KL03Align(NotKnownLayout<AU32>, u8); + + let expected = DstLayout::for_type::<KL03Align>(); + + assert_eq!(<KL03Align as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL03Align as KnownLayout>::LAYOUT, sized_layout(64, 64)); + + // ... with `packed`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(packed)] + struct KL03Packed(NotKnownLayout<AU32>, u8); + + let expected = DstLayout::for_type::<KL03Packed>(); + + assert_eq!(<KL03Packed as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL03Packed as KnownLayout>::LAYOUT, sized_layout(1, 5)); + + // ... with `packed(N)` + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(packed(2))] + struct KL03PackedN(NotKnownLayout<AU32>, u8); + + assert_impl_all!(KL03PackedN: KnownLayout); + + let expected = DstLayout::for_type::<KL03PackedN>(); + + assert_eq!(<KL03PackedN as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL03PackedN as KnownLayout>::LAYOUT, sized_layout(2, 6)); + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | N | Y | N | Y | KL05 | + #[allow(dead_code)] + #[derive(KnownLayout)] + struct KL05<T>(u8, T); + + fn _test_kl05<T>(t: T) -> impl KnownLayout { + KL05(0u8, t) + } + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | N | Y | Y | Y | KL07 | + #[allow(dead_code)] + #[derive(KnownLayout)] + struct KL07<T: KnownLayout>(u8, T); + + fn _test_kl07<T: KnownLayout>(t: T) -> impl KnownLayout { + let _ = KL07(0u8, t); + } + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | Y | N | Y | N | KL10 | + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C)] + struct KL10(NotKnownLayout<AU32>, [u8]); + + let expected = DstLayout::new_zst(None) + .extend(DstLayout::for_type::<NotKnownLayout<AU32>>(), None) + .extend(<[u8] as KnownLayout>::LAYOUT, None) + .pad_to_align(); + + assert_eq!(<KL10 as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL10 as KnownLayout>::LAYOUT, unsized_layout(4, 1, 4, false)); + + // ...with `align(N)`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C, align(64))] + struct KL10Align(NotKnownLayout<AU32>, [u8]); + + let repr_align = NonZeroUsize::new(64); + + let expected = DstLayout::new_zst(repr_align) + .extend(DstLayout::for_type::<NotKnownLayout<AU32>>(), None) + .extend(<[u8] as KnownLayout>::LAYOUT, None) + .pad_to_align(); + + assert_eq!(<KL10Align as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL10Align as KnownLayout>::LAYOUT, unsized_layout(64, 1, 4, false)); + + // ...with `packed`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C, packed)] + struct KL10Packed(NotKnownLayout<AU32>, [u8]); + + let repr_packed = NonZeroUsize::new(1); + + let expected = DstLayout::new_zst(None) + .extend(DstLayout::for_type::<NotKnownLayout<AU32>>(), repr_packed) + .extend(<[u8] as KnownLayout>::LAYOUT, repr_packed) + .pad_to_align(); + + assert_eq!(<KL10Packed as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL10Packed as KnownLayout>::LAYOUT, unsized_layout(1, 1, 4, false)); + + // ...with `packed(N)`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C, packed(2))] + struct KL10PackedN(NotKnownLayout<AU32>, [u8]); + + let repr_packed = NonZeroUsize::new(2); + + let expected = DstLayout::new_zst(None) + .extend(DstLayout::for_type::<NotKnownLayout<AU32>>(), repr_packed) + .extend(<[u8] as KnownLayout>::LAYOUT, repr_packed) + .pad_to_align(); + + assert_eq!(<KL10PackedN as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL10PackedN as KnownLayout>::LAYOUT, unsized_layout(2, 1, 4, false)); + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | Y | N | Y | Y | KL11 | + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C)] + struct KL11(NotKnownLayout<AU64>, u8); + + let expected = DstLayout::new_zst(None) + .extend(DstLayout::for_type::<NotKnownLayout<AU64>>(), None) + .extend(<u8 as KnownLayout>::LAYOUT, None) + .pad_to_align(); + + assert_eq!(<KL11 as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL11 as KnownLayout>::LAYOUT, sized_layout(8, 16)); + + // ...with `align(N)`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C, align(64))] + struct KL11Align(NotKnownLayout<AU64>, u8); + + let repr_align = NonZeroUsize::new(64); + + let expected = DstLayout::new_zst(repr_align) + .extend(DstLayout::for_type::<NotKnownLayout<AU64>>(), None) + .extend(<u8 as KnownLayout>::LAYOUT, None) + .pad_to_align(); + + assert_eq!(<KL11Align as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL11Align as KnownLayout>::LAYOUT, sized_layout(64, 64)); + + // ...with `packed`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C, packed)] + struct KL11Packed(NotKnownLayout<AU64>, u8); + + let repr_packed = NonZeroUsize::new(1); + + let expected = DstLayout::new_zst(None) + .extend(DstLayout::for_type::<NotKnownLayout<AU64>>(), repr_packed) + .extend(<u8 as KnownLayout>::LAYOUT, repr_packed) + .pad_to_align(); + + assert_eq!(<KL11Packed as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL11Packed as KnownLayout>::LAYOUT, sized_layout(1, 9)); + + // ...with `packed(N)`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C, packed(2))] + struct KL11PackedN(NotKnownLayout<AU64>, u8); + + let repr_packed = NonZeroUsize::new(2); + + let expected = DstLayout::new_zst(None) + .extend(DstLayout::for_type::<NotKnownLayout<AU64>>(), repr_packed) + .extend(<u8 as KnownLayout>::LAYOUT, repr_packed) + .pad_to_align(); + + assert_eq!(<KL11PackedN as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL11PackedN as KnownLayout>::LAYOUT, sized_layout(2, 10)); + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | Y | Y | Y | N | KL14 | + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C)] + struct KL14<T: ?Sized + KnownLayout>(u8, T); + + fn _test_kl14<T: ?Sized + KnownLayout>(kl: &KL14<T>) { + _assert_kl(kl) + } + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | Y | Y | Y | Y | KL15 | + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C)] + struct KL15<T: KnownLayout>(u8, T); + + fn _test_kl15<T: KnownLayout>(t: T) -> impl KnownLayout { + let _ = KL15(0u8, t); + } + + // Test a variety of combinations of field types: + // - () + // - u8 + // - AU16 + // - [()] + // - [u8] + // - [AU16] + + #[allow(clippy::upper_case_acronyms, dead_code)] + #[derive(KnownLayout)] + #[repr(C)] + struct KLTU<T, U: ?Sized>(T, U); + + assert_eq!(<KLTU<(), ()> as KnownLayout>::LAYOUT, sized_layout(1, 0)); + + assert_eq!(<KLTU<(), u8> as KnownLayout>::LAYOUT, sized_layout(1, 1)); + + assert_eq!(<KLTU<(), AU16> as KnownLayout>::LAYOUT, sized_layout(2, 2)); + + assert_eq!(<KLTU<(), [()]> as KnownLayout>::LAYOUT, unsized_layout(1, 0, 0, false)); + + assert_eq!(<KLTU<(), [u8]> as KnownLayout>::LAYOUT, unsized_layout(1, 1, 0, false)); + + assert_eq!(<KLTU<(), [AU16]> as KnownLayout>::LAYOUT, unsized_layout(2, 2, 0, false)); + + assert_eq!(<KLTU<u8, ()> as KnownLayout>::LAYOUT, sized_layout(1, 1)); + + assert_eq!(<KLTU<u8, u8> as KnownLayout>::LAYOUT, sized_layout(1, 2)); + + assert_eq!(<KLTU<u8, AU16> as KnownLayout>::LAYOUT, sized_layout(2, 4)); + + assert_eq!(<KLTU<u8, [()]> as KnownLayout>::LAYOUT, unsized_layout(1, 0, 1, false)); + + assert_eq!(<KLTU<u8, [u8]> as KnownLayout>::LAYOUT, unsized_layout(1, 1, 1, false)); + + assert_eq!(<KLTU<u8, [AU16]> as KnownLayout>::LAYOUT, unsized_layout(2, 2, 2, false)); + + assert_eq!(<KLTU<AU16, ()> as KnownLayout>::LAYOUT, sized_layout(2, 2)); + + assert_eq!(<KLTU<AU16, u8> as KnownLayout>::LAYOUT, sized_layout(2, 4)); + + assert_eq!(<KLTU<AU16, AU16> as KnownLayout>::LAYOUT, sized_layout(2, 4)); + + assert_eq!(<KLTU<AU16, [()]> as KnownLayout>::LAYOUT, unsized_layout(2, 0, 2, false)); + + assert_eq!(<KLTU<AU16, [u8]> as KnownLayout>::LAYOUT, unsized_layout(2, 1, 2, false)); + + assert_eq!(<KLTU<AU16, [AU16]> as KnownLayout>::LAYOUT, unsized_layout(2, 2, 2, false)); + + // Test a variety of field counts. + + #[derive(KnownLayout)] + #[repr(C)] + struct KLF0; + + assert_eq!(<KLF0 as KnownLayout>::LAYOUT, sized_layout(1, 0)); + + #[derive(KnownLayout)] + #[repr(C)] + struct KLF1([u8]); + + assert_eq!(<KLF1 as KnownLayout>::LAYOUT, unsized_layout(1, 1, 0, true)); + + #[derive(KnownLayout)] + #[repr(C)] + struct KLF2(NotKnownLayout<u8>, [u8]); + + assert_eq!(<KLF2 as KnownLayout>::LAYOUT, unsized_layout(1, 1, 1, false)); + + #[derive(KnownLayout)] + #[repr(C)] + struct KLF3(NotKnownLayout<u8>, NotKnownLayout<AU16>, [u8]); + + assert_eq!(<KLF3 as KnownLayout>::LAYOUT, unsized_layout(2, 1, 4, false)); + + #[derive(KnownLayout)] + #[repr(C)] + struct KLF4(NotKnownLayout<u8>, NotKnownLayout<AU16>, NotKnownLayout<AU32>, [u8]); + + assert_eq!(<KLF4 as KnownLayout>::LAYOUT, unsized_layout(4, 1, 8, false)); + } + + #[test] + fn test_object_safety() { + fn _takes_immutable(_: &dyn Immutable) {} + fn _takes_unaligned(_: &dyn Unaligned) {} + } + + #[test] + fn test_from_zeros_only() { + // Test types that implement `FromZeros` but not `FromBytes`. + + assert!(!bool::new_zeroed()); + assert_eq!(char::new_zeroed(), '\0'); + + #[cfg(feature = "alloc")] + { + assert_eq!(bool::new_box_zeroed(), Ok(Box::new(false))); + assert_eq!(char::new_box_zeroed(), Ok(Box::new('\0'))); + + assert_eq!( + <[bool]>::new_box_zeroed_with_elems(3).unwrap().as_ref(), + [false, false, false] + ); + assert_eq!( + <[char]>::new_box_zeroed_with_elems(3).unwrap().as_ref(), + ['\0', '\0', '\0'] + ); + + assert_eq!(bool::new_vec_zeroed(3).unwrap().as_ref(), [false, false, false]); + assert_eq!(char::new_vec_zeroed(3).unwrap().as_ref(), ['\0', '\0', '\0']); + } + + let mut string = "hello".to_string(); + let s: &mut str = string.as_mut(); + assert_eq!(s, "hello"); + s.zero(); + assert_eq!(s, "\0\0\0\0\0"); + } + + #[test] + fn test_zst_count_preserved() { + // Test that, when an explicit count is provided to for a type with a + // ZST trailing slice element, that count is preserved. This is + // important since, for such types, all element counts result in objects + // of the same size, and so the correct behavior is ambiguous. However, + // preserving the count as requested by the user is the behavior that we + // document publicly. + + // FromZeros methods + #[cfg(feature = "alloc")] + assert_eq!(<[()]>::new_box_zeroed_with_elems(3).unwrap().len(), 3); + #[cfg(feature = "alloc")] + assert_eq!(<()>::new_vec_zeroed(3).unwrap().len(), 3); + + // FromBytes methods + assert_eq!(<[()]>::ref_from_bytes_with_elems(&[][..], 3).unwrap().len(), 3); + assert_eq!(<[()]>::ref_from_prefix_with_elems(&[][..], 3).unwrap().0.len(), 3); + assert_eq!(<[()]>::ref_from_suffix_with_elems(&[][..], 3).unwrap().1.len(), 3); + assert_eq!(<[()]>::mut_from_bytes_with_elems(&mut [][..], 3).unwrap().len(), 3); + assert_eq!(<[()]>::mut_from_prefix_with_elems(&mut [][..], 3).unwrap().0.len(), 3); + assert_eq!(<[()]>::mut_from_suffix_with_elems(&mut [][..], 3).unwrap().1.len(), 3); + } + + #[test] + fn test_read_write() { + const VAL: u64 = 0x12345678; + #[cfg(target_endian = "big")] + const VAL_BYTES: [u8; 8] = VAL.to_be_bytes(); + #[cfg(target_endian = "little")] + const VAL_BYTES: [u8; 8] = VAL.to_le_bytes(); + const ZEROS: [u8; 8] = [0u8; 8]; + + // Test `FromBytes::{read_from, read_from_prefix, read_from_suffix}`. + + assert_eq!(u64::read_from_bytes(&VAL_BYTES[..]), Ok(VAL)); + // The first 8 bytes are from `VAL_BYTES` and the second 8 bytes are all + // zeros. + let bytes_with_prefix: [u8; 16] = transmute!([VAL_BYTES, [0; 8]]); + assert_eq!(u64::read_from_prefix(&bytes_with_prefix[..]), Ok((VAL, &ZEROS[..]))); + assert_eq!(u64::read_from_suffix(&bytes_with_prefix[..]), Ok((&VAL_BYTES[..], 0))); + // The first 8 bytes are all zeros and the second 8 bytes are from + // `VAL_BYTES` + let bytes_with_suffix: [u8; 16] = transmute!([[0; 8], VAL_BYTES]); + assert_eq!(u64::read_from_prefix(&bytes_with_suffix[..]), Ok((0, &VAL_BYTES[..]))); + assert_eq!(u64::read_from_suffix(&bytes_with_suffix[..]), Ok((&ZEROS[..], VAL))); + + // Test `IntoBytes::{write_to, write_to_prefix, write_to_suffix}`. + + let mut bytes = [0u8; 8]; + assert_eq!(VAL.write_to(&mut bytes[..]), Ok(())); + assert_eq!(bytes, VAL_BYTES); + let mut bytes = [0u8; 16]; + assert_eq!(VAL.write_to_prefix(&mut bytes[..]), Ok(())); + let want: [u8; 16] = transmute!([VAL_BYTES, [0; 8]]); + assert_eq!(bytes, want); + let mut bytes = [0u8; 16]; + assert_eq!(VAL.write_to_suffix(&mut bytes[..]), Ok(())); + let want: [u8; 16] = transmute!([[0; 8], VAL_BYTES]); + assert_eq!(bytes, want); + } + + #[test] + #[cfg(feature = "std")] + fn test_read_io_with_padding_soundness() { + // This test is designed to exhibit potential UB in + // `FromBytes::read_from_io`. (see #2319, #2320). + + // On most platforms (where `align_of::<u16>() == 2`), `WithPadding` + // will have inter-field padding between `x` and `y`. + #[derive(FromBytes)] + #[repr(C)] + struct WithPadding { + x: u8, + y: u16, + } + struct ReadsInRead; + impl std::io::Read for ReadsInRead { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> { + // This body branches on every byte of `buf`, ensuring that it + // exhibits UB if any byte of `buf` is uninitialized. + if buf.iter().all(|&x| x == 0) { + Ok(buf.len()) + } else { + buf.iter_mut().for_each(|x| *x = 0); + Ok(buf.len()) + } + } + } + assert!(matches!(WithPadding::read_from_io(ReadsInRead), Ok(WithPadding { x: 0, y: 0 }))); + } + + #[test] + #[cfg(feature = "std")] + fn test_read_write_io() { + let mut long_buffer = [0, 0, 0, 0]; + assert!(matches!(u16::MAX.write_to_io(&mut long_buffer[..]), Ok(()))); + assert_eq!(long_buffer, [255, 255, 0, 0]); + assert!(matches!(u16::read_from_io(&long_buffer[..]), Ok(u16::MAX))); + + let mut short_buffer = [0, 0]; + assert!(u32::MAX.write_to_io(&mut short_buffer[..]).is_err()); + assert_eq!(short_buffer, [255, 255]); + assert!(u32::read_from_io(&short_buffer[..]).is_err()); + } + + #[test] + fn test_try_from_bytes_try_read_from() { + assert_eq!(<bool as TryFromBytes>::try_read_from_bytes(&[0]), Ok(false)); + assert_eq!(<bool as TryFromBytes>::try_read_from_bytes(&[1]), Ok(true)); + + assert_eq!(<bool as TryFromBytes>::try_read_from_prefix(&[0, 2]), Ok((false, &[2][..]))); + assert_eq!(<bool as TryFromBytes>::try_read_from_prefix(&[1, 2]), Ok((true, &[2][..]))); + + assert_eq!(<bool as TryFromBytes>::try_read_from_suffix(&[2, 0]), Ok((&[2][..], false))); + assert_eq!(<bool as TryFromBytes>::try_read_from_suffix(&[2, 1]), Ok((&[2][..], true))); + + // If we don't pass enough bytes, it fails. + assert!(matches!( + <u8 as TryFromBytes>::try_read_from_bytes(&[]), + Err(TryReadError::Size(_)) + )); + assert!(matches!( + <u8 as TryFromBytes>::try_read_from_prefix(&[]), + Err(TryReadError::Size(_)) + )); + assert!(matches!( + <u8 as TryFromBytes>::try_read_from_suffix(&[]), + Err(TryReadError::Size(_)) + )); + + // If we pass too many bytes, it fails. + assert!(matches!( + <u8 as TryFromBytes>::try_read_from_bytes(&[0, 0]), + Err(TryReadError::Size(_)) + )); + + // If we pass an invalid value, it fails. + assert!(matches!( + <bool as TryFromBytes>::try_read_from_bytes(&[2]), + Err(TryReadError::Validity(_)) + )); + assert!(matches!( + <bool as TryFromBytes>::try_read_from_prefix(&[2, 0]), + Err(TryReadError::Validity(_)) + )); + assert!(matches!( + <bool as TryFromBytes>::try_read_from_suffix(&[0, 2]), + Err(TryReadError::Validity(_)) + )); + + // Reading from a misaligned buffer should still succeed. Since `AU64`'s + // alignment is 8, and since we read from two adjacent addresses one + // byte apart, it is guaranteed that at least one of them (though + // possibly both) will be misaligned. + let bytes: [u8; 9] = [0, 0, 0, 0, 0, 0, 0, 0, 0]; + assert_eq!(<AU64 as TryFromBytes>::try_read_from_bytes(&bytes[..8]), Ok(AU64(0))); + assert_eq!(<AU64 as TryFromBytes>::try_read_from_bytes(&bytes[1..9]), Ok(AU64(0))); + + assert_eq!( + <AU64 as TryFromBytes>::try_read_from_prefix(&bytes[..8]), + Ok((AU64(0), &[][..])) + ); + assert_eq!( + <AU64 as TryFromBytes>::try_read_from_prefix(&bytes[1..9]), + Ok((AU64(0), &[][..])) + ); + + assert_eq!( + <AU64 as TryFromBytes>::try_read_from_suffix(&bytes[..8]), + Ok((&[][..], AU64(0))) + ); + assert_eq!( + <AU64 as TryFromBytes>::try_read_from_suffix(&bytes[1..9]), + Ok((&[][..], AU64(0))) + ); + } + + #[test] + fn test_ref_from_mut_from_bytes() { + // Test `FromBytes::{ref_from_bytes, mut_from_bytes}{,_prefix,Suffix}` + // success cases. Exhaustive coverage for these methods is covered by + // the `Ref` tests above, which these helper methods defer to. + + let mut buf = + Align::<[u8; 16], AU64>::new([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + + assert_eq!( + AU64::ref_from_bytes(&buf.t[8..]).unwrap().0.to_ne_bytes(), + [8, 9, 10, 11, 12, 13, 14, 15] + ); + let suffix = AU64::mut_from_bytes(&mut buf.t[8..]).unwrap(); + suffix.0 = 0x0101010101010101; + // The `[u8:9]` is a non-half size of the full buffer, which would catch + // `from_prefix` having the same implementation as `from_suffix` (issues #506, #511). + assert_eq!( + <[u8; 9]>::ref_from_suffix(&buf.t[..]).unwrap(), + (&[0, 1, 2, 3, 4, 5, 6][..], &[7u8, 1, 1, 1, 1, 1, 1, 1, 1]) + ); + let (prefix, suffix) = AU64::mut_from_suffix(&mut buf.t[1..]).unwrap(); + assert_eq!(prefix, &mut [1u8, 2, 3, 4, 5, 6, 7][..]); + suffix.0 = 0x0202020202020202; + let (prefix, suffix) = <[u8; 10]>::mut_from_suffix(&mut buf.t[..]).unwrap(); + assert_eq!(prefix, &mut [0u8, 1, 2, 3, 4, 5][..]); + suffix[0] = 42; + assert_eq!( + <[u8; 9]>::ref_from_prefix(&buf.t[..]).unwrap(), + (&[0u8, 1, 2, 3, 4, 5, 42, 7, 2], &[2u8, 2, 2, 2, 2, 2, 2][..]) + ); + <[u8; 2]>::mut_from_prefix(&mut buf.t[..]).unwrap().0[1] = 30; + assert_eq!(buf.t, [0, 30, 2, 3, 4, 5, 42, 7, 2, 2, 2, 2, 2, 2, 2, 2]); + } + + #[test] + fn test_ref_from_mut_from_bytes_error() { + // Test `FromBytes::{ref_from_bytes, mut_from_bytes}{,_prefix,Suffix}` + // error cases. + + // Fail because the buffer is too large. + let mut buf = Align::<[u8; 16], AU64>::default(); + // `buf.t` should be aligned to 8, so only the length check should fail. + assert!(AU64::ref_from_bytes(&buf.t[..]).is_err()); + assert!(AU64::mut_from_bytes(&mut buf.t[..]).is_err()); + assert!(<[u8; 8]>::ref_from_bytes(&buf.t[..]).is_err()); + assert!(<[u8; 8]>::mut_from_bytes(&mut buf.t[..]).is_err()); + + // Fail because the buffer is too small. + let mut buf = Align::<[u8; 4], AU64>::default(); + assert!(AU64::ref_from_bytes(&buf.t[..]).is_err()); + assert!(AU64::mut_from_bytes(&mut buf.t[..]).is_err()); + assert!(<[u8; 8]>::ref_from_bytes(&buf.t[..]).is_err()); + assert!(<[u8; 8]>::mut_from_bytes(&mut buf.t[..]).is_err()); + assert!(AU64::ref_from_prefix(&buf.t[..]).is_err()); + assert!(AU64::mut_from_prefix(&mut buf.t[..]).is_err()); + assert!(AU64::ref_from_suffix(&buf.t[..]).is_err()); + assert!(AU64::mut_from_suffix(&mut buf.t[..]).is_err()); + assert!(<[u8; 8]>::ref_from_prefix(&buf.t[..]).is_err()); + assert!(<[u8; 8]>::mut_from_prefix(&mut buf.t[..]).is_err()); + assert!(<[u8; 8]>::ref_from_suffix(&buf.t[..]).is_err()); + assert!(<[u8; 8]>::mut_from_suffix(&mut buf.t[..]).is_err()); + + // Fail because the alignment is insufficient. + let mut buf = Align::<[u8; 13], AU64>::default(); + assert!(AU64::ref_from_bytes(&buf.t[1..]).is_err()); + assert!(AU64::mut_from_bytes(&mut buf.t[1..]).is_err()); + assert!(AU64::ref_from_bytes(&buf.t[1..]).is_err()); + assert!(AU64::mut_from_bytes(&mut buf.t[1..]).is_err()); + assert!(AU64::ref_from_prefix(&buf.t[1..]).is_err()); + assert!(AU64::mut_from_prefix(&mut buf.t[1..]).is_err()); + assert!(AU64::ref_from_suffix(&buf.t[..]).is_err()); + assert!(AU64::mut_from_suffix(&mut buf.t[..]).is_err()); + } + + #[test] + fn test_to_methods() { + /// Run a series of tests by calling `IntoBytes` methods on `t`. + /// + /// `bytes` is the expected byte sequence returned from `t.as_bytes()` + /// before `t` has been modified. `post_mutation` is the expected + /// sequence returned from `t.as_bytes()` after `t.as_mut_bytes()[0]` + /// has had its bits flipped (by applying `^= 0xFF`). + /// + /// `N` is the size of `t` in bytes. + fn test<T: FromBytes + IntoBytes + Immutable + Debug + Eq + ?Sized, const N: usize>( + t: &mut T, + bytes: &[u8], + post_mutation: &T, + ) { + // Test that we can access the underlying bytes, and that we get the + // right bytes and the right number of bytes. + assert_eq!(t.as_bytes(), bytes); + + // Test that changes to the underlying byte slices are reflected in + // the original object. + t.as_mut_bytes()[0] ^= 0xFF; + assert_eq!(t, post_mutation); + t.as_mut_bytes()[0] ^= 0xFF; + + // `write_to` rejects slices that are too small or too large. + assert!(t.write_to(&mut vec![0; N - 1][..]).is_err()); + assert!(t.write_to(&mut vec![0; N + 1][..]).is_err()); + + // `write_to` works as expected. + let mut bytes = [0; N]; + assert_eq!(t.write_to(&mut bytes[..]), Ok(())); + assert_eq!(bytes, t.as_bytes()); + + // `write_to_prefix` rejects slices that are too small. + assert!(t.write_to_prefix(&mut vec![0; N - 1][..]).is_err()); + + // `write_to_prefix` works with exact-sized slices. + let mut bytes = [0; N]; + assert_eq!(t.write_to_prefix(&mut bytes[..]), Ok(())); + assert_eq!(bytes, t.as_bytes()); + + // `write_to_prefix` works with too-large slices, and any bytes past + // the prefix aren't modified. + let mut too_many_bytes = vec![0; N + 1]; + too_many_bytes[N] = 123; + assert_eq!(t.write_to_prefix(&mut too_many_bytes[..]), Ok(())); + assert_eq!(&too_many_bytes[..N], t.as_bytes()); + assert_eq!(too_many_bytes[N], 123); + + // `write_to_suffix` rejects slices that are too small. + assert!(t.write_to_suffix(&mut vec![0; N - 1][..]).is_err()); + + // `write_to_suffix` works with exact-sized slices. + let mut bytes = [0; N]; + assert_eq!(t.write_to_suffix(&mut bytes[..]), Ok(())); + assert_eq!(bytes, t.as_bytes()); + + // `write_to_suffix` works with too-large slices, and any bytes + // before the suffix aren't modified. + let mut too_many_bytes = vec![0; N + 1]; + too_many_bytes[0] = 123; + assert_eq!(t.write_to_suffix(&mut too_many_bytes[..]), Ok(())); + assert_eq!(&too_many_bytes[1..], t.as_bytes()); + assert_eq!(too_many_bytes[0], 123); + } + + #[derive(Debug, Eq, PartialEq, FromBytes, IntoBytes, Immutable)] + #[repr(C)] + struct Foo { + a: u32, + b: Wrapping<u32>, + c: Option<NonZeroU32>, + } + + let expected_bytes: Vec<u8> = if cfg!(target_endian = "little") { + vec![1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] + } else { + vec![0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0] + }; + let post_mutation_expected_a = + if cfg!(target_endian = "little") { 0x00_00_00_FE } else { 0xFF_00_00_01 }; + test::<_, 12>( + &mut Foo { a: 1, b: Wrapping(2), c: None }, + expected_bytes.as_bytes(), + &Foo { a: post_mutation_expected_a, b: Wrapping(2), c: None }, + ); + test::<_, 3>( + Unsized::from_mut_slice(&mut [1, 2, 3]), + &[1, 2, 3], + Unsized::from_mut_slice(&mut [0xFE, 2, 3]), + ); + } + + #[test] + fn test_array() { + #[derive(FromBytes, IntoBytes, Immutable)] + #[repr(C)] + struct Foo { + a: [u16; 33], + } + + let foo = Foo { a: [0xFFFF; 33] }; + let expected = [0xFFu8; 66]; + assert_eq!(foo.as_bytes(), &expected[..]); + } + + #[test] + fn test_new_zeroed() { + assert!(!bool::new_zeroed()); + assert_eq!(u64::new_zeroed(), 0); + // This test exists in order to exercise unsafe code, especially when + // running under Miri. + #[allow(clippy::unit_cmp)] + { + assert_eq!(<()>::new_zeroed(), ()); + } + } + + #[test] + fn test_transparent_packed_generic_struct() { + #[derive(IntoBytes, FromBytes, Unaligned)] + #[repr(transparent)] + #[allow(dead_code)] // We never construct this type + struct Foo<T> { + _t: T, + _phantom: PhantomData<()>, + } + + assert_impl_all!(Foo<u32>: FromZeros, FromBytes, IntoBytes); + assert_impl_all!(Foo<u8>: Unaligned); + + #[derive(IntoBytes, FromBytes, Unaligned)] + #[repr(C, packed)] + #[allow(dead_code)] // We never construct this type + struct Bar<T, U> { + _t: T, + _u: U, + } + + assert_impl_all!(Bar<u8, AU64>: FromZeros, FromBytes, IntoBytes, Unaligned); + } + + #[cfg(feature = "alloc")] + mod alloc { + use super::*; + + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[test] + fn test_extend_vec_zeroed() { + // Test extending when there is an existing allocation. + let mut v = vec![100u16, 200, 300]; + FromZeros::extend_vec_zeroed(&mut v, 3).unwrap(); + assert_eq!(v.len(), 6); + assert_eq!(&*v, &[100, 200, 300, 0, 0, 0]); + drop(v); + + // Test extending when there is no existing allocation. + let mut v: Vec<u64> = Vec::new(); + FromZeros::extend_vec_zeroed(&mut v, 3).unwrap(); + assert_eq!(v.len(), 3); + assert_eq!(&*v, &[0, 0, 0]); + drop(v); + } + + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[test] + fn test_extend_vec_zeroed_zst() { + // Test extending when there is an existing (fake) allocation. + let mut v = vec![(), (), ()]; + <()>::extend_vec_zeroed(&mut v, 3).unwrap(); + assert_eq!(v.len(), 6); + assert_eq!(&*v, &[(), (), (), (), (), ()]); + drop(v); + + // Test extending when there is no existing (fake) allocation. + let mut v: Vec<()> = Vec::new(); + <()>::extend_vec_zeroed(&mut v, 3).unwrap(); + assert_eq!(&*v, &[(), (), ()]); + drop(v); + } + + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[test] + fn test_insert_vec_zeroed() { + // Insert at start (no existing allocation). + let mut v: Vec<u64> = Vec::new(); + u64::insert_vec_zeroed(&mut v, 0, 2).unwrap(); + assert_eq!(v.len(), 2); + assert_eq!(&*v, &[0, 0]); + drop(v); + + // Insert at start. + let mut v = vec![100u64, 200, 300]; + u64::insert_vec_zeroed(&mut v, 0, 2).unwrap(); + assert_eq!(v.len(), 5); + assert_eq!(&*v, &[0, 0, 100, 200, 300]); + drop(v); + + // Insert at middle. + let mut v = vec![100u64, 200, 300]; + u64::insert_vec_zeroed(&mut v, 1, 1).unwrap(); + assert_eq!(v.len(), 4); + assert_eq!(&*v, &[100, 0, 200, 300]); + drop(v); + + // Insert at end. + let mut v = vec![100u64, 200, 300]; + u64::insert_vec_zeroed(&mut v, 3, 1).unwrap(); + assert_eq!(v.len(), 4); + assert_eq!(&*v, &[100, 200, 300, 0]); + drop(v); + } + + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[test] + fn test_insert_vec_zeroed_zst() { + // Insert at start (no existing fake allocation). + let mut v: Vec<()> = Vec::new(); + <()>::insert_vec_zeroed(&mut v, 0, 2).unwrap(); + assert_eq!(v.len(), 2); + assert_eq!(&*v, &[(), ()]); + drop(v); + + // Insert at start. + let mut v = vec![(), (), ()]; + <()>::insert_vec_zeroed(&mut v, 0, 2).unwrap(); + assert_eq!(v.len(), 5); + assert_eq!(&*v, &[(), (), (), (), ()]); + drop(v); + + // Insert at middle. + let mut v = vec![(), (), ()]; + <()>::insert_vec_zeroed(&mut v, 1, 1).unwrap(); + assert_eq!(v.len(), 4); + assert_eq!(&*v, &[(), (), (), ()]); + drop(v); + + // Insert at end. + let mut v = vec![(), (), ()]; + <()>::insert_vec_zeroed(&mut v, 3, 1).unwrap(); + assert_eq!(v.len(), 4); + assert_eq!(&*v, &[(), (), (), ()]); + drop(v); + } + + #[test] + fn test_new_box_zeroed() { + assert_eq!(u64::new_box_zeroed(), Ok(Box::new(0))); + } + + #[test] + fn test_new_box_zeroed_array() { + drop(<[u32; 0x1000]>::new_box_zeroed()); + } + + #[test] + fn test_new_box_zeroed_zst() { + // This test exists in order to exercise unsafe code, especially + // when running under Miri. + #[allow(clippy::unit_cmp)] + { + assert_eq!(<()>::new_box_zeroed(), Ok(Box::new(()))); + } + } + + #[test] + fn test_new_box_zeroed_with_elems() { + let mut s: Box<[u64]> = <[u64]>::new_box_zeroed_with_elems(3).unwrap(); + assert_eq!(s.len(), 3); + assert_eq!(&*s, &[0, 0, 0]); + s[1] = 3; + assert_eq!(&*s, &[0, 3, 0]); + } + + #[test] + fn test_new_box_zeroed_with_elems_empty() { + let s: Box<[u64]> = <[u64]>::new_box_zeroed_with_elems(0).unwrap(); + assert_eq!(s.len(), 0); + } + + #[test] + fn test_new_box_zeroed_with_elems_zst() { + let mut s: Box<[()]> = <[()]>::new_box_zeroed_with_elems(3).unwrap(); + assert_eq!(s.len(), 3); + assert!(s.get(10).is_none()); + // This test exists in order to exercise unsafe code, especially + // when running under Miri. + #[allow(clippy::unit_cmp)] + { + assert_eq!(s[1], ()); + } + s[2] = (); + } + + #[test] + fn test_new_box_zeroed_with_elems_zst_empty() { + let s: Box<[()]> = <[()]>::new_box_zeroed_with_elems(0).unwrap(); + assert_eq!(s.len(), 0); + } + + #[test] + fn new_box_zeroed_with_elems_errors() { + assert_eq!(<[u16]>::new_box_zeroed_with_elems(usize::MAX), Err(AllocError)); + + let max = <usize as core::convert::TryFrom<_>>::try_from(isize::MAX).unwrap(); + assert_eq!( + <[u16]>::new_box_zeroed_with_elems((max / mem::size_of::<u16>()) + 1), + Err(AllocError) + ); + } + } + + #[test] + #[allow(deprecated)] + fn test_deprecated_from_bytes() { + let val = 0u32; + let bytes = val.as_bytes(); + + assert!(u32::ref_from(bytes).is_some()); + // mut_from needs mut bytes + let mut val = 0u32; + let mut_bytes = val.as_mut_bytes(); + assert!(u32::mut_from(mut_bytes).is_some()); + + assert!(u32::read_from(bytes).is_some()); + + let (slc, rest) = <u32>::slice_from_prefix(bytes, 0).unwrap(); + assert!(slc.is_empty()); + assert_eq!(rest.len(), 4); + + let (rest, slc) = <u32>::slice_from_suffix(bytes, 0).unwrap(); + assert!(slc.is_empty()); + assert_eq!(rest.len(), 4); + + let (slc, rest) = <u32>::mut_slice_from_prefix(mut_bytes, 0).unwrap(); + assert!(slc.is_empty()); + assert_eq!(rest.len(), 4); + + let (rest, slc) = <u32>::mut_slice_from_suffix(mut_bytes, 0).unwrap(); + assert!(slc.is_empty()); + assert_eq!(rest.len(), 4); + } + + #[test] + fn test_try_ref_from_prefix_suffix() { + use crate::util::testutil::Align; + let bytes = &Align::<[u8; 4], u32>::new([0u8; 4]).t[..]; + let (r, rest): (&u32, &[u8]) = u32::try_ref_from_prefix(bytes).unwrap(); + assert_eq!(*r, 0); + assert_eq!(rest.len(), 0); + + let (rest, r): (&[u8], &u32) = u32::try_ref_from_suffix(bytes).unwrap(); + assert_eq!(*r, 0); + assert_eq!(rest.len(), 0); + } + + #[test] + fn test_raw_dangling() { + use crate::util::AsAddress; + let ptr: NonNull<u32> = u32::raw_dangling(); + assert_eq!(AsAddress::addr(ptr), 1); + + let ptr: NonNull<[u32]> = <[u32]>::raw_dangling(); + assert_eq!(AsAddress::addr(ptr), 1); + } + + #[test] + fn test_try_ref_from_prefix_with_elems() { + use crate::util::testutil::Align; + let bytes = &Align::<[u8; 8], u32>::new([0u8; 8]).t[..]; + let (r, rest): (&[u32], &[u8]) = <[u32]>::try_ref_from_prefix_with_elems(bytes, 2).unwrap(); + assert_eq!(r.len(), 2); + assert_eq!(rest.len(), 0); + } + + #[test] + fn test_try_ref_from_suffix_with_elems() { + use crate::util::testutil::Align; + let bytes = &Align::<[u8; 8], u32>::new([0u8; 8]).t[..]; + let (rest, r): (&[u8], &[u32]) = <[u32]>::try_ref_from_suffix_with_elems(bytes, 2).unwrap(); + assert_eq!(r.len(), 2); + assert_eq!(rest.len(), 0); + } +} diff --git a/rust/zerocopy/src/macros.rs b/rust/zerocopy/src/macros.rs new file mode 100644 index 000000000000..b801d86a8fa6 --- /dev/null +++ b/rust/zerocopy/src/macros.rs @@ -0,0 +1,1825 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2024 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License <LICENSE-BSD or +// https://opensource.org/license/bsd-2-clause>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +/// Safely transmutes a value of one type to a value of another type of the same +/// size. +/// +/// This macro behaves like an invocation of this function: +/// +/// ```ignore +/// const fn transmute<Src, Dst>(src: Src) -> Dst +/// where +/// Src: IntoBytes, +/// Dst: FromBytes, +/// size_of::<Src>() == size_of::<Dst>(), +/// { +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// However, unlike a function, this macro can only be invoked when the types of +/// `Src` and `Dst` are completely concrete. The types `Src` and `Dst` are +/// inferred from the calling context; they cannot be explicitly specified in +/// the macro invocation. +/// +/// Note that the `Src` produced by the expression `$e` will *not* be dropped. +/// Semantically, its bits will be copied into a new value of type `Dst`, the +/// original `Src` will be forgotten, and the value of type `Dst` will be +/// returned. +/// +/// # `#![allow(shrink)]` +/// +/// If `#![allow(shrink)]` is provided, `transmute!` additionally supports +/// transmutations that shrink the size of the value; e.g.: +/// +/// ``` +/// # use zerocopy::transmute; +/// let u: u32 = transmute!(#![allow(shrink)] 0u64); +/// assert_eq!(u, 0u32); +/// ``` +/// +/// # Examples +/// +/// ``` +/// # use zerocopy::transmute; +/// let one_dimensional: [u8; 8] = [0, 1, 2, 3, 4, 5, 6, 7]; +/// +/// let two_dimensional: [[u8; 4]; 2] = transmute!(one_dimensional); +/// +/// assert_eq!(two_dimensional, [[0, 1, 2, 3], [4, 5, 6, 7]]); +/// ``` +/// +/// # Use in `const` contexts +/// +/// This macro can be invoked in `const` contexts. +/// +#[doc = codegen_section!( + header = "h2", + bench = "transmute", + format = "coco_static_size", +)] +#[macro_export] +macro_rules! transmute { + // NOTE: This must be a macro (rather than a function with trait bounds) + // because there's no way, in a generic context, to enforce that two types + // have the same size. `core::mem::transmute` uses compiler magic to enforce + // this so long as the types are concrete. + (#![allow(shrink)] $e:expr) => {{ + let mut e = $e; + if false { + // This branch, though never taken, ensures that the type of `e` is + // `IntoBytes` and that the type of the outer macro invocation + // expression is `FromBytes`. + + fn transmute<Src, Dst>(src: Src) -> Dst + where + Src: $crate::IntoBytes, + Dst: $crate::FromBytes, + { + let _ = src; + loop {} + } + loop {} + #[allow(unreachable_code)] + transmute(e) + } else { + use $crate::util::macro_util::core_reexport::mem::ManuallyDrop; + + // NOTE: `repr(packed)` is important! It ensures that the size of + // `Transmute` won't be rounded up to accommodate `Src`'s or `Dst`'s + // alignment, which would break the size comparison logic below. + // + // As an example of why this is problematic, consider `Src = [u8; + // 5]`, `Dst = u32`. The total size of `Transmute<Src, Dst>` would + // be 8, and so we would reject a `[u8; 5]` to `u32` transmute as + // being size-increasing, which it isn't. + #[repr(C, packed)] + union Transmute<Src, Dst> { + src: ManuallyDrop<Src>, + dst: ManuallyDrop<Dst>, + } + + // SAFETY: `Transmute` is a `repr(C)` union whose `src` field has + // type `ManuallyDrop<Src>`. Thus, the `src` field starts at byte + // offset 0 within `Transmute` [1]. `ManuallyDrop<T>` has the same + // layout and bit validity as `T`, so it is sound to transmute `Src` + // to `Transmute`. + // + // [1] https://doc.rust-lang.org/1.85.0/reference/type-layout.html#reprc-unions + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/mem/struct.ManuallyDrop.html: + // + // `ManuallyDrop<T>` is guaranteed to have the same layout and bit + // validity as `T` + let u: Transmute<_, _> = unsafe { + // Clippy: We can't annotate the types; this macro is designed + // to infer the types from the calling context. + #[allow(clippy::missing_transmute_annotations)] + $crate::util::macro_util::core_reexport::mem::transmute(e) + }; + + if false { + // SAFETY: This code is never executed. + e = ManuallyDrop::into_inner(unsafe { u.src }); + // Suppress the `unused_assignments` lint on the previous line. + let _ = e; + loop {} + } else { + // SAFETY: Per the safety comment on `let u` above, the `dst` + // field in `Transmute` starts at byte offset 0, and has the + // same layout and bit validity as `Dst`. + // + // Transmuting `Src` to `Transmute<Src, Dst>` above using + // `core::mem::transmute` ensures that `size_of::<Src>() == + // size_of::<Transmute<Src, Dst>>()`. A `#[repr(C, packed)]` + // union has the maximum size of all of its fields [1], so this + // is equivalent to `size_of::<Src>() >= size_of::<Dst>()`. + // + // The outer `if`'s `false` branch ensures that `Src: IntoBytes` + // and `Dst: FromBytes`. This, combined with the size bound, + // ensures that this transmute is sound. + // + // [1] Per https://doc.rust-lang.org/1.85.0/reference/type-layout.html#reprc-unions: + // + // The union will have a size of the maximum size of all of + // its fields rounded to its alignment + let dst = unsafe { u.dst }; + $crate::util::macro_util::must_use(ManuallyDrop::into_inner(dst)) + } + } + }}; + ($e:expr) => {{ + let e = $e; + if false { + // This branch, though never taken, ensures that the type of `e` is + // `IntoBytes` and that the type of the outer macro invocation + // expression is `FromBytes`. + + fn transmute<Src, Dst>(src: Src) -> Dst + where + Src: $crate::IntoBytes, + Dst: $crate::FromBytes, + { + let _ = src; + loop {} + } + loop {} + #[allow(unreachable_code)] + transmute(e) + } else { + // SAFETY: `core::mem::transmute` ensures that the type of `e` and + // the type of this macro invocation expression have the same size. + // We know this transmute is safe thanks to the `IntoBytes` and + // `FromBytes` bounds enforced by the `false` branch. + let u = unsafe { + // Clippy: We can't annotate the types; this macro is designed + // to infer the types from the calling context. + #[allow(clippy::missing_transmute_annotations, unnecessary_transmutes)] + $crate::util::macro_util::core_reexport::mem::transmute(e) + }; + $crate::util::macro_util::must_use(u) + } + }}; +} + +/// Safely transmutes a mutable or immutable reference of one type to an +/// immutable reference of another type of the same size and compatible +/// alignment. +/// +/// This macro behaves like an invocation of this function: +/// +/// ```ignore +/// fn transmute_ref<'src, 'dst, Src, Dst>(src: &'src Src) -> &'dst Dst +/// where +/// 'src: 'dst, +/// Src: IntoBytes + Immutable + ?Sized, +/// Dst: FromBytes + Immutable + ?Sized, +/// align_of::<Src>() >= align_of::<Dst>(), +/// size_compatible::<Src, Dst>(), +/// { +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// The types `Src` and `Dst` are inferred from the calling context; they cannot +/// be explicitly specified in the macro invocation. +/// +/// # Size compatibility +/// +/// `transmute_ref!` supports transmuting between `Sized` types, between unsized +/// (i.e., `?Sized`) types, and from a `Sized` type to an unsized type. It +/// supports any transmutation that preserves the number of bytes of the +/// referent, even if doing so requires updating the metadata stored in an +/// unsized "fat" reference: +/// +/// ``` +/// # use zerocopy::transmute_ref; +/// # use core::mem::size_of_val; // Not in the prelude on our MSRV +/// let src: &[[u8; 2]] = &[[0, 1], [2, 3]][..]; +/// let dst: &[u8] = transmute_ref!(src); +/// +/// assert_eq!(src.len(), 2); +/// assert_eq!(dst.len(), 4); +/// assert_eq!(dst, [0, 1, 2, 3]); +/// assert_eq!(size_of_val(src), size_of_val(dst)); +/// ``` +/// +/// # Errors +/// +/// Violations of the alignment and size compatibility checks are detected +/// *after* the compiler performs monomorphization. This has two important +/// consequences. +/// +/// First, it means that generic code will *never* fail these conditions: +/// +/// ``` +/// # use zerocopy::{transmute_ref, FromBytes, IntoBytes, Immutable}; +/// fn transmute_ref<Src, Dst>(src: &Src) -> &Dst +/// where +/// Src: IntoBytes + Immutable, +/// Dst: FromBytes + Immutable, +/// { +/// transmute_ref!(src) +/// } +/// ``` +/// +/// Instead, failures will only be detected once generic code is instantiated +/// with concrete types: +/// +/// ```compile_fail,E0080 +/// # use zerocopy::{transmute_ref, FromBytes, IntoBytes, Immutable}; +/// # +/// # fn transmute_ref<Src, Dst>(src: &Src) -> &Dst +/// # where +/// # Src: IntoBytes + Immutable, +/// # Dst: FromBytes + Immutable, +/// # { +/// # transmute_ref!(src) +/// # } +/// let src: &u16 = &0; +/// let dst: &u8 = transmute_ref(src); +/// ``` +/// +/// Second, the fact that violations are detected after monomorphization means +/// that `cargo check` will usually not detect errors, even when types are +/// concrete. Instead, `cargo build` must be used to detect such errors. +/// +/// # Examples +/// +/// Transmuting between `Sized` types: +/// +/// ``` +/// # use zerocopy::transmute_ref; +/// let one_dimensional: [u8; 8] = [0, 1, 2, 3, 4, 5, 6, 7]; +/// +/// let two_dimensional: &[[u8; 4]; 2] = transmute_ref!(&one_dimensional); +/// +/// assert_eq!(two_dimensional, &[[0, 1, 2, 3], [4, 5, 6, 7]]); +/// ``` +/// +/// Transmuting between unsized types: +/// +/// ``` +/// # use {zerocopy::*, zerocopy_derive::*}; +/// # type u16 = zerocopy::byteorder::native_endian::U16; +/// # type u32 = zerocopy::byteorder::native_endian::U32; +/// #[derive(KnownLayout, FromBytes, IntoBytes, Immutable)] +/// #[repr(C)] +/// struct SliceDst<T, U> { +/// t: T, +/// u: [U], +/// } +/// +/// type Src = SliceDst<u32, u16>; +/// type Dst = SliceDst<u16, u8>; +/// +/// let src = Src::ref_from_bytes(&[0, 1, 2, 3, 4, 5, 6, 7]).unwrap(); +/// let dst: &Dst = transmute_ref!(src); +/// +/// assert_eq!(src.t.as_bytes(), [0, 1, 2, 3]); +/// assert_eq!(src.u.len(), 2); +/// assert_eq!(src.u.as_bytes(), [4, 5, 6, 7]); +/// +/// assert_eq!(dst.t.as_bytes(), [0, 1]); +/// assert_eq!(dst.u, [2, 3, 4, 5, 6, 7]); +/// ``` +/// +/// # Use in `const` contexts +/// +/// This macro can be invoked in `const` contexts only when `Src: Sized` and +/// `Dst: Sized`. +/// +#[doc = codegen_section!( + header = "h2", + bench = "transmute_ref", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] +)] +#[macro_export] +macro_rules! transmute_ref { + ($e:expr) => {{ + // NOTE: This must be a macro (rather than a function with trait bounds) + // because there's no way, in a generic context, to enforce that two + // types have the same size or alignment. + + // Ensure that the source type is a reference or a mutable reference + // (note that mutable references are implicitly reborrowed here). + let e: &_ = $e; + + #[allow(unused, clippy::diverging_sub_expression)] + if false { + // This branch, though never taken, ensures that the type of `e` is + // `&T` where `T: IntoBytes + Immutable`, and that the type of this + // macro expression is `&U` where `U: FromBytes + Immutable`. + + struct AssertSrcIsIntoBytes<'a, T: ?::core::marker::Sized + $crate::IntoBytes>(&'a T); + struct AssertSrcIsImmutable<'a, T: ?::core::marker::Sized + $crate::Immutable>(&'a T); + struct AssertDstIsFromBytes<'a, U: ?::core::marker::Sized + $crate::FromBytes>(&'a U); + struct AssertDstIsImmutable<'a, T: ?::core::marker::Sized + $crate::Immutable>(&'a T); + + let _ = AssertSrcIsIntoBytes(e); + let _ = AssertSrcIsImmutable(e); + + if true { + #[allow(unused, unreachable_code)] + let u = AssertDstIsFromBytes(loop {}); + u.0 + } else { + #[allow(unused, unreachable_code)] + let u = AssertDstIsImmutable(loop {}); + u.0 + } + } else { + use $crate::util::macro_util::TransmuteRefDst; + let t = $crate::util::macro_util::Wrap::new(e); + + if false { + // This branch exists solely to force the compiler to infer the + // type of `Dst` *before* it attempts to resolve the method call + // to `transmute_ref` in the `else` branch. + // + // Without this, if `Src` is `Sized` but `Dst` is `!Sized`, the + // compiler will eagerly select the inherent impl of + // `transmute_ref` (which requires `Dst: Sized`) because inherent + // methods take priority over trait methods. It does this before + // it realizes `Dst` is `!Sized`, leading to a compile error when + // it checks the bounds later. + // + // By calling this helper (which returns `&Dst`), we force `Dst` + // to be fully resolved. By the time it gets to the `else` + // branch, the compiler knows `Dst` is `!Sized`, properly + // disqualifies the inherent method, and falls back to the trait + // implementation. + t.transmute_ref_inference_helper() + } else { + // SAFETY: The outer `if false` branch ensures that: + // - `Src: IntoBytes + Immutable` + // - `Dst: FromBytes + Immutable` + unsafe { + t.transmute_ref() + } + } + } + }} +} + +/// Safely transmutes a mutable reference of one type to a mutable reference of +/// another type of the same size and compatible alignment. +/// +/// This macro behaves like an invocation of this function: +/// +/// ```ignore +/// const fn transmute_mut<'src, 'dst, Src, Dst>(src: &'src mut Src) -> &'dst mut Dst +/// where +/// 'src: 'dst, +/// Src: FromBytes + IntoBytes + ?Sized, +/// Dst: FromBytes + IntoBytes + ?Sized, +/// align_of::<Src>() >= align_of::<Dst>(), +/// size_compatible::<Src, Dst>(), +/// { +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// The types `Src` and `Dst` are inferred from the calling context; they cannot +/// be explicitly specified in the macro invocation. +/// +/// # Size compatibility +/// +/// `transmute_mut!` supports transmuting between `Sized` types, between unsized +/// (i.e., `?Sized`) types, and from a `Sized` type to an unsized type. It +/// supports any transmutation that preserves the number of bytes of the +/// referent, even if doing so requires updating the metadata stored in an +/// unsized "fat" reference: +/// +/// ``` +/// # use zerocopy::transmute_mut; +/// # use core::mem::size_of_val; // Not in the prelude on our MSRV +/// let src: &mut [[u8; 2]] = &mut [[0, 1], [2, 3]][..]; +/// let dst: &mut [u8] = transmute_mut!(src); +/// +/// assert_eq!(dst.len(), 4); +/// assert_eq!(dst, [0, 1, 2, 3]); +/// let dst_size = size_of_val(dst); +/// assert_eq!(src.len(), 2); +/// assert_eq!(size_of_val(src), dst_size); +/// ``` +/// +/// # Errors +/// +/// Violations of the alignment and size compatibility checks are detected +/// *after* the compiler performs monomorphization. This has two important +/// consequences. +/// +/// First, it means that generic code will *never* fail these conditions: +/// +/// ``` +/// # use zerocopy::{transmute_mut, FromBytes, IntoBytes, Immutable}; +/// fn transmute_mut<Src, Dst>(src: &mut Src) -> &mut Dst +/// where +/// Src: FromBytes + IntoBytes, +/// Dst: FromBytes + IntoBytes, +/// { +/// transmute_mut!(src) +/// } +/// ``` +/// +/// Instead, failures will only be detected once generic code is instantiated +/// with concrete types: +/// +/// ```compile_fail,E0080 +/// # use zerocopy::{transmute_mut, FromBytes, IntoBytes, Immutable}; +/// # +/// # fn transmute_mut<Src, Dst>(src: &mut Src) -> &mut Dst +/// # where +/// # Src: FromBytes + IntoBytes, +/// # Dst: FromBytes + IntoBytes, +/// # { +/// # transmute_mut!(src) +/// # } +/// let src: &mut u16 = &mut 0; +/// let dst: &mut u8 = transmute_mut(src); +/// ``` +/// +/// Second, the fact that violations are detected after monomorphization means +/// that `cargo check` will usually not detect errors, even when types are +/// concrete. Instead, `cargo build` must be used to detect such errors. +/// +/// +/// # Examples +/// +/// Transmuting between `Sized` types: +/// +/// ``` +/// # use zerocopy::transmute_mut; +/// let mut one_dimensional: [u8; 8] = [0, 1, 2, 3, 4, 5, 6, 7]; +/// +/// let two_dimensional: &mut [[u8; 4]; 2] = transmute_mut!(&mut one_dimensional); +/// +/// assert_eq!(two_dimensional, &[[0, 1, 2, 3], [4, 5, 6, 7]]); +/// +/// two_dimensional.reverse(); +/// +/// assert_eq!(one_dimensional, [4, 5, 6, 7, 0, 1, 2, 3]); +/// ``` +/// +/// Transmuting between unsized types: +/// +/// ``` +/// # use {zerocopy::*, zerocopy_derive::*}; +/// # type u16 = zerocopy::byteorder::native_endian::U16; +/// # type u32 = zerocopy::byteorder::native_endian::U32; +/// #[derive(KnownLayout, FromBytes, IntoBytes, Immutable)] +/// #[repr(C)] +/// struct SliceDst<T, U> { +/// t: T, +/// u: [U], +/// } +/// +/// type Src = SliceDst<u32, u16>; +/// type Dst = SliceDst<u16, u8>; +/// +/// let mut bytes = [0, 1, 2, 3, 4, 5, 6, 7]; +/// let src = Src::mut_from_bytes(&mut bytes[..]).unwrap(); +/// let dst: &mut Dst = transmute_mut!(src); +/// +/// assert_eq!(dst.t.as_bytes(), [0, 1]); +/// assert_eq!(dst.u, [2, 3, 4, 5, 6, 7]); +/// +/// assert_eq!(src.t.as_bytes(), [0, 1, 2, 3]); +/// assert_eq!(src.u.len(), 2); +/// assert_eq!(src.u.as_bytes(), [4, 5, 6, 7]); +/// ``` +#[macro_export] +macro_rules! transmute_mut { + ($e:expr) => {{ + // NOTE: This must be a macro (rather than a function with trait bounds) + // because, for backwards-compatibility on v0.8.x, we use the autoref + // specialization trick to dispatch to different `transmute_mut` + // implementations: one which doesn't require `Src: KnownLayout + Dst: + // KnownLayout` when `Src: Sized + Dst: Sized`, and one which requires + // `KnownLayout` bounds otherwise. + + // Ensure that the source type is a mutable reference. + let e: &mut _ = $e; + + #[allow(unused)] + use $crate::util::macro_util::TransmuteMutDst as _; + let t = $crate::util::macro_util::Wrap::new(e); + if false { + // This branch exists solely to force the compiler to infer the type + // of `Dst` *before* it attempts to resolve the method call to + // `transmute_mut` in the `else` branch. + // + // Without this, if `Src` is `Sized` but `Dst` is `!Sized`, the + // compiler will eagerly select the inherent impl of `transmute_mut` + // (which requires `Dst: Sized`) because inherent methods take + // priority over trait methods. It does this before it realizes + // `Dst` is `!Sized`, leading to a compile error when it checks the + // bounds later. + // + // By calling this helper (which returns `&mut Dst`), we force `Dst` + // to be fully resolved. By the time it gets to the `else` branch, + // the compiler knows `Dst` is `!Sized`, properly disqualifies the + // inherent method, and falls back to the trait implementation. + t.transmute_mut_inference_helper() + } else { + t.transmute_mut() + } + }} +} + +/// Conditionally transmutes a value of one type to a value of another type of +/// the same size. +/// +/// This macro behaves like an invocation of this function: +/// +/// ```ignore +/// fn try_transmute<Src, Dst>(src: Src) -> Result<Dst, ValidityError<Src, Dst>> +/// where +/// Src: IntoBytes, +/// Dst: TryFromBytes, +/// size_of::<Src>() == size_of::<Dst>(), +/// { +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// However, unlike a function, this macro can only be invoked when the types of +/// `Src` and `Dst` are completely concrete. The types `Src` and `Dst` are +/// inferred from the calling context; they cannot be explicitly specified in +/// the macro invocation. +/// +/// Note that the `Src` produced by the expression `$e` will *not* be dropped. +/// Semantically, its bits will be copied into a new value of type `Dst`, the +/// original `Src` will be forgotten, and the value of type `Dst` will be +/// returned. +/// +/// # Examples +/// +/// ``` +/// # use zerocopy::*; +/// // 0u8 → bool = false +/// assert_eq!(try_transmute!(0u8), Ok(false)); +/// +/// // 1u8 → bool = true +/// assert_eq!(try_transmute!(1u8), Ok(true)); +/// +/// // 2u8 → bool = error +/// assert!(matches!( +/// try_transmute!(2u8), +/// Result::<bool, _>::Err(ValidityError { .. }) +/// )); +/// ``` +/// +#[doc = codegen_section!( + header = "h2", + bench = "try_transmute", + format = "coco_static_size", +)] +#[macro_export] +macro_rules! try_transmute { + ($e:expr) => {{ + // NOTE: This must be a macro (rather than a function with trait bounds) + // because there's no way, in a generic context, to enforce that two + // types have the same size. `core::mem::transmute` uses compiler magic + // to enforce this so long as the types are concrete. + + let e = $e; + if false { + // Check that the sizes of the source and destination types are + // equal. + + // SAFETY: This code is never executed. + Ok(unsafe { + // Clippy: We can't annotate the types; this macro is designed + // to infer the types from the calling context. + #[allow(clippy::missing_transmute_annotations)] + $crate::util::macro_util::core_reexport::mem::transmute(e) + }) + } else { + $crate::util::macro_util::try_transmute::<_, _>(e) + } + }} +} + +/// Conditionally transmutes a mutable or immutable reference of one type to an +/// immutable reference of another type of the same size and compatible +/// alignment. +/// +/// *Note that while the **value** of the referent is checked for validity at +/// runtime, the **size** and **alignment** are checked at compile time. For +/// conversions which are fallible with respect to size and alignment, see the +/// methods on [`TryFromBytes`].* +/// +/// This macro behaves like an invocation of this function: +/// +/// ```ignore +/// fn try_transmute_ref<Src, Dst>(src: &Src) -> Result<&Dst, ValidityError<&Src, Dst>> +/// where +/// Src: IntoBytes + Immutable + ?Sized, +/// Dst: TryFromBytes + Immutable + ?Sized, +/// align_of::<Src>() >= align_of::<Dst>(), +/// size_compatible::<Src, Dst>(), +/// { +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// The types `Src` and `Dst` are inferred from the calling context; they cannot +/// be explicitly specified in the macro invocation. +/// +/// [`TryFromBytes`]: crate::TryFromBytes +/// +/// # Size compatibility +/// +/// `try_transmute_ref!` supports transmuting between `Sized` types, between +/// unsized (i.e., `?Sized`) types, and from a `Sized` type to an unsized type. +/// It supports any transmutation that preserves the number of bytes of the +/// referent, even if doing so requires updating the metadata stored in an +/// unsized "fat" reference: +/// +/// ``` +/// # use zerocopy::try_transmute_ref; +/// # use core::mem::size_of_val; // Not in the prelude on our MSRV +/// let src: &[[u8; 2]] = &[[0, 1], [2, 3]][..]; +/// let dst: &[u8] = try_transmute_ref!(src).unwrap(); +/// +/// assert_eq!(src.len(), 2); +/// assert_eq!(dst.len(), 4); +/// assert_eq!(dst, [0, 1, 2, 3]); +/// assert_eq!(size_of_val(src), size_of_val(dst)); +/// ``` +/// +/// # Examples +/// +/// Transmuting between `Sized` types: +/// +/// ``` +/// # use zerocopy::*; +/// // 0u8 → bool = false +/// assert_eq!(try_transmute_ref!(&0u8), Ok(&false)); +/// +/// // 1u8 → bool = true +/// assert_eq!(try_transmute_ref!(&1u8), Ok(&true)); +/// +/// // 2u8 → bool = error +/// assert!(matches!( +/// try_transmute_ref!(&2u8), +/// Result::<&bool, _>::Err(ValidityError { .. }) +/// )); +/// ``` +/// +/// Transmuting between unsized types: +/// +/// ``` +/// # use {zerocopy::*, zerocopy_derive::*}; +/// # type u16 = zerocopy::byteorder::native_endian::U16; +/// # type u32 = zerocopy::byteorder::native_endian::U32; +/// #[derive(KnownLayout, FromBytes, IntoBytes, Immutable)] +/// #[repr(C)] +/// struct SliceDst<T, U> { +/// t: T, +/// u: [U], +/// } +/// +/// type Src = SliceDst<u32, u16>; +/// type Dst = SliceDst<u16, bool>; +/// +/// let src = Src::ref_from_bytes(&[0, 1, 0, 1, 0, 1, 0, 1]).unwrap(); +/// let dst: &Dst = try_transmute_ref!(src).unwrap(); +/// +/// assert_eq!(src.t.as_bytes(), [0, 1, 0, 1]); +/// assert_eq!(src.u.len(), 2); +/// assert_eq!(src.u.as_bytes(), [0, 1, 0, 1]); +/// +/// assert_eq!(dst.t.as_bytes(), [0, 1]); +/// assert_eq!(dst.u, [false, true, false, true, false, true]); +/// ``` +/// +#[doc = codegen_section!( + header = "h2", + bench = "try_transmute_ref", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] +)] +#[macro_export] +macro_rules! try_transmute_ref { + ($e:expr) => {{ + // Ensure that the source type is a reference or a mutable reference + // (note that mutable references are implicitly reborrowed here). + let e: &_ = $e; + + #[allow(unused_imports)] + use $crate::util::macro_util::TryTransmuteRefDst as _; + let t = $crate::util::macro_util::Wrap::new(e); + if false { + // This branch exists solely to force the compiler to infer the type + // of `Dst` *before* it attempts to resolve the method call to + // `try_transmute_ref` in the `else` branch. + // + // Without this, if `Src` is `Sized` but `Dst` is `!Sized`, the + // compiler will eagerly select the inherent impl of + // `try_transmute_ref` (which requires `Dst: Sized`) because + // inherent methods take priority over trait methods. It does this + // before it realizes `Dst` is `!Sized`, leading to a compile error + // when it checks the bounds later. + // + // By calling this helper (which returns `&Dst`), we force `Dst` + // to be fully resolved. By the time it gets to the `else` + // branch, the compiler knows `Dst` is `!Sized`, properly + // disqualifies the inherent method, and falls back to the trait + // implementation. + Ok(t.transmute_ref_inference_helper()) + } else { + t.try_transmute_ref() + } + }} +} + +/// Conditionally transmutes a mutable reference of one type to a mutable +/// reference of another type of the same size and compatible alignment. +/// +/// *Note that while the **value** of the referent is checked for validity at +/// runtime, the **size** and **alignment** are checked at compile time. For +/// conversions which are fallible with respect to size and alignment, see the +/// methods on [`TryFromBytes`].* +/// +/// This macro behaves like an invocation of this function: +/// +/// ```ignore +/// fn try_transmute_mut<Src, Dst>(src: &mut Src) -> Result<&mut Dst, ValidityError<&mut Src, Dst>> +/// where +/// Src: FromBytes + IntoBytes + ?Sized, +/// Dst: TryFromBytes + IntoBytes + ?Sized, +/// align_of::<Src>() >= align_of::<Dst>(), +/// size_compatible::<Src, Dst>(), +/// { +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// The types `Src` and `Dst` are inferred from the calling context; they cannot +/// be explicitly specified in the macro invocation. +/// +/// [`TryFromBytes`]: crate::TryFromBytes +/// +/// # Size compatibility +/// +/// `try_transmute_mut!` supports transmuting between `Sized` types, between +/// unsized (i.e., `?Sized`) types, and from a `Sized` type to an unsized type. +/// It supports any transmutation that preserves the number of bytes of the +/// referent, even if doing so requires updating the metadata stored in an +/// unsized "fat" reference: +/// +/// ``` +/// # use zerocopy::try_transmute_mut; +/// # use core::mem::size_of_val; // Not in the prelude on our MSRV +/// let src: &mut [[u8; 2]] = &mut [[0, 1], [2, 3]][..]; +/// let dst: &mut [u8] = try_transmute_mut!(src).unwrap(); +/// +/// assert_eq!(dst.len(), 4); +/// assert_eq!(dst, [0, 1, 2, 3]); +/// let dst_size = size_of_val(dst); +/// assert_eq!(src.len(), 2); +/// assert_eq!(size_of_val(src), dst_size); +/// ``` +/// +/// # Examples +/// +/// Transmuting between `Sized` types: +/// +/// ``` +/// # use zerocopy::*; +/// // 0u8 → bool = false +/// let src = &mut 0u8; +/// assert_eq!(try_transmute_mut!(src), Ok(&mut false)); +/// +/// // 1u8 → bool = true +/// let src = &mut 1u8; +/// assert_eq!(try_transmute_mut!(src), Ok(&mut true)); +/// +/// // 2u8 → bool = error +/// let src = &mut 2u8; +/// assert!(matches!( +/// try_transmute_mut!(src), +/// Result::<&mut bool, _>::Err(ValidityError { .. }) +/// )); +/// ``` +/// +/// Transmuting between unsized types: +/// +/// ``` +/// # use {zerocopy::*, zerocopy_derive::*}; +/// # type u16 = zerocopy::byteorder::native_endian::U16; +/// # type u32 = zerocopy::byteorder::native_endian::U32; +/// #[derive(KnownLayout, FromBytes, IntoBytes, Immutable)] +/// #[repr(C)] +/// struct SliceDst<T, U> { +/// t: T, +/// u: [U], +/// } +/// +/// type Src = SliceDst<u32, u16>; +/// type Dst = SliceDst<u16, bool>; +/// +/// let mut bytes = [0, 1, 0, 1, 0, 1, 0, 1]; +/// let src = Src::mut_from_bytes(&mut bytes).unwrap(); +/// +/// assert_eq!(src.t.as_bytes(), [0, 1, 0, 1]); +/// assert_eq!(src.u.len(), 2); +/// assert_eq!(src.u.as_bytes(), [0, 1, 0, 1]); +/// +/// let dst: &Dst = try_transmute_mut!(src).unwrap(); +/// +/// assert_eq!(dst.t.as_bytes(), [0, 1]); +/// assert_eq!(dst.u, [false, true, false, true, false, true]); +/// ``` +#[macro_export] +macro_rules! try_transmute_mut { + ($e:expr) => {{ + // Ensure that the source type is a mutable reference. + let e: &mut _ = $e; + + #[allow(unused_imports)] + use $crate::util::macro_util::TryTransmuteMutDst as _; + let t = $crate::util::macro_util::Wrap::new(e); + if false { + // This branch exists solely to force the compiler to infer the type + // of `Dst` *before* it attempts to resolve the method call to + // `try_transmute_mut` in the `else` branch. + // + // Without this, if `Src` is `Sized` but `Dst` is `!Sized`, the + // compiler will eagerly select the inherent impl of + // `try_transmute_mut` (which requires `Dst: Sized`) because + // inherent methods take priority over trait methods. It does this + // before it realizes `Dst` is `!Sized`, leading to a compile error + // when it checks the bounds later. + // + // By calling this helper (which returns `&Dst`), we force `Dst` + // to be fully resolved. By the time it gets to the `else` + // branch, the compiler knows `Dst` is `!Sized`, properly + // disqualifies the inherent method, and falls back to the trait + // implementation. + Ok(t.transmute_mut_inference_helper()) + } else { + t.try_transmute_mut() + } + }} +} + +/// Includes a file and safely transmutes it to a value of an arbitrary type. +/// +/// The file will be included as a byte array, `[u8; N]`, which will be +/// transmuted to another type, `T`. `T` is inferred from the calling context, +/// and must implement [`FromBytes`]. +/// +/// The file is located relative to the current file (similarly to how modules +/// are found). The provided path is interpreted in a platform-specific way at +/// compile time. So, for instance, an invocation with a Windows path containing +/// backslashes `\` would not compile correctly on Unix. +/// +/// `include_value!` is ignorant of byte order. For byte order-aware types, see +/// the [`byteorder`] module. +/// +/// [`FromBytes`]: crate::FromBytes +/// [`byteorder`]: crate::byteorder +/// +/// # Examples +/// +/// Assume there are two files in the same directory with the following +/// contents: +/// +/// File `data` (no trailing newline): +/// +/// ```text +/// abcd +/// ``` +/// +/// File `main.rs`: +/// +/// ```rust +/// use zerocopy::include_value; +/// # macro_rules! include_value { +/// # ($file:expr) => { zerocopy::include_value!(concat!("../testdata/include_value/", $file)) }; +/// # } +/// +/// fn main() { +/// let as_u32: u32 = include_value!("data"); +/// assert_eq!(as_u32, u32::from_ne_bytes([b'a', b'b', b'c', b'd'])); +/// let as_i32: i32 = include_value!("data"); +/// assert_eq!(as_i32, i32::from_ne_bytes([b'a', b'b', b'c', b'd'])); +/// } +/// ``` +/// +/// # Use in `const` contexts +/// +/// This macro can be invoked in `const` contexts. +#[doc(alias("include_bytes", "include_data", "include_type"))] +#[macro_export] +macro_rules! include_value { + ($file:expr $(,)?) => { + $crate::transmute!(*::core::include_bytes!($file)) + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! cryptocorrosion_derive_traits { + ( + #[repr($repr:ident)] + $(#[$attr:meta])* + $vis:vis struct $name:ident $(<$($tyvar:ident),*>)? + $( + ( + $($tuple_field_vis:vis $tuple_field_ty:ty),* + ); + )? + + $( + { + $($field_vis:vis $field_name:ident: $field_ty:ty,)* + } + )? + ) => { + $crate::cryptocorrosion_derive_traits!(@assert_allowed_struct_repr #[repr($repr)]); + + $(#[$attr])* + #[repr($repr)] + $vis struct $name $(<$($tyvar),*>)? + $( + ( + $($tuple_field_vis $tuple_field_ty),* + ); + )? + + $( + { + $($field_vis $field_name: $field_ty,)* + } + )? + + // SAFETY: See inline. + unsafe impl $(<$($tyvar),*>)? $crate::TryFromBytes for $name$(<$($tyvar),*>)? + where + $( + $($tuple_field_ty: $crate::FromBytes,)* + )? + + $( + $($field_ty: $crate::FromBytes,)* + )? + { + #[inline(always)] + fn is_bit_valid<A>(_: $crate::Maybe<'_, Self, A>) -> bool + where + A: $crate::invariant::Alignment, + { + // SAFETY: This macro only accepts `#[repr(C)]` and + // `#[repr(transparent)]` structs, and this `impl` block + // requires all field types to be `FromBytes`. Thus, all + // initialized byte sequences constitutes valid instances of + // `Self`. + true + } + + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` and + // `#[repr(transparent)]` structs, and this `impl` block requires all + // field types to be `FromBytes`, which is a sub-trait of `FromZeros`. + unsafe impl $(<$($tyvar),*>)? $crate::FromZeros for $name$(<$($tyvar),*>)? + where + $( + $($tuple_field_ty: $crate::FromBytes,)* + )? + + $( + $($field_ty: $crate::FromBytes,)* + )? + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` and + // `#[repr(transparent)]` structs, and this `impl` block requires all + // field types to be `FromBytes`. + unsafe impl $(<$($tyvar),*>)? $crate::FromBytes for $name$(<$($tyvar),*>)? + where + $( + $($tuple_field_ty: $crate::FromBytes,)* + )? + + $( + $($field_ty: $crate::FromBytes,)* + )? + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` and + // `#[repr(transparent)]` structs, this `impl` block requires all field + // types to be `IntoBytes`, and a padding check is used to ensures that + // there are no padding bytes. + unsafe impl $(<$($tyvar),*>)? $crate::IntoBytes for $name$(<$($tyvar),*>)? + where + $( + $($tuple_field_ty: $crate::IntoBytes,)* + )? + + $( + $($field_ty: $crate::IntoBytes,)* + )? + + (): $crate::util::macro_util::PaddingFree< + Self, + { + $crate::cryptocorrosion_derive_traits!( + @struct_padding_check #[repr($repr)] + $(($($tuple_field_ty),*))? + $({$($field_ty),*})? + ) + }, + >, + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` and + // `#[repr(transparent)]` structs, and this `impl` block requires all + // field types to be `Immutable`. + unsafe impl $(<$($tyvar),*>)? $crate::Immutable for $name$(<$($tyvar),*>)? + where + $( + $($tuple_field_ty: $crate::Immutable,)* + )? + + $( + $($field_ty: $crate::Immutable,)* + )? + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + }; + (@assert_allowed_struct_repr #[repr(transparent)]) => {}; + (@assert_allowed_struct_repr #[repr(C)]) => {}; + (@assert_allowed_struct_repr #[$_attr:meta]) => { + compile_error!("repr must be `#[repr(transparent)]` or `#[repr(C)]`"); + }; + ( + @struct_padding_check #[repr(transparent)] + $(($($tuple_field_ty:ty),*))? + $({$($field_ty:ty),*})? + ) => { + // SAFETY: `#[repr(transparent)]` structs cannot have the same layout as + // their single non-zero-sized field, and so cannot have any padding + // outside of that field. + 0 + }; + ( + @struct_padding_check #[repr(C)] + $(($($tuple_field_ty:ty),*))? + $({$($field_ty:ty),*})? + ) => { + $crate::struct_padding!( + Self, + None, + None, + [ + $($($tuple_field_ty),*)? + $($($field_ty),*)? + ] + ) + }; + ( + #[repr(C)] + $(#[$attr:meta])* + $vis:vis union $name:ident { + $( + $field_name:ident: $field_ty:ty, + )* + } + ) => { + $(#[$attr])* + #[repr(C)] + $vis union $name { + $( + $field_name: $field_ty, + )* + } + + // SAFETY: See inline. + unsafe impl $crate::TryFromBytes for $name + where + $( + $field_ty: $crate::FromBytes, + )* + { + #[inline(always)] + fn is_bit_valid<A>(_: $crate::Maybe<'_, Self, A>) -> bool + where + A: $crate::invariant::Alignment, + { + // SAFETY: This macro only accepts `#[repr(C)]` unions, and this + // `impl` block requires all field types to be `FromBytes`. + // Thus, all initialized byte sequences constitutes valid + // instances of `Self`. + true + } + + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` unions, and this `impl` + // block requires all field types to be `FromBytes`, which is a + // sub-trait of `FromZeros`. + unsafe impl $crate::FromZeros for $name + where + $( + $field_ty: $crate::FromBytes, + )* + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` unions, and this `impl` + // block requires all field types to be `FromBytes`. + unsafe impl $crate::FromBytes for $name + where + $( + $field_ty: $crate::FromBytes, + )* + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` unions, this `impl` + // block requires all field types to be `IntoBytes`, and a padding check + // is used to ensures that there are no padding bytes before or after + // any field. + unsafe impl $crate::IntoBytes for $name + where + $( + $field_ty: $crate::IntoBytes, + )* + (): $crate::util::macro_util::PaddingFree< + Self, + { + $crate::union_padding!( + Self, + None::<usize>, + None::<usize>, + [$($field_ty),*] + ) + }, + >, + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` unions, and this `impl` + // block requires all field types to be `Immutable`. + unsafe impl $crate::Immutable for $name + where + $( + $field_ty: $crate::Immutable, + )* + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + }; +} + +#[cfg(test)] +mod tests { + use crate::{ + byteorder::native_endian::{U16, U32}, + util::testutil::*, + *, + }; + + #[derive(KnownLayout, Immutable, FromBytes, IntoBytes, PartialEq, Debug)] + #[repr(C)] + struct SliceDst<T, U> { + a: T, + b: [U], + } + + #[test] + fn test_transmute() { + // Test that memory is transmuted as expected. + let array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let x: [[u8; 2]; 4] = transmute!(array_of_u8s); + assert_eq!(x, array_of_arrays); + let x: [u8; 8] = transmute!(array_of_arrays); + assert_eq!(x, array_of_u8s); + + // Test that memory is transmuted as expected when shrinking. + let x: [[u8; 2]; 3] = transmute!(#![allow(shrink)] array_of_u8s); + assert_eq!(x, [[0u8, 1], [2, 3], [4, 5]]); + + // Test that the source expression's value is forgotten rather than + // dropped. + #[derive(IntoBytes)] + #[repr(transparent)] + struct PanicOnDrop(()); + impl Drop for PanicOnDrop { + fn drop(&mut self) { + panic!("PanicOnDrop::drop"); + } + } + #[allow(clippy::let_unit_value)] + let _: () = transmute!(PanicOnDrop(())); + #[allow(clippy::let_unit_value)] + let _: () = transmute!(#![allow(shrink)] PanicOnDrop(())); + + // Test that `transmute!` is legal in a const context. + const ARRAY_OF_U8S: [u8; 8] = [0u8, 1, 2, 3, 4, 5, 6, 7]; + const ARRAY_OF_ARRAYS: [[u8; 2]; 4] = [[0, 1], [2, 3], [4, 5], [6, 7]]; + const X: [[u8; 2]; 4] = transmute!(ARRAY_OF_U8S); + assert_eq!(X, ARRAY_OF_ARRAYS); + const X_SHRINK: [[u8; 2]; 3] = transmute!(#![allow(shrink)] ARRAY_OF_U8S); + assert_eq!(X_SHRINK, [[0u8, 1], [2, 3], [4, 5]]); + + // Test that `transmute!` works with `!Immutable` types. + let x: usize = transmute!(UnsafeCell::new(1usize)); + assert_eq!(x, 1); + let x: UnsafeCell<usize> = transmute!(1usize); + assert_eq!(x.into_inner(), 1); + let x: UnsafeCell<isize> = transmute!(UnsafeCell::new(1usize)); + assert_eq!(x.into_inner(), 1); + } + + // A `Sized` type which doesn't implement `KnownLayout` (it is "not + // `KnownLayout`", or `Nkl`). + // + // This permits us to test that `transmute_ref!` and `transmute_mut!` work + // for types which are `Sized + !KnownLayout`. When we added support for + // slice DSTs in #1924, this new support relied on `KnownLayout`, but we + // need to make sure to remain backwards-compatible with code which uses + // these macros with types which are `!KnownLayout`. + #[derive(FromBytes, IntoBytes, Immutable, PartialEq, Eq, Debug)] + #[repr(transparent)] + struct Nkl<T>(T); + + #[test] + fn test_transmute_ref() { + // Test that memory is transmuted as expected. + let array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let x: &[[u8; 2]; 4] = transmute_ref!(&array_of_u8s); + assert_eq!(*x, array_of_arrays); + let x: &[u8; 8] = transmute_ref!(&array_of_arrays); + assert_eq!(*x, array_of_u8s); + + // Test that `transmute_ref!` is legal in a const context. + const ARRAY_OF_U8S: [u8; 8] = [0u8, 1, 2, 3, 4, 5, 6, 7]; + const ARRAY_OF_ARRAYS: [[u8; 2]; 4] = [[0, 1], [2, 3], [4, 5], [6, 7]]; + #[allow(clippy::redundant_static_lifetimes)] + const X: &'static [[u8; 2]; 4] = transmute_ref!(&ARRAY_OF_U8S); + assert_eq!(*X, ARRAY_OF_ARRAYS); + + // Test sized -> unsized transmutation. + let array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let slice_of_arrays = &array_of_arrays[..]; + let x: &[[u8; 2]] = transmute_ref!(&array_of_u8s); + assert_eq!(x, slice_of_arrays); + + // Before 1.61.0, we can't define the `const fn transmute_ref` function + // that we do on and after 1.61.0. + #[cfg(no_zerocopy_generic_bounds_in_const_fn_1_61_0)] + { + // Test that `transmute_ref!` supports non-`KnownLayout` `Sized` + // types. + const ARRAY_OF_NKL_U8S: Nkl<[u8; 8]> = Nkl([0u8, 1, 2, 3, 4, 5, 6, 7]); + const ARRAY_OF_NKL_ARRAYS: Nkl<[[u8; 2]; 4]> = Nkl([[0, 1], [2, 3], [4, 5], [6, 7]]); + const X_NKL: &Nkl<[[u8; 2]; 4]> = transmute_ref!(&ARRAY_OF_NKL_U8S); + assert_eq!(*X_NKL, ARRAY_OF_NKL_ARRAYS); + } + + #[cfg(not(no_zerocopy_generic_bounds_in_const_fn_1_61_0))] + { + // Call through a generic function to make sure our autoref + // specialization trick works even when types are generic. + const fn transmute_ref<T, U>(t: &T) -> &U + where + T: IntoBytes + Immutable, + U: FromBytes + Immutable, + { + transmute_ref!(t) + } + + // Test that `transmute_ref!` supports non-`KnownLayout` `Sized` + // types. + const ARRAY_OF_NKL_U8S: Nkl<[u8; 8]> = Nkl([0u8, 1, 2, 3, 4, 5, 6, 7]); + const ARRAY_OF_NKL_ARRAYS: Nkl<[[u8; 2]; 4]> = Nkl([[0, 1], [2, 3], [4, 5], [6, 7]]); + const X_NKL: &Nkl<[[u8; 2]; 4]> = transmute_ref(&ARRAY_OF_NKL_U8S); + assert_eq!(*X_NKL, ARRAY_OF_NKL_ARRAYS); + } + + // Test that `transmute_ref!` works on slice DSTs in and that memory is + // transmuted as expected. + let slice_dst_of_u8s = + SliceDst::<U16, [u8; 2]>::ref_from_bytes(&[0, 1, 2, 3, 4, 5][..]).unwrap(); + let slice_dst_of_u16s = + SliceDst::<U16, U16>::ref_from_bytes(&[0, 1, 2, 3, 4, 5][..]).unwrap(); + let x: &SliceDst<U16, U16> = transmute_ref!(slice_dst_of_u8s); + assert_eq!(x, slice_dst_of_u16s); + + let slice_dst_of_u8s = + SliceDst::<U16, u8>::ref_from_bytes(&[0, 1, 2, 3, 4, 5][..]).unwrap(); + let x: &[u8] = transmute_ref!(slice_dst_of_u8s); + assert_eq!(x, [0, 1, 2, 3, 4, 5]); + + let x: &[u8] = transmute_ref!(slice_dst_of_u16s); + assert_eq!(x, [0, 1, 2, 3, 4, 5]); + + let x: &[U16] = transmute_ref!(slice_dst_of_u16s); + let slice_of_u16s: &[U16] = <[U16]>::ref_from_bytes(&[0, 1, 2, 3, 4, 5][..]).unwrap(); + assert_eq!(x, slice_of_u16s); + + // Test that transmuting from a type with larger trailing slice offset + // and larger trailing slice element works. + let bytes = &[0, 1, 2, 3, 4, 5, 6, 7][..]; + let slice_dst_big = SliceDst::<U32, U16>::ref_from_bytes(bytes).unwrap(); + let slice_dst_small = SliceDst::<U16, u8>::ref_from_bytes(bytes).unwrap(); + let x: &SliceDst<U16, u8> = transmute_ref!(slice_dst_big); + assert_eq!(x, slice_dst_small); + + // Test that it's legal to transmute a reference while shrinking the + // lifetime (note that `X` has the lifetime `'static`). + let x: &[u8; 8] = transmute_ref!(X); + assert_eq!(*x, ARRAY_OF_U8S); + + // Test that `transmute_ref!` supports decreasing alignment. + let u = AU64(0); + let array = [0, 0, 0, 0, 0, 0, 0, 0]; + let x: &[u8; 8] = transmute_ref!(&u); + assert_eq!(*x, array); + + // Test that a mutable reference can be turned into an immutable one. + let mut x = 0u8; + #[allow(clippy::useless_transmute)] + let y: &u8 = transmute_ref!(&mut x); + assert_eq!(*y, 0); + } + + #[test] + fn test_try_transmute() { + // Test that memory is transmuted with `try_transmute` as expected. + let array_of_bools = [false, true, false, true, false, true, false, true]; + let array_of_arrays = [[0, 1], [0, 1], [0, 1], [0, 1]]; + let x: Result<[[u8; 2]; 4], _> = try_transmute!(array_of_bools); + assert_eq!(x, Ok(array_of_arrays)); + let x: Result<[bool; 8], _> = try_transmute!(array_of_arrays); + assert_eq!(x, Ok(array_of_bools)); + + // Test that `try_transmute!` works with `!Immutable` types. + let x: Result<usize, _> = try_transmute!(UnsafeCell::new(1usize)); + assert_eq!(x.unwrap(), 1); + let x: Result<UnsafeCell<usize>, _> = try_transmute!(1usize); + assert_eq!(x.unwrap().into_inner(), 1); + let x: Result<UnsafeCell<isize>, _> = try_transmute!(UnsafeCell::new(1usize)); + assert_eq!(x.unwrap().into_inner(), 1); + + #[derive(FromBytes, IntoBytes, Debug, PartialEq)] + #[repr(transparent)] + struct PanicOnDrop<T>(T); + + impl<T> Drop for PanicOnDrop<T> { + fn drop(&mut self) { + panic!("PanicOnDrop dropped"); + } + } + + // Since `try_transmute!` semantically moves its argument on failure, + // the `PanicOnDrop` is not dropped, and thus this shouldn't panic. + let x: Result<usize, _> = try_transmute!(PanicOnDrop(1usize)); + assert_eq!(x, Ok(1)); + + // Since `try_transmute!` semantically returns ownership of its argument + // on failure, the `PanicOnDrop` is returned rather than dropped, and + // thus this shouldn't panic. + let y: Result<bool, _> = try_transmute!(PanicOnDrop(2u8)); + // We have to use `map_err` instead of comparing against + // `Err(PanicOnDrop(2u8))` because the latter would create and then drop + // its `PanicOnDrop` temporary, which would cause a panic. + assert_eq!(y.as_ref().map_err(|p| &p.src.0), Err::<&bool, _>(&2u8)); + mem::forget(y); + } + + #[test] + fn test_try_transmute_ref() { + // Test that memory is transmuted with `try_transmute_ref` as expected. + let array_of_bools = &[false, true, false, true, false, true, false, true]; + let array_of_arrays = &[[0, 1], [0, 1], [0, 1], [0, 1]]; + let x: Result<&[[u8; 2]; 4], _> = try_transmute_ref!(array_of_bools); + assert_eq!(x, Ok(array_of_arrays)); + let x: Result<&[bool; 8], _> = try_transmute_ref!(array_of_arrays); + assert_eq!(x, Ok(array_of_bools)); + + // Test that it's legal to transmute a reference while shrinking the + // lifetime. + { + let x: Result<&[[u8; 2]; 4], _> = try_transmute_ref!(array_of_bools); + assert_eq!(x, Ok(array_of_arrays)); + } + + // Test that `try_transmute_ref!` supports decreasing alignment. + let u = AU64(0); + let array = [0u8, 0, 0, 0, 0, 0, 0, 0]; + let x: Result<&[u8; 8], _> = try_transmute_ref!(&u); + assert_eq!(x, Ok(&array)); + + // Test that a mutable reference can be turned into an immutable one. + let mut x = 0u8; + #[allow(clippy::useless_transmute)] + let y: Result<&u8, _> = try_transmute_ref!(&mut x); + assert_eq!(y, Ok(&0)); + + // Test that sized types work which don't implement `KnownLayout`. + let array_of_nkl_u8s = Nkl([0u8, 1, 2, 3, 4, 5, 6, 7]); + let array_of_nkl_arrays = Nkl([[0, 1], [2, 3], [4, 5], [6, 7]]); + let x: Result<&Nkl<[[u8; 2]; 4]>, _> = try_transmute_ref!(&array_of_nkl_u8s); + assert_eq!(x, Ok(&array_of_nkl_arrays)); + + // Test sized -> unsized transmutation. + let array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let slice_of_arrays = &array_of_arrays[..]; + let x: Result<&[[u8; 2]], _> = try_transmute_ref!(&array_of_u8s); + assert_eq!(x, Ok(slice_of_arrays)); + + // Test unsized -> unsized transmutation. + let slice_dst_of_u8s = + SliceDst::<U16, [u8; 2]>::ref_from_bytes(&[0, 1, 2, 3, 4, 5][..]).unwrap(); + let slice_dst_of_u16s = + SliceDst::<U16, U16>::ref_from_bytes(&[0, 1, 2, 3, 4, 5][..]).unwrap(); + let x: Result<&SliceDst<U16, U16>, _> = try_transmute_ref!(slice_dst_of_u8s); + assert_eq!(x, Ok(slice_dst_of_u16s)); + } + + #[test] + fn test_try_transmute_mut() { + // Test that memory is transmuted with `try_transmute_mut` as expected. + let array_of_u8s = &mut [0u8, 1, 0, 1, 0, 1, 0, 1]; + let array_of_arrays = &mut [[0u8, 1], [0, 1], [0, 1], [0, 1]]; + let x: Result<&mut [[u8; 2]; 4], _> = try_transmute_mut!(array_of_u8s); + assert_eq!(x, Ok(array_of_arrays)); + + let array_of_bools = &mut [false, true, false, true, false, true, false, true]; + let array_of_arrays = &mut [[0u8, 1], [0, 1], [0, 1], [0, 1]]; + let x: Result<&mut [bool; 8], _> = try_transmute_mut!(array_of_arrays); + assert_eq!(x, Ok(array_of_bools)); + + // Test that it's legal to transmute a reference while shrinking the + // lifetime. + let array_of_bools = &mut [false, true, false, true, false, true, false, true]; + let array_of_arrays = &mut [[0u8, 1], [0, 1], [0, 1], [0, 1]]; + { + let x: Result<&mut [bool; 8], _> = try_transmute_mut!(array_of_arrays); + assert_eq!(x, Ok(array_of_bools)); + } + + // Test that `try_transmute_mut!` supports decreasing alignment. + let u = &mut AU64(0); + let array = &mut [0u8, 0, 0, 0, 0, 0, 0, 0]; + let x: Result<&mut [u8; 8], _> = try_transmute_mut!(u); + assert_eq!(x, Ok(array)); + + // Test that a mutable reference can be turned into an immutable one. + let mut x = 0u8; + #[allow(clippy::useless_transmute)] + let y: Result<&mut u8, _> = try_transmute_mut!(&mut x); + assert_eq!(y, Ok(&mut 0)); + + // Test that sized types work which don't implement `KnownLayout`. + let mut array_of_nkl_u8s = Nkl([0u8, 1, 2, 3, 4, 5, 6, 7]); + let mut array_of_nkl_arrays = Nkl([[0, 1], [2, 3], [4, 5], [6, 7]]); + let x: Result<&mut Nkl<[[u8; 2]; 4]>, _> = try_transmute_mut!(&mut array_of_nkl_u8s); + assert_eq!(x, Ok(&mut array_of_nkl_arrays)); + + // Test sized -> unsized transmutation. + let mut array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let mut array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let slice_of_arrays = &mut array_of_arrays[..]; + let x: Result<&mut [[u8; 2]], _> = try_transmute_mut!(&mut array_of_u8s); + assert_eq!(x, Ok(slice_of_arrays)); + + // Test unsized -> unsized transmutation. + let mut bytes = [0, 1, 2, 3, 4, 5, 6]; + let slice_dst_of_u8s = SliceDst::<u8, [u8; 2]>::mut_from_bytes(&mut bytes[..]).unwrap(); + let mut bytes = [0, 1, 2, 3, 4, 5, 6]; + let slice_dst_of_u16s = SliceDst::<u8, U16>::mut_from_bytes(&mut bytes[..]).unwrap(); + let x: Result<&mut SliceDst<u8, U16>, _> = try_transmute_mut!(slice_dst_of_u8s); + assert_eq!(x, Ok(slice_dst_of_u16s)); + } + + #[test] + fn test_transmute_mut() { + // Test that memory is transmuted as expected. + let mut array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let mut array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let x: &mut [[u8; 2]; 4] = transmute_mut!(&mut array_of_u8s); + assert_eq!(*x, array_of_arrays); + let x: &mut [u8; 8] = transmute_mut!(&mut array_of_arrays); + assert_eq!(*x, array_of_u8s); + + { + // Test that it's legal to transmute a reference while shrinking the + // lifetime. + let x: &mut [u8; 8] = transmute_mut!(&mut array_of_arrays); + assert_eq!(*x, array_of_u8s); + } + + // Test that `transmute_mut!` supports non-`KnownLayout` types. + let mut array_of_u8s = Nkl([0u8, 1, 2, 3, 4, 5, 6, 7]); + let mut array_of_arrays = Nkl([[0, 1], [2, 3], [4, 5], [6, 7]]); + let x: &mut Nkl<[[u8; 2]; 4]> = transmute_mut!(&mut array_of_u8s); + assert_eq!(*x, array_of_arrays); + let x: &mut Nkl<[u8; 8]> = transmute_mut!(&mut array_of_arrays); + assert_eq!(*x, array_of_u8s); + + // Test that `transmute_mut!` supports decreasing alignment. + let mut u = AU64(0); + let array = [0, 0, 0, 0, 0, 0, 0, 0]; + let x: &[u8; 8] = transmute_mut!(&mut u); + assert_eq!(*x, array); + + // Test that a mutable reference can be turned into an immutable one. + let mut x = 0u8; + #[allow(clippy::useless_transmute)] + let y: &u8 = transmute_mut!(&mut x); + assert_eq!(*y, 0); + + // Test that `transmute_mut!` works on slice DSTs in and that memory is + // transmuted as expected. + let mut bytes = [0, 1, 2, 3, 4, 5, 6]; + let slice_dst_of_u8s = SliceDst::<u8, [u8; 2]>::mut_from_bytes(&mut bytes[..]).unwrap(); + let mut bytes = [0, 1, 2, 3, 4, 5, 6]; + let slice_dst_of_u16s = SliceDst::<u8, U16>::mut_from_bytes(&mut bytes[..]).unwrap(); + let x: &mut SliceDst<u8, U16> = transmute_mut!(slice_dst_of_u8s); + assert_eq!(x, slice_dst_of_u16s); + + // Test that `transmute_mut!` works on slices that memory is transmuted + // as expected. + let array_of_u16s: &mut [u16] = &mut [0u16, 1, 2]; + let array_of_i16s: &mut [i16] = &mut [0i16, 1, 2]; + let x: &mut [i16] = transmute_mut!(array_of_u16s); + assert_eq!(x, array_of_i16s); + + // Test that transmuting from a type with larger trailing slice offset + // and larger trailing slice element works. + let mut bytes = [0, 1, 2, 3, 4, 5, 6, 7]; + let slice_dst_big = SliceDst::<U32, U16>::mut_from_bytes(&mut bytes[..]).unwrap(); + let mut bytes = [0, 1, 2, 3, 4, 5, 6, 7]; + let slice_dst_small = SliceDst::<U16, u8>::mut_from_bytes(&mut bytes[..]).unwrap(); + let x: &mut SliceDst<U16, u8> = transmute_mut!(slice_dst_big); + assert_eq!(x, slice_dst_small); + + // Test sized -> unsized transmutation. + let mut array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let mut array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let slice_of_arrays = &mut array_of_arrays[..]; + let x: &mut [[u8; 2]] = transmute_mut!(&mut array_of_u8s); + assert_eq!(x, slice_of_arrays); + } + + #[test] + fn test_macros_evaluate_args_once() { + let mut ctr = 0; + #[allow(clippy::useless_transmute)] + let _: usize = transmute!({ + ctr += 1; + 0usize + }); + assert_eq!(ctr, 1); + + let mut ctr = 0; + let _: &usize = transmute_ref!({ + ctr += 1; + &0usize + }); + assert_eq!(ctr, 1); + + let mut ctr: usize = 0; + let _: &mut usize = transmute_mut!({ + ctr += 1; + &mut ctr + }); + assert_eq!(ctr, 1); + + let mut ctr = 0; + #[allow(clippy::useless_transmute)] + let _: usize = try_transmute!({ + ctr += 1; + 0usize + }) + .unwrap(); + assert_eq!(ctr, 1); + } + + #[test] + fn test_include_value() { + const AS_U32: u32 = include_value!("../testdata/include_value/data"); + assert_eq!(AS_U32, u32::from_ne_bytes([b'a', b'b', b'c', b'd'])); + const AS_I32: i32 = include_value!("../testdata/include_value/data"); + assert_eq!(AS_I32, i32::from_ne_bytes([b'a', b'b', b'c', b'd'])); + } + + #[test] + #[allow(non_camel_case_types, unreachable_pub, dead_code)] + fn test_cryptocorrosion_derive_traits() { + // Test the set of invocations added in + // https://github.com/cryptocorrosion/cryptocorrosion/pull/85 + + fn assert_impls<T: FromBytes + IntoBytes + Immutable>() {} + + cryptocorrosion_derive_traits! { + #[repr(C)] + #[derive(Clone, Copy)] + pub union vec128_storage { + d: [u32; 4], + q: [u64; 2], + } + } + + assert_impls::<vec128_storage>(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct u32x4_generic([u32; 4]); + } + + assert_impls::<u32x4_generic>(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct u64x2_generic([u64; 2]); + } + + assert_impls::<u64x2_generic>(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct u128x1_generic([u128; 1]); + } + + assert_impls::<u128x1_generic>(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[derive(Copy, Clone, Default)] + #[allow(non_camel_case_types)] + pub struct x2<W, G>(pub [W; 2], PhantomData<G>); + } + + enum NotZerocopy {} + assert_impls::<x2<(), NotZerocopy>>(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[derive(Copy, Clone, Default)] + #[allow(non_camel_case_types)] + pub struct x4<W>(pub [W; 4]); + } + + assert_impls::<x4<()>>(); + + #[cfg(feature = "simd")] + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + #[cfg(target_arch = "x86")] + use core::arch::x86::{__m128i, __m256i}; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::{__m128i, __m256i}; + + cryptocorrosion_derive_traits! { + #[repr(C)] + #[derive(Copy, Clone)] + pub struct X4(__m128i, __m128i, __m128i, __m128i); + } + + assert_impls::<X4>(); + + cryptocorrosion_derive_traits! { + #[repr(C)] + /// Generic wrapper for unparameterized storage of any of the + /// possible impls. Converting into and out of this type should + /// be essentially free, although it may be more aligned than a + /// particular impl requires. + #[allow(non_camel_case_types)] + #[derive(Copy, Clone)] + pub union vec128_storage { + u32x4: [u32; 4], + u64x2: [u64; 2], + u128x1: [u128; 1], + sse2: __m128i, + } + } + + assert_impls::<vec128_storage>(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[allow(non_camel_case_types)] + #[derive(Copy, Clone)] + pub struct vec<S3, S4, NI> { + x: __m128i, + s3: PhantomData<S3>, + s4: PhantomData<S4>, + ni: PhantomData<NI>, + } + } + + assert_impls::<vec<NotZerocopy, NotZerocopy, NotZerocopy>>(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[derive(Copy, Clone)] + pub struct u32x4x2_avx2<NI> { + x: __m256i, + ni: PhantomData<NI>, + } + } + + assert_impls::<u32x4x2_avx2<NotZerocopy>>(); + } + + // Make sure that our derive works for `#[repr(C)]` structs even though + // cryptocorrosion doesn't currently have any. + cryptocorrosion_derive_traits! { + #[repr(C)] + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct ReprC(u8, u8, u16); + } + } +} diff --git a/rust/zerocopy/src/pointer/inner.rs b/rust/zerocopy/src/pointer/inner.rs new file mode 100644 index 000000000000..5db08080141f --- /dev/null +++ b/rust/zerocopy/src/pointer/inner.rs @@ -0,0 +1,754 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2024 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use core::{marker::PhantomData, ops::Range, ptr::NonNull}; + +pub use _def::PtrInner; + +#[allow(unused_imports)] +use crate::util::polyfills::NumExt as _; +use crate::{ + layout::{CastType, MetadataCastError}, + pointer::cast, + util::AsAddress, + AlignmentError, CastError, KnownLayout, MetadataOf, SizeError, SplitAt, +}; + +mod _def { + use super::*; + /// The inner pointer stored inside a [`Ptr`][crate::Ptr]. + /// + /// `PtrInner<'a, T>` is [covariant] in `'a` and invariant in `T`. + /// + /// [covariant]: https://doc.rust-lang.org/reference/subtyping.html + #[allow(missing_debug_implementations)] + pub struct PtrInner<'a, T> + where + T: ?Sized, + { + /// # Invariants + /// + /// 0. If `ptr`'s referent is not zero sized, then `ptr` has valid + /// provenance for its referent, which is entirely contained in some + /// Rust allocation, `A`. + /// 1. If `ptr`'s referent is not zero sized, `A` is guaranteed to live + /// for at least `'a`. + /// + /// # Postconditions + /// + /// By virtue of these invariants, code may assume the following, which + /// are logical implications of the invariants: + /// - `ptr`'s referent is not larger than `isize::MAX` bytes \[1\] + /// - `ptr`'s referent does not wrap around the address space \[1\] + /// + /// \[1\] Per <https://doc.rust-lang.org/1.85.0/std/ptr/index.html#allocated-object>: + /// + /// For any allocated object with `base` address, `size`, and a set of + /// `addresses`, the following are guaranteed: + /// ... + /// - `size <= isize::MAX` + /// + /// As a consequence of these guarantees, given any address `a` within + /// the set of addresses of an allocated object: + /// ... + /// - It is guaranteed that, given `o = a - base` (i.e., the offset of + /// `a` within the allocated object), `base + o` will not wrap + /// around the address space (in other words, will not overflow + /// `usize`) + ptr: NonNull<T>, + // SAFETY: `&'a UnsafeCell<T>` is covariant in `'a` and invariant in `T` + // [1]. We use this construction rather than the equivalent `&mut T`, + // because our MSRV of 1.65 prohibits `&mut` types in const contexts. + // + // [1] https://doc.rust-lang.org/1.81.0/reference/subtyping.html#variance + _marker: PhantomData<&'a core::cell::UnsafeCell<T>>, + } + + impl<'a, T: 'a + ?Sized> Copy for PtrInner<'a, T> {} + impl<'a, T: 'a + ?Sized> Clone for PtrInner<'a, T> { + #[inline(always)] + fn clone(&self) -> PtrInner<'a, T> { + // SAFETY: None of the invariants on `ptr` are affected by having + // multiple copies of a `PtrInner`. + *self + } + } + + impl<'a, T: 'a + ?Sized> PtrInner<'a, T> { + /// Constructs a `Ptr` from a [`NonNull`]. + /// + /// # Safety + /// + /// The caller promises that: + /// + /// 0. If `ptr`'s referent is not zero sized, then `ptr` has valid + /// provenance for its referent, which is entirely contained in some + /// Rust allocation, `A`. + /// 1. If `ptr`'s referent is not zero sized, `A` is guaranteed to live + /// for at least `'a`. + #[inline(always)] + #[must_use] + pub const unsafe fn new(ptr: NonNull<T>) -> PtrInner<'a, T> { + // SAFETY: The caller has promised to satisfy all safety invariants + // of `PtrInner`. + Self { ptr, _marker: PhantomData } + } + + /// Converts this `PtrInner<T>` to a [`NonNull<T>`]. + /// + /// Note that this method does not consume `self`. The caller should + /// watch out for `unsafe` code which uses the returned `NonNull` in a + /// way that violates the safety invariants of `self`. + #[inline(always)] + #[must_use] + pub const fn as_non_null(&self) -> NonNull<T> { + self.ptr + } + + /// Converts this `PtrInner<T>` to a [`*mut T`]. + /// + /// Note that this method does not consume `self`. The caller should + /// watch out for `unsafe` code which uses the returned `*mut T` in a + /// way that violates the safety invariants of `self`. + #[inline(always)] + #[must_use] + pub const fn as_ptr(&self) -> *mut T { + self.ptr.as_ptr() + } + } +} + +impl<'a, T: ?Sized> PtrInner<'a, T> { + /// Constructs a `PtrInner` from a reference. + #[inline] + pub fn from_ref(ptr: &'a T) -> Self { + let ptr = NonNull::from(ptr); + // SAFETY: + // 0. If `ptr`'s referent is not zero sized, then `ptr`, by invariant on + // `&'a T` [1], has valid provenance for its referent, which is + // entirely contained in some Rust allocation, `A`. + // 1. If `ptr`'s referent is not zero sized, then `A`, by invariant on + // `&'a T`, is guaranteed to live for at least `'a`. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/primitive.reference.html#safety: + // + // For all types, `T: ?Sized`, and for all `t: &T` or `t: &mut T`, + // when such values cross an API boundary, the following invariants + // must generally be upheld: + // ... + // - if `size_of_val(t) > 0`, then `t` is dereferenceable for + // `size_of_val(t)` many bytes + // + // If `t` points at address `a`, being “dereferenceable” for N bytes + // means that the memory range `[a, a + N)` is all contained within a + // single allocated object. + unsafe { Self::new(ptr) } + } + + /// Constructs a `PtrInner` from a mutable reference. + #[inline] + pub fn from_mut(ptr: &'a mut T) -> Self { + let ptr = NonNull::from(ptr); + // SAFETY: + // 0. If `ptr`'s referent is not zero sized, then `ptr`, by invariant on + // `&'a mut T` [1], has valid provenance for its referent, which is + // entirely contained in some Rust allocation, `A`. + // 1. If `ptr`'s referent is not zero sized, then `A`, by invariant on + // `&'a mut T`, is guaranteed to live for at least `'a`. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/primitive.reference.html#safety: + // + // For all types, `T: ?Sized`, and for all `t: &T` or `t: &mut T`, + // when such values cross an API boundary, the following invariants + // must generally be upheld: + // ... + // - if `size_of_val(t) > 0`, then `t` is dereferenceable for + // `size_of_val(t)` many bytes + // + // If `t` points at address `a`, being “dereferenceable” for N bytes + // means that the memory range `[a, a + N)` is all contained within a + // single allocated object. + unsafe { Self::new(ptr) } + } + + /// # Safety + /// + /// The caller may assume that the resulting `PtrInner` addresses the subset + /// of the bytes of `self`'s referent addressed by `C::project(self)`. + #[must_use] + #[inline(always)] + pub fn project<U: ?Sized, C: cast::Project<T, U>>(self) -> PtrInner<'a, U> { + let projected_raw = C::project(self); + + // SAFETY: `self`'s referent lives at a `NonNull` address, and is either + // zero-sized or lives in an allocation. In either case, it does not + // wrap around the address space [1], and so none of the addresses + // contained in it or one-past-the-end of it are null. + // + // By invariant on `C: Project`, `C::project` is a provenance-preserving + // projection which preserves or shrinks the set of referent bytes, so + // `projected_raw` references a subset of `self`'s referent, and so it + // cannot be null. + // + // [1] https://doc.rust-lang.org/1.92.0/std/ptr/index.html#allocation + let projected_non_null = unsafe { NonNull::new_unchecked(projected_raw) }; + + // SAFETY: As described in the preceding safety comment, `projected_raw`, + // and thus `projected_non_null`, addresses a subset of `self`'s + // referent. Thus, `projected_non_null` either: + // - Addresses zero bytes or, + // - Addresses a subset of the referent of `self`. In this case, `self` + // has provenance for its referent, which lives in an allocation. + // Since `projected_non_null` was constructed using a sequence of + // provenance-preserving operations, it also has provenance for its + // referent and that referent lives in an allocation. By invariant on + // `self`, that allocation lives for `'a`. + unsafe { PtrInner::new(projected_non_null) } + } +} + +#[allow(clippy::needless_lifetimes)] +impl<'a, T> PtrInner<'a, T> +where + T: ?Sized + KnownLayout, +{ + /// Extracts the metadata of this `ptr`. + #[inline] + #[must_use] + pub fn meta(self) -> MetadataOf<T> { + let meta = T::pointer_to_metadata(self.as_ptr()); + // SAFETY: By invariant on `PtrInner`, `self.as_non_null()` addresses no + // more than `isize::MAX` bytes. + unsafe { MetadataOf::new_unchecked(meta) } + } + + /// Produces a `PtrInner` with the same address and provenance as `self` but + /// the given `meta`. + /// + /// # Safety + /// + /// The caller promises that if `self`'s referent is not zero sized, then + /// a pointer constructed from its address with the given `meta` metadata + /// will address a subset of the allocation pointed to by `self`. + #[inline] + #[must_use] + pub unsafe fn with_meta(self, meta: T::PointerMetadata) -> Self + where + T: KnownLayout, + { + let raw = T::raw_from_ptr_len(self.as_non_null().cast(), meta); + + // SAFETY: + // + // Lemma 0: `raw` either addresses zero bytes, or addresses a subset of + // the allocation pointed to by `self` and has the same + // provenance as `self`. Proof: `raw` is constructed using + // provenance-preserving operations, and the caller has + // promised that, if `self`'s referent is not zero-sized, the + // resulting pointer addresses a subset of the allocation + // pointed to by `self`. + // + // 0. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `ptr` is derived from some valid Rust allocation, + // `A`. + // 1. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `ptr` has valid provenance for `A`. + // 2. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `ptr` addresses a byte range which is entirely + // contained in `A`. + // 3. Per Lemma 0 and by invariant on `self`, `ptr` addresses a byte + // range whose length fits in an `isize`. + // 4. Per Lemma 0 and by invariant on `self`, `ptr` addresses a byte + // range which does not wrap around the address space. + // 5. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `A` is guaranteed to live for at least `'a`. + unsafe { PtrInner::new(raw) } + } +} + +#[allow(clippy::needless_lifetimes)] +impl<'a, T> PtrInner<'a, T> +where + T: ?Sized + KnownLayout<PointerMetadata = usize>, +{ + /// Splits `T` in two. + /// + /// # Safety + /// + /// The caller promises that: + /// - `l_len.get() <= self.meta()`. + /// + /// ## (Non-)Overlap + /// + /// Given `let (left, right) = ptr.split_at(l_len)`, it is guaranteed that + /// `left` and `right` are contiguous and non-overlapping if + /// `l_len.padding_needed_for() == 0`. This is true for all `[T]`. + /// + /// If `l_len.padding_needed_for() != 0`, then the left pointer will overlap + /// the right pointer to satisfy `T`'s padding requirements. + #[inline] + #[must_use] + pub unsafe fn split_at_unchecked( + self, + l_len: crate::util::MetadataOf<T>, + ) -> (Self, PtrInner<'a, [T::Elem]>) + where + T: SplitAt, + { + let l_len = l_len.get(); + + // SAFETY: The caller promises that `l_len.get() <= self.meta()`. + // Trivially, `0 <= l_len`. + let left = unsafe { self.with_meta(l_len) }; + + let right = self.trailing_slice(); + // SAFETY: The caller promises that `l_len <= self.meta() = slf.meta()`. + // Trivially, `slf.meta() <= slf.meta()`. + let right = unsafe { right.slice_unchecked(l_len..self.meta().get()) }; + + // SAFETY: If `l_len.padding_needed_for() == 0`, then `left` and `right` + // are non-overlapping. Proof: `left` is constructed `slf` with `l_len` + // as its (exclusive) upper bound. If `l_len.padding_needed_for() == 0`, + // then `left` requires no trailing padding following its final element. + // Since `right` is constructed from `slf`'s trailing slice with `l_len` + // as its (inclusive) lower bound, no byte is referred to by both + // pointers. + // + // Conversely, `l_len.padding_needed_for() == N`, where `N + // > 0`, `left` requires `N` bytes of trailing padding following its + // final element. Since `right` is constructed from the trailing slice + // of `slf` with `l_len` as its (inclusive) lower bound, the first `N` + // bytes of `right` are aliased by `left`. + (left, right) + } + + /// Produces the trailing slice of `self`. + #[inline] + #[must_use] + pub fn trailing_slice(self) -> PtrInner<'a, [T::Elem]> + where + T: SplitAt, + { + let offset = crate::trailing_slice_layout::<T>().offset; + + let bytes = self.as_non_null().cast::<u8>().as_ptr(); + + // SAFETY: + // - By invariant on `T: KnownLayout`, `T::LAYOUT` describes `T`'s + // layout. `offset` is the offset of the trailing slice within `T`, + // which is by definition in-bounds or one byte past the end of any + // `T`, regardless of metadata. By invariant on `PtrInner`, `self` + // (and thus `bytes`) points to a byte range of size `<= isize::MAX`, + // and so `offset <= isize::MAX`. Since `size_of::<u8>() == 1`, + // `offset * size_of::<u8>() <= isize::MAX`. + // - If `offset > 0`, then by invariant on `PtrInner`, `self` (and thus + // `bytes`) points to a byte range entirely contained within the same + // allocated object as `self`. As explained above, this offset results + // in a pointer to or one byte past the end of this allocated object. + let bytes = unsafe { bytes.add(offset) }; + + // SAFETY: By the preceding safety argument, `bytes` is within or one + // byte past the end of the same allocated object as `self`, which + // ensures that it is non-null. + let bytes = unsafe { NonNull::new_unchecked(bytes) }; + + let ptr = KnownLayout::raw_from_ptr_len(bytes, self.meta().get()); + + // SAFETY: + // 0. If `ptr`'s referent is not zero sized, then `ptr` is derived from + // some valid Rust allocation, `A`, because `ptr` is derived from + // the same allocated object as `self`. + // 1. If `ptr`'s referent is not zero sized, then `ptr` has valid + // provenance for `A` because `raw` is derived from the same + // allocated object as `self` via provenance-preserving operations. + // 2. If `ptr`'s referent is not zero sized, then `ptr` addresses a byte + // range which is entirely contained in `A`, by previous safety proof + // on `bytes`. + // 3. `ptr` addresses a byte range whose length fits in an `isize`, by + // consequence of #2. + // 4. `ptr` addresses a byte range which does not wrap around the + // address space, by consequence of #2. + // 5. If `ptr`'s referent is not zero sized, then `A` is guaranteed to + // live for at least `'a`, because `ptr` is derived from `self`. + unsafe { PtrInner::new(ptr) } + } +} + +#[allow(clippy::needless_lifetimes)] +impl<'a, T> PtrInner<'a, [T]> { + /// Creates a pointer which addresses the given `range` of self. + /// + /// # Safety + /// + /// `range` is a valid range (`start <= end`) and `end <= self.meta()`. + #[inline] + #[must_use] + pub unsafe fn slice_unchecked(self, range: Range<usize>) -> Self { + let base = self.as_non_null().cast::<T>().as_ptr(); + + // SAFETY: The caller promises that `start <= end <= self.meta()`. By + // invariant, if `self`'s referent is not zero-sized, then `self` refers + // to a byte range which is contained within a single allocation, which + // is no more than `isize::MAX` bytes long, and which does not wrap + // around the address space. Thus, this pointer arithmetic remains + // in-bounds of the same allocation, and does not wrap around the + // address space. The offset (in bytes) does not overflow `isize`. + // + // If `self`'s referent is zero-sized, then these conditions are + // trivially satisfied. + let base = unsafe { base.add(range.start) }; + + // SAFETY: The caller promises that `start <= end`, and so this will not + // underflow. + #[allow(unstable_name_collisions)] + let len = unsafe { range.end.unchecked_sub(range.start) }; + + let ptr = core::ptr::slice_from_raw_parts_mut(base, len); + + // SAFETY: By invariant, `self`'s referent is either a ZST or lives + // entirely in an allocation. `ptr` points inside of or one byte past + // the end of that referent. Thus, in either case, `ptr` is non-null. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; + + // SAFETY: + // + // Lemma 0: `ptr` addresses a subset of the bytes addressed by `self`, + // and has the same provenance. Proof: The caller guarantees + // that `start <= end <= self.meta()`. Thus, `base` is + // in-bounds of `self`, and `base + (end - start)` is also + // in-bounds of self. Finally, `ptr` is constructed using + // provenance-preserving operations. + // + // 0. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `ptr` has valid provenance for its referent, + // which is entirely contained in some Rust allocation, `A`. + // 1. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `A` is guaranteed to live for at least `'a`. + unsafe { PtrInner::new(ptr) } + } + + /// Iteratively projects the elements `PtrInner<T>` from `PtrInner<[T]>`. + #[inline] + pub fn iter(&self) -> impl Iterator<Item = PtrInner<'a, T>> { + // FIXME(#429): Once `NonNull::cast` documents that it preserves + // provenance, cite those docs. + let base = self.as_non_null().cast::<T>().as_ptr(); + (0..self.meta().get()).map(move |i| { + // FIXME(https://github.com/rust-lang/rust/issues/74265): Use + // `NonNull::get_unchecked_mut`. + + // SAFETY: If the following conditions are not satisfied + // `pointer::cast` may induce Undefined Behavior [1]: + // + // > - The computed offset, `count * size_of::<T>()` bytes, must not + // > overflow `isize``. + // > - If the computed offset is non-zero, then `self` must be + // > derived from a pointer to some allocated object, and the + // > entire memory range between `self` and the result must be in + // > bounds of that allocated object. In particular, this range + // > must not “wrap around” the edge of the address space. + // + // [1] https://doc.rust-lang.org/std/primitive.pointer.html#method.add + // + // We satisfy both of these conditions here: + // - By invariant on `Ptr`, `self` addresses a byte range whose + // length fits in an `isize`. Since `elem` is contained in `self`, + // the computed offset of `elem` must fit within `isize.` + // - If the computed offset is non-zero, then this means that the + // referent is not zero-sized. In this case, `base` points to an + // allocated object (by invariant on `self`). Thus: + // - By contract, `self.meta()` accurately reflects the number of + // elements in the slice. `i` is in bounds of `c.meta()` by + // construction, and so the result of this addition cannot + // overflow past the end of the allocation referred to by `c`. + // - By invariant on `Ptr`, `self` addresses a byte range which + // does not wrap around the address space. Since `elem` is + // contained in `self`, the computed offset of `elem` must wrap + // around the address space. + // + // FIXME(#429): Once `pointer::add` documents that it preserves + // provenance, cite those docs. + let elem = unsafe { base.add(i) }; + + // SAFETY: `elem` must not be null. `base` is constructed from a + // `NonNull` pointer, and the addition that produces `elem` must not + // overflow or wrap around, so `elem >= base > 0`. + // + // FIXME(#429): Once `NonNull::new_unchecked` documents that it + // preserves provenance, cite those docs. + let elem = unsafe { NonNull::new_unchecked(elem) }; + + // SAFETY: The safety invariants of `Ptr::new` (see definition) are + // satisfied: + // 0. If `elem`'s referent is not zero sized, then `elem` has valid + // provenance for its referent, because it derived from `self` + // using a series of provenance-preserving operations, and + // because `self` has valid provenance for its referent. By the + // same argument, `elem`'s referent is entirely contained within + // the same allocated object as `self`'s referent. + // 1. If `elem`'s referent is not zero sized, then the allocation of + // `elem` is guaranteed to live for at least `'a`, because `elem` + // is entirely contained in `self`, which lives for at least `'a` + // by invariant on `Ptr`. + unsafe { PtrInner::new(elem) } + }) + } +} + +impl<'a, T, const N: usize> PtrInner<'a, [T; N]> { + /// Casts this pointer-to-array into a slice. + /// + /// # Safety + /// + /// Callers may assume that the returned `PtrInner` references the same + /// address and length as `self`. + #[allow(clippy::wrong_self_convention)] + #[inline] + #[must_use] + pub fn as_slice(self) -> PtrInner<'a, [T]> { + let start = self.as_non_null().cast::<T>().as_ptr(); + let slice = core::ptr::slice_from_raw_parts_mut(start, N); + // SAFETY: `slice` is not null, because it is derived from `start` + // which is non-null. + let slice = unsafe { NonNull::new_unchecked(slice) }; + // SAFETY: Lemma: In the following safety arguments, note that `slice` + // is derived from `self` in two steps: first, by casting `self: [T; N]` + // to `start: T`, then by constructing a pointer to a slice starting at + // `start` of length `N`. As a result, `slice` references exactly the + // same allocation as `self`, if any. + // + // 0. By the above lemma, if `slice`'s referent is not zero sized, then + // `slice` has the same referent as `self`. By invariant on `self`, + // this referent is entirely contained within some allocation, `A`. + // Because `slice` was constructed using provenance-preserving + // operations, it has provenance for its entire referent. + // 1. By the above lemma, if `slice`'s referent is not zero sized, then + // `A` is guaranteed to live for at least `'a`, because it is derived + // from the same allocation as `self`, which, by invariant on + // `PtrInner`, lives for at least `'a`. + unsafe { PtrInner::new(slice) } + } +} + +impl<'a> PtrInner<'a, [u8]> { + /// Attempts to cast `self` to a `U` using the given cast type. + /// + /// If `U` is a slice DST and pointer metadata (`meta`) is provided, then + /// the cast will only succeed if it would produce an object with the given + /// metadata. + /// + /// Returns `None` if the resulting `U` would be invalidly-aligned, if no + /// `U` can fit in `self`, or if the provided pointer metadata describes an + /// invalid instance of `U`. On success, returns a pointer to the + /// largest-possible `U` which fits in `self`. + /// + /// # Safety + /// + /// The caller may assume that this implementation is correct, and may rely + /// on that assumption for the soundness of their code. In particular, the + /// caller may assume that, if `try_cast_into` returns `Some((ptr, + /// remainder))`, then `ptr` and `remainder` refer to non-overlapping byte + /// ranges within `self`, and that `ptr` and `remainder` entirely cover + /// `self`. Finally: + /// - If this is a prefix cast, `ptr` has the same address as `self`. + /// - If this is a suffix cast, `remainder` has the same address as `self`. + #[inline] + pub fn try_cast_into<U>( + self, + cast_type: CastType, + meta: Option<U::PointerMetadata>, + ) -> Result<(PtrInner<'a, U>, PtrInner<'a, [u8]>), CastError<Self, U>> + where + U: 'a + ?Sized + KnownLayout, + { + // PANICS: By invariant, the byte range addressed by + // `self.as_non_null()` does not wrap around the address space. This + // implies that the sum of the address (represented as a `usize`) and + // length do not overflow `usize`, as required by + // `validate_cast_and_convert_metadata`. Thus, this call to + // `validate_cast_and_convert_metadata` will only panic if `U` is a DST + // whose trailing slice element is zero-sized. + let maybe_metadata = MetadataOf::<U>::validate_cast_and_convert_metadata( + AsAddress::addr(self.as_ptr()), + self.meta(), + cast_type, + meta, + ); + + let (elems, split_at) = match maybe_metadata { + Ok((elems, split_at)) => (elems, split_at), + Err(MetadataCastError::Alignment) => { + // SAFETY: Since `validate_cast_and_convert_metadata` returned + // an alignment error, `U` must have an alignment requirement + // greater than one. + let err = unsafe { AlignmentError::<_, U>::new_unchecked(self) }; + return Err(CastError::Alignment(err)); + } + Err(MetadataCastError::Size) => return Err(CastError::Size(SizeError::new(self))), + }; + + // SAFETY: `validate_cast_and_convert_metadata` promises to return + // `split_at <= self.meta()`. + // + // Lemma 0: `l_slice` and `r_slice` are non-overlapping. Proof: By + // contract on `PtrInner::split_at_unchecked`, the produced `PtrInner`s + // are always non-overlapping if `self` is a `[T]`; here it is a `[u8]`. + let (l_slice, r_slice) = unsafe { self.split_at_unchecked(split_at) }; + + let (target, remainder) = match cast_type { + CastType::Prefix => (l_slice, r_slice), + CastType::Suffix => (r_slice, l_slice), + }; + + let base = target.as_non_null().cast::<u8>(); + + let ptr = U::raw_from_ptr_len(base, elems.get()); + + // SAFETY: + // 0. By invariant, if `target`'s referent is not zero sized, then + // `target` has provenance valid for some Rust allocation, `A`. + // Because `ptr` is derived from `target` via provenance-preserving + // operations, `ptr` will also have provenance valid for its entire + // referent. + // 1. `validate_cast_and_convert_metadata` promises that the object + // described by `elems` and `split_at` lives at a byte range which is + // a subset of the input byte range. Thus, by invariant, if + // `target`'s referent is not zero sized, then `target` refers to an + // allocation which is guaranteed to live for at least `'a`, and thus + // so does `ptr`. + Ok((unsafe { PtrInner::new(ptr) }, remainder)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::*; + + #[test] + fn test_meta() { + let arr = [1; 16]; + let dst = <[u8]>::ref_from_bytes(&arr[..]).unwrap(); + let ptr = PtrInner::from_ref(dst); + assert_eq!(ptr.meta().get(), 16); + + // SAFETY: 8 is less than 16 + let ptr = unsafe { ptr.with_meta(8) }; + + assert_eq!(ptr.meta().get(), 8); + } + + #[test] + fn test_split_at() { + fn test_split_at<const OFFSET: usize, const BUFFER_SIZE: usize>() { + #[derive(FromBytes, KnownLayout, SplitAt, Immutable)] + #[repr(C)] + struct SliceDst<const OFFSET: usize> { + prefix: [u8; OFFSET], + trailing: [u8], + } + + let n: usize = BUFFER_SIZE - OFFSET; + let arr = [1; BUFFER_SIZE]; + let dst = SliceDst::<OFFSET>::ref_from_bytes(&arr[..]).unwrap(); + let ptr = PtrInner::from_ref(dst); + for i in 0..=n { + assert_eq!(ptr.meta().get(), n); + // SAFETY: `i` is in bounds by construction. + let i = unsafe { MetadataOf::new_unchecked(i) }; + // SAFETY: `i` is in bounds by construction. + let (l, r) = unsafe { ptr.split_at_unchecked(i) }; + // SAFETY: Points to a valid value by construction. + #[allow(clippy::undocumented_unsafe_blocks, clippy::as_conversions)] + // Clippy false positive + let l_sum: usize = l + .trailing_slice() + .iter() + .map( + #[inline(always)] + |ptr| unsafe { core::ptr::read_unaligned(ptr.as_ptr()) } as usize, + ) + .sum(); + // SAFETY: Points to a valid value by construction. + #[allow(clippy::undocumented_unsafe_blocks, clippy::as_conversions)] + // Clippy false positive + let r_sum: usize = r + .iter() + .map( + #[inline(always)] + |ptr| unsafe { core::ptr::read_unaligned(ptr.as_ptr()) } as usize, + ) + .sum(); + assert_eq!(l_sum, i.get()); + assert_eq!(r_sum, n - i.get()); + assert_eq!(l_sum + r_sum, n); + } + } + + test_split_at::<0, 16>(); + test_split_at::<1, 17>(); + test_split_at::<2, 18>(); + } + + #[test] + fn test_trailing_slice() { + fn test_trailing_slice<const OFFSET: usize, const BUFFER_SIZE: usize>() { + #[derive(FromBytes, KnownLayout, SplitAt, Immutable)] + #[repr(C)] + struct SliceDst<const OFFSET: usize> { + prefix: [u8; OFFSET], + trailing: [u8], + } + + let n: usize = BUFFER_SIZE - OFFSET; + let arr = [1; BUFFER_SIZE]; + let dst = SliceDst::<OFFSET>::ref_from_bytes(&arr[..]).unwrap(); + let ptr = PtrInner::from_ref(dst); + + assert_eq!(ptr.meta().get(), n); + let trailing = ptr.trailing_slice(); + assert_eq!(trailing.meta().get(), n); + + assert_eq!( + // SAFETY: We assume this to be sound for the sake of this test, + // which will fail, here, in miri, if the safety precondition of + // `offset_of` is not satisfied. + unsafe { + #[allow(clippy::as_conversions)] + let offset = (trailing.as_ptr() as *mut u8).offset_from(ptr.as_ptr() as *mut _); + offset + }, + isize::try_from(OFFSET).unwrap(), + ); + + // SAFETY: Points to a valid value by construction. + #[allow(clippy::undocumented_unsafe_blocks, clippy::as_conversions)] + // Clippy false positive + let trailing: usize = trailing + .iter() + .map(|ptr| unsafe { core::ptr::read_unaligned(ptr.as_ptr()) } as usize) + .sum(); + + assert_eq!(trailing, n); + } + + test_trailing_slice::<0, 16>(); + test_trailing_slice::<1, 17>(); + test_trailing_slice::<2, 18>(); + } + #[test] + fn test_ptr_inner_clone() { + let mut x = 0u8; + let p = PtrInner::from_mut(&mut x); + #[allow(clippy::clone_on_copy)] + let p2 = p.clone(); + assert_eq!(p.as_non_null(), p2.as_non_null()); + } +} diff --git a/rust/zerocopy/src/pointer/invariant.rs b/rust/zerocopy/src/pointer/invariant.rs new file mode 100644 index 000000000000..1802d23563db --- /dev/null +++ b/rust/zerocopy/src/pointer/invariant.rs @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2024 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +#![allow(missing_copy_implementations, missing_debug_implementations, missing_docs)] + +//! The parameterized invariants of a [`Ptr`][super::Ptr]. +//! +//! Invariants are encoded as ([`Aliasing`], [`Alignment`], [`Validity`]) +//! triples implementing the [`Invariants`] trait. + +/// The invariants of a [`Ptr`][super::Ptr]. +pub trait Invariants: Sealed { + type Aliasing: Aliasing; + type Alignment: Alignment; + type Validity: Validity; +} + +impl<A: Aliasing, AA: Alignment, V: Validity> Invariants for (A, AA, V) { + type Aliasing = A; + type Alignment = AA; + type Validity = V; +} + +/// The aliasing invariant of a [`Ptr`][super::Ptr]. +/// +/// All aliasing invariants must permit reading from the bytes of a pointer's +/// referent which are not covered by [`UnsafeCell`]s. +/// +/// [`UnsafeCell`]: core::cell::UnsafeCell +pub trait Aliasing: Sealed { + /// Is `Self` [`Exclusive`]? + #[doc(hidden)] + const IS_EXCLUSIVE: bool; +} + +/// The alignment invariant of a [`Ptr`][super::Ptr]. +pub trait Alignment: Sealed { + #[doc(hidden)] + #[must_use] + fn read<T, I, R>(ptr: crate::Ptr<'_, T, I>) -> T + where + T: Copy + Read<I::Aliasing, R>, + I: Invariants<Alignment = Self, Validity = Valid>, + I::Aliasing: Reference; +} + +/// The validity invariant of a [`Ptr`][super::Ptr]. +/// +/// # Safety +/// +/// In this section, we will use `Ptr<T, V>` as a shorthand for `Ptr<T, I: +/// Invariants<Validity = V>>` for brevity. +/// +/// Each `V: Validity` defines a set of bit values which may appear in the +/// referent of a `Ptr<T, V>`, denoted `S(T, V)`. Each `V: Validity`, in its +/// documentation, provides a definition of `S(T, V)` which must be valid for +/// all `T: ?Sized`. Any `V: Validity` must guarantee that this set is only a +/// function of the *bit validity* of the referent type, `T`, and not of any +/// other property of `T`. As a consequence, given `V: Validity`, `T`, and `U` +/// where `T` and `U` have the same bit validity, `S(V, T) = S(V, U)`. +/// +/// It is guaranteed that the referent of any `ptr: Ptr<T, V>` is a member of +/// `S(T, V)`. Unsafe code must ensure that this guarantee will be upheld for +/// any existing `Ptr`s or any `Ptr`s that that code creates. +/// +/// An important implication of this guarantee is that it restricts what +/// transmutes are sound, where "transmute" is used in this context to refer to +/// changing the referent type or validity invariant of a `Ptr`, as either +/// change may change the set of bit values permitted to appear in the referent. +/// In particular, the following are necessary (but not sufficient) conditions +/// in order for a transmute from `src: Ptr<T, V>` to `dst: Ptr<U, W>` to be +/// sound: +/// - If `S(T, V) = S(U, W)`, then no restrictions apply; otherwise, +/// - If `dst` permits mutation of its referent (e.g. via `Exclusive` aliasing +/// or interior mutation under `Shared` aliasing), then it must hold that +/// `S(T, V) ⊇ S(U, W)` - in other words, the transmute must not expand the +/// set of allowed referent bit patterns. A violation of this requirement +/// would permit using `dst` to write `x` where `x ∈ S(U, W)` but `x ∉ S(T, +/// V)`, which would violate the guarantee that `src`'s referent may only +/// contain values in `S(T, V)`. +/// - If the referent may be mutated without going through `dst` while `dst` is +/// live (e.g. via interior mutation on a `Shared`-aliased `Ptr` or `&` +/// reference), then it must hold that `S(T, V) ⊆ S(U, W)` - in other words, +/// the transmute must not shrink the set of allowed referent bit patterns. A +/// violation of this requirement would permit using `src` or another +/// mechanism (e.g. a `&` reference used to derive `src`) to write `x` where +/// `x ∈ S(T, V)` but `x ∉ S(U, W)`, which would violate the guarantee that +/// `dst`'s referent may only contain values in `S(U, W)`. +pub unsafe trait Validity: Sealed { + const KIND: ValidityKind; +} + +pub enum ValidityKind { + Uninit, + AsInitialized, + Initialized, + Valid, +} + +/// An [`Aliasing`] invariant which is either [`Shared`] or [`Exclusive`]. +/// +/// # Safety +/// +/// Given `A: Reference`, callers may assume that either `A = Shared` or `A = +/// Exclusive`. +pub trait Reference: Aliasing + Sealed {} + +/// The `Ptr<'a, T>` adheres to the aliasing rules of a `&'a T`. +/// +/// The referent of a shared-aliased `Ptr` may be concurrently referenced by any +/// number of shared-aliased `Ptr` or `&T` references, or by any number of +/// `Ptr<U>` or `&U` references as permitted by `T`'s library safety invariants, +/// and may not be concurrently referenced by any exclusively-aliased `Ptr`s or +/// `&mut` references. The referent must not be mutated, except via +/// [`UnsafeCell`]s, and only when permitted by `T`'s library safety invariants. +/// +/// [`UnsafeCell`]: core::cell::UnsafeCell +pub enum Shared {} +impl Aliasing for Shared { + const IS_EXCLUSIVE: bool = false; +} +impl Reference for Shared {} + +/// The `Ptr<'a, T>` adheres to the aliasing rules of a `&'a mut T`. +/// +/// The referent of an exclusively-aliased `Ptr` may not be concurrently +/// referenced by any other `Ptr`s or references, and may not be accessed (read +/// or written) other than via this `Ptr`. +pub enum Exclusive {} +impl Aliasing for Exclusive { + const IS_EXCLUSIVE: bool = true; +} +impl Reference for Exclusive {} + +/// It is unknown whether the pointer is aligned. +pub enum Unaligned {} + +impl Alignment for Unaligned { + #[inline(always)] + fn read<T, I, R>(ptr: crate::Ptr<'_, T, I>) -> T + where + T: Copy + Read<I::Aliasing, R>, + I: Invariants<Alignment = Self, Validity = Valid>, + I::Aliasing: Reference, + { + (*ptr.into_unalign().as_ref()).into_inner() + } +} + +/// The referent is aligned: for `Ptr<T>`, the referent's address is a multiple +/// of the `T`'s alignment. +pub enum Aligned {} +impl Alignment for Aligned { + #[inline(always)] + fn read<T, I, R>(ptr: crate::Ptr<'_, T, I>) -> T + where + T: Copy + Read<I::Aliasing, R>, + I: Invariants<Alignment = Self, Validity = Valid>, + I::Aliasing: Reference, + { + *ptr.as_ref() + } +} + +/// Any bit pattern is allowed in the `Ptr`'s referent, including uninitialized +/// bytes. +pub enum Uninit {} +// SAFETY: `Uninit`'s validity is well-defined for all `T: ?Sized`, and is not a +// function of any property of `T` other than its bit validity (in fact, it's +// not even a property of `T`'s bit validity, but this is more than we are +// required to uphold). +unsafe impl Validity for Uninit { + const KIND: ValidityKind = ValidityKind::Uninit; +} + +/// The byte ranges initialized in `T` are also initialized in the referent of a +/// `Ptr<T>`. +/// +/// Formally: uninitialized bytes may only be present in `Ptr<T>`'s referent +/// where they are guaranteed to be present in `T`. This is a dynamic property: +/// if, at a particular byte offset, a valid enum discriminant is set, the +/// subsequent bytes may only have uninitialized bytes as specified by the +/// corresponding enum. +/// +/// Formally, given `len = size_of_val_raw(ptr)`, at every byte offset, `b`, in +/// the range `[0, len)`: +/// - If, in any instance `t: T` of length `len`, the byte at offset `b` in `t` +/// is initialized, then the byte at offset `b` within `*ptr` must be +/// initialized. +/// - Let `c` be the contents of the byte range `[0, b)` in `*ptr`. Let `S` be +/// the subset of valid instances of `T` of length `len` which contain `c` in +/// the offset range `[0, b)`. If, in any instance of `t: T` in `S`, the byte +/// at offset `b` in `t` is initialized, then the byte at offset `b` in `*ptr` +/// must be initialized. +/// +/// Pragmatically, this means that if `*ptr` is guaranteed to contain an enum +/// type at a particular offset, and the enum discriminant stored in `*ptr` +/// corresponds to a valid variant of that enum type, then it is guaranteed +/// that the appropriate bytes of `*ptr` are initialized as defined by that +/// variant's bit validity (although note that the variant may contain another +/// enum type, in which case the same rules apply depending on the state of +/// its discriminant, and so on recursively). +pub enum AsInitialized {} +// SAFETY: `AsInitialized`'s validity is well-defined for all `T: ?Sized`, and +// is not a function of any property of `T` other than its bit validity. +unsafe impl Validity for AsInitialized { + const KIND: ValidityKind = ValidityKind::AsInitialized; +} + +/// The byte ranges in the referent are fully initialized. In other words, if +/// the referent is `N` bytes long, then it contains a bit-valid `[u8; N]`. +pub enum Initialized {} +// SAFETY: `Initialized`'s validity is well-defined for all `T: ?Sized`, and is +// not a function of any property of `T` other than its bit validity (in fact, +// it's not even a property of `T`'s bit validity, but this is more than we are +// required to uphold). +unsafe impl Validity for Initialized { + const KIND: ValidityKind = ValidityKind::Initialized; +} + +/// The referent of a `Ptr<T>` is valid for `T`, upholding bit validity and any +/// library safety invariants. +pub enum Valid {} +// SAFETY: `Valid`'s validity is well-defined for all `T: ?Sized`, and is not a +// function of any property of `T` other than its bit validity. +unsafe impl Validity for Valid { + const KIND: ValidityKind = ValidityKind::Valid; +} + +/// # Safety +/// +/// `DT: CastableFrom<ST, SV, DV>` is sound if `SV = DV = Uninit` or `SV = DV = +/// Initialized`. +pub unsafe trait CastableFrom<ST: ?Sized, SV, DV> {} + +// SAFETY: `SV = DV = Uninit`. +unsafe impl<ST: ?Sized, DT: ?Sized> CastableFrom<ST, Uninit, Uninit> for DT {} +// SAFETY: `SV = DV = Initialized`. +unsafe impl<ST: ?Sized, DT: ?Sized> CastableFrom<ST, Initialized, Initialized> for DT {} + +/// [`Ptr`](crate::Ptr) referents that permit unsynchronized read operations. +/// +/// `T: Read<A, R>` implies that a pointer to `T` with aliasing `A` permits +/// unsynchronized read operations. This can be because `A` is [`Exclusive`] or +/// because `T` does not permit interior mutation. +/// +/// # Safety +/// +/// `T: Read<A, R>` if either of the following conditions holds: +/// - `A` is [`Exclusive`] +/// - `T` implements [`Immutable`](crate::Immutable) +/// +/// As a consequence, if `T: Read<A, R>`, then any `Ptr<T, (A, ...)>` is +/// permitted to perform unsynchronized reads from its referent. +pub trait Read<A: Aliasing, R> {} + +impl<A: Aliasing, T: ?Sized + crate::Immutable> Read<A, BecauseImmutable> for T {} +impl<T: ?Sized> Read<Exclusive, BecauseExclusive> for T {} + +/// Unsynchronized reads are permitted because only one live [`Ptr`](crate::Ptr) +/// or reference may exist to the referent bytes at a time. +#[derive(Copy, Clone, Debug)] +pub enum BecauseExclusive {} + +/// Unsynchronized reads are permitted because no live [`Ptr`](crate::Ptr)s or +/// references permit interior mutation. +#[derive(Copy, Clone, Debug)] +pub enum BecauseImmutable {} + +use sealed::Sealed; +mod sealed { + use super::*; + + pub trait Sealed {} + + impl Sealed for Shared {} + impl Sealed for Exclusive {} + + impl Sealed for Unaligned {} + impl Sealed for Aligned {} + + impl Sealed for Uninit {} + impl Sealed for AsInitialized {} + impl Sealed for Initialized {} + impl Sealed for Valid {} + + impl<A: Sealed, AA: Sealed, V: Sealed> Sealed for (A, AA, V) {} + + impl Sealed for BecauseImmutable {} + impl Sealed for BecauseExclusive {} +} diff --git a/rust/zerocopy/src/pointer/mod.rs b/rust/zerocopy/src/pointer/mod.rs new file mode 100644 index 000000000000..3461f7f5ca80 --- /dev/null +++ b/rust/zerocopy/src/pointer/mod.rs @@ -0,0 +1,410 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2023 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Abstractions over raw pointers. + +#![allow(missing_docs)] + +mod inner; +pub mod invariant; +mod ptr; +pub mod transmute; + +pub use inner::PtrInner; +pub use invariant::{BecauseExclusive, BecauseImmutable, Read}; +pub use ptr::{Ptr, TryWithError}; +pub use transmute::*; + +use crate::wrappers::ReadOnly; + +/// A shorthand for a maybe-valid, maybe-aligned reference. Used as the argument +/// to [`TryFromBytes::is_bit_valid`]. +/// +/// [`TryFromBytes::is_bit_valid`]: crate::TryFromBytes::is_bit_valid +pub type Maybe<'a, T, Alignment = invariant::Unaligned> = + Ptr<'a, ReadOnly<T>, (invariant::Shared, Alignment, invariant::Initialized)>; + +/// Checks if the referent is zeroed. +pub(crate) fn is_zeroed<T, I>(ptr: Ptr<'_, T, I>) -> bool +where + T: crate::Immutable + crate::KnownLayout, + I: invariant::Invariants<Validity = invariant::Initialized>, + I::Aliasing: invariant::Reference, +{ + ptr.as_bytes().as_ref().iter().all( + #[inline(always)] + |&byte| byte == 0, + ) +} + +pub mod cast { + use core::{marker::PhantomData, mem}; + + use crate::{ + layout::{SizeInfo, TrailingSliceLayout}, + HasField, KnownLayout, PtrInner, + }; + + /// A pointer cast or projection. + /// + /// # Safety + /// + /// The implementation of `project` must satisfy its safety post-condition. + pub unsafe trait Project<Src: ?Sized, Dst: ?Sized> { + /// Projects a pointer from `Src` to `Dst`. + /// + /// Users should generally not call `project` directly, and instead + /// should use high-level APIs like [`PtrInner::project`] or + /// [`Ptr::project`]. + /// + /// [`Ptr::project`]: crate::pointer::Ptr::project + /// + /// # Safety + /// + /// The returned pointer refers to a non-strict subset of the bytes of + /// `src`'s referent, and has the same provenance as `src`. + fn project(src: PtrInner<'_, Src>) -> *mut Dst; + } + + /// A [`Project`] which preserves the address of the referent – a pointer + /// cast. + /// + /// # Safety + /// + /// A `Cast` projection must preserve the address of the referent. It may + /// shrink the set of referent bytes, and it may change the referent's type. + pub unsafe trait Cast<Src: ?Sized, Dst: ?Sized>: Project<Src, Dst> {} + + /// A [`Cast`] which does not shrink the set of referent bytes. + /// + /// # Safety + /// + /// A `CastExact` projection must preserve the set of referent bytes. + pub unsafe trait CastExact<Src: ?Sized, Dst: ?Sized>: Cast<Src, Dst> {} + + /// A no-op pointer cast. + #[derive(Default, Copy, Clone)] + #[allow(missing_debug_implementations)] + pub struct IdCast; + + // SAFETY: `project` returns its argument unchanged, and so it is a + // provenance-preserving projection which preserves the set of referent + // bytes. + unsafe impl<T: ?Sized> Project<T, T> for IdCast { + #[inline(always)] + fn project(src: PtrInner<'_, T>) -> *mut T { + src.as_ptr() + } + } + + // SAFETY: The `Project::project` impl preserves referent address. + unsafe impl<T: ?Sized> Cast<T, T> for IdCast {} + + // SAFETY: The `Project::project` impl preserves referent size. + unsafe impl<T: ?Sized> CastExact<T, T> for IdCast {} + + /// A pointer cast which preserves or shrinks the set of referent bytes of + /// a statically-sized referent. + /// + /// # Safety + /// + /// The implementation of [`Project`] uses a compile-time assertion to + /// guarantee that `Dst` is no larger than `Src`. Thus, `CastSized` has a + /// sound implementation of [`Project`] for all `Src` and `Dst` – the caller + /// may pass any `Src` and `Dst` without being responsible for soundness. + #[allow(missing_debug_implementations, missing_copy_implementations)] + pub enum CastSized {} + + // SAFETY: By the `static_assert!`, `Dst` is no larger than `Src`, + // and so all casts preserve or shrink the set of referent bytes. All + // operations preserve provenance. + unsafe impl<Src, Dst> Project<Src, Dst> for CastSized { + #[inline(always)] + fn project(src: PtrInner<'_, Src>) -> *mut Dst { + static_assert!(Src, Dst => mem::size_of::<Src>() >= mem::size_of::<Dst>()); + src.as_ptr().cast::<Dst>() + } + } + + // SAFETY: The `Project::project` impl preserves referent address. + unsafe impl<Src, Dst> Cast<Src, Dst> for CastSized {} + + /// A pointer cast which preserves the set of referent bytes of a + /// statically-sized referent. + /// + /// # Safety + /// + /// The implementation of [`Project`] uses a compile-time assertion to + /// guarantee that `Dst` has the same size as `Src`. Thus, `CastSizedExact` + /// has a sound implementation of [`Project`] for all `Src` and `Dst` – the + /// caller may pass any `Src` and `Dst` without being responsible for + /// soundness. + #[allow(missing_debug_implementations, missing_copy_implementations)] + pub enum CastSizedExact {} + + // SAFETY: By the `static_assert!`, `Dst` has the same size as `Src`, + // and so all casts preserve the set of referent bytes. All operations + // preserve provenance. + unsafe impl<Src, Dst> Project<Src, Dst> for CastSizedExact { + #[inline(always)] + fn project(src: PtrInner<'_, Src>) -> *mut Dst { + static_assert!(Src, Dst => mem::size_of::<Src>() == mem::size_of::<Dst>()); + src.as_ptr().cast::<Dst>() + } + } + + // SAFETY: The `Project::project_raw` impl preserves referent address. + unsafe impl<Src, Dst> Cast<Src, Dst> for CastSizedExact {} + + // SAFETY: By the `static_assert!`, `Project::project_raw` impl preserves + // referent size. + unsafe impl<Src, Dst> CastExact<Src, Dst> for CastSizedExact {} + + /// A pointer cast which preserves or shrinks the set of referent bytes of + /// a dynamically-sized referent. + /// + /// # Safety + /// + /// The implementation of [`Project`] uses a compile-time assertion to + /// guarantee that the cast preserves the set of referent bytes. Thus, + /// `CastUnsized` has a sound implementation of [`Project`] for all `Src` + /// and `Dst` – the caller may pass any `Src` and `Dst` without being + /// responsible for soundness. + #[allow(missing_debug_implementations, missing_copy_implementations)] + pub enum CastUnsized {} + + // SAFETY: By the `static_assert!`, `Src` and `Dst` are either: + // - Both sized and equal in size + // - Both slice DSTs with the same trailing slice offset and element size + // and with align_of::<Src>() == align_of::<Dst>(). These ensure that any + // given pointer metadata encodes the same size for both `Src` and `Dst` + // (note that the alignment is required as it affects the amount of + // trailing padding). Thus, `project` preserves the set of referent bytes. + unsafe impl<Src, Dst> Project<Src, Dst> for CastUnsized + where + Src: ?Sized + KnownLayout, + Dst: ?Sized + KnownLayout<PointerMetadata = Src::PointerMetadata>, + { + #[inline(always)] + fn project(src: PtrInner<'_, Src>) -> *mut Dst { + // FIXME: Do we want this to support shrinking casts as well? If so, + // we'll need to remove the `CastExact` impl. + static_assert!(Src: ?Sized + KnownLayout, Dst: ?Sized + KnownLayout => { + let src = <Src as KnownLayout>::LAYOUT; + let dst = <Dst as KnownLayout>::LAYOUT; + match (src.size_info, dst.size_info) { + (SizeInfo::Sized { size: src_size }, SizeInfo::Sized { size: dst_size }) => src_size == dst_size, + ( + SizeInfo::SliceDst(TrailingSliceLayout { offset: src_offset, elem_size: src_elem_size }), + SizeInfo::SliceDst(TrailingSliceLayout { offset: dst_offset, elem_size: dst_elem_size }) + ) => src.align.get() == dst.align.get() && src_offset == dst_offset && src_elem_size == dst_elem_size, + _ => false, + } + }); + + let metadata = Src::pointer_to_metadata(src.as_ptr()); + Dst::raw_from_ptr_len(src.as_non_null().cast::<u8>(), metadata).as_ptr() + } + } + + // SAFETY: The `Project::project` impl preserves referent address. + unsafe impl<Src, Dst> Cast<Src, Dst> for CastUnsized + where + Src: ?Sized + KnownLayout, + Dst: ?Sized + KnownLayout<PointerMetadata = Src::PointerMetadata>, + { + } + + // SAFETY: By the `static_assert!` in `Project::project`, `Src` and `Dst` + // are either: + // - Both sized and equal in size + // - Both slice DSTs with the same alignment, trailing slice offset, and + // element size. These ensure that any given pointer metadata encodes the + // same size for both `Src` and `Dst` (note that the alignment is required + // as it affects the amount of trailing padding). + unsafe impl<Src, Dst> CastExact<Src, Dst> for CastUnsized + where + Src: ?Sized + KnownLayout, + Dst: ?Sized + KnownLayout<PointerMetadata = Src::PointerMetadata>, + { + } + + /// A field projection + /// + /// A `Projection` is a [`Project`] which implements projection by + /// delegating to an implementation of [`HasField::project`]. + #[allow(missing_debug_implementations, missing_copy_implementations)] + pub struct Projection<F: ?Sized, const VARIANT_ID: i128, const FIELD_ID: i128> { + _never: core::convert::Infallible, + _phantom: PhantomData<F>, + } + + // SAFETY: `HasField::project` has the same safety post-conditions as + // `Project::project`. + unsafe impl<T: ?Sized, F, const VARIANT_ID: i128, const FIELD_ID: i128> Project<T, T::Type> + for Projection<F, VARIANT_ID, FIELD_ID> + where + T: HasField<F, VARIANT_ID, FIELD_ID>, + { + #[inline(always)] + fn project(src: PtrInner<'_, T>) -> *mut T::Type { + T::project(src) + } + } + + // SAFETY: All `repr(C)` union fields exist at offset 0 within the union [1], + // and so any union projection is actually a cast (ie, preserves address). + // + // [1] Per + // https://doc.rust-lang.org/1.92.0/reference/type-layout.html#reprc-unions, + // it's not *technically* guaranteed that non-maximally-sized fields + // are at offset 0, but it's clear that this is the intention of `repr(C)` + // unions. It says: + // + // > A union declared with `#[repr(C)]` will have the same size and + // > alignment as an equivalent C union declaration in the C language for + // > the target platform. + // + // Note that this only mentions size and alignment, not layout. However, + // C unions *do* guarantee that all fields start at offset 0. [2] + // + // This is also reinforced by + // https://doc.rust-lang.org/1.92.0/reference/items/unions.html#r-items.union.fields.offset: + // + // > Fields might have a non-zero offset (except when the C + // > representation is used); in that case the bits starting at the + // > offset of the fields are read + // + // [2] Per https://port70.net/~nsz/c/c11/n1570.html#6.7.2.1p16: + // + // > The size of a union is sufficient to contain the largest of its + // > members. The value of at most one of the members can be stored in a + // > union object at any time. A pointer to a union object, suitably + // > converted, points to each of its members (or if a member is a + // > bit-field, then to the unit in which it resides), and vice versa. + // + // FIXME(https://github.com/rust-lang/unsafe-code-guidelines/issues/595): + // Cite the documentation once it's updated. + unsafe impl<T: ?Sized, F, const FIELD_ID: i128> Cast<T, T::Type> + for Projection<F, { crate::REPR_C_UNION_VARIANT_ID }, FIELD_ID> + where + T: HasField<F, { crate::REPR_C_UNION_VARIANT_ID }, FIELD_ID>, + { + } + + /// A transitive sequence of projections. + /// + /// Given `TU: Project` and `UV: Project`, `TransitiveProject<_, TU, UV>` is + /// a [`Project`] which projects by applying `TU` followed by `UV`. + /// + /// If `TU: Cast` and `UV: Cast`, then `TransitiveProject<_, TU, UV>: Cast`. + #[allow(missing_debug_implementations)] + pub struct TransitiveProject<U: ?Sized, TU, UV> { + _never: core::convert::Infallible, + _projections: PhantomData<(TU, UV)>, + // On our MSRV (1.56), the debuginfo for a tuple containing both an + // uninhabited type and a DST causes an ICE. We split `U` from `TU` and + // `UV` to avoid this situation. + _u: PhantomData<U>, + } + + // SAFETY: Since `TU::project` and `UV::project` are each + // provenance-preserving operations which preserve or shrink the set of + // referent bytes, so is their composition. + unsafe impl<T, U, V, TU, UV> Project<T, V> for TransitiveProject<U, TU, UV> + where + T: ?Sized, + U: ?Sized, + V: ?Sized, + TU: Project<T, U>, + UV: Project<U, V>, + { + #[inline(always)] + fn project(t: PtrInner<'_, T>) -> *mut V { + t.project::<_, TU>().project::<_, UV>().as_ptr() + } + } + + // SAFETY: Since the `Project::project` impl delegates to `TU::project` and + // `UV::project`, and since `TU` and `UV` are `Cast`, the `Project::project` + // impl preserves the address of the referent. + unsafe impl<T, U, V, TU, UV> Cast<T, V> for TransitiveProject<U, TU, UV> + where + T: ?Sized, + U: ?Sized, + V: ?Sized, + TU: Cast<T, U>, + UV: Cast<U, V>, + { + } + + // SAFETY: Since the `Project::project` impl delegates to `TU::project` and + // `UV::project`, and since `TU` and `UV` are `CastExact`, the `Project::project` + // impl preserves the set of referent bytes. + unsafe impl<T, U, V, TU, UV> CastExact<T, V> for TransitiveProject<U, TU, UV> + where + T: ?Sized, + U: ?Sized, + V: ?Sized, + TU: CastExact<T, U>, + UV: CastExact<U, V>, + { + } + + /// A cast from `T` to `[u8]`. + #[allow(missing_copy_implementations, missing_debug_implementations)] + pub struct AsBytesCast; + + // SAFETY: `project` constructs a pointer with the same address as `src` + // and with a referent of the same size as `*src`. It does this using + // provenance-preserving operations. + // + // FIXME(https://github.com/rust-lang/unsafe-code-guidelines/issues/594): + // Technically, this proof assumes that `*src` is contiguous (the same is + // true of other proofs in this codebase). Is this guaranteed anywhere? + unsafe impl<T: ?Sized + KnownLayout> Project<T, [u8]> for AsBytesCast { + #[inline(always)] + fn project(src: PtrInner<'_, T>) -> *mut [u8] { + let bytes = match T::size_of_val_raw(src.as_non_null()) { + Some(bytes) => bytes, + // SAFETY: `KnownLayout::size_of_val_raw` promises to always + // return `Some` so long as the resulting size fits in a + // `usize`. By invariant on `PtrInner`, `src` refers to a range + // of bytes whose size fits in an `isize`, which implies that it + // also fits in a `usize`. + None => unsafe { core::hint::unreachable_unchecked() }, + }; + + core::ptr::slice_from_raw_parts_mut(src.as_ptr().cast::<u8>(), bytes) + } + } + + // SAFETY: The `Project::project` impl preserves referent address. + unsafe impl<T: ?Sized + KnownLayout> Cast<T, [u8]> for AsBytesCast {} + + // SAFETY: The `Project::project` impl preserves the set of referent bytes. + unsafe impl<T: ?Sized + KnownLayout> CastExact<T, [u8]> for AsBytesCast {} + + /// A cast from any type to `()`. + #[allow(missing_copy_implementations, missing_debug_implementations)] + pub struct CastToUnit; + + // SAFETY: The `project` implementation projects to a subset of its + // argument's referent using provenance-preserving operations. + unsafe impl<T: ?Sized> Project<T, ()> for CastToUnit { + #[inline(always)] + fn project(src: PtrInner<'_, T>) -> *mut () { + src.as_ptr().cast::<()>() + } + } + + // SAFETY: The `project` implementation preserves referent address. + unsafe impl<T: ?Sized> Cast<T, ()> for CastToUnit {} +} diff --git a/rust/zerocopy/src/pointer/ptr.rs b/rust/zerocopy/src/pointer/ptr.rs new file mode 100644 index 000000000000..b7c4ea56d2b2 --- /dev/null +++ b/rust/zerocopy/src/pointer/ptr.rs @@ -0,0 +1,1586 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2023 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +#![allow(missing_docs)] + +use core::{ + fmt::{Debug, Formatter}, + marker::PhantomData, +}; + +use crate::{ + pointer::{ + inner::PtrInner, + invariant::*, + transmute::{MutationCompatible, SizeEq, TransmuteFromPtr}, + }, + AlignmentError, CastError, CastType, KnownLayout, SizeError, TryFromBytes, ValidityError, +}; + +/// Module used to gate access to [`Ptr`]'s fields. +mod def { + #[cfg(doc)] + use super::super::invariant; + use super::*; + + /// A raw pointer with more restrictions. + /// + /// `Ptr<T>` is similar to [`NonNull<T>`], but it is more restrictive in the + /// following ways (note that these requirements only hold of non-zero-sized + /// referents): + /// - It must derive from a valid allocation. + /// - It must reference a byte range which is contained inside the + /// allocation from which it derives. + /// - As a consequence, the byte range it references must have a size + /// which does not overflow `isize`. + /// + /// Depending on how `Ptr` is parameterized, it may have additional + /// invariants: + /// - `ptr` conforms to the aliasing invariant of + /// [`I::Aliasing`](invariant::Aliasing). + /// - `ptr` conforms to the alignment invariant of + /// [`I::Alignment`](invariant::Alignment). + /// - `ptr` conforms to the validity invariant of + /// [`I::Validity`](invariant::Validity). + /// + /// `Ptr<'a, T>` is [covariant] in `'a` and invariant in `T`. + /// + /// [`NonNull<T>`]: core::ptr::NonNull + /// [covariant]: https://doc.rust-lang.org/reference/subtyping.html + pub struct Ptr<'a, T, I> + where + T: ?Sized, + I: Invariants, + { + /// # Invariants + /// + /// 0. `ptr` conforms to the aliasing invariant of + /// [`I::Aliasing`](invariant::Aliasing). + /// 1. `ptr` conforms to the alignment invariant of + /// [`I::Alignment`](invariant::Alignment). + /// 2. `ptr` conforms to the validity invariant of + /// [`I::Validity`](invariant::Validity). + // SAFETY: `PtrInner<'a, T>` is covariant in `'a` and invariant in `T`. + ptr: PtrInner<'a, T>, + _invariants: PhantomData<I>, + } + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + { + /// Constructs a new `Ptr` from a [`PtrInner`]. + /// + /// # Safety + /// + /// The caller promises that: + /// + /// 0. `ptr` conforms to the aliasing invariant of + /// [`I::Aliasing`](invariant::Aliasing). + /// 1. `ptr` conforms to the alignment invariant of + /// [`I::Alignment`](invariant::Alignment). + /// 2. `ptr` conforms to the validity invariant of + /// [`I::Validity`](invariant::Validity). + pub(crate) unsafe fn from_inner(ptr: PtrInner<'a, T>) -> Ptr<'a, T, I> { + // SAFETY: The caller has promised to satisfy all safety invariants + // of `Ptr`. + Self { ptr, _invariants: PhantomData } + } + + /// Converts this `Ptr<T>` to a [`PtrInner<T>`]. + /// + /// Note that this method does not consume `self`. The caller should + /// watch out for `unsafe` code which uses the returned value in a way + /// that violates the safety invariants of `self`. + #[inline] + #[must_use] + pub fn as_inner(&self) -> PtrInner<'a, T> { + self.ptr + } + } +} + +#[allow(unreachable_pub)] // This is a false positive on our MSRV toolchain. +pub use def::Ptr; + +/// External trait implementations on [`Ptr`]. +mod _external { + use super::*; + + /// SAFETY: Shared pointers are safely `Copy`. `Ptr`'s other invariants + /// (besides aliasing) are unaffected by the number of references that exist + /// to `Ptr`'s referent. The notable cases are: + /// - Alignment is a property of the referent type (`T`) and the address, + /// both of which are unchanged + /// - Let `S(T, V)` be the set of bit values permitted to appear in the + /// referent of a `Ptr<T, I: Invariants<Validity = V>>`. Since this copy + /// does not change `I::Validity` or `T`, `S(T, I::Validity)` is also + /// unchanged. + /// + /// We are required to guarantee that the referents of the original `Ptr` + /// and of the copy (which, of course, are actually the same since they + /// live in the same byte address range) both remain in the set `S(T, + /// I::Validity)`. Since this invariant holds on the original `Ptr`, it + /// cannot be violated by the original `Ptr`, and thus the original `Ptr` + /// cannot be used to violate this invariant on the copy. The inverse + /// holds as well. + impl<'a, T, I> Copy for Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants<Aliasing = Shared>, + { + } + + /// SAFETY: See the safety comment on `Copy`. + impl<'a, T, I> Clone for Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants<Aliasing = Shared>, + { + #[inline] + fn clone(&self) -> Self { + *self + } + } + + impl<'a, T, I> Debug for Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + self.as_inner().as_non_null().fmt(f) + } + } +} + +/// Methods for converting to and from `Ptr` and Rust's safe reference types. +mod _conversions { + use super::*; + use crate::pointer::cast::{CastExact, CastSized, IdCast}; + + /// `&'a T` → `Ptr<'a, T>` + impl<'a, T> Ptr<'a, T, (Shared, Aligned, Valid)> + where + T: 'a + ?Sized, + { + /// Constructs a `Ptr` from a shared reference. + #[inline(always)] + pub fn from_ref(ptr: &'a T) -> Self { + let inner = PtrInner::from_ref(ptr); + // SAFETY: + // 0. `ptr`, by invariant on `&'a T`, conforms to the aliasing + // invariant of `Shared`. + // 1. `ptr`, by invariant on `&'a T`, conforms to the alignment + // invariant of `Aligned`. + // 2. `ptr`'s referent, by invariant on `&'a T`, is a bit-valid `T`. + // This satisfies the requirement that a `Ptr<T, (_, _, Valid)>` + // point to a bit-valid `T`. Even if `T` permits interior + // mutation, this invariant guarantees that the returned `Ptr` + // can only ever be used to modify the referent to store + // bit-valid `T`s, which ensures that the returned `Ptr` cannot + // be used to violate the soundness of the original `ptr: &'a T` + // or of any other references that may exist to the same + // referent. + unsafe { Self::from_inner(inner) } + } + } + + /// `&'a mut T` → `Ptr<'a, T>` + impl<'a, T> Ptr<'a, T, (Exclusive, Aligned, Valid)> + where + T: 'a + ?Sized, + { + /// Constructs a `Ptr` from an exclusive reference. + #[inline(always)] + pub fn from_mut(ptr: &'a mut T) -> Self { + let inner = PtrInner::from_mut(ptr); + // SAFETY: + // 0. `ptr`, by invariant on `&'a mut T`, conforms to the aliasing + // invariant of `Exclusive`. + // 1. `ptr`, by invariant on `&'a mut T`, conforms to the alignment + // invariant of `Aligned`. + // 2. `ptr`'s referent, by invariant on `&'a mut T`, is a bit-valid + // `T`. This satisfies the requirement that a `Ptr<T, (_, _, + // Valid)>` point to a bit-valid `T`. This invariant guarantees + // that the returned `Ptr` can only ever be used to modify the + // referent to store bit-valid `T`s, which ensures that the + // returned `Ptr` cannot be used to violate the soundness of the + // original `ptr: &'a mut T`. + unsafe { Self::from_inner(inner) } + } + } + + /// `Ptr<'a, T>` → `&'a T` + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants<Alignment = Aligned, Validity = Valid>, + I::Aliasing: Reference, + { + /// Converts `self` to a shared reference. + // This consumes `self`, not `&self`, because `self` is, logically, a + // pointer. For `I::Aliasing = invariant::Shared`, `Self: Copy`, and so + // this doesn't prevent the caller from still using the pointer after + // calling `as_ref`. + #[allow(clippy::wrong_self_convention)] + #[inline] + #[must_use] + pub fn as_ref(self) -> &'a T { + let raw = self.as_inner().as_non_null(); + // SAFETY: `self` satisfies the `Aligned` invariant, so we know that + // `raw` is validly-aligned for `T`. + #[cfg(miri)] + unsafe { + crate::util::miri_promise_symbolic_alignment( + raw.as_ptr().cast(), + core::mem::align_of_val_raw(raw.as_ptr()), + ); + } + // SAFETY: This invocation of `NonNull::as_ref` satisfies its + // documented safety preconditions: + // + // 1. The pointer is properly aligned. This is ensured by-contract + // on `Ptr`, because the `I::Alignment` is `Aligned`. + // + // 2. If the pointer's referent is not zero-sized, then the pointer + // must be “dereferenceable” in the sense defined in the module + // documentation; i.e.: + // + // > The memory range of the given size starting at the pointer + // > must all be within the bounds of a single allocated object. + // > [2] + // + // This is ensured by contract on all `PtrInner`s. + // + // 3. The pointer must point to a validly-initialized instance of + // `T`. This is ensured by-contract on `Ptr`, because the + // `I::Validity` is `Valid`. + // + // 4. You must enforce Rust’s aliasing rules. This is ensured by + // contract on `Ptr`, because `I::Aliasing: Reference`. Either it + // is `Shared` or `Exclusive`. If it is `Shared`, other + // references may not mutate the referent outside of + // `UnsafeCell`s. + // + // [1]: https://doc.rust-lang.org/std/ptr/struct.NonNull.html#method.as_ref + // [2]: https://doc.rust-lang.org/std/ptr/index.html#safety + unsafe { raw.as_ref() } + } + } + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + I::Aliasing: Reference, + { + /// Reborrows `self`, producing another `Ptr`. + /// + /// Since `self` is borrowed mutably, this prevents any methods from + /// being called on `self` as long as the returned `Ptr` exists. + #[inline] + #[must_use] + #[allow(clippy::needless_lifetimes)] // Allows us to name the lifetime in the safety comment below. + pub fn reborrow<'b>(&'b mut self) -> Ptr<'b, T, I> + where + 'a: 'b, + { + // SAFETY: The following all hold by invariant on `self`, and thus + // hold of `ptr = self.as_inner()`: + // 0. SEE BELOW. + // 1. `ptr` conforms to the alignment invariant of + // [`I::Alignment`](invariant::Alignment). + // 2. `ptr` conforms to the validity invariant of + // [`I::Validity`](invariant::Validity). `self` and the returned + // `Ptr` permit the same bit values in their referents since they + // have the same referent type (`T`) and the same validity + // (`I::Validity`). Thus, regardless of what mutation is + // permitted (`Exclusive` aliasing or `Shared`-aliased interior + // mutation), neither can be used to write a value to the + // referent which violates the other's validity invariant. + // + // For aliasing (0 above), since `I::Aliasing: Reference`, + // there are two cases for `I::Aliasing`: + // - For `invariant::Shared`: `'a` outlives `'b`, and so the + // returned `Ptr` does not permit accessing the referent any + // longer than is possible via `self`. For shared aliasing, it is + // sound for multiple `Ptr`s to exist simultaneously which + // reference the same memory, so creating a new one is not + // problematic. + // - For `invariant::Exclusive`: Since `self` is `&'b mut` and we + // return a `Ptr` with lifetime `'b`, `self` is inaccessible to + // the caller for the lifetime `'b` - in other words, `self` is + // inaccessible to the caller as long as the returned `Ptr` + // exists. Since `self` is an exclusive `Ptr`, no other live + // references or `Ptr`s may exist which refer to the same memory + // while `self` is live. Thus, as long as the returned `Ptr` + // exists, no other references or `Ptr`s which refer to the same + // memory may be live. + unsafe { Ptr::from_inner(self.as_inner()) } + } + + /// Reborrows `self` as shared, producing another `Ptr` with `Shared` + /// aliasing. + /// + /// Since `self` is borrowed mutably, this prevents any methods from + /// being called on `self` as long as the returned `Ptr` exists. + #[inline] + #[must_use] + #[allow(clippy::needless_lifetimes)] // Allows us to name the lifetime in the safety comment below. + pub fn reborrow_shared<'b>(&'b mut self) -> Ptr<'b, T, (Shared, I::Alignment, I::Validity)> + where + 'a: 'b, + { + // SAFETY: The following all hold by invariant on `self`, and thus + // hold of `ptr = self.as_inner()`: + // 0. SEE BELOW. + // 1. `ptr` conforms to the alignment invariant of + // [`I::Alignment`](invariant::Alignment). + // 2. `ptr` conforms to the validity invariant of + // [`I::Validity`](invariant::Validity). `self` and the returned + // `Ptr` permit the same bit values in their referents since they + // have the same referent type (`T`) and the same validity + // (`I::Validity`). Thus, regardless of what mutation is + // permitted (`Exclusive` aliasing or `Shared`-aliased interior + // mutation), neither can be used to write a value to the + // referent which violates the other's validity invariant. + // + // For aliasing (0 above), since `I::Aliasing: Reference`, + // there are two cases for `I::Aliasing`: + // - For `invariant::Shared`: `'a` outlives `'b`, and so the + // returned `Ptr` does not permit accessing the referent any + // longer than is possible via `self`. For shared aliasing, it is + // sound for multiple `Ptr`s to exist simultaneously which + // reference the same memory, so creating a new one is not + // problematic. + // - For `invariant::Exclusive`: Since `self` is `&'b mut` and we + // return a `Ptr` with lifetime `'b`, `self` is inaccessible to + // the caller for the lifetime `'b` - in other words, `self` is + // inaccessible to the caller as long as the returned `Ptr` + // exists. Since `self` is an exclusive `Ptr`, no other live + // references or `Ptr`s may exist which refer to the same memory + // while `self` is live. Thus, as long as the returned `Ptr` + // exists, no other references or `Ptr`s which refer to the same + // memory may be live. + unsafe { Ptr::from_inner(self.as_inner()) } + } + } + + /// `Ptr<'a, T>` → `&'a mut T` + impl<'a, T> Ptr<'a, T, (Exclusive, Aligned, Valid)> + where + T: 'a + ?Sized, + { + /// Converts `self` to a mutable reference. + #[allow(clippy::wrong_self_convention)] + #[inline] + #[must_use] + pub fn as_mut(self) -> &'a mut T { + let mut raw = self.as_inner().as_non_null(); + // SAFETY: `self` satisfies the `Aligned` invariant, so we know that + // `raw` is validly-aligned for `T`. + #[cfg(miri)] + unsafe { + crate::util::miri_promise_symbolic_alignment( + raw.as_ptr().cast(), + core::mem::align_of_val_raw(raw.as_ptr()), + ); + } + // SAFETY: This invocation of `NonNull::as_mut` satisfies its + // documented safety preconditions: + // + // 1. The pointer is properly aligned. This is ensured by-contract + // on `Ptr`, because the `ALIGNMENT_INVARIANT` is `Aligned`. + // + // 2. If the pointer's referent is not zero-sized, then the pointer + // must be “dereferenceable” in the sense defined in the module + // documentation; i.e.: + // + // > The memory range of the given size starting at the pointer + // > must all be within the bounds of a single allocated object. + // > [2] + // + // This is ensured by contract on all `PtrInner`s. + // + // 3. The pointer must point to a validly-initialized instance of + // `T`. This is ensured by-contract on `Ptr`, because the + // validity invariant is `Valid`. + // + // 4. You must enforce Rust’s aliasing rules. This is ensured by + // contract on `Ptr`, because the `ALIASING_INVARIANT` is + // `Exclusive`. + // + // [1]: https://doc.rust-lang.org/std/ptr/struct.NonNull.html#method.as_mut + // [2]: https://doc.rust-lang.org/std/ptr/index.html#safety + unsafe { raw.as_mut() } + } + } + + /// `Ptr<'a, T>` → `Ptr<'a, U>` + impl<'a, T: ?Sized, I> Ptr<'a, T, I> + where + I: Invariants, + { + #[must_use] + #[inline(always)] + pub fn transmute<U, V, R>(self) -> Ptr<'a, U, (I::Aliasing, Unaligned, V)> + where + V: Validity, + U: TransmuteFromPtr<T, I::Aliasing, I::Validity, V, <U as SizeEq<T>>::CastFrom, R> + + SizeEq<T> + + ?Sized, + { + self.transmute_with::<U, V, <U as SizeEq<T>>::CastFrom, R>() + } + + #[inline] + #[must_use] + pub fn transmute_with<U, V, C, R>(self) -> Ptr<'a, U, (I::Aliasing, Unaligned, V)> + where + V: Validity, + U: TransmuteFromPtr<T, I::Aliasing, I::Validity, V, C, R> + ?Sized, + C: CastExact<T, U>, + { + // SAFETY: + // - By `C: CastExact`, `C` preserves referent address, and so we + // don't need to consider projections in the following safety + // arguments. + // - If aliasing is `Shared`, then by `U: TransmuteFromPtr<T>`, at + // least one of the following holds: + // - `T: Immutable` and `U: Immutable`, in which case it is + // trivially sound for shared code to operate on a `&T` and `&U` + // at the same time, as neither can perform interior mutation + // - It is directly guaranteed that it is sound for shared code to + // operate on these references simultaneously + // - By `U: TransmuteFromPtr<T, I::Aliasing, I::Validity, C, V>`, it + // is sound to perform this transmute using `C`. + unsafe { self.project_transmute_unchecked::<_, _, C>() } + } + + #[inline] + #[must_use] + pub fn recall_validity<V, R>(self) -> Ptr<'a, T, (I::Aliasing, I::Alignment, V)> + where + V: Validity, + T: TransmuteFromPtr<T, I::Aliasing, I::Validity, V, IdCast, R>, + { + let ptr = self.transmute_with::<T, V, IdCast, R>(); + // SAFETY: `self` and `ptr` have the same address and referent type. + // Therefore, if `self` satisfies `I::Alignment`, then so does + // `ptr`. + unsafe { ptr.assume_alignment::<I::Alignment>() } + } + + /// Projects and/or transmutes to a different (unsized) referent type + /// without checking interior mutability. + /// + /// Callers should prefer [`cast`] or [`project`] where possible. + /// + /// [`cast`]: Ptr::cast + /// [`project`]: Ptr::project + /// + /// # Safety + /// + /// The caller promises that: + /// - If `I::Aliasing` is [`Shared`], it must not be possible for safe + /// code, operating on a `&T` and `&U`, with the referents of `self` + /// and `self.project_transmute_unchecked()`, respectively, to cause + /// undefined behavior. + /// - It is sound to project and/or transmute a pointer of type `T` with + /// aliasing `I::Aliasing` and validity `I::Validity` to a pointer of + /// type `U` with aliasing `I::Aliasing` and validity `V`. This is a + /// subtle soundness requirement that is a function of `T`, `U`, + /// `I::Aliasing`, `I::Validity`, and `V`, and may depend upon the + /// presence, absence, or specific location of `UnsafeCell`s in `T` + /// and/or `U`, and on whether interior mutation is ever permitted via + /// those `UnsafeCell`s. See [`Validity`] for more details. + #[inline] + #[must_use] + pub unsafe fn project_transmute_unchecked<U: ?Sized, V, P>( + self, + ) -> Ptr<'a, U, (I::Aliasing, Unaligned, V)> + where + V: Validity, + P: crate::pointer::cast::Project<T, U>, + { + let ptr = self.as_inner().project::<_, P>(); + + // SAFETY: + // + // The following safety arguments rely on the fact that `P: Project` + // guarantees that `P` is a referent-preserving or -shrinking + // projection. Thus, `ptr` addresses a subset of the bytes of + // `*self`, and so certain properties that hold of `*self` also hold + // of `*ptr`. + // + // 0. `ptr` conforms to the aliasing invariant of `I::Aliasing`: + // - `Exclusive`: `self` is the only `Ptr` or reference which is + // permitted to read or modify the referent for the lifetime + // `'a`. Since we consume `self` by value, the returned pointer + // remains the only `Ptr` or reference which is permitted to + // read or modify the referent for the lifetime `'a`. + // - `Shared`: Since `self` has aliasing `Shared`, we know that + // no other code may mutate the referent during the lifetime + // `'a`, except via `UnsafeCell`s, and except as permitted by + // `T`'s library safety invariants. The caller promises that + // any safe operations which can be permitted on a `&T` and a + // `&U` simultaneously must be sound. Thus, no operations on a + // `&U` could violate `&T`'s library safety invariants, and + // vice-versa. Since any mutation via shared references outside + // of `UnsafeCell`s is unsound, this must be impossible using + // `&T` and `&U`. + // - `Inaccessible`: There are no restrictions we need to uphold. + // 1. `ptr` trivially satisfies the alignment invariant `Unaligned`. + // 2. The caller promises that the returned pointer satisfies the + // validity invariant `V` with respect to its referent type, `U`. + unsafe { Ptr::from_inner(ptr) } + } + } + + /// `Ptr<'a, T, (_, _, _)>` → `Ptr<'a, Unalign<T>, (_, Aligned, _)>` + impl<'a, T, I> Ptr<'a, T, I> + where + I: Invariants, + { + /// Converts a `Ptr` an unaligned `T` into a `Ptr` to an aligned + /// `Unalign<T>`. + #[inline] + #[must_use] + pub fn into_unalign( + self, + ) -> Ptr<'a, crate::Unalign<T>, (I::Aliasing, Aligned, I::Validity)> { + // FIXME(#1359): This should be a `transmute_with` call. + // Unfortunately, to avoid blanket impl conflicts, we only implement + // `TransmuteFrom<T>` for `Unalign<T>` (and vice versa) specifically + // for `Valid` validity, not for all validity types. + + // SAFETY: + // - By `CastSized: Cast`, `CastSized` preserves referent address, + // and so we don't need to consider projections in the following + // safety arguments. + // - Since `Unalign<T>` has the same layout as `T`, the returned + // pointer refers to `UnsafeCell`s at the same locations as + // `self`. + // - `Unalign<T>` promises to have the same bit validity as `T`. By + // invariant on `Validity`, the set of bit patterns allowed in the + // referent of a `Ptr<X, (_, _, V)>` is only a function of the + // validity of `X` and of `V`. Thus, the set of bit patterns + // allowed in the referent of a `Ptr<T, (_, _, I::Validity)>` is + // the same as the set of bit patterns allowed in the referent of + // a `Ptr<Unalign<T>, (_, _, I::Validity)>`. As a result, `self` + // and the returned `Ptr` permit the same set of bit patterns in + // their referents, and so neither can be used to violate the + // validity of the other. + let ptr = unsafe { self.project_transmute_unchecked::<_, _, CastSized>() }; + ptr.bikeshed_recall_aligned() + } + } + + impl<'a, T, I> Ptr<'a, T, I> + where + T: ?Sized, + I: Invariants<Validity = Valid>, + I::Aliasing: Reference, + { + /// Reads the referent. + #[must_use] + #[inline(always)] + pub fn read<R>(self) -> T + where + T: Copy, + T: Read<I::Aliasing, R>, + { + <I::Alignment as Alignment>::read(self) + } + + /// Views the value as an aligned reference. + /// + /// This is only available if `T` is [`Unaligned`]. + #[must_use] + #[inline] + pub fn unaligned_as_ref(self) -> &'a T + where + T: crate::Unaligned, + { + self.bikeshed_recall_aligned().as_ref() + } + } +} + +/// State transitions between invariants. +mod _transitions { + use super::*; + use crate::{ + pointer::{cast::IdCast, transmute::TryTransmuteFromPtr}, + ReadOnly, + }; + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + { + /// Assumes that `self` satisfies the invariants `H`. + /// + /// # Safety + /// + /// The caller promises that `self` satisfies the invariants `H`. + unsafe fn assume_invariants<H: Invariants>(self) -> Ptr<'a, T, H> { + // SAFETY: The caller has promised to satisfy all parameterized + // invariants of `Ptr`. `Ptr`'s other invariants are satisfied + // by-contract by the source `Ptr`. + unsafe { Ptr::from_inner(self.as_inner()) } + } + + /// Helps the type system unify two distinct invariant types which are + /// actually the same. + #[inline] + #[must_use] + pub fn unify_invariants< + H: Invariants<Aliasing = I::Aliasing, Alignment = I::Alignment, Validity = I::Validity>, + >( + self, + ) -> Ptr<'a, T, H> { + // SAFETY: The associated type bounds on `H` ensure that the + // invariants are unchanged. + unsafe { self.assume_invariants::<H>() } + } + + /// Assumes that `self`'s referent is validly-aligned for `T` if + /// required by `A`. + /// + /// # Safety + /// + /// The caller promises that `self`'s referent conforms to the alignment + /// invariant of `T` if required by `A`. + #[inline] + pub(crate) unsafe fn assume_alignment<A: Alignment>( + self, + ) -> Ptr<'a, T, (I::Aliasing, A, I::Validity)> { + // SAFETY: The caller promises that `self`'s referent is + // well-aligned for `T` if required by `A` . + unsafe { self.assume_invariants() } + } + + /// Checks the `self`'s alignment at runtime, returning an aligned `Ptr` + /// on success. + #[inline] + pub fn try_into_aligned( + self, + ) -> Result<Ptr<'a, T, (I::Aliasing, Aligned, I::Validity)>, AlignmentError<Self, T>> + where + T: Sized, + { + if let Err(err) = + crate::util::validate_aligned_to::<_, T>(self.as_inner().as_non_null()) + { + return Err(err.with_src(self)); + } + + // SAFETY: We just checked the alignment. + Ok(unsafe { self.assume_alignment::<Aligned>() }) + } + + /// Recalls that `self`'s referent is validly-aligned for `T`. + #[inline] + // FIXME(#859): Reconsider the name of this method before making it + // public. + #[must_use] + pub fn bikeshed_recall_aligned(self) -> Ptr<'a, T, (I::Aliasing, Aligned, I::Validity)> + where + T: crate::Unaligned, + { + // SAFETY: The bound `T: Unaligned` ensures that `T` has no + // non-trivial alignment requirement. + unsafe { self.assume_alignment::<Aligned>() } + } + + /// Assumes that `self`'s referent conforms to the validity requirement + /// of `V`. + /// + /// # Safety + /// + /// The caller promises that `self`'s referent conforms to the validity + /// requirement of `V`. + #[must_use] + #[inline] + pub unsafe fn assume_validity<V: Validity>( + self, + ) -> Ptr<'a, T, (I::Aliasing, I::Alignment, V)> { + // SAFETY: The caller promises that `self`'s referent conforms to + // the validity requirement of `V`. + unsafe { self.assume_invariants() } + } + + /// A shorthand for `self.assume_validity<invariant::Initialized>()`. + /// + /// # Safety + /// + /// The caller promises to uphold the safety preconditions of + /// `self.assume_validity<invariant::Initialized>()`. + #[must_use] + #[inline] + pub unsafe fn assume_initialized( + self, + ) -> Ptr<'a, T, (I::Aliasing, I::Alignment, Initialized)> { + // SAFETY: The caller has promised to uphold the safety + // preconditions. + unsafe { self.assume_validity::<Initialized>() } + } + + /// A shorthand for `self.assume_validity<Valid>()`. + /// + /// # Safety + /// + /// The caller promises to uphold the safety preconditions of + /// `self.assume_validity<Valid>()`. + #[must_use] + #[inline] + pub unsafe fn assume_valid(self) -> Ptr<'a, T, (I::Aliasing, I::Alignment, Valid)> { + // SAFETY: The caller has promised to uphold the safety + // preconditions. + unsafe { self.assume_validity::<Valid>() } + } + + /// Checks that `self`'s referent is validly initialized for `T`, + /// returning a `Ptr` with `Valid` on success. + /// + /// # Panics + /// + /// This method will panic if + /// [`T::is_bit_valid`][TryFromBytes::is_bit_valid] panics. + /// + /// # Safety + /// + /// On error, unsafe code may rely on this method's returned + /// `ValidityError` containing `self`. + #[inline] + pub fn try_into_valid<R, S>( + mut self, + ) -> Result<Ptr<'a, T, (I::Aliasing, I::Alignment, Valid)>, ValidityError<Self, T>> + where + T: TryFromBytes + + Read<I::Aliasing, R> + + TryTransmuteFromPtr<T, I::Aliasing, I::Validity, Valid, IdCast, S>, + ReadOnly<T>: Read<I::Aliasing, R>, + I::Aliasing: Reference, + I: Invariants<Validity = Initialized>, + { + // This call may panic. If that happens, it doesn't cause any + // soundness issues, as we have not generated any invalid state + // which we need to fix before returning. + if T::is_bit_valid(self.reborrow().transmute::<_, _, _>().reborrow_shared()) { + // SAFETY: If `T::is_bit_valid`, code may assume that `self` + // contains a bit-valid instance of `T`. By `T: + // TryTransmuteFromPtr<T, I::Aliasing, I::Validity, Valid>`, so + // long as `self`'s referent conforms to the `Valid` validity + // for `T` (which we just confirmed), then this transmute is + // sound. + Ok(unsafe { self.assume_valid() }) + } else { + Err(ValidityError::new(self)) + } + } + + /// Forgets that `self`'s referent is validly-aligned for `T`. + #[inline] + #[must_use] + pub fn forget_aligned(self) -> Ptr<'a, T, (I::Aliasing, Unaligned, I::Validity)> { + // SAFETY: `Unaligned` is less restrictive than `Aligned`. + unsafe { self.assume_invariants() } + } + } +} + +/// Casts of the referent type. +#[cfg_attr(not(zerocopy_unstable_ptr), allow(unreachable_pub))] +pub use _casts::TryWithError; +mod _casts { + use core::cell::UnsafeCell; + + use super::*; + use crate::{ + pointer::cast::{AsBytesCast, Cast}, + HasTag, ProjectField, + }; + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + { + /// Casts to a different referent type without checking interior + /// mutability. + /// + /// Callers should prefer [`cast`][Ptr::cast] where possible. + /// + /// # Safety + /// + /// If `I::Aliasing` is [`Shared`], it must not be possible for safe + /// code, operating on a `&T` and `&U` with the same referent + /// simultaneously, to cause undefined behavior. + #[inline] + #[must_use] + pub unsafe fn cast_unchecked<U, C: Cast<T, U>>( + self, + ) -> Ptr<'a, U, (I::Aliasing, Unaligned, I::Validity)> + where + U: 'a + CastableFrom<T, I::Validity, I::Validity> + ?Sized, + { + // SAFETY: + // - By `C: Cast`, `C` preserves the address of the referent. + // - If `I::Aliasing` is [`Shared`], the caller promises that it + // is not possible for safe code, operating on a `&T` and `&U` + // with the same referent simultaneously, to cause undefined + // behavior. + // - By `U: CastableFrom<T, I::Validity, I::Validity>`, + // `I::Validity` is either `Uninit` or `Initialized`. In both + // cases, the bit validity `I::Validity` has the same semantics + // regardless of referent type. In other words, the set of allowed + // referent values for `Ptr<T, (_, _, I::Validity)>` and `Ptr<U, + // (_, _, I::Validity)>` are identical. As a consequence, neither + // `self` nor the returned `Ptr` can be used to write values which + // are invalid for the other. + unsafe { self.project_transmute_unchecked::<_, _, C>() } + } + + /// Casts to a different referent type. + #[inline] + #[must_use] + pub fn cast<U, C, R>(self) -> Ptr<'a, U, (I::Aliasing, Unaligned, I::Validity)> + where + T: MutationCompatible<U, I::Aliasing, I::Validity, I::Validity, R>, + U: 'a + ?Sized + CastableFrom<T, I::Validity, I::Validity>, + C: Cast<T, U>, + { + // SAFETY: Because `T: MutationCompatible<U, I::Aliasing, R>`, one + // of the following holds: + // - `T: Read<I::Aliasing>` and `U: Read<I::Aliasing>`, in which + // case one of the following holds: + // - `I::Aliasing` is `Exclusive` + // - `T` and `U` are both `Immutable` + // - It is sound for safe code to operate on `&T` and `&U` with the + // same referent simultaneously. + unsafe { self.cast_unchecked::<_, C>() } + } + + #[inline(always)] + pub fn project<F, const VARIANT_ID: i128, const FIELD_ID: i128>( + mut self, + ) -> Result<Ptr<'a, T::Type, T::Invariants>, T::Error> + where + T: ProjectField<F, I, VARIANT_ID, FIELD_ID>, + I::Aliasing: Reference, + { + use crate::pointer::cast::Projection; + match T::is_projectable(self.reborrow().project_tag()) { + Ok(()) => { + let inner = self.as_inner(); + let projected = inner.project::<_, Projection<F, VARIANT_ID, FIELD_ID>>(); + // SAFETY: By `T: ProjectField<F, I, VARIANT_ID, FIELD_ID>`, + // for `self: Ptr<'_, T, I>` such that `T::is_projectable` + // (which we've verified in this match arm), + // `T::project(self.as_inner())` conforms to + // `T::Invariants`. The `projected` pointer satisfies these + // invariants because it is produced by way of an + // abstraction that is equivalent to + // `T::project(ptr.as_inner())`: by invariant on + // `PtrInner::project`, `projected` is guaranteed to address + // the subset of the bytes of `inner`'s referent addressed + // by `Projection::project(inner)`, and by invariant on + // `Projection`, `Projection::project` is implemented by + // delegating to an implementation of `HasField::project`. + Ok(unsafe { Ptr::from_inner(projected) }) + } + Err(err) => Err(err), + } + } + + #[must_use] + #[inline(always)] + pub(crate) fn project_tag(self) -> Ptr<'a, T::Tag, I> + where + T: HasTag, + { + // SAFETY: By invariant on `Self::ProjectToTag`, this is a sound + // projection. + let tag = unsafe { self.project_transmute_unchecked::<_, _, T::ProjectToTag>() }; + // SAFETY: By invariant on `Self::ProjectToTag`, the projected + // pointer has the same alignment as `ptr`. + let tag = unsafe { tag.assume_alignment() }; + tag.unify_invariants() + } + + /// Attempts to transform the pointer, restoring the original on + /// failure. + /// + /// # Safety + /// + /// If `I::Aliasing != Shared`, then if `f` returns `Err(err)`, no copy + /// of `f`'s argument must exist outside of `err`. + #[inline(always)] + pub(crate) unsafe fn try_with_unchecked<U, J, E, F>( + self, + f: F, + ) -> Result<Ptr<'a, U, J>, E::Mapped> + where + U: 'a + ?Sized, + J: Invariants<Aliasing = I::Aliasing>, + E: TryWithError<Self>, + F: FnOnce(Ptr<'a, T, I>) -> Result<Ptr<'a, U, J>, E>, + { + let old_inner = self.as_inner(); + #[rustfmt::skip] + let res = f(self).map_err(#[inline(always)] move |err: E| { + err.map(#[inline(always)] |src| { + drop(src); + + // SAFETY: + // 0. Aliasing is either `Shared` or `Exclusive`: + // - If aliasing is `Shared`, then it cannot violate + // aliasing make another copy of this pointer (in fact, + // using `I::Aliasing = Shared`, we could have just + // cloned `self`). + // - If aliasing is `Exclusive`, then `f` is not allowed + // to make another copy of `self`. In `map_err`, we are + // consuming the only value in the returned `Result`. + // By invariant on `E: TryWithError<Self>`, that `err: + // E` only contains a single `Self` and no other + // non-ZST fields which could be `Ptr`s or references + // to `self`'s referent. By the same invariant, `map` + // consumes this single `Self` and passes it to this + // closure. Since `self` was, by invariant on + // `Exclusive`, the only `Ptr` or reference live for + // `'a` with this referent, and since we `drop(src)` + // above, there are no copies left, and so we are + // creating the only copy. + // 1. `self` conforms to `I::Aliasing` by invariant on + // `Ptr`, and `old_inner` has the same address, so it + // does too. + // 2. `f` could not have violated `self`'s validity without + // itself being unsound. Assuming that `f` is sound, the + // referent of `self` is still valid for `T`. + unsafe { Ptr::from_inner(old_inner) } + }) + }); + res + } + + /// Attempts to transform the pointer, restoring the original on + /// failure. + #[inline(always)] + pub fn try_with<U, J, E, F>(self, f: F) -> Result<Ptr<'a, U, J>, E::Mapped> + where + U: 'a + ?Sized, + J: Invariants<Aliasing = I::Aliasing>, + E: TryWithError<Self>, + F: FnOnce(Ptr<'a, T, I>) -> Result<Ptr<'a, U, J>, E>, + I: Invariants<Aliasing = Shared>, + { + // SAFETY: `I::Aliasing = Shared`, so the safety condition does not + // apply. + unsafe { self.try_with_unchecked(f) } + } + } + + /// # Safety + /// + /// `Self` only contains a single `Self::Inner`, and `Self::Mapped` only + /// contains a single `MappedInner`. Other than that, `Self` and + /// `Self::Mapped` contain no non-ZST fields. + /// + /// `map` must pass ownership of `self`'s sole `Self::Inner` to `f`. + pub unsafe trait TryWithError<MappedInner> { + type Inner; + type Mapped; + fn map<F: FnOnce(Self::Inner) -> MappedInner>(self, f: F) -> Self::Mapped; + } + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + KnownLayout + ?Sized, + I: Invariants, + { + /// Casts this pointer-to-initialized into a pointer-to-bytes. + #[allow(clippy::wrong_self_convention)] + #[must_use] + #[inline] + pub fn as_bytes<R>(self) -> Ptr<'a, [u8], (I::Aliasing, Aligned, Valid)> + where + [u8]: TransmuteFromPtr<T, I::Aliasing, I::Validity, Valid, AsBytesCast, R>, + { + self.transmute_with::<[u8], Valid, AsBytesCast, _>().bikeshed_recall_aligned() + } + } + + impl<'a, T, I, const N: usize> Ptr<'a, [T; N], I> + where + T: 'a, + I: Invariants, + { + /// Casts this pointer-to-array into a slice. + #[allow(clippy::wrong_self_convention)] + #[inline] + #[must_use] + pub fn as_slice(self) -> Ptr<'a, [T], I> { + let slice = self.as_inner().as_slice(); + // SAFETY: Note that, by post-condition on `PtrInner::as_slice`, + // `slice` refers to the same byte range as `self.as_inner()`. + // + // 0. Thus, `slice` conforms to the aliasing invariant of + // `I::Aliasing` because `self` does. + // 1. By the above lemma, `slice` conforms to the alignment + // invariant of `I::Alignment` because `self` does. + // 2. Since `[T; N]` and `[T]` have the same bit validity [1][2], + // and since `self` and the returned `Ptr` have the same validity + // invariant, neither `self` nor the returned `Ptr` can be used + // to write a value to the referent which violates the other's + // validity invariant. + // + // [1] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#array-layout: + // + // An array of `[T; N]` has a size of `size_of::<T>() * N` and the + // same alignment of `T`. Arrays are laid out so that the + // zero-based `nth` element of the array is offset from the start + // of the array by `n * size_of::<T>()` bytes. + // + // ... + // + // Slices have the same layout as the section of the array they + // slice. + // + // [2] Per https://doc.rust-lang.org/1.81.0/reference/types/array.html#array-types: + // + // All elements of arrays are always initialized + unsafe { Ptr::from_inner(slice) } + } + } + + /// For caller convenience, these methods are generic over alignment + /// invariant. In practice, the referent is always well-aligned, because the + /// alignment of `[u8]` is 1. + impl<'a, I> Ptr<'a, [u8], I> + where + I: Invariants<Validity = Valid>, + { + /// Attempts to cast `self` to a `U` using the given cast type. + /// + /// If `U` is a slice DST and pointer metadata (`meta`) is provided, + /// then the cast will only succeed if it would produce an object with + /// the given metadata. + /// + /// Returns `None` if the resulting `U` would be invalidly-aligned, if + /// no `U` can fit in `self`, or if the provided pointer metadata + /// describes an invalid instance of `U`. On success, returns a pointer + /// to the largest-possible `U` which fits in `self`. + /// + /// # Safety + /// + /// The caller may assume that this implementation is correct, and may + /// rely on that assumption for the soundness of their code. In + /// particular, the caller may assume that, if `try_cast_into` returns + /// `Some((ptr, remainder))`, then `ptr` and `remainder` refer to + /// non-overlapping byte ranges within `self`, and that `ptr` and + /// `remainder` entirely cover `self`. Finally: + /// - If this is a prefix cast, `ptr` has the same address as `self`. + /// - If this is a suffix cast, `remainder` has the same address as + /// `self`. + #[inline(always)] + pub fn try_cast_into<U, R>( + self, + cast_type: CastType, + meta: Option<U::PointerMetadata>, + ) -> Result< + (Ptr<'a, U, (I::Aliasing, Aligned, Initialized)>, Ptr<'a, [u8], I>), + CastError<Self, U>, + > + where + I::Aliasing: Reference, + U: 'a + ?Sized + KnownLayout + Read<I::Aliasing, R>, + { + let (inner, remainder) = self.as_inner().try_cast_into(cast_type, meta).map_err( + #[inline(always)] + |err| { + err.map_src( + #[inline(always)] + |inner| + // SAFETY: `PtrInner::try_cast_into` promises to return its + // original argument on error, which was originally produced + // by `self.as_inner()`, which is guaranteed to satisfy + // `Ptr`'s invariants. + unsafe { Ptr::from_inner(inner) }, + ) + }, + )?; + + // SAFETY: + // 0. Since `U: Read<I::Aliasing, _>`, either: + // - `I::Aliasing` is `Exclusive`, in which case both `src` and + // `ptr` conform to `Exclusive` + // - `I::Aliasing` is `Shared` and `U` is `Immutable` (we already + // know that `[u8]: Immutable`). In this case, neither `U` nor + // `[u8]` permit mutation, and so `Shared` aliasing is + // satisfied. + // 1. `ptr` conforms to the alignment invariant of `Aligned` because + // it is derived from `try_cast_into`, which promises that the + // object described by `target` is validly aligned for `U`. + // 2. By trait bound, `self` - and thus `target` - is a bit-valid + // `[u8]`. `Ptr<[u8], (_, _, Valid)>` and `Ptr<_, (_, _, + // Initialized)>` have the same bit validity, and so neither + // `self` nor `res` can be used to write a value to the referent + // which violates the other's validity invariant. + let res = unsafe { Ptr::from_inner(inner) }; + + // SAFETY: + // 0. `self` and `remainder` both have the type `[u8]`. Thus, they + // have `UnsafeCell`s at the same locations. Type casting does + // not affect aliasing. + // 1. `[u8]` has no alignment requirement. + // 2. `self` has validity `Valid` and has type `[u8]`. Since + // `remainder` references a subset of `self`'s referent, it is + // also a bit-valid `[u8]`. Thus, neither `self` nor `remainder` + // can be used to write a value to the referent which violates + // the other's validity invariant. + let remainder = unsafe { Ptr::from_inner(remainder) }; + + Ok((res, remainder)) + } + + /// Attempts to cast `self` into a `U`, failing if all of the bytes of + /// `self` cannot be treated as a `U`. + /// + /// In particular, this method fails if `self` is not validly-aligned + /// for `U` or if `self`'s size is not a valid size for `U`. + /// + /// # Safety + /// + /// On success, the caller may assume that the returned pointer + /// references the same byte range as `self`. + #[allow(unused)] + #[inline(always)] + pub fn try_cast_into_no_leftover<U, R>( + self, + meta: Option<U::PointerMetadata>, + ) -> Result<Ptr<'a, U, (I::Aliasing, Aligned, Initialized)>, CastError<Self, U>> + where + I::Aliasing: Reference, + U: 'a + ?Sized + KnownLayout + Read<I::Aliasing, R>, + [u8]: Read<I::Aliasing, R>, + { + // SAFETY: The provided closure returns the only copy of `slf`. + unsafe { + self.try_with_unchecked( + #[inline(always)] + |slf| match slf.try_cast_into(CastType::Prefix, meta) { + Ok((slf, remainder)) => { + if remainder.is_empty() { + Ok(slf) + } else { + Err(CastError::Size(SizeError::<_, U>::new(()))) + } + } + Err(err) => Err(err.map_src( + #[inline(always)] + |_slf| (), + )), + }, + ) + } + } + } + + impl<'a, T, I> Ptr<'a, UnsafeCell<T>, I> + where + T: 'a + ?Sized, + I: Invariants<Aliasing = Exclusive>, + { + /// Converts this `Ptr` into a pointer to the underlying data. + /// + /// This call borrows the `UnsafeCell` mutably (at compile-time) which + /// guarantees that we possess the only reference. + /// + /// This is like [`UnsafeCell::get_mut`], but for `Ptr`. + /// + /// [`UnsafeCell::get_mut`]: core::cell::UnsafeCell::get_mut + #[must_use] + #[inline(always)] + pub fn get_mut(self) -> Ptr<'a, T, I> { + // SAFETY: As described below, `UnsafeCell<T>` has the same size + // as `T: ?Sized` (same static size or same DST layout). Thus, + // `*const UnsafeCell<T> as *const T` is a size-preserving cast. + define_cast!(unsafe { Cast<T: ?Sized> = UnsafeCell<T> => T }); + + // SAFETY: + // - Aliasing is `Exclusive`, and so we are not required to promise + // anything about the locations of `UnsafeCell`s. + // - `UnsafeCell<T>` has the same bit validity as `T` [1]. + // Technically the term "representation" doesn't guarantee this, + // but the subsequent sentence in the documentation makes it clear + // that this is the intention. + // + // By invariant on `Validity`, since `T` and `UnsafeCell<T>` have + // the same bit validity, then the set of values which may appear + // in the referent of a `Ptr<T, (_, _, V)>` is the same as the set + // which may appear in the referent of a `Ptr<UnsafeCell<T>, (_, + // _, V)>`. Thus, neither `self` nor `ptr` may be used to write a + // value to the referent which would violate the other's validity + // invariant. + // + // [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout: + // + // `UnsafeCell<T>` has the same in-memory representation as its + // inner type `T`. A consequence of this guarantee is that it is + // possible to convert between `T` and `UnsafeCell<T>`. + let ptr = unsafe { self.project_transmute_unchecked::<_, _, Cast>() }; + + // SAFETY: `UnsafeCell<T>` has the same alignment as `T` [1], + // and so if `self` is guaranteed to be aligned, then so is the + // returned `Ptr`. + // + // [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout: + // + // `UnsafeCell<T>` has the same in-memory representation as + // its inner type `T`. A consequence of this guarantee is that + // it is possible to convert between `T` and `UnsafeCell<T>`. + let ptr = unsafe { ptr.assume_alignment::<I::Alignment>() }; + ptr.unify_invariants() + } + } +} + +/// Projections through the referent. +mod _project { + use super::*; + + impl<'a, T, I> Ptr<'a, [T], I> + where + T: 'a, + I: Invariants, + I::Aliasing: Reference, + { + /// Iteratively projects the elements `Ptr<T>` from `Ptr<[T]>`. + #[inline] + pub fn iter(self) -> impl Iterator<Item = Ptr<'a, T, I>> { + // SAFETY: + // 0. `elem` conforms to the aliasing invariant of `I::Aliasing`: + // - `Exclusive`: `self` is consumed by value, and therefore + // cannot be used to access the slice while any yielded + // element `Ptr` is live. Each non-zero-sized element is a + // disjoint byte range within the slice, and zero-sized + // elements address no bytes, so distinct yielded element + // `Ptr`s do not alias each other. + // - `Shared`: It is sound for multiple shared `Ptr`s to exist + // simultaneously which reference the same memory. + // 1. `elem`, conditionally, conforms to the validity invariant of + // `I::Alignment`. If `elem` is projected from data well-aligned + // for `[T]`, `elem` will be valid for `T`. + // 2. `elem` conforms to the validity invariant of `I::Validity`. + // Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#array-layout: + // + // Slices have the same layout as the section of the array they + // slice. + // + // Arrays are laid out so that the zero-based `nth` element of + // the array is offset from the start of the array by `n * + // size_of::<T>()` bytes. Thus, `elem` addresses a valid `T` + // within the slice. Since `self` satisfies `I::Validity`, `elem` + // also satisfies `I::Validity`. + self.as_inner().iter().map( + #[inline(always)] + |elem| unsafe { Ptr::from_inner(elem) }, + ) + } + } + + #[allow(clippy::needless_lifetimes)] + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized + KnownLayout<PointerMetadata = usize>, + I: Invariants, + { + /// The number of slice elements in the object referenced by `self`. + #[inline] + #[must_use] + pub fn len(&self) -> usize { + self.as_inner().meta().get() + } + + /// Returns `true` if the slice pointer has a length of 0. + #[inline] + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + } +} + +#[cfg(test)] +mod tests { + use core::mem::{self, MaybeUninit}; + + use super::*; + #[allow(unused)] // Needed on our MSRV, but considered unused on later toolchains. + use crate::util::AsAddress; + use crate::{pointer::BecauseImmutable, util::testutil::AU64, FromBytes, Immutable}; + + mod test_ptr_try_cast_into_soundness { + use super::*; + + // This test is designed so that if `Ptr::try_cast_into_xxx` are + // buggy, it will manifest as unsoundness that Miri can detect. + + // - If `size_of::<T>() == 0`, `N == 4` + // - Else, `N == 4 * size_of::<T>()` + // + // Each test will be run for each metadata in `metas`. + fn test<T, I, const N: usize>(metas: I) + where + T: ?Sized + KnownLayout + Immutable + FromBytes, + I: IntoIterator<Item = Option<T::PointerMetadata>> + Clone, + { + let mut bytes = [MaybeUninit::<u8>::uninit(); N]; + let initialized = [MaybeUninit::new(0u8); N]; + for start in 0..=bytes.len() { + for end in start..=bytes.len() { + // Set all bytes to uninitialized other than those in + // the range we're going to pass to `try_cast_from`. + // This allows Miri to detect out-of-bounds reads + // because they read uninitialized memory. Without this, + // some out-of-bounds reads would still be in-bounds of + // `bytes`, and so might spuriously be accepted. + bytes = [MaybeUninit::<u8>::uninit(); N]; + let bytes = &mut bytes[start..end]; + // Initialize only the byte range we're going to pass to + // `try_cast_from`. + bytes.copy_from_slice(&initialized[start..end]); + + let bytes = { + let bytes: *const [MaybeUninit<u8>] = bytes; + #[allow(clippy::as_conversions)] + let bytes = bytes as *const [u8]; + // SAFETY: We just initialized these bytes to valid + // `u8`s. + unsafe { &*bytes } + }; + + // SAFETY: The bytes in `slf` must be initialized. + unsafe fn validate_and_get_len< + T: ?Sized + KnownLayout + FromBytes + Immutable, + >( + slf: Ptr<'_, T, (Shared, Aligned, Initialized)>, + ) -> usize { + let t = slf.recall_validity().as_ref(); + + let bytes = { + let len = mem::size_of_val(t); + let t: *const T = t; + // SAFETY: + // - We know `t`'s bytes are all initialized + // because we just read it from `slf`, which + // points to an initialized range of bytes. If + // there's a bug and this doesn't hold, then + // that's exactly what we're hoping Miri will + // catch! + // - Since `T: FromBytes`, `T` doesn't contain + // any `UnsafeCell`s, so it's okay for `t: T` + // and a `&[u8]` to the same memory to be + // alive concurrently. + unsafe { core::slice::from_raw_parts(t.cast::<u8>(), len) } + }; + + // This assertion ensures that `t`'s bytes are read + // and compared to another value, which in turn + // ensures that Miri gets a chance to notice if any + // of `t`'s bytes are uninitialized, which they + // shouldn't be (see the comment above). + assert_eq!(bytes, vec![0u8; bytes.len()]); + + mem::size_of_val(t) + } + + for meta in metas.clone().into_iter() { + for cast_type in [CastType::Prefix, CastType::Suffix] { + if let Ok((slf, remaining)) = Ptr::from_ref(bytes) + .try_cast_into::<T, BecauseImmutable>(cast_type, meta) + { + // SAFETY: All bytes in `bytes` have been + // initialized. + let len = unsafe { validate_and_get_len(slf) }; + assert_eq!(remaining.len(), bytes.len() - len); + #[allow(unstable_name_collisions)] + let bytes_addr = bytes.as_ptr().addr(); + #[allow(unstable_name_collisions)] + let remaining_addr = remaining.as_inner().as_ptr().addr(); + match cast_type { + CastType::Prefix => { + assert_eq!(remaining_addr, bytes_addr + len) + } + CastType::Suffix => assert_eq!(remaining_addr, bytes_addr), + } + + if let Some(want) = meta { + let got = + KnownLayout::pointer_to_metadata(slf.as_inner().as_ptr()); + assert_eq!(got, want); + } + } + } + + if let Ok(slf) = Ptr::from_ref(bytes) + .try_cast_into_no_leftover::<T, BecauseImmutable>(meta) + { + // SAFETY: All bytes in `bytes` have been + // initialized. + let len = unsafe { validate_and_get_len(slf) }; + assert_eq!(len, bytes.len()); + + if let Some(want) = meta { + let got = KnownLayout::pointer_to_metadata(slf.as_inner().as_ptr()); + assert_eq!(got, want); + } + } + } + } + } + } + + #[derive(FromBytes, KnownLayout, Immutable)] + #[repr(C)] + struct SliceDst<T> { + a: u8, + trailing: [T], + } + + // Each test case becomes its own `#[test]` function. We do this because + // this test in particular takes far, far longer to execute under Miri + // than all of our other tests combined. Previously, we had these + // execute sequentially in a single test function. We run Miri tests in + // parallel in CI, but this test being sequential meant that most of + // that parallelism was wasted, as all other tests would finish in a + // fraction of the total execution time, leaving this test to execute on + // a single thread for the remainder of the test. By putting each test + // case in its own function, we permit better use of available + // parallelism. + macro_rules! test { + ($test_name:ident: $ty:ty) => { + #[test] + #[allow(non_snake_case)] + fn $test_name() { + const S: usize = core::mem::size_of::<$ty>(); + const N: usize = if S == 0 { 4 } else { S * 4 }; + test::<$ty, _, N>([None]); + + // If `$ty` is a ZST, then we can't pass `None` as the + // pointer metadata, or else computing the correct trailing + // slice length will panic. + if S == 0 { + test::<[$ty], _, N>([Some(0), Some(1), Some(2), Some(3)]); + test::<SliceDst<$ty>, _, N>([Some(0), Some(1), Some(2), Some(3)]); + } else { + test::<[$ty], _, N>([None, Some(0), Some(1), Some(2), Some(3)]); + test::<SliceDst<$ty>, _, N>([None, Some(0), Some(1), Some(2), Some(3)]); + } + } + }; + ($ty:ident) => { + test!($ty: $ty); + }; + ($($ty:ident),*) => { $(test!($ty);)* } + } + + test!(empty_tuple: ()); + test!(u8, u16, u32, u64, usize, AU64); + test!(i8, i16, i32, i64, isize); + test!(f32, f64); + } + + #[test] + fn test_try_cast_into_explicit_count() { + macro_rules! test { + ($ty:ty, $bytes:expr, $elems:expr, $expect:expr) => {{ + let bytes = [0u8; $bytes]; + let ptr = Ptr::from_ref(&bytes[..]); + let res = + ptr.try_cast_into::<$ty, BecauseImmutable>(CastType::Prefix, Some($elems)); + if let Some(expect) = $expect { + let (ptr, _) = res.unwrap(); + assert_eq!(KnownLayout::pointer_to_metadata(ptr.as_inner().as_ptr()), expect); + } else { + let _ = res.unwrap_err(); + } + }}; + } + + #[derive(KnownLayout, Immutable)] + #[repr(C)] + struct ZstDst { + u: [u8; 8], + slc: [()], + } + + test!(ZstDst, 8, 0, Some(0)); + test!(ZstDst, 7, 0, None); + + test!(ZstDst, 8, usize::MAX, Some(usize::MAX)); + test!(ZstDst, 7, usize::MAX, None); + + #[derive(KnownLayout, Immutable)] + #[repr(C)] + struct Dst { + u: [u8; 8], + slc: [u8], + } + + test!(Dst, 8, 0, Some(0)); + test!(Dst, 7, 0, None); + + test!(Dst, 9, 1, Some(1)); + test!(Dst, 8, 1, None); + + // If we didn't properly check for overflow, this would cause the + // metadata to overflow to 0, and thus the cast would spuriously + // succeed. + test!(Dst, 8, usize::MAX - 8 + 1, None); + } + + #[test] + fn test_try_cast_into_no_leftover_restores_original_slice() { + let bytes = [0u8; 4]; + let ptr = Ptr::from_ref(&bytes[..]); + let res = ptr.try_cast_into_no_leftover::<[u8; 2], BecauseImmutable>(None); + match res { + Ok(_) => panic!("should have failed due to leftover bytes"), + Err(CastError::Size(e)) => { + assert_eq!(e.into_src().len(), 4, "Should return original slice length"); + } + Err(e) => panic!("wrong error type: {:?}", e), + } + } + + #[test] + fn test_iter_exclusive_yields_disjoint_ptrs() { + let mut arr = [0u8, 1, 2, 3]; + + { + let mut iter = Ptr::from_mut(&mut arr[..]).iter(); + let first = iter.next().unwrap().as_mut(); + let second = iter.next().unwrap().as_mut(); + + *first = 10; + *second = 20; + *first = 30; + } + + assert_eq!(arr, [30, 20, 2, 3]); + } +} diff --git a/rust/zerocopy/src/pointer/transmute.rs b/rust/zerocopy/src/pointer/transmute.rs new file mode 100644 index 000000000000..a534984b70d3 --- /dev/null +++ b/rust/zerocopy/src/pointer/transmute.rs @@ -0,0 +1,522 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2025 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +#![allow(missing_docs)] + +use core::{ + cell::{Cell, UnsafeCell}, + mem::{ManuallyDrop, MaybeUninit}, + num::Wrapping, +}; + +use crate::{ + pointer::{ + cast::{self, CastExact, CastSizedExact}, + invariant::*, + }, + FromBytes, Immutable, IntoBytes, Unalign, +}; + +/// Transmutations which are sound to attempt, conditional on validating the bit +/// validity of the destination type. +/// +/// If a `Ptr` transmutation is `TryTransmuteFromPtr`, then it is sound to +/// perform that transmutation so long as some additional mechanism is used to +/// validate that the referent is bit-valid for the destination type. That +/// validation mechanism could be a type bound (such as `TransmuteFrom`) or a +/// runtime validity check. +/// +/// # Safety +/// +/// ## Post-conditions +/// +/// Given `Dst: TryTransmuteFromPtr<Src, A, SV, DV, C, _>`, callers may assume +/// the following: +/// +/// Given `src: Ptr<'a, Src, (A, _, SV)>`, if the referent of `src` is +/// `DV`-valid for `Dst`, then it is sound to transmute `src` into `dst: Ptr<'a, +/// Dst, (A, Unaligned, DV)>` using `C`. +/// +/// ## Pre-conditions +/// +/// Given `src: Ptr<Src, (A, _, SV)>` and `dst: Ptr<Dst, (A, Unaligned, DV)>`, +/// `Dst: TryTransmuteFromPtr<Src, A, SV, DV, C, _>` is sound if all of the +/// following hold: +/// - Forwards transmutation: Either of the following hold: +/// - So long as `dst` is active, no mutation of `dst`'s referent is allowed +/// except via `dst` itself +/// - The set of `DV`-valid referents of `dst` is a superset of the set of +/// `SV`-valid referents of `src` (NOTE: this condition effectively bans +/// shrinking or overwriting transmutes, which cannot satisfy this +/// condition) +/// - Reverse transmutation: Either of the following hold: +/// - `dst` does not permit mutation of its referent +/// - The set of `DV`-valid referents of `dst` is a subset of the set of +/// `SV`-valid referents of `src` (NOTE: this condition effectively bans +/// shrinking or overwriting transmutes, which cannot satisfy this +/// condition) +/// - No safe code, given access to `src` and `dst`, can cause undefined +/// behavior: Any of the following hold: +/// - `A` is `Exclusive` +/// - `Src: Immutable` and `Dst: Immutable` +/// - It is sound for shared code to operate on a `&Src` and `&Dst` which +/// reference the same byte range at the same time +/// +/// ## Proof +/// +/// Given: +/// - `src: Ptr<'a, Src, (A, _, SV)>` +/// - `src`'s referent is `DV`-valid for `Dst` +/// +/// We are trying to prove that it is sound to perform a cast from `src` to a +/// `dst: Ptr<'a, Dst, (A, Unaligned, DV)>` using `C`. We need to prove that +/// such a cast does not violate any of `src`'s invariants, and that it +/// satisfies all invariants of the destination `Ptr` type. +/// +/// First, by `C: CastExact`, `src`'s address is unchanged, so it still satisfies +/// its alignment. Since `dst`'s alignment is `Unaligned`, it trivially satisfies +/// its alignment. +/// +/// Second, aliasing is either `Exclusive` or `Shared`: +/// - If it is `Exclusive`, then both `src` and `dst` satisfy `Exclusive` +/// aliasing trivially: since `src` and `dst` have the same lifetime, `src` is +/// inaccessible so long as `dst` is alive, and no other live `Ptr`s or +/// references may reference the same referent. +/// - If it is `Shared`, then either: +/// - `Src: Immutable` and `Dst: Immutable`, and so neither `src` nor `dst` +/// permit interior mutation. +/// - It is explicitly sound for safe code to operate on a `&Src` and a `&Dst` +/// pointing to the same byte range at the same time. +/// +/// Third, `src`'s validity is satisfied. By invariant, `src`'s referent began +/// as an `SV`-valid `Src`. It is guaranteed to remain so, as either of the +/// following hold: +/// - `dst` does not permit mutation of its referent. +/// - The set of `DV`-valid referents of `dst` is a subset of the set of +/// `SV`-valid referents of `src`. Thus, any value written via `dst` is +/// guaranteed to be an `SV`-valid referent of `src`. +/// +/// Fourth, `dst`'s validity is satisfied. It is a given of this proof that the +/// referent is `DV`-valid for `Dst`. It is guaranteed to remain so, as either +/// of the following hold: +/// - So long as `dst` is active, no mutation of the referent is allowed except +/// via `dst` itself. +/// - The set of `DV`-valid referents of `dst` is a superset of the set of +/// `SV`-valid referents of `src`. Thus, any value written via `src` is +/// guaranteed to be a `DV`-valid referent of `dst`. +pub unsafe trait TryTransmuteFromPtr< + Src: ?Sized, + A: Aliasing, + SV: Validity, + DV: Validity, + C: CastExact<Src, Self>, + R, +> +{ +} + +#[allow(missing_copy_implementations, missing_debug_implementations)] +pub enum BecauseMutationCompatible {} + +// SAFETY: +// - Forwards transmutation: By `Dst: MutationCompatible<Src, A, SV, DV, _>`, we +// know that at least one of the following holds: +// - So long as `dst: Ptr<Dst>` is active, no mutation of its referent is +// allowed except via `dst` itself if either of the following hold: +// - Aliasing is `Exclusive`, in which case, so long as the `Dst` `Ptr` +// exists, no mutation is permitted except via that `Ptr` +// - Aliasing is `Shared`, `Src: Immutable`, and `Dst: Immutable`, in which +// case no mutation is possible via either `Ptr` +// - Since the underlying cast is size-preserving, `dst` addresses the same +// referent as `src`. By `Dst: TransmuteFrom<Src, SV, DV>`, the set of +// `DV`-valid referents of `dst` is a superset of the set of `SV`-valid +// referents of `src`. +// - Reverse transmutation: Since the underlying cast is size-preserving, `dst` +// addresses the same referent as `src`. By `Src: TransmuteFrom<Dst, DV, SV>`, +// the set of `DV`-valid referents of `src` is a subset of the set of +// `SV`-valid referents of `dst`. +// - No safe code, given access to `src` and `dst`, can cause undefined +// behavior: By `Dst: MutationCompatible<Src, A, SV, DV, _>`, at least one of +// the following holds: +// - `A` is `Exclusive` +// - `Src: Immutable` and `Dst: Immutable` +// - `Dst: InvariantsEq<Src>`, which guarantees that `Src` and `Dst` have the +// same invariants, and permit interior mutation on the same byte ranges +unsafe impl<Src, Dst, SV, DV, A, C, R> + TryTransmuteFromPtr<Src, A, SV, DV, C, (BecauseMutationCompatible, R)> for Dst +where + A: Aliasing, + SV: Validity, + DV: Validity, + Src: TransmuteFrom<Dst, DV, SV> + ?Sized, + Dst: MutationCompatible<Src, A, SV, DV, R> + ?Sized, + C: CastExact<Src, Dst>, +{ +} + +// SAFETY: +// - Forwards transmutation: Since aliasing is `Shared` and `Src: Immutable`, +// `src` does not permit mutation of its referent. +// - Reverse transmutation: Since aliasing is `Shared` and `Dst: Immutable`, +// `dst` does not permit mutation of its referent. +// - No safe code, given access to `src` and `dst`, can cause undefined +// behavior: `Src: Immutable` and `Dst: Immutable` +unsafe impl<Src, Dst, SV, DV, C> TryTransmuteFromPtr<Src, Shared, SV, DV, C, BecauseImmutable> + for Dst +where + SV: Validity, + DV: Validity, + Src: Immutable + ?Sized, + Dst: Immutable + ?Sized, + C: CastExact<Src, Dst>, +{ +} + +/// Denotes that `src: Ptr<Src, (A, _, SV)>` and `dst: Ptr<Self, (A, _, DV)>`, +/// referencing the same referent at the same time, cannot be used by safe code +/// to break library safety invariants of `Src` or `Self`. +/// +/// # Safety +/// +/// At least one of the following must hold: +/// - `Src: Read<A, _>` and `Self: Read<A, _>` +/// - `Self: InvariantsEq<Src>`, and, for some `V`: +/// - `Dst: TransmuteFrom<Src, V, V>` +/// - `Src: TransmuteFrom<Dst, V, V>` +pub unsafe trait MutationCompatible<Src: ?Sized, A: Aliasing, SV, DV, R> {} + +#[allow(missing_copy_implementations, missing_debug_implementations)] +pub enum BecauseRead {} + +// SAFETY: `Src: Read<A, _>` and `Dst: Read<A, _>`. +unsafe impl<Src: ?Sized, Dst: ?Sized, A: Aliasing, SV: Validity, DV: Validity, R> + MutationCompatible<Src, A, SV, DV, (BecauseRead, R)> for Dst +where + Src: Read<A, R>, + Dst: Read<A, R>, +{ +} + +/// Denotes that two types have the same invariants. +/// +/// # Safety +/// +/// It is sound for safe code to operate on a `&T` and a `&Self` pointing to the +/// same referent at the same time - no such safe code can cause undefined +/// behavior. +pub unsafe trait InvariantsEq<T: ?Sized> {} + +// SAFETY: Trivially sound to have multiple `&T` pointing to the same referent. +unsafe impl<T: ?Sized> InvariantsEq<T> for T {} + +// SAFETY: `Dst: InvariantsEq<Src> + TransmuteFrom<Src, SV, DV>`, and `Src: +// TransmuteFrom<Dst, DV, SV>`. +unsafe impl<Src: ?Sized, Dst: ?Sized, A: Aliasing, SV: Validity, DV: Validity> + MutationCompatible<Src, A, SV, DV, BecauseInvariantsEq> for Dst +where + Src: TransmuteFrom<Dst, DV, SV>, + Dst: TransmuteFrom<Src, SV, DV> + InvariantsEq<Src>, +{ +} + +#[allow(missing_debug_implementations, missing_copy_implementations)] +pub enum BecauseInvariantsEq {} + +macro_rules! unsafe_impl_invariants_eq { + ($tyvar:ident => $t:ty, $u:ty) => {{ + crate::util::macros::__unsafe(); + // SAFETY: The caller promises that this is sound. + unsafe impl<$tyvar> InvariantsEq<$t> for $u {} + // SAFETY: The caller promises that this is sound. + unsafe impl<$tyvar> InvariantsEq<$u> for $t {} + }}; +} + +impl_transitive_transmute_from!(T => MaybeUninit<T> => T => Wrapping<T>); +impl_transitive_transmute_from!(T => Wrapping<T> => T => MaybeUninit<T>); + +// SAFETY: `ManuallyDrop<T>` has the same size and bit validity as `T` [1], and +// implements `Deref<Target = T>` [2]. Thus, it is already possible for safe +// code to obtain a `&T` and a `&ManuallyDrop<T>` to the same referent at the +// same time. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/mem/struct.ManuallyDrop.html: +// +// `ManuallyDrop<T>` is guaranteed to have the same layout and bit +// validity as `T` +// +// [2] https://doc.rust-lang.org/1.81.0/std/mem/struct.ManuallyDrop.html#impl-Deref-for-ManuallyDrop%3CT%3E +unsafe impl<T: ?Sized> InvariantsEq<T> for ManuallyDrop<T> {} +// SAFETY: See previous safety comment. +unsafe impl<T: ?Sized> InvariantsEq<ManuallyDrop<T>> for T {} + +/// Transmutations which are always sound. +/// +/// `TransmuteFromPtr` is a shorthand for [`TryTransmuteFromPtr`] and +/// [`TransmuteFrom`]. +/// +/// # Safety +/// +/// `Dst: TransmuteFromPtr<Src, A, SV, DV, _>` is equivalent to `Dst: +/// TryTransmuteFromPtr<Src, A, SV, DV, _> + TransmuteFrom<Src, SV, DV>`. +pub unsafe trait TransmuteFromPtr< + Src: ?Sized, + A: Aliasing, + SV: Validity, + DV: Validity, + C: CastExact<Src, Self>, + R, +>: TryTransmuteFromPtr<Src, A, SV, DV, C, R> + TransmuteFrom<Src, SV, DV> +{ +} + +// SAFETY: The `where` bounds are equivalent to the safety invariant on +// `TransmuteFromPtr`. +unsafe impl< + Src: ?Sized, + Dst: ?Sized, + A: Aliasing, + SV: Validity, + DV: Validity, + C: CastExact<Src, Dst>, + R, + > TransmuteFromPtr<Src, A, SV, DV, C, R> for Dst +where + Dst: TransmuteFrom<Src, SV, DV> + TryTransmuteFromPtr<Src, A, SV, DV, C, R>, +{ +} + +/// Denotes that any `SV`-valid `Src` may soundly be transmuted into a +/// `DV`-valid `Self`. +/// +/// # Safety +/// +/// Given `src: Ptr<Src, (_, _, SV)>` and `dst: Ptr<Dst, (_, _, DV)>`, if the +/// referents of `src` and `dst` are the same size, then the set of bit patterns +/// allowed to appear in `src`'s referent must be a subset of the set allowed to +/// appear in `dst`'s referent. +/// +/// If the referents are not the same size, then `Dst: TransmuteFrom<Src, SV, +/// DV>` conveys no safety guarantee. +pub unsafe trait TransmuteFrom<Src: ?Sized, SV, DV> {} + +/// Carries the ability to perform a size-preserving cast or conversion from a +/// raw pointer to `Src` to a raw pointer to `Self`. +/// +/// The cast/conversion is carried by the associated [`CastFrom`] type, and +/// may be a no-op cast (without updating pointer metadata) or a conversion +/// which updates pointer metadata. +/// +/// # Safety +/// +/// `SizeEq` on its own conveys no safety guarantee. Any safety guarantees come +/// from the safety invariants on the associated [`CastFrom`] type, specifically +/// the [`CastExact`] bound. +/// +/// [`CastFrom`]: SizeEq::CastFrom +/// [`CastExact`]: CastExact +pub trait SizeEq<Src: ?Sized> { + type CastFrom: CastExact<Src, Self>; +} + +impl<T: ?Sized> SizeEq<T> for T { + type CastFrom = cast::IdCast; +} + +// SAFETY: Since `Src: IntoBytes`, the set of valid `Src`'s is the set of +// initialized bit patterns, which is exactly the set allowed in the referent of +// any `Initialized` `Ptr`. +unsafe impl<Src, Dst> TransmuteFrom<Src, Valid, Initialized> for Dst +where + Src: IntoBytes + ?Sized, + Dst: ?Sized, +{ +} + +// SAFETY: Since `Dst: FromBytes`, any initialized bit pattern may appear in the +// referent of a `Ptr<Dst, (_, _, Valid)>`. This is exactly equal to the set of +// bit patterns which may appear in the referent of any `Initialized` `Ptr`. +unsafe impl<Src, Dst> TransmuteFrom<Src, Initialized, Valid> for Dst +where + Src: ?Sized, + Dst: FromBytes + ?Sized, +{ +} + +// FIXME(#2354): This seems like a smell - the soundness of this bound has +// nothing to do with `Src` or `Dst` - we're basically just saying `[u8; N]` is +// transmutable into `[u8; N]`. + +// SAFETY: The set of allowed bit patterns in the referent of any `Initialized` +// `Ptr` is the same regardless of referent type. +unsafe impl<Src, Dst> TransmuteFrom<Src, Initialized, Initialized> for Dst +where + Src: ?Sized, + Dst: ?Sized, +{ +} + +// FIXME(#2354): This seems like a smell - the soundness of this bound has +// nothing to do with `Dst` - we're basically just saying that any type is +// transmutable into `MaybeUninit<[u8; N]>`. + +// SAFETY: A `Dst` with validity `Uninit` permits any byte sequence, and +// therefore can be transmuted from any value. +unsafe impl<Src, Dst, V> TransmuteFrom<Src, V, Uninit> for Dst +where + Src: ?Sized, + Dst: ?Sized, + V: Validity, +{ +} + +// SAFETY: +// - `ManuallyDrop<T>` has the same size as `T` [1] +// - `ManuallyDrop<T>` has the same validity as `T` [1] +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/mem/struct.ManuallyDrop.html: +// +// `ManuallyDrop<T>` is guaranteed to have the same layout and bit validity as +// `T` +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T: ?Sized => ManuallyDrop<T>) }; + +// SAFETY: +// - `Unalign<T>` promises to have the same size as `T`. +// - `Unalign<T>` promises to have the same validity as `T`. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T => Unalign<T>) }; +// SAFETY: `Unalign<T>` promises to have the same size and validity as `T`. +// Given `u: &Unalign<T>`, it is already possible to obtain `let t = +// u.try_deref().unwrap()`. Because `Unalign<T>` has the same size as `T`, the +// returned `&T` must point to the same referent as `u`, and thus it must be +// sound for these two references to exist at the same time since it's already +// possible for safe code to get into this state. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_invariants_eq!(T => T, Unalign<T>) }; + +// SAFETY: +// - `Wrapping<T>` has the same size as `T` [1]. +// - `Wrapping<T>` has only one field, which is `pub` [2]. We are also +// guaranteed per that `Wrapping<T>` has the same layout as `T` [1]. The only +// way for both of these to be true simultaneously is for `Wrapping<T>` to +// have the same bit validity as `T`. In particular, in order to change the +// bit validity, one of the following would need to happen: +// - `Wrapping` could change its `repr`, but this would violate the layout +// guarantee. +// - `Wrapping` could add or change its fields, but this would be a +// stability-breaking change. +// +// [1] Per https://doc.rust-lang.org/1.85.0/core/num/struct.Wrapping.html#layout-1: +// +// `Wrapping<T>` is guaranteed to have the same layout and ABI as `T`. +// +// [2] Definition from https://doc.rust-lang.org/1.85.0/core/num/struct.Wrapping.html: +// +// ``` +// #[repr(transparent)] +// pub struct Wrapping<T>(pub T); +// ``` +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T => Wrapping<T>) }; + +// SAFETY: By the preceding safety proof, `Wrapping<T>` and `T` have the same +// layout and bit validity. Since a `Wrapping<T>`'s `T` field is `pub`, given +// `w: &Wrapping<T>`, it's possible to do `let t = &w.t`, which means that it's +// already possible for safe code to obtain a `&Wrapping<T>` and a `&T` pointing +// to the same referent at the same time. Thus, this must be sound. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_invariants_eq!(T => T, Wrapping<T>) }; + +// SAFETY: +// - `UnsafeCell<T>` has the same size as `T` [1]. +// - Per [1], `UnsafeCell<T>` has the same bit validity as `T`. Technically the +// term "representation" doesn't guarantee this, but the subsequent sentence +// in the documentation makes it clear that this is the intention. +// +// [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout: +// +// `UnsafeCell<T>` has the same in-memory representation as its inner type +// `T`. A consequence of this guarantee is that it is possible to convert +// between `T` and `UnsafeCell<T>`. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T: ?Sized => UnsafeCell<T>) }; + +// SAFETY: +// - `Cell<T>` has the same size as `T` [1]. +// - Per [1], `Cell<T>` has the same bit validity as `T`. Technically the term +// "representation" doesn't guarantee this, but it does promise to have the +// "same memory layout and caveats as `UnsafeCell<T>`." The `UnsafeCell` docs +// [2] make it clear that bit validity is the intention even if that phrase +// isn't used. +// +// [1] Per https://doc.rust-lang.org/1.85.0/std/cell/struct.Cell.html#memory-layout: +// +// `Cell<T>` has the same memory layout and caveats as `UnsafeCell<T>`. In +// particular, this means that `Cell<T>` has the same in-memory representation +// as its inner type `T`. +// +// [2] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout: +// +// `UnsafeCell<T>` has the same in-memory representation as its inner type +// `T`. A consequence of this guarantee is that it is possible to convert +// between `T` and `UnsafeCell<T>`. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T: ?Sized => Cell<T>) }; + +impl_transitive_transmute_from!(T: ?Sized => Cell<T> => T => UnsafeCell<T>); +impl_transitive_transmute_from!(T: ?Sized => UnsafeCell<T> => T => Cell<T>); + +// SAFETY: `MaybeUninit<T>` has no validity requirements. Currently this is not +// explicitly guaranteed, but it's obvious from `MaybeUninit`'s documentation +// that this is the intention: +// https://doc.rust-lang.org/1.85.0/core/mem/union.MaybeUninit.html +unsafe impl<T> TransmuteFrom<T, Uninit, Valid> for MaybeUninit<T> {} + +impl<T> SizeEq<T> for MaybeUninit<T> { + type CastFrom = CastSizedExact; +} + +impl<T> SizeEq<MaybeUninit<T>> for T { + type CastFrom = CastSizedExact; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::pointer::cast::Project as _; + + fn test_size_eq<Src, Dst: SizeEq<Src>>(mut src: Src) { + let _: *mut Dst = + <Dst as SizeEq<Src>>::CastFrom::project(crate::pointer::PtrInner::from_mut(&mut src)); + } + + #[test] + fn test_transmute_coverage() { + // SizeEq<T> for MaybeUninit<T> + test_size_eq::<u8, MaybeUninit<u8>>(0u8); + + // SizeEq<MaybeUninit<T>> for T + test_size_eq::<MaybeUninit<u8>, u8>(MaybeUninit::<u8>::new(0)); + + // Transitive: MaybeUninit<T> -> Wrapping<T> + // T => MaybeUninit<T> => T => Wrapping<T> + test_size_eq::<u8, Wrapping<u8>>(0u8); + + // T => Wrapping<T> => T => MaybeUninit<T> + test_size_eq::<Wrapping<u8>, MaybeUninit<u8>>(Wrapping(0u8)); + + // T: ?Sized => Cell<T> => T => UnsafeCell<T> + test_size_eq::<Cell<u8>, UnsafeCell<u8>>(Cell::new(0u8)); + + // T: ?Sized => UnsafeCell<T> => T => Cell<T> + test_size_eq::<UnsafeCell<u8>, Cell<u8>>(UnsafeCell::new(0u8)); + } +} diff --git a/rust/zerocopy/src/ref.rs b/rust/zerocopy/src/ref.rs new file mode 100644 index 000000000000..860066d75196 --- /dev/null +++ b/rust/zerocopy/src/ref.rs @@ -0,0 +1,1358 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2024 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License <LICENSE-BSD or +// https://opensource.org/license/bsd-2-clause>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. +use super::*; +use crate::pointer::{ + BecauseInvariantsEq, BecauseMutationCompatible, MutationCompatible, TransmuteFromPtr, +}; + +mod def { + use core::marker::PhantomData; + + use crate::{ + ByteSlice, ByteSliceMut, CloneableByteSlice, CopyableByteSlice, IntoByteSlice, + IntoByteSliceMut, + }; + + /// A typed reference derived from a byte slice. + /// + /// A `Ref<B, T>` is a reference to a `T` which is stored in a byte slice, `B`. + /// Unlike a native reference (`&T` or `&mut T`), `Ref<B, T>` has the same + /// mutability as the byte slice it was constructed from (`B`). + /// + /// # Examples + /// + /// `Ref` can be used to treat a sequence of bytes as a structured type, and + /// to read and write the fields of that type as if the byte slice reference + /// were simply a reference to that type. + /// + /// ```rust + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] + /// #[repr(C)] + /// struct UdpHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] + /// #[repr(C, packed)] + /// struct UdpPacket { + /// header: UdpHeader, + /// body: [u8], + /// } + /// + /// impl UdpPacket { + /// pub fn parse<B: ByteSlice>(bytes: B) -> Option<Ref<B, UdpPacket>> { + /// Ref::from_bytes(bytes).ok() + /// } + /// } + /// ``` + pub struct Ref<B, T: ?Sized>( + // INVARIANTS: The referent (via `.deref`, `.deref_mut`, `.into`) byte + // slice is aligned to `T`'s alignment and its size corresponds to a + // valid size for `T`. + B, + PhantomData<T>, + ); + + impl<B, T: ?Sized> Ref<B, T> { + /// Constructs a new `Ref`. + /// + /// # Safety + /// + /// `bytes` dereferences (via [`deref`], [`deref_mut`], and [`into`]) to + /// a byte slice which is aligned to `T`'s alignment and whose size is a + /// valid size for `T`. + /// + /// [`deref`]: core::ops::Deref::deref + /// [`deref_mut`]: core::ops::DerefMut::deref_mut + /// [`into`]: core::convert::Into::into + pub(crate) unsafe fn new_unchecked(bytes: B) -> Ref<B, T> { + // INVARIANTS: The caller has promised that `bytes`'s referent is + // validly-aligned and has a valid size. + Ref(bytes, PhantomData) + } + } + + impl<B: ByteSlice, T: ?Sized> Ref<B, T> { + /// Access the byte slice as a [`ByteSlice`]. + /// + /// # Safety + /// + /// The caller promises not to call methods on the returned + /// [`ByteSlice`] other than `ByteSlice` methods (for example, via + /// `Any::downcast_ref`). + /// + /// `as_byte_slice` promises to return a `ByteSlice` whose referent is + /// validly-aligned for `T` and has a valid size for `T`. + pub(crate) unsafe fn as_byte_slice(&self) -> &impl ByteSlice { + // INVARIANTS: The caller promises not to call methods other than + // those on `ByteSlice`. Since `B: ByteSlice`, dereference stability + // guarantees that calling `ByteSlice` methods will not change the + // address or length of `self.0`'s referent. + // + // SAFETY: By invariant on `self.0`, the alignment and size + // post-conditions are upheld. + &self.0 + } + } + + impl<B: ByteSliceMut, T: ?Sized> Ref<B, T> { + /// Access the byte slice as a [`ByteSliceMut`]. + /// + /// # Safety + /// + /// The caller promises not to call methods on the returned + /// [`ByteSliceMut`] other than `ByteSliceMut` methods (for example, via + /// `Any::downcast_mut`). + /// + /// `as_byte_slice` promises to return a `ByteSlice` whose referent is + /// validly-aligned for `T` and has a valid size for `T`. + pub(crate) unsafe fn as_byte_slice_mut(&mut self) -> &mut impl ByteSliceMut { + // INVARIANTS: The caller promises not to call methods other than + // those on `ByteSliceMut`. Since `B: ByteSlice`, dereference + // stability guarantees that calling `ByteSlice` methods will not + // change the address or length of `self.0`'s referent. + // + // SAFETY: By invariant on `self.0`, the alignment and size + // post-conditions are upheld. + &mut self.0 + } + } + + impl<'a, B: IntoByteSlice<'a>, T: ?Sized> Ref<B, T> { + /// Access the byte slice as an [`IntoByteSlice`]. + /// + /// # Safety + /// + /// The caller promises not to call methods on the returned + /// [`IntoByteSlice`] other than `IntoByteSlice` methods (for example, + /// via `Any::downcast_ref`). + /// + /// `as_byte_slice` promises to return a `ByteSlice` whose referent is + /// validly-aligned for `T` and has a valid size for `T`. + pub(crate) unsafe fn into_byte_slice(self) -> impl IntoByteSlice<'a> { + // INVARIANTS: The caller promises not to call methods other than + // those on `IntoByteSlice`. Since `B: ByteSlice`, dereference + // stability guarantees that calling `ByteSlice` methods will not + // change the address or length of `self.0`'s referent. + // + // SAFETY: By invariant on `self.0`, the alignment and size + // post-conditions are upheld. + self.0 + } + } + + impl<'a, B: IntoByteSliceMut<'a>, T: ?Sized> Ref<B, T> { + /// Access the byte slice as an [`IntoByteSliceMut`]. + /// + /// # Safety + /// + /// The caller promises not to call methods on the returned + /// [`IntoByteSliceMut`] other than `IntoByteSliceMut` methods (for + /// example, via `Any::downcast_mut`). + /// + /// `as_byte_slice` promises to return a `ByteSlice` whose referent is + /// validly-aligned for `T` and has a valid size for `T`. + pub(crate) unsafe fn into_byte_slice_mut(self) -> impl IntoByteSliceMut<'a> { + // INVARIANTS: The caller promises not to call methods other than + // those on `IntoByteSliceMut`. Since `B: ByteSlice`, dereference + // stability guarantees that calling `ByteSlice` methods will not + // change the address or length of `self.0`'s referent. + // + // SAFETY: By invariant on `self.0`, the alignment and size + // post-conditions are upheld. + self.0 + } + } + + impl<B: CloneableByteSlice + Clone, T: ?Sized> Clone for Ref<B, T> { + #[inline] + fn clone(&self) -> Ref<B, T> { + // INVARIANTS: Since `B: CloneableByteSlice`, `self.0.clone()` has + // the same address and length as `self.0`. Since `self.0` upholds + // the field invariants, so does `self.0.clone()`. + Ref(self.0.clone(), PhantomData) + } + } + + // INVARIANTS: Since `B: CopyableByteSlice`, the copied `Ref`'s `.0` has the + // same address and length as the original `Ref`'s `.0`. Since the original + // upholds the field invariants, so does the copy. + impl<B: CopyableByteSlice + Copy, T: ?Sized> Copy for Ref<B, T> {} +} + +#[allow(unreachable_pub)] // This is a false positive on our MSRV toolchain. +pub use def::Ref; + +use crate::pointer::{ + invariant::{Aligned, BecauseExclusive, Initialized, Unaligned, Valid}, + BecauseRead, PtrInner, +}; + +impl<B, T> Ref<B, T> +where + B: ByteSlice, +{ + #[must_use = "has no side effects"] + pub(crate) fn sized_from(bytes: B) -> Result<Ref<B, T>, CastError<B, T>> { + if bytes.len() != mem::size_of::<T>() { + return Err(SizeError::new(bytes).into()); + } + if let Err(err) = util::validate_aligned_to::<_, T>(bytes.deref()) { + return Err(err.with_src(bytes).into()); + } + + // SAFETY: We just validated size and alignment. + Ok(unsafe { Ref::new_unchecked(bytes) }) + } +} + +impl<B, T> Ref<B, T> +where + B: SplitByteSlice, +{ + #[must_use = "has no side effects"] + pub(crate) fn sized_from_prefix(bytes: B) -> Result<(Ref<B, T>, B), CastError<B, T>> { + if bytes.len() < mem::size_of::<T>() { + return Err(SizeError::new(bytes).into()); + } + if let Err(err) = util::validate_aligned_to::<_, T>(bytes.deref()) { + return Err(err.with_src(bytes).into()); + } + let (bytes, suffix) = bytes.split_at(mem::size_of::<T>()).map_err( + #[inline(always)] + |b| SizeError::new(b).into(), + )?; + // SAFETY: We just validated alignment and that `bytes` is at least as + // large as `T`. `bytes.split_at(mem::size_of::<T>())?` ensures that the + // new `bytes` is exactly the size of `T`. By safety postcondition on + // `SplitByteSlice::split_at` we can rely on `split_at` to produce the + // correct `bytes` and `suffix`. + let r = unsafe { Ref::new_unchecked(bytes) }; + Ok((r, suffix)) + } + + #[must_use = "has no side effects"] + pub(crate) fn sized_from_suffix(bytes: B) -> Result<(B, Ref<B, T>), CastError<B, T>> { + let bytes_len = bytes.len(); + let split_at = if let Some(split_at) = bytes_len.checked_sub(mem::size_of::<T>()) { + split_at + } else { + return Err(SizeError::new(bytes).into()); + }; + let (prefix, bytes) = bytes.split_at(split_at).map_err(|b| SizeError::new(b).into())?; + if let Err(err) = util::validate_aligned_to::<_, T>(bytes.deref()) { + return Err(err.with_src(bytes).into()); + } + // SAFETY: Since `split_at` is defined as `bytes_len - size_of::<T>()`, + // the `bytes` which results from `let (prefix, bytes) = + // bytes.split_at(split_at)?` has length `size_of::<T>()`. After + // constructing `bytes`, we validate that it has the proper alignment. + // By safety postcondition on `SplitByteSlice::split_at` we can rely on + // `split_at` to produce the correct `prefix` and `bytes`. + let r = unsafe { Ref::new_unchecked(bytes) }; + Ok((prefix, r)) + } +} + +impl<B, T> Ref<B, T> +where + B: ByteSlice, + T: KnownLayout + Immutable + ?Sized, +{ + /// Constructs a `Ref` from a byte slice. + /// + /// If the length of `source` is not a [valid size of `T`][valid-size], or + /// if `source` is not appropriately aligned for `T`, this returns `Err`. If + /// [`T: Unaligned`][t-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// `T` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [t-unaligned]: crate::Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = Ref::<_, ZSTy>::from_bytes(&b"UU"[..]); // ⚠ Compile Error! + /// ``` + #[must_use = "has no side effects"] + #[inline] + pub fn from_bytes(source: B) -> Result<Ref<B, T>, CastError<B, T>> { + static_assert_dst_is_not_zst!(T); + if let Err(e) = + Ptr::from_ref(source.deref()).try_cast_into_no_leftover::<T, BecauseImmutable>(None) + { + return Err(e.with_src(()).with_src(source)); + } + // SAFETY: `try_cast_into_no_leftover` validates size and alignment. + Ok(unsafe { Ref::new_unchecked(source) }) + } +} + +impl<B, T> Ref<B, T> +where + B: SplitByteSlice, + T: KnownLayout + Immutable + ?Sized, +{ + /// Constructs a `Ref` from the prefix of a byte slice. + /// + /// This method computes the [largest possible size of `T`][valid-size] that + /// can fit in the leading bytes of `source`, then attempts to return both a + /// `Ref` to those bytes, and a reference to the remaining bytes. If there + /// are insufficient bytes, or if `source` is not appropriately aligned, + /// this returns `Err`. If [`T: Unaligned`][t-unaligned], you can + /// [infallibly discard the alignment error][size-error-from]. + /// + /// `T` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [t-unaligned]: crate::Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = Ref::<_, ZSTy>::from_prefix(&b"UU"[..]); // ⚠ Compile Error! + /// ``` + #[must_use = "has no side effects"] + #[inline] + pub fn from_prefix(source: B) -> Result<(Ref<B, T>, B), CastError<B, T>> { + static_assert_dst_is_not_zst!(T); + let remainder = match Ptr::from_ref(source.deref()) + .try_cast_into::<T, BecauseImmutable>(CastType::Prefix, None) + { + Ok((_, remainder)) => remainder, + Err(e) => { + return Err(e.with_src(()).with_src(source)); + } + }; + + // SAFETY: `remainder` is constructed as a subset of `source`, and so it + // cannot have a larger size than `source`. Both of their `len` methods + // measure bytes (`source` deref's to `[u8]`, and `remainder` is a + // `Ptr<[u8]>`), so `source.len() >= remainder.len()`. Thus, this cannot + // underflow. + #[allow(unstable_name_collisions)] + let split_at = unsafe { source.len().unchecked_sub(remainder.len()) }; + let (bytes, suffix) = source.split_at(split_at).map_err(|b| SizeError::new(b).into())?; + // SAFETY: `try_cast_into` validates size and alignment, and returns a + // `split_at` that indicates how many bytes of `source` correspond to a + // valid `T`. By safety postcondition on `SplitByteSlice::split_at` we + // can rely on `split_at` to produce the correct `source` and `suffix`. + let r = unsafe { Ref::new_unchecked(bytes) }; + Ok((r, suffix)) + } + + /// Constructs a `Ref` from the suffix of a byte slice. + /// + /// This method computes the [largest possible size of `T`][valid-size] that + /// can fit in the trailing bytes of `source`, then attempts to return both + /// a `Ref` to those bytes, and a reference to the preceding bytes. If there + /// are insufficient bytes, or if that suffix of `source` is not + /// appropriately aligned, this returns `Err`. If [`T: + /// Unaligned`][t-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// `T` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [t-unaligned]: crate::Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = Ref::<_, ZSTy>::from_suffix(&b"UU"[..]); // ⚠ Compile Error! + /// ``` + #[must_use = "has no side effects"] + #[inline] + pub fn from_suffix(source: B) -> Result<(B, Ref<B, T>), CastError<B, T>> { + static_assert_dst_is_not_zst!(T); + let remainder = match Ptr::from_ref(source.deref()) + .try_cast_into::<T, BecauseImmutable>(CastType::Suffix, None) + { + Ok((_, remainder)) => remainder, + Err(e) => { + let e = e.with_src(()); + return Err(e.with_src(source)); + } + }; + + let split_at = remainder.len(); + let (prefix, bytes) = source.split_at(split_at).map_err(|b| SizeError::new(b).into())?; + // SAFETY: `try_cast_into` validates size and alignment, and returns a + // `split_at` that indicates how many bytes of `source` correspond to a + // valid `T`. By safety postcondition on `SplitByteSlice::split_at` we + // can rely on `split_at` to produce the correct `prefix` and `bytes`. + let r = unsafe { Ref::new_unchecked(bytes) }; + Ok((prefix, r)) + } +} + +impl<B, T> Ref<B, T> +where + B: ByteSlice, + T: KnownLayout<PointerMetadata = usize> + Immutable + ?Sized, +{ + /// Constructs a `Ref` from the given bytes with DST length equal to `count` + /// without copying. + /// + /// This method attempts to return a `Ref` to the prefix of `source` + /// interpreted as a `T` with `count` trailing elements, and a reference to + /// the remaining bytes. If the length of `source` is not equal to the size + /// of `Self` with `count` elements, or if `source` is not appropriately + /// aligned, this returns `Err`. If [`T: Unaligned`][t-unaligned], you can + /// [infallibly discard the alignment error][size-error-from]. + /// + /// [t-unaligned]: crate::Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = Ref::<_, ZSTy>::from_bytes_with_elems(&b"UU"[..], 42); // ⚠ Compile Error! + /// ``` + #[inline] + pub fn from_bytes_with_elems(source: B, count: usize) -> Result<Ref<B, T>, CastError<B, T>> { + static_assert_dst_is_not_zst!(T); + let expected_len = match T::size_for_metadata(count) { + Some(len) => len, + None => return Err(SizeError::new(source).into()), + }; + if source.len() != expected_len { + return Err(SizeError::new(source).into()); + } + Self::from_bytes(source) + } +} + +impl<B, T> Ref<B, T> +where + B: SplitByteSlice, + T: KnownLayout<PointerMetadata = usize> + Immutable + ?Sized, +{ + /// Constructs a `Ref` from the prefix of the given bytes with DST + /// length equal to `count` without copying. + /// + /// This method attempts to return a `Ref` to the prefix of `source` + /// interpreted as a `T` with `count` trailing elements, and a reference to + /// the remaining bytes. If there are insufficient bytes, or if `source` is + /// not appropriately aligned, this returns `Err`. If [`T: + /// Unaligned`][t-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// [t-unaligned]: crate::Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = Ref::<_, ZSTy>::from_prefix_with_elems(&b"UU"[..], 42); // ⚠ Compile Error! + /// ``` + #[inline] + pub fn from_prefix_with_elems( + source: B, + count: usize, + ) -> Result<(Ref<B, T>, B), CastError<B, T>> { + static_assert_dst_is_not_zst!(T); + let expected_len = match T::size_for_metadata(count) { + Some(len) => len, + None => return Err(SizeError::new(source).into()), + }; + let (prefix, bytes) = source.split_at(expected_len).map_err(SizeError::new)?; + Self::from_bytes(prefix).map(move |l| (l, bytes)) + } + + /// Constructs a `Ref` from the suffix of the given bytes with DST length + /// equal to `count` without copying. + /// + /// This method attempts to return a `Ref` to the suffix of `source` + /// interpreted as a `T` with `count` trailing elements, and a reference to + /// the preceding bytes. If there are insufficient bytes, or if that suffix + /// of `source` is not appropriately aligned, this returns `Err`. If [`T: + /// Unaligned`][t-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// [t-unaligned]: crate::Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = Ref::<_, ZSTy>::from_suffix_with_elems(&b"UU"[..], 42); // ⚠ Compile Error! + /// ``` + #[inline] + pub fn from_suffix_with_elems( + source: B, + count: usize, + ) -> Result<(B, Ref<B, T>), CastError<B, T>> { + static_assert_dst_is_not_zst!(T); + let expected_len = match T::size_for_metadata(count) { + Some(len) => len, + None => return Err(SizeError::new(source).into()), + }; + let split_at = if let Some(split_at) = source.len().checked_sub(expected_len) { + split_at + } else { + return Err(SizeError::new(source).into()); + }; + // SAFETY: The preceding `source.len().checked_sub(expected_len)` + // guarantees that `split_at` is in-bounds. + let (bytes, suffix) = unsafe { source.split_at_unchecked(split_at) }; + Self::from_bytes(suffix).map(move |l| (bytes, l)) + } +} + +impl<'a, B, T> Ref<B, T> +where + B: 'a + IntoByteSlice<'a>, + T: FromBytes + KnownLayout + Immutable + ?Sized, +{ + /// Converts this `Ref` into a reference. + /// + /// `into_ref` consumes the `Ref`, and returns a reference to `T`. + /// + /// Note: this is an associated function, which means that you have to call + /// it as `Ref::into_ref(r)` instead of `r.into_ref()`. This is so that + /// there is no conflict with a method on the inner type. + #[must_use = "has no side effects"] + #[inline(always)] + pub fn into_ref(r: Self) -> &'a T { + // Presumably unreachable, since we've guarded each constructor of `Ref`. + static_assert_dst_is_not_zst!(T); + + // SAFETY: We don't call any methods on `b` other than those provided by + // `IntoByteSlice`. + let b = unsafe { r.into_byte_slice() }; + let b = b.into_byte_slice(); + + if let crate::layout::SizeInfo::Sized { .. } = T::LAYOUT.size_info { + let ptr = Ptr::from_ref(b); + // SAFETY: We just checked that `T: Sized`. By invariant on `r`, + // `b`'s size is equal to `size_of::<T>()`. + let ptr = unsafe { cast_for_sized::<T, _, _, _>(ptr) }; + + // SAFETY: None of the preceding transformations modifies the + // address of the pointer, and by invariant on `r`, we know that it + // is validly-aligned. + let ptr = unsafe { ptr.assume_alignment::<Aligned>() }; + return ptr.as_ref(); + } + + // PANICS: By post-condition on `into_byte_slice`, `b`'s size and + // alignment are valid for `T`. By post-condition, `b.into_byte_slice()` + // produces a byte slice with identical address and length to that + // produced by `b.deref()`. + let ptr = Ptr::from_ref(b.into_byte_slice()) + .try_cast_into_no_leftover::<T, BecauseImmutable>(None) + .expect("zerocopy internal error: into_ref should be infallible"); + let ptr = ptr.recall_validity(); + ptr.as_ref() + } +} + +impl<'a, B, T> Ref<B, T> +where + B: 'a + IntoByteSliceMut<'a>, + T: FromBytes + IntoBytes + KnownLayout + ?Sized, +{ + /// Converts this `Ref` into a mutable reference. + /// + /// `into_mut` consumes the `Ref`, and returns a mutable reference to `T`. + /// + /// Note: this is an associated function, which means that you have to call + /// it as `Ref::into_mut(r)` instead of `r.into_mut()`. This is so that + /// there is no conflict with a method on the inner type. + #[must_use = "has no side effects"] + #[inline(always)] + pub fn into_mut(r: Self) -> &'a mut T { + // Presumably unreachable, since we've guarded each constructor of `Ref`. + static_assert_dst_is_not_zst!(T); + + // SAFETY: We don't call any methods on `b` other than those provided by + // `IntoByteSliceMut`. + let b = unsafe { r.into_byte_slice_mut() }; + let b = b.into_byte_slice_mut(); + + if let crate::layout::SizeInfo::Sized { .. } = T::LAYOUT.size_info { + let ptr = Ptr::from_mut(b); + // SAFETY: We just checked that `T: Sized`. By invariant on `r`, + // `b`'s size is equal to `size_of::<T>()`. + let ptr = unsafe { + cast_for_sized::< + T, + _, + (BecauseRead, BecauseExclusive), + (BecauseMutationCompatible, BecauseInvariantsEq), + >(ptr) + }; + + // SAFETY: None of the preceding transformations modifies the + // address of the pointer, and by invariant on `r`, we know that it + // is validly-aligned. + let ptr = unsafe { ptr.assume_alignment::<Aligned>() }; + return ptr.as_mut(); + } + + // PANICS: By post-condition on `into_byte_slice_mut`, `b`'s size and + // alignment are valid for `T`. By post-condition, + // `b.into_byte_slice_mut()` produces a byte slice with identical + // address and length to that produced by `b.deref_mut()`. + let ptr = Ptr::from_mut(b.into_byte_slice_mut()) + .try_cast_into_no_leftover::<T, BecauseExclusive>(None) + .expect("zerocopy internal error: into_ref should be infallible"); + let ptr = ptr.recall_validity::<_, (_, (_, _))>(); + ptr.as_mut() + } +} + +impl<B, T> Ref<B, T> +where + B: ByteSlice, + T: ?Sized, +{ + /// Gets the underlying bytes. + /// + /// Note: this is an associated function, which means that you have to call + /// it as `Ref::bytes(r)` instead of `r.bytes()`. This is so that there is + /// no conflict with a method on the inner type. + #[inline] + pub fn bytes(r: &Self) -> &[u8] { + // SAFETY: We don't call any methods on `b` other than those provided by + // `ByteSlice`. + unsafe { r.as_byte_slice().deref() } + } +} + +impl<B, T> Ref<B, T> +where + B: ByteSliceMut, + T: ?Sized, +{ + /// Gets the underlying bytes mutably. + /// + /// Note: this is an associated function, which means that you have to call + /// it as `Ref::bytes_mut(r)` instead of `r.bytes_mut()`. This is so that + /// there is no conflict with a method on the inner type. + #[inline] + pub fn bytes_mut(r: &mut Self) -> &mut [u8] { + // SAFETY: We don't call any methods on `b` other than those provided by + // `ByteSliceMut`. + unsafe { r.as_byte_slice_mut().deref_mut() } + } +} + +impl<B, T> Ref<B, T> +where + B: ByteSlice, + T: FromBytes, +{ + /// Reads a copy of `T`. + /// + /// Note: this is an associated function, which means that you have to call + /// it as `Ref::read(r)` instead of `r.read()`. This is so that there is no + /// conflict with a method on the inner type. + #[must_use = "has no side effects"] + #[inline] + pub fn read(r: &Self) -> T { + // SAFETY: We don't call any methods on `b` other than those provided by + // `ByteSlice`. + let b = unsafe { r.as_byte_slice() }; + + // SAFETY: By postcondition on `as_byte_slice`, we know that `b` is a + // valid size and alignment for `T`. By safety invariant on `ByteSlice`, + // we know that this is preserved via `.deref()`. Because `T: + // FromBytes`, it is sound to interpret these bytes as a `T`. + unsafe { ptr::read(b.deref().as_ptr().cast::<T>()) } + } +} + +impl<B, T> Ref<B, T> +where + B: ByteSliceMut, + T: IntoBytes, +{ + /// Writes the bytes of `t` and then forgets `t`. + /// + /// Note: this is an associated function, which means that you have to call + /// it as `Ref::write(r, t)` instead of `r.write(t)`. This is so that there + /// is no conflict with a method on the inner type. + #[inline] + pub fn write(r: &mut Self, t: T) { + // SAFETY: We don't call any methods on `b` other than those provided by + // `ByteSliceMut`. + let b = unsafe { r.as_byte_slice_mut() }; + + // SAFETY: By postcondition on `as_byte_slice_mut`, we know that `b` is + // a valid size and alignment for `T`. By safety invariant on + // `ByteSlice`, we know that this is preserved via `.deref()`. Writing + // `t` to the buffer will allow all of the bytes of `t` to be accessed + // as a `[u8]`, but because `T: IntoBytes`, we know that this is sound. + unsafe { ptr::write(b.deref_mut().as_mut_ptr().cast::<T>(), t) } + } +} + +impl<B, T> Deref for Ref<B, T> +where + B: ByteSlice, + T: FromBytes + KnownLayout + Immutable + ?Sized, +{ + type Target = T; + #[inline] + fn deref(&self) -> &T { + // Presumably unreachable, since we've guarded each constructor of `Ref`. + static_assert_dst_is_not_zst!(T); + + // SAFETY: We don't call any methods on `b` other than those provided by + // `ByteSlice`. + let b = unsafe { self.as_byte_slice() }; + let b = b.deref(); + + if let crate::layout::SizeInfo::Sized { .. } = T::LAYOUT.size_info { + let ptr = Ptr::from_ref(b); + // SAFETY: We just checked that `T: Sized`. By invariant on `r`, + // `b`'s size is equal to `size_of::<T>()`. + let ptr = unsafe { cast_for_sized::<T, _, _, _>(ptr) }; + + // SAFETY: None of the preceding transformations modifies the + // address of the pointer, and by invariant on `r`, we know that it + // is validly-aligned. + let ptr = unsafe { ptr.assume_alignment::<Aligned>() }; + return ptr.as_ref(); + } + + // PANICS: By postcondition on `as_byte_slice`, `b`'s size and alignment + // are valid for `T`, and by invariant on `ByteSlice`, these are + // preserved through `.deref()`, so this `unwrap` will not panic. + let ptr = Ptr::from_ref(b) + .try_cast_into_no_leftover::<T, BecauseImmutable>(None) + .expect("zerocopy internal error: Deref::deref should be infallible"); + let ptr = ptr.recall_validity(); + ptr.as_ref() + } +} + +impl<B, T> DerefMut for Ref<B, T> +where + B: ByteSliceMut, + // FIXME(#251): We can't remove `Immutable` here because it's required by + // the impl of `Deref`, which is a super-trait of `DerefMut`. Maybe we can + // add a separate inherent method for this? + T: FromBytes + IntoBytes + KnownLayout + Immutable + ?Sized, +{ + #[inline] + fn deref_mut(&mut self) -> &mut T { + // Presumably unreachable, since we've guarded each constructor of `Ref`. + static_assert_dst_is_not_zst!(T); + + // SAFETY: We don't call any methods on `b` other than those provided by + // `ByteSliceMut`. + let b = unsafe { self.as_byte_slice_mut() }; + let b = b.deref_mut(); + + if let crate::layout::SizeInfo::Sized { .. } = T::LAYOUT.size_info { + let ptr = Ptr::from_mut(b); + // SAFETY: We just checked that `T: Sized`. By invariant on `r`, + // `b`'s size is equal to `size_of::<T>()`. + let ptr = unsafe { + cast_for_sized::< + T, + _, + (BecauseRead, BecauseExclusive), + (BecauseMutationCompatible, BecauseInvariantsEq), + >(ptr) + }; + + // SAFETY: None of the preceding transformations modifies the + // address of the pointer, and by invariant on `r`, we know that it + // is validly-aligned. + let ptr = unsafe { ptr.assume_alignment::<Aligned>() }; + return ptr.as_mut(); + } + + // PANICS: By postcondition on `as_byte_slice_mut`, `b`'s size and + // alignment are valid for `T`, and by invariant on `ByteSlice`, these + // are preserved through `.deref_mut()`, so this `unwrap` will not + // panic. + let ptr = Ptr::from_mut(b) + .try_cast_into_no_leftover::<T, BecauseExclusive>(None) + .expect("zerocopy internal error: DerefMut::deref_mut should be infallible"); + let ptr = ptr.recall_validity::<_, (_, (_, BecauseExclusive))>(); + ptr.as_mut() + } +} + +impl<T, B> Display for Ref<B, T> +where + B: ByteSlice, + T: FromBytes + Display + KnownLayout + Immutable + ?Sized, +{ + #[inline] + fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { + let inner: &T = self; + inner.fmt(fmt) + } +} + +impl<T, B> Debug for Ref<B, T> +where + B: ByteSlice, + T: FromBytes + Debug + KnownLayout + Immutable + ?Sized, +{ + #[inline] + fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { + let inner: &T = self; + fmt.debug_tuple("Ref").field(&inner).finish() + } +} + +impl<T, B> Eq for Ref<B, T> +where + B: ByteSlice, + T: FromBytes + Eq + KnownLayout + Immutable + ?Sized, +{ +} + +impl<T, B> PartialEq for Ref<B, T> +where + B: ByteSlice, + T: FromBytes + PartialEq + KnownLayout + Immutable + ?Sized, +{ + #[inline] + fn eq(&self, other: &Self) -> bool { + self.deref().eq(other.deref()) + } +} + +impl<T, B> Ord for Ref<B, T> +where + B: ByteSlice, + T: FromBytes + Ord + KnownLayout + Immutable + ?Sized, +{ + #[inline] + fn cmp(&self, other: &Self) -> Ordering { + let inner: &T = self; + let other_inner: &T = other; + inner.cmp(other_inner) + } +} + +impl<T, B> PartialOrd for Ref<B, T> +where + B: ByteSlice, + T: FromBytes + PartialOrd + KnownLayout + Immutable + ?Sized, +{ + #[inline] + fn partial_cmp(&self, other: &Self) -> Option<Ordering> { + let inner: &T = self; + let other_inner: &T = other; + inner.partial_cmp(other_inner) + } +} + +/// # Safety +/// +/// `T: Sized` and `ptr`'s referent must have size `size_of::<T>()`. +#[inline(always)] +unsafe fn cast_for_sized<'a, T, A, R, S>( + ptr: Ptr<'a, [u8], (A, Aligned, Valid)>, +) -> Ptr<'a, T, (A, Unaligned, Valid)> +where + T: FromBytes + KnownLayout + ?Sized, + A: crate::invariant::Aliasing, + [u8]: MutationCompatible<T, A, Initialized, Initialized, R>, + T: TransmuteFromPtr<T, A, Initialized, Valid, crate::pointer::cast::IdCast, S>, +{ + use crate::pointer::cast::{Cast, Project}; + + enum CastForSized {} + + // SAFETY: `CastForSized` is only used below with the input `ptr`, which the + // caller promises has size `size_of::<T>()`. Thus, the referent produced in + // this cast has the same size as `ptr`'s referent. All operations preserve + // provenance. + unsafe impl<T: ?Sized + KnownLayout> Project<[u8], T> for CastForSized { + #[inline(always)] + fn project(src: PtrInner<'_, [u8]>) -> *mut T { + T::raw_from_ptr_len( + src.as_non_null().cast(), + <T::PointerMetadata as crate::PointerMetadata>::from_elem_count(0), + ) + .as_ptr() + } + } + + // SAFETY: The `Project::project` impl preserves referent address. + unsafe impl<T: ?Sized + KnownLayout> Cast<[u8], T> for CastForSized {} + + ptr.recall_validity::<Initialized, (_, (_, _))>() + .cast::<_, CastForSized, _>() + .recall_validity::<Valid, _>() +} + +#[cfg(test)] +#[allow(clippy::assertions_on_result_states)] +mod tests { + use core::convert::TryInto as _; + + use super::*; + use crate::util::testutil::*; + + #[test] + fn test_mut_slice_into_ref() { + // Prior to #1260/#1299, calling `into_ref` on a `&mut [u8]`-backed + // `Ref` was not supported. + let mut buf = [0u8]; + let r = Ref::<&mut [u8], u8>::from_bytes(&mut buf).unwrap(); + assert_eq!(Ref::into_ref(r), &0); + } + + #[test] + fn test_address() { + // Test that the `Deref` and `DerefMut` implementations return a + // reference which points to the right region of memory. + + let buf = [0]; + let r = Ref::<_, u8>::from_bytes(&buf[..]).unwrap(); + let buf_ptr = buf.as_ptr(); + let deref_ptr: *const u8 = r.deref(); + assert_eq!(buf_ptr, deref_ptr); + + let buf = [0]; + let r = Ref::<_, [u8]>::from_bytes(&buf[..]).unwrap(); + let buf_ptr = buf.as_ptr(); + let deref_ptr = r.deref().as_ptr(); + assert_eq!(buf_ptr, deref_ptr); + } + + // Verify that values written to a `Ref` are properly shared between the + // typed and untyped representations, that reads via `deref` and `read` + // behave the same, and that writes via `deref_mut` and `write` behave the + // same. + fn test_new_helper(mut r: Ref<&mut [u8], AU64>) { + // assert that the value starts at 0 + assert_eq!(*r, AU64(0)); + assert_eq!(Ref::read(&r), AU64(0)); + + // Assert that values written to the typed value are reflected in the + // byte slice. + const VAL1: AU64 = AU64(0xFF00FF00FF00FF00); + *r = VAL1; + assert_eq!(Ref::bytes(&r), &VAL1.to_bytes()); + *r = AU64(0); + Ref::write(&mut r, VAL1); + assert_eq!(Ref::bytes(&r), &VAL1.to_bytes()); + + // Assert that values written to the byte slice are reflected in the + // typed value. + const VAL2: AU64 = AU64(!VAL1.0); // different from `VAL1` + Ref::bytes_mut(&mut r).copy_from_slice(&VAL2.to_bytes()[..]); + assert_eq!(*r, VAL2); + assert_eq!(Ref::read(&r), VAL2); + } + + // Verify that values written to a `Ref` are properly shared between the + // typed and untyped representations; pass a value with `typed_len` `AU64`s + // backed by an array of `typed_len * 8` bytes. + fn test_new_helper_slice(mut r: Ref<&mut [u8], [AU64]>, typed_len: usize) { + // Assert that the value starts out zeroed. + assert_eq!(&*r, vec![AU64(0); typed_len].as_slice()); + + // Check the backing storage is the exact same slice. + let untyped_len = typed_len * 8; + assert_eq!(Ref::bytes(&r).len(), untyped_len); + assert_eq!(Ref::bytes(&r).as_ptr(), r.as_ptr().cast::<u8>()); + + // Assert that values written to the typed value are reflected in the + // byte slice. + const VAL1: AU64 = AU64(0xFF00FF00FF00FF00); + for typed in &mut *r { + *typed = VAL1; + } + assert_eq!(Ref::bytes(&r), VAL1.0.to_ne_bytes().repeat(typed_len).as_slice()); + + // Assert that values written to the byte slice are reflected in the + // typed value. + const VAL2: AU64 = AU64(!VAL1.0); // different from VAL1 + Ref::bytes_mut(&mut r).copy_from_slice(&VAL2.0.to_ne_bytes().repeat(typed_len)); + assert!(r.iter().copied().all(|x| x == VAL2)); + } + + #[test] + fn test_new_aligned_sized() { + // Test that a properly-aligned, properly-sized buffer works for new, + // new_from_prefix, and new_from_suffix, and that new_from_prefix and + // new_from_suffix return empty slices. Test that a properly-aligned + // buffer whose length is a multiple of the element size works for + // new_slice. + + // A buffer with an alignment of 8. + let mut buf = Align::<[u8; 8], AU64>::default(); + // `buf.t` should be aligned to 8, so this should always succeed. + test_new_helper(Ref::<_, AU64>::from_bytes(&mut buf.t[..]).unwrap()); + { + // In a block so that `r` and `suffix` don't live too long. + buf.set_default(); + let (r, suffix) = Ref::<_, AU64>::from_prefix(&mut buf.t[..]).unwrap(); + assert!(suffix.is_empty()); + test_new_helper(r); + } + { + buf.set_default(); + let (prefix, r) = Ref::<_, AU64>::from_suffix(&mut buf.t[..]).unwrap(); + assert!(prefix.is_empty()); + test_new_helper(r); + } + + // A buffer with alignment 8 and length 24. We choose this length very + // intentionally: if we instead used length 16, then the prefix and + // suffix lengths would be identical. In the past, we used length 16, + // which resulted in this test failing to discover the bug uncovered in + // #506. + let mut buf = Align::<[u8; 24], AU64>::default(); + // `buf.t` should be aligned to 8 and have a length which is a multiple + // of `size_of::<AU64>()`, so this should always succeed. + test_new_helper_slice(Ref::<_, [AU64]>::from_bytes(&mut buf.t[..]).unwrap(), 3); + buf.set_default(); + let r = Ref::<_, [AU64]>::from_bytes_with_elems(&mut buf.t[..], 3).unwrap(); + test_new_helper_slice(r, 3); + + let ascending: [u8; 24] = (0..24).collect::<Vec<_>>().try_into().unwrap(); + // 16 ascending bytes followed by 8 zeros. + let mut ascending_prefix = ascending; + ascending_prefix[16..].copy_from_slice(&[0, 0, 0, 0, 0, 0, 0, 0]); + // 8 zeros followed by 16 ascending bytes. + let mut ascending_suffix = ascending; + ascending_suffix[..8].copy_from_slice(&[0, 0, 0, 0, 0, 0, 0, 0]); + { + buf.t = ascending_suffix; + let (r, suffix) = Ref::<_, [AU64]>::from_prefix_with_elems(&mut buf.t[..], 1).unwrap(); + assert_eq!(suffix, &ascending[8..]); + test_new_helper_slice(r, 1); + } + { + buf.t = ascending_prefix; + let (prefix, r) = Ref::<_, [AU64]>::from_suffix_with_elems(&mut buf.t[..], 1).unwrap(); + assert_eq!(prefix, &ascending[..16]); + test_new_helper_slice(r, 1); + } + } + + #[test] + fn test_new_oversized() { + // Test that a properly-aligned, overly-sized buffer works for + // `new_from_prefix` and `new_from_suffix`, and that they return the + // remainder and prefix of the slice respectively. + + let mut buf = Align::<[u8; 16], AU64>::default(); + { + // In a block so that `r` and `suffix` don't live too long. `buf.t` + // should be aligned to 8, so this should always succeed. + let (r, suffix) = Ref::<_, AU64>::from_prefix(&mut buf.t[..]).unwrap(); + assert_eq!(suffix.len(), 8); + test_new_helper(r); + } + { + buf.set_default(); + // `buf.t` should be aligned to 8, so this should always succeed. + let (prefix, r) = Ref::<_, AU64>::from_suffix(&mut buf.t[..]).unwrap(); + assert_eq!(prefix.len(), 8); + test_new_helper(r); + } + } + + #[test] + #[allow(clippy::cognitive_complexity)] + fn test_new_error() { + // Fail because the buffer is too large. + + // A buffer with an alignment of 8. + let buf = Align::<[u8; 16], AU64>::default(); + // `buf.t` should be aligned to 8, so only the length check should fail. + assert!(Ref::<_, AU64>::from_bytes(&buf.t[..]).is_err()); + + // Fail because the buffer is too small. + + // A buffer with an alignment of 8. + let buf = Align::<[u8; 4], AU64>::default(); + // `buf.t` should be aligned to 8, so only the length check should fail. + assert!(Ref::<_, AU64>::from_bytes(&buf.t[..]).is_err()); + assert!(Ref::<_, AU64>::from_prefix(&buf.t[..]).is_err()); + assert!(Ref::<_, AU64>::from_suffix(&buf.t[..]).is_err()); + + // Fail because the length is not a multiple of the element size. + + let buf = Align::<[u8; 12], AU64>::default(); + // `buf.t` has length 12, but element size is 8. + assert!(Ref::<_, [AU64]>::from_bytes(&buf.t[..]).is_err()); + + // Fail because the buffer is too short. + let buf = Align::<[u8; 12], AU64>::default(); + // `buf.t` has length 12, but the element size is 8 (and we're expecting + // two of them). For each function, we test with a length that would + // cause the size to overflow `usize`, and with a normal length that + // will fail thanks to the buffer being too short; these are different + // error paths, and while the error types are the same, the distinction + // shows up in code coverage metrics. + let n = (usize::MAX / mem::size_of::<AU64>()) + 1; + assert!(Ref::<_, [AU64]>::from_bytes_with_elems(&buf.t[..], n).is_err()); + assert!(Ref::<_, [AU64]>::from_bytes_with_elems(&buf.t[..], 2).is_err()); + assert!(Ref::<_, [AU64]>::from_prefix_with_elems(&buf.t[..], n).is_err()); + assert!(Ref::<_, [AU64]>::from_prefix_with_elems(&buf.t[..], 2).is_err()); + assert!(Ref::<_, [AU64]>::from_suffix_with_elems(&buf.t[..], n).is_err()); + assert!(Ref::<_, [AU64]>::from_suffix_with_elems(&buf.t[..], 2).is_err()); + + // Fail because the alignment is insufficient. + + // A buffer with an alignment of 8. An odd buffer size is chosen so that + // the last byte of the buffer has odd alignment. + let buf = Align::<[u8; 13], AU64>::default(); + // Slicing from 1, we get a buffer with size 12 (so the length check + // should succeed) but an alignment of only 1, which is insufficient. + assert!(Ref::<_, AU64>::from_bytes(&buf.t[1..]).is_err()); + assert!(Ref::<_, AU64>::from_prefix(&buf.t[1..]).is_err()); + assert!(Ref::<_, [AU64]>::from_bytes(&buf.t[1..]).is_err()); + assert!(Ref::<_, [AU64]>::from_bytes_with_elems(&buf.t[1..], 1).is_err()); + assert!(Ref::<_, [AU64]>::from_prefix_with_elems(&buf.t[1..], 1).is_err()); + assert!(Ref::<_, [AU64]>::from_suffix_with_elems(&buf.t[1..], 1).is_err()); + // Slicing is unnecessary here because `new_from_suffix` uses the suffix + // of the slice, which has odd alignment. + assert!(Ref::<_, AU64>::from_suffix(&buf.t[..]).is_err()); + + // Fail due to arithmetic overflow. + + let buf = Align::<[u8; 16], AU64>::default(); + let unreasonable_len = usize::MAX / mem::size_of::<AU64>() + 1; + assert!(Ref::<_, [AU64]>::from_prefix_with_elems(&buf.t[..], unreasonable_len).is_err()); + assert!(Ref::<_, [AU64]>::from_suffix_with_elems(&buf.t[..], unreasonable_len).is_err()); + } + + #[test] + #[allow(unstable_name_collisions)] + #[allow(clippy::as_conversions)] + fn test_into_ref_mut() { + #[allow(unused)] + use crate::util::AsAddress as _; + + let mut buf = Align::<[u8; 8], u64>::default(); + let r = Ref::<_, u64>::from_bytes(&buf.t[..]).unwrap(); + let rf = Ref::into_ref(r); + assert_eq!(rf, &0u64); + let buf_addr = (&buf.t as *const [u8; 8]).addr(); + assert_eq!((rf as *const u64).addr(), buf_addr); + + let r = Ref::<_, u64>::from_bytes(&mut buf.t[..]).unwrap(); + let rf = Ref::into_mut(r); + assert_eq!(rf, &mut 0u64); + assert_eq!((rf as *mut u64).addr(), buf_addr); + + *rf = u64::MAX; + assert_eq!(buf.t, [0xFF; 8]); + } + + #[test] + fn test_display_debug() { + let buf = Align::<[u8; 8], u64>::default(); + let r = Ref::<_, u64>::from_bytes(&buf.t[..]).unwrap(); + assert_eq!(format!("{}", r), "0"); + assert_eq!(format!("{:?}", r), "Ref(0)"); + + let buf = Align::<[u8; 8], u64>::default(); + let r = Ref::<_, [u64]>::from_bytes(&buf.t[..]).unwrap(); + assert_eq!(format!("{:?}", r), "Ref([0])"); + } + + #[test] + fn test_eq() { + let buf1 = 0_u64; + let r1 = Ref::<_, u64>::from_bytes(buf1.as_bytes()).unwrap(); + let buf2 = 0_u64; + let r2 = Ref::<_, u64>::from_bytes(buf2.as_bytes()).unwrap(); + assert_eq!(r1, r2); + } + + #[test] + fn test_ne() { + let buf1 = 0_u64; + let r1 = Ref::<_, u64>::from_bytes(buf1.as_bytes()).unwrap(); + let buf2 = 1_u64; + let r2 = Ref::<_, u64>::from_bytes(buf2.as_bytes()).unwrap(); + assert_ne!(r1, r2); + } + + #[test] + fn test_ord() { + let buf1 = 0_u64; + let r1 = Ref::<_, u64>::from_bytes(buf1.as_bytes()).unwrap(); + let buf2 = 1_u64; + let r2 = Ref::<_, u64>::from_bytes(buf2.as_bytes()).unwrap(); + assert!(r1 < r2); + assert_eq!(PartialOrd::partial_cmp(&r1, &r2), Some(Ordering::Less)); + assert_eq!(Ord::cmp(&r1, &r2), Ordering::Less); + } +} + +#[cfg(all(test, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS))] +mod benches { + use test::{self, Bencher}; + + use super::*; + use crate::util::testutil::*; + + #[bench] + fn bench_from_bytes_sized(b: &mut Bencher) { + let buf = Align::<[u8; 8], AU64>::default(); + // `buf.t` should be aligned to 8, so this should always succeed. + let bytes = &buf.t[..]; + b.iter(|| test::black_box(Ref::<_, AU64>::from_bytes(test::black_box(bytes)).unwrap())); + } + + #[bench] + fn bench_into_ref_sized(b: &mut Bencher) { + let buf = Align::<[u8; 8], AU64>::default(); + let bytes = &buf.t[..]; + let r = Ref::<_, AU64>::from_bytes(bytes).unwrap(); + b.iter(|| test::black_box(Ref::into_ref(test::black_box(r)))); + } + + #[bench] + fn bench_into_mut_sized(b: &mut Bencher) { + let mut buf = Align::<[u8; 8], AU64>::default(); + let buf = &mut buf.t[..]; + let _ = Ref::<_, AU64>::from_bytes(&mut *buf).unwrap(); + b.iter(move || { + // SAFETY: The preceding `from_bytes` succeeded, and so we know that + // `buf` is validly-aligned and has the correct length. + let r = unsafe { Ref::<&mut [u8], AU64>::new_unchecked(&mut *buf) }; + test::black_box(Ref::into_mut(test::black_box(r))); + }); + } + + #[bench] + fn bench_deref_sized(b: &mut Bencher) { + let buf = Align::<[u8; 8], AU64>::default(); + let bytes = &buf.t[..]; + let r = Ref::<_, AU64>::from_bytes(bytes).unwrap(); + b.iter(|| { + let temp = test::black_box(r); + test::black_box(temp.deref()); + }); + } + + #[bench] + fn bench_deref_mut_sized(b: &mut Bencher) { + let mut buf = Align::<[u8; 8], AU64>::default(); + let buf = &mut buf.t[..]; + let _ = Ref::<_, AU64>::from_bytes(&mut *buf).unwrap(); + b.iter(|| { + // SAFETY: The preceding `from_bytes` succeeded, and so we know that + // `buf` is validly-aligned and has the correct length. + let r = unsafe { Ref::<&mut [u8], AU64>::new_unchecked(&mut *buf) }; + let mut temp = test::black_box(r); + test::black_box(temp.deref_mut()); + }); + } +} diff --git a/rust/zerocopy/src/split_at.rs b/rust/zerocopy/src/split_at.rs new file mode 100644 index 000000000000..9a67d5acbb0d --- /dev/null +++ b/rust/zerocopy/src/split_at.rs @@ -0,0 +1,1090 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2025 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License <LICENSE-BSD or +// https://opensource.org/license/bsd-2-clause>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use super::*; +use crate::pointer::invariant::{Aligned, Exclusive, Invariants, Shared, Valid}; + +/// Types that can be split in two. +/// +/// This trait generalizes Rust's existing support for splitting slices to +/// support slices and slice-based dynamically-sized types ("slice DSTs"). +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(SplitAt)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{SplitAt, KnownLayout}; +/// #[derive(SplitAt, KnownLayout)] +/// #[repr(C)] +/// struct MyStruct<T: ?Sized> { +/// # /* +/// ..., +/// # */ +/// // `SplitAt` types must have at least one field. +/// field: T, +/// } +/// ``` +/// +/// This derive performs a sophisticated, compile-time safety analysis to +/// determine whether a type is `SplitAt`. +/// +/// # Safety +/// +/// This trait does not convey any safety guarantees to code outside this crate. +/// +/// You must not rely on the `#[doc(hidden)]` internals of `SplitAt`. Future +/// releases of zerocopy may make backwards-breaking changes to these items, +/// including changes that only affect soundness, which may cause code which +/// uses those items to silently become unsound. +/// +#[cfg_attr(feature = "derive", doc = "[derive]: zerocopy_derive::SplitAt")] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.SplitAt.html"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(SplitAt)]` to `{Self}`") +)] +// # Safety +// +// The trailing slice is well-aligned for its element type. `Self` is `[T]`, or +// a `repr(C)` or `repr(transparent)` slice DST. +pub unsafe trait SplitAt: KnownLayout<PointerMetadata = usize> { + /// The element type of the trailing slice. + type Elem; + + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// Unsafely splits `self` in two. + /// + /// # Safety + /// + /// The caller promises that `l_len` is not greater than the length of + /// `self`'s trailing slice. + /// + #[doc = codegen_section!( + header = "h5", + bench = "split_at_unchecked", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[inline] + #[must_use] + unsafe fn split_at_unchecked(&self, l_len: usize) -> Split<&Self> { + // SAFETY: By precondition on the caller, `l_len <= self.len()`. + unsafe { Split::<&Self>::new(self, l_len) } + } + + /// Attempts to split `self` in two. + /// + /// Returns `None` if `l_len` is greater than the length of `self`'s + /// trailing slice. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// length: u8, + /// body: [u8], + /// } + /// + /// // These bytes encode a `Packet`. + /// let bytes = &[4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::ref_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at(packet.length as usize).unwrap(); + /// + /// // Use the `Immutable` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_immutable(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "split_at", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[inline] + #[must_use = "has no side effects"] + fn split_at(&self, l_len: usize) -> Option<Split<&Self>> { + MetadataOf::new_in_bounds(self, l_len).map( + #[inline(always)] + |l_len| { + // SAFETY: We have ensured that `l_len <= self.len()` (by + // post-condition on `MetadataOf::new_in_bounds`) + unsafe { Split::new(self, l_len.get()) } + }, + ) + } + + /// Unsafely splits `self` in two. + /// + /// # Safety + /// + /// The caller promises that `l_len` is not greater than the length of + /// `self`'s trailing slice. + /// + #[doc = codegen_header!("h5", "split_at_mut_unchecked")] + /// + /// See [`SplitAt::split_at_unchecked`](#method.split_at_unchecked.codegen). + #[inline] + #[must_use] + unsafe fn split_at_mut_unchecked(&mut self, l_len: usize) -> Split<&mut Self> { + // SAFETY: By precondition on the caller, `l_len <= self.len()`. + unsafe { Split::<&mut Self>::new(self, l_len) } + } + + /// Attempts to split `self` in two. + /// + /// Returns `None` if `l_len` is greater than the length of `self`'s + /// trailing slice, or if the given `l_len` would result in [the trailing + /// padding](KnownLayout#slice-dst-layout) of the left portion overlapping + /// the right portion. + /// + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, IntoBytes)] + /// #[repr(C)] + /// struct Packet<B: ?Sized> { + /// length: u8, + /// body: B, + /// } + /// + /// // These bytes encode a `Packet`. + /// let mut bytes = &mut [4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::<[u8]>::mut_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// { + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at_mut(packet.length as usize).unwrap(); + /// + /// // Use the `IntoBytes` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_into_bytes(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// + /// rest.fill(0); + /// } + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 0, 0, 0, 0, 0]); + /// ``` + /// + #[doc = codegen_header!("h5", "split_at_mut")] + /// + /// See [`SplitAt::split_at`](#method.split_at.codegen). + #[inline] + fn split_at_mut(&mut self, l_len: usize) -> Option<Split<&mut Self>> { + MetadataOf::new_in_bounds(self, l_len).map( + #[inline(always)] + |l_len| { + // SAFETY: We have ensured that `l_len <= self.len()` (by + // post-condition on `MetadataOf::new_in_bounds`) + unsafe { Split::new(self, l_len.get()) } + }, + ) + } +} + +// SAFETY: `[T]`'s trailing slice is `[T]`, which is trivially aligned. +unsafe impl<T> SplitAt for [T] { + type Elem = T; + + #[inline] + #[allow(dead_code)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } +} + +/// A `T` that has been split into two possibly-overlapping parts. +/// +/// For some dynamically sized types, the padding that appears after the +/// trailing slice field [is a dynamic function of the trailing slice +/// length](KnownLayout#slice-dst-layout). If `T` is split at a length that +/// requires trailing padding, the trailing padding of the left part of the +/// split `T` will overlap the right part. If `T` is a mutable reference or +/// permits interior mutation, you must ensure that the left and right parts do +/// not overlap. You can do this at zero-cost using using +/// [`Self::via_immutable`], [`Self::via_into_bytes`], or +/// [`Self::via_unaligned`], or with a dynamic check by using +/// [`Self::via_runtime_check`]. +#[derive(Debug)] +pub struct Split<T> { + /// A pointer to the source slice DST. + source: T, + /// The length of the future left half of `source`. + /// + /// # Safety + /// + /// If `source` is a pointer to a slice DST, `l_len` is no greater than + /// `source`'s length. + l_len: usize, +} + +impl<T> Split<T> { + /// Produces a `Split` of `source` with `l_len`. + /// + /// # Safety + /// + /// `l_len` is no greater than `source`'s length. + #[inline(always)] + unsafe fn new(source: T, l_len: usize) -> Self { + Self { source, l_len } + } +} + +impl<'a, T> Split<&'a T> +where + T: ?Sized + SplitAt, +{ + #[inline(always)] + fn into_ptr(self) -> Split<Ptr<'a, T, (Shared, Aligned, Valid)>> { + let source = Ptr::from_ref(self.source); + // SAFETY: `Ptr::from_ref(self.source)` points to exactly `self.source` + // and thus maintains the invariants of `self` with respect to `l_len`. + unsafe { Split::new(source, self.l_len) } + } + + /// Produces the split parts of `self`, using [`Immutable`] to ensure that + /// it is sound to have concurrent references to both parts. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// length: u8, + /// body: [u8], + /// } + /// + /// // These bytes encode a `Packet`. + /// let bytes = &[4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::ref_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at(packet.length as usize).unwrap(); + /// + /// // Use the `Immutable` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_immutable(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "split_via_immutable", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_immutable(self) -> (&'a T, &'a [T::Elem]) + where + T: Immutable, + { + let (l, r) = self.into_ptr().via_immutable(); + (l.as_ref(), r.as_ref()) + } + + /// Produces the split parts of `self`, using [`IntoBytes`] to ensure that + /// it is sound to have concurrent references to both parts. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, Immutable, IntoBytes)] + /// #[repr(C)] + /// struct Packet<B: ?Sized> { + /// length: u8, + /// body: B, + /// } + /// + /// // These bytes encode a `Packet`. + /// let bytes = &[4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::<[u8]>::ref_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at(packet.length as usize).unwrap(); + /// + /// // Use the `IntoBytes` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_into_bytes(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// ``` + /// + #[doc = codegen_header!("h5", "split_via_into_bytes")] + /// + /// See [`Split::via_immutable`](#method.split_via_immutable.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_into_bytes(self) -> (&'a T, &'a [T::Elem]) + where + T: IntoBytes, + { + let (l, r) = self.into_ptr().via_into_bytes(); + (l.as_ref(), r.as_ref()) + } + + /// Produces the split parts of `self`, using [`Unaligned`] to ensure that + /// it is sound to have concurrent references to both parts. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, Immutable, Unaligned)] + /// #[repr(C)] + /// struct Packet { + /// length: u8, + /// body: [u8], + /// } + /// + /// // These bytes encode a `Packet`. + /// let bytes = &[4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::ref_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at(packet.length as usize).unwrap(); + /// + /// // Use the `Unaligned` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_unaligned(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// ``` + /// + #[doc = codegen_header!("h5", "split_via_unaligned")] + /// + /// See [`Split::via_immutable`](#method.split_via_immutable.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_unaligned(self) -> (&'a T, &'a [T::Elem]) + where + T: Unaligned, + { + let (l, r) = self.into_ptr().via_unaligned(); + (l.as_ref(), r.as_ref()) + } + + /// Produces the split parts of `self`, using a dynamic check to ensure that + /// it is sound to have concurrent references to both parts. You should + /// prefer using [`Self::via_immutable`], [`Self::via_into_bytes`], or + /// [`Self::via_unaligned`], which have no runtime cost. + /// + /// Note that this check is overly conservative if `T` is [`Immutable`]; for + /// some types, this check will reject some splits which + /// [`Self::via_immutable`] will accept. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes, IntoBytes, network_endian::U16}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, Immutable, Debug)] + /// #[repr(C, align(2))] + /// struct Packet { + /// length: U16, + /// body: [u8], + /// } + /// + /// // These bytes encode a `Packet`. + /// let bytes = [ + /// 4u16.to_be(), + /// 1u16.to_be(), + /// 2u16.to_be(), + /// 3u16.to_be(), + /// 4u16.to_be() + /// ]; + /// + /// let packet = Packet::ref_from_bytes(bytes.as_bytes()).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [0, 1, 0, 2, 0, 3, 0, 4]); + /// + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at(packet.length.into()).unwrap(); + /// + /// // Use a dynamic check to prove that it's okay to return concurrent + /// // references to `packet` and `rest`. + /// let (packet, rest) = split.via_runtime_check().unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [0, 1, 0, 2]); + /// assert_eq!(rest, [0, 3, 0, 4]); + /// + /// // Attempt to split `packet` at `length - 1`. + /// let idx = packet.length.get() - 1; + /// let split = packet.split_at(idx as usize).unwrap(); + /// + /// // Attempt (and fail) to use a dynamic check to prove that it's okay + /// // to return concurrent references to `packet` and `rest`. Note that + /// // this is a case of `via_runtime_check` being overly conservative. + /// // Although the left and right parts indeed overlap, the `Immutable` + /// // bound ensures that concurrently referencing these overlapping + /// // parts is sound. + /// assert!(split.via_runtime_check().is_err()); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "split_via_runtime_check", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_runtime_check(self) -> Result<(&'a T, &'a [T::Elem]), Self> { + match self.into_ptr().via_runtime_check() { + Ok((l, r)) => Ok((l.as_ref(), r.as_ref())), + Err(s) => Err(s.into_ref()), + } + } + + /// Unsafely produces the split parts of `self`. + /// + /// # Safety + /// + /// If `T` permits interior mutation, the trailing padding bytes of the left + /// portion must not overlap the right portion. For some dynamically sized + /// types, the padding that appears after the trailing slice field [is a + /// dynamic function of the trailing slice + /// length](KnownLayout#slice-dst-layout). Thus, for some types, this + /// condition is dependent on the length of the left portion. + /// + #[doc = codegen_section!( + header = "h5", + bench = "split_via_unchecked", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[inline(always)] + pub unsafe fn via_unchecked(self) -> (&'a T, &'a [T::Elem]) { + // SAFETY: The aliasing of `self.into_ptr()` is not `Exclusive`, but the + // caller has promised that if `T` permits interior mutation then the + // left and right portions of `self` split at `l_len` do not overlap. + let (l, r) = unsafe { self.into_ptr().via_unchecked() }; + (l.as_ref(), r.as_ref()) + } +} + +impl<'a, T> Split<&'a mut T> +where + T: ?Sized + SplitAt, +{ + #[inline(always)] + fn into_ptr(self) -> Split<Ptr<'a, T, (Exclusive, Aligned, Valid)>> { + let source = Ptr::from_mut(self.source); + // SAFETY: `Ptr::from_mut(self.source)` points to exactly `self.source`, + // and thus maintains the invariants of `self` with respect to `l_len`. + unsafe { Split::new(source, self.l_len) } + } + + /// Produces the split parts of `self`, using [`IntoBytes`] to ensure that + /// it is sound to have concurrent references to both parts. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, IntoBytes)] + /// #[repr(C)] + /// struct Packet<B: ?Sized> { + /// length: u8, + /// body: B, + /// } + /// + /// // These bytes encode a `Packet`. + /// let mut bytes = &mut [4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::<[u8]>::mut_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// { + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at_mut(packet.length as usize).unwrap(); + /// + /// // Use the `IntoBytes` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_into_bytes(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// + /// rest.fill(0); + /// } + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 0, 0, 0, 0, 0]); + /// ``` + /// + /// # Code Generation + /// + /// See [`Split::via_immutable`](#method.split_via_immutable.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_into_bytes(self) -> (&'a mut T, &'a mut [T::Elem]) + where + T: IntoBytes, + { + let (l, r) = self.into_ptr().via_into_bytes(); + (l.as_mut(), r.as_mut()) + } + + /// Produces the split parts of `self`, using [`Unaligned`] to ensure that + /// it is sound to have concurrent references to both parts. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, IntoBytes, Unaligned)] + /// #[repr(C)] + /// struct Packet<B: ?Sized> { + /// length: u8, + /// body: B, + /// } + /// + /// // These bytes encode a `Packet`. + /// let mut bytes = &mut [4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::<[u8]>::mut_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// { + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at_mut(packet.length as usize).unwrap(); + /// + /// // Use the `Unaligned` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_unaligned(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// + /// rest.fill(0); + /// } + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 0, 0, 0, 0, 0]); + /// ``` + /// + /// # Code Generation + /// + /// See [`Split::via_immutable`](#method.split_via_immutable.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_unaligned(self) -> (&'a mut T, &'a mut [T::Elem]) + where + T: Unaligned, + { + let (l, r) = self.into_ptr().via_unaligned(); + (l.as_mut(), r.as_mut()) + } + + /// Produces the split parts of `self`, using a dynamic check to ensure that + /// it is sound to have concurrent references to both parts. You should + /// prefer using [`Self::via_into_bytes`] or [`Self::via_unaligned`], which + /// have no runtime cost. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, IntoBytes, Debug)] + /// #[repr(C)] + /// struct Packet<B: ?Sized> { + /// length: u8, + /// body: B, + /// } + /// + /// // These bytes encode a `Packet`. + /// let mut bytes = &mut [4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::<[u8]>::mut_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// { + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at_mut(packet.length as usize).unwrap(); + /// + /// // Use a dynamic check to prove that it's okay to return concurrent + /// // references to `packet` and `rest`. + /// let (packet, rest) = split.via_runtime_check().unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// + /// rest.fill(0); + /// } + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 0, 0, 0, 0, 0]); + /// ``` + /// + /// # Code Generation + /// + /// See [`Split::via_runtime_check`](#method.split_via_runtime_check.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_runtime_check(self) -> Result<(&'a mut T, &'a mut [T::Elem]), Self> { + match self.into_ptr().via_runtime_check() { + Ok((l, r)) => Ok((l.as_mut(), r.as_mut())), + Err(s) => Err(s.into_mut()), + } + } + + /// Unsafely produces the split parts of `self`. + /// + /// # Safety + /// + /// The trailing padding bytes of the left portion must not overlap the + /// right portion. For some dynamically sized types, the padding that + /// appears after the trailing slice field [is a dynamic function of the + /// trailing slice length](KnownLayout#slice-dst-layout). Thus, for some + /// types, this condition is dependent on the length of the left portion. + /// + /// # Code Generation + /// + /// See [`Split::via_unchecked`](#method.split_via_unchecked.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub unsafe fn via_unchecked(self) -> (&'a mut T, &'a mut [T::Elem]) { + // SAFETY: The aliasing of `self.into_ptr()` is `Exclusive`, and the + // caller has promised that the left and right portions of `self` split + // at `l_len` do not overlap. + let (l, r) = unsafe { self.into_ptr().via_unchecked() }; + (l.as_mut(), r.as_mut()) + } +} + +impl<'a, T, I> Split<Ptr<'a, T, I>> +where + T: ?Sized + SplitAt, + I: Invariants<Alignment = Aligned, Validity = Valid>, +{ + fn into_ref(self) -> Split<&'a T> + where + I: Invariants<Aliasing = Shared>, + { + // SAFETY: `self.source.as_ref()` points to exactly the same referent as + // `self.source` and thus maintains the invariants of `self` with + // respect to `l_len`. + unsafe { Split::new(self.source.as_ref(), self.l_len) } + } + + fn into_mut(self) -> Split<&'a mut T> + where + I: Invariants<Aliasing = Exclusive>, + { + // SAFETY: `self.source.as_mut()` points to exactly the same referent as + // `self.source` and thus maintains the invariants of `self` with + // respect to `l_len`. + unsafe { Split::new(self.source.unify_invariants().as_mut(), self.l_len) } + } + + /// Produces the length of `self`'s left part. + #[inline(always)] + fn l_len(&self) -> MetadataOf<T> { + // SAFETY: By invariant on `Split`, `self.l_len` is not greater than the + // length of `self.source`. + unsafe { MetadataOf::<T>::new_unchecked(self.l_len) } + } + + /// Produces the split parts of `self`, using [`Immutable`] to ensure that + /// it is sound to have concurrent references to both parts. + #[inline(always)] + fn via_immutable(self) -> (Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>) + where + T: Immutable, + I: Invariants<Aliasing = Shared>, + { + // SAFETY: `Aliasing = Shared` and `T: Immutable`. + unsafe { self.via_unchecked() } + } + + /// Produces the split parts of `self`, using [`IntoBytes`] to ensure that + /// it is sound to have concurrent references to both parts. + #[inline(always)] + fn via_into_bytes(self) -> (Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>) + where + T: IntoBytes, + { + // SAFETY: By `T: IntoBytes`, `T` has no padding for any length. + // Consequently, `T` can be split into non-overlapping parts at any + // index. + unsafe { self.via_unchecked() } + } + + /// Produces the split parts of `self`, using [`Unaligned`] to ensure that + /// it is sound to have concurrent references to both parts. + #[inline(always)] + fn via_unaligned(self) -> (Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>) + where + T: Unaligned, + { + // SAFETY: By `T: SplitAt + Unaligned`, `T` is either a slice or a + // `repr(C)` or `repr(transparent)` slice DST that is well-aligned at + // any address and length. If `T` is a slice DST with alignment 1, + // `repr(C)` or `repr(transparent)` ensures that no padding is placed + // after the final element of the trailing slice. Consequently, `T` can + // be split into strictly non-overlapping parts any any index. + unsafe { self.via_unchecked() } + } + + /// Produces the split parts of `self`, using a dynamic check to ensure that + /// it is sound to have concurrent references to both parts. You should + /// prefer using [`Self::via_immutable`], [`Self::via_into_bytes`], or + /// [`Self::via_unaligned`], which have no runtime cost. + #[inline(always)] + fn via_runtime_check(self) -> Result<(Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>), Self> { + let l_len = self.l_len(); + // FIXME(#1290): Once we require `KnownLayout` on all fields, add an + // `IS_IMMUTABLE` associated const, and add `T::IS_IMMUTABLE ||` to the + // below check. + if l_len.padding_needed_for() == 0 { + // SAFETY: By `T: SplitAt`, `T` is either `[T]`, or a `repr(C)` or + // `repr(transparent)` slice DST, for which the trailing padding + // needed to accommodate `l_len` trailing elements is + // `l_len.padding_needed_for()`. If no trailing padding is required, + // the left and right parts are strictly non-overlapping. + Ok(unsafe { self.via_unchecked() }) + } else { + Err(self) + } + } + + /// Unsafely produces the split parts of `self`. + /// + /// # Safety + /// + /// The caller promises that if `I::Aliasing` is [`Exclusive`] or `T` + /// permits interior mutation, then `l_len.padding_needed_for() == 0`. + #[inline(always)] + unsafe fn via_unchecked(self) -> (Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>) { + let l_len = self.l_len(); + let inner = self.source.as_inner(); + + // SAFETY: By invariant on `Self::l_len`, `l_len` is not greater than + // the length of `inner`'s trailing slice. + let (left, right) = unsafe { inner.split_at_unchecked(l_len) }; + + // Lemma 0: `left` and `right` conform to the aliasing invariant + // `I::Aliasing`. Proof: If `I::Aliasing` is `Exclusive` or `T` permits + // interior mutation, the caller promises that `l_len.padding_needed_for() + // == 0`. Consequently, by post-condition on `PtrInner::split_at_unchecked`, + // there is no trailing padding after `left`'s final element that would + // overlap into `right`. If `I::Aliasing` is shared and `T` forbids interior + // mutation, then overlap between their referents is permissible. + + // SAFETY: + // 0. `left` conforms to the aliasing invariant of `I::Aliasing`, by Lemma 0. + // 1. `left` conforms to the alignment invariant of `I::Alignment, because + // the referents of `left` and `Self` have the same address and type + // (and, thus, alignment requirement). + // 2. `left` conforms to the validity invariant of `I::Validity`, neither + // the type nor bytes of `left`'s referent have been changed. + let left = unsafe { Ptr::from_inner(left) }; + + // SAFETY: + // 0. `right` conforms to the aliasing invariant of `I::Aliasing`, by Lemma + // 0. + // 1. `right` conforms to the alignment invariant of `I::Alignment, because + // if `ptr` with `I::Alignment = Aligned`, then by invariant on `T: + // SplitAt`, the trailing slice of `ptr` (from which `right` is derived) + // will also be well-aligned. + // 2. `right` conforms to the validity invariant of `I::Validity`, + // because `right: [T::Elem]` is derived from the trailing slice of + // `ptr`, which, by contract on `T: SplitAt::Elem`, has type + // `[T::Elem]`. The `left` part cannot be used to invalidate `right`, + // because the caller promises that if `I::Aliasing` is `Exclusive` + // or `T` permits interior mutation, then `l_len.padding_needed_for() + // == 0` and thus the parts will be non-overlapping. + let right = unsafe { Ptr::from_inner(right) }; + + (left, right) + } +} + +#[cfg(test)] +mod tests { + #[cfg(feature = "derive")] + #[test] + fn test_split_at() { + use crate::{FromBytes, Immutable, IntoBytes, KnownLayout, SplitAt}; + + #[derive(FromBytes, KnownLayout, SplitAt, IntoBytes, Immutable, Debug)] + #[repr(C)] + struct SliceDst<const OFFSET: usize> { + prefix: [u8; OFFSET], + trailing: [u8], + } + + #[allow(clippy::as_conversions)] + fn test_split_at<const OFFSET: usize, const BUFFER_SIZE: usize>() { + // Test `split_at` + let n: usize = BUFFER_SIZE - OFFSET; + let arr = [1; BUFFER_SIZE]; + let dst = SliceDst::<OFFSET>::ref_from_bytes(&arr[..]).unwrap(); + for i in 0..=n { + let (l, r) = dst.split_at(i).unwrap().via_runtime_check().unwrap(); + let l_sum: u8 = l.trailing.iter().sum(); + let r_sum: u8 = r.iter().sum(); + assert_eq!(l_sum, i as u8); + assert_eq!(r_sum, (n - i) as u8); + assert_eq!(l_sum + r_sum, n as u8); + } + + // Test `split_at_mut` + let n: usize = BUFFER_SIZE - OFFSET; + let mut arr = [1; BUFFER_SIZE]; + let dst = SliceDst::<OFFSET>::mut_from_bytes(&mut arr[..]).unwrap(); + for i in 0..=n { + let (l, r) = dst.split_at_mut(i).unwrap().via_runtime_check().unwrap(); + let l_sum: u8 = l.trailing.iter().sum(); + let r_sum: u8 = r.iter().sum(); + assert_eq!(l_sum, i as u8); + assert_eq!(r_sum, (n - i) as u8); + assert_eq!(l_sum + r_sum, n as u8); + } + } + + test_split_at::<0, 16>(); + test_split_at::<1, 17>(); + test_split_at::<2, 18>(); + } + + #[cfg(feature = "derive")] + #[test] + #[allow(clippy::as_conversions)] + fn test_split_at_overlapping() { + use crate::{FromBytes, Immutable, IntoBytes, KnownLayout, SplitAt}; + + #[derive(FromBytes, KnownLayout, SplitAt, Immutable)] + #[repr(C, align(2))] + struct SliceDst { + prefix: u8, + trailing: [u8], + } + + const N: usize = 16; + + let arr = [1u16; N]; + let dst = SliceDst::ref_from_bytes(arr.as_bytes()).unwrap(); + + for i in 0..N { + let split = dst.split_at(i).unwrap().via_runtime_check(); + if i % 2 == 1 { + assert!(split.is_ok()); + } else { + assert!(split.is_err()); + } + } + } + #[test] + fn test_split_at_unchecked() { + use crate::SplitAt; + let mut arr = [1, 2, 3, 4]; + let slice = &arr[..]; + // SAFETY: 2 <= arr.len() (4) + let split = unsafe { SplitAt::split_at_unchecked(slice, 2) }; + // SAFETY: SplitAt::split_at_unchecked guarantees that the split is valid. + let (l, r) = unsafe { split.via_unchecked() }; + assert_eq!(l, &[1, 2]); + assert_eq!(r, &[3, 4]); + + let slice_mut = &mut arr[..]; + // SAFETY: 2 <= arr.len() (4) + let split = unsafe { SplitAt::split_at_mut_unchecked(slice_mut, 2) }; + // SAFETY: SplitAt::split_at_mut_unchecked guarantees that the split is valid. + let (l, r) = unsafe { split.via_unchecked() }; + assert_eq!(l, &mut [1, 2]); + assert_eq!(r, &mut [3, 4]); + } + + #[test] + fn test_split_at_via_methods() { + use crate::{FromBytes, Immutable, IntoBytes, KnownLayout, SplitAt}; + #[derive(FromBytes, KnownLayout, SplitAt, IntoBytes, Immutable, Debug)] + #[repr(C)] + struct Packet { + length: u8, + body: [u8], + } + + let arr = [1, 2, 3, 4]; + let packet = Packet::ref_from_bytes(&arr[..]).unwrap(); + + let split1 = packet.split_at(2).unwrap(); + let (l, r) = split1.via_immutable(); + assert_eq!(l.length, 1); + assert_eq!(r, &[4]); + + let split2 = packet.split_at(2).unwrap(); + let (l, r) = split2.via_into_bytes(); + assert_eq!(l.length, 1); + assert_eq!(r, &[4]); + } + #[test] + fn test_split_at_via_unaligned() { + use crate::{FromBytes, Immutable, IntoBytes, KnownLayout, SplitAt, Unaligned}; + #[derive(FromBytes, KnownLayout, SplitAt, IntoBytes, Immutable, Unaligned)] + #[repr(C)] + struct Packet { + length: u8, + body: [u8], + } + + let arr = [1, 2, 3, 4]; + let packet = Packet::ref_from_bytes(&arr[..]).unwrap(); + + let split = packet.split_at(2).unwrap(); + let (l, r) = split.via_unaligned(); + assert_eq!(l.length, 1); + assert_eq!(r, &[4]); + } +} diff --git a/rust/zerocopy/src/util/macro_util.rs b/rust/zerocopy/src/util/macro_util.rs new file mode 100644 index 000000000000..1abb0fbeb46e --- /dev/null +++ b/rust/zerocopy/src/util/macro_util.rs @@ -0,0 +1,1310 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2022 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Utilities used by macros and by `zerocopy-derive`. +//! +//! These are defined here `zerocopy` rather than in code generated by macros or +//! by `zerocopy-derive` so that they can be compiled once rather than +//! recompiled for every invocation (e.g., if they were defined in generated +//! code, then deriving `IntoBytes` and `FromBytes` on three different types +//! would result in the code in question being emitted and compiled six +//! different times). + +#![allow(missing_debug_implementations)] + +// FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): Remove +// this `cfg` when `size_of_val_raw` is stabilized. +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] +#[cfg(not(target_pointer_width = "16"))] +use core::ptr::{self, NonNull}; +use core::{marker::PhantomData, mem, num::Wrapping}; + +use crate::{ + pointer::{ + cast::CastSized, + invariant::{Aligned, Initialized, Valid}, + BecauseImmutable, + }, + FromBytes, Immutable, IntoBytes, KnownLayout, Ptr, ReadOnly, TryFromBytes, ValidityError, +}; + +/// Projects the type of the field at `Index` in `Self` without regard for field +/// privacy. +/// +/// The `Index` parameter is any sort of handle that identifies the field; its +/// definition is the obligation of the implementer. +/// +/// # Safety +/// +/// Unsafe code may assume that this accurately reflects the definition of +/// `Self`. +pub unsafe trait Field<Index> { + /// The type of the field at `Index`. + type Type: ?Sized; +} + +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented( + message = "`{T}` has {PADDING_BYTES} total byte(s) of padding", + label = "types with padding cannot implement `IntoBytes`", + note = "consider using `zerocopy::Unalign` to lower the alignment of individual fields", + note = "consider adding explicit fields where padding would be", + note = "consider using `#[repr(packed)]` to remove padding" + ) +)] +pub trait PaddingFree<T: ?Sized, const PADDING_BYTES: usize> {} +impl<T: ?Sized> PaddingFree<T, 0> for () {} + +// FIXME(#1112): In the slice DST case, we should delegate to *both* +// `PaddingFree` *and* `DynamicPaddingFree` (and probably rename `PaddingFree` +// to `StaticPaddingFree` or something - or introduce a third trait with that +// name) so that we can have more clear error messages. + +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented( + message = "`{T}` has one or more padding bytes", + label = "types with padding cannot implement `IntoBytes`", + note = "consider using `zerocopy::Unalign` to lower the alignment of individual fields", + note = "consider adding explicit fields where padding would be", + note = "consider using `#[repr(packed)]` to remove padding" + ) +)] +pub trait DynamicPaddingFree<T: ?Sized, const HAS_PADDING: bool> {} +impl<T: ?Sized> DynamicPaddingFree<T, false> for () {} + +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] +#[cfg(not(target_pointer_width = "16"))] +const _64K: usize = 1 << 16; + +// FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): Remove +// this `cfg` when `size_of_val_raw` is stabilized. +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] +#[cfg(not(target_pointer_width = "16"))] +#[repr(C, align(65536))] +struct Aligned64kAllocation([u8; _64K]); + +/// A pointer to an aligned allocation of size 2^16. +/// +/// # Safety +/// +/// `ALIGNED_64K_ALLOCATION` is guaranteed to point to the entirety of an +/// allocation with size and alignment 2^16, and to have valid provenance. +// FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): Remove +// this `cfg` when `size_of_val_raw` is stabilized. +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] +#[cfg(not(target_pointer_width = "16"))] +pub const ALIGNED_64K_ALLOCATION: NonNull<[u8]> = { + const REF: &Aligned64kAllocation = &Aligned64kAllocation([0; _64K]); + let ptr: *const Aligned64kAllocation = REF; + let ptr: *const [u8] = ptr::slice_from_raw_parts(ptr.cast(), _64K); + // SAFETY: + // - `ptr` is derived from a Rust reference, which is guaranteed to be + // non-null. + // - `ptr` is derived from an `&Aligned64kAllocation`, which has size and + // alignment `_64K` as promised. Its length is initialized to `_64K`, + // which means that it refers to the entire allocation. + // - `ptr` is derived from a Rust reference, which is guaranteed to have + // valid provenance. + // + // FIXME(#429): Once `NonNull::new_unchecked` docs document that it + // preserves provenance, cite those docs. + // FIXME: Replace this `as` with `ptr.cast_mut()` once our MSRV >= 1.65 + #[allow(clippy::as_conversions)] + unsafe { + NonNull::new_unchecked(ptr as *mut _) + } +}; + +/// Computes the offset of the base of the field `$trailing_field_name` within +/// the type `$ty`. +/// +/// `trailing_field_offset!` produces code which is valid in a `const` context. +// FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): Remove +// this `cfg` when `size_of_val_raw` is stabilized. +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! trailing_field_offset { + ($ty:ty, $trailing_field_name:tt) => {{ + let min_size = { + let zero_elems: *const [()] = + $crate::util::macro_util::core_reexport::ptr::slice_from_raw_parts( + $crate::util::macro_util::core_reexport::ptr::NonNull::<()>::dangling() + .as_ptr() + .cast_const(), + 0, + ); + // SAFETY: + // - If `$ty` is `Sized`, `size_of_val_raw` is always safe to call. + // - Otherwise: + // - If `$ty` is not a slice DST, this pointer conversion will + // fail due to "mismatched vtable kinds", and compilation will + // fail. + // - If `$ty` is a slice DST, we have constructed `zero_elems` to + // have zero trailing slice elements. Per the `size_of_val_raw` + // docs, "For the special case where the dynamic tail length is + // 0, this function is safe to call." [1] + // + // [1] https://doc.rust-lang.org/nightly/std/mem/fn.size_of_val_raw.html + unsafe { + #[allow(clippy::as_conversions)] + $crate::util::macro_util::core_reexport::mem::size_of_val_raw( + zero_elems as *const $ty, + ) + } + }; + + assert!(min_size <= _64K); + + #[allow(clippy::as_conversions)] + let ptr = ALIGNED_64K_ALLOCATION.as_ptr() as *const $ty; + + // SAFETY: + // - Thanks to the preceding `assert!`, we know that the value with zero + // elements fits in `_64K` bytes, and thus in the allocation addressed + // by `ALIGNED_64K_ALLOCATION`. The offset of the trailing field is + // guaranteed to be no larger than this size, so this field projection + // is guaranteed to remain in-bounds of its allocation. + // - Because the minimum size is no larger than `_64K` bytes, and + // because an object's size must always be a multiple of its alignment + // [1], we know that `$ty`'s alignment is no larger than `_64K`. The + // allocation addressed by `ALIGNED_64K_ALLOCATION` is guaranteed to + // be aligned to `_64K`, so `ptr` is guaranteed to satisfy `$ty`'s + // alignment. + // - As required by `addr_of!`, we do not write through `field`. + // + // Note that, as of [2], this requirement is technically unnecessary + // for Rust versions >= 1.75.0, but no harm in guaranteeing it anyway + // until we bump our MSRV. + // + // [1] Per https://doc.rust-lang.org/reference/type-layout.html: + // + // The size of a value is always a multiple of its alignment. + // + // [2] https://github.com/rust-lang/reference/pull/1387 + let field = unsafe { + $crate::util::macro_util::core_reexport::ptr::addr_of!((*ptr).$trailing_field_name) + }; + // SAFETY: + // - Both `ptr` and `field` are derived from the same allocated object. + // - By the preceding safety comment, `field` is in bounds of that + // allocated object. + // - The distance, in bytes, between `ptr` and `field` is required to be + // a multiple of the size of `u8`, which is trivially true because + // `u8`'s size is 1. + // - The distance, in bytes, cannot overflow `isize`. This is guaranteed + // because no allocated object can have a size larger than can fit in + // `isize`. [1] + // - The distance being in-bounds cannot rely on wrapping around the + // address space. This is guaranteed because the same is guaranteed of + // allocated objects. [1] + // + // [1] FIXME(#429), FIXME(https://github.com/rust-lang/rust/pull/116675): + // Once these are guaranteed in the Reference, cite it. + let offset = unsafe { field.cast::<u8>().offset_from(ptr.cast::<u8>()) }; + // Guaranteed not to be lossy: `field` comes after `ptr`, so the offset + // from `ptr` to `field` is guaranteed to be positive. + assert!(offset >= 0); + Some( + #[allow(clippy::as_conversions)] + { + offset as usize + }, + ) + }}; +} + +/// Computes alignment of `$ty: ?Sized`. +/// +/// `align_of!` produces code which is valid in a `const` context. +// FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): Remove +// this `cfg` when `size_of_val_raw` is stabilized. +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! align_of { + ($ty:ty) => {{ + // SAFETY: `OffsetOfTrailingIsAlignment` is `repr(C)`, and its layout is + // guaranteed [1] to begin with the single-byte layout for `_byte`, + // followed by the padding needed to align `_trailing`, then the layout + // for `_trailing`, and finally any trailing padding bytes needed to + // correctly-align the entire struct. + // + // This macro computes the alignment of `$ty` by counting the number of + // bytes preceding `_trailing`. For instance, if the alignment of `$ty` + // is `1`, then no padding is required align `_trailing` and it will be + // located immediately after `_byte` at offset 1. If the alignment of + // `$ty` is 2, then a single padding byte is required before + // `_trailing`, and `_trailing` will be located at offset 2. + + // This correspondence between offset and alignment holds for all valid + // Rust alignments, and we confirm this exhaustively (or, at least up to + // the maximum alignment supported by `trailing_field_offset!`) in + // `test_align_of_dst`. + // + // [1]: https://doc.rust-lang.org/nomicon/other-reprs.html#reprc + + #[repr(C)] + struct OffsetOfTrailingIsAlignment { + _byte: u8, + _trailing: $ty, + } + + trailing_field_offset!(OffsetOfTrailingIsAlignment, _trailing) + }}; +} + +mod size_to_tag { + pub trait SizeToTag<const SIZE: usize> { + type Tag; + } + + impl SizeToTag<1> for () { + type Tag = u8; + } + impl SizeToTag<2> for () { + type Tag = u16; + } + impl SizeToTag<4> for () { + type Tag = u32; + } + impl SizeToTag<8> for () { + type Tag = u64; + } + impl SizeToTag<16> for () { + type Tag = u128; + } +} + +/// An alias for the unsigned integer of the given size in bytes. +#[doc(hidden)] +pub type SizeToTag<const SIZE: usize> = <() as size_to_tag::SizeToTag<SIZE>>::Tag; + +// We put `Sized` in its own module so it can have the same name as the standard +// library `Sized` without shadowing it in the parent module. +#[cfg(not(no_zerocopy_diagnostic_on_unimplemented_1_78_0))] +mod __size_of { + #[diagnostic::on_unimplemented( + message = "`{Self}` is unsized", + label = "`IntoBytes` needs all field types to be `Sized` in order to determine whether there is padding", + note = "consider using `#[repr(packed)]` to remove padding", + note = "`IntoBytes` does not require the fields of `#[repr(packed)]` types to be `Sized`" + )] + pub trait Sized: core::marker::Sized {} + impl<T: core::marker::Sized> Sized for T {} + + #[inline(always)] + #[must_use] + #[allow(clippy::needless_maybe_sized)] + pub const fn size_of<T: Sized + ?core::marker::Sized>() -> usize { + core::mem::size_of::<T>() + } +} + +#[cfg(no_zerocopy_diagnostic_on_unimplemented_1_78_0)] +pub use core::mem::size_of; + +#[cfg(not(no_zerocopy_diagnostic_on_unimplemented_1_78_0))] +pub use __size_of::size_of; + +/// How many padding bytes does the struct type `$t` have? +/// +/// `$ts` is the list of the type of every field in `$t`. `$t` must be a struct +/// type, or else `struct_padding!`'s result may be meaningless. +/// +/// Note that `struct_padding!`'s results are independent of `repcr` since they +/// only consider the size of the type and the sizes of the fields. Whatever the +/// repr, the size of the type already takes into account any padding that the +/// compiler has decided to add. Structs with well-defined representations (such +/// as `repr(C)`) can use this macro to check for padding. Note that while this +/// may yield some consistent value for some `repr(Rust)` structs, it is not +/// guaranteed across platforms or compilations. +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! struct_padding { + ($t:ty, $_align:expr, $_packed:expr, [$($ts:ty),*]) => {{ + // The `align` and `packed` directives can be ignored here. Regardless + // of if and how they are set, comparing the size of `$t` to the sum of + // its field sizes is a reliable indicator of the presence of padding. + $crate::util::macro_util::size_of::<$t>() - (0 $(+ $crate::util::macro_util::size_of::<$ts>())*) + }}; +} + +/// Does the `repr(C)` struct type `$t` have padding? +/// +/// `$ts` is the list of the type of every field in `$t`. `$t` must be a +/// `repr(C)` struct type, or else `struct_has_padding!`'s result may be +/// meaningless. +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! repr_c_struct_has_padding { + ($t:ty, $align:expr, $packed:expr, [$($ts:tt),*]) => {{ + let layout = $crate::DstLayout::for_repr_c_struct( + $align, + $packed, + &[$($crate::repr_c_struct_has_padding!(@field $ts),)*] + ); + layout.requires_static_padding() || layout.requires_dynamic_padding() + }}; + (@field ([$t:ty])) => { + <[$t] as $crate::KnownLayout>::LAYOUT + }; + (@field ($t:ty)) => { + $crate::DstLayout::for_unpadded_type::<$t>() + }; + (@field [$t:ty]) => { + <[$t] as $crate::KnownLayout>::LAYOUT + }; + (@field $t:ty) => { + $crate::DstLayout::for_unpadded_type::<$t>() + }; +} + +/// Does the union type `$t` have padding? +/// +/// `$ts` is the list of the type of every field in `$t`. `$t` must be a union +/// type, or else `union_padding!`'s result may be meaningless. +/// +/// Note that `union_padding!`'s results are independent of `repr` since they +/// only consider the size of the type and the sizes of the fields. Whatever the +/// repr, the size of the type already takes into account any padding that the +/// compiler has decided to add. Unions with well-defined representations (such +/// as `repr(C)`) can use this macro to check for padding. Note that while this +/// may yield some consistent value for some `repr(Rust)` unions, it is not +/// guaranteed across platforms or compilations. +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! union_padding { + ($t:ty, $_align:expr, $_packed:expr, [$($ts:ty),*]) => {{ + // The `align` and `packed` directives can be ignored here. Regardless + // of if and how they are set, comparing the size of `$t` to each of its + // field sizes is a reliable indicator of the presence of padding. + let mut max = 0; + $({ + let padding = $crate::util::macro_util::size_of::<$t>() - $crate::util::macro_util::size_of::<$ts>(); + if padding > max { + max = padding; + } + })* + max + }}; +} + +/// How many padding bytes does the enum type `$t` have? +/// +/// `$disc` is the type of the enum tag, and `$ts` is a list of fields in each +/// square-bracket-delimited variant. `$t` must be an enum, or else +/// `enum_padding!`'s result may be meaningless. An enum has padding if any of +/// its variant structs [1][2] contain padding, and so all of the variants of an +/// enum must be "full" in order for the enum to not have padding. +/// +/// The results of `enum_padding!` require that the enum is not `repr(Rust)`, as +/// `repr(Rust)` enums may niche the enum's tag and reduce the total number of +/// bytes required to represent the enum as a result. As long as the enum is +/// `repr(C)`, `repr(int)`, or `repr(C, int)`, this will consistently return +/// whether the enum contains any padding bytes. +/// +/// [1]: https://doc.rust-lang.org/1.81.0/reference/type-layout.html#reprc-enums-with-fields +/// [2]: https://doc.rust-lang.org/1.81.0/reference/type-layout.html#primitive-representation-of-enums-with-fields +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! enum_padding { + ($t:ty, $_align:expr, $packed:expr, $disc:ty, $([$($ts:ty),*]),*) => {{ + // The `align` and `packed` directives are irrelevant. `$align` can be + // ignored because regardless of if and how it is set, comparing the + // size of `$t` to each of its field sizes is a reliable indicator of + // the presence of padding. `$packed` is irrelevant because it is + // forbidden on enums. + #[allow(clippy::as_conversions)] + const _: [(); 1] = [(); $packed.is_none() as usize]; + let mut max = 0; + $({ + let padding = $crate::util::macro_util::size_of::<$t>() + - ( + $crate::util::macro_util::size_of::<$disc>() + $(+ $crate::util::macro_util::size_of::<$ts>())* + ); + if padding > max { + max = padding; + } + })* + max + }}; +} + +/// Unwraps an infallible `Result`. +#[doc(hidden)] +#[macro_export] +macro_rules! into_inner { + ($e:expr) => { + match $e { + $crate::util::macro_util::core_reexport::result::Result::Ok(e) => e, + $crate::util::macro_util::core_reexport::result::Result::Err(i) => match i {}, + } + }; +} + +/// Translates an identifier or tuple index into a numeric identifier. +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! ident_id { + ($field:ident) => { + $crate::util::macro_util::hash_name(stringify!($field)) + }; + ($field:literal) => { + $field + }; +} + +/// Computes the hash of a string. +/// +/// NOTE(#2749) on hash collisions: This function's output only needs to be +/// deterministic within a particular compilation. Thus, if a user ever reports +/// a hash collision (very unlikely given the <= 16-byte special case), we can +/// strengthen the hash function at that point and publish a new version. Since +/// this is computed at compile time on small strings, we can easily use more +/// expensive and higher-quality hash functions if need be. +#[inline(always)] +#[must_use] +#[allow(clippy::as_conversions, clippy::indexing_slicing, clippy::arithmetic_side_effects)] +pub const fn hash_name(name: &str) -> i128 { + let name = name.as_bytes(); + + // We guarantee freedom from hash collisions between any two strings of + // length 16 or less by having the hashes of such strings be equal to + // their value. There is still a possibility that such strings will have + // the same value as the hash of a string of length > 16. + if name.len() <= size_of::<u128>() { + let mut bytes = [0u8; 16]; + + let mut i = 0; + while i < name.len() { + bytes[i] = name[i]; + i += 1; + } + + return i128::from_ne_bytes(bytes); + }; + + // An implementation of FxHasher, although returning a u128. Probably + // not as strong as it could be, but probably more collision resistant + // than normal 64-bit FxHasher. + let mut hash = 0u128; + let mut i = 0; + while i < name.len() { + // This is just FxHasher's `0x517cc1b727220a95` constant + // concatenated back-to-back. + const K: u128 = 0x517cc1b727220a95517cc1b727220a95; + hash = (hash.rotate_left(5) ^ (name[i] as u128)).wrapping_mul(K); + i += 1; + } + i128::from_ne_bytes(hash.to_ne_bytes()) +} + +/// Attempts to transmute `Src` into `Dst`. +/// +/// A helper for `try_transmute!`. +/// +/// # Panics +/// +/// `try_transmute` may either produce a post-monomorphization error or a panic +/// if `Dst` is bigger than `Src`. Otherwise, `try_transmute` panics under the +/// same circumstances as [`is_bit_valid`]. +/// +/// [`is_bit_valid`]: TryFromBytes::is_bit_valid +#[inline(always)] +pub fn try_transmute<Src, Dst>(src: Src) -> Result<Dst, ValidityError<Src, Dst>> +where + Src: IntoBytes, + Dst: TryFromBytes, +{ + static_assert!(Src, Dst => mem::size_of::<Dst>() == mem::size_of::<Src>()); + + let mu_src = mem::MaybeUninit::new(src); + // SAFETY: `MaybeUninit` has no validity requirements. + let mu_dst: mem::MaybeUninit<ReadOnly<Dst>> = + unsafe { crate::util::transmute_unchecked(mu_src) }; + + let ptr = Ptr::from_ref(&mu_dst); + + // SAFETY: Since `Src: IntoBytes`, and since `size_of::<Src>() == + // size_of::<Dst>()` by the preceding assertion, all of `mu_dst`'s bytes are + // initialized. `MaybeUninit` has no validity requirements, so even if + // `ptr` is used to mutate its referent (which it actually can't be - it's + // a shared `ReadOnly` pointer), that won't violate its referent's validity. + let ptr = unsafe { ptr.assume_validity::<Initialized>() }; + if Dst::is_bit_valid(ptr.cast::<_, CastSized, _>()) { + // SAFETY: Since `Dst::is_bit_valid`, we know that `ptr`'s referent is + // bit-valid for `Dst`. `ptr` points to `mu_dst`, and no intervening + // operations have mutated it, so it is a bit-valid `Dst`. + Ok(ReadOnly::into_inner(unsafe { mu_dst.assume_init() })) + } else { + // SAFETY: `MaybeUninit` has no validity requirements. + let mu_src: mem::MaybeUninit<Src> = unsafe { crate::util::transmute_unchecked(mu_dst) }; + // SAFETY: `mu_dst`/`mu_src` was constructed from `src` and never + // modified, so it is still bit-valid. + Err(ValidityError::new(unsafe { mu_src.assume_init() })) + } +} + +/// See `try_transmute_ref!` documentation. +pub trait TryTransmuteRefDst<'a> { + type Dst: ?Sized; + + /// See `try_transmute_ref!` documentation. + fn try_transmute_ref(self) -> Result<&'a Self::Dst, ValidityError<&'a Self::Src, Self::Dst>> + where + Self: TryTransmuteRefSrc<'a>, + Self::Src: IntoBytes + Immutable + KnownLayout, + Self::Dst: TryFromBytes + Immutable + KnownLayout; +} + +pub trait TryTransmuteRefSrc<'a> { + type Src: ?Sized; +} + +impl<'a, Src, Dst> TryTransmuteRefSrc<'a> for Wrap<&'a Src, &'a Dst> +where + Src: ?Sized, + Dst: ?Sized, +{ + type Src = Src; +} + +impl<'a, Src, Dst> TryTransmuteRefDst<'a> for Wrap<&'a Src, &'a Dst> +where + Src: IntoBytes + Immutable + KnownLayout + ?Sized, + Dst: TryFromBytes + Immutable + KnownLayout + ?Sized, +{ + type Dst = Dst; + + #[inline(always)] + fn try_transmute_ref( + self, + ) -> Result< + &'a Dst, + ValidityError<&'a <Wrap<&'a Src, &'a Dst> as TryTransmuteRefSrc<'a>>::Src, Dst>, + > { + let ptr = Ptr::from_ref(self.0); + #[rustfmt::skip] + let res = ptr.try_with(#[inline(always)] |ptr| { + let ptr = ptr.recall_validity::<Initialized, _>(); + let ptr = ptr.cast::<_, crate::layout::CastFrom<Dst>, _>(); + ptr.try_into_valid() + }); + match res { + Ok(ptr) => { + static_assert!(Src: ?Sized + KnownLayout, Dst: ?Sized + KnownLayout => { + Src::LAYOUT.align.get() >= Dst::LAYOUT.align.get() + }, "cannot transmute reference when destination type has higher alignment than source type"); + // SAFETY: We have checked that `Dst` does not have a stricter + // alignment requirement than `Src`. + let ptr = unsafe { ptr.assume_alignment::<Aligned>() }; + Ok(ptr.as_ref()) + } + Err(err) => Err(err.map_src(Ptr::as_ref)), + } + } +} + +pub trait TryTransmuteMutDst<'a> { + type Dst: ?Sized; + + /// See `try_transmute_mut!` documentation. + fn try_transmute_mut( + self, + ) -> Result<&'a mut Self::Dst, ValidityError<&'a mut Self::Src, Self::Dst>> + where + Self: TryTransmuteMutSrc<'a>, + Self::Src: IntoBytes, + Self::Dst: TryFromBytes; +} + +pub trait TryTransmuteMutSrc<'a> { + type Src: ?Sized; +} + +impl<'a, Src, Dst> TryTransmuteMutSrc<'a> for Wrap<&'a mut Src, &'a mut Dst> +where + Src: ?Sized, + Dst: ?Sized, +{ + type Src = Src; +} + +impl<'a, Src, Dst> TryTransmuteMutDst<'a> for Wrap<&'a mut Src, &'a mut Dst> +where + Src: FromBytes + IntoBytes + KnownLayout + ?Sized, + Dst: TryFromBytes + IntoBytes + KnownLayout + ?Sized, +{ + type Dst = Dst; + + #[inline(always)] + fn try_transmute_mut( + self, + ) -> Result< + &'a mut Dst, + ValidityError<&'a mut <Wrap<&'a mut Src, &'a mut Dst> as TryTransmuteMutSrc<'a>>::Src, Dst>, + > { + let ptr = Ptr::from_mut(self.0); + // SAFETY: The provided closure returns the only copy of `ptr`. + #[rustfmt::skip] + let res = unsafe { + ptr.try_with_unchecked(#[inline(always)] |ptr| { + let ptr = ptr.recall_validity::<Initialized, (_, (_, _))>(); + let ptr = ptr.cast::<_, crate::layout::CastFrom<Dst>, _>(); + ptr.try_into_valid() + }) + }; + match res { + Ok(ptr) => { + static_assert!(Src: ?Sized + KnownLayout, Dst: ?Sized + KnownLayout => { + Src::LAYOUT.align.get() >= Dst::LAYOUT.align.get() + }, "cannot transmute reference when destination type has higher alignment than source type"); + // SAFETY: We have checked that `Dst` does not have a stricter + // alignment requirement than `Src`. + let ptr = unsafe { ptr.assume_alignment::<Aligned>() }; + Ok(ptr.as_mut()) + } + Err(err) => Err(err.map_src(Ptr::as_mut)), + } + } +} + +// Used in `transmute_ref!` and friends. +// +// This permits us to use the autoref specialization trick to dispatch to +// associated functions for `transmute_ref` and `transmute_mut` when both `Src` +// and `Dst` are `Sized`, and to trait methods otherwise. The associated +// functions, unlike the trait methods, do not require a `KnownLayout` bound. +// This permits us to add support for transmuting references to unsized types +// without breaking backwards-compatibility (on v0.8.x) with the old +// implementation, which did not require a `KnownLayout` bound to transmute +// sized types. +#[derive(Copy, Clone)] +pub struct Wrap<Src, Dst>(pub Src, pub PhantomData<Dst>); + +impl<Src, Dst> Wrap<Src, Dst> { + #[inline(always)] + pub const fn new(src: Src) -> Self { + Wrap(src, PhantomData) + } +} + +impl<'a, Src, Dst> Wrap<&'a Src, &'a Dst> +where + Src: ?Sized, + Dst: ?Sized, +{ + #[allow(clippy::must_use_candidate, clippy::missing_inline_in_public_items, clippy::empty_loop)] + pub const fn transmute_ref_inference_helper(self) -> &'a Dst { + loop {} + } +} + +impl<'a, Src, Dst> Wrap<&'a Src, &'a Dst> { + /// # Safety + /// The caller must guarantee that: + /// - `Src: IntoBytes + Immutable` + /// - `Dst: FromBytes + Immutable` + /// + /// # PME + /// + /// Instantiating this method PMEs unless both: + /// - `mem::size_of::<Dst>() == mem::size_of::<Src>()` + /// - `mem::align_of::<Dst>() <= mem::align_of::<Src>()` + #[inline(always)] + #[must_use] + pub const unsafe fn transmute_ref(self) -> &'a Dst { + static_assert!(Src, Dst => mem::size_of::<Dst>() == mem::size_of::<Src>()); + static_assert!(Src, Dst => mem::align_of::<Dst>() <= mem::align_of::<Src>()); + + let src: *const Src = self.0; + let dst = src.cast::<Dst>(); + // SAFETY: + // - We know that it is sound to view the target type of the input + // reference (`Src`) as the target type of the output reference + // (`Dst`) because the caller has guaranteed that `Src: IntoBytes`, + // `Dst: FromBytes`, and `size_of::<Src>() == size_of::<Dst>()`. + // - We know that there are no `UnsafeCell`s, and thus we don't have to + // worry about `UnsafeCell` overlap, because `Src: Immutable` and + // `Dst: Immutable`. + // - The caller has guaranteed that alignment is not increased. + // - We know that the returned lifetime will not outlive the input + // lifetime thanks to the lifetime bounds on this function. + // + // FIXME(#67): Once our MSRV is 1.58, replace this `transmute` with + // `&*dst`. + #[allow(clippy::transmute_ptr_to_ref)] + unsafe { + mem::transmute(dst) + } + } + + #[inline(always)] + pub fn try_transmute_ref(self) -> Result<&'a Dst, ValidityError<&'a Src, Dst>> + where + Src: IntoBytes + Immutable, + Dst: TryFromBytes + Immutable, + { + static_assert!(Src => mem::align_of::<Src>() == mem::align_of::<Wrapping<Src>>()); + static_assert!(Dst => mem::align_of::<Dst>() == mem::align_of::<Wrapping<Dst>>()); + + // SAFETY: By the preceding assert, `Src` and `Wrapping<Src>` have the + // same alignment. + let src: &Wrapping<Src> = + unsafe { crate::util::transmute_ref::<_, _, BecauseImmutable>(self.0) }; + let src = Wrap::new(src); + <Wrap<&'a Wrapping<Src>, &'a Wrapping<Dst>> as TryTransmuteRefDst<'a>>::try_transmute_ref( + src, + ) + .map( + // SAFETY: By the preceding assert, `Dst` and `Wrapping<Dst>` have + // the same alignment. + #[inline(always)] + |dst| unsafe { crate::util::transmute_ref::<_, _, BecauseImmutable>(dst) }, + ) + .map_err( + #[inline(always)] + |err| { + // SAFETY: By the preceding assert, `Src` and `Wrapping<Src>` have the + // same alignment. + ValidityError::new(unsafe { + crate::util::transmute_ref::<_, _, BecauseImmutable>(err.into_src()) + }) + }, + ) + } +} + +impl<'a, Src, Dst> Wrap<&'a mut Src, &'a mut Dst> +where + Src: ?Sized, + Dst: ?Sized, +{ + #[allow(clippy::must_use_candidate, clippy::missing_inline_in_public_items, clippy::empty_loop)] + pub fn transmute_mut_inference_helper(self) -> &'a mut Dst { + loop {} + } +} + +impl<'a, Src, Dst> Wrap<&'a mut Src, &'a mut Dst> { + /// Transmutes a mutable reference of one type to a mutable reference of + /// another type. + /// + /// # PME + /// + /// Instantiating this method PMEs unless both: + /// - `mem::size_of::<Dst>() == mem::size_of::<Src>()` + /// - `mem::align_of::<Dst>() <= mem::align_of::<Src>()` + #[inline(always)] + #[must_use] + pub fn transmute_mut(self) -> &'a mut Dst + where + Src: FromBytes + IntoBytes, + Dst: FromBytes + IntoBytes, + { + static_assert!(Src, Dst => mem::size_of::<Dst>() == mem::size_of::<Src>()); + static_assert!(Src, Dst => mem::align_of::<Dst>() <= mem::align_of::<Src>()); + + let src: *mut Src = self.0; + let dst = src.cast::<Dst>(); + // SAFETY: + // - We know that it is sound to view the target type of the input + // reference (`Src`) as the target type of the output reference + // (`Dst`) and vice-versa because `Src: FromBytes + IntoBytes`, `Dst: + // FromBytes + IntoBytes`, and (as asserted above) `size_of::<Src>() + // == size_of::<Dst>()`. + // - We asserted above that alignment will not increase. + // - We know that the returned lifetime will not outlive the input + // lifetime thanks to the lifetime bounds on this function. + unsafe { &mut *dst } + } + + #[inline(always)] + pub fn try_transmute_mut(self) -> Result<&'a mut Dst, ValidityError<&'a mut Src, Dst>> + where + Src: FromBytes + IntoBytes, + Dst: TryFromBytes + IntoBytes, + { + static_assert!(Src => mem::align_of::<Src>() == mem::align_of::<Wrapping<Src>>()); + static_assert!(Dst => mem::align_of::<Dst>() == mem::align_of::<Wrapping<Dst>>()); + + // SAFETY: By the preceding assert, `Src` and `Wrapping<Src>` have the + // same alignment. + let src: &mut Wrapping<Src> = + unsafe { crate::util::transmute_mut::<_, _, (_, (_, _))>(self.0) }; + let src = Wrap::new(src); + <Wrap<&'a mut Wrapping<Src>, &'a mut Wrapping<Dst>> as TryTransmuteMutDst<'a>> + ::try_transmute_mut(src) + // SAFETY: By the preceding assert, `Dst` and `Wrapping<Dst>` have the + // same alignment. + .map(|dst| unsafe { crate::util::transmute_mut::<_, _, (_, (_, _))>(dst) }) + .map_err(|err| { + // SAFETY: By the preceding assert, `Src` and `Wrapping<Src>` have the + // same alignment. + ValidityError::new(unsafe { + crate::util::transmute_mut::<_, _, (_, (_, _))>(err.into_src()) + }) + }) + } +} + +pub trait TransmuteRefDst<'a> { + type Dst: ?Sized; + + #[must_use] + fn transmute_ref(self) -> &'a Self::Dst; +} + +impl<'a, Src: ?Sized, Dst: ?Sized> TransmuteRefDst<'a> for Wrap<&'a Src, &'a Dst> +where + Src: KnownLayout + IntoBytes + Immutable, + Dst: KnownLayout<PointerMetadata = usize> + FromBytes + Immutable, +{ + type Dst = Dst; + + #[inline(always)] + fn transmute_ref(self) -> &'a Dst { + let ptr = Ptr::from_ref(self.0) + .recall_validity::<Initialized, _>() + .transmute_with::<Dst, Initialized, crate::layout::CastFrom<Dst>, (crate::pointer::BecauseMutationCompatible, _)>() + .recall_validity::<Valid, _>(); + + static_assert!(Src: ?Sized + KnownLayout, Dst: ?Sized + KnownLayout => { + Src::LAYOUT.align.get() >= Dst::LAYOUT.align.get() + }, "cannot transmute reference when destination type has higher alignment than source type"); + + // SAFETY: The preceding `static_assert!` ensures that + // `Src::LAYOUT.align >= Dst::LAYOUT.align`. Since `self` is + // validly-aligned for `Src`, it is also validly-aligned for `Dst`. + let ptr = unsafe { ptr.assume_alignment() }; + + ptr.as_ref() + } +} + +pub trait TransmuteMutDst<'a> { + type Dst: ?Sized; + #[must_use] + fn transmute_mut(self) -> &'a mut Self::Dst; +} + +impl<'a, Src: ?Sized, Dst: ?Sized> TransmuteMutDst<'a> for Wrap<&'a mut Src, &'a mut Dst> +where + Src: KnownLayout + FromBytes + IntoBytes, + Dst: KnownLayout<PointerMetadata = usize> + FromBytes + IntoBytes, +{ + type Dst = Dst; + + #[inline(always)] + fn transmute_mut(self) -> &'a mut Dst { + let ptr = Ptr::from_mut(self.0) + .recall_validity::<Initialized, (_, (_, _))>() + .transmute_with::<Dst, Initialized, crate::layout::CastFrom<Dst>, _>() + .recall_validity::<Valid, (_, (_, _))>(); + + static_assert!(Src: ?Sized + KnownLayout, Dst: ?Sized + KnownLayout => { + Src::LAYOUT.align.get() >= Dst::LAYOUT.align.get() + }, "cannot transmute reference when destination type has higher alignment than source type"); + + // SAFETY: The preceding `static_assert!` ensures that + // `Src::LAYOUT.align >= Dst::LAYOUT.align`. Since `self` is + // validly-aligned for `Src`, it is also validly-aligned for `Dst`. + let ptr = unsafe { ptr.assume_alignment() }; + + ptr.as_mut() + } +} + +/// A function which emits a warning if its return value is not used. +#[must_use] +#[inline(always)] +pub const fn must_use<T>(t: T) -> T { + t +} + +// NOTE: We can't change this to a `pub use core as core_reexport` until [1] is +// fixed or we update to a semver-breaking version (as of this writing, 0.8.0) +// on the `main` branch. +// +// [1] https://github.com/obi1kenobi/cargo-semver-checks/issues/573 +pub mod core_reexport { + pub use core::*; + + pub mod mem { + pub use core::mem::*; + } +} + +#[cfg(test)] +mod tests { + use core::num::NonZeroUsize; + + use crate::util::testutil::*; + + #[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] + mod nightly { + use super::super::*; + use crate::util::testutil::*; + + // FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): + // Remove this `cfg` when `size_of_val_raw` is stabilized. + #[allow(clippy::decimal_literal_representation)] + #[test] + fn test_trailing_field_offset() { + assert_eq!(mem::align_of::<Aligned64kAllocation>(), _64K); + + macro_rules! test { + (#[$cfg:meta] ($($ts:ty),* ; $trailing_field_ty:ty) => $expect:expr) => {{ + #[$cfg] + struct Test($(#[allow(dead_code)] $ts,)* #[allow(dead_code)] $trailing_field_ty); + assert_eq!(test!(@offset $($ts),* ; $trailing_field_ty), $expect); + }}; + (#[$cfg:meta] $(#[$cfgs:meta])* ($($ts:ty),* ; $trailing_field_ty:ty) => $expect:expr) => { + test!(#[$cfg] ($($ts),* ; $trailing_field_ty) => $expect); + test!($(#[$cfgs])* ($($ts),* ; $trailing_field_ty) => $expect); + }; + (@offset ; $_trailing:ty) => { trailing_field_offset!(Test, 0) }; + (@offset $_t:ty ; $_trailing:ty) => { trailing_field_offset!(Test, 1) }; + } + + test!(#[repr(C)] #[repr(transparent)] #[repr(packed)](; u8) => Some(0)); + test!(#[repr(C)] #[repr(transparent)] #[repr(packed)](; [u8]) => Some(0)); + test!(#[repr(C)] #[repr(C, packed)] (u8; u8) => Some(1)); + test!(#[repr(C)] (; AU64) => Some(0)); + test!(#[repr(C)] (; [AU64]) => Some(0)); + test!(#[repr(C)] (u8; AU64) => Some(8)); + test!(#[repr(C)] (u8; [AU64]) => Some(8)); + + #[derive( + Immutable, FromBytes, Eq, PartialEq, Ord, PartialOrd, Default, Debug, Copy, Clone, + )] + #[repr(C)] + pub(crate) struct Nested<T, U: ?Sized> { + _t: T, + _u: U, + } + + test!(#[repr(C)] (; Nested<u8, AU64>) => Some(0)); + test!(#[repr(C)] (; Nested<u8, [AU64]>) => Some(0)); + test!(#[repr(C)] (u8; Nested<u8, AU64>) => Some(8)); + test!(#[repr(C)] (u8; Nested<u8, [AU64]>) => Some(8)); + + // Test that `packed(N)` limits the offset of the trailing field. + test!(#[repr(C, packed( 1))] (u8; elain::Align< 2>) => Some( 1)); + test!(#[repr(C, packed( 2))] (u8; elain::Align< 4>) => Some( 2)); + test!(#[repr(C, packed( 4))] (u8; elain::Align< 8>) => Some( 4)); + test!(#[repr(C, packed( 8))] (u8; elain::Align< 16>) => Some( 8)); + test!(#[repr(C, packed( 16))] (u8; elain::Align< 32>) => Some( 16)); + test!(#[repr(C, packed( 32))] (u8; elain::Align< 64>) => Some( 32)); + test!(#[repr(C, packed( 64))] (u8; elain::Align< 128>) => Some( 64)); + test!(#[repr(C, packed( 128))] (u8; elain::Align< 256>) => Some( 128)); + test!(#[repr(C, packed( 256))] (u8; elain::Align< 512>) => Some( 256)); + test!(#[repr(C, packed( 512))] (u8; elain::Align< 1024>) => Some( 512)); + test!(#[repr(C, packed( 1024))] (u8; elain::Align< 2048>) => Some( 1024)); + test!(#[repr(C, packed( 2048))] (u8; elain::Align< 4096>) => Some( 2048)); + test!(#[repr(C, packed( 4096))] (u8; elain::Align< 8192>) => Some( 4096)); + test!(#[repr(C, packed( 8192))] (u8; elain::Align< 16384>) => Some( 8192)); + test!(#[repr(C, packed( 16384))] (u8; elain::Align< 32768>) => Some( 16384)); + test!(#[repr(C, packed( 32768))] (u8; elain::Align< 65536>) => Some( 32768)); + test!(#[repr(C, packed( 65536))] (u8; elain::Align< 131072>) => Some( 65536)); + /* Alignments above 65536 are not yet supported. + test!(#[repr(C, packed( 131072))] (u8; elain::Align< 262144>) => Some( 131072)); + test!(#[repr(C, packed( 262144))] (u8; elain::Align< 524288>) => Some( 262144)); + test!(#[repr(C, packed( 524288))] (u8; elain::Align< 1048576>) => Some( 524288)); + test!(#[repr(C, packed( 1048576))] (u8; elain::Align< 2097152>) => Some( 1048576)); + test!(#[repr(C, packed( 2097152))] (u8; elain::Align< 4194304>) => Some( 2097152)); + test!(#[repr(C, packed( 4194304))] (u8; elain::Align< 8388608>) => Some( 4194304)); + test!(#[repr(C, packed( 8388608))] (u8; elain::Align< 16777216>) => Some( 8388608)); + test!(#[repr(C, packed( 16777216))] (u8; elain::Align< 33554432>) => Some( 16777216)); + test!(#[repr(C, packed( 33554432))] (u8; elain::Align< 67108864>) => Some( 33554432)); + test!(#[repr(C, packed( 67108864))] (u8; elain::Align< 33554432>) => Some( 67108864)); + test!(#[repr(C, packed( 33554432))] (u8; elain::Align<134217728>) => Some( 33554432)); + test!(#[repr(C, packed(134217728))] (u8; elain::Align<268435456>) => Some(134217728)); + test!(#[repr(C, packed(268435456))] (u8; elain::Align<268435456>) => Some(268435456)); + */ + + // Test that `align(N)` does not limit the offset of the trailing field. + test!(#[repr(C, align( 1))] (u8; elain::Align< 2>) => Some( 2)); + test!(#[repr(C, align( 2))] (u8; elain::Align< 4>) => Some( 4)); + test!(#[repr(C, align( 4))] (u8; elain::Align< 8>) => Some( 8)); + test!(#[repr(C, align( 8))] (u8; elain::Align< 16>) => Some( 16)); + test!(#[repr(C, align( 16))] (u8; elain::Align< 32>) => Some( 32)); + test!(#[repr(C, align( 32))] (u8; elain::Align< 64>) => Some( 64)); + test!(#[repr(C, align( 64))] (u8; elain::Align< 128>) => Some( 128)); + test!(#[repr(C, align( 128))] (u8; elain::Align< 256>) => Some( 256)); + test!(#[repr(C, align( 256))] (u8; elain::Align< 512>) => Some( 512)); + test!(#[repr(C, align( 512))] (u8; elain::Align< 1024>) => Some( 1024)); + test!(#[repr(C, align( 1024))] (u8; elain::Align< 2048>) => Some( 2048)); + test!(#[repr(C, align( 2048))] (u8; elain::Align< 4096>) => Some( 4096)); + test!(#[repr(C, align( 4096))] (u8; elain::Align< 8192>) => Some( 8192)); + test!(#[repr(C, align( 8192))] (u8; elain::Align< 16384>) => Some( 16384)); + test!(#[repr(C, align( 16384))] (u8; elain::Align< 32768>) => Some( 32768)); + test!(#[repr(C, align( 32768))] (u8; elain::Align< 65536>) => Some( 65536)); + /* Alignments above 65536 are not yet supported. + test!(#[repr(C, align( 65536))] (u8; elain::Align< 131072>) => Some( 131072)); + test!(#[repr(C, align( 131072))] (u8; elain::Align< 262144>) => Some( 262144)); + test!(#[repr(C, align( 262144))] (u8; elain::Align< 524288>) => Some( 524288)); + test!(#[repr(C, align( 524288))] (u8; elain::Align< 1048576>) => Some( 1048576)); + test!(#[repr(C, align( 1048576))] (u8; elain::Align< 2097152>) => Some( 2097152)); + test!(#[repr(C, align( 2097152))] (u8; elain::Align< 4194304>) => Some( 4194304)); + test!(#[repr(C, align( 4194304))] (u8; elain::Align< 8388608>) => Some( 8388608)); + test!(#[repr(C, align( 8388608))] (u8; elain::Align< 16777216>) => Some( 16777216)); + test!(#[repr(C, align( 16777216))] (u8; elain::Align< 33554432>) => Some( 33554432)); + test!(#[repr(C, align( 33554432))] (u8; elain::Align< 67108864>) => Some( 67108864)); + test!(#[repr(C, align( 67108864))] (u8; elain::Align< 33554432>) => Some( 33554432)); + test!(#[repr(C, align( 33554432))] (u8; elain::Align<134217728>) => Some(134217728)); + test!(#[repr(C, align(134217728))] (u8; elain::Align<268435456>) => Some(268435456)); + */ + } + + // FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): + // Remove this `cfg` when `size_of_val_raw` is stabilized. + #[allow(clippy::decimal_literal_representation)] + #[test] + fn test_align_of_dst() { + // Test that `align_of!` correctly computes the alignment of DSTs. + assert_eq!(align_of!([elain::Align<1>]), Some(1)); + assert_eq!(align_of!([elain::Align<2>]), Some(2)); + assert_eq!(align_of!([elain::Align<4>]), Some(4)); + assert_eq!(align_of!([elain::Align<8>]), Some(8)); + assert_eq!(align_of!([elain::Align<16>]), Some(16)); + assert_eq!(align_of!([elain::Align<32>]), Some(32)); + assert_eq!(align_of!([elain::Align<64>]), Some(64)); + assert_eq!(align_of!([elain::Align<128>]), Some(128)); + assert_eq!(align_of!([elain::Align<256>]), Some(256)); + assert_eq!(align_of!([elain::Align<512>]), Some(512)); + assert_eq!(align_of!([elain::Align<1024>]), Some(1024)); + assert_eq!(align_of!([elain::Align<2048>]), Some(2048)); + assert_eq!(align_of!([elain::Align<4096>]), Some(4096)); + assert_eq!(align_of!([elain::Align<8192>]), Some(8192)); + assert_eq!(align_of!([elain::Align<16384>]), Some(16384)); + assert_eq!(align_of!([elain::Align<32768>]), Some(32768)); + assert_eq!(align_of!([elain::Align<65536>]), Some(65536)); + /* Alignments above 65536 are not yet supported. + assert_eq!(align_of!([elain::Align<131072>]), Some(131072)); + assert_eq!(align_of!([elain::Align<262144>]), Some(262144)); + assert_eq!(align_of!([elain::Align<524288>]), Some(524288)); + assert_eq!(align_of!([elain::Align<1048576>]), Some(1048576)); + assert_eq!(align_of!([elain::Align<2097152>]), Some(2097152)); + assert_eq!(align_of!([elain::Align<4194304>]), Some(4194304)); + assert_eq!(align_of!([elain::Align<8388608>]), Some(8388608)); + assert_eq!(align_of!([elain::Align<16777216>]), Some(16777216)); + assert_eq!(align_of!([elain::Align<33554432>]), Some(33554432)); + assert_eq!(align_of!([elain::Align<67108864>]), Some(67108864)); + assert_eq!(align_of!([elain::Align<33554432>]), Some(33554432)); + assert_eq!(align_of!([elain::Align<134217728>]), Some(134217728)); + assert_eq!(align_of!([elain::Align<268435456>]), Some(268435456)); + */ + } + } + + #[test] + fn test_enum_casts() { + // Test that casting the variants of enums with signed integer reprs to + // unsigned integers obeys expected signed -> unsigned casting rules. + + #[repr(i8)] + enum ReprI8 { + MinusOne = -1, + Zero = 0, + Min = i8::MIN, + Max = i8::MAX, + } + + #[allow(clippy::as_conversions)] + let x = ReprI8::MinusOne as u8; + assert_eq!(x, u8::MAX); + + #[allow(clippy::as_conversions)] + let x = ReprI8::Zero as u8; + assert_eq!(x, 0); + + #[allow(clippy::as_conversions)] + let x = ReprI8::Min as u8; + assert_eq!(x, 128); + + #[allow(clippy::as_conversions)] + let x = ReprI8::Max as u8; + assert_eq!(x, 127); + } + + #[test] + fn test_struct_padding() { + // Test that, for each provided repr, `struct_padding!` reports the + // expected value. + macro_rules! test { + (#[$cfg:meta] ($($ts:ty),*) => $expect:expr) => {{ + #[$cfg] + #[allow(dead_code)] + struct Test($($ts),*); + assert_eq!(struct_padding!(Test, None::<NonZeroUsize>, None::<NonZeroUsize>, [$($ts),*]), $expect); + }}; + (#[$cfg:meta] $(#[$cfgs:meta])* ($($ts:ty),*) => $expect:expr) => { + test!(#[$cfg] ($($ts),*) => $expect); + test!($(#[$cfgs])* ($($ts),*) => $expect); + }; + } + + test!(#[repr(C)] #[repr(transparent)] #[repr(packed)] () => 0); + test!(#[repr(C)] #[repr(transparent)] #[repr(packed)] (u8) => 0); + test!(#[repr(C)] #[repr(transparent)] #[repr(packed)] (u8, ()) => 0); + test!(#[repr(C)] #[repr(packed)] (u8, u8) => 0); + + test!(#[repr(C)] (u8, AU64) => 7); + // Rust won't let you put `#[repr(packed)]` on a type which contains a + // `#[repr(align(n > 1))]` type (`AU64`), so we have to use `u64` here. + // It's not ideal, but it definitely has align > 1 on /some/ of our CI + // targets, and this isn't a particularly complex macro we're testing + // anyway. + test!(#[repr(packed)] (u8, u64) => 0); + } + + #[test] + fn test_repr_c_struct_padding() { + // Test that, for each provided repr, `repr_c_struct_padding!` reports + // the expected value. + macro_rules! test { + (($($ts:tt),*) => $expect:expr) => {{ + #[repr(C)] + #[allow(dead_code)] + struct Test($($ts),*); + assert_eq!(repr_c_struct_has_padding!(Test, None::<NonZeroUsize>, None::<NonZeroUsize>, [$($ts),*]), $expect); + }}; + } + + // Test static padding + test!(() => false); + test!(([u8]) => false); + test!((u8) => false); + test!((u8, [u8]) => false); + test!((u8, ()) => false); + test!((u8, (), [u8]) => false); + test!((u8, u8) => false); + test!((u8, u8, [u8]) => false); + + test!((u8, AU64) => true); + test!((u8, AU64, [u8]) => true); + + // Test dynamic padding + test!((AU64, [AU64]) => false); + test!((u8, [AU64]) => true); + + #[repr(align(4))] + struct AU32(#[allow(unused)] u32); + test!((AU64, [AU64]) => false); + test!((AU64, [AU32]) => true); + } + + #[test] + fn test_union_padding() { + // Test that, for each provided repr, `union_padding!` reports the + // expected value. + macro_rules! test { + (#[$cfg:meta] {$($fs:ident: $ts:ty),*} => $expect:expr) => {{ + #[$cfg] + #[allow(unused)] // fields are never read + union Test{ $($fs: $ts),* } + assert_eq!(union_padding!(Test, None::<NonZeroUsize>, None::<usize>, [$($ts),*]), $expect); + }}; + (#[$cfg:meta] $(#[$cfgs:meta])* {$($fs:ident: $ts:ty),*} => $expect:expr) => { + test!(#[$cfg] {$($fs: $ts),*} => $expect); + test!($(#[$cfgs])* {$($fs: $ts),*} => $expect); + }; + } + + test!(#[repr(C)] #[repr(packed)] {a: u8} => 0); + test!(#[repr(C)] #[repr(packed)] {a: u8, b: u8} => 0); + + // Rust won't let you put `#[repr(packed)]` on a type which contains a + // `#[repr(align(n > 1))]` type (`AU64`), so we have to use `u64` here. + // It's not ideal, but it definitely has align > 1 on /some/ of our CI + // targets, and this isn't a particularly complex macro we're testing + // anyway. + test!(#[repr(C)] #[repr(packed)] {a: u8, b: u64} => 7); + } + + #[test] + fn test_enum_padding() { + // Test that, for each provided repr, `enum_has_padding!` reports the + // expected value. + macro_rules! test { + (#[repr($disc:ident $(, $c:ident)?)] { $($vs:ident ($($ts:ty),*),)* } => $expect:expr) => { + test!(@case #[repr($disc $(, $c)?)] { $($vs ($($ts),*),)* } => $expect); + }; + (#[repr($disc:ident $(, $c:ident)?)] #[$cfg:meta] $(#[$cfgs:meta])* { $($vs:ident ($($ts:ty),*),)* } => $expect:expr) => { + test!(@case #[repr($disc $(, $c)?)] #[$cfg] { $($vs ($($ts),*),)* } => $expect); + test!(#[repr($disc $(, $c)?)] $(#[$cfgs])* { $($vs ($($ts),*),)* } => $expect); + }; + (@case #[repr($disc:ident $(, $c:ident)?)] $(#[$cfg:meta])? { $($vs:ident ($($ts:ty),*),)* } => $expect:expr) => {{ + #[repr($disc $(, $c)?)] + $(#[$cfg])? + #[allow(unused)] // variants and fields are never used + enum Test { + $($vs ($($ts),*),)* + } + assert_eq!( + enum_padding!(Test, None::<NonZeroUsize>, None::<NonZeroUsize>, $disc, $([$($ts),*]),*), + $expect + ); + }}; + } + + #[allow(unused)] + #[repr(align(2))] + struct U16(u16); + + #[allow(unused)] + #[repr(align(4))] + struct U32(u32); + + test!(#[repr(u8)] #[repr(C)] { + A(u8), + } => 0); + test!(#[repr(u16)] #[repr(C)] { + A(u8, u8), + B(U16), + } => 0); + test!(#[repr(u32)] #[repr(C)] { + A(u8, u8, u8, u8), + B(U16, u8, u8), + C(u8, u8, U16), + D(U16, U16), + E(U32), + } => 0); + + // `repr(int)` can pack the discriminant more efficiently + test!(#[repr(u8)] { + A(u8, U16), + } => 0); + test!(#[repr(u8)] { + A(u8, U16, U32), + } => 0); + + // `repr(C)` cannot + test!(#[repr(u8, C)] { + A(u8, U16), + } => 2); + test!(#[repr(u8, C)] { + A(u8, u8, u8, U32), + } => 4); + + // And field ordering can always cause problems + test!(#[repr(u8)] #[repr(C)] { + A(U16, u8), + } => 2); + test!(#[repr(u8)] #[repr(C)] { + A(U32, u8, u8, u8), + } => 4); + } +} diff --git a/rust/zerocopy/src/util/macros.rs b/rust/zerocopy/src/util/macros.rs new file mode 100644 index 000000000000..43e4fd64ee15 --- /dev/null +++ b/rust/zerocopy/src/util/macros.rs @@ -0,0 +1,1067 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2023 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +/// Unsafely implements trait(s) for a type. +/// +/// # Safety +/// +/// The trait impl must be sound. +/// +/// When implementing `TryFromBytes`: +/// - If no `is_bit_valid` impl is provided, then it must be valid for +/// `is_bit_valid` to unconditionally return `true`. In other words, it must +/// be the case that any initialized sequence of bytes constitutes a valid +/// instance of `$ty`. +/// - If an `is_bit_valid` impl is provided, then the impl of `is_bit_valid` +/// must only return `true` if its argument refers to a valid `$ty`. +macro_rules! unsafe_impl { + // Implement `$trait` for `$ty` with no bounds. + ($(#[$attr:meta])* $ty:ty: $trait:ident $(; |$candidate:ident| $is_bit_valid:expr)?) => {{ + crate::util::macros::__unsafe(); + + $(#[$attr])* + // SAFETY: The caller promises that this is sound. + unsafe impl $trait for $ty { + unsafe_impl!(@method $trait $(; |$candidate| $is_bit_valid)?); + } + }}; + + // Implement all `$traits` for `$ty` with no bounds. + // + // The 2 arms under this one are there so we can apply + // N attributes for each one of M trait implementations. + // The simple solution of: + // + // ($(#[$attrs:meta])* $ty:ty: $($traits:ident),*) => { + // $( unsafe_impl!( $(#[$attrs])* $ty: $traits ) );* + // } + // + // Won't work. The macro processor sees that the outer repetition + // contains both $attrs and $traits and expects them to match the same + // amount of fragments. + // + // To solve this we must: + // 1. Pack the attributes into a single token tree fragment we can match over. + // 2. Expand the traits. + // 3. Unpack and expand the attributes. + ($(#[$attrs:meta])* $ty:ty: $($traits:ident),*) => { + unsafe_impl!(@impl_traits_with_packed_attrs { $(#[$attrs])* } $ty: $($traits),*) + }; + + (@impl_traits_with_packed_attrs $attrs:tt $ty:ty: $($traits:ident),*) => {{ + $( unsafe_impl!(@unpack_attrs $attrs $ty: $traits); )* + }}; + + (@unpack_attrs { $(#[$attrs:meta])* } $ty:ty: $traits:ident) => { + unsafe_impl!($(#[$attrs])* $ty: $traits); + }; + + // This arm is identical to the following one, except it contains a + // preceding `const`. If we attempt to handle these with a single arm, there + // is an inherent ambiguity between `const` (the keyword) and `const` (the + // ident match for `$tyvar:ident`). + // + // To explain how this works, consider the following invocation: + // + // unsafe_impl!(const N: usize, T: ?Sized + Copy => Clone for Foo<T>); + // + // In this invocation, here are the assignments to meta-variables: + // + // |---------------|------------| + // | Meta-variable | Assignment | + // |---------------|------------| + // | $constname | N | + // | $constty | usize | + // | $tyvar | T | + // | $optbound | Sized | + // | $bound | Copy | + // | $trait | Clone | + // | $ty | Foo<T> | + // |---------------|------------| + // + // The following arm has the same behavior with the exception of the lack of + // support for a leading `const` parameter. + ( + $(#[$attr:meta])* + const $constname:ident : $constty:ident $(,)? + $($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?),* + => $trait:ident for $ty:ty $(; |$candidate:ident| $is_bit_valid:expr)? + ) => { + unsafe_impl!( + @inner + $(#[$attr])* + @const $constname: $constty, + $($tyvar $(: $(? $optbound +)* + $($bound +)*)?,)* + => $trait for $ty $(; |$candidate| $is_bit_valid)? + ); + }; + ( + $(#[$attr:meta])* + $($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?),* + => $trait:ident for $ty:ty $(; |$candidate:ident| $is_bit_valid:expr)? + ) => {{ + unsafe_impl!( + @inner + $(#[$attr])* + $($tyvar $(: $(? $optbound +)* + $($bound +)*)?,)* + => $trait for $ty $(; |$candidate| $is_bit_valid)? + ); + }}; + ( + @inner + $(#[$attr:meta])* + $(@const $constname:ident : $constty:ident,)* + $($tyvar:ident $(: $(? $optbound:ident +)* + $($bound:ident +)* )?,)* + => $trait:ident for $ty:ty $(; |$candidate:ident| $is_bit_valid:expr)? + ) => {{ + crate::util::macros::__unsafe(); + + $(#[$attr])* + #[allow(non_local_definitions)] + // SAFETY: The caller promises that this is sound. + unsafe impl<$($tyvar $(: $(? $optbound +)* $($bound +)*)?),* $(, const $constname: $constty,)*> $trait for $ty { + unsafe_impl!(@method $trait $(; |$candidate| $is_bit_valid)?); + } + }}; + + (@method TryFromBytes ; |$candidate:ident| $is_bit_valid:expr) => { + #[allow(clippy::missing_inline_in_public_items, dead_code)] + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn only_derive_is_allowed_to_implement_this_trait() {} + + #[inline] + fn is_bit_valid<Alignment>($candidate: Maybe<'_, Self, Alignment>) -> bool + where + Alignment: crate::invariant::Alignment, + { + $is_bit_valid + } + }; + (@method TryFromBytes) => { + #[allow(clippy::missing_inline_in_public_items)] + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn only_derive_is_allowed_to_implement_this_trait() {} + #[inline(always)] + fn is_bit_valid<Alignment>(_candidate: Maybe<'_, Self, Alignment>) -> bool + where + Alignment: crate::invariant::Alignment, + { + true + } + }; + (@method $trait:ident) => { + #[allow(clippy::missing_inline_in_public_items, dead_code)] + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn only_derive_is_allowed_to_implement_this_trait() {} + }; + (@method $trait:ident; |$_candidate:ident| $_is_bit_valid:expr) => { + compile_error!("Can't provide `is_bit_valid` impl for trait other than `TryFromBytes`"); + }; +} + +/// Implements `$trait` for `$ty` where `$ty: TransmuteFrom<$repr>` (and +/// vice-versa). +/// +/// Calling this macro is safe; the internals of the macro emit appropriate +/// trait bounds which ensure that the given impl is sound. +macro_rules! impl_for_transmute_from { + ( + $(#[$attr:meta])* + $($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?)? + => $trait:ident for $ty:ty [$repr:ty] + ) => { + const _: () = { + $(#[$attr])* + #[allow(non_local_definitions)] + + // SAFETY: `is_trait<T, R>` (defined and used below) requires `T: + // TransmuteFrom<R>`, `R: TransmuteFrom<T>`, and `R: $trait`. It is + // called using `$ty` and `$repr`, ensuring that `$ty` and `$repr` + // have equivalent bit validity, and ensuring that `$repr: $trait`. + // The supported traits - `TryFromBytes`, `FromZeros`, `FromBytes`, + // and `IntoBytes` - are defined only in terms of the bit validity + // of a type. Therefore, `$repr: $trait` ensures that `$ty: $trait` + // is sound. + unsafe impl<$($tyvar $(: $(? $optbound +)* $($bound +)*)?)?> $trait for $ty { + #[allow(dead_code, clippy::missing_inline_in_public_items)] + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn only_derive_is_allowed_to_implement_this_trait() { + use crate::pointer::{*, invariant::Valid}; + + impl_for_transmute_from!(@assert_is_supported_trait $trait); + + fn is_trait<T, R>() + where + T: TransmuteFrom<R, Valid, Valid> + ?Sized, + R: TransmuteFrom<T, Valid, Valid> + ?Sized, + R: $trait, + { + } + + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn f<$($tyvar $(: $(? $optbound +)* $($bound +)*)?)?>() { + is_trait::<$ty, $repr>(); + } + } + + impl_for_transmute_from!( + @is_bit_valid + $(<$tyvar $(: $(? $optbound +)* $($bound +)*)?>)? + $trait for $ty [$repr] + ); + } + }; + }; + (@assert_is_supported_trait TryFromBytes) => {}; + (@assert_is_supported_trait FromZeros) => {}; + (@assert_is_supported_trait FromBytes) => {}; + (@assert_is_supported_trait IntoBytes) => {}; + ( + @is_bit_valid + $(<$tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?>)? + TryFromBytes for $ty:ty [$repr:ty] + ) => { + #[inline(always)] + fn is_bit_valid<Alignment>(candidate: $crate::Maybe<'_, Self, Alignment>) -> bool + where + Alignment: $crate::invariant::Alignment, + { + // SAFETY: This macro ensures that `$repr` and `Self` have the same + // size and bit validity. Thus, a bit-valid instance of `$repr` is + // also a bit-valid instance of `Self`. + <$repr as TryFromBytes>::is_bit_valid(candidate.transmute::<_, _, BecauseImmutable>()) + } + }; + ( + @is_bit_valid + $(<$tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?>)? + $trait:ident for $ty:ty [$repr:ty] + ) => { + // Trait other than `TryFromBytes`; no `is_bit_valid` impl. + }; +} + +/// Implements a trait for a type, bounding on each member of the power set of +/// a set of type variables. This is useful for implementing traits for tuples +/// or `fn` types. +/// +/// The last argument is the name of a macro which will be called in every +/// `impl` block, and is expected to expand to the name of the type for which to +/// implement the trait. +/// +/// For example, the invocation: +/// ```ignore +/// unsafe_impl_for_power_set!(A, B => Foo for type!(...)) +/// ``` +/// ...expands to: +/// ```ignore +/// unsafe impl Foo for type!() { ... } +/// unsafe impl<B> Foo for type!(B) { ... } +/// unsafe impl<A, B> Foo for type!(A, B) { ... } +/// ``` +macro_rules! unsafe_impl_for_power_set { + ( + $first:ident $(, $rest:ident)* $(-> $ret:ident)? => $trait:ident for $macro:ident!(...) + $(; |$candidate:ident| $is_bit_valid:expr)? + ) => { + unsafe_impl_for_power_set!( + $($rest),* $(-> $ret)? => $trait for $macro!(...) + $(; |$candidate| $is_bit_valid)? + ); + unsafe_impl_for_power_set!( + @impl $first $(, $rest)* $(-> $ret)? => $trait for $macro!(...) + $(; |$candidate| $is_bit_valid)? + ); + }; + ( + $(-> $ret:ident)? => $trait:ident for $macro:ident!(...) + $(; |$candidate:ident| $is_bit_valid:expr)? + ) => { + unsafe_impl_for_power_set!( + @impl $(-> $ret)? => $trait for $macro!(...) + $(; |$candidate| $is_bit_valid)? + ); + }; + ( + @impl $($vars:ident),* $(-> $ret:ident)? => $trait:ident for $macro:ident!(...) + $(; |$candidate:ident| $is_bit_valid:expr)? + ) => { + unsafe_impl!( + $($vars,)* $($ret)? => $trait for $macro!($($vars),* $(-> $ret)?) + $(; |$candidate| $is_bit_valid)? + ); + }; +} + +/// Expands to an `Option<extern "C" fn>` type with the given argument types and +/// return type. Designed for use with `unsafe_impl_for_power_set`. +macro_rules! opt_extern_c_fn { + ($($args:ident),* -> $ret:ident) => { Option<extern "C" fn($($args),*) -> $ret> }; +} + +/// Expands to an `Option<unsafe extern "C" fn>` type with the given argument +/// types and return type. Designed for use with `unsafe_impl_for_power_set`. +macro_rules! opt_unsafe_extern_c_fn { + ($($args:ident),* -> $ret:ident) => { Option<unsafe extern "C" fn($($args),*) -> $ret> }; +} + +/// Expands to an `Option<fn>` type with the given argument types and return +/// type. Designed for use with `unsafe_impl_for_power_set`. +macro_rules! opt_fn { + ($($args:ident),* -> $ret:ident) => { Option<fn($($args),*) -> $ret> }; +} + +/// Expands to an `Option<unsafe fn>` type with the given argument types and +/// return type. Designed for use with `unsafe_impl_for_power_set`. +macro_rules! opt_unsafe_fn { + ($($args:ident),* -> $ret:ident) => { Option<unsafe fn($($args),*) -> $ret> }; +} + +// This `allow` is needed because, when testing, we export this macro so it can +// be used in `doctests`. +#[allow(rustdoc::private_intra_doc_links)] +/// Implements trait(s) for a type or verifies the given implementation by +/// referencing an existing (derived) implementation. +/// +/// This macro exists so that we can provide zerocopy-derive as an optional +/// dependency and still get the benefit of using its derives to validate that +/// our trait impls are sound. +/// +/// When compiling without `--cfg 'feature = "derive"` and without `--cfg test`, +/// `impl_or_verify!` emits the provided trait impl. When compiling with either +/// of those cfgs, it is expected that the type in question is deriving the +/// traits instead. In this case, `impl_or_verify!` emits code which validates +/// that the given trait impl is at least as restrictive as the the impl emitted +/// by the custom derive. This has the effect of confirming that the impl which +/// is emitted when the `derive` feature is disabled is actually sound (on the +/// assumption that the impl emitted by the custom derive is sound). +/// +/// The caller is still required to provide a safety comment (e.g. using the +/// `const _: () = unsafe` macro). The reason for this restriction is that, +/// while `impl_or_verify!` can guarantee that the provided impl is sound when +/// it is compiled with the appropriate cfgs, there is no way to guarantee that +/// it is ever compiled with those cfgs. In particular, it would be possible to +/// accidentally place an `impl_or_verify!` call in a context that is only ever +/// compiled when the `derive` feature is disabled. If that were to happen, +/// there would be nothing to prevent an unsound trait impl from being emitted. +/// Requiring a safety comment reduces the likelihood of emitting an unsound +/// impl in this case, and also provides useful documentation for readers of the +/// code. +/// +/// Finally, if a `TryFromBytes::is_bit_valid` impl is provided, it must adhere +/// to the safety preconditions of [`unsafe_impl!`]. +/// +/// ## Example +/// +/// ```rust,ignore +/// // Note that these derives are gated by `feature = "derive"` +/// #[cfg_attr(any(feature = "derive", test), derive(FromZeros, FromBytes, IntoBytes, Unaligned))] +/// #[repr(transparent)] +/// struct Wrapper<T>(T); +/// +/// const _: () = unsafe { +/// /// SAFETY: +/// /// `Wrapper<T>` is `repr(transparent)`, so it is sound to implement any +/// /// zerocopy trait if `T` implements that trait. +/// impl_or_verify!(T: FromZeros => FromZeros for Wrapper<T>); +/// impl_or_verify!(T: FromBytes => FromBytes for Wrapper<T>); +/// impl_or_verify!(T: IntoBytes => IntoBytes for Wrapper<T>); +/// impl_or_verify!(T: Unaligned => Unaligned for Wrapper<T>); +/// } +/// ``` +#[cfg_attr(__ZEROCOPY_INTERNAL_USE_ONLY_DEV_MODE, macro_export)] // Used in `doctests.rs` +#[doc(hidden)] +macro_rules! impl_or_verify { + // The following two match arms follow the same pattern as their + // counterparts in `unsafe_impl!`; see the documentation on those arms for + // more details. + ( + const $constname:ident : $constty:ident $(,)? + $($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?),* + => $trait:ident for $ty:ty + ) => { + impl_or_verify!(@impl { unsafe_impl!( + const $constname: $constty, $($tyvar $(: $(? $optbound +)* $($bound +)*)?),* => $trait for $ty + ); }); + impl_or_verify!(@verify $trait, { + impl<const $constname: $constty, $($tyvar $(: $(? $optbound +)* $($bound +)*)?),*> Subtrait for $ty {} + }); + }; + ( + $($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?),* + => $trait:ident for $ty:ty $(; |$candidate:ident| $is_bit_valid:expr)? + ) => { + impl_or_verify!(@impl { unsafe_impl!( + $($tyvar $(: $(? $optbound +)* $($bound +)*)?),* => $trait for $ty + $(; |$candidate| $is_bit_valid)? + ); }); + impl_or_verify!(@verify $trait, { + impl<$($tyvar $(: $(? $optbound +)* $($bound +)*)?),*> Subtrait for $ty {} + }); + }; + (@impl $impl_block:tt) => { + #[cfg(not(any(feature = "derive", test)))] + { $impl_block }; + }; + (@verify $trait:ident, $impl_block:tt) => { + #[cfg(any(feature = "derive", test))] + { + // On some toolchains, `Subtrait` triggers the `dead_code` lint + // because it is implemented but never used. + #[allow(dead_code)] + trait Subtrait: $trait {} + $impl_block + }; + }; +} + +/// Implements `KnownLayout` for a sized type. +macro_rules! impl_known_layout { + ($(const $constvar:ident : $constty:ty, $tyvar:ident $(: ?$optbound:ident)? => $ty:ty),* $(,)?) => { + $(impl_known_layout!(@inner const $constvar: $constty, $tyvar $(: ?$optbound)? => $ty);)* + }; + ($($tyvar:ident $(: ?$optbound:ident)? => $ty:ty),* $(,)?) => { + $(impl_known_layout!(@inner , $tyvar $(: ?$optbound)? => $ty);)* + }; + ($($(#[$attrs:meta])* $ty:ty),*) => { $(impl_known_layout!(@inner , => $(#[$attrs])* $ty);)* }; + (@inner $(const $constvar:ident : $constty:ty)? , $($tyvar:ident $(: ?$optbound:ident)?)? => $(#[$attrs:meta])* $ty:ty) => { + const _: () = { + use core::ptr::NonNull; + + #[allow(non_local_definitions)] + $(#[$attrs])* + // SAFETY: Delegates safety to `DstLayout::for_type`. + unsafe impl<$($tyvar $(: ?$optbound)?)? $(, const $constvar : $constty)?> KnownLayout for $ty { + #[allow(clippy::missing_inline_in_public_items)] + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn only_derive_is_allowed_to_implement_this_trait() where Self: Sized {} + + type PointerMetadata = (); + + // SAFETY: `CoreMaybeUninit<T>::LAYOUT` and `T::LAYOUT` are + // identical because `CoreMaybeUninit<T>` has the same size and + // alignment as `T` [1], and `CoreMaybeUninit` admits + // uninitialized bytes in all positions. + // + // [1] Per https://doc.rust-lang.org/1.81.0/std/mem/union.MaybeUninit.html#layout-1: + // + // `MaybeUninit<T>` is guaranteed to have the same size, + // alignment, and ABI as `T` + type MaybeUninit = core::mem::MaybeUninit<Self>; + + const LAYOUT: crate::DstLayout = crate::DstLayout::for_type::<$ty>(); + + // SAFETY: `.cast` preserves address and provenance. + // + // FIXME(#429): Add documentation to `.cast` that promises that + // it preserves provenance. + #[inline(always)] + fn raw_from_ptr_len(bytes: NonNull<u8>, _meta: ()) -> NonNull<Self> { + bytes.cast::<Self>() + } + + #[inline(always)] + fn pointer_to_metadata(_ptr: *mut Self) -> () { + } + } + }; + }; +} + +/// Implements `KnownLayout` for a type in terms of the implementation of +/// another type with the same representation. +/// +/// # Safety +/// +/// - `$ty` and `$repr` must have the same: +/// - Fixed prefix size +/// - Alignment +/// - (For DSTs) trailing slice element size +/// - It must be valid to perform an `as` cast from `*mut $repr` to `*mut $ty`, +/// and this operation must preserve referent size (ie, `size_of_val_raw`). +macro_rules! unsafe_impl_known_layout { + ($($tyvar:ident: ?Sized + KnownLayout =>)? #[repr($repr:ty)] $ty:ty) => {{ + use core::ptr::NonNull; + + crate::util::macros::__unsafe(); + + #[allow(non_local_definitions)] + // SAFETY: The caller promises that this is sound. + unsafe impl<$($tyvar: ?Sized + KnownLayout)?> KnownLayout for $ty { + #[allow(clippy::missing_inline_in_public_items, dead_code)] + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn only_derive_is_allowed_to_implement_this_trait() {} + + type PointerMetadata = <$repr as KnownLayout>::PointerMetadata; + type MaybeUninit = <$repr as KnownLayout>::MaybeUninit; + + const LAYOUT: DstLayout = <$repr as KnownLayout>::LAYOUT; + + // SAFETY: All operations preserve address and provenance. Caller + // has promised that the `as` cast preserves size. + // + // FIXME(#429): Add documentation to `NonNull::new_unchecked` that + // it preserves provenance. + #[inline(always)] + fn raw_from_ptr_len(bytes: NonNull<u8>, meta: <$repr as KnownLayout>::PointerMetadata) -> NonNull<Self> { + #[allow(clippy::as_conversions)] + let ptr = <$repr>::raw_from_ptr_len(bytes, meta).as_ptr() as *mut Self; + // SAFETY: `ptr` was converted from `bytes`, which is non-null. + unsafe { NonNull::new_unchecked(ptr) } + } + + #[inline(always)] + fn pointer_to_metadata(ptr: *mut Self) -> Self::PointerMetadata { + #[allow(clippy::as_conversions)] + let ptr = ptr as *mut $repr; + <$repr>::pointer_to_metadata(ptr) + } + } + }}; +} + +/// Uses `align_of` to confirm that a type or set of types have alignment 1. +/// +/// Note that `align_of<T>` requires `T: Sized`, so this macro doesn't work for +/// unsized types. +macro_rules! assert_unaligned { + ($($tys:ty),*) => { + $( + // We only compile this assertion under `cfg(test)` to avoid taking + // an extra non-dev dependency (and making this crate more expensive + // to compile for our dependents). + #[cfg(test)] + static_assertions::const_assert_eq!(core::mem::align_of::<$tys>(), 1); + )* + }; +} + +/// Emits a function definition as either `const fn` or `fn` depending on +/// whether the current toolchain version supports `const fn` with generic trait +/// bounds. +macro_rules! maybe_const_trait_bounded_fn { + // This case handles both `self` methods (where `self` is by value) and + // non-method functions. Each `$args` may optionally be followed by `: + // $arg_tys:ty`, which can be omitted for `self`. + ($(#[$attr:meta])* $vis:vis const fn $name:ident($($args:ident $(: $arg_tys:ty)?),* $(,)?) $(-> $ret_ty:ty)? $body:block) => { + #[cfg(not(no_zerocopy_generic_bounds_in_const_fn_1_61_0))] + $(#[$attr])* $vis const fn $name($($args $(: $arg_tys)?),*) $(-> $ret_ty)? $body + + #[cfg(no_zerocopy_generic_bounds_in_const_fn_1_61_0)] + $(#[$attr])* $vis fn $name($($args $(: $arg_tys)?),*) $(-> $ret_ty)? $body + }; +} + +/// Either panic (if the current Rust toolchain supports panicking in `const +/// fn`) or evaluate a constant that will cause an array indexing error whose +/// error message will include the format string. +/// +/// The type that this expression evaluates to must be `Copy`, or else the +/// non-panicking desugaring will fail to compile. +macro_rules! const_panic { + (@non_panic $($_arg:tt)+) => {{ + // This will type check to whatever type is expected based on the call + // site. + let panic: [_; 0] = []; + // This will always fail (since we're indexing into an array of size 0. + #[allow(unconditional_panic)] + panic[0] + }}; + ($($arg:tt)+) => {{ + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + panic!($($arg)+); + #[cfg(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0)] + const_panic!(@non_panic $($arg)+) + }}; +} + +/// Either assert (if the current Rust toolchain supports panicking in `const +/// fn`) or evaluate the expression and, if it evaluates to `false`, call +/// `const_panic!`. This is used in place of `assert!` in const contexts to +/// accommodate old toolchains. +macro_rules! const_assert { + ($e:expr) => {{ + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + assert!($e); + #[cfg(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0)] + { + let e = $e; + if !e { + let _: () = const_panic!(@non_panic concat!("assertion failed: ", stringify!($e))); + } + } + }}; + ($e:expr, $($args:tt)+) => {{ + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + assert!($e, $($args)+); + #[cfg(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0)] + { + let e = $e; + if !e { + let _: () = const_panic!(@non_panic concat!("assertion failed: ", stringify!($e), ": ", stringify!($arg)), $($args)*); + } + } + }}; +} + +/// Like `const_assert!`, but relative to `debug_assert!`. +macro_rules! const_debug_assert { + ($e:expr $(, $msg:expr)?) => {{ + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + debug_assert!($e $(, $msg)?); + #[cfg(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0)] + { + // Use this (rather than `#[cfg(debug_assertions)]`) to ensure that + // `$e` is always compiled even if it will never be evaluated at + // runtime. + if cfg!(debug_assertions) { + let e = $e; + if !e { + let _: () = const_panic!(@non_panic concat!("assertion failed: ", stringify!($e) $(, ": ", $msg)?)); + } + } + } + }} +} + +/// Either invoke `unreachable!()` or `loop {}` depending on whether the Rust +/// toolchain supports panicking in `const fn`. +macro_rules! const_unreachable { + () => {{ + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + unreachable!(); + + #[cfg(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0)] + loop {} + }}; +} + +/// Asserts at compile time that `$condition` is true for `Self` or the given +/// `$tyvar`s. Unlike `const_assert`, this is *strictly* a compile-time check; +/// it cannot be evaluated in a runtime context. The condition is checked after +/// monomorphization and, upon failure, emits a compile error. +macro_rules! static_assert { + (Self $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )? => $condition:expr $(, $args:tt)*) => {{ + trait StaticAssert { + const ASSERT: bool; + } + + impl<T $(: $(? $optbound +)* $($bound +)*)?> StaticAssert for T { + const ASSERT: bool = { + const_assert!($condition $(, $args)*); + $condition + }; + } + + const_assert!(<Self as StaticAssert>::ASSERT); + }}; + ($($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?),* => $condition:expr $(, $args:tt)*) => {{ + trait StaticAssert { + const ASSERT: bool; + } + + // NOTE: We use `PhantomData` so we can support unsized types. + impl<$($tyvar $(: $(? $optbound +)* $($bound +)*)?,)*> StaticAssert for ($(core::marker::PhantomData<$tyvar>,)*) { + const ASSERT: bool = { + const_assert!($condition $(, $args)*); + $condition + }; + } + + const_assert!(<($(core::marker::PhantomData<$tyvar>,)*) as StaticAssert>::ASSERT); + }}; +} + +/// Assert at compile time that `tyvar` does not have a zero-sized DST +/// component. +macro_rules! static_assert_dst_is_not_zst { + ($tyvar:ident) => {{ + use crate::KnownLayout; + static_assert!($tyvar: ?Sized + KnownLayout => { + let dst_is_zst = match $tyvar::LAYOUT.size_info { + crate::SizeInfo::Sized { .. } => false, + crate::SizeInfo::SliceDst(TrailingSliceLayout { elem_size, .. }) => { + elem_size == 0 + } + }; + !dst_is_zst + }, "cannot call this method on a dynamically-sized type whose trailing slice element is zero-sized"); + }} +} + +/// Defines a named [`Cast`] implementation. +/// +/// # Safety +/// +/// The caller must ensure that, given `src: *mut $src`, `src as *mut $dst` is a +/// size-preserving or size-shrinking cast. +/// +/// [`Cast`]: crate::pointer::cast::Cast +#[macro_export] +#[doc(hidden)] +macro_rules! define_cast { + // We require the caller to provide an `unsafe` block as part of the input + // syntax since a call to `define_cast!` is useless inside of an `unsafe` + // block (since it would introduce a type which can't be named outside of + // the context of that block). + (unsafe { $vis:vis $name:ident $(<$tyvar:ident $(: ?$optbound:ident)?>)? = $src:ty => $dst:ty }) => { + #[allow(missing_debug_implementations, missing_copy_implementations, unreachable_pub)] + $vis enum $name {} + + // SAFETY: The caller promises that `src as *mut $src` is a size- + // preserving or size-shrinking cast. All operations preserve + // provenance. + unsafe impl $(<$tyvar $(: ?$optbound)?>)? $crate::pointer::cast::Project<$src, $dst> for $name { + fn project(src: $crate::pointer::PtrInner<'_, $src>) -> *mut $dst { + #[allow(clippy::as_conversions)] + return src.as_ptr() as *mut $dst; + } + } + + // SAFETY: The impl of `Project::project` preserves referent address. + unsafe impl $(<$tyvar $(: ?$optbound)?>)? $crate::pointer::cast::Cast<$src, $dst> for $name {} + }; +} + +/// Implements `TransmuteFrom` and `SizeEq` for `T` and `$wrapper<T>`. +/// +/// # Safety +/// +/// `T` and `$wrapper<T>` must have the same bit validity, and must have the +/// same size in the sense of `CastExact` (specifically, both a +/// `T`-to-`$wrapper<T>` cast and a `$wrapper<T>`-to-`T` cast must be +/// size-preserving). +macro_rules! unsafe_impl_for_transparent_wrapper { + ($vis:vis T $(: ?$optbound:ident)? => $wrapper:ident<T>) => {{ + crate::util::macros::__unsafe(); + + use crate::pointer::{TransmuteFrom, cast::{CastExact, TransitiveProject}, SizeEq, invariant::Valid}; + use crate::wrappers::ReadOnly; + + // SAFETY: The caller promises that `T` and `$wrapper<T>` have the same + // bit validity. + unsafe impl<T $(: ?$optbound)?> TransmuteFrom<T, Valid, Valid> for $wrapper<T> {} + // SAFETY: See previous safety comment. + unsafe impl<T $(: ?$optbound)?> TransmuteFrom<$wrapper<T>, Valid, Valid> for T {} + // SAFETY: The caller promises that a `T` to `$wrapper<T>` cast is + // size-preserving. + define_cast!(unsafe { $vis CastToWrapper<T $(: ?$optbound)? > = T => $wrapper<T> }); + // SAFETY: The caller promises that a `T` to `$wrapper<T>` cast is + // size-preserving. + unsafe impl<T $(: ?$optbound)?> CastExact<T, $wrapper<T>> for CastToWrapper {} + // SAFETY: The caller promises that a `$wrapper<T>` to `T` cast is + // size-preserving. + define_cast!(unsafe { $vis CastFromWrapper<T $(: ?$optbound)? > = $wrapper<T> => T }); + // SAFETY: The caller promises that a `$wrapper<T>` to `T` cast is + // size-preserving. + unsafe impl<T $(: ?$optbound)?> CastExact<$wrapper<T>, T> for CastFromWrapper {} + + impl<T $(: ?$optbound)?> SizeEq<T> for $wrapper<T> { + type CastFrom = CastToWrapper; + } + impl<T $(: ?$optbound)?> SizeEq<$wrapper<T>> for T { + type CastFrom = CastFromWrapper; + } + + impl<T $(: ?$optbound)?> SizeEq<ReadOnly<T>> for $wrapper<T> { + type CastFrom = TransitiveProject< + T, + <T as SizeEq<ReadOnly<T>>>::CastFrom, + CastToWrapper, + >; + } + impl<T $(: ?$optbound)?> SizeEq<$wrapper<T>> for ReadOnly<T> { + type CastFrom = TransitiveProject< + T, + CastFromWrapper, + <ReadOnly<T> as SizeEq<T>>::CastFrom, + >; + } + + impl<T $(: ?$optbound)?> SizeEq<ReadOnly<T>> for ReadOnly<$wrapper<T>> { + type CastFrom = TransitiveProject< + $wrapper<T>, + <$wrapper<T> as SizeEq<ReadOnly<T>>>::CastFrom, + <ReadOnly<$wrapper<T>> as SizeEq<$wrapper<T>>>::CastFrom, + >; + } + impl<T $(: ?$optbound)?> SizeEq<ReadOnly<$wrapper<T>>> for ReadOnly<T> { + type CastFrom = TransitiveProject< + $wrapper<T>, + <$wrapper<T> as SizeEq<ReadOnly<$wrapper<T>>>>::CastFrom, + <ReadOnly<T> as SizeEq<$wrapper<T>>>::CastFrom, + >; + } + }}; +} + +macro_rules! impl_transitive_transmute_from { + ($($tyvar:ident $(: ?$optbound:ident)?)? => $t:ty => $u:ty => $v:ty) => { + const _: () = { + use crate::pointer::{TransmuteFrom, SizeEq, invariant::Valid}; + + impl<$($tyvar $(: ?$optbound)?)?> SizeEq<$t> for $v + where + $u: SizeEq<$t>, + $v: SizeEq<$u>, + { + type CastFrom = cast::TransitiveProject< + $u, + <$u as SizeEq<$t>>::CastFrom, + <$v as SizeEq<$u>>::CastFrom + >; + } + + // SAFETY: Since `$u: TransmuteFrom<$t, Valid, Valid>`, it is sound + // to transmute a bit-valid `$t` to a bit-valid `$u`. Since `$v: + // TransmuteFrom<$u, Valid, Valid>`, it is sound to transmute that + // bit-valid `$u` to a bit-valid `$v`. + unsafe impl<$($tyvar $(: ?$optbound)?)?> TransmuteFrom<$t, Valid, Valid> for $v + where + $u: TransmuteFrom<$t, Valid, Valid>, + $v: TransmuteFrom<$u, Valid, Valid>, + {} + }; + }; +} + +/// A no-op `unsafe fn` for use in macro expansions. +/// +/// Calling this function in a macro expansion ensures that the macro's caller +/// must wrap the call in `unsafe { ... }`. +#[inline(always)] +pub(crate) const unsafe fn __unsafe() {} + +/// Extracts the contents of doc comments. +#[allow(unused)] +macro_rules! docstring { + ($(#[doc = $content:expr])*) => { + concat!($($content, "\n",)*) + } +} + +/// Generate a rustdoc-style header with `$name` as the HTML ID for the 'Code +/// Generation' section of documentation. +#[allow(unused)] +macro_rules! codegen_header { + ($level:expr, $name:expr) => { + concat!( + " +<", + $level, + " id='method.", + $name, + ".codegen'> + <a class='doc-anchor' href='#method.", + $name, + ".codegen'>§</a> + Code Generation +</", + $level, + "> +" + ) + }; +} + +/// Generates HTML tabs. +#[rustfmt::skip] +#[allow(unused)] +macro_rules! tabs { + ( + name = $name:expr, + arity = $arity:literal, + $([ + $($open:ident)? + @index $n:literal + @title $title:literal + $(#[doc = $content:expr])* + ]),* + ) => { + concat!(" +<div class='codegen-tabs' style='--arity: ", $arity ,"'>", $(concat!(" + <details name='tab-", $name,"' style='--n: ", $n ,"'", $(stringify!($open),)*"> + <summary><h6>", $title, "</h6></summary> + <div> + +", $($content, "\n",)* " +\ + </div> + </details>"),)* +"</div>") + } +} + +/// Generates the HTML for a single benchmark example. +#[allow(unused)] +macro_rules! codegen_example { + (format = $format:expr, bench = $bench:expr) => { + tabs!( + name = $bench, + arity = 4, + [ + @index 1 + @title "Format" + /// ```ignore + #[doc = include_str!(concat!("../benches/formats/", $format, ".rs"))] + /// ``` + ], + [ + @index 2 + @title "Benchmark" + /// ```ignore + #[doc = include_str!(concat!("../benches/", $bench, ".rs"))] + /// ``` + ], + [ + open + @index 3 + @title "Assembly" + /// ```plain + #[doc = include_str!(concat!("../benches/", $bench, ".x86-64"))] + /// ``` + ], + [ + @index 4 + @title "Machine Code Analysis" + /// ```plain + #[doc = include_str!(concat!("../benches/", $bench, ".x86-64.mca"))] + /// ``` + ] + ) + } +} + +/// Generate the HTML for a suite of benchmark examples. +#[allow(unused)] +macro_rules! codegen_example_suite { + ( + bench = $bench:expr, + format = $format:expr, + arity = $arity:literal, + $([ + $($open:ident)? + @index $index:literal + @title $title:literal + @variant $variant:literal + ]),* + ) => { + tabs!( + name = $bench, + arity = $arity, + $([ + $($open)* + @index $index + @title $title + #[doc = codegen_example!( + format = concat!($format, "_", $variant), + bench = concat!($bench, "_", $variant) + )] + ]),* + ) + } +} + +/// Generates the string for code generation preamble. +#[allow(unused)] +macro_rules! codegen_preamble { + () => { + docstring!( + /// + /// This abstraction is safe and cheap, but does not necessarily + /// have zero runtime cost. The codegen you experience in practice + /// will depend on optimization level, the layout of the destination + /// type, and what the compiler can prove about the source. + /// + ) + } +} + +/// Stub for rendering codegen documentation; used to break build dependency +/// between benches and zerocopy when re-blessing codegen tests. +#[allow(unused)] +#[cfg(not(doc))] +macro_rules! codegen_section { + ( + header = $level:expr, + bench = $bench:expr, + format = $format:expr, + arity = $arity:literal, + $([ + $($open:ident)? + @index $index:literal + @title $title:literal + @variant $variant:literal + ]),* + ) => { + "" + }; + ( + header = $level:expr, + bench = $bench:expr, + format = $format:expr, + ) => { + "" + }; +} + +/// Generates the HTML for code generation documentation. +#[allow(unused)] +#[cfg(doc)] +macro_rules! codegen_section { + ( + header = $level:expr, + bench = $bench:expr, + format = $format:expr, + arity = $arity:literal, + $([ + $($open:ident)? + @index $index:literal + @title $title:literal + @variant $variant:literal + ]),* + ) => { + concat!( + codegen_header!($level, $bench), + codegen_preamble!(), + docstring!( + /// + /// The below examples illustrate typical codegen for + /// increasingly complex types: + /// + ), + codegen_example_suite!( + bench = $bench, + format = $format, + arity = $arity, + $([ + $($open)* + @index $index + @title $title + @variant $variant + ]),* + ) + ) + }; + ( + header = $level:expr, + bench = $bench:expr, + format = $format:expr, + ) => { + concat!( + codegen_header!($level, $bench), + codegen_preamble!(), + codegen_example!( + format = $format, + bench = $bench + ) + ) + } +} diff --git a/rust/zerocopy/src/util/mod.rs b/rust/zerocopy/src/util/mod.rs new file mode 100644 index 000000000000..d6d4c6c2fcd9 --- /dev/null +++ b/rust/zerocopy/src/util/mod.rs @@ -0,0 +1,938 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2023 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +#[macro_use] +pub(crate) mod macros; + +#[doc(hidden)] +pub mod macro_util; + +use core::{ + marker::PhantomData, + mem::{self, ManuallyDrop}, + num::NonZeroUsize, + ptr::NonNull, +}; + +use super::*; +use crate::pointer::{ + invariant::{Exclusive, Shared, Valid}, + SizeEq, TransmuteFromPtr, +}; + +/// Like [`PhantomData`], but [`Send`] and [`Sync`] regardless of whether the +/// wrapped `T` is. +pub(crate) struct SendSyncPhantomData<T: ?Sized>(PhantomData<T>); + +// SAFETY: `SendSyncPhantomData` does not enable any behavior which isn't sound +// to be called from multiple threads. +unsafe impl<T: ?Sized> Send for SendSyncPhantomData<T> {} +// SAFETY: `SendSyncPhantomData` does not enable any behavior which isn't sound +// to be called from multiple threads. +unsafe impl<T: ?Sized> Sync for SendSyncPhantomData<T> {} + +impl<T: ?Sized> Default for SendSyncPhantomData<T> { + fn default() -> SendSyncPhantomData<T> { + SendSyncPhantomData(PhantomData) + } +} + +impl<T: ?Sized> PartialEq for SendSyncPhantomData<T> { + fn eq(&self, _other: &Self) -> bool { + true + } +} + +impl<T: ?Sized> Eq for SendSyncPhantomData<T> {} + +impl<T: ?Sized> Clone for SendSyncPhantomData<T> { + fn clone(&self) -> Self { + SendSyncPhantomData(PhantomData) + } +} + +#[cfg(miri)] +extern "Rust" { + /// Miri-provided intrinsic that marks the pointer `ptr` as aligned to + /// `align`. + /// + /// This intrinsic is used to inform Miri's symbolic alignment checker that + /// a pointer is aligned, even if Miri cannot statically deduce that fact. + /// This is often required when performing raw pointer arithmetic or casts + /// where the alignment is guaranteed by runtime checks or invariants that + /// Miri is not aware of. + pub(crate) fn miri_promise_symbolic_alignment(ptr: *const (), align: usize); +} + +pub(crate) trait AsAddress { + fn addr(self) -> usize; +} + +impl<T: ?Sized> AsAddress for &T { + #[inline(always)] + fn addr(self) -> usize { + let ptr: *const T = self; + AsAddress::addr(ptr) + } +} + +impl<T: ?Sized> AsAddress for &mut T { + #[inline(always)] + fn addr(self) -> usize { + let ptr: *const T = self; + AsAddress::addr(ptr) + } +} + +impl<T: ?Sized> AsAddress for NonNull<T> { + #[inline(always)] + fn addr(self) -> usize { + AsAddress::addr(self.as_ptr()) + } +} + +impl<T: ?Sized> AsAddress for *const T { + #[inline(always)] + fn addr(self) -> usize { + // FIXME(#181), FIXME(https://github.com/rust-lang/rust/issues/95228): + // Use `.addr()` instead of `as usize` once it's stable, and get rid of + // this `allow`. Currently, `as usize` is the only way to accomplish + // this. + #[allow(clippy::as_conversions)] + #[cfg_attr( + __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, + allow(lossy_provenance_casts) + )] + return self.cast::<()>() as usize; + } +} + +impl<T: ?Sized> AsAddress for *mut T { + #[inline(always)] + fn addr(self) -> usize { + let ptr: *const T = self; + AsAddress::addr(ptr) + } +} + +/// Validates that `t` is aligned to `align_of::<U>()`. +#[inline(always)] +pub(crate) fn validate_aligned_to<T: AsAddress, U>(t: T) -> Result<(), AlignmentError<(), U>> { + // `mem::align_of::<U>()` is guaranteed to return a non-zero value, which in + // turn guarantees that this mod operation will not panic. + #[allow(clippy::arithmetic_side_effects)] + let remainder = t.addr() % mem::align_of::<U>(); + if remainder == 0 { + Ok(()) + } else { + // SAFETY: We just confirmed that `t.addr() % align_of::<U>() != 0`. + // That's only possible if `align_of::<U>() > 1`. + Err(unsafe { AlignmentError::new_unchecked(()) }) + } +} + +/// Returns the bytes needed to pad `len` to the next multiple of `align`. +/// +/// This function assumes that align is a power of two; there are no guarantees +/// on the answer it gives if this is not the case. +#[cfg_attr( + kani, + kani::requires(len <= DstLayout::MAX_SIZE), + kani::requires(align.is_power_of_two()), + kani::ensures(|&p| (len + p) % align.get() == 0), + // Ensures that we add the minimum required padding. + kani::ensures(|&p| p < align.get()), +)] +pub(crate) const fn padding_needed_for(len: usize, align: NonZeroUsize) -> usize { + #[cfg(kani)] + #[kani::proof_for_contract(padding_needed_for)] + fn proof() { + padding_needed_for(kani::any(), kani::any()); + } + + // Abstractly, we want to compute: + // align - (len % align). + // Handling the case where len%align is 0. + // Because align is a power of two, len % align = len & (align-1). + // Guaranteed not to underflow as align is nonzero. + #[allow(clippy::arithmetic_side_effects)] + let mask = align.get() - 1; + + // To efficiently subtract this value from align, we can use the bitwise + // complement. + // Note that ((!len) & (align-1)) gives us a number that with (len & + // (align-1)) sums to align-1. So subtracting 1 from x before taking the + // complement subtracts `len` from `align`. Some quick inspection of + // cases shows that this also handles the case where `len % align = 0` + // correctly too: len-1 % align then equals align-1, so the complement mod + // align will be 0, as desired. + // + // The following reasoning can be verified quickly by an SMT solver + // supporting the theory of bitvectors: + // ```smtlib + // ; Naive implementation of padding + // (define-fun padding1 ( + // (len (_ BitVec 32)) + // (align (_ BitVec 32))) (_ BitVec 32) + // (ite + // (= (_ bv0 32) (bvand len (bvsub align (_ bv1 32)))) + // (_ bv0 32) + // (bvsub align (bvand len (bvsub align (_ bv1 32)))))) + // + // ; The implementation below + // (define-fun padding2 ( + // (len (_ BitVec 32)) + // (align (_ BitVec 32))) (_ BitVec 32) + // (bvand (bvnot (bvsub len (_ bv1 32))) (bvsub align (_ bv1 32)))) + // + // (define-fun is-power-of-two ((x (_ BitVec 32))) Bool + // (= (_ bv0 32) (bvand x (bvsub x (_ bv1 32))))) + // + // (declare-const len (_ BitVec 32)) + // (declare-const align (_ BitVec 32)) + // ; Search for a case where align is a power of two and padding2 disagrees + // ; with padding1 + // (assert (and (is-power-of-two align) + // (not (= (padding1 len align) (padding2 len align))))) + // (simplify (padding1 (_ bv300 32) (_ bv32 32))) ; 20 + // (simplify (padding2 (_ bv300 32) (_ bv32 32))) ; 20 + // (simplify (padding1 (_ bv322 32) (_ bv32 32))) ; 30 + // (simplify (padding2 (_ bv322 32) (_ bv32 32))) ; 30 + // (simplify (padding1 (_ bv8 32) (_ bv8 32))) ; 0 + // (simplify (padding2 (_ bv8 32) (_ bv8 32))) ; 0 + // (check-sat) ; unsat, also works for 64-bit bitvectors + // ``` + !(len.wrapping_sub(1)) & mask +} + +/// Rounds `n` down to the largest value `m` such that `m <= n` and `m % align +/// == 0`. +/// +/// # Panics +/// +/// May panic if `align` is not a power of two. Even if it doesn't panic in this +/// case, it will produce nonsense results. +#[inline(always)] +#[cfg_attr( + kani, + kani::requires(align.is_power_of_two()), + kani::ensures(|&m| m <= n && m % align.get() == 0), + // Guarantees that `m` is the *largest* value such that `m % align == 0`. + kani::ensures(|&m| { + // If this `checked_add` fails, then the next multiple would wrap + // around, which trivially satisfies the "largest value" requirement. + m.checked_add(align.get()).map(|next_mul| next_mul > n).unwrap_or(true) + }) +)] +pub(crate) const fn round_down_to_next_multiple_of_alignment( + n: usize, + align: NonZeroUsize, +) -> usize { + #[cfg(kani)] + #[kani::proof_for_contract(round_down_to_next_multiple_of_alignment)] + fn proof() { + round_down_to_next_multiple_of_alignment(kani::any(), kani::any()); + } + + let align = align.get(); + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + debug_assert!(align.is_power_of_two()); + + // Subtraction can't underflow because `align.get() >= 1`. + #[allow(clippy::arithmetic_side_effects)] + let mask = !(align - 1); + n & mask +} + +pub(crate) const fn max(a: NonZeroUsize, b: NonZeroUsize) -> NonZeroUsize { + if a.get() < b.get() { + b + } else { + a + } +} + +pub(crate) const fn min(a: NonZeroUsize, b: NonZeroUsize) -> NonZeroUsize { + if a.get() > b.get() { + b + } else { + a + } +} + +/// Copies `src` into the prefix of `dst`. +/// +/// # Safety +/// +/// The caller guarantees that `src.len() <= dst.len()`. +#[inline(always)] +pub(crate) unsafe fn copy_unchecked(src: &[u8], dst: &mut [u8]) { + debug_assert!(src.len() <= dst.len()); + // SAFETY: This invocation satisfies the safety contract of + // copy_nonoverlapping [1]: + // - `src.as_ptr()` is trivially valid for reads of `src.len()` bytes + // - `dst.as_ptr()` is valid for writes of `src.len()` bytes, because the + // caller has promised that `src.len() <= dst.len()` + // - `src` and `dst` are, trivially, properly aligned + // - the region of memory beginning at `src` with a size of `src.len()` + // bytes does not overlap with the region of memory beginning at `dst` + // with the same size, because `dst` is derived from an exclusive + // reference. + unsafe { + core::ptr::copy_nonoverlapping(src.as_ptr(), dst.as_mut_ptr(), src.len()); + }; +} + +/// Unsafely transmutes the given `src` into a type `Dst`. +/// +/// # Safety +/// +/// The value `src` must be a valid instance of `Dst`. +#[inline(always)] +pub(crate) const unsafe fn transmute_unchecked<Src, Dst>(src: Src) -> Dst { + static_assert!(Src, Dst => core::mem::size_of::<Src>() == core::mem::size_of::<Dst>()); + + #[repr(C)] + union Transmute<Src, Dst> { + src: ManuallyDrop<Src>, + dst: ManuallyDrop<Dst>, + } + + // SAFETY: Since `Transmute<Src, Dst>` is `#[repr(C)]`, its `src` and `dst` + // fields both start at the same offset and the types of those fields are + // transparent wrappers around `Src` and `Dst` [1]. Consequently, + // initializing `Transmute` with with `src` and then reading out `dst` is + // equivalent to transmuting from `Src` to `Dst` [2]. Transmuting from `src` + // to `Dst` is valid because — by contract on the caller — `src` is a valid + // instance of `Dst`. + // + // [1] Per https://doc.rust-lang.org/1.82.0/std/mem/struct.ManuallyDrop.html: + // + // `ManuallyDrop<T>` is guaranteed to have the same layout and bit + // validity as `T`, and is subject to the same layout optimizations as + // `T`. + // + // [2] Per https://doc.rust-lang.org/1.82.0/reference/items/unions.html#reading-and-writing-union-fields: + // + // Effectively, writing to and then reading from a union with the C + // representation is analogous to a transmute from the type used for + // writing to the type used for reading. + unsafe { ManuallyDrop::into_inner(Transmute { src: ManuallyDrop::new(src) }.dst) } +} + +/// # Safety +/// +/// `Src` must have a greater or equal alignment to `Dst`. +pub(crate) unsafe fn transmute_ref<Src, Dst, R>(src: &Src) -> &Dst +where + Src: ?Sized, + Dst: SizeEq<Src> + + TransmuteFromPtr<Src, Shared, Valid, Valid, <Dst as SizeEq<Src>>::CastFrom, R> + + ?Sized, +{ + let dst = Ptr::from_ref(src).transmute(); + // SAFETY: The caller promises that `Src`'s alignment is at least as large + // as `Dst`'s alignment. + let dst = unsafe { dst.assume_alignment() }; + dst.as_ref() +} + +/// # Safety +/// +/// `Src` must have a greater or equal alignment to `Dst`. +pub(crate) unsafe fn transmute_mut<Src, Dst, R>(src: &mut Src) -> &mut Dst +where + Src: ?Sized, + Dst: SizeEq<Src> + + TransmuteFromPtr<Src, Exclusive, Valid, Valid, <Dst as SizeEq<Src>>::CastFrom, R> + + ?Sized, +{ + let dst = Ptr::from_mut(src).transmute(); + // SAFETY: The caller promises that `Src`'s alignment is at least as large + // as `Dst`'s alignment. + let dst = unsafe { dst.assume_alignment() }; + dst.as_mut() +} + +/// Uses `allocate` to create a `Box<T>`. +/// +/// # Errors +/// +/// Returns an error on allocation failure. Allocation failure is guaranteed +/// never to cause a panic or an abort. +/// +/// # Safety +/// +/// `allocate` must be either `alloc::alloc::alloc` or +/// `alloc::alloc::alloc_zeroed`. The referent of the box returned by `new_box` +/// has the same bit-validity as the referent of the pointer returned by the +/// given `allocate` and sufficient size to store `T` with `meta`. +#[must_use = "has no side effects (other than allocation)"] +#[cfg(feature = "alloc")] +#[inline] +pub(crate) unsafe fn new_box<T>( + meta: T::PointerMetadata, + allocate: unsafe fn(core::alloc::Layout) -> *mut u8, +) -> Result<alloc::boxed::Box<T>, AllocError> +where + T: ?Sized + crate::KnownLayout, +{ + let align = T::LAYOUT.align.get(); + if !T::is_valid_metadata(meta) { + return Err(AllocError); + } + let size = match T::size_for_metadata(meta) { + Some(size) => size, + // Thanks to the `!T::is_valid_metadata(meta)` check + // above, this branch is unreachable. Fortunately, the + // optimizer recognizes this, so replacing this branch + // with `unreachable_unchecked` produces no codegen + // improvements. + None => return Err(AllocError), + }; + let ptr = if size != 0 { + // SAFETY: + // - `align` is derived from a `NonZeroUsize` and is thus non-zero. + // - `align` is a power of two because, by invariant on + // `KnownLayout::LAYOUT` `<T as KnownLayout>::LAYOUT` accurately + // reflects the layout of `T`. + // - `size`, by invariant on `size_for_metadata` is well-aligned for + // `align` and, by the check on `T::is_valid_metadata(meta)`, is less + // than `isize::MAX`. + let layout: Layout = unsafe { Layout::from_size_align_unchecked(size, align) }; + // SAFETY: By contract on the caller, `allocate` is either + // `alloc::alloc::alloc` or `alloc::alloc::alloc_zeroed`. The above + // check ensures their shared safety precondition: that the supplied + // layout is not zero-sized type [1]. + // + // [1] Per https://doc.rust-lang.org/1.81.0/std/alloc/trait.GlobalAlloc.html#tymethod.alloc: + // + // This function is unsafe because undefined behavior can result if + // the caller does not ensure that layout has non-zero size. + let ptr = unsafe { allocate(layout) }; + match NonNull::new(ptr) { + Some(ptr) => ptr, + None => return Err(AllocError), + } + } else { + // We use `transmute` instead of an `as` cast since Miri (with strict + // provenance enabled) notices and complains that an `as` cast creates a + // pointer with no provenance. Miri isn't smart enough to realize that + // we're only executing this branch when we're constructing a zero-sized + // `Box`, which doesn't require provenance. + // + // SAFETY: any initialized bit sequence is a bit-valid `*mut u8`. All + // bits of a `usize` are initialized. + // + // `#[allow(unknown_lints)]` is for `integer_to_ptr_transmutes` + #[allow(unknown_lints)] + #[allow(clippy::useless_transmute, integer_to_ptr_transmutes)] + let dangling = unsafe { mem::transmute::<usize, *mut u8>(align) }; + // SAFETY: `dangling` is constructed from `align`, which is derived from + // a `NonZeroUsize`, which is guaranteed to be non-zero. + // + // `Box<[T]>` does not allocate when `T` is zero-sized or when `len` is + // zero, but it does require a non-null dangling pointer for its + // allocation. + // + // FIXME(https://github.com/rust-lang/rust/issues/95228): Use + // `std::ptr::without_provenance` once it's stable. That may optimize + // better. As written, Rust may assume that this consumes "exposed" + // provenance, and thus Rust may have to assume that this may consume + // provenance from any pointer whose provenance has been exposed. + unsafe { NonNull::new_unchecked(dangling) } + }; + + let ptr = T::raw_from_ptr_len(ptr, meta); + + // FIXME(#429): Add a "SAFETY" comment and remove this `allow`. Make sure to + // include a justification that `ptr.as_ptr()` is validly-aligned in the ZST + // case (in which we manually construct a dangling pointer) and to justify + // why `Box` is safe to drop (it's because `allocate` uses the system + // allocator). + #[allow(clippy::undocumented_unsafe_blocks)] + Ok(unsafe { alloc::boxed::Box::from_raw(ptr.as_ptr()) }) +} + +mod len_of { + use super::*; + + /// A witness type for metadata of a valid instance of `&T`. + pub struct MetadataOf<T: ?Sized + KnownLayout> { + /// # Safety + /// + /// The size of an instance of `&T` with the given metadata is not + /// larger than `isize::MAX`. + meta: T::PointerMetadata, + _p: PhantomData<T>, + } + + impl<T: ?Sized + KnownLayout> Copy for MetadataOf<T> {} + impl<T: ?Sized + KnownLayout> Clone for MetadataOf<T> { + #[inline] + fn clone(&self) -> Self { + *self + } + } + + impl<T: ?Sized + KnownLayout> core::fmt::Debug for MetadataOf<T> + where + T::PointerMetadata: core::fmt::Debug, + { + #[inline] + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("MetadataOf").field("meta", &self.meta).finish() + } + } + + impl<T: ?Sized> MetadataOf<T> + where + T: KnownLayout, + { + /// Returns `None` if `meta` is greater than `t`'s metadata. + #[inline(always)] + pub(crate) fn new_in_bounds(t: &T, meta: usize) -> Option<Self> + where + T: KnownLayout<PointerMetadata = usize>, + { + if meta <= Ptr::from_ref(t).len() { + // SAFETY: We have checked that `meta` is not greater than `t`'s + // metadata, which, by invariant on `&T`, addresses no more than + // `isize::MAX` bytes [1][2]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/primitive.reference.html#safety: + // + // For all types, `T: ?Sized`, and for all `t: &T` or `t: + // &mut T`, when such values cross an API boundary, the + // following invariants must generally be upheld: + // + // * `t` is non-null + // * `t` is aligned to `align_of_val(t)` + // * if `size_of_val(t) > 0`, then `t` is dereferenceable for + // `size_of_val(t)` many bytes + // + // If `t` points at address `a`, being "dereferenceable" for + // N bytes means that the memory range `[a, a + N)` is all + // contained within a single allocated object. + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/ptr/index.html#allocated-object: + // + // For any allocated object with `base` address, `size`, and + // a set of `addresses`, the following are guaranteed: + // - For all addresses `a` in `addresses`, `a` is in the + // range `base .. (base + size)` (note that this requires + // `a < base + size`, not `a <= base + size`) + // - `base` is not equal to [`null()`] (i.e., the address + // with the numerical value 0) + // - `base + size <= usize::MAX` + // - `size <= isize::MAX` + Some(unsafe { Self::new_unchecked(meta) }) + } else { + None + } + } + + /// # Safety + /// + /// The size of an instance of `&T` with the given metadata is not + /// larger than `isize::MAX`. + pub(crate) unsafe fn new_unchecked(meta: T::PointerMetadata) -> Self { + // SAFETY: The caller has promised that the size of an instance of + // `&T` with the given metadata is not larger than `isize::MAX`. + Self { meta, _p: PhantomData } + } + + pub(crate) fn get(&self) -> T::PointerMetadata + where + T::PointerMetadata: Copy, + { + self.meta + } + + #[inline] + pub(crate) fn padding_needed_for(&self) -> usize + where + T: KnownLayout<PointerMetadata = usize>, + { + let trailing_slice_layout = crate::trailing_slice_layout::<T>(); + + // FIXME(#67): Remove this allow. See NumExt for more details. + #[allow( + unstable_name_collisions, + clippy::incompatible_msrv, + clippy::multiple_unsafe_ops_per_block + )] + // SAFETY: By invariant on `self`, a `&T` with metadata `self.meta` + // describes an object of size `<= isize::MAX`. This computes the + // size of such a `&T` without any trailing padding, and so neither + // the multiplication nor the addition will overflow. + let unpadded_size = unsafe { + let trailing_size = self.meta.unchecked_mul(trailing_slice_layout.elem_size); + trailing_size.unchecked_add(trailing_slice_layout.offset) + }; + + util::padding_needed_for(unpadded_size, T::LAYOUT.align) + } + + #[inline(always)] + pub(crate) fn validate_cast_and_convert_metadata( + addr: usize, + bytes_len: MetadataOf<[u8]>, + cast_type: CastType, + meta: Option<T::PointerMetadata>, + ) -> Result<(MetadataOf<T>, MetadataOf<[u8]>), MetadataCastError> { + let layout = match meta { + None => T::LAYOUT, + // This can return `Err(MetadataCastError::Size)` if the + // metadata describes an object which can't fit in an `isize`. + Some(meta) => { + if !T::is_valid_metadata(meta) { + return Err(MetadataCastError::Size); + } + let size = match T::size_for_metadata(meta) { + Some(size) => size, + // Thanks to the `!T::is_valid_metadata(meta)` check + // above, this branch is unreachable. Fortunately, the + // optimizer recognizes this, so replacing this branch + // with `unreachable_unchecked` produces no codegen + // improvements. + None => return Err(MetadataCastError::Size), + }; + DstLayout { + align: T::LAYOUT.align, + size_info: crate::SizeInfo::Sized { size }, + statically_shallow_unpadded: false, + } + } + }; + // Lemma 0: By contract on `validate_cast_and_convert_metadata`, if + // the result is `Ok(..)`, then a `&T` with `elems` trailing slice + // elements is no larger in size than `bytes_len.get()`. + let (elems, split_at) = + layout.validate_cast_and_convert_metadata(addr, bytes_len.get(), cast_type)?; + let elems = T::PointerMetadata::from_elem_count(elems); + + // For a slice DST type, if `meta` is `Some(elems)`, then we + // synthesize `layout` to describe a sized type whose size is equal + // to the size of the instance that we are asked to cast. For sized + // types, `validate_cast_and_convert_metadata` returns `elems == 0`. + // Thus, in this case, we need to use the `elems` passed by the + // caller, not the one returned by + // `validate_cast_and_convert_metadata`. + // + // Lemma 1: A `&T` with `elems` trailing slice elements is no larger + // in size than `bytes_len.get()`. Proof: + // - If `meta` is `None`, then `elems` satisfies this condition by + // Lemma 0. + // - If `meta` is `Some(meta)`, then `layout` describes an object + // whose size is equal to the size of an `&T` with `meta` + // metadata. By Lemma 0, that size is not larger than + // `bytes_len.get()`. + // + // Lemma 2: A `&T` with `elems` trailing slice elements is no larger + // than `isize::MAX` bytes. Proof: By Lemma 1, a `&T` with metadata + // `elems` is not larger in size than `bytes_len.get()`. By + // invariant on `MetadataOf<[u8]>`, a `&[u8]` with metadata + // `bytes_len` is not larger than `isize::MAX`. Because + // `size_of::<u8>()` is `1`, a `&[u8]` with metadata `bytes_len` has + // size `bytes_len.get()` bytes. Therefore, a `&T` with metadata + // `elems` has size not larger than `isize::MAX`. + let elems = meta.unwrap_or(elems); + + // SAFETY: See Lemma 2. + let elems = unsafe { MetadataOf::new_unchecked(elems) }; + + // SAFETY: Let `size` be the size of a `&T` with metadata `elems`. + // By post-condition on `validate_cast_and_convert_metadata`, one of + // the following conditions holds: + // - `split_at == size`, in which case, by Lemma 2, `split_at <= + // isize::MAX`. Since `size_of::<u8>() == 1`, a `[u8]` with + // `split_at` elems has size not larger than `isize::MAX`. + // - `split_at == bytes_len - size`. Since `bytes_len: + // MetadataOf<u8>`, and since `size` is non-negative, `split_at` + // addresses no more bytes than `bytes_len` does. Since + // `bytes_len: MetadataOf<u8>`, `bytes_len` describes a `[u8]` + // which has no more than `isize::MAX` bytes, and thus so does + // `split_at`. + let split_at = unsafe { MetadataOf::<[u8]>::new_unchecked(split_at) }; + Ok((elems, split_at)) + } + } +} + +pub use len_of::MetadataOf; + +/// Since we support multiple versions of Rust, there are often features which +/// have been stabilized in the most recent stable release which do not yet +/// exist (stably) on our MSRV. This module provides polyfills for those +/// features so that we can write more "modern" code, and just remove the +/// polyfill once our MSRV supports the corresponding feature. Without this, +/// we'd have to write worse/more verbose code and leave FIXME comments +/// sprinkled throughout the codebase to update to the new pattern once it's +/// stabilized. +/// +/// Each trait is imported as `_` at the crate root; each polyfill should "just +/// work" at usage sites. +pub(crate) mod polyfills { + use core::ptr::{self, NonNull}; + + // A polyfill for `NonNull::slice_from_raw_parts` that we can use before our + // MSRV is 1.70, when that function was stabilized. + // + // The `#[allow(unused)]` is necessary because, on sufficiently recent + // toolchain versions, `ptr.slice_from_raw_parts()` resolves to the inherent + // method rather than to this trait, and so this trait is considered unused. + // + // FIXME(#67): Once our MSRV is 1.70, remove this. + #[allow(unused)] + pub(crate) trait NonNullExt<T> { + fn slice_from_raw_parts(data: Self, len: usize) -> NonNull<[T]>; + } + + impl<T> NonNullExt<T> for NonNull<T> { + // NOTE on coverage: this will never be tested in nightly since it's a + // polyfill for a feature which has been stabilized on our nightly + // toolchain. + #[cfg_attr( + all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + coverage(off) + )] + #[inline(always)] + fn slice_from_raw_parts(data: Self, len: usize) -> NonNull<[T]> { + let ptr = ptr::slice_from_raw_parts_mut(data.as_ptr(), len); + // SAFETY: `ptr` is converted from `data`, which is non-null. + unsafe { NonNull::new_unchecked(ptr) } + } + } + + // A polyfill for `Self::unchecked_sub` that we can use until methods like + // `usize::unchecked_sub` is stabilized. + // + // The `#[allow(unused)]` is necessary because, on sufficiently recent + // toolchain versions, `ptr.slice_from_raw_parts()` resolves to the inherent + // method rather than to this trait, and so this trait is considered unused. + // + // FIXME(#67): Once our MSRV is high enough, remove this. + #[allow(unused)] + pub(crate) trait NumExt { + /// Add without checking for overflow. + /// + /// # Safety + /// + /// The caller promises that the addition will not overflow. + unsafe fn unchecked_add(self, rhs: Self) -> Self; + + /// Subtract without checking for underflow. + /// + /// # Safety + /// + /// The caller promises that the subtraction will not underflow. + unsafe fn unchecked_sub(self, rhs: Self) -> Self; + + /// Multiply without checking for overflow. + /// + /// # Safety + /// + /// The caller promises that the multiplication will not overflow. + unsafe fn unchecked_mul(self, rhs: Self) -> Self; + } + + // NOTE on coverage: these will never be tested in nightly since they're + // polyfills for a feature which has been stabilized on our nightly + // toolchain. + impl NumExt for usize { + #[cfg_attr( + all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + coverage(off) + )] + #[inline(always)] + unsafe fn unchecked_add(self, rhs: usize) -> usize { + match self.checked_add(rhs) { + Some(x) => x, + None => { + // SAFETY: The caller promises that the addition will not + // underflow. + unsafe { core::hint::unreachable_unchecked() } + } + } + } + + #[cfg_attr( + all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + coverage(off) + )] + #[inline(always)] + unsafe fn unchecked_sub(self, rhs: usize) -> usize { + match self.checked_sub(rhs) { + Some(x) => x, + None => { + // SAFETY: The caller promises that the subtraction will not + // underflow. + unsafe { core::hint::unreachable_unchecked() } + } + } + } + + #[cfg_attr( + all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + coverage(off) + )] + #[inline(always)] + unsafe fn unchecked_mul(self, rhs: usize) -> usize { + match self.checked_mul(rhs) { + Some(x) => x, + None => { + // SAFETY: The caller promises that the multiplication will + // not overflow. + unsafe { core::hint::unreachable_unchecked() } + } + } + } + } +} + +#[cfg(test)] +pub(crate) mod testutil { + use crate::*; + + /// A `T` which is aligned to at least `align_of::<A>()`. + #[derive(Default)] + pub(crate) struct Align<T, A> { + pub(crate) t: T, + _a: [A; 0], + } + + impl<T: Default, A> Align<T, A> { + pub(crate) fn set_default(&mut self) { + self.t = T::default(); + } + } + + impl<T, A> Align<T, A> { + pub(crate) const fn new(t: T) -> Align<T, A> { + Align { t, _a: [] } + } + } + + /// A `T` which is guaranteed not to satisfy `align_of::<A>()`. + /// + /// It must be the case that `align_of::<T>() < align_of::<A>()` in order + /// for this type to work properly. + #[repr(C)] + pub(crate) struct ForceUnalign<T: Unaligned, A> { + // The outer struct is aligned to `A`, and, thanks to `repr(C)`, `t` is + // placed at the minimum offset that guarantees its alignment. If + // `align_of::<T>() < align_of::<A>()`, then that offset will be + // guaranteed *not* to satisfy `align_of::<A>()`. + // + // Note that we need `T: Unaligned` in order to guarantee that there is + // no padding between `_u` and `t`. + _u: u8, + pub(crate) t: T, + _a: [A; 0], + } + + impl<T: Unaligned, A> ForceUnalign<T, A> { + pub(crate) fn new(t: T) -> ForceUnalign<T, A> { + ForceUnalign { _u: 0, t, _a: [] } + } + } + // A `u64` with alignment 8. + // + // Though `u64` has alignment 8 on some platforms, it's not guaranteed. By + // contrast, `AU64` is guaranteed to have alignment 8 on all platforms. + #[derive( + KnownLayout, + Immutable, + FromBytes, + IntoBytes, + Eq, + PartialEq, + Ord, + PartialOrd, + Default, + Debug, + Copy, + Clone, + )] + #[repr(C, align(8))] + pub(crate) struct AU64(pub(crate) u64); + + impl AU64 { + // Converts this `AU64` to bytes using this platform's endianness. + pub(crate) fn to_bytes(self) -> [u8; 8] { + crate::transmute!(self) + } + } + + impl Display for AU64 { + #[cfg_attr( + all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + coverage(off) + )] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + Display::fmt(&self.0, f) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_round_down_to_next_multiple_of_alignment() { + fn alt_impl(n: usize, align: NonZeroUsize) -> usize { + let mul = n / align.get(); + mul * align.get() + } + + for align in [1, 2, 4, 8, 16] { + for n in 0..256 { + let align = NonZeroUsize::new(align).unwrap(); + let want = alt_impl(n, align); + let got = round_down_to_next_multiple_of_alignment(n, align); + assert_eq!(got, want, "round_down_to_next_multiple_of_alignment({}, {})", n, align); + } + } + } + + #[rustversion::since(1.57.0)] + #[test] + #[should_panic] + fn test_round_down_to_next_multiple_of_alignment_zerocopy_panic_in_const_and_vec_try_reserve() { + round_down_to_next_multiple_of_alignment(0, NonZeroUsize::new(3).unwrap()); + } + #[test] + fn test_send_sync_phantom_data() { + let x = SendSyncPhantomData::<u8>::default(); + let y = x.clone(); + assert!(x == y); + assert!(x == SendSyncPhantomData::<u8>::default()); + } + + #[test] + #[allow(clippy::as_conversions)] + fn test_as_address() { + let x = 0u8; + let r = &x; + let mut x_mut = 0u8; + let rm = &mut x_mut; + let p = r as *const u8; + let pm = rm as *mut u8; + let nn = NonNull::new(p as *mut u8).unwrap(); + + assert_eq!(AsAddress::addr(r), p as usize); + assert_eq!(AsAddress::addr(rm), pm as usize); + assert_eq!(AsAddress::addr(p), p as usize); + assert_eq!(AsAddress::addr(pm), pm as usize); + assert_eq!(AsAddress::addr(nn), p as usize); + } +} diff --git a/rust/zerocopy/src/wrappers.rs b/rust/zerocopy/src/wrappers.rs new file mode 100644 index 000000000000..266aec25fa58 --- /dev/null +++ b/rust/zerocopy/src/wrappers.rs @@ -0,0 +1,1034 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT + +// Copyright 2023 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use core::{fmt, hash::Hash}; + +use super::*; +use crate::pointer::{invariant::Valid, SizeEq, TransmuteFrom}; + +/// A type with no alignment requirement. +/// +/// An `Unalign` wraps a `T`, removing any alignment requirement. `Unalign<T>` +/// has the same size and bit validity as `T`, but not necessarily the same +/// alignment [or ABI]. This is useful if a type with an alignment requirement +/// needs to be read from a chunk of memory which provides no alignment +/// guarantees. +/// +/// Since `Unalign` has no alignment requirement, the inner `T` may not be +/// properly aligned in memory. There are five ways to access the inner `T`: +/// - by value, using [`get`] or [`into_inner`] +/// - by reference inside of a callback, using [`update`] +/// - fallibly by reference, using [`try_deref`] or [`try_deref_mut`]; these can +/// fail if the `Unalign` does not satisfy `T`'s alignment requirement at +/// runtime +/// - unsafely by reference, using [`deref_unchecked`] or +/// [`deref_mut_unchecked`]; it is the caller's responsibility to ensure that +/// the `Unalign` satisfies `T`'s alignment requirement +/// - (where `T: Unaligned`) infallibly by reference, using [`Deref::deref`] or +/// [`DerefMut::deref_mut`] +/// +/// [or ABI]: https://github.com/google/zerocopy/issues/164 +/// [`get`]: Unalign::get +/// [`into_inner`]: Unalign::into_inner +/// [`update`]: Unalign::update +/// [`try_deref`]: Unalign::try_deref +/// [`try_deref_mut`]: Unalign::try_deref_mut +/// [`deref_unchecked`]: Unalign::deref_unchecked +/// [`deref_mut_unchecked`]: Unalign::deref_mut_unchecked +/// +/// # Example +/// +/// In this example, we need `EthernetFrame` to have no alignment requirement - +/// and thus implement [`Unaligned`]. `EtherType` is `#[repr(u16)]` and so +/// cannot implement `Unaligned`. We use `Unalign` to relax `EtherType`'s +/// alignment requirement so that `EthernetFrame` has no alignment requirement +/// and can implement `Unaligned`. +/// +/// ```rust +/// use zerocopy::*; +/// # use zerocopy_derive::*; +/// # #[derive(FromBytes, KnownLayout, Immutable, Unaligned)] #[repr(C)] struct Mac([u8; 6]); +/// +/// # #[derive(PartialEq, Copy, Clone, Debug)] +/// #[derive(TryFromBytes, KnownLayout, Immutable)] +/// #[repr(u16)] +/// enum EtherType { +/// Ipv4 = 0x0800u16.to_be(), +/// Arp = 0x0806u16.to_be(), +/// Ipv6 = 0x86DDu16.to_be(), +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(TryFromBytes, KnownLayout, Immutable, Unaligned)] +/// #[repr(C)] +/// struct EthernetFrame { +/// src: Mac, +/// dst: Mac, +/// ethertype: Unalign<EtherType>, +/// payload: [u8], +/// } +/// +/// let bytes = &[ +/// # 0, 1, 2, 3, 4, 5, +/// # 6, 7, 8, 9, 10, 11, +/// # /* +/// ... +/// # */ +/// 0x86, 0xDD, // EtherType +/// 0xDE, 0xAD, 0xBE, 0xEF // Payload +/// ][..]; +/// +/// // PANICS: Guaranteed not to panic because `bytes` is of the right +/// // length, has the right contents, and `EthernetFrame` has no +/// // alignment requirement. +/// let packet = EthernetFrame::try_ref_from_bytes(&bytes).unwrap(); +/// +/// assert_eq!(packet.ethertype.get(), EtherType::Ipv6); +/// assert_eq!(packet.payload, [0xDE, 0xAD, 0xBE, 0xEF]); +/// ``` +/// +/// # Safety +/// +/// `Unalign<T>` is guaranteed to have the same size and bit validity as `T`, +/// and to have [`UnsafeCell`]s covering the same byte ranges as `T`. +/// `Unalign<T>` is guaranteed to have alignment 1. +// NOTE: This type is sound to use with types that need to be dropped. The +// reason is that the compiler-generated drop code automatically moves all +// values to aligned memory slots before dropping them in-place. This is not +// well-documented, but it's hinted at in places like [1] and [2]. However, this +// also means that `T` must be `Sized`; unless something changes, we can never +// support unsized `T`. [3] +// +// [1] https://github.com/rust-lang/rust/issues/54148#issuecomment-420529646 +// [2] https://github.com/google/zerocopy/pull/126#discussion_r1018512323 +// [3] https://github.com/google/zerocopy/issues/209 +#[allow(missing_debug_implementations)] +#[derive(Default, Copy)] +#[cfg_attr(any(feature = "derive", test), derive(Immutable, FromBytes, IntoBytes, Unaligned))] +#[repr(C, packed)] +pub struct Unalign<T>(T); + +// We do not use `derive(KnownLayout)` on `Unalign`, because the derive is not +// smart enough to realize that `Unalign<T>` is always sized and thus emits a +// `KnownLayout` impl bounded on `T: KnownLayout.` This is overly restrictive. +impl_known_layout!(T => Unalign<T>); + +// FIXME(https://github.com/rust-lang/rust-clippy/issues/16087): Move these +// attributes below the comment once this Clippy bug is fixed. +#[cfg_attr( + all(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, any(feature = "derive", test)), + expect(unused_unsafe) +)] +#[cfg_attr( + all( + not(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + any(feature = "derive", test) + ), + allow(unused_unsafe) +)] +// SAFETY: +// - `Unalign<T>` promises to have alignment 1, and so we don't require that `T: +// Unaligned`. +// - `Unalign<T>` has the same bit validity as `T`, and so it is `FromZeros`, +// `FromBytes`, or `IntoBytes` exactly when `T` is as well. +// - `Immutable`: `Unalign<T>` has the same fields as `T`, so it permits +// interior mutation exactly when `T` does. +// - `TryFromBytes`: `Unalign<T>` has the same the same bit validity as `T`, so +// `T::is_bit_valid` is a sound implementation of `is_bit_valid`. +// +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + impl_or_verify!(T => Unaligned for Unalign<T>); + impl_or_verify!(T: Immutable => Immutable for Unalign<T>); + impl_or_verify!( + T: TryFromBytes => TryFromBytes for Unalign<T>; + |c| T::is_bit_valid(c.transmute::<_, _, BecauseImmutable>()) + ); + impl_or_verify!(T: FromZeros => FromZeros for Unalign<T>); + impl_or_verify!(T: FromBytes => FromBytes for Unalign<T>); + impl_or_verify!(T: IntoBytes => IntoBytes for Unalign<T>); +}; + +// Note that `Unalign: Clone` only if `T: Copy`. Since the inner `T` may not be +// aligned, there's no way to safely call `T::clone`, and so a `T: Clone` bound +// is not sufficient to implement `Clone` for `Unalign`. +impl<T: Copy> Clone for Unalign<T> { + #[inline(always)] + fn clone(&self) -> Unalign<T> { + *self + } +} + +impl<T> Unalign<T> { + /// Constructs a new `Unalign`. + #[inline(always)] + pub const fn new(val: T) -> Unalign<T> { + Unalign(val) + } + + /// Consumes `self`, returning the inner `T`. + #[inline(always)] + pub const fn into_inner(self) -> T { + // SAFETY: Since `Unalign` is `#[repr(C, packed)]`, it has the same size + // and bit validity as `T`. + // + // We do this instead of just destructuring in order to prevent + // `Unalign`'s `Drop::drop` from being run, since dropping is not + // supported in `const fn`s. + // + // FIXME(https://github.com/rust-lang/rust/issues/73255): Destructure + // instead of using unsafe. + unsafe { crate::util::transmute_unchecked(self) } + } + + /// Attempts to return a reference to the wrapped `T`, failing if `self` is + /// not properly aligned. + /// + /// If `self` does not satisfy `align_of::<T>()`, then `try_deref` returns + /// `Err`. + /// + /// If `T: Unaligned`, then `Unalign<T>` implements [`Deref`], and callers + /// may prefer [`Deref::deref`], which is infallible. + #[inline(always)] + pub fn try_deref(&self) -> Result<&T, AlignmentError<&Self, T>> { + let inner = Ptr::from_ref(self).transmute(); + match inner.try_into_aligned() { + Ok(aligned) => Ok(aligned.as_ref()), + Err(err) => Err(err.map_src( + #[inline(always)] + |src| src.into_unalign().as_ref(), + )), + } + } + + /// Attempts to return a mutable reference to the wrapped `T`, failing if + /// `self` is not properly aligned. + /// + /// If `self` does not satisfy `align_of::<T>()`, then `try_deref` returns + /// `Err`. + /// + /// If `T: Unaligned`, then `Unalign<T>` implements [`DerefMut`], and + /// callers may prefer [`DerefMut::deref_mut`], which is infallible. + #[inline(always)] + pub fn try_deref_mut(&mut self) -> Result<&mut T, AlignmentError<&mut Self, T>> { + let inner = Ptr::from_mut(self).transmute::<_, _, (_, (_, _))>(); + match inner.try_into_aligned() { + Ok(aligned) => Ok(aligned.as_mut()), + Err(err) => Err(err.map_src(|src| src.into_unalign().as_mut())), + } + } + + /// Returns a reference to the wrapped `T` without checking alignment. + /// + /// If `T: Unaligned`, then `Unalign<T>` implements[ `Deref`], and callers + /// may prefer [`Deref::deref`], which is safe. + /// + /// # Safety + /// + /// The caller must guarantee that `self` satisfies `align_of::<T>()`. + #[inline(always)] + pub const unsafe fn deref_unchecked(&self) -> &T { + // SAFETY: `Unalign<T>` is `repr(transparent)`, so there is a valid `T` + // at the same memory location as `self`. It has no alignment guarantee, + // but the caller has promised that `self` is properly aligned, so we + // know that it is sound to create a reference to `T` at this memory + // location. + // + // We use `mem::transmute` instead of `&*self.get_ptr()` because + // dereferencing pointers is not stable in `const` on our current MSRV + // (1.56 as of this writing). + unsafe { mem::transmute(self) } + } + + /// Returns a mutable reference to the wrapped `T` without checking + /// alignment. + /// + /// If `T: Unaligned`, then `Unalign<T>` implements[ `DerefMut`], and + /// callers may prefer [`DerefMut::deref_mut`], which is safe. + /// + /// # Safety + /// + /// The caller must guarantee that `self` satisfies `align_of::<T>()`. + #[inline(always)] + pub unsafe fn deref_mut_unchecked(&mut self) -> &mut T { + // SAFETY: `self.get_mut_ptr()` returns a raw pointer to a valid `T` at + // the same memory location as `self`. It has no alignment guarantee, + // but the caller has promised that `self` is properly aligned, so we + // know that the pointer itself is aligned, and thus that it is sound to + // create a reference to a `T` at this memory location. + unsafe { &mut *self.get_mut_ptr() } + } + + /// Gets an unaligned raw pointer to the inner `T`. + /// + /// # Safety + /// + /// The returned raw pointer is not necessarily aligned to + /// `align_of::<T>()`. Most functions which operate on raw pointers require + /// those pointers to be aligned, so calling those functions with the result + /// of `get_ptr` will result in undefined behavior if alignment is not + /// guaranteed using some out-of-band mechanism. In general, the only + /// functions which are safe to call with this pointer are those which are + /// explicitly documented as being sound to use with an unaligned pointer, + /// such as [`read_unaligned`]. + /// + /// Even if the caller is permitted to mutate `self` (e.g. they have + /// ownership or a mutable borrow), it is not guaranteed to be sound to + /// write through the returned pointer. If writing is required, prefer + /// [`get_mut_ptr`] instead. + /// + /// [`read_unaligned`]: core::ptr::read_unaligned + /// [`get_mut_ptr`]: Unalign::get_mut_ptr + #[inline(always)] + pub const fn get_ptr(&self) -> *const T { + ptr::addr_of!(self.0) + } + + /// Gets an unaligned mutable raw pointer to the inner `T`. + /// + /// # Safety + /// + /// The returned raw pointer is not necessarily aligned to + /// `align_of::<T>()`. Most functions which operate on raw pointers require + /// those pointers to be aligned, so calling those functions with the result + /// of `get_ptr` will result in undefined behavior if alignment is not + /// guaranteed using some out-of-band mechanism. In general, the only + /// functions which are safe to call with this pointer are those which are + /// explicitly documented as being sound to use with an unaligned pointer, + /// such as [`read_unaligned`]. + /// + /// [`read_unaligned`]: core::ptr::read_unaligned + // FIXME(https://github.com/rust-lang/rust/issues/57349): Make this `const`. + #[inline(always)] + pub fn get_mut_ptr(&mut self) -> *mut T { + ptr::addr_of_mut!(self.0) + } + + /// Sets the inner `T`, dropping the previous value. + // FIXME(https://github.com/rust-lang/rust/issues/57349): Make this `const`. + #[inline(always)] + pub fn set(&mut self, t: T) { + *self = Unalign::new(t); + } + + /// Updates the inner `T` by calling a function on it. + /// + /// If [`T: Unaligned`], then `Unalign<T>` implements [`DerefMut`], and that + /// impl should be preferred over this method when performing updates, as it + /// will usually be faster and more ergonomic. + /// + /// For large types, this method may be expensive, as it requires copying + /// `2 * size_of::<T>()` bytes. \[1\] + /// + /// \[1\] Since the inner `T` may not be aligned, it would not be sound to + /// invoke `f` on it directly. Instead, `update` moves it into a + /// properly-aligned location in the local stack frame, calls `f` on it, and + /// then moves it back to its original location in `self`. + /// + /// [`T: Unaligned`]: Unaligned + #[inline] + pub fn update<O, F: FnOnce(&mut T) -> O>(&mut self, f: F) -> O { + if mem::align_of::<T>() == 1 { + // While we advise callers to use `DerefMut` when `T: Unaligned`, + // not all callers will be able to guarantee `T: Unaligned` in all + // cases. In particular, callers who are themselves providing an API + // which is generic over `T` may sometimes be called by *their* + // callers with `T` such that `align_of::<T>() == 1`, but cannot + // guarantee this in the general case. Thus, this optimization may + // sometimes be helpful. + + // SAFETY: Since `T`'s alignment is 1, `self` satisfies its + // alignment by definition. + let t = unsafe { self.deref_mut_unchecked() }; + return f(t); + } + + // On drop, this moves `copy` out of itself and uses `ptr::write` to + // overwrite `slf`. + struct WriteBackOnDrop<T> { + copy: ManuallyDrop<T>, + slf: *mut Unalign<T>, + } + + impl<T> Drop for WriteBackOnDrop<T> { + fn drop(&mut self) { + // SAFETY: We never use `copy` again as required by + // `ManuallyDrop::take`. + let copy = unsafe { ManuallyDrop::take(&mut self.copy) }; + // SAFETY: `slf` is the raw pointer value of `self`. We know it + // is valid for writes and properly aligned because `self` is a + // mutable reference, which guarantees both of these properties. + unsafe { ptr::write(self.slf, Unalign::new(copy)) }; + } + } + + // SAFETY: We know that `self` is valid for reads, properly aligned, and + // points to an initialized `Unalign<T>` because it is a mutable + // reference, which guarantees all of these properties. + // + // Since `T: !Copy`, it would be unsound in the general case to allow + // both the original `Unalign<T>` and the copy to be used by safe code. + // We guarantee that the copy is used to overwrite the original in the + // `Drop::drop` impl of `WriteBackOnDrop`. So long as this `drop` is + // called before any other safe code executes, soundness is upheld. + // While this method can terminate in two ways (by returning normally or + // by unwinding due to a panic in `f`), in both cases, `write_back` is + // dropped - and its `drop` called - before any other safe code can + // execute. + let copy = unsafe { ptr::read(self) }.into_inner(); + let mut write_back = WriteBackOnDrop { copy: ManuallyDrop::new(copy), slf: self }; + + let ret = f(&mut write_back.copy); + + drop(write_back); + ret + } +} + +impl<T: Copy> Unalign<T> { + /// Gets a copy of the inner `T`. + // FIXME(https://github.com/rust-lang/rust/issues/57349): Make this `const`. + #[inline(always)] + pub fn get(&self) -> T { + let Unalign(val) = *self; + val + } +} + +impl<T: Unaligned> Deref for Unalign<T> { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &T { + Ptr::from_ref(self).transmute().bikeshed_recall_aligned().as_ref() + } +} + +impl<T: Unaligned> DerefMut for Unalign<T> { + #[inline(always)] + fn deref_mut(&mut self) -> &mut T { + Ptr::from_mut(self).transmute::<_, _, (_, (_, _))>().bikeshed_recall_aligned().as_mut() + } +} + +impl<T: Unaligned + PartialOrd> PartialOrd<Unalign<T>> for Unalign<T> { + #[inline(always)] + fn partial_cmp(&self, other: &Unalign<T>) -> Option<Ordering> { + PartialOrd::partial_cmp(self.deref(), other.deref()) + } +} + +impl<T: Unaligned + Ord> Ord for Unalign<T> { + #[inline(always)] + fn cmp(&self, other: &Unalign<T>) -> Ordering { + Ord::cmp(self.deref(), other.deref()) + } +} + +impl<T: Unaligned + PartialEq> PartialEq<Unalign<T>> for Unalign<T> { + #[inline(always)] + fn eq(&self, other: &Unalign<T>) -> bool { + PartialEq::eq(self.deref(), other.deref()) + } +} + +impl<T: Unaligned + Eq> Eq for Unalign<T> {} + +impl<T: Unaligned + Hash> Hash for Unalign<T> { + #[inline(always)] + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.deref().hash(state); + } +} + +impl<T: Unaligned + Debug> Debug for Unalign<T> { + #[inline(always)] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + Debug::fmt(self.deref(), f) + } +} + +impl<T: Unaligned + Display> Display for Unalign<T> { + #[inline(always)] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + Display::fmt(self.deref(), f) + } +} + +/// A wrapper type to construct uninitialized instances of `T`. +/// +/// `MaybeUninit` is identical to the [standard library +/// `MaybeUninit`][core-maybe-uninit] type except that it supports unsized +/// types. +/// +/// # Layout +/// +/// The same layout guarantees and caveats apply to `MaybeUninit<T>` as apply to +/// the [standard library `MaybeUninit`][core-maybe-uninit] with one exception: +/// for `T: !Sized`, there is no single value for `T`'s size. Instead, for such +/// types, the following are guaranteed: +/// - Every [valid size][valid-size] for `T` is a valid size for +/// `MaybeUninit<T>` and vice versa +/// - Given `t: *const T` and `m: *const MaybeUninit<T>` with identical fat +/// pointer metadata, `t` and `m` address the same number of bytes (and +/// likewise for `*mut`) +/// +/// [core-maybe-uninit]: core::mem::MaybeUninit +/// [valid-size]: crate::KnownLayout#what-is-a-valid-size +#[repr(transparent)] +#[doc(hidden)] +pub struct MaybeUninit<T: ?Sized + KnownLayout>( + // SAFETY: `MaybeUninit<T>` has the same size as `T`, because (by invariant + // on `T::MaybeUninit`) `T::MaybeUninit` has `T::LAYOUT` identical to `T`, + // and because (invariant on `T::LAYOUT`) we can trust that `LAYOUT` + // accurately reflects the layout of `T`. By invariant on `T::MaybeUninit`, + // it admits uninitialized bytes in all positions. Because `MaybeUninit` is + // marked `repr(transparent)`, these properties additionally hold true for + // `Self`. + T::MaybeUninit, +); + +#[doc(hidden)] +impl<T: ?Sized + KnownLayout> MaybeUninit<T> { + /// Constructs a `MaybeUninit<T>` initialized with the given value. + #[inline(always)] + pub fn new(val: T) -> Self + where + T: Sized, + Self: Sized, + { + // SAFETY: It is valid to transmute `val` to `MaybeUninit<T>` because it + // is both valid to transmute `val` to `T::MaybeUninit`, and it is valid + // to transmute from `T::MaybeUninit` to `MaybeUninit<T>`. + // + // First, it is valid to transmute `val` to `T::MaybeUninit` because, by + // invariant on `T::MaybeUninit`: + // - For `T: Sized`, `T` and `T::MaybeUninit` have the same size. + // - All byte sequences of the correct size are valid values of + // `T::MaybeUninit`. + // + // Second, it is additionally valid to transmute from `T::MaybeUninit` + // to `MaybeUninit<T>`, because `MaybeUninit<T>` is a + // `repr(transparent)` wrapper around `T::MaybeUninit`. + // + // These two transmutes are collapsed into one so we don't need to add a + // `T::MaybeUninit: Sized` bound to this function's `where` clause. + unsafe { crate::util::transmute_unchecked(val) } + } + + /// Constructs an uninitialized `MaybeUninit<T>`. + #[must_use] + #[inline(always)] + pub fn uninit() -> Self + where + T: Sized, + Self: Sized, + { + let uninit = CoreMaybeUninit::<T>::uninit(); + // SAFETY: It is valid to transmute from `CoreMaybeUninit<T>` to + // `MaybeUninit<T>` since they both admit uninitialized bytes in all + // positions, and they have the same size (i.e., that of `T`). + // + // `MaybeUninit<T>` has the same size as `T`, because (by invariant on + // `T::MaybeUninit`) `T::MaybeUninit` has `T::LAYOUT` identical to `T`, + // and because (invariant on `T::LAYOUT`) we can trust that `LAYOUT` + // accurately reflects the layout of `T`. + // + // `CoreMaybeUninit<T>` has the same size as `T` [1] and admits + // uninitialized bytes in all positions. + // + // [1] Per https://doc.rust-lang.org/1.81.0/std/mem/union.MaybeUninit.html#layout-1: + // + // `MaybeUninit<T>` is guaranteed to have the same size, alignment, + // and ABI as `T` + unsafe { crate::util::transmute_unchecked(uninit) } + } + + /// Creates a `Box<MaybeUninit<T>>`. + /// + /// This function is useful for allocating large, uninit values on the heap + /// without ever creating a temporary instance of `Self` on the stack. + /// + /// # Errors + /// + /// Returns an error on allocation failure. Allocation failure is guaranteed + /// never to cause a panic or an abort. + #[cfg(feature = "alloc")] + #[inline] + pub fn new_boxed_uninit(meta: T::PointerMetadata) -> Result<Box<Self>, AllocError> { + // SAFETY: `alloc::alloc::alloc_zeroed` is a valid argument of + // `new_box`. The referent of the pointer returned by `alloc` (and, + // consequently, the `Box` derived from it) is a valid instance of + // `Self`, because `Self` is `MaybeUninit` and thus admits arbitrary + // (un)initialized bytes. + unsafe { crate::util::new_box(meta, alloc::alloc::alloc) } + } + + /// Extracts the value from the `MaybeUninit<T>` container. + /// + /// # Safety + /// + /// The caller must ensure that `self` is in an bit-valid state. Depending + /// on subsequent use, it may also need to be in a library-valid state. + #[inline(always)] + pub unsafe fn assume_init(self) -> T + where + T: Sized, + Self: Sized, + { + // SAFETY: The caller guarantees that `self` is in an bit-valid state. + unsafe { crate::util::transmute_unchecked(self) } + } +} + +impl<T: ?Sized + KnownLayout> fmt::Debug for MaybeUninit<T> { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad(core::any::type_name::<Self>()) + } +} + +#[allow(unreachable_pub)] // False positive on MSRV +#[doc(hidden)] +pub use read_only_def::*; +mod read_only_def { + /// A read-only wrapper. + /// + /// A `ReadOnly<T>` disables any interior mutability in `T`, ensuring that + /// a `&ReadOnly<T>` is genuinely read-only. Thus, `ReadOnly<T>` is + /// [`Immutable`] regardless of whether `T` is. + /// + /// Note that `&mut ReadOnly<T>` still permits mutation – the read-only + /// property only applies to shared references. + /// + /// [`Immutable`]: crate::Immutable + #[repr(transparent)] + pub struct ReadOnly<T: ?Sized> { + // INVARIANT: `inner` is never mutated through a `&ReadOnly<T>` + // reference. + inner: T, + } + + impl<T> ReadOnly<T> { + /// Creates a new `ReadOnly`. + #[must_use] + #[inline(always)] + pub const fn new(t: T) -> ReadOnly<T> { + ReadOnly { inner: t } + } + + /// Returns the inner value. + #[must_use] + #[inline(always)] + pub fn into_inner(r: ReadOnly<T>) -> T { + r.inner + } + } + + impl<T: ?Sized> ReadOnly<T> { + #[inline(always)] + pub(crate) fn as_mut(r: &mut ReadOnly<T>) -> &mut T { + // SAFETY: `r: &mut ReadOnly`, so this doesn't violate the invariant + // that `inner` is never mutated through a `&ReadOnly<T>` reference. + &mut r.inner + } + + /// # Safety + /// + /// The caller promises not to mutate the referent (i.e., via interior + /// mutation). + pub(crate) const unsafe fn as_ref_unchecked(r: &ReadOnly<T>) -> &T { + // SAFETY: The caller promises not to mutate the referent. + &r.inner + } + } +} + +// SAFETY: `ReadOnly<T>` is a `#[repr(transparent)` wrapper around `T`. +const _: () = unsafe { + unsafe_impl_known_layout!(T: ?Sized + KnownLayout => #[repr(T)] ReadOnly<T>); +}; + +#[allow(clippy::multiple_unsafe_ops_per_block)] +// SAFETY: +// - `ReadOnly<T>` has the same alignment as `T`, and so it is `Unaligned` +// exactly when `T` is as well. +// - `ReadOnly<T>` has the same bit validity as `T`, and so this `is_bit_valid` +// implementation is correct, and thus the `TryFromBytes` impl is sound. +// - `ReadOnly<T>` has the same bit validity as `T`, and so it is `FromZeros`, +// `FromBytes`, and `IntoBytes` exactly when `T` is as well. +const _: () = unsafe { + unsafe_impl!(T: ?Sized + Unaligned => Unaligned for ReadOnly<T>); + unsafe_impl!( + T: ?Sized + TryFromBytes => TryFromBytes for ReadOnly<T>; + |c| T::is_bit_valid(c.cast::<_, <ReadOnly<T> as SizeEq<ReadOnly<ReadOnly<T>>>>::CastFrom, _>()) + ); + unsafe_impl!(T: ?Sized + FromZeros => FromZeros for ReadOnly<T>); + unsafe_impl!(T: ?Sized + FromBytes => FromBytes for ReadOnly<T>); + unsafe_impl!(T: ?Sized + IntoBytes => IntoBytes for ReadOnly<T>); +}; + +// SAFETY: By invariant, `inner` is never mutated through a `&ReadOnly<T>` +// reference. +const _: () = unsafe { + unsafe_impl!(T: ?Sized => Immutable for ReadOnly<T>); +}; + +const _: () = { + use crate::pointer::cast::CastExact; + + // SAFETY: `ReadOnly<T>` has the same layout as `T`. + define_cast!(unsafe { pub CastFromReadOnly<T: ?Sized> = ReadOnly<T> => T}); + // SAFETY: `ReadOnly<T>` has the same layout as `T`. + unsafe impl<T: ?Sized> CastExact<ReadOnly<T>, T> for CastFromReadOnly {} + // SAFETY: `ReadOnly<T>` has the same layout as `T`. + define_cast!(unsafe { pub CastToReadOnly<T: ?Sized> = T => ReadOnly<T>}); + // SAFETY: `ReadOnly<T>` has the same layout as `T`. + unsafe impl<T: ?Sized> CastExact<T, ReadOnly<T>> for CastToReadOnly {} + + impl<T: ?Sized> SizeEq<ReadOnly<T>> for T { + type CastFrom = CastFromReadOnly; + } + + impl<T: ?Sized> SizeEq<T> for ReadOnly<T> { + type CastFrom = CastToReadOnly; + } +}; + +// SAFETY: `ReadOnly<T>` is a `#[repr(transparent)]` wrapper around `T`, and so +// it has the same bit validity as `T`. +unsafe impl<T: ?Sized> TransmuteFrom<T, Valid, Valid> for ReadOnly<T> {} + +// SAFETY: `ReadOnly<T>` is a `#[repr(transparent)]` wrapper around `T`, and so +// it has the same bit validity as `T`. +unsafe impl<T: ?Sized> TransmuteFrom<ReadOnly<T>, Valid, Valid> for T {} + +impl<'a, T: ?Sized + Immutable> From<&'a T> for &'a ReadOnly<T> { + #[inline(always)] + fn from(t: &'a T) -> &'a ReadOnly<T> { + let ro = Ptr::from_ref(t).transmute::<_, _, (_, _)>(); + // SAFETY: `ReadOnly<T>` has the same alignment as `T`, and + // `Ptr::from_ref` produces an aligned `Ptr`. + let ro = unsafe { ro.assume_alignment() }; + ro.as_ref() + } +} + +impl<T: ?Sized + Immutable> Deref for ReadOnly<T> { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &Self::Target { + // SAFETY: By `T: Immutable`, `&T` doesn't permit interior mutation. + unsafe { ReadOnly::as_ref_unchecked(self) } + } +} + +impl<T: ?Sized + Immutable> DerefMut for ReadOnly<T> { + #[inline(always)] + fn deref_mut(&mut self) -> &mut Self::Target { + ReadOnly::as_mut(self) + } +} + +impl<T: ?Sized + Immutable + Debug> Debug for ReadOnly<T> { + #[inline(always)] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + self.deref().fmt(f) + } +} + +// SAFETY: See safety comment on `ProjectToTag`. +unsafe impl<T: HasTag + ?Sized> HasTag for ReadOnly<T> { + #[allow(clippy::missing_inline_in_public_items)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + type Tag = T::Tag; + + // SAFETY: `<T as SizeEq<ReadOnly<T>>>::CastFrom` is a no-op projection that + // produces a pointer with the same referent. By invariant, for any `Ptr<'_, + // T, I>` it is sound to use `T::ProjectToTag` to project to a `Ptr<'_, + // T::Tag, I>`. Since `ReadOnly<T>` has the same layout and validity as `T`, + // the same is true of projecting from a `Ptr<'_, ReadOnly<T>, I>`. + type ProjectToTag = crate::pointer::cast::TransitiveProject< + T, + <T as SizeEq<ReadOnly<T>>>::CastFrom, + T::ProjectToTag, + >; +} + +// SAFETY: `ReadOnly<T>` is a `#[repr(transparent)]` wrapper around `T`, and so +// has the same fields at the same offsets. Thus, it satisfies the safety +// invariants of `HasField<Field, VARIANT_ID, FIELD_ID>` for field `f` exactly +// when `T` does, as guaranteed by the `T: HasField` bound: +// - If `VARIANT_ID` is `STRUCT_VARIANT_ID` or `UNION_VARIANT_ID`, then `T` has +// the layout of a struct or union type. Since `ReadOnly<T>` is a transparent +// wrapper around `T`, it does too. Otherwise, if `VARIANT_ID` is an enum +// variant index, then `T` has the layout of an enum type, and `ReadOnly<T>` +// does too. +// - By `T: HasField<_, _, FIELD_ID>`: +// - `T` has a field `f` with name `n` such that +// `FIELD_ID = zerocopy::ident_id!(n)` or at index `i` such that +// `FIELD_ID = zerocopy::ident_id!(i)`. +// - `Field` has the same visibility as `f`. +// - `T::Type` has the same type as `f`. Thus, `ReadOnly<T::Type>` has the +// same type as `f`, wrapped in `ReadOnly`. +// +// `project` satisfies its post-condition – namely, that the returned pointer +// refers to a non-strict subset of the bytes of `slf`'s referent, and has the +// same provenance as `slf` – because all intermediate operations satisfy those +// same conditions. +unsafe impl<T, Field, const VARIANT_ID: i128, const FIELD_ID: i128> + HasField<Field, VARIANT_ID, FIELD_ID> for ReadOnly<T> +where + T: HasField<Field, VARIANT_ID, FIELD_ID> + ?Sized, +{ + #[allow(clippy::missing_inline_in_public_items)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + type Type = ReadOnly<T::Type>; + + #[inline(always)] + fn project(slf: PtrInner<'_, Self>) -> *mut ReadOnly<T::Type> { + slf.project::<_, <T as SizeEq<ReadOnly<T>>>::CastFrom>() + .project::<_, crate::pointer::cast::Projection<Field, VARIANT_ID, FIELD_ID>>() + .project::<_, <ReadOnly<T::Type> as SizeEq<T::Type>>::CastFrom>() + .as_non_null() + .as_ptr() + } +} + +// SAFETY: `ReadOnly<T>` is a `#[repr(transparent)]` wrapper around `T`, and so +// has the same fields at the same offsets. `is_projectable` simply delegates to +// `T::is_projectable`, which is sound because a `Ptr<'_, ReadOnly<T>, I>` will +// be projectable exactly when a `Ptr<'_, T, I>` referent is. +unsafe impl<T, Field, I, const VARIANT_ID: i128, const FIELD_ID: i128> + ProjectField<Field, I, VARIANT_ID, FIELD_ID> for ReadOnly<T> +where + T: ProjectField<Field, I, VARIANT_ID, FIELD_ID> + ?Sized, + I: invariant::Invariants, +{ + #[allow(clippy::missing_inline_in_public_items)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + type Invariants = T::Invariants; + + type Error = T::Error; + + #[inline(always)] + fn is_projectable<'a>(ptr: Ptr<'a, Self::Tag, I>) -> Result<(), Self::Error> { + T::is_projectable(ptr) + } +} + +#[cfg(test)] +mod tests { + use core::panic::AssertUnwindSafe; + + use super::*; + use crate::util::testutil::*; + + #[test] + fn test_unalign() { + // Test methods that don't depend on alignment. + let mut u = Unalign::new(AU64(123)); + assert_eq!(u.get(), AU64(123)); + assert_eq!(u.into_inner(), AU64(123)); + assert_eq!(u.get_ptr(), <*const _>::cast::<AU64>(&u)); + assert_eq!(u.get_mut_ptr(), <*mut _>::cast::<AU64>(&mut u)); + u.set(AU64(321)); + assert_eq!(u.get(), AU64(321)); + + // Test methods that depend on alignment (when alignment is satisfied). + let mut u: Align<_, AU64> = Align::new(Unalign::new(AU64(123))); + assert_eq!(u.t.try_deref().unwrap(), &AU64(123)); + assert_eq!(u.t.try_deref_mut().unwrap(), &mut AU64(123)); + // SAFETY: The `Align<_, AU64>` guarantees proper alignment. + assert_eq!(unsafe { u.t.deref_unchecked() }, &AU64(123)); + // SAFETY: The `Align<_, AU64>` guarantees proper alignment. + assert_eq!(unsafe { u.t.deref_mut_unchecked() }, &mut AU64(123)); + *u.t.try_deref_mut().unwrap() = AU64(321); + assert_eq!(u.t.get(), AU64(321)); + + // Test methods that depend on alignment (when alignment is not + // satisfied). + let mut u: ForceUnalign<_, AU64> = ForceUnalign::new(Unalign::new(AU64(123))); + assert!(matches!(u.t.try_deref(), Err(AlignmentError { .. }))); + assert!(matches!(u.t.try_deref_mut(), Err(AlignmentError { .. }))); + + // Test methods that depend on `T: Unaligned`. + let mut u = Unalign::new(123u8); + assert_eq!(u.try_deref(), Ok(&123)); + assert_eq!(u.try_deref_mut(), Ok(&mut 123)); + assert_eq!(u.deref(), &123); + assert_eq!(u.deref_mut(), &mut 123); + *u = 21; + assert_eq!(u.get(), 21); + + // Test that some `Unalign` functions and methods are `const`. + const _UNALIGN: Unalign<u64> = Unalign::new(0); + const _UNALIGN_PTR: *const u64 = _UNALIGN.get_ptr(); + const _U64: u64 = _UNALIGN.into_inner(); + // Make sure all code is considered "used". + // + // FIXME(https://github.com/rust-lang/rust/issues/104084): Remove this + // attribute. + #[allow(dead_code)] + const _: () = { + let x: Align<_, AU64> = Align::new(Unalign::new(AU64(123))); + // Make sure that `deref_unchecked` is `const`. + // + // SAFETY: The `Align<_, AU64>` guarantees proper alignment. + let au64 = unsafe { x.t.deref_unchecked() }; + match au64 { + AU64(123) => {} + _ => const_unreachable!(), + } + }; + } + + #[test] + fn test_unalign_update() { + let mut u = Unalign::new(AU64(123)); + u.update(|a| a.0 += 1); + assert_eq!(u.get(), AU64(124)); + + // Test that, even if the callback panics, the original is still + // correctly overwritten. Use a `Box` so that Miri is more likely to + // catch any unsoundness (which would likely result in two `Box`es for + // the same heap object, which is the sort of thing that Miri would + // probably catch). + let mut u = Unalign::new(Box::new(AU64(123))); + let res = std::panic::catch_unwind(AssertUnwindSafe(|| { + u.update(|a| { + a.0 += 1; + panic!(); + }) + })); + assert!(res.is_err()); + assert_eq!(u.into_inner(), Box::new(AU64(124))); + + // Test the align_of::<T>() == 1 optimization. + let mut u = Unalign::new([0u8, 1]); + u.update(|a| a[0] += 1); + assert_eq!(u.get(), [1u8, 1]); + } + + #[test] + fn test_unalign_copy_clone() { + // Test that `Copy` and `Clone` do not cause soundness issues. This test + // is mainly meant to exercise UB that would be caught by Miri. + + // `u.t` is definitely not validly-aligned for `AU64`'s alignment of 8. + let u = ForceUnalign::<_, AU64>::new(Unalign::new(AU64(123))); + #[allow(clippy::clone_on_copy)] + let v = u.t.clone(); + let w = u.t; + assert_eq!(u.t.get(), v.get()); + assert_eq!(u.t.get(), w.get()); + assert_eq!(v.get(), w.get()); + } + + #[test] + fn test_unalign_trait_impls() { + let zero = Unalign::new(0u8); + let one = Unalign::new(1u8); + + assert!(zero < one); + assert_eq!(PartialOrd::partial_cmp(&zero, &one), Some(Ordering::Less)); + assert_eq!(Ord::cmp(&zero, &one), Ordering::Less); + + assert_ne!(zero, one); + assert_eq!(zero, zero); + assert!(!PartialEq::eq(&zero, &one)); + assert!(PartialEq::eq(&zero, &zero)); + + fn hash<T: Hash>(t: &T) -> u64 { + let mut h = std::collections::hash_map::DefaultHasher::new(); + t.hash(&mut h); + h.finish() + } + + assert_eq!(hash(&zero), hash(&0u8)); + assert_eq!(hash(&one), hash(&1u8)); + + assert_eq!(format!("{:?}", zero), format!("{:?}", 0u8)); + assert_eq!(format!("{:?}", one), format!("{:?}", 1u8)); + assert_eq!(format!("{}", zero), format!("{}", 0u8)); + assert_eq!(format!("{}", one), format!("{}", 1u8)); + } + + #[test] + #[allow(clippy::as_conversions)] + fn test_maybe_uninit() { + // int + { + let input = 42; + let uninit = MaybeUninit::new(input); + // SAFETY: `uninit` is in an initialized state + let output = unsafe { uninit.assume_init() }; + assert_eq!(input, output); + } + + // thin ref + { + let input = 42; + let uninit = MaybeUninit::new(&input); + // SAFETY: `uninit` is in an initialized state + let output = unsafe { uninit.assume_init() }; + assert_eq!(&input as *const _, output as *const _); + assert_eq!(input, *output); + } + + // wide ref + { + let input = [1, 2, 3, 4]; + let uninit = MaybeUninit::new(&input[..]); + // SAFETY: `uninit` is in an initialized state + let output = unsafe { uninit.assume_init() }; + assert_eq!(&input[..] as *const _, output as *const _); + assert_eq!(input, *output); + } + } + #[test] + fn test_maybe_uninit_uninit() { + let _uninit = MaybeUninit::<u8>::uninit(); + // Cannot check value, but can check it compiles and runs + } + + #[test] + #[cfg(feature = "alloc")] + fn test_maybe_uninit_new_boxed_uninit() { + let _boxed = MaybeUninit::<u8>::new_boxed_uninit(()).unwrap(); + } + + #[test] + fn test_maybe_uninit_debug() { + let uninit = MaybeUninit::<u8>::uninit(); + assert!(format!("{:?}", uninit).contains("MaybeUninit")); + } +} diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 3498d25b15e8..911745743246 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -329,6 +329,7 @@ rust_common_cmd = \ -Zcrate-attr=no_std \ -Zcrate-attr='feature($(rust_allowed_features))' \ -Zunstable-options --extern pin_init --extern kernel \ + --extern zerocopy --extern zerocopy_derive \ --crate-type rlib -L $(objtree)/rust/ \ --sysroot=/dev/null \ --out-dir $(dir $@) --emit=dep-info=$(depfile) diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py index d5f9a0ca742c..dc1219736f77 100755 --- a/scripts/generate_rust_analyzer.py +++ b/scripts/generate_rust_analyzer.py @@ -26,6 +26,14 @@ def args_crates_cfgs(cfgs: List[str]) -> Dict[str, List[str]]: return crates_cfgs +def args_crates_envs(envs: List[str]) -> Dict[str, Dict[str, str]]: + crates_envs = {} + for env in envs: + crate, vals = env.split("=", 1) + crates_envs[crate] = dict(v.split("=", 1) for v in vals.split()) + + return crates_envs + class Dependency(TypedDict): crate: int name: str @@ -61,6 +69,7 @@ def generate_crates( sysroot_src: pathlib.Path, external_src: Optional[pathlib.Path], cfgs: List[str], + envs: List[str], core_edition: str, ) -> List[Crate]: # Generate the configuration list. @@ -74,6 +83,7 @@ def generate_crates( # Now fill the crates list. crates: List[Crate] = [] crates_cfgs = args_crates_cfgs(cfgs) + crates_envs = args_crates_envs(envs) def get_crate_name(path: pathlib.Path) -> str: return invoke_rustc(["--print", "crate-name", str(path)]) @@ -92,6 +102,10 @@ def generate_crates( is_workspace_member if is_workspace_member is not None else True ) edition = edition if edition is not None else "2021" + crate_env = { + "RUST_MODFILE": "This is only for rust-analyzer", + **crates_envs.get(display_name, {}), + } return { "display_name": display_name, "root_module": str(root_module), @@ -99,9 +113,7 @@ def generate_crates( "deps": deps, "cfg": cfg, "edition": edition, - "env": { - "RUST_MODFILE": "This is only for rust-analyzer" - } + "env": crate_env, } def append_proc_macro_crate( @@ -240,6 +252,12 @@ def generate_crates( [std, proc_macro, proc_macro2, quote, syn], ) + zerocopy_derive = append_proc_macro_crate( + "zerocopy_derive", + srctree / "rust" / "zerocopy-derive" / "lib.rs", + [std, proc_macro, proc_macro2, quote, syn], + ) + build_error = append_crate( "build_error", srctree / "rust" / "build_error.rs", @@ -264,6 +282,12 @@ def generate_crates( [core, compiler_builtins], ) + zerocopy = append_crate( + "zerocopy", + srctree / "rust" / "zerocopy" / "src" / "lib.rs", + [core, compiler_builtins], + ) + def append_crate_with_generated( display_name: str, deps: List[Dependency], @@ -292,7 +316,7 @@ def generate_crates( bindings = append_crate_with_generated("bindings", [core, ffi, pin_init]) uapi = append_crate_with_generated("uapi", [core, ffi, pin_init]) kernel = append_crate_with_generated( - "kernel", [core, macros, build_error, pin_init, ffi, bindings, uapi] + "kernel", [core, macros, build_error, pin_init, ffi, bindings, uapi, zerocopy, zerocopy_derive] ) scripts = srctree / "scripts" @@ -337,7 +361,7 @@ def generate_crates( append_crate( crate_name, path, - [core, kernel, pin_init], + [core, kernel, pin_init, zerocopy, zerocopy_derive], cfg=generated_cfg, ) @@ -347,6 +371,7 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument('--verbose', '-v', action='store_true') parser.add_argument('--cfgs', action='append', default=[]) + parser.add_argument('--envs', action='append', default=[]) parser.add_argument("core_edition") parser.add_argument("srctree", type=pathlib.Path) parser.add_argument("objtree", type=pathlib.Path) @@ -357,6 +382,7 @@ def main() -> None: class Args(argparse.Namespace): verbose: bool cfgs: List[str] + envs: List[str] srctree: pathlib.Path objtree: pathlib.Path sysroot: pathlib.Path @@ -372,7 +398,7 @@ def main() -> None: ) rust_project = { - "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree, args.cfgs, args.core_edition), + "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree, args.cfgs, args.envs, args.core_edition), "sysroot": str(args.sysroot), } |