diff options
Diffstat (limited to 'rust')
104 files changed, 9252 insertions, 2162 deletions
diff --git a/rust/.kunitconfig b/rust/.kunitconfig new file mode 100644 index 000000000000..9e72a5ab03c9 --- /dev/null +++ b/rust/.kunitconfig @@ -0,0 +1,3 @@ +CONFIG_KUNIT=y +CONFIG_RUST=y +CONFIG_RUST_KERNEL_DOCTESTS=y diff --git a/rust/Makefile b/rust/Makefile index 43cd7f845a9a..d62b58d0a55c 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -11,11 +11,8 @@ always-$(CONFIG_RUST) += exports_core_generated.h obj-$(CONFIG_RUST) += helpers/helpers.o CFLAGS_REMOVE_helpers/helpers.o = -Wmissing-prototypes -Wmissing-declarations -always-$(CONFIG_RUST) += libmacros.so -no-clean-files += libmacros.so - always-$(CONFIG_RUST) += bindings/bindings_generated.rs bindings/bindings_helpers_generated.rs -obj-$(CONFIG_RUST) += bindings.o kernel.o +obj-$(CONFIG_RUST) += bindings.o pin_init.o kernel.o always-$(CONFIG_RUST) += exports_helpers_generated.h \ exports_bindings_generated.h exports_kernel_generated.h @@ -38,9 +35,17 @@ obj-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.o always-$(subst y,$(CONFIG_RUST),$(CONFIG_JUMP_LABEL)) += kernel/generated_arch_static_branch_asm.rs -# Avoids running `$(RUSTC)` for the sysroot when it may not be available. +# Avoids running `$(RUSTC)` when it may not be available. ifdef CONFIG_RUST +libmacros_name := $(shell MAKEFLAGS= $(RUSTC) --print file-names --crate-name macros --crate-type proc-macro - </dev/null) +libmacros_extension := $(patsubst libmacros.%,%,$(libmacros_name)) + +libpin_init_internal_name := $(shell MAKEFLAGS= $(RUSTC) --print file-names --crate-name pin_init_internal --crate-type proc-macro - </dev/null) +libpin_init_internal_extension := $(patsubst libpin_init_internal.%,%,$(libpin_init_internal_name)) + +always-$(CONFIG_RUST) += $(libmacros_name) $(libpin_init_internal_name) + # `$(rust_flags)` is passed in case the user added `--sysroot`. rustc_sysroot := $(shell MAKEFLAGS= $(RUSTC) $(rust_flags) --print sysroot) rustc_host_target := $(shell $(RUSTC) --version --verbose | grep -F 'host: ' | cut -d' ' -f2) @@ -55,10 +60,16 @@ endif core-cfgs = \ --cfg no_fp_fmt_parse +core-edition := $(if $(call rustc-min-version,108700),2024,2021) + +# `rustc` recognizes `--remap-path-prefix` since 1.26.0, but `rustdoc` only +# since Rust 1.81.0. Moreover, `rustdoc` ICEs on out-of-tree builds since Rust +# 1.82.0 (https://github.com/rust-lang/rust/issues/138520). Thus workaround both +# issues skipping the flag. The former also applies to `RUSTDOC TK`. quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< cmd_rustdoc = \ OBJTREE=$(abspath $(objtree)) \ - $(RUSTDOC) $(filter-out $(skip_flags),$(if $(rustdoc_host),$(rust_common_flags),$(rust_flags))) \ + $(RUSTDOC) $(filter-out $(skip_flags) --remap-path-prefix=%,$(if $(rustdoc_host),$(rust_common_flags),$(rust_flags))) \ $(rustc_target_flags) -L$(objtree)/$(obj) \ -Zunstable-options --generate-link-to-definition \ --output $(rustdoc_output) \ @@ -78,7 +89,7 @@ quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< # command-like flags to solve the issue. Meanwhile, we use the non-custom case # and then retouch the generated files. rustdoc: rustdoc-core rustdoc-macros rustdoc-compiler_builtins \ - rustdoc-kernel + rustdoc-kernel rustdoc-pin_init $(Q)cp $(srctree)/Documentation/images/logo.svg $(rustdoc_output)/static.files/ $(Q)cp $(srctree)/Documentation/images/COPYING-logo $(rustdoc_output)/static.files/ $(Q)find $(rustdoc_output) -name '*.html' -type f -print0 | xargs -0 sed -Ei \ @@ -97,8 +108,8 @@ rustdoc-macros: $(src)/macros/lib.rs FORCE # Starting with Rust 1.82.0, skipping `-Wrustdoc::unescaped_backticks` should # not be needed -- see https://github.com/rust-lang/rust/pull/128307. -rustdoc-core: private skip_flags = -Wrustdoc::unescaped_backticks -rustdoc-core: private rustc_target_flags = $(core-cfgs) +rustdoc-core: private skip_flags = --edition=2021 -Wrustdoc::unescaped_backticks +rustdoc-core: private rustc_target_flags = --edition=$(core-edition) $(core-cfgs) rustdoc-core: $(RUST_LIB_SRC)/core/src/lib.rs FORCE +$(call if_changed,rustdoc) @@ -108,18 +119,31 @@ rustdoc-compiler_builtins: $(src)/compiler_builtins.rs rustdoc-core FORCE rustdoc-ffi: $(src)/ffi.rs rustdoc-core FORCE +$(call if_changed,rustdoc) -rustdoc-kernel: private rustc_target_flags = --extern ffi \ - --extern build_error --extern macros=$(objtree)/$(obj)/libmacros.so \ +rustdoc-pin_init_internal: private rustdoc_host = yes +rustdoc-pin_init_internal: private rustc_target_flags = --cfg kernel \ + --extern proc_macro --crate-type proc-macro +rustdoc-pin_init_internal: $(src)/pin-init/internal/src/lib.rs FORCE + +$(call if_changed,rustdoc) + +rustdoc-pin_init: private rustdoc_host = yes +rustdoc-pin_init: private rustc_target_flags = --extern pin_init_internal \ + --extern macros --extern alloc --cfg kernel --cfg feature=\"alloc\" +rustdoc-pin_init: $(src)/pin-init/src/lib.rs rustdoc-pin_init_internal \ + rustdoc-macros FORCE + +$(call if_changed,rustdoc) + +rustdoc-kernel: private rustc_target_flags = --extern ffi --extern pin_init \ + --extern build_error --extern macros \ --extern bindings --extern uapi rustdoc-kernel: $(src)/kernel/lib.rs rustdoc-core rustdoc-ffi rustdoc-macros \ - rustdoc-compiler_builtins $(obj)/libmacros.so \ + rustdoc-pin_init rustdoc-compiler_builtins $(obj)/$(libmacros_name) \ $(obj)/bindings.o FORCE +$(call if_changed,rustdoc) -quiet_cmd_rustc_test_library = RUSTC TL $< +quiet_cmd_rustc_test_library = $(RUSTC_OR_CLIPPY_QUIET) TL $< cmd_rustc_test_library = \ OBJTREE=$(abspath $(objtree)) \ - $(RUSTC) $(rust_common_flags) \ + $(RUSTC_OR_CLIPPY) $(rust_common_flags) \ @$(objtree)/include/generated/rustc_cfg $(rustc_target_flags) \ --crate-type $(if $(rustc_test_library_proc),proc-macro,rlib) \ --out-dir $(objtree)/$(obj)/test --cfg testlib \ @@ -137,12 +161,24 @@ rusttestlib-macros: private rustc_test_library_proc = yes rusttestlib-macros: $(src)/macros/lib.rs FORCE +$(call if_changed,rustc_test_library) +rusttestlib-pin_init_internal: private rustc_target_flags = --cfg kernel \ + --extern proc_macro +rusttestlib-pin_init_internal: private rustc_test_library_proc = yes +rusttestlib-pin_init_internal: $(src)/pin-init/internal/src/lib.rs FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-pin_init: private rustc_target_flags = --extern pin_init_internal \ + --extern macros --cfg kernel +rusttestlib-pin_init: $(src)/pin-init/src/lib.rs rusttestlib-macros \ + rusttestlib-pin_init_internal $(obj)/$(libpin_init_internal_name) FORCE + +$(call if_changed,rustc_test_library) + rusttestlib-kernel: private rustc_target_flags = --extern ffi \ - --extern build_error --extern macros \ + --extern build_error --extern macros --extern pin_init \ --extern bindings --extern uapi -rusttestlib-kernel: $(src)/kernel/lib.rs \ - rusttestlib-bindings rusttestlib-uapi rusttestlib-build_error \ - $(obj)/libmacros.so $(obj)/bindings.o FORCE +rusttestlib-kernel: $(src)/kernel/lib.rs rusttestlib-bindings rusttestlib-uapi \ + rusttestlib-build_error rusttestlib-pin_init $(obj)/$(libmacros_name) \ + $(obj)/bindings.o FORCE +$(call if_changed,rustc_test_library) rusttestlib-bindings: private rustc_target_flags = --extern ffi @@ -169,9 +205,9 @@ quiet_cmd_rustdoc_test_kernel = RUSTDOC TK $< rm -rf $(objtree)/$(obj)/test/doctests/kernel; \ mkdir -p $(objtree)/$(obj)/test/doctests/kernel; \ OBJTREE=$(abspath $(objtree)) \ - $(RUSTDOC) --test $(rust_flags) \ - -L$(objtree)/$(obj) --extern ffi --extern kernel \ - --extern build_error --extern macros \ + $(RUSTDOC) --test $(filter-out --remap-path-prefix=%,$(rust_flags)) \ + -L$(objtree)/$(obj) --extern ffi --extern pin_init \ + --extern kernel --extern build_error --extern macros \ --extern bindings --extern uapi \ --no-run --crate-name kernel -Zunstable-options \ --sysroot=/dev/null \ @@ -187,10 +223,10 @@ quiet_cmd_rustdoc_test_kernel = RUSTDOC TK $< # We cannot use `-Zpanic-abort-tests` because some tests are dynamic, # so for the moment we skip `-Cpanic=abort`. -quiet_cmd_rustc_test = RUSTC T $< +quiet_cmd_rustc_test = $(RUSTC_OR_CLIPPY_QUIET) T $< cmd_rustc_test = \ OBJTREE=$(abspath $(objtree)) \ - $(RUSTC) --test $(rust_common_flags) \ + $(RUSTC_OR_CLIPPY) --test $(rust_common_flags) \ @$(objtree)/include/generated/rustc_cfg \ $(rustc_target_flags) --out-dir $(objtree)/$(obj)/test \ -L$(objtree)/$(obj)/test \ @@ -201,18 +237,18 @@ quiet_cmd_rustc_test = RUSTC T $< rusttest: rusttest-macros rusttest-kernel rusttest-macros: private rustc_target_flags = --extern proc_macro \ - --extern macros --extern kernel + --extern macros --extern kernel --extern pin_init rusttest-macros: private rustdoc_test_target_flags = --crate-type proc-macro rusttest-macros: $(src)/macros/lib.rs \ - rusttestlib-macros rusttestlib-kernel FORCE + rusttestlib-macros rusttestlib-kernel rusttestlib-pin_init FORCE +$(call if_changed,rustc_test) +$(call if_changed,rustdoc_test) -rusttest-kernel: private rustc_target_flags = --extern ffi \ +rusttest-kernel: private rustc_target_flags = --extern ffi --extern pin_init \ --extern build_error --extern macros --extern bindings --extern uapi rusttest-kernel: $(src)/kernel/lib.rs rusttestlib-ffi rusttestlib-kernel \ rusttestlib-build_error rusttestlib-macros rusttestlib-bindings \ - rusttestlib-uapi FORCE + rusttestlib-uapi rusttestlib-pin_init FORCE +$(call if_changed,rustc_test) ifdef CONFIG_CC_IS_CLANG @@ -230,7 +266,8 @@ bindgen_skip_c_flags := -mno-fp-ret-in-387 -mpreferred-stack-boundary=% \ -mfunction-return=thunk-extern -mrecord-mcount -mabi=lp64 \ -mindirect-branch-cs-prefix -mstack-protector-guard% -mtraceback=no \ -mno-pointers-to-nested-functions -mno-string \ - -mno-strict-align -mstrict-align \ + -mno-strict-align -mstrict-align -mdirect-extern-access \ + -mexplicit-relocs -mno-check-zero-division \ -fconserve-stack -falign-jumps=% -falign-loops=% \ -femit-struct-debug-baseonly -fno-ipa-cp-clone -fno-ipa-sra \ -fno-partial-inlining -fplugin-arg-arm_ssp_per_task_plugin-% \ @@ -238,12 +275,15 @@ bindgen_skip_c_flags := -mno-fp-ret-in-387 -mpreferred-stack-boundary=% \ -fzero-call-used-regs=% -fno-stack-clash-protection \ -fno-inline-functions-called-once -fsanitize=bounds-strict \ -fstrict-flex-arrays=% -fmin-function-alignment=% \ - -fzero-init-padding-bits=% \ + -fzero-init-padding-bits=% -mno-fdpic \ --param=% --param asan-% # Derived from `scripts/Makefile.clang`. BINDGEN_TARGET_x86 := x86_64-linux-gnu BINDGEN_TARGET_arm64 := aarch64-linux-gnu +BINDGEN_TARGET_arm := arm-linux-gnueabi +BINDGEN_TARGET_loongarch := loongarch64-linux-gnusf +BINDGEN_TARGET_um := $(BINDGEN_TARGET_$(SUBARCH)) BINDGEN_TARGET := $(BINDGEN_TARGET_$(SRCARCH)) # All warnings are inhibited since GCC builds are very experimental, @@ -330,10 +370,11 @@ $(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_extra = ; $(obj)/bindings/bindings_helpers_generated.rs: $(src)/helpers/helpers.c FORCE $(call if_changed_dep,bindgen) +rust_exports = $(NM) -p --defined-only $(1) | awk '$$2~/(T|R|D|B)/ && $$3!~/__(pfx|cfi|odr_asan)/ { printf $(2),$$3 }' + quiet_cmd_exports = EXPORTS $@ cmd_exports = \ - $(NM) -p --defined-only $< \ - | awk '$$2~/(T|R|D|B)/ && $$3!~/__cfi/ {printf "EXPORT_SYMBOL_RUST_GPL(%s);\n",$$3}' > $@ + $(call rust_exports,$<,"EXPORT_SYMBOL_RUST_GPL(%s);\n") > $@ $(obj)/exports_core_generated.h: $(obj)/core.o FORCE $(call if_changed,exports) @@ -358,22 +399,26 @@ $(obj)/exports_kernel_generated.h: $(obj)/kernel.o FORCE quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@ cmd_rustc_procmacro = \ - $(RUSTC_OR_CLIPPY) $(rust_common_flags) \ + $(RUSTC_OR_CLIPPY) $(rust_common_flags) $(rustc_target_flags) \ -Clinker-flavor=gcc -Clinker=$(HOSTCC) \ - -Clink-args='$(call escsq,$(KBUILD_HOSTLDFLAGS))' \ + -Clink-args='$(call escsq,$(KBUILD_PROCMACROLDFLAGS))' \ --emit=dep-info=$(depfile) --emit=link=$@ --extern proc_macro \ --crate-type proc-macro \ - --crate-name $(patsubst lib%.so,%,$(notdir $@)) $< + --crate-name $(patsubst lib%.$(libmacros_extension),%,$(notdir $@)) $< # Procedural macros can only be used with the `rustc` that compiled it. -$(obj)/libmacros.so: $(src)/macros/lib.rs FORCE +$(obj)/$(libmacros_name): $(src)/macros/lib.rs FORCE + +$(call if_changed_dep,rustc_procmacro) + +$(obj)/$(libpin_init_internal_name): private rustc_target_flags = --cfg kernel +$(obj)/$(libpin_init_internal_name): $(src)/pin-init/internal/src/lib.rs FORCE +$(call if_changed_dep,rustc_procmacro) quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L $@ cmd_rustc_library = \ OBJTREE=$(abspath $(objtree)) \ $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \ - $(filter-out $(skip_flags),$(rust_flags) $(rustc_target_flags)) \ + $(filter-out $(skip_flags),$(rust_flags)) $(rustc_target_flags) \ --emit=dep-info=$(depfile) --emit=obj=$@ \ --emit=metadata=$(dir $@)$(patsubst %.o,lib%.rmeta,$(notdir $@)) \ --crate-type rlib -L$(objtree)/$(obj) \ @@ -383,8 +428,8 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L $(cmd_objtool) rust-analyzer: - $(Q)$(srctree)/scripts/generate_rust_analyzer.py \ - --cfgs='core=$(core-cfgs)' \ + $(Q)MAKEFLAGS= $(srctree)/scripts/generate_rust_analyzer.py \ + --cfgs='core=$(core-cfgs)' $(core-edition) \ $(realpath $(srctree)) $(realpath $(objtree)) \ $(rustc_sysroot) $(RUST_LIB_SRC) $(if $(KBUILD_EXTMOD),$(srcroot)) \ > rust-project.json @@ -395,6 +440,13 @@ redirect-intrinsics = \ __muloti4 __multi3 \ __udivmodti4 __udivti3 __umodti3 +ifdef CONFIG_ARM + # Add eabi initrinsics for ARM 32-bit + redirect-intrinsics += \ + __aeabi_fadd __aeabi_fmul __aeabi_fcmpeq __aeabi_fcmple __aeabi_fcmplt __aeabi_fcmpun \ + __aeabi_dadd __aeabi_dmul __aeabi_dcmple __aeabi_dcmplt __aeabi_dcmpun \ + __aeabi_uldivmod +endif ifneq ($(or $(CONFIG_ARM64),$(and $(CONFIG_RISCV),$(CONFIG_64BIT))),) # These intrinsics are defined for ARM64 and RISCV64 redirect-intrinsics += \ @@ -402,15 +454,40 @@ ifneq ($(or $(CONFIG_ARM64),$(and $(CONFIG_RISCV),$(CONFIG_64BIT))),) __ashlti3 __lshrti3 endif +ifdef CONFIG_MODVERSIONS +cmd_gendwarfksyms = $(if $(skip_gendwarfksyms),, \ + $(call rust_exports,$@,"%s\n") | \ + scripts/gendwarfksyms/gendwarfksyms \ + $(if $(KBUILD_GENDWARFKSYMS_STABLE), --stable) \ + $(if $(KBUILD_SYMTYPES), --symtypes $(@:.o=.symtypes),) \ + $@ >> $(dot-target).cmd) +endif + define rule_rustc_library $(call cmd_and_fixdep,rustc_library) $(call cmd,gen_objtooldep) + $(call cmd,gendwarfksyms) +endef + +define rule_rust_cc_library + $(call if_changed_rule,cc_o_c) + $(call cmd,force_checksrc) + $(call cmd,gendwarfksyms) endef +# helpers.o uses the same export mechanism as Rust libraries, so ensure symbol +# versions are calculated for the helpers too. +$(obj)/helpers/helpers.o: $(src)/helpers/helpers.c $(recordmcount_source) FORCE + +$(call if_changed_rule,rust_cc_library) + +# Disable symbol versioning for exports.o to avoid conflicts with the actual +# symbol versions generated from Rust objects. +$(obj)/exports.o: private skip_gendwarfksyms = 1 + $(obj)/core.o: private skip_clippy = 1 -$(obj)/core.o: private skip_flags = -Wunreachable_pub +$(obj)/core.o: private skip_flags = --edition=2021 -Wunreachable_pub $(obj)/core.o: private rustc_objcopy = $(foreach sym,$(redirect-intrinsics),--redefine-sym $(sym)=__rust$(sym)) -$(obj)/core.o: private rustc_target_flags = $(core-cfgs) +$(obj)/core.o: private rustc_target_flags = --edition=$(core-edition) $(core-cfgs) $(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs \ $(wildcard $(objtree)/include/config/RUSTC_VERSION_TEXT) FORCE +$(call if_changed_rule,rustc_library) @@ -418,13 +495,23 @@ ifneq ($(or $(CONFIG_X86_64),$(CONFIG_X86_32)),) $(obj)/core.o: scripts/target.json endif +$(obj)/compiler_builtins.o: private skip_gendwarfksyms = 1 $(obj)/compiler_builtins.o: private rustc_objcopy = -w -W '__*' $(obj)/compiler_builtins.o: $(src)/compiler_builtins.rs $(obj)/core.o FORCE +$(call if_changed_rule,rustc_library) +$(obj)/pin_init.o: private skip_gendwarfksyms = 1 +$(obj)/pin_init.o: private rustc_target_flags = --extern pin_init_internal \ + --extern macros --cfg kernel +$(obj)/pin_init.o: $(src)/pin-init/src/lib.rs $(obj)/compiler_builtins.o \ + $(obj)/$(libpin_init_internal_name) $(obj)/$(libmacros_name) FORCE + +$(call if_changed_rule,rustc_library) + +$(obj)/build_error.o: private skip_gendwarfksyms = 1 $(obj)/build_error.o: $(src)/build_error.rs $(obj)/compiler_builtins.o FORCE +$(call if_changed_rule,rustc_library) +$(obj)/ffi.o: private skip_gendwarfksyms = 1 $(obj)/ffi.o: $(src)/ffi.rs $(obj)/compiler_builtins.o FORCE +$(call if_changed_rule,rustc_library) @@ -436,15 +523,16 @@ $(obj)/bindings.o: $(src)/bindings/lib.rs \ +$(call if_changed_rule,rustc_library) $(obj)/uapi.o: private rustc_target_flags = --extern ffi +$(obj)/uapi.o: private skip_gendwarfksyms = 1 $(obj)/uapi.o: $(src)/uapi/lib.rs \ $(obj)/ffi.o \ $(obj)/uapi/uapi_generated.rs FORCE +$(call if_changed_rule,rustc_library) -$(obj)/kernel.o: private rustc_target_flags = --extern ffi \ +$(obj)/kernel.o: private rustc_target_flags = --extern ffi --extern pin_init \ --extern build_error --extern macros --extern bindings --extern uapi -$(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/build_error.o \ - $(obj)/libmacros.so $(obj)/bindings.o $(obj)/uapi.o FORCE +$(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/build_error.o $(obj)/pin_init.o \ + $(obj)/$(libmacros_name) $(obj)/bindings.o $(obj)/uapi.o FORCE +$(call if_changed_rule,rustc_library) ifdef CONFIG_JUMP_LABEL diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 5c4dfe22f41a..3f66570b8756 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -10,7 +10,11 @@ #include <linux/blk-mq.h> #include <linux/blk_types.h> #include <linux/blkdev.h> +#include <linux/completion.h> +#include <linux/cpumask.h> #include <linux/cred.h> +#include <linux/device/faux.h> +#include <linux/dma-mapping.h> #include <linux/errname.h> #include <linux/ethtool.h> #include <linux/file.h> @@ -20,9 +24,13 @@ #include <linux/jump_label.h> #include <linux/mdio.h> #include <linux/miscdevice.h> +#include <linux/of_device.h> +#include <linux/pci.h> #include <linux/phy.h> #include <linux/pid_namespace.h> +#include <linux/platform_device.h> #include <linux/poll.h> +#include <linux/property.h> #include <linux/refcount.h> #include <linux/sched.h> #include <linux/security.h> @@ -32,6 +40,11 @@ #include <linux/workqueue.h> #include <trace/events/rust_sample.h> +#if defined(CONFIG_DRM_PANIC_SCREEN_QR_CODE) +// Used by `#[export]` in `drivers/gpu/drm/drm_panic_qr.rs`. +#include <drm/drm_panic.h> +#endif + /* `bindgen` gets confused at certain things. */ const size_t RUST_CONST_HELPER_ARCH_SLAB_MINALIGN = ARCH_SLAB_MINALIGN; const size_t RUST_CONST_HELPER_PAGE_SIZE = PAGE_SIZE; diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs index 014af0d1fc70..a08eb5518cac 100644 --- a/rust/bindings/lib.rs +++ b/rust/bindings/lib.rs @@ -26,6 +26,7 @@ #[allow(dead_code)] #[allow(clippy::undocumented_unsafe_blocks)] +#[cfg_attr(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES, allow(unnecessary_transmutes))] mod bindings_raw { // Manual definition for blocklisted types. type __kernel_size_t = usize; diff --git a/rust/compiler_builtins.rs b/rust/compiler_builtins.rs index f14b8d7caf89..dd16c1dc899c 100644 --- a/rust/compiler_builtins.rs +++ b/rust/compiler_builtins.rs @@ -73,5 +73,29 @@ define_panicking_intrinsics!("`u128` should not be used", { __umodti3, }); +#[cfg(target_arch = "arm")] +define_panicking_intrinsics!("`f32` should not be used", { + __aeabi_fadd, + __aeabi_fmul, + __aeabi_fcmpeq, + __aeabi_fcmple, + __aeabi_fcmplt, + __aeabi_fcmpun, +}); + +#[cfg(target_arch = "arm")] +define_panicking_intrinsics!("`f64` should not be used", { + __aeabi_dadd, + __aeabi_dmul, + __aeabi_dcmple, + __aeabi_dcmplt, + __aeabi_dcmpun, +}); + +#[cfg(target_arch = "arm")] +define_panicking_intrinsics!("`u64` division/modulo should not be used", { + __aeabi_uldivmod, +}); + // NOTE: if you are adding a new intrinsic here, you should also add it to // `redirect-intrinsics` in `rust/Makefile`. diff --git a/rust/helpers/completion.c b/rust/helpers/completion.c new file mode 100644 index 000000000000..b2443262a2ae --- /dev/null +++ b/rust/helpers/completion.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/completion.h> + +void rust_helper_init_completion(struct completion *x) +{ + init_completion(x); +} diff --git a/rust/helpers/cpumask.c b/rust/helpers/cpumask.c new file mode 100644 index 000000000000..2d380a86c34a --- /dev/null +++ b/rust/helpers/cpumask.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/cpumask.h> + +void rust_helper_cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) +{ + cpumask_set_cpu(cpu, dstp); +} + +void rust_helper_cpumask_clear_cpu(int cpu, struct cpumask *dstp) +{ + cpumask_clear_cpu(cpu, dstp); +} + +void rust_helper_cpumask_setall(struct cpumask *dstp) +{ + cpumask_setall(dstp); +} + +unsigned int rust_helper_cpumask_weight(struct cpumask *srcp) +{ + return cpumask_weight(srcp); +} + +void rust_helper_cpumask_copy(struct cpumask *dstp, const struct cpumask *srcp) +{ + cpumask_copy(dstp, srcp); +} + +bool rust_helper_alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) +{ + return alloc_cpumask_var(mask, flags); +} + +bool rust_helper_zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) +{ + return zalloc_cpumask_var(mask, flags); +} + +#ifndef CONFIG_CPUMASK_OFFSTACK +void rust_helper_free_cpumask_var(cpumask_var_t mask) +{ + free_cpumask_var(mask); +} +#endif diff --git a/rust/helpers/device.c b/rust/helpers/device.c new file mode 100644 index 000000000000..b2135c6686b0 --- /dev/null +++ b/rust/helpers/device.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/device.h> + +int rust_helper_devm_add_action(struct device *dev, + void (*action)(void *), + void *data) +{ + return devm_add_action(dev, action, data); +} diff --git a/rust/helpers/dma.c b/rust/helpers/dma.c new file mode 100644 index 000000000000..df8b8a77355a --- /dev/null +++ b/rust/helpers/dma.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/dma-mapping.h> + +void *rust_helper_dma_alloc_attrs(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t flag, + unsigned long attrs) +{ + return dma_alloc_attrs(dev, size, dma_handle, flag, attrs); +} + +void rust_helper_dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t dma_handle, unsigned long attrs) +{ + dma_free_attrs(dev, size, cpu_addr, dma_handle, attrs); +} diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index dcf827a61b52..97cb759d92d4 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -11,20 +11,29 @@ #include "bug.c" #include "build_assert.c" #include "build_bug.c" +#include "completion.c" +#include "cpumask.c" #include "cred.c" +#include "device.c" +#include "dma.c" #include "err.c" #include "fs.c" +#include "io.c" #include "jump_label.c" #include "kunit.c" #include "mutex.c" #include "page.c" +#include "platform.c" +#include "pci.c" #include "pid_namespace.c" #include "rbtree.c" +#include "rcu.c" #include "refcount.c" #include "security.c" #include "signal.c" #include "slab.c" #include "spinlock.c" +#include "sync.c" #include "task.c" #include "uaccess.c" #include "vmalloc.c" diff --git a/rust/helpers/io.c b/rust/helpers/io.c new file mode 100644 index 000000000000..15ea187c5466 --- /dev/null +++ b/rust/helpers/io.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/io.h> + +void __iomem *rust_helper_ioremap(phys_addr_t offset, size_t size) +{ + return ioremap(offset, size); +} + +void rust_helper_iounmap(void __iomem *addr) +{ + iounmap(addr); +} + +u8 rust_helper_readb(const void __iomem *addr) +{ + return readb(addr); +} + +u16 rust_helper_readw(const void __iomem *addr) +{ + return readw(addr); +} + +u32 rust_helper_readl(const void __iomem *addr) +{ + return readl(addr); +} + +#ifdef CONFIG_64BIT +u64 rust_helper_readq(const void __iomem *addr) +{ + return readq(addr); +} +#endif + +void rust_helper_writeb(u8 value, void __iomem *addr) +{ + writeb(value, addr); +} + +void rust_helper_writew(u16 value, void __iomem *addr) +{ + writew(value, addr); +} + +void rust_helper_writel(u32 value, void __iomem *addr) +{ + writel(value, addr); +} + +#ifdef CONFIG_64BIT +void rust_helper_writeq(u64 value, void __iomem *addr) +{ + writeq(value, addr); +} +#endif + +u8 rust_helper_readb_relaxed(const void __iomem *addr) +{ + return readb_relaxed(addr); +} + +u16 rust_helper_readw_relaxed(const void __iomem *addr) +{ + return readw_relaxed(addr); +} + +u32 rust_helper_readl_relaxed(const void __iomem *addr) +{ + return readl_relaxed(addr); +} + +#ifdef CONFIG_64BIT +u64 rust_helper_readq_relaxed(const void __iomem *addr) +{ + return readq_relaxed(addr); +} +#endif + +void rust_helper_writeb_relaxed(u8 value, void __iomem *addr) +{ + writeb_relaxed(value, addr); +} + +void rust_helper_writew_relaxed(u16 value, void __iomem *addr) +{ + writew_relaxed(value, addr); +} + +void rust_helper_writel_relaxed(u32 value, void __iomem *addr) +{ + writel_relaxed(value, addr); +} + +#ifdef CONFIG_64BIT +void rust_helper_writeq_relaxed(u64 value, void __iomem *addr) +{ + writeq_relaxed(value, addr); +} +#endif diff --git a/rust/helpers/mutex.c b/rust/helpers/mutex.c index 7e00680958ef..06575553eda5 100644 --- a/rust/helpers/mutex.c +++ b/rust/helpers/mutex.c @@ -12,3 +12,8 @@ void rust_helper___mutex_init(struct mutex *mutex, const char *name, { __mutex_init(mutex, name, key); } + +void rust_helper_mutex_assert_is_held(struct mutex *mutex) +{ + lockdep_assert_held(mutex); +} diff --git a/rust/helpers/pci.c b/rust/helpers/pci.c new file mode 100644 index 000000000000..8ba22f911459 --- /dev/null +++ b/rust/helpers/pci.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/pci.h> + +void rust_helper_pci_set_drvdata(struct pci_dev *pdev, void *data) +{ + pci_set_drvdata(pdev, data); +} + +void *rust_helper_pci_get_drvdata(struct pci_dev *pdev) +{ + return pci_get_drvdata(pdev); +} + +resource_size_t rust_helper_pci_resource_len(struct pci_dev *pdev, int bar) +{ + return pci_resource_len(pdev, bar); +} diff --git a/rust/helpers/platform.c b/rust/helpers/platform.c new file mode 100644 index 000000000000..ab9b9f317301 --- /dev/null +++ b/rust/helpers/platform.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/platform_device.h> + +void *rust_helper_platform_get_drvdata(const struct platform_device *pdev) +{ + return platform_get_drvdata(pdev); +} + +void rust_helper_platform_set_drvdata(struct platform_device *pdev, void *data) +{ + platform_set_drvdata(pdev, data); +} diff --git a/rust/helpers/rcu.c b/rust/helpers/rcu.c new file mode 100644 index 000000000000..f1cec6583513 --- /dev/null +++ b/rust/helpers/rcu.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/rcupdate.h> + +void rust_helper_rcu_read_lock(void) +{ + rcu_read_lock(); +} + +void rust_helper_rcu_read_unlock(void) +{ + rcu_read_unlock(); +} diff --git a/rust/helpers/security.c b/rust/helpers/security.c index 239e5b4745fe..0c4c2065df28 100644 --- a/rust/helpers/security.c +++ b/rust/helpers/security.c @@ -8,13 +8,13 @@ void rust_helper_security_cred_getsecid(const struct cred *c, u32 *secid) security_cred_getsecid(c, secid); } -int rust_helper_security_secid_to_secctx(u32 secid, char **secdata, u32 *seclen) +int rust_helper_security_secid_to_secctx(u32 secid, struct lsm_context *cp) { - return security_secid_to_secctx(secid, secdata, seclen); + return security_secid_to_secctx(secid, cp); } -void rust_helper_security_release_secctx(char *secdata, u32 seclen) +void rust_helper_security_release_secctx(struct lsm_context *cp) { - security_release_secctx(secdata, seclen); + security_release_secctx(cp); } #endif diff --git a/rust/helpers/spinlock.c b/rust/helpers/spinlock.c index 5971fdf6f755..42c4bf01a23e 100644 --- a/rust/helpers/spinlock.c +++ b/rust/helpers/spinlock.c @@ -30,3 +30,8 @@ int rust_helper_spin_trylock(spinlock_t *lock) { return spin_trylock(lock); } + +void rust_helper_spin_assert_is_held(spinlock_t *lock) +{ + lockdep_assert_held(lock); +} diff --git a/rust/helpers/sync.c b/rust/helpers/sync.c new file mode 100644 index 000000000000..ff7e68b48810 --- /dev/null +++ b/rust/helpers/sync.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/lockdep.h> + +void rust_helper_lockdep_register_key(struct lock_class_key *k) +{ + lockdep_register_key(k); +} + +void rust_helper_lockdep_unregister_key(struct lock_class_key *k) +{ + lockdep_unregister_key(k); +} diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index f2f7f3a53d29..fc9c9c41cd79 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -123,7 +123,7 @@ pub mod flags { /// [`Allocator`] is designed to be implemented as a ZST; [`Allocator`] functions do not operate on /// an object instance. /// -/// In order to be able to support `#[derive(SmartPointer)]` later on, we need to avoid a design +/// In order to be able to support `#[derive(CoercePointee)]` later on, we need to avoid a design /// that requires an `Allocator` to be instantiated, hence its functions must not contain any kind /// of `self` parameter. /// diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 439985e29fbc..aa2dfa9dca4c 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -80,6 +80,7 @@ impl ReallocFunc { /// This method has the same guarantees as `Allocator::realloc`. Additionally /// - it accepts any pointer to a valid memory allocation allocated by this function. /// - memory allocated by this function remains valid until it is passed to this function. + #[inline] unsafe fn call( &self, ptr: Option<NonNull<u8>>, diff --git a/rust/kernel/alloc/allocator_test.rs b/rust/kernel/alloc/allocator_test.rs index e3240d16040b..c37d4c0c64e9 100644 --- a/rust/kernel/alloc/allocator_test.rs +++ b/rust/kernel/alloc/allocator_test.rs @@ -62,6 +62,24 @@ unsafe impl Allocator for Cmalloc { )); } + // ISO C (ISO/IEC 9899:2011) defines `aligned_alloc`: + // + // > The value of alignment shall be a valid alignment supported by the implementation + // [...]. + // + // As an example of the "supported by the implementation" requirement, POSIX.1-2001 (IEEE + // 1003.1-2001) defines `posix_memalign`: + // + // > The value of alignment shall be a power of two multiple of sizeof (void *). + // + // and POSIX-based implementations of `aligned_alloc` inherit this requirement. At the time + // of writing, this is known to be the case on macOS (but not in glibc). + // + // Satisfy the stricter requirement to avoid spurious test failures on some platforms. + let min_align = core::mem::size_of::<*const crate::ffi::c_void>(); + let layout = layout.align_to(min_align).map_err(|_| AllocError)?; + let layout = layout.pad_to_align(); + // SAFETY: Returns either NULL or a pointer to a memory allocation that satisfies or // exceeds the given size and alignment requirements. let dst = unsafe { libc_aligned_alloc(layout.align(), layout.size()) } as *mut u8; diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index 9ce414361c2c..b77d32f3a58b 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -15,8 +15,9 @@ use core::pin::Pin; use core::ptr::NonNull; use core::result::Result; -use crate::init::{InPlaceInit, InPlaceWrite, Init, PinInit}; +use crate::init::InPlaceInit; use crate::types::ForeignOwnable; +use pin_init::{InPlaceWrite, Init, PinInit, ZeroableOption}; /// The kernel's [`Box`] type -- a heap allocation for a single value of type `T`. /// @@ -99,6 +100,10 @@ pub type VBox<T> = Box<T, super::allocator::Vmalloc>; /// ``` pub type KVBox<T> = Box<T, super::allocator::KVmalloc>; +// SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee: +// https://doc.rust-lang.org/stable/std/option/index.html#representation). +unsafe impl<T, A: Allocator> ZeroableOption for Box<T, A> {} + // SAFETY: `Box` is `Send` if `T` is `Send` because the `Box` owns a `T`. unsafe impl<T, A> Send for Box<T, A> where @@ -245,6 +250,12 @@ where Ok(Self::new(x, flags)?.into()) } + /// Convert a [`Box<T,A>`] to a [`Pin<Box<T,A>>`]. If `T` does not implement + /// [`Unpin`], then `x` will be pinned in memory and can't be moved. + pub fn into_pin(this: Self) -> Pin<Self> { + this.into() + } + /// Forgets the contents (does not run the destructor), but keeps the allocation. fn forget_contents(this: Self) -> Box<MaybeUninit<T>, A> { let ptr = Self::into_raw(this); @@ -354,22 +365,30 @@ where A: Allocator, { type Borrowed<'a> = &'a T; + type BorrowedMut<'a> = &'a mut T; - fn into_foreign(self) -> *const crate::ffi::c_void { - Box::into_raw(self) as _ + fn into_foreign(self) -> *mut crate::ffi::c_void { + Box::into_raw(self).cast() } - unsafe fn from_foreign(ptr: *const crate::ffi::c_void) -> Self { + unsafe fn from_foreign(ptr: *mut crate::ffi::c_void) -> Self { // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous // call to `Self::into_foreign`. - unsafe { Box::from_raw(ptr as _) } + unsafe { Box::from_raw(ptr.cast()) } } - unsafe fn borrow<'a>(ptr: *const crate::ffi::c_void) -> &'a T { + unsafe fn borrow<'a>(ptr: *mut crate::ffi::c_void) -> &'a T { // SAFETY: The safety requirements of this method ensure that the object remains alive and // immutable for the duration of 'a. unsafe { &*ptr.cast() } } + + unsafe fn borrow_mut<'a>(ptr: *mut crate::ffi::c_void) -> &'a mut T { + let ptr = ptr.cast(); + // SAFETY: The safety requirements of this method ensure that the pointer is valid and that + // nothing else will access the value for the duration of 'a. + unsafe { &mut *ptr } + } } impl<T: 'static, A> ForeignOwnable for Pin<Box<T, A>> @@ -377,19 +396,20 @@ where A: Allocator, { type Borrowed<'a> = Pin<&'a T>; + type BorrowedMut<'a> = Pin<&'a mut T>; - fn into_foreign(self) -> *const crate::ffi::c_void { + fn into_foreign(self) -> *mut crate::ffi::c_void { // SAFETY: We are still treating the box as pinned. - Box::into_raw(unsafe { Pin::into_inner_unchecked(self) }) as _ + Box::into_raw(unsafe { Pin::into_inner_unchecked(self) }).cast() } - unsafe fn from_foreign(ptr: *const crate::ffi::c_void) -> Self { + unsafe fn from_foreign(ptr: *mut crate::ffi::c_void) -> Self { // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous // call to `Self::into_foreign`. - unsafe { Pin::new_unchecked(Box::from_raw(ptr as _)) } + unsafe { Pin::new_unchecked(Box::from_raw(ptr.cast())) } } - unsafe fn borrow<'a>(ptr: *const crate::ffi::c_void) -> Pin<&'a T> { + unsafe fn borrow<'a>(ptr: *mut crate::ffi::c_void) -> Pin<&'a T> { // SAFETY: The safety requirements for this function ensure that the object is still alive, // so it is safe to dereference the raw pointer. // The safety requirements of `from_foreign` also ensure that the object remains alive for @@ -399,6 +419,18 @@ where // SAFETY: This pointer originates from a `Pin<Box<T>>`. unsafe { Pin::new_unchecked(r) } } + + unsafe fn borrow_mut<'a>(ptr: *mut crate::ffi::c_void) -> Pin<&'a mut T> { + let ptr = ptr.cast(); + // SAFETY: The safety requirements for this function ensure that the object is still alive, + // so it is safe to dereference the raw pointer. + // The safety requirements of `from_foreign` also ensure that the object remains alive for + // the lifetime of the returned value. + let r = unsafe { &mut *ptr }; + + // SAFETY: This pointer originates from a `Pin<Box<T>>`. + unsafe { Pin::new_unchecked(r) } + } } impl<T, A> Deref for Box<T, A> @@ -427,13 +459,23 @@ where } } +impl<T, A> fmt::Display for Box<T, A> +where + T: ?Sized + fmt::Display, + A: Allocator, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + <T as fmt::Display>::fmt(&**self, f) + } +} + impl<T, A> fmt::Debug for Box<T, A> where T: ?Sized + fmt::Debug, A: Allocator, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&**self, f) + <T as fmt::Debug>::fmt(&**self, f) } } diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index ae9d072741ce..f62204fe563f 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -2,6 +2,9 @@ //! Implementation of [`Vec`]. +// May not be needed in Rust 1.87.0 (pending beta backport). +#![allow(clippy::ptr_eq)] + use super::{ allocator::{KVmalloc, Kmalloc, Vmalloc}, layout::ArrayLayout, @@ -193,6 +196,9 @@ where #[inline] pub unsafe fn set_len(&mut self, new_len: usize) { debug_assert!(new_len <= self.capacity()); + + // INVARIANT: By the safety requirements of this method `new_len` represents the exact + // number of elements stored within `self`. self.len = new_len; } diff --git a/rust/kernel/alloc/layout.rs b/rust/kernel/alloc/layout.rs index 4b3cd7fdc816..93ed514f7cc7 100644 --- a/rust/kernel/alloc/layout.rs +++ b/rust/kernel/alloc/layout.rs @@ -43,6 +43,25 @@ impl<T> ArrayLayout<T> { /// # Errors /// /// When `len * size_of::<T>()` overflows or when `len * size_of::<T>() > isize::MAX`. + /// + /// # Examples + /// + /// ``` + /// # use kernel::alloc::layout::{ArrayLayout, LayoutError}; + /// let layout = ArrayLayout::<i32>::new(15)?; + /// assert_eq!(layout.len(), 15); + /// + /// // Errors because `len * size_of::<T>()` overflows. + /// let layout = ArrayLayout::<i32>::new(isize::MAX as usize); + /// assert!(layout.is_err()); + /// + /// // Errors because `len * size_of::<i32>() > isize::MAX`, + /// // even though `len < isize::MAX`. + /// let layout = ArrayLayout::<i32>::new(isize::MAX as usize / 2); + /// assert!(layout.is_err()); + /// + /// # Ok::<(), Error>(()) + /// ``` pub const fn new(len: usize) -> Result<Self, LayoutError> { match len.checked_mul(core::mem::size_of::<T>()) { Some(size) if size <= ISIZE_MAX => { diff --git a/rust/kernel/block/mq/gen_disk.rs b/rust/kernel/block/mq/gen_disk.rs index 798c4ae0bded..14806e1997fd 100644 --- a/rust/kernel/block/mq/gen_disk.rs +++ b/rust/kernel/block/mq/gen_disk.rs @@ -174,9 +174,9 @@ impl GenDiskBuilder { /// /// # Invariants /// -/// - `gendisk` must always point to an initialized and valid `struct gendisk`. -/// - `gendisk` was added to the VFS through a call to -/// `bindings::device_add_disk`. +/// - `gendisk` must always point to an initialized and valid `struct gendisk`. +/// - `gendisk` was added to the VFS through a call to +/// `bindings::device_add_disk`. pub struct GenDisk<T: Operations> { _tagset: Arc<TagSet<T>>, gendisk: *mut bindings::gendisk, diff --git a/rust/kernel/block/mq/operations.rs b/rust/kernel/block/mq/operations.rs index c8646d0d9866..864ff379dc91 100644 --- a/rust/kernel/block/mq/operations.rs +++ b/rust/kernel/block/mq/operations.rs @@ -9,6 +9,7 @@ use crate::{ block::mq::request::RequestDataWrapper, block::mq::Request, error::{from_result, Result}, + prelude::*, types::ARef, }; use core::{marker::PhantomData, sync::atomic::AtomicU64, sync::atomic::Ordering}; @@ -35,7 +36,7 @@ pub trait Operations: Sized { /// Called by the kernel to poll the device for completed requests. Only /// used for poll queues. fn poll() -> bool { - crate::build_error(crate::error::VTABLE_DEFAULT_ERROR) + build_error!(crate::error::VTABLE_DEFAULT_ERROR) } } diff --git a/rust/kernel/block/mq/request.rs b/rust/kernel/block/mq/request.rs index 7943f43b9575..4a5b7ec914ef 100644 --- a/rust/kernel/block/mq/request.rs +++ b/rust/kernel/block/mq/request.rs @@ -12,7 +12,7 @@ use crate::{ }; use core::{ marker::PhantomData, - ptr::{addr_of_mut, NonNull}, + ptr::NonNull, sync::atomic::{AtomicU64, Ordering}, }; @@ -187,7 +187,7 @@ impl RequestDataWrapper { pub(crate) unsafe fn refcount_ptr(this: *mut Self) -> *mut AtomicU64 { // SAFETY: Because of the safety requirements of this function, the // field projection is safe. - unsafe { addr_of_mut!((*this).refcount) } + unsafe { &raw mut (*this).refcount } } } diff --git a/rust/kernel/block/mq/tag_set.rs b/rust/kernel/block/mq/tag_set.rs index d7f175a05d99..bcf4214ad149 100644 --- a/rust/kernel/block/mq/tag_set.rs +++ b/rust/kernel/block/mq/tag_set.rs @@ -10,12 +10,11 @@ use crate::{ bindings, block::mq::{operations::OperationsVTable, request::RequestDataWrapper, Operations}, error, - prelude::PinInit, - try_pin_init, + prelude::try_pin_init, types::Opaque, }; use core::{convert::TryInto, marker::PhantomData}; -use macros::{pin_data, pinned_drop}; +use pin_init::{pin_data, pinned_drop, PinInit}; /// A wrapper for the C `struct blk_mq_tag_set`. /// @@ -52,7 +51,7 @@ impl<T: Operations> TagSet<T> { numa_node: bindings::NUMA_NO_NODE, queue_depth: num_tags, cmd_size, - flags: bindings::BLK_MQ_F_SHOULD_MERGE, + flags: 0, driver_data: core::ptr::null_mut::<crate::ffi::c_void>(), nr_maps: num_maps, ..tag_set diff --git a/rust/kernel/build_assert.rs b/rust/kernel/build_assert.rs index 9e37120bc69c..6331b15d7c4d 100644 --- a/rust/kernel/build_assert.rs +++ b/rust/kernel/build_assert.rs @@ -2,6 +2,9 @@ //! Build-time assert. +#[doc(hidden)] +pub use build_error::build_error; + /// Fails the build if the code path calling `build_error!` can possibly be executed. /// /// If the macro is executed in const context, `build_error!` will panic. @@ -11,7 +14,6 @@ /// # Examples /// /// ``` -/// # use kernel::build_error; /// #[inline] /// fn foo(a: usize) -> usize { /// a.checked_add(1).unwrap_or_else(|| build_error!("overflow")) @@ -23,10 +25,10 @@ #[macro_export] macro_rules! build_error { () => {{ - $crate::build_error("") + $crate::build_assert::build_error("") }}; ($msg:expr) => {{ - $crate::build_error($msg) + $crate::build_assert::build_error($msg) }}; } @@ -73,12 +75,12 @@ macro_rules! build_error { macro_rules! build_assert { ($cond:expr $(,)?) => {{ if !$cond { - $crate::build_error(concat!("assertion failed: ", stringify!($cond))); + $crate::build_assert::build_error(concat!("assertion failed: ", stringify!($cond))); } }}; ($cond:expr, $msg:expr) => {{ if !$cond { - $crate::build_error($msg); + $crate::build_assert::build_error($msg); } }}; } diff --git a/rust/kernel/cred.rs b/rust/kernel/cred.rs index 81d67789b16f..2599f01e8b28 100644 --- a/rust/kernel/cred.rs +++ b/rust/kernel/cred.rs @@ -47,6 +47,7 @@ impl Credential { /// /// The caller must ensure that `ptr` is valid and remains valid for the lifetime of the /// returned [`Credential`] reference. + #[inline] pub unsafe fn from_ptr<'a>(ptr: *const bindings::cred) -> &'a Credential { // SAFETY: The safety requirements guarantee the validity of the dereference, while the // `Credential` type being transparent makes the cast ok. @@ -54,6 +55,7 @@ impl Credential { } /// Get the id for this security context. + #[inline] pub fn get_secid(&self) -> u32 { let mut secid = 0; // SAFETY: The invariants of this type ensures that the pointer is valid. @@ -62,6 +64,7 @@ impl Credential { } /// Returns the effective UID of the given credential. + #[inline] pub fn euid(&self) -> Kuid { // SAFETY: By the type invariant, we know that `self.0` is valid. Furthermore, the `euid` // field of a credential is never changed after initialization, so there is no potential @@ -72,11 +75,13 @@ impl Credential { // SAFETY: The type invariants guarantee that `Credential` is always ref-counted. unsafe impl AlwaysRefCounted for Credential { + #[inline] fn inc_ref(&self) { // SAFETY: The existence of a shared reference means that the refcount is nonzero. unsafe { bindings::get_cred(self.0.get()) }; } + #[inline] unsafe fn dec_ref(obj: core::ptr::NonNull<Credential>) { // SAFETY: The safety requirements guarantee that the refcount is nonzero. The cast is okay // because `Credential` has the same representation as `struct cred`. diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index d5e6a19ff6b7..21b343a1dc4d 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -6,6 +6,7 @@ use crate::{ bindings, + str::CStr, types::{ARef, Opaque}, }; use core::{fmt, ptr}; @@ -180,6 +181,12 @@ impl Device { ) }; } + + /// Checks if property is present or not. + pub fn property_present(&self, name: &CStr) -> bool { + // SAFETY: By the invariant of `CStr`, `name` is null-terminated. + unsafe { bindings::device_property_present(self.as_raw().cast_const(), name.as_char_ptr()) } + } } // SAFETY: Instances of `Device` are always reference-counted. @@ -202,6 +209,32 @@ unsafe impl Send for Device {} // synchronization in `struct device`. unsafe impl Sync for Device {} +/// Marker trait for the context of a bus specific device. +/// +/// Some functions of a bus specific device should only be called from a certain context, i.e. bus +/// callbacks, such as `probe()`. +/// +/// This is the marker trait for structures representing the context of a bus specific device. +pub trait DeviceContext: private::Sealed {} + +/// The [`Normal`] context is the context of a bus specific device when it is not an argument of +/// any bus callback. +pub struct Normal; + +/// The [`Core`] context is the context of a bus specific device when it is supplied as argument of +/// any of the bus callbacks, such as `probe()`. +pub struct Core; + +mod private { + pub trait Sealed {} + + impl Sealed for super::Core {} + impl Sealed for super::Normal {} +} + +impl DeviceContext for Core {} +impl DeviceContext for Normal {} + #[doc(hidden)] #[macro_export] macro_rules! dev_printk { diff --git a/rust/kernel/device_id.rs b/rust/kernel/device_id.rs new file mode 100644 index 000000000000..e5859217a579 --- /dev/null +++ b/rust/kernel/device_id.rs @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic implementation of device IDs. +//! +//! Each bus / subsystem that matches device and driver through a bus / subsystem specific ID is +//! expected to implement [`RawDeviceId`]. + +use core::mem::MaybeUninit; + +/// Marker trait to indicate a Rust device ID type represents a corresponding C device ID type. +/// +/// This is meant to be implemented by buses/subsystems so that they can use [`IdTable`] to +/// guarantee (at compile-time) zero-termination of device id tables provided by drivers. +/// +/// # Safety +/// +/// Implementers must ensure that: +/// - `Self` is layout-compatible with [`RawDeviceId::RawType`]; i.e. it's safe to transmute to +/// `RawDeviceId`. +/// +/// This requirement is needed so `IdArray::new` can convert `Self` to `RawType` when building +/// the ID table. +/// +/// Ideally, this should be achieved using a const function that does conversion instead of +/// transmute; however, const trait functions relies on `const_trait_impl` unstable feature, +/// which is broken/gone in Rust 1.73. +/// +/// - `DRIVER_DATA_OFFSET` is the offset of context/data field of the device ID (usually named +/// `driver_data`) of the device ID, the field is suitable sized to write a `usize` value. +/// +/// Similar to the previous requirement, the data should ideally be added during `Self` to +/// `RawType` conversion, but there's currently no way to do it when using traits in const. +pub unsafe trait RawDeviceId { + /// The raw type that holds the device id. + /// + /// Id tables created from [`Self`] are going to hold this type in its zero-terminated array. + type RawType: Copy; + + /// The offset to the context/data field. + const DRIVER_DATA_OFFSET: usize; + + /// The index stored at `DRIVER_DATA_OFFSET` of the implementor of the [`RawDeviceId`] trait. + fn index(&self) -> usize; +} + +/// A zero-terminated device id array. +#[repr(C)] +pub struct RawIdArray<T: RawDeviceId, const N: usize> { + ids: [T::RawType; N], + sentinel: MaybeUninit<T::RawType>, +} + +impl<T: RawDeviceId, const N: usize> RawIdArray<T, N> { + #[doc(hidden)] + pub const fn size(&self) -> usize { + core::mem::size_of::<Self>() + } +} + +/// A zero-terminated device id array, followed by context data. +#[repr(C)] +pub struct IdArray<T: RawDeviceId, U, const N: usize> { + raw_ids: RawIdArray<T, N>, + id_infos: [U; N], +} + +impl<T: RawDeviceId, U, const N: usize> IdArray<T, U, N> { + /// Creates a new instance of the array. + /// + /// The contents are derived from the given identifiers and context information. + pub const fn new(ids: [(T, U); N]) -> Self { + let mut raw_ids = [const { MaybeUninit::<T::RawType>::uninit() }; N]; + let mut infos = [const { MaybeUninit::uninit() }; N]; + + let mut i = 0usize; + while i < N { + // SAFETY: by the safety requirement of `RawDeviceId`, we're guaranteed that `T` is + // layout-wise compatible with `RawType`. + raw_ids[i] = unsafe { core::mem::transmute_copy(&ids[i].0) }; + // SAFETY: by the safety requirement of `RawDeviceId`, this would be effectively + // `raw_ids[i].driver_data = i;`. + unsafe { + raw_ids[i] + .as_mut_ptr() + .byte_offset(T::DRIVER_DATA_OFFSET as _) + .cast::<usize>() + .write(i); + } + + // SAFETY: this is effectively a move: `infos[i] = ids[i].1`. We make a copy here but + // later forget `ids`. + infos[i] = MaybeUninit::new(unsafe { core::ptr::read(&ids[i].1) }); + i += 1; + } + + core::mem::forget(ids); + + Self { + raw_ids: RawIdArray { + // SAFETY: this is effectively `array_assume_init`, which is unstable, so we use + // `transmute_copy` instead. We have initialized all elements of `raw_ids` so this + // `array_assume_init` is safe. + ids: unsafe { core::mem::transmute_copy(&raw_ids) }, + sentinel: MaybeUninit::zeroed(), + }, + // SAFETY: We have initialized all elements of `infos` so this `array_assume_init` is + // safe. + id_infos: unsafe { core::mem::transmute_copy(&infos) }, + } + } + + /// Reference to the contained [`RawIdArray`]. + pub const fn raw_ids(&self) -> &RawIdArray<T, N> { + &self.raw_ids + } +} + +/// A device id table. +/// +/// This trait is only implemented by `IdArray`. +/// +/// The purpose of this trait is to allow `&'static dyn IdArray<T, U>` to be in context when `N` in +/// `IdArray` doesn't matter. +pub trait IdTable<T: RawDeviceId, U> { + /// Obtain the pointer to the ID table. + fn as_ptr(&self) -> *const T::RawType; + + /// Obtain the pointer to the bus specific device ID from an index. + fn id(&self, index: usize) -> &T::RawType; + + /// Obtain the pointer to the driver-specific information from an index. + fn info(&self, index: usize) -> &U; +} + +impl<T: RawDeviceId, U, const N: usize> IdTable<T, U> for IdArray<T, U, N> { + fn as_ptr(&self) -> *const T::RawType { + // This cannot be `self.ids.as_ptr()`, as the return pointer must have correct provenance + // to access the sentinel. + (self as *const Self).cast() + } + + fn id(&self, index: usize) -> &T::RawType { + &self.raw_ids.ids[index] + } + + fn info(&self, index: usize) -> &U { + &self.id_infos[index] + } +} + +/// Create device table alias for modpost. +#[macro_export] +macro_rules! module_device_table { + ($table_type: literal, $module_table_name:ident, $table_name:ident) => { + #[rustfmt::skip] + #[export_name = + concat!("__mod_device_table__", $table_type, + "__", module_path!(), + "_", line!(), + "_", stringify!($table_name)) + ] + static $module_table_name: [core::mem::MaybeUninit<u8>; $table_name.raw_ids().size()] = + unsafe { core::mem::transmute_copy($table_name.raw_ids()) }; + }; +} diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs new file mode 100644 index 000000000000..dc6ea014ee60 --- /dev/null +++ b/rust/kernel/devres.rs @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Devres abstraction +//! +//! [`Devres`] represents an abstraction for the kernel devres (device resource management) +//! implementation. + +use crate::{ + alloc::Flags, + bindings, + device::Device, + error::{Error, Result}, + ffi::c_void, + prelude::*, + revocable::{Revocable, RevocableGuard}, + sync::{rcu, Arc, Completion}, + types::ARef, +}; + +#[pin_data] +struct DevresInner<T> { + dev: ARef<Device>, + callback: unsafe extern "C" fn(*mut c_void), + #[pin] + data: Revocable<T>, + #[pin] + revoke: Completion, +} + +/// This abstraction is meant to be used by subsystems to containerize [`Device`] bound resources to +/// manage their lifetime. +/// +/// [`Device`] bound resources should be freed when either the resource goes out of scope or the +/// [`Device`] is unbound respectively, depending on what happens first. In any case, it is always +/// guaranteed that revoking the device resource is completed before the corresponding [`Device`] +/// is unbound. +/// +/// To achieve that [`Devres`] registers a devres callback on creation, which is called once the +/// [`Device`] is unbound, revoking access to the encapsulated resource (see also [`Revocable`]). +/// +/// After the [`Devres`] has been unbound it is not possible to access the encapsulated resource +/// anymore. +/// +/// [`Devres`] users should make sure to simply free the corresponding backing resource in `T`'s +/// [`Drop`] implementation. +/// +/// # Example +/// +/// ```no_run +/// # use kernel::{bindings, c_str, device::Device, devres::Devres, io::{Io, IoRaw}}; +/// # use core::ops::Deref; +/// +/// // See also [`pci::Bar`] for a real example. +/// struct IoMem<const SIZE: usize>(IoRaw<SIZE>); +/// +/// impl<const SIZE: usize> IoMem<SIZE> { +/// /// # Safety +/// /// +/// /// [`paddr`, `paddr` + `SIZE`) must be a valid MMIO region that is mappable into the CPUs +/// /// virtual address space. +/// unsafe fn new(paddr: usize) -> Result<Self>{ +/// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is +/// // valid for `ioremap`. +/// let addr = unsafe { bindings::ioremap(paddr as _, SIZE as _) }; +/// if addr.is_null() { +/// return Err(ENOMEM); +/// } +/// +/// Ok(IoMem(IoRaw::new(addr as _, SIZE)?)) +/// } +/// } +/// +/// impl<const SIZE: usize> Drop for IoMem<SIZE> { +/// fn drop(&mut self) { +/// // SAFETY: `self.0.addr()` is guaranteed to be properly mapped by `Self::new`. +/// unsafe { bindings::iounmap(self.0.addr() as _); }; +/// } +/// } +/// +/// impl<const SIZE: usize> Deref for IoMem<SIZE> { +/// type Target = Io<SIZE>; +/// +/// fn deref(&self) -> &Self::Target { +/// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`. +/// unsafe { Io::from_raw(&self.0) } +/// } +/// } +/// # fn no_run() -> Result<(), Error> { +/// # // SAFETY: Invalid usage; just for the example to get an `ARef<Device>` instance. +/// # let dev = unsafe { Device::get_device(core::ptr::null_mut()) }; +/// +/// // SAFETY: Invalid usage for example purposes. +/// let iomem = unsafe { IoMem::<{ core::mem::size_of::<u32>() }>::new(0xBAAAAAAD)? }; +/// let devres = Devres::new(&dev, iomem, GFP_KERNEL)?; +/// +/// let res = devres.try_access().ok_or(ENXIO)?; +/// res.write8(0x42, 0x0); +/// # Ok(()) +/// # } +/// ``` +pub struct Devres<T>(Arc<DevresInner<T>>); + +impl<T> DevresInner<T> { + fn new(dev: &Device, data: T, flags: Flags) -> Result<Arc<DevresInner<T>>> { + let inner = Arc::pin_init( + pin_init!( DevresInner { + dev: dev.into(), + callback: Self::devres_callback, + data <- Revocable::new(data), + revoke <- Completion::new(), + }), + flags, + )?; + + // Convert `Arc<DevresInner>` into a raw pointer and make devres own this reference until + // `Self::devres_callback` is called. + let data = inner.clone().into_raw(); + + // SAFETY: `devm_add_action` guarantees to call `Self::devres_callback` once `dev` is + // detached. + let ret = + unsafe { bindings::devm_add_action(dev.as_raw(), Some(inner.callback), data as _) }; + + if ret != 0 { + // SAFETY: We just created another reference to `inner` in order to pass it to + // `bindings::devm_add_action`. If `bindings::devm_add_action` fails, we have to drop + // this reference accordingly. + let _ = unsafe { Arc::from_raw(data) }; + return Err(Error::from_errno(ret)); + } + + Ok(inner) + } + + fn as_ptr(&self) -> *const Self { + self as _ + } + + fn remove_action(this: &Arc<Self>) -> bool { + // SAFETY: + // - `self.inner.dev` is a valid `Device`, + // - the `action` and `data` pointers are the exact same ones as given to devm_add_action() + // previously, + // - `self` is always valid, even if the action has been released already. + let success = unsafe { + bindings::devm_remove_action_nowarn( + this.dev.as_raw(), + Some(this.callback), + this.as_ptr() as _, + ) + } == 0; + + if success { + // SAFETY: We leaked an `Arc` reference to devm_add_action() in `DevresInner::new`; if + // devm_remove_action_nowarn() was successful we can (and have to) claim back ownership + // of this reference. + let _ = unsafe { Arc::from_raw(this.as_ptr()) }; + } + + success + } + + #[allow(clippy::missing_safety_doc)] + unsafe extern "C" fn devres_callback(ptr: *mut kernel::ffi::c_void) { + let ptr = ptr as *mut DevresInner<T>; + // Devres owned this memory; now that we received the callback, drop the `Arc` and hence the + // reference. + // SAFETY: Safe, since we leaked an `Arc` reference to devm_add_action() in + // `DevresInner::new`. + let inner = unsafe { Arc::from_raw(ptr) }; + + if !inner.data.revoke() { + // If `revoke()` returns false, it means that `Devres::drop` already started revoking + // `inner.data` for us. Hence we have to wait until `Devres::drop()` signals that it + // completed revoking `inner.data`. + inner.revoke.wait_for_completion(); + } + } +} + +impl<T> Devres<T> { + /// Creates a new [`Devres`] instance of the given `data`. The `data` encapsulated within the + /// returned `Devres` instance' `data` will be revoked once the device is detached. + pub fn new(dev: &Device, data: T, flags: Flags) -> Result<Self> { + let inner = DevresInner::new(dev, data, flags)?; + + Ok(Devres(inner)) + } + + /// Same as [`Devres::new`], but does not return a `Devres` instance. Instead the given `data` + /// is owned by devres and will be revoked / dropped, once the device is detached. + pub fn new_foreign_owned(dev: &Device, data: T, flags: Flags) -> Result { + let _ = DevresInner::new(dev, data, flags)?; + + Ok(()) + } + + /// [`Devres`] accessor for [`Revocable::try_access`]. + pub fn try_access(&self) -> Option<RevocableGuard<'_, T>> { + self.0.data.try_access() + } + + /// [`Devres`] accessor for [`Revocable::try_access_with_guard`]. + pub fn try_access_with_guard<'a>(&'a self, guard: &'a rcu::Guard) -> Option<&'a T> { + self.0.data.try_access_with_guard(guard) + } +} + +impl<T> Drop for Devres<T> { + fn drop(&mut self) { + // SAFETY: When `drop` runs, it is guaranteed that nobody is accessing the revocable data + // anymore, hence it is safe not to wait for the grace period to finish. + if unsafe { self.0.data.revoke_nosync() } { + // We revoked `self.0.data` before the devres action did, hence try to remove it. + if !DevresInner::remove_action(&self.0) { + // We could not remove the devres action, which means that it now runs concurrently, + // hence signal that `self.0.data` has been revoked successfully. + self.0.revoke.complete_all(); + } + } + } +} diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs new file mode 100644 index 000000000000..8cdc76043ee7 --- /dev/null +++ b/rust/kernel/dma.rs @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Direct memory access (DMA). +//! +//! C header: [`include/linux/dma-mapping.h`](srctree/include/linux/dma-mapping.h) + +use crate::{ + bindings, build_assert, + device::Device, + error::code::*, + error::Result, + transmute::{AsBytes, FromBytes}, + types::ARef, +}; + +/// Possible attributes associated with a DMA mapping. +/// +/// They can be combined with the operators `|`, `&`, and `!`. +/// +/// Values can be used from the [`attrs`] module. +/// +/// # Examples +/// +/// ``` +/// use kernel::device::Device; +/// use kernel::dma::{attrs::*, CoherentAllocation}; +/// +/// # fn test(dev: &Device) -> Result { +/// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN; +/// let c: CoherentAllocation<u64> = +/// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, attribs)?; +/// # Ok::<(), Error>(()) } +/// ``` +#[derive(Clone, Copy, PartialEq)] +#[repr(transparent)] +pub struct Attrs(u32); + +impl Attrs { + /// Get the raw representation of this attribute. + pub(crate) fn as_raw(self) -> crate::ffi::c_ulong { + self.0 as _ + } + + /// Check whether `flags` is contained in `self`. + pub fn contains(self, flags: Attrs) -> bool { + (self & flags) == flags + } +} + +impl core::ops::BitOr for Attrs { + type Output = Self; + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } +} + +impl core::ops::BitAnd for Attrs { + type Output = Self; + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } +} + +impl core::ops::Not for Attrs { + type Output = Self; + fn not(self) -> Self::Output { + Self(!self.0) + } +} + +/// DMA mapping attributes. +pub mod attrs { + use super::Attrs; + + /// Specifies that reads and writes to the mapping may be weakly ordered, that is that reads + /// and writes may pass each other. + pub const DMA_ATTR_WEAK_ORDERING: Attrs = Attrs(bindings::DMA_ATTR_WEAK_ORDERING); + + /// Specifies that writes to the mapping may be buffered to improve performance. + pub const DMA_ATTR_WRITE_COMBINE: Attrs = Attrs(bindings::DMA_ATTR_WRITE_COMBINE); + + /// Lets the platform to avoid creating a kernel virtual mapping for the allocated buffer. + pub const DMA_ATTR_NO_KERNEL_MAPPING: Attrs = Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING); + + /// Allows platform code to skip synchronization of the CPU cache for the given buffer assuming + /// that it has been already transferred to 'device' domain. + pub const DMA_ATTR_SKIP_CPU_SYNC: Attrs = Attrs(bindings::DMA_ATTR_SKIP_CPU_SYNC); + + /// Forces contiguous allocation of the buffer in physical memory. + pub const DMA_ATTR_FORCE_CONTIGUOUS: Attrs = Attrs(bindings::DMA_ATTR_FORCE_CONTIGUOUS); + + /// This is a hint to the DMA-mapping subsystem that it's probably not worth the time to try + /// to allocate memory to in a way that gives better TLB efficiency. + pub const DMA_ATTR_ALLOC_SINGLE_PAGES: Attrs = Attrs(bindings::DMA_ATTR_ALLOC_SINGLE_PAGES); + + /// This tells the DMA-mapping subsystem to suppress allocation failure reports (similarly to + /// __GFP_NOWARN). + pub const DMA_ATTR_NO_WARN: Attrs = Attrs(bindings::DMA_ATTR_NO_WARN); + + /// Used to indicate that the buffer is fully accessible at an elevated privilege level (and + /// ideally inaccessible or at least read-only at lesser-privileged levels). + pub const DMA_ATTR_PRIVILEGED: Attrs = Attrs(bindings::DMA_ATTR_PRIVILEGED); +} + +/// An abstraction of the `dma_alloc_coherent` API. +/// +/// This is an abstraction around the `dma_alloc_coherent` API which is used to allocate and map +/// large consistent DMA regions. +/// +/// A [`CoherentAllocation`] instance contains a pointer to the allocated region (in the +/// processor's virtual address space) and the device address which can be given to the device +/// as the DMA address base of the region. The region is released once [`CoherentAllocation`] +/// is dropped. +/// +/// # Invariants +/// +/// For the lifetime of an instance of [`CoherentAllocation`], the `cpu_addr` is a valid pointer +/// to an allocated region of consistent memory and `dma_handle` is the DMA address base of +/// the region. +// TODO +// +// DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness +// reasons DMA allocation would need to be embedded in a `Devres` container, in order to ensure +// that device resources can never survive device unbind. +// +// However, it is neither desirable nor necessary to protect the allocated memory of the DMA +// allocation from surviving device unbind; it would require RCU read side critical sections to +// access the memory, which may require subsequent unnecessary copies. +// +// Hence, find a way to revoke the device resources of a `CoherentAllocation`, but not the +// entire `CoherentAllocation` including the allocated memory itself. +pub struct CoherentAllocation<T: AsBytes + FromBytes> { + dev: ARef<Device>, + dma_handle: bindings::dma_addr_t, + count: usize, + cpu_addr: *mut T, + dma_attrs: Attrs, +} + +impl<T: AsBytes + FromBytes> CoherentAllocation<T> { + /// Allocates a region of `size_of::<T> * count` of consistent memory. + /// + /// # Examples + /// + /// ``` + /// use kernel::device::Device; + /// use kernel::dma::{attrs::*, CoherentAllocation}; + /// + /// # fn test(dev: &Device) -> Result { + /// let c: CoherentAllocation<u64> = + /// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// # Ok::<(), Error>(()) } + /// ``` + pub fn alloc_attrs( + dev: &Device, + count: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result<CoherentAllocation<T>> { + build_assert!( + core::mem::size_of::<T>() > 0, + "It doesn't make sense for the allocated type to be a ZST" + ); + + let size = count + .checked_mul(core::mem::size_of::<T>()) + .ok_or(EOVERFLOW)?; + let mut dma_handle = 0; + // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. + let ret = unsafe { + bindings::dma_alloc_attrs( + dev.as_raw(), + size, + &mut dma_handle, + gfp_flags.as_raw(), + dma_attrs.as_raw(), + ) + }; + if ret.is_null() { + return Err(ENOMEM); + } + // INVARIANT: We just successfully allocated a coherent region which is accessible for + // `count` elements, hence the cpu address is valid. We also hold a refcounted reference + // to the device. + Ok(Self { + dev: dev.into(), + dma_handle, + count, + cpu_addr: ret as *mut T, + dma_attrs, + }) + } + + /// Performs the same functionality as [`CoherentAllocation::alloc_attrs`], except the + /// `dma_attrs` is 0 by default. + pub fn alloc_coherent( + dev: &Device, + count: usize, + gfp_flags: kernel::alloc::Flags, + ) -> Result<CoherentAllocation<T>> { + CoherentAllocation::alloc_attrs(dev, count, gfp_flags, Attrs(0)) + } + + /// Returns the base address to the allocated region in the CPU's virtual address space. + pub fn start_ptr(&self) -> *const T { + self.cpu_addr + } + + /// Returns the base address to the allocated region in the CPU's virtual address space as + /// a mutable pointer. + pub fn start_ptr_mut(&mut self) -> *mut T { + self.cpu_addr + } + + /// Returns a DMA handle which may given to the device as the DMA address base of + /// the region. + pub fn dma_handle(&self) -> bindings::dma_addr_t { + self.dma_handle + } + + /// Returns a pointer to an element from the region with bounds checking. `offset` is in + /// units of `T`, not the number of bytes. + /// + /// Public but hidden since it should only be used from [`dma_read`] and [`dma_write`] macros. + #[doc(hidden)] + pub fn item_from_index(&self, offset: usize) -> Result<*mut T> { + if offset >= self.count { + return Err(EINVAL); + } + // SAFETY: + // - The pointer is valid due to type invariant on `CoherentAllocation` + // and we've just checked that the range and index is within bounds. + // - `offset` can't overflow since it is smaller than `self.count` and we've checked + // that `self.count` won't overflow early in the constructor. + Ok(unsafe { self.cpu_addr.add(offset) }) + } + + /// Reads the value of `field` and ensures that its type is [`FromBytes`]. + /// + /// # Safety + /// + /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is + /// validated beforehand. + /// + /// Public but hidden since it should only be used from [`dma_read`] macro. + #[doc(hidden)] + pub unsafe fn field_read<F: FromBytes>(&self, field: *const F) -> F { + // SAFETY: + // - By the safety requirements field is valid. + // - Using read_volatile() here is not sound as per the usual rules, the usage here is + // a special exception with the following notes in place. When dealing with a potential + // race from a hardware or code outside kernel (e.g. user-space program), we need that + // read on a valid memory is not UB. Currently read_volatile() is used for this, and the + // rationale behind is that it should generate the same code as READ_ONCE() which the + // kernel already relies on to avoid UB on data races. Note that the usage of + // read_volatile() is limited to this particular case, it cannot be used to prevent + // the UB caused by racing between two kernel functions nor do they provide atomicity. + unsafe { field.read_volatile() } + } + + /// Writes a value to `field` and ensures that its type is [`AsBytes`]. + /// + /// # Safety + /// + /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is + /// validated beforehand. + /// + /// Public but hidden since it should only be used from [`dma_write`] macro. + #[doc(hidden)] + pub unsafe fn field_write<F: AsBytes>(&self, field: *mut F, val: F) { + // SAFETY: + // - By the safety requirements field is valid. + // - Using write_volatile() here is not sound as per the usual rules, the usage here is + // a special exception with the following notes in place. When dealing with a potential + // race from a hardware or code outside kernel (e.g. user-space program), we need that + // write on a valid memory is not UB. Currently write_volatile() is used for this, and the + // rationale behind is that it should generate the same code as WRITE_ONCE() which the + // kernel already relies on to avoid UB on data races. Note that the usage of + // write_volatile() is limited to this particular case, it cannot be used to prevent + // the UB caused by racing between two kernel functions nor do they provide atomicity. + unsafe { field.write_volatile(val) } + } +} + +/// Note that the device configured to do DMA must be halted before this object is dropped. +impl<T: AsBytes + FromBytes> Drop for CoherentAllocation<T> { + fn drop(&mut self) { + let size = self.count * core::mem::size_of::<T>(); + // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. + // The cpu address, and the dma handle are valid due to the type invariants on + // `CoherentAllocation`. + unsafe { + bindings::dma_free_attrs( + self.dev.as_raw(), + size, + self.cpu_addr as _, + self.dma_handle, + self.dma_attrs.as_raw(), + ) + } + } +} + +// SAFETY: It is safe to send a `CoherentAllocation` to another thread if `T` +// can be sent to another thread. +unsafe impl<T: AsBytes + FromBytes + Send> Send for CoherentAllocation<T> {} + +/// Reads a field of an item from an allocated region of structs. +/// +/// # Examples +/// +/// ``` +/// use kernel::device::Device; +/// use kernel::dma::{attrs::*, CoherentAllocation}; +/// +/// struct MyStruct { field: u32, } +/// +/// // SAFETY: All bit patterns are acceptable values for `MyStruct`. +/// unsafe impl kernel::transmute::FromBytes for MyStruct{}; +/// // SAFETY: Instances of `MyStruct` have no uninitialized portions. +/// unsafe impl kernel::transmute::AsBytes for MyStruct{}; +/// +/// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result { +/// let whole = kernel::dma_read!(alloc[2]); +/// let field = kernel::dma_read!(alloc[1].field); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +macro_rules! dma_read { + ($dma:expr, $idx: expr, $($field:tt)*) => {{ + let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; + // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be + // dereferenced. The compiler also further validates the expression on whether `field` + // is a member of `item` when expanded by the macro. + unsafe { + let ptr_field = ::core::ptr::addr_of!((*item) $($field)*); + $crate::dma::CoherentAllocation::field_read(&$dma, ptr_field) + } + }}; + ($dma:ident [ $idx:expr ] $($field:tt)* ) => { + $crate::dma_read!($dma, $idx, $($field)*); + }; + ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => { + $crate::dma_read!($($dma).*, $idx, $($field)*); + }; +} + +/// Writes to a field of an item from an allocated region of structs. +/// +/// # Examples +/// +/// ``` +/// use kernel::device::Device; +/// use kernel::dma::{attrs::*, CoherentAllocation}; +/// +/// struct MyStruct { member: u32, } +/// +/// // SAFETY: All bit patterns are acceptable values for `MyStruct`. +/// unsafe impl kernel::transmute::FromBytes for MyStruct{}; +/// // SAFETY: Instances of `MyStruct` have no uninitialized portions. +/// unsafe impl kernel::transmute::AsBytes for MyStruct{}; +/// +/// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result { +/// kernel::dma_write!(alloc[2].member = 0xf); +/// kernel::dma_write!(alloc[1] = MyStruct { member: 0xf }); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +macro_rules! dma_write { + ($dma:ident [ $idx:expr ] $($field:tt)*) => {{ + $crate::dma_write!($dma, $idx, $($field)*); + }}; + ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {{ + $crate::dma_write!($($dma).*, $idx, $($field)*); + }}; + ($dma:expr, $idx: expr, = $val:expr) => { + let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; + // SAFETY: `item_from_index` ensures that `item` is always a valid item. + unsafe { $crate::dma::CoherentAllocation::field_write(&$dma, item, $val) } + }; + ($dma:expr, $idx: expr, $(.$field:ident)* = $val:expr) => { + let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; + // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be + // dereferenced. The compiler also further validates the expression on whether `field` + // is a member of `item` when expanded by the macro. + unsafe { + let ptr_field = ::core::ptr::addr_of_mut!((*item) $(.$field)*); + $crate::dma::CoherentAllocation::field_write(&$dma, ptr_field, $val) + } + }; +} diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs new file mode 100644 index 000000000000..ec9166cedfa7 --- /dev/null +++ b/rust/kernel/driver.rs @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic support for drivers of different buses (e.g., PCI, Platform, Amba, etc.). +//! +//! Each bus / subsystem is expected to implement [`RegistrationOps`], which allows drivers to +//! register using the [`Registration`] class. + +use crate::error::{Error, Result}; +use crate::{device, of, str::CStr, try_pin_init, types::Opaque, ThisModule}; +use core::pin::Pin; +use pin_init::{pin_data, pinned_drop, PinInit}; + +/// The [`RegistrationOps`] trait serves as generic interface for subsystems (e.g., PCI, Platform, +/// Amba, etc.) to provide the corresponding subsystem specific implementation to register / +/// unregister a driver of the particular type (`RegType`). +/// +/// For instance, the PCI subsystem would set `RegType` to `bindings::pci_driver` and call +/// `bindings::__pci_register_driver` from `RegistrationOps::register` and +/// `bindings::pci_unregister_driver` from `RegistrationOps::unregister`. +/// +/// # Safety +/// +/// A call to [`RegistrationOps::unregister`] for a given instance of `RegType` is only valid if a +/// preceding call to [`RegistrationOps::register`] has been successful. +pub unsafe trait RegistrationOps { + /// The type that holds information about the registration. This is typically a struct defined + /// by the C portion of the kernel. + type RegType: Default; + + /// Registers a driver. + /// + /// # Safety + /// + /// On success, `reg` must remain pinned and valid until the matching call to + /// [`RegistrationOps::unregister`]. + unsafe fn register( + reg: &Opaque<Self::RegType>, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result; + + /// Unregisters a driver previously registered with [`RegistrationOps::register`]. + /// + /// # Safety + /// + /// Must only be called after a preceding successful call to [`RegistrationOps::register`] for + /// the same `reg`. + unsafe fn unregister(reg: &Opaque<Self::RegType>); +} + +/// A [`Registration`] is a generic type that represents the registration of some driver type (e.g. +/// `bindings::pci_driver`). Therefore a [`Registration`] must be initialized with a type that +/// implements the [`RegistrationOps`] trait, such that the generic `T::register` and +/// `T::unregister` calls result in the subsystem specific registration calls. +/// +///Once the `Registration` structure is dropped, the driver is unregistered. +#[pin_data(PinnedDrop)] +pub struct Registration<T: RegistrationOps> { + #[pin] + reg: Opaque<T::RegType>, +} + +// SAFETY: `Registration` has no fields or methods accessible via `&Registration`, so it is safe to +// share references to it with multiple threads as nothing can be done. +unsafe impl<T: RegistrationOps> Sync for Registration<T> {} + +// SAFETY: Both registration and unregistration are implemented in C and safe to be performed from +// any thread, so `Registration` is `Send`. +unsafe impl<T: RegistrationOps> Send for Registration<T> {} + +impl<T: RegistrationOps> Registration<T> { + /// Creates a new instance of the registration object. + pub fn new(name: &'static CStr, module: &'static ThisModule) -> impl PinInit<Self, Error> { + try_pin_init!(Self { + reg <- Opaque::try_ffi_init(|ptr: *mut T::RegType| { + // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write. + unsafe { ptr.write(T::RegType::default()) }; + + // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write, and it has + // just been initialised above, so it's also valid for read. + let drv = unsafe { &*(ptr as *const Opaque<T::RegType>) }; + + // SAFETY: `drv` is guaranteed to be pinned until `T::unregister`. + unsafe { T::register(drv, name, module) } + }), + }) + } +} + +#[pinned_drop] +impl<T: RegistrationOps> PinnedDrop for Registration<T> { + fn drop(self: Pin<&mut Self>) { + // SAFETY: The existence of `self` guarantees that `self.reg` has previously been + // successfully registered with `T::register` + unsafe { T::unregister(&self.reg) }; + } +} + +/// Declares a kernel module that exposes a single driver. +/// +/// It is meant to be used as a helper by other subsystems so they can more easily expose their own +/// macros. +#[macro_export] +macro_rules! module_driver { + (<$gen_type:ident>, $driver_ops:ty, { type: $type:ty, $($f:tt)* }) => { + type Ops<$gen_type> = $driver_ops; + + #[$crate::prelude::pin_data] + struct DriverModule { + #[pin] + _driver: $crate::driver::Registration<Ops<$type>>, + } + + impl $crate::InPlaceModule for DriverModule { + fn init( + module: &'static $crate::ThisModule + ) -> impl ::pin_init::PinInit<Self, $crate::error::Error> { + $crate::try_pin_init!(Self { + _driver <- $crate::driver::Registration::new( + <Self as $crate::ModuleMetadata>::NAME, + module, + ), + }) + } + } + + $crate::prelude::module! { + type: DriverModule, + $($f)* + } + } +} + +/// The bus independent adapter to match a drivers and a devices. +/// +/// This trait should be implemented by the bus specific adapter, which represents the connection +/// of a device and a driver. +/// +/// It provides bus independent functions for device / driver interactions. +pub trait Adapter { + /// The type holding driver private data about each device id supported by the driver. + type IdInfo: 'static; + + /// The [`of::IdTable`] of the corresponding driver. + fn of_id_table() -> Option<of::IdTable<Self::IdInfo>>; + + /// Returns the driver's private data from the matching entry in the [`of::IdTable`], if any. + /// + /// If this returns `None`, it means there is no match with an entry in the [`of::IdTable`]. + #[cfg(CONFIG_OF)] + fn of_id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { + let table = Self::of_id_table()?; + + // SAFETY: + // - `table` has static lifetime, hence it's valid for read, + // - `dev` is guaranteed to be valid while it's alive, and so is `pdev.as_ref().as_raw()`. + let raw_id = unsafe { bindings::of_match_device(table.as_ptr(), dev.as_raw()) }; + + if raw_id.is_null() { + None + } else { + // SAFETY: `DeviceId` is a `#[repr(transparent)` wrapper of `struct of_device_id` and + // does not add additional invariants, so it's safe to transmute. + let id = unsafe { &*raw_id.cast::<of::DeviceId>() }; + + Some(table.info(<of::DeviceId as crate::device_id::RawDeviceId>::index(id))) + } + } + + #[cfg(not(CONFIG_OF))] + #[allow(missing_docs)] + fn of_id_info(_dev: &device::Device) -> Option<&'static Self::IdInfo> { + None + } + + /// Returns the driver's private data from the matching entry of any of the ID tables, if any. + /// + /// If this returns `None`, it means that there is no match in any of the ID tables directly + /// associated with a [`device::Device`]. + fn id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { + let id = Self::of_id_info(dev); + if id.is_some() { + return id; + } + + None + } +} diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 5fece574ec02..3dee3139fcd4 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -4,9 +4,10 @@ //! //! C header: [`include/uapi/asm-generic/errno-base.h`](srctree/include/uapi/asm-generic/errno-base.h) -use crate::{alloc::AllocError, str::CStr}; - -use core::alloc::LayoutError; +use crate::{ + alloc::{layout::LayoutError, AllocError}, + str::CStr, +}; use core::fmt; use core::num::NonZeroI32; @@ -63,6 +64,7 @@ pub mod code { declare_err!(EPIPE, "Broken pipe."); declare_err!(EDOM, "Math argument out of domain of func."); declare_err!(ERANGE, "Math result not representable."); + declare_err!(EOVERFLOW, "Value too large for defined data type."); declare_err!(ERESTARTSYS, "Restart the system call."); declare_err!(ERESTARTNOINTR, "System call was interrupted by a signal and will be restarted."); declare_err!(ERESTARTNOHAND, "Restart if no handler."); @@ -101,19 +103,16 @@ impl Error { /// It is a bug to pass an out-of-range `errno`. `EINVAL` would /// be returned in such a case. pub fn from_errno(errno: crate::ffi::c_int) -> Error { - if errno < -(bindings::MAX_ERRNO as i32) || errno >= 0 { + if let Some(error) = Self::try_from_errno(errno) { + error + } else { // TODO: Make it a `WARN_ONCE` once available. crate::pr_warn!( - "attempted to create `Error` with out of range `errno`: {}", + "attempted to create `Error` with out of range `errno`: {}\n", errno ); - return code::EINVAL; + code::EINVAL } - - // INVARIANT: The check above ensures the type invariant - // will hold. - // SAFETY: `errno` is checked above to be in a valid range. - unsafe { Error::from_errno_unchecked(errno) } } /// Creates an [`Error`] from a kernel error code. @@ -250,8 +249,129 @@ impl From<core::convert::Infallible> for Error { /// [`Error`] as its error type. /// /// Note that even if a function does not return anything when it succeeds, -/// it should still be modeled as returning a `Result` rather than +/// it should still be modeled as returning a [`Result`] rather than /// just an [`Error`]. +/// +/// Calling a function that returns [`Result`] forces the caller to handle +/// the returned [`Result`]. +/// +/// This can be done "manually" by using [`match`]. Using [`match`] to decode +/// the [`Result`] is similar to C where all the return value decoding and the +/// error handling is done explicitly by writing handling code for each +/// error to cover. Using [`match`] the error and success handling can be +/// implemented in all detail as required. For example (inspired by +/// [`samples/rust/rust_minimal.rs`]): +/// +/// ``` +/// # #[allow(clippy::single_match)] +/// fn example() -> Result { +/// let mut numbers = KVec::new(); +/// +/// match numbers.push(72, GFP_KERNEL) { +/// Err(e) => { +/// pr_err!("Error pushing 72: {e:?}"); +/// return Err(e.into()); +/// } +/// // Do nothing, continue. +/// Ok(()) => (), +/// } +/// +/// match numbers.push(108, GFP_KERNEL) { +/// Err(e) => { +/// pr_err!("Error pushing 108: {e:?}"); +/// return Err(e.into()); +/// } +/// // Do nothing, continue. +/// Ok(()) => (), +/// } +/// +/// match numbers.push(200, GFP_KERNEL) { +/// Err(e) => { +/// pr_err!("Error pushing 200: {e:?}"); +/// return Err(e.into()); +/// } +/// // Do nothing, continue. +/// Ok(()) => (), +/// } +/// +/// Ok(()) +/// } +/// # example()?; +/// # Ok::<(), Error>(()) +/// ``` +/// +/// An alternative to be more concise is the [`if let`] syntax: +/// +/// ``` +/// fn example() -> Result { +/// let mut numbers = KVec::new(); +/// +/// if let Err(e) = numbers.push(72, GFP_KERNEL) { +/// pr_err!("Error pushing 72: {e:?}"); +/// return Err(e.into()); +/// } +/// +/// if let Err(e) = numbers.push(108, GFP_KERNEL) { +/// pr_err!("Error pushing 108: {e:?}"); +/// return Err(e.into()); +/// } +/// +/// if let Err(e) = numbers.push(200, GFP_KERNEL) { +/// pr_err!("Error pushing 200: {e:?}"); +/// return Err(e.into()); +/// } +/// +/// Ok(()) +/// } +/// # example()?; +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Instead of these verbose [`match`]/[`if let`], the [`?`] operator can +/// be used to handle the [`Result`]. Using the [`?`] operator is often +/// the best choice to handle [`Result`] in a non-verbose way as done in +/// [`samples/rust/rust_minimal.rs`]: +/// +/// ``` +/// fn example() -> Result { +/// let mut numbers = KVec::new(); +/// +/// numbers.push(72, GFP_KERNEL)?; +/// numbers.push(108, GFP_KERNEL)?; +/// numbers.push(200, GFP_KERNEL)?; +/// +/// Ok(()) +/// } +/// # example()?; +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Another possibility is to call [`unwrap()`](Result::unwrap) or +/// [`expect()`](Result::expect). However, use of these functions is +/// *heavily discouraged* in the kernel because they trigger a Rust +/// [`panic!`] if an error happens, which may destabilize the system or +/// entirely break it as a result -- just like the C [`BUG()`] macro. +/// Please see the documentation for the C macro [`BUG()`] for guidance +/// on when to use these functions. +/// +/// Alternatively, depending on the use case, using [`unwrap_or()`], +/// [`unwrap_or_else()`], [`unwrap_or_default()`] or [`unwrap_unchecked()`] +/// might be an option, as well. +/// +/// For even more details, please see the [Rust documentation]. +/// +/// [`match`]: https://doc.rust-lang.org/reference/expressions/match-expr.html +/// [`samples/rust/rust_minimal.rs`]: srctree/samples/rust/rust_minimal.rs +/// [`if let`]: https://doc.rust-lang.org/reference/expressions/if-expr.html#if-let-expressions +/// [`?`]: https://doc.rust-lang.org/reference/expressions/operator-expr.html#the-question-mark-operator +/// [`unwrap()`]: Result::unwrap +/// [`expect()`]: Result::expect +/// [`BUG()`]: https://docs.kernel.org/process/deprecated.html#bug-and-bug-on +/// [`unwrap_or()`]: Result::unwrap_or +/// [`unwrap_or_else()`]: Result::unwrap_or_else +/// [`unwrap_or_default()`]: Result::unwrap_or_default +/// [`unwrap_unchecked()`]: Result::unwrap_unchecked +/// [Rust documentation]: https://doc.rust-lang.org/book/ch09-02-recoverable-errors-with-result.html pub type Result<T = (), E = Error> = core::result::Result<T, E>; /// Converts an integer as returned by a C kernel function to an error if it's negative, and diff --git a/rust/kernel/faux.rs b/rust/kernel/faux.rs new file mode 100644 index 000000000000..8a50fcd4c9bb --- /dev/null +++ b/rust/kernel/faux.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0-only + +//! Abstractions for the faux bus. +//! +//! This module provides bindings for working with faux devices in kernel modules. +//! +//! C header: [`include/linux/device/faux.h`] + +use crate::{bindings, device, error::code::*, prelude::*}; +use core::ptr::{addr_of_mut, null, null_mut, NonNull}; + +/// The registration of a faux device. +/// +/// This type represents the registration of a [`struct faux_device`]. When an instance of this type +/// is dropped, its respective faux device will be unregistered from the system. +/// +/// # Invariants +/// +/// `self.0` always holds a valid pointer to an initialized and registered [`struct faux_device`]. +/// +/// [`struct faux_device`]: srctree/include/linux/device/faux.h +pub struct Registration(NonNull<bindings::faux_device>); + +impl Registration { + /// Create and register a new faux device with the given name. + #[inline] + pub fn new(name: &CStr, parent: Option<&device::Device>) -> Result<Self> { + // SAFETY: + // - `name` is copied by this function into its own storage + // - `faux_ops` is safe to leave NULL according to the C API + // - `parent` can be either NULL or a pointer to a `struct device`, and `faux_device_create` + // will take a reference to `parent` using `device_add` - ensuring that it remains valid + // for the lifetime of the faux device. + let dev = unsafe { + bindings::faux_device_create( + name.as_char_ptr(), + parent.map_or(null_mut(), |p| p.as_raw()), + null(), + ) + }; + + // The above function will return either a valid device, or NULL on failure + // INVARIANT: The device will remain registered until faux_device_destroy() is called, which + // happens in our Drop implementation. + Ok(Self(NonNull::new(dev).ok_or(ENODEV)?)) + } + + fn as_raw(&self) -> *mut bindings::faux_device { + self.0.as_ptr() + } +} + +impl AsRef<device::Device> for Registration { + fn as_ref(&self) -> &device::Device { + // SAFETY: The underlying `device` in `faux_device` is guaranteed by the C API to be + // a valid initialized `device`. + unsafe { device::Device::as_ref(addr_of_mut!((*self.as_raw()).dev)) } + } +} + +impl Drop for Registration { + #[inline] + fn drop(&mut self) { + // SAFETY: `self.0` is a valid registered faux_device via our type invariants. + unsafe { bindings::faux_device_destroy(self.as_raw()) } + } +} + +// SAFETY: The faux device API is thread-safe as guaranteed by the device core, as long as +// faux_device_destroy() is guaranteed to only be called once - which is guaranteed by our type not +// having Copy/Clone. +unsafe impl Send for Registration {} + +// SAFETY: The faux device API is thread-safe as guaranteed by the device core, as long as +// faux_device_destroy() is guaranteed to only be called once - which is guaranteed by our type not +// having Copy/Clone. +unsafe impl Sync for Registration {} diff --git a/rust/kernel/firmware.rs b/rust/kernel/firmware.rs index c5162fdc95ff..2494c96e105f 100644 --- a/rust/kernel/firmware.rs +++ b/rust/kernel/firmware.rs @@ -4,7 +4,7 @@ //! //! C header: [`include/linux/firmware.h`](srctree/include/linux/firmware.h) -use crate::{bindings, device::Device, error::Error, error::Result, str::CStr}; +use crate::{bindings, device::Device, error::Error, error::Result, ffi, str::CStr}; use core::ptr::NonNull; /// # Invariants @@ -12,7 +12,11 @@ use core::ptr::NonNull; /// One of the following: `bindings::request_firmware`, `bindings::firmware_request_nowarn`, /// `bindings::firmware_request_platform`, `bindings::request_firmware_direct`. struct FwFunc( - unsafe extern "C" fn(*mut *const bindings::firmware, *const u8, *mut bindings::device) -> i32, + unsafe extern "C" fn( + *mut *const bindings::firmware, + *const ffi::c_char, + *mut bindings::device, + ) -> i32, ); impl FwFunc { @@ -115,3 +119,219 @@ unsafe impl Send for Firmware {} // SAFETY: `Firmware` only holds a pointer to a C `struct firmware`, references to which are safe to // be used from any thread. unsafe impl Sync for Firmware {} + +/// Create firmware .modinfo entries. +/// +/// This macro is the counterpart of the C macro `MODULE_FIRMWARE()`, but instead of taking a +/// simple string literals, which is already covered by the `firmware` field of +/// [`crate::prelude::module!`], it allows the caller to pass a builder type, based on the +/// [`ModInfoBuilder`], which can create the firmware modinfo strings in a more flexible way. +/// +/// Drivers should extend the [`ModInfoBuilder`] with their own driver specific builder type. +/// +/// The `builder` argument must be a type which implements the following function. +/// +/// `const fn create(module_name: &'static CStr) -> ModInfoBuilder` +/// +/// `create` should pass the `module_name` to the [`ModInfoBuilder`] and, with the help of +/// it construct the corresponding firmware modinfo. +/// +/// Typically, such contracts would be enforced by a trait, however traits do not (yet) support +/// const functions. +/// +/// # Example +/// +/// ``` +/// # mod module_firmware_test { +/// # use kernel::firmware; +/// # use kernel::prelude::*; +/// # +/// # struct MyModule; +/// # +/// # impl kernel::Module for MyModule { +/// # fn init(_module: &'static ThisModule) -> Result<Self> { +/// # Ok(Self) +/// # } +/// # } +/// # +/// # +/// struct Builder<const N: usize>; +/// +/// impl<const N: usize> Builder<N> { +/// const DIR: &'static str = "vendor/chip/"; +/// const FILES: [&'static str; 3] = [ "foo", "bar", "baz" ]; +/// +/// const fn create(module_name: &'static kernel::str::CStr) -> firmware::ModInfoBuilder<N> { +/// let mut builder = firmware::ModInfoBuilder::new(module_name); +/// +/// let mut i = 0; +/// while i < Self::FILES.len() { +/// builder = builder.new_entry() +/// .push(Self::DIR) +/// .push(Self::FILES[i]) +/// .push(".bin"); +/// +/// i += 1; +/// } +/// +/// builder +/// } +/// } +/// +/// module! { +/// type: MyModule, +/// name: "module_firmware_test", +/// author: "Rust for Linux", +/// description: "module_firmware! test module", +/// license: "GPL", +/// } +/// +/// kernel::module_firmware!(Builder); +/// # } +/// ``` +#[macro_export] +macro_rules! module_firmware { + // The argument is the builder type without the const generic, since it's deferred from within + // this macro. Hence, we can neither use `expr` nor `ty`. + ($($builder:tt)*) => { + const _: () = { + const __MODULE_FIRMWARE_PREFIX: &'static $crate::str::CStr = if cfg!(MODULE) { + $crate::c_str!("") + } else { + <LocalModule as $crate::ModuleMetadata>::NAME + }; + + #[link_section = ".modinfo"] + #[used] + static __MODULE_FIRMWARE: [u8; $($builder)*::create(__MODULE_FIRMWARE_PREFIX) + .build_length()] = $($builder)*::create(__MODULE_FIRMWARE_PREFIX).build(); + }; + }; +} + +/// Builder for firmware module info. +/// +/// [`ModInfoBuilder`] is a helper component to flexibly compose firmware paths strings for the +/// .modinfo section in const context. +/// +/// Therefore the [`ModInfoBuilder`] provides the methods [`ModInfoBuilder::new_entry`] and +/// [`ModInfoBuilder::push`], where the latter is used to push path components and the former to +/// mark the beginning of a new path string. +/// +/// [`ModInfoBuilder`] is meant to be used in combination with [`kernel::module_firmware!`]. +/// +/// The const generic `N` as well as the `module_name` parameter of [`ModInfoBuilder::new`] is an +/// internal implementation detail and supplied through the above macro. +pub struct ModInfoBuilder<const N: usize> { + buf: [u8; N], + n: usize, + module_name: &'static CStr, +} + +impl<const N: usize> ModInfoBuilder<N> { + /// Create an empty builder instance. + pub const fn new(module_name: &'static CStr) -> Self { + Self { + buf: [0; N], + n: 0, + module_name, + } + } + + const fn push_internal(mut self, bytes: &[u8]) -> Self { + let mut j = 0; + + if N == 0 { + self.n += bytes.len(); + return self; + } + + while j < bytes.len() { + if self.n < N { + self.buf[self.n] = bytes[j]; + } + self.n += 1; + j += 1; + } + self + } + + /// Push an additional path component. + /// + /// Append path components to the [`ModInfoBuilder`] instance. Paths need to be separated + /// with [`ModInfoBuilder::new_entry`]. + /// + /// # Example + /// + /// ``` + /// use kernel::firmware::ModInfoBuilder; + /// + /// # const DIR: &str = "vendor/chip/"; + /// # const fn no_run<const N: usize>(builder: ModInfoBuilder<N>) { + /// let builder = builder.new_entry() + /// .push(DIR) + /// .push("foo.bin") + /// .new_entry() + /// .push(DIR) + /// .push("bar.bin"); + /// # } + /// ``` + pub const fn push(self, s: &str) -> Self { + // Check whether there has been an initial call to `next_entry()`. + if N != 0 && self.n == 0 { + crate::build_error!("Must call next_entry() before push()."); + } + + self.push_internal(s.as_bytes()) + } + + const fn push_module_name(self) -> Self { + let mut this = self; + let module_name = this.module_name; + + if !this.module_name.is_empty() { + this = this.push_internal(module_name.as_bytes_with_nul()); + + if N != 0 { + // Re-use the space taken by the NULL terminator and swap it with the '.' separator. + this.buf[this.n - 1] = b'.'; + } + } + + this + } + + /// Prepare the [`ModInfoBuilder`] for the next entry. + /// + /// This method acts as a separator between module firmware path entries. + /// + /// Must be called before constructing a new entry with subsequent calls to + /// [`ModInfoBuilder::push`]. + /// + /// See [`ModInfoBuilder::push`] for an example. + pub const fn new_entry(self) -> Self { + self.push_internal(b"\0") + .push_module_name() + .push_internal(b"firmware=") + } + + /// Build the byte array. + pub const fn build(self) -> [u8; N] { + // Add the final NULL terminator. + let this = self.push_internal(b"\0"); + + if this.n == N { + this.buf + } else { + crate::build_error!("Length mismatch."); + } + } +} + +impl ModInfoBuilder<0> { + /// Return the length of the byte array to build. + pub const fn build_length(self) -> usize { + // Compensate for the NULL terminator added by `build`. + self.n + 1 + } +} diff --git a/rust/kernel/fs/file.rs b/rust/kernel/fs/file.rs index e03dbe14d62a..138693bdeb3f 100644 --- a/rust/kernel/fs/file.rs +++ b/rust/kernel/fs/file.rs @@ -219,6 +219,7 @@ unsafe impl AlwaysRefCounted for File { /// must be on the same thread as this file. /// /// [`assume_no_fdget_pos`]: LocalFile::assume_no_fdget_pos +#[repr(transparent)] pub struct LocalFile { inner: Opaque<bindings::file>, } @@ -267,7 +268,7 @@ impl LocalFile { /// # Safety /// /// * The caller must ensure that `ptr` points at a valid file and that the file's refcount is - /// positive for the duration of 'a. + /// positive for the duration of `'a`. /// * The caller must ensure that if there is an active call to `fdget_pos` that did not take /// the `f_pos_lock` mutex, then that call is on the current thread. #[inline] @@ -341,7 +342,7 @@ impl File { /// # Safety /// /// * The caller must ensure that `ptr` points at a valid file and that the file's refcount is - /// positive for the duration of 'a. + /// positive for the duration of `'a`. /// * The caller must ensure that if there are active `fdget_pos` calls on this file, then they /// took the `f_pos_lock` mutex. #[inline] @@ -392,6 +393,7 @@ pub struct FileDescriptorReservation { impl FileDescriptorReservation { /// Creates a new file descriptor reservation. + #[inline] pub fn get_unused_fd_flags(flags: u32) -> Result<Self> { // SAFETY: FFI call, there are no safety requirements on `flags`. let fd: i32 = unsafe { bindings::get_unused_fd_flags(flags) }; @@ -405,6 +407,7 @@ impl FileDescriptorReservation { } /// Returns the file descriptor number that was reserved. + #[inline] pub fn reserved_fd(&self) -> u32 { self.fd } @@ -413,6 +416,7 @@ impl FileDescriptorReservation { /// /// The previously reserved file descriptor is bound to `file`. This method consumes the /// [`FileDescriptorReservation`], so it will not be usable after this call. + #[inline] pub fn fd_install(self, file: ARef<File>) { // SAFETY: `self.fd` was previously returned by `get_unused_fd_flags`. We have not yet used // the fd, so it is still valid, and `current` still refers to the same task, as this type @@ -433,6 +437,7 @@ impl FileDescriptorReservation { } impl Drop for FileDescriptorReservation { + #[inline] fn drop(&mut self) { // SAFETY: By the type invariants of this type, `self.fd` was previously returned by // `get_unused_fd_flags`. We have not yet used the fd, so it is still valid, and `current` diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index c962029f96e1..8d228c237954 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -1,131 +1,77 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT +// SPDX-License-Identifier: GPL-2.0 -//! API to safely and fallibly initialize pinned `struct`s using in-place constructors. -//! -//! It also allows in-place initialization of big `struct`s that would otherwise produce a stack -//! overflow. +//! Extensions to the [`pin-init`] crate. //! //! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential //! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move. //! -//! # Overview +//! The [`pin-init`] crate is the way such structs are initialized on the Rust side. Please refer +//! to its documentation to better understand how to use it. Additionally, there are many examples +//! throughout the kernel, such as the types from the [`sync`] module. And the ones presented +//! below. +//! +//! [`sync`]: crate::sync +//! [pinning]: https://doc.rust-lang.org/std/pin/index.html +//! [`pin-init`]: https://rust.docs.kernel.org/pin_init/ //! -//! To initialize a `struct` with an in-place constructor you will need two things: -//! - an in-place constructor, -//! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`], -//! [`UniqueArc<T>`], [`KBox<T>`] or any other smart pointer that implements [`InPlaceInit`]). +//! # [`Opaque<T>`] //! -//! To get an in-place constructor there are generally three options: -//! - directly creating an in-place constructor using the [`pin_init!`] macro, -//! - a custom function/macro returning an in-place constructor provided by someone else, -//! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer. +//! For the special case where initializing a field is a single FFI-function call that cannot fail, +//! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single +//! [`Opaque<T>`] field by just delegating to the supplied closure. You can use these in +//! combination with [`pin_init!`]. //! -//! Aside from pinned initialization, this API also supports in-place construction without pinning, -//! the macros/types/functions are generally named like the pinned variants without the `pin` -//! prefix. +//! [`Opaque<T>`]: crate::types::Opaque +//! [`Opaque::ffi_init`]: crate::types::Opaque::ffi_init +//! [`pin_init!`]: pin_init::pin_init //! //! # Examples //! -//! ## Using the [`pin_init!`] macro +//! ## General Examples //! -//! If you want to use [`PinInit`], then you will have to annotate your `struct` with -//! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for -//! [structurally pinned fields]. After doing this, you can then create an in-place constructor via -//! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is -//! that you need to write `<-` instead of `:` for fields that you want to initialize in-place. +//! ```rust,ignore +//! # #![allow(clippy::disallowed_names)] +//! use kernel::types::Opaque; +//! use pin_init::pin_init_from_closure; //! -//! ```rust -//! # #![expect(clippy::disallowed_names)] -//! use kernel::sync::{new_mutex, Mutex}; -//! # use core::pin::Pin; -//! #[pin_data] -//! struct Foo { -//! #[pin] -//! a: Mutex<usize>, -//! b: u32, +//! // assume we have some `raw_foo` type in C: +//! #[repr(C)] +//! struct RawFoo([u8; 16]); +//! extern { +//! fn init_foo(_: *mut RawFoo); //! } //! -//! let foo = pin_init!(Foo { -//! a <- new_mutex!(42, "Foo::a"), -//! b: 24, -//! }); -//! ``` -//! -//! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like -//! (or just the stack) to actually initialize a `Foo`: -//! -//! ```rust -//! # #![expect(clippy::disallowed_names)] -//! # use kernel::sync::{new_mutex, Mutex}; -//! # use core::pin::Pin; -//! # #[pin_data] -//! # struct Foo { -//! # #[pin] -//! # a: Mutex<usize>, -//! # b: u32, -//! # } -//! # let foo = pin_init!(Foo { -//! # a <- new_mutex!(42, "Foo::a"), -//! # b: 24, -//! # }); -//! let foo: Result<Pin<KBox<Foo>>> = KBox::pin_init(foo, GFP_KERNEL); -//! ``` -//! -//! For more information see the [`pin_init!`] macro. -//! -//! ## Using a custom function/macro that returns an initializer -//! -//! Many types from the kernel supply a function/macro that returns an initializer, because the -//! above method only works for types where you can access the fields. -//! -//! ```rust -//! # use kernel::sync::{new_mutex, Arc, Mutex}; -//! let mtx: Result<Arc<Mutex<usize>>> = -//! Arc::pin_init(new_mutex!(42, "example::mtx"), GFP_KERNEL); -//! ``` -//! -//! To declare an init macro/function you just return an [`impl PinInit<T, E>`]: -//! -//! ```rust -//! # use kernel::{sync::Mutex, new_mutex, init::PinInit, try_pin_init}; //! #[pin_data] -//! struct DriverData { +//! struct Foo { //! #[pin] -//! status: Mutex<i32>, -//! buffer: KBox<[u8; 1_000_000]>, +//! raw: Opaque<RawFoo>, //! } //! -//! impl DriverData { -//! fn new() -> impl PinInit<Self, Error> { -//! try_pin_init!(Self { -//! status <- new_mutex!(0, "DriverData::status"), -//! buffer: KBox::init(kernel::init::zeroed(), GFP_KERNEL)?, -//! }) +//! impl Foo { +//! fn setup(self: Pin<&mut Self>) { +//! pr_info!("Setting up foo\n"); //! } //! } -//! ``` //! -//! ## Manual creation of an initializer -//! -//! Often when working with primitives the previous approaches are not sufficient. That is where -//! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a -//! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure -//! actually does the initialization in the correct way. Here are the things to look out for -//! (we are calling the parameter to the closure `slot`): -//! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so -//! `slot` now contains a valid bit pattern for the type `T`, -//! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so -//! you need to take care to clean up anything if your initialization fails mid-way, -//! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of -//! `slot` gets called. +//! let foo = pin_init!(Foo { +//! raw <- unsafe { +//! Opaque::ffi_init(|s| { +//! // note that this cannot fail. +//! init_foo(s); +//! }) +//! }, +//! }).pin_chain(|foo| { +//! foo.setup(); +//! Ok(()) +//! }); +//! ``` //! -//! ```rust -//! # #![expect(unreachable_pub, clippy::disallowed_names)] -//! use kernel::{init, types::Opaque}; +//! ```rust,ignore +//! # #![allow(unreachable_pub, clippy::disallowed_names)] +//! use kernel::{prelude::*, types::Opaque}; //! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin}; //! # mod bindings { -//! # #![expect(non_camel_case_types)] -//! # #![expect(clippy::missing_safety_doc)] +//! # #![allow(non_camel_case_types)] //! # pub struct foo; //! # pub unsafe fn init_foo(_ptr: *mut foo) {} //! # pub unsafe fn destroy_foo(_ptr: *mut foo) {} @@ -133,7 +79,7 @@ //! # } //! # // `Error::from_errno` is `pub(crate)` in the `kernel` crate, thus provide a workaround. //! # trait FromErrno { -//! # fn from_errno(errno: kernel::ffi::c_int) -> Error { +//! # fn from_errno(errno: core::ffi::c_int) -> Error { //! # // Dummy error that can be constructed outside the `kernel` crate. //! # Error::from(core::fmt::Error) //! # } @@ -157,7 +103,7 @@ //! // enabled `foo`, //! // - when it returns `Err(e)`, then it has cleaned up before //! unsafe { -//! init::pin_init_from_closure(move |slot: *mut Self| { +//! pin_init::pin_init_from_closure(move |slot: *mut Self| { //! // `slot` contains uninit memory, avoid creating a reference. //! let foo = addr_of_mut!((*slot).foo); //! @@ -187,385 +133,114 @@ //! } //! } //! ``` -//! -//! For the special case where initializing a field is a single FFI-function call that cannot fail, -//! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single -//! [`Opaque`] field by just delegating to the supplied closure. You can use these in combination -//! with [`pin_init!`]. -//! -//! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside -//! the `kernel` crate. The [`sync`] module is a good starting point. -//! -//! [`sync`]: kernel::sync -//! [pinning]: https://doc.rust-lang.org/std/pin/index.html -//! [structurally pinned fields]: -//! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field -//! [stack]: crate::stack_pin_init -//! [`Arc<T>`]: crate::sync::Arc -//! [`impl PinInit<Foo>`]: PinInit -//! [`impl PinInit<T, E>`]: PinInit -//! [`impl Init<T, E>`]: Init -//! [`Opaque`]: kernel::types::Opaque -//! [`Opaque::ffi_init`]: kernel::types::Opaque::ffi_init -//! [`pin_data`]: ::macros::pin_data -//! [`pin_init!`]: crate::pin_init! use crate::{ - alloc::{AllocError, Flags, KBox}, + alloc::{AllocError, Flags}, error::{self, Error}, - sync::Arc, - sync::UniqueArc, - types::{Opaque, ScopeGuard}, -}; -use core::{ - cell::UnsafeCell, - convert::Infallible, - marker::PhantomData, - mem::MaybeUninit, - num::*, - pin::Pin, - ptr::{self, NonNull}, }; +use pin_init::{init_from_closure, pin_init_from_closure, Init, PinInit}; -#[doc(hidden)] -pub mod __internal; -#[doc(hidden)] -pub mod macros; +/// Smart pointer that can initialize memory in-place. +pub trait InPlaceInit<T>: Sized { + /// Pinned version of `Self`. + /// + /// If a type already implicitly pins its pointee, `Pin<Self>` is unnecessary. In this case use + /// `Self`, otherwise just use `Pin<Self>`. + type PinnedSelf; -/// Initialize and pin a type directly on the stack. -/// -/// # Examples -/// -/// ```rust -/// # #![expect(clippy::disallowed_names)] -/// # use kernel::{init, macros::pin_data, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex}; -/// # use core::pin::Pin; -/// #[pin_data] -/// struct Foo { -/// #[pin] -/// a: Mutex<usize>, -/// b: Bar, -/// } -/// -/// #[pin_data] -/// struct Bar { -/// x: u32, -/// } -/// -/// stack_pin_init!(let foo = pin_init!(Foo { -/// a <- new_mutex!(42), -/// b: Bar { -/// x: 64, -/// }, -/// })); -/// let foo: Pin<&mut Foo> = foo; -/// pr_info!("a: {}", &*foo.a.lock()); -/// ``` -/// -/// # Syntax -/// -/// A normal `let` binding with optional type annotation. The expression is expected to implement -/// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error -/// type, then use [`stack_try_pin_init!`]. -/// -/// [`stack_try_pin_init!`]: crate::stack_try_pin_init! -#[macro_export] -macro_rules! stack_pin_init { - (let $var:ident $(: $t:ty)? = $val:expr) => { - let val = $val; - let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit()); - let mut $var = match $crate::init::__internal::StackInit::init($var, val) { - Ok(res) => res, - Err(x) => { - let x: ::core::convert::Infallible = x; - match x {} - } + /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this + /// type. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E> + where + E: From<AllocError>; + + /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this + /// type. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Self::PinnedSelf> + where + Error: From<E>, + { + // SAFETY: We delegate to `init` and only change the error type. + let init = unsafe { + pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) }; - }; -} + Self::try_pin_init(init, flags) + } -/// Initialize and pin a type directly on the stack. -/// -/// # Examples -/// -/// ```rust,ignore -/// # #![expect(clippy::disallowed_names)] -/// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex}; -/// # use macros::pin_data; -/// # use core::{alloc::AllocError, pin::Pin}; -/// #[pin_data] -/// struct Foo { -/// #[pin] -/// a: Mutex<usize>, -/// b: KBox<Bar>, -/// } -/// -/// struct Bar { -/// x: u32, -/// } -/// -/// stack_try_pin_init!(let foo: Result<Pin<&mut Foo>, AllocError> = pin_init!(Foo { -/// a <- new_mutex!(42), -/// b: KBox::new(Bar { -/// x: 64, -/// }, GFP_KERNEL)?, -/// })); -/// let foo = foo.unwrap(); -/// pr_info!("a: {}", &*foo.a.lock()); -/// ``` -/// -/// ```rust,ignore -/// # #![expect(clippy::disallowed_names)] -/// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex}; -/// # use macros::pin_data; -/// # use core::{alloc::AllocError, pin::Pin}; -/// #[pin_data] -/// struct Foo { -/// #[pin] -/// a: Mutex<usize>, -/// b: KBox<Bar>, -/// } -/// -/// struct Bar { -/// x: u32, -/// } -/// -/// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo { -/// a <- new_mutex!(42), -/// b: KBox::new(Bar { -/// x: 64, -/// }, GFP_KERNEL)?, -/// })); -/// pr_info!("a: {}", &*foo.a.lock()); -/// # Ok::<_, AllocError>(()) -/// ``` -/// -/// # Syntax -/// -/// A normal `let` binding with optional type annotation. The expression is expected to implement -/// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the -/// `=` will propagate this error. -#[macro_export] -macro_rules! stack_try_pin_init { - (let $var:ident $(: $t:ty)? = $val:expr) => { - let val = $val; - let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit()); - let mut $var = $crate::init::__internal::StackInit::init($var, val); - }; - (let $var:ident $(: $t:ty)? =? $val:expr) => { - let val = $val; - let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit()); - let mut $var = $crate::init::__internal::StackInit::init($var, val)?; - }; + /// Use the given initializer to in-place initialize a `T`. + fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E> + where + E: From<AllocError>; + + /// Use the given initializer to in-place initialize a `T`. + fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self> + where + Error: From<E>, + { + // SAFETY: We delegate to `init` and only change the error type. + let init = unsafe { + init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) + }; + Self::try_init(init, flags) + } } -/// Construct an in-place, pinned initializer for `struct`s. -/// -/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use -/// [`try_pin_init!`]. -/// -/// The syntax is almost identical to that of a normal `struct` initializer: -/// -/// ```rust -/// # use kernel::{init, pin_init, macros::pin_data, init::*}; -/// # use core::pin::Pin; -/// #[pin_data] -/// struct Foo { -/// a: usize, -/// b: Bar, -/// } -/// -/// #[pin_data] -/// struct Bar { -/// x: u32, -/// } -/// -/// # fn demo() -> impl PinInit<Foo> { -/// let a = 42; -/// -/// let initializer = pin_init!(Foo { -/// a, -/// b: Bar { -/// x: 64, -/// }, -/// }); -/// # initializer } -/// # KBox::pin_init(demo(), GFP_KERNEL).unwrap(); -/// ``` -/// -/// Arbitrary Rust expressions can be used to set the value of a variable. -/// -/// The fields are initialized in the order that they appear in the initializer. So it is possible -/// to read already initialized fields using raw pointers. -/// -/// IMPORTANT: You are not allowed to create references to fields of the struct inside of the -/// initializer. -/// -/// # Init-functions -/// -/// When working with this API it is often desired to let others construct your types without -/// giving access to all fields. This is where you would normally write a plain function `new` -/// that would return a new instance of your type. With this API that is also possible. -/// However, there are a few extra things to keep in mind. -/// -/// To create an initializer function, simply declare it like this: -/// -/// ```rust -/// # use kernel::{init, pin_init, init::*}; -/// # use core::pin::Pin; -/// # #[pin_data] -/// # struct Foo { -/// # a: usize, -/// # b: Bar, -/// # } -/// # #[pin_data] -/// # struct Bar { -/// # x: u32, -/// # } -/// impl Foo { -/// fn new() -> impl PinInit<Self> { -/// pin_init!(Self { -/// a: 42, -/// b: Bar { -/// x: 64, -/// }, -/// }) -/// } -/// } -/// ``` +/// Construct an in-place fallible initializer for `struct`s. /// -/// Users of `Foo` can now create it like this: +/// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use +/// [`init!`]. /// -/// ```rust -/// # #![expect(clippy::disallowed_names)] -/// # use kernel::{init, pin_init, macros::pin_data, init::*}; -/// # use core::pin::Pin; -/// # #[pin_data] -/// # struct Foo { -/// # a: usize, -/// # b: Bar, -/// # } -/// # #[pin_data] -/// # struct Bar { -/// # x: u32, -/// # } -/// # impl Foo { -/// # fn new() -> impl PinInit<Self> { -/// # pin_init!(Self { -/// # a: 42, -/// # b: Bar { -/// # x: 64, -/// # }, -/// # }) -/// # } -/// # } -/// let foo = KBox::pin_init(Foo::new(), GFP_KERNEL); -/// ``` +/// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error, +/// append `? $type` after the `struct` initializer. +/// The safety caveats from [`try_pin_init!`] also apply: +/// - `unsafe` code must guarantee either full initialization or return an error and allow +/// deallocation of the memory. +/// - the fields are initialized in the order given in the initializer. +/// - no references to fields are allowed to be created inside of the initializer. /// -/// They can also easily embed it into their own `struct`s: +/// # Examples /// /// ```rust -/// # use kernel::{init, pin_init, macros::pin_data, init::*}; -/// # use core::pin::Pin; -/// # #[pin_data] -/// # struct Foo { -/// # a: usize, -/// # b: Bar, -/// # } -/// # #[pin_data] -/// # struct Bar { -/// # x: u32, -/// # } -/// # impl Foo { -/// # fn new() -> impl PinInit<Self> { -/// # pin_init!(Self { -/// # a: 42, -/// # b: Bar { -/// # x: 64, -/// # }, -/// # }) -/// # } -/// # } -/// #[pin_data] -/// struct FooContainer { -/// #[pin] -/// foo1: Foo, -/// #[pin] -/// foo2: Foo, -/// other: u32, +/// use kernel::error::Error; +/// use pin_init::zeroed; +/// struct BigBuf { +/// big: KBox<[u8; 1024 * 1024 * 1024]>, +/// small: [u8; 1024 * 1024], /// } /// -/// impl FooContainer { -/// fn new(other: u32) -> impl PinInit<Self> { -/// pin_init!(Self { -/// foo1 <- Foo::new(), -/// foo2 <- Foo::new(), -/// other, -/// }) +/// impl BigBuf { +/// fn new() -> impl Init<Self, Error> { +/// try_init!(Self { +/// big: KBox::init(zeroed(), GFP_KERNEL)?, +/// small: [0; 1024 * 1024], +/// }? Error) /// } /// } /// ``` /// -/// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`. -/// This signifies that the given field is initialized in-place. As with `struct` initializers, just -/// writing the field (in this case `other`) without `:` or `<-` means `other: other,`. -/// -/// # Syntax -/// -/// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with -/// the following modifications is expected: -/// - Fields that you want to initialize in-place have to use `<-` instead of `:`. -/// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`] -/// pointer named `this` inside of the initializer. -/// - Using struct update syntax one can place `..Zeroable::zeroed()` at the very end of the -/// struct, this initializes every field with 0 and then runs all initializers specified in the -/// body. This can only be done if [`Zeroable`] is implemented for the struct. -/// -/// For instance: -/// -/// ```rust -/// # use kernel::{macros::{Zeroable, pin_data}, pin_init}; -/// # use core::{ptr::addr_of_mut, marker::PhantomPinned}; -/// #[pin_data] -/// #[derive(Zeroable)] -/// struct Buf { -/// // `ptr` points into `buf`. -/// ptr: *mut u8, -/// buf: [u8; 64], -/// #[pin] -/// pin: PhantomPinned, -/// } -/// pin_init!(&this in Buf { -/// buf: [0; 64], -/// // SAFETY: TODO. -/// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() }, -/// pin: PhantomPinned, -/// }); -/// pin_init!(Buf { -/// buf: [1; 64], -/// ..Zeroable::zeroed() -/// }); -/// ``` -/// -/// [`try_pin_init!`]: kernel::try_pin_init -/// [`NonNull<Self>`]: core::ptr::NonNull -// For a detailed example of how this macro works, see the module documentation of the hidden -// module `__internal` inside of `init/__internal.rs`. +/// [`Infallible`]: core::convert::Infallible +/// [`init!`]: pin_init::init +/// [`try_pin_init!`]: crate::try_pin_init! +/// [`Error`]: crate::error::Error #[macro_export] -macro_rules! pin_init { +macro_rules! try_init { ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - $crate::__init_internal!( - @this($($this)?), - @typ($t $(::<$($generics),*>)?), - @fields($($fields)*), - @error(::core::convert::Infallible), - @data(PinData, use_data), - @has_data(HasPinData, __pin_data), - @construct_closure(pin_init_from_closure), - @munch_fields($($fields)*), - ) + ::pin_init::try_init!($(&$this in)? $t $(::<$($generics),* $(,)?>)? { + $($fields)* + }? $crate::error::Error) + }; + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + $($fields:tt)* + }? $err:ty) => { + ::pin_init::try_init!($(&$this in)? $t $(::<$($generics),* $(,)?>)? { + $($fields)* + }? $err) }; } @@ -587,7 +262,9 @@ macro_rules! pin_init { /// # Examples /// /// ```rust -/// use kernel::{init::{self, PinInit}, error::Error}; +/// # #![feature(new_uninit)] +/// use kernel::error::Error; +/// use pin_init::zeroed; /// #[pin_data] /// struct BigBuf { /// big: KBox<[u8; 1024 * 1024 * 1024]>, @@ -598,844 +275,31 @@ macro_rules! pin_init { /// impl BigBuf { /// fn new() -> impl PinInit<Self, Error> { /// try_pin_init!(Self { -/// big: KBox::init(init::zeroed(), GFP_KERNEL)?, +/// big: KBox::init(zeroed(), GFP_KERNEL)?, /// small: [0; 1024 * 1024], /// ptr: core::ptr::null_mut(), /// }? Error) /// } /// } /// ``` -// For a detailed example of how this macro works, see the module documentation of the hidden -// module `__internal` inside of `init/__internal.rs`. -#[macro_export] -macro_rules! try_pin_init { - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { - $($fields:tt)* - }) => { - $crate::__init_internal!( - @this($($this)?), - @typ($t $(::<$($generics),*>)? ), - @fields($($fields)*), - @error($crate::error::Error), - @data(PinData, use_data), - @has_data(HasPinData, __pin_data), - @construct_closure(pin_init_from_closure), - @munch_fields($($fields)*), - ) - }; - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { - $($fields:tt)* - }? $err:ty) => { - $crate::__init_internal!( - @this($($this)?), - @typ($t $(::<$($generics),*>)? ), - @fields($($fields)*), - @error($err), - @data(PinData, use_data), - @has_data(HasPinData, __pin_data), - @construct_closure(pin_init_from_closure), - @munch_fields($($fields)*), - ) - }; -} - -/// Construct an in-place initializer for `struct`s. -/// -/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use -/// [`try_init!`]. -/// -/// The syntax is identical to [`pin_init!`] and its safety caveats also apply: -/// - `unsafe` code must guarantee either full initialization or return an error and allow -/// deallocation of the memory. -/// - the fields are initialized in the order given in the initializer. -/// - no references to fields are allowed to be created inside of the initializer. -/// -/// This initializer is for initializing data in-place that might later be moved. If you want to -/// pin-initialize, use [`pin_init!`]. -/// -/// [`try_init!`]: crate::try_init! -// For a detailed example of how this macro works, see the module documentation of the hidden -// module `__internal` inside of `init/__internal.rs`. -#[macro_export] -macro_rules! init { - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { - $($fields:tt)* - }) => { - $crate::__init_internal!( - @this($($this)?), - @typ($t $(::<$($generics),*>)?), - @fields($($fields)*), - @error(::core::convert::Infallible), - @data(InitData, /*no use_data*/), - @has_data(HasInitData, __init_data), - @construct_closure(init_from_closure), - @munch_fields($($fields)*), - ) - } -} - -/// Construct an in-place fallible initializer for `struct`s. -/// -/// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use -/// [`init!`]. -/// -/// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error, -/// append `? $type` after the `struct` initializer. -/// The safety caveats from [`try_pin_init!`] also apply: -/// - `unsafe` code must guarantee either full initialization or return an error and allow -/// deallocation of the memory. -/// - the fields are initialized in the order given in the initializer. -/// - no references to fields are allowed to be created inside of the initializer. -/// -/// # Examples -/// -/// ```rust -/// use kernel::{alloc::KBox, init::{PinInit, zeroed}, error::Error}; -/// struct BigBuf { -/// big: KBox<[u8; 1024 * 1024 * 1024]>, -/// small: [u8; 1024 * 1024], -/// } /// -/// impl BigBuf { -/// fn new() -> impl Init<Self, Error> { -/// try_init!(Self { -/// big: KBox::init(zeroed(), GFP_KERNEL)?, -/// small: [0; 1024 * 1024], -/// }? Error) -/// } -/// } -/// ``` -// For a detailed example of how this macro works, see the module documentation of the hidden -// module `__internal` inside of `init/__internal.rs`. +/// [`Infallible`]: core::convert::Infallible +/// [`pin_init!`]: pin_init::pin_init +/// [`Error`]: crate::error::Error #[macro_export] -macro_rules! try_init { +macro_rules! try_pin_init { ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - $crate::__init_internal!( - @this($($this)?), - @typ($t $(::<$($generics),*>)?), - @fields($($fields)*), - @error($crate::error::Error), - @data(InitData, /*no use_data*/), - @has_data(HasInitData, __init_data), - @construct_closure(init_from_closure), - @munch_fields($($fields)*), - ) + ::pin_init::try_pin_init!($(&$this in)? $t $(::<$($generics),* $(,)?>)? { + $($fields)* + }? $crate::error::Error) }; ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }? $err:ty) => { - $crate::__init_internal!( - @this($($this)?), - @typ($t $(::<$($generics),*>)?), - @fields($($fields)*), - @error($err), - @data(InitData, /*no use_data*/), - @has_data(HasInitData, __init_data), - @construct_closure(init_from_closure), - @munch_fields($($fields)*), - ) + ::pin_init::try_pin_init!($(&$this in)? $t $(::<$($generics),* $(,)?>)? { + $($fields)* + }? $err) }; } - -/// Asserts that a field on a struct using `#[pin_data]` is marked with `#[pin]` ie. that it is -/// structurally pinned. -/// -/// # Example -/// -/// This will succeed: -/// ``` -/// use kernel::assert_pinned; -/// #[pin_data] -/// struct MyStruct { -/// #[pin] -/// some_field: u64, -/// } -/// -/// assert_pinned!(MyStruct, some_field, u64); -/// ``` -/// -/// This will fail: -// TODO: replace with `compile_fail` when supported. -/// ```ignore -/// use kernel::assert_pinned; -/// #[pin_data] -/// struct MyStruct { -/// some_field: u64, -/// } -/// -/// assert_pinned!(MyStruct, some_field, u64); -/// ``` -/// -/// Some uses of the macro may trigger the `can't use generic parameters from outer item` error. To -/// work around this, you may pass the `inline` parameter to the macro. The `inline` parameter can -/// only be used when the macro is invoked from a function body. -/// ``` -/// use kernel::assert_pinned; -/// #[pin_data] -/// struct Foo<T> { -/// #[pin] -/// elem: T, -/// } -/// -/// impl<T> Foo<T> { -/// fn project(self: Pin<&mut Self>) -> Pin<&mut T> { -/// assert_pinned!(Foo<T>, elem, T, inline); -/// -/// // SAFETY: The field is structurally pinned. -/// unsafe { self.map_unchecked_mut(|me| &mut me.elem) } -/// } -/// } -/// ``` -#[macro_export] -macro_rules! assert_pinned { - ($ty:ty, $field:ident, $field_ty:ty, inline) => { - let _ = move |ptr: *mut $field_ty| { - // SAFETY: This code is unreachable. - let data = unsafe { <$ty as $crate::init::__internal::HasPinData>::__pin_data() }; - let init = $crate::init::__internal::AlwaysFail::<$field_ty>::new(); - // SAFETY: This code is unreachable. - unsafe { data.$field(ptr, init) }.ok(); - }; - }; - - ($ty:ty, $field:ident, $field_ty:ty) => { - const _: () = { - $crate::assert_pinned!($ty, $field, $field_ty, inline); - }; - }; -} - -/// A pin-initializer for the type `T`. -/// -/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can -/// be [`KBox<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use -/// the [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc<T>`] on this. -/// -/// Also see the [module description](self). -/// -/// # Safety -/// -/// When implementing this trait you will need to take great care. Also there are probably very few -/// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible. -/// -/// The [`PinInit::__pinned_init`] function: -/// - returns `Ok(())` if it initialized every field of `slot`, -/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: -/// - `slot` can be deallocated without UB occurring, -/// - `slot` does not need to be dropped, -/// - `slot` is not partially initialized. -/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. -/// -/// [`Arc<T>`]: crate::sync::Arc -/// [`Arc::pin_init`]: crate::sync::Arc::pin_init -#[must_use = "An initializer must be used in order to create its value."] -pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { - /// Initializes `slot`. - /// - /// # Safety - /// - /// - `slot` is a valid pointer to uninitialized memory. - /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to - /// deallocate. - /// - `slot` will not move until it is dropped, i.e. it will be pinned. - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>; - - /// First initializes the value using `self` then calls the function `f` with the initialized - /// value. - /// - /// If `f` returns an error the value is dropped and the initializer will forward the error. - /// - /// # Examples - /// - /// ```rust - /// # #![expect(clippy::disallowed_names)] - /// use kernel::{types::Opaque, init::pin_init_from_closure}; - /// #[repr(C)] - /// struct RawFoo([u8; 16]); - /// extern "C" { - /// fn init_foo(_: *mut RawFoo); - /// } - /// - /// #[pin_data] - /// struct Foo { - /// #[pin] - /// raw: Opaque<RawFoo>, - /// } - /// - /// impl Foo { - /// fn setup(self: Pin<&mut Self>) { - /// pr_info!("Setting up foo"); - /// } - /// } - /// - /// let foo = pin_init!(Foo { - /// // SAFETY: TODO. - /// raw <- unsafe { - /// Opaque::ffi_init(|s| { - /// init_foo(s); - /// }) - /// }, - /// }).pin_chain(|foo| { - /// foo.setup(); - /// Ok(()) - /// }); - /// ``` - fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E> - where - F: FnOnce(Pin<&mut T>) -> Result<(), E>, - { - ChainPinInit(self, f, PhantomData) - } -} - -/// An initializer returned by [`PinInit::pin_chain`]. -pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, KBox<T>)>); - -// SAFETY: The `__pinned_init` function is implemented such that it -// - returns `Ok(())` on successful initialization, -// - returns `Err(err)` on error and in this case `slot` will be dropped. -// - considers `slot` pinned. -unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainPinInit<I, F, T, E> -where - I: PinInit<T, E>, - F: FnOnce(Pin<&mut T>) -> Result<(), E>, -{ - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: All requirements fulfilled since this function is `__pinned_init`. - unsafe { self.0.__pinned_init(slot)? }; - // SAFETY: The above call initialized `slot` and we still have unique access. - let val = unsafe { &mut *slot }; - // SAFETY: `slot` is considered pinned. - let val = unsafe { Pin::new_unchecked(val) }; - // SAFETY: `slot` was initialized above. - (self.1)(val).inspect_err(|_| unsafe { core::ptr::drop_in_place(slot) }) - } -} - -/// An initializer for `T`. -/// -/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can -/// be [`KBox<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use -/// the [`InPlaceInit::init`] function of a smart pointer like [`Arc<T>`] on this. Because -/// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well. -/// -/// Also see the [module description](self). -/// -/// # Safety -/// -/// When implementing this trait you will need to take great care. Also there are probably very few -/// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible. -/// -/// The [`Init::__init`] function: -/// - returns `Ok(())` if it initialized every field of `slot`, -/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: -/// - `slot` can be deallocated without UB occurring, -/// - `slot` does not need to be dropped, -/// - `slot` is not partially initialized. -/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. -/// -/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same -/// code as `__init`. -/// -/// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to -/// move the pointee after initialization. -/// -/// [`Arc<T>`]: crate::sync::Arc -#[must_use = "An initializer must be used in order to create its value."] -pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { - /// Initializes `slot`. - /// - /// # Safety - /// - /// - `slot` is a valid pointer to uninitialized memory. - /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to - /// deallocate. - unsafe fn __init(self, slot: *mut T) -> Result<(), E>; - - /// First initializes the value using `self` then calls the function `f` with the initialized - /// value. - /// - /// If `f` returns an error the value is dropped and the initializer will forward the error. - /// - /// # Examples - /// - /// ```rust - /// # #![expect(clippy::disallowed_names)] - /// use kernel::{types::Opaque, init::{self, init_from_closure}}; - /// struct Foo { - /// buf: [u8; 1_000_000], - /// } - /// - /// impl Foo { - /// fn setup(&mut self) { - /// pr_info!("Setting up foo"); - /// } - /// } - /// - /// let foo = init!(Foo { - /// buf <- init::zeroed() - /// }).chain(|foo| { - /// foo.setup(); - /// Ok(()) - /// }); - /// ``` - fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E> - where - F: FnOnce(&mut T) -> Result<(), E>, - { - ChainInit(self, f, PhantomData) - } -} - -/// An initializer returned by [`Init::chain`]. -pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, KBox<T>)>); - -// SAFETY: The `__init` function is implemented such that it -// - returns `Ok(())` on successful initialization, -// - returns `Err(err)` on error and in this case `slot` will be dropped. -unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E> -where - I: Init<T, E>, - F: FnOnce(&mut T) -> Result<(), E>, -{ - unsafe fn __init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: All requirements fulfilled since this function is `__init`. - unsafe { self.0.__pinned_init(slot)? }; - // SAFETY: The above call initialized `slot` and we still have unique access. - (self.1)(unsafe { &mut *slot }).inspect_err(|_| - // SAFETY: `slot` was initialized above. - unsafe { core::ptr::drop_in_place(slot) }) - } -} - -// SAFETY: `__pinned_init` behaves exactly the same as `__init`. -unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E> -where - I: Init<T, E>, - F: FnOnce(&mut T) -> Result<(), E>, -{ - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: `__init` has less strict requirements compared to `__pinned_init`. - unsafe { self.__init(slot) } - } -} - -/// Creates a new [`PinInit<T, E>`] from the given closure. -/// -/// # Safety -/// -/// The closure: -/// - returns `Ok(())` if it initialized every field of `slot`, -/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: -/// - `slot` can be deallocated without UB occurring, -/// - `slot` does not need to be dropped, -/// - `slot` is not partially initialized. -/// - may assume that the `slot` does not move if `T: !Unpin`, -/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. -#[inline] -pub const unsafe fn pin_init_from_closure<T: ?Sized, E>( - f: impl FnOnce(*mut T) -> Result<(), E>, -) -> impl PinInit<T, E> { - __internal::InitClosure(f, PhantomData) -} - -/// Creates a new [`Init<T, E>`] from the given closure. -/// -/// # Safety -/// -/// The closure: -/// - returns `Ok(())` if it initialized every field of `slot`, -/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: -/// - `slot` can be deallocated without UB occurring, -/// - `slot` does not need to be dropped, -/// - `slot` is not partially initialized. -/// - the `slot` may move after initialization. -/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. -#[inline] -pub const unsafe fn init_from_closure<T: ?Sized, E>( - f: impl FnOnce(*mut T) -> Result<(), E>, -) -> impl Init<T, E> { - __internal::InitClosure(f, PhantomData) -} - -/// An initializer that leaves the memory uninitialized. -/// -/// The initializer is a no-op. The `slot` memory is not changed. -#[inline] -pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { - // SAFETY: The memory is allowed to be uninitialized. - unsafe { init_from_closure(|_| Ok(())) } -} - -/// Initializes an array by initializing each element via the provided initializer. -/// -/// # Examples -/// -/// ```rust -/// use kernel::{alloc::KBox, error::Error, init::init_array_from_fn}; -/// let array: KBox<[usize; 1_000]> = -/// KBox::init::<Error>(init_array_from_fn(|i| i), GFP_KERNEL).unwrap(); -/// assert_eq!(array.len(), 1_000); -/// ``` -pub fn init_array_from_fn<I, const N: usize, T, E>( - mut make_init: impl FnMut(usize) -> I, -) -> impl Init<[T; N], E> -where - I: Init<T, E>, -{ - let init = move |slot: *mut [T; N]| { - let slot = slot.cast::<T>(); - // Counts the number of initialized elements and when dropped drops that many elements from - // `slot`. - let mut init_count = ScopeGuard::new_with_data(0, |i| { - // We now free every element that has been initialized before. - // SAFETY: The loop initialized exactly the values from 0..i and since we - // return `Err` below, the caller will consider the memory at `slot` as - // uninitialized. - unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; - }); - for i in 0..N { - let init = make_init(i); - // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. - let ptr = unsafe { slot.add(i) }; - // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` - // requirements. - unsafe { init.__init(ptr) }?; - *init_count += 1; - } - init_count.dismiss(); - Ok(()) - }; - // SAFETY: The initializer above initializes every element of the array. On failure it drops - // any initialized elements and returns `Err`. - unsafe { init_from_closure(init) } -} - -/// Initializes an array by initializing each element via the provided initializer. -/// -/// # Examples -/// -/// ```rust -/// use kernel::{sync::{Arc, Mutex}, init::pin_init_array_from_fn, new_mutex}; -/// let array: Arc<[Mutex<usize>; 1_000]> = -/// Arc::pin_init(pin_init_array_from_fn(|i| new_mutex!(i)), GFP_KERNEL).unwrap(); -/// assert_eq!(array.len(), 1_000); -/// ``` -pub fn pin_init_array_from_fn<I, const N: usize, T, E>( - mut make_init: impl FnMut(usize) -> I, -) -> impl PinInit<[T; N], E> -where - I: PinInit<T, E>, -{ - let init = move |slot: *mut [T; N]| { - let slot = slot.cast::<T>(); - // Counts the number of initialized elements and when dropped drops that many elements from - // `slot`. - let mut init_count = ScopeGuard::new_with_data(0, |i| { - // We now free every element that has been initialized before. - // SAFETY: The loop initialized exactly the values from 0..i and since we - // return `Err` below, the caller will consider the memory at `slot` as - // uninitialized. - unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; - }); - for i in 0..N { - let init = make_init(i); - // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. - let ptr = unsafe { slot.add(i) }; - // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` - // requirements. - unsafe { init.__pinned_init(ptr) }?; - *init_count += 1; - } - init_count.dismiss(); - Ok(()) - }; - // SAFETY: The initializer above initializes every element of the array. On failure it drops - // any initialized elements and returns `Err`. - unsafe { pin_init_from_closure(init) } -} - -// SAFETY: Every type can be initialized by-value. -unsafe impl<T, E> Init<T, E> for T { - unsafe fn __init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: TODO. - unsafe { slot.write(self) }; - Ok(()) - } -} - -// SAFETY: Every type can be initialized by-value. `__pinned_init` calls `__init`. -unsafe impl<T, E> PinInit<T, E> for T { - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: TODO. - unsafe { self.__init(slot) } - } -} - -/// Smart pointer that can initialize memory in-place. -pub trait InPlaceInit<T>: Sized { - /// Pinned version of `Self`. - /// - /// If a type already implicitly pins its pointee, `Pin<Self>` is unnecessary. In this case use - /// `Self`, otherwise just use `Pin<Self>`. - type PinnedSelf; - - /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this - /// type. - /// - /// If `T: !Unpin` it will not be able to move afterwards. - fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E> - where - E: From<AllocError>; - - /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this - /// type. - /// - /// If `T: !Unpin` it will not be able to move afterwards. - fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Self::PinnedSelf> - where - Error: From<E>, - { - // SAFETY: We delegate to `init` and only change the error type. - let init = unsafe { - pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) - }; - Self::try_pin_init(init, flags) - } - - /// Use the given initializer to in-place initialize a `T`. - fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E> - where - E: From<AllocError>; - - /// Use the given initializer to in-place initialize a `T`. - fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self> - where - Error: From<E>, - { - // SAFETY: We delegate to `init` and only change the error type. - let init = unsafe { - init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) - }; - Self::try_init(init, flags) - } -} - -impl<T> InPlaceInit<T> for Arc<T> { - type PinnedSelf = Self; - - #[inline] - fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E> - where - E: From<AllocError>, - { - UniqueArc::try_pin_init(init, flags).map(|u| u.into()) - } - - #[inline] - fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E> - where - E: From<AllocError>, - { - UniqueArc::try_init(init, flags).map(|u| u.into()) - } -} - -impl<T> InPlaceInit<T> for UniqueArc<T> { - type PinnedSelf = Pin<Self>; - - #[inline] - fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E> - where - E: From<AllocError>, - { - UniqueArc::new_uninit(flags)?.write_pin_init(init) - } - - #[inline] - fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E> - where - E: From<AllocError>, - { - UniqueArc::new_uninit(flags)?.write_init(init) - } -} - -/// Smart pointer containing uninitialized memory and that can write a value. -pub trait InPlaceWrite<T> { - /// The type `Self` turns into when the contents are initialized. - type Initialized; - - /// Use the given initializer to write a value into `self`. - /// - /// Does not drop the current value and considers it as uninitialized memory. - fn write_init<E>(self, init: impl Init<T, E>) -> Result<Self::Initialized, E>; - - /// Use the given pin-initializer to write a value into `self`. - /// - /// Does not drop the current value and considers it as uninitialized memory. - fn write_pin_init<E>(self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E>; -} - -impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> { - type Initialized = UniqueArc<T>; - - fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> { - let slot = self.as_mut_ptr(); - // SAFETY: When init errors/panics, slot will get deallocated but not dropped, - // slot is valid. - unsafe { init.__init(slot)? }; - // SAFETY: All fields have been initialized. - Ok(unsafe { self.assume_init() }) - } - - fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> { - let slot = self.as_mut_ptr(); - // SAFETY: When init errors/panics, slot will get deallocated but not dropped, - // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; - // SAFETY: All fields have been initialized. - Ok(unsafe { self.assume_init() }.into()) - } -} - -/// Trait facilitating pinned destruction. -/// -/// Use [`pinned_drop`] to implement this trait safely: -/// -/// ```rust -/// # use kernel::sync::Mutex; -/// use kernel::macros::pinned_drop; -/// use core::pin::Pin; -/// #[pin_data(PinnedDrop)] -/// struct Foo { -/// #[pin] -/// mtx: Mutex<usize>, -/// } -/// -/// #[pinned_drop] -/// impl PinnedDrop for Foo { -/// fn drop(self: Pin<&mut Self>) { -/// pr_info!("Foo is being dropped!"); -/// } -/// } -/// ``` -/// -/// # Safety -/// -/// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl. -/// -/// [`pinned_drop`]: kernel::macros::pinned_drop -pub unsafe trait PinnedDrop: __internal::HasPinData { - /// Executes the pinned destructor of this type. - /// - /// While this function is marked safe, it is actually unsafe to call it manually. For this - /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code - /// and thus prevents this function from being called where it should not. - /// - /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute - /// automatically. - fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop); -} - -/// Marker trait for types that can be initialized by writing just zeroes. -/// -/// # Safety -/// -/// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words, -/// this is not UB: -/// -/// ```rust,ignore -/// let val: Self = unsafe { core::mem::zeroed() }; -/// ``` -pub unsafe trait Zeroable {} - -/// Create a new zeroed T. -/// -/// The returned initializer will write `0x00` to every byte of the given `slot`. -#[inline] -pub fn zeroed<T: Zeroable>() -> impl Init<T> { - // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T` - // and because we write all zeroes, the memory is initialized. - unsafe { - init_from_closure(|slot: *mut T| { - slot.write_bytes(0, 1); - Ok(()) - }) - } -} - -macro_rules! impl_zeroable { - ($($({$($generics:tt)*})? $t:ty, )*) => { - // SAFETY: Safety comments written in the macro invocation. - $(unsafe impl$($($generics)*)? Zeroable for $t {})* - }; -} - -impl_zeroable! { - // SAFETY: All primitives that are allowed to be zero. - bool, - char, - u8, u16, u32, u64, u128, usize, - i8, i16, i32, i64, i128, isize, - f32, f64, - - // Note: do not add uninhabited types (such as `!` or `core::convert::Infallible`) to this list; - // creating an instance of an uninhabited type is immediate undefined behavior. For more on - // uninhabited/empty types, consult The Rustonomicon: - // <https://doc.rust-lang.org/stable/nomicon/exotic-sizes.html#empty-types>. The Rust Reference - // also has information on undefined behavior: - // <https://doc.rust-lang.org/stable/reference/behavior-considered-undefined.html>. - // - // SAFETY: These are inhabited ZSTs; there is nothing to zero and a valid value exists. - {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, (), - - // SAFETY: Type is allowed to take any value, including all zeros. - {<T>} MaybeUninit<T>, - // SAFETY: Type is allowed to take any value, including all zeros. - {<T>} Opaque<T>, - - // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`. - {<T: ?Sized + Zeroable>} UnsafeCell<T>, - - // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee). - Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>, - Option<NonZeroU128>, Option<NonZeroUsize>, - Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>, - Option<NonZeroI128>, Option<NonZeroIsize>, - - // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee). - // - // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant. - {<T: ?Sized>} Option<NonNull<T>>, - {<T: ?Sized>} Option<KBox<T>>, - - // SAFETY: `null` pointer is valid. - // - // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be - // null. - // - // When `Pointee` gets stabilized, we could use - // `T: ?Sized where <T as Pointee>::Metadata: Zeroable` - {<T>} *mut T, {<T>} *const T, - - // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be - // zero. - {<T>} *mut [T], {<T>} *const [T], *mut str, *const str, - - // SAFETY: `T` is `Zeroable`. - {<const N: usize, T: Zeroable>} [T; N], {<T: Zeroable>} Wrapping<T>, -} - -macro_rules! impl_tuple_zeroable { - ($(,)?) => {}; - ($first:ident, $($t:ident),* $(,)?) => { - // SAFETY: All elements are zeroable and padding can be zero. - unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {} - impl_tuple_zeroable!($($t),* ,); - } -} - -impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J); diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs new file mode 100644 index 000000000000..72d80a6f131e --- /dev/null +++ b/rust/kernel/io.rs @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Memory-mapped IO. +//! +//! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h) + +use crate::error::{code::EINVAL, Result}; +use crate::{bindings, build_assert}; + +/// Raw representation of an MMIO region. +/// +/// By itself, the existence of an instance of this structure does not provide any guarantees that +/// the represented MMIO region does exist or is properly mapped. +/// +/// Instead, the bus specific MMIO implementation must convert this raw representation into an `Io` +/// instance providing the actual memory accessors. Only by the conversion into an `Io` structure +/// any guarantees are given. +pub struct IoRaw<const SIZE: usize = 0> { + addr: usize, + maxsize: usize, +} + +impl<const SIZE: usize> IoRaw<SIZE> { + /// Returns a new `IoRaw` instance on success, an error otherwise. + pub fn new(addr: usize, maxsize: usize) -> Result<Self> { + if maxsize < SIZE { + return Err(EINVAL); + } + + Ok(Self { addr, maxsize }) + } + + /// Returns the base address of the MMIO region. + #[inline] + pub fn addr(&self) -> usize { + self.addr + } + + /// Returns the maximum size of the MMIO region. + #[inline] + pub fn maxsize(&self) -> usize { + self.maxsize + } +} + +/// IO-mapped memory, starting at the base address @addr and spanning @maxlen bytes. +/// +/// The creator (usually a subsystem / bus such as PCI) is responsible for creating the +/// mapping, performing an additional region request etc. +/// +/// # Invariant +/// +/// `addr` is the start and `maxsize` the length of valid I/O mapped memory region of size +/// `maxsize`. +/// +/// # Examples +/// +/// ```no_run +/// # use kernel::{bindings, io::{Io, IoRaw}}; +/// # use core::ops::Deref; +/// +/// // See also [`pci::Bar`] for a real example. +/// struct IoMem<const SIZE: usize>(IoRaw<SIZE>); +/// +/// impl<const SIZE: usize> IoMem<SIZE> { +/// /// # Safety +/// /// +/// /// [`paddr`, `paddr` + `SIZE`) must be a valid MMIO region that is mappable into the CPUs +/// /// virtual address space. +/// unsafe fn new(paddr: usize) -> Result<Self>{ +/// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is +/// // valid for `ioremap`. +/// let addr = unsafe { bindings::ioremap(paddr as _, SIZE as _) }; +/// if addr.is_null() { +/// return Err(ENOMEM); +/// } +/// +/// Ok(IoMem(IoRaw::new(addr as _, SIZE)?)) +/// } +/// } +/// +/// impl<const SIZE: usize> Drop for IoMem<SIZE> { +/// fn drop(&mut self) { +/// // SAFETY: `self.0.addr()` is guaranteed to be properly mapped by `Self::new`. +/// unsafe { bindings::iounmap(self.0.addr() as _); }; +/// } +/// } +/// +/// impl<const SIZE: usize> Deref for IoMem<SIZE> { +/// type Target = Io<SIZE>; +/// +/// fn deref(&self) -> &Self::Target { +/// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`. +/// unsafe { Io::from_raw(&self.0) } +/// } +/// } +/// +///# fn no_run() -> Result<(), Error> { +/// // SAFETY: Invalid usage for example purposes. +/// let iomem = unsafe { IoMem::<{ core::mem::size_of::<u32>() }>::new(0xBAAAAAAD)? }; +/// iomem.write32(0x42, 0x0); +/// assert!(iomem.try_write32(0x42, 0x0).is_ok()); +/// assert!(iomem.try_write32(0x42, 0x4).is_err()); +/// # Ok(()) +/// # } +/// ``` +#[repr(transparent)] +pub struct Io<const SIZE: usize = 0>(IoRaw<SIZE>); + +macro_rules! define_read { + ($(#[$attr:meta])* $name:ident, $try_name:ident, $c_fn:ident -> $type_name:ty) => { + /// Read IO data from a given offset known at compile time. + /// + /// Bound checks are performed on compile time, hence if the offset is not known at compile + /// time, the build will fail. + $(#[$attr])* + #[inline] + pub fn $name(&self, offset: usize) -> $type_name { + let addr = self.io_addr_assert::<$type_name>(offset); + + // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. + unsafe { bindings::$c_fn(addr as _) } + } + + /// Read IO data from a given offset. + /// + /// Bound checks are performed on runtime, it fails if the offset (plus the type size) is + /// out of bounds. + $(#[$attr])* + pub fn $try_name(&self, offset: usize) -> Result<$type_name> { + let addr = self.io_addr::<$type_name>(offset)?; + + // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. + Ok(unsafe { bindings::$c_fn(addr as _) }) + } + }; +} + +macro_rules! define_write { + ($(#[$attr:meta])* $name:ident, $try_name:ident, $c_fn:ident <- $type_name:ty) => { + /// Write IO data from a given offset known at compile time. + /// + /// Bound checks are performed on compile time, hence if the offset is not known at compile + /// time, the build will fail. + $(#[$attr])* + #[inline] + pub fn $name(&self, value: $type_name, offset: usize) { + let addr = self.io_addr_assert::<$type_name>(offset); + + // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. + unsafe { bindings::$c_fn(value, addr as _, ) } + } + + /// Write IO data from a given offset. + /// + /// Bound checks are performed on runtime, it fails if the offset (plus the type size) is + /// out of bounds. + $(#[$attr])* + pub fn $try_name(&self, value: $type_name, offset: usize) -> Result { + let addr = self.io_addr::<$type_name>(offset)?; + + // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. + unsafe { bindings::$c_fn(value, addr as _) } + Ok(()) + } + }; +} + +impl<const SIZE: usize> Io<SIZE> { + /// Converts an `IoRaw` into an `Io` instance, providing the accessors to the MMIO mapping. + /// + /// # Safety + /// + /// Callers must ensure that `addr` is the start of a valid I/O mapped memory region of size + /// `maxsize`. + pub unsafe fn from_raw(raw: &IoRaw<SIZE>) -> &Self { + // SAFETY: `Io` is a transparent wrapper around `IoRaw`. + unsafe { &*core::ptr::from_ref(raw).cast() } + } + + /// Returns the base address of this mapping. + #[inline] + pub fn addr(&self) -> usize { + self.0.addr() + } + + /// Returns the maximum size of this mapping. + #[inline] + pub fn maxsize(&self) -> usize { + self.0.maxsize() + } + + #[inline] + const fn offset_valid<U>(offset: usize, size: usize) -> bool { + let type_size = core::mem::size_of::<U>(); + if let Some(end) = offset.checked_add(type_size) { + end <= size && offset % type_size == 0 + } else { + false + } + } + + #[inline] + fn io_addr<U>(&self, offset: usize) -> Result<usize> { + if !Self::offset_valid::<U>(offset, self.maxsize()) { + return Err(EINVAL); + } + + // Probably no need to check, since the safety requirements of `Self::new` guarantee that + // this can't overflow. + self.addr().checked_add(offset).ok_or(EINVAL) + } + + #[inline] + fn io_addr_assert<U>(&self, offset: usize) -> usize { + build_assert!(Self::offset_valid::<U>(offset, SIZE)); + + self.addr() + offset + } + + define_read!(read8, try_read8, readb -> u8); + define_read!(read16, try_read16, readw -> u16); + define_read!(read32, try_read32, readl -> u32); + define_read!( + #[cfg(CONFIG_64BIT)] + read64, + try_read64, + readq -> u64 + ); + + define_read!(read8_relaxed, try_read8_relaxed, readb_relaxed -> u8); + define_read!(read16_relaxed, try_read16_relaxed, readw_relaxed -> u16); + define_read!(read32_relaxed, try_read32_relaxed, readl_relaxed -> u32); + define_read!( + #[cfg(CONFIG_64BIT)] + read64_relaxed, + try_read64_relaxed, + readq_relaxed -> u64 + ); + + define_write!(write8, try_write8, writeb <- u8); + define_write!(write16, try_write16, writew <- u16); + define_write!(write32, try_write32, writel <- u32); + define_write!( + #[cfg(CONFIG_64BIT)] + write64, + try_write64, + writeq <- u64 + ); + + define_write!(write8_relaxed, try_write8_relaxed, writeb_relaxed <- u8); + define_write!(write16_relaxed, try_write16_relaxed, writew_relaxed <- u16); + define_write!(write32_relaxed, try_write32_relaxed, writel_relaxed <- u32); + define_write!( + #[cfg(CONFIG_64BIT)] + write64_relaxed, + try_write64_relaxed, + writeq_relaxed <- u64 + ); +} diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs index 824da0e9738a..1604fb6a5b1b 100644 --- a/rust/kernel/kunit.rs +++ b/rust/kernel/kunit.rs @@ -40,6 +40,8 @@ pub fn info(args: fmt::Arguments<'_>) { } } +use macros::kunit_tests; + /// Asserts that a boolean expression is `true` at runtime. /// /// Public but hidden since it should only be used from generated tests. @@ -161,3 +163,172 @@ macro_rules! kunit_assert_eq { $crate::kunit_assert!($name, $file, $diff, $left == $right); }}; } + +/// Represents an individual test case. +/// +/// The [`kunit_unsafe_test_suite!`] macro expects a NULL-terminated list of valid test cases. +/// Use [`kunit_case_null`] to generate such a delimiter. +#[doc(hidden)] +pub const fn kunit_case( + name: &'static kernel::str::CStr, + run_case: unsafe extern "C" fn(*mut kernel::bindings::kunit), +) -> kernel::bindings::kunit_case { + kernel::bindings::kunit_case { + run_case: Some(run_case), + name: name.as_char_ptr(), + attr: kernel::bindings::kunit_attributes { + speed: kernel::bindings::kunit_speed_KUNIT_SPEED_NORMAL, + }, + generate_params: None, + status: kernel::bindings::kunit_status_KUNIT_SUCCESS, + module_name: core::ptr::null_mut(), + log: core::ptr::null_mut(), + } +} + +/// Represents the NULL test case delimiter. +/// +/// The [`kunit_unsafe_test_suite!`] macro expects a NULL-terminated list of test cases. This +/// function returns such a delimiter. +#[doc(hidden)] +pub const fn kunit_case_null() -> kernel::bindings::kunit_case { + kernel::bindings::kunit_case { + run_case: None, + name: core::ptr::null_mut(), + generate_params: None, + attr: kernel::bindings::kunit_attributes { + speed: kernel::bindings::kunit_speed_KUNIT_SPEED_NORMAL, + }, + status: kernel::bindings::kunit_status_KUNIT_SUCCESS, + module_name: core::ptr::null_mut(), + log: core::ptr::null_mut(), + } +} + +/// Registers a KUnit test suite. +/// +/// # Safety +/// +/// `test_cases` must be a NULL terminated array of valid test cases, +/// whose lifetime is at least that of the test suite (i.e., static). +/// +/// # Examples +/// +/// ```ignore +/// extern "C" fn test_fn(_test: *mut kernel::bindings::kunit) { +/// let actual = 1 + 1; +/// let expected = 2; +/// assert_eq!(actual, expected); +/// } +/// +/// static mut KUNIT_TEST_CASES: [kernel::bindings::kunit_case; 2] = [ +/// kernel::kunit::kunit_case(kernel::c_str!("name"), test_fn), +/// kernel::kunit::kunit_case_null(), +/// ]; +/// kernel::kunit_unsafe_test_suite!(suite_name, KUNIT_TEST_CASES); +/// ``` +#[doc(hidden)] +#[macro_export] +macro_rules! kunit_unsafe_test_suite { + ($name:ident, $test_cases:ident) => { + const _: () = { + const KUNIT_TEST_SUITE_NAME: [::kernel::ffi::c_char; 256] = { + let name_u8 = ::core::stringify!($name).as_bytes(); + let mut ret = [0; 256]; + + if name_u8.len() > 255 { + panic!(concat!( + "The test suite name `", + ::core::stringify!($name), + "` exceeds the maximum length of 255 bytes." + )); + } + + let mut i = 0; + while i < name_u8.len() { + ret[i] = name_u8[i] as ::kernel::ffi::c_char; + i += 1; + } + + ret + }; + + static mut KUNIT_TEST_SUITE: ::kernel::bindings::kunit_suite = + ::kernel::bindings::kunit_suite { + name: KUNIT_TEST_SUITE_NAME, + #[allow(unused_unsafe)] + // SAFETY: `$test_cases` is passed in by the user, and + // (as documented) must be valid for the lifetime of + // the suite (i.e., static). + test_cases: unsafe { + ::core::ptr::addr_of_mut!($test_cases) + .cast::<::kernel::bindings::kunit_case>() + }, + suite_init: None, + suite_exit: None, + init: None, + exit: None, + attr: ::kernel::bindings::kunit_attributes { + speed: ::kernel::bindings::kunit_speed_KUNIT_SPEED_NORMAL, + }, + status_comment: [0; 256usize], + debugfs: ::core::ptr::null_mut(), + log: ::core::ptr::null_mut(), + suite_init_err: 0, + is_init: false, + }; + + #[used] + #[allow(unused_unsafe)] + #[cfg_attr(not(target_os = "macos"), link_section = ".kunit_test_suites")] + static mut KUNIT_TEST_SUITE_ENTRY: *const ::kernel::bindings::kunit_suite = + // SAFETY: `KUNIT_TEST_SUITE` is static. + unsafe { ::core::ptr::addr_of_mut!(KUNIT_TEST_SUITE) }; + }; + }; +} + +/// Returns whether we are currently running a KUnit test. +/// +/// In some cases, you need to call test-only code from outside the test case, for example, to +/// create a function mock. This function allows to change behavior depending on whether we are +/// currently running a KUnit test or not. +/// +/// # Examples +/// +/// This example shows how a function can be mocked to return a well-known value while testing: +/// +/// ``` +/// # use kernel::kunit::in_kunit_test; +/// fn fn_mock_example(n: i32) -> i32 { +/// if in_kunit_test() { +/// return 100; +/// } +/// +/// n + 1 +/// } +/// +/// let mock_res = fn_mock_example(5); +/// assert_eq!(mock_res, 100); +/// ``` +pub fn in_kunit_test() -> bool { + // SAFETY: `kunit_get_current_test()` is always safe to call (it has fallbacks for + // when KUnit is not enabled). + !unsafe { bindings::kunit_get_current_test() }.is_null() +} + +#[kunit_tests(rust_kernel_kunit)] +mod tests { + use super::*; + + #[test] + fn rust_test_kunit_example_test() { + #![expect(clippy::eq_op)] + assert_eq!(1 + 1, 2); + } + + #[test] + fn rust_test_kunit_in_kunit_test() { + assert!(in_kunit_test()); + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index e1065a7551a3..de07aadd1ff5 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -6,18 +6,26 @@ //! usage by Rust code in the kernel and is shared by all of them. //! //! In other words, all the rest of the Rust code in the kernel (e.g. kernel -//! modules written in Rust) depends on [`core`], [`alloc`] and this crate. +//! modules written in Rust) depends on [`core`] and this crate. //! //! If you need a kernel C API that is not ported or wrapped yet here, then //! do so first instead of bypassing this crate. #![no_std] #![feature(arbitrary_self_types)] -#![feature(coerce_unsized)] -#![feature(dispatch_from_dyn)] +#![cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, feature(derive_coerce_pointee))] +#![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(coerce_unsized))] +#![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(dispatch_from_dyn))] +#![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(unsize))] #![feature(inline_const)] #![feature(lint_reasons)] -#![feature(unsize)] +// Stable in Rust 1.82 +#![feature(raw_ref_op)] +// Stable in Rust 1.83 +#![feature(const_maybe_uninit_as_mut_ptr)] +#![feature(const_mut_refs)] +#![feature(const_ptr_write)] +#![feature(const_refs_to_cell)] // Ensure conditional compilation based on the kernel configuration works; // otherwise we may silently break things like initcall handling. @@ -32,14 +40,21 @@ pub use ffi; pub mod alloc; #[cfg(CONFIG_BLOCK)] pub mod block; -mod build_assert; +#[doc(hidden)] +pub mod build_assert; pub mod cred; pub mod device; +pub mod device_id; +pub mod devres; +pub mod dma; +pub mod driver; pub mod error; +pub mod faux; #[cfg(CONFIG_RUST_FW_LOADER_ABSTRACTIONS)] pub mod firmware; pub mod fs; pub mod init; +pub mod io; pub mod ioctl; pub mod jump_label; #[cfg(CONFIG_KUNIT)] @@ -48,11 +63,16 @@ pub mod list; pub mod miscdevice; #[cfg(CONFIG_NET)] pub mod net; +pub mod of; pub mod page; +#[cfg(CONFIG_PCI)] +pub mod pci; pub mod pid_namespace; +pub mod platform; pub mod prelude; pub mod print; pub mod rbtree; +pub mod revocable; pub mod security; pub mod seq_file; pub mod sizes; @@ -74,9 +94,6 @@ pub use bindings; pub use macros; pub use uapi; -#[doc(hidden)] -pub use build_error::build_error; - /// Prefix to appear before log messages printed from within the `kernel` crate. const __LOG_PREFIX: &[u8] = b"rust_kernel\0"; @@ -98,11 +115,11 @@ pub trait InPlaceModule: Sync + Send { /// Creates an initialiser for the module. /// /// It is called when the module is loaded. - fn init(module: &'static ThisModule) -> impl init::PinInit<Self, error::Error>; + fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error>; } impl<T: Module> InPlaceModule for T { - fn init(module: &'static ThisModule) -> impl init::PinInit<Self, error::Error> { + fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error> { let initer = move |slot: *mut Self| { let m = <Self as Module>::init(module)?; @@ -112,10 +129,16 @@ impl<T: Module> InPlaceModule for T { }; // SAFETY: On success, `initer` always fully initialises an instance of `Self`. - unsafe { init::pin_init_from_closure(initer) } + unsafe { pin_init::pin_init_from_closure(initer) } } } +/// Metadata attached to a [`Module`] or [`InPlaceModule`]. +pub trait ModuleMetadata { + /// The name of the module as specified in the `module!` macro. + const NAME: &'static crate::str::CStr; +} + /// Equivalent to `THIS_MODULE` in the C API. /// /// C header: [`include/linux/init.h`](srctree/include/linux/init.h) diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs index fb93330f4af4..2054682c5724 100644 --- a/rust/kernel/list.rs +++ b/rust/kernel/list.rs @@ -4,12 +4,15 @@ //! A linked list implementation. -use crate::init::PinInit; +// May not be needed in Rust 1.87.0 (pending beta backport). +#![allow(clippy::ptr_eq)] + use crate::sync::ArcBorrow; use crate::types::Opaque; use core::iter::{DoubleEndedIterator, FusedIterator}; use core::marker::PhantomData; use core::ptr; +use pin_init::PinInit; mod impl_list_item_mod; pub use self::impl_list_item_mod::{ @@ -245,8 +248,20 @@ impl<T: ?Sized + ListItem<ID>, const ID: u64> List<T, ID> { self.first.is_null() } - /// Add the provided item to the back of the list. - pub fn push_back(&mut self, item: ListArc<T, ID>) { + /// Inserts `item` before `next` in the cycle. + /// + /// Returns a pointer to the newly inserted element. Never changes `self.first` unless the list + /// is empty. + /// + /// # Safety + /// + /// * `next` must be an element in this list or null. + /// * if `next` is null, then the list must be empty. + unsafe fn insert_inner( + &mut self, + item: ListArc<T, ID>, + next: *mut ListLinksFields, + ) -> *mut ListLinksFields { let raw_item = ListArc::into_raw(item); // SAFETY: // * We just got `raw_item` from a `ListArc`, so it's in an `Arc`. @@ -259,16 +274,16 @@ impl<T: ?Sized + ListItem<ID>, const ID: u64> List<T, ID> { // SAFETY: We have not yet called `post_remove`, so `list_links` is still valid. let item = unsafe { ListLinks::fields(list_links) }; - if self.first.is_null() { - self.first = item; + // Check if the list is empty. + if next.is_null() { // SAFETY: The caller just gave us ownership of these fields. // INVARIANT: A linked list with one item should be cyclic. unsafe { (*item).next = item; (*item).prev = item; } + self.first = item; } else { - let next = self.first; // SAFETY: By the type invariant, this pointer is valid or null. We just checked that // it's not null, so it must be valid. let prev = unsafe { (*next).prev }; @@ -282,45 +297,27 @@ impl<T: ?Sized + ListItem<ID>, const ID: u64> List<T, ID> { (*next).prev = item; } } + + item + } + + /// Add the provided item to the back of the list. + pub fn push_back(&mut self, item: ListArc<T, ID>) { + // SAFETY: + // * `self.first` is null or in the list. + // * `self.first` is only null if the list is empty. + unsafe { self.insert_inner(item, self.first) }; } /// Add the provided item to the front of the list. pub fn push_front(&mut self, item: ListArc<T, ID>) { - let raw_item = ListArc::into_raw(item); // SAFETY: - // * We just got `raw_item` from a `ListArc`, so it's in an `Arc`. - // * If this requirement is violated, then the previous caller of `prepare_to_insert` - // violated the safety requirement that they can't give up ownership of the `ListArc` - // until they call `post_remove`. - // * We own the `ListArc`. - // * Removing items] from this list is always done using `remove_internal_inner`, which - // calls `post_remove` before giving up ownership. - let list_links = unsafe { T::prepare_to_insert(raw_item) }; - // SAFETY: We have not yet called `post_remove`, so `list_links` is still valid. - let item = unsafe { ListLinks::fields(list_links) }; + // * `self.first` is null or in the list. + // * `self.first` is only null if the list is empty. + let new_elem = unsafe { self.insert_inner(item, self.first) }; - if self.first.is_null() { - // SAFETY: The caller just gave us ownership of these fields. - // INVARIANT: A linked list with one item should be cyclic. - unsafe { - (*item).next = item; - (*item).prev = item; - } - } else { - let next = self.first; - // SAFETY: We just checked that `next` is non-null. - let prev = unsafe { (*next).prev }; - // SAFETY: Pointers in a linked list are never dangling, and the caller just gave us - // ownership of the fields on `item`. - // INVARIANT: This correctly inserts `item` between `prev` and `next`. - unsafe { - (*item).next = next; - (*item).prev = prev; - (*prev).next = item; - (*next).prev = item; - } - } - self.first = item; + // INVARIANT: `new_elem` is in the list because we just inserted it. + self.first = new_elem; } /// Removes the last item from this list. @@ -489,17 +486,21 @@ impl<T: ?Sized + ListItem<ID>, const ID: u64> List<T, ID> { other.first = ptr::null_mut(); } - /// Returns a cursor to the first element of the list. - /// - /// If the list is empty, this returns `None`. - pub fn cursor_front(&mut self) -> Option<Cursor<'_, T, ID>> { - if self.first.is_null() { - None - } else { - Some(Cursor { - current: self.first, - list: self, - }) + /// Returns a cursor that points before the first element of the list. + pub fn cursor_front(&mut self) -> Cursor<'_, T, ID> { + // INVARIANT: `self.first` is in this list. + Cursor { + next: self.first, + list: self, + } + } + + /// Returns a cursor that points after the last element in the list. + pub fn cursor_back(&mut self) -> Cursor<'_, T, ID> { + // INVARIANT: `next` is allowed to be null. + Cursor { + next: core::ptr::null_mut(), + list: self, } } @@ -579,69 +580,358 @@ impl<'a, T: ?Sized + ListItem<ID>, const ID: u64> Iterator for Iter<'a, T, ID> { /// A cursor into a [`List`]. /// +/// A cursor always rests between two elements in the list. This means that a cursor has a previous +/// and next element, but no current element. It also means that it's possible to have a cursor +/// into an empty list. +/// +/// # Examples +/// +/// ``` +/// use kernel::prelude::*; +/// use kernel::list::{List, ListArc, ListLinks}; +/// +/// #[pin_data] +/// struct ListItem { +/// value: u32, +/// #[pin] +/// links: ListLinks, +/// } +/// +/// impl ListItem { +/// fn new(value: u32) -> Result<ListArc<Self>> { +/// ListArc::pin_init(try_pin_init!(Self { +/// value, +/// links <- ListLinks::new(), +/// }), GFP_KERNEL) +/// } +/// } +/// +/// kernel::list::impl_has_list_links! { +/// impl HasListLinks<0> for ListItem { self.links } +/// } +/// kernel::list::impl_list_arc_safe! { +/// impl ListArcSafe<0> for ListItem { untracked; } +/// } +/// kernel::list::impl_list_item! { +/// impl ListItem<0> for ListItem { using ListLinks; } +/// } +/// +/// // Use a cursor to remove the first element with the given value. +/// fn remove_first(list: &mut List<ListItem>, value: u32) -> Option<ListArc<ListItem>> { +/// let mut cursor = list.cursor_front(); +/// while let Some(next) = cursor.peek_next() { +/// if next.value == value { +/// return Some(next.remove()); +/// } +/// cursor.move_next(); +/// } +/// None +/// } +/// +/// // Use a cursor to remove the last element with the given value. +/// fn remove_last(list: &mut List<ListItem>, value: u32) -> Option<ListArc<ListItem>> { +/// let mut cursor = list.cursor_back(); +/// while let Some(prev) = cursor.peek_prev() { +/// if prev.value == value { +/// return Some(prev.remove()); +/// } +/// cursor.move_prev(); +/// } +/// None +/// } +/// +/// // Use a cursor to remove all elements with the given value. The removed elements are moved to +/// // a new list. +/// fn remove_all(list: &mut List<ListItem>, value: u32) -> List<ListItem> { +/// let mut out = List::new(); +/// let mut cursor = list.cursor_front(); +/// while let Some(next) = cursor.peek_next() { +/// if next.value == value { +/// out.push_back(next.remove()); +/// } else { +/// cursor.move_next(); +/// } +/// } +/// out +/// } +/// +/// // Use a cursor to insert a value at a specific index. Returns an error if the index is out of +/// // bounds. +/// fn insert_at(list: &mut List<ListItem>, new: ListArc<ListItem>, idx: usize) -> Result { +/// let mut cursor = list.cursor_front(); +/// for _ in 0..idx { +/// if !cursor.move_next() { +/// return Err(EINVAL); +/// } +/// } +/// cursor.insert_next(new); +/// Ok(()) +/// } +/// +/// // Merge two sorted lists into a single sorted list. +/// fn merge_sorted(list: &mut List<ListItem>, merge: List<ListItem>) { +/// let mut cursor = list.cursor_front(); +/// for to_insert in merge { +/// while let Some(next) = cursor.peek_next() { +/// if to_insert.value < next.value { +/// break; +/// } +/// cursor.move_next(); +/// } +/// cursor.insert_prev(to_insert); +/// } +/// } +/// +/// let mut list = List::new(); +/// list.push_back(ListItem::new(14)?); +/// list.push_back(ListItem::new(12)?); +/// list.push_back(ListItem::new(10)?); +/// list.push_back(ListItem::new(12)?); +/// list.push_back(ListItem::new(15)?); +/// list.push_back(ListItem::new(14)?); +/// assert_eq!(remove_all(&mut list, 12).iter().count(), 2); +/// // [14, 10, 15, 14] +/// assert!(remove_first(&mut list, 14).is_some()); +/// // [10, 15, 14] +/// insert_at(&mut list, ListItem::new(12)?, 2)?; +/// // [10, 15, 12, 14] +/// assert!(remove_last(&mut list, 15).is_some()); +/// // [10, 12, 14] +/// +/// let mut list2 = List::new(); +/// list2.push_back(ListItem::new(11)?); +/// list2.push_back(ListItem::new(13)?); +/// merge_sorted(&mut list, list2); +/// +/// let mut items = list.into_iter(); +/// assert_eq!(items.next().unwrap().value, 10); +/// assert_eq!(items.next().unwrap().value, 11); +/// assert_eq!(items.next().unwrap().value, 12); +/// assert_eq!(items.next().unwrap().value, 13); +/// assert_eq!(items.next().unwrap().value, 14); +/// assert!(items.next().is_none()); +/// # Result::<(), Error>::Ok(()) +/// ``` +/// /// # Invariants /// -/// The `current` pointer points a value in `list`. +/// The `next` pointer is null or points a value in `list`. pub struct Cursor<'a, T: ?Sized + ListItem<ID>, const ID: u64 = 0> { - current: *mut ListLinksFields, list: &'a mut List<T, ID>, + /// Points at the element after this cursor, or null if the cursor is after the last element. + next: *mut ListLinksFields, } impl<'a, T: ?Sized + ListItem<ID>, const ID: u64> Cursor<'a, T, ID> { - /// Access the current element of this cursor. - pub fn current(&self) -> ArcBorrow<'_, T> { - // SAFETY: The `current` pointer points a value in the list. - let me = unsafe { T::view_value(ListLinks::from_fields(self.current)) }; - // SAFETY: - // * All values in a list are stored in an `Arc`. - // * The value cannot be removed from the list for the duration of the lifetime annotated - // on the returned `ArcBorrow`, because removing it from the list would require mutable - // access to the cursor or the list. However, the `ArcBorrow` holds an immutable borrow - // on the cursor, which in turn holds a mutable borrow on the list, so any such - // mutable access requires first releasing the immutable borrow on the cursor. - // * Values in a list never have a `UniqueArc` reference, because the list has a `ListArc` - // reference, and `UniqueArc` references must be unique. - unsafe { ArcBorrow::from_raw(me) } + /// Returns a pointer to the element before the cursor. + /// + /// Returns null if there is no element before the cursor. + fn prev_ptr(&self) -> *mut ListLinksFields { + let mut next = self.next; + let first = self.list.first; + if next == first { + // We are before the first element. + return core::ptr::null_mut(); + } + + if next.is_null() { + // We are after the last element, so we need a pointer to the last element, which is + // the same as `(*first).prev`. + next = first; + } + + // SAFETY: `next` can't be null, because then `first` must also be null, but in that case + // we would have exited at the `next == first` check. Thus, `next` is an element in the + // list, so we can access its `prev` pointer. + unsafe { (*next).prev } + } + + /// Access the element after this cursor. + pub fn peek_next(&mut self) -> Option<CursorPeek<'_, 'a, T, true, ID>> { + if self.next.is_null() { + return None; + } + + // INVARIANT: + // * We just checked that `self.next` is non-null, so it must be in `self.list`. + // * `ptr` is equal to `self.next`. + Some(CursorPeek { + ptr: self.next, + cursor: self, + }) + } + + /// Access the element before this cursor. + pub fn peek_prev(&mut self) -> Option<CursorPeek<'_, 'a, T, false, ID>> { + let prev = self.prev_ptr(); + + if prev.is_null() { + return None; + } + + // INVARIANT: + // * We just checked that `prev` is non-null, so it must be in `self.list`. + // * `self.prev_ptr()` never returns `self.next`. + Some(CursorPeek { + ptr: prev, + cursor: self, + }) } - /// Move the cursor to the next element. - pub fn next(self) -> Option<Cursor<'a, T, ID>> { - // SAFETY: The `current` field is always in a list. - let next = unsafe { (*self.current).next }; + /// Move the cursor one element forward. + /// + /// If the cursor is after the last element, then this call does nothing. This call returns + /// `true` if the cursor's position was changed. + pub fn move_next(&mut self) -> bool { + if self.next.is_null() { + return false; + } + + // SAFETY: `self.next` is an element in the list and we borrow the list mutably, so we can + // access the `next` field. + let mut next = unsafe { (*self.next).next }; if next == self.list.first { - None - } else { - // INVARIANT: Since `self.current` is in the `list`, its `next` pointer is also in the - // `list`. - Some(Cursor { - current: next, - list: self.list, - }) + next = core::ptr::null_mut(); } + + // INVARIANT: `next` is either null or the next element after an element in the list. + self.next = next; + true } - /// Move the cursor to the previous element. - pub fn prev(self) -> Option<Cursor<'a, T, ID>> { - // SAFETY: The `current` field is always in a list. - let prev = unsafe { (*self.current).prev }; + /// Move the cursor one element backwards. + /// + /// If the cursor is before the first element, then this call does nothing. This call returns + /// `true` if the cursor's position was changed. + pub fn move_prev(&mut self) -> bool { + if self.next == self.list.first { + return false; + } - if self.current == self.list.first { - None + // INVARIANT: `prev_ptr()` always returns a pointer that is null or in the list. + self.next = self.prev_ptr(); + true + } + + /// Inserts an element where the cursor is pointing and get a pointer to the new element. + fn insert_inner(&mut self, item: ListArc<T, ID>) -> *mut ListLinksFields { + let ptr = if self.next.is_null() { + self.list.first } else { - // INVARIANT: Since `self.current` is in the `list`, its `prev` pointer is also in the - // `list`. - Some(Cursor { - current: prev, - list: self.list, - }) + self.next + }; + // SAFETY: + // * `ptr` is an element in the list or null. + // * if `ptr` is null, then `self.list.first` is null so the list is empty. + let item = unsafe { self.list.insert_inner(item, ptr) }; + if self.next == self.list.first { + // INVARIANT: We just inserted `item`, so it's a member of list. + self.list.first = item; } + item } - /// Remove the current element from the list. + /// Insert an element at this cursor's location. + pub fn insert(mut self, item: ListArc<T, ID>) { + // This is identical to `insert_prev`, but consumes the cursor. This is helpful because it + // reduces confusion when the last operation on the cursor is an insertion; in that case, + // you just want to insert the element at the cursor, and it is confusing that the call + // involves the word prev or next. + self.insert_inner(item); + } + + /// Inserts an element after this cursor. + /// + /// After insertion, the new element will be after the cursor. + pub fn insert_next(&mut self, item: ListArc<T, ID>) { + self.next = self.insert_inner(item); + } + + /// Inserts an element before this cursor. + /// + /// After insertion, the new element will be before the cursor. + pub fn insert_prev(&mut self, item: ListArc<T, ID>) { + self.insert_inner(item); + } + + /// Remove the next element from the list. + pub fn remove_next(&mut self) -> Option<ListArc<T, ID>> { + self.peek_next().map(|v| v.remove()) + } + + /// Remove the previous element from the list. + pub fn remove_prev(&mut self) -> Option<ListArc<T, ID>> { + self.peek_prev().map(|v| v.remove()) + } +} + +/// References the element in the list next to the cursor. +/// +/// # Invariants +/// +/// * `ptr` is an element in `self.cursor.list`. +/// * `ISNEXT == (self.ptr == self.cursor.next)`. +pub struct CursorPeek<'a, 'b, T: ?Sized + ListItem<ID>, const ISNEXT: bool, const ID: u64> { + cursor: &'a mut Cursor<'b, T, ID>, + ptr: *mut ListLinksFields, +} + +impl<'a, 'b, T: ?Sized + ListItem<ID>, const ISNEXT: bool, const ID: u64> + CursorPeek<'a, 'b, T, ISNEXT, ID> +{ + /// Remove the element from the list. pub fn remove(self) -> ListArc<T, ID> { - // SAFETY: The `current` pointer always points at a member of the list. - unsafe { self.list.remove_internal(self.current) } + if ISNEXT { + self.cursor.move_next(); + } + + // INVARIANT: `self.ptr` is not equal to `self.cursor.next` due to the above `move_next` + // call. + // SAFETY: By the type invariants of `Self`, `next` is not null, so `next` is an element of + // `self.cursor.list` by the type invariants of `Cursor`. + unsafe { self.cursor.list.remove_internal(self.ptr) } + } + + /// Access this value as an [`ArcBorrow`]. + pub fn arc(&self) -> ArcBorrow<'_, T> { + // SAFETY: `self.ptr` points at an element in `self.cursor.list`. + let me = unsafe { T::view_value(ListLinks::from_fields(self.ptr)) }; + // SAFETY: + // * All values in a list are stored in an `Arc`. + // * The value cannot be removed from the list for the duration of the lifetime annotated + // on the returned `ArcBorrow`, because removing it from the list would require mutable + // access to the `CursorPeek`, the `Cursor` or the `List`. However, the `ArcBorrow` holds + // an immutable borrow on the `CursorPeek`, which in turn holds a mutable borrow on the + // `Cursor`, which in turn holds a mutable borrow on the `List`, so any such mutable + // access requires first releasing the immutable borrow on the `CursorPeek`. + // * Values in a list never have a `UniqueArc` reference, because the list has a `ListArc` + // reference, and `UniqueArc` references must be unique. + unsafe { ArcBorrow::from_raw(me) } + } +} + +impl<'a, 'b, T: ?Sized + ListItem<ID>, const ISNEXT: bool, const ID: u64> core::ops::Deref + for CursorPeek<'a, 'b, T, ISNEXT, ID> +{ + // If you change the `ptr` field to have type `ArcBorrow<'a, T>`, it might seem like you could + // get rid of the `CursorPeek::arc` method and change the deref target to `ArcBorrow<'a, T>`. + // However, that doesn't work because 'a is too long. You could obtain an `ArcBorrow<'a, T>` + // and then call `CursorPeek::remove` without giving up the `ArcBorrow<'a, T>`, which would be + // unsound. + type Target = T; + + fn deref(&self) -> &T { + // SAFETY: `self.ptr` points at an element in `self.cursor.list`. + let me = unsafe { T::view_value(ListLinks::from_fields(self.ptr)) }; + + // SAFETY: The value cannot be removed from the list for the duration of the lifetime + // annotated on the returned `&T`, because removing it from the list would require mutable + // access to the `CursorPeek`, the `Cursor` or the `List`. However, the `&T` holds an + // immutable borrow on the `CursorPeek`, which in turn holds a mutable borrow on the + // `Cursor`, which in turn holds a mutable borrow on the `List`, so any such mutable access + // requires first releasing the immutable borrow on the `CursorPeek`. + unsafe { &*me } } } diff --git a/rust/kernel/list/arc.rs b/rust/kernel/list/arc.rs index 3483d8c232c4..a88a2dc65aa7 100644 --- a/rust/kernel/list/arc.rs +++ b/rust/kernel/list/arc.rs @@ -7,7 +7,7 @@ use crate::alloc::{AllocError, Flags}; use crate::prelude::*; use crate::sync::{Arc, ArcBorrow, UniqueArc}; -use core::marker::{PhantomPinned, Unsize}; +use core::marker::PhantomPinned; use core::ops::Deref; use core::pin::Pin; use core::sync::atomic::{AtomicBool, Ordering}; @@ -96,7 +96,7 @@ macro_rules! impl_list_arc_safe { } $($rest:tt)*) => { impl$(<$($generics)*>)? $crate::list::ListArcSafe<$num> for $t { unsafe fn on_create_list_arc_from_unique(self: ::core::pin::Pin<&mut Self>) { - $crate::assert_pinned!($t, $field, $fty, inline); + ::pin_init::assert_pinned!($t, $field, $fty, inline); // SAFETY: This field is structurally pinned as per the above assertion. let field = unsafe { @@ -159,6 +159,7 @@ pub use impl_list_arc_safe; /// /// [`List`]: crate::list::List #[repr(transparent)] +#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))] pub struct ListArc<T, const ID: u64 = 0> where T: ListArcSafe<ID> + ?Sized, @@ -443,18 +444,20 @@ where // This is to allow coercion from `ListArc<T>` to `ListArc<U>` if `T` can be converted to the // dynamically-sized type (DST) `U`. +#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))] impl<T, U, const ID: u64> core::ops::CoerceUnsized<ListArc<U, ID>> for ListArc<T, ID> where - T: ListArcSafe<ID> + Unsize<U> + ?Sized, + T: ListArcSafe<ID> + core::marker::Unsize<U> + ?Sized, U: ListArcSafe<ID> + ?Sized, { } // This is to allow `ListArc<U>` to be dispatched on when `ListArc<T>` can be coerced into // `ListArc<U>`. +#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))] impl<T, U, const ID: u64> core::ops::DispatchFromDyn<ListArc<U, ID>> for ListArc<T, ID> where - T: ListArcSafe<ID> + Unsize<U> + ?Sized, + T: ListArcSafe<ID> + core::marker::Unsize<U> + ?Sized, U: ListArcSafe<ID> + ?Sized, { } diff --git a/rust/kernel/miscdevice.rs b/rust/kernel/miscdevice.rs index 8f88891fb1d2..15d10e5c1db7 100644 --- a/rust/kernel/miscdevice.rs +++ b/rust/kernel/miscdevice.rs @@ -10,9 +10,12 @@ use crate::{ bindings, + device::Device, error::{to_result, Error, Result, VTABLE_DEFAULT_ERROR}, ffi::{c_int, c_long, c_uint, c_ulong}, + fs::File, prelude::*, + seq_file::SeqFile, str::CStr, types::{ForeignOwnable, Opaque}, }; @@ -32,7 +35,7 @@ impl MiscDeviceOptions { let mut result: bindings::miscdevice = unsafe { MaybeUninit::zeroed().assume_init() }; result.minor = bindings::MISC_DYNAMIC_MINOR as _; result.name = self.name.as_char_ptr(); - result.fops = create_vtable::<T>(); + result.fops = MiscdeviceVTable::<T>::build(); result } } @@ -80,6 +83,16 @@ impl<T: MiscDevice> MiscDeviceRegistration<T> { pub fn as_raw(&self) -> *mut bindings::miscdevice { self.inner.get() } + + /// Access the `this_device` field. + pub fn device(&self) -> &Device { + // SAFETY: This can only be called after a successful register(), which always + // initialises `this_device` with a valid device. Furthermore, the signature of this + // function tells the borrow-checker that the `&Device` reference must not outlive the + // `&MiscDeviceRegistration<T>` used to obtain it, so the last use of the reference must be + // before the underlying `struct miscdevice` is destroyed. + unsafe { Device::as_ref((*self.as_raw()).this_device) } + } } #[pinned_drop] @@ -92,31 +105,32 @@ impl<T> PinnedDrop for MiscDeviceRegistration<T> { /// Trait implemented by the private data of an open misc device. #[vtable] -pub trait MiscDevice { +pub trait MiscDevice: Sized { /// What kind of pointer should `Self` be wrapped in. type Ptr: ForeignOwnable + Send + Sync; /// Called when the misc device is opened. /// /// The returned pointer will be stored as the private data for the file. - fn open() -> Result<Self::Ptr>; + fn open(_file: &File, _misc: &MiscDeviceRegistration<Self>) -> Result<Self::Ptr>; /// Called when the misc device is released. - fn release(device: Self::Ptr) { + fn release(device: Self::Ptr, _file: &File) { drop(device); } /// Handler for ioctls. /// - /// The `cmd` argument is usually manipulated using the utilties in [`kernel::ioctl`]. + /// The `cmd` argument is usually manipulated using the utilities in [`kernel::ioctl`]. /// /// [`kernel::ioctl`]: mod@crate::ioctl fn ioctl( _device: <Self::Ptr as ForeignOwnable>::Borrowed<'_>, + _file: &File, _cmd: u32, _arg: usize, ) -> Result<isize> { - kernel::build_error(VTABLE_DEFAULT_ERROR) + build_error!(VTABLE_DEFAULT_ERROR) } /// Handler for ioctls. @@ -129,124 +143,177 @@ pub trait MiscDevice { #[cfg(CONFIG_COMPAT)] fn compat_ioctl( _device: <Self::Ptr as ForeignOwnable>::Borrowed<'_>, + _file: &File, _cmd: u32, _arg: usize, ) -> Result<isize> { - kernel::build_error(VTABLE_DEFAULT_ERROR) + build_error!(VTABLE_DEFAULT_ERROR) + } + + /// Show info for this fd. + fn show_fdinfo( + _device: <Self::Ptr as ForeignOwnable>::Borrowed<'_>, + _m: &SeqFile, + _file: &File, + ) { + build_error!(VTABLE_DEFAULT_ERROR) } } -const fn create_vtable<T: MiscDevice>() -> &'static bindings::file_operations { - const fn maybe_fn<T: Copy>(check: bool, func: T) -> Option<T> { - if check { - Some(func) - } else { - None +/// A vtable for the file operations of a Rust miscdevice. +struct MiscdeviceVTable<T: MiscDevice>(PhantomData<T>); + +impl<T: MiscDevice> MiscdeviceVTable<T> { + /// # Safety + /// + /// `file` and `inode` must be the file and inode for a file that is undergoing initialization. + /// The file must be associated with a `MiscDeviceRegistration<T>`. + unsafe extern "C" fn open(inode: *mut bindings::inode, raw_file: *mut bindings::file) -> c_int { + // SAFETY: The pointers are valid and for a file being opened. + let ret = unsafe { bindings::generic_file_open(inode, raw_file) }; + if ret != 0 { + return ret; } - } - struct VtableHelper<T: MiscDevice> { - _t: PhantomData<T>, - } - impl<T: MiscDevice> VtableHelper<T> { - const VTABLE: bindings::file_operations = bindings::file_operations { - open: Some(fops_open::<T>), - release: Some(fops_release::<T>), - unlocked_ioctl: maybe_fn(T::HAS_IOCTL, fops_ioctl::<T>), - #[cfg(CONFIG_COMPAT)] - compat_ioctl: if T::HAS_COMPAT_IOCTL { - Some(fops_compat_ioctl::<T>) - } else if T::HAS_IOCTL { - Some(bindings::compat_ptr_ioctl) - } else { - None - }, - // SAFETY: All zeros is a valid value for `bindings::file_operations`. - ..unsafe { MaybeUninit::zeroed().assume_init() } + // SAFETY: The open call of a file can access the private data. + let misc_ptr = unsafe { (*raw_file).private_data }; + + // SAFETY: This is a miscdevice, so `misc_open()` set the private data to a pointer to the + // associated `struct miscdevice` before calling into this method. Furthermore, + // `misc_open()` ensures that the miscdevice can't be unregistered and freed during this + // call to `fops_open`. + let misc = unsafe { &*misc_ptr.cast::<MiscDeviceRegistration<T>>() }; + + // SAFETY: + // * This underlying file is valid for (much longer than) the duration of `T::open`. + // * There is no active fdget_pos region on the file on this thread. + let file = unsafe { File::from_raw_file(raw_file) }; + + let ptr = match T::open(file, misc) { + Ok(ptr) => ptr, + Err(err) => return err.to_errno(), }; + + // This overwrites the private data with the value specified by the user, changing the type + // of this file's private data. All future accesses to the private data is performed by + // other fops_* methods in this file, which all correctly cast the private data to the new + // type. + // + // SAFETY: The open call of a file can access the private data. + unsafe { (*raw_file).private_data = ptr.into_foreign() }; + + 0 } - &VtableHelper::<T>::VTABLE -} + /// # Safety + /// + /// `file` and `inode` must be the file and inode for a file that is being released. The file + /// must be associated with a `MiscDeviceRegistration<T>`. + unsafe extern "C" fn release(_inode: *mut bindings::inode, file: *mut bindings::file) -> c_int { + // SAFETY: The release call of a file owns the private data. + let private = unsafe { (*file).private_data }; + // SAFETY: The release call of a file owns the private data. + let ptr = unsafe { <T::Ptr as ForeignOwnable>::from_foreign(private) }; -/// # Safety -/// -/// `file` and `inode` must be the file and inode for a file that is undergoing initialization. -/// The file must be associated with a `MiscDeviceRegistration<T>`. -unsafe extern "C" fn fops_open<T: MiscDevice>( - inode: *mut bindings::inode, - file: *mut bindings::file, -) -> c_int { - // SAFETY: The pointers are valid and for a file being opened. - let ret = unsafe { bindings::generic_file_open(inode, file) }; - if ret != 0 { - return ret; + // SAFETY: + // * The file is valid for the duration of this call. + // * There is no active fdget_pos region on the file on this thread. + T::release(ptr, unsafe { File::from_raw_file(file) }); + + 0 } - let ptr = match T::open() { - Ok(ptr) => ptr, - Err(err) => return err.to_errno(), - }; + /// # Safety + /// + /// `file` must be a valid file that is associated with a `MiscDeviceRegistration<T>`. + unsafe extern "C" fn ioctl(file: *mut bindings::file, cmd: c_uint, arg: c_ulong) -> c_long { + // SAFETY: The ioctl call of a file can access the private data. + let private = unsafe { (*file).private_data }; + // SAFETY: Ioctl calls can borrow the private data of the file. + let device = unsafe { <T::Ptr as ForeignOwnable>::borrow(private) }; - // SAFETY: The open call of a file owns the private data. - unsafe { (*file).private_data = ptr.into_foreign().cast_mut() }; + // SAFETY: + // * The file is valid for the duration of this call. + // * There is no active fdget_pos region on the file on this thread. + let file = unsafe { File::from_raw_file(file) }; - 0 -} + match T::ioctl(device, file, cmd, arg) { + Ok(ret) => ret as c_long, + Err(err) => err.to_errno() as c_long, + } + } -/// # Safety -/// -/// `file` and `inode` must be the file and inode for a file that is being released. The file must -/// be associated with a `MiscDeviceRegistration<T>`. -unsafe extern "C" fn fops_release<T: MiscDevice>( - _inode: *mut bindings::inode, - file: *mut bindings::file, -) -> c_int { - // SAFETY: The release call of a file owns the private data. - let private = unsafe { (*file).private_data }; - // SAFETY: The release call of a file owns the private data. - let ptr = unsafe { <T::Ptr as ForeignOwnable>::from_foreign(private) }; - - T::release(ptr); - - 0 -} + /// # Safety + /// + /// `file` must be a valid file that is associated with a `MiscDeviceRegistration<T>`. + #[cfg(CONFIG_COMPAT)] + unsafe extern "C" fn compat_ioctl( + file: *mut bindings::file, + cmd: c_uint, + arg: c_ulong, + ) -> c_long { + // SAFETY: The compat ioctl call of a file can access the private data. + let private = unsafe { (*file).private_data }; + // SAFETY: Ioctl calls can borrow the private data of the file. + let device = unsafe { <T::Ptr as ForeignOwnable>::borrow(private) }; -/// # Safety -/// -/// `file` must be a valid file that is associated with a `MiscDeviceRegistration<T>`. -unsafe extern "C" fn fops_ioctl<T: MiscDevice>( - file: *mut bindings::file, - cmd: c_uint, - arg: c_ulong, -) -> c_long { - // SAFETY: The ioctl call of a file can access the private data. - let private = unsafe { (*file).private_data }; - // SAFETY: Ioctl calls can borrow the private data of the file. - let device = unsafe { <T::Ptr as ForeignOwnable>::borrow(private) }; - - match T::ioctl(device, cmd, arg) { - Ok(ret) => ret as c_long, - Err(err) => err.to_errno() as c_long, + // SAFETY: + // * The file is valid for the duration of this call. + // * There is no active fdget_pos region on the file on this thread. + let file = unsafe { File::from_raw_file(file) }; + + match T::compat_ioctl(device, file, cmd, arg) { + Ok(ret) => ret as c_long, + Err(err) => err.to_errno() as c_long, + } } -} -/// # Safety -/// -/// `file` must be a valid file that is associated with a `MiscDeviceRegistration<T>`. -#[cfg(CONFIG_COMPAT)] -unsafe extern "C" fn fops_compat_ioctl<T: MiscDevice>( - file: *mut bindings::file, - cmd: c_uint, - arg: c_ulong, -) -> c_long { - // SAFETY: The compat ioctl call of a file can access the private data. - let private = unsafe { (*file).private_data }; - // SAFETY: Ioctl calls can borrow the private data of the file. - let device = unsafe { <T::Ptr as ForeignOwnable>::borrow(private) }; - - match T::compat_ioctl(device, cmd, arg) { - Ok(ret) => ret as c_long, - Err(err) => err.to_errno() as c_long, + /// # Safety + /// + /// - `file` must be a valid file that is associated with a `MiscDeviceRegistration<T>`. + /// - `seq_file` must be a valid `struct seq_file` that we can write to. + unsafe extern "C" fn show_fdinfo(seq_file: *mut bindings::seq_file, file: *mut bindings::file) { + // SAFETY: The release call of a file owns the private data. + let private = unsafe { (*file).private_data }; + // SAFETY: Ioctl calls can borrow the private data of the file. + let device = unsafe { <T::Ptr as ForeignOwnable>::borrow(private) }; + // SAFETY: + // * The file is valid for the duration of this call. + // * There is no active fdget_pos region on the file on this thread. + let file = unsafe { File::from_raw_file(file) }; + // SAFETY: The caller ensures that the pointer is valid and exclusive for the duration in + // which this method is called. + let m = unsafe { SeqFile::from_raw(seq_file) }; + + T::show_fdinfo(device, m, file); + } + + const VTABLE: bindings::file_operations = bindings::file_operations { + open: Some(Self::open), + release: Some(Self::release), + unlocked_ioctl: if T::HAS_IOCTL { + Some(Self::ioctl) + } else { + None + }, + #[cfg(CONFIG_COMPAT)] + compat_ioctl: if T::HAS_COMPAT_IOCTL { + Some(Self::compat_ioctl) + } else if T::HAS_IOCTL { + Some(bindings::compat_ptr_ioctl) + } else { + None + }, + show_fdinfo: if T::HAS_SHOW_FDINFO { + Some(Self::show_fdinfo) + } else { + None + }, + // SAFETY: All zeros is a valid value for `bindings::file_operations`. + ..unsafe { MaybeUninit::zeroed().assume_init() } + }; + + const fn build() -> &'static bindings::file_operations { + &Self::VTABLE } } diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs index 2fbfb6a94c11..a59469c785e3 100644 --- a/rust/kernel/net/phy.rs +++ b/rust/kernel/net/phy.rs @@ -587,17 +587,17 @@ pub trait Driver { /// Issues a PHY software reset. fn soft_reset(_dev: &mut Device) -> Result { - kernel::build_error(VTABLE_DEFAULT_ERROR) + build_error!(VTABLE_DEFAULT_ERROR) } /// Sets up device-specific structures during discovery. fn probe(_dev: &mut Device) -> Result { - kernel::build_error(VTABLE_DEFAULT_ERROR) + build_error!(VTABLE_DEFAULT_ERROR) } /// Probes the hardware to determine what abilities it has. fn get_features(_dev: &mut Device) -> Result { - kernel::build_error(VTABLE_DEFAULT_ERROR) + build_error!(VTABLE_DEFAULT_ERROR) } /// Returns true if this is a suitable driver for the given phydev. @@ -609,32 +609,32 @@ pub trait Driver { /// Configures the advertisement and resets auto-negotiation /// if auto-negotiation is enabled. fn config_aneg(_dev: &mut Device) -> Result { - kernel::build_error(VTABLE_DEFAULT_ERROR) + build_error!(VTABLE_DEFAULT_ERROR) } /// Determines the negotiated speed and duplex. fn read_status(_dev: &mut Device) -> Result<u16> { - kernel::build_error(VTABLE_DEFAULT_ERROR) + build_error!(VTABLE_DEFAULT_ERROR) } /// Suspends the hardware, saving state if needed. fn suspend(_dev: &mut Device) -> Result { - kernel::build_error(VTABLE_DEFAULT_ERROR) + build_error!(VTABLE_DEFAULT_ERROR) } /// Resumes the hardware, restoring state if needed. fn resume(_dev: &mut Device) -> Result { - kernel::build_error(VTABLE_DEFAULT_ERROR) + build_error!(VTABLE_DEFAULT_ERROR) } /// Overrides the default MMD read function for reading a MMD register. fn read_mmd(_dev: &mut Device, _devnum: u8, _regnum: u16) -> Result<u16> { - kernel::build_error(VTABLE_DEFAULT_ERROR) + build_error!(VTABLE_DEFAULT_ERROR) } /// Overrides the default MMD write function for writing a MMD register. fn write_mmd(_dev: &mut Device, _devnum: u8, _regnum: u16, _val: u16) -> Result { - kernel::build_error(VTABLE_DEFAULT_ERROR) + build_error!(VTABLE_DEFAULT_ERROR) } /// Callback for notification of link change. @@ -790,7 +790,7 @@ impl DeviceMask { /// DeviceId::new_with_driver::<PhySample>() /// ], /// name: "rust_sample_phy", -/// author: "Rust for Linux Contributors", +/// authors: ["Rust for Linux Contributors"], /// description: "Rust sample PHYs driver", /// license: "GPL", /// } @@ -819,7 +819,7 @@ impl DeviceMask { /// module! { /// type: Module, /// name: "rust_sample_phy", -/// author: "Rust for Linux Contributors", +/// authors: ["Rust for Linux Contributors"], /// description: "Rust sample PHYs driver", /// license: "GPL", /// } @@ -837,7 +837,7 @@ impl DeviceMask { /// [::kernel::net::phy::create_phy_driver::<PhySample>()]; /// /// impl ::kernel::Module for Module { -/// fn init(module: &'static ThisModule) -> Result<Self> { +/// fn init(module: &'static ::kernel::ThisModule) -> Result<Self> { /// let drivers = unsafe { &mut DRIVERS }; /// let mut reg = ::kernel::net::phy::Registration::register( /// module, @@ -903,7 +903,7 @@ macro_rules! module_phy_driver { [$($crate::net::phy::create_phy_driver::<$driver>()),+]; impl $crate::Module for Module { - fn init(module: &'static ThisModule) -> Result<Self> { + fn init(module: &'static $crate::ThisModule) -> Result<Self> { // SAFETY: The anonymous constant guarantees that nobody else can access // the `DRIVERS` static. The array is used only in the C side. let drivers = unsafe { &mut DRIVERS }; diff --git a/rust/kernel/of.rs b/rust/kernel/of.rs new file mode 100644 index 000000000000..04f2d8ef29cb --- /dev/null +++ b/rust/kernel/of.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Device Tree / Open Firmware abstractions. + +use crate::{bindings, device_id::RawDeviceId, prelude::*}; + +/// IdTable type for OF drivers. +pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>; + +/// An open firmware device id. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::of_device_id); + +// SAFETY: +// * `DeviceId` is a `#[repr(transparent)` wrapper of `struct of_device_id` and does not add +// additional invariants, so it's safe to transmute to `RawType`. +// * `DRIVER_DATA_OFFSET` is the offset to the `data` field. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::of_device_id; + + const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::of_device_id, data); + + fn index(&self) -> usize { + self.0.data as _ + } +} + +impl DeviceId { + /// Create a new device id from an OF 'compatible' string. + pub const fn new(compatible: &'static CStr) -> Self { + let src = compatible.as_bytes_with_nul(); + // Replace with `bindings::of_device_id::default()` once stabilized for `const`. + // SAFETY: FFI type is valid to be zero-initialized. + let mut of: bindings::of_device_id = unsafe { core::mem::zeroed() }; + + // TODO: Use `clone_from_slice` once the corresponding types do match. + let mut i = 0; + while i < src.len() { + of.compatible[i] = src[i] as _; + i += 1; + } + + Self(of) + } +} + +/// Create an OF `IdTable` with an "alias" for modpost. +#[macro_export] +macro_rules! of_device_table { + ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { + const $table_name: $crate::device_id::IdArray< + $crate::of::DeviceId, + $id_info_type, + { $table_data.len() }, + > = $crate::device_id::IdArray::new($table_data); + + $crate::module_device_table!("of", $module_table_name, $table_name); + }; +} diff --git a/rust/kernel/page.rs b/rust/kernel/page.rs index fdac6c375fe4..f6126aca33a6 100644 --- a/rust/kernel/page.rs +++ b/rust/kernel/page.rs @@ -57,9 +57,8 @@ impl Page { /// ``` /// use kernel::page::Page; /// - /// # fn dox() -> Result<(), kernel::alloc::AllocError> { /// let page = Page::alloc_page(GFP_KERNEL)?; - /// # Ok(()) } + /// # Ok::<(), kernel::alloc::AllocError>(()) /// ``` /// /// Allocate memory for a page and zero its contents. @@ -67,9 +66,8 @@ impl Page { /// ``` /// use kernel::page::Page; /// - /// # fn dox() -> Result<(), kernel::alloc::AllocError> { /// let page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; - /// # Ok(()) } + /// # Ok::<(), kernel::alloc::AllocError>(()) /// ``` pub fn alloc_page(flags: Flags) -> Result<Self, AllocError> { // SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs new file mode 100644 index 000000000000..bbc453c6d9ea --- /dev/null +++ b/rust/kernel/pci.rs @@ -0,0 +1,477 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Abstractions for the PCI bus. +//! +//! C header: [`include/linux/pci.h`](srctree/include/linux/pci.h) + +use crate::{ + alloc::flags::*, + bindings, device, + device_id::RawDeviceId, + devres::Devres, + driver, + error::{to_result, Result}, + io::Io, + io::IoRaw, + str::CStr, + types::{ARef, ForeignOwnable, Opaque}, + ThisModule, +}; +use core::{ + marker::PhantomData, + ops::Deref, + ptr::{addr_of_mut, NonNull}, +}; +use kernel::prelude::*; + +/// An adapter for the registration of PCI drivers. +pub struct Adapter<T: Driver>(T); + +// SAFETY: A call to `unregister` for a given instance of `RegType` is guaranteed to be valid if +// a preceding call to `register` has been successful. +unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { + type RegType = bindings::pci_driver; + + unsafe fn register( + pdrv: &Opaque<Self::RegType>, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + // SAFETY: It's safe to set the fields of `struct pci_driver` on initialization. + unsafe { + (*pdrv.get()).name = name.as_char_ptr(); + (*pdrv.get()).probe = Some(Self::probe_callback); + (*pdrv.get()).remove = Some(Self::remove_callback); + (*pdrv.get()).id_table = T::ID_TABLE.as_ptr(); + } + + // SAFETY: `pdrv` is guaranteed to be a valid `RegType`. + to_result(unsafe { + bindings::__pci_register_driver(pdrv.get(), module.0, name.as_char_ptr()) + }) + } + + unsafe fn unregister(pdrv: &Opaque<Self::RegType>) { + // SAFETY: `pdrv` is guaranteed to be a valid `RegType`. + unsafe { bindings::pci_unregister_driver(pdrv.get()) } + } +} + +impl<T: Driver + 'static> Adapter<T> { + extern "C" fn probe_callback( + pdev: *mut bindings::pci_dev, + id: *const bindings::pci_device_id, + ) -> kernel::ffi::c_int { + // SAFETY: The PCI bus only ever calls the probe callback with a valid pointer to a + // `struct pci_dev`. + // + // INVARIANT: `pdev` is valid for the duration of `probe_callback()`. + let pdev = unsafe { &*pdev.cast::<Device<device::Core>>() }; + + // SAFETY: `DeviceId` is a `#[repr(transparent)` wrapper of `struct pci_device_id` and + // does not add additional invariants, so it's safe to transmute. + let id = unsafe { &*id.cast::<DeviceId>() }; + let info = T::ID_TABLE.info(id.index()); + + match T::probe(pdev, info) { + Ok(data) => { + // Let the `struct pci_dev` own a reference of the driver's private data. + // SAFETY: By the type invariant `pdev.as_raw` returns a valid pointer to a + // `struct pci_dev`. + unsafe { bindings::pci_set_drvdata(pdev.as_raw(), data.into_foreign() as _) }; + } + Err(err) => return Error::to_errno(err), + } + + 0 + } + + extern "C" fn remove_callback(pdev: *mut bindings::pci_dev) { + // SAFETY: The PCI bus only ever calls the remove callback with a valid pointer to a + // `struct pci_dev`. + let ptr = unsafe { bindings::pci_get_drvdata(pdev) }; + + // SAFETY: `remove_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `ptr` points to a valid and initialized + // `KBox<T>` pointer created through `KBox::into_foreign`. + let _ = unsafe { KBox::<T>::from_foreign(ptr) }; + } +} + +/// Declares a kernel module that exposes a single PCI driver. +/// +/// # Example +/// +///```ignore +/// kernel::module_pci_driver! { +/// type: MyDriver, +/// name: "Module name", +/// authors: ["Author name"], +/// description: "Description", +/// license: "GPL v2", +/// } +///``` +#[macro_export] +macro_rules! module_pci_driver { +($($f:tt)*) => { + $crate::module_driver!(<T>, $crate::pci::Adapter<T>, { $($f)* }); +}; +} + +/// Abstraction for the PCI device ID structure ([`struct pci_device_id`]). +/// +/// [`struct pci_device_id`]: https://docs.kernel.org/PCI/pci.html#c.pci_device_id +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::pci_device_id); + +impl DeviceId { + const PCI_ANY_ID: u32 = !0; + + /// Equivalent to C's `PCI_DEVICE` macro. + /// + /// Create a new `pci::DeviceId` from a vendor and device ID number. + pub const fn from_id(vendor: u32, device: u32) -> Self { + Self(bindings::pci_device_id { + vendor, + device, + subvendor: DeviceId::PCI_ANY_ID, + subdevice: DeviceId::PCI_ANY_ID, + class: 0, + class_mask: 0, + driver_data: 0, + override_only: 0, + }) + } + + /// Equivalent to C's `PCI_DEVICE_CLASS` macro. + /// + /// Create a new `pci::DeviceId` from a class number and mask. + pub const fn from_class(class: u32, class_mask: u32) -> Self { + Self(bindings::pci_device_id { + vendor: DeviceId::PCI_ANY_ID, + device: DeviceId::PCI_ANY_ID, + subvendor: DeviceId::PCI_ANY_ID, + subdevice: DeviceId::PCI_ANY_ID, + class, + class_mask, + driver_data: 0, + override_only: 0, + }) + } +} + +// SAFETY: +// * `DeviceId` is a `#[repr(transparent)` wrapper of `pci_device_id` and does not add +// additional invariants, so it's safe to transmute to `RawType`. +// * `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::pci_device_id; + + const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::pci_device_id, driver_data); + + fn index(&self) -> usize { + self.0.driver_data as _ + } +} + +/// `IdTable` type for PCI. +pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>; + +/// Create a PCI `IdTable` with its alias for modpost. +#[macro_export] +macro_rules! pci_device_table { + ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { + const $table_name: $crate::device_id::IdArray< + $crate::pci::DeviceId, + $id_info_type, + { $table_data.len() }, + > = $crate::device_id::IdArray::new($table_data); + + $crate::module_device_table!("pci", $module_table_name, $table_name); + }; +} + +/// The PCI driver trait. +/// +/// # Example +/// +///``` +/// # use kernel::{bindings, device::Core, pci}; +/// +/// struct MyDriver; +/// +/// kernel::pci_device_table!( +/// PCI_TABLE, +/// MODULE_PCI_TABLE, +/// <MyDriver as pci::Driver>::IdInfo, +/// [ +/// (pci::DeviceId::from_id(bindings::PCI_VENDOR_ID_REDHAT, bindings::PCI_ANY_ID as _), ()) +/// ] +/// ); +/// +/// impl pci::Driver for MyDriver { +/// type IdInfo = (); +/// const ID_TABLE: pci::IdTable<Self::IdInfo> = &PCI_TABLE; +/// +/// fn probe( +/// _pdev: &pci::Device<Core>, +/// _id_info: &Self::IdInfo, +/// ) -> Result<Pin<KBox<Self>>> { +/// Err(ENODEV) +/// } +/// } +///``` +/// Drivers must implement this trait in order to get a PCI driver registered. Please refer to the +/// `Adapter` documentation for an example. +pub trait Driver: Send { + /// The type holding information about each device id supported by the driver. + // TODO: Use `associated_type_defaults` once stabilized: + // + // ``` + // type IdInfo: 'static = (); + // ``` + type IdInfo: 'static; + + /// The table of device ids supported by the driver. + const ID_TABLE: IdTable<Self::IdInfo>; + + /// PCI driver probe. + /// + /// Called when a new platform device is added or discovered. + /// Implementers should attempt to initialize the device here. + fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> Result<Pin<KBox<Self>>>; +} + +/// The PCI device representation. +/// +/// This structure represents the Rust abstraction for a C `struct pci_dev`. The implementation +/// abstracts the usage of an already existing C `struct pci_dev` within Rust code that we get +/// passed from the C side. +/// +/// # Invariants +/// +/// A [`Device`] instance represents a valid `struct device` created by the C portion of the kernel. +#[repr(transparent)] +pub struct Device<Ctx: device::DeviceContext = device::Normal>( + Opaque<bindings::pci_dev>, + PhantomData<Ctx>, +); + +/// A PCI BAR to perform I/O-Operations on. +/// +/// # Invariants +/// +/// `Bar` always holds an `IoRaw` inststance that holds a valid pointer to the start of the I/O +/// memory mapped PCI bar and its size. +pub struct Bar<const SIZE: usize = 0> { + pdev: ARef<Device>, + io: IoRaw<SIZE>, + num: i32, +} + +impl<const SIZE: usize> Bar<SIZE> { + fn new(pdev: &Device, num: u32, name: &CStr) -> Result<Self> { + let len = pdev.resource_len(num)?; + if len == 0 { + return Err(ENOMEM); + } + + // Convert to `i32`, since that's what all the C bindings use. + let num = i32::try_from(num)?; + + // SAFETY: + // `pdev` is valid by the invariants of `Device`. + // `num` is checked for validity by a previous call to `Device::resource_len`. + // `name` is always valid. + let ret = unsafe { bindings::pci_request_region(pdev.as_raw(), num, name.as_char_ptr()) }; + if ret != 0 { + return Err(EBUSY); + } + + // SAFETY: + // `pdev` is valid by the invariants of `Device`. + // `num` is checked for validity by a previous call to `Device::resource_len`. + // `name` is always valid. + let ioptr: usize = unsafe { bindings::pci_iomap(pdev.as_raw(), num, 0) } as usize; + if ioptr == 0 { + // SAFETY: + // `pdev` valid by the invariants of `Device`. + // `num` is checked for validity by a previous call to `Device::resource_len`. + unsafe { bindings::pci_release_region(pdev.as_raw(), num) }; + return Err(ENOMEM); + } + + let io = match IoRaw::new(ioptr, len as usize) { + Ok(io) => io, + Err(err) => { + // SAFETY: + // `pdev` is valid by the invariants of `Device`. + // `ioptr` is guaranteed to be the start of a valid I/O mapped memory region. + // `num` is checked for validity by a previous call to `Device::resource_len`. + unsafe { Self::do_release(pdev, ioptr, num) }; + return Err(err); + } + }; + + Ok(Bar { + pdev: pdev.into(), + io, + num, + }) + } + + /// # Safety + /// + /// `ioptr` must be a valid pointer to the memory mapped PCI bar number `num`. + unsafe fn do_release(pdev: &Device, ioptr: usize, num: i32) { + // SAFETY: + // `pdev` is valid by the invariants of `Device`. + // `ioptr` is valid by the safety requirements. + // `num` is valid by the safety requirements. + unsafe { + bindings::pci_iounmap(pdev.as_raw(), ioptr as _); + bindings::pci_release_region(pdev.as_raw(), num); + } + } + + fn release(&self) { + // SAFETY: The safety requirements are guaranteed by the type invariant of `self.pdev`. + unsafe { Self::do_release(&self.pdev, self.io.addr(), self.num) }; + } +} + +impl Bar { + fn index_is_valid(index: u32) -> bool { + // A `struct pci_dev` owns an array of resources with at most `PCI_NUM_RESOURCES` entries. + index < bindings::PCI_NUM_RESOURCES + } +} + +impl<const SIZE: usize> Drop for Bar<SIZE> { + fn drop(&mut self) { + self.release(); + } +} + +impl<const SIZE: usize> Deref for Bar<SIZE> { + type Target = Io<SIZE>; + + fn deref(&self) -> &Self::Target { + // SAFETY: By the type invariant of `Self`, the MMIO range in `self.io` is properly mapped. + unsafe { Io::from_raw(&self.io) } + } +} + +impl Device { + fn as_raw(&self) -> *mut bindings::pci_dev { + self.0.get() + } + + /// Returns the PCI vendor ID. + pub fn vendor_id(&self) -> u16 { + // SAFETY: `self.as_raw` is a valid pointer to a `struct pci_dev`. + unsafe { (*self.as_raw()).vendor } + } + + /// Returns the PCI device ID. + pub fn device_id(&self) -> u16 { + // SAFETY: `self.as_raw` is a valid pointer to a `struct pci_dev`. + unsafe { (*self.as_raw()).device } + } + + /// Returns the size of the given PCI bar resource. + pub fn resource_len(&self, bar: u32) -> Result<bindings::resource_size_t> { + if !Bar::index_is_valid(bar) { + return Err(EINVAL); + } + + // SAFETY: + // - `bar` is a valid bar number, as guaranteed by the above call to `Bar::index_is_valid`, + // - by its type invariant `self.as_raw` is always a valid pointer to a `struct pci_dev`. + Ok(unsafe { bindings::pci_resource_len(self.as_raw(), bar.try_into()?) }) + } + + /// Mapps an entire PCI-BAR after performing a region-request on it. I/O operation bound checks + /// can be performed on compile time for offsets (plus the requested type size) < SIZE. + pub fn iomap_region_sized<const SIZE: usize>( + &self, + bar: u32, + name: &CStr, + ) -> Result<Devres<Bar<SIZE>>> { + let bar = Bar::<SIZE>::new(self, bar, name)?; + let devres = Devres::new(self.as_ref(), bar, GFP_KERNEL)?; + + Ok(devres) + } + + /// Mapps an entire PCI-BAR after performing a region-request on it. + pub fn iomap_region(&self, bar: u32, name: &CStr) -> Result<Devres<Bar>> { + self.iomap_region_sized::<0>(bar, name) + } +} + +impl Device<device::Core> { + /// Enable memory resources for this device. + pub fn enable_device_mem(&self) -> Result { + // SAFETY: `self.as_raw` is guaranteed to be a pointer to a valid `struct pci_dev`. + to_result(unsafe { bindings::pci_enable_device_mem(self.as_raw()) }) + } + + /// Enable bus-mastering for this device. + pub fn set_master(&self) { + // SAFETY: `self.as_raw` is guaranteed to be a pointer to a valid `struct pci_dev`. + unsafe { bindings::pci_set_master(self.as_raw()) }; + } +} + +impl Deref for Device<device::Core> { + type Target = Device; + + fn deref(&self) -> &Self::Target { + let ptr: *const Self = self; + + // CAST: `Device<Ctx>` is a transparent wrapper of `Opaque<bindings::pci_dev>`. + let ptr = ptr.cast::<Device>(); + + // SAFETY: `ptr` was derived from `&self`. + unsafe { &*ptr } + } +} + +impl From<&Device<device::Core>> for ARef<Device> { + fn from(dev: &Device<device::Core>) -> Self { + (&**dev).into() + } +} + +// SAFETY: Instances of `Device` are always reference-counted. +unsafe impl crate::types::AlwaysRefCounted for Device { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. + unsafe { bindings::pci_dev_get(self.as_raw()) }; + } + + unsafe fn dec_ref(obj: NonNull<Self>) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::pci_dev_put(obj.cast().as_ptr()) } + } +} + +impl AsRef<device::Device> for Device { + fn as_ref(&self) -> &device::Device { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct pci_dev`. + let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::as_ref(dev) } + } +} + +// SAFETY: A `Device` is always reference-counted and can be released from any thread. +unsafe impl Send for Device {} + +// SAFETY: `Device` can be shared among threads because all methods of `Device` +// (i.e. `Device<Normal>) are thread safe. +unsafe impl Sync for Device {} diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs new file mode 100644 index 000000000000..4917cb34e2fe --- /dev/null +++ b/rust/kernel/platform.rs @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Abstractions for the platform bus. +//! +//! C header: [`include/linux/platform_device.h`](srctree/include/linux/platform_device.h) + +use crate::{ + bindings, device, driver, + error::{to_result, Result}, + of, + prelude::*, + str::CStr, + types::{ARef, ForeignOwnable, Opaque}, + ThisModule, +}; + +use core::{ + marker::PhantomData, + ops::Deref, + ptr::{addr_of_mut, NonNull}, +}; + +/// An adapter for the registration of platform drivers. +pub struct Adapter<T: Driver>(T); + +// SAFETY: A call to `unregister` for a given instance of `RegType` is guaranteed to be valid if +// a preceding call to `register` has been successful. +unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { + type RegType = bindings::platform_driver; + + unsafe fn register( + pdrv: &Opaque<Self::RegType>, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + let of_table = match T::OF_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + // SAFETY: It's safe to set the fields of `struct platform_driver` on initialization. + unsafe { + (*pdrv.get()).driver.name = name.as_char_ptr(); + (*pdrv.get()).probe = Some(Self::probe_callback); + (*pdrv.get()).remove = Some(Self::remove_callback); + (*pdrv.get()).driver.of_match_table = of_table; + } + + // SAFETY: `pdrv` is guaranteed to be a valid `RegType`. + to_result(unsafe { bindings::__platform_driver_register(pdrv.get(), module.0) }) + } + + unsafe fn unregister(pdrv: &Opaque<Self::RegType>) { + // SAFETY: `pdrv` is guaranteed to be a valid `RegType`. + unsafe { bindings::platform_driver_unregister(pdrv.get()) }; + } +} + +impl<T: Driver + 'static> Adapter<T> { + extern "C" fn probe_callback(pdev: *mut bindings::platform_device) -> kernel::ffi::c_int { + // SAFETY: The platform bus only ever calls the probe callback with a valid pointer to a + // `struct platform_device`. + // + // INVARIANT: `pdev` is valid for the duration of `probe_callback()`. + let pdev = unsafe { &*pdev.cast::<Device<device::Core>>() }; + + let info = <Self as driver::Adapter>::id_info(pdev.as_ref()); + match T::probe(pdev, info) { + Ok(data) => { + // Let the `struct platform_device` own a reference of the driver's private data. + // SAFETY: By the type invariant `pdev.as_raw` returns a valid pointer to a + // `struct platform_device`. + unsafe { bindings::platform_set_drvdata(pdev.as_raw(), data.into_foreign() as _) }; + } + Err(err) => return Error::to_errno(err), + } + + 0 + } + + extern "C" fn remove_callback(pdev: *mut bindings::platform_device) { + // SAFETY: `pdev` is a valid pointer to a `struct platform_device`. + let ptr = unsafe { bindings::platform_get_drvdata(pdev) }; + + // SAFETY: `remove_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `ptr` points to a valid and initialized + // `KBox<T>` pointer created through `KBox::into_foreign`. + let _ = unsafe { KBox::<T>::from_foreign(ptr) }; + } +} + +impl<T: Driver + 'static> driver::Adapter for Adapter<T> { + type IdInfo = T::IdInfo; + + fn of_id_table() -> Option<of::IdTable<Self::IdInfo>> { + T::OF_ID_TABLE + } +} + +/// Declares a kernel module that exposes a single platform driver. +/// +/// # Examples +/// +/// ```ignore +/// kernel::module_platform_driver! { +/// type: MyDriver, +/// name: "Module name", +/// authors: ["Author name"], +/// description: "Description", +/// license: "GPL v2", +/// } +/// ``` +#[macro_export] +macro_rules! module_platform_driver { + ($($f:tt)*) => { + $crate::module_driver!(<T>, $crate::platform::Adapter<T>, { $($f)* }); + }; +} + +/// The platform driver trait. +/// +/// Drivers must implement this trait in order to get a platform driver registered. +/// +/// # Example +/// +///``` +/// # use kernel::{bindings, c_str, device::Core, of, platform}; +/// +/// struct MyDriver; +/// +/// kernel::of_device_table!( +/// OF_TABLE, +/// MODULE_OF_TABLE, +/// <MyDriver as platform::Driver>::IdInfo, +/// [ +/// (of::DeviceId::new(c_str!("test,device")), ()) +/// ] +/// ); +/// +/// impl platform::Driver for MyDriver { +/// type IdInfo = (); +/// const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = Some(&OF_TABLE); +/// +/// fn probe( +/// _pdev: &platform::Device<Core>, +/// _id_info: Option<&Self::IdInfo>, +/// ) -> Result<Pin<KBox<Self>>> { +/// Err(ENODEV) +/// } +/// } +///``` +pub trait Driver: Send { + /// The type holding driver private data about each device id supported by the driver. + /// + /// TODO: Use associated_type_defaults once stabilized: + /// + /// type IdInfo: 'static = (); + type IdInfo: 'static; + + /// The table of OF device ids supported by the driver. + const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>>; + + /// Platform driver probe. + /// + /// Called when a new platform device is added or discovered. + /// Implementers should attempt to initialize the device here. + fn probe(dev: &Device<device::Core>, id_info: Option<&Self::IdInfo>) + -> Result<Pin<KBox<Self>>>; +} + +/// The platform device representation. +/// +/// This structure represents the Rust abstraction for a C `struct platform_device`. The +/// implementation abstracts the usage of an already existing C `struct platform_device` within Rust +/// code that we get passed from the C side. +/// +/// # Invariants +/// +/// A [`Device`] instance represents a valid `struct platform_device` created by the C portion of +/// the kernel. +#[repr(transparent)] +pub struct Device<Ctx: device::DeviceContext = device::Normal>( + Opaque<bindings::platform_device>, + PhantomData<Ctx>, +); + +impl Device { + fn as_raw(&self) -> *mut bindings::platform_device { + self.0.get() + } +} + +impl Deref for Device<device::Core> { + type Target = Device; + + fn deref(&self) -> &Self::Target { + let ptr: *const Self = self; + + // CAST: `Device<Ctx>` is a transparent wrapper of `Opaque<bindings::platform_device>`. + let ptr = ptr.cast::<Device>(); + + // SAFETY: `ptr` was derived from `&self`. + unsafe { &*ptr } + } +} + +impl From<&Device<device::Core>> for ARef<Device> { + fn from(dev: &Device<device::Core>) -> Self { + (&**dev).into() + } +} + +// SAFETY: Instances of `Device` are always reference-counted. +unsafe impl crate::types::AlwaysRefCounted for Device { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. + unsafe { bindings::get_device(self.as_ref().as_raw()) }; + } + + unsafe fn dec_ref(obj: NonNull<Self>) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::platform_device_put(obj.cast().as_ptr()) } + } +} + +impl AsRef<device::Device> for Device { + fn as_ref(&self) -> &device::Device { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct platform_device`. + let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::as_ref(dev) } + } +} + +// SAFETY: A `Device` is always reference-counted and can be released from any thread. +unsafe impl Send for Device {} + +// SAFETY: `Device` can be shared among threads because all methods of `Device` +// (i.e. `Device<Normal>) are thread safe. +unsafe impl Sync for Device {} diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index 9ab4e0b6cbc9..baa774a351ce 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -17,9 +17,11 @@ pub use core::pin::Pin; pub use crate::alloc::{flags::*, Box, KBox, KVBox, KVVec, KVec, VBox, VVec, Vec}; #[doc(no_inline)] -pub use macros::{module, pin_data, pinned_drop, vtable, Zeroable}; +pub use macros::{export, module, vtable}; -pub use super::build_assert; +pub use pin_init::{init, pin_data, pin_init, pinned_drop, InPlaceWrite, Init, PinInit, Zeroable}; + +pub use super::{build_assert, build_error}; // `super::std_vendor` is hidden, which makes the macro inline for some reason. #[doc(no_inline)] @@ -28,7 +30,7 @@ pub use super::fmt; pub use super::{dev_alert, dev_crit, dev_dbg, dev_emerg, dev_err, dev_info, dev_notice, dev_warn}; pub use super::{pr_alert, pr_crit, pr_debug, pr_emerg, pr_err, pr_info, pr_notice, pr_warn}; -pub use super::{init, pin_init, try_init, try_pin_init}; +pub use super::{try_init, try_pin_init}; pub use super::static_assert; @@ -36,6 +38,6 @@ pub use super::error::{code::*, Error, Result}; pub use super::{str::CStr, ThisModule}; -pub use super::init::{InPlaceInit, InPlaceWrite, Init, PinInit}; +pub use super::init::InPlaceInit; pub use super::current; diff --git a/rust/kernel/print.rs b/rust/kernel/print.rs index b19ee490be58..cf4714242e14 100644 --- a/rust/kernel/print.rs +++ b/rust/kernel/print.rs @@ -6,16 +6,16 @@ //! //! Reference: <https://docs.kernel.org/core-api/printk-basics.html> -use core::{ +use crate::{ ffi::{c_char, c_void}, - fmt, + prelude::*, + str::RawFormatter, }; - -use crate::str::RawFormatter; +use core::fmt; // Called from `vsprintf` with format specifier `%pA`. #[expect(clippy::missing_safety_doc)] -#[no_mangle] +#[export] unsafe extern "C" fn rust_fmt_argument( buf: *mut c_char, end: *mut c_char, diff --git a/rust/kernel/rbtree.rs b/rust/kernel/rbtree.rs index 571e27efe544..5246b2c8a4ff 100644 --- a/rust/kernel/rbtree.rs +++ b/rust/kernel/rbtree.rs @@ -36,17 +36,17 @@ use core::{ /// /// // Check the nodes we just inserted. /// { -/// assert_eq!(tree.get(&10).unwrap(), &100); -/// assert_eq!(tree.get(&20).unwrap(), &200); -/// assert_eq!(tree.get(&30).unwrap(), &300); +/// assert_eq!(tree.get(&10), Some(&100)); +/// assert_eq!(tree.get(&20), Some(&200)); +/// assert_eq!(tree.get(&30), Some(&300)); /// } /// /// // Iterate over the nodes we just inserted. /// { /// let mut iter = tree.iter(); -/// assert_eq!(iter.next().unwrap(), (&10, &100)); -/// assert_eq!(iter.next().unwrap(), (&20, &200)); -/// assert_eq!(iter.next().unwrap(), (&30, &300)); +/// assert_eq!(iter.next(), Some((&10, &100))); +/// assert_eq!(iter.next(), Some((&20, &200))); +/// assert_eq!(iter.next(), Some((&30, &300))); /// assert!(iter.next().is_none()); /// } /// @@ -61,9 +61,9 @@ use core::{ /// // Check that the tree reflects the replacement. /// { /// let mut iter = tree.iter(); -/// assert_eq!(iter.next().unwrap(), (&10, &1000)); -/// assert_eq!(iter.next().unwrap(), (&20, &200)); -/// assert_eq!(iter.next().unwrap(), (&30, &300)); +/// assert_eq!(iter.next(), Some((&10, &1000))); +/// assert_eq!(iter.next(), Some((&20, &200))); +/// assert_eq!(iter.next(), Some((&30, &300))); /// assert!(iter.next().is_none()); /// } /// @@ -73,9 +73,9 @@ use core::{ /// // Check that the tree reflects the update. /// { /// let mut iter = tree.iter(); -/// assert_eq!(iter.next().unwrap(), (&10, &1000)); -/// assert_eq!(iter.next().unwrap(), (&20, &200)); -/// assert_eq!(iter.next().unwrap(), (&30, &3000)); +/// assert_eq!(iter.next(), Some((&10, &1000))); +/// assert_eq!(iter.next(), Some((&20, &200))); +/// assert_eq!(iter.next(), Some((&30, &3000))); /// assert!(iter.next().is_none()); /// } /// @@ -85,8 +85,8 @@ use core::{ /// // Check that the tree reflects the removal. /// { /// let mut iter = tree.iter(); -/// assert_eq!(iter.next().unwrap(), (&20, &200)); -/// assert_eq!(iter.next().unwrap(), (&30, &3000)); +/// assert_eq!(iter.next(), Some((&20, &200))); +/// assert_eq!(iter.next(), Some((&30, &3000))); /// assert!(iter.next().is_none()); /// } /// @@ -128,20 +128,20 @@ use core::{ /// // Check the nodes we just inserted. /// { /// let mut iter = tree.iter(); -/// assert_eq!(iter.next().unwrap(), (&10, &100)); -/// assert_eq!(iter.next().unwrap(), (&20, &200)); -/// assert_eq!(iter.next().unwrap(), (&30, &300)); +/// assert_eq!(iter.next(), Some((&10, &100))); +/// assert_eq!(iter.next(), Some((&20, &200))); +/// assert_eq!(iter.next(), Some((&30, &300))); /// assert!(iter.next().is_none()); /// } /// /// // Remove a node, getting back ownership of it. -/// let existing = tree.remove(&30).unwrap(); +/// let existing = tree.remove(&30); /// /// // Check that the tree reflects the removal. /// { /// let mut iter = tree.iter(); -/// assert_eq!(iter.next().unwrap(), (&10, &100)); -/// assert_eq!(iter.next().unwrap(), (&20, &200)); +/// assert_eq!(iter.next(), Some((&10, &100))); +/// assert_eq!(iter.next(), Some((&20, &200))); /// assert!(iter.next().is_none()); /// } /// @@ -155,9 +155,9 @@ use core::{ /// // Check that the tree reflect the new insertion. /// { /// let mut iter = tree.iter(); -/// assert_eq!(iter.next().unwrap(), (&10, &100)); -/// assert_eq!(iter.next().unwrap(), (&15, &150)); -/// assert_eq!(iter.next().unwrap(), (&20, &200)); +/// assert_eq!(iter.next(), Some((&10, &100))); +/// assert_eq!(iter.next(), Some((&15, &150))); +/// assert_eq!(iter.next(), Some((&20, &200))); /// assert!(iter.next().is_none()); /// } /// @@ -886,7 +886,7 @@ impl<'a, K, V> Cursor<'a, K, V> { /// # Safety /// /// - `node` must be a valid pointer to a node in an [`RBTree`]. - /// - The caller has immutable access to `node` for the duration of 'b. + /// - The caller has immutable access to `node` for the duration of `'b`. unsafe fn to_key_value<'b>(node: NonNull<bindings::rb_node>) -> (&'b K, &'b V) { // SAFETY: the caller guarantees that `node` is a valid pointer in an `RBTree`. let (k, v) = unsafe { Self::to_key_value_raw(node) }; @@ -897,7 +897,7 @@ impl<'a, K, V> Cursor<'a, K, V> { /// # Safety /// /// - `node` must be a valid pointer to a node in an [`RBTree`]. - /// - The caller has mutable access to `node` for the duration of 'b. + /// - The caller has mutable access to `node` for the duration of `'b`. unsafe fn to_key_value_mut<'b>(node: NonNull<bindings::rb_node>) -> (&'b K, &'b mut V) { // SAFETY: the caller guarantees that `node` is a valid pointer in an `RBTree`. let (k, v) = unsafe { Self::to_key_value_raw(node) }; @@ -908,7 +908,7 @@ impl<'a, K, V> Cursor<'a, K, V> { /// # Safety /// /// - `node` must be a valid pointer to a node in an [`RBTree`]. - /// - The caller has immutable access to the key for the duration of 'b. + /// - The caller has immutable access to the key for the duration of `'b`. unsafe fn to_key_value_raw<'b>(node: NonNull<bindings::rb_node>) -> (&'b K, *mut V) { // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` // point to the links field of `Node<K, V>` objects. @@ -1168,12 +1168,12 @@ impl<'a, K, V> RawVacantEntry<'a, K, V> { fn insert(self, node: RBTreeNode<K, V>) -> &'a mut V { let node = KBox::into_raw(node.node); - // SAFETY: `node` is valid at least until we call `Box::from_raw`, which only happens when + // SAFETY: `node` is valid at least until we call `KBox::from_raw`, which only happens when // the node is removed or replaced. let node_links = unsafe { addr_of_mut!((*node).links) }; // INVARIANT: We are linking in a new node, which is valid. It remains valid because we - // "forgot" it with `Box::into_raw`. + // "forgot" it with `KBox::into_raw`. // SAFETY: The type invariants of `RawVacantEntry` are exactly the safety requirements of `rb_link_node`. unsafe { bindings::rb_link_node(node_links, self.parent, self.child_field_of_parent) }; @@ -1259,7 +1259,7 @@ impl<'a, K, V> OccupiedEntry<'a, K, V> { fn replace(self, node: RBTreeNode<K, V>) -> RBTreeNode<K, V> { let node = KBox::into_raw(node.node); - // SAFETY: `node` is valid at least until we call `Box::from_raw`, which only happens when + // SAFETY: `node` is valid at least until we call `KBox::from_raw`, which only happens when // the node is removed or replaced. let new_node_links = unsafe { addr_of_mut!((*node).links) }; diff --git a/rust/kernel/revocable.rs b/rust/kernel/revocable.rs new file mode 100644 index 000000000000..3f0fbee4acb5 --- /dev/null +++ b/rust/kernel/revocable.rs @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Revocable objects. +//! +//! The [`Revocable`] type wraps other types and allows access to them to be revoked. The existence +//! of a [`RevocableGuard`] ensures that objects remain valid. + +use crate::{bindings, prelude::*, sync::rcu, types::Opaque}; +use core::{ + marker::PhantomData, + ops::Deref, + ptr::drop_in_place, + sync::atomic::{AtomicBool, Ordering}, +}; + +/// An object that can become inaccessible at runtime. +/// +/// Once access is revoked and all concurrent users complete (i.e., all existing instances of +/// [`RevocableGuard`] are dropped), the wrapped object is also dropped. +/// +/// # Examples +/// +/// ``` +/// # use kernel::revocable::Revocable; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn add_two(v: &Revocable<Example>) -> Option<u32> { +/// let guard = v.try_access()?; +/// Some(guard.a + guard.b) +/// } +/// +/// let v = KBox::pin_init(Revocable::new(Example { a: 10, b: 20 }), GFP_KERNEL).unwrap(); +/// assert_eq!(add_two(&v), Some(30)); +/// v.revoke(); +/// assert_eq!(add_two(&v), None); +/// ``` +/// +/// Sample example as above, but explicitly using the rcu read side lock. +/// +/// ``` +/// # use kernel::revocable::Revocable; +/// use kernel::sync::rcu; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn add_two(v: &Revocable<Example>) -> Option<u32> { +/// let guard = rcu::read_lock(); +/// let e = v.try_access_with_guard(&guard)?; +/// Some(e.a + e.b) +/// } +/// +/// let v = KBox::pin_init(Revocable::new(Example { a: 10, b: 20 }), GFP_KERNEL).unwrap(); +/// assert_eq!(add_two(&v), Some(30)); +/// v.revoke(); +/// assert_eq!(add_two(&v), None); +/// ``` +#[pin_data(PinnedDrop)] +pub struct Revocable<T> { + is_available: AtomicBool, + #[pin] + data: Opaque<T>, +} + +// SAFETY: `Revocable` is `Send` if the wrapped object is also `Send`. This is because while the +// functionality exposed by `Revocable` can be accessed from any thread/CPU, it is possible that +// this isn't supported by the wrapped object. +unsafe impl<T: Send> Send for Revocable<T> {} + +// SAFETY: `Revocable` is `Sync` if the wrapped object is both `Send` and `Sync`. We require `Send` +// from the wrapped object as well because of `Revocable::revoke`, which can trigger the `Drop` +// implementation of the wrapped object from an arbitrary thread. +unsafe impl<T: Sync + Send> Sync for Revocable<T> {} + +impl<T> Revocable<T> { + /// Creates a new revocable instance of the given data. + pub fn new(data: impl PinInit<T>) -> impl PinInit<Self> { + pin_init!(Self { + is_available: AtomicBool::new(true), + data <- Opaque::pin_init(data), + }) + } + + /// Tries to access the revocable wrapped object. + /// + /// Returns `None` if the object has been revoked and is therefore no longer accessible. + /// + /// Returns a guard that gives access to the object otherwise; the object is guaranteed to + /// remain accessible while the guard is alive. In such cases, callers are not allowed to sleep + /// because another CPU may be waiting to complete the revocation of this object. + pub fn try_access(&self) -> Option<RevocableGuard<'_, T>> { + let guard = rcu::read_lock(); + if self.is_available.load(Ordering::Relaxed) { + // Since `self.is_available` is true, data is initialised and has to remain valid + // because the RCU read side lock prevents it from being dropped. + Some(RevocableGuard::new(self.data.get(), guard)) + } else { + None + } + } + + /// Tries to access the revocable wrapped object. + /// + /// Returns `None` if the object has been revoked and is therefore no longer accessible. + /// + /// Returns a shared reference to the object otherwise; the object is guaranteed to + /// remain accessible while the rcu read side guard is alive. In such cases, callers are not + /// allowed to sleep because another CPU may be waiting to complete the revocation of this + /// object. + pub fn try_access_with_guard<'a>(&'a self, _guard: &'a rcu::Guard) -> Option<&'a T> { + if self.is_available.load(Ordering::Relaxed) { + // SAFETY: Since `self.is_available` is true, data is initialised and has to remain + // valid because the RCU read side lock prevents it from being dropped. + Some(unsafe { &*self.data.get() }) + } else { + None + } + } + + /// # Safety + /// + /// Callers must ensure that there are no more concurrent users of the revocable object. + unsafe fn revoke_internal<const SYNC: bool>(&self) -> bool { + let revoke = self.is_available.swap(false, Ordering::Relaxed); + + if revoke { + if SYNC { + // SAFETY: Just an FFI call, there are no further requirements. + unsafe { bindings::synchronize_rcu() }; + } + + // SAFETY: We know `self.data` is valid because only one CPU can succeed the + // `compare_exchange` above that takes `is_available` from `true` to `false`. + unsafe { drop_in_place(self.data.get()) }; + } + + revoke + } + + /// Revokes access to and drops the wrapped object. + /// + /// Access to the object is revoked immediately to new callers of [`Revocable::try_access`], + /// expecting that there are no concurrent users of the object. + /// + /// Returns `true` if `&self` has been revoked with this call, `false` if it was revoked + /// already. + /// + /// # Safety + /// + /// Callers must ensure that there are no more concurrent users of the revocable object. + pub unsafe fn revoke_nosync(&self) -> bool { + // SAFETY: By the safety requirement of this function, the caller ensures that nobody is + // accessing the data anymore and hence we don't have to wait for the grace period to + // finish. + unsafe { self.revoke_internal::<false>() } + } + + /// Revokes access to and drops the wrapped object. + /// + /// Access to the object is revoked immediately to new callers of [`Revocable::try_access`]. + /// + /// If there are concurrent users of the object (i.e., ones that called + /// [`Revocable::try_access`] beforehand and still haven't dropped the returned guard), this + /// function waits for the concurrent access to complete before dropping the wrapped object. + /// + /// Returns `true` if `&self` has been revoked with this call, `false` if it was revoked + /// already. + pub fn revoke(&self) -> bool { + // SAFETY: By passing `true` we ask `revoke_internal` to wait for the grace period to + // finish. + unsafe { self.revoke_internal::<true>() } + } +} + +#[pinned_drop] +impl<T> PinnedDrop for Revocable<T> { + fn drop(self: Pin<&mut Self>) { + // Drop only if the data hasn't been revoked yet (in which case it has already been + // dropped). + // SAFETY: We are not moving out of `p`, only dropping in place + let p = unsafe { self.get_unchecked_mut() }; + if *p.is_available.get_mut() { + // SAFETY: We know `self.data` is valid because no other CPU has changed + // `is_available` to `false` yet, and no other CPU can do it anymore because this CPU + // holds the only reference (mutable) to `self` now. + unsafe { drop_in_place(p.data.get()) }; + } + } +} + +/// A guard that allows access to a revocable object and keeps it alive. +/// +/// CPUs may not sleep while holding on to [`RevocableGuard`] because it's in atomic context +/// holding the RCU read-side lock. +/// +/// # Invariants +/// +/// The RCU read-side lock is held while the guard is alive. +pub struct RevocableGuard<'a, T> { + data_ref: *const T, + _rcu_guard: rcu::Guard, + _p: PhantomData<&'a ()>, +} + +impl<T> RevocableGuard<'_, T> { + fn new(data_ref: *const T, rcu_guard: rcu::Guard) -> Self { + Self { + data_ref, + _rcu_guard: rcu_guard, + _p: PhantomData, + } + } +} + +impl<T> Deref for RevocableGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: By the type invariants, we hold the rcu read-side lock, so the object is + // guaranteed to remain valid. + unsafe { &*self.data_ref } + } +} diff --git a/rust/kernel/security.rs b/rust/kernel/security.rs index ea4c58c81703..0c63e9e7e564 100644 --- a/rust/kernel/security.rs +++ b/rust/kernel/security.rs @@ -15,60 +15,60 @@ use crate::{ /// /// # Invariants /// -/// The `secdata` and `seclen` fields correspond to a valid security context as returned by a -/// successful call to `security_secid_to_secctx`, that has not yet been destroyed by calling -/// `security_release_secctx`. +/// The `ctx` field corresponds to a valid security context as returned by a successful call to +/// `security_secid_to_secctx`, that has not yet been released by `security_release_secctx`. pub struct SecurityCtx { - secdata: *mut crate::ffi::c_char, - seclen: usize, + ctx: bindings::lsm_context, } impl SecurityCtx { /// Get the security context given its id. + #[inline] pub fn from_secid(secid: u32) -> Result<Self> { - let mut secdata = core::ptr::null_mut(); - let mut seclen = 0u32; - // SAFETY: Just a C FFI call. The pointers are valid for writes. - to_result(unsafe { bindings::security_secid_to_secctx(secid, &mut secdata, &mut seclen) })?; + // SAFETY: `struct lsm_context` can be initialized to all zeros. + let mut ctx: bindings::lsm_context = unsafe { core::mem::zeroed() }; + + // SAFETY: Just a C FFI call. The pointer is valid for writes. + to_result(unsafe { bindings::security_secid_to_secctx(secid, &mut ctx) })?; // INVARIANT: If the above call did not fail, then we have a valid security context. - Ok(Self { - secdata, - seclen: seclen as usize, - }) + Ok(Self { ctx }) } /// Returns whether the security context is empty. + #[inline] pub fn is_empty(&self) -> bool { - self.seclen == 0 + self.ctx.len == 0 } /// Returns the length of this security context. + #[inline] pub fn len(&self) -> usize { - self.seclen + self.ctx.len as usize } /// Returns the bytes for this security context. + #[inline] pub fn as_bytes(&self) -> &[u8] { - let ptr = self.secdata; + let ptr = self.ctx.context; if ptr.is_null() { - debug_assert_eq!(self.seclen, 0); + debug_assert_eq!(self.len(), 0); // We can't pass a null pointer to `slice::from_raw_parts` even if the length is zero. return &[]; } // SAFETY: The call to `security_secid_to_secctx` guarantees that the pointer is valid for - // `seclen` bytes. Furthermore, if the length is zero, then we have ensured that the + // `self.len()` bytes. Furthermore, if the length is zero, then we have ensured that the // pointer is not null. - unsafe { core::slice::from_raw_parts(ptr.cast(), self.seclen) } + unsafe { core::slice::from_raw_parts(ptr.cast(), self.len()) } } } impl Drop for SecurityCtx { + #[inline] fn drop(&mut self) { - // SAFETY: By the invariant of `Self`, this frees a pointer that came from a successful - // call to `security_secid_to_secctx` and has not yet been destroyed by - // `security_release_secctx`. - unsafe { bindings::security_release_secctx(self.secdata, self.seclen as u32) }; + // SAFETY: By the invariant of `Self`, this releases an lsm context that came from a + // successful call to `security_secid_to_secctx` and has not yet been released. + unsafe { bindings::security_release_secctx(&mut self.ctx) }; } } diff --git a/rust/kernel/seq_file.rs b/rust/kernel/seq_file.rs index 04947c672979..7a9403eb6e5b 100644 --- a/rust/kernel/seq_file.rs +++ b/rust/kernel/seq_file.rs @@ -18,7 +18,7 @@ impl SeqFile { /// /// # Safety /// - /// The caller must ensure that for the duration of 'a the following is satisfied: + /// The caller must ensure that for the duration of `'a` the following is satisfied: /// * The pointer points at a valid `struct seq_file`. /// * The `struct seq_file` is not accessed from any other thread. pub unsafe fn from_raw<'a>(ptr: *mut bindings::seq_file) -> &'a SeqFile { @@ -30,6 +30,7 @@ impl SeqFile { } /// Used by the [`seq_print`] macro. + #[inline] pub fn call_printf(&self, args: core::fmt::Arguments<'_>) { // SAFETY: Passing a void pointer to `Arguments` is valid for `%pA`. unsafe { diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index 0f2765463dc8..fb61ce81ea28 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -31,6 +31,23 @@ impl BStr { // SAFETY: `BStr` is transparent to `[u8]`. unsafe { &*(bytes as *const [u8] as *const BStr) } } + + /// Strip a prefix from `self`. Delegates to [`slice::strip_prefix`]. + /// + /// # Examples + /// + /// ``` + /// # use kernel::b_str; + /// assert_eq!(Some(b_str!("bar")), b_str!("foobar").strip_prefix(b_str!("foo"))); + /// assert_eq!(None, b_str!("foobar").strip_prefix(b_str!("bar"))); + /// assert_eq!(Some(b_str!("foobar")), b_str!("foobar").strip_prefix(b_str!(""))); + /// assert_eq!(Some(b_str!("")), b_str!("foobar").strip_prefix(b_str!("foobar"))); + /// ``` + pub fn strip_prefix(&self, pattern: impl AsRef<Self>) -> Option<&BStr> { + self.deref() + .strip_prefix(pattern.as_ref().deref()) + .map(Self::from_bytes) + } } impl fmt::Display for BStr { @@ -39,12 +56,13 @@ impl fmt::Display for BStr { /// ``` /// # use kernel::{fmt, b_str, str::{BStr, CString}}; /// let ascii = b_str!("Hello, BStr!"); - /// let s = CString::try_from_fmt(fmt!("{}", ascii)).unwrap(); + /// let s = CString::try_from_fmt(fmt!("{}", ascii))?; /// assert_eq!(s.as_bytes(), "Hello, BStr!".as_bytes()); /// /// let non_ascii = b_str!("🦀"); - /// let s = CString::try_from_fmt(fmt!("{}", non_ascii)).unwrap(); + /// let s = CString::try_from_fmt(fmt!("{}", non_ascii))?; /// assert_eq!(s.as_bytes(), "\\xf0\\x9f\\xa6\\x80".as_bytes()); + /// # Ok::<(), kernel::error::Error>(()) /// ``` fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { for &b in &self.0 { @@ -55,7 +73,7 @@ impl fmt::Display for BStr { b'\r' => f.write_str("\\r")?, // Printable characters. 0x20..=0x7e => f.write_char(b as char)?, - _ => write!(f, "\\x{:02x}", b)?, + _ => write!(f, "\\x{b:02x}")?, } } Ok(()) @@ -70,12 +88,13 @@ impl fmt::Debug for BStr { /// # use kernel::{fmt, b_str, str::{BStr, CString}}; /// // Embedded double quotes are escaped. /// let ascii = b_str!("Hello, \"BStr\"!"); - /// let s = CString::try_from_fmt(fmt!("{:?}", ascii)).unwrap(); + /// let s = CString::try_from_fmt(fmt!("{:?}", ascii))?; /// assert_eq!(s.as_bytes(), "\"Hello, \\\"BStr\\\"!\"".as_bytes()); /// /// let non_ascii = b_str!("😺"); - /// let s = CString::try_from_fmt(fmt!("{:?}", non_ascii)).unwrap(); + /// let s = CString::try_from_fmt(fmt!("{:?}", non_ascii))?; /// assert_eq!(s.as_bytes(), "\"\\xf0\\x9f\\x98\\xba\"".as_bytes()); + /// # Ok::<(), kernel::error::Error>(()) /// ``` fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_char('"')?; @@ -90,7 +109,7 @@ impl fmt::Debug for BStr { b'\\' => f.write_str("\\\\")?, // Printable characters. 0x20..=0x7e => f.write_char(b as char)?, - _ => write!(f, "\\x{:02x}", b)?, + _ => write!(f, "\\x{b:02x}")?, } } f.write_char('"') @@ -106,6 +125,35 @@ impl Deref for BStr { } } +impl PartialEq for BStr { + fn eq(&self, other: &Self) -> bool { + self.deref().eq(other.deref()) + } +} + +impl<Idx> Index<Idx> for BStr +where + [u8]: Index<Idx, Output = [u8]>, +{ + type Output = Self; + + fn index(&self, index: Idx) -> &Self::Output { + BStr::from_bytes(&self.0[index]) + } +} + +impl AsRef<BStr> for [u8] { + fn as_ref(&self) -> &BStr { + BStr::from_bytes(self) + } +} + +impl AsRef<BStr> for BStr { + fn as_ref(&self) -> &BStr { + self + } +} + /// Creates a new [`BStr`] from a string literal. /// /// `b_str!` converts the supplied string literal to byte string, so non-ASCII @@ -273,8 +321,9 @@ impl CStr { /// /// ``` /// # use kernel::str::CStr; - /// let cstr = CStr::from_bytes_with_nul(b"foo\0").unwrap(); + /// let cstr = CStr::from_bytes_with_nul(b"foo\0")?; /// assert_eq!(cstr.to_str(), Ok("foo")); + /// # Ok::<(), kernel::error::Error>(()) /// ``` #[inline] pub fn to_str(&self) -> Result<&str, core::str::Utf8Error> { @@ -384,12 +433,13 @@ impl fmt::Display for CStr { /// # use kernel::str::CStr; /// # use kernel::str::CString; /// let penguin = c_str!("🐧"); - /// let s = CString::try_from_fmt(fmt!("{}", penguin)).unwrap(); + /// let s = CString::try_from_fmt(fmt!("{}", penguin))?; /// assert_eq!(s.as_bytes_with_nul(), "\\xf0\\x9f\\x90\\xa7\0".as_bytes()); /// /// let ascii = c_str!("so \"cool\""); - /// let s = CString::try_from_fmt(fmt!("{}", ascii)).unwrap(); + /// let s = CString::try_from_fmt(fmt!("{}", ascii))?; /// assert_eq!(s.as_bytes_with_nul(), "so \"cool\"\0".as_bytes()); + /// # Ok::<(), kernel::error::Error>(()) /// ``` fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { for &c in self.as_bytes() { @@ -397,7 +447,7 @@ impl fmt::Display for CStr { // Printable character. f.write_char(c as char)?; } else { - write!(f, "\\x{:02x}", c)?; + write!(f, "\\x{c:02x}")?; } } Ok(()) @@ -413,13 +463,14 @@ impl fmt::Debug for CStr { /// # use kernel::str::CStr; /// # use kernel::str::CString; /// let penguin = c_str!("🐧"); - /// let s = CString::try_from_fmt(fmt!("{:?}", penguin)).unwrap(); + /// let s = CString::try_from_fmt(fmt!("{:?}", penguin))?; /// assert_eq!(s.as_bytes_with_nul(), "\"\\xf0\\x9f\\x90\\xa7\"\0".as_bytes()); /// /// // Embedded double quotes are escaped. /// let ascii = c_str!("so \"cool\""); - /// let s = CString::try_from_fmt(fmt!("{:?}", ascii)).unwrap(); + /// let s = CString::try_from_fmt(fmt!("{:?}", ascii))?; /// assert_eq!(s.as_bytes_with_nul(), "\"so \\\"cool\\\"\"\0".as_bytes()); + /// # Ok::<(), kernel::error::Error>(()) /// ``` fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str("\"")?; @@ -428,7 +479,7 @@ impl fmt::Debug for CStr { // Printable characters. b'\"' => f.write_str("\\\"")?, 0x20..=0x7e => f.write_char(c as char)?, - _ => write!(f, "\\x{:02x}", c)?, + _ => write!(f, "\\x{c:02x}")?, } } f.write_str("\"") @@ -522,6 +573,7 @@ macro_rules! c_str { } #[cfg(test)] +#[expect(clippy::items_after_test_module)] mod tests { use super::*; @@ -547,7 +599,7 @@ mod tests { }) } - const ALL_ASCII_CHARS: &'static str = + const ALL_ASCII_CHARS: &str = "\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\x09\\x0a\\x0b\\x0c\\x0d\\x0e\\x0f\ \\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a\\x1b\\x1c\\x1d\\x1e\\x1f \ !\"#$%&'()*+,-./0123456789:;<=>?@\ @@ -581,6 +633,7 @@ mod tests { fn test_cstr_as_str_unchecked() { let good_bytes = b"\xf0\x9f\x90\xA7\0"; let checked_cstr = CStr::from_bytes_with_nul(good_bytes).unwrap(); + // SAFETY: The contents come from a string literal which contains valid UTF-8. let unchecked_str = unsafe { checked_cstr.as_str_unchecked() }; assert_eq!(unchecked_str, "🐧"); } @@ -588,13 +641,13 @@ mod tests { #[test] fn test_cstr_display() { let hello_world = CStr::from_bytes_with_nul(b"hello, world!\0").unwrap(); - assert_eq!(format!("{}", hello_world), "hello, world!"); + assert_eq!(format!("{hello_world}"), "hello, world!"); let non_printables = CStr::from_bytes_with_nul(b"\x01\x09\x0a\0").unwrap(); - assert_eq!(format!("{}", non_printables), "\\x01\\x09\\x0a"); + assert_eq!(format!("{non_printables}"), "\\x01\\x09\\x0a"); let non_ascii = CStr::from_bytes_with_nul(b"d\xe9j\xe0 vu\0").unwrap(); - assert_eq!(format!("{}", non_ascii), "d\\xe9j\\xe0 vu"); + assert_eq!(format!("{non_ascii}"), "d\\xe9j\\xe0 vu"); let good_bytes = CStr::from_bytes_with_nul(b"\xf0\x9f\xa6\x80\0").unwrap(); - assert_eq!(format!("{}", good_bytes), "\\xf0\\x9f\\xa6\\x80"); + assert_eq!(format!("{good_bytes}"), "\\xf0\\x9f\\xa6\\x80"); } #[test] @@ -605,47 +658,47 @@ mod tests { bytes[i as usize] = i.wrapping_add(1); } let cstr = CStr::from_bytes_with_nul(&bytes).unwrap(); - assert_eq!(format!("{}", cstr), ALL_ASCII_CHARS); + assert_eq!(format!("{cstr}"), ALL_ASCII_CHARS); } #[test] fn test_cstr_debug() { let hello_world = CStr::from_bytes_with_nul(b"hello, world!\0").unwrap(); - assert_eq!(format!("{:?}", hello_world), "\"hello, world!\""); + assert_eq!(format!("{hello_world:?}"), "\"hello, world!\""); let non_printables = CStr::from_bytes_with_nul(b"\x01\x09\x0a\0").unwrap(); - assert_eq!(format!("{:?}", non_printables), "\"\\x01\\x09\\x0a\""); + assert_eq!(format!("{non_printables:?}"), "\"\\x01\\x09\\x0a\""); let non_ascii = CStr::from_bytes_with_nul(b"d\xe9j\xe0 vu\0").unwrap(); - assert_eq!(format!("{:?}", non_ascii), "\"d\\xe9j\\xe0 vu\""); + assert_eq!(format!("{non_ascii:?}"), "\"d\\xe9j\\xe0 vu\""); let good_bytes = CStr::from_bytes_with_nul(b"\xf0\x9f\xa6\x80\0").unwrap(); - assert_eq!(format!("{:?}", good_bytes), "\"\\xf0\\x9f\\xa6\\x80\""); + assert_eq!(format!("{good_bytes:?}"), "\"\\xf0\\x9f\\xa6\\x80\""); } #[test] fn test_bstr_display() { let hello_world = BStr::from_bytes(b"hello, world!"); - assert_eq!(format!("{}", hello_world), "hello, world!"); + assert_eq!(format!("{hello_world}"), "hello, world!"); let escapes = BStr::from_bytes(b"_\t_\n_\r_\\_\'_\"_"); - assert_eq!(format!("{}", escapes), "_\\t_\\n_\\r_\\_'_\"_"); + assert_eq!(format!("{escapes}"), "_\\t_\\n_\\r_\\_'_\"_"); let others = BStr::from_bytes(b"\x01"); - assert_eq!(format!("{}", others), "\\x01"); + assert_eq!(format!("{others}"), "\\x01"); let non_ascii = BStr::from_bytes(b"d\xe9j\xe0 vu"); - assert_eq!(format!("{}", non_ascii), "d\\xe9j\\xe0 vu"); + assert_eq!(format!("{non_ascii}"), "d\\xe9j\\xe0 vu"); let good_bytes = BStr::from_bytes(b"\xf0\x9f\xa6\x80"); - assert_eq!(format!("{}", good_bytes), "\\xf0\\x9f\\xa6\\x80"); + assert_eq!(format!("{good_bytes}"), "\\xf0\\x9f\\xa6\\x80"); } #[test] fn test_bstr_debug() { let hello_world = BStr::from_bytes(b"hello, world!"); - assert_eq!(format!("{:?}", hello_world), "\"hello, world!\""); + assert_eq!(format!("{hello_world:?}"), "\"hello, world!\""); let escapes = BStr::from_bytes(b"_\t_\n_\r_\\_\'_\"_"); - assert_eq!(format!("{:?}", escapes), "\"_\\t_\\n_\\r_\\\\_'_\\\"_\""); + assert_eq!(format!("{escapes:?}"), "\"_\\t_\\n_\\r_\\\\_'_\\\"_\""); let others = BStr::from_bytes(b"\x01"); - assert_eq!(format!("{:?}", others), "\"\\x01\""); + assert_eq!(format!("{others:?}"), "\"\\x01\""); let non_ascii = BStr::from_bytes(b"d\xe9j\xe0 vu"); - assert_eq!(format!("{:?}", non_ascii), "\"d\\xe9j\\xe0 vu\""); + assert_eq!(format!("{non_ascii:?}"), "\"d\\xe9j\\xe0 vu\""); let good_bytes = BStr::from_bytes(b"\xf0\x9f\xa6\x80"); - assert_eq!(format!("{:?}", good_bytes), "\"\\xf0\\x9f\\xa6\\x80\""); + assert_eq!(format!("{good_bytes:?}"), "\"\\xf0\\x9f\\xa6\\x80\""); } } @@ -799,16 +852,17 @@ impl fmt::Write for Formatter { /// ``` /// use kernel::{str::CString, fmt}; /// -/// let s = CString::try_from_fmt(fmt!("{}{}{}", "abc", 10, 20)).unwrap(); +/// let s = CString::try_from_fmt(fmt!("{}{}{}", "abc", 10, 20))?; /// assert_eq!(s.as_bytes_with_nul(), "abc1020\0".as_bytes()); /// /// let tmp = "testing"; -/// let s = CString::try_from_fmt(fmt!("{tmp}{}", 123)).unwrap(); +/// let s = CString::try_from_fmt(fmt!("{tmp}{}", 123))?; /// assert_eq!(s.as_bytes_with_nul(), "testing123\0".as_bytes()); /// /// // This fails because it has an embedded `NUL` byte. /// let s = CString::try_from_fmt(fmt!("a\0b{}", 123)); /// assert_eq!(s.is_ok(), false); +/// # Ok::<(), kernel::error::Error>(()) /// ``` pub struct CString { buf: KVec<u8>, diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 1eab7ebf25fd..c23a12639924 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -5,43 +5,87 @@ //! This module contains the kernel APIs related to synchronisation that have been ported or //! wrapped for usage by Rust code in the kernel. +use crate::prelude::*; use crate::types::Opaque; +use pin_init; mod arc; +pub mod completion; mod condvar; pub mod lock; mod locked_by; pub mod poll; +pub mod rcu; pub use arc::{Arc, ArcBorrow, UniqueArc}; +pub use completion::Completion; pub use condvar::{new_condvar, CondVar, CondVarTimeoutResult}; pub use lock::global::{global_lock, GlobalGuard, GlobalLock, GlobalLockBackend, GlobalLockedBy}; -pub use lock::mutex::{new_mutex, Mutex}; -pub use lock::spinlock::{new_spinlock, SpinLock}; +pub use lock::mutex::{new_mutex, Mutex, MutexGuard}; +pub use lock::spinlock::{new_spinlock, SpinLock, SpinLockGuard}; pub use locked_by::LockedBy; /// Represents a lockdep class. It's a wrapper around C's `lock_class_key`. #[repr(transparent)] -pub struct LockClassKey(Opaque<bindings::lock_class_key>); +#[pin_data(PinnedDrop)] +pub struct LockClassKey { + #[pin] + inner: Opaque<bindings::lock_class_key>, +} // SAFETY: `bindings::lock_class_key` is designed to be used concurrently from multiple threads and // provides its own synchronization. unsafe impl Sync for LockClassKey {} impl LockClassKey { - /// Creates a new lock class key. - pub const fn new() -> Self { - Self(Opaque::uninit()) + /// Initializes a dynamically allocated lock class key. In the common case of using a + /// statically allocated lock class key, the static_lock_class! macro should be used instead. + /// + /// # Example + /// ``` + /// # use kernel::c_str; + /// # use kernel::alloc::KBox; + /// # use kernel::types::ForeignOwnable; + /// # use kernel::sync::{LockClassKey, SpinLock}; + /// # use pin_init::stack_pin_init; + /// + /// let key = KBox::pin_init(LockClassKey::new_dynamic(), GFP_KERNEL)?; + /// let key_ptr = key.into_foreign(); + /// + /// { + /// stack_pin_init!(let num: SpinLock<u32> = SpinLock::new( + /// 0, + /// c_str!("my_spinlock"), + /// // SAFETY: `key_ptr` is returned by the above `into_foreign()`, whose + /// // `from_foreign()` has not yet been called. + /// unsafe { <Pin<KBox<LockClassKey>> as ForeignOwnable>::borrow(key_ptr) } + /// )); + /// } + /// + /// // SAFETY: We dropped `num`, the only use of the key, so the result of the previous + /// // `borrow` has also been dropped. Thus, it's safe to use from_foreign. + /// unsafe { drop(<Pin<KBox<LockClassKey>> as ForeignOwnable>::from_foreign(key_ptr)) }; + /// + /// # Ok::<(), Error>(()) + /// ``` + pub fn new_dynamic() -> impl PinInit<Self> { + pin_init!(Self { + // SAFETY: lockdep_register_key expects an uninitialized block of memory + inner <- Opaque::ffi_init(|slot| unsafe { bindings::lockdep_register_key(slot) }) + }) } pub(crate) fn as_ptr(&self) -> *mut bindings::lock_class_key { - self.0.get() + self.inner.get() } } -impl Default for LockClassKey { - fn default() -> Self { - Self::new() +#[pinned_drop] +impl PinnedDrop for LockClassKey { + fn drop(self: Pin<&mut Self>) { + // SAFETY: self.as_ptr was registered with lockdep and self is pinned, so the address + // hasn't changed. Thus, it's safe to pass to unregister. + unsafe { bindings::lockdep_unregister_key(self.as_ptr()) } } } @@ -50,8 +94,11 @@ impl Default for LockClassKey { #[macro_export] macro_rules! static_lock_class { () => {{ - static CLASS: $crate::sync::LockClassKey = $crate::sync::LockClassKey::new(); - &CLASS + static CLASS: $crate::sync::LockClassKey = + // SAFETY: lockdep expects uninitialized memory when it's handed a statically allocated + // lock_class_key + unsafe { ::core::mem::MaybeUninit::uninit().assume_init() }; + $crate::prelude::Pin::static_ref(&CLASS) }}; } diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index fa4509406ee9..8484c814609a 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -19,20 +19,20 @@ use crate::{ alloc::{AllocError, Flags, KBox}, bindings, - init::{self, InPlaceInit, Init, PinInit}, + init::InPlaceInit, try_init, types::{ForeignOwnable, Opaque}, }; use core::{ alloc::Layout, fmt, - marker::{PhantomData, Unsize}, + marker::PhantomData, mem::{ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut}, pin::Pin, ptr::NonNull, }; -use macros::pin_data; +use pin_init::{self, pin_data, InPlaceWrite, Init, PinInit}; mod std_vendor; @@ -125,8 +125,18 @@ mod std_vendor; /// let coerced: Arc<dyn MyTrait> = obj; /// # Ok::<(), Error>(()) /// ``` +#[repr(transparent)] +#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))] pub struct Arc<T: ?Sized> { ptr: NonNull<ArcInner<T>>, + // NB: this informs dropck that objects of type `ArcInner<T>` may be used in `<Arc<T> as + // Drop>::drop`. Note that dropck already assumes that objects of type `T` may be used in + // `<Arc<T> as Drop>::drop` and the distinction between `T` and `ArcInner<T>` is not presently + // meaningful with respect to dropck - but this may change in the future so this is left here + // out of an abundance of caution. + // + // See https://doc.rust-lang.org/nomicon/phantom-data.html#generic-parameters-and-drop-checking + // for more detail on the semantics of dropck in the presence of `PhantomData`. _p: PhantomData<ArcInner<T>>, } @@ -172,10 +182,12 @@ impl<T: ?Sized> ArcInner<T> { // This is to allow coercion from `Arc<T>` to `Arc<U>` if `T` can be converted to the // dynamically-sized type (DST) `U`. -impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {} +#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))] +impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {} // This is to allow `Arc<U>` to be dispatched on when `Arc<T>` can be coerced into `Arc<U>`. -impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<Arc<U>> for Arc<T> {} +#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))] +impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<Arc<U>> for Arc<T> {} // SAFETY: It is safe to send `Arc<T>` to another thread when the underlying `T` is `Sync` because // it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs @@ -190,6 +202,26 @@ unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {} // the reference count reaches zero and `T` is dropped. unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {} +impl<T> InPlaceInit<T> for Arc<T> { + type PinnedSelf = Self; + + #[inline] + fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E> + where + E: From<AllocError>, + { + UniqueArc::try_pin_init(init, flags).map(|u| u.into()) + } + + #[inline] + fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E> + where + E: From<AllocError>, + { + UniqueArc::try_init(init, flags).map(|u| u.into()) + } +} + impl<T> Arc<T> { /// Constructs a new reference counted instance of `T`. pub fn new(contents: T, flags: Flags) -> Result<Self, AllocError> { @@ -201,10 +233,11 @@ impl<T> Arc<T> { }; let inner = KBox::new(value, flags)?; + let inner = KBox::leak(inner).into(); // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new // `Arc` object. - Ok(unsafe { Self::from_inner(KBox::leak(inner).into()) }) + Ok(unsafe { Self::from_inner(inner) }) } } @@ -233,6 +266,15 @@ impl<T: ?Sized> Arc<T> { unsafe { core::ptr::addr_of!((*ptr).data) } } + /// Return a raw pointer to the data in this arc. + pub fn as_ptr(this: &Self) -> *const T { + let ptr = this.ptr.as_ptr(); + + // SAFETY: As `ptr` points to a valid allocation of type `ArcInner`, + // field projection to `data`is within bounds of the allocation. + unsafe { core::ptr::addr_of!((*ptr).data) } + } + /// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`]. /// /// # Safety @@ -331,26 +373,37 @@ impl<T: ?Sized> Arc<T> { impl<T: 'static> ForeignOwnable for Arc<T> { type Borrowed<'a> = ArcBorrow<'a, T>; + type BorrowedMut<'a> = Self::Borrowed<'a>; + + fn into_foreign(self) -> *mut crate::ffi::c_void { + ManuallyDrop::new(self).ptr.as_ptr().cast() + } + + unsafe fn from_foreign(ptr: *mut crate::ffi::c_void) -> Self { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) }; - fn into_foreign(self) -> *const crate::ffi::c_void { - ManuallyDrop::new(self).ptr.as_ptr() as _ + // SAFETY: By the safety requirement of this function, we know that `ptr` came from + // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and + // holds a reference count increment that is transferrable to us. + unsafe { Self::from_inner(inner) } } - unsafe fn borrow<'a>(ptr: *const crate::ffi::c_void) -> ArcBorrow<'a, T> { - // By the safety requirement of this function, we know that `ptr` came from - // a previous call to `Arc::into_foreign`. - let inner = NonNull::new(ptr as *mut ArcInner<T>).unwrap(); + unsafe fn borrow<'a>(ptr: *mut crate::ffi::c_void) -> ArcBorrow<'a, T> { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) }; // SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive // for the lifetime of the returned value. unsafe { ArcBorrow::new(inner) } } - unsafe fn from_foreign(ptr: *const crate::ffi::c_void) -> Self { - // SAFETY: By the safety requirement of this function, we know that `ptr` came from - // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and - // holds a reference count increment that is transferrable to us. - unsafe { Self::from_inner(NonNull::new(ptr as _).unwrap()) } + unsafe fn borrow_mut<'a>(ptr: *mut crate::ffi::c_void) -> ArcBorrow<'a, T> { + // SAFETY: The safety requirements for `borrow_mut` are a superset of the safety + // requirements for `borrow`. + unsafe { Self::borrow(ptr) } } } @@ -372,10 +425,14 @@ impl<T: ?Sized> AsRef<T> for Arc<T> { impl<T: ?Sized> Clone for Arc<T> { fn clone(&self) -> Self { + // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is + // safe to dereference it. + let refcount = unsafe { self.ptr.as_ref() }.refcount.get(); + // INVARIANT: C `refcount_inc` saturates the refcount, so it cannot overflow to zero. // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is // safe to increment the refcount. - unsafe { bindings::refcount_inc(self.ptr.as_ref().refcount.get()) }; + unsafe { bindings::refcount_inc(refcount) }; // SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`. unsafe { Self::from_inner(self.ptr) } @@ -471,6 +528,8 @@ impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> { /// obj.as_arc_borrow().use_reference(); /// # Ok::<(), Error>(()) /// ``` +#[repr(transparent)] +#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))] pub struct ArcBorrow<'a, T: ?Sized + 'a> { inner: NonNull<ArcInner<T>>, _p: PhantomData<&'a ()>, @@ -478,7 +537,8 @@ pub struct ArcBorrow<'a, T: ?Sized + 'a> { // This is to allow `ArcBorrow<U>` to be dispatched on when `ArcBorrow<T>` can be coerced into // `ArcBorrow<U>`. -impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<ArcBorrow<'_, U>> +#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))] +impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<ArcBorrow<'_, U>> for ArcBorrow<'_, T> { } @@ -508,11 +568,11 @@ impl<T: ?Sized> ArcBorrow<'_, T> { } /// Creates an [`ArcBorrow`] to an [`Arc`] that has previously been deconstructed with - /// [`Arc::into_raw`]. + /// [`Arc::into_raw`] or [`Arc::as_ptr`]. /// /// # Safety /// - /// * The provided pointer must originate from a call to [`Arc::into_raw`]. + /// * The provided pointer must originate from a call to [`Arc::into_raw`] or [`Arc::as_ptr`]. /// * For the duration of the lifetime annotated on this `ArcBorrow`, the reference count must /// not hit zero. /// * For the duration of the lifetime annotated on this `ArcBorrow`, there must not be a @@ -628,6 +688,48 @@ pub struct UniqueArc<T: ?Sized> { inner: Arc<T>, } +impl<T> InPlaceInit<T> for UniqueArc<T> { + type PinnedSelf = Pin<Self>; + + #[inline] + fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E> + where + E: From<AllocError>, + { + UniqueArc::new_uninit(flags)?.write_pin_init(init) + } + + #[inline] + fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E> + where + E: From<AllocError>, + { + UniqueArc::new_uninit(flags)?.write_init(init) + } +} + +impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> { + type Initialized = UniqueArc<T>; + + fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid. + unsafe { init.__init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { self.assume_init() }) + } + + fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { init.__pinned_init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { self.assume_init() }.into()) + } +} + impl<T> UniqueArc<T> { /// Tries to allocate a new [`UniqueArc`] instance. pub fn new(value: T, flags: Flags) -> Result<Self, AllocError> { @@ -644,7 +746,7 @@ impl<T> UniqueArc<T> { try_init!(ArcInner { // SAFETY: There are no safety requirements for this FFI call. refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), - data <- init::uninit::<T, AllocError>(), + data <- pin_init::uninit::<T, AllocError>(), }? AllocError), flags, )?; diff --git a/rust/kernel/sync/completion.rs b/rust/kernel/sync/completion.rs new file mode 100644 index 000000000000..c50012a940a3 --- /dev/null +++ b/rust/kernel/sync/completion.rs @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Completion support. +//! +//! Reference: <https://docs.kernel.org/scheduler/completion.html> +//! +//! C header: [`include/linux/completion.h`](srctree/include/linux/completion.h) + +use crate::{bindings, prelude::*, types::Opaque}; + +/// Synchronization primitive to signal when a certain task has been completed. +/// +/// The [`Completion`] synchronization primitive signals when a certain task has been completed by +/// waking up other tasks that have been queued up to wait for the [`Completion`] to be completed. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::{Arc, Completion}; +/// use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem}; +/// +/// #[pin_data] +/// struct MyTask { +/// #[pin] +/// work: Work<MyTask>, +/// #[pin] +/// done: Completion, +/// } +/// +/// impl_has_work! { +/// impl HasWork<Self> for MyTask { self.work } +/// } +/// +/// impl MyTask { +/// fn new() -> Result<Arc<Self>> { +/// let this = Arc::pin_init(pin_init!(MyTask { +/// work <- new_work!("MyTask::work"), +/// done <- Completion::new(), +/// }), GFP_KERNEL)?; +/// +/// let _ = workqueue::system().enqueue(this.clone()); +/// +/// Ok(this) +/// } +/// +/// fn wait_for_completion(&self) { +/// self.done.wait_for_completion(); +/// +/// pr_info!("Completion: task complete\n"); +/// } +/// } +/// +/// impl WorkItem for MyTask { +/// type Pointer = Arc<MyTask>; +/// +/// fn run(this: Arc<MyTask>) { +/// // process this task +/// this.done.complete_all(); +/// } +/// } +/// +/// let task = MyTask::new()?; +/// task.wait_for_completion(); +/// # Ok::<(), Error>(()) +/// ``` +#[pin_data] +pub struct Completion { + #[pin] + inner: Opaque<bindings::completion>, +} + +// SAFETY: `Completion` is safe to be send to any task. +unsafe impl Send for Completion {} + +// SAFETY: `Completion` is safe to be accessed concurrently. +unsafe impl Sync for Completion {} + +impl Completion { + /// Create an initializer for a new [`Completion`]. + pub fn new() -> impl PinInit<Self> { + pin_init!(Self { + inner <- Opaque::ffi_init(|slot: *mut bindings::completion| { + // SAFETY: `slot` is a valid pointer to an uninitialized `struct completion`. + unsafe { bindings::init_completion(slot) }; + }), + }) + } + + fn as_raw(&self) -> *mut bindings::completion { + self.inner.get() + } + + /// Signal all tasks waiting on this completion. + /// + /// This method wakes up all tasks waiting on this completion; after this operation the + /// completion is permanently done, i.e. signals all current and future waiters. + pub fn complete_all(&self) { + // SAFETY: `self.as_raw()` is a pointer to a valid `struct completion`. + unsafe { bindings::complete_all(self.as_raw()) }; + } + + /// Wait for completion of a task. + /// + /// This method waits for the completion of a task; it is not interruptible and there is no + /// timeout. + /// + /// See also [`Completion::complete_all`]. + pub fn wait_for_completion(&self) { + // SAFETY: `self.as_raw()` is a pointer to a valid `struct completion`. + unsafe { bindings::wait_for_completion(self.as_raw()) }; + } +} diff --git a/rust/kernel/sync/condvar.rs b/rust/kernel/sync/condvar.rs index 7df565038d7d..caebf03f553b 100644 --- a/rust/kernel/sync/condvar.rs +++ b/rust/kernel/sync/condvar.rs @@ -8,16 +8,15 @@ use super::{lock::Backend, lock::Guard, LockClassKey}; use crate::{ ffi::{c_int, c_long}, - init::PinInit, - pin_init, str::CStr, - task::{MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE, TASK_NORMAL, TASK_UNINTERRUPTIBLE}, + task::{ + MAX_SCHEDULE_TIMEOUT, TASK_FREEZABLE, TASK_INTERRUPTIBLE, TASK_NORMAL, TASK_UNINTERRUPTIBLE, + }, time::Jiffies, types::Opaque, }; -use core::marker::PhantomPinned; -use core::ptr; -use macros::pin_data; +use core::{marker::PhantomPinned, pin::Pin, ptr}; +use pin_init::{pin_data, pin_init, PinInit}; /// Creates a [`CondVar`] initialiser with the given name and a newly-created lock class. #[macro_export] @@ -37,7 +36,7 @@ pub use new_condvar; /// spuriously. /// /// Instances of [`CondVar`] need a lock class and to be pinned. The recommended way to create such -/// instances is with the [`pin_init`](crate::pin_init) and [`new_condvar`] macros. +/// instances is with the [`pin_init`](crate::pin_init!) and [`new_condvar`] macros. /// /// # Examples /// @@ -101,7 +100,7 @@ unsafe impl Sync for CondVar {} impl CondVar { /// Constructs a new condvar initialiser. - pub fn new(name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> { + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self> { pin_init!(Self { _pin: PhantomPinned, // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have @@ -159,6 +158,25 @@ impl CondVar { crate::current!().signal_pending() } + /// Releases the lock and waits for a notification in interruptible and freezable mode. + /// + /// The process is allowed to be frozen during this sleep. No lock should be held when calling + /// this function, and there is a lockdep assertion for this. Freezing a task that holds a lock + /// can trivially deadlock vs another task that needs that lock to complete before it too can + /// hit freezable. + #[must_use = "wait_interruptible_freezable returns if a signal is pending, so the caller must check the return value"] + pub fn wait_interruptible_freezable<T: ?Sized, B: Backend>( + &self, + guard: &mut Guard<'_, T, B>, + ) -> bool { + self.wait_internal( + TASK_INTERRUPTIBLE | TASK_FREEZABLE, + guard, + MAX_SCHEDULE_TIMEOUT, + ); + crate::current!().signal_pending() + } + /// Releases the lock and waits for a notification in interruptible mode. /// /// Atomically releases the given lock (whose ownership is proven by the guard) and puts the diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index 41dcddac69e2..e82fa5be289c 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -7,13 +7,11 @@ use super::LockClassKey; use crate::{ - init::PinInit, - pin_init, str::CStr, types::{NotThreadSafe, Opaque, ScopeGuard}, }; -use core::{cell::UnsafeCell, marker::PhantomPinned}; -use macros::pin_data; +use core::{cell::UnsafeCell, marker::PhantomPinned, pin::Pin}; +use pin_init::{pin_data, pin_init, PinInit}; pub mod mutex; pub mod spinlock; @@ -90,12 +88,20 @@ pub unsafe trait Backend { // SAFETY: The safety requirements ensure that the lock is initialised. *guard_state = unsafe { Self::lock(ptr) }; } + + /// Asserts that the lock is held using lockdep. + /// + /// # Safety + /// + /// Callers must ensure that [`Backend::init`] has been previously called. + unsafe fn assert_is_held(ptr: *mut Self::State); } /// A mutual exclusion primitive. /// /// Exposes one of the kernel locking primitives. Which one is exposed depends on the lock /// [`Backend`] specified as the generic parameter `B`. +#[repr(C)] #[pin_data] pub struct Lock<T: ?Sized, B: Backend> { /// The kernel lock object. @@ -121,7 +127,7 @@ unsafe impl<T: ?Sized + Send, B: Backend> Sync for Lock<T, B> {} impl<T, B: Backend> Lock<T, B> { /// Constructs a new lock initialiser. - pub fn new(t: T, name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> { + pub fn new(t: T, name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self> { pin_init!(Self { data: UnsafeCell::new(t), _pin: PhantomPinned, @@ -134,6 +140,28 @@ impl<T, B: Backend> Lock<T, B> { } } +impl<B: Backend> Lock<(), B> { + /// Constructs a [`Lock`] from a raw pointer. + /// + /// This can be useful for interacting with a lock which was initialised outside of Rust. + /// + /// # Safety + /// + /// The caller promises that `ptr` points to a valid initialised instance of [`State`] during + /// the whole lifetime of `'a`. + /// + /// [`State`]: Backend::State + pub unsafe fn from_raw<'a>(ptr: *mut B::State) -> &'a Self { + // SAFETY: + // - By the safety contract `ptr` must point to a valid initialised instance of `B::State` + // - Since the lock data type is `()` which is a ZST, `state` is the only non-ZST member of + // the struct + // - Combined with `#[repr(C)]`, this guarantees `Self` has an equivalent data layout to + // `B::State`. + unsafe { &*ptr.cast() } + } +} + impl<T: ?Sized, B: Backend> Lock<T, B> { /// Acquires the lock and gives the caller access to the data protected by it. pub fn lock(&self) -> Guard<'_, T, B> { @@ -169,7 +197,37 @@ pub struct Guard<'a, T: ?Sized, B: Backend> { // SAFETY: `Guard` is sync when the data protected by the lock is also sync. unsafe impl<T: Sync + ?Sized, B: Backend> Sync for Guard<'_, T, B> {} -impl<T: ?Sized, B: Backend> Guard<'_, T, B> { +impl<'a, T: ?Sized, B: Backend> Guard<'a, T, B> { + /// Returns the lock that this guard originates from. + /// + /// # Examples + /// + /// The following example shows how to use [`Guard::lock_ref()`] to assert the corresponding + /// lock is held. + /// + /// ``` + /// # use kernel::{new_spinlock, sync::lock::{Backend, Guard, Lock}}; + /// # use pin_init::stack_pin_init; + /// + /// fn assert_held<T, B: Backend>(guard: &Guard<'_, T, B>, lock: &Lock<T, B>) { + /// // Address-equal means the same lock. + /// assert!(core::ptr::eq(guard.lock_ref(), lock)); + /// } + /// + /// // Creates a new lock on the stack. + /// stack_pin_init!{ + /// let l = new_spinlock!(42) + /// } + /// + /// let g = l.lock(); + /// + /// // `g` originates from `l`. + /// assert_held(&g, &l); + /// ``` + pub fn lock_ref(&self) -> &'a Lock<T, B> { + self.lock + } + pub(crate) fn do_unlocked<U>(&mut self, cb: impl FnOnce() -> U) -> U { // SAFETY: The caller owns the lock, so it is safe to unlock it. unsafe { B::unlock(self.lock.state.get(), &self.state) }; @@ -211,7 +269,10 @@ impl<'a, T: ?Sized, B: Backend> Guard<'a, T, B> { /// # Safety /// /// The caller must ensure that it owns the lock. - pub(crate) unsafe fn new(lock: &'a Lock<T, B>, state: B::GuardState) -> Self { + pub unsafe fn new(lock: &'a Lock<T, B>, state: B::GuardState) -> Self { + // SAFETY: The caller can only hold the lock if `Backend::init` has already been called. + unsafe { B::assert_is_held(lock.state.get()) }; + Self { lock, state, diff --git a/rust/kernel/sync/lock/global.rs b/rust/kernel/sync/lock/global.rs index 480ee724e3cc..d65f94b5caf2 100644 --- a/rust/kernel/sync/lock/global.rs +++ b/rust/kernel/sync/lock/global.rs @@ -13,6 +13,7 @@ use crate::{ use core::{ cell::UnsafeCell, marker::{PhantomData, PhantomPinned}, + pin::Pin, }; /// Trait implemented for marker types for global locks. @@ -26,7 +27,7 @@ pub trait GlobalLockBackend { /// The backend used for this global lock. type Backend: Backend + 'static; /// The class for this global lock. - fn get_lock_class() -> &'static LockClassKey; + fn get_lock_class() -> Pin<&'static LockClassKey>; } /// Type used for global locks. @@ -270,7 +271,7 @@ macro_rules! global_lock { type Item = $valuety; type Backend = $crate::global_lock_inner!(backend $kind); - fn get_lock_class() -> &'static $crate::sync::LockClassKey { + fn get_lock_class() -> Pin<&'static $crate::sync::LockClassKey> { $crate::static_lock_class!() } } diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs index 0e946ebefce1..581cee7ab842 100644 --- a/rust/kernel/sync/lock/mutex.rs +++ b/rust/kernel/sync/lock/mutex.rs @@ -26,7 +26,7 @@ pub use new_mutex; /// Since it may block, [`Mutex`] needs to be used with care in atomic contexts. /// /// Instances of [`Mutex`] need a lock class and to be pinned. The recommended way to create such -/// instances is with the [`pin_init`](crate::pin_init) and [`new_mutex`] macros. +/// instances is with the [`pin_init`](pin_init::pin_init) and [`new_mutex`] macros. /// /// # Examples /// @@ -86,6 +86,14 @@ pub use new_mutex; /// [`struct mutex`]: srctree/include/linux/mutex.h pub type Mutex<T> = super::Lock<T, MutexBackend>; +/// A [`Guard`] acquired from locking a [`Mutex`]. +/// +/// This is simply a type alias for a [`Guard`] returned from locking a [`Mutex`]. It will unlock +/// the [`Mutex`] upon being dropped. +/// +/// [`Guard`]: super::Guard +pub type MutexGuard<'a, T> = super::Guard<'a, T, MutexBackend>; + /// A kernel `struct mutex` lock backend. pub struct MutexBackend; @@ -126,4 +134,9 @@ unsafe impl super::Backend for MutexBackend { None } } + + unsafe fn assert_is_held(ptr: *mut Self::State) { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + unsafe { bindings::mutex_assert_is_held(ptr) } + } } diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index 9f4d128bed98..d7be38ccbdc7 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -24,7 +24,7 @@ pub use new_spinlock; /// unlocked, at which point another CPU will be allowed to make progress. /// /// Instances of [`SpinLock`] need a lock class and to be pinned. The recommended way to create such -/// instances is with the [`pin_init`](crate::pin_init) and [`new_spinlock`] macros. +/// instances is with the [`pin_init`](pin_init::pin_init) and [`new_spinlock`] macros. /// /// # Examples /// @@ -87,6 +87,14 @@ pub type SpinLock<T> = super::Lock<T, SpinLockBackend>; /// A kernel `spinlock_t` lock backend. pub struct SpinLockBackend; +/// A [`Guard`] acquired from locking a [`SpinLock`]. +/// +/// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLock`]. It will unlock +/// the [`SpinLock`] upon being dropped. +/// +/// [`Guard`]: super::Guard +pub type SpinLockGuard<'a, T> = super::Guard<'a, T, SpinLockBackend>; + // SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the // default implementation that always calls the same locking method. unsafe impl super::Backend for SpinLockBackend { @@ -125,4 +133,9 @@ unsafe impl super::Backend for SpinLockBackend { None } } + + unsafe fn assert_is_held(ptr: *mut Self::State) { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + unsafe { bindings::spin_assert_is_held(ptr) } + } } diff --git a/rust/kernel/sync/locked_by.rs b/rust/kernel/sync/locked_by.rs index a7b244675c2b..61f100a45b35 100644 --- a/rust/kernel/sync/locked_by.rs +++ b/rust/kernel/sync/locked_by.rs @@ -55,7 +55,7 @@ use core::{cell::UnsafeCell, mem::size_of, ptr}; /// fn print_bytes_used(dir: &Directory, file: &File) { /// let guard = dir.inner.lock(); /// let inner_file = file.inner.access(&guard); -/// pr_info!("{} {}", guard.bytes_used, inner_file.bytes_used); +/// pr_info!("{} {}\n", guard.bytes_used, inner_file.bytes_used); /// } /// /// /// Increments `bytes_used` for both the directory and file. diff --git a/rust/kernel/sync/poll.rs b/rust/kernel/sync/poll.rs index d5f17153b424..d7e6e59e124b 100644 --- a/rust/kernel/sync/poll.rs +++ b/rust/kernel/sync/poll.rs @@ -43,11 +43,11 @@ impl PollTable { /// /// # Safety /// - /// The caller must ensure that for the duration of 'a, the pointer will point at a valid poll + /// The caller must ensure that for the duration of `'a`, the pointer will point at a valid poll /// table (as defined in the type invariants). /// /// The caller must also ensure that the `poll_table` is only accessed via the returned - /// reference for the duration of 'a. + /// reference for the duration of `'a`. pub unsafe fn from_ptr<'a>(ptr: *mut bindings::poll_table) -> &'a mut PollTable { // SAFETY: The safety requirements guarantee the validity of the dereference, while the // `PollTable` type being transparent makes the cast ok. @@ -89,7 +89,7 @@ pub struct PollCondVar { impl PollCondVar { /// Constructs a new condvar initialiser. - pub fn new(name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> { + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self> { pin_init!(Self { inner <- CondVar::new(name, key), }) diff --git a/rust/kernel/sync/rcu.rs b/rust/kernel/sync/rcu.rs new file mode 100644 index 000000000000..b51d9150ffe2 --- /dev/null +++ b/rust/kernel/sync/rcu.rs @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! RCU support. +//! +//! C header: [`include/linux/rcupdate.h`](srctree/include/linux/rcupdate.h) + +use crate::{bindings, types::NotThreadSafe}; + +/// Evidence that the RCU read side lock is held on the current thread/CPU. +/// +/// The type is explicitly not `Send` because this property is per-thread/CPU. +/// +/// # Invariants +/// +/// The RCU read side lock is actually held while instances of this guard exist. +pub struct Guard(NotThreadSafe); + +impl Guard { + /// Acquires the RCU read side lock and returns a guard. + pub fn new() -> Self { + // SAFETY: An FFI call with no additional requirements. + unsafe { bindings::rcu_read_lock() }; + // INVARIANT: The RCU read side lock was just acquired above. + Self(NotThreadSafe) + } + + /// Explicitly releases the RCU read side lock. + pub fn unlock(self) {} +} + +impl Default for Guard { + fn default() -> Self { + Self::new() + } +} + +impl Drop for Guard { + fn drop(&mut self) { + // SAFETY: By the type invariants, the RCU read side is locked, so it is ok to unlock it. + unsafe { bindings::rcu_read_unlock() }; + } +} + +/// Acquires the RCU read side lock. +pub fn read_lock() -> Guard { + Guard::new() +} diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs index 07bc22a7645c..9e6f6854948d 100644 --- a/rust/kernel/task.rs +++ b/rust/kernel/task.rs @@ -23,6 +23,8 @@ pub const MAX_SCHEDULE_TIMEOUT: c_long = c_long::MAX; pub const TASK_INTERRUPTIBLE: c_int = bindings::TASK_INTERRUPTIBLE as c_int; /// Bitmask for tasks that are sleeping in an uninterruptible state. pub const TASK_UNINTERRUPTIBLE: c_int = bindings::TASK_UNINTERRUPTIBLE as c_int; +/// Bitmask for tasks that are sleeping in a freezable state. +pub const TASK_FREEZABLE: c_int = bindings::TASK_FREEZABLE as c_int; /// Convenience constant for waking up tasks regardless of whether they are in interruptible or /// uninterruptible sleep. pub const TASK_NORMAL: c_uint = bindings::TASK_NORMAL as c_uint; @@ -106,7 +108,7 @@ unsafe impl Send for Task {} unsafe impl Sync for Task {} /// The type of process identifiers (PIDs). -type Pid = bindings::pid_t; +pub type Pid = bindings::pid_t; /// The type of user identifiers (UIDs). #[derive(Copy, Clone)] @@ -320,7 +322,7 @@ impl Task { /// Wakes up the task. pub fn wake_up(&self) { - // SAFETY: It's always safe to call `signal_pending` on a valid task, even if the task + // SAFETY: It's always safe to call `wake_up_process` on a valid task, even if the task // running. unsafe { bindings::wake_up_process(self.as_ptr()) }; } diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs index 379c0f5772e5..f509cb0eb71e 100644 --- a/rust/kernel/time.rs +++ b/rust/kernel/time.rs @@ -8,6 +8,8 @@ //! C header: [`include/linux/jiffies.h`](srctree/include/linux/jiffies.h). //! C header: [`include/linux/ktime.h`](srctree/include/linux/ktime.h). +pub mod hrtimer; + /// The number of nanoseconds per millisecond. pub const NSEC_PER_MSEC: i64 = bindings::NSEC_PER_MSEC as i64; @@ -81,3 +83,69 @@ impl core::ops::Sub for Ktime { } } } + +/// An identifier for a clock. Used when specifying clock sources. +/// +/// +/// Selection of the clock depends on the use case. In some cases the usage of a +/// particular clock is mandatory, e.g. in network protocols, filesystems.In other +/// cases the user of the clock has to decide which clock is best suited for the +/// purpose. In most scenarios clock [`ClockId::Monotonic`] is the best choice as it +/// provides a accurate monotonic notion of time (leap second smearing ignored). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[repr(u32)] +pub enum ClockId { + /// A settable system-wide clock that measures real (i.e., wall-clock) time. + /// + /// Setting this clock requires appropriate privileges. This clock is + /// affected by discontinuous jumps in the system time (e.g., if the system + /// administrator manually changes the clock), and by frequency adjustments + /// performed by NTP and similar applications via adjtime(3), adjtimex(2), + /// clock_adjtime(2), and ntp_adjtime(3). This clock normally counts the + /// number of seconds since 1970-01-01 00:00:00 Coordinated Universal Time + /// (UTC) except that it ignores leap seconds; near a leap second it may be + /// adjusted by leap second smearing to stay roughly in sync with UTC. Leap + /// second smearing applies frequency adjustments to the clock to speed up + /// or slow down the clock to account for the leap second without + /// discontinuities in the clock. If leap second smearing is not applied, + /// the clock will experience discontinuity around leap second adjustment. + RealTime = bindings::CLOCK_REALTIME, + /// A monotonically increasing clock. + /// + /// A nonsettable system-wide clock that represents monotonic time since—as + /// described by POSIX—"some unspecified point in the past". On Linux, that + /// point corresponds to the number of seconds that the system has been + /// running since it was booted. + /// + /// The CLOCK_MONOTONIC clock is not affected by discontinuous jumps in the + /// CLOCK_REAL (e.g., if the system administrator manually changes the + /// clock), but is affected by frequency adjustments. This clock does not + /// count time that the system is suspended. + Monotonic = bindings::CLOCK_MONOTONIC, + /// A monotonic that ticks while system is suspended. + /// + /// A nonsettable system-wide clock that is identical to CLOCK_MONOTONIC, + /// except that it also includes any time that the system is suspended. This + /// allows applications to get a suspend-aware monotonic clock without + /// having to deal with the complications of CLOCK_REALTIME, which may have + /// discontinuities if the time is changed using settimeofday(2) or similar. + BootTime = bindings::CLOCK_BOOTTIME, + /// International Atomic Time. + /// + /// A system-wide clock derived from wall-clock time but counting leap seconds. + /// + /// This clock is coupled to CLOCK_REALTIME and will be set when CLOCK_REALTIME is + /// set, or when the offset to CLOCK_REALTIME is changed via adjtimex(2). This + /// usually happens during boot and **should** not happen during normal operations. + /// However, if NTP or another application adjusts CLOCK_REALTIME by leap second + /// smearing, this clock will not be precise during leap second smearing. + /// + /// The acronym TAI refers to International Atomic Time. + TAI = bindings::CLOCK_TAI, +} + +impl ClockId { + fn into_c(self) -> bindings::clockid_t { + self as bindings::clockid_t + } +} diff --git a/rust/kernel/time/hrtimer.rs b/rust/kernel/time/hrtimer.rs new file mode 100644 index 000000000000..ce53f8579d18 --- /dev/null +++ b/rust/kernel/time/hrtimer.rs @@ -0,0 +1,520 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Intrusive high resolution timers. +//! +//! Allows running timer callbacks without doing allocations at the time of +//! starting the timer. For now, only one timer per type is allowed. +//! +//! # Vocabulary +//! +//! States: +//! +//! - Stopped: initialized but not started, or cancelled, or not restarted. +//! - Started: initialized and started or restarted. +//! - Running: executing the callback. +//! +//! Operations: +//! +//! * Start +//! * Cancel +//! * Restart +//! +//! Events: +//! +//! * Expire +//! +//! ## State Diagram +//! +//! ```text +//! Return NoRestart +//! +---------------------------------------------------------------------+ +//! | | +//! | | +//! | | +//! | Return Restart | +//! | +------------------------+ | +//! | | | | +//! | | | | +//! v v | | +//! +-----------------+ Start +------------------+ +--------+-----+--+ +//! | +---------------->| | | | +//! Init | | | | Expire | | +//! --------->| Stopped | | Started +---------->| Running | +//! | | Cancel | | | | +//! | |<----------------+ | | | +//! +-----------------+ +---------------+--+ +-----------------+ +//! ^ | +//! | | +//! +---------+ +//! Restart +//! ``` +//! +//! +//! A timer is initialized in the **stopped** state. A stopped timer can be +//! **started** by the `start` operation, with an **expiry** time. After the +//! `start` operation, the timer is in the **started** state. When the timer +//! **expires**, the timer enters the **running** state and the handler is +//! executed. After the handler has returned, the timer may enter the +//! **started* or **stopped** state, depending on the return value of the +//! handler. A timer in the **started** or **running** state may be **canceled** +//! by the `cancel` operation. A timer that is cancelled enters the **stopped** +//! state. +//! +//! A `cancel` or `restart` operation on a timer in the **running** state takes +//! effect after the handler has returned and the timer has transitioned +//! out of the **running** state. +//! +//! A `restart` operation on a timer in the **stopped** state is equivalent to a +//! `start` operation. + +use super::ClockId; +use crate::{prelude::*, time::Ktime, types::Opaque}; +use core::marker::PhantomData; +use pin_init::PinInit; + +/// A timer backed by a C `struct hrtimer`. +/// +/// # Invariants +/// +/// * `self.timer` is initialized by `bindings::hrtimer_setup`. +#[pin_data] +#[repr(C)] +pub struct HrTimer<T> { + #[pin] + timer: Opaque<bindings::hrtimer>, + mode: HrTimerMode, + _t: PhantomData<T>, +} + +// SAFETY: Ownership of an `HrTimer` can be moved to other threads and +// used/dropped from there. +unsafe impl<T> Send for HrTimer<T> {} + +// SAFETY: Timer operations are locked on the C side, so it is safe to operate +// on a timer from multiple threads. +unsafe impl<T> Sync for HrTimer<T> {} + +impl<T> HrTimer<T> { + /// Return an initializer for a new timer instance. + pub fn new(mode: HrTimerMode, clock: ClockId) -> impl PinInit<Self> + where + T: HrTimerCallback, + { + pin_init!(Self { + // INVARIANT: We initialize `timer` with `hrtimer_setup` below. + timer <- Opaque::ffi_init(move |place: *mut bindings::hrtimer| { + // SAFETY: By design of `pin_init!`, `place` is a pointer to a + // live allocation. hrtimer_setup will initialize `place` and + // does not require `place` to be initialized prior to the call. + unsafe { + bindings::hrtimer_setup( + place, + Some(T::Pointer::run), + clock.into_c(), + mode.into_c(), + ); + } + }), + mode: mode, + _t: PhantomData, + }) + } + + /// Get a pointer to the contained `bindings::hrtimer`. + /// + /// This function is useful to get access to the value without creating + /// intermediate references. + /// + /// # Safety + /// + /// `this` must point to a live allocation of at least the size of `Self`. + unsafe fn raw_get(this: *const Self) -> *mut bindings::hrtimer { + // SAFETY: The field projection to `timer` does not go out of bounds, + // because the caller of this function promises that `this` points to an + // allocation of at least the size of `Self`. + unsafe { Opaque::raw_get(core::ptr::addr_of!((*this).timer)) } + } + + /// Cancel an initialized and potentially running timer. + /// + /// If the timer handler is running, this function will block until the + /// handler returns. + /// + /// Note that the timer might be started by a concurrent start operation. If + /// so, the timer might not be in the **stopped** state when this function + /// returns. + /// + /// Users of the `HrTimer` API would not usually call this method directly. + /// Instead they would use the safe [`HrTimerHandle::cancel`] on the handle + /// returned when the timer was started. + /// + /// This function is useful to get access to the value without creating + /// intermediate references. + /// + /// # Safety + /// + /// `this` must point to a valid `Self`. + pub(crate) unsafe fn raw_cancel(this: *const Self) -> bool { + // SAFETY: `this` points to an allocation of at least `HrTimer` size. + let c_timer_ptr = unsafe { HrTimer::raw_get(this) }; + + // If the handler is running, this will wait for the handler to return + // before returning. + // SAFETY: `c_timer_ptr` is initialized and valid. Synchronization is + // handled on the C side. + unsafe { bindings::hrtimer_cancel(c_timer_ptr) != 0 } + } +} + +/// Implemented by pointer types that point to structs that contain a [`HrTimer`]. +/// +/// `Self` must be [`Sync`] because it is passed to timer callbacks in another +/// thread of execution (hard or soft interrupt context). +/// +/// Starting a timer returns a [`HrTimerHandle`] that can be used to manipulate +/// the timer. Note that it is OK to call the start function repeatedly, and +/// that more than one [`HrTimerHandle`] associated with a [`HrTimerPointer`] may +/// exist. A timer can be manipulated through any of the handles, and a handle +/// may represent a cancelled timer. +pub trait HrTimerPointer: Sync + Sized { + /// A handle representing a started or restarted timer. + /// + /// If the timer is running or if the timer callback is executing when the + /// handle is dropped, the drop method of [`HrTimerHandle`] should not return + /// until the timer is stopped and the callback has completed. + /// + /// Note: When implementing this trait, consider that it is not unsafe to + /// leak the handle. + type TimerHandle: HrTimerHandle; + + /// Start the timer with expiry after `expires` time units. If the timer was + /// already running, it is restarted with the new expiry time. + fn start(self, expires: Ktime) -> Self::TimerHandle; +} + +/// Unsafe version of [`HrTimerPointer`] for situations where leaking the +/// [`HrTimerHandle`] returned by `start` would be unsound. This is the case for +/// stack allocated timers. +/// +/// Typical implementers are pinned references such as [`Pin<&T>`]. +/// +/// # Safety +/// +/// Implementers of this trait must ensure that instances of types implementing +/// [`UnsafeHrTimerPointer`] outlives any associated [`HrTimerPointer::TimerHandle`] +/// instances. +pub unsafe trait UnsafeHrTimerPointer: Sync + Sized { + /// A handle representing a running timer. + /// + /// # Safety + /// + /// If the timer is running, or if the timer callback is executing when the + /// handle is dropped, the drop method of [`Self::TimerHandle`] must not return + /// until the timer is stopped and the callback has completed. + type TimerHandle: HrTimerHandle; + + /// Start the timer after `expires` time units. If the timer was already + /// running, it is restarted at the new expiry time. + /// + /// # Safety + /// + /// Caller promises keep the timer structure alive until the timer is dead. + /// Caller can ensure this by not leaking the returned [`Self::TimerHandle`]. + unsafe fn start(self, expires: Ktime) -> Self::TimerHandle; +} + +/// A trait for stack allocated timers. +/// +/// # Safety +/// +/// Implementers must ensure that `start_scoped` does not return until the +/// timer is dead and the timer handler is not running. +pub unsafe trait ScopedHrTimerPointer { + /// Start the timer to run after `expires` time units and immediately + /// after call `f`. When `f` returns, the timer is cancelled. + fn start_scoped<T, F>(self, expires: Ktime, f: F) -> T + where + F: FnOnce() -> T; +} + +// SAFETY: By the safety requirement of [`UnsafeHrTimerPointer`], dropping the +// handle returned by [`UnsafeHrTimerPointer::start`] ensures that the timer is +// killed. +unsafe impl<T> ScopedHrTimerPointer for T +where + T: UnsafeHrTimerPointer, +{ + fn start_scoped<U, F>(self, expires: Ktime, f: F) -> U + where + F: FnOnce() -> U, + { + // SAFETY: We drop the timer handle below before returning. + let handle = unsafe { UnsafeHrTimerPointer::start(self, expires) }; + let t = f(); + drop(handle); + t + } +} + +/// Implemented by [`HrTimerPointer`] implementers to give the C timer callback a +/// function to call. +// This is split from `HrTimerPointer` to make it easier to specify trait bounds. +pub trait RawHrTimerCallback { + /// Type of the parameter passed to [`HrTimerCallback::run`]. It may be + /// [`Self`], or a pointer type derived from [`Self`]. + type CallbackTarget<'a>; + + /// Callback to be called from C when timer fires. + /// + /// # Safety + /// + /// Only to be called by C code in the `hrtimer` subsystem. `this` must point + /// to the `bindings::hrtimer` structure that was used to start the timer. + unsafe extern "C" fn run(this: *mut bindings::hrtimer) -> bindings::hrtimer_restart; +} + +/// Implemented by structs that can be the target of a timer callback. +pub trait HrTimerCallback { + /// The type whose [`RawHrTimerCallback::run`] method will be invoked when + /// the timer expires. + type Pointer<'a>: RawHrTimerCallback; + + /// Called by the timer logic when the timer fires. + fn run(this: <Self::Pointer<'_> as RawHrTimerCallback>::CallbackTarget<'_>) -> HrTimerRestart + where + Self: Sized; +} + +/// A handle representing a potentially running timer. +/// +/// More than one handle representing the same timer might exist. +/// +/// # Safety +/// +/// When dropped, the timer represented by this handle must be cancelled, if it +/// is running. If the timer handler is running when the handle is dropped, the +/// drop method must wait for the handler to return before returning. +/// +/// Note: One way to satisfy the safety requirement is to call `Self::cancel` in +/// the drop implementation for `Self.` +pub unsafe trait HrTimerHandle { + /// Cancel the timer. If the timer is in the running state, block till the + /// handler has returned. + /// + /// Note that the timer might be started by a concurrent start operation. If + /// so, the timer might not be in the **stopped** state when this function + /// returns. + fn cancel(&mut self) -> bool; +} + +/// Implemented by structs that contain timer nodes. +/// +/// Clients of the timer API would usually safely implement this trait by using +/// the [`crate::impl_has_hr_timer`] macro. +/// +/// # Safety +/// +/// Implementers of this trait must ensure that the implementer has a +/// [`HrTimer`] field and that all trait methods are implemented according to +/// their documentation. All the methods of this trait must operate on the same +/// field. +pub unsafe trait HasHrTimer<T> { + /// Return a pointer to the [`HrTimer`] within `Self`. + /// + /// This function is useful to get access to the value without creating + /// intermediate references. + /// + /// # Safety + /// + /// `this` must be a valid pointer. + unsafe fn raw_get_timer(this: *const Self) -> *const HrTimer<T>; + + /// Return a pointer to the struct that is containing the [`HrTimer`] pointed + /// to by `ptr`. + /// + /// This function is useful to get access to the value without creating + /// intermediate references. + /// + /// # Safety + /// + /// `ptr` must point to a [`HrTimer<T>`] field in a struct of type `Self`. + unsafe fn timer_container_of(ptr: *mut HrTimer<T>) -> *mut Self + where + Self: Sized; + + /// Get pointer to the contained `bindings::hrtimer` struct. + /// + /// This function is useful to get access to the value without creating + /// intermediate references. + /// + /// # Safety + /// + /// `this` must be a valid pointer. + unsafe fn c_timer_ptr(this: *const Self) -> *const bindings::hrtimer { + // SAFETY: `this` is a valid pointer to a `Self`. + let timer_ptr = unsafe { Self::raw_get_timer(this) }; + + // SAFETY: timer_ptr points to an allocation of at least `HrTimer` size. + unsafe { HrTimer::raw_get(timer_ptr) } + } + + /// Start the timer contained in the `Self` pointed to by `self_ptr`. If + /// it is already running it is removed and inserted. + /// + /// # Safety + /// + /// - `this` must point to a valid `Self`. + /// - Caller must ensure that the pointee of `this` lives until the timer + /// fires or is canceled. + unsafe fn start(this: *const Self, expires: Ktime) { + // SAFETY: By function safety requirement, `this` is a valid `Self`. + unsafe { + bindings::hrtimer_start_range_ns( + Self::c_timer_ptr(this).cast_mut(), + expires.to_ns(), + 0, + (*Self::raw_get_timer(this)).mode.into_c(), + ); + } + } +} + +/// Restart policy for timers. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +#[repr(u32)] +pub enum HrTimerRestart { + /// Timer should not be restarted. + #[allow(clippy::unnecessary_cast)] + NoRestart = bindings::hrtimer_restart_HRTIMER_NORESTART as u32, + /// Timer should be restarted. + #[allow(clippy::unnecessary_cast)] + Restart = bindings::hrtimer_restart_HRTIMER_RESTART as u32, +} + +impl HrTimerRestart { + fn into_c(self) -> bindings::hrtimer_restart { + self as bindings::hrtimer_restart + } +} + +/// Operational mode of [`HrTimer`]. +// NOTE: Some of these have the same encoding on the C side, so we keep +// `repr(Rust)` and convert elsewhere. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum HrTimerMode { + /// Timer expires at the given expiration time. + Absolute, + /// Timer expires after the given expiration time interpreted as a duration from now. + Relative, + /// Timer does not move between CPU cores. + Pinned, + /// Timer handler is executed in soft irq context. + Soft, + /// Timer handler is executed in hard irq context. + Hard, + /// Timer expires at the given expiration time. + /// Timer does not move between CPU cores. + AbsolutePinned, + /// Timer expires after the given expiration time interpreted as a duration from now. + /// Timer does not move between CPU cores. + RelativePinned, + /// Timer expires at the given expiration time. + /// Timer handler is executed in soft irq context. + AbsoluteSoft, + /// Timer expires after the given expiration time interpreted as a duration from now. + /// Timer handler is executed in soft irq context. + RelativeSoft, + /// Timer expires at the given expiration time. + /// Timer does not move between CPU cores. + /// Timer handler is executed in soft irq context. + AbsolutePinnedSoft, + /// Timer expires after the given expiration time interpreted as a duration from now. + /// Timer does not move between CPU cores. + /// Timer handler is executed in soft irq context. + RelativePinnedSoft, + /// Timer expires at the given expiration time. + /// Timer handler is executed in hard irq context. + AbsoluteHard, + /// Timer expires after the given expiration time interpreted as a duration from now. + /// Timer handler is executed in hard irq context. + RelativeHard, + /// Timer expires at the given expiration time. + /// Timer does not move between CPU cores. + /// Timer handler is executed in hard irq context. + AbsolutePinnedHard, + /// Timer expires after the given expiration time interpreted as a duration from now. + /// Timer does not move between CPU cores. + /// Timer handler is executed in hard irq context. + RelativePinnedHard, +} + +impl HrTimerMode { + fn into_c(self) -> bindings::hrtimer_mode { + use bindings::*; + match self { + HrTimerMode::Absolute => hrtimer_mode_HRTIMER_MODE_ABS, + HrTimerMode::Relative => hrtimer_mode_HRTIMER_MODE_REL, + HrTimerMode::Pinned => hrtimer_mode_HRTIMER_MODE_PINNED, + HrTimerMode::Soft => hrtimer_mode_HRTIMER_MODE_SOFT, + HrTimerMode::Hard => hrtimer_mode_HRTIMER_MODE_HARD, + HrTimerMode::AbsolutePinned => hrtimer_mode_HRTIMER_MODE_ABS_PINNED, + HrTimerMode::RelativePinned => hrtimer_mode_HRTIMER_MODE_REL_PINNED, + HrTimerMode::AbsoluteSoft => hrtimer_mode_HRTIMER_MODE_ABS_SOFT, + HrTimerMode::RelativeSoft => hrtimer_mode_HRTIMER_MODE_REL_SOFT, + HrTimerMode::AbsolutePinnedSoft => hrtimer_mode_HRTIMER_MODE_ABS_PINNED_SOFT, + HrTimerMode::RelativePinnedSoft => hrtimer_mode_HRTIMER_MODE_REL_PINNED_SOFT, + HrTimerMode::AbsoluteHard => hrtimer_mode_HRTIMER_MODE_ABS_HARD, + HrTimerMode::RelativeHard => hrtimer_mode_HRTIMER_MODE_REL_HARD, + HrTimerMode::AbsolutePinnedHard => hrtimer_mode_HRTIMER_MODE_ABS_PINNED_HARD, + HrTimerMode::RelativePinnedHard => hrtimer_mode_HRTIMER_MODE_REL_PINNED_HARD, + } + } +} + +/// Use to implement the [`HasHrTimer<T>`] trait. +/// +/// See [`module`] documentation for an example. +/// +/// [`module`]: crate::time::hrtimer +#[macro_export] +macro_rules! impl_has_hr_timer { + ( + impl$({$($generics:tt)*})? + HasHrTimer<$timer_type:ty> + for $self:ty + { self.$field:ident } + $($rest:tt)* + ) => { + // SAFETY: This implementation of `raw_get_timer` only compiles if the + // field has the right type. + unsafe impl$(<$($generics)*>)? $crate::time::hrtimer::HasHrTimer<$timer_type> for $self { + + #[inline] + unsafe fn raw_get_timer( + this: *const Self, + ) -> *const $crate::time::hrtimer::HrTimer<$timer_type> { + // SAFETY: The caller promises that the pointer is not dangling. + unsafe { ::core::ptr::addr_of!((*this).$field) } + } + + #[inline] + unsafe fn timer_container_of( + ptr: *mut $crate::time::hrtimer::HrTimer<$timer_type>, + ) -> *mut Self { + // SAFETY: As per the safety requirement of this function, `ptr` + // is pointing inside a `$timer_type`. + unsafe { ::kernel::container_of!(ptr, $timer_type, $field).cast_mut() } + } + } + } +} + +mod arc; +pub use arc::ArcHrTimerHandle; +mod pin; +pub use pin::PinHrTimerHandle; +mod pin_mut; +pub use pin_mut::PinMutHrTimerHandle; +// `box` is a reserved keyword, so prefix with `t` for timer +mod tbox; +pub use tbox::BoxHrTimerHandle; diff --git a/rust/kernel/time/hrtimer/arc.rs b/rust/kernel/time/hrtimer/arc.rs new file mode 100644 index 000000000000..4a984d85b4a1 --- /dev/null +++ b/rust/kernel/time/hrtimer/arc.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0 + +use super::HasHrTimer; +use super::HrTimer; +use super::HrTimerCallback; +use super::HrTimerHandle; +use super::HrTimerPointer; +use super::RawHrTimerCallback; +use crate::sync::Arc; +use crate::sync::ArcBorrow; +use crate::time::Ktime; + +/// A handle for an `Arc<HasHrTimer<T>>` returned by a call to +/// [`HrTimerPointer::start`]. +pub struct ArcHrTimerHandle<T> +where + T: HasHrTimer<T>, +{ + pub(crate) inner: Arc<T>, +} + +// SAFETY: We implement drop below, and we cancel the timer in the drop +// implementation. +unsafe impl<T> HrTimerHandle for ArcHrTimerHandle<T> +where + T: HasHrTimer<T>, +{ + fn cancel(&mut self) -> bool { + let self_ptr = Arc::as_ptr(&self.inner); + + // SAFETY: As we obtained `self_ptr` from a valid reference above, it + // must point to a valid `T`. + let timer_ptr = unsafe { <T as HasHrTimer<T>>::raw_get_timer(self_ptr) }; + + // SAFETY: As `timer_ptr` points into `T` and `T` is valid, `timer_ptr` + // must point to a valid `HrTimer` instance. + unsafe { HrTimer::<T>::raw_cancel(timer_ptr) } + } +} + +impl<T> Drop for ArcHrTimerHandle<T> +where + T: HasHrTimer<T>, +{ + fn drop(&mut self) { + self.cancel(); + } +} + +impl<T> HrTimerPointer for Arc<T> +where + T: 'static, + T: Send + Sync, + T: HasHrTimer<T>, + T: for<'a> HrTimerCallback<Pointer<'a> = Self>, +{ + type TimerHandle = ArcHrTimerHandle<T>; + + fn start(self, expires: Ktime) -> ArcHrTimerHandle<T> { + // SAFETY: + // - We keep `self` alive by wrapping it in a handle below. + // - Since we generate the pointer passed to `start` from a valid + // reference, it is a valid pointer. + unsafe { T::start(Arc::as_ptr(&self), expires) }; + ArcHrTimerHandle { inner: self } + } +} + +impl<T> RawHrTimerCallback for Arc<T> +where + T: 'static, + T: HasHrTimer<T>, + T: for<'a> HrTimerCallback<Pointer<'a> = Self>, +{ + type CallbackTarget<'a> = ArcBorrow<'a, T>; + + unsafe extern "C" fn run(ptr: *mut bindings::hrtimer) -> bindings::hrtimer_restart { + // `HrTimer` is `repr(C)` + let timer_ptr = ptr.cast::<super::HrTimer<T>>(); + + // SAFETY: By C API contract `ptr` is the pointer we passed when + // queuing the timer, so it is a `HrTimer<T>` embedded in a `T`. + let data_ptr = unsafe { T::timer_container_of(timer_ptr) }; + + // SAFETY: + // - `data_ptr` is derived form the pointer to the `T` that was used to + // queue the timer. + // - As per the safety requirements of the trait `HrTimerHandle`, the + // `ArcHrTimerHandle` associated with this timer is guaranteed to + // be alive until this method returns. That handle borrows the `T` + // behind `data_ptr` thus guaranteeing the validity of + // the `ArcBorrow` created below. + // - We own one refcount in the `ArcTimerHandle` associated with this + // timer, so it is not possible to get a `UniqueArc` to this + // allocation from other `Arc` clones. + let receiver = unsafe { ArcBorrow::from_raw(data_ptr) }; + + T::run(receiver).into_c() + } +} diff --git a/rust/kernel/time/hrtimer/pin.rs b/rust/kernel/time/hrtimer/pin.rs new file mode 100644 index 000000000000..f760db265c7b --- /dev/null +++ b/rust/kernel/time/hrtimer/pin.rs @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 + +use super::HasHrTimer; +use super::HrTimer; +use super::HrTimerCallback; +use super::HrTimerHandle; +use super::RawHrTimerCallback; +use super::UnsafeHrTimerPointer; +use crate::time::Ktime; +use core::pin::Pin; + +/// A handle for a `Pin<&HasHrTimer>`. When the handle exists, the timer might be +/// running. +pub struct PinHrTimerHandle<'a, T> +where + T: HasHrTimer<T>, +{ + pub(crate) inner: Pin<&'a T>, +} + +// SAFETY: We cancel the timer when the handle is dropped. The implementation of +// the `cancel` method will block if the timer handler is running. +unsafe impl<'a, T> HrTimerHandle for PinHrTimerHandle<'a, T> +where + T: HasHrTimer<T>, +{ + fn cancel(&mut self) -> bool { + let self_ptr: *const T = self.inner.get_ref(); + + // SAFETY: As we got `self_ptr` from a reference above, it must point to + // a valid `T`. + let timer_ptr = unsafe { <T as HasHrTimer<T>>::raw_get_timer(self_ptr) }; + + // SAFETY: As `timer_ptr` is derived from a reference, it must point to + // a valid and initialized `HrTimer`. + unsafe { HrTimer::<T>::raw_cancel(timer_ptr) } + } +} + +impl<'a, T> Drop for PinHrTimerHandle<'a, T> +where + T: HasHrTimer<T>, +{ + fn drop(&mut self) { + self.cancel(); + } +} + +// SAFETY: We capture the lifetime of `Self` when we create a `PinHrTimerHandle`, +// so `Self` will outlive the handle. +unsafe impl<'a, T> UnsafeHrTimerPointer for Pin<&'a T> +where + T: Send + Sync, + T: HasHrTimer<T>, + T: HrTimerCallback<Pointer<'a> = Self>, +{ + type TimerHandle = PinHrTimerHandle<'a, T>; + + unsafe fn start(self, expires: Ktime) -> Self::TimerHandle { + // Cast to pointer + let self_ptr: *const T = self.get_ref(); + + // SAFETY: + // - As we derive `self_ptr` from a reference above, it must point to a + // valid `T`. + // - We keep `self` alive by wrapping it in a handle below. + unsafe { T::start(self_ptr, expires) }; + + PinHrTimerHandle { inner: self } + } +} + +impl<'a, T> RawHrTimerCallback for Pin<&'a T> +where + T: HasHrTimer<T>, + T: HrTimerCallback<Pointer<'a> = Self>, +{ + type CallbackTarget<'b> = Self; + + unsafe extern "C" fn run(ptr: *mut bindings::hrtimer) -> bindings::hrtimer_restart { + // `HrTimer` is `repr(C)` + let timer_ptr = ptr as *mut HrTimer<T>; + + // SAFETY: By the safety requirement of this function, `timer_ptr` + // points to a `HrTimer<T>` contained in an `T`. + let receiver_ptr = unsafe { T::timer_container_of(timer_ptr) }; + + // SAFETY: + // - By the safety requirement of this function, `timer_ptr` + // points to a `HrTimer<T>` contained in an `T`. + // - As per the safety requirements of the trait `HrTimerHandle`, the + // `PinHrTimerHandle` associated with this timer is guaranteed to + // be alive until this method returns. That handle borrows the `T` + // behind `receiver_ptr`, thus guaranteeing the validity of + // the reference created below. + let receiver_ref = unsafe { &*receiver_ptr }; + + // SAFETY: `receiver_ref` only exists as pinned, so it is safe to pin it + // here. + let receiver_pin = unsafe { Pin::new_unchecked(receiver_ref) }; + + T::run(receiver_pin).into_c() + } +} diff --git a/rust/kernel/time/hrtimer/pin_mut.rs b/rust/kernel/time/hrtimer/pin_mut.rs new file mode 100644 index 000000000000..90c0351d62e4 --- /dev/null +++ b/rust/kernel/time/hrtimer/pin_mut.rs @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0 + +use super::{ + HasHrTimer, HrTimer, HrTimerCallback, HrTimerHandle, RawHrTimerCallback, UnsafeHrTimerPointer, +}; +use crate::time::Ktime; +use core::{marker::PhantomData, pin::Pin, ptr::NonNull}; + +/// A handle for a `Pin<&mut HasHrTimer>`. When the handle exists, the timer might +/// be running. +pub struct PinMutHrTimerHandle<'a, T> +where + T: HasHrTimer<T>, +{ + pub(crate) inner: NonNull<T>, + _p: PhantomData<&'a mut T>, +} + +// SAFETY: We cancel the timer when the handle is dropped. The implementation of +// the `cancel` method will block if the timer handler is running. +unsafe impl<'a, T> HrTimerHandle for PinMutHrTimerHandle<'a, T> +where + T: HasHrTimer<T>, +{ + fn cancel(&mut self) -> bool { + let self_ptr = self.inner.as_ptr(); + + // SAFETY: As we got `self_ptr` from a reference above, it must point to + // a valid `T`. + let timer_ptr = unsafe { <T as HasHrTimer<T>>::raw_get_timer(self_ptr) }; + + // SAFETY: As `timer_ptr` is derived from a reference, it must point to + // a valid and initialized `HrTimer`. + unsafe { HrTimer::<T>::raw_cancel(timer_ptr) } + } +} + +impl<'a, T> Drop for PinMutHrTimerHandle<'a, T> +where + T: HasHrTimer<T>, +{ + fn drop(&mut self) { + self.cancel(); + } +} + +// SAFETY: We capture the lifetime of `Self` when we create a +// `PinMutHrTimerHandle`, so `Self` will outlive the handle. +unsafe impl<'a, T> UnsafeHrTimerPointer for Pin<&'a mut T> +where + T: Send + Sync, + T: HasHrTimer<T>, + T: HrTimerCallback<Pointer<'a> = Self>, +{ + type TimerHandle = PinMutHrTimerHandle<'a, T>; + + unsafe fn start(mut self, expires: Ktime) -> Self::TimerHandle { + // SAFETY: + // - We promise not to move out of `self`. We only pass `self` + // back to the caller as a `Pin<&mut self>`. + // - The return value of `get_unchecked_mut` is guaranteed not to be null. + let self_ptr = unsafe { NonNull::new_unchecked(self.as_mut().get_unchecked_mut()) }; + + // SAFETY: + // - As we derive `self_ptr` from a reference above, it must point to a + // valid `T`. + // - We keep `self` alive by wrapping it in a handle below. + unsafe { T::start(self_ptr.as_ptr(), expires) }; + + PinMutHrTimerHandle { + inner: self_ptr, + _p: PhantomData, + } + } +} + +impl<'a, T> RawHrTimerCallback for Pin<&'a mut T> +where + T: HasHrTimer<T>, + T: HrTimerCallback<Pointer<'a> = Self>, +{ + type CallbackTarget<'b> = Self; + + unsafe extern "C" fn run(ptr: *mut bindings::hrtimer) -> bindings::hrtimer_restart { + // `HrTimer` is `repr(C)` + let timer_ptr = ptr as *mut HrTimer<T>; + + // SAFETY: By the safety requirement of this function, `timer_ptr` + // points to a `HrTimer<T>` contained in an `T`. + let receiver_ptr = unsafe { T::timer_container_of(timer_ptr) }; + + // SAFETY: + // - By the safety requirement of this function, `timer_ptr` + // points to a `HrTimer<T>` contained in an `T`. + // - As per the safety requirements of the trait `HrTimerHandle`, the + // `PinMutHrTimerHandle` associated with this timer is guaranteed to + // be alive until this method returns. That handle borrows the `T` + // behind `receiver_ptr` mutably thus guaranteeing the validity of + // the reference created below. + let receiver_ref = unsafe { &mut *receiver_ptr }; + + // SAFETY: `receiver_ref` only exists as pinned, so it is safe to pin it + // here. + let receiver_pin = unsafe { Pin::new_unchecked(receiver_ref) }; + + T::run(receiver_pin).into_c() + } +} diff --git a/rust/kernel/time/hrtimer/tbox.rs b/rust/kernel/time/hrtimer/tbox.rs new file mode 100644 index 000000000000..2071cae07234 --- /dev/null +++ b/rust/kernel/time/hrtimer/tbox.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 + +use super::HasHrTimer; +use super::HrTimer; +use super::HrTimerCallback; +use super::HrTimerHandle; +use super::HrTimerPointer; +use super::RawHrTimerCallback; +use crate::prelude::*; +use crate::time::Ktime; +use core::ptr::NonNull; + +/// A handle for a [`Box<HasHrTimer<T>>`] returned by a call to +/// [`HrTimerPointer::start`]. +/// +/// # Invariants +/// +/// - `self.inner` comes from a `Box::into_raw` call. +pub struct BoxHrTimerHandle<T, A> +where + T: HasHrTimer<T>, + A: crate::alloc::Allocator, +{ + pub(crate) inner: NonNull<T>, + _p: core::marker::PhantomData<A>, +} + +// SAFETY: We implement drop below, and we cancel the timer in the drop +// implementation. +unsafe impl<T, A> HrTimerHandle for BoxHrTimerHandle<T, A> +where + T: HasHrTimer<T>, + A: crate::alloc::Allocator, +{ + fn cancel(&mut self) -> bool { + // SAFETY: As we obtained `self.inner` from a valid reference when we + // created `self`, it must point to a valid `T`. + let timer_ptr = unsafe { <T as HasHrTimer<T>>::raw_get_timer(self.inner.as_ptr()) }; + + // SAFETY: As `timer_ptr` points into `T` and `T` is valid, `timer_ptr` + // must point to a valid `HrTimer` instance. + unsafe { HrTimer::<T>::raw_cancel(timer_ptr) } + } +} + +impl<T, A> Drop for BoxHrTimerHandle<T, A> +where + T: HasHrTimer<T>, + A: crate::alloc::Allocator, +{ + fn drop(&mut self) { + self.cancel(); + // SAFETY: By type invariant, `self.inner` came from a `Box::into_raw` + // call. + drop(unsafe { Box::<T, A>::from_raw(self.inner.as_ptr()) }) + } +} + +impl<T, A> HrTimerPointer for Pin<Box<T, A>> +where + T: 'static, + T: Send + Sync, + T: HasHrTimer<T>, + T: for<'a> HrTimerCallback<Pointer<'a> = Pin<Box<T, A>>>, + A: crate::alloc::Allocator, +{ + type TimerHandle = BoxHrTimerHandle<T, A>; + + fn start(self, expires: Ktime) -> Self::TimerHandle { + // SAFETY: + // - We will not move out of this box during timer callback (we pass an + // immutable reference to the callback). + // - `Box::into_raw` is guaranteed to return a valid pointer. + let inner = + unsafe { NonNull::new_unchecked(Box::into_raw(Pin::into_inner_unchecked(self))) }; + + // SAFETY: + // - We keep `self` alive by wrapping it in a handle below. + // - Since we generate the pointer passed to `start` from a valid + // reference, it is a valid pointer. + unsafe { T::start(inner.as_ptr(), expires) }; + + // INVARIANT: `inner` came from `Box::into_raw` above. + BoxHrTimerHandle { + inner, + _p: core::marker::PhantomData, + } + } +} + +impl<T, A> RawHrTimerCallback for Pin<Box<T, A>> +where + T: 'static, + T: HasHrTimer<T>, + T: for<'a> HrTimerCallback<Pointer<'a> = Pin<Box<T, A>>>, + A: crate::alloc::Allocator, +{ + type CallbackTarget<'a> = Pin<&'a mut T>; + + unsafe extern "C" fn run(ptr: *mut bindings::hrtimer) -> bindings::hrtimer_restart { + // `HrTimer` is `repr(C)` + let timer_ptr = ptr.cast::<super::HrTimer<T>>(); + + // SAFETY: By C API contract `ptr` is the pointer we passed when + // queuing the timer, so it is a `HrTimer<T>` embedded in a `T`. + let data_ptr = unsafe { T::timer_container_of(timer_ptr) }; + + // SAFETY: + // - As per the safety requirements of the trait `HrTimerHandle`, the + // `BoxHrTimerHandle` associated with this timer is guaranteed to + // be alive until this method returns. That handle owns the `T` + // behind `data_ptr` thus guaranteeing the validity of + // the reference created below. + // - As `data_ptr` comes from a `Pin<Box<T>>`, only pinned references to + // `data_ptr` exist. + let data_mut_ref = unsafe { Pin::new_unchecked(&mut *data_ptr) }; + + T::run(data_mut_ref).into_c() + } +} diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index ec6457bb3084..9d0471afc964 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -2,7 +2,6 @@ //! Kernel types. -use crate::init::{self, PinInit}; use core::{ cell::UnsafeCell, marker::{PhantomData, PhantomPinned}, @@ -10,6 +9,7 @@ use core::{ ops::{Deref, DerefMut}, ptr::NonNull, }; +use pin_init::{PinInit, Zeroable}; /// Used to transfer ownership to and from foreign (non-Rust) languages. /// @@ -19,35 +19,34 @@ use core::{ /// This trait is meant to be used in cases when Rust objects are stored in C objects and /// eventually "freed" back to Rust. pub trait ForeignOwnable: Sized { - /// Type of values borrowed between calls to [`ForeignOwnable::into_foreign`] and - /// [`ForeignOwnable::from_foreign`]. + /// Type used to immutably borrow a value that is currently foreign-owned. type Borrowed<'a>; + /// Type used to mutably borrow a value that is currently foreign-owned. + type BorrowedMut<'a>; + /// Converts a Rust-owned object to a foreign-owned one. /// /// The foreign representation is a pointer to void. There are no guarantees for this pointer. /// For example, it might be invalid, dangling or pointing to uninitialized memory. Using it in - /// any way except for [`ForeignOwnable::from_foreign`], [`ForeignOwnable::borrow`], - /// [`ForeignOwnable::try_from_foreign`] can result in undefined behavior. - fn into_foreign(self) -> *const crate::ffi::c_void; - - /// Borrows a foreign-owned object. - /// - /// # Safety + /// any way except for [`from_foreign`], [`try_from_foreign`], [`borrow`], or [`borrow_mut`] can + /// result in undefined behavior. /// - /// `ptr` must have been returned by a previous call to [`ForeignOwnable::into_foreign`] for - /// which a previous matching [`ForeignOwnable::from_foreign`] hasn't been called yet. - unsafe fn borrow<'a>(ptr: *const crate::ffi::c_void) -> Self::Borrowed<'a>; + /// [`from_foreign`]: Self::from_foreign + /// [`try_from_foreign`]: Self::try_from_foreign + /// [`borrow`]: Self::borrow + /// [`borrow_mut`]: Self::borrow_mut + fn into_foreign(self) -> *mut crate::ffi::c_void; /// Converts a foreign-owned object back to a Rust-owned one. /// /// # Safety /// - /// `ptr` must have been returned by a previous call to [`ForeignOwnable::into_foreign`] for - /// which a previous matching [`ForeignOwnable::from_foreign`] hasn't been called yet. - /// Additionally, all instances (if any) of values returned by [`ForeignOwnable::borrow`] for - /// this object must have been dropped. - unsafe fn from_foreign(ptr: *const crate::ffi::c_void) -> Self; + /// The provided pointer must have been returned by a previous call to [`into_foreign`], and it + /// must not be passed to `from_foreign` more than once. + /// + /// [`into_foreign`]: Self::into_foreign + unsafe fn from_foreign(ptr: *mut crate::ffi::c_void) -> Self; /// Tries to convert a foreign-owned object back to a Rust-owned one. /// @@ -56,9 +55,10 @@ pub trait ForeignOwnable: Sized { /// /// # Safety /// - /// `ptr` must either be null or satisfy the safety requirements for - /// [`ForeignOwnable::from_foreign`]. - unsafe fn try_from_foreign(ptr: *const crate::ffi::c_void) -> Option<Self> { + /// `ptr` must either be null or satisfy the safety requirements for [`from_foreign`]. + /// + /// [`from_foreign`]: Self::from_foreign + unsafe fn try_from_foreign(ptr: *mut crate::ffi::c_void) -> Option<Self> { if ptr.is_null() { None } else { @@ -67,18 +67,63 @@ pub trait ForeignOwnable: Sized { unsafe { Some(Self::from_foreign(ptr)) } } } + + /// Borrows a foreign-owned object immutably. + /// + /// This method provides a way to access a foreign-owned value from Rust immutably. It provides + /// you with exactly the same abilities as an `&Self` when the value is Rust-owned. + /// + /// # Safety + /// + /// The provided pointer must have been returned by a previous call to [`into_foreign`], and if + /// the pointer is ever passed to [`from_foreign`], then that call must happen after the end of + /// the lifetime `'a`. + /// + /// [`into_foreign`]: Self::into_foreign + /// [`from_foreign`]: Self::from_foreign + unsafe fn borrow<'a>(ptr: *mut crate::ffi::c_void) -> Self::Borrowed<'a>; + + /// Borrows a foreign-owned object mutably. + /// + /// This method provides a way to access a foreign-owned value from Rust mutably. It provides + /// you with exactly the same abilities as an `&mut Self` when the value is Rust-owned, except + /// that the address of the object must not be changed. + /// + /// Note that for types like [`Arc`], an `&mut Arc<T>` only gives you immutable access to the + /// inner value, so this method also only provides immutable access in that case. + /// + /// In the case of `Box<T>`, this method gives you the ability to modify the inner `T`, but it + /// does not let you change the box itself. That is, you cannot change which allocation the box + /// points at. + /// + /// # Safety + /// + /// The provided pointer must have been returned by a previous call to [`into_foreign`], and if + /// the pointer is ever passed to [`from_foreign`], then that call must happen after the end of + /// the lifetime `'a`. + /// + /// The lifetime `'a` must not overlap with the lifetime of any other call to [`borrow`] or + /// `borrow_mut` on the same object. + /// + /// [`into_foreign`]: Self::into_foreign + /// [`from_foreign`]: Self::from_foreign + /// [`borrow`]: Self::borrow + /// [`Arc`]: crate::sync::Arc + unsafe fn borrow_mut<'a>(ptr: *mut crate::ffi::c_void) -> Self::BorrowedMut<'a>; } impl ForeignOwnable for () { type Borrowed<'a> = (); + type BorrowedMut<'a> = (); - fn into_foreign(self) -> *const crate::ffi::c_void { + fn into_foreign(self) -> *mut crate::ffi::c_void { core::ptr::NonNull::dangling().as_ptr() } - unsafe fn borrow<'a>(_: *const crate::ffi::c_void) -> Self::Borrowed<'a> {} + unsafe fn from_foreign(_: *mut crate::ffi::c_void) -> Self {} - unsafe fn from_foreign(_: *const crate::ffi::c_void) -> Self {} + unsafe fn borrow<'a>(_: *mut crate::ffi::c_void) -> Self::Borrowed<'a> {} + unsafe fn borrow_mut<'a>(_: *mut crate::ffi::c_void) -> Self::BorrowedMut<'a> {} } /// Runs a cleanup function/closure when dropped. @@ -206,7 +251,7 @@ impl<T, F: FnOnce(T)> Drop for ScopeGuard<T, F> { /// Stores an opaque value. /// -/// `Opaque<T>` is meant to be used with FFI objects that are never interpreted by Rust code. +/// [`Opaque<T>`] is meant to be used with FFI objects that are never interpreted by Rust code. /// /// It is used to wrap structs from the C side, like for example `Opaque<bindings::mutex>`. /// It gets rid of all the usual assumptions that Rust has for a value: @@ -221,7 +266,7 @@ impl<T, F: FnOnce(T)> Drop for ScopeGuard<T, F> { /// This has to be used for all values that the C side has access to, because it can't be ensured /// that the C side is adhering to the usual constraints that Rust needs. /// -/// Using `Opaque<T>` allows to continue to use references on the Rust side even for values shared +/// Using [`Opaque<T>`] allows to continue to use references on the Rust side even for values shared /// with C. /// /// # Examples @@ -264,6 +309,9 @@ pub struct Opaque<T> { _pin: PhantomPinned, } +// SAFETY: `Opaque<T>` allows the inner value to be any bit pattern, including all zeros. +unsafe impl<T> Zeroable for Opaque<T> {} + impl<T> Opaque<T> { /// Creates a new opaque value. pub const fn new(value: T) -> Self { @@ -281,6 +329,17 @@ impl<T> Opaque<T> { } } + /// Create an opaque pin-initializer from the given pin-initializer. + pub fn pin_init(slot: impl PinInit<T>) -> impl PinInit<Self> { + Self::ffi_init(|ptr: *mut T| { + // SAFETY: + // - `ptr` is a valid pointer to uninitialized memory, + // - `slot` is not accessed on error; the call is infallible, + // - `slot` is pinned in memory. + let _ = unsafe { PinInit::<T>::__pinned_init(slot, ptr) }; + }) + } + /// Creates a pin-initializer from the given initializer closure. /// /// The returned initializer calls the given closure with the pointer to the inner `T` of this @@ -293,7 +352,7 @@ impl<T> Opaque<T> { // SAFETY: We contain a `MaybeUninit`, so it is OK for the `init_func` to not fully // initialize the `T`. unsafe { - init::pin_init_from_closure::<_, ::core::convert::Infallible>(move |slot| { + pin_init::pin_init_from_closure::<_, ::core::convert::Infallible>(move |slot| { init_func(Self::raw_get(slot)); Ok(()) }) @@ -313,7 +372,9 @@ impl<T> Opaque<T> { ) -> impl PinInit<Self, E> { // SAFETY: We contain a `MaybeUninit`, so it is OK for the `init_func` to not fully // initialize the `T`. - unsafe { init::pin_init_from_closure::<_, E>(move |slot| init_func(Self::raw_get(slot))) } + unsafe { + pin_init::pin_init_from_closure::<_, E>(move |slot| init_func(Self::raw_get(slot))) + } } /// Returns a raw pointer to the opaque data. @@ -434,7 +495,7 @@ impl<T: AlwaysRefCounted> ARef<T> { /// } /// /// let mut data = Empty {}; - /// let ptr = NonNull::<Empty>::new(&mut data as *mut _).unwrap(); + /// let ptr = NonNull::<Empty>::new(&mut data).unwrap(); /// # // SAFETY: TODO. /// let data_ref: ARef<Empty> = unsafe { ARef::from_raw(ptr) }; /// let raw_ptr: NonNull<Empty> = ARef::into_raw(data_ref); diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs index cc044924867b..80a9782b1c6e 100644 --- a/rust/kernel/uaccess.rs +++ b/rust/kernel/uaccess.rs @@ -5,7 +5,7 @@ //! C header: [`include/linux/uaccess.h`](srctree/include/linux/uaccess.h) use crate::{ - alloc::Flags, + alloc::{Allocator, Flags}, bindings, error::Result, ffi::c_void, @@ -127,7 +127,7 @@ impl UserSlice { /// Reads the entirety of the user slice, appending it to the end of the provided buffer. /// /// Fails with [`EFAULT`] if the read happens on a bad address. - pub fn read_all(self, buf: &mut KVec<u8>, flags: Flags) -> Result { + pub fn read_all<A: Allocator>(self, buf: &mut Vec<u8, A>, flags: Flags) -> Result { self.reader().read_all(buf, flags) } @@ -281,12 +281,11 @@ impl UserSliceReader { /// Reads the entirety of the user slice, appending it to the end of the provided buffer. /// /// Fails with [`EFAULT`] if the read happens on a bad address. - pub fn read_all(mut self, buf: &mut KVec<u8>, flags: Flags) -> Result { + pub fn read_all<A: Allocator>(mut self, buf: &mut Vec<u8, A>, flags: Flags) -> Result { let len = self.length; buf.reserve(len, flags)?; - // The call to `try_reserve` was successful, so the spare capacity is at least `len` bytes - // long. + // The call to `reserve` was successful, so the spare capacity is at least `len` bytes long. self.read_raw(&mut buf.spare_capacity_mut()[..len])?; // SAFETY: Since the call to `read_raw` was successful, so the next `len` bytes of the diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index fd3e97192ed8..f98bd02b838f 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -60,7 +60,7 @@ //! type Pointer = Arc<MyStruct>; //! //! fn run(this: Arc<MyStruct>) { -//! pr_info!("The value is: {}", this.value); +//! pr_info!("The value is: {}\n", this.value); //! } //! } //! @@ -69,6 +69,7 @@ //! fn print_later(val: Arc<MyStruct>) { //! let _ = workqueue::system().enqueue(val); //! } +//! # print_later(MyStruct::new(42).unwrap()); //! ``` //! //! The following example shows how multiple `work_struct` fields can be used: @@ -107,7 +108,7 @@ //! type Pointer = Arc<MyStruct>; //! //! fn run(this: Arc<MyStruct>) { -//! pr_info!("The value is: {}", this.value_1); +//! pr_info!("The value is: {}\n", this.value_1); //! } //! } //! @@ -115,7 +116,7 @@ //! type Pointer = Arc<MyStruct>; //! //! fn run(this: Arc<MyStruct>) { -//! pr_info!("The second value is: {}", this.value_2); +//! pr_info!("The second value is: {}\n", this.value_2); //! } //! } //! @@ -126,6 +127,8 @@ //! fn print_2_later(val: Arc<MyStruct>) { //! let _ = workqueue::system().enqueue::<Arc<MyStruct>, 2>(val); //! } +//! # print_1_later(MyStruct::new(24, 25).unwrap()); +//! # print_2_later(MyStruct::new(41, 42).unwrap()); //! ``` //! //! C header: [`include/linux/workqueue.h`](srctree/include/linux/workqueue.h) @@ -366,7 +369,7 @@ unsafe impl<T: ?Sized, const ID: u64> Sync for Work<T, ID> {} impl<T: ?Sized, const ID: u64> Work<T, ID> { /// Creates a new instance of [`Work`]. #[inline] - pub fn new(name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self> where T: WorkItem<ID>, { @@ -700,3 +703,21 @@ pub fn system_freezable_power_efficient() -> &'static Queue { // SAFETY: `system_freezable_power_efficient_wq` is a C global, always available. unsafe { Queue::from_raw(bindings::system_freezable_power_efficient_wq) } } + +/// Returns the system bottom halves work queue (`system_bh_wq`). +/// +/// It is similar to the one returned by [`system`] but for work items which +/// need to run from a softirq context. +pub fn system_bh() -> &'static Queue { + // SAFETY: `system_bh_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_bh_wq) } +} + +/// Returns the system bottom halves high-priority work queue (`system_bh_highpri_wq`). +/// +/// It is similar to the one returned by [`system_bh`] but for work items which +/// require higher scheduling priority. +pub fn system_bh_highpri() -> &'static Queue { + // SAFETY: `system_bh_highpri_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_bh_highpri_wq) } +} diff --git a/rust/macros/export.rs b/rust/macros/export.rs new file mode 100644 index 000000000000..a08f6337d5c8 --- /dev/null +++ b/rust/macros/export.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 + +use crate::helpers::function_name; +use proc_macro::TokenStream; + +/// Please see [`crate::export`] for documentation. +pub(crate) fn export(_attr: TokenStream, ts: TokenStream) -> TokenStream { + let Some(name) = function_name(ts.clone()) else { + return "::core::compile_error!(\"The #[export] attribute must be used on a function.\");" + .parse::<TokenStream>() + .unwrap(); + }; + + // This verifies that the function has the same signature as the declaration generated by + // bindgen. It makes use of the fact that all branches of an if/else must have the same type. + let signature_check = quote!( + const _: () = { + if true { + ::kernel::bindings::#name + } else { + #name + }; + }; + ); + + let no_mangle = quote!(#[no_mangle]); + + TokenStream::from_iter([signature_check, no_mangle, ts]) +} diff --git a/rust/macros/helpers.rs b/rust/macros/helpers.rs index 563dcd2b7ace..a3ee27e29a6f 100644 --- a/rust/macros/helpers.rs +++ b/rust/macros/helpers.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 -use proc_macro::{token_stream, Group, TokenStream, TokenTree}; +use proc_macro::{token_stream, Group, Ident, TokenStream, TokenTree}; pub(crate) fn try_ident(it: &mut token_stream::IntoIter) -> Option<String> { if let Some(TokenTree::Ident(ident)) = it.next() { @@ -70,148 +70,19 @@ pub(crate) fn expect_end(it: &mut token_stream::IntoIter) { } } -/// Parsed generics. -/// -/// See the field documentation for an explanation what each of the fields represents. -/// -/// # Examples -/// -/// ```rust,ignore -/// # let input = todo!(); -/// let (Generics { decl_generics, impl_generics, ty_generics }, rest) = parse_generics(input); -/// quote! { -/// struct Foo<$($decl_generics)*> { -/// // ... -/// } -/// -/// impl<$impl_generics> Foo<$ty_generics> { -/// fn foo() { -/// // ... -/// } -/// } -/// } -/// ``` -pub(crate) struct Generics { - /// The generics with bounds and default values (e.g. `T: Clone, const N: usize = 0`). - /// - /// Use this on type definitions e.g. `struct Foo<$decl_generics> ...` (or `union`/`enum`). - pub(crate) decl_generics: Vec<TokenTree>, - /// The generics with bounds (e.g. `T: Clone, const N: usize`). - /// - /// Use this on `impl` blocks e.g. `impl<$impl_generics> Trait for ...`. - pub(crate) impl_generics: Vec<TokenTree>, - /// The generics without bounds and without default values (e.g. `T, N`). - /// - /// Use this when you use the type that is declared with these generics e.g. - /// `Foo<$ty_generics>`. - pub(crate) ty_generics: Vec<TokenTree>, -} - -/// Parses the given `TokenStream` into `Generics` and the rest. -/// -/// The generics are not present in the rest, but a where clause might remain. -pub(crate) fn parse_generics(input: TokenStream) -> (Generics, Vec<TokenTree>) { - // The generics with bounds and default values. - let mut decl_generics = vec![]; - // `impl_generics`, the declared generics with their bounds. - let mut impl_generics = vec![]; - // Only the names of the generics, without any bounds. - let mut ty_generics = vec![]; - // Tokens not related to the generics e.g. the `where` token and definition. - let mut rest = vec![]; - // The current level of `<`. - let mut nesting = 0; - let mut toks = input.into_iter(); - // If we are at the beginning of a generic parameter. - let mut at_start = true; - let mut skip_until_comma = false; - while let Some(tt) = toks.next() { - if nesting == 1 && matches!(&tt, TokenTree::Punct(p) if p.as_char() == '>') { - // Found the end of the generics. - break; - } else if nesting >= 1 { - decl_generics.push(tt.clone()); - } - match tt.clone() { - TokenTree::Punct(p) if p.as_char() == '<' => { - if nesting >= 1 && !skip_until_comma { - // This is inside of the generics and part of some bound. - impl_generics.push(tt); - } - nesting += 1; - } - TokenTree::Punct(p) if p.as_char() == '>' => { - // This is a parsing error, so we just end it here. - if nesting == 0 { - break; - } else { - nesting -= 1; - if nesting >= 1 && !skip_until_comma { - // We are still inside of the generics and part of some bound. - impl_generics.push(tt); - } - } - } - TokenTree::Punct(p) if skip_until_comma && p.as_char() == ',' => { - if nesting == 1 { - impl_generics.push(tt.clone()); - impl_generics.push(tt); - skip_until_comma = false; - } - } - _ if !skip_until_comma => { - match nesting { - // If we haven't entered the generics yet, we still want to keep these tokens. - 0 => rest.push(tt), - 1 => { - // Here depending on the token, it might be a generic variable name. - match tt.clone() { - TokenTree::Ident(i) if at_start && i.to_string() == "const" => { - let Some(name) = toks.next() else { - // Parsing error. - break; - }; - impl_generics.push(tt); - impl_generics.push(name.clone()); - ty_generics.push(name.clone()); - decl_generics.push(name); - at_start = false; - } - TokenTree::Ident(_) if at_start => { - impl_generics.push(tt.clone()); - ty_generics.push(tt); - at_start = false; - } - TokenTree::Punct(p) if p.as_char() == ',' => { - impl_generics.push(tt.clone()); - ty_generics.push(tt); - at_start = true; - } - // Lifetimes begin with `'`. - TokenTree::Punct(p) if p.as_char() == '\'' && at_start => { - impl_generics.push(tt.clone()); - ty_generics.push(tt); - } - // Generics can have default values, we skip these. - TokenTree::Punct(p) if p.as_char() == '=' => { - skip_until_comma = true; - } - _ => impl_generics.push(tt), - } - } - _ => impl_generics.push(tt), +/// Given a function declaration, finds the name of the function. +pub(crate) fn function_name(input: TokenStream) -> Option<Ident> { + let mut input = input.into_iter(); + while let Some(token) = input.next() { + match token { + TokenTree::Ident(i) if i.to_string() == "fn" => { + if let Some(TokenTree::Ident(i)) = input.next() { + return Some(i); } + return None; } - _ => {} + _ => continue, } } - rest.extend(toks); - ( - Generics { - impl_generics, - decl_generics, - ty_generics, - }, - rest, - ) + None } diff --git a/rust/macros/kunit.rs b/rust/macros/kunit.rs new file mode 100644 index 000000000000..99ccac82edde --- /dev/null +++ b/rust/macros/kunit.rs @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Procedural macro to run KUnit tests using a user-space like syntax. +//! +//! Copyright (c) 2023 José Expósito <jose.exposito89@gmail.com> + +use proc_macro::{Delimiter, Group, TokenStream, TokenTree}; +use std::fmt::Write; + +pub(crate) fn kunit_tests(attr: TokenStream, ts: TokenStream) -> TokenStream { + let attr = attr.to_string(); + + if attr.is_empty() { + panic!("Missing test name in `#[kunit_tests(test_name)]` macro") + } + + if attr.len() > 255 { + panic!("The test suite name `{attr}` exceeds the maximum length of 255 bytes") + } + + let mut tokens: Vec<_> = ts.into_iter().collect(); + + // Scan for the `mod` keyword. + tokens + .iter() + .find_map(|token| match token { + TokenTree::Ident(ident) => match ident.to_string().as_str() { + "mod" => Some(true), + _ => None, + }, + _ => None, + }) + .expect("`#[kunit_tests(test_name)]` attribute should only be applied to modules"); + + // Retrieve the main body. The main body should be the last token tree. + let body = match tokens.pop() { + Some(TokenTree::Group(group)) if group.delimiter() == Delimiter::Brace => group, + _ => panic!("Cannot locate main body of module"), + }; + + // Get the functions set as tests. Search for `[test]` -> `fn`. + let mut body_it = body.stream().into_iter(); + let mut tests = Vec::new(); + while let Some(token) = body_it.next() { + match token { + TokenTree::Group(ident) if ident.to_string() == "[test]" => match body_it.next() { + Some(TokenTree::Ident(ident)) if ident.to_string() == "fn" => { + let test_name = match body_it.next() { + Some(TokenTree::Ident(ident)) => ident.to_string(), + _ => continue, + }; + tests.push(test_name); + } + _ => continue, + }, + _ => (), + } + } + + // Add `#[cfg(CONFIG_KUNIT)]` before the module declaration. + let config_kunit = "#[cfg(CONFIG_KUNIT)]".to_owned().parse().unwrap(); + tokens.insert( + 0, + TokenTree::Group(Group::new(Delimiter::None, config_kunit)), + ); + + // Generate the test KUnit test suite and a test case for each `#[test]`. + // The code generated for the following test module: + // + // ``` + // #[kunit_tests(kunit_test_suit_name)] + // mod tests { + // #[test] + // fn foo() { + // assert_eq!(1, 1); + // } + // + // #[test] + // fn bar() { + // assert_eq!(2, 2); + // } + // } + // ``` + // + // Looks like: + // + // ``` + // unsafe extern "C" fn kunit_rust_wrapper_foo(_test: *mut kernel::bindings::kunit) { foo(); } + // unsafe extern "C" fn kunit_rust_wrapper_bar(_test: *mut kernel::bindings::kunit) { bar(); } + // + // static mut TEST_CASES: [kernel::bindings::kunit_case; 3] = [ + // kernel::kunit::kunit_case(kernel::c_str!("foo"), kunit_rust_wrapper_foo), + // kernel::kunit::kunit_case(kernel::c_str!("bar"), kunit_rust_wrapper_bar), + // kernel::kunit::kunit_case_null(), + // ]; + // + // kernel::kunit_unsafe_test_suite!(kunit_test_suit_name, TEST_CASES); + // ``` + let mut kunit_macros = "".to_owned(); + let mut test_cases = "".to_owned(); + for test in &tests { + let kunit_wrapper_fn_name = format!("kunit_rust_wrapper_{test}"); + let kunit_wrapper = format!( + "unsafe extern \"C\" fn {kunit_wrapper_fn_name}(_test: *mut kernel::bindings::kunit) {{ {test}(); }}" + ); + writeln!(kunit_macros, "{kunit_wrapper}").unwrap(); + writeln!( + test_cases, + " kernel::kunit::kunit_case(kernel::c_str!(\"{test}\"), {kunit_wrapper_fn_name})," + ) + .unwrap(); + } + + writeln!(kunit_macros).unwrap(); + writeln!( + kunit_macros, + "static mut TEST_CASES: [kernel::bindings::kunit_case; {}] = [\n{test_cases} kernel::kunit::kunit_case_null(),\n];", + tests.len() + 1 + ) + .unwrap(); + + writeln!( + kunit_macros, + "kernel::kunit_unsafe_test_suite!({attr}, TEST_CASES);" + ) + .unwrap(); + + // Remove the `#[test]` macros. + // We do this at a token level, in order to preserve span information. + let mut new_body = vec![]; + let mut body_it = body.stream().into_iter(); + + while let Some(token) = body_it.next() { + match token { + TokenTree::Punct(ref c) if c.as_char() == '#' => match body_it.next() { + Some(TokenTree::Group(group)) if group.to_string() == "[test]" => (), + Some(next) => { + new_body.extend([token, next]); + } + _ => { + new_body.push(token); + } + }, + _ => { + new_body.push(token); + } + } + } + + let mut new_body = TokenStream::from_iter(new_body); + new_body.extend::<TokenStream>(kunit_macros.parse().unwrap()); + + tokens.push(TokenTree::Group(Group::new(Delimiter::Brace, new_body))); + + tokens.into_iter().collect() +} diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index 4ab94e44adfe..9acaa68c974e 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -9,13 +9,12 @@ #[macro_use] mod quote; mod concat_idents; +mod export; mod helpers; +mod kunit; mod module; mod paste; -mod pin_data; -mod pinned_drop; mod vtable; -mod zeroable; use proc_macro::TokenStream; @@ -36,7 +35,7 @@ use proc_macro::TokenStream; /// module!{ /// type: MyModule, /// name: "my_kernel_module", -/// author: "Rust for Linux Contributors", +/// authors: ["Rust for Linux Contributors"], /// description: "My very own kernel module!", /// license: "GPL", /// alias: ["alternate_module_name"], @@ -69,7 +68,7 @@ use proc_macro::TokenStream; /// module!{ /// type: MyDeviceDriverModule, /// name: "my_device_driver_module", -/// author: "Rust for Linux Contributors", +/// authors: ["Rust for Linux Contributors"], /// description: "My device driver requires firmware", /// license: "GPL", /// firmware: ["my_device_firmware1.bin", "my_device_firmware2.bin"], @@ -88,7 +87,7 @@ use proc_macro::TokenStream; /// # Supported argument types /// - `type`: type which implements the [`Module`] trait (required). /// - `name`: ASCII string literal of the name of the kernel module (required). -/// - `author`: string literal of the author of the kernel module. +/// - `authors`: array of ASCII string literals of the authors of the kernel module. /// - `description`: string literal of the description of the kernel module. /// - `license`: ASCII string literal of the license of the kernel module (required). /// - `alias`: array of ASCII string literals of the alias names of the kernel module. @@ -123,12 +122,12 @@ pub fn module(ts: TokenStream) -> TokenStream { /// used on the Rust side, it should not be possible to call the default /// implementation. This is done to ensure that we call the vtable methods /// through the C vtable, and not through the Rust vtable. Therefore, the -/// default implementation should call `kernel::build_error`, which prevents +/// default implementation should call `build_error!`, which prevents /// calls to this function at compile time: /// /// ```compile_fail /// # // Intentionally missing `use`s to simplify `rusttest`. -/// kernel::build_error(VTABLE_DEFAULT_ERROR) +/// build_error!(VTABLE_DEFAULT_ERROR) /// ``` /// /// Note that you might need to import [`kernel::error::VTABLE_DEFAULT_ERROR`]. @@ -145,11 +144,11 @@ pub fn module(ts: TokenStream) -> TokenStream { /// #[vtable] /// pub trait Operations: Send + Sync + Sized { /// fn foo(&self) -> Result<()> { -/// kernel::build_error(VTABLE_DEFAULT_ERROR) +/// build_error!(VTABLE_DEFAULT_ERROR) /// } /// /// fn bar(&self) -> Result<()> { -/// kernel::build_error(VTABLE_DEFAULT_ERROR) +/// build_error!(VTABLE_DEFAULT_ERROR) /// } /// } /// @@ -174,6 +173,29 @@ pub fn vtable(attr: TokenStream, ts: TokenStream) -> TokenStream { vtable::vtable(attr, ts) } +/// Export a function so that C code can call it via a header file. +/// +/// Functions exported using this macro can be called from C code using the declaration in the +/// appropriate header file. It should only be used in cases where C calls the function through a +/// header file; cases where C calls into Rust via a function pointer in a vtable (such as +/// `file_operations`) should not use this macro. +/// +/// This macro has the following effect: +/// +/// * Disables name mangling for this function. +/// * Verifies at compile-time that the function signature matches the declaration in the header +/// file. +/// +/// You must declare the signature of the Rust function in a header file that is included by +/// `rust/bindings/bindings_helper.h`. +/// +/// This macro is *not* the same as the C macros `EXPORT_SYMBOL_*`. All Rust symbols are currently +/// automatically exported with `EXPORT_SYMBOL_GPL`. +#[proc_macro_attribute] +pub fn export(attr: TokenStream, ts: TokenStream) -> TokenStream { + export::export(attr, ts) +} + /// Concatenate two identifiers. /// /// This is useful in macros that need to declare or reference items with names @@ -232,106 +254,6 @@ pub fn concat_idents(ts: TokenStream) -> TokenStream { concat_idents::concat_idents(ts) } -/// Used to specify the pinning information of the fields of a struct. -/// -/// This is somewhat similar in purpose as -/// [pin-project-lite](https://crates.io/crates/pin-project-lite). -/// Place this macro on a struct definition and then `#[pin]` in front of the attributes of each -/// field you want to structurally pin. -/// -/// This macro enables the use of the [`pin_init!`] macro. When pin-initializing a `struct`, -/// then `#[pin]` directs the type of initializer that is required. -/// -/// If your `struct` implements `Drop`, then you need to add `PinnedDrop` as arguments to this -/// macro, and change your `Drop` implementation to `PinnedDrop` annotated with -/// `#[`[`macro@pinned_drop`]`]`, since dropping pinned values requires extra care. -/// -/// # Examples -/// -/// ``` -/// # #![feature(lint_reasons)] -/// # use kernel::prelude::*; -/// # use std::{sync::Mutex, process::Command}; -/// # use kernel::macros::pin_data; -/// #[pin_data] -/// struct DriverData { -/// #[pin] -/// queue: Mutex<KVec<Command>>, -/// buf: KBox<[u8; 1024 * 1024]>, -/// } -/// ``` -/// -/// ``` -/// # #![feature(lint_reasons)] -/// # use kernel::prelude::*; -/// # use std::{sync::Mutex, process::Command}; -/// # use core::pin::Pin; -/// # pub struct Info; -/// # mod bindings { -/// # pub unsafe fn destroy_info(_ptr: *mut super::Info) {} -/// # } -/// use kernel::macros::{pin_data, pinned_drop}; -/// -/// #[pin_data(PinnedDrop)] -/// struct DriverData { -/// #[pin] -/// queue: Mutex<KVec<Command>>, -/// buf: KBox<[u8; 1024 * 1024]>, -/// raw_info: *mut Info, -/// } -/// -/// #[pinned_drop] -/// impl PinnedDrop for DriverData { -/// fn drop(self: Pin<&mut Self>) { -/// unsafe { bindings::destroy_info(self.raw_info) }; -/// } -/// } -/// # fn main() {} -/// ``` -/// -/// [`pin_init!`]: ../kernel/macro.pin_init.html -// ^ cannot use direct link, since `kernel` is not a dependency of `macros`. -#[proc_macro_attribute] -pub fn pin_data(inner: TokenStream, item: TokenStream) -> TokenStream { - pin_data::pin_data(inner, item) -} - -/// Used to implement `PinnedDrop` safely. -/// -/// Only works on structs that are annotated via `#[`[`macro@pin_data`]`]`. -/// -/// # Examples -/// -/// ``` -/// # #![feature(lint_reasons)] -/// # use kernel::prelude::*; -/// # use macros::{pin_data, pinned_drop}; -/// # use std::{sync::Mutex, process::Command}; -/// # use core::pin::Pin; -/// # mod bindings { -/// # pub struct Info; -/// # pub unsafe fn destroy_info(_ptr: *mut Info) {} -/// # } -/// #[pin_data(PinnedDrop)] -/// struct DriverData { -/// #[pin] -/// queue: Mutex<KVec<Command>>, -/// buf: KBox<[u8; 1024 * 1024]>, -/// raw_info: *mut bindings::Info, -/// } -/// -/// #[pinned_drop] -/// impl PinnedDrop for DriverData { -/// fn drop(self: Pin<&mut Self>) { -/// unsafe { bindings::destroy_info(self.raw_info) }; -/// } -/// } -/// ``` -#[proc_macro_attribute] -pub fn pinned_drop(args: TokenStream, input: TokenStream) -> TokenStream { - pinned_drop::pinned_drop(args, input) -} - /// Paste identifiers together. /// /// Within the `paste!` macro, identifiers inside `[<` and `>]` are concatenated together to form a @@ -472,23 +394,29 @@ pub fn paste(input: TokenStream) -> TokenStream { tokens.into_iter().collect() } -/// Derives the [`Zeroable`] trait for the given struct. +/// Registers a KUnit test suite and its test cases using a user-space like syntax. /// -/// This can only be used for structs where every field implements the [`Zeroable`] trait. +/// This macro should be used on modules. If `CONFIG_KUNIT` (in `.config`) is `n`, the target module +/// is ignored. /// /// # Examples /// -/// ``` -/// use kernel::macros::Zeroable; +/// ```ignore +/// # use macros::kunit_tests; +/// #[kunit_tests(kunit_test_suit_name)] +/// mod tests { +/// #[test] +/// fn foo() { +/// assert_eq!(1, 1); +/// } /// -/// #[derive(Zeroable)] -/// pub struct DriverData { -/// id: i64, -/// buf_ptr: *mut u8, -/// len: usize, +/// #[test] +/// fn bar() { +/// assert_eq!(2, 2); +/// } /// } /// ``` -#[proc_macro_derive(Zeroable)] -pub fn derive_zeroable(input: TokenStream) -> TokenStream { - zeroable::derive(input) +#[proc_macro_attribute] +pub fn kunit_tests(attr: TokenStream, ts: TokenStream) -> TokenStream { + kunit::kunit_tests(attr, ts) } diff --git a/rust/macros/module.rs b/rust/macros/module.rs index 2587f41b0d39..44e5cb108cea 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -48,7 +48,7 @@ impl<'a> ModInfoBuilder<'a> { ) } else { // Loadable modules' modinfo strings go as-is. - format!("{field}={content}\0", field = field, content = content) + format!("{field}={content}\0") }; write!( @@ -56,7 +56,7 @@ impl<'a> ModInfoBuilder<'a> { " {cfg} #[doc(hidden)] - #[link_section = \".modinfo\"] + #[cfg_attr(not(target_os = \"macos\"), link_section = \".modinfo\")] #[used] pub static __{module}_{counter}: [u8; {length}] = *{string}; ", @@ -95,6 +95,7 @@ struct ModuleInfo { license: String, name: String, author: Option<String>, + authors: Option<Vec<String>>, description: Option<String>, alias: Option<Vec<String>>, firmware: Option<Vec<String>>, @@ -108,6 +109,7 @@ impl ModuleInfo { "type", "name", "author", + "authors", "description", "license", "alias", @@ -124,10 +126,7 @@ impl ModuleInfo { }; if seen_keys.contains(&key) { - panic!( - "Duplicated key \"{}\". Keys can only be specified once.", - key - ); + panic!("Duplicated key \"{key}\". Keys can only be specified once."); } assert_eq!(expect_punct(it), ':'); @@ -136,14 +135,12 @@ impl ModuleInfo { "type" => info.type_ = expect_ident(it), "name" => info.name = expect_string_ascii(it), "author" => info.author = Some(expect_string(it)), + "authors" => info.authors = Some(expect_string_array(it)), "description" => info.description = Some(expect_string(it)), "license" => info.license = expect_string_ascii(it), "alias" => info.alias = Some(expect_string_array(it)), "firmware" => info.firmware = Some(expect_string_array(it)), - _ => panic!( - "Unknown key \"{}\". Valid keys are: {:?}.", - key, EXPECTED_KEYS - ), + _ => panic!("Unknown key \"{key}\". Valid keys are: {EXPECTED_KEYS:?}."), } assert_eq!(expect_punct(it), ','); @@ -155,7 +152,7 @@ impl ModuleInfo { for key in REQUIRED_KEYS { if !seen_keys.iter().any(|e| e == key) { - panic!("Missing required key \"{}\".", key); + panic!("Missing required key \"{key}\"."); } } @@ -167,10 +164,7 @@ impl ModuleInfo { } if seen_keys != ordered_keys { - panic!( - "Keys are not ordered as expected. Order them like: {:?}.", - ordered_keys - ); + panic!("Keys are not ordered as expected. Order them like: {ordered_keys:?}."); } info @@ -186,6 +180,11 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { if let Some(author) = info.author { modinfo.emit("author", &author); } + if let Some(authors) = info.authors { + for author in authors { + modinfo.emit("author", &author); + } + } if let Some(description) = info.description { modinfo.emit("description", &description); } @@ -228,11 +227,19 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { kernel::ThisModule::from_ptr(core::ptr::null_mut()) }}; + /// The `LocalModule` type is the type of the module created by `module!`, + /// `module_pci_driver!`, `module_platform_driver!`, etc. + type LocalModule = {type_}; + + impl kernel::ModuleMetadata for {type_} {{ + const NAME: &'static kernel::str::CStr = kernel::c_str!(\"{name}\"); + }} + // Double nested modules, since then nobody can access the public items inside. mod __module_init {{ mod __module_init {{ use super::super::{type_}; - use kernel::init::PinInit; + use pin_init::PinInit; /// The \"Rust loadable module\" mark. // @@ -271,6 +278,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { #[cfg(MODULE)] #[doc(hidden)] #[no_mangle] + #[link_section = \".exit.text\"] pub extern \"C\" fn cleanup_module() {{ // SAFETY: // - This function is inaccessible to the outside due to the double diff --git a/rust/macros/paste.rs b/rust/macros/paste.rs index 6529a387673f..cce712d19855 100644 --- a/rust/macros/paste.rs +++ b/rust/macros/paste.rs @@ -50,7 +50,7 @@ fn concat_helper(tokens: &[TokenTree]) -> Vec<(String, Span)> { let tokens = group.stream().into_iter().collect::<Vec<TokenTree>>(); segments.append(&mut concat_helper(tokens.as_slice())); } - token => panic!("unexpected token in paste segments: {:?}", token), + token => panic!("unexpected token in paste segments: {token:?}"), }; } diff --git a/rust/macros/quote.rs b/rust/macros/quote.rs index 33a199e4f176..92cacc4067c9 100644 --- a/rust/macros/quote.rs +++ b/rust/macros/quote.rs @@ -2,6 +2,7 @@ use proc_macro::{TokenStream, TokenTree}; +#[allow(dead_code)] pub(crate) trait ToTokens { fn to_tokens(&self, tokens: &mut TokenStream); } @@ -20,6 +21,12 @@ impl ToTokens for proc_macro::Group { } } +impl ToTokens for proc_macro::Ident { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.extend([TokenTree::from(self.clone())]); + } +} + impl ToTokens for TokenTree { fn to_tokens(&self, tokens: &mut TokenStream) { tokens.extend([self.clone()]); @@ -40,7 +47,7 @@ impl ToTokens for TokenStream { /// `quote` crate but provides only just enough functionality needed by the current `macros` crate. macro_rules! quote_spanned { ($span:expr => $($tt:tt)*) => {{ - let mut tokens; + let mut tokens: ::std::vec::Vec<::proc_macro::TokenTree>; #[allow(clippy::vec_init_then_push)] { tokens = ::std::vec::Vec::new(); @@ -65,7 +72,8 @@ macro_rules! quote_spanned { quote_spanned!(@proc $v $span $($tt)*); }; (@proc $v:ident $span:ident ( $($inner:tt)* ) $($tt:tt)*) => { - let mut tokens = ::std::vec::Vec::new(); + #[allow(unused_mut)] + let mut tokens = ::std::vec::Vec::<::proc_macro::TokenTree>::new(); quote_spanned!(@proc tokens $span $($inner)*); $v.push(::proc_macro::TokenTree::Group(::proc_macro::Group::new( ::proc_macro::Delimiter::Parenthesis, @@ -136,6 +144,22 @@ macro_rules! quote_spanned { )); quote_spanned!(@proc $v $span $($tt)*); }; + (@proc $v:ident $span:ident = $($tt:tt)*) => { + $v.push(::proc_macro::TokenTree::Punct( + ::proc_macro::Punct::new('=', ::proc_macro::Spacing::Alone) + )); + quote_spanned!(@proc $v $span $($tt)*); + }; + (@proc $v:ident $span:ident # $($tt:tt)*) => { + $v.push(::proc_macro::TokenTree::Punct( + ::proc_macro::Punct::new('#', ::proc_macro::Spacing::Alone) + )); + quote_spanned!(@proc $v $span $($tt)*); + }; + (@proc $v:ident $span:ident _ $($tt:tt)*) => { + $v.push(::proc_macro::TokenTree::Ident(::proc_macro::Ident::new("_", $span))); + quote_spanned!(@proc $v $span $($tt)*); + }; (@proc $v:ident $span:ident $id:ident $($tt:tt)*) => { $v.push(::proc_macro::TokenTree::Ident(::proc_macro::Ident::new(stringify!($id), $span))); quote_spanned!(@proc $v $span $($tt)*); diff --git a/rust/pin-init/CONTRIBUTING.md b/rust/pin-init/CONTRIBUTING.md new file mode 100644 index 000000000000..16c899a7ae0b --- /dev/null +++ b/rust/pin-init/CONTRIBUTING.md @@ -0,0 +1,72 @@ +# Contributing to `pin-init` + +Thanks for showing interest in contributing to `pin-init`! This document outlines the guidelines for +contributing to `pin-init`. + +All contributions are double-licensed under Apache 2.0 and MIT. You can find the respective licenses +in the `LICENSE-APACHE` and `LICENSE-MIT` files. + +## Non-Code Contributions + +### Bug Reports + +For any type of bug report, please submit an issue using the bug report issue template. + +If the issue is a soundness issue, please privately report it as a security vulnerability via the +GitHub web interface. + +### Feature Requests + +If you have any feature requests, please submit an issue using the feature request issue template. + +### Questions and Getting Help + +You can ask questions in the Discussions page of the GitHub repository. If you're encountering +problems or just have questions related to `pin-init` in the Linux kernel, you can also ask your +questions in the [Rust-for-Linux Zulip](https://rust-for-linux.zulipchat.com/) or see +<https://rust-for-linux.com/contact>. + +## Contributing Code + +### Linux Kernel + +`pin-init` is used by the Linux kernel and all commits are synchronized to it. For this reason, the +same requirements for commits apply to `pin-init`. See [the kernel's documentation] for details. The +rest of this document will also cover some of the rules listed there and additional ones. + +[the kernel's documentation]: https://docs.kernel.org/process/submitting-patches.html + +Contributions to `pin-init` ideally go through the [GitHub repository], because that repository runs +a CI with lots of tests not present in the kernel. However, patches are also accepted (though not +preferred). Do note that there are some files that are only present in the GitHub repository such as +tests, licenses and cargo related files. Making changes to them can only happen via GitHub. + +[GitHub repository]: https://github.com/Rust-for-Linux/pin-init + +### Commit Style + +Everything must compile without errors or warnings and all tests must pass after **every commit**. +This is important for bisection and also required by the kernel. + +Each commit should be a single, logically cohesive change. Of course it's best to keep the changes +small and digestible, but logically linked changes should be made in the same commit. For example, +when fixing typos, create a single commit that fixes all of them instead of one commit per typo. + +Commits must have a meaningful commit title. Commits with changes to files in the `internal` +directory should have a title prefixed with `internal:`. The commit message should explain the +change and its rationale. You also have to add your `Signed-off-by` tag, see [Developer's +Certificate of Origin]. This has to be done for both mailing list submissions as well as GitHub +submissions. + +[Developer's Certificate of Origin]: https://docs.kernel.org/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin + +Any changes made to public APIs must be documented not only in the commit message, but also in the +`CHANGELOG.md` file. This is especially important for breaking changes, as those warrant a major +version bump. + +If you make changes to the top-level crate documentation, you also need to update the `README.md` +via `cargo rdme`. + +Some of these rules can be ignored if the change is done solely to files that are not present in the +kernel version of this library. Those files are documented in the `sync-kernel.sh` script at the +very bottom in the `--exclude` flag given to the `git am` command. diff --git a/rust/pin-init/README.md b/rust/pin-init/README.md new file mode 100644 index 000000000000..3d04796b212b --- /dev/null +++ b/rust/pin-init/README.md @@ -0,0 +1,228 @@ +[](https://crates.io/crates/pin-init) +[](https://docs.rs/pin-init/) +[](https://deps.rs/repo/github/Rust-for-Linux/pin-init) + +[](#nightly-only) + +# `pin-init` + +<!-- cargo-rdme start --> + +Library to safely and fallibly initialize pinned `struct`s using in-place constructors. + +[Pinning][pinning] is Rust's way of ensuring data does not move. + +It also allows in-place initialization of big `struct`s that would otherwise produce a stack +overflow. + +This library's main use-case is in [Rust-for-Linux]. Although this version can be used +standalone. + +There are cases when you want to in-place initialize a struct. For example when it is very big +and moving it from the stack is not an option, because it is bigger than the stack itself. +Another reason would be that you need the address of the object to initialize it. This stands +in direct conflict with Rust's normal process of first initializing an object and then moving +it into it's final memory location. For more information, see +<https://rust-for-linux.com/the-safe-pinned-initialization-problem>. + +This library allows you to do in-place initialization safely. + +### Nightly Needed for `alloc` feature + +This library requires the [`allocator_api` unstable feature] when the `alloc` feature is +enabled and thus this feature can only be used with a nightly compiler. When enabling the +`alloc` feature, the user will be required to activate `allocator_api` as well. + +[`allocator_api` unstable feature]: https://doc.rust-lang.org/nightly/unstable-book/library-features/allocator-api.html + +The feature is enabled by default, thus by default `pin-init` will require a nightly compiler. +However, using the crate on stable compilers is possible by disabling `alloc`. In practice this +will require the `std` feature, because stable compilers have neither `Box` nor `Arc` in no-std +mode. + +## Overview + +To initialize a `struct` with an in-place constructor you will need two things: +- an in-place constructor, +- a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`], + [`Box<T>`] or any other smart pointer that supports this library). + +To get an in-place constructor there are generally three options: +- directly creating an in-place constructor using the [`pin_init!`] macro, +- a custom function/macro returning an in-place constructor provided by someone else, +- using the unsafe function [`pin_init_from_closure()`] to manually create an initializer. + +Aside from pinned initialization, this library also supports in-place construction without +pinning, the macros/types/functions are generally named like the pinned variants without the +`pin_` prefix. + +## Examples + +Throughout the examples we will often make use of the `CMutex` type which can be found in +`../examples/mutex.rs`. It is essentially a userland rebuild of the `struct mutex` type from +the Linux kernel. It also uses a wait list and a basic spinlock. Importantly the wait list +requires it to be pinned to be locked and thus is a prime candidate for using this library. + +### Using the [`pin_init!`] macro + +If you want to use [`PinInit`], then you will have to annotate your `struct` with +`#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for +[structurally pinned fields]. After doing this, you can then create an in-place constructor via +[`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is +that you need to write `<-` instead of `:` for fields that you want to initialize in-place. + +```rust +use pin_init::{pin_data, pin_init, InPlaceInit}; + +#[pin_data] +struct Foo { + #[pin] + a: CMutex<usize>, + b: u32, +} + +let foo = pin_init!(Foo { + a <- CMutex::new(42), + b: 24, +}); +``` + +`foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like +(or just the stack) to actually initialize a `Foo`: + +```rust +let foo: Result<Pin<Box<Foo>>, AllocError> = Box::pin_init(foo); +``` + +For more information see the [`pin_init!`] macro. + +### Using a custom function/macro that returns an initializer + +Many types that use this library supply a function/macro that returns an initializer, because +the above method only works for types where you can access the fields. + +```rust +let mtx: Result<Pin<Arc<CMutex<usize>>>, _> = Arc::pin_init(CMutex::new(42)); +``` + +To declare an init macro/function you just return an [`impl PinInit<T, E>`]: + +```rust +#[pin_data] +struct DriverData { + #[pin] + status: CMutex<i32>, + buffer: Box<[u8; 1_000_000]>, +} + +impl DriverData { + fn new() -> impl PinInit<Self, Error> { + try_pin_init!(Self { + status <- CMutex::new(0), + buffer: Box::init(pin_init::zeroed())?, + }? Error) + } +} +``` + +### Manual creation of an initializer + +Often when working with primitives the previous approaches are not sufficient. That is where +[`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a +[`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure +actually does the initialization in the correct way. Here are the things to look out for +(we are calling the parameter to the closure `slot`): +- when the closure returns `Ok(())`, then it has completed the initialization successfully, so + `slot` now contains a valid bit pattern for the type `T`, +- when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so + you need to take care to clean up anything if your initialization fails mid-way, +- you may assume that `slot` will stay pinned even after the closure returns until `drop` of + `slot` gets called. + +```rust +use pin_init::{pin_data, pinned_drop, PinInit, PinnedDrop, pin_init_from_closure}; +use core::{ + ptr::addr_of_mut, + marker::PhantomPinned, + cell::UnsafeCell, + pin::Pin, + mem::MaybeUninit, +}; +mod bindings { + #[repr(C)] + pub struct foo { + /* fields from C ... */ + } + extern "C" { + pub fn init_foo(ptr: *mut foo); + pub fn destroy_foo(ptr: *mut foo); + #[must_use = "you must check the error return code"] + pub fn enable_foo(ptr: *mut foo, flags: u32) -> i32; + } +} + +/// # Invariants +/// +/// `foo` is always initialized +#[pin_data(PinnedDrop)] +pub struct RawFoo { + #[pin] + _p: PhantomPinned, + #[pin] + foo: UnsafeCell<MaybeUninit<bindings::foo>>, +} + +impl RawFoo { + pub fn new(flags: u32) -> impl PinInit<Self, i32> { + // SAFETY: + // - when the closure returns `Ok(())`, then it has successfully initialized and + // enabled `foo`, + // - when it returns `Err(e)`, then it has cleaned up before + unsafe { + pin_init_from_closure(move |slot: *mut Self| { + // `slot` contains uninit memory, avoid creating a reference. + let foo = addr_of_mut!((*slot).foo); + let foo = UnsafeCell::raw_get(foo).cast::<bindings::foo>(); + + // Initialize the `foo` + bindings::init_foo(foo); + + // Try to enable it. + let err = bindings::enable_foo(foo, flags); + if err != 0 { + // Enabling has failed, first clean up the foo and then return the error. + bindings::destroy_foo(foo); + Err(err) + } else { + // All fields of `RawFoo` have been initialized, since `_p` is a ZST. + Ok(()) + } + }) + } + } +} + +#[pinned_drop] +impl PinnedDrop for RawFoo { + fn drop(self: Pin<&mut Self>) { + // SAFETY: Since `foo` is initialized, destroying is safe. + unsafe { bindings::destroy_foo(self.foo.get().cast::<bindings::foo>()) }; + } +} +``` + +For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside +the `kernel` crate. The [`sync`] module is a good starting point. + +[`sync`]: https://rust.docs.kernel.org/kernel/sync/index.html +[pinning]: https://doc.rust-lang.org/std/pin/index.html +[structurally pinned fields]: https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field +[stack]: https://docs.rs/pin-init/latest/pin_init/macro.stack_pin_init.html +[`Arc<T>`]: https://doc.rust-lang.org/stable/alloc/sync/struct.Arc.html +[`Box<T>`]: https://doc.rust-lang.org/stable/alloc/boxed/struct.Box.html +[`impl PinInit<Foo>`]: https://docs.rs/pin-init/latest/pin_init/trait.PinInit.html +[`impl PinInit<T, E>`]: https://docs.rs/pin-init/latest/pin_init/trait.PinInit.html +[`impl Init<T, E>`]: https://docs.rs/pin-init/latest/pin_init/trait.Init.html +[Rust-for-Linux]: https://rust-for-linux.com/ + +<!-- cargo-rdme end --> diff --git a/rust/pin-init/examples/big_struct_in_place.rs b/rust/pin-init/examples/big_struct_in_place.rs new file mode 100644 index 000000000000..30d44a334ffd --- /dev/null +++ b/rust/pin-init/examples/big_struct_in_place.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use pin_init::*; + +// Struct with size over 1GiB +#[derive(Debug)] +pub struct BigStruct { + buf: [u8; 1024 * 1024 * 1024], + a: u64, + b: u64, + c: u64, + d: u64, + managed_buf: ManagedBuf, +} + +#[derive(Debug)] +pub struct ManagedBuf { + buf: [u8; 1024 * 1024], +} + +impl ManagedBuf { + pub fn new() -> impl Init<Self> { + init!(ManagedBuf { buf <- zeroed() }) + } +} + +fn main() { + // we want to initialize the struct in-place, otherwise we would get a stackoverflow + let buf: Box<BigStruct> = Box::init(init!(BigStruct { + buf <- zeroed(), + a: 7, + b: 186, + c: 7789, + d: 34, + managed_buf <- ManagedBuf::new(), + })) + .unwrap(); + println!("{}", core::mem::size_of_val(&*buf)); +} diff --git a/rust/pin-init/examples/error.rs b/rust/pin-init/examples/error.rs new file mode 100644 index 000000000000..e0cc258746ce --- /dev/null +++ b/rust/pin-init/examples/error.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![cfg_attr(feature = "alloc", feature(allocator_api))] + +use core::convert::Infallible; + +#[cfg(feature = "alloc")] +use std::alloc::AllocError; + +#[derive(Debug)] +pub struct Error; + +impl From<Infallible> for Error { + fn from(e: Infallible) -> Self { + match e {} + } +} + +#[cfg(feature = "alloc")] +impl From<AllocError> for Error { + fn from(_: AllocError) -> Self { + Self + } +} + +#[allow(dead_code)] +fn main() {} diff --git a/rust/pin-init/examples/linked_list.rs b/rust/pin-init/examples/linked_list.rs new file mode 100644 index 000000000000..6d7eb0a0ec0d --- /dev/null +++ b/rust/pin-init/examples/linked_list.rs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![allow(clippy::undocumented_unsafe_blocks)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] + +use core::{ + cell::Cell, + convert::Infallible, + marker::PhantomPinned, + pin::Pin, + ptr::{self, NonNull}, +}; + +use pin_init::*; + +#[expect(unused_attributes)] +mod error; +use error::Error; + +#[pin_data(PinnedDrop)] +#[repr(C)] +#[derive(Debug)] +pub struct ListHead { + next: Link, + prev: Link, + #[pin] + pin: PhantomPinned, +} + +impl ListHead { + #[inline] + pub fn new() -> impl PinInit<Self, Infallible> { + try_pin_init!(&this in Self { + next: unsafe { Link::new_unchecked(this) }, + prev: unsafe { Link::new_unchecked(this) }, + pin: PhantomPinned, + }? Infallible) + } + + #[inline] + pub fn insert_next(list: &ListHead) -> impl PinInit<Self, Infallible> + '_ { + try_pin_init!(&this in Self { + prev: list.next.prev().replace(unsafe { Link::new_unchecked(this)}), + next: list.next.replace(unsafe { Link::new_unchecked(this)}), + pin: PhantomPinned, + }? Infallible) + } + + #[inline] + pub fn insert_prev(list: &ListHead) -> impl PinInit<Self, Infallible> + '_ { + try_pin_init!(&this in Self { + next: list.prev.next().replace(unsafe { Link::new_unchecked(this)}), + prev: list.prev.replace(unsafe { Link::new_unchecked(this)}), + pin: PhantomPinned, + }? Infallible) + } + + #[inline] + pub fn next(&self) -> Option<NonNull<Self>> { + if ptr::eq(self.next.as_ptr(), self) { + None + } else { + Some(unsafe { NonNull::new_unchecked(self.next.as_ptr() as *mut Self) }) + } + } + + #[allow(dead_code)] + pub fn size(&self) -> usize { + let mut size = 1; + let mut cur = self.next.clone(); + while !ptr::eq(self, cur.cur()) { + cur = cur.next().clone(); + size += 1; + } + size + } +} + +#[pinned_drop] +impl PinnedDrop for ListHead { + //#[inline] + fn drop(self: Pin<&mut Self>) { + if !ptr::eq(self.next.as_ptr(), &*self) { + let next = unsafe { &*self.next.as_ptr() }; + let prev = unsafe { &*self.prev.as_ptr() }; + next.prev.set(&self.prev); + prev.next.set(&self.next); + } + } +} + +#[repr(transparent)] +#[derive(Clone, Debug)] +struct Link(Cell<NonNull<ListHead>>); + +impl Link { + /// # Safety + /// + /// The contents of the pointer should form a consistent circular + /// linked list; for example, a "next" link should be pointed back + /// by the target `ListHead`'s "prev" link and a "prev" link should be + /// pointed back by the target `ListHead`'s "next" link. + #[inline] + unsafe fn new_unchecked(ptr: NonNull<ListHead>) -> Self { + Self(Cell::new(ptr)) + } + + #[inline] + fn next(&self) -> &Link { + unsafe { &(*self.0.get().as_ptr()).next } + } + + #[inline] + fn prev(&self) -> &Link { + unsafe { &(*self.0.get().as_ptr()).prev } + } + + #[allow(dead_code)] + fn cur(&self) -> &ListHead { + unsafe { &*self.0.get().as_ptr() } + } + + #[inline] + fn replace(&self, other: Link) -> Link { + unsafe { Link::new_unchecked(self.0.replace(other.0.get())) } + } + + #[inline] + fn as_ptr(&self) -> *const ListHead { + self.0.get().as_ptr() + } + + #[inline] + fn set(&self, val: &Link) { + self.0.set(val.0.get()); + } +} + +#[allow(dead_code)] +#[cfg_attr(test, test)] +fn main() -> Result<(), Error> { + let a = Box::pin_init(ListHead::new())?; + stack_pin_init!(let b = ListHead::insert_next(&a)); + stack_pin_init!(let c = ListHead::insert_next(&a)); + stack_pin_init!(let d = ListHead::insert_next(&b)); + let e = Box::pin_init(ListHead::insert_next(&b))?; + println!("a ({a:p}): {a:?}"); + println!("b ({b:p}): {b:?}"); + println!("c ({c:p}): {c:?}"); + println!("d ({d:p}): {d:?}"); + println!("e ({e:p}): {e:?}"); + let mut inspect = &*a; + while let Some(next) = inspect.next() { + println!("({inspect:p}): {inspect:?}"); + inspect = unsafe { &*next.as_ptr() }; + if core::ptr::eq(inspect, &*a) { + break; + } + } + Ok(()) +} diff --git a/rust/pin-init/examples/mutex.rs b/rust/pin-init/examples/mutex.rs new file mode 100644 index 000000000000..073bb79341d1 --- /dev/null +++ b/rust/pin-init/examples/mutex.rs @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![allow(clippy::undocumented_unsafe_blocks)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] +#![allow(clippy::missing_safety_doc)] + +use core::{ + cell::{Cell, UnsafeCell}, + marker::PhantomPinned, + ops::{Deref, DerefMut}, + pin::Pin, + sync::atomic::{AtomicBool, Ordering}, +}; +use std::{ + sync::Arc, + thread::{self, park, sleep, Builder, Thread}, + time::Duration, +}; + +use pin_init::*; +#[expect(unused_attributes)] +#[path = "./linked_list.rs"] +pub mod linked_list; +use linked_list::*; + +pub struct SpinLock { + inner: AtomicBool, +} + +impl SpinLock { + #[inline] + pub fn acquire(&self) -> SpinLockGuard<'_> { + while self + .inner + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + while self.inner.load(Ordering::Relaxed) { + thread::yield_now(); + } + } + SpinLockGuard(self) + } + + #[inline] + #[allow(clippy::new_without_default)] + pub const fn new() -> Self { + Self { + inner: AtomicBool::new(false), + } + } +} + +pub struct SpinLockGuard<'a>(&'a SpinLock); + +impl Drop for SpinLockGuard<'_> { + #[inline] + fn drop(&mut self) { + self.0.inner.store(false, Ordering::Release); + } +} + +#[pin_data] +pub struct CMutex<T> { + #[pin] + wait_list: ListHead, + spin_lock: SpinLock, + locked: Cell<bool>, + #[pin] + data: UnsafeCell<T>, +} + +impl<T> CMutex<T> { + #[inline] + pub fn new(val: impl PinInit<T>) -> impl PinInit<Self> { + pin_init!(CMutex { + wait_list <- ListHead::new(), + spin_lock: SpinLock::new(), + locked: Cell::new(false), + data <- unsafe { + pin_init_from_closure(|slot: *mut UnsafeCell<T>| { + val.__pinned_init(slot.cast::<T>()) + }) + }, + }) + } + + #[inline] + pub fn lock(&self) -> Pin<CMutexGuard<'_, T>> { + let mut sguard = self.spin_lock.acquire(); + if self.locked.get() { + stack_pin_init!(let wait_entry = WaitEntry::insert_new(&self.wait_list)); + // println!("wait list length: {}", self.wait_list.size()); + while self.locked.get() { + drop(sguard); + park(); + sguard = self.spin_lock.acquire(); + } + // This does have an effect, as the ListHead inside wait_entry implements Drop! + #[expect(clippy::drop_non_drop)] + drop(wait_entry); + } + self.locked.set(true); + unsafe { + Pin::new_unchecked(CMutexGuard { + mtx: self, + _pin: PhantomPinned, + }) + } + } + + #[allow(dead_code)] + pub fn get_data_mut(self: Pin<&mut Self>) -> &mut T { + // SAFETY: we have an exclusive reference and thus nobody has access to data. + unsafe { &mut *self.data.get() } + } +} + +unsafe impl<T: Send> Send for CMutex<T> {} +unsafe impl<T: Send> Sync for CMutex<T> {} + +pub struct CMutexGuard<'a, T> { + mtx: &'a CMutex<T>, + _pin: PhantomPinned, +} + +impl<T> Drop for CMutexGuard<'_, T> { + #[inline] + fn drop(&mut self) { + let sguard = self.mtx.spin_lock.acquire(); + self.mtx.locked.set(false); + if let Some(list_field) = self.mtx.wait_list.next() { + let wait_entry = list_field.as_ptr().cast::<WaitEntry>(); + unsafe { (*wait_entry).thread.unpark() }; + } + drop(sguard); + } +} + +impl<T> Deref for CMutexGuard<'_, T> { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + unsafe { &*self.mtx.data.get() } + } +} + +impl<T> DerefMut for CMutexGuard<'_, T> { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.mtx.data.get() } + } +} + +#[pin_data] +#[repr(C)] +struct WaitEntry { + #[pin] + wait_list: ListHead, + thread: Thread, +} + +impl WaitEntry { + #[inline] + fn insert_new(list: &ListHead) -> impl PinInit<Self> + '_ { + pin_init!(Self { + thread: thread::current(), + wait_list <- ListHead::insert_prev(list), + }) + } +} + +#[cfg(not(any(feature = "std", feature = "alloc")))] +fn main() {} + +#[allow(dead_code)] +#[cfg_attr(test, test)] +#[cfg(any(feature = "std", feature = "alloc"))] +fn main() { + let mtx: Pin<Arc<CMutex<usize>>> = Arc::pin_init(CMutex::new(0)).unwrap(); + let mut handles = vec![]; + let thread_count = 20; + let workload = if cfg!(miri) { 100 } else { 1_000 }; + for i in 0..thread_count { + let mtx = mtx.clone(); + handles.push( + Builder::new() + .name(format!("worker #{i}")) + .spawn(move || { + for _ in 0..workload { + *mtx.lock() += 1; + } + println!("{i} halfway"); + sleep(Duration::from_millis((i as u64) * 10)); + for _ in 0..workload { + *mtx.lock() += 1; + } + println!("{i} finished"); + }) + .expect("should not fail"), + ); + } + for h in handles { + h.join().expect("thread panicked"); + } + println!("{:?}", &*mtx.lock()); + assert_eq!(*mtx.lock(), workload * thread_count * 2); +} diff --git a/rust/pin-init/examples/pthread_mutex.rs b/rust/pin-init/examples/pthread_mutex.rs new file mode 100644 index 000000000000..5ac22f1880d2 --- /dev/null +++ b/rust/pin-init/examples/pthread_mutex.rs @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// inspired by <https://github.com/nbdd0121/pin-init/blob/trunk/examples/pthread_mutex.rs> +#![allow(clippy::undocumented_unsafe_blocks)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] +#[cfg(not(windows))] +mod pthread_mtx { + #[cfg(feature = "alloc")] + use core::alloc::AllocError; + use core::{ + cell::UnsafeCell, + marker::PhantomPinned, + mem::MaybeUninit, + ops::{Deref, DerefMut}, + pin::Pin, + }; + use pin_init::*; + use std::convert::Infallible; + + #[pin_data(PinnedDrop)] + pub struct PThreadMutex<T> { + #[pin] + raw: UnsafeCell<libc::pthread_mutex_t>, + data: UnsafeCell<T>, + #[pin] + pin: PhantomPinned, + } + + unsafe impl<T: Send> Send for PThreadMutex<T> {} + unsafe impl<T: Send> Sync for PThreadMutex<T> {} + + #[pinned_drop] + impl<T> PinnedDrop for PThreadMutex<T> { + fn drop(self: Pin<&mut Self>) { + unsafe { + libc::pthread_mutex_destroy(self.raw.get()); + } + } + } + + #[derive(Debug)] + pub enum Error { + #[expect(dead_code)] + IO(std::io::Error), + Alloc, + } + + impl From<Infallible> for Error { + fn from(e: Infallible) -> Self { + match e {} + } + } + + #[cfg(feature = "alloc")] + impl From<AllocError> for Error { + fn from(_: AllocError) -> Self { + Self::Alloc + } + } + + impl<T> PThreadMutex<T> { + pub fn new(data: T) -> impl PinInit<Self, Error> { + fn init_raw() -> impl PinInit<UnsafeCell<libc::pthread_mutex_t>, Error> { + let init = |slot: *mut UnsafeCell<libc::pthread_mutex_t>| { + // we can cast, because `UnsafeCell` has the same layout as T. + let slot: *mut libc::pthread_mutex_t = slot.cast(); + let mut attr = MaybeUninit::uninit(); + let attr = attr.as_mut_ptr(); + // SAFETY: ptr is valid + let ret = unsafe { libc::pthread_mutexattr_init(attr) }; + if ret != 0 { + return Err(Error::IO(std::io::Error::from_raw_os_error(ret))); + } + // SAFETY: attr is initialized + let ret = unsafe { + libc::pthread_mutexattr_settype(attr, libc::PTHREAD_MUTEX_NORMAL) + }; + if ret != 0 { + // SAFETY: attr is initialized + unsafe { libc::pthread_mutexattr_destroy(attr) }; + return Err(Error::IO(std::io::Error::from_raw_os_error(ret))); + } + // SAFETY: slot is valid + unsafe { slot.write(libc::PTHREAD_MUTEX_INITIALIZER) }; + // SAFETY: attr and slot are valid ptrs and attr is initialized + let ret = unsafe { libc::pthread_mutex_init(slot, attr) }; + // SAFETY: attr was initialized + unsafe { libc::pthread_mutexattr_destroy(attr) }; + if ret != 0 { + return Err(Error::IO(std::io::Error::from_raw_os_error(ret))); + } + Ok(()) + }; + // SAFETY: mutex has been initialized + unsafe { pin_init_from_closure(init) } + } + try_pin_init!(Self { + data: UnsafeCell::new(data), + raw <- init_raw(), + pin: PhantomPinned, + }? Error) + } + + pub fn lock(&self) -> PThreadMutexGuard<'_, T> { + // SAFETY: raw is always initialized + unsafe { libc::pthread_mutex_lock(self.raw.get()) }; + PThreadMutexGuard { mtx: self } + } + } + + pub struct PThreadMutexGuard<'a, T> { + mtx: &'a PThreadMutex<T>, + } + + impl<T> Drop for PThreadMutexGuard<'_, T> { + fn drop(&mut self) { + // SAFETY: raw is always initialized + unsafe { libc::pthread_mutex_unlock(self.mtx.raw.get()) }; + } + } + + impl<T> Deref for PThreadMutexGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.mtx.data.get() } + } + } + + impl<T> DerefMut for PThreadMutexGuard<'_, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.mtx.data.get() } + } + } +} + +#[cfg_attr(test, test)] +fn main() { + #[cfg(all(any(feature = "std", feature = "alloc"), not(windows)))] + { + use core::pin::Pin; + use pin_init::*; + use pthread_mtx::*; + use std::{ + sync::Arc, + thread::{sleep, Builder}, + time::Duration, + }; + let mtx: Pin<Arc<PThreadMutex<usize>>> = Arc::try_pin_init(PThreadMutex::new(0)).unwrap(); + let mut handles = vec![]; + let thread_count = 20; + let workload = 1_000_000; + for i in 0..thread_count { + let mtx = mtx.clone(); + handles.push( + Builder::new() + .name(format!("worker #{i}")) + .spawn(move || { + for _ in 0..workload { + *mtx.lock() += 1; + } + println!("{i} halfway"); + sleep(Duration::from_millis((i as u64) * 10)); + for _ in 0..workload { + *mtx.lock() += 1; + } + println!("{i} finished"); + }) + .expect("should not fail"), + ); + } + for h in handles { + h.join().expect("thread panicked"); + } + println!("{:?}", &*mtx.lock()); + assert_eq!(*mtx.lock(), workload * thread_count * 2); + } +} diff --git a/rust/pin-init/examples/static_init.rs b/rust/pin-init/examples/static_init.rs new file mode 100644 index 000000000000..3487d761aa26 --- /dev/null +++ b/rust/pin-init/examples/static_init.rs @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![allow(clippy::undocumented_unsafe_blocks)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] + +use core::{ + cell::{Cell, UnsafeCell}, + mem::MaybeUninit, + ops, + pin::Pin, + time::Duration, +}; +use pin_init::*; +use std::{ + sync::Arc, + thread::{sleep, Builder}, +}; + +#[expect(unused_attributes)] +mod mutex; +use mutex::*; + +pub struct StaticInit<T, I> { + cell: UnsafeCell<MaybeUninit<T>>, + init: Cell<Option<I>>, + lock: SpinLock, + present: Cell<bool>, +} + +unsafe impl<T: Sync, I> Sync for StaticInit<T, I> {} +unsafe impl<T: Send, I> Send for StaticInit<T, I> {} + +impl<T, I: PinInit<T>> StaticInit<T, I> { + pub const fn new(init: I) -> Self { + Self { + cell: UnsafeCell::new(MaybeUninit::uninit()), + init: Cell::new(Some(init)), + lock: SpinLock::new(), + present: Cell::new(false), + } + } +} + +impl<T, I: PinInit<T>> ops::Deref for StaticInit<T, I> { + type Target = T; + fn deref(&self) -> &Self::Target { + if self.present.get() { + unsafe { (*self.cell.get()).assume_init_ref() } + } else { + println!("acquire spinlock on static init"); + let _guard = self.lock.acquire(); + println!("rechecking present..."); + std::thread::sleep(std::time::Duration::from_millis(200)); + if self.present.get() { + return unsafe { (*self.cell.get()).assume_init_ref() }; + } + println!("doing init"); + let ptr = self.cell.get().cast::<T>(); + match self.init.take() { + Some(f) => unsafe { f.__pinned_init(ptr).unwrap() }, + None => unsafe { core::hint::unreachable_unchecked() }, + } + self.present.set(true); + unsafe { (*self.cell.get()).assume_init_ref() } + } + } +} + +pub struct CountInit; + +unsafe impl PinInit<CMutex<usize>> for CountInit { + unsafe fn __pinned_init( + self, + slot: *mut CMutex<usize>, + ) -> Result<(), core::convert::Infallible> { + let init = CMutex::new(0); + std::thread::sleep(std::time::Duration::from_millis(1000)); + unsafe { init.__pinned_init(slot) } + } +} + +pub static COUNT: StaticInit<CMutex<usize>, CountInit> = StaticInit::new(CountInit); + +#[cfg(not(any(feature = "std", feature = "alloc")))] +fn main() {} + +#[cfg(any(feature = "std", feature = "alloc"))] +fn main() { + let mtx: Pin<Arc<CMutex<usize>>> = Arc::pin_init(CMutex::new(0)).unwrap(); + let mut handles = vec![]; + let thread_count = 20; + let workload = 1_000; + for i in 0..thread_count { + let mtx = mtx.clone(); + handles.push( + Builder::new() + .name(format!("worker #{i}")) + .spawn(move || { + for _ in 0..workload { + *COUNT.lock() += 1; + std::thread::sleep(std::time::Duration::from_millis(10)); + *mtx.lock() += 1; + std::thread::sleep(std::time::Duration::from_millis(10)); + *COUNT.lock() += 1; + } + println!("{i} halfway"); + sleep(Duration::from_millis((i as u64) * 10)); + for _ in 0..workload { + std::thread::sleep(std::time::Duration::from_millis(10)); + *mtx.lock() += 1; + } + println!("{i} finished"); + }) + .expect("should not fail"), + ); + } + for h in handles { + h.join().expect("thread panicked"); + } + println!("{:?}, {:?}", &*mtx.lock(), &*COUNT.lock()); + assert_eq!(*mtx.lock(), workload * thread_count * 2); +} diff --git a/rust/pin-init/internal/src/helpers.rs b/rust/pin-init/internal/src/helpers.rs new file mode 100644 index 000000000000..236f989a50f2 --- /dev/null +++ b/rust/pin-init/internal/src/helpers.rs @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[cfg(not(kernel))] +use proc_macro2 as proc_macro; + +use proc_macro::{TokenStream, TokenTree}; + +/// Parsed generics. +/// +/// See the field documentation for an explanation what each of the fields represents. +/// +/// # Examples +/// +/// ```rust,ignore +/// # let input = todo!(); +/// let (Generics { decl_generics, impl_generics, ty_generics }, rest) = parse_generics(input); +/// quote! { +/// struct Foo<$($decl_generics)*> { +/// // ... +/// } +/// +/// impl<$impl_generics> Foo<$ty_generics> { +/// fn foo() { +/// // ... +/// } +/// } +/// } +/// ``` +pub(crate) struct Generics { + /// The generics with bounds and default values (e.g. `T: Clone, const N: usize = 0`). + /// + /// Use this on type definitions e.g. `struct Foo<$decl_generics> ...` (or `union`/`enum`). + pub(crate) decl_generics: Vec<TokenTree>, + /// The generics with bounds (e.g. `T: Clone, const N: usize`). + /// + /// Use this on `impl` blocks e.g. `impl<$impl_generics> Trait for ...`. + pub(crate) impl_generics: Vec<TokenTree>, + /// The generics without bounds and without default values (e.g. `T, N`). + /// + /// Use this when you use the type that is declared with these generics e.g. + /// `Foo<$ty_generics>`. + pub(crate) ty_generics: Vec<TokenTree>, +} + +/// Parses the given `TokenStream` into `Generics` and the rest. +/// +/// The generics are not present in the rest, but a where clause might remain. +pub(crate) fn parse_generics(input: TokenStream) -> (Generics, Vec<TokenTree>) { + // The generics with bounds and default values. + let mut decl_generics = vec![]; + // `impl_generics`, the declared generics with their bounds. + let mut impl_generics = vec![]; + // Only the names of the generics, without any bounds. + let mut ty_generics = vec![]; + // Tokens not related to the generics e.g. the `where` token and definition. + let mut rest = vec![]; + // The current level of `<`. + let mut nesting = 0; + let mut toks = input.into_iter(); + // If we are at the beginning of a generic parameter. + let mut at_start = true; + let mut skip_until_comma = false; + while let Some(tt) = toks.next() { + if nesting == 1 && matches!(&tt, TokenTree::Punct(p) if p.as_char() == '>') { + // Found the end of the generics. + break; + } else if nesting >= 1 { + decl_generics.push(tt.clone()); + } + match tt.clone() { + TokenTree::Punct(p) if p.as_char() == '<' => { + if nesting >= 1 && !skip_until_comma { + // This is inside of the generics and part of some bound. + impl_generics.push(tt); + } + nesting += 1; + } + TokenTree::Punct(p) if p.as_char() == '>' => { + // This is a parsing error, so we just end it here. + if nesting == 0 { + break; + } else { + nesting -= 1; + if nesting >= 1 && !skip_until_comma { + // We are still inside of the generics and part of some bound. + impl_generics.push(tt); + } + } + } + TokenTree::Punct(p) if skip_until_comma && p.as_char() == ',' => { + if nesting == 1 { + impl_generics.push(tt.clone()); + impl_generics.push(tt); + skip_until_comma = false; + } + } + _ if !skip_until_comma => { + match nesting { + // If we haven't entered the generics yet, we still want to keep these tokens. + 0 => rest.push(tt), + 1 => { + // Here depending on the token, it might be a generic variable name. + match tt.clone() { + TokenTree::Ident(i) if at_start && i.to_string() == "const" => { + let Some(name) = toks.next() else { + // Parsing error. + break; + }; + impl_generics.push(tt); + impl_generics.push(name.clone()); + ty_generics.push(name.clone()); + decl_generics.push(name); + at_start = false; + } + TokenTree::Ident(_) if at_start => { + impl_generics.push(tt.clone()); + ty_generics.push(tt); + at_start = false; + } + TokenTree::Punct(p) if p.as_char() == ',' => { + impl_generics.push(tt.clone()); + ty_generics.push(tt); + at_start = true; + } + // Lifetimes begin with `'`. + TokenTree::Punct(p) if p.as_char() == '\'' && at_start => { + impl_generics.push(tt.clone()); + ty_generics.push(tt); + } + // Generics can have default values, we skip these. + TokenTree::Punct(p) if p.as_char() == '=' => { + skip_until_comma = true; + } + _ => impl_generics.push(tt), + } + } + _ => impl_generics.push(tt), + } + } + _ => {} + } + } + rest.extend(toks); + ( + Generics { + impl_generics, + decl_generics, + ty_generics, + }, + rest, + ) +} diff --git a/rust/pin-init/internal/src/lib.rs b/rust/pin-init/internal/src/lib.rs new file mode 100644 index 000000000000..babe5e878550 --- /dev/null +++ b/rust/pin-init/internal/src/lib.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// When fixdep scans this, it will find this string `CONFIG_RUSTC_VERSION_TEXT` +// and thus add a dependency on `include/config/RUSTC_VERSION_TEXT`, which is +// touched by Kconfig when the version string from the compiler changes. + +//! `pin-init` proc macros. + +#![cfg_attr(not(RUSTC_LINT_REASONS_IS_STABLE), feature(lint_reasons))] +// Allow `.into()` to convert +// - `proc_macro2::TokenStream` into `proc_macro::TokenStream` in the user-space version. +// - `proc_macro::TokenStream` into `proc_macro::TokenStream` in the kernel version. +// Clippy warns on this conversion, but it's required by the user-space version. +// +// Remove once we have `proc_macro2` in the kernel. +#![allow(clippy::useless_conversion)] +// Documentation is done in the pin-init crate instead. +#![allow(missing_docs)] + +use proc_macro::TokenStream; + +#[cfg(kernel)] +#[path = "../../../macros/quote.rs"] +#[macro_use] +mod quote; +#[cfg(not(kernel))] +#[macro_use] +extern crate quote; + +mod helpers; +mod pin_data; +mod pinned_drop; +mod zeroable; + +#[proc_macro_attribute] +pub fn pin_data(inner: TokenStream, item: TokenStream) -> TokenStream { + pin_data::pin_data(inner.into(), item.into()).into() +} + +#[proc_macro_attribute] +pub fn pinned_drop(args: TokenStream, input: TokenStream) -> TokenStream { + pinned_drop::pinned_drop(args.into(), input.into()).into() +} + +#[proc_macro_derive(Zeroable)] +pub fn derive_zeroable(input: TokenStream) -> TokenStream { + zeroable::derive(input.into()).into() +} diff --git a/rust/macros/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 1d4a3547c684..87d4a7eb1d35 100644 --- a/rust/macros/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -1,11 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT +#[cfg(not(kernel))] +use proc_macro2 as proc_macro; + use crate::helpers::{parse_generics, Generics}; use proc_macro::{Group, Punct, Spacing, TokenStream, TokenTree}; pub(crate) fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream { // This proc-macro only does some pre-parsing and then delegates the actual parsing to - // `kernel::__pin_data!`. + // `pin_init::__pin_data!`. let ( Generics { @@ -71,7 +74,7 @@ pub(crate) fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream { .collect::<Vec<_>>(); // This should be the body of the struct `{...}`. let last = rest.pop(); - let mut quoted = quote!(::kernel::__pin_data! { + let mut quoted = quote!(::pin_init::__pin_data! { parse_input: @args(#args), @sig(#(#rest)*), diff --git a/rust/macros/pinned_drop.rs b/rust/pin-init/internal/src/pinned_drop.rs index 88fb72b20660..c4ca7a70b726 100644 --- a/rust/macros/pinned_drop.rs +++ b/rust/pin-init/internal/src/pinned_drop.rs @@ -1,5 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT +#[cfg(not(kernel))] +use proc_macro2 as proc_macro; + use proc_macro::{TokenStream, TokenTree}; pub(crate) fn pinned_drop(_args: TokenStream, input: TokenStream) -> TokenStream { @@ -25,8 +28,7 @@ pub(crate) fn pinned_drop(_args: TokenStream, input: TokenStream) -> TokenStream // Found the end of the generics, this should be `PinnedDrop`. assert!( matches!(tt, TokenTree::Ident(i) if i.to_string() == "PinnedDrop"), - "expected 'PinnedDrop', found: '{:?}'", - tt + "expected 'PinnedDrop', found: '{tt:?}'" ); pinned_drop_idx = Some(i); break; @@ -35,11 +37,11 @@ pub(crate) fn pinned_drop(_args: TokenStream, input: TokenStream) -> TokenStream let idx = pinned_drop_idx .unwrap_or_else(|| panic!("Expected an `impl` block implementing `PinnedDrop`.")); // Fully qualify the `PinnedDrop`, as to avoid any tampering. - toks.splice(idx..idx, quote!(::kernel::init::)); + toks.splice(idx..idx, quote!(::pin_init::)); // Take the `{}` body and call the declarative macro. if let Some(TokenTree::Group(last)) = toks.pop() { let last = last.stream(); - quote!(::kernel::__pinned_drop! { + quote!(::pin_init::__pinned_drop! { @impl_sig(#(#toks)*), @impl_body(#last), }) diff --git a/rust/macros/zeroable.rs b/rust/pin-init/internal/src/zeroable.rs index cfee2cec18d5..acc94008c152 100644 --- a/rust/macros/zeroable.rs +++ b/rust/pin-init/internal/src/zeroable.rs @@ -1,5 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 +#[cfg(not(kernel))] +use proc_macro2 as proc_macro; + use crate::helpers::{parse_generics, Generics}; use proc_macro::{TokenStream, TokenTree}; @@ -27,7 +30,7 @@ pub(crate) fn derive(input: TokenStream) -> TokenStream { // If we find a `,`, then we have finished a generic/constant/lifetime parameter. TokenTree::Punct(p) if nested == 0 && p.as_char() == ',' => { if in_generic && !inserted { - new_impl_generics.extend(quote! { : ::kernel::init::Zeroable }); + new_impl_generics.extend(quote! { : ::pin_init::Zeroable }); } in_generic = true; inserted = false; @@ -41,7 +44,7 @@ pub(crate) fn derive(input: TokenStream) -> TokenStream { TokenTree::Punct(p) if nested == 0 && p.as_char() == ':' => { new_impl_generics.push(tt); if in_generic { - new_impl_generics.extend(quote! { ::kernel::init::Zeroable + }); + new_impl_generics.extend(quote! { ::pin_init::Zeroable + }); inserted = true; } } @@ -59,10 +62,10 @@ pub(crate) fn derive(input: TokenStream) -> TokenStream { } assert_eq!(nested, 0); if in_generic && !inserted { - new_impl_generics.extend(quote! { : ::kernel::init::Zeroable }); + new_impl_generics.extend(quote! { : ::pin_init::Zeroable }); } quote! { - ::kernel::__derive_zeroable!( + ::pin_init::__derive_zeroable!( parse_input: @sig(#(#rest)*), @impl_generics(#(#new_impl_generics)*), diff --git a/rust/kernel/init/__internal.rs b/rust/pin-init/src/__internal.rs index 74329cc3262c..557b5948cddc 100644 --- a/rust/kernel/init/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -1,19 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT -//! This module contains API-internal items for pin-init. +//! This module contains library internal items. //! -//! These items must not be used outside of -//! - `kernel/init.rs` -//! - `macros/pin_data.rs` -//! - `macros/pinned_drop.rs` +//! These items must not be used outside of this crate and the pin-init-internal crate located at +//! `../internal`. use super::*; /// See the [nomicon] for what subtyping is. See also [this table]. /// +/// The reason for not using `PhantomData<*mut T>` is that that type never implements [`Send`] and +/// [`Sync`]. Hence `fn(*mut T) -> *mut T` is used, as that type always implements them. +/// /// [nomicon]: https://doc.rust-lang.org/nomicon/subtyping.html /// [this table]: https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns -pub(super) type Invariant<T> = PhantomData<fn(*mut T) -> *mut T>; +pub(crate) type Invariant<T> = PhantomData<fn(*mut T) -> *mut T>; /// Module-internal type implementing `PinInit` and `Init`. /// @@ -105,7 +106,7 @@ pub unsafe trait InitData: Copy { } } -pub struct AllData<T: ?Sized>(PhantomData<fn(KBox<T>) -> KBox<T>>); +pub struct AllData<T: ?Sized>(Invariant<T>); impl<T: ?Sized> Clone for AllData<T> { fn clone(&self) -> Self { @@ -135,7 +136,7 @@ unsafe impl<T: ?Sized> HasInitData for T { /// /// If `self.is_init` is true, then `self.value` is initialized. /// -/// [`stack_pin_init`]: kernel::stack_pin_init +/// [`stack_pin_init`]: crate::stack_pin_init pub struct StackInit<T> { value: MaybeUninit<T>, is_init: bool, @@ -156,7 +157,7 @@ impl<T> StackInit<T> { /// Creates a new [`StackInit<T>`] that is uninitialized. Use [`stack_pin_init`] instead of this /// primitive. /// - /// [`stack_pin_init`]: kernel::stack_pin_init + /// [`stack_pin_init`]: crate::stack_pin_init #[inline] pub fn uninit() -> Self { Self { @@ -186,6 +187,33 @@ impl<T> StackInit<T> { } } +#[test] +fn stack_init_reuse() { + use ::std::{borrow::ToOwned, println, string::String}; + use core::pin::pin; + + #[derive(Debug)] + struct Foo { + a: usize, + b: String, + } + let mut slot: Pin<&mut StackInit<Foo>> = pin!(StackInit::uninit()); + let value: Result<Pin<&mut Foo>, core::convert::Infallible> = + slot.as_mut().init(crate::init!(Foo { + a: 42, + b: "Hello".to_owned(), + })); + let value = value.unwrap(); + println!("{value:?}"); + let value: Result<Pin<&mut Foo>, core::convert::Infallible> = + slot.as_mut().init(crate::init!(Foo { + a: 24, + b: "world!".to_owned(), + })); + let value = value.unwrap(); + println!("{value:?}"); +} + /// When a value of this type is dropped, it drops a `T`. /// /// Can be forgotten to prevent the drop. diff --git a/rust/pin-init/src/alloc.rs b/rust/pin-init/src/alloc.rs new file mode 100644 index 000000000000..5017f57442d8 --- /dev/null +++ b/rust/pin-init/src/alloc.rs @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[cfg(all(feature = "alloc", not(feature = "std")))] +use alloc::{boxed::Box, sync::Arc}; +#[cfg(feature = "alloc")] +use core::alloc::AllocError; +use core::{mem::MaybeUninit, pin::Pin}; +#[cfg(feature = "std")] +use std::sync::Arc; + +#[cfg(not(feature = "alloc"))] +type AllocError = core::convert::Infallible; + +use crate::{ + init_from_closure, pin_init_from_closure, InPlaceWrite, Init, PinInit, ZeroableOption, +}; + +pub extern crate alloc; + +// SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee: +// <https://doc.rust-lang.org/stable/std/option/index.html#representation>). +unsafe impl<T> ZeroableOption for Box<T> {} + +/// Smart pointer that can initialize memory in-place. +pub trait InPlaceInit<T>: Sized { + /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this + /// type. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> + where + E: From<AllocError>; + + /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this + /// type. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + fn pin_init(init: impl PinInit<T>) -> Result<Pin<Self>, AllocError> { + // SAFETY: We delegate to `init` and only change the error type. + let init = unsafe { + pin_init_from_closure(|slot| match init.__pinned_init(slot) { + Ok(()) => Ok(()), + Err(i) => match i {}, + }) + }; + Self::try_pin_init(init) + } + + /// Use the given initializer to in-place initialize a `T`. + fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> + where + E: From<AllocError>; + + /// Use the given initializer to in-place initialize a `T`. + fn init(init: impl Init<T>) -> Result<Self, AllocError> { + // SAFETY: We delegate to `init` and only change the error type. + let init = unsafe { + init_from_closure(|slot| match init.__init(slot) { + Ok(()) => Ok(()), + Err(i) => match i {}, + }) + }; + Self::try_init(init) + } +} + +#[cfg(feature = "alloc")] +macro_rules! try_new_uninit { + ($type:ident) => { + $type::try_new_uninit()? + }; +} +#[cfg(all(feature = "std", not(feature = "alloc")))] +macro_rules! try_new_uninit { + ($type:ident) => { + $type::new_uninit() + }; +} + +impl<T> InPlaceInit<T> for Box<T> { + #[inline] + fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> + where + E: From<AllocError>, + { + try_new_uninit!(Box).write_pin_init(init) + } + + #[inline] + fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> + where + E: From<AllocError>, + { + try_new_uninit!(Box).write_init(init) + } +} + +impl<T> InPlaceInit<T> for Arc<T> { + #[inline] + fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> + where + E: From<AllocError>, + { + let mut this = try_new_uninit!(Arc); + let Some(slot) = Arc::get_mut(&mut this) else { + // SAFETY: the Arc has just been created and has no external references + unsafe { core::hint::unreachable_unchecked() } + }; + let slot = slot.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { init.__pinned_init(slot)? }; + // SAFETY: All fields have been initialized and this is the only `Arc` to that data. + Ok(unsafe { Pin::new_unchecked(this.assume_init()) }) + } + + #[inline] + fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> + where + E: From<AllocError>, + { + let mut this = try_new_uninit!(Arc); + let Some(slot) = Arc::get_mut(&mut this) else { + // SAFETY: the Arc has just been created and has no external references + unsafe { core::hint::unreachable_unchecked() } + }; + let slot = slot.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid. + unsafe { init.__init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { this.assume_init() }) + } +} + +impl<T> InPlaceWrite<T> for Box<MaybeUninit<T>> { + type Initialized = Box<T>; + + fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid. + unsafe { init.__init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { self.assume_init() }) + } + + fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { init.__pinned_init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { self.assume_init() }.into()) + } +} diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs new file mode 100644 index 000000000000..0806c689f693 --- /dev/null +++ b/rust/pin-init/src/lib.rs @@ -0,0 +1,1483 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Library to safely and fallibly initialize pinned `struct`s using in-place constructors. +//! +//! [Pinning][pinning] is Rust's way of ensuring data does not move. +//! +//! It also allows in-place initialization of big `struct`s that would otherwise produce a stack +//! overflow. +//! +//! This library's main use-case is in [Rust-for-Linux]. Although this version can be used +//! standalone. +//! +//! There are cases when you want to in-place initialize a struct. For example when it is very big +//! and moving it from the stack is not an option, because it is bigger than the stack itself. +//! Another reason would be that you need the address of the object to initialize it. This stands +//! in direct conflict with Rust's normal process of first initializing an object and then moving +//! it into it's final memory location. For more information, see +//! <https://rust-for-linux.com/the-safe-pinned-initialization-problem>. +//! +//! This library allows you to do in-place initialization safely. +//! +//! ## Nightly Needed for `alloc` feature +//! +//! This library requires the [`allocator_api` unstable feature] when the `alloc` feature is +//! enabled and thus this feature can only be used with a nightly compiler. When enabling the +//! `alloc` feature, the user will be required to activate `allocator_api` as well. +//! +//! [`allocator_api` unstable feature]: https://doc.rust-lang.org/nightly/unstable-book/library-features/allocator-api.html +//! +//! The feature is enabled by default, thus by default `pin-init` will require a nightly compiler. +//! However, using the crate on stable compilers is possible by disabling `alloc`. In practice this +//! will require the `std` feature, because stable compilers have neither `Box` nor `Arc` in no-std +//! mode. +//! +//! # Overview +//! +//! To initialize a `struct` with an in-place constructor you will need two things: +//! - an in-place constructor, +//! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`], +//! [`Box<T>`] or any other smart pointer that supports this library). +//! +//! To get an in-place constructor there are generally three options: +//! - directly creating an in-place constructor using the [`pin_init!`] macro, +//! - a custom function/macro returning an in-place constructor provided by someone else, +//! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer. +//! +//! Aside from pinned initialization, this library also supports in-place construction without +//! pinning, the macros/types/functions are generally named like the pinned variants without the +//! `pin_` prefix. +//! +//! # Examples +//! +//! Throughout the examples we will often make use of the `CMutex` type which can be found in +//! `../examples/mutex.rs`. It is essentially a userland rebuild of the `struct mutex` type from +//! the Linux kernel. It also uses a wait list and a basic spinlock. Importantly the wait list +//! requires it to be pinned to be locked and thus is a prime candidate for using this library. +//! +//! ## Using the [`pin_init!`] macro +//! +//! If you want to use [`PinInit`], then you will have to annotate your `struct` with +//! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for +//! [structurally pinned fields]. After doing this, you can then create an in-place constructor via +//! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is +//! that you need to write `<-` instead of `:` for fields that you want to initialize in-place. +//! +//! ```rust +//! # #![expect(clippy::disallowed_names)] +//! # #![feature(allocator_api)] +//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +//! # use core::pin::Pin; +//! use pin_init::{pin_data, pin_init, InPlaceInit}; +//! +//! #[pin_data] +//! struct Foo { +//! #[pin] +//! a: CMutex<usize>, +//! b: u32, +//! } +//! +//! let foo = pin_init!(Foo { +//! a <- CMutex::new(42), +//! b: 24, +//! }); +//! # let _ = Box::pin_init(foo); +//! ``` +//! +//! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like +//! (or just the stack) to actually initialize a `Foo`: +//! +//! ```rust +//! # #![expect(clippy::disallowed_names)] +//! # #![feature(allocator_api)] +//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +//! # use core::{alloc::AllocError, pin::Pin}; +//! # use pin_init::*; +//! # +//! # #[pin_data] +//! # struct Foo { +//! # #[pin] +//! # a: CMutex<usize>, +//! # b: u32, +//! # } +//! # +//! # let foo = pin_init!(Foo { +//! # a <- CMutex::new(42), +//! # b: 24, +//! # }); +//! let foo: Result<Pin<Box<Foo>>, AllocError> = Box::pin_init(foo); +//! ``` +//! +//! For more information see the [`pin_init!`] macro. +//! +//! ## Using a custom function/macro that returns an initializer +//! +//! Many types that use this library supply a function/macro that returns an initializer, because +//! the above method only works for types where you can access the fields. +//! +//! ```rust +//! # #![feature(allocator_api)] +//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +//! # use pin_init::*; +//! # use std::sync::Arc; +//! # use core::pin::Pin; +//! let mtx: Result<Pin<Arc<CMutex<usize>>>, _> = Arc::pin_init(CMutex::new(42)); +//! ``` +//! +//! To declare an init macro/function you just return an [`impl PinInit<T, E>`]: +//! +//! ```rust +//! # #![feature(allocator_api)] +//! # use pin_init::*; +//! # #[path = "../examples/error.rs"] mod error; use error::Error; +//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +//! #[pin_data] +//! struct DriverData { +//! #[pin] +//! status: CMutex<i32>, +//! buffer: Box<[u8; 1_000_000]>, +//! } +//! +//! impl DriverData { +//! fn new() -> impl PinInit<Self, Error> { +//! try_pin_init!(Self { +//! status <- CMutex::new(0), +//! buffer: Box::init(pin_init::zeroed())?, +//! }? Error) +//! } +//! } +//! ``` +//! +//! ## Manual creation of an initializer +//! +//! Often when working with primitives the previous approaches are not sufficient. That is where +//! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a +//! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure +//! actually does the initialization in the correct way. Here are the things to look out for +//! (we are calling the parameter to the closure `slot`): +//! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so +//! `slot` now contains a valid bit pattern for the type `T`, +//! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so +//! you need to take care to clean up anything if your initialization fails mid-way, +//! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of +//! `slot` gets called. +//! +//! ```rust +//! # #![feature(extern_types)] +//! use pin_init::{pin_data, pinned_drop, PinInit, PinnedDrop, pin_init_from_closure}; +//! use core::{ +//! ptr::addr_of_mut, +//! marker::PhantomPinned, +//! cell::UnsafeCell, +//! pin::Pin, +//! mem::MaybeUninit, +//! }; +//! mod bindings { +//! #[repr(C)] +//! pub struct foo { +//! /* fields from C ... */ +//! } +//! extern "C" { +//! pub fn init_foo(ptr: *mut foo); +//! pub fn destroy_foo(ptr: *mut foo); +//! #[must_use = "you must check the error return code"] +//! pub fn enable_foo(ptr: *mut foo, flags: u32) -> i32; +//! } +//! } +//! +//! /// # Invariants +//! /// +//! /// `foo` is always initialized +//! #[pin_data(PinnedDrop)] +//! pub struct RawFoo { +//! #[pin] +//! _p: PhantomPinned, +//! #[pin] +//! foo: UnsafeCell<MaybeUninit<bindings::foo>>, +//! } +//! +//! impl RawFoo { +//! pub fn new(flags: u32) -> impl PinInit<Self, i32> { +//! // SAFETY: +//! // - when the closure returns `Ok(())`, then it has successfully initialized and +//! // enabled `foo`, +//! // - when it returns `Err(e)`, then it has cleaned up before +//! unsafe { +//! pin_init_from_closure(move |slot: *mut Self| { +//! // `slot` contains uninit memory, avoid creating a reference. +//! let foo = addr_of_mut!((*slot).foo); +//! let foo = UnsafeCell::raw_get(foo).cast::<bindings::foo>(); +//! +//! // Initialize the `foo` +//! bindings::init_foo(foo); +//! +//! // Try to enable it. +//! let err = bindings::enable_foo(foo, flags); +//! if err != 0 { +//! // Enabling has failed, first clean up the foo and then return the error. +//! bindings::destroy_foo(foo); +//! Err(err) +//! } else { +//! // All fields of `RawFoo` have been initialized, since `_p` is a ZST. +//! Ok(()) +//! } +//! }) +//! } +//! } +//! } +//! +//! #[pinned_drop] +//! impl PinnedDrop for RawFoo { +//! fn drop(self: Pin<&mut Self>) { +//! // SAFETY: Since `foo` is initialized, destroying is safe. +//! unsafe { bindings::destroy_foo(self.foo.get().cast::<bindings::foo>()) }; +//! } +//! } +//! ``` +//! +//! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside +//! the `kernel` crate. The [`sync`] module is a good starting point. +//! +//! [`sync`]: https://rust.docs.kernel.org/kernel/sync/index.html +//! [pinning]: https://doc.rust-lang.org/std/pin/index.html +//! [structurally pinned fields]: +//! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field +//! [stack]: crate::stack_pin_init +#![cfg_attr( + kernel, + doc = "[`Arc<T>`]: https://rust.docs.kernel.org/kernel/sync/struct.Arc.html" +)] +#![cfg_attr( + kernel, + doc = "[`Box<T>`]: https://rust.docs.kernel.org/kernel/alloc/kbox/struct.Box.html" +)] +#![cfg_attr(not(kernel), doc = "[`Arc<T>`]: alloc::alloc::sync::Arc")] +#![cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")] +//! [`impl PinInit<Foo>`]: crate::PinInit +//! [`impl PinInit<T, E>`]: crate::PinInit +//! [`impl Init<T, E>`]: crate::Init +//! [Rust-for-Linux]: https://rust-for-linux.com/ + +#![cfg_attr(not(RUSTC_LINT_REASONS_IS_STABLE), feature(lint_reasons))] +#![cfg_attr( + all( + any(feature = "alloc", feature = "std"), + not(RUSTC_NEW_UNINIT_IS_STABLE) + ), + feature(new_uninit) +)] +#![forbid(missing_docs, unsafe_op_in_unsafe_fn)] +#![cfg_attr(not(feature = "std"), no_std)] +#![cfg_attr(feature = "alloc", feature(allocator_api))] + +use core::{ + cell::UnsafeCell, + convert::Infallible, + marker::PhantomData, + mem::MaybeUninit, + num::*, + pin::Pin, + ptr::{self, NonNull}, +}; + +#[doc(hidden)] +pub mod __internal; +#[doc(hidden)] +pub mod macros; + +#[cfg(any(feature = "std", feature = "alloc"))] +mod alloc; +#[cfg(any(feature = "std", feature = "alloc"))] +pub use alloc::InPlaceInit; + +/// Used to specify the pinning information of the fields of a struct. +/// +/// This is somewhat similar in purpose as +/// [pin-project-lite](https://crates.io/crates/pin-project-lite). +/// Place this macro on a struct definition and then `#[pin]` in front of the attributes of each +/// field you want to structurally pin. +/// +/// This macro enables the use of the [`pin_init!`] macro. When pin-initializing a `struct`, +/// then `#[pin]` directs the type of initializer that is required. +/// +/// If your `struct` implements `Drop`, then you need to add `PinnedDrop` as arguments to this +/// macro, and change your `Drop` implementation to `PinnedDrop` annotated with +/// `#[`[`macro@pinned_drop`]`]`, since dropping pinned values requires extra care. +/// +/// # Examples +/// +/// ``` +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// use pin_init::pin_data; +/// +/// enum Command { +/// /* ... */ +/// } +/// +/// #[pin_data] +/// struct DriverData { +/// #[pin] +/// queue: CMutex<Vec<Command>>, +/// buf: Box<[u8; 1024 * 1024]>, +/// } +/// ``` +/// +/// ``` +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # mod bindings { pub struct info; pub unsafe fn destroy_info(_: *mut info) {} } +/// use core::pin::Pin; +/// use pin_init::{pin_data, pinned_drop, PinnedDrop}; +/// +/// enum Command { +/// /* ... */ +/// } +/// +/// #[pin_data(PinnedDrop)] +/// struct DriverData { +/// #[pin] +/// queue: CMutex<Vec<Command>>, +/// buf: Box<[u8; 1024 * 1024]>, +/// raw_info: *mut bindings::info, +/// } +/// +/// #[pinned_drop] +/// impl PinnedDrop for DriverData { +/// fn drop(self: Pin<&mut Self>) { +/// unsafe { bindings::destroy_info(self.raw_info) }; +/// } +/// } +/// ``` +pub use ::pin_init_internal::pin_data; + +/// Used to implement `PinnedDrop` safely. +/// +/// Only works on structs that are annotated via `#[`[`macro@pin_data`]`]`. +/// +/// # Examples +/// +/// ``` +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # mod bindings { pub struct info; pub unsafe fn destroy_info(_: *mut info) {} } +/// use core::pin::Pin; +/// use pin_init::{pin_data, pinned_drop, PinnedDrop}; +/// +/// enum Command { +/// /* ... */ +/// } +/// +/// #[pin_data(PinnedDrop)] +/// struct DriverData { +/// #[pin] +/// queue: CMutex<Vec<Command>>, +/// buf: Box<[u8; 1024 * 1024]>, +/// raw_info: *mut bindings::info, +/// } +/// +/// #[pinned_drop] +/// impl PinnedDrop for DriverData { +/// fn drop(self: Pin<&mut Self>) { +/// unsafe { bindings::destroy_info(self.raw_info) }; +/// } +/// } +/// ``` +pub use ::pin_init_internal::pinned_drop; + +/// Derives the [`Zeroable`] trait for the given struct. +/// +/// This can only be used for structs where every field implements the [`Zeroable`] trait. +/// +/// # Examples +/// +/// ``` +/// use pin_init::Zeroable; +/// +/// #[derive(Zeroable)] +/// pub struct DriverData { +/// id: i64, +/// buf_ptr: *mut u8, +/// len: usize, +/// } +/// ``` +pub use ::pin_init_internal::Zeroable; + +/// Initialize and pin a type directly on the stack. +/// +/// # Examples +/// +/// ```rust +/// # #![expect(clippy::disallowed_names)] +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// # use core::pin::Pin; +/// #[pin_data] +/// struct Foo { +/// #[pin] +/// a: CMutex<usize>, +/// b: Bar, +/// } +/// +/// #[pin_data] +/// struct Bar { +/// x: u32, +/// } +/// +/// stack_pin_init!(let foo = pin_init!(Foo { +/// a <- CMutex::new(42), +/// b: Bar { +/// x: 64, +/// }, +/// })); +/// let foo: Pin<&mut Foo> = foo; +/// println!("a: {}", &*foo.a.lock()); +/// ``` +/// +/// # Syntax +/// +/// A normal `let` binding with optional type annotation. The expression is expected to implement +/// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error +/// type, then use [`stack_try_pin_init!`]. +#[macro_export] +macro_rules! stack_pin_init { + (let $var:ident $(: $t:ty)? = $val:expr) => { + let val = $val; + let mut $var = ::core::pin::pin!($crate::__internal::StackInit$(::<$t>)?::uninit()); + let mut $var = match $crate::__internal::StackInit::init($var, val) { + Ok(res) => res, + Err(x) => { + let x: ::core::convert::Infallible = x; + match x {} + } + }; + }; +} + +/// Initialize and pin a type directly on the stack. +/// +/// # Examples +/// +/// ```rust +/// # #![expect(clippy::disallowed_names)] +/// # #![feature(allocator_api)] +/// # #[path = "../examples/error.rs"] mod error; use error::Error; +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// #[pin_data] +/// struct Foo { +/// #[pin] +/// a: CMutex<usize>, +/// b: Box<Bar>, +/// } +/// +/// struct Bar { +/// x: u32, +/// } +/// +/// stack_try_pin_init!(let foo: Foo = try_pin_init!(Foo { +/// a <- CMutex::new(42), +/// b: Box::try_new(Bar { +/// x: 64, +/// })?, +/// }? Error)); +/// let foo = foo.unwrap(); +/// println!("a: {}", &*foo.a.lock()); +/// ``` +/// +/// ```rust +/// # #![expect(clippy::disallowed_names)] +/// # #![feature(allocator_api)] +/// # #[path = "../examples/error.rs"] mod error; use error::Error; +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// #[pin_data] +/// struct Foo { +/// #[pin] +/// a: CMutex<usize>, +/// b: Box<Bar>, +/// } +/// +/// struct Bar { +/// x: u32, +/// } +/// +/// stack_try_pin_init!(let foo: Foo =? try_pin_init!(Foo { +/// a <- CMutex::new(42), +/// b: Box::try_new(Bar { +/// x: 64, +/// })?, +/// }? Error)); +/// println!("a: {}", &*foo.a.lock()); +/// # Ok::<_, Error>(()) +/// ``` +/// +/// # Syntax +/// +/// A normal `let` binding with optional type annotation. The expression is expected to implement +/// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the +/// `=` will propagate this error. +#[macro_export] +macro_rules! stack_try_pin_init { + (let $var:ident $(: $t:ty)? = $val:expr) => { + let val = $val; + let mut $var = ::core::pin::pin!($crate::__internal::StackInit$(::<$t>)?::uninit()); + let mut $var = $crate::__internal::StackInit::init($var, val); + }; + (let $var:ident $(: $t:ty)? =? $val:expr) => { + let val = $val; + let mut $var = ::core::pin::pin!($crate::__internal::StackInit$(::<$t>)?::uninit()); + let mut $var = $crate::__internal::StackInit::init($var, val)?; + }; +} + +/// Construct an in-place, pinned initializer for `struct`s. +/// +/// This macro defaults the error to [`Infallible`]. If you need a different error, then use +/// [`try_pin_init!`]. +/// +/// The syntax is almost identical to that of a normal `struct` initializer: +/// +/// ```rust +/// # use pin_init::*; +/// # use core::pin::Pin; +/// #[pin_data] +/// struct Foo { +/// a: usize, +/// b: Bar, +/// } +/// +/// #[pin_data] +/// struct Bar { +/// x: u32, +/// } +/// +/// # fn demo() -> impl PinInit<Foo> { +/// let a = 42; +/// +/// let initializer = pin_init!(Foo { +/// a, +/// b: Bar { +/// x: 64, +/// }, +/// }); +/// # initializer } +/// # Box::pin_init(demo()).unwrap(); +/// ``` +/// +/// Arbitrary Rust expressions can be used to set the value of a variable. +/// +/// The fields are initialized in the order that they appear in the initializer. So it is possible +/// to read already initialized fields using raw pointers. +/// +/// IMPORTANT: You are not allowed to create references to fields of the struct inside of the +/// initializer. +/// +/// # Init-functions +/// +/// When working with this library it is often desired to let others construct your types without +/// giving access to all fields. This is where you would normally write a plain function `new` that +/// would return a new instance of your type. With this library that is also possible. However, +/// there are a few extra things to keep in mind. +/// +/// To create an initializer function, simply declare it like this: +/// +/// ```rust +/// # use pin_init::*; +/// # use core::pin::Pin; +/// # #[pin_data] +/// # struct Foo { +/// # a: usize, +/// # b: Bar, +/// # } +/// # #[pin_data] +/// # struct Bar { +/// # x: u32, +/// # } +/// impl Foo { +/// fn new() -> impl PinInit<Self> { +/// pin_init!(Self { +/// a: 42, +/// b: Bar { +/// x: 64, +/// }, +/// }) +/// } +/// } +/// ``` +/// +/// Users of `Foo` can now create it like this: +/// +/// ```rust +/// # #![expect(clippy::disallowed_names)] +/// # use pin_init::*; +/// # use core::pin::Pin; +/// # #[pin_data] +/// # struct Foo { +/// # a: usize, +/// # b: Bar, +/// # } +/// # #[pin_data] +/// # struct Bar { +/// # x: u32, +/// # } +/// # impl Foo { +/// # fn new() -> impl PinInit<Self> { +/// # pin_init!(Self { +/// # a: 42, +/// # b: Bar { +/// # x: 64, +/// # }, +/// # }) +/// # } +/// # } +/// let foo = Box::pin_init(Foo::new()); +/// ``` +/// +/// They can also easily embed it into their own `struct`s: +/// +/// ```rust +/// # use pin_init::*; +/// # use core::pin::Pin; +/// # #[pin_data] +/// # struct Foo { +/// # a: usize, +/// # b: Bar, +/// # } +/// # #[pin_data] +/// # struct Bar { +/// # x: u32, +/// # } +/// # impl Foo { +/// # fn new() -> impl PinInit<Self> { +/// # pin_init!(Self { +/// # a: 42, +/// # b: Bar { +/// # x: 64, +/// # }, +/// # }) +/// # } +/// # } +/// #[pin_data] +/// struct FooContainer { +/// #[pin] +/// foo1: Foo, +/// #[pin] +/// foo2: Foo, +/// other: u32, +/// } +/// +/// impl FooContainer { +/// fn new(other: u32) -> impl PinInit<Self> { +/// pin_init!(Self { +/// foo1 <- Foo::new(), +/// foo2 <- Foo::new(), +/// other, +/// }) +/// } +/// } +/// ``` +/// +/// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`. +/// This signifies that the given field is initialized in-place. As with `struct` initializers, just +/// writing the field (in this case `other`) without `:` or `<-` means `other: other,`. +/// +/// # Syntax +/// +/// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with +/// the following modifications is expected: +/// - Fields that you want to initialize in-place have to use `<-` instead of `:`. +/// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`] +/// pointer named `this` inside of the initializer. +/// - Using struct update syntax one can place `..Zeroable::zeroed()` at the very end of the +/// struct, this initializes every field with 0 and then runs all initializers specified in the +/// body. This can only be done if [`Zeroable`] is implemented for the struct. +/// +/// For instance: +/// +/// ```rust +/// # use pin_init::*; +/// # use core::{ptr::addr_of_mut, marker::PhantomPinned}; +/// #[pin_data] +/// #[derive(Zeroable)] +/// struct Buf { +/// // `ptr` points into `buf`. +/// ptr: *mut u8, +/// buf: [u8; 64], +/// #[pin] +/// pin: PhantomPinned, +/// } +/// +/// let init = pin_init!(&this in Buf { +/// buf: [0; 64], +/// // SAFETY: TODO. +/// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() }, +/// pin: PhantomPinned, +/// }); +/// let init = pin_init!(Buf { +/// buf: [1; 64], +/// ..Zeroable::zeroed() +/// }); +/// ``` +/// +/// [`NonNull<Self>`]: core::ptr::NonNull +// For a detailed example of how this macro works, see the module documentation of the hidden +// module `macros` inside of `macros.rs`. +#[macro_export] +macro_rules! pin_init { + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + $($fields:tt)* + }) => { + $crate::try_pin_init!($(&$this in)? $t $(::<$($generics),*>)? { + $($fields)* + }? ::core::convert::Infallible) + }; +} + +/// Construct an in-place, fallible pinned initializer for `struct`s. +/// +/// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`]. +/// +/// You can use the `?` operator or use `return Err(err)` inside the initializer to stop +/// initialization and return the error. +/// +/// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when +/// initialization fails, the memory can be safely deallocated without any further modifications. +/// +/// The syntax is identical to [`pin_init!`] with the following exception: you must append `? $type` +/// after the `struct` initializer to specify the error type you want to use. +/// +/// # Examples +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/error.rs"] mod error; use error::Error; +/// use pin_init::{pin_data, try_pin_init, PinInit, InPlaceInit, zeroed}; +/// +/// #[pin_data] +/// struct BigBuf { +/// big: Box<[u8; 1024 * 1024 * 1024]>, +/// small: [u8; 1024 * 1024], +/// ptr: *mut u8, +/// } +/// +/// impl BigBuf { +/// fn new() -> impl PinInit<Self, Error> { +/// try_pin_init!(Self { +/// big: Box::init(zeroed())?, +/// small: [0; 1024 * 1024], +/// ptr: core::ptr::null_mut(), +/// }? Error) +/// } +/// } +/// # let _ = Box::pin_init(BigBuf::new()); +/// ``` +// For a detailed example of how this macro works, see the module documentation of the hidden +// module `macros` inside of `macros.rs`. +#[macro_export] +macro_rules! try_pin_init { + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + $($fields:tt)* + }? $err:ty) => { + $crate::__init_internal!( + @this($($this)?), + @typ($t $(::<$($generics),*>)? ), + @fields($($fields)*), + @error($err), + @data(PinData, use_data), + @has_data(HasPinData, __pin_data), + @construct_closure(pin_init_from_closure), + @munch_fields($($fields)*), + ) + } +} + +/// Construct an in-place initializer for `struct`s. +/// +/// This macro defaults the error to [`Infallible`]. If you need a different error, then use +/// [`try_init!`]. +/// +/// The syntax is identical to [`pin_init!`] and its safety caveats also apply: +/// - `unsafe` code must guarantee either full initialization or return an error and allow +/// deallocation of the memory. +/// - the fields are initialized in the order given in the initializer. +/// - no references to fields are allowed to be created inside of the initializer. +/// +/// This initializer is for initializing data in-place that might later be moved. If you want to +/// pin-initialize, use [`pin_init!`]. +/// +/// # Examples +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/error.rs"] mod error; use error::Error; +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::InPlaceInit; +/// use pin_init::{init, Init, zeroed}; +/// +/// struct BigBuf { +/// small: [u8; 1024 * 1024], +/// } +/// +/// impl BigBuf { +/// fn new() -> impl Init<Self> { +/// init!(Self { +/// small <- zeroed(), +/// }) +/// } +/// } +/// # let _ = Box::init(BigBuf::new()); +/// ``` +// For a detailed example of how this macro works, see the module documentation of the hidden +// module `macros` inside of `macros.rs`. +#[macro_export] +macro_rules! init { + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + $($fields:tt)* + }) => { + $crate::try_init!($(&$this in)? $t $(::<$($generics),*>)? { + $($fields)* + }? ::core::convert::Infallible) + } +} + +/// Construct an in-place fallible initializer for `struct`s. +/// +/// If the initialization can complete without error (or [`Infallible`]), then use +/// [`init!`]. +/// +/// The syntax is identical to [`try_pin_init!`]. You need to specify a custom error +/// via `? $type` after the `struct` initializer. +/// The safety caveats from [`try_pin_init!`] also apply: +/// - `unsafe` code must guarantee either full initialization or return an error and allow +/// deallocation of the memory. +/// - the fields are initialized in the order given in the initializer. +/// - no references to fields are allowed to be created inside of the initializer. +/// +/// # Examples +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # use core::alloc::AllocError; +/// # use pin_init::InPlaceInit; +/// use pin_init::{try_init, Init, zeroed}; +/// +/// struct BigBuf { +/// big: Box<[u8; 1024 * 1024 * 1024]>, +/// small: [u8; 1024 * 1024], +/// } +/// +/// impl BigBuf { +/// fn new() -> impl Init<Self, AllocError> { +/// try_init!(Self { +/// big: Box::init(zeroed())?, +/// small: [0; 1024 * 1024], +/// }? AllocError) +/// } +/// } +/// # let _ = Box::init(BigBuf::new()); +/// ``` +// For a detailed example of how this macro works, see the module documentation of the hidden +// module `macros` inside of `macros.rs`. +#[macro_export] +macro_rules! try_init { + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + $($fields:tt)* + }? $err:ty) => { + $crate::__init_internal!( + @this($($this)?), + @typ($t $(::<$($generics),*>)?), + @fields($($fields)*), + @error($err), + @data(InitData, /*no use_data*/), + @has_data(HasInitData, __init_data), + @construct_closure(init_from_closure), + @munch_fields($($fields)*), + ) + } +} + +/// Asserts that a field on a struct using `#[pin_data]` is marked with `#[pin]` ie. that it is +/// structurally pinned. +/// +/// # Example +/// +/// This will succeed: +/// ``` +/// use pin_init::{pin_data, assert_pinned}; +/// +/// #[pin_data] +/// struct MyStruct { +/// #[pin] +/// some_field: u64, +/// } +/// +/// assert_pinned!(MyStruct, some_field, u64); +/// ``` +/// +/// This will fail: +/// ```compile_fail +/// use pin_init::{pin_data, assert_pinned}; +/// +/// #[pin_data] +/// struct MyStruct { +/// some_field: u64, +/// } +/// +/// assert_pinned!(MyStruct, some_field, u64); +/// ``` +/// +/// Some uses of the macro may trigger the `can't use generic parameters from outer item` error. To +/// work around this, you may pass the `inline` parameter to the macro. The `inline` parameter can +/// only be used when the macro is invoked from a function body. +/// ``` +/// # use core::pin::Pin; +/// use pin_init::{pin_data, assert_pinned}; +/// +/// #[pin_data] +/// struct Foo<T> { +/// #[pin] +/// elem: T, +/// } +/// +/// impl<T> Foo<T> { +/// fn project(self: Pin<&mut Self>) -> Pin<&mut T> { +/// assert_pinned!(Foo<T>, elem, T, inline); +/// +/// // SAFETY: The field is structurally pinned. +/// unsafe { self.map_unchecked_mut(|me| &mut me.elem) } +/// } +/// } +/// ``` +#[macro_export] +macro_rules! assert_pinned { + ($ty:ty, $field:ident, $field_ty:ty, inline) => { + let _ = move |ptr: *mut $field_ty| { + // SAFETY: This code is unreachable. + let data = unsafe { <$ty as $crate::__internal::HasPinData>::__pin_data() }; + let init = $crate::__internal::AlwaysFail::<$field_ty>::new(); + // SAFETY: This code is unreachable. + unsafe { data.$field(ptr, init) }.ok(); + }; + }; + + ($ty:ty, $field:ident, $field_ty:ty) => { + const _: () = { + $crate::assert_pinned!($ty, $field, $field_ty, inline); + }; + }; +} + +/// A pin-initializer for the type `T`. +/// +/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can +/// be [`Box<T>`], [`Arc<T>`] or even the stack (see [`stack_pin_init!`]). +/// +/// Also see the [module description](self). +/// +/// # Safety +/// +/// When implementing this trait you will need to take great care. Also there are probably very few +/// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible. +/// +/// The [`PinInit::__pinned_init`] function: +/// - returns `Ok(())` if it initialized every field of `slot`, +/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: +/// - `slot` can be deallocated without UB occurring, +/// - `slot` does not need to be dropped, +/// - `slot` is not partially initialized. +/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. +/// +#[cfg_attr( + kernel, + doc = "[`Arc<T>`]: https://rust.docs.kernel.org/kernel/sync/struct.Arc.html" +)] +#[cfg_attr( + kernel, + doc = "[`Box<T>`]: https://rust.docs.kernel.org/kernel/alloc/kbox/struct.Box.html" +)] +#[cfg_attr(not(kernel), doc = "[`Arc<T>`]: alloc::alloc::sync::Arc")] +#[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")] +#[must_use = "An initializer must be used in order to create its value."] +pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { + /// Initializes `slot`. + /// + /// # Safety + /// + /// - `slot` is a valid pointer to uninitialized memory. + /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to + /// deallocate. + /// - `slot` will not move until it is dropped, i.e. it will be pinned. + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>; + + /// First initializes the value using `self` then calls the function `f` with the initialized + /// value. + /// + /// If `f` returns an error the value is dropped and the initializer will forward the error. + /// + /// # Examples + /// + /// ```rust + /// # #![feature(allocator_api)] + /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; + /// # use pin_init::*; + /// let mtx_init = CMutex::new(42); + /// // Make the initializer print the value. + /// let mtx_init = mtx_init.pin_chain(|mtx| { + /// println!("{:?}", mtx.get_data_mut()); + /// Ok(()) + /// }); + /// ``` + fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E> + where + F: FnOnce(Pin<&mut T>) -> Result<(), E>, + { + ChainPinInit(self, f, PhantomData) + } +} + +/// An initializer returned by [`PinInit::pin_chain`]. +pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, T)>); + +// SAFETY: The `__pinned_init` function is implemented such that it +// - returns `Ok(())` on successful initialization, +// - returns `Err(err)` on error and in this case `slot` will be dropped. +// - considers `slot` pinned. +unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainPinInit<I, F, T, E> +where + I: PinInit<T, E>, + F: FnOnce(Pin<&mut T>) -> Result<(), E>, +{ + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: All requirements fulfilled since this function is `__pinned_init`. + unsafe { self.0.__pinned_init(slot)? }; + // SAFETY: The above call initialized `slot` and we still have unique access. + let val = unsafe { &mut *slot }; + // SAFETY: `slot` is considered pinned. + let val = unsafe { Pin::new_unchecked(val) }; + // SAFETY: `slot` was initialized above. + (self.1)(val).inspect_err(|_| unsafe { core::ptr::drop_in_place(slot) }) + } +} + +/// An initializer for `T`. +/// +/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can +/// be [`Box<T>`], [`Arc<T>`] or even the stack (see [`stack_pin_init!`]). Because +/// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well. +/// +/// Also see the [module description](self). +/// +/// # Safety +/// +/// When implementing this trait you will need to take great care. Also there are probably very few +/// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible. +/// +/// The [`Init::__init`] function: +/// - returns `Ok(())` if it initialized every field of `slot`, +/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: +/// - `slot` can be deallocated without UB occurring, +/// - `slot` does not need to be dropped, +/// - `slot` is not partially initialized. +/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. +/// +/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same +/// code as `__init`. +/// +/// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to +/// move the pointee after initialization. +/// +#[cfg_attr( + kernel, + doc = "[`Arc<T>`]: https://rust.docs.kernel.org/kernel/sync/struct.Arc.html" +)] +#[cfg_attr( + kernel, + doc = "[`Box<T>`]: https://rust.docs.kernel.org/kernel/alloc/kbox/struct.Box.html" +)] +#[cfg_attr(not(kernel), doc = "[`Arc<T>`]: alloc::alloc::sync::Arc")] +#[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")] +#[must_use = "An initializer must be used in order to create its value."] +pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { + /// Initializes `slot`. + /// + /// # Safety + /// + /// - `slot` is a valid pointer to uninitialized memory. + /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to + /// deallocate. + unsafe fn __init(self, slot: *mut T) -> Result<(), E>; + + /// First initializes the value using `self` then calls the function `f` with the initialized + /// value. + /// + /// If `f` returns an error the value is dropped and the initializer will forward the error. + /// + /// # Examples + /// + /// ```rust + /// # #![expect(clippy::disallowed_names)] + /// use pin_init::{init, zeroed, Init}; + /// + /// struct Foo { + /// buf: [u8; 1_000_000], + /// } + /// + /// impl Foo { + /// fn setup(&mut self) { + /// println!("Setting up foo"); + /// } + /// } + /// + /// let foo = init!(Foo { + /// buf <- zeroed() + /// }).chain(|foo| { + /// foo.setup(); + /// Ok(()) + /// }); + /// ``` + fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E> + where + F: FnOnce(&mut T) -> Result<(), E>, + { + ChainInit(self, f, PhantomData) + } +} + +/// An initializer returned by [`Init::chain`]. +pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, T)>); + +// SAFETY: The `__init` function is implemented such that it +// - returns `Ok(())` on successful initialization, +// - returns `Err(err)` on error and in this case `slot` will be dropped. +unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E> +where + I: Init<T, E>, + F: FnOnce(&mut T) -> Result<(), E>, +{ + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: All requirements fulfilled since this function is `__init`. + unsafe { self.0.__pinned_init(slot)? }; + // SAFETY: The above call initialized `slot` and we still have unique access. + (self.1)(unsafe { &mut *slot }).inspect_err(|_| + // SAFETY: `slot` was initialized above. + unsafe { core::ptr::drop_in_place(slot) }) + } +} + +// SAFETY: `__pinned_init` behaves exactly the same as `__init`. +unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E> +where + I: Init<T, E>, + F: FnOnce(&mut T) -> Result<(), E>, +{ + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: `__init` has less strict requirements compared to `__pinned_init`. + unsafe { self.__init(slot) } + } +} + +/// Creates a new [`PinInit<T, E>`] from the given closure. +/// +/// # Safety +/// +/// The closure: +/// - returns `Ok(())` if it initialized every field of `slot`, +/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: +/// - `slot` can be deallocated without UB occurring, +/// - `slot` does not need to be dropped, +/// - `slot` is not partially initialized. +/// - may assume that the `slot` does not move if `T: !Unpin`, +/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. +#[inline] +pub const unsafe fn pin_init_from_closure<T: ?Sized, E>( + f: impl FnOnce(*mut T) -> Result<(), E>, +) -> impl PinInit<T, E> { + __internal::InitClosure(f, PhantomData) +} + +/// Creates a new [`Init<T, E>`] from the given closure. +/// +/// # Safety +/// +/// The closure: +/// - returns `Ok(())` if it initialized every field of `slot`, +/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: +/// - `slot` can be deallocated without UB occurring, +/// - `slot` does not need to be dropped, +/// - `slot` is not partially initialized. +/// - the `slot` may move after initialization. +/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. +#[inline] +pub const unsafe fn init_from_closure<T: ?Sized, E>( + f: impl FnOnce(*mut T) -> Result<(), E>, +) -> impl Init<T, E> { + __internal::InitClosure(f, PhantomData) +} + +/// An initializer that leaves the memory uninitialized. +/// +/// The initializer is a no-op. The `slot` memory is not changed. +#[inline] +pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { + // SAFETY: The memory is allowed to be uninitialized. + unsafe { init_from_closure(|_| Ok(())) } +} + +/// Initializes an array by initializing each element via the provided initializer. +/// +/// # Examples +/// +/// ```rust +/// # use pin_init::*; +/// use pin_init::init_array_from_fn; +/// let array: Box<[usize; 1_000]> = Box::init(init_array_from_fn(|i| i)).unwrap(); +/// assert_eq!(array.len(), 1_000); +/// ``` +pub fn init_array_from_fn<I, const N: usize, T, E>( + mut make_init: impl FnMut(usize) -> I, +) -> impl Init<[T; N], E> +where + I: Init<T, E>, +{ + let init = move |slot: *mut [T; N]| { + let slot = slot.cast::<T>(); + for i in 0..N { + let init = make_init(i); + // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. + let ptr = unsafe { slot.add(i) }; + // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` + // requirements. + if let Err(e) = unsafe { init.__init(ptr) } { + // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return + // `Err` below, `slot` will be considered uninitialized memory. + unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; + return Err(e); + } + } + Ok(()) + }; + // SAFETY: The initializer above initializes every element of the array. On failure it drops + // any initialized elements and returns `Err`. + unsafe { init_from_closure(init) } +} + +/// Initializes an array by initializing each element via the provided initializer. +/// +/// # Examples +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// # use core::pin::Pin; +/// use pin_init::pin_init_array_from_fn; +/// use std::sync::Arc; +/// let array: Pin<Arc<[CMutex<usize>; 1_000]>> = +/// Arc::pin_init(pin_init_array_from_fn(|i| CMutex::new(i))).unwrap(); +/// assert_eq!(array.len(), 1_000); +/// ``` +pub fn pin_init_array_from_fn<I, const N: usize, T, E>( + mut make_init: impl FnMut(usize) -> I, +) -> impl PinInit<[T; N], E> +where + I: PinInit<T, E>, +{ + let init = move |slot: *mut [T; N]| { + let slot = slot.cast::<T>(); + for i in 0..N { + let init = make_init(i); + // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. + let ptr = unsafe { slot.add(i) }; + // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` + // requirements. + if let Err(e) = unsafe { init.__pinned_init(ptr) } { + // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return + // `Err` below, `slot` will be considered uninitialized memory. + unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; + return Err(e); + } + } + Ok(()) + }; + // SAFETY: The initializer above initializes every element of the array. On failure it drops + // any initialized elements and returns `Err`. + unsafe { pin_init_from_closure(init) } +} + +// SAFETY: Every type can be initialized by-value. +unsafe impl<T, E> Init<T, E> for T { + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: TODO. + unsafe { slot.write(self) }; + Ok(()) + } +} + +// SAFETY: Every type can be initialized by-value. `__pinned_init` calls `__init`. +unsafe impl<T, E> PinInit<T, E> for T { + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: TODO. + unsafe { self.__init(slot) } + } +} + +/// Smart pointer containing uninitialized memory and that can write a value. +pub trait InPlaceWrite<T> { + /// The type `Self` turns into when the contents are initialized. + type Initialized; + + /// Use the given initializer to write a value into `self`. + /// + /// Does not drop the current value and considers it as uninitialized memory. + fn write_init<E>(self, init: impl Init<T, E>) -> Result<Self::Initialized, E>; + + /// Use the given pin-initializer to write a value into `self`. + /// + /// Does not drop the current value and considers it as uninitialized memory. + fn write_pin_init<E>(self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E>; +} + +/// Trait facilitating pinned destruction. +/// +/// Use [`pinned_drop`] to implement this trait safely: +/// +/// ```rust +/// # #![feature(allocator_api)] +/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; +/// # use pin_init::*; +/// use core::pin::Pin; +/// #[pin_data(PinnedDrop)] +/// struct Foo { +/// #[pin] +/// mtx: CMutex<usize>, +/// } +/// +/// #[pinned_drop] +/// impl PinnedDrop for Foo { +/// fn drop(self: Pin<&mut Self>) { +/// println!("Foo is being dropped!"); +/// } +/// } +/// ``` +/// +/// # Safety +/// +/// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl. +pub unsafe trait PinnedDrop: __internal::HasPinData { + /// Executes the pinned destructor of this type. + /// + /// While this function is marked safe, it is actually unsafe to call it manually. For this + /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code + /// and thus prevents this function from being called where it should not. + /// + /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute + /// automatically. + fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop); +} + +/// Marker trait for types that can be initialized by writing just zeroes. +/// +/// # Safety +/// +/// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words, +/// this is not UB: +/// +/// ```rust,ignore +/// let val: Self = unsafe { core::mem::zeroed() }; +/// ``` +pub unsafe trait Zeroable {} + +/// Marker trait for types that allow `Option<Self>` to be set to all zeroes in order to write +/// `None` to that location. +/// +/// # Safety +/// +/// The implementer needs to ensure that `unsafe impl Zeroable for Option<Self> {}` is sound. +pub unsafe trait ZeroableOption {} + +// SAFETY: by the safety requirement of `ZeroableOption`, this is valid. +unsafe impl<T: ZeroableOption> Zeroable for Option<T> {} + +/// Create a new zeroed T. +/// +/// The returned initializer will write `0x00` to every byte of the given `slot`. +#[inline] +pub fn zeroed<T: Zeroable>() -> impl Init<T> { + // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T` + // and because we write all zeroes, the memory is initialized. + unsafe { + init_from_closure(|slot: *mut T| { + slot.write_bytes(0, 1); + Ok(()) + }) + } +} + +macro_rules! impl_zeroable { + ($($({$($generics:tt)*})? $t:ty, )*) => { + // SAFETY: Safety comments written in the macro invocation. + $(unsafe impl$($($generics)*)? Zeroable for $t {})* + }; +} + +impl_zeroable! { + // SAFETY: All primitives that are allowed to be zero. + bool, + char, + u8, u16, u32, u64, u128, usize, + i8, i16, i32, i64, i128, isize, + f32, f64, + + // Note: do not add uninhabited types (such as `!` or `core::convert::Infallible`) to this list; + // creating an instance of an uninhabited type is immediate undefined behavior. For more on + // uninhabited/empty types, consult The Rustonomicon: + // <https://doc.rust-lang.org/stable/nomicon/exotic-sizes.html#empty-types>. The Rust Reference + // also has information on undefined behavior: + // <https://doc.rust-lang.org/stable/reference/behavior-considered-undefined.html>. + // + // SAFETY: These are inhabited ZSTs; there is nothing to zero and a valid value exists. + {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, (), + + // SAFETY: Type is allowed to take any value, including all zeros. + {<T>} MaybeUninit<T>, + + // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`. + {<T: ?Sized + Zeroable>} UnsafeCell<T>, + + // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee: + // <https://doc.rust-lang.org/stable/std/option/index.html#representation>). + Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>, + Option<NonZeroU128>, Option<NonZeroUsize>, + Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>, + Option<NonZeroI128>, Option<NonZeroIsize>, + {<T>} Option<NonNull<T>>, + + // SAFETY: `null` pointer is valid. + // + // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be + // null. + // + // When `Pointee` gets stabilized, we could use + // `T: ?Sized where <T as Pointee>::Metadata: Zeroable` + {<T>} *mut T, {<T>} *const T, + + // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be + // zero. + {<T>} *mut [T], {<T>} *const [T], *mut str, *const str, + + // SAFETY: `T` is `Zeroable`. + {<const N: usize, T: Zeroable>} [T; N], {<T: Zeroable>} Wrapping<T>, +} + +macro_rules! impl_tuple_zeroable { + ($(,)?) => {}; + ($first:ident, $($t:ident),* $(,)?) => { + // SAFETY: All elements are zeroable and padding can be zero. + unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {} + impl_tuple_zeroable!($($t),* ,); + } +} + +impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J); diff --git a/rust/kernel/init/macros.rs b/rust/pin-init/src/macros.rs index 1fd146a83241..361623324d5c 100644 --- a/rust/kernel/init/macros.rs +++ b/rust/pin-init/src/macros.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT //! This module provides the macros that actually implement the proc-macros `pin_data` and -//! `pinned_drop`. It also contains `__init_internal` the implementation of the `{try_}{pin_}init!` -//! macros. +//! `pinned_drop`. It also contains `__init_internal`, the implementation of the +//! `{try_}{pin_}init!` macros. //! //! These macros should never be called directly, since they expect their input to be //! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in @@ -11,16 +11,17 @@ //! This architecture has been chosen because the kernel does not yet have access to `syn` which //! would make matters a lot easier for implementing these as proc-macros. //! +//! Since this library and the kernel implementation should diverge as little as possible, the same +//! approach has been taken here. +//! //! # Macro expansion example //! //! This section is intended for readers trying to understand the macros in this module and the -//! `pin_init!` macros from `init.rs`. +//! `[try_][pin_]init!` macros from `lib.rs`. //! //! We will look at the following example: //! //! ```rust,ignore -//! # use kernel::init::*; -//! # use core::pin::Pin; //! #[pin_data] //! #[repr(C)] //! struct Bar<T> { @@ -45,7 +46,7 @@ //! #[pinned_drop] //! impl PinnedDrop for Foo { //! fn drop(self: Pin<&mut Self>) { -//! pr_info!("{self:p} is getting dropped."); +//! println!("{self:p} is getting dropped."); //! } //! } //! @@ -75,7 +76,6 @@ //! Here is the definition of `Bar` from our example: //! //! ```rust,ignore -//! # use kernel::init::*; //! #[pin_data] //! #[repr(C)] //! struct Bar<T> { @@ -121,22 +121,22 @@ //! self, //! slot: *mut T, //! // Since `t` is `#[pin]`, this is `PinInit`. -//! init: impl ::kernel::init::PinInit<T, E>, +//! init: impl ::pin_init::PinInit<T, E>, //! ) -> ::core::result::Result<(), E> { -//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) } +//! unsafe { ::pin_init::PinInit::__pinned_init(init, slot) } //! } //! pub unsafe fn x<E>( //! self, //! slot: *mut usize, //! // Since `x` is not `#[pin]`, this is `Init`. -//! init: impl ::kernel::init::Init<usize, E>, +//! init: impl ::pin_init::Init<usize, E>, //! ) -> ::core::result::Result<(), E> { -//! unsafe { ::kernel::init::Init::__init(init, slot) } +//! unsafe { ::pin_init::Init::__init(init, slot) } //! } //! } //! // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct //! // that we constructed above. -//! unsafe impl<T> ::kernel::init::__internal::HasPinData for Bar<T> { +//! unsafe impl<T> ::pin_init::__internal::HasPinData for Bar<T> { //! type PinData = __ThePinData<T>; //! unsafe fn __pin_data() -> Self::PinData { //! __ThePinData { @@ -147,7 +147,7 @@ //! // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data //! // struct. This is important to ensure that no user can implement a rogue `__pin_data` //! // function without using `unsafe`. -//! unsafe impl<T> ::kernel::init::__internal::PinData for __ThePinData<T> { +//! unsafe impl<T> ::pin_init::__internal::PinData for __ThePinData<T> { //! type Datee = Bar<T>; //! } //! // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is @@ -191,7 +191,7 @@ //! #[expect(non_camel_case_types)] //! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} //! impl< -//! T: ::kernel::init::PinnedDrop, +//! T: ::pin_init::PinnedDrop, //! > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} //! impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {} //! }; @@ -227,11 +227,11 @@ //! // - we `use` the `HasPinData` trait in the block, it is only available in that //! // scope. //! let data = unsafe { -//! use ::kernel::init::__internal::HasPinData; +//! use ::pin_init::__internal::HasPinData; //! Self::__pin_data() //! }; //! // Ensure that `data` really is of type `PinData` and help with type inference: -//! let init = ::kernel::init::__internal::PinData::make_closure::< +//! let init = ::pin_init::__internal::PinData::make_closure::< //! _, //! __InitOk, //! ::core::convert::Infallible, @@ -251,7 +251,7 @@ //! // is an error later. This `DropGuard` will drop the field when it gets //! // dropped and has not yet been forgotten. //! let __t_guard = unsafe { -//! ::pinned_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t)) +//! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t)) //! }; //! // Expansion of `x: 0,`: //! // Since this can be an arbitrary expression we cannot place it inside @@ -262,7 +262,7 @@ //! } //! // We again create a `DropGuard`. //! let __x_guard = unsafe { -//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x)) +//! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x)) //! }; //! // Since initialization has successfully completed, we can now forget //! // the guards. This is not `mem::forget`, since we only have @@ -303,7 +303,7 @@ //! }; //! // Construct the initializer. //! let init = unsafe { -//! ::kernel::init::pin_init_from_closure::< +//! ::pin_init::pin_init_from_closure::< //! _, //! ::core::convert::Infallible, //! >(init) @@ -350,19 +350,19 @@ //! unsafe fn b<E>( //! self, //! slot: *mut Bar<u32>, -//! init: impl ::kernel::init::PinInit<Bar<u32>, E>, +//! init: impl ::pin_init::PinInit<Bar<u32>, E>, //! ) -> ::core::result::Result<(), E> { -//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) } +//! unsafe { ::pin_init::PinInit::__pinned_init(init, slot) } //! } //! unsafe fn a<E>( //! self, //! slot: *mut usize, -//! init: impl ::kernel::init::Init<usize, E>, +//! init: impl ::pin_init::Init<usize, E>, //! ) -> ::core::result::Result<(), E> { -//! unsafe { ::kernel::init::Init::__init(init, slot) } +//! unsafe { ::pin_init::Init::__init(init, slot) } //! } //! } -//! unsafe impl ::kernel::init::__internal::HasPinData for Foo { +//! unsafe impl ::pin_init::__internal::HasPinData for Foo { //! type PinData = __ThePinData; //! unsafe fn __pin_data() -> Self::PinData { //! __ThePinData { @@ -370,7 +370,7 @@ //! } //! } //! } -//! unsafe impl ::kernel::init::__internal::PinData for __ThePinData { +//! unsafe impl ::pin_init::__internal::PinData for __ThePinData { //! type Datee = Foo; //! } //! #[allow(dead_code)] @@ -394,8 +394,8 @@ //! let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) }; //! // Create the unsafe token that proves that we are inside of a destructor, this //! // type is only allowed to be created in a destructor. -//! let token = unsafe { ::kernel::init::__internal::OnlyCallFromDrop::new() }; -//! ::kernel::init::PinnedDrop::drop(pinned, token); +//! let token = unsafe { ::pin_init::__internal::OnlyCallFromDrop::new() }; +//! ::pin_init::PinnedDrop::drop(pinned, token); //! } //! } //! }; @@ -412,7 +412,7 @@ //! #[pinned_drop] //! impl PinnedDrop for Foo { //! fn drop(self: Pin<&mut Self>) { -//! pr_info!("{self:p} is getting dropped."); +//! println!("{self:p} is getting dropped."); //! } //! } //! ``` @@ -421,9 +421,9 @@ //! //! ```rust,ignore //! // `unsafe`, full path and the token parameter are added, everything else stays the same. -//! unsafe impl ::kernel::init::PinnedDrop for Foo { -//! fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) { -//! pr_info!("{self:p} is getting dropped."); +//! unsafe impl ::pin_init::PinnedDrop for Foo { +//! fn drop(self: Pin<&mut Self>, _: ::pin_init::__internal::OnlyCallFromDrop) { +//! println!("{self:p} is getting dropped."); //! } //! } //! ``` @@ -448,10 +448,10 @@ //! let initializer = { //! struct __InitOk; //! let data = unsafe { -//! use ::kernel::init::__internal::HasPinData; +//! use ::pin_init::__internal::HasPinData; //! Foo::__pin_data() //! }; -//! let init = ::kernel::init::__internal::PinData::make_closure::< +//! let init = ::pin_init::__internal::PinData::make_closure::< //! _, //! __InitOk, //! ::core::convert::Infallible, @@ -462,12 +462,12 @@ //! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) }; //! } //! let __a_guard = unsafe { -//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a)) +//! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a)) //! }; //! let init = Bar::new(36); //! unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? }; //! let __b_guard = unsafe { -//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b)) +//! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b)) //! }; //! ::core::mem::forget(__b_guard); //! ::core::mem::forget(__a_guard); @@ -492,12 +492,17 @@ //! init(slot).map(|__InitOk| ()) //! }; //! let init = unsafe { -//! ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init) +//! ::pin_init::pin_init_from_closure::<_, ::core::convert::Infallible>(init) //! }; //! init //! }; //! ``` +#[cfg(kernel)] +pub use ::macros::paste; +#[cfg(not(kernel))] +pub use ::paste::paste; + /// Creates a `unsafe impl<...> PinnedDrop for $type` block. /// /// See [`PinnedDrop`] for more information. @@ -517,7 +522,7 @@ macro_rules! __pinned_drop { unsafe $($impl_sig)* { // Inherit all attributes and the type/ident tokens for the signature. $(#[$($attr)*])* - fn drop($($sig)*, _: $crate::init::__internal::OnlyCallFromDrop) { + fn drop($($sig)*, _: $crate::__internal::OnlyCallFromDrop) { $($inner)* } } @@ -863,7 +868,7 @@ macro_rules! __pin_data { // SAFETY: We have added the correct projection functions above to `__ThePinData` and // we also use the least restrictive generics possible. unsafe impl<$($impl_generics)*> - $crate::init::__internal::HasPinData for $name<$($ty_generics)*> + $crate::__internal::HasPinData for $name<$($ty_generics)*> where $($whr)* { type PinData = __ThePinData<$($ty_generics)*>; @@ -875,7 +880,7 @@ macro_rules! __pin_data { // SAFETY: TODO. unsafe impl<$($impl_generics)*> - $crate::init::__internal::PinData for __ThePinData<$($ty_generics)*> + $crate::__internal::PinData for __ThePinData<$($ty_generics)*> where $($whr)* { type Datee = $name<$($ty_generics)*>; @@ -934,7 +939,7 @@ macro_rules! __pin_data { // `PinnedDrop` as the parameter to `#[pin_data]`. #[expect(non_camel_case_types)] trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} - impl<T: $crate::init::PinnedDrop> + impl<T: $crate::PinnedDrop> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} impl<$($impl_generics)*> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*> @@ -957,8 +962,8 @@ macro_rules! __pin_data { let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) }; // SAFETY: Since this is a drop function, we can create this token to call the // pinned destructor of this type. - let token = unsafe { $crate::init::__internal::OnlyCallFromDrop::new() }; - $crate::init::PinnedDrop::drop(pinned, token); + let token = unsafe { $crate::__internal::OnlyCallFromDrop::new() }; + $crate::PinnedDrop::drop(pinned, token); } } }; @@ -998,10 +1003,10 @@ macro_rules! __pin_data { $pvis unsafe fn $p_field<E>( self, slot: *mut $p_type, - init: impl $crate::init::PinInit<$p_type, E>, + init: impl $crate::PinInit<$p_type, E>, ) -> ::core::result::Result<(), E> { // SAFETY: TODO. - unsafe { $crate::init::PinInit::__pinned_init(init, slot) } + unsafe { $crate::PinInit::__pinned_init(init, slot) } } )* $( @@ -1009,10 +1014,10 @@ macro_rules! __pin_data { $fvis unsafe fn $field<E>( self, slot: *mut $type, - init: impl $crate::init::Init<$type, E>, + init: impl $crate::Init<$type, E>, ) -> ::core::result::Result<(), E> { // SAFETY: TODO. - unsafe { $crate::init::Init::__init(init, slot) } + unsafe { $crate::Init::__init(init, slot) } } )* } @@ -1129,15 +1134,15 @@ macro_rules! __init_internal { // // SAFETY: TODO. let data = unsafe { - use $crate::init::__internal::$has_data; + use $crate::__internal::$has_data; // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal // information that is associated to already parsed fragments, so a path fragment // cannot be used in this position. Doing the retokenization results in valid rust // code. - ::kernel::macros::paste!($t::$get_data()) + $crate::macros::paste!($t::$get_data()) }; // Ensure that `data` really is of type `$data` and help with type inference: - let init = $crate::init::__internal::$data::make_closure::<_, __InitOk, $err>( + let init = $crate::__internal::$data::make_closure::<_, __InitOk, $err>( data, move |slot| { { @@ -1147,7 +1152,7 @@ macro_rules! __init_internal { // error when fields are missing (since they will be zeroed). We also have to // check that the type actually implements `Zeroable`. $({ - fn assert_zeroable<T: $crate::init::Zeroable>(_: *mut T) {} + fn assert_zeroable<T: $crate::Zeroable>(_: *mut T) {} // Ensure that the struct is indeed `Zeroable`. assert_zeroable(slot); // SAFETY: The type implements `Zeroable` by the check above. @@ -1184,7 +1189,7 @@ macro_rules! __init_internal { init(slot).map(|__InitOk| ()) }; // SAFETY: TODO. - let init = unsafe { $crate::init::$construct_closure::<_, $err>(init) }; + let init = unsafe { $crate::$construct_closure::<_, $err>(init) }; init }}; (init_slot($($use_data:ident)?): @@ -1215,10 +1220,10 @@ macro_rules! __init_internal { // // We rely on macro hygiene to make it impossible for users to access this local variable. // We use `paste!` to create new hygiene for `$field`. - ::kernel::macros::paste! { + $crate::macros::paste! { // SAFETY: We forget the guard later when initialization has succeeded. let [< __ $field _guard >] = unsafe { - $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) + $crate::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) }; $crate::__init_internal!(init_slot($use_data): @@ -1241,15 +1246,15 @@ macro_rules! __init_internal { // // SAFETY: `slot` is valid, because we are inside of an initializer closure, we // return when an error/panic occurs. - unsafe { $crate::init::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? }; + unsafe { $crate::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? }; // Create the drop guard: // // We rely on macro hygiene to make it impossible for users to access this local variable. // We use `paste!` to create new hygiene for `$field`. - ::kernel::macros::paste! { + $crate::macros::paste! { // SAFETY: We forget the guard later when initialization has succeeded. let [< __ $field _guard >] = unsafe { - $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) + $crate::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) }; $crate::__init_internal!(init_slot(): @@ -1278,10 +1283,10 @@ macro_rules! __init_internal { // // We rely on macro hygiene to make it impossible for users to access this local variable. // We use `paste!` to create new hygiene for `$field`. - ::kernel::macros::paste! { + $crate::macros::paste! { // SAFETY: We forget the guard later when initialization has succeeded. let [< __ $field _guard >] = unsafe { - $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) + $crate::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) }; $crate::__init_internal!(init_slot($($use_data)?): @@ -1315,7 +1320,7 @@ macro_rules! __init_internal { // information that is associated to already parsed fragments, so a path fragment // cannot be used in this position. Doing the retokenization results in valid rust // code. - ::kernel::macros::paste!( + $crate::macros::paste!( ::core::ptr::write($slot, $t { $($acc)* ..zeroed @@ -1339,7 +1344,7 @@ macro_rules! __init_internal { // information that is associated to already parsed fragments, so a path fragment // cannot be used in this position. Doing the retokenization results in valid rust // code. - ::kernel::macros::paste!( + $crate::macros::paste!( ::core::ptr::write($slot, $t { $($acc)* }); @@ -1394,12 +1399,12 @@ macro_rules! __derive_zeroable { ) => { // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero. #[automatically_derived] - unsafe impl<$($impl_generics)*> $crate::init::Zeroable for $name<$($ty_generics)*> + unsafe impl<$($impl_generics)*> $crate::Zeroable for $name<$($ty_generics)*> where $($($whr)*)? {} const _: () = { - fn assert_zeroable<T: ?::core::marker::Sized + $crate::init::Zeroable>() {} + fn assert_zeroable<T: ?::core::marker::Sized + $crate::Zeroable>() {} fn ensure_zeroable<$($impl_generics)*>() where $($($whr)*)? { diff --git a/rust/uapi/lib.rs b/rust/uapi/lib.rs index 13495910271f..c98d7a8cde77 100644 --- a/rust/uapi/lib.rs +++ b/rust/uapi/lib.rs @@ -24,6 +24,7 @@ unreachable_pub, unsafe_op_in_unsafe_fn )] +#![cfg_attr(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES, allow(unnecessary_transmutes))] // Manual definition of blocklisted types. type __kernel_size_t = usize; |