Merge branch 'next' into for-linus

Prepare input updates for 7.1 merge window.
author: Dmitry Torokhov <dmitry.torokhov@gmail.com> 2026-04-20 04:28:57 +0300
committer: Dmitry Torokhov <dmitry.torokhov@gmail.com> 2026-04-20 04:28:57 +0300
commit: f4b369c6fe0ceaba2da2daff8c9eb415f85926dd (patch)
tree: 30465d0a429b2c224685b5d8e804bf053c4d129a /lib
parent: ff14dafde15c11403fac61367a34fea08926e9ee (diff)
parent: 2ca45e57ea027fffe3350ae5e21ad9cecb0dce74 (diff)
download: linux-f4b369c6fe0ceaba2da2daff8c9eb415f85926dd.tar.xz
227 files changed, 27842 insertions, 2323 deletions
diff --git a/lib/Kconfig b/lib/Kconfig
index e629449dd2a3..0f2fb9610647 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -430,19 +430,6 @@ config GLOB
 	  are compiling an out-of tree driver which tells you that it
 	  depends on this.
 
-config GLOB_SELFTEST
-	tristate "glob self-test on init"
-	depends on GLOB
-	help
-	  This option enables a simple self-test of the glob_match
-	  function on startup.	It is primarily useful for people
-	  working on the code to ensure they haven't introduced any
-	  regressions.
-
-	  It only adds a little bit of code and slows kernel boot (or
-	  module load) by a small amount, so you're welcome to play with
-	  it, but you probably don't need it.
-
 #
 # Netlink attribute parsing support is select'ed if needed
 #
@@ -542,6 +529,9 @@ config MEMREGION
 config ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION
 	bool
 
+config GENERIC_CPU_CACHE_MAINTENANCE
+	bool
+
 config ARCH_HAS_MEMREMAP_COMPAT_ALIGN
 	bool
 
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 713cc94caa02..93f356d2b3d9 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -35,6 +35,18 @@ config PRINTK_CALLER
 	  no option to enable/disable at the kernel command line parameter or
 	  sysfs interface.
 
+config PRINTK_EXECUTION_CTX
+	bool
+	depends on PRINTK
+	help
+	  This option extends struct printk_info to include extra execution
+	  context in printk, such as task name and CPU number from where the
+	  message originated. This is useful for correlating printk messages
+	  with specific execution contexts.
+
+	  This is automatically enabled when a console driver that supports
+	  execution context is selected.
+
 config STACKTRACE_BUILD_ID
 	bool "Show build ID information in stacktraces"
 	depends on PRINTK
@@ -206,6 +218,16 @@ config DEBUG_BUGVERBOSE
 	  of the BUG call as well as the EIP and oops trace.  This aids
 	  debugging but costs about 70-100K of memory.
 
+config DEBUG_BUGVERBOSE_DETAILED
+	bool "Verbose WARN_ON_ONCE() reporting (adds 100K)" if DEBUG_BUGVERBOSE
+	help
+	  Say Y here to make WARN_ON_ONCE() output the condition string of the
+	  warning, in addition to the file name and line number.
+	  This helps debugging, but costs about 100K of memory.
+
+	  Say N if unsure.
+
+
 endmenu # "printk and dmesg options"
 
 config DEBUG_KERNEL
@@ -332,8 +354,7 @@ config DEBUG_INFO_COMPRESSED_ZLIB
 	depends on $(cc-option,-gz=zlib)
 	depends on $(ld-option,--compress-debug-sections=zlib)
 	help
-	  Compress the debug information using zlib.  Requires GCC 5.0+ or Clang
-	  5.0+, binutils 2.26+, and zlib.
+	  Compress the debug information using zlib.
 
 	  Users of dpkg-deb via debian/rules may find an increase in
 	  size of their debug .deb packages with this config set, due to the
@@ -379,18 +400,13 @@ config DEBUG_INFO_BTF
 	depends on !DEBUG_INFO_SPLIT && !DEBUG_INFO_REDUCED
 	depends on !GCC_PLUGIN_RANDSTRUCT || COMPILE_TEST
 	depends on BPF_SYSCALL
-	depends on PAHOLE_VERSION >= 116
-	depends on DEBUG_INFO_DWARF4 || PAHOLE_VERSION >= 121
+	depends on PAHOLE_VERSION >= 122
 	# pahole uses elfutils, which does not have support for Hexagon relocations
 	depends on !HEXAGON
 	help
 	  Generate deduplicated BTF type information from DWARF debug info.
-	  Turning this on requires pahole v1.16 or later (v1.21 or later to
-	  support DWARF 5), which will convert DWARF type info into equivalent
-	  deduplicated BTF type info.
-
-config PAHOLE_HAS_SPLIT_BTF
-	def_bool PAHOLE_VERSION >= 119
+	  Turning this on requires pahole v1.22 or later, which will convert
+	  DWARF type info into equivalent deduplicated BTF type info.
 
 config PAHOLE_HAS_BTF_TAG
 	def_bool PAHOLE_VERSION >= 123
@@ -412,7 +428,7 @@ config PAHOLE_HAS_LANG_EXCLUDE
 config DEBUG_INFO_BTF_MODULES
 	bool "Generate BTF type information for kernel modules"
 	default y
-	depends on DEBUG_INFO_BTF && MODULES && PAHOLE_HAS_SPLIT_BTF
+	depends on DEBUG_INFO_BTF && MODULES
 	help
 	  Generate compact split BTF type information for kernel modules.
 
@@ -483,23 +499,23 @@ config DEBUG_SECTION_MISMATCH
 	bool "Enable full Section mismatch analysis"
 	depends on CC_IS_GCC
 	help
-	  The section mismatch analysis checks if there are illegal
-	  references from one section to another section.
-	  During linktime or runtime, some sections are dropped;
-	  any use of code/data previously in these sections would
-	  most likely result in an oops.
-	  In the code, functions and variables are annotated with
-	  __init,, etc. (see the full list in include/linux/init.h),
-	  which results in the code/data being placed in specific sections.
+	  The section mismatch analysis checks if there are illegal references
+	  from one section to another. During linktime or runtime, some
+	  sections are dropped; any use of code/data previously in these
+	  sections would most likely result in an oops.
+
+	  In the code, functions and variables are annotated with __init,
+	  __initdata, and so on (see the full list in include/linux/init.h).
+	  This directs the toolchain to place code/data in specific sections.
+
 	  The section mismatch analysis is always performed after a full
-	  kernel build, and enabling this option causes the following
-	  additional step to occur:
-	  - Add the option -fno-inline-functions-called-once to gcc commands.
-	    When inlining a function annotated with __init in a non-init
-	    function, we would lose the section information and thus
-	    the analysis would not catch the illegal reference.
-	    This option tells gcc to inline less (but it does result in
-	    a larger kernel).
+	  kernel build, and enabling this option causes the option
+	  -fno-inline-functions-called-once to be added to gcc commands.
+
+	  However, when inlining a function annotated with __init in
+	  a non-init function, we would lose the section information and thus
+	  the analysis would not catch the illegal reference.  This option
+	  tells gcc to inline less (but it does result in a larger kernel).
 
 config SECTION_MISMATCH_WARN_ONLY
 	bool "Make section mismatch errors non-fatal"
@@ -612,6 +628,36 @@ config DEBUG_FORCE_WEAK_PER_CPU
 	  To ensure that generic code follows the above rules, this
 	  option forces all percpu variables to be defined as weak.
 
+config WARN_CONTEXT_ANALYSIS
+	bool "Compiler context-analysis warnings"
+	depends on CC_IS_CLANG && CLANG_VERSION >= 220100
+	# Branch profiling re-defines "if", which messes with the compiler's
+	# ability to analyze __cond_acquires(..), resulting in false positives.
+	depends on !TRACE_BRANCH_PROFILING
+	default y
+	help
+	  Context Analysis is a language extension, which enables statically
+	  checking that required contexts are active (or inactive) by acquiring
+	  and releasing user-definable "context locks".
+
+	  Clang's name of the feature is "Thread Safety Analysis". Requires
+	  Clang 22.1.0 or later.
+
+	  Produces warnings by default. Select CONFIG_WERROR if you wish to
+	  turn these warnings into errors.
+
+	  For more details, see Documentation/dev-tools/context-analysis.rst.
+
+config WARN_CONTEXT_ANALYSIS_ALL
+	bool "Enable context analysis for all source files"
+	depends on WARN_CONTEXT_ANALYSIS
+	depends on EXPERT && !COMPILE_TEST
+	help
+	  Enable tree-wide context analysis. This is likely to produce a
+	  large number of false positives - enable at your own risk.
+
+	  If unsure, say N.
+
 endmenu # "Compiler options"
 
 menu "Generic Kernel Debugging Instruments"
@@ -678,7 +724,7 @@ choice
 	help
 	  This selects the default access restrictions for debugfs.
 	  It can be overridden with kernel command line option
-	  debugfs=[on,no-mount,off]. The restrictions apply for API access
+	  debugfs=[on,off]. The restrictions apply for API access
 	  and filesystem registration.
 
 config DEBUG_FS_ALLOW_ALL
@@ -687,13 +733,6 @@ config DEBUG_FS_ALLOW_ALL
 	  No restrictions apply. Both API and filesystem registration
 	  is on. This is the normal default operation.
 
-config DEBUG_FS_DISALLOW_MOUNT
-	bool "Do not register debugfs as filesystem"
-	help
-	  The API is open but filesystem is not loaded. Clients can still do
-	  their work and read with debug tools that do not need
-	  debugfs filesystem.
-
 config DEBUG_FS_ALLOW_NONE
 	bool "No access"
 	help
@@ -721,6 +760,7 @@ source "mm/Kconfig.debug"
 
 config DEBUG_OBJECTS
 	bool "Debug object operations"
+	depends on PREEMPT_COUNT || !DEFERRED_STRUCT_PAGE_INIT
 	depends on DEBUG_KERNEL
 	help
 	  If you say Y here, additional code will be inserted into the
@@ -1108,13 +1148,14 @@ config SOFTLOCKUP_DETECTOR_INTR_STORM
 	  the CPU stats and the interrupt counts during the "soft lockups".
 
 config BOOTPARAM_SOFTLOCKUP_PANIC
-	bool "Panic (Reboot) On Soft Lockups"
+	int "Panic (Reboot) On Soft Lockups"
 	depends on SOFTLOCKUP_DETECTOR
+	default 0
 	help
-	  Say Y here to enable the kernel to panic on "soft lockups",
-	  which are bugs that cause the kernel to loop in kernel
-	  mode for more than 20 seconds (configurable using the watchdog_thresh
-	  sysctl), without giving other tasks a chance to run.
+	  Set to a non-zero value N to enable the kernel to panic on "soft
+	  lockups", which are bugs that cause the kernel to loop in kernel
+	  mode for more than (N * 20 seconds) (configurable using the
+	  watchdog_thresh sysctl), without giving other tasks a chance to run.
 
 	  The panic can be used in combination with panic_timeout,
 	  to cause the system to reboot automatically after a
@@ -1122,7 +1163,7 @@ config BOOTPARAM_SOFTLOCKUP_PANIC
 	  high-availability systems that have uptime guarantees and
 	  where a lockup must be resolved ASAP.
 
-	  Say N if unsure.
+	  Say 0 if unsure.
 
 config HAVE_HARDLOCKUP_DETECTOR_BUDDY
 	bool
@@ -1257,12 +1298,13 @@ config DEFAULT_HUNG_TASK_TIMEOUT
 	  Keeping the default should be fine in most cases.
 
 config BOOTPARAM_HUNG_TASK_PANIC
-	bool "Panic (Reboot) On Hung Tasks"
+	int "Number of hung tasks to trigger kernel panic"
 	depends on DETECT_HUNG_TASK
+	default 0
 	help
-	  Say Y here to enable the kernel to panic on "hung tasks",
-	  which are bugs that cause the kernel to leave a task stuck
-	  in uninterruptible "D" state.
+	  When set to a non-zero value, a kernel panic will be triggered
+	  if the number of hung tasks found during a single scan reaches
+	  this value.
 
 	  The panic can be used in combination with panic_timeout,
 	  to cause the system to reboot automatically after a
@@ -1270,7 +1312,7 @@ config BOOTPARAM_HUNG_TASK_PANIC
 	  high-availability systems that have uptime guarantees and
 	  where a hung tasks must be resolved ASAP.
 
-	  Say N if unsure.
+	  Say 0 if unsure.
 
 config DETECT_HUNG_TASK_BLOCKER
 	bool "Dump Hung Tasks Blocker"
@@ -1294,6 +1336,29 @@ config WQ_WATCHDOG
 	  state.  This can be configured through kernel parameter
 	  "workqueue.watchdog_thresh" and its sysfs counterpart.
 
+config BOOTPARAM_WQ_STALL_PANIC
+	int "Panic on Nth workqueue stall"
+	default 0
+	range 0 100
+	depends on WQ_WATCHDOG
+	help
+	  Set the number of workqueue stalls to trigger a kernel panic.
+	  A workqueue stall occurs when a worker pool doesn't make forward
+	  progress on a pending work item for over 30 seconds (configurable
+	  using the workqueue.watchdog_thresh parameter).
+
+	  If n = 0, the kernel will not panic on stall. If n > 0, the kernel
+	  will panic after n stall warnings.
+
+	  The panic can be used in combination with panic_timeout,
+	  to cause the system to reboot automatically after a
+	  stall has been detected. This feature is useful for
+	  high-availability systems that have uptime guarantees and
+	  where a stall must be resolved ASAP.
+
+	  This setting can be overridden at runtime via the
+	  workqueue.panic_on_stall kernel parameter.
+
 config WQ_CPU_INTENSIVE_REPORT
 	bool "Report per-cpu work items which hog CPU for too long"
 	depends on DEBUG_KERNEL
@@ -1356,6 +1421,24 @@ config DEBUG_PREEMPT
 	  depending on workload as it triggers debugging routines for each
 	  this_cpu operation. It should only be used for debugging purposes.
 
+config DEBUG_ATOMIC
+	bool "Debug atomic variables"
+	depends on DEBUG_KERNEL
+	help
+	  If you say Y here then the kernel will add a runtime alignment check
+	  to atomic accesses. Useful for architectures that do not have trap on
+	  mis-aligned access.
+
+	  This option has potentially significant overhead.
+
+config DEBUG_ATOMIC_LARGEST_ALIGN
+	bool "Check alignment only up to __aligned_largest"
+	depends on DEBUG_ATOMIC
+	help
+	  If you say Y here then the check for natural alignment of
+	  atomic accesses will be constrained to the compiler's largest
+	  alignment for scalar types.
+
 menu "Lock Debugging (spinlocks, mutexes, etc...)"
 
 config LOCK_DEBUGGING_SUPPORT
@@ -1684,33 +1767,6 @@ config STACKTRACE
 	  It is also used by various kernel debugging features that require
 	  stack trace generation.
 
-config WARN_ALL_UNSEEDED_RANDOM
-	bool "Warn for all uses of unseeded randomness"
-	default n
-	help
-	  Some parts of the kernel contain bugs relating to their use of
-	  cryptographically secure random numbers before it's actually possible
-	  to generate those numbers securely. This setting ensures that these
-	  flaws don't go unnoticed, by enabling a message, should this ever
-	  occur. This will allow people with obscure setups to know when things
-	  are going wrong, so that they might contact developers about fixing
-	  it.
-
-	  Unfortunately, on some models of some architectures getting
-	  a fully seeded CRNG is extremely difficult, and so this can
-	  result in dmesg getting spammed for a surprisingly long
-	  time.  This is really bad from a security perspective, and
-	  so architecture maintainers really need to do what they can
-	  to get the CRNG seeded sooner after the system is booted.
-	  However, since users cannot do anything actionable to
-	  address this, by default this option is disabled.
-
-	  Say Y here if you want to receive warnings for all uses of
-	  unseeded randomness.  This will be of use primarily for
-	  those developers interested in improving the security of
-	  Linux kernels running on their architecture (or
-	  subarchitecture).
-
 config DEBUG_KOBJECT
 	bool "kobject debugging"
 	depends on DEBUG_KERNEL
@@ -2274,16 +2330,6 @@ config TEST_LIST_SORT
 
 	  If unsure, say N.
 
-config TEST_MIN_HEAP
-	tristate "Min heap test"
-	depends on DEBUG_KERNEL || m
-	help
-	  Enable this to turn on min heap function tests. This test is
-	  executed only once during system boot (so affects only boot time),
-	  or at module load time.
-
-	  If unsure, say N.
-
 config TEST_SORT
 	tristate "Array-based sort test" if !KUNIT_ALL_TESTS
 	depends on KUNIT
@@ -2496,9 +2542,6 @@ config TEST_BITMAP
 
 	  If unsure, say N.
 
-config TEST_UUID
-	tristate "Test functions located in the uuid module at runtime"
-
 config TEST_XARRAY
 	tristate "Test the XArray code at runtime"
 
@@ -2548,14 +2591,6 @@ config TEST_PARMAN
 
 	  If unsure, say N.
 
-config TEST_IRQ_TIMINGS
-	bool "IRQ timings selftest"
-	depends on IRQ_TIMINGS
-	help
-	  Enable this option to test the irq timings code on boot.
-
-	  If unsure, say N.
-
 config TEST_LKM
 	tristate "Test module loading with 'hello world' module"
 	depends on m
@@ -2649,6 +2684,19 @@ config TEST_SYSCTL
 
 	  If unsure, say N.
 
+config BITOPS_KUNIT
+	tristate "KUnit test for bitops" if !KUNIT_ALL_TESTS
+	depends on KUNIT
+	default KUNIT_ALL_TESTS
+	help
+	  This option enables the KUnit test for the bitops library
+	  which provides functions for bit operations.
+
+	  Note that this is derived from the original test_bitops module.
+	  For micro-benchmarks and compiler warning checks, enable TEST_BITOPS.
+
+	  If unsure, say N.
+
 config BITFIELD_KUNIT
 	tristate "KUnit test bitfield functions at runtime" if !KUNIT_ALL_TESTS
 	depends on KUNIT
@@ -2777,6 +2825,20 @@ config LIST_KUNIT_TEST
 
 	  If unsure, say N.
 
+config LIST_PRIVATE_KUNIT_TEST
+	tristate "KUnit Test for Kernel Private Linked-list structures" if !KUNIT_ALL_TESTS
+	depends on KUNIT
+	default KUNIT_ALL_TESTS
+	help
+	  This builds the KUnit test for the private linked-list primitives
+	  defined in include/linux/list_private.h.
+
+	  These primitives allow manipulation of list_head members that are
+	  marked as private and require special accessors (ACCESS_PRIVATE)
+	  to strip qualifiers or handle encapsulation.
+
+	  If unsure, say N.
+
 config HASHTABLE_KUNIT_TEST
 	tristate "KUnit Test for Kernel Hashtable structures" if !KUNIT_ALL_TESTS
 	depends on KUNIT
@@ -2802,6 +2864,43 @@ config LINEAR_RANGES_TEST
 
 	  If unsure, say N.
 
+config CONTEXT_ANALYSIS_TEST
+	bool "Compiler context-analysis warnings test"
+	depends on EXPERT
+	help
+	  This builds the test for compiler-based context analysis. The test
+	  does not add executable code to the kernel, but is meant to test that
+	  common patterns supported by the analysis do not result in false
+	  positive warnings.
+
+	  When adding support for new context locks, it is strongly recommended
+	  to add supported patterns to this test.
+
+	  If unsure, say N.
+
+config LIVEUPDATE_TEST
+	bool "Live Update Kernel Test"
+	default n
+	depends on LIVEUPDATE
+	help
+	  Enable a built-in kernel test module for the Live Update
+	  Orchestrator.
+
+	  This module validates the File-Lifecycle-Bound subsystem by
+	  registering a set of mock FLB objects with any real file handlers
+	  that support live update (such as the memfd handler).
+
+	  When live update operations are performed, this test module will
+	  output messages to the kernel log (dmesg), confirming that its
+	  registration and various callback functions (preserve, retrieve,
+	  finish, etc.) are being invoked correctly.
+
+	  This is a debugging and regression testing tool for developers
+	  working on the Live Update subsystem. It should not be enabled in
+	  production kernels.
+
+	  If unsure, say N
+
 config CMDLINE_KUNIT_TEST
 	tristate "KUnit test for cmdline API" if !KUNIT_ALL_TESTS
 	depends on KUNIT
@@ -2814,8 +2913,25 @@ config CMDLINE_KUNIT_TEST
 
 	  If unsure, say N.
 
+config BASE64_KUNIT
+	tristate "KUnit test for base64 decoding and encoding" if !KUNIT_ALL_TESTS
+	depends on KUNIT
+	default KUNIT_ALL_TESTS
+	help
+	  This builds the base64 unit tests.
+
+	  The tests cover the encoding and decoding logic of Base64 functions
+	  in the kernel.
+	  In addition to correctness checks, simple performance benchmarks
+	  for both encoding and decoding are also included.
+
+	  For more information on KUnit and unit tests in general please refer
+	  to the KUnit documentation in Documentation/dev-tools/kunit/.
+
+	  If unsure, say N.
+
 config BITS_TEST
-	tristate "KUnit test for bits.h" if !KUNIT_ALL_TESTS
+	tristate "KUnit test for bit functions and macros" if !KUNIT_ALL_TESTS
 	depends on KUNIT
 	default KUNIT_ALL_TESTS
 	help
@@ -2860,6 +2976,17 @@ config MEMCPY_KUNIT_TEST
 
 	  If unsure, say N.
 
+config MIN_HEAP_KUNIT_TEST
+	tristate "Min heap test" if !KUNIT_ALL_TESTS
+	depends on KUNIT
+	default KUNIT_ALL_TESTS
+	help
+	  This option enables the KUnit test suite for the min heap library
+	  which provides functions for creating and managing min heaps.
+	  The test suite checks the functionality of the min heap library.
+
+	  If unsure, say N
+
 config IS_SIGNED_TYPE_KUNIT_TEST
 	tristate "Test is_signed_type() macro" if !KUNIT_ALL_TESTS
 	depends on KUNIT
@@ -3265,6 +3392,17 @@ config RATELIMIT_KUNIT_TEST
 
 	  If unsure, say N.
 
+config UUID_KUNIT_TEST
+	tristate "KUnit test for UUID" if !KUNIT_ALL_TESTS
+	depends on KUNIT
+	default KUNIT_ALL_TESTS
+	help
+	  This option enables the KUnit test suite for the uuid library,
+	  which provides functions for generating and parsing UUID and GUID.
+	  The test suite checks parsing of UUID and GUID strings.
+
+	  If unsure, say N.
+
 config INT_POW_KUNIT_TEST
 	tristate "Integer exponentiation (int_pow) test" if !KUNIT_ALL_TESTS
 	depends on KUNIT
@@ -3334,6 +3472,19 @@ config PRIME_NUMBERS_KUNIT_TEST
 
 	  If unsure, say N
 
+config GLOB_KUNIT_TEST
+	tristate "Glob matching test" if !KUNIT_ALL_TESTS
+	depends on GLOB
+	depends on KUNIT
+	default KUNIT_ALL_TESTS
+	help
+	  Enable this option to test the glob functions at runtime.
+
+	  This test suite verifies the correctness of glob_match() across various
+	  scenarios, including edge cases.
+
+	  If unsure, say N
+
 endif # RUNTIME_TESTING_MENU
 
 config ARCH_USE_MEMTEST
diff --git a/lib/Kconfig.ubsan b/lib/Kconfig.ubsan
index 744121178815..1ecaae7064d2 100644
--- a/lib/Kconfig.ubsan
+++ b/lib/Kconfig.ubsan
@@ -104,7 +104,7 @@ config UBSAN_DIV_ZERO
 	  This option enables -fsanitize=integer-divide-by-zero which checks
 	  for integer division by zero. This is effectively redundant with the
 	  kernel's existing exception handling, though it can provide greater
-	  debugging information under CONFIG_UBSAN_REPORT_FULL.
+	  debugging information.
 
 config UBSAN_UNREACHABLE
 	bool "Perform checking for unreachable code"
diff --git a/lib/Makefile b/lib/Makefile
index 1ab2c4be3b66..1b9ee167517f 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -50,6 +50,8 @@ lib-$(CONFIG_MIN_HEAP) += min_heap.o
 lib-y	+= kobject.o klist.o
 obj-y	+= lockref.o
 
+CONTEXT_ANALYSIS_rhashtable.o := y
+
 obj-y += bcd.o sort.o parser.o debug_locks.o random32.o \
 	 bust_spinlocks.o kasprintf.o bitmap.o scatterlist.o \
 	 list_sort.o uuid.o iov_iter.o clz_ctz.o \
@@ -75,7 +77,6 @@ obj-$(CONFIG_TEST_UBSAN) += test_ubsan.o
 CFLAGS_test_ubsan.o += $(call cc-disable-warning, unused-but-set-variable)
 UBSAN_SANITIZE_test_ubsan.o := y
 obj-$(CONFIG_TEST_KSTRTOX) += test-kstrtox.o
-obj-$(CONFIG_TEST_MIN_HEAP) += test_min_heap.o
 obj-$(CONFIG_TEST_LKM) += test_module.o
 obj-$(CONFIG_TEST_VMALLOC) += test_vmalloc.o
 obj-$(CONFIG_TEST_RHASHTABLE) += test_rhashtable.o
@@ -89,7 +90,6 @@ ifeq ($(CONFIG_CC_IS_CLANG)$(CONFIG_KASAN),yy)
 GCOV_PROFILE_test_bitmap.o := n
 endif
 
-obj-$(CONFIG_TEST_UUID) += test_uuid.o
 obj-$(CONFIG_TEST_XARRAY) += test_xarray.o
 obj-$(CONFIG_TEST_MAPLE_TREE) += test_maple_tree.o
 obj-$(CONFIG_TEST_PARMAN) += test_parman.o
@@ -127,6 +127,8 @@ obj-$(CONFIG_HAS_IOMEM) += iomap_copy.o devres.o
 obj-$(CONFIG_CHECK_SIGNATURE) += check_signature.o
 obj-$(CONFIG_DEBUG_LOCKING_API_SELFTESTS) += locking-selftest.o
 
+obj-$(CONFIG_GENERIC_CPU_CACHE_MAINTENANCE) += cache_maint.o
+
 lib-y += logic_pio.o
 
 lib-$(CONFIG_INDIRECT_IOMEM) += logic_iomem.o
@@ -224,7 +226,6 @@ obj-$(CONFIG_CLOSURES) += closure.o
 obj-$(CONFIG_DQL) += dynamic_queue_limits.o
 
 obj-$(CONFIG_GLOB) += glob.o
-obj-$(CONFIG_GLOB_SELFTEST) += globtest.o
 
 obj-$(CONFIG_DIMLIB) += dim/
 obj-$(CONFIG_SIGNATURE) += digsig.o
@@ -248,6 +249,7 @@ obj-$(CONFIG_POLYNOMIAL) += polynomial.o
 # Prevent the compiler from calling builtins like memcmp() or bcmp() from this
 # file.
 CFLAGS_stackdepot.o += -fno-builtin
+CONTEXT_ANALYSIS_stackdepot.o := y
 obj-$(CONFIG_STACKDEPOT) += stackdepot.o
 KASAN_SANITIZE_stackdepot.o := n
 # In particular, instrumenting stackdepot.c with KMSAN will result in infinite
@@ -329,4 +331,7 @@ obj-$(CONFIG_GENERIC_LIB_DEVMEM_IS_ALLOWED) += devmem_is_allowed.o
 
 obj-$(CONFIG_FIRMWARE_TABLE) += fw_table.o
 
+CONTEXT_ANALYSIS_test_context-analysis.o := y
+obj-$(CONFIG_CONTEXT_ANALYSIS_TEST) += test_context-analysis.o
+
 subdir-$(CONFIG_FORTIFY_SOURCE) += test_fortify
diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c
index f26456988445..58991ab09d84 100644
--- a/lib/alloc_tag.c
+++ b/lib/alloc_tag.c
@@ -669,8 +669,9 @@ static int __init alloc_mod_tags_mem(void)
 		return -ENOMEM;
 	}
 
-	vm_module_tags->pages = kmalloc_array(get_vm_area_size(vm_module_tags) >> PAGE_SHIFT,
-					sizeof(struct page *), GFP_KERNEL | __GFP_ZERO);
+	vm_module_tags->pages = kmalloc_objs(struct page *,
+					     get_vm_area_size(vm_module_tags) >> PAGE_SHIFT,
+					     GFP_KERNEL | __GFP_ZERO);
 	if (!vm_module_tags->pages) {
 		free_vm_area(vm_module_tags);
 		return -ENOMEM;
@@ -776,31 +777,38 @@ EXPORT_SYMBOL(page_alloc_tagging_ops);
 static int proc_mem_profiling_handler(const struct ctl_table *table, int write,
 				      void *buffer, size_t *lenp, loff_t *ppos)
 {
-	if (!mem_profiling_support && write)
-		return -EINVAL;
+	if (write) {
+		/*
+		 * Call from do_sysctl_args() which is a no-op since the same
+		 * value was already set by setup_early_mem_profiling.
+		 * Return success to avoid warnings from do_sysctl_args().
+		 */
+		if (!current->mm)
+			return 0;
+
+#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
+		/* User can't toggle profiling while debugging */
+		return -EACCES;
+#endif
+		if (!mem_profiling_support)
+			return -EINVAL;
+	}
 
 	return proc_do_static_key(table, write, buffer, lenp, ppos);
 }
 
 
-static struct ctl_table memory_allocation_profiling_sysctls[] = {
+static const struct ctl_table memory_allocation_profiling_sysctls[] = {
 	{
 		.procname	= "mem_profiling",
 		.data		= &mem_alloc_profiling_key,
-#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
-		.mode		= 0444,
-#else
 		.mode		= 0644,
-#endif
 		.proc_handler	= proc_mem_profiling_handler,
 	},
 };
 
 static void __init sysctl_init(void)
 {
-	if (!mem_profiling_support)
-		memory_allocation_profiling_sysctls[0].mode = 0444;
-
 	register_sysctl_init("vm", memory_allocation_profiling_sysctls);
 }
 #else /* CONFIG_SYSCTL */
@@ -845,7 +853,7 @@ static int __init alloc_tag_init(void)
 
 	alloc_tag_cttype = codetag_register_type(&desc);
 	if (IS_ERR(alloc_tag_cttype)) {
-		pr_err("Allocation tags registration failed, errno = %ld\n", PTR_ERR(alloc_tag_cttype));
+		pr_err("Allocation tags registration failed, errno = %pe\n", alloc_tag_cttype);
 		free_mod_tags_mem();
 		shutdown_mem_profiling(true);
 		return PTR_ERR(alloc_tag_cttype);
diff --git a/lib/assoc_array.c b/lib/assoc_array.c
index 388e656ac974..bcc6e0a013eb 100644
--- a/lib/assoc_array.c
+++ b/lib/assoc_array.c
@@ -454,7 +454,7 @@ static bool assoc_array_insert_in_empty_tree(struct assoc_array_edit *edit)
 
 	pr_devel("-->%s()\n", __func__);
 
-	new_n0 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
+	new_n0 = kzalloc_obj(struct assoc_array_node);
 	if (!new_n0)
 		return false;
 
@@ -536,11 +536,11 @@ static bool assoc_array_insert_into_terminal_node(struct assoc_array_edit *edit,
 	 * those now.  We may also need a new shortcut, but we deal with that
 	 * when we need it.
 	 */
-	new_n0 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
+	new_n0 = kzalloc_obj(struct assoc_array_node);
 	if (!new_n0)
 		return false;
 	edit->new_meta[0] = assoc_array_node_to_ptr(new_n0);
-	new_n1 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
+	new_n1 = kzalloc_obj(struct assoc_array_node);
 	if (!new_n1)
 		return false;
 	edit->new_meta[1] = assoc_array_node_to_ptr(new_n1);
@@ -741,7 +741,7 @@ all_leaves_cluster_together:
 	keylen = round_up(diff, ASSOC_ARRAY_KEY_CHUNK_SIZE);
 	keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT;
 
-	new_s0 = kzalloc(struct_size(new_s0, index_key, keylen), GFP_KERNEL);
+	new_s0 = kzalloc_flex(*new_s0, index_key, keylen);
 	if (!new_s0)
 		return false;
 	edit->new_meta[2] = assoc_array_shortcut_to_ptr(new_s0);
@@ -832,7 +832,7 @@ static bool assoc_array_insert_mid_shortcut(struct assoc_array_edit *edit,
 	edit->excised_meta[0] = assoc_array_shortcut_to_ptr(shortcut);
 
 	/* Create a new node now since we're going to need it anyway */
-	new_n0 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
+	new_n0 = kzalloc_obj(struct assoc_array_node);
 	if (!new_n0)
 		return false;
 	edit->new_meta[0] = assoc_array_node_to_ptr(new_n0);
@@ -848,8 +848,7 @@ static bool assoc_array_insert_mid_shortcut(struct assoc_array_edit *edit,
 		keylen = round_up(diff, ASSOC_ARRAY_KEY_CHUNK_SIZE);
 		keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT;
 
-		new_s0 = kzalloc(struct_size(new_s0, index_key, keylen),
-				 GFP_KERNEL);
+		new_s0 = kzalloc_flex(*new_s0, index_key, keylen);
 		if (!new_s0)
 			return false;
 		edit->new_meta[1] = assoc_array_shortcut_to_ptr(new_s0);
@@ -898,8 +897,7 @@ static bool assoc_array_insert_mid_shortcut(struct assoc_array_edit *edit,
 		keylen = round_up(shortcut->skip_to_level, ASSOC_ARRAY_KEY_CHUNK_SIZE);
 		keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT;
 
-		new_s1 = kzalloc(struct_size(new_s1, index_key, keylen),
-				 GFP_KERNEL);
+		new_s1 = kzalloc_flex(*new_s1, index_key, keylen);
 		if (!new_s1)
 			return false;
 		edit->new_meta[2] = assoc_array_shortcut_to_ptr(new_s1);
@@ -977,7 +975,7 @@ struct assoc_array_edit *assoc_array_insert(struct assoc_array *array,
 	 */
 	BUG_ON(assoc_array_ptr_is_meta(object));
 
-	edit = kzalloc(sizeof(struct assoc_array_edit), GFP_KERNEL);
+	edit = kzalloc_obj(struct assoc_array_edit);
 	if (!edit)
 		return ERR_PTR(-ENOMEM);
 	edit->array = array;
@@ -1089,7 +1087,7 @@ struct assoc_array_edit *assoc_array_delete(struct assoc_array *array,
 
 	pr_devel("-->%s()\n", __func__);
 
-	edit = kzalloc(sizeof(struct assoc_array_edit), GFP_KERNEL);
+	edit = kzalloc_obj(struct assoc_array_edit);
 	if (!edit)
 		return ERR_PTR(-ENOMEM);
 	edit->array = array;
@@ -1206,7 +1204,7 @@ found_leaf:
 			node = parent;
 
 			/* Create a new node to collapse into */
-			new_n0 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
+			new_n0 = kzalloc_obj(struct assoc_array_node);
 			if (!new_n0)
 				goto enomem;
 			edit->new_meta[0] = assoc_array_node_to_ptr(new_n0);
@@ -1281,7 +1279,7 @@ struct assoc_array_edit *assoc_array_clear(struct assoc_array *array,
 	if (!array->root)
 		return NULL;
 
-	edit = kzalloc(sizeof(struct assoc_array_edit), GFP_KERNEL);
+	edit = kzalloc_obj(struct assoc_array_edit);
 	if (!edit)
 		return ERR_PTR(-ENOMEM);
 	edit->array = array;
@@ -1469,7 +1467,7 @@ int assoc_array_gc(struct assoc_array *array,
 	if (!array->root)
 		return 0;
 
-	edit = kzalloc(sizeof(struct assoc_array_edit), GFP_KERNEL);
+	edit = kzalloc_obj(struct assoc_array_edit);
 	if (!edit)
 		return -ENOMEM;
 	edit->array = array;
@@ -1490,8 +1488,7 @@ descend:
 		shortcut = assoc_array_ptr_to_shortcut(cursor);
 		keylen = round_up(shortcut->skip_to_level, ASSOC_ARRAY_KEY_CHUNK_SIZE);
 		keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT;
-		new_s = kmalloc(struct_size(new_s, index_key, keylen),
-				GFP_KERNEL);
+		new_s = kmalloc_flex(*new_s, index_key, keylen);
 		if (!new_s)
 			goto enomem;
 		pr_devel("dup shortcut %p -> %p\n", shortcut, new_s);
@@ -1505,7 +1502,7 @@ descend:
 
 	/* Duplicate the node at this position */
 	node = assoc_array_ptr_to_node(cursor);
-	new_n = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
+	new_n = kzalloc_obj(struct assoc_array_node);
 	if (!new_n)
 		goto enomem;
 	pr_devel("dup node %p -> %p\n", node, new_n);
diff --git a/lib/base64.c b/lib/base64.c
index b736a7a431c5..41961a444028 100644
--- a/lib/base64.c
+++ b/lib/base64.c
@@ -1,12 +1,12 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * base64.c - RFC4648-compliant base64 encoding
+ * base64.c - Base64 with support for multiple variants
  *
  * Copyright (c) 2020 Hannes Reinecke, SUSE
  *
  * Based on the base64url routines from fs/crypto/fname.c
- * (which are using the URL-safe base64 encoding),
- * modified to use the standard coding table from RFC4648 section 4.
+ * (which are using the URL-safe Base64 encoding),
+ * modified to support multiple Base64 variants.
  */
 
 #include <linux/kernel.h>
@@ -15,89 +15,170 @@
 #include <linux/string.h>
 #include <linux/base64.h>
 
-static const char base64_table[65] =
-	"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+static const char base64_tables[][65] = {
+	[BASE64_STD] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/",
+	[BASE64_URLSAFE] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_",
+	[BASE64_IMAP] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,",
+};
 
+/*
+ * Initialize the base64 reverse mapping for a single character
+ * This macro maps a character to its corresponding base64 value,
+ * returning -1 if the character is invalid.
+ * char 'A'-'Z' maps to 0-25, 'a'-'z' maps to 26-51, '0'-'9' maps to 52-61,
+ * ch_62 maps to 62, ch_63 maps to 63, and other characters return -1
+ */
+#define INIT_1(v, ch_62, ch_63) \
+	[v] = (v) >= 'A' && (v) <= 'Z' ? (v) - 'A' \
+		: (v) >= 'a' && (v) <= 'z' ? (v) - 'a' + 26 \
+		: (v) >= '0' && (v) <= '9' ? (v) - '0' + 52 \
+		: (v) == (ch_62) ? 62 : (v) == (ch_63) ? 63 : -1
+
+/*
+ * Recursive macros to generate multiple Base64 reverse mapping table entries.
+ * Each macro generates a sequence of entries in the lookup table:
+ * INIT_2 generates 2 entries, INIT_4 generates 4, INIT_8 generates 8, and so on up to INIT_32.
+ */
+#define INIT_2(v, ...) INIT_1(v, __VA_ARGS__), INIT_1((v) + 1, __VA_ARGS__)
+#define INIT_4(v, ...) INIT_2(v, __VA_ARGS__), INIT_2((v) + 2, __VA_ARGS__)
+#define INIT_8(v, ...) INIT_4(v, __VA_ARGS__), INIT_4((v) + 4, __VA_ARGS__)
+#define INIT_16(v, ...) INIT_8(v, __VA_ARGS__), INIT_8((v) + 8, __VA_ARGS__)
+#define INIT_32(v, ...) INIT_16(v, __VA_ARGS__), INIT_16((v) + 16, __VA_ARGS__)
+
+#define BASE64_REV_INIT(ch_62, ch_63) { \
+	[0 ... 0x1f] = -1, \
+	INIT_32(0x20, ch_62, ch_63), \
+	INIT_32(0x40, ch_62, ch_63), \
+	INIT_32(0x60, ch_62, ch_63), \
+	[0x80 ... 0xff] = -1 }
+
+static const s8 base64_rev_maps[][256] = {
+	[BASE64_STD] = BASE64_REV_INIT('+', '/'),
+	[BASE64_URLSAFE] = BASE64_REV_INIT('-', '_'),
+	[BASE64_IMAP] = BASE64_REV_INIT('+', ',')
+};
+
+#undef BASE64_REV_INIT
+#undef INIT_32
+#undef INIT_16
+#undef INIT_8
+#undef INIT_4
+#undef INIT_2
+#undef INIT_1
 /**
- * base64_encode() - base64-encode some binary data
+ * base64_encode() - Base64-encode some binary data
  * @src: the binary data to encode
  * @srclen: the length of @src in bytes
- * @dst: (output) the base64-encoded string.  Not NUL-terminated.
+ * @dst: (output) the Base64-encoded string.  Not NUL-terminated.
+ * @padding: whether to append '=' padding characters
+ * @variant: which base64 variant to use
  *
- * Encodes data using base64 encoding, i.e. the "Base 64 Encoding" specified
- * by RFC 4648, including the  '='-padding.
+ * Encodes data using the selected Base64 variant.
  *
- * Return: the length of the resulting base64-encoded string in bytes.
+ * Return: the length of the resulting Base64-encoded string in bytes.
  */
-int base64_encode(const u8 *src, int srclen, char *dst)
+int base64_encode(const u8 *src, int srclen, char *dst, bool padding, enum base64_variant variant)
 {
 	u32 ac = 0;
-	int bits = 0;
-	int i;
 	char *cp = dst;
+	const char *base64_table = base64_tables[variant];
 
-	for (i = 0; i < srclen; i++) {
-		ac = (ac << 8) | src[i];
-		bits += 8;
-		do {
-			bits -= 6;
-			*cp++ = base64_table[(ac >> bits) & 0x3f];
-		} while (bits >= 6);
-	}
-	if (bits) {
-		*cp++ = base64_table[(ac << (6 - bits)) & 0x3f];
-		bits -= 6;
+	while (srclen >= 3) {
+		ac = src[0] << 16 | src[1] << 8 | src[2];
+		*cp++ = base64_table[ac >> 18];
+		*cp++ = base64_table[(ac >> 12) & 0x3f];
+		*cp++ = base64_table[(ac >> 6) & 0x3f];
+		*cp++ = base64_table[ac & 0x3f];
+
+		src += 3;
+		srclen -= 3;
 	}
-	while (bits < 0) {
-		*cp++ = '=';
-		bits += 2;
+
+	switch (srclen) {
+	case 2:
+		ac = src[0] << 16 | src[1] << 8;
+		*cp++ = base64_table[ac >> 18];
+		*cp++ = base64_table[(ac >> 12) & 0x3f];
+		*cp++ = base64_table[(ac >> 6) & 0x3f];
+		if (padding)
+			*cp++ = '=';
+		break;
+	case 1:
+		ac = src[0] << 16;
+		*cp++ = base64_table[ac >> 18];
+		*cp++ = base64_table[(ac >> 12) & 0x3f];
+		if (padding) {
+			*cp++ = '=';
+			*cp++ = '=';
+		}
+		break;
 	}
 	return cp - dst;
 }
 EXPORT_SYMBOL_GPL(base64_encode);
 
 /**
- * base64_decode() - base64-decode a string
+ * base64_decode() - Base64-decode a string
  * @src: the string to decode.  Doesn't need to be NUL-terminated.
  * @srclen: the length of @src in bytes
  * @dst: (output) the decoded binary data
+ * @padding: whether to append '=' padding characters
+ * @variant: which base64 variant to use
  *
- * Decodes a string using base64 encoding, i.e. the "Base 64 Encoding"
- * specified by RFC 4648, including the  '='-padding.
- *
- * This implementation hasn't been optimized for performance.
+ * Decodes a string using the selected Base64 variant.
  *
  * Return: the length of the resulting decoded binary data in bytes,
- *	   or -1 if the string isn't a valid base64 string.
+ *	   or -1 if the string isn't a valid Base64 string.
  */
-int base64_decode(const char *src, int srclen, u8 *dst)
+int base64_decode(const char *src, int srclen, u8 *dst, bool padding, enum base64_variant variant)
 {
-	u32 ac = 0;
-	int bits = 0;
-	int i;
 	u8 *bp = dst;
+	s8 input[4];
+	s32 val;
+	const u8 *s = (const u8 *)src;
+	const s8 *base64_rev_tables = base64_rev_maps[variant];
 
-	for (i = 0; i < srclen; i++) {
-		const char *p = strchr(base64_table, src[i]);
+	while (srclen >= 4) {
+		input[0] = base64_rev_tables[s[0]];
+		input[1] = base64_rev_tables[s[1]];
+		input[2] = base64_rev_tables[s[2]];
+		input[3] = base64_rev_tables[s[3]];
 
-		if (src[i] == '=') {
-			ac = (ac << 6);
-			bits += 6;
-			if (bits >= 8)
-				bits -= 8;
-			continue;
-		}
-		if (p == NULL || src[i] == 0)
-			return -1;
-		ac = (ac << 6) | (p - base64_table);
-		bits += 6;
-		if (bits >= 8) {
-			bits -= 8;
-			*bp++ = (u8)(ac >> bits);
+		val = input[0] << 18 | input[1] << 12 | input[2] << 6 | input[3];
+
+		if (unlikely(val < 0)) {
+			if (!padding || srclen != 4 || s[3] != '=')
+				return -1;
+			padding = 0;
+			srclen = s[2] == '=' ? 2 : 3;
+			break;
 		}
+
+		*bp++ = val >> 16;
+		*bp++ = val >> 8;
+		*bp++ = val;
+
+		s += 4;
+		srclen -= 4;
 	}
-	if (ac & ((1 << bits) - 1))
+
+	if (likely(!srclen))
+		return bp - dst;
+	if (padding || srclen == 1)
 		return -1;
+
+	val = (base64_rev_tables[s[0]] << 12) | (base64_rev_tables[s[1]] << 6);
+	*bp++ = val >> 10;
+
+	if (srclen == 2) {
+		if (val & 0x800003ff)
+			return -1;
+	} else {
+		val |= base64_rev_tables[s[2]];
+		if (val & 0x80000003)
+			return -1;
+		*bp++ = val >> 2;
+	}
 	return bp - dst;
 }
 EXPORT_SYMBOL_GPL(base64_decode);
diff --git a/lib/bch.c b/lib/bch.c
index 1c0cb07cdfeb..9561c0828802 100644
--- a/lib/bch.c
+++ b/lib/bch.c
@@ -1320,7 +1320,7 @@ struct bch_control *bch_init(int m, int t, unsigned int prim_poly,
 	if (prim_poly == 0)
 		prim_poly = prim_poly_tab[m-min_m];
 
-	bch = kzalloc(sizeof(*bch), GFP_KERNEL);
+	bch = kzalloc_obj(*bch);
 	if (bch == NULL)
 		goto fail;
 
diff --git a/lib/bitmap.c b/lib/bitmap.c
index b97692854966..9dc526507875 100644
--- a/lib/bitmap.c
+++ b/lib/bitmap.c
@@ -355,6 +355,12 @@ unsigned int __bitmap_weight_andnot(const unsigned long *bitmap1,
 }
 EXPORT_SYMBOL(__bitmap_weight_andnot);
 
+unsigned int __bitmap_weighted_or(unsigned long *dst, const unsigned long *bitmap1,
+				  const unsigned long *bitmap2, unsigned int bits)
+{
+	return BITMAP_WEIGHT(({dst[idx] = bitmap1[idx] | bitmap2[idx]; dst[idx]; }), bits);
+}
+
 void __bitmap_set(unsigned long *map, unsigned int start, int len)
 {
 	unsigned long *p = map + BIT_WORD(start);
diff --git a/lib/bootconfig.c b/lib/bootconfig.c
index 81f29c29f47b..e88d0221a826 100644
--- a/lib/bootconfig.c
+++ b/lib/bootconfig.c
@@ -316,7 +316,7 @@ int __init xbc_node_compose_key_after(struct xbc_node *root,
 			       depth ? "." : "");
 		if (ret < 0)
 			return ret;
-		if (ret > size) {
+		if (ret >= size) {
 			size = 0;
 		} else {
 			size -= ret;
@@ -532,9 +532,9 @@ static char *skip_spaces_until_newline(char *p)
 static int __init __xbc_open_brace(char *p)
 {
 	/* Push the last key as open brace */
-	open_brace[brace_index++] = xbc_node_index(last_parent);
 	if (brace_index >= XBC_DEPTH_MAX)
 		return xbc_parse_error("Exceed max depth of braces", p);
+	open_brace[brace_index++] = xbc_node_index(last_parent);
 
 	return 0;
 }
@@ -557,17 +557,13 @@ static int __init __xbc_close_brace(char *p)
 /*
  * Return delimiter or error, no node added. As same as lib/cmdline.c,
  * you can use " around spaces, but can't escape " for value.
+ * *@__v must point real value string. (not including spaces before value.)
  */
 static int __init __xbc_parse_value(char **__v, char **__n)
 {
 	char *p, *v = *__v;
 	int c, quotes = 0;
 
-	v = skip_spaces(v);
-	while (*v == '#') {
-		v = skip_comment(v);
-		v = skip_spaces(v);
-	}
 	if (*v == '"' || *v == '\'') {
 		quotes = *v;
 		v++;
@@ -617,6 +613,13 @@ static int __init xbc_parse_array(char **__v)
 		last_parent = xbc_node_get_child(last_parent);
 
 	do {
+		/* Search the next array value beyond comments and empty lines */
+		next = skip_spaces(*__v);
+		while (*next == '#') {
+			next = skip_comment(next);
+			next = skip_spaces(next);
+		}
+		*__v = next;
 		c = __xbc_parse_value(__v, &next);
 		if (c < 0)
 			return c;
@@ -701,9 +704,17 @@ static int __init xbc_parse_kv(char **k, char *v, int op)
 	if (ret)
 		return ret;
 
-	c = __xbc_parse_value(&v, &next);
-	if (c < 0)
-		return c;
+	v = skip_spaces_until_newline(v);
+	/* If there is a comment, this has an empty value. */
+	if (*v == '#') {
+		next = skip_comment(v);
+		*v = '\0';
+		c = '\n';
+	} else {
+		c = __xbc_parse_value(&v, &next);
+		if (c < 0)
+			return c;
+	}
 
 	child = xbc_node_get_child(last_parent);
 	if (child && xbc_node_is_value(child)) {
@@ -712,7 +723,8 @@ static int __init xbc_parse_kv(char **k, char *v, int op)
 		if (op == ':') {
 			unsigned short nidx = child->next;
 
-			xbc_init_node(child, v, XBC_VALUE);
+			if (xbc_init_node(child, v, XBC_VALUE) < 0)
+				return xbc_parse_error("Failed to override value", v);
 			child->next = nidx;	/* keep subkeys */
 			goto array;
 		}
@@ -791,7 +803,7 @@ static int __init xbc_verify_tree(void)
 
 	/* Brace closing */
 	if (brace_index) {
-		n = &xbc_nodes[open_brace[brace_index]];
+		n = &xbc_nodes[open_brace[brace_index - 1]];
 		return xbc_parse_error("Brace is not closed",
 					xbc_node_get_data(n));
 	}
diff --git a/lib/bucket_locks.c b/lib/bucket_locks.c
index 64b92e1dbace..d29516ca0554 100644
--- a/lib/bucket_locks.c
+++ b/lib/bucket_locks.c
@@ -31,7 +31,7 @@ int __alloc_bucket_spinlocks(spinlock_t **locks, unsigned int *locks_mask,
 	}
 
 	if (sizeof(spinlock_t) != 0) {
-		tlocks = kvmalloc_array(size, sizeof(spinlock_t), gfp);
+		tlocks = kvmalloc_objs(spinlock_t, size, gfp);
 		if (!tlocks)
 			return -ENOMEM;
 		for (i = 0; i < size; i++) {
diff --git a/lib/bug.c b/lib/bug.c
index b1f07459c2ee..623c467a8b76 100644
--- a/lib/bug.c
+++ b/lib/bug.c
@@ -139,6 +139,29 @@ void bug_get_file_line(struct bug_entry *bug, const char **file,
 #endif
 }
 
+static const char *bug_get_format(struct bug_entry *bug)
+{
+	const char *format = NULL;
+#ifdef HAVE_ARCH_BUG_FORMAT
+#ifdef CONFIG_GENERIC_BUG_RELATIVE_POINTERS
+	/*
+	 * Allow an architecture to:
+	 *  - relative encode NULL (difficult vs KASLR);
+	 *  - use a literal 0 (there are no valid objects inside
+	 *    the __bug_table itself to refer to after all);
+	 *  - use an empty string.
+	 */
+	if (bug->format_disp)
+		format = (const char *)&bug->format_disp + bug->format_disp;
+	if (format && format[0] == '\0')
+		format = NULL;
+#else
+	format = bug->format;
+#endif
+#endif
+	return format;
+}
+
 struct bug_entry *find_bug(unsigned long bugaddr)
 {
 	struct bug_entry *bug;
@@ -150,26 +173,55 @@ struct bug_entry *find_bug(unsigned long bugaddr)
 	return module_find_bug(bugaddr);
 }
 
-static enum bug_trap_type __report_bug(unsigned long bugaddr, struct pt_regs *regs)
+__diag_push();
+__diag_ignore(GCC, all, "-Wsuggest-attribute=format",
+	      "Not a valid __printf() conversion candidate.");
+static void __warn_printf(const char *fmt, struct pt_regs *regs)
 {
-	struct bug_entry *bug;
-	const char *file;
-	unsigned line, warning, once, done;
+	if (!fmt)
+		return;
+
+#ifdef HAVE_ARCH_BUG_FORMAT_ARGS
+	if (regs) {
+		struct arch_va_list _args;
+		va_list *args = __warn_args(&_args, regs);
+
+		if (args) {
+			vprintk(fmt, *args);
+			return;
+		}
+	}
+#endif
+
+	printk("%s", fmt);
+}
+__diag_pop();
 
-	if (!is_valid_bugaddr(bugaddr))
-		return BUG_TRAP_TYPE_NONE;
+static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long bugaddr, struct pt_regs *regs)
+{
+	bool warning, once, done, no_cut, has_args;
+	const char *file, *fmt;
+	unsigned line;
+
+	if (!bug) {
+		if (!is_valid_bugaddr(bugaddr))
+			return BUG_TRAP_TYPE_NONE;
 
-	bug = find_bug(bugaddr);
-	if (!bug)
-		return BUG_TRAP_TYPE_NONE;
+		bug = find_bug(bugaddr);
+		if (!bug)
+			return BUG_TRAP_TYPE_NONE;
+	}
 
 	disable_trace_on_warning();
 
 	bug_get_file_line(bug, &file, &line);
+	fmt = bug_get_format(bug);
 
-	warning = (bug->flags & BUGFLAG_WARNING) != 0;
-	once = (bug->flags & BUGFLAG_ONCE) != 0;
-	done = (bug->flags & BUGFLAG_DONE) != 0;
+	warning  = bug->flags & BUGFLAG_WARNING;
+	once     = bug->flags & BUGFLAG_ONCE;
+	done     = bug->flags & BUGFLAG_DONE;
+	no_cut   = bug->flags & BUGFLAG_NO_CUT_HERE;
+	has_args = bug->flags & BUGFLAG_ARGS;
 
 	if (warning && once) {
 		if (done)
@@ -187,8 +239,10 @@ static enum bug_trap_type __report_bug(unsigned long bugaddr, struct pt_regs *re
 	 * "cut here" line now. WARN() issues its own "cut here" before the
 	 * extra debugging message it writes before triggering the handler.
 	 */
-	if ((bug->flags & BUGFLAG_NO_CUT_HERE) == 0)
+	if (!no_cut) {
 		printk(KERN_DEFAULT CUT_HERE);
+		__warn_printf(fmt, has_args ? regs : NULL);
+	}
 
 	if (warning) {
 		/* this is a WARN_ON rather than BUG/BUG_ON */
@@ -206,13 +260,25 @@ static enum bug_trap_type __report_bug(unsigned long bugaddr, struct pt_regs *re
 	return BUG_TRAP_TYPE_BUG;
 }
 
+enum bug_trap_type report_bug_entry(struct bug_entry *bug, struct pt_regs *regs)
+{
+	enum bug_trap_type ret;
+	bool rcu = false;
+
+	rcu = warn_rcu_enter();
+	ret = __report_bug(bug, bug_addr(bug), regs);
+	warn_rcu_exit(rcu);
+
+	return ret;
+}
+
 enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs)
 {
 	enum bug_trap_type ret;
 	bool rcu = false;
 
 	rcu = warn_rcu_enter();
-	ret = __report_bug(bugaddr, regs);
+	ret = __report_bug(NULL, bugaddr, regs);
 	warn_rcu_exit(rcu);
 
 	return ret;
diff --git a/lib/build_OID_registry b/lib/build_OID_registry
index 8267e8d71338..30493ac190c0 100755
--- a/lib/build_OID_registry
+++ b/lib/build_OID_registry
@@ -60,10 +60,12 @@ for (my $i = 0; $i <= $#names; $i++) {
     # Determine the encoded length of this OID
     my $size = $#components;
     for (my $loop = 2; $loop <= $#components; $loop++) {
-	my $c = $components[$loop];
+	$ENV{'BC_LINE_LENGTH'} = "0";
+	my $c = `echo "ibase=10; obase=2; $components[$loop]" | bc`;
+	chomp($c);
 
 	# We will base128 encode the number
-	my $tmp = ($c == 0) ? 0 : int(log($c)/log(2));
+	my $tmp = length($c) - 1;
 	$tmp = int($tmp / 7);
 	$size += $tmp;
     }
@@ -100,16 +102,24 @@ for (my $i = 0; $i <= $#names; $i++) {
     push @octets, $components[0] * 40 + $components[1];
 
     for (my $loop = 2; $loop <= $#components; $loop++) {
-	my $c = $components[$loop];
+	# get the base 2 representation of the component
+	$ENV{'BC_LINE_LENGTH'} = "0";
+	my $c = `echo "ibase=10; obase=2; $components[$loop]" | bc`;
+	chomp($c);
 
-	# Base128 encode the number
-	my $tmp = ($c == 0) ? 0 : int(log($c)/log(2));
+	my $tmp = length($c) - 1;
 	$tmp = int($tmp / 7);
 
-	for (; $tmp > 0; $tmp--) {
-	    push @octets, (($c >> $tmp * 7) & 0x7f) | 0x80;
+	# zero pad upto length multiple of 7
+	$c = substr("0000000", 0, ($tmp + 1) * 7 - length($c)).$c;
+
+	# Base128 encode the number
+	for (my $j = 0; $j < $tmp; $j++) {
+	    my $b = oct("0b".substr($c, $j * 7, 7));
+
+	    push @octets, $b | 0x80;
 	}
-	push @octets, $c & 0x7f;
+	push @octets, oct("0b".substr($c, $tmp * 7, 7));
     }
 
     push @encoded_oids, \@octets;
diff --git a/lib/buildid.c b/lib/buildid.c
index c4b0f376fb34..c4b737640621 100644
--- a/lib/buildid.c
+++ b/lib/buildid.c
@@ -5,33 +5,15 @@
 #include <linux/elf.h>
 #include <linux/kernel.h>
 #include <linux/pagemap.h>
+#include <linux/fs.h>
 #include <linux/secretmem.h>
 
 #define BUILD_ID 3
 
 #define MAX_PHDR_CNT 256
 
-struct freader {
-	void *buf;
-	u32 buf_sz;
-	int err;
-	union {
-		struct {
-			struct file *file;
-			struct folio *folio;
-			void *addr;
-			loff_t folio_off;
-			bool may_fault;
-		};
-		struct {
-			const char *data;
-			u64 data_sz;
-		};
-	};
-};
-
-static void freader_init_from_file(struct freader *r, void *buf, u32 buf_sz,
-				   struct file *file, bool may_fault)
+void freader_init_from_file(struct freader *r, void *buf, u32 buf_sz,
+			    struct file *file, bool may_fault)
 {
 	memset(r, 0, sizeof(*r));
 	r->buf = buf;
@@ -40,7 +22,7 @@ static void freader_init_from_file(struct freader *r, void *buf, u32 buf_sz,
 	r->may_fault = may_fault;
 }
 
-static void freader_init_from_mem(struct freader *r, const char *data, u64 data_sz)
+void freader_init_from_mem(struct freader *r, const char *data, u64 data_sz)
 {
 	memset(r, 0, sizeof(*r));
 	r->data = data;
@@ -65,20 +47,9 @@ static int freader_get_folio(struct freader *r, loff_t file_off)
 
 	freader_put_folio(r);
 
-	/* reject secretmem folios created with memfd_secret() */
-	if (secretmem_mapping(r->file->f_mapping))
-		return -EFAULT;
-
+	/* only use page cache lookup - fail if not already cached */
 	r->folio = filemap_get_folio(r->file->f_mapping, file_off >> PAGE_SHIFT);
 
-	/* if sleeping is allowed, wait for the page, if necessary */
-	if (r->may_fault && (IS_ERR(r->folio) || !folio_test_uptodate(r->folio))) {
-		filemap_invalidate_lock_shared(r->file->f_mapping);
-		r->folio = read_cache_folio(r->file->f_mapping, file_off >> PAGE_SHIFT,
-					    NULL, r->file);
-		filemap_invalidate_unlock_shared(r->file->f_mapping);
-	}
-
 	if (IS_ERR(r->folio) || !folio_test_uptodate(r->folio)) {
 		if (!IS_ERR(r->folio))
 			folio_put(r->folio);
@@ -92,7 +63,7 @@ static int freader_get_folio(struct freader *r, loff_t file_off)
 	return 0;
 }
 
-static const void *freader_fetch(struct freader *r, loff_t file_off, size_t sz)
+const void *freader_fetch(struct freader *r, loff_t file_off, size_t sz)
 {
 	size_t folio_sz;
 
@@ -116,6 +87,24 @@ static const void *freader_fetch(struct freader *r, loff_t file_off, size_t sz)
 		return r->data + file_off;
 	}
 
+	/* reject secretmem folios created with memfd_secret() */
+	if (secretmem_mapping(r->file->f_mapping)) {
+		r->err = -EFAULT;
+		return NULL;
+	}
+
+	/* use __kernel_read() for sleepable context */
+	if (r->may_fault) {
+		ssize_t ret;
+
+		ret = __kernel_read(r->file, r->buf, sz, &file_off);
+		if (ret != sz) {
+			r->err = (ret < 0) ? ret : -EIO;
+			return NULL;
+		}
+		return r->buf;
+	}
+
 	/* fetch or reuse folio for given file offset */
 	r->err = freader_get_folio(r, file_off);
 	if (r->err)
@@ -127,18 +116,21 @@ static const void *freader_fetch(struct freader *r, loff_t file_off, size_t sz)
 	 */
 	folio_sz = folio_size(r->folio);
 	if (file_off + sz > r->folio_off + folio_sz) {
-		int part_sz = r->folio_off + folio_sz - file_off;
-
-		/* copy the part that resides in the current folio */
-		memcpy(r->buf, r->addr + (file_off - r->folio_off), part_sz);
-
-		/* fetch next folio */
-		r->err = freader_get_folio(r, r->folio_off + folio_sz);
-		if (r->err)
-			return NULL;
-
-		/* copy the rest of requested data */
-		memcpy(r->buf + part_sz, r->addr, sz - part_sz);
+		u64 part_sz = r->folio_off + folio_sz - file_off, off;
+
+		memcpy(r->buf, r->addr + file_off - r->folio_off, part_sz);
+		off = part_sz;
+
+		while (off < sz) {
+			/* fetch next folio */
+			r->err = freader_get_folio(r, r->folio_off + folio_sz);
+			if (r->err)
+				return NULL;
+			folio_sz = folio_size(r->folio);
+			part_sz = min_t(u64, sz - off, folio_sz);
+			memcpy(r->buf + off, r->addr, part_sz);
+			off += part_sz;
+		}
 
 		return r->buf;
 	}
@@ -147,7 +139,7 @@ static const void *freader_fetch(struct freader *r, loff_t file_off, size_t sz)
 	return r->addr + (file_off - r->folio_off);
 }
 
-static void freader_cleanup(struct freader *r)
+void freader_cleanup(struct freader *r)
 {
 	if (!r->buf)
 		return; /* non-file-backed mode */
@@ -287,7 +279,7 @@ static int get_build_id_64(struct freader *r, unsigned char *build_id, __u32 *si
 /* enough for Elf64_Ehdr, Elf64_Phdr, and all the smaller requests */
 #define MAX_FREADER_BUF_SZ 64
 
-static int __build_id_parse(struct vm_area_struct *vma, unsigned char *build_id,
+static int __build_id_parse(struct file *file, unsigned char *build_id,
 			    __u32 *size, bool may_fault)
 {
 	const Elf32_Ehdr *ehdr;
@@ -295,11 +287,7 @@ static int __build_id_parse(struct vm_area_struct *vma, unsigned char *build_id,
 	char buf[MAX_FREADER_BUF_SZ];
 	int ret;
 
-	/* only works for page backed storage  */
-	if (!vma->vm_file)
-		return -EINVAL;
-
-	freader_init_from_file(&r, buf, sizeof(buf), vma->vm_file, may_fault);
+	freader_init_from_file(&r, buf, sizeof(buf), file, may_fault);
 
 	/* fetch first 18 bytes of ELF header for checks */
 	ehdr = freader_fetch(&r, 0, offsetofend(Elf32_Ehdr, e_type));
@@ -327,8 +315,8 @@ out:
 	return ret;
 }
 
-/*
- * Parse build ID of ELF file mapped to vma
+/**
+ * build_id_parse_nofault() - Parse build ID of ELF file mapped to vma
  * @vma:      vma object
  * @build_id: buffer to store build id, at least BUILD_ID_SIZE long
  * @size:     returns actual build id size in case of success
@@ -340,11 +328,14 @@ out:
  */
 int build_id_parse_nofault(struct vm_area_struct *vma, unsigned char *build_id, __u32 *size)
 {
-	return __build_id_parse(vma, build_id, size, false /* !may_fault */);
+	if (!vma->vm_file)
+		return -EINVAL;
+
+	return __build_id_parse(vma->vm_file, build_id, size, false /* !may_fault */);
 }
 
-/*
- * Parse build ID of ELF file mapped to VMA
+/**
+ * build_id_parse() - Parse build ID of ELF file mapped to VMA
  * @vma:      vma object
  * @build_id: buffer to store build id, at least BUILD_ID_SIZE long
  * @size:     returns actual build id size in case of success
@@ -356,7 +347,26 @@ int build_id_parse_nofault(struct vm_area_struct *vma, unsigned char *build_id,
  */
 int build_id_parse(struct vm_area_struct *vma, unsigned char *build_id, __u32 *size)
 {
-	return __build_id_parse(vma, build_id, size, true /* may_fault */);
+	if (!vma->vm_file)
+		return -EINVAL;
+
+	return __build_id_parse(vma->vm_file, build_id, size, true /* may_fault */);
+}
+
+/**
+ * build_id_parse_file() - Parse build ID of ELF file
+ * @file:      file object
+ * @build_id: buffer to store build id, at least BUILD_ID_SIZE long
+ * @size:     returns actual build id size in case of success
+ *
+ * Assumes faultable context and can cause page faults to bring in file data
+ * into page cache.
+ *
+ * Return: 0 on success; negative error, otherwise
+ */
+int build_id_parse_file(struct file *file, unsigned char *build_id, __u32 *size)
+{
+	return __build_id_parse(file, build_id, size, true /* may_fault */);
 }
 
 /**
diff --git a/lib/cache_maint.c b/lib/cache_maint.c
new file mode 100644
index 000000000000..9256a9ffc34c
--- /dev/null
+++ b/lib/cache_maint.c
@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Generic support for Memory System Cache Maintenance operations.
+ *
+ * Coherency maintenance drivers register with this simple framework that will
+ * iterate over each registered instance to first kick off invalidation and
+ * then to wait until it is complete.
+ *
+ * If no implementations are registered yet cpu_cache_has_invalidate_memregion()
+ * will return false. If this runs concurrently with unregistration then a
+ * race exists but this is no worse than the case where the operations instance
+ * responsible for a given memory region has not yet registered.
+ */
+#include <linux/cache_coherency.h>
+#include <linux/cleanup.h>
+#include <linux/container_of.h>
+#include <linux/export.h>
+#include <linux/kref.h>
+#include <linux/list.h>
+#include <linux/memregion.h>
+#include <linux/module.h>
+#include <linux/rwsem.h>
+#include <linux/slab.h>
+
+static LIST_HEAD(cache_ops_instance_list);
+static DECLARE_RWSEM(cache_ops_instance_list_lock);
+
+static void __cache_coherency_ops_instance_free(struct kref *kref)
+{
+	struct cache_coherency_ops_inst *cci =
+		container_of(kref, struct cache_coherency_ops_inst, kref);
+	kfree(cci);
+}
+
+void cache_coherency_ops_instance_put(struct cache_coherency_ops_inst *cci)
+{
+	kref_put(&cci->kref, __cache_coherency_ops_instance_free);
+}
+EXPORT_SYMBOL_GPL(cache_coherency_ops_instance_put);
+
+static int cache_inval_one(struct cache_coherency_ops_inst *cci, void *data)
+{
+	if (!cci->ops)
+		return -EINVAL;
+
+	return cci->ops->wbinv(cci, data);
+}
+
+static int cache_inval_done_one(struct cache_coherency_ops_inst *cci)
+{
+	if (!cci->ops)
+		return -EINVAL;
+
+	if (!cci->ops->done)
+		return 0;
+
+	return cci->ops->done(cci);
+}
+
+static int cache_invalidate_memregion(phys_addr_t addr, size_t size)
+{
+	int ret;
+	struct cache_coherency_ops_inst *cci;
+	struct cc_inval_params params = {
+		.addr = addr,
+		.size = size,
+	};
+
+	guard(rwsem_read)(&cache_ops_instance_list_lock);
+	list_for_each_entry(cci, &cache_ops_instance_list, node) {
+		ret = cache_inval_one(cci, &params);
+		if (ret)
+			return ret;
+	}
+	list_for_each_entry(cci, &cache_ops_instance_list, node) {
+		ret = cache_inval_done_one(cci);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+struct cache_coherency_ops_inst *
+_cache_coherency_ops_instance_alloc(const struct cache_coherency_ops *ops,
+				    size_t size)
+{
+	struct cache_coherency_ops_inst *cci;
+
+	if (!ops || !ops->wbinv)
+		return NULL;
+
+	cci = kzalloc(size, GFP_KERNEL);
+	if (!cci)
+		return NULL;
+
+	cci->ops = ops;
+	INIT_LIST_HEAD(&cci->node);
+	kref_init(&cci->kref);
+
+	return cci;
+}
+EXPORT_SYMBOL_NS_GPL(_cache_coherency_ops_instance_alloc, "CACHE_COHERENCY");
+
+int cache_coherency_ops_instance_register(struct cache_coherency_ops_inst *cci)
+{
+	guard(rwsem_write)(&cache_ops_instance_list_lock);
+	list_add(&cci->node, &cache_ops_instance_list);
+
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(cache_coherency_ops_instance_register, "CACHE_COHERENCY");
+
+void cache_coherency_ops_instance_unregister(struct cache_coherency_ops_inst *cci)
+{
+	guard(rwsem_write)(&cache_ops_instance_list_lock);
+	list_del(&cci->node);
+}
+EXPORT_SYMBOL_NS_GPL(cache_coherency_ops_instance_unregister, "CACHE_COHERENCY");
+
+int cpu_cache_invalidate_memregion(phys_addr_t start, size_t len)
+{
+	return cache_invalidate_memregion(start, len);
+}
+EXPORT_SYMBOL_NS_GPL(cpu_cache_invalidate_memregion, "DEVMEM");
+
+/*
+ * Used for optimization / debug purposes only as removal can race
+ *
+ * Machines that do not support invalidation, e.g. VMs, will not have any
+ * operations instance to register and so this will always return false.
+ */
+bool cpu_cache_has_invalidate_memregion(void)
+{
+	guard(rwsem_read)(&cache_ops_instance_list_lock);
+	return !list_empty(&cache_ops_instance_list);
+}
+EXPORT_SYMBOL_NS_GPL(cpu_cache_has_invalidate_memregion, "DEVMEM");
diff --git a/lib/codetag.c b/lib/codetag.c
index 545911cebd25..304667897ad4 100644
--- a/lib/codetag.c
+++ b/lib/codetag.c
@@ -193,7 +193,7 @@ static int codetag_module_init(struct codetag_type *cttype, struct module *mod)
 
 	BUG_ON(range.start > range.stop);
 
-	cmod = kmalloc(sizeof(*cmod), GFP_KERNEL);
+	cmod = kmalloc_obj(*cmod);
 	if (unlikely(!cmod))
 		return -ENOMEM;
 
@@ -383,7 +383,7 @@ codetag_register_type(const struct codetag_type_desc *desc)
 
 	BUG_ON(desc->tag_size <= 0);
 
-	cttype = kzalloc(sizeof(*cttype), GFP_KERNEL);
+	cttype = kzalloc_obj(*cttype);
 	if (unlikely(!cttype))
 		return ERR_PTR(-ENOMEM);
 
diff --git a/lib/cpu_rmap.c b/lib/cpu_rmap.c
index f03d9be3f06b..c86ab6e55d17 100644
--- a/lib/cpu_rmap.c
+++ b/lib/cpu_rmap.c
@@ -309,7 +309,7 @@ EXPORT_SYMBOL(irq_cpu_rmap_remove);
  */
 int irq_cpu_rmap_add(struct cpu_rmap *rmap, int irq)
 {
-	struct irq_glue *glue = kzalloc(sizeof(*glue), GFP_KERNEL);
+	struct irq_glue *glue = kzalloc_obj(*glue);
 	int rc;
 
 	if (!glue)
diff --git a/lib/crc/arm/crc-t10dif.h b/lib/crc/arm/crc-t10dif.h
index 63441de5e3f1..afc0ebf97f19 100644
--- a/lib/crc/arm/crc-t10dif.h
+++ b/lib/crc/arm/crc-t10dif.h
@@ -5,7 +5,6 @@
  * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
  */
 
-#include <asm/neon.h>
 #include <asm/simd.h>
 
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
@@ -19,22 +18,16 @@ asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len,
 
 static inline u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
 {
-	if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) {
+	if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && likely(may_use_simd())) {
 		if (static_branch_likely(&have_pmull)) {
-			if (likely(may_use_simd())) {
-				kernel_neon_begin();
-				crc = crc_t10dif_pmull64(crc, data, length);
-				kernel_neon_end();
-				return crc;
-			}
+			scoped_ksimd()
+				return crc_t10dif_pmull64(crc, data, length);
 		} else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE &&
-			   static_branch_likely(&have_neon) &&
-			   likely(may_use_simd())) {
+			   static_branch_likely(&have_neon)) {
 			u8 buf[16] __aligned(16);
 
-			kernel_neon_begin();
-			crc_t10dif_pmull8(crc, data, length, buf);
-			kernel_neon_end();
+			scoped_ksimd()
+				crc_t10dif_pmull8(crc, data, length, buf);
 
 			return crc_t10dif_generic(0, buf, sizeof(buf));
 		}
diff --git a/lib/crc/arm/crc32.h b/lib/crc/arm/crc32.h
index 7b76f52f6907..f33de6b22cd4 100644
--- a/lib/crc/arm/crc32.h
+++ b/lib/crc/arm/crc32.h
@@ -8,7 +8,6 @@
 #include <linux/cpufeature.h>
 
 #include <asm/hwcap.h>
-#include <asm/neon.h>
 #include <asm/simd.h>
 
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32);
@@ -42,9 +41,8 @@ static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
 			len -= n;
 		}
 		n = round_down(len, 16);
-		kernel_neon_begin();
-		crc = crc32_pmull_le(p, n, crc);
-		kernel_neon_end();
+		scoped_ksimd()
+			crc = crc32_pmull_le(p, n, crc);
 		p += n;
 		len -= n;
 	}
@@ -71,9 +69,8 @@ static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
 			len -= n;
 		}
 		n = round_down(len, 16);
-		kernel_neon_begin();
-		crc = crc32c_pmull_le(p, n, crc);
-		kernel_neon_end();
+		scoped_ksimd()
+			crc = crc32c_pmull_le(p, n, crc);
 		p += n;
 		len -= n;
 	}
diff --git a/lib/crc/arm64/crc-t10dif.h b/lib/crc/arm64/crc-t10dif.h
index f88db2971805..b8338139ed77 100644
--- a/lib/crc/arm64/crc-t10dif.h
+++ b/lib/crc/arm64/crc-t10dif.h
@@ -7,7 +7,6 @@
 
 #include <linux/cpufeature.h>
 
-#include <asm/neon.h>
 #include <asm/simd.h>
 
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_asimd);
@@ -21,22 +20,16 @@ asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len);
 
 static inline u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
 {
-	if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) {
+	if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && likely(may_use_simd())) {
 		if (static_branch_likely(&have_pmull)) {
-			if (likely(may_use_simd())) {
-				kernel_neon_begin();
-				crc = crc_t10dif_pmull_p64(crc, data, length);
-				kernel_neon_end();
-				return crc;
-			}
+			scoped_ksimd()
+				return crc_t10dif_pmull_p64(crc, data, length);
 		} else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE &&
-			   static_branch_likely(&have_asimd) &&
-			   likely(may_use_simd())) {
+			   static_branch_likely(&have_asimd)) {
 			u8 buf[16];
 
-			kernel_neon_begin();
-			crc_t10dif_pmull_p8(crc, data, length, buf);
-			kernel_neon_end();
+			scoped_ksimd()
+				crc_t10dif_pmull_p8(crc, data, length, buf);
 
 			return crc_t10dif_generic(0, buf, sizeof(buf));
 		}
diff --git a/lib/crc/arm64/crc32.h b/lib/crc/arm64/crc32.h
index 31e649cd40a2..1939a5dee477 100644
--- a/lib/crc/arm64/crc32.h
+++ b/lib/crc/arm64/crc32.h
@@ -2,7 +2,6 @@
 
 #include <asm/alternative.h>
 #include <asm/cpufeature.h>
-#include <asm/neon.h>
 #include <asm/simd.h>
 
 // The minimum input length to consider the 4-way interleaved code path
@@ -23,9 +22,8 @@ static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
 
 	if (len >= min_len && cpu_have_named_feature(PMULL) &&
 	    likely(may_use_simd())) {
-		kernel_neon_begin();
-		crc = crc32_le_arm64_4way(crc, p, len);
-		kernel_neon_end();
+		scoped_ksimd()
+			crc = crc32_le_arm64_4way(crc, p, len);
 
 		p += round_down(len, 64);
 		len %= 64;
@@ -44,9 +42,8 @@ static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
 
 	if (len >= min_len && cpu_have_named_feature(PMULL) &&
 	    likely(may_use_simd())) {
-		kernel_neon_begin();
-		crc = crc32c_le_arm64_4way(crc, p, len);
-		kernel_neon_end();
+		scoped_ksimd()
+			crc = crc32c_le_arm64_4way(crc, p, len);
 
 		p += round_down(len, 64);
 		len %= 64;
@@ -65,9 +62,8 @@ static inline u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
 
 	if (len >= min_len && cpu_have_named_feature(PMULL) &&
 	    likely(may_use_simd())) {
-		kernel_neon_begin();
-		crc = crc32_be_arm64_4way(crc, p, len);
-		kernel_neon_end();
+		scoped_ksimd()
+			crc = crc32_be_arm64_4way(crc, p, len);
 
 		p += round_down(len, 64);
 		len %= 64;
diff --git a/lib/crypto/.kunitconfig b/lib/crypto/.kunitconfig
new file mode 100644
index 000000000000..6b2ce28ae509
--- /dev/null
+++ b/lib/crypto/.kunitconfig
@@ -0,0 +1,34 @@
+CONFIG_KUNIT=y
+
+# These kconfig options select all the CONFIG_CRYPTO_LIB_* symbols that have a
+# corresponding KUnit test.  Those symbols cannot be directly enabled here,
+# since they are hidden symbols.
+CONFIG_CRYPTO=y
+CONFIG_CRYPTO_ADIANTUM=y
+CONFIG_CRYPTO_BLAKE2B=y
+CONFIG_CRYPTO_CHACHA20POLY1305=y
+CONFIG_CRYPTO_HCTR2=y
+CONFIG_CRYPTO_MD5=y
+CONFIG_CRYPTO_MLDSA=y
+CONFIG_CRYPTO_SHA1=y
+CONFIG_CRYPTO_SHA256=y
+CONFIG_CRYPTO_SHA512=y
+CONFIG_CRYPTO_SHA3=y
+CONFIG_INET=y
+CONFIG_IPV6=y
+CONFIG_NET=y
+CONFIG_NETDEVICES=y
+CONFIG_WIREGUARD=y
+
+CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST=y
+CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST=y
+CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST=y
+CONFIG_CRYPTO_LIB_MD5_KUNIT_TEST=y
+CONFIG_CRYPTO_LIB_MLDSA_KUNIT_TEST=y
+CONFIG_CRYPTO_LIB_NH_KUNIT_TEST=y
+CONFIG_CRYPTO_LIB_POLY1305_KUNIT_TEST=y
+CONFIG_CRYPTO_LIB_POLYVAL_KUNIT_TEST=y
+CONFIG_CRYPTO_LIB_SHA1_KUNIT_TEST=y
+CONFIG_CRYPTO_LIB_SHA256_KUNIT_TEST=y
+CONFIG_CRYPTO_LIB_SHA512_KUNIT_TEST=y
+CONFIG_CRYPTO_LIB_SHA3_KUNIT_TEST=y
diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
index 16859c6226dd..032f9755f999 100644
--- a/lib/crypto/Kconfig
+++ b/lib/crypto/Kconfig
@@ -11,6 +11,18 @@ config CRYPTO_LIB_UTILS
 config CRYPTO_LIB_AES
 	tristate
 
+config CRYPTO_LIB_AES_ARCH
+	bool
+	depends on CRYPTO_LIB_AES && !UML && !KMSAN
+	default y if ARM
+	default y if ARM64
+	default y if PPC && (SPE || (PPC64 && VSX))
+	default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
+		     RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
+	default y if S390
+	default y if SPARC64
+	default y if X86
+
 config CRYPTO_LIB_AESCFB
 	tristate
 	select CRYPTO_LIB_AES
@@ -28,6 +40,17 @@ config CRYPTO_LIB_ARC4
 config CRYPTO_LIB_GF128MUL
 	tristate
 
+config CRYPTO_LIB_BLAKE2B
+	tristate
+	help
+	  The BLAKE2b library functions.  Select this if your module uses any of
+	  the functions from <crypto/blake2b.h>.
+
+config CRYPTO_LIB_BLAKE2B_ARCH
+	bool
+	depends on CRYPTO_LIB_BLAKE2B && !UML
+	default y if ARM && KERNEL_MODE_NEON
+
 # BLAKE2s support is always built-in, so there's no CRYPTO_LIB_BLAKE2S option.
 
 config CRYPTO_LIB_BLAKE2S_ARCH
@@ -50,7 +73,8 @@ config CRYPTO_LIB_CHACHA_ARCH
 	default y if ARM64 && KERNEL_MODE_NEON
 	default y if MIPS && CPU_MIPS32_R2
 	default y if PPC64 && CPU_LITTLE_ENDIAN && VSX
-	default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
+	default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
+		     RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
 	default y if S390
 	default y if X86_64
 
@@ -89,6 +113,26 @@ config CRYPTO_LIB_MD5_ARCH
 	default y if PPC
 	default y if SPARC64
 
+config CRYPTO_LIB_MLDSA
+	tristate
+	select CRYPTO_LIB_SHA3
+	help
+	  The ML-DSA library functions.  Select this if your module uses any of
+	  the functions from <crypto/mldsa.h>.
+
+config CRYPTO_LIB_NH
+	tristate
+	help
+	  Implementation of the NH almost-universal hash function, specifically
+	  the variant of NH used in Adiantum.
+
+config CRYPTO_LIB_NH_ARCH
+	bool
+	depends on CRYPTO_LIB_NH && !UML && !KMSAN
+	default y if ARM && KERNEL_MODE_NEON
+	default y if ARM64 && KERNEL_MODE_NEON
+	default y if X86_64
+
 config CRYPTO_LIB_POLY1305
 	tristate
 	help
@@ -124,6 +168,18 @@ config CRYPTO_LIB_POLY1305_RSIZE
 	default 9 if ARM || ARM64
 	default 1
 
+config CRYPTO_LIB_POLYVAL
+	tristate
+	help
+	  The POLYVAL library functions.  Select this if your module uses any of
+	  the functions from <crypto/polyval.h>.
+
+config CRYPTO_LIB_POLYVAL_ARCH
+	bool
+	depends on CRYPTO_LIB_POLYVAL && !UML
+	default y if ARM64 && KERNEL_MODE_NEON
+	default y if X86_64
+
 config CRYPTO_LIB_CHACHA20POLY1305
 	tristate
 	select CRYPTO_LIB_CHACHA
@@ -161,7 +217,8 @@ config CRYPTO_LIB_SHA256_ARCH
 	default y if ARM64
 	default y if MIPS && CPU_CAVIUM_OCTEON
 	default y if PPC && SPE
-	default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
+	default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
+		     RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
 	default y if S390
 	default y if SPARC64
 	default y if X86_64
@@ -179,11 +236,25 @@ config CRYPTO_LIB_SHA512_ARCH
 	default y if ARM && !CPU_V7M
 	default y if ARM64
 	default y if MIPS && CPU_CAVIUM_OCTEON
-	default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
+	default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
+		     RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
 	default y if S390
 	default y if SPARC64
 	default y if X86_64
 
+config CRYPTO_LIB_SHA3
+	tristate
+	select CRYPTO_LIB_UTILS
+	help
+	  The SHA3 library functions.  Select this if your module uses any of
+	  the functions from <crypto/sha3.h>.
+
+config CRYPTO_LIB_SHA3_ARCH
+	bool
+	depends on CRYPTO_LIB_SHA3 && !UML
+	default y if ARM64 && KERNEL_MODE_NEON
+	default y if S390
+
 config CRYPTO_LIB_SM3
 	tristate
 
diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
index d2845b214585..dc7a56f7287d 100644
--- a/lib/crypto/Makefile
+++ b/lib/crypto/Makefile
@@ -15,8 +15,50 @@ obj-$(CONFIG_CRYPTO_HASH_INFO)			+= hash_info.o
 obj-$(CONFIG_CRYPTO_LIB_UTILS)			+= libcryptoutils.o
 libcryptoutils-y				:= memneq.o utils.o
 
-obj-$(CONFIG_CRYPTO_LIB_AES)			+= libaes.o
-libaes-y					:= aes.o
+################################################################################
+
+obj-$(CONFIG_CRYPTO_LIB_AES) += libaes.o
+libaes-y := aes.o
+ifeq ($(CONFIG_CRYPTO_LIB_AES_ARCH),y)
+CFLAGS_aes.o += -I$(src)/$(SRCARCH)
+
+libaes-$(CONFIG_ARM) += arm/aes-cipher-core.o
+
+ifeq ($(CONFIG_ARM64),y)
+libaes-y += arm64/aes-cipher-core.o
+libaes-$(CONFIG_KERNEL_MODE_NEON) += arm64/aes-ce-core.o
+endif
+
+ifeq ($(CONFIG_PPC),y)
+ifeq ($(CONFIG_SPE),y)
+libaes-y += powerpc/aes-spe-core.o \
+	    powerpc/aes-spe-keys.o \
+	    powerpc/aes-spe-modes.o \
+	    powerpc/aes-tab-4k.o
+else
+libaes-y += powerpc/aesp8-ppc.o
+aes-perlasm-flavour-y := linux-ppc64
+aes-perlasm-flavour-$(CONFIG_PPC64_ELF_ABI_V2) := linux-ppc64-elfv2
+aes-perlasm-flavour-$(CONFIG_CPU_LITTLE_ENDIAN) := linux-ppc64le
+quiet_cmd_perlasm_aes = PERLASM $@
+      cmd_perlasm_aes = $(PERL) $< $(aes-perlasm-flavour-y) $@
+# Use if_changed instead of cmd, in case the flavour changed.
+$(obj)/powerpc/aesp8-ppc.S: $(src)/powerpc/aesp8-ppc.pl FORCE
+	$(call if_changed,perlasm_aes)
+targets += powerpc/aesp8-ppc.S
+OBJECT_FILES_NON_STANDARD_powerpc/aesp8-ppc.o := y
+endif # !CONFIG_SPE
+endif # CONFIG_PPC
+
+libaes-$(CONFIG_RISCV) += riscv/aes-riscv64-zvkned.o
+libaes-$(CONFIG_SPARC) += sparc/aes_asm.o
+libaes-$(CONFIG_X86) += x86/aes-aesni.o
+endif # CONFIG_CRYPTO_LIB_AES_ARCH
+
+# clean-files must be defined unconditionally
+clean-files += powerpc/aesp8-ppc.S
+
+################################################################################
 
 obj-$(CONFIG_CRYPTO_LIB_AESCFB)			+= libaescfb.o
 libaescfb-y					:= aescfb.o
@@ -31,6 +73,15 @@ obj-$(CONFIG_CRYPTO_LIB_GF128MUL)		+= gf128mul.o
 
 ################################################################################
 
+obj-$(CONFIG_CRYPTO_LIB_BLAKE2B) += libblake2b.o
+libblake2b-y := blake2b.o
+ifeq ($(CONFIG_CRYPTO_LIB_BLAKE2B_ARCH),y)
+CFLAGS_blake2b.o += -I$(src)/$(SRCARCH)
+libblake2b-$(CONFIG_ARM) += arm/blake2b-neon-core.o
+endif # CONFIG_CRYPTO_LIB_BLAKE2B_ARCH
+
+################################################################################
+
 # blake2s is used by the /dev/random driver which is always builtin
 obj-y += blake2s.o
 ifeq ($(CONFIG_CRYPTO_LIB_BLAKE2S_ARCH),y)
@@ -117,6 +168,22 @@ endif # CONFIG_CRYPTO_LIB_MD5_ARCH
 
 ################################################################################
 
+obj-$(CONFIG_CRYPTO_LIB_MLDSA) += libmldsa.o
+libmldsa-y := mldsa.o
+
+################################################################################
+
+obj-$(CONFIG_CRYPTO_LIB_NH) += libnh.o
+libnh-y := nh.o
+ifeq ($(CONFIG_CRYPTO_LIB_NH_ARCH),y)
+CFLAGS_nh.o += -I$(src)/$(SRCARCH)
+libnh-$(CONFIG_ARM) += arm/nh-neon-core.o
+libnh-$(CONFIG_ARM64) += arm64/nh-neon-core.o
+libnh-$(CONFIG_X86) += x86/nh-sse2.o x86/nh-avx2.o
+endif
+
+################################################################################
+
 obj-$(CONFIG_CRYPTO_LIB_POLY1305) += libpoly1305.o
 libpoly1305-y := poly1305.o
 ifeq ($(CONFIG_ARCH_SUPPORTS_INT128),y)
@@ -188,6 +255,16 @@ clean-files += arm/poly1305-core.S \
 
 ################################################################################
 
+obj-$(CONFIG_CRYPTO_LIB_POLYVAL) += libpolyval.o
+libpolyval-y := polyval.o
+ifeq ($(CONFIG_CRYPTO_LIB_POLYVAL_ARCH),y)
+CFLAGS_polyval.o += -I$(src)/$(SRCARCH)
+libpolyval-$(CONFIG_ARM64) += arm64/polyval-ce-core.o
+libpolyval-$(CONFIG_X86) += x86/polyval-pclmul-avx.o
+endif
+
+################################################################################
+
 obj-$(CONFIG_CRYPTO_LIB_SHA1) += libsha1.o
 libsha1-y := sha1.o
 ifeq ($(CONFIG_CRYPTO_LIB_SHA1_ARCH),y)
@@ -268,6 +345,16 @@ endif # CONFIG_CRYPTO_LIB_SHA512_ARCH
 
 ################################################################################
 
+obj-$(CONFIG_CRYPTO_LIB_SHA3) += libsha3.o
+libsha3-y := sha3.o
+
+ifeq ($(CONFIG_CRYPTO_LIB_SHA3_ARCH),y)
+CFLAGS_sha3.o += -I$(src)/$(SRCARCH)
+libsha3-$(CONFIG_ARM64) += arm64/sha3-ce-core.o
+endif # CONFIG_CRYPTO_LIB_SHA3_ARCH
+
+################################################################################
+
 obj-$(CONFIG_MPILIB) += mpi/
 
 obj-$(CONFIG_CRYPTO_SELFTESTS_FULL)		+= simd.o
diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c
index b57fda3460f1..b73e19f1bb95 100644
--- a/lib/crypto/aes.c
+++ b/lib/crypto/aes.c
@@ -1,19 +1,17 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) 2017-2019 Linaro Ltd <ard.biesheuvel@linaro.org>
+ * Copyright 2026 Google LLC
  */
 
 #include <crypto/aes.h>
+#include <linux/cache.h>
 #include <linux/crypto.h>
 #include <linux/export.h>
 #include <linux/module.h>
 #include <linux/unaligned.h>
 
-/*
- * Emit the sbox as volatile const to prevent the compiler from doing
- * constant folding on sbox references involving fixed indexes.
- */
-static volatile const u8 __cacheline_aligned aes_sbox[] = {
+static const u8 ____cacheline_aligned aes_sbox[] = {
 	0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5,
 	0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76,
 	0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0,
@@ -48,7 +46,7 @@ static volatile const u8 __cacheline_aligned aes_sbox[] = {
 	0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16,
 };
 
-static volatile const u8 __cacheline_aligned aes_inv_sbox[] = {
+static const u8 ____cacheline_aligned aes_inv_sbox[] = {
 	0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38,
 	0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb,
 	0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87,
@@ -89,6 +87,110 @@ extern const u8 crypto_aes_inv_sbox[256] __alias(aes_inv_sbox);
 EXPORT_SYMBOL(crypto_aes_sbox);
 EXPORT_SYMBOL(crypto_aes_inv_sbox);
 
+/* aes_enc_tab[i] contains MixColumn([SubByte(i), 0, 0, 0]). */
+const u32 ____cacheline_aligned aes_enc_tab[256] = {
+	0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6, 0x0df2f2ff, 0xbd6b6bd6,
+	0xb16f6fde, 0x54c5c591, 0x50303060, 0x03010102, 0xa96767ce, 0x7d2b2b56,
+	0x19fefee7, 0x62d7d7b5, 0xe6abab4d, 0x9a7676ec, 0x45caca8f, 0x9d82821f,
+	0x40c9c989, 0x877d7dfa, 0x15fafaef, 0xeb5959b2, 0xc947478e, 0x0bf0f0fb,
+	0xecadad41, 0x67d4d4b3, 0xfda2a25f, 0xeaafaf45, 0xbf9c9c23, 0xf7a4a453,
+	0x967272e4, 0x5bc0c09b, 0xc2b7b775, 0x1cfdfde1, 0xae93933d, 0x6a26264c,
+	0x5a36366c, 0x413f3f7e, 0x02f7f7f5, 0x4fcccc83, 0x5c343468, 0xf4a5a551,
+	0x34e5e5d1, 0x08f1f1f9, 0x937171e2, 0x73d8d8ab, 0x53313162, 0x3f15152a,
+	0x0c040408, 0x52c7c795, 0x65232346, 0x5ec3c39d, 0x28181830, 0xa1969637,
+	0x0f05050a, 0xb59a9a2f, 0x0907070e, 0x36121224, 0x9b80801b, 0x3de2e2df,
+	0x26ebebcd, 0x6927274e, 0xcdb2b27f, 0x9f7575ea, 0x1b090912, 0x9e83831d,
+	0x742c2c58, 0x2e1a1a34, 0x2d1b1b36, 0xb26e6edc, 0xee5a5ab4, 0xfba0a05b,
+	0xf65252a4, 0x4d3b3b76, 0x61d6d6b7, 0xceb3b37d, 0x7b292952, 0x3ee3e3dd,
+	0x712f2f5e, 0x97848413, 0xf55353a6, 0x68d1d1b9, 0x00000000, 0x2cededc1,
+	0x60202040, 0x1ffcfce3, 0xc8b1b179, 0xed5b5bb6, 0xbe6a6ad4, 0x46cbcb8d,
+	0xd9bebe67, 0x4b393972, 0xde4a4a94, 0xd44c4c98, 0xe85858b0, 0x4acfcf85,
+	0x6bd0d0bb, 0x2aefefc5, 0xe5aaaa4f, 0x16fbfbed, 0xc5434386, 0xd74d4d9a,
+	0x55333366, 0x94858511, 0xcf45458a, 0x10f9f9e9, 0x06020204, 0x817f7ffe,
+	0xf05050a0, 0x443c3c78, 0xba9f9f25, 0xe3a8a84b, 0xf35151a2, 0xfea3a35d,
+	0xc0404080, 0x8a8f8f05, 0xad92923f, 0xbc9d9d21, 0x48383870, 0x04f5f5f1,
+	0xdfbcbc63, 0xc1b6b677, 0x75dadaaf, 0x63212142, 0x30101020, 0x1affffe5,
+	0x0ef3f3fd, 0x6dd2d2bf, 0x4ccdcd81, 0x140c0c18, 0x35131326, 0x2fececc3,
+	0xe15f5fbe, 0xa2979735, 0xcc444488, 0x3917172e, 0x57c4c493, 0xf2a7a755,
+	0x827e7efc, 0x473d3d7a, 0xac6464c8, 0xe75d5dba, 0x2b191932, 0x957373e6,
+	0xa06060c0, 0x98818119, 0xd14f4f9e, 0x7fdcdca3, 0x66222244, 0x7e2a2a54,
+	0xab90903b, 0x8388880b, 0xca46468c, 0x29eeeec7, 0xd3b8b86b, 0x3c141428,
+	0x79dedea7, 0xe25e5ebc, 0x1d0b0b16, 0x76dbdbad, 0x3be0e0db, 0x56323264,
+	0x4e3a3a74, 0x1e0a0a14, 0xdb494992, 0x0a06060c, 0x6c242448, 0xe45c5cb8,
+	0x5dc2c29f, 0x6ed3d3bd, 0xefacac43, 0xa66262c4, 0xa8919139, 0xa4959531,
+	0x37e4e4d3, 0x8b7979f2, 0x32e7e7d5, 0x43c8c88b, 0x5937376e, 0xb76d6dda,
+	0x8c8d8d01, 0x64d5d5b1, 0xd24e4e9c, 0xe0a9a949, 0xb46c6cd8, 0xfa5656ac,
+	0x07f4f4f3, 0x25eaeacf, 0xaf6565ca, 0x8e7a7af4, 0xe9aeae47, 0x18080810,
+	0xd5baba6f, 0x887878f0, 0x6f25254a, 0x722e2e5c, 0x241c1c38, 0xf1a6a657,
+	0xc7b4b473, 0x51c6c697, 0x23e8e8cb, 0x7cdddda1, 0x9c7474e8, 0x211f1f3e,
+	0xdd4b4b96, 0xdcbdbd61, 0x868b8b0d, 0x858a8a0f, 0x907070e0, 0x423e3e7c,
+	0xc4b5b571, 0xaa6666cc, 0xd8484890, 0x05030306, 0x01f6f6f7, 0x120e0e1c,
+	0xa36161c2, 0x5f35356a, 0xf95757ae, 0xd0b9b969, 0x91868617, 0x58c1c199,
+	0x271d1d3a, 0xb99e9e27, 0x38e1e1d9, 0x13f8f8eb, 0xb398982b, 0x33111122,
+	0xbb6969d2, 0x70d9d9a9, 0x898e8e07, 0xa7949433, 0xb69b9b2d, 0x221e1e3c,
+	0x92878715, 0x20e9e9c9, 0x49cece87, 0xff5555aa, 0x78282850, 0x7adfdfa5,
+	0x8f8c8c03, 0xf8a1a159, 0x80898909, 0x170d0d1a, 0xdabfbf65, 0x31e6e6d7,
+	0xc6424284, 0xb86868d0, 0xc3414182, 0xb0999929, 0x772d2d5a, 0x110f0f1e,
+	0xcbb0b07b, 0xfc5454a8, 0xd6bbbb6d, 0x3a16162c,
+};
+EXPORT_SYMBOL(aes_enc_tab);
+
+/* aes_dec_tab[i] contains InvMixColumn([InvSubByte(i), 0, 0, 0]). */
+const u32 ____cacheline_aligned aes_dec_tab[256] = {
+	0x50a7f451, 0x5365417e, 0xc3a4171a, 0x965e273a, 0xcb6bab3b, 0xf1459d1f,
+	0xab58faac, 0x9303e34b, 0x55fa3020, 0xf66d76ad, 0x9176cc88, 0x254c02f5,
+	0xfcd7e54f, 0xd7cb2ac5, 0x80443526, 0x8fa362b5, 0x495ab1de, 0x671bba25,
+	0x980eea45, 0xe1c0fe5d, 0x02752fc3, 0x12f04c81, 0xa397468d, 0xc6f9d36b,
+	0xe75f8f03, 0x959c9215, 0xeb7a6dbf, 0xda595295, 0x2d83bed4, 0xd3217458,
+	0x2969e049, 0x44c8c98e, 0x6a89c275, 0x78798ef4, 0x6b3e5899, 0xdd71b927,
+	0xb64fe1be, 0x17ad88f0, 0x66ac20c9, 0xb43ace7d, 0x184adf63, 0x82311ae5,
+	0x60335197, 0x457f5362, 0xe07764b1, 0x84ae6bbb, 0x1ca081fe, 0x942b08f9,
+	0x58684870, 0x19fd458f, 0x876cde94, 0xb7f87b52, 0x23d373ab, 0xe2024b72,
+	0x578f1fe3, 0x2aab5566, 0x0728ebb2, 0x03c2b52f, 0x9a7bc586, 0xa50837d3,
+	0xf2872830, 0xb2a5bf23, 0xba6a0302, 0x5c8216ed, 0x2b1ccf8a, 0x92b479a7,
+	0xf0f207f3, 0xa1e2694e, 0xcdf4da65, 0xd5be0506, 0x1f6234d1, 0x8afea6c4,
+	0x9d532e34, 0xa055f3a2, 0x32e18a05, 0x75ebf6a4, 0x39ec830b, 0xaaef6040,
+	0x069f715e, 0x51106ebd, 0xf98a213e, 0x3d06dd96, 0xae053edd, 0x46bde64d,
+	0xb58d5491, 0x055dc471, 0x6fd40604, 0xff155060, 0x24fb9819, 0x97e9bdd6,
+	0xcc434089, 0x779ed967, 0xbd42e8b0, 0x888b8907, 0x385b19e7, 0xdbeec879,
+	0x470a7ca1, 0xe90f427c, 0xc91e84f8, 0x00000000, 0x83868009, 0x48ed2b32,
+	0xac70111e, 0x4e725a6c, 0xfbff0efd, 0x5638850f, 0x1ed5ae3d, 0x27392d36,
+	0x64d90f0a, 0x21a65c68, 0xd1545b9b, 0x3a2e3624, 0xb1670a0c, 0x0fe75793,
+	0xd296eeb4, 0x9e919b1b, 0x4fc5c080, 0xa220dc61, 0x694b775a, 0x161a121c,
+	0x0aba93e2, 0xe52aa0c0, 0x43e0223c, 0x1d171b12, 0x0b0d090e, 0xadc78bf2,
+	0xb9a8b62d, 0xc8a91e14, 0x8519f157, 0x4c0775af, 0xbbdd99ee, 0xfd607fa3,
+	0x9f2601f7, 0xbcf5725c, 0xc53b6644, 0x347efb5b, 0x7629438b, 0xdcc623cb,
+	0x68fcedb6, 0x63f1e4b8, 0xcadc31d7, 0x10856342, 0x40229713, 0x2011c684,
+	0x7d244a85, 0xf83dbbd2, 0x1132f9ae, 0x6da129c7, 0x4b2f9e1d, 0xf330b2dc,
+	0xec52860d, 0xd0e3c177, 0x6c16b32b, 0x99b970a9, 0xfa489411, 0x2264e947,
+	0xc48cfca8, 0x1a3ff0a0, 0xd82c7d56, 0xef903322, 0xc74e4987, 0xc1d138d9,
+	0xfea2ca8c, 0x360bd498, 0xcf81f5a6, 0x28de7aa5, 0x268eb7da, 0xa4bfad3f,
+	0xe49d3a2c, 0x0d927850, 0x9bcc5f6a, 0x62467e54, 0xc2138df6, 0xe8b8d890,
+	0x5ef7392e, 0xf5afc382, 0xbe805d9f, 0x7c93d069, 0xa92dd56f, 0xb31225cf,
+	0x3b99acc8, 0xa77d1810, 0x6e639ce8, 0x7bbb3bdb, 0x097826cd, 0xf418596e,
+	0x01b79aec, 0xa89a4f83, 0x656e95e6, 0x7ee6ffaa, 0x08cfbc21, 0xe6e815ef,
+	0xd99be7ba, 0xce366f4a, 0xd4099fea, 0xd67cb029, 0xafb2a431, 0x31233f2a,
+	0x3094a5c6, 0xc066a235, 0x37bc4e74, 0xa6ca82fc, 0xb0d090e0, 0x15d8a733,
+	0x4a9804f1, 0xf7daec41, 0x0e50cd7f, 0x2ff69117, 0x8dd64d76, 0x4db0ef43,
+	0x544daacc, 0xdf0496e4, 0xe3b5d19e, 0x1b886a4c, 0xb81f2cc1, 0x7f516546,
+	0x04ea5e9d, 0x5d358c01, 0x737487fa, 0x2e410bfb, 0x5a1d67b3, 0x52d2db92,
+	0x335610e9, 0x1347d66d, 0x8c61d79a, 0x7a0ca137, 0x8e14f859, 0x893c13eb,
+	0xee27a9ce, 0x35c961b7, 0xede51ce1, 0x3cb1477a, 0x59dfd29c, 0x3f73f255,
+	0x79ce1418, 0xbf37c773, 0xeacdf753, 0x5baafd5f, 0x146f3ddf, 0x86db4478,
+	0x81f3afca, 0x3ec468b9, 0x2c342438, 0x5f40a3c2, 0x72c31d16, 0x0c25e2bc,
+	0x8b493c28, 0x41950dff, 0x7101a839, 0xdeb30c08, 0x9ce4b4d8, 0x90c15664,
+	0x6184cb7b, 0x70b632d5, 0x745c6c48, 0x4257b8d0,
+};
+EXPORT_SYMBOL(aes_dec_tab);
+
+/* Prefetch data into L1 cache.  @mem should be cacheline-aligned. */
+static __always_inline void aes_prefetch(const void *mem, size_t len)
+{
+	for (size_t i = 0; i < len; i += L1_CACHE_BYTES)
+		*(volatile const u8 *)(mem + i);
+	barrier();
+}
+
 static u32 mul_by_x(u32 w)
 {
 	u32 x = w & 0x7f7f7f7f;
@@ -145,22 +247,6 @@ static u32 inv_mix_columns(u32 x)
 	return mix_columns(x ^ y ^ ror32(y, 16));
 }
 
-static __always_inline u32 subshift(u32 in[], int pos)
-{
-	return (aes_sbox[in[pos] & 0xff]) ^
-	       (aes_sbox[(in[(pos + 1) % 4] >>  8) & 0xff] <<  8) ^
-	       (aes_sbox[(in[(pos + 2) % 4] >> 16) & 0xff] << 16) ^
-	       (aes_sbox[(in[(pos + 3) % 4] >> 24) & 0xff] << 24);
-}
-
-static __always_inline u32 inv_subshift(u32 in[], int pos)
-{
-	return (aes_inv_sbox[in[pos] & 0xff]) ^
-	       (aes_inv_sbox[(in[(pos + 3) % 4] >>  8) & 0xff] <<  8) ^
-	       (aes_inv_sbox[(in[(pos + 2) % 4] >> 16) & 0xff] << 16) ^
-	       (aes_inv_sbox[(in[(pos + 1) % 4] >> 24) & 0xff] << 24);
-}
-
 static u32 subw(u32 in)
 {
 	return (aes_sbox[in & 0xff]) ^
@@ -169,38 +255,17 @@ static u32 subw(u32 in)
 	       (aes_sbox[(in >> 24) & 0xff] << 24);
 }
 
-/**
- * aes_expandkey - Expands the AES key as described in FIPS-197
- * @ctx:	The location where the computed key will be stored.
- * @in_key:	The supplied key.
- * @key_len:	The length of the supplied key.
- *
- * Returns 0 on success. The function fails only if an invalid key size (or
- * pointer) is supplied.
- * The expanded key size is 240 bytes (max of 14 rounds with a unique 16 bytes
- * key schedule plus a 16 bytes key which is used before the first round).
- * The decryption key is prepared for the "Equivalent Inverse Cipher" as
- * described in FIPS-197. The first slot (16 bytes) of each key (enc or dec) is
- * for the initial combination, the second slot for the first round and so on.
- */
-int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
-		  unsigned int key_len)
+static void aes_expandkey_generic(u32 rndkeys[], u32 *inv_rndkeys,
+				  const u8 *in_key, int key_len)
 {
 	u32 kwords = key_len / sizeof(u32);
 	u32 rc, i, j;
-	int err;
-
-	err = aes_check_keylen(key_len);
-	if (err)
-		return err;
-
-	ctx->key_length = key_len;
 
 	for (i = 0; i < kwords; i++)
-		ctx->key_enc[i] = get_unaligned_le32(in_key + i * sizeof(u32));
+		rndkeys[i] = get_unaligned_le32(&in_key[i * sizeof(u32)]);
 
 	for (i = 0, rc = 1; i < 10; i++, rc = mul_by_x(rc)) {
-		u32 *rki = ctx->key_enc + (i * kwords);
+		u32 *rki = &rndkeys[i * kwords];
 		u32 *rko = rki + kwords;
 
 		rko[0] = ror32(subw(rki[kwords - 1]), 8) ^ rc ^ rki[0];
@@ -229,129 +294,239 @@ int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
 	 * the Inverse Mix Columns transformation to all but the first and
 	 * the last one.
 	 */
-	ctx->key_dec[0] = ctx->key_enc[key_len + 24];
-	ctx->key_dec[1] = ctx->key_enc[key_len + 25];
-	ctx->key_dec[2] = ctx->key_enc[key_len + 26];
-	ctx->key_dec[3] = ctx->key_enc[key_len + 27];
-
-	for (i = 4, j = key_len + 20; j > 0; i += 4, j -= 4) {
-		ctx->key_dec[i]     = inv_mix_columns(ctx->key_enc[j]);
-		ctx->key_dec[i + 1] = inv_mix_columns(ctx->key_enc[j + 1]);
-		ctx->key_dec[i + 2] = inv_mix_columns(ctx->key_enc[j + 2]);
-		ctx->key_dec[i + 3] = inv_mix_columns(ctx->key_enc[j + 3]);
-	}
+	if (inv_rndkeys) {
+		inv_rndkeys[0] = rndkeys[key_len + 24];
+		inv_rndkeys[1] = rndkeys[key_len + 25];
+		inv_rndkeys[2] = rndkeys[key_len + 26];
+		inv_rndkeys[3] = rndkeys[key_len + 27];
+
+		for (i = 4, j = key_len + 20; j > 0; i += 4, j -= 4) {
+			inv_rndkeys[i]     = inv_mix_columns(rndkeys[j]);
+			inv_rndkeys[i + 1] = inv_mix_columns(rndkeys[j + 1]);
+			inv_rndkeys[i + 2] = inv_mix_columns(rndkeys[j + 2]);
+			inv_rndkeys[i + 3] = inv_mix_columns(rndkeys[j + 3]);
+		}
 
-	ctx->key_dec[i]     = ctx->key_enc[0];
-	ctx->key_dec[i + 1] = ctx->key_enc[1];
-	ctx->key_dec[i + 2] = ctx->key_enc[2];
-	ctx->key_dec[i + 3] = ctx->key_enc[3];
+		inv_rndkeys[i]     = rndkeys[0];
+		inv_rndkeys[i + 1] = rndkeys[1];
+		inv_rndkeys[i + 2] = rndkeys[2];
+		inv_rndkeys[i + 3] = rndkeys[3];
+	}
+}
 
+int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
+		  unsigned int key_len)
+{
+	if (aes_check_keylen(key_len) != 0)
+		return -EINVAL;
+	ctx->key_length = key_len;
+	aes_expandkey_generic(ctx->key_enc, ctx->key_dec, in_key, key_len);
 	return 0;
 }
 EXPORT_SYMBOL(aes_expandkey);
 
-/**
- * aes_encrypt - Encrypt a single AES block
- * @ctx:	Context struct containing the key schedule
- * @out:	Buffer to store the ciphertext
- * @in:		Buffer containing the plaintext
- */
-void aes_encrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in)
+static __always_inline u32 enc_quarterround(const u32 w[4], int i, u32 rk)
 {
-	const u32 *rkp = ctx->key_enc + 4;
-	int rounds = 6 + ctx->key_length / 4;
-	u32 st0[4], st1[4];
-	int round;
+	return rk ^ aes_enc_tab[(u8)w[i]] ^
+	       rol32(aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)], 8) ^
+	       rol32(aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^
+	       rol32(aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)], 24);
+}
+
+static __always_inline u32 enclast_quarterround(const u32 w[4], int i, u32 rk)
+{
+	return rk ^ ((aes_enc_tab[(u8)w[i]] & 0x0000ff00) >> 8) ^
+	       (aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)] & 0x0000ff00) ^
+	       ((aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)] & 0x0000ff00) << 8) ^
+	       ((aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)] & 0x0000ff00) << 16);
+}
+
+static void __maybe_unused aes_encrypt_generic(const u32 rndkeys[], int nrounds,
+					       u8 out[AES_BLOCK_SIZE],
+					       const u8 in[AES_BLOCK_SIZE])
+{
+	const u32 *rkp = rndkeys;
+	int n = nrounds - 1;
+	u32 w[4];
 
-	st0[0] = ctx->key_enc[0] ^ get_unaligned_le32(in);
-	st0[1] = ctx->key_enc[1] ^ get_unaligned_le32(in + 4);
-	st0[2] = ctx->key_enc[2] ^ get_unaligned_le32(in + 8);
-	st0[3] = ctx->key_enc[3] ^ get_unaligned_le32(in + 12);
+	w[0] = get_unaligned_le32(&in[0]) ^ *rkp++;
+	w[1] = get_unaligned_le32(&in[4]) ^ *rkp++;
+	w[2] = get_unaligned_le32(&in[8]) ^ *rkp++;
+	w[3] = get_unaligned_le32(&in[12]) ^ *rkp++;
 
 	/*
-	 * Force the compiler to emit data independent Sbox references,
-	 * by xoring the input with Sbox values that are known to add up
-	 * to zero. This pulls the entire Sbox into the D-cache before any
-	 * data dependent lookups are done.
+	 * Prefetch the table before doing data and key-dependent loads from it.
+	 *
+	 * This is intended only as a basic constant-time hardening measure that
+	 * avoids interfering with performance too much.  Its effectiveness is
+	 * not guaranteed.  For proper constant-time AES, a CPU that supports
+	 * AES instructions should be used instead.
 	 */
-	st0[0] ^= aes_sbox[ 0] ^ aes_sbox[ 64] ^ aes_sbox[134] ^ aes_sbox[195];
-	st0[1] ^= aes_sbox[16] ^ aes_sbox[ 82] ^ aes_sbox[158] ^ aes_sbox[221];
-	st0[2] ^= aes_sbox[32] ^ aes_sbox[ 96] ^ aes_sbox[160] ^ aes_sbox[234];
-	st0[3] ^= aes_sbox[48] ^ aes_sbox[112] ^ aes_sbox[186] ^ aes_sbox[241];
-
-	for (round = 0;; round += 2, rkp += 8) {
-		st1[0] = mix_columns(subshift(st0, 0)) ^ rkp[0];
-		st1[1] = mix_columns(subshift(st0, 1)) ^ rkp[1];
-		st1[2] = mix_columns(subshift(st0, 2)) ^ rkp[2];
-		st1[3] = mix_columns(subshift(st0, 3)) ^ rkp[3];
-
-		if (round == rounds - 2)
-			break;
-
-		st0[0] = mix_columns(subshift(st1, 0)) ^ rkp[4];
-		st0[1] = mix_columns(subshift(st1, 1)) ^ rkp[5];
-		st0[2] = mix_columns(subshift(st1, 2)) ^ rkp[6];
-		st0[3] = mix_columns(subshift(st1, 3)) ^ rkp[7];
-	}
+	aes_prefetch(aes_enc_tab, sizeof(aes_enc_tab));
+
+	do {
+		u32 w0 = enc_quarterround(w, 0, *rkp++);
+		u32 w1 = enc_quarterround(w, 1, *rkp++);
+		u32 w2 = enc_quarterround(w, 2, *rkp++);
+		u32 w3 = enc_quarterround(w, 3, *rkp++);
+
+		w[0] = w0;
+		w[1] = w1;
+		w[2] = w2;
+		w[3] = w3;
+	} while (--n);
+
+	put_unaligned_le32(enclast_quarterround(w, 0, *rkp++), &out[0]);
+	put_unaligned_le32(enclast_quarterround(w, 1, *rkp++), &out[4]);
+	put_unaligned_le32(enclast_quarterround(w, 2, *rkp++), &out[8]);
+	put_unaligned_le32(enclast_quarterround(w, 3, *rkp++), &out[12]);
+}
 
-	put_unaligned_le32(subshift(st1, 0) ^ rkp[4], out);
-	put_unaligned_le32(subshift(st1, 1) ^ rkp[5], out + 4);
-	put_unaligned_le32(subshift(st1, 2) ^ rkp[6], out + 8);
-	put_unaligned_le32(subshift(st1, 3) ^ rkp[7], out + 12);
+static __always_inline u32 dec_quarterround(const u32 w[4], int i, u32 rk)
+{
+	return rk ^ aes_dec_tab[(u8)w[i]] ^
+	       rol32(aes_dec_tab[(u8)(w[(i + 3) % 4] >> 8)], 8) ^
+	       rol32(aes_dec_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^
+	       rol32(aes_dec_tab[(u8)(w[(i + 1) % 4] >> 24)], 24);
+}
+
+static __always_inline u32 declast_quarterround(const u32 w[4], int i, u32 rk)
+{
+	return rk ^ aes_inv_sbox[(u8)w[i]] ^
+	       ((u32)aes_inv_sbox[(u8)(w[(i + 3) % 4] >> 8)] << 8) ^
+	       ((u32)aes_inv_sbox[(u8)(w[(i + 2) % 4] >> 16)] << 16) ^
+	       ((u32)aes_inv_sbox[(u8)(w[(i + 1) % 4] >> 24)] << 24);
+}
+
+static void __maybe_unused aes_decrypt_generic(const u32 inv_rndkeys[],
+					       int nrounds,
+					       u8 out[AES_BLOCK_SIZE],
+					       const u8 in[AES_BLOCK_SIZE])
+{
+	const u32 *rkp = inv_rndkeys;
+	int n = nrounds - 1;
+	u32 w[4];
+
+	w[0] = get_unaligned_le32(&in[0]) ^ *rkp++;
+	w[1] = get_unaligned_le32(&in[4]) ^ *rkp++;
+	w[2] = get_unaligned_le32(&in[8]) ^ *rkp++;
+	w[3] = get_unaligned_le32(&in[12]) ^ *rkp++;
+
+	aes_prefetch(aes_dec_tab, sizeof(aes_dec_tab));
+
+	do {
+		u32 w0 = dec_quarterround(w, 0, *rkp++);
+		u32 w1 = dec_quarterround(w, 1, *rkp++);
+		u32 w2 = dec_quarterround(w, 2, *rkp++);
+		u32 w3 = dec_quarterround(w, 3, *rkp++);
+
+		w[0] = w0;
+		w[1] = w1;
+		w[2] = w2;
+		w[3] = w3;
+	} while (--n);
+
+	aes_prefetch(aes_inv_sbox, sizeof(aes_inv_sbox));
+	put_unaligned_le32(declast_quarterround(w, 0, *rkp++), &out[0]);
+	put_unaligned_le32(declast_quarterround(w, 1, *rkp++), &out[4]);
+	put_unaligned_le32(declast_quarterround(w, 2, *rkp++), &out[8]);
+	put_unaligned_le32(declast_quarterround(w, 3, *rkp++), &out[12]);
 }
-EXPORT_SYMBOL(aes_encrypt);
 
-/**
- * aes_decrypt - Decrypt a single AES block
- * @ctx:	Context struct containing the key schedule
- * @out:	Buffer to store the plaintext
- * @in:		Buffer containing the ciphertext
+/*
+ * Note: the aes_prepare*key_* names reflect the fact that the implementation
+ * might not actually expand the key.  (The s390 code for example doesn't.)
+ * Where the key is expanded we use the more specific names aes_expandkey_*.
+ *
+ * aes_preparekey_arch() is passed an optional pointer 'inv_k' which points to
+ * the area to store the prepared decryption key.  It will be NULL if the user
+ * is requesting encryption-only.  aes_preparekey_arch() is also passed a valid
+ * 'key_len' and 'nrounds', corresponding to AES-128, AES-192, or AES-256.
  */
-void aes_decrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in)
+#ifdef CONFIG_CRYPTO_LIB_AES_ARCH
+/* An arch-specific implementation of AES is available.  Include it. */
+#include "aes.h" /* $(SRCARCH)/aes.h */
+#else
+/* No arch-specific implementation of AES is available.  Use generic code. */
+
+static void aes_preparekey_arch(union aes_enckey_arch *k,
+				union aes_invkey_arch *inv_k,
+				const u8 *in_key, int key_len, int nrounds)
 {
-	const u32 *rkp = ctx->key_dec + 4;
-	int rounds = 6 + ctx->key_length / 4;
-	u32 st0[4], st1[4];
-	int round;
+	aes_expandkey_generic(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL,
+			      in_key, key_len);
+}
 
-	st0[0] = ctx->key_dec[0] ^ get_unaligned_le32(in);
-	st0[1] = ctx->key_dec[1] ^ get_unaligned_le32(in + 4);
-	st0[2] = ctx->key_dec[2] ^ get_unaligned_le32(in + 8);
-	st0[3] = ctx->key_dec[3] ^ get_unaligned_le32(in + 12);
+static void aes_encrypt_arch(const struct aes_enckey *key,
+			     u8 out[AES_BLOCK_SIZE],
+			     const u8 in[AES_BLOCK_SIZE])
+{
+	aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in);
+}
 
-	/*
-	 * Force the compiler to emit data independent Sbox references,
-	 * by xoring the input with Sbox values that are known to add up
-	 * to zero. This pulls the entire Sbox into the D-cache before any
-	 * data dependent lookups are done.
-	 */
-	st0[0] ^= aes_inv_sbox[ 0] ^ aes_inv_sbox[ 64] ^ aes_inv_sbox[129] ^ aes_inv_sbox[200];
-	st0[1] ^= aes_inv_sbox[16] ^ aes_inv_sbox[ 83] ^ aes_inv_sbox[150] ^ aes_inv_sbox[212];
-	st0[2] ^= aes_inv_sbox[32] ^ aes_inv_sbox[ 96] ^ aes_inv_sbox[160] ^ aes_inv_sbox[236];
-	st0[3] ^= aes_inv_sbox[48] ^ aes_inv_sbox[112] ^ aes_inv_sbox[187] ^ aes_inv_sbox[247];
-
-	for (round = 0;; round += 2, rkp += 8) {
-		st1[0] = inv_mix_columns(inv_subshift(st0, 0)) ^ rkp[0];
-		st1[1] = inv_mix_columns(inv_subshift(st0, 1)) ^ rkp[1];
-		st1[2] = inv_mix_columns(inv_subshift(st0, 2)) ^ rkp[2];
-		st1[3] = inv_mix_columns(inv_subshift(st0, 3)) ^ rkp[3];
-
-		if (round == rounds - 2)
-			break;
-
-		st0[0] = inv_mix_columns(inv_subshift(st1, 0)) ^ rkp[4];
-		st0[1] = inv_mix_columns(inv_subshift(st1, 1)) ^ rkp[5];
-		st0[2] = inv_mix_columns(inv_subshift(st1, 2)) ^ rkp[6];
-		st0[3] = inv_mix_columns(inv_subshift(st1, 3)) ^ rkp[7];
-	}
+static void aes_decrypt_arch(const struct aes_key *key,
+			     u8 out[AES_BLOCK_SIZE],
+			     const u8 in[AES_BLOCK_SIZE])
+{
+	aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds, out, in);
+}
+#endif
 
-	put_unaligned_le32(inv_subshift(st1, 0) ^ rkp[4], out);
-	put_unaligned_le32(inv_subshift(st1, 1) ^ rkp[5], out + 4);
-	put_unaligned_le32(inv_subshift(st1, 2) ^ rkp[6], out + 8);
-	put_unaligned_le32(inv_subshift(st1, 3) ^ rkp[7], out + 12);
+static int __aes_preparekey(struct aes_enckey *enc_key,
+			    union aes_invkey_arch *inv_k,
+			    const u8 *in_key, size_t key_len)
+{
+	if (aes_check_keylen(key_len) != 0)
+		return -EINVAL;
+	enc_key->len = key_len;
+	enc_key->nrounds = 6 + key_len / 4;
+	aes_preparekey_arch(&enc_key->k, inv_k, in_key, key_len,
+			    enc_key->nrounds);
+	return 0;
+}
+
+int aes_preparekey(struct aes_key *key, const u8 *in_key, size_t key_len)
+{
+	return __aes_preparekey((struct aes_enckey *)key, &key->inv_k,
+				in_key, key_len);
+}
+EXPORT_SYMBOL(aes_preparekey);
+
+int aes_prepareenckey(struct aes_enckey *key, const u8 *in_key, size_t key_len)
+{
+	return __aes_preparekey(key, NULL, in_key, key_len);
+}
+EXPORT_SYMBOL(aes_prepareenckey);
+
+void aes_encrypt(aes_encrypt_arg key, u8 out[AES_BLOCK_SIZE],
+		 const u8 in[AES_BLOCK_SIZE])
+{
+	aes_encrypt_arch(key.enc_key, out, in);
+}
+EXPORT_SYMBOL(aes_encrypt);
+
+void aes_decrypt(const struct aes_key *key, u8 out[AES_BLOCK_SIZE],
+		 const u8 in[AES_BLOCK_SIZE])
+{
+	aes_decrypt_arch(key, out, in);
 }
 EXPORT_SYMBOL(aes_decrypt);
 
-MODULE_DESCRIPTION("Generic AES library");
+#ifdef aes_mod_init_arch
+static int __init aes_mod_init(void)
+{
+	aes_mod_init_arch();
+	return 0;
+}
+subsys_initcall(aes_mod_init);
+
+static void __exit aes_mod_exit(void)
+{
+}
+module_exit(aes_mod_exit);
+#endif
+
+MODULE_DESCRIPTION("AES block cipher");
 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_AUTHOR("Eric Biggers <ebiggers@kernel.org>");
 MODULE_LICENSE("GPL v2");
diff --git a/lib/crypto/aescfb.c b/lib/crypto/aescfb.c
index 0f294c8cbf3c..147e5211728f 100644
--- a/lib/crypto/aescfb.c
+++ b/lib/crypto/aescfb.c
@@ -11,7 +11,7 @@
 #include <linux/module.h>
 #include <asm/irqflags.h>
 
-static void aescfb_encrypt_block(const struct crypto_aes_ctx *ctx, void *dst,
+static void aescfb_encrypt_block(const struct aes_enckey *key, void *dst,
 				 const void *src)
 {
 	unsigned long flags;
@@ -25,27 +25,27 @@ static void aescfb_encrypt_block(const struct crypto_aes_ctx *ctx, void *dst,
 	 * interrupts disabled.
 	 */
 	local_irq_save(flags);
-	aes_encrypt(ctx, dst, src);
+	aes_encrypt(key, dst, src);
 	local_irq_restore(flags);
 }
 
 /**
  * aescfb_encrypt - Perform AES-CFB encryption on a block of data
  *
- * @ctx:	The AES-CFB key schedule
+ * @key:	The AES-CFB key schedule
  * @dst:	Pointer to the ciphertext output buffer
  * @src:	Pointer the plaintext (may equal @dst for encryption in place)
  * @len:	The size in bytes of the plaintext and ciphertext.
  * @iv:		The initialization vector (IV) to use for this block of data
  */
-void aescfb_encrypt(const struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src,
+void aescfb_encrypt(const struct aes_enckey *key, u8 *dst, const u8 *src,
 		    int len, const u8 iv[AES_BLOCK_SIZE])
 {
 	u8 ks[AES_BLOCK_SIZE];
 	const u8 *v = iv;
 
 	while (len > 0) {
-		aescfb_encrypt_block(ctx, ks, v);
+		aescfb_encrypt_block(key, ks, v);
 		crypto_xor_cpy(dst, src, ks, min(len, AES_BLOCK_SIZE));
 		v = dst;
 
@@ -61,18 +61,18 @@ EXPORT_SYMBOL(aescfb_encrypt);
 /**
  * aescfb_decrypt - Perform AES-CFB decryption on a block of data
  *
- * @ctx:	The AES-CFB key schedule
+ * @key:	The AES-CFB key schedule
  * @dst:	Pointer to the plaintext output buffer
  * @src:	Pointer the ciphertext (may equal @dst for decryption in place)
  * @len:	The size in bytes of the plaintext and ciphertext.
  * @iv:		The initialization vector (IV) to use for this block of data
  */
-void aescfb_decrypt(const struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src,
+void aescfb_decrypt(const struct aes_enckey *key, u8 *dst, const u8 *src,
 		    int len, const u8 iv[AES_BLOCK_SIZE])
 {
 	u8 ks[2][AES_BLOCK_SIZE];
 
-	aescfb_encrypt_block(ctx, ks[0], iv);
+	aescfb_encrypt_block(key, ks[0], iv);
 
 	for (int i = 0; len > 0; i ^= 1) {
 		if (len > AES_BLOCK_SIZE)
@@ -81,7 +81,7 @@ void aescfb_decrypt(const struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src,
 			 * performing the XOR, as that may update in place and
 			 * overwrite the ciphertext.
 			 */
-			aescfb_encrypt_block(ctx, ks[!i], src);
+			aescfb_encrypt_block(key, ks[!i], src);
 
 		crypto_xor_cpy(dst, src, ks[i], min(len, AES_BLOCK_SIZE));
 
@@ -214,15 +214,15 @@ static struct {
 static int __init libaescfb_init(void)
 {
 	for (int i = 0; i < ARRAY_SIZE(aescfb_tv); i++) {
-		struct crypto_aes_ctx ctx;
+		struct aes_enckey key;
 		u8 buf[64];
 
-		if (aes_expandkey(&ctx, aescfb_tv[i].key, aescfb_tv[i].klen)) {
-			pr_err("aes_expandkey() failed on vector %d\n", i);
+		if (aes_prepareenckey(&key, aescfb_tv[i].key, aescfb_tv[i].klen)) {
+			pr_err("aes_prepareenckey() failed on vector %d\n", i);
 			return -ENODEV;
 		}
 
-		aescfb_encrypt(&ctx, buf, aescfb_tv[i].ptext, aescfb_tv[i].len,
+		aescfb_encrypt(&key, buf, aescfb_tv[i].ptext, aescfb_tv[i].len,
 			       aescfb_tv[i].iv);
 		if (memcmp(buf, aescfb_tv[i].ctext, aescfb_tv[i].len)) {
 			pr_err("aescfb_encrypt() #1 failed on vector %d\n", i);
@@ -230,14 +230,14 @@ static int __init libaescfb_init(void)
 		}
 
 		/* decrypt in place */
-		aescfb_decrypt(&ctx, buf, buf, aescfb_tv[i].len, aescfb_tv[i].iv);
+		aescfb_decrypt(&key, buf, buf, aescfb_tv[i].len, aescfb_tv[i].iv);
 		if (memcmp(buf, aescfb_tv[i].ptext, aescfb_tv[i].len)) {
 			pr_err("aescfb_decrypt() failed on vector %d\n", i);
 			return -ENODEV;
 		}
 
 		/* encrypt in place */
-		aescfb_encrypt(&ctx, buf, buf, aescfb_tv[i].len, aescfb_tv[i].iv);
+		aescfb_encrypt(&key, buf, buf, aescfb_tv[i].len, aescfb_tv[i].iv);
 		if (memcmp(buf, aescfb_tv[i].ctext, aescfb_tv[i].len)) {
 			pr_err("aescfb_encrypt() #2 failed on vector %d\n", i);
 
diff --git a/lib/crypto/aesgcm.c b/lib/crypto/aesgcm.c
index ac0b2fcfd606..02f5b5f32c76 100644
--- a/lib/crypto/aesgcm.c
+++ b/lib/crypto/aesgcm.c
@@ -12,7 +12,7 @@
 #include <linux/module.h>
 #include <asm/irqflags.h>
 
-static void aesgcm_encrypt_block(const struct crypto_aes_ctx *ctx, void *dst,
+static void aesgcm_encrypt_block(const struct aes_enckey *key, void *dst,
 				 const void *src)
 {
 	unsigned long flags;
@@ -26,7 +26,7 @@ static void aesgcm_encrypt_block(const struct crypto_aes_ctx *ctx, void *dst,
 	 * effective when running with interrupts disabled.
 	 */
 	local_irq_save(flags);
-	aes_encrypt(ctx, dst, src);
+	aes_encrypt(key, dst, src);
 	local_irq_restore(flags);
 }
 
@@ -49,12 +49,12 @@ int aesgcm_expandkey(struct aesgcm_ctx *ctx, const u8 *key,
 	int ret;
 
 	ret = crypto_gcm_check_authsize(authsize) ?:
-	      aes_expandkey(&ctx->aes_ctx, key, keysize);
+	      aes_prepareenckey(&ctx->aes_key, key, keysize);
 	if (ret)
 		return ret;
 
 	ctx->authsize = authsize;
-	aesgcm_encrypt_block(&ctx->aes_ctx, &ctx->ghash_key, kin);
+	aesgcm_encrypt_block(&ctx->aes_key, &ctx->ghash_key, kin);
 
 	return 0;
 }
@@ -97,7 +97,7 @@ static void aesgcm_mac(const struct aesgcm_ctx *ctx, const u8 *src, int src_len,
 	aesgcm_ghash(&ghash, &ctx->ghash_key, &tail, sizeof(tail));
 
 	ctr[3] = cpu_to_be32(1);
-	aesgcm_encrypt_block(&ctx->aes_ctx, buf, ctr);
+	aesgcm_encrypt_block(&ctx->aes_key, buf, ctr);
 	crypto_xor_cpy(authtag, buf, (u8 *)&ghash, ctx->authsize);
 
 	memzero_explicit(&ghash, sizeof(ghash));
@@ -119,7 +119,7 @@ static void aesgcm_crypt(const struct aesgcm_ctx *ctx, u8 *dst, const u8 *src,
 		 * len', this cannot happen, so no explicit test is necessary.
 		 */
 		ctr[3] = cpu_to_be32(n++);
-		aesgcm_encrypt_block(&ctx->aes_ctx, buf, ctr);
+		aesgcm_encrypt_block(&ctx->aes_key, buf, ctr);
 		crypto_xor_cpy(dst, src, buf, min(len, AES_BLOCK_SIZE));
 
 		dst += AES_BLOCK_SIZE;
diff --git a/lib/crypto/arm/aes-cipher-core.S b/lib/crypto/arm/aes-cipher-core.S
new file mode 100644
index 000000000000..87567d6822ba
--- /dev/null
+++ b/lib/crypto/arm/aes-cipher-core.S
@@ -0,0 +1,201 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Scalar AES core transform
+ *
+ * Copyright (C) 2017 Linaro Ltd.
+ * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+#include <asm/cache.h>
+
+	.text
+	.align		5
+
+	rk		.req	r0
+	rounds		.req	r1
+	in		.req	r2
+	out		.req	r3
+	ttab		.req	ip
+
+	t0		.req	lr
+	t1		.req	r2
+	t2		.req	r3
+
+	.macro		__select, out, in, idx
+	.if		__LINUX_ARM_ARCH__ < 7
+	and		\out, \in, #0xff << (8 * \idx)
+	.else
+	ubfx		\out, \in, #(8 * \idx), #8
+	.endif
+	.endm
+
+	.macro		__load, out, in, idx, sz, op
+	.if		__LINUX_ARM_ARCH__ < 7 && \idx > 0
+	ldr\op		\out, [ttab, \in, lsr #(8 * \idx) - \sz]
+	.else
+	ldr\op		\out, [ttab, \in, lsl #\sz]
+	.endif
+	.endm
+
+	.macro		__hround, out0, out1, in0, in1, in2, in3, t3, t4, enc, sz, op, oldcpsr
+	__select	\out0, \in0, 0
+	__select	t0, \in1, 1
+	__load		\out0, \out0, 0, \sz, \op
+	__load		t0, t0, 1, \sz, \op
+
+	.if		\enc
+	__select	\out1, \in1, 0
+	__select	t1, \in2, 1
+	.else
+	__select	\out1, \in3, 0
+	__select	t1, \in0, 1
+	.endif
+	__load		\out1, \out1, 0, \sz, \op
+	__select	t2, \in2, 2
+	__load		t1, t1, 1, \sz, \op
+	__load		t2, t2, 2, \sz, \op
+
+	eor		\out0, \out0, t0, ror #24
+
+	__select	t0, \in3, 3
+	.if		\enc
+	__select	\t3, \in3, 2
+	__select	\t4, \in0, 3
+	.else
+	__select	\t3, \in1, 2
+	__select	\t4, \in2, 3
+	.endif
+	__load		\t3, \t3, 2, \sz, \op
+	__load		t0, t0, 3, \sz, \op
+	__load		\t4, \t4, 3, \sz, \op
+
+	.ifnb		\oldcpsr
+	/*
+	 * This is the final round and we're done with all data-dependent table
+	 * lookups, so we can safely re-enable interrupts.
+	 */
+	restore_irqs	\oldcpsr
+	.endif
+
+	eor		\out1, \out1, t1, ror #24
+	eor		\out0, \out0, t2, ror #16
+	ldm		rk!, {t1, t2}
+	eor		\out1, \out1, \t3, ror #16
+	eor		\out0, \out0, t0, ror #8
+	eor		\out1, \out1, \t4, ror #8
+	eor		\out0, \out0, t1
+	eor		\out1, \out1, t2
+	.endm
+
+	.macro		fround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op, oldcpsr
+	__hround	\out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1, \sz, \op
+	__hround	\out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1, \sz, \op, \oldcpsr
+	.endm
+
+	.macro		iround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op, oldcpsr
+	__hround	\out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0, \sz, \op
+	__hround	\out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0, \sz, \op, \oldcpsr
+	.endm
+
+	.macro		do_crypt, round, ttab, ltab, bsz
+	push		{r3-r11, lr}
+
+	// Load keys first, to reduce latency in case they're not cached yet.
+	ldm		rk!, {r8-r11}
+
+	ldr		r4, [in]
+	ldr		r5, [in, #4]
+	ldr		r6, [in, #8]
+	ldr		r7, [in, #12]
+
+#ifdef CONFIG_CPU_BIG_ENDIAN
+	rev_l		r4, t0
+	rev_l		r5, t0
+	rev_l		r6, t0
+	rev_l		r7, t0
+#endif
+
+	eor		r4, r4, r8
+	eor		r5, r5, r9
+	eor		r6, r6, r10
+	eor		r7, r7, r11
+
+	mov_l		ttab, \ttab
+	/*
+	 * Disable interrupts and prefetch the 1024-byte 'ft' or 'it' table into
+	 * L1 cache, assuming cacheline size >= 32.  This is a hardening measure
+	 * intended to make cache-timing attacks more difficult.  They may not
+	 * be fully prevented, however; see the paper
+	 * https://cr.yp.to/antiforgery/cachetiming-20050414.pdf
+	 * ("Cache-timing attacks on AES") for a discussion of the many
+	 * difficulties involved in writing truly constant-time AES software.
+	 */
+	 save_and_disable_irqs	t0
+	.set		i, 0
+	.rept		1024 / 128
+	ldr		r8, [ttab, #i + 0]
+	ldr		r9, [ttab, #i + 32]
+	ldr		r10, [ttab, #i + 64]
+	ldr		r11, [ttab, #i + 96]
+	.set		i, i + 128
+	.endr
+	push		{t0}		// oldcpsr
+
+	tst		rounds, #2
+	bne		1f
+
+0:	\round		r8, r9, r10, r11, r4, r5, r6, r7
+	\round		r4, r5, r6, r7, r8, r9, r10, r11
+
+1:	subs		rounds, rounds, #4
+	\round		r8, r9, r10, r11, r4, r5, r6, r7
+	bls		2f
+	\round		r4, r5, r6, r7, r8, r9, r10, r11
+	b		0b
+
+2:	.ifb		\ltab
+	add		ttab, ttab, #1
+	.else
+	mov_l		ttab, \ltab
+	// Prefetch inverse S-box for final round; see explanation above
+	.set		i, 0
+	.rept		256 / 64
+	ldr		t0, [ttab, #i + 0]
+	ldr		t1, [ttab, #i + 32]
+	.set		i, i + 64
+	.endr
+	.endif
+
+	pop		{rounds}	// oldcpsr
+	\round		r4, r5, r6, r7, r8, r9, r10, r11, \bsz, b, rounds
+
+#ifdef CONFIG_CPU_BIG_ENDIAN
+	rev_l		r4, t0
+	rev_l		r5, t0
+	rev_l		r6, t0
+	rev_l		r7, t0
+#endif
+
+	ldr		out, [sp]
+
+	str		r4, [out]
+	str		r5, [out, #4]
+	str		r6, [out, #8]
+	str		r7, [out, #12]
+
+	pop		{r3-r11, pc}
+
+	.align		3
+	.ltorg
+	.endm
+
+ENTRY(__aes_arm_encrypt)
+	do_crypt	fround, aes_enc_tab,, 2
+ENDPROC(__aes_arm_encrypt)
+
+	.align		5
+ENTRY(__aes_arm_decrypt)
+	do_crypt	iround, aes_dec_tab, crypto_aes_inv_sbox, 0
+ENDPROC(__aes_arm_decrypt)
diff --git a/lib/crypto/arm/aes.h b/lib/crypto/arm/aes.h
new file mode 100644
index 000000000000..1dd7dfa657bb
--- /dev/null
+++ b/lib/crypto/arm/aes.h
@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * AES block cipher, optimized for ARM
+ *
+ * Copyright (C) 2017 Linaro Ltd.
+ * Copyright 2026 Google LLC
+ */
+
+asmlinkage void __aes_arm_encrypt(const u32 rk[], int rounds,
+				  const u8 in[AES_BLOCK_SIZE],
+				  u8 out[AES_BLOCK_SIZE]);
+asmlinkage void __aes_arm_decrypt(const u32 inv_rk[], int rounds,
+				  const u8 in[AES_BLOCK_SIZE],
+				  u8 out[AES_BLOCK_SIZE]);
+
+static void aes_preparekey_arch(union aes_enckey_arch *k,
+				union aes_invkey_arch *inv_k,
+				const u8 *in_key, int key_len, int nrounds)
+{
+	aes_expandkey_generic(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL,
+			      in_key, key_len);
+}
+
+static void aes_encrypt_arch(const struct aes_enckey *key,
+			     u8 out[AES_BLOCK_SIZE],
+			     const u8 in[AES_BLOCK_SIZE])
+{
+	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
+	    !IS_ALIGNED((uintptr_t)out | (uintptr_t)in, 4)) {
+		u8 bounce_buf[AES_BLOCK_SIZE] __aligned(4);
+
+		memcpy(bounce_buf, in, AES_BLOCK_SIZE);
+		__aes_arm_encrypt(key->k.rndkeys, key->nrounds, bounce_buf,
+				  bounce_buf);
+		memcpy(out, bounce_buf, AES_BLOCK_SIZE);
+		return;
+	}
+	__aes_arm_encrypt(key->k.rndkeys, key->nrounds, in, out);
+}
+
+static void aes_decrypt_arch(const struct aes_key *key,
+			     u8 out[AES_BLOCK_SIZE],
+			     const u8 in[AES_BLOCK_SIZE])
+{
+	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
+	    !IS_ALIGNED((uintptr_t)out | (uintptr_t)in, 4)) {
+		u8 bounce_buf[AES_BLOCK_SIZE] __aligned(4);
+
+		memcpy(bounce_buf, in, AES_BLOCK_SIZE);
+		__aes_arm_decrypt(key->inv_k.inv_rndkeys, key->nrounds,
+				  bounce_buf, bounce_buf);
+		memcpy(out, bounce_buf, AES_BLOCK_SIZE);
+		return;
+	}
+	__aes_arm_decrypt(key->inv_k.inv_rndkeys, key->nrounds, in, out);
+}
diff --git a/lib/crypto/arm/blake2b-neon-core.S b/lib/crypto/arm/blake2b-neon-core.S
new file mode 100644
index 000000000000..b55c37f0b88f
--- /dev/null
+++ b/lib/crypto/arm/blake2b-neon-core.S
@@ -0,0 +1,350 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * BLAKE2b digest algorithm optimized with ARM NEON instructions.  On ARM
+ * processors that have NEON support but not the ARMv8 Crypto Extensions,
+ * typically this BLAKE2b implementation is much faster than the SHA-2 family
+ * and slightly faster than SHA-1.
+ *
+ * Copyright 2020 Google LLC
+ *
+ * Author: Eric Biggers <ebiggers@google.com>
+ */
+
+#include <linux/linkage.h>
+
+	.text
+	.fpu		neon
+
+	// The arguments to blake2b_compress_neon()
+	CTX		.req	r0
+	DATA		.req	r1
+	NBLOCKS		.req	r2
+	INC		.req	r3
+
+	// Pointers to the rotation tables
+	ROR24_TABLE	.req	r4
+	ROR16_TABLE	.req	r5
+
+	// The original stack pointer
+	ORIG_SP		.req	r6
+
+	// NEON registers which contain the message words of the current block.
+	// M_0-M_3 are occasionally used for other purposes too.
+	M_0		.req	d16
+	M_1		.req	d17
+	M_2		.req	d18
+	M_3		.req	d19
+	M_4		.req	d20
+	M_5		.req	d21
+	M_6		.req	d22
+	M_7		.req	d23
+	M_8		.req	d24
+	M_9		.req	d25
+	M_10		.req	d26
+	M_11		.req	d27
+	M_12		.req	d28
+	M_13		.req	d29
+	M_14		.req	d30
+	M_15		.req	d31
+
+	.align		4
+	// Tables for computing ror64(x, 24) and ror64(x, 16) using the vtbl.8
+	// instruction.  This is the most efficient way to implement these
+	// rotation amounts with NEON.  (On Cortex-A53 it's the same speed as
+	// vshr.u64 + vsli.u64, while on Cortex-A7 it's faster.)
+.Lror24_table:
+	.byte		3, 4, 5, 6, 7, 0, 1, 2
+.Lror16_table:
+	.byte		2, 3, 4, 5, 6, 7, 0, 1
+	// The BLAKE2b initialization vector
+.Lblake2b_IV:
+	.quad		0x6a09e667f3bcc908, 0xbb67ae8584caa73b
+	.quad		0x3c6ef372fe94f82b, 0xa54ff53a5f1d36f1
+	.quad		0x510e527fade682d1, 0x9b05688c2b3e6c1f
+	.quad		0x1f83d9abfb41bd6b, 0x5be0cd19137e2179
+
+// Execute one round of BLAKE2b by updating the state matrix v[0..15] in the
+// NEON registers q0-q7.  The message block is in q8..q15 (M_0-M_15).  The stack
+// pointer points to a 32-byte aligned buffer containing a copy of q8 and q9
+// (M_0-M_3), so that they can be reloaded if they are used as temporary
+// registers.  The macro arguments s0-s15 give the order in which the message
+// words are used in this round.  'final' is 1 if this is the final round.
+.macro	_blake2b_round	s0, s1, s2, s3, s4, s5, s6, s7, \
+			s8, s9, s10, s11, s12, s13, s14, s15, final=0
+
+	// Mix the columns:
+	// (v[0], v[4], v[8], v[12]), (v[1], v[5], v[9], v[13]),
+	// (v[2], v[6], v[10], v[14]), and (v[3], v[7], v[11], v[15]).
+
+	// a += b + m[blake2b_sigma[r][2*i + 0]];
+	vadd.u64	q0, q0, q2
+	vadd.u64	q1, q1, q3
+	vadd.u64	d0, d0, M_\s0
+	vadd.u64	d1, d1, M_\s2
+	vadd.u64	d2, d2, M_\s4
+	vadd.u64	d3, d3, M_\s6
+
+	// d = ror64(d ^ a, 32);
+	veor		q6, q6, q0
+	veor		q7, q7, q1
+	vrev64.32	q6, q6
+	vrev64.32	q7, q7
+
+	// c += d;
+	vadd.u64	q4, q4, q6
+	vadd.u64	q5, q5, q7
+
+	// b = ror64(b ^ c, 24);
+	vld1.8		{M_0}, [ROR24_TABLE, :64]
+	veor		q2, q2, q4
+	veor		q3, q3, q5
+	vtbl.8		d4, {d4}, M_0
+	vtbl.8		d5, {d5}, M_0
+	vtbl.8		d6, {d6}, M_0
+	vtbl.8		d7, {d7}, M_0
+
+	// a += b + m[blake2b_sigma[r][2*i + 1]];
+	//
+	// M_0 got clobbered above, so we have to reload it if any of the four
+	// message words this step needs happens to be M_0.  Otherwise we don't
+	// need to reload it here, as it will just get clobbered again below.
+.if \s1 == 0 || \s3 == 0 || \s5 == 0 || \s7 == 0
+	vld1.8		{M_0}, [sp, :64]
+.endif
+	vadd.u64	q0, q0, q2
+	vadd.u64	q1, q1, q3
+	vadd.u64	d0, d0, M_\s1
+	vadd.u64	d1, d1, M_\s3
+	vadd.u64	d2, d2, M_\s5
+	vadd.u64	d3, d3, M_\s7
+
+	// d = ror64(d ^ a, 16);
+	vld1.8		{M_0}, [ROR16_TABLE, :64]
+	veor		q6, q6, q0
+	veor		q7, q7, q1
+	vtbl.8		d12, {d12}, M_0
+	vtbl.8		d13, {d13}, M_0
+	vtbl.8		d14, {d14}, M_0
+	vtbl.8		d15, {d15}, M_0
+
+	// c += d;
+	vadd.u64	q4, q4, q6
+	vadd.u64	q5, q5, q7
+
+	// b = ror64(b ^ c, 63);
+	//
+	// This rotation amount isn't a multiple of 8, so it has to be
+	// implemented using a pair of shifts, which requires temporary
+	// registers.  Use q8-q9 (M_0-M_3) for this, and reload them afterwards.
+	veor		q8, q2, q4
+	veor		q9, q3, q5
+	vshr.u64	q2, q8, #63
+	vshr.u64	q3, q9, #63
+	vsli.u64	q2, q8, #1
+	vsli.u64	q3, q9, #1
+	vld1.8		{q8-q9}, [sp, :256]
+
+	// Mix the diagonals:
+	// (v[0], v[5], v[10], v[15]), (v[1], v[6], v[11], v[12]),
+	// (v[2], v[7], v[8], v[13]), and (v[3], v[4], v[9], v[14]).
+	//
+	// There are two possible ways to do this: use 'vext' instructions to
+	// shift the rows of the matrix so that the diagonals become columns,
+	// and undo it afterwards; or just use 64-bit operations on 'd'
+	// registers instead of 128-bit operations on 'q' registers.  We use the
+	// latter approach, as it performs much better on Cortex-A7.
+
+	// a += b + m[blake2b_sigma[r][2*i + 0]];
+	vadd.u64	d0, d0, d5
+	vadd.u64	d1, d1, d6
+	vadd.u64	d2, d2, d7
+	vadd.u64	d3, d3, d4
+	vadd.u64	d0, d0, M_\s8
+	vadd.u64	d1, d1, M_\s10
+	vadd.u64	d2, d2, M_\s12
+	vadd.u64	d3, d3, M_\s14
+
+	// d = ror64(d ^ a, 32);
+	veor		d15, d15, d0
+	veor		d12, d12, d1
+	veor		d13, d13, d2
+	veor		d14, d14, d3
+	vrev64.32	d15, d15
+	vrev64.32	d12, d12
+	vrev64.32	d13, d13
+	vrev64.32	d14, d14
+
+	// c += d;
+	vadd.u64	d10, d10, d15
+	vadd.u64	d11, d11, d12
+	vadd.u64	d8, d8, d13
+	vadd.u64	d9, d9, d14
+
+	// b = ror64(b ^ c, 24);
+	vld1.8		{M_0}, [ROR24_TABLE, :64]
+	veor		d5, d5, d10
+	veor		d6, d6, d11
+	veor		d7, d7, d8
+	veor		d4, d4, d9
+	vtbl.8		d5, {d5}, M_0
+	vtbl.8		d6, {d6}, M_0
+	vtbl.8		d7, {d7}, M_0
+	vtbl.8		d4, {d4}, M_0
+
+	// a += b + m[blake2b_sigma[r][2*i + 1]];
+.if \s9 == 0 || \s11 == 0 || \s13 == 0 || \s15 == 0
+	vld1.8		{M_0}, [sp, :64]
+.endif
+	vadd.u64	d0, d0, d5
+	vadd.u64	d1, d1, d6
+	vadd.u64	d2, d2, d7
+	vadd.u64	d3, d3, d4
+	vadd.u64	d0, d0, M_\s9
+	vadd.u64	d1, d1, M_\s11
+	vadd.u64	d2, d2, M_\s13
+	vadd.u64	d3, d3, M_\s15
+
+	// d = ror64(d ^ a, 16);
+	vld1.8		{M_0}, [ROR16_TABLE, :64]
+	veor		d15, d15, d0
+	veor		d12, d12, d1
+	veor		d13, d13, d2
+	veor		d14, d14, d3
+	vtbl.8		d12, {d12}, M_0
+	vtbl.8		d13, {d13}, M_0
+	vtbl.8		d14, {d14}, M_0
+	vtbl.8		d15, {d15}, M_0
+
+	// c += d;
+	vadd.u64	d10, d10, d15
+	vadd.u64	d11, d11, d12
+	vadd.u64	d8, d8, d13
+	vadd.u64	d9, d9, d14
+
+	// b = ror64(b ^ c, 63);
+	veor		d16, d4, d9
+	veor		d17, d5, d10
+	veor		d18, d6, d11
+	veor		d19, d7, d8
+	vshr.u64	q2, q8, #63
+	vshr.u64	q3, q9, #63
+	vsli.u64	q2, q8, #1
+	vsli.u64	q3, q9, #1
+	// Reloading q8-q9 can be skipped on the final round.
+.if ! \final
+	vld1.8		{q8-q9}, [sp, :256]
+.endif
+.endm
+
+//
+// void blake2b_compress_neon(struct blake2b_ctx *ctx,
+//			      const u8 *data, size_t nblocks, u32 inc);
+//
+// Only the first three fields of struct blake2b_ctx are used:
+//	u64 h[8];	(inout)
+//	u64 t[2];	(inout)
+//	u64 f[2];	(in)
+//
+	.align		5
+ENTRY(blake2b_compress_neon)
+	push		{r4-r10}
+
+	// Allocate a 32-byte stack buffer that is 32-byte aligned.
+	mov		ORIG_SP, sp
+	sub		ip, sp, #32
+	bic		ip, ip, #31
+	mov		sp, ip
+
+	adr		ROR24_TABLE, .Lror24_table
+	adr		ROR16_TABLE, .Lror16_table
+
+	mov		ip, CTX
+	vld1.64		{q0-q1}, [ip]!		// Load h[0..3]
+	vld1.64		{q2-q3}, [ip]!		// Load h[4..7]
+.Lnext_block:
+	  adr		r10, .Lblake2b_IV
+	vld1.64		{q14-q15}, [ip]		// Load t[0..1] and f[0..1]
+	vld1.64		{q4-q5}, [r10]!		// Load IV[0..3]
+	  vmov		r7, r8, d28		// Copy t[0] to (r7, r8)
+	vld1.64		{q6-q7}, [r10]		// Load IV[4..7]
+	  adds		r7, r7, INC		// Increment counter
+	bcs		.Lslow_inc_ctr
+	vmov.i32	d28[0], r7
+	vst1.64		{d28}, [ip]		// Update t[0]
+.Linc_ctr_done:
+
+	// Load the next message block and finish initializing the state matrix
+	// 'v'.  Fortunately, there are exactly enough NEON registers to fit the
+	// entire state matrix in q0-q7 and the entire message block in q8-15.
+	//
+	// However, _blake2b_round also needs some extra registers for rotates,
+	// so we have to spill some registers.  It's better to spill the message
+	// registers than the state registers, as the message doesn't change.
+	// Therefore we store a copy of the first 32 bytes of the message block
+	// (q8-q9) in an aligned buffer on the stack so that they can be
+	// reloaded when needed.  (We could just reload directly from the
+	// message buffer, but it's faster to use aligned loads.)
+	vld1.8		{q8-q9}, [DATA]!
+	  veor		q6, q6, q14	// v[12..13] = IV[4..5] ^ t[0..1]
+	vld1.8		{q10-q11}, [DATA]!
+	  veor		q7, q7, q15	// v[14..15] = IV[6..7] ^ f[0..1]
+	vld1.8		{q12-q13}, [DATA]!
+	vst1.8		{q8-q9}, [sp, :256]
+	  mov		ip, CTX
+	vld1.8		{q14-q15}, [DATA]!
+
+	// Execute the rounds.  Each round is provided the order in which it
+	// needs to use the message words.
+	_blake2b_round	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
+	_blake2b_round	14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3
+	_blake2b_round	11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4
+	_blake2b_round	7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8
+	_blake2b_round	9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13
+	_blake2b_round	2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9
+	_blake2b_round	12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11
+	_blake2b_round	13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10
+	_blake2b_round	6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5
+	_blake2b_round	10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0
+	_blake2b_round	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
+	_blake2b_round	14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 \
+			final=1
+
+	// Fold the final state matrix into the hash chaining value:
+	//
+	//	for (i = 0; i < 8; i++)
+	//		h[i] ^= v[i] ^ v[i + 8];
+	//
+	  vld1.64	{q8-q9}, [ip]!		// Load old h[0..3]
+	veor		q0, q0, q4		// v[0..1] ^= v[8..9]
+	veor		q1, q1, q5		// v[2..3] ^= v[10..11]
+	  vld1.64	{q10-q11}, [ip]		// Load old h[4..7]
+	veor		q2, q2, q6		// v[4..5] ^= v[12..13]
+	veor		q3, q3, q7		// v[6..7] ^= v[14..15]
+	veor		q0, q0, q8		// v[0..1] ^= h[0..1]
+	veor		q1, q1, q9		// v[2..3] ^= h[2..3]
+	  mov		ip, CTX
+	  subs		NBLOCKS, NBLOCKS, #1	// nblocks--
+	  vst1.64	{q0-q1}, [ip]!		// Store new h[0..3]
+	veor		q2, q2, q10		// v[4..5] ^= h[4..5]
+	veor		q3, q3, q11		// v[6..7] ^= h[6..7]
+	  vst1.64	{q2-q3}, [ip]!		// Store new h[4..7]
+
+	// Advance to the next block, if there is one.
+	bne		.Lnext_block		// nblocks != 0?
+
+	mov		sp, ORIG_SP
+	pop		{r4-r10}
+	mov		pc, lr
+
+.Lslow_inc_ctr:
+	// Handle the case where the counter overflowed its low 32 bits, by
+	// carrying the overflow bit into the full 128-bit counter.
+	vmov		r9, r10, d29
+	adcs		r8, r8, #0
+	adcs		r9, r9, #0
+	adc		r10, r10, #0
+	vmov		d28, r7, r8
+	vmov		d29, r9, r10
+	vst1.64		{q14}, [ip]		// Update t[0] and t[1]
+	b		.Linc_ctr_done
+ENDPROC(blake2b_compress_neon)
diff --git a/lib/crypto/arm/blake2b.h b/lib/crypto/arm/blake2b.h
new file mode 100644
index 000000000000..5c76498521e6
--- /dev/null
+++ b/lib/crypto/arm/blake2b.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * BLAKE2b digest algorithm, NEON accelerated
+ *
+ * Copyright 2020 Google LLC
+ */
+
+#include <asm/neon.h>
+#include <asm/simd.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
+
+asmlinkage void blake2b_compress_neon(struct blake2b_ctx *ctx,
+				      const u8 *data, size_t nblocks, u32 inc);
+
+static void blake2b_compress(struct blake2b_ctx *ctx,
+			     const u8 *data, size_t nblocks, u32 inc)
+{
+	if (!static_branch_likely(&have_neon) || !may_use_simd()) {
+		blake2b_compress_generic(ctx, data, nblocks, inc);
+		return;
+	}
+	do {
+		const size_t blocks = min_t(size_t, nblocks,
+					    SZ_4K / BLAKE2B_BLOCK_SIZE);
+
+		scoped_ksimd()
+			blake2b_compress_neon(ctx, data, blocks, inc);
+
+		data += blocks * BLAKE2B_BLOCK_SIZE;
+		nblocks -= blocks;
+	} while (nblocks);
+}
+
+#define blake2b_mod_init_arch blake2b_mod_init_arch
+static void blake2b_mod_init_arch(void)
+{
+	if (elf_hwcap & HWCAP_NEON)
+		static_branch_enable(&have_neon);
+}
diff --git a/lib/crypto/arm/blake2s-core.S b/lib/crypto/arm/blake2s-core.S
index 293f44fa8f31..933f0558b7cd 100644
--- a/lib/crypto/arm/blake2s-core.S
+++ b/lib/crypto/arm/blake2s-core.S
@@ -115,7 +115,7 @@
 
 // Execute one round of BLAKE2s by updating the state matrix v[0..15].  v[0..9]
 // are in r0..r9.  The stack pointer points to 8 bytes of scratch space for
-// spilling v[8..9], then to v[9..15], then to the message block.  r10-r12 and
+// spilling v[8..9], then to v[10..15], then to the message block.  r10-r12 and
 // r14 are free to use.  The macro arguments s0-s15 give the order in which the
 // message words are used in this round.
 //
@@ -170,10 +170,10 @@
 .endm
 
 //
-// void blake2s_compress(struct blake2s_state *state,
-//			 const u8 *block, size_t nblocks, u32 inc);
+// void blake2s_compress(struct blake2s_ctx *ctx,
+//			 const u8 *data, size_t nblocks, u32 inc);
 //
-// Only the first three fields of struct blake2s_state are used:
+// Only the first three fields of struct blake2s_ctx are used:
 //	u32 h[8];	(inout)
 //	u32 t[2];	(inout)
 //	u32 f[2];	(in)
@@ -183,8 +183,8 @@ ENTRY(blake2s_compress)
 	push		{r0-r2,r4-r11,lr}	// keep this an even number
 
 .Lnext_block:
-	// r0 is 'state'
-	// r1 is 'block'
+	// r0 is 'ctx'
+	// r1 is 'data'
 	// r3 is 'inc'
 
 	// Load and increment the counter t[0..1].
@@ -209,18 +209,18 @@ ENTRY(blake2s_compress)
 .Lcopy_block_done:
 	str		r1, [sp, #68]		// Update message pointer
 
-	// Calculate v[8..15].  Push v[9..15] onto the stack, and leave space
+	// Calculate v[8..15].  Push v[10..15] onto the stack, and leave space
 	// for spilling v[8..9].  Leave v[8..9] in r8-r9.
-	mov		r14, r0			// r14 = state
+	mov		r14, r0			// r14 = ctx
 	adr		r12, .Lblake2s_IV
 	ldmia		r12!, {r8-r9}		// load IV[0..1]
 	__ldrd		r0, r1, r14, 40		// load f[0..1]
-	ldm		r12, {r2-r7}		// load IV[3..7]
+	ldm		r12, {r2-r7}		// load IV[2..7]
 	eor		r4, r4, r10		// v[12] = IV[4] ^ t[0]
 	eor		r5, r5, r11		// v[13] = IV[5] ^ t[1]
 	eor		r6, r6, r0		// v[14] = IV[6] ^ f[0]
 	eor		r7, r7, r1		// v[15] = IV[7] ^ f[1]
-	push		{r2-r7}			// push v[9..15]
+	push		{r2-r7}			// push v[10..15]
 	sub		sp, sp, #8		// leave space for v[8..9]
 
 	// Load h[0..7] == v[0..7].
@@ -275,7 +275,7 @@ ENTRY(blake2s_compress)
 	// Advance to the next block, if there is one.  Note that if there are
 	// multiple blocks, then 'inc' (the counter increment amount) must be
 	// 64.  So we can simply set it to 64 without re-loading it.
-	ldm		sp, {r0, r1, r2}	// load (state, block, nblocks)
+	ldm		sp, {r0, r1, r2}	// load (ctx, data, nblocks)
 	mov		r3, #64			// set 'inc'
 	subs		r2, r2, #1		// nblocks--
 	str		r2, [sp, #8]
diff --git a/lib/crypto/arm/blake2s.h b/lib/crypto/arm/blake2s.h
index aa7a97139ea7..42c04440c191 100644
--- a/lib/crypto/arm/blake2s.h
+++ b/lib/crypto/arm/blake2s.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0-or-later */
 
 /* defined in blake2s-core.S */
-void blake2s_compress(struct blake2s_state *state, const u8 *block,
-		      size_t nblocks, u32 inc);
+void blake2s_compress(struct blake2s_ctx *ctx,
+		      const u8 *data, size_t nblocks, u32 inc);
diff --git a/lib/crypto/arm/chacha.h b/lib/crypto/arm/chacha.h
index 0cae30f8ee5d..836e49088e98 100644
--- a/lib/crypto/arm/chacha.h
+++ b/lib/crypto/arm/chacha.h
@@ -12,7 +12,6 @@
 
 #include <asm/cputype.h>
 #include <asm/hwcap.h>
-#include <asm/neon.h>
 #include <asm/simd.h>
 
 asmlinkage void chacha_block_xor_neon(const struct chacha_state *state,
@@ -68,9 +67,8 @@ static void hchacha_block_arch(const struct chacha_state *state,
 	if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable()) {
 		hchacha_block_arm(state, out, nrounds);
 	} else {
-		kernel_neon_begin();
-		hchacha_block_neon(state, out, nrounds);
-		kernel_neon_end();
+		scoped_ksimd()
+			hchacha_block_neon(state, out, nrounds);
 	}
 }
 
@@ -87,9 +85,8 @@ static void chacha_crypt_arch(struct chacha_state *state, u8 *dst,
 	do {
 		unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
 
-		kernel_neon_begin();
-		chacha_doneon(state, dst, src, todo, nrounds);
-		kernel_neon_end();
+		scoped_ksimd()
+			chacha_doneon(state, dst, src, todo, nrounds);
 
 		bytes -= todo;
 		src += todo;
diff --git a/lib/crypto/arm/curve25519.h b/lib/crypto/arm/curve25519.h
index f6d66494eb8f..b1a566885e95 100644
--- a/lib/crypto/arm/curve25519.h
+++ b/lib/crypto/arm/curve25519.h
@@ -25,9 +25,8 @@ static void curve25519_arch(u8 out[CURVE25519_KEY_SIZE],
 			    const u8 point[CURVE25519_KEY_SIZE])
 {
 	if (static_branch_likely(&have_neon) && crypto_simd_usable()) {
-		kernel_neon_begin();
-		curve25519_neon(out, scalar, point);
-		kernel_neon_end();
+		scoped_ksimd()
+			curve25519_neon(out, scalar, point);
 	} else {
 		curve25519_generic(out, scalar, point);
 	}
diff --git a/lib/crypto/arm/nh-neon-core.S b/lib/crypto/arm/nh-neon-core.S
new file mode 100644
index 000000000000..01620a0782ca
--- /dev/null
+++ b/lib/crypto/arm/nh-neon-core.S
@@ -0,0 +1,116 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * NH - ε-almost-universal hash function, NEON accelerated version
+ *
+ * Copyright 2018 Google LLC
+ *
+ * Author: Eric Biggers <ebiggers@google.com>
+ */
+
+#include <linux/linkage.h>
+
+	.text
+	.fpu		neon
+
+	KEY		.req	r0
+	MESSAGE		.req	r1
+	MESSAGE_LEN	.req	r2
+	HASH		.req	r3
+
+	PASS0_SUMS	.req	q0
+	PASS0_SUM_A	.req	d0
+	PASS0_SUM_B	.req	d1
+	PASS1_SUMS	.req	q1
+	PASS1_SUM_A	.req	d2
+	PASS1_SUM_B	.req	d3
+	PASS2_SUMS	.req	q2
+	PASS2_SUM_A	.req	d4
+	PASS2_SUM_B	.req	d5
+	PASS3_SUMS	.req	q3
+	PASS3_SUM_A	.req	d6
+	PASS3_SUM_B	.req	d7
+	K0		.req	q4
+	K1		.req	q5
+	K2		.req	q6
+	K3		.req	q7
+	T0		.req	q8
+	T0_L		.req	d16
+	T0_H		.req	d17
+	T1		.req	q9
+	T1_L		.req	d18
+	T1_H		.req	d19
+	T2		.req	q10
+	T2_L		.req	d20
+	T2_H		.req	d21
+	T3		.req	q11
+	T3_L		.req	d22
+	T3_H		.req	d23
+
+.macro _nh_stride	k0, k1, k2, k3
+
+	// Load next message stride
+	vld1.8		{T3}, [MESSAGE]!
+
+	// Load next key stride
+	vld1.32		{\k3}, [KEY]!
+
+	// Add message words to key words
+	vadd.u32	T0, T3, \k0
+	vadd.u32	T1, T3, \k1
+	vadd.u32	T2, T3, \k2
+	vadd.u32	T3, T3, \k3
+
+	// Multiply 32x32 => 64 and accumulate
+	vmlal.u32	PASS0_SUMS, T0_L, T0_H
+	vmlal.u32	PASS1_SUMS, T1_L, T1_H
+	vmlal.u32	PASS2_SUMS, T2_L, T2_H
+	vmlal.u32	PASS3_SUMS, T3_L, T3_H
+.endm
+
+/*
+ * void nh_neon(const u32 *key, const u8 *message, size_t message_len,
+ *		__le64 hash[NH_NUM_PASSES])
+ *
+ * It's guaranteed that message_len % 16 == 0.
+ */
+ENTRY(nh_neon)
+
+	vld1.32		{K0,K1}, [KEY]!
+	  vmov.u64	PASS0_SUMS, #0
+	  vmov.u64	PASS1_SUMS, #0
+	vld1.32		{K2}, [KEY]!
+	  vmov.u64	PASS2_SUMS, #0
+	  vmov.u64	PASS3_SUMS, #0
+
+	subs		MESSAGE_LEN, MESSAGE_LEN, #64
+	blt		.Lloop4_done
+.Lloop4:
+	_nh_stride	K0, K1, K2, K3
+	_nh_stride	K1, K2, K3, K0
+	_nh_stride	K2, K3, K0, K1
+	_nh_stride	K3, K0, K1, K2
+	subs		MESSAGE_LEN, MESSAGE_LEN, #64
+	bge		.Lloop4
+
+.Lloop4_done:
+	ands		MESSAGE_LEN, MESSAGE_LEN, #63
+	beq		.Ldone
+	_nh_stride	K0, K1, K2, K3
+
+	subs		MESSAGE_LEN, MESSAGE_LEN, #16
+	beq		.Ldone
+	_nh_stride	K1, K2, K3, K0
+
+	subs		MESSAGE_LEN, MESSAGE_LEN, #16
+	beq		.Ldone
+	_nh_stride	K2, K3, K0, K1
+
+.Ldone:
+	// Sum the accumulators for each pass, then store the sums to 'hash'
+	vadd.u64	T0_L, PASS0_SUM_A, PASS0_SUM_B
+	vadd.u64	T0_H, PASS1_SUM_A, PASS1_SUM_B
+	vadd.u64	T1_L, PASS2_SUM_A, PASS2_SUM_B
+	vadd.u64	T1_H, PASS3_SUM_A, PASS3_SUM_B
+	vst1.8		{T0-T1}, [HASH]
+	bx		lr
+ENDPROC(nh_neon)
diff --git a/lib/crypto/arm/nh.h b/lib/crypto/arm/nh.h
new file mode 100644
index 000000000000..c9f39d819336
--- /dev/null
+++ b/lib/crypto/arm/nh.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * ARM32 accelerated implementation of NH
+ *
+ * Copyright 2018 Google LLC
+ */
+
+#include <asm/neon.h>
+#include <asm/simd.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
+
+asmlinkage void nh_neon(const u32 *key, const u8 *message, size_t message_len,
+			__le64 hash[NH_NUM_PASSES]);
+
+static bool nh_arch(const u32 *key, const u8 *message, size_t message_len,
+		    __le64 hash[NH_NUM_PASSES])
+{
+	if (static_branch_likely(&have_neon) && message_len >= 64 &&
+	    may_use_simd()) {
+		scoped_ksimd()
+			nh_neon(key, message, message_len, hash);
+		return true;
+	}
+	return false;
+}
+
+#define nh_mod_init_arch nh_mod_init_arch
+static void nh_mod_init_arch(void)
+{
+	if (elf_hwcap & HWCAP_NEON)
+		static_branch_enable(&have_neon);
+}
diff --git a/lib/crypto/arm/poly1305.h b/lib/crypto/arm/poly1305.h
index 0021cf368307..0fe903d8de55 100644
--- a/lib/crypto/arm/poly1305.h
+++ b/lib/crypto/arm/poly1305.h
@@ -6,7 +6,6 @@
  */
 
 #include <asm/hwcap.h>
-#include <asm/neon.h>
 #include <asm/simd.h>
 #include <linux/cpufeature.h>
 #include <linux/jump_label.h>
@@ -32,9 +31,8 @@ static void poly1305_blocks(struct poly1305_block_state *state, const u8 *src,
 		do {
 			unsigned int todo = min_t(unsigned int, len, SZ_4K);
 
-			kernel_neon_begin();
-			poly1305_blocks_neon(state, src, todo, padbit);
-			kernel_neon_end();
+			scoped_ksimd()
+				poly1305_blocks_neon(state, src, todo, padbit);
 
 			len -= todo;
 			src += todo;
diff --git a/lib/crypto/arm/sha1-armv7-neon.S b/lib/crypto/arm/sha1-armv7-neon.S
index 6edba3ab62e8..a0323fa5c58a 100644
--- a/lib/crypto/arm/sha1-armv7-neon.S
+++ b/lib/crypto/arm/sha1-armv7-neon.S
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0-or-later */
-/* sha1-armv7-neon.S - ARM/NEON accelerated SHA-1 transform function
+/* ARM/NEON accelerated SHA-1 transform function
  *
  * Copyright © 2013-2014 Jussi Kivilinna <jussi.kivilinna@iki.fi>
  */
diff --git a/lib/crypto/arm/sha1-ce-core.S b/lib/crypto/arm/sha1-ce-core.S
index 2de40dd25e47..7d6b2631ca8d 100644
--- a/lib/crypto/arm/sha1-ce-core.S
+++ b/lib/crypto/arm/sha1-ce-core.S
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * sha1-ce-core.S - SHA-1 secure hash using ARMv8 Crypto Extensions
+ * SHA-1 secure hash using ARMv8 Crypto Extensions
  *
  * Copyright (C) 2015 Linaro Ltd.
  * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
diff --git a/lib/crypto/arm/sha1.h b/lib/crypto/arm/sha1.h
index 29f8bcad0447..3e2d8c7cab9f 100644
--- a/lib/crypto/arm/sha1.h
+++ b/lib/crypto/arm/sha1.h
@@ -4,7 +4,6 @@
  *
  * Copyright 2025 Google LLC
  */
-#include <asm/neon.h>
 #include <asm/simd.h>
 
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
@@ -22,12 +21,12 @@ static void sha1_blocks(struct sha1_block_state *state,
 {
 	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
 	    static_branch_likely(&have_neon) && likely(may_use_simd())) {
-		kernel_neon_begin();
-		if (static_branch_likely(&have_ce))
-			sha1_ce_transform(state, data, nblocks);
-		else
-			sha1_transform_neon(state, data, nblocks);
-		kernel_neon_end();
+		scoped_ksimd() {
+			if (static_branch_likely(&have_ce))
+				sha1_ce_transform(state, data, nblocks);
+			else
+				sha1_transform_neon(state, data, nblocks);
+		}
 	} else {
 		sha1_block_data_order(state, data, nblocks);
 	}
diff --git a/lib/crypto/arm/sha256-ce.S b/lib/crypto/arm/sha256-ce.S
index 7481ac8e6c0d..144ee805f64a 100644
--- a/lib/crypto/arm/sha256-ce.S
+++ b/lib/crypto/arm/sha256-ce.S
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * sha256-ce.S - SHA-224/256 secure hash using ARMv8 Crypto Extensions
+ * SHA-224/256 secure hash using ARMv8 Crypto Extensions
  *
  * Copyright (C) 2015 Linaro Ltd.
  * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
diff --git a/lib/crypto/arm/sha256.h b/lib/crypto/arm/sha256.h
index 7556457b3094..ae7e52dd6e3b 100644
--- a/lib/crypto/arm/sha256.h
+++ b/lib/crypto/arm/sha256.h
@@ -22,12 +22,12 @@ static void sha256_blocks(struct sha256_block_state *state,
 {
 	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
 	    static_branch_likely(&have_neon) && likely(may_use_simd())) {
-		kernel_neon_begin();
-		if (static_branch_likely(&have_ce))
-			sha256_ce_transform(state, data, nblocks);
-		else
-			sha256_block_data_order_neon(state, data, nblocks);
-		kernel_neon_end();
+		scoped_ksimd() {
+			if (static_branch_likely(&have_ce))
+				sha256_ce_transform(state, data, nblocks);
+			else
+				sha256_block_data_order_neon(state, data, nblocks);
+		}
 	} else {
 		sha256_block_data_order(state, data, nblocks);
 	}
diff --git a/lib/crypto/arm/sha512.h b/lib/crypto/arm/sha512.h
index d1b485dd275d..ed9bd81d6d78 100644
--- a/lib/crypto/arm/sha512.h
+++ b/lib/crypto/arm/sha512.h
@@ -19,9 +19,8 @@ static void sha512_blocks(struct sha512_block_state *state,
 {
 	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
 	    static_branch_likely(&have_neon) && likely(may_use_simd())) {
-		kernel_neon_begin();
-		sha512_block_data_order_neon(state, data, nblocks);
-		kernel_neon_end();
+		scoped_ksimd()
+			sha512_block_data_order_neon(state, data, nblocks);
 	} else {
 		sha512_block_data_order(state, data, nblocks);
 	}
diff --git a/lib/crypto/arm64/aes-ce-core.S b/lib/crypto/arm64/aes-ce-core.S
new file mode 100644
index 000000000000..e52e13eb8fdb
--- /dev/null
+++ b/lib/crypto/arm64/aes-ce-core.S
@@ -0,0 +1,84 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+	.arch		armv8-a+crypto
+
+SYM_FUNC_START(__aes_ce_encrypt)
+	sub		w3, w3, #2
+	ld1		{v0.16b}, [x2]
+	ld1		{v1.4s}, [x0], #16
+	cmp		w3, #10
+	bmi		0f
+	bne		3f
+	mov		v3.16b, v1.16b
+	b		2f
+0:	mov		v2.16b, v1.16b
+	ld1		{v3.4s}, [x0], #16
+1:	aese		v0.16b, v2.16b
+	aesmc		v0.16b, v0.16b
+2:	ld1		{v1.4s}, [x0], #16
+	aese		v0.16b, v3.16b
+	aesmc		v0.16b, v0.16b
+3:	ld1		{v2.4s}, [x0], #16
+	subs		w3, w3, #3
+	aese		v0.16b, v1.16b
+	aesmc		v0.16b, v0.16b
+	ld1		{v3.4s}, [x0], #16
+	bpl		1b
+	aese		v0.16b, v2.16b
+	eor		v0.16b, v0.16b, v3.16b
+	st1		{v0.16b}, [x1]
+	ret
+SYM_FUNC_END(__aes_ce_encrypt)
+
+SYM_FUNC_START(__aes_ce_decrypt)
+	sub		w3, w3, #2
+	ld1		{v0.16b}, [x2]
+	ld1		{v1.4s}, [x0], #16
+	cmp		w3, #10
+	bmi		0f
+	bne		3f
+	mov		v3.16b, v1.16b
+	b		2f
+0:	mov		v2.16b, v1.16b
+	ld1		{v3.4s}, [x0], #16
+1:	aesd		v0.16b, v2.16b
+	aesimc		v0.16b, v0.16b
+2:	ld1		{v1.4s}, [x0], #16
+	aesd		v0.16b, v3.16b
+	aesimc		v0.16b, v0.16b
+3:	ld1		{v2.4s}, [x0], #16
+	subs		w3, w3, #3
+	aesd		v0.16b, v1.16b
+	aesimc		v0.16b, v0.16b
+	ld1		{v3.4s}, [x0], #16
+	bpl		1b
+	aesd		v0.16b, v2.16b
+	eor		v0.16b, v0.16b, v3.16b
+	st1		{v0.16b}, [x1]
+	ret
+SYM_FUNC_END(__aes_ce_decrypt)
+
+/*
+ * __aes_ce_sub() - use the aese instruction to perform the AES sbox
+ *                  substitution on each byte in 'input'
+ */
+SYM_FUNC_START(__aes_ce_sub)
+	dup		v1.4s, w0
+	movi		v0.16b, #0
+	aese		v0.16b, v1.16b
+	umov		w0, v0.s[0]
+	ret
+SYM_FUNC_END(__aes_ce_sub)
+
+SYM_FUNC_START(__aes_ce_invert)
+	ld1		{v0.4s}, [x1]
+	aesimc		v1.16b, v0.16b
+	st1		{v1.4s}, [x0]
+	ret
+SYM_FUNC_END(__aes_ce_invert)
diff --git a/lib/crypto/arm64/aes-cipher-core.S b/lib/crypto/arm64/aes-cipher-core.S
new file mode 100644
index 000000000000..651f701c56a8
--- /dev/null
+++ b/lib/crypto/arm64/aes-cipher-core.S
@@ -0,0 +1,132 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Scalar AES core transform
+ *
+ * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+#include <asm/cache.h>
+
+	.text
+
+	rk		.req	x0
+	out		.req	x1
+	in		.req	x2
+	rounds		.req	x3
+	tt		.req	x2
+
+	.macro		__pair1, sz, op, reg0, reg1, in0, in1e, in1d, shift
+	.ifc		\op\shift, b0
+	ubfiz		\reg0, \in0, #2, #8
+	ubfiz		\reg1, \in1e, #2, #8
+	.else
+	ubfx		\reg0, \in0, #\shift, #8
+	ubfx		\reg1, \in1e, #\shift, #8
+	.endif
+
+	/*
+	 * AArch64 cannot do byte size indexed loads from a table containing
+	 * 32-bit quantities, i.e., 'ldrb w12, [tt, w12, uxtw #2]' is not a
+	 * valid instruction. So perform the shift explicitly first for the
+	 * high bytes (the low byte is shifted implicitly by using ubfiz rather
+	 * than ubfx above)
+	 */
+	.ifnc		\op, b
+	ldr		\reg0, [tt, \reg0, uxtw #2]
+	ldr		\reg1, [tt, \reg1, uxtw #2]
+	.else
+	.if		\shift > 0
+	lsl		\reg0, \reg0, #2
+	lsl		\reg1, \reg1, #2
+	.endif
+	ldrb		\reg0, [tt, \reg0, uxtw]
+	ldrb		\reg1, [tt, \reg1, uxtw]
+	.endif
+	.endm
+
+	.macro		__pair0, sz, op, reg0, reg1, in0, in1e, in1d, shift
+	ubfx		\reg0, \in0, #\shift, #8
+	ubfx		\reg1, \in1d, #\shift, #8
+	ldr\op		\reg0, [tt, \reg0, uxtw #\sz]
+	ldr\op		\reg1, [tt, \reg1, uxtw #\sz]
+	.endm
+
+	.macro		__hround, out0, out1, in0, in1, in2, in3, t0, t1, enc, sz, op
+	ldp		\out0, \out1, [rk], #8
+
+	__pair\enc	\sz, \op, w12, w13, \in0, \in1, \in3, 0
+	__pair\enc	\sz, \op, w14, w15, \in1, \in2, \in0, 8
+	__pair\enc	\sz, \op, w16, w17, \in2, \in3, \in1, 16
+	__pair\enc	\sz, \op, \t0, \t1, \in3, \in0, \in2, 24
+
+	eor		\out0, \out0, w12
+	eor		\out1, \out1, w13
+	eor		\out0, \out0, w14, ror #24
+	eor		\out1, \out1, w15, ror #24
+	eor		\out0, \out0, w16, ror #16
+	eor		\out1, \out1, w17, ror #16
+	eor		\out0, \out0, \t0, ror #8
+	eor		\out1, \out1, \t1, ror #8
+	.endm
+
+	.macro		fround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op
+	__hround	\out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1, \sz, \op
+	__hround	\out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1, \sz, \op
+	.endm
+
+	.macro		iround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op
+	__hround	\out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0, \sz, \op
+	__hround	\out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0, \sz, \op
+	.endm
+
+	.macro		do_crypt, round, ttab, ltab, bsz
+	ldp		w4, w5, [in]
+	ldp		w6, w7, [in, #8]
+	ldp		w8, w9, [rk], #16
+	ldp		w10, w11, [rk, #-8]
+
+CPU_BE(	rev		w4, w4		)
+CPU_BE(	rev		w5, w5		)
+CPU_BE(	rev		w6, w6		)
+CPU_BE(	rev		w7, w7		)
+
+	eor		w4, w4, w8
+	eor		w5, w5, w9
+	eor		w6, w6, w10
+	eor		w7, w7, w11
+
+	adr_l		tt, \ttab
+
+	tbnz		rounds, #1, 1f
+
+0:	\round		w8, w9, w10, w11, w4, w5, w6, w7
+	\round		w4, w5, w6, w7, w8, w9, w10, w11
+
+1:	subs		rounds, rounds, #4
+	\round		w8, w9, w10, w11, w4, w5, w6, w7
+	b.ls		3f
+2:	\round		w4, w5, w6, w7, w8, w9, w10, w11
+	b		0b
+3:	adr_l		tt, \ltab
+	\round		w4, w5, w6, w7, w8, w9, w10, w11, \bsz, b
+
+CPU_BE(	rev		w4, w4		)
+CPU_BE(	rev		w5, w5		)
+CPU_BE(	rev		w6, w6		)
+CPU_BE(	rev		w7, w7		)
+
+	stp		w4, w5, [out]
+	stp		w6, w7, [out, #8]
+	ret
+	.endm
+
+SYM_FUNC_START(__aes_arm64_encrypt)
+	do_crypt	fround, aes_enc_tab, aes_enc_tab + 1, 2
+SYM_FUNC_END(__aes_arm64_encrypt)
+
+	.align		5
+SYM_FUNC_START(__aes_arm64_decrypt)
+	do_crypt	iround, aes_dec_tab, crypto_aes_inv_sbox, 0
+SYM_FUNC_END(__aes_arm64_decrypt)
diff --git a/lib/crypto/arm64/aes.h b/lib/crypto/arm64/aes.h
new file mode 100644
index 000000000000..63eea6271ef9
--- /dev/null
+++ b/lib/crypto/arm64/aes.h
@@ -0,0 +1,164 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * AES block cipher, optimized for ARM64
+ *
+ * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
+ * Copyright 2026 Google LLC
+ */
+
+#include <asm/neon.h>
+#include <asm/simd.h>
+#include <linux/unaligned.h>
+#include <linux/cpufeature.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes);
+
+struct aes_block {
+	u8 b[AES_BLOCK_SIZE];
+};
+
+asmlinkage void __aes_arm64_encrypt(const u32 rk[], u8 out[AES_BLOCK_SIZE],
+				    const u8 in[AES_BLOCK_SIZE], int rounds);
+asmlinkage void __aes_arm64_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE],
+				    const u8 in[AES_BLOCK_SIZE], int rounds);
+asmlinkage void __aes_ce_encrypt(const u32 rk[], u8 out[AES_BLOCK_SIZE],
+				 const u8 in[AES_BLOCK_SIZE], int rounds);
+asmlinkage void __aes_ce_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE],
+				 const u8 in[AES_BLOCK_SIZE], int rounds);
+asmlinkage u32 __aes_ce_sub(u32 l);
+asmlinkage void __aes_ce_invert(struct aes_block *out,
+				const struct aes_block *in);
+
+/*
+ * Expand an AES key using the crypto extensions if supported and usable or
+ * generic code otherwise.  The expanded key format is compatible between the
+ * two cases.  The outputs are @rndkeys (required) and @inv_rndkeys (optional).
+ */
+static void aes_expandkey_arm64(u32 rndkeys[], u32 *inv_rndkeys,
+				const u8 *in_key, int key_len, int nrounds)
+{
+	/*
+	 * The AES key schedule round constants
+	 */
+	static u8 const rcon[] = {
+		0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,
+	};
+
+	u32 kwords = key_len / sizeof(u32);
+	struct aes_block *key_enc, *key_dec;
+	int i, j;
+
+	if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) ||
+	    !static_branch_likely(&have_aes) || unlikely(!may_use_simd())) {
+		aes_expandkey_generic(rndkeys, inv_rndkeys, in_key, key_len);
+		return;
+	}
+
+	for (i = 0; i < kwords; i++)
+		rndkeys[i] = get_unaligned_le32(&in_key[i * sizeof(u32)]);
+
+	scoped_ksimd() {
+		for (i = 0; i < sizeof(rcon); i++) {
+			u32 *rki = &rndkeys[i * kwords];
+			u32 *rko = rki + kwords;
+
+			rko[0] = ror32(__aes_ce_sub(rki[kwords - 1]), 8) ^
+				 rcon[i] ^ rki[0];
+			rko[1] = rko[0] ^ rki[1];
+			rko[2] = rko[1] ^ rki[2];
+			rko[3] = rko[2] ^ rki[3];
+
+			if (key_len == AES_KEYSIZE_192) {
+				if (i >= 7)
+					break;
+				rko[4] = rko[3] ^ rki[4];
+				rko[5] = rko[4] ^ rki[5];
+			} else if (key_len == AES_KEYSIZE_256) {
+				if (i >= 6)
+					break;
+				rko[4] = __aes_ce_sub(rko[3]) ^ rki[4];
+				rko[5] = rko[4] ^ rki[5];
+				rko[6] = rko[5] ^ rki[6];
+				rko[7] = rko[6] ^ rki[7];
+			}
+		}
+
+		/*
+		 * Generate the decryption keys for the Equivalent Inverse
+		 * Cipher.  This involves reversing the order of the round
+		 * keys, and applying the Inverse Mix Columns transformation on
+		 * all but the first and the last one.
+		 */
+		if (inv_rndkeys) {
+			key_enc = (struct aes_block *)rndkeys;
+			key_dec = (struct aes_block *)inv_rndkeys;
+			j = nrounds;
+
+			key_dec[0] = key_enc[j];
+			for (i = 1, j--; j > 0; i++, j--)
+				__aes_ce_invert(key_dec + i, key_enc + j);
+			key_dec[i] = key_enc[0];
+		}
+	}
+}
+
+static void aes_preparekey_arch(union aes_enckey_arch *k,
+				union aes_invkey_arch *inv_k,
+				const u8 *in_key, int key_len, int nrounds)
+{
+	aes_expandkey_arm64(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL,
+			    in_key, key_len, nrounds);
+}
+
+/*
+ * This is here temporarily until the remaining AES mode implementations are
+ * migrated from arch/arm64/crypto/ to lib/crypto/arm64/.
+ */
+int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
+		     unsigned int key_len)
+{
+	if (aes_check_keylen(key_len) != 0)
+		return -EINVAL;
+	ctx->key_length = key_len;
+	aes_expandkey_arm64(ctx->key_enc, ctx->key_dec, in_key, key_len,
+			    6 + key_len / 4);
+	return 0;
+}
+EXPORT_SYMBOL(ce_aes_expandkey);
+
+static void aes_encrypt_arch(const struct aes_enckey *key,
+			     u8 out[AES_BLOCK_SIZE],
+			     const u8 in[AES_BLOCK_SIZE])
+{
+	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
+	    static_branch_likely(&have_aes) && likely(may_use_simd())) {
+		scoped_ksimd()
+			__aes_ce_encrypt(key->k.rndkeys, out, in, key->nrounds);
+	} else {
+		__aes_arm64_encrypt(key->k.rndkeys, out, in, key->nrounds);
+	}
+}
+
+static void aes_decrypt_arch(const struct aes_key *key,
+			     u8 out[AES_BLOCK_SIZE],
+			     const u8 in[AES_BLOCK_SIZE])
+{
+	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
+	    static_branch_likely(&have_aes) && likely(may_use_simd())) {
+		scoped_ksimd()
+			__aes_ce_decrypt(key->inv_k.inv_rndkeys, out, in,
+					 key->nrounds);
+	} else {
+		__aes_arm64_decrypt(key->inv_k.inv_rndkeys, out, in,
+				    key->nrounds);
+	}
+}
+
+#ifdef CONFIG_KERNEL_MODE_NEON
+#define aes_mod_init_arch aes_mod_init_arch
+static void aes_mod_init_arch(void)
+{
+	if (cpu_have_named_feature(AES))
+		static_branch_enable(&have_aes);
+}
+#endif /* CONFIG_KERNEL_MODE_NEON */
diff --git a/lib/crypto/arm64/chacha.h b/lib/crypto/arm64/chacha.h
index ba6c22d46086..ca8c6a8b0578 100644
--- a/lib/crypto/arm64/chacha.h
+++ b/lib/crypto/arm64/chacha.h
@@ -23,7 +23,6 @@
 #include <linux/kernel.h>
 
 #include <asm/hwcap.h>
-#include <asm/neon.h>
 #include <asm/simd.h>
 
 asmlinkage void chacha_block_xor_neon(const struct chacha_state *state,
@@ -65,9 +64,8 @@ static void hchacha_block_arch(const struct chacha_state *state,
 	if (!static_branch_likely(&have_neon) || !crypto_simd_usable()) {
 		hchacha_block_generic(state, out, nrounds);
 	} else {
-		kernel_neon_begin();
-		hchacha_block_neon(state, out, nrounds);
-		kernel_neon_end();
+		scoped_ksimd()
+			hchacha_block_neon(state, out, nrounds);
 	}
 }
 
@@ -81,9 +79,8 @@ static void chacha_crypt_arch(struct chacha_state *state, u8 *dst,
 	do {
 		unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
 
-		kernel_neon_begin();
-		chacha_doneon(state, dst, src, todo, nrounds);
-		kernel_neon_end();
+		scoped_ksimd()
+			chacha_doneon(state, dst, src, todo, nrounds);
 
 		bytes -= todo;
 		src += todo;
diff --git a/lib/crypto/arm64/nh-neon-core.S b/lib/crypto/arm64/nh-neon-core.S
new file mode 100644
index 000000000000..6fa57fce8085
--- /dev/null
+++ b/lib/crypto/arm64/nh-neon-core.S
@@ -0,0 +1,103 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * NH - ε-almost-universal hash function, ARM64 NEON accelerated version
+ *
+ * Copyright 2018 Google LLC
+ *
+ * Author: Eric Biggers <ebiggers@google.com>
+ */
+
+#include <linux/linkage.h>
+
+	KEY		.req	x0
+	MESSAGE		.req	x1
+	MESSAGE_LEN	.req	x2
+	HASH		.req	x3
+
+	PASS0_SUMS	.req	v0
+	PASS1_SUMS	.req	v1
+	PASS2_SUMS	.req	v2
+	PASS3_SUMS	.req	v3
+	K0		.req	v4
+	K1		.req	v5
+	K2		.req	v6
+	K3		.req	v7
+	T0		.req	v8
+	T1		.req	v9
+	T2		.req	v10
+	T3		.req	v11
+	T4		.req	v12
+	T5		.req	v13
+	T6		.req	v14
+	T7		.req	v15
+
+.macro _nh_stride	k0, k1, k2, k3
+
+	// Load next message stride
+	ld1		{T3.16b}, [MESSAGE], #16
+
+	// Load next key stride
+	ld1		{\k3\().4s}, [KEY], #16
+
+	// Add message words to key words
+	add		T0.4s, T3.4s, \k0\().4s
+	add		T1.4s, T3.4s, \k1\().4s
+	add		T2.4s, T3.4s, \k2\().4s
+	add		T3.4s, T3.4s, \k3\().4s
+
+	// Multiply 32x32 => 64 and accumulate
+	mov		T4.d[0], T0.d[1]
+	mov		T5.d[0], T1.d[1]
+	mov		T6.d[0], T2.d[1]
+	mov		T7.d[0], T3.d[1]
+	umlal		PASS0_SUMS.2d, T0.2s, T4.2s
+	umlal		PASS1_SUMS.2d, T1.2s, T5.2s
+	umlal		PASS2_SUMS.2d, T2.2s, T6.2s
+	umlal		PASS3_SUMS.2d, T3.2s, T7.2s
+.endm
+
+/*
+ * void nh_neon(const u32 *key, const u8 *message, size_t message_len,
+ *		__le64 hash[NH_NUM_PASSES])
+ *
+ * It's guaranteed that message_len % 16 == 0.
+ */
+SYM_FUNC_START(nh_neon)
+
+	ld1		{K0.4s,K1.4s}, [KEY], #32
+	  movi		PASS0_SUMS.2d, #0
+	  movi		PASS1_SUMS.2d, #0
+	ld1		{K2.4s}, [KEY], #16
+	  movi		PASS2_SUMS.2d, #0
+	  movi		PASS3_SUMS.2d, #0
+
+	subs		MESSAGE_LEN, MESSAGE_LEN, #64
+	blt		.Lloop4_done
+.Lloop4:
+	_nh_stride	K0, K1, K2, K3
+	_nh_stride	K1, K2, K3, K0
+	_nh_stride	K2, K3, K0, K1
+	_nh_stride	K3, K0, K1, K2
+	subs		MESSAGE_LEN, MESSAGE_LEN, #64
+	bge		.Lloop4
+
+.Lloop4_done:
+	ands		MESSAGE_LEN, MESSAGE_LEN, #63
+	beq		.Ldone
+	_nh_stride	K0, K1, K2, K3
+
+	subs		MESSAGE_LEN, MESSAGE_LEN, #16
+	beq		.Ldone
+	_nh_stride	K1, K2, K3, K0
+
+	subs		MESSAGE_LEN, MESSAGE_LEN, #16
+	beq		.Ldone
+	_nh_stride	K2, K3, K0, K1
+
+.Ldone:
+	// Sum the accumulators for each pass, then store the sums to 'hash'
+	addp		T0.2d, PASS0_SUMS.2d, PASS1_SUMS.2d
+	addp		T1.2d, PASS2_SUMS.2d, PASS3_SUMS.2d
+	st1		{T0.16b,T1.16b}, [HASH]
+	ret
+SYM_FUNC_END(nh_neon)
diff --git a/lib/crypto/arm64/nh.h b/lib/crypto/arm64/nh.h
new file mode 100644
index 000000000000..08902630bdd1
--- /dev/null
+++ b/lib/crypto/arm64/nh.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * ARM64 accelerated implementation of NH
+ *
+ * Copyright 2018 Google LLC
+ */
+
+#include <asm/hwcap.h>
+#include <asm/simd.h>
+#include <linux/cpufeature.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
+
+asmlinkage void nh_neon(const u32 *key, const u8 *message, size_t message_len,
+			__le64 hash[NH_NUM_PASSES]);
+
+static bool nh_arch(const u32 *key, const u8 *message, size_t message_len,
+		    __le64 hash[NH_NUM_PASSES])
+{
+	if (static_branch_likely(&have_neon) && message_len >= 64 &&
+	    may_use_simd()) {
+		scoped_ksimd()
+			nh_neon(key, message, message_len, hash);
+		return true;
+	}
+	return false;
+}
+
+#define nh_mod_init_arch nh_mod_init_arch
+static void nh_mod_init_arch(void)
+{
+	if (cpu_have_named_feature(ASIMD))
+		static_branch_enable(&have_neon);
+}
diff --git a/lib/crypto/arm64/poly1305.h b/lib/crypto/arm64/poly1305.h
index aed5921ccd9a..b77669767cd6 100644
--- a/lib/crypto/arm64/poly1305.h
+++ b/lib/crypto/arm64/poly1305.h
@@ -6,7 +6,6 @@
  */
 
 #include <asm/hwcap.h>
-#include <asm/neon.h>
 #include <asm/simd.h>
 #include <linux/cpufeature.h>
 #include <linux/jump_label.h>
@@ -31,9 +30,8 @@ static void poly1305_blocks(struct poly1305_block_state *state, const u8 *src,
 		do {
 			unsigned int todo = min_t(unsigned int, len, SZ_4K);
 
-			kernel_neon_begin();
-			poly1305_blocks_neon(state, src, todo, padbit);
-			kernel_neon_end();
+			scoped_ksimd()
+				poly1305_blocks_neon(state, src, todo, padbit);
 
 			len -= todo;
 			src += todo;
diff --git a/lib/crypto/arm64/polyval-ce-core.S b/lib/crypto/arm64/polyval-ce-core.S
new file mode 100644
index 000000000000..7c731a044d02
--- /dev/null
+++ b/lib/crypto/arm64/polyval-ce-core.S
@@ -0,0 +1,359 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Implementation of POLYVAL using ARMv8 Crypto Extensions.
+ *
+ * Copyright 2021 Google LLC
+ */
+/*
+ * This is an efficient implementation of POLYVAL using ARMv8 Crypto Extensions
+ * It works on 8 blocks at a time, by precomputing the first 8 keys powers h^8,
+ * ..., h^1 in the POLYVAL finite field. This precomputation allows us to split
+ * finite field multiplication into two steps.
+ *
+ * In the first step, we consider h^i, m_i as normal polynomials of degree less
+ * than 128. We then compute p(x) = h^8m_0 + ... + h^1m_7 where multiplication
+ * is simply polynomial multiplication.
+ *
+ * In the second step, we compute the reduction of p(x) modulo the finite field
+ * modulus g(x) = x^128 + x^127 + x^126 + x^121 + 1.
+ *
+ * This two step process is equivalent to computing h^8m_0 + ... + h^1m_7 where
+ * multiplication is finite field multiplication. The advantage is that the
+ * two-step process  only requires 1 finite field reduction for every 8
+ * polynomial multiplications. Further parallelism is gained by interleaving the
+ * multiplications and polynomial reductions.
+ */
+
+#include <linux/linkage.h>
+#define STRIDE_BLOCKS 8
+
+ACCUMULATOR	.req	x0
+KEY_POWERS	.req	x1
+MSG		.req	x2
+BLOCKS_LEFT	.req	x3
+KEY_START	.req	x10
+EXTRA_BYTES	.req	x11
+TMP	.req	x13
+
+M0	.req	v0
+M1	.req	v1
+M2	.req	v2
+M3	.req	v3
+M4	.req	v4
+M5	.req	v5
+M6	.req	v6
+M7	.req	v7
+KEY8	.req	v8
+KEY7	.req	v9
+KEY6	.req	v10
+KEY5	.req	v11
+KEY4	.req	v12
+KEY3	.req	v13
+KEY2	.req	v14
+KEY1	.req	v15
+PL	.req	v16
+PH	.req	v17
+TMP_V	.req	v18
+LO	.req	v20
+MI	.req	v21
+HI	.req	v22
+SUM	.req	v23
+GSTAR	.req	v24
+
+	.text
+
+	.arch	armv8-a+crypto
+	.align	4
+
+.Lgstar:
+	.quad	0xc200000000000000, 0xc200000000000000
+
+/*
+ * Computes the product of two 128-bit polynomials in X and Y and XORs the
+ * components of the 256-bit product into LO, MI, HI.
+ *
+ * Given:
+ *  X = [X_1 : X_0]
+ *  Y = [Y_1 : Y_0]
+ *
+ * We compute:
+ *  LO += X_0 * Y_0
+ *  MI += (X_0 + X_1) * (Y_0 + Y_1)
+ *  HI += X_1 * Y_1
+ *
+ * Later, the 256-bit result can be extracted as:
+ *   [HI_1 : HI_0 + HI_1 + MI_1 + LO_1 : LO_1 + HI_0 + MI_0 + LO_0 : LO_0]
+ * This step is done when computing the polynomial reduction for efficiency
+ * reasons.
+ *
+ * Karatsuba multiplication is used instead of Schoolbook multiplication because
+ * it was found to be slightly faster on ARM64 CPUs.
+ *
+ */
+.macro karatsuba1 X Y
+	X .req \X
+	Y .req \Y
+	ext	v25.16b, X.16b, X.16b, #8
+	ext	v26.16b, Y.16b, Y.16b, #8
+	eor	v25.16b, v25.16b, X.16b
+	eor	v26.16b, v26.16b, Y.16b
+	pmull2	v28.1q, X.2d, Y.2d
+	pmull	v29.1q, X.1d, Y.1d
+	pmull	v27.1q, v25.1d, v26.1d
+	eor	HI.16b, HI.16b, v28.16b
+	eor	LO.16b, LO.16b, v29.16b
+	eor	MI.16b, MI.16b, v27.16b
+	.unreq X
+	.unreq Y
+.endm
+
+/*
+ * Same as karatsuba1, except overwrites HI, LO, MI rather than XORing into
+ * them.
+ */
+.macro karatsuba1_store X Y
+	X .req \X
+	Y .req \Y
+	ext	v25.16b, X.16b, X.16b, #8
+	ext	v26.16b, Y.16b, Y.16b, #8
+	eor	v25.16b, v25.16b, X.16b
+	eor	v26.16b, v26.16b, Y.16b
+	pmull2	HI.1q, X.2d, Y.2d
+	pmull	LO.1q, X.1d, Y.1d
+	pmull	MI.1q, v25.1d, v26.1d
+	.unreq X
+	.unreq Y
+.endm
+
+/*
+ * Computes the 256-bit polynomial represented by LO, HI, MI. Stores
+ * the result in PL, PH.
+ * [PH : PL] =
+ *   [HI_1 : HI_1 + HI_0 + MI_1 + LO_1 : HI_0 + MI_0 + LO_1 + LO_0 : LO_0]
+ */
+.macro karatsuba2
+	// v4 = [HI_1 + MI_1 : HI_0 + MI_0]
+	eor	v4.16b, HI.16b, MI.16b
+	// v4 = [HI_1 + MI_1 + LO_1 : HI_0 + MI_0 + LO_0]
+	eor	v4.16b, v4.16b, LO.16b
+	// v5 = [HI_0 : LO_1]
+	ext	v5.16b, LO.16b, HI.16b, #8
+	// v4 = [HI_1 + HI_0 + MI_1 + LO_1 : HI_0 + MI_0 + LO_1 + LO_0]
+	eor	v4.16b, v4.16b, v5.16b
+	// HI = [HI_0 : HI_1]
+	ext	HI.16b, HI.16b, HI.16b, #8
+	// LO = [LO_0 : LO_1]
+	ext	LO.16b, LO.16b, LO.16b, #8
+	// PH = [HI_1 : HI_1 + HI_0 + MI_1 + LO_1]
+	ext	PH.16b, v4.16b, HI.16b, #8
+	// PL = [HI_0 + MI_0 + LO_1 + LO_0 : LO_0]
+	ext	PL.16b, LO.16b, v4.16b, #8
+.endm
+
+/*
+ * Computes the 128-bit reduction of PH : PL. Stores the result in dest.
+ *
+ * This macro computes p(x) mod g(x) where p(x) is in montgomery form and g(x) =
+ * x^128 + x^127 + x^126 + x^121 + 1.
+ *
+ * We have a 256-bit polynomial PH : PL = P_3 : P_2 : P_1 : P_0 that is the
+ * product of two 128-bit polynomials in Montgomery form.  We need to reduce it
+ * mod g(x).  Also, since polynomials in Montgomery form have an "extra" factor
+ * of x^128, this product has two extra factors of x^128.  To get it back into
+ * Montgomery form, we need to remove one of these factors by dividing by x^128.
+ *
+ * To accomplish both of these goals, we add multiples of g(x) that cancel out
+ * the low 128 bits P_1 : P_0, leaving just the high 128 bits. Since the low
+ * bits are zero, the polynomial division by x^128 can be done by right
+ * shifting.
+ *
+ * Since the only nonzero term in the low 64 bits of g(x) is the constant term,
+ * the multiple of g(x) needed to cancel out P_0 is P_0 * g(x).  The CPU can
+ * only do 64x64 bit multiplications, so split P_0 * g(x) into x^128 * P_0 +
+ * x^64 * g*(x) * P_0 + P_0, where g*(x) is bits 64-127 of g(x).  Adding this to
+ * the original polynomial gives P_3 : P_2 + P_0 + T_1 : P_1 + T_0 : 0, where T
+ * = T_1 : T_0 = g*(x) * P_0.  Thus, bits 0-63 got "folded" into bits 64-191.
+ *
+ * Repeating this same process on the next 64 bits "folds" bits 64-127 into bits
+ * 128-255, giving the answer in bits 128-255. This time, we need to cancel P_1
+ * + T_0 in bits 64-127. The multiple of g(x) required is (P_1 + T_0) * g(x) *
+ * x^64. Adding this to our previous computation gives P_3 + P_1 + T_0 + V_1 :
+ * P_2 + P_0 + T_1 + V_0 : 0 : 0, where V = V_1 : V_0 = g*(x) * (P_1 + T_0).
+ *
+ * So our final computation is:
+ *   T = T_1 : T_0 = g*(x) * P_0
+ *   V = V_1 : V_0 = g*(x) * (P_1 + T_0)
+ *   p(x) / x^{128} mod g(x) = P_3 + P_1 + T_0 + V_1 : P_2 + P_0 + T_1 + V_0
+ *
+ * The implementation below saves a XOR instruction by computing P_1 + T_0 : P_0
+ * + T_1 and XORing into dest, rather than separately XORing P_1 : P_0 and T_0 :
+ * T_1 into dest.  This allows us to reuse P_1 + T_0 when computing V.
+ */
+.macro montgomery_reduction dest
+	DEST .req \dest
+	// TMP_V = T_1 : T_0 = P_0 * g*(x)
+	pmull	TMP_V.1q, PL.1d, GSTAR.1d
+	// TMP_V = T_0 : T_1
+	ext	TMP_V.16b, TMP_V.16b, TMP_V.16b, #8
+	// TMP_V = P_1 + T_0 : P_0 + T_1
+	eor	TMP_V.16b, PL.16b, TMP_V.16b
+	// PH = P_3 + P_1 + T_0 : P_2 + P_0 + T_1
+	eor	PH.16b, PH.16b, TMP_V.16b
+	// TMP_V = V_1 : V_0 = (P_1 + T_0) * g*(x)
+	pmull2	TMP_V.1q, TMP_V.2d, GSTAR.2d
+	eor	DEST.16b, PH.16b, TMP_V.16b
+	.unreq DEST
+.endm
+
+/*
+ * Compute Polyval on 8 blocks.
+ *
+ * If reduce is set, also computes the montgomery reduction of the
+ * previous full_stride call and XORs with the first message block.
+ * (m_0 + REDUCE(PL, PH))h^8 + ... + m_7h^1.
+ * I.e., the first multiplication uses m_0 + REDUCE(PL, PH) instead of m_0.
+ *
+ * Sets PL, PH.
+ */
+.macro full_stride reduce
+	eor		LO.16b, LO.16b, LO.16b
+	eor		MI.16b, MI.16b, MI.16b
+	eor		HI.16b, HI.16b, HI.16b
+
+	ld1		{M0.16b, M1.16b, M2.16b, M3.16b}, [MSG], #64
+	ld1		{M4.16b, M5.16b, M6.16b, M7.16b}, [MSG], #64
+
+	karatsuba1 M7 KEY1
+	.if \reduce
+	pmull	TMP_V.1q, PL.1d, GSTAR.1d
+	.endif
+
+	karatsuba1 M6 KEY2
+	.if \reduce
+	ext	TMP_V.16b, TMP_V.16b, TMP_V.16b, #8
+	.endif
+
+	karatsuba1 M5 KEY3
+	.if \reduce
+	eor	TMP_V.16b, PL.16b, TMP_V.16b
+	.endif
+
+	karatsuba1 M4 KEY4
+	.if \reduce
+	eor	PH.16b, PH.16b, TMP_V.16b
+	.endif
+
+	karatsuba1 M3 KEY5
+	.if \reduce
+	pmull2	TMP_V.1q, TMP_V.2d, GSTAR.2d
+	.endif
+
+	karatsuba1 M2 KEY6
+	.if \reduce
+	eor	SUM.16b, PH.16b, TMP_V.16b
+	.endif
+
+	karatsuba1 M1 KEY7
+	eor	M0.16b, M0.16b, SUM.16b
+
+	karatsuba1 M0 KEY8
+	karatsuba2
+.endm
+
+/*
+ * Handle any extra blocks after full_stride loop.
+ */
+.macro partial_stride
+	add	KEY_POWERS, KEY_START, #(STRIDE_BLOCKS << 4)
+	sub	KEY_POWERS, KEY_POWERS, BLOCKS_LEFT, lsl #4
+	ld1	{KEY1.16b}, [KEY_POWERS], #16
+
+	ld1	{TMP_V.16b}, [MSG], #16
+	eor	SUM.16b, SUM.16b, TMP_V.16b
+	karatsuba1_store KEY1 SUM
+	sub	BLOCKS_LEFT, BLOCKS_LEFT, #1
+
+	tst	BLOCKS_LEFT, #4
+	beq	.Lpartial4BlocksDone
+	ld1	{M0.16b, M1.16b,  M2.16b, M3.16b}, [MSG], #64
+	ld1	{KEY8.16b, KEY7.16b, KEY6.16b,	KEY5.16b}, [KEY_POWERS], #64
+	karatsuba1 M0 KEY8
+	karatsuba1 M1 KEY7
+	karatsuba1 M2 KEY6
+	karatsuba1 M3 KEY5
+.Lpartial4BlocksDone:
+	tst	BLOCKS_LEFT, #2
+	beq	.Lpartial2BlocksDone
+	ld1	{M0.16b, M1.16b}, [MSG], #32
+	ld1	{KEY8.16b, KEY7.16b}, [KEY_POWERS], #32
+	karatsuba1 M0 KEY8
+	karatsuba1 M1 KEY7
+.Lpartial2BlocksDone:
+	tst	BLOCKS_LEFT, #1
+	beq	.LpartialDone
+	ld1	{M0.16b}, [MSG], #16
+	ld1	{KEY8.16b}, [KEY_POWERS], #16
+	karatsuba1 M0 KEY8
+.LpartialDone:
+	karatsuba2
+	montgomery_reduction SUM
+.endm
+
+/*
+ * Computes a = a * b * x^{-128} mod x^128 + x^127 + x^126 + x^121 + 1.
+ *
+ * void polyval_mul_pmull(struct polyval_elem *a,
+ *			  const struct polyval_elem *b);
+ */
+SYM_FUNC_START(polyval_mul_pmull)
+	adr	TMP, .Lgstar
+	ld1	{GSTAR.2d}, [TMP]
+	ld1	{v0.16b}, [x0]
+	ld1	{v1.16b}, [x1]
+	karatsuba1_store v0 v1
+	karatsuba2
+	montgomery_reduction SUM
+	st1	{SUM.16b}, [x0]
+	ret
+SYM_FUNC_END(polyval_mul_pmull)
+
+/*
+ * Perform polynomial evaluation as specified by POLYVAL.  This computes:
+ *	h^n * accumulator + h^n * m_0 + ... + h^1 * m_{n-1}
+ * where n=nblocks, h is the hash key, and m_i are the message blocks.
+ *
+ * x0 - pointer to accumulator
+ * x1 - pointer to precomputed key powers h^8 ... h^1
+ * x2 - pointer to message blocks
+ * x3 - number of blocks to hash
+ *
+ * void polyval_blocks_pmull(struct polyval_elem *acc,
+ *			     const struct polyval_key *key,
+ *			     const u8 *data, size_t nblocks);
+ */
+SYM_FUNC_START(polyval_blocks_pmull)
+	adr	TMP, .Lgstar
+	mov	KEY_START, KEY_POWERS
+	ld1	{GSTAR.2d}, [TMP]
+	ld1	{SUM.16b}, [ACCUMULATOR]
+	subs	BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
+	blt .LstrideLoopExit
+	ld1	{KEY8.16b, KEY7.16b, KEY6.16b, KEY5.16b}, [KEY_POWERS], #64
+	ld1	{KEY4.16b, KEY3.16b, KEY2.16b, KEY1.16b}, [KEY_POWERS], #64
+	full_stride 0
+	subs	BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
+	blt .LstrideLoopExitReduce
+.LstrideLoop:
+	full_stride 1
+	subs	BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
+	bge	.LstrideLoop
+.LstrideLoopExitReduce:
+	montgomery_reduction SUM
+.LstrideLoopExit:
+	adds	BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
+	beq	.LskipPartial
+	partial_stride
+.LskipPartial:
+	st1	{SUM.16b}, [ACCUMULATOR]
+	ret
+SYM_FUNC_END(polyval_blocks_pmull)
diff --git a/lib/crypto/arm64/polyval.h b/lib/crypto/arm64/polyval.h
new file mode 100644
index 000000000000..a39763395e9b
--- /dev/null
+++ b/lib/crypto/arm64/polyval.h
@@ -0,0 +1,80 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * POLYVAL library functions, arm64 optimized
+ *
+ * Copyright 2025 Google LLC
+ */
+#include <asm/simd.h>
+#include <linux/cpufeature.h>
+
+#define NUM_H_POWERS 8
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
+
+asmlinkage void polyval_mul_pmull(struct polyval_elem *a,
+				  const struct polyval_elem *b);
+asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc,
+				     const struct polyval_key *key,
+				     const u8 *data, size_t nblocks);
+
+static void polyval_preparekey_arch(struct polyval_key *key,
+				    const u8 raw_key[POLYVAL_BLOCK_SIZE])
+{
+	static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS);
+	memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE);
+	if (static_branch_likely(&have_pmull) && may_use_simd()) {
+		scoped_ksimd() {
+			for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
+				key->h_powers[i] = key->h_powers[i + 1];
+				polyval_mul_pmull(
+					&key->h_powers[i],
+					&key->h_powers[NUM_H_POWERS - 1]);
+			}
+		}
+	} else {
+		for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
+			key->h_powers[i] = key->h_powers[i + 1];
+			polyval_mul_generic(&key->h_powers[i],
+					    &key->h_powers[NUM_H_POWERS - 1]);
+		}
+	}
+}
+
+static void polyval_mul_arch(struct polyval_elem *acc,
+			     const struct polyval_key *key)
+{
+	if (static_branch_likely(&have_pmull) && may_use_simd()) {
+		scoped_ksimd()
+			polyval_mul_pmull(acc, &key->h_powers[NUM_H_POWERS - 1]);
+	} else {
+		polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]);
+	}
+}
+
+static void polyval_blocks_arch(struct polyval_elem *acc,
+				const struct polyval_key *key,
+				const u8 *data, size_t nblocks)
+{
+	if (static_branch_likely(&have_pmull) && may_use_simd()) {
+		do {
+			/* Allow rescheduling every 4 KiB. */
+			size_t n = min_t(size_t, nblocks,
+					 4096 / POLYVAL_BLOCK_SIZE);
+
+			scoped_ksimd()
+				polyval_blocks_pmull(acc, key, data, n);
+			data += n * POLYVAL_BLOCK_SIZE;
+			nblocks -= n;
+		} while (nblocks);
+	} else {
+		polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1],
+				       data, nblocks);
+	}
+}
+
+#define polyval_mod_init_arch polyval_mod_init_arch
+static void polyval_mod_init_arch(void)
+{
+	if (cpu_have_named_feature(PMULL))
+		static_branch_enable(&have_pmull);
+}
diff --git a/lib/crypto/arm64/sha1-ce-core.S b/lib/crypto/arm64/sha1-ce-core.S
index 21efbbafd7d6..8fbd4767f0f0 100644
--- a/lib/crypto/arm64/sha1-ce-core.S
+++ b/lib/crypto/arm64/sha1-ce-core.S
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * sha1-ce-core.S - SHA-1 secure hash using ARMv8 Crypto Extensions
+ * SHA-1 secure hash using ARMv8 Crypto Extensions
  *
  * Copyright (C) 2014 Linaro Ltd <ard.biesheuvel@linaro.org>
  */
diff --git a/lib/crypto/arm64/sha1.h b/lib/crypto/arm64/sha1.h
index aaef4ebfc5e3..bc7071f1be09 100644
--- a/lib/crypto/arm64/sha1.h
+++ b/lib/crypto/arm64/sha1.h
@@ -4,7 +4,6 @@
  *
  * Copyright 2025 Google LLC
  */
-#include <asm/neon.h>
 #include <asm/simd.h>
 #include <linux/cpufeature.h>
 
@@ -20,9 +19,9 @@ static void sha1_blocks(struct sha1_block_state *state,
 		do {
 			size_t rem;
 
-			kernel_neon_begin();
-			rem = __sha1_ce_transform(state, data, nblocks);
-			kernel_neon_end();
+			scoped_ksimd()
+				rem = __sha1_ce_transform(state, data, nblocks);
+
 			data += (nblocks - rem) * SHA1_BLOCK_SIZE;
 			nblocks = rem;
 		} while (nblocks);
diff --git a/lib/crypto/arm64/sha256-ce.S b/lib/crypto/arm64/sha256-ce.S
index 410174ba5237..e4bfe42a61a9 100644
--- a/lib/crypto/arm64/sha256-ce.S
+++ b/lib/crypto/arm64/sha256-ce.S
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * sha2-ce-core.S - core SHA-224/SHA-256 transform using v8 Crypto Extensions
+ * Core SHA-224/SHA-256 transform using v8 Crypto Extensions
  *
  * Copyright (C) 2014 Linaro Ltd <ard.biesheuvel@linaro.org>
  */
diff --git a/lib/crypto/arm64/sha256.h b/lib/crypto/arm64/sha256.h
index 80d06df27d3a..568dff0f276a 100644
--- a/lib/crypto/arm64/sha256.h
+++ b/lib/crypto/arm64/sha256.h
@@ -4,7 +4,6 @@
  *
  * Copyright 2025 Google LLC
  */
-#include <asm/neon.h>
 #include <asm/simd.h>
 #include <linux/cpufeature.h>
 
@@ -27,17 +26,16 @@ static void sha256_blocks(struct sha256_block_state *state,
 			do {
 				size_t rem;
 
-				kernel_neon_begin();
-				rem = __sha256_ce_transform(state,
-							    data, nblocks);
-				kernel_neon_end();
+				scoped_ksimd()
+					rem = __sha256_ce_transform(state, data,
+								    nblocks);
+
 				data += (nblocks - rem) * SHA256_BLOCK_SIZE;
 				nblocks = rem;
 			} while (nblocks);
 		} else {
-			kernel_neon_begin();
-			sha256_block_neon(state, data, nblocks);
-			kernel_neon_end();
+			scoped_ksimd()
+				sha256_block_neon(state, data, nblocks);
 		}
 	} else {
 		sha256_block_data_order(state, data, nblocks);
@@ -66,9 +64,8 @@ static bool sha256_finup_2x_arch(const struct __sha256_ctx *ctx,
 	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
 	    static_branch_likely(&have_ce) && len >= SHA256_BLOCK_SIZE &&
 	    len <= 65536 && likely(may_use_simd())) {
-		kernel_neon_begin();
-		sha256_ce_finup2x(ctx, data1, data2, len, out1, out2);
-		kernel_neon_end();
+		scoped_ksimd()
+			sha256_ce_finup2x(ctx, data1, data2, len, out1, out2);
 		kmsan_unpoison_memory(out1, SHA256_DIGEST_SIZE);
 		kmsan_unpoison_memory(out2, SHA256_DIGEST_SIZE);
 		return true;
diff --git a/lib/crypto/arm64/sha3-ce-core.S b/lib/crypto/arm64/sha3-ce-core.S
new file mode 100644
index 000000000000..ace90b506490
--- /dev/null
+++ b/lib/crypto/arm64/sha3-ce-core.S
@@ -0,0 +1,213 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Core SHA-3 transform using v8.2 Crypto Extensions
+ *
+ * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+	.irp	b,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
+	.set	.Lv\b\().2d, \b
+	.set	.Lv\b\().16b, \b
+	.endr
+
+	/*
+	 * ARMv8.2 Crypto Extensions instructions
+	 */
+	.macro	eor3, rd, rn, rm, ra
+	.inst	0xce000000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16)
+	.endm
+
+	.macro	rax1, rd, rn, rm
+	.inst	0xce608c00 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
+	.endm
+
+	.macro	bcax, rd, rn, rm, ra
+	.inst	0xce200000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16)
+	.endm
+
+	.macro	xar, rd, rn, rm, imm6
+	.inst	0xce800000 | .L\rd | (.L\rn << 5) | ((\imm6) << 10) | (.L\rm << 16)
+	.endm
+
+	/*
+	 * size_t sha3_ce_transform(struct sha3_state *state, const u8 *data,
+	 *			    size_t nblocks, size_t block_size)
+	 *
+	 * block_size is assumed to be one of 72 (SHA3-512), 104 (SHA3-384), 136
+	 * (SHA3-256 and SHAKE256), 144 (SHA3-224), or 168 (SHAKE128).
+	 */
+	.text
+SYM_FUNC_START(sha3_ce_transform)
+	/* load state */
+	add	x8, x0, #32
+	ld1	{ v0.1d- v3.1d}, [x0]
+	ld1	{ v4.1d- v7.1d}, [x8], #32
+	ld1	{ v8.1d-v11.1d}, [x8], #32
+	ld1	{v12.1d-v15.1d}, [x8], #32
+	ld1	{v16.1d-v19.1d}, [x8], #32
+	ld1	{v20.1d-v23.1d}, [x8], #32
+	ld1	{v24.1d}, [x8]
+
+0:	sub	x2, x2, #1
+	mov	w8, #24
+	adr_l	x9, .Lsha3_rcon
+
+	/* load input */
+	ld1	{v25.8b-v28.8b}, [x1], #32
+	ld1	{v29.8b}, [x1], #8
+	eor	v0.8b, v0.8b, v25.8b
+	eor	v1.8b, v1.8b, v26.8b
+	eor	v2.8b, v2.8b, v27.8b
+	eor	v3.8b, v3.8b, v28.8b
+	eor	v4.8b, v4.8b, v29.8b
+
+	ld1	{v25.8b-v28.8b}, [x1], #32
+	eor	v5.8b, v5.8b, v25.8b
+	eor	v6.8b, v6.8b, v26.8b
+	eor	v7.8b, v7.8b, v27.8b
+	eor	v8.8b, v8.8b, v28.8b
+	cmp	x3, #72
+	b.eq	3f	/* SHA3-512 (block_size=72)? */
+
+	ld1	{v25.8b-v28.8b}, [x1], #32
+	eor	v9.8b, v9.8b, v25.8b
+	eor	v10.8b, v10.8b, v26.8b
+	eor	v11.8b, v11.8b, v27.8b
+	eor	v12.8b, v12.8b, v28.8b
+	cmp	x3, #104
+	b.eq	3f	/* SHA3-384 (block_size=104)? */
+
+	ld1	{v25.8b-v28.8b}, [x1], #32
+	eor	v13.8b, v13.8b, v25.8b
+	eor	v14.8b, v14.8b, v26.8b
+	eor	v15.8b, v15.8b, v27.8b
+	eor	v16.8b, v16.8b, v28.8b
+	cmp	x3, #144
+	b.lt	3f	/* SHA3-256 or SHAKE256 (block_size=136)? */
+	b.eq	2f	/* SHA3-224 (block_size=144)? */
+
+	/* SHAKE128 (block_size=168) */
+	ld1	{v25.8b-v28.8b}, [x1], #32
+	eor	v17.8b, v17.8b, v25.8b
+	eor	v18.8b, v18.8b, v26.8b
+	eor	v19.8b, v19.8b, v27.8b
+	eor	v20.8b, v20.8b, v28.8b
+	b	3f
+2:
+	/* SHA3-224 (block_size=144) */
+	ld1	{v25.8b}, [x1], #8
+	eor	v17.8b, v17.8b, v25.8b
+
+3:	sub	w8, w8, #1
+
+	eor3	v29.16b,  v4.16b,  v9.16b, v14.16b
+	eor3	v26.16b,  v1.16b,  v6.16b, v11.16b
+	eor3	v28.16b,  v3.16b,  v8.16b, v13.16b
+	eor3	v25.16b,  v0.16b,  v5.16b, v10.16b
+	eor3	v27.16b,  v2.16b,  v7.16b, v12.16b
+	eor3	v29.16b, v29.16b, v19.16b, v24.16b
+	eor3	v26.16b, v26.16b, v16.16b, v21.16b
+	eor3	v28.16b, v28.16b, v18.16b, v23.16b
+	eor3	v25.16b, v25.16b, v15.16b, v20.16b
+	eor3	v27.16b, v27.16b, v17.16b, v22.16b
+
+	rax1	v30.2d, v29.2d, v26.2d	// bc[0]
+	rax1	v26.2d, v26.2d, v28.2d	// bc[2]
+	rax1	v28.2d, v28.2d, v25.2d	// bc[4]
+	rax1	v25.2d, v25.2d, v27.2d	// bc[1]
+	rax1	v27.2d, v27.2d, v29.2d	// bc[3]
+
+	eor	 v0.16b,  v0.16b, v30.16b
+	xar	 v29.2d,   v1.2d,  v25.2d, (64 - 1)
+	xar	  v1.2d,   v6.2d,  v25.2d, (64 - 44)
+	xar	  v6.2d,   v9.2d,  v28.2d, (64 - 20)
+	xar	  v9.2d,  v22.2d,  v26.2d, (64 - 61)
+	xar	 v22.2d,  v14.2d,  v28.2d, (64 - 39)
+	xar	 v14.2d,  v20.2d,  v30.2d, (64 - 18)
+	xar	 v31.2d,   v2.2d,  v26.2d, (64 - 62)
+	xar	  v2.2d,  v12.2d,  v26.2d, (64 - 43)
+	xar	 v12.2d,  v13.2d,  v27.2d, (64 - 25)
+	xar	 v13.2d,  v19.2d,  v28.2d, (64 - 8)
+	xar	 v19.2d,  v23.2d,  v27.2d, (64 - 56)
+	xar	 v23.2d,  v15.2d,  v30.2d, (64 - 41)
+	xar	 v15.2d,   v4.2d,  v28.2d, (64 - 27)
+	xar	 v28.2d,  v24.2d,  v28.2d, (64 - 14)
+	xar	 v24.2d,  v21.2d,  v25.2d, (64 - 2)
+	xar	  v8.2d,   v8.2d,  v27.2d, (64 - 55)
+	xar	  v4.2d,  v16.2d,  v25.2d, (64 - 45)
+	xar	 v16.2d,   v5.2d,  v30.2d, (64 - 36)
+	xar	  v5.2d,   v3.2d,  v27.2d, (64 - 28)
+	xar	 v27.2d,  v18.2d,  v27.2d, (64 - 21)
+	xar	  v3.2d,  v17.2d,  v26.2d, (64 - 15)
+	xar	 v25.2d,  v11.2d,  v25.2d, (64 - 10)
+	xar	 v26.2d,   v7.2d,  v26.2d, (64 - 6)
+	xar	 v30.2d,  v10.2d,  v30.2d, (64 - 3)
+
+	bcax	v20.16b, v31.16b, v22.16b,  v8.16b
+	bcax	v21.16b,  v8.16b, v23.16b, v22.16b
+	bcax	v22.16b, v22.16b, v24.16b, v23.16b
+	bcax	v23.16b, v23.16b, v31.16b, v24.16b
+	bcax	v24.16b, v24.16b,  v8.16b, v31.16b
+
+	ld1r	{v31.2d}, [x9], #8
+
+	bcax	v17.16b, v25.16b, v19.16b,  v3.16b
+	bcax	v18.16b,  v3.16b, v15.16b, v19.16b
+	bcax	v19.16b, v19.16b, v16.16b, v15.16b
+	bcax	v15.16b, v15.16b, v25.16b, v16.16b
+	bcax	v16.16b, v16.16b,  v3.16b, v25.16b
+
+	bcax	v10.16b, v29.16b, v12.16b, v26.16b
+	bcax	v11.16b, v26.16b, v13.16b, v12.16b
+	bcax	v12.16b, v12.16b, v14.16b, v13.16b
+	bcax	v13.16b, v13.16b, v29.16b, v14.16b
+	bcax	v14.16b, v14.16b, v26.16b, v29.16b
+
+	bcax	 v7.16b, v30.16b,  v9.16b,  v4.16b
+	bcax	 v8.16b,  v4.16b,  v5.16b,  v9.16b
+	bcax	 v9.16b,  v9.16b,  v6.16b,  v5.16b
+	bcax	 v5.16b,  v5.16b, v30.16b,  v6.16b
+	bcax	 v6.16b,  v6.16b,  v4.16b, v30.16b
+
+	bcax	 v3.16b, v27.16b,  v0.16b, v28.16b
+	bcax	 v4.16b, v28.16b,  v1.16b,  v0.16b
+	bcax	 v0.16b,  v0.16b,  v2.16b,  v1.16b
+	bcax	 v1.16b,  v1.16b, v27.16b,  v2.16b
+	bcax	 v2.16b,  v2.16b, v28.16b, v27.16b
+
+	eor	 v0.16b,  v0.16b, v31.16b
+
+	cbnz	w8, 3b
+	cond_yield 4f, x8, x9
+	cbnz	x2, 0b
+
+	/* save state */
+4:	st1	{ v0.1d- v3.1d}, [x0], #32
+	st1	{ v4.1d- v7.1d}, [x0], #32
+	st1	{ v8.1d-v11.1d}, [x0], #32
+	st1	{v12.1d-v15.1d}, [x0], #32
+	st1	{v16.1d-v19.1d}, [x0], #32
+	st1	{v20.1d-v23.1d}, [x0], #32
+	st1	{v24.1d}, [x0]
+	mov	x0, x2
+	ret
+SYM_FUNC_END(sha3_ce_transform)
+
+	.section	".rodata", "a"
+	.align		8
+.Lsha3_rcon:
+	.quad	0x0000000000000001, 0x0000000000008082, 0x800000000000808a
+	.quad	0x8000000080008000, 0x000000000000808b, 0x0000000080000001
+	.quad	0x8000000080008081, 0x8000000000008009, 0x000000000000008a
+	.quad	0x0000000000000088, 0x0000000080008009, 0x000000008000000a
+	.quad	0x000000008000808b, 0x800000000000008b, 0x8000000000008089
+	.quad	0x8000000000008003, 0x8000000000008002, 0x8000000000000080
+	.quad	0x000000000000800a, 0x800000008000000a, 0x8000000080008081
+	.quad	0x8000000000008080, 0x0000000080000001, 0x8000000080008008
diff --git a/lib/crypto/arm64/sha3.h b/lib/crypto/arm64/sha3.h
new file mode 100644
index 000000000000..b602f1b3b282
--- /dev/null
+++ b/lib/crypto/arm64/sha3.h
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <asm/simd.h>
+#include <linux/cpufeature.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3);
+
+asmlinkage size_t sha3_ce_transform(struct sha3_state *state, const u8 *data,
+				    size_t nblocks, size_t block_size);
+
+static void sha3_absorb_blocks(struct sha3_state *state, const u8 *data,
+			       size_t nblocks, size_t block_size)
+{
+	if (static_branch_likely(&have_sha3) && likely(may_use_simd())) {
+		do {
+			size_t rem;
+
+			scoped_ksimd()
+				rem = sha3_ce_transform(state, data, nblocks,
+							block_size);
+			data += (nblocks - rem) * block_size;
+			nblocks = rem;
+		} while (nblocks);
+	} else {
+		sha3_absorb_blocks_generic(state, data, nblocks, block_size);
+	}
+}
+
+static void sha3_keccakf(struct sha3_state *state)
+{
+	if (static_branch_likely(&have_sha3) && likely(may_use_simd())) {
+		/*
+		 * Passing zeroes into sha3_ce_transform() gives the plain
+		 * Keccak-f permutation, which is what we want here.  Any
+		 * supported block size may be used.  Use SHA3_512_BLOCK_SIZE
+		 * since it's the shortest.
+		 */
+		static const u8 zeroes[SHA3_512_BLOCK_SIZE];
+
+		scoped_ksimd()
+			sha3_ce_transform(state, zeroes, 1, sizeof(zeroes));
+	} else {
+		sha3_keccakf_generic(state);
+	}
+}
+
+#define sha3_mod_init_arch sha3_mod_init_arch
+static void sha3_mod_init_arch(void)
+{
+	if (cpu_have_named_feature(SHA3))
+		static_branch_enable(&have_sha3);
+}
diff --git a/lib/crypto/arm64/sha512-ce-core.S b/lib/crypto/arm64/sha512-ce-core.S
index 22f1ded89bc8..ffd51acfd1ee 100644
--- a/lib/crypto/arm64/sha512-ce-core.S
+++ b/lib/crypto/arm64/sha512-ce-core.S
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
- * sha512-ce-core.S - core SHA-384/SHA-512 transform using v8 Crypto Extensions
+ * Core SHA-384/SHA-512 transform using v8 Crypto Extensions
  *
  * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
  *
diff --git a/lib/crypto/arm64/sha512.h b/lib/crypto/arm64/sha512.h
index ddb0d256f73a..7eb7ef04d268 100644
--- a/lib/crypto/arm64/sha512.h
+++ b/lib/crypto/arm64/sha512.h
@@ -4,7 +4,7 @@
  *
  * Copyright 2025 Google LLC
  */
-#include <asm/neon.h>
+
 #include <asm/simd.h>
 #include <linux/cpufeature.h>
 
@@ -24,9 +24,9 @@ static void sha512_blocks(struct sha512_block_state *state,
 		do {
 			size_t rem;
 
-			kernel_neon_begin();
-			rem = __sha512_ce_transform(state, data, nblocks);
-			kernel_neon_end();
+			scoped_ksimd()
+				rem = __sha512_ce_transform(state, data, nblocks);
+
 			data += (nblocks - rem) * SHA512_BLOCK_SIZE;
 			nblocks = rem;
 		} while (nblocks);
diff --git a/lib/crypto/blake2b.c b/lib/crypto/blake2b.c
new file mode 100644
index 000000000000..581b7f8486fa
--- /dev/null
+++ b/lib/crypto/blake2b.c
@@ -0,0 +1,170 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ * Copyright 2025 Google LLC
+ *
+ * This is an implementation of the BLAKE2b hash and PRF functions.
+ *
+ * Information: https://blake2.net/
+ */
+
+#include <crypto/blake2b.h>
+#include <linux/bug.h>
+#include <linux/export.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/string.h>
+#include <linux/unroll.h>
+#include <linux/types.h>
+
+static const u8 blake2b_sigma[12][16] = {
+	{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },
+	{ 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 },
+	{ 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 },
+	{ 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 },
+	{ 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 },
+	{ 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 },
+	{ 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 },
+	{ 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 },
+	{ 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 },
+	{ 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 },
+	{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },
+	{ 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }
+};
+
+static inline void blake2b_increment_counter(struct blake2b_ctx *ctx, u32 inc)
+{
+	ctx->t[0] += inc;
+	ctx->t[1] += (ctx->t[0] < inc);
+}
+
+static void __maybe_unused
+blake2b_compress_generic(struct blake2b_ctx *ctx,
+			 const u8 *data, size_t nblocks, u32 inc)
+{
+	u64 m[16];
+	u64 v[16];
+	int i;
+
+	WARN_ON(IS_ENABLED(DEBUG) &&
+		(nblocks > 1 && inc != BLAKE2B_BLOCK_SIZE));
+
+	while (nblocks > 0) {
+		blake2b_increment_counter(ctx, inc);
+		memcpy(m, data, BLAKE2B_BLOCK_SIZE);
+		le64_to_cpu_array(m, ARRAY_SIZE(m));
+		memcpy(v, ctx->h, 64);
+		v[ 8] = BLAKE2B_IV0;
+		v[ 9] = BLAKE2B_IV1;
+		v[10] = BLAKE2B_IV2;
+		v[11] = BLAKE2B_IV3;
+		v[12] = BLAKE2B_IV4 ^ ctx->t[0];
+		v[13] = BLAKE2B_IV5 ^ ctx->t[1];
+		v[14] = BLAKE2B_IV6 ^ ctx->f[0];
+		v[15] = BLAKE2B_IV7 ^ ctx->f[1];
+
+#define G(r, i, a, b, c, d) do { \
+	a += b + m[blake2b_sigma[r][2 * i + 0]]; \
+	d = ror64(d ^ a, 32); \
+	c += d; \
+	b = ror64(b ^ c, 24); \
+	a += b + m[blake2b_sigma[r][2 * i + 1]]; \
+	d = ror64(d ^ a, 16); \
+	c += d; \
+	b = ror64(b ^ c, 63); \
+} while (0)
+
+#ifdef CONFIG_64BIT
+		/*
+		 * Unroll the rounds loop to enable constant-folding of the
+		 * blake2b_sigma values.  Seems worthwhile on 64-bit kernels.
+		 * Not worthwhile on 32-bit kernels because the code size is
+		 * already so large there due to BLAKE2b using 64-bit words.
+		 */
+		unrolled_full
+#endif
+		for (int r = 0; r < 12; r++) {
+			G(r, 0, v[0], v[4], v[8], v[12]);
+			G(r, 1, v[1], v[5], v[9], v[13]);
+			G(r, 2, v[2], v[6], v[10], v[14]);
+			G(r, 3, v[3], v[7], v[11], v[15]);
+			G(r, 4, v[0], v[5], v[10], v[15]);
+			G(r, 5, v[1], v[6], v[11], v[12]);
+			G(r, 6, v[2], v[7], v[8], v[13]);
+			G(r, 7, v[3], v[4], v[9], v[14]);
+		}
+#undef G
+
+		for (i = 0; i < 8; ++i)
+			ctx->h[i] ^= v[i] ^ v[i + 8];
+
+		data += BLAKE2B_BLOCK_SIZE;
+		--nblocks;
+	}
+}
+
+#ifdef CONFIG_CRYPTO_LIB_BLAKE2B_ARCH
+#include "blake2b.h" /* $(SRCARCH)/blake2b.h */
+#else
+#define blake2b_compress blake2b_compress_generic
+#endif
+
+static inline void blake2b_set_lastblock(struct blake2b_ctx *ctx)
+{
+	ctx->f[0] = -1;
+}
+
+void blake2b_update(struct blake2b_ctx *ctx, const u8 *in, size_t inlen)
+{
+	const size_t fill = BLAKE2B_BLOCK_SIZE - ctx->buflen;
+
+	if (unlikely(!inlen))
+		return;
+	if (inlen > fill) {
+		memcpy(ctx->buf + ctx->buflen, in, fill);
+		blake2b_compress(ctx, ctx->buf, 1, BLAKE2B_BLOCK_SIZE);
+		ctx->buflen = 0;
+		in += fill;
+		inlen -= fill;
+	}
+	if (inlen > BLAKE2B_BLOCK_SIZE) {
+		const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2B_BLOCK_SIZE);
+
+		blake2b_compress(ctx, in, nblocks - 1, BLAKE2B_BLOCK_SIZE);
+		in += BLAKE2B_BLOCK_SIZE * (nblocks - 1);
+		inlen -= BLAKE2B_BLOCK_SIZE * (nblocks - 1);
+	}
+	memcpy(ctx->buf + ctx->buflen, in, inlen);
+	ctx->buflen += inlen;
+}
+EXPORT_SYMBOL(blake2b_update);
+
+void blake2b_final(struct blake2b_ctx *ctx, u8 *out)
+{
+	WARN_ON(IS_ENABLED(DEBUG) && !out);
+	blake2b_set_lastblock(ctx);
+	memset(ctx->buf + ctx->buflen, 0,
+	       BLAKE2B_BLOCK_SIZE - ctx->buflen); /* Padding */
+	blake2b_compress(ctx, ctx->buf, 1, ctx->buflen);
+	cpu_to_le64_array(ctx->h, ARRAY_SIZE(ctx->h));
+	memcpy(out, ctx->h, ctx->outlen);
+	memzero_explicit(ctx, sizeof(*ctx));
+}
+EXPORT_SYMBOL(blake2b_final);
+
+#ifdef blake2b_mod_init_arch
+static int __init blake2b_mod_init(void)
+{
+	blake2b_mod_init_arch();
+	return 0;
+}
+subsys_initcall(blake2b_mod_init);
+
+static void __exit blake2b_mod_exit(void)
+{
+}
+module_exit(blake2b_mod_exit);
+#endif
+
+MODULE_DESCRIPTION("BLAKE2b hash function");
+MODULE_LICENSE("GPL");
diff --git a/lib/crypto/blake2s.c b/lib/crypto/blake2s.c
index 5638ed9d882d..71578a084742 100644
--- a/lib/crypto/blake2s.c
+++ b/lib/crypto/blake2s.c
@@ -14,6 +14,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/string.h>
+#include <linux/unroll.h>
 #include <linux/types.h>
 
 static const u8 blake2s_sigma[10][16] = {
@@ -29,16 +30,15 @@ static const u8 blake2s_sigma[10][16] = {
 	{ 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 },
 };
 
-static inline void blake2s_increment_counter(struct blake2s_state *state,
-					     const u32 inc)
+static inline void blake2s_increment_counter(struct blake2s_ctx *ctx, u32 inc)
 {
-	state->t[0] += inc;
-	state->t[1] += (state->t[0] < inc);
+	ctx->t[0] += inc;
+	ctx->t[1] += (ctx->t[0] < inc);
 }
 
 static void __maybe_unused
-blake2s_compress_generic(struct blake2s_state *state, const u8 *block,
-			 size_t nblocks, const u32 inc)
+blake2s_compress_generic(struct blake2s_ctx *ctx,
+			 const u8 *data, size_t nblocks, u32 inc)
 {
 	u32 m[16];
 	u32 v[16];
@@ -48,18 +48,18 @@ blake2s_compress_generic(struct blake2s_state *state, const u8 *block,
 		(nblocks > 1 && inc != BLAKE2S_BLOCK_SIZE));
 
 	while (nblocks > 0) {
-		blake2s_increment_counter(state, inc);
-		memcpy(m, block, BLAKE2S_BLOCK_SIZE);
+		blake2s_increment_counter(ctx, inc);
+		memcpy(m, data, BLAKE2S_BLOCK_SIZE);
 		le32_to_cpu_array(m, ARRAY_SIZE(m));
-		memcpy(v, state->h, 32);
+		memcpy(v, ctx->h, 32);
 		v[ 8] = BLAKE2S_IV0;
 		v[ 9] = BLAKE2S_IV1;
 		v[10] = BLAKE2S_IV2;
 		v[11] = BLAKE2S_IV3;
-		v[12] = BLAKE2S_IV4 ^ state->t[0];
-		v[13] = BLAKE2S_IV5 ^ state->t[1];
-		v[14] = BLAKE2S_IV6 ^ state->f[0];
-		v[15] = BLAKE2S_IV7 ^ state->f[1];
+		v[12] = BLAKE2S_IV4 ^ ctx->t[0];
+		v[13] = BLAKE2S_IV5 ^ ctx->t[1];
+		v[14] = BLAKE2S_IV6 ^ ctx->f[0];
+		v[15] = BLAKE2S_IV7 ^ ctx->f[1];
 
 #define G(r, i, a, b, c, d) do { \
 	a += b + m[blake2s_sigma[r][2 * i + 0]]; \
@@ -72,34 +72,27 @@ blake2s_compress_generic(struct blake2s_state *state, const u8 *block,
 	b = ror32(b ^ c, 7); \
 } while (0)
 
-#define ROUND(r) do { \
-	G(r, 0, v[0], v[ 4], v[ 8], v[12]); \
-	G(r, 1, v[1], v[ 5], v[ 9], v[13]); \
-	G(r, 2, v[2], v[ 6], v[10], v[14]); \
-	G(r, 3, v[3], v[ 7], v[11], v[15]); \
-	G(r, 4, v[0], v[ 5], v[10], v[15]); \
-	G(r, 5, v[1], v[ 6], v[11], v[12]); \
-	G(r, 6, v[2], v[ 7], v[ 8], v[13]); \
-	G(r, 7, v[3], v[ 4], v[ 9], v[14]); \
-} while (0)
-		ROUND(0);
-		ROUND(1);
-		ROUND(2);
-		ROUND(3);
-		ROUND(4);
-		ROUND(5);
-		ROUND(6);
-		ROUND(7);
-		ROUND(8);
-		ROUND(9);
-
+		/*
+		 * Unroll the rounds loop to enable constant-folding of the
+		 * blake2s_sigma values.
+		 */
+		unrolled_full
+		for (int r = 0; r < 10; r++) {
+			G(r, 0, v[0], v[4], v[8], v[12]);
+			G(r, 1, v[1], v[5], v[9], v[13]);
+			G(r, 2, v[2], v[6], v[10], v[14]);
+			G(r, 3, v[3], v[7], v[11], v[15]);
+			G(r, 4, v[0], v[5], v[10], v[15]);
+			G(r, 5, v[1], v[6], v[11], v[12]);
+			G(r, 6, v[2], v[7], v[8], v[13]);
+			G(r, 7, v[3], v[4], v[9], v[14]);
+		}
 #undef G
-#undef ROUND
 
 		for (i = 0; i < 8; ++i)
-			state->h[i] ^= v[i] ^ v[i + 8];
+			ctx->h[i] ^= v[i] ^ v[i + 8];
 
-		block += BLAKE2S_BLOCK_SIZE;
+		data += BLAKE2S_BLOCK_SIZE;
 		--nblocks;
 	}
 }
@@ -110,45 +103,46 @@ blake2s_compress_generic(struct blake2s_state *state, const u8 *block,
 #define blake2s_compress blake2s_compress_generic
 #endif
 
-static inline void blake2s_set_lastblock(struct blake2s_state *state)
+static inline void blake2s_set_lastblock(struct blake2s_ctx *ctx)
 {
-	state->f[0] = -1;
+	ctx->f[0] = -1;
 }
 
-void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen)
+void blake2s_update(struct blake2s_ctx *ctx, const u8 *in, size_t inlen)
 {
-	const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen;
+	const size_t fill = BLAKE2S_BLOCK_SIZE - ctx->buflen;
 
 	if (unlikely(!inlen))
 		return;
 	if (inlen > fill) {
-		memcpy(state->buf + state->buflen, in, fill);
-		blake2s_compress(state, state->buf, 1, BLAKE2S_BLOCK_SIZE);
-		state->buflen = 0;
+		memcpy(ctx->buf + ctx->buflen, in, fill);
+		blake2s_compress(ctx, ctx->buf, 1, BLAKE2S_BLOCK_SIZE);
+		ctx->buflen = 0;
 		in += fill;
 		inlen -= fill;
 	}
 	if (inlen > BLAKE2S_BLOCK_SIZE) {
 		const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE);
-		blake2s_compress(state, in, nblocks - 1, BLAKE2S_BLOCK_SIZE);
+
+		blake2s_compress(ctx, in, nblocks - 1, BLAKE2S_BLOCK_SIZE);
 		in += BLAKE2S_BLOCK_SIZE * (nblocks - 1);
 		inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1);
 	}
-	memcpy(state->buf + state->buflen, in, inlen);
-	state->buflen += inlen;
+	memcpy(ctx->buf + ctx->buflen, in, inlen);
+	ctx->buflen += inlen;
 }
 EXPORT_SYMBOL(blake2s_update);
 
-void blake2s_final(struct blake2s_state *state, u8 *out)
+void blake2s_final(struct blake2s_ctx *ctx, u8 *out)
 {
 	WARN_ON(IS_ENABLED(DEBUG) && !out);
-	blake2s_set_lastblock(state);
-	memset(state->buf + state->buflen, 0,
-	       BLAKE2S_BLOCK_SIZE - state->buflen); /* Padding */
-	blake2s_compress(state, state->buf, 1, state->buflen);
-	cpu_to_le32_array(state->h, ARRAY_SIZE(state->h));
-	memcpy(out, state->h, state->outlen);
-	memzero_explicit(state, sizeof(*state));
+	blake2s_set_lastblock(ctx);
+	memset(ctx->buf + ctx->buflen, 0,
+	       BLAKE2S_BLOCK_SIZE - ctx->buflen); /* Padding */
+	blake2s_compress(ctx, ctx->buf, 1, ctx->buflen);
+	cpu_to_le32_array(ctx->h, ARRAY_SIZE(ctx->h));
+	memcpy(out, ctx->h, ctx->outlen);
+	memzero_explicit(ctx, sizeof(*ctx));
 }
 EXPORT_SYMBOL(blake2s_final);
 
diff --git a/lib/crypto/chacha20poly1305.c b/lib/crypto/chacha20poly1305.c
index 0b49d6aedefd..212ce33562af 100644
--- a/lib/crypto/chacha20poly1305.c
+++ b/lib/crypto/chacha20poly1305.c
@@ -89,7 +89,7 @@ __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
 void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
 			      const u8 *ad, const size_t ad_len,
 			      const u64 nonce,
-			      const u8 key[CHACHA20POLY1305_KEY_SIZE])
+			      const u8 key[at_least CHACHA20POLY1305_KEY_SIZE])
 {
 	struct chacha_state chacha_state;
 	u32 k[CHACHA_KEY_WORDS];
@@ -111,8 +111,8 @@ EXPORT_SYMBOL(chacha20poly1305_encrypt);
 
 void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
 			       const u8 *ad, const size_t ad_len,
-			       const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
-			       const u8 key[CHACHA20POLY1305_KEY_SIZE])
+			       const u8 nonce[at_least XCHACHA20POLY1305_NONCE_SIZE],
+			       const u8 key[at_least CHACHA20POLY1305_KEY_SIZE])
 {
 	struct chacha_state chacha_state;
 
@@ -170,7 +170,7 @@ __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
 bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
 			      const u8 *ad, const size_t ad_len,
 			      const u64 nonce,
-			      const u8 key[CHACHA20POLY1305_KEY_SIZE])
+			      const u8 key[at_least CHACHA20POLY1305_KEY_SIZE])
 {
 	struct chacha_state chacha_state;
 	u32 k[CHACHA_KEY_WORDS];
@@ -195,8 +195,8 @@ EXPORT_SYMBOL(chacha20poly1305_decrypt);
 
 bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
 			       const u8 *ad, const size_t ad_len,
-			       const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
-			       const u8 key[CHACHA20POLY1305_KEY_SIZE])
+			       const u8 nonce[at_least XCHACHA20POLY1305_NONCE_SIZE],
+			       const u8 key[at_least CHACHA20POLY1305_KEY_SIZE])
 {
 	struct chacha_state chacha_state;
 
@@ -211,7 +211,7 @@ bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,
 				       const size_t src_len,
 				       const u8 *ad, const size_t ad_len,
 				       const u64 nonce,
-				       const u8 key[CHACHA20POLY1305_KEY_SIZE],
+				       const u8 key[at_least CHACHA20POLY1305_KEY_SIZE],
 				       int encrypt)
 {
 	const u8 *pad0 = page_address(ZERO_PAGE(0));
@@ -335,7 +335,7 @@ bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,
 bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len,
 					 const u8 *ad, const size_t ad_len,
 					 const u64 nonce,
-					 const u8 key[CHACHA20POLY1305_KEY_SIZE])
+					 const u8 key[at_least CHACHA20POLY1305_KEY_SIZE])
 {
 	return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len,
 						 nonce, key, 1);
@@ -345,7 +345,7 @@ EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);
 bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len,
 					 const u8 *ad, const size_t ad_len,
 					 const u64 nonce,
-					 const u8 key[CHACHA20POLY1305_KEY_SIZE])
+					 const u8 key[at_least CHACHA20POLY1305_KEY_SIZE])
 {
 	if (unlikely(src_len < POLY1305_DIGEST_SIZE))
 		return false;
diff --git a/lib/crypto/fips-mldsa.h b/lib/crypto/fips-mldsa.h
new file mode 100644
index 000000000000..1b9a60a59e68
--- /dev/null
+++ b/lib/crypto/fips-mldsa.h
@@ -0,0 +1,458 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/* ML-DSA test vector extracted from leancrypto */
+
+#include <linux/fips.h>
+
+static const u8 fips_test_mldsa65_signature[] __initconst __maybe_unused = {
+	0xda, 0xcf, 0x8d, 0x67, 0x59, 0x60, 0x6c, 0x39, 0x2d, 0x89, 0xb6, 0xa1,
+	0xf3, 0x8c, 0x70, 0xcf, 0x25, 0x86, 0x21, 0xa1, 0x9f, 0x20, 0x9e, 0xf5,
+	0xd2, 0xdd, 0xbd, 0x99, 0xfa, 0xe4, 0xab, 0x77, 0x31, 0x65, 0x18, 0xa1,
+	0xd1, 0x3f, 0x21, 0x70, 0x36, 0xe1, 0xf9, 0x5c, 0x28, 0xb6, 0x7d, 0x34,
+	0xae, 0x66, 0xc9, 0x1c, 0x8e, 0xc6, 0xf9, 0x45, 0x8c, 0xa9, 0xb2, 0xfb,
+	0x0f, 0x5b, 0xb8, 0xf9, 0xf5, 0xe2, 0x37, 0x79, 0x12, 0xda, 0xa7, 0x72,
+	0x9e, 0x0d, 0xf8, 0x88, 0x5b, 0x34, 0x49, 0x6c, 0xed, 0xa3, 0x7f, 0x86,
+	0xd3, 0xd9, 0x2f, 0x44, 0x08, 0x0d, 0xb7, 0xdb, 0x4a, 0xce, 0x02, 0x14,
+	0x02, 0xd6, 0x40, 0x75, 0xe3, 0xc0, 0x97, 0xfc, 0x6c, 0x6a, 0x88, 0x29,
+	0x0c, 0xe2, 0x3a, 0x2b, 0x28, 0x82, 0x8f, 0x27, 0x09, 0x69, 0x91, 0xc6,
+	0xc3, 0xb7, 0x07, 0x61, 0x86, 0x8d, 0x89, 0x8a, 0xd5, 0x00, 0x3b, 0x4b,
+	0xfc, 0x6f, 0xb3, 0x3f, 0x4c, 0x93, 0x31, 0xfc, 0x88, 0x53, 0x26, 0xea,
+	0xe5, 0x3a, 0xfc, 0xc1, 0x59, 0x16, 0xf0, 0xb7, 0xac, 0xde, 0x1e, 0xd8,
+	0x74, 0x85, 0x72, 0xd9, 0xbb, 0xbe, 0x76, 0x32, 0x25, 0x9d, 0x21, 0xbc,
+	0xfd, 0x8d, 0x32, 0xfe, 0xae, 0x24, 0xe5, 0x4a, 0xcc, 0x5d, 0x15, 0x23,
+	0xd3, 0x57, 0xe7, 0xa9, 0x2c, 0x31, 0xd7, 0xc5, 0x6b, 0x70, 0x6c, 0x22,
+	0x5a, 0x13, 0x1f, 0x76, 0x13, 0x78, 0x6f, 0xac, 0x42, 0x4c, 0x46, 0x81,
+	0xa2, 0x20, 0x91, 0x30, 0xed, 0xcb, 0x90, 0xfe, 0x3c, 0xa3, 0xc7, 0xb4,
+	0x1f, 0x21, 0x1d, 0x98, 0x74, 0x6a, 0x3e, 0xc8, 0xcc, 0xd2, 0x68, 0x87,
+	0x69, 0xa9, 0xdf, 0x50, 0xd5, 0x0a, 0x8e, 0x10, 0x54, 0xab, 0xea, 0x65,
+	0x2a, 0x52, 0xd7, 0x22, 0xae, 0x2f, 0x1e, 0xc3, 0x16, 0x58, 0x20, 0x18,
+	0x6d, 0x35, 0x46, 0x31, 0x43, 0x5d, 0x62, 0xfb, 0xb1, 0x47, 0x32, 0xfa,
+	0x14, 0xcc, 0x51, 0xa3, 0xcd, 0x99, 0x4f, 0x97, 0x0f, 0xca, 0x24, 0x93,
+	0x17, 0xea, 0xa3, 0xf3, 0x1f, 0xbe, 0xb5, 0xa3, 0xac, 0x80, 0xcc, 0x20,
+	0x3b, 0xa6, 0xd3, 0x32, 0x72, 0x4e, 0xd9, 0x25, 0xf9, 0xc2, 0x24, 0x15,
+	0xbd, 0x1e, 0x1e, 0x41, 0x8c, 0x18, 0x8c, 0x58, 0xe8, 0x75, 0x20, 0xff,
+	0xa3, 0xf4, 0xd4, 0xab, 0x75, 0x78, 0x4e, 0xbb, 0x7c, 0x94, 0x93, 0x28,
+	0x5b, 0x07, 0x3a, 0x3c, 0xc9, 0xf1, 0x55, 0x3e, 0x33, 0xed, 0xf8, 0x72,
+	0x55, 0xab, 0x5a, 0xea, 0xbe, 0x65, 0xfa, 0x81, 0x50, 0xc0, 0x9d, 0x2d,
+	0xfb, 0x04, 0x25, 0x7c, 0xb9, 0xee, 0xe2, 0xa3, 0x00, 0x44, 0xd3, 0x9d,
+	0xee, 0x4f, 0x80, 0x77, 0xfb, 0x26, 0x6b, 0x07, 0xd0, 0xff, 0x82, 0x39,
+	0x0e, 0x2b, 0x47, 0xa3, 0xe7, 0x3e, 0xc5, 0x4e, 0x15, 0x8a, 0x48, 0x28,
+	0xfb, 0xf7, 0xa4, 0x86, 0xfb, 0x77, 0x60, 0xcd, 0xc5, 0x68, 0x96, 0xd7,
+	0x4c, 0x3c, 0xf2, 0x51, 0x71, 0x79, 0x2e, 0x2e, 0x57, 0x10, 0xa7, 0xfc,
+	0xd1, 0xd4, 0x61, 0x71, 0x81, 0x85, 0x74, 0x09, 0x7d, 0x80, 0xd0, 0xc2,
+	0xe9, 0xff, 0xb7, 0x88, 0x53, 0x74, 0x1e, 0xb0, 0xca, 0x65, 0x48, 0x8e,
+	0xdb, 0x59, 0x3a, 0xcb, 0x80, 0xeb, 0xfd, 0xd2, 0xc9, 0x38, 0x43, 0xae,
+	0x76, 0xf2, 0xbb, 0x51, 0xb2, 0xcb, 0xe6, 0x85, 0x31, 0xb5, 0x62, 0xd4,
+	0x5e, 0x48, 0x08, 0xf1, 0x40, 0x5b, 0x16, 0x83, 0x5e, 0xa5, 0x9c, 0x6b,
+	0x91, 0x49, 0x44, 0xff, 0x3b, 0xa9, 0x2b, 0xf3, 0x06, 0x33, 0x9e, 0x6e,
+	0x3c, 0x66, 0x7e, 0x27, 0xa2, 0x59, 0x7b, 0xe3, 0xb6, 0xb4, 0x28, 0xeb,
+	0x93, 0x35, 0x87, 0xac, 0x0e, 0x0b, 0x7e, 0xbc, 0x35, 0x28, 0x72, 0x1f,
+	0x26, 0x59, 0xd0, 0x1f, 0x63, 0xe4, 0x86, 0x5d, 0x70, 0xf3, 0xa8, 0xa4,
+	0xb8, 0xcd, 0xb3, 0xf8, 0x8d, 0xaa, 0x41, 0xd2, 0xcc, 0x0b, 0x15, 0x66,
+	0x22, 0x83, 0x92, 0xe3, 0x0b, 0xf9, 0xea, 0xa0, 0x33, 0xa1, 0x4e, 0x92,
+	0xae, 0x81, 0x95, 0xa4, 0x58, 0x3f, 0xa9, 0x15, 0x52, 0xf9, 0xda, 0xb7,
+	0x10, 0x8d, 0xc6, 0xab, 0x77, 0xe9, 0xbe, 0xad, 0xc9, 0x3a, 0x6a, 0x8d,
+	0x92, 0x6c, 0x69, 0xff, 0x31, 0x49, 0x25, 0x04, 0xc8, 0x93, 0x6f, 0xc8,
+	0xe7, 0x60, 0x7a, 0x76, 0xb5, 0xc1, 0x07, 0xef, 0xa3, 0x39, 0xa6, 0xf2,
+	0x36, 0x04, 0xde, 0x3c, 0x4a, 0x4e, 0x96, 0xbd, 0x64, 0x26, 0x80, 0x01,
+	0x88, 0x47, 0xd2, 0xa4, 0x46, 0xcd, 0xe1, 0x30, 0x7f, 0xa3, 0x00, 0x11,
+	0x38, 0x55, 0xfa, 0xeb, 0x10, 0xeb, 0xa0, 0x65, 0x04, 0x09, 0xc8, 0xde,
+	0x9c, 0x73, 0xba, 0x0c, 0xbd, 0xd3, 0xa5, 0x84, 0x5e, 0xb9, 0x3b, 0xd4,
+	0x94, 0xbd, 0xa6, 0x53, 0xbe, 0x93, 0x69, 0x3e, 0xaa, 0x32, 0x31, 0x06,
+	0xc8, 0x1b, 0x4a, 0x48, 0xb5, 0x17, 0x85, 0xbf, 0x72, 0xec, 0xf5, 0x29,
+	0x8a, 0xd8, 0xeb, 0x99, 0x8b, 0x74, 0x84, 0x57, 0x8c, 0xe1, 0x85, 0x94,
+	0xa0, 0xbc, 0x7a, 0x14, 0xf0, 0xf4, 0x8b, 0x25, 0x37, 0x43, 0xa1, 0x34,
+	0x09, 0x71, 0xca, 0x5c, 0x9f, 0x08, 0x38, 0xd9, 0x9c, 0x0c, 0x0e, 0xcb,
+	0xe4, 0xad, 0x4b, 0x2a, 0x89, 0x67, 0xf8, 0x29, 0x6c, 0x69, 0x0e, 0x5d,
+	0xca, 0xfa, 0xa6, 0x6b, 0x0e, 0xb5, 0x94, 0x17, 0x71, 0xf0, 0xc9, 0xcd,
+	0x02, 0x1d, 0xa5, 0xd5, 0xc6, 0xa7, 0xbc, 0x5f, 0x6e, 0x67, 0x43, 0x68,
+	0xce, 0xac, 0x54, 0x81, 0x2a, 0x25, 0x22, 0x52, 0x35, 0xad, 0x7b, 0xd5,
+	0x06, 0x8c, 0x00, 0xfb, 0xca, 0xc4, 0x0a, 0x49, 0x1e, 0xc8, 0xeb, 0x77,
+	0xc1, 0x63, 0x23, 0x96, 0xbd, 0x35, 0xfa, 0x13, 0xae, 0xbf, 0x1d, 0x1e,
+	0x69, 0x8d, 0xb3, 0xe3, 0x07, 0xde, 0x4e, 0xd0, 0x12, 0xa9, 0xc3, 0x36,
+	0x30, 0x46, 0xef, 0x92, 0x76, 0x17, 0x8f, 0x10, 0xe7, 0xba, 0x99, 0x4b,
+	0xdf, 0xad, 0xb8, 0x11, 0x80, 0xdf, 0xe7, 0xfd, 0x80, 0x64, 0xf7, 0x2a,
+	0xac, 0x60, 0x2a, 0x54, 0x8f, 0x4f, 0xaf, 0xaf, 0x60, 0xf9, 0x67, 0x20,
+	0x80, 0x53, 0x5c, 0xb6, 0x81, 0xa6, 0x2a, 0x74, 0x2d, 0xc5, 0x74, 0x2a,
+	0x95, 0x26, 0x13, 0x17, 0x01, 0xdd, 0x31, 0xac, 0x5a, 0x05, 0xda, 0xde,
+	0xba, 0xf6, 0x37, 0x13, 0x8d, 0xe4, 0xa8, 0x93, 0x46, 0x9e, 0xa9, 0x82,
+	0x24, 0x7e, 0xc8, 0xda, 0x63, 0x89, 0xcd, 0x33, 0xc9, 0xf7, 0xf9, 0x71,
+	0x35, 0xe6, 0xa5, 0x5f, 0x6b, 0x3b, 0xbb, 0x0c, 0xe0, 0xa4, 0x0b, 0xe3,
+	0x29, 0xc0, 0xae, 0x8e, 0xce, 0x03, 0x09, 0x73, 0x0e, 0x1e, 0x9c, 0xe9,
+	0x59, 0xb6, 0x8b, 0x78, 0x67, 0x32, 0x8b, 0xf1, 0x93, 0xcc, 0x72, 0x1b,
+	0x6f, 0xa2, 0xf1, 0x04, 0x9c, 0xfa, 0x98, 0x02, 0xca, 0xdf, 0x35, 0x3c,
+	0x38, 0xac, 0xa8, 0xdb, 0x90, 0xae, 0xaa, 0xf9, 0x70, 0xfb, 0xed, 0xbd,
+	0xa6, 0x25, 0x14, 0x58, 0x09, 0x8a, 0x36, 0xaf, 0x41, 0x09, 0x19, 0xcb,
+	0xd3, 0x25, 0x5d, 0x0e, 0xe6, 0x20, 0x14, 0x71, 0x24, 0x79, 0x19, 0x55,
+	0xaf, 0x51, 0x5b, 0xa4, 0xc0, 0x93, 0x9e, 0xdd, 0x88, 0x31, 0x13, 0x96,
+	0xbf, 0xca, 0x0a, 0xd7, 0xbc, 0xc4, 0x00, 0xa1, 0x10, 0x2d, 0x92, 0x79,
+	0xf9, 0x14, 0xdb, 0xd2, 0xba, 0x74, 0xfa, 0xa8, 0xe5, 0x40, 0x14, 0xc2,
+	0x56, 0x3c, 0x7f, 0x50, 0x07, 0x60, 0x86, 0x93, 0x51, 0x2e, 0xf9, 0x70,
+	0x61, 0x70, 0x0e, 0xa4, 0x87, 0x75, 0xcc, 0x6c, 0x72, 0xb7, 0x68, 0x23,
+	0xb7, 0x3d, 0x76, 0xaf, 0x96, 0x9b, 0x4a, 0xe5, 0x12, 0x28, 0x4a, 0x8f,
+	0x79, 0x34, 0xff, 0xec, 0x92, 0xeb, 0x6b, 0xaf, 0xc9, 0xbd, 0xc1, 0x77,
+	0x07, 0xd0, 0xfa, 0x55, 0x57, 0x10, 0x0c, 0xad, 0x29, 0x2a, 0x79, 0xd6,
+	0x09, 0x9e, 0x7d, 0x18, 0xd4, 0xd6, 0xdd, 0x72, 0x1a, 0x8f, 0x24, 0x11,
+	0x70, 0xd2, 0x52, 0x36, 0x0f, 0x38, 0x79, 0x38, 0x4a, 0x02, 0x4f, 0x73,
+	0x2a, 0xaa, 0x6a, 0xb5, 0x0c, 0x72, 0x32, 0x85, 0x21, 0x76, 0x1a, 0x8a,
+	0x7d, 0x51, 0x0e, 0xf1, 0xf9, 0x19, 0xfa, 0x6b, 0x9b, 0x22, 0x71, 0x8c,
+	0x13, 0xcc, 0xba, 0x7d, 0xee, 0xd8, 0x34, 0xf6, 0x85, 0x60, 0xe1, 0xe4,
+	0x59, 0x6e, 0x32, 0x60, 0xd9, 0xfa, 0xb7, 0x56, 0x54, 0x25, 0xd1, 0x73,
+	0x6a, 0xf2, 0xa0, 0xc7, 0xa0, 0x67, 0x10, 0x89, 0x9c, 0x27, 0x5f, 0x7f,
+	0x2e, 0x5a, 0x29, 0x70, 0x7a, 0x7b, 0xaf, 0x21, 0xd0, 0xf4, 0x06, 0xb9,
+	0x2d, 0xf1, 0xb8, 0x32, 0xed, 0xc5, 0xc9, 0xac, 0x2f, 0x54, 0x0a, 0xf9,
+	0x08, 0x39, 0x39, 0x7d, 0x1d, 0xaf, 0xb4, 0x5f, 0x4d, 0x75, 0xc3, 0xe8,
+	0x52, 0x3a, 0x47, 0x72, 0x2c, 0xa9, 0x2d, 0xcb, 0x74, 0x06, 0xfe, 0x69,
+	0xd3, 0xf3, 0x1a, 0xb2, 0xd3, 0x01, 0xed, 0x6c, 0xc1, 0xca, 0x4f, 0xaf,
+	0x11, 0x9b, 0xa2, 0x27, 0x2a, 0x59, 0x56, 0x58, 0xdf, 0x79, 0x8b, 0xc9,
+	0x87, 0xe9, 0x58, 0x81, 0x48, 0xc6, 0xb6, 0x7d, 0x60, 0x54, 0x87, 0x9c,
+	0x61, 0xbb, 0x4b, 0xbb, 0x61, 0xac, 0x0a, 0x5a, 0x66, 0x7e, 0x70, 0x8b,
+	0xfd, 0x92, 0x76, 0x4a, 0xa9, 0xa5, 0xc3, 0xf4, 0xf2, 0x93, 0x48, 0xc4,
+	0xf3, 0x91, 0x2b, 0x60, 0x04, 0x0e, 0xb0, 0x6b, 0x60, 0x5e, 0xf0, 0xf1,
+	0x54, 0x41, 0x56, 0xdc, 0x25, 0x57, 0xc3, 0xb6, 0x0b, 0x5e, 0x15, 0xb5,
+	0x2a, 0x36, 0x4f, 0xe7, 0x1d, 0x70, 0xa8, 0xa7, 0xec, 0xd6, 0x74, 0xba,
+	0xa4, 0x79, 0x83, 0x7c, 0x9e, 0x1a, 0x5d, 0x32, 0xc8, 0xcb, 0x41, 0xca,
+	0x04, 0xec, 0x0b, 0x18, 0x54, 0xe1, 0x67, 0xbf, 0xa8, 0x7a, 0xc3, 0x0f,
+	0x27, 0x2a, 0xaf, 0x2a, 0x41, 0x19, 0x1f, 0xe8, 0xa2, 0xe8, 0xfa, 0xfc,
+	0x88, 0x41, 0x46, 0xc3, 0x1c, 0x44, 0xe5, 0xee, 0x47, 0xec, 0xfe, 0xbf,
+	0xb8, 0x29, 0x2e, 0xae, 0x47, 0x0a, 0x42, 0x69, 0x8a, 0x9a, 0x94, 0x97,
+	0x9e, 0xf5, 0xb6, 0x37, 0x1c, 0x10, 0xc2, 0x99, 0xa8, 0xe9, 0x9e, 0x0e,
+	0x6e, 0xb5, 0xbe, 0xba, 0x1f, 0x77, 0xa6, 0x35, 0x02, 0x1e, 0x8c, 0xe6,
+	0x02, 0x53, 0xe2, 0x9a, 0xdd, 0x09, 0x6e, 0x9b, 0x7a, 0x36, 0x4f, 0x38,
+	0x8d, 0x4c, 0xa4, 0xb4, 0xff, 0x90, 0x76, 0x0d, 0x11, 0x7d, 0xe1, 0xe9,
+	0x7f, 0x2a, 0x4a, 0x80, 0xe0, 0xd8, 0x3c, 0x23, 0xd2, 0xa5, 0xe5, 0x39,
+	0x77, 0xbf, 0x3d, 0x71, 0x0d, 0x45, 0xbb, 0x39, 0x66, 0x1a, 0x4d, 0x59,
+	0xb7, 0xd0, 0x0a, 0xee, 0x87, 0xee, 0x1f, 0xcf, 0x6f, 0xc2, 0x50, 0xb1,
+	0xa5, 0x4c, 0xee, 0x40, 0x69, 0xd7, 0x36, 0x38, 0x14, 0xcd, 0x6a, 0x9a,
+	0x90, 0x40, 0xad, 0x76, 0xf1, 0xa6, 0xd4, 0x3c, 0x75, 0x10, 0xba, 0xcb,
+	0xab, 0x22, 0x28, 0x5f, 0x0c, 0xe0, 0xee, 0xf4, 0xfd, 0x61, 0x52, 0x0a,
+	0x59, 0xfe, 0x61, 0xc5, 0x40, 0xf9, 0x91, 0x8e, 0x36, 0x29, 0x63, 0x6c,
+	0x6e, 0x45, 0xa5, 0x42, 0xe3, 0x36, 0x90, 0xe7, 0x90, 0x9f, 0x58, 0xbb,
+	0xf9, 0x1b, 0xee, 0x2c, 0xbb, 0x3a, 0xfd, 0x3d, 0xbe, 0x3d, 0x45, 0xf0,
+	0xc2, 0x18, 0xaa, 0x46, 0x10, 0x23, 0xe9, 0x63, 0xba, 0x7f, 0xc2, 0xe1,
+	0xf4, 0x05, 0xdd, 0x4a, 0x7c, 0xa8, 0xab, 0xa9, 0xbd, 0x6f, 0xdf, 0x48,
+	0x59, 0x11, 0xd4, 0xba, 0x75, 0xb6, 0x22, 0xd4, 0xd7, 0x35, 0x6f, 0x27,
+	0x70, 0xc7, 0x3d, 0x90, 0x06, 0x39, 0x2a, 0x16, 0xd0, 0x8b, 0xd7, 0xfb,
+	0x5e, 0x85, 0x2e, 0xb0, 0xd8, 0xc7, 0xdb, 0xe5, 0x24, 0x3a, 0x6e, 0xc4,
+	0x5e, 0xd4, 0x22, 0x25, 0x14, 0xee, 0xa5, 0x30, 0x8b, 0xd6, 0x27, 0x61,
+	0x33, 0x13, 0x46, 0x0b, 0x26, 0x45, 0xa6, 0xb4, 0xfa, 0x8d, 0xa3, 0xf2,
+	0x27, 0xd2, 0xc5, 0x04, 0xaa, 0x96, 0xa4, 0x55, 0xfa, 0x40, 0xf1, 0xfc,
+	0x66, 0x33, 0x9e, 0x4b, 0x39, 0x75, 0xae, 0x7f, 0x52, 0x87, 0x7b, 0x8a,
+	0xf9, 0x7d, 0x5f, 0x8a, 0x7e, 0xf7, 0xfe, 0xc4, 0x7f, 0xf4, 0xf6, 0x9a,
+	0x86, 0x78, 0x21, 0x02, 0x94, 0x9e, 0x50, 0x2d, 0xdc, 0xd6, 0xa5, 0x53,
+	0xf1, 0xef, 0x06, 0xe8, 0xb5, 0x46, 0x81, 0xcc, 0x91, 0x4f, 0x37, 0xee,
+	0x27, 0xcb, 0x91, 0xad, 0xff, 0x1d, 0xd1, 0x00, 0xa8, 0x96, 0x22, 0xaa,
+	0x63, 0x23, 0x2a, 0x7a, 0x75, 0x6f, 0xe9, 0x2d, 0x26, 0xde, 0x11, 0x97,
+	0x4b, 0x17, 0x3f, 0xde, 0x51, 0x1a, 0x22, 0xed, 0x38, 0x6f, 0x3e, 0x7a,
+	0xd0, 0xd6, 0x60, 0x06, 0x7e, 0x3f, 0xa4, 0x29, 0xfa, 0x18, 0x91, 0xda,
+	0x73, 0x38, 0xe3, 0xe3, 0xb5, 0xc0, 0x5b, 0x4e, 0xe8, 0x94, 0xea, 0x45,
+	0x6e, 0x5b, 0x50, 0xaa, 0x38, 0xb6, 0x6f, 0xdb, 0x90, 0x1b, 0x3b, 0x82,
+	0xbb, 0x0d, 0x38, 0xe3, 0xca, 0xd9, 0xf1, 0x2e, 0x27, 0x4c, 0x2c, 0x5a,
+	0x42, 0xdf, 0x44, 0xc8, 0x07, 0xe4, 0x95, 0xb5, 0xec, 0x91, 0x34, 0x1c,
+	0x9a, 0x0c, 0x50, 0x1a, 0xce, 0x67, 0xe4, 0x4b, 0x87, 0x61, 0x43, 0x95,
+	0x95, 0xb8, 0x8a, 0xf4, 0xc9, 0x92, 0x33, 0x33, 0xe3, 0xfe, 0x98, 0x2a,
+	0xae, 0x8e, 0xf2, 0x6b, 0x13, 0x7c, 0xe4, 0x44, 0x40, 0x66, 0xea, 0x0c,
+	0xe4, 0xdb, 0x16, 0x65, 0xa8, 0x8b, 0x37, 0x08, 0xec, 0x1e, 0xfc, 0xa6,
+	0xd0, 0x9b, 0x9e, 0x0a, 0xd2, 0xe3, 0xcf, 0x5d, 0xb2, 0xaf, 0x8e, 0x05,
+	0x7d, 0x8d, 0x84, 0xbc, 0x9f, 0xb1, 0xe6, 0x6a, 0x2e, 0x4b, 0x6d, 0x64,
+	0x91, 0x17, 0x9d, 0xb5, 0x35, 0x15, 0x02, 0xe9, 0x1b, 0x85, 0xc1, 0x89,
+	0xc2, 0x5a, 0x32, 0x3a, 0x80, 0x78, 0x5e, 0xcc, 0x50, 0x26, 0xf5, 0x11,
+	0x01, 0x79, 0xf3, 0xaf, 0xb6, 0x40, 0x00, 0x73, 0x8f, 0xeb, 0x5a, 0xd1,
+	0x26, 0x00, 0xe2, 0xa3, 0xcd, 0xfd, 0xaa, 0x15, 0x5b, 0x98, 0x2a, 0x76,
+	0x41, 0x07, 0xc2, 0xde, 0xb6, 0x71, 0xe7, 0xc3, 0xe9, 0x92, 0xb3, 0xd8,
+	0xfe, 0xaf, 0x12, 0x61, 0x86, 0x5b, 0x6e, 0x74, 0x45, 0x7b, 0x9b, 0x6f,
+	0x1a, 0x13, 0x84, 0xf6, 0x31, 0x5f, 0x5b, 0x6c, 0xde, 0x47, 0xb8, 0x73,
+	0x32, 0xc7, 0x94, 0x92, 0xa5, 0xc3, 0x65, 0xdf, 0x96, 0x6c, 0xfd, 0xb7,
+	0x80, 0xfb, 0x47, 0xba, 0x6e, 0x43, 0xb3, 0x7e, 0x86, 0xc9, 0x97, 0x45,
+	0xde, 0x3f, 0x3a, 0xf6, 0xb0, 0x9e, 0x9a, 0xcb, 0xfd, 0xf2, 0x5c, 0xba,
+	0x6e, 0x3f, 0xed, 0xfa, 0x74, 0x84, 0xe2, 0xb1, 0xae, 0x66, 0x57, 0x0b,
+	0x96, 0x6c, 0x77, 0xe4, 0x8a, 0x67, 0x97, 0xc7, 0xe0, 0x44, 0xb2, 0x83,
+	0x2d, 0x3c, 0x2e, 0x01, 0x19, 0x2e, 0x4c, 0x74, 0xe1, 0x35, 0x73, 0xeb,
+	0x85, 0x63, 0x8c, 0x3a, 0xb8, 0xbc, 0x25, 0x6a, 0x8d, 0xaf, 0xd2, 0xfb,
+	0xef, 0xd3, 0x12, 0x93, 0x0b, 0x39, 0xfa, 0x66, 0xbe, 0x3b, 0xfd, 0x6c,
+	0x0b, 0xbb, 0xb2, 0x5a, 0x78, 0xa1, 0xcf, 0x8c, 0x7d, 0x60, 0x55, 0xeb,
+	0x33, 0x4e, 0x8e, 0xf9, 0x19, 0x4d, 0x42, 0xd4, 0xf8, 0xd8, 0xba, 0xad,
+	0x0a, 0x6e, 0x62, 0xd4, 0xe1, 0x6a, 0xcc, 0xea, 0x09, 0x91, 0x8e, 0x62,
+	0xc9, 0x1e, 0x9e, 0x48, 0xaa, 0xde, 0xf7, 0xa2, 0x5a, 0xcb, 0x83, 0x20,
+	0xe8, 0xf5, 0xd1, 0xfe, 0x9d, 0x18, 0x2f, 0xd6, 0xf8, 0x97, 0x17, 0xce,
+	0xc2, 0x05, 0x08, 0xef, 0x61, 0x70, 0x9d, 0x95, 0x79, 0x59, 0x4c, 0x06,
+	0x24, 0x3d, 0x24, 0x69, 0xff, 0x46, 0xda, 0xbc, 0x71, 0x7a, 0x74, 0x93,
+	0x58, 0xf5, 0xc8, 0x91, 0xfb, 0x66, 0xed, 0x78, 0x8f, 0xf8, 0x28, 0xa8,
+	0x1d, 0xa5, 0x3a, 0x13, 0x76, 0xc2, 0xcc, 0xba, 0xb9, 0x56, 0x29, 0x74,
+	0xd6, 0x14, 0x75, 0x58, 0xe6, 0x2e, 0x79, 0x6e, 0x9d, 0x41, 0x94, 0x8a,
+	0xcf, 0xf1, 0xb1, 0xe0, 0x36, 0xe5, 0x89, 0x9a, 0x95, 0xa1, 0x11, 0xd1,
+	0xbe, 0x45, 0xe4, 0xb3, 0xb0, 0x62, 0x32, 0x1d, 0xba, 0xe0, 0xde, 0x57,
+	0x81, 0x0e, 0x01, 0x9b, 0x52, 0x3d, 0xd5, 0xde, 0x3b, 0x3a, 0xdd, 0x8f,
+	0xe3, 0x2e, 0xce, 0x1e, 0x89, 0x4d, 0x81, 0xf0, 0xf6, 0x20, 0x63, 0x7a,
+	0x4c, 0xbb, 0x66, 0xe0, 0xbe, 0x2b, 0xee, 0xd0, 0x3b, 0x60, 0x1e, 0x65,
+	0xd1, 0x2c, 0x7c, 0x5c, 0x6c, 0x16, 0x5b, 0x90, 0xc8, 0x05, 0x10, 0xf2,
+	0xde, 0x33, 0x90, 0x35, 0x69, 0x24, 0x3f, 0xc1, 0x8f, 0x1e, 0x4a, 0x60,
+	0xf1, 0x03, 0x65, 0x46, 0x40, 0x76, 0xe9, 0x83, 0x97, 0xda, 0x0b, 0xb8,
+	0x22, 0xfa, 0x55, 0x99, 0xfd, 0x18, 0x24, 0xd2, 0x66, 0xb0, 0x7b, 0x70,
+	0x56, 0x93, 0xad, 0x09, 0x95, 0x8e, 0x1f, 0x2f, 0xe8, 0x12, 0x55, 0xd4,
+	0x1f, 0xde, 0x09, 0x85, 0x05, 0xd1, 0xd5, 0x10, 0x2c, 0x8c, 0x6b, 0x53,
+	0x28, 0xce, 0x06, 0xc5, 0x52, 0x0f, 0xfa, 0x09, 0x09, 0x23, 0x1b, 0xe3,
+	0xbf, 0xb1, 0x89, 0x72, 0x26, 0x0d, 0xa6, 0xbb, 0x7d, 0x9e, 0xdc, 0xf8,
+	0xf5, 0x0b, 0x8c, 0xe0, 0xbc, 0x97, 0x3b, 0x72, 0xdd, 0xf5, 0x9d, 0xc5,
+	0xb6, 0x37, 0x2c, 0x76, 0x5b, 0x58, 0x67, 0xdb, 0xed, 0x3b, 0x6e, 0xe5,
+	0xe5, 0x6d, 0x6f, 0x0d, 0x7e, 0xff, 0xa9, 0x57, 0x4a, 0x84, 0x85, 0x82,
+	0xac, 0x00, 0x50, 0xa3, 0x4f, 0x87, 0xfe, 0x2a, 0x40, 0x52, 0x54, 0x81,
+	0x69, 0x42, 0x0b, 0x0c, 0xd7, 0x18, 0x98, 0x01, 0x8c, 0x5a, 0xa2, 0xf4,
+	0xe8, 0x61, 0xd1, 0x38, 0xfd, 0x0f, 0x63, 0x75, 0xd3, 0x4b, 0x1d, 0xdc,
+	0xdf, 0xb2, 0xeb, 0x94, 0x97, 0x5c, 0x2a, 0xb4, 0x12, 0x5c, 0x49, 0x2b,
+	0xfc, 0xd0, 0x8d, 0xfb, 0xe7, 0xb3, 0xcb, 0x0f, 0x3c, 0x2e, 0x04, 0x36,
+	0xa8, 0x03, 0xc9, 0xd7, 0x11, 0x2d, 0x2a, 0x93, 0xff, 0xda, 0x26, 0xb0,
+	0x54, 0x7e, 0xaf, 0x30, 0x7d, 0xce, 0x46, 0x8a, 0x3d, 0x7c, 0xa4, 0x7a,
+	0x2c, 0xfa, 0xba, 0xa1, 0xc9, 0x41, 0xd3, 0xb8, 0x84, 0x03, 0x78, 0xdd,
+	0xe9, 0x57, 0x19, 0x62, 0x62, 0xff, 0x5b, 0x3b, 0x48, 0x62, 0x0e, 0xee,
+	0x19, 0xb0, 0x32, 0x6e, 0x6a, 0x07, 0xd8, 0x4e, 0x25, 0x76, 0xa7, 0xe3,
+	0x98, 0xa1, 0x6f, 0xb6, 0x99, 0x32, 0x67, 0x7d, 0x46, 0x42, 0x4a, 0x82,
+	0xd1, 0x29, 0x1b, 0x87, 0xeb, 0x4b, 0x9e, 0xdf, 0x69, 0x75, 0xbd, 0x4f,
+	0xd3, 0xde, 0xc9, 0x83, 0xe6, 0xd6, 0xea, 0x03, 0x81, 0x12, 0xf3, 0x5d,
+	0x99, 0xf1, 0xb1, 0xd9, 0x3e, 0xbe, 0xf3, 0xa8, 0xdc, 0xb6, 0xf8, 0x4b,
+	0x9e, 0x26, 0x3f, 0xf0, 0x7c, 0xb3, 0xf4, 0xca, 0x00, 0x6c, 0x6c, 0xe5,
+	0x43, 0xa1, 0xfd, 0x3a, 0xf8, 0x8e, 0xe3, 0x9f, 0x88, 0xc5, 0x44, 0xfd,
+	0x24, 0x69, 0x76, 0xd5, 0xcb, 0xdc, 0x9d, 0x12, 0xf3, 0x13, 0x7e, 0xe7,
+	0xc3, 0xa8, 0x6a, 0xb2, 0xe0, 0xb3, 0x1d, 0xab, 0x3b, 0xc9, 0x77, 0x3d,
+	0x0f, 0xc3, 0xbe, 0x4b, 0x8b, 0x28, 0xbd, 0x7c, 0xe6, 0xb2, 0x06, 0x1f,
+	0xf9, 0x8f, 0x16, 0x62, 0xbf, 0xc7, 0x55, 0x73, 0xd4, 0xf1, 0x5a, 0x95,
+	0x80, 0xa3, 0x4e, 0xaa, 0x60, 0x17, 0x3c, 0xc9, 0x5e, 0xd4, 0x0c, 0x56,
+	0x7a, 0x77, 0x8e, 0x7f, 0x67, 0x08, 0x2f, 0xd9, 0x21, 0x19, 0xfd, 0x86,
+	0x8c, 0x23, 0x8d, 0xf6, 0x92, 0x1f, 0x36, 0x2c, 0x7c, 0x83, 0xbd, 0x2f,
+	0x6c, 0x63, 0x7c, 0xb7, 0x93, 0x74, 0x1b, 0xc2, 0x95, 0x34, 0x26, 0x1e,
+	0x07, 0x87, 0x3a, 0xb6, 0xe2, 0x39, 0x71, 0x9b, 0x20, 0xcd, 0x63, 0xf0,
+	0xbf, 0x48, 0xb5, 0x0e, 0x49, 0x86, 0x50, 0x80, 0xbd, 0xd6, 0x0e, 0xab,
+	0xd5, 0x69, 0x1b, 0xa4, 0xb3, 0x63, 0x3c, 0x8f, 0xcb, 0x42, 0xdb, 0xd7,
+	0x1a, 0xf4, 0xdf, 0x9e, 0x25, 0xfc, 0xd4, 0x00, 0xcb, 0xec, 0x57, 0x69,
+	0x30, 0x15, 0x4d, 0x7a, 0x69, 0x28, 0x2f, 0x2b, 0x34, 0x26, 0xd1, 0xe7,
+	0x01, 0x42, 0x5e, 0x02, 0xe2, 0x75, 0xe8, 0x52, 0x8a, 0xb4, 0x71, 0xfa,
+	0xc3, 0x3d, 0xe6, 0xac, 0xeb, 0xf3, 0x93, 0xe0, 0x37, 0xcd, 0x66, 0x92,
+	0x66, 0x2c, 0xfe, 0x4b, 0xd6, 0x3c, 0xf1, 0x57, 0xe5, 0xcf, 0xf5, 0xd0,
+	0xdb, 0x0e, 0x1f, 0x82, 0x65, 0x3b, 0xab, 0x69, 0x42, 0x53, 0x7d, 0xa4,
+	0x7c, 0xb7, 0x86, 0xeb, 0x23, 0x45, 0xa8, 0x4a, 0x73, 0xfc, 0x38, 0xc6,
+	0xe5, 0x2c, 0xab, 0x80, 0xfb, 0x23, 0xb2, 0x0c, 0x53, 0x28, 0x21, 0x37,
+	0x54, 0x9c, 0x72, 0x51, 0x0f, 0x44, 0x50, 0xd3, 0xe1, 0xd5, 0xb2, 0x27,
+	0x83, 0xb6, 0xe9, 0x4d, 0x64, 0x5c, 0x17, 0x0f, 0xe0, 0x13, 0xe4, 0x26,
+	0x6b, 0xd0, 0xd8, 0x25, 0xe3, 0x69, 0x6a, 0x95, 0x3f, 0x4a, 0x4e, 0xa0,
+	0x58, 0xbc, 0x28, 0x47, 0x8b, 0x68, 0xe4, 0x41, 0x90, 0x46, 0x1b, 0x84,
+	0xa0, 0x7b, 0x46, 0x46, 0x03, 0xee, 0x21, 0x0d, 0x34, 0xed, 0xff, 0x15,
+	0x57, 0x06, 0xdf, 0x71, 0x09, 0xb2, 0x66, 0x0d, 0x6e, 0xcc, 0xa5, 0x0c,
+	0xaf, 0x3f, 0x24, 0x8f, 0xd1, 0xc8, 0x44, 0x86, 0xaf, 0xbf, 0xeb, 0x2f,
+	0xb9, 0xee, 0xa7, 0xcf, 0xe4, 0xe8, 0xec, 0x47, 0x09, 0xd8, 0x95, 0x9e,
+	0x3c, 0xda, 0x92, 0x41, 0x61, 0xf5, 0xc3, 0xec, 0x00, 0xe4, 0xa3, 0x0d,
+	0x4a, 0xb3, 0xf6, 0x82, 0x05, 0x38, 0x70, 0x6a, 0xd1, 0x28, 0x2c, 0xb3,
+	0xc6, 0xbb, 0x38, 0xb3, 0x06, 0x7f, 0xd6, 0x4c, 0xe7, 0xfb, 0xef, 0x0d,
+	0x52, 0x66, 0xbe, 0xd8, 0xa6, 0x6f, 0xe8, 0xd9, 0x42, 0x4f, 0xad, 0xe8,
+	0xe8, 0x6c, 0xf9, 0xe9, 0x42, 0xd9, 0x66, 0x6e, 0xec, 0xfe, 0xf5, 0x91,
+	0xbf, 0x0a, 0x98, 0xd8, 0x7b, 0x23, 0x12, 0xa6, 0x04, 0xa8, 0xb3, 0x61,
+	0x13, 0x65, 0xc0, 0xe2, 0x82, 0xb9, 0xb2, 0x38, 0x07, 0x06, 0xca, 0x64,
+	0x6c, 0x23, 0x93, 0x60, 0x1d, 0x4d, 0x38, 0x5e, 0x8e, 0x90, 0x16, 0x4a,
+	0xfd, 0xb3, 0xcd, 0x84, 0x9c, 0xa5, 0xfa, 0x73, 0x2d, 0xcb, 0x87, 0x31,
+	0x3d, 0xf8, 0xfc, 0xeb, 0xa7, 0x56, 0x2f, 0x5b, 0x95, 0x9a, 0xc6, 0x82,
+	0x29, 0x86, 0x47, 0xe2, 0xc2, 0x84, 0x01, 0xaf, 0xc8, 0x0b, 0x2d, 0xfb,
+	0x34, 0xba, 0x5d, 0x9d, 0xd1, 0x85, 0xd5, 0x1e, 0x63, 0xcb, 0x3c, 0xa8,
+	0xfa, 0x79, 0xef, 0x12, 0xa6, 0xb5, 0xdb, 0xc5, 0x1d, 0x6a, 0xa7, 0x54,
+	0x58, 0x0c, 0xbe, 0x61, 0xe5, 0x96, 0x7f, 0x4a, 0x3b, 0x59, 0x32, 0x2d,
+	0x06, 0x44, 0x83, 0x5c, 0xad, 0xe9, 0xfe, 0x7c, 0xd7, 0x5b, 0x34, 0xa1,
+	0xa3, 0xad, 0x9a, 0xbf, 0xd5, 0x30, 0xf0, 0x22, 0xfc, 0x94, 0x7f, 0xd4,
+	0xa4, 0xca, 0x88, 0x31, 0xe7, 0xf2, 0x89, 0x2d, 0xda, 0xe6, 0x91, 0xa6,
+	0x27, 0x22, 0x74, 0x9f, 0xc6, 0x72, 0x4f, 0xf6, 0xa9, 0xfe, 0x7a, 0xf0,
+	0xa8, 0x6b, 0x6c, 0x9f, 0xe9, 0x2a, 0x9b, 0x23, 0x9e, 0xb8, 0x2b, 0x29,
+	0x65, 0xa7, 0x5d, 0xbd, 0x10, 0xe4, 0x56, 0x02, 0x94, 0xdd, 0xd1, 0xab,
+	0x9b, 0x82, 0x2d, 0x8d, 0xf6, 0xd3, 0x65, 0x63, 0x4a, 0xc4, 0x86, 0x61,
+	0x37, 0x9f, 0xdb, 0x4b, 0x34, 0x20, 0x0a, 0xca, 0x45, 0x6c, 0x06, 0xc4,
+	0x9c, 0x74, 0x4d, 0x83, 0x6a, 0x8d, 0xad, 0xc6, 0x61, 0x3a, 0x8d, 0xde,
+	0x6c, 0xf9, 0x8e, 0x33, 0xa2, 0xee, 0x99, 0xc7, 0xe4, 0x52, 0xb2, 0x44,
+	0x6f, 0x2f, 0x0f, 0x41, 0xa9, 0x1a, 0xd3, 0x96, 0x42, 0xc6, 0x49, 0x12,
+	0x6a, 0xf0, 0x29, 0xa9, 0x0c, 0x9c, 0x50, 0x5d, 0x1d, 0xd1, 0x42, 0x7e,
+	0x6f, 0x36, 0x48, 0x0f, 0x58, 0x14, 0x94, 0xc0, 0x10, 0x1e, 0xe0, 0xb2,
+	0xdd, 0xba, 0x57, 0x91, 0x4d, 0xd5, 0xdc, 0xa6, 0x4c, 0x68, 0x00, 0x6c,
+	0xb3, 0x5d, 0x32, 0x13, 0xbe, 0xa8, 0xc3, 0xfb, 0xd4, 0x19, 0x40, 0xf5,
+	0x6f, 0x63, 0xa1, 0x07, 0xbf, 0xa2, 0x8b, 0xfc, 0xfe, 0xf8, 0xa1, 0x33,
+	0x70, 0x07, 0x6d, 0xc5, 0x72, 0xa0, 0x39, 0xd6, 0xd7, 0x76, 0x6c, 0xfa,
+	0x1f, 0x04, 0xd6, 0x23, 0xbf, 0x66, 0x78, 0x92, 0x00, 0x11, 0x8a, 0x75,
+	0x67, 0x44, 0xa6, 0x7c, 0xd0, 0x14, 0xe6, 0xd0, 0x31, 0x6d, 0xdb, 0xc5,
+	0xb1, 0xa7, 0x99, 0xc3, 0xaf, 0x18, 0x7a, 0x26, 0x46, 0xad, 0x6d, 0x0c,
+	0xb6, 0xb5, 0xad, 0xc1, 0xcf, 0x60, 0x99, 0xf5, 0x9f, 0x88, 0xaf, 0x0e,
+	0x37, 0x15, 0xf9, 0x2b, 0x1a, 0x5f, 0xfb, 0xc9, 0xf8, 0xd4, 0xf0, 0x97,
+	0xd2, 0x91, 0xf4, 0x94, 0xa2, 0xd3, 0x3b, 0x8b, 0x0c, 0x22, 0xa0, 0xac,
+	0xb3, 0xb5, 0xdf, 0xf2, 0x27, 0x38, 0x47, 0x53, 0x5b, 0x6e, 0x8f, 0x98,
+	0x9e, 0xad, 0xb6, 0xf5, 0x0e, 0x17, 0x20, 0x35, 0x54, 0x6b, 0x73, 0xa6,
+	0x64, 0x65, 0xac, 0xb8, 0xc1, 0xd3, 0xf7, 0x07, 0x82, 0x93, 0x9d, 0xcb,
+	0xcc, 0xe9, 0x0c, 0x51, 0x52, 0x85, 0x8b, 0x95, 0xa6, 0xb1, 0xce, 0xdc,
+	0xfa, 0x00, 0x00, 0x08, 0x14, 0x1c, 0x23, 0x2a, 0x35,
+};
+
+static const u8 fips_test_mldsa65_public_key[] __initconst __maybe_unused = {
+	0x9f, 0x55, 0x1e, 0x7f, 0x9c, 0x08, 0xb2, 0x83, 0xfd, 0x5b, 0xa2, 0xac,
+	0x4f, 0x26, 0xc2, 0xf5, 0x06, 0x05, 0x96, 0x08, 0x24, 0xad, 0xec, 0xe4,
+	0x99, 0xcc, 0x6c, 0xbd, 0x55, 0x37, 0x15, 0x94, 0xab, 0x31, 0x9e, 0x56,
+	0xe5, 0xe4, 0x55, 0xec, 0x4d, 0x49, 0x5b, 0x5a, 0x7a, 0xe8, 0xc3, 0x4a,
+	0x08, 0x44, 0x4a, 0xc2, 0x2d, 0xe4, 0x61, 0x33, 0x90, 0x20, 0x71, 0x45,
+	0xa5, 0x45, 0xd0, 0x83, 0x2b, 0x32, 0x6c, 0xa7, 0x9e, 0x76, 0xcd, 0xfb,
+	0x58, 0x15, 0x9e, 0x74, 0x0d, 0x67, 0x57, 0xb1, 0x06, 0x5b, 0x5d, 0xd5,
+	0x1c, 0xbb, 0x95, 0x40, 0x1c, 0x71, 0x31, 0x03, 0xef, 0xff, 0x04, 0x6b,
+	0xdd, 0xa2, 0xf0, 0x32, 0x00, 0x72, 0xbc, 0x87, 0xb6, 0x2c, 0x1f, 0x90,
+	0x7f, 0x92, 0xa0, 0xb2, 0x04, 0xdd, 0xa9, 0xaf, 0x7f, 0x01, 0x28, 0x4c,
+	0xb2, 0x57, 0x2d, 0x56, 0x93, 0xd0, 0xc7, 0x54, 0x02, 0x90, 0x57, 0x70,
+	0x23, 0x57, 0xe8, 0xe7, 0x33, 0x32, 0x98, 0xfc, 0x9b, 0x8e, 0x6e, 0x7b,
+	0xaa, 0x5d, 0xb5, 0x4e, 0xe0, 0x5d, 0x97, 0xa3, 0xea, 0x43, 0x7e, 0xb3,
+	0xa4, 0x8c, 0xcf, 0xdc, 0xc0, 0x51, 0xa7, 0x99, 0x45, 0x3d, 0x3c, 0xa0,
+	0xba, 0xc5, 0xff, 0xe1, 0x89, 0xb3, 0x7d, 0xc3, 0xdc, 0xe2, 0x23, 0x81,
+	0xff, 0xa9, 0xc7, 0x93, 0xc6, 0x67, 0xad, 0x94, 0xcf, 0xeb, 0x91, 0x78,
+	0x15, 0x25, 0xf7, 0xf5, 0x06, 0x08, 0x2f, 0x0c, 0xee, 0x0b, 0x6a, 0x06,
+	0x59, 0xe0, 0x1f, 0x2e, 0x5a, 0x12, 0x06, 0xf5, 0xf4, 0x8e, 0x75, 0x57,
+	0xa9, 0x33, 0x23, 0x0f, 0xc2, 0x6f, 0x02, 0xf8, 0x68, 0x0f, 0x62, 0x02,
+	0x81, 0xfe, 0x03, 0x7c, 0xaf, 0xd7, 0x42, 0x5b, 0xcc, 0xe7, 0x2b, 0xea,
+	0x49, 0xab, 0x03, 0x6d, 0x0a, 0x02, 0xae, 0x47, 0x79, 0xce, 0xfd, 0x18,
+	0x76, 0x07, 0x9e, 0xa6, 0xbf, 0x7e, 0x8d, 0x73, 0xf9, 0x44, 0xeb, 0x8c,
+	0xc5, 0x59, 0xb7, 0x19, 0xf6, 0x73, 0x53, 0x42, 0x2a, 0x55, 0x7b, 0xb4,
+	0x56, 0x49, 0x08, 0x9e, 0x9a, 0x65, 0x60, 0x70, 0x1d, 0xbd, 0xc6, 0x85,
+	0x29, 0xde, 0xfe, 0x44, 0xae, 0xdf, 0x25, 0xfd, 0x5b, 0x74, 0x6c, 0x96,
+	0xe6, 0x81, 0x37, 0x80, 0xe0, 0x9e, 0xf3, 0x75, 0x63, 0xb4, 0xc9, 0x2f,
+	0x71, 0xe6, 0xeb, 0xdf, 0xaf, 0x7e, 0xff, 0x9e, 0xe0, 0xbf, 0xca, 0xca,
+	0x11, 0xed, 0xc6, 0x04, 0xd8, 0x49, 0x13, 0x2c, 0x63, 0xf1, 0xb3, 0x17,
+	0x74, 0xd9, 0x50, 0x3f, 0xb9, 0x29, 0x0e, 0x48, 0xa7, 0xf0, 0xdc, 0x78,
+	0x18, 0x0e, 0x9f, 0xb7, 0xde, 0x36, 0x79, 0x67, 0xa4, 0x23, 0x08, 0xe7,
+	0x62, 0xe8, 0xa4, 0xe5, 0xcf, 0xff, 0x35, 0x55, 0x36, 0x2e, 0x3a, 0xe4,
+	0x45, 0x6a, 0x80, 0xf2, 0xca, 0xe7, 0x40, 0x79, 0x14, 0xc4, 0x62, 0x38,
+	0xbb, 0xd0, 0x4e, 0x6c, 0xb5, 0x85, 0x42, 0x3f, 0x35, 0xf7, 0xd7, 0x54,
+	0xb8, 0x2b, 0x8b, 0xd5, 0x6f, 0x16, 0x61, 0x27, 0x23, 0xac, 0xdb, 0xea,
+	0x9b, 0x3b, 0x99, 0xcd, 0x79, 0xe6, 0x12, 0x09, 0x99, 0x09, 0xa4, 0xe1,
+	0x88, 0x25, 0x00, 0x9e, 0x60, 0x16, 0x63, 0xd7, 0x42, 0x9b, 0xcc, 0x36,
+	0x9a, 0x8d, 0xa3, 0x75, 0x36, 0xa1, 0xa8, 0xfc, 0xa2, 0xfe, 0x29, 0x26,
+	0x4c, 0x93, 0x21, 0x44, 0x6b, 0x1c, 0xba, 0xbd, 0xef, 0xff, 0x6d, 0x1f,
+	0x2b, 0x6c, 0x66, 0x81, 0x9a, 0x3a, 0x1d, 0x0b, 0xd7, 0x24, 0xd4, 0xb8,
+	0x93, 0xb5, 0x22, 0xf9, 0xd2, 0xf4, 0xa5, 0x05, 0x78, 0x38, 0xae, 0x58,
+	0xf6, 0x50, 0x8f, 0x47, 0x1d, 0xf3, 0xfb, 0x0d, 0x04, 0x14, 0xd1, 0xd6,
+	0xd8, 0x2e, 0xf2, 0xbd, 0xf5, 0x71, 0x86, 0x4c, 0xdd, 0x61, 0x24, 0x18,
+	0x5b, 0x54, 0xf5, 0xcd, 0x99, 0x89, 0x01, 0x8e, 0xd1, 0x19, 0x52, 0xbc,
+	0x45, 0xed, 0x0e, 0xec, 0x72, 0x2f, 0x5a, 0xe7, 0xdf, 0x36, 0x1c, 0x57,
+	0x9f, 0xb2, 0x8b, 0xf2, 0x78, 0x1b, 0x3e, 0xc5, 0x48, 0x1f, 0x27, 0x04,
+	0x76, 0x10, 0x44, 0xee, 0x5c, 0x68, 0x8f, 0xca, 0xd7, 0x31, 0xfc, 0x5c,
+	0x40, 0x03, 0x2e, 0xbd, 0x1d, 0x59, 0x13, 0x57, 0xbc, 0x33, 0xc6, 0xa1,
+	0xa3, 0xe5, 0x55, 0x79, 0x9b, 0x7e, 0x49, 0xbb, 0x23, 0x96, 0xc3, 0x1c,
+	0xfe, 0x66, 0xeb, 0x5b, 0x5f, 0xe5, 0x03, 0xc9, 0xa4, 0xac, 0x4d, 0xc4,
+	0x50, 0xbb, 0xd3, 0xc1, 0x91, 0x48, 0xe0, 0x93, 0x92, 0x2a, 0xdb, 0x41,
+	0x37, 0x98, 0xbc, 0xa2, 0x7a, 0x09, 0x92, 0x0b, 0x1c, 0xe6, 0x4b, 0x1e,
+	0x8e, 0x78, 0x81, 0x74, 0x7d, 0x6b, 0x71, 0xd5, 0xe7, 0x0e, 0x7b, 0xc2,
+	0x74, 0x5d, 0x89, 0xf1, 0xfa, 0x59, 0xaa, 0xf7, 0x86, 0x66, 0x7e, 0xc2,
+	0x9c, 0xf4, 0xd5, 0x8d, 0xc0, 0xb7, 0xb7, 0xa2, 0xd5, 0xcd, 0x51, 0xc3,
+	0x7d, 0xa9, 0x5e, 0x46, 0xba, 0x06, 0xa3, 0x4d, 0x60, 0xd6, 0x68, 0xc6,
+	0xf9, 0x63, 0x88, 0x17, 0x5c, 0x20, 0xe1, 0xc4, 0x0f, 0x3f, 0xc1, 0xa9,
+	0xa7, 0x3e, 0x39, 0xef, 0x2f, 0xaf, 0xc4, 0x69, 0x29, 0xe3, 0xd4, 0x8d,
+	0xe0, 0x0e, 0x88, 0xc2, 0x93, 0x43, 0xfb, 0x28, 0xcf, 0x5d, 0x85, 0x50,
+	0xf7, 0xeb, 0x42, 0xf5, 0x87, 0xde, 0xa5, 0x65, 0xef, 0x43, 0x0c, 0x57,
+	0x76, 0x09, 0xf4, 0x5f, 0xde, 0x81, 0x0a, 0xd9, 0x59, 0x41, 0xa4, 0x6a,
+	0xb7, 0x05, 0xc7, 0xa5, 0xfe, 0x49, 0xd5, 0x9b, 0x57, 0x13, 0x14, 0x66,
+	0xe2, 0xb9, 0xcc, 0x09, 0x35, 0xd4, 0xb0, 0xe0, 0xd1, 0x0d, 0x7e, 0x50,
+	0x48, 0x45, 0x21, 0x00, 0x67, 0xb2, 0xad, 0xa7, 0x46, 0xe2, 0x6f, 0x70,
+	0xe5, 0x3c, 0x88, 0x04, 0xaa, 0x21, 0xde, 0x03, 0xb6, 0x6f, 0xfe, 0x43,
+	0x51, 0xdc, 0x2e, 0x5c, 0x6c, 0x77, 0x8f, 0x8e, 0x9d, 0x1a, 0x5b, 0x35,
+	0xc5, 0xe4, 0x48, 0x82, 0x17, 0x4b, 0xf0, 0xea, 0xc9, 0x0e, 0xd2, 0x8f,
+	0xcd, 0xd5, 0x01, 0xbd, 0x7f, 0x0f, 0xf5, 0xae, 0x92, 0x28, 0x1e, 0x2c,
+	0xf4, 0xe9, 0x03, 0xf7, 0x0a, 0xeb, 0x84, 0x18, 0xa1, 0x37, 0x38, 0x8a,
+	0x11, 0xa2, 0x5d, 0x8c, 0xf6, 0xe4, 0x3f, 0x5b, 0x87, 0x07, 0x6b, 0xb4,
+	0x07, 0xe0, 0x8f, 0x30, 0xc4, 0xfa, 0x27, 0xae, 0xfc, 0x02, 0xd1, 0x21,
+	0x5c, 0xbc, 0x0b, 0x93, 0x6e, 0x7e, 0xf9, 0x6b, 0x80, 0x7a, 0x25, 0x84,
+	0x20, 0xf1, 0x6a, 0xfa, 0x75, 0xed, 0x57, 0x61, 0x62, 0xa7, 0xf6, 0x5b,
+	0xe1, 0xb0, 0x38, 0xc8, 0xe9, 0x6d, 0x3f, 0xef, 0x1e, 0x99, 0x0b, 0xb7,
+	0xc8, 0x9f, 0x76, 0x5c, 0x04, 0x1f, 0x02, 0x92, 0x00, 0xa7, 0x38, 0x3d,
+	0x00, 0x3b, 0xa7, 0xbc, 0x39, 0x6e, 0xab, 0xf5, 0x10, 0xa8, 0xba, 0xd6,
+	0x28, 0x6b, 0x0e, 0x00, 0x48, 0xf9, 0x3b, 0x5c, 0xde, 0x59, 0x93, 0x46,
+	0xd6, 0x61, 0x52, 0x81, 0x71, 0x0f, 0x0e, 0x61, 0xac, 0xc6, 0x7f, 0x15,
+	0x93, 0xa7, 0xc1, 0x16, 0xb5, 0xef, 0x85, 0xd1, 0xa7, 0x61, 0xc2, 0x85,
+	0x1d, 0x61, 0xc6, 0xae, 0xb3, 0x9e, 0x8d, 0x23, 0xa3, 0xc8, 0xd5, 0xf2,
+	0xc7, 0x1b, 0x7e, 0xef, 0xd2, 0xdf, 0x25, 0xaf, 0x4e, 0x81, 0x15, 0x59,
+	0xe5, 0x36, 0xb1, 0xf1, 0xd5, 0xda, 0x58, 0xd8, 0xd9, 0x0d, 0x6d, 0xc9,
+	0x25, 0xb5, 0xe8, 0x1d, 0x3b, 0xca, 0x2d, 0xab, 0xf2, 0xe2, 0xe9, 0x55,
+	0xd7, 0xf4, 0xc7, 0xd0, 0x57, 0x7a, 0x86, 0x15, 0x0a, 0x5a, 0x8b, 0xd7,
+	0x3f, 0x66, 0x0f, 0x80, 0xb4, 0xe0, 0x5c, 0x33, 0xed, 0xaf, 0x1b, 0x3b,
+	0x6d, 0x1c, 0xd9, 0x8c, 0xb5, 0x96, 0xa3, 0xfb, 0xcf, 0xcc, 0x97, 0x1c,
+	0xae, 0x06, 0x19, 0x41, 0x61, 0xf8, 0x97, 0x6b, 0x82, 0x5e, 0x1c, 0xbf,
+	0x6f, 0x43, 0x3d, 0xe5, 0x00, 0xf5, 0xfe, 0x66, 0x48, 0x26, 0x31, 0xa1,
+	0x72, 0x67, 0x6e, 0xd4, 0x5b, 0x6f, 0x66, 0xde, 0x70, 0x8b, 0x2b, 0xc3,
+	0xa2, 0x30, 0xe9, 0x55, 0xc8, 0xff, 0xf8, 0xd0, 0xdd, 0xa9, 0x21, 0x85,
+	0x6e, 0x6c, 0x82, 0x66, 0xcc, 0x52, 0xf0, 0x9e, 0x1e, 0xb5, 0x3a, 0xff,
+	0x4c, 0xf3, 0xae, 0x02, 0xc3, 0x4b, 0x76, 0x25, 0xbd, 0xb0, 0x21, 0x54,
+	0x61, 0xda, 0x16, 0xd3, 0x23, 0x86, 0x41, 0xa1, 0x4c, 0x59, 0x15, 0x95,
+	0x65, 0x85, 0xb6, 0x8e, 0xa6, 0x37, 0xc0, 0xa2, 0x71, 0x1d, 0x67, 0x44,
+	0x7b, 0xe5, 0x4c, 0x4f, 0xb6, 0x2c, 0x46, 0xf7, 0x29, 0xa5, 0xf2, 0xd3,
+	0x51, 0x19, 0x91, 0x4d, 0xa7, 0xb5, 0x05, 0xb9, 0x6e, 0x61, 0x6e, 0xf8,
+	0xc0, 0x01, 0xe5, 0x41, 0x0a, 0x89, 0x64, 0x77, 0xf2, 0xc8, 0x63, 0x2d,
+	0x9d, 0x27, 0x7f, 0x47, 0x30, 0x39, 0xdf, 0xb6, 0x6e, 0x4f, 0x00, 0x3f,
+	0x15, 0xc6, 0xaf, 0x62, 0xdf, 0x3f, 0x47, 0xe8, 0x42, 0x90, 0x77, 0x23,
+	0x7a, 0xaa, 0x99, 0x53, 0x03, 0x63, 0x60, 0x59, 0x07, 0x52, 0x3c, 0xb5,
+	0x67, 0x59, 0xfe, 0x08, 0xe6, 0x43, 0x0f, 0x3b, 0x08, 0x7c, 0xc7, 0x07,
+	0x3c, 0xfa, 0x65, 0xea, 0x69, 0x51, 0x41, 0x31, 0xb3, 0x05, 0x69, 0xba,
+	0x2c, 0xbf, 0x89, 0x25, 0x9e, 0xfe, 0x07, 0x13, 0x78, 0x0e, 0x16, 0x54,
+	0xdf, 0x23, 0xdf, 0x10, 0x69, 0x79, 0xd0, 0x33, 0xd7, 0x21, 0x8b, 0xc8,
+	0x2a, 0xd0, 0x74, 0x0a, 0xfa, 0xb1, 0x6f, 0xa3, 0xcb, 0x1d, 0xca, 0x4f,
+	0x00, 0x46, 0x6c, 0x42, 0x09, 0xe0, 0x30, 0x89, 0x08, 0x33, 0x9b, 0x7b,
+	0x7b, 0x0f, 0x69, 0x5c, 0x0d, 0x34, 0x91, 0xfc, 0xfe, 0x22, 0x82, 0x02,
+	0xcd, 0xfa, 0x97, 0xe8, 0x28, 0x1d, 0xbc, 0x13, 0x0b, 0xfd, 0x47, 0xa1,
+	0x7e, 0xa2, 0x86, 0x4d, 0x6f, 0x12, 0x51, 0x35, 0x7d, 0x76, 0x8a, 0x58,
+	0x05, 0xb6, 0x39, 0xa1, 0x2f, 0xd7, 0xda, 0xaf, 0x00, 0xa0, 0x1a, 0x94,
+	0xd8, 0x23, 0x34, 0x99, 0x5c, 0xaf, 0xcc, 0x15, 0x4b, 0x56, 0xb2, 0xd2,
+	0x81, 0x07, 0xd3, 0xf3, 0x47, 0xa2, 0x45, 0x93, 0xcb, 0xae, 0xa7, 0x6b,
+	0x3f, 0xf9, 0xea, 0xfc, 0x0e, 0x64, 0xf2, 0x93, 0x7f, 0x24, 0x22, 0x73,
+	0x86, 0xc7, 0x2d, 0x75, 0x9b, 0x41, 0x8b, 0xfb, 0x3b, 0x26, 0x2a, 0xe5,
+	0x0b, 0xd4, 0x00, 0xe3, 0x2c, 0x69, 0x49, 0x62, 0x6c, 0x13, 0x58, 0x6e,
+	0xac, 0x43, 0xe5, 0x2b, 0x3b, 0x88, 0xdc, 0xd4, 0x41, 0xe8, 0xee, 0x4e,
+	0xc3, 0x28, 0x91, 0x17, 0x9a, 0x5a, 0xdb, 0x80, 0x8b, 0x4d, 0x64, 0xcc,
+	0xbe, 0x66, 0xa4, 0x62, 0xfb, 0x13, 0x44, 0x10, 0xd9, 0xe4, 0xd5, 0xa5,
+	0xae, 0x9e, 0x42, 0x50, 0xfc, 0x78, 0xad, 0xfa, 0xc4, 0xd0, 0x5a, 0x60,
+	0x9b, 0x45, 0x2b, 0x61, 0x5c, 0x57, 0xb5, 0x92, 0x28, 0xe9, 0xf5, 0x35,
+	0x67, 0xc1, 0x5e, 0xa8, 0x1c, 0x99, 0x36, 0x38, 0xb8, 0x5c, 0xff, 0x3d,
+	0xa0, 0xfc, 0xb0, 0xbc, 0x3d, 0x2c, 0xb4, 0x36, 0x17, 0xb4, 0x6d, 0xb5,
+	0x39, 0x45, 0xa9, 0x2a, 0x6b, 0xa2, 0x24, 0x44, 0x30, 0xab, 0x2c, 0x82,
+	0x36, 0xdc, 0xd6, 0x36, 0x5d, 0x0a, 0xdc, 0xee, 0x0f, 0x2b, 0x28, 0x99,
+	0xdc, 0x67, 0x0d, 0xea, 0x6e, 0x42, 0xb9, 0x45, 0x7f, 0xd2, 0x96, 0x1e,
+	0x60, 0x42, 0xeb, 0x1e, 0x5f, 0x8e, 0xa9, 0xdc, 0xd3, 0x8a, 0xd6, 0xbd,
+	0x4e, 0x1f, 0x42, 0x75, 0x1d, 0xe2, 0xc6, 0x11, 0xc9, 0x80, 0x1f, 0xfe,
+	0x99, 0x52, 0x4d, 0x7b, 0x35, 0xf7, 0xb7, 0xc3, 0xee, 0xd6, 0x94, 0xf5,
+	0x74, 0xa0, 0x69, 0xcd, 0x1f, 0x2b, 0xd0, 0x87, 0xf7, 0x8c, 0x69, 0xc5,
+	0x96, 0x70, 0x91, 0xe8, 0x3d, 0xd2, 0xcc, 0xf1, 0x4c, 0xcd, 0xe2, 0x14,
+	0x00, 0x10, 0x4a, 0xd9, 0x6a, 0x5d, 0x65, 0x2c, 0x4b, 0x79, 0x0c, 0xc4,
+	0x78, 0x5e, 0xc8, 0xc5, 0x37, 0x74, 0x6d, 0x50, 0x5c, 0x34, 0x1f, 0xe0,
+	0xf4, 0xe3, 0xe1, 0x86, 0x68, 0xb1, 0xea, 0x70, 0xf0, 0xae, 0xe4, 0x59,
+	0xa1, 0x08, 0x7e, 0x35, 0xa3, 0x16, 0xd2, 0xb0, 0xa3, 0xd4, 0xb0, 0x74,
+	0x8c, 0x05, 0x79, 0x73, 0xfb, 0xe6, 0x65, 0x96, 0x15, 0x07, 0xd5, 0xaf,
+	0x88, 0x9e, 0x6b, 0xf0, 0xbb, 0x3f, 0xe6, 0xd1, 0x6a, 0xe7, 0xc9, 0xae,
+	0xd9, 0xb0, 0x16, 0x1c, 0x40, 0xeb, 0xdb, 0xc1, 0xbf, 0x83, 0xdb, 0x8a,
+	0x4f, 0x96, 0xca, 0xd7, 0x22, 0x06, 0x87, 0x08, 0x9d, 0x65, 0x2f, 0xd9,
+	0x8e, 0x95, 0x6c, 0xcc, 0xbf, 0x76, 0x2a, 0xea, 0x5c, 0x8e, 0x5b, 0x17,
+	0x0f, 0x75, 0x7b, 0xfa, 0xf9, 0xfb, 0xaa, 0x92, 0xc7, 0x7e, 0x63, 0x63,
+	0x54, 0xa4, 0xff, 0xf6, 0xc0, 0xc0, 0xf5, 0x70, 0xd8, 0xe3, 0xa4, 0x79,
+	0x16, 0xf0, 0x6f, 0x90, 0x5e, 0xb7, 0xab, 0x6f, 0xab, 0x75, 0x3b, 0xe1,
+	0x4c, 0xa8, 0x0b, 0x72, 0x5f, 0x5f, 0x11, 0x22, 0x36, 0x71, 0x20, 0xd3,
+	0x5b, 0x5e, 0x07, 0x06, 0x76, 0x1a, 0xcc, 0x5e, 0x7c, 0x97, 0x7d, 0xb2,
+	0x6b, 0xf8, 0x39, 0x89, 0x37, 0xb6, 0x6d, 0xea, 0x74, 0x57, 0x28, 0xd7,
+	0x0e, 0x9b, 0xeb, 0x28, 0x88, 0x90, 0xfd, 0x2d, 0x16, 0x21, 0x74, 0x26,
+	0xc5, 0xb8, 0x44, 0xad, 0x9f, 0x97, 0xf9, 0x65, 0x36, 0xd8, 0x00, 0x59,
+	0x17, 0x49, 0xf9, 0xc7, 0xb3, 0x84, 0xb9, 0xe2, 0x95, 0xe0, 0xd1, 0x7f,
+	0x5f, 0xaa, 0xd7, 0xfd, 0x6a, 0x6a, 0x83, 0x14, 0x46, 0x1d, 0x12, 0x8d,
+	0x09, 0xc3, 0xa5, 0xca, 0x72, 0xa3, 0x25, 0x65, 0xb6, 0x40, 0x25, 0x04,
+	0x51, 0xab, 0x22, 0xeb, 0xd7, 0x69, 0xc9, 0x22, 0x9c, 0xa0, 0x19, 0x5c,
+	0x1a, 0xfd, 0x41, 0x8f, 0x98, 0xc5, 0x71, 0xb8, 0x6f, 0x76, 0xae, 0xfa,
+	0x9b, 0x03, 0xab, 0x43, 0x81, 0x3b, 0x66, 0xae, 0xf0, 0xd2, 0xb7, 0xee,
+	0x9a, 0xe3, 0xae, 0x45, 0xc1, 0x86, 0xb0, 0xce, 0x9e, 0x2b, 0xec, 0xb8,
+	0xcf, 0xca, 0x0e, 0x8c, 0x33, 0xfa, 0xa7, 0xef, 0xf7, 0xfc, 0xa1, 0x41,
+	0x49, 0xd3, 0x6d, 0xb5, 0x58, 0xe4, 0x0e, 0x24, 0xd2, 0x8a, 0x74, 0xc9,
+	0x56, 0x2e, 0x53, 0xc7, 0x7a, 0x38, 0x0f, 0x4b, 0xd9, 0xf9, 0x2f, 0xfa,
+	0x7d, 0xee, 0x14, 0x18, 0xce, 0x75, 0x42, 0x6c, 0x03, 0x34, 0xce, 0x80,
+	0xec, 0xf2, 0x05, 0xf0, 0xdf, 0xcd, 0xf8, 0xdb, 0x26, 0x7d, 0xb6, 0x3d,
+	0x28, 0x24, 0x7e, 0x7e, 0x39, 0x9f, 0xa6, 0xc6, 0xeb, 0x2a, 0xc8, 0x17,
+	0x94, 0xa9, 0x89, 0xf5, 0xdf, 0xcb, 0x77, 0xfd, 0xc9, 0x9e, 0x68, 0x98,
+	0x7d, 0x04, 0x50, 0x3c, 0x64, 0x1d, 0x66, 0xb0, 0x97, 0x06, 0xb6, 0x08,
+	0x5b, 0xe4, 0x17, 0x44, 0xd6, 0x94, 0x39, 0x6b, 0x03, 0x2c, 0xcb, 0x5a,
+	0x8d, 0x86, 0x08, 0x23, 0x4f, 0x95, 0xa8, 0x1a,
+};
+
+static const u8 fips_test_mldsa65_message[] __initconst __maybe_unused = {
+	0x1a, 0x84, 0x21, 0x0d, 0x35, 0x7a, 0x88, 0xc8, 0x6a, 0x11, 0xe3,
+	0x15, 0x24, 0xec, 0x0d, 0x2e, 0x76, 0xb9, 0xcf, 0x2b, 0x04, 0x25,
+	0x16, 0xae, 0x62, 0x42, 0xa0, 0x20, 0x68, 0x25, 0x3e, 0xb4, 0x75,
+	0xa7, 0x1d, 0x64, 0xc3, 0xd1, 0x08, 0x07, 0x67, 0xb6, 0xf7, 0x76,
+	0x76, 0xf6, 0xd6, 0x62, 0x66, 0x04, 0x89, 0x0c, 0x8f, 0x07, 0xac,
+	0xc8, 0x51, 0x77, 0xd9, 0x47, 0x5e, 0xb5, 0x22, 0x20,
+};
diff --git a/lib/crypto/fips.h b/lib/crypto/fips.h
new file mode 100644
index 000000000000..023410c2e0db
--- /dev/null
+++ b/lib/crypto/fips.h
@@ -0,0 +1,45 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* This file was generated by: gen-fips-testvecs.py */
+
+#include <linux/fips.h>
+
+static const u8 fips_test_data[] __initconst __maybe_unused = {
+	0x66, 0x69, 0x70, 0x73, 0x20, 0x74, 0x65, 0x73,
+	0x74, 0x20, 0x64, 0x61, 0x74, 0x61, 0x00, 0x00,
+};
+
+static const u8 fips_test_key[] __initconst __maybe_unused = {
+	0x66, 0x69, 0x70, 0x73, 0x20, 0x74, 0x65, 0x73,
+	0x74, 0x20, 0x6b, 0x65, 0x79, 0x00, 0x00, 0x00,
+};
+
+static const u8 fips_test_hmac_sha1_value[] __initconst __maybe_unused = {
+	0x29, 0xa9, 0x88, 0xb8, 0x5c, 0xb4, 0xaf, 0x4b,
+	0x97, 0x2a, 0xee, 0x87, 0x5b, 0x0a, 0x02, 0x55,
+	0x99, 0xbf, 0x86, 0x78,
+};
+
+static const u8 fips_test_hmac_sha256_value[] __initconst __maybe_unused = {
+	0x59, 0x25, 0x85, 0xcc, 0x40, 0xe9, 0x64, 0x2f,
+	0xe9, 0xbf, 0x82, 0xb7, 0xd3, 0x15, 0x3d, 0x43,
+	0x22, 0x0b, 0x4c, 0x00, 0x90, 0x14, 0x25, 0xcf,
+	0x9e, 0x13, 0x2b, 0xc2, 0x30, 0xe6, 0xe8, 0x93,
+};
+
+static const u8 fips_test_hmac_sha512_value[] __initconst __maybe_unused = {
+	0x6b, 0xea, 0x5d, 0x27, 0x49, 0x5b, 0x3f, 0xea,
+	0xde, 0x2d, 0xfa, 0x32, 0x75, 0xdb, 0x77, 0xc8,
+	0x26, 0xe9, 0x4e, 0x95, 0x4d, 0xad, 0x88, 0x02,
+	0x87, 0xf9, 0x52, 0x0a, 0xd1, 0x92, 0x80, 0x1d,
+	0x92, 0x7e, 0x3c, 0xbd, 0xb1, 0x3c, 0x49, 0x98,
+	0x44, 0x9c, 0x8f, 0xee, 0x3f, 0x02, 0x71, 0x51,
+	0x57, 0x0b, 0x15, 0x38, 0x95, 0xd8, 0xa3, 0x81,
+	0xba, 0xb3, 0x15, 0x37, 0x5c, 0x6d, 0x57, 0x2b,
+};
+
+static const u8 fips_test_sha3_256_value[] __initconst __maybe_unused = {
+	0x77, 0xc4, 0x8b, 0x69, 0x70, 0x5f, 0x0a, 0xb1,
+	0xb1, 0xa5, 0x82, 0x0a, 0x22, 0x2b, 0x49, 0x31,
+	0xba, 0x9b, 0xb6, 0xaa, 0x32, 0xa7, 0x97, 0x00,
+	0x98, 0xdb, 0xff, 0xe7, 0xc6, 0xde, 0xb5, 0x82,
+};
diff --git a/lib/crypto/gf128mul.c b/lib/crypto/gf128mul.c
index 2a34590fe3f1..e5a727b15f07 100644
--- a/lib/crypto/gf128mul.c
+++ b/lib/crypto/gf128mul.c
@@ -245,12 +245,12 @@ struct gf128mul_64k *gf128mul_init_64k_bbe(const be128 *g)
 	struct gf128mul_64k *t;
 	int i, j, k;
 
-	t = kzalloc(sizeof(*t), GFP_KERNEL);
+	t = kzalloc_obj(*t);
 	if (!t)
 		goto out;
 
 	for (i = 0; i < 16; i++) {
-		t->t[i] = kzalloc(sizeof(*t->t[i]), GFP_KERNEL);
+		t->t[i] = kzalloc_obj(*t->t[i]);
 		if (!t->t[i]) {
 			gf128mul_free_64k(t);
 			t = NULL;
@@ -326,7 +326,7 @@ struct gf128mul_4k *gf128mul_init_4k_lle(const be128 *g)
 	struct gf128mul_4k *t;
 	int j, k;
 
-	t = kzalloc(sizeof(*t), GFP_KERNEL);
+	t = kzalloc_obj(*t);
 	if (!t)
 		goto out;
 
diff --git a/lib/crypto/md5.c b/lib/crypto/md5.c
index c0610ea1370e..c4af57db0ea8 100644
--- a/lib/crypto/md5.c
+++ b/lib/crypto/md5.c
@@ -29,7 +29,7 @@ static const struct md5_block_state md5_iv = {
 #define F4(x, y, z) (y ^ (x | ~z))
 
 #define MD5STEP(f, w, x, y, z, in, s) \
-	(w += f(x, y, z) + in, w = (w << s | w >> (32 - s)) + x)
+	(w += f(x, y, z) + in, w = rol32(w, s) + x)
 
 static void md5_block_generic(struct md5_block_state *state,
 			      const u8 data[MD5_BLOCK_SIZE])
diff --git a/lib/crypto/mldsa.c b/lib/crypto/mldsa.c
new file mode 100644
index 000000000000..c96fddc4e7dc
--- /dev/null
+++ b/lib/crypto/mldsa.c
@@ -0,0 +1,682 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Support for verifying ML-DSA signatures
+ *
+ * Copyright 2025 Google LLC
+ */
+
+#include <crypto/mldsa.h>
+#include <crypto/sha3.h>
+#include <kunit/visibility.h>
+#include <linux/export.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/unaligned.h>
+#include "fips-mldsa.h"
+
+#define Q 8380417 /* The prime q = 2^23 - 2^13 + 1 */
+#define QINV_MOD_2_32 58728449 /* Multiplicative inverse of q mod 2^32 */
+#define N 256 /* Number of components per ring element */
+#define D 13 /* Number of bits dropped from the public key vector t */
+#define RHO_LEN 32 /* Length of the public random seed in bytes */
+#define MAX_W1_ENCODED_LEN 192 /* Max encoded length of one element of w'_1 */
+
+/*
+ * The zetas array in Montgomery form, i.e. with extra factor of 2^32.
+ * Reference: FIPS 204 Section 7.5 "NTT and NTT^-1"
+ * Generated by the following Python code:
+ * q=8380417; [a%q - q*(a%q > q//2) for a in [1753**(int(f'{i:08b}'[::-1], 2)) << 32 for i in range(256)]]
+ */
+static const s32 zetas_times_2_32[N] = {
+	-4186625, 25847,    -2608894, -518909,	237124,	  -777960,  -876248,
+	466468,	  1826347,  2353451,  -359251,	-2091905, 3119733,  -2884855,
+	3111497,  2680103,  2725464,  1024112,	-1079900, 3585928,  -549488,
+	-1119584, 2619752,  -2108549, -2118186, -3859737, -1399561, -3277672,
+	1757237,  -19422,   4010497,  280005,	2706023,  95776,    3077325,
+	3530437,  -1661693, -3592148, -2537516, 3915439,  -3861115, -3043716,
+	3574422,  -2867647, 3539968,  -300467,	2348700,  -539299,  -1699267,
+	-1643818, 3505694,  -3821735, 3507263,	-2140649, -1600420, 3699596,
+	811944,	  531354,   954230,   3881043,	3900724,  -2556880, 2071892,
+	-2797779, -3930395, -1528703, -3677745, -3041255, -1452451, 3475950,
+	2176455,  -1585221, -1257611, 1939314,	-4083598, -1000202, -3190144,
+	-3157330, -3632928, 126922,   3412210,	-983419,  2147896,  2715295,
+	-2967645, -3693493, -411027,  -2477047, -671102,  -1228525, -22981,
+	-1308169, -381987,  1349076,  1852771,	-1430430, -3343383, 264944,
+	508951,	  3097992,  44288,    -1100098, 904516,	  3958618,  -3724342,
+	-8578,	  1653064,  -3249728, 2389356,	-210977,  759969,   -1316856,
+	189548,	  -3553272, 3159746,  -1851402, -2409325, -177440,  1315589,
+	1341330,  1285669,  -1584928, -812732,	-1439742, -3019102, -3881060,
+	-3628969, 3839961,  2091667,  3407706,	2316500,  3817976,  -3342478,
+	2244091,  -2446433, -3562462, 266997,	2434439,  -1235728, 3513181,
+	-3520352, -3759364, -1197226, -3193378, 900702,	  1859098,  909542,
+	819034,	  495491,   -1613174, -43260,	-522500,  -655327,  -3122442,
+	2031748,  3207046,  -3556995, -525098,	-768622,  -3595838, 342297,
+	286988,	  -2437823, 4108315,  3437287,	-3342277, 1735879,  203044,
+	2842341,  2691481,  -2590150, 1265009,	4055324,  1247620,  2486353,
+	1595974,  -3767016, 1250494,  2635921,	-3548272, -2994039, 1869119,
+	1903435,  -1050970, -1333058, 1237275,	-3318210, -1430225, -451100,
+	1312455,  3306115,  -1962642, -1279661, 1917081,  -2546312, -1374803,
+	1500165,  777191,   2235880,  3406031,	-542412,  -2831860, -1671176,
+	-1846953, -2584293, -3724270, 594136,	-3776993, -2013608, 2432395,
+	2454455,  -164721,  1957272,  3369112,	185531,	  -1207385, -3183426,
+	162844,	  1616392,  3014001,  810149,	1652634,  -3694233, -1799107,
+	-3038916, 3523897,  3866901,  269760,	2213111,  -975884,  1717735,
+	472078,	  -426683,  1723600,  -1803090, 1910376,  -1667432, -1104333,
+	-260646,  -3833893, -2939036, -2235985, -420899,  -2286327, 183443,
+	-976891,  1612842,  -3545687, -554416,	3919660,  -48306,   -1362209,
+	3937738,  1400424,  -846154,  1976782
+};
+
+/* Reference: FIPS 204 Section 4 "Parameter Sets" */
+static const struct mldsa_parameter_set {
+	u8 k; /* num rows in the matrix A */
+	u8 l; /* num columns in the matrix A */
+	u8 ctilde_len; /* length of commitment hash ctilde in bytes; lambda/4 */
+	u8 omega; /* max num of 1's in the hint vector h */
+	u8 tau; /* num of +-1's in challenge c */
+	u8 beta; /* tau times eta */
+	u16 pk_len; /* length of public keys in bytes */
+	u16 sig_len; /* length of signatures in bytes */
+	s32 gamma1; /* coefficient range of y */
+} mldsa_parameter_sets[] = {
+	[MLDSA44] = {
+		.k = 4,
+		.l = 4,
+		.ctilde_len = 32,
+		.omega = 80,
+		.tau = 39,
+		.beta = 78,
+		.pk_len = MLDSA44_PUBLIC_KEY_SIZE,
+		.sig_len = MLDSA44_SIGNATURE_SIZE,
+		.gamma1 = 1 << 17,
+	},
+	[MLDSA65] = {
+		.k = 6,
+		.l = 5,
+		.ctilde_len = 48,
+		.omega = 55,
+		.tau = 49,
+		.beta = 196,
+		.pk_len = MLDSA65_PUBLIC_KEY_SIZE,
+		.sig_len = MLDSA65_SIGNATURE_SIZE,
+		.gamma1 = 1 << 19,
+	},
+	[MLDSA87] = {
+		.k = 8,
+		.l = 7,
+		.ctilde_len = 64,
+		.omega = 75,
+		.tau = 60,
+		.beta = 120,
+		.pk_len = MLDSA87_PUBLIC_KEY_SIZE,
+		.sig_len = MLDSA87_SIGNATURE_SIZE,
+		.gamma1 = 1 << 19,
+	},
+};
+
+/*
+ * An element of the ring R_q (normal form) or the ring T_q (NTT form).  It
+ * consists of N integers mod q: either the polynomial coefficients of the R_q
+ * element or the components of the T_q element.  In either case, whether they
+ * are fully reduced to [0, q - 1] varies in the different parts of the code.
+ */
+struct mldsa_ring_elem {
+	s32 x[N];
+};
+
+struct mldsa_verification_workspace {
+	/* SHAKE context for computing c, mu, and ctildeprime */
+	struct shake_ctx shake;
+	/* The fields in this union are used in their order of declaration. */
+	union {
+		/* The hash of the public key */
+		u8 tr[64];
+		/* The message representative mu */
+		u8 mu[64];
+		/* Temporary space for rej_ntt_poly() */
+		u8 block[SHAKE128_BLOCK_SIZE + 1];
+		/* Encoded element of w'_1 */
+		u8 w1_encoded[MAX_W1_ENCODED_LEN];
+		/* The commitment hash.  Real length is params->ctilde_len */
+		u8 ctildeprime[64];
+	};
+	/* SHAKE context for generating elements of the matrix A */
+	struct shake_ctx a_shake;
+	/*
+	 * An element of the matrix A generated from the public seed, or an
+	 * element of the vector t_1 decoded from the public key and pre-scaled
+	 * by 2^d.  Both are in NTT form.  To reduce memory usage, we generate
+	 * or decode these elements only as needed.
+	 */
+	union {
+		struct mldsa_ring_elem a;
+		struct mldsa_ring_elem t1_scaled;
+	};
+	/* The challenge c, generated from ctilde */
+	struct mldsa_ring_elem c;
+	/* A temporary element used during calculations */
+	struct mldsa_ring_elem tmp;
+
+	/* The following fields are variable-length: */
+
+	/* The signer's response vector */
+	struct mldsa_ring_elem z[/* l */];
+
+	/* The signer's hint vector */
+	/* u8 h[k * N]; */
+};
+
+/*
+ * Compute a * b * 2^-32 mod q.  a * b must be in the range [-2^31 * q, 2^31 * q
+ * - 1] before reduction.  The return value is in the range [-q + 1, q - 1].
+ *
+ * To reduce mod q efficiently, this uses Montgomery reduction with R=2^32.
+ * That's where the factor of 2^-32 comes from.  The caller must include a
+ * factor of 2^32 at some point to compensate for that.
+ *
+ * To keep the input and output ranges very close to symmetric, this
+ * specifically does a "signed" Montgomery reduction.  That is, when computing
+ * d = c * q^-1 mod 2^32, this chooses a representative in [S32_MIN, S32_MAX]
+ * rather than [0, U32_MAX], i.e. s32 rather than u32.  This matters in the
+ * wider multiplication d * Q when d keeps its value via sign extension.
+ *
+ * Reference: FIPS 204 Appendix A "Montgomery Multiplication".  But, it doesn't
+ * explain it properly: it has an off-by-one error in the upper end of the input
+ * range, it doesn't clarify that the signed version should be used, and it
+ * gives an unnecessarily large output range.  A better citation is perhaps the
+ * Dilithium reference code, which functionally matches the below code and
+ * merely has the (benign) off-by-one error in its documentation.
+ */
+static inline s32 Zq_mult(s32 a, s32 b)
+{
+	/* Compute the unreduced product c. */
+	s64 c = (s64)a * b;
+
+	/*
+	 * Compute d = c * q^-1 mod 2^32.  Generate a signed result, as
+	 * explained above, but do the actual multiplication using an unsigned
+	 * type to avoid signed integer overflow which is undefined behavior.
+	 */
+	s32 d = (u32)c * QINV_MOD_2_32;
+
+	/*
+	 * Compute e = c - d * q.  This makes the low 32 bits zero, since
+	 *   c - (c * q^-1) * q mod 2^32
+	 * = c - c * (q^-1 * q) mod 2^32
+	 * = c - c * 1 mod 2^32
+	 * = c - c mod 2^32
+	 * = 0 mod 2^32
+	 */
+	s64 e = c - (s64)d * Q;
+
+	/* Finally, return e * 2^-32. */
+	return e >> 32;
+}
+
+/*
+ * Convert @w to its number-theoretically-transformed representation in-place.
+ * Reference: FIPS 204 Algorithm 41, NTT
+ *
+ * To prevent intermediate overflows, all input coefficients must have absolute
+ * value < q.  All output components have absolute value < 9*q.
+ */
+static void ntt(struct mldsa_ring_elem *w)
+{
+	int m = 0; /* index in zetas_times_2_32 */
+
+	for (int len = 128; len >= 1; len /= 2) {
+		for (int start = 0; start < 256; start += 2 * len) {
+			const s32 z = zetas_times_2_32[++m];
+
+			for (int j = start; j < start + len; j++) {
+				s32 t = Zq_mult(z, w->x[j + len]);
+
+				w->x[j + len] = w->x[j] - t;
+				w->x[j] += t;
+			}
+		}
+	}
+}
+
+/*
+ * Convert @w from its number-theoretically-transformed representation in-place.
+ * Reference: FIPS 204 Algorithm 42, NTT^-1
+ *
+ * This also multiplies the coefficients by 2^32, undoing an extra factor of
+ * 2^-32 introduced earlier, and reduces the coefficients to [0, q - 1].
+ */
+static void invntt_and_mul_2_32(struct mldsa_ring_elem *w)
+{
+	int m = 256; /* index in zetas_times_2_32 */
+
+	/* Prevent intermediate overflows. */
+	for (int j = 0; j < 256; j++)
+		w->x[j] %= Q;
+
+	for (int len = 1; len < 256; len *= 2) {
+		for (int start = 0; start < 256; start += 2 * len) {
+			const s32 z = -zetas_times_2_32[--m];
+
+			for (int j = start; j < start + len; j++) {
+				s32 t = w->x[j];
+
+				w->x[j] = t + w->x[j + len];
+				w->x[j + len] = Zq_mult(z, t - w->x[j + len]);
+			}
+		}
+	}
+	/*
+	 * Multiply by 2^32 * 256^-1.  2^32 cancels the factor of 2^-32 from
+	 * earlier Montgomery multiplications.  256^-1 is for NTT^-1.  This
+	 * itself uses Montgomery multiplication, so *another* 2^32 is needed.
+	 * Thus the actual multiplicand is 2^32 * 2^32 * 256^-1 mod q = 41978.
+	 *
+	 * Finally, also reduce from [-q + 1, q - 1] to [0, q - 1].
+	 */
+	for (int j = 0; j < 256; j++) {
+		w->x[j] = Zq_mult(w->x[j], 41978);
+		w->x[j] += (w->x[j] >> 31) & Q;
+	}
+}
+
+/*
+ * Decode an element of t_1, i.e. the high d bits of t = A*s_1 + s_2.
+ * Reference: FIPS 204 Algorithm 23, pkDecode.
+ * Also multiply it by 2^d and convert it to NTT form.
+ */
+static const u8 *decode_t1_elem(struct mldsa_ring_elem *out,
+				const u8 *t1_encoded)
+{
+	for (int j = 0; j < N; j += 4, t1_encoded += 5) {
+		u32 v = get_unaligned_le32(t1_encoded);
+
+		out->x[j + 0] = ((v >> 0) & 0x3ff) << D;
+		out->x[j + 1] = ((v >> 10) & 0x3ff) << D;
+		out->x[j + 2] = ((v >> 20) & 0x3ff) << D;
+		out->x[j + 3] = ((v >> 30) | (t1_encoded[4] << 2)) << D;
+		static_assert(0x3ff << D < Q); /* All coefficients < q. */
+	}
+	ntt(out);
+	return t1_encoded; /* Return updated pointer. */
+}
+
+/*
+ * Decode the signer's response vector 'z' from the signature.
+ * Reference: FIPS 204 Algorithm 27, sigDecode.
+ *
+ * This also validates that the coefficients of z are in range, corresponding
+ * the infinity norm check at the end of Algorithm 8, ML-DSA.Verify_internal.
+ *
+ * Finally, this also converts z to NTT form.
+ */
+static bool decode_z(struct mldsa_ring_elem z[/* l */], int l, s32 gamma1,
+		     int beta, const u8 **sig_ptr)
+{
+	const u8 *sig = *sig_ptr;
+
+	for (int i = 0; i < l; i++) {
+		if (l == 4) { /* ML-DSA-44? */
+			/* 18-bit coefficients: decode 4 from 9 bytes. */
+			for (int j = 0; j < N; j += 4, sig += 9) {
+				u64 v = get_unaligned_le64(sig);
+
+				z[i].x[j + 0] = (v >> 0) & 0x3ffff;
+				z[i].x[j + 1] = (v >> 18) & 0x3ffff;
+				z[i].x[j + 2] = (v >> 36) & 0x3ffff;
+				z[i].x[j + 3] = (v >> 54) | (sig[8] << 10);
+			}
+		} else {
+			/* 20-bit coefficients: decode 4 from 10 bytes. */
+			for (int j = 0; j < N; j += 4, sig += 10) {
+				u64 v = get_unaligned_le64(sig);
+
+				z[i].x[j + 0] = (v >> 0) & 0xfffff;
+				z[i].x[j + 1] = (v >> 20) & 0xfffff;
+				z[i].x[j + 2] = (v >> 40) & 0xfffff;
+				z[i].x[j + 3] =
+					(v >> 60) |
+					(get_unaligned_le16(&sig[8]) << 4);
+			}
+		}
+		for (int j = 0; j < N; j++) {
+			z[i].x[j] = gamma1 - z[i].x[j];
+			if (z[i].x[j] <= -(gamma1 - beta) ||
+			    z[i].x[j] >= gamma1 - beta)
+				return false;
+		}
+		ntt(&z[i]);
+	}
+	*sig_ptr = sig; /* Return updated pointer. */
+	return true;
+}
+
+/*
+ * Decode the signer's hint vector 'h' from the signature.
+ * Reference: FIPS 204 Algorithm 21, HintBitUnpack
+ *
+ * Note that there are several ways in which the hint vector can be malformed.
+ */
+static bool decode_hint_vector(u8 h[/* k * N */], int k, int omega, const u8 *y)
+{
+	int index = 0;
+
+	memset(h, 0, k * N);
+	for (int i = 0; i < k; i++) {
+		int count = y[omega + i]; /* num 1's in elems 0 through i */
+		int prev = -1;
+
+		/* Cumulative count mustn't decrease or exceed omega. */
+		if (count < index || count > omega)
+			return false;
+		for (; index < count; index++) {
+			if (prev >= y[index]) /* Coefficients out of order? */
+				return false;
+			prev = y[index];
+			h[i * N + y[index]] = 1;
+		}
+	}
+	return mem_is_zero(&y[index], omega - index);
+}
+
+/*
+ * Expand @seed into an element of R_q @c with coefficients in {-1, 0, 1},
+ * exactly @tau of them nonzero.  Reference: FIPS 204 Algorithm 29, SampleInBall
+ */
+static void sample_in_ball(struct mldsa_ring_elem *c, const u8 *seed,
+			   size_t seed_len, int tau, struct shake_ctx *shake)
+{
+	u64 signs;
+	u8 j;
+
+	shake256_init(shake);
+	shake_update(shake, seed, seed_len);
+	shake_squeeze(shake, (u8 *)&signs, sizeof(signs));
+	le64_to_cpus(&signs);
+	*c = (struct mldsa_ring_elem){};
+	for (int i = N - tau; i < N; i++, signs >>= 1) {
+		do {
+			shake_squeeze(shake, &j, 1);
+		} while (j > i);
+		c->x[i] = c->x[j];
+		c->x[j] = 1 - 2 * (s32)(signs & 1);
+	}
+}
+
+/*
+ * Expand the public seed @rho and @row_and_column into an element of T_q @out.
+ * Reference: FIPS 204 Algorithm 30, RejNTTPoly
+ *
+ * @shake and @block are temporary space used by the expansion.  @block has
+ * space for one SHAKE128 block, plus an extra byte to allow reading a u32 from
+ * the final 3-byte group without reading out-of-bounds.
+ */
+static void rej_ntt_poly(struct mldsa_ring_elem *out, const u8 rho[RHO_LEN],
+			 __le16 row_and_column, struct shake_ctx *shake,
+			 u8 block[SHAKE128_BLOCK_SIZE + 1])
+{
+	shake128_init(shake);
+	shake_update(shake, rho, RHO_LEN);
+	shake_update(shake, (u8 *)&row_and_column, sizeof(row_and_column));
+	for (int i = 0; i < N;) {
+		shake_squeeze(shake, block, SHAKE128_BLOCK_SIZE);
+		block[SHAKE128_BLOCK_SIZE] = 0; /* for KMSAN */
+		static_assert(SHAKE128_BLOCK_SIZE % 3 == 0);
+		for (int j = 0; j < SHAKE128_BLOCK_SIZE && i < N; j += 3) {
+			u32 x = get_unaligned_le32(&block[j]) & 0x7fffff;
+
+			if (x < Q) /* Ignore values >= q. */
+				out->x[i++] = x;
+		}
+	}
+}
+
+/*
+ * Return the HighBits of r adjusted according to hint h
+ * Reference: FIPS 204 Algorithm 40, UseHint
+ *
+ * This is needed because of the public key compression in ML-DSA.
+ *
+ * h is either 0 or 1, r is in [0, q - 1], and gamma2 is either (q - 1) / 88 or
+ * (q - 1) / 32.  Except when invoked via the unit test interface, gamma2 is a
+ * compile-time constant, so compilers will optimize the code accordingly.
+ */
+static __always_inline s32 use_hint(u8 h, s32 r, const s32 gamma2)
+{
+	const s32 m = (Q - 1) / (2 * gamma2); /* 44 or 16, compile-time const */
+	s32 r1;
+
+	/*
+	 * Handle the special case where r - (r mod+- (2 * gamma2)) == q - 1,
+	 * i.e. r >= q - gamma2.  This is also exactly where the computation of
+	 * r1 below would produce 'm' and would need a correction.
+	 */
+	if (r >= Q - gamma2)
+		return h == 0 ? 0 : m - 1;
+
+	/*
+	 * Compute the (non-hint-adjusted) HighBits r1 as:
+	 *
+	 *  r1 = (r - (r mod+- (2 * gamma2))) / (2 * gamma2)
+	 *     = floor((r + gamma2 - 1) / (2 * gamma2))
+	 *
+	 * Note that when '2 * gamma2' is a compile-time constant, compilers
+	 * optimize the division to a reciprocal multiplication and shift.
+	 */
+	r1 = (u32)(r + gamma2 - 1) / (2 * gamma2);
+
+	/*
+	 * Return the HighBits r1:
+	 *	+ 0 if the hint is 0;
+	 *	+ 1 (mod m) if the hint is 1 and the LowBits are positive;
+	 *	- 1 (mod m) if the hint is 1 and the LowBits are negative or 0.
+	 *
+	 * r1 is in (and remains in) [0, m - 1].  Note that when 'm' is a
+	 * compile-time constant, compilers optimize the '% m' accordingly.
+	 */
+	if (h == 0)
+		return r1;
+	if (r > r1 * (2 * gamma2))
+		return (u32)(r1 + 1) % m;
+	return (u32)(r1 + m - 1) % m;
+}
+
+static __always_inline void use_hint_elem(struct mldsa_ring_elem *w,
+					  const u8 h[N], const s32 gamma2)
+{
+	for (int j = 0; j < N; j++)
+		w->x[j] = use_hint(h[j], w->x[j], gamma2);
+}
+
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_MLDSA_KUNIT_TEST)
+/* Allow the __always_inline function use_hint() to be unit-tested. */
+s32 mldsa_use_hint(u8 h, s32 r, s32 gamma2)
+{
+	return use_hint(h, r, gamma2);
+}
+EXPORT_SYMBOL_IF_KUNIT(mldsa_use_hint);
+#endif
+
+/*
+ * Encode one element of the commitment vector w'_1 into a byte string.
+ * Reference: FIPS 204 Algorithm 28, w1Encode.
+ * Return the number of bytes used: 192 for ML-DSA-44 and 128 for the others.
+ */
+static size_t encode_w1(u8 out[MAX_W1_ENCODED_LEN],
+			const struct mldsa_ring_elem *w1, int k)
+{
+	size_t pos = 0;
+
+	static_assert(N * 6 / 8 == MAX_W1_ENCODED_LEN);
+	if (k == 4) { /* ML-DSA-44? */
+		/* 6 bits per coefficient.  Pack 4 at a time. */
+		for (int j = 0; j < N; j += 4) {
+			u32 v = (w1->x[j + 0] << 0) | (w1->x[j + 1] << 6) |
+				(w1->x[j + 2] << 12) | (w1->x[j + 3] << 18);
+			out[pos++] = v >> 0;
+			out[pos++] = v >> 8;
+			out[pos++] = v >> 16;
+		}
+	} else {
+		/* 4 bits per coefficient.  Pack 2 at a time. */
+		for (int j = 0; j < N; j += 2)
+			out[pos++] = w1->x[j] | (w1->x[j + 1] << 4);
+	}
+	return pos;
+}
+
+int mldsa_verify(enum mldsa_alg alg, const u8 *sig, size_t sig_len,
+		 const u8 *msg, size_t msg_len, const u8 *pk, size_t pk_len)
+{
+	const struct mldsa_parameter_set *params = &mldsa_parameter_sets[alg];
+	const int k = params->k, l = params->l;
+	/* For now this just does pure ML-DSA with an empty context string. */
+	static const u8 msg_prefix[2] = { /* dom_sep= */ 0, /* ctx_len= */ 0 };
+	const u8 *ctilde; /* The signer's commitment hash */
+	const u8 *t1_encoded = &pk[RHO_LEN]; /* Next encoded element of t_1 */
+	u8 *h; /* The signer's hint vector, length k * N */
+	size_t w1_enc_len;
+
+	/* Validate the public key and signature lengths. */
+	if (pk_len != params->pk_len || sig_len != params->sig_len)
+		return -EBADMSG;
+
+	/*
+	 * Allocate the workspace, including variable-length fields.  Its size
+	 * depends only on the ML-DSA parameter set, not the other inputs.
+	 *
+	 * For freeing it, use kfree_sensitive() rather than kfree().  This is
+	 * mainly to comply with FIPS 204 Section 3.6.3 "Intermediate Values".
+	 * In reality it's a bit gratuitous, as this is a public key operation.
+	 */
+	struct mldsa_verification_workspace *ws __free(kfree_sensitive) =
+		kmalloc(sizeof(*ws) + (l * sizeof(ws->z[0])) + (k * N),
+			GFP_KERNEL);
+	if (!ws)
+		return -ENOMEM;
+	h = (u8 *)&ws->z[l];
+
+	/* Decode the signature.  Reference: FIPS 204 Algorithm 27, sigDecode */
+	ctilde = sig;
+	sig += params->ctilde_len;
+	if (!decode_z(ws->z, l, params->gamma1, params->beta, &sig))
+		return -EBADMSG;
+	if (!decode_hint_vector(h, k, params->omega, sig))
+		return -EBADMSG;
+
+	/* Recreate the challenge c from the signer's commitment hash. */
+	sample_in_ball(&ws->c, ctilde, params->ctilde_len, params->tau,
+		       &ws->shake);
+	ntt(&ws->c);
+
+	/* Compute the message representative mu. */
+	shake256(pk, pk_len, ws->tr, sizeof(ws->tr));
+	shake256_init(&ws->shake);
+	shake_update(&ws->shake, ws->tr, sizeof(ws->tr));
+	shake_update(&ws->shake, msg_prefix, sizeof(msg_prefix));
+	shake_update(&ws->shake, msg, msg_len);
+	shake_squeeze(&ws->shake, ws->mu, sizeof(ws->mu));
+
+	/* Start computing ctildeprime = H(mu || w1Encode(w'_1)). */
+	shake256_init(&ws->shake);
+	shake_update(&ws->shake, ws->mu, sizeof(ws->mu));
+
+	/*
+	 * Compute the commitment w'_1 from A, z, c, t_1, and h.
+	 *
+	 * The computation is the same for each of the k rows.  Just do each row
+	 * before moving on to the next, resulting in only one loop over k.
+	 */
+	for (int i = 0; i < k; i++) {
+		/*
+		 * tmp = NTT(A) * NTT(z) * 2^-32
+		 * To reduce memory use, generate each element of NTT(A)
+		 * on-demand.  Note that each element is used only once.
+		 */
+		ws->tmp = (struct mldsa_ring_elem){};
+		for (int j = 0; j < l; j++) {
+			rej_ntt_poly(&ws->a, pk /* rho is first field of pk */,
+				     cpu_to_le16((i << 8) | j), &ws->a_shake,
+				     ws->block);
+			for (int n = 0; n < N; n++)
+				ws->tmp.x[n] +=
+					Zq_mult(ws->a.x[n], ws->z[j].x[n]);
+		}
+		/* All components of tmp now have abs value < l*q. */
+
+		/* Decode the next element of t_1. */
+		t1_encoded = decode_t1_elem(&ws->t1_scaled, t1_encoded);
+
+		/*
+		 * tmp -= NTT(c) * NTT(t_1 * 2^d) * 2^-32
+		 *
+		 * Taking a conservative bound for the output of ntt(), the
+		 * multiplicands can have absolute value up to 9*q.  That
+		 * corresponds to a product with absolute value 81*q^2.  That is
+		 * within the limits of Zq_mult() which needs < ~256*q^2.
+		 */
+		for (int j = 0; j < N; j++)
+			ws->tmp.x[j] -= Zq_mult(ws->c.x[j], ws->t1_scaled.x[j]);
+		/* All components of tmp now have abs value < (l+1)*q. */
+
+		/* tmp = w'_Approx = NTT^-1(tmp) * 2^32 */
+		invntt_and_mul_2_32(&ws->tmp);
+		/* All coefficients of tmp are now in [0, q - 1]. */
+
+		/*
+		 * tmp = w'_1 = UseHint(h, w'_Approx)
+		 * For efficiency, set gamma2 to a compile-time constant.
+		 */
+		if (k == 4)
+			use_hint_elem(&ws->tmp, &h[i * N], (Q - 1) / 88);
+		else
+			use_hint_elem(&ws->tmp, &h[i * N], (Q - 1) / 32);
+
+		/* Encode and hash the next element of w'_1. */
+		w1_enc_len = encode_w1(ws->w1_encoded, &ws->tmp, k);
+		shake_update(&ws->shake, ws->w1_encoded, w1_enc_len);
+	}
+
+	/* Finish computing ctildeprime. */
+	shake_squeeze(&ws->shake, ws->ctildeprime, params->ctilde_len);
+
+	/* Verify that ctilde == ctildeprime. */
+	if (memcmp(ws->ctildeprime, ctilde, params->ctilde_len) != 0)
+		return -EKEYREJECTED;
+	/* ||z||_infinity < gamma1 - beta was already checked in decode_z(). */
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mldsa_verify);
+
+#ifdef CONFIG_CRYPTO_FIPS
+static int __init mldsa_mod_init(void)
+{
+	if (fips_enabled) {
+		/*
+		 * FIPS cryptographic algorithm self-test.  As per the FIPS
+		 * Implementation Guidance, testing any ML-DSA parameter set
+		 * satisfies the test requirement for all of them, and only a
+		 * positive test is required.
+		 */
+		int err = mldsa_verify(MLDSA65, fips_test_mldsa65_signature,
+				       sizeof(fips_test_mldsa65_signature),
+				       fips_test_mldsa65_message,
+				       sizeof(fips_test_mldsa65_message),
+				       fips_test_mldsa65_public_key,
+				       sizeof(fips_test_mldsa65_public_key));
+		if (err)
+			panic("mldsa: FIPS self-test failed; err=%pe\n",
+			      ERR_PTR(err));
+	}
+	return 0;
+}
+subsys_initcall(mldsa_mod_init);
+
+static void __exit mldsa_mod_exit(void)
+{
+}
+module_exit(mldsa_mod_exit);
+#endif /* CONFIG_CRYPTO_FIPS */
+
+MODULE_DESCRIPTION("ML-DSA signature verification");
+MODULE_LICENSE("GPL");
diff --git a/lib/crypto/mpi/mpicoder.c b/lib/crypto/mpi/mpicoder.c
index 47f6939599b3..bf716a03c704 100644
--- a/lib/crypto/mpi/mpicoder.c
+++ b/lib/crypto/mpi/mpicoder.c
@@ -398,7 +398,7 @@ MPI mpi_read_raw_from_sgl(struct scatterlist *sgl, unsigned int nbytes)
 
 	while (sg_miter_next(&miter)) {
 		buff = miter.addr;
-		len = min_t(unsigned, miter.length, nbytes);
+		len = min(miter.length, nbytes);
 		nbytes -= len;
 
 		for (x = 0; x < len; x++) {
diff --git a/lib/crypto/mpi/mpih-mul.c b/lib/crypto/mpi/mpih-mul.c
index a93647564054..29dd80609c47 100644
--- a/lib/crypto/mpi/mpih-mul.c
+++ b/lib/crypto/mpi/mpih-mul.c
@@ -372,7 +372,7 @@ mpihelp_mul_karatsuba_case(mpi_ptr_t prodp,
 				return -ENOMEM;
 		} else {
 			if (!ctx->next) {
-				ctx->next = kzalloc(sizeof *ctx, GFP_KERNEL);
+				ctx->next = kzalloc_obj(*ctx);
 				if (!ctx->next)
 					return -ENOMEM;
 			}
diff --git a/lib/crypto/mpi/mpiutil.c b/lib/crypto/mpi/mpiutil.c
index 7f2db830f404..f4faf7c903f9 100644
--- a/lib/crypto/mpi/mpiutil.c
+++ b/lib/crypto/mpi/mpiutil.c
@@ -33,7 +33,7 @@ MPI mpi_alloc(unsigned nlimbs)
 {
 	MPI a;
 
-	a = kmalloc(sizeof *a, GFP_KERNEL);
+	a = kmalloc_obj(*a);
 	if (!a)
 		return a;
 
@@ -93,14 +93,14 @@ int mpi_resize(MPI a, unsigned nlimbs)
 		return 0;	/* no need to do it */
 
 	if (a->d) {
-		p = kcalloc(nlimbs, sizeof(mpi_limb_t), GFP_KERNEL);
+		p = kzalloc_objs(mpi_limb_t, nlimbs);
 		if (!p)
 			return -ENOMEM;
 		memcpy(p, a->d, a->alloced * sizeof(mpi_limb_t));
 		kfree_sensitive(a->d);
 		a->d = p;
 	} else {
-		a->d = kcalloc(nlimbs, sizeof(mpi_limb_t), GFP_KERNEL);
+		a->d = kzalloc_objs(mpi_limb_t, nlimbs);
 		if (!a->d)
 			return -ENOMEM;
 	}
diff --git a/lib/crypto/nh.c b/lib/crypto/nh.c
new file mode 100644
index 000000000000..e1d0095b5289
--- /dev/null
+++ b/lib/crypto/nh.c
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2018 Google LLC
+ */
+
+/*
+ * Implementation of the NH almost-universal hash function, specifically the
+ * variant of NH used in Adiantum.  This is *not* a cryptographic hash function.
+ *
+ * Reference: section 6.3 of "Adiantum: length-preserving encryption for
+ * entry-level processors" (https://eprint.iacr.org/2018/720.pdf).
+ */
+
+#include <crypto/nh.h>
+#include <linux/export.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/unaligned.h>
+
+#ifdef CONFIG_CRYPTO_LIB_NH_ARCH
+#include "nh.h" /* $(SRCARCH)/nh.h */
+#else
+static bool nh_arch(const u32 *key, const u8 *message, size_t message_len,
+		    __le64 hash[NH_NUM_PASSES])
+{
+	return false;
+}
+#endif
+
+void nh(const u32 *key, const u8 *message, size_t message_len,
+	__le64 hash[NH_NUM_PASSES])
+{
+	u64 sums[4] = { 0, 0, 0, 0 };
+
+	if (nh_arch(key, message, message_len, hash))
+		return;
+
+	static_assert(NH_PAIR_STRIDE == 2);
+	static_assert(NH_NUM_PASSES == 4);
+
+	while (message_len) {
+		u32 m0 = get_unaligned_le32(message + 0);
+		u32 m1 = get_unaligned_le32(message + 4);
+		u32 m2 = get_unaligned_le32(message + 8);
+		u32 m3 = get_unaligned_le32(message + 12);
+
+		sums[0] += (u64)(u32)(m0 + key[0]) * (u32)(m2 + key[2]);
+		sums[1] += (u64)(u32)(m0 + key[4]) * (u32)(m2 + key[6]);
+		sums[2] += (u64)(u32)(m0 + key[8]) * (u32)(m2 + key[10]);
+		sums[3] += (u64)(u32)(m0 + key[12]) * (u32)(m2 + key[14]);
+		sums[0] += (u64)(u32)(m1 + key[1]) * (u32)(m3 + key[3]);
+		sums[1] += (u64)(u32)(m1 + key[5]) * (u32)(m3 + key[7]);
+		sums[2] += (u64)(u32)(m1 + key[9]) * (u32)(m3 + key[11]);
+		sums[3] += (u64)(u32)(m1 + key[13]) * (u32)(m3 + key[15]);
+		key += NH_MESSAGE_UNIT / sizeof(key[0]);
+		message += NH_MESSAGE_UNIT;
+		message_len -= NH_MESSAGE_UNIT;
+	}
+
+	hash[0] = cpu_to_le64(sums[0]);
+	hash[1] = cpu_to_le64(sums[1]);
+	hash[2] = cpu_to_le64(sums[2]);
+	hash[3] = cpu_to_le64(sums[3]);
+}
+EXPORT_SYMBOL_GPL(nh);
+
+#ifdef nh_mod_init_arch
+static int __init nh_mod_init(void)
+{
+	nh_mod_init_arch();
+	return 0;
+}
+subsys_initcall(nh_mod_init);
+
+static void __exit nh_mod_exit(void)
+{
+}
+module_exit(nh_mod_exit);
+#endif
+
+MODULE_DESCRIPTION("NH almost-universal hash function");
+MODULE_LICENSE("GPL");
diff --git a/lib/crypto/polyval.c b/lib/crypto/polyval.c
new file mode 100644
index 000000000000..5796275f574a
--- /dev/null
+++ b/lib/crypto/polyval.c
@@ -0,0 +1,307 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * POLYVAL library functions
+ *
+ * Copyright 2025 Google LLC
+ */
+
+#include <crypto/polyval.h>
+#include <linux/export.h>
+#include <linux/module.h>
+#include <linux/string.h>
+#include <linux/unaligned.h>
+
+/*
+ * POLYVAL is an almost-XOR-universal hash function.  Similar to GHASH, POLYVAL
+ * interprets the message as the coefficients of a polynomial in GF(2^128) and
+ * evaluates that polynomial at a secret point.  POLYVAL has a simple
+ * mathematical relationship with GHASH, but it uses a better field convention
+ * which makes it easier and faster to implement.
+ *
+ * POLYVAL is not a cryptographic hash function, and it should be used only by
+ * algorithms that are specifically designed to use it.
+ *
+ * POLYVAL is specified by "AES-GCM-SIV: Nonce Misuse-Resistant Authenticated
+ * Encryption" (https://datatracker.ietf.org/doc/html/rfc8452)
+ *
+ * POLYVAL is also used by HCTR2.  See "Length-preserving encryption with HCTR2"
+ * (https://eprint.iacr.org/2021/1441.pdf).
+ *
+ * This file provides a library API for POLYVAL.  This API can delegate to
+ * either a generic implementation or an architecture-optimized implementation.
+ *
+ * For the generic implementation, we don't use the traditional table approach
+ * to GF(2^128) multiplication.  That approach is not constant-time and requires
+ * a lot of memory.  Instead, we use a different approach which emulates
+ * carryless multiplication using standard multiplications by spreading the data
+ * bits apart using "holes".  This allows the carries to spill harmlessly.  This
+ * approach is borrowed from BoringSSL, which in turn credits BearSSL's
+ * documentation (https://bearssl.org/constanttime.html#ghash-for-gcm) for the
+ * "holes" trick and a presentation by Shay Gueron
+ * (https://crypto.stanford.edu/RealWorldCrypto/slides/gueron.pdf) for the
+ * 256-bit => 128-bit reduction algorithm.
+ */
+
+#ifdef CONFIG_ARCH_SUPPORTS_INT128
+
+/* Do a 64 x 64 => 128 bit carryless multiplication. */
+static void clmul64(u64 a, u64 b, u64 *out_lo, u64 *out_hi)
+{
+	/*
+	 * With 64-bit multiplicands and one term every 4 bits, there would be
+	 * up to 64 / 4 = 16 one bits per column when each multiplication is
+	 * written out as a series of additions in the schoolbook manner.
+	 * Unfortunately, that doesn't work since the value 16 is 1 too large to
+	 * fit in 4 bits.  Carries would sometimes overflow into the next term.
+	 *
+	 * Using one term every 5 bits would work.  However, that would cost
+	 * 5 x 5 = 25 multiplications instead of 4 x 4 = 16.
+	 *
+	 * Instead, mask off 4 bits from one multiplicand, giving a max of 15
+	 * one bits per column.  Then handle those 4 bits separately.
+	 */
+	u64 a0 = a & 0x1111111111111110;
+	u64 a1 = a & 0x2222222222222220;
+	u64 a2 = a & 0x4444444444444440;
+	u64 a3 = a & 0x8888888888888880;
+
+	u64 b0 = b & 0x1111111111111111;
+	u64 b1 = b & 0x2222222222222222;
+	u64 b2 = b & 0x4444444444444444;
+	u64 b3 = b & 0x8888888888888888;
+
+	/* Multiply the high 60 bits of @a by @b. */
+	u128 c0 = (a0 * (u128)b0) ^ (a1 * (u128)b3) ^
+		  (a2 * (u128)b2) ^ (a3 * (u128)b1);
+	u128 c1 = (a0 * (u128)b1) ^ (a1 * (u128)b0) ^
+		  (a2 * (u128)b3) ^ (a3 * (u128)b2);
+	u128 c2 = (a0 * (u128)b2) ^ (a1 * (u128)b1) ^
+		  (a2 * (u128)b0) ^ (a3 * (u128)b3);
+	u128 c3 = (a0 * (u128)b3) ^ (a1 * (u128)b2) ^
+		  (a2 * (u128)b1) ^ (a3 * (u128)b0);
+
+	/* Multiply the low 4 bits of @a by @b. */
+	u64 e0 = -(a & 1) & b;
+	u64 e1 = -((a >> 1) & 1) & b;
+	u64 e2 = -((a >> 2) & 1) & b;
+	u64 e3 = -((a >> 3) & 1) & b;
+	u64 extra_lo = e0 ^ (e1 << 1) ^ (e2 << 2) ^ (e3 << 3);
+	u64 extra_hi = (e1 >> 63) ^ (e2 >> 62) ^ (e3 >> 61);
+
+	/* Add all the intermediate products together. */
+	*out_lo = (((u64)c0) & 0x1111111111111111) ^
+		  (((u64)c1) & 0x2222222222222222) ^
+		  (((u64)c2) & 0x4444444444444444) ^
+		  (((u64)c3) & 0x8888888888888888) ^ extra_lo;
+	*out_hi = (((u64)(c0 >> 64)) & 0x1111111111111111) ^
+		  (((u64)(c1 >> 64)) & 0x2222222222222222) ^
+		  (((u64)(c2 >> 64)) & 0x4444444444444444) ^
+		  (((u64)(c3 >> 64)) & 0x8888888888888888) ^ extra_hi;
+}
+
+#else /* CONFIG_ARCH_SUPPORTS_INT128 */
+
+/* Do a 32 x 32 => 64 bit carryless multiplication. */
+static u64 clmul32(u32 a, u32 b)
+{
+	/*
+	 * With 32-bit multiplicands and one term every 4 bits, there are up to
+	 * 32 / 4 = 8 one bits per column when each multiplication is written
+	 * out as a series of additions in the schoolbook manner.  The value 8
+	 * fits in 4 bits, so the carries don't overflow into the next term.
+	 */
+	u32 a0 = a & 0x11111111;
+	u32 a1 = a & 0x22222222;
+	u32 a2 = a & 0x44444444;
+	u32 a3 = a & 0x88888888;
+
+	u32 b0 = b & 0x11111111;
+	u32 b1 = b & 0x22222222;
+	u32 b2 = b & 0x44444444;
+	u32 b3 = b & 0x88888888;
+
+	u64 c0 = (a0 * (u64)b0) ^ (a1 * (u64)b3) ^
+		 (a2 * (u64)b2) ^ (a3 * (u64)b1);
+	u64 c1 = (a0 * (u64)b1) ^ (a1 * (u64)b0) ^
+		 (a2 * (u64)b3) ^ (a3 * (u64)b2);
+	u64 c2 = (a0 * (u64)b2) ^ (a1 * (u64)b1) ^
+		 (a2 * (u64)b0) ^ (a3 * (u64)b3);
+	u64 c3 = (a0 * (u64)b3) ^ (a1 * (u64)b2) ^
+		 (a2 * (u64)b1) ^ (a3 * (u64)b0);
+
+	/* Add all the intermediate products together. */
+	return (c0 & 0x1111111111111111) ^
+	       (c1 & 0x2222222222222222) ^
+	       (c2 & 0x4444444444444444) ^
+	       (c3 & 0x8888888888888888);
+}
+
+/* Do a 64 x 64 => 128 bit carryless multiplication. */
+static void clmul64(u64 a, u64 b, u64 *out_lo, u64 *out_hi)
+{
+	u32 a_lo = (u32)a;
+	u32 a_hi = a >> 32;
+	u32 b_lo = (u32)b;
+	u32 b_hi = b >> 32;
+
+	/* Karatsuba multiplication */
+	u64 lo = clmul32(a_lo, b_lo);
+	u64 hi = clmul32(a_hi, b_hi);
+	u64 mi = clmul32(a_lo ^ a_hi, b_lo ^ b_hi) ^ lo ^ hi;
+
+	*out_lo = lo ^ (mi << 32);
+	*out_hi = hi ^ (mi >> 32);
+}
+#endif /* !CONFIG_ARCH_SUPPORTS_INT128 */
+
+/* Compute @a = @a * @b * x^-128 in the POLYVAL field. */
+static void __maybe_unused
+polyval_mul_generic(struct polyval_elem *a, const struct polyval_elem *b)
+{
+	u64 c0, c1, c2, c3, mi0, mi1;
+
+	/*
+	 * Carryless-multiply @a by @b using Karatsuba multiplication.  Store
+	 * the 256-bit product in @c0 (low) through @c3 (high).
+	 */
+	clmul64(le64_to_cpu(a->lo), le64_to_cpu(b->lo), &c0, &c1);
+	clmul64(le64_to_cpu(a->hi), le64_to_cpu(b->hi), &c2, &c3);
+	clmul64(le64_to_cpu(a->lo ^ a->hi), le64_to_cpu(b->lo ^ b->hi),
+		&mi0, &mi1);
+	mi0 ^= c0 ^ c2;
+	mi1 ^= c1 ^ c3;
+	c1 ^= mi0;
+	c2 ^= mi1;
+
+	/*
+	 * Cancel out the low 128 bits of the product by adding multiples of
+	 * G(x) = x^128 + x^127 + x^126 + x^121 + 1.  Do this in two steps, each
+	 * of which cancels out 64 bits.  Note that we break G(x) into three
+	 * parts: 1, x^64 * (x^63 + x^62 + x^57), and x^128 * 1.
+	 */
+
+	/*
+	 * First, add G(x) times c0 as follows:
+	 *
+	 * (c0, c1, c2) = (0,
+	 *                 c1 + (c0 * (x^63 + x^62 + x^57) mod x^64),
+	 *		   c2 + c0 + floor((c0 * (x^63 + x^62 + x^57)) / x^64))
+	 */
+	c1 ^= (c0 << 63) ^ (c0 << 62) ^ (c0 << 57);
+	c2 ^= c0 ^ (c0 >> 1) ^ (c0 >> 2) ^ (c0 >> 7);
+
+	/*
+	 * Second, add G(x) times the new c1:
+	 *
+	 * (c1, c2, c3) = (0,
+	 *                 c2 + (c1 * (x^63 + x^62 + x^57) mod x^64),
+	 *		   c3 + c1 + floor((c1 * (x^63 + x^62 + x^57)) / x^64))
+	 */
+	c2 ^= (c1 << 63) ^ (c1 << 62) ^ (c1 << 57);
+	c3 ^= c1 ^ (c1 >> 1) ^ (c1 >> 2) ^ (c1 >> 7);
+
+	/* Return (c2, c3).  This implicitly multiplies by x^-128. */
+	a->lo = cpu_to_le64(c2);
+	a->hi = cpu_to_le64(c3);
+}
+
+static void __maybe_unused
+polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key,
+		       const u8 *data, size_t nblocks)
+{
+	do {
+		acc->lo ^= get_unaligned((__le64 *)data);
+		acc->hi ^= get_unaligned((__le64 *)(data + 8));
+		polyval_mul_generic(acc, key);
+		data += POLYVAL_BLOCK_SIZE;
+	} while (--nblocks);
+}
+
+/* Include the arch-optimized implementation of POLYVAL, if one is available. */
+#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH
+#include "polyval.h" /* $(SRCARCH)/polyval.h */
+void polyval_preparekey(struct polyval_key *key,
+			const u8 raw_key[POLYVAL_BLOCK_SIZE])
+{
+	polyval_preparekey_arch(key, raw_key);
+}
+EXPORT_SYMBOL_GPL(polyval_preparekey);
+#endif /* Else, polyval_preparekey() is an inline function. */
+
+/*
+ * polyval_mul_generic() and polyval_blocks_generic() take the key as a
+ * polyval_elem rather than a polyval_key, so that arch-optimized
+ * implementations with a different key format can use it as a fallback (if they
+ * have H^1 stored somewhere in their struct).  Thus, the following dispatch
+ * code is needed to pass the appropriate key argument.
+ */
+
+static void polyval_mul(struct polyval_ctx *ctx)
+{
+#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH
+	polyval_mul_arch(&ctx->acc, ctx->key);
+#else
+	polyval_mul_generic(&ctx->acc, &ctx->key->h);
+#endif
+}
+
+static void polyval_blocks(struct polyval_ctx *ctx,
+			   const u8 *data, size_t nblocks)
+{
+#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH
+	polyval_blocks_arch(&ctx->acc, ctx->key, data, nblocks);
+#else
+	polyval_blocks_generic(&ctx->acc, &ctx->key->h, data, nblocks);
+#endif
+}
+
+void polyval_update(struct polyval_ctx *ctx, const u8 *data, size_t len)
+{
+	if (unlikely(ctx->partial)) {
+		size_t n = min(len, POLYVAL_BLOCK_SIZE - ctx->partial);
+
+		len -= n;
+		while (n--)
+			ctx->acc.bytes[ctx->partial++] ^= *data++;
+		if (ctx->partial < POLYVAL_BLOCK_SIZE)
+			return;
+		polyval_mul(ctx);
+	}
+	if (len >= POLYVAL_BLOCK_SIZE) {
+		size_t nblocks = len / POLYVAL_BLOCK_SIZE;
+
+		polyval_blocks(ctx, data, nblocks);
+		data += len & ~(POLYVAL_BLOCK_SIZE - 1);
+		len &= POLYVAL_BLOCK_SIZE - 1;
+	}
+	for (size_t i = 0; i < len; i++)
+		ctx->acc.bytes[i] ^= data[i];
+	ctx->partial = len;
+}
+EXPORT_SYMBOL_GPL(polyval_update);
+
+void polyval_final(struct polyval_ctx *ctx, u8 out[POLYVAL_BLOCK_SIZE])
+{
+	if (unlikely(ctx->partial))
+		polyval_mul(ctx);
+	memcpy(out, &ctx->acc, POLYVAL_BLOCK_SIZE);
+	memzero_explicit(ctx, sizeof(*ctx));
+}
+EXPORT_SYMBOL_GPL(polyval_final);
+
+#ifdef polyval_mod_init_arch
+static int __init polyval_mod_init(void)
+{
+	polyval_mod_init_arch();
+	return 0;
+}
+subsys_initcall(polyval_mod_init);
+
+static void __exit polyval_mod_exit(void)
+{
+}
+module_exit(polyval_mod_exit);
+#endif
+
+MODULE_DESCRIPTION("POLYVAL almost-XOR-universal hash function");
+MODULE_LICENSE("GPL");
diff --git a/lib/crypto/powerpc/.gitignore b/lib/crypto/powerpc/.gitignore
new file mode 100644
index 000000000000..598ca7aff6b1
--- /dev/null
+++ b/lib/crypto/powerpc/.gitignore
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0-only
+aesp8-ppc.S
diff --git a/lib/crypto/powerpc/aes-spe-core.S b/lib/crypto/powerpc/aes-spe-core.S
new file mode 100644
index 000000000000..8e00eccc352b
--- /dev/null
+++ b/lib/crypto/powerpc/aes-spe-core.S
@@ -0,0 +1,346 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Fast AES implementation for SPE instruction set (PPC)
+ *
+ * This code makes use of the SPE SIMD instruction set as defined in
+ * http://cache.freescale.com/files/32bit/doc/ref_manual/SPEPIM.pdf
+ * Implementation is based on optimization guide notes from
+ * http://cache.freescale.com/files/32bit/doc/app_note/AN2665.pdf
+ *
+ * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
+ */
+
+#include <asm/ppc_asm.h>
+#include "aes-spe-regs.h"
+
+#define	EAD(in, bpos) \
+	rlwimi		rT0,in,28-((bpos+3)%4)*8,20,27;
+
+#define DAD(in, bpos) \
+	rlwimi		rT1,in,24-((bpos+3)%4)*8,24,31;
+
+#define LWH(out, off) \
+	evlwwsplat	out,off(rT0);	/* load word high		*/
+
+#define LWL(out, off) \
+	lwz		out,off(rT0);	/* load word low		*/
+
+#define LBZ(out, tab, off) \
+	lbz		out,off(tab);	/* load byte			*/
+
+#define LAH(out, in, bpos, off) \
+	EAD(in, bpos)			/* calc addr + load word high	*/ \
+	LWH(out, off)
+
+#define LAL(out, in, bpos, off) \
+	EAD(in, bpos)			/* calc addr + load word low	*/ \
+	LWL(out, off)
+
+#define LAE(out, in, bpos) \
+	EAD(in, bpos)			/* calc addr + load enc byte	*/ \
+	LBZ(out, rT0, 8)
+
+#define LBE(out) \
+	LBZ(out, rT0, 8)		/* load enc byte		*/
+
+#define LAD(out, in, bpos) \
+	DAD(in, bpos)			/* calc addr + load dec byte	*/ \
+	LBZ(out, rT1, 0)
+
+#define LBD(out) \
+	LBZ(out, rT1, 0)
+
+/*
+ * ppc_encrypt_block: The central encryption function for a single 16 bytes
+ * block. It does no stack handling or register saving to support fast calls
+ * via bl/blr. It expects that caller has pre-xored input data with first
+ * 4 words of encryption key into rD0-rD3. Pointer/counter registers must
+ * have also been set up before (rT0, rKP, CTR). Output is stored in rD0-rD3
+ * and rW0-rW3 and caller must execute a final xor on the output registers.
+ * All working registers rD0-rD3 & rW0-rW7 are overwritten during processing.
+ *
+ */
+_GLOBAL(ppc_encrypt_block)
+	LAH(rW4, rD1, 2, 4)
+	LAH(rW6, rD0, 3, 0)
+	LAH(rW3, rD0, 1, 8)
+ppc_encrypt_block_loop:
+	LAH(rW0, rD3, 0, 12)
+	LAL(rW0, rD0, 0, 12)
+	LAH(rW1, rD1, 0, 12)
+	LAH(rW2, rD2, 1, 8)
+	LAL(rW2, rD3, 1, 8)
+	LAL(rW3, rD1, 1, 8)
+	LAL(rW4, rD2, 2, 4)
+	LAL(rW6, rD1, 3, 0)
+	LAH(rW5, rD3, 2, 4)
+	LAL(rW5, rD0, 2, 4)
+	LAH(rW7, rD2, 3, 0)
+	evldw		rD1,16(rKP)
+	EAD(rD3, 3)
+	evxor		rW2,rW2,rW4
+	LWL(rW7, 0)
+	evxor		rW2,rW2,rW6
+	EAD(rD2, 0)
+	evxor		rD1,rD1,rW2
+	LWL(rW1, 12)
+	evxor		rD1,rD1,rW0
+	evldw		rD3,24(rKP)
+	evmergehi	rD0,rD0,rD1
+	EAD(rD1, 2)
+	evxor		rW3,rW3,rW5
+	LWH(rW4, 4)
+	evxor		rW3,rW3,rW7
+	EAD(rD0, 3)
+	evxor		rD3,rD3,rW3
+	LWH(rW6, 0)
+	evxor		rD3,rD3,rW1
+	EAD(rD0, 1)
+	evmergehi	rD2,rD2,rD3
+	LWH(rW3, 8)
+	LAH(rW0, rD3, 0, 12)
+	LAL(rW0, rD0, 0, 12)
+	LAH(rW1, rD1, 0, 12)
+	LAH(rW2, rD2, 1, 8)
+	LAL(rW2, rD3, 1, 8)
+	LAL(rW3, rD1, 1, 8)
+	LAL(rW4, rD2, 2, 4)
+	LAL(rW6, rD1, 3, 0)
+	LAH(rW5, rD3, 2, 4)
+	LAL(rW5, rD0, 2, 4)
+	LAH(rW7, rD2, 3, 0)
+	evldw		rD1,32(rKP)
+	EAD(rD3, 3)
+	evxor		rW2,rW2,rW4
+	LWL(rW7, 0)
+	evxor		rW2,rW2,rW6
+	EAD(rD2, 0)
+	evxor		rD1,rD1,rW2
+	LWL(rW1, 12)
+	evxor		rD1,rD1,rW0
+	evldw		rD3,40(rKP)
+	evmergehi	rD0,rD0,rD1
+	EAD(rD1, 2)
+	evxor		rW3,rW3,rW5
+	LWH(rW4, 4)
+	evxor		rW3,rW3,rW7
+	EAD(rD0, 3)
+	evxor		rD3,rD3,rW3
+	LWH(rW6, 0)
+	evxor		rD3,rD3,rW1
+	EAD(rD0, 1)
+	evmergehi	rD2,rD2,rD3
+	LWH(rW3, 8)
+	addi		rKP,rKP,32
+	bdnz		ppc_encrypt_block_loop
+	LAH(rW0, rD3, 0, 12)
+	LAL(rW0, rD0, 0, 12)
+	LAH(rW1, rD1, 0, 12)
+	LAH(rW2, rD2, 1, 8)
+	LAL(rW2, rD3, 1, 8)
+	LAL(rW3, rD1, 1, 8)
+	LAL(rW4, rD2, 2, 4)
+	LAH(rW5, rD3, 2, 4)
+	LAL(rW6, rD1, 3, 0)
+	LAL(rW5, rD0, 2, 4)
+	LAH(rW7, rD2, 3, 0)
+	evldw		rD1,16(rKP)
+	EAD(rD3, 3)
+	evxor		rW2,rW2,rW4
+	LWL(rW7, 0)
+	evxor		rW2,rW2,rW6
+	EAD(rD2, 0)
+	evxor		rD1,rD1,rW2
+	LWL(rW1, 12)
+	evxor		rD1,rD1,rW0
+	evldw		rD3,24(rKP)
+	evmergehi	rD0,rD0,rD1
+	EAD(rD1, 0)
+	evxor		rW3,rW3,rW5
+	LBE(rW2)
+	evxor		rW3,rW3,rW7
+	EAD(rD0, 1)
+	evxor		rD3,rD3,rW3
+	LBE(rW6)
+	evxor		rD3,rD3,rW1
+	EAD(rD0, 0)
+	evmergehi	rD2,rD2,rD3
+	LBE(rW1)
+	LAE(rW0, rD3, 0)
+	LAE(rW1, rD0, 0)
+	LAE(rW4, rD2, 1)
+	LAE(rW5, rD3, 1)
+	LAE(rW3, rD2, 0)
+	LAE(rW7, rD1, 1)
+	rlwimi		rW0,rW4,8,16,23
+	rlwimi		rW1,rW5,8,16,23
+	LAE(rW4, rD1, 2)
+	LAE(rW5, rD2, 2)
+	rlwimi		rW2,rW6,8,16,23
+	rlwimi		rW3,rW7,8,16,23
+	LAE(rW6, rD3, 2)
+	LAE(rW7, rD0, 2)
+	rlwimi		rW0,rW4,16,8,15
+	rlwimi		rW1,rW5,16,8,15
+	LAE(rW4, rD0, 3)
+	LAE(rW5, rD1, 3)
+	rlwimi		rW2,rW6,16,8,15
+	lwz		rD0,32(rKP)
+	rlwimi		rW3,rW7,16,8,15
+	lwz		rD1,36(rKP)
+	LAE(rW6, rD2, 3)
+	LAE(rW7, rD3, 3)
+	rlwimi		rW0,rW4,24,0,7
+	lwz		rD2,40(rKP)
+	rlwimi		rW1,rW5,24,0,7
+	lwz		rD3,44(rKP)
+	rlwimi		rW2,rW6,24,0,7
+	rlwimi		rW3,rW7,24,0,7
+	blr
+
+/*
+ * ppc_decrypt_block: The central decryption function for a single 16 bytes
+ * block. It does no stack handling or register saving to support fast calls
+ * via bl/blr. It expects that caller has pre-xored input data with first
+ * 4 words of encryption key into rD0-rD3. Pointer/counter registers must
+ * have also been set up before (rT0, rKP, CTR). Output is stored in rD0-rD3
+ * and rW0-rW3 and caller must execute a final xor on the output registers.
+ * All working registers rD0-rD3 & rW0-rW7 are overwritten during processing.
+ *
+ */
+_GLOBAL(ppc_decrypt_block)
+	LAH(rW0, rD1, 0, 12)
+	LAH(rW6, rD0, 3, 0)
+	LAH(rW3, rD0, 1, 8)
+ppc_decrypt_block_loop:
+	LAH(rW1, rD3, 0, 12)
+	LAL(rW0, rD2, 0, 12)
+	LAH(rW2, rD2, 1, 8)
+	LAL(rW2, rD3, 1, 8)
+	LAH(rW4, rD3, 2, 4)
+	LAL(rW4, rD0, 2, 4)
+	LAL(rW6, rD1, 3, 0)
+	LAH(rW5, rD1, 2, 4)
+	LAH(rW7, rD2, 3, 0)
+	LAL(rW7, rD3, 3, 0)
+	LAL(rW3, rD1, 1, 8)
+	evldw		rD1,16(rKP)
+	EAD(rD0, 0)
+	evxor		rW4,rW4,rW6
+	LWL(rW1, 12)
+	evxor		rW0,rW0,rW4
+	EAD(rD2, 2)
+	evxor		rW0,rW0,rW2
+	LWL(rW5, 4)
+	evxor		rD1,rD1,rW0
+	evldw		rD3,24(rKP)
+	evmergehi	rD0,rD0,rD1
+	EAD(rD1, 0)
+	evxor		rW3,rW3,rW7
+	LWH(rW0, 12)
+	evxor		rW3,rW3,rW1
+	EAD(rD0, 3)
+	evxor		rD3,rD3,rW3
+	LWH(rW6, 0)
+	evxor		rD3,rD3,rW5
+	EAD(rD0, 1)
+	evmergehi	rD2,rD2,rD3
+	LWH(rW3, 8)
+	LAH(rW1, rD3, 0, 12)
+	LAL(rW0, rD2, 0, 12)
+	LAH(rW2, rD2, 1, 8)
+	LAL(rW2, rD3, 1, 8)
+	LAH(rW4, rD3, 2, 4)
+	LAL(rW4, rD0, 2, 4)
+	LAL(rW6, rD1, 3, 0)
+	LAH(rW5, rD1, 2, 4)
+	LAH(rW7, rD2, 3, 0)
+	LAL(rW7, rD3, 3, 0)
+	LAL(rW3, rD1, 1, 8)
+	evldw		 rD1,32(rKP)
+	EAD(rD0, 0)
+	evxor		rW4,rW4,rW6
+	LWL(rW1, 12)
+	evxor		rW0,rW0,rW4
+	EAD(rD2, 2)
+	evxor		rW0,rW0,rW2
+	LWL(rW5, 4)
+	evxor		rD1,rD1,rW0
+	evldw		rD3,40(rKP)
+	evmergehi	rD0,rD0,rD1
+	EAD(rD1, 0)
+	evxor		rW3,rW3,rW7
+	LWH(rW0, 12)
+	evxor		rW3,rW3,rW1
+	EAD(rD0, 3)
+	evxor		rD3,rD3,rW3
+	LWH(rW6, 0)
+	evxor		rD3,rD3,rW5
+	EAD(rD0, 1)
+	evmergehi	rD2,rD2,rD3
+	LWH(rW3, 8)
+	addi		rKP,rKP,32
+	bdnz		ppc_decrypt_block_loop
+	LAH(rW1, rD3, 0, 12)
+	LAL(rW0, rD2, 0, 12)
+	LAH(rW2, rD2, 1, 8)
+	LAL(rW2, rD3, 1, 8)
+	LAH(rW4, rD3, 2, 4)
+	LAL(rW4, rD0, 2, 4)
+	LAL(rW6, rD1, 3, 0)
+	LAH(rW5, rD1, 2, 4)
+	LAH(rW7, rD2, 3, 0)
+	LAL(rW7, rD3, 3, 0)
+	LAL(rW3, rD1, 1, 8)
+	evldw		 rD1,16(rKP)
+	EAD(rD0, 0)
+	evxor		rW4,rW4,rW6
+	LWL(rW1, 12)
+	evxor		rW0,rW0,rW4
+	EAD(rD2, 2)
+	evxor		rW0,rW0,rW2
+	LWL(rW5, 4)
+	evxor		rD1,rD1,rW0
+	evldw		rD3,24(rKP)
+	evmergehi	rD0,rD0,rD1
+	DAD(rD1, 0)
+	evxor		rW3,rW3,rW7
+	LBD(rW0)
+	evxor		rW3,rW3,rW1
+	DAD(rD0, 1)
+	evxor		rD3,rD3,rW3
+	LBD(rW6)
+	evxor		rD3,rD3,rW5
+	DAD(rD0, 0)
+	evmergehi	rD2,rD2,rD3
+	LBD(rW3)
+	LAD(rW2, rD3, 0)
+	LAD(rW1, rD2, 0)
+	LAD(rW4, rD2, 1)
+	LAD(rW5, rD3, 1)
+	LAD(rW7, rD1, 1)
+	rlwimi		rW0,rW4,8,16,23
+	rlwimi		rW1,rW5,8,16,23
+	LAD(rW4, rD3, 2)
+	LAD(rW5, rD0, 2)
+	rlwimi		rW2,rW6,8,16,23
+	rlwimi		rW3,rW7,8,16,23
+	LAD(rW6, rD1, 2)
+	LAD(rW7, rD2, 2)
+	rlwimi		rW0,rW4,16,8,15
+	rlwimi		rW1,rW5,16,8,15
+	LAD(rW4, rD0, 3)
+	LAD(rW5, rD1, 3)
+	rlwimi		rW2,rW6,16,8,15
+	lwz		rD0,32(rKP)
+	rlwimi		rW3,rW7,16,8,15
+	lwz		rD1,36(rKP)
+	LAD(rW6, rD2, 3)
+	LAD(rW7, rD3, 3)
+	rlwimi		rW0,rW4,24,0,7
+	lwz		rD2,40(rKP)
+	rlwimi		rW1,rW5,24,0,7
+	lwz		rD3,44(rKP)
+	rlwimi		rW2,rW6,24,0,7
+	rlwimi		rW3,rW7,24,0,7
+	blr
diff --git a/lib/crypto/powerpc/aes-spe-keys.S b/lib/crypto/powerpc/aes-spe-keys.S
new file mode 100644
index 000000000000..2e1bc0d099bf
--- /dev/null
+++ b/lib/crypto/powerpc/aes-spe-keys.S
@@ -0,0 +1,278 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Key handling functions for PPC AES implementation
+ *
+ * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
+ */
+
+#include <asm/ppc_asm.h>
+
+#ifdef __BIG_ENDIAN__
+#define LOAD_KEY(d, s, off) \
+	lwz		d,off(s);
+#else
+#define LOAD_KEY(d, s, off) \
+	li		r0,off; \
+	lwbrx		d,s,r0;
+#endif
+
+#define INITIALIZE_KEY \
+	stwu		r1,-32(r1);	/* create stack frame		*/ \
+	stw		r14,8(r1);	/* save registers		*/ \
+	stw		r15,12(r1);					   \
+	stw		r16,16(r1);
+
+#define FINALIZE_KEY \
+	lwz		r14,8(r1);	/* restore registers		*/ \
+	lwz		r15,12(r1);					   \
+	lwz		r16,16(r1);					   \
+	xor		r5,r5,r5;	/* clear sensitive data		*/ \
+	xor		r6,r6,r6;					   \
+	xor		r7,r7,r7;					   \
+	xor		r8,r8,r8;					   \
+	xor		r9,r9,r9;					   \
+	xor		r10,r10,r10;					   \
+	xor		r11,r11,r11;					   \
+	xor		r12,r12,r12;					   \
+	addi		r1,r1,32;	/* cleanup stack		*/
+
+#define LS_BOX(r, t1, t2) \
+	lis		t2,PPC_AES_4K_ENCTAB@h;				   \
+	ori		t2,t2,PPC_AES_4K_ENCTAB@l;			   \
+	rlwimi		t2,r,4,20,27;					   \
+	lbz		t1,8(t2);					   \
+	rlwimi		r,t1,0,24,31;					   \
+	rlwimi		t2,r,28,20,27;					   \
+	lbz		t1,8(t2);					   \
+	rlwimi		r,t1,8,16,23;					   \
+	rlwimi		t2,r,20,20,27;					   \
+	lbz		t1,8(t2);					   \
+	rlwimi		r,t1,16,8,15;					   \
+	rlwimi		t2,r,12,20,27;					   \
+	lbz		t1,8(t2);					   \
+	rlwimi		r,t1,24,0,7;
+
+#define GF8_MUL(out, in, t1, t2) \
+	lis t1,0x8080;			/* multiplication in GF8	*/ \
+	ori t1,t1,0x8080; 						   \
+	and t1,t1,in; 							   \
+	srwi t1,t1,7; 							   \
+	mulli t1,t1,0x1b; 						   \
+	lis t2,0x7f7f; 							   \
+	ori t2,t2,0x7f7f; 						   \
+	and t2,t2,in; 							   \
+	slwi t2,t2,1; 							   \
+	xor out,t1,t2;
+
+/*
+ * ppc_expand_key_128(u32 *key_enc, const u8 *key)
+ *
+ * Expand 128 bit key into 176 bytes encryption key. It consists of
+ * key itself plus 10 rounds with 16 bytes each
+ *
+ */
+_GLOBAL(ppc_expand_key_128)
+	INITIALIZE_KEY
+	LOAD_KEY(r5,r4,0)
+	LOAD_KEY(r6,r4,4)
+	LOAD_KEY(r7,r4,8)
+	LOAD_KEY(r8,r4,12)
+	stw		r5,0(r3)	/* key[0..3] = input data	*/
+	stw		r6,4(r3)
+	stw		r7,8(r3)
+	stw		r8,12(r3)
+	li		r16,10		/* 10 expansion rounds		*/
+	lis		r0,0x0100	/* RCO(1)			*/
+ppc_expand_128_loop:
+	addi		r3,r3,16
+	mr		r14,r8		/* apply LS_BOX to 4th temp	*/
+	rotlwi		r14,r14,8
+	LS_BOX(r14, r15, r4)
+	xor		r14,r14,r0
+	xor		r5,r5,r14	/* xor next 4 keys		*/
+	xor		r6,r6,r5
+	xor		r7,r7,r6
+	xor		r8,r8,r7
+	stw		r5,0(r3)	/* store next 4 keys		*/
+	stw		r6,4(r3)
+	stw		r7,8(r3)
+	stw		r8,12(r3)
+	GF8_MUL(r0, r0, r4, r14)	/* multiply RCO by 2 in GF	*/
+	subi		r16,r16,1
+	cmpwi		r16,0
+	bt		eq,ppc_expand_128_end
+	b		ppc_expand_128_loop
+ppc_expand_128_end:
+	FINALIZE_KEY
+	blr
+
+/*
+ * ppc_expand_key_192(u32 *key_enc, const u8 *key)
+ *
+ * Expand 192 bit key into 208 bytes encryption key. It consists of key
+ * itself plus 12 rounds with 16 bytes each
+ *
+ */
+_GLOBAL(ppc_expand_key_192)
+	INITIALIZE_KEY
+	LOAD_KEY(r5,r4,0)
+	LOAD_KEY(r6,r4,4)
+	LOAD_KEY(r7,r4,8)
+	LOAD_KEY(r8,r4,12)
+	LOAD_KEY(r9,r4,16)
+	LOAD_KEY(r10,r4,20)
+	stw		r5,0(r3)
+	stw		r6,4(r3)
+	stw		r7,8(r3)
+	stw		r8,12(r3)
+	stw		r9,16(r3)
+	stw		r10,20(r3)
+	li		r16,8		/* 8 expansion rounds		*/
+	lis		r0,0x0100	/* RCO(1)			*/
+ppc_expand_192_loop:
+	addi		r3,r3,24
+	mr		r14,r10		/* apply LS_BOX to 6th temp	*/
+	rotlwi		r14,r14,8
+	LS_BOX(r14, r15, r4)
+	xor		r14,r14,r0
+	xor		r5,r5,r14	/* xor next 6 keys		*/
+	xor		r6,r6,r5
+	xor		r7,r7,r6
+	xor		r8,r8,r7
+	xor		r9,r9,r8
+	xor		r10,r10,r9
+	stw		r5,0(r3)
+	stw		r6,4(r3)
+	stw		r7,8(r3)
+	stw		r8,12(r3)
+	subi		r16,r16,1
+	cmpwi		r16,0		/* last round early kick out	*/
+	bt		eq,ppc_expand_192_end
+	stw		r9,16(r3)
+	stw		r10,20(r3)
+	GF8_MUL(r0, r0, r4, r14)	/* multiply RCO GF8		*/
+	b		ppc_expand_192_loop
+ppc_expand_192_end:
+	FINALIZE_KEY
+	blr
+
+/*
+ * ppc_expand_key_256(u32 *key_enc, const u8 *key)
+ *
+ * Expand 256 bit key into 240 bytes encryption key. It consists of key
+ * itself plus 14 rounds with 16 bytes each
+ *
+ */
+_GLOBAL(ppc_expand_key_256)
+	INITIALIZE_KEY
+	LOAD_KEY(r5,r4,0)
+	LOAD_KEY(r6,r4,4)
+	LOAD_KEY(r7,r4,8)
+	LOAD_KEY(r8,r4,12)
+	LOAD_KEY(r9,r4,16)
+	LOAD_KEY(r10,r4,20)
+	LOAD_KEY(r11,r4,24)
+	LOAD_KEY(r12,r4,28)
+	stw		r5,0(r3)
+	stw		r6,4(r3)
+	stw		r7,8(r3)
+	stw		r8,12(r3)
+	stw		r9,16(r3)
+	stw		r10,20(r3)
+	stw		r11,24(r3)
+	stw		r12,28(r3)
+	li		r16,7		/* 7 expansion rounds		*/
+	lis		r0,0x0100	/* RCO(1)			*/
+ppc_expand_256_loop:
+	addi		r3,r3,32
+	mr		r14,r12		/* apply LS_BOX to 8th temp	*/
+	rotlwi		r14,r14,8
+	LS_BOX(r14, r15, r4)
+	xor		r14,r14,r0
+	xor		r5,r5,r14	/* xor 4 keys			*/
+	xor		r6,r6,r5
+	xor		r7,r7,r6
+	xor		r8,r8,r7
+	mr		r14,r8
+	LS_BOX(r14, r15, r4)		/* apply LS_BOX to 4th temp	*/
+	xor		r9,r9,r14	/* xor 4 keys			*/
+	xor		r10,r10,r9
+	xor		r11,r11,r10
+	xor		r12,r12,r11
+	stw		r5,0(r3)
+	stw		r6,4(r3)
+	stw		r7,8(r3)
+	stw		r8,12(r3)
+	subi		r16,r16,1
+	cmpwi		r16,0		/* last round early kick out	*/
+	bt		eq,ppc_expand_256_end
+	stw		r9,16(r3)
+	stw		r10,20(r3)
+	stw		r11,24(r3)
+	stw		r12,28(r3)
+	GF8_MUL(r0, r0, r4, r14)
+	b		ppc_expand_256_loop
+ppc_expand_256_end:
+	FINALIZE_KEY
+	blr
+
+/*
+ * ppc_generate_decrypt_key: derive decryption key from encryption key
+ * number of bytes to handle are calculated from length of key (16/24/32)
+ *
+ */
+_GLOBAL(ppc_generate_decrypt_key)
+	addi		r6,r5,24
+	slwi		r6,r6,2
+	lwzx		r7,r4,r6	/* first/last 4 words are same	*/
+	stw		r7,0(r3)
+	lwz		r7,0(r4)
+	stwx		r7,r3,r6
+	addi		r6,r6,4
+	lwzx		r7,r4,r6
+	stw		r7,4(r3)
+	lwz		r7,4(r4)
+	stwx		r7,r3,r6
+	addi		r6,r6,4
+	lwzx		r7,r4,r6
+	stw		r7,8(r3)
+	lwz		r7,8(r4)
+	stwx		r7,r3,r6
+	addi		r6,r6,4
+	lwzx		r7,r4,r6
+	stw		r7,12(r3)
+	lwz		r7,12(r4)
+	stwx		r7,r3,r6
+	addi		r3,r3,16
+	add		r4,r4,r6
+	subi		r4,r4,28
+	addi		r5,r5,20
+	srwi		r5,r5,2
+ppc_generate_decrypt_block:
+	li	r6,4
+	mtctr	r6
+ppc_generate_decrypt_word:
+	lwz		r6,0(r4)
+	GF8_MUL(r7, r6, r0, r7)
+	GF8_MUL(r8, r7, r0, r8)
+	GF8_MUL(r9, r8, r0, r9)
+	xor		r10,r9,r6
+	xor		r11,r7,r8
+	xor		r11,r11,r9
+	xor		r12,r7,r10
+	rotrwi		r12,r12,24
+	xor		r11,r11,r12
+	xor		r12,r8,r10
+	rotrwi		r12,r12,16
+	xor		r11,r11,r12
+	rotrwi		r12,r10,8
+	xor		r11,r11,r12
+	stw		r11,0(r3)
+	addi		r3,r3,4
+	addi		r4,r4,4
+	bdnz		ppc_generate_decrypt_word
+	subi		r4,r4,32
+	subi		r5,r5,1
+	cmpwi		r5,0
+	bt		gt,ppc_generate_decrypt_block
+	blr
diff --git a/lib/crypto/powerpc/aes-spe-modes.S b/lib/crypto/powerpc/aes-spe-modes.S
new file mode 100644
index 000000000000..3f92a6a85785
--- /dev/null
+++ b/lib/crypto/powerpc/aes-spe-modes.S
@@ -0,0 +1,625 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * AES modes (ECB/CBC/CTR/XTS) for PPC AES implementation
+ *
+ * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
+ */
+
+#include <asm/ppc_asm.h>
+#include "aes-spe-regs.h"
+
+#ifdef __BIG_ENDIAN__			/* Macros for big endian builds	*/
+
+#define LOAD_DATA(reg, off) \
+	lwz		reg,off(rSP);	/* load with offset		*/
+#define SAVE_DATA(reg, off) \
+	stw		reg,off(rDP);	/* save with offset		*/
+#define NEXT_BLOCK \
+	addi		rSP,rSP,16;	/* increment pointers per bloc	*/ \
+	addi		rDP,rDP,16;
+#define LOAD_IV(reg, off) \
+	lwz		reg,off(rIP);	/* IV loading with offset	*/
+#define SAVE_IV(reg, off) \
+	stw		reg,off(rIP);	/* IV saving with offset	*/
+#define START_IV			/* nothing to reset		*/
+#define CBC_DEC 16			/* CBC decrement per block	*/
+#define CTR_DEC 1			/* CTR decrement one byte	*/
+
+#else					/* Macros for little endian	*/
+
+#define LOAD_DATA(reg, off) \
+	lwbrx		reg,0,rSP;	/* load reversed		*/ \
+	addi		rSP,rSP,4;	/* and increment pointer	*/
+#define SAVE_DATA(reg, off) \
+	stwbrx		reg,0,rDP;	/* save reversed		*/ \
+	addi		rDP,rDP,4;	/* and increment pointer	*/
+#define NEXT_BLOCK			/* nothing todo			*/
+#define LOAD_IV(reg, off) \
+	lwbrx		reg,0,rIP;	/* load reversed		*/ \
+	addi		rIP,rIP,4;	/* and increment pointer	*/
+#define SAVE_IV(reg, off) \
+	stwbrx		reg,0,rIP;	/* load reversed		*/ \
+	addi		rIP,rIP,4;	/* and increment pointer	*/
+#define START_IV \
+	subi		rIP,rIP,16;	/* must reset pointer		*/
+#define CBC_DEC 32			/* 2 blocks because of incs	*/
+#define CTR_DEC 17			/* 1 block because of incs	*/
+
+#endif
+
+#define SAVE_0_REGS
+#define LOAD_0_REGS
+
+#define SAVE_4_REGS \
+	stw		rI0,96(r1);	/* save 32 bit registers	*/ \
+	stw		rI1,100(r1);					   \
+	stw		rI2,104(r1);					   \
+	stw		rI3,108(r1);
+
+#define LOAD_4_REGS \
+	lwz		rI0,96(r1);	/* restore 32 bit registers	*/ \
+	lwz		rI1,100(r1);					   \
+	lwz		rI2,104(r1);					   \
+	lwz		rI3,108(r1);
+
+#define SAVE_8_REGS \
+	SAVE_4_REGS							   \
+	stw		rG0,112(r1);	/* save 32 bit registers	*/ \
+	stw		rG1,116(r1);					   \
+	stw		rG2,120(r1);					   \
+	stw		rG3,124(r1);
+
+#define LOAD_8_REGS \
+	LOAD_4_REGS							   \
+	lwz		rG0,112(r1);	/* restore 32 bit registers	*/ \
+	lwz		rG1,116(r1);					   \
+	lwz		rG2,120(r1);					   \
+	lwz		rG3,124(r1);
+
+#define INITIALIZE_CRYPT(tab,nr32bitregs) \
+	mflr		r0;						   \
+	stwu		r1,-160(r1);	/* create stack frame		*/ \
+	lis		rT0,tab@h;	/* en-/decryption table pointer	*/ \
+	stw		r0,8(r1);	/* save link register		*/ \
+	ori		rT0,rT0,tab@l;					   \
+	evstdw		r14,16(r1);					   \
+	mr		rKS,rKP;					   \
+	evstdw		r15,24(r1);	/* We must save non volatile	*/ \
+	evstdw		r16,32(r1);	/* registers. Take the chance	*/ \
+	evstdw		r17,40(r1);	/* and save the SPE part too	*/ \
+	evstdw		r18,48(r1);					   \
+	evstdw		r19,56(r1);					   \
+	evstdw		r20,64(r1);					   \
+	evstdw		r21,72(r1);					   \
+	evstdw		r22,80(r1);					   \
+	evstdw		r23,88(r1);					   \
+	SAVE_##nr32bitregs##_REGS
+
+#define FINALIZE_CRYPT(nr32bitregs) \
+	lwz		r0,8(r1);					   \
+	evldw		r14,16(r1);	/* restore SPE registers	*/ \
+	evldw		r15,24(r1);					   \
+	evldw		r16,32(r1);					   \
+	evldw		r17,40(r1);					   \
+	evldw		r18,48(r1);					   \
+	evldw		r19,56(r1);					   \
+	evldw		r20,64(r1);					   \
+	evldw		r21,72(r1);					   \
+	evldw		r22,80(r1);					   \
+	evldw		r23,88(r1);					   \
+	LOAD_##nr32bitregs##_REGS					   \
+	mtlr		r0;		/* restore link register	*/ \
+	xor		r0,r0,r0;					   \
+	stw		r0,16(r1);	/* delete sensitive data	*/ \
+	stw		r0,24(r1);	/* that we might have pushed	*/ \
+	stw		r0,32(r1);	/* from other context that runs	*/ \
+	stw		r0,40(r1);	/* the same code		*/ \
+	stw		r0,48(r1);					   \
+	stw		r0,56(r1);					   \
+	stw		r0,64(r1);					   \
+	stw		r0,72(r1);					   \
+	stw		r0,80(r1);					   \
+	stw		r0,88(r1);					   \
+	addi		r1,r1,160;	/* cleanup stack frame		*/
+
+#define ENDIAN_SWAP(t0, t1, s0, s1) \
+	rotrwi		t0,s0,8;	/* swap endianness for 2 GPRs	*/ \
+	rotrwi		t1,s1,8;					   \
+	rlwimi		t0,s0,8,8,15;					   \
+	rlwimi		t1,s1,8,8,15;					   \
+	rlwimi		t0,s0,8,24,31;					   \
+	rlwimi		t1,s1,8,24,31;
+
+#define GF128_MUL(d0, d1, d2, d3, t0) \
+	li		t0,0x87;	/* multiplication in GF128	*/ \
+	cmpwi		d3,-1;						   \
+	iselgt		t0,0,t0;					   \
+	rlwimi		d3,d2,0,0,0;	/* propagate "carry" bits	*/ \
+	rotlwi		d3,d3,1;					   \
+	rlwimi		d2,d1,0,0,0;					   \
+	rotlwi		d2,d2,1;					   \
+	rlwimi		d1,d0,0,0,0;					   \
+	slwi		d0,d0,1;	/* shift left 128 bit		*/ \
+	rotlwi		d1,d1,1;					   \
+	xor		d0,d0,t0;
+
+#define START_KEY(d0, d1, d2, d3) \
+	lwz		rW0,0(rKP);					   \
+	mtctr		rRR;						   \
+	lwz		rW1,4(rKP);					   \
+	lwz		rW2,8(rKP);					   \
+	lwz		rW3,12(rKP);					   \
+	xor		rD0,d0,rW0;					   \
+	xor		rD1,d1,rW1;					   \
+	xor		rD2,d2,rW2;					   \
+	xor		rD3,d3,rW3;
+
+/*
+ * ppc_encrypt_aes(u8 *out, const u8 *in, u32 *key_enc,
+ *		   u32 rounds)
+ *
+ * called from glue layer to encrypt a single 16 byte block
+ * round values are AES128 = 4, AES192 = 5, AES256 = 6
+ *
+ */
+_GLOBAL(ppc_encrypt_aes)
+	INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 0)
+	LOAD_DATA(rD0, 0)
+	LOAD_DATA(rD1, 4)
+	LOAD_DATA(rD2, 8)
+	LOAD_DATA(rD3, 12)
+	START_KEY(rD0, rD1, rD2, rD3)
+	bl		ppc_encrypt_block
+	xor		rD0,rD0,rW0
+	SAVE_DATA(rD0, 0)
+	xor		rD1,rD1,rW1
+	SAVE_DATA(rD1, 4)
+	xor		rD2,rD2,rW2
+	SAVE_DATA(rD2, 8)
+	xor		rD3,rD3,rW3
+	SAVE_DATA(rD3, 12)
+	FINALIZE_CRYPT(0)
+	blr
+
+/*
+ * ppc_decrypt_aes(u8 *out, const u8 *in, u32 *key_dec,
+ *		   u32 rounds)
+ *
+ * called from glue layer to decrypt a single 16 byte block
+ * round values are AES128 = 4, AES192 = 5, AES256 = 6
+ *
+ */
+_GLOBAL(ppc_decrypt_aes)
+	INITIALIZE_CRYPT(PPC_AES_4K_DECTAB,0)
+	LOAD_DATA(rD0, 0)
+	addi		rT1,rT0,4096
+	LOAD_DATA(rD1, 4)
+	LOAD_DATA(rD2, 8)
+	LOAD_DATA(rD3, 12)
+	START_KEY(rD0, rD1, rD2, rD3)
+	bl		ppc_decrypt_block
+	xor		rD0,rD0,rW0
+	SAVE_DATA(rD0, 0)
+	xor		rD1,rD1,rW1
+	SAVE_DATA(rD1, 4)
+	xor		rD2,rD2,rW2
+	SAVE_DATA(rD2, 8)
+	xor		rD3,rD3,rW3
+	SAVE_DATA(rD3, 12)
+	FINALIZE_CRYPT(0)
+	blr
+
+/*
+ * ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc,
+ *		   u32 rounds, u32 bytes);
+ *
+ * called from glue layer to encrypt multiple blocks via ECB
+ * Bytes must be larger or equal 16 and only whole blocks are
+ * processed. round values are AES128 = 4, AES192 = 5 and
+ * AES256 = 6
+ *
+ */
+_GLOBAL(ppc_encrypt_ecb)
+	INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 0)
+ppc_encrypt_ecb_loop:
+	LOAD_DATA(rD0, 0)
+	mr		rKP,rKS
+	LOAD_DATA(rD1, 4)
+	subi		rLN,rLN,16
+	LOAD_DATA(rD2, 8)
+	cmpwi		rLN,15
+	LOAD_DATA(rD3, 12)
+	START_KEY(rD0, rD1, rD2, rD3)
+	bl		ppc_encrypt_block
+	xor		rD0,rD0,rW0
+	SAVE_DATA(rD0, 0)
+	xor		rD1,rD1,rW1
+	SAVE_DATA(rD1, 4)
+	xor		rD2,rD2,rW2
+	SAVE_DATA(rD2, 8)
+	xor		rD3,rD3,rW3
+	SAVE_DATA(rD3, 12)
+	NEXT_BLOCK
+	bt		gt,ppc_encrypt_ecb_loop
+	FINALIZE_CRYPT(0)
+	blr
+
+/*
+ * ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec,
+ *		   u32 rounds, u32 bytes);
+ *
+ * called from glue layer to decrypt multiple blocks via ECB
+ * Bytes must be larger or equal 16 and only whole blocks are
+ * processed. round values are AES128 = 4, AES192 = 5 and
+ * AES256 = 6
+ *
+ */
+_GLOBAL(ppc_decrypt_ecb)
+	INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 0)
+	addi		rT1,rT0,4096
+ppc_decrypt_ecb_loop:
+	LOAD_DATA(rD0, 0)
+	mr		rKP,rKS
+	LOAD_DATA(rD1, 4)
+	subi		rLN,rLN,16
+	LOAD_DATA(rD2, 8)
+	cmpwi		rLN,15
+	LOAD_DATA(rD3, 12)
+	START_KEY(rD0, rD1, rD2, rD3)
+	bl		ppc_decrypt_block
+	xor		rD0,rD0,rW0
+	SAVE_DATA(rD0, 0)
+	xor		rD1,rD1,rW1
+	SAVE_DATA(rD1, 4)
+	xor		rD2,rD2,rW2
+	SAVE_DATA(rD2, 8)
+	xor		rD3,rD3,rW3
+	SAVE_DATA(rD3, 12)
+	NEXT_BLOCK
+	bt		gt,ppc_decrypt_ecb_loop
+	FINALIZE_CRYPT(0)
+	blr
+
+/*
+ * ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc,
+ *		   32 rounds, u32 bytes, u8 *iv);
+ *
+ * called from glue layer to encrypt multiple blocks via CBC
+ * Bytes must be larger or equal 16 and only whole blocks are
+ * processed. round values are AES128 = 4, AES192 = 5 and
+ * AES256 = 6
+ *
+ */
+_GLOBAL(ppc_encrypt_cbc)
+	INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 4)
+	LOAD_IV(rI0, 0)
+	LOAD_IV(rI1, 4)
+	LOAD_IV(rI2, 8)
+	LOAD_IV(rI3, 12)
+ppc_encrypt_cbc_loop:
+	LOAD_DATA(rD0, 0)
+	mr		rKP,rKS
+	LOAD_DATA(rD1, 4)
+	subi		rLN,rLN,16
+	LOAD_DATA(rD2, 8)
+	cmpwi		rLN,15
+	LOAD_DATA(rD3, 12)
+	xor		rD0,rD0,rI0
+	xor		rD1,rD1,rI1
+	xor		rD2,rD2,rI2
+	xor		rD3,rD3,rI3
+	START_KEY(rD0, rD1, rD2, rD3)
+	bl		ppc_encrypt_block
+	xor		rI0,rD0,rW0
+	SAVE_DATA(rI0, 0)
+	xor		rI1,rD1,rW1
+	SAVE_DATA(rI1, 4)
+	xor		rI2,rD2,rW2
+	SAVE_DATA(rI2, 8)
+	xor		rI3,rD3,rW3
+	SAVE_DATA(rI3, 12)
+	NEXT_BLOCK
+	bt		gt,ppc_encrypt_cbc_loop
+	START_IV
+	SAVE_IV(rI0, 0)
+	SAVE_IV(rI1, 4)
+	SAVE_IV(rI2, 8)
+	SAVE_IV(rI3, 12)
+	FINALIZE_CRYPT(4)
+	blr
+
+/*
+ * ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec,
+ *		   u32 rounds, u32 bytes, u8 *iv);
+ *
+ * called from glue layer to decrypt multiple blocks via CBC
+ * round values are AES128 = 4, AES192 = 5, AES256 = 6
+ *
+ */
+_GLOBAL(ppc_decrypt_cbc)
+	INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 4)
+	li		rT1,15
+	LOAD_IV(rI0, 0)
+	andc		rLN,rLN,rT1
+	LOAD_IV(rI1, 4)
+	subi		rLN,rLN,16
+	LOAD_IV(rI2, 8)
+	add		rSP,rSP,rLN	/* reverse processing		*/
+	LOAD_IV(rI3, 12)
+	add		rDP,rDP,rLN
+	LOAD_DATA(rD0, 0)
+	addi		rT1,rT0,4096
+	LOAD_DATA(rD1, 4)
+	LOAD_DATA(rD2, 8)
+	LOAD_DATA(rD3, 12)
+	START_IV
+	SAVE_IV(rD0, 0)
+	SAVE_IV(rD1, 4)
+	SAVE_IV(rD2, 8)
+	cmpwi		rLN,16
+	SAVE_IV(rD3, 12)
+	bt		lt,ppc_decrypt_cbc_end
+ppc_decrypt_cbc_loop:
+	mr		rKP,rKS
+	START_KEY(rD0, rD1, rD2, rD3)
+	bl		ppc_decrypt_block
+	subi		rLN,rLN,16
+	subi		rSP,rSP,CBC_DEC
+	xor		rW0,rD0,rW0
+	LOAD_DATA(rD0, 0)
+	xor		rW1,rD1,rW1
+	LOAD_DATA(rD1, 4)
+	xor		rW2,rD2,rW2
+	LOAD_DATA(rD2, 8)
+	xor		rW3,rD3,rW3
+	LOAD_DATA(rD3, 12)
+	xor		rW0,rW0,rD0
+	SAVE_DATA(rW0, 0)
+	xor		rW1,rW1,rD1
+	SAVE_DATA(rW1, 4)
+	xor		rW2,rW2,rD2
+	SAVE_DATA(rW2, 8)
+	xor		rW3,rW3,rD3
+	SAVE_DATA(rW3, 12)
+	cmpwi		rLN,15
+	subi		rDP,rDP,CBC_DEC
+	bt		gt,ppc_decrypt_cbc_loop
+ppc_decrypt_cbc_end:
+	mr		rKP,rKS
+	START_KEY(rD0, rD1, rD2, rD3)
+	bl		ppc_decrypt_block
+	xor		rW0,rW0,rD0
+	xor		rW1,rW1,rD1
+	xor		rW2,rW2,rD2
+	xor		rW3,rW3,rD3
+	xor		rW0,rW0,rI0	/* decrypt with initial IV	*/
+	SAVE_DATA(rW0, 0)
+	xor		rW1,rW1,rI1
+	SAVE_DATA(rW1, 4)
+	xor		rW2,rW2,rI2
+	SAVE_DATA(rW2, 8)
+	xor		rW3,rW3,rI3
+	SAVE_DATA(rW3, 12)
+	FINALIZE_CRYPT(4)
+	blr
+
+/*
+ * ppc_crypt_ctr(u8 *out, const u8 *in, u32 *key_enc,
+ *		 u32 rounds, u32 bytes, u8 *iv);
+ *
+ * called from glue layer to encrypt/decrypt multiple blocks
+ * via CTR. Number of bytes does not need to be a multiple of
+ * 16. Round values are AES128 = 4, AES192 = 5, AES256 = 6
+ *
+ */
+_GLOBAL(ppc_crypt_ctr)
+	INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 4)
+	LOAD_IV(rI0, 0)
+	LOAD_IV(rI1, 4)
+	LOAD_IV(rI2, 8)
+	cmpwi		rLN,16
+	LOAD_IV(rI3, 12)
+	START_IV
+	bt		lt,ppc_crypt_ctr_partial
+ppc_crypt_ctr_loop:
+	mr		rKP,rKS
+	START_KEY(rI0, rI1, rI2, rI3)
+	bl		ppc_encrypt_block
+	xor		rW0,rD0,rW0
+	xor		rW1,rD1,rW1
+	xor		rW2,rD2,rW2
+	xor		rW3,rD3,rW3
+	LOAD_DATA(rD0, 0)
+	subi		rLN,rLN,16
+	LOAD_DATA(rD1, 4)
+	LOAD_DATA(rD2, 8)
+	LOAD_DATA(rD3, 12)
+	xor		rD0,rD0,rW0
+	SAVE_DATA(rD0, 0)
+	xor		rD1,rD1,rW1
+	SAVE_DATA(rD1, 4)
+	xor		rD2,rD2,rW2
+	SAVE_DATA(rD2, 8)
+	xor		rD3,rD3,rW3
+	SAVE_DATA(rD3, 12)
+	addic		rI3,rI3,1	/* increase counter			*/
+	addze		rI2,rI2
+	addze		rI1,rI1
+	addze		rI0,rI0
+	NEXT_BLOCK
+	cmpwi		rLN,15
+	bt		gt,ppc_crypt_ctr_loop
+ppc_crypt_ctr_partial:
+	cmpwi		rLN,0
+	bt		eq,ppc_crypt_ctr_end
+	mr		rKP,rKS
+	START_KEY(rI0, rI1, rI2, rI3)
+	bl		ppc_encrypt_block
+	xor		rW0,rD0,rW0
+	SAVE_IV(rW0, 0)
+	xor		rW1,rD1,rW1
+	SAVE_IV(rW1, 4)
+	xor		rW2,rD2,rW2
+	SAVE_IV(rW2, 8)
+	xor		rW3,rD3,rW3
+	SAVE_IV(rW3, 12)
+	mtctr		rLN
+	subi		rIP,rIP,CTR_DEC
+	subi		rSP,rSP,1
+	subi		rDP,rDP,1
+ppc_crypt_ctr_xorbyte:
+	lbzu		rW4,1(rIP)	/* bytewise xor for partial block	*/
+	lbzu		rW5,1(rSP)
+	xor		rW4,rW4,rW5
+	stbu		rW4,1(rDP)
+	bdnz		ppc_crypt_ctr_xorbyte
+	subf		rIP,rLN,rIP
+	addi		rIP,rIP,1
+	addic		rI3,rI3,1
+	addze		rI2,rI2
+	addze		rI1,rI1
+	addze		rI0,rI0
+ppc_crypt_ctr_end:
+	SAVE_IV(rI0, 0)
+	SAVE_IV(rI1, 4)
+	SAVE_IV(rI2, 8)
+	SAVE_IV(rI3, 12)
+	FINALIZE_CRYPT(4)
+	blr
+
+/*
+ * ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc,
+ *		   u32 rounds, u32 bytes, u8 *iv, u32 *key_twk);
+ *
+ * called from glue layer to encrypt multiple blocks via XTS
+ * If key_twk is given, the initial IV encryption will be
+ * processed too. Round values are AES128 = 4, AES192 = 5,
+ * AES256 = 6
+ *
+ */
+_GLOBAL(ppc_encrypt_xts)
+	INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 8)
+	LOAD_IV(rI0, 0)
+	LOAD_IV(rI1, 4)
+	LOAD_IV(rI2, 8)
+	cmpwi		rKT,0
+	LOAD_IV(rI3, 12)
+	bt		eq,ppc_encrypt_xts_notweak
+	mr		rKP,rKT
+	START_KEY(rI0, rI1, rI2, rI3)
+	bl		ppc_encrypt_block
+	xor		rI0,rD0,rW0
+	xor		rI1,rD1,rW1
+	xor		rI2,rD2,rW2
+	xor		rI3,rD3,rW3
+ppc_encrypt_xts_notweak:
+	ENDIAN_SWAP(rG0, rG1, rI0, rI1)
+	ENDIAN_SWAP(rG2, rG3, rI2, rI3)
+ppc_encrypt_xts_loop:
+	LOAD_DATA(rD0, 0)
+	mr		rKP,rKS
+	LOAD_DATA(rD1, 4)
+	subi		rLN,rLN,16
+	LOAD_DATA(rD2, 8)
+	LOAD_DATA(rD3, 12)
+	xor		rD0,rD0,rI0
+	xor		rD1,rD1,rI1
+	xor		rD2,rD2,rI2
+	xor		rD3,rD3,rI3
+	START_KEY(rD0, rD1, rD2, rD3)
+	bl		ppc_encrypt_block
+	xor		rD0,rD0,rW0
+	xor		rD1,rD1,rW1
+	xor		rD2,rD2,rW2
+	xor		rD3,rD3,rW3
+	xor		rD0,rD0,rI0
+	SAVE_DATA(rD0, 0)
+	xor		rD1,rD1,rI1
+	SAVE_DATA(rD1, 4)
+	xor		rD2,rD2,rI2
+	SAVE_DATA(rD2, 8)
+	xor		rD3,rD3,rI3
+	SAVE_DATA(rD3, 12)
+	GF128_MUL(rG0, rG1, rG2, rG3, rW0)
+	ENDIAN_SWAP(rI0, rI1, rG0, rG1)
+	ENDIAN_SWAP(rI2, rI3, rG2, rG3)
+	cmpwi		rLN,0
+	NEXT_BLOCK
+	bt		gt,ppc_encrypt_xts_loop
+	START_IV
+	SAVE_IV(rI0, 0)
+	SAVE_IV(rI1, 4)
+	SAVE_IV(rI2, 8)
+	SAVE_IV(rI3, 12)
+	FINALIZE_CRYPT(8)
+	blr
+
+/*
+ * ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec,
+ *		   u32 rounds, u32 blocks, u8 *iv, u32 *key_twk);
+ *
+ * called from glue layer to decrypt multiple blocks via XTS
+ * If key_twk is given, the initial IV encryption will be
+ * processed too. Round values are AES128 = 4, AES192 = 5,
+ * AES256 = 6
+ *
+ */
+_GLOBAL(ppc_decrypt_xts)
+	INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 8)
+	LOAD_IV(rI0, 0)
+	addi		rT1,rT0,4096
+	LOAD_IV(rI1, 4)
+	LOAD_IV(rI2, 8)
+	cmpwi		rKT,0
+	LOAD_IV(rI3, 12)
+	bt		eq,ppc_decrypt_xts_notweak
+	subi		rT0,rT0,4096
+	mr		rKP,rKT
+	START_KEY(rI0, rI1, rI2, rI3)
+	bl		ppc_encrypt_block
+	xor		rI0,rD0,rW0
+	xor		rI1,rD1,rW1
+	xor		rI2,rD2,rW2
+	xor		rI3,rD3,rW3
+	addi		rT0,rT0,4096
+ppc_decrypt_xts_notweak:
+	ENDIAN_SWAP(rG0, rG1, rI0, rI1)
+	ENDIAN_SWAP(rG2, rG3, rI2, rI3)
+ppc_decrypt_xts_loop:
+	LOAD_DATA(rD0, 0)
+	mr		rKP,rKS
+	LOAD_DATA(rD1, 4)
+	subi		rLN,rLN,16
+	LOAD_DATA(rD2, 8)
+	LOAD_DATA(rD3, 12)
+	xor		rD0,rD0,rI0
+	xor		rD1,rD1,rI1
+	xor		rD2,rD2,rI2
+	xor		rD3,rD3,rI3
+	START_KEY(rD0, rD1, rD2, rD3)
+	bl		ppc_decrypt_block
+	xor		rD0,rD0,rW0
+	xor		rD1,rD1,rW1
+	xor		rD2,rD2,rW2
+	xor		rD3,rD3,rW3
+	xor		rD0,rD0,rI0
+	SAVE_DATA(rD0, 0)
+	xor		rD1,rD1,rI1
+	SAVE_DATA(rD1, 4)
+	xor		rD2,rD2,rI2
+	SAVE_DATA(rD2, 8)
+	xor		rD3,rD3,rI3
+	SAVE_DATA(rD3, 12)
+	GF128_MUL(rG0, rG1, rG2, rG3, rW0)
+	ENDIAN_SWAP(rI0, rI1, rG0, rG1)
+	ENDIAN_SWAP(rI2, rI3, rG2, rG3)
+	cmpwi		rLN,0
+	NEXT_BLOCK
+	bt		gt,ppc_decrypt_xts_loop
+	START_IV
+	SAVE_IV(rI0, 0)
+	SAVE_IV(rI1, 4)
+	SAVE_IV(rI2, 8)
+	SAVE_IV(rI3, 12)
+	FINALIZE_CRYPT(8)
+	blr
diff --git a/lib/crypto/powerpc/aes-spe-regs.h b/lib/crypto/powerpc/aes-spe-regs.h
new file mode 100644
index 000000000000..2eb4c9b94152
--- /dev/null
+++ b/lib/crypto/powerpc/aes-spe-regs.h
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Common registers for PPC AES implementation
+ *
+ * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
+ */
+
+#define rKS r0	/* copy of en-/decryption key pointer			*/
+#define rDP r3	/* destination pointer					*/
+#define rSP r4	/* source pointer					*/
+#define rKP r5	/* pointer to en-/decryption key pointer		*/
+#define rRR r6	/* en-/decryption rounds				*/
+#define rLN r7	/* length of data to be processed			*/
+#define rIP r8	/* potiner to IV (CBC/CTR/XTS modes)			*/
+#define rKT r9	/* pointer to tweak key (XTS mode)			*/
+#define rT0 r11	/* pointers to en-/decryption tables			*/
+#define rT1 r10
+#define rD0 r9	/* data 						*/
+#define rD1 r14
+#define rD2 r12
+#define rD3 r15
+#define rW0 r16	/* working registers					*/
+#define rW1 r17
+#define rW2 r18
+#define rW3 r19
+#define rW4 r20
+#define rW5 r21
+#define rW6 r22
+#define rW7 r23
+#define rI0 r24	/* IV							*/
+#define rI1 r25
+#define rI2 r26
+#define rI3 r27
+#define rG0 r28	/* endian reversed tweak (XTS mode)			*/
+#define rG1 r29
+#define rG2 r30
+#define rG3 r31
diff --git a/lib/crypto/powerpc/aes-tab-4k.S b/lib/crypto/powerpc/aes-tab-4k.S
new file mode 100644
index 000000000000..ceb604bc6f72
--- /dev/null
+++ b/lib/crypto/powerpc/aes-tab-4k.S
@@ -0,0 +1,326 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * 4K AES tables for PPC AES implementation
+ *
+ * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
+ */
+
+/*
+ * These big endian AES encryption/decryption tables have been taken from
+ * crypto/aes_generic.c and are designed to be simply accessed by a combination
+ * of rlwimi/lwz instructions with a minimum of table registers (usually only
+ * one required). Thus they are aligned to 4K. The locality of rotated values
+ * is derived from the reduced offsets that are available in the SPE load
+ * instructions. E.g. evldw, evlwwsplat, ...
+ *
+ * For the safety-conscious it has to be noted that they might be vulnerable
+ * to cache timing attacks because of their size. Nevertheless in contrast to
+ * the generic tables they have been reduced from 16KB to 8KB + 256 bytes.
+ * This is a quite good tradeoff for low power devices (e.g. routers) without
+ * dedicated encryption hardware where we usually have no multiuser
+ * environment.
+ *
+ */
+
+#define R(a, b, c, d) \
+	0x##a##b##c##d, 0x##d##a##b##c, 0x##c##d##a##b, 0x##b##c##d##a
+
+.data
+.align 12
+.globl PPC_AES_4K_ENCTAB
+PPC_AES_4K_ENCTAB:
+/* encryption table, same as crypto_ft_tab in crypto/aes-generic.c */
+	.long R(c6, 63, 63, a5), R(f8, 7c, 7c, 84)
+	.long R(ee, 77, 77, 99), R(f6, 7b, 7b, 8d)
+	.long R(ff, f2, f2, 0d), R(d6, 6b, 6b, bd)
+	.long R(de, 6f, 6f, b1), R(91, c5, c5, 54)
+	.long R(60, 30, 30, 50), R(02, 01, 01, 03)
+	.long R(ce, 67, 67, a9), R(56, 2b, 2b, 7d)
+	.long R(e7, fe, fe, 19), R(b5, d7, d7, 62)
+	.long R(4d, ab, ab, e6), R(ec, 76, 76, 9a)
+	.long R(8f, ca, ca, 45), R(1f, 82, 82, 9d)
+	.long R(89, c9, c9, 40), R(fa, 7d, 7d, 87)
+	.long R(ef, fa, fa, 15), R(b2, 59, 59, eb)
+	.long R(8e, 47, 47, c9), R(fb, f0, f0, 0b)
+	.long R(41, ad, ad, ec), R(b3, d4, d4, 67)
+	.long R(5f, a2, a2, fd), R(45, af, af, ea)
+	.long R(23, 9c, 9c, bf), R(53, a4, a4, f7)
+	.long R(e4, 72, 72, 96), R(9b, c0, c0, 5b)
+	.long R(75, b7, b7, c2), R(e1, fd, fd, 1c)
+	.long R(3d, 93, 93, ae), R(4c, 26, 26, 6a)
+	.long R(6c, 36, 36, 5a), R(7e, 3f, 3f, 41)
+	.long R(f5, f7, f7, 02), R(83, cc, cc, 4f)
+	.long R(68, 34, 34, 5c), R(51, a5, a5, f4)
+	.long R(d1, e5, e5, 34), R(f9, f1, f1, 08)
+	.long R(e2, 71, 71, 93), R(ab, d8, d8, 73)
+	.long R(62, 31, 31, 53), R(2a, 15, 15, 3f)
+	.long R(08, 04, 04, 0c), R(95, c7, c7, 52)
+	.long R(46, 23, 23, 65), R(9d, c3, c3, 5e)
+	.long R(30, 18, 18, 28), R(37, 96, 96, a1)
+	.long R(0a, 05, 05, 0f), R(2f, 9a, 9a, b5)
+	.long R(0e, 07, 07, 09), R(24, 12, 12, 36)
+	.long R(1b, 80, 80, 9b), R(df, e2, e2, 3d)
+	.long R(cd, eb, eb, 26), R(4e, 27, 27, 69)
+	.long R(7f, b2, b2, cd), R(ea, 75, 75, 9f)
+	.long R(12, 09, 09, 1b), R(1d, 83, 83, 9e)
+	.long R(58, 2c, 2c, 74), R(34, 1a, 1a, 2e)
+	.long R(36, 1b, 1b, 2d), R(dc, 6e, 6e, b2)
+	.long R(b4, 5a, 5a, ee), R(5b, a0, a0, fb)
+	.long R(a4, 52, 52, f6), R(76, 3b, 3b, 4d)
+	.long R(b7, d6, d6, 61), R(7d, b3, b3, ce)
+	.long R(52, 29, 29, 7b), R(dd, e3, e3, 3e)
+	.long R(5e, 2f, 2f, 71), R(13, 84, 84, 97)
+	.long R(a6, 53, 53, f5), R(b9, d1, d1, 68)
+	.long R(00, 00, 00, 00), R(c1, ed, ed, 2c)
+	.long R(40, 20, 20, 60), R(e3, fc, fc, 1f)
+	.long R(79, b1, b1, c8), R(b6, 5b, 5b, ed)
+	.long R(d4, 6a, 6a, be), R(8d, cb, cb, 46)
+	.long R(67, be, be, d9), R(72, 39, 39, 4b)
+	.long R(94, 4a, 4a, de), R(98, 4c, 4c, d4)
+	.long R(b0, 58, 58, e8), R(85, cf, cf, 4a)
+	.long R(bb, d0, d0, 6b), R(c5, ef, ef, 2a)
+	.long R(4f, aa, aa, e5), R(ed, fb, fb, 16)
+	.long R(86, 43, 43, c5), R(9a, 4d, 4d, d7)
+	.long R(66, 33, 33, 55), R(11, 85, 85, 94)
+	.long R(8a, 45, 45, cf), R(e9, f9, f9, 10)
+	.long R(04, 02, 02, 06), R(fe, 7f, 7f, 81)
+	.long R(a0, 50, 50, f0), R(78, 3c, 3c, 44)
+	.long R(25, 9f, 9f, ba), R(4b, a8, a8, e3)
+	.long R(a2, 51, 51, f3), R(5d, a3, a3, fe)
+	.long R(80, 40, 40, c0), R(05, 8f, 8f, 8a)
+	.long R(3f, 92, 92, ad), R(21, 9d, 9d, bc)
+	.long R(70, 38, 38, 48), R(f1, f5, f5, 04)
+	.long R(63, bc, bc, df), R(77, b6, b6, c1)
+	.long R(af, da, da, 75), R(42, 21, 21, 63)
+	.long R(20, 10, 10, 30), R(e5, ff, ff, 1a)
+	.long R(fd, f3, f3, 0e), R(bf, d2, d2, 6d)
+	.long R(81, cd, cd, 4c), R(18, 0c, 0c, 14)
+	.long R(26, 13, 13, 35), R(c3, ec, ec, 2f)
+	.long R(be, 5f, 5f, e1), R(35, 97, 97, a2)
+	.long R(88, 44, 44, cc), R(2e, 17, 17, 39)
+	.long R(93, c4, c4, 57), R(55, a7, a7, f2)
+	.long R(fc, 7e, 7e, 82), R(7a, 3d, 3d, 47)
+	.long R(c8, 64, 64, ac), R(ba, 5d, 5d, e7)
+	.long R(32, 19, 19, 2b), R(e6, 73, 73, 95)
+	.long R(c0, 60, 60, a0), R(19, 81, 81, 98)
+	.long R(9e, 4f, 4f, d1), R(a3, dc, dc, 7f)
+	.long R(44, 22, 22, 66), R(54, 2a, 2a, 7e)
+	.long R(3b, 90, 90, ab), R(0b, 88, 88, 83)
+	.long R(8c, 46, 46, ca), R(c7, ee, ee, 29)
+	.long R(6b, b8, b8, d3), R(28, 14, 14, 3c)
+	.long R(a7, de, de, 79), R(bc, 5e, 5e, e2)
+	.long R(16, 0b, 0b, 1d), R(ad, db, db, 76)
+	.long R(db, e0, e0, 3b), R(64, 32, 32, 56)
+	.long R(74, 3a, 3a, 4e), R(14, 0a, 0a, 1e)
+	.long R(92, 49, 49, db), R(0c, 06, 06, 0a)
+	.long R(48, 24, 24, 6c), R(b8, 5c, 5c, e4)
+	.long R(9f, c2, c2, 5d), R(bd, d3, d3, 6e)
+	.long R(43, ac, ac, ef), R(c4, 62, 62, a6)
+	.long R(39, 91, 91, a8), R(31, 95, 95, a4)
+	.long R(d3, e4, e4, 37), R(f2, 79, 79, 8b)
+	.long R(d5, e7, e7, 32), R(8b, c8, c8, 43)
+	.long R(6e, 37, 37, 59), R(da, 6d, 6d, b7)
+	.long R(01, 8d, 8d, 8c), R(b1, d5, d5, 64)
+	.long R(9c, 4e, 4e, d2), R(49, a9, a9, e0)
+	.long R(d8, 6c, 6c, b4), R(ac, 56, 56, fa)
+	.long R(f3, f4, f4, 07), R(cf, ea, ea, 25)
+	.long R(ca, 65, 65, af), R(f4, 7a, 7a, 8e)
+	.long R(47, ae, ae, e9), R(10, 08, 08, 18)
+	.long R(6f, ba, ba, d5), R(f0, 78, 78, 88)
+	.long R(4a, 25, 25, 6f), R(5c, 2e, 2e, 72)
+	.long R(38, 1c, 1c, 24), R(57, a6, a6, f1)
+	.long R(73, b4, b4, c7), R(97, c6, c6, 51)
+	.long R(cb, e8, e8, 23), R(a1, dd, dd, 7c)
+	.long R(e8, 74, 74, 9c), R(3e, 1f, 1f, 21)
+	.long R(96, 4b, 4b, dd), R(61, bd, bd, dc)
+	.long R(0d, 8b, 8b, 86), R(0f, 8a, 8a, 85)
+	.long R(e0, 70, 70, 90), R(7c, 3e, 3e, 42)
+	.long R(71, b5, b5, c4), R(cc, 66, 66, aa)
+	.long R(90, 48, 48, d8), R(06, 03, 03, 05)
+	.long R(f7, f6, f6, 01), R(1c, 0e, 0e, 12)
+	.long R(c2, 61, 61, a3), R(6a, 35, 35, 5f)
+	.long R(ae, 57, 57, f9), R(69, b9, b9, d0)
+	.long R(17, 86, 86, 91), R(99, c1, c1, 58)
+	.long R(3a, 1d, 1d, 27), R(27, 9e, 9e, b9)
+	.long R(d9, e1, e1, 38), R(eb, f8, f8, 13)
+	.long R(2b, 98, 98, b3), R(22, 11, 11, 33)
+	.long R(d2, 69, 69, bb), R(a9, d9, d9, 70)
+	.long R(07, 8e, 8e, 89), R(33, 94, 94, a7)
+	.long R(2d, 9b, 9b, b6), R(3c, 1e, 1e, 22)
+	.long R(15, 87, 87, 92), R(c9, e9, e9, 20)
+	.long R(87, ce, ce, 49), R(aa, 55, 55, ff)
+	.long R(50, 28, 28, 78), R(a5, df, df, 7a)
+	.long R(03, 8c, 8c, 8f), R(59, a1, a1, f8)
+	.long R(09, 89, 89, 80), R(1a, 0d, 0d, 17)
+	.long R(65, bf, bf, da), R(d7, e6, e6, 31)
+	.long R(84, 42, 42, c6), R(d0, 68, 68, b8)
+	.long R(82, 41, 41, c3), R(29, 99, 99, b0)
+	.long R(5a, 2d, 2d, 77), R(1e, 0f, 0f, 11)
+	.long R(7b, b0, b0, cb), R(a8, 54, 54, fc)
+	.long R(6d, bb, bb, d6), R(2c, 16, 16, 3a)
+.globl PPC_AES_4K_DECTAB
+PPC_AES_4K_DECTAB:
+/* decryption table, same as crypto_it_tab in crypto/aes-generic.c */
+	.long R(51, f4, a7, 50), R(7e, 41, 65, 53)
+	.long R(1a, 17, a4, c3), R(3a, 27, 5e, 96)
+	.long R(3b, ab, 6b, cb), R(1f, 9d, 45, f1)
+	.long R(ac, fa, 58, ab), R(4b, e3, 03, 93)
+	.long R(20, 30, fa, 55), R(ad, 76, 6d, f6)
+	.long R(88, cc, 76, 91), R(f5, 02, 4c, 25)
+	.long R(4f, e5, d7, fc), R(c5, 2a, cb, d7)
+	.long R(26, 35, 44, 80), R(b5, 62, a3, 8f)
+	.long R(de, b1, 5a, 49), R(25, ba, 1b, 67)
+	.long R(45, ea, 0e, 98), R(5d, fe, c0, e1)
+	.long R(c3, 2f, 75, 02), R(81, 4c, f0, 12)
+	.long R(8d, 46, 97, a3), R(6b, d3, f9, c6)
+	.long R(03, 8f, 5f, e7), R(15, 92, 9c, 95)
+	.long R(bf, 6d, 7a, eb), R(95, 52, 59, da)
+	.long R(d4, be, 83, 2d), R(58, 74, 21, d3)
+	.long R(49, e0, 69, 29), R(8e, c9, c8, 44)
+	.long R(75, c2, 89, 6a), R(f4, 8e, 79, 78)
+	.long R(99, 58, 3e, 6b), R(27, b9, 71, dd)
+	.long R(be, e1, 4f, b6), R(f0, 88, ad, 17)
+	.long R(c9, 20, ac, 66), R(7d, ce, 3a, b4)
+	.long R(63, df, 4a, 18), R(e5, 1a, 31, 82)
+	.long R(97, 51, 33, 60), R(62, 53, 7f, 45)
+	.long R(b1, 64, 77, e0), R(bb, 6b, ae, 84)
+	.long R(fe, 81, a0, 1c), R(f9, 08, 2b, 94)
+	.long R(70, 48, 68, 58), R(8f, 45, fd, 19)
+	.long R(94, de, 6c, 87), R(52, 7b, f8, b7)
+	.long R(ab, 73, d3, 23), R(72, 4b, 02, e2)
+	.long R(e3, 1f, 8f, 57), R(66, 55, ab, 2a)
+	.long R(b2, eb, 28, 07), R(2f, b5, c2, 03)
+	.long R(86, c5, 7b, 9a), R(d3, 37, 08, a5)
+	.long R(30, 28, 87, f2), R(23, bf, a5, b2)
+	.long R(02, 03, 6a, ba), R(ed, 16, 82, 5c)
+	.long R(8a, cf, 1c, 2b), R(a7, 79, b4, 92)
+	.long R(f3, 07, f2, f0), R(4e, 69, e2, a1)
+	.long R(65, da, f4, cd), R(06, 05, be, d5)
+	.long R(d1, 34, 62, 1f), R(c4, a6, fe, 8a)
+	.long R(34, 2e, 53, 9d), R(a2, f3, 55, a0)
+	.long R(05, 8a, e1, 32), R(a4, f6, eb, 75)
+	.long R(0b, 83, ec, 39), R(40, 60, ef, aa)
+	.long R(5e, 71, 9f, 06), R(bd, 6e, 10, 51)
+	.long R(3e, 21, 8a, f9), R(96, dd, 06, 3d)
+	.long R(dd, 3e, 05, ae), R(4d, e6, bd, 46)
+	.long R(91, 54, 8d, b5), R(71, c4, 5d, 05)
+	.long R(04, 06, d4, 6f), R(60, 50, 15, ff)
+	.long R(19, 98, fb, 24), R(d6, bd, e9, 97)
+	.long R(89, 40, 43, cc), R(67, d9, 9e, 77)
+	.long R(b0, e8, 42, bd), R(07, 89, 8b, 88)
+	.long R(e7, 19, 5b, 38), R(79, c8, ee, db)
+	.long R(a1, 7c, 0a, 47), R(7c, 42, 0f, e9)
+	.long R(f8, 84, 1e, c9), R(00, 00, 00, 00)
+	.long R(09, 80, 86, 83), R(32, 2b, ed, 48)
+	.long R(1e, 11, 70, ac), R(6c, 5a, 72, 4e)
+	.long R(fd, 0e, ff, fb), R(0f, 85, 38, 56)
+	.long R(3d, ae, d5, 1e), R(36, 2d, 39, 27)
+	.long R(0a, 0f, d9, 64), R(68, 5c, a6, 21)
+	.long R(9b, 5b, 54, d1), R(24, 36, 2e, 3a)
+	.long R(0c, 0a, 67, b1), R(93, 57, e7, 0f)
+	.long R(b4, ee, 96, d2), R(1b, 9b, 91, 9e)
+	.long R(80, c0, c5, 4f), R(61, dc, 20, a2)
+	.long R(5a, 77, 4b, 69), R(1c, 12, 1a, 16)
+	.long R(e2, 93, ba, 0a), R(c0, a0, 2a, e5)
+	.long R(3c, 22, e0, 43), R(12, 1b, 17, 1d)
+	.long R(0e, 09, 0d, 0b), R(f2, 8b, c7, ad)
+	.long R(2d, b6, a8, b9), R(14, 1e, a9, c8)
+	.long R(57, f1, 19, 85), R(af, 75, 07, 4c)
+	.long R(ee, 99, dd, bb), R(a3, 7f, 60, fd)
+	.long R(f7, 01, 26, 9f), R(5c, 72, f5, bc)
+	.long R(44, 66, 3b, c5), R(5b, fb, 7e, 34)
+	.long R(8b, 43, 29, 76), R(cb, 23, c6, dc)
+	.long R(b6, ed, fc, 68), R(b8, e4, f1, 63)
+	.long R(d7, 31, dc, ca), R(42, 63, 85, 10)
+	.long R(13, 97, 22, 40), R(84, c6, 11, 20)
+	.long R(85, 4a, 24, 7d), R(d2, bb, 3d, f8)
+	.long R(ae, f9, 32, 11), R(c7, 29, a1, 6d)
+	.long R(1d, 9e, 2f, 4b), R(dc, b2, 30, f3)
+	.long R(0d, 86, 52, ec), R(77, c1, e3, d0)
+	.long R(2b, b3, 16, 6c), R(a9, 70, b9, 99)
+	.long R(11, 94, 48, fa), R(47, e9, 64, 22)
+	.long R(a8, fc, 8c, c4), R(a0, f0, 3f, 1a)
+	.long R(56, 7d, 2c, d8), R(22, 33, 90, ef)
+	.long R(87, 49, 4e, c7), R(d9, 38, d1, c1)
+	.long R(8c, ca, a2, fe), R(98, d4, 0b, 36)
+	.long R(a6, f5, 81, cf), R(a5, 7a, de, 28)
+	.long R(da, b7, 8e, 26), R(3f, ad, bf, a4)
+	.long R(2c, 3a, 9d, e4), R(50, 78, 92, 0d)
+	.long R(6a, 5f, cc, 9b), R(54, 7e, 46, 62)
+	.long R(f6, 8d, 13, c2), R(90, d8, b8, e8)
+	.long R(2e, 39, f7, 5e), R(82, c3, af, f5)
+	.long R(9f, 5d, 80, be), R(69, d0, 93, 7c)
+	.long R(6f, d5, 2d, a9), R(cf, 25, 12, b3)
+	.long R(c8, ac, 99, 3b), R(10, 18, 7d, a7)
+	.long R(e8, 9c, 63, 6e), R(db, 3b, bb, 7b)
+	.long R(cd, 26, 78, 09), R(6e, 59, 18, f4)
+	.long R(ec, 9a, b7, 01), R(83, 4f, 9a, a8)
+	.long R(e6, 95, 6e, 65), R(aa, ff, e6, 7e)
+	.long R(21, bc, cf, 08), R(ef, 15, e8, e6)
+	.long R(ba, e7, 9b, d9), R(4a, 6f, 36, ce)
+	.long R(ea, 9f, 09, d4), R(29, b0, 7c, d6)
+	.long R(31, a4, b2, af), R(2a, 3f, 23, 31)
+	.long R(c6, a5, 94, 30), R(35, a2, 66, c0)
+	.long R(74, 4e, bc, 37), R(fc, 82, ca, a6)
+	.long R(e0, 90, d0, b0), R(33, a7, d8, 15)
+	.long R(f1, 04, 98, 4a), R(41, ec, da, f7)
+	.long R(7f, cd, 50, 0e), R(17, 91, f6, 2f)
+	.long R(76, 4d, d6, 8d), R(43, ef, b0, 4d)
+	.long R(cc, aa, 4d, 54), R(e4, 96, 04, df)
+	.long R(9e, d1, b5, e3), R(4c, 6a, 88, 1b)
+	.long R(c1, 2c, 1f, b8), R(46, 65, 51, 7f)
+	.long R(9d, 5e, ea, 04), R(01, 8c, 35, 5d)
+	.long R(fa, 87, 74, 73), R(fb, 0b, 41, 2e)
+	.long R(b3, 67, 1d, 5a), R(92, db, d2, 52)
+	.long R(e9, 10, 56, 33), R(6d, d6, 47, 13)
+	.long R(9a, d7, 61, 8c), R(37, a1, 0c, 7a)
+	.long R(59, f8, 14, 8e), R(eb, 13, 3c, 89)
+	.long R(ce, a9, 27, ee), R(b7, 61, c9, 35)
+	.long R(e1, 1c, e5, ed), R(7a, 47, b1, 3c)
+	.long R(9c, d2, df, 59), R(55, f2, 73, 3f)
+	.long R(18, 14, ce, 79), R(73, c7, 37, bf)
+	.long R(53, f7, cd, ea), R(5f, fd, aa, 5b)
+	.long R(df, 3d, 6f, 14), R(78, 44, db, 86)
+	.long R(ca, af, f3, 81), R(b9, 68, c4, 3e)
+	.long R(38, 24, 34, 2c), R(c2, a3, 40, 5f)
+	.long R(16, 1d, c3, 72), R(bc, e2, 25, 0c)
+	.long R(28, 3c, 49, 8b), R(ff, 0d, 95, 41)
+	.long R(39, a8, 01, 71), R(08, 0c, b3, de)
+	.long R(d8, b4, e4, 9c), R(64, 56, c1, 90)
+	.long R(7b, cb, 84, 61), R(d5, 32, b6, 70)
+	.long R(48, 6c, 5c, 74), R(d0, b8, 57, 42)
+.globl PPC_AES_4K_DECTAB2
+PPC_AES_4K_DECTAB2:
+/* decryption table, same as crypto_il_tab in crypto/aes-generic.c */
+	.byte 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38
+	.byte 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb
+	.byte 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87
+	.byte 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb
+	.byte 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d
+	.byte 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e
+	.byte 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2
+	.byte 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25
+	.byte 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16
+	.byte 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92
+	.byte 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda
+	.byte 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84
+	.byte 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a
+	.byte 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06
+	.byte 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02
+	.byte 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b
+	.byte 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea
+	.byte 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73
+	.byte 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85
+	.byte 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e
+	.byte 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89
+	.byte 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b
+	.byte 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20
+	.byte 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4
+	.byte 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31
+	.byte 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f
+	.byte 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d
+	.byte 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef
+	.byte 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0
+	.byte 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61
+	.byte 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26
+	.byte 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d
diff --git a/lib/crypto/powerpc/aes.h b/lib/crypto/powerpc/aes.h
new file mode 100644
index 000000000000..5a36b637e6b9
--- /dev/null
+++ b/lib/crypto/powerpc/aes.h
@@ -0,0 +1,240 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
+ * Copyright (C) 2015 International Business Machines Inc.
+ * Copyright 2026 Google LLC
+ */
+#include <asm/simd.h>
+#include <asm/switch_to.h>
+#include <linux/cpufeature.h>
+#include <linux/jump_label.h>
+#include <linux/preempt.h>
+#include <linux/uaccess.h>
+
+#ifdef CONFIG_SPE
+
+EXPORT_SYMBOL_GPL(ppc_expand_key_128);
+EXPORT_SYMBOL_GPL(ppc_expand_key_192);
+EXPORT_SYMBOL_GPL(ppc_expand_key_256);
+EXPORT_SYMBOL_GPL(ppc_generate_decrypt_key);
+EXPORT_SYMBOL_GPL(ppc_encrypt_ecb);
+EXPORT_SYMBOL_GPL(ppc_decrypt_ecb);
+EXPORT_SYMBOL_GPL(ppc_encrypt_cbc);
+EXPORT_SYMBOL_GPL(ppc_decrypt_cbc);
+EXPORT_SYMBOL_GPL(ppc_crypt_ctr);
+EXPORT_SYMBOL_GPL(ppc_encrypt_xts);
+EXPORT_SYMBOL_GPL(ppc_decrypt_xts);
+
+void ppc_encrypt_aes(u8 *out, const u8 *in, const u32 *key_enc, u32 rounds);
+void ppc_decrypt_aes(u8 *out, const u8 *in, const u32 *key_dec, u32 rounds);
+
+static void spe_begin(void)
+{
+	/* disable preemption and save users SPE registers if required */
+	preempt_disable();
+	enable_kernel_spe();
+}
+
+static void spe_end(void)
+{
+	disable_kernel_spe();
+	/* reenable preemption */
+	preempt_enable();
+}
+
+static void aes_preparekey_arch(union aes_enckey_arch *k,
+				union aes_invkey_arch *inv_k,
+				const u8 *in_key, int key_len, int nrounds)
+{
+	if (key_len == AES_KEYSIZE_128)
+		ppc_expand_key_128(k->spe_enc_key, in_key);
+	else if (key_len == AES_KEYSIZE_192)
+		ppc_expand_key_192(k->spe_enc_key, in_key);
+	else
+		ppc_expand_key_256(k->spe_enc_key, in_key);
+
+	if (inv_k)
+		ppc_generate_decrypt_key(inv_k->spe_dec_key, k->spe_enc_key,
+					 key_len);
+}
+
+static void aes_encrypt_arch(const struct aes_enckey *key,
+			     u8 out[AES_BLOCK_SIZE],
+			     const u8 in[AES_BLOCK_SIZE])
+{
+	spe_begin();
+	ppc_encrypt_aes(out, in, key->k.spe_enc_key, key->nrounds / 2 - 1);
+	spe_end();
+}
+
+static void aes_decrypt_arch(const struct aes_key *key,
+			     u8 out[AES_BLOCK_SIZE],
+			     const u8 in[AES_BLOCK_SIZE])
+{
+	spe_begin();
+	ppc_decrypt_aes(out, in, key->inv_k.spe_dec_key, key->nrounds / 2 - 1);
+	spe_end();
+}
+
+#else /* CONFIG_SPE */
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto);
+
+EXPORT_SYMBOL_GPL(aes_p8_set_encrypt_key);
+EXPORT_SYMBOL_GPL(aes_p8_set_decrypt_key);
+EXPORT_SYMBOL_GPL(aes_p8_encrypt);
+EXPORT_SYMBOL_GPL(aes_p8_decrypt);
+EXPORT_SYMBOL_GPL(aes_p8_cbc_encrypt);
+EXPORT_SYMBOL_GPL(aes_p8_ctr32_encrypt_blocks);
+EXPORT_SYMBOL_GPL(aes_p8_xts_encrypt);
+EXPORT_SYMBOL_GPL(aes_p8_xts_decrypt);
+
+static inline bool is_vsx_format(const struct p8_aes_key *key)
+{
+	return key->nrounds != 0;
+}
+
+/*
+ * Convert a round key from VSX to generic format by reflecting all 16 bytes (if
+ * little endian) or reflecting the bytes in each 4-byte word (if big endian),
+ * and (if apply_inv_mix=true) applying InvMixColumn to each column.
+ *
+ * It would be nice if the VSX and generic key formats would be compatible.  But
+ * that's very difficult to do, with the assembly code having been borrowed from
+ * OpenSSL and also targeted to POWER8 rather than POWER9.
+ *
+ * Fortunately, this conversion should only be needed in extremely rare cases,
+ * possibly not at all in practice.  It's just included for full correctness.
+ */
+static void rndkey_from_vsx(u32 out[4], const u32 in[4], bool apply_inv_mix)
+{
+	const bool be = IS_ENABLED(CONFIG_CPU_BIG_ENDIAN);
+	u32 k0 = swab32(in[0]);
+	u32 k1 = swab32(in[1]);
+	u32 k2 = swab32(in[2]);
+	u32 k3 = swab32(in[3]);
+
+	if (apply_inv_mix) {
+		k0 = inv_mix_columns(k0);
+		k1 = inv_mix_columns(k1);
+		k2 = inv_mix_columns(k2);
+		k3 = inv_mix_columns(k3);
+	}
+	out[0] = be ? k0 : k3;
+	out[1] = be ? k1 : k2;
+	out[2] = be ? k2 : k1;
+	out[3] = be ? k3 : k0;
+}
+
+static void aes_preparekey_arch(union aes_enckey_arch *k,
+				union aes_invkey_arch *inv_k,
+				const u8 *in_key, int key_len, int nrounds)
+{
+	const int keybits = 8 * key_len;
+	int ret;
+
+	if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) {
+		preempt_disable();
+		pagefault_disable();
+		enable_kernel_vsx();
+		ret = aes_p8_set_encrypt_key(in_key, keybits, &k->p8);
+		/*
+		 * aes_p8_set_encrypt_key() should never fail here, since the
+		 * key length was already validated.
+		 */
+		WARN_ON_ONCE(ret);
+		if (inv_k) {
+			ret = aes_p8_set_decrypt_key(in_key, keybits,
+						     &inv_k->p8);
+			/* ... and likewise for aes_p8_set_decrypt_key(). */
+			WARN_ON_ONCE(ret);
+		}
+		disable_kernel_vsx();
+		pagefault_enable();
+		preempt_enable();
+	} else {
+		aes_expandkey_generic(k->rndkeys,
+				      inv_k ? inv_k->inv_rndkeys : NULL,
+				      in_key, key_len);
+		/* Mark the key as using the generic format. */
+		k->p8.nrounds = 0;
+		if (inv_k)
+			inv_k->p8.nrounds = 0;
+	}
+}
+
+static void aes_encrypt_arch(const struct aes_enckey *key,
+			     u8 out[AES_BLOCK_SIZE],
+			     const u8 in[AES_BLOCK_SIZE])
+{
+	if (static_branch_likely(&have_vec_crypto) &&
+	    likely(is_vsx_format(&key->k.p8) && may_use_simd())) {
+		preempt_disable();
+		pagefault_disable();
+		enable_kernel_vsx();
+		aes_p8_encrypt(in, out, &key->k.p8);
+		disable_kernel_vsx();
+		pagefault_enable();
+		preempt_enable();
+	} else if (unlikely(is_vsx_format(&key->k.p8))) {
+		/*
+		 * This handles (the hopefully extremely rare) case where a key
+		 * was prepared using the VSX optimized format, then encryption
+		 * is done in a context that cannot use VSX instructions.
+		 */
+		u32 rndkeys[AES_MAX_KEYLENGTH_U32];
+
+		for (int i = 0; i < 4 * (key->nrounds + 1); i += 4)
+			rndkey_from_vsx(&rndkeys[i],
+					&key->k.p8.rndkeys[i], false);
+		aes_encrypt_generic(rndkeys, key->nrounds, out, in);
+	} else {
+		aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in);
+	}
+}
+
+static void aes_decrypt_arch(const struct aes_key *key, u8 out[AES_BLOCK_SIZE],
+			     const u8 in[AES_BLOCK_SIZE])
+{
+	if (static_branch_likely(&have_vec_crypto) &&
+	    likely(is_vsx_format(&key->inv_k.p8) && may_use_simd())) {
+		preempt_disable();
+		pagefault_disable();
+		enable_kernel_vsx();
+		aes_p8_decrypt(in, out, &key->inv_k.p8);
+		disable_kernel_vsx();
+		pagefault_enable();
+		preempt_enable();
+	} else if (unlikely(is_vsx_format(&key->inv_k.p8))) {
+		/*
+		 * This handles (the hopefully extremely rare) case where a key
+		 * was prepared using the VSX optimized format, then decryption
+		 * is done in a context that cannot use VSX instructions.
+		 */
+		u32 inv_rndkeys[AES_MAX_KEYLENGTH_U32];
+		int i;
+
+		rndkey_from_vsx(&inv_rndkeys[0],
+				&key->inv_k.p8.rndkeys[0], false);
+		for (i = 4; i < 4 * key->nrounds; i += 4) {
+			rndkey_from_vsx(&inv_rndkeys[i],
+					&key->inv_k.p8.rndkeys[i], true);
+		}
+		rndkey_from_vsx(&inv_rndkeys[i],
+				&key->inv_k.p8.rndkeys[i], false);
+		aes_decrypt_generic(inv_rndkeys, key->nrounds, out, in);
+	} else {
+		aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds,
+				    out, in);
+	}
+}
+
+#define aes_mod_init_arch aes_mod_init_arch
+static void aes_mod_init_arch(void)
+{
+	if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
+	    (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
+		static_branch_enable(&have_vec_crypto);
+}
+
+#endif /* !CONFIG_SPE */
diff --git a/lib/crypto/powerpc/aesp8-ppc.pl b/lib/crypto/powerpc/aesp8-ppc.pl
new file mode 100644
index 000000000000..253a06758057
--- /dev/null
+++ b/lib/crypto/powerpc/aesp8-ppc.pl
@@ -0,0 +1,3890 @@
+#! /usr/bin/env perl
+# SPDX-License-Identifier: GPL-2.0
+
+# This code is taken from CRYPTOGAMs[1] and is included here using the option
+# in the license to distribute the code under the GPL. Therefore this program
+# is free software; you can redistribute it and/or modify it under the terms of
+# the GNU General Public License version 2 as published by the Free Software
+# Foundation.
+#
+# [1] https://www.openssl.org/~appro/cryptogams/
+
+# Copyright (c) 2006-2017, CRYPTOGAMS by <appro@openssl.org>
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+#       * Redistributions of source code must retain copyright notices,
+#         this list of conditions and the following disclaimer.
+#
+#       * Redistributions in binary form must reproduce the above
+#         copyright notice, this list of conditions and the following
+#         disclaimer in the documentation and/or other materials
+#         provided with the distribution.
+#
+#       * Neither the name of the CRYPTOGAMS nor the names of its
+#         copyright holder and contributors may be used to endorse or
+#         promote products derived from this software without specific
+#         prior written permission.
+#
+# ALTERNATIVELY, provided that this notice is retained in full, this
+# product may be distributed under the terms of the GNU General Public
+# License (GPL), in which case the provisions of the GPL apply INSTEAD OF
+# those given above.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# ====================================================================
+# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+# project. The module is, however, dual licensed under OpenSSL and
+# CRYPTOGAMS licenses depending on where you obtain it. For further
+# details see https://www.openssl.org/~appro/cryptogams/.
+# ====================================================================
+#
+# This module implements support for AES instructions as per PowerISA
+# specification version 2.07, first implemented by POWER8 processor.
+# The module is endian-agnostic in sense that it supports both big-
+# and little-endian cases. Data alignment in parallelizable modes is
+# handled with VSX loads and stores, which implies MSR.VSX flag being
+# set. It should also be noted that ISA specification doesn't prohibit
+# alignment exceptions for these instructions on page boundaries.
+# Initially alignment was handled in pure AltiVec/VMX way [when data
+# is aligned programmatically, which in turn guarantees exception-
+# free execution], but it turned to hamper performance when vcipher
+# instructions are interleaved. It's reckoned that eventual
+# misalignment penalties at page boundaries are in average lower
+# than additional overhead in pure AltiVec approach.
+#
+# May 2016
+#
+# Add XTS subroutine, 9x on little- and 12x improvement on big-endian
+# systems were measured.
+#
+######################################################################
+# Current large-block performance in cycles per byte processed with
+# 128-bit key (less is better).
+#
+#		CBC en-/decrypt	CTR	XTS
+# POWER8[le]	3.96/0.72	0.74	1.1
+# POWER8[be]	3.75/0.65	0.66	1.0
+
+$flavour = shift;
+
+if ($flavour =~ /64/) {
+	$SIZE_T	=8;
+	$LRSAVE	=2*$SIZE_T;
+	$STU	="stdu";
+	$POP	="ld";
+	$PUSH	="std";
+	$UCMP	="cmpld";
+	$SHL	="sldi";
+} elsif ($flavour =~ /32/) {
+	$SIZE_T	=4;
+	$LRSAVE	=$SIZE_T;
+	$STU	="stwu";
+	$POP	="lwz";
+	$PUSH	="stw";
+	$UCMP	="cmplw";
+	$SHL	="slwi";
+} else { die "nonsense $flavour"; }
+
+$LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0;
+
+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
+( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
+( $xlate="${dir}../../../arch/powerpc/crypto/ppc-xlate.pl" and -f $xlate) or
+die "can't locate ppc-xlate.pl";
+
+open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
+
+$FRAME=8*$SIZE_T;
+$prefix="aes_p8";
+
+$sp="r1";
+$vrsave="r12";
+
+#########################################################################
+{{{	# Key setup procedures						#
+my ($inp,$bits,$out,$ptr,$cnt,$rounds)=map("r$_",(3..8));
+my ($zero,$in0,$in1,$key,$rcon,$mask,$tmp)=map("v$_",(0..6));
+my ($stage,$outperm,$outmask,$outhead,$outtail)=map("v$_",(7..11));
+
+$code.=<<___;
+.machine	"any"
+
+.text
+
+.align	7
+rcon:
+.long	0x01000000, 0x01000000, 0x01000000, 0x01000000	?rev
+.long	0x1b000000, 0x1b000000, 0x1b000000, 0x1b000000	?rev
+.long	0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c	?rev
+.long	0,0,0,0						?asis
+.long	0x0f102132, 0x43546576, 0x8798a9ba, 0xcbdcedfe
+Lconsts:
+	mflr	r0
+	bcl	20,31,\$+4
+	mflr	$ptr	 #vvvvv "distance between . and rcon
+	addi	$ptr,$ptr,-0x58
+	mtlr	r0
+	blr
+	.long	0
+	.byte	0,12,0x14,0,0,0,0,0
+.asciz	"AES for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"
+
+.globl	.${prefix}_set_encrypt_key
+Lset_encrypt_key:
+	mflr		r11
+	$PUSH		r11,$LRSAVE($sp)
+
+	li		$ptr,-1
+	${UCMP}i	$inp,0
+	beq-		Lenc_key_abort		# if ($inp==0) return -1;
+	${UCMP}i	$out,0
+	beq-		Lenc_key_abort		# if ($out==0) return -1;
+	li		$ptr,-2
+	cmpwi		$bits,128
+	blt-		Lenc_key_abort
+	cmpwi		$bits,256
+	bgt-		Lenc_key_abort
+	andi.		r0,$bits,0x3f
+	bne-		Lenc_key_abort
+
+	lis		r0,0xfff0
+	mfspr		$vrsave,256
+	mtspr		256,r0
+
+	bl		Lconsts
+	mtlr		r11
+
+	neg		r9,$inp
+	lvx		$in0,0,$inp
+	addi		$inp,$inp,15		# 15 is not typo
+	lvsr		$key,0,r9		# borrow $key
+	li		r8,0x20
+	cmpwi		$bits,192
+	lvx		$in1,0,$inp
+	le?vspltisb	$mask,0x0f		# borrow $mask
+	lvx		$rcon,0,$ptr
+	le?vxor		$key,$key,$mask		# adjust for byte swap
+	lvx		$mask,r8,$ptr
+	addi		$ptr,$ptr,0x10
+	vperm		$in0,$in0,$in1,$key	# align [and byte swap in LE]
+	li		$cnt,8
+	vxor		$zero,$zero,$zero
+	mtctr		$cnt
+
+	?lvsr		$outperm,0,$out
+	vspltisb	$outmask,-1
+	lvx		$outhead,0,$out
+	?vperm		$outmask,$zero,$outmask,$outperm
+
+	blt		Loop128
+	addi		$inp,$inp,8
+	beq		L192
+	addi		$inp,$inp,8
+	b		L256
+
+.align	4
+Loop128:
+	vperm		$key,$in0,$in0,$mask	# rotate-n-splat
+	vsldoi		$tmp,$zero,$in0,12	# >>32
+	 vperm		$outtail,$in0,$in0,$outperm	# rotate
+	 vsel		$stage,$outhead,$outtail,$outmask
+	 vmr		$outhead,$outtail
+	vcipherlast	$key,$key,$rcon
+	 stvx		$stage,0,$out
+	 addi		$out,$out,16
+
+	vxor		$in0,$in0,$tmp
+	vsldoi		$tmp,$zero,$tmp,12	# >>32
+	vxor		$in0,$in0,$tmp
+	vsldoi		$tmp,$zero,$tmp,12	# >>32
+	vxor		$in0,$in0,$tmp
+	 vadduwm	$rcon,$rcon,$rcon
+	vxor		$in0,$in0,$key
+	bdnz		Loop128
+
+	lvx		$rcon,0,$ptr		# last two round keys
+
+	vperm		$key,$in0,$in0,$mask	# rotate-n-splat
+	vsldoi		$tmp,$zero,$in0,12	# >>32
+	 vperm		$outtail,$in0,$in0,$outperm	# rotate
+	 vsel		$stage,$outhead,$outtail,$outmask
+	 vmr		$outhead,$outtail
+	vcipherlast	$key,$key,$rcon
+	 stvx		$stage,0,$out
+	 addi		$out,$out,16
+
+	vxor		$in0,$in0,$tmp
+	vsldoi		$tmp,$zero,$tmp,12	# >>32
+	vxor		$in0,$in0,$tmp
+	vsldoi		$tmp,$zero,$tmp,12	# >>32
+	vxor		$in0,$in0,$tmp
+	 vadduwm	$rcon,$rcon,$rcon
+	vxor		$in0,$in0,$key
+
+	vperm		$key,$in0,$in0,$mask	# rotate-n-splat
+	vsldoi		$tmp,$zero,$in0,12	# >>32
+	 vperm		$outtail,$in0,$in0,$outperm	# rotate
+	 vsel		$stage,$outhead,$outtail,$outmask
+	 vmr		$outhead,$outtail
+	vcipherlast	$key,$key,$rcon
+	 stvx		$stage,0,$out
+	 addi		$out,$out,16
+
+	vxor		$in0,$in0,$tmp
+	vsldoi		$tmp,$zero,$tmp,12	# >>32
+	vxor		$in0,$in0,$tmp
+	vsldoi		$tmp,$zero,$tmp,12	# >>32
+	vxor		$in0,$in0,$tmp
+	vxor		$in0,$in0,$key
+	 vperm		$outtail,$in0,$in0,$outperm	# rotate
+	 vsel		$stage,$outhead,$outtail,$outmask
+	 vmr		$outhead,$outtail
+	 stvx		$stage,0,$out
+
+	addi		$inp,$out,15		# 15 is not typo
+	addi		$out,$out,0x50
+
+	li		$rounds,10
+	b		Ldone
+
+.align	4
+L192:
+	lvx		$tmp,0,$inp
+	li		$cnt,4
+	 vperm		$outtail,$in0,$in0,$outperm	# rotate
+	 vsel		$stage,$outhead,$outtail,$outmask
+	 vmr		$outhead,$outtail
+	 stvx		$stage,0,$out
+	 addi		$out,$out,16
+	vperm		$in1,$in1,$tmp,$key	# align [and byte swap in LE]
+	vspltisb	$key,8			# borrow $key
+	mtctr		$cnt
+	vsububm		$mask,$mask,$key	# adjust the mask
+
+Loop192:
+	vperm		$key,$in1,$in1,$mask	# roate-n-splat
+	vsldoi		$tmp,$zero,$in0,12	# >>32
+	vcipherlast	$key,$key,$rcon
+
+	vxor		$in0,$in0,$tmp
+	vsldoi		$tmp,$zero,$tmp,12	# >>32
+	vxor		$in0,$in0,$tmp
+	vsldoi		$tmp,$zero,$tmp,12	# >>32
+	vxor		$in0,$in0,$tmp
+
+	 vsldoi		$stage,$zero,$in1,8
+	vspltw		$tmp,$in0,3
+	vxor		$tmp,$tmp,$in1
+	vsldoi		$in1,$zero,$in1,12	# >>32
+	 vadduwm	$rcon,$rcon,$rcon
+	vxor		$in1,$in1,$tmp
+	vxor		$in0,$in0,$key
+	vxor		$in1,$in1,$key
+	 vsldoi		$stage,$stage,$in0,8
+
+	vperm		$key,$in1,$in1,$mask	# rotate-n-splat
+	vsldoi		$tmp,$zero,$in0,12	# >>32
+	 vperm		$outtail,$stage,$stage,$outperm	# rotate
+	 vsel		$stage,$outhead,$outtail,$outmask
+	 vmr		$outhead,$outtail
+	vcipherlast	$key,$key,$rcon
+	 stvx		$stage,0,$out
+	 addi		$out,$out,16
+
+	 vsldoi		$stage,$in0,$in1,8
+	vxor		$in0,$in0,$tmp
+	vsldoi		$tmp,$zero,$tmp,12	# >>32
+	 vperm		$outtail,$stage,$stage,$outperm	# rotate
+	 vsel		$stage,$outhead,$outtail,$outmask
+	 vmr		$outhead,$outtail
+	vxor		$in0,$in0,$tmp
+	vsldoi		$tmp,$zero,$tmp,12	# >>32
+	vxor		$in0,$in0,$tmp
+	 stvx		$stage,0,$out
+	 addi		$out,$out,16
+
+	vspltw		$tmp,$in0,3
+	vxor		$tmp,$tmp,$in1
+	vsldoi		$in1,$zero,$in1,12	# >>32
+	 vadduwm	$rcon,$rcon,$rcon
+	vxor		$in1,$in1,$tmp
+	vxor		$in0,$in0,$key
+	vxor		$in1,$in1,$key
+	 vperm		$outtail,$in0,$in0,$outperm	# rotate
+	 vsel		$stage,$outhead,$outtail,$outmask
+	 vmr		$outhead,$outtail
+	 stvx		$stage,0,$out
+	 addi		$inp,$out,15		# 15 is not typo
+	 addi		$out,$out,16
+	bdnz		Loop192
+
+	li		$rounds,12
+	addi		$out,$out,0x20
+	b		Ldone
+
+.align	4
+L256:
+	lvx		$tmp,0,$inp
+	li		$cnt,7
+	li		$rounds,14
+	 vperm		$outtail,$in0,$in0,$outperm	# rotate
+	 vsel		$stage,$outhead,$outtail,$outmask
+	 vmr		$outhead,$outtail
+	 stvx		$stage,0,$out
+	 addi		$out,$out,16
+	vperm		$in1,$in1,$tmp,$key	# align [and byte swap in LE]
+	mtctr		$cnt
+
+Loop256:
+	vperm		$key,$in1,$in1,$mask	# rotate-n-splat
+	vsldoi		$tmp,$zero,$in0,12	# >>32
+	 vperm		$outtail,$in1,$in1,$outperm	# rotate
+	 vsel		$stage,$outhead,$outtail,$outmask
+	 vmr		$outhead,$outtail
+	vcipherlast	$key,$key,$rcon
+	 stvx		$stage,0,$out
+	 addi		$out,$out,16
+
+	vxor		$in0,$in0,$tmp
+	vsldoi		$tmp,$zero,$tmp,12	# >>32
+	vxor		$in0,$in0,$tmp
+	vsldoi		$tmp,$zero,$tmp,12	# >>32
+	vxor		$in0,$in0,$tmp
+	 vadduwm	$rcon,$rcon,$rcon
+	vxor		$in0,$in0,$key
+	 vperm		$outtail,$in0,$in0,$outperm	# rotate
+	 vsel		$stage,$outhead,$outtail,$outmask
+	 vmr		$outhead,$outtail
+	 stvx		$stage,0,$out
+	 addi		$inp,$out,15		# 15 is not typo
+	 addi		$out,$out,16
+	bdz		Ldone
+
+	vspltw		$key,$in0,3		# just splat
+	vsldoi		$tmp,$zero,$in1,12	# >>32
+	vsbox		$key,$key
+
+	vxor		$in1,$in1,$tmp
+	vsldoi		$tmp,$zero,$tmp,12	# >>32
+	vxor		$in1,$in1,$tmp
+	vsldoi		$tmp,$zero,$tmp,12	# >>32
+	vxor		$in1,$in1,$tmp
+
+	vxor		$in1,$in1,$key
+	b		Loop256
+
+.align	4
+Ldone:
+	lvx		$in1,0,$inp		# redundant in aligned case
+	vsel		$in1,$outhead,$in1,$outmask
+	stvx		$in1,0,$inp
+	li		$ptr,0
+	mtspr		256,$vrsave
+	stw		$rounds,0($out)
+
+Lenc_key_abort:
+	mr		r3,$ptr
+	blr
+	.long		0
+	.byte		0,12,0x14,1,0,0,3,0
+	.long		0
+.size	.${prefix}_set_encrypt_key,.-.${prefix}_set_encrypt_key
+
+.globl	.${prefix}_set_decrypt_key
+	$STU		$sp,-$FRAME($sp)
+	mflr		r10
+	$PUSH		r10,$FRAME+$LRSAVE($sp)
+	bl		Lset_encrypt_key
+	mtlr		r10
+
+	cmpwi		r3,0
+	bne-		Ldec_key_abort
+
+	slwi		$cnt,$rounds,4
+	subi		$inp,$out,240		# first round key
+	srwi		$rounds,$rounds,1
+	add		$out,$inp,$cnt		# last round key
+	mtctr		$rounds
+
+Ldeckey:
+	lwz		r0, 0($inp)
+	lwz		r6, 4($inp)
+	lwz		r7, 8($inp)
+	lwz		r8, 12($inp)
+	addi		$inp,$inp,16
+	lwz		r9, 0($out)
+	lwz		r10,4($out)
+	lwz		r11,8($out)
+	lwz		r12,12($out)
+	stw		r0, 0($out)
+	stw		r6, 4($out)
+	stw		r7, 8($out)
+	stw		r8, 12($out)
+	subi		$out,$out,16
+	stw		r9, -16($inp)
+	stw		r10,-12($inp)
+	stw		r11,-8($inp)
+	stw		r12,-4($inp)
+	bdnz		Ldeckey
+
+	xor		r3,r3,r3		# return value
+Ldec_key_abort:
+	addi		$sp,$sp,$FRAME
+	blr
+	.long		0
+	.byte		0,12,4,1,0x80,0,3,0
+	.long		0
+.size	.${prefix}_set_decrypt_key,.-.${prefix}_set_decrypt_key
+___
+}}}
+#########################################################################
+{{{	# Single block en- and decrypt procedures			#
+sub gen_block () {
+my $dir = shift;
+my $n   = $dir eq "de" ? "n" : "";
+my ($inp,$out,$key,$rounds,$idx)=map("r$_",(3..7));
+
+$code.=<<___;
+.globl	.${prefix}_${dir}crypt
+	lwz		$rounds,240($key)
+	lis		r0,0xfc00
+	mfspr		$vrsave,256
+	li		$idx,15			# 15 is not typo
+	mtspr		256,r0
+
+	lvx		v0,0,$inp
+	neg		r11,$out
+	lvx		v1,$idx,$inp
+	lvsl		v2,0,$inp		# inpperm
+	le?vspltisb	v4,0x0f
+	?lvsl		v3,0,r11		# outperm
+	le?vxor		v2,v2,v4
+	li		$idx,16
+	vperm		v0,v0,v1,v2		# align [and byte swap in LE]
+	lvx		v1,0,$key
+	?lvsl		v5,0,$key		# keyperm
+	srwi		$rounds,$rounds,1
+	lvx		v2,$idx,$key
+	addi		$idx,$idx,16
+	subi		$rounds,$rounds,1
+	?vperm		v1,v1,v2,v5		# align round key
+
+	vxor		v0,v0,v1
+	lvx		v1,$idx,$key
+	addi		$idx,$idx,16
+	mtctr		$rounds
+
+Loop_${dir}c:
+	?vperm		v2,v2,v1,v5
+	v${n}cipher	v0,v0,v2
+	lvx		v2,$idx,$key
+	addi		$idx,$idx,16
+	?vperm		v1,v1,v2,v5
+	v${n}cipher	v0,v0,v1
+	lvx		v1,$idx,$key
+	addi		$idx,$idx,16
+	bdnz		Loop_${dir}c
+
+	?vperm		v2,v2,v1,v5
+	v${n}cipher	v0,v0,v2
+	lvx		v2,$idx,$key
+	?vperm		v1,v1,v2,v5
+	v${n}cipherlast	v0,v0,v1
+
+	vspltisb	v2,-1
+	vxor		v1,v1,v1
+	li		$idx,15			# 15 is not typo
+	?vperm		v2,v1,v2,v3		# outmask
+	le?vxor		v3,v3,v4
+	lvx		v1,0,$out		# outhead
+	vperm		v0,v0,v0,v3		# rotate [and byte swap in LE]
+	vsel		v1,v1,v0,v2
+	lvx		v4,$idx,$out
+	stvx		v1,0,$out
+	vsel		v0,v0,v4,v2
+	stvx		v0,$idx,$out
+
+	mtspr		256,$vrsave
+	blr
+	.long		0
+	.byte		0,12,0x14,0,0,0,3,0
+	.long		0
+.size	.${prefix}_${dir}crypt,.-.${prefix}_${dir}crypt
+___
+}
+&gen_block("en");
+&gen_block("de");
+}}}
+#########################################################################
+{{{	# CBC en- and decrypt procedures				#
+my ($inp,$out,$len,$key,$ivp,$enc,$rounds,$idx)=map("r$_",(3..10));
+my ($rndkey0,$rndkey1,$inout,$tmp)=		map("v$_",(0..3));
+my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm)=
+						map("v$_",(4..10));
+$code.=<<___;
+.globl	.${prefix}_cbc_encrypt
+	${UCMP}i	$len,16
+	bltlr-
+
+	cmpwi		$enc,0			# test direction
+	lis		r0,0xffe0
+	mfspr		$vrsave,256
+	mtspr		256,r0
+
+	li		$idx,15
+	vxor		$rndkey0,$rndkey0,$rndkey0
+	le?vspltisb	$tmp,0x0f
+
+	lvx		$ivec,0,$ivp		# load [unaligned] iv
+	lvsl		$inpperm,0,$ivp
+	lvx		$inptail,$idx,$ivp
+	le?vxor		$inpperm,$inpperm,$tmp
+	vperm		$ivec,$ivec,$inptail,$inpperm
+
+	neg		r11,$inp
+	?lvsl		$keyperm,0,$key		# prepare for unaligned key
+	lwz		$rounds,240($key)
+
+	lvsr		$inpperm,0,r11		# prepare for unaligned load
+	lvx		$inptail,0,$inp
+	addi		$inp,$inp,15		# 15 is not typo
+	le?vxor		$inpperm,$inpperm,$tmp
+
+	?lvsr		$outperm,0,$out		# prepare for unaligned store
+	vspltisb	$outmask,-1
+	lvx		$outhead,0,$out
+	?vperm		$outmask,$rndkey0,$outmask,$outperm
+	le?vxor		$outperm,$outperm,$tmp
+
+	srwi		$rounds,$rounds,1
+	li		$idx,16
+	subi		$rounds,$rounds,1
+	beq		Lcbc_dec
+
+Lcbc_enc:
+	vmr		$inout,$inptail
+	lvx		$inptail,0,$inp
+	addi		$inp,$inp,16
+	mtctr		$rounds
+	subi		$len,$len,16		# len-=16
+
+	lvx		$rndkey0,0,$key
+	 vperm		$inout,$inout,$inptail,$inpperm
+	lvx		$rndkey1,$idx,$key
+	addi		$idx,$idx,16
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vxor		$inout,$inout,$rndkey0
+	lvx		$rndkey0,$idx,$key
+	addi		$idx,$idx,16
+	vxor		$inout,$inout,$ivec
+
+Loop_cbc_enc:
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
+	vcipher		$inout,$inout,$rndkey1
+	lvx		$rndkey1,$idx,$key
+	addi		$idx,$idx,16
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vcipher		$inout,$inout,$rndkey0
+	lvx		$rndkey0,$idx,$key
+	addi		$idx,$idx,16
+	bdnz		Loop_cbc_enc
+
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
+	vcipher		$inout,$inout,$rndkey1
+	lvx		$rndkey1,$idx,$key
+	li		$idx,16
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vcipherlast	$ivec,$inout,$rndkey0
+	${UCMP}i	$len,16
+
+	vperm		$tmp,$ivec,$ivec,$outperm
+	vsel		$inout,$outhead,$tmp,$outmask
+	vmr		$outhead,$tmp
+	stvx		$inout,0,$out
+	addi		$out,$out,16
+	bge		Lcbc_enc
+
+	b		Lcbc_done
+
+.align	4
+Lcbc_dec:
+	${UCMP}i	$len,128
+	bge		_aesp8_cbc_decrypt8x
+	vmr		$tmp,$inptail
+	lvx		$inptail,0,$inp
+	addi		$inp,$inp,16
+	mtctr		$rounds
+	subi		$len,$len,16		# len-=16
+
+	lvx		$rndkey0,0,$key
+	 vperm		$tmp,$tmp,$inptail,$inpperm
+	lvx		$rndkey1,$idx,$key
+	addi		$idx,$idx,16
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vxor		$inout,$tmp,$rndkey0
+	lvx		$rndkey0,$idx,$key
+	addi		$idx,$idx,16
+
+Loop_cbc_dec:
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
+	vncipher	$inout,$inout,$rndkey1
+	lvx		$rndkey1,$idx,$key
+	addi		$idx,$idx,16
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vncipher	$inout,$inout,$rndkey0
+	lvx		$rndkey0,$idx,$key
+	addi		$idx,$idx,16
+	bdnz		Loop_cbc_dec
+
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
+	vncipher	$inout,$inout,$rndkey1
+	lvx		$rndkey1,$idx,$key
+	li		$idx,16
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vncipherlast	$inout,$inout,$rndkey0
+	${UCMP}i	$len,16
+
+	vxor		$inout,$inout,$ivec
+	vmr		$ivec,$tmp
+	vperm		$tmp,$inout,$inout,$outperm
+	vsel		$inout,$outhead,$tmp,$outmask
+	vmr		$outhead,$tmp
+	stvx		$inout,0,$out
+	addi		$out,$out,16
+	bge		Lcbc_dec
+
+Lcbc_done:
+	addi		$out,$out,-1
+	lvx		$inout,0,$out		# redundant in aligned case
+	vsel		$inout,$outhead,$inout,$outmask
+	stvx		$inout,0,$out
+
+	neg		$enc,$ivp		# write [unaligned] iv
+	li		$idx,15			# 15 is not typo
+	vxor		$rndkey0,$rndkey0,$rndkey0
+	vspltisb	$outmask,-1
+	le?vspltisb	$tmp,0x0f
+	?lvsl		$outperm,0,$enc
+	?vperm		$outmask,$rndkey0,$outmask,$outperm
+	le?vxor		$outperm,$outperm,$tmp
+	lvx		$outhead,0,$ivp
+	vperm		$ivec,$ivec,$ivec,$outperm
+	vsel		$inout,$outhead,$ivec,$outmask
+	lvx		$inptail,$idx,$ivp
+	stvx		$inout,0,$ivp
+	vsel		$inout,$ivec,$inptail,$outmask
+	stvx		$inout,$idx,$ivp
+
+	mtspr		256,$vrsave
+	blr
+	.long		0
+	.byte		0,12,0x14,0,0,0,6,0
+	.long		0
+___
+#########################################################################
+{{	# Optimized CBC decrypt procedure				#
+my $key_="r11";
+my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
+my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10..13));
+my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(14..21));
+my $rndkey0="v23";	# v24-v25 rotating buffer for first found keys
+			# v26-v31 last 6 round keys
+my ($tmp,$keyperm)=($in3,$in4);	# aliases with "caller", redundant assignment
+
+$code.=<<___;
+.align	5
+_aesp8_cbc_decrypt8x:
+	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
+	li		r10,`$FRAME+8*16+15`
+	li		r11,`$FRAME+8*16+31`
+	stvx		v20,r10,$sp		# ABI says so
+	addi		r10,r10,32
+	stvx		v21,r11,$sp
+	addi		r11,r11,32
+	stvx		v22,r10,$sp
+	addi		r10,r10,32
+	stvx		v23,r11,$sp
+	addi		r11,r11,32
+	stvx		v24,r10,$sp
+	addi		r10,r10,32
+	stvx		v25,r11,$sp
+	addi		r11,r11,32
+	stvx		v26,r10,$sp
+	addi		r10,r10,32
+	stvx		v27,r11,$sp
+	addi		r11,r11,32
+	stvx		v28,r10,$sp
+	addi		r10,r10,32
+	stvx		v29,r11,$sp
+	addi		r11,r11,32
+	stvx		v30,r10,$sp
+	stvx		v31,r11,$sp
+	li		r0,-1
+	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
+	li		$x10,0x10
+	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
+	li		$x20,0x20
+	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
+	li		$x30,0x30
+	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
+	li		$x40,0x40
+	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
+	li		$x50,0x50
+	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
+	li		$x60,0x60
+	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
+	li		$x70,0x70
+	mtspr		256,r0
+
+	subi		$rounds,$rounds,3	# -4 in total
+	subi		$len,$len,128		# bias
+
+	lvx		$rndkey0,$x00,$key	# load key schedule
+	lvx		v30,$x10,$key
+	addi		$key,$key,0x20
+	lvx		v31,$x00,$key
+	?vperm		$rndkey0,$rndkey0,v30,$keyperm
+	addi		$key_,$sp,$FRAME+15
+	mtctr		$rounds
+
+Load_cbc_dec_key:
+	?vperm		v24,v30,v31,$keyperm
+	lvx		v30,$x10,$key
+	addi		$key,$key,0x20
+	stvx		v24,$x00,$key_		# off-load round[1]
+	?vperm		v25,v31,v30,$keyperm
+	lvx		v31,$x00,$key
+	stvx		v25,$x10,$key_		# off-load round[2]
+	addi		$key_,$key_,0x20
+	bdnz		Load_cbc_dec_key
+
+	lvx		v26,$x10,$key
+	?vperm		v24,v30,v31,$keyperm
+	lvx		v27,$x20,$key
+	stvx		v24,$x00,$key_		# off-load round[3]
+	?vperm		v25,v31,v26,$keyperm
+	lvx		v28,$x30,$key
+	stvx		v25,$x10,$key_		# off-load round[4]
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
+	?vperm		v26,v26,v27,$keyperm
+	lvx		v29,$x40,$key
+	?vperm		v27,v27,v28,$keyperm
+	lvx		v30,$x50,$key
+	?vperm		v28,v28,v29,$keyperm
+	lvx		v31,$x60,$key
+	?vperm		v29,v29,v30,$keyperm
+	lvx		$out0,$x70,$key		# borrow $out0
+	?vperm		v30,v30,v31,$keyperm
+	lvx		v24,$x00,$key_		# pre-load round[1]
+	?vperm		v31,v31,$out0,$keyperm
+	lvx		v25,$x10,$key_		# pre-load round[2]
+
+	#lvx		$inptail,0,$inp		# "caller" already did this
+	#addi		$inp,$inp,15		# 15 is not typo
+	subi		$inp,$inp,15		# undo "caller"
+
+	 le?li		$idx,8
+	lvx_u		$in0,$x00,$inp		# load first 8 "words"
+	 le?lvsl	$inpperm,0,$idx
+	 le?vspltisb	$tmp,0x0f
+	lvx_u		$in1,$x10,$inp
+	 le?vxor	$inpperm,$inpperm,$tmp	# transform for lvx_u/stvx_u
+	lvx_u		$in2,$x20,$inp
+	 le?vperm	$in0,$in0,$in0,$inpperm
+	lvx_u		$in3,$x30,$inp
+	 le?vperm	$in1,$in1,$in1,$inpperm
+	lvx_u		$in4,$x40,$inp
+	 le?vperm	$in2,$in2,$in2,$inpperm
+	vxor		$out0,$in0,$rndkey0
+	lvx_u		$in5,$x50,$inp
+	 le?vperm	$in3,$in3,$in3,$inpperm
+	vxor		$out1,$in1,$rndkey0
+	lvx_u		$in6,$x60,$inp
+	 le?vperm	$in4,$in4,$in4,$inpperm
+	vxor		$out2,$in2,$rndkey0
+	lvx_u		$in7,$x70,$inp
+	addi		$inp,$inp,0x80
+	 le?vperm	$in5,$in5,$in5,$inpperm
+	vxor		$out3,$in3,$rndkey0
+	 le?vperm	$in6,$in6,$in6,$inpperm
+	vxor		$out4,$in4,$rndkey0
+	 le?vperm	$in7,$in7,$in7,$inpperm
+	vxor		$out5,$in5,$rndkey0
+	vxor		$out6,$in6,$rndkey0
+	vxor		$out7,$in7,$rndkey0
+
+	mtctr		$rounds
+	b		Loop_cbc_dec8x
+.align	5
+Loop_cbc_dec8x:
+	vncipher	$out0,$out0,v24
+	vncipher	$out1,$out1,v24
+	vncipher	$out2,$out2,v24
+	vncipher	$out3,$out3,v24
+	vncipher	$out4,$out4,v24
+	vncipher	$out5,$out5,v24
+	vncipher	$out6,$out6,v24
+	vncipher	$out7,$out7,v24
+	lvx		v24,$x20,$key_		# round[3]
+	addi		$key_,$key_,0x20
+
+	vncipher	$out0,$out0,v25
+	vncipher	$out1,$out1,v25
+	vncipher	$out2,$out2,v25
+	vncipher	$out3,$out3,v25
+	vncipher	$out4,$out4,v25
+	vncipher	$out5,$out5,v25
+	vncipher	$out6,$out6,v25
+	vncipher	$out7,$out7,v25
+	lvx		v25,$x10,$key_		# round[4]
+	bdnz		Loop_cbc_dec8x
+
+	subic		$len,$len,128		# $len-=128
+	vncipher	$out0,$out0,v24
+	vncipher	$out1,$out1,v24
+	vncipher	$out2,$out2,v24
+	vncipher	$out3,$out3,v24
+	vncipher	$out4,$out4,v24
+	vncipher	$out5,$out5,v24
+	vncipher	$out6,$out6,v24
+	vncipher	$out7,$out7,v24
+
+	subfe.		r0,r0,r0		# borrow?-1:0
+	vncipher	$out0,$out0,v25
+	vncipher	$out1,$out1,v25
+	vncipher	$out2,$out2,v25
+	vncipher	$out3,$out3,v25
+	vncipher	$out4,$out4,v25
+	vncipher	$out5,$out5,v25
+	vncipher	$out6,$out6,v25
+	vncipher	$out7,$out7,v25
+
+	and		r0,r0,$len
+	vncipher	$out0,$out0,v26
+	vncipher	$out1,$out1,v26
+	vncipher	$out2,$out2,v26
+	vncipher	$out3,$out3,v26
+	vncipher	$out4,$out4,v26
+	vncipher	$out5,$out5,v26
+	vncipher	$out6,$out6,v26
+	vncipher	$out7,$out7,v26
+
+	add		$inp,$inp,r0		# $inp is adjusted in such
+						# way that at exit from the
+						# loop inX-in7 are loaded
+						# with last "words"
+	vncipher	$out0,$out0,v27
+	vncipher	$out1,$out1,v27
+	vncipher	$out2,$out2,v27
+	vncipher	$out3,$out3,v27
+	vncipher	$out4,$out4,v27
+	vncipher	$out5,$out5,v27
+	vncipher	$out6,$out6,v27
+	vncipher	$out7,$out7,v27
+
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
+	vncipher	$out0,$out0,v28
+	vncipher	$out1,$out1,v28
+	vncipher	$out2,$out2,v28
+	vncipher	$out3,$out3,v28
+	vncipher	$out4,$out4,v28
+	vncipher	$out5,$out5,v28
+	vncipher	$out6,$out6,v28
+	vncipher	$out7,$out7,v28
+	lvx		v24,$x00,$key_		# re-pre-load round[1]
+
+	vncipher	$out0,$out0,v29
+	vncipher	$out1,$out1,v29
+	vncipher	$out2,$out2,v29
+	vncipher	$out3,$out3,v29
+	vncipher	$out4,$out4,v29
+	vncipher	$out5,$out5,v29
+	vncipher	$out6,$out6,v29
+	vncipher	$out7,$out7,v29
+	lvx		v25,$x10,$key_		# re-pre-load round[2]
+
+	vncipher	$out0,$out0,v30
+	 vxor		$ivec,$ivec,v31		# xor with last round key
+	vncipher	$out1,$out1,v30
+	 vxor		$in0,$in0,v31
+	vncipher	$out2,$out2,v30
+	 vxor		$in1,$in1,v31
+	vncipher	$out3,$out3,v30
+	 vxor		$in2,$in2,v31
+	vncipher	$out4,$out4,v30
+	 vxor		$in3,$in3,v31
+	vncipher	$out5,$out5,v30
+	 vxor		$in4,$in4,v31
+	vncipher	$out6,$out6,v30
+	 vxor		$in5,$in5,v31
+	vncipher	$out7,$out7,v30
+	 vxor		$in6,$in6,v31
+
+	vncipherlast	$out0,$out0,$ivec
+	vncipherlast	$out1,$out1,$in0
+	 lvx_u		$in0,$x00,$inp		# load next input block
+	vncipherlast	$out2,$out2,$in1
+	 lvx_u		$in1,$x10,$inp
+	vncipherlast	$out3,$out3,$in2
+	 le?vperm	$in0,$in0,$in0,$inpperm
+	 lvx_u		$in2,$x20,$inp
+	vncipherlast	$out4,$out4,$in3
+	 le?vperm	$in1,$in1,$in1,$inpperm
+	 lvx_u		$in3,$x30,$inp
+	vncipherlast	$out5,$out5,$in4
+	 le?vperm	$in2,$in2,$in2,$inpperm
+	 lvx_u		$in4,$x40,$inp
+	vncipherlast	$out6,$out6,$in5
+	 le?vperm	$in3,$in3,$in3,$inpperm
+	 lvx_u		$in5,$x50,$inp
+	vncipherlast	$out7,$out7,$in6
+	 le?vperm	$in4,$in4,$in4,$inpperm
+	 lvx_u		$in6,$x60,$inp
+	vmr		$ivec,$in7
+	 le?vperm	$in5,$in5,$in5,$inpperm
+	 lvx_u		$in7,$x70,$inp
+	 addi		$inp,$inp,0x80
+
+	le?vperm	$out0,$out0,$out0,$inpperm
+	le?vperm	$out1,$out1,$out1,$inpperm
+	stvx_u		$out0,$x00,$out
+	 le?vperm	$in6,$in6,$in6,$inpperm
+	 vxor		$out0,$in0,$rndkey0
+	le?vperm	$out2,$out2,$out2,$inpperm
+	stvx_u		$out1,$x10,$out
+	 le?vperm	$in7,$in7,$in7,$inpperm
+	 vxor		$out1,$in1,$rndkey0
+	le?vperm	$out3,$out3,$out3,$inpperm
+	stvx_u		$out2,$x20,$out
+	 vxor		$out2,$in2,$rndkey0
+	le?vperm	$out4,$out4,$out4,$inpperm
+	stvx_u		$out3,$x30,$out
+	 vxor		$out3,$in3,$rndkey0
+	le?vperm	$out5,$out5,$out5,$inpperm
+	stvx_u		$out4,$x40,$out
+	 vxor		$out4,$in4,$rndkey0
+	le?vperm	$out6,$out6,$out6,$inpperm
+	stvx_u		$out5,$x50,$out
+	 vxor		$out5,$in5,$rndkey0
+	le?vperm	$out7,$out7,$out7,$inpperm
+	stvx_u		$out6,$x60,$out
+	 vxor		$out6,$in6,$rndkey0
+	stvx_u		$out7,$x70,$out
+	addi		$out,$out,0x80
+	 vxor		$out7,$in7,$rndkey0
+
+	mtctr		$rounds
+	beq		Loop_cbc_dec8x		# did $len-=128 borrow?
+
+	addic.		$len,$len,128
+	beq		Lcbc_dec8x_done
+	nop
+	nop
+
+Loop_cbc_dec8x_tail:				# up to 7 "words" tail...
+	vncipher	$out1,$out1,v24
+	vncipher	$out2,$out2,v24
+	vncipher	$out3,$out3,v24
+	vncipher	$out4,$out4,v24
+	vncipher	$out5,$out5,v24
+	vncipher	$out6,$out6,v24
+	vncipher	$out7,$out7,v24
+	lvx		v24,$x20,$key_		# round[3]
+	addi		$key_,$key_,0x20
+
+	vncipher	$out1,$out1,v25
+	vncipher	$out2,$out2,v25
+	vncipher	$out3,$out3,v25
+	vncipher	$out4,$out4,v25
+	vncipher	$out5,$out5,v25
+	vncipher	$out6,$out6,v25
+	vncipher	$out7,$out7,v25
+	lvx		v25,$x10,$key_		# round[4]
+	bdnz		Loop_cbc_dec8x_tail
+
+	vncipher	$out1,$out1,v24
+	vncipher	$out2,$out2,v24
+	vncipher	$out3,$out3,v24
+	vncipher	$out4,$out4,v24
+	vncipher	$out5,$out5,v24
+	vncipher	$out6,$out6,v24
+	vncipher	$out7,$out7,v24
+
+	vncipher	$out1,$out1,v25
+	vncipher	$out2,$out2,v25
+	vncipher	$out3,$out3,v25
+	vncipher	$out4,$out4,v25
+	vncipher	$out5,$out5,v25
+	vncipher	$out6,$out6,v25
+	vncipher	$out7,$out7,v25
+
+	vncipher	$out1,$out1,v26
+	vncipher	$out2,$out2,v26
+	vncipher	$out3,$out3,v26
+	vncipher	$out4,$out4,v26
+	vncipher	$out5,$out5,v26
+	vncipher	$out6,$out6,v26
+	vncipher	$out7,$out7,v26
+
+	vncipher	$out1,$out1,v27
+	vncipher	$out2,$out2,v27
+	vncipher	$out3,$out3,v27
+	vncipher	$out4,$out4,v27
+	vncipher	$out5,$out5,v27
+	vncipher	$out6,$out6,v27
+	vncipher	$out7,$out7,v27
+
+	vncipher	$out1,$out1,v28
+	vncipher	$out2,$out2,v28
+	vncipher	$out3,$out3,v28
+	vncipher	$out4,$out4,v28
+	vncipher	$out5,$out5,v28
+	vncipher	$out6,$out6,v28
+	vncipher	$out7,$out7,v28
+
+	vncipher	$out1,$out1,v29
+	vncipher	$out2,$out2,v29
+	vncipher	$out3,$out3,v29
+	vncipher	$out4,$out4,v29
+	vncipher	$out5,$out5,v29
+	vncipher	$out6,$out6,v29
+	vncipher	$out7,$out7,v29
+
+	vncipher	$out1,$out1,v30
+	 vxor		$ivec,$ivec,v31		# last round key
+	vncipher	$out2,$out2,v30
+	 vxor		$in1,$in1,v31
+	vncipher	$out3,$out3,v30
+	 vxor		$in2,$in2,v31
+	vncipher	$out4,$out4,v30
+	 vxor		$in3,$in3,v31
+	vncipher	$out5,$out5,v30
+	 vxor		$in4,$in4,v31
+	vncipher	$out6,$out6,v30
+	 vxor		$in5,$in5,v31
+	vncipher	$out7,$out7,v30
+	 vxor		$in6,$in6,v31
+
+	cmplwi		$len,32			# switch($len)
+	blt		Lcbc_dec8x_one
+	nop
+	beq		Lcbc_dec8x_two
+	cmplwi		$len,64
+	blt		Lcbc_dec8x_three
+	nop
+	beq		Lcbc_dec8x_four
+	cmplwi		$len,96
+	blt		Lcbc_dec8x_five
+	nop
+	beq		Lcbc_dec8x_six
+
+Lcbc_dec8x_seven:
+	vncipherlast	$out1,$out1,$ivec
+	vncipherlast	$out2,$out2,$in1
+	vncipherlast	$out3,$out3,$in2
+	vncipherlast	$out4,$out4,$in3
+	vncipherlast	$out5,$out5,$in4
+	vncipherlast	$out6,$out6,$in5
+	vncipherlast	$out7,$out7,$in6
+	vmr		$ivec,$in7
+
+	le?vperm	$out1,$out1,$out1,$inpperm
+	le?vperm	$out2,$out2,$out2,$inpperm
+	stvx_u		$out1,$x00,$out
+	le?vperm	$out3,$out3,$out3,$inpperm
+	stvx_u		$out2,$x10,$out
+	le?vperm	$out4,$out4,$out4,$inpperm
+	stvx_u		$out3,$x20,$out
+	le?vperm	$out5,$out5,$out5,$inpperm
+	stvx_u		$out4,$x30,$out
+	le?vperm	$out6,$out6,$out6,$inpperm
+	stvx_u		$out5,$x40,$out
+	le?vperm	$out7,$out7,$out7,$inpperm
+	stvx_u		$out6,$x50,$out
+	stvx_u		$out7,$x60,$out
+	addi		$out,$out,0x70
+	b		Lcbc_dec8x_done
+
+.align	5
+Lcbc_dec8x_six:
+	vncipherlast	$out2,$out2,$ivec
+	vncipherlast	$out3,$out3,$in2
+	vncipherlast	$out4,$out4,$in3
+	vncipherlast	$out5,$out5,$in4
+	vncipherlast	$out6,$out6,$in5
+	vncipherlast	$out7,$out7,$in6
+	vmr		$ivec,$in7
+
+	le?vperm	$out2,$out2,$out2,$inpperm
+	le?vperm	$out3,$out3,$out3,$inpperm
+	stvx_u		$out2,$x00,$out
+	le?vperm	$out4,$out4,$out4,$inpperm
+	stvx_u		$out3,$x10,$out
+	le?vperm	$out5,$out5,$out5,$inpperm
+	stvx_u		$out4,$x20,$out
+	le?vperm	$out6,$out6,$out6,$inpperm
+	stvx_u		$out5,$x30,$out
+	le?vperm	$out7,$out7,$out7,$inpperm
+	stvx_u		$out6,$x40,$out
+	stvx_u		$out7,$x50,$out
+	addi		$out,$out,0x60
+	b		Lcbc_dec8x_done
+
+.align	5
+Lcbc_dec8x_five:
+	vncipherlast	$out3,$out3,$ivec
+	vncipherlast	$out4,$out4,$in3
+	vncipherlast	$out5,$out5,$in4
+	vncipherlast	$out6,$out6,$in5
+	vncipherlast	$out7,$out7,$in6
+	vmr		$ivec,$in7
+
+	le?vperm	$out3,$out3,$out3,$inpperm
+	le?vperm	$out4,$out4,$out4,$inpperm
+	stvx_u		$out3,$x00,$out
+	le?vperm	$out5,$out5,$out5,$inpperm
+	stvx_u		$out4,$x10,$out
+	le?vperm	$out6,$out6,$out6,$inpperm
+	stvx_u		$out5,$x20,$out
+	le?vperm	$out7,$out7,$out7,$inpperm
+	stvx_u		$out6,$x30,$out
+	stvx_u		$out7,$x40,$out
+	addi		$out,$out,0x50
+	b		Lcbc_dec8x_done
+
+.align	5
+Lcbc_dec8x_four:
+	vncipherlast	$out4,$out4,$ivec
+	vncipherlast	$out5,$out5,$in4
+	vncipherlast	$out6,$out6,$in5
+	vncipherlast	$out7,$out7,$in6
+	vmr		$ivec,$in7
+
+	le?vperm	$out4,$out4,$out4,$inpperm
+	le?vperm	$out5,$out5,$out5,$inpperm
+	stvx_u		$out4,$x00,$out
+	le?vperm	$out6,$out6,$out6,$inpperm
+	stvx_u		$out5,$x10,$out
+	le?vperm	$out7,$out7,$out7,$inpperm
+	stvx_u		$out6,$x20,$out
+	stvx_u		$out7,$x30,$out
+	addi		$out,$out,0x40
+	b		Lcbc_dec8x_done
+
+.align	5
+Lcbc_dec8x_three:
+	vncipherlast	$out5,$out5,$ivec
+	vncipherlast	$out6,$out6,$in5
+	vncipherlast	$out7,$out7,$in6
+	vmr		$ivec,$in7
+
+	le?vperm	$out5,$out5,$out5,$inpperm
+	le?vperm	$out6,$out6,$out6,$inpperm
+	stvx_u		$out5,$x00,$out
+	le?vperm	$out7,$out7,$out7,$inpperm
+	stvx_u		$out6,$x10,$out
+	stvx_u		$out7,$x20,$out
+	addi		$out,$out,0x30
+	b		Lcbc_dec8x_done
+
+.align	5
+Lcbc_dec8x_two:
+	vncipherlast	$out6,$out6,$ivec
+	vncipherlast	$out7,$out7,$in6
+	vmr		$ivec,$in7
+
+	le?vperm	$out6,$out6,$out6,$inpperm
+	le?vperm	$out7,$out7,$out7,$inpperm
+	stvx_u		$out6,$x00,$out
+	stvx_u		$out7,$x10,$out
+	addi		$out,$out,0x20
+	b		Lcbc_dec8x_done
+
+.align	5
+Lcbc_dec8x_one:
+	vncipherlast	$out7,$out7,$ivec
+	vmr		$ivec,$in7
+
+	le?vperm	$out7,$out7,$out7,$inpperm
+	stvx_u		$out7,0,$out
+	addi		$out,$out,0x10
+
+Lcbc_dec8x_done:
+	le?vperm	$ivec,$ivec,$ivec,$inpperm
+	stvx_u		$ivec,0,$ivp		# write [unaligned] iv
+
+	li		r10,`$FRAME+15`
+	li		r11,`$FRAME+31`
+	stvx		$inpperm,r10,$sp	# wipe copies of round keys
+	addi		r10,r10,32
+	stvx		$inpperm,r11,$sp
+	addi		r11,r11,32
+	stvx		$inpperm,r10,$sp
+	addi		r10,r10,32
+	stvx		$inpperm,r11,$sp
+	addi		r11,r11,32
+	stvx		$inpperm,r10,$sp
+	addi		r10,r10,32
+	stvx		$inpperm,r11,$sp
+	addi		r11,r11,32
+	stvx		$inpperm,r10,$sp
+	addi		r10,r10,32
+	stvx		$inpperm,r11,$sp
+	addi		r11,r11,32
+
+	mtspr		256,$vrsave
+	lvx		v20,r10,$sp		# ABI says so
+	addi		r10,r10,32
+	lvx		v21,r11,$sp
+	addi		r11,r11,32
+	lvx		v22,r10,$sp
+	addi		r10,r10,32
+	lvx		v23,r11,$sp
+	addi		r11,r11,32
+	lvx		v24,r10,$sp
+	addi		r10,r10,32
+	lvx		v25,r11,$sp
+	addi		r11,r11,32
+	lvx		v26,r10,$sp
+	addi		r10,r10,32
+	lvx		v27,r11,$sp
+	addi		r11,r11,32
+	lvx		v28,r10,$sp
+	addi		r10,r10,32
+	lvx		v29,r11,$sp
+	addi		r11,r11,32
+	lvx		v30,r10,$sp
+	lvx		v31,r11,$sp
+	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
+	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
+	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
+	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
+	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
+	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
+	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`
+	blr
+	.long		0
+	.byte		0,12,0x14,0,0x80,6,6,0
+	.long		0
+.size	.${prefix}_cbc_encrypt,.-.${prefix}_cbc_encrypt
+___
+}}	}}}
+
+#########################################################################
+{{{	# CTR procedure[s]						#
+
+####################### WARNING: Here be dragons! #######################
+#
+# This code is written as 'ctr32', based on a 32-bit counter used
+# upstream. The kernel does *not* use a 32-bit counter. The kernel uses
+# a 128-bit counter.
+#
+# This leads to subtle changes from the upstream code: the counter
+# is incremented with vaddu_q_m rather than vaddu_w_m. This occurs in
+# both the bulk (8 blocks at a time) path, and in the individual block
+# path. Be aware of this when doing updates.
+#
+# See:
+# 1d4aa0b4c181 ("crypto: vmx - Fixing AES-CTR counter bug")
+# 009b30ac7444 ("crypto: vmx - CTR: always increment IV as quadword")
+# https://github.com/openssl/openssl/pull/8942
+#
+#########################################################################
+my ($inp,$out,$len,$key,$ivp,$x10,$rounds,$idx)=map("r$_",(3..10));
+my ($rndkey0,$rndkey1,$inout,$tmp)=		map("v$_",(0..3));
+my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm,$one)=
+						map("v$_",(4..11));
+my $dat=$tmp;
+
+$code.=<<___;
+.globl	.${prefix}_ctr32_encrypt_blocks
+	${UCMP}i	$len,1
+	bltlr-
+
+	lis		r0,0xfff0
+	mfspr		$vrsave,256
+	mtspr		256,r0
+
+	li		$idx,15
+	vxor		$rndkey0,$rndkey0,$rndkey0
+	le?vspltisb	$tmp,0x0f
+
+	lvx		$ivec,0,$ivp		# load [unaligned] iv
+	lvsl		$inpperm,0,$ivp
+	lvx		$inptail,$idx,$ivp
+	 vspltisb	$one,1
+	le?vxor		$inpperm,$inpperm,$tmp
+	vperm		$ivec,$ivec,$inptail,$inpperm
+	 vsldoi		$one,$rndkey0,$one,1
+
+	neg		r11,$inp
+	?lvsl		$keyperm,0,$key		# prepare for unaligned key
+	lwz		$rounds,240($key)
+
+	lvsr		$inpperm,0,r11		# prepare for unaligned load
+	lvx		$inptail,0,$inp
+	addi		$inp,$inp,15		# 15 is not typo
+	le?vxor		$inpperm,$inpperm,$tmp
+
+	srwi		$rounds,$rounds,1
+	li		$idx,16
+	subi		$rounds,$rounds,1
+
+	${UCMP}i	$len,8
+	bge		_aesp8_ctr32_encrypt8x
+
+	?lvsr		$outperm,0,$out		# prepare for unaligned store
+	vspltisb	$outmask,-1
+	lvx		$outhead,0,$out
+	?vperm		$outmask,$rndkey0,$outmask,$outperm
+	le?vxor		$outperm,$outperm,$tmp
+
+	lvx		$rndkey0,0,$key
+	mtctr		$rounds
+	lvx		$rndkey1,$idx,$key
+	addi		$idx,$idx,16
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vxor		$inout,$ivec,$rndkey0
+	lvx		$rndkey0,$idx,$key
+	addi		$idx,$idx,16
+	b		Loop_ctr32_enc
+
+.align	5
+Loop_ctr32_enc:
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
+	vcipher		$inout,$inout,$rndkey1
+	lvx		$rndkey1,$idx,$key
+	addi		$idx,$idx,16
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vcipher		$inout,$inout,$rndkey0
+	lvx		$rndkey0,$idx,$key
+	addi		$idx,$idx,16
+	bdnz		Loop_ctr32_enc
+
+	vadduqm		$ivec,$ivec,$one	# Kernel change for 128-bit
+	 vmr		$dat,$inptail
+	 lvx		$inptail,0,$inp
+	 addi		$inp,$inp,16
+	 subic.		$len,$len,1		# blocks--
+
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
+	vcipher		$inout,$inout,$rndkey1
+	lvx		$rndkey1,$idx,$key
+	 vperm		$dat,$dat,$inptail,$inpperm
+	 li		$idx,16
+	?vperm		$rndkey1,$rndkey0,$rndkey1,$keyperm
+	 lvx		$rndkey0,0,$key
+	vxor		$dat,$dat,$rndkey1	# last round key
+	vcipherlast	$inout,$inout,$dat
+
+	 lvx		$rndkey1,$idx,$key
+	 addi		$idx,$idx,16
+	vperm		$inout,$inout,$inout,$outperm
+	vsel		$dat,$outhead,$inout,$outmask
+	 mtctr		$rounds
+	 ?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vmr		$outhead,$inout
+	 vxor		$inout,$ivec,$rndkey0
+	 lvx		$rndkey0,$idx,$key
+	 addi		$idx,$idx,16
+	stvx		$dat,0,$out
+	addi		$out,$out,16
+	bne		Loop_ctr32_enc
+
+	addi		$out,$out,-1
+	lvx		$inout,0,$out		# redundant in aligned case
+	vsel		$inout,$outhead,$inout,$outmask
+	stvx		$inout,0,$out
+
+	mtspr		256,$vrsave
+	blr
+	.long		0
+	.byte		0,12,0x14,0,0,0,6,0
+	.long		0
+___
+#########################################################################
+{{	# Optimized CTR procedure					#
+my $key_="r11";
+my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
+my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10,12..14));
+my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(15..22));
+my $rndkey0="v23";	# v24-v25 rotating buffer for first found keys
+			# v26-v31 last 6 round keys
+my ($tmp,$keyperm)=($in3,$in4);	# aliases with "caller", redundant assignment
+my ($two,$three,$four)=($outhead,$outperm,$outmask);
+
+$code.=<<___;
+.align	5
+_aesp8_ctr32_encrypt8x:
+	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
+	li		r10,`$FRAME+8*16+15`
+	li		r11,`$FRAME+8*16+31`
+	stvx		v20,r10,$sp		# ABI says so
+	addi		r10,r10,32
+	stvx		v21,r11,$sp
+	addi		r11,r11,32
+	stvx		v22,r10,$sp
+	addi		r10,r10,32
+	stvx		v23,r11,$sp
+	addi		r11,r11,32
+	stvx		v24,r10,$sp
+	addi		r10,r10,32
+	stvx		v25,r11,$sp
+	addi		r11,r11,32
+	stvx		v26,r10,$sp
+	addi		r10,r10,32
+	stvx		v27,r11,$sp
+	addi		r11,r11,32
+	stvx		v28,r10,$sp
+	addi		r10,r10,32
+	stvx		v29,r11,$sp
+	addi		r11,r11,32
+	stvx		v30,r10,$sp
+	stvx		v31,r11,$sp
+	li		r0,-1
+	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
+	li		$x10,0x10
+	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
+	li		$x20,0x20
+	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
+	li		$x30,0x30
+	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
+	li		$x40,0x40
+	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
+	li		$x50,0x50
+	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
+	li		$x60,0x60
+	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
+	li		$x70,0x70
+	mtspr		256,r0
+
+	subi		$rounds,$rounds,3	# -4 in total
+
+	lvx		$rndkey0,$x00,$key	# load key schedule
+	lvx		v30,$x10,$key
+	addi		$key,$key,0x20
+	lvx		v31,$x00,$key
+	?vperm		$rndkey0,$rndkey0,v30,$keyperm
+	addi		$key_,$sp,$FRAME+15
+	mtctr		$rounds
+
+Load_ctr32_enc_key:
+	?vperm		v24,v30,v31,$keyperm
+	lvx		v30,$x10,$key
+	addi		$key,$key,0x20
+	stvx		v24,$x00,$key_		# off-load round[1]
+	?vperm		v25,v31,v30,$keyperm
+	lvx		v31,$x00,$key
+	stvx		v25,$x10,$key_		# off-load round[2]
+	addi		$key_,$key_,0x20
+	bdnz		Load_ctr32_enc_key
+
+	lvx		v26,$x10,$key
+	?vperm		v24,v30,v31,$keyperm
+	lvx		v27,$x20,$key
+	stvx		v24,$x00,$key_		# off-load round[3]
+	?vperm		v25,v31,v26,$keyperm
+	lvx		v28,$x30,$key
+	stvx		v25,$x10,$key_		# off-load round[4]
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
+	?vperm		v26,v26,v27,$keyperm
+	lvx		v29,$x40,$key
+	?vperm		v27,v27,v28,$keyperm
+	lvx		v30,$x50,$key
+	?vperm		v28,v28,v29,$keyperm
+	lvx		v31,$x60,$key
+	?vperm		v29,v29,v30,$keyperm
+	lvx		$out0,$x70,$key		# borrow $out0
+	?vperm		v30,v30,v31,$keyperm
+	lvx		v24,$x00,$key_		# pre-load round[1]
+	?vperm		v31,v31,$out0,$keyperm
+	lvx		v25,$x10,$key_		# pre-load round[2]
+
+	vadduqm		$two,$one,$one
+	subi		$inp,$inp,15		# undo "caller"
+	$SHL		$len,$len,4
+
+	vadduqm		$out1,$ivec,$one	# counter values ...
+	vadduqm		$out2,$ivec,$two	# (do all ctr adds as 128-bit)
+	vxor		$out0,$ivec,$rndkey0	# ... xored with rndkey[0]
+	 le?li		$idx,8
+	vadduqm		$out3,$out1,$two
+	vxor		$out1,$out1,$rndkey0
+	 le?lvsl	$inpperm,0,$idx
+	vadduqm		$out4,$out2,$two
+	vxor		$out2,$out2,$rndkey0
+	 le?vspltisb	$tmp,0x0f
+	vadduqm		$out5,$out3,$two
+	vxor		$out3,$out3,$rndkey0
+	 le?vxor	$inpperm,$inpperm,$tmp	# transform for lvx_u/stvx_u
+	vadduqm		$out6,$out4,$two
+	vxor		$out4,$out4,$rndkey0
+	vadduqm		$out7,$out5,$two
+	vxor		$out5,$out5,$rndkey0
+	vadduqm		$ivec,$out6,$two	# next counter value
+	vxor		$out6,$out6,$rndkey0
+	vxor		$out7,$out7,$rndkey0
+
+	mtctr		$rounds
+	b		Loop_ctr32_enc8x
+.align	5
+Loop_ctr32_enc8x:
+	vcipher 	$out0,$out0,v24
+	vcipher 	$out1,$out1,v24
+	vcipher 	$out2,$out2,v24
+	vcipher 	$out3,$out3,v24
+	vcipher 	$out4,$out4,v24
+	vcipher 	$out5,$out5,v24
+	vcipher 	$out6,$out6,v24
+	vcipher 	$out7,$out7,v24
+Loop_ctr32_enc8x_middle:
+	lvx		v24,$x20,$key_		# round[3]
+	addi		$key_,$key_,0x20
+
+	vcipher 	$out0,$out0,v25
+	vcipher 	$out1,$out1,v25
+	vcipher 	$out2,$out2,v25
+	vcipher 	$out3,$out3,v25
+	vcipher 	$out4,$out4,v25
+	vcipher 	$out5,$out5,v25
+	vcipher 	$out6,$out6,v25
+	vcipher 	$out7,$out7,v25
+	lvx		v25,$x10,$key_		# round[4]
+	bdnz		Loop_ctr32_enc8x
+
+	subic		r11,$len,256		# $len-256, borrow $key_
+	vcipher 	$out0,$out0,v24
+	vcipher 	$out1,$out1,v24
+	vcipher 	$out2,$out2,v24
+	vcipher 	$out3,$out3,v24
+	vcipher 	$out4,$out4,v24
+	vcipher 	$out5,$out5,v24
+	vcipher 	$out6,$out6,v24
+	vcipher 	$out7,$out7,v24
+
+	subfe		r0,r0,r0		# borrow?-1:0
+	vcipher 	$out0,$out0,v25
+	vcipher 	$out1,$out1,v25
+	vcipher 	$out2,$out2,v25
+	vcipher 	$out3,$out3,v25
+	vcipher 	$out4,$out4,v25
+	vcipher		$out5,$out5,v25
+	vcipher		$out6,$out6,v25
+	vcipher		$out7,$out7,v25
+
+	and		r0,r0,r11
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
+	vcipher		$out0,$out0,v26
+	vcipher		$out1,$out1,v26
+	vcipher		$out2,$out2,v26
+	vcipher		$out3,$out3,v26
+	vcipher		$out4,$out4,v26
+	vcipher		$out5,$out5,v26
+	vcipher		$out6,$out6,v26
+	vcipher		$out7,$out7,v26
+	lvx		v24,$x00,$key_		# re-pre-load round[1]
+
+	subic		$len,$len,129		# $len-=129
+	vcipher		$out0,$out0,v27
+	addi		$len,$len,1		# $len-=128 really
+	vcipher		$out1,$out1,v27
+	vcipher		$out2,$out2,v27
+	vcipher		$out3,$out3,v27
+	vcipher		$out4,$out4,v27
+	vcipher		$out5,$out5,v27
+	vcipher		$out6,$out6,v27
+	vcipher		$out7,$out7,v27
+	lvx		v25,$x10,$key_		# re-pre-load round[2]
+
+	vcipher		$out0,$out0,v28
+	 lvx_u		$in0,$x00,$inp		# load input
+	vcipher		$out1,$out1,v28
+	 lvx_u		$in1,$x10,$inp
+	vcipher		$out2,$out2,v28
+	 lvx_u		$in2,$x20,$inp
+	vcipher		$out3,$out3,v28
+	 lvx_u		$in3,$x30,$inp
+	vcipher		$out4,$out4,v28
+	 lvx_u		$in4,$x40,$inp
+	vcipher		$out5,$out5,v28
+	 lvx_u		$in5,$x50,$inp
+	vcipher		$out6,$out6,v28
+	 lvx_u		$in6,$x60,$inp
+	vcipher		$out7,$out7,v28
+	 lvx_u		$in7,$x70,$inp
+	 addi		$inp,$inp,0x80
+
+	vcipher		$out0,$out0,v29
+	 le?vperm	$in0,$in0,$in0,$inpperm
+	vcipher		$out1,$out1,v29
+	 le?vperm	$in1,$in1,$in1,$inpperm
+	vcipher		$out2,$out2,v29
+	 le?vperm	$in2,$in2,$in2,$inpperm
+	vcipher		$out3,$out3,v29
+	 le?vperm	$in3,$in3,$in3,$inpperm
+	vcipher		$out4,$out4,v29
+	 le?vperm	$in4,$in4,$in4,$inpperm
+	vcipher		$out5,$out5,v29
+	 le?vperm	$in5,$in5,$in5,$inpperm
+	vcipher		$out6,$out6,v29
+	 le?vperm	$in6,$in6,$in6,$inpperm
+	vcipher		$out7,$out7,v29
+	 le?vperm	$in7,$in7,$in7,$inpperm
+
+	add		$inp,$inp,r0		# $inp is adjusted in such
+						# way that at exit from the
+						# loop inX-in7 are loaded
+						# with last "words"
+	subfe.		r0,r0,r0		# borrow?-1:0
+	vcipher		$out0,$out0,v30
+	 vxor		$in0,$in0,v31		# xor with last round key
+	vcipher		$out1,$out1,v30
+	 vxor		$in1,$in1,v31
+	vcipher		$out2,$out2,v30
+	 vxor		$in2,$in2,v31
+	vcipher		$out3,$out3,v30
+	 vxor		$in3,$in3,v31
+	vcipher		$out4,$out4,v30
+	 vxor		$in4,$in4,v31
+	vcipher		$out5,$out5,v30
+	 vxor		$in5,$in5,v31
+	vcipher		$out6,$out6,v30
+	 vxor		$in6,$in6,v31
+	vcipher		$out7,$out7,v30
+	 vxor		$in7,$in7,v31
+
+	bne		Lctr32_enc8x_break	# did $len-129 borrow?
+
+	vcipherlast	$in0,$out0,$in0
+	vcipherlast	$in1,$out1,$in1
+	 vadduqm	$out1,$ivec,$one	# counter values ...
+	vcipherlast	$in2,$out2,$in2
+	 vadduqm	$out2,$ivec,$two
+	 vxor		$out0,$ivec,$rndkey0	# ... xored with rndkey[0]
+	vcipherlast	$in3,$out3,$in3
+	 vadduqm	$out3,$out1,$two
+	 vxor		$out1,$out1,$rndkey0
+	vcipherlast	$in4,$out4,$in4
+	 vadduqm	$out4,$out2,$two
+	 vxor		$out2,$out2,$rndkey0
+	vcipherlast	$in5,$out5,$in5
+	 vadduqm	$out5,$out3,$two
+	 vxor		$out3,$out3,$rndkey0
+	vcipherlast	$in6,$out6,$in6
+	 vadduqm	$out6,$out4,$two
+	 vxor		$out4,$out4,$rndkey0
+	vcipherlast	$in7,$out7,$in7
+	 vadduqm	$out7,$out5,$two
+	 vxor		$out5,$out5,$rndkey0
+	le?vperm	$in0,$in0,$in0,$inpperm
+	 vadduqm	$ivec,$out6,$two	# next counter value
+	 vxor		$out6,$out6,$rndkey0
+	le?vperm	$in1,$in1,$in1,$inpperm
+	 vxor		$out7,$out7,$rndkey0
+	mtctr		$rounds
+
+	 vcipher	$out0,$out0,v24
+	stvx_u		$in0,$x00,$out
+	le?vperm	$in2,$in2,$in2,$inpperm
+	 vcipher	$out1,$out1,v24
+	stvx_u		$in1,$x10,$out
+	le?vperm	$in3,$in3,$in3,$inpperm
+	 vcipher	$out2,$out2,v24
+	stvx_u		$in2,$x20,$out
+	le?vperm	$in4,$in4,$in4,$inpperm
+	 vcipher	$out3,$out3,v24
+	stvx_u		$in3,$x30,$out
+	le?vperm	$in5,$in5,$in5,$inpperm
+	 vcipher	$out4,$out4,v24
+	stvx_u		$in4,$x40,$out
+	le?vperm	$in6,$in6,$in6,$inpperm
+	 vcipher	$out5,$out5,v24
+	stvx_u		$in5,$x50,$out
+	le?vperm	$in7,$in7,$in7,$inpperm
+	 vcipher	$out6,$out6,v24
+	stvx_u		$in6,$x60,$out
+	 vcipher	$out7,$out7,v24
+	stvx_u		$in7,$x70,$out
+	addi		$out,$out,0x80
+
+	b		Loop_ctr32_enc8x_middle
+
+.align	5
+Lctr32_enc8x_break:
+	cmpwi		$len,-0x60
+	blt		Lctr32_enc8x_one
+	nop
+	beq		Lctr32_enc8x_two
+	cmpwi		$len,-0x40
+	blt		Lctr32_enc8x_three
+	nop
+	beq		Lctr32_enc8x_four
+	cmpwi		$len,-0x20
+	blt		Lctr32_enc8x_five
+	nop
+	beq		Lctr32_enc8x_six
+	cmpwi		$len,0x00
+	blt		Lctr32_enc8x_seven
+
+Lctr32_enc8x_eight:
+	vcipherlast	$out0,$out0,$in0
+	vcipherlast	$out1,$out1,$in1
+	vcipherlast	$out2,$out2,$in2
+	vcipherlast	$out3,$out3,$in3
+	vcipherlast	$out4,$out4,$in4
+	vcipherlast	$out5,$out5,$in5
+	vcipherlast	$out6,$out6,$in6
+	vcipherlast	$out7,$out7,$in7
+
+	le?vperm	$out0,$out0,$out0,$inpperm
+	le?vperm	$out1,$out1,$out1,$inpperm
+	stvx_u		$out0,$x00,$out
+	le?vperm	$out2,$out2,$out2,$inpperm
+	stvx_u		$out1,$x10,$out
+	le?vperm	$out3,$out3,$out3,$inpperm
+	stvx_u		$out2,$x20,$out
+	le?vperm	$out4,$out4,$out4,$inpperm
+	stvx_u		$out3,$x30,$out
+	le?vperm	$out5,$out5,$out5,$inpperm
+	stvx_u		$out4,$x40,$out
+	le?vperm	$out6,$out6,$out6,$inpperm
+	stvx_u		$out5,$x50,$out
+	le?vperm	$out7,$out7,$out7,$inpperm
+	stvx_u		$out6,$x60,$out
+	stvx_u		$out7,$x70,$out
+	addi		$out,$out,0x80
+	b		Lctr32_enc8x_done
+
+.align	5
+Lctr32_enc8x_seven:
+	vcipherlast	$out0,$out0,$in1
+	vcipherlast	$out1,$out1,$in2
+	vcipherlast	$out2,$out2,$in3
+	vcipherlast	$out3,$out3,$in4
+	vcipherlast	$out4,$out4,$in5
+	vcipherlast	$out5,$out5,$in6
+	vcipherlast	$out6,$out6,$in7
+
+	le?vperm	$out0,$out0,$out0,$inpperm
+	le?vperm	$out1,$out1,$out1,$inpperm
+	stvx_u		$out0,$x00,$out
+	le?vperm	$out2,$out2,$out2,$inpperm
+	stvx_u		$out1,$x10,$out
+	le?vperm	$out3,$out3,$out3,$inpperm
+	stvx_u		$out2,$x20,$out
+	le?vperm	$out4,$out4,$out4,$inpperm
+	stvx_u		$out3,$x30,$out
+	le?vperm	$out5,$out5,$out5,$inpperm
+	stvx_u		$out4,$x40,$out
+	le?vperm	$out6,$out6,$out6,$inpperm
+	stvx_u		$out5,$x50,$out
+	stvx_u		$out6,$x60,$out
+	addi		$out,$out,0x70
+	b		Lctr32_enc8x_done
+
+.align	5
+Lctr32_enc8x_six:
+	vcipherlast	$out0,$out0,$in2
+	vcipherlast	$out1,$out1,$in3
+	vcipherlast	$out2,$out2,$in4
+	vcipherlast	$out3,$out3,$in5
+	vcipherlast	$out4,$out4,$in6
+	vcipherlast	$out5,$out5,$in7
+
+	le?vperm	$out0,$out0,$out0,$inpperm
+	le?vperm	$out1,$out1,$out1,$inpperm
+	stvx_u		$out0,$x00,$out
+	le?vperm	$out2,$out2,$out2,$inpperm
+	stvx_u		$out1,$x10,$out
+	le?vperm	$out3,$out3,$out3,$inpperm
+	stvx_u		$out2,$x20,$out
+	le?vperm	$out4,$out4,$out4,$inpperm
+	stvx_u		$out3,$x30,$out
+	le?vperm	$out5,$out5,$out5,$inpperm
+	stvx_u		$out4,$x40,$out
+	stvx_u		$out5,$x50,$out
+	addi		$out,$out,0x60
+	b		Lctr32_enc8x_done
+
+.align	5
+Lctr32_enc8x_five:
+	vcipherlast	$out0,$out0,$in3
+	vcipherlast	$out1,$out1,$in4
+	vcipherlast	$out2,$out2,$in5
+	vcipherlast	$out3,$out3,$in6
+	vcipherlast	$out4,$out4,$in7
+
+	le?vperm	$out0,$out0,$out0,$inpperm
+	le?vperm	$out1,$out1,$out1,$inpperm
+	stvx_u		$out0,$x00,$out
+	le?vperm	$out2,$out2,$out2,$inpperm
+	stvx_u		$out1,$x10,$out
+	le?vperm	$out3,$out3,$out3,$inpperm
+	stvx_u		$out2,$x20,$out
+	le?vperm	$out4,$out4,$out4,$inpperm
+	stvx_u		$out3,$x30,$out
+	stvx_u		$out4,$x40,$out
+	addi		$out,$out,0x50
+	b		Lctr32_enc8x_done
+
+.align	5
+Lctr32_enc8x_four:
+	vcipherlast	$out0,$out0,$in4
+	vcipherlast	$out1,$out1,$in5
+	vcipherlast	$out2,$out2,$in6
+	vcipherlast	$out3,$out3,$in7
+
+	le?vperm	$out0,$out0,$out0,$inpperm
+	le?vperm	$out1,$out1,$out1,$inpperm
+	stvx_u		$out0,$x00,$out
+	le?vperm	$out2,$out2,$out2,$inpperm
+	stvx_u		$out1,$x10,$out
+	le?vperm	$out3,$out3,$out3,$inpperm
+	stvx_u		$out2,$x20,$out
+	stvx_u		$out3,$x30,$out
+	addi		$out,$out,0x40
+	b		Lctr32_enc8x_done
+
+.align	5
+Lctr32_enc8x_three:
+	vcipherlast	$out0,$out0,$in5
+	vcipherlast	$out1,$out1,$in6
+	vcipherlast	$out2,$out2,$in7
+
+	le?vperm	$out0,$out0,$out0,$inpperm
+	le?vperm	$out1,$out1,$out1,$inpperm
+	stvx_u		$out0,$x00,$out
+	le?vperm	$out2,$out2,$out2,$inpperm
+	stvx_u		$out1,$x10,$out
+	stvx_u		$out2,$x20,$out
+	addi		$out,$out,0x30
+	b		Lctr32_enc8x_done
+
+.align	5
+Lctr32_enc8x_two:
+	vcipherlast	$out0,$out0,$in6
+	vcipherlast	$out1,$out1,$in7
+
+	le?vperm	$out0,$out0,$out0,$inpperm
+	le?vperm	$out1,$out1,$out1,$inpperm
+	stvx_u		$out0,$x00,$out
+	stvx_u		$out1,$x10,$out
+	addi		$out,$out,0x20
+	b		Lctr32_enc8x_done
+
+.align	5
+Lctr32_enc8x_one:
+	vcipherlast	$out0,$out0,$in7
+
+	le?vperm	$out0,$out0,$out0,$inpperm
+	stvx_u		$out0,0,$out
+	addi		$out,$out,0x10
+
+Lctr32_enc8x_done:
+	li		r10,`$FRAME+15`
+	li		r11,`$FRAME+31`
+	stvx		$inpperm,r10,$sp	# wipe copies of round keys
+	addi		r10,r10,32
+	stvx		$inpperm,r11,$sp
+	addi		r11,r11,32
+	stvx		$inpperm,r10,$sp
+	addi		r10,r10,32
+	stvx		$inpperm,r11,$sp
+	addi		r11,r11,32
+	stvx		$inpperm,r10,$sp
+	addi		r10,r10,32
+	stvx		$inpperm,r11,$sp
+	addi		r11,r11,32
+	stvx		$inpperm,r10,$sp
+	addi		r10,r10,32
+	stvx		$inpperm,r11,$sp
+	addi		r11,r11,32
+
+	mtspr		256,$vrsave
+	lvx		v20,r10,$sp		# ABI says so
+	addi		r10,r10,32
+	lvx		v21,r11,$sp
+	addi		r11,r11,32
+	lvx		v22,r10,$sp
+	addi		r10,r10,32
+	lvx		v23,r11,$sp
+	addi		r11,r11,32
+	lvx		v24,r10,$sp
+	addi		r10,r10,32
+	lvx		v25,r11,$sp
+	addi		r11,r11,32
+	lvx		v26,r10,$sp
+	addi		r10,r10,32
+	lvx		v27,r11,$sp
+	addi		r11,r11,32
+	lvx		v28,r10,$sp
+	addi		r10,r10,32
+	lvx		v29,r11,$sp
+	addi		r11,r11,32
+	lvx		v30,r10,$sp
+	lvx		v31,r11,$sp
+	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
+	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
+	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
+	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
+	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
+	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
+	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`
+	blr
+	.long		0
+	.byte		0,12,0x14,0,0x80,6,6,0
+	.long		0
+.size	.${prefix}_ctr32_encrypt_blocks,.-.${prefix}_ctr32_encrypt_blocks
+___
+}}	}}}
+
+#########################################################################
+{{{	# XTS procedures						#
+# int aes_p8_xts_[en|de]crypt(const char *inp, char *out, size_t len,	#
+#                             const AES_KEY *key1, const AES_KEY *key2,	#
+#                             [const] unsigned char iv[16]);		#
+# If $key2 is NULL, then a "tweak chaining" mode is engaged, in which	#
+# input tweak value is assumed to be encrypted already, and last tweak	#
+# value, one suitable for consecutive call on same chunk of data, is	#
+# written back to original buffer. In addition, in "tweak chaining"	#
+# mode only complete input blocks are processed.			#
+
+my ($inp,$out,$len,$key1,$key2,$ivp,$rounds,$idx) =	map("r$_",(3..10));
+my ($rndkey0,$rndkey1,$inout) =				map("v$_",(0..2));
+my ($output,$inptail,$inpperm,$leperm,$keyperm) =	map("v$_",(3..7));
+my ($tweak,$seven,$eighty7,$tmp,$tweak1) =		map("v$_",(8..12));
+my $taillen = $key2;
+
+   ($inp,$idx) = ($idx,$inp);				# reassign
+
+$code.=<<___;
+.globl	.${prefix}_xts_encrypt
+	mr		$inp,r3				# reassign
+	li		r3,-1
+	${UCMP}i	$len,16
+	bltlr-
+
+	lis		r0,0xfff0
+	mfspr		r12,256				# save vrsave
+	li		r11,0
+	mtspr		256,r0
+
+	vspltisb	$seven,0x07			# 0x070707..07
+	le?lvsl		$leperm,r11,r11
+	le?vspltisb	$tmp,0x0f
+	le?vxor		$leperm,$leperm,$seven
+
+	li		$idx,15
+	lvx		$tweak,0,$ivp			# load [unaligned] iv
+	lvsl		$inpperm,0,$ivp
+	lvx		$inptail,$idx,$ivp
+	le?vxor		$inpperm,$inpperm,$tmp
+	vperm		$tweak,$tweak,$inptail,$inpperm
+
+	neg		r11,$inp
+	lvsr		$inpperm,0,r11			# prepare for unaligned load
+	lvx		$inout,0,$inp
+	addi		$inp,$inp,15			# 15 is not typo
+	le?vxor		$inpperm,$inpperm,$tmp
+
+	${UCMP}i	$key2,0				# key2==NULL?
+	beq		Lxts_enc_no_key2
+
+	?lvsl		$keyperm,0,$key2		# prepare for unaligned key
+	lwz		$rounds,240($key2)
+	srwi		$rounds,$rounds,1
+	subi		$rounds,$rounds,1
+	li		$idx,16
+
+	lvx		$rndkey0,0,$key2
+	lvx		$rndkey1,$idx,$key2
+	addi		$idx,$idx,16
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vxor		$tweak,$tweak,$rndkey0
+	lvx		$rndkey0,$idx,$key2
+	addi		$idx,$idx,16
+	mtctr		$rounds
+
+Ltweak_xts_enc:
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
+	vcipher		$tweak,$tweak,$rndkey1
+	lvx		$rndkey1,$idx,$key2
+	addi		$idx,$idx,16
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vcipher		$tweak,$tweak,$rndkey0
+	lvx		$rndkey0,$idx,$key2
+	addi		$idx,$idx,16
+	bdnz		Ltweak_xts_enc
+
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
+	vcipher		$tweak,$tweak,$rndkey1
+	lvx		$rndkey1,$idx,$key2
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vcipherlast	$tweak,$tweak,$rndkey0
+
+	li		$ivp,0				# don't chain the tweak
+	b		Lxts_enc
+
+Lxts_enc_no_key2:
+	li		$idx,-16
+	and		$len,$len,$idx			# in "tweak chaining"
+							# mode only complete
+							# blocks are processed
+Lxts_enc:
+	lvx		$inptail,0,$inp
+	addi		$inp,$inp,16
+
+	?lvsl		$keyperm,0,$key1		# prepare for unaligned key
+	lwz		$rounds,240($key1)
+	srwi		$rounds,$rounds,1
+	subi		$rounds,$rounds,1
+	li		$idx,16
+
+	vslb		$eighty7,$seven,$seven		# 0x808080..80
+	vor		$eighty7,$eighty7,$seven	# 0x878787..87
+	vspltisb	$tmp,1				# 0x010101..01
+	vsldoi		$eighty7,$eighty7,$tmp,15	# 0x870101..01
+
+	${UCMP}i	$len,96
+	bge		_aesp8_xts_encrypt6x
+
+	andi.		$taillen,$len,15
+	subic		r0,$len,32
+	subi		$taillen,$taillen,16
+	subfe		r0,r0,r0
+	and		r0,r0,$taillen
+	add		$inp,$inp,r0
+
+	lvx		$rndkey0,0,$key1
+	lvx		$rndkey1,$idx,$key1
+	addi		$idx,$idx,16
+	vperm		$inout,$inout,$inptail,$inpperm
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vxor		$inout,$inout,$tweak
+	vxor		$inout,$inout,$rndkey0
+	lvx		$rndkey0,$idx,$key1
+	addi		$idx,$idx,16
+	mtctr		$rounds
+	b		Loop_xts_enc
+
+.align	5
+Loop_xts_enc:
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
+	vcipher		$inout,$inout,$rndkey1
+	lvx		$rndkey1,$idx,$key1
+	addi		$idx,$idx,16
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vcipher		$inout,$inout,$rndkey0
+	lvx		$rndkey0,$idx,$key1
+	addi		$idx,$idx,16
+	bdnz		Loop_xts_enc
+
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
+	vcipher		$inout,$inout,$rndkey1
+	lvx		$rndkey1,$idx,$key1
+	li		$idx,16
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vxor		$rndkey0,$rndkey0,$tweak
+	vcipherlast	$output,$inout,$rndkey0
+
+	le?vperm	$tmp,$output,$output,$leperm
+	be?nop
+	le?stvx_u	$tmp,0,$out
+	be?stvx_u	$output,0,$out
+	addi		$out,$out,16
+
+	subic.		$len,$len,16
+	beq		Lxts_enc_done
+
+	vmr		$inout,$inptail
+	lvx		$inptail,0,$inp
+	addi		$inp,$inp,16
+	lvx		$rndkey0,0,$key1
+	lvx		$rndkey1,$idx,$key1
+	addi		$idx,$idx,16
+
+	subic		r0,$len,32
+	subfe		r0,r0,r0
+	and		r0,r0,$taillen
+	add		$inp,$inp,r0
+
+	vsrab		$tmp,$tweak,$seven		# next tweak value
+	vaddubm		$tweak,$tweak,$tweak
+	vsldoi		$tmp,$tmp,$tmp,15
+	vand		$tmp,$tmp,$eighty7
+	vxor		$tweak,$tweak,$tmp
+
+	vperm		$inout,$inout,$inptail,$inpperm
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vxor		$inout,$inout,$tweak
+	vxor		$output,$output,$rndkey0	# just in case $len<16
+	vxor		$inout,$inout,$rndkey0
+	lvx		$rndkey0,$idx,$key1
+	addi		$idx,$idx,16
+
+	mtctr		$rounds
+	${UCMP}i	$len,16
+	bge		Loop_xts_enc
+
+	vxor		$output,$output,$tweak
+	lvsr		$inpperm,0,$len			# $inpperm is no longer needed
+	vxor		$inptail,$inptail,$inptail	# $inptail is no longer needed
+	vspltisb	$tmp,-1
+	vperm		$inptail,$inptail,$tmp,$inpperm
+	vsel		$inout,$inout,$output,$inptail
+
+	subi		r11,$out,17
+	subi		$out,$out,16
+	mtctr		$len
+	li		$len,16
+Loop_xts_enc_steal:
+	lbzu		r0,1(r11)
+	stb		r0,16(r11)
+	bdnz		Loop_xts_enc_steal
+
+	mtctr		$rounds
+	b		Loop_xts_enc			# one more time...
+
+Lxts_enc_done:
+	${UCMP}i	$ivp,0
+	beq		Lxts_enc_ret
+
+	vsrab		$tmp,$tweak,$seven		# next tweak value
+	vaddubm		$tweak,$tweak,$tweak
+	vsldoi		$tmp,$tmp,$tmp,15
+	vand		$tmp,$tmp,$eighty7
+	vxor		$tweak,$tweak,$tmp
+
+	le?vperm	$tweak,$tweak,$tweak,$leperm
+	stvx_u		$tweak,0,$ivp
+
+Lxts_enc_ret:
+	mtspr		256,r12				# restore vrsave
+	li		r3,0
+	blr
+	.long		0
+	.byte		0,12,0x04,0,0x80,6,6,0
+	.long		0
+.size	.${prefix}_xts_encrypt,.-.${prefix}_xts_encrypt
+
+.globl	.${prefix}_xts_decrypt
+	mr		$inp,r3				# reassign
+	li		r3,-1
+	${UCMP}i	$len,16
+	bltlr-
+
+	lis		r0,0xfff8
+	mfspr		r12,256				# save vrsave
+	li		r11,0
+	mtspr		256,r0
+
+	andi.		r0,$len,15
+	neg		r0,r0
+	andi.		r0,r0,16
+	sub		$len,$len,r0
+
+	vspltisb	$seven,0x07			# 0x070707..07
+	le?lvsl		$leperm,r11,r11
+	le?vspltisb	$tmp,0x0f
+	le?vxor		$leperm,$leperm,$seven
+
+	li		$idx,15
+	lvx		$tweak,0,$ivp			# load [unaligned] iv
+	lvsl		$inpperm,0,$ivp
+	lvx		$inptail,$idx,$ivp
+	le?vxor		$inpperm,$inpperm,$tmp
+	vperm		$tweak,$tweak,$inptail,$inpperm
+
+	neg		r11,$inp
+	lvsr		$inpperm,0,r11			# prepare for unaligned load
+	lvx		$inout,0,$inp
+	addi		$inp,$inp,15			# 15 is not typo
+	le?vxor		$inpperm,$inpperm,$tmp
+
+	${UCMP}i	$key2,0				# key2==NULL?
+	beq		Lxts_dec_no_key2
+
+	?lvsl		$keyperm,0,$key2		# prepare for unaligned key
+	lwz		$rounds,240($key2)
+	srwi		$rounds,$rounds,1
+	subi		$rounds,$rounds,1
+	li		$idx,16
+
+	lvx		$rndkey0,0,$key2
+	lvx		$rndkey1,$idx,$key2
+	addi		$idx,$idx,16
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vxor		$tweak,$tweak,$rndkey0
+	lvx		$rndkey0,$idx,$key2
+	addi		$idx,$idx,16
+	mtctr		$rounds
+
+Ltweak_xts_dec:
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
+	vcipher		$tweak,$tweak,$rndkey1
+	lvx		$rndkey1,$idx,$key2
+	addi		$idx,$idx,16
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vcipher		$tweak,$tweak,$rndkey0
+	lvx		$rndkey0,$idx,$key2
+	addi		$idx,$idx,16
+	bdnz		Ltweak_xts_dec
+
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
+	vcipher		$tweak,$tweak,$rndkey1
+	lvx		$rndkey1,$idx,$key2
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vcipherlast	$tweak,$tweak,$rndkey0
+
+	li		$ivp,0				# don't chain the tweak
+	b		Lxts_dec
+
+Lxts_dec_no_key2:
+	neg		$idx,$len
+	andi.		$idx,$idx,15
+	add		$len,$len,$idx			# in "tweak chaining"
+							# mode only complete
+							# blocks are processed
+Lxts_dec:
+	lvx		$inptail,0,$inp
+	addi		$inp,$inp,16
+
+	?lvsl		$keyperm,0,$key1		# prepare for unaligned key
+	lwz		$rounds,240($key1)
+	srwi		$rounds,$rounds,1
+	subi		$rounds,$rounds,1
+	li		$idx,16
+
+	vslb		$eighty7,$seven,$seven		# 0x808080..80
+	vor		$eighty7,$eighty7,$seven	# 0x878787..87
+	vspltisb	$tmp,1				# 0x010101..01
+	vsldoi		$eighty7,$eighty7,$tmp,15	# 0x870101..01
+
+	${UCMP}i	$len,96
+	bge		_aesp8_xts_decrypt6x
+
+	lvx		$rndkey0,0,$key1
+	lvx		$rndkey1,$idx,$key1
+	addi		$idx,$idx,16
+	vperm		$inout,$inout,$inptail,$inpperm
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vxor		$inout,$inout,$tweak
+	vxor		$inout,$inout,$rndkey0
+	lvx		$rndkey0,$idx,$key1
+	addi		$idx,$idx,16
+	mtctr		$rounds
+
+	${UCMP}i	$len,16
+	blt		Ltail_xts_dec
+	be?b		Loop_xts_dec
+
+.align	5
+Loop_xts_dec:
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
+	vncipher	$inout,$inout,$rndkey1
+	lvx		$rndkey1,$idx,$key1
+	addi		$idx,$idx,16
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vncipher	$inout,$inout,$rndkey0
+	lvx		$rndkey0,$idx,$key1
+	addi		$idx,$idx,16
+	bdnz		Loop_xts_dec
+
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
+	vncipher	$inout,$inout,$rndkey1
+	lvx		$rndkey1,$idx,$key1
+	li		$idx,16
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vxor		$rndkey0,$rndkey0,$tweak
+	vncipherlast	$output,$inout,$rndkey0
+
+	le?vperm	$tmp,$output,$output,$leperm
+	be?nop
+	le?stvx_u	$tmp,0,$out
+	be?stvx_u	$output,0,$out
+	addi		$out,$out,16
+
+	subic.		$len,$len,16
+	beq		Lxts_dec_done
+
+	vmr		$inout,$inptail
+	lvx		$inptail,0,$inp
+	addi		$inp,$inp,16
+	lvx		$rndkey0,0,$key1
+	lvx		$rndkey1,$idx,$key1
+	addi		$idx,$idx,16
+
+	vsrab		$tmp,$tweak,$seven		# next tweak value
+	vaddubm		$tweak,$tweak,$tweak
+	vsldoi		$tmp,$tmp,$tmp,15
+	vand		$tmp,$tmp,$eighty7
+	vxor		$tweak,$tweak,$tmp
+
+	vperm		$inout,$inout,$inptail,$inpperm
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vxor		$inout,$inout,$tweak
+	vxor		$inout,$inout,$rndkey0
+	lvx		$rndkey0,$idx,$key1
+	addi		$idx,$idx,16
+
+	mtctr		$rounds
+	${UCMP}i	$len,16
+	bge		Loop_xts_dec
+
+Ltail_xts_dec:
+	vsrab		$tmp,$tweak,$seven		# next tweak value
+	vaddubm		$tweak1,$tweak,$tweak
+	vsldoi		$tmp,$tmp,$tmp,15
+	vand		$tmp,$tmp,$eighty7
+	vxor		$tweak1,$tweak1,$tmp
+
+	subi		$inp,$inp,16
+	add		$inp,$inp,$len
+
+	vxor		$inout,$inout,$tweak		# :-(
+	vxor		$inout,$inout,$tweak1		# :-)
+
+Loop_xts_dec_short:
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
+	vncipher	$inout,$inout,$rndkey1
+	lvx		$rndkey1,$idx,$key1
+	addi		$idx,$idx,16
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vncipher	$inout,$inout,$rndkey0
+	lvx		$rndkey0,$idx,$key1
+	addi		$idx,$idx,16
+	bdnz		Loop_xts_dec_short
+
+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
+	vncipher	$inout,$inout,$rndkey1
+	lvx		$rndkey1,$idx,$key1
+	li		$idx,16
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+	vxor		$rndkey0,$rndkey0,$tweak1
+	vncipherlast	$output,$inout,$rndkey0
+
+	le?vperm	$tmp,$output,$output,$leperm
+	be?nop
+	le?stvx_u	$tmp,0,$out
+	be?stvx_u	$output,0,$out
+
+	vmr		$inout,$inptail
+	lvx		$inptail,0,$inp
+	#addi		$inp,$inp,16
+	lvx		$rndkey0,0,$key1
+	lvx		$rndkey1,$idx,$key1
+	addi		$idx,$idx,16
+	vperm		$inout,$inout,$inptail,$inpperm
+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
+
+	lvsr		$inpperm,0,$len			# $inpperm is no longer needed
+	vxor		$inptail,$inptail,$inptail	# $inptail is no longer needed
+	vspltisb	$tmp,-1
+	vperm		$inptail,$inptail,$tmp,$inpperm
+	vsel		$inout,$inout,$output,$inptail
+
+	vxor		$rndkey0,$rndkey0,$tweak
+	vxor		$inout,$inout,$rndkey0
+	lvx		$rndkey0,$idx,$key1
+	addi		$idx,$idx,16
+
+	subi		r11,$out,1
+	mtctr		$len
+	li		$len,16
+Loop_xts_dec_steal:
+	lbzu		r0,1(r11)
+	stb		r0,16(r11)
+	bdnz		Loop_xts_dec_steal
+
+	mtctr		$rounds
+	b		Loop_xts_dec			# one more time...
+
+Lxts_dec_done:
+	${UCMP}i	$ivp,0
+	beq		Lxts_dec_ret
+
+	vsrab		$tmp,$tweak,$seven		# next tweak value
+	vaddubm		$tweak,$tweak,$tweak
+	vsldoi		$tmp,$tmp,$tmp,15
+	vand		$tmp,$tmp,$eighty7
+	vxor		$tweak,$tweak,$tmp
+
+	le?vperm	$tweak,$tweak,$tweak,$leperm
+	stvx_u		$tweak,0,$ivp
+
+Lxts_dec_ret:
+	mtspr		256,r12				# restore vrsave
+	li		r3,0
+	blr
+	.long		0
+	.byte		0,12,0x04,0,0x80,6,6,0
+	.long		0
+.size	.${prefix}_xts_decrypt,.-.${prefix}_xts_decrypt
+___
+#########################################################################
+{{	# Optimized XTS procedures					#
+my $key_=$key2;
+my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,3,26..31));
+    $x00=0 if ($flavour =~ /osx/);
+my ($in0,  $in1,  $in2,  $in3,  $in4,  $in5 )=map("v$_",(0..5));
+my ($out0, $out1, $out2, $out3, $out4, $out5)=map("v$_",(7,12..16));
+my ($twk0, $twk1, $twk2, $twk3, $twk4, $twk5)=map("v$_",(17..22));
+my $rndkey0="v23";	# v24-v25 rotating buffer for first found keys
+			# v26-v31 last 6 round keys
+my ($keyperm)=($out0);	# aliases with "caller", redundant assignment
+my $taillen=$x70;
+
+$code.=<<___;
+.align	5
+_aesp8_xts_encrypt6x:
+	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
+	mflr		r11
+	li		r7,`$FRAME+8*16+15`
+	li		r3,`$FRAME+8*16+31`
+	$PUSH		r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
+	stvx		v20,r7,$sp		# ABI says so
+	addi		r7,r7,32
+	stvx		v21,r3,$sp
+	addi		r3,r3,32
+	stvx		v22,r7,$sp
+	addi		r7,r7,32
+	stvx		v23,r3,$sp
+	addi		r3,r3,32
+	stvx		v24,r7,$sp
+	addi		r7,r7,32
+	stvx		v25,r3,$sp
+	addi		r3,r3,32
+	stvx		v26,r7,$sp
+	addi		r7,r7,32
+	stvx		v27,r3,$sp
+	addi		r3,r3,32
+	stvx		v28,r7,$sp
+	addi		r7,r7,32
+	stvx		v29,r3,$sp
+	addi		r3,r3,32
+	stvx		v30,r7,$sp
+	stvx		v31,r3,$sp
+	li		r0,-1
+	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
+	li		$x10,0x10
+	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
+	li		$x20,0x20
+	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
+	li		$x30,0x30
+	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
+	li		$x40,0x40
+	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
+	li		$x50,0x50
+	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
+	li		$x60,0x60
+	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
+	li		$x70,0x70
+	mtspr		256,r0
+
+	xxlor		2, 32+$eighty7, 32+$eighty7
+	vsldoi		$eighty7,$tmp,$eighty7,1        # 0x010101..87
+	xxlor		1, 32+$eighty7, 32+$eighty7
+
+	# Load XOR Lconsts.
+	mr		$x70, r6
+	bl		Lconsts
+	lxvw4x		0, $x40, r6		# load XOR contents
+	mr		r6, $x70
+	li		$x70,0x70
+
+	subi		$rounds,$rounds,3	# -4 in total
+
+	lvx		$rndkey0,$x00,$key1	# load key schedule
+	lvx		v30,$x10,$key1
+	addi		$key1,$key1,0x20
+	lvx		v31,$x00,$key1
+	?vperm		$rndkey0,$rndkey0,v30,$keyperm
+	addi		$key_,$sp,$FRAME+15
+	mtctr		$rounds
+
+Load_xts_enc_key:
+	?vperm		v24,v30,v31,$keyperm
+	lvx		v30,$x10,$key1
+	addi		$key1,$key1,0x20
+	stvx		v24,$x00,$key_		# off-load round[1]
+	?vperm		v25,v31,v30,$keyperm
+	lvx		v31,$x00,$key1
+	stvx		v25,$x10,$key_		# off-load round[2]
+	addi		$key_,$key_,0x20
+	bdnz		Load_xts_enc_key
+
+	lvx		v26,$x10,$key1
+	?vperm		v24,v30,v31,$keyperm
+	lvx		v27,$x20,$key1
+	stvx		v24,$x00,$key_		# off-load round[3]
+	?vperm		v25,v31,v26,$keyperm
+	lvx		v28,$x30,$key1
+	stvx		v25,$x10,$key_		# off-load round[4]
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
+	?vperm		v26,v26,v27,$keyperm
+	lvx		v29,$x40,$key1
+	?vperm		v27,v27,v28,$keyperm
+	lvx		v30,$x50,$key1
+	?vperm		v28,v28,v29,$keyperm
+	lvx		v31,$x60,$key1
+	?vperm		v29,v29,v30,$keyperm
+	lvx		$twk5,$x70,$key1	# borrow $twk5
+	?vperm		v30,v30,v31,$keyperm
+	lvx		v24,$x00,$key_		# pre-load round[1]
+	?vperm		v31,v31,$twk5,$keyperm
+	lvx		v25,$x10,$key_		# pre-load round[2]
+
+	# Switch to use the following codes with 0x010101..87 to generate tweak.
+	#     eighty7 = 0x010101..87
+	# vsrab         tmp, tweak, seven       # next tweak value, right shift 7 bits
+	# vand          tmp, tmp, eighty7       # last byte with carry
+	# vaddubm       tweak, tweak, tweak     # left shift 1 bit (x2)
+	# xxlor         vsx, 0, 0
+	# vpermxor      tweak, tweak, tmp, vsx
+
+	 vperm		$in0,$inout,$inptail,$inpperm
+	 subi		$inp,$inp,31		# undo "caller"
+	vxor		$twk0,$tweak,$rndkey0
+	vsrab		$tmp,$tweak,$seven	# next tweak value
+	vaddubm		$tweak,$tweak,$tweak
+	vand		$tmp,$tmp,$eighty7
+	 vxor		$out0,$in0,$twk0
+	xxlor		32+$in1, 0, 0
+	vpermxor	$tweak, $tweak, $tmp, $in1
+
+	 lvx_u		$in1,$x10,$inp
+	vxor		$twk1,$tweak,$rndkey0
+	vsrab		$tmp,$tweak,$seven	# next tweak value
+	vaddubm		$tweak,$tweak,$tweak
+	 le?vperm	$in1,$in1,$in1,$leperm
+	vand		$tmp,$tmp,$eighty7
+	 vxor		$out1,$in1,$twk1
+	xxlor		32+$in2, 0, 0
+	vpermxor	$tweak, $tweak, $tmp, $in2
+
+	 lvx_u		$in2,$x20,$inp
+	 andi.		$taillen,$len,15
+	vxor		$twk2,$tweak,$rndkey0
+	vsrab		$tmp,$tweak,$seven	# next tweak value
+	vaddubm		$tweak,$tweak,$tweak
+	 le?vperm	$in2,$in2,$in2,$leperm
+	vand		$tmp,$tmp,$eighty7
+	 vxor		$out2,$in2,$twk2
+	xxlor		32+$in3, 0, 0
+	vpermxor	$tweak, $tweak, $tmp, $in3
+
+	 lvx_u		$in3,$x30,$inp
+	 sub		$len,$len,$taillen
+	vxor		$twk3,$tweak,$rndkey0
+	vsrab		$tmp,$tweak,$seven	# next tweak value
+	vaddubm		$tweak,$tweak,$tweak
+	 le?vperm	$in3,$in3,$in3,$leperm
+	vand		$tmp,$tmp,$eighty7
+	 vxor		$out3,$in3,$twk3
+	xxlor		32+$in4, 0, 0
+	vpermxor	$tweak, $tweak, $tmp, $in4
+
+	 lvx_u		$in4,$x40,$inp
+	 subi		$len,$len,0x60
+	vxor		$twk4,$tweak,$rndkey0
+	vsrab		$tmp,$tweak,$seven	# next tweak value
+	vaddubm		$tweak,$tweak,$tweak
+	 le?vperm	$in4,$in4,$in4,$leperm
+	vand		$tmp,$tmp,$eighty7
+	 vxor		$out4,$in4,$twk4
+	xxlor		32+$in5, 0, 0
+	vpermxor	$tweak, $tweak, $tmp, $in5
+
+	 lvx_u		$in5,$x50,$inp
+	 addi		$inp,$inp,0x60
+	vxor		$twk5,$tweak,$rndkey0
+	vsrab		$tmp,$tweak,$seven	# next tweak value
+	vaddubm		$tweak,$tweak,$tweak
+	 le?vperm	$in5,$in5,$in5,$leperm
+	vand		$tmp,$tmp,$eighty7
+	 vxor		$out5,$in5,$twk5
+	xxlor		32+$in0, 0, 0
+	vpermxor	$tweak, $tweak, $tmp, $in0
+
+	vxor		v31,v31,$rndkey0
+	mtctr		$rounds
+	b		Loop_xts_enc6x
+
+.align	5
+Loop_xts_enc6x:
+	vcipher		$out0,$out0,v24
+	vcipher		$out1,$out1,v24
+	vcipher		$out2,$out2,v24
+	vcipher		$out3,$out3,v24
+	vcipher		$out4,$out4,v24
+	vcipher		$out5,$out5,v24
+	lvx		v24,$x20,$key_		# round[3]
+	addi		$key_,$key_,0x20
+
+	vcipher		$out0,$out0,v25
+	vcipher		$out1,$out1,v25
+	vcipher		$out2,$out2,v25
+	vcipher		$out3,$out3,v25
+	vcipher		$out4,$out4,v25
+	vcipher		$out5,$out5,v25
+	lvx		v25,$x10,$key_		# round[4]
+	bdnz		Loop_xts_enc6x
+
+	xxlor		32+$eighty7, 1, 1	# 0x010101..87
+
+	subic		$len,$len,96		# $len-=96
+	 vxor		$in0,$twk0,v31		# xor with last round key
+	vcipher		$out0,$out0,v24
+	vcipher		$out1,$out1,v24
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
+	 vxor		$twk0,$tweak,$rndkey0
+	 vaddubm	$tweak,$tweak,$tweak
+	vcipher		$out2,$out2,v24
+	vcipher		$out3,$out3,v24
+	vcipher		$out4,$out4,v24
+	vcipher		$out5,$out5,v24
+
+	subfe.		r0,r0,r0		# borrow?-1:0
+	 vand		$tmp,$tmp,$eighty7
+	vcipher		$out0,$out0,v25
+	vcipher		$out1,$out1,v25
+	 xxlor		32+$in1, 0, 0
+	 vpermxor	$tweak, $tweak, $tmp, $in1
+	vcipher		$out2,$out2,v25
+	vcipher		$out3,$out3,v25
+	 vxor		$in1,$twk1,v31
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
+	 vxor		$twk1,$tweak,$rndkey0
+	vcipher		$out4,$out4,v25
+	vcipher		$out5,$out5,v25
+
+	and		r0,r0,$len
+	 vaddubm	$tweak,$tweak,$tweak
+	vcipher		$out0,$out0,v26
+	vcipher		$out1,$out1,v26
+	 vand		$tmp,$tmp,$eighty7
+	vcipher		$out2,$out2,v26
+	vcipher		$out3,$out3,v26
+	 xxlor		32+$in2, 0, 0
+	 vpermxor	$tweak, $tweak, $tmp, $in2
+	vcipher		$out4,$out4,v26
+	vcipher		$out5,$out5,v26
+
+	add		$inp,$inp,r0		# $inp is adjusted in such
+						# way that at exit from the
+						# loop inX-in5 are loaded
+						# with last "words"
+	 vxor		$in2,$twk2,v31
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
+	 vxor		$twk2,$tweak,$rndkey0
+	 vaddubm	$tweak,$tweak,$tweak
+	vcipher		$out0,$out0,v27
+	vcipher		$out1,$out1,v27
+	vcipher		$out2,$out2,v27
+	vcipher		$out3,$out3,v27
+	 vand		$tmp,$tmp,$eighty7
+	vcipher		$out4,$out4,v27
+	vcipher		$out5,$out5,v27
+
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
+	 xxlor		32+$in3, 0, 0
+	 vpermxor	$tweak, $tweak, $tmp, $in3
+	vcipher		$out0,$out0,v28
+	vcipher		$out1,$out1,v28
+	 vxor		$in3,$twk3,v31
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
+	 vxor		$twk3,$tweak,$rndkey0
+	vcipher		$out2,$out2,v28
+	vcipher		$out3,$out3,v28
+	 vaddubm	$tweak,$tweak,$tweak
+	vcipher		$out4,$out4,v28
+	vcipher		$out5,$out5,v28
+	lvx		v24,$x00,$key_		# re-pre-load round[1]
+	 vand		$tmp,$tmp,$eighty7
+
+	vcipher		$out0,$out0,v29
+	vcipher		$out1,$out1,v29
+	 xxlor		32+$in4, 0, 0
+	 vpermxor	$tweak, $tweak, $tmp, $in4
+	vcipher		$out2,$out2,v29
+	vcipher		$out3,$out3,v29
+	 vxor		$in4,$twk4,v31
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
+	 vxor		$twk4,$tweak,$rndkey0
+	vcipher		$out4,$out4,v29
+	vcipher		$out5,$out5,v29
+	lvx		v25,$x10,$key_		# re-pre-load round[2]
+	 vaddubm	$tweak,$tweak,$tweak
+
+	vcipher		$out0,$out0,v30
+	vcipher		$out1,$out1,v30
+	 vand		$tmp,$tmp,$eighty7
+	vcipher		$out2,$out2,v30
+	vcipher		$out3,$out3,v30
+	 xxlor		32+$in5, 0, 0
+	 vpermxor	$tweak, $tweak, $tmp, $in5
+	vcipher		$out4,$out4,v30
+	vcipher		$out5,$out5,v30
+	 vxor		$in5,$twk5,v31
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
+	 vxor		$twk5,$tweak,$rndkey0
+
+	vcipherlast	$out0,$out0,$in0
+	 lvx_u		$in0,$x00,$inp		# load next input block
+	 vaddubm	$tweak,$tweak,$tweak
+	vcipherlast	$out1,$out1,$in1
+	 lvx_u		$in1,$x10,$inp
+	vcipherlast	$out2,$out2,$in2
+	 le?vperm	$in0,$in0,$in0,$leperm
+	 lvx_u		$in2,$x20,$inp
+	 vand		$tmp,$tmp,$eighty7
+	vcipherlast	$out3,$out3,$in3
+	 le?vperm	$in1,$in1,$in1,$leperm
+	 lvx_u		$in3,$x30,$inp
+	vcipherlast	$out4,$out4,$in4
+	 le?vperm	$in2,$in2,$in2,$leperm
+	 lvx_u		$in4,$x40,$inp
+	 xxlor		10, 32+$in0, 32+$in0
+	 xxlor		32+$in0, 0, 0
+	 vpermxor	$tweak, $tweak, $tmp, $in0
+	 xxlor		32+$in0, 10, 10
+	vcipherlast	$tmp,$out5,$in5		# last block might be needed
+						# in stealing mode
+	 le?vperm	$in3,$in3,$in3,$leperm
+	 lvx_u		$in5,$x50,$inp
+	 addi		$inp,$inp,0x60
+	 le?vperm	$in4,$in4,$in4,$leperm
+	 le?vperm	$in5,$in5,$in5,$leperm
+
+	le?vperm	$out0,$out0,$out0,$leperm
+	le?vperm	$out1,$out1,$out1,$leperm
+	stvx_u		$out0,$x00,$out		# store output
+	 vxor		$out0,$in0,$twk0
+	le?vperm	$out2,$out2,$out2,$leperm
+	stvx_u		$out1,$x10,$out
+	 vxor		$out1,$in1,$twk1
+	le?vperm	$out3,$out3,$out3,$leperm
+	stvx_u		$out2,$x20,$out
+	 vxor		$out2,$in2,$twk2
+	le?vperm	$out4,$out4,$out4,$leperm
+	stvx_u		$out3,$x30,$out
+	 vxor		$out3,$in3,$twk3
+	le?vperm	$out5,$tmp,$tmp,$leperm
+	stvx_u		$out4,$x40,$out
+	 vxor		$out4,$in4,$twk4
+	le?stvx_u	$out5,$x50,$out
+	be?stvx_u	$tmp, $x50,$out
+	 vxor		$out5,$in5,$twk5
+	addi		$out,$out,0x60
+
+	mtctr		$rounds
+	beq		Loop_xts_enc6x		# did $len-=96 borrow?
+
+	xxlor		32+$eighty7, 2, 2	# 0x010101..87
+
+	addic.		$len,$len,0x60
+	beq		Lxts_enc6x_zero
+	cmpwi		$len,0x20
+	blt		Lxts_enc6x_one
+	nop
+	beq		Lxts_enc6x_two
+	cmpwi		$len,0x40
+	blt		Lxts_enc6x_three
+	nop
+	beq		Lxts_enc6x_four
+
+Lxts_enc6x_five:
+	vxor		$out0,$in1,$twk0
+	vxor		$out1,$in2,$twk1
+	vxor		$out2,$in3,$twk2
+	vxor		$out3,$in4,$twk3
+	vxor		$out4,$in5,$twk4
+
+	bl		_aesp8_xts_enc5x
+
+	le?vperm	$out0,$out0,$out0,$leperm
+	vmr		$twk0,$twk5		# unused tweak
+	le?vperm	$out1,$out1,$out1,$leperm
+	stvx_u		$out0,$x00,$out		# store output
+	le?vperm	$out2,$out2,$out2,$leperm
+	stvx_u		$out1,$x10,$out
+	le?vperm	$out3,$out3,$out3,$leperm
+	stvx_u		$out2,$x20,$out
+	vxor		$tmp,$out4,$twk5	# last block prep for stealing
+	le?vperm	$out4,$out4,$out4,$leperm
+	stvx_u		$out3,$x30,$out
+	stvx_u		$out4,$x40,$out
+	addi		$out,$out,0x50
+	bne		Lxts_enc6x_steal
+	b		Lxts_enc6x_done
+
+.align	4
+Lxts_enc6x_four:
+	vxor		$out0,$in2,$twk0
+	vxor		$out1,$in3,$twk1
+	vxor		$out2,$in4,$twk2
+	vxor		$out3,$in5,$twk3
+	vxor		$out4,$out4,$out4
+
+	bl		_aesp8_xts_enc5x
+
+	le?vperm	$out0,$out0,$out0,$leperm
+	vmr		$twk0,$twk4		# unused tweak
+	le?vperm	$out1,$out1,$out1,$leperm
+	stvx_u		$out0,$x00,$out		# store output
+	le?vperm	$out2,$out2,$out2,$leperm
+	stvx_u		$out1,$x10,$out
+	vxor		$tmp,$out3,$twk4	# last block prep for stealing
+	le?vperm	$out3,$out3,$out3,$leperm
+	stvx_u		$out2,$x20,$out
+	stvx_u		$out3,$x30,$out
+	addi		$out,$out,0x40
+	bne		Lxts_enc6x_steal
+	b		Lxts_enc6x_done
+
+.align	4
+Lxts_enc6x_three:
+	vxor		$out0,$in3,$twk0
+	vxor		$out1,$in4,$twk1
+	vxor		$out2,$in5,$twk2
+	vxor		$out3,$out3,$out3
+	vxor		$out4,$out4,$out4
+
+	bl		_aesp8_xts_enc5x
+
+	le?vperm	$out0,$out0,$out0,$leperm
+	vmr		$twk0,$twk3		# unused tweak
+	le?vperm	$out1,$out1,$out1,$leperm
+	stvx_u		$out0,$x00,$out		# store output
+	vxor		$tmp,$out2,$twk3	# last block prep for stealing
+	le?vperm	$out2,$out2,$out2,$leperm
+	stvx_u		$out1,$x10,$out
+	stvx_u		$out2,$x20,$out
+	addi		$out,$out,0x30
+	bne		Lxts_enc6x_steal
+	b		Lxts_enc6x_done
+
+.align	4
+Lxts_enc6x_two:
+	vxor		$out0,$in4,$twk0
+	vxor		$out1,$in5,$twk1
+	vxor		$out2,$out2,$out2
+	vxor		$out3,$out3,$out3
+	vxor		$out4,$out4,$out4
+
+	bl		_aesp8_xts_enc5x
+
+	le?vperm	$out0,$out0,$out0,$leperm
+	vmr		$twk0,$twk2		# unused tweak
+	vxor		$tmp,$out1,$twk2	# last block prep for stealing
+	le?vperm	$out1,$out1,$out1,$leperm
+	stvx_u		$out0,$x00,$out		# store output
+	stvx_u		$out1,$x10,$out
+	addi		$out,$out,0x20
+	bne		Lxts_enc6x_steal
+	b		Lxts_enc6x_done
+
+.align	4
+Lxts_enc6x_one:
+	vxor		$out0,$in5,$twk0
+	nop
+Loop_xts_enc1x:
+	vcipher		$out0,$out0,v24
+	lvx		v24,$x20,$key_		# round[3]
+	addi		$key_,$key_,0x20
+
+	vcipher		$out0,$out0,v25
+	lvx		v25,$x10,$key_		# round[4]
+	bdnz		Loop_xts_enc1x
+
+	add		$inp,$inp,$taillen
+	cmpwi		$taillen,0
+	vcipher		$out0,$out0,v24
+
+	subi		$inp,$inp,16
+	vcipher		$out0,$out0,v25
+
+	lvsr		$inpperm,0,$taillen
+	vcipher		$out0,$out0,v26
+
+	lvx_u		$in0,0,$inp
+	vcipher		$out0,$out0,v27
+
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
+	vcipher		$out0,$out0,v28
+	lvx		v24,$x00,$key_		# re-pre-load round[1]
+
+	vcipher		$out0,$out0,v29
+	lvx		v25,$x10,$key_		# re-pre-load round[2]
+	 vxor		$twk0,$twk0,v31
+
+	le?vperm	$in0,$in0,$in0,$leperm
+	vcipher		$out0,$out0,v30
+
+	vperm		$in0,$in0,$in0,$inpperm
+	vcipherlast	$out0,$out0,$twk0
+
+	vmr		$twk0,$twk1		# unused tweak
+	vxor		$tmp,$out0,$twk1	# last block prep for stealing
+	le?vperm	$out0,$out0,$out0,$leperm
+	stvx_u		$out0,$x00,$out		# store output
+	addi		$out,$out,0x10
+	bne		Lxts_enc6x_steal
+	b		Lxts_enc6x_done
+
+.align	4
+Lxts_enc6x_zero:
+	cmpwi		$taillen,0
+	beq		Lxts_enc6x_done
+
+	add		$inp,$inp,$taillen
+	subi		$inp,$inp,16
+	lvx_u		$in0,0,$inp
+	lvsr		$inpperm,0,$taillen	# $in5 is no more
+	le?vperm	$in0,$in0,$in0,$leperm
+	vperm		$in0,$in0,$in0,$inpperm
+	vxor		$tmp,$tmp,$twk0
+Lxts_enc6x_steal:
+	vxor		$in0,$in0,$twk0
+	vxor		$out0,$out0,$out0
+	vspltisb	$out1,-1
+	vperm		$out0,$out0,$out1,$inpperm
+	vsel		$out0,$in0,$tmp,$out0	# $tmp is last block, remember?
+
+	subi		r30,$out,17
+	subi		$out,$out,16
+	mtctr		$taillen
+Loop_xts_enc6x_steal:
+	lbzu		r0,1(r30)
+	stb		r0,16(r30)
+	bdnz		Loop_xts_enc6x_steal
+
+	li		$taillen,0
+	mtctr		$rounds
+	b		Loop_xts_enc1x		# one more time...
+
+.align	4
+Lxts_enc6x_done:
+	${UCMP}i	$ivp,0
+	beq		Lxts_enc6x_ret
+
+	vxor		$tweak,$twk0,$rndkey0
+	le?vperm	$tweak,$tweak,$tweak,$leperm
+	stvx_u		$tweak,0,$ivp
+
+Lxts_enc6x_ret:
+	mtlr		r11
+	li		r10,`$FRAME+15`
+	li		r11,`$FRAME+31`
+	stvx		$seven,r10,$sp		# wipe copies of round keys
+	addi		r10,r10,32
+	stvx		$seven,r11,$sp
+	addi		r11,r11,32
+	stvx		$seven,r10,$sp
+	addi		r10,r10,32
+	stvx		$seven,r11,$sp
+	addi		r11,r11,32
+	stvx		$seven,r10,$sp
+	addi		r10,r10,32
+	stvx		$seven,r11,$sp
+	addi		r11,r11,32
+	stvx		$seven,r10,$sp
+	addi		r10,r10,32
+	stvx		$seven,r11,$sp
+	addi		r11,r11,32
+
+	mtspr		256,$vrsave
+	lvx		v20,r10,$sp		# ABI says so
+	addi		r10,r10,32
+	lvx		v21,r11,$sp
+	addi		r11,r11,32
+	lvx		v22,r10,$sp
+	addi		r10,r10,32
+	lvx		v23,r11,$sp
+	addi		r11,r11,32
+	lvx		v24,r10,$sp
+	addi		r10,r10,32
+	lvx		v25,r11,$sp
+	addi		r11,r11,32
+	lvx		v26,r10,$sp
+	addi		r10,r10,32
+	lvx		v27,r11,$sp
+	addi		r11,r11,32
+	lvx		v28,r10,$sp
+	addi		r10,r10,32
+	lvx		v29,r11,$sp
+	addi		r11,r11,32
+	lvx		v30,r10,$sp
+	lvx		v31,r11,$sp
+	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
+	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
+	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
+	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
+	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
+	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
+	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`
+	blr
+	.long		0
+	.byte		0,12,0x04,1,0x80,6,6,0
+	.long		0
+
+.align	5
+_aesp8_xts_enc5x:
+	vcipher		$out0,$out0,v24
+	vcipher		$out1,$out1,v24
+	vcipher		$out2,$out2,v24
+	vcipher		$out3,$out3,v24
+	vcipher		$out4,$out4,v24
+	lvx		v24,$x20,$key_		# round[3]
+	addi		$key_,$key_,0x20
+
+	vcipher		$out0,$out0,v25
+	vcipher		$out1,$out1,v25
+	vcipher		$out2,$out2,v25
+	vcipher		$out3,$out3,v25
+	vcipher		$out4,$out4,v25
+	lvx		v25,$x10,$key_		# round[4]
+	bdnz		_aesp8_xts_enc5x
+
+	add		$inp,$inp,$taillen
+	cmpwi		$taillen,0
+	vcipher		$out0,$out0,v24
+	vcipher		$out1,$out1,v24
+	vcipher		$out2,$out2,v24
+	vcipher		$out3,$out3,v24
+	vcipher		$out4,$out4,v24
+
+	subi		$inp,$inp,16
+	vcipher		$out0,$out0,v25
+	vcipher		$out1,$out1,v25
+	vcipher		$out2,$out2,v25
+	vcipher		$out3,$out3,v25
+	vcipher		$out4,$out4,v25
+	 vxor		$twk0,$twk0,v31
+
+	vcipher		$out0,$out0,v26
+	lvsr		$inpperm,r0,$taillen	# $in5 is no more
+	vcipher		$out1,$out1,v26
+	vcipher		$out2,$out2,v26
+	vcipher		$out3,$out3,v26
+	vcipher		$out4,$out4,v26
+	 vxor		$in1,$twk1,v31
+
+	vcipher		$out0,$out0,v27
+	lvx_u		$in0,0,$inp
+	vcipher		$out1,$out1,v27
+	vcipher		$out2,$out2,v27
+	vcipher		$out3,$out3,v27
+	vcipher		$out4,$out4,v27
+	 vxor		$in2,$twk2,v31
+
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
+	vcipher		$out0,$out0,v28
+	vcipher		$out1,$out1,v28
+	vcipher		$out2,$out2,v28
+	vcipher		$out3,$out3,v28
+	vcipher		$out4,$out4,v28
+	lvx		v24,$x00,$key_		# re-pre-load round[1]
+	 vxor		$in3,$twk3,v31
+
+	vcipher		$out0,$out0,v29
+	le?vperm	$in0,$in0,$in0,$leperm
+	vcipher		$out1,$out1,v29
+	vcipher		$out2,$out2,v29
+	vcipher		$out3,$out3,v29
+	vcipher		$out4,$out4,v29
+	lvx		v25,$x10,$key_		# re-pre-load round[2]
+	 vxor		$in4,$twk4,v31
+
+	vcipher		$out0,$out0,v30
+	vperm		$in0,$in0,$in0,$inpperm
+	vcipher		$out1,$out1,v30
+	vcipher		$out2,$out2,v30
+	vcipher		$out3,$out3,v30
+	vcipher		$out4,$out4,v30
+
+	vcipherlast	$out0,$out0,$twk0
+	vcipherlast	$out1,$out1,$in1
+	vcipherlast	$out2,$out2,$in2
+	vcipherlast	$out3,$out3,$in3
+	vcipherlast	$out4,$out4,$in4
+	blr
+        .long   	0
+        .byte   	0,12,0x14,0,0,0,0,0
+
+.align	5
+_aesp8_xts_decrypt6x:
+	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
+	mflr		r11
+	li		r7,`$FRAME+8*16+15`
+	li		r3,`$FRAME+8*16+31`
+	$PUSH		r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
+	stvx		v20,r7,$sp		# ABI says so
+	addi		r7,r7,32
+	stvx		v21,r3,$sp
+	addi		r3,r3,32
+	stvx		v22,r7,$sp
+	addi		r7,r7,32
+	stvx		v23,r3,$sp
+	addi		r3,r3,32
+	stvx		v24,r7,$sp
+	addi		r7,r7,32
+	stvx		v25,r3,$sp
+	addi		r3,r3,32
+	stvx		v26,r7,$sp
+	addi		r7,r7,32
+	stvx		v27,r3,$sp
+	addi		r3,r3,32
+	stvx		v28,r7,$sp
+	addi		r7,r7,32
+	stvx		v29,r3,$sp
+	addi		r3,r3,32
+	stvx		v30,r7,$sp
+	stvx		v31,r3,$sp
+	li		r0,-1
+	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
+	li		$x10,0x10
+	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
+	li		$x20,0x20
+	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
+	li		$x30,0x30
+	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
+	li		$x40,0x40
+	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
+	li		$x50,0x50
+	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
+	li		$x60,0x60
+	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
+	li		$x70,0x70
+	mtspr		256,r0
+
+	xxlor		2, 32+$eighty7, 32+$eighty7
+	vsldoi		$eighty7,$tmp,$eighty7,1        # 0x010101..87
+	xxlor		1, 32+$eighty7, 32+$eighty7
+
+	# Load XOR Lconsts.
+	mr		$x70, r6
+	bl		Lconsts
+	lxvw4x		0, $x40, r6		# load XOR contents
+	mr		r6, $x70
+	li		$x70,0x70
+
+	subi		$rounds,$rounds,3	# -4 in total
+
+	lvx		$rndkey0,$x00,$key1	# load key schedule
+	lvx		v30,$x10,$key1
+	addi		$key1,$key1,0x20
+	lvx		v31,$x00,$key1
+	?vperm		$rndkey0,$rndkey0,v30,$keyperm
+	addi		$key_,$sp,$FRAME+15
+	mtctr		$rounds
+
+Load_xts_dec_key:
+	?vperm		v24,v30,v31,$keyperm
+	lvx		v30,$x10,$key1
+	addi		$key1,$key1,0x20
+	stvx		v24,$x00,$key_		# off-load round[1]
+	?vperm		v25,v31,v30,$keyperm
+	lvx		v31,$x00,$key1
+	stvx		v25,$x10,$key_		# off-load round[2]
+	addi		$key_,$key_,0x20
+	bdnz		Load_xts_dec_key
+
+	lvx		v26,$x10,$key1
+	?vperm		v24,v30,v31,$keyperm
+	lvx		v27,$x20,$key1
+	stvx		v24,$x00,$key_		# off-load round[3]
+	?vperm		v25,v31,v26,$keyperm
+	lvx		v28,$x30,$key1
+	stvx		v25,$x10,$key_		# off-load round[4]
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
+	?vperm		v26,v26,v27,$keyperm
+	lvx		v29,$x40,$key1
+	?vperm		v27,v27,v28,$keyperm
+	lvx		v30,$x50,$key1
+	?vperm		v28,v28,v29,$keyperm
+	lvx		v31,$x60,$key1
+	?vperm		v29,v29,v30,$keyperm
+	lvx		$twk5,$x70,$key1	# borrow $twk5
+	?vperm		v30,v30,v31,$keyperm
+	lvx		v24,$x00,$key_		# pre-load round[1]
+	?vperm		v31,v31,$twk5,$keyperm
+	lvx		v25,$x10,$key_		# pre-load round[2]
+
+	 vperm		$in0,$inout,$inptail,$inpperm
+	 subi		$inp,$inp,31		# undo "caller"
+	vxor		$twk0,$tweak,$rndkey0
+	vsrab		$tmp,$tweak,$seven	# next tweak value
+	vaddubm		$tweak,$tweak,$tweak
+	vand		$tmp,$tmp,$eighty7
+	 vxor		$out0,$in0,$twk0
+	xxlor		32+$in1, 0, 0
+	vpermxor	$tweak, $tweak, $tmp, $in1
+
+	 lvx_u		$in1,$x10,$inp
+	vxor		$twk1,$tweak,$rndkey0
+	vsrab		$tmp,$tweak,$seven	# next tweak value
+	vaddubm		$tweak,$tweak,$tweak
+	 le?vperm	$in1,$in1,$in1,$leperm
+	vand		$tmp,$tmp,$eighty7
+	 vxor		$out1,$in1,$twk1
+	xxlor		32+$in2, 0, 0
+	vpermxor	$tweak, $tweak, $tmp, $in2
+
+	 lvx_u		$in2,$x20,$inp
+	 andi.		$taillen,$len,15
+	vxor		$twk2,$tweak,$rndkey0
+	vsrab		$tmp,$tweak,$seven	# next tweak value
+	vaddubm		$tweak,$tweak,$tweak
+	 le?vperm	$in2,$in2,$in2,$leperm
+	vand		$tmp,$tmp,$eighty7
+	 vxor		$out2,$in2,$twk2
+	xxlor		32+$in3, 0, 0
+	vpermxor	$tweak, $tweak, $tmp, $in3
+
+	 lvx_u		$in3,$x30,$inp
+	 sub		$len,$len,$taillen
+	vxor		$twk3,$tweak,$rndkey0
+	vsrab		$tmp,$tweak,$seven	# next tweak value
+	vaddubm		$tweak,$tweak,$tweak
+	 le?vperm	$in3,$in3,$in3,$leperm
+	vand		$tmp,$tmp,$eighty7
+	 vxor		$out3,$in3,$twk3
+	xxlor		32+$in4, 0, 0
+	vpermxor	$tweak, $tweak, $tmp, $in4
+
+	 lvx_u		$in4,$x40,$inp
+	 subi		$len,$len,0x60
+	vxor		$twk4,$tweak,$rndkey0
+	vsrab		$tmp,$tweak,$seven	# next tweak value
+	vaddubm		$tweak,$tweak,$tweak
+	 le?vperm	$in4,$in4,$in4,$leperm
+	vand		$tmp,$tmp,$eighty7
+	 vxor		$out4,$in4,$twk4
+	xxlor		32+$in5, 0, 0
+	vpermxor	$tweak, $tweak, $tmp, $in5
+
+	 lvx_u		$in5,$x50,$inp
+	 addi		$inp,$inp,0x60
+	vxor		$twk5,$tweak,$rndkey0
+	vsrab		$tmp,$tweak,$seven	# next tweak value
+	vaddubm		$tweak,$tweak,$tweak
+	 le?vperm	$in5,$in5,$in5,$leperm
+	vand		$tmp,$tmp,$eighty7
+	 vxor		$out5,$in5,$twk5
+	xxlor		32+$in0, 0, 0
+	vpermxor	$tweak, $tweak, $tmp, $in0
+
+	vxor		v31,v31,$rndkey0
+	mtctr		$rounds
+	b		Loop_xts_dec6x
+
+.align	5
+Loop_xts_dec6x:
+	vncipher	$out0,$out0,v24
+	vncipher	$out1,$out1,v24
+	vncipher	$out2,$out2,v24
+	vncipher	$out3,$out3,v24
+	vncipher	$out4,$out4,v24
+	vncipher	$out5,$out5,v24
+	lvx		v24,$x20,$key_		# round[3]
+	addi		$key_,$key_,0x20
+
+	vncipher	$out0,$out0,v25
+	vncipher	$out1,$out1,v25
+	vncipher	$out2,$out2,v25
+	vncipher	$out3,$out3,v25
+	vncipher	$out4,$out4,v25
+	vncipher	$out5,$out5,v25
+	lvx		v25,$x10,$key_		# round[4]
+	bdnz		Loop_xts_dec6x
+
+	xxlor		32+$eighty7, 1, 1	# 0x010101..87
+
+	subic		$len,$len,96		# $len-=96
+	 vxor		$in0,$twk0,v31		# xor with last round key
+	vncipher	$out0,$out0,v24
+	vncipher	$out1,$out1,v24
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
+	 vxor		$twk0,$tweak,$rndkey0
+	 vaddubm	$tweak,$tweak,$tweak
+	vncipher	$out2,$out2,v24
+	vncipher	$out3,$out3,v24
+	vncipher	$out4,$out4,v24
+	vncipher	$out5,$out5,v24
+
+	subfe.		r0,r0,r0		# borrow?-1:0
+	 vand		$tmp,$tmp,$eighty7
+	vncipher	$out0,$out0,v25
+	vncipher	$out1,$out1,v25
+	 xxlor		32+$in1, 0, 0
+	 vpermxor	$tweak, $tweak, $tmp, $in1
+	vncipher	$out2,$out2,v25
+	vncipher	$out3,$out3,v25
+	 vxor		$in1,$twk1,v31
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
+	 vxor		$twk1,$tweak,$rndkey0
+	vncipher	$out4,$out4,v25
+	vncipher	$out5,$out5,v25
+
+	and		r0,r0,$len
+	 vaddubm	$tweak,$tweak,$tweak
+	vncipher	$out0,$out0,v26
+	vncipher	$out1,$out1,v26
+	 vand		$tmp,$tmp,$eighty7
+	vncipher	$out2,$out2,v26
+	vncipher	$out3,$out3,v26
+	 xxlor		32+$in2, 0, 0
+	 vpermxor	$tweak, $tweak, $tmp, $in2
+	vncipher	$out4,$out4,v26
+	vncipher	$out5,$out5,v26
+
+	add		$inp,$inp,r0		# $inp is adjusted in such
+						# way that at exit from the
+						# loop inX-in5 are loaded
+						# with last "words"
+	 vxor		$in2,$twk2,v31
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
+	 vxor		$twk2,$tweak,$rndkey0
+	 vaddubm	$tweak,$tweak,$tweak
+	vncipher	$out0,$out0,v27
+	vncipher	$out1,$out1,v27
+	vncipher	$out2,$out2,v27
+	vncipher	$out3,$out3,v27
+	 vand		$tmp,$tmp,$eighty7
+	vncipher	$out4,$out4,v27
+	vncipher	$out5,$out5,v27
+
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
+	 xxlor		32+$in3, 0, 0
+	 vpermxor	$tweak, $tweak, $tmp, $in3
+	vncipher	$out0,$out0,v28
+	vncipher	$out1,$out1,v28
+	 vxor		$in3,$twk3,v31
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
+	 vxor		$twk3,$tweak,$rndkey0
+	vncipher	$out2,$out2,v28
+	vncipher	$out3,$out3,v28
+	 vaddubm	$tweak,$tweak,$tweak
+	vncipher	$out4,$out4,v28
+	vncipher	$out5,$out5,v28
+	lvx		v24,$x00,$key_		# re-pre-load round[1]
+	 vand		$tmp,$tmp,$eighty7
+
+	vncipher	$out0,$out0,v29
+	vncipher	$out1,$out1,v29
+	 xxlor		32+$in4, 0, 0
+	 vpermxor	$tweak, $tweak, $tmp, $in4
+	vncipher	$out2,$out2,v29
+	vncipher	$out3,$out3,v29
+	 vxor		$in4,$twk4,v31
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
+	 vxor		$twk4,$tweak,$rndkey0
+	vncipher	$out4,$out4,v29
+	vncipher	$out5,$out5,v29
+	lvx		v25,$x10,$key_		# re-pre-load round[2]
+	 vaddubm	$tweak,$tweak,$tweak
+
+	vncipher	$out0,$out0,v30
+	vncipher	$out1,$out1,v30
+	 vand		$tmp,$tmp,$eighty7
+	vncipher	$out2,$out2,v30
+	vncipher	$out3,$out3,v30
+	 xxlor		32+$in5, 0, 0
+	 vpermxor	$tweak, $tweak, $tmp, $in5
+	vncipher	$out4,$out4,v30
+	vncipher	$out5,$out5,v30
+	 vxor		$in5,$twk5,v31
+	 vsrab		$tmp,$tweak,$seven	# next tweak value
+	 vxor		$twk5,$tweak,$rndkey0
+
+	vncipherlast	$out0,$out0,$in0
+	 lvx_u		$in0,$x00,$inp		# load next input block
+	 vaddubm	$tweak,$tweak,$tweak
+	vncipherlast	$out1,$out1,$in1
+	 lvx_u		$in1,$x10,$inp
+	vncipherlast	$out2,$out2,$in2
+	 le?vperm	$in0,$in0,$in0,$leperm
+	 lvx_u		$in2,$x20,$inp
+	 vand		$tmp,$tmp,$eighty7
+	vncipherlast	$out3,$out3,$in3
+	 le?vperm	$in1,$in1,$in1,$leperm
+	 lvx_u		$in3,$x30,$inp
+	vncipherlast	$out4,$out4,$in4
+	 le?vperm	$in2,$in2,$in2,$leperm
+	 lvx_u		$in4,$x40,$inp
+	 xxlor		10, 32+$in0, 32+$in0
+	 xxlor		32+$in0, 0, 0
+	 vpermxor	$tweak, $tweak, $tmp, $in0
+	 xxlor		32+$in0, 10, 10
+	vncipherlast	$out5,$out5,$in5
+	 le?vperm	$in3,$in3,$in3,$leperm
+	 lvx_u		$in5,$x50,$inp
+	 addi		$inp,$inp,0x60
+	 le?vperm	$in4,$in4,$in4,$leperm
+	 le?vperm	$in5,$in5,$in5,$leperm
+
+	le?vperm	$out0,$out0,$out0,$leperm
+	le?vperm	$out1,$out1,$out1,$leperm
+	stvx_u		$out0,$x00,$out		# store output
+	 vxor		$out0,$in0,$twk0
+	le?vperm	$out2,$out2,$out2,$leperm
+	stvx_u		$out1,$x10,$out
+	 vxor		$out1,$in1,$twk1
+	le?vperm	$out3,$out3,$out3,$leperm
+	stvx_u		$out2,$x20,$out
+	 vxor		$out2,$in2,$twk2
+	le?vperm	$out4,$out4,$out4,$leperm
+	stvx_u		$out3,$x30,$out
+	 vxor		$out3,$in3,$twk3
+	le?vperm	$out5,$out5,$out5,$leperm
+	stvx_u		$out4,$x40,$out
+	 vxor		$out4,$in4,$twk4
+	stvx_u		$out5,$x50,$out
+	 vxor		$out5,$in5,$twk5
+	addi		$out,$out,0x60
+
+	mtctr		$rounds
+	beq		Loop_xts_dec6x		# did $len-=96 borrow?
+
+	xxlor		32+$eighty7, 2, 2	# 0x010101..87
+
+	addic.		$len,$len,0x60
+	beq		Lxts_dec6x_zero
+	cmpwi		$len,0x20
+	blt		Lxts_dec6x_one
+	nop
+	beq		Lxts_dec6x_two
+	cmpwi		$len,0x40
+	blt		Lxts_dec6x_three
+	nop
+	beq		Lxts_dec6x_four
+
+Lxts_dec6x_five:
+	vxor		$out0,$in1,$twk0
+	vxor		$out1,$in2,$twk1
+	vxor		$out2,$in3,$twk2
+	vxor		$out3,$in4,$twk3
+	vxor		$out4,$in5,$twk4
+
+	bl		_aesp8_xts_dec5x
+
+	le?vperm	$out0,$out0,$out0,$leperm
+	vmr		$twk0,$twk5		# unused tweak
+	vxor		$twk1,$tweak,$rndkey0
+	le?vperm	$out1,$out1,$out1,$leperm
+	stvx_u		$out0,$x00,$out		# store output
+	vxor		$out0,$in0,$twk1
+	le?vperm	$out2,$out2,$out2,$leperm
+	stvx_u		$out1,$x10,$out
+	le?vperm	$out3,$out3,$out3,$leperm
+	stvx_u		$out2,$x20,$out
+	le?vperm	$out4,$out4,$out4,$leperm
+	stvx_u		$out3,$x30,$out
+	stvx_u		$out4,$x40,$out
+	addi		$out,$out,0x50
+	bne		Lxts_dec6x_steal
+	b		Lxts_dec6x_done
+
+.align	4
+Lxts_dec6x_four:
+	vxor		$out0,$in2,$twk0
+	vxor		$out1,$in3,$twk1
+	vxor		$out2,$in4,$twk2
+	vxor		$out3,$in5,$twk3
+	vxor		$out4,$out4,$out4
+
+	bl		_aesp8_xts_dec5x
+
+	le?vperm	$out0,$out0,$out0,$leperm
+	vmr		$twk0,$twk4		# unused tweak
+	vmr		$twk1,$twk5
+	le?vperm	$out1,$out1,$out1,$leperm
+	stvx_u		$out0,$x00,$out		# store output
+	vxor		$out0,$in0,$twk5
+	le?vperm	$out2,$out2,$out2,$leperm
+	stvx_u		$out1,$x10,$out
+	le?vperm	$out3,$out3,$out3,$leperm
+	stvx_u		$out2,$x20,$out
+	stvx_u		$out3,$x30,$out
+	addi		$out,$out,0x40
+	bne		Lxts_dec6x_steal
+	b		Lxts_dec6x_done
+
+.align	4
+Lxts_dec6x_three:
+	vxor		$out0,$in3,$twk0
+	vxor		$out1,$in4,$twk1
+	vxor		$out2,$in5,$twk2
+	vxor		$out3,$out3,$out3
+	vxor		$out4,$out4,$out4
+
+	bl		_aesp8_xts_dec5x
+
+	le?vperm	$out0,$out0,$out0,$leperm
+	vmr		$twk0,$twk3		# unused tweak
+	vmr		$twk1,$twk4
+	le?vperm	$out1,$out1,$out1,$leperm
+	stvx_u		$out0,$x00,$out		# store output
+	vxor		$out0,$in0,$twk4
+	le?vperm	$out2,$out2,$out2,$leperm
+	stvx_u		$out1,$x10,$out
+	stvx_u		$out2,$x20,$out
+	addi		$out,$out,0x30
+	bne		Lxts_dec6x_steal
+	b		Lxts_dec6x_done
+
+.align	4
+Lxts_dec6x_two:
+	vxor		$out0,$in4,$twk0
+	vxor		$out1,$in5,$twk1
+	vxor		$out2,$out2,$out2
+	vxor		$out3,$out3,$out3
+	vxor		$out4,$out4,$out4
+
+	bl		_aesp8_xts_dec5x
+
+	le?vperm	$out0,$out0,$out0,$leperm
+	vmr		$twk0,$twk2		# unused tweak
+	vmr		$twk1,$twk3
+	le?vperm	$out1,$out1,$out1,$leperm
+	stvx_u		$out0,$x00,$out		# store output
+	vxor		$out0,$in0,$twk3
+	stvx_u		$out1,$x10,$out
+	addi		$out,$out,0x20
+	bne		Lxts_dec6x_steal
+	b		Lxts_dec6x_done
+
+.align	4
+Lxts_dec6x_one:
+	vxor		$out0,$in5,$twk0
+	nop
+Loop_xts_dec1x:
+	vncipher	$out0,$out0,v24
+	lvx		v24,$x20,$key_		# round[3]
+	addi		$key_,$key_,0x20
+
+	vncipher	$out0,$out0,v25
+	lvx		v25,$x10,$key_		# round[4]
+	bdnz		Loop_xts_dec1x
+
+	subi		r0,$taillen,1
+	vncipher	$out0,$out0,v24
+
+	andi.		r0,r0,16
+	cmpwi		$taillen,0
+	vncipher	$out0,$out0,v25
+
+	sub		$inp,$inp,r0
+	vncipher	$out0,$out0,v26
+
+	lvx_u		$in0,0,$inp
+	vncipher	$out0,$out0,v27
+
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
+	vncipher	$out0,$out0,v28
+	lvx		v24,$x00,$key_		# re-pre-load round[1]
+
+	vncipher	$out0,$out0,v29
+	lvx		v25,$x10,$key_		# re-pre-load round[2]
+	 vxor		$twk0,$twk0,v31
+
+	le?vperm	$in0,$in0,$in0,$leperm
+	vncipher	$out0,$out0,v30
+
+	mtctr		$rounds
+	vncipherlast	$out0,$out0,$twk0
+
+	vmr		$twk0,$twk1		# unused tweak
+	vmr		$twk1,$twk2
+	le?vperm	$out0,$out0,$out0,$leperm
+	stvx_u		$out0,$x00,$out		# store output
+	addi		$out,$out,0x10
+	vxor		$out0,$in0,$twk2
+	bne		Lxts_dec6x_steal
+	b		Lxts_dec6x_done
+
+.align	4
+Lxts_dec6x_zero:
+	cmpwi		$taillen,0
+	beq		Lxts_dec6x_done
+
+	lvx_u		$in0,0,$inp
+	le?vperm	$in0,$in0,$in0,$leperm
+	vxor		$out0,$in0,$twk1
+Lxts_dec6x_steal:
+	vncipher	$out0,$out0,v24
+	lvx		v24,$x20,$key_		# round[3]
+	addi		$key_,$key_,0x20
+
+	vncipher	$out0,$out0,v25
+	lvx		v25,$x10,$key_		# round[4]
+	bdnz		Lxts_dec6x_steal
+
+	add		$inp,$inp,$taillen
+	vncipher	$out0,$out0,v24
+
+	cmpwi		$taillen,0
+	vncipher	$out0,$out0,v25
+
+	lvx_u		$in0,0,$inp
+	vncipher	$out0,$out0,v26
+
+	lvsr		$inpperm,0,$taillen	# $in5 is no more
+	vncipher	$out0,$out0,v27
+
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
+	vncipher	$out0,$out0,v28
+	lvx		v24,$x00,$key_		# re-pre-load round[1]
+
+	vncipher	$out0,$out0,v29
+	lvx		v25,$x10,$key_		# re-pre-load round[2]
+	 vxor		$twk1,$twk1,v31
+
+	le?vperm	$in0,$in0,$in0,$leperm
+	vncipher	$out0,$out0,v30
+
+	vperm		$in0,$in0,$in0,$inpperm
+	vncipherlast	$tmp,$out0,$twk1
+
+	le?vperm	$out0,$tmp,$tmp,$leperm
+	le?stvx_u	$out0,0,$out
+	be?stvx_u	$tmp,0,$out
+
+	vxor		$out0,$out0,$out0
+	vspltisb	$out1,-1
+	vperm		$out0,$out0,$out1,$inpperm
+	vsel		$out0,$in0,$tmp,$out0
+	vxor		$out0,$out0,$twk0
+
+	subi		r30,$out,1
+	mtctr		$taillen
+Loop_xts_dec6x_steal:
+	lbzu		r0,1(r30)
+	stb		r0,16(r30)
+	bdnz		Loop_xts_dec6x_steal
+
+	li		$taillen,0
+	mtctr		$rounds
+	b		Loop_xts_dec1x		# one more time...
+
+.align	4
+Lxts_dec6x_done:
+	${UCMP}i	$ivp,0
+	beq		Lxts_dec6x_ret
+
+	vxor		$tweak,$twk0,$rndkey0
+	le?vperm	$tweak,$tweak,$tweak,$leperm
+	stvx_u		$tweak,0,$ivp
+
+Lxts_dec6x_ret:
+	mtlr		r11
+	li		r10,`$FRAME+15`
+	li		r11,`$FRAME+31`
+	stvx		$seven,r10,$sp		# wipe copies of round keys
+	addi		r10,r10,32
+	stvx		$seven,r11,$sp
+	addi		r11,r11,32
+	stvx		$seven,r10,$sp
+	addi		r10,r10,32
+	stvx		$seven,r11,$sp
+	addi		r11,r11,32
+	stvx		$seven,r10,$sp
+	addi		r10,r10,32
+	stvx		$seven,r11,$sp
+	addi		r11,r11,32
+	stvx		$seven,r10,$sp
+	addi		r10,r10,32
+	stvx		$seven,r11,$sp
+	addi		r11,r11,32
+
+	mtspr		256,$vrsave
+	lvx		v20,r10,$sp		# ABI says so
+	addi		r10,r10,32
+	lvx		v21,r11,$sp
+	addi		r11,r11,32
+	lvx		v22,r10,$sp
+	addi		r10,r10,32
+	lvx		v23,r11,$sp
+	addi		r11,r11,32
+	lvx		v24,r10,$sp
+	addi		r10,r10,32
+	lvx		v25,r11,$sp
+	addi		r11,r11,32
+	lvx		v26,r10,$sp
+	addi		r10,r10,32
+	lvx		v27,r11,$sp
+	addi		r11,r11,32
+	lvx		v28,r10,$sp
+	addi		r10,r10,32
+	lvx		v29,r11,$sp
+	addi		r11,r11,32
+	lvx		v30,r10,$sp
+	lvx		v31,r11,$sp
+	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
+	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
+	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
+	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
+	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
+	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
+	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`
+	blr
+	.long		0
+	.byte		0,12,0x04,1,0x80,6,6,0
+	.long		0
+
+.align	5
+_aesp8_xts_dec5x:
+	vncipher	$out0,$out0,v24
+	vncipher	$out1,$out1,v24
+	vncipher	$out2,$out2,v24
+	vncipher	$out3,$out3,v24
+	vncipher	$out4,$out4,v24
+	lvx		v24,$x20,$key_		# round[3]
+	addi		$key_,$key_,0x20
+
+	vncipher	$out0,$out0,v25
+	vncipher	$out1,$out1,v25
+	vncipher	$out2,$out2,v25
+	vncipher	$out3,$out3,v25
+	vncipher	$out4,$out4,v25
+	lvx		v25,$x10,$key_		# round[4]
+	bdnz		_aesp8_xts_dec5x
+
+	subi		r0,$taillen,1
+	vncipher	$out0,$out0,v24
+	vncipher	$out1,$out1,v24
+	vncipher	$out2,$out2,v24
+	vncipher	$out3,$out3,v24
+	vncipher	$out4,$out4,v24
+
+	andi.		r0,r0,16
+	cmpwi		$taillen,0
+	vncipher	$out0,$out0,v25
+	vncipher	$out1,$out1,v25
+	vncipher	$out2,$out2,v25
+	vncipher	$out3,$out3,v25
+	vncipher	$out4,$out4,v25
+	 vxor		$twk0,$twk0,v31
+
+	sub		$inp,$inp,r0
+	vncipher	$out0,$out0,v26
+	vncipher	$out1,$out1,v26
+	vncipher	$out2,$out2,v26
+	vncipher	$out3,$out3,v26
+	vncipher	$out4,$out4,v26
+	 vxor		$in1,$twk1,v31
+
+	vncipher	$out0,$out0,v27
+	lvx_u		$in0,0,$inp
+	vncipher	$out1,$out1,v27
+	vncipher	$out2,$out2,v27
+	vncipher	$out3,$out3,v27
+	vncipher	$out4,$out4,v27
+	 vxor		$in2,$twk2,v31
+
+	addi		$key_,$sp,$FRAME+15	# rewind $key_
+	vncipher	$out0,$out0,v28
+	vncipher	$out1,$out1,v28
+	vncipher	$out2,$out2,v28
+	vncipher	$out3,$out3,v28
+	vncipher	$out4,$out4,v28
+	lvx		v24,$x00,$key_		# re-pre-load round[1]
+	 vxor		$in3,$twk3,v31
+
+	vncipher	$out0,$out0,v29
+	le?vperm	$in0,$in0,$in0,$leperm
+	vncipher	$out1,$out1,v29
+	vncipher	$out2,$out2,v29
+	vncipher	$out3,$out3,v29
+	vncipher	$out4,$out4,v29
+	lvx		v25,$x10,$key_		# re-pre-load round[2]
+	 vxor		$in4,$twk4,v31
+
+	vncipher	$out0,$out0,v30
+	vncipher	$out1,$out1,v30
+	vncipher	$out2,$out2,v30
+	vncipher	$out3,$out3,v30
+	vncipher	$out4,$out4,v30
+
+	vncipherlast	$out0,$out0,$twk0
+	vncipherlast	$out1,$out1,$in1
+	vncipherlast	$out2,$out2,$in2
+	vncipherlast	$out3,$out3,$in3
+	vncipherlast	$out4,$out4,$in4
+	mtctr		$rounds
+	blr
+        .long   	0
+        .byte   	0,12,0x14,0,0,0,0,0
+___
+}}	}}}
+
+my $consts=1;
+foreach(split("\n",$code)) {
+        s/\`([^\`]*)\`/eval($1)/geo;
+
+	# constants table endian-specific conversion
+	if ($consts && m/\.(long|byte)\s+(.+)\s+(\?[a-z]*)$/o) {
+	    my $conv=$3;
+	    my @bytes=();
+
+	    # convert to endian-agnostic format
+	    if ($1 eq "long") {
+	      foreach (split(/,\s*/,$2)) {
+		my $l = /^0/?oct:int;
+		push @bytes,($l>>24)&0xff,($l>>16)&0xff,($l>>8)&0xff,$l&0xff;
+	      }
+	    } else {
+		@bytes = map(/^0/?oct:int,split(/,\s*/,$2));
+	    }
+
+	    # little-endian conversion
+	    if ($flavour =~ /le$/o) {
+		SWITCH: for($conv)  {
+		    /\?inv/ && do   { @bytes=map($_^0xf,@bytes); last; };
+		    /\?rev/ && do   { @bytes=reverse(@bytes);    last; };
+		}
+	    }
+
+	    #emit
+	    print ".byte\t",join(',',map (sprintf("0x%02x",$_),@bytes)),"\n";
+	    next;
+	}
+	$consts=0 if (m/Lconsts:/o);	# end of table
+
+	# instructions prefixed with '?' are endian-specific and need
+	# to be adjusted accordingly...
+	if ($flavour =~ /le$/o) {	# little-endian
+	    s/le\?//o		or
+	    s/be\?/#be#/o	or
+	    s/\?lvsr/lvsl/o	or
+	    s/\?lvsl/lvsr/o	or
+	    s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/o or
+	    s/\?(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/o or
+	    s/\?(vspltw\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9])/$1$2 3-$3/o;
+	} else {			# big-endian
+	    s/le\?/#le#/o	or
+	    s/be\?//o		or
+	    s/\?([a-z]+)/$1/o;
+	}
+
+        print $_,"\n";
+}
+
+close STDOUT;
diff --git a/lib/crypto/riscv/.gitignore b/lib/crypto/riscv/.gitignore
new file mode 100644
index 000000000000..0d47d4f21c6d
--- /dev/null
+++ b/lib/crypto/riscv/.gitignore
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0-only
+poly1305-core.S
diff --git a/lib/crypto/riscv/aes-riscv64-zvkned.S b/lib/crypto/riscv/aes-riscv64-zvkned.S
new file mode 100644
index 000000000000..0d988bc3d37b
--- /dev/null
+++ b/lib/crypto/riscv/aes-riscv64-zvkned.S
@@ -0,0 +1,84 @@
+/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */
+//
+// This file is dual-licensed, meaning that you can use it under your
+// choice of either of the following two licenses:
+//
+// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
+//
+// Licensed under the Apache License 2.0 (the "License"). You can obtain
+// a copy in the file LICENSE in the source distribution or at
+// https://www.openssl.org/source/license.html
+//
+// or
+//
+// Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu>
+// Copyright (c) 2023, Phoebe Chen <phoebe.chen@sifive.com>
+// Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com>
+// Copyright 2024 Google LLC
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The generated code of this file depends on the following RISC-V extensions:
+// - RV64I
+// - RISC-V Vector ('V') with VLEN >= 128
+// - RISC-V Vector AES block cipher extension ('Zvkned')
+
+#include <linux/linkage.h>
+
+.text
+.option arch, +zvkned
+
+#include "../../arch/riscv/crypto/aes-macros.S"
+
+#define RNDKEYS		a0
+#define KEY_LEN		a1
+#define OUTP		a2
+#define INP		a3
+
+.macro	__aes_crypt_zvkned	enc, keybits
+	vle32.v		v16, (INP)
+	aes_crypt	v16, \enc, \keybits
+	vse32.v		v16, (OUTP)
+	ret
+.endm
+
+.macro	aes_crypt_zvkned	enc
+	aes_begin	RNDKEYS, 128f, 192f, KEY_LEN
+	__aes_crypt_zvkned	\enc, 256
+128:
+	__aes_crypt_zvkned	\enc, 128
+192:
+	__aes_crypt_zvkned	\enc, 192
+.endm
+
+// void aes_encrypt_zvkned(const u32 rndkeys[], int key_len,
+//			   u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]);
+SYM_FUNC_START(aes_encrypt_zvkned)
+	aes_crypt_zvkned	1
+SYM_FUNC_END(aes_encrypt_zvkned)
+
+// void aes_decrypt_zvkned(const u32 rndkeys[], int key_len,
+//			   u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]);
+SYM_FUNC_START(aes_decrypt_zvkned)
+	aes_crypt_zvkned	0
+SYM_FUNC_END(aes_decrypt_zvkned)
diff --git a/lib/crypto/riscv/aes.h b/lib/crypto/riscv/aes.h
new file mode 100644
index 000000000000..0b26f58faf2b
--- /dev/null
+++ b/lib/crypto/riscv/aes.h
@@ -0,0 +1,63 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2023 VRULL GmbH
+ * Copyright (C) 2023 SiFive, Inc.
+ * Copyright 2024 Google LLC
+ */
+
+#include <asm/simd.h>
+#include <asm/vector.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkned);
+
+void aes_encrypt_zvkned(const u32 rndkeys[], int key_len,
+			u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]);
+void aes_decrypt_zvkned(const u32 rndkeys[], int key_len,
+			u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]);
+
+static void aes_preparekey_arch(union aes_enckey_arch *k,
+				union aes_invkey_arch *inv_k,
+				const u8 *in_key, int key_len, int nrounds)
+{
+	aes_expandkey_generic(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL,
+			      in_key, key_len);
+}
+
+static void aes_encrypt_arch(const struct aes_enckey *key,
+			     u8 out[AES_BLOCK_SIZE],
+			     const u8 in[AES_BLOCK_SIZE])
+{
+	if (static_branch_likely(&have_zvkned) && likely(may_use_simd())) {
+		kernel_vector_begin();
+		aes_encrypt_zvkned(key->k.rndkeys, key->len, out, in);
+		kernel_vector_end();
+	} else {
+		aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in);
+	}
+}
+
+static void aes_decrypt_arch(const struct aes_key *key,
+			     u8 out[AES_BLOCK_SIZE],
+			     const u8 in[AES_BLOCK_SIZE])
+{
+	/*
+	 * Note that the Zvkned code uses the standard round keys, while the
+	 * fallback uses the inverse round keys.  Thus both must be present.
+	 */
+	if (static_branch_likely(&have_zvkned) && likely(may_use_simd())) {
+		kernel_vector_begin();
+		aes_decrypt_zvkned(key->k.rndkeys, key->len, out, in);
+		kernel_vector_end();
+	} else {
+		aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds,
+				    out, in);
+	}
+}
+
+#define aes_mod_init_arch aes_mod_init_arch
+static void aes_mod_init_arch(void)
+{
+	if (riscv_isa_extension_available(NULL, ZVKNED) &&
+	    riscv_vector_vlen() >= 128)
+		static_branch_enable(&have_zvkned);
+}
diff --git a/lib/crypto/riscv/chacha-riscv64-zvkb.S b/lib/crypto/riscv/chacha-riscv64-zvkb.S
index b777d0b4e379..3d183ec818f5 100644
--- a/lib/crypto/riscv/chacha-riscv64-zvkb.S
+++ b/lib/crypto/riscv/chacha-riscv64-zvkb.S
@@ -60,7 +60,8 @@
 #define VL		t2
 #define STRIDE		t3
 #define ROUND_CTR	t4
-#define KEY0		s0
+#define KEY0		t5
+// Avoid s0/fp to allow for unwinding
 #define KEY1		s1
 #define KEY2		s2
 #define KEY3		s3
@@ -143,7 +144,6 @@
 // The updated 32-bit counter is written back to state->x[12] before returning.
 SYM_FUNC_START(chacha_zvkb)
 	addi		sp, sp, -96
-	sd		s0, 0(sp)
 	sd		s1, 8(sp)
 	sd		s2, 16(sp)
 	sd		s3, 24(sp)
@@ -280,7 +280,6 @@ SYM_FUNC_START(chacha_zvkb)
 	bnez		NBLOCKS, .Lblock_loop
 
 	sw		COUNTER, 48(STATEP)
-	ld		s0, 0(sp)
 	ld		s1, 8(sp)
 	ld		s2, 16(sp)
 	ld		s3, 24(sp)
diff --git a/lib/crypto/s390/aes.h b/lib/crypto/s390/aes.h
new file mode 100644
index 000000000000..5466f6ecbce7
--- /dev/null
+++ b/lib/crypto/s390/aes.h
@@ -0,0 +1,106 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * AES optimized using the CP Assist for Cryptographic Functions (CPACF)
+ *
+ * Copyright 2026 Google LLC
+ */
+#include <asm/cpacf.h>
+#include <linux/cpufeature.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_aes128);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_aes192);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_aes256);
+
+/*
+ * When the CPU supports CPACF AES for the requested key length, we need only
+ * save a copy of the raw AES key, as that's what the CPACF instructions need.
+ *
+ * When unsupported, fall back to the generic key expansion and en/decryption.
+ */
+static void aes_preparekey_arch(union aes_enckey_arch *k,
+				union aes_invkey_arch *inv_k,
+				const u8 *in_key, int key_len, int nrounds)
+{
+	if (key_len == AES_KEYSIZE_128) {
+		if (static_branch_likely(&have_cpacf_aes128)) {
+			memcpy(k->raw_key, in_key, AES_KEYSIZE_128);
+			return;
+		}
+	} else if (key_len == AES_KEYSIZE_192) {
+		if (static_branch_likely(&have_cpacf_aes192)) {
+			memcpy(k->raw_key, in_key, AES_KEYSIZE_192);
+			return;
+		}
+	} else {
+		if (static_branch_likely(&have_cpacf_aes256)) {
+			memcpy(k->raw_key, in_key, AES_KEYSIZE_256);
+			return;
+		}
+	}
+	aes_expandkey_generic(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL,
+			      in_key, key_len);
+}
+
+static inline bool aes_crypt_s390(const struct aes_enckey *key,
+				  u8 out[AES_BLOCK_SIZE],
+				  const u8 in[AES_BLOCK_SIZE], int decrypt)
+{
+	if (key->len == AES_KEYSIZE_128) {
+		if (static_branch_likely(&have_cpacf_aes128)) {
+			cpacf_km(CPACF_KM_AES_128 | decrypt,
+				 (void *)key->k.raw_key, out, in,
+				 AES_BLOCK_SIZE);
+			return true;
+		}
+	} else if (key->len == AES_KEYSIZE_192) {
+		if (static_branch_likely(&have_cpacf_aes192)) {
+			cpacf_km(CPACF_KM_AES_192 | decrypt,
+				 (void *)key->k.raw_key, out, in,
+				 AES_BLOCK_SIZE);
+			return true;
+		}
+	} else {
+		if (static_branch_likely(&have_cpacf_aes256)) {
+			cpacf_km(CPACF_KM_AES_256 | decrypt,
+				 (void *)key->k.raw_key, out, in,
+				 AES_BLOCK_SIZE);
+			return true;
+		}
+	}
+	return false;
+}
+
+static void aes_encrypt_arch(const struct aes_enckey *key,
+			     u8 out[AES_BLOCK_SIZE],
+			     const u8 in[AES_BLOCK_SIZE])
+{
+	if (likely(aes_crypt_s390(key, out, in, 0)))
+		return;
+	aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in);
+}
+
+static void aes_decrypt_arch(const struct aes_key *key,
+			     u8 out[AES_BLOCK_SIZE],
+			     const u8 in[AES_BLOCK_SIZE])
+{
+	if (likely(aes_crypt_s390((const struct aes_enckey *)key, out, in,
+				  CPACF_DECRYPT)))
+		return;
+	aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds, out, in);
+}
+
+#define aes_mod_init_arch aes_mod_init_arch
+static void aes_mod_init_arch(void)
+{
+	if (cpu_have_feature(S390_CPU_FEATURE_MSA)) {
+		cpacf_mask_t km_functions;
+
+		cpacf_query(CPACF_KM, &km_functions);
+		if (cpacf_test_func(&km_functions, CPACF_KM_AES_128))
+			static_branch_enable(&have_cpacf_aes128);
+		if (cpacf_test_func(&km_functions, CPACF_KM_AES_192))
+			static_branch_enable(&have_cpacf_aes192);
+		if (cpacf_test_func(&km_functions, CPACF_KM_AES_256))
+			static_branch_enable(&have_cpacf_aes256);
+	}
+}
diff --git a/lib/crypto/s390/sha3.h b/lib/crypto/s390/sha3.h
new file mode 100644
index 000000000000..85471404775a
--- /dev/null
+++ b/lib/crypto/s390/sha3.h
@@ -0,0 +1,151 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * SHA-3 optimized using the CP Assist for Cryptographic Functions (CPACF)
+ *
+ * Copyright 2025 Google LLC
+ */
+#include <asm/cpacf.h>
+#include <linux/cpufeature.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3_init_optim);
+
+static void sha3_absorb_blocks(struct sha3_state *state, const u8 *data,
+			       size_t nblocks, size_t block_size)
+{
+	if (static_branch_likely(&have_sha3)) {
+		/*
+		 * Note that KIMD assumes little-endian order of the state
+		 * words.  sha3_state already uses that order, though, so
+		 * there's no need for a byteswap.
+		 */
+		switch (block_size) {
+		case SHA3_224_BLOCK_SIZE:
+			cpacf_kimd(CPACF_KIMD_SHA3_224, state,
+				   data, nblocks * block_size);
+			return;
+		case SHA3_256_BLOCK_SIZE:
+			/*
+			 * This case handles both SHA3-256 and SHAKE256, since
+			 * they have the same block size.
+			 */
+			cpacf_kimd(CPACF_KIMD_SHA3_256, state,
+				   data, nblocks * block_size);
+			return;
+		case SHA3_384_BLOCK_SIZE:
+			cpacf_kimd(CPACF_KIMD_SHA3_384, state,
+				   data, nblocks * block_size);
+			return;
+		case SHA3_512_BLOCK_SIZE:
+			cpacf_kimd(CPACF_KIMD_SHA3_512, state,
+				   data, nblocks * block_size);
+			return;
+		}
+	}
+	sha3_absorb_blocks_generic(state, data, nblocks, block_size);
+}
+
+static void sha3_keccakf(struct sha3_state *state)
+{
+	if (static_branch_likely(&have_sha3)) {
+		/*
+		 * Passing zeroes into any of CPACF_KIMD_SHA3_* gives the plain
+		 * Keccak-f permutation, which is what we want here.  Use
+		 * SHA3-512 since it has the smallest block size.
+		 */
+		static const u8 zeroes[SHA3_512_BLOCK_SIZE];
+
+		cpacf_kimd(CPACF_KIMD_SHA3_512, state, zeroes, sizeof(zeroes));
+	} else {
+		sha3_keccakf_generic(state);
+	}
+}
+
+static inline bool s390_sha3(int func, const u8 *in, size_t in_len,
+			     u8 *out, size_t out_len)
+{
+	struct sha3_state state;
+
+	if (!static_branch_likely(&have_sha3))
+		return false;
+
+	if (static_branch_likely(&have_sha3_init_optim))
+		func |= CPACF_KLMD_NIP | CPACF_KLMD_DUFOP;
+	else
+		memset(&state, 0, sizeof(state));
+
+	cpacf_klmd(func, &state, in, in_len);
+
+	if (static_branch_likely(&have_sha3_init_optim))
+		kmsan_unpoison_memory(&state, out_len);
+
+	memcpy(out, &state, out_len);
+	memzero_explicit(&state, sizeof(state));
+	return true;
+}
+
+#define sha3_224_arch sha3_224_arch
+static bool sha3_224_arch(const u8 *in, size_t in_len,
+			  u8 out[SHA3_224_DIGEST_SIZE])
+{
+	return s390_sha3(CPACF_KLMD_SHA3_224, in, in_len,
+			 out, SHA3_224_DIGEST_SIZE);
+}
+
+#define sha3_256_arch sha3_256_arch
+static bool sha3_256_arch(const u8 *in, size_t in_len,
+			  u8 out[SHA3_256_DIGEST_SIZE])
+{
+	return s390_sha3(CPACF_KLMD_SHA3_256, in, in_len,
+			 out, SHA3_256_DIGEST_SIZE);
+}
+
+#define sha3_384_arch sha3_384_arch
+static bool sha3_384_arch(const u8 *in, size_t in_len,
+			  u8 out[SHA3_384_DIGEST_SIZE])
+{
+	return s390_sha3(CPACF_KLMD_SHA3_384, in, in_len,
+			 out, SHA3_384_DIGEST_SIZE);
+}
+
+#define sha3_512_arch sha3_512_arch
+static bool sha3_512_arch(const u8 *in, size_t in_len,
+			  u8 out[SHA3_512_DIGEST_SIZE])
+{
+	return s390_sha3(CPACF_KLMD_SHA3_512, in, in_len,
+			 out, SHA3_512_DIGEST_SIZE);
+}
+
+#define sha3_mod_init_arch sha3_mod_init_arch
+static void sha3_mod_init_arch(void)
+{
+	int num_present = 0;
+	int num_possible = 0;
+
+	if (!cpu_have_feature(S390_CPU_FEATURE_MSA))
+		return;
+	/*
+	 * Since all the SHA-3 functions are in Message-Security-Assist
+	 * Extension 6, just treat them as all or nothing.  This way we need
+	 * only one static_key.
+	 */
+#define QUERY(opcode, func) \
+	({ num_present += !!cpacf_query_func(opcode, func); num_possible++; })
+	QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_224);
+	QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_256);
+	QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_384);
+	QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_512);
+	QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_224);
+	QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_256);
+	QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_384);
+	QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_512);
+#undef QUERY
+
+	if (num_present == num_possible) {
+		static_branch_enable(&have_sha3);
+		if (test_facility(86))
+			static_branch_enable(&have_sha3_init_optim);
+	} else if (num_present != 0) {
+		pr_warn("Unsupported combination of SHA-3 facilities\n");
+	}
+}
diff --git a/lib/crypto/sha1.c b/lib/crypto/sha1.c
index 5904e4ae85d2..daf18c862fdf 100644
--- a/lib/crypto/sha1.c
+++ b/lib/crypto/sha1.c
@@ -12,6 +12,7 @@
 #include <linux/string.h>
 #include <linux/unaligned.h>
 #include <linux/wordpart.h>
+#include "fips.h"
 
 static const struct sha1_block_state sha1_iv = {
 	.h = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 },
@@ -48,7 +49,7 @@ static const struct sha1_block_state sha1_iv = {
 #endif
 
 /* This "rolls" over the 512-bit array */
-#define W(x) (array[(x)&15])
+#define W(x) (workspace[(x)&15])
 
 /*
  * Where do we get the source from? The first 16 iterations get it from
@@ -69,34 +70,20 @@ static const struct sha1_block_state sha1_iv = {
 #define T_40_59(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, ((B&C)+(D&(B^C))) , 0x8f1bbcdc, A, B, C, D, E )
 #define T_60_79(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, (B^C^D) ,  0xca62c1d6, A, B, C, D, E )
 
-/**
- * sha1_transform - single block SHA1 transform (deprecated)
- *
- * @digest: 160 bit digest to update
- * @data:   512 bits of data to hash
- * @array:  16 words of workspace (see note)
- *
- * This function executes SHA-1's internal compression function.  It updates the
- * 160-bit internal state (@digest) with a single 512-bit data block (@data).
- *
- * Don't use this function.  SHA-1 is no longer considered secure.  And even if
- * you do have to use SHA-1, this isn't the correct way to hash something with
- * SHA-1 as this doesn't handle padding and finalization.
- *
- * Note: If the hash is security sensitive, the caller should be sure
- * to clear the workspace. This is left to the caller to avoid
- * unnecessary clears between chained hashing operations.
- */
-void sha1_transform(__u32 *digest, const char *data, __u32 *array)
+#define SHA1_WORKSPACE_WORDS 16
+
+static void sha1_block_generic(struct sha1_block_state *state,
+			       const u8 data[SHA1_BLOCK_SIZE],
+			       u32 workspace[SHA1_WORKSPACE_WORDS])
 {
 	__u32 A, B, C, D, E;
 	unsigned int i = 0;
 
-	A = digest[0];
-	B = digest[1];
-	C = digest[2];
-	D = digest[3];
-	E = digest[4];
+	A = state->h[0];
+	B = state->h[1];
+	C = state->h[2];
+	D = state->h[3];
+	E = state->h[4];
 
 	/* Round 1 - iterations 0-16 take their input from 'data' */
 	for (; i < 16; ++i)
@@ -118,27 +105,12 @@ void sha1_transform(__u32 *digest, const char *data, __u32 *array)
 	for (; i < 80; ++i)
 		T_60_79(i, A, B, C, D, E);
 
-	digest[0] += A;
-	digest[1] += B;
-	digest[2] += C;
-	digest[3] += D;
-	digest[4] += E;
-}
-EXPORT_SYMBOL(sha1_transform);
-
-/**
- * sha1_init_raw - initialize the vectors for a SHA1 digest
- * @buf: vector to initialize
- */
-void sha1_init_raw(__u32 *buf)
-{
-	buf[0] = 0x67452301;
-	buf[1] = 0xefcdab89;
-	buf[2] = 0x98badcfe;
-	buf[3] = 0x10325476;
-	buf[4] = 0xc3d2e1f0;
+	state->h[0] += A;
+	state->h[1] += B;
+	state->h[2] += C;
+	state->h[3] += D;
+	state->h[4] += E;
 }
-EXPORT_SYMBOL(sha1_init_raw);
 
 static void __maybe_unused sha1_blocks_generic(struct sha1_block_state *state,
 					       const u8 *data, size_t nblocks)
@@ -146,7 +118,7 @@ static void __maybe_unused sha1_blocks_generic(struct sha1_block_state *state,
 	u32 workspace[SHA1_WORKSPACE_WORDS];
 
 	do {
-		sha1_transform(state->h, data, workspace);
+		sha1_block_generic(state, data, workspace);
 		data += SHA1_BLOCK_SIZE;
 	} while (--nblocks);
 
@@ -330,10 +302,26 @@ void hmac_sha1_usingrawkey(const u8 *raw_key, size_t raw_key_len,
 }
 EXPORT_SYMBOL_GPL(hmac_sha1_usingrawkey);
 
-#ifdef sha1_mod_init_arch
+#if defined(sha1_mod_init_arch) || defined(CONFIG_CRYPTO_FIPS)
 static int __init sha1_mod_init(void)
 {
+#ifdef sha1_mod_init_arch
 	sha1_mod_init_arch();
+#endif
+	if (fips_enabled) {
+		/*
+		 * FIPS cryptographic algorithm self-test.  As per the FIPS
+		 * Implementation Guidance, testing HMAC-SHA1 satisfies the test
+		 * requirement for SHA-1 too.
+		 */
+		u8 mac[SHA1_DIGEST_SIZE];
+
+		hmac_sha1_usingrawkey(fips_test_key, sizeof(fips_test_key),
+				      fips_test_data, sizeof(fips_test_data),
+				      mac);
+		if (memcmp(fips_test_hmac_sha1_value, mac, sizeof(mac)) != 0)
+			panic("sha1: FIPS self-test failed\n");
+	}
 	return 0;
 }
 subsys_initcall(sha1_mod_init);
diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c
index 881b935418ce..5d6b77e7e141 100644
--- a/lib/crypto/sha256.c
+++ b/lib/crypto/sha256.c
@@ -17,6 +17,7 @@
 #include <linux/string.h>
 #include <linux/unaligned.h>
 #include <linux/wordpart.h>
+#include "fips.h"
 
 static const struct sha256_block_state sha224_iv = {
 	.h = {
@@ -269,8 +270,8 @@ void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE])
 EXPORT_SYMBOL(sha256);
 
 /*
- * Pre-boot environment (as indicated by __DISABLE_EXPORTS being defined)
- * doesn't need either HMAC support or interleaved hashing support
+ * Pre-boot environments (as indicated by __DISABLE_EXPORTS being defined) just
+ * need the generic SHA-256 code.  Omit all other features from them.
  */
 #ifndef __DISABLE_EXPORTS
 
@@ -477,12 +478,27 @@ void hmac_sha256_usingrawkey(const u8 *raw_key, size_t raw_key_len,
 	hmac_sha256_final(&ctx, out);
 }
 EXPORT_SYMBOL_GPL(hmac_sha256_usingrawkey);
-#endif /* !__DISABLE_EXPORTS */
 
-#ifdef sha256_mod_init_arch
+#if defined(sha256_mod_init_arch) || defined(CONFIG_CRYPTO_FIPS)
 static int __init sha256_mod_init(void)
 {
+#ifdef sha256_mod_init_arch
 	sha256_mod_init_arch();
+#endif
+	if (fips_enabled) {
+		/*
+		 * FIPS cryptographic algorithm self-test.  As per the FIPS
+		 * Implementation Guidance, testing HMAC-SHA256 satisfies the
+		 * test requirement for SHA-224, SHA-256, and HMAC-SHA224 too.
+		 */
+		u8 mac[SHA256_DIGEST_SIZE];
+
+		hmac_sha256_usingrawkey(fips_test_key, sizeof(fips_test_key),
+					fips_test_data, sizeof(fips_test_data),
+					mac);
+		if (memcmp(fips_test_hmac_sha256_value, mac, sizeof(mac)) != 0)
+			panic("sha256: FIPS self-test failed\n");
+	}
 	return 0;
 }
 subsys_initcall(sha256_mod_init);
@@ -493,5 +509,7 @@ static void __exit sha256_mod_exit(void)
 module_exit(sha256_mod_exit);
 #endif
 
+#endif /* !__DISABLE_EXPORTS */
+
 MODULE_DESCRIPTION("SHA-224, SHA-256, HMAC-SHA224, and HMAC-SHA256 library functions");
 MODULE_LICENSE("GPL");
diff --git a/lib/crypto/sha3.c b/lib/crypto/sha3.c
new file mode 100644
index 000000000000..32b7074de792
--- /dev/null
+++ b/lib/crypto/sha3.c
@@ -0,0 +1,411 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * SHA-3, as specified in
+ * https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.202.pdf
+ *
+ * SHA-3 code by Jeff Garzik <jeff@garzik.org>
+ *               Ard Biesheuvel <ard.biesheuvel@linaro.org>
+ *               David Howells <dhowells@redhat.com>
+ *
+ * See also Documentation/crypto/sha3.rst
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <crypto/sha3.h>
+#include <crypto/utils.h>
+#include <linux/export.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/unaligned.h>
+#include "fips.h"
+
+/*
+ * On some 32-bit architectures, such as h8300, GCC ends up using over 1 KB of
+ * stack if the round calculation gets inlined into the loop in
+ * sha3_keccakf_generic().  On the other hand, on 64-bit architectures with
+ * plenty of [64-bit wide] general purpose registers, not inlining it severely
+ * hurts performance.  So let's use 64-bitness as a heuristic to decide whether
+ * to inline or not.
+ */
+#ifdef CONFIG_64BIT
+#define SHA3_INLINE inline
+#else
+#define SHA3_INLINE noinline
+#endif
+
+#define SHA3_KECCAK_ROUNDS 24
+
+static const u64 sha3_keccakf_rndc[SHA3_KECCAK_ROUNDS] = {
+	0x0000000000000001ULL, 0x0000000000008082ULL, 0x800000000000808aULL,
+	0x8000000080008000ULL, 0x000000000000808bULL, 0x0000000080000001ULL,
+	0x8000000080008081ULL, 0x8000000000008009ULL, 0x000000000000008aULL,
+	0x0000000000000088ULL, 0x0000000080008009ULL, 0x000000008000000aULL,
+	0x000000008000808bULL, 0x800000000000008bULL, 0x8000000000008089ULL,
+	0x8000000000008003ULL, 0x8000000000008002ULL, 0x8000000000000080ULL,
+	0x000000000000800aULL, 0x800000008000000aULL, 0x8000000080008081ULL,
+	0x8000000000008080ULL, 0x0000000080000001ULL, 0x8000000080008008ULL
+};
+
+/*
+ * Perform a single round of Keccak mixing.
+ */
+static SHA3_INLINE void sha3_keccakf_one_round_generic(u64 st[25], int round)
+{
+	u64 t[5], tt, bc[5];
+
+	/* Theta */
+	bc[0] = st[0] ^ st[5] ^ st[10] ^ st[15] ^ st[20];
+	bc[1] = st[1] ^ st[6] ^ st[11] ^ st[16] ^ st[21];
+	bc[2] = st[2] ^ st[7] ^ st[12] ^ st[17] ^ st[22];
+	bc[3] = st[3] ^ st[8] ^ st[13] ^ st[18] ^ st[23];
+	bc[4] = st[4] ^ st[9] ^ st[14] ^ st[19] ^ st[24];
+
+	t[0] = bc[4] ^ rol64(bc[1], 1);
+	t[1] = bc[0] ^ rol64(bc[2], 1);
+	t[2] = bc[1] ^ rol64(bc[3], 1);
+	t[3] = bc[2] ^ rol64(bc[4], 1);
+	t[4] = bc[3] ^ rol64(bc[0], 1);
+
+	st[0] ^= t[0];
+
+	/* Rho Pi */
+	tt = st[1];
+	st[ 1] = rol64(st[ 6] ^ t[1], 44);
+	st[ 6] = rol64(st[ 9] ^ t[4], 20);
+	st[ 9] = rol64(st[22] ^ t[2], 61);
+	st[22] = rol64(st[14] ^ t[4], 39);
+	st[14] = rol64(st[20] ^ t[0], 18);
+	st[20] = rol64(st[ 2] ^ t[2], 62);
+	st[ 2] = rol64(st[12] ^ t[2], 43);
+	st[12] = rol64(st[13] ^ t[3], 25);
+	st[13] = rol64(st[19] ^ t[4],  8);
+	st[19] = rol64(st[23] ^ t[3], 56);
+	st[23] = rol64(st[15] ^ t[0], 41);
+	st[15] = rol64(st[ 4] ^ t[4], 27);
+	st[ 4] = rol64(st[24] ^ t[4], 14);
+	st[24] = rol64(st[21] ^ t[1],  2);
+	st[21] = rol64(st[ 8] ^ t[3], 55);
+	st[ 8] = rol64(st[16] ^ t[1], 45);
+	st[16] = rol64(st[ 5] ^ t[0], 36);
+	st[ 5] = rol64(st[ 3] ^ t[3], 28);
+	st[ 3] = rol64(st[18] ^ t[3], 21);
+	st[18] = rol64(st[17] ^ t[2], 15);
+	st[17] = rol64(st[11] ^ t[1], 10);
+	st[11] = rol64(st[ 7] ^ t[2],  6);
+	st[ 7] = rol64(st[10] ^ t[0],  3);
+	st[10] = rol64(    tt ^ t[1],  1);
+
+	/* Chi */
+	bc[ 0] = ~st[ 1] & st[ 2];
+	bc[ 1] = ~st[ 2] & st[ 3];
+	bc[ 2] = ~st[ 3] & st[ 4];
+	bc[ 3] = ~st[ 4] & st[ 0];
+	bc[ 4] = ~st[ 0] & st[ 1];
+	st[ 0] ^= bc[ 0];
+	st[ 1] ^= bc[ 1];
+	st[ 2] ^= bc[ 2];
+	st[ 3] ^= bc[ 3];
+	st[ 4] ^= bc[ 4];
+
+	bc[ 0] = ~st[ 6] & st[ 7];
+	bc[ 1] = ~st[ 7] & st[ 8];
+	bc[ 2] = ~st[ 8] & st[ 9];
+	bc[ 3] = ~st[ 9] & st[ 5];
+	bc[ 4] = ~st[ 5] & st[ 6];
+	st[ 5] ^= bc[ 0];
+	st[ 6] ^= bc[ 1];
+	st[ 7] ^= bc[ 2];
+	st[ 8] ^= bc[ 3];
+	st[ 9] ^= bc[ 4];
+
+	bc[ 0] = ~st[11] & st[12];
+	bc[ 1] = ~st[12] & st[13];
+	bc[ 2] = ~st[13] & st[14];
+	bc[ 3] = ~st[14] & st[10];
+	bc[ 4] = ~st[10] & st[11];
+	st[10] ^= bc[ 0];
+	st[11] ^= bc[ 1];
+	st[12] ^= bc[ 2];
+	st[13] ^= bc[ 3];
+	st[14] ^= bc[ 4];
+
+	bc[ 0] = ~st[16] & st[17];
+	bc[ 1] = ~st[17] & st[18];
+	bc[ 2] = ~st[18] & st[19];
+	bc[ 3] = ~st[19] & st[15];
+	bc[ 4] = ~st[15] & st[16];
+	st[15] ^= bc[ 0];
+	st[16] ^= bc[ 1];
+	st[17] ^= bc[ 2];
+	st[18] ^= bc[ 3];
+	st[19] ^= bc[ 4];
+
+	bc[ 0] = ~st[21] & st[22];
+	bc[ 1] = ~st[22] & st[23];
+	bc[ 2] = ~st[23] & st[24];
+	bc[ 3] = ~st[24] & st[20];
+	bc[ 4] = ~st[20] & st[21];
+	st[20] ^= bc[ 0];
+	st[21] ^= bc[ 1];
+	st[22] ^= bc[ 2];
+	st[23] ^= bc[ 3];
+	st[24] ^= bc[ 4];
+
+	/* Iota */
+	st[0] ^= sha3_keccakf_rndc[round];
+}
+
+/* Generic implementation of the Keccak-f[1600] permutation */
+static void sha3_keccakf_generic(struct sha3_state *state)
+{
+	/*
+	 * Temporarily convert the state words from little-endian to native-
+	 * endian so that they can be operated on.  Note that on little-endian
+	 * machines this conversion is a no-op and is optimized out.
+	 */
+
+	for (int i = 0; i < ARRAY_SIZE(state->words); i++)
+		state->native_words[i] = le64_to_cpu(state->words[i]);
+
+	for (int round = 0; round < SHA3_KECCAK_ROUNDS; round++)
+		sha3_keccakf_one_round_generic(state->native_words, round);
+
+	for (int i = 0; i < ARRAY_SIZE(state->words); i++)
+		state->words[i] = cpu_to_le64(state->native_words[i]);
+}
+
+/*
+ * Generic implementation of absorbing the given nonzero number of full blocks
+ * into the sponge function Keccak[r=8*block_size, c=1600-8*block_size].
+ */
+static void __maybe_unused
+sha3_absorb_blocks_generic(struct sha3_state *state, const u8 *data,
+			   size_t nblocks, size_t block_size)
+{
+	do {
+		for (size_t i = 0; i < block_size; i += 8)
+			state->words[i / 8] ^= get_unaligned((__le64 *)&data[i]);
+		sha3_keccakf_generic(state);
+		data += block_size;
+	} while (--nblocks);
+}
+
+#ifdef CONFIG_CRYPTO_LIB_SHA3_ARCH
+#include "sha3.h" /* $(SRCARCH)/sha3.h */
+#else
+#define sha3_keccakf		sha3_keccakf_generic
+#define sha3_absorb_blocks	sha3_absorb_blocks_generic
+#endif
+
+void __sha3_update(struct __sha3_ctx *ctx, const u8 *in, size_t in_len)
+{
+	const size_t block_size = ctx->block_size;
+	size_t absorb_offset = ctx->absorb_offset;
+
+	/* Warn if squeezing has already begun. */
+	WARN_ON_ONCE(absorb_offset >= block_size);
+
+	if (absorb_offset && absorb_offset + in_len >= block_size) {
+		crypto_xor(&ctx->state.bytes[absorb_offset], in,
+			   block_size - absorb_offset);
+		in += block_size - absorb_offset;
+		in_len -= block_size - absorb_offset;
+		sha3_keccakf(&ctx->state);
+		absorb_offset = 0;
+	}
+
+	if (in_len >= block_size) {
+		size_t nblocks = in_len / block_size;
+
+		sha3_absorb_blocks(&ctx->state, in, nblocks, block_size);
+		in += nblocks * block_size;
+		in_len -= nblocks * block_size;
+	}
+
+	if (in_len) {
+		crypto_xor(&ctx->state.bytes[absorb_offset], in, in_len);
+		absorb_offset += in_len;
+	}
+	ctx->absorb_offset = absorb_offset;
+}
+EXPORT_SYMBOL_GPL(__sha3_update);
+
+void sha3_final(struct sha3_ctx *sha3_ctx, u8 *out)
+{
+	struct __sha3_ctx *ctx = &sha3_ctx->ctx;
+
+	ctx->state.bytes[ctx->absorb_offset] ^= 0x06;
+	ctx->state.bytes[ctx->block_size - 1] ^= 0x80;
+	sha3_keccakf(&ctx->state);
+	memcpy(out, ctx->state.bytes, ctx->digest_size);
+	sha3_zeroize_ctx(sha3_ctx);
+}
+EXPORT_SYMBOL_GPL(sha3_final);
+
+void shake_squeeze(struct shake_ctx *shake_ctx, u8 *out, size_t out_len)
+{
+	struct __sha3_ctx *ctx = &shake_ctx->ctx;
+	const size_t block_size = ctx->block_size;
+	size_t squeeze_offset = ctx->squeeze_offset;
+
+	if (ctx->absorb_offset < block_size) {
+		/* First squeeze: */
+
+		/* Add the domain separation suffix and padding. */
+		ctx->state.bytes[ctx->absorb_offset] ^= 0x1f;
+		ctx->state.bytes[block_size - 1] ^= 0x80;
+
+		/* Indicate that squeezing has begun. */
+		ctx->absorb_offset = block_size;
+
+		/*
+		 * Indicate that no output is pending yet, i.e. sha3_keccakf()
+		 * will need to be called before the first copy.
+		 */
+		squeeze_offset = block_size;
+	}
+	while (out_len) {
+		if (squeeze_offset == block_size) {
+			sha3_keccakf(&ctx->state);
+			squeeze_offset = 0;
+		}
+		size_t copy = min(out_len, block_size - squeeze_offset);
+
+		memcpy(out, &ctx->state.bytes[squeeze_offset], copy);
+		out += copy;
+		out_len -= copy;
+		squeeze_offset += copy;
+	}
+	ctx->squeeze_offset = squeeze_offset;
+}
+EXPORT_SYMBOL_GPL(shake_squeeze);
+
+#ifndef sha3_224_arch
+static inline bool sha3_224_arch(const u8 *in, size_t in_len,
+				 u8 out[SHA3_224_DIGEST_SIZE])
+{
+	return false;
+}
+#endif
+#ifndef sha3_256_arch
+static inline bool sha3_256_arch(const u8 *in, size_t in_len,
+				 u8 out[SHA3_256_DIGEST_SIZE])
+{
+	return false;
+}
+#endif
+#ifndef sha3_384_arch
+static inline bool sha3_384_arch(const u8 *in, size_t in_len,
+				 u8 out[SHA3_384_DIGEST_SIZE])
+{
+	return false;
+}
+#endif
+#ifndef sha3_512_arch
+static inline bool sha3_512_arch(const u8 *in, size_t in_len,
+				 u8 out[SHA3_512_DIGEST_SIZE])
+{
+	return false;
+}
+#endif
+
+void sha3_224(const u8 *in, size_t in_len, u8 out[SHA3_224_DIGEST_SIZE])
+{
+	struct sha3_ctx ctx;
+
+	if (sha3_224_arch(in, in_len, out))
+		return;
+	sha3_224_init(&ctx);
+	sha3_update(&ctx, in, in_len);
+	sha3_final(&ctx, out);
+}
+EXPORT_SYMBOL_GPL(sha3_224);
+
+void sha3_256(const u8 *in, size_t in_len, u8 out[SHA3_256_DIGEST_SIZE])
+{
+	struct sha3_ctx ctx;
+
+	if (sha3_256_arch(in, in_len, out))
+		return;
+	sha3_256_init(&ctx);
+	sha3_update(&ctx, in, in_len);
+	sha3_final(&ctx, out);
+}
+EXPORT_SYMBOL_GPL(sha3_256);
+
+void sha3_384(const u8 *in, size_t in_len, u8 out[SHA3_384_DIGEST_SIZE])
+{
+	struct sha3_ctx ctx;
+
+	if (sha3_384_arch(in, in_len, out))
+		return;
+	sha3_384_init(&ctx);
+	sha3_update(&ctx, in, in_len);
+	sha3_final(&ctx, out);
+}
+EXPORT_SYMBOL_GPL(sha3_384);
+
+void sha3_512(const u8 *in, size_t in_len, u8 out[SHA3_512_DIGEST_SIZE])
+{
+	struct sha3_ctx ctx;
+
+	if (sha3_512_arch(in, in_len, out))
+		return;
+	sha3_512_init(&ctx);
+	sha3_update(&ctx, in, in_len);
+	sha3_final(&ctx, out);
+}
+EXPORT_SYMBOL_GPL(sha3_512);
+
+void shake128(const u8 *in, size_t in_len, u8 *out, size_t out_len)
+{
+	struct shake_ctx ctx;
+
+	shake128_init(&ctx);
+	shake_update(&ctx, in, in_len);
+	shake_squeeze(&ctx, out, out_len);
+	shake_zeroize_ctx(&ctx);
+}
+EXPORT_SYMBOL_GPL(shake128);
+
+void shake256(const u8 *in, size_t in_len, u8 *out, size_t out_len)
+{
+	struct shake_ctx ctx;
+
+	shake256_init(&ctx);
+	shake_update(&ctx, in, in_len);
+	shake_squeeze(&ctx, out, out_len);
+	shake_zeroize_ctx(&ctx);
+}
+EXPORT_SYMBOL_GPL(shake256);
+
+#if defined(sha3_mod_init_arch) || defined(CONFIG_CRYPTO_FIPS)
+static int __init sha3_mod_init(void)
+{
+#ifdef sha3_mod_init_arch
+	sha3_mod_init_arch();
+#endif
+	if (fips_enabled) {
+		/*
+		 * FIPS cryptographic algorithm self-test.  As per the FIPS
+		 * Implementation Guidance, testing any SHA-3 algorithm
+		 * satisfies the test requirement for all of them.
+		 */
+		u8 hash[SHA3_256_DIGEST_SIZE];
+
+		sha3_256(fips_test_data, sizeof(fips_test_data), hash);
+		if (memcmp(fips_test_sha3_256_value, hash, sizeof(hash)) != 0)
+			panic("sha3: FIPS self-test failed\n");
+	}
+	return 0;
+}
+subsys_initcall(sha3_mod_init);
+
+static void __exit sha3_mod_exit(void)
+{
+}
+module_exit(sha3_mod_exit);
+#endif
+
+MODULE_DESCRIPTION("SHA-3 library functions");
+MODULE_LICENSE("GPL");
diff --git a/lib/crypto/sha512.c b/lib/crypto/sha512.c
index d8062188be98..605eab51aabd 100644
--- a/lib/crypto/sha512.c
+++ b/lib/crypto/sha512.c
@@ -17,6 +17,7 @@
 #include <linux/string.h>
 #include <linux/unaligned.h>
 #include <linux/wordpart.h>
+#include "fips.h"
 
 static const struct sha512_block_state sha384_iv = {
 	.h = {
@@ -405,10 +406,26 @@ void hmac_sha512_usingrawkey(const u8 *raw_key, size_t raw_key_len,
 }
 EXPORT_SYMBOL_GPL(hmac_sha512_usingrawkey);
 
-#ifdef sha512_mod_init_arch
+#if defined(sha512_mod_init_arch) || defined(CONFIG_CRYPTO_FIPS)
 static int __init sha512_mod_init(void)
 {
+#ifdef sha512_mod_init_arch
 	sha512_mod_init_arch();
+#endif
+	if (fips_enabled) {
+		/*
+		 * FIPS cryptographic algorithm self-test.  As per the FIPS
+		 * Implementation Guidance, testing HMAC-SHA512 satisfies the
+		 * test requirement for SHA-384, SHA-512, and HMAC-SHA384 too.
+		 */
+		u8 mac[SHA512_DIGEST_SIZE];
+
+		hmac_sha512_usingrawkey(fips_test_key, sizeof(fips_test_key),
+					fips_test_data, sizeof(fips_test_data),
+					mac);
+		if (memcmp(fips_test_hmac_sha512_value, mac, sizeof(mac)) != 0)
+			panic("sha512: FIPS self-test failed\n");
+	}
 	return 0;
 }
 subsys_initcall(sha512_mod_init);
diff --git a/lib/crypto/sparc/aes.h b/lib/crypto/sparc/aes.h
new file mode 100644
index 000000000000..e354aa507ee0
--- /dev/null
+++ b/lib/crypto/sparc/aes.h
@@ -0,0 +1,149 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * AES accelerated using the sparc64 aes opcodes
+ *
+ * Copyright (C) 2008, Intel Corp.
+ * Copyright (c) 2010, Intel Corporation.
+ * Copyright 2026 Google LLC
+ */
+
+#include <asm/fpumacro.h>
+#include <asm/opcodes.h>
+#include <asm/pstate.h>
+#include <asm/elf.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes_opcodes);
+
+EXPORT_SYMBOL_GPL(aes_sparc64_key_expand);
+EXPORT_SYMBOL_GPL(aes_sparc64_load_encrypt_keys_128);
+EXPORT_SYMBOL_GPL(aes_sparc64_load_encrypt_keys_192);
+EXPORT_SYMBOL_GPL(aes_sparc64_load_encrypt_keys_256);
+EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_128);
+EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_192);
+EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_256);
+EXPORT_SYMBOL_GPL(aes_sparc64_ecb_encrypt_128);
+EXPORT_SYMBOL_GPL(aes_sparc64_ecb_encrypt_192);
+EXPORT_SYMBOL_GPL(aes_sparc64_ecb_encrypt_256);
+EXPORT_SYMBOL_GPL(aes_sparc64_ecb_decrypt_128);
+EXPORT_SYMBOL_GPL(aes_sparc64_ecb_decrypt_192);
+EXPORT_SYMBOL_GPL(aes_sparc64_ecb_decrypt_256);
+EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_128);
+EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_192);
+EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_256);
+EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_128);
+EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_192);
+EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_256);
+EXPORT_SYMBOL_GPL(aes_sparc64_ctr_crypt_128);
+EXPORT_SYMBOL_GPL(aes_sparc64_ctr_crypt_192);
+EXPORT_SYMBOL_GPL(aes_sparc64_ctr_crypt_256);
+
+void aes_sparc64_encrypt_128(const u64 *key, const u32 *input, u32 *output);
+void aes_sparc64_encrypt_192(const u64 *key, const u32 *input, u32 *output);
+void aes_sparc64_encrypt_256(const u64 *key, const u32 *input, u32 *output);
+void aes_sparc64_decrypt_128(const u64 *key, const u32 *input, u32 *output);
+void aes_sparc64_decrypt_192(const u64 *key, const u32 *input, u32 *output);
+void aes_sparc64_decrypt_256(const u64 *key, const u32 *input, u32 *output);
+
+static void aes_preparekey_arch(union aes_enckey_arch *k,
+				union aes_invkey_arch *inv_k,
+				const u8 *in_key, int key_len, int nrounds)
+{
+	if (static_branch_likely(&have_aes_opcodes)) {
+		u32 aligned_key[AES_MAX_KEY_SIZE / 4];
+
+		if (IS_ALIGNED((uintptr_t)in_key, 4)) {
+			aes_sparc64_key_expand((const u32 *)in_key,
+					       k->sparc_rndkeys, key_len);
+		} else {
+			memcpy(aligned_key, in_key, key_len);
+			aes_sparc64_key_expand(aligned_key,
+					       k->sparc_rndkeys, key_len);
+			memzero_explicit(aligned_key, key_len);
+		}
+		/*
+		 * Note that nothing needs to be written to inv_k (if it's
+		 * non-NULL) here, since the SPARC64 assembly code uses
+		 * k->sparc_rndkeys for both encryption and decryption.
+		 */
+	} else {
+		aes_expandkey_generic(k->rndkeys,
+				      inv_k ? inv_k->inv_rndkeys : NULL,
+				      in_key, key_len);
+	}
+}
+
+static void aes_sparc64_encrypt(const struct aes_enckey *key,
+				const u32 *input, u32 *output)
+{
+	if (key->len == AES_KEYSIZE_128)
+		aes_sparc64_encrypt_128(key->k.sparc_rndkeys, input, output);
+	else if (key->len == AES_KEYSIZE_192)
+		aes_sparc64_encrypt_192(key->k.sparc_rndkeys, input, output);
+	else
+		aes_sparc64_encrypt_256(key->k.sparc_rndkeys, input, output);
+}
+
+static void aes_encrypt_arch(const struct aes_enckey *key,
+			     u8 out[AES_BLOCK_SIZE],
+			     const u8 in[AES_BLOCK_SIZE])
+{
+	u32 bounce_buf[AES_BLOCK_SIZE / 4];
+
+	if (static_branch_likely(&have_aes_opcodes)) {
+		if (IS_ALIGNED((uintptr_t)in | (uintptr_t)out, 4)) {
+			aes_sparc64_encrypt(key, (const u32 *)in, (u32 *)out);
+		} else {
+			memcpy(bounce_buf, in, AES_BLOCK_SIZE);
+			aes_sparc64_encrypt(key, bounce_buf, bounce_buf);
+			memcpy(out, bounce_buf, AES_BLOCK_SIZE);
+		}
+	} else {
+		aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in);
+	}
+}
+
+static void aes_sparc64_decrypt(const struct aes_key *key,
+				const u32 *input, u32 *output)
+{
+	if (key->len == AES_KEYSIZE_128)
+		aes_sparc64_decrypt_128(key->k.sparc_rndkeys, input, output);
+	else if (key->len == AES_KEYSIZE_192)
+		aes_sparc64_decrypt_192(key->k.sparc_rndkeys, input, output);
+	else
+		aes_sparc64_decrypt_256(key->k.sparc_rndkeys, input, output);
+}
+
+static void aes_decrypt_arch(const struct aes_key *key,
+			     u8 out[AES_BLOCK_SIZE],
+			     const u8 in[AES_BLOCK_SIZE])
+{
+	u32 bounce_buf[AES_BLOCK_SIZE / 4];
+
+	if (static_branch_likely(&have_aes_opcodes)) {
+		if (IS_ALIGNED((uintptr_t)in | (uintptr_t)out, 4)) {
+			aes_sparc64_decrypt(key, (const u32 *)in, (u32 *)out);
+		} else {
+			memcpy(bounce_buf, in, AES_BLOCK_SIZE);
+			aes_sparc64_decrypt(key, bounce_buf, bounce_buf);
+			memcpy(out, bounce_buf, AES_BLOCK_SIZE);
+		}
+	} else {
+		aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds,
+				    out, in);
+	}
+}
+
+#define aes_mod_init_arch aes_mod_init_arch
+static void aes_mod_init_arch(void)
+{
+	unsigned long cfr;
+
+	if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
+		return;
+
+	__asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
+	if (!(cfr & CFR_AES))
+		return;
+
+	static_branch_enable(&have_aes_opcodes);
+}
diff --git a/lib/crypto/sparc/aes_asm.S b/lib/crypto/sparc/aes_asm.S
new file mode 100644
index 000000000000..f291174a72a1
--- /dev/null
+++ b/lib/crypto/sparc/aes_asm.S
@@ -0,0 +1,1543 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#include <linux/linkage.h>
+#include <asm/opcodes.h>
+#include <asm/visasm.h>
+
+#define ENCRYPT_TWO_ROUNDS(KEY_BASE, I0, I1, T0, T1) \
+	AES_EROUND01(KEY_BASE +  0, I0, I1, T0) \
+	AES_EROUND23(KEY_BASE +  2, I0, I1, T1) \
+	AES_EROUND01(KEY_BASE +  4, T0, T1, I0) \
+	AES_EROUND23(KEY_BASE +  6, T0, T1, I1)
+
+#define ENCRYPT_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
+	AES_EROUND01(KEY_BASE +  0, I0, I1, T0) \
+	AES_EROUND23(KEY_BASE +  2, I0, I1, T1) \
+	AES_EROUND01(KEY_BASE +  0, I2, I3, T2) \
+	AES_EROUND23(KEY_BASE +  2, I2, I3, T3) \
+	AES_EROUND01(KEY_BASE +  4, T0, T1, I0) \
+	AES_EROUND23(KEY_BASE +  6, T0, T1, I1) \
+	AES_EROUND01(KEY_BASE +  4, T2, T3, I2) \
+	AES_EROUND23(KEY_BASE +  6, T2, T3, I3)
+
+#define ENCRYPT_TWO_ROUNDS_LAST(KEY_BASE, I0, I1, T0, T1) \
+	AES_EROUND01(KEY_BASE +  0, I0, I1, T0) \
+	AES_EROUND23(KEY_BASE +  2, I0, I1, T1) \
+	AES_EROUND01_L(KEY_BASE +  4, T0, T1, I0) \
+	AES_EROUND23_L(KEY_BASE +  6, T0, T1, I1)
+
+#define ENCRYPT_TWO_ROUNDS_LAST_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
+	AES_EROUND01(KEY_BASE +  0, I0, I1, T0) \
+	AES_EROUND23(KEY_BASE +  2, I0, I1, T1) \
+	AES_EROUND01(KEY_BASE +  0, I2, I3, T2) \
+	AES_EROUND23(KEY_BASE +  2, I2, I3, T3) \
+	AES_EROUND01_L(KEY_BASE +  4, T0, T1, I0) \
+	AES_EROUND23_L(KEY_BASE +  6, T0, T1, I1) \
+	AES_EROUND01_L(KEY_BASE +  4, T2, T3, I2) \
+	AES_EROUND23_L(KEY_BASE +  6, T2, T3, I3)
+
+	/* 10 rounds */
+#define ENCRYPT_128(KEY_BASE, I0, I1, T0, T1) \
+	ENCRYPT_TWO_ROUNDS(KEY_BASE +  0, I0, I1, T0, T1) \
+	ENCRYPT_TWO_ROUNDS(KEY_BASE +  8, I0, I1, T0, T1) \
+	ENCRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
+	ENCRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
+	ENCRYPT_TWO_ROUNDS_LAST(KEY_BASE + 32, I0, I1, T0, T1)
+
+#define ENCRYPT_128_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
+	ENCRYPT_TWO_ROUNDS_2(KEY_BASE +  0, I0, I1, I2, I3, T0, T1, T2, T3) \
+	ENCRYPT_TWO_ROUNDS_2(KEY_BASE +  8, I0, I1, I2, I3, T0, T1, T2, T3) \
+	ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, T0, T1, T2, T3) \
+	ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, T0, T1, T2, T3) \
+	ENCRYPT_TWO_ROUNDS_LAST_2(KEY_BASE + 32, I0, I1, I2, I3, T0, T1, T2, T3)
+
+	/* 12 rounds */
+#define ENCRYPT_192(KEY_BASE, I0, I1, T0, T1) \
+	ENCRYPT_TWO_ROUNDS(KEY_BASE +  0, I0, I1, T0, T1) \
+	ENCRYPT_TWO_ROUNDS(KEY_BASE +  8, I0, I1, T0, T1) \
+	ENCRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
+	ENCRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
+	ENCRYPT_TWO_ROUNDS(KEY_BASE + 32, I0, I1, T0, T1) \
+	ENCRYPT_TWO_ROUNDS_LAST(KEY_BASE + 40, I0, I1, T0, T1)
+
+#define ENCRYPT_192_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
+	ENCRYPT_TWO_ROUNDS_2(KEY_BASE +  0, I0, I1, I2, I3, T0, T1, T2, T3) \
+	ENCRYPT_TWO_ROUNDS_2(KEY_BASE +  8, I0, I1, I2, I3, T0, T1, T2, T3) \
+	ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, T0, T1, T2, T3) \
+	ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, T0, T1, T2, T3) \
+	ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 32, I0, I1, I2, I3, T0, T1, T2, T3) \
+	ENCRYPT_TWO_ROUNDS_LAST_2(KEY_BASE + 40, I0, I1, I2, I3, T0, T1, T2, T3)
+
+	/* 14 rounds */
+#define ENCRYPT_256(KEY_BASE, I0, I1, T0, T1) \
+	ENCRYPT_TWO_ROUNDS(KEY_BASE +  0, I0, I1, T0, T1) \
+	ENCRYPT_TWO_ROUNDS(KEY_BASE +  8, I0, I1, T0, T1) \
+	ENCRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
+	ENCRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
+	ENCRYPT_TWO_ROUNDS(KEY_BASE + 32, I0, I1, T0, T1) \
+	ENCRYPT_TWO_ROUNDS(KEY_BASE + 40, I0, I1, T0, T1) \
+	ENCRYPT_TWO_ROUNDS_LAST(KEY_BASE + 48, I0, I1, T0, T1)
+
+#define ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, TMP_BASE) \
+	ENCRYPT_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, \
+			     TMP_BASE + 0, TMP_BASE + 2, TMP_BASE + 4, TMP_BASE + 6)
+
+#define ENCRYPT_256_2(KEY_BASE, I0, I1, I2, I3) \
+	ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE +  0, I0, I1, I2, I3, KEY_BASE + 48) \
+	ldd	[%o0 + 0xd0], %f56; \
+	ldd	[%o0 + 0xd8], %f58; \
+	ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE +  8, I0, I1, I2, I3, KEY_BASE +  0) \
+	ldd	[%o0 + 0xe0], %f60; \
+	ldd	[%o0 + 0xe8], %f62; \
+	ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, KEY_BASE +  0) \
+	ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, KEY_BASE +  0) \
+	ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 32, I0, I1, I2, I3, KEY_BASE +  0) \
+	ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 40, I0, I1, I2, I3, KEY_BASE +  0) \
+	AES_EROUND01(KEY_BASE +  48, I0, I1, KEY_BASE + 0) \
+	AES_EROUND23(KEY_BASE +  50, I0, I1, KEY_BASE + 2) \
+	AES_EROUND01(KEY_BASE +  48, I2, I3, KEY_BASE + 4) \
+	AES_EROUND23(KEY_BASE +  50, I2, I3, KEY_BASE + 6) \
+	AES_EROUND01_L(KEY_BASE +  52, KEY_BASE + 0, KEY_BASE + 2, I0) \
+	AES_EROUND23_L(KEY_BASE +  54, KEY_BASE + 0, KEY_BASE + 2, I1) \
+	ldd	[%o0 + 0x10], %f8; \
+	ldd	[%o0 + 0x18], %f10; \
+	AES_EROUND01_L(KEY_BASE +  52, KEY_BASE + 4, KEY_BASE + 6, I2) \
+	AES_EROUND23_L(KEY_BASE +  54, KEY_BASE + 4, KEY_BASE + 6, I3) \
+	ldd	[%o0 + 0x20], %f12; \
+	ldd	[%o0 + 0x28], %f14;
+
+#define DECRYPT_TWO_ROUNDS(KEY_BASE, I0, I1, T0, T1) \
+	AES_DROUND23(KEY_BASE +  0, I0, I1, T1) \
+	AES_DROUND01(KEY_BASE +  2, I0, I1, T0) \
+	AES_DROUND23(KEY_BASE +  4, T0, T1, I1) \
+	AES_DROUND01(KEY_BASE +  6, T0, T1, I0)
+
+#define DECRYPT_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
+	AES_DROUND23(KEY_BASE +  0, I0, I1, T1) \
+	AES_DROUND01(KEY_BASE +  2, I0, I1, T0) \
+	AES_DROUND23(KEY_BASE +  0, I2, I3, T3) \
+	AES_DROUND01(KEY_BASE +  2, I2, I3, T2) \
+	AES_DROUND23(KEY_BASE +  4, T0, T1, I1) \
+	AES_DROUND01(KEY_BASE +  6, T0, T1, I0) \
+	AES_DROUND23(KEY_BASE +  4, T2, T3, I3) \
+	AES_DROUND01(KEY_BASE +  6, T2, T3, I2)
+
+#define DECRYPT_TWO_ROUNDS_LAST(KEY_BASE, I0, I1, T0, T1) \
+	AES_DROUND23(KEY_BASE +  0, I0, I1, T1) \
+	AES_DROUND01(KEY_BASE +  2, I0, I1, T0) \
+	AES_DROUND23_L(KEY_BASE +  4, T0, T1, I1) \
+	AES_DROUND01_L(KEY_BASE +  6, T0, T1, I0)
+
+#define DECRYPT_TWO_ROUNDS_LAST_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
+	AES_DROUND23(KEY_BASE +  0, I0, I1, T1) \
+	AES_DROUND01(KEY_BASE +  2, I0, I1, T0) \
+	AES_DROUND23(KEY_BASE +  0, I2, I3, T3) \
+	AES_DROUND01(KEY_BASE +  2, I2, I3, T2) \
+	AES_DROUND23_L(KEY_BASE +  4, T0, T1, I1) \
+	AES_DROUND01_L(KEY_BASE +  6, T0, T1, I0) \
+	AES_DROUND23_L(KEY_BASE +  4, T2, T3, I3) \
+	AES_DROUND01_L(KEY_BASE +  6, T2, T3, I2)
+
+	/* 10 rounds */
+#define DECRYPT_128(KEY_BASE, I0, I1, T0, T1) \
+	DECRYPT_TWO_ROUNDS(KEY_BASE +  0, I0, I1, T0, T1) \
+	DECRYPT_TWO_ROUNDS(KEY_BASE +  8, I0, I1, T0, T1) \
+	DECRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
+	DECRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
+	DECRYPT_TWO_ROUNDS_LAST(KEY_BASE + 32, I0, I1, T0, T1)
+
+#define DECRYPT_128_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
+	DECRYPT_TWO_ROUNDS_2(KEY_BASE +  0, I0, I1, I2, I3, T0, T1, T2, T3) \
+	DECRYPT_TWO_ROUNDS_2(KEY_BASE +  8, I0, I1, I2, I3, T0, T1, T2, T3) \
+	DECRYPT_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, T0, T1, T2, T3) \
+	DECRYPT_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, T0, T1, T2, T3) \
+	DECRYPT_TWO_ROUNDS_LAST_2(KEY_BASE + 32, I0, I1, I2, I3, T0, T1, T2, T3)
+
+	/* 12 rounds */
+#define DECRYPT_192(KEY_BASE, I0, I1, T0, T1) \
+	DECRYPT_TWO_ROUNDS(KEY_BASE +  0, I0, I1, T0, T1) \
+	DECRYPT_TWO_ROUNDS(KEY_BASE +  8, I0, I1, T0, T1) \
+	DECRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
+	DECRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
+	DECRYPT_TWO_ROUNDS(KEY_BASE + 32, I0, I1, T0, T1) \
+	DECRYPT_TWO_ROUNDS_LAST(KEY_BASE + 40, I0, I1, T0, T1)
+
+#define DECRYPT_192_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
+	DECRYPT_TWO_ROUNDS_2(KEY_BASE +  0, I0, I1, I2, I3, T0, T1, T2, T3) \
+	DECRYPT_TWO_ROUNDS_2(KEY_BASE +  8, I0, I1, I2, I3, T0, T1, T2, T3) \
+	DECRYPT_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, T0, T1, T2, T3) \
+	DECRYPT_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, T0, T1, T2, T3) \
+	DECRYPT_TWO_ROUNDS_2(KEY_BASE + 32, I0, I1, I2, I3, T0, T1, T2, T3) \
+	DECRYPT_TWO_ROUNDS_LAST_2(KEY_BASE + 40, I0, I1, I2, I3, T0, T1, T2, T3)
+
+	/* 14 rounds */
+#define DECRYPT_256(KEY_BASE, I0, I1, T0, T1) \
+	DECRYPT_TWO_ROUNDS(KEY_BASE +  0, I0, I1, T0, T1) \
+	DECRYPT_TWO_ROUNDS(KEY_BASE +  8, I0, I1, T0, T1) \
+	DECRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
+	DECRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
+	DECRYPT_TWO_ROUNDS(KEY_BASE + 32, I0, I1, T0, T1) \
+	DECRYPT_TWO_ROUNDS(KEY_BASE + 40, I0, I1, T0, T1) \
+	DECRYPT_TWO_ROUNDS_LAST(KEY_BASE + 48, I0, I1, T0, T1)
+
+#define DECRYPT_256_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, TMP_BASE) \
+	DECRYPT_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, \
+			     TMP_BASE + 0, TMP_BASE + 2, TMP_BASE + 4, TMP_BASE + 6)
+
+#define DECRYPT_256_2(KEY_BASE, I0, I1, I2, I3) \
+	DECRYPT_256_TWO_ROUNDS_2(KEY_BASE +  0, I0, I1, I2, I3, KEY_BASE + 48) \
+	ldd	[%o0 + 0x18], %f56; \
+	ldd	[%o0 + 0x10], %f58; \
+	DECRYPT_256_TWO_ROUNDS_2(KEY_BASE +  8, I0, I1, I2, I3, KEY_BASE +  0) \
+	ldd	[%o0 + 0x08], %f60; \
+	ldd	[%o0 + 0x00], %f62; \
+	DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, KEY_BASE +  0) \
+	DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, KEY_BASE +  0) \
+	DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 32, I0, I1, I2, I3, KEY_BASE +  0) \
+	DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 40, I0, I1, I2, I3, KEY_BASE +  0) \
+	AES_DROUND23(KEY_BASE +  48, I0, I1, KEY_BASE + 2) \
+	AES_DROUND01(KEY_BASE +  50, I0, I1, KEY_BASE + 0) \
+	AES_DROUND23(KEY_BASE +  48, I2, I3, KEY_BASE + 6) \
+	AES_DROUND01(KEY_BASE +  50, I2, I3, KEY_BASE + 4) \
+	AES_DROUND23_L(KEY_BASE +  52, KEY_BASE + 0, KEY_BASE + 2, I1) \
+	AES_DROUND01_L(KEY_BASE +  54, KEY_BASE + 0, KEY_BASE + 2, I0) \
+	ldd	[%o0 + 0xd8], %f8; \
+	ldd	[%o0 + 0xd0], %f10; \
+	AES_DROUND23_L(KEY_BASE +  52, KEY_BASE + 4, KEY_BASE + 6, I3) \
+	AES_DROUND01_L(KEY_BASE +  54, KEY_BASE + 4, KEY_BASE + 6, I2) \
+	ldd	[%o0 + 0xc8], %f12; \
+	ldd	[%o0 + 0xc0], %f14;
+
+	.align	32
+ENTRY(aes_sparc64_key_expand)
+	/* %o0=input_key, %o1=output_key, %o2=key_len */
+	VISEntry
+	ld	[%o0 + 0x00], %f0
+	ld	[%o0 + 0x04], %f1
+	ld	[%o0 + 0x08], %f2
+	ld	[%o0 + 0x0c], %f3
+
+	std	%f0, [%o1 + 0x00]
+	std	%f2, [%o1 + 0x08]
+	add	%o1, 0x10, %o1
+
+	cmp	%o2, 24
+	bl	2f
+	 nop
+
+	be	1f
+	 nop
+
+	/* 256-bit key expansion */
+	ld	[%o0 + 0x10], %f4
+	ld	[%o0 + 0x14], %f5
+	ld	[%o0 + 0x18], %f6
+	ld	[%o0 + 0x1c], %f7
+
+	std	%f4, [%o1 + 0x00]
+	std	%f6, [%o1 + 0x08]
+	add	%o1, 0x10, %o1
+
+	AES_KEXPAND1(0, 6, 0x0, 8)
+	AES_KEXPAND2(2, 8, 10)
+	AES_KEXPAND0(4, 10, 12)
+	AES_KEXPAND2(6, 12, 14)
+	AES_KEXPAND1(8, 14, 0x1, 16)
+	AES_KEXPAND2(10, 16, 18)
+	AES_KEXPAND0(12, 18, 20)
+	AES_KEXPAND2(14, 20, 22)
+	AES_KEXPAND1(16, 22, 0x2, 24)
+	AES_KEXPAND2(18, 24, 26)
+	AES_KEXPAND0(20, 26, 28)
+	AES_KEXPAND2(22, 28, 30)
+	AES_KEXPAND1(24, 30, 0x3, 32)
+	AES_KEXPAND2(26, 32, 34)
+	AES_KEXPAND0(28, 34, 36)
+	AES_KEXPAND2(30, 36, 38)
+	AES_KEXPAND1(32, 38, 0x4, 40)
+	AES_KEXPAND2(34, 40, 42)
+	AES_KEXPAND0(36, 42, 44)
+	AES_KEXPAND2(38, 44, 46)
+	AES_KEXPAND1(40, 46, 0x5, 48)
+	AES_KEXPAND2(42, 48, 50)
+	AES_KEXPAND0(44, 50, 52)
+	AES_KEXPAND2(46, 52, 54)
+	AES_KEXPAND1(48, 54, 0x6, 56)
+	AES_KEXPAND2(50, 56, 58)
+
+	std	%f8, [%o1 + 0x00]
+	std	%f10, [%o1 + 0x08]
+	std	%f12, [%o1 + 0x10]
+	std	%f14, [%o1 + 0x18]
+	std	%f16, [%o1 + 0x20]
+	std	%f18, [%o1 + 0x28]
+	std	%f20, [%o1 + 0x30]
+	std	%f22, [%o1 + 0x38]
+	std	%f24, [%o1 + 0x40]
+	std	%f26, [%o1 + 0x48]
+	std	%f28, [%o1 + 0x50]
+	std	%f30, [%o1 + 0x58]
+	std	%f32, [%o1 + 0x60]
+	std	%f34, [%o1 + 0x68]
+	std	%f36, [%o1 + 0x70]
+	std	%f38, [%o1 + 0x78]
+	std	%f40, [%o1 + 0x80]
+	std	%f42, [%o1 + 0x88]
+	std	%f44, [%o1 + 0x90]
+	std	%f46, [%o1 + 0x98]
+	std	%f48, [%o1 + 0xa0]
+	std	%f50, [%o1 + 0xa8]
+	std	%f52, [%o1 + 0xb0]
+	std	%f54, [%o1 + 0xb8]
+	std	%f56, [%o1 + 0xc0]
+	ba,pt	%xcc, 80f
+	 std	%f58, [%o1 + 0xc8]
+
+1:	
+	/* 192-bit key expansion */
+	ld	[%o0 + 0x10], %f4
+	ld	[%o0 + 0x14], %f5
+
+	std	%f4, [%o1 + 0x00]
+	add	%o1, 0x08, %o1
+
+	AES_KEXPAND1(0, 4, 0x0, 6)
+	AES_KEXPAND2(2, 6, 8)
+	AES_KEXPAND2(4, 8, 10)
+	AES_KEXPAND1(6, 10, 0x1, 12)
+	AES_KEXPAND2(8, 12, 14)
+	AES_KEXPAND2(10, 14, 16)
+	AES_KEXPAND1(12, 16, 0x2, 18)
+	AES_KEXPAND2(14, 18, 20)
+	AES_KEXPAND2(16, 20, 22)
+	AES_KEXPAND1(18, 22, 0x3, 24)
+	AES_KEXPAND2(20, 24, 26)
+	AES_KEXPAND2(22, 26, 28)
+	AES_KEXPAND1(24, 28, 0x4, 30)
+	AES_KEXPAND2(26, 30, 32)
+	AES_KEXPAND2(28, 32, 34)
+	AES_KEXPAND1(30, 34, 0x5, 36)
+	AES_KEXPAND2(32, 36, 38)
+	AES_KEXPAND2(34, 38, 40)
+	AES_KEXPAND1(36, 40, 0x6, 42)
+	AES_KEXPAND2(38, 42, 44)
+	AES_KEXPAND2(40, 44, 46)
+	AES_KEXPAND1(42, 46, 0x7, 48)
+	AES_KEXPAND2(44, 48, 50)
+
+	std	%f6, [%o1 + 0x00]
+	std	%f8, [%o1 + 0x08]
+	std	%f10, [%o1 + 0x10]
+	std	%f12, [%o1 + 0x18]
+	std	%f14, [%o1 + 0x20]
+	std	%f16, [%o1 + 0x28]
+	std	%f18, [%o1 + 0x30]
+	std	%f20, [%o1 + 0x38]
+	std	%f22, [%o1 + 0x40]
+	std	%f24, [%o1 + 0x48]
+	std	%f26, [%o1 + 0x50]
+	std	%f28, [%o1 + 0x58]
+	std	%f30, [%o1 + 0x60]
+	std	%f32, [%o1 + 0x68]
+	std	%f34, [%o1 + 0x70]
+	std	%f36, [%o1 + 0x78]
+	std	%f38, [%o1 + 0x80]
+	std	%f40, [%o1 + 0x88]
+	std	%f42, [%o1 + 0x90]
+	std	%f44, [%o1 + 0x98]
+	std	%f46, [%o1 + 0xa0]
+	std	%f48, [%o1 + 0xa8]
+	ba,pt	%xcc, 80f
+	 std	%f50, [%o1 + 0xb0]
+
+2:
+	/* 128-bit key expansion */
+	AES_KEXPAND1(0, 2, 0x0, 4)
+	AES_KEXPAND2(2, 4, 6)
+	AES_KEXPAND1(4, 6, 0x1, 8)
+	AES_KEXPAND2(6, 8, 10)
+	AES_KEXPAND1(8, 10, 0x2, 12)
+	AES_KEXPAND2(10, 12, 14)
+	AES_KEXPAND1(12, 14, 0x3, 16)
+	AES_KEXPAND2(14, 16, 18)
+	AES_KEXPAND1(16, 18, 0x4, 20)
+	AES_KEXPAND2(18, 20, 22)
+	AES_KEXPAND1(20, 22, 0x5, 24)
+	AES_KEXPAND2(22, 24, 26)
+	AES_KEXPAND1(24, 26, 0x6, 28)
+	AES_KEXPAND2(26, 28, 30)
+	AES_KEXPAND1(28, 30, 0x7, 32)
+	AES_KEXPAND2(30, 32, 34)
+	AES_KEXPAND1(32, 34, 0x8, 36)
+	AES_KEXPAND2(34, 36, 38)
+	AES_KEXPAND1(36, 38, 0x9, 40)
+	AES_KEXPAND2(38, 40, 42)
+
+	std	%f4, [%o1 + 0x00]
+	std	%f6, [%o1 + 0x08]
+	std	%f8, [%o1 + 0x10]
+	std	%f10, [%o1 + 0x18]
+	std	%f12, [%o1 + 0x20]
+	std	%f14, [%o1 + 0x28]
+	std	%f16, [%o1 + 0x30]
+	std	%f18, [%o1 + 0x38]
+	std	%f20, [%o1 + 0x40]
+	std	%f22, [%o1 + 0x48]
+	std	%f24, [%o1 + 0x50]
+	std	%f26, [%o1 + 0x58]
+	std	%f28, [%o1 + 0x60]
+	std	%f30, [%o1 + 0x68]
+	std	%f32, [%o1 + 0x70]
+	std	%f34, [%o1 + 0x78]
+	std	%f36, [%o1 + 0x80]
+	std	%f38, [%o1 + 0x88]
+	std	%f40, [%o1 + 0x90]
+	std	%f42, [%o1 + 0x98]
+80:
+	retl
+	 VISExit
+ENDPROC(aes_sparc64_key_expand)
+
+	.align		32
+ENTRY(aes_sparc64_encrypt_128)
+	/* %o0=key, %o1=input, %o2=output */
+	VISEntry
+	ld		[%o1 + 0x00], %f4
+	ld		[%o1 + 0x04], %f5
+	ld		[%o1 + 0x08], %f6
+	ld		[%o1 + 0x0c], %f7
+	ldd		[%o0 + 0x00], %f8
+	ldd		[%o0 + 0x08], %f10
+	ldd		[%o0 + 0x10], %f12
+	ldd		[%o0 + 0x18], %f14
+	ldd		[%o0 + 0x20], %f16
+	ldd		[%o0 + 0x28], %f18
+	ldd		[%o0 + 0x30], %f20
+	ldd		[%o0 + 0x38], %f22
+	ldd		[%o0 + 0x40], %f24
+	ldd		[%o0 + 0x48], %f26
+	ldd		[%o0 + 0x50], %f28
+	ldd		[%o0 + 0x58], %f30
+	ldd		[%o0 + 0x60], %f32
+	ldd		[%o0 + 0x68], %f34
+	ldd		[%o0 + 0x70], %f36
+	ldd		[%o0 + 0x78], %f38
+	ldd		[%o0 + 0x80], %f40
+	ldd		[%o0 + 0x88], %f42
+	ldd		[%o0 + 0x90], %f44
+	ldd		[%o0 + 0x98], %f46
+	ldd		[%o0 + 0xa0], %f48
+	ldd		[%o0 + 0xa8], %f50
+	fxor		%f8, %f4, %f4
+	fxor		%f10, %f6, %f6
+	ENCRYPT_128(12, 4, 6, 0, 2)
+	st		%f4, [%o2 + 0x00]
+	st		%f5, [%o2 + 0x04]
+	st		%f6, [%o2 + 0x08]
+	st		%f7, [%o2 + 0x0c]
+	retl
+	 VISExit
+ENDPROC(aes_sparc64_encrypt_128)
+
+	.align		32
+ENTRY(aes_sparc64_encrypt_192)
+	/* %o0=key, %o1=input, %o2=output */
+	VISEntry
+	ld		[%o1 + 0x00], %f4
+	ld		[%o1 + 0x04], %f5
+	ld		[%o1 + 0x08], %f6
+	ld		[%o1 + 0x0c], %f7
+
+	ldd		[%o0 + 0x00], %f8
+	ldd		[%o0 + 0x08], %f10
+
+	fxor		%f8, %f4, %f4
+	fxor		%f10, %f6, %f6
+
+	ldd		[%o0 + 0x10], %f8
+	ldd		[%o0 + 0x18], %f10
+	ldd		[%o0 + 0x20], %f12
+	ldd		[%o0 + 0x28], %f14
+	add		%o0, 0x20, %o0
+
+	ENCRYPT_TWO_ROUNDS(8, 4, 6, 0, 2)
+
+	ldd		[%o0 + 0x10], %f12
+	ldd		[%o0 + 0x18], %f14
+	ldd		[%o0 + 0x20], %f16
+	ldd		[%o0 + 0x28], %f18
+	ldd		[%o0 + 0x30], %f20
+	ldd		[%o0 + 0x38], %f22
+	ldd		[%o0 + 0x40], %f24
+	ldd		[%o0 + 0x48], %f26
+	ldd		[%o0 + 0x50], %f28
+	ldd		[%o0 + 0x58], %f30
+	ldd		[%o0 + 0x60], %f32
+	ldd		[%o0 + 0x68], %f34
+	ldd		[%o0 + 0x70], %f36
+	ldd		[%o0 + 0x78], %f38
+	ldd		[%o0 + 0x80], %f40
+	ldd		[%o0 + 0x88], %f42
+	ldd		[%o0 + 0x90], %f44
+	ldd		[%o0 + 0x98], %f46
+	ldd		[%o0 + 0xa0], %f48
+	ldd		[%o0 + 0xa8], %f50
+
+
+	ENCRYPT_128(12, 4, 6, 0, 2)
+
+	st		%f4, [%o2 + 0x00]
+	st		%f5, [%o2 + 0x04]
+	st		%f6, [%o2 + 0x08]
+	st		%f7, [%o2 + 0x0c]
+
+	retl
+	 VISExit
+ENDPROC(aes_sparc64_encrypt_192)
+
+	.align		32
+ENTRY(aes_sparc64_encrypt_256)
+	/* %o0=key, %o1=input, %o2=output */
+	VISEntry
+	ld		[%o1 + 0x00], %f4
+	ld		[%o1 + 0x04], %f5
+	ld		[%o1 + 0x08], %f6
+	ld		[%o1 + 0x0c], %f7
+
+	ldd		[%o0 + 0x00], %f8
+	ldd		[%o0 + 0x08], %f10
+
+	fxor		%f8, %f4, %f4
+	fxor		%f10, %f6, %f6
+
+	ldd		[%o0 + 0x10], %f8
+
+	ldd		[%o0 + 0x18], %f10
+	ldd		[%o0 + 0x20], %f12
+	ldd		[%o0 + 0x28], %f14
+	add		%o0, 0x20, %o0
+
+	ENCRYPT_TWO_ROUNDS(8, 4, 6, 0, 2)
+
+	ldd		[%o0 + 0x10], %f8
+
+	ldd		[%o0 + 0x18], %f10
+	ldd		[%o0 + 0x20], %f12
+	ldd		[%o0 + 0x28], %f14
+	add		%o0, 0x20, %o0
+
+	ENCRYPT_TWO_ROUNDS(8, 4, 6, 0, 2)
+
+	ldd		[%o0 + 0x10], %f12
+	ldd		[%o0 + 0x18], %f14
+	ldd		[%o0 + 0x20], %f16
+	ldd		[%o0 + 0x28], %f18
+	ldd		[%o0 + 0x30], %f20
+	ldd		[%o0 + 0x38], %f22
+	ldd		[%o0 + 0x40], %f24
+	ldd		[%o0 + 0x48], %f26
+	ldd		[%o0 + 0x50], %f28
+	ldd		[%o0 + 0x58], %f30
+	ldd		[%o0 + 0x60], %f32
+	ldd		[%o0 + 0x68], %f34
+	ldd		[%o0 + 0x70], %f36
+	ldd		[%o0 + 0x78], %f38
+	ldd		[%o0 + 0x80], %f40
+	ldd		[%o0 + 0x88], %f42
+	ldd		[%o0 + 0x90], %f44
+	ldd		[%o0 + 0x98], %f46
+	ldd		[%o0 + 0xa0], %f48
+	ldd		[%o0 + 0xa8], %f50
+
+	ENCRYPT_128(12, 4, 6, 0, 2)
+
+	st		%f4, [%o2 + 0x00]
+	st		%f5, [%o2 + 0x04]
+	st		%f6, [%o2 + 0x08]
+	st		%f7, [%o2 + 0x0c]
+
+	retl
+	 VISExit
+ENDPROC(aes_sparc64_encrypt_256)
+
+	.align		32
+ENTRY(aes_sparc64_decrypt_128)
+	/* %o0=key, %o1=input, %o2=output */
+	VISEntry
+	ld		[%o1 + 0x00], %f4
+	ld		[%o1 + 0x04], %f5
+	ld		[%o1 + 0x08], %f6
+	ld		[%o1 + 0x0c], %f7
+	ldd		[%o0 + 0xa0], %f8
+	ldd		[%o0 + 0xa8], %f10
+	ldd		[%o0 + 0x98], %f12
+	ldd		[%o0 + 0x90], %f14
+	ldd		[%o0 + 0x88], %f16
+	ldd		[%o0 + 0x80], %f18
+	ldd		[%o0 + 0x78], %f20
+	ldd		[%o0 + 0x70], %f22
+	ldd		[%o0 + 0x68], %f24
+	ldd		[%o0 + 0x60], %f26
+	ldd		[%o0 + 0x58], %f28
+	ldd		[%o0 + 0x50], %f30
+	ldd		[%o0 + 0x48], %f32
+	ldd		[%o0 + 0x40], %f34
+	ldd		[%o0 + 0x38], %f36
+	ldd		[%o0 + 0x30], %f38
+	ldd		[%o0 + 0x28], %f40
+	ldd		[%o0 + 0x20], %f42
+	ldd		[%o0 + 0x18], %f44
+	ldd		[%o0 + 0x10], %f46
+	ldd		[%o0 + 0x08], %f48
+	ldd		[%o0 + 0x00], %f50
+	fxor		%f8, %f4, %f4
+	fxor		%f10, %f6, %f6
+	DECRYPT_128(12, 4, 6, 0, 2)
+	st		%f4, [%o2 + 0x00]
+	st		%f5, [%o2 + 0x04]
+	st		%f6, [%o2 + 0x08]
+	st		%f7, [%o2 + 0x0c]
+	retl
+	 VISExit
+ENDPROC(aes_sparc64_decrypt_128)
+
+	.align		32
+ENTRY(aes_sparc64_decrypt_192)
+	/* %o0=key, %o1=input, %o2=output */
+	VISEntry
+	ld		[%o1 + 0x00], %f4
+	ld		[%o1 + 0x04], %f5
+	ld		[%o1 + 0x08], %f6
+	ld		[%o1 + 0x0c], %f7
+	ldd		[%o0 + 0xc0], %f8
+	ldd		[%o0 + 0xc8], %f10
+	ldd		[%o0 + 0xb8], %f12
+	ldd		[%o0 + 0xb0], %f14
+	ldd		[%o0 + 0xa8], %f16
+	ldd		[%o0 + 0xa0], %f18
+	fxor		%f8, %f4, %f4
+	fxor		%f10, %f6, %f6
+	ldd		[%o0 + 0x98], %f20
+	ldd		[%o0 + 0x90], %f22
+	ldd		[%o0 + 0x88], %f24
+	ldd		[%o0 + 0x80], %f26
+	DECRYPT_TWO_ROUNDS(12, 4, 6, 0, 2)
+	ldd		[%o0 + 0x78], %f28
+	ldd		[%o0 + 0x70], %f30
+	ldd		[%o0 + 0x68], %f32
+	ldd		[%o0 + 0x60], %f34
+	ldd		[%o0 + 0x58], %f36
+	ldd		[%o0 + 0x50], %f38
+	ldd		[%o0 + 0x48], %f40
+	ldd		[%o0 + 0x40], %f42
+	ldd		[%o0 + 0x38], %f44
+	ldd		[%o0 + 0x30], %f46
+	ldd		[%o0 + 0x28], %f48
+	ldd		[%o0 + 0x20], %f50
+	ldd		[%o0 + 0x18], %f52
+	ldd		[%o0 + 0x10], %f54
+	ldd		[%o0 + 0x08], %f56
+	ldd		[%o0 + 0x00], %f58
+	DECRYPT_128(20, 4, 6, 0, 2)
+	st		%f4, [%o2 + 0x00]
+	st		%f5, [%o2 + 0x04]
+	st		%f6, [%o2 + 0x08]
+	st		%f7, [%o2 + 0x0c]
+	retl
+	 VISExit
+ENDPROC(aes_sparc64_decrypt_192)
+
+	.align		32
+ENTRY(aes_sparc64_decrypt_256)
+	/* %o0=key, %o1=input, %o2=output */
+	VISEntry
+	ld		[%o1 + 0x00], %f4
+	ld		[%o1 + 0x04], %f5
+	ld		[%o1 + 0x08], %f6
+	ld		[%o1 + 0x0c], %f7
+	ldd		[%o0 + 0xe0], %f8
+	ldd		[%o0 + 0xe8], %f10
+	ldd		[%o0 + 0xd8], %f12
+	ldd		[%o0 + 0xd0], %f14
+	ldd		[%o0 + 0xc8], %f16
+	fxor		%f8, %f4, %f4
+	ldd		[%o0 + 0xc0], %f18
+	fxor		%f10, %f6, %f6
+	ldd		[%o0 + 0xb8], %f20
+	AES_DROUND23(12, 4, 6, 2)
+	ldd		[%o0 + 0xb0], %f22
+	AES_DROUND01(14, 4, 6, 0)
+	ldd		[%o0 + 0xa8], %f24
+	AES_DROUND23(16, 0, 2, 6)
+	ldd		[%o0 + 0xa0], %f26
+	AES_DROUND01(18, 0, 2, 4)
+	ldd		[%o0 + 0x98], %f12
+	AES_DROUND23(20, 4, 6, 2)
+	ldd		[%o0 + 0x90], %f14
+	AES_DROUND01(22, 4, 6, 0)
+	ldd		[%o0 + 0x88], %f16
+	AES_DROUND23(24, 0, 2, 6)
+	ldd		[%o0 + 0x80], %f18
+	AES_DROUND01(26, 0, 2, 4)
+	ldd		[%o0 + 0x78], %f20
+	AES_DROUND23(12, 4, 6, 2)
+	ldd		[%o0 + 0x70], %f22
+	AES_DROUND01(14, 4, 6, 0)
+	ldd		[%o0 + 0x68], %f24
+	AES_DROUND23(16, 0, 2, 6)
+	ldd		[%o0 + 0x60], %f26
+	AES_DROUND01(18, 0, 2, 4)
+	ldd		[%o0 + 0x58], %f28
+	AES_DROUND23(20, 4, 6, 2)
+	ldd		[%o0 + 0x50], %f30
+	AES_DROUND01(22, 4, 6, 0)
+	ldd		[%o0 + 0x48], %f32
+	AES_DROUND23(24, 0, 2, 6)
+	ldd		[%o0 + 0x40], %f34
+	AES_DROUND01(26, 0, 2, 4)
+	ldd		[%o0 + 0x38], %f36
+	AES_DROUND23(28, 4, 6, 2)
+	ldd		[%o0 + 0x30], %f38
+	AES_DROUND01(30, 4, 6, 0)
+	ldd		[%o0 + 0x28], %f40
+	AES_DROUND23(32, 0, 2, 6)
+	ldd		[%o0 + 0x20], %f42
+	AES_DROUND01(34, 0, 2, 4)
+	ldd		[%o0 + 0x18], %f44
+	AES_DROUND23(36, 4, 6, 2)
+	ldd		[%o0 + 0x10], %f46
+	AES_DROUND01(38, 4, 6, 0)
+	ldd		[%o0 + 0x08], %f48
+	AES_DROUND23(40, 0, 2, 6)
+	ldd		[%o0 + 0x00], %f50
+	AES_DROUND01(42, 0, 2, 4)
+	AES_DROUND23(44, 4, 6, 2)
+	AES_DROUND01(46, 4, 6, 0)
+	AES_DROUND23_L(48, 0, 2, 6)
+	AES_DROUND01_L(50, 0, 2, 4)
+	st		%f4, [%o2 + 0x00]
+	st		%f5, [%o2 + 0x04]
+	st		%f6, [%o2 + 0x08]
+	st		%f7, [%o2 + 0x0c]
+	retl
+	 VISExit
+ENDPROC(aes_sparc64_decrypt_256)
+
+	.align		32
+ENTRY(aes_sparc64_load_encrypt_keys_128)
+	/* %o0=key */
+	VISEntry
+	ldd		[%o0 + 0x10], %f8
+	ldd		[%o0 + 0x18], %f10
+	ldd		[%o0 + 0x20], %f12
+	ldd		[%o0 + 0x28], %f14
+	ldd		[%o0 + 0x30], %f16
+	ldd		[%o0 + 0x38], %f18
+	ldd		[%o0 + 0x40], %f20
+	ldd		[%o0 + 0x48], %f22
+	ldd		[%o0 + 0x50], %f24
+	ldd		[%o0 + 0x58], %f26
+	ldd		[%o0 + 0x60], %f28
+	ldd		[%o0 + 0x68], %f30
+	ldd		[%o0 + 0x70], %f32
+	ldd		[%o0 + 0x78], %f34
+	ldd		[%o0 + 0x80], %f36
+	ldd		[%o0 + 0x88], %f38
+	ldd		[%o0 + 0x90], %f40
+	ldd		[%o0 + 0x98], %f42
+	ldd		[%o0 + 0xa0], %f44
+	retl
+	 ldd		[%o0 + 0xa8], %f46
+ENDPROC(aes_sparc64_load_encrypt_keys_128)
+
+	.align		32
+ENTRY(aes_sparc64_load_encrypt_keys_192)
+	/* %o0=key */
+	VISEntry
+	ldd		[%o0 + 0x10], %f8
+	ldd		[%o0 + 0x18], %f10
+	ldd		[%o0 + 0x20], %f12
+	ldd		[%o0 + 0x28], %f14
+	ldd		[%o0 + 0x30], %f16
+	ldd		[%o0 + 0x38], %f18
+	ldd		[%o0 + 0x40], %f20
+	ldd		[%o0 + 0x48], %f22
+	ldd		[%o0 + 0x50], %f24
+	ldd		[%o0 + 0x58], %f26
+	ldd		[%o0 + 0x60], %f28
+	ldd		[%o0 + 0x68], %f30
+	ldd		[%o0 + 0x70], %f32
+	ldd		[%o0 + 0x78], %f34
+	ldd		[%o0 + 0x80], %f36
+	ldd		[%o0 + 0x88], %f38
+	ldd		[%o0 + 0x90], %f40
+	ldd		[%o0 + 0x98], %f42
+	ldd		[%o0 + 0xa0], %f44
+	ldd		[%o0 + 0xa8], %f46
+	ldd		[%o0 + 0xb0], %f48
+	ldd		[%o0 + 0xb8], %f50
+	ldd		[%o0 + 0xc0], %f52
+	retl
+	 ldd		[%o0 + 0xc8], %f54
+ENDPROC(aes_sparc64_load_encrypt_keys_192)
+
+	.align		32
+ENTRY(aes_sparc64_load_encrypt_keys_256)
+	/* %o0=key */
+	VISEntry
+	ldd		[%o0 + 0x10], %f8
+	ldd		[%o0 + 0x18], %f10
+	ldd		[%o0 + 0x20], %f12
+	ldd		[%o0 + 0x28], %f14
+	ldd		[%o0 + 0x30], %f16
+	ldd		[%o0 + 0x38], %f18
+	ldd		[%o0 + 0x40], %f20
+	ldd		[%o0 + 0x48], %f22
+	ldd		[%o0 + 0x50], %f24
+	ldd		[%o0 + 0x58], %f26
+	ldd		[%o0 + 0x60], %f28
+	ldd		[%o0 + 0x68], %f30
+	ldd		[%o0 + 0x70], %f32
+	ldd		[%o0 + 0x78], %f34
+	ldd		[%o0 + 0x80], %f36
+	ldd		[%o0 + 0x88], %f38
+	ldd		[%o0 + 0x90], %f40
+	ldd		[%o0 + 0x98], %f42
+	ldd		[%o0 + 0xa0], %f44
+	ldd		[%o0 + 0xa8], %f46
+	ldd		[%o0 + 0xb0], %f48
+	ldd		[%o0 + 0xb8], %f50
+	ldd		[%o0 + 0xc0], %f52
+	ldd		[%o0 + 0xc8], %f54
+	ldd		[%o0 + 0xd0], %f56
+	ldd		[%o0 + 0xd8], %f58
+	ldd		[%o0 + 0xe0], %f60
+	retl
+	 ldd		[%o0 + 0xe8], %f62
+ENDPROC(aes_sparc64_load_encrypt_keys_256)
+
+	.align		32
+ENTRY(aes_sparc64_load_decrypt_keys_128)
+	/* %o0=key */
+	VISEntry
+	ldd		[%o0 + 0x98], %f8
+	ldd		[%o0 + 0x90], %f10
+	ldd		[%o0 + 0x88], %f12
+	ldd		[%o0 + 0x80], %f14
+	ldd		[%o0 + 0x78], %f16
+	ldd		[%o0 + 0x70], %f18
+	ldd		[%o0 + 0x68], %f20
+	ldd		[%o0 + 0x60], %f22
+	ldd		[%o0 + 0x58], %f24
+	ldd		[%o0 + 0x50], %f26
+	ldd		[%o0 + 0x48], %f28
+	ldd		[%o0 + 0x40], %f30
+	ldd		[%o0 + 0x38], %f32
+	ldd		[%o0 + 0x30], %f34
+	ldd		[%o0 + 0x28], %f36
+	ldd		[%o0 + 0x20], %f38
+	ldd		[%o0 + 0x18], %f40
+	ldd		[%o0 + 0x10], %f42
+	ldd		[%o0 + 0x08], %f44
+	retl
+	 ldd		[%o0 + 0x00], %f46
+ENDPROC(aes_sparc64_load_decrypt_keys_128)
+
+	.align		32
+ENTRY(aes_sparc64_load_decrypt_keys_192)
+	/* %o0=key */
+	VISEntry
+	ldd		[%o0 + 0xb8], %f8
+	ldd		[%o0 + 0xb0], %f10
+	ldd		[%o0 + 0xa8], %f12
+	ldd		[%o0 + 0xa0], %f14
+	ldd		[%o0 + 0x98], %f16
+	ldd		[%o0 + 0x90], %f18
+	ldd		[%o0 + 0x88], %f20
+	ldd		[%o0 + 0x80], %f22
+	ldd		[%o0 + 0x78], %f24
+	ldd		[%o0 + 0x70], %f26
+	ldd		[%o0 + 0x68], %f28
+	ldd		[%o0 + 0x60], %f30
+	ldd		[%o0 + 0x58], %f32
+	ldd		[%o0 + 0x50], %f34
+	ldd		[%o0 + 0x48], %f36
+	ldd		[%o0 + 0x40], %f38
+	ldd		[%o0 + 0x38], %f40
+	ldd		[%o0 + 0x30], %f42
+	ldd		[%o0 + 0x28], %f44
+	ldd		[%o0 + 0x20], %f46
+	ldd		[%o0 + 0x18], %f48
+	ldd		[%o0 + 0x10], %f50
+	ldd		[%o0 + 0x08], %f52
+	retl
+	 ldd		[%o0 + 0x00], %f54
+ENDPROC(aes_sparc64_load_decrypt_keys_192)
+
+	.align		32
+ENTRY(aes_sparc64_load_decrypt_keys_256)
+	/* %o0=key */
+	VISEntry
+	ldd		[%o0 + 0xd8], %f8
+	ldd		[%o0 + 0xd0], %f10
+	ldd		[%o0 + 0xc8], %f12
+	ldd		[%o0 + 0xc0], %f14
+	ldd		[%o0 + 0xb8], %f16
+	ldd		[%o0 + 0xb0], %f18
+	ldd		[%o0 + 0xa8], %f20
+	ldd		[%o0 + 0xa0], %f22
+	ldd		[%o0 + 0x98], %f24
+	ldd		[%o0 + 0x90], %f26
+	ldd		[%o0 + 0x88], %f28
+	ldd		[%o0 + 0x80], %f30
+	ldd		[%o0 + 0x78], %f32
+	ldd		[%o0 + 0x70], %f34
+	ldd		[%o0 + 0x68], %f36
+	ldd		[%o0 + 0x60], %f38
+	ldd		[%o0 + 0x58], %f40
+	ldd		[%o0 + 0x50], %f42
+	ldd		[%o0 + 0x48], %f44
+	ldd		[%o0 + 0x40], %f46
+	ldd		[%o0 + 0x38], %f48
+	ldd		[%o0 + 0x30], %f50
+	ldd		[%o0 + 0x28], %f52
+	ldd		[%o0 + 0x20], %f54
+	ldd		[%o0 + 0x18], %f56
+	ldd		[%o0 + 0x10], %f58
+	ldd		[%o0 + 0x08], %f60
+	retl
+	 ldd		[%o0 + 0x00], %f62
+ENDPROC(aes_sparc64_load_decrypt_keys_256)
+
+	.align		32
+ENTRY(aes_sparc64_ecb_encrypt_128)
+	/* %o0=key, %o1=input, %o2=output, %o3=len */
+	ldx		[%o0 + 0x00], %g1
+	subcc		%o3, 0x10, %o3
+	be		10f
+	 ldx		[%o0 + 0x08], %g2
+1:	ldx		[%o1 + 0x00], %g3
+	ldx		[%o1 + 0x08], %g7
+	ldx		[%o1 + 0x10], %o4
+	ldx		[%o1 + 0x18], %o5
+	xor		%g1, %g3, %g3
+	xor		%g2, %g7, %g7
+	MOVXTOD_G3_F4
+	MOVXTOD_G7_F6
+	xor		%g1, %o4, %g3
+	xor		%g2, %o5, %g7
+	MOVXTOD_G3_F60
+	MOVXTOD_G7_F62
+	ENCRYPT_128_2(8, 4, 6, 60, 62, 0, 2, 56, 58)
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+	std		%f60, [%o2 + 0x10]
+	std		%f62, [%o2 + 0x18]
+	sub		%o3, 0x20, %o3
+	add		%o1, 0x20, %o1
+	brgz		%o3, 1b
+	 add		%o2, 0x20, %o2
+	brlz,pt		%o3, 11f
+	 nop
+10:	ldx		[%o1 + 0x00], %g3
+	ldx		[%o1 + 0x08], %g7
+	xor		%g1, %g3, %g3
+	xor		%g2, %g7, %g7
+	MOVXTOD_G3_F4
+	MOVXTOD_G7_F6
+	ENCRYPT_128(8, 4, 6, 0, 2)
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+11:	retl
+	 nop
+ENDPROC(aes_sparc64_ecb_encrypt_128)
+
+	.align		32
+ENTRY(aes_sparc64_ecb_encrypt_192)
+	/* %o0=key, %o1=input, %o2=output, %o3=len */
+	ldx		[%o0 + 0x00], %g1
+	subcc		%o3, 0x10, %o3
+	be		10f
+	 ldx		[%o0 + 0x08], %g2
+1:	ldx		[%o1 + 0x00], %g3
+	ldx		[%o1 + 0x08], %g7
+	ldx		[%o1 + 0x10], %o4
+	ldx		[%o1 + 0x18], %o5
+	xor		%g1, %g3, %g3
+	xor		%g2, %g7, %g7
+	MOVXTOD_G3_F4
+	MOVXTOD_G7_F6
+	xor		%g1, %o4, %g3
+	xor		%g2, %o5, %g7
+	MOVXTOD_G3_F60
+	MOVXTOD_G7_F62
+	ENCRYPT_192_2(8, 4, 6, 60, 62, 0, 2, 56, 58)
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+	std		%f60, [%o2 + 0x10]
+	std		%f62, [%o2 + 0x18]
+	sub		%o3, 0x20, %o3
+	add		%o1, 0x20, %o1
+	brgz		%o3, 1b
+	 add		%o2, 0x20, %o2
+	brlz,pt		%o3, 11f
+	 nop
+10:	ldx		[%o1 + 0x00], %g3
+	ldx		[%o1 + 0x08], %g7
+	xor		%g1, %g3, %g3
+	xor		%g2, %g7, %g7
+	MOVXTOD_G3_F4
+	MOVXTOD_G7_F6
+	ENCRYPT_192(8, 4, 6, 0, 2)
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+11:	retl
+	 nop
+ENDPROC(aes_sparc64_ecb_encrypt_192)
+
+	.align		32
+ENTRY(aes_sparc64_ecb_encrypt_256)
+	/* %o0=key, %o1=input, %o2=output, %o3=len */
+	ldx		[%o0 + 0x00], %g1
+	subcc		%o3, 0x10, %o3
+	be		10f
+	 ldx		[%o0 + 0x08], %g2
+1:	ldx		[%o1 + 0x00], %g3
+	ldx		[%o1 + 0x08], %g7
+	ldx		[%o1 + 0x10], %o4
+	ldx		[%o1 + 0x18], %o5
+	xor		%g1, %g3, %g3
+	xor		%g2, %g7, %g7
+	MOVXTOD_G3_F4
+	MOVXTOD_G7_F6
+	xor		%g1, %o4, %g3
+	xor		%g2, %o5, %g7
+	MOVXTOD_G3_F0
+	MOVXTOD_G7_F2
+	ENCRYPT_256_2(8, 4, 6, 0, 2)
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+	std		%f0, [%o2 + 0x10]
+	std		%f2, [%o2 + 0x18]
+	sub		%o3, 0x20, %o3
+	add		%o1, 0x20, %o1
+	brgz		%o3, 1b
+	 add		%o2, 0x20, %o2
+	brlz,pt		%o3, 11f
+	 nop
+10:	ldd		[%o0 + 0xd0], %f56
+	ldd		[%o0 + 0xd8], %f58
+	ldd		[%o0 + 0xe0], %f60
+	ldd		[%o0 + 0xe8], %f62
+	ldx		[%o1 + 0x00], %g3
+	ldx		[%o1 + 0x08], %g7
+	xor		%g1, %g3, %g3
+	xor		%g2, %g7, %g7
+	MOVXTOD_G3_F4
+	MOVXTOD_G7_F6
+	ENCRYPT_256(8, 4, 6, 0, 2)
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+11:	retl
+	 nop
+ENDPROC(aes_sparc64_ecb_encrypt_256)
+
+	.align		32
+ENTRY(aes_sparc64_ecb_decrypt_128)
+	/* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */
+	ldx		[%o0 - 0x10], %g1
+	subcc		%o3, 0x10, %o3
+	be		10f
+	 ldx		[%o0 - 0x08], %g2
+1:	ldx		[%o1 + 0x00], %g3
+	ldx		[%o1 + 0x08], %g7
+	ldx		[%o1 + 0x10], %o4
+	ldx		[%o1 + 0x18], %o5
+	xor		%g1, %g3, %g3
+	xor		%g2, %g7, %g7
+	MOVXTOD_G3_F4
+	MOVXTOD_G7_F6
+	xor		%g1, %o4, %g3
+	xor		%g2, %o5, %g7
+	MOVXTOD_G3_F60
+	MOVXTOD_G7_F62
+	DECRYPT_128_2(8, 4, 6, 60, 62, 0, 2, 56, 58)
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+	std		%f60, [%o2 + 0x10]
+	std		%f62, [%o2 + 0x18]
+	sub		%o3, 0x20, %o3
+	add		%o1, 0x20, %o1
+	brgz,pt		%o3, 1b
+	 add		%o2, 0x20, %o2
+	brlz,pt		%o3, 11f
+	 nop
+10:	ldx		[%o1 + 0x00], %g3
+	ldx		[%o1 + 0x08], %g7
+	xor		%g1, %g3, %g3
+	xor		%g2, %g7, %g7
+	MOVXTOD_G3_F4
+	MOVXTOD_G7_F6
+	DECRYPT_128(8, 4, 6, 0, 2)
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+11:	retl
+	 nop
+ENDPROC(aes_sparc64_ecb_decrypt_128)
+
+	.align		32
+ENTRY(aes_sparc64_ecb_decrypt_192)
+	/* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */
+	ldx		[%o0 - 0x10], %g1
+	subcc		%o3, 0x10, %o3
+	be		10f
+	 ldx		[%o0 - 0x08], %g2
+1:	ldx		[%o1 + 0x00], %g3
+	ldx		[%o1 + 0x08], %g7
+	ldx		[%o1 + 0x10], %o4
+	ldx		[%o1 + 0x18], %o5
+	xor		%g1, %g3, %g3
+	xor		%g2, %g7, %g7
+	MOVXTOD_G3_F4
+	MOVXTOD_G7_F6
+	xor		%g1, %o4, %g3
+	xor		%g2, %o5, %g7
+	MOVXTOD_G3_F60
+	MOVXTOD_G7_F62
+	DECRYPT_192_2(8, 4, 6, 60, 62, 0, 2, 56, 58)
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+	std		%f60, [%o2 + 0x10]
+	std		%f62, [%o2 + 0x18]
+	sub		%o3, 0x20, %o3
+	add		%o1, 0x20, %o1
+	brgz,pt		%o3, 1b
+	 add		%o2, 0x20, %o2
+	brlz,pt		%o3, 11f
+	 nop
+10:	ldx		[%o1 + 0x00], %g3
+	ldx		[%o1 + 0x08], %g7
+	xor		%g1, %g3, %g3
+	xor		%g2, %g7, %g7
+	MOVXTOD_G3_F4
+	MOVXTOD_G7_F6
+	DECRYPT_192(8, 4, 6, 0, 2)
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+11:	retl
+	 nop
+ENDPROC(aes_sparc64_ecb_decrypt_192)
+
+	.align		32
+ENTRY(aes_sparc64_ecb_decrypt_256)
+	/* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */
+	ldx		[%o0 - 0x10], %g1
+	subcc		%o3, 0x10, %o3
+	ldx		[%o0 - 0x08], %g2
+	be		10f
+	 sub		%o0, 0xf0, %o0
+1:	ldx		[%o1 + 0x00], %g3
+	ldx		[%o1 + 0x08], %g7
+	ldx		[%o1 + 0x10], %o4
+	ldx		[%o1 + 0x18], %o5
+	xor		%g1, %g3, %g3
+	xor		%g2, %g7, %g7
+	MOVXTOD_G3_F4
+	MOVXTOD_G7_F6
+	xor		%g1, %o4, %g3
+	xor		%g2, %o5, %g7
+	MOVXTOD_G3_F0
+	MOVXTOD_G7_F2
+	DECRYPT_256_2(8, 4, 6, 0, 2)
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+	std		%f0, [%o2 + 0x10]
+	std		%f2, [%o2 + 0x18]
+	sub		%o3, 0x20, %o3
+	add		%o1, 0x20, %o1
+	brgz,pt		%o3, 1b
+	 add		%o2, 0x20, %o2
+	brlz,pt		%o3, 11f
+	 nop
+10:	ldd		[%o0 + 0x18], %f56
+	ldd		[%o0 + 0x10], %f58
+	ldd		[%o0 + 0x08], %f60
+	ldd		[%o0 + 0x00], %f62
+	ldx		[%o1 + 0x00], %g3
+	ldx		[%o1 + 0x08], %g7
+	xor		%g1, %g3, %g3
+	xor		%g2, %g7, %g7
+	MOVXTOD_G3_F4
+	MOVXTOD_G7_F6
+	DECRYPT_256(8, 4, 6, 0, 2)
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+11:	retl
+	 nop
+ENDPROC(aes_sparc64_ecb_decrypt_256)
+
+	.align		32
+ENTRY(aes_sparc64_cbc_encrypt_128)
+	/* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
+	ldd		[%o4 + 0x00], %f4
+	ldd		[%o4 + 0x08], %f6
+	ldx		[%o0 + 0x00], %g1
+	ldx		[%o0 + 0x08], %g2
+1:	ldx		[%o1 + 0x00], %g3
+	ldx		[%o1 + 0x08], %g7
+	add		%o1, 0x10, %o1
+	xor		%g1, %g3, %g3
+	xor		%g2, %g7, %g7
+	MOVXTOD_G3_F0
+	MOVXTOD_G7_F2
+	fxor		%f4, %f0, %f4
+	fxor		%f6, %f2, %f6
+	ENCRYPT_128(8, 4, 6, 0, 2)
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+	subcc		%o3, 0x10, %o3
+	bne,pt		%xcc, 1b
+	 add		%o2, 0x10, %o2
+	std		%f4, [%o4 + 0x00]
+	std		%f6, [%o4 + 0x08]
+	retl
+	 nop
+ENDPROC(aes_sparc64_cbc_encrypt_128)
+
+	.align		32
+ENTRY(aes_sparc64_cbc_encrypt_192)
+	/* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
+	ldd		[%o4 + 0x00], %f4
+	ldd		[%o4 + 0x08], %f6
+	ldx		[%o0 + 0x00], %g1
+	ldx		[%o0 + 0x08], %g2
+1:	ldx		[%o1 + 0x00], %g3
+	ldx		[%o1 + 0x08], %g7
+	add		%o1, 0x10, %o1
+	xor		%g1, %g3, %g3
+	xor		%g2, %g7, %g7
+	MOVXTOD_G3_F0
+	MOVXTOD_G7_F2
+	fxor		%f4, %f0, %f4
+	fxor		%f6, %f2, %f6
+	ENCRYPT_192(8, 4, 6, 0, 2)
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+	subcc		%o3, 0x10, %o3
+	bne,pt		%xcc, 1b
+	 add		%o2, 0x10, %o2
+	std		%f4, [%o4 + 0x00]
+	std		%f6, [%o4 + 0x08]
+	retl
+	 nop
+ENDPROC(aes_sparc64_cbc_encrypt_192)
+
+	.align		32
+ENTRY(aes_sparc64_cbc_encrypt_256)
+	/* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
+	ldd		[%o4 + 0x00], %f4
+	ldd		[%o4 + 0x08], %f6
+	ldx		[%o0 + 0x00], %g1
+	ldx		[%o0 + 0x08], %g2
+1:	ldx		[%o1 + 0x00], %g3
+	ldx		[%o1 + 0x08], %g7
+	add		%o1, 0x10, %o1
+	xor		%g1, %g3, %g3
+	xor		%g2, %g7, %g7
+	MOVXTOD_G3_F0
+	MOVXTOD_G7_F2
+	fxor		%f4, %f0, %f4
+	fxor		%f6, %f2, %f6
+	ENCRYPT_256(8, 4, 6, 0, 2)
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+	subcc		%o3, 0x10, %o3
+	bne,pt		%xcc, 1b
+	 add		%o2, 0x10, %o2
+	std		%f4, [%o4 + 0x00]
+	std		%f6, [%o4 + 0x08]
+	retl
+	 nop
+ENDPROC(aes_sparc64_cbc_encrypt_256)
+
+	.align		32
+ENTRY(aes_sparc64_cbc_decrypt_128)
+	/* %o0=&key[key_len], %o1=input, %o2=output, %o3=len, %o4=iv */
+	ldx		[%o0 - 0x10], %g1
+	ldx		[%o0 - 0x08], %g2
+	ldx		[%o4 + 0x00], %o0
+	ldx		[%o4 + 0x08], %o5
+1:	ldx		[%o1 + 0x00], %g3
+	ldx		[%o1 + 0x08], %g7
+	add		%o1, 0x10, %o1
+	xor		%g1, %g3, %g3
+	xor		%g2, %g7, %g7
+	MOVXTOD_G3_F4
+	MOVXTOD_G7_F6
+	DECRYPT_128(8, 4, 6, 0, 2)
+	MOVXTOD_O0_F0
+	MOVXTOD_O5_F2
+	xor		%g1, %g3, %o0
+	xor		%g2, %g7, %o5
+	fxor		%f4, %f0, %f4
+	fxor		%f6, %f2, %f6
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+	subcc		%o3, 0x10, %o3
+	bne,pt		%xcc, 1b
+	 add		%o2, 0x10, %o2
+	stx		%o0, [%o4 + 0x00]
+	stx		%o5, [%o4 + 0x08]
+	retl
+	 nop
+ENDPROC(aes_sparc64_cbc_decrypt_128)
+
+	.align		32
+ENTRY(aes_sparc64_cbc_decrypt_192)
+	/* %o0=&key[key_len], %o1=input, %o2=output, %o3=len, %o4=iv */
+	ldx		[%o0 - 0x10], %g1
+	ldx		[%o0 - 0x08], %g2
+	ldx		[%o4 + 0x00], %o0
+	ldx		[%o4 + 0x08], %o5
+1:	ldx		[%o1 + 0x00], %g3
+	ldx		[%o1 + 0x08], %g7
+	add		%o1, 0x10, %o1
+	xor		%g1, %g3, %g3
+	xor		%g2, %g7, %g7
+	MOVXTOD_G3_F4
+	MOVXTOD_G7_F6
+	DECRYPT_192(8, 4, 6, 0, 2)
+	MOVXTOD_O0_F0
+	MOVXTOD_O5_F2
+	xor		%g1, %g3, %o0
+	xor		%g2, %g7, %o5
+	fxor		%f4, %f0, %f4
+	fxor		%f6, %f2, %f6
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+	subcc		%o3, 0x10, %o3
+	bne,pt		%xcc, 1b
+	 add		%o2, 0x10, %o2
+	stx		%o0, [%o4 + 0x00]
+	stx		%o5, [%o4 + 0x08]
+	retl
+	 nop
+ENDPROC(aes_sparc64_cbc_decrypt_192)
+
+	.align		32
+ENTRY(aes_sparc64_cbc_decrypt_256)
+	/* %o0=&key[key_len], %o1=input, %o2=output, %o3=len, %o4=iv */
+	ldx		[%o0 - 0x10], %g1
+	ldx		[%o0 - 0x08], %g2
+	ldx		[%o4 + 0x00], %o0
+	ldx		[%o4 + 0x08], %o5
+1:	ldx		[%o1 + 0x00], %g3
+	ldx		[%o1 + 0x08], %g7
+	add		%o1, 0x10, %o1
+	xor		%g1, %g3, %g3
+	xor		%g2, %g7, %g7
+	MOVXTOD_G3_F4
+	MOVXTOD_G7_F6
+	DECRYPT_256(8, 4, 6, 0, 2)
+	MOVXTOD_O0_F0
+	MOVXTOD_O5_F2
+	xor		%g1, %g3, %o0
+	xor		%g2, %g7, %o5
+	fxor		%f4, %f0, %f4
+	fxor		%f6, %f2, %f6
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+	subcc		%o3, 0x10, %o3
+	bne,pt		%xcc, 1b
+	 add		%o2, 0x10, %o2
+	stx		%o0, [%o4 + 0x00]
+	stx		%o5, [%o4 + 0x08]
+	retl
+	 nop
+ENDPROC(aes_sparc64_cbc_decrypt_256)
+
+	.align		32
+ENTRY(aes_sparc64_ctr_crypt_128)
+	/* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
+	ldx		[%o4 + 0x00], %g3
+	ldx		[%o4 + 0x08], %g7
+	subcc		%o3, 0x10, %o3
+	ldx		[%o0 + 0x00], %g1
+	be		10f
+	 ldx		[%o0 + 0x08], %g2
+1:	xor		%g1, %g3, %o5
+	MOVXTOD_O5_F0
+	xor		%g2, %g7, %o5
+	MOVXTOD_O5_F2
+	add		%g7, 1, %g7
+	add		%g3, 1, %o5
+	movrz		%g7, %o5, %g3
+	xor		%g1, %g3, %o5
+	MOVXTOD_O5_F4
+	xor		%g2, %g7, %o5
+	MOVXTOD_O5_F6
+	add		%g7, 1, %g7
+	add		%g3, 1, %o5
+	movrz		%g7, %o5, %g3
+	ENCRYPT_128_2(8, 0, 2, 4, 6, 56, 58, 60, 62)
+	ldd		[%o1 + 0x00], %f56
+	ldd		[%o1 + 0x08], %f58
+	ldd		[%o1 + 0x10], %f60
+	ldd		[%o1 + 0x18], %f62
+	fxor		%f56, %f0, %f56
+	fxor		%f58, %f2, %f58
+	fxor		%f60, %f4, %f60
+	fxor		%f62, %f6, %f62
+	std		%f56, [%o2 + 0x00]
+	std		%f58, [%o2 + 0x08]
+	std		%f60, [%o2 + 0x10]
+	std		%f62, [%o2 + 0x18]
+	subcc		%o3, 0x20, %o3
+	add		%o1, 0x20, %o1
+	brgz		%o3, 1b
+	 add		%o2, 0x20, %o2
+	brlz,pt		%o3, 11f
+	 nop
+10:	xor		%g1, %g3, %o5
+	MOVXTOD_O5_F0
+	xor		%g2, %g7, %o5
+	MOVXTOD_O5_F2
+	add		%g7, 1, %g7
+	add		%g3, 1, %o5
+	movrz		%g7, %o5, %g3
+	ENCRYPT_128(8, 0, 2, 4, 6)
+	ldd		[%o1 + 0x00], %f4
+	ldd		[%o1 + 0x08], %f6
+	fxor		%f4, %f0, %f4
+	fxor		%f6, %f2, %f6
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+11:	stx		%g3, [%o4 + 0x00]
+	retl
+	 stx		%g7, [%o4 + 0x08]
+ENDPROC(aes_sparc64_ctr_crypt_128)
+
+	.align		32
+ENTRY(aes_sparc64_ctr_crypt_192)
+	/* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
+	ldx		[%o4 + 0x00], %g3
+	ldx		[%o4 + 0x08], %g7
+	subcc		%o3, 0x10, %o3
+	ldx		[%o0 + 0x00], %g1
+	be		10f
+	 ldx		[%o0 + 0x08], %g2
+1:	xor		%g1, %g3, %o5
+	MOVXTOD_O5_F0
+	xor		%g2, %g7, %o5
+	MOVXTOD_O5_F2
+	add		%g7, 1, %g7
+	add		%g3, 1, %o5
+	movrz		%g7, %o5, %g3
+	xor		%g1, %g3, %o5
+	MOVXTOD_O5_F4
+	xor		%g2, %g7, %o5
+	MOVXTOD_O5_F6
+	add		%g7, 1, %g7
+	add		%g3, 1, %o5
+	movrz		%g7, %o5, %g3
+	ENCRYPT_192_2(8, 0, 2, 4, 6, 56, 58, 60, 62)
+	ldd		[%o1 + 0x00], %f56
+	ldd		[%o1 + 0x08], %f58
+	ldd		[%o1 + 0x10], %f60
+	ldd		[%o1 + 0x18], %f62
+	fxor		%f56, %f0, %f56
+	fxor		%f58, %f2, %f58
+	fxor		%f60, %f4, %f60
+	fxor		%f62, %f6, %f62
+	std		%f56, [%o2 + 0x00]
+	std		%f58, [%o2 + 0x08]
+	std		%f60, [%o2 + 0x10]
+	std		%f62, [%o2 + 0x18]
+	subcc		%o3, 0x20, %o3
+	add		%o1, 0x20, %o1
+	brgz		%o3, 1b
+	 add		%o2, 0x20, %o2
+	brlz,pt		%o3, 11f
+	 nop
+10:	xor		%g1, %g3, %o5
+	MOVXTOD_O5_F0
+	xor		%g2, %g7, %o5
+	MOVXTOD_O5_F2
+	add		%g7, 1, %g7
+	add		%g3, 1, %o5
+	movrz		%g7, %o5, %g3
+	ENCRYPT_192(8, 0, 2, 4, 6)
+	ldd		[%o1 + 0x00], %f4
+	ldd		[%o1 + 0x08], %f6
+	fxor		%f4, %f0, %f4
+	fxor		%f6, %f2, %f6
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+11:	stx		%g3, [%o4 + 0x00]
+	retl
+	 stx		%g7, [%o4 + 0x08]
+ENDPROC(aes_sparc64_ctr_crypt_192)
+
+	.align		32
+ENTRY(aes_sparc64_ctr_crypt_256)
+	/* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
+	ldx		[%o4 + 0x00], %g3
+	ldx		[%o4 + 0x08], %g7
+	subcc		%o3, 0x10, %o3
+	ldx		[%o0 + 0x00], %g1
+	be		10f
+	 ldx		[%o0 + 0x08], %g2
+1:	xor		%g1, %g3, %o5
+	MOVXTOD_O5_F0
+	xor		%g2, %g7, %o5
+	MOVXTOD_O5_F2
+	add		%g7, 1, %g7
+	add		%g3, 1, %o5
+	movrz		%g7, %o5, %g3
+	xor		%g1, %g3, %o5
+	MOVXTOD_O5_F4
+	xor		%g2, %g7, %o5
+	MOVXTOD_O5_F6
+	add		%g7, 1, %g7
+	add		%g3, 1, %o5
+	movrz		%g7, %o5, %g3
+	ENCRYPT_256_2(8, 0, 2, 4, 6)
+	ldd		[%o1 + 0x00], %f56
+	ldd		[%o1 + 0x08], %f58
+	ldd		[%o1 + 0x10], %f60
+	ldd		[%o1 + 0x18], %f62
+	fxor		%f56, %f0, %f56
+	fxor		%f58, %f2, %f58
+	fxor		%f60, %f4, %f60
+	fxor		%f62, %f6, %f62
+	std		%f56, [%o2 + 0x00]
+	std		%f58, [%o2 + 0x08]
+	std		%f60, [%o2 + 0x10]
+	std		%f62, [%o2 + 0x18]
+	subcc		%o3, 0x20, %o3
+	add		%o1, 0x20, %o1
+	brgz		%o3, 1b
+	 add		%o2, 0x20, %o2
+	brlz,pt		%o3, 11f
+	 nop
+10:	ldd		[%o0 + 0xd0], %f56
+	ldd		[%o0 + 0xd8], %f58
+	ldd		[%o0 + 0xe0], %f60
+	ldd		[%o0 + 0xe8], %f62
+	xor		%g1, %g3, %o5
+	MOVXTOD_O5_F0
+	xor		%g2, %g7, %o5
+	MOVXTOD_O5_F2
+	add		%g7, 1, %g7
+	add		%g3, 1, %o5
+	movrz		%g7, %o5, %g3
+	ENCRYPT_256(8, 0, 2, 4, 6)
+	ldd		[%o1 + 0x00], %f4
+	ldd		[%o1 + 0x08], %f6
+	fxor		%f4, %f0, %f4
+	fxor		%f6, %f2, %f6
+	std		%f4, [%o2 + 0x00]
+	std		%f6, [%o2 + 0x08]
+11:	stx		%g3, [%o4 + 0x00]
+	retl
+	 stx		%g7, [%o4 + 0x08]
+ENDPROC(aes_sparc64_ctr_crypt_256)
diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig
index 578af717e13a..0de289b429a9 100644
--- a/lib/crypto/tests/Kconfig
+++ b/lib/crypto/tests/Kconfig
@@ -1,49 +1,76 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
+config CRYPTO_LIB_BLAKE2B_KUNIT_TEST
+	tristate "KUnit tests for BLAKE2b" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_BLAKE2B
+	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the BLAKE2b cryptographic hash function.
+
 config CRYPTO_LIB_BLAKE2S_KUNIT_TEST
 	tristate "KUnit tests for BLAKE2s" if !KUNIT_ALL_TESTS
 	depends on KUNIT
 	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
-	# No need to select CRYPTO_LIB_BLAKE2S here, as that option doesn't
+	# No need to depend on CRYPTO_LIB_BLAKE2S here, as that option doesn't
 	# exist; the BLAKE2s code is always built-in for the /dev/random driver.
 	help
 	  KUnit tests for the BLAKE2s cryptographic hash function.
 
 config CRYPTO_LIB_CURVE25519_KUNIT_TEST
 	tristate "KUnit tests for Curve25519" if !KUNIT_ALL_TESTS
-	depends on KUNIT
+	depends on KUNIT && CRYPTO_LIB_CURVE25519
 	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
-	select CRYPTO_LIB_CURVE25519
 	help
 	  KUnit tests for the Curve25519 Diffie-Hellman function.
 
 config CRYPTO_LIB_MD5_KUNIT_TEST
 	tristate "KUnit tests for MD5" if !KUNIT_ALL_TESTS
-	depends on KUNIT
+	depends on KUNIT && CRYPTO_LIB_MD5
 	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
-	select CRYPTO_LIB_MD5
 	help
 	  KUnit tests for the MD5 cryptographic hash function and its
 	  corresponding HMAC.
 
+config CRYPTO_LIB_MLDSA_KUNIT_TEST
+	tristate "KUnit tests for ML-DSA" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_MLDSA
+	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the ML-DSA digital signature algorithm.
+
+config CRYPTO_LIB_NH_KUNIT_TEST
+	tristate "KUnit tests for NH" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_NH
+	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	help
+	  KUnit tests for the NH almost-universal hash function.
+
 config CRYPTO_LIB_POLY1305_KUNIT_TEST
 	tristate "KUnit tests for Poly1305" if !KUNIT_ALL_TESTS
-	depends on KUNIT
+	depends on KUNIT && CRYPTO_LIB_POLY1305
 	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
-	select CRYPTO_LIB_POLY1305
 	help
 	  KUnit tests for the Poly1305 library functions.
 
+config CRYPTO_LIB_POLYVAL_KUNIT_TEST
+	tristate "KUnit tests for POLYVAL" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_POLYVAL
+	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the POLYVAL library functions.
+
 config CRYPTO_LIB_SHA1_KUNIT_TEST
 	tristate "KUnit tests for SHA-1" if !KUNIT_ALL_TESTS
-	depends on KUNIT
+	depends on KUNIT && CRYPTO_LIB_SHA1
 	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
-	select CRYPTO_LIB_SHA1
 	help
 	  KUnit tests for the SHA-1 cryptographic hash function and its
 	  corresponding HMAC.
@@ -52,10 +79,9 @@ config CRYPTO_LIB_SHA1_KUNIT_TEST
 # included, for consistency with the naming used elsewhere (e.g. CRYPTO_SHA256).
 config CRYPTO_LIB_SHA256_KUNIT_TEST
 	tristate "KUnit tests for SHA-224 and SHA-256" if !KUNIT_ALL_TESTS
-	depends on KUNIT
+	depends on KUNIT && CRYPTO_LIB_SHA256
 	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
-	select CRYPTO_LIB_SHA256
 	help
 	  KUnit tests for the SHA-224 and SHA-256 cryptographic hash functions
 	  and their corresponding HMACs.
@@ -64,14 +90,23 @@ config CRYPTO_LIB_SHA256_KUNIT_TEST
 # included, for consistency with the naming used elsewhere (e.g. CRYPTO_SHA512).
 config CRYPTO_LIB_SHA512_KUNIT_TEST
 	tristate "KUnit tests for SHA-384 and SHA-512" if !KUNIT_ALL_TESTS
-	depends on KUNIT
+	depends on KUNIT && CRYPTO_LIB_SHA512
 	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
-	select CRYPTO_LIB_SHA512
 	help
 	  KUnit tests for the SHA-384 and SHA-512 cryptographic hash functions
 	  and their corresponding HMACs.
 
+config CRYPTO_LIB_SHA3_KUNIT_TEST
+	tristate "KUnit tests for SHA-3" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_SHA3
+	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the SHA3 cryptographic hash and XOF functions,
+	  including SHA3-224, SHA3-256, SHA3-384, SHA3-512, SHAKE128 and
+	  SHAKE256.
+
 config CRYPTO_LIB_BENCHMARK_VISIBLE
 	bool
 
diff --git a/lib/crypto/tests/Makefile b/lib/crypto/tests/Makefile
index a71fad19922b..f4262379f56c 100644
--- a/lib/crypto/tests/Makefile
+++ b/lib/crypto/tests/Makefile
@@ -1,9 +1,14 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
+obj-$(CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST) += blake2b_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST) += blake2s_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST) += curve25519_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_MD5_KUNIT_TEST) += md5_kunit.o
+obj-$(CONFIG_CRYPTO_LIB_MLDSA_KUNIT_TEST) += mldsa_kunit.o
+obj-$(CONFIG_CRYPTO_LIB_NH_KUNIT_TEST) += nh_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_POLY1305_KUNIT_TEST) += poly1305_kunit.o
+obj-$(CONFIG_CRYPTO_LIB_POLYVAL_KUNIT_TEST) += polyval_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_SHA1_KUNIT_TEST) += sha1_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_SHA256_KUNIT_TEST) += sha224_kunit.o sha256_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_SHA512_KUNIT_TEST) += sha384_kunit.o sha512_kunit.o
+obj-$(CONFIG_CRYPTO_LIB_SHA3_KUNIT_TEST) += sha3_kunit.o
diff --git a/lib/crypto/tests/blake2b-testvecs.h b/lib/crypto/tests/blake2b-testvecs.h
new file mode 100644
index 000000000000..9e407dbc219c
--- /dev/null
+++ b/lib/crypto/tests/blake2b-testvecs.h
@@ -0,0 +1,342 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py blake2b */
+
+static const struct {
+	size_t data_len;
+	u8 digest[BLAKE2B_HASH_SIZE];
+} hash_testvecs[] = {
+	{
+		.data_len = 0,
+		.digest = {
+			0x78, 0x6a, 0x02, 0xf7, 0x42, 0x01, 0x59, 0x03,
+			0xc6, 0xc6, 0xfd, 0x85, 0x25, 0x52, 0xd2, 0x72,
+			0x91, 0x2f, 0x47, 0x40, 0xe1, 0x58, 0x47, 0x61,
+			0x8a, 0x86, 0xe2, 0x17, 0xf7, 0x1f, 0x54, 0x19,
+			0xd2, 0x5e, 0x10, 0x31, 0xaf, 0xee, 0x58, 0x53,
+			0x13, 0x89, 0x64, 0x44, 0x93, 0x4e, 0xb0, 0x4b,
+			0x90, 0x3a, 0x68, 0x5b, 0x14, 0x48, 0xb7, 0x55,
+			0xd5, 0x6f, 0x70, 0x1a, 0xfe, 0x9b, 0xe2, 0xce,
+		},
+	},
+	{
+		.data_len = 1,
+		.digest = {
+			0x6f, 0x2e, 0xcc, 0x83, 0x53, 0xa3, 0x20, 0x16,
+			0x5b, 0xda, 0xd0, 0x04, 0xd3, 0xcb, 0xe4, 0x37,
+			0x5b, 0xf0, 0x84, 0x36, 0xe1, 0xad, 0x45, 0xcc,
+			0x4d, 0x7f, 0x09, 0x68, 0xb2, 0x62, 0x93, 0x7f,
+			0x72, 0x32, 0xe8, 0xa7, 0x2f, 0x1f, 0x6f, 0xc6,
+			0x14, 0xd6, 0x70, 0xae, 0x0c, 0xf0, 0xf3, 0xce,
+			0x64, 0x4d, 0x22, 0xdf, 0xc7, 0xa7, 0xf8, 0xa8,
+			0x18, 0x23, 0xd8, 0x6c, 0xaf, 0x65, 0xa2, 0x54,
+		},
+	},
+	{
+		.data_len = 2,
+		.digest = {
+			0x04, 0x13, 0xe2, 0x10, 0xbe, 0x65, 0xde, 0xce,
+			0x61, 0xa8, 0xe0, 0xd6, 0x35, 0xb1, 0xb8, 0x88,
+			0xd2, 0xea, 0x45, 0x3a, 0xe1, 0x8d, 0x94, 0xb5,
+			0x66, 0x06, 0x98, 0x96, 0x39, 0xf8, 0x0e, 0xcb,
+			0x34, 0xa6, 0xa8, 0x17, 0xfe, 0x56, 0xbc, 0xa9,
+			0x5e, 0x1b, 0xb1, 0xde, 0x3c, 0xc7, 0x78, 0x4f,
+			0x39, 0xc6, 0xfc, 0xa8, 0xb3, 0x27, 0x66, 0x3e,
+			0x4e, 0xb5, 0x5d, 0x08, 0x89, 0xee, 0xd1, 0xe0,
+		},
+	},
+	{
+		.data_len = 3,
+		.digest = {
+			0x2b, 0x4a, 0xa3, 0x4e, 0x2b, 0x7a, 0x47, 0x20,
+			0x30, 0x5b, 0x09, 0x17, 0x3a, 0xf4, 0xcc, 0xf0,
+			0xf7, 0x7b, 0x97, 0x68, 0x98, 0x9f, 0x4f, 0x09,
+			0x46, 0x25, 0xe7, 0xd6, 0x53, 0x6b, 0xf9, 0x68,
+			0x48, 0x12, 0x44, 0x8c, 0x9a, 0xc8, 0xd4, 0x42,
+			0xeb, 0x2c, 0x5f, 0x41, 0xba, 0x17, 0xd0, 0xc3,
+			0xad, 0xfd, 0xfb, 0x42, 0x33, 0xcb, 0x08, 0x5d,
+			0xd2, 0x5c, 0x3d, 0xde, 0x87, 0x4d, 0xd6, 0xe4,
+		},
+	},
+	{
+		.data_len = 16,
+		.digest = {
+			0xbf, 0x40, 0xf2, 0x38, 0x44, 0x8e, 0x24, 0x5e,
+			0xbc, 0x67, 0xbb, 0xf0, 0x10, 0x9a, 0x79, 0xbb,
+			0x36, 0x55, 0xce, 0xd2, 0xba, 0x04, 0x0d, 0xe8,
+			0x30, 0x29, 0x5c, 0x2a, 0xa6, 0x3a, 0x4f, 0x37,
+			0xac, 0x5f, 0xd4, 0x13, 0xa2, 0xf4, 0xfe, 0x80,
+			0x61, 0xd7, 0x58, 0x66, 0x0c, 0x7f, 0xa2, 0x56,
+			0x6b, 0x52, 0x7c, 0x22, 0x73, 0x7f, 0x17, 0xaa,
+			0x91, 0x5a, 0x22, 0x06, 0xd9, 0x00, 0x48, 0x12,
+		},
+	},
+	{
+		.data_len = 32,
+		.digest = {
+			0x41, 0x04, 0x65, 0x93, 0x81, 0x9a, 0x20, 0x0a,
+			0x00, 0x60, 0x00, 0x64, 0x4c, 0x04, 0x3d, 0xe0,
+			0x6b, 0x17, 0x0c, 0xe1, 0x0e, 0x28, 0x8b, 0xa0,
+			0x76, 0xd2, 0x79, 0xb0, 0x33, 0x60, 0x61, 0x27,
+			0xf2, 0x64, 0xf1, 0x8a, 0xe5, 0x3e, 0xaa, 0x37,
+			0x60, 0xad, 0x2d, 0x75, 0x13, 0xae, 0xd8, 0x9e,
+			0xec, 0xe0, 0xe4, 0x40, 0x2f, 0x59, 0x44, 0xb0,
+			0x66, 0x7a, 0x68, 0x38, 0xce, 0x21, 0x99, 0x2a,
+		},
+	},
+	{
+		.data_len = 48,
+		.digest = {
+			0x19, 0x6f, 0x9d, 0xc7, 0x87, 0x12, 0x5c, 0xa3,
+			0xe2, 0xd3, 0xf1, 0x82, 0xec, 0xf3, 0x55, 0x9c,
+			0x86, 0xd1, 0x6d, 0xde, 0xcf, 0x5b, 0xec, 0x4c,
+			0x43, 0x25, 0x85, 0x90, 0xef, 0xe8, 0xe3, 0x5f,
+			0x2c, 0x3a, 0x84, 0x07, 0xb8, 0x55, 0xfd, 0x5e,
+			0xa4, 0x45, 0xf2, 0xac, 0xe4, 0xbd, 0xc7, 0x96,
+			0x80, 0x59, 0x3e, 0xc9, 0xb1, 0x60, 0xb1, 0x2b,
+			0x17, 0x49, 0x7d, 0x3e, 0x7d, 0x4d, 0x70, 0x24,
+		},
+	},
+	{
+		.data_len = 49,
+		.digest = {
+			0x73, 0x72, 0xd5, 0x0a, 0x97, 0xb4, 0x7d, 0xdb,
+			0x05, 0x14, 0x8e, 0x40, 0xc2, 0x9a, 0x8a, 0x74,
+			0x4b, 0xda, 0x7e, 0xfc, 0x97, 0x57, 0x23, 0x39,
+			0xdc, 0x57, 0x09, 0x13, 0x24, 0xfc, 0xf3, 0x23,
+			0x55, 0x48, 0xdd, 0xe5, 0x07, 0x9a, 0x6f, 0x7b,
+			0x62, 0xea, 0x4d, 0x79, 0xb4, 0xb9, 0xc5, 0x86,
+			0xc0, 0x34, 0xd6, 0xd2, 0x6c, 0xc3, 0x94, 0xfb,
+			0x34, 0xd6, 0x62, 0xae, 0xb8, 0x99, 0xf1, 0x38,
+		},
+	},
+	{
+		.data_len = 63,
+		.digest = {
+			0x42, 0x3a, 0xe3, 0xa2, 0xae, 0x5a, 0x28, 0xce,
+			0xf1, 0x3c, 0x97, 0xc2, 0x34, 0xf6, 0xb5, 0x1e,
+			0xfc, 0x31, 0xb4, 0x04, 0x61, 0xb7, 0x54, 0x0b,
+			0x0d, 0x1a, 0x22, 0x9c, 0x04, 0x67, 0x5c, 0x4c,
+			0x75, 0x1b, 0x10, 0x0b, 0x99, 0xe2, 0xb1, 0x5e,
+			0x5d, 0x4b, 0x7a, 0xe6, 0xf6, 0xb5, 0x62, 0xee,
+			0x2d, 0x44, 0x57, 0xb2, 0x96, 0x73, 0x5e, 0xb9,
+			0x6a, 0xb2, 0xb3, 0x16, 0xa3, 0xd9, 0x6a, 0x60,
+		},
+	},
+	{
+		.data_len = 64,
+		.digest = {
+			0x50, 0xb9, 0xbe, 0xb2, 0x69, 0x07, 0x45, 0x5b,
+			0x59, 0xde, 0x8d, 0xbf, 0x08, 0xdc, 0x2e, 0x7f,
+			0x93, 0x29, 0xc1, 0x91, 0xe8, 0x74, 0x03, 0x89,
+			0x20, 0xfb, 0xb2, 0x4b, 0xe8, 0x68, 0x6f, 0xe1,
+			0xb4, 0x30, 0xbe, 0x11, 0x3c, 0x43, 0x19, 0x66,
+			0x72, 0x78, 0xb7, 0xf4, 0xe9, 0x09, 0x18, 0x4e,
+			0xae, 0x4a, 0x24, 0xe0, 0x6f, 0x44, 0x02, 0xe3,
+			0xfd, 0xda, 0xb3, 0x3e, 0x3c, 0x6d, 0x54, 0x2e,
+		},
+	},
+	{
+		.data_len = 65,
+		.digest = {
+			0xd6, 0xf2, 0xa9, 0x61, 0x3f, 0xce, 0x2a, 0x68,
+			0x19, 0x86, 0xff, 0xd1, 0xee, 0x89, 0x3b, 0xa4,
+			0x10, 0x9a, 0x91, 0x50, 0x35, 0x48, 0x9e, 0xf5,
+			0x9c, 0x95, 0xe0, 0xfb, 0x92, 0x0f, 0xa8, 0xf7,
+			0x6c, 0x43, 0x85, 0xf1, 0x6e, 0x11, 0x4e, 0x67,
+			0x78, 0xd7, 0x53, 0x25, 0x0c, 0xf8, 0xce, 0x38,
+			0x74, 0x08, 0xb0, 0x3c, 0x53, 0x20, 0x4d, 0xc4,
+			0x9a, 0xf5, 0x78, 0xe8, 0x41, 0x8f, 0xed, 0x1f,
+		},
+	},
+	{
+		.data_len = 127,
+		.digest = {
+			0xe8, 0xb2, 0xc5, 0xa7, 0xf5, 0xfa, 0xee, 0xa0,
+			0x57, 0xba, 0x58, 0xf9, 0x0a, 0xf2, 0x64, 0x16,
+			0xa8, 0xa6, 0x03, 0x85, 0x3b, 0xb8, 0x6f, 0xca,
+			0x76, 0xc3, 0xa1, 0x2b, 0xec, 0xef, 0xc4, 0x66,
+			0x11, 0xdf, 0x03, 0x85, 0x9d, 0x0c, 0x37, 0x7b,
+			0xa9, 0x7b, 0x44, 0xfb, 0x11, 0x8f, 0x3f, 0x71,
+			0xcd, 0x81, 0x43, 0x2e, 0x71, 0x5c, 0x54, 0x9f,
+			0xca, 0x0f, 0x01, 0x91, 0xca, 0xaa, 0x93, 0xe9,
+		},
+	},
+	{
+		.data_len = 128,
+		.digest = {
+			0x05, 0x8e, 0x9d, 0xdc, 0xe9, 0x36, 0x3e, 0x73,
+			0x63, 0x59, 0x69, 0x81, 0x0b, 0x8c, 0xc7, 0x9e,
+			0xcc, 0xe7, 0x9c, 0x19, 0x54, 0xa7, 0x2f, 0x86,
+			0xb5, 0xea, 0xae, 0x6d, 0xfe, 0x4e, 0x6e, 0x83,
+			0x8d, 0x1a, 0x1c, 0x70, 0x3f, 0x34, 0xa1, 0x04,
+			0x59, 0xd1, 0xbb, 0xaa, 0x58, 0xf7, 0xce, 0xfb,
+			0x86, 0x66, 0x22, 0xfc, 0x78, 0x74, 0x6e, 0x85,
+			0xf1, 0x59, 0x7d, 0x9e, 0x1c, 0x3b, 0xc6, 0x65,
+		},
+	},
+	{
+		.data_len = 129,
+		.digest = {
+			0x6b, 0x1f, 0x7c, 0x9a, 0x65, 0x7f, 0x09, 0x61,
+			0xe5, 0x04, 0x9a, 0xf1, 0x4b, 0x36, 0x8e, 0x41,
+			0x86, 0xcf, 0x86, 0x19, 0xd8, 0xc9, 0x34, 0x70,
+			0x67, 0xd1, 0x03, 0x72, 0x12, 0xf7, 0x27, 0x92,
+			0x2e, 0x3d, 0x2b, 0x54, 0x9a, 0x48, 0xa4, 0xc2,
+			0x61, 0xea, 0x6a, 0xe8, 0xdd, 0x07, 0x41, 0x85,
+			0x58, 0x6d, 0xcd, 0x12, 0x0d, 0xbc, 0xb1, 0x23,
+			0xb2, 0xdb, 0x24, 0x1f, 0xc4, 0xa7, 0xae, 0xda,
+		},
+	},
+	{
+		.data_len = 256,
+		.digest = {
+			0x50, 0xd8, 0xdc, 0xb2, 0x50, 0x24, 0x7a, 0x49,
+			0xb1, 0x00, 0x73, 0x16, 0x1f, 0xce, 0xf9, 0xe8,
+			0x77, 0x0a, 0x27, 0x74, 0xc7, 0xeb, 0xf0, 0x62,
+			0xb9, 0xf3, 0x24, 0xa6, 0x03, 0x18, 0x40, 0xde,
+			0x9b, 0x1d, 0xa8, 0xd0, 0xbf, 0x66, 0xa3, 0xc1,
+			0x31, 0x04, 0x95, 0xc7, 0xc3, 0xb7, 0x11, 0xe2,
+			0x1e, 0x31, 0x49, 0x98, 0x06, 0xab, 0xf0, 0xe6,
+			0x5c, 0xac, 0x88, 0x28, 0x0b, 0x3d, 0xb2, 0xc2,
+		},
+	},
+	{
+		.data_len = 511,
+		.digest = {
+			0xd4, 0x2b, 0x6b, 0x9e, 0xfc, 0x44, 0xc0, 0x90,
+			0x64, 0x77, 0x5d, 0xf3, 0x44, 0xb6, 0x92, 0x8f,
+			0x80, 0xe2, 0xe4, 0x9b, 0xaf, 0x49, 0x04, 0xea,
+			0x29, 0xf7, 0x4a, 0x33, 0x3f, 0xc7, 0x3b, 0xab,
+			0xa1, 0x71, 0x7f, 0xa2, 0x8e, 0x03, 0xa0, 0xd6,
+			0xa7, 0xcd, 0xe0, 0xf8, 0xd7, 0x3b, 0xa4, 0x0d,
+			0x84, 0x79, 0x12, 0x72, 0x3f, 0x8e, 0x48, 0x35,
+			0x76, 0x4f, 0x56, 0xe9, 0x21, 0x40, 0x19, 0xbe,
+		},
+	},
+	{
+		.data_len = 513,
+		.digest = {
+			0x84, 0xd4, 0xd8, 0x6c, 0x60, 0x3d, 0x6e, 0xfd,
+			0x84, 0xb7, 0xdf, 0xba, 0x13, 0x5e, 0x07, 0x94,
+			0x5b, 0x6b, 0x62, 0x1d, 0x82, 0x02, 0xa7, 0xb3,
+			0x21, 0xdf, 0x42, 0x20, 0x85, 0xa8, 0x6f, 0x30,
+			0xf7, 0x03, 0xba, 0x66, 0x0e, 0xa6, 0x42, 0x21,
+			0x37, 0xe8, 0xed, 0x5b, 0x22, 0xf5, 0x4e, 0xa5,
+			0xe5, 0x80, 0x1b, 0x47, 0xf0, 0x49, 0xb3, 0xe5,
+			0x6e, 0xd9, 0xd9, 0x95, 0x3d, 0x2e, 0x42, 0x13,
+		},
+	},
+	{
+		.data_len = 1000,
+		.digest = {
+			0x71, 0x17, 0xab, 0x93, 0xfe, 0x3b, 0xa4, 0xe6,
+			0xcb, 0xb0, 0xea, 0x95, 0xe7, 0x1a, 0x01, 0xc0,
+			0x12, 0x33, 0xfe, 0xcc, 0x79, 0x15, 0xae, 0x56,
+			0xd2, 0x70, 0x44, 0x60, 0x54, 0x42, 0xa8, 0x69,
+			0x7e, 0xc3, 0x90, 0xa0, 0x0c, 0x63, 0x39, 0xff,
+			0x55, 0x53, 0xb8, 0x46, 0xef, 0x06, 0xcb, 0xba,
+			0x73, 0xf4, 0x76, 0x22, 0xf1, 0x60, 0x98, 0xbc,
+			0xbf, 0x76, 0x95, 0x85, 0x13, 0x1d, 0x11, 0x3b,
+		},
+	},
+	{
+		.data_len = 3333,
+		.digest = {
+			0x3a, 0xaa, 0x85, 0xa0, 0x8c, 0x8e, 0xe1, 0x9c,
+			0x9b, 0x43, 0x72, 0x7f, 0x40, 0x88, 0x3b, 0xd1,
+			0xc4, 0xd8, 0x2b, 0x69, 0xa6, 0x74, 0x47, 0x69,
+			0x5f, 0x7d, 0xab, 0x75, 0xa9, 0xf9, 0x88, 0x54,
+			0xce, 0x57, 0xcc, 0x9d, 0xac, 0x13, 0x91, 0xdb,
+			0x6d, 0x5c, 0xd8, 0xf4, 0x35, 0xc9, 0x30, 0xf0,
+			0x4b, 0x91, 0x25, 0xab, 0x92, 0xa8, 0xc8, 0x6f,
+			0xa0, 0xeb, 0x71, 0x56, 0x95, 0xab, 0xfd, 0xd7,
+		},
+	},
+	{
+		.data_len = 4096,
+		.digest = {
+			0xe1, 0xe9, 0xbe, 0x6c, 0x96, 0xe2, 0xe8, 0xa6,
+			0x53, 0xcd, 0x79, 0x77, 0x57, 0x51, 0x2f, 0xb2,
+			0x9f, 0xfc, 0x09, 0xaa, 0x2c, 0xbc, 0x6c, 0x5f,
+			0xb0, 0xf2, 0x12, 0x39, 0x54, 0xd7, 0x27, 0xf8,
+			0x33, 0x5d, 0xd4, 0x8a, 0xca, 0xd8, 0x2e, 0xbb,
+			0x02, 0x82, 0xca, 0x1b, 0x54, 0xfa, 0xd6, 0xf4,
+			0x49, 0x63, 0xfc, 0xc8, 0x73, 0xd4, 0x26, 0x8d,
+			0x4f, 0x1c, 0x56, 0xa7, 0xf4, 0x58, 0x6f, 0x51,
+		},
+	},
+	{
+		.data_len = 4128,
+		.digest = {
+			0xf2, 0xf6, 0xe1, 0x16, 0x98, 0x69, 0x74, 0x5f,
+			0x6c, 0xc4, 0x9d, 0x34, 0xa2, 0x84, 0x5d, 0x47,
+			0xac, 0x39, 0xe0, 0x14, 0x2d, 0x78, 0xfa, 0x27,
+			0xd5, 0x18, 0xaf, 0x26, 0x89, 0xa4, 0x69, 0xd3,
+			0x56, 0xde, 0xfe, 0x4b, 0x9f, 0x0c, 0x9d, 0x5a,
+			0x9a, 0x73, 0x3e, 0x3c, 0x76, 0x4b, 0x96, 0xca,
+			0x49, 0xda, 0x05, 0x8c, 0x53, 0xbb, 0x85, 0x89,
+			0x60, 0xc7, 0xe0, 0xb3, 0x51, 0x18, 0xd2, 0xd2,
+		},
+	},
+	{
+		.data_len = 4160,
+		.digest = {
+			0xfc, 0x5c, 0xcf, 0xbf, 0x29, 0xe3, 0x01, 0xef,
+			0x4b, 0x40, 0x70, 0x01, 0xca, 0x4d, 0x46, 0xce,
+			0xa9, 0x95, 0x5d, 0xb4, 0xf1, 0x79, 0x29, 0xdb,
+			0xac, 0x32, 0x3d, 0xd9, 0x60, 0x9e, 0x6b, 0xb8,
+			0x28, 0x62, 0xb7, 0x4a, 0xbb, 0x33, 0xb9, 0xd0,
+			0x83, 0xe0, 0xd7, 0x5a, 0x2d, 0x01, 0x4c, 0x61,
+			0x9e, 0x7d, 0x2d, 0x2d, 0x60, 0x29, 0x5e, 0x60,
+			0x10, 0xb7, 0x41, 0x00, 0x3f, 0xe5, 0xf7, 0x52,
+		},
+	},
+	{
+		.data_len = 4224,
+		.digest = {
+			0xf8, 0xe5, 0x4b, 0xe5, 0x89, 0xf9, 0x1b, 0x43,
+			0xbb, 0x65, 0x3d, 0xa0, 0xb4, 0xdc, 0x04, 0x26,
+			0x68, 0x15, 0xae, 0x4d, 0xd6, 0x03, 0xb7, 0x27,
+			0x06, 0x8c, 0x2a, 0x82, 0x51, 0x96, 0xbf, 0x83,
+			0x38, 0x96, 0x21, 0x8a, 0xd9, 0xf9, 0x4e, 0x38,
+			0xc6, 0xb3, 0xbd, 0xfe, 0xd3, 0x49, 0x90, 0xbc,
+			0xa1, 0x77, 0xd0, 0xa0, 0x3c, 0x2b, 0x4e, 0x10,
+			0x34, 0xc3, 0x17, 0x85, 0x3d, 0xec, 0xa8, 0x05,
+		},
+	},
+	{
+		.data_len = 16384,
+		.digest = {
+			0x38, 0x56, 0xaf, 0x83, 0x68, 0x9c, 0xba, 0xe3,
+			0xec, 0x51, 0xf5, 0xf4, 0x93, 0x48, 0x1d, 0xe6,
+			0xad, 0xa8, 0x8c, 0x70, 0x2a, 0xd9, 0xaa, 0x43,
+			0x04, 0x40, 0x95, 0xc1, 0xe6, 0x8a, 0xf5, 0x01,
+			0x6b, 0x79, 0xd9, 0xb4, 0xd0, 0x1d, 0x93, 0x26,
+			0xfe, 0xf5, 0x07, 0x57, 0xda, 0x08, 0x0a, 0x82,
+			0xc9, 0x17, 0x13, 0x5b, 0x9e, 0x11, 0x96, 0xa5,
+			0xd0, 0x92, 0xcd, 0xf1, 0xa3, 0x5b, 0x43, 0x21,
+		},
+	},
+};
+
+static const u8 hash_testvec_consolidated[BLAKE2B_HASH_SIZE] = {
+	0xa4, 0xf8, 0xf6, 0xa1, 0x36, 0x89, 0xc0, 0x2a,
+	0xc3, 0x42, 0x32, 0x71, 0xe5, 0xea, 0x14, 0x77,
+	0xf3, 0x99, 0x91, 0x87, 0x49, 0xc2, 0x8d, 0xa5,
+	0x2f, 0xed, 0x01, 0x35, 0x39, 0x64, 0x09, 0x25,
+	0xe3, 0xa8, 0x50, 0x97, 0x35, 0x8b, 0xf5, 0x19,
+	0x1e, 0xd5, 0x9f, 0x03, 0x0b, 0x65, 0x55, 0x0e,
+	0xa0, 0xb7, 0xda, 0x18, 0x7b, 0x7f, 0x88, 0x55,
+	0x1f, 0xdb, 0x82, 0x6b, 0x98, 0x90, 0x1c, 0xdd,
+};
+
+static const u8 blake2b_keyed_testvec_consolidated[BLAKE2B_HASH_SIZE] = {
+	0x2b, 0x89, 0x36, 0x3a, 0x36, 0xe4, 0x18, 0x38,
+	0xc4, 0x5b, 0x5c, 0xa5, 0x9a, 0xed, 0xf2, 0xee,
+	0x5a, 0xb6, 0x82, 0x6c, 0x63, 0xf2, 0x29, 0x57,
+	0xc7, 0xd5, 0x32, 0x27, 0xba, 0x88, 0xb1, 0xab,
+	0xf2, 0x2a, 0xc1, 0xea, 0xf3, 0x91, 0x89, 0x66,
+	0x47, 0x1e, 0x5b, 0xc6, 0x98, 0x12, 0xe9, 0x25,
+	0xbf, 0x72, 0xd2, 0x3f, 0x88, 0x97, 0x17, 0x51,
+	0xed, 0x96, 0xfb, 0xe9, 0xca, 0x52, 0x42, 0xc9,
+};
diff --git a/lib/crypto/tests/blake2b_kunit.c b/lib/crypto/tests/blake2b_kunit.c
new file mode 100644
index 000000000000..bc0be7da1e76
--- /dev/null
+++ b/lib/crypto/tests/blake2b_kunit.c
@@ -0,0 +1,133 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright 2025 Google LLC
+ */
+#include <crypto/blake2b.h>
+#include "blake2b-testvecs.h"
+
+/*
+ * The following are compatibility functions that present BLAKE2b as an unkeyed
+ * hash function that produces hashes of fixed length BLAKE2B_HASH_SIZE, so that
+ * hash-test-template.h can be reused to test it.
+ */
+
+static void blake2b_default(const u8 *data, size_t len,
+			    u8 out[BLAKE2B_HASH_SIZE])
+{
+	blake2b(NULL, 0, data, len, out, BLAKE2B_HASH_SIZE);
+}
+
+static void blake2b_init_default(struct blake2b_ctx *ctx)
+{
+	blake2b_init(ctx, BLAKE2B_HASH_SIZE);
+}
+
+/*
+ * Generate the HASH_KUNIT_CASES using hash-test-template.h.  These test BLAKE2b
+ * with a key length of 0 and a hash length of BLAKE2B_HASH_SIZE.
+ */
+#define HASH blake2b_default
+#define HASH_CTX blake2b_ctx
+#define HASH_SIZE BLAKE2B_HASH_SIZE
+#define HASH_INIT blake2b_init_default
+#define HASH_UPDATE blake2b_update
+#define HASH_FINAL blake2b_final
+#include "hash-test-template.h"
+
+/*
+ * BLAKE2b specific test case which tests all possible combinations of key
+ * length and hash length.
+ */
+static void test_blake2b_all_key_and_hash_lens(struct kunit *test)
+{
+	const size_t data_len = 100;
+	u8 *data = &test_buf[0];
+	u8 *key = data + data_len;
+	u8 *hash = key + BLAKE2B_KEY_SIZE;
+	struct blake2b_ctx main_ctx;
+	u8 main_hash[BLAKE2B_HASH_SIZE];
+
+	rand_bytes_seeded_from_len(data, data_len);
+	blake2b_init(&main_ctx, BLAKE2B_HASH_SIZE);
+	for (int key_len = 0; key_len <= BLAKE2B_KEY_SIZE; key_len++) {
+		rand_bytes_seeded_from_len(key, key_len);
+		for (int out_len = 1; out_len <= BLAKE2B_HASH_SIZE; out_len++) {
+			blake2b(key, key_len, data, data_len, hash, out_len);
+			blake2b_update(&main_ctx, hash, out_len);
+		}
+	}
+	blake2b_final(&main_ctx, main_hash);
+	KUNIT_ASSERT_MEMEQ(test, main_hash, blake2b_keyed_testvec_consolidated,
+			   BLAKE2B_HASH_SIZE);
+}
+
+/*
+ * BLAKE2b specific test case which tests using a guarded buffer for all allowed
+ * key lengths.  Also tests both blake2b() and blake2b_init_key().
+ */
+static void test_blake2b_with_guarded_key_buf(struct kunit *test)
+{
+	const size_t data_len = 100;
+
+	rand_bytes(test_buf, data_len);
+	for (int key_len = 0; key_len <= BLAKE2B_KEY_SIZE; key_len++) {
+		u8 key[BLAKE2B_KEY_SIZE];
+		u8 *guarded_key = &test_buf[TEST_BUF_LEN - key_len];
+		u8 hash1[BLAKE2B_HASH_SIZE];
+		u8 hash2[BLAKE2B_HASH_SIZE];
+		struct blake2b_ctx ctx;
+
+		rand_bytes(key, key_len);
+		memcpy(guarded_key, key, key_len);
+
+		blake2b(key, key_len, test_buf, data_len,
+			hash1, BLAKE2B_HASH_SIZE);
+		blake2b(guarded_key, key_len, test_buf, data_len,
+			hash2, BLAKE2B_HASH_SIZE);
+		KUNIT_ASSERT_MEMEQ(test, hash1, hash2, BLAKE2B_HASH_SIZE);
+
+		blake2b_init_key(&ctx, BLAKE2B_HASH_SIZE, guarded_key, key_len);
+		blake2b_update(&ctx, test_buf, data_len);
+		blake2b_final(&ctx, hash2);
+		KUNIT_ASSERT_MEMEQ(test, hash1, hash2, BLAKE2B_HASH_SIZE);
+	}
+}
+
+/*
+ * BLAKE2b specific test case which tests using a guarded output buffer for all
+ * allowed output lengths.
+ */
+static void test_blake2b_with_guarded_out_buf(struct kunit *test)
+{
+	const size_t data_len = 100;
+
+	rand_bytes(test_buf, data_len);
+	for (int out_len = 1; out_len <= BLAKE2B_HASH_SIZE; out_len++) {
+		u8 hash[BLAKE2B_HASH_SIZE];
+		u8 *guarded_hash = &test_buf[TEST_BUF_LEN - out_len];
+
+		blake2b(NULL, 0, test_buf, data_len, hash, out_len);
+		blake2b(NULL, 0, test_buf, data_len, guarded_hash, out_len);
+		KUNIT_ASSERT_MEMEQ(test, hash, guarded_hash, out_len);
+	}
+}
+
+static struct kunit_case blake2b_test_cases[] = {
+	HASH_KUNIT_CASES,
+	KUNIT_CASE(test_blake2b_all_key_and_hash_lens),
+	KUNIT_CASE(test_blake2b_with_guarded_key_buf),
+	KUNIT_CASE(test_blake2b_with_guarded_out_buf),
+	KUNIT_CASE(benchmark_hash),
+	{},
+};
+
+static struct kunit_suite blake2b_test_suite = {
+	.name = "blake2b",
+	.test_cases = blake2b_test_cases,
+	.suite_init = hash_suite_init,
+	.suite_exit = hash_suite_exit,
+};
+kunit_test_suite(blake2b_test_suite);
+
+MODULE_DESCRIPTION("KUnit tests and benchmark for BLAKE2b");
+MODULE_LICENSE("GPL");
diff --git a/lib/crypto/tests/blake2s_kunit.c b/lib/crypto/tests/blake2s_kunit.c
index 057c40132246..6832d9aa7b82 100644
--- a/lib/crypto/tests/blake2s_kunit.c
+++ b/lib/crypto/tests/blake2s_kunit.c
@@ -14,12 +14,12 @@
 static void blake2s_default(const u8 *data, size_t len,
 			    u8 out[BLAKE2S_HASH_SIZE])
 {
-	blake2s(out, data, NULL, BLAKE2S_HASH_SIZE, len, 0);
+	blake2s(NULL, 0, data, len, out, BLAKE2S_HASH_SIZE);
 }
 
-static void blake2s_init_default(struct blake2s_state *state)
+static void blake2s_init_default(struct blake2s_ctx *ctx)
 {
-	blake2s_init(state, BLAKE2S_HASH_SIZE);
+	blake2s_init(ctx, BLAKE2S_HASH_SIZE);
 }
 
 /*
@@ -27,7 +27,7 @@ static void blake2s_init_default(struct blake2s_state *state)
  * with a key length of 0 and a hash length of BLAKE2S_HASH_SIZE.
  */
 #define HASH blake2s_default
-#define HASH_CTX blake2s_state
+#define HASH_CTX blake2s_ctx
 #define HASH_SIZE BLAKE2S_HASH_SIZE
 #define HASH_INIT blake2s_init_default
 #define HASH_UPDATE blake2s_update
@@ -44,19 +44,19 @@ static void test_blake2s_all_key_and_hash_lens(struct kunit *test)
 	u8 *data = &test_buf[0];
 	u8 *key = data + data_len;
 	u8 *hash = key + BLAKE2S_KEY_SIZE;
-	struct blake2s_state main_state;
+	struct blake2s_ctx main_ctx;
 	u8 main_hash[BLAKE2S_HASH_SIZE];
 
 	rand_bytes_seeded_from_len(data, data_len);
-	blake2s_init(&main_state, BLAKE2S_HASH_SIZE);
+	blake2s_init(&main_ctx, BLAKE2S_HASH_SIZE);
 	for (int key_len = 0; key_len <= BLAKE2S_KEY_SIZE; key_len++) {
 		rand_bytes_seeded_from_len(key, key_len);
 		for (int out_len = 1; out_len <= BLAKE2S_HASH_SIZE; out_len++) {
-			blake2s(hash, data, key, out_len, data_len, key_len);
-			blake2s_update(&main_state, hash, out_len);
+			blake2s(key, key_len, data, data_len, hash, out_len);
+			blake2s_update(&main_ctx, hash, out_len);
 		}
 	}
-	blake2s_final(&main_state, main_hash);
+	blake2s_final(&main_ctx, main_hash);
 	KUNIT_ASSERT_MEMEQ(test, main_hash, blake2s_keyed_testvec_consolidated,
 			   BLAKE2S_HASH_SIZE);
 }
@@ -75,21 +75,20 @@ static void test_blake2s_with_guarded_key_buf(struct kunit *test)
 		u8 *guarded_key = &test_buf[TEST_BUF_LEN - key_len];
 		u8 hash1[BLAKE2S_HASH_SIZE];
 		u8 hash2[BLAKE2S_HASH_SIZE];
-		struct blake2s_state state;
+		struct blake2s_ctx ctx;
 
 		rand_bytes(key, key_len);
 		memcpy(guarded_key, key, key_len);
 
-		blake2s(hash1, test_buf, key,
-			BLAKE2S_HASH_SIZE, data_len, key_len);
-		blake2s(hash2, test_buf, guarded_key,
-			BLAKE2S_HASH_SIZE, data_len, key_len);
+		blake2s(key, key_len, test_buf, data_len,
+			hash1, BLAKE2S_HASH_SIZE);
+		blake2s(guarded_key, key_len, test_buf, data_len,
+			hash2, BLAKE2S_HASH_SIZE);
 		KUNIT_ASSERT_MEMEQ(test, hash1, hash2, BLAKE2S_HASH_SIZE);
 
-		blake2s_init_key(&state, BLAKE2S_HASH_SIZE,
-				 guarded_key, key_len);
-		blake2s_update(&state, test_buf, data_len);
-		blake2s_final(&state, hash2);
+		blake2s_init_key(&ctx, BLAKE2S_HASH_SIZE, guarded_key, key_len);
+		blake2s_update(&ctx, test_buf, data_len);
+		blake2s_final(&ctx, hash2);
 		KUNIT_ASSERT_MEMEQ(test, hash1, hash2, BLAKE2S_HASH_SIZE);
 	}
 }
@@ -107,8 +106,8 @@ static void test_blake2s_with_guarded_out_buf(struct kunit *test)
 		u8 hash[BLAKE2S_HASH_SIZE];
 		u8 *guarded_hash = &test_buf[TEST_BUF_LEN - out_len];
 
-		blake2s(hash, test_buf, NULL, out_len, data_len, 0);
-		blake2s(guarded_hash, test_buf, NULL, out_len, data_len, 0);
+		blake2s(NULL, 0, test_buf, data_len, hash, out_len);
+		blake2s(NULL, 0, test_buf, data_len, guarded_hash, out_len);
 		KUNIT_ASSERT_MEMEQ(test, hash, guarded_hash, out_len);
 	}
 }
diff --git a/lib/crypto/tests/mldsa-testvecs.h b/lib/crypto/tests/mldsa-testvecs.h
new file mode 100644
index 000000000000..d2d6a042a829
--- /dev/null
+++ b/lib/crypto/tests/mldsa-testvecs.h
@@ -0,0 +1,1887 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/* ML-DSA test vectors extracted from leancrypto */
+
+struct mldsa_testvector {
+	enum mldsa_alg alg;
+	int sig_len;
+	int msg_len;
+	int pk_len;
+	const u8 *sig;
+	const u8 *msg;
+	const u8 *pk;
+};
+
+static const struct mldsa_testvector mldsa44_testvector = {
+	.alg = MLDSA44,
+	.pk_len = MLDSA44_PUBLIC_KEY_SIZE,
+	.pk =
+		(const u8[MLDSA44_PUBLIC_KEY_SIZE]) {
+			0x8f, 0x61, 0x67, 0xa9, 0x7c, 0x61, 0xc2, 0xf2, 0x87,
+			0xe2, 0x28, 0xf8, 0x44, 0x80, 0x6f, 0xb0, 0x10, 0xc1,
+			0x14, 0xf6, 0x88, 0x42, 0x76, 0xbe, 0x05, 0xd2, 0x56,
+			0xa0, 0xb7, 0x46, 0xcf, 0xc5, 0x76, 0x0a, 0x52, 0xfe,
+			0xa3, 0x3c, 0x05, 0x6e, 0xd5, 0xd3, 0xbd, 0x80, 0x03,
+			0x29, 0x25, 0x96, 0xdf, 0xa9, 0x5b, 0x12, 0x42, 0x89,
+			0x6e, 0x03, 0x2c, 0x42, 0x64, 0xee, 0xc7, 0xf7, 0x55,
+			0xfe, 0xfd, 0x15, 0x74, 0xee, 0x2d, 0xb3, 0xbf, 0xf2,
+			0x24, 0xe7, 0x35, 0x45, 0x77, 0x67, 0x44, 0xd2, 0x0c,
+			0x43, 0xfc, 0x7b, 0x47, 0x56, 0xad, 0xc5, 0xe7, 0x37,
+			0x49, 0x21, 0x12, 0x57, 0x7f, 0xca, 0x7f, 0x5d, 0xac,
+			0x62, 0x5b, 0x8d, 0xbf, 0xdb, 0x64, 0xac, 0x12, 0x1d,
+			0x7f, 0x7a, 0x2f, 0xa0, 0x2c, 0xfe, 0x95, 0x4b, 0x78,
+			0xdb, 0xf0, 0x98, 0x97, 0x70, 0x62, 0xfc, 0x64, 0x4a,
+			0x1b, 0xbc, 0x30, 0x1b, 0x51, 0x13, 0x47, 0x6e, 0x83,
+			0xd2, 0xa2, 0xcf, 0x39, 0xba, 0xd3, 0x00, 0x62, 0x7c,
+			0x5b, 0xe6, 0x14, 0x55, 0xd5, 0xfc, 0xbf, 0x15, 0x65,
+			0xd7, 0x1f, 0xbb, 0xf6, 0x2b, 0x46, 0x38, 0x1e, 0xf3,
+			0x8f, 0x0d, 0x57, 0x8a, 0x41, 0xfb, 0x47, 0x19, 0xe1,
+			0x79, 0xca, 0x98, 0x1a, 0x73, 0x4d, 0x8a, 0xc0, 0xa1,
+			0xa7, 0x4a, 0x28, 0x4a, 0x92, 0x6c, 0x77, 0x4f, 0x18,
+			0xe1, 0xce, 0x11, 0x14, 0xd5, 0xf6, 0xc0, 0xa8, 0x1e,
+			0x26, 0x25, 0xe4, 0x30, 0xb2, 0x6f, 0x89, 0x6e, 0xc6,
+			0x44, 0x1a, 0xd3, 0xca, 0xe1, 0x05, 0x0d, 0x61, 0x80,
+			0xd7, 0xbc, 0x56, 0x0a, 0x57, 0x9f, 0x16, 0x40, 0x84,
+			0x1e, 0xc0, 0x5e, 0xcd, 0xb6, 0xc8, 0x5d, 0x87, 0xd6,
+			0xee, 0xcb, 0x21, 0x2f, 0x22, 0x9a, 0x7e, 0xea, 0x2e,
+			0xf5, 0x87, 0xf0, 0x44, 0x10, 0x3d, 0x42, 0x6d, 0x3f,
+			0x4f, 0xa0, 0x21, 0x8e, 0x40, 0x75, 0x3b, 0xaa, 0xd4,
+			0xd9, 0x37, 0x0a, 0x35, 0x5b, 0xba, 0xbd, 0x11, 0x17,
+			0x7b, 0x7e, 0xfc, 0xb1, 0x1c, 0x5c, 0x71, 0xce, 0xde,
+			0xa5, 0xec, 0xd6, 0x8b, 0x50, 0x64, 0x02, 0xca, 0x84,
+			0x26, 0xfe, 0x03, 0x9e, 0xaf, 0x13, 0x05, 0x1c, 0x85,
+			0x32, 0x92, 0x85, 0x84, 0x83, 0xb6, 0x76, 0x27, 0xa7,
+			0xd8, 0x0c, 0xa4, 0xb5, 0x65, 0x7a, 0xbf, 0x2e, 0x64,
+			0xce, 0x82, 0x0d, 0x27, 0xd3, 0x2c, 0x9d, 0xb5, 0xa5,
+			0x51, 0xe3, 0xab, 0xbd, 0xe2, 0xe0, 0x64, 0x71, 0x0f,
+			0x99, 0x09, 0x6a, 0x40, 0x9d, 0x8c, 0x6d, 0x63, 0x70,
+			0x42, 0xe2, 0xa8, 0x5c, 0x71, 0xd2, 0x88, 0x10, 0xbc,
+			0x3b, 0x9d, 0xc8, 0x3a, 0x8a, 0xf3, 0x81, 0x0a, 0x0e,
+			0xd8, 0xf6, 0x13, 0x90, 0xc2, 0xda, 0x4c, 0x4f, 0x6f,
+			0x1f, 0x61, 0x0d, 0x46, 0xac, 0x5c, 0x4a, 0xcc, 0x23,
+			0xfb, 0xf2, 0xf8, 0x9f, 0x48, 0x1f, 0xaf, 0x24, 0xb4,
+			0xc7, 0xcb, 0x6f, 0x80, 0x74, 0xa3, 0xdb, 0xa8, 0x61,
+			0x12, 0x60, 0x08, 0x56, 0x85, 0x47, 0x05, 0x32, 0x93,
+			0x43, 0x9b, 0xa6, 0xf6, 0x56, 0x3b, 0xab, 0x93, 0x28,
+			0x19, 0xda, 0xad, 0xb5, 0xaa, 0x2e, 0x83, 0x3d, 0x37,
+			0x0e, 0x83, 0xf2, 0xfe, 0xdd, 0xe8, 0xd9, 0x53, 0x36,
+			0x3b, 0x5d, 0x8e, 0x24, 0xa9, 0x3a, 0x8f, 0x85, 0x4b,
+			0x50, 0xf7, 0x61, 0x0f, 0x20, 0x92, 0x09, 0x1f, 0xe4,
+			0x24, 0x98, 0x9f, 0xa4, 0x52, 0x12, 0x2a, 0xae, 0x6f,
+			0xd6, 0x0d, 0xb5, 0x22, 0x72, 0x83, 0xfb, 0x4f, 0xef,
+			0xa6, 0x55, 0x9a, 0x14, 0xab, 0x82, 0x89, 0x1f, 0xf2,
+			0x0b, 0x14, 0x76, 0xb7, 0xf7, 0x14, 0xdd, 0xd6, 0xc1,
+			0xe8, 0xb2, 0x99, 0x23, 0x28, 0xe8, 0xa4, 0x69, 0x18,
+			0xf6, 0x3e, 0xb1, 0xff, 0xde, 0xf2, 0x2c, 0x7c, 0x73,
+			0x93, 0x32, 0x52, 0x06, 0xeb, 0x59, 0xb2, 0x8f, 0x2f,
+			0x1d, 0x6a, 0x85, 0x74, 0xd3, 0xe5, 0xa1, 0x95, 0xe4,
+			0x96, 0x1e, 0x75, 0x16, 0xe4, 0x5c, 0x40, 0xf0, 0x20,
+			0xb1, 0x10, 0xe7, 0x2a, 0x70, 0x41, 0xac, 0x49, 0x40,
+			0x55, 0xef, 0xd1, 0x58, 0x24, 0x6a, 0xa6, 0x20, 0xdc,
+			0x23, 0xdc, 0x66, 0x45, 0x4f, 0x6a, 0x52, 0x3c, 0x61,
+			0xc8, 0xfb, 0x28, 0x1e, 0x8c, 0x3f, 0xfc, 0xc7, 0x73,
+			0x2f, 0xf1, 0xe2, 0x31, 0xee, 0xa0, 0x5f, 0x12, 0x3f,
+			0x94, 0xe5, 0x26, 0xc0, 0x62, 0xcc, 0x67, 0x8f, 0x5d,
+			0xfd, 0x3d, 0x8f, 0x16, 0xae, 0x4e, 0x04, 0x54, 0x5b,
+			0x02, 0x66, 0x00, 0x70, 0xe5, 0xcc, 0xb9, 0x51, 0x8c,
+			0x1a, 0x5d, 0xf0, 0xfe, 0x7d, 0x1b, 0x2f, 0x0a, 0x48,
+			0x94, 0xda, 0x4b, 0xb3, 0x05, 0x75, 0x3b, 0x25, 0xda,
+			0xf2, 0x12, 0x47, 0xe2, 0xd7, 0xbb, 0xe7, 0x05, 0xa2,
+			0x4e, 0xaf, 0x2f, 0x29, 0x6c, 0xbd, 0x80, 0x82, 0xdf,
+			0xbb, 0x4b, 0x9d, 0x29, 0x9b, 0xef, 0xff, 0x17, 0x2a,
+			0xa7, 0x2f, 0x4a, 0x2e, 0xbf, 0x29, 0x38, 0xcb, 0x94,
+			0x9c, 0x10, 0x87, 0x69, 0x57, 0x7d, 0xd7, 0xcf, 0xc1,
+			0x57, 0x0f, 0xc2, 0x9c, 0x3b, 0x1e, 0xbc, 0x55, 0x08,
+			0xb1, 0x50, 0xb0, 0x28, 0x79, 0x1d, 0xd9, 0x2f, 0xa5,
+			0x12, 0xcc, 0xa7, 0x5b, 0xcb, 0x67, 0x57, 0x13, 0x4a,
+			0xb3, 0xd5, 0x34, 0xea, 0xd9, 0x6b, 0x15, 0x17, 0x1f,
+			0xe7, 0x52, 0x2a, 0x11, 0xdf, 0xab, 0x3a, 0x8d, 0x1b,
+			0xaa, 0xfa, 0x54, 0xf0, 0x20, 0x94, 0x40, 0x2d, 0x3e,
+			0xf2, 0xa2, 0x8b, 0xbb, 0x86, 0xce, 0x29, 0x87, 0xb1,
+			0xfa, 0x27, 0xf2, 0x5a, 0x79, 0xc2, 0xf8, 0xe9, 0xf2,
+			0x7c, 0x5a, 0xcd, 0x45, 0x8e, 0x8b, 0xba, 0xad, 0x13,
+			0x52, 0x79, 0xd5, 0x0f, 0x59, 0xfb, 0x07, 0x15, 0x1f,
+			0xb3, 0xe6, 0xc8, 0x66, 0xf7, 0x37, 0x19, 0xd8, 0x33,
+			0xdd, 0xc1, 0xb3, 0x96, 0x4b, 0x28, 0x43, 0xf8, 0x5e,
+			0xc7, 0xe8, 0x81, 0x6e, 0xd0, 0xb2, 0x5a, 0x3d, 0xf1,
+			0x68, 0x8f, 0xf0, 0xf5, 0x1a, 0x6d, 0xc3, 0xaa, 0x5b,
+			0x72, 0x27, 0xbd, 0xd6, 0x7c, 0x9b, 0xbf, 0x89, 0x6a,
+			0x09, 0xf0, 0x48, 0xf0, 0x8c, 0x27, 0x69, 0x28, 0xf3,
+			0x5f, 0x53, 0xe3, 0x4d, 0x60, 0x4a, 0xb8, 0xc5, 0xf9,
+			0x85, 0x07, 0x3e, 0xfb, 0xd7, 0x21, 0x69, 0xd5, 0xec,
+			0x18, 0x68, 0xb6, 0x55, 0x15, 0xac, 0x2e, 0x0f, 0x5c,
+			0x2e, 0x9e, 0x12, 0x10, 0x5e, 0xc6, 0xb3, 0xcd, 0xe6,
+			0x3a, 0x2f, 0x43, 0xf6, 0x75, 0x31, 0x3c, 0x90, 0x34,
+			0x1c, 0x3d, 0x45, 0xaa, 0x8e, 0x08, 0xcf, 0x58, 0x67,
+			0x34, 0xd4, 0x24, 0xb8, 0x20, 0x69, 0xfe, 0xf0, 0x33,
+			0xe9, 0x3e, 0xbd, 0xe5, 0x18, 0x9f, 0x66, 0xc9, 0x0d,
+			0x6f, 0x47, 0x99, 0xf7, 0x0c, 0xdd, 0xad, 0x8c, 0x6b,
+			0x80, 0xed, 0x19, 0x61, 0x8a, 0xba, 0x62, 0x2f, 0xcc,
+			0x0e, 0x47, 0xe0, 0xc6, 0x1f, 0x2f, 0x44, 0x40, 0x17,
+			0xb6, 0x89, 0xc6, 0xf5, 0xc2, 0x97, 0x89, 0x38, 0x6c,
+			0x8e, 0x2c, 0x46, 0x0b, 0x6e, 0x01, 0x47, 0xd3, 0x2f,
+			0x58, 0xf8, 0xc7, 0x3b, 0x49, 0xb2, 0x35, 0x9f, 0x67,
+			0xc9, 0x6c, 0xde, 0x30, 0x3b, 0x50, 0x19, 0x65, 0xbb,
+			0x73, 0xa1, 0x47, 0x01, 0x6a, 0x3d, 0xe3, 0x85, 0x4d,
+			0xd0, 0x72, 0x0d, 0xbc, 0x81, 0x52, 0xe8, 0x1a, 0x8b,
+			0x85, 0x4c, 0x6f, 0x0e, 0xfc, 0x59, 0x84, 0x7a, 0xf2,
+			0x28, 0x3e, 0x02, 0xcb, 0xe2, 0x4a, 0xbf, 0xac, 0x22,
+			0x34, 0x86, 0xe4, 0x7d, 0x6b, 0xa3, 0x52, 0xac, 0xff,
+			0xe5, 0xbe, 0x0c, 0x8d, 0xf5, 0xd8, 0xfd, 0x5a, 0x3d,
+			0xad, 0x0d, 0xc0, 0x02, 0xd0, 0x58, 0x8e, 0x7c, 0x50,
+			0x7c, 0x09, 0xc0, 0xdb, 0xd7, 0xdf, 0xe0, 0xb2, 0x6f,
+			0xb2, 0x79, 0x0d, 0xc1, 0xa0, 0xb1, 0x7e, 0xe3, 0x0d,
+			0xfc, 0x93, 0x43, 0x8c, 0x86, 0x68, 0xa6, 0x51, 0x93,
+			0x42, 0xb7, 0xcd, 0x13, 0x56, 0x6e, 0xdc, 0x63, 0x51,
+			0x23, 0xcf, 0x29, 0xc5, 0x5a, 0x66, 0x43, 0x80, 0xfe,
+			0x28, 0x15, 0x71, 0x52, 0x87, 0xe2, 0x18, 0x10, 0xad,
+			0x94, 0x47, 0x2f, 0xe3, 0x5d, 0x7a, 0x02, 0x6d, 0x31,
+			0x58, 0xae, 0x2f, 0x96, 0x53, 0x1b, 0x6c, 0x0e, 0x25,
+			0xea, 0x41, 0xd3, 0x29, 0x27, 0x22, 0x24, 0x8d, 0x1d,
+			0x8e, 0xd1, 0x33, 0x76, 0x67, 0x7d, 0x9d, 0xd9, 0xa7,
+			0x3e, 0x61, 0xd0, 0xad, 0x93, 0xb9, 0xdf, 0x87, 0x3a,
+			0x7b, 0x89, 0xc7, 0x1d, 0x91, 0xec, 0x43, 0xa4, 0xdc,
+			0x02, 0x88, 0x2e, 0xaa, 0xb2, 0x58, 0xa5, 0xd3, 0x68,
+			0x9c, 0x9f, 0x60, 0x12, 0xc8, 0x7e, 0x7d, 0x40, 0x80,
+			0xfd, 0xb4, 0xbf, 0x56, 0xf4, 0x01, 0x39, 0x3d, 0xa0,
+			0x34, 0x48, 0x79, 0x75, 0xe2, 0x0f, 0x60, 0x09, 0x42,
+			0x11, 0x6f, 0xa5, 0x31, 0x46, 0xb7, 0x03, 0xc8, 0x61,
+			0x53, 0x39, 0x1e, 0xf4, 0x99, 0x36, 0x7d, 0xc0, 0x87,
+			0xda, 0x63, 0x71, 0x9b, 0x29, 0x7b, 0x4e, 0x6f, 0x09,
+			0xa2, 0x2c, 0xa5, 0xc2, 0xb9, 0xe7, 0xe0, 0x56, 0x8b,
+			0x1d, 0xbb, 0xcc, 0x34, 0x8c, 0xbc, 0xb6, 0x0a, 0xc9,
+			0xfa, 0x4a, 0x31, 0x63, 0x0d, 0x30, 0xff, 0x59, 0x3d,
+			0x8c, 0x4d, 0x74, 0x28, 0xf4, 0xe9, 0x97, 0x43, 0x05,
+			0x3a, 0x33, 0x51, 0x51, 0xe4, 0x0e, 0x33, 0xae, 0x2c,
+			0xda, 0x28, 0x83, 0x93, 0x4e, 0xfe, 0x37, 0x1d, 0x6c,
+			0x25, 0x1e, 0x24, 0xbc, 0x3a, 0x5c, 0x68, 0xac, 0x54,
+			0x3a, 0x47, 0x74, 0x35, 0xff, 0x37, 0x80, 0x12, 0x30,
+			0xd7, 0x31, 0x2a, 0x49, 0x51, 0x2d, 0x4f, 0xd2, 0x9c,
+			0xca, 0x55, 0x87, 0xd0, 0x41, 0x86, 0xc7, 0xf2, 0xda,
+			0xf8, 0x4b, 0x08, 0x23, 0xb3, 0x00, 0xb7, 0xb6, 0x4f,
+			0x2e, 0xaf, 0xb8, 0x8e, 0xb1, 0x44, 0xe1, 0xed, 0x67,
+			0xf8, 0x80, 0xa7, 0x04, 0xa0, 0x66, 0xe6, 0xb5, 0x69,
+			0xca, 0x95, 0x71, 0xc8, 0x0d, 0x3d, 0xf6, 0x77, 0xfd,
+			0x2c, 0x95, 0xed, 0xe5, 0x22, 0x43, 0xd9,
+		},
+	.msg_len = 64,
+	.msg =
+		(const u8[64]) {
+			0x6d, 0xb3, 0x8e, 0x80, 0xaf, 0x5f, 0x19, 0xd9,
+			0xb0, 0xcf, 0xad, 0x58, 0xc7, 0x27, 0xae, 0x12,
+			0x4e, 0x7d, 0xa3, 0x1a, 0xe3, 0x85, 0xc6, 0xaa,
+			0xf6, 0xa1, 0x9a, 0xb1, 0xe9, 0xe0, 0xfe, 0x89,
+			0x1e, 0xc5, 0x6f, 0x10, 0x18, 0x24, 0xab, 0xa8,
+			0x6d, 0x03, 0xd0, 0x3d, 0xc3, 0xff, 0x67, 0xe7,
+			0x3a, 0x95, 0x94, 0xc8, 0x49, 0x51, 0x8f, 0xa0,
+			0x65, 0xcb, 0x20, 0x29, 0x2a, 0x6d, 0xf7, 0xf2,
+		},
+	.sig_len = MLDSA44_SIGNATURE_SIZE,
+	.sig =
+		(const u8[MLDSA44_SIGNATURE_SIZE]) {
+			0x5e, 0x05, 0x37, 0xe2, 0xc1, 0x20, 0xce, 0x7b, 0x8a,
+			0xdf, 0xf2, 0x22, 0x61, 0x17, 0x58, 0xaa, 0x3c, 0xe4,
+			0x82, 0x9c, 0x0e, 0xb6, 0x1f, 0xb4, 0x98, 0x0f, 0xba,
+			0x8e, 0x51, 0x15, 0x67, 0x76, 0x0b, 0x98, 0x63, 0xda,
+			0x17, 0xd3, 0xbb, 0xbe, 0x16, 0x29, 0x71, 0xab, 0xba,
+			0x99, 0xed, 0x3f, 0xd4, 0xc2, 0x16, 0x71, 0xb6, 0x21,
+			0x87, 0x48, 0xaa, 0xb5, 0x39, 0x5e, 0xfb, 0x5d, 0x68,
+			0x3b, 0xd3, 0x60, 0xf4, 0x5b, 0x85, 0x2a, 0x5b, 0xb5,
+			0xce, 0x6e, 0xf3, 0x39, 0xc3, 0xbe, 0x96, 0xa7, 0x61,
+			0xc9, 0xbf, 0xdf, 0x33, 0x1d, 0xec, 0xb9, 0x2b, 0x7a,
+			0x05, 0xce, 0x1e, 0xd9, 0x46, 0x70, 0xca, 0x54, 0xbf,
+			0xdc, 0x46, 0x9e, 0x2f, 0x29, 0x18, 0x57, 0x96, 0x84,
+			0xac, 0xe9, 0xd7, 0x74, 0xeb, 0x8e, 0x6b, 0xec, 0x46,
+			0x9a, 0x2a, 0xfa, 0xde, 0x80, 0x09, 0x53, 0xd9, 0xeb,
+			0x9d, 0xf7, 0xaa, 0xe2, 0xe5, 0xdc, 0xc3, 0xd9, 0x70,
+			0xe5, 0x8b, 0xa8, 0xba, 0x2b, 0x41, 0x72, 0x92, 0x25,
+			0xaf, 0xd9, 0xb4, 0x5a, 0x53, 0xb7, 0xcc, 0x1d, 0x69,
+			0xf1, 0x53, 0x5b, 0x52, 0x38, 0xbc, 0x47, 0x24, 0x8c,
+			0x1d, 0x28, 0x5d, 0x5c, 0x1c, 0xc9, 0x9d, 0xea, 0x1c,
+			0xb1, 0xb3, 0x49, 0x68, 0xd5, 0xad, 0xdc, 0x47, 0x58,
+			0x6d, 0x38, 0x33, 0xe7, 0x9b, 0xaa, 0x89, 0xb1, 0x96,
+			0x0b, 0xcb, 0xc4, 0x24, 0x73, 0xf2, 0xe7, 0xb6, 0xca,
+			0x74, 0x55, 0x1b, 0xb5, 0xb7, 0x9e, 0x2e, 0xe3, 0x3a,
+			0x32, 0x5d, 0x1d, 0x6e, 0x15, 0xe6, 0xb8, 0xfb, 0xce,
+			0x57, 0x81, 0x15, 0xb5, 0xcf, 0x67, 0x2b, 0x55, 0x4c,
+			0x85, 0x6f, 0x28, 0xa6, 0xbb, 0xb4, 0x28, 0x76, 0x91,
+			0xa4, 0x29, 0xa1, 0x50, 0x7c, 0xed, 0x9a, 0xfc, 0xe4,
+			0xbc, 0xd7, 0x28, 0x62, 0x28, 0x61, 0x4d, 0x8d, 0x8c,
+			0x5a, 0x5e, 0x4d, 0x1d, 0x5e, 0x73, 0xcc, 0x0b, 0x9d,
+			0x56, 0x73, 0xc7, 0xf2, 0x26, 0xf7, 0x7e, 0x61, 0xa4,
+			0x86, 0xf5, 0x1c, 0xd1, 0x00, 0xd0, 0x31, 0xc5, 0x03,
+			0x17, 0x1c, 0xec, 0x04, 0xe5, 0xc7, 0x13, 0xb6, 0x81,
+			0x78, 0x3d, 0x27, 0x87, 0x36, 0xf3, 0x2a, 0x59, 0x96,
+			0xeb, 0x44, 0xfd, 0xb9, 0x95, 0xb7, 0x76, 0xb1, 0x08,
+			0xc4, 0x98, 0xb1, 0x08, 0x36, 0x2a, 0x63, 0x72, 0x4f,
+			0xef, 0x47, 0xfc, 0x84, 0x09, 0x18, 0x60, 0xb7, 0x8a,
+			0xff, 0xae, 0x32, 0x3c, 0x79, 0xdf, 0xd6, 0x24, 0xbe,
+			0x9c, 0x38, 0x68, 0x92, 0xde, 0x81, 0x80, 0x22, 0x06,
+			0xf2, 0xe4, 0xde, 0x75, 0x4e, 0xd6, 0x36, 0x93, 0x44,
+			0xd1, 0xa4, 0x2e, 0x2e, 0x05, 0x87, 0xbd, 0xf7, 0xc5,
+			0xc8, 0x1c, 0x7b, 0x00, 0xe8, 0x11, 0x7f, 0xc2, 0x39,
+			0x4b, 0x7b, 0x97, 0x11, 0x92, 0x6c, 0xff, 0x89, 0x7f,
+			0x26, 0x89, 0x4f, 0x38, 0xfd, 0xdd, 0x08, 0xa7, 0xce,
+			0x6f, 0xe8, 0x57, 0x9b, 0x46, 0xe5, 0xdb, 0x72, 0x03,
+			0x1e, 0x7d, 0xb0, 0x77, 0xb9, 0xcc, 0xdb, 0x6c, 0xa7,
+			0xd8, 0x30, 0x34, 0xad, 0xa7, 0xe4, 0x63, 0xf0, 0x19,
+			0x0e, 0x5d, 0x3b, 0xe0, 0xff, 0x40, 0x1c, 0xa5, 0xb3,
+			0xb9, 0x87, 0x6e, 0x2c, 0xf3, 0x5f, 0xcd, 0x54, 0x2a,
+			0xc0, 0x6e, 0x2b, 0xd9, 0x2d, 0xcc, 0xd5, 0x68, 0x95,
+			0x4a, 0x4a, 0x84, 0x60, 0x54, 0xee, 0xa0, 0x21, 0x9e,
+			0x8d, 0x20, 0xcb, 0xe8, 0xc5, 0x5a, 0xba, 0xe2, 0xaa,
+			0x6e, 0x1c, 0xb1, 0xdf, 0x18, 0x9f, 0x94, 0xc7, 0x77,
+			0x5a, 0x2c, 0x0e, 0x05, 0xaa, 0x2a, 0x54, 0x58, 0x6c,
+			0xb3, 0x2e, 0x2f, 0xa4, 0x6e, 0x98, 0xbb, 0x6f, 0x41,
+			0x6d, 0xbd, 0x71, 0x95, 0xe4, 0xbc, 0x13, 0x37, 0x99,
+			0x0d, 0xac, 0x27, 0x69, 0xb9, 0x0b, 0x14, 0x5f, 0x6e,
+			0xd2, 0x2b, 0xe2, 0x0c, 0xc6, 0xbc, 0x10, 0x11, 0x47,
+			0xb7, 0x37, 0x2c, 0x0e, 0x88, 0xcd, 0xbb, 0xf7, 0x28,
+			0xd6, 0x4a, 0x9d, 0xff, 0x3c, 0x2f, 0x7d, 0x2b, 0xe8,
+			0xe8, 0x9e, 0xae, 0x7b, 0xe6, 0x2a, 0xb3, 0x4e, 0x20,
+			0xcc, 0xf1, 0x81, 0x8e, 0xed, 0x6d, 0xe2, 0x99, 0xf5,
+			0xb5, 0x1a, 0x30, 0x95, 0x52, 0x34, 0xf5, 0x3f, 0xc3,
+			0x31, 0xd6, 0xbe, 0xa2, 0xc8, 0xdc, 0xe4, 0x1c, 0xf6,
+			0x0f, 0x4d, 0x0b, 0x89, 0x8e, 0x66, 0x93, 0x88, 0xb8,
+			0xad, 0xbc, 0xdc, 0x96, 0x01, 0x9f, 0x16, 0x70, 0xf5,
+			0x4f, 0xa4, 0x0e, 0x0f, 0xc3, 0xf6, 0x9c, 0xe1, 0xa1,
+			0xe3, 0xec, 0x9d, 0x09, 0xcd, 0x90, 0x52, 0x26, 0x09,
+			0xd9, 0x9c, 0xde, 0xbd, 0xdf, 0xbb, 0xf4, 0x50, 0xd1,
+			0x89, 0x68, 0xf6, 0x86, 0x53, 0x33, 0x5a, 0xf6, 0x54,
+			0xb5, 0x7c, 0xe1, 0xd7, 0x8d, 0xb8, 0x58, 0xf5, 0xda,
+			0x14, 0xc8, 0x35, 0x1c, 0xcf, 0x44, 0x62, 0xbc, 0xd7,
+			0xe3, 0xd8, 0x32, 0xcf, 0x16, 0xf5, 0x2f, 0x55, 0x23,
+			0xc0, 0x1b, 0xc4, 0xe3, 0x28, 0xc8, 0xc8, 0x97, 0x70,
+			0x8b, 0x06, 0x98, 0xfb, 0xf6, 0x33, 0x6b, 0x86, 0x1c,
+			0xdb, 0x2a, 0x3c, 0x08, 0x08, 0x57, 0xd3, 0x4a, 0xf8,
+			0x22, 0x26, 0x78, 0x65, 0x5f, 0xa6, 0xf8, 0x9f, 0x22,
+			0x76, 0x62, 0xb0, 0x08, 0x68, 0x70, 0xea, 0x72, 0x57,
+			0x6b, 0xe7, 0xf7, 0xc6, 0x12, 0x9a, 0x49, 0x50, 0xa9,
+			0xa5, 0x6c, 0xe7, 0xda, 0xb4, 0xbf, 0xb6, 0xbf, 0x4f,
+			0xdf, 0x9e, 0x9b, 0xb4, 0xb3, 0x8d, 0x1a, 0x12, 0x16,
+			0x68, 0xd2, 0x63, 0xae, 0x92, 0x77, 0x1f, 0x03, 0xa5,
+			0xed, 0x58, 0x3b, 0xe9, 0x0b, 0xfe, 0xfc, 0xae, 0x53,
+			0x0b, 0x5f, 0x13, 0xf2, 0xd2, 0xe2, 0x0b, 0xec, 0x75,
+			0x85, 0x68, 0x0c, 0x57, 0xde, 0x1b, 0x6d, 0x78, 0x0b,
+			0x19, 0x66, 0xa8, 0xf5, 0x45, 0x72, 0x2b, 0x01, 0x06,
+			0xf6, 0xd1, 0x47, 0x21, 0x24, 0x07, 0xf7, 0x71, 0x03,
+			0xbc, 0xb0, 0x7c, 0x5b, 0x5c, 0x24, 0xff, 0x74, 0x47,
+			0x62, 0x81, 0xc3, 0x0b, 0x31, 0x76, 0x90, 0x5b, 0xef,
+			0x95, 0xa8, 0xa7, 0x02, 0xa1, 0xbf, 0xe1, 0xf4, 0x16,
+			0x06, 0x8a, 0x97, 0x39, 0x35, 0xcf, 0xf3, 0xa7, 0x4a,
+			0x43, 0xba, 0x05, 0x95, 0x7d, 0x73, 0x76, 0x7a, 0x53,
+			0xef, 0xf8, 0x4e, 0xcb, 0x04, 0x70, 0x4c, 0xee, 0xff,
+			0x82, 0xbd, 0xcd, 0xc1, 0xbe, 0x3d, 0x83, 0x71, 0x03,
+			0xf0, 0xc0, 0x2b, 0x98, 0xf9, 0x60, 0x54, 0x02, 0x7d,
+			0xa6, 0x41, 0xcc, 0xa3, 0xd7, 0x8d, 0xfd, 0xce, 0x28,
+			0xae, 0x0f, 0x48, 0x17, 0x2a, 0xaf, 0xe9, 0xb9, 0x4a,
+			0x8a, 0x22, 0xd2, 0x4d, 0xd3, 0x1b, 0xa3, 0x39, 0x88,
+			0x8a, 0x8f, 0x5b, 0x44, 0x97, 0xb9, 0x04, 0x1b, 0x58,
+			0x67, 0x74, 0x2f, 0x07, 0x7a, 0x52, 0xa9, 0x9d, 0xa4,
+			0x41, 0x28, 0xf2, 0x35, 0xca, 0x68, 0x4e, 0x4a, 0x3a,
+			0x66, 0xb9, 0x88, 0x2e, 0x65, 0x1d, 0x47, 0x04, 0xed,
+			0xdb, 0xe1, 0x40, 0x12, 0x06, 0x13, 0x62, 0x28, 0x3d,
+			0x0b, 0x35, 0x06, 0xc6, 0x2b, 0xb3, 0x71, 0x3c, 0xfa,
+			0x77, 0xec, 0x47, 0x93, 0x78, 0x36, 0x25, 0x19, 0xd7,
+			0x70, 0x30, 0x8a, 0x4c, 0x94, 0xdc, 0x3e, 0xeb, 0x61,
+			0x25, 0xbc, 0xa0, 0x27, 0xd9, 0x17, 0xa5, 0x19, 0x4f,
+			0xf4, 0x93, 0x32, 0x56, 0x9a, 0x0b, 0x77, 0xb4, 0x55,
+			0x1b, 0x8f, 0x9e, 0x69, 0x5b, 0xe2, 0x6d, 0x70, 0x15,
+			0x79, 0x5c, 0xf6, 0xb6, 0x04, 0xa2, 0x01, 0x37, 0x74,
+			0x20, 0xb8, 0x62, 0xf6, 0x37, 0x3c, 0xab, 0xca, 0x71,
+			0xa5, 0x8a, 0x56, 0x5d, 0x6a, 0x4a, 0x61, 0x2e, 0xb8,
+			0x62, 0x7d, 0x47, 0x34, 0x7d, 0xcd, 0x4d, 0x70, 0x23,
+			0xf5, 0xaa, 0xd1, 0xa5, 0xf0, 0x4c, 0x38, 0xc3, 0x98,
+			0x79, 0x4c, 0x0b, 0x6b, 0xcc, 0xe7, 0xd7, 0x09, 0xae,
+			0x23, 0x9b, 0x2f, 0xde, 0x70, 0xc6, 0xad, 0x0f, 0x66,
+			0xb5, 0x78, 0x6b, 0x0b, 0xb0, 0x2e, 0x94, 0xf2, 0xa8,
+			0x8b, 0x74, 0xf0, 0x03, 0x47, 0xd8, 0xec, 0xe8, 0x1f,
+			0xa3, 0x7b, 0x38, 0x9e, 0x0e, 0xc0, 0x47, 0xd2, 0x0f,
+			0x8e, 0x7f, 0xb1, 0x83, 0xd3, 0x86, 0x79, 0x3c, 0xa1,
+			0xae, 0xc4, 0xaf, 0xae, 0x9d, 0x83, 0xc0, 0xd1, 0x2b,
+			0x2b, 0xda, 0x50, 0x8c, 0xea, 0x41, 0x97, 0x9b, 0x0f,
+			0x15, 0xc2, 0xe2, 0x8f, 0x39, 0x0b, 0x92, 0xdd, 0xde,
+			0x52, 0x62, 0x74, 0xdc, 0xda, 0x11, 0x87, 0x4d, 0xa9,
+			0x4a, 0xc5, 0x2f, 0xae, 0xaf, 0xc1, 0xc3, 0x05, 0xfa,
+			0x38, 0xcc, 0x5c, 0xb1, 0x9f, 0xe0, 0x82, 0x90, 0xb3,
+			0xd5, 0xdc, 0xf4, 0x55, 0xdb, 0xea, 0x94, 0x06, 0x7c,
+			0x2c, 0x82, 0x78, 0xeb, 0xa5, 0x01, 0xf0, 0x3d, 0x4b,
+			0x87, 0xdd, 0xd5, 0x91, 0x4f, 0xf3, 0xa7, 0xdf, 0xa1,
+			0xd8, 0x31, 0xde, 0x05, 0x99, 0x67, 0x3d, 0xa4, 0x6b,
+			0x19, 0xa3, 0xe8, 0x55, 0xb7, 0xf5, 0xc3, 0x63, 0x5e,
+			0xd4, 0x38, 0xf9, 0x24, 0x64, 0x7d, 0x17, 0xc1, 0x07,
+			0xbe, 0x39, 0x54, 0x1b, 0x44, 0xe5, 0xc6, 0x3c, 0x02,
+			0xb1, 0x6f, 0xff, 0x8c, 0xcb, 0x79, 0xe2, 0xec, 0x4d,
+			0x01, 0xfa, 0x7f, 0x88, 0x1d, 0xc3, 0x4c, 0x6a, 0xfb,
+			0x0b, 0xc6, 0x57, 0xc3, 0xd8, 0x24, 0x47, 0x41, 0xbd,
+			0x27, 0xc4, 0xd4, 0x49, 0xfb, 0x52, 0xe6, 0x77, 0x5f,
+			0x0a, 0xdf, 0xea, 0xd5, 0xd3, 0x22, 0xc3, 0x53, 0x16,
+			0xf3, 0x1b, 0x7b, 0x09, 0xd7, 0x10, 0x0e, 0x23, 0xae,
+			0x16, 0x8a, 0x93, 0xcb, 0xc9, 0xb7, 0xb8, 0xff, 0xd2,
+			0x50, 0x1f, 0x25, 0xa7, 0x71, 0x8f, 0x3f, 0xc0, 0xe1,
+			0x37, 0x10, 0x0b, 0x43, 0x6e, 0x2b, 0x16, 0x59, 0x8f,
+			0x77, 0x77, 0x6b, 0x77, 0xce, 0x76, 0x6b, 0x37, 0x81,
+			0xaf, 0x83, 0x42, 0x92, 0x93, 0xe5, 0x39, 0xca, 0xd2,
+			0x20, 0x2e, 0xcf, 0x24, 0x26, 0x4c, 0x51, 0x1c, 0x58,
+			0xc5, 0x8d, 0x05, 0x11, 0xdf, 0xae, 0x51, 0x38, 0xde,
+			0xab, 0x4e, 0x04, 0xc8, 0x24, 0x24, 0x0d, 0xd5, 0x9c,
+			0x5b, 0x2b, 0xe6, 0x0d, 0x83, 0x95, 0xcd, 0x1c, 0x89,
+			0xa1, 0xaf, 0x67, 0x47, 0xfb, 0x08, 0x02, 0xf8, 0x8b,
+			0x63, 0x05, 0x73, 0x20, 0x64, 0xd7, 0x52, 0x15, 0xa4,
+			0x5d, 0x63, 0x73, 0x73, 0x12, 0x0b, 0xdd, 0xfe, 0x9f,
+			0xb7, 0xe8, 0xa8, 0x94, 0x3a, 0x86, 0xff, 0xcf, 0x7d,
+			0x24, 0xbd, 0xb9, 0xea, 0x68, 0x23, 0xf4, 0x07, 0xc3,
+			0xfe, 0x63, 0xd0, 0xab, 0x65, 0x8a, 0xf0, 0x6d, 0x81,
+			0x8c, 0xc8, 0x0e, 0xc6, 0x6b, 0xdd, 0x2e, 0x65, 0x9b,
+			0x17, 0xcf, 0x82, 0x69, 0x46, 0xba, 0x62, 0x5d, 0x31,
+			0x33, 0x60, 0x18, 0x94, 0xa5, 0x77, 0x24, 0xc6, 0x45,
+			0xe5, 0xb3, 0xd5, 0x12, 0x10, 0xc9, 0x22, 0x98, 0xf9,
+			0xca, 0x20, 0x89, 0x79, 0x04, 0x08, 0xf5, 0x1c, 0xf8,
+			0x50, 0x8c, 0x25, 0xaa, 0x90, 0x90, 0x44, 0xbc, 0xfb,
+			0x5d, 0x3f, 0xf8, 0x38, 0x64, 0xca, 0x8d, 0xff, 0x17,
+			0xce, 0x70, 0x51, 0x90, 0x75, 0x6b, 0x7d, 0x64, 0x43,
+			0x56, 0xcd, 0xf8, 0x85, 0x93, 0x65, 0x09, 0x81, 0x30,
+			0x76, 0x79, 0xcc, 0xdf, 0x9c, 0x6d, 0xff, 0x89, 0x38,
+			0x60, 0xbf, 0x07, 0xcb, 0x2f, 0xc9, 0x87, 0xd7, 0xac,
+			0x74, 0x19, 0x57, 0x90, 0x5e, 0x69, 0x61, 0xf6, 0xca,
+			0xea, 0x45, 0x6b, 0xe2, 0xfe, 0x2c, 0xff, 0x1b, 0x23,
+			0x15, 0x52, 0xdd, 0x57, 0xfe, 0x1d, 0x10, 0xea, 0x0f,
+			0xce, 0x98, 0xe7, 0x47, 0x27, 0xec, 0x36, 0xe5, 0x68,
+			0x17, 0xcf, 0xdc, 0xb9, 0xef, 0x6a, 0xbc, 0xec, 0x78,
+			0x08, 0x64, 0x06, 0xe1, 0x1c, 0xc6, 0x87, 0xd6, 0x0a,
+			0xb1, 0x81, 0xc6, 0xb6, 0xf8, 0x8b, 0xe3, 0x19, 0x8c,
+			0xce, 0x46, 0x40, 0xc5, 0xc2, 0xae, 0x50, 0x26, 0x4a,
+			0x90, 0x91, 0x8d, 0xfe, 0x6b, 0x7d, 0x0a, 0x54, 0x4a,
+			0x4b, 0x48, 0x74, 0x4a, 0x37, 0x21, 0x7f, 0xdd, 0x87,
+			0xa3, 0x1e, 0xac, 0xcd, 0xf5, 0x9e, 0x75, 0xa2, 0x52,
+			0x63, 0x76, 0xca, 0x9e, 0x02, 0xeb, 0xe6, 0xa6, 0x73,
+			0xad, 0xea, 0xe8, 0x3e, 0x6f, 0x44, 0xed, 0xe8, 0x01,
+			0x29, 0x19, 0x6a, 0x20, 0x35, 0xa7, 0xf0, 0xf1, 0xaf,
+			0xc0, 0x3b, 0xb1, 0xd5, 0xe4, 0xfb, 0xf7, 0xd7, 0x2f,
+			0x33, 0x6c, 0x73, 0xfd, 0xe5, 0x5c, 0x63, 0xf6, 0x1c,
+			0x06, 0x13, 0xaf, 0xc1, 0x80, 0x55, 0x07, 0xae, 0x8c,
+			0x13, 0x74, 0xf6, 0xe0, 0x54, 0x15, 0xd8, 0xe0, 0xa5,
+			0x03, 0xcf, 0x22, 0xbe, 0x18, 0xef, 0x26, 0xad, 0x9c,
+			0x9d, 0x51, 0xb1, 0x3b, 0x37, 0x03, 0xbf, 0xf0, 0xc5,
+			0xcb, 0x6c, 0x5d, 0x30, 0xa9, 0x5a, 0x10, 0x90, 0xfa,
+			0xb4, 0xd4, 0x0e, 0x6b, 0x4a, 0x0a, 0x6c, 0x9d, 0x2e,
+			0x69, 0xe8, 0xec, 0x69, 0xe2, 0x50, 0xab, 0x2f, 0xdc,
+			0xff, 0xaf, 0xac, 0x65, 0xe7, 0xf0, 0xc1, 0x6f, 0x7c,
+			0x2d, 0xa1, 0xeb, 0x97, 0x90, 0x7c, 0x1e, 0xa8, 0x53,
+			0x1b, 0x87, 0xc5, 0xa9, 0xa1, 0xcf, 0x86, 0x7e, 0x11,
+			0xf8, 0xd6, 0x14, 0xda, 0x19, 0x81, 0x19, 0xb4, 0x45,
+			0x1c, 0x7c, 0xb9, 0x96, 0xa2, 0xac, 0x79, 0x24, 0x94,
+			0x7c, 0xb2, 0x1e, 0x83, 0xea, 0xc4, 0xb9, 0xd2, 0x0d,
+			0x4c, 0x55, 0x3d, 0x15, 0x7b, 0x65, 0xd8, 0xff, 0x03,
+			0x5d, 0xed, 0x3c, 0x94, 0x76, 0x19, 0x40, 0x3b, 0xcc,
+			0x45, 0xbe, 0x91, 0x19, 0x8c, 0x75, 0xe5, 0xd2, 0xbe,
+			0x67, 0x40, 0xb1, 0x67, 0x8e, 0x2a, 0x34, 0xd5, 0x99,
+			0xee, 0xd9, 0x4f, 0x89, 0x7c, 0xf0, 0xd6, 0x93, 0x59,
+			0x4b, 0x3a, 0x8d, 0xe6, 0xbd, 0xde, 0xce, 0xef, 0x8b,
+			0x3c, 0xe3, 0xf7, 0x06, 0x33, 0x27, 0x8d, 0xd9, 0x22,
+			0x1a, 0x65, 0x40, 0xfc, 0x69, 0x1b, 0x7d, 0xf0, 0xed,
+			0xe4, 0xe0, 0x7f, 0x6d, 0x23, 0xed, 0x11, 0xd1, 0x07,
+			0xb0, 0x2f, 0x8a, 0xbf, 0x51, 0x37, 0x22, 0x04, 0xed,
+			0x93, 0xea, 0x1d, 0x0b, 0x30, 0x15, 0x89, 0x22, 0x7a,
+			0x45, 0x56, 0x99, 0xc6, 0xac, 0xd6, 0xce, 0x61, 0xea,
+			0xb2, 0x59, 0xe8, 0xb5, 0xfc, 0x87, 0xa7, 0xfe, 0x09,
+			0xa2, 0x0d, 0x5e, 0xbe, 0xb9, 0xd4, 0x9a, 0x1b, 0x60,
+			0xda, 0xb9, 0x32, 0xf1, 0x30, 0x3e, 0xb2, 0x45, 0x6d,
+			0x55, 0x0c, 0x2c, 0x4b, 0x9a, 0xc0, 0xbb, 0x8e, 0xac,
+			0x9c, 0x95, 0x5f, 0x08, 0x88, 0xa0, 0x53, 0x05, 0x75,
+			0x8d, 0x9e, 0x9d, 0x3f, 0x0f, 0xdd, 0x50, 0x0d, 0xf8,
+			0x11, 0xbd, 0xf9, 0xfb, 0x22, 0x5c, 0x7b, 0x9e, 0x7c,
+			0x8e, 0x2f, 0x0e, 0xdb, 0xb8, 0x1d, 0x0c, 0x5e, 0x82,
+			0xf3, 0x8e, 0xec, 0x32, 0x1c, 0x59, 0x73, 0xa5, 0xf3,
+			0x5b, 0x47, 0x00, 0x64, 0x89, 0x68, 0x3b, 0xaf, 0xe8,
+			0xe2, 0x9b, 0xa6, 0xac, 0x2c, 0xf9, 0x2b, 0x92, 0xf3,
+			0xf0, 0x5b, 0xcc, 0x75, 0x22, 0xd5, 0xf4, 0x2b, 0x06,
+			0x96, 0xc8, 0x50, 0xee, 0xac, 0x62, 0x16, 0x45, 0x9e,
+			0xbc, 0xcc, 0x8f, 0x5a, 0x66, 0xc6, 0x30, 0x7c, 0xe0,
+			0x22, 0xcc, 0xb9, 0xda, 0x0b, 0x0a, 0xbd, 0x2a, 0x2e,
+			0x46, 0x7d, 0xb6, 0x86, 0x70, 0xa3, 0x16, 0x49, 0x85,
+			0x28, 0x7b, 0xe9, 0x00, 0x6b, 0xfa, 0x06, 0xb0, 0xeb,
+			0xbd, 0x67, 0x28, 0x6f, 0x27, 0xd7, 0x9c, 0x7f, 0xda,
+			0xec, 0xf4, 0x7e, 0x55, 0xe1, 0x0c, 0x29, 0x61, 0x7a,
+			0xf5, 0xb6, 0xb8, 0xa5, 0xef, 0x36, 0x6a, 0xad, 0x59,
+			0x22, 0xbd, 0x3d, 0xad, 0x86, 0xe7, 0x4c, 0x69, 0x26,
+			0x0f, 0xbf, 0x67, 0xad, 0x65, 0x32, 0xbd, 0x21, 0xd6,
+			0x59, 0x6b, 0xe3, 0xda, 0xc1, 0x6f, 0x82, 0x41, 0x2c,
+			0xaa, 0xe4, 0x8c, 0xfc, 0x7c, 0x61, 0x28, 0x51, 0x52,
+			0x3d, 0xf1, 0x84, 0xb5, 0x0b, 0xfd, 0x1f, 0x2a, 0x06,
+			0x2e, 0x30, 0xed, 0x63, 0x43, 0xc9, 0x83, 0x97, 0xb1,
+			0xd4, 0x80, 0x6f, 0x2c, 0x50, 0xec, 0x20, 0x95, 0x42,
+			0xa0, 0x34, 0x94, 0x1a, 0xa9, 0x5e, 0x5b, 0x59, 0xe3,
+			0x39, 0xac, 0xbd, 0x2f, 0x77, 0x36, 0x59, 0x9c, 0xc3,
+			0x3c, 0x66, 0x87, 0xf5, 0x81, 0x4b, 0xb0, 0x10, 0x4a,
+			0xe6, 0x46, 0xe7, 0xce, 0x93, 0x7b, 0x24, 0x6b, 0x2e,
+			0xc1, 0xe5, 0xaf, 0x4b, 0x71, 0x22, 0xad, 0x88, 0xda,
+			0x55, 0xcb, 0xe0, 0x73, 0xd1, 0x65, 0x7d, 0xa5, 0x7f,
+			0x36, 0xbc, 0x42, 0xc2, 0x78, 0x9f, 0x88, 0xe8, 0xdb,
+			0xff, 0x8a, 0x5a, 0x80, 0x34, 0x3a, 0x23, 0x4c, 0x8a,
+			0x81, 0xff, 0xbd, 0xb7, 0x88, 0xd0, 0x73, 0x07, 0x8a,
+			0x4e, 0xa7, 0x4a, 0x61, 0x0f, 0x1f, 0x1c, 0xe7, 0x34,
+			0x37, 0x1c, 0x53, 0x90, 0x3b, 0xa4, 0x32, 0x6c, 0x6d,
+			0xe8, 0x00, 0xde, 0xe0, 0x0c, 0x5e, 0x06, 0xef, 0xb8,
+			0x48, 0x2e, 0xb3, 0xda, 0xac, 0x92, 0x4d, 0x0d, 0x95,
+			0x75, 0x44, 0x01, 0x6f, 0x97, 0xc3, 0x29, 0x76, 0x33,
+			0x36, 0x9a, 0xae, 0xfb, 0x1b, 0x43, 0xe5, 0xb1, 0x54,
+			0x3a, 0x9c, 0x76, 0x7f, 0x76, 0x83, 0xc9, 0x9c, 0xd6,
+			0x56, 0x59, 0x83, 0xa9, 0xde, 0xd7, 0xb0, 0xf3, 0x34,
+			0x11, 0x31, 0x06, 0x8e, 0xe9, 0xd4, 0x79, 0xd5, 0x3d,
+			0x31, 0x6b, 0x59, 0xe9, 0x54, 0x69, 0x12, 0xfd, 0x44,
+			0x59, 0x4e, 0x1b, 0x3b, 0xb4, 0x12, 0xe9, 0xfb, 0xb0,
+			0xb4, 0x84, 0xb9, 0x7d, 0xea, 0x4f, 0xd1, 0x5f, 0xd0,
+			0x3e, 0xce, 0xef, 0x5c, 0xf7, 0xea, 0x55, 0xa0, 0x8f,
+			0xa8, 0xa7, 0x98, 0xe7, 0xa1, 0x6b, 0x3f, 0xba, 0x5a,
+			0x32, 0x4b, 0xfa, 0x31, 0xb6, 0x63, 0x86, 0x19, 0x00,
+			0xa2, 0x6d, 0x7d, 0x15, 0x56, 0x05, 0x68, 0xa3, 0xe0,
+			0xf3, 0xd4, 0x82, 0xcf, 0xeb, 0xd4, 0x1c, 0xd0, 0xb6,
+			0x14, 0x5e, 0x9e, 0x6b, 0xed, 0x7a, 0x02, 0x1a, 0xcd,
+			0x09, 0xdc, 0x26, 0x98, 0x50, 0x11, 0x34, 0x39, 0x50,
+			0x5a, 0x70, 0x79, 0x85, 0xca, 0xd2, 0xf2, 0x0c, 0x0d,
+			0x12, 0x1f, 0x2e, 0x41, 0x46, 0x51, 0x72, 0x75, 0x78,
+			0x8c, 0xa4, 0xaf, 0xba, 0xca, 0xd3, 0xdf, 0xea, 0xf8,
+			0x09, 0x0b, 0x36, 0x45, 0x4f, 0x77, 0x83, 0xae, 0xbc,
+			0xc5, 0xce, 0xe1, 0xf6, 0x1d, 0x1e, 0x38, 0x56, 0x9c,
+			0x9f, 0xb1, 0xbd, 0xda, 0xe7, 0xf0, 0xf4, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x0b, 0x1f, 0x2c, 0x38,
+		},
+};
+
+static const struct mldsa_testvector mldsa65_testvector = {
+	.alg = MLDSA65,
+	.pk_len = MLDSA65_PUBLIC_KEY_SIZE,
+	.pk =
+		(const u8[MLDSA65_PUBLIC_KEY_SIZE]) {
+			0x9f, 0x55, 0x1e, 0x7f, 0x9c, 0x08, 0xb2, 0x83, 0xfd,
+			0x5b, 0xa2, 0xac, 0x4f, 0x26, 0xc2, 0xf5, 0x06, 0x05,
+			0x96, 0x08, 0x24, 0xad, 0xec, 0xe4, 0x99, 0xcc, 0x6c,
+			0xbd, 0x55, 0x37, 0x15, 0x94, 0xab, 0x31, 0x9e, 0x56,
+			0xe5, 0xe4, 0x55, 0xec, 0x4d, 0x49, 0x5b, 0x5a, 0x7a,
+			0xe8, 0xc3, 0x4a, 0x08, 0x44, 0x4a, 0xc2, 0x2d, 0xe4,
+			0x61, 0x33, 0x90, 0x20, 0x71, 0x45, 0xa5, 0x45, 0xd0,
+			0x83, 0x2b, 0x32, 0x6c, 0xa7, 0x9e, 0x76, 0xcd, 0xfb,
+			0x58, 0x15, 0x9e, 0x74, 0x0d, 0x67, 0x57, 0xb1, 0x06,
+			0x5b, 0x5d, 0xd5, 0x1c, 0xbb, 0x95, 0x40, 0x1c, 0x71,
+			0x31, 0x03, 0xef, 0xff, 0x04, 0x6b, 0xdd, 0xa2, 0xf0,
+			0x32, 0x00, 0x72, 0xbc, 0x87, 0xb6, 0x2c, 0x1f, 0x90,
+			0x7f, 0x92, 0xa0, 0xb2, 0x04, 0xdd, 0xa9, 0xaf, 0x7f,
+			0x01, 0x28, 0x4c, 0xb2, 0x57, 0x2d, 0x56, 0x93, 0xd0,
+			0xc7, 0x54, 0x02, 0x90, 0x57, 0x70, 0x23, 0x57, 0xe8,
+			0xe7, 0x33, 0x32, 0x98, 0xfc, 0x9b, 0x8e, 0x6e, 0x7b,
+			0xaa, 0x5d, 0xb5, 0x4e, 0xe0, 0x5d, 0x97, 0xa3, 0xea,
+			0x43, 0x7e, 0xb3, 0xa4, 0x8c, 0xcf, 0xdc, 0xc0, 0x51,
+			0xa7, 0x99, 0x45, 0x3d, 0x3c, 0xa0, 0xba, 0xc5, 0xff,
+			0xe1, 0x89, 0xb3, 0x7d, 0xc3, 0xdc, 0xe2, 0x23, 0x81,
+			0xff, 0xa9, 0xc7, 0x93, 0xc6, 0x67, 0xad, 0x94, 0xcf,
+			0xeb, 0x91, 0x78, 0x15, 0x25, 0xf7, 0xf5, 0x06, 0x08,
+			0x2f, 0x0c, 0xee, 0x0b, 0x6a, 0x06, 0x59, 0xe0, 0x1f,
+			0x2e, 0x5a, 0x12, 0x06, 0xf5, 0xf4, 0x8e, 0x75, 0x57,
+			0xa9, 0x33, 0x23, 0x0f, 0xc2, 0x6f, 0x02, 0xf8, 0x68,
+			0x0f, 0x62, 0x02, 0x81, 0xfe, 0x03, 0x7c, 0xaf, 0xd7,
+			0x42, 0x5b, 0xcc, 0xe7, 0x2b, 0xea, 0x49, 0xab, 0x03,
+			0x6d, 0x0a, 0x02, 0xae, 0x47, 0x79, 0xce, 0xfd, 0x18,
+			0x76, 0x07, 0x9e, 0xa6, 0xbf, 0x7e, 0x8d, 0x73, 0xf9,
+			0x44, 0xeb, 0x8c, 0xc5, 0x59, 0xb7, 0x19, 0xf6, 0x73,
+			0x53, 0x42, 0x2a, 0x55, 0x7b, 0xb4, 0x56, 0x49, 0x08,
+			0x9e, 0x9a, 0x65, 0x60, 0x70, 0x1d, 0xbd, 0xc6, 0x85,
+			0x29, 0xde, 0xfe, 0x44, 0xae, 0xdf, 0x25, 0xfd, 0x5b,
+			0x74, 0x6c, 0x96, 0xe6, 0x81, 0x37, 0x80, 0xe0, 0x9e,
+			0xf3, 0x75, 0x63, 0xb4, 0xc9, 0x2f, 0x71, 0xe6, 0xeb,
+			0xdf, 0xaf, 0x7e, 0xff, 0x9e, 0xe0, 0xbf, 0xca, 0xca,
+			0x11, 0xed, 0xc6, 0x04, 0xd8, 0x49, 0x13, 0x2c, 0x63,
+			0xf1, 0xb3, 0x17, 0x74, 0xd9, 0x50, 0x3f, 0xb9, 0x29,
+			0x0e, 0x48, 0xa7, 0xf0, 0xdc, 0x78, 0x18, 0x0e, 0x9f,
+			0xb7, 0xde, 0x36, 0x79, 0x67, 0xa4, 0x23, 0x08, 0xe7,
+			0x62, 0xe8, 0xa4, 0xe5, 0xcf, 0xff, 0x35, 0x55, 0x36,
+			0x2e, 0x3a, 0xe4, 0x45, 0x6a, 0x80, 0xf2, 0xca, 0xe7,
+			0x40, 0x79, 0x14, 0xc4, 0x62, 0x38, 0xbb, 0xd0, 0x4e,
+			0x6c, 0xb5, 0x85, 0x42, 0x3f, 0x35, 0xf7, 0xd7, 0x54,
+			0xb8, 0x2b, 0x8b, 0xd5, 0x6f, 0x16, 0x61, 0x27, 0x23,
+			0xac, 0xdb, 0xea, 0x9b, 0x3b, 0x99, 0xcd, 0x79, 0xe6,
+			0x12, 0x09, 0x99, 0x09, 0xa4, 0xe1, 0x88, 0x25, 0x00,
+			0x9e, 0x60, 0x16, 0x63, 0xd7, 0x42, 0x9b, 0xcc, 0x36,
+			0x9a, 0x8d, 0xa3, 0x75, 0x36, 0xa1, 0xa8, 0xfc, 0xa2,
+			0xfe, 0x29, 0x26, 0x4c, 0x93, 0x21, 0x44, 0x6b, 0x1c,
+			0xba, 0xbd, 0xef, 0xff, 0x6d, 0x1f, 0x2b, 0x6c, 0x66,
+			0x81, 0x9a, 0x3a, 0x1d, 0x0b, 0xd7, 0x24, 0xd4, 0xb8,
+			0x93, 0xb5, 0x22, 0xf9, 0xd2, 0xf4, 0xa5, 0x05, 0x78,
+			0x38, 0xae, 0x58, 0xf6, 0x50, 0x8f, 0x47, 0x1d, 0xf3,
+			0xfb, 0x0d, 0x04, 0x14, 0xd1, 0xd6, 0xd8, 0x2e, 0xf2,
+			0xbd, 0xf5, 0x71, 0x86, 0x4c, 0xdd, 0x61, 0x24, 0x18,
+			0x5b, 0x54, 0xf5, 0xcd, 0x99, 0x89, 0x01, 0x8e, 0xd1,
+			0x19, 0x52, 0xbc, 0x45, 0xed, 0x0e, 0xec, 0x72, 0x2f,
+			0x5a, 0xe7, 0xdf, 0x36, 0x1c, 0x57, 0x9f, 0xb2, 0x8b,
+			0xf2, 0x78, 0x1b, 0x3e, 0xc5, 0x48, 0x1f, 0x27, 0x04,
+			0x76, 0x10, 0x44, 0xee, 0x5c, 0x68, 0x8f, 0xca, 0xd7,
+			0x31, 0xfc, 0x5c, 0x40, 0x03, 0x2e, 0xbd, 0x1d, 0x59,
+			0x13, 0x57, 0xbc, 0x33, 0xc6, 0xa1, 0xa3, 0xe5, 0x55,
+			0x79, 0x9b, 0x7e, 0x49, 0xbb, 0x23, 0x96, 0xc3, 0x1c,
+			0xfe, 0x66, 0xeb, 0x5b, 0x5f, 0xe5, 0x03, 0xc9, 0xa4,
+			0xac, 0x4d, 0xc4, 0x50, 0xbb, 0xd3, 0xc1, 0x91, 0x48,
+			0xe0, 0x93, 0x92, 0x2a, 0xdb, 0x41, 0x37, 0x98, 0xbc,
+			0xa2, 0x7a, 0x09, 0x92, 0x0b, 0x1c, 0xe6, 0x4b, 0x1e,
+			0x8e, 0x78, 0x81, 0x74, 0x7d, 0x6b, 0x71, 0xd5, 0xe7,
+			0x0e, 0x7b, 0xc2, 0x74, 0x5d, 0x89, 0xf1, 0xfa, 0x59,
+			0xaa, 0xf7, 0x86, 0x66, 0x7e, 0xc2, 0x9c, 0xf4, 0xd5,
+			0x8d, 0xc0, 0xb7, 0xb7, 0xa2, 0xd5, 0xcd, 0x51, 0xc3,
+			0x7d, 0xa9, 0x5e, 0x46, 0xba, 0x06, 0xa3, 0x4d, 0x60,
+			0xd6, 0x68, 0xc6, 0xf9, 0x63, 0x88, 0x17, 0x5c, 0x20,
+			0xe1, 0xc4, 0x0f, 0x3f, 0xc1, 0xa9, 0xa7, 0x3e, 0x39,
+			0xef, 0x2f, 0xaf, 0xc4, 0x69, 0x29, 0xe3, 0xd4, 0x8d,
+			0xe0, 0x0e, 0x88, 0xc2, 0x93, 0x43, 0xfb, 0x28, 0xcf,
+			0x5d, 0x85, 0x50, 0xf7, 0xeb, 0x42, 0xf5, 0x87, 0xde,
+			0xa5, 0x65, 0xef, 0x43, 0x0c, 0x57, 0x76, 0x09, 0xf4,
+			0x5f, 0xde, 0x81, 0x0a, 0xd9, 0x59, 0x41, 0xa4, 0x6a,
+			0xb7, 0x05, 0xc7, 0xa5, 0xfe, 0x49, 0xd5, 0x9b, 0x57,
+			0x13, 0x14, 0x66, 0xe2, 0xb9, 0xcc, 0x09, 0x35, 0xd4,
+			0xb0, 0xe0, 0xd1, 0x0d, 0x7e, 0x50, 0x48, 0x45, 0x21,
+			0x00, 0x67, 0xb2, 0xad, 0xa7, 0x46, 0xe2, 0x6f, 0x70,
+			0xe5, 0x3c, 0x88, 0x04, 0xaa, 0x21, 0xde, 0x03, 0xb6,
+			0x6f, 0xfe, 0x43, 0x51, 0xdc, 0x2e, 0x5c, 0x6c, 0x77,
+			0x8f, 0x8e, 0x9d, 0x1a, 0x5b, 0x35, 0xc5, 0xe4, 0x48,
+			0x82, 0x17, 0x4b, 0xf0, 0xea, 0xc9, 0x0e, 0xd2, 0x8f,
+			0xcd, 0xd5, 0x01, 0xbd, 0x7f, 0x0f, 0xf5, 0xae, 0x92,
+			0x28, 0x1e, 0x2c, 0xf4, 0xe9, 0x03, 0xf7, 0x0a, 0xeb,
+			0x84, 0x18, 0xa1, 0x37, 0x38, 0x8a, 0x11, 0xa2, 0x5d,
+			0x8c, 0xf6, 0xe4, 0x3f, 0x5b, 0x87, 0x07, 0x6b, 0xb4,
+			0x07, 0xe0, 0x8f, 0x30, 0xc4, 0xfa, 0x27, 0xae, 0xfc,
+			0x02, 0xd1, 0x21, 0x5c, 0xbc, 0x0b, 0x93, 0x6e, 0x7e,
+			0xf9, 0x6b, 0x80, 0x7a, 0x25, 0x84, 0x20, 0xf1, 0x6a,
+			0xfa, 0x75, 0xed, 0x57, 0x61, 0x62, 0xa7, 0xf6, 0x5b,
+			0xe1, 0xb0, 0x38, 0xc8, 0xe9, 0x6d, 0x3f, 0xef, 0x1e,
+			0x99, 0x0b, 0xb7, 0xc8, 0x9f, 0x76, 0x5c, 0x04, 0x1f,
+			0x02, 0x92, 0x00, 0xa7, 0x38, 0x3d, 0x00, 0x3b, 0xa7,
+			0xbc, 0x39, 0x6e, 0xab, 0xf5, 0x10, 0xa8, 0xba, 0xd6,
+			0x28, 0x6b, 0x0e, 0x00, 0x48, 0xf9, 0x3b, 0x5c, 0xde,
+			0x59, 0x93, 0x46, 0xd6, 0x61, 0x52, 0x81, 0x71, 0x0f,
+			0x0e, 0x61, 0xac, 0xc6, 0x7f, 0x15, 0x93, 0xa7, 0xc1,
+			0x16, 0xb5, 0xef, 0x85, 0xd1, 0xa7, 0x61, 0xc2, 0x85,
+			0x1d, 0x61, 0xc6, 0xae, 0xb3, 0x9e, 0x8d, 0x23, 0xa3,
+			0xc8, 0xd5, 0xf2, 0xc7, 0x1b, 0x7e, 0xef, 0xd2, 0xdf,
+			0x25, 0xaf, 0x4e, 0x81, 0x15, 0x59, 0xe5, 0x36, 0xb1,
+			0xf1, 0xd5, 0xda, 0x58, 0xd8, 0xd9, 0x0d, 0x6d, 0xc9,
+			0x25, 0xb5, 0xe8, 0x1d, 0x3b, 0xca, 0x2d, 0xab, 0xf2,
+			0xe2, 0xe9, 0x55, 0xd7, 0xf4, 0xc7, 0xd0, 0x57, 0x7a,
+			0x86, 0x15, 0x0a, 0x5a, 0x8b, 0xd7, 0x3f, 0x66, 0x0f,
+			0x80, 0xb4, 0xe0, 0x5c, 0x33, 0xed, 0xaf, 0x1b, 0x3b,
+			0x6d, 0x1c, 0xd9, 0x8c, 0xb5, 0x96, 0xa3, 0xfb, 0xcf,
+			0xcc, 0x97, 0x1c, 0xae, 0x06, 0x19, 0x41, 0x61, 0xf8,
+			0x97, 0x6b, 0x82, 0x5e, 0x1c, 0xbf, 0x6f, 0x43, 0x3d,
+			0xe5, 0x00, 0xf5, 0xfe, 0x66, 0x48, 0x26, 0x31, 0xa1,
+			0x72, 0x67, 0x6e, 0xd4, 0x5b, 0x6f, 0x66, 0xde, 0x70,
+			0x8b, 0x2b, 0xc3, 0xa2, 0x30, 0xe9, 0x55, 0xc8, 0xff,
+			0xf8, 0xd0, 0xdd, 0xa9, 0x21, 0x85, 0x6e, 0x6c, 0x82,
+			0x66, 0xcc, 0x52, 0xf0, 0x9e, 0x1e, 0xb5, 0x3a, 0xff,
+			0x4c, 0xf3, 0xae, 0x02, 0xc3, 0x4b, 0x76, 0x25, 0xbd,
+			0xb0, 0x21, 0x54, 0x61, 0xda, 0x16, 0xd3, 0x23, 0x86,
+			0x41, 0xa1, 0x4c, 0x59, 0x15, 0x95, 0x65, 0x85, 0xb6,
+			0x8e, 0xa6, 0x37, 0xc0, 0xa2, 0x71, 0x1d, 0x67, 0x44,
+			0x7b, 0xe5, 0x4c, 0x4f, 0xb6, 0x2c, 0x46, 0xf7, 0x29,
+			0xa5, 0xf2, 0xd3, 0x51, 0x19, 0x91, 0x4d, 0xa7, 0xb5,
+			0x05, 0xb9, 0x6e, 0x61, 0x6e, 0xf8, 0xc0, 0x01, 0xe5,
+			0x41, 0x0a, 0x89, 0x64, 0x77, 0xf2, 0xc8, 0x63, 0x2d,
+			0x9d, 0x27, 0x7f, 0x47, 0x30, 0x39, 0xdf, 0xb6, 0x6e,
+			0x4f, 0x00, 0x3f, 0x15, 0xc6, 0xaf, 0x62, 0xdf, 0x3f,
+			0x47, 0xe8, 0x42, 0x90, 0x77, 0x23, 0x7a, 0xaa, 0x99,
+			0x53, 0x03, 0x63, 0x60, 0x59, 0x07, 0x52, 0x3c, 0xb5,
+			0x67, 0x59, 0xfe, 0x08, 0xe6, 0x43, 0x0f, 0x3b, 0x08,
+			0x7c, 0xc7, 0x07, 0x3c, 0xfa, 0x65, 0xea, 0x69, 0x51,
+			0x41, 0x31, 0xb3, 0x05, 0x69, 0xba, 0x2c, 0xbf, 0x89,
+			0x25, 0x9e, 0xfe, 0x07, 0x13, 0x78, 0x0e, 0x16, 0x54,
+			0xdf, 0x23, 0xdf, 0x10, 0x69, 0x79, 0xd0, 0x33, 0xd7,
+			0x21, 0x8b, 0xc8, 0x2a, 0xd0, 0x74, 0x0a, 0xfa, 0xb1,
+			0x6f, 0xa3, 0xcb, 0x1d, 0xca, 0x4f, 0x00, 0x46, 0x6c,
+			0x42, 0x09, 0xe0, 0x30, 0x89, 0x08, 0x33, 0x9b, 0x7b,
+			0x7b, 0x0f, 0x69, 0x5c, 0x0d, 0x34, 0x91, 0xfc, 0xfe,
+			0x22, 0x82, 0x02, 0xcd, 0xfa, 0x97, 0xe8, 0x28, 0x1d,
+			0xbc, 0x13, 0x0b, 0xfd, 0x47, 0xa1, 0x7e, 0xa2, 0x86,
+			0x4d, 0x6f, 0x12, 0x51, 0x35, 0x7d, 0x76, 0x8a, 0x58,
+			0x05, 0xb6, 0x39, 0xa1, 0x2f, 0xd7, 0xda, 0xaf, 0x00,
+			0xa0, 0x1a, 0x94, 0xd8, 0x23, 0x34, 0x99, 0x5c, 0xaf,
+			0xcc, 0x15, 0x4b, 0x56, 0xb2, 0xd2, 0x81, 0x07, 0xd3,
+			0xf3, 0x47, 0xa2, 0x45, 0x93, 0xcb, 0xae, 0xa7, 0x6b,
+			0x3f, 0xf9, 0xea, 0xfc, 0x0e, 0x64, 0xf2, 0x93, 0x7f,
+			0x24, 0x22, 0x73, 0x86, 0xc7, 0x2d, 0x75, 0x9b, 0x41,
+			0x8b, 0xfb, 0x3b, 0x26, 0x2a, 0xe5, 0x0b, 0xd4, 0x00,
+			0xe3, 0x2c, 0x69, 0x49, 0x62, 0x6c, 0x13, 0x58, 0x6e,
+			0xac, 0x43, 0xe5, 0x2b, 0x3b, 0x88, 0xdc, 0xd4, 0x41,
+			0xe8, 0xee, 0x4e, 0xc3, 0x28, 0x91, 0x17, 0x9a, 0x5a,
+			0xdb, 0x80, 0x8b, 0x4d, 0x64, 0xcc, 0xbe, 0x66, 0xa4,
+			0x62, 0xfb, 0x13, 0x44, 0x10, 0xd9, 0xe4, 0xd5, 0xa5,
+			0xae, 0x9e, 0x42, 0x50, 0xfc, 0x78, 0xad, 0xfa, 0xc4,
+			0xd0, 0x5a, 0x60, 0x9b, 0x45, 0x2b, 0x61, 0x5c, 0x57,
+			0xb5, 0x92, 0x28, 0xe9, 0xf5, 0x35, 0x67, 0xc1, 0x5e,
+			0xa8, 0x1c, 0x99, 0x36, 0x38, 0xb8, 0x5c, 0xff, 0x3d,
+			0xa0, 0xfc, 0xb0, 0xbc, 0x3d, 0x2c, 0xb4, 0x36, 0x17,
+			0xb4, 0x6d, 0xb5, 0x39, 0x45, 0xa9, 0x2a, 0x6b, 0xa2,
+			0x24, 0x44, 0x30, 0xab, 0x2c, 0x82, 0x36, 0xdc, 0xd6,
+			0x36, 0x5d, 0x0a, 0xdc, 0xee, 0x0f, 0x2b, 0x28, 0x99,
+			0xdc, 0x67, 0x0d, 0xea, 0x6e, 0x42, 0xb9, 0x45, 0x7f,
+			0xd2, 0x96, 0x1e, 0x60, 0x42, 0xeb, 0x1e, 0x5f, 0x8e,
+			0xa9, 0xdc, 0xd3, 0x8a, 0xd6, 0xbd, 0x4e, 0x1f, 0x42,
+			0x75, 0x1d, 0xe2, 0xc6, 0x11, 0xc9, 0x80, 0x1f, 0xfe,
+			0x99, 0x52, 0x4d, 0x7b, 0x35, 0xf7, 0xb7, 0xc3, 0xee,
+			0xd6, 0x94, 0xf5, 0x74, 0xa0, 0x69, 0xcd, 0x1f, 0x2b,
+			0xd0, 0x87, 0xf7, 0x8c, 0x69, 0xc5, 0x96, 0x70, 0x91,
+			0xe8, 0x3d, 0xd2, 0xcc, 0xf1, 0x4c, 0xcd, 0xe2, 0x14,
+			0x00, 0x10, 0x4a, 0xd9, 0x6a, 0x5d, 0x65, 0x2c, 0x4b,
+			0x79, 0x0c, 0xc4, 0x78, 0x5e, 0xc8, 0xc5, 0x37, 0x74,
+			0x6d, 0x50, 0x5c, 0x34, 0x1f, 0xe0, 0xf4, 0xe3, 0xe1,
+			0x86, 0x68, 0xb1, 0xea, 0x70, 0xf0, 0xae, 0xe4, 0x59,
+			0xa1, 0x08, 0x7e, 0x35, 0xa3, 0x16, 0xd2, 0xb0, 0xa3,
+			0xd4, 0xb0, 0x74, 0x8c, 0x05, 0x79, 0x73, 0xfb, 0xe6,
+			0x65, 0x96, 0x15, 0x07, 0xd5, 0xaf, 0x88, 0x9e, 0x6b,
+			0xf0, 0xbb, 0x3f, 0xe6, 0xd1, 0x6a, 0xe7, 0xc9, 0xae,
+			0xd9, 0xb0, 0x16, 0x1c, 0x40, 0xeb, 0xdb, 0xc1, 0xbf,
+			0x83, 0xdb, 0x8a, 0x4f, 0x96, 0xca, 0xd7, 0x22, 0x06,
+			0x87, 0x08, 0x9d, 0x65, 0x2f, 0xd9, 0x8e, 0x95, 0x6c,
+			0xcc, 0xbf, 0x76, 0x2a, 0xea, 0x5c, 0x8e, 0x5b, 0x17,
+			0x0f, 0x75, 0x7b, 0xfa, 0xf9, 0xfb, 0xaa, 0x92, 0xc7,
+			0x7e, 0x63, 0x63, 0x54, 0xa4, 0xff, 0xf6, 0xc0, 0xc0,
+			0xf5, 0x70, 0xd8, 0xe3, 0xa4, 0x79, 0x16, 0xf0, 0x6f,
+			0x90, 0x5e, 0xb7, 0xab, 0x6f, 0xab, 0x75, 0x3b, 0xe1,
+			0x4c, 0xa8, 0x0b, 0x72, 0x5f, 0x5f, 0x11, 0x22, 0x36,
+			0x71, 0x20, 0xd3, 0x5b, 0x5e, 0x07, 0x06, 0x76, 0x1a,
+			0xcc, 0x5e, 0x7c, 0x97, 0x7d, 0xb2, 0x6b, 0xf8, 0x39,
+			0x89, 0x37, 0xb6, 0x6d, 0xea, 0x74, 0x57, 0x28, 0xd7,
+			0x0e, 0x9b, 0xeb, 0x28, 0x88, 0x90, 0xfd, 0x2d, 0x16,
+			0x21, 0x74, 0x26, 0xc5, 0xb8, 0x44, 0xad, 0x9f, 0x97,
+			0xf9, 0x65, 0x36, 0xd8, 0x00, 0x59, 0x17, 0x49, 0xf9,
+			0xc7, 0xb3, 0x84, 0xb9, 0xe2, 0x95, 0xe0, 0xd1, 0x7f,
+			0x5f, 0xaa, 0xd7, 0xfd, 0x6a, 0x6a, 0x83, 0x14, 0x46,
+			0x1d, 0x12, 0x8d, 0x09, 0xc3, 0xa5, 0xca, 0x72, 0xa3,
+			0x25, 0x65, 0xb6, 0x40, 0x25, 0x04, 0x51, 0xab, 0x22,
+			0xeb, 0xd7, 0x69, 0xc9, 0x22, 0x9c, 0xa0, 0x19, 0x5c,
+			0x1a, 0xfd, 0x41, 0x8f, 0x98, 0xc5, 0x71, 0xb8, 0x6f,
+			0x76, 0xae, 0xfa, 0x9b, 0x03, 0xab, 0x43, 0x81, 0x3b,
+			0x66, 0xae, 0xf0, 0xd2, 0xb7, 0xee, 0x9a, 0xe3, 0xae,
+			0x45, 0xc1, 0x86, 0xb0, 0xce, 0x9e, 0x2b, 0xec, 0xb8,
+			0xcf, 0xca, 0x0e, 0x8c, 0x33, 0xfa, 0xa7, 0xef, 0xf7,
+			0xfc, 0xa1, 0x41, 0x49, 0xd3, 0x6d, 0xb5, 0x58, 0xe4,
+			0x0e, 0x24, 0xd2, 0x8a, 0x74, 0xc9, 0x56, 0x2e, 0x53,
+			0xc7, 0x7a, 0x38, 0x0f, 0x4b, 0xd9, 0xf9, 0x2f, 0xfa,
+			0x7d, 0xee, 0x14, 0x18, 0xce, 0x75, 0x42, 0x6c, 0x03,
+			0x34, 0xce, 0x80, 0xec, 0xf2, 0x05, 0xf0, 0xdf, 0xcd,
+			0xf8, 0xdb, 0x26, 0x7d, 0xb6, 0x3d, 0x28, 0x24, 0x7e,
+			0x7e, 0x39, 0x9f, 0xa6, 0xc6, 0xeb, 0x2a, 0xc8, 0x17,
+			0x94, 0xa9, 0x89, 0xf5, 0xdf, 0xcb, 0x77, 0xfd, 0xc9,
+			0x9e, 0x68, 0x98, 0x7d, 0x04, 0x50, 0x3c, 0x64, 0x1d,
+			0x66, 0xb0, 0x97, 0x06, 0xb6, 0x08, 0x5b, 0xe4, 0x17,
+			0x44, 0xd6, 0x94, 0x39, 0x6b, 0x03, 0x2c, 0xcb, 0x5a,
+			0x8d, 0x86, 0x08, 0x23, 0x4f, 0x95, 0xa8, 0x1a,
+		},
+	.msg_len = 64,
+	.msg =
+		(const u8[64]) {
+			0x1a, 0x84, 0x21, 0x0d, 0x35, 0x7a, 0x88, 0xc8,
+			0x6a, 0x11, 0xe3, 0x15, 0x24, 0xec, 0x0d, 0x2e,
+			0x76, 0xb9, 0xcf, 0x2b, 0x04, 0x25, 0x16, 0xae,
+			0x62, 0x42, 0xa0, 0x20, 0x68, 0x25, 0x3e, 0xb4,
+			0x75, 0xa7, 0x1d, 0x64, 0xc3, 0xd1, 0x08, 0x07,
+			0x67, 0xb6, 0xf7, 0x76, 0x76, 0xf6, 0xd6, 0x62,
+			0x66, 0x04, 0x89, 0x0c, 0x8f, 0x07, 0xac, 0xc8,
+			0x51, 0x77, 0xd9, 0x47, 0x5e, 0xb5, 0x22, 0x20,
+		},
+	.sig_len = MLDSA65_SIGNATURE_SIZE,
+	.sig =
+		(const u8[MLDSA65_SIGNATURE_SIZE]) {
+			0xda, 0xcf, 0x8d, 0x67, 0x59, 0x60, 0x6c, 0x39, 0x2d,
+			0x89, 0xb6, 0xa1, 0xf3, 0x8c, 0x70, 0xcf, 0x25, 0x86,
+			0x21, 0xa1, 0x9f, 0x20, 0x9e, 0xf5, 0xd2, 0xdd, 0xbd,
+			0x99, 0xfa, 0xe4, 0xab, 0x77, 0x31, 0x65, 0x18, 0xa1,
+			0xd1, 0x3f, 0x21, 0x70, 0x36, 0xe1, 0xf9, 0x5c, 0x28,
+			0xb6, 0x7d, 0x34, 0xae, 0x66, 0xc9, 0x1c, 0x8e, 0xc6,
+			0xf9, 0x45, 0x8c, 0xa9, 0xb2, 0xfb, 0x0f, 0x5b, 0xb8,
+			0xf9, 0xf5, 0xe2, 0x37, 0x79, 0x12, 0xda, 0xa7, 0x72,
+			0x9e, 0x0d, 0xf8, 0x88, 0x5b, 0x34, 0x49, 0x6c, 0xed,
+			0xa3, 0x7f, 0x86, 0xd3, 0xd9, 0x2f, 0x44, 0x08, 0x0d,
+			0xb7, 0xdb, 0x4a, 0xce, 0x02, 0x14, 0x02, 0xd6, 0x40,
+			0x75, 0xe3, 0xc0, 0x97, 0xfc, 0x6c, 0x6a, 0x88, 0x29,
+			0x0c, 0xe2, 0x3a, 0x2b, 0x28, 0x82, 0x8f, 0x27, 0x09,
+			0x69, 0x91, 0xc6, 0xc3, 0xb7, 0x07, 0x61, 0x86, 0x8d,
+			0x89, 0x8a, 0xd5, 0x00, 0x3b, 0x4b, 0xfc, 0x6f, 0xb3,
+			0x3f, 0x4c, 0x93, 0x31, 0xfc, 0x88, 0x53, 0x26, 0xea,
+			0xe5, 0x3a, 0xfc, 0xc1, 0x59, 0x16, 0xf0, 0xb7, 0xac,
+			0xde, 0x1e, 0xd8, 0x74, 0x85, 0x72, 0xd9, 0xbb, 0xbe,
+			0x76, 0x32, 0x25, 0x9d, 0x21, 0xbc, 0xfd, 0x8d, 0x32,
+			0xfe, 0xae, 0x24, 0xe5, 0x4a, 0xcc, 0x5d, 0x15, 0x23,
+			0xd3, 0x57, 0xe7, 0xa9, 0x2c, 0x31, 0xd7, 0xc5, 0x6b,
+			0x70, 0x6c, 0x22, 0x5a, 0x13, 0x1f, 0x76, 0x13, 0x78,
+			0x6f, 0xac, 0x42, 0x4c, 0x46, 0x81, 0xa2, 0x20, 0x91,
+			0x30, 0xed, 0xcb, 0x90, 0xfe, 0x3c, 0xa3, 0xc7, 0xb4,
+			0x1f, 0x21, 0x1d, 0x98, 0x74, 0x6a, 0x3e, 0xc8, 0xcc,
+			0xd2, 0x68, 0x87, 0x69, 0xa9, 0xdf, 0x50, 0xd5, 0x0a,
+			0x8e, 0x10, 0x54, 0xab, 0xea, 0x65, 0x2a, 0x52, 0xd7,
+			0x22, 0xae, 0x2f, 0x1e, 0xc3, 0x16, 0x58, 0x20, 0x18,
+			0x6d, 0x35, 0x46, 0x31, 0x43, 0x5d, 0x62, 0xfb, 0xb1,
+			0x47, 0x32, 0xfa, 0x14, 0xcc, 0x51, 0xa3, 0xcd, 0x99,
+			0x4f, 0x97, 0x0f, 0xca, 0x24, 0x93, 0x17, 0xea, 0xa3,
+			0xf3, 0x1f, 0xbe, 0xb5, 0xa3, 0xac, 0x80, 0xcc, 0x20,
+			0x3b, 0xa6, 0xd3, 0x32, 0x72, 0x4e, 0xd9, 0x25, 0xf9,
+			0xc2, 0x24, 0x15, 0xbd, 0x1e, 0x1e, 0x41, 0x8c, 0x18,
+			0x8c, 0x58, 0xe8, 0x75, 0x20, 0xff, 0xa3, 0xf4, 0xd4,
+			0xab, 0x75, 0x78, 0x4e, 0xbb, 0x7c, 0x94, 0x93, 0x28,
+			0x5b, 0x07, 0x3a, 0x3c, 0xc9, 0xf1, 0x55, 0x3e, 0x33,
+			0xed, 0xf8, 0x72, 0x55, 0xab, 0x5a, 0xea, 0xbe, 0x65,
+			0xfa, 0x81, 0x50, 0xc0, 0x9d, 0x2d, 0xfb, 0x04, 0x25,
+			0x7c, 0xb9, 0xee, 0xe2, 0xa3, 0x00, 0x44, 0xd3, 0x9d,
+			0xee, 0x4f, 0x80, 0x77, 0xfb, 0x26, 0x6b, 0x07, 0xd0,
+			0xff, 0x82, 0x39, 0x0e, 0x2b, 0x47, 0xa3, 0xe7, 0x3e,
+			0xc5, 0x4e, 0x15, 0x8a, 0x48, 0x28, 0xfb, 0xf7, 0xa4,
+			0x86, 0xfb, 0x77, 0x60, 0xcd, 0xc5, 0x68, 0x96, 0xd7,
+			0x4c, 0x3c, 0xf2, 0x51, 0x71, 0x79, 0x2e, 0x2e, 0x57,
+			0x10, 0xa7, 0xfc, 0xd1, 0xd4, 0x61, 0x71, 0x81, 0x85,
+			0x74, 0x09, 0x7d, 0x80, 0xd0, 0xc2, 0xe9, 0xff, 0xb7,
+			0x88, 0x53, 0x74, 0x1e, 0xb0, 0xca, 0x65, 0x48, 0x8e,
+			0xdb, 0x59, 0x3a, 0xcb, 0x80, 0xeb, 0xfd, 0xd2, 0xc9,
+			0x38, 0x43, 0xae, 0x76, 0xf2, 0xbb, 0x51, 0xb2, 0xcb,
+			0xe6, 0x85, 0x31, 0xb5, 0x62, 0xd4, 0x5e, 0x48, 0x08,
+			0xf1, 0x40, 0x5b, 0x16, 0x83, 0x5e, 0xa5, 0x9c, 0x6b,
+			0x91, 0x49, 0x44, 0xff, 0x3b, 0xa9, 0x2b, 0xf3, 0x06,
+			0x33, 0x9e, 0x6e, 0x3c, 0x66, 0x7e, 0x27, 0xa2, 0x59,
+			0x7b, 0xe3, 0xb6, 0xb4, 0x28, 0xeb, 0x93, 0x35, 0x87,
+			0xac, 0x0e, 0x0b, 0x7e, 0xbc, 0x35, 0x28, 0x72, 0x1f,
+			0x26, 0x59, 0xd0, 0x1f, 0x63, 0xe4, 0x86, 0x5d, 0x70,
+			0xf3, 0xa8, 0xa4, 0xb8, 0xcd, 0xb3, 0xf8, 0x8d, 0xaa,
+			0x41, 0xd2, 0xcc, 0x0b, 0x15, 0x66, 0x22, 0x83, 0x92,
+			0xe3, 0x0b, 0xf9, 0xea, 0xa0, 0x33, 0xa1, 0x4e, 0x92,
+			0xae, 0x81, 0x95, 0xa4, 0x58, 0x3f, 0xa9, 0x15, 0x52,
+			0xf9, 0xda, 0xb7, 0x10, 0x8d, 0xc6, 0xab, 0x77, 0xe9,
+			0xbe, 0xad, 0xc9, 0x3a, 0x6a, 0x8d, 0x92, 0x6c, 0x69,
+			0xff, 0x31, 0x49, 0x25, 0x04, 0xc8, 0x93, 0x6f, 0xc8,
+			0xe7, 0x60, 0x7a, 0x76, 0xb5, 0xc1, 0x07, 0xef, 0xa3,
+			0x39, 0xa6, 0xf2, 0x36, 0x04, 0xde, 0x3c, 0x4a, 0x4e,
+			0x96, 0xbd, 0x64, 0x26, 0x80, 0x01, 0x88, 0x47, 0xd2,
+			0xa4, 0x46, 0xcd, 0xe1, 0x30, 0x7f, 0xa3, 0x00, 0x11,
+			0x38, 0x55, 0xfa, 0xeb, 0x10, 0xeb, 0xa0, 0x65, 0x04,
+			0x09, 0xc8, 0xde, 0x9c, 0x73, 0xba, 0x0c, 0xbd, 0xd3,
+			0xa5, 0x84, 0x5e, 0xb9, 0x3b, 0xd4, 0x94, 0xbd, 0xa6,
+			0x53, 0xbe, 0x93, 0x69, 0x3e, 0xaa, 0x32, 0x31, 0x06,
+			0xc8, 0x1b, 0x4a, 0x48, 0xb5, 0x17, 0x85, 0xbf, 0x72,
+			0xec, 0xf5, 0x29, 0x8a, 0xd8, 0xeb, 0x99, 0x8b, 0x74,
+			0x84, 0x57, 0x8c, 0xe1, 0x85, 0x94, 0xa0, 0xbc, 0x7a,
+			0x14, 0xf0, 0xf4, 0x8b, 0x25, 0x37, 0x43, 0xa1, 0x34,
+			0x09, 0x71, 0xca, 0x5c, 0x9f, 0x08, 0x38, 0xd9, 0x9c,
+			0x0c, 0x0e, 0xcb, 0xe4, 0xad, 0x4b, 0x2a, 0x89, 0x67,
+			0xf8, 0x29, 0x6c, 0x69, 0x0e, 0x5d, 0xca, 0xfa, 0xa6,
+			0x6b, 0x0e, 0xb5, 0x94, 0x17, 0x71, 0xf0, 0xc9, 0xcd,
+			0x02, 0x1d, 0xa5, 0xd5, 0xc6, 0xa7, 0xbc, 0x5f, 0x6e,
+			0x67, 0x43, 0x68, 0xce, 0xac, 0x54, 0x81, 0x2a, 0x25,
+			0x22, 0x52, 0x35, 0xad, 0x7b, 0xd5, 0x06, 0x8c, 0x00,
+			0xfb, 0xca, 0xc4, 0x0a, 0x49, 0x1e, 0xc8, 0xeb, 0x77,
+			0xc1, 0x63, 0x23, 0x96, 0xbd, 0x35, 0xfa, 0x13, 0xae,
+			0xbf, 0x1d, 0x1e, 0x69, 0x8d, 0xb3, 0xe3, 0x07, 0xde,
+			0x4e, 0xd0, 0x12, 0xa9, 0xc3, 0x36, 0x30, 0x46, 0xef,
+			0x92, 0x76, 0x17, 0x8f, 0x10, 0xe7, 0xba, 0x99, 0x4b,
+			0xdf, 0xad, 0xb8, 0x11, 0x80, 0xdf, 0xe7, 0xfd, 0x80,
+			0x64, 0xf7, 0x2a, 0xac, 0x60, 0x2a, 0x54, 0x8f, 0x4f,
+			0xaf, 0xaf, 0x60, 0xf9, 0x67, 0x20, 0x80, 0x53, 0x5c,
+			0xb6, 0x81, 0xa6, 0x2a, 0x74, 0x2d, 0xc5, 0x74, 0x2a,
+			0x95, 0x26, 0x13, 0x17, 0x01, 0xdd, 0x31, 0xac, 0x5a,
+			0x05, 0xda, 0xde, 0xba, 0xf6, 0x37, 0x13, 0x8d, 0xe4,
+			0xa8, 0x93, 0x46, 0x9e, 0xa9, 0x82, 0x24, 0x7e, 0xc8,
+			0xda, 0x63, 0x89, 0xcd, 0x33, 0xc9, 0xf7, 0xf9, 0x71,
+			0x35, 0xe6, 0xa5, 0x5f, 0x6b, 0x3b, 0xbb, 0x0c, 0xe0,
+			0xa4, 0x0b, 0xe3, 0x29, 0xc0, 0xae, 0x8e, 0xce, 0x03,
+			0x09, 0x73, 0x0e, 0x1e, 0x9c, 0xe9, 0x59, 0xb6, 0x8b,
+			0x78, 0x67, 0x32, 0x8b, 0xf1, 0x93, 0xcc, 0x72, 0x1b,
+			0x6f, 0xa2, 0xf1, 0x04, 0x9c, 0xfa, 0x98, 0x02, 0xca,
+			0xdf, 0x35, 0x3c, 0x38, 0xac, 0xa8, 0xdb, 0x90, 0xae,
+			0xaa, 0xf9, 0x70, 0xfb, 0xed, 0xbd, 0xa6, 0x25, 0x14,
+			0x58, 0x09, 0x8a, 0x36, 0xaf, 0x41, 0x09, 0x19, 0xcb,
+			0xd3, 0x25, 0x5d, 0x0e, 0xe6, 0x20, 0x14, 0x71, 0x24,
+			0x79, 0x19, 0x55, 0xaf, 0x51, 0x5b, 0xa4, 0xc0, 0x93,
+			0x9e, 0xdd, 0x88, 0x31, 0x13, 0x96, 0xbf, 0xca, 0x0a,
+			0xd7, 0xbc, 0xc4, 0x00, 0xa1, 0x10, 0x2d, 0x92, 0x79,
+			0xf9, 0x14, 0xdb, 0xd2, 0xba, 0x74, 0xfa, 0xa8, 0xe5,
+			0x40, 0x14, 0xc2, 0x56, 0x3c, 0x7f, 0x50, 0x07, 0x60,
+			0x86, 0x93, 0x51, 0x2e, 0xf9, 0x70, 0x61, 0x70, 0x0e,
+			0xa4, 0x87, 0x75, 0xcc, 0x6c, 0x72, 0xb7, 0x68, 0x23,
+			0xb7, 0x3d, 0x76, 0xaf, 0x96, 0x9b, 0x4a, 0xe5, 0x12,
+			0x28, 0x4a, 0x8f, 0x79, 0x34, 0xff, 0xec, 0x92, 0xeb,
+			0x6b, 0xaf, 0xc9, 0xbd, 0xc1, 0x77, 0x07, 0xd0, 0xfa,
+			0x55, 0x57, 0x10, 0x0c, 0xad, 0x29, 0x2a, 0x79, 0xd6,
+			0x09, 0x9e, 0x7d, 0x18, 0xd4, 0xd6, 0xdd, 0x72, 0x1a,
+			0x8f, 0x24, 0x11, 0x70, 0xd2, 0x52, 0x36, 0x0f, 0x38,
+			0x79, 0x38, 0x4a, 0x02, 0x4f, 0x73, 0x2a, 0xaa, 0x6a,
+			0xb5, 0x0c, 0x72, 0x32, 0x85, 0x21, 0x76, 0x1a, 0x8a,
+			0x7d, 0x51, 0x0e, 0xf1, 0xf9, 0x19, 0xfa, 0x6b, 0x9b,
+			0x22, 0x71, 0x8c, 0x13, 0xcc, 0xba, 0x7d, 0xee, 0xd8,
+			0x34, 0xf6, 0x85, 0x60, 0xe1, 0xe4, 0x59, 0x6e, 0x32,
+			0x60, 0xd9, 0xfa, 0xb7, 0x56, 0x54, 0x25, 0xd1, 0x73,
+			0x6a, 0xf2, 0xa0, 0xc7, 0xa0, 0x67, 0x10, 0x89, 0x9c,
+			0x27, 0x5f, 0x7f, 0x2e, 0x5a, 0x29, 0x70, 0x7a, 0x7b,
+			0xaf, 0x21, 0xd0, 0xf4, 0x06, 0xb9, 0x2d, 0xf1, 0xb8,
+			0x32, 0xed, 0xc5, 0xc9, 0xac, 0x2f, 0x54, 0x0a, 0xf9,
+			0x08, 0x39, 0x39, 0x7d, 0x1d, 0xaf, 0xb4, 0x5f, 0x4d,
+			0x75, 0xc3, 0xe8, 0x52, 0x3a, 0x47, 0x72, 0x2c, 0xa9,
+			0x2d, 0xcb, 0x74, 0x06, 0xfe, 0x69, 0xd3, 0xf3, 0x1a,
+			0xb2, 0xd3, 0x01, 0xed, 0x6c, 0xc1, 0xca, 0x4f, 0xaf,
+			0x11, 0x9b, 0xa2, 0x27, 0x2a, 0x59, 0x56, 0x58, 0xdf,
+			0x79, 0x8b, 0xc9, 0x87, 0xe9, 0x58, 0x81, 0x48, 0xc6,
+			0xb6, 0x7d, 0x60, 0x54, 0x87, 0x9c, 0x61, 0xbb, 0x4b,
+			0xbb, 0x61, 0xac, 0x0a, 0x5a, 0x66, 0x7e, 0x70, 0x8b,
+			0xfd, 0x92, 0x76, 0x4a, 0xa9, 0xa5, 0xc3, 0xf4, 0xf2,
+			0x93, 0x48, 0xc4, 0xf3, 0x91, 0x2b, 0x60, 0x04, 0x0e,
+			0xb0, 0x6b, 0x60, 0x5e, 0xf0, 0xf1, 0x54, 0x41, 0x56,
+			0xdc, 0x25, 0x57, 0xc3, 0xb6, 0x0b, 0x5e, 0x15, 0xb5,
+			0x2a, 0x36, 0x4f, 0xe7, 0x1d, 0x70, 0xa8, 0xa7, 0xec,
+			0xd6, 0x74, 0xba, 0xa4, 0x79, 0x83, 0x7c, 0x9e, 0x1a,
+			0x5d, 0x32, 0xc8, 0xcb, 0x41, 0xca, 0x04, 0xec, 0x0b,
+			0x18, 0x54, 0xe1, 0x67, 0xbf, 0xa8, 0x7a, 0xc3, 0x0f,
+			0x27, 0x2a, 0xaf, 0x2a, 0x41, 0x19, 0x1f, 0xe8, 0xa2,
+			0xe8, 0xfa, 0xfc, 0x88, 0x41, 0x46, 0xc3, 0x1c, 0x44,
+			0xe5, 0xee, 0x47, 0xec, 0xfe, 0xbf, 0xb8, 0x29, 0x2e,
+			0xae, 0x47, 0x0a, 0x42, 0x69, 0x8a, 0x9a, 0x94, 0x97,
+			0x9e, 0xf5, 0xb6, 0x37, 0x1c, 0x10, 0xc2, 0x99, 0xa8,
+			0xe9, 0x9e, 0x0e, 0x6e, 0xb5, 0xbe, 0xba, 0x1f, 0x77,
+			0xa6, 0x35, 0x02, 0x1e, 0x8c, 0xe6, 0x02, 0x53, 0xe2,
+			0x9a, 0xdd, 0x09, 0x6e, 0x9b, 0x7a, 0x36, 0x4f, 0x38,
+			0x8d, 0x4c, 0xa4, 0xb4, 0xff, 0x90, 0x76, 0x0d, 0x11,
+			0x7d, 0xe1, 0xe9, 0x7f, 0x2a, 0x4a, 0x80, 0xe0, 0xd8,
+			0x3c, 0x23, 0xd2, 0xa5, 0xe5, 0x39, 0x77, 0xbf, 0x3d,
+			0x71, 0x0d, 0x45, 0xbb, 0x39, 0x66, 0x1a, 0x4d, 0x59,
+			0xb7, 0xd0, 0x0a, 0xee, 0x87, 0xee, 0x1f, 0xcf, 0x6f,
+			0xc2, 0x50, 0xb1, 0xa5, 0x4c, 0xee, 0x40, 0x69, 0xd7,
+			0x36, 0x38, 0x14, 0xcd, 0x6a, 0x9a, 0x90, 0x40, 0xad,
+			0x76, 0xf1, 0xa6, 0xd4, 0x3c, 0x75, 0x10, 0xba, 0xcb,
+			0xab, 0x22, 0x28, 0x5f, 0x0c, 0xe0, 0xee, 0xf4, 0xfd,
+			0x61, 0x52, 0x0a, 0x59, 0xfe, 0x61, 0xc5, 0x40, 0xf9,
+			0x91, 0x8e, 0x36, 0x29, 0x63, 0x6c, 0x6e, 0x45, 0xa5,
+			0x42, 0xe3, 0x36, 0x90, 0xe7, 0x90, 0x9f, 0x58, 0xbb,
+			0xf9, 0x1b, 0xee, 0x2c, 0xbb, 0x3a, 0xfd, 0x3d, 0xbe,
+			0x3d, 0x45, 0xf0, 0xc2, 0x18, 0xaa, 0x46, 0x10, 0x23,
+			0xe9, 0x63, 0xba, 0x7f, 0xc2, 0xe1, 0xf4, 0x05, 0xdd,
+			0x4a, 0x7c, 0xa8, 0xab, 0xa9, 0xbd, 0x6f, 0xdf, 0x48,
+			0x59, 0x11, 0xd4, 0xba, 0x75, 0xb6, 0x22, 0xd4, 0xd7,
+			0x35, 0x6f, 0x27, 0x70, 0xc7, 0x3d, 0x90, 0x06, 0x39,
+			0x2a, 0x16, 0xd0, 0x8b, 0xd7, 0xfb, 0x5e, 0x85, 0x2e,
+			0xb0, 0xd8, 0xc7, 0xdb, 0xe5, 0x24, 0x3a, 0x6e, 0xc4,
+			0x5e, 0xd4, 0x22, 0x25, 0x14, 0xee, 0xa5, 0x30, 0x8b,
+			0xd6, 0x27, 0x61, 0x33, 0x13, 0x46, 0x0b, 0x26, 0x45,
+			0xa6, 0xb4, 0xfa, 0x8d, 0xa3, 0xf2, 0x27, 0xd2, 0xc5,
+			0x04, 0xaa, 0x96, 0xa4, 0x55, 0xfa, 0x40, 0xf1, 0xfc,
+			0x66, 0x33, 0x9e, 0x4b, 0x39, 0x75, 0xae, 0x7f, 0x52,
+			0x87, 0x7b, 0x8a, 0xf9, 0x7d, 0x5f, 0x8a, 0x7e, 0xf7,
+			0xfe, 0xc4, 0x7f, 0xf4, 0xf6, 0x9a, 0x86, 0x78, 0x21,
+			0x02, 0x94, 0x9e, 0x50, 0x2d, 0xdc, 0xd6, 0xa5, 0x53,
+			0xf1, 0xef, 0x06, 0xe8, 0xb5, 0x46, 0x81, 0xcc, 0x91,
+			0x4f, 0x37, 0xee, 0x27, 0xcb, 0x91, 0xad, 0xff, 0x1d,
+			0xd1, 0x00, 0xa8, 0x96, 0x22, 0xaa, 0x63, 0x23, 0x2a,
+			0x7a, 0x75, 0x6f, 0xe9, 0x2d, 0x26, 0xde, 0x11, 0x97,
+			0x4b, 0x17, 0x3f, 0xde, 0x51, 0x1a, 0x22, 0xed, 0x38,
+			0x6f, 0x3e, 0x7a, 0xd0, 0xd6, 0x60, 0x06, 0x7e, 0x3f,
+			0xa4, 0x29, 0xfa, 0x18, 0x91, 0xda, 0x73, 0x38, 0xe3,
+			0xe3, 0xb5, 0xc0, 0x5b, 0x4e, 0xe8, 0x94, 0xea, 0x45,
+			0x6e, 0x5b, 0x50, 0xaa, 0x38, 0xb6, 0x6f, 0xdb, 0x90,
+			0x1b, 0x3b, 0x82, 0xbb, 0x0d, 0x38, 0xe3, 0xca, 0xd9,
+			0xf1, 0x2e, 0x27, 0x4c, 0x2c, 0x5a, 0x42, 0xdf, 0x44,
+			0xc8, 0x07, 0xe4, 0x95, 0xb5, 0xec, 0x91, 0x34, 0x1c,
+			0x9a, 0x0c, 0x50, 0x1a, 0xce, 0x67, 0xe4, 0x4b, 0x87,
+			0x61, 0x43, 0x95, 0x95, 0xb8, 0x8a, 0xf4, 0xc9, 0x92,
+			0x33, 0x33, 0xe3, 0xfe, 0x98, 0x2a, 0xae, 0x8e, 0xf2,
+			0x6b, 0x13, 0x7c, 0xe4, 0x44, 0x40, 0x66, 0xea, 0x0c,
+			0xe4, 0xdb, 0x16, 0x65, 0xa8, 0x8b, 0x37, 0x08, 0xec,
+			0x1e, 0xfc, 0xa6, 0xd0, 0x9b, 0x9e, 0x0a, 0xd2, 0xe3,
+			0xcf, 0x5d, 0xb2, 0xaf, 0x8e, 0x05, 0x7d, 0x8d, 0x84,
+			0xbc, 0x9f, 0xb1, 0xe6, 0x6a, 0x2e, 0x4b, 0x6d, 0x64,
+			0x91, 0x17, 0x9d, 0xb5, 0x35, 0x15, 0x02, 0xe9, 0x1b,
+			0x85, 0xc1, 0x89, 0xc2, 0x5a, 0x32, 0x3a, 0x80, 0x78,
+			0x5e, 0xcc, 0x50, 0x26, 0xf5, 0x11, 0x01, 0x79, 0xf3,
+			0xaf, 0xb6, 0x40, 0x00, 0x73, 0x8f, 0xeb, 0x5a, 0xd1,
+			0x26, 0x00, 0xe2, 0xa3, 0xcd, 0xfd, 0xaa, 0x15, 0x5b,
+			0x98, 0x2a, 0x76, 0x41, 0x07, 0xc2, 0xde, 0xb6, 0x71,
+			0xe7, 0xc3, 0xe9, 0x92, 0xb3, 0xd8, 0xfe, 0xaf, 0x12,
+			0x61, 0x86, 0x5b, 0x6e, 0x74, 0x45, 0x7b, 0x9b, 0x6f,
+			0x1a, 0x13, 0x84, 0xf6, 0x31, 0x5f, 0x5b, 0x6c, 0xde,
+			0x47, 0xb8, 0x73, 0x32, 0xc7, 0x94, 0x92, 0xa5, 0xc3,
+			0x65, 0xdf, 0x96, 0x6c, 0xfd, 0xb7, 0x80, 0xfb, 0x47,
+			0xba, 0x6e, 0x43, 0xb3, 0x7e, 0x86, 0xc9, 0x97, 0x45,
+			0xde, 0x3f, 0x3a, 0xf6, 0xb0, 0x9e, 0x9a, 0xcb, 0xfd,
+			0xf2, 0x5c, 0xba, 0x6e, 0x3f, 0xed, 0xfa, 0x74, 0x84,
+			0xe2, 0xb1, 0xae, 0x66, 0x57, 0x0b, 0x96, 0x6c, 0x77,
+			0xe4, 0x8a, 0x67, 0x97, 0xc7, 0xe0, 0x44, 0xb2, 0x83,
+			0x2d, 0x3c, 0x2e, 0x01, 0x19, 0x2e, 0x4c, 0x74, 0xe1,
+			0x35, 0x73, 0xeb, 0x85, 0x63, 0x8c, 0x3a, 0xb8, 0xbc,
+			0x25, 0x6a, 0x8d, 0xaf, 0xd2, 0xfb, 0xef, 0xd3, 0x12,
+			0x93, 0x0b, 0x39, 0xfa, 0x66, 0xbe, 0x3b, 0xfd, 0x6c,
+			0x0b, 0xbb, 0xb2, 0x5a, 0x78, 0xa1, 0xcf, 0x8c, 0x7d,
+			0x60, 0x55, 0xeb, 0x33, 0x4e, 0x8e, 0xf9, 0x19, 0x4d,
+			0x42, 0xd4, 0xf8, 0xd8, 0xba, 0xad, 0x0a, 0x6e, 0x62,
+			0xd4, 0xe1, 0x6a, 0xcc, 0xea, 0x09, 0x91, 0x8e, 0x62,
+			0xc9, 0x1e, 0x9e, 0x48, 0xaa, 0xde, 0xf7, 0xa2, 0x5a,
+			0xcb, 0x83, 0x20, 0xe8, 0xf5, 0xd1, 0xfe, 0x9d, 0x18,
+			0x2f, 0xd6, 0xf8, 0x97, 0x17, 0xce, 0xc2, 0x05, 0x08,
+			0xef, 0x61, 0x70, 0x9d, 0x95, 0x79, 0x59, 0x4c, 0x06,
+			0x24, 0x3d, 0x24, 0x69, 0xff, 0x46, 0xda, 0xbc, 0x71,
+			0x7a, 0x74, 0x93, 0x58, 0xf5, 0xc8, 0x91, 0xfb, 0x66,
+			0xed, 0x78, 0x8f, 0xf8, 0x28, 0xa8, 0x1d, 0xa5, 0x3a,
+			0x13, 0x76, 0xc2, 0xcc, 0xba, 0xb9, 0x56, 0x29, 0x74,
+			0xd6, 0x14, 0x75, 0x58, 0xe6, 0x2e, 0x79, 0x6e, 0x9d,
+			0x41, 0x94, 0x8a, 0xcf, 0xf1, 0xb1, 0xe0, 0x36, 0xe5,
+			0x89, 0x9a, 0x95, 0xa1, 0x11, 0xd1, 0xbe, 0x45, 0xe4,
+			0xb3, 0xb0, 0x62, 0x32, 0x1d, 0xba, 0xe0, 0xde, 0x57,
+			0x81, 0x0e, 0x01, 0x9b, 0x52, 0x3d, 0xd5, 0xde, 0x3b,
+			0x3a, 0xdd, 0x8f, 0xe3, 0x2e, 0xce, 0x1e, 0x89, 0x4d,
+			0x81, 0xf0, 0xf6, 0x20, 0x63, 0x7a, 0x4c, 0xbb, 0x66,
+			0xe0, 0xbe, 0x2b, 0xee, 0xd0, 0x3b, 0x60, 0x1e, 0x65,
+			0xd1, 0x2c, 0x7c, 0x5c, 0x6c, 0x16, 0x5b, 0x90, 0xc8,
+			0x05, 0x10, 0xf2, 0xde, 0x33, 0x90, 0x35, 0x69, 0x24,
+			0x3f, 0xc1, 0x8f, 0x1e, 0x4a, 0x60, 0xf1, 0x03, 0x65,
+			0x46, 0x40, 0x76, 0xe9, 0x83, 0x97, 0xda, 0x0b, 0xb8,
+			0x22, 0xfa, 0x55, 0x99, 0xfd, 0x18, 0x24, 0xd2, 0x66,
+			0xb0, 0x7b, 0x70, 0x56, 0x93, 0xad, 0x09, 0x95, 0x8e,
+			0x1f, 0x2f, 0xe8, 0x12, 0x55, 0xd4, 0x1f, 0xde, 0x09,
+			0x85, 0x05, 0xd1, 0xd5, 0x10, 0x2c, 0x8c, 0x6b, 0x53,
+			0x28, 0xce, 0x06, 0xc5, 0x52, 0x0f, 0xfa, 0x09, 0x09,
+			0x23, 0x1b, 0xe3, 0xbf, 0xb1, 0x89, 0x72, 0x26, 0x0d,
+			0xa6, 0xbb, 0x7d, 0x9e, 0xdc, 0xf8, 0xf5, 0x0b, 0x8c,
+			0xe0, 0xbc, 0x97, 0x3b, 0x72, 0xdd, 0xf5, 0x9d, 0xc5,
+			0xb6, 0x37, 0x2c, 0x76, 0x5b, 0x58, 0x67, 0xdb, 0xed,
+			0x3b, 0x6e, 0xe5, 0xe5, 0x6d, 0x6f, 0x0d, 0x7e, 0xff,
+			0xa9, 0x57, 0x4a, 0x84, 0x85, 0x82, 0xac, 0x00, 0x50,
+			0xa3, 0x4f, 0x87, 0xfe, 0x2a, 0x40, 0x52, 0x54, 0x81,
+			0x69, 0x42, 0x0b, 0x0c, 0xd7, 0x18, 0x98, 0x01, 0x8c,
+			0x5a, 0xa2, 0xf4, 0xe8, 0x61, 0xd1, 0x38, 0xfd, 0x0f,
+			0x63, 0x75, 0xd3, 0x4b, 0x1d, 0xdc, 0xdf, 0xb2, 0xeb,
+			0x94, 0x97, 0x5c, 0x2a, 0xb4, 0x12, 0x5c, 0x49, 0x2b,
+			0xfc, 0xd0, 0x8d, 0xfb, 0xe7, 0xb3, 0xcb, 0x0f, 0x3c,
+			0x2e, 0x04, 0x36, 0xa8, 0x03, 0xc9, 0xd7, 0x11, 0x2d,
+			0x2a, 0x93, 0xff, 0xda, 0x26, 0xb0, 0x54, 0x7e, 0xaf,
+			0x30, 0x7d, 0xce, 0x46, 0x8a, 0x3d, 0x7c, 0xa4, 0x7a,
+			0x2c, 0xfa, 0xba, 0xa1, 0xc9, 0x41, 0xd3, 0xb8, 0x84,
+			0x03, 0x78, 0xdd, 0xe9, 0x57, 0x19, 0x62, 0x62, 0xff,
+			0x5b, 0x3b, 0x48, 0x62, 0x0e, 0xee, 0x19, 0xb0, 0x32,
+			0x6e, 0x6a, 0x07, 0xd8, 0x4e, 0x25, 0x76, 0xa7, 0xe3,
+			0x98, 0xa1, 0x6f, 0xb6, 0x99, 0x32, 0x67, 0x7d, 0x46,
+			0x42, 0x4a, 0x82, 0xd1, 0x29, 0x1b, 0x87, 0xeb, 0x4b,
+			0x9e, 0xdf, 0x69, 0x75, 0xbd, 0x4f, 0xd3, 0xde, 0xc9,
+			0x83, 0xe6, 0xd6, 0xea, 0x03, 0x81, 0x12, 0xf3, 0x5d,
+			0x99, 0xf1, 0xb1, 0xd9, 0x3e, 0xbe, 0xf3, 0xa8, 0xdc,
+			0xb6, 0xf8, 0x4b, 0x9e, 0x26, 0x3f, 0xf0, 0x7c, 0xb3,
+			0xf4, 0xca, 0x00, 0x6c, 0x6c, 0xe5, 0x43, 0xa1, 0xfd,
+			0x3a, 0xf8, 0x8e, 0xe3, 0x9f, 0x88, 0xc5, 0x44, 0xfd,
+			0x24, 0x69, 0x76, 0xd5, 0xcb, 0xdc, 0x9d, 0x12, 0xf3,
+			0x13, 0x7e, 0xe7, 0xc3, 0xa8, 0x6a, 0xb2, 0xe0, 0xb3,
+			0x1d, 0xab, 0x3b, 0xc9, 0x77, 0x3d, 0x0f, 0xc3, 0xbe,
+			0x4b, 0x8b, 0x28, 0xbd, 0x7c, 0xe6, 0xb2, 0x06, 0x1f,
+			0xf9, 0x8f, 0x16, 0x62, 0xbf, 0xc7, 0x55, 0x73, 0xd4,
+			0xf1, 0x5a, 0x95, 0x80, 0xa3, 0x4e, 0xaa, 0x60, 0x17,
+			0x3c, 0xc9, 0x5e, 0xd4, 0x0c, 0x56, 0x7a, 0x77, 0x8e,
+			0x7f, 0x67, 0x08, 0x2f, 0xd9, 0x21, 0x19, 0xfd, 0x86,
+			0x8c, 0x23, 0x8d, 0xf6, 0x92, 0x1f, 0x36, 0x2c, 0x7c,
+			0x83, 0xbd, 0x2f, 0x6c, 0x63, 0x7c, 0xb7, 0x93, 0x74,
+			0x1b, 0xc2, 0x95, 0x34, 0x26, 0x1e, 0x07, 0x87, 0x3a,
+			0xb6, 0xe2, 0x39, 0x71, 0x9b, 0x20, 0xcd, 0x63, 0xf0,
+			0xbf, 0x48, 0xb5, 0x0e, 0x49, 0x86, 0x50, 0x80, 0xbd,
+			0xd6, 0x0e, 0xab, 0xd5, 0x69, 0x1b, 0xa4, 0xb3, 0x63,
+			0x3c, 0x8f, 0xcb, 0x42, 0xdb, 0xd7, 0x1a, 0xf4, 0xdf,
+			0x9e, 0x25, 0xfc, 0xd4, 0x00, 0xcb, 0xec, 0x57, 0x69,
+			0x30, 0x15, 0x4d, 0x7a, 0x69, 0x28, 0x2f, 0x2b, 0x34,
+			0x26, 0xd1, 0xe7, 0x01, 0x42, 0x5e, 0x02, 0xe2, 0x75,
+			0xe8, 0x52, 0x8a, 0xb4, 0x71, 0xfa, 0xc3, 0x3d, 0xe6,
+			0xac, 0xeb, 0xf3, 0x93, 0xe0, 0x37, 0xcd, 0x66, 0x92,
+			0x66, 0x2c, 0xfe, 0x4b, 0xd6, 0x3c, 0xf1, 0x57, 0xe5,
+			0xcf, 0xf5, 0xd0, 0xdb, 0x0e, 0x1f, 0x82, 0x65, 0x3b,
+			0xab, 0x69, 0x42, 0x53, 0x7d, 0xa4, 0x7c, 0xb7, 0x86,
+			0xeb, 0x23, 0x45, 0xa8, 0x4a, 0x73, 0xfc, 0x38, 0xc6,
+			0xe5, 0x2c, 0xab, 0x80, 0xfb, 0x23, 0xb2, 0x0c, 0x53,
+			0x28, 0x21, 0x37, 0x54, 0x9c, 0x72, 0x51, 0x0f, 0x44,
+			0x50, 0xd3, 0xe1, 0xd5, 0xb2, 0x27, 0x83, 0xb6, 0xe9,
+			0x4d, 0x64, 0x5c, 0x17, 0x0f, 0xe0, 0x13, 0xe4, 0x26,
+			0x6b, 0xd0, 0xd8, 0x25, 0xe3, 0x69, 0x6a, 0x95, 0x3f,
+			0x4a, 0x4e, 0xa0, 0x58, 0xbc, 0x28, 0x47, 0x8b, 0x68,
+			0xe4, 0x41, 0x90, 0x46, 0x1b, 0x84, 0xa0, 0x7b, 0x46,
+			0x46, 0x03, 0xee, 0x21, 0x0d, 0x34, 0xed, 0xff, 0x15,
+			0x57, 0x06, 0xdf, 0x71, 0x09, 0xb2, 0x66, 0x0d, 0x6e,
+			0xcc, 0xa5, 0x0c, 0xaf, 0x3f, 0x24, 0x8f, 0xd1, 0xc8,
+			0x44, 0x86, 0xaf, 0xbf, 0xeb, 0x2f, 0xb9, 0xee, 0xa7,
+			0xcf, 0xe4, 0xe8, 0xec, 0x47, 0x09, 0xd8, 0x95, 0x9e,
+			0x3c, 0xda, 0x92, 0x41, 0x61, 0xf5, 0xc3, 0xec, 0x00,
+			0xe4, 0xa3, 0x0d, 0x4a, 0xb3, 0xf6, 0x82, 0x05, 0x38,
+			0x70, 0x6a, 0xd1, 0x28, 0x2c, 0xb3, 0xc6, 0xbb, 0x38,
+			0xb3, 0x06, 0x7f, 0xd6, 0x4c, 0xe7, 0xfb, 0xef, 0x0d,
+			0x52, 0x66, 0xbe, 0xd8, 0xa6, 0x6f, 0xe8, 0xd9, 0x42,
+			0x4f, 0xad, 0xe8, 0xe8, 0x6c, 0xf9, 0xe9, 0x42, 0xd9,
+			0x66, 0x6e, 0xec, 0xfe, 0xf5, 0x91, 0xbf, 0x0a, 0x98,
+			0xd8, 0x7b, 0x23, 0x12, 0xa6, 0x04, 0xa8, 0xb3, 0x61,
+			0x13, 0x65, 0xc0, 0xe2, 0x82, 0xb9, 0xb2, 0x38, 0x07,
+			0x06, 0xca, 0x64, 0x6c, 0x23, 0x93, 0x60, 0x1d, 0x4d,
+			0x38, 0x5e, 0x8e, 0x90, 0x16, 0x4a, 0xfd, 0xb3, 0xcd,
+			0x84, 0x9c, 0xa5, 0xfa, 0x73, 0x2d, 0xcb, 0x87, 0x31,
+			0x3d, 0xf8, 0xfc, 0xeb, 0xa7, 0x56, 0x2f, 0x5b, 0x95,
+			0x9a, 0xc6, 0x82, 0x29, 0x86, 0x47, 0xe2, 0xc2, 0x84,
+			0x01, 0xaf, 0xc8, 0x0b, 0x2d, 0xfb, 0x34, 0xba, 0x5d,
+			0x9d, 0xd1, 0x85, 0xd5, 0x1e, 0x63, 0xcb, 0x3c, 0xa8,
+			0xfa, 0x79, 0xef, 0x12, 0xa6, 0xb5, 0xdb, 0xc5, 0x1d,
+			0x6a, 0xa7, 0x54, 0x58, 0x0c, 0xbe, 0x61, 0xe5, 0x96,
+			0x7f, 0x4a, 0x3b, 0x59, 0x32, 0x2d, 0x06, 0x44, 0x83,
+			0x5c, 0xad, 0xe9, 0xfe, 0x7c, 0xd7, 0x5b, 0x34, 0xa1,
+			0xa3, 0xad, 0x9a, 0xbf, 0xd5, 0x30, 0xf0, 0x22, 0xfc,
+			0x94, 0x7f, 0xd4, 0xa4, 0xca, 0x88, 0x31, 0xe7, 0xf2,
+			0x89, 0x2d, 0xda, 0xe6, 0x91, 0xa6, 0x27, 0x22, 0x74,
+			0x9f, 0xc6, 0x72, 0x4f, 0xf6, 0xa9, 0xfe, 0x7a, 0xf0,
+			0xa8, 0x6b, 0x6c, 0x9f, 0xe9, 0x2a, 0x9b, 0x23, 0x9e,
+			0xb8, 0x2b, 0x29, 0x65, 0xa7, 0x5d, 0xbd, 0x10, 0xe4,
+			0x56, 0x02, 0x94, 0xdd, 0xd1, 0xab, 0x9b, 0x82, 0x2d,
+			0x8d, 0xf6, 0xd3, 0x65, 0x63, 0x4a, 0xc4, 0x86, 0x61,
+			0x37, 0x9f, 0xdb, 0x4b, 0x34, 0x20, 0x0a, 0xca, 0x45,
+			0x6c, 0x06, 0xc4, 0x9c, 0x74, 0x4d, 0x83, 0x6a, 0x8d,
+			0xad, 0xc6, 0x61, 0x3a, 0x8d, 0xde, 0x6c, 0xf9, 0x8e,
+			0x33, 0xa2, 0xee, 0x99, 0xc7, 0xe4, 0x52, 0xb2, 0x44,
+			0x6f, 0x2f, 0x0f, 0x41, 0xa9, 0x1a, 0xd3, 0x96, 0x42,
+			0xc6, 0x49, 0x12, 0x6a, 0xf0, 0x29, 0xa9, 0x0c, 0x9c,
+			0x50, 0x5d, 0x1d, 0xd1, 0x42, 0x7e, 0x6f, 0x36, 0x48,
+			0x0f, 0x58, 0x14, 0x94, 0xc0, 0x10, 0x1e, 0xe0, 0xb2,
+			0xdd, 0xba, 0x57, 0x91, 0x4d, 0xd5, 0xdc, 0xa6, 0x4c,
+			0x68, 0x00, 0x6c, 0xb3, 0x5d, 0x32, 0x13, 0xbe, 0xa8,
+			0xc3, 0xfb, 0xd4, 0x19, 0x40, 0xf5, 0x6f, 0x63, 0xa1,
+			0x07, 0xbf, 0xa2, 0x8b, 0xfc, 0xfe, 0xf8, 0xa1, 0x33,
+			0x70, 0x07, 0x6d, 0xc5, 0x72, 0xa0, 0x39, 0xd6, 0xd7,
+			0x76, 0x6c, 0xfa, 0x1f, 0x04, 0xd6, 0x23, 0xbf, 0x66,
+			0x78, 0x92, 0x00, 0x11, 0x8a, 0x75, 0x67, 0x44, 0xa6,
+			0x7c, 0xd0, 0x14, 0xe6, 0xd0, 0x31, 0x6d, 0xdb, 0xc5,
+			0xb1, 0xa7, 0x99, 0xc3, 0xaf, 0x18, 0x7a, 0x26, 0x46,
+			0xad, 0x6d, 0x0c, 0xb6, 0xb5, 0xad, 0xc1, 0xcf, 0x60,
+			0x99, 0xf5, 0x9f, 0x88, 0xaf, 0x0e, 0x37, 0x15, 0xf9,
+			0x2b, 0x1a, 0x5f, 0xfb, 0xc9, 0xf8, 0xd4, 0xf0, 0x97,
+			0xd2, 0x91, 0xf4, 0x94, 0xa2, 0xd3, 0x3b, 0x8b, 0x0c,
+			0x22, 0xa0, 0xac, 0xb3, 0xb5, 0xdf, 0xf2, 0x27, 0x38,
+			0x47, 0x53, 0x5b, 0x6e, 0x8f, 0x98, 0x9e, 0xad, 0xb6,
+			0xf5, 0x0e, 0x17, 0x20, 0x35, 0x54, 0x6b, 0x73, 0xa6,
+			0x64, 0x65, 0xac, 0xb8, 0xc1, 0xd3, 0xf7, 0x07, 0x82,
+			0x93, 0x9d, 0xcb, 0xcc, 0xe9, 0x0c, 0x51, 0x52, 0x85,
+			0x8b, 0x95, 0xa6, 0xb1, 0xce, 0xdc, 0xfa, 0x00, 0x00,
+			0x08, 0x14, 0x1c, 0x23, 0x2a, 0x35,
+		},
+};
+
+static const struct mldsa_testvector mldsa87_testvector = {
+	.alg = MLDSA87,
+	.pk_len = MLDSA87_PUBLIC_KEY_SIZE,
+	.pk =
+		(const u8[MLDSA87_PUBLIC_KEY_SIZE]) {
+			0xd4, 0x9d, 0xdc, 0x3d, 0xa4, 0xa5, 0x87, 0xa5, 0x54,
+			0x61, 0xf3, 0xf4, 0xe0, 0x11, 0xc9, 0x1c, 0x78, 0x0a,
+			0xf1, 0x8a, 0xa8, 0xb2, 0xff, 0xb2, 0x9a, 0x2c, 0xe0,
+			0x86, 0x5c, 0xaa, 0x86, 0xe0, 0xd9, 0x42, 0x54, 0x18,
+			0x3e, 0x4c, 0x96, 0x1a, 0xb4, 0xc7, 0x18, 0xcf, 0x7d,
+			0xca, 0xe2, 0x74, 0x6c, 0x81, 0x3e, 0xcb, 0xf8, 0x7b,
+			0xc4, 0x90, 0x50, 0xd5, 0xe8, 0xd3, 0xbc, 0x8b, 0xa8,
+			0x3e, 0xb0, 0x96, 0x65, 0xd7, 0xbb, 0xa9, 0xab, 0x9c,
+			0x82, 0x5e, 0x6e, 0x8d, 0xf8, 0xc3, 0x6d, 0xe9, 0xbf,
+			0xbd, 0x30, 0xc9, 0xca, 0x47, 0x85, 0xae, 0x6f, 0x5d,
+			0x09, 0x4d, 0xd7, 0xdd, 0x05, 0x51, 0xe0, 0x9e, 0x94,
+			0x3b, 0x1d, 0xfa, 0x30, 0x57, 0xdc, 0x58, 0x48, 0xe5,
+			0x45, 0xf3, 0x34, 0x8a, 0x73, 0x66, 0x8a, 0xe0, 0x62,
+			0x3d, 0x0d, 0xfb, 0x13, 0x52, 0x25, 0xf5, 0xd3, 0x94,
+			0x15, 0x9d, 0xac, 0x6a, 0x74, 0x8a, 0x64, 0x91, 0x5b,
+			0xa3, 0xe2, 0xd0, 0x5c, 0xd2, 0xee, 0x52, 0xc0, 0x0c,
+			0x6c, 0x81, 0x34, 0x94, 0xfb, 0x87, 0xf6, 0x6b, 0x0f,
+			0x7e, 0x99, 0xa7, 0xaf, 0xb3, 0x74, 0xa7, 0xb6, 0x64,
+			0xd7, 0x36, 0x39, 0x3f, 0x7d, 0x0e, 0xc3, 0x76, 0xb3,
+			0x3a, 0xc9, 0x94, 0x6f, 0xc0, 0xaa, 0x92, 0x2a, 0xf1,
+			0x38, 0x8c, 0x62, 0xa1, 0x9a, 0xbe, 0x8d, 0x32, 0xec,
+			0x05, 0xb9, 0x8d, 0xb0, 0xdb, 0x37, 0x8d, 0x61, 0xfa,
+			0x79, 0x55, 0x1d, 0xf2, 0xc3, 0x19, 0x15, 0x3b, 0x26,
+			0xef, 0xe5, 0xf3, 0x29, 0xa1, 0x13, 0x60, 0x68, 0xd3,
+			0x22, 0x57, 0xcd, 0x99, 0xe6, 0x22, 0x54, 0xdf, 0x02,
+			0xe4, 0xd5, 0xd2, 0x66, 0xce, 0x5a, 0x90, 0x13, 0x7e,
+			0x7c, 0xad, 0x4c, 0x4c, 0xb6, 0x93, 0x65, 0xda, 0x4b,
+			0xb6, 0x13, 0x0e, 0x1b, 0x36, 0xaa, 0x2c, 0xbd, 0x60,
+			0x57, 0x98, 0x35, 0xa9, 0xe9, 0xad, 0xc5, 0x8e, 0x33,
+			0x47, 0x00, 0xe9, 0xf2, 0x1b, 0xe7, 0x5a, 0xd0, 0x55,
+			0x3b, 0x0e, 0x77, 0x09, 0x7e, 0x1f, 0x11, 0x0b, 0xf0,
+			0xe4, 0x07, 0x27, 0xe7, 0x72, 0xe9, 0x50, 0x9c, 0x9c,
+			0x84, 0xb0, 0x5c, 0xe0, 0x13, 0x79, 0xfa, 0x3f, 0x80,
+			0x96, 0xe2, 0x04, 0xde, 0x35, 0x5f, 0xa4, 0x4e, 0x8a,
+			0x12, 0x7a, 0xba, 0x53, 0x20, 0x36, 0x3d, 0xf0, 0x63,
+			0xb6, 0x14, 0xab, 0xce, 0x6b, 0x24, 0xfd, 0xc9, 0xd2,
+			0x52, 0xd9, 0xc0, 0x40, 0xdf, 0xdd, 0xd5, 0xc2, 0xa0,
+			0xf5, 0x74, 0x46, 0x17, 0xdc, 0xf5, 0x81, 0x68, 0xbd,
+			0x1a, 0x01, 0x33, 0xd0, 0x2f, 0xdc, 0x3a, 0x43, 0x4a,
+			0x08, 0x07, 0x98, 0x9f, 0x75, 0x5d, 0x70, 0x15, 0x1d,
+			0x58, 0x7f, 0x26, 0x30, 0x28, 0xe1, 0x17, 0x6d, 0x14,
+			0x0f, 0x01, 0x4a, 0x88, 0x3b, 0xf5, 0x70, 0x1d, 0x97,
+			0xd5, 0xda, 0x18, 0xa8, 0xb3, 0xa4, 0x0e, 0x04, 0x3f,
+			0xe8, 0x50, 0xf4, 0x87, 0x65, 0x75, 0x5a, 0x5d, 0x2f,
+			0x94, 0x63, 0x7a, 0xd0, 0x06, 0xce, 0xfb, 0xeb, 0x41,
+			0x65, 0xc9, 0x55, 0x38, 0x54, 0xd3, 0xc4, 0xac, 0x24,
+			0x46, 0x06, 0x2a, 0x87, 0xb1, 0x0f, 0x06, 0xd8, 0x1e,
+			0xa7, 0x35, 0xc4, 0xa3, 0xdc, 0x60, 0x80, 0x83, 0xe7,
+			0xf0, 0x74, 0xf6, 0xd6, 0xa7, 0x1d, 0x50, 0xff, 0xba,
+			0x82, 0xc0, 0xca, 0x72, 0x6b, 0xda, 0x4a, 0xcb, 0x3b,
+			0xe8, 0xa3, 0xaa, 0x32, 0x86, 0xc0, 0x94, 0x75, 0x2b,
+			0x2f, 0x44, 0xad, 0x5b, 0x8d, 0xab, 0xc2, 0x03, 0x55,
+			0xb1, 0x7e, 0x67, 0x07, 0x39, 0x33, 0x83, 0x67, 0xec,
+			0xbf, 0x52, 0xad, 0x55, 0x37, 0x3e, 0xa1, 0x41, 0xed,
+			0xa0, 0x91, 0xbf, 0x28, 0x1b, 0x04, 0x32, 0xbf, 0xf1,
+			0xb0, 0x0a, 0x11, 0x99, 0x98, 0x77, 0xee, 0x14, 0x13,
+			0x15, 0x13, 0xad, 0x1f, 0xb6, 0x6b, 0xcd, 0x3b, 0xb8,
+			0x75, 0x9a, 0x55, 0x9b, 0x0c, 0x6c, 0xf7, 0x7c, 0x21,
+			0x06, 0xb7, 0xf0, 0x43, 0x41, 0x96, 0xc1, 0x73, 0x44,
+			0x53, 0xd8, 0x18, 0x3e, 0x09, 0x4e, 0xc2, 0x5f, 0xa0,
+			0xd5, 0x18, 0xdb, 0x9f, 0xf4, 0xa0, 0xf8, 0x4d, 0xa7,
+			0x72, 0x7e, 0x85, 0xbb, 0xb7, 0xcc, 0x7b, 0x51, 0xb0,
+			0xf1, 0x5e, 0x03, 0xcd, 0xe8, 0x5b, 0x83, 0x3f, 0x95,
+			0xe2, 0x0b, 0xa1, 0xc7, 0x6f, 0x74, 0x98, 0xcd, 0x95,
+			0xf4, 0xd4, 0xb8, 0x40, 0xff, 0x75, 0x54, 0x83, 0x3a,
+			0x2c, 0x64, 0x38, 0x10, 0xcb, 0x6d, 0xad, 0xf9, 0x91,
+			0xcb, 0xcf, 0xbe, 0xf6, 0xf7, 0x94, 0x15, 0xea, 0xaf,
+			0x37, 0x65, 0x7c, 0xd2, 0xff, 0x99, 0x79, 0xf8, 0x95,
+			0x27, 0x75, 0x09, 0x60, 0xa4, 0x6f, 0x06, 0x0f, 0x6c,
+			0x13, 0xdd, 0x32, 0x79, 0xa7, 0x40, 0xa5, 0xdd, 0x2c,
+			0x22, 0xc1, 0xee, 0xc3, 0x31, 0x59, 0xb1, 0x3d, 0xa1,
+			0x77, 0x69, 0xb5, 0xd6, 0xae, 0xd7, 0x86, 0xab, 0xa4,
+			0xdf, 0x9f, 0x36, 0xb4, 0xb2, 0xe6, 0x88, 0xd6, 0x45,
+			0x9b, 0x8b, 0x87, 0xbd, 0x27, 0xe9, 0x55, 0xd5, 0xac,
+			0xe9, 0x33, 0x80, 0x00, 0x1d, 0x00, 0x21, 0x74, 0xe2,
+			0x0a, 0x5b, 0xd2, 0x37, 0xab, 0x6f, 0x48, 0x5e, 0x14,
+			0x76, 0x3a, 0x84, 0xf9, 0x34, 0x03, 0x74, 0x8d, 0x6c,
+			0xd8, 0xd4, 0x0f, 0xc7, 0xc8, 0x0c, 0xb5, 0x18, 0x12,
+			0xa5, 0x5a, 0x76, 0x4b, 0x5d, 0x1e, 0x75, 0x89, 0x3a,
+			0x01, 0x60, 0xf1, 0x1e, 0x5b, 0x98, 0x71, 0x7a, 0x15,
+			0x79, 0xff, 0x3d, 0x37, 0x5e, 0xc9, 0x56, 0xa5, 0x43,
+			0xe7, 0xd0, 0x2b, 0x63, 0xfb, 0x2f, 0x1a, 0x55, 0x37,
+			0x15, 0x02, 0x7f, 0x9a, 0x84, 0xf0, 0x07, 0x45, 0x2b,
+			0xa0, 0x9a, 0xb4, 0x60, 0x4d, 0x48, 0xd5, 0xee, 0x64,
+			0xf9, 0xe7, 0x4b, 0x1c, 0xb9, 0x05, 0xd1, 0xd1, 0x47,
+			0x4f, 0xca, 0x26, 0x32, 0xad, 0x7b, 0x2e, 0x7e, 0x54,
+			0x05, 0x96, 0x12, 0x29, 0xbc, 0xd3, 0x45, 0x4e, 0x05,
+			0x83, 0x6d, 0x30, 0x38, 0xd0, 0x3c, 0x46, 0xcd, 0xb6,
+			0x21, 0xb6, 0x5c, 0x2a, 0x40, 0x7a, 0x2e, 0x97, 0xd7,
+			0xbf, 0xa6, 0x55, 0x2a, 0xa6, 0x28, 0x54, 0xc4, 0x66,
+			0xbe, 0xf6, 0xea, 0xc1, 0x19, 0xcb, 0xcc, 0x52, 0x07,
+			0xcd, 0x90, 0x3f, 0x09, 0x23, 0x77, 0xf7, 0xc6, 0x03,
+			0xcc, 0x8b, 0x18, 0xac, 0x63, 0x6d, 0x2a, 0x01, 0x82,
+			0xad, 0x20, 0x11, 0x82, 0xf3, 0xfe, 0x2d, 0x53, 0x3f,
+			0x18, 0xb5, 0xbc, 0xf7, 0x0c, 0xa7, 0xca, 0x63, 0xfa,
+			0x77, 0x8a, 0x3c, 0xc1, 0xc4, 0xb0, 0x15, 0x4b, 0x08,
+			0xdd, 0xb3, 0xd2, 0x33, 0xc8, 0xc2, 0xaa, 0x1a, 0xe4,
+			0x9b, 0x1b, 0x18, 0x78, 0x07, 0x6c, 0x08, 0x28, 0x3f,
+			0xc6, 0xb7, 0x46, 0x8f, 0x8e, 0xc6, 0xff, 0xb1, 0x2d,
+			0x63, 0x12, 0x7c, 0xe5, 0xd8, 0xa5, 0xf8, 0x64, 0x76,
+			0xf7, 0x1b, 0xbc, 0x41, 0xad, 0xee, 0xe8, 0x6f, 0xd4,
+			0x13, 0xea, 0x59, 0x72, 0xfa, 0x04, 0xb4, 0x6e, 0x52,
+			0x3a, 0xff, 0x60, 0xa0, 0xd8, 0xdf, 0x41, 0x1a, 0x4e,
+			0xc8, 0x80, 0x2d, 0x29, 0x13, 0xd3, 0xd0, 0x55, 0x20,
+			0xb5, 0x33, 0x4f, 0x02, 0xd2, 0xea, 0xca, 0x94, 0x99,
+			0xdf, 0x3c, 0xfd, 0xe5, 0x2b, 0x45, 0x04, 0x85, 0xfe,
+			0x87, 0x42, 0x53, 0x11, 0x62, 0x2f, 0xcf, 0x05, 0x35,
+			0x29, 0x6a, 0xa5, 0x37, 0x80, 0x9b, 0x4a, 0x44, 0x3c,
+			0x6d, 0xe1, 0xd3, 0x88, 0x6a, 0xe7, 0x6f, 0x45, 0xfc,
+			0x99, 0x33, 0xe4, 0x18, 0x62, 0x19, 0x36, 0x37, 0x6f,
+			0xe9, 0x37, 0xbb, 0x86, 0xa2, 0x07, 0xec, 0xbc, 0x33,
+			0x0d, 0x0a, 0x18, 0x30, 0x13, 0xcf, 0x36, 0x31, 0x2f,
+			0xd2, 0xf6, 0x7e, 0xa8, 0xa4, 0x95, 0x71, 0xc8, 0x40,
+			0x1f, 0x34, 0xf1, 0x95, 0xde, 0x3e, 0xe1, 0xb7, 0xef,
+			0x26, 0x1e, 0x20, 0x78, 0xf8, 0x9a, 0x24, 0xbc, 0x7f,
+			0x64, 0x30, 0x42, 0x63, 0x77, 0x89, 0x96, 0xf8, 0x9b,
+			0x60, 0x51, 0xb3, 0x02, 0x7a, 0x64, 0x19, 0xd3, 0x13,
+			0x98, 0xcd, 0xb6, 0x1e, 0x68, 0x82, 0x9b, 0x23, 0x44,
+			0x3e, 0xa8, 0x57, 0xf5, 0x4d, 0xd8, 0xea, 0x93, 0x84,
+			0x4e, 0x56, 0x18, 0xb7, 0x94, 0x02, 0xfa, 0x6e, 0x8e,
+			0xa0, 0x75, 0xd2, 0xb4, 0xbb, 0x50, 0x6f, 0x3f, 0x58,
+			0xe2, 0x28, 0xb6, 0x87, 0xd6, 0xc5, 0xe0, 0x76, 0xa6,
+			0xc6, 0x5b, 0xed, 0x6b, 0x48, 0xd4, 0x25, 0xd3, 0x5f,
+			0x64, 0xab, 0xe3, 0x5a, 0xae, 0x69, 0xe8, 0x06, 0xff,
+			0xef, 0x67, 0x0b, 0x49, 0xe4, 0xe8, 0x31, 0xee, 0x1a,
+			0x35, 0x86, 0x0c, 0x81, 0x8a, 0x1f, 0x7f, 0x06, 0x20,
+			0x66, 0x81, 0x8b, 0xfb, 0xdc, 0x99, 0x78, 0x7b, 0x43,
+			0x84, 0x35, 0xa7, 0x4f, 0x27, 0xb5, 0xc9, 0x44, 0xbf,
+			0x05, 0x59, 0xce, 0x4c, 0xe5, 0xaa, 0x18, 0x37, 0x32,
+			0x5b, 0x41, 0x62, 0x89, 0x0a, 0xec, 0x0a, 0xae, 0x67,
+			0xcf, 0x0b, 0xef, 0xf6, 0x3a, 0x3b, 0x13, 0x1d, 0xf8,
+			0x8b, 0x20, 0x5f, 0x2c, 0x55, 0x75, 0xec, 0xf4, 0x9d,
+			0x2d, 0x0d, 0x0d, 0xb7, 0x88, 0x71, 0xf7, 0xc6, 0xf8,
+			0x65, 0x1a, 0xba, 0xdb, 0xdc, 0x53, 0xd2, 0x80, 0x92,
+			0x72, 0x14, 0xce, 0x68, 0xb8, 0xf4, 0x78, 0x4b, 0x92,
+			0xee, 0x15, 0xfc, 0xd4, 0xe0, 0x89, 0x49, 0x82, 0xf8,
+			0x7f, 0xce, 0xf6, 0xce, 0x86, 0xfa, 0xa5, 0xd9, 0x3a,
+			0xfb, 0xef, 0xa8, 0xd9, 0x3b, 0x6e, 0xb5, 0xe2, 0x29,
+			0x31, 0x6f, 0x5f, 0xa7, 0x34, 0x58, 0x49, 0x07, 0x9c,
+			0x4d, 0x74, 0x46, 0xb6, 0xd5, 0x0c, 0x3c, 0x26, 0x4b,
+			0xee, 0x37, 0x9c, 0x92, 0xa2, 0x37, 0x95, 0x35, 0x71,
+			0xee, 0x46, 0xdd, 0xef, 0xd0, 0xf2, 0xd4, 0xef, 0xe0,
+			0xdb, 0x28, 0xf4, 0x5d, 0x12, 0xe0, 0x85, 0x4a, 0x3c,
+			0x5b, 0x2f, 0xd8, 0x9c, 0x9a, 0xcf, 0x6f, 0x01, 0xe7,
+			0x1b, 0x2e, 0x41, 0x5d, 0x5d, 0xdd, 0xdd, 0xb6, 0x6d,
+			0xb3, 0x7d, 0xbc, 0x33, 0x50, 0xc5, 0xaf, 0x1b, 0xee,
+			0x33, 0xc4, 0x42, 0x46, 0xca, 0xe2, 0x00, 0xe9, 0xc9,
+			0xec, 0x9e, 0x9b, 0x67, 0xba, 0x39, 0x3c, 0xc5, 0xa1,
+			0x7d, 0xac, 0x9e, 0xbe, 0x67, 0x92, 0x20, 0x5b, 0x36,
+			0x85, 0xde, 0xdd, 0xd3, 0xec, 0x4e, 0x82, 0x03, 0x4a,
+			0x44, 0x6e, 0xee, 0x9c, 0xd0, 0x13, 0x9e, 0x98, 0x6f,
+			0x77, 0xd8, 0xfe, 0xb7, 0x54, 0x01, 0x15, 0xcd, 0xd3,
+			0x31, 0xf5, 0xd7, 0x74, 0x2d, 0x4d, 0x4f, 0x18, 0xd2,
+			0x8f, 0x80, 0xb6, 0x46, 0x80, 0xa7, 0xa8, 0x8c, 0xf2,
+			0x64, 0x4f, 0x40, 0x8a, 0x64, 0x96, 0x2c, 0x1e, 0xd2,
+			0x3c, 0xe9, 0x50, 0xc4, 0x86, 0xd9, 0x19, 0x93, 0x7a,
+			0xba, 0x95, 0x35, 0x35, 0xe3, 0x4d, 0x83, 0xda, 0x15,
+			0x09, 0x87, 0xb8, 0x1a, 0x54, 0x6f, 0x61, 0xdd, 0x55,
+			0x01, 0xe1, 0x39, 0x1c, 0xb2, 0xac, 0x8c, 0x15, 0x82,
+			0x7c, 0xa6, 0x69, 0x54, 0x64, 0x2f, 0x5c, 0x0c, 0x79,
+			0xaf, 0x85, 0x71, 0x2d, 0xff, 0xc0, 0xf2, 0x40, 0xbb,
+			0x05, 0x31, 0x31, 0x00, 0x8d, 0x0c, 0x40, 0x11, 0x88,
+			0xf2, 0x91, 0x9f, 0x7a, 0x8e, 0x99, 0x82, 0x72, 0xd8,
+			0x45, 0x08, 0x83, 0x5c, 0x2b, 0x73, 0x46, 0xd6, 0x54,
+			0x9a, 0x3d, 0x42, 0x48, 0x67, 0x19, 0x2b, 0x19, 0xae,
+			0x0c, 0x16, 0x08, 0xa4, 0xec, 0x7e, 0x15, 0xad, 0x4f,
+			0xa9, 0xbd, 0x4f, 0x09, 0xf0, 0xc2, 0x9f, 0xb5, 0xb7,
+			0xff, 0x96, 0xf9, 0xf9, 0xc6, 0x5d, 0x57, 0x07, 0xe8,
+			0xf2, 0x7c, 0x95, 0xa0, 0x8a, 0x15, 0x8b, 0x34, 0x84,
+			0xb5, 0x28, 0x7a, 0xf5, 0x04, 0xb8, 0xcb, 0x5e, 0x77,
+			0x4d, 0xdd, 0x14, 0x8d, 0xa6, 0xe2, 0x45, 0x44, 0xfd,
+			0xd0, 0x9c, 0x41, 0x10, 0xfc, 0xe5, 0xdd, 0x2c, 0xda,
+			0x6b, 0xfe, 0xf9, 0xff, 0x65, 0xe7, 0x66, 0x40, 0x69,
+			0x0c, 0x09, 0x99, 0xaf, 0x02, 0xd0, 0x85, 0x55, 0xcc,
+			0x0d, 0x59, 0xaa, 0xb6, 0x37, 0x9b, 0x58, 0xfa, 0xb5,
+			0xc5, 0xcd, 0xd5, 0x1c, 0xf9, 0x9f, 0x31, 0x4c, 0x8a,
+			0x78, 0xca, 0x1e, 0x22, 0xdc, 0x33, 0x64, 0xce, 0x56,
+			0x6d, 0x57, 0xe9, 0xbd, 0xe0, 0xdd, 0x23, 0x22, 0x9c,
+			0x70, 0x5a, 0xe7, 0x46, 0x08, 0xc5, 0xa6, 0x85, 0x34,
+			0x5f, 0x9c, 0x96, 0xbb, 0xbe, 0xb4, 0x2f, 0xbb, 0x47,
+			0xdc, 0x38, 0xad, 0xb7, 0x92, 0x08, 0x0d, 0x93, 0xf4,
+			0x7f, 0xe6, 0x64, 0xa5, 0xb3, 0x9e, 0xdd, 0xdb, 0x99,
+			0x8e, 0xca, 0x82, 0x4f, 0x77, 0xf7, 0xff, 0x92, 0xd3,
+			0x48, 0xe0, 0x80, 0x71, 0x14, 0x61, 0x6b, 0x77, 0x1b,
+			0x58, 0xbd, 0xb6, 0xb6, 0xe5, 0xb0, 0xa8, 0x16, 0xe7,
+			0x4c, 0xc2, 0x5e, 0x40, 0xbc, 0x9f, 0x77, 0x76, 0x98,
+			0xf7, 0x00, 0xd1, 0xdf, 0x3c, 0x2e, 0x16, 0x4f, 0xe7,
+			0x91, 0xec, 0xeb, 0xfb, 0xa1, 0x9f, 0x66, 0x6d, 0xef,
+			0xaf, 0x36, 0xd8, 0xb6, 0x40, 0x7e, 0x2f, 0x51, 0x3b,
+			0x75, 0x56, 0x1d, 0x33, 0xee, 0x4b, 0xcf, 0x98, 0xb3,
+			0x9f, 0x4f, 0xff, 0xfd, 0x2a, 0xb9, 0x4e, 0x4b, 0xe0,
+			0x39, 0x60, 0xd5, 0x7c, 0xaa, 0x28, 0x7e, 0x7e, 0x27,
+			0xd8, 0x09, 0x85, 0x6d, 0xfa, 0x57, 0x8a, 0xee, 0x84,
+			0xbe, 0x48, 0xf9, 0x62, 0x78, 0x8a, 0x4d, 0x2a, 0x80,
+			0x2f, 0x6f, 0xc9, 0xa7, 0xb5, 0x56, 0xbb, 0x58, 0xc2,
+			0xb9, 0xd0, 0xfe, 0x8d, 0x68, 0x86, 0x2b, 0x7b, 0xf0,
+			0x01, 0x57, 0xff, 0xaf, 0x88, 0xad, 0x53, 0xf9, 0x63,
+			0xfe, 0xf4, 0x5a, 0x97, 0xb7, 0x8f, 0x92, 0x0f, 0x90,
+			0x63, 0xcf, 0xfe, 0x08, 0xbc, 0x8c, 0x5a, 0x9c, 0xac,
+			0x8b, 0x0a, 0x94, 0x17, 0x25, 0xae, 0x87, 0xa7, 0x1d,
+			0x3f, 0x28, 0xfe, 0x45, 0x3e, 0x48, 0xef, 0x19, 0x4a,
+			0xbe, 0xc6, 0xd5, 0x64, 0xe1, 0xf2, 0xf5, 0xd2, 0x3a,
+			0x03, 0xe0, 0x50, 0x31, 0x7e, 0xdf, 0xd5, 0x15, 0xc6,
+			0xb6, 0xb2, 0x1e, 0x44, 0x6a, 0x0f, 0x67, 0xf2, 0x34,
+			0x69, 0xb9, 0xd2, 0x51, 0xeb, 0x86, 0xab, 0x7e, 0x60,
+			0x68, 0xe4, 0xc2, 0x4b, 0x8b, 0xa2, 0xc6, 0x78, 0xb2,
+			0x61, 0xa1, 0xe5, 0xb6, 0x33, 0x47, 0x3f, 0xfa, 0xde,
+			0x5b, 0x6b, 0x95, 0x89, 0x51, 0xbb, 0xd3, 0x8a, 0x00,
+			0x1d, 0xad, 0xfc, 0x7a, 0x47, 0xc8, 0xa0, 0x69, 0xa6,
+			0xa3, 0x3f, 0xa9, 0xdb, 0xd8, 0xe4, 0x0d, 0x2c, 0xba,
+			0x26, 0x2c, 0xa6, 0x82, 0x62, 0x5a, 0x1e, 0x9f, 0xa8,
+			0x24, 0x9d, 0xe5, 0xc7, 0xcf, 0x6d, 0x2f, 0x68, 0xe1,
+			0xc0, 0x04, 0xad, 0xde, 0x00, 0x5b, 0x57, 0x35, 0x2b,
+			0x9a, 0x9b, 0xe5, 0x90, 0x31, 0x8a, 0x5b, 0xea, 0xe5,
+			0xf6, 0x73, 0x02, 0xc1, 0x33, 0xaf, 0xe7, 0x61, 0x6c,
+			0x04, 0x8c, 0x08, 0xb2, 0xfb, 0xa2, 0x18, 0x51, 0x39,
+			0x1a, 0x61, 0x2c, 0x8c, 0x7d, 0x11, 0xb8, 0x2e, 0x38,
+			0xb7, 0xe0, 0xde, 0x5e, 0x6f, 0xc5, 0xf4, 0x31, 0x72,
+			0xcb, 0xd6, 0xec, 0xbc, 0xe9, 0x36, 0x2e, 0x2b, 0x63,
+			0xea, 0xa1, 0xab, 0xf8, 0x11, 0xe7, 0x31, 0xad, 0xe4,
+			0xcb, 0x23, 0xe4, 0x64, 0x54, 0x50, 0x4f, 0x15, 0x8b,
+			0x58, 0xa4, 0xa1, 0x35, 0xea, 0x73, 0x3e, 0xc3, 0x3e,
+			0x31, 0x81, 0x80, 0x53, 0x6a, 0x2c, 0xfe, 0xd7, 0xd8,
+			0x8f, 0xe7, 0x50, 0x2d, 0x99, 0x69, 0x32, 0xfa, 0x4e,
+			0xb4, 0xf9, 0x89, 0x18, 0xbe, 0x5a, 0x95, 0x7a, 0x0f,
+			0x34, 0xc5, 0xd9, 0x28, 0xff, 0x93, 0x3a, 0x13, 0x22,
+			0xe6, 0xdb, 0x5c, 0xfe, 0x51, 0x05, 0xaf, 0xa3, 0xcd,
+			0x17, 0x01, 0xd1, 0x84, 0x76, 0x23, 0xff, 0x91, 0xc4,
+			0x6b, 0xc7, 0xa8, 0x77, 0xe0, 0xda, 0x45, 0xef, 0xd7,
+			0x13, 0xe3, 0x02, 0x7c, 0x74, 0x7e, 0x66, 0xbb, 0x9f,
+			0x48, 0x35, 0x56, 0xa8, 0x08, 0x41, 0xf7, 0xa7, 0xe3,
+			0x60, 0xa6, 0xce, 0x73, 0xdf, 0x73, 0x99, 0xc6, 0xbd,
+			0x7b, 0xb4, 0xde, 0x31, 0xe6, 0x35, 0xaf, 0x3e, 0x93,
+			0xd2, 0x69, 0xb5, 0xb9, 0xbe, 0x9f, 0xf4, 0x0d, 0xd8,
+			0x82, 0xc1, 0x4f, 0xf6, 0xf3, 0x23, 0x1c, 0xe7, 0xff,
+			0xac, 0xb2, 0x79, 0xe3, 0x62, 0x6a, 0xb7, 0x85, 0x08,
+			0xda, 0x4a, 0x3f, 0x22, 0x62, 0x2b, 0xcd, 0xb7, 0x59,
+			0xf5, 0x82, 0xef, 0x08, 0x7f, 0xb5, 0xa0, 0x6c, 0xba,
+			0x61, 0x45, 0x8f, 0xf4, 0xa2, 0x66, 0x83, 0x72, 0x65,
+			0x34, 0x34, 0x68, 0x99, 0xf5, 0x99, 0xc9, 0x52, 0x6a,
+			0xc2, 0x7f, 0xde, 0x25, 0xa3, 0x7b, 0xce, 0xa3, 0xde,
+			0x6d, 0x62, 0x98, 0x84, 0xe4, 0x2a, 0xe1, 0x1c, 0xa9,
+			0x0b, 0x7e, 0x45, 0xd3, 0x78, 0x40, 0x85, 0x61, 0x9d,
+			0xce, 0xcd, 0xfd, 0x7b, 0x66, 0x92, 0x5f, 0xb9, 0x37,
+			0x04, 0xea, 0xec, 0x37, 0x32, 0xbe, 0xcd, 0x3e, 0x5e,
+			0x37, 0xcd, 0xe3, 0x81, 0x92, 0x69, 0xdf, 0xa1, 0xb9,
+			0x8c, 0x44, 0x43, 0xae, 0xcd, 0xe5, 0xd5, 0x4e, 0xb9,
+			0x2f, 0x1e, 0xbc, 0xf2, 0xd1, 0x88, 0x93, 0xde, 0x6a,
+			0x8a, 0x43, 0xa1, 0x6b, 0xdc, 0x5b, 0xd2, 0xdd, 0xe8,
+			0xca, 0x0a, 0x95, 0x45, 0x9f, 0xd8, 0x8c, 0xfd, 0xe0,
+			0xc3, 0xbd, 0x0e, 0xa0, 0xbd, 0x72, 0xa7, 0x77, 0x9a,
+			0x79, 0xd9, 0x15, 0xad, 0x50, 0x13, 0xa7, 0x41, 0xc7,
+			0xd1, 0xec, 0x2f, 0x45, 0x14, 0x02, 0xb7, 0x8e, 0x51,
+			0x76, 0xd4, 0x2f, 0x1c, 0xea, 0x9c, 0x6d, 0x35, 0x1c,
+			0xa7, 0xeb, 0xd7, 0x79, 0xe4, 0x35, 0xbc, 0x02, 0xa6,
+			0xd0, 0x58, 0xcb, 0xdf, 0xea, 0x90, 0x16, 0x53, 0x3c,
+			0xc5, 0xda, 0x36, 0x4f, 0x61, 0x4a, 0x1b, 0xfa, 0x15,
+			0x52, 0x0d, 0x3d, 0x73, 0xc5, 0xfe, 0x2a, 0xa2, 0x4a,
+			0x6e, 0x95, 0x99, 0xfc, 0xbf, 0xb2, 0xfc, 0x22, 0xbe,
+			0x45, 0x10, 0x5a, 0xc5, 0x38, 0xa0, 0xc1, 0xba, 0x46,
+			0x6d, 0x8e, 0xbe, 0x0b, 0x5f, 0xf6, 0x6f, 0xbb, 0x4d,
+			0x22, 0xf5, 0x95, 0x4d, 0x12, 0xc2, 0x6f, 0x8d, 0xde,
+			0x86, 0xaf, 0x3f, 0x08, 0x24, 0x56, 0xa3, 0xcd, 0xfc,
+			0xeb, 0xa1, 0x3c, 0x22, 0x66, 0x45, 0x03, 0x8b, 0x04,
+			0x12, 0x80, 0x34, 0xf9, 0xf4, 0x64, 0xbf, 0xb9, 0x3d,
+			0xd7, 0x34, 0xe4, 0xc8, 0x8e, 0x86, 0xc9, 0x73, 0x21,
+			0x7f, 0x30, 0x01, 0xc8, 0xd2, 0x91, 0x8d, 0xc4, 0xca,
+			0xd5, 0x59, 0xac, 0xd7, 0xb4, 0xfb, 0x98, 0xfb, 0xc4,
+			0x48, 0x82, 0x93, 0xc6, 0x58, 0x8b, 0x0c, 0xdf, 0x3f,
+			0x07, 0x5a, 0x94, 0x89, 0xc6, 0xbe, 0x1a, 0x19, 0x63,
+			0x8b, 0xa8, 0x0c, 0x20, 0xae, 0x9a, 0x1b, 0xd3, 0x45,
+			0xa0, 0xd9, 0x8c, 0xbb, 0x67, 0xa0, 0x54, 0x86, 0x9b,
+			0x7b, 0xa8, 0x16, 0x6f, 0xa4, 0x59, 0x45, 0x1e, 0x34,
+			0xb5, 0xa7, 0x1b, 0x3f, 0xda, 0xc6, 0x80, 0x19, 0xdc,
+			0xc2, 0xee, 0xeb, 0xd9, 0x96, 0xdc, 0x4f, 0xec, 0x02,
+			0xde, 0x34, 0x43, 0x5a, 0x37, 0xc8, 0xdf, 0x3b, 0x53,
+			0x82, 0xfa, 0xf8, 0x8a, 0x32, 0x97, 0x1e, 0xf6, 0x0e,
+			0x7d, 0x39, 0xe0, 0xdb, 0x0b, 0x22, 0xa4, 0xd0, 0x56,
+			0x13, 0xbc, 0xe9, 0x38, 0x92, 0x76, 0xff, 0x2b, 0xc2,
+			0xa0, 0x98, 0x7d, 0x8c, 0x58, 0x8e, 0x7f, 0x6d, 0x80,
+			0x06, 0xff, 0xc7, 0xb3, 0xb7, 0x1b, 0xac, 0xc4, 0x68,
+			0x61, 0x11, 0x15, 0xb7, 0xee, 0x8d, 0x66, 0x34, 0x9f,
+			0x18, 0x03, 0x88, 0xd1, 0x04, 0xd0, 0x86, 0xc0, 0xca,
+			0x2a, 0xe1, 0x1b, 0x47, 0x18, 0xc9, 0x97, 0xed, 0x1f,
+			0x99, 0xc6, 0xa9, 0x20, 0x3f, 0xf6, 0x43, 0x96, 0x30,
+			0x8b, 0xa5, 0xcb, 0x2e, 0xc5, 0xdb, 0x26, 0xc0, 0xf6,
+			0x48, 0x07, 0x3a, 0x28, 0x3e, 0x35, 0x38, 0x50, 0x80,
+			0xdf, 0x80, 0xfa, 0x12, 0x92, 0x83, 0x55, 0xde, 0x31,
+		},
+	.msg_len = 64,
+	.msg =
+		(const u8[64]) {
+			0x48, 0x11, 0x92, 0xb3, 0xaa, 0x7e, 0xb3, 0x14,
+			0xdc, 0x46, 0xd6, 0xbf, 0x26, 0xb2, 0x66, 0xc5,
+			0xd8, 0xc4, 0x69, 0x9f, 0x67, 0x50, 0x67, 0xc0,
+			0x3b, 0x8d, 0xc5, 0xb5, 0x68, 0x59, 0x6f, 0x9c,
+			0xd4, 0xf2, 0x49, 0xfb, 0xa4, 0xd0, 0xbf, 0xc5,
+			0x3c, 0xee, 0x3b, 0x03, 0x25, 0x6c, 0x51, 0x94,
+			0xb3, 0xd4, 0xdb, 0x2a, 0xbd, 0x26, 0xef, 0x58,
+			0xb3, 0x91, 0x62, 0x81, 0x1b, 0x29, 0x9c, 0x12,
+		},
+	.sig_len = MLDSA87_SIGNATURE_SIZE,
+	.sig =
+		(const u8[MLDSA87_SIGNATURE_SIZE]) {
+			0xba, 0x08, 0xf3, 0x0c, 0xa1, 0x48, 0xad, 0x67, 0x12,
+			0x68, 0x51, 0x9b, 0xa4, 0x21, 0x66, 0x8d, 0xea, 0x0a,
+			0x71, 0x4f, 0x4c, 0xab, 0x1b, 0x42, 0x84, 0xb7, 0x9c,
+			0x78, 0xa9, 0x18, 0xd9, 0xe8, 0x08, 0x8f, 0xee, 0xf7,
+			0x4b, 0xcf, 0x0e, 0xcd, 0xf6, 0x36, 0xcf, 0xd5, 0xa1,
+			0x5e, 0x59, 0xc6, 0x8c, 0x6c, 0x4d, 0xfe, 0xfa, 0xfa,
+			0xbc, 0x52, 0x3a, 0x35, 0xba, 0xd9, 0xe1, 0xf2, 0x26,
+			0x73, 0xbd, 0x64, 0x93, 0xe7, 0x83, 0x76, 0xc4, 0xf8,
+			0x4b, 0xa9, 0x91, 0x8c, 0xd2, 0xf5, 0x08, 0xe6, 0x58,
+			0xcd, 0xfb, 0x90, 0xc1, 0x84, 0x72, 0x77, 0x72, 0x47,
+			0x5f, 0xcb, 0x7f, 0x74, 0x85, 0x56, 0xba, 0x73, 0x86,
+			0xc1, 0x32, 0x1c, 0xaf, 0xc5, 0x60, 0x2c, 0x91, 0x99,
+			0x09, 0x03, 0xcb, 0xe6, 0xa4, 0xae, 0x0b, 0xdc, 0x15,
+			0x70, 0x4c, 0x91, 0xa9, 0x9e, 0x5e, 0x63, 0x5c, 0x24,
+			0xc9, 0x26, 0x39, 0x56, 0xe4, 0x6c, 0x2c, 0x65, 0xf8,
+			0x0b, 0x4f, 0x56, 0x8f, 0x1e, 0x74, 0xda, 0x31, 0xad,
+			0x8f, 0x1b, 0xa7, 0xb1, 0xb3, 0x9b, 0x48, 0x09, 0x08,
+			0xdd, 0xe1, 0x25, 0x20, 0xa4, 0xc7, 0x12, 0xd4, 0xb0,
+			0xa0, 0xcb, 0x90, 0xf5, 0x9f, 0xc1, 0x8b, 0xbc, 0x9d,
+			0xcf, 0x78, 0xfc, 0xa1, 0x04, 0x4b, 0x0b, 0x7e, 0x82,
+			0x94, 0x80, 0x0e, 0x70, 0x77, 0xce, 0x63, 0x1c, 0x43,
+			0x41, 0x3f, 0x5d, 0x61, 0x1a, 0x44, 0xaa, 0xd0, 0xb7,
+			0x18, 0xd8, 0x91, 0x83, 0xa4, 0xa7, 0xa1, 0x86, 0x13,
+			0x54, 0xaf, 0xa8, 0xdb, 0x1a, 0xd7, 0xa9, 0x54, 0xca,
+			0x67, 0xb5, 0xb6, 0xb4, 0x22, 0x1a, 0x24, 0x8e, 0x65,
+			0x01, 0xa8, 0xd2, 0x15, 0x9d, 0x98, 0x59, 0x70, 0x94,
+			0xb4, 0x96, 0xeb, 0xf2, 0x95, 0x41, 0x9a, 0x89, 0x02,
+			0xb7, 0x6a, 0x20, 0x2a, 0x64, 0xf1, 0x1c, 0x1e, 0x3d,
+			0xd8, 0xdb, 0x6d, 0xef, 0xf2, 0x1a, 0x24, 0xa4, 0xd0,
+			0xb9, 0x98, 0xd3, 0xe5, 0xbd, 0x7a, 0xa3, 0xc3, 0x19,
+			0x97, 0x54, 0x77, 0x97, 0xb5, 0x57, 0x93, 0x71, 0x24,
+			0x91, 0x81, 0xcf, 0x5f, 0x38, 0xb9, 0xd5, 0x7e, 0x8b,
+			0x0f, 0xea, 0x56, 0x49, 0xac, 0x25, 0xc4, 0x6c, 0x5e,
+			0x4e, 0x08, 0x31, 0xc8, 0xdf, 0x0b, 0x19, 0xb9, 0xbc,
+			0xec, 0x3b, 0x89, 0xf8, 0x98, 0xe3, 0x6c, 0xf6, 0x0f,
+			0x5b, 0x7e, 0x03, 0x10, 0x68, 0x78, 0x2c, 0xfa, 0x39,
+			0x5c, 0x64, 0x5b, 0xa1, 0xfc, 0x8d, 0x14, 0x87, 0x62,
+			0x04, 0x57, 0xc0, 0x5b, 0x6f, 0x6e, 0x3c, 0xea, 0xe6,
+			0x23, 0xde, 0x9b, 0x1d, 0x89, 0x69, 0x38, 0xfb, 0x10,
+			0xab, 0xb9, 0xda, 0xee, 0xa8, 0x8d, 0x96, 0xc0, 0xa5,
+			0x40, 0xce, 0x16, 0x1e, 0xf5, 0xf4, 0x74, 0x6f, 0x0e,
+			0x38, 0xe5, 0x1d, 0xe6, 0x6c, 0x19, 0xa5, 0x71, 0x48,
+			0xda, 0x3e, 0xe9, 0x10, 0x57, 0x16, 0x89, 0xd3, 0x5b,
+			0x4a, 0x5b, 0xa9, 0xb1, 0x13, 0xad, 0x77, 0xaf, 0x1f,
+			0x8b, 0x71, 0x3f, 0xff, 0x7c, 0xc4, 0x31, 0xb2, 0xa7,
+			0x18, 0x2f, 0x4b, 0x99, 0xe2, 0x14, 0xbf, 0x1e, 0x69,
+			0x8f, 0x56, 0xe2, 0x2c, 0x84, 0x34, 0xd6, 0x76, 0x27,
+			0x06, 0x7f, 0x2e, 0x9d, 0xbe, 0x90, 0x95, 0xda, 0x5a,
+			0xfb, 0xaa, 0x8a, 0x51, 0x3d, 0x97, 0xb8, 0x75, 0xe9,
+			0xba, 0x2a, 0x03, 0x7e, 0xb0, 0xe3, 0xdc, 0xe3, 0x3b,
+			0x2a, 0xd8, 0x77, 0xa5, 0xf3, 0x76, 0x69, 0xf1, 0xd2,
+			0xd5, 0xb3, 0xba, 0xfa, 0x97, 0x55, 0x5f, 0xd1, 0xc3,
+			0x10, 0x55, 0x9a, 0xb6, 0xce, 0x32, 0x80, 0x7f, 0x48,
+			0xb7, 0x15, 0x85, 0x3f, 0x22, 0x58, 0x00, 0x51, 0x12,
+			0x5c, 0xcc, 0xd0, 0x77, 0x26, 0x57, 0x4b, 0xbb, 0x67,
+			0x21, 0x0d, 0x92, 0x80, 0xe0, 0xec, 0x2f, 0x58, 0x44,
+			0x8b, 0x98, 0x70, 0x16, 0x13, 0x91, 0x38, 0xee, 0x9e,
+			0x4f, 0xf2, 0x6f, 0x95, 0x49, 0xe8, 0xcc, 0xc5, 0x6d,
+			0x20, 0x20, 0x4d, 0x1b, 0x08, 0xf4, 0xa5, 0x8b, 0xf3,
+			0xf6, 0x89, 0x2c, 0x17, 0xa6, 0xa8, 0xa8, 0xfb, 0xab,
+			0x6f, 0xa0, 0xe3, 0xcd, 0xc5, 0xfb, 0xac, 0x91, 0x42,
+			0x8c, 0xf2, 0x28, 0xd1, 0x13, 0xdb, 0x63, 0x59, 0x69,
+			0xab, 0x16, 0x5a, 0xaf, 0x0a, 0xfa, 0x43, 0x6d, 0x53,
+			0x7f, 0x14, 0x88, 0xc8, 0xa1, 0x71, 0x45, 0x54, 0x30,
+			0xa7, 0x91, 0x55, 0x44, 0x07, 0x03, 0x78, 0xf3, 0xf6,
+			0x23, 0xc0, 0x21, 0x4b, 0x39, 0xc4, 0x9a, 0x5a, 0xe8,
+			0xad, 0x28, 0x6b, 0xd2, 0xd7, 0xc9, 0xed, 0xf4, 0x30,
+			0xff, 0xf8, 0xc4, 0x7a, 0xfe, 0x49, 0x1c, 0x28, 0xbb,
+			0x89, 0xfa, 0x9d, 0x64, 0xcc, 0x37, 0x76, 0x6a, 0xf3,
+			0x8b, 0x94, 0x60, 0xe0, 0xb2, 0x29, 0xf8, 0x5a, 0xc6,
+			0x3e, 0xcc, 0x6f, 0x13, 0x85, 0x7c, 0xaa, 0xc4, 0x35,
+			0x56, 0x9a, 0xb6, 0x96, 0x76, 0x41, 0xe2, 0x11, 0xbe,
+			0x07, 0xba, 0x0c, 0x21, 0x32, 0xc3, 0xb3, 0xf3, 0x96,
+			0x49, 0xa2, 0xc4, 0x49, 0x48, 0xfe, 0x17, 0xef, 0xdf,
+			0x1e, 0xf3, 0xde, 0x5f, 0xa0, 0x49, 0xfc, 0xa5, 0xb5,
+			0xf3, 0x4c, 0x99, 0x5f, 0xcb, 0x33, 0xc6, 0x54, 0x52,
+			0xa0, 0x85, 0x16, 0x75, 0x2c, 0x5f, 0x87, 0x61, 0x58,
+			0xc9, 0x23, 0x0a, 0x36, 0x66, 0x3b, 0x78, 0x65, 0xcc,
+			0xd4, 0x8e, 0xe1, 0x27, 0x9a, 0x74, 0x37, 0x79, 0x4c,
+			0x5c, 0x35, 0xf9, 0xa5, 0x7f, 0x79, 0x35, 0xa8, 0x7d,
+			0xcf, 0x64, 0xa3, 0x44, 0xb9, 0xee, 0x1e, 0x07, 0x5c,
+			0xda, 0xcd, 0x93, 0xfc, 0x33, 0x77, 0x68, 0xfc, 0xd6,
+			0xea, 0xf4, 0xcd, 0x0a, 0x3a, 0xe3, 0xb3, 0xef, 0xf9,
+			0xb5, 0x1a, 0x4b, 0x50, 0x2e, 0xcb, 0x88, 0x48, 0x60,
+			0xc3, 0x5c, 0x1b, 0xb7, 0xcc, 0x7a, 0x49, 0x2e, 0xf0,
+			0x9e, 0x96, 0x47, 0x7d, 0x82, 0x85, 0xcb, 0x37, 0x85,
+			0x26, 0xfd, 0xe1, 0x13, 0x57, 0xad, 0x51, 0xdc, 0x02,
+			0x8d, 0x3c, 0x4e, 0xdd, 0xf3, 0xdf, 0xbe, 0xab, 0x0d,
+			0x99, 0xf8, 0x13, 0x37, 0x0b, 0x1b, 0xaf, 0x08, 0x9b,
+			0x38, 0x5b, 0x3a, 0x91, 0x71, 0x9a, 0xe6, 0xc1, 0x7d,
+			0xd4, 0xd8, 0xf2, 0x66, 0x57, 0x99, 0x29, 0x58, 0xb5,
+			0x9d, 0x8f, 0x86, 0x7f, 0xe9, 0x17, 0xff, 0x3c, 0xfd,
+			0x0a, 0xfd, 0x15, 0x5c, 0x2f, 0x77, 0x6c, 0x2b, 0x86,
+			0x72, 0xcb, 0x00, 0x4d, 0x31, 0x98, 0xfe, 0x28, 0xc5,
+			0x54, 0x36, 0x97, 0x8c, 0xc9, 0x31, 0xae, 0xff, 0xfc,
+			0xa6, 0x15, 0xf9, 0x9d, 0x49, 0x83, 0x47, 0xb4, 0xe5,
+			0x8c, 0x55, 0x2f, 0x35, 0x6d, 0x7c, 0x47, 0xb9, 0xeb,
+			0x0a, 0x64, 0xea, 0x22, 0xae, 0xe3, 0x6b, 0xae, 0x0e,
+			0xf8, 0xc4, 0x64, 0x18, 0x56, 0xa7, 0x78, 0x92, 0x20,
+			0xe0, 0x03, 0x93, 0x42, 0x1f, 0x1a, 0x48, 0xa1, 0x11,
+			0xdd, 0x83, 0x99, 0x67, 0xc5, 0xf2, 0x0a, 0x9e, 0xfa,
+			0xab, 0x68, 0x54, 0x1d, 0xa7, 0xe8, 0x44, 0x5f, 0x46,
+			0x23, 0xea, 0xdd, 0xb2, 0xf5, 0x83, 0x76, 0x84, 0x13,
+			0xd6, 0xe4, 0xb2, 0x59, 0x92, 0x92, 0xf4, 0xb5, 0x56,
+			0xd7, 0x29, 0xc7, 0x05, 0x00, 0x79, 0x17, 0x59, 0x20,
+			0xc6, 0x6a, 0xdc, 0x45, 0x93, 0xe7, 0xfc, 0x29, 0x37,
+			0xa7, 0x7e, 0xbc, 0x21, 0xe7, 0xe1, 0xff, 0x18, 0x54,
+			0xf9, 0xd3, 0x0c, 0x78, 0xe1, 0xc9, 0x40, 0x36, 0x36,
+			0x68, 0x46, 0x3c, 0x9a, 0x4d, 0xa6, 0xbc, 0xc3, 0xc7,
+			0xc9, 0x8c, 0x7e, 0xc6, 0x07, 0x2a, 0xd0, 0x4d, 0x45,
+			0x3d, 0x65, 0xef, 0x56, 0x7f, 0xf1, 0xcd, 0x6d, 0x97,
+			0x0c, 0x11, 0x66, 0xf4, 0x27, 0xfc, 0xe3, 0xf3, 0x93,
+			0x0d, 0xa1, 0x5a, 0x95, 0x86, 0xcc, 0x0f, 0xf4, 0x97,
+			0xbc, 0x7c, 0x62, 0x78, 0xe5, 0x01, 0xe0, 0xc7, 0x0c,
+			0xe7, 0xa9, 0x84, 0xa6, 0x17, 0xaa, 0x36, 0x06, 0x2a,
+			0x0f, 0x2f, 0x7e, 0x3d, 0xe8, 0x70, 0x8d, 0x36, 0x73,
+			0x9a, 0x7f, 0x47, 0x95, 0xc5, 0x5d, 0xc5, 0xfa, 0x9c,
+			0x64, 0x52, 0xcb, 0x52, 0x74, 0xda, 0xb0, 0x1f, 0x82,
+			0x01, 0x43, 0x98, 0xa6, 0xe7, 0xf2, 0x89, 0x51, 0x92,
+			0xf8, 0x37, 0x82, 0x8e, 0x91, 0x58, 0x40, 0x2c, 0xac,
+			0xcf, 0xde, 0x92, 0x63, 0x5a, 0x64, 0xf9, 0x9c, 0xa4,
+			0x47, 0x6d, 0x13, 0x43, 0xd3, 0xa6, 0x89, 0xbb, 0x7b,
+			0x32, 0x45, 0x2d, 0x0e, 0x52, 0x25, 0x67, 0xda, 0x1d,
+			0x9e, 0x42, 0x38, 0xfc, 0x92, 0x84, 0x4d, 0x13, 0x53,
+			0xef, 0x46, 0x81, 0xdd, 0xf6, 0x79, 0x9f, 0xba, 0x45,
+			0x49, 0x4b, 0x93, 0x4a, 0x54, 0x1c, 0x72, 0xa0, 0xfd,
+			0xa1, 0x13, 0x1c, 0x5e, 0xa5, 0x4e, 0xd5, 0xdf, 0xd1,
+			0x6a, 0x40, 0x1d, 0x5e, 0x57, 0xf8, 0xe3, 0x3a, 0xe8,
+			0x35, 0xf4, 0x67, 0xa2, 0x37, 0xf2, 0xe0, 0xd1, 0xfe,
+			0x08, 0x5e, 0x5c, 0x41, 0x17, 0x87, 0x24, 0xae, 0xf3,
+			0xe2, 0xc8, 0xb4, 0xfc, 0x3b, 0xef, 0xdb, 0x5e, 0x27,
+			0x93, 0x0f, 0x01, 0xf9, 0x35, 0xcf, 0x67, 0x4f, 0x65,
+			0xde, 0xca, 0xb6, 0xf1, 0x12, 0xa0, 0xd7, 0x53, 0x09,
+			0xec, 0x83, 0x9a, 0xb6, 0x8f, 0xf6, 0x03, 0xdf, 0x92,
+			0xa9, 0xc4, 0x85, 0x55, 0x5a, 0x74, 0x66, 0x9e, 0xc0,
+			0x46, 0xc7, 0x50, 0x7b, 0xda, 0x56, 0x4a, 0xf5, 0x38,
+			0x37, 0xd5, 0x82, 0x81, 0xf3, 0x2d, 0x51, 0xa6, 0xa3,
+			0x34, 0xf9, 0x24, 0xed, 0x35, 0x80, 0x72, 0xae, 0x3d,
+			0x8b, 0xf8, 0x66, 0xb2, 0x6c, 0x3f, 0x4e, 0x2c, 0xdb,
+			0x89, 0x76, 0x95, 0x8f, 0x1a, 0xbe, 0xfc, 0x6c, 0x37,
+			0xec, 0xf9, 0x80, 0x32, 0xac, 0x94, 0x97, 0x85, 0xe4,
+			0xa5, 0x19, 0x32, 0x46, 0x1f, 0x98, 0x89, 0x4c, 0x85,
+			0xf1, 0xfc, 0xed, 0x01, 0x98, 0x4d, 0x4a, 0xd4, 0xd1,
+			0x4e, 0x82, 0x0b, 0x32, 0x0d, 0x48, 0x34, 0x64, 0x98,
+			0xea, 0x6c, 0xf9, 0x11, 0x25, 0xbc, 0x48, 0xc3, 0xf4,
+			0x96, 0xfc, 0x5e, 0x71, 0x5d, 0xfa, 0xf7, 0x9f, 0x75,
+			0x89, 0x1d, 0xe7, 0x3b, 0x97, 0x39, 0x18, 0x90, 0xca,
+			0xa5, 0x46, 0x01, 0x2a, 0x13, 0x54, 0x5c, 0x2a, 0x14,
+			0xed, 0x1b, 0x65, 0x0a, 0xb0, 0xd9, 0x95, 0x01, 0xd0,
+			0xb3, 0x78, 0xd6, 0xc4, 0x06, 0xf3, 0x7f, 0x51, 0xe0,
+			0x39, 0xed, 0x2c, 0xf3, 0xfa, 0xc3, 0xfa, 0x1d, 0x7c,
+			0x82, 0x0f, 0x45, 0xb7, 0x15, 0xcf, 0x99, 0x53, 0x59,
+			0x9d, 0x54, 0x31, 0x6a, 0xf8, 0xb8, 0x73, 0x2d, 0xd1,
+			0xab, 0x1b, 0x69, 0xc9, 0x86, 0xdc, 0x7a, 0xca, 0xb9,
+			0x6a, 0xaf, 0x1e, 0x8f, 0x83, 0xa8, 0xf0, 0xae, 0x44,
+			0x85, 0x0d, 0x1f, 0x6c, 0x3a, 0x90, 0x50, 0xef, 0x7c,
+			0x3c, 0x1c, 0xc4, 0xb8, 0x66, 0x1c, 0x14, 0xf7, 0xf2,
+			0x5f, 0x5c, 0x70, 0x28, 0xa7, 0x55, 0x93, 0xdb, 0xa0,
+			0x95, 0x9c, 0x1b, 0xb7, 0xb6, 0x44, 0x5e, 0x98, 0x88,
+			0x89, 0xb8, 0x5a, 0xee, 0xba, 0xfc, 0xc6, 0x87, 0x2a,
+			0xdb, 0xb7, 0xcf, 0xb2, 0x4b, 0x96, 0x2c, 0x91, 0x54,
+			0x34, 0x33, 0xfb, 0x57, 0xbe, 0xf5, 0x7b, 0xbe, 0x6b,
+			0xa0, 0x3b, 0x4d, 0x09, 0x18, 0x9b, 0x75, 0x2c, 0xde,
+			0x28, 0x87, 0xd3, 0x7b, 0xf5, 0xbe, 0x0d, 0x33, 0x5a,
+			0x3b, 0x92, 0x41, 0x35, 0x50, 0x65, 0xc9, 0xa9, 0xfa,
+			0xab, 0x22, 0x4b, 0x8c, 0x3a, 0xb1, 0x9e, 0x31, 0x41,
+			0xd2, 0x2f, 0xca, 0x9e, 0xd0, 0x96, 0x0c, 0x03, 0x08,
+			0xec, 0x94, 0xa5, 0xb1, 0x65, 0x2b, 0xe1, 0xe4, 0x48,
+			0x11, 0xd6, 0x8c, 0x60, 0x6d, 0xf4, 0x64, 0x89, 0xa6,
+			0x69, 0x4b, 0xa0, 0xbe, 0xae, 0xf5, 0xde, 0xba, 0x45,
+			0x12, 0xc8, 0x28, 0xe0, 0x7a, 0xb5, 0xd4, 0x75, 0x7a,
+			0x24, 0x59, 0x4f, 0xbf, 0x0a, 0xa4, 0x53, 0x9a, 0x59,
+			0xca, 0x06, 0x51, 0x6d, 0xcb, 0x93, 0x63, 0x6e, 0x24,
+			0xe2, 0x41, 0xa5, 0xa9, 0x19, 0xc3, 0x59, 0x9a, 0x0c,
+			0xb6, 0x4e, 0x65, 0x60, 0xa0, 0x49, 0xf0, 0x7f, 0x50,
+			0x96, 0xed, 0xe9, 0xfc, 0x33, 0x5b, 0x52, 0x51, 0x7e,
+			0x77, 0xd7, 0xbb, 0xaf, 0xdd, 0x12, 0x4b, 0xbc, 0xc6,
+			0xae, 0xca, 0x92, 0x3a, 0x9f, 0xc1, 0x0c, 0x8f, 0x01,
+			0xbe, 0x4c, 0x78, 0x23, 0x7c, 0x8b, 0x82, 0xb7, 0x45,
+			0x80, 0x44, 0x34, 0x9d, 0x13, 0xdb, 0xc4, 0x3f, 0x1b,
+			0x39, 0xb0, 0xee, 0xa9, 0xac, 0x5e, 0x3d, 0xc3, 0x59,
+			0x92, 0xcc, 0x6e, 0x6c, 0x97, 0xcc, 0xba, 0x15, 0xa6,
+			0x53, 0x92, 0x76, 0x35, 0x01, 0x70, 0x16, 0xde, 0xde,
+			0x56, 0xc3, 0xb3, 0x48, 0x8b, 0x4d, 0xfa, 0xa7, 0xcb,
+			0x43, 0xb5, 0x76, 0x9a, 0x56, 0x03, 0xd6, 0x41, 0xbe,
+			0x73, 0x89, 0x1f, 0x6f, 0x02, 0x3f, 0xcc, 0xaf, 0x49,
+			0xe3, 0x5e, 0xb3, 0xf4, 0xbc, 0x61, 0xf9, 0x9e, 0x9e,
+			0xec, 0x10, 0x98, 0x0d, 0x5a, 0x6d, 0xcf, 0xbf, 0xe3,
+			0xe5, 0xf5, 0x1b, 0x17, 0x46, 0x7a, 0x32, 0xc7, 0x18,
+			0x40, 0x20, 0x8e, 0x06, 0x62, 0x2d, 0x1b, 0x90, 0x54,
+			0x4f, 0x7f, 0xe4, 0x50, 0x09, 0xdc, 0x57, 0x22, 0xe9,
+			0x7c, 0x3c, 0x77, 0x57, 0xc2, 0x8b, 0xc9, 0x6a, 0x2b,
+			0x31, 0xa1, 0x7f, 0x14, 0xdf, 0x22, 0x7d, 0xed, 0xc9,
+			0x0d, 0xbb, 0x93, 0xba, 0x72, 0x85, 0x33, 0x58, 0x80,
+			0xc0, 0xf5, 0x75, 0x5d, 0x7e, 0x5f, 0xfb, 0x41, 0xbc,
+			0x5f, 0x62, 0x13, 0xb9, 0x16, 0x1e, 0xfe, 0x9e, 0x53,
+			0x53, 0xce, 0xa0, 0xfc, 0x7f, 0xf6, 0x18, 0xc2, 0x0f,
+			0xb9, 0x7f, 0xbf, 0x09, 0xc3, 0x37, 0x08, 0xb0, 0x1f,
+			0x16, 0x80, 0x5e, 0xf6, 0xcb, 0xfe, 0x22, 0x7c, 0x07,
+			0x99, 0xb3, 0x12, 0x69, 0xa4, 0x05, 0xc9, 0x29, 0xc7,
+			0xd0, 0x57, 0x05, 0x33, 0x85, 0x0c, 0xd4, 0xfe, 0xfc,
+			0x54, 0xa8, 0xc9, 0x7b, 0x92, 0x67, 0xa6, 0xa7, 0xbf,
+			0x5f, 0xfc, 0xe9, 0x65, 0x31, 0x4f, 0x85, 0x8f, 0x89,
+			0xbe, 0xa1, 0xdb, 0x6d, 0x96, 0x52, 0x09, 0x56, 0x12,
+			0xc6, 0xb2, 0x28, 0x26, 0x07, 0x39, 0x21, 0x39, 0xce,
+			0x55, 0xf9, 0x82, 0x23, 0xc2, 0x3f, 0x3a, 0x66, 0xc4,
+			0xee, 0x10, 0x4b, 0xf1, 0x1e, 0x1d, 0x59, 0xfa, 0x28,
+			0x20, 0x34, 0x26, 0x49, 0x7c, 0xc3, 0x85, 0x2f, 0x80,
+			0xbe, 0x2c, 0xbc, 0x94, 0x81, 0x74, 0x2a, 0x21, 0x49,
+			0x1d, 0xa7, 0xff, 0xf0, 0x21, 0x56, 0x98, 0x68, 0x8f,
+			0x52, 0x2d, 0xa0, 0xf2, 0x84, 0x73, 0x2f, 0xe2, 0x90,
+			0x8c, 0x56, 0xd1, 0xbf, 0x67, 0xc3, 0x19, 0x47, 0x14,
+			0x71, 0x62, 0x25, 0xdc, 0x11, 0xbc, 0x24, 0xa0, 0xb2,
+			0x42, 0x72, 0x37, 0x78, 0xe6, 0xe0, 0xf6, 0x8c, 0x66,
+			0xe5, 0x1b, 0x7a, 0x79, 0x46, 0xc0, 0x76, 0xcd, 0xa9,
+			0x3c, 0xb2, 0x17, 0x8c, 0xc9, 0xb1, 0xc4, 0x2a, 0x1f,
+			0xdb, 0xf7, 0xeb, 0x8a, 0x5d, 0x29, 0xd1, 0xd1, 0x98,
+			0x43, 0x9e, 0x22, 0xf3, 0x96, 0x1e, 0x83, 0xeb, 0x44,
+			0x08, 0xc4, 0xa0, 0xd2, 0xd3, 0x36, 0xf8, 0x94, 0xab,
+			0x3c, 0x4b, 0x68, 0xf3, 0x7c, 0x1a, 0x16, 0x1a, 0x66,
+			0x77, 0x67, 0xee, 0x64, 0xc2, 0xdb, 0x53, 0xae, 0x1f,
+			0xeb, 0x44, 0x77, 0x3b, 0x5f, 0x74, 0x48, 0xe9, 0x05,
+			0xfa, 0x3a, 0x03, 0x1b, 0x54, 0x36, 0xb0, 0x28, 0x79,
+			0x70, 0xe6, 0x2f, 0xb4, 0xf5, 0x28, 0x60, 0x1e, 0x63,
+			0xf6, 0x03, 0x09, 0x4f, 0x0f, 0x7f, 0x01, 0xe5, 0x02,
+			0x6a, 0x76, 0x96, 0xb9, 0x58, 0x39, 0xf9, 0xc4, 0x05,
+			0xac, 0x93, 0x2f, 0x30, 0xab, 0xc5, 0x2b, 0xad, 0xed,
+			0x3c, 0xb0, 0x6c, 0x66, 0x94, 0x7e, 0x79, 0x01, 0x90,
+			0xf0, 0x10, 0xb5, 0xcd, 0x6b, 0x07, 0xc2, 0xe7, 0x8a,
+			0xac, 0xf7, 0x5b, 0xb5, 0x11, 0xf8, 0x1a, 0x52, 0xc7,
+			0x5d, 0x2c, 0xc1, 0xce, 0x8f, 0x0c, 0xc1, 0x6a, 0x95,
+			0x80, 0x25, 0xf9, 0x95, 0x28, 0x18, 0x65, 0xd3, 0x15,
+			0x0e, 0xbf, 0x6f, 0xa6, 0x86, 0xd9, 0xea, 0xdc, 0xb8,
+			0x89, 0x02, 0x08, 0x6b, 0xd1, 0x96, 0xb1, 0x6f, 0xb1,
+			0x28, 0x6b, 0xc4, 0xe3, 0xa8, 0x53, 0xd3, 0xd0, 0x67,
+			0x00, 0x7c, 0xa1, 0x0a, 0xd1, 0x9a, 0x89, 0x12, 0xc9,
+			0xa7, 0x3f, 0x03, 0x03, 0xad, 0x0f, 0x34, 0x2c, 0xd8,
+			0xdf, 0x70, 0xed, 0x38, 0xf4, 0xb1, 0xf7, 0xb1, 0x72,
+			0x5e, 0x2d, 0xa6, 0xb7, 0x8d, 0x9c, 0x7f, 0x96, 0x73,
+			0xc8, 0x47, 0x46, 0x20, 0x2f, 0x44, 0x2c, 0x2b, 0xa2,
+			0x30, 0xe4, 0x81, 0x91, 0x7e, 0xce, 0x4c, 0x8b, 0xcc,
+			0x5f, 0xfd, 0xff, 0x2f, 0x94, 0xbd, 0xe5, 0xaf, 0x09,
+			0xcd, 0xb6, 0xbc, 0x65, 0x83, 0xc3, 0xcd, 0x79, 0xc4,
+			0x68, 0xda, 0x46, 0x36, 0x97, 0x52, 0x35, 0x90, 0x4f,
+			0x1d, 0x8d, 0x66, 0x1f, 0xea, 0x5d, 0xef, 0xdf, 0x8c,
+			0xa8, 0x6b, 0xb0, 0xdc, 0x1a, 0x1e, 0x79, 0x43, 0xe9,
+			0x29, 0x71, 0x11, 0xbd, 0xe5, 0xd6, 0x71, 0xf4, 0xa7,
+			0xf8, 0x64, 0x40, 0x73, 0x46, 0x5b, 0x2a, 0xb5, 0x9a,
+			0xa1, 0x9a, 0x64, 0x4c, 0x5e, 0xa5, 0x8d, 0xfd, 0x49,
+			0x91, 0xba, 0xe3, 0xa7, 0x19, 0xac, 0x2b, 0xe9, 0xea,
+			0xf1, 0x1c, 0x85, 0x91, 0xea, 0xfb, 0xc7, 0x5c, 0x46,
+			0x7e, 0xa8, 0x0a, 0x92, 0xbf, 0x61, 0x2d, 0xc2, 0x20,
+			0x7e, 0xf7, 0x0d, 0xb8, 0x65, 0x41, 0x48, 0x39, 0xb9,
+			0xb0, 0xd4, 0xc1, 0x84, 0xee, 0x21, 0xba, 0x9c, 0x74,
+			0xf3, 0x6d, 0x0a, 0x33, 0x87, 0xab, 0x2e, 0x15, 0x1b,
+			0xad, 0xac, 0x98, 0xae, 0x7c, 0xd8, 0xae, 0x03, 0x50,
+			0xfa, 0xae, 0xfe, 0x8c, 0x6d, 0x51, 0x67, 0x76, 0x70,
+			0xa6, 0x2f, 0x1e, 0x6a, 0x4f, 0xf0, 0xbe, 0x51, 0xbb,
+			0xa6, 0x20, 0x1f, 0x56, 0x72, 0x34, 0xd0, 0x13, 0x6e,
+			0x0f, 0x1a, 0xcb, 0x34, 0xe1, 0xfb, 0x3c, 0xf1, 0x43,
+			0x1b, 0x73, 0x40, 0xa0, 0x1f, 0x5d, 0x40, 0x51, 0x9f,
+			0x97, 0x56, 0x1f, 0x26, 0x54, 0x03, 0xb8, 0x2e, 0xfd,
+			0xc7, 0x52, 0xea, 0x85, 0x5d, 0x45, 0x71, 0xa3, 0x5c,
+			0x86, 0x23, 0xcb, 0x08, 0xc5, 0xd8, 0x21, 0xfd, 0x10,
+			0x49, 0xac, 0xaa, 0xd5, 0x97, 0x37, 0xbb, 0xad, 0xd5,
+			0x84, 0xc2, 0x65, 0xeb, 0xdd, 0x92, 0x7c, 0x50, 0x64,
+			0x4f, 0x08, 0x32, 0x56, 0x81, 0x39, 0xa4, 0xfc, 0x3f,
+			0x04, 0xbb, 0xde, 0xe9, 0xcd, 0xb6, 0xbb, 0x29, 0x6b,
+			0x75, 0x64, 0x32, 0x5f, 0xca, 0xd0, 0x3a, 0x19, 0x3d,
+			0x01, 0x8c, 0x0b, 0xd5, 0x76, 0xe6, 0x97, 0x5d, 0x73,
+			0x7b, 0xb3, 0xcd, 0xdf, 0xbc, 0x29, 0xda, 0xa5, 0x22,
+			0xfb, 0x1d, 0xf6, 0xf1, 0x50, 0x71, 0x50, 0xd3, 0xf8,
+			0x0a, 0xbf, 0x37, 0x28, 0xca, 0x3b, 0xab, 0xec, 0xf5,
+			0xee, 0x2d, 0x1e, 0x21, 0x35, 0x39, 0x56, 0x55, 0x3c,
+			0x9e, 0x5a, 0x58, 0x94, 0x85, 0xa7, 0xba, 0x0d, 0xa4,
+			0x0b, 0x4c, 0x73, 0xb2, 0x56, 0x68, 0x5a, 0x5d, 0x6c,
+			0x18, 0xce, 0xc5, 0x25, 0x14, 0xc9, 0xb3, 0xc9, 0x94,
+			0x57, 0x62, 0x95, 0xd5, 0x1c, 0xf6, 0x93, 0x12, 0x45,
+			0x28, 0xa1, 0x7e, 0x96, 0x53, 0x68, 0xf5, 0x6a, 0xca,
+			0xbe, 0x04, 0x94, 0x97, 0xcf, 0xb5, 0xcf, 0x70, 0xe1,
+			0x7e, 0x15, 0x2d, 0x6a, 0x67, 0x0c, 0x86, 0x07, 0xd1,
+			0xe2, 0x95, 0xc3, 0xd8, 0x22, 0xdf, 0x6a, 0x88, 0xc1,
+			0xad, 0x83, 0x57, 0x30, 0xd0, 0x59, 0xaf, 0xf4, 0x9f,
+			0x8d, 0x89, 0x7d, 0x0b, 0xee, 0xb7, 0x1c, 0xaa, 0xd2,
+			0x4c, 0x7a, 0xa4, 0xfb, 0x99, 0xe0, 0x0a, 0xd7, 0x4d,
+			0xc4, 0xe9, 0x88, 0x18, 0x27, 0x1d, 0xb2, 0xe9, 0x43,
+			0x16, 0x25, 0x29, 0x04, 0x98, 0x81, 0xfd, 0xaf, 0xdd,
+			0x3f, 0xf1, 0x61, 0x32, 0x20, 0x51, 0x96, 0xe4, 0xd2,
+			0xab, 0xc4, 0x00, 0x33, 0xb4, 0x7b, 0x3c, 0xc2, 0x2d,
+			0x61, 0x11, 0x4d, 0x72, 0x16, 0x6d, 0xfb, 0x24, 0x67,
+			0x7c, 0xea, 0xf0, 0x79, 0xa3, 0x50, 0x93, 0xe3, 0x9b,
+			0x7f, 0x89, 0xf0, 0xa6, 0x8c, 0xb0, 0xaf, 0x1c, 0xe0,
+			0x91, 0xce, 0x3f, 0xe5, 0x43, 0x90, 0x41, 0x95, 0xc7,
+			0x1f, 0x8f, 0x0a, 0xb2, 0x1e, 0x84, 0x3e, 0x86, 0xf0,
+			0x81, 0x87, 0x8b, 0x04, 0x52, 0xa6, 0x46, 0xb0, 0xe4,
+			0xf4, 0x5b, 0x97, 0x7b, 0x7b, 0x32, 0xb6, 0x17, 0x96,
+			0x48, 0x12, 0xb6, 0x31, 0xf1, 0x5c, 0xd4, 0x8b, 0x93,
+			0x60, 0xc4, 0x00, 0xea, 0x12, 0xe1, 0x93, 0x6f, 0x53,
+			0x82, 0x2a, 0x48, 0x1d, 0xc1, 0x91, 0x79, 0x7c, 0x30,
+			0x43, 0xe5, 0x1e, 0x10, 0x7e, 0x7f, 0xbc, 0x18, 0x60,
+			0x03, 0x2a, 0xe5, 0xbb, 0x92, 0x7f, 0x24, 0x54, 0x12,
+			0x21, 0x77, 0x89, 0xc9, 0x22, 0xfb, 0x57, 0xaa, 0x69,
+			0xdd, 0x69, 0x06, 0xb0, 0xe5, 0x5b, 0x74, 0x1f, 0x02,
+			0x99, 0x62, 0x56, 0x21, 0x29, 0xd2, 0x26, 0x0f, 0x6a,
+			0x3a, 0x05, 0x96, 0x92, 0xb9, 0xff, 0xe9, 0x3b, 0x24,
+			0x70, 0xd4, 0x13, 0xa4, 0xb2, 0x6c, 0x76, 0xe3, 0x55,
+			0xe5, 0xc8, 0x18, 0x66, 0x60, 0x91, 0xba, 0x98, 0x5d,
+			0x67, 0x13, 0x97, 0xce, 0x49, 0xaa, 0x52, 0x7b, 0x20,
+			0x65, 0x65, 0x0a, 0x4f, 0x7b, 0x0a, 0x26, 0x62, 0xa7,
+			0xb5, 0xab, 0x2d, 0x5d, 0x37, 0xad, 0xb8, 0x97, 0xcf,
+			0xf5, 0x4f, 0xef, 0x03, 0xed, 0xce, 0x73, 0x68, 0x5d,
+			0x9f, 0x6c, 0x3d, 0x3f, 0xd5, 0xc1, 0xca, 0x5f, 0xef,
+			0xe4, 0xf8, 0xfb, 0xc7, 0xbb, 0x49, 0x30, 0x1a, 0xe1,
+			0x3c, 0xd1, 0x7f, 0x7c, 0x26, 0xa2, 0x87, 0x6e, 0x80,
+			0xa6, 0x47, 0x2c, 0x0f, 0xa6, 0x68, 0xf6, 0xde, 0x65,
+			0x1f, 0x02, 0xa7, 0x09, 0x08, 0x34, 0xbe, 0x2c, 0xc7,
+			0xed, 0x4c, 0xb0, 0xa1, 0x63, 0x63, 0x35, 0xe0, 0x11,
+			0xfd, 0xe1, 0x2d, 0xb4, 0xcf, 0x9e, 0x18, 0x78, 0xa8,
+			0x63, 0xbf, 0xb7, 0xe8, 0x22, 0x82, 0xc5, 0x9c, 0x0b,
+			0x3a, 0x1c, 0x2f, 0x73, 0x80, 0xf1, 0xbb, 0x9a, 0x84,
+			0x51, 0x2c, 0x1c, 0x2e, 0xda, 0xb8, 0xa3, 0x43, 0x4f,
+			0x95, 0x8c, 0x06, 0x12, 0xdc, 0x75, 0x53, 0x01, 0xc4,
+			0x50, 0x95, 0x31, 0x7e, 0x75, 0x33, 0x69, 0x3e, 0x82,
+			0xad, 0x51, 0x1a, 0x4d, 0xa4, 0xe7, 0xbb, 0x7c, 0x9e,
+			0x3c, 0xbc, 0x1a, 0xf7, 0x7d, 0x0f, 0xdd, 0x93, 0xb5,
+			0x91, 0x61, 0x28, 0x28, 0x3b, 0x8f, 0xdd, 0xfa, 0xe6,
+			0xf4, 0xcc, 0x21, 0x82, 0x0c, 0xe2, 0xe4, 0xc7, 0x39,
+			0x69, 0x8e, 0xd6, 0xc0, 0x45, 0x94, 0x26, 0x27, 0x1b,
+			0xca, 0x02, 0x39, 0x00, 0x12, 0xef, 0xad, 0x0f, 0xf2,
+			0xdb, 0x19, 0x39, 0x6e, 0x4e, 0xfd, 0x14, 0x56, 0x3c,
+			0xd3, 0xfb, 0x43, 0x2d, 0x14, 0xb1, 0x62, 0xd2, 0xc8,
+			0x09, 0xe3, 0xb4, 0xf3, 0x24, 0x4e, 0xeb, 0xe5, 0xd6,
+			0x42, 0x48, 0xa5, 0xf8, 0xd8, 0xcc, 0x0b, 0xd6, 0x23,
+			0x59, 0x46, 0xaf, 0x15, 0xb2, 0x36, 0x2d, 0x52, 0xbe,
+			0x4a, 0x5f, 0xc6, 0xb2, 0x3d, 0xc5, 0x53, 0x57, 0x35,
+			0x2a, 0xa1, 0x06, 0xc8, 0x93, 0xa8, 0x3e, 0xc6, 0xcb,
+			0xa4, 0xd9, 0xe4, 0x53, 0xa9, 0xe8, 0x6d, 0xe1, 0x73,
+			0x97, 0xb9, 0x6f, 0xb3, 0xf8, 0x1d, 0xb4, 0xae, 0xd8,
+			0x77, 0xa7, 0xef, 0xe1, 0xeb, 0x83, 0x3d, 0xc3, 0x35,
+			0xf0, 0xc6, 0xd8, 0x66, 0xb5, 0x29, 0x7f, 0x74, 0xdb,
+			0xa1, 0xf2, 0x16, 0xc9, 0x07, 0x0f, 0x5c, 0x3e, 0x18,
+			0x64, 0x82, 0xfd, 0xae, 0x80, 0xca, 0xcf, 0xf9, 0x7e,
+			0x7f, 0x36, 0x58, 0xd5, 0x4d, 0x2d, 0x5a, 0x27, 0x27,
+			0xcc, 0x7e, 0x45, 0x0d, 0xf8, 0xd6, 0x5f, 0xee, 0x83,
+			0x42, 0x83, 0x48, 0x0b, 0x64, 0xc6, 0x03, 0x4d, 0x1e,
+			0x24, 0x90, 0xba, 0x2a, 0x92, 0x7d, 0x43, 0x77, 0x17,
+			0xc8, 0xcc, 0x9f, 0x31, 0x80, 0x4e, 0x19, 0x7f, 0x8f,
+			0xcf, 0xd7, 0x27, 0x48, 0x10, 0xea, 0x3d, 0xd6, 0x0f,
+			0x70, 0x7a, 0xb9, 0xed, 0x53, 0x67, 0x1c, 0x56, 0xb1,
+			0x78, 0xfc, 0x0c, 0xcc, 0x1f, 0xf8, 0x89, 0x45, 0x57,
+			0x4a, 0x72, 0xd2, 0xa0, 0x07, 0x53, 0x73, 0x3d, 0x0e,
+			0x45, 0x5a, 0x31, 0x94, 0x8f, 0x5f, 0x11, 0xe4, 0x36,
+			0xe2, 0x19, 0xc4, 0x84, 0x94, 0xb9, 0xc0, 0xe6, 0x5d,
+			0x66, 0x7e, 0x4c, 0x05, 0xc6, 0xcd, 0x9c, 0x6c, 0x8f,
+			0x32, 0x68, 0xfb, 0x66, 0x86, 0x22, 0xf5, 0xfa, 0x9a,
+			0x46, 0x26, 0xb5, 0xd5, 0xc1, 0x7d, 0xeb, 0x7b, 0x82,
+			0x3e, 0x62, 0x8c, 0x76, 0x0b, 0x08, 0x1d, 0xe0, 0x75,
+			0x34, 0x30, 0x4b, 0xc2, 0x4b, 0xcb, 0x49, 0xa3, 0x65,
+			0x0e, 0x90, 0xd3, 0x57, 0xe5, 0xf5, 0x83, 0x1b, 0xa2,
+			0xbb, 0x4a, 0x23, 0x63, 0x25, 0x64, 0xd7, 0xf1, 0x85,
+			0x9b, 0xbe, 0x2e, 0xda, 0xad, 0xc8, 0x60, 0x37, 0x3c,
+			0x78, 0x33, 0xb5, 0x76, 0x7c, 0xc6, 0xe5, 0x97, 0x86,
+			0x6d, 0x70, 0xdc, 0x4f, 0x79, 0x39, 0xe6, 0x27, 0x78,
+			0x57, 0x98, 0x69, 0x79, 0xaf, 0x0a, 0xc0, 0x64, 0x0c,
+			0x1a, 0x09, 0xa0, 0x98, 0x06, 0x41, 0x5c, 0x29, 0x27,
+			0xbd, 0x2e, 0xbe, 0xe8, 0x08, 0x5f, 0xdf, 0xed, 0xd9,
+			0x31, 0x97, 0xdb, 0x0a, 0x1f, 0x4e, 0x1b, 0x67, 0x32,
+			0xef, 0x84, 0xa2, 0x14, 0xa7, 0x83, 0x33, 0xdc, 0x40,
+			0x55, 0x76, 0x27, 0x01, 0x9f, 0x98, 0x90, 0xa9, 0xfa,
+			0xfb, 0x73, 0x86, 0x38, 0x6b, 0xa8, 0x57, 0xc9, 0xaa,
+			0x06, 0xbf, 0x84, 0x97, 0xfa, 0xc8, 0xde, 0x67, 0x6b,
+			0xa6, 0x2c, 0xa1, 0x90, 0x6c, 0x6d, 0xcd, 0xb5, 0x73,
+			0x3c, 0x16, 0x68, 0xb4, 0x00, 0x55, 0xf6, 0xd8, 0x35,
+			0xb0, 0xf9, 0x8e, 0x2e, 0xfb, 0x54, 0xc1, 0x8e, 0x51,
+			0x90, 0x2d, 0x28, 0x83, 0x5d, 0x81, 0xd6, 0x00, 0x00,
+			0x41, 0xf2, 0x97, 0xdf, 0x2f, 0x8d, 0xa8, 0xc2, 0x1d,
+			0xe3, 0xbd, 0x1d, 0x2a, 0x6f, 0x4a, 0xfd, 0xd3, 0xcd,
+			0x0b, 0xac, 0x28, 0x93, 0x87, 0x3e, 0xd9, 0x7b, 0x17,
+			0xe2, 0x66, 0x0a, 0x6e, 0xcb, 0xa8, 0x3e, 0x3d, 0x9b,
+			0x3c, 0xbe, 0x78, 0xe1, 0x2a, 0xa5, 0x42, 0x5f, 0xdc,
+			0xe0, 0x0e, 0x06, 0x10, 0x59, 0xac, 0x4a, 0xd3, 0x63,
+			0x95, 0x92, 0x31, 0x1c, 0x7d, 0xb1, 0x5e, 0x48, 0x01,
+			0x23, 0x16, 0x61, 0xb3, 0xfc, 0xe6, 0x9a, 0x5d, 0x39,
+			0xf9, 0x2f, 0xb5, 0xc6, 0x6d, 0xeb, 0xb2, 0x6e, 0x93,
+			0x9d, 0x8b, 0x19, 0x74, 0xa9, 0xc4, 0x57, 0x96, 0xbc,
+			0x1c, 0xbb, 0x88, 0x5a, 0xd6, 0xf7, 0xc0, 0xe2, 0xb6,
+			0xf2, 0xb3, 0xf0, 0x56, 0x5b, 0x2c, 0x9e, 0xdf, 0x96,
+			0x6d, 0xcc, 0x41, 0x06, 0xce, 0x1e, 0x1b, 0x3a, 0x39,
+			0x67, 0xe1, 0x7e, 0x81, 0xfc, 0xff, 0x0a, 0xc0, 0xf1,
+			0x76, 0x82, 0x49, 0x76, 0x88, 0x38, 0x10, 0x5c, 0x90,
+			0x83, 0x4b, 0x3e, 0x8d, 0xb9, 0x35, 0x77, 0xf5, 0x4c,
+			0x7e, 0x3e, 0x12, 0x54, 0xf0, 0x33, 0x56, 0xbc, 0x9b,
+			0x0e, 0x10, 0x7d, 0x4a, 0xb6, 0x2d, 0x02, 0x35, 0x79,
+			0xe8, 0x0c, 0x15, 0xc1, 0xde, 0xc7, 0x2a, 0x7d, 0xe1,
+			0x27, 0xd4, 0x99, 0x50, 0x6b, 0xde, 0x6d, 0x02, 0x65,
+			0x12, 0x88, 0xe6, 0x80, 0x13, 0x5e, 0xad, 0xb1, 0xef,
+			0xe7, 0x77, 0x19, 0xc9, 0x63, 0xcf, 0x14, 0x41, 0x40,
+			0x76, 0xc8, 0xa9, 0xc3, 0x08, 0x88, 0xdb, 0x44, 0x6c,
+			0x10, 0x74, 0x5c, 0xf6, 0x86, 0x8c, 0x61, 0x95, 0x19,
+			0x76, 0x1f, 0x3e, 0xcb, 0xdf, 0x6e, 0x08, 0x9f, 0x43,
+			0xb9, 0x66, 0x2b, 0x88, 0x27, 0x48, 0xbe, 0x5d, 0x9e,
+			0x83, 0x72, 0x9a, 0x43, 0x06, 0x8e, 0xf3, 0x9e, 0xba,
+			0xd5, 0xd8, 0x70, 0x1b, 0xe4, 0x4e, 0x39, 0x0a, 0x37,
+			0x5b, 0x2f, 0x8c, 0xa8, 0xbd, 0x3c, 0xfe, 0xb2, 0x59,
+			0x5b, 0x5f, 0x5a, 0x05, 0x67, 0xc8, 0xec, 0x71, 0x5d,
+			0xb5, 0x77, 0xc7, 0xb3, 0x1c, 0xb7, 0xac, 0x90, 0xf6,
+			0xd3, 0xb7, 0x61, 0x65, 0xbe, 0x29, 0x8f, 0x27, 0x65,
+			0x9a, 0x9a, 0x1c, 0x00, 0x82, 0xee, 0x10, 0xb9, 0x60,
+			0xff, 0x9f, 0x5a, 0xbe, 0x09, 0x72, 0x63, 0x79, 0x60,
+			0x50, 0x84, 0x18, 0xc9, 0xc5, 0x0d, 0x43, 0xce, 0x29,
+			0x80, 0x72, 0x83, 0x12, 0x73, 0x13, 0x44, 0xec, 0x25,
+			0xe5, 0x15, 0x70, 0xe2, 0x7f, 0x6b, 0x54, 0x5b, 0x58,
+			0x57, 0x06, 0xdc, 0xae, 0x1f, 0x9c, 0x38, 0x4c, 0xbf,
+			0x1c, 0x42, 0xb2, 0x63, 0x9c, 0xfb, 0xae, 0xc0, 0xdf,
+			0xd3, 0xcc, 0xcc, 0xb9, 0x9b, 0xe9, 0x66, 0x77, 0x70,
+			0x47, 0x6c, 0x8b, 0x5b, 0xc1, 0x87, 0x7a, 0x63, 0xa3,
+			0x13, 0xfc, 0x2d, 0x16, 0xd3, 0xe2, 0x00, 0xe9, 0x94,
+			0x11, 0xf2, 0x28, 0xda, 0x0b, 0x58, 0x09, 0x2d, 0x59,
+			0x2d, 0xb5, 0x25, 0x99, 0xca, 0x63, 0xc9, 0x96, 0xa3,
+			0x17, 0x2a, 0x0e, 0x22, 0x2c, 0x5f, 0x8d, 0x53, 0xc4,
+			0xd1, 0x3d, 0x4e, 0x20, 0xdc, 0x44, 0x18, 0xd2, 0xd8,
+			0x5c, 0x9a, 0x78, 0x0a, 0x53, 0xa8, 0x73, 0x7d, 0x98,
+			0xc0, 0xba, 0xfb, 0xfe, 0x27, 0xd4, 0x82, 0x87, 0xd4,
+			0x4d, 0xc0, 0xe3, 0xd1, 0xd0, 0xff, 0xf0, 0xd3, 0x04,
+			0xfd, 0x11, 0xf4, 0x57, 0xc2, 0xcc, 0xcf, 0xa9, 0xe6,
+			0xc5, 0x09, 0x4d, 0xcc, 0x36, 0x69, 0x26, 0x35, 0x8c,
+			0xe8, 0xe2, 0xe2, 0xdd, 0x5a, 0x5f, 0xaf, 0xda, 0x38,
+			0x07, 0x33, 0xfb, 0xf8, 0xfb, 0xf9, 0x62, 0x4a, 0x75,
+			0xad, 0xc3, 0x95, 0xd3, 0x6e, 0x5b, 0xe3, 0x10, 0x2f,
+			0x31, 0xaf, 0xcd, 0x57, 0xb9, 0x84, 0x3d, 0xa4, 0x66,
+			0x3f, 0x10, 0xeb, 0xa2, 0xe2, 0xfa, 0xc7, 0xd3, 0x91,
+			0x83, 0x48, 0xbb, 0x3a, 0x6c, 0xe4, 0x12, 0x1c, 0x36,
+			0x44, 0xc5, 0x07, 0x07, 0x9b, 0x9d, 0x51, 0x3e, 0x74,
+			0xc5, 0x35, 0x8a, 0x10, 0x9c, 0x0d, 0x81, 0xf3, 0x10,
+			0x0f, 0x29, 0xaf, 0x01, 0x5d, 0x6f, 0x7f, 0x50, 0x9a,
+			0xf8, 0xbf, 0x1c, 0x57, 0xfc, 0x25, 0x5c, 0x02, 0x15,
+			0x43, 0x0b, 0x60, 0x21, 0xf3, 0xb1, 0x43, 0x9a, 0x41,
+			0xa0, 0xc1, 0x1b, 0x7a, 0x88, 0x97, 0xfd, 0x27, 0x56,
+			0xf7, 0x48, 0x1f, 0xb6, 0x9a, 0xa2, 0xb1, 0xc5, 0xab,
+			0x30, 0xa1, 0x14, 0x91, 0xef, 0x14, 0xe3, 0x01, 0x73,
+			0xf3, 0x8d, 0xfa, 0xd7, 0xae, 0x2f, 0x13, 0x5d, 0x66,
+			0xfb, 0x1c, 0x4b, 0x2e, 0x94, 0xdb, 0x9f, 0x10, 0x5b,
+			0x2a, 0x5e, 0x85, 0x98, 0x73, 0x9d, 0xcd, 0x7c, 0xa0,
+			0x79, 0xb6, 0xef, 0x8e, 0xf8, 0xdd, 0x11, 0x32, 0x0d,
+			0xec, 0x6a, 0x04, 0xea, 0x81, 0xd7, 0x63, 0xbd, 0x01,
+			0x77, 0x3e, 0x52, 0x30, 0x88, 0x3a, 0x9e, 0x04, 0xdf,
+			0x42, 0x8b, 0x40, 0x23, 0xa9, 0x5a, 0x97, 0x21, 0x55,
+			0xce, 0x72, 0x21, 0x45, 0x40, 0x01, 0x66, 0x63, 0x1d,
+			0xab, 0x37, 0x8d, 0xd5, 0x75, 0x56, 0xc2, 0x7c, 0xa0,
+			0x9e, 0xa5, 0x85, 0xc2, 0x0b, 0x51, 0x80, 0xae, 0x0c,
+			0xb3, 0xde, 0xa4, 0xb5, 0xa3, 0xa3, 0x00, 0xe1, 0xc9,
+			0x09, 0xee, 0xab, 0xbb, 0x36, 0x78, 0x5b, 0x3f, 0x45,
+			0x3d, 0xf6, 0xe8, 0x7e, 0xa3, 0xd7, 0x24, 0x78, 0xdf,
+			0xbc, 0x9f, 0xe4, 0x4c, 0x4d, 0x22, 0xb8, 0x76, 0xb0,
+			0x6a, 0xc1, 0x35, 0x66, 0xfe, 0xaf, 0xf1, 0xb4, 0x44,
+			0x5e, 0xe0, 0xff, 0xcf, 0x80, 0xf8, 0xa8, 0x5f, 0x59,
+			0x91, 0xa4, 0x4f, 0xc0, 0x58, 0xb1, 0x2d, 0xc9, 0x6a,
+			0xcd, 0x62, 0xcb, 0x41, 0x89, 0x9d, 0x3b, 0x3e, 0x48,
+			0x5f, 0x23, 0x46, 0x21, 0x8a, 0x49, 0xc0, 0x16, 0x07,
+			0x4d, 0xd9, 0x26, 0xbe, 0x86, 0xfb, 0x06, 0xbd, 0x41,
+			0xc5, 0xd8, 0x87, 0xf6, 0x94, 0xb1, 0xef, 0xed, 0xab,
+			0xfb, 0x93, 0x48, 0x5a, 0xe7, 0x22, 0xbc, 0x2b, 0x24,
+			0xb4, 0x86, 0x33, 0x9b, 0xd6, 0x09, 0x46, 0x85, 0x81,
+			0x89, 0xa9, 0x30, 0x13, 0xfa, 0x08, 0x55, 0xfc, 0xe5,
+			0xfb, 0xc8, 0xb2, 0xa8, 0x31, 0x4c, 0x9e, 0x1c, 0xff,
+			0x80, 0x10, 0x77, 0x5f, 0xf1, 0x07, 0x14, 0xf7, 0x88,
+			0xb9, 0xeb, 0x48, 0xfe, 0x18, 0x48, 0xf3, 0xaf, 0xfd,
+			0x7d, 0x0c, 0x9f, 0x30, 0x11, 0xcb, 0xed, 0xfc, 0x31,
+			0xf5, 0x6e, 0x72, 0xfa, 0x9a, 0x3a, 0xa5, 0x1d, 0x89,
+			0x89, 0x8b, 0xc6, 0x2c, 0xaf, 0xa6, 0xef, 0xbd, 0xb4,
+			0xc2, 0xa0, 0x97, 0x74, 0xd0, 0x4d, 0x72, 0xb7, 0xe8,
+			0x24, 0xba, 0x5e, 0x53, 0x4b, 0x91, 0xc1, 0xe4, 0x14,
+			0x22, 0x50, 0xb2, 0xc3, 0x84, 0xf4, 0xa6, 0xb3, 0xa4,
+			0x99, 0xda, 0x82, 0x55, 0x8e, 0x38, 0xb4, 0x0c, 0x21,
+			0xe5, 0x91, 0x30, 0xc7, 0x80, 0x55, 0xf1, 0x69, 0xde,
+			0xd7, 0x82, 0xf3, 0x08, 0xd3, 0x16, 0x69, 0xe7, 0xd4,
+			0xeb, 0xfc, 0x9b, 0x8f, 0xd6, 0xfc, 0xf9, 0x9f, 0x95,
+			0xb3, 0x90, 0xa6, 0x12, 0x21, 0x53, 0x50, 0xd5, 0x3e,
+			0x2b, 0x46, 0x14, 0x52, 0x20, 0x1c, 0x33, 0xed, 0xdd,
+			0x58, 0x81, 0x92, 0x37, 0xe5, 0x99, 0x77, 0xfa, 0x29,
+			0x28, 0x49, 0x07, 0xc8, 0xe2, 0x41, 0x8f, 0x77, 0x3b,
+			0xb4, 0x8c, 0x34, 0x3e, 0x80, 0xcb, 0xcf, 0x73, 0x4d,
+			0xfc, 0x78, 0x60, 0x9e, 0xd0, 0x86, 0xc3, 0x4d, 0xa7,
+			0x40, 0x9c, 0x03, 0x72, 0x46, 0xc3, 0xc3, 0x74, 0xdf,
+			0x5a, 0x47, 0x8d, 0xbc, 0xc1, 0x5b, 0x10, 0x3e, 0x7e,
+			0x09, 0x3d, 0xdb, 0x0c, 0xc4, 0x3b, 0x12, 0xbf, 0x66,
+			0xe2, 0xb7, 0x4c, 0x1a, 0x81, 0x58, 0x63, 0x39, 0x72,
+			0xde, 0xfa, 0x5d, 0xcc, 0x2c, 0x69, 0x11, 0x0a, 0x5f,
+			0xb9, 0xbc, 0xda, 0x80, 0x03, 0x8e, 0xa2, 0x02, 0x6c,
+			0xc1, 0x15, 0xcc, 0xbf, 0x01, 0x14, 0xd0, 0x36, 0xad,
+			0xdb, 0xb7, 0x77, 0xd6, 0x99, 0x94, 0xf5, 0xa5, 0xac,
+			0xce, 0x9a, 0x23, 0x9b, 0x1e, 0xa4, 0x97, 0x7e, 0x41,
+			0xc1, 0x65, 0xc7, 0x94, 0xfa, 0x07, 0x1d, 0x9a, 0x87,
+			0xba, 0xee, 0xef, 0x99, 0x92, 0xaa, 0x41, 0x29, 0x03,
+			0xe3, 0x01, 0x17, 0x42, 0xed, 0x11, 0x68, 0x44, 0x02,
+			0xca, 0x81, 0x4b, 0x33, 0x81, 0x64, 0x30, 0x95, 0x7f,
+			0x81, 0x17, 0xe6, 0x59, 0x0b, 0x4e, 0xf3, 0x4a, 0x72,
+			0x0c, 0x45, 0x1e, 0x1d, 0x44, 0xe2, 0x9e, 0x45, 0x29,
+			0x88, 0xb4, 0xda, 0x9b, 0x9f, 0x3e, 0x92, 0x9a, 0x0b,
+			0x21, 0x2a, 0xd4, 0xa0, 0xd8, 0xd8, 0xbd, 0x06, 0x6d,
+			0x71, 0x3c, 0xcc, 0xd8, 0xdc, 0x19, 0xd7, 0x8f, 0x21,
+			0x01, 0x25, 0xa5, 0x15, 0x62, 0x43, 0xfa, 0xdf, 0xb9,
+			0x33, 0x23, 0xd3, 0xa2, 0xab, 0xb1, 0x7e, 0x4e, 0x5e,
+			0x46, 0xbd, 0x76, 0x6e, 0x18, 0x88, 0x5f, 0x85, 0x03,
+			0x43, 0x57, 0xd6, 0x3a, 0xf2, 0xcc, 0x9d, 0xbc, 0x2c,
+			0x67, 0xa8, 0xcd, 0xda, 0xe8, 0x39, 0x61, 0x68, 0x6d,
+			0xcf, 0xeb, 0x07, 0x1f, 0x58, 0x61, 0x68, 0x6d, 0x78,
+			0xa6, 0xad, 0xb6, 0x16, 0x70, 0x89, 0x49, 0x59, 0x62,
+			0x76, 0xc1, 0xc3, 0xc8, 0xce, 0xe2, 0xfc, 0x42, 0x51,
+			0x8d, 0xc8, 0xf1, 0x03, 0x17, 0x36, 0x55, 0x8c, 0x8e,
+			0x98, 0xd1, 0x33, 0x3c, 0x4c, 0x63, 0x81, 0x97, 0xc5,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x06, 0x0c, 0x16, 0x19, 0x23, 0x28, 0x30,
+			0x37,
+		},
+};
diff --git a/lib/crypto/tests/mldsa_kunit.c b/lib/crypto/tests/mldsa_kunit.c
new file mode 100644
index 000000000000..67f8f93e3dc6
--- /dev/null
+++ b/lib/crypto/tests/mldsa_kunit.c
@@ -0,0 +1,438 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * KUnit tests and benchmark for ML-DSA
+ *
+ * Copyright 2025 Google LLC
+ */
+#include <crypto/mldsa.h>
+#include <kunit/test.h>
+#include <linux/random.h>
+#include <linux/unaligned.h>
+
+#define Q 8380417 /* The prime q = 2^23 - 2^13 + 1 */
+
+/* ML-DSA parameters that the tests use */
+static const struct {
+	int sig_len;
+	int pk_len;
+	int k;
+	int lambda;
+	int gamma1;
+	int beta;
+	int omega;
+} params[] = {
+	[MLDSA44] = {
+		.sig_len = MLDSA44_SIGNATURE_SIZE,
+		.pk_len = MLDSA44_PUBLIC_KEY_SIZE,
+		.k = 4,
+		.lambda = 128,
+		.gamma1 = 1 << 17,
+		.beta = 78,
+		.omega = 80,
+	},
+	[MLDSA65] = {
+		.sig_len = MLDSA65_SIGNATURE_SIZE,
+		.pk_len = MLDSA65_PUBLIC_KEY_SIZE,
+		.k = 6,
+		.lambda = 192,
+		.gamma1 = 1 << 19,
+		.beta = 196,
+		.omega = 55,
+	},
+	[MLDSA87] = {
+		.sig_len = MLDSA87_SIGNATURE_SIZE,
+		.pk_len = MLDSA87_PUBLIC_KEY_SIZE,
+		.k = 8,
+		.lambda = 256,
+		.gamma1 = 1 << 19,
+		.beta = 120,
+		.omega = 75,
+	},
+};
+
+#include "mldsa-testvecs.h"
+
+static void do_mldsa_and_assert_success(struct kunit *test,
+					const struct mldsa_testvector *tv)
+{
+	int err = mldsa_verify(tv->alg, tv->sig, tv->sig_len, tv->msg,
+			       tv->msg_len, tv->pk, tv->pk_len);
+	KUNIT_ASSERT_EQ(test, err, 0);
+}
+
+static u8 *kunit_kmemdup_or_fail(struct kunit *test, const u8 *src, size_t len)
+{
+	u8 *dst = kunit_kmalloc(test, len, GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, dst);
+	return memcpy(dst, src, len);
+}
+
+/*
+ * Test that changing coefficients in a valid signature's z vector results in
+ * the following behavior from mldsa_verify():
+ *
+ *  * -EBADMSG if a coefficient is changed to have an out-of-range value, i.e.
+ *    absolute value >= gamma1 - beta, corresponding to the verifier detecting
+ *    the out-of-range coefficient and rejecting the signature as malformed
+ *
+ *  * -EKEYREJECTED if a coefficient is changed to a different in-range value,
+ *    i.e. absolute value < gamma1 - beta, corresponding to the verifier
+ *    continuing to the "real" signature check and that check failing
+ */
+static void test_mldsa_z_range(struct kunit *test,
+			       const struct mldsa_testvector *tv)
+{
+	u8 *sig = kunit_kmemdup_or_fail(test, tv->sig, tv->sig_len);
+	const int lambda = params[tv->alg].lambda;
+	const s32 gamma1 = params[tv->alg].gamma1;
+	const int beta = params[tv->alg].beta;
+	/*
+	 * We just modify the first coefficient.  The coefficient is gamma1
+	 * minus either the first 18 or 20 bits of the u32, depending on gamma1.
+	 *
+	 * The layout of ML-DSA signatures is ctilde || z || h.  ctilde is
+	 * lambda / 4 bytes, so z starts at &sig[lambda / 4].
+	 */
+	u8 *z_ptr = &sig[lambda / 4];
+	const u32 z_data = get_unaligned_le32(z_ptr);
+	const u32 mask = (gamma1 << 1) - 1;
+	/* These are the four boundaries of the out-of-range values. */
+	const s32 out_of_range_coeffs[] = {
+		-gamma1 + 1,
+		-(gamma1 - beta),
+		gamma1,
+		gamma1 - beta,
+	};
+	/*
+	 * These are the two boundaries of the valid range, along with 0.  We
+	 * assume that none of these matches the original coefficient.
+	 */
+	const s32 in_range_coeffs[] = {
+		-(gamma1 - beta - 1),
+		0,
+		gamma1 - beta - 1,
+	};
+
+	/* Initially the signature is valid. */
+	do_mldsa_and_assert_success(test, tv);
+
+	/* Test some out-of-range coefficients. */
+	for (int i = 0; i < ARRAY_SIZE(out_of_range_coeffs); i++) {
+		const s32 c = out_of_range_coeffs[i];
+
+		put_unaligned_le32((z_data & ~mask) | (mask & (gamma1 - c)),
+				   z_ptr);
+		KUNIT_ASSERT_EQ(test, -EBADMSG,
+				mldsa_verify(tv->alg, sig, tv->sig_len, tv->msg,
+					     tv->msg_len, tv->pk, tv->pk_len));
+	}
+
+	/* Test some in-range coefficients. */
+	for (int i = 0; i < ARRAY_SIZE(in_range_coeffs); i++) {
+		const s32 c = in_range_coeffs[i];
+
+		put_unaligned_le32((z_data & ~mask) | (mask & (gamma1 - c)),
+				   z_ptr);
+		KUNIT_ASSERT_EQ(test, -EKEYREJECTED,
+				mldsa_verify(tv->alg, sig, tv->sig_len, tv->msg,
+					     tv->msg_len, tv->pk, tv->pk_len));
+	}
+}
+
+/* Test that mldsa_verify() rejects malformed hint vectors with -EBADMSG. */
+static void test_mldsa_bad_hints(struct kunit *test,
+				 const struct mldsa_testvector *tv)
+{
+	const int omega = params[tv->alg].omega;
+	const int k = params[tv->alg].k;
+	u8 *sig = kunit_kmemdup_or_fail(test, tv->sig, tv->sig_len);
+	/* Pointer to the encoded hint vector in the signature */
+	u8 *hintvec = &sig[tv->sig_len - omega - k];
+	u8 h;
+
+	/* Initially the signature is valid. */
+	do_mldsa_and_assert_success(test, tv);
+
+	/* Cumulative hint count exceeds omega */
+	memcpy(sig, tv->sig, tv->sig_len);
+	hintvec[omega + k - 1] = omega + 1;
+	KUNIT_ASSERT_EQ(test, -EBADMSG,
+			mldsa_verify(tv->alg, sig, tv->sig_len, tv->msg,
+				     tv->msg_len, tv->pk, tv->pk_len));
+
+	/* Cumulative hint count decreases */
+	memcpy(sig, tv->sig, tv->sig_len);
+	KUNIT_ASSERT_GE(test, hintvec[omega + k - 2], 1);
+	hintvec[omega + k - 1] = hintvec[omega + k - 2] - 1;
+	KUNIT_ASSERT_EQ(test, -EBADMSG,
+			mldsa_verify(tv->alg, sig, tv->sig_len, tv->msg,
+				     tv->msg_len, tv->pk, tv->pk_len));
+
+	/*
+	 * Hint indices out of order.  To test this, swap hintvec[0] and
+	 * hintvec[1].  This assumes that the original valid signature had at
+	 * least two nonzero hints in the first element (asserted below).
+	 */
+	memcpy(sig, tv->sig, tv->sig_len);
+	KUNIT_ASSERT_GE(test, hintvec[omega], 2);
+	h = hintvec[0];
+	hintvec[0] = hintvec[1];
+	hintvec[1] = h;
+	KUNIT_ASSERT_EQ(test, -EBADMSG,
+			mldsa_verify(tv->alg, sig, tv->sig_len, tv->msg,
+				     tv->msg_len, tv->pk, tv->pk_len));
+
+	/*
+	 * Extra hint indices given.  For this test to work, the original valid
+	 * signature must have fewer than omega nonzero hints (asserted below).
+	 */
+	memcpy(sig, tv->sig, tv->sig_len);
+	KUNIT_ASSERT_LT(test, hintvec[omega + k - 1], omega);
+	hintvec[omega - 1] = 0xff;
+	KUNIT_ASSERT_EQ(test, -EBADMSG,
+			mldsa_verify(tv->alg, sig, tv->sig_len, tv->msg,
+				     tv->msg_len, tv->pk, tv->pk_len));
+}
+
+static void test_mldsa_mutation(struct kunit *test,
+				const struct mldsa_testvector *tv)
+{
+	const int sig_len = tv->sig_len;
+	const int msg_len = tv->msg_len;
+	const int pk_len = tv->pk_len;
+	const int num_iter = 200;
+	u8 *sig = kunit_kmemdup_or_fail(test, tv->sig, sig_len);
+	u8 *msg = kunit_kmemdup_or_fail(test, tv->msg, msg_len);
+	u8 *pk = kunit_kmemdup_or_fail(test, tv->pk, pk_len);
+
+	/* Initially the signature is valid. */
+	do_mldsa_and_assert_success(test, tv);
+
+	/* Changing any bit in the signature should invalidate the signature */
+	for (int i = 0; i < num_iter; i++) {
+		size_t pos = get_random_u32_below(sig_len);
+		u8 b = 1 << get_random_u32_below(8);
+
+		sig[pos] ^= b;
+		KUNIT_ASSERT_NE(test, 0,
+				mldsa_verify(tv->alg, sig, sig_len, msg,
+					     msg_len, pk, pk_len));
+		sig[pos] ^= b;
+	}
+
+	/* Changing any bit in the message should invalidate the signature */
+	for (int i = 0; i < num_iter; i++) {
+		size_t pos = get_random_u32_below(msg_len);
+		u8 b = 1 << get_random_u32_below(8);
+
+		msg[pos] ^= b;
+		KUNIT_ASSERT_NE(test, 0,
+				mldsa_verify(tv->alg, sig, sig_len, msg,
+					     msg_len, pk, pk_len));
+		msg[pos] ^= b;
+	}
+
+	/* Changing any bit in the public key should invalidate the signature */
+	for (int i = 0; i < num_iter; i++) {
+		size_t pos = get_random_u32_below(pk_len);
+		u8 b = 1 << get_random_u32_below(8);
+
+		pk[pos] ^= b;
+		KUNIT_ASSERT_NE(test, 0,
+				mldsa_verify(tv->alg, sig, sig_len, msg,
+					     msg_len, pk, pk_len));
+		pk[pos] ^= b;
+	}
+
+	/* All changes should have been undone. */
+	KUNIT_ASSERT_EQ(test, 0,
+			mldsa_verify(tv->alg, sig, sig_len, msg, msg_len, pk,
+				     pk_len));
+}
+
+static void test_mldsa(struct kunit *test, const struct mldsa_testvector *tv)
+{
+	/* Valid signature */
+	KUNIT_ASSERT_EQ(test, tv->sig_len, params[tv->alg].sig_len);
+	KUNIT_ASSERT_EQ(test, tv->pk_len, params[tv->alg].pk_len);
+	do_mldsa_and_assert_success(test, tv);
+
+	/* Signature too short */
+	KUNIT_ASSERT_EQ(test, -EBADMSG,
+			mldsa_verify(tv->alg, tv->sig, tv->sig_len - 1, tv->msg,
+				     tv->msg_len, tv->pk, tv->pk_len));
+
+	/* Signature too long */
+	KUNIT_ASSERT_EQ(test, -EBADMSG,
+			mldsa_verify(tv->alg, tv->sig, tv->sig_len + 1, tv->msg,
+				     tv->msg_len, tv->pk, tv->pk_len));
+
+	/* Public key too short */
+	KUNIT_ASSERT_EQ(test, -EBADMSG,
+			mldsa_verify(tv->alg, tv->sig, tv->sig_len, tv->msg,
+				     tv->msg_len, tv->pk, tv->pk_len - 1));
+
+	/* Public key too long */
+	KUNIT_ASSERT_EQ(test, -EBADMSG,
+			mldsa_verify(tv->alg, tv->sig, tv->sig_len, tv->msg,
+				     tv->msg_len, tv->pk, tv->pk_len + 1));
+
+	/*
+	 * Message too short.  Error is EKEYREJECTED because it gets rejected by
+	 * the "real" signature check rather than the well-formedness checks.
+	 */
+	KUNIT_ASSERT_EQ(test, -EKEYREJECTED,
+			mldsa_verify(tv->alg, tv->sig, tv->sig_len, tv->msg,
+				     tv->msg_len - 1, tv->pk, tv->pk_len));
+	/*
+	 * Can't simply try (tv->msg, tv->msg_len + 1) too, as tv->msg would be
+	 * accessed out of bounds.  However, ML-DSA just hashes the message and
+	 * doesn't handle different message lengths differently anyway.
+	 */
+
+	/* Test the validity checks on the z vector. */
+	test_mldsa_z_range(test, tv);
+
+	/* Test the validity checks on the hint vector. */
+	test_mldsa_bad_hints(test, tv);
+
+	/* Test randomly mutating the inputs. */
+	test_mldsa_mutation(test, tv);
+}
+
+static void test_mldsa44(struct kunit *test)
+{
+	test_mldsa(test, &mldsa44_testvector);
+}
+
+static void test_mldsa65(struct kunit *test)
+{
+	test_mldsa(test, &mldsa65_testvector);
+}
+
+static void test_mldsa87(struct kunit *test)
+{
+	test_mldsa(test, &mldsa87_testvector);
+}
+
+static s32 mod(s32 a, s32 m)
+{
+	a %= m;
+	if (a < 0)
+		a += m;
+	return a;
+}
+
+static s32 symmetric_mod(s32 a, s32 m)
+{
+	a = mod(a, m);
+	if (a > m / 2)
+		a -= m;
+	return a;
+}
+
+/* Mechanical, inefficient translation of FIPS 204 Algorithm 36, Decompose */
+static void decompose_ref(s32 r, s32 gamma2, s32 *r0, s32 *r1)
+{
+	s32 rplus = mod(r, Q);
+
+	*r0 = symmetric_mod(rplus, 2 * gamma2);
+	if (rplus - *r0 == Q - 1) {
+		*r1 = 0;
+		*r0 = *r0 - 1;
+	} else {
+		*r1 = (rplus - *r0) / (2 * gamma2);
+	}
+}
+
+/* Mechanical, inefficient translation of FIPS 204 Algorithm 40, UseHint */
+static s32 use_hint_ref(u8 h, s32 r, s32 gamma2)
+{
+	s32 m = (Q - 1) / (2 * gamma2);
+	s32 r0, r1;
+
+	decompose_ref(r, gamma2, &r0, &r1);
+	if (h == 1 && r0 > 0)
+		return mod(r1 + 1, m);
+	if (h == 1 && r0 <= 0)
+		return mod(r1 - 1, m);
+	return r1;
+}
+
+/*
+ * Test that for all possible inputs, mldsa_use_hint() gives the same output as
+ * a mechanical translation of the pseudocode from FIPS 204.
+ */
+static void test_mldsa_use_hint(struct kunit *test)
+{
+	for (int i = 0; i < 2; i++) {
+		const s32 gamma2 = (Q - 1) / (i == 0 ? 88 : 32);
+
+		for (u8 h = 0; h < 2; h++) {
+			for (s32 r = 0; r < Q; r++) {
+				KUNIT_ASSERT_EQ(test,
+						mldsa_use_hint(h, r, gamma2),
+						use_hint_ref(h, r, gamma2));
+			}
+		}
+	}
+}
+
+static void benchmark_mldsa(struct kunit *test,
+			    const struct mldsa_testvector *tv)
+{
+	const int warmup_niter = 200;
+	const int benchmark_niter = 200;
+	u64 t0, t1;
+
+	if (!IS_ENABLED(CONFIG_CRYPTO_LIB_BENCHMARK))
+		kunit_skip(test, "not enabled");
+
+	for (int i = 0; i < warmup_niter; i++)
+		do_mldsa_and_assert_success(test, tv);
+
+	t0 = ktime_get_ns();
+	for (int i = 0; i < benchmark_niter; i++)
+		do_mldsa_and_assert_success(test, tv);
+	t1 = ktime_get_ns();
+	kunit_info(test, "%llu ops/s",
+		   div64_u64((u64)benchmark_niter * NSEC_PER_SEC,
+			     t1 - t0 ?: 1));
+}
+
+static void benchmark_mldsa44(struct kunit *test)
+{
+	benchmark_mldsa(test, &mldsa44_testvector);
+}
+
+static void benchmark_mldsa65(struct kunit *test)
+{
+	benchmark_mldsa(test, &mldsa65_testvector);
+}
+
+static void benchmark_mldsa87(struct kunit *test)
+{
+	benchmark_mldsa(test, &mldsa87_testvector);
+}
+
+static struct kunit_case mldsa_kunit_cases[] = {
+	KUNIT_CASE(test_mldsa44),
+	KUNIT_CASE(test_mldsa65),
+	KUNIT_CASE(test_mldsa87),
+	KUNIT_CASE(test_mldsa_use_hint),
+	KUNIT_CASE(benchmark_mldsa44),
+	KUNIT_CASE(benchmark_mldsa65),
+	KUNIT_CASE(benchmark_mldsa87),
+	{},
+};
+
+static struct kunit_suite mldsa_kunit_suite = {
+	.name = "mldsa",
+	.test_cases = mldsa_kunit_cases,
+};
+kunit_test_suite(mldsa_kunit_suite);
+
+MODULE_DESCRIPTION("KUnit tests and benchmark for ML-DSA");
+MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
+MODULE_LICENSE("GPL");
diff --git a/lib/crypto/tests/nh-testvecs.h b/lib/crypto/tests/nh-testvecs.h
new file mode 100644
index 000000000000..b393e1f90f31
--- /dev/null
+++ b/lib/crypto/tests/nh-testvecs.h
@@ -0,0 +1,298 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py nh */
+
+static const u8 nh_test_key[NH_KEY_BYTES] = {
+	0x04, 0x59, 0x66, 0x92, 0x81, 0xd7, 0xe9, 0x25,
+	0x68, 0xfa, 0xb0, 0xca, 0x9f, 0xea, 0x98, 0xca,
+	0xcd, 0xbf, 0x6d, 0xa5, 0x0c, 0x22, 0xc3, 0x57,
+	0xdc, 0x35, 0x05, 0xdd, 0x5b, 0xb0, 0xce, 0xf6,
+	0xb2, 0x4c, 0x77, 0x2e, 0xd2, 0x63, 0xf0, 0x17,
+	0x60, 0xd8, 0xd3, 0xd9, 0xed, 0x34, 0xb6, 0xed,
+	0x6a, 0x11, 0xc0, 0x25, 0xda, 0xba, 0x7e, 0xef,
+	0x49, 0x13, 0xf7, 0xd9, 0xfc, 0xb6, 0xfd, 0x58,
+	0xe9, 0x5f, 0xc5, 0xc4, 0x69, 0x89, 0xba, 0xa6,
+	0x2b, 0x58, 0x8d, 0x36, 0x6c, 0xb9, 0x90, 0x1e,
+	0x64, 0xc7, 0x44, 0x84, 0x03, 0x70, 0x30, 0x47,
+	0xdd, 0x58, 0xf4, 0x87, 0x61, 0xfd, 0x9c, 0x6b,
+	0x51, 0x1b, 0x39, 0x1d, 0x6d, 0x50, 0xae, 0x19,
+	0x71, 0x03, 0xc7, 0xa7, 0x42, 0x82, 0x8f, 0xa5,
+	0x63, 0x6a, 0xe2, 0x8a, 0xad, 0x4b, 0x40, 0xa7,
+	0x3f, 0x8b, 0xe4, 0xae, 0xb2, 0x8a, 0x14, 0x78,
+	0x91, 0x07, 0xba, 0x02, 0x08, 0xc1, 0x34, 0xb8,
+	0xda, 0x61, 0x67, 0xf6, 0x98, 0x97, 0x1a, 0xcb,
+	0x0f, 0x82, 0x80, 0xff, 0x02, 0x54, 0x16, 0x57,
+	0x18, 0x35, 0xaf, 0x16, 0x17, 0x68, 0xcc, 0xc7,
+	0x52, 0xac, 0x31, 0x39, 0x60, 0xe4, 0xb4, 0xcb,
+	0x0e, 0xf9, 0x57, 0xe9, 0x96, 0xff, 0x99, 0xd6,
+	0x10, 0x96, 0x09, 0xab, 0x28, 0x92, 0x1b, 0x9f,
+	0x10, 0xde, 0x3e, 0x87, 0xb8, 0x9d, 0x2d, 0xa0,
+	0x3c, 0x91, 0x85, 0x8c, 0x9e, 0xc0, 0x97, 0x9a,
+	0xb4, 0x54, 0x7f, 0x4a, 0x63, 0xc2, 0x75, 0x0f,
+	0x0d, 0x2f, 0x62, 0x56, 0x48, 0x0e, 0xb6, 0xc7,
+	0xcf, 0x0d, 0x78, 0xca, 0xbd, 0x31, 0x9e, 0x4c,
+	0xf7, 0x3f, 0x9e, 0xc2, 0xea, 0x5e, 0x44, 0x6d,
+	0x76, 0xf9, 0xc5, 0xe0, 0x29, 0xea, 0x15, 0xbf,
+	0xaf, 0xd4, 0x75, 0xc8, 0x89, 0xcf, 0x4f, 0x17,
+	0xfd, 0x4a, 0x45, 0xa5, 0x4d, 0x2d, 0x87, 0x11,
+	0x2b, 0x3e, 0x64, 0xa2, 0x6b, 0xc5, 0x23, 0x8c,
+	0xfa, 0x71, 0x13, 0x72, 0x0e, 0x7c, 0xe1, 0x2c,
+	0x9f, 0x0e, 0x29, 0xc9, 0x15, 0xde, 0x4e, 0xd7,
+	0x42, 0x1f, 0x8e, 0xe1, 0x91, 0x99, 0x50, 0x38,
+	0x7f, 0x15, 0xc0, 0xf6, 0x4b, 0xfd, 0x9d, 0x40,
+	0xe9, 0x44, 0x51, 0xca, 0x3b, 0x83, 0x41, 0x9f,
+	0x82, 0x64, 0x66, 0x22, 0x12, 0x43, 0x1c, 0x4f,
+	0x45, 0x11, 0x3a, 0x46, 0xb1, 0x7c, 0x62, 0x0a,
+	0x9d, 0x4c, 0x99, 0x85, 0xb0, 0x10, 0x19, 0xcf,
+	0xeb, 0xf9, 0x65, 0xaf, 0xd8, 0x05, 0x9e, 0x61,
+	0x03, 0x5f, 0x15, 0x99, 0xa9, 0x05, 0x20, 0xc8,
+	0xaf, 0xab, 0x31, 0x9d, 0xd5, 0xdf, 0x24, 0xce,
+	0x2b, 0x6d, 0xd7, 0x17, 0xc3, 0x04, 0xff, 0x82,
+	0xa7, 0x18, 0x39, 0xe9, 0x0d, 0x0a, 0x5f, 0xb9,
+	0xc9, 0x86, 0x1d, 0xf8, 0x02, 0x2d, 0xc3, 0x88,
+	0x28, 0x73, 0x5c, 0xac, 0x25, 0xc9, 0xfe, 0xcb,
+	0xd2, 0xfd, 0x63, 0x74, 0xac, 0xe1, 0xb8, 0xa2,
+	0xc6, 0x2b, 0xb5, 0x40, 0x01, 0x9b, 0xed, 0xee,
+	0x7b, 0x63, 0x66, 0x05, 0x45, 0xc2, 0x6c, 0xd8,
+	0x58, 0xf1, 0xa1, 0x3d, 0xc8, 0x43, 0x59, 0x4b,
+	0x39, 0x87, 0x24, 0x64, 0x92, 0xb0, 0xab, 0x75,
+	0xf1, 0xb7, 0xbf, 0x7c, 0xde, 0xc0, 0xaf, 0x4a,
+	0xc2, 0x7b, 0xd9, 0x8a, 0x99, 0xcd, 0x83, 0x01,
+	0xe6, 0xae, 0xeb, 0x16, 0xe7, 0x54, 0x9c, 0x95,
+	0x0a, 0x91, 0x02, 0xaf, 0x9f, 0x79, 0x40, 0x45,
+	0xce, 0x47, 0x41, 0x65, 0xca, 0x80, 0x0d, 0x14,
+	0x46, 0x58, 0x5d, 0x4d, 0x28, 0x55, 0x70, 0x49,
+	0x7c, 0x32, 0x1f, 0x01, 0xaa, 0x05, 0x2f, 0xf1,
+	0xeb, 0xa3, 0xe6, 0x1d, 0xf9, 0x43, 0xe0, 0x58,
+	0x05, 0x61, 0x22, 0xc3, 0xee, 0xe4, 0x6f, 0x94,
+	0xaf, 0x82, 0xda, 0x18, 0x18, 0x63, 0x9c, 0xfa,
+	0xc0, 0x04, 0x27, 0xc5, 0x39, 0x5e, 0x7a, 0xa6,
+	0x85, 0x46, 0xb7, 0x76, 0xc9, 0x16, 0xf2, 0xf8,
+	0x40, 0x8d, 0x4b, 0x5e, 0x72, 0xf3, 0x3e, 0x12,
+	0xa4, 0x80, 0x39, 0xb2, 0x92, 0xfe, 0x6e, 0x5b,
+	0x5b, 0xad, 0xea, 0x29, 0xbc, 0x66, 0xe6, 0xfe,
+	0x80, 0x02, 0x5d, 0x83, 0x37, 0xfc, 0xde, 0x6c,
+	0x25, 0x54, 0xa2, 0xff, 0x7d, 0xb6, 0xe1, 0xd6,
+	0xcf, 0xdb, 0x60, 0xe3, 0xbe, 0x2f, 0x4e, 0xb4,
+	0xf5, 0xb4, 0x51, 0xf7, 0x5a, 0x25, 0xda, 0x40,
+	0x84, 0x5e, 0xc0, 0x0a, 0x6b, 0xfa, 0x0c, 0xfb,
+	0x5e, 0x3e, 0x12, 0x6c, 0x39, 0x35, 0xc0, 0x28,
+	0xd6, 0x1b, 0x3a, 0x72, 0xc3, 0xfe, 0xa5, 0x4c,
+	0x35, 0xa2, 0x42, 0xf6, 0x3d, 0xa5, 0xbf, 0xb5,
+	0x39, 0xe3, 0xc9, 0xd5, 0x8c, 0x1b, 0xe5, 0xef,
+	0x91, 0xd2, 0x80, 0x6f, 0xcc, 0x77, 0x44, 0x50,
+	0x62, 0xc7, 0xac, 0x29, 0xcb, 0x72, 0xda, 0x6d,
+	0xc5, 0xfe, 0xa7, 0xee, 0x8b, 0xeb, 0xfc, 0xa3,
+	0x46, 0x18, 0x5f, 0xaa, 0xc3, 0x65, 0xd0, 0x8f,
+	0x67, 0x98, 0xd6, 0xce, 0x5f, 0x84, 0xd4, 0x96,
+	0x1b, 0x67, 0xa0, 0xcf, 0xfc, 0x94, 0x55, 0x5e,
+	0x4b, 0x51, 0x68, 0xa7, 0x6d, 0x02, 0xf9, 0x53,
+	0x54, 0x86, 0x6b, 0x53, 0x39, 0xe0, 0x36, 0x23,
+	0x87, 0x1a, 0xfb, 0x53, 0x1a, 0x65, 0xd8, 0x42,
+	0xa8, 0x85, 0xfd, 0x2c, 0x7f, 0x6b, 0x7f, 0x67,
+	0x70, 0x23, 0x6c, 0xe9, 0x0b, 0xf0, 0x1e, 0x0d,
+	0x0b, 0xb4, 0xd4, 0x96, 0x14, 0x95, 0x7e, 0xf3,
+	0x9b, 0xdd, 0xd7, 0xc4, 0x24, 0x22, 0xb9, 0x9d,
+	0xb3, 0xa6, 0xac, 0x09, 0x7c, 0x00, 0xbf, 0xd0,
+	0xdc, 0xfb, 0x9b, 0x7c, 0x8c, 0xbd, 0xd4, 0x1a,
+	0x13, 0x2b, 0x82, 0x3d, 0x7c, 0x8c, 0x10, 0x47,
+	0x49, 0x6c, 0x53, 0xeb, 0xa7, 0xc2, 0xde, 0xed,
+	0xe2, 0x55, 0x93, 0x2c, 0x1a, 0x5a, 0x7d, 0xe1,
+	0x37, 0x62, 0xdd, 0x29, 0x1a, 0x72, 0x82, 0xc0,
+	0x14, 0x73, 0x5d, 0x0e, 0x9b, 0xcc, 0x54, 0x68,
+	0x3a, 0x4d, 0x56, 0x8f, 0xc9, 0x4e, 0xaf, 0x7b,
+	0xde, 0x17, 0x9c, 0x5e, 0x83, 0x82, 0x22, 0xe3,
+	0x28, 0xdf, 0x1b, 0xb6, 0xdb, 0x17, 0x90, 0x48,
+	0xb5, 0x13, 0x4e, 0xd3, 0x97, 0x5e, 0xb3, 0x9c,
+	0x16, 0x08, 0xc8, 0x77, 0xb3, 0xcd, 0x94, 0x90,
+	0x4f, 0x77, 0xaf, 0x67, 0xdd, 0x80, 0x15, 0x1c,
+	0x59, 0xfb, 0x3c, 0xec, 0xf8, 0xb3, 0x67, 0xfb,
+	0xa0, 0x94, 0x3c, 0x53, 0x99, 0x49, 0x94, 0x2c,
+	0x85, 0x26, 0x92, 0x6d, 0x8d, 0x48, 0xf6, 0x72,
+	0xdd, 0xfb, 0xb2, 0x10, 0x51, 0x5b, 0xbe, 0xd5,
+	0x70, 0x3d, 0x28, 0x94, 0x98, 0x4f, 0x6e, 0x20,
+	0x7b, 0x7d, 0x0f, 0x56, 0xc9, 0x96, 0x5f, 0x60,
+	0x2e, 0x2f, 0x9b, 0x38, 0x7f, 0xc7, 0x3c, 0x6b,
+	0x2f, 0x2b, 0x8f, 0x1f, 0x07, 0x1c, 0x85, 0x57,
+	0x16, 0x2e, 0xc7, 0x74, 0xe5, 0xf2, 0x0d, 0xfe,
+	0xef, 0x57, 0xb0, 0xa4, 0x4f, 0x4c, 0x7d, 0x81,
+	0xbb, 0xaa, 0xcb, 0xa0, 0xb0, 0x51, 0xcf, 0xc2,
+	0xee, 0x90, 0x2e, 0x5e, 0x27, 0xca, 0xd3, 0xe8,
+	0xf3, 0x55, 0x02, 0x56, 0x06, 0xa5, 0xad, 0xdf,
+	0xa3, 0xa9, 0x06, 0x05, 0x53, 0x74, 0x55, 0xd5,
+	0xd2, 0x20, 0x0a, 0x6d, 0x4a, 0xef, 0x16, 0xbf,
+	0xc3, 0xb2, 0x75, 0x93, 0xd8, 0x6e, 0x0f, 0xd2,
+	0xae, 0x3b, 0xc0, 0x00, 0x22, 0x6f, 0xb5, 0x0a,
+	0x41, 0xfc, 0xf9, 0x41, 0xfc, 0x16, 0x4f, 0xa6,
+	0x1c, 0x18, 0x41, 0x67, 0x73, 0xa8, 0x79, 0xa9,
+	0x54, 0x18, 0x4e, 0x88, 0x44, 0x0f, 0xa1, 0x5b,
+	0xf0, 0x68, 0xea, 0x3c, 0x62, 0x59, 0x8d, 0xc7,
+	0x6f, 0xd7, 0x72, 0x20, 0x74, 0x39, 0xd4, 0x3a,
+	0x41, 0x1b, 0x58, 0x57, 0x54, 0x85, 0x60, 0xca,
+	0x49, 0x4b, 0xa1, 0x04, 0x91, 0xb6, 0xf2, 0xcd,
+	0x62, 0x63, 0x67, 0xd1, 0xee, 0x6b, 0x9e, 0x5d,
+	0xd6, 0xc4, 0x58, 0x6b, 0xe1, 0xe6, 0x4a, 0xdb,
+	0xe8, 0xb1, 0x35, 0x03, 0x15, 0x8d, 0x34, 0x69,
+	0x4c, 0xd2, 0x54, 0xce, 0xe8, 0x6a, 0x69, 0x6f,
+	0xaa, 0xb5, 0x1f, 0x86, 0xed, 0xac, 0x4f, 0x16,
+	0x1e, 0x48, 0x93, 0xe8, 0x6c, 0x24, 0x1c, 0xd0,
+	0xbb, 0x61, 0xc2, 0x34, 0xdd, 0xc9, 0x5c, 0xce,
+};
+
+static const u8 nh_test_msg[NH_MESSAGE_BYTES] = {
+	0x99, 0x57, 0x61, 0x41, 0xad, 0x08, 0x7e, 0x17,
+	0xd4, 0xef, 0x0b, 0x23, 0xff, 0x0b, 0x96, 0x0a,
+	0x6c, 0x98, 0xac, 0x78, 0x5e, 0xb6, 0xb2, 0x67,
+	0x0f, 0x48, 0xf4, 0xa1, 0xe5, 0x1e, 0xfe, 0x83,
+	0xe4, 0x56, 0x2a, 0x03, 0x64, 0xff, 0x7a, 0xf3,
+	0x03, 0xfe, 0xa7, 0x86, 0xdc, 0x35, 0x79, 0x13,
+	0xf8, 0xe1, 0x59, 0x19, 0x04, 0x43, 0x24, 0x82,
+	0x44, 0x82, 0x41, 0x2b, 0xc7, 0xcf, 0xf5, 0xa4,
+	0xdc, 0xca, 0xf5, 0x34, 0xc4, 0x23, 0x3c, 0x1f,
+	0xa8, 0x84, 0x1f, 0x2a, 0xcd, 0xae, 0x9d, 0x5e,
+	0x05, 0xe2, 0xfb, 0x0c, 0x68, 0x81, 0x90, 0x11,
+	0x44, 0xf6, 0xdd, 0x5b, 0x51, 0xd3, 0xe0, 0xab,
+	0x29, 0x3a, 0xa9, 0x9c, 0xf6, 0x7e, 0x2d, 0xe3,
+	0x6c, 0x09, 0x59, 0xd7, 0xfa, 0x7f, 0x6a, 0x33,
+	0x3b, 0x23, 0x7b, 0x1b, 0xb2, 0x79, 0x5f, 0x5c,
+	0xb6, 0x2d, 0xb0, 0xf8, 0xab, 0x33, 0x28, 0xe0,
+	0x72, 0x2e, 0x2f, 0x03, 0x22, 0x16, 0xb4, 0x87,
+	0xf7, 0x14, 0x3f, 0x55, 0x8a, 0xb0, 0x47, 0xdb,
+	0x42, 0x2d, 0xc0, 0x0c, 0x0a, 0x33, 0xf8, 0xab,
+	0x44, 0xae, 0xa3, 0xc9, 0xfc, 0xf6, 0x34, 0x8c,
+	0x60, 0x30, 0x6d, 0x31, 0x70, 0xf3, 0x39, 0x53,
+	0xf1, 0x2d, 0xb9, 0x6c, 0xa6, 0x48, 0x9c, 0x9c,
+	0xc2, 0x88, 0xb3, 0xa9, 0x98, 0xb6, 0xc3, 0x47,
+	0x94, 0x02, 0x9d, 0x98, 0x6e, 0x25, 0x6c, 0xf5,
+	0x9b, 0xc6, 0x4d, 0xee, 0x07, 0x1e, 0x25, 0x8f,
+	0x01, 0xde, 0xad, 0xe5, 0x77, 0x4f, 0xd1, 0xc0,
+	0x62, 0xbb, 0x3a, 0xb9, 0x83, 0x0b, 0x29, 0x76,
+	0x4f, 0xb1, 0x86, 0x2c, 0x27, 0xc7, 0x38, 0x65,
+	0xcb, 0x78, 0xb7, 0x02, 0x10, 0x9e, 0xde, 0x83,
+	0xd1, 0xac, 0x05, 0x86, 0x23, 0xce, 0x4f, 0x8d,
+	0xcc, 0x4e, 0x3f, 0x04, 0xf4, 0x39, 0x91, 0x81,
+	0x1c, 0x42, 0x47, 0x4d, 0x50, 0xe5, 0x01, 0x22,
+	0x98, 0xcf, 0x91, 0x36, 0xb3, 0x7c, 0xcf, 0x78,
+	0x07, 0x22, 0xa9, 0x18, 0xd2, 0xcd, 0x7d, 0x4d,
+	0xa6, 0xcb, 0xaa, 0x52, 0x13, 0x49, 0x64, 0xb0,
+	0xa5, 0x3d, 0xc7, 0xc3, 0x10, 0x87, 0x2e, 0x76,
+	0xa9, 0x52, 0xc5, 0x50, 0x18, 0xc0, 0x5d, 0xb4,
+	0x4c, 0xc6, 0x7f, 0x64, 0xae, 0x53, 0xc3, 0x46,
+	0x99, 0xb7, 0x61, 0x6b, 0x08, 0x43, 0x08, 0x4c,
+	0x90, 0x2c, 0xee, 0x56, 0x91, 0xb4, 0x28, 0xa8,
+	0xa8, 0x8b, 0x3b, 0x1a, 0x67, 0x71, 0xf2, 0x81,
+	0x48, 0x20, 0x71, 0x30, 0xdd, 0x69, 0x8a, 0xc2,
+	0x4c, 0x9d, 0x4e, 0x17, 0xfb, 0x2e, 0xe7, 0x9b,
+	0x86, 0x94, 0xa5, 0xce, 0xf9, 0x74, 0x56, 0xff,
+	0x3b, 0xff, 0xd9, 0x5a, 0xc8, 0x98, 0xf5, 0x25,
+	0xa2, 0xb9, 0x66, 0x46, 0x89, 0x17, 0x39, 0x08,
+	0x69, 0x03, 0x59, 0x1e, 0x13, 0x12, 0x68, 0xe7,
+	0x2f, 0x00, 0xd3, 0xf3, 0x71, 0xd1, 0x20, 0xc5,
+	0x0b, 0x38, 0x89, 0xda, 0x62, 0x3c, 0xce, 0xea,
+	0x04, 0x19, 0x47, 0x6d, 0xd8, 0x64, 0x38, 0x60,
+	0x96, 0x71, 0x68, 0x48, 0x79, 0xf8, 0xf4, 0x76,
+	0x33, 0xf6, 0x60, 0x8d, 0x21, 0xd0, 0xee, 0x41,
+	0xc0, 0xbe, 0x33, 0x61, 0x5e, 0x66, 0xe6, 0x16,
+	0x14, 0xc7, 0xfb, 0x6c, 0xf3, 0x58, 0xef, 0x12,
+	0x7c, 0x70, 0x65, 0x5d, 0x55, 0xe8, 0xf2, 0x92,
+	0x3a, 0xfe, 0x34, 0x64, 0x31, 0x7c, 0x29, 0xbb,
+	0x01, 0x18, 0xbd, 0xb6, 0xe4, 0x1e, 0xa4, 0xf3,
+	0x7b, 0x4c, 0x6a, 0x0d, 0x01, 0xfc, 0xc7, 0x66,
+	0xc3, 0x88, 0x37, 0x25, 0xcf, 0xe9, 0xca, 0x82,
+	0xeb, 0xa1, 0x38, 0x40, 0xc9, 0xdb, 0x38, 0x7b,
+	0x78, 0xcf, 0x11, 0xa3, 0x1c, 0x6b, 0x70, 0xc8,
+	0xe1, 0x2f, 0x7c, 0x17, 0x2c, 0x58, 0x28, 0xa4,
+	0x13, 0x40, 0xc7, 0x69, 0x0f, 0x04, 0xe5, 0x8e,
+	0xf0, 0x67, 0x53, 0xea, 0x10, 0xf5, 0x83, 0xc9,
+	0xcb, 0x6b, 0x16, 0xef, 0x2e, 0x55, 0xb3, 0xdd,
+	0xed, 0xf9, 0x1a, 0x52, 0x9a, 0x73, 0x78, 0x14,
+	0x14, 0x21, 0xfc, 0xef, 0x3c, 0x40, 0xa9, 0xfe,
+	0xef, 0xd7, 0x6e, 0x28, 0x2f, 0xd3, 0x73, 0xed,
+	0xa3, 0x73, 0xb5, 0x62, 0x41, 0xe6, 0xd4, 0x79,
+	0x49, 0x31, 0x2b, 0x86, 0x74, 0x56, 0x21, 0xfe,
+	0x6d, 0xb2, 0xbe, 0x81, 0x80, 0xa6, 0x81, 0x19,
+	0x90, 0x79, 0x6f, 0xc4, 0x4e, 0x7d, 0x6f, 0x2f,
+	0xa8, 0x6f, 0xd5, 0xc4, 0x7e, 0x23, 0x3b, 0xe6,
+	0x9b, 0x60, 0x97, 0x7b, 0xe2, 0x08, 0x8a, 0xaa,
+	0xc7, 0x7c, 0xf6, 0xe5, 0x01, 0x3e, 0xd2, 0x29,
+	0x7d, 0xd7, 0x40, 0x84, 0x95, 0xfa, 0xdf, 0xd8,
+	0x81, 0xe9, 0x5e, 0xdd, 0x0d, 0x17, 0x51, 0x6b,
+	0x8c, 0x0e, 0x47, 0xf9, 0x0c, 0x92, 0x1b, 0x60,
+	0xca, 0x06, 0x8a, 0xe5, 0xe8, 0x0f, 0x06, 0x75,
+	0x5d, 0x76, 0xc9, 0x32, 0x2c, 0x52, 0x2c, 0x2e,
+	0xd8, 0x66, 0x38, 0x75, 0x16, 0xc7, 0x7d, 0x51,
+	0xc4, 0xc2, 0x22, 0xc8, 0x19, 0xfc, 0x3d, 0x69,
+	0x1e, 0xd9, 0x64, 0x47, 0x5d, 0x21, 0x84, 0x46,
+	0xd7, 0xe1, 0xf0, 0x95, 0x3a, 0x8f, 0xbd, 0x7a,
+	0x53, 0x71, 0x4c, 0x54, 0xc1, 0x3e, 0x27, 0xde,
+	0xeb, 0x04, 0x11, 0xb0, 0x33, 0x4d, 0x57, 0x0b,
+	0x6b, 0x7d, 0x6c, 0xd5, 0x87, 0x7e, 0xb4, 0xe2,
+	0x94, 0x9e, 0x9f, 0x74, 0xe8, 0xb7, 0xfa, 0x05,
+	0x9b, 0x8f, 0x81, 0x43, 0x35, 0x82, 0xb8, 0x5b,
+	0xa8, 0x5e, 0xfa, 0x7a, 0x80, 0x8d, 0xd2, 0x90,
+	0x58, 0x79, 0x89, 0x56, 0x90, 0x2b, 0xff, 0x92,
+	0x3c, 0x35, 0xbe, 0x99, 0x5f, 0xd2, 0x4b, 0x15,
+	0x58, 0x4b, 0xbf, 0x08, 0x9b, 0x9b, 0x97, 0x10,
+	0xa4, 0x55, 0xc7, 0xec, 0x29, 0xc5, 0x14, 0x3e,
+	0x8f, 0x56, 0xa3, 0x92, 0x9e, 0x33, 0xcc, 0x9e,
+	0x77, 0x2f, 0x33, 0xcb, 0xc4, 0xe9, 0x19, 0xf4,
+	0x32, 0x2b, 0xef, 0x6c, 0x1c, 0x92, 0x2c, 0x45,
+	0x88, 0x74, 0x5f, 0xcf, 0x56, 0xfd, 0x87, 0x5f,
+	0xb6, 0x9b, 0xa2, 0x51, 0xda, 0x9b, 0x83, 0x4f,
+	0xec, 0x14, 0xe8, 0xd2, 0x42, 0x03, 0xcb, 0xe8,
+	0xd0, 0xb7, 0xf8, 0x38, 0xde, 0x6f, 0xdf, 0x43,
+	0xfa, 0x41, 0xab, 0xec, 0x2e, 0x3c, 0x93, 0x39,
+	0x76, 0xd1, 0x6f, 0x5b, 0x6c, 0x6e, 0x8d, 0xeb,
+	0x45, 0x6b, 0xc5, 0x76, 0x00, 0x29, 0xca, 0x3b,
+	0xdb, 0x78, 0xc2, 0x32, 0x09, 0x39, 0x19, 0x50,
+	0xa2, 0x44, 0x92, 0x09, 0xdb, 0x8b, 0x9e, 0x16,
+	0x76, 0x7f, 0xf1, 0x78, 0x7b, 0xb2, 0x51, 0xbc,
+	0x28, 0xbd, 0xb0, 0x7f, 0x25, 0x63, 0x7d, 0x34,
+	0xfb, 0xf6, 0x36, 0x24, 0xc7, 0xf9, 0x41, 0xb6,
+	0x2a, 0x06, 0xfc, 0xf0, 0x83, 0xf2, 0x12, 0x3d,
+	0x60, 0x2e, 0x10, 0x70, 0x31, 0x6f, 0x37, 0x08,
+	0x3e, 0x91, 0x93, 0xb5, 0xda, 0xb8, 0x4c, 0x1b,
+	0xd8, 0xb8, 0x3b, 0xd5, 0x3e, 0xb6, 0xc0, 0xbb,
+	0x38, 0x0f, 0xd2, 0x68, 0x4f, 0x78, 0x56, 0xf6,
+	0xda, 0x65, 0xb4, 0x0b, 0xb4, 0xaf, 0xa8, 0x19,
+	0x2f, 0x70, 0x55, 0xe0, 0x47, 0x31, 0x9f, 0x37,
+	0x1a, 0x47, 0xb9, 0x0c, 0x97, 0x79, 0xfc, 0xa9,
+	0x76, 0xe6, 0xfa, 0x38, 0x67, 0x25, 0xd3, 0x89,
+	0x8d, 0xad, 0xc6, 0x11, 0x2d, 0x77, 0x0b, 0x35,
+	0xa2, 0xe2, 0xdf, 0xc8, 0x94, 0xd5, 0xdf, 0xd2,
+	0x69, 0x2a, 0x99, 0x93, 0xfa, 0x4a, 0x5f, 0xc7,
+	0x8a, 0x14, 0x5f, 0x2a, 0xf3, 0x02, 0xf0, 0x3e,
+	0x21, 0x8e, 0x2e, 0x4b, 0xc4, 0xd2, 0xc8, 0xa6,
+	0x41, 0x6e, 0x17, 0x36, 0xe9, 0xad, 0x73, 0x33,
+	0x6c, 0xea, 0xc2, 0x31, 0x8f, 0x30, 0x51, 0x5c,
+	0x1c, 0x20, 0xe6, 0x05, 0x1a, 0x17, 0x15, 0x5d,
+	0x3e, 0x8f, 0xd2, 0x7f, 0xa1, 0xc5, 0x47, 0xb3,
+	0xb2, 0x9c, 0xe8, 0xf0, 0x6d, 0xc1, 0xc3, 0xa2,
+};
+
+static const u8 nh_test_val16[NH_HASH_BYTES] = {
+	0x30, 0x77, 0x55, 0x7c, 0x45, 0xd8, 0xce, 0xf7,
+	0x2a, 0xb5, 0x14, 0x8c, 0x35, 0x7e, 0xaa, 0x00,
+	0x50, 0xbc, 0x50, 0x7c, 0xd3, 0x20, 0x7c, 0x9c,
+	0xb4, 0xf1, 0x91, 0x26, 0x81, 0x03, 0xa5, 0x68,
+};
+
+static const u8 nh_test_val96[NH_HASH_BYTES] = {
+	0xd2, 0x19, 0xca, 0xa5, 0x6c, 0x0c, 0xdf, 0x2f,
+	0x69, 0xfa, 0x75, 0xc1, 0x63, 0xdb, 0xfa, 0x4d,
+	0x45, 0x2b, 0xb8, 0xdb, 0xac, 0xee, 0x61, 0xc6,
+	0x7a, 0x83, 0xb6, 0x0f, 0x32, 0x82, 0xe4, 0xd0,
+};
+
+static const u8 nh_test_val256[NH_HASH_BYTES] = {
+	0x33, 0x8f, 0xb4, 0x96, 0xf1, 0xb6, 0xf1, 0xb5,
+	0x05, 0x19, 0xbb, 0x6b, 0xda, 0xd9, 0x95, 0x75,
+	0x96, 0x3f, 0x8b, 0x42, 0xb6, 0xcd, 0xb7, 0xb7,
+	0xe7, 0x97, 0xb5, 0xa9, 0x0b, 0xd7, 0xdd, 0x33,
+};
+
+static const u8 nh_test_val1024[NH_HASH_BYTES] = {
+	0x32, 0x3d, 0x51, 0xe1, 0x77, 0xb6, 0xac, 0x06,
+	0x84, 0x67, 0xb7, 0xf2, 0x24, 0xe7, 0xec, 0xfd,
+	0x96, 0x64, 0xff, 0x55, 0xc7, 0x1b, 0xf9, 0xdc,
+	0xa3, 0xc7, 0x32, 0x06, 0x79, 0xcf, 0xca, 0xb6,
+};
diff --git a/lib/crypto/tests/nh_kunit.c b/lib/crypto/tests/nh_kunit.c
new file mode 100644
index 000000000000..a8a3c3f345cb
--- /dev/null
+++ b/lib/crypto/tests/nh_kunit.c
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright 2025 Google LLC
+ */
+#include <crypto/nh.h>
+#include <kunit/test.h>
+#include "nh-testvecs.h"
+
+static void test_nh(struct kunit *test)
+{
+	u32 *key = kunit_kmalloc(test, NH_KEY_BYTES, GFP_KERNEL);
+	__le64 hash[NH_NUM_PASSES];
+
+	KUNIT_ASSERT_NOT_NULL(test, key);
+	memcpy(key, nh_test_key, NH_KEY_BYTES);
+	le32_to_cpu_array(key, NH_KEY_WORDS);
+
+	nh(key, nh_test_msg, 16, hash);
+	KUNIT_ASSERT_MEMEQ(test, hash, nh_test_val16, NH_HASH_BYTES);
+
+	nh(key, nh_test_msg, 96, hash);
+	KUNIT_ASSERT_MEMEQ(test, hash, nh_test_val96, NH_HASH_BYTES);
+
+	nh(key, nh_test_msg, 256, hash);
+	KUNIT_ASSERT_MEMEQ(test, hash, nh_test_val256, NH_HASH_BYTES);
+
+	nh(key, nh_test_msg, 1024, hash);
+	KUNIT_ASSERT_MEMEQ(test, hash, nh_test_val1024, NH_HASH_BYTES);
+}
+
+static struct kunit_case nh_test_cases[] = {
+	KUNIT_CASE(test_nh),
+	{},
+};
+
+static struct kunit_suite nh_test_suite = {
+	.name = "nh",
+	.test_cases = nh_test_cases,
+};
+kunit_test_suite(nh_test_suite);
+
+MODULE_DESCRIPTION("KUnit tests for NH");
+MODULE_LICENSE("GPL");
diff --git a/lib/crypto/tests/polyval-testvecs.h b/lib/crypto/tests/polyval-testvecs.h
new file mode 100644
index 000000000000..3d33f60d58bb
--- /dev/null
+++ b/lib/crypto/tests/polyval-testvecs.h
@@ -0,0 +1,186 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py polyval */
+
+static const struct {
+	size_t data_len;
+	u8 digest[POLYVAL_DIGEST_SIZE];
+} hash_testvecs[] = {
+	{
+		.data_len = 0,
+		.digest = {
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		},
+	},
+	{
+		.data_len = 1,
+		.digest = {
+			0xb5, 0x51, 0x69, 0x89, 0xd4, 0x3c, 0x59, 0xca,
+			0x6a, 0x1c, 0x2a, 0xe9, 0xa1, 0x9c, 0x6c, 0x83,
+		},
+	},
+	{
+		.data_len = 2,
+		.digest = {
+			0xf4, 0x50, 0xaf, 0x07, 0xda, 0x42, 0xa7, 0x41,
+			0x4d, 0x24, 0x88, 0x87, 0xe3, 0x40, 0x73, 0x7c,
+		},
+	},
+	{
+		.data_len = 3,
+		.digest = {
+			0x9e, 0x88, 0x78, 0x71, 0x4c, 0x55, 0x87, 0xe8,
+			0xb4, 0x96, 0x3d, 0x56, 0xc8, 0xb2, 0xe1, 0x68,
+		},
+	},
+	{
+		.data_len = 16,
+		.digest = {
+			0x9e, 0x81, 0x37, 0x8f, 0x49, 0xf7, 0xa2, 0xe4,
+			0x04, 0x45, 0x12, 0x78, 0x45, 0x42, 0x27, 0xad,
+		},
+	},
+	{
+		.data_len = 32,
+		.digest = {
+			0x60, 0x19, 0xd0, 0xa4, 0xf0, 0xde, 0x9e, 0xe7,
+			0x6a, 0x89, 0x1a, 0xea, 0x80, 0x14, 0xa9, 0xa3,
+		},
+	},
+	{
+		.data_len = 48,
+		.digest = {
+			0x0c, 0xa2, 0x70, 0x4d, 0x7c, 0x89, 0xac, 0x41,
+			0xc2, 0x9e, 0x0d, 0x07, 0x07, 0x6a, 0x7f, 0xd5,
+		},
+	},
+	{
+		.data_len = 49,
+		.digest = {
+			0x91, 0xd3, 0xa9, 0x5c, 0x79, 0x3d, 0x6b, 0x84,
+			0x99, 0x54, 0xa7, 0xb4, 0x06, 0x66, 0xfd, 0x1c,
+		},
+	},
+	{
+		.data_len = 63,
+		.digest = {
+			0x29, 0x37, 0xb8, 0xe5, 0xd8, 0x27, 0x4d, 0xfb,
+			0x83, 0x4f, 0x67, 0xf7, 0xf9, 0xc1, 0x0a, 0x9d,
+		},
+	},
+	{
+		.data_len = 64,
+		.digest = {
+			0x17, 0xa9, 0x06, 0x2c, 0xf3, 0xe8, 0x2e, 0xa6,
+			0x6b, 0xb2, 0x1f, 0x5d, 0x94, 0x3c, 0x02, 0xa2,
+		},
+	},
+	{
+		.data_len = 65,
+		.digest = {
+			0x7c, 0x80, 0x74, 0xd7, 0xa1, 0x37, 0x30, 0x64,
+			0x3b, 0xa4, 0xa3, 0x98, 0xde, 0x47, 0x10, 0x23,
+		},
+	},
+	{
+		.data_len = 127,
+		.digest = {
+			0x27, 0x3a, 0xcf, 0xf5, 0xaf, 0x9f, 0xd8, 0xd8,
+			0x2d, 0x6a, 0x91, 0xfb, 0xb8, 0xfa, 0xbe, 0x0c,
+		},
+	},
+	{
+		.data_len = 128,
+		.digest = {
+			0x97, 0x6e, 0xc4, 0xbe, 0x6b, 0x15, 0xa6, 0x7c,
+			0xc4, 0xa2, 0xb8, 0x0a, 0x0e, 0x9c, 0xc7, 0x3a,
+		},
+	},
+	{
+		.data_len = 129,
+		.digest = {
+			0x2b, 0xc3, 0x98, 0xba, 0x6e, 0x42, 0xf8, 0x18,
+			0x85, 0x69, 0x15, 0x37, 0x10, 0x60, 0xe6, 0xac,
+		},
+	},
+	{
+		.data_len = 256,
+		.digest = {
+			0x88, 0x21, 0x77, 0x89, 0xd7, 0x93, 0x90, 0xfc,
+			0xf3, 0xb0, 0xe3, 0xfb, 0x14, 0xe2, 0xcf, 0x74,
+		},
+	},
+	{
+		.data_len = 511,
+		.digest = {
+			0x66, 0x3d, 0x3e, 0x08, 0xa0, 0x49, 0x81, 0x68,
+			0x3e, 0x3b, 0xc8, 0x80, 0x55, 0xd4, 0x15, 0xe9,
+		},
+	},
+	{
+		.data_len = 513,
+		.digest = {
+			0x05, 0xf5, 0x06, 0x66, 0xe7, 0x11, 0x08, 0x84,
+			0xff, 0x94, 0x50, 0x85, 0x65, 0x95, 0x2a, 0x20,
+		},
+	},
+	{
+		.data_len = 1000,
+		.digest = {
+			0xd3, 0xa0, 0x51, 0x69, 0xb5, 0x38, 0xae, 0x1b,
+			0xe1, 0xa2, 0x89, 0xc6, 0x8d, 0x2b, 0x62, 0x37,
+		},
+	},
+	{
+		.data_len = 3333,
+		.digest = {
+			0x37, 0x6d, 0x6a, 0x14, 0xdc, 0xa5, 0x37, 0xfc,
+			0xfe, 0x67, 0x76, 0xb2, 0x64, 0x68, 0x64, 0x05,
+		},
+	},
+	{
+		.data_len = 4096,
+		.digest = {
+			0xe3, 0x12, 0x0c, 0x58, 0x46, 0x45, 0x27, 0x7a,
+			0x0e, 0xa2, 0xfa, 0x2c, 0x35, 0x73, 0x6c, 0x94,
+		},
+	},
+	{
+		.data_len = 4128,
+		.digest = {
+			0x63, 0x0d, 0xa1, 0xbc, 0x6e, 0x3e, 0xd3, 0x1d,
+			0x28, 0x52, 0xd2, 0xf4, 0x30, 0x2d, 0xff, 0xc4,
+		},
+	},
+	{
+		.data_len = 4160,
+		.digest = {
+			0xb2, 0x91, 0x49, 0xe2, 0x02, 0x98, 0x00, 0x79,
+			0x71, 0xb9, 0xd7, 0xd4, 0xb5, 0x94, 0x6d, 0x7d,
+		},
+	},
+	{
+		.data_len = 4224,
+		.digest = {
+			0x58, 0x96, 0x48, 0x69, 0x05, 0x17, 0xe1, 0x6d,
+			0xbc, 0xf2, 0x3d, 0x10, 0x96, 0x00, 0x74, 0x58,
+		},
+	},
+	{
+		.data_len = 16384,
+		.digest = {
+			0x99, 0x3c, 0xcb, 0x4d, 0x64, 0xc9, 0xa9, 0x41,
+			0x52, 0x93, 0xfd, 0x65, 0xc4, 0xcc, 0xa5, 0xe5,
+		},
+	},
+};
+
+static const u8 hash_testvec_consolidated[POLYVAL_DIGEST_SIZE] = {
+	0xdf, 0x68, 0x52, 0x99, 0x92, 0xc3, 0xe8, 0x88,
+	0x29, 0x13, 0xc8, 0x35, 0x67, 0xa3, 0xd3, 0xad,
+};
+
+static const u8 polyval_allones_hashofhashes[POLYVAL_DIGEST_SIZE] = {
+	0xd5, 0xf7, 0xfd, 0xb2, 0xa6, 0xef, 0x0b, 0x85,
+	0x0d, 0x0a, 0x06, 0x10, 0xbc, 0x64, 0x94, 0x73,
+};
diff --git a/lib/crypto/tests/polyval_kunit.c b/lib/crypto/tests/polyval_kunit.c
new file mode 100644
index 000000000000..f47f41a39a41
--- /dev/null
+++ b/lib/crypto/tests/polyval_kunit.c
@@ -0,0 +1,223 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright 2025 Google LLC
+ */
+#include <crypto/polyval.h>
+#include "polyval-testvecs.h"
+
+/*
+ * A fixed key used when presenting POLYVAL as an unkeyed hash function in order
+ * to reuse hash-test-template.h.  At the beginning of the test suite, this is
+ * initialized to a key prepared from bytes generated from a fixed seed.
+ */
+static struct polyval_key test_key;
+
+static void polyval_init_withtestkey(struct polyval_ctx *ctx)
+{
+	polyval_init(ctx, &test_key);
+}
+
+static void polyval_withtestkey(const u8 *data, size_t len,
+				u8 out[POLYVAL_BLOCK_SIZE])
+{
+	polyval(&test_key, data, len, out);
+}
+
+/* Generate the HASH_KUNIT_CASES using hash-test-template.h. */
+#define HASH polyval_withtestkey
+#define HASH_CTX polyval_ctx
+#define HASH_SIZE POLYVAL_BLOCK_SIZE
+#define HASH_INIT polyval_init_withtestkey
+#define HASH_UPDATE polyval_update
+#define HASH_FINAL polyval_final
+#include "hash-test-template.h"
+
+/*
+ * Test an example from RFC8452 ("AES-GCM-SIV: Nonce Misuse-Resistant
+ * Authenticated Encryption") to ensure compatibility with that.
+ */
+static void test_polyval_rfc8452_testvec(struct kunit *test)
+{
+	static const u8 raw_key[POLYVAL_BLOCK_SIZE] =
+		"\x31\x07\x28\xd9\x91\x1f\x1f\x38"
+		"\x37\xb2\x43\x16\xc3\xfa\xb9\xa0";
+	static const u8 data[48] =
+		"\x65\x78\x61\x6d\x70\x6c\x65\x00"
+		"\x00\x00\x00\x00\x00\x00\x00\x00"
+		"\x48\x65\x6c\x6c\x6f\x20\x77\x6f"
+		"\x72\x6c\x64\x00\x00\x00\x00\x00"
+		"\x38\x00\x00\x00\x00\x00\x00\x00"
+		"\x58\x00\x00\x00\x00\x00\x00\x00";
+	static const u8 expected_hash[POLYVAL_BLOCK_SIZE] =
+		"\xad\x7f\xcf\x0b\x51\x69\x85\x16"
+		"\x62\x67\x2f\x3c\x5f\x95\x13\x8f";
+	u8 hash[POLYVAL_BLOCK_SIZE];
+	struct polyval_key key;
+
+	polyval_preparekey(&key, raw_key);
+	polyval(&key, data, sizeof(data), hash);
+	KUNIT_ASSERT_MEMEQ(test, hash, expected_hash, sizeof(hash));
+}
+
+/*
+ * Test a key and messages containing all one bits.  This is useful to detect
+ * overflow bugs in implementations that emulate carryless multiplication using
+ * a series of standard multiplications with the bits spread out.
+ */
+static void test_polyval_allones_key_and_message(struct kunit *test)
+{
+	struct polyval_key key;
+	struct polyval_ctx hashofhashes_ctx;
+	u8 hash[POLYVAL_BLOCK_SIZE];
+
+	static_assert(TEST_BUF_LEN >= 4096);
+	memset(test_buf, 0xff, 4096);
+
+	polyval_preparekey(&key, test_buf);
+	polyval_init(&hashofhashes_ctx, &key);
+	for (size_t len = 0; len <= 4096; len += 16) {
+		polyval(&key, test_buf, len, hash);
+		polyval_update(&hashofhashes_ctx, hash, sizeof(hash));
+	}
+	polyval_final(&hashofhashes_ctx, hash);
+	KUNIT_ASSERT_MEMEQ(test, hash, polyval_allones_hashofhashes,
+			   sizeof(hash));
+}
+
+#define MAX_LEN_FOR_KEY_CHECK 1024
+
+/*
+ * Given two prepared keys which should be identical (but may differ in
+ * alignment and/or whether they are followed by a guard page or not), verify
+ * that they produce consistent results on various data lengths.
+ */
+static void check_key_consistency(struct kunit *test,
+				  const struct polyval_key *key1,
+				  const struct polyval_key *key2)
+{
+	u8 *data = test_buf;
+	u8 hash1[POLYVAL_BLOCK_SIZE];
+	u8 hash2[POLYVAL_BLOCK_SIZE];
+
+	rand_bytes(data, MAX_LEN_FOR_KEY_CHECK);
+	KUNIT_ASSERT_MEMEQ(test, key1, key2, sizeof(*key1));
+
+	for (int i = 0; i < 100; i++) {
+		size_t len = rand_length(MAX_LEN_FOR_KEY_CHECK);
+
+		polyval(key1, data, len, hash1);
+		polyval(key2, data, len, hash2);
+		KUNIT_ASSERT_MEMEQ(test, hash1, hash2, sizeof(hash1));
+	}
+}
+
+/* Test that no buffer overreads occur on either raw_key or polyval_key. */
+static void test_polyval_with_guarded_key(struct kunit *test)
+{
+	u8 raw_key[POLYVAL_BLOCK_SIZE];
+	u8 *guarded_raw_key = &test_buf[TEST_BUF_LEN - sizeof(raw_key)];
+	struct polyval_key key1, key2;
+	struct polyval_key *guarded_key =
+		(struct polyval_key *)&test_buf[TEST_BUF_LEN - sizeof(key1)];
+
+	/* Prepare with regular buffers. */
+	rand_bytes(raw_key, sizeof(raw_key));
+	polyval_preparekey(&key1, raw_key);
+
+	/* Prepare with guarded raw_key, then check that it works. */
+	memcpy(guarded_raw_key, raw_key, sizeof(raw_key));
+	polyval_preparekey(&key2, guarded_raw_key);
+	check_key_consistency(test, &key1, &key2);
+
+	/* Prepare guarded polyval_key, then check that it works. */
+	polyval_preparekey(guarded_key, raw_key);
+	check_key_consistency(test, &key1, guarded_key);
+}
+
+/*
+ * Test that polyval_key only needs to be aligned to
+ * __alignof__(struct polyval_key), i.e. 8 bytes.  The assembly code may prefer
+ * 16-byte or higher alignment, but it musn't require it.
+ */
+static void test_polyval_with_minimally_aligned_key(struct kunit *test)
+{
+	u8 raw_key[POLYVAL_BLOCK_SIZE];
+	struct polyval_key key;
+	struct polyval_key *minaligned_key =
+		(struct polyval_key *)&test_buf[MAX_LEN_FOR_KEY_CHECK +
+						__alignof__(struct polyval_key)];
+
+	KUNIT_ASSERT_TRUE(test, IS_ALIGNED((uintptr_t)minaligned_key,
+					   __alignof__(struct polyval_key)));
+	KUNIT_ASSERT_TRUE(test,
+			  !IS_ALIGNED((uintptr_t)minaligned_key,
+				      2 * __alignof__(struct polyval_key)));
+
+	rand_bytes(raw_key, sizeof(raw_key));
+	polyval_preparekey(&key, raw_key);
+	polyval_preparekey(minaligned_key, raw_key);
+	check_key_consistency(test, &key, minaligned_key);
+}
+
+struct polyval_irq_test_state {
+	struct polyval_key expected_key;
+	u8 raw_key[POLYVAL_BLOCK_SIZE];
+};
+
+static bool polyval_irq_test_func(void *state_)
+{
+	struct polyval_irq_test_state *state = state_;
+	struct polyval_key key;
+
+	polyval_preparekey(&key, state->raw_key);
+	return memcmp(&key, &state->expected_key, sizeof(key)) == 0;
+}
+
+/*
+ * Test that polyval_preparekey() produces the same output regardless of whether
+ * FPU or vector registers are usable when it is called.
+ */
+static void test_polyval_preparekey_in_irqs(struct kunit *test)
+{
+	struct polyval_irq_test_state state;
+
+	rand_bytes(state.raw_key, sizeof(state.raw_key));
+	polyval_preparekey(&state.expected_key, state.raw_key);
+	kunit_run_irq_test(test, polyval_irq_test_func, 200000, &state);
+}
+
+static int polyval_suite_init(struct kunit_suite *suite)
+{
+	u8 raw_key[POLYVAL_BLOCK_SIZE];
+
+	rand_bytes_seeded_from_len(raw_key, sizeof(raw_key));
+	polyval_preparekey(&test_key, raw_key);
+	return hash_suite_init(suite);
+}
+
+static void polyval_suite_exit(struct kunit_suite *suite)
+{
+	hash_suite_exit(suite);
+}
+
+static struct kunit_case polyval_test_cases[] = {
+	HASH_KUNIT_CASES,
+	KUNIT_CASE(test_polyval_rfc8452_testvec),
+	KUNIT_CASE(test_polyval_allones_key_and_message),
+	KUNIT_CASE(test_polyval_with_guarded_key),
+	KUNIT_CASE(test_polyval_with_minimally_aligned_key),
+	KUNIT_CASE(test_polyval_preparekey_in_irqs),
+	KUNIT_CASE(benchmark_hash),
+	{},
+};
+
+static struct kunit_suite polyval_test_suite = {
+	.name = "polyval",
+	.test_cases = polyval_test_cases,
+	.suite_init = polyval_suite_init,
+	.suite_exit = polyval_suite_exit,
+};
+kunit_test_suite(polyval_test_suite);
+
+MODULE_DESCRIPTION("KUnit tests and benchmark for POLYVAL");
+MODULE_LICENSE("GPL");
diff --git a/lib/crypto/tests/sha3-testvecs.h b/lib/crypto/tests/sha3-testvecs.h
new file mode 100644
index 000000000000..8d614a5fa0c3
--- /dev/null
+++ b/lib/crypto/tests/sha3-testvecs.h
@@ -0,0 +1,249 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py sha3 */
+
+/* SHA3-256 test vectors */
+
+static const struct {
+	size_t data_len;
+	u8 digest[SHA3_256_DIGEST_SIZE];
+} hash_testvecs[] = {
+	{
+		.data_len = 0,
+		.digest = {
+			0xa7, 0xff, 0xc6, 0xf8, 0xbf, 0x1e, 0xd7, 0x66,
+			0x51, 0xc1, 0x47, 0x56, 0xa0, 0x61, 0xd6, 0x62,
+			0xf5, 0x80, 0xff, 0x4d, 0xe4, 0x3b, 0x49, 0xfa,
+			0x82, 0xd8, 0x0a, 0x4b, 0x80, 0xf8, 0x43, 0x4a,
+		},
+	},
+	{
+		.data_len = 1,
+		.digest = {
+			0x11, 0x03, 0xe7, 0x84, 0x51, 0x50, 0x86, 0x35,
+			0x71, 0x8a, 0x70, 0xe3, 0xc4, 0x26, 0x7b, 0x21,
+			0x02, 0x13, 0xa0, 0x81, 0xe8, 0xe6, 0x14, 0x25,
+			0x07, 0x34, 0xe5, 0xc5, 0x40, 0x06, 0xf2, 0x8b,
+		},
+	},
+	{
+		.data_len = 2,
+		.digest = {
+			0x2f, 0x6f, 0x6d, 0x47, 0x48, 0x52, 0x11, 0xb9,
+			0xe4, 0x3d, 0xc8, 0x71, 0xcf, 0xb2, 0xee, 0xae,
+			0x5b, 0xf4, 0x12, 0x84, 0x5b, 0x1c, 0xec, 0x6c,
+			0xc1, 0x66, 0x88, 0xaa, 0xc3, 0x40, 0xbd, 0x7e,
+		},
+	},
+	{
+		.data_len = 3,
+		.digest = {
+			0xec, 0x02, 0xe8, 0x81, 0x4f, 0x84, 0x41, 0x69,
+			0x06, 0xd8, 0xdc, 0x1d, 0x01, 0x78, 0xd7, 0xcb,
+			0x39, 0xdf, 0xd3, 0x12, 0x1c, 0x99, 0xfd, 0xf3,
+			0x5c, 0x83, 0xc9, 0xc2, 0x7a, 0x7b, 0x6a, 0x05,
+		},
+	},
+	{
+		.data_len = 16,
+		.digest = {
+			0xff, 0x6f, 0xc3, 0x41, 0xc3, 0x5f, 0x34, 0x6d,
+			0xa7, 0xdf, 0x3e, 0xc2, 0x8b, 0x29, 0xb6, 0xf1,
+			0xf8, 0x67, 0xfd, 0xcd, 0xb1, 0x9f, 0x38, 0x08,
+			0x1d, 0x8d, 0xd9, 0xc2, 0x43, 0x66, 0x18, 0x6c,
+		},
+	},
+	{
+		.data_len = 32,
+		.digest = {
+			0xe4, 0xb1, 0x06, 0x17, 0xf8, 0x8b, 0x91, 0x95,
+			0xe7, 0x57, 0x66, 0xac, 0x08, 0xb2, 0x03, 0x3e,
+			0xf7, 0x84, 0x1f, 0xe3, 0x25, 0xa3, 0x11, 0xd2,
+			0x11, 0xa4, 0x78, 0x74, 0x2a, 0x43, 0x20, 0xa5,
+		},
+	},
+	{
+		.data_len = 48,
+		.digest = {
+			0xeb, 0x57, 0x5f, 0x20, 0xa3, 0x6b, 0xc7, 0xb4,
+			0x66, 0x2a, 0xa0, 0x30, 0x3b, 0x52, 0x00, 0xc9,
+			0xce, 0x6a, 0xd8, 0x1e, 0xbe, 0xed, 0xa1, 0xd1,
+			0xbe, 0x63, 0xc7, 0xe1, 0xe2, 0x66, 0x67, 0x0c,
+		},
+	},
+	{
+		.data_len = 49,
+		.digest = {
+			0xf0, 0x67, 0xad, 0x66, 0xbe, 0xec, 0x5a, 0xfd,
+			0x29, 0xd2, 0x4f, 0x1d, 0xb2, 0x24, 0xb8, 0x90,
+			0x05, 0x28, 0x0e, 0x66, 0x67, 0x74, 0x2d, 0xee,
+			0x66, 0x25, 0x11, 0xd1, 0x76, 0xa2, 0xfc, 0x3a,
+		},
+	},
+	{
+		.data_len = 63,
+		.digest = {
+			0x57, 0x56, 0x21, 0xb3, 0x2d, 0x2d, 0xe1, 0x9d,
+			0xbf, 0x2c, 0x82, 0xa8, 0xad, 0x7e, 0x6c, 0x46,
+			0xfb, 0x30, 0xeb, 0xce, 0xcf, 0xed, 0x2d, 0x65,
+			0xe7, 0xe4, 0x96, 0x69, 0xe0, 0x48, 0xd2, 0xb6,
+		},
+	},
+	{
+		.data_len = 64,
+		.digest = {
+			0x7b, 0xba, 0x67, 0x15, 0xe5, 0x21, 0xc4, 0x69,
+			0xd3, 0xef, 0x5c, 0x97, 0x9f, 0x5b, 0xba, 0x9c,
+			0xfa, 0x55, 0x64, 0xec, 0xb5, 0x37, 0x53, 0x1b,
+			0x3f, 0x4c, 0x0a, 0xed, 0x51, 0x98, 0x2b, 0x52,
+		},
+	},
+	{
+		.data_len = 65,
+		.digest = {
+			0x44, 0xb6, 0x6b, 0x83, 0x09, 0x83, 0x55, 0x83,
+			0xde, 0x1f, 0xcc, 0x33, 0xef, 0xdc, 0x05, 0xbb,
+			0x3b, 0x63, 0x76, 0x45, 0xe4, 0x8e, 0x14, 0x7a,
+			0x2d, 0xae, 0x90, 0xce, 0x68, 0xc3, 0xa4, 0xf2,
+		},
+	},
+	{
+		.data_len = 127,
+		.digest = {
+			0x50, 0x3e, 0x99, 0x4e, 0x28, 0x2b, 0xc9, 0xf4,
+			0xf5, 0xeb, 0x2b, 0x16, 0x04, 0x2d, 0xf5, 0xbe,
+			0xc0, 0x91, 0x41, 0x2a, 0x8e, 0x69, 0x5e, 0x39,
+			0x53, 0x2c, 0xc1, 0x18, 0xa5, 0xeb, 0xd8, 0xda,
+		},
+	},
+	{
+		.data_len = 128,
+		.digest = {
+			0x90, 0x0b, 0xa6, 0x92, 0x84, 0x30, 0xaf, 0xee,
+			0x38, 0x59, 0x83, 0x83, 0xe9, 0xfe, 0xab, 0x86,
+			0x79, 0x1b, 0xcd, 0xe7, 0x0a, 0x0f, 0x58, 0x53,
+			0x36, 0xab, 0x12, 0xe1, 0x5c, 0x97, 0xc1, 0xfb,
+		},
+	},
+	{
+		.data_len = 129,
+		.digest = {
+			0x2b, 0x52, 0x1e, 0x54, 0xbe, 0x38, 0x4c, 0x3e,
+			0x73, 0x37, 0x18, 0xf5, 0x25, 0x2c, 0xc8, 0xc7,
+			0xda, 0x7e, 0xb6, 0x47, 0x9d, 0xf4, 0x46, 0xce,
+			0xfa, 0x80, 0x20, 0x6b, 0xbd, 0xfd, 0x2a, 0xd8,
+		},
+	},
+	{
+		.data_len = 256,
+		.digest = {
+			0x45, 0xf0, 0xf5, 0x9b, 0xd9, 0x91, 0x26, 0xd5,
+			0x91, 0x3b, 0xf8, 0x87, 0x8b, 0x34, 0x02, 0x31,
+			0x64, 0xab, 0xf4, 0x1c, 0x6e, 0x34, 0x72, 0xdf,
+			0x32, 0x6d, 0xe5, 0xd2, 0x67, 0x5e, 0x86, 0x93,
+		},
+	},
+	{
+		.data_len = 511,
+		.digest = {
+			0xb3, 0xaf, 0x71, 0x64, 0xfa, 0xd4, 0xf1, 0x07,
+			0x38, 0xef, 0x04, 0x8e, 0x89, 0xf4, 0x02, 0xd2,
+			0xa5, 0xaf, 0x3b, 0xf5, 0x67, 0x56, 0xcf, 0xa9,
+			0x8e, 0x43, 0xf5, 0xb5, 0xe3, 0x91, 0x8e, 0xe7,
+		},
+	},
+	{
+		.data_len = 513,
+		.digest = {
+			0x51, 0xac, 0x0a, 0x65, 0xb7, 0x96, 0x20, 0xcf,
+			0x88, 0xf6, 0x97, 0x35, 0x89, 0x0d, 0x31, 0x0f,
+			0xbe, 0x17, 0xbe, 0x62, 0x03, 0x67, 0xc0, 0xee,
+			0x4f, 0xc1, 0xe3, 0x7f, 0x6f, 0xab, 0xac, 0xb4,
+		},
+	},
+	{
+		.data_len = 1000,
+		.digest = {
+			0x7e, 0xea, 0xa8, 0xd7, 0xde, 0x20, 0x1b, 0x58,
+			0x24, 0xd8, 0x26, 0x40, 0x36, 0x5f, 0x3f, 0xaa,
+			0xe5, 0x5a, 0xea, 0x98, 0x58, 0xd4, 0xd6, 0xfc,
+			0x20, 0x4c, 0x5c, 0x4f, 0xaf, 0x56, 0xc7, 0xc3,
+		},
+	},
+	{
+		.data_len = 3333,
+		.digest = {
+			0x61, 0xb1, 0xb1, 0x3e, 0x0e, 0x7e, 0x90, 0x3d,
+			0x31, 0x54, 0xbd, 0xc9, 0x0d, 0x53, 0x62, 0xf1,
+			0xcd, 0x18, 0x80, 0xf9, 0x91, 0x75, 0x41, 0xb3,
+			0x51, 0x39, 0x57, 0xa7, 0xa8, 0x1e, 0xfb, 0xc9,
+		},
+	},
+	{
+		.data_len = 4096,
+		.digest = {
+			0xab, 0x29, 0xda, 0x10, 0xc4, 0x11, 0x2d, 0x5c,
+			0xd1, 0xce, 0x1c, 0x95, 0xfa, 0xc6, 0xc7, 0xb0,
+			0x1b, 0xd1, 0xdc, 0x6f, 0xa0, 0x9d, 0x1b, 0x23,
+			0xfb, 0x6e, 0x90, 0x97, 0xd0, 0x75, 0x44, 0x7a,
+		},
+	},
+	{
+		.data_len = 4128,
+		.digest = {
+			0x02, 0x45, 0x95, 0xf4, 0x19, 0xb5, 0x93, 0x29,
+			0x90, 0xf2, 0x63, 0x3f, 0x89, 0xe8, 0xa5, 0x31,
+			0x76, 0xf2, 0x89, 0x79, 0x66, 0xd3, 0x96, 0xdf,
+			0x33, 0xd1, 0xa6, 0x17, 0x73, 0xb1, 0xd0, 0x45,
+		},
+	},
+	{
+		.data_len = 4160,
+		.digest = {
+			0xd1, 0x8e, 0x22, 0xea, 0x44, 0x87, 0x6e, 0x9d,
+			0xfb, 0x36, 0x02, 0x20, 0x63, 0xb7, 0x69, 0x45,
+			0x25, 0x41, 0x69, 0xe0, 0x9b, 0x87, 0xcf, 0xa3,
+			0x51, 0xbb, 0xfc, 0x8d, 0xf7, 0x29, 0xa7, 0xea,
+		},
+	},
+	{
+		.data_len = 4224,
+		.digest = {
+			0x11, 0x86, 0x7d, 0x84, 0xf9, 0x8c, 0x6e, 0xc4,
+			0x64, 0x36, 0xc6, 0xf3, 0x42, 0x92, 0x31, 0x2b,
+			0x1e, 0x12, 0xe6, 0x4d, 0xbe, 0xfa, 0x77, 0x3f,
+			0x89, 0x41, 0x33, 0x58, 0x1c, 0x98, 0x16, 0x0a,
+		},
+	},
+	{
+		.data_len = 16384,
+		.digest = {
+			0xb2, 0xba, 0x0c, 0x8c, 0x9d, 0xbb, 0x1e, 0xb0,
+			0x03, 0xb5, 0xdf, 0x4f, 0xf5, 0x35, 0xdb, 0xec,
+			0x60, 0xf2, 0x5b, 0xb6, 0xd0, 0x49, 0xd3, 0xed,
+			0x55, 0xc0, 0x7a, 0xd7, 0xaf, 0xa1, 0xea, 0x53,
+		},
+	},
+};
+
+static const u8 hash_testvec_consolidated[SHA3_256_DIGEST_SIZE] = {
+	0x3b, 0x33, 0x67, 0xf8, 0xea, 0x92, 0x78, 0x62,
+	0xdd, 0xbe, 0x72, 0x15, 0xbd, 0x6f, 0xfa, 0xe5,
+	0x5e, 0xab, 0x9f, 0xb1, 0xe4, 0x23, 0x7c, 0x2c,
+	0x80, 0xcf, 0x09, 0x75, 0xf8, 0xe2, 0xfa, 0x30,
+};
+
+/* SHAKE test vectors */
+
+static const u8 shake128_testvec_consolidated[SHA3_256_DIGEST_SIZE] = {
+	0x89, 0x88, 0x3a, 0x44, 0xec, 0xfe, 0x3c, 0xeb,
+	0x2f, 0x1c, 0x1d, 0xda, 0x9e, 0x36, 0x64, 0xf0,
+	0x85, 0x4c, 0x49, 0x12, 0x76, 0x5a, 0x4d, 0xe7,
+	0xa8, 0xfd, 0xcd, 0xbe, 0x45, 0xb4, 0x6f, 0xb0,
+};
+
+static const u8 shake256_testvec_consolidated[SHA3_256_DIGEST_SIZE] = {
+	0x5a, 0xfd, 0x66, 0x62, 0x5c, 0x37, 0x2b, 0x41,
+	0x77, 0x1c, 0x01, 0x5d, 0x64, 0x7c, 0x63, 0x7a,
+	0x7c, 0x76, 0x9e, 0xa8, 0xd1, 0xb0, 0x8e, 0x02,
+	0x16, 0x9b, 0xfe, 0x0e, 0xb5, 0xd8, 0x6a, 0xb5,
+};
diff --git a/lib/crypto/tests/sha3_kunit.c b/lib/crypto/tests/sha3_kunit.c
new file mode 100644
index 000000000000..ed5fbe80337f
--- /dev/null
+++ b/lib/crypto/tests/sha3_kunit.c
@@ -0,0 +1,422 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2025 Red Hat, Inc. All Rights Reserved.
+ * Written by David Howells (dhowells@redhat.com)
+ */
+#include <crypto/sha3.h>
+#include "sha3-testvecs.h"
+
+#define HASH		sha3_256
+#define HASH_CTX	sha3_ctx
+#define HASH_SIZE	SHA3_256_DIGEST_SIZE
+#define HASH_INIT	sha3_256_init
+#define HASH_UPDATE	sha3_update
+#define HASH_FINAL	sha3_final
+#include "hash-test-template.h"
+
+/*
+ * Sample message and the output generated for various algorithms by passing it
+ * into "openssl sha3-224" etc..
+ */
+static const u8 test_sha3_sample[] =
+	"The quick red fox jumped over the lazy brown dog!\n"
+	"The quick red fox jumped over the lazy brown dog!\n"
+	"The quick red fox jumped over the lazy brown dog!\n"
+	"The quick red fox jumped over the lazy brown dog!\n";
+
+static const u8 test_sha3_224[8 + SHA3_224_DIGEST_SIZE + 8] = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* Write-before guard */
+	0xd6, 0xe8, 0xd8, 0x80, 0xfa, 0x42, 0x80, 0x70,
+	0x7e, 0x7f, 0xd7, 0xd2, 0xd7, 0x7a, 0x35, 0x65,
+	0xf0, 0x0b, 0x4f, 0x9f, 0x2a, 0x33, 0xca, 0x0a,
+	0xef, 0xa6, 0x4c, 0xb8,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* Write-after guard */
+};
+
+static const u8 test_sha3_256[8 + SHA3_256_DIGEST_SIZE + 8] = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* Write-before guard */
+	0xdb, 0x3b, 0xb0, 0xb8, 0x8d, 0x15, 0x78, 0xe5,
+	0x78, 0x76, 0x8e, 0x39, 0x7e, 0x89, 0x86, 0xb9,
+	0x14, 0x3a, 0x1e, 0xe7, 0x96, 0x7c, 0xf3, 0x25,
+	0x70, 0xbd, 0xc3, 0xa9, 0xae, 0x63, 0x71, 0x1d,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* Write-after guard */
+};
+
+static const u8 test_sha3_384[8 + SHA3_384_DIGEST_SIZE + 8] = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* Write-before guard */
+	0x2d, 0x4b, 0x29, 0x85, 0x19, 0x94, 0xaa, 0x31,
+	0x9b, 0x04, 0x9d, 0x6e, 0x79, 0x66, 0xc7, 0x56,
+	0x8a, 0x2e, 0x99, 0x84, 0x06, 0xcf, 0x10, 0x2d,
+	0xec, 0xf0, 0x03, 0x04, 0x1f, 0xd5, 0x99, 0x63,
+	0x2f, 0xc3, 0x2b, 0x0d, 0xd9, 0x45, 0xf7, 0xbb,
+	0x0a, 0xc3, 0x46, 0xab, 0xfe, 0x4d, 0x94, 0xc2,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* Write-after guard */
+};
+
+static const u8 test_sha3_512[8 + SHA3_512_DIGEST_SIZE + 8] = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* Write-before guard */
+	0xdd, 0x71, 0x3b, 0x44, 0xb6, 0x6c, 0xd7, 0x78,
+	0xe7, 0x93, 0xa1, 0x4c, 0xd7, 0x24, 0x16, 0xf1,
+	0xfd, 0xa2, 0x82, 0x4e, 0xed, 0x59, 0xe9, 0x83,
+	0x15, 0x38, 0x89, 0x7d, 0x39, 0x17, 0x0c, 0xb2,
+	0xcf, 0x12, 0x80, 0x78, 0xa1, 0x78, 0x41, 0xeb,
+	0xed, 0x21, 0x4c, 0xa4, 0x4a, 0x5f, 0x30, 0x1a,
+	0x70, 0x98, 0x4f, 0x14, 0xa2, 0xd1, 0x64, 0x1b,
+	0xc2, 0x0a, 0xff, 0x3b, 0xe8, 0x26, 0x41, 0x8f,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* Write-after guard */
+};
+
+static const u8 test_shake128[8 + SHAKE128_DEFAULT_SIZE + 8] = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* Write-before guard */
+	0x41, 0xd6, 0xb8, 0x9c, 0xf8, 0xe8, 0x54, 0xf2,
+	0x5c, 0xde, 0x51, 0x12, 0xaf, 0x9e, 0x0d, 0x91,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* Write-after guard */
+};
+
+static const u8 test_shake256[8 + SHAKE256_DEFAULT_SIZE + 8] = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* Write-before guard */
+	0xab, 0x06, 0xd4, 0xf9, 0x8b, 0xfd, 0xb2, 0xc4,
+	0xfe, 0xf1, 0xcc, 0xe2, 0x40, 0x45, 0xdd, 0x15,
+	0xcb, 0xdd, 0x02, 0x8d, 0xb7, 0x9f, 0x1e, 0x67,
+	0xd6, 0x7f, 0x98, 0x5e, 0x1b, 0x19, 0xf8, 0x01,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* Write-after guard */
+};
+
+static void test_sha3_224_basic(struct kunit *test)
+{
+	u8 out[8 + SHA3_224_DIGEST_SIZE + 8];
+
+	BUILD_BUG_ON(sizeof(out) != sizeof(test_sha3_224));
+
+	memset(out, 0, sizeof(out));
+	sha3_224(test_sha3_sample, sizeof(test_sha3_sample) - 1, out + 8);
+
+	KUNIT_ASSERT_MEMEQ_MSG(test, out, test_sha3_224, sizeof(test_sha3_224),
+			       "SHA3-224 gives wrong output");
+}
+
+static void test_sha3_256_basic(struct kunit *test)
+{
+	u8 out[8 + SHA3_256_DIGEST_SIZE + 8];
+
+	BUILD_BUG_ON(sizeof(out) != sizeof(test_sha3_256));
+
+	memset(out, 0, sizeof(out));
+	sha3_256(test_sha3_sample, sizeof(test_sha3_sample) - 1, out + 8);
+
+	KUNIT_ASSERT_MEMEQ_MSG(test, out, test_sha3_256, sizeof(test_sha3_256),
+			       "SHA3-256 gives wrong output");
+}
+
+static void test_sha3_384_basic(struct kunit *test)
+{
+	u8 out[8 + SHA3_384_DIGEST_SIZE + 8];
+
+	BUILD_BUG_ON(sizeof(out) != sizeof(test_sha3_384));
+
+	memset(out, 0, sizeof(out));
+	sha3_384(test_sha3_sample, sizeof(test_sha3_sample) - 1, out + 8);
+
+	KUNIT_ASSERT_MEMEQ_MSG(test, out, test_sha3_384, sizeof(test_sha3_384),
+			       "SHA3-384 gives wrong output");
+}
+
+static void test_sha3_512_basic(struct kunit *test)
+{
+	u8 out[8 + SHA3_512_DIGEST_SIZE + 8];
+
+	BUILD_BUG_ON(sizeof(out) != sizeof(test_sha3_512));
+
+	memset(out, 0, sizeof(out));
+	sha3_512(test_sha3_sample, sizeof(test_sha3_sample) - 1, out + 8);
+
+	KUNIT_ASSERT_MEMEQ_MSG(test, out, test_sha3_512, sizeof(test_sha3_512),
+			       "SHA3-512 gives wrong output");
+}
+
+static void test_shake128_basic(struct kunit *test)
+{
+	u8 out[8 + SHAKE128_DEFAULT_SIZE + 8];
+
+	BUILD_BUG_ON(sizeof(out) != sizeof(test_shake128));
+
+	memset(out, 0, sizeof(out));
+	shake128(test_sha3_sample, sizeof(test_sha3_sample) - 1,
+		 out + 8, sizeof(out) - 16);
+
+	KUNIT_ASSERT_MEMEQ_MSG(test, out, test_shake128, sizeof(test_shake128),
+			       "SHAKE128 gives wrong output");
+}
+
+static void test_shake256_basic(struct kunit *test)
+{
+	u8 out[8 + SHAKE256_DEFAULT_SIZE + 8];
+
+	BUILD_BUG_ON(sizeof(out) != sizeof(test_shake256));
+
+	memset(out, 0, sizeof(out));
+	shake256(test_sha3_sample, sizeof(test_sha3_sample) - 1,
+		 out + 8, sizeof(out) - 16);
+
+	KUNIT_ASSERT_MEMEQ_MSG(test, out, test_shake256, sizeof(test_shake256),
+			       "SHAKE256 gives wrong output");
+}
+
+/*
+ * Usable NIST tests.
+ *
+ * From: https://csrc.nist.gov/projects/cryptographic-standards-and-guidelines/example-values
+ */
+static const u8 test_nist_1600_sample[] = {
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3,
+	0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3, 0xa3
+};
+
+static const u8 test_shake128_nist_0[] = {
+	0x7f, 0x9c, 0x2b, 0xa4, 0xe8, 0x8f, 0x82, 0x7d,
+	0x61, 0x60, 0x45, 0x50, 0x76, 0x05, 0x85, 0x3e
+};
+
+static const u8 test_shake128_nist_1600[] = {
+	0x13, 0x1a, 0xb8, 0xd2, 0xb5, 0x94, 0x94, 0x6b,
+	0x9c, 0x81, 0x33, 0x3f, 0x9b, 0xb6, 0xe0, 0xce,
+};
+
+static const u8 test_shake256_nist_0[] = {
+	0x46, 0xb9, 0xdd, 0x2b, 0x0b, 0xa8, 0x8d, 0x13,
+	0x23, 0x3b, 0x3f, 0xeb, 0x74, 0x3e, 0xeb, 0x24,
+	0x3f, 0xcd, 0x52, 0xea, 0x62, 0xb8, 0x1b, 0x82,
+	0xb5, 0x0c, 0x27, 0x64, 0x6e, 0xd5, 0x76, 0x2f
+};
+
+static const u8 test_shake256_nist_1600[] = {
+	0xcd, 0x8a, 0x92, 0x0e, 0xd1, 0x41, 0xaa, 0x04,
+	0x07, 0xa2, 0x2d, 0x59, 0x28, 0x86, 0x52, 0xe9,
+	0xd9, 0xf1, 0xa7, 0xee, 0x0c, 0x1e, 0x7c, 0x1c,
+	0xa6, 0x99, 0x42, 0x4d, 0xa8, 0x4a, 0x90, 0x4d,
+};
+
+static void test_shake128_nist(struct kunit *test)
+{
+	u8 out[SHAKE128_DEFAULT_SIZE];
+
+	shake128("", 0, out, sizeof(out));
+	KUNIT_ASSERT_MEMEQ_MSG(test, out, test_shake128_nist_0, sizeof(out),
+			       "SHAKE128 gives wrong output for NIST.0");
+
+	shake128(test_nist_1600_sample, sizeof(test_nist_1600_sample),
+		 out, sizeof(out));
+	KUNIT_ASSERT_MEMEQ_MSG(test, out, test_shake128_nist_1600, sizeof(out),
+			       "SHAKE128 gives wrong output for NIST.1600");
+}
+
+static void test_shake256_nist(struct kunit *test)
+{
+	u8 out[SHAKE256_DEFAULT_SIZE];
+
+	shake256("", 0, out, sizeof(out));
+	KUNIT_ASSERT_MEMEQ_MSG(test, out, test_shake256_nist_0, sizeof(out),
+			       "SHAKE256 gives wrong output for NIST.0");
+
+	shake256(test_nist_1600_sample, sizeof(test_nist_1600_sample),
+		 out, sizeof(out));
+	KUNIT_ASSERT_MEMEQ_MSG(test, out, test_shake256_nist_1600, sizeof(out),
+			       "SHAKE256 gives wrong output for NIST.1600");
+}
+
+static void shake(int alg, const u8 *in, size_t in_len, u8 *out, size_t out_len)
+{
+	if (alg == 0)
+		shake128(in, in_len, out, out_len);
+	else
+		shake256(in, in_len, out, out_len);
+}
+
+static void shake_init(struct shake_ctx *ctx, int alg)
+{
+	if (alg == 0)
+		shake128_init(ctx);
+	else
+		shake256_init(ctx);
+}
+
+/*
+ * Test each of SHAKE128 and SHAKE256 with all input lengths 0 through 4096, for
+ * both input and output.  The input and output lengths cycle through the values
+ * together, so we do 4096 tests total.  To verify all the SHAKE outputs,
+ * compute and verify the SHA3-256 digest of all of them concatenated together.
+ */
+static void test_shake_all_lens_up_to_4096(struct kunit *test)
+{
+	struct sha3_ctx main_ctx;
+	const size_t max_len = 4096;
+	u8 *const in = test_buf;
+	u8 *const out = &test_buf[TEST_BUF_LEN - max_len];
+	u8 main_hash[SHA3_256_DIGEST_SIZE];
+
+	KUNIT_ASSERT_LE(test, 2 * max_len, TEST_BUF_LEN);
+
+	rand_bytes_seeded_from_len(in, max_len);
+	for (int alg = 0; alg < 2; alg++) {
+		sha3_256_init(&main_ctx);
+		for (size_t in_len = 0; in_len <= max_len; in_len++) {
+			size_t out_len = (in_len * 293) % (max_len + 1);
+
+			shake(alg, in, in_len, out, out_len);
+			sha3_update(&main_ctx, out, out_len);
+		}
+		sha3_final(&main_ctx, main_hash);
+		if (alg == 0)
+			KUNIT_ASSERT_MEMEQ_MSG(test, main_hash,
+					       shake128_testvec_consolidated,
+					       sizeof(main_hash),
+					       "shake128() gives wrong output");
+		else
+			KUNIT_ASSERT_MEMEQ_MSG(test, main_hash,
+					       shake256_testvec_consolidated,
+					       sizeof(main_hash),
+					       "shake256() gives wrong output");
+	}
+}
+
+/*
+ * Test that a sequence of SHAKE squeezes gives the same output as a single
+ * squeeze of the same total length.
+ */
+static void test_shake_multiple_squeezes(struct kunit *test)
+{
+	const size_t max_len = 512;
+	u8 *ref_out;
+
+	KUNIT_ASSERT_GE(test, TEST_BUF_LEN, 2 * max_len);
+
+	ref_out = kunit_kzalloc(test, max_len, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ref_out);
+
+	for (int i = 0; i < 2000; i++) {
+		const int alg = rand32() % 2;
+		const size_t in_len = rand_length(max_len);
+		const size_t out_len = rand_length(max_len);
+		const size_t in_offs = rand_offset(max_len - in_len);
+		const size_t out_offs = rand_offset(max_len - out_len);
+		u8 *const in = &test_buf[in_offs];
+		u8 *const out = &test_buf[out_offs];
+		struct shake_ctx ctx;
+		size_t remaining_len, j, num_parts;
+
+		rand_bytes(in, in_len);
+		rand_bytes(out, out_len);
+
+		/* Compute the output using the one-shot function. */
+		shake(alg, in, in_len, ref_out, out_len);
+
+		/* Compute the output using a random sequence of squeezes. */
+		shake_init(&ctx, alg);
+		shake_update(&ctx, in, in_len);
+		remaining_len = out_len;
+		j = 0;
+		num_parts = 0;
+		while (rand_bool()) {
+			size_t part_len = rand_length(remaining_len);
+
+			shake_squeeze(&ctx, &out[j], part_len);
+			num_parts++;
+			j += part_len;
+			remaining_len -= part_len;
+		}
+		if (remaining_len != 0 || rand_bool()) {
+			shake_squeeze(&ctx, &out[j], remaining_len);
+			num_parts++;
+		}
+
+		/* Verify that the outputs are the same. */
+		KUNIT_ASSERT_MEMEQ_MSG(
+			test, out, ref_out, out_len,
+			"Multi-squeeze test failed with in_len=%zu in_offs=%zu out_len=%zu out_offs=%zu num_parts=%zu alg=%d",
+			in_len, in_offs, out_len, out_offs, num_parts, alg);
+	}
+}
+
+/*
+ * Test that SHAKE operations on buffers immediately followed by an unmapped
+ * page work as expected.  This catches out-of-bounds memory accesses even if
+ * they occur in assembly code.
+ */
+static void test_shake_with_guarded_bufs(struct kunit *test)
+{
+	const size_t max_len = 512;
+	u8 *reg_buf;
+
+	KUNIT_ASSERT_GE(test, TEST_BUF_LEN, max_len);
+
+	reg_buf = kunit_kzalloc(test, max_len, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, reg_buf);
+
+	for (int alg = 0; alg < 2; alg++) {
+		for (size_t len = 0; len <= max_len; len++) {
+			u8 *guarded_buf = &test_buf[TEST_BUF_LEN - len];
+
+			rand_bytes(reg_buf, len);
+			memcpy(guarded_buf, reg_buf, len);
+
+			shake(alg, reg_buf, len, reg_buf, len);
+			shake(alg, guarded_buf, len, guarded_buf, len);
+
+			KUNIT_ASSERT_MEMEQ_MSG(
+				test, reg_buf, guarded_buf, len,
+				"Guard page test failed with len=%zu alg=%d",
+				len, alg);
+		}
+	}
+}
+
+static struct kunit_case sha3_test_cases[] = {
+	HASH_KUNIT_CASES,
+	KUNIT_CASE(test_sha3_224_basic),
+	KUNIT_CASE(test_sha3_256_basic),
+	KUNIT_CASE(test_sha3_384_basic),
+	KUNIT_CASE(test_sha3_512_basic),
+	KUNIT_CASE(test_shake128_basic),
+	KUNIT_CASE(test_shake256_basic),
+	KUNIT_CASE(test_shake128_nist),
+	KUNIT_CASE(test_shake256_nist),
+	KUNIT_CASE(test_shake_all_lens_up_to_4096),
+	KUNIT_CASE(test_shake_multiple_squeezes),
+	KUNIT_CASE(test_shake_with_guarded_bufs),
+	KUNIT_CASE(benchmark_hash),
+	{},
+};
+
+static struct kunit_suite sha3_test_suite = {
+	.name = "sha3",
+	.test_cases = sha3_test_cases,
+	.suite_init = hash_suite_init,
+	.suite_exit = hash_suite_exit,
+};
+kunit_test_suite(sha3_test_suite);
+
+MODULE_DESCRIPTION("KUnit tests and benchmark for SHA3");
+MODULE_LICENSE("GPL");
diff --git a/lib/crypto/x86/aes-aesni.S b/lib/crypto/x86/aes-aesni.S
new file mode 100644
index 000000000000..b8c3e104a3be
--- /dev/null
+++ b/lib/crypto/x86/aes-aesni.S
@@ -0,0 +1,261 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+//
+// AES block cipher using AES-NI instructions
+//
+// Copyright 2026 Google LLC
+//
+// The code in this file supports 32-bit and 64-bit CPUs, and it doesn't require
+// AVX.  It does use up to SSE4.1, which all CPUs with AES-NI have.
+#include <linux/linkage.h>
+
+.section .rodata
+#ifdef __x86_64__
+#define RODATA(label)	label(%rip)
+#else
+#define RODATA(label)	label
+#endif
+
+	// A mask for pshufb that extracts the last dword, rotates it right by 8
+	// bits, and copies the result to all four dwords.
+.p2align 4
+.Lmask:
+	.byte	13, 14, 15, 12, 13, 14, 15, 12, 13, 14, 15, 12, 13, 14, 15, 12
+
+	// The AES round constants, used during key expansion
+.Lrcon:
+	.long	0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36
+
+.text
+
+// Transform four dwords [a0, a1, a2, a3] in \a into
+// [a0, a0^a1, a0^a1^a2, a0^a1^a2^a3].  \tmp is a temporary xmm register.
+//
+// Note: this could be done in four instructions, shufps + pxor + shufps + pxor,
+// if the temporary register were zero-initialized ahead of time.  We instead do
+// it in an easier-to-understand way that doesn't require zero-initialization
+// and avoids the unusual shufps instruction.  movdqa is usually "free" anyway.
+.macro	_prefix_sum	a, tmp
+	movdqa		\a, \tmp	// [a0, a1, a2, a3]
+	pslldq		$4, \a		// [0, a0, a1, a2]
+	pxor		\tmp, \a	// [a0, a0^a1, a1^a2, a2^a3]
+	movdqa		\a, \tmp
+	pslldq		$8, \a		// [0, 0, a0, a0^a1]
+	pxor		\tmp, \a	// [a0, a0^a1, a0^a1^a2, a0^a1^a2^a3]
+.endm
+
+.macro	_gen_round_key	a, b
+	// Compute four copies of rcon[i] ^ SubBytes(ror32(w, 8)), where w is
+	// the last dword of the previous round key (given in \b).
+	//
+	// 'aesenclast src, dst' does dst = src XOR SubBytes(ShiftRows(dst)).
+	// It is used here solely for the SubBytes and the XOR.  The ShiftRows
+	// is a no-op because all four columns are the same here.
+	//
+	// Don't use the 'aeskeygenassist' instruction, since:
+	//  - On most Intel CPUs it is microcoded, making it have a much higher
+	//    latency and use more execution ports than 'aesenclast'.
+	//  - It cannot be used in a loop, since it requires an immediate.
+	//  - It doesn't do much more than 'aesenclast' in the first place.
+	movdqa		\b, %xmm2
+	pshufb		MASK, %xmm2
+	aesenclast	RCON, %xmm2
+
+	// XOR in the prefix sum of the four dwords of \a, which is the
+	// previous round key (AES-128) or the first round key in the previous
+	// pair of round keys (AES-256).  The result is the next round key.
+	_prefix_sum	\a, tmp=%xmm3
+	pxor		%xmm2, \a
+
+	// Store the next round key to memory.  Also leave it in \a.
+	movdqu		\a, (RNDKEYS)
+.endm
+
+.macro	_aes_expandkey_aesni	is_aes128
+#ifdef __x86_64__
+	// Arguments
+	.set	RNDKEYS,	%rdi
+	.set	INV_RNDKEYS,	%rsi
+	.set	IN_KEY,		%rdx
+
+	// Other local variables
+	.set	RCON_PTR,	%rcx
+	.set	COUNTER,	%eax
+#else
+	// Arguments, assuming -mregparm=3
+	.set	RNDKEYS,	%eax
+	.set	INV_RNDKEYS,	%edx
+	.set	IN_KEY,		%ecx
+
+	// Other local variables
+	.set	RCON_PTR,	%ebx
+	.set	COUNTER,	%esi
+#endif
+	.set	RCON,		%xmm6
+	.set	MASK,		%xmm7
+
+#ifdef __i386__
+	push		%ebx
+	push		%esi
+#endif
+
+.if \is_aes128
+	// AES-128: the first round key is simply a copy of the raw key.
+	movdqu		(IN_KEY), %xmm0
+	movdqu		%xmm0, (RNDKEYS)
+.else
+	// AES-256: the first two round keys are simply a copy of the raw key.
+	movdqu		(IN_KEY), %xmm0
+	movdqu		%xmm0, (RNDKEYS)
+	movdqu		16(IN_KEY), %xmm1
+	movdqu		%xmm1, 16(RNDKEYS)
+	add		$32, RNDKEYS
+.endif
+
+	// Generate the remaining round keys.
+	movdqa		RODATA(.Lmask), MASK
+.if \is_aes128
+	lea		RODATA(.Lrcon), RCON_PTR
+	mov		$10, COUNTER
+.Lgen_next_aes128_round_key:
+	add		$16, RNDKEYS
+	movd		(RCON_PTR), RCON
+	pshufd		$0x00, RCON, RCON
+	add		$4, RCON_PTR
+	_gen_round_key	%xmm0, %xmm0
+	dec		COUNTER
+	jnz		.Lgen_next_aes128_round_key
+.else
+	// AES-256: only the first 7 round constants are needed, so instead of
+	// loading each one from memory, just start by loading [1, 1, 1, 1] and
+	// then generate the rest by doubling.
+	pshufd		$0x00, RODATA(.Lrcon), RCON
+	pxor		%xmm5, %xmm5	// All-zeroes
+	mov		$7, COUNTER
+.Lgen_next_aes256_round_key_pair:
+	// Generate the next AES-256 round key: either the first of a pair of
+	// two, or the last one.
+	_gen_round_key	%xmm0, %xmm1
+
+	dec		COUNTER
+	jz		.Lgen_aes256_round_keys_done
+
+	// Generate the second AES-256 round key of the pair.  Compared to the
+	// first, there's no rotation and no XOR of a round constant.
+	pshufd		$0xff, %xmm0, %xmm2	// Get four copies of last dword
+	aesenclast	%xmm5, %xmm2		// Just does SubBytes
+	_prefix_sum	%xmm1, tmp=%xmm3
+	pxor		%xmm2, %xmm1
+	movdqu		%xmm1, 16(RNDKEYS)
+	add		$32, RNDKEYS
+	paddd		RCON, RCON		// RCON <<= 1
+	jmp		.Lgen_next_aes256_round_key_pair
+.Lgen_aes256_round_keys_done:
+.endif
+
+	// If INV_RNDKEYS is non-NULL, write the round keys for the Equivalent
+	// Inverse Cipher to it.  To do that, reverse the standard round keys,
+	// and apply aesimc (InvMixColumn) to each except the first and last.
+	test		INV_RNDKEYS, INV_RNDKEYS
+	jz		.Ldone\@
+	movdqu		(RNDKEYS), %xmm0	// Last standard round key
+	movdqu		%xmm0, (INV_RNDKEYS)	// => First inverse round key
+.if \is_aes128
+	mov		$9, COUNTER
+.else
+	mov		$13, COUNTER
+.endif
+.Lgen_next_inv_round_key\@:
+	sub		$16, RNDKEYS
+	add		$16, INV_RNDKEYS
+	movdqu		(RNDKEYS), %xmm0
+	aesimc		%xmm0, %xmm0
+	movdqu		%xmm0, (INV_RNDKEYS)
+	dec		COUNTER
+	jnz		.Lgen_next_inv_round_key\@
+	movdqu		-16(RNDKEYS), %xmm0	// First standard round key
+	movdqu		%xmm0, 16(INV_RNDKEYS)	// => Last inverse round key
+
+.Ldone\@:
+#ifdef __i386__
+	pop		%esi
+	pop		%ebx
+#endif
+	RET
+.endm
+
+// void aes128_expandkey_aesni(u32 rndkeys[], u32 *inv_rndkeys,
+//			       const u8 in_key[AES_KEYSIZE_128]);
+SYM_FUNC_START(aes128_expandkey_aesni)
+	_aes_expandkey_aesni	1
+SYM_FUNC_END(aes128_expandkey_aesni)
+
+// void aes256_expandkey_aesni(u32 rndkeys[], u32 *inv_rndkeys,
+//			       const u8 in_key[AES_KEYSIZE_256]);
+SYM_FUNC_START(aes256_expandkey_aesni)
+	_aes_expandkey_aesni	0
+SYM_FUNC_END(aes256_expandkey_aesni)
+
+.macro	_aes_crypt_aesni	enc
+#ifdef __x86_64__
+	.set	RNDKEYS,	%rdi
+	.set	NROUNDS,	%esi
+	.set	OUT,		%rdx
+	.set	IN,		%rcx
+#else
+	// Assuming -mregparm=3
+	.set	RNDKEYS,	%eax
+	.set	NROUNDS,	%edx
+	.set	OUT,		%ecx
+	.set	IN,		%ebx	// Passed on stack
+#endif
+
+#ifdef __i386__
+	push		%ebx
+	mov		8(%esp), %ebx
+#endif
+
+	// Zero-th round
+	movdqu		(IN), %xmm0
+	movdqu		(RNDKEYS), %xmm1
+	pxor		%xmm1, %xmm0
+
+	// Normal rounds
+	add		$16, RNDKEYS
+	dec		NROUNDS
+.Lnext_round\@:
+	movdqu		(RNDKEYS), %xmm1
+.if \enc
+	aesenc		%xmm1, %xmm0
+.else
+	aesdec		%xmm1, %xmm0
+.endif
+	add		$16, RNDKEYS
+	dec		NROUNDS
+	jne		.Lnext_round\@
+
+	// Last round
+	movdqu		(RNDKEYS), %xmm1
+.if \enc
+	aesenclast	%xmm1, %xmm0
+.else
+	aesdeclast	%xmm1, %xmm0
+.endif
+	movdqu		%xmm0, (OUT)
+
+#ifdef __i386__
+	pop		%ebx
+#endif
+	RET
+.endm
+
+// void aes_encrypt_aesni(const u32 rndkeys[], int nrounds,
+//			  u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]);
+SYM_FUNC_START(aes_encrypt_aesni)
+	_aes_crypt_aesni	1
+SYM_FUNC_END(aes_encrypt_aesni)
+
+// void aes_decrypt_aesni(const u32 inv_rndkeys[], int nrounds,
+//			  u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]);
+SYM_FUNC_START(aes_decrypt_aesni)
+	_aes_crypt_aesni	0
+SYM_FUNC_END(aes_decrypt_aesni)
diff --git a/lib/crypto/x86/aes.h b/lib/crypto/x86/aes.h
new file mode 100644
index 000000000000..b047dee94f57
--- /dev/null
+++ b/lib/crypto/x86/aes.h
@@ -0,0 +1,85 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * AES block cipher using AES-NI instructions
+ *
+ * Copyright 2026 Google LLC
+ */
+
+#include <asm/fpu/api.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes);
+
+void aes128_expandkey_aesni(u32 rndkeys[], u32 *inv_rndkeys,
+			    const u8 in_key[AES_KEYSIZE_128]);
+void aes256_expandkey_aesni(u32 rndkeys[], u32 *inv_rndkeys,
+			    const u8 in_key[AES_KEYSIZE_256]);
+void aes_encrypt_aesni(const u32 rndkeys[], int nrounds,
+		       u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]);
+void aes_decrypt_aesni(const u32 inv_rndkeys[], int nrounds,
+		       u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]);
+
+/*
+ * Expand an AES key using AES-NI if supported and usable or generic code
+ * otherwise.  The expanded key format is compatible between the two cases.  The
+ * outputs are @k->rndkeys (required) and @inv_k->inv_rndkeys (optional).
+ *
+ * We could just always use the generic key expansion code.  AES key expansion
+ * is usually less performance-critical than AES en/decryption.  However,
+ * there's still *some* value in speed here, as well as in non-key-dependent
+ * execution time which AES-NI provides.  So, do use AES-NI to expand AES-128
+ * and AES-256 keys.  (Don't bother with AES-192, as it's almost never used.)
+ */
+static void aes_preparekey_arch(union aes_enckey_arch *k,
+				union aes_invkey_arch *inv_k,
+				const u8 *in_key, int key_len, int nrounds)
+{
+	u32 *rndkeys = k->rndkeys;
+	u32 *inv_rndkeys = inv_k ? inv_k->inv_rndkeys : NULL;
+
+	if (static_branch_likely(&have_aes) && key_len != AES_KEYSIZE_192 &&
+	    irq_fpu_usable()) {
+		kernel_fpu_begin();
+		if (key_len == AES_KEYSIZE_128)
+			aes128_expandkey_aesni(rndkeys, inv_rndkeys, in_key);
+		else
+			aes256_expandkey_aesni(rndkeys, inv_rndkeys, in_key);
+		kernel_fpu_end();
+	} else {
+		aes_expandkey_generic(rndkeys, inv_rndkeys, in_key, key_len);
+	}
+}
+
+static void aes_encrypt_arch(const struct aes_enckey *key,
+			     u8 out[AES_BLOCK_SIZE],
+			     const u8 in[AES_BLOCK_SIZE])
+{
+	if (static_branch_likely(&have_aes) && irq_fpu_usable()) {
+		kernel_fpu_begin();
+		aes_encrypt_aesni(key->k.rndkeys, key->nrounds, out, in);
+		kernel_fpu_end();
+	} else {
+		aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in);
+	}
+}
+
+static void aes_decrypt_arch(const struct aes_key *key,
+			     u8 out[AES_BLOCK_SIZE],
+			     const u8 in[AES_BLOCK_SIZE])
+{
+	if (static_branch_likely(&have_aes) && irq_fpu_usable()) {
+		kernel_fpu_begin();
+		aes_decrypt_aesni(key->inv_k.inv_rndkeys, key->nrounds,
+				  out, in);
+		kernel_fpu_end();
+	} else {
+		aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds,
+				    out, in);
+	}
+}
+
+#define aes_mod_init_arch aes_mod_init_arch
+static void aes_mod_init_arch(void)
+{
+	if (boot_cpu_has(X86_FEATURE_AES))
+		static_branch_enable(&have_aes);
+}
diff --git a/lib/crypto/x86/blake2s-core.S b/lib/crypto/x86/blake2s-core.S
index ef8e9f427aab..7b1d98ca7482 100644
--- a/lib/crypto/x86/blake2s-core.S
+++ b/lib/crypto/x86/blake2s-core.S
@@ -6,19 +6,25 @@
 
 #include <linux/linkage.h>
 
-.section .rodata.cst32.BLAKE2S_IV, "aM", @progbits, 32
+.section .rodata.cst32.iv, "aM", @progbits, 32
 .align 32
-IV:	.octa 0xA54FF53A3C6EF372BB67AE856A09E667
+.Liv:
+	.octa 0xA54FF53A3C6EF372BB67AE856A09E667
 	.octa 0x5BE0CD191F83D9AB9B05688C510E527F
-.section .rodata.cst16.ROT16, "aM", @progbits, 16
+
+.section .rodata.cst16.ror16, "aM", @progbits, 16
 .align 16
-ROT16:	.octa 0x0D0C0F0E09080B0A0504070601000302
-.section .rodata.cst16.ROR328, "aM", @progbits, 16
+.Lror16:
+	.octa 0x0D0C0F0E09080B0A0504070601000302
+
+.section .rodata.cst16.ror8, "aM", @progbits, 16
 .align 16
-ROR328:	.octa 0x0C0F0E0D080B0A090407060500030201
-.section .rodata.cst64.BLAKE2S_SIGMA, "aM", @progbits, 160
+.Lror8:
+	.octa 0x0C0F0E0D080B0A090407060500030201
+
+.section .rodata.cst64.sigma, "aM", @progbits, 160
 .align 64
-SIGMA:
+.Lsigma:
 .byte  0,  2,  4,  6,  1,  3,  5,  7, 14,  8, 10, 12, 15,  9, 11, 13
 .byte 14,  4,  9, 13, 10,  8, 15,  6,  5,  1,  0, 11,  3, 12,  2,  7
 .byte 11, 12,  5, 15,  8,  0,  2, 13,  9, 10,  3,  7,  4, 14,  6,  1
@@ -29,9 +35,10 @@ SIGMA:
 .byte 13,  7, 12,  3, 11, 14,  1,  9,  2,  5, 15,  8, 10,  0,  4,  6
 .byte  6, 14, 11,  0, 15,  9,  3,  8, 10, 12, 13,  1,  5,  2,  7,  4
 .byte 10,  8,  7,  1,  2,  4,  6,  5, 13, 15,  9,  3,  0, 11, 14, 12
-.section .rodata.cst64.BLAKE2S_SIGMA2, "aM", @progbits, 160
+
+.section .rodata.cst64.sigma2, "aM", @progbits, 160
 .align 64
-SIGMA2:
+.Lsigma2:
 .byte  0,  2,  4,  6,  1,  3,  5,  7, 14,  8, 10, 12, 15,  9, 11, 13
 .byte  8,  2, 13, 15, 10,  9, 12,  3,  6,  4,  0, 14,  5, 11,  1,  7
 .byte 11, 13,  8,  6,  5, 10, 14,  3,  2,  4, 12, 15,  1,  0,  7,  9
@@ -43,36 +50,52 @@ SIGMA2:
 .byte 15,  5,  4, 13, 10,  7,  3, 11, 12,  2,  0,  6,  9,  8,  1, 14
 .byte  8,  7, 14, 11, 13, 15,  0, 12, 10,  4,  5,  6,  3,  2,  1,  9
 
+#define CTX		%rdi
+#define DATA		%rsi
+#define NBLOCKS		%rdx
+#define INC		%ecx
+
 .text
+//
+// void blake2s_compress_ssse3(struct blake2s_ctx *ctx,
+//			       const u8 *data, size_t nblocks, u32 inc);
+//
+// Only the first three fields of struct blake2s_ctx are used:
+//	u32 h[8];	(inout)
+//	u32 t[2];	(inout)
+//	u32 f[2];	(in)
+//
 SYM_FUNC_START(blake2s_compress_ssse3)
-	testq		%rdx,%rdx
-	je		.Lendofloop
-	movdqu		(%rdi),%xmm0
-	movdqu		0x10(%rdi),%xmm1
-	movdqa		ROT16(%rip),%xmm12
-	movdqa		ROR328(%rip),%xmm13
-	movdqu		0x20(%rdi),%xmm14
-	movq		%rcx,%xmm15
-	leaq		SIGMA+0xa0(%rip),%r8
-	jmp		.Lbeginofloop
+	movdqu		(CTX),%xmm0		// Load h[0..3]
+	movdqu		16(CTX),%xmm1		// Load h[4..7]
+	movdqa		.Lror16(%rip),%xmm12
+	movdqa		.Lror8(%rip),%xmm13
+	movdqu		32(CTX),%xmm14		// Load t and f
+	movd		INC,%xmm15		// Load inc
+	leaq		.Lsigma+160(%rip),%r8
+	jmp		.Lssse3_mainloop
+
 	.align		32
-.Lbeginofloop:
-	movdqa		%xmm0,%xmm10
-	movdqa		%xmm1,%xmm11
-	paddq		%xmm15,%xmm14
-	movdqa		IV(%rip),%xmm2
+.Lssse3_mainloop:
+	// Main loop: each iteration processes one 64-byte block.
+	movdqa		%xmm0,%xmm10		// Save h[0..3] and let v[0..3] = h[0..3]
+	movdqa		%xmm1,%xmm11		// Save h[4..7] and let v[4..7] = h[4..7]
+	paddq		%xmm15,%xmm14		// t += inc (64-bit addition)
+	movdqa		.Liv(%rip),%xmm2	// v[8..11] = iv[0..3]
 	movdqa		%xmm14,%xmm3
-	pxor		IV+0x10(%rip),%xmm3
-	leaq		SIGMA(%rip),%rcx
-.Lroundloop:
+	pxor		.Liv+16(%rip),%xmm3	// v[12..15] = iv[4..7] ^ [t, f]
+	leaq		.Lsigma(%rip),%rcx
+
+.Lssse3_roundloop:
+	// Round loop: each iteration does 1 round (of 10 rounds total).
 	movzbl		(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm4
-	movzbl		0x1(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm5
-	movzbl		0x2(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm6
-	movzbl		0x3(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm7
+	movd		(DATA,%rax,4),%xmm4
+	movzbl		1(%rcx),%eax
+	movd		(DATA,%rax,4),%xmm5
+	movzbl		2(%rcx),%eax
+	movd		(DATA,%rax,4),%xmm6
+	movzbl		3(%rcx),%eax
+	movd		(DATA,%rax,4),%xmm7
 	punpckldq	%xmm5,%xmm4
 	punpckldq	%xmm7,%xmm6
 	punpcklqdq	%xmm6,%xmm4
@@ -83,17 +106,17 @@ SYM_FUNC_START(blake2s_compress_ssse3)
 	paddd		%xmm3,%xmm2
 	pxor		%xmm2,%xmm1
 	movdqa		%xmm1,%xmm8
-	psrld		$0xc,%xmm1
-	pslld		$0x14,%xmm8
+	psrld		$12,%xmm1
+	pslld		$20,%xmm8
 	por		%xmm8,%xmm1
-	movzbl		0x4(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm5
-	movzbl		0x5(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm6
-	movzbl		0x6(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm7
-	movzbl		0x7(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm4
+	movzbl		4(%rcx),%eax
+	movd		(DATA,%rax,4),%xmm5
+	movzbl		5(%rcx),%eax
+	movd		(DATA,%rax,4),%xmm6
+	movzbl		6(%rcx),%eax
+	movd		(DATA,%rax,4),%xmm7
+	movzbl		7(%rcx),%eax
+	movd		(DATA,%rax,4),%xmm4
 	punpckldq	%xmm6,%xmm5
 	punpckldq	%xmm4,%xmm7
 	punpcklqdq	%xmm7,%xmm5
@@ -104,20 +127,20 @@ SYM_FUNC_START(blake2s_compress_ssse3)
 	paddd		%xmm3,%xmm2
 	pxor		%xmm2,%xmm1
 	movdqa		%xmm1,%xmm8
-	psrld		$0x7,%xmm1
-	pslld		$0x19,%xmm8
+	psrld		$7,%xmm1
+	pslld		$25,%xmm8
 	por		%xmm8,%xmm1
 	pshufd		$0x93,%xmm0,%xmm0
 	pshufd		$0x4e,%xmm3,%xmm3
 	pshufd		$0x39,%xmm2,%xmm2
-	movzbl		0x8(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm6
-	movzbl		0x9(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm7
-	movzbl		0xa(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm4
-	movzbl		0xb(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm5
+	movzbl		8(%rcx),%eax
+	movd		(DATA,%rax,4),%xmm6
+	movzbl		9(%rcx),%eax
+	movd		(DATA,%rax,4),%xmm7
+	movzbl		10(%rcx),%eax
+	movd		(DATA,%rax,4),%xmm4
+	movzbl		11(%rcx),%eax
+	movd		(DATA,%rax,4),%xmm5
 	punpckldq	%xmm7,%xmm6
 	punpckldq	%xmm5,%xmm4
 	punpcklqdq	%xmm4,%xmm6
@@ -128,17 +151,17 @@ SYM_FUNC_START(blake2s_compress_ssse3)
 	paddd		%xmm3,%xmm2
 	pxor		%xmm2,%xmm1
 	movdqa		%xmm1,%xmm8
-	psrld		$0xc,%xmm1
-	pslld		$0x14,%xmm8
+	psrld		$12,%xmm1
+	pslld		$20,%xmm8
 	por		%xmm8,%xmm1
-	movzbl		0xc(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm7
-	movzbl		0xd(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm4
-	movzbl		0xe(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm5
-	movzbl		0xf(%rcx),%eax
-	movd		(%rsi,%rax,4),%xmm6
+	movzbl		12(%rcx),%eax
+	movd		(DATA,%rax,4),%xmm7
+	movzbl		13(%rcx),%eax
+	movd		(DATA,%rax,4),%xmm4
+	movzbl		14(%rcx),%eax
+	movd		(DATA,%rax,4),%xmm5
+	movzbl		15(%rcx),%eax
+	movd		(DATA,%rax,4),%xmm6
 	punpckldq	%xmm4,%xmm7
 	punpckldq	%xmm6,%xmm5
 	punpcklqdq	%xmm5,%xmm7
@@ -149,53 +172,68 @@ SYM_FUNC_START(blake2s_compress_ssse3)
 	paddd		%xmm3,%xmm2
 	pxor		%xmm2,%xmm1
 	movdqa		%xmm1,%xmm8
-	psrld		$0x7,%xmm1
-	pslld		$0x19,%xmm8
+	psrld		$7,%xmm1
+	pslld		$25,%xmm8
 	por		%xmm8,%xmm1
 	pshufd		$0x39,%xmm0,%xmm0
 	pshufd		$0x4e,%xmm3,%xmm3
 	pshufd		$0x93,%xmm2,%xmm2
-	addq		$0x10,%rcx
+	addq		$16,%rcx
 	cmpq		%r8,%rcx
-	jnz		.Lroundloop
+	jnz		.Lssse3_roundloop
+
+	// Compute the new h: h[0..7] ^= v[0..7] ^ v[8..15]
 	pxor		%xmm2,%xmm0
 	pxor		%xmm3,%xmm1
 	pxor		%xmm10,%xmm0
 	pxor		%xmm11,%xmm1
-	addq		$0x40,%rsi
-	decq		%rdx
-	jnz		.Lbeginofloop
-	movdqu		%xmm0,(%rdi)
-	movdqu		%xmm1,0x10(%rdi)
-	movdqu		%xmm14,0x20(%rdi)
-.Lendofloop:
+	addq		$64,DATA
+	decq		NBLOCKS
+	jnz		.Lssse3_mainloop
+
+	movdqu		%xmm0,(CTX)		// Store new h[0..3]
+	movdqu		%xmm1,16(CTX)		// Store new h[4..7]
+	movq		%xmm14,32(CTX)		// Store new t (f is unchanged)
 	RET
 SYM_FUNC_END(blake2s_compress_ssse3)
 
+//
+// void blake2s_compress_avx512(struct blake2s_ctx *ctx,
+//				const u8 *data, size_t nblocks, u32 inc);
+//
+// Only the first three fields of struct blake2s_ctx are used:
+//	u32 h[8];	(inout)
+//	u32 t[2];	(inout)
+//	u32 f[2];	(in)
+//
 SYM_FUNC_START(blake2s_compress_avx512)
-	vmovdqu		(%rdi),%xmm0
-	vmovdqu		0x10(%rdi),%xmm1
-	vmovdqu		0x20(%rdi),%xmm4
-	vmovq		%rcx,%xmm5
-	vmovdqa		IV(%rip),%xmm14
-	vmovdqa		IV+16(%rip),%xmm15
-	jmp		.Lblake2s_compress_avx512_mainloop
-.align 32
-.Lblake2s_compress_avx512_mainloop:
-	vmovdqa		%xmm0,%xmm10
-	vmovdqa		%xmm1,%xmm11
-	vpaddq		%xmm5,%xmm4,%xmm4
-	vmovdqa		%xmm14,%xmm2
-	vpxor		%xmm15,%xmm4,%xmm3
-	vmovdqu		(%rsi),%ymm6
-	vmovdqu		0x20(%rsi),%ymm7
-	addq		$0x40,%rsi
-	leaq		SIGMA2(%rip),%rax
-	movb		$0xa,%cl
-.Lblake2s_compress_avx512_roundloop:
+	vmovdqu		(CTX),%xmm0		// Load h[0..3]
+	vmovdqu		16(CTX),%xmm1		// Load h[4..7]
+	vmovdqu		32(CTX),%xmm4		// Load t and f
+	vmovd		INC,%xmm5		// Load inc
+	vmovdqa		.Liv(%rip),%xmm14	// Load iv[0..3]
+	vmovdqa		.Liv+16(%rip),%xmm15	// Load iv[4..7]
+	jmp		.Lavx512_mainloop
+
+	.align		32
+.Lavx512_mainloop:
+	// Main loop: each iteration processes one 64-byte block.
+	vmovdqa		%xmm0,%xmm10		// Save h[0..3] and let v[0..3] = h[0..3]
+	vmovdqa		%xmm1,%xmm11		// Save h[4..7] and let v[4..7] = h[4..7]
+	vpaddq		%xmm5,%xmm4,%xmm4	// t += inc (64-bit addition)
+	vmovdqa		%xmm14,%xmm2		// v[8..11] = iv[0..3]
+	vpxor		%xmm15,%xmm4,%xmm3	// v[12..15] = iv[4..7] ^ [t, f]
+	vmovdqu		(DATA),%ymm6		// Load first 8 data words
+	vmovdqu		32(DATA),%ymm7		// Load second 8 data words
+	addq		$64,DATA
+	leaq		.Lsigma2(%rip),%rax
+	movb		$10,%cl			// Set num rounds remaining
+
+.Lavx512_roundloop:
+	// Round loop: each iteration does 1 round (of 10 rounds total).
 	vpmovzxbd	(%rax),%ymm8
-	vpmovzxbd	0x8(%rax),%ymm9
-	addq		$0x10,%rax
+	vpmovzxbd	8(%rax),%ymm9
+	addq		$16,%rax
 	vpermi2d	%ymm7,%ymm6,%ymm8
 	vpermi2d	%ymm7,%ymm6,%ymm9
 	vmovdqa		%ymm8,%ymm6
@@ -203,50 +241,51 @@ SYM_FUNC_START(blake2s_compress_avx512)
 	vpaddd		%xmm8,%xmm0,%xmm0
 	vpaddd		%xmm1,%xmm0,%xmm0
 	vpxor		%xmm0,%xmm3,%xmm3
-	vprord		$0x10,%xmm3,%xmm3
+	vprord		$16,%xmm3,%xmm3
 	vpaddd		%xmm3,%xmm2,%xmm2
 	vpxor		%xmm2,%xmm1,%xmm1
-	vprord		$0xc,%xmm1,%xmm1
-	vextracti128	$0x1,%ymm8,%xmm8
+	vprord		$12,%xmm1,%xmm1
+	vextracti128	$1,%ymm8,%xmm8
 	vpaddd		%xmm8,%xmm0,%xmm0
 	vpaddd		%xmm1,%xmm0,%xmm0
 	vpxor		%xmm0,%xmm3,%xmm3
-	vprord		$0x8,%xmm3,%xmm3
+	vprord		$8,%xmm3,%xmm3
 	vpaddd		%xmm3,%xmm2,%xmm2
 	vpxor		%xmm2,%xmm1,%xmm1
-	vprord		$0x7,%xmm1,%xmm1
+	vprord		$7,%xmm1,%xmm1
 	vpshufd		$0x93,%xmm0,%xmm0
 	vpshufd		$0x4e,%xmm3,%xmm3
 	vpshufd		$0x39,%xmm2,%xmm2
 	vpaddd		%xmm9,%xmm0,%xmm0
 	vpaddd		%xmm1,%xmm0,%xmm0
 	vpxor		%xmm0,%xmm3,%xmm3
-	vprord		$0x10,%xmm3,%xmm3
+	vprord		$16,%xmm3,%xmm3
 	vpaddd		%xmm3,%xmm2,%xmm2
 	vpxor		%xmm2,%xmm1,%xmm1
-	vprord		$0xc,%xmm1,%xmm1
-	vextracti128	$0x1,%ymm9,%xmm9
+	vprord		$12,%xmm1,%xmm1
+	vextracti128	$1,%ymm9,%xmm9
 	vpaddd		%xmm9,%xmm0,%xmm0
 	vpaddd		%xmm1,%xmm0,%xmm0
 	vpxor		%xmm0,%xmm3,%xmm3
-	vprord		$0x8,%xmm3,%xmm3
+	vprord		$8,%xmm3,%xmm3
 	vpaddd		%xmm3,%xmm2,%xmm2
 	vpxor		%xmm2,%xmm1,%xmm1
-	vprord		$0x7,%xmm1,%xmm1
+	vprord		$7,%xmm1,%xmm1
 	vpshufd		$0x39,%xmm0,%xmm0
 	vpshufd		$0x4e,%xmm3,%xmm3
 	vpshufd		$0x93,%xmm2,%xmm2
 	decb		%cl
-	jne		.Lblake2s_compress_avx512_roundloop
-	vpxor		%xmm10,%xmm0,%xmm0
-	vpxor		%xmm11,%xmm1,%xmm1
-	vpxor		%xmm2,%xmm0,%xmm0
-	vpxor		%xmm3,%xmm1,%xmm1
-	decq		%rdx
-	jne		.Lblake2s_compress_avx512_mainloop
-	vmovdqu		%xmm0,(%rdi)
-	vmovdqu		%xmm1,0x10(%rdi)
-	vmovdqu		%xmm4,0x20(%rdi)
+	jne		.Lavx512_roundloop
+
+	// Compute the new h: h[0..7] ^= v[0..7] ^ v[8..15]
+	vpternlogd	$0x96,%xmm10,%xmm2,%xmm0
+	vpternlogd	$0x96,%xmm11,%xmm3,%xmm1
+	decq		NBLOCKS
+	jne		.Lavx512_mainloop
+
+	vmovdqu		%xmm0,(CTX)		// Store new h[0..3]
+	vmovdqu		%xmm1,16(CTX)		// Store new h[4..7]
+	vmovq		%xmm4,32(CTX)		// Store new t (f is unchanged)
 	vzeroupper
 	RET
 SYM_FUNC_END(blake2s_compress_avx512)
diff --git a/lib/crypto/x86/blake2s.h b/lib/crypto/x86/blake2s.h
index b6d30d2fa045..f8eed6cb042e 100644
--- a/lib/crypto/x86/blake2s.h
+++ b/lib/crypto/x86/blake2s.h
@@ -11,24 +11,22 @@
 #include <linux/kernel.h>
 #include <linux/sizes.h>
 
-asmlinkage void blake2s_compress_ssse3(struct blake2s_state *state,
-				       const u8 *block, const size_t nblocks,
-				       const u32 inc);
-asmlinkage void blake2s_compress_avx512(struct blake2s_state *state,
-					const u8 *block, const size_t nblocks,
-					const u32 inc);
+asmlinkage void blake2s_compress_ssse3(struct blake2s_ctx *ctx,
+				       const u8 *data, size_t nblocks, u32 inc);
+asmlinkage void blake2s_compress_avx512(struct blake2s_ctx *ctx,
+					const u8 *data, size_t nblocks, u32 inc);
 
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_ssse3);
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_avx512);
 
-static void blake2s_compress(struct blake2s_state *state, const u8 *block,
-			     size_t nblocks, const u32 inc)
+static void blake2s_compress(struct blake2s_ctx *ctx,
+			     const u8 *data, size_t nblocks, u32 inc)
 {
 	/* SIMD disables preemption, so relax after processing each page. */
 	BUILD_BUG_ON(SZ_4K / BLAKE2S_BLOCK_SIZE < 8);
 
 	if (!static_branch_likely(&blake2s_use_ssse3) || !may_use_simd()) {
-		blake2s_compress_generic(state, block, nblocks, inc);
+		blake2s_compress_generic(ctx, data, nblocks, inc);
 		return;
 	}
 
@@ -38,13 +36,13 @@ static void blake2s_compress(struct blake2s_state *state, const u8 *block,
 
 		kernel_fpu_begin();
 		if (static_branch_likely(&blake2s_use_avx512))
-			blake2s_compress_avx512(state, block, blocks, inc);
+			blake2s_compress_avx512(ctx, data, blocks, inc);
 		else
-			blake2s_compress_ssse3(state, block, blocks, inc);
+			blake2s_compress_ssse3(ctx, data, blocks, inc);
 		kernel_fpu_end();
 
+		data += blocks * BLAKE2S_BLOCK_SIZE;
 		nblocks -= blocks;
-		block += blocks * BLAKE2S_BLOCK_SIZE;
 	} while (nblocks);
 }
 
diff --git a/lib/crypto/x86/nh-avx2.S b/lib/crypto/x86/nh-avx2.S
new file mode 100644
index 000000000000..9c085a31b137
--- /dev/null
+++ b/lib/crypto/x86/nh-avx2.S
@@ -0,0 +1,158 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * NH - ε-almost-universal hash function, x86_64 AVX2 accelerated
+ *
+ * Copyright 2018 Google LLC
+ *
+ * Author: Eric Biggers <ebiggers@google.com>
+ */
+
+#include <linux/linkage.h>
+
+#define		PASS0_SUMS	%ymm0
+#define		PASS1_SUMS	%ymm1
+#define		PASS2_SUMS	%ymm2
+#define		PASS3_SUMS	%ymm3
+#define		K0		%ymm4
+#define		K0_XMM		%xmm4
+#define		K1		%ymm5
+#define		K1_XMM		%xmm5
+#define		K2		%ymm6
+#define		K2_XMM		%xmm6
+#define		K3		%ymm7
+#define		K3_XMM		%xmm7
+#define		T0		%ymm8
+#define		T1		%ymm9
+#define		T2		%ymm10
+#define		T2_XMM		%xmm10
+#define		T3		%ymm11
+#define		T3_XMM		%xmm11
+#define		T4		%ymm12
+#define		T5		%ymm13
+#define		T6		%ymm14
+#define		T7		%ymm15
+#define		KEY		%rdi
+#define		MESSAGE		%rsi
+#define		MESSAGE_LEN	%rdx
+#define		HASH		%rcx
+
+.macro _nh_2xstride	k0, k1, k2, k3
+
+	// Add message words to key words
+	vpaddd		\k0, T3, T0
+	vpaddd		\k1, T3, T1
+	vpaddd		\k2, T3, T2
+	vpaddd		\k3, T3, T3
+
+	// Multiply 32x32 => 64 and accumulate
+	vpshufd		$0x10, T0, T4
+	vpshufd		$0x32, T0, T0
+	vpshufd		$0x10, T1, T5
+	vpshufd		$0x32, T1, T1
+	vpshufd		$0x10, T2, T6
+	vpshufd		$0x32, T2, T2
+	vpshufd		$0x10, T3, T7
+	vpshufd		$0x32, T3, T3
+	vpmuludq	T4, T0, T0
+	vpmuludq	T5, T1, T1
+	vpmuludq	T6, T2, T2
+	vpmuludq	T7, T3, T3
+	vpaddq		T0, PASS0_SUMS, PASS0_SUMS
+	vpaddq		T1, PASS1_SUMS, PASS1_SUMS
+	vpaddq		T2, PASS2_SUMS, PASS2_SUMS
+	vpaddq		T3, PASS3_SUMS, PASS3_SUMS
+.endm
+
+/*
+ * void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
+ *		__le64 hash[NH_NUM_PASSES])
+ *
+ * It's guaranteed that message_len % 16 == 0.
+ */
+SYM_FUNC_START(nh_avx2)
+
+	vmovdqu		0x00(KEY), K0
+	vmovdqu		0x10(KEY), K1
+	add		$0x20, KEY
+	vpxor		PASS0_SUMS, PASS0_SUMS, PASS0_SUMS
+	vpxor		PASS1_SUMS, PASS1_SUMS, PASS1_SUMS
+	vpxor		PASS2_SUMS, PASS2_SUMS, PASS2_SUMS
+	vpxor		PASS3_SUMS, PASS3_SUMS, PASS3_SUMS
+
+	sub		$0x40, MESSAGE_LEN
+	jl		.Lloop4_done
+.Lloop4:
+	vmovdqu		(MESSAGE), T3
+	vmovdqu		0x00(KEY), K2
+	vmovdqu		0x10(KEY), K3
+	_nh_2xstride	K0, K1, K2, K3
+
+	vmovdqu		0x20(MESSAGE), T3
+	vmovdqu		0x20(KEY), K0
+	vmovdqu		0x30(KEY), K1
+	_nh_2xstride	K2, K3, K0, K1
+
+	add		$0x40, MESSAGE
+	add		$0x40, KEY
+	sub		$0x40, MESSAGE_LEN
+	jge		.Lloop4
+
+.Lloop4_done:
+	and		$0x3f, MESSAGE_LEN
+	jz		.Ldone
+
+	cmp		$0x20, MESSAGE_LEN
+	jl		.Llast
+
+	// 2 or 3 strides remain; do 2 more.
+	vmovdqu		(MESSAGE), T3
+	vmovdqu		0x00(KEY), K2
+	vmovdqu		0x10(KEY), K3
+	_nh_2xstride	K0, K1, K2, K3
+	add		$0x20, MESSAGE
+	add		$0x20, KEY
+	sub		$0x20, MESSAGE_LEN
+	jz		.Ldone
+	vmovdqa		K2, K0
+	vmovdqa		K3, K1
+.Llast:
+	// Last stride.  Zero the high 128 bits of the message and keys so they
+	// don't affect the result when processing them like 2 strides.
+	vmovdqu		(MESSAGE), T3_XMM
+	vmovdqa		K0_XMM, K0_XMM
+	vmovdqa		K1_XMM, K1_XMM
+	vmovdqu		0x00(KEY), K2_XMM
+	vmovdqu		0x10(KEY), K3_XMM
+	_nh_2xstride	K0, K1, K2, K3
+
+.Ldone:
+	// Sum the accumulators for each pass, then store the sums to 'hash'
+
+	// PASS0_SUMS is (0A 0B 0C 0D)
+	// PASS1_SUMS is (1A 1B 1C 1D)
+	// PASS2_SUMS is (2A 2B 2C 2D)
+	// PASS3_SUMS is (3A 3B 3C 3D)
+	// We need the horizontal sums:
+	//     (0A + 0B + 0C + 0D,
+	//	1A + 1B + 1C + 1D,
+	//	2A + 2B + 2C + 2D,
+	//	3A + 3B + 3C + 3D)
+	//
+
+	vpunpcklqdq	PASS1_SUMS, PASS0_SUMS, T0	// T0 = (0A 1A 0C 1C)
+	vpunpckhqdq	PASS1_SUMS, PASS0_SUMS, T1	// T1 = (0B 1B 0D 1D)
+	vpunpcklqdq	PASS3_SUMS, PASS2_SUMS, T2	// T2 = (2A 3A 2C 3C)
+	vpunpckhqdq	PASS3_SUMS, PASS2_SUMS, T3	// T3 = (2B 3B 2D 3D)
+
+	vinserti128	$0x1, T2_XMM, T0, T4		// T4 = (0A 1A 2A 3A)
+	vinserti128	$0x1, T3_XMM, T1, T5		// T5 = (0B 1B 2B 3B)
+	vperm2i128	$0x31, T2, T0, T0		// T0 = (0C 1C 2C 3C)
+	vperm2i128	$0x31, T3, T1, T1		// T1 = (0D 1D 2D 3D)
+
+	vpaddq		T5, T4, T4
+	vpaddq		T1, T0, T0
+	vpaddq		T4, T0, T0
+	vmovdqu		T0, (HASH)
+	vzeroupper
+	RET
+SYM_FUNC_END(nh_avx2)
diff --git a/lib/crypto/x86/nh-sse2.S b/lib/crypto/x86/nh-sse2.S
new file mode 100644
index 000000000000..d36c0e6d5556
--- /dev/null
+++ b/lib/crypto/x86/nh-sse2.S
@@ -0,0 +1,123 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * NH - ε-almost-universal hash function, x86_64 SSE2 accelerated
+ *
+ * Copyright 2018 Google LLC
+ *
+ * Author: Eric Biggers <ebiggers@google.com>
+ */
+
+#include <linux/linkage.h>
+
+#define		PASS0_SUMS	%xmm0
+#define		PASS1_SUMS	%xmm1
+#define		PASS2_SUMS	%xmm2
+#define		PASS3_SUMS	%xmm3
+#define		K0		%xmm4
+#define		K1		%xmm5
+#define		K2		%xmm6
+#define		K3		%xmm7
+#define		T0		%xmm8
+#define		T1		%xmm9
+#define		T2		%xmm10
+#define		T3		%xmm11
+#define		T4		%xmm12
+#define		T5		%xmm13
+#define		T6		%xmm14
+#define		T7		%xmm15
+#define		KEY		%rdi
+#define		MESSAGE		%rsi
+#define		MESSAGE_LEN	%rdx
+#define		HASH		%rcx
+
+.macro _nh_stride	k0, k1, k2, k3, offset
+
+	// Load next message stride
+	movdqu		\offset(MESSAGE), T1
+
+	// Load next key stride
+	movdqu		\offset(KEY), \k3
+
+	// Add message words to key words
+	movdqa		T1, T2
+	movdqa		T1, T3
+	paddd		T1, \k0    // reuse k0 to avoid a move
+	paddd		\k1, T1
+	paddd		\k2, T2
+	paddd		\k3, T3
+
+	// Multiply 32x32 => 64 and accumulate
+	pshufd		$0x10, \k0, T4
+	pshufd		$0x32, \k0, \k0
+	pshufd		$0x10, T1, T5
+	pshufd		$0x32, T1, T1
+	pshufd		$0x10, T2, T6
+	pshufd		$0x32, T2, T2
+	pshufd		$0x10, T3, T7
+	pshufd		$0x32, T3, T3
+	pmuludq		T4, \k0
+	pmuludq		T5, T1
+	pmuludq		T6, T2
+	pmuludq		T7, T3
+	paddq		\k0, PASS0_SUMS
+	paddq		T1, PASS1_SUMS
+	paddq		T2, PASS2_SUMS
+	paddq		T3, PASS3_SUMS
+.endm
+
+/*
+ * void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
+ *		__le64 hash[NH_NUM_PASSES])
+ *
+ * It's guaranteed that message_len % 16 == 0.
+ */
+SYM_FUNC_START(nh_sse2)
+
+	movdqu		0x00(KEY), K0
+	movdqu		0x10(KEY), K1
+	movdqu		0x20(KEY), K2
+	add		$0x30, KEY
+	pxor		PASS0_SUMS, PASS0_SUMS
+	pxor		PASS1_SUMS, PASS1_SUMS
+	pxor		PASS2_SUMS, PASS2_SUMS
+	pxor		PASS3_SUMS, PASS3_SUMS
+
+	sub		$0x40, MESSAGE_LEN
+	jl		.Lloop4_done
+.Lloop4:
+	_nh_stride	K0, K1, K2, K3, 0x00
+	_nh_stride	K1, K2, K3, K0, 0x10
+	_nh_stride	K2, K3, K0, K1, 0x20
+	_nh_stride	K3, K0, K1, K2, 0x30
+	add		$0x40, KEY
+	add		$0x40, MESSAGE
+	sub		$0x40, MESSAGE_LEN
+	jge		.Lloop4
+
+.Lloop4_done:
+	and		$0x3f, MESSAGE_LEN
+	jz		.Ldone
+	_nh_stride	K0, K1, K2, K3, 0x00
+
+	sub		$0x10, MESSAGE_LEN
+	jz		.Ldone
+	_nh_stride	K1, K2, K3, K0, 0x10
+
+	sub		$0x10, MESSAGE_LEN
+	jz		.Ldone
+	_nh_stride	K2, K3, K0, K1, 0x20
+
+.Ldone:
+	// Sum the accumulators for each pass, then store the sums to 'hash'
+	movdqa		PASS0_SUMS, T0
+	movdqa		PASS2_SUMS, T1
+	punpcklqdq	PASS1_SUMS, T0		// => (PASS0_SUM_A PASS1_SUM_A)
+	punpcklqdq	PASS3_SUMS, T1		// => (PASS2_SUM_A PASS3_SUM_A)
+	punpckhqdq	PASS1_SUMS, PASS0_SUMS	// => (PASS0_SUM_B PASS1_SUM_B)
+	punpckhqdq	PASS3_SUMS, PASS2_SUMS	// => (PASS2_SUM_B PASS3_SUM_B)
+	paddq		PASS0_SUMS, T0
+	paddq		PASS2_SUMS, T1
+	movdqu		T0, 0x00(HASH)
+	movdqu		T1, 0x10(HASH)
+	RET
+SYM_FUNC_END(nh_sse2)
diff --git a/lib/crypto/x86/nh.h b/lib/crypto/x86/nh.h
new file mode 100644
index 000000000000..83361c2e9783
--- /dev/null
+++ b/lib/crypto/x86/nh.h
@@ -0,0 +1,45 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * x86_64 accelerated implementation of NH
+ *
+ * Copyright 2018 Google LLC
+ */
+
+#include <asm/fpu/api.h>
+#include <linux/static_call.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sse2);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_avx2);
+
+asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
+			__le64 hash[NH_NUM_PASSES]);
+asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
+			__le64 hash[NH_NUM_PASSES]);
+
+static bool nh_arch(const u32 *key, const u8 *message, size_t message_len,
+		    __le64 hash[NH_NUM_PASSES])
+{
+	if (message_len >= 64 && static_branch_likely(&have_sse2) &&
+	    irq_fpu_usable()) {
+		kernel_fpu_begin();
+		if (static_branch_likely(&have_avx2))
+			nh_avx2(key, message, message_len, hash);
+		else
+			nh_sse2(key, message, message_len, hash);
+		kernel_fpu_end();
+		return true;
+	}
+	return false;
+}
+
+#define nh_mod_init_arch nh_mod_init_arch
+static void nh_mod_init_arch(void)
+{
+	if (boot_cpu_has(X86_FEATURE_XMM2)) {
+		static_branch_enable(&have_sse2);
+		if (boot_cpu_has(X86_FEATURE_AVX2) &&
+		    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
+				      NULL))
+			static_branch_enable(&have_avx2);
+	}
+}
diff --git a/lib/crypto/x86/polyval-pclmul-avx.S b/lib/crypto/x86/polyval-pclmul-avx.S
new file mode 100644
index 000000000000..7f739465ad35
--- /dev/null
+++ b/lib/crypto/x86/polyval-pclmul-avx.S
@@ -0,0 +1,319 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2021 Google LLC
+ */
+/*
+ * This is an efficient implementation of POLYVAL using intel PCLMULQDQ-NI
+ * instructions. It works on 8 blocks at a time, by precomputing the first 8
+ * keys powers h^8, ..., h^1 in the POLYVAL finite field. This precomputation
+ * allows us to split finite field multiplication into two steps.
+ *
+ * In the first step, we consider h^i, m_i as normal polynomials of degree less
+ * than 128. We then compute p(x) = h^8m_0 + ... + h^1m_7 where multiplication
+ * is simply polynomial multiplication.
+ *
+ * In the second step, we compute the reduction of p(x) modulo the finite field
+ * modulus g(x) = x^128 + x^127 + x^126 + x^121 + 1.
+ *
+ * This two step process is equivalent to computing h^8m_0 + ... + h^1m_7 where
+ * multiplication is finite field multiplication. The advantage is that the
+ * two-step process  only requires 1 finite field reduction for every 8
+ * polynomial multiplications. Further parallelism is gained by interleaving the
+ * multiplications and polynomial reductions.
+ */
+
+#include <linux/linkage.h>
+#include <asm/frame.h>
+
+#define STRIDE_BLOCKS 8
+
+#define GSTAR %xmm7
+#define PL %xmm8
+#define PH %xmm9
+#define TMP_XMM %xmm11
+#define LO %xmm12
+#define HI %xmm13
+#define MI %xmm14
+#define SUM %xmm15
+
+#define ACCUMULATOR %rdi
+#define KEY_POWERS %rsi
+#define MSG %rdx
+#define BLOCKS_LEFT %rcx
+#define TMP %rax
+
+.section    .rodata.cst16.gstar, "aM", @progbits, 16
+.align 16
+
+.Lgstar:
+	.quad 0xc200000000000000, 0xc200000000000000
+
+.text
+
+/*
+ * Performs schoolbook1_iteration on two lists of 128-bit polynomials of length
+ * count pointed to by MSG and KEY_POWERS.
+ */
+.macro schoolbook1 count
+	.set i, 0
+	.rept (\count)
+		schoolbook1_iteration i 0
+		.set i, (i +1)
+	.endr
+.endm
+
+/*
+ * Computes the product of two 128-bit polynomials at the memory locations
+ * specified by (MSG + 16*i) and (KEY_POWERS + 16*i) and XORs the components of
+ * the 256-bit product into LO, MI, HI.
+ *
+ * Given:
+ *   X = [X_1 : X_0]
+ *   Y = [Y_1 : Y_0]
+ *
+ * We compute:
+ *   LO += X_0 * Y_0
+ *   MI += X_0 * Y_1 + X_1 * Y_0
+ *   HI += X_1 * Y_1
+ *
+ * Later, the 256-bit result can be extracted as:
+ *   [HI_1 : HI_0 + MI_1 : LO_1 + MI_0 : LO_0]
+ * This step is done when computing the polynomial reduction for efficiency
+ * reasons.
+ *
+ * If xor_sum == 1, then also XOR the value of SUM into m_0.  This avoids an
+ * extra multiplication of SUM and h^8.
+ */
+.macro schoolbook1_iteration i xor_sum
+	movups (16*\i)(MSG), %xmm0
+	.if (\i == 0 && \xor_sum == 1)
+		pxor SUM, %xmm0
+	.endif
+	vpclmulqdq $0x01, (16*\i)(KEY_POWERS), %xmm0, %xmm2
+	vpclmulqdq $0x00, (16*\i)(KEY_POWERS), %xmm0, %xmm1
+	vpclmulqdq $0x10, (16*\i)(KEY_POWERS), %xmm0, %xmm3
+	vpclmulqdq $0x11, (16*\i)(KEY_POWERS), %xmm0, %xmm4
+	vpxor %xmm2, MI, MI
+	vpxor %xmm1, LO, LO
+	vpxor %xmm4, HI, HI
+	vpxor %xmm3, MI, MI
+.endm
+
+/*
+ * Performs the same computation as schoolbook1_iteration, except we expect the
+ * arguments to already be loaded into xmm0 and xmm1 and we set the result
+ * registers LO, MI, and HI directly rather than XOR'ing into them.
+ */
+.macro schoolbook1_noload
+	vpclmulqdq $0x01, %xmm0, %xmm1, MI
+	vpclmulqdq $0x10, %xmm0, %xmm1, %xmm2
+	vpclmulqdq $0x00, %xmm0, %xmm1, LO
+	vpclmulqdq $0x11, %xmm0, %xmm1, HI
+	vpxor %xmm2, MI, MI
+.endm
+
+/*
+ * Computes the 256-bit polynomial represented by LO, HI, MI. Stores
+ * the result in PL, PH.
+ *   [PH : PL] = [HI_1 : HI_0 + MI_1 : LO_1 + MI_0 : LO_0]
+ */
+.macro schoolbook2
+	vpslldq $8, MI, PL
+	vpsrldq $8, MI, PH
+	pxor LO, PL
+	pxor HI, PH
+.endm
+
+/*
+ * Computes the 128-bit reduction of PH : PL. Stores the result in dest.
+ *
+ * This macro computes p(x) mod g(x) where p(x) is in montgomery form and g(x) =
+ * x^128 + x^127 + x^126 + x^121 + 1.
+ *
+ * We have a 256-bit polynomial PH : PL = P_3 : P_2 : P_1 : P_0 that is the
+ * product of two 128-bit polynomials in Montgomery form.  We need to reduce it
+ * mod g(x).  Also, since polynomials in Montgomery form have an "extra" factor
+ * of x^128, this product has two extra factors of x^128.  To get it back into
+ * Montgomery form, we need to remove one of these factors by dividing by x^128.
+ *
+ * To accomplish both of these goals, we add multiples of g(x) that cancel out
+ * the low 128 bits P_1 : P_0, leaving just the high 128 bits. Since the low
+ * bits are zero, the polynomial division by x^128 can be done by right shifting.
+ *
+ * Since the only nonzero term in the low 64 bits of g(x) is the constant term,
+ * the multiple of g(x) needed to cancel out P_0 is P_0 * g(x).  The CPU can
+ * only do 64x64 bit multiplications, so split P_0 * g(x) into x^128 * P_0 +
+ * x^64 * g*(x) * P_0 + P_0, where g*(x) is bits 64-127 of g(x).  Adding this to
+ * the original polynomial gives P_3 : P_2 + P_0 + T_1 : P_1 + T_0 : 0, where T
+ * = T_1 : T_0 = g*(x) * P_0.  Thus, bits 0-63 got "folded" into bits 64-191.
+ *
+ * Repeating this same process on the next 64 bits "folds" bits 64-127 into bits
+ * 128-255, giving the answer in bits 128-255. This time, we need to cancel P_1
+ * + T_0 in bits 64-127. The multiple of g(x) required is (P_1 + T_0) * g(x) *
+ * x^64. Adding this to our previous computation gives P_3 + P_1 + T_0 + V_1 :
+ * P_2 + P_0 + T_1 + V_0 : 0 : 0, where V = V_1 : V_0 = g*(x) * (P_1 + T_0).
+ *
+ * So our final computation is:
+ *   T = T_1 : T_0 = g*(x) * P_0
+ *   V = V_1 : V_0 = g*(x) * (P_1 + T_0)
+ *   p(x) / x^{128} mod g(x) = P_3 + P_1 + T_0 + V_1 : P_2 + P_0 + T_1 + V_0
+ *
+ * The implementation below saves a XOR instruction by computing P_1 + T_0 : P_0
+ * + T_1 and XORing into dest, rather than separately XORing P_1 : P_0 and T_0 :
+ * T_1 into dest.  This allows us to reuse P_1 + T_0 when computing V.
+ */
+.macro montgomery_reduction dest
+	vpclmulqdq $0x00, PL, GSTAR, TMP_XMM	# TMP_XMM = T_1 : T_0 = P_0 * g*(x)
+	pshufd $0b01001110, TMP_XMM, TMP_XMM	# TMP_XMM = T_0 : T_1
+	pxor PL, TMP_XMM			# TMP_XMM = P_1 + T_0 : P_0 + T_1
+	pxor TMP_XMM, PH			# PH = P_3 + P_1 + T_0 : P_2 + P_0 + T_1
+	pclmulqdq $0x11, GSTAR, TMP_XMM		# TMP_XMM = V_1 : V_0 = V = [(P_1 + T_0) * g*(x)]
+	vpxor TMP_XMM, PH, \dest
+.endm
+
+/*
+ * Compute schoolbook multiplication for 8 blocks
+ * m_0h^8 + ... + m_7h^1
+ *
+ * If reduce is set, also computes the montgomery reduction of the
+ * previous full_stride call and XORs with the first message block.
+ * (m_0 + REDUCE(PL, PH))h^8 + ... + m_7h^1.
+ * I.e., the first multiplication uses m_0 + REDUCE(PL, PH) instead of m_0.
+ */
+.macro full_stride reduce
+	pxor LO, LO
+	pxor HI, HI
+	pxor MI, MI
+
+	schoolbook1_iteration 7 0
+	.if \reduce
+		vpclmulqdq $0x00, PL, GSTAR, TMP_XMM
+	.endif
+
+	schoolbook1_iteration 6 0
+	.if \reduce
+		pshufd $0b01001110, TMP_XMM, TMP_XMM
+	.endif
+
+	schoolbook1_iteration 5 0
+	.if \reduce
+		pxor PL, TMP_XMM
+	.endif
+
+	schoolbook1_iteration 4 0
+	.if \reduce
+		pxor TMP_XMM, PH
+	.endif
+
+	schoolbook1_iteration 3 0
+	.if \reduce
+		pclmulqdq $0x11, GSTAR, TMP_XMM
+	.endif
+
+	schoolbook1_iteration 2 0
+	.if \reduce
+		vpxor TMP_XMM, PH, SUM
+	.endif
+
+	schoolbook1_iteration 1 0
+
+	schoolbook1_iteration 0 1
+
+	addq $(8*16), MSG
+	schoolbook2
+.endm
+
+/*
+ * Process BLOCKS_LEFT blocks, where 0 < BLOCKS_LEFT < STRIDE_BLOCKS
+ */
+.macro partial_stride
+	mov BLOCKS_LEFT, TMP
+	shlq $4, TMP
+	addq $(16*STRIDE_BLOCKS), KEY_POWERS
+	subq TMP, KEY_POWERS
+
+	movups (MSG), %xmm0
+	pxor SUM, %xmm0
+	movups (KEY_POWERS), %xmm1
+	schoolbook1_noload
+	dec BLOCKS_LEFT
+	addq $16, MSG
+	addq $16, KEY_POWERS
+
+	test $4, BLOCKS_LEFT
+	jz .Lpartial4BlocksDone
+	schoolbook1 4
+	addq $(4*16), MSG
+	addq $(4*16), KEY_POWERS
+.Lpartial4BlocksDone:
+	test $2, BLOCKS_LEFT
+	jz .Lpartial2BlocksDone
+	schoolbook1 2
+	addq $(2*16), MSG
+	addq $(2*16), KEY_POWERS
+.Lpartial2BlocksDone:
+	test $1, BLOCKS_LEFT
+	jz .LpartialDone
+	schoolbook1 1
+.LpartialDone:
+	schoolbook2
+	montgomery_reduction SUM
+.endm
+
+/*
+ * Computes a = a * b * x^{-128} mod x^128 + x^127 + x^126 + x^121 + 1.
+ *
+ * void polyval_mul_pclmul_avx(struct polyval_elem *a,
+ *			       const struct polyval_elem *b);
+ */
+SYM_FUNC_START(polyval_mul_pclmul_avx)
+	FRAME_BEGIN
+	vmovdqa .Lgstar(%rip), GSTAR
+	movups (%rdi), %xmm0
+	movups (%rsi), %xmm1
+	schoolbook1_noload
+	schoolbook2
+	montgomery_reduction SUM
+	movups SUM, (%rdi)
+	FRAME_END
+	RET
+SYM_FUNC_END(polyval_mul_pclmul_avx)
+
+/*
+ * Perform polynomial evaluation as specified by POLYVAL.  This computes:
+ *	h^n * accumulator + h^n * m_0 + ... + h^1 * m_{n-1}
+ * where n=nblocks, h is the hash key, and m_i are the message blocks.
+ *
+ * rdi - pointer to the accumulator
+ * rsi - pointer to precomputed key powers h^8 ... h^1
+ * rdx - pointer to message blocks
+ * rcx - number of blocks to hash
+ *
+ * void polyval_blocks_pclmul_avx(struct polyval_elem *acc,
+ *				  const struct polyval_key *key,
+ *				  const u8 *data, size_t nblocks);
+ */
+SYM_FUNC_START(polyval_blocks_pclmul_avx)
+	FRAME_BEGIN
+	vmovdqa .Lgstar(%rip), GSTAR
+	movups (ACCUMULATOR), SUM
+	subq $STRIDE_BLOCKS, BLOCKS_LEFT
+	js .LstrideLoopExit
+	full_stride 0
+	subq $STRIDE_BLOCKS, BLOCKS_LEFT
+	js .LstrideLoopExitReduce
+.LstrideLoop:
+	full_stride 1
+	subq $STRIDE_BLOCKS, BLOCKS_LEFT
+	jns .LstrideLoop
+.LstrideLoopExitReduce:
+	montgomery_reduction SUM
+.LstrideLoopExit:
+	add $STRIDE_BLOCKS, BLOCKS_LEFT
+	jz .LskipPartial
+	partial_stride
+.LskipPartial:
+	movups SUM, (ACCUMULATOR)
+	FRAME_END
+	RET
+SYM_FUNC_END(polyval_blocks_pclmul_avx)
diff --git a/lib/crypto/x86/polyval.h b/lib/crypto/x86/polyval.h
new file mode 100644
index 000000000000..ef8797521420
--- /dev/null
+++ b/lib/crypto/x86/polyval.h
@@ -0,0 +1,83 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * POLYVAL library functions, x86_64 optimized
+ *
+ * Copyright 2025 Google LLC
+ */
+#include <asm/fpu/api.h>
+#include <linux/cpufeature.h>
+
+#define NUM_H_POWERS 8
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul_avx);
+
+asmlinkage void polyval_mul_pclmul_avx(struct polyval_elem *a,
+				       const struct polyval_elem *b);
+asmlinkage void polyval_blocks_pclmul_avx(struct polyval_elem *acc,
+					  const struct polyval_key *key,
+					  const u8 *data, size_t nblocks);
+
+static void polyval_preparekey_arch(struct polyval_key *key,
+				    const u8 raw_key[POLYVAL_BLOCK_SIZE])
+{
+	static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS);
+	memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE);
+	if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) {
+		kernel_fpu_begin();
+		for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
+			key->h_powers[i] = key->h_powers[i + 1];
+			polyval_mul_pclmul_avx(
+				&key->h_powers[i],
+				&key->h_powers[NUM_H_POWERS - 1]);
+		}
+		kernel_fpu_end();
+	} else {
+		for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
+			key->h_powers[i] = key->h_powers[i + 1];
+			polyval_mul_generic(&key->h_powers[i],
+					    &key->h_powers[NUM_H_POWERS - 1]);
+		}
+	}
+}
+
+static void polyval_mul_arch(struct polyval_elem *acc,
+			     const struct polyval_key *key)
+{
+	if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) {
+		kernel_fpu_begin();
+		polyval_mul_pclmul_avx(acc, &key->h_powers[NUM_H_POWERS - 1]);
+		kernel_fpu_end();
+	} else {
+		polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]);
+	}
+}
+
+static void polyval_blocks_arch(struct polyval_elem *acc,
+				const struct polyval_key *key,
+				const u8 *data, size_t nblocks)
+{
+	if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) {
+		do {
+			/* Allow rescheduling every 4 KiB. */
+			size_t n = min_t(size_t, nblocks,
+					 4096 / POLYVAL_BLOCK_SIZE);
+
+			kernel_fpu_begin();
+			polyval_blocks_pclmul_avx(acc, key, data, n);
+			kernel_fpu_end();
+			data += n * POLYVAL_BLOCK_SIZE;
+			nblocks -= n;
+		} while (nblocks);
+	} else {
+		polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1],
+				       data, nblocks);
+	}
+}
+
+#define polyval_mod_init_arch polyval_mod_init_arch
+static void polyval_mod_init_arch(void)
+{
+	if (boot_cpu_has(X86_FEATURE_PCLMULQDQ) &&
+	    boot_cpu_has(X86_FEATURE_AVX))
+		static_branch_enable(&have_pclmul_avx);
+}
diff --git a/lib/debugobjects.c b/lib/debugobjects.c
index 7f50c4480a4e..12f50de85b62 100644
--- a/lib/debugobjects.c
+++ b/lib/debugobjects.c
@@ -2,7 +2,7 @@
 /*
  * Generic infrastructure for lifetime debugging of objects.
  *
- * Copyright (C) 2008, Thomas Gleixner <tglx@linutronix.de>
+ * Copyright (C) 2008, Linutronix GmbH, Thomas Gleixner <tglx@kernel.org>
  */
 
 #define pr_fmt(fmt) "ODEBUG: " fmt
@@ -398,9 +398,26 @@ static void fill_pool(void)
 
 	atomic_inc(&cpus_allocating);
 	while (pool_should_refill(&pool_global)) {
+		gfp_t gfp = __GFP_HIGH | __GFP_NOWARN;
 		HLIST_HEAD(head);
 
-		if (!kmem_alloc_batch(&head, obj_cache, __GFP_HIGH | __GFP_NOWARN))
+		/*
+		 * Allow reclaim only in preemptible context and during
+		 * early boot. If not preemptible, the caller might hold
+		 * locks causing a deadlock in the allocator.
+		 *
+		 * If the reclaim flag is not set during early boot then
+		 * allocations, which happen before deferred page
+		 * initialization has completed, will fail.
+		 *
+		 * In preemptible context the flag is harmless and not a
+		 * performance issue as that's usually invoked from slow
+		 * path initialization context.
+		 */
+		if (preemptible() || system_state < SYSTEM_SCHEDULING)
+			gfp |= __GFP_KSWAPD_RECLAIM;
+
+		if (!kmem_alloc_batch(&head, obj_cache, gfp))
 			break;
 
 		guard(raw_spinlock_irqsave)(&pool_lock);
@@ -714,13 +731,13 @@ static void debug_objects_fill_pool(void)
 	 * raw_spinlock_t are basically the same type and this lock-type
 	 * inversion works just fine.
 	 */
-	if (!IS_ENABLED(CONFIG_PREEMPT_RT) || preemptible()) {
+	if (!IS_ENABLED(CONFIG_PREEMPT_RT) || preemptible() || system_state < SYSTEM_SCHEDULING) {
 		/*
 		 * Annotate away the spinlock_t inside raw_spinlock_t warning
-		 * by temporarily raising the wait-type to WAIT_SLEEP, matching
+		 * by temporarily raising the wait-type to LD_WAIT_CONFIG, matching
 		 * the preemptible() condition above.
 		 */
-		static DEFINE_WAIT_OVERRIDE_MAP(fill_pool_map, LD_WAIT_SLEEP);
+		static DEFINE_WAIT_OVERRIDE_MAP(fill_pool_map, LD_WAIT_CONFIG);
 		lock_map_acquire_try(&fill_pool_map);
 		fill_pool();
 		lock_map_release(&fill_pool_map);
diff --git a/lib/dec_and_lock.c b/lib/dec_and_lock.c
index 1dcca8f2e194..8c7c398fd770 100644
--- a/lib/dec_and_lock.c
+++ b/lib/dec_and_lock.c
@@ -18,7 +18,7 @@
  * because the spin-lock and the decrement must be
  * "atomic".
  */
-int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
+int atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
 {
 	/* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
 	if (atomic_add_unless(atomic, -1, 1))
@@ -32,7 +32,7 @@ int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
 	return 0;
 }
 
-EXPORT_SYMBOL(_atomic_dec_and_lock);
+EXPORT_SYMBOL(atomic_dec_and_lock);
 
 int _atomic_dec_and_lock_irqsave(atomic_t *atomic, spinlock_t *lock,
 				 unsigned long *flags)
@@ -50,7 +50,7 @@ int _atomic_dec_and_lock_irqsave(atomic_t *atomic, spinlock_t *lock,
 }
 EXPORT_SYMBOL(_atomic_dec_and_lock_irqsave);
 
-int _atomic_dec_and_raw_lock(atomic_t *atomic, raw_spinlock_t *lock)
+int atomic_dec_and_raw_lock(atomic_t *atomic, raw_spinlock_t *lock)
 {
 	/* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
 	if (atomic_add_unless(atomic, -1, 1))
@@ -63,7 +63,7 @@ int _atomic_dec_and_raw_lock(atomic_t *atomic, raw_spinlock_t *lock)
 	raw_spin_unlock(lock);
 	return 0;
 }
-EXPORT_SYMBOL(_atomic_dec_and_raw_lock);
+EXPORT_SYMBOL(atomic_dec_and_raw_lock);
 
 int _atomic_dec_and_raw_lock_irqsave(atomic_t *atomic, raw_spinlock_t *lock,
 				     unsigned long *flags)
diff --git a/lib/decompress_unxz.c b/lib/decompress_unxz.c
index 32138bb8ef77..05d5cb490a44 100644
--- a/lib/decompress_unxz.c
+++ b/lib/decompress_unxz.c
@@ -157,11 +157,11 @@
  * when XZ_DYNALLOC is used, but the pre-boot free() doesn't support it.
  * Workaround it here because the other decompressors don't need it.
  */
-#undef kmalloc
+#undef kmalloc_obj
 #undef kfree
 #undef vmalloc
 #undef vfree
-#define kmalloc(size, flags) malloc(size)
+#define kmalloc_obj(type) malloc(sizeof(type))
 #define kfree(ptr) free(ptr)
 #define vmalloc(size) malloc(size)
 #define vfree(ptr) do { if (ptr != NULL) free(ptr); } while (0)
diff --git a/lib/dhry_1.c b/lib/dhry_1.c
index ca6c87232c58..134cc1c746c2 100644
--- a/lib/dhry_1.c
+++ b/lib/dhry_1.c
@@ -139,11 +139,11 @@ int dhry(int n)
 
 	/* Initializations */
 
-	Next_Ptr_Glob = (Rec_Pointer)kzalloc(sizeof(Rec_Type), GFP_ATOMIC);
+	Next_Ptr_Glob = (Rec_Pointer) kzalloc_obj(Rec_Type, GFP_ATOMIC);
 	if (!Next_Ptr_Glob)
 		return -ENOMEM;
 
-	Ptr_Glob = (Rec_Pointer)kzalloc(sizeof(Rec_Type), GFP_ATOMIC);
+	Ptr_Glob = (Rec_Pointer) kzalloc_obj(Rec_Type, GFP_ATOMIC);
 	if (!Ptr_Glob) {
 		kfree(Next_Ptr_Glob);
 		return -ENOMEM;
diff --git a/lib/dim/net_dim.c b/lib/dim/net_dim.c
index d6aa09a979b3..d8d4f6553559 100644
--- a/lib/dim/net_dim.c
+++ b/lib/dim/net_dim.c
@@ -105,7 +105,7 @@ int net_dim_init_irq_moder(struct net_device *dev, u8 profile_flags,
 	struct dim_irq_moder *moder;
 	int len;
 
-	dev->irq_moder = kzalloc(sizeof(*dev->irq_moder), GFP_KERNEL);
+	dev->irq_moder = kzalloc_obj(*dev->irq_moder);
 	if (!dev->irq_moder)
 		return -ENOMEM;
 
diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c
index 5a007952f7f2..18a71a9108d3 100644
--- a/lib/dynamic_debug.c
+++ b/lib/dynamic_debug.c
@@ -95,6 +95,7 @@ static const struct { unsigned flag:8; char opt_char; } opt_array[] = {
 	{ _DPRINTK_FLAGS_INCL_SOURCENAME, 's' },
 	{ _DPRINTK_FLAGS_INCL_LINENO, 'l' },
 	{ _DPRINTK_FLAGS_INCL_TID, 't' },
+	{ _DPRINTK_FLAGS_INCL_STACK, 'd' },
 	{ _DPRINTK_FLAGS_NONE, '_' },
 };
 
@@ -1240,7 +1241,7 @@ static int ddebug_add_module(struct _ddebug_info *di, const char *modname)
 		return 0;
 	}
 
-	dt = kzalloc(sizeof(*dt), GFP_KERNEL);
+	dt = kzalloc_obj(*dt);
 	if (dt == NULL) {
 		pr_err("error adding module: %s\n", modname);
 		return -ENOMEM;
diff --git a/lib/error-inject.c b/lib/error-inject.c
index 887acd9a6ea6..f3d1b70be605 100644
--- a/lib/error-inject.c
+++ b/lib/error-inject.c
@@ -80,7 +80,7 @@ static void populate_error_injection_list(struct error_injection_entry *start,
 			continue;
 		}
 
-		ent = kmalloc(sizeof(*ent), GFP_KERNEL);
+		ent = kmalloc_obj(*ent);
 		if (!ent)
 			break;
 		ent->start_addr = entry;
diff --git a/lib/find_bit.c b/lib/find_bit.c
index d4b5a29e3e72..5a0066c26d9a 100644
--- a/lib/find_bit.c
+++ b/lib/find_bit.c
@@ -71,7 +71,7 @@ out:										\
 
 #define FIND_NTH_BIT(FETCH, size, num)						\
 ({										\
-	unsigned long sz = (size), nr = (num), idx, w, tmp;			\
+	unsigned long sz = (size), nr = (num), idx, w, tmp = 0;			\
 										\
 	for (idx = 0; (idx + 1) * BITS_PER_LONG <= sz; idx++) {			\
 		if (idx * BITS_PER_LONG + nr >= sz)				\
diff --git a/lib/flex_proportions.c b/lib/flex_proportions.c
index 84ecccddc771..012d5614efb9 100644
--- a/lib/flex_proportions.c
+++ b/lib/flex_proportions.c
@@ -64,13 +64,14 @@ void fprop_global_destroy(struct fprop_global *p)
 bool fprop_new_period(struct fprop_global *p, int periods)
 {
 	s64 events = percpu_counter_sum(&p->events);
+	unsigned long flags;
 
 	/*
 	 * Don't do anything if there are no events.
 	 */
 	if (events <= 1)
 		return false;
-	preempt_disable_nested();
+	local_irq_save(flags);
 	write_seqcount_begin(&p->sequence);
 	if (periods < 64)
 		events -= events >> periods;
@@ -78,7 +79,7 @@ bool fprop_new_period(struct fprop_global *p, int periods)
 	percpu_counter_add(&p->events, -events);
 	p->period += periods;
 	write_seqcount_end(&p->sequence);
-	preempt_enable_nested();
+	local_irq_restore(flags);
 
 	return true;
 }
diff --git a/lib/fonts/Kconfig b/lib/fonts/Kconfig
index ae59b5b4e225..7d03823e46dc 100644
--- a/lib/fonts/Kconfig
+++ b/lib/fonts/Kconfig
@@ -112,6 +112,17 @@ config FONT_SUN12x22
 	  big letters (like the letters used in the SPARC PROM). If the
 	  standard font is unreadable for you, say Y, otherwise say N.
 
+config FONT_TER10x18
+	bool "Terminus 10x18 font (not supported by all drivers)"
+	depends on FRAMEBUFFER_CONSOLE || DRM_PANIC
+	depends on !SPARC && FONTS || SPARC
+	help
+	  Terminus Font is a clean, fixed width bitmap font, designed
+	  for long (8 and more hours per day) work with computers.
+	  This is the high resolution version made for use with 13-16" laptops.
+	  It fits between the normal 8x16 font and Terminus 16x32.
+	  If other fonts are unreadable for you, say Y, otherwise say N.
+
 config FONT_TER16x32
 	bool "Terminus 16x32 font (not supported by all drivers)"
 	depends on FRAMEBUFFER_CONSOLE || DRM_PANIC
@@ -140,6 +151,7 @@ config FONT_AUTOSELECT
 	depends on !FONT_SUN8x16
 	depends on !FONT_SUN12x22
 	depends on !FONT_10x18
+	depends on !FONT_TER10x18
 	depends on !FONT_TER16x32
 	depends on !FONT_6x8
 	select FONT_8x16
diff --git a/lib/fonts/Makefile b/lib/fonts/Makefile
index e16f68492174..30a85a4292fa 100644
--- a/lib/fonts/Makefile
+++ b/lib/fonts/Makefile
@@ -14,6 +14,7 @@ font-objs-$(CONFIG_FONT_PEARL_8x8) += font_pearl_8x8.o
 font-objs-$(CONFIG_FONT_ACORN_8x8) += font_acorn_8x8.o
 font-objs-$(CONFIG_FONT_MINI_4x6)  += font_mini_4x6.o
 font-objs-$(CONFIG_FONT_6x10)      += font_6x10.o
+font-objs-$(CONFIG_FONT_TER10x18)  += font_ter10x18.o
 font-objs-$(CONFIG_FONT_TER16x32)  += font_ter16x32.o
 font-objs-$(CONFIG_FONT_6x8)       += font_6x8.o
 
diff --git a/lib/fonts/font_ter10x18.c b/lib/fonts/font_ter10x18.c
new file mode 100644
index 000000000000..80356e9d56c7
--- /dev/null
+++ b/lib/fonts/font_ter10x18.c
@@ -0,0 +1,5143 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/font.h>
+#include <linux/module.h>
+
+#define FONTDATAMAX 9216
+
+static const struct font_data fontdata_ter10x18 = {
+	{ 0, 0, FONTDATAMAX, 0 }, {
+	/* 0 0x00 '^@' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 1 0x01 '^A' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x80, 0x40, /* #--------# */
+	0x80, 0x40, /* #--------# */
+	0xb3, 0x40, /* #-##--##-# */
+	0xb3, 0x40, /* #-##--##-# */
+	0x80, 0x40, /* #--------# */
+	0x80, 0x40, /* #--------# */
+	0xbf, 0x40, /* #-######-# */
+	0x9e, 0x40, /* #--####--# */
+	0x80, 0x40, /* #--------# */
+	0x80, 0x40, /* #--------# */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 2 0x02 '^B' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xcc, 0xc0, /* ##--##--## */
+	0xcc, 0xc0, /* ##--##--## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xc0, 0xc0, /* ##------## */
+	0xe1, 0xc0, /* ###----### */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 3 0x03 '^C' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x73, 0x80, /* -###--###- */
+	0xf3, 0xc0, /* ####--#### */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x7f, 0x80, /* -########- */
+	0x3f, 0x00, /* --######-- */
+	0x1e, 0x00, /* ---####--- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 4 0x04 '^D' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x1e, 0x00, /* ---####--- */
+	0x3f, 0x00, /* --######-- */
+	0x7f, 0x80, /* -########- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x7f, 0x80, /* -########- */
+	0x3f, 0x00, /* --######-- */
+	0x1e, 0x00, /* ---####--- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 5 0x05 '^E' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1e, 0x00, /* ---####--- */
+	0x3f, 0x00, /* --######-- */
+	0x3f, 0x00, /* --######-- */
+	0x1e, 0x00, /* ---####--- */
+	0x0c, 0x00, /* ----##---- */
+	0x6d, 0x80, /* -##-##-##- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x6d, 0x80, /* -##-##-##- */
+	0x0c, 0x00, /* ----##---- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 6 0x06 '^F' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x1e, 0x00, /* ---####--- */
+	0x3f, 0x00, /* --######-- */
+	0x7f, 0x80, /* -########- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x6d, 0x80, /* -##-##-##- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 7 0x07 '^G' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x1e, 0x00, /* ---####--- */
+	0x1e, 0x00, /* ---####--- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 8 0x08 '^H' */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xf3, 0xc0, /* ####--#### */
+	0xe1, 0xc0, /* ###----### */
+	0xe1, 0xc0, /* ###----### */
+	0xf3, 0xc0, /* ####--#### */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+
+	/* 9 0x09 '^I' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1e, 0x00, /* ---####--- */
+	0x33, 0x00, /* --##--##-- */
+	0x21, 0x00, /* --#----#-- */
+	0x21, 0x00, /* --#----#-- */
+	0x33, 0x00, /* --##--##-- */
+	0x1e, 0x00, /* ---####--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 10 0x0a '^J' */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xe1, 0xc0, /* ###----### */
+	0xcc, 0xc0, /* ##--##--## */
+	0xde, 0xc0, /* ##-####-## */
+	0xde, 0xc0, /* ##-####-## */
+	0xcc, 0xc0, /* ##--##--## */
+	0xe1, 0xc0, /* ###----### */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+
+	/* 11 0x0b '^K' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0f, 0x80, /* ----#####- */
+	0x03, 0x80, /* ------###- */
+	0x06, 0x80, /* -----##-#- */
+	0x0c, 0x80, /* ----##--#- */
+	0x3e, 0x00, /* --#####--- */
+	0x63, 0x00, /* -##---##-- */
+	0x63, 0x00, /* -##---##-- */
+	0x63, 0x00, /* -##---##-- */
+	0x63, 0x00, /* -##---##-- */
+	0x63, 0x00, /* -##---##-- */
+	0x63, 0x00, /* -##---##-- */
+	0x3e, 0x00, /* --#####--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 12 0x0c '^L' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x7f, 0x80, /* -########- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 13 0x0d '^M' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x80, /* --#######- */
+	0x31, 0x80, /* --##---##- */
+	0x31, 0x80, /* --##---##- */
+	0x3f, 0x80, /* --#######- */
+	0x30, 0x00, /* --##------ */
+	0x30, 0x00, /* --##------ */
+	0x30, 0x00, /* --##------ */
+	0x30, 0x00, /* --##------ */
+	0x30, 0x00, /* --##------ */
+	0x30, 0x00, /* --##------ */
+	0xf0, 0x00, /* ####------ */
+	0xe0, 0x00, /* ###------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 14 0x0e '^N' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x7f, 0x80, /* -########- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x63, 0x80, /* -##---###- */
+	0xe3, 0x00, /* ###---##-- */
+	0xc0, 0x00, /* ##-------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 15 0x0f '^O' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0xcc, 0xc0, /* ##--##--## */
+	0x6d, 0x80, /* -##-##-##- */
+	0x3f, 0x00, /* --######-- */
+	0x1e, 0x00, /* ---####--- */
+	0xf3, 0xc0, /* ####--#### */
+	0x1e, 0x00, /* ---####--- */
+	0x3f, 0x00, /* --######-- */
+	0x6d, 0x80, /* -##-##-##- */
+	0xcc, 0xc0, /* ##--##--## */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 16 0x10 '^P' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0xc0, 0x00, /* ##-------- */
+	0xf0, 0x00, /* ####------ */
+	0xfc, 0x00, /* ######---- */
+	0xff, 0x00, /* ########-- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0x00, /* ########-- */
+	0xfc, 0x00, /* ######---- */
+	0xf0, 0x00, /* ####------ */
+	0xc0, 0x00, /* ##-------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 17 0x11 '^Q' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0xc0, /* --------## */
+	0x03, 0xc0, /* ------#### */
+	0x0f, 0xc0, /* ----###### */
+	0x3f, 0xc0, /* --######## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x3f, 0xc0, /* --######## */
+	0x0f, 0xc0, /* ----###### */
+	0x03, 0xc0, /* ------#### */
+	0x00, 0xc0, /* --------## */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 18 0x12 '^R' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x1e, 0x00, /* ---####--- */
+	0x3f, 0x00, /* --######-- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x3f, 0x00, /* --######-- */
+	0x1e, 0x00, /* ---####--- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 19 0x13 '^S' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 20 0x14 '^T' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x80, /* --#######- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x3d, 0x80, /* --####-##- */
+	0x0d, 0x80, /* ----##-##- */
+	0x0d, 0x80, /* ----##-##- */
+	0x0d, 0x80, /* ----##-##- */
+	0x0d, 0x80, /* ----##-##- */
+	0x0d, 0x80, /* ----##-##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 21 0x15 '^U' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1e, 0x00, /* ---####--- */
+	0x33, 0x00, /* --##--##-- */
+	0x30, 0x00, /* --##------ */
+	0x1c, 0x00, /* ---###---- */
+	0x36, 0x00, /* --##-##--- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x1b, 0x00, /* ---##-##-- */
+	0x0e, 0x00, /* ----###--- */
+	0x03, 0x00, /* ------##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x1e, 0x00, /* ---####--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 22 0x16 '^V' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0xff, 0x80, /* #########- */
+	0xff, 0x80, /* #########- */
+	0xff, 0x80, /* #########- */
+	0xff, 0x80, /* #########- */
+	0xff, 0x80, /* #########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 23 0x17 '^W' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x1e, 0x00, /* ---####--- */
+	0x3f, 0x00, /* --######-- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x3f, 0x00, /* --######-- */
+	0x1e, 0x00, /* ---####--- */
+	0x0c, 0x00, /* ----##---- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 24 0x18 '^X' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x1e, 0x00, /* ---####--- */
+	0x3f, 0x00, /* --######-- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 25 0x19 '^Y' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x3f, 0x00, /* --######-- */
+	0x1e, 0x00, /* ---####--- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 26 0x1a '^Z' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x04, 0x00, /* -----#---- */
+	0x06, 0x00, /* -----##--- */
+	0x03, 0x00, /* ------##-- */
+	0xff, 0x80, /* #########- */
+	0xff, 0x80, /* #########- */
+	0x03, 0x00, /* ------##-- */
+	0x06, 0x00, /* -----##--- */
+	0x04, 0x00, /* -----#---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 27 0x1b '^[' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x10, 0x00, /* ---#------ */
+	0x30, 0x00, /* --##------ */
+	0x60, 0x00, /* -##------- */
+	0xff, 0x80, /* #########- */
+	0xff, 0x80, /* #########- */
+	0x60, 0x00, /* -##------- */
+	0x30, 0x00, /* --##------ */
+	0x10, 0x00, /* ---#------ */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 28 0x1c '^\' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 29 0x1d '^]' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x12, 0x00, /* ---#--#--- */
+	0x33, 0x00, /* --##--##-- */
+	0x61, 0x80, /* -##----##- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x61, 0x80, /* -##----##- */
+	0x33, 0x00, /* --##--##-- */
+	0x12, 0x00, /* ---#--#--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 30 0x1e '^^' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x1e, 0x00, /* ---####--- */
+	0x1e, 0x00, /* ---####--- */
+	0x3f, 0x00, /* --######-- */
+	0x3f, 0x00, /* --######-- */
+	0x7f, 0x80, /* -########- */
+	0x7f, 0x80, /* -########- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 31 0x1f '^_' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x7f, 0x80, /* -########- */
+	0x7f, 0x80, /* -########- */
+	0x3f, 0x00, /* --######-- */
+	0x3f, 0x00, /* --######-- */
+	0x1e, 0x00, /* ---####--- */
+	0x1e, 0x00, /* ---####--- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 32 0x20 ' ' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 33 0x21 '!' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 34 0x22 '"' */
+	0x00, 0x00, /* ---------- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 35 0x23 '#' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x7f, 0x80, /* -########- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x7f, 0x80, /* -########- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 36 0x24 '$' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x3f, 0x00, /* --######-- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6c, 0x00, /* -##-##---- */
+	0x6c, 0x00, /* -##-##---- */
+	0x6c, 0x00, /* -##-##---- */
+	0x3f, 0x00, /* --######-- */
+	0x0d, 0x80, /* ----##-##- */
+	0x0d, 0x80, /* ----##-##- */
+	0x0d, 0x80, /* ----##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x3f, 0x00, /* --######-- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+
+	/* 37 0x25 '%' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x73, 0x00, /* -###--##-- */
+	0x53, 0x00, /* -#-#--##-- */
+	0x76, 0x00, /* -###-##--- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x30, 0x00, /* --##------ */
+	0x37, 0x00, /* --##-###-- */
+	0x65, 0x00, /* -##--#-#-- */
+	0x67, 0x00, /* -##--###-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 38 0x26 '&' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3c, 0x00, /* --####---- */
+	0x66, 0x00, /* -##--##--- */
+	0x66, 0x00, /* -##--##--- */
+	0x66, 0x00, /* -##--##--- */
+	0x3c, 0x00, /* --####---- */
+	0x39, 0x80, /* --###--##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0xc7, 0x00, /* ##---###-- */
+	0xc3, 0x00, /* ##----##-- */
+	0xc3, 0x00, /* ##----##-- */
+	0x67, 0x80, /* -##--####- */
+	0x3d, 0x80, /* --####-##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 39 0x27 ''' */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 40 0x28 '(' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x06, 0x00, /* -----##--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 41 0x29 ')' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x18, 0x00, /* ---##----- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x06, 0x00, /* -----##--- */
+	0x06, 0x00, /* -----##--- */
+	0x06, 0x00, /* -----##--- */
+	0x06, 0x00, /* -----##--- */
+	0x06, 0x00, /* -----##--- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x18, 0x00, /* ---##----- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 42 0x2a '*' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x63, 0x00, /* -##---##-- */
+	0x36, 0x00, /* --##-##--- */
+	0x1c, 0x00, /* ---###---- */
+	0xff, 0x80, /* #########- */
+	0x1c, 0x00, /* ---###---- */
+	0x36, 0x00, /* --##-##--- */
+	0x63, 0x00, /* -##---##-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 43 0x2b '+' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x7f, 0x80, /* -########- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 44 0x2c ',' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x18, 0x00, /* ---##----- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 45 0x2d '-' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 46 0x2e '.' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 47 0x2f '/' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x06, 0x00, /* -----##--- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x30, 0x00, /* --##------ */
+	0x30, 0x00, /* --##------ */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 48 0x30 '0' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x63, 0x80, /* -##---###- */
+	0x67, 0x80, /* -##--####- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x79, 0x80, /* -####--##- */
+	0x71, 0x80, /* -###---##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 49 0x31 '1' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x1c, 0x00, /* ---###---- */
+	0x3c, 0x00, /* --####---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 50 0x32 '2' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x01, 0x80, /* -------##- */
+	0x03, 0x00, /* ------##-- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x18, 0x00, /* ---##----- */
+	0x30, 0x00, /* --##------ */
+	0x60, 0x00, /* -##------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 51 0x33 '3' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x1f, 0x00, /* ---#####-- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 52 0x34 '4' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x01, 0x80, /* -------##- */
+	0x03, 0x80, /* ------###- */
+	0x07, 0x80, /* -----####- */
+	0x0d, 0x80, /* ----##-##- */
+	0x19, 0x80, /* ---##--##- */
+	0x31, 0x80, /* --##---##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x7f, 0x80, /* -########- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 53 0x35 '5' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x7f, 0x00, /* -#######-- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 54 0x36 '6' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1f, 0x00, /* ---#####-- */
+	0x30, 0x00, /* --##------ */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x7f, 0x00, /* -#######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 55 0x37 '7' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x01, 0x80, /* -------##- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x06, 0x00, /* -----##--- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 56 0x38 '8' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 57 0x39 '9' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x80, /* --#######- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x03, 0x00, /* ------##-- */
+	0x3e, 0x00, /* --#####--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 58 0x3a ':' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 59 0x3b ';' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x18, 0x00, /* ---##----- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 60 0x3c '<' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x03, 0x00, /* ------##-- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x18, 0x00, /* ---##----- */
+	0x30, 0x00, /* --##------ */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x30, 0x00, /* --##------ */
+	0x18, 0x00, /* ---##----- */
+	0x0c, 0x00, /* ----##---- */
+	0x06, 0x00, /* -----##--- */
+	0x03, 0x00, /* ------##-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 61 0x3d '=' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 62 0x3e '>' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x60, 0x00, /* -##------- */
+	0x30, 0x00, /* --##------ */
+	0x18, 0x00, /* ---##----- */
+	0x0c, 0x00, /* ----##---- */
+	0x06, 0x00, /* -----##--- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x18, 0x00, /* ---##----- */
+	0x30, 0x00, /* --##------ */
+	0x60, 0x00, /* -##------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 63 0x3f '?' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1e, 0x00, /* ---####--- */
+	0x33, 0x00, /* --##--##-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x01, 0x80, /* -------##- */
+	0x03, 0x00, /* ------##-- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 64 0x40 '@' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x00, /* -#######-- */
+	0xc1, 0x80, /* ##-----##- */
+	0xc1, 0x80, /* ##-----##- */
+	0xcf, 0x80, /* ##--#####- */
+	0xd9, 0x80, /* ##-##--##- */
+	0xd9, 0x80, /* ##-##--##- */
+	0xd9, 0x80, /* ##-##--##- */
+	0xd9, 0x80, /* ##-##--##- */
+	0xcf, 0x80, /* ##--#####- */
+	0xc0, 0x00, /* ##-------- */
+	0xc0, 0x00, /* ##-------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 65 0x41 'A' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x7f, 0x80, /* -########- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 66 0x42 'B' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x00, /* -#######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x7f, 0x00, /* -#######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x7f, 0x00, /* -#######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 67 0x43 'C' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 68 0x44 'D' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7e, 0x00, /* -######--- */
+	0x63, 0x00, /* -##---##-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x63, 0x00, /* -##---##-- */
+	0x7e, 0x00, /* -######--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 69 0x45 'E' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x7e, 0x00, /* -######--- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 70 0x46 'F' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x7e, 0x00, /* -######--- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 71 0x47 'G' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x67, 0x80, /* -##--####- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 72 0x48 'H' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x7f, 0x80, /* -########- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 73 0x49 'I' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1e, 0x00, /* ---####--- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x1e, 0x00, /* ---####--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 74 0x4a 'J' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x07, 0x80, /* -----####- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x63, 0x00, /* -##---##-- */
+	0x63, 0x00, /* -##---##-- */
+	0x63, 0x00, /* -##---##-- */
+	0x3e, 0x00, /* --#####--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 75 0x4b 'K' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x63, 0x00, /* -##---##-- */
+	0x66, 0x00, /* -##--##--- */
+	0x6c, 0x00, /* -##-##---- */
+	0x78, 0x00, /* -####----- */
+	0x78, 0x00, /* -####----- */
+	0x6c, 0x00, /* -##-##---- */
+	0x66, 0x00, /* -##--##--- */
+	0x63, 0x00, /* -##---##-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 76 0x4c 'L' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 77 0x4d 'M' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x80, 0x80, /* #-------#- */
+	0xc1, 0x80, /* ##-----##- */
+	0xe3, 0x80, /* ###---###- */
+	0xf7, 0x80, /* ####-####- */
+	0xdd, 0x80, /* ##-###-##- */
+	0xc9, 0x80, /* ##--#--##- */
+	0xc1, 0x80, /* ##-----##- */
+	0xc1, 0x80, /* ##-----##- */
+	0xc1, 0x80, /* ##-----##- */
+	0xc1, 0x80, /* ##-----##- */
+	0xc1, 0x80, /* ##-----##- */
+	0xc1, 0x80, /* ##-----##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 78 0x4e 'N' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x71, 0x80, /* -###---##- */
+	0x79, 0x80, /* -####--##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x67, 0x80, /* -##--####- */
+	0x63, 0x80, /* -##---###- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 79 0x4f 'O' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 80 0x50 'P' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x00, /* -#######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x7f, 0x00, /* -#######-- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 81 0x51 'Q' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x67, 0x80, /* -##--####- */
+	0x3f, 0x00, /* --######-- */
+	0x03, 0x00, /* ------##-- */
+	0x01, 0x80, /* -------##- */
+	0x00, 0x00, /* ---------- */
+
+	/* 82 0x52 'R' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x00, /* -#######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x7f, 0x00, /* -#######-- */
+	0x78, 0x00, /* -####----- */
+	0x6c, 0x00, /* -##-##---- */
+	0x66, 0x00, /* -##--##--- */
+	0x63, 0x00, /* -##---##-- */
+	0x61, 0x80, /* -##----##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 83 0x53 'S' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x3f, 0x00, /* --######-- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 84 0x54 'T' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 85 0x55 'U' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 86 0x56 'V' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x1e, 0x00, /* ---####--- */
+	0x1e, 0x00, /* ---####--- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 87 0x57 'W' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0xc1, 0x80, /* ##-----##- */
+	0xc1, 0x80, /* ##-----##- */
+	0xc1, 0x80, /* ##-----##- */
+	0xc1, 0x80, /* ##-----##- */
+	0xc1, 0x80, /* ##-----##- */
+	0xc1, 0x80, /* ##-----##- */
+	0xc9, 0x80, /* ##--#--##- */
+	0xdd, 0x80, /* ##-###-##- */
+	0xf7, 0x80, /* ####-####- */
+	0xe3, 0x80, /* ###---###- */
+	0xc1, 0x80, /* ##-----##- */
+	0x80, 0x80, /* #-------#- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 88 0x58 'X' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x1e, 0x00, /* ---####--- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x1e, 0x00, /* ---####--- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 89 0x59 'Y' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x1e, 0x00, /* ---####--- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 90 0x5a 'Z' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x03, 0x00, /* ------##-- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x18, 0x00, /* ---##----- */
+	0x30, 0x00, /* --##------ */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 91 0x5b '[' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1e, 0x00, /* ---####--- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x1e, 0x00, /* ---####--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 92 0x5c '\' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x30, 0x00, /* --##------ */
+	0x30, 0x00, /* --##------ */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x06, 0x00, /* -----##--- */
+	0x06, 0x00, /* -----##--- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 93 0x5d ']' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1e, 0x00, /* ---####--- */
+	0x06, 0x00, /* -----##--- */
+	0x06, 0x00, /* -----##--- */
+	0x06, 0x00, /* -----##--- */
+	0x06, 0x00, /* -----##--- */
+	0x06, 0x00, /* -----##--- */
+	0x06, 0x00, /* -----##--- */
+	0x06, 0x00, /* -----##--- */
+	0x06, 0x00, /* -----##--- */
+	0x06, 0x00, /* -----##--- */
+	0x06, 0x00, /* -----##--- */
+	0x1e, 0x00, /* ---####--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 94 0x5e '^' */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x1e, 0x00, /* ---####--- */
+	0x33, 0x00, /* --##--##-- */
+	0x61, 0x80, /* -##----##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 95 0x5f '_' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+
+	/* 96 0x60 '`' */
+	0x30, 0x00, /* --##------ */
+	0x18, 0x00, /* ---##----- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 97 0x61 'a' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x3f, 0x80, /* --#######- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x80, /* --#######- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 98 0x62 'b' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x7f, 0x00, /* -#######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x7f, 0x00, /* -#######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 99 0x63 'c' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 100 0x64 'd' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x3f, 0x80, /* --#######- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x80, /* --#######- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 101 0x65 'e' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x7f, 0x80, /* -########- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 102 0x66 'f' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x07, 0x80, /* -----####- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x3f, 0x00, /* --######-- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 103 0x67 'g' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x80, /* --#######- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x80, /* --#######- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x3f, 0x00, /* --######-- */
+
+	/* 104 0x68 'h' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x7f, 0x00, /* -#######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 105 0x69 'i' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x1c, 0x00, /* ---###---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x1e, 0x00, /* ---####--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 106 0x6a 'j' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x00, 0x00, /* ---------- */
+	0x07, 0x00, /* -----###-- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x1e, 0x00, /* ---####--- */
+
+	/* 107 0x6b 'k' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x61, 0x80, /* -##----##- */
+	0x63, 0x00, /* -##---##-- */
+	0x66, 0x00, /* -##--##--- */
+	0x6c, 0x00, /* -##-##---- */
+	0x78, 0x00, /* -####----- */
+	0x6c, 0x00, /* -##-##---- */
+	0x66, 0x00, /* -##--##--- */
+	0x63, 0x00, /* -##---##-- */
+	0x61, 0x80, /* -##----##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 108 0x6c 'l' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1c, 0x00, /* ---###---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x1e, 0x00, /* ---####--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 109 0x6d 'm' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x00, /* -#######-- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 110 0x6e 'n' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x00, /* -#######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 111 0x6f 'o' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 112 0x70 'p' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x00, /* -#######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x7f, 0x00, /* -#######-- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+
+	/* 113 0x71 'q' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x80, /* --#######- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x80, /* --#######- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+
+	/* 114 0x72 'r' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x6f, 0x80, /* -##-#####- */
+	0x78, 0x00, /* -####----- */
+	0x70, 0x00, /* -###------ */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 115 0x73 's' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x3f, 0x00, /* --######-- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 116 0x74 't' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x7e, 0x00, /* -######--- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x18, 0x00, /* ---##----- */
+	0x0f, 0x00, /* ----####-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 117 0x75 'u' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x80, /* --#######- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 118 0x76 'v' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x1e, 0x00, /* ---####--- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 119 0x77 'w' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 120 0x78 'x' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x33, 0x00, /* --##--##-- */
+	0x1e, 0x00, /* ---####--- */
+	0x0c, 0x00, /* ----##---- */
+	0x1e, 0x00, /* ---####--- */
+	0x33, 0x00, /* --##--##-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 121 0x79 'y' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x80, /* --#######- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x3f, 0x00, /* --######-- */
+
+	/* 122 0x7a 'z' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x01, 0x80, /* -------##- */
+	0x03, 0x00, /* ------##-- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x18, 0x00, /* ---##----- */
+	0x30, 0x00, /* --##------ */
+	0x60, 0x00, /* -##------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 123 0x7b '{' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x07, 0x00, /* -----###-- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x38, 0x00, /* --###----- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x07, 0x00, /* -----###-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 124 0x7c '|' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 125 0x7d '}' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x38, 0x00, /* --###----- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x07, 0x00, /* -----###-- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x38, 0x00, /* --###----- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 126 0x7e '~' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x39, 0x80, /* --###--##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x67, 0x00, /* -##--###-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 127 0x7f '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x08, 0x00, /* ----#----- */
+	0x1c, 0x00, /* ---###---- */
+	0x36, 0x00, /* --##-##--- */
+	0x63, 0x00, /* -##---##-- */
+	0xc1, 0x80, /* ##-----##- */
+	0xc1, 0x80, /* ##-----##- */
+	0xc1, 0x80, /* ##-----##- */
+	0xc1, 0x80, /* ##-----##- */
+	0xc1, 0x80, /* ##-----##- */
+	0xff, 0x80, /* #########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 128 0x80 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x18, 0x00, /* ---##----- */
+
+	/* 129 0x81 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x80, /* --#######- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 130 0x82 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x7f, 0x80, /* -########- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 131 0x83 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1e, 0x00, /* ---####--- */
+	0x33, 0x00, /* --##--##-- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x3f, 0x80, /* --#######- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x80, /* --#######- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 132 0x84 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x3f, 0x80, /* --#######- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x80, /* --#######- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 133 0x85 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x18, 0x00, /* ---##----- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x3f, 0x80, /* --#######- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x80, /* --#######- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 134 0x86 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1e, 0x00, /* ---####--- */
+	0x33, 0x00, /* --##--##-- */
+	0x1e, 0x00, /* ---####--- */
+	0x3f, 0x00, /* --######-- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x3f, 0x80, /* --#######- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x80, /* --#######- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 135 0x87 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x18, 0x00, /* ---##----- */
+
+	/* 136 0x88 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1e, 0x00, /* ---####--- */
+	0x33, 0x00, /* --##--##-- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x7f, 0x80, /* -########- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 137 0x89 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x7f, 0x80, /* -########- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 138 0x8a '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x18, 0x00, /* ---##----- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x7f, 0x80, /* -########- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 139 0x8b '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x00, 0x00, /* ---------- */
+	0x1c, 0x00, /* ---###---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x1e, 0x00, /* ---####--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 140 0x8c '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1e, 0x00, /* ---####--- */
+	0x33, 0x00, /* --##--##-- */
+	0x00, 0x00, /* ---------- */
+	0x1c, 0x00, /* ---###---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x1e, 0x00, /* ---####--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 141 0x8d '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x18, 0x00, /* ---##----- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x1c, 0x00, /* ---###---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x1e, 0x00, /* ---####--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 142 0x8e '' */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x7f, 0x80, /* -########- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 143 0x8f '' */
+	0x1e, 0x00, /* ---####--- */
+	0x33, 0x00, /* --##--##-- */
+	0x1e, 0x00, /* ---####--- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x7f, 0x80, /* -########- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 144 0x90 '' */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x7e, 0x00, /* -######--- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 145 0x91 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7b, 0x80, /* -####-###- */
+	0x0c, 0xc0, /* ----##--## */
+	0x0c, 0xc0, /* ----##--## */
+	0x7c, 0xc0, /* -#####--## */
+	0xcf, 0xc0, /* ##--###### */
+	0xcc, 0x00, /* ##--##---- */
+	0xcc, 0x00, /* ##--##---- */
+	0xcc, 0xc0, /* ##--##--## */
+	0x77, 0x80, /* -###-####- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 146 0x92 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0xc0, /* -######### */
+	0xc6, 0x00, /* ##---##--- */
+	0xc6, 0x00, /* ##---##--- */
+	0xc6, 0x00, /* ##---##--- */
+	0xc6, 0x00, /* ##---##--- */
+	0xff, 0xc0, /* ########## */
+	0xc6, 0x00, /* ##---##--- */
+	0xc6, 0x00, /* ##---##--- */
+	0xc6, 0x00, /* ##---##--- */
+	0xc6, 0x00, /* ##---##--- */
+	0xc6, 0x00, /* ##---##--- */
+	0xc7, 0xc0, /* ##---##### */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 147 0x93 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1e, 0x00, /* ---####--- */
+	0x33, 0x00, /* --##--##-- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 148 0x94 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 149 0x95 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x18, 0x00, /* ---##----- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 150 0x96 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1e, 0x00, /* ---####--- */
+	0x33, 0x00, /* --##--##-- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x80, /* --#######- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 151 0x97 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x18, 0x00, /* ---##----- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x80, /* --#######- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 152 0x98 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x80, /* --#######- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x3f, 0x00, /* --######-- */
+
+	/* 153 0x99 '' */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 154 0x9a '' */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 155 0x9b '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x3f, 0x00, /* --######-- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6c, 0x00, /* -##-##---- */
+	0x6c, 0x00, /* -##-##---- */
+	0x6c, 0x00, /* -##-##---- */
+	0x6c, 0x00, /* -##-##---- */
+	0x6c, 0x00, /* -##-##---- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x3f, 0x00, /* --######-- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+
+	/* 156 0x9c '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1e, 0x00, /* ---####--- */
+	0x33, 0x00, /* --##--##-- */
+	0x30, 0x00, /* --##------ */
+	0x30, 0x00, /* --##------ */
+	0x30, 0x00, /* --##------ */
+	0x7e, 0x00, /* -######--- */
+	0x30, 0x00, /* --##------ */
+	0x30, 0x00, /* --##------ */
+	0x30, 0x00, /* --##------ */
+	0x30, 0x00, /* --##------ */
+	0x31, 0x80, /* --##---##- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 157 0x9d '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x1e, 0x00, /* ---####--- */
+	0x1e, 0x00, /* ---####--- */
+	0x0c, 0x00, /* ----##---- */
+	0x3f, 0x00, /* --######-- */
+	0x0c, 0x00, /* ----##---- */
+	0x3f, 0x00, /* --######-- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 158 0x9e '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0xf8, 0x00, /* #####----- */
+	0xcc, 0x00, /* ##--##---- */
+	0xcc, 0x00, /* ##--##---- */
+	0xcc, 0x00, /* ##--##---- */
+	0xcc, 0x00, /* ##--##---- */
+	0xfb, 0x00, /* #####-##-- */
+	0xc3, 0x00, /* ##----##-- */
+	0xc7, 0x80, /* ##---####- */
+	0xc3, 0x00, /* ##----##-- */
+	0xc3, 0x00, /* ##----##-- */
+	0xc3, 0x00, /* ##----##-- */
+	0xc1, 0x80, /* ##-----##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 159 0x9f '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x07, 0x00, /* -----###-- */
+	0x0d, 0x80, /* ----##-##- */
+	0x0d, 0x80, /* ----##-##- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x3f, 0x00, /* --######-- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x6c, 0x00, /* -##-##---- */
+	0x6c, 0x00, /* -##-##---- */
+	0x38, 0x00, /* --###----- */
+
+	/* 160 0xa0 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x3f, 0x80, /* --#######- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x80, /* --#######- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 161 0xa1 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x1c, 0x00, /* ---###---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x1e, 0x00, /* ---####--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 162 0xa2 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 163 0xa3 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x80, /* --#######- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 164 0xa4 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3b, 0x80, /* --###-###- */
+	0x6e, 0x00, /* -##-###--- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x00, /* -#######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 165 0xa5 '' */
+	0x3b, 0x80, /* --###-###- */
+	0x6e, 0x00, /* -##-###--- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x71, 0x80, /* -###---##- */
+	0x79, 0x80, /* -####--##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x67, 0x80, /* -##--####- */
+	0x63, 0x80, /* -##---###- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 166 0xa6 '' */
+	0x00, 0x00, /* ---------- */
+	0x3e, 0x00, /* --#####--- */
+	0x03, 0x00, /* ------##-- */
+	0x3f, 0x00, /* --######-- */
+	0x63, 0x00, /* -##---##-- */
+	0x63, 0x00, /* -##---##-- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x00, /* -#######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 167 0xa7 '' */
+	0x00, 0x00, /* ---------- */
+	0x3e, 0x00, /* --#####--- */
+	0x63, 0x00, /* -##---##-- */
+	0x63, 0x00, /* -##---##-- */
+	0x63, 0x00, /* -##---##-- */
+	0x63, 0x00, /* -##---##-- */
+	0x3e, 0x00, /* --#####--- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x00, /* -#######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 168 0xa8 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x18, 0x00, /* ---##----- */
+	0x30, 0x00, /* --##------ */
+	0x60, 0x00, /* -##------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x33, 0x00, /* --##--##-- */
+	0x1e, 0x00, /* ---####--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 169 0xa9 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 170 0xaa '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 171 0xab '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x30, 0x00, /* --##------ */
+	0x70, 0x00, /* -###------ */
+	0x30, 0x80, /* --##----#- */
+	0x31, 0x80, /* --##---##- */
+	0x33, 0x00, /* --##--##-- */
+	0x36, 0x00, /* --##-##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x18, 0x00, /* ---##----- */
+	0x37, 0x00, /* --##-###-- */
+	0x6d, 0x80, /* -##-##-##- */
+	0xc1, 0x80, /* ##-----##- */
+	0x83, 0x00, /* #-----##-- */
+	0x06, 0x00, /* -----##--- */
+	0x0f, 0x80, /* ----#####- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 172 0xac '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x30, 0x00, /* --##------ */
+	0x70, 0x00, /* -###------ */
+	0x30, 0x80, /* --##----#- */
+	0x31, 0x80, /* --##---##- */
+	0x33, 0x00, /* --##--##-- */
+	0x36, 0x00, /* --##-##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x19, 0x80, /* ---##--##- */
+	0x33, 0x80, /* --##--###- */
+	0x67, 0x80, /* -##--####- */
+	0xcd, 0x80, /* ##--##-##- */
+	0x8f, 0x80, /* #---#####- */
+	0x01, 0x80, /* -------##- */
+	0x01, 0x80, /* -------##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 173 0xad '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 174 0xae '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0xc0, /* ----##--## */
+	0x19, 0x80, /* ---##--##- */
+	0x33, 0x00, /* --##--##-- */
+	0x66, 0x00, /* -##--##--- */
+	0xcc, 0x00, /* ##--##---- */
+	0x66, 0x00, /* -##--##--- */
+	0x33, 0x00, /* --##--##-- */
+	0x19, 0x80, /* ---##--##- */
+	0x0c, 0xc0, /* ----##--## */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 175 0xaf '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0xcc, 0x00, /* ##--##---- */
+	0x66, 0x00, /* -##--##--- */
+	0x33, 0x00, /* --##--##-- */
+	0x19, 0x80, /* ---##--##- */
+	0x0c, 0xc0, /* ----##--## */
+	0x19, 0x80, /* ---##--##- */
+	0x33, 0x00, /* --##--##-- */
+	0x66, 0x00, /* -##--##--- */
+	0xcc, 0x00, /* ##--##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 176 0xb0 '' */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0x00, 0x00, /* ---------- */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0x00, 0x00, /* ---------- */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0x00, 0x00, /* ---------- */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0x00, 0x00, /* ---------- */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0x00, 0x00, /* ---------- */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0x00, 0x00, /* ---------- */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0x00, 0x00, /* ---------- */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0x00, 0x00, /* ---------- */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0x00, 0x00, /* ---------- */
+
+	/* 177 0xb1 '' */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0x55, 0x40, /* -#-#-#-#-# */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0x55, 0x40, /* -#-#-#-#-# */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0x55, 0x40, /* -#-#-#-#-# */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0x55, 0x40, /* -#-#-#-#-# */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0x55, 0x40, /* -#-#-#-#-# */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0x55, 0x40, /* -#-#-#-#-# */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0x55, 0x40, /* -#-#-#-#-# */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0x55, 0x40, /* -#-#-#-#-# */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0x55, 0x40, /* -#-#-#-#-# */
+
+	/* 178 0xb2 '' */
+	0xff, 0xc0, /* ########## */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0xff, 0xc0, /* ########## */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0xff, 0xc0, /* ########## */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0xff, 0xc0, /* ########## */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0xff, 0xc0, /* ########## */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0xff, 0xc0, /* ########## */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0xff, 0xc0, /* ########## */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0xff, 0xc0, /* ########## */
+	0xaa, 0x80, /* #-#-#-#-#- */
+	0xff, 0xc0, /* ########## */
+	0xaa, 0x80, /* #-#-#-#-#- */
+
+	/* 179 0xb3 '' */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+
+	/* 180 0xb4 '' */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0xfc, 0x00, /* ######---- */
+	0xfc, 0x00, /* ######---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+
+	/* 181 0xb5 '' */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0xfc, 0x00, /* ######---- */
+	0xfc, 0x00, /* ######---- */
+	0x0c, 0x00, /* ----##---- */
+	0xfc, 0x00, /* ######---- */
+	0xfc, 0x00, /* ######---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+
+	/* 182 0xb6 '' */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0xf6, 0x00, /* ####-##--- */
+	0xf6, 0x00, /* ####-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+
+	/* 183 0xb7 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0xfe, 0x00, /* #######--- */
+	0xfe, 0x00, /* #######--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+
+	/* 184 0xb8 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0xfc, 0x00, /* ######---- */
+	0xfc, 0x00, /* ######---- */
+	0x0c, 0x00, /* ----##---- */
+	0xfc, 0x00, /* ######---- */
+	0xfc, 0x00, /* ######---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+
+	/* 185 0xb9 '' */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0xf6, 0x00, /* ####-##--- */
+	0xf6, 0x00, /* ####-##--- */
+	0x06, 0x00, /* -----##--- */
+	0xf6, 0x00, /* ####-##--- */
+	0xf6, 0x00, /* ####-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+
+	/* 186 0xba '' */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+
+	/* 187 0xbb '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0xfe, 0x00, /* #######--- */
+	0xfe, 0x00, /* #######--- */
+	0x06, 0x00, /* -----##--- */
+	0xf6, 0x00, /* ####-##--- */
+	0xf6, 0x00, /* ####-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+
+	/* 188 0xbc '' */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0xf6, 0x00, /* ####-##--- */
+	0xf6, 0x00, /* ####-##--- */
+	0x06, 0x00, /* -----##--- */
+	0xfe, 0x00, /* #######--- */
+	0xfe, 0x00, /* #######--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 189 0xbd '' */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0xfe, 0x00, /* #######--- */
+	0xfe, 0x00, /* #######--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 190 0xbe '' */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0xfc, 0x00, /* ######---- */
+	0xfc, 0x00, /* ######---- */
+	0x0c, 0x00, /* ----##---- */
+	0xfc, 0x00, /* ######---- */
+	0xfc, 0x00, /* ######---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 191 0xbf '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0xfc, 0x00, /* ######---- */
+	0xfc, 0x00, /* ######---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+
+	/* 192 0xc0 '' */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0f, 0xc0, /* ----###### */
+	0x0f, 0xc0, /* ----###### */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 193 0xc1 '' */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 194 0xc2 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+
+	/* 195 0xc3 '' */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0f, 0xc0, /* ----###### */
+	0x0f, 0xc0, /* ----###### */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+
+	/* 196 0xc4 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 197 0xc5 '' */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+
+	/* 198 0xc6 '' */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0f, 0xc0, /* ----###### */
+	0x0f, 0xc0, /* ----###### */
+	0x0c, 0x00, /* ----##---- */
+	0x0f, 0xc0, /* ----###### */
+	0x0f, 0xc0, /* ----###### */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+
+	/* 199 0xc7 '' */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x37, 0xc0, /* --##-##### */
+	0x37, 0xc0, /* --##-##### */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+
+	/* 200 0xc8 '' */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x37, 0xc0, /* --##-##### */
+	0x37, 0xc0, /* --##-##### */
+	0x30, 0x00, /* --##------ */
+	0x3f, 0xc0, /* --######## */
+	0x3f, 0xc0, /* --######## */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 201 0xc9 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0xc0, /* --######## */
+	0x3f, 0xc0, /* --######## */
+	0x30, 0x00, /* --##------ */
+	0x37, 0xc0, /* --##-##### */
+	0x37, 0xc0, /* --##-##### */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+
+	/* 202 0xca '' */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0xf7, 0xc0, /* ####-##### */
+	0xf7, 0xc0, /* ####-##### */
+	0x00, 0x00, /* ---------- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 203 0xcb '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x00, 0x00, /* ---------- */
+	0xf7, 0xc0, /* ####-##### */
+	0xf7, 0xc0, /* ####-##### */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+
+	/* 204 0xcc '' */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x37, 0xc0, /* --##-##### */
+	0x37, 0xc0, /* --##-##### */
+	0x30, 0x00, /* --##------ */
+	0x37, 0xc0, /* --##-##### */
+	0x37, 0xc0, /* --##-##### */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+
+	/* 205 0xcd '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x00, 0x00, /* ---------- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 206 0xce '' */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0xf7, 0xc0, /* ####-##### */
+	0xf7, 0xc0, /* ####-##### */
+	0x00, 0x00, /* ---------- */
+	0xf7, 0xc0, /* ####-##### */
+	0xf7, 0xc0, /* ####-##### */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+
+	/* 207 0xcf '' */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x00, 0x00, /* ---------- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 208 0xd0 '' */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 209 0xd1 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x00, 0x00, /* ---------- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+
+	/* 210 0xd2 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+
+	/* 211 0xd3 '' */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x3f, 0xc0, /* --######## */
+	0x3f, 0xc0, /* --######## */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 212 0xd4 '' */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0f, 0xc0, /* ----###### */
+	0x0f, 0xc0, /* ----###### */
+	0x0c, 0x00, /* ----##---- */
+	0x0f, 0xc0, /* ----###### */
+	0x0f, 0xc0, /* ----###### */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 213 0xd5 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0f, 0xc0, /* ----###### */
+	0x0f, 0xc0, /* ----###### */
+	0x0c, 0x00, /* ----##---- */
+	0x0f, 0xc0, /* ----###### */
+	0x0f, 0xc0, /* ----###### */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+
+	/* 214 0xd6 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0xc0, /* --######## */
+	0x3f, 0xc0, /* --######## */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+
+	/* 215 0xd7 '' */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+	0x36, 0x00, /* --##-##--- */
+
+	/* 216 0xd8 '' */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x0c, 0x00, /* ----##---- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+
+	/* 217 0xd9 '' */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0xfc, 0x00, /* ######---- */
+	0xfc, 0x00, /* ######---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 218 0xda '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0f, 0xc0, /* ----###### */
+	0x0f, 0xc0, /* ----###### */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+
+	/* 219 0xdb '' */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+
+	/* 220 0xdc '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+
+	/* 221 0xdd '' */
+	0xf8, 0x00, /* #####----- */
+	0xf8, 0x00, /* #####----- */
+	0xf8, 0x00, /* #####----- */
+	0xf8, 0x00, /* #####----- */
+	0xf8, 0x00, /* #####----- */
+	0xf8, 0x00, /* #####----- */
+	0xf8, 0x00, /* #####----- */
+	0xf8, 0x00, /* #####----- */
+	0xf8, 0x00, /* #####----- */
+	0xf8, 0x00, /* #####----- */
+	0xf8, 0x00, /* #####----- */
+	0xf8, 0x00, /* #####----- */
+	0xf8, 0x00, /* #####----- */
+	0xf8, 0x00, /* #####----- */
+	0xf8, 0x00, /* #####----- */
+	0xf8, 0x00, /* #####----- */
+	0xf8, 0x00, /* #####----- */
+	0xf8, 0x00, /* #####----- */
+
+	/* 222 0xde '' */
+	0x07, 0xc0, /* -----##### */
+	0x07, 0xc0, /* -----##### */
+	0x07, 0xc0, /* -----##### */
+	0x07, 0xc0, /* -----##### */
+	0x07, 0xc0, /* -----##### */
+	0x07, 0xc0, /* -----##### */
+	0x07, 0xc0, /* -----##### */
+	0x07, 0xc0, /* -----##### */
+	0x07, 0xc0, /* -----##### */
+	0x07, 0xc0, /* -----##### */
+	0x07, 0xc0, /* -----##### */
+	0x07, 0xc0, /* -----##### */
+	0x07, 0xc0, /* -----##### */
+	0x07, 0xc0, /* -----##### */
+	0x07, 0xc0, /* -----##### */
+	0x07, 0xc0, /* -----##### */
+	0x07, 0xc0, /* -----##### */
+	0x07, 0xc0, /* -----##### */
+
+	/* 223 0xdf '' */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0xff, 0xc0, /* ########## */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 224 0xe0 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7d, 0x80, /* -#####-##- */
+	0xc7, 0x00, /* ##---###-- */
+	0xc6, 0x00, /* ##---##--- */
+	0xc6, 0x00, /* ##---##--- */
+	0xc6, 0x00, /* ##---##--- */
+	0xc6, 0x00, /* ##---##--- */
+	0xc6, 0x00, /* ##---##--- */
+	0xc7, 0x00, /* ##---###-- */
+	0x7d, 0x80, /* -#####-##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 225 0xe1 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3e, 0x00, /* --#####--- */
+	0x63, 0x00, /* -##---##-- */
+	0x63, 0x00, /* -##---##-- */
+	0x63, 0x00, /* -##---##-- */
+	0x62, 0x00, /* -##---#--- */
+	0x7f, 0x00, /* -#######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x7f, 0x00, /* -#######-- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+
+	/* 226 0xe2 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 227 0xe3 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 228 0xe4 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x60, 0x00, /* -##------- */
+	0x30, 0x00, /* --##------ */
+	0x18, 0x00, /* ---##----- */
+	0x0c, 0x00, /* ----##---- */
+	0x06, 0x00, /* -----##--- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x18, 0x00, /* ---##----- */
+	0x30, 0x00, /* --##------ */
+	0x60, 0x00, /* -##------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 229 0xe5 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1f, 0xc0, /* ---####### */
+	0x33, 0x00, /* --##--##-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x33, 0x00, /* --##--##-- */
+	0x1e, 0x00, /* ---####--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 230 0xe6 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x63, 0x80, /* -##---###- */
+	0x7d, 0x80, /* -#####-##- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+
+	/* 231 0xe7 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x07, 0x00, /* -----###-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 232 0xe8 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x3f, 0x00, /* --######-- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x3f, 0x00, /* --######-- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 233 0xe9 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 234 0xea '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x73, 0x80, /* -###--###- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 235 0xeb '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x18, 0x00, /* ---##----- */
+	0x0c, 0x00, /* ----##---- */
+	0x1e, 0x00, /* ---####--- */
+	0x33, 0x00, /* --##--##-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x33, 0x00, /* --##--##-- */
+	0x1e, 0x00, /* ---####--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 236 0xec '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 237 0xed '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x01, 0x80, /* -------##- */
+	0x03, 0x00, /* ------##-- */
+	0x3f, 0x00, /* --######-- */
+	0x67, 0x80, /* -##--####- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x79, 0x80, /* -####--##- */
+	0x3f, 0x00, /* --######-- */
+	0x30, 0x00, /* --##------ */
+	0x60, 0x00, /* -##------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 238 0xee '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1f, 0x80, /* ---######- */
+	0x30, 0x00, /* --##------ */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x7f, 0x80, /* -########- */
+	0x60, 0x00, /* -##------- */
+	0x60, 0x00, /* -##------- */
+	0x30, 0x00, /* --##------ */
+	0x1f, 0x80, /* ---######- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 239 0xef '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1e, 0x00, /* ---####--- */
+	0x33, 0x00, /* --##--##-- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x61, 0x80, /* -##----##- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 240 0xf0 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 241 0xf1 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x7f, 0x80, /* -########- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 242 0xf2 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x30, 0x00, /* --##------ */
+	0x18, 0x00, /* ---##----- */
+	0x0c, 0x00, /* ----##---- */
+	0x06, 0x00, /* -----##--- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x18, 0x00, /* ---##----- */
+	0x30, 0x00, /* --##------ */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 243 0xf3 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x03, 0x00, /* ------##-- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x18, 0x00, /* ---##----- */
+	0x30, 0x00, /* --##------ */
+	0x30, 0x00, /* --##------ */
+	0x18, 0x00, /* ---##----- */
+	0x0c, 0x00, /* ----##---- */
+	0x06, 0x00, /* -----##--- */
+	0x03, 0x00, /* ------##-- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 244 0xf4 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x07, 0x00, /* -----###-- */
+	0x0d, 0x80, /* ----##-##- */
+	0x0d, 0x80, /* ----##-##- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+
+	/* 245 0xf5 '' */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x6c, 0x00, /* -##-##---- */
+	0x6c, 0x00, /* -##-##---- */
+	0x38, 0x00, /* --###----- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 246 0xf6 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x7f, 0x80, /* -########- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 247 0xf7 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x39, 0x80, /* --###--##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x67, 0x00, /* -##--###-- */
+	0x00, 0x00, /* ---------- */
+	0x39, 0x80, /* --###--##- */
+	0x6d, 0x80, /* -##-##-##- */
+	0x67, 0x00, /* -##--###-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 248 0xf8 '' */
+	0x00, 0x00, /* ---------- */
+	0x1e, 0x00, /* ---####--- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x1e, 0x00, /* ---####--- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 249 0xf9 '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x1c, 0x00, /* ---###---- */
+	0x1c, 0x00, /* ---###---- */
+	0x1c, 0x00, /* ---###---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 250 0xfa '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x0c, 0x00, /* ----##---- */
+	0x0c, 0x00, /* ----##---- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 251 0xfb '' */
+	0x00, 0x00, /* ---------- */
+	0x03, 0x80, /* ------###- */
+	0x03, 0x80, /* ------###- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x03, 0x00, /* ------##-- */
+	0x63, 0x00, /* -##---##-- */
+	0x63, 0x00, /* -##---##-- */
+	0x63, 0x00, /* -##---##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x1b, 0x00, /* ---##-##-- */
+	0x0f, 0x00, /* ----####-- */
+	0x07, 0x00, /* -----###-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 252 0xfc '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3e, 0x00, /* --#####--- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 253 0xfd '' */
+	0x00, 0x00, /* ---------- */
+	0x1e, 0x00, /* ---####--- */
+	0x33, 0x00, /* --##--##-- */
+	0x33, 0x00, /* --##--##-- */
+	0x06, 0x00, /* -----##--- */
+	0x0c, 0x00, /* ----##---- */
+	0x18, 0x00, /* ---##----- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 254 0xfe '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x3f, 0x00, /* --######-- */
+	0x3f, 0x00, /* --######-- */
+	0x3f, 0x00, /* --######-- */
+	0x3f, 0x00, /* --######-- */
+	0x3f, 0x00, /* --######-- */
+	0x3f, 0x00, /* --######-- */
+	0x3f, 0x00, /* --######-- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+
+	/* 255 0xff '' */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+	0x00, 0x00, /* ---------- */
+} };
+
+
+const struct font_desc font_ter_10x18 = {
+	.idx	= TER10x18_IDX,
+	.name	= "TER10x18",
+	.width	= 10,
+	.height = 18,
+	.charcount = 256,
+	.data	= fontdata_ter10x18.data,
+#ifdef __sparc__
+	.pref	= 5,
+#else
+	.pref	= -1,
+#endif
+};
diff --git a/lib/fonts/fonts.c b/lib/fonts/fonts.c
index 47e34950b665..a7f118b30171 100644
--- a/lib/fonts/fonts.c
+++ b/lib/fonts/fonts.c
@@ -54,6 +54,9 @@ static const struct font_desc *fonts[] = {
 #ifdef CONFIG_FONT_6x10
 	&font_6x10,
 #endif
+#ifdef CONFIG_FONT_TER10x18
+	&font_ter_10x18,
+#endif
 #ifdef CONFIG_FONT_TER16x32
 	&font_ter_16x32,
 #endif
diff --git a/lib/globtest.c b/lib/globtest.c
deleted file mode 100644
index d8e97d43b905..000000000000
--- a/lib/globtest.c
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Extracted fronm glob.c
- */
-
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/glob.h>
-#include <linux/printk.h>
-
-/* Boot with "glob.verbose=1" to show successful tests, too */
-static bool verbose = false;
-module_param(verbose, bool, 0);
-
-struct glob_test {
-	char const *pat, *str;
-	bool expected;
-};
-
-static bool __pure __init test(char const *pat, char const *str, bool expected)
-{
-	bool match = glob_match(pat, str);
-	bool success = match == expected;
-
-	/* Can't get string literals into a particular section, so... */
-	static char const msg_error[] __initconst =
-		KERN_ERR "glob: \"%s\" vs. \"%s\": %s *** ERROR ***\n";
-	static char const msg_ok[] __initconst =
-		KERN_DEBUG "glob: \"%s\" vs. \"%s\": %s OK\n";
-	static char const mismatch[] __initconst = "mismatch";
-	char const *message;
-
-	if (!success)
-		message = msg_error;
-	else if (verbose)
-		message = msg_ok;
-	else
-		return success;
-
-	printk(message, pat, str, mismatch + 3*match);
-	return success;
-}
-
-/*
- * The tests are all jammed together in one array to make it simpler
- * to place that array in the .init.rodata section.  The obvious
- * "array of structures containing char *" has no way to force the
- * pointed-to strings to be in a particular section.
- *
- * Anyway, a test consists of:
- * 1. Expected glob_match result: '1' or '0'.
- * 2. Pattern to match: null-terminated string
- * 3. String to match against: null-terminated string
- *
- * The list of tests is terminated with a final '\0' instead of
- * a glob_match result character.
- */
-static char const glob_tests[] __initconst =
-	/* Some basic tests */
-	"1" "a\0" "a\0"
-	"0" "a\0" "b\0"
-	"0" "a\0" "aa\0"
-	"0" "a\0" "\0"
-	"1" "\0" "\0"
-	"0" "\0" "a\0"
-	/* Simple character class tests */
-	"1" "[a]\0" "a\0"
-	"0" "[a]\0" "b\0"
-	"0" "[!a]\0" "a\0"
-	"1" "[!a]\0" "b\0"
-	"1" "[ab]\0" "a\0"
-	"1" "[ab]\0" "b\0"
-	"0" "[ab]\0" "c\0"
-	"1" "[!ab]\0" "c\0"
-	"1" "[a-c]\0" "b\0"
-	"0" "[a-c]\0" "d\0"
-	/* Corner cases in character class parsing */
-	"1" "[a-c-e-g]\0" "-\0"
-	"0" "[a-c-e-g]\0" "d\0"
-	"1" "[a-c-e-g]\0" "f\0"
-	"1" "[]a-ceg-ik[]\0" "a\0"
-	"1" "[]a-ceg-ik[]\0" "]\0"
-	"1" "[]a-ceg-ik[]\0" "[\0"
-	"1" "[]a-ceg-ik[]\0" "h\0"
-	"0" "[]a-ceg-ik[]\0" "f\0"
-	"0" "[!]a-ceg-ik[]\0" "h\0"
-	"0" "[!]a-ceg-ik[]\0" "]\0"
-	"1" "[!]a-ceg-ik[]\0" "f\0"
-	/* Simple wild cards */
-	"1" "?\0" "a\0"
-	"0" "?\0" "aa\0"
-	"0" "??\0" "a\0"
-	"1" "?x?\0" "axb\0"
-	"0" "?x?\0" "abx\0"
-	"0" "?x?\0" "xab\0"
-	/* Asterisk wild cards (backtracking) */
-	"0" "*??\0" "a\0"
-	"1" "*??\0" "ab\0"
-	"1" "*??\0" "abc\0"
-	"1" "*??\0" "abcd\0"
-	"0" "??*\0" "a\0"
-	"1" "??*\0" "ab\0"
-	"1" "??*\0" "abc\0"
-	"1" "??*\0" "abcd\0"
-	"0" "?*?\0" "a\0"
-	"1" "?*?\0" "ab\0"
-	"1" "?*?\0" "abc\0"
-	"1" "?*?\0" "abcd\0"
-	"1" "*b\0" "b\0"
-	"1" "*b\0" "ab\0"
-	"0" "*b\0" "ba\0"
-	"1" "*b\0" "bb\0"
-	"1" "*b\0" "abb\0"
-	"1" "*b\0" "bab\0"
-	"1" "*bc\0" "abbc\0"
-	"1" "*bc\0" "bc\0"
-	"1" "*bc\0" "bbc\0"
-	"1" "*bc\0" "bcbc\0"
-	/* Multiple asterisks (complex backtracking) */
-	"1" "*ac*\0" "abacadaeafag\0"
-	"1" "*ac*ae*ag*\0" "abacadaeafag\0"
-	"1" "*a*b*[bc]*[ef]*g*\0" "abacadaeafag\0"
-	"0" "*a*b*[ef]*[cd]*g*\0" "abacadaeafag\0"
-	"1" "*abcd*\0" "abcabcabcabcdefg\0"
-	"1" "*ab*cd*\0" "abcabcabcabcdefg\0"
-	"1" "*abcd*abcdef*\0" "abcabcdabcdeabcdefg\0"
-	"0" "*abcd*\0" "abcabcabcabcefg\0"
-	"0" "*ab*cd*\0" "abcabcabcabcefg\0";
-
-static int __init glob_init(void)
-{
-	unsigned successes = 0;
-	unsigned n = 0;
-	char const *p = glob_tests;
-	static char const message[] __initconst =
-		KERN_INFO "glob: %u self-tests passed, %u failed\n";
-
-	/*
-	 * Tests are jammed together in a string.  The first byte is '1'
-	 * or '0' to indicate the expected outcome, or '\0' to indicate the
-	 * end of the tests.  Then come two null-terminated strings: the
-	 * pattern and the string to match it against.
-	 */
-	while (*p) {
-		bool expected = *p++ & 1;
-		char const *pat = p;
-
-		p += strlen(p) + 1;
-		successes += test(pat, p, expected);
-		p += strlen(p) + 1;
-		n++;
-	}
-
-	n -= successes;
-	printk(message, successes, n);
-
-	/* What's the errno for "kernel bug detected"?  Guess... */
-	return n ? -ECANCELED : 0;
-}
-
-/* We need a dummy exit function to allow unload */
-static void __exit glob_fini(void) { }
-
-module_init(glob_init);
-module_exit(glob_fini);
-
-MODULE_DESCRIPTION("glob(7) matching tests");
-MODULE_LICENSE("Dual MIT/GPL");
diff --git a/lib/group_cpus.c b/lib/group_cpus.c
index 6d08ac05f371..e6e18d7a49bb 100644
--- a/lib/group_cpus.c
+++ b/lib/group_cpus.c
@@ -47,7 +47,7 @@ static cpumask_var_t *alloc_node_to_cpumask(void)
 	cpumask_var_t *masks;
 	int node;
 
-	masks = kcalloc(nr_node_ids, sizeof(cpumask_var_t), GFP_KERNEL);
+	masks = kzalloc_objs(cpumask_var_t, nr_node_ids);
 	if (!masks)
 		return NULL;
 
@@ -114,48 +114,15 @@ static int ncpus_cmp_func(const void *l, const void *r)
 	return ln->ncpus - rn->ncpus;
 }
 
-/*
- * Allocate group number for each node, so that for each node:
- *
- * 1) the allocated number is >= 1
- *
- * 2) the allocated number is <= active CPU number of this node
- *
- * The actual allocated total groups may be less than @numgrps when
- * active total CPU number is less than @numgrps.
- *
- * Active CPUs means the CPUs in '@cpu_mask AND @node_to_cpumask[]'
- * for each node.
- */
-static void alloc_nodes_groups(unsigned int numgrps,
-			       cpumask_var_t *node_to_cpumask,
-			       const struct cpumask *cpu_mask,
-			       const nodemask_t nodemsk,
-			       struct cpumask *nmsk,
-			       struct node_groups *node_groups)
+static void alloc_groups_to_nodes(unsigned int numgrps,
+				  unsigned int numcpus,
+				  struct node_groups *node_groups,
+				  unsigned int num_nodes)
 {
-	unsigned n, remaining_ncpus = 0;
-
-	for (n = 0; n < nr_node_ids; n++) {
-		node_groups[n].id = n;
-		node_groups[n].ncpus = UINT_MAX;
-	}
-
-	for_each_node_mask(n, nodemsk) {
-		unsigned ncpus;
-
-		cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]);
-		ncpus = cpumask_weight(nmsk);
-
-		if (!ncpus)
-			continue;
-		remaining_ncpus += ncpus;
-		node_groups[n].ncpus = ncpus;
-	}
+	unsigned int n, remaining_ncpus = numcpus;
+	unsigned int  ngroups, ncpus;
 
-	numgrps = min_t(unsigned, remaining_ncpus, numgrps);
-
-	sort(node_groups, nr_node_ids, sizeof(node_groups[0]),
+	sort(node_groups, num_nodes, sizeof(node_groups[0]),
 	     ncpus_cmp_func, NULL);
 
 	/*
@@ -226,9 +193,8 @@ static void alloc_nodes_groups(unsigned int numgrps,
 	 * finally for each node X: grps(X) <= ncpu(X).
 	 *
 	 */
-	for (n = 0; n < nr_node_ids; n++) {
-		unsigned ngroups, ncpus;
 
+	for (n = 0; n < num_nodes; n++) {
 		if (node_groups[n].ncpus == UINT_MAX)
 			continue;
 
@@ -246,12 +212,201 @@ static void alloc_nodes_groups(unsigned int numgrps,
 	}
 }
 
+/*
+ * Allocate group number for each node, so that for each node:
+ *
+ * 1) the allocated number is >= 1
+ *
+ * 2) the allocated number is <= active CPU number of this node
+ *
+ * The actual allocated total groups may be less than @numgrps when
+ * active total CPU number is less than @numgrps.
+ *
+ * Active CPUs means the CPUs in '@cpu_mask AND @node_to_cpumask[]'
+ * for each node.
+ */
+static void alloc_nodes_groups(unsigned int numgrps,
+			       cpumask_var_t *node_to_cpumask,
+			       const struct cpumask *cpu_mask,
+			       const nodemask_t nodemsk,
+			       struct cpumask *nmsk,
+			       struct node_groups *node_groups)
+{
+	unsigned int n, numcpus = 0;
+
+	for (n = 0; n < nr_node_ids; n++) {
+		node_groups[n].id = n;
+		node_groups[n].ncpus = UINT_MAX;
+	}
+
+	for_each_node_mask(n, nodemsk) {
+		unsigned int ncpus;
+
+		cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]);
+		ncpus = cpumask_weight(nmsk);
+
+		if (!ncpus)
+			continue;
+		numcpus += ncpus;
+		node_groups[n].ncpus = ncpus;
+	}
+
+	numgrps = min_t(unsigned int, numcpus, numgrps);
+	alloc_groups_to_nodes(numgrps, numcpus, node_groups, nr_node_ids);
+}
+
+static void assign_cpus_to_groups(unsigned int ncpus,
+				  struct cpumask *nmsk,
+				  struct node_groups *nv,
+				  struct cpumask *masks,
+				  unsigned int *curgrp,
+				  unsigned int last_grp)
+{
+	unsigned int v, cpus_per_grp, extra_grps;
+	/* Account for rounding errors */
+	extra_grps = ncpus - nv->ngroups * (ncpus / nv->ngroups);
+
+	/* Spread allocated groups on CPUs of the current node */
+	for (v = 0; v < nv->ngroups; v++, *curgrp += 1) {
+		cpus_per_grp = ncpus / nv->ngroups;
+
+		/* Account for extra groups to compensate rounding errors */
+		if (extra_grps) {
+			cpus_per_grp++;
+			--extra_grps;
+		}
+
+		/*
+		 * wrapping has to be considered given 'startgrp'
+		 * may start anywhere
+		 */
+		if (*curgrp >= last_grp)
+			*curgrp = 0;
+		grp_spread_init_one(&masks[*curgrp], nmsk, cpus_per_grp);
+	}
+}
+
+static int alloc_cluster_groups(unsigned int ncpus,
+				unsigned int ngroups,
+				struct cpumask *node_cpumask,
+				cpumask_var_t msk,
+				const struct cpumask ***clusters_ptr,
+				struct node_groups **cluster_groups_ptr)
+{
+	unsigned int ncluster = 0;
+	unsigned int cpu, nc, n;
+	const struct cpumask *cluster_mask;
+	const struct cpumask **clusters;
+	struct node_groups *cluster_groups;
+
+	cpumask_copy(msk, node_cpumask);
+
+	/* Probe how many clusters in this node. */
+	while (1) {
+		cpu = cpumask_first(msk);
+		if (cpu >= nr_cpu_ids)
+			break;
+
+		cluster_mask = topology_cluster_cpumask(cpu);
+		if (!cpumask_weight(cluster_mask))
+			goto no_cluster;
+		/* Clean out CPUs on the same cluster. */
+		cpumask_andnot(msk, msk, cluster_mask);
+		ncluster++;
+	}
+
+	/* If ngroups < ncluster, cross cluster is inevitable, skip. */
+	if (ncluster == 0 || ncluster > ngroups)
+		goto no_cluster;
+
+	/* Allocate memory based on cluster number. */
+	clusters = kzalloc_objs(*clusters, ncluster);
+	if (!clusters)
+		goto no_cluster;
+	cluster_groups = kzalloc_objs(struct node_groups, ncluster);
+	if (!cluster_groups)
+		goto fail_cluster_groups;
+
+	/* Filling cluster info for later process. */
+	cpumask_copy(msk, node_cpumask);
+	for (n = 0; n < ncluster; n++) {
+		cpu = cpumask_first(msk);
+		cluster_mask = topology_cluster_cpumask(cpu);
+		nc = cpumask_weight_and(cluster_mask, node_cpumask);
+		clusters[n] = cluster_mask;
+		cluster_groups[n].id = n;
+		cluster_groups[n].ncpus = nc;
+		cpumask_andnot(msk, msk, cluster_mask);
+	}
+
+	alloc_groups_to_nodes(ngroups, ncpus, cluster_groups, ncluster);
+
+	*clusters_ptr = clusters;
+	*cluster_groups_ptr = cluster_groups;
+	return ncluster;
+
+ fail_cluster_groups:
+	kfree(clusters);
+ no_cluster:
+	return 0;
+}
+
+/*
+ * Try group CPUs evenly for cluster locality within a NUMA node.
+ *
+ * Return: true if success, false otherwise.
+ */
+static bool __try_group_cluster_cpus(unsigned int ncpus,
+				     unsigned int ngroups,
+				     struct cpumask *node_cpumask,
+				     struct cpumask *masks,
+				     unsigned int *curgrp,
+				     unsigned int last_grp)
+{
+	struct node_groups *cluster_groups;
+	const struct cpumask **clusters;
+	unsigned int ncluster;
+	bool ret = false;
+	cpumask_var_t nmsk;
+	unsigned int i, nc;
+
+	if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL))
+		goto fail_nmsk_alloc;
+
+	ncluster = alloc_cluster_groups(ncpus, ngroups, node_cpumask, nmsk,
+					&clusters, &cluster_groups);
+
+	if (ncluster == 0)
+		goto fail_no_clusters;
+
+	for (i = 0; i < ncluster; i++) {
+		struct node_groups *nv = &cluster_groups[i];
+
+		/* Get the cpus on this cluster. */
+		cpumask_and(nmsk, node_cpumask, clusters[nv->id]);
+		nc = cpumask_weight(nmsk);
+		if (!nc)
+			continue;
+		WARN_ON_ONCE(nv->ngroups > nc);
+
+		assign_cpus_to_groups(nc, nmsk, nv, masks, curgrp, last_grp);
+	}
+
+	ret = true;
+	kfree(cluster_groups);
+	kfree(clusters);
+ fail_no_clusters:
+	free_cpumask_var(nmsk);
+ fail_nmsk_alloc:
+	return ret;
+}
+
 static int __group_cpus_evenly(unsigned int startgrp, unsigned int numgrps,
 			       cpumask_var_t *node_to_cpumask,
 			       const struct cpumask *cpu_mask,
 			       struct cpumask *nmsk, struct cpumask *masks)
 {
-	unsigned int i, n, nodes, cpus_per_grp, extra_grps, done = 0;
+	unsigned int i, n, nodes, done = 0;
 	unsigned int last_grp = numgrps;
 	unsigned int curgrp = startgrp;
 	nodemask_t nodemsk = NODE_MASK_NONE;
@@ -277,9 +432,7 @@ static int __group_cpus_evenly(unsigned int startgrp, unsigned int numgrps,
 		return numgrps;
 	}
 
-	node_groups = kcalloc(nr_node_ids,
-			       sizeof(struct node_groups),
-			       GFP_KERNEL);
+	node_groups = kzalloc_objs(struct node_groups, nr_node_ids);
 	if (!node_groups)
 		return -ENOMEM;
 
@@ -287,7 +440,7 @@ static int __group_cpus_evenly(unsigned int startgrp, unsigned int numgrps,
 	alloc_nodes_groups(numgrps, node_to_cpumask, cpu_mask,
 			   nodemsk, nmsk, node_groups);
 	for (i = 0; i < nr_node_ids; i++) {
-		unsigned int ncpus, v;
+		unsigned int ncpus;
 		struct node_groups *nv = &node_groups[i];
 
 		if (nv->ngroups == UINT_MAX)
@@ -301,28 +454,14 @@ static int __group_cpus_evenly(unsigned int startgrp, unsigned int numgrps,
 
 		WARN_ON_ONCE(nv->ngroups > ncpus);
 
-		/* Account for rounding errors */
-		extra_grps = ncpus - nv->ngroups * (ncpus / nv->ngroups);
-
-		/* Spread allocated groups on CPUs of the current node */
-		for (v = 0; v < nv->ngroups; v++, curgrp++) {
-			cpus_per_grp = ncpus / nv->ngroups;
-
-			/* Account for extra groups to compensate rounding errors */
-			if (extra_grps) {
-				cpus_per_grp++;
-				--extra_grps;
-			}
-
-			/*
-			 * wrapping has to be considered given 'startgrp'
-			 * may start anywhere
-			 */
-			if (curgrp >= last_grp)
-				curgrp = 0;
-			grp_spread_init_one(&masks[curgrp], nmsk,
-						cpus_per_grp);
+		if (__try_group_cluster_cpus(ncpus, nv->ngroups, nmsk,
+					     masks, &curgrp, last_grp)) {
+			done += nv->ngroups;
+			continue;
 		}
+
+		assign_cpus_to_groups(ncpus, nmsk, nv, masks, &curgrp,
+				      last_grp);
 		done += nv->ngroups;
 	}
 	kfree(node_groups);
@@ -367,7 +506,7 @@ struct cpumask *group_cpus_evenly(unsigned int numgrps, unsigned int *nummasks)
 	if (!node_to_cpumask)
 		goto fail_npresmsk;
 
-	masks = kcalloc(numgrps, sizeof(*masks), GFP_KERNEL);
+	masks = kzalloc_objs(*masks, numgrps);
 	if (!masks)
 		goto fail_node_to_cpumask;
 
@@ -433,7 +572,7 @@ struct cpumask *group_cpus_evenly(unsigned int numgrps, unsigned int *nummasks)
 	if (numgrps == 0)
 		return NULL;
 
-	masks = kcalloc(numgrps, sizeof(*masks), GFP_KERNEL);
+	masks = kzalloc_objs(*masks, numgrps);
 	if (!masks)
 		return NULL;
 
diff --git a/lib/hexdump.c b/lib/hexdump.c
index c3db7c3a7643..2e5cd8c24769 100644
--- a/lib/hexdump.c
+++ b/lib/hexdump.c
@@ -6,6 +6,7 @@
 #include <linux/types.h>
 #include <linux/ctype.h>
 #include <linux/errno.h>
+#include <linux/hex.h>
 #include <linux/kernel.h>
 #include <linux/minmax.h>
 #include <linux/export.h>
diff --git a/lib/hweight.c b/lib/hweight.c
index c94586b62551..0dfcafc3fd39 100644
--- a/lib/hweight.c
+++ b/lib/hweight.c
@@ -4,8 +4,8 @@
 #include <asm/types.h>
 
 /**
- * hweightN - returns the hamming weight of a N-bit word
- * @x: the word to weigh
+ * DOC: __sw_hweightN - returns the hamming weight of a N-bit word
+ * @w: the word to weigh
  *
  * The Hamming Weight of a number is the total number of bits set in it.
  */
diff --git a/lib/idr.c b/lib/idr.c
index e2adc457abb4..69bee5369670 100644
--- a/lib/idr.c
+++ b/lib/idr.c
@@ -40,6 +40,8 @@ int idr_alloc_u32(struct idr *idr, void *ptr, u32 *nextid,
 
 	if (WARN_ON_ONCE(!(idr->idr_rt.xa_flags & ROOT_IS_IDR)))
 		idr->idr_rt.xa_flags |= IDR_RT_MARKER;
+	if (max < base)
+		return -ENOSPC;
 
 	id = (id < base) ? 0 : id - base;
 	radix_tree_iter_init(&iter, id);
@@ -415,7 +417,7 @@ next:
 		}
 		bitmap = alloc;
 		if (!bitmap)
-			bitmap = kzalloc(sizeof(*bitmap), GFP_NOWAIT);
+			bitmap = kzalloc_obj(*bitmap, GFP_NOWAIT);
 		if (!bitmap)
 			goto alloc;
 		bitmap->bitmap[0] = tmp;
@@ -442,7 +444,7 @@ next:
 		} else {
 			bitmap = alloc;
 			if (!bitmap)
-				bitmap = kzalloc(sizeof(*bitmap), GFP_NOWAIT);
+				bitmap = kzalloc_obj(*bitmap, GFP_NOWAIT);
 			if (!bitmap)
 				goto alloc;
 			__set_bit(bit, bitmap->bitmap);
@@ -463,7 +465,7 @@ out:
 	return xas.xa_index * IDA_BITMAP_BITS + bit;
 alloc:
 	xas_unlock_irqrestore(&xas, flags);
-	alloc = kzalloc(sizeof(*bitmap), gfp);
+	alloc = kzalloc_obj(*bitmap, gfp);
 	if (!alloc)
 		return -ENOMEM;
 	xas_set(&xas, min / IDA_BITMAP_BITS);
diff --git a/lib/interval_tree.c b/lib/interval_tree.c
index 324766e9bf63..9ceb084b6b4e 100644
--- a/lib/interval_tree.c
+++ b/lib/interval_tree.c
@@ -13,6 +13,7 @@ INTERVAL_TREE_DEFINE(struct interval_tree_node, rb,
 
 EXPORT_SYMBOL_GPL(interval_tree_insert);
 EXPORT_SYMBOL_GPL(interval_tree_remove);
+EXPORT_SYMBOL_GPL(interval_tree_subtree_search);
 EXPORT_SYMBOL_GPL(interval_tree_iter_first);
 EXPORT_SYMBOL_GPL(interval_tree_iter_next);
 
diff --git a/lib/interval_tree_test.c b/lib/interval_tree_test.c
index 5fd62656f42e..16200feacbf3 100644
--- a/lib/interval_tree_test.c
+++ b/lib/interval_tree_test.c
@@ -311,8 +311,7 @@ static inline int span_iteration_check(void) {return 0; }
 
 static int interval_tree_test_init(void)
 {
-	nodes = kmalloc_array(nnodes, sizeof(struct interval_tree_node),
-			      GFP_KERNEL);
+	nodes = kmalloc_objs(struct interval_tree_node, nnodes);
 	if (!nodes)
 		return -ENOMEM;
 
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 2fe66a6b8789..0a63c7fba313 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -49,12 +49,24 @@ size_t copy_from_user_iter(void __user *iter_from, size_t progress,
 
 	if (should_fail_usercopy())
 		return len;
-	if (access_ok(iter_from, len)) {
-		to += progress;
-		instrument_copy_from_user_before(to, iter_from, len);
-		res = raw_copy_from_user(to, iter_from, len);
-		instrument_copy_from_user_after(to, iter_from, len, res);
+	if (can_do_masked_user_access()) {
+		iter_from = mask_user_address(iter_from);
+	} else {
+		if (!access_ok(iter_from, len))
+			return res;
+
+		/*
+		 * Ensure that bad access_ok() speculation will not
+		 * lead to nasty side effects *after* the copy is
+		 * finished:
+		 */
+		barrier_nospec();
 	}
+	to += progress;
+	instrument_copy_from_user_before(to, iter_from, len);
+	res = raw_copy_from_user(to, iter_from, len);
+	instrument_copy_from_user_after(to, iter_from, len, res);
+
 	return res;
 }
 
@@ -891,7 +903,7 @@ static int want_pages_array(struct page ***res, size_t size,
 		count = maxpages;
 	WARN_ON(!count);	// caller should've prevented that
 	if (!*res) {
-		*res = kvmalloc_array(count, sizeof(struct page *), GFP_KERNEL);
+		*res = kvmalloc_objs(struct page *, count);
 		if (!*res)
 			return 0;
 	}
@@ -1306,7 +1318,7 @@ struct iovec *iovec_from_user(const struct iovec __user *uvec,
 	if (nr_segs > UIO_MAXIOV)
 		return ERR_PTR(-EINVAL);
 	if (nr_segs > fast_segs) {
-		iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);
+		iov = kmalloc_objs(struct iovec, nr_segs);
 		if (!iov)
 			return ERR_PTR(-ENOMEM);
 	}
@@ -1833,3 +1845,101 @@ ssize_t iov_iter_extract_pages(struct iov_iter *i,
 	return -EFAULT;
 }
 EXPORT_SYMBOL_GPL(iov_iter_extract_pages);
+
+static unsigned int get_contig_folio_len(struct page **pages,
+		unsigned int *num_pages, size_t left, size_t offset)
+{
+	struct folio *folio = page_folio(pages[0]);
+	size_t contig_sz = min_t(size_t, PAGE_SIZE - offset, left);
+	unsigned int max_pages, i;
+	size_t folio_offset, len;
+
+	folio_offset = PAGE_SIZE * folio_page_idx(folio, pages[0]) + offset;
+	len = min(folio_size(folio) - folio_offset, left);
+
+	/*
+	 * We might COW a single page in the middle of a large folio, so we have
+	 * to check that all pages belong to the same folio.
+	 */
+	left -= contig_sz;
+	max_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE);
+	for (i = 1; i < max_pages; i++) {
+		size_t next = min_t(size_t, PAGE_SIZE, left);
+
+		if (page_folio(pages[i]) != folio ||
+		    pages[i] != pages[i - 1] + 1)
+			break;
+		contig_sz += next;
+		left -= next;
+	}
+
+	*num_pages = i;
+	return contig_sz;
+}
+
+#define PAGE_PTRS_PER_BVEC     (sizeof(struct bio_vec) / sizeof(struct page *))
+
+/**
+ * iov_iter_extract_bvecs - Extract bvecs from an iterator
+ * @iter:	the iterator to extract from
+ * @bv:		bvec return array
+ * @max_size:	maximum size to extract from @iter
+ * @nr_vecs:	number of vectors in @bv (on in and output)
+ * @max_vecs:	maximum vectors in @bv, including those filled before calling
+ * @extraction_flags: flags to qualify request
+ *
+ * Like iov_iter_extract_pages(), but returns physically contiguous ranges
+ * contained in a single folio as a single bvec instead of multiple entries.
+ *
+ * Returns the number of bytes extracted when successful, or a negative errno.
+ * If @nr_vecs was non-zero on entry, the number of successfully extracted bytes
+ * can be 0.
+ */
+ssize_t iov_iter_extract_bvecs(struct iov_iter *iter, struct bio_vec *bv,
+		size_t max_size, unsigned short *nr_vecs,
+		unsigned short max_vecs, iov_iter_extraction_t extraction_flags)
+{
+	unsigned short entries_left = max_vecs - *nr_vecs;
+	unsigned short nr_pages, i = 0;
+	size_t left, offset, len;
+	struct page **pages;
+	ssize_t size;
+
+	/*
+	 * Move page array up in the allocated memory for the bio vecs as far as
+	 * possible so that we can start filling biovecs from the beginning
+	 * without overwriting the temporary page array.
+	 */
+	BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2);
+	pages = (struct page **)(bv + *nr_vecs) +
+		entries_left * (PAGE_PTRS_PER_BVEC - 1);
+
+	size = iov_iter_extract_pages(iter, &pages, max_size, entries_left,
+			extraction_flags, &offset);
+	if (unlikely(size <= 0))
+		return size ? size : -EFAULT;
+
+	nr_pages = DIV_ROUND_UP(offset + size, PAGE_SIZE);
+	for (left = size; left > 0; left -= len) {
+		unsigned int nr_to_add;
+
+		if (*nr_vecs > 0 &&
+		    !zone_device_pages_have_same_pgmap(bv[*nr_vecs - 1].bv_page,
+				pages[i]))
+			break;
+
+		len = get_contig_folio_len(&pages[i], &nr_to_add, left, offset);
+		bvec_set_page(&bv[*nr_vecs], pages[i], len, offset);
+		i += nr_to_add;
+		(*nr_vecs)++;
+		offset = 0;
+	}
+
+	iov_iter_revert(iter, left);
+	if (iov_iter_extract_will_pin(iter)) {
+		while (i < nr_pages)
+			unpin_user_page(pages[i++]);
+	}
+	return size - left;
+}
+EXPORT_SYMBOL_GPL(iov_iter_extract_bvecs);
diff --git a/lib/kfifo.c b/lib/kfifo.c
index a8b2eed90599..2633f9cc336c 100644
--- a/lib/kfifo.c
+++ b/lib/kfifo.c
@@ -22,8 +22,8 @@ static inline unsigned int kfifo_unused(struct __kfifo *fifo)
 	return (fifo->mask + 1) - (fifo->in - fifo->out);
 }
 
-int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
-		size_t esize, gfp_t gfp_mask)
+int __kfifo_alloc_node(struct __kfifo *fifo, unsigned int size,
+		size_t esize, gfp_t gfp_mask, int node)
 {
 	/*
 	 * round up to the next power of 2, since our 'let the indices
@@ -41,7 +41,7 @@ int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
 		return -EINVAL;
 	}
 
-	fifo->data = kmalloc_array(esize, size, gfp_mask);
+	fifo->data = kmalloc_array_node(size, esize, gfp_mask, node);
 
 	if (!fifo->data) {
 		fifo->mask = 0;
@@ -51,7 +51,7 @@ int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
 
 	return 0;
 }
-EXPORT_SYMBOL(__kfifo_alloc);
+EXPORT_SYMBOL(__kfifo_alloc_node);
 
 void __kfifo_free(struct __kfifo *fifo)
 {
diff --git a/lib/kobject.c b/lib/kobject.c
index abe5f5b856ce..cfdb2c3f20a2 100644
--- a/lib/kobject.c
+++ b/lib/kobject.c
@@ -765,7 +765,7 @@ static struct kobject *kobject_create(void)
 {
 	struct kobject *kobj;
 
-	kobj = kzalloc(sizeof(*kobj), GFP_KERNEL);
+	kobj = kzalloc_obj(*kobj);
 	if (!kobj)
 		return NULL;
 
@@ -962,7 +962,7 @@ static struct kset *kset_create(const char *name,
 	struct kset *kset;
 	int retval;
 
-	kset = kzalloc(sizeof(*kset), GFP_KERNEL);
+	kset = kzalloc_obj(*kset);
 	if (!kset)
 		return NULL;
 	retval = kobject_set_name(&kset->kobj, "%s", name);
diff --git a/lib/kobject_uevent.c b/lib/kobject_uevent.c
index 78e16b95d210..871941c9830c 100644
--- a/lib/kobject_uevent.c
+++ b/lib/kobject_uevent.c
@@ -124,7 +124,7 @@ static int kobject_action_args(const char *buf, size_t count,
 	if (!count)
 		return -EINVAL;
 
-	env = kzalloc(sizeof(*env), GFP_KERNEL);
+	env = kzalloc_obj(*env);
 	if (!env)
 		return -ENOMEM;
 
@@ -537,7 +537,7 @@ int kobject_uevent_env(struct kobject *kobj, enum kobject_action action,
 	}
 
 	/* environment buffer */
-	env = kzalloc(sizeof(struct kobj_uevent_env), GFP_KERNEL);
+	env = kzalloc_obj(struct kobj_uevent_env);
 	if (!env)
 		return -ENOMEM;
 
@@ -776,7 +776,7 @@ static int uevent_net_init(struct net *net)
 		.flags	= NL_CFG_F_NONROOT_RECV
 	};
 
-	ue_sk = kzalloc(sizeof(*ue_sk), GFP_KERNEL);
+	ue_sk = kzalloc_obj(*ue_sk);
 	if (!ue_sk)
 		return -ENOMEM;
 
diff --git a/lib/kstrtox.c b/lib/kstrtox.c
index bdde40cd69d7..97be2a39f537 100644
--- a/lib/kstrtox.c
+++ b/lib/kstrtox.c
@@ -340,8 +340,8 @@ EXPORT_SYMBOL(kstrtos8);
  * @s: input string
  * @res: result
  *
- * This routine returns 0 iff the first character is one of 'YyTt1NnFf0', or
- * [oO][NnFf] for "on" and "off". Otherwise it will return -EINVAL.  Value
+ * This routine returns 0 iff the first character is one of 'EeYyTt1DdNnFf0',
+ * or [oO][NnFf] for "on" and "off". Otherwise it will return -EINVAL.  Value
  * pointed to by res is updated upon finding a match.
  */
 noinline
diff --git a/lib/kunit/Kconfig b/lib/kunit/Kconfig
index 7a6af361d2fc..498cc51e493d 100644
--- a/lib/kunit/Kconfig
+++ b/lib/kunit/Kconfig
@@ -28,7 +28,7 @@ config KUNIT_FAULT_TEST
 	bool "Enable KUnit tests which print BUG stacktraces"
 	depends on KUNIT_TEST
 	depends on !UML
-	default y
+	default !PANIC_ON_OOPS
 	help
 	  Enables fault handling tests for the KUnit framework. These tests may
 	  trigger a kernel BUG(), and the associated stack trace, even when they
@@ -93,6 +93,30 @@ config KUNIT_AUTORUN_ENABLED
 	  In most cases this should be left as Y. Only if additional opt-in
 	  behavior is needed should this be set to N.
 
+config KUNIT_DEFAULT_FILTER_GLOB
+	string "Default value of the filter_glob module parameter"
+	help
+	  Sets the default value of kunit.filter_glob. If set to a non-empty
+	  string only matching tests are executed.
+
+	  If unsure, leave empty so all tests are executed.
+
+config KUNIT_DEFAULT_FILTER
+	string "Default value of the filter module parameter"
+	help
+	  Sets the default value of kunit.filter. If set to a non-empty
+	  string only matching tests are executed.
+
+	  If unsure, leave empty so all tests are executed.
+
+config KUNIT_DEFAULT_FILTER_ACTION
+	string "Default value of the filter_action module parameter"
+	help
+	  Sets the default value of kunit.filter_action. If set to a non-empty
+	  string only matching tests are executed.
+
+	  If unsure, leave empty so all tests are executed.
+
 config KUNIT_DEFAULT_TIMEOUT
 	int "Default value of the timeout module parameter"
 	default 300
diff --git a/lib/kunit/assert.c b/lib/kunit/assert.c
index 867aa5c4bccf..4c751ad8506a 100644
--- a/lib/kunit/assert.c
+++ b/lib/kunit/assert.c
@@ -51,7 +51,7 @@ void kunit_unary_assert_format(const struct kunit_assert *assert,
 			       const struct va_format *message,
 			       struct string_stream *stream)
 {
-	struct kunit_unary_assert *unary_assert;
+	const struct kunit_unary_assert *unary_assert;
 
 	unary_assert = container_of(assert, struct kunit_unary_assert, assert);
 
@@ -71,7 +71,7 @@ void kunit_ptr_not_err_assert_format(const struct kunit_assert *assert,
 				     const struct va_format *message,
 				     struct string_stream *stream)
 {
-	struct kunit_ptr_not_err_assert *ptr_assert;
+	const struct kunit_ptr_not_err_assert *ptr_assert;
 
 	ptr_assert = container_of(assert, struct kunit_ptr_not_err_assert,
 				  assert);
@@ -117,7 +117,7 @@ void kunit_binary_assert_format(const struct kunit_assert *assert,
 				const struct va_format *message,
 				struct string_stream *stream)
 {
-	struct kunit_binary_assert *binary_assert;
+	const struct kunit_binary_assert *binary_assert;
 
 	binary_assert = container_of(assert, struct kunit_binary_assert,
 				     assert);
@@ -145,7 +145,7 @@ void kunit_binary_ptr_assert_format(const struct kunit_assert *assert,
 				    const struct va_format *message,
 				    struct string_stream *stream)
 {
-	struct kunit_binary_ptr_assert *binary_assert;
+	const struct kunit_binary_ptr_assert *binary_assert;
 
 	binary_assert = container_of(assert, struct kunit_binary_ptr_assert,
 				     assert);
@@ -185,7 +185,7 @@ void kunit_binary_str_assert_format(const struct kunit_assert *assert,
 				    const struct va_format *message,
 				    struct string_stream *stream)
 {
-	struct kunit_binary_str_assert *binary_assert;
+	const struct kunit_binary_str_assert *binary_assert;
 
 	binary_assert = container_of(assert, struct kunit_binary_str_assert,
 				     assert);
@@ -237,7 +237,7 @@ void kunit_mem_assert_format(const struct kunit_assert *assert,
 			     const struct va_format *message,
 			     struct string_stream *stream)
 {
-	struct kunit_mem_assert *mem_assert;
+	const struct kunit_mem_assert *mem_assert;
 
 	mem_assert = container_of(assert, struct kunit_mem_assert,
 				  assert);
diff --git a/lib/kunit/attributes.c b/lib/kunit/attributes.c
index 2cf04cc09372..6d7a53af94a9 100644
--- a/lib/kunit/attributes.c
+++ b/lib/kunit/attributes.c
@@ -410,7 +410,7 @@ struct kunit_suite *kunit_filter_attr_tests(const struct kunit_suite *const suit
 
 	kunit_suite_for_each_test_case(suite, test_case) { n++; }
 
-	filtered = kcalloc(n + 1, sizeof(*filtered), GFP_KERNEL);
+	filtered = kzalloc_objs(*filtered, n + 1);
 	if (!filtered) {
 		kfree(copy);
 		return ERR_PTR(-ENOMEM);
diff --git a/lib/kunit/device.c b/lib/kunit/device.c
index 520c1fccee8a..85d57ad34045 100644
--- a/lib/kunit/device.c
+++ b/lib/kunit/device.c
@@ -106,13 +106,12 @@ EXPORT_SYMBOL_GPL(kunit_driver_create);
 
 /* Helper which creates a kunit_device, attaches it to the kunit_bus*/
 static struct kunit_device *kunit_device_register_internal(struct kunit *test,
-							   const char *name,
-							   const struct device_driver *drv)
+							   const char *name)
 {
 	struct kunit_device *kunit_dev;
 	int err = -ENOMEM;
 
-	kunit_dev = kzalloc(sizeof(*kunit_dev), GFP_KERNEL);
+	kunit_dev = kzalloc_obj(*kunit_dev);
 	if (!kunit_dev)
 		return ERR_PTR(err);
 
@@ -150,7 +149,7 @@ struct device *kunit_device_register_with_driver(struct kunit *test,
 						 const char *name,
 						 const struct device_driver *drv)
 {
-	struct kunit_device *kunit_dev = kunit_device_register_internal(test, name, drv);
+	struct kunit_device *kunit_dev = kunit_device_register_internal(test, name);
 
 	if (IS_ERR_OR_NULL(kunit_dev))
 		return ERR_CAST(kunit_dev);
@@ -172,7 +171,7 @@ struct device *kunit_device_register(struct kunit *test, const char *name)
 	if (IS_ERR(drv))
 		return ERR_CAST(drv);
 
-	dev = kunit_device_register_internal(test, name, drv);
+	dev = kunit_device_register_internal(test, name);
 	if (IS_ERR(dev)) {
 		kunit_release_action(test, driver_unregister_wrapper, (void *)drv);
 		return ERR_CAST(dev);
diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c
index 0061d4c7e351..1fef217de11d 100644
--- a/lib/kunit/executor.c
+++ b/lib/kunit/executor.c
@@ -45,9 +45,11 @@ bool kunit_autorun(void)
 	return autorun_param;
 }
 
-static char *filter_glob_param;
-static char *filter_param;
-static char *filter_action_param;
+#define PARAM_FROM_CONFIG(config) (config[0] ? config : NULL)
+
+static char *filter_glob_param = PARAM_FROM_CONFIG(CONFIG_KUNIT_DEFAULT_FILTER_GLOB);
+static char *filter_param = PARAM_FROM_CONFIG(CONFIG_KUNIT_DEFAULT_FILTER);
+static char *filter_action_param = PARAM_FROM_CONFIG(CONFIG_KUNIT_DEFAULT_FILTER_ACTION);
 
 module_param_named(filter_glob, filter_glob_param, charp, 0600);
 MODULE_PARM_DESC(filter_glob,
@@ -129,7 +131,7 @@ kunit_filter_glob_tests(const struct kunit_suite *const suite, const char *test_
 	if (!copy)
 		return ERR_PTR(-ENOMEM);
 
-	filtered = kcalloc(n + 1, sizeof(*filtered), GFP_KERNEL);
+	filtered = kzalloc_objs(*filtered, n + 1);
 	if (!filtered) {
 		kfree(copy);
 		return ERR_PTR(-ENOMEM);
@@ -177,7 +179,7 @@ kunit_filter_suites(const struct kunit_suite_set *suite_set,
 
 	const size_t max = suite_set->end - suite_set->start;
 
-	copy = kcalloc(max, sizeof(*copy), GFP_KERNEL);
+	copy = kzalloc_objs(*copy, max);
 	if (!copy) { /* won't be able to run anything, return an empty set */
 		return filtered;
 	}
@@ -192,7 +194,7 @@ kunit_filter_suites(const struct kunit_suite_set *suite_set,
 	/* Parse attribute filters */
 	if (filters) {
 		filter_count = kunit_get_filter_count(filters);
-		parsed_filters = kcalloc(filter_count, sizeof(*parsed_filters), GFP_KERNEL);
+		parsed_filters = kzalloc_objs(*parsed_filters, filter_count);
 		if (!parsed_filters) {
 			*err = -ENOMEM;
 			goto free_parsed_glob;
diff --git a/lib/kunit/executor_test.c b/lib/kunit/executor_test.c
index f0090c2729cd..4cb119ad8f64 100644
--- a/lib/kunit/executor_test.c
+++ b/lib/kunit/executor_test.c
@@ -272,7 +272,7 @@ static void free_suite_set_at_end(struct kunit *test, const void *to_free)
 	if (!((struct kunit_suite_set *)to_free)->start)
 		return;
 
-	free = kzalloc(sizeof(struct kunit_suite_set), GFP_KERNEL);
+	free = kzalloc_obj(struct kunit_suite_set);
 	*free = *(struct kunit_suite_set *)to_free;
 
 	kunit_add_action(test, free_suite_set, (void *)free);
diff --git a/lib/kunit/kunit-example-test.c b/lib/kunit/kunit-example-test.c
index 9452b163956f..0bae7b7ca0b0 100644
--- a/lib/kunit/kunit-example-test.c
+++ b/lib/kunit/kunit-example-test.c
@@ -283,7 +283,7 @@ static void example_slow_test(struct kunit *test)
  */
 static int example_resource_init(struct kunit_resource *res, void *context)
 {
-	int *info = kmalloc(sizeof(*info), GFP_KERNEL);
+	int *info = kmalloc_obj(*info);
 
 	if (!info)
 		return -ENOMEM;
diff --git a/lib/kunit/kunit-test.c b/lib/kunit/kunit-test.c
index 63130a48e237..126e30879dad 100644
--- a/lib/kunit/kunit-test.c
+++ b/lib/kunit/kunit-test.c
@@ -538,8 +538,7 @@ static void kunit_resource_test_action_ordering(struct kunit *test)
 
 static int kunit_resource_test_init(struct kunit *test)
 {
-	struct kunit_test_resource_context *ctx =
-			kzalloc(sizeof(*ctx), GFP_KERNEL);
+	struct kunit_test_resource_context *ctx = kzalloc_obj(*ctx);
 
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
 
diff --git a/lib/kunit/resource.c b/lib/kunit/resource.c
index f0209252b179..45e55238ccf6 100644
--- a/lib/kunit/resource.c
+++ b/lib/kunit/resource.c
@@ -98,7 +98,7 @@ int kunit_add_action(struct kunit *test, void (*action)(void *), void *ctx)
 
 	KUNIT_ASSERT_NOT_NULL_MSG(test, action, "Tried to action a NULL function!");
 
-	action_ctx = kzalloc(sizeof(*action_ctx), GFP_KERNEL);
+	action_ctx = kzalloc_obj(*action_ctx);
 	if (!action_ctx)
 		return -ENOMEM;
 
diff --git a/lib/kunit/static_stub.c b/lib/kunit/static_stub.c
index 484fd85251b4..d9dd6377aa38 100644
--- a/lib/kunit/static_stub.c
+++ b/lib/kunit/static_stub.c
@@ -111,7 +111,7 @@ void __kunit_activate_static_stub(struct kunit *test,
 		/* We got an extra reference from find_resource(), so put it. */
 		kunit_put_resource(res);
 	} else {
-		ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
+		ctx = kmalloc_obj(*ctx);
 		KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
 		ctx->real_fn_addr = real_fn_addr;
 		ctx->replacement_addr = replacement_addr;
diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c
index 54f4fdcbfac8..0d8f1b30559b 100644
--- a/lib/kunit/string-stream.c
+++ b/lib/kunit/string-stream.c
@@ -18,7 +18,7 @@ static struct string_stream_fragment *alloc_string_stream_fragment(int len, gfp_
 {
 	struct string_stream_fragment *frag;
 
-	frag = kzalloc(sizeof(*frag), gfp);
+	frag = kzalloc_obj(*frag, gfp);
 	if (!frag)
 		return ERR_PTR(-ENOMEM);
 
@@ -158,7 +158,7 @@ struct string_stream *alloc_string_stream(gfp_t gfp)
 {
 	struct string_stream *stream;
 
-	stream = kzalloc(sizeof(*stream), gfp);
+	stream = kzalloc_obj(*stream, gfp);
 	if (!stream)
 		return ERR_PTR(-ENOMEM);
 
diff --git a/lib/kunit/test.c b/lib/kunit/test.c
index 62eb529824c6..41e1c89799b6 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c
@@ -94,7 +94,7 @@ struct kunit_result_stats {
 	unsigned long total;
 };
 
-static bool kunit_should_print_stats(struct kunit_result_stats stats)
+static bool kunit_should_print_stats(struct kunit_result_stats *stats)
 {
 	if (kunit_stats_enabled == 0)
 		return false;
@@ -102,11 +102,11 @@ static bool kunit_should_print_stats(struct kunit_result_stats stats)
 	if (kunit_stats_enabled == 2)
 		return true;
 
-	return (stats.total > 1);
+	return (stats->total > 1);
 }
 
 static void kunit_print_test_stats(struct kunit *test,
-				   struct kunit_result_stats stats)
+				   struct kunit_result_stats *stats)
 {
 	if (!kunit_should_print_stats(stats))
 		return;
@@ -115,10 +115,10 @@ static void kunit_print_test_stats(struct kunit *test,
 		  KUNIT_SUBTEST_INDENT
 		  "# %s: pass:%lu fail:%lu skip:%lu total:%lu",
 		  test->name,
-		  stats.passed,
-		  stats.failed,
-		  stats.skipped,
-		  stats.total);
+		  stats->passed,
+		  stats->failed,
+		  stats->skipped,
+		  stats->total);
 }
 
 /* Append formatted message to log. */
@@ -600,26 +600,26 @@ static void kunit_run_case_catch_errors(struct kunit_suite *suite,
 }
 
 static void kunit_print_suite_stats(struct kunit_suite *suite,
-				    struct kunit_result_stats suite_stats,
-				    struct kunit_result_stats param_stats)
+				    struct kunit_result_stats *suite_stats,
+				    struct kunit_result_stats *param_stats)
 {
 	if (kunit_should_print_stats(suite_stats)) {
 		kunit_log(KERN_INFO, suite,
 			  "# %s: pass:%lu fail:%lu skip:%lu total:%lu",
 			  suite->name,
-			  suite_stats.passed,
-			  suite_stats.failed,
-			  suite_stats.skipped,
-			  suite_stats.total);
+			  suite_stats->passed,
+			  suite_stats->failed,
+			  suite_stats->skipped,
+			  suite_stats->total);
 	}
 
 	if (kunit_should_print_stats(param_stats)) {
 		kunit_log(KERN_INFO, suite,
 			  "# Totals: pass:%lu fail:%lu skip:%lu total:%lu",
-			  param_stats.passed,
-			  param_stats.failed,
-			  param_stats.skipped,
-			  param_stats.total);
+			  param_stats->passed,
+			  param_stats->failed,
+			  param_stats->skipped,
+			  param_stats->total);
 	}
 }
 
@@ -681,13 +681,116 @@ static void kunit_init_parent_param_test(struct kunit_case *test_case, struct ku
 	}
 }
 
-int kunit_run_tests(struct kunit_suite *suite)
+static noinline_for_stack void
+kunit_run_param_test(struct kunit_suite *suite, struct kunit_case *test_case,
+		     struct kunit *test,
+		     struct kunit_result_stats *suite_stats,
+		     struct kunit_result_stats *total_stats,
+		     struct kunit_result_stats *param_stats)
 {
 	char param_desc[KUNIT_PARAM_DESC_SIZE];
+	const void *curr_param;
+
+	kunit_init_parent_param_test(test_case, test);
+	if (test_case->status == KUNIT_FAILURE) {
+		kunit_update_stats(param_stats, test->status);
+		return;
+	}
+	/* Get initial param. */
+	param_desc[0] = '\0';
+	/* TODO: Make generate_params try-catch */
+	curr_param = test_case->generate_params(test, NULL, param_desc);
+	test_case->status = KUNIT_SKIPPED;
+	kunit_log(KERN_INFO, test, KUNIT_SUBTEST_INDENT KUNIT_SUBTEST_INDENT
+		  "KTAP version 1\n");
+	kunit_log(KERN_INFO, test, KUNIT_SUBTEST_INDENT KUNIT_SUBTEST_INDENT
+		  "# Subtest: %s", test_case->name);
+	if (test->params_array.params &&
+	    test_case->generate_params == kunit_array_gen_params) {
+		kunit_log(KERN_INFO, test, KUNIT_SUBTEST_INDENT
+			  KUNIT_SUBTEST_INDENT "1..%zd\n",
+			  test->params_array.num_params);
+	}
+
+	while (curr_param) {
+		struct kunit param_test = {
+			.param_value = curr_param,
+			.param_index = ++test->param_index,
+			.parent = test,
+		};
+		kunit_init_test(&param_test, test_case->name, NULL);
+		param_test.log = test_case->log;
+		kunit_run_case_catch_errors(suite, test_case, &param_test);
+
+		if (param_desc[0] == '\0') {
+			snprintf(param_desc, sizeof(param_desc),
+				 "param-%d", param_test.param_index);
+		}
+
+		kunit_print_ok_not_ok(&param_test, KUNIT_LEVEL_CASE_PARAM,
+				      param_test.status,
+				      param_test.param_index,
+				      param_desc,
+				      param_test.status_comment);
+
+		kunit_update_stats(param_stats, param_test.status);
+
+		/* Get next param. */
+		param_desc[0] = '\0';
+		curr_param = test_case->generate_params(test, curr_param,
+							param_desc);
+	}
+	/*
+	 * TODO: Put into a try catch. Since we don't need suite->exit
+	 * for it we can't reuse kunit_try_run_cleanup for this yet.
+	 */
+	if (test_case->param_exit)
+		test_case->param_exit(test);
+	/* TODO: Put this kunit_cleanup into a try-catch. */
+	kunit_cleanup(test);
+}
+
+static noinline_for_stack void
+kunit_run_one_test(struct kunit_suite *suite, struct kunit_case *test_case,
+		   struct kunit_result_stats *suite_stats,
+		   struct kunit_result_stats *total_stats)
+{
+	struct kunit test = { .param_value = NULL, .param_index = 0 };
+	struct kunit_result_stats param_stats = { 0 };
+
+	kunit_init_test(&test, test_case->name, test_case->log);
+	if (test_case->status == KUNIT_SKIPPED) {
+		/* Test marked as skip */
+		test.status = KUNIT_SKIPPED;
+		kunit_update_stats(&param_stats, test.status);
+	} else if (!test_case->generate_params) {
+		/* Non-parameterised test. */
+		test_case->status = KUNIT_SKIPPED;
+		kunit_run_case_catch_errors(suite, test_case, &test);
+		kunit_update_stats(&param_stats, test.status);
+	} else {
+		kunit_run_param_test(suite, test_case, &test, suite_stats,
+				     total_stats, &param_stats);
+	}
+	kunit_print_attr((void *)test_case, true, KUNIT_LEVEL_CASE);
+
+	kunit_print_test_stats(&test, &param_stats);
+
+	kunit_print_ok_not_ok(&test, KUNIT_LEVEL_CASE, test_case->status,
+			      kunit_test_case_num(suite, test_case),
+			      test_case->name,
+			      test.status_comment);
+
+	kunit_update_stats(suite_stats, test_case->status);
+	kunit_accumulate_stats(total_stats, param_stats);
+}
+
+
+int kunit_run_tests(struct kunit_suite *suite)
+{
 	struct kunit_case *test_case;
 	struct kunit_result_stats suite_stats = { 0 };
 	struct kunit_result_stats total_stats = { 0 };
-	const void *curr_param;
 
 	/* Taint the kernel so we know we've run tests. */
 	add_taint(TAINT_TEST, LOCKDEP_STILL_OK);
@@ -703,97 +806,13 @@ int kunit_run_tests(struct kunit_suite *suite)
 
 	kunit_print_suite_start(suite);
 
-	kunit_suite_for_each_test_case(suite, test_case) {
-		struct kunit test = { .param_value = NULL, .param_index = 0 };
-		struct kunit_result_stats param_stats = { 0 };
-
-		kunit_init_test(&test, test_case->name, test_case->log);
-		if (test_case->status == KUNIT_SKIPPED) {
-			/* Test marked as skip */
-			test.status = KUNIT_SKIPPED;
-			kunit_update_stats(&param_stats, test.status);
-		} else if (!test_case->generate_params) {
-			/* Non-parameterised test. */
-			test_case->status = KUNIT_SKIPPED;
-			kunit_run_case_catch_errors(suite, test_case, &test);
-			kunit_update_stats(&param_stats, test.status);
-		} else {
-			kunit_init_parent_param_test(test_case, &test);
-			if (test_case->status == KUNIT_FAILURE) {
-				kunit_update_stats(&param_stats, test.status);
-				goto test_case_end;
-			}
-			/* Get initial param. */
-			param_desc[0] = '\0';
-			/* TODO: Make generate_params try-catch */
-			curr_param = test_case->generate_params(&test, NULL, param_desc);
-			test_case->status = KUNIT_SKIPPED;
-			kunit_log(KERN_INFO, &test, KUNIT_SUBTEST_INDENT KUNIT_SUBTEST_INDENT
-				  "KTAP version 1\n");
-			kunit_log(KERN_INFO, &test, KUNIT_SUBTEST_INDENT KUNIT_SUBTEST_INDENT
-				  "# Subtest: %s", test_case->name);
-			if (test.params_array.params &&
-			    test_case->generate_params == kunit_array_gen_params) {
-				kunit_log(KERN_INFO, &test, KUNIT_SUBTEST_INDENT
-					  KUNIT_SUBTEST_INDENT "1..%zd\n",
-					  test.params_array.num_params);
-			}
-
-			while (curr_param) {
-				struct kunit param_test = {
-					.param_value = curr_param,
-					.param_index = ++test.param_index,
-					.parent = &test,
-				};
-				kunit_init_test(&param_test, test_case->name, NULL);
-				param_test.log = test_case->log;
-				kunit_run_case_catch_errors(suite, test_case, &param_test);
-
-				if (param_desc[0] == '\0') {
-					snprintf(param_desc, sizeof(param_desc),
-						 "param-%d", param_test.param_index);
-				}
-
-				kunit_print_ok_not_ok(&param_test, KUNIT_LEVEL_CASE_PARAM,
-						      param_test.status,
-						      param_test.param_index,
-						      param_desc,
-						      param_test.status_comment);
-
-				kunit_update_stats(&param_stats, param_test.status);
-
-				/* Get next param. */
-				param_desc[0] = '\0';
-				curr_param = test_case->generate_params(&test, curr_param,
-									param_desc);
-			}
-			/*
-			 * TODO: Put into a try catch. Since we don't need suite->exit
-			 * for it we can't reuse kunit_try_run_cleanup for this yet.
-			 */
-			if (test_case->param_exit)
-				test_case->param_exit(&test);
-			/* TODO: Put this kunit_cleanup into a try-catch. */
-			kunit_cleanup(&test);
-		}
-test_case_end:
-		kunit_print_attr((void *)test_case, true, KUNIT_LEVEL_CASE);
-
-		kunit_print_test_stats(&test, param_stats);
-
-		kunit_print_ok_not_ok(&test, KUNIT_LEVEL_CASE, test_case->status,
-				      kunit_test_case_num(suite, test_case),
-				      test_case->name,
-				      test.status_comment);
-
-		kunit_update_stats(&suite_stats, test_case->status);
-		kunit_accumulate_stats(&total_stats, param_stats);
-	}
+	kunit_suite_for_each_test_case(suite, test_case)
+		kunit_run_one_test(suite, test_case, &suite_stats, &total_stats);
 
 	if (suite->suite_exit)
 		suite->suite_exit(suite);
 
-	kunit_print_suite_stats(suite, suite_stats, total_stats);
+	kunit_print_suite_stats(suite, &suite_stats, &total_stats);
 suite_end:
 	kunit_print_suite_end(suite);
 
diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c
index ed99344317f5..d939403331b5 100644
--- a/lib/locking-selftest.c
+++ b/lib/locking-selftest.c
@@ -202,7 +202,7 @@ static void init_shared_classes(void)
 	local_irq_disable();			\
 	__irq_enter();				\
 	lockdep_hardirq_threaded();		\
-	WARN_ON(!in_irq());
+	WARN_ON(!in_hardirq());
 
 #define HARDIRQ_EXIT()				\
 	__irq_exit();				\
@@ -2512,7 +2512,7 @@ DEFINE_LOCK_GUARD_0(NOTTHREADED_HARDIRQ,
 	do {
 		local_irq_disable();
 		__irq_enter();
-		WARN_ON(!in_irq());
+		WARN_ON(!in_hardirq());
 	} while(0), HARDIRQ_EXIT())
 DEFINE_LOCK_GUARD_0(SOFTIRQ, SOFTIRQ_ENTER(), SOFTIRQ_EXIT())
 
diff --git a/lib/logic_iomem.c b/lib/logic_iomem.c
index b247d412ddef..42a571d05670 100644
--- a/lib/logic_iomem.c
+++ b/lib/logic_iomem.c
@@ -48,7 +48,7 @@ int logic_iomem_add_region(struct resource *resource,
 	if (WARN_ON((resource->flags & IORESOURCE_TYPE_BITS) != IORESOURCE_MEM))
 		return -EINVAL;
 
-	rreg = kzalloc(sizeof(*rreg), GFP_KERNEL);
+	rreg = kzalloc_obj(*rreg);
 	if (!rreg)
 		return -ENOMEM;
 
diff --git a/lib/lru_cache.c b/lib/lru_cache.c
index 9e0d469c7658..82f775044056 100644
--- a/lib/lru_cache.c
+++ b/lib/lru_cache.c
@@ -94,14 +94,14 @@ struct lru_cache *lc_create(const char *name, struct kmem_cache *cache,
 	if (e_count > LC_MAX_ACTIVE)
 		return NULL;
 
-	slot = kcalloc(e_count, sizeof(struct hlist_head), GFP_KERNEL);
+	slot = kzalloc_objs(struct hlist_head, e_count);
 	if (!slot)
 		goto out_fail;
-	element = kcalloc(e_count, sizeof(struct lc_element *), GFP_KERNEL);
+	element = kzalloc_objs(struct lc_element *, e_count);
 	if (!element)
 		goto out_fail;
 
-	lc = kzalloc(sizeof(*lc), GFP_KERNEL);
+	lc = kzalloc_obj(*lc);
 	if (!lc)
 		goto out_fail;
 
diff --git a/lib/lwq.c b/lib/lwq.c
index 57d080a4d53d..c1e11ba6f254 100644
--- a/lib/lwq.c
+++ b/lib/lwq.c
@@ -110,7 +110,7 @@ static int lwq_test(void)
 	for (i = 0; i < ARRAY_SIZE(threads); i++)
 		threads[i] = kthread_run(lwq_exercise, &q, "lwq-test-%d", i);
 	for (i = 0; i < 100; i++) {
-		t = kmalloc(sizeof(*t), GFP_KERNEL);
+		t = kmalloc_obj(*t);
 		if (!t)
 			break;
 		t->i = i;
diff --git a/lib/math/div64.c b/lib/math/div64.c
index bf77b9843175..d1e92ea24fce 100644
--- a/lib/math/div64.c
+++ b/lib/math/div64.c
@@ -177,94 +177,157 @@ EXPORT_SYMBOL(div64_s64);
  * Iterative div/mod for use when dividend is not expected to be much
  * bigger than divisor.
  */
+#ifndef iter_div_u64_rem
 u32 iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder)
 {
 	return __iter_div_u64_rem(dividend, divisor, remainder);
 }
 EXPORT_SYMBOL(iter_div_u64_rem);
+#endif
 
-#ifndef mul_u64_u64_div_u64
-u64 mul_u64_u64_div_u64(u64 a, u64 b, u64 c)
-{
-	if (ilog2(a) + ilog2(b) <= 62)
-		return div64_u64(a * b, c);
+#if !defined(mul_u64_add_u64_div_u64) || defined(test_mul_u64_add_u64_div_u64)
 
-#if defined(__SIZEOF_INT128__)
+#define mul_add(a, b, c) add_u64_u32(mul_u32_u32(a, b), c)
 
+#if defined(__SIZEOF_INT128__) && !defined(test_mul_u64_add_u64_div_u64)
+static inline u64 mul_u64_u64_add_u64(u64 *p_lo, u64 a, u64 b, u64 c)
+{
 	/* native 64x64=128 bits multiplication */
-	u128 prod = (u128)a * b;
-	u64 n_lo = prod, n_hi = prod >> 64;
+	u128 prod = (u128)a * b + c;
 
+	*p_lo = prod;
+	return prod >> 64;
+}
 #else
-
-	/* perform a 64x64=128 bits multiplication manually */
-	u32 a_lo = a, a_hi = a >> 32, b_lo = b, b_hi = b >> 32;
+static inline u64 mul_u64_u64_add_u64(u64 *p_lo, u64 a, u64 b, u64 c)
+{
+	/* perform a 64x64=128 bits multiplication in 32bit chunks */
 	u64 x, y, z;
 
-	x = (u64)a_lo * b_lo;
-	y = (u64)a_lo * b_hi + (u32)(x >> 32);
-	z = (u64)a_hi * b_hi + (u32)(y >> 32);
-	y = (u64)a_hi * b_lo + (u32)y;
-	z += (u32)(y >> 32);
-	x = (y << 32) + (u32)x;
-
-	u64 n_lo = x, n_hi = z;
+	/* Since (x-1)(x-1) + 2(x-1) == x.x - 1 two u32 can be added to a u64 */
+	x = mul_add(a, b, c);
+	y = mul_add(a, b >> 32, c >> 32);
+	y = add_u64_u32(y, x >> 32);
+	z = mul_add(a >> 32, b >> 32, y >> 32);
+	y = mul_add(a >> 32, b, y);
+	*p_lo = (y << 32) + (u32)x;
+	return add_u64_u32(z, y >> 32);
+}
+#endif
 
+#ifndef BITS_PER_ITER
+#define BITS_PER_ITER (__LONG_WIDTH__ >= 64 ? 32 : 16)
 #endif
 
-	/* make sure c is not zero, trigger runtime exception otherwise */
-	if (unlikely(c == 0)) {
-		unsigned long zero = 0;
+#if BITS_PER_ITER == 32
+#define mul_u64_long_add_u64(p_lo, a, b, c) mul_u64_u64_add_u64(p_lo, a, b, c)
+#define add_u64_long(a, b) ((a) + (b))
+#else
+#undef BITS_PER_ITER
+#define BITS_PER_ITER 16
+static inline u32 mul_u64_long_add_u64(u64 *p_lo, u64 a, u32 b, u64 c)
+{
+	u64 n_lo = mul_add(a, b, c);
+	u64 n_med = mul_add(a >> 32, b, c >> 32);
 
-		OPTIMIZER_HIDE_VAR(zero);
-		return ~0UL/zero;
-	}
+	n_med = add_u64_u32(n_med, n_lo >> 32);
+	*p_lo = n_med << 32 | (u32)n_lo;
+	return n_med >> 32;
+}
+
+#define add_u64_long(a, b) add_u64_u32(a, b)
+#endif
+
+u64 mul_u64_add_u64_div_u64(u64 a, u64 b, u64 c, u64 d)
+{
+	unsigned long d_msig, q_digit;
+	unsigned int reps, d_z_hi;
+	u64 quotient, n_lo, n_hi;
+	u32 overflow;
 
-	int shift = __builtin_ctzll(c);
+	n_hi = mul_u64_u64_add_u64(&n_lo, a, b, c);
 
-	/* try reducing the fraction in case the dividend becomes <= 64 bits */
-	if ((n_hi >> shift) == 0) {
-		u64 n = shift ? (n_lo >> shift) | (n_hi << (64 - shift)) : n_lo;
+	if (!n_hi)
+		return div64_u64(n_lo, d);
 
-		return div64_u64(n, c >> shift);
-		/*
-		 * The remainder value if needed would be:
-		 *   res = div64_u64_rem(n, c >> shift, &rem);
-		 *   rem = (rem << shift) + (n_lo - (n << shift));
-		 */
-	}
+	if (unlikely(n_hi >= d)) {
+		/* trigger runtime exception if divisor is zero */
+		if (d == 0) {
+			unsigned long zero = 0;
 
-	if (n_hi >= c) {
+			OPTIMIZER_HIDE_VAR(zero);
+			return ~0UL/zero;
+		}
 		/* overflow: result is unrepresentable in a u64 */
-		return -1;
+		return ~0ULL;
 	}
 
-	/* Do the full 128 by 64 bits division */
-
-	shift = __builtin_clzll(c);
-	c <<= shift;
+	/* Left align the divisor, shifting the dividend to match */
+	d_z_hi = __builtin_clzll(d);
+	if (d_z_hi) {
+		d <<= d_z_hi;
+		n_hi = n_hi << d_z_hi | n_lo >> (64 - d_z_hi);
+		n_lo <<= d_z_hi;
+	}
 
-	int p = 64 + shift;
-	u64 res = 0;
-	bool carry;
+	reps = 64 / BITS_PER_ITER;
+	/* Optimise loop count for small dividends */
+	if (!(u32)(n_hi >> 32)) {
+		reps -= 32 / BITS_PER_ITER;
+		n_hi = n_hi << 32 | n_lo >> 32;
+		n_lo <<= 32;
+	}
+#if BITS_PER_ITER == 16
+	if (!(u32)(n_hi >> 48)) {
+		reps--;
+		n_hi = add_u64_u32(n_hi << 16, n_lo >> 48);
+		n_lo <<= 16;
+	}
+#endif
 
-	do {
-		carry = n_hi >> 63;
-		shift = carry ? 1 : __builtin_clzll(n_hi);
-		if (p < shift)
-			break;
-		p -= shift;
-		n_hi <<= shift;
-		n_hi |= n_lo >> (64 - shift);
-		n_lo <<= shift;
-		if (carry || (n_hi >= c)) {
-			n_hi -= c;
-			res |= 1ULL << p;
+	/* Invert the dividend so we can use add instead of subtract. */
+	n_lo = ~n_lo;
+	n_hi = ~n_hi;
+
+	/*
+	 * Get the most significant BITS_PER_ITER bits of the divisor.
+	 * This is used to get a low 'guestimate' of the quotient digit.
+	 */
+	d_msig = (d >> (64 - BITS_PER_ITER)) + 1;
+
+	/*
+	 * Now do a 'long division' with BITS_PER_ITER bit 'digits'.
+	 * The 'guess' quotient digit can be low and BITS_PER_ITER+1 bits.
+	 * The worst case is dividing ~0 by 0x8000 which requires two subtracts.
+	 */
+	quotient = 0;
+	while (reps--) {
+		q_digit = (unsigned long)(~n_hi >> (64 - 2 * BITS_PER_ITER)) / d_msig;
+		/* Shift 'n' left to align with the product q_digit * d */
+		overflow = n_hi >> (64 - BITS_PER_ITER);
+		n_hi = add_u64_u32(n_hi << BITS_PER_ITER, n_lo >> (64 - BITS_PER_ITER));
+		n_lo <<= BITS_PER_ITER;
+		/* Add product to negated divisor */
+		overflow += mul_u64_long_add_u64(&n_hi, d, q_digit, n_hi);
+		/* Adjust for the q_digit 'guestimate' being low */
+		while (overflow < 0xffffffff >> (32 - BITS_PER_ITER)) {
+			q_digit++;
+			n_hi += d;
+			overflow += n_hi < d;
 		}
-	} while (n_hi);
-	/* The remainder value if needed would be n_hi << p */
+		quotient = add_u64_long(quotient << BITS_PER_ITER, q_digit);
+	}
 
-	return res;
+	/*
+	 * The above only ensures the remainder doesn't overflow,
+	 * it can still be possible to add (aka subtract) another copy
+	 * of the divisor.
+	 */
+	if ((n_hi + d) > n_hi)
+		quotient++;
+	return quotient;
 }
-EXPORT_SYMBOL(mul_u64_u64_div_u64);
+#if !defined(test_mul_u64_add_u64_div_u64)
+EXPORT_SYMBOL(mul_u64_add_u64_div_u64);
+#endif
 #endif
diff --git a/lib/math/test_mul_u64_u64_div_u64.c b/lib/math/test_mul_u64_u64_div_u64.c
index 58d058de4e73..338d014f0c73 100644
--- a/lib/math/test_mul_u64_u64_div_u64.c
+++ b/lib/math/test_mul_u64_u64_div_u64.c
@@ -10,80 +10,141 @@
 #include <linux/printk.h>
 #include <linux/math64.h>
 
-typedef struct { u64 a; u64 b; u64 c; u64 result; } test_params;
+typedef struct { u64 a; u64 b; u64 d; u64 result; uint round_up;} test_params;
 
 static test_params test_values[] = {
 /* this contains many edge values followed by a couple random values */
-{                0xb,                0x7,                0x3,               0x19 },
-{         0xffff0000,         0xffff0000,                0xf, 0x1110eeef00000000 },
-{         0xffffffff,         0xffffffff,                0x1, 0xfffffffe00000001 },
-{         0xffffffff,         0xffffffff,                0x2, 0x7fffffff00000000 },
-{        0x1ffffffff,         0xffffffff,                0x2, 0xfffffffe80000000 },
-{        0x1ffffffff,         0xffffffff,                0x3, 0xaaaaaaa9aaaaaaab },
-{        0x1ffffffff,        0x1ffffffff,                0x4, 0xffffffff00000000 },
-{ 0xffff000000000000, 0xffff000000000000, 0xffff000000000001, 0xfffeffffffffffff },
-{ 0x3333333333333333, 0x3333333333333333, 0x5555555555555555, 0x1eb851eb851eb851 },
-{ 0x7fffffffffffffff,                0x2,                0x3, 0x5555555555555554 },
-{ 0xffffffffffffffff,                0x2, 0x8000000000000000,                0x3 },
-{ 0xffffffffffffffff,                0x2, 0xc000000000000000,                0x2 },
-{ 0xffffffffffffffff, 0x4000000000000004, 0x8000000000000000, 0x8000000000000007 },
-{ 0xffffffffffffffff, 0x4000000000000001, 0x8000000000000000, 0x8000000000000001 },
-{ 0xffffffffffffffff, 0x8000000000000001, 0xffffffffffffffff, 0x8000000000000001 },
-{ 0xfffffffffffffffe, 0x8000000000000001, 0xffffffffffffffff, 0x8000000000000000 },
-{ 0xffffffffffffffff, 0x8000000000000001, 0xfffffffffffffffe, 0x8000000000000001 },
-{ 0xffffffffffffffff, 0x8000000000000001, 0xfffffffffffffffd, 0x8000000000000002 },
-{ 0x7fffffffffffffff, 0xffffffffffffffff, 0xc000000000000000, 0xaaaaaaaaaaaaaaa8 },
-{ 0xffffffffffffffff, 0x7fffffffffffffff, 0xa000000000000000, 0xccccccccccccccca },
-{ 0xffffffffffffffff, 0x7fffffffffffffff, 0x9000000000000000, 0xe38e38e38e38e38b },
-{ 0x7fffffffffffffff, 0x7fffffffffffffff, 0x5000000000000000, 0xccccccccccccccc9 },
-{ 0xffffffffffffffff, 0xfffffffffffffffe, 0xffffffffffffffff, 0xfffffffffffffffe },
-{ 0xe6102d256d7ea3ae, 0x70a77d0be4c31201, 0xd63ec35ab3220357, 0x78f8bf8cc86c6e18 },
-{ 0xf53bae05cb86c6e1, 0x3847b32d2f8d32e0, 0xcfd4f55a647f403c, 0x42687f79d8998d35 },
-{ 0x9951c5498f941092, 0x1f8c8bfdf287a251, 0xa3c8dc5f81ea3fe2, 0x1d887cb25900091f },
-{ 0x374fee9daa1bb2bb, 0x0d0bfbff7b8ae3ef, 0xc169337bd42d5179, 0x03bb2dbaffcbb961 },
-{ 0xeac0d03ac10eeaf0, 0x89be05dfa162ed9b, 0x92bb1679a41f0e4b, 0xdc5f5cc9e270d216 },
+{                0xb,                0x7,                0x3,               0x19, 1 },
+{         0xffff0000,         0xffff0000,                0xf, 0x1110eeef00000000, 0 },
+{         0xffffffff,         0xffffffff,                0x1, 0xfffffffe00000001, 0 },
+{         0xffffffff,         0xffffffff,                0x2, 0x7fffffff00000000, 1 },
+{        0x1ffffffff,         0xffffffff,                0x2, 0xfffffffe80000000, 1 },
+{        0x1ffffffff,         0xffffffff,                0x3, 0xaaaaaaa9aaaaaaab, 0 },
+{        0x1ffffffff,        0x1ffffffff,                0x4, 0xffffffff00000000, 1 },
+{ 0xffff000000000000, 0xffff000000000000, 0xffff000000000001, 0xfffeffffffffffff, 1 },
+{ 0x3333333333333333, 0x3333333333333333, 0x5555555555555555, 0x1eb851eb851eb851, 1 },
+{ 0x7fffffffffffffff,                0x2,                0x3, 0x5555555555555554, 1 },
+{ 0xffffffffffffffff,                0x2, 0x8000000000000000,                0x3, 1 },
+{ 0xffffffffffffffff,                0x2, 0xc000000000000000,                0x2, 1 },
+{ 0xffffffffffffffff, 0x4000000000000004, 0x8000000000000000, 0x8000000000000007, 1 },
+{ 0xffffffffffffffff, 0x4000000000000001, 0x8000000000000000, 0x8000000000000001, 1 },
+{ 0xffffffffffffffff, 0x8000000000000001, 0xffffffffffffffff, 0x8000000000000001, 0 },
+{ 0xfffffffffffffffe, 0x8000000000000001, 0xffffffffffffffff, 0x8000000000000000, 1 },
+{ 0xffffffffffffffff, 0x8000000000000001, 0xfffffffffffffffe, 0x8000000000000001, 1 },
+{ 0xffffffffffffffff, 0x8000000000000001, 0xfffffffffffffffd, 0x8000000000000002, 1 },
+{ 0x7fffffffffffffff, 0xffffffffffffffff, 0xc000000000000000, 0xaaaaaaaaaaaaaaa8, 1 },
+{ 0xffffffffffffffff, 0x7fffffffffffffff, 0xa000000000000000, 0xccccccccccccccca, 1 },
+{ 0xffffffffffffffff, 0x7fffffffffffffff, 0x9000000000000000, 0xe38e38e38e38e38b, 1 },
+{ 0x7fffffffffffffff, 0x7fffffffffffffff, 0x5000000000000000, 0xccccccccccccccc9, 1 },
+{ 0xffffffffffffffff, 0xfffffffffffffffe, 0xffffffffffffffff, 0xfffffffffffffffe, 0 },
+{ 0xe6102d256d7ea3ae, 0x70a77d0be4c31201, 0xd63ec35ab3220357, 0x78f8bf8cc86c6e18, 1 },
+{ 0xf53bae05cb86c6e1, 0x3847b32d2f8d32e0, 0xcfd4f55a647f403c, 0x42687f79d8998d35, 1 },
+{ 0x9951c5498f941092, 0x1f8c8bfdf287a251, 0xa3c8dc5f81ea3fe2, 0x1d887cb25900091f, 1 },
+{ 0x374fee9daa1bb2bb, 0x0d0bfbff7b8ae3ef, 0xc169337bd42d5179, 0x03bb2dbaffcbb961, 1 },
+{ 0xeac0d03ac10eeaf0, 0x89be05dfa162ed9b, 0x92bb1679a41f0e4b, 0xdc5f5cc9e270d216, 1 },
 };
 
 /*
  * The above table can be verified with the following shell script:
- *
- * #!/bin/sh
- * sed -ne 's/^{ \+\(.*\), \+\(.*\), \+\(.*\), \+\(.*\) },$/\1 \2 \3 \4/p' \
- *     lib/math/test_mul_u64_u64_div_u64.c |
- * while read a b c r; do
- *   expected=$( printf "obase=16; ibase=16; %X * %X / %X\n" $a $b $c | bc )
- *   given=$( printf "%X\n" $r )
- *   if [ "$expected" = "$given" ]; then
- *     echo "$a * $b / $c = $r OK"
- *   else
- *     echo "$a * $b / $c = $r is wrong" >&2
- *     echo "should be equivalent to 0x$expected" >&2
- *     exit 1
- *   fi
- * done
+
+#!/bin/sh
+sed -ne 's/^{ \+\(.*\), \+\(.*\), \+\(.*\), \+\(.*\), \+\(.*\) },$/\1 \2 \3 \4 \5/p' \
+    lib/math/test_mul_u64_u64_div_u64.c |
+while read a b d r e; do
+  expected=$( printf "obase=16; ibase=16; %X * %X / %X\n" $a $b $d | bc )
+  given=$( printf "%X\n" $r )
+  if [ "$expected" = "$given" ]; then
+    echo "$a * $b  / $d = $r OK"
+  else
+    echo "$a * $b  / $d = $r is wrong" >&2
+    echo "should be equivalent to 0x$expected" >&2
+    exit 1
+  fi
+  expected=$( printf "obase=16; ibase=16; (%X * %X + %X) / %X\n" $a $b $((d-1)) $d | bc )
+  given=$( printf "%X\n" $((r + e)) )
+  if [ "$expected" = "$given" ]; then
+    echo "$a * $b +/ $d = $(printf '%#x' $((r + e))) OK"
+  else
+    echo "$a * $b +/ $d = $(printf '%#x' $((r + e))) is wrong" >&2
+    echo "should be equivalent to 0x$expected" >&2
+    exit 1
+  fi
+done
+
  */
 
-static int __init test_init(void)
+static u64 test_mul_u64_add_u64_div_u64(u64 a, u64 b, u64 c, u64 d);
+#if __LONG_WIDTH__ >= 64
+#define TEST_32BIT_DIV
+static u64 test_mul_u64_add_u64_div_u64_32bit(u64 a, u64 b, u64 c, u64 d);
+#endif
+
+static int __init test_run(unsigned int fn_no, const char *fn_name)
 {
+	u64 start_time;
+	int errors = 0;
+	int tests = 0;
 	int i;
 
-	pr_info("Starting mul_u64_u64_div_u64() test\n");
+	start_time = ktime_get_ns();
 
 	for (i = 0; i < ARRAY_SIZE(test_values); i++) {
 		u64 a = test_values[i].a;
 		u64 b = test_values[i].b;
-		u64 c = test_values[i].c;
+		u64 d = test_values[i].d;
 		u64 expected_result = test_values[i].result;
-		u64 result = mul_u64_u64_div_u64(a, b, c);
+		u64 result, result_up;
+
+		switch (fn_no) {
+		default:
+			result = mul_u64_u64_div_u64(a, b, d);
+			result_up = mul_u64_u64_div_u64_roundup(a, b, d);
+			break;
+		case 1:
+			result = test_mul_u64_add_u64_div_u64(a, b, 0, d);
+			result_up = test_mul_u64_add_u64_div_u64(a, b, d - 1, d);
+			break;
+#ifdef TEST_32BIT_DIV
+		case 2:
+			result = test_mul_u64_add_u64_div_u64_32bit(a, b, 0, d);
+			result_up = test_mul_u64_add_u64_div_u64_32bit(a, b, d - 1, d);
+			break;
+#endif
+		}
+
+		tests += 2;
 
 		if (result != expected_result) {
-			pr_err("ERROR: 0x%016llx * 0x%016llx / 0x%016llx\n", a, b, c);
+			pr_err("ERROR: 0x%016llx * 0x%016llx / 0x%016llx\n", a, b, d);
 			pr_err("ERROR: expected result: %016llx\n", expected_result);
 			pr_err("ERROR: obtained result: %016llx\n", result);
+			errors++;
+		}
+		expected_result += test_values[i].round_up;
+		if (result_up != expected_result) {
+			pr_err("ERROR: 0x%016llx * 0x%016llx +/ 0x%016llx\n", a, b, d);
+			pr_err("ERROR: expected result: %016llx\n", expected_result);
+			pr_err("ERROR: obtained result: %016llx\n", result_up);
+			errors++;
 		}
 	}
 
-	pr_info("Completed mul_u64_u64_div_u64() test\n");
+	pr_info("Completed %s() test, %d tests, %d errors, %llu ns\n",
+		fn_name, tests, errors, ktime_get_ns() - start_time);
+	return errors;
+}
+
+static int __init test_init(void)
+{
+	pr_info("Starting mul_u64_u64_div_u64() test\n");
+	if (test_run(0, "mul_u64_u64_div_u64"))
+		return -EINVAL;
+	if (test_run(1, "test_mul_u64_u64_div_u64"))
+		return -EINVAL;
+#ifdef TEST_32BIT_DIV
+	if (test_run(2, "test_mul_u64_u64_div_u64_32bit"))
+		return -EINVAL;
+#endif
 	return 0;
 }
 
@@ -91,6 +152,36 @@ static void __exit test_exit(void)
 {
 }
 
+/* Compile the generic mul_u64_add_u64_div_u64() code */
+#undef __div64_32
+#define __div64_32 __div64_32
+#define div_s64_rem div_s64_rem
+#define div64_u64_rem div64_u64_rem
+#define div64_u64 div64_u64
+#define div64_s64 div64_s64
+#define iter_div_u64_rem iter_div_u64_rem
+
+#undef mul_u64_add_u64_div_u64
+#define mul_u64_add_u64_div_u64 test_mul_u64_add_u64_div_u64
+#define test_mul_u64_add_u64_div_u64 test_mul_u64_add_u64_div_u64
+
+#include "div64.c"
+
+#ifdef TEST_32BIT_DIV
+/* Recompile the generic code for 32bit long */
+#undef test_mul_u64_add_u64_div_u64
+#define test_mul_u64_add_u64_div_u64 test_mul_u64_add_u64_div_u64_32bit
+#undef BITS_PER_ITER
+#define BITS_PER_ITER 16
+
+#define mul_u64_u64_add_u64 mul_u64_u64_add_u64_32bit
+#undef mul_u64_long_add_u64
+#undef add_u64_long
+#undef mul_add
+
+#include "div64.c"
+#endif
+
 module_init(test_init);
 module_exit(test_exit);
 
diff --git a/lib/objagg.c b/lib/objagg.c
index 363e43e849ac..23c7105a3f9f 100644
--- a/lib/objagg.c
+++ b/lib/objagg.c
@@ -525,7 +525,7 @@ struct objagg *objagg_create(const struct objagg_ops *ops,
 		    !ops->delta_destroy))
 		return ERR_PTR(-EINVAL);
 
-	objagg = kzalloc(sizeof(*objagg), GFP_KERNEL);
+	objagg = kzalloc_obj(*objagg);
 	if (!objagg)
 		return ERR_PTR(-ENOMEM);
 	objagg->ops = ops;
@@ -610,8 +610,8 @@ const struct objagg_stats *objagg_stats_get(struct objagg *objagg)
 	struct objagg_obj *objagg_obj;
 	int i;
 
-	objagg_stats = kzalloc(struct_size(objagg_stats, stats_info,
-					   objagg->obj_count), GFP_KERNEL);
+	objagg_stats = kzalloc_flex(*objagg_stats, stats_info,
+				    objagg->obj_count);
 	if (!objagg_stats)
 		return ERR_PTR(-ENOMEM);
 
@@ -786,11 +786,11 @@ static struct objagg_tmp_graph *objagg_tmp_graph_create(struct objagg *objagg)
 	struct objagg_obj *objagg_obj;
 	int i, j;
 
-	graph = kzalloc(sizeof(*graph), GFP_KERNEL);
+	graph = kzalloc_obj(*graph);
 	if (!graph)
 		return NULL;
 
-	graph->nodes = kcalloc(nodes_count, sizeof(*graph->nodes), GFP_KERNEL);
+	graph->nodes = kzalloc_objs(*graph->nodes, nodes_count);
 	if (!graph->nodes)
 		goto err_nodes_alloc;
 	graph->nodes_count = nodes_count;
@@ -930,7 +930,7 @@ struct objagg_hints *objagg_hints_get(struct objagg *objagg,
 	struct objagg_hints *objagg_hints;
 	int err;
 
-	objagg_hints = kzalloc(sizeof(*objagg_hints), GFP_KERNEL);
+	objagg_hints = kzalloc_obj(*objagg_hints);
 	if (!objagg_hints)
 		return ERR_PTR(-ENOMEM);
 
@@ -1010,9 +1010,8 @@ objagg_hints_stats_get(struct objagg_hints *objagg_hints)
 	struct objagg_hints_node *hnode;
 	int i;
 
-	objagg_stats = kzalloc(struct_size(objagg_stats, stats_info,
-					   objagg_hints->node_count),
-			       GFP_KERNEL);
+	objagg_stats = kzalloc_flex(*objagg_stats, stats_info,
+				    objagg_hints->node_count);
 	if (!objagg_stats)
 		return ERR_PTR(-ENOMEM);
 
diff --git a/lib/objpool.c b/lib/objpool.c
index b998b720c732..d98fadf1de16 100644
--- a/lib/objpool.c
+++ b/lib/objpool.c
@@ -142,7 +142,7 @@ int objpool_init(struct objpool_head *pool, int nr_objs, int object_size,
 	pool->gfp = gfp & ~__GFP_ZERO;
 	pool->context = context;
 	pool->release = release;
-	slot_size = nr_cpu_ids * sizeof(struct objpool_slot);
+	slot_size = nr_cpu_ids * sizeof(struct objpool_slot *);
 	pool->cpu_slots = kzalloc(slot_size, pool->gfp);
 	if (!pool->cpu_slots)
 		return -ENOMEM;
diff --git a/lib/once.c b/lib/once.c
index 2c306f0e891e..d801bfa945e6 100644
--- a/lib/once.c
+++ b/lib/once.c
@@ -26,7 +26,7 @@ static void once_disable_jump(struct static_key_true *key, struct module *mod)
 {
 	struct once_work *w;
 
-	w = kmalloc(sizeof(*w), GFP_ATOMIC);
+	w = kmalloc_obj(*w, GFP_ATOMIC);
 	if (!w)
 		return;
 
@@ -93,6 +93,6 @@ void __do_once_sleepable_done(bool *done, struct static_key_true *once_key,
 {
 	*done = true;
 	mutex_unlock(&once_mutex);
-	once_disable_jump(once_key, mod);
+	static_branch_disable(once_key);
 }
 EXPORT_SYMBOL(__do_once_sleepable_done);
diff --git a/lib/parman.c b/lib/parman.c
index 3f8f8d422e62..0de691c202ab 100644
--- a/lib/parman.c
+++ b/lib/parman.c
@@ -268,7 +268,7 @@ struct parman *parman_create(const struct parman_ops *ops, void *priv)
 {
 	struct parman *parman;
 
-	parman = kzalloc(sizeof(*parman), GFP_KERNEL);
+	parman = kzalloc_obj(*parman);
 	if (!parman)
 		return NULL;
 	INIT_LIST_HEAD(&parman->prio_list);
diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c
index 668f6aa6a75d..97772e42b9b2 100644
--- a/lib/percpu-refcount.c
+++ b/lib/percpu-refcount.c
@@ -73,7 +73,7 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release,
 	if (!ref->percpu_count_ptr)
 		return -ENOMEM;
 
-	data = kzalloc(sizeof(*ref->data), gfp);
+	data = kzalloc_obj(*ref->data, gfp);
 	if (!data) {
 		free_percpu((void __percpu *)ref->percpu_count_ptr);
 		ref->percpu_count_ptr = 0;
diff --git a/lib/pldmfw/pldmfw.c b/lib/pldmfw/pldmfw.c
index b45ceb725780..e4612ea147bb 100644
--- a/lib/pldmfw/pldmfw.c
+++ b/lib/pldmfw/pldmfw.c
@@ -287,7 +287,7 @@ pldm_parse_desc_tlvs(struct pldmfw_priv *data, struct pldmfw_record *record, u8
 		if (err)
 			return err;
 
-		desc = kzalloc(sizeof(*desc), GFP_KERNEL);
+		desc = kzalloc_obj(*desc);
 		if (!desc)
 			return -ENOMEM;
 
@@ -328,7 +328,7 @@ pldm_parse_one_record(struct pldmfw_priv *data,
 	int i;
 
 	/* Make a copy and insert it into the record list */
-	record = kzalloc(sizeof(*record), GFP_KERNEL);
+	record = kzalloc_obj(*record);
 	if (!record)
 		return -ENOMEM;
 
@@ -465,7 +465,7 @@ static int pldm_parse_components(struct pldmfw_priv *data)
 		if (err)
 			return err;
 
-		component = kzalloc(sizeof(*component), GFP_KERNEL);
+		component = kzalloc_obj(*component);
 		if (!component)
 			return -ENOMEM;
 
@@ -848,7 +848,7 @@ int pldmfw_flash_image(struct pldmfw *context, const struct firmware *fw)
 	struct pldmfw_priv *data;
 	int err;
 
-	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	data = kzalloc_obj(*data);
 	if (!data)
 		return -ENOMEM;
 
diff --git a/lib/plist.c b/lib/plist.c
index 330febb4bd7d..a5bef38add43 100644
--- a/lib/plist.c
+++ b/lib/plist.c
@@ -10,7 +10,7 @@
  * 2001-2005 (c) MontaVista Software, Inc.
  * Daniel Walker <dwalker@mvista.com>
  *
- * (C) 2005 Thomas Gleixner <tglx@linutronix.de>
+ * (C) 2005 Linutronix GmbH, Thomas Gleixner <tglx@kernel.org>
  *
  * Simplifications of the original code by
  * Oleg Nesterov <oleg@tv-sign.ru>
@@ -47,8 +47,8 @@ static void plist_check_list(struct list_head *top)
 
 	plist_check_prev_next(top, prev, next);
 	while (next != top) {
-		WRITE_ONCE(prev, next);
-		WRITE_ONCE(next, prev->next);
+		prev = next;
+		next = prev->next;
 		plist_check_prev_next(top, prev, next);
 	}
 }
diff --git a/lib/raid6/neon.c b/lib/raid6/neon.c
index 0a2e76035ea9..6d9474ce6da9 100644
--- a/lib/raid6/neon.c
+++ b/lib/raid6/neon.c
@@ -8,10 +8,9 @@
 #include <linux/raid/pq.h>
 
 #ifdef __KERNEL__
-#include <asm/neon.h>
+#include <asm/simd.h>
 #else
-#define kernel_neon_begin()
-#define kernel_neon_end()
+#define scoped_ksimd()
 #define cpu_has_neon()		(1)
 #endif
 
@@ -32,10 +31,9 @@
 	{								\
 		void raid6_neon ## _n  ## _gen_syndrome_real(int,	\
 						unsigned long, void**);	\
-		kernel_neon_begin();					\
-		raid6_neon ## _n ## _gen_syndrome_real(disks,		\
+		scoped_ksimd()						\
+			raid6_neon ## _n ## _gen_syndrome_real(disks,	\
 					(unsigned long)bytes, ptrs);	\
-		kernel_neon_end();					\
 	}								\
 	static void raid6_neon ## _n ## _xor_syndrome(int disks,	\
 					int start, int stop, 		\
@@ -43,10 +41,9 @@
 	{								\
 		void raid6_neon ## _n  ## _xor_syndrome_real(int,	\
 				int, int, unsigned long, void**);	\
-		kernel_neon_begin();					\
-		raid6_neon ## _n ## _xor_syndrome_real(disks,		\
-			start, stop, (unsigned long)bytes, ptrs);	\
-		kernel_neon_end();					\
+		scoped_ksimd()						\
+			raid6_neon ## _n ## _xor_syndrome_real(disks,	\
+				start, stop, (unsigned long)bytes, ptrs);\
 	}								\
 	struct raid6_calls const raid6_neonx ## _n = {			\
 		raid6_neon ## _n ## _gen_syndrome,			\
diff --git a/lib/raid6/recov_neon.c b/lib/raid6/recov_neon.c
index 70e1404c1512..9d99aeabd31a 100644
--- a/lib/raid6/recov_neon.c
+++ b/lib/raid6/recov_neon.c
@@ -7,11 +7,10 @@
 #include <linux/raid/pq.h>
 
 #ifdef __KERNEL__
-#include <asm/neon.h>
+#include <asm/simd.h>
 #include "neon.h"
 #else
-#define kernel_neon_begin()
-#define kernel_neon_end()
+#define scoped_ksimd()
 #define cpu_has_neon()		(1)
 #endif
 
@@ -55,9 +54,8 @@ static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
 	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
 					 raid6_gfexp[failb]]];
 
-	kernel_neon_begin();
-	__raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul);
-	kernel_neon_end();
+	scoped_ksimd()
+		__raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul);
 }
 
 static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
@@ -86,9 +84,8 @@ static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
 	/* Now, pick the proper data tables */
 	qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
 
-	kernel_neon_begin();
-	__raid6_datap_recov_neon(bytes, p, q, dq, qmul);
-	kernel_neon_end();
+	scoped_ksimd()
+		__raid6_datap_recov_neon(bytes, p, q, dq, qmul);
 }
 
 const struct raid6_recov_calls raid6_recov_neon = {
diff --git a/lib/raid6/recov_rvv.c b/lib/raid6/recov_rvv.c
index 5f779719c3d3..40c393206b6a 100644
--- a/lib/raid6/recov_rvv.c
+++ b/lib/raid6/recov_rvv.c
@@ -4,13 +4,8 @@
  * Author: Chunyan Zhang <zhangchunyan@iscas.ac.cn>
  */
 
-#include <asm/vector.h>
 #include <linux/raid/pq.h>
-
-static int rvv_has_vector(void)
-{
-	return has_vector();
-}
+#include "rvv.h"
 
 static void __raid6_2data_recov_rvv(int bytes, u8 *p, u8 *q, u8 *dp,
 				    u8 *dq, const u8 *pbmul,
diff --git a/lib/raid6/rvv.c b/lib/raid6/rvv.c
index 89da5fc247aa..75c9dafedb28 100644
--- a/lib/raid6/rvv.c
+++ b/lib/raid6/rvv.c
@@ -9,22 +9,17 @@
  *	Copyright 2002-2004 H. Peter Anvin
  */
 
-#include <asm/vector.h>
-#include <linux/raid/pq.h>
 #include "rvv.h"
 
-#define NSIZE	(riscv_v_vsize / 32) /* NSIZE = vlenb */
-
-static int rvv_has_vector(void)
-{
-	return has_vector();
-}
+#ifdef __riscv_vector
+#error "This code must be built without compiler support for vector"
+#endif
 
 static void raid6_rvv1_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
 {
 	u8 **dptr = (u8 **)ptrs;
 	u8 *p, *q;
-	unsigned long vl, d;
+	unsigned long vl, d, nsize;
 	int z, z0;
 
 	z0 = disks - 3;		/* Highest data disk */
@@ -38,8 +33,10 @@ static void raid6_rvv1_gen_syndrome_real(int disks, unsigned long bytes, void **
 		      : "=&r" (vl)
 	);
 
+	nsize = vl;
+
 	 /* v0:wp0, v1:wq0, v2:wd0/w20, v3:w10 */
-	for (d = 0; d < bytes; d += NSIZE * 1) {
+	for (d = 0; d < bytes; d += nsize * 1) {
 		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
 		asm volatile (".option	push\n"
 			      ".option	arch,+v\n"
@@ -47,7 +44,7 @@ static void raid6_rvv1_gen_syndrome_real(int disks, unsigned long bytes, void **
 			      "vmv.v.v	v1, v0\n"
 			      ".option	pop\n"
 			      : :
-			      [wp0]"r"(&dptr[z0][d + 0 * NSIZE])
+			      [wp0]"r"(&dptr[z0][d + 0 * nsize])
 		);
 
 		for (z = z0 - 1 ; z >= 0 ; z--) {
@@ -71,7 +68,7 @@ static void raid6_rvv1_gen_syndrome_real(int disks, unsigned long bytes, void **
 				      "vxor.vv	v0, v0, v2\n"
 				      ".option	pop\n"
 				      : :
-				      [wd0]"r"(&dptr[z][d + 0 * NSIZE]),
+				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
 				      [x1d]"r"(0x1d)
 			);
 		}
@@ -86,8 +83,8 @@ static void raid6_rvv1_gen_syndrome_real(int disks, unsigned long bytes, void **
 			      "vse8.v	v1, (%[wq0])\n"
 			      ".option	pop\n"
 			      : :
-			      [wp0]"r"(&p[d + NSIZE * 0]),
-			      [wq0]"r"(&q[d + NSIZE * 0])
+			      [wp0]"r"(&p[d + nsize * 0]),
+			      [wq0]"r"(&q[d + nsize * 0])
 		);
 	}
 }
@@ -97,7 +94,7 @@ static void raid6_rvv1_xor_syndrome_real(int disks, int start, int stop,
 {
 	u8 **dptr = (u8 **)ptrs;
 	u8 *p, *q;
-	unsigned long vl, d;
+	unsigned long vl, d, nsize;
 	int z, z0;
 
 	z0 = stop;		/* P/Q right side optimization */
@@ -111,8 +108,10 @@ static void raid6_rvv1_xor_syndrome_real(int disks, int start, int stop,
 		      : "=&r" (vl)
 	);
 
+	nsize = vl;
+
 	/* v0:wp0, v1:wq0, v2:wd0/w20, v3:w10 */
-	for (d = 0 ; d < bytes ; d += NSIZE * 1) {
+	for (d = 0 ; d < bytes ; d += nsize * 1) {
 		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
 		asm volatile (".option	push\n"
 			      ".option	arch,+v\n"
@@ -120,7 +119,7 @@ static void raid6_rvv1_xor_syndrome_real(int disks, int start, int stop,
 			      "vmv.v.v	v1, v0\n"
 			      ".option	pop\n"
 			      : :
-			      [wp0]"r"(&dptr[z0][d + 0 * NSIZE])
+			      [wp0]"r"(&dptr[z0][d + 0 * nsize])
 		);
 
 		/* P/Q data pages */
@@ -145,7 +144,7 @@ static void raid6_rvv1_xor_syndrome_real(int disks, int start, int stop,
 				      "vxor.vv	v0, v0, v2\n"
 				      ".option	pop\n"
 				      : :
-				      [wd0]"r"(&dptr[z][d + 0 * NSIZE]),
+				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
 				      [x1d]"r"(0x1d)
 			);
 		}
@@ -185,8 +184,8 @@ static void raid6_rvv1_xor_syndrome_real(int disks, int start, int stop,
 			      "vse8.v	v3, (%[wq0])\n"
 			      ".option	pop\n"
 			      : :
-			      [wp0]"r"(&p[d + NSIZE * 0]),
-			      [wq0]"r"(&q[d + NSIZE * 0])
+			      [wp0]"r"(&p[d + nsize * 0]),
+			      [wq0]"r"(&q[d + nsize * 0])
 		);
 	}
 }
@@ -195,7 +194,7 @@ static void raid6_rvv2_gen_syndrome_real(int disks, unsigned long bytes, void **
 {
 	u8 **dptr = (u8 **)ptrs;
 	u8 *p, *q;
-	unsigned long vl, d;
+	unsigned long vl, d, nsize;
 	int z, z0;
 
 	z0 = disks - 3;		/* Highest data disk */
@@ -209,11 +208,13 @@ static void raid6_rvv2_gen_syndrome_real(int disks, unsigned long bytes, void **
 		      : "=&r" (vl)
 	);
 
+	nsize = vl;
+
 	/*
 	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
 	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
 	 */
-	for (d = 0; d < bytes; d += NSIZE * 2) {
+	for (d = 0; d < bytes; d += nsize * 2) {
 		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
 		asm volatile (".option	push\n"
 			      ".option	arch,+v\n"
@@ -223,8 +224,8 @@ static void raid6_rvv2_gen_syndrome_real(int disks, unsigned long bytes, void **
 			      "vmv.v.v	v5, v4\n"
 			      ".option	pop\n"
 			      : :
-			      [wp0]"r"(&dptr[z0][d + 0 * NSIZE]),
-			      [wp1]"r"(&dptr[z0][d + 1 * NSIZE])
+			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
+			      [wp1]"r"(&dptr[z0][d + 1 * nsize])
 		);
 
 		for (z = z0 - 1; z >= 0; z--) {
@@ -256,8 +257,8 @@ static void raid6_rvv2_gen_syndrome_real(int disks, unsigned long bytes, void **
 				      "vxor.vv	v4, v4, v6\n"
 				      ".option	pop\n"
 				      : :
-				      [wd0]"r"(&dptr[z][d + 0 * NSIZE]),
-				      [wd1]"r"(&dptr[z][d + 1 * NSIZE]),
+				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
+				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
 				      [x1d]"r"(0x1d)
 			);
 		}
@@ -274,10 +275,10 @@ static void raid6_rvv2_gen_syndrome_real(int disks, unsigned long bytes, void **
 			      "vse8.v	v5, (%[wq1])\n"
 			      ".option	pop\n"
 			      : :
-			      [wp0]"r"(&p[d + NSIZE * 0]),
-			      [wq0]"r"(&q[d + NSIZE * 0]),
-			      [wp1]"r"(&p[d + NSIZE * 1]),
-			      [wq1]"r"(&q[d + NSIZE * 1])
+			      [wp0]"r"(&p[d + nsize * 0]),
+			      [wq0]"r"(&q[d + nsize * 0]),
+			      [wp1]"r"(&p[d + nsize * 1]),
+			      [wq1]"r"(&q[d + nsize * 1])
 		);
 	}
 }
@@ -287,7 +288,7 @@ static void raid6_rvv2_xor_syndrome_real(int disks, int start, int stop,
 {
 	u8 **dptr = (u8 **)ptrs;
 	u8 *p, *q;
-	unsigned long vl, d;
+	unsigned long vl, d, nsize;
 	int z, z0;
 
 	z0 = stop;		/* P/Q right side optimization */
@@ -301,11 +302,13 @@ static void raid6_rvv2_xor_syndrome_real(int disks, int start, int stop,
 		      : "=&r" (vl)
 	);
 
+	nsize = vl;
+
 	/*
 	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
 	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
 	 */
-	for (d = 0; d < bytes; d += NSIZE * 2) {
+	for (d = 0; d < bytes; d += nsize * 2) {
 		 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
 		asm volatile (".option	push\n"
 			      ".option	arch,+v\n"
@@ -315,8 +318,8 @@ static void raid6_rvv2_xor_syndrome_real(int disks, int start, int stop,
 			      "vmv.v.v	v5, v4\n"
 			      ".option	pop\n"
 			      : :
-			      [wp0]"r"(&dptr[z0][d + 0 * NSIZE]),
-			      [wp1]"r"(&dptr[z0][d + 1 * NSIZE])
+			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
+			      [wp1]"r"(&dptr[z0][d + 1 * nsize])
 		);
 
 		/* P/Q data pages */
@@ -349,8 +352,8 @@ static void raid6_rvv2_xor_syndrome_real(int disks, int start, int stop,
 				      "vxor.vv	v4, v4, v6\n"
 				      ".option	pop\n"
 				      : :
-				      [wd0]"r"(&dptr[z][d + 0 * NSIZE]),
-				      [wd1]"r"(&dptr[z][d + 1 * NSIZE]),
+				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
+				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
 				      [x1d]"r"(0x1d)
 			);
 		}
@@ -403,10 +406,10 @@ static void raid6_rvv2_xor_syndrome_real(int disks, int start, int stop,
 			      "vse8.v	v7, (%[wq1])\n"
 			      ".option	pop\n"
 			      : :
-			      [wp0]"r"(&p[d + NSIZE * 0]),
-			      [wq0]"r"(&q[d + NSIZE * 0]),
-			      [wp1]"r"(&p[d + NSIZE * 1]),
-			      [wq1]"r"(&q[d + NSIZE * 1])
+			      [wp0]"r"(&p[d + nsize * 0]),
+			      [wq0]"r"(&q[d + nsize * 0]),
+			      [wp1]"r"(&p[d + nsize * 1]),
+			      [wq1]"r"(&q[d + nsize * 1])
 		);
 	}
 }
@@ -415,7 +418,7 @@ static void raid6_rvv4_gen_syndrome_real(int disks, unsigned long bytes, void **
 {
 	u8 **dptr = (u8 **)ptrs;
 	u8 *p, *q;
-	unsigned long vl, d;
+	unsigned long vl, d, nsize;
 	int z, z0;
 
 	z0 = disks - 3;	/* Highest data disk */
@@ -429,13 +432,15 @@ static void raid6_rvv4_gen_syndrome_real(int disks, unsigned long bytes, void **
 		      : "=&r" (vl)
 	);
 
+	nsize = vl;
+
 	/*
 	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
 	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
 	 * v8:wp2, v9:wq2, v10:wd2/w22, v11:w12
 	 * v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
 	 */
-	for (d = 0; d < bytes; d += NSIZE * 4) {
+	for (d = 0; d < bytes; d += nsize * 4) {
 		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
 		asm volatile (".option	push\n"
 			      ".option	arch,+v\n"
@@ -449,10 +454,10 @@ static void raid6_rvv4_gen_syndrome_real(int disks, unsigned long bytes, void **
 			      "vmv.v.v	v13, v12\n"
 			      ".option	pop\n"
 			      : :
-			      [wp0]"r"(&dptr[z0][d + 0 * NSIZE]),
-			      [wp1]"r"(&dptr[z0][d + 1 * NSIZE]),
-			      [wp2]"r"(&dptr[z0][d + 2 * NSIZE]),
-			      [wp3]"r"(&dptr[z0][d + 3 * NSIZE])
+			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
+			      [wp1]"r"(&dptr[z0][d + 1 * nsize]),
+			      [wp2]"r"(&dptr[z0][d + 2 * nsize]),
+			      [wp3]"r"(&dptr[z0][d + 3 * nsize])
 		);
 
 		for (z = z0 - 1; z >= 0; z--) {
@@ -500,10 +505,10 @@ static void raid6_rvv4_gen_syndrome_real(int disks, unsigned long bytes, void **
 				      "vxor.vv	v12, v12, v14\n"
 				      ".option	pop\n"
 				      : :
-				      [wd0]"r"(&dptr[z][d + 0 * NSIZE]),
-				      [wd1]"r"(&dptr[z][d + 1 * NSIZE]),
-				      [wd2]"r"(&dptr[z][d + 2 * NSIZE]),
-				      [wd3]"r"(&dptr[z][d + 3 * NSIZE]),
+				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
+				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
+				      [wd2]"r"(&dptr[z][d + 2 * nsize]),
+				      [wd3]"r"(&dptr[z][d + 3 * nsize]),
 				      [x1d]"r"(0x1d)
 			);
 		}
@@ -524,14 +529,14 @@ static void raid6_rvv4_gen_syndrome_real(int disks, unsigned long bytes, void **
 			      "vse8.v	v13, (%[wq3])\n"
 			      ".option	pop\n"
 			      : :
-			      [wp0]"r"(&p[d + NSIZE * 0]),
-			      [wq0]"r"(&q[d + NSIZE * 0]),
-			      [wp1]"r"(&p[d + NSIZE * 1]),
-			      [wq1]"r"(&q[d + NSIZE * 1]),
-			      [wp2]"r"(&p[d + NSIZE * 2]),
-			      [wq2]"r"(&q[d + NSIZE * 2]),
-			      [wp3]"r"(&p[d + NSIZE * 3]),
-			      [wq3]"r"(&q[d + NSIZE * 3])
+			      [wp0]"r"(&p[d + nsize * 0]),
+			      [wq0]"r"(&q[d + nsize * 0]),
+			      [wp1]"r"(&p[d + nsize * 1]),
+			      [wq1]"r"(&q[d + nsize * 1]),
+			      [wp2]"r"(&p[d + nsize * 2]),
+			      [wq2]"r"(&q[d + nsize * 2]),
+			      [wp3]"r"(&p[d + nsize * 3]),
+			      [wq3]"r"(&q[d + nsize * 3])
 		);
 	}
 }
@@ -541,7 +546,7 @@ static void raid6_rvv4_xor_syndrome_real(int disks, int start, int stop,
 {
 	u8 **dptr = (u8 **)ptrs;
 	u8 *p, *q;
-	unsigned long vl, d;
+	unsigned long vl, d, nsize;
 	int z, z0;
 
 	z0 = stop;		/* P/Q right side optimization */
@@ -555,13 +560,15 @@ static void raid6_rvv4_xor_syndrome_real(int disks, int start, int stop,
 		      : "=&r" (vl)
 	);
 
+	nsize = vl;
+
 	/*
 	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
 	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
 	 * v8:wp2, v9:wq2, v10:wd2/w22, v11:w12
 	 * v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
 	 */
-	for (d = 0; d < bytes; d += NSIZE * 4) {
+	for (d = 0; d < bytes; d += nsize * 4) {
 		 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
 		asm volatile (".option	push\n"
 			      ".option	arch,+v\n"
@@ -575,10 +582,10 @@ static void raid6_rvv4_xor_syndrome_real(int disks, int start, int stop,
 			      "vmv.v.v	v13, v12\n"
 			      ".option	pop\n"
 			      : :
-			      [wp0]"r"(&dptr[z0][d + 0 * NSIZE]),
-			      [wp1]"r"(&dptr[z0][d + 1 * NSIZE]),
-			      [wp2]"r"(&dptr[z0][d + 2 * NSIZE]),
-			      [wp3]"r"(&dptr[z0][d + 3 * NSIZE])
+			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
+			      [wp1]"r"(&dptr[z0][d + 1 * nsize]),
+			      [wp2]"r"(&dptr[z0][d + 2 * nsize]),
+			      [wp3]"r"(&dptr[z0][d + 3 * nsize])
 		);
 
 		/* P/Q data pages */
@@ -627,10 +634,10 @@ static void raid6_rvv4_xor_syndrome_real(int disks, int start, int stop,
 				      "vxor.vv	v12, v12, v14\n"
 				      ".option	pop\n"
 				      : :
-				      [wd0]"r"(&dptr[z][d + 0 * NSIZE]),
-				      [wd1]"r"(&dptr[z][d + 1 * NSIZE]),
-				      [wd2]"r"(&dptr[z][d + 2 * NSIZE]),
-				      [wd3]"r"(&dptr[z][d + 3 * NSIZE]),
+				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
+				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
+				      [wd2]"r"(&dptr[z][d + 2 * nsize]),
+				      [wd3]"r"(&dptr[z][d + 3 * nsize]),
 				      [x1d]"r"(0x1d)
 			);
 		}
@@ -709,14 +716,14 @@ static void raid6_rvv4_xor_syndrome_real(int disks, int start, int stop,
 			      "vse8.v	v15, (%[wq3])\n"
 			      ".option	pop\n"
 			      : :
-			      [wp0]"r"(&p[d + NSIZE * 0]),
-			      [wq0]"r"(&q[d + NSIZE * 0]),
-			      [wp1]"r"(&p[d + NSIZE * 1]),
-			      [wq1]"r"(&q[d + NSIZE * 1]),
-			      [wp2]"r"(&p[d + NSIZE * 2]),
-			      [wq2]"r"(&q[d + NSIZE * 2]),
-			      [wp3]"r"(&p[d + NSIZE * 3]),
-			      [wq3]"r"(&q[d + NSIZE * 3])
+			      [wp0]"r"(&p[d + nsize * 0]),
+			      [wq0]"r"(&q[d + nsize * 0]),
+			      [wp1]"r"(&p[d + nsize * 1]),
+			      [wq1]"r"(&q[d + nsize * 1]),
+			      [wp2]"r"(&p[d + nsize * 2]),
+			      [wq2]"r"(&q[d + nsize * 2]),
+			      [wp3]"r"(&p[d + nsize * 3]),
+			      [wq3]"r"(&q[d + nsize * 3])
 		);
 	}
 }
@@ -725,7 +732,7 @@ static void raid6_rvv8_gen_syndrome_real(int disks, unsigned long bytes, void **
 {
 	u8 **dptr = (u8 **)ptrs;
 	u8 *p, *q;
-	unsigned long vl, d;
+	unsigned long vl, d, nsize;
 	int z, z0;
 
 	z0 = disks - 3;	/* Highest data disk */
@@ -739,6 +746,8 @@ static void raid6_rvv8_gen_syndrome_real(int disks, unsigned long bytes, void **
 		      : "=&r" (vl)
 	);
 
+	nsize = vl;
+
 	/*
 	 * v0:wp0,   v1:wq0,  v2:wd0/w20,  v3:w10
 	 * v4:wp1,   v5:wq1,  v6:wd1/w21,  v7:w11
@@ -749,7 +758,7 @@ static void raid6_rvv8_gen_syndrome_real(int disks, unsigned long bytes, void **
 	 * v24:wp6, v25:wq6, v26:wd6/w26, v27:w16
 	 * v28:wp7, v29:wq7, v30:wd7/w27, v31:w17
 	 */
-	for (d = 0; d < bytes; d += NSIZE * 8) {
+	for (d = 0; d < bytes; d += nsize * 8) {
 		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
 		asm volatile (".option	push\n"
 			      ".option	arch,+v\n"
@@ -771,14 +780,14 @@ static void raid6_rvv8_gen_syndrome_real(int disks, unsigned long bytes, void **
 			      "vmv.v.v	v29, v28\n"
 			      ".option	pop\n"
 			      : :
-			      [wp0]"r"(&dptr[z0][d + 0 * NSIZE]),
-			      [wp1]"r"(&dptr[z0][d + 1 * NSIZE]),
-			      [wp2]"r"(&dptr[z0][d + 2 * NSIZE]),
-			      [wp3]"r"(&dptr[z0][d + 3 * NSIZE]),
-			      [wp4]"r"(&dptr[z0][d + 4 * NSIZE]),
-			      [wp5]"r"(&dptr[z0][d + 5 * NSIZE]),
-			      [wp6]"r"(&dptr[z0][d + 6 * NSIZE]),
-			      [wp7]"r"(&dptr[z0][d + 7 * NSIZE])
+			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
+			      [wp1]"r"(&dptr[z0][d + 1 * nsize]),
+			      [wp2]"r"(&dptr[z0][d + 2 * nsize]),
+			      [wp3]"r"(&dptr[z0][d + 3 * nsize]),
+			      [wp4]"r"(&dptr[z0][d + 4 * nsize]),
+			      [wp5]"r"(&dptr[z0][d + 5 * nsize]),
+			      [wp6]"r"(&dptr[z0][d + 6 * nsize]),
+			      [wp7]"r"(&dptr[z0][d + 7 * nsize])
 		);
 
 		for (z = z0 - 1; z >= 0; z--) {
@@ -858,14 +867,14 @@ static void raid6_rvv8_gen_syndrome_real(int disks, unsigned long bytes, void **
 				      "vxor.vv	v28, v28, v30\n"
 				      ".option	pop\n"
 				      : :
-				      [wd0]"r"(&dptr[z][d + 0 * NSIZE]),
-				      [wd1]"r"(&dptr[z][d + 1 * NSIZE]),
-				      [wd2]"r"(&dptr[z][d + 2 * NSIZE]),
-				      [wd3]"r"(&dptr[z][d + 3 * NSIZE]),
-				      [wd4]"r"(&dptr[z][d + 4 * NSIZE]),
-				      [wd5]"r"(&dptr[z][d + 5 * NSIZE]),
-				      [wd6]"r"(&dptr[z][d + 6 * NSIZE]),
-				      [wd7]"r"(&dptr[z][d + 7 * NSIZE]),
+				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
+				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
+				      [wd2]"r"(&dptr[z][d + 2 * nsize]),
+				      [wd3]"r"(&dptr[z][d + 3 * nsize]),
+				      [wd4]"r"(&dptr[z][d + 4 * nsize]),
+				      [wd5]"r"(&dptr[z][d + 5 * nsize]),
+				      [wd6]"r"(&dptr[z][d + 6 * nsize]),
+				      [wd7]"r"(&dptr[z][d + 7 * nsize]),
 				      [x1d]"r"(0x1d)
 			);
 		}
@@ -894,22 +903,22 @@ static void raid6_rvv8_gen_syndrome_real(int disks, unsigned long bytes, void **
 			      "vse8.v	v29, (%[wq7])\n"
 			      ".option	pop\n"
 			      : :
-			      [wp0]"r"(&p[d + NSIZE * 0]),
-			      [wq0]"r"(&q[d + NSIZE * 0]),
-			      [wp1]"r"(&p[d + NSIZE * 1]),
-			      [wq1]"r"(&q[d + NSIZE * 1]),
-			      [wp2]"r"(&p[d + NSIZE * 2]),
-			      [wq2]"r"(&q[d + NSIZE * 2]),
-			      [wp3]"r"(&p[d + NSIZE * 3]),
-			      [wq3]"r"(&q[d + NSIZE * 3]),
-			      [wp4]"r"(&p[d + NSIZE * 4]),
-			      [wq4]"r"(&q[d + NSIZE * 4]),
-			      [wp5]"r"(&p[d + NSIZE * 5]),
-			      [wq5]"r"(&q[d + NSIZE * 5]),
-			      [wp6]"r"(&p[d + NSIZE * 6]),
-			      [wq6]"r"(&q[d + NSIZE * 6]),
-			      [wp7]"r"(&p[d + NSIZE * 7]),
-			      [wq7]"r"(&q[d + NSIZE * 7])
+			      [wp0]"r"(&p[d + nsize * 0]),
+			      [wq0]"r"(&q[d + nsize * 0]),
+			      [wp1]"r"(&p[d + nsize * 1]),
+			      [wq1]"r"(&q[d + nsize * 1]),
+			      [wp2]"r"(&p[d + nsize * 2]),
+			      [wq2]"r"(&q[d + nsize * 2]),
+			      [wp3]"r"(&p[d + nsize * 3]),
+			      [wq3]"r"(&q[d + nsize * 3]),
+			      [wp4]"r"(&p[d + nsize * 4]),
+			      [wq4]"r"(&q[d + nsize * 4]),
+			      [wp5]"r"(&p[d + nsize * 5]),
+			      [wq5]"r"(&q[d + nsize * 5]),
+			      [wp6]"r"(&p[d + nsize * 6]),
+			      [wq6]"r"(&q[d + nsize * 6]),
+			      [wp7]"r"(&p[d + nsize * 7]),
+			      [wq7]"r"(&q[d + nsize * 7])
 		);
 	}
 }
@@ -919,7 +928,7 @@ static void raid6_rvv8_xor_syndrome_real(int disks, int start, int stop,
 {
 	u8 **dptr = (u8 **)ptrs;
 	u8 *p, *q;
-	unsigned long vl, d;
+	unsigned long vl, d, nsize;
 	int z, z0;
 
 	z0 = stop;		/* P/Q right side optimization */
@@ -933,6 +942,8 @@ static void raid6_rvv8_xor_syndrome_real(int disks, int start, int stop,
 		      : "=&r" (vl)
 	);
 
+	nsize = vl;
+
 	/*
 	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
 	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
@@ -943,7 +954,7 @@ static void raid6_rvv8_xor_syndrome_real(int disks, int start, int stop,
 	 * v24:wp6, v25:wq6, v26:wd6/w26, v27:w16
 	 * v28:wp7, v29:wq7, v30:wd7/w27, v31:w17
 	 */
-	for (d = 0; d < bytes; d += NSIZE * 8) {
+	for (d = 0; d < bytes; d += nsize * 8) {
 		 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
 		asm volatile (".option	push\n"
 			      ".option	arch,+v\n"
@@ -965,14 +976,14 @@ static void raid6_rvv8_xor_syndrome_real(int disks, int start, int stop,
 			      "vmv.v.v	v29, v28\n"
 			      ".option	pop\n"
 			      : :
-			      [wp0]"r"(&dptr[z0][d + 0 * NSIZE]),
-			      [wp1]"r"(&dptr[z0][d + 1 * NSIZE]),
-			      [wp2]"r"(&dptr[z0][d + 2 * NSIZE]),
-			      [wp3]"r"(&dptr[z0][d + 3 * NSIZE]),
-			      [wp4]"r"(&dptr[z0][d + 4 * NSIZE]),
-			      [wp5]"r"(&dptr[z0][d + 5 * NSIZE]),
-			      [wp6]"r"(&dptr[z0][d + 6 * NSIZE]),
-			      [wp7]"r"(&dptr[z0][d + 7 * NSIZE])
+			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
+			      [wp1]"r"(&dptr[z0][d + 1 * nsize]),
+			      [wp2]"r"(&dptr[z0][d + 2 * nsize]),
+			      [wp3]"r"(&dptr[z0][d + 3 * nsize]),
+			      [wp4]"r"(&dptr[z0][d + 4 * nsize]),
+			      [wp5]"r"(&dptr[z0][d + 5 * nsize]),
+			      [wp6]"r"(&dptr[z0][d + 6 * nsize]),
+			      [wp7]"r"(&dptr[z0][d + 7 * nsize])
 		);
 
 		/* P/Q data pages */
@@ -1053,14 +1064,14 @@ static void raid6_rvv8_xor_syndrome_real(int disks, int start, int stop,
 				      "vxor.vv	v28, v28, v30\n"
 				      ".option	pop\n"
 				      : :
-				      [wd0]"r"(&dptr[z][d + 0 * NSIZE]),
-				      [wd1]"r"(&dptr[z][d + 1 * NSIZE]),
-				      [wd2]"r"(&dptr[z][d + 2 * NSIZE]),
-				      [wd3]"r"(&dptr[z][d + 3 * NSIZE]),
-				      [wd4]"r"(&dptr[z][d + 4 * NSIZE]),
-				      [wd5]"r"(&dptr[z][d + 5 * NSIZE]),
-				      [wd6]"r"(&dptr[z][d + 6 * NSIZE]),
-				      [wd7]"r"(&dptr[z][d + 7 * NSIZE]),
+				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
+				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
+				      [wd2]"r"(&dptr[z][d + 2 * nsize]),
+				      [wd3]"r"(&dptr[z][d + 3 * nsize]),
+				      [wd4]"r"(&dptr[z][d + 4 * nsize]),
+				      [wd5]"r"(&dptr[z][d + 5 * nsize]),
+				      [wd6]"r"(&dptr[z][d + 6 * nsize]),
+				      [wd7]"r"(&dptr[z][d + 7 * nsize]),
 				      [x1d]"r"(0x1d)
 			);
 		}
@@ -1191,22 +1202,22 @@ static void raid6_rvv8_xor_syndrome_real(int disks, int start, int stop,
 			      "vse8.v	v31, (%[wq7])\n"
 			      ".option	pop\n"
 			      : :
-			      [wp0]"r"(&p[d + NSIZE * 0]),
-			      [wq0]"r"(&q[d + NSIZE * 0]),
-			      [wp1]"r"(&p[d + NSIZE * 1]),
-			      [wq1]"r"(&q[d + NSIZE * 1]),
-			      [wp2]"r"(&p[d + NSIZE * 2]),
-			      [wq2]"r"(&q[d + NSIZE * 2]),
-			      [wp3]"r"(&p[d + NSIZE * 3]),
-			      [wq3]"r"(&q[d + NSIZE * 3]),
-			      [wp4]"r"(&p[d + NSIZE * 4]),
-			      [wq4]"r"(&q[d + NSIZE * 4]),
-			      [wp5]"r"(&p[d + NSIZE * 5]),
-			      [wq5]"r"(&q[d + NSIZE * 5]),
-			      [wp6]"r"(&p[d + NSIZE * 6]),
-			      [wq6]"r"(&q[d + NSIZE * 6]),
-			      [wp7]"r"(&p[d + NSIZE * 7]),
-			      [wq7]"r"(&q[d + NSIZE * 7])
+			      [wp0]"r"(&p[d + nsize * 0]),
+			      [wq0]"r"(&q[d + nsize * 0]),
+			      [wp1]"r"(&p[d + nsize * 1]),
+			      [wq1]"r"(&q[d + nsize * 1]),
+			      [wp2]"r"(&p[d + nsize * 2]),
+			      [wq2]"r"(&q[d + nsize * 2]),
+			      [wp3]"r"(&p[d + nsize * 3]),
+			      [wq3]"r"(&q[d + nsize * 3]),
+			      [wp4]"r"(&p[d + nsize * 4]),
+			      [wq4]"r"(&q[d + nsize * 4]),
+			      [wp5]"r"(&p[d + nsize * 5]),
+			      [wq5]"r"(&q[d + nsize * 5]),
+			      [wp6]"r"(&p[d + nsize * 6]),
+			      [wq6]"r"(&q[d + nsize * 6]),
+			      [wp7]"r"(&p[d + nsize * 7]),
+			      [wq7]"r"(&q[d + nsize * 7])
 		);
 	}
 }
diff --git a/lib/raid6/rvv.h b/lib/raid6/rvv.h
index 94044a1b707b..6d0708a2c8a4 100644
--- a/lib/raid6/rvv.h
+++ b/lib/raid6/rvv.h
@@ -7,6 +7,23 @@
  * Definitions for RISC-V RAID-6 code
  */
 
+#ifdef __KERNEL__
+#include <asm/vector.h>
+#else
+#define kernel_vector_begin()
+#define kernel_vector_end()
+#include <sys/auxv.h>
+#include <asm/hwcap.h>
+#define has_vector() (getauxval(AT_HWCAP) & COMPAT_HWCAP_ISA_V)
+#endif
+
+#include <linux/raid/pq.h>
+
+static int rvv_has_vector(void)
+{
+	return has_vector();
+}
+
 #define RAID6_RVV_WRAPPER(_n)						\
 	static void raid6_rvv ## _n ## _gen_syndrome(int disks,		\
 					size_t bytes, void **ptrs)	\
diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile
index 8f2dd2210ba8..09bbe2b14cce 100644
--- a/lib/raid6/test/Makefile
+++ b/lib/raid6/test/Makefile
@@ -35,6 +35,11 @@ ifeq ($(ARCH),aarch64)
         HAS_NEON = yes
 endif
 
+ifeq ($(findstring riscv,$(ARCH)),riscv)
+        CFLAGS += -I../../../arch/riscv/include -DCONFIG_RISCV=1
+        HAS_RVV = yes
+endif
+
 ifeq ($(findstring ppc,$(ARCH)),ppc)
         CFLAGS += -I../../../arch/powerpc/include
         HAS_ALTIVEC := $(shell printf '$(pound)include <altivec.h>\nvector int a;\n' |\
@@ -63,6 +68,9 @@ else ifeq ($(HAS_ALTIVEC),yes)
                 vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o
 else ifeq ($(ARCH),loongarch64)
         OBJS += loongarch_simd.o recov_loongarch_simd.o
+else ifeq ($(HAS_RVV),yes)
+        OBJS   += rvv.o recov_rvv.o
+        CFLAGS += -DCONFIG_RISCV_ISA_V=1
 endif
 
 .c.o:
diff --git a/lib/ratelimit.c b/lib/ratelimit.c
index 859c251b23ce..e2d65d3b1c35 100644
--- a/lib/ratelimit.c
+++ b/lib/ratelimit.c
@@ -27,7 +27,7 @@
 int ___ratelimit(struct ratelimit_state *rs, const char *func)
 {
 	/* Paired with WRITE_ONCE() in .proc_handler().
-	 * Changing two values seperately could be inconsistent
+	 * Changing two values separately could be inconsistent
 	 * and some message could be lost.  (See: net_ratelimit_state).
 	 */
 	int interval = READ_ONCE(rs->interval);
diff --git a/lib/rbtree.c b/lib/rbtree.c
index 5114eda6309c..18d42bcf4ec9 100644
--- a/lib/rbtree.c
+++ b/lib/rbtree.c
@@ -460,35 +460,6 @@ void __rb_insert_augmented(struct rb_node *node, struct rb_root *root,
 }
 EXPORT_SYMBOL(__rb_insert_augmented);
 
-/*
- * This function returns the first node (in sort order) of the tree.
- */
-struct rb_node *rb_first(const struct rb_root *root)
-{
-	struct rb_node	*n;
-
-	n = root->rb_node;
-	if (!n)
-		return NULL;
-	while (n->rb_left)
-		n = n->rb_left;
-	return n;
-}
-EXPORT_SYMBOL(rb_first);
-
-struct rb_node *rb_last(const struct rb_root *root)
-{
-	struct rb_node	*n;
-
-	n = root->rb_node;
-	if (!n)
-		return NULL;
-	while (n->rb_right)
-		n = n->rb_right;
-	return n;
-}
-EXPORT_SYMBOL(rb_last);
-
 struct rb_node *rb_next(const struct rb_node *node)
 {
 	struct rb_node *parent;
diff --git a/lib/rbtree_test.c b/lib/rbtree_test.c
index 690cede46ac2..768c5e6453f3 100644
--- a/lib/rbtree_test.c
+++ b/lib/rbtree_test.c
@@ -399,7 +399,7 @@ static int augmented_check(void)
 
 static int __init rbtree_test_init(void)
 {
-	nodes = kmalloc_array(nnodes, sizeof(*nodes), GFP_KERNEL);
+	nodes = kmalloc_objs(*nodes, nnodes);
 	if (!nodes)
 		return -ENOMEM;
 
diff --git a/lib/reed_solomon/decode_rs.c b/lib/reed_solomon/decode_rs.c
index 805de84ae83d..ef86ee2aec58 100644
--- a/lib/reed_solomon/decode_rs.c
+++ b/lib/reed_solomon/decode_rs.c
@@ -5,7 +5,7 @@
  * Copyright 2002, Phil Karn, KA9Q
  * May be used under the terms of the GNU General Public License (GPL)
  *
- * Adaption to the kernel by Thomas Gleixner (tglx@linutronix.de)
+ * Adaption to the kernel by Thomas Gleixner (tglx@kernel.org)
  *
  * Generic data width independent code which is included by the wrappers.
  */
diff --git a/lib/reed_solomon/encode_rs.c b/lib/reed_solomon/encode_rs.c
index 9112d46e869e..1d9e51dcc83d 100644
--- a/lib/reed_solomon/encode_rs.c
+++ b/lib/reed_solomon/encode_rs.c
@@ -5,7 +5,7 @@
  * Copyright 2002, Phil Karn, KA9Q
  * May be used under the terms of the GNU General Public License (GPL)
  *
- * Adaption to the kernel by Thomas Gleixner (tglx@linutronix.de)
+ * Adaption to the kernel by Thomas Gleixner (tglx@kernel.org)
  *
  * Generic data width independent code which is included by the wrappers.
  */
diff --git a/lib/reed_solomon/reed_solomon.c b/lib/reed_solomon/reed_solomon.c
index bbc01bad3053..864484c01827 100644
--- a/lib/reed_solomon/reed_solomon.c
+++ b/lib/reed_solomon/reed_solomon.c
@@ -2,7 +2,7 @@
 /*
  * Generic Reed Solomon encoder / decoder library
  *
- * Copyright (C) 2004 Thomas Gleixner (tglx@linutronix.de)
+ * Copyright (C) 2004 Thomas Gleixner (tglx@kernel.org)
  *
  * Reed Solomon code lifted from reed solomon library written by Phil Karn
  * Copyright 2002 Phil Karn, KA9Q
@@ -73,7 +73,7 @@ static struct rs_codec *codec_init(int symsize, int gfpoly, int (*gffunc)(int),
 	int i, j, sr, root, iprim;
 	struct rs_codec *rs;
 
-	rs = kzalloc(sizeof(*rs), gfp);
+	rs = kzalloc_obj(*rs, gfp);
 	if (!rs)
 		return NULL;
 
diff --git a/lib/reed_solomon/test_rslib.c b/lib/reed_solomon/test_rslib.c
index 75cb1adac884..42b856e6c8a8 100644
--- a/lib/reed_solomon/test_rslib.c
+++ b/lib/reed_solomon/test_rslib.c
@@ -111,7 +111,7 @@ static struct wspace *alloc_ws(struct rs_codec *rs)
 	struct wspace *ws;
 	int nn = rs->nn;
 
-	ws = kzalloc(sizeof(*ws), GFP_KERNEL);
+	ws = kzalloc_obj(*ws);
 	if (!ws)
 		return NULL;
 
@@ -124,7 +124,7 @@ static struct wspace *alloc_ws(struct rs_codec *rs)
 	ws->s = ws->r + nn;
 	ws->corr = ws->s + nroots;
 
-	ws->errlocs = kmalloc_array(nn + nroots, sizeof(int), GFP_KERNEL);
+	ws->errlocs = kmalloc_objs(int, nn + nroots);
 	if (!ws->errlocs)
 		goto err;
 
diff --git a/lib/ref_tracker.c b/lib/ref_tracker.c
index 258fb0e7abdf..30c999d57b10 100644
--- a/lib/ref_tracker.c
+++ b/lib/ref_tracker.c
@@ -74,8 +74,7 @@ ref_tracker_get_stats(struct ref_tracker_dir *dir, unsigned int limit)
 	struct ref_tracker_dir_stats *stats;
 	struct ref_tracker *tracker;
 
-	stats = kmalloc(struct_size(stats, stacks, limit),
-			GFP_NOWAIT);
+	stats = kmalloc_flex(*stats, stacks, limit, GFP_NOWAIT);
 	if (!stats)
 		return ERR_PTR(-ENOMEM);
 	stats->total = 0;
@@ -268,7 +267,7 @@ int ref_tracker_alloc(struct ref_tracker_dir *dir,
 	}
 	if (gfp & __GFP_DIRECT_RECLAIM)
 		gfp_mask |= __GFP_NOFAIL;
-	*trackerp = tracker = kzalloc(sizeof(*tracker), gfp_mask);
+	*trackerp = tracker = kzalloc_obj(*tracker, gfp_mask);
 	if (unlikely(!tracker)) {
 		pr_err_once("memory allocation failure, unreliable refcount tracker.\n");
 		refcount_inc(&dir->untracked);
diff --git a/lib/rhashtable.c b/lib/rhashtable.c
index fde0f0e556f8..6074ed5f66f3 100644
--- a/lib/rhashtable.c
+++ b/lib/rhashtable.c
@@ -358,6 +358,7 @@ static int rhashtable_rehash_table(struct rhashtable *ht)
 static int rhashtable_rehash_alloc(struct rhashtable *ht,
 				   struct bucket_table *old_tbl,
 				   unsigned int size)
+	__must_hold(&ht->mutex)
 {
 	struct bucket_table *new_tbl;
 	int err;
@@ -392,6 +393,7 @@ static int rhashtable_rehash_alloc(struct rhashtable *ht,
  * bucket locks or concurrent RCU protected lookups and traversals.
  */
 static int rhashtable_shrink(struct rhashtable *ht)
+	__must_hold(&ht->mutex)
 {
 	struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);
 	unsigned int nelems = atomic_read(&ht->nelems);
@@ -724,7 +726,7 @@ EXPORT_SYMBOL_GPL(rhashtable_walk_exit);
  * resize events and always continue.
  */
 int rhashtable_walk_start_check(struct rhashtable_iter *iter)
-	__acquires(RCU)
+	__acquires_shared(RCU)
 {
 	struct rhashtable *ht = iter->ht;
 	bool rhlist = ht->rhlist;
@@ -940,7 +942,6 @@ EXPORT_SYMBOL_GPL(rhashtable_walk_peek);
  * hash table.
  */
 void rhashtable_walk_stop(struct rhashtable_iter *iter)
-	__releases(RCU)
 {
 	struct rhashtable *ht;
 	struct bucket_table *tbl = iter->walker.tbl;
diff --git a/lib/scatterlist.c b/lib/scatterlist.c
index 4af1c8b0775a..d773720d11bf 100644
--- a/lib/scatterlist.c
+++ b/lib/scatterlist.c
@@ -65,6 +65,32 @@ int sg_nents_for_len(struct scatterlist *sg, u64 len)
 EXPORT_SYMBOL(sg_nents_for_len);
 
 /**
+ * sg_nents_for_dma - return the count of DMA-capable entries in scatterlist
+ * @sgl:	The scatterlist
+ * @sglen:	The current number of entries
+ * @len:	The maximum length of DMA-capable block
+ *
+ * Description:
+ * Determines the number of entries in @sgl which would be permitted in
+ * DMA-capable transfer if list had been split accordingly, taking into
+ * account chaining as well.
+ *
+ * Returns:
+ *   the number of sgl entries needed
+ *
+ **/
+int sg_nents_for_dma(struct scatterlist *sgl, unsigned int sglen, size_t len)
+{
+	struct scatterlist *sg;
+	int i, nents = 0;
+
+	for_each_sg(sgl, sg, sglen, i)
+		nents += DIV_ROUND_UP(sg_dma_len(sg), len);
+	return nents;
+}
+EXPORT_SYMBOL(sg_nents_for_dma);
+
+/**
  * sg_last - return the last scatterlist entry in a list
  * @sgl:	First entry in the scatterlist
  * @nents:	Number of entries in the scatterlist
@@ -142,8 +168,7 @@ static struct scatterlist *sg_kmalloc(unsigned int nents, gfp_t gfp_mask)
 		kmemleak_alloc(ptr, PAGE_SIZE, 1, gfp_mask);
 		return ptr;
 	} else
-		return kmalloc_array(nents, sizeof(struct scatterlist),
-				     gfp_mask);
+		return kmalloc_objs(struct scatterlist, nents, gfp_mask);
 }
 
 static void sg_kfree(struct scatterlist *sg, unsigned int nents)
@@ -606,8 +631,7 @@ struct scatterlist *sgl_alloc_order(unsigned long long length,
 			return NULL;
 		nalloc++;
 	}
-	sgl = kmalloc_array(nalloc, sizeof(struct scatterlist),
-			    gfp & ~GFP_DMA);
+	sgl = kmalloc_objs(struct scatterlist, nalloc, gfp & ~GFP_DMA);
 	if (!sgl)
 		return NULL;
 
diff --git a/lib/sg_split.c b/lib/sg_split.c
index 0f89aab5c671..24e8f5e48e63 100644
--- a/lib/sg_split.c
+++ b/lib/sg_split.c
@@ -152,7 +152,7 @@ int sg_split(struct scatterlist *in, const int in_mapped_nents,
 	int i, ret;
 	struct sg_splitter *splitters;
 
-	splitters = kcalloc(nb_splits, sizeof(*splitters), gfp_mask);
+	splitters = kzalloc_objs(*splitters, nb_splits, gfp_mask);
 	if (!splitters)
 		return -ENOMEM;
 
@@ -163,9 +163,8 @@ int sg_split(struct scatterlist *in, const int in_mapped_nents,
 
 	ret = -ENOMEM;
 	for (i = 0; i < nb_splits; i++) {
-		splitters[i].out_sg = kmalloc_array(splitters[i].nents,
-						    sizeof(struct scatterlist),
-						    gfp_mask);
+		splitters[i].out_sg = kmalloc_objs(struct scatterlist,
+						   splitters[i].nents, gfp_mask);
 		if (!splitters[i].out_sg)
 			goto err;
 	}
diff --git a/lib/stackdepot.c b/lib/stackdepot.c
index de0b0025af2b..dd2717ff94bf 100644
--- a/lib/stackdepot.c
+++ b/lib/stackdepot.c
@@ -61,18 +61,18 @@ static unsigned int stack_bucket_number_order;
 /* Hash mask for indexing the table. */
 static unsigned int stack_hash_mask;
 
+/* The lock must be held when performing pool or freelist modifications. */
+static DEFINE_RAW_SPINLOCK(pool_lock);
 /* Array of memory regions that store stack records. */
-static void **stack_pools;
+static void **stack_pools __pt_guarded_by(&pool_lock);
 /* Newly allocated pool that is not yet added to stack_pools. */
 static void *new_pool;
 /* Number of pools in stack_pools. */
 static int pools_num;
 /* Offset to the unused space in the currently used pool. */
-static size_t pool_offset = DEPOT_POOL_SIZE;
+static size_t pool_offset __guarded_by(&pool_lock) = DEPOT_POOL_SIZE;
 /* Freelist of stack records within stack_pools. */
-static LIST_HEAD(free_stacks);
-/* The lock must be held when performing pool or freelist modifications. */
-static DEFINE_RAW_SPINLOCK(pool_lock);
+static __guarded_by(&pool_lock) LIST_HEAD(free_stacks);
 
 /* Statistics counters for debugfs. */
 enum depot_counter_id {
@@ -260,7 +260,7 @@ int stack_depot_init(void)
 		entries = 1UL << STACK_BUCKET_NUMBER_ORDER_MAX;
 
 	pr_info("allocating hash table of %lu entries via kvcalloc\n", entries);
-	stack_table = kvcalloc(entries, sizeof(struct list_head), GFP_KERNEL);
+	stack_table = kvzalloc_objs(struct list_head, entries);
 	if (!stack_table) {
 		pr_err("hash table allocation failed, disabling\n");
 		stack_depot_disabled = true;
@@ -291,6 +291,7 @@ EXPORT_SYMBOL_GPL(stack_depot_init);
  * Initializes new stack pool, and updates the list of pools.
  */
 static bool depot_init_pool(void **prealloc)
+	__must_hold(&pool_lock)
 {
 	lockdep_assert_held(&pool_lock);
 
@@ -338,6 +339,7 @@ static bool depot_init_pool(void **prealloc)
 
 /* Keeps the preallocated memory to be used for a new stack depot pool. */
 static void depot_keep_new_pool(void **prealloc)
+	__must_hold(&pool_lock)
 {
 	lockdep_assert_held(&pool_lock);
 
@@ -357,6 +359,7 @@ static void depot_keep_new_pool(void **prealloc)
  * the current pre-allocation.
  */
 static struct stack_record *depot_pop_free_pool(void **prealloc, size_t size)
+	__must_hold(&pool_lock)
 {
 	struct stack_record *stack;
 	void *current_pool;
@@ -391,6 +394,7 @@ static struct stack_record *depot_pop_free_pool(void **prealloc, size_t size)
 
 /* Try to find next free usable entry from the freelist. */
 static struct stack_record *depot_pop_free(void)
+	__must_hold(&pool_lock)
 {
 	struct stack_record *stack;
 
@@ -428,6 +432,7 @@ static inline size_t depot_stack_record_size(struct stack_record *s, unsigned in
 /* Allocates a new stack in a stack depot pool. */
 static struct stack_record *
 depot_alloc_stack(unsigned long *entries, unsigned int nr_entries, u32 hash, depot_flags_t flags, void **prealloc)
+	__must_hold(&pool_lock)
 {
 	struct stack_record *stack = NULL;
 	size_t record_size;
@@ -486,6 +491,7 @@ depot_alloc_stack(unsigned long *entries, unsigned int nr_entries, u32 hash, dep
 }
 
 static struct stack_record *depot_fetch_stack(depot_stack_handle_t handle)
+	__must_not_hold(&pool_lock)
 {
 	const int pools_num_cached = READ_ONCE(pools_num);
 	union handle_parts parts = { .handle = handle };
@@ -502,7 +508,8 @@ static struct stack_record *depot_fetch_stack(depot_stack_handle_t handle)
 		return NULL;
 	}
 
-	pool = stack_pools[pool_index];
+	/* @pool_index either valid, or user passed in corrupted value. */
+	pool = context_unsafe(stack_pools[pool_index]);
 	if (WARN_ON(!pool))
 		return NULL;
 
@@ -515,6 +522,7 @@ static struct stack_record *depot_fetch_stack(depot_stack_handle_t handle)
 
 /* Links stack into the freelist. */
 static void depot_free_stack(struct stack_record *stack)
+	__must_not_hold(&pool_lock)
 {
 	unsigned long flags;
 
diff --git a/lib/string_helpers.c b/lib/string_helpers.c
index ffb8ead6d4cd..169eaf583494 100644
--- a/lib/string_helpers.c
+++ b/lib/string_helpers.c
@@ -13,6 +13,7 @@
 #include <linux/device.h>
 #include <linux/errno.h>
 #include <linux/fs.h>
+#include <linux/hex.h>
 #include <linux/limits.h>
 #include <linux/mm.h>
 #include <linux/slab.h>
@@ -146,7 +147,7 @@ int parse_int_array(const char *buf, size_t count, int **array)
 	if (!nints)
 		return -ENOENT;
 
-	ints = kcalloc(nints + 1, sizeof(*ints), GFP_KERNEL);
+	ints = kzalloc_objs(*ints, nints + 1);
 	if (!ints)
 		return -ENOMEM;
 
diff --git a/lib/strncpy_from_user.c b/lib/strncpy_from_user.c
index 6dc234913dd5..5bb752ff7c61 100644
--- a/lib/strncpy_from_user.c
+++ b/lib/strncpy_from_user.c
@@ -126,7 +126,7 @@ long strncpy_from_user(char *dst, const char __user *src, long count)
 	if (can_do_masked_user_access()) {
 		long retval;
 
-		src = masked_user_access_begin(src);
+		src = masked_user_read_access_begin(src);
 		retval = do_strncpy_from_user(dst, src, count, count);
 		user_read_access_end();
 		return retval;
diff --git a/lib/strnlen_user.c b/lib/strnlen_user.c
index 6e489f9e90f1..4a6574b67f82 100644
--- a/lib/strnlen_user.c
+++ b/lib/strnlen_user.c
@@ -99,7 +99,7 @@ long strnlen_user(const char __user *str, long count)
 	if (can_do_masked_user_access()) {
 		long retval;
 
-		str = masked_user_access_begin(str);
+		str = masked_user_read_access_begin(str);
 		retval = do_strnlen_user(str, count, count);
 		user_read_access_end();
 		return retval;
diff --git a/lib/sys_info.c b/lib/sys_info.c
index 496f9151c9b6..f32a06ec9ed4 100644
--- a/lib/sys_info.c
+++ b/lib/sys_info.c
@@ -1,31 +1,35 @@
 // SPDX-License-Identifier: GPL-2.0-only
-#include <linux/sched/debug.h>
+#include <linux/array_size.h>
+#include <linux/bitops.h>
+#include <linux/cleanup.h>
 #include <linux/console.h>
+#include <linux/log2.h>
 #include <linux/kernel.h>
 #include <linux/ftrace.h>
-#include <linux/sysctl.h>
 #include <linux/nmi.h>
+#include <linux/sched/debug.h>
+#include <linux/string.h>
+#include <linux/sysctl.h>
 
 #include <linux/sys_info.h>
 
-struct sys_info_name {
-	unsigned long bit;
-	const char *name;
+static const char * const si_names[] = {
+	[ilog2(SYS_INFO_TASKS)]			= "tasks",
+	[ilog2(SYS_INFO_MEM)]			= "mem",
+	[ilog2(SYS_INFO_TIMERS)]		= "timers",
+	[ilog2(SYS_INFO_LOCKS)]			= "locks",
+	[ilog2(SYS_INFO_FTRACE)]		= "ftrace",
+	[ilog2(SYS_INFO_PANIC_CONSOLE_REPLAY)]	= "",
+	[ilog2(SYS_INFO_ALL_BT)]		= "all_bt",
+	[ilog2(SYS_INFO_BLOCKED_TASKS)]		= "blocked_tasks",
 };
 
 /*
- * When 'si_names' gets updated,  please make sure the 'sys_info_avail'
- * below is updated accordingly.
+ * Default kernel sys_info mask.
+ * If a kernel module calls sys_info() with "parameter == 0", then
+ * this mask will be used.
  */
-static const struct sys_info_name  si_names[] = {
-	{ SYS_INFO_TASKS,		"tasks" },
-	{ SYS_INFO_MEM,			"mem" },
-	{ SYS_INFO_TIMERS,		"timers" },
-	{ SYS_INFO_LOCKS,		"locks" },
-	{ SYS_INFO_FTRACE,		"ftrace" },
-	{ SYS_INFO_ALL_CPU_BT,		"all_bt" },
-	{ SYS_INFO_BLOCKED_TASKS,	"blocked_tasks" },
-};
+static unsigned long kernel_si_mask;
 
 /* Expecting string like "xxx_sys_info=tasks,mem,timers,locks,ftrace,..." */
 unsigned long sys_info_parse_param(char *str)
@@ -36,12 +40,9 @@ unsigned long sys_info_parse_param(char *str)
 
 	s = str;
 	while ((name = strsep(&s, ",")) && *name) {
-		for (i = 0; i < ARRAY_SIZE(si_names); i++) {
-			if (!strcmp(name, si_names[i].name)) {
-				si_bits |= si_names[i].bit;
-				break;
-			}
-		}
+		i = match_string(si_names, ARRAY_SIZE(si_names), name);
+		if (i >= 0)
+			__set_bit(i, &si_bits);
 	}
 
 	return si_bits;
@@ -49,56 +50,93 @@ unsigned long sys_info_parse_param(char *str)
 
 #ifdef CONFIG_SYSCTL
 
-static const char sys_info_avail[] __maybe_unused = "tasks,mem,timers,locks,ftrace,all_bt,blocked_tasks";
+static int sys_info_write_handler(const struct ctl_table *table,
+				  void *buffer, size_t *lenp, loff_t *ppos,
+				  unsigned long *si_bits_global)
+{
+	unsigned long si_bits;
+	int ret;
+
+	ret = proc_dostring(table, 1, buffer, lenp, ppos);
+	if (ret)
+		return ret;
+
+	si_bits = sys_info_parse_param(table->data);
+
+	/* The access to the global value is not synchronized. */
+	WRITE_ONCE(*si_bits_global, si_bits);
+
+	return 0;
+}
+
+static int sys_info_read_handler(const struct ctl_table *table,
+				 void *buffer, size_t *lenp, loff_t *ppos,
+				 unsigned long *si_bits_global)
+{
+	unsigned long si_bits;
+	unsigned int len = 0;
+	char *delim = "";
+	unsigned int i;
+
+	/* The access to the global value is not synchronized. */
+	si_bits = READ_ONCE(*si_bits_global);
+
+	for_each_set_bit(i, &si_bits, ARRAY_SIZE(si_names)) {
+		if (*si_names[i]) {
+			len += scnprintf(table->data + len, table->maxlen - len,
+					 "%s%s", delim, si_names[i]);
+			delim = ",";
+		}
+	}
+
+	return proc_dostring(table, 0, buffer, lenp, ppos);
+}
 
 int sysctl_sys_info_handler(const struct ctl_table *ro_table, int write,
 					  void *buffer, size_t *lenp,
 					  loff_t *ppos)
 {
-	char names[sizeof(sys_info_avail)];
 	struct ctl_table table;
-	unsigned long *si_bits_global;
-
-	si_bits_global = ro_table->data;
-
-	if (write) {
-		unsigned long si_bits;
-		int ret;
-
-		table = *ro_table;
-		table.data = names;
-		table.maxlen = sizeof(names);
-		ret = proc_dostring(&table, write, buffer, lenp, ppos);
-		if (ret)
-			return ret;
-
-		si_bits = sys_info_parse_param(names);
-		/* The access to the global value is not synchronized. */
-		WRITE_ONCE(*si_bits_global, si_bits);
-		return 0;
-	} else {
-		/* for 'read' operation */
-		char *delim = "";
-		int i, len = 0;
-
-		names[0] = '\0';
-		for (i = 0; i < ARRAY_SIZE(si_names); i++) {
-			if (*si_bits_global & si_names[i].bit) {
-				len += scnprintf(names + len, sizeof(names) - len,
-					"%s%s", delim, si_names[i].name);
-				delim = ",";
-			}
-		}
+	unsigned int i;
+	size_t maxlen;
 
-		table = *ro_table;
-		table.data = names;
-		table.maxlen = sizeof(names);
-		return proc_dostring(&table, write, buffer, lenp, ppos);
-	}
+	maxlen = 0;
+	for (i = 0; i < ARRAY_SIZE(si_names); i++)
+		maxlen += strlen(si_names[i]) + 1;
+
+	char *names __free(kfree) = kzalloc(maxlen, GFP_KERNEL);
+	if (!names)
+		return -ENOMEM;
+
+	table = *ro_table;
+	table.data = names;
+	table.maxlen = maxlen;
+
+	if (write)
+		return sys_info_write_handler(&table, buffer, lenp, ppos, ro_table->data);
+	else
+		return sys_info_read_handler(&table, buffer, lenp, ppos, ro_table->data);
+}
+
+static const struct ctl_table sys_info_sysctls[] = {
+	{
+		.procname	= "kernel_sys_info",
+		.data		= &kernel_si_mask,
+		.maxlen         = sizeof(kernel_si_mask),
+		.mode		= 0644,
+		.proc_handler	= sysctl_sys_info_handler,
+	},
+};
+
+static int __init sys_info_sysctl_init(void)
+{
+	register_sysctl_init("kernel", sys_info_sysctls);
+	return 0;
 }
+subsys_initcall(sys_info_sysctl_init);
 #endif
 
-void sys_info(unsigned long si_mask)
+static void __sys_info(unsigned long si_mask)
 {
 	if (si_mask & SYS_INFO_TASKS)
 		show_state();
@@ -115,9 +153,14 @@ void sys_info(unsigned long si_mask)
 	if (si_mask & SYS_INFO_FTRACE)
 		ftrace_dump(DUMP_ALL);
 
-	if (si_mask & SYS_INFO_ALL_CPU_BT)
+	if (si_mask & SYS_INFO_ALL_BT)
 		trigger_all_cpu_backtrace();
 
 	if (si_mask & SYS_INFO_BLOCKED_TASKS)
 		show_state_filter(TASK_UNINTERRUPTIBLE);
 }
+
+void sys_info(unsigned long si_mask)
+{
+	__sys_info(si_mask ? : kernel_si_mask);
+}
diff --git a/lib/test_bpf.c b/lib/test_bpf.c
index af0041df2b72..5892c0f17ddc 100644
--- a/lib/test_bpf.c
+++ b/lib/test_bpf.c
@@ -94,7 +94,7 @@ static int bpf_fill_maxinsns1(struct bpf_test *self)
 	__u32 k = ~0;
 	int i;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -113,7 +113,7 @@ static int bpf_fill_maxinsns2(struct bpf_test *self)
 	struct sock_filter *insn;
 	int i;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -133,7 +133,7 @@ static int bpf_fill_maxinsns3(struct bpf_test *self)
 	struct rnd_state rnd;
 	int i;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -159,7 +159,7 @@ static int bpf_fill_maxinsns4(struct bpf_test *self)
 	struct sock_filter *insn;
 	int i;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -178,7 +178,7 @@ static int bpf_fill_maxinsns5(struct bpf_test *self)
 	struct sock_filter *insn;
 	int i;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -201,7 +201,7 @@ static int bpf_fill_maxinsns6(struct bpf_test *self)
 	struct sock_filter *insn;
 	int i;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -223,7 +223,7 @@ static int bpf_fill_maxinsns7(struct bpf_test *self)
 	struct sock_filter *insn;
 	int i;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -249,7 +249,7 @@ static int bpf_fill_maxinsns8(struct bpf_test *self)
 	struct sock_filter *insn;
 	int i, jmp_off = len - 3;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -272,7 +272,7 @@ static int bpf_fill_maxinsns9(struct bpf_test *self)
 	struct bpf_insn *insn;
 	int i;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -298,7 +298,7 @@ static int bpf_fill_maxinsns10(struct bpf_test *self)
 	struct bpf_insn *insn;
 	int i;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -324,7 +324,7 @@ static int __bpf_fill_ja(struct bpf_test *self, unsigned int len,
 	unsigned int rlen;
 	int i, j;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -358,7 +358,7 @@ static int bpf_fill_maxinsns12(struct bpf_test *self)
 	struct sock_filter *insn;
 	int i = 0;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -381,7 +381,7 @@ static int bpf_fill_maxinsns13(struct bpf_test *self)
 	struct sock_filter *insn;
 	int i = 0;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -410,7 +410,7 @@ static int bpf_fill_ld_abs_get_processor_id(struct bpf_test *self)
 	struct sock_filter *insn;
 	int i;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -434,7 +434,7 @@ static int __bpf_fill_stxdw(struct bpf_test *self, int size)
 	struct bpf_insn *insn;
 	int i;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -484,7 +484,7 @@ static int __bpf_fill_max_jmp(struct bpf_test *self, int jmp, int imm, bool alu3
 	int len = S16_MAX + 5;
 	int i;
 
-	insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL);
+	insns = kmalloc_objs(*insns, len);
 	if (!insns)
 		return -ENOMEM;
 
@@ -626,7 +626,7 @@ static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op,
 	int imm, k;
 	int i = 0;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -759,7 +759,7 @@ static int __bpf_fill_alu_shift_same_reg(struct bpf_test *self, u8 op,
 	int i = 0;
 	u64 val;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -1244,7 +1244,7 @@ static int __bpf_fill_alu_imm_regs(struct bpf_test *self, u8 op, bool alu32)
 	u32 imm;
 	int rd;
 
-	insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL);
+	insns = kmalloc_objs(*insns, len);
 	if (!insns)
 		return -ENOMEM;
 
@@ -1426,7 +1426,7 @@ static int __bpf_fill_alu_reg_pairs(struct bpf_test *self, u8 op, bool alu32)
 	int rd, rs;
 	int i = 0;
 
-	insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL);
+	insns = kmalloc_objs(*insns, len);
 	if (!insns)
 		return -ENOMEM;
 
@@ -1917,7 +1917,7 @@ static int __bpf_fill_atomic_reg_pairs(struct bpf_test *self, u8 width, u8 op)
 	u64 mem, upd, res;
 	int rd, rs, i = 0;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -2163,7 +2163,7 @@ static int bpf_fill_ld_imm64_magn(struct bpf_test *self)
 	int bit, adj, sign;
 	int i = 0;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -2217,7 +2217,7 @@ static int __bpf_fill_ld_imm64_bytes(struct bpf_test *self,
 	u32 rand = 1;
 	int i = 0;
 
-	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+	insn = kmalloc_objs(*insn, len);
 	if (!insn)
 		return -ENOMEM;
 
@@ -2724,7 +2724,7 @@ static int __bpf_fill_staggered_jumps(struct bpf_test *self,
 	struct bpf_insn *insns;
 	int off, ind;
 
-	insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL);
+	insns = kmalloc_objs(*insns, len);
 	if (!insns)
 		return -ENOMEM;
 
@@ -15461,7 +15461,7 @@ static __init int prepare_tail_call_tests(struct bpf_array **pprogs)
 	int which, err;
 
 	/* Allocate the table of programs to be used for tail calls */
-	progs = kzalloc(struct_size(progs, ptrs, ntests + 1), GFP_KERNEL);
+	progs = kzalloc_flex(*progs, ptrs, ntests + 1);
 	if (!progs)
 		goto out_nomem;
 
diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c
new file mode 100644
index 000000000000..140efa8a9763
--- /dev/null
+++ b/lib/test_context-analysis.c
@@ -0,0 +1,598 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Compile-only tests for common patterns that should not generate false
+ * positive errors when compiled with Clang's context analysis.
+ */
+
+#include <linux/bit_spinlock.h>
+#include <linux/build_bug.h>
+#include <linux/local_lock.h>
+#include <linux/mutex.h>
+#include <linux/percpu.h>
+#include <linux/rcupdate.h>
+#include <linux/rwsem.h>
+#include <linux/seqlock.h>
+#include <linux/spinlock.h>
+#include <linux/srcu.h>
+#include <linux/ww_mutex.h>
+
+/*
+ * Test that helper macros work as expected.
+ */
+static void __used test_common_helpers(void)
+{
+	BUILD_BUG_ON(context_unsafe(3) != 3); /* plain expression */
+	BUILD_BUG_ON(context_unsafe((void)2; 3) != 3); /* does not swallow semi-colon */
+	BUILD_BUG_ON(context_unsafe((void)2, 3) != 3); /* does not swallow commas */
+	context_unsafe(do { } while (0)); /* works with void statements */
+}
+
+#define TEST_SPINLOCK_COMMON(class, type, type_init, type_lock, type_unlock, type_trylock, op)	\
+	struct test_##class##_data {								\
+		type lock;									\
+		int counter __guarded_by(&lock);						\
+		int *pointer __pt_guarded_by(&lock);						\
+	};											\
+	static void __used test_##class##_init(struct test_##class##_data *d)			\
+	{											\
+		guard(type_init)(&d->lock);							\
+		d->counter = 0;									\
+	}											\
+	static void __used test_##class(struct test_##class##_data *d)				\
+	{											\
+		unsigned long flags;								\
+		d->pointer++;									\
+		type_lock(&d->lock);								\
+		op(d->counter);									\
+		op(*d->pointer);								\
+		type_unlock(&d->lock);								\
+		type_lock##_irq(&d->lock);							\
+		op(d->counter);									\
+		op(*d->pointer);								\
+		type_unlock##_irq(&d->lock);							\
+		type_lock##_bh(&d->lock);							\
+		op(d->counter);									\
+		op(*d->pointer);								\
+		type_unlock##_bh(&d->lock);							\
+		type_lock##_irqsave(&d->lock, flags);						\
+		op(d->counter);									\
+		op(*d->pointer);								\
+		type_unlock##_irqrestore(&d->lock, flags);					\
+	}											\
+	static void __used test_##class##_trylock(struct test_##class##_data *d)		\
+	{											\
+		if (type_trylock(&d->lock)) {							\
+			op(d->counter);								\
+			type_unlock(&d->lock);							\
+		}										\
+	}											\
+	static void __used test_##class##_assert(struct test_##class##_data *d)			\
+	{											\
+		lockdep_assert_held(&d->lock);							\
+		op(d->counter);									\
+	}											\
+	static void __used test_##class##_guard(struct test_##class##_data *d)			\
+	{											\
+		{ guard(class)(&d->lock);		op(d->counter); }			\
+		{ guard(class##_irq)(&d->lock);		op(d->counter); }			\
+		{ guard(class##_irqsave)(&d->lock);	op(d->counter); }			\
+	}
+
+#define TEST_OP_RW(x) (x)++
+#define TEST_OP_RO(x) ((void)(x))
+
+TEST_SPINLOCK_COMMON(raw_spinlock,
+		     raw_spinlock_t,
+		     raw_spinlock_init,
+		     raw_spin_lock,
+		     raw_spin_unlock,
+		     raw_spin_trylock,
+		     TEST_OP_RW);
+static void __used test_raw_spinlock_trylock_extra(struct test_raw_spinlock_data *d)
+{
+	unsigned long flags;
+
+	data_race(d->counter++); /* no warning */
+
+	if (raw_spin_trylock_irq(&d->lock)) {
+		d->counter++;
+		raw_spin_unlock_irq(&d->lock);
+	}
+	if (raw_spin_trylock_irqsave(&d->lock, flags)) {
+		d->counter++;
+		raw_spin_unlock_irqrestore(&d->lock, flags);
+	}
+	scoped_cond_guard(raw_spinlock_try, return, &d->lock) {
+		d->counter++;
+	}
+}
+
+TEST_SPINLOCK_COMMON(spinlock,
+		     spinlock_t,
+		     spinlock_init,
+		     spin_lock,
+		     spin_unlock,
+		     spin_trylock,
+		     TEST_OP_RW);
+static void __used test_spinlock_trylock_extra(struct test_spinlock_data *d)
+{
+	unsigned long flags;
+
+	if (spin_trylock_irq(&d->lock)) {
+		d->counter++;
+		spin_unlock_irq(&d->lock);
+	}
+	if (spin_trylock_irqsave(&d->lock, flags)) {
+		d->counter++;
+		spin_unlock_irqrestore(&d->lock, flags);
+	}
+	scoped_cond_guard(spinlock_try, return, &d->lock) {
+		d->counter++;
+	}
+}
+
+TEST_SPINLOCK_COMMON(write_lock,
+		     rwlock_t,
+		     rwlock_init,
+		     write_lock,
+		     write_unlock,
+		     write_trylock,
+		     TEST_OP_RW);
+static void __used test_write_trylock_extra(struct test_write_lock_data *d)
+{
+	unsigned long flags;
+
+	if (write_trylock_irqsave(&d->lock, flags)) {
+		d->counter++;
+		write_unlock_irqrestore(&d->lock, flags);
+	}
+}
+
+TEST_SPINLOCK_COMMON(read_lock,
+		     rwlock_t,
+		     rwlock_init,
+		     read_lock,
+		     read_unlock,
+		     read_trylock,
+		     TEST_OP_RO);
+
+struct test_mutex_data {
+	struct mutex mtx;
+	int counter __guarded_by(&mtx);
+};
+
+static void __used test_mutex_init(struct test_mutex_data *d)
+{
+	guard(mutex_init)(&d->mtx);
+	d->counter = 0;
+}
+
+static void __used test_mutex_lock(struct test_mutex_data *d)
+{
+	mutex_lock(&d->mtx);
+	d->counter++;
+	mutex_unlock(&d->mtx);
+	mutex_lock_io(&d->mtx);
+	d->counter++;
+	mutex_unlock(&d->mtx);
+}
+
+static void __used test_mutex_trylock(struct test_mutex_data *d, atomic_t *a)
+{
+	if (!mutex_lock_interruptible(&d->mtx)) {
+		d->counter++;
+		mutex_unlock(&d->mtx);
+	}
+	if (!mutex_lock_killable(&d->mtx)) {
+		d->counter++;
+		mutex_unlock(&d->mtx);
+	}
+	if (mutex_trylock(&d->mtx)) {
+		d->counter++;
+		mutex_unlock(&d->mtx);
+	}
+	if (atomic_dec_and_mutex_lock(a, &d->mtx)) {
+		d->counter++;
+		mutex_unlock(&d->mtx);
+	}
+}
+
+static void __used test_mutex_assert(struct test_mutex_data *d)
+{
+	lockdep_assert_held(&d->mtx);
+	d->counter++;
+}
+
+static void __used test_mutex_guard(struct test_mutex_data *d)
+{
+	guard(mutex)(&d->mtx);
+	d->counter++;
+}
+
+static void __used test_mutex_cond_guard(struct test_mutex_data *d)
+{
+	scoped_cond_guard(mutex_try, return, &d->mtx) {
+		d->counter++;
+	}
+	scoped_cond_guard(mutex_intr, return, &d->mtx) {
+		d->counter++;
+	}
+}
+
+struct test_seqlock_data {
+	seqlock_t sl;
+	int counter __guarded_by(&sl);
+};
+
+static void __used test_seqlock_init(struct test_seqlock_data *d)
+{
+	guard(seqlock_init)(&d->sl);
+	d->counter = 0;
+}
+
+static void __used test_seqlock_reader(struct test_seqlock_data *d)
+{
+	unsigned int seq;
+
+	do {
+		seq = read_seqbegin(&d->sl);
+		(void)d->counter;
+	} while (read_seqretry(&d->sl, seq));
+}
+
+static void __used test_seqlock_writer(struct test_seqlock_data *d)
+{
+	unsigned long flags;
+
+	write_seqlock(&d->sl);
+	d->counter++;
+	write_sequnlock(&d->sl);
+
+	write_seqlock_irq(&d->sl);
+	d->counter++;
+	write_sequnlock_irq(&d->sl);
+
+	write_seqlock_bh(&d->sl);
+	d->counter++;
+	write_sequnlock_bh(&d->sl);
+
+	write_seqlock_irqsave(&d->sl, flags);
+	d->counter++;
+	write_sequnlock_irqrestore(&d->sl, flags);
+}
+
+static void __used test_seqlock_scoped(struct test_seqlock_data *d)
+{
+	scoped_seqlock_read (&d->sl, ss_lockless) {
+		(void)d->counter;
+	}
+}
+
+struct test_rwsem_data {
+	struct rw_semaphore sem;
+	int counter __guarded_by(&sem);
+};
+
+static void __used test_rwsem_init(struct test_rwsem_data *d)
+{
+	guard(rwsem_init)(&d->sem);
+	d->counter = 0;
+}
+
+static void __used test_rwsem_reader(struct test_rwsem_data *d)
+{
+	down_read(&d->sem);
+	(void)d->counter;
+	up_read(&d->sem);
+
+	if (down_read_trylock(&d->sem)) {
+		(void)d->counter;
+		up_read(&d->sem);
+	}
+}
+
+static void __used test_rwsem_writer(struct test_rwsem_data *d)
+{
+	down_write(&d->sem);
+	d->counter++;
+	up_write(&d->sem);
+
+	down_write(&d->sem);
+	d->counter++;
+	downgrade_write(&d->sem);
+	(void)d->counter;
+	up_read(&d->sem);
+
+	if (down_write_trylock(&d->sem)) {
+		d->counter++;
+		up_write(&d->sem);
+	}
+}
+
+static void __used test_rwsem_assert(struct test_rwsem_data *d)
+{
+	rwsem_assert_held_nolockdep(&d->sem);
+	d->counter++;
+}
+
+static void __used test_rwsem_guard(struct test_rwsem_data *d)
+{
+	{ guard(rwsem_read)(&d->sem); (void)d->counter; }
+	{ guard(rwsem_write)(&d->sem); d->counter++; }
+}
+
+static void __used test_rwsem_cond_guard(struct test_rwsem_data *d)
+{
+	scoped_cond_guard(rwsem_read_try, return, &d->sem) {
+		(void)d->counter;
+	}
+	scoped_cond_guard(rwsem_write_try, return, &d->sem) {
+		d->counter++;
+	}
+}
+
+struct test_bit_spinlock_data {
+	unsigned long bits;
+	int counter __guarded_by(__bitlock(3, &bits));
+};
+
+static void __used test_bit_spin_lock(struct test_bit_spinlock_data *d)
+{
+	/*
+	 * Note, the analysis seems to have false negatives, because it won't
+	 * precisely recognize the bit of the fake __bitlock() token.
+	 */
+	bit_spin_lock(3, &d->bits);
+	d->counter++;
+	bit_spin_unlock(3, &d->bits);
+
+	bit_spin_lock(3, &d->bits);
+	d->counter++;
+	__bit_spin_unlock(3, &d->bits);
+
+	if (bit_spin_trylock(3, &d->bits)) {
+		d->counter++;
+		bit_spin_unlock(3, &d->bits);
+	}
+}
+
+/*
+ * Test that we can mark a variable guarded by RCU, and we can dereference and
+ * write to the pointer with RCU's primitives.
+ */
+struct test_rcu_data {
+	long __rcu_guarded *data;
+};
+
+static void __used test_rcu_guarded_reader(struct test_rcu_data *d)
+{
+	rcu_read_lock();
+	(void)rcu_dereference(d->data);
+	rcu_read_unlock();
+
+	rcu_read_lock_bh();
+	(void)rcu_dereference(d->data);
+	rcu_read_unlock_bh();
+
+	rcu_read_lock_sched();
+	(void)rcu_dereference(d->data);
+	rcu_read_unlock_sched();
+}
+
+static void __used test_rcu_guard(struct test_rcu_data *d)
+{
+	guard(rcu)();
+	(void)rcu_dereference(d->data);
+}
+
+static void __used test_rcu_guarded_updater(struct test_rcu_data *d)
+{
+	rcu_assign_pointer(d->data, NULL);
+	RCU_INIT_POINTER(d->data, NULL);
+	(void)unrcu_pointer(d->data);
+}
+
+static void wants_rcu_held(void)	__must_hold_shared(RCU)       { }
+static void wants_rcu_held_bh(void)	__must_hold_shared(RCU_BH)    { }
+static void wants_rcu_held_sched(void)	__must_hold_shared(RCU_SCHED) { }
+
+static void __used test_rcu_lock_variants(void)
+{
+	rcu_read_lock();
+	wants_rcu_held();
+	rcu_read_unlock();
+
+	rcu_read_lock_bh();
+	wants_rcu_held_bh();
+	rcu_read_unlock_bh();
+
+	rcu_read_lock_sched();
+	wants_rcu_held_sched();
+	rcu_read_unlock_sched();
+}
+
+static void __used test_rcu_lock_reentrant(void)
+{
+	rcu_read_lock();
+	rcu_read_lock();
+	rcu_read_lock_bh();
+	rcu_read_lock_bh();
+	rcu_read_lock_sched();
+	rcu_read_lock_sched();
+
+	rcu_read_unlock_sched();
+	rcu_read_unlock_sched();
+	rcu_read_unlock_bh();
+	rcu_read_unlock_bh();
+	rcu_read_unlock();
+	rcu_read_unlock();
+}
+
+static void __used test_rcu_assert_variants(void)
+{
+	lockdep_assert_in_rcu_read_lock();
+	wants_rcu_held();
+
+	lockdep_assert_in_rcu_read_lock_bh();
+	wants_rcu_held_bh();
+
+	lockdep_assert_in_rcu_read_lock_sched();
+	wants_rcu_held_sched();
+}
+
+struct test_srcu_data {
+	struct srcu_struct srcu;
+	long __rcu_guarded *data;
+};
+
+static void __used test_srcu(struct test_srcu_data *d)
+{
+	init_srcu_struct(&d->srcu);
+
+	int idx = srcu_read_lock(&d->srcu);
+	long *data = srcu_dereference(d->data, &d->srcu);
+	(void)data;
+	srcu_read_unlock(&d->srcu, idx);
+
+	rcu_assign_pointer(d->data, NULL);
+}
+
+static void __used test_srcu_guard(struct test_srcu_data *d)
+{
+	{ guard(srcu)(&d->srcu); (void)srcu_dereference(d->data, &d->srcu); }
+	{ guard(srcu_fast)(&d->srcu); (void)srcu_dereference(d->data, &d->srcu); }
+	{ guard(srcu_fast_notrace)(&d->srcu); (void)srcu_dereference(d->data, &d->srcu); }
+}
+
+struct test_local_lock_data {
+	local_lock_t lock;
+	int counter __guarded_by(&lock);
+};
+
+static DEFINE_PER_CPU(struct test_local_lock_data, test_local_lock_data) = {
+	.lock = INIT_LOCAL_LOCK(lock),
+};
+
+static void __used test_local_lock_init(struct test_local_lock_data *d)
+{
+	guard(local_lock_init)(&d->lock);
+	d->counter = 0;
+}
+
+static void __used test_local_lock(void)
+{
+	unsigned long flags;
+
+	local_lock(&test_local_lock_data.lock);
+	this_cpu_add(test_local_lock_data.counter, 1);
+	local_unlock(&test_local_lock_data.lock);
+
+	local_lock_irq(&test_local_lock_data.lock);
+	this_cpu_add(test_local_lock_data.counter, 1);
+	local_unlock_irq(&test_local_lock_data.lock);
+
+	local_lock_irqsave(&test_local_lock_data.lock, flags);
+	this_cpu_add(test_local_lock_data.counter, 1);
+	local_unlock_irqrestore(&test_local_lock_data.lock, flags);
+
+	local_lock_nested_bh(&test_local_lock_data.lock);
+	this_cpu_add(test_local_lock_data.counter, 1);
+	local_unlock_nested_bh(&test_local_lock_data.lock);
+}
+
+static void __used test_local_lock_guard(void)
+{
+	{ guard(local_lock)(&test_local_lock_data.lock); this_cpu_add(test_local_lock_data.counter, 1); }
+	{ guard(local_lock_irq)(&test_local_lock_data.lock); this_cpu_add(test_local_lock_data.counter, 1); }
+	{ guard(local_lock_irqsave)(&test_local_lock_data.lock); this_cpu_add(test_local_lock_data.counter, 1); }
+	{ guard(local_lock_nested_bh)(&test_local_lock_data.lock); this_cpu_add(test_local_lock_data.counter, 1); }
+}
+
+struct test_local_trylock_data {
+	local_trylock_t lock;
+	int counter __guarded_by(&lock);
+};
+
+static DEFINE_PER_CPU(struct test_local_trylock_data, test_local_trylock_data) = {
+	.lock = INIT_LOCAL_TRYLOCK(lock),
+};
+
+static void __used test_local_trylock_init(struct test_local_trylock_data *d)
+{
+	guard(local_trylock_init)(&d->lock);
+	d->counter = 0;
+}
+
+static void __used test_local_trylock(void)
+{
+	local_lock(&test_local_trylock_data.lock);
+	this_cpu_add(test_local_trylock_data.counter, 1);
+	local_unlock(&test_local_trylock_data.lock);
+
+	if (local_trylock(&test_local_trylock_data.lock)) {
+		this_cpu_add(test_local_trylock_data.counter, 1);
+		local_unlock(&test_local_trylock_data.lock);
+	}
+}
+
+static DEFINE_WD_CLASS(ww_class);
+
+struct test_ww_mutex_data {
+	struct ww_mutex mtx;
+	int counter __guarded_by(&mtx);
+};
+
+static void __used test_ww_mutex_lock_noctx(struct test_ww_mutex_data *d)
+{
+	if (!ww_mutex_lock(&d->mtx, NULL)) {
+		d->counter++;
+		ww_mutex_unlock(&d->mtx);
+	}
+
+	if (!ww_mutex_lock_interruptible(&d->mtx, NULL)) {
+		d->counter++;
+		ww_mutex_unlock(&d->mtx);
+	}
+
+	if (ww_mutex_trylock(&d->mtx, NULL)) {
+		d->counter++;
+		ww_mutex_unlock(&d->mtx);
+	}
+
+	ww_mutex_lock_slow(&d->mtx, NULL);
+	d->counter++;
+	ww_mutex_unlock(&d->mtx);
+
+	ww_mutex_destroy(&d->mtx);
+}
+
+static void __used test_ww_mutex_lock_ctx(struct test_ww_mutex_data *d)
+{
+	struct ww_acquire_ctx ctx;
+
+	ww_acquire_init(&ctx, &ww_class);
+
+	if (!ww_mutex_lock(&d->mtx, &ctx)) {
+		d->counter++;
+		ww_mutex_unlock(&d->mtx);
+	}
+
+	if (!ww_mutex_lock_interruptible(&d->mtx, &ctx)) {
+		d->counter++;
+		ww_mutex_unlock(&d->mtx);
+	}
+
+	if (ww_mutex_trylock(&d->mtx, &ctx)) {
+		d->counter++;
+		ww_mutex_unlock(&d->mtx);
+	}
+
+	ww_mutex_lock_slow(&d->mtx, &ctx);
+	d->counter++;
+	ww_mutex_unlock(&d->mtx);
+
+	ww_acquire_done(&ctx);
+	ww_acquire_fini(&ctx);
+
+	ww_mutex_destroy(&d->mtx);
+}
diff --git a/lib/test_debug_virtual.c b/lib/test_debug_virtual.c
index b7cc0aaee173..518ee8d213cd 100644
--- a/lib/test_debug_virtual.c
+++ b/lib/test_debug_virtual.c
@@ -29,7 +29,7 @@ static int __init test_debug_virtual_init(void)
 
 	pr_info("PA: %pa for VA: 0x%lx\n", &pa, (unsigned long)va);
 
-	foo = kzalloc(sizeof(*foo), GFP_KERNEL);
+	foo = kzalloc_obj(*foo);
 	if (!foo)
 		return -ENOMEM;
 
diff --git a/lib/test_firmware.c b/lib/test_firmware.c
index be4f93124901..b471d720879a 100644
--- a/lib/test_firmware.c
+++ b/lib/test_firmware.c
@@ -1309,7 +1309,7 @@ static ssize_t upload_register_store(struct device *dev,
 		goto free_name;
 	}
 
-	tst = kzalloc(sizeof(*tst), GFP_KERNEL);
+	tst = kzalloc_obj(*tst);
 	if (!tst) {
 		ret = -ENOMEM;
 		goto free_name;
@@ -1526,7 +1526,7 @@ static int __init test_firmware_init(void)
 {
 	int rc;
 
-	test_fw_config = kzalloc(sizeof(struct test_config), GFP_KERNEL);
+	test_fw_config = kzalloc_obj(struct test_config);
 	if (!test_fw_config)
 		return -ENOMEM;
 
diff --git a/lib/test_fortify/test_fortify.sh b/lib/test_fortify/test_fortify.sh
index c2688ab8281d..ad6dd44fa31c 100644
--- a/lib/test_fortify/test_fortify.sh
+++ b/lib/test_fortify/test_fortify.sh
@@ -17,7 +17,7 @@ WANT="__${FILE%%-*}"
 # Argument 2: Where to write the build log.
 OUT="$1"
 shift
-TMP="${OUT}.tmp"
+TMP="${OUT%/*}/.${OUT##*/}.tmp"
 
 # Argument 3: Path to "nm" tool.
 NM="$1"
@@ -29,7 +29,7 @@ shift
 __cleanup() {
 	rm -f "$TMP"
 }
-trap __cleanup EXIT
+trap __cleanup EXIT HUP INT QUIT TERM
 
 # Function names in warnings are wrapped in backticks under UTF-8 locales.
 # Run the commands with LANG=C so that grep output will not change.
diff --git a/lib/test_hmm.c b/lib/test_hmm.c
index 83e3d8208a54..0964d53365e6 100644
--- a/lib/test_hmm.c
+++ b/lib/test_hmm.c
@@ -92,6 +92,7 @@ struct dmirror {
 	struct xarray			pt;
 	struct mmu_interval_notifier	notifier;
 	struct mutex			mutex;
+	__u64			flags;
 };
 
 /*
@@ -119,6 +120,7 @@ struct dmirror_device {
 	unsigned long		calloc;
 	unsigned long		cfree;
 	struct page		*free_pages;
+	struct folio		*free_folios;
 	spinlock_t		lock;		/* protects the above */
 };
 
@@ -164,7 +166,7 @@ static int dmirror_fops_open(struct inode *inode, struct file *filp)
 	int ret;
 
 	/* Mirror this process address space */
-	dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL);
+	dmirror = kzalloc_obj(*dmirror);
 	if (dmirror == NULL)
 		return -ENOMEM;
 
@@ -492,7 +494,7 @@ fini:
 }
 
 static int dmirror_allocate_chunk(struct dmirror_device *mdevice,
-				   struct page **ppage)
+				  struct page **ppage, bool is_large)
 {
 	struct dmirror_chunk *devmem;
 	struct resource *res = NULL;
@@ -502,7 +504,7 @@ static int dmirror_allocate_chunk(struct dmirror_device *mdevice,
 	void *ptr;
 	int ret = -ENOMEM;
 
-	devmem = kzalloc(sizeof(*devmem), GFP_KERNEL);
+	devmem = kzalloc_obj(*devmem);
 	if (!devmem)
 		return ret;
 
@@ -572,20 +574,45 @@ static int dmirror_allocate_chunk(struct dmirror_device *mdevice,
 		pfn_first, pfn_last);
 
 	spin_lock(&mdevice->lock);
-	for (pfn = pfn_first; pfn < pfn_last; pfn++) {
+	for (pfn = pfn_first; pfn < pfn_last; ) {
 		struct page *page = pfn_to_page(pfn);
 
+		if (is_large && IS_ALIGNED(pfn, HPAGE_PMD_NR)
+			&& (pfn + HPAGE_PMD_NR <= pfn_last)) {
+			page->zone_device_data = mdevice->free_folios;
+			mdevice->free_folios = page_folio(page);
+			pfn += HPAGE_PMD_NR;
+			continue;
+		}
+
 		page->zone_device_data = mdevice->free_pages;
 		mdevice->free_pages = page;
+		pfn++;
 	}
+
+	ret = 0;
 	if (ppage) {
-		*ppage = mdevice->free_pages;
-		mdevice->free_pages = (*ppage)->zone_device_data;
-		mdevice->calloc++;
+		if (is_large) {
+			if (!mdevice->free_folios) {
+				ret = -ENOMEM;
+				goto err_unlock;
+			}
+			*ppage = folio_page(mdevice->free_folios, 0);
+			mdevice->free_folios = (*ppage)->zone_device_data;
+			mdevice->calloc += HPAGE_PMD_NR;
+		} else if (mdevice->free_pages) {
+			*ppage = mdevice->free_pages;
+			mdevice->free_pages = (*ppage)->zone_device_data;
+			mdevice->calloc++;
+		} else {
+			ret = -ENOMEM;
+			goto err_unlock;
+		}
 	}
+err_unlock:
 	spin_unlock(&mdevice->lock);
 
-	return 0;
+	return ret;
 
 err_release:
 	mutex_unlock(&mdevice->devmem_lock);
@@ -598,10 +625,13 @@ err_devmem:
 	return ret;
 }
 
-static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice)
+static struct page *dmirror_devmem_alloc_page(struct dmirror *dmirror,
+					      bool is_large)
 {
 	struct page *dpage = NULL;
 	struct page *rpage = NULL;
+	unsigned int order = is_large ? HPAGE_PMD_ORDER : 0;
+	struct dmirror_device *mdevice = dmirror->mdevice;
 
 	/*
 	 * For ZONE_DEVICE private type, this is a fake device so we allocate
@@ -610,49 +640,57 @@ static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice)
 	 * data and ignore rpage.
 	 */
 	if (dmirror_is_private_zone(mdevice)) {
-		rpage = alloc_page(GFP_HIGHUSER);
+		rpage = folio_page(folio_alloc(GFP_HIGHUSER, order), 0);
 		if (!rpage)
 			return NULL;
 	}
 	spin_lock(&mdevice->lock);
 
-	if (mdevice->free_pages) {
+	if (is_large && mdevice->free_folios) {
+		dpage = folio_page(mdevice->free_folios, 0);
+		mdevice->free_folios = dpage->zone_device_data;
+		mdevice->calloc += 1 << order;
+		spin_unlock(&mdevice->lock);
+	} else if (!is_large && mdevice->free_pages) {
 		dpage = mdevice->free_pages;
 		mdevice->free_pages = dpage->zone_device_data;
 		mdevice->calloc++;
 		spin_unlock(&mdevice->lock);
 	} else {
 		spin_unlock(&mdevice->lock);
-		if (dmirror_allocate_chunk(mdevice, &dpage))
+		if (dmirror_allocate_chunk(mdevice, &dpage, is_large))
 			goto error;
 	}
 
-	zone_device_page_init(dpage);
+	zone_device_folio_init(page_folio(dpage),
+			       page_pgmap(folio_page(page_folio(dpage), 0)),
+			       order);
 	dpage->zone_device_data = rpage;
 	return dpage;
 
 error:
 	if (rpage)
-		__free_page(rpage);
+		__free_pages(rpage, order);
 	return NULL;
 }
 
 static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args,
 					   struct dmirror *dmirror)
 {
-	struct dmirror_device *mdevice = dmirror->mdevice;
 	const unsigned long *src = args->src;
 	unsigned long *dst = args->dst;
 	unsigned long addr;
 
-	for (addr = args->start; addr < args->end; addr += PAGE_SIZE,
-						   src++, dst++) {
+	for (addr = args->start; addr < args->end; ) {
 		struct page *spage;
 		struct page *dpage;
 		struct page *rpage;
+		bool is_large = *src & MIGRATE_PFN_COMPOUND;
+		int write = (*src & MIGRATE_PFN_WRITE) ? MIGRATE_PFN_WRITE : 0;
+		unsigned long nr = 1;
 
 		if (!(*src & MIGRATE_PFN_MIGRATE))
-			continue;
+			goto next;
 
 		/*
 		 * Note that spage might be NULL which is OK since it is an
@@ -662,17 +700,50 @@ static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args,
 		if (WARN(spage && is_zone_device_page(spage),
 		     "page already in device spage pfn: 0x%lx\n",
 		     page_to_pfn(spage)))
+			goto next;
+
+		if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) {
+			dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC;
+			dpage = NULL;
+		} else
+			dpage = dmirror_devmem_alloc_page(dmirror, is_large);
+
+		if (!dpage) {
+			struct folio *folio;
+			unsigned long i;
+			unsigned long spfn = *src >> MIGRATE_PFN_SHIFT;
+			struct page *src_page;
+
+			if (!is_large)
+				goto next;
+
+			if (!spage && is_large) {
+				nr = HPAGE_PMD_NR;
+			} else {
+				folio = page_folio(spage);
+				nr = folio_nr_pages(folio);
+			}
+
+			for (i = 0; i < nr && addr < args->end; i++) {
+				dpage = dmirror_devmem_alloc_page(dmirror, false);
+				rpage = BACKING_PAGE(dpage);
+				rpage->zone_device_data = dmirror;
+
+				*dst = migrate_pfn(page_to_pfn(dpage)) | write;
+				src_page = pfn_to_page(spfn + i);
+
+				if (spage)
+					copy_highpage(rpage, src_page);
+				else
+					clear_highpage(rpage);
+				src++;
+				dst++;
+				addr += PAGE_SIZE;
+			}
 			continue;
-
-		dpage = dmirror_devmem_alloc_page(mdevice);
-		if (!dpage)
-			continue;
+		}
 
 		rpage = BACKING_PAGE(dpage);
-		if (spage)
-			copy_highpage(rpage, spage);
-		else
-			clear_highpage(rpage);
 
 		/*
 		 * Normally, a device would use the page->zone_device_data to
@@ -684,10 +755,42 @@ static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args,
 
 		pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n",
 			 page_to_pfn(spage), page_to_pfn(dpage));
-		*dst = migrate_pfn(page_to_pfn(dpage));
-		if ((*src & MIGRATE_PFN_WRITE) ||
-		    (!spage && args->vma->vm_flags & VM_WRITE))
-			*dst |= MIGRATE_PFN_WRITE;
+
+		*dst = migrate_pfn(page_to_pfn(dpage)) | write;
+
+		if (is_large) {
+			int i;
+			struct folio *folio = page_folio(dpage);
+			*dst |= MIGRATE_PFN_COMPOUND;
+
+			if (folio_test_large(folio)) {
+				for (i = 0; i < folio_nr_pages(folio); i++) {
+					struct page *dst_page =
+						pfn_to_page(page_to_pfn(rpage) + i);
+					struct page *src_page =
+						pfn_to_page(page_to_pfn(spage) + i);
+
+					if (spage)
+						copy_highpage(dst_page, src_page);
+					else
+						clear_highpage(dst_page);
+					src++;
+					dst++;
+					addr += PAGE_SIZE;
+				}
+				continue;
+			}
+		}
+
+		if (spage)
+			copy_highpage(rpage, spage);
+		else
+			clear_highpage(rpage);
+
+next:
+		src++;
+		dst++;
+		addr += PAGE_SIZE;
 	}
 }
 
@@ -734,14 +837,17 @@ static int dmirror_migrate_finalize_and_map(struct migrate_vma *args,
 	const unsigned long *src = args->src;
 	const unsigned long *dst = args->dst;
 	unsigned long pfn;
+	const unsigned long start_pfn = start >> PAGE_SHIFT;
+	const unsigned long end_pfn = end >> PAGE_SHIFT;
 
 	/* Map the migrated pages into the device's page tables. */
 	mutex_lock(&dmirror->mutex);
 
-	for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++,
-								src++, dst++) {
+	for (pfn = start_pfn; pfn < end_pfn; pfn++, src++, dst++) {
 		struct page *dpage;
 		void *entry;
+		int nr, i;
+		struct page *rpage;
 
 		if (!(*src & MIGRATE_PFN_MIGRATE))
 			continue;
@@ -750,13 +856,25 @@ static int dmirror_migrate_finalize_and_map(struct migrate_vma *args,
 		if (!dpage)
 			continue;
 
-		entry = BACKING_PAGE(dpage);
-		if (*dst & MIGRATE_PFN_WRITE)
-			entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);
-		entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
-		if (xa_is_err(entry)) {
-			mutex_unlock(&dmirror->mutex);
-			return xa_err(entry);
+		if (*dst & MIGRATE_PFN_COMPOUND)
+			nr = folio_nr_pages(page_folio(dpage));
+		else
+			nr = 1;
+
+		WARN_ON_ONCE(end_pfn < start_pfn + nr);
+
+		rpage = BACKING_PAGE(dpage);
+		VM_WARN_ON(folio_nr_pages(page_folio(rpage)) != nr);
+
+		for (i = 0; i < nr; i++) {
+			entry = folio_page(page_folio(rpage), i);
+			if (*dst & MIGRATE_PFN_WRITE)
+				entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);
+			entry = xa_store(&dmirror->pt, pfn + i, entry, GFP_ATOMIC);
+			if (xa_is_err(entry)) {
+				mutex_unlock(&dmirror->mutex);
+				return xa_err(entry);
+			}
 		}
 	}
 
@@ -829,31 +947,77 @@ static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args,
 	unsigned long start = args->start;
 	unsigned long end = args->end;
 	unsigned long addr;
+	unsigned int order = 0;
+	int i;
 
-	for (addr = start; addr < end; addr += PAGE_SIZE,
-				       src++, dst++) {
+	for (addr = start; addr < end; ) {
 		struct page *dpage, *spage;
 
 		spage = migrate_pfn_to_page(*src);
-		if (!spage || !(*src & MIGRATE_PFN_MIGRATE))
-			continue;
+		if (!spage || !(*src & MIGRATE_PFN_MIGRATE)) {
+			addr += PAGE_SIZE;
+			goto next;
+		}
 
 		if (WARN_ON(!is_device_private_page(spage) &&
-			    !is_device_coherent_page(spage)))
-			continue;
-		spage = BACKING_PAGE(spage);
-		dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr);
-		if (!dpage)
-			continue;
-		pr_debug("migrating from dev to sys pfn src: 0x%lx pfn dst: 0x%lx\n",
-			 page_to_pfn(spage), page_to_pfn(dpage));
+			    !is_device_coherent_page(spage))) {
+			addr += PAGE_SIZE;
+			goto next;
+		}
 
-		lock_page(dpage);
-		xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);
-		copy_highpage(dpage, spage);
-		*dst = migrate_pfn(page_to_pfn(dpage));
+		spage = BACKING_PAGE(spage);
+		order = folio_order(page_folio(spage));
+		if (order)
+			*dst = MIGRATE_PFN_COMPOUND;
 		if (*src & MIGRATE_PFN_WRITE)
 			*dst |= MIGRATE_PFN_WRITE;
+
+		if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) {
+			dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC;
+			*dst &= ~MIGRATE_PFN_COMPOUND;
+			dpage = NULL;
+		} else if (order) {
+			dpage = folio_page(vma_alloc_folio(GFP_HIGHUSER_MOVABLE,
+						order, args->vma, addr), 0);
+		} else {
+			dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr);
+		}
+
+		if (!dpage && !order)
+			return VM_FAULT_OOM;
+
+		pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n",
+				page_to_pfn(spage), page_to_pfn(dpage));
+
+		if (dpage) {
+			lock_page(dpage);
+			*dst |= migrate_pfn(page_to_pfn(dpage));
+		}
+
+		for (i = 0; i < (1 << order); i++) {
+			struct page *src_page;
+			struct page *dst_page;
+
+			/* Try with smaller pages if large allocation fails */
+			if (!dpage && order) {
+				dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr);
+				lock_page(dpage);
+				dst[i] = migrate_pfn(page_to_pfn(dpage));
+				dst_page = pfn_to_page(page_to_pfn(dpage));
+				dpage = NULL; /* For the next iteration */
+			} else {
+				dst_page = pfn_to_page(page_to_pfn(dpage) + i);
+			}
+
+			src_page = pfn_to_page(page_to_pfn(spage) + i);
+
+			xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);
+			addr += PAGE_SIZE;
+			copy_highpage(dst_page, src_page);
+		}
+next:
+		src += 1 << order;
+		dst += 1 << order;
 	}
 	return 0;
 }
@@ -879,11 +1043,14 @@ static int dmirror_migrate_to_system(struct dmirror *dmirror,
 	unsigned long size = cmd->npages << PAGE_SHIFT;
 	struct mm_struct *mm = dmirror->notifier.mm;
 	struct vm_area_struct *vma;
-	unsigned long src_pfns[32] = { 0 };
-	unsigned long dst_pfns[32] = { 0 };
 	struct migrate_vma args = { 0 };
 	unsigned long next;
 	int ret;
+	unsigned long *src_pfns;
+	unsigned long *dst_pfns;
+
+	src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL);
+	dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL);
 
 	start = cmd->addr;
 	end = start + size;
@@ -902,7 +1069,7 @@ static int dmirror_migrate_to_system(struct dmirror *dmirror,
 			ret = -EINVAL;
 			goto out;
 		}
-		next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT));
+		next = min(end, addr + (PTRS_PER_PTE << PAGE_SHIFT));
 		if (next > vma->vm_end)
 			next = vma->vm_end;
 
@@ -912,7 +1079,7 @@ static int dmirror_migrate_to_system(struct dmirror *dmirror,
 		args.start = addr;
 		args.end = next;
 		args.pgmap_owner = dmirror->mdevice;
-		args.flags = dmirror_select_device(dmirror);
+		args.flags = dmirror_select_device(dmirror) | MIGRATE_VMA_SELECT_COMPOUND;
 
 		ret = migrate_vma_setup(&args);
 		if (ret)
@@ -928,6 +1095,8 @@ static int dmirror_migrate_to_system(struct dmirror *dmirror,
 out:
 	mmap_read_unlock(mm);
 	mmput(mm);
+	kvfree(src_pfns);
+	kvfree(dst_pfns);
 
 	return ret;
 }
@@ -939,12 +1108,12 @@ static int dmirror_migrate_to_device(struct dmirror *dmirror,
 	unsigned long size = cmd->npages << PAGE_SHIFT;
 	struct mm_struct *mm = dmirror->notifier.mm;
 	struct vm_area_struct *vma;
-	unsigned long src_pfns[32] = { 0 };
-	unsigned long dst_pfns[32] = { 0 };
 	struct dmirror_bounce bounce;
 	struct migrate_vma args = { 0 };
 	unsigned long next;
 	int ret;
+	unsigned long *src_pfns = NULL;
+	unsigned long *dst_pfns = NULL;
 
 	start = cmd->addr;
 	end = start + size;
@@ -955,6 +1124,18 @@ static int dmirror_migrate_to_device(struct dmirror *dmirror,
 	if (!mmget_not_zero(mm))
 		return -EINVAL;
 
+	ret = -ENOMEM;
+	src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns),
+			  GFP_KERNEL | __GFP_NOFAIL);
+	if (!src_pfns)
+		goto free_mem;
+
+	dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns),
+			  GFP_KERNEL | __GFP_NOFAIL);
+	if (!dst_pfns)
+		goto free_mem;
+
+	ret = 0;
 	mmap_read_lock(mm);
 	for (addr = start; addr < end; addr = next) {
 		vma = vma_lookup(mm, addr);
@@ -962,7 +1143,7 @@ static int dmirror_migrate_to_device(struct dmirror *dmirror,
 			ret = -EINVAL;
 			goto out;
 		}
-		next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT));
+		next = min(end, addr + (PTRS_PER_PTE << PAGE_SHIFT));
 		if (next > vma->vm_end)
 			next = vma->vm_end;
 
@@ -972,7 +1153,8 @@ static int dmirror_migrate_to_device(struct dmirror *dmirror,
 		args.start = addr;
 		args.end = next;
 		args.pgmap_owner = dmirror->mdevice;
-		args.flags = MIGRATE_VMA_SELECT_SYSTEM;
+		args.flags = MIGRATE_VMA_SELECT_SYSTEM |
+				MIGRATE_VMA_SELECT_COMPOUND;
 		ret = migrate_vma_setup(&args);
 		if (ret)
 			goto out;
@@ -992,7 +1174,7 @@ static int dmirror_migrate_to_device(struct dmirror *dmirror,
 	 */
 	ret = dmirror_bounce_init(&bounce, start, size);
 	if (ret)
-		return ret;
+		goto free_mem;
 	mutex_lock(&dmirror->mutex);
 	ret = dmirror_do_read(dmirror, start, end, &bounce);
 	mutex_unlock(&dmirror->mutex);
@@ -1003,11 +1185,14 @@ static int dmirror_migrate_to_device(struct dmirror *dmirror,
 	}
 	cmd->cpages = bounce.cpages;
 	dmirror_bounce_fini(&bounce);
-	return ret;
+	goto free_mem;
 
 out:
 	mmap_read_unlock(mm);
 	mmput(mm);
+free_mem:
+	kfree(src_pfns);
+	kfree(dst_pfns);
 	return ret;
 }
 
@@ -1200,6 +1385,7 @@ static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk)
 	unsigned long i;
 	unsigned long *src_pfns;
 	unsigned long *dst_pfns;
+	unsigned int order = 0;
 
 	src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL);
 	dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL);
@@ -1215,13 +1401,25 @@ static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk)
 		if (WARN_ON(!is_device_private_page(spage) &&
 			    !is_device_coherent_page(spage)))
 			continue;
+
+		order = folio_order(page_folio(spage));
 		spage = BACKING_PAGE(spage);
-		dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL);
+		if (src_pfns[i] & MIGRATE_PFN_COMPOUND) {
+			dpage = folio_page(folio_alloc(GFP_HIGHUSER_MOVABLE,
+					      order), 0);
+		} else {
+			dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL);
+			order = 0;
+		}
+
+		/* TODO Support splitting here */
 		lock_page(dpage);
-		copy_highpage(dpage, spage);
 		dst_pfns[i] = migrate_pfn(page_to_pfn(dpage));
 		if (src_pfns[i] & MIGRATE_PFN_WRITE)
 			dst_pfns[i] |= MIGRATE_PFN_WRITE;
+		if (order)
+			dst_pfns[i] |= MIGRATE_PFN_COMPOUND;
+		folio_copy(page_folio(dpage), page_folio(spage));
 	}
 	migrate_device_pages(src_pfns, dst_pfns, npages);
 	migrate_device_finalize(src_pfns, dst_pfns, npages);
@@ -1234,7 +1432,12 @@ static void dmirror_remove_free_pages(struct dmirror_chunk *devmem)
 {
 	struct dmirror_device *mdevice = devmem->mdevice;
 	struct page *page;
+	struct folio *folio;
 
+
+	for (folio = mdevice->free_folios; folio; folio = folio_zone_device_data(folio))
+		if (dmirror_page_to_chunk(folio_page(folio, 0)) == devmem)
+			mdevice->free_folios = folio_zone_device_data(folio);
 	for (page = mdevice->free_pages; page; page = page->zone_device_data)
 		if (dmirror_page_to_chunk(page) == devmem)
 			mdevice->free_pages = page->zone_device_data;
@@ -1265,6 +1468,7 @@ static void dmirror_device_remove_chunks(struct dmirror_device *mdevice)
 		mdevice->devmem_count = 0;
 		mdevice->devmem_capacity = 0;
 		mdevice->free_pages = NULL;
+		mdevice->free_folios = NULL;
 		kfree(mdevice->devmem_chunks);
 		mdevice->devmem_chunks = NULL;
 	}
@@ -1329,6 +1533,10 @@ static long dmirror_fops_unlocked_ioctl(struct file *filp,
 		dmirror_device_remove_chunks(dmirror->mdevice);
 		ret = 0;
 		break;
+	case HMM_DMIRROR_FLAGS:
+		dmirror->flags = cmd.npages;
+		ret = 0;
+		break;
 
 	default:
 		return -EINVAL;
@@ -1374,22 +1582,35 @@ static const struct file_operations dmirror_fops = {
 	.owner		= THIS_MODULE,
 };
 
-static void dmirror_devmem_free(struct page *page)
+static void dmirror_devmem_free(struct folio *folio)
 {
+	struct page *page = &folio->page;
 	struct page *rpage = BACKING_PAGE(page);
 	struct dmirror_device *mdevice;
+	struct folio *rfolio = page_folio(rpage);
+	unsigned int order = folio_order(rfolio);
 
-	if (rpage != page)
-		__free_page(rpage);
+	if (rpage != page) {
+		if (order)
+			__free_pages(rpage, order);
+		else
+			__free_page(rpage);
+		rpage = NULL;
+	}
 
 	mdevice = dmirror_page_to_device(page);
 	spin_lock(&mdevice->lock);
 
 	/* Return page to our allocator if not freeing the chunk */
 	if (!dmirror_page_to_chunk(page)->remove) {
-		mdevice->cfree++;
-		page->zone_device_data = mdevice->free_pages;
-		mdevice->free_pages = page;
+		mdevice->cfree += 1 << order;
+		if (order) {
+			page->zone_device_data = mdevice->free_folios;
+			mdevice->free_folios = page_folio(page);
+		} else {
+			page->zone_device_data = mdevice->free_pages;
+			mdevice->free_pages = page;
+		}
 	}
 	spin_unlock(&mdevice->lock);
 }
@@ -1397,36 +1618,61 @@ static void dmirror_devmem_free(struct page *page)
 static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf)
 {
 	struct migrate_vma args = { 0 };
-	unsigned long src_pfns = 0;
-	unsigned long dst_pfns = 0;
 	struct page *rpage;
 	struct dmirror *dmirror;
-	vm_fault_t ret;
+	vm_fault_t ret = 0;
+	unsigned int order, nr;
 
 	/*
 	 * Normally, a device would use the page->zone_device_data to point to
 	 * the mirror but here we use it to hold the page for the simulated
 	 * device memory and that page holds the pointer to the mirror.
 	 */
-	rpage = vmf->page->zone_device_data;
+	rpage = folio_zone_device_data(page_folio(vmf->page));
 	dmirror = rpage->zone_device_data;
 
 	/* FIXME demonstrate how we can adjust migrate range */
+	order = folio_order(page_folio(vmf->page));
+	nr = 1 << order;
+
+	/*
+	 * When folios are partially mapped, we can't rely on the folio
+	 * order of vmf->page as the folio might not be fully split yet
+	 */
+	if (vmf->pte) {
+		order = 0;
+		nr = 1;
+	}
+
+	/*
+	 * Consider a per-cpu cache of src and dst pfns, but with
+	 * large number of cpus that might not scale well.
+	 */
+	args.start = ALIGN_DOWN(vmf->address, (PAGE_SIZE << order));
 	args.vma = vmf->vma;
-	args.start = vmf->address;
-	args.end = args.start + PAGE_SIZE;
-	args.src = &src_pfns;
-	args.dst = &dst_pfns;
+	args.end = args.start + (PAGE_SIZE << order);
+
+	nr = (args.end - args.start) >> PAGE_SHIFT;
+	args.src = kcalloc(nr, sizeof(unsigned long), GFP_KERNEL);
+	args.dst = kcalloc(nr, sizeof(unsigned long), GFP_KERNEL);
 	args.pgmap_owner = dmirror->mdevice;
 	args.flags = dmirror_select_device(dmirror);
 	args.fault_page = vmf->page;
 
+	if (!args.src || !args.dst) {
+		ret = VM_FAULT_OOM;
+		goto err;
+	}
+
+	if (order)
+		args.flags |= MIGRATE_VMA_SELECT_COMPOUND;
+
 	if (migrate_vma_setup(&args))
 		return VM_FAULT_SIGBUS;
 
 	ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror);
 	if (ret)
-		return ret;
+		goto err;
 	migrate_vma_pages(&args);
 	/*
 	 * No device finalize step is needed since
@@ -1434,12 +1680,50 @@ static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf)
 	 * invalidated the device page table.
 	 */
 	migrate_vma_finalize(&args);
-	return 0;
+err:
+	kfree(args.src);
+	kfree(args.dst);
+	return ret;
+}
+
+static void dmirror_devmem_folio_split(struct folio *head, struct folio *tail)
+{
+	struct page *rpage = BACKING_PAGE(folio_page(head, 0));
+	struct page *rpage_tail;
+	struct folio *rfolio;
+	unsigned long offset = 0;
+
+	if (!rpage) {
+		tail->page.zone_device_data = NULL;
+		return;
+	}
+
+	rfolio = page_folio(rpage);
+
+	if (tail == NULL) {
+		folio_reset_order(rfolio);
+		rfolio->mapping = NULL;
+		folio_set_count(rfolio, 1);
+		return;
+	}
+
+	offset = folio_pfn(tail) - folio_pfn(head);
+
+	rpage_tail = folio_page(rfolio, offset);
+	tail->page.zone_device_data = rpage_tail;
+	rpage_tail->zone_device_data = rpage->zone_device_data;
+	clear_compound_head(rpage_tail);
+	rpage_tail->mapping = NULL;
+
+	folio_page(tail, 0)->mapping = folio_page(head, 0)->mapping;
+	tail->pgmap = head->pgmap;
+	folio_set_count(page_folio(rpage_tail), 1);
 }
 
 static const struct dev_pagemap_ops dmirror_devmem_ops = {
-	.page_free	= dmirror_devmem_free,
+	.folio_free	= dmirror_devmem_free,
 	.migrate_to_ram	= dmirror_devmem_fault,
+	.folio_split	= dmirror_devmem_folio_split,
 };
 
 static int dmirror_device_init(struct dmirror_device *mdevice, int id)
@@ -1458,20 +1742,25 @@ static int dmirror_device_init(struct dmirror_device *mdevice, int id)
 
 	ret = dev_set_name(&mdevice->device, "hmm_dmirror%u", id);
 	if (ret)
-		return ret;
+		goto put_device;
 
 	ret = cdev_device_add(&mdevice->cdevice, &mdevice->device);
 	if (ret)
-		return ret;
+		goto put_device;
 
 	/* Build a list of free ZONE_DEVICE struct pages */
-	return dmirror_allocate_chunk(mdevice, NULL);
+	return dmirror_allocate_chunk(mdevice, NULL, false);
+
+put_device:
+	put_device(&mdevice->device);
+	return ret;
 }
 
 static void dmirror_device_remove(struct dmirror_device *mdevice)
 {
 	dmirror_device_remove_chunks(mdevice);
 	cdev_device_del(&mdevice->cdevice, &mdevice->device);
+	put_device(&mdevice->device);
 }
 
 static int __init hmm_dmirror_init(void)
diff --git a/lib/test_hmm_uapi.h b/lib/test_hmm_uapi.h
index 8c818a2cf4f6..f94c6d457338 100644
--- a/lib/test_hmm_uapi.h
+++ b/lib/test_hmm_uapi.h
@@ -37,6 +37,9 @@ struct hmm_dmirror_cmd {
 #define HMM_DMIRROR_EXCLUSIVE		_IOWR('H', 0x05, struct hmm_dmirror_cmd)
 #define HMM_DMIRROR_CHECK_EXCLUSIVE	_IOWR('H', 0x06, struct hmm_dmirror_cmd)
 #define HMM_DMIRROR_RELEASE		_IOWR('H', 0x07, struct hmm_dmirror_cmd)
+#define HMM_DMIRROR_FLAGS		_IOWR('H', 0x08, struct hmm_dmirror_cmd)
+
+#define HMM_DMIRROR_FLAG_FAIL_ALLOC	(1ULL << 0)
 
 /*
  * Values returned in hmm_dmirror_cmd.ptr for HMM_DMIRROR_SNAPSHOT.
diff --git a/lib/test_kho.c b/lib/test_kho.c
index fff018e5548d..7ef9e4061869 100644
--- a/lib/test_kho.c
+++ b/lib/test_kho.c
@@ -19,6 +19,7 @@
 #include <linux/printk.h>
 #include <linux/vmalloc.h>
 #include <linux/kexec_handover.h>
+#include <linux/kho/abi/kexec_handover.h>
 
 #include <net/checksum.h>
 
@@ -33,44 +34,28 @@ struct kho_test_state {
 	unsigned int nr_folios;
 	struct folio **folios;
 	phys_addr_t *folios_info;
+	struct kho_vmalloc folios_info_phys;
+	int nr_folios_preserved;
 	struct folio *fdt;
 	__wsum csum;
 };
 
 static struct kho_test_state kho_test_state;
 
-static int kho_test_notifier(struct notifier_block *self, unsigned long cmd,
-			     void *v)
+static void kho_test_unpreserve_data(struct kho_test_state *state)
 {
-	struct kho_test_state *state = &kho_test_state;
-	struct kho_serialization *ser = v;
-	int err = 0;
-
-	switch (cmd) {
-	case KEXEC_KHO_ABORT:
-		return NOTIFY_DONE;
-	case KEXEC_KHO_FINALIZE:
-		/* Handled below */
-		break;
-	default:
-		return NOTIFY_BAD;
-	}
-
-	err |= kho_preserve_folio(state->fdt);
-	err |= kho_add_subtree(ser, KHO_TEST_FDT, folio_address(state->fdt));
+	for (int i = 0; i < state->nr_folios_preserved; i++)
+		kho_unpreserve_folio(state->folios[i]);
 
-	return err ? NOTIFY_BAD : NOTIFY_DONE;
+	kho_unpreserve_vmalloc(&state->folios_info_phys);
+	vfree(state->folios_info);
 }
 
-static struct notifier_block kho_test_nb = {
-	.notifier_call = kho_test_notifier,
-};
-
-static int kho_test_save_data(struct kho_test_state *state, void *fdt)
+static int kho_test_preserve_data(struct kho_test_state *state)
 {
-	phys_addr_t *folios_info __free(kvfree) = NULL;
 	struct kho_vmalloc folios_info_phys;
-	int err = 0;
+	phys_addr_t *folios_info;
+	int err;
 
 	folios_info = vmalloc_array(state->nr_folios, sizeof(*folios_info));
 	if (!folios_info)
@@ -78,62 +63,98 @@ static int kho_test_save_data(struct kho_test_state *state, void *fdt)
 
 	err = kho_preserve_vmalloc(folios_info, &folios_info_phys);
 	if (err)
-		return err;
+		goto err_free_info;
+
+	state->folios_info_phys = folios_info_phys;
+	state->folios_info = folios_info;
 
 	for (int i = 0; i < state->nr_folios; i++) {
 		struct folio *folio = state->folios[i];
 		unsigned int order = folio_order(folio);
 
 		folios_info[i] = virt_to_phys(folio_address(folio)) | order;
-
 		err = kho_preserve_folio(folio);
 		if (err)
-			break;
+			goto err_unpreserve;
+		state->nr_folios_preserved++;
 	}
 
+	return 0;
+
+err_unpreserve:
+	/*
+	 * kho_test_unpreserve_data frees folio_info, bail out immediately to
+	 * avoid double free
+	 */
+	kho_test_unpreserve_data(state);
+	return err;
+
+err_free_info:
+	vfree(folios_info);
+	return err;
+}
+
+static int kho_test_prepare_fdt(struct kho_test_state *state, ssize_t fdt_size)
+{
+	const char compatible[] = KHO_TEST_COMPAT;
+	unsigned int magic = KHO_TEST_MAGIC;
+	void *fdt = folio_address(state->fdt);
+	int err;
+
+	err = fdt_create(fdt, fdt_size);
+	err |= fdt_finish_reservemap(fdt);
+	err |= fdt_begin_node(fdt, "");
+	err |= fdt_property(fdt, "compatible", compatible, sizeof(compatible));
+	err |= fdt_property(fdt, "magic", &magic, sizeof(magic));
+
 	err |= fdt_begin_node(fdt, "data");
 	err |= fdt_property(fdt, "nr_folios", &state->nr_folios,
 			    sizeof(state->nr_folios));
-	err |= fdt_property(fdt, "folios_info", &folios_info_phys,
-			    sizeof(folios_info_phys));
+	err |= fdt_property(fdt, "folios_info", &state->folios_info_phys,
+			    sizeof(state->folios_info_phys));
 	err |= fdt_property(fdt, "csum", &state->csum, sizeof(state->csum));
 	err |= fdt_end_node(fdt);
 
-	if (!err)
-		state->folios_info = no_free_ptr(folios_info);
+	err |= fdt_end_node(fdt);
+	err |= fdt_finish(fdt);
 
 	return err;
 }
 
-static int kho_test_prepare_fdt(struct kho_test_state *state)
+static int kho_test_preserve(struct kho_test_state *state)
 {
-	const char compatible[] = KHO_TEST_COMPAT;
-	unsigned int magic = KHO_TEST_MAGIC;
 	ssize_t fdt_size;
-	int err = 0;
-	void *fdt;
+	int err;
 
 	fdt_size = state->nr_folios * sizeof(phys_addr_t) + PAGE_SIZE;
 	state->fdt = folio_alloc(GFP_KERNEL, get_order(fdt_size));
 	if (!state->fdt)
 		return -ENOMEM;
 
-	fdt = folio_address(state->fdt);
-
-	err |= fdt_create(fdt, fdt_size);
-	err |= fdt_finish_reservemap(fdt);
+	err = kho_preserve_folio(state->fdt);
+	if (err)
+		goto err_free_fdt;
 
-	err |= fdt_begin_node(fdt, "");
-	err |= fdt_property(fdt, "compatible", compatible, sizeof(compatible));
-	err |= fdt_property(fdt, "magic", &magic, sizeof(magic));
-	err |= kho_test_save_data(state, fdt);
-	err |= fdt_end_node(fdt);
+	err = kho_test_preserve_data(state);
+	if (err)
+		goto err_unpreserve_fdt;
 
-	err |= fdt_finish(fdt);
+	err = kho_test_prepare_fdt(state, fdt_size);
+	if (err)
+		goto err_unpreserve_data;
 
+	err = kho_add_subtree(KHO_TEST_FDT, folio_address(state->fdt));
 	if (err)
-		folio_put(state->fdt);
+		goto err_unpreserve_data;
+
+	return 0;
 
+err_unpreserve_data:
+	kho_test_unpreserve_data(state);
+err_unpreserve_fdt:
+	kho_unpreserve_folio(state->fdt);
+err_free_fdt:
+	folio_put(state->fdt);
 	return err;
 }
 
@@ -190,7 +211,7 @@ static int kho_test_save(void)
 	max_mem = PAGE_ALIGN(max_mem);
 	max_nr = max_mem >> PAGE_SHIFT;
 
-	folios = kvmalloc_array(max_nr, sizeof(*state->folios), GFP_KERNEL);
+	folios = kvmalloc_objs(*state->folios, max_nr);
 	if (!folios)
 		return -ENOMEM;
 	state->folios = folios;
@@ -199,18 +220,12 @@ static int kho_test_save(void)
 	if (err)
 		goto err_free_folios;
 
-	err = kho_test_prepare_fdt(state);
+	err = kho_test_preserve(state);
 	if (err)
 		goto err_free_folios;
 
-	err = register_kho_notifier(&kho_test_nb);
-	if (err)
-		goto err_free_fdt;
-
 	return 0;
 
-err_free_fdt:
-	folio_put(state->fdt);
 err_free_folios:
 	kvfree(folios);
 	return err;
@@ -292,7 +307,6 @@ static int kho_test_restore(phys_addr_t fdt_phys)
 	if (err)
 		return err;
 
-	pr_info("KHO restore succeeded\n");
 	return 0;
 }
 
@@ -305,8 +319,15 @@ static int __init kho_test_init(void)
 		return 0;
 
 	err = kho_retrieve_subtree(KHO_TEST_FDT, &fdt_phys);
-	if (!err)
-		return kho_test_restore(fdt_phys);
+	if (!err) {
+		err = kho_test_restore(fdt_phys);
+		if (err)
+			pr_err("KHO restore failed\n");
+		else
+			pr_info("KHO restore succeeded\n");
+
+		return err;
+	}
 
 	if (err != -ENOENT) {
 		pr_warn("failed to retrieve %s FDT: %d\n", KHO_TEST_FDT, err);
@@ -319,17 +340,21 @@ module_init(kho_test_init);
 
 static void kho_test_cleanup(void)
 {
+	/* unpreserve and free the data stored in folios */
+	kho_test_unpreserve_data(&kho_test_state);
 	for (int i = 0; i < kho_test_state.nr_folios; i++)
 		folio_put(kho_test_state.folios[i]);
 
 	kvfree(kho_test_state.folios);
-	vfree(kho_test_state.folios_info);
+
+	/* Unpreserve and release the FDT folio */
+	kho_unpreserve_folio(kho_test_state.fdt);
 	folio_put(kho_test_state.fdt);
 }
 
 static void __exit kho_test_exit(void)
 {
-	unregister_kho_notifier(&kho_test_nb);
+	kho_remove_subtree(folio_address(kho_test_state.fdt));
 	kho_test_cleanup();
 }
 module_exit(kho_test_exit);
diff --git a/lib/test_memcat_p.c b/lib/test_memcat_p.c
index 7e0797a6bebf..62f1633b30f8 100644
--- a/lib/test_memcat_p.c
+++ b/lib/test_memcat_p.c
@@ -24,20 +24,20 @@ static int __init test_memcat_p_init(void)
 	struct test_struct **in0, **in1, **out, **p;
 	int err = -ENOMEM, i, r, total = 0;
 
-	in0 = kcalloc(INPUT_MAX, sizeof(*in0), GFP_KERNEL);
+	in0 = kzalloc_objs(*in0, INPUT_MAX);
 	if (!in0)
 		return err;
 
-	in1 = kcalloc(INPUT_MAX, sizeof(*in1), GFP_KERNEL);
+	in1 = kzalloc_objs(*in1, INPUT_MAX);
 	if (!in1)
 		goto err_free_in0;
 
 	for (i = 0, r = 1; i < INPUT_MAX - 1; i++) {
-		in0[i] = kmalloc(sizeof(**in0), GFP_KERNEL);
+		in0[i] = kmalloc_obj(**in0);
 		if (!in0[i])
 			goto err_free_elements;
 
-		in1[i] = kmalloc(sizeof(**in1), GFP_KERNEL);
+		in1[i] = kmalloc_obj(**in1);
 		if (!in1[i]) {
 			kfree(in0[i]);
 			goto err_free_elements;
diff --git a/lib/test_objagg.c b/lib/test_objagg.c
index ce5c4c36a084..f21e3ae01395 100644
--- a/lib/test_objagg.c
+++ b/lib/test_objagg.c
@@ -107,7 +107,7 @@ static void *delta_create(void *priv, void *parent_obj, void *obj)
 	if (!delta_check(priv, parent_obj, obj))
 		return ERR_PTR(-EINVAL);
 
-	delta = kzalloc(sizeof(*delta), GFP_KERNEL);
+	delta = kzalloc_obj(*delta);
 	if (!delta)
 		return ERR_PTR(-ENOMEM);
 	delta->key_id_diff = diff;
@@ -130,7 +130,7 @@ static void *root_create(void *priv, void *obj, unsigned int id)
 	struct tokey *key = obj;
 	struct root *root;
 
-	root = kzalloc(sizeof(*root), GFP_KERNEL);
+	root = kzalloc_obj(*root);
 	if (!root)
 		return ERR_PTR(-ENOMEM);
 	memcpy(&root->key, key, sizeof(root->key));
diff --git a/lib/test_parman.c b/lib/test_parman.c
index f9b97426a337..28f0951189ab 100644
--- a/lib/test_parman.c
+++ b/lib/test_parman.c
@@ -219,7 +219,7 @@ static struct test_parman *test_parman_create(const struct parman_ops *ops)
 	struct test_parman *test_parman;
 	int err;
 
-	test_parman = kzalloc(sizeof(*test_parman), GFP_KERNEL);
+	test_parman = kzalloc_obj(*test_parman);
 	if (!test_parman)
 		return ERR_PTR(-ENOMEM);
 	err = test_parman_resize(test_parman, TEST_PARMAN_BASE_COUNT);
diff --git a/lib/test_rhashtable.c b/lib/test_rhashtable.c
index c63db03ebb9d..0b33559a910b 100644
--- a/lib/test_rhashtable.c
+++ b/lib/test_rhashtable.c
@@ -524,7 +524,7 @@ static int __init test_insert_dup(struct test_obj_rhl *rhl_test_objects,
 	const char *key;
 	int err = 0;
 
-	rhlt = kmalloc(sizeof(*rhlt), GFP_KERNEL);
+	rhlt = kmalloc_obj(*rhlt);
 	if (WARN_ON(!rhlt))
 		return -EINVAL;
 
diff --git a/lib/test_uuid.c b/lib/test_uuid.c
deleted file mode 100644
index 0124fad5d72c..000000000000
--- a/lib/test_uuid.c
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Test cases for lib/uuid.c module.
- */
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/init.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/string.h>
-#include <linux/uuid.h>
-
-struct test_uuid_data {
-	const char *uuid;
-	guid_t le;
-	uuid_t be;
-};
-
-static const struct test_uuid_data test_uuid_test_data[] = {
-	{
-		.uuid = "c33f4995-3701-450e-9fbf-206a2e98e576",
-		.le = GUID_INIT(0xc33f4995, 0x3701, 0x450e, 0x9f, 0xbf, 0x20, 0x6a, 0x2e, 0x98, 0xe5, 0x76),
-		.be = UUID_INIT(0xc33f4995, 0x3701, 0x450e, 0x9f, 0xbf, 0x20, 0x6a, 0x2e, 0x98, 0xe5, 0x76),
-	},
-	{
-		.uuid = "64b4371c-77c1-48f9-8221-29f054fc023b",
-		.le = GUID_INIT(0x64b4371c, 0x77c1, 0x48f9, 0x82, 0x21, 0x29, 0xf0, 0x54, 0xfc, 0x02, 0x3b),
-		.be = UUID_INIT(0x64b4371c, 0x77c1, 0x48f9, 0x82, 0x21, 0x29, 0xf0, 0x54, 0xfc, 0x02, 0x3b),
-	},
-	{
-		.uuid = "0cb4ddff-a545-4401-9d06-688af53e7f84",
-		.le = GUID_INIT(0x0cb4ddff, 0xa545, 0x4401, 0x9d, 0x06, 0x68, 0x8a, 0xf5, 0x3e, 0x7f, 0x84),
-		.be = UUID_INIT(0x0cb4ddff, 0xa545, 0x4401, 0x9d, 0x06, 0x68, 0x8a, 0xf5, 0x3e, 0x7f, 0x84),
-	},
-};
-
-static const char * const test_uuid_wrong_data[] = {
-	"c33f4995-3701-450e-9fbf206a2e98e576 ",	/* no hyphen(s) */
-	"64b4371c-77c1-48f9-8221-29f054XX023b",	/* invalid character(s) */
-	"0cb4ddff-a545-4401-9d06-688af53e",	/* not enough data */
-};
-
-static unsigned total_tests __initdata;
-static unsigned failed_tests __initdata;
-
-static void __init test_uuid_failed(const char *prefix, bool wrong, bool be,
-				    const char *data, const char *actual)
-{
-	pr_err("%s test #%u %s %s data: '%s'\n",
-	       prefix,
-	       total_tests,
-	       wrong ? "passed on wrong" : "failed on",
-	       be ? "BE" : "LE",
-	       data);
-	if (actual && *actual)
-		pr_err("%s test #%u actual data: '%s'\n",
-		       prefix,
-		       total_tests,
-		       actual);
-	failed_tests++;
-}
-
-static void __init test_uuid_test(const struct test_uuid_data *data)
-{
-	guid_t le;
-	uuid_t be;
-	char buf[48];
-
-	/* LE */
-	total_tests++;
-	if (guid_parse(data->uuid, &le))
-		test_uuid_failed("conversion", false, false, data->uuid, NULL);
-
-	total_tests++;
-	if (!guid_equal(&data->le, &le)) {
-		sprintf(buf, "%pUl", &le);
-		test_uuid_failed("cmp", false, false, data->uuid, buf);
-	}
-
-	/* BE */
-	total_tests++;
-	if (uuid_parse(data->uuid, &be))
-		test_uuid_failed("conversion", false, true, data->uuid, NULL);
-
-	total_tests++;
-	if (!uuid_equal(&data->be, &be)) {
-		sprintf(buf, "%pUb", &be);
-		test_uuid_failed("cmp", false, true, data->uuid, buf);
-	}
-}
-
-static void __init test_uuid_wrong(const char *data)
-{
-	guid_t le;
-	uuid_t be;
-
-	/* LE */
-	total_tests++;
-	if (!guid_parse(data, &le))
-		test_uuid_failed("negative", true, false, data, NULL);
-
-	/* BE */
-	total_tests++;
-	if (!uuid_parse(data, &be))
-		test_uuid_failed("negative", true, true, data, NULL);
-}
-
-static int __init test_uuid_init(void)
-{
-	unsigned int i;
-
-	for (i = 0; i < ARRAY_SIZE(test_uuid_test_data); i++)
-		test_uuid_test(&test_uuid_test_data[i]);
-
-	for (i = 0; i < ARRAY_SIZE(test_uuid_wrong_data); i++)
-		test_uuid_wrong(test_uuid_wrong_data[i]);
-
-	if (failed_tests == 0)
-		pr_info("all %u tests passed\n", total_tests);
-	else
-		pr_err("failed %u out of %u tests\n", failed_tests, total_tests);
-
-	return failed_tests ? -EINVAL : 0;
-}
-module_init(test_uuid_init);
-
-static void __exit test_uuid_exit(void)
-{
-	/* do nothing */
-}
-module_exit(test_uuid_exit);
-
-MODULE_AUTHOR("Andy Shevchenko <andriy.shevchenko@linux.intel.com>");
-MODULE_DESCRIPTION("Test cases for lib/uuid.c module");
-MODULE_LICENSE("Dual BSD/GPL");
diff --git a/lib/test_vmalloc.c b/lib/test_vmalloc.c
index 2815658ccc37..876c72c18a0c 100644
--- a/lib/test_vmalloc.c
+++ b/lib/test_vmalloc.c
@@ -54,9 +54,13 @@ __param(int, run_test_mask, 7,
 		"\t\tid: 256,  name: kvfree_rcu_1_arg_vmalloc_test\n"
 		"\t\tid: 512,  name: kvfree_rcu_2_arg_vmalloc_test\n"
 		"\t\tid: 1024, name: vm_map_ram_test\n"
+		"\t\tid: 2048, name: no_block_alloc_test\n"
 		/* Add a new test case description here. */
 );
 
+__param(int, nr_pcpu_objects, 35000,
+	"Number of pcpu objects to allocate for pcpu_alloc_test");
+
 /*
  * This is for synchronization of setup phase.
  */
@@ -283,6 +287,30 @@ static int fix_size_alloc_test(void)
 	return 0;
 }
 
+static int no_block_alloc_test(void)
+{
+	void *ptr;
+	int i;
+
+	for (i = 0; i < test_loop_count; i++) {
+		bool use_atomic = !!(get_random_u8() % 2);
+		gfp_t gfp = use_atomic ? GFP_ATOMIC : GFP_NOWAIT;
+		unsigned long size = (nr_pages > 0 ? nr_pages : 1) * PAGE_SIZE;
+
+		preempt_disable();
+		ptr = __vmalloc(size, gfp);
+		preempt_enable();
+
+		if (!ptr)
+			return -1;
+
+		*((__u8 *)ptr) = 0;
+		vfree(ptr);
+	}
+
+	return 0;
+}
+
 static int
 pcpu_alloc_test(void)
 {
@@ -292,24 +320,24 @@ pcpu_alloc_test(void)
 	size_t size, align;
 	int i;
 
-	pcpu = vmalloc(sizeof(void __percpu *) * 35000);
+	pcpu = vmalloc(sizeof(void __percpu *) * nr_pcpu_objects);
 	if (!pcpu)
 		return -1;
 
-	for (i = 0; i < 35000; i++) {
+	for (i = 0; i < nr_pcpu_objects; i++) {
 		size = get_random_u32_inclusive(1, PAGE_SIZE / 4);
 
 		/*
 		 * Maximum PAGE_SIZE
 		 */
-		align = 1 << get_random_u32_inclusive(1, 11);
+		align = 1 << get_random_u32_inclusive(1, PAGE_SHIFT - 1);
 
 		pcpu[i] = __alloc_percpu(size, align);
 		if (!pcpu[i])
 			rv = -1;
 	}
 
-	for (i = 0; i < 35000; i++)
+	for (i = 0; i < nr_pcpu_objects; i++)
 		free_percpu(pcpu[i]);
 
 	vfree(pcpu);
@@ -368,7 +396,7 @@ vm_map_ram_test(void)
 	int i;
 
 	map_nr_pages = nr_pages > 0 ? nr_pages:1;
-	pages = kcalloc(map_nr_pages, sizeof(struct page *), GFP_KERNEL);
+	pages = kzalloc_objs(struct page *, map_nr_pages);
 	if (!pages)
 		return -1;
 
@@ -411,6 +439,7 @@ static struct test_case_desc test_case_array[] = {
 	{ "kvfree_rcu_1_arg_vmalloc_test", kvfree_rcu_1_arg_vmalloc_test, },
 	{ "kvfree_rcu_2_arg_vmalloc_test", kvfree_rcu_2_arg_vmalloc_test, },
 	{ "vm_map_ram_test", vm_map_ram_test, },
+	{ "no_block_alloc_test", no_block_alloc_test, true },
 	/* Add a new test case here. */
 };
 
@@ -474,7 +503,7 @@ static int test_func(void *private)
 		for (j = 0; j < test_repeat_count; j++) {
 			ret = test_case_array[index].test_func();
 
-			if (!ret && !test_case_array[index].xfail)
+			if (!ret)
 				t->data[index].test_passed++;
 			else if (ret && test_case_array[index].xfail)
 				t->data[index].test_xfailed++;
@@ -513,7 +542,7 @@ init_test_configuration(void)
 	nr_threads = clamp(nr_threads, 1, (int) USHRT_MAX);
 
 	/* Allocate the space for test instances. */
-	tdriver = kvcalloc(nr_threads, sizeof(*tdriver), GFP_KERNEL);
+	tdriver = kvzalloc_objs(*tdriver, nr_threads);
 	if (tdriver == NULL)
 		return -1;
 
diff --git a/lib/tests/Makefile b/lib/tests/Makefile
index f7460831cfdd..05f74edbc62b 100644
--- a/lib/tests/Makefile
+++ b/lib/tests/Makefile
@@ -4,6 +4,8 @@
 
 # KUnit tests
 CFLAGS_bitfield_kunit.o := $(DISABLE_STRUCTLEAK_PLUGIN)
+obj-$(CONFIG_BASE64_KUNIT) += base64_kunit.o
+obj-$(CONFIG_BITOPS_KUNIT) += bitops_kunit.o
 obj-$(CONFIG_BITFIELD_KUNIT) += bitfield_kunit.o
 obj-$(CONFIG_BITS_TEST) += test_bits.o
 obj-$(CONFIG_BLACKHOLE_DEV_KUNIT_TEST) += blackhole_dev_kunit.o
@@ -18,20 +20,24 @@ CFLAGS_fortify_kunit.o += $(DISABLE_STRUCTLEAK_PLUGIN)
 obj-$(CONFIG_FORTIFY_KUNIT_TEST) += fortify_kunit.o
 CFLAGS_test_fprobe.o += $(CC_FLAGS_FTRACE)
 obj-$(CONFIG_FPROBE_SANITY_TEST) += test_fprobe.o
+obj-$(CONFIG_GLOB_KUNIT_TEST) += glob_kunit.o
 obj-$(CONFIG_HASHTABLE_KUNIT_TEST) += hashtable_test.o
 obj-$(CONFIG_HASH_KUNIT_TEST) += test_hash.o
 obj-$(CONFIG_TEST_IOV_ITER) += kunit_iov_iter.o
 obj-$(CONFIG_IS_SIGNED_TYPE_KUNIT_TEST) += is_signed_type_kunit.o
 obj-$(CONFIG_KPROBES_SANITY_TEST) += test_kprobes.o
 obj-$(CONFIG_LIST_KUNIT_TEST) += list-test.o
+obj-$(CONFIG_LIST_PRIVATE_KUNIT_TEST) += list-private-test.o
 obj-$(CONFIG_KFIFO_KUNIT_TEST) += kfifo_kunit.o
 obj-$(CONFIG_TEST_LIST_SORT) += test_list_sort.o
 obj-$(CONFIG_LINEAR_RANGES_TEST) += test_linear_ranges.o
+obj-$(CONFIG_LIVEUPDATE_TEST) += liveupdate.o
 
 CFLAGS_longest_symbol_kunit.o += $(call cc-disable-warning, missing-prototypes)
 obj-$(CONFIG_LONGEST_SYM_KUNIT_TEST) += longest_symbol_kunit.o
 
 obj-$(CONFIG_MEMCPY_KUNIT_TEST) += memcpy_kunit.o
+obj-$(CONFIG_MIN_HEAP_KUNIT_TEST) += min_heap_kunit.o
 CFLAGS_overflow_kunit.o = $(call cc-disable-warning, tautological-constant-out-of-range-compare)
 obj-$(CONFIG_OVERFLOW_KUNIT_TEST) += overflow_kunit.o
 obj-$(CONFIG_PRINTF_KUNIT_TEST) += printf_kunit.o
@@ -48,5 +54,6 @@ obj-$(CONFIG_STRING_HELPERS_KUNIT_TEST) += string_helpers_kunit.o
 obj-$(CONFIG_USERCOPY_KUNIT_TEST) += usercopy_kunit.o
 obj-$(CONFIG_UTIL_MACROS_KUNIT) += util_macros_kunit.o
 obj-$(CONFIG_RATELIMIT_KUNIT_TEST) += test_ratelimit.o
+obj-$(CONFIG_UUID_KUNIT_TEST) += uuid_kunit.o
 
 obj-$(CONFIG_TEST_RUNTIME_MODULE)		+= module/
diff --git a/lib/tests/base64_kunit.c b/lib/tests/base64_kunit.c
new file mode 100644
index 000000000000..f7252070c359
--- /dev/null
+++ b/lib/tests/base64_kunit.c
@@ -0,0 +1,294 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * base64_kunit_test.c - KUnit tests for base64 encoding and decoding functions
+ *
+ * Copyright (c) 2025, Guan-Chun Wu <409411716@gms.tku.edu.tw>
+ */
+
+#include <kunit/test.h>
+#include <linux/base64.h>
+
+/* ---------- Benchmark helpers ---------- */
+static u64 bench_encode_ns(const u8 *data, int len, char *dst, int reps,
+			   enum base64_variant variant)
+{
+	u64 t0, t1;
+
+	t0 = ktime_get_ns();
+	for (int i = 0; i < reps; i++)
+		base64_encode(data, len, dst, true, variant);
+	t1 = ktime_get_ns();
+
+	return div64_u64(t1 - t0, (u64)reps);
+}
+
+static u64 bench_decode_ns(const char *data, int len, u8 *dst, int reps,
+			   enum base64_variant variant)
+{
+	u64 t0, t1;
+
+	t0 = ktime_get_ns();
+	for (int i = 0; i < reps; i++)
+		base64_decode(data, len, dst, true, variant);
+	t1 = ktime_get_ns();
+
+	return div64_u64(t1 - t0, (u64)reps);
+}
+
+static void run_perf_and_check(struct kunit *test, const char *label, int size,
+			       enum base64_variant variant)
+{
+	const int reps = 1000;
+	size_t outlen = DIV_ROUND_UP(size, 3) * 4;
+	u8 *in = kmalloc(size, GFP_KERNEL);
+	char *enc = kmalloc(outlen, GFP_KERNEL);
+	u8 *decoded = kmalloc(size, GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, in);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, enc);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, decoded);
+
+	get_random_bytes(in, size);
+	int enc_len = base64_encode(in, size, enc, true, variant);
+	int dec_len = base64_decode(enc, enc_len, decoded, true, variant);
+
+	/* correctness sanity check */
+	KUNIT_EXPECT_EQ(test, dec_len, size);
+	KUNIT_EXPECT_MEMEQ(test, decoded, in, size);
+
+	/* benchmark encode */
+
+	u64 t1 = bench_encode_ns(in, size, enc, reps, variant);
+
+	kunit_info(test, "[%s] encode run : %lluns", label, t1);
+
+	u64 t2 = bench_decode_ns(enc, enc_len, decoded, reps, variant);
+
+	kunit_info(test, "[%s] decode run : %lluns", label, t2);
+
+	kfree(in);
+	kfree(enc);
+	kfree(decoded);
+}
+
+static void base64_performance_tests(struct kunit *test)
+{
+	/* run on STD variant only */
+	run_perf_and_check(test, "64B", 64, BASE64_STD);
+	run_perf_and_check(test, "1KB", 1024, BASE64_STD);
+}
+
+/* ---------- Helpers for encode ---------- */
+static void expect_encode_ok(struct kunit *test, const u8 *src, int srclen,
+			     const char *expected, bool padding,
+			     enum base64_variant variant)
+{
+	char buf[128];
+	int encoded_len = base64_encode(src, srclen, buf, padding, variant);
+
+	buf[encoded_len] = '\0';
+
+	KUNIT_EXPECT_EQ(test, encoded_len, strlen(expected));
+	KUNIT_EXPECT_STREQ(test, buf, expected);
+}
+
+/* ---------- Helpers for decode ---------- */
+static void expect_decode_ok(struct kunit *test, const char *src,
+			     const u8 *expected, int expected_len, bool padding,
+			     enum base64_variant variant)
+{
+	u8 buf[128];
+	int decoded_len = base64_decode(src, strlen(src), buf, padding, variant);
+
+	KUNIT_EXPECT_EQ(test, decoded_len, expected_len);
+	KUNIT_EXPECT_MEMEQ(test, buf, expected, expected_len);
+}
+
+static void expect_decode_err(struct kunit *test, const char *src,
+			      int srclen, bool padding,
+			      enum base64_variant variant)
+{
+	u8 buf[64];
+	int decoded_len = base64_decode(src, srclen, buf, padding, variant);
+
+	KUNIT_EXPECT_EQ(test, decoded_len, -1);
+}
+
+/* ---------- Encode Tests ---------- */
+static void base64_std_encode_tests(struct kunit *test)
+{
+	/* With padding */
+	expect_encode_ok(test, (const u8 *)"", 0, "", true, BASE64_STD);
+	expect_encode_ok(test, (const u8 *)"f", 1, "Zg==", true, BASE64_STD);
+	expect_encode_ok(test, (const u8 *)"fo", 2, "Zm8=", true, BASE64_STD);
+	expect_encode_ok(test, (const u8 *)"foo", 3, "Zm9v", true, BASE64_STD);
+	expect_encode_ok(test, (const u8 *)"foob", 4, "Zm9vYg==", true, BASE64_STD);
+	expect_encode_ok(test, (const u8 *)"fooba", 5, "Zm9vYmE=", true, BASE64_STD);
+	expect_encode_ok(test, (const u8 *)"foobar", 6, "Zm9vYmFy", true, BASE64_STD);
+
+	/* Extra cases with padding */
+	expect_encode_ok(test, (const u8 *)"Hello, world!", 13, "SGVsbG8sIHdvcmxkIQ==",
+			 true, BASE64_STD);
+	expect_encode_ok(test, (const u8 *)"ABCDEFGHIJKLMNOPQRSTUVWXYZ", 26,
+			 "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo=", true, BASE64_STD);
+	expect_encode_ok(test, (const u8 *)"abcdefghijklmnopqrstuvwxyz", 26,
+			 "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo=", true, BASE64_STD);
+	expect_encode_ok(test, (const u8 *)"0123456789+/", 12, "MDEyMzQ1Njc4OSsv",
+			 true, BASE64_STD);
+
+	/* Without padding */
+	expect_encode_ok(test, (const u8 *)"", 0, "", false, BASE64_STD);
+	expect_encode_ok(test, (const u8 *)"f", 1, "Zg", false, BASE64_STD);
+	expect_encode_ok(test, (const u8 *)"fo", 2, "Zm8", false, BASE64_STD);
+	expect_encode_ok(test, (const u8 *)"foo", 3, "Zm9v", false, BASE64_STD);
+	expect_encode_ok(test, (const u8 *)"foob", 4, "Zm9vYg", false, BASE64_STD);
+	expect_encode_ok(test, (const u8 *)"fooba", 5, "Zm9vYmE", false, BASE64_STD);
+	expect_encode_ok(test, (const u8 *)"foobar", 6, "Zm9vYmFy", false, BASE64_STD);
+
+	/* Extra cases without padding */
+	expect_encode_ok(test, (const u8 *)"Hello, world!", 13, "SGVsbG8sIHdvcmxkIQ",
+			 false, BASE64_STD);
+	expect_encode_ok(test, (const u8 *)"ABCDEFGHIJKLMNOPQRSTUVWXYZ", 26,
+			 "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo", false, BASE64_STD);
+	expect_encode_ok(test, (const u8 *)"abcdefghijklmnopqrstuvwxyz", 26,
+			 "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo", false, BASE64_STD);
+	expect_encode_ok(test, (const u8 *)"0123456789+/", 12, "MDEyMzQ1Njc4OSsv",
+			 false, BASE64_STD);
+}
+
+/* ---------- Decode Tests ---------- */
+static void base64_std_decode_tests(struct kunit *test)
+{
+	/* -------- With padding --------*/
+	expect_decode_ok(test, "", (const u8 *)"", 0, true, BASE64_STD);
+	expect_decode_ok(test, "Zg==", (const u8 *)"f", 1, true, BASE64_STD);
+	expect_decode_ok(test, "Zm8=", (const u8 *)"fo", 2, true, BASE64_STD);
+	expect_decode_ok(test, "Zm9v", (const u8 *)"foo", 3, true, BASE64_STD);
+	expect_decode_ok(test, "Zm9vYg==", (const u8 *)"foob", 4, true, BASE64_STD);
+	expect_decode_ok(test, "Zm9vYmE=", (const u8 *)"fooba", 5, true, BASE64_STD);
+	expect_decode_ok(test, "Zm9vYmFy", (const u8 *)"foobar", 6, true, BASE64_STD);
+	expect_decode_ok(test, "SGVsbG8sIHdvcmxkIQ==", (const u8 *)"Hello, world!", 13,
+			 true, BASE64_STD);
+	expect_decode_ok(test, "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo=",
+			 (const u8 *)"ABCDEFGHIJKLMNOPQRSTUVWXYZ", 26, true, BASE64_STD);
+	expect_decode_ok(test, "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo=",
+			 (const u8 *)"abcdefghijklmnopqrstuvwxyz", 26, true, BASE64_STD);
+
+	/* Error cases */
+	expect_decode_err(test, "Zg=!", 4, true, BASE64_STD);
+	expect_decode_err(test, "Zm$=", 4, true, BASE64_STD);
+	expect_decode_err(test, "Z===", 4, true, BASE64_STD);
+	expect_decode_err(test, "Zg", 2, true, BASE64_STD);
+	expect_decode_err(test, "Zm9v====", 8, true, BASE64_STD);
+	expect_decode_err(test, "Zm==A", 5, true, BASE64_STD);
+
+	{
+		char with_nul[4] = { 'Z', 'g', '\0', '=' };
+
+		expect_decode_err(test, with_nul, 4, true, BASE64_STD);
+	}
+
+	/* -------- Without padding --------*/
+	expect_decode_ok(test, "", (const u8 *)"", 0, false, BASE64_STD);
+	expect_decode_ok(test, "Zg", (const u8 *)"f", 1, false, BASE64_STD);
+	expect_decode_ok(test, "Zm8", (const u8 *)"fo", 2, false, BASE64_STD);
+	expect_decode_ok(test, "Zm9v", (const u8 *)"foo", 3, false, BASE64_STD);
+	expect_decode_ok(test, "Zm9vYg", (const u8 *)"foob", 4, false, BASE64_STD);
+	expect_decode_ok(test, "Zm9vYmE", (const u8 *)"fooba", 5, false, BASE64_STD);
+	expect_decode_ok(test, "Zm9vYmFy", (const u8 *)"foobar", 6, false, BASE64_STD);
+	expect_decode_ok(test, "TWFu", (const u8 *)"Man", 3, false, BASE64_STD);
+	expect_decode_ok(test, "SGVsbG8sIHdvcmxkIQ", (const u8 *)"Hello, world!", 13,
+			 false, BASE64_STD);
+	expect_decode_ok(test, "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo",
+			 (const u8 *)"ABCDEFGHIJKLMNOPQRSTUVWXYZ", 26, false, BASE64_STD);
+	expect_decode_ok(test, "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo",
+			 (const u8 *)"abcdefghijklmnopqrstuvwxyz", 26, false, BASE64_STD);
+	expect_decode_ok(test, "MDEyMzQ1Njc4OSsv", (const u8 *)"0123456789+/", 12,
+			 false, BASE64_STD);
+
+	/* Error cases */
+	expect_decode_err(test, "Zg=!", 4, false, BASE64_STD);
+	expect_decode_err(test, "Zm$=", 4, false, BASE64_STD);
+	expect_decode_err(test, "Z===", 4, false, BASE64_STD);
+	expect_decode_err(test, "Zg=", 3, false, BASE64_STD);
+	expect_decode_err(test, "Zm9v====", 8, false, BASE64_STD);
+	expect_decode_err(test, "Zm==v", 4, false, BASE64_STD);
+
+	{
+		char with_nul[4] = { 'Z', 'g', '\0', '=' };
+
+		expect_decode_err(test, with_nul, 4, false, BASE64_STD);
+	}
+}
+
+/* ---------- Variant tests (URLSAFE / IMAP) ---------- */
+static void base64_variant_tests(struct kunit *test)
+{
+	const u8 sample1[] = { 0x00, 0xfb, 0xff, 0x7f, 0x80 };
+	char std_buf[128], url_buf[128], imap_buf[128];
+	u8 back[128];
+	int n_std, n_url, n_imap, m;
+	int i;
+
+	n_std = base64_encode(sample1, sizeof(sample1), std_buf, false, BASE64_STD);
+	n_url = base64_encode(sample1, sizeof(sample1), url_buf, false, BASE64_URLSAFE);
+	std_buf[n_std] = '\0';
+	url_buf[n_url] = '\0';
+
+	for (i = 0; i < n_std; i++) {
+		if (std_buf[i] == '+')
+			std_buf[i] = '-';
+		else if (std_buf[i] == '/')
+			std_buf[i] = '_';
+	}
+	KUNIT_EXPECT_STREQ(test, std_buf, url_buf);
+
+	m = base64_decode(url_buf, n_url, back, false, BASE64_URLSAFE);
+	KUNIT_EXPECT_EQ(test, m, (int)sizeof(sample1));
+	KUNIT_EXPECT_MEMEQ(test, back, sample1, sizeof(sample1));
+
+	n_std  = base64_encode(sample1, sizeof(sample1), std_buf, false, BASE64_STD);
+	n_imap = base64_encode(sample1, sizeof(sample1), imap_buf, false, BASE64_IMAP);
+	std_buf[n_std]   = '\0';
+	imap_buf[n_imap] = '\0';
+
+	for (i = 0; i < n_std; i++)
+		if (std_buf[i] == '/')
+			std_buf[i] = ',';
+	KUNIT_EXPECT_STREQ(test, std_buf, imap_buf);
+
+	m = base64_decode(imap_buf, n_imap, back, false, BASE64_IMAP);
+	KUNIT_EXPECT_EQ(test, m, (int)sizeof(sample1));
+	KUNIT_EXPECT_MEMEQ(test, back, sample1, sizeof(sample1));
+
+	{
+		const char *bad = "Zg==";
+		u8 tmp[8];
+
+		m = base64_decode(bad, strlen(bad), tmp, false, BASE64_URLSAFE);
+		KUNIT_EXPECT_EQ(test, m, -1);
+
+		m = base64_decode(bad, strlen(bad), tmp, false, BASE64_IMAP);
+		KUNIT_EXPECT_EQ(test, m, -1);
+	}
+}
+
+/* ---------- Test registration ---------- */
+static struct kunit_case base64_test_cases[] = {
+	KUNIT_CASE(base64_performance_tests),
+	KUNIT_CASE(base64_std_encode_tests),
+	KUNIT_CASE(base64_std_decode_tests),
+	KUNIT_CASE(base64_variant_tests),
+	{}
+};
+
+static struct kunit_suite base64_test_suite = {
+	.name = "base64",
+	.test_cases = base64_test_cases,
+};
+
+kunit_test_suite(base64_test_suite);
+
+MODULE_AUTHOR("Guan-Chun Wu <409411716@gms.tku.edu.tw>");
+MODULE_DESCRIPTION("KUnit tests for Base64 encoding/decoding, including performance checks");
+MODULE_LICENSE("GPL");
diff --git a/lib/tests/bitops_kunit.c b/lib/tests/bitops_kunit.c
new file mode 100644
index 000000000000..7fd9d697f131
--- /dev/null
+++ b/lib/tests/bitops_kunit.c
@@ -0,0 +1,205 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 Intel Corporation
+ * Copyright (C) 2026 Ryota Sakamoto <sakamo.ryota@gmail.com>
+ */
+
+#include <linux/bitops.h>
+#include <linux/module.h>
+#include <kunit/test.h>
+
+/* use an enum because that's the most common BITMAP usage */
+enum bitops_fun {
+	BITOPS_4 = 4,
+	BITOPS_7 = 7,
+	BITOPS_11 = 11,
+	BITOPS_31 = 31,
+	BITOPS_88 = 88,
+	BITOPS_LENGTH = 256
+};
+
+struct bitops_test_case {
+	const char *str;
+	const long nr;
+};
+
+static struct bitops_test_case bitops_cases[] = {
+	{
+		.str = "BITOPS_4",
+		.nr = BITOPS_4,
+	},
+	{
+		.str = "BITOPS_7",
+		.nr = BITOPS_7,
+	},
+	{
+		.str = "BITOPS_11",
+		.nr = BITOPS_11,
+	},
+	{
+		.str = "BITOPS_31",
+		.nr = BITOPS_31,
+	},
+	{
+		.str = "BITOPS_88",
+		.nr = BITOPS_88,
+	},
+};
+
+KUNIT_ARRAY_PARAM_DESC(bitops, bitops_cases, str);
+
+static void test_set_bit_clear_bit(struct kunit *test)
+{
+	const struct bitops_test_case *params = test->param_value;
+	DECLARE_BITMAP(bitmap, BITOPS_LENGTH);
+	int bit_set;
+
+	bitmap_zero(bitmap, BITOPS_LENGTH);
+
+	set_bit(params->nr, bitmap);
+	KUNIT_EXPECT_TRUE(test, test_bit(params->nr, bitmap));
+
+	clear_bit(params->nr, bitmap);
+	KUNIT_EXPECT_FALSE(test, test_bit(params->nr, bitmap));
+
+	bit_set = find_first_bit(bitmap, BITOPS_LENGTH);
+	KUNIT_EXPECT_EQ(test, bit_set, BITOPS_LENGTH);
+}
+
+static void test_change_bit(struct kunit *test)
+{
+	const struct bitops_test_case *params = test->param_value;
+	DECLARE_BITMAP(bitmap, BITOPS_LENGTH);
+	int bit_set;
+
+	bitmap_zero(bitmap, BITOPS_LENGTH);
+
+	change_bit(params->nr, bitmap);
+	KUNIT_EXPECT_TRUE(test, test_bit(params->nr, bitmap));
+
+	change_bit(params->nr, bitmap);
+	KUNIT_EXPECT_FALSE(test, test_bit(params->nr, bitmap));
+
+	bit_set = find_first_bit(bitmap, BITOPS_LENGTH);
+	KUNIT_EXPECT_EQ(test, bit_set, BITOPS_LENGTH);
+}
+
+static void test_test_and_set_bit_test_and_clear_bit(struct kunit *test)
+{
+	const struct bitops_test_case *params = test->param_value;
+	DECLARE_BITMAP(bitmap, BITOPS_LENGTH);
+	int bit_set;
+
+	bitmap_zero(bitmap, BITOPS_LENGTH);
+
+	KUNIT_EXPECT_FALSE(test, test_and_set_bit(params->nr, bitmap));
+	KUNIT_EXPECT_TRUE(test, test_bit(params->nr, bitmap));
+
+	KUNIT_EXPECT_TRUE(test, test_and_set_bit(params->nr, bitmap));
+	KUNIT_EXPECT_TRUE(test, test_bit(params->nr, bitmap));
+
+	KUNIT_EXPECT_TRUE(test, test_and_clear_bit(params->nr, bitmap));
+	KUNIT_EXPECT_FALSE(test, test_bit(params->nr, bitmap));
+
+	KUNIT_EXPECT_FALSE(test, test_and_clear_bit(params->nr, bitmap));
+	KUNIT_EXPECT_FALSE(test, test_bit(params->nr, bitmap));
+
+	bit_set = find_first_bit(bitmap, BITOPS_LENGTH);
+	KUNIT_EXPECT_EQ(test, bit_set, BITOPS_LENGTH);
+}
+
+static void test_test_and_change_bit(struct kunit *test)
+{
+	const struct bitops_test_case *params = test->param_value;
+	DECLARE_BITMAP(bitmap, BITOPS_LENGTH);
+	int bit_set;
+
+	bitmap_zero(bitmap, BITOPS_LENGTH);
+
+	KUNIT_EXPECT_FALSE(test, test_and_change_bit(params->nr, bitmap));
+	KUNIT_EXPECT_TRUE(test, test_bit(params->nr, bitmap));
+
+	KUNIT_EXPECT_TRUE(test, test_and_change_bit(params->nr, bitmap));
+	KUNIT_EXPECT_FALSE(test, test_bit(params->nr, bitmap));
+
+	bit_set = find_first_bit(bitmap, BITOPS_LENGTH);
+	KUNIT_EXPECT_EQ(test, bit_set, BITOPS_LENGTH);
+}
+
+struct order_test_case {
+	const char *str;
+	const unsigned int count;
+	const int expected;
+};
+
+static struct order_test_case order_test_cases[] = {
+	{"0x00000003", 0x00000003,  2},
+	{"0x00000004", 0x00000004,  2},
+	{"0x00001fff", 0x00001fff, 13},
+	{"0x00002000", 0x00002000, 13},
+	{"0x50000000", 0x50000000, 31},
+	{"0x80000000", 0x80000000, 31},
+	{"0x80003000", 0x80003000, 32},
+};
+
+KUNIT_ARRAY_PARAM_DESC(order, order_test_cases, str);
+
+static void test_get_count_order(struct kunit *test)
+{
+	const struct order_test_case *params = test->param_value;
+
+	KUNIT_EXPECT_EQ(test, get_count_order(params->count), params->expected);
+	KUNIT_EXPECT_EQ(test, get_count_order_long(params->count), params->expected);
+}
+
+#ifdef CONFIG_64BIT
+struct order_long_test_case {
+	const char *str;
+	const unsigned long count;
+	const int expected;
+};
+
+static struct order_long_test_case order_long_test_cases[] = {
+	{"0x0000000300000000", 0x0000000300000000, 34},
+	{"0x0000000400000000", 0x0000000400000000, 34},
+	{"0x00001fff00000000", 0x00001fff00000000, 45},
+	{"0x0000200000000000", 0x0000200000000000, 45},
+	{"0x5000000000000000", 0x5000000000000000, 63},
+	{"0x8000000000000000", 0x8000000000000000, 63},
+	{"0x8000300000000000", 0x8000300000000000, 64},
+};
+
+KUNIT_ARRAY_PARAM_DESC(order_long, order_long_test_cases, str);
+
+static void test_get_count_order_long(struct kunit *test)
+{
+	const struct order_long_test_case *params = test->param_value;
+
+	KUNIT_EXPECT_EQ(test, get_count_order_long(params->count), params->expected);
+}
+#endif
+
+static struct kunit_case bitops_test_cases[] = {
+	KUNIT_CASE_PARAM(test_set_bit_clear_bit, bitops_gen_params),
+	KUNIT_CASE_PARAM(test_change_bit, bitops_gen_params),
+	KUNIT_CASE_PARAM(test_test_and_set_bit_test_and_clear_bit, bitops_gen_params),
+	KUNIT_CASE_PARAM(test_test_and_change_bit, bitops_gen_params),
+	KUNIT_CASE_PARAM(test_get_count_order, order_gen_params),
+#ifdef CONFIG_64BIT
+	KUNIT_CASE_PARAM(test_get_count_order_long, order_long_gen_params),
+#endif
+	{},
+};
+
+static struct kunit_suite bitops_test_suite = {
+	.name = "bitops",
+	.test_cases = bitops_test_cases,
+};
+
+kunit_test_suite(bitops_test_suite);
+
+MODULE_AUTHOR("Jesse Brandeburg <jesse.brandeburg@intel.com>");
+MODULE_AUTHOR("Wei Yang <richard.weiyang@gmail.com>");
+MODULE_AUTHOR("Ryota Sakamoto <sakamo.ryota@gmail.com>");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Bit testing module");
diff --git a/lib/tests/glob_kunit.c b/lib/tests/glob_kunit.c
new file mode 100644
index 000000000000..362b1eda8e5b
--- /dev/null
+++ b/lib/tests/glob_kunit.c
@@ -0,0 +1,125 @@
+// SPDX-License-Identifier: MIT OR GPL-2.0
+/*
+ * Test cases for glob functions.
+ */
+
+#include <kunit/test.h>
+#include <linux/glob.h>
+#include <linux/module.h>
+
+/**
+ * struct glob_test_case - Test case for glob matching.
+ * @pat: Pattern to match.
+ * @str: String to match against.
+ * @expected: Expected glob_match result, true if matched.
+ */
+struct glob_test_case {
+	const char *pat;
+	const char *str;
+	bool expected;
+};
+
+static const struct glob_test_case glob_test_cases[] = {
+	/* Some basic tests */
+	{ .pat = "a", .str = "a", .expected = true },
+	{ .pat = "a", .str = "b", .expected = false },
+	{ .pat = "a", .str = "aa", .expected = false },
+	{ .pat = "a", .str = "", .expected = false },
+	{ .pat = "", .str = "", .expected = true },
+	{ .pat = "", .str = "a", .expected = false },
+	/* Simple character class tests */
+	{ .pat = "[a]", .str = "a", .expected = true },
+	{ .pat = "[a]", .str = "b", .expected = false },
+	{ .pat = "[!a]", .str = "a", .expected = false },
+	{ .pat = "[!a]", .str = "b", .expected = true },
+	{ .pat = "[ab]", .str = "a", .expected = true },
+	{ .pat = "[ab]", .str = "b", .expected = true },
+	{ .pat = "[ab]", .str = "c", .expected = false },
+	{ .pat = "[!ab]", .str = "c", .expected = true },
+	{ .pat = "[a-c]", .str = "b", .expected = true },
+	{ .pat = "[a-c]", .str = "d", .expected = false },
+	/* Corner cases in character class parsing */
+	{ .pat = "[a-c-e-g]", .str = "-", .expected = true },
+	{ .pat = "[a-c-e-g]", .str = "d", .expected = false },
+	{ .pat = "[a-c-e-g]", .str = "f", .expected = true },
+	{ .pat = "[]a-ceg-ik[]", .str = "a", .expected = true },
+	{ .pat = "[]a-ceg-ik[]", .str = "]", .expected = true },
+	{ .pat = "[]a-ceg-ik[]", .str = "[", .expected = true },
+	{ .pat = "[]a-ceg-ik[]", .str = "h", .expected = true },
+	{ .pat = "[]a-ceg-ik[]", .str = "f", .expected = false },
+	{ .pat = "[!]a-ceg-ik[]", .str = "h", .expected = false },
+	{ .pat = "[!]a-ceg-ik[]", .str = "]", .expected = false },
+	{ .pat = "[!]a-ceg-ik[]", .str = "f", .expected = true },
+	/* Simple wild cards */
+	{ .pat = "?", .str = "a", .expected = true },
+	{ .pat = "?", .str = "aa", .expected = false },
+	{ .pat = "??", .str = "a", .expected = false },
+	{ .pat = "?x?", .str = "axb", .expected = true },
+	{ .pat = "?x?", .str = "abx", .expected = false },
+	{ .pat = "?x?", .str = "xab", .expected = false },
+	/* Asterisk wild cards (backtracking) */
+	{ .pat = "*??", .str = "a", .expected = false },
+	{ .pat = "*??", .str = "ab", .expected = true },
+	{ .pat = "*??", .str = "abc", .expected = true },
+	{ .pat = "*??", .str = "abcd", .expected = true },
+	{ .pat = "??*", .str = "a", .expected = false },
+	{ .pat = "??*", .str = "ab", .expected = true },
+	{ .pat = "??*", .str = "abc", .expected = true },
+	{ .pat = "??*", .str = "abcd", .expected = true },
+	{ .pat = "?*?", .str = "a", .expected = false },
+	{ .pat = "?*?", .str = "ab", .expected = true },
+	{ .pat = "?*?", .str = "abc", .expected = true },
+	{ .pat = "?*?", .str = "abcd", .expected = true },
+	{ .pat = "*b", .str = "b", .expected = true },
+	{ .pat = "*b", .str = "ab", .expected = true },
+	{ .pat = "*b", .str = "ba", .expected = false },
+	{ .pat = "*b", .str = "bb", .expected = true },
+	{ .pat = "*b", .str = "abb", .expected = true },
+	{ .pat = "*b", .str = "bab", .expected = true },
+	{ .pat = "*bc", .str = "abbc", .expected = true },
+	{ .pat = "*bc", .str = "bc", .expected = true },
+	{ .pat = "*bc", .str = "bbc", .expected = true },
+	{ .pat = "*bc", .str = "bcbc", .expected = true },
+	/* Multiple asterisks (complex backtracking) */
+	{ .pat = "*ac*", .str = "abacadaeafag", .expected = true },
+	{ .pat = "*ac*ae*ag*", .str = "abacadaeafag", .expected = true },
+	{ .pat = "*a*b*[bc]*[ef]*g*", .str = "abacadaeafag", .expected = true },
+	{ .pat = "*a*b*[ef]*[cd]*g*", .str = "abacadaeafag", .expected = false },
+	{ .pat = "*abcd*", .str = "abcabcabcabcdefg", .expected = true },
+	{ .pat = "*ab*cd*", .str = "abcabcabcabcdefg", .expected = true },
+	{ .pat = "*abcd*abcdef*", .str = "abcabcdabcdeabcdefg", .expected = true },
+	{ .pat = "*abcd*", .str = "abcabcabcabcefg", .expected = false },
+	{ .pat = "*ab*cd*", .str = "abcabcabcabcefg", .expected = false },
+};
+
+static void glob_case_to_desc(const struct glob_test_case *t, char *desc)
+{
+	snprintf(desc, KUNIT_PARAM_DESC_SIZE, "pat:\"%s\" str:\"%s\"", t->pat, t->str);
+}
+
+KUNIT_ARRAY_PARAM(glob, glob_test_cases, glob_case_to_desc);
+
+static void glob_test_match(struct kunit *test)
+{
+	const struct glob_test_case *params = test->param_value;
+
+	KUNIT_EXPECT_EQ_MSG(test,
+			    glob_match(params->pat, params->str),
+			    params->expected,
+			    "Pattern: \"%s\", String: \"%s\", Expected: %d",
+			    params->pat, params->str, params->expected);
+}
+
+static struct kunit_case glob_kunit_test_cases[] = {
+	KUNIT_CASE_PARAM(glob_test_match, glob_gen_params),
+	{}
+};
+
+static struct kunit_suite glob_test_suite = {
+	.name = "glob",
+	.test_cases = glob_kunit_test_cases,
+};
+
+kunit_test_suite(glob_test_suite);
+MODULE_DESCRIPTION("Test cases for glob functions");
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/lib/tests/kunit_iov_iter.c b/lib/tests/kunit_iov_iter.c
index 48342736d016..bb847e5010eb 100644
--- a/lib/tests/kunit_iov_iter.c
+++ b/lib/tests/kunit_iov_iter.c
@@ -387,7 +387,7 @@ static void __init iov_kunit_load_folioq(struct kunit *test,
 
 	for (i = 0; i < npages; i++) {
 		if (folioq_full(p)) {
-			p->next = kzalloc(sizeof(struct folio_queue), GFP_KERNEL);
+			p->next = kzalloc_obj(struct folio_queue);
 			KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p->next);
 			folioq_init(p->next, 0);
 			p->next->prev = p;
@@ -403,7 +403,7 @@ static struct folio_queue *iov_kunit_create_folioq(struct kunit *test)
 {
 	struct folio_queue *folioq;
 
-	folioq = kzalloc(sizeof(struct folio_queue), GFP_KERNEL);
+	folioq = kzalloc_obj(struct folio_queue);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folioq);
 	kunit_add_action_or_reset(test, iov_kunit_destroy_folioq, folioq);
 	folioq_init(folioq, 0);
@@ -565,7 +565,7 @@ static struct xarray *iov_kunit_create_xarray(struct kunit *test)
 {
 	struct xarray *xarray;
 
-	xarray = kzalloc(sizeof(struct xarray), GFP_KERNEL);
+	xarray = kzalloc_obj(struct xarray);
 	xa_init(xarray);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xarray);
 	kunit_add_action_or_reset(test, iov_kunit_destroy_xarray, xarray);
diff --git a/lib/tests/list-private-test.c b/lib/tests/list-private-test.c
new file mode 100644
index 000000000000..3bd62939ae67
--- /dev/null
+++ b/lib/tests/list-private-test.c
@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KUnit compilation/smoke test for Private list primitives.
+ *
+ * Copyright (c) 2025, Google LLC.
+ * Pasha Tatashin <pasha.tatashin@soleen.com>
+ */
+#include <linux/list_private.h>
+#include <kunit/test.h>
+
+/*
+ * This forces compiler to warn if you access it directly, because list
+ * primitives expect (struct list_head *), not (volatile struct list_head *).
+ */
+#undef __private
+#define __private volatile
+
+/* Redefine ACCESS_PRIVATE for this test. */
+#undef ACCESS_PRIVATE
+#define ACCESS_PRIVATE(p, member) \
+	(*((struct list_head *)((unsigned long)&((p)->member))))
+
+struct list_test_struct {
+	int data;
+	struct list_head __private list;
+};
+
+static void list_private_compile_test(struct kunit *test)
+{
+	struct list_test_struct entry;
+	struct list_test_struct *pos, *n;
+	LIST_HEAD(head);
+
+	INIT_LIST_HEAD(&ACCESS_PRIVATE(&entry, list));
+	list_add(&ACCESS_PRIVATE(&entry, list), &head);
+	pos = &entry;
+
+	pos = list_private_entry(&ACCESS_PRIVATE(&entry, list), struct list_test_struct, list);
+	pos = list_private_first_entry(&head, struct list_test_struct, list);
+	pos = list_private_last_entry(&head, struct list_test_struct, list);
+	pos = list_private_next_entry(pos, list);
+	pos = list_private_prev_entry(pos, list);
+	pos = list_private_next_entry_circular(pos, &head, list);
+	pos = list_private_prev_entry_circular(pos, &head, list);
+
+	if (list_private_entry_is_head(pos, &head, list))
+		return;
+
+	list_private_for_each_entry(pos, &head, list) { }
+	list_private_for_each_entry_reverse(pos, &head, list) { }
+	list_private_for_each_entry_continue(pos, &head, list) { }
+	list_private_for_each_entry_continue_reverse(pos, &head, list) { }
+	list_private_for_each_entry_from(pos, &head, list) { }
+	list_private_for_each_entry_from_reverse(pos, &head, list) { }
+
+	list_private_for_each_entry_safe(pos, n, &head, list)
+		list_private_safe_reset_next(pos, n, list);
+	list_private_for_each_entry_safe_continue(pos, n, &head, list) { }
+	list_private_for_each_entry_safe_from(pos, n, &head, list) { }
+	list_private_for_each_entry_safe_reverse(pos, n, &head, list) { }
+}
+
+static struct kunit_case list_private_test_cases[] = {
+	KUNIT_CASE(list_private_compile_test),
+	{},
+};
+
+static struct kunit_suite list_private_test_module = {
+	.name = "list-private-kunit-test",
+	.test_cases = list_private_test_cases,
+};
+
+kunit_test_suite(list_private_test_module);
+
+MODULE_DESCRIPTION("KUnit compilation test for private list primitives");
+MODULE_LICENSE("GPL");
diff --git a/lib/tests/list-test.c b/lib/tests/list-test.c
index 9135cdc1bb39..6d9227a2b204 100644
--- a/lib/tests/list-test.c
+++ b/lib/tests/list-test.c
@@ -26,10 +26,10 @@ static void list_test_list_init(struct kunit *test)
 
 	INIT_LIST_HEAD(&list2);
 
-	list4 = kzalloc(sizeof(*list4), GFP_KERNEL | __GFP_NOFAIL);
+	list4 = kzalloc_obj(*list4, GFP_KERNEL | __GFP_NOFAIL);
 	INIT_LIST_HEAD(list4);
 
-	list5 = kmalloc(sizeof(*list5), GFP_KERNEL | __GFP_NOFAIL);
+	list5 = kmalloc_obj(*list5, GFP_KERNEL | __GFP_NOFAIL);
 	memset(list5, 0xFF, sizeof(*list5));
 	INIT_LIST_HEAD(list5);
 
@@ -829,10 +829,10 @@ static void hlist_test_init(struct kunit *test)
 
 	INIT_HLIST_HEAD(&list2);
 
-	list4 = kzalloc(sizeof(*list4), GFP_KERNEL | __GFP_NOFAIL);
+	list4 = kzalloc_obj(*list4, GFP_KERNEL | __GFP_NOFAIL);
 	INIT_HLIST_HEAD(list4);
 
-	list5 = kmalloc(sizeof(*list5), GFP_KERNEL | __GFP_NOFAIL);
+	list5 = kmalloc_obj(*list5, GFP_KERNEL | __GFP_NOFAIL);
 	memset(list5, 0xFF, sizeof(*list5));
 	INIT_HLIST_HEAD(list5);
 
diff --git a/lib/tests/liveupdate.c b/lib/tests/liveupdate.c
new file mode 100644
index 000000000000..496d6ef91a30
--- /dev/null
+++ b/lib/tests/liveupdate.c
@@ -0,0 +1,158 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (c) 2025, Google LLC.
+ * Pasha Tatashin <pasha.tatashin@soleen.com>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME " test: " fmt
+
+#include <linux/cleanup.h>
+#include <linux/errno.h>
+#include <linux/init.h>
+#include <linux/liveupdate.h>
+#include <linux/module.h>
+#include "../../kernel/liveupdate/luo_internal.h"
+
+static const struct liveupdate_flb_ops test_flb_ops;
+#define DEFINE_TEST_FLB(i) {						\
+	.ops = &test_flb_ops,						\
+	.compatible = LIVEUPDATE_TEST_FLB_COMPATIBLE(i),		\
+}
+
+/* Number of Test FLBs to register with every file handler */
+#define TEST_NFLBS 3
+static struct liveupdate_flb test_flbs[TEST_NFLBS] = {
+	DEFINE_TEST_FLB(0),
+	DEFINE_TEST_FLB(1),
+	DEFINE_TEST_FLB(2),
+};
+
+#define TEST_FLB_MAGIC_BASE 0xFEEDF00DCAFEBEE0ULL
+
+static int test_flb_preserve(struct liveupdate_flb_op_args *argp)
+{
+	ptrdiff_t index = argp->flb - test_flbs;
+
+	pr_info("%s: preserve was triggered\n", argp->flb->compatible);
+	argp->data = TEST_FLB_MAGIC_BASE + index;
+
+	return 0;
+}
+
+static void test_flb_unpreserve(struct liveupdate_flb_op_args *argp)
+{
+	pr_info("%s: unpreserve was triggered\n", argp->flb->compatible);
+}
+
+static int test_flb_retrieve(struct liveupdate_flb_op_args *argp)
+{
+	ptrdiff_t index = argp->flb - test_flbs;
+	u64 expected_data = TEST_FLB_MAGIC_BASE + index;
+
+	if (argp->data == expected_data) {
+		pr_info("%s: found flb data from the previous boot\n",
+			argp->flb->compatible);
+		argp->obj = (void *)argp->data;
+	} else {
+		pr_err("%s: ERROR - incorrect data handle: %llx, expected %llx\n",
+		       argp->flb->compatible, argp->data, expected_data);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void test_flb_finish(struct liveupdate_flb_op_args *argp)
+{
+	ptrdiff_t index = argp->flb - test_flbs;
+	void *expected_obj = (void *)(TEST_FLB_MAGIC_BASE + index);
+
+	if (argp->obj == expected_obj) {
+		pr_info("%s: finish was triggered\n", argp->flb->compatible);
+	} else {
+		pr_err("%s: ERROR - finish called with invalid object\n",
+		       argp->flb->compatible);
+	}
+}
+
+static const struct liveupdate_flb_ops test_flb_ops = {
+	.preserve	= test_flb_preserve,
+	.unpreserve	= test_flb_unpreserve,
+	.retrieve	= test_flb_retrieve,
+	.finish		= test_flb_finish,
+	.owner		= THIS_MODULE,
+};
+
+static void liveupdate_test_init(void)
+{
+	static DEFINE_MUTEX(init_lock);
+	static bool initialized;
+	int i;
+
+	guard(mutex)(&init_lock);
+
+	if (initialized)
+		return;
+
+	for (i = 0; i < TEST_NFLBS; i++) {
+		struct liveupdate_flb *flb = &test_flbs[i];
+		void *obj;
+		int err;
+
+		err = liveupdate_flb_get_incoming(flb, &obj);
+		if (err && err != -ENODATA && err != -ENOENT) {
+			pr_err("liveupdate_flb_get_incoming for %s failed: %pe\n",
+			       flb->compatible, ERR_PTR(err));
+		}
+	}
+	initialized = true;
+}
+
+void liveupdate_test_register(struct liveupdate_file_handler *fh)
+{
+	int err, i;
+
+	liveupdate_test_init();
+
+	for (i = 0; i < TEST_NFLBS; i++) {
+		struct liveupdate_flb *flb = &test_flbs[i];
+
+		err = liveupdate_register_flb(fh, flb);
+		if (err) {
+			pr_err("Failed to register %s %pe\n",
+			       flb->compatible, ERR_PTR(err));
+		}
+	}
+
+	err = liveupdate_register_flb(fh, &test_flbs[0]);
+	if (!err || err != -EEXIST) {
+		pr_err("Failed: %s should be already registered, but got err: %pe\n",
+		       test_flbs[0].compatible, ERR_PTR(err));
+	}
+
+	pr_info("Registered %d FLBs with file handler: [%s]\n",
+		TEST_NFLBS, fh->compatible);
+}
+
+void liveupdate_test_unregister(struct liveupdate_file_handler *fh)
+{
+	int err, i;
+
+	for (i = 0; i < TEST_NFLBS; i++) {
+		struct liveupdate_flb *flb = &test_flbs[i];
+
+		err = liveupdate_unregister_flb(fh, flb);
+		if (err) {
+			pr_err("Failed to unregister %s %pe\n",
+			       flb->compatible, ERR_PTR(err));
+		}
+	}
+
+	pr_info("Unregistered %d FLBs from file handler: [%s]\n",
+		TEST_NFLBS, fh->compatible);
+}
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Pasha Tatashin <pasha.tatashin@soleen.com>");
+MODULE_DESCRIPTION("In-kernel test for LUO mechanism");
diff --git a/lib/test_min_heap.c b/lib/tests/min_heap_kunit.c
index a9c4a74d3898..9c1122661698 100644
--- a/lib/test_min_heap.c
+++ b/lib/tests/min_heap_kunit.c
@@ -1,60 +1,66 @@
 // SPDX-License-Identifier: GPL-2.0-only
-#define pr_fmt(fmt) "min_heap_test: " fmt
-
 /*
  * Test cases for the min max heap.
  */
 
-#include <linux/log2.h>
+#include <kunit/test.h>
 #include <linux/min_heap.h>
 #include <linux/module.h>
-#include <linux/printk.h>
 #include <linux/random.h>
 
+struct min_heap_test_case {
+	const char *str;
+	bool min_heap;
+};
+
+static struct min_heap_test_case min_heap_cases[] = {
+	{
+		.str = "min",
+		.min_heap = true,
+	},
+	{
+		.str = "max",
+		.min_heap = false,
+	},
+};
+
+KUNIT_ARRAY_PARAM_DESC(min_heap, min_heap_cases, str);
+
 DEFINE_MIN_HEAP(int, min_heap_test);
 
-static __init bool less_than(const void *lhs, const void *rhs, void __always_unused *args)
+static bool less_than(const void *lhs, const void *rhs, void __always_unused *args)
 {
 	return *(int *)lhs < *(int *)rhs;
 }
 
-static __init bool greater_than(const void *lhs, const void *rhs, void __always_unused *args)
+static bool greater_than(const void *lhs, const void *rhs, void __always_unused *args)
 {
 	return *(int *)lhs > *(int *)rhs;
 }
 
-static __init int pop_verify_heap(bool min_heap,
-				struct min_heap_test *heap,
-				const struct min_heap_callbacks *funcs)
+static void pop_verify_heap(struct kunit *test,
+			    bool min_heap,
+			    struct min_heap_test *heap,
+			    const struct min_heap_callbacks *funcs)
 {
 	int *values = heap->data;
-	int err = 0;
 	int last;
 
 	last = values[0];
 	min_heap_pop_inline(heap, funcs, NULL);
 	while (heap->nr > 0) {
-		if (min_heap) {
-			if (last > values[0]) {
-				pr_err("error: expected %d <= %d\n", last,
-					values[0]);
-				err++;
-			}
-		} else {
-			if (last < values[0]) {
-				pr_err("error: expected %d >= %d\n", last,
-					values[0]);
-				err++;
-			}
-		}
+		if (min_heap)
+			KUNIT_EXPECT_LE(test, last, values[0]);
+		else
+			KUNIT_EXPECT_GE(test, last, values[0]);
 		last = values[0];
 		min_heap_pop_inline(heap, funcs, NULL);
 	}
-	return err;
 }
 
-static __init int test_heapify_all(bool min_heap)
+static void test_heapify_all(struct kunit *test)
 {
+	const struct min_heap_test_case *params = test->param_value;
 	int values[] = { 3, 1, 2, 4, 0x8000000, 0x7FFFFFF, 0,
 			 -3, -1, -2, -4, 0x8000000, 0x7FFFFFF };
 	struct min_heap_test heap = {
@@ -63,15 +69,14 @@ static __init int test_heapify_all(bool min_heap)
 		.size =  ARRAY_SIZE(values),
 	};
 	struct min_heap_callbacks funcs = {
-		.less = min_heap ? less_than : greater_than,
+		.less = params->min_heap ? less_than : greater_than,
 		.swp = NULL,
 	};
-	int i, err;
+	int i;
 
 	/* Test with known set of values. */
 	min_heapify_all_inline(&heap, &funcs, NULL);
-	err = pop_verify_heap(min_heap, &heap, &funcs);
-
+	pop_verify_heap(test, params->min_heap, &heap, &funcs);
 
 	/* Test with randomly generated values. */
 	heap.nr = ARRAY_SIZE(values);
@@ -79,13 +84,12 @@ static __init int test_heapify_all(bool min_heap)
 		values[i] = get_random_u32();
 
 	min_heapify_all_inline(&heap, &funcs, NULL);
-	err += pop_verify_heap(min_heap, &heap, &funcs);
-
-	return err;
+	pop_verify_heap(test, params->min_heap, &heap, &funcs);
 }
 
-static __init int test_heap_push(bool min_heap)
+static void test_heap_push(struct kunit *test)
 {
+	const struct min_heap_test_case *params = test->param_value;
 	const int data[] = { 3, 1, 2, 4, 0x80000000, 0x7FFFFFFF, 0,
 			     -3, -1, -2, -4, 0x80000000, 0x7FFFFFFF };
 	int values[ARRAY_SIZE(data)];
@@ -95,29 +99,28 @@ static __init int test_heap_push(bool min_heap)
 		.size =  ARRAY_SIZE(values),
 	};
 	struct min_heap_callbacks funcs = {
-		.less = min_heap ? less_than : greater_than,
+		.less = params->min_heap ? less_than : greater_than,
 		.swp = NULL,
 	};
-	int i, temp, err;
+	int i, temp;
 
 	/* Test with known set of values copied from data. */
 	for (i = 0; i < ARRAY_SIZE(data); i++)
 		min_heap_push_inline(&heap, &data[i], &funcs, NULL);
 
-	err = pop_verify_heap(min_heap, &heap, &funcs);
+	pop_verify_heap(test, params->min_heap, &heap, &funcs);
 
 	/* Test with randomly generated values. */
 	while (heap.nr < heap.size) {
 		temp = get_random_u32();
 		min_heap_push_inline(&heap, &temp, &funcs, NULL);
 	}
-	err += pop_verify_heap(min_heap, &heap, &funcs);
-
-	return err;
+	pop_verify_heap(test, params->min_heap, &heap, &funcs);
 }
 
-static __init int test_heap_pop_push(bool min_heap)
+static void test_heap_pop_push(struct kunit *test)
 {
+	const struct min_heap_test_case *params = test->param_value;
 	const int data[] = { 3, 1, 2, 4, 0x80000000, 0x7FFFFFFF, 0,
 			     -3, -1, -2, -4, 0x80000000, 0x7FFFFFFF };
 	int values[ARRAY_SIZE(data)];
@@ -127,13 +130,13 @@ static __init int test_heap_pop_push(bool min_heap)
 		.size =  ARRAY_SIZE(values),
 	};
 	struct min_heap_callbacks funcs = {
-		.less = min_heap ? less_than : greater_than,
+		.less = params->min_heap ? less_than : greater_than,
 		.swp = NULL,
 	};
-	int i, temp, err;
+	int i, temp;
 
 	/* Fill values with data to pop and replace. */
-	temp = min_heap ? 0x80000000 : 0x7FFFFFFF;
+	temp = params->min_heap ? 0x80000000 : 0x7FFFFFFF;
 	for (i = 0; i < ARRAY_SIZE(data); i++)
 		min_heap_push_inline(&heap, &temp, &funcs, NULL);
 
@@ -141,7 +144,7 @@ static __init int test_heap_pop_push(bool min_heap)
 	for (i = 0; i < ARRAY_SIZE(data); i++)
 		min_heap_pop_push_inline(&heap, &data[i], &funcs, NULL);
 
-	err = pop_verify_heap(min_heap, &heap, &funcs);
+	pop_verify_heap(test, params->min_heap, &heap, &funcs);
 
 	heap.nr = 0;
 	for (i = 0; i < ARRAY_SIZE(data); i++)
@@ -152,13 +155,12 @@ static __init int test_heap_pop_push(bool min_heap)
 		temp = get_random_u32();
 		min_heap_pop_push_inline(&heap, &temp, &funcs, NULL);
 	}
-	err += pop_verify_heap(min_heap, &heap, &funcs);
-
-	return err;
+	pop_verify_heap(test, params->min_heap, &heap, &funcs);
 }
 
-static __init int test_heap_del(bool min_heap)
+static void test_heap_del(struct kunit *test)
 {
+	const struct min_heap_test_case *params = test->param_value;
 	int values[] = { 3, 1, 2, 4, 0x8000000, 0x7FFFFFF, 0,
 			 -3, -1, -2, -4, 0x8000000, 0x7FFFFFF };
 	struct min_heap_test heap;
@@ -166,17 +168,16 @@ static __init int test_heap_del(bool min_heap)
 	min_heap_init_inline(&heap, values, ARRAY_SIZE(values));
 	heap.nr = ARRAY_SIZE(values);
 	struct min_heap_callbacks funcs = {
-		.less = min_heap ? less_than : greater_than,
+		.less = params->min_heap ? less_than : greater_than,
 		.swp = NULL,
 	};
-	int i, err;
+	int i;
 
 	/* Test with known set of values. */
 	min_heapify_all_inline(&heap, &funcs, NULL);
 	for (i = 0; i < ARRAY_SIZE(values) / 2; i++)
 		min_heap_del_inline(&heap, get_random_u32() % heap.nr, &funcs, NULL);
-	err = pop_verify_heap(min_heap, &heap, &funcs);
-
+	pop_verify_heap(test, params->min_heap, &heap, &funcs);
 
 	/* Test with randomly generated values. */
 	heap.nr = ARRAY_SIZE(values);
@@ -186,37 +187,23 @@ static __init int test_heap_del(bool min_heap)
 
 	for (i = 0; i < ARRAY_SIZE(values) / 2; i++)
 		min_heap_del_inline(&heap, get_random_u32() % heap.nr, &funcs, NULL);
-	err += pop_verify_heap(min_heap, &heap, &funcs);
-
-	return err;
+	pop_verify_heap(test, params->min_heap, &heap, &funcs);
 }
 
-static int __init test_min_heap_init(void)
-{
-	int err = 0;
-
-	err += test_heapify_all(true);
-	err += test_heapify_all(false);
-	err += test_heap_push(true);
-	err += test_heap_push(false);
-	err += test_heap_pop_push(true);
-	err += test_heap_pop_push(false);
-	err += test_heap_del(true);
-	err += test_heap_del(false);
-	if (err) {
-		pr_err("test failed with %d errors\n", err);
-		return -EINVAL;
-	}
-	pr_info("test passed\n");
-	return 0;
-}
-module_init(test_min_heap_init);
+static struct kunit_case min_heap_test_cases[] = {
+	KUNIT_CASE_PARAM(test_heapify_all, min_heap_gen_params),
+	KUNIT_CASE_PARAM(test_heap_push, min_heap_gen_params),
+	KUNIT_CASE_PARAM(test_heap_pop_push, min_heap_gen_params),
+	KUNIT_CASE_PARAM(test_heap_del, min_heap_gen_params),
+	{},
+};
 
-static void __exit test_min_heap_exit(void)
-{
-	/* do nothing */
-}
-module_exit(test_min_heap_exit);
+static struct kunit_suite min_heap_test_suite = {
+	.name = "min_heap",
+	.test_cases = min_heap_test_cases,
+};
+
+kunit_test_suite(min_heap_test_suite);
 
 MODULE_DESCRIPTION("Test cases for the min max heap");
 MODULE_LICENSE("GPL");
diff --git a/lib/tests/printf_kunit.c b/lib/tests/printf_kunit.c
index bc54cca2d7a6..f6f21b445ece 100644
--- a/lib/tests/printf_kunit.c
+++ b/lib/tests/printf_kunit.c
@@ -266,15 +266,17 @@ hash_pointer(struct kunit *kunittest)
 	KUNIT_EXPECT_MEMNEQ(kunittest, buf, PTR_STR, PTR_WIDTH);
 }
 
-static void
-test_hashed(struct kunit *kunittest, const char *fmt, const void *p)
-{
-	char buf[PLAIN_BUF_SIZE];
-
-	plain_hash_to_buffer(kunittest, p, buf, PLAIN_BUF_SIZE);
-
-	test(buf, fmt, p);
-}
+/*
+ * This is a macro so that the compiler can compare its arguments to the
+ * __printf() attribute on __test(). This cannot be a function with a __printf()
+ * attribute because GCC requires __printf() functions to be variadic.
+ */
+#define test_hashed(kunittest, fmt, p)						\
+	do {									\
+		char buf[PLAIN_BUF_SIZE];					\
+		plain_hash_to_buffer(kunittest, p, buf, PLAIN_BUF_SIZE);	\
+		test(buf, fmt, p);						\
+	} while (0)
 
 /*
  * NULL pointers aren't hashed.
@@ -504,6 +506,7 @@ time_and_date(struct kunit *kunittest)
 	};
 	/* 2019-01-04T15:32:23 */
 	time64_t t = 1546615943;
+	struct timespec64 ts = { .tv_sec = t, .tv_nsec = 11235813 };
 
 	test("(%pt?)", "%pt", &tm);
 	test("2018-11-26T05:35:43", "%ptR", &tm);
@@ -522,6 +525,9 @@ time_and_date(struct kunit *kunittest)
 	test("0119-00-04 15:32:23", "%ptTsr", &t);
 	test("15:32:23|2019-01-04", "%ptTts|%ptTds", &t, &t);
 	test("15:32:23|0119-00-04", "%ptTtrs|%ptTdrs", &t, &t);
+
+	test("2019-01-04T15:32:23.011235813", "%ptS", &ts);
+	test("1546615943.011235813", "%ptSp", &ts);
 }
 
 static void
diff --git a/lib/tests/string_kunit.c b/lib/tests/string_kunit.c
index 0ed7448a26d3..f9a8e557ba77 100644
--- a/lib/tests/string_kunit.c
+++ b/lib/tests/string_kunit.c
@@ -602,6 +602,18 @@ static void string_test_memtostr(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, dest[7], '\0');
 }
 
+static void string_test_strends(struct kunit *test)
+{
+	KUNIT_EXPECT_TRUE(test, strends("foo-bar", "bar"));
+	KUNIT_EXPECT_TRUE(test, strends("foo-bar", "-bar"));
+	KUNIT_EXPECT_TRUE(test, strends("foobar", "foobar"));
+	KUNIT_EXPECT_TRUE(test, strends("foobar", ""));
+	KUNIT_EXPECT_FALSE(test, strends("bar", "foobar"));
+	KUNIT_EXPECT_FALSE(test, strends("", "foo"));
+	KUNIT_EXPECT_FALSE(test, strends("foobar", "ba"));
+	KUNIT_EXPECT_TRUE(test, strends("", ""));
+}
+
 static struct kunit_case string_test_cases[] = {
 	KUNIT_CASE(string_test_memset16),
 	KUNIT_CASE(string_test_memset32),
@@ -623,6 +635,7 @@ static struct kunit_case string_test_cases[] = {
 	KUNIT_CASE(string_test_strlcat),
 	KUNIT_CASE(string_test_strtomem),
 	KUNIT_CASE(string_test_memtostr),
+	KUNIT_CASE(string_test_strends),
 	{}
 };
 
diff --git a/lib/tests/test_fprobe.c b/lib/tests/test_fprobe.c
index cf92111b5c79..108c7aa33cb4 100644
--- a/lib/tests/test_fprobe.c
+++ b/lib/tests/test_fprobe.c
@@ -12,7 +12,8 @@
 
 static struct kunit *current_test;
 
-static u32 rand1, entry_val, exit_val;
+static u32 rand1, entry_only_val, entry_val, exit_val;
+static u32 entry_only_count, entry_count, exit_count;
 
 /* Use indirect calls to avoid inlining the target functions */
 static u32 (*target)(u32 value);
@@ -190,6 +191,101 @@ static void test_fprobe_skip(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp));
 }
 
+/* Handler for fprobe entry only case */
+static notrace int entry_only_handler(struct fprobe *fp, unsigned long ip,
+				      unsigned long ret_ip,
+				      struct ftrace_regs *fregs, void *data)
+{
+	KUNIT_EXPECT_FALSE(current_test, preemptible());
+	KUNIT_EXPECT_EQ(current_test, ip, target_ip);
+
+	entry_only_count++;
+	entry_only_val = (rand1 / div_factor);
+
+	return 0;
+}
+
+static notrace int fprobe_entry_multi_handler(struct fprobe *fp, unsigned long ip,
+					      unsigned long ret_ip,
+					      struct ftrace_regs *fregs,
+					      void *data)
+{
+	KUNIT_EXPECT_FALSE(current_test, preemptible());
+	KUNIT_EXPECT_EQ(current_test, ip, target_ip);
+
+	entry_count++;
+	entry_val = (rand1 / div_factor);
+
+	return 0;
+}
+
+static notrace void fprobe_exit_multi_handler(struct fprobe *fp, unsigned long ip,
+					      unsigned long ret_ip,
+					      struct ftrace_regs *fregs,
+					      void *data)
+{
+	unsigned long ret = ftrace_regs_get_return_value(fregs);
+
+	KUNIT_EXPECT_FALSE(current_test, preemptible());
+	KUNIT_EXPECT_EQ(current_test, ip, target_ip);
+	KUNIT_EXPECT_EQ(current_test, ret, (rand1 / div_factor));
+
+	exit_count++;
+	exit_val = ret;
+}
+
+static void check_fprobe_multi(struct kunit *test)
+{
+	entry_only_count = entry_count = exit_count = 0;
+	entry_only_val = entry_val = exit_val = 0;
+
+	target(rand1);
+
+	/* Verify all handlers were called */
+	KUNIT_EXPECT_EQ(test, 1, entry_only_count);
+	KUNIT_EXPECT_EQ(test, 1, entry_count);
+	KUNIT_EXPECT_EQ(test, 1, exit_count);
+
+	/* Verify values are correct */
+	KUNIT_EXPECT_EQ(test, (rand1 / div_factor), entry_only_val);
+	KUNIT_EXPECT_EQ(test, (rand1 / div_factor), entry_val);
+	KUNIT_EXPECT_EQ(test, (rand1 / div_factor), exit_val);
+}
+
+/* Test multiple fprobes hooking the same target function */
+static void test_fprobe_multi(struct kunit *test)
+{
+	struct fprobe fp1 = {
+		.entry_handler = fprobe_entry_multi_handler,
+		.exit_handler = fprobe_exit_multi_handler,
+	};
+	struct fprobe fp2 = {
+		.entry_handler = entry_only_handler,
+	};
+
+	current_test = test;
+
+	/* Test Case 1: Register in order 1 -> 2 */
+	KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp1, "fprobe_selftest_target", NULL));
+	KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp2, "fprobe_selftest_target", NULL));
+
+	check_fprobe_multi(test);
+
+	/* Unregister all */
+	KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp1));
+	KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp2));
+
+	/* Test Case 2: Register in order 2 -> 1 */
+	KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp2, "fprobe_selftest_target", NULL));
+	KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp1, "fprobe_selftest_target", NULL));
+
+	check_fprobe_multi(test);
+
+	/* Unregister all */
+	KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp1));
+	KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp2));
+}
+
 static unsigned long get_ftrace_location(void *func)
 {
 	unsigned long size, addr = (unsigned long)func;
@@ -217,6 +313,7 @@ static struct kunit_case fprobe_testcases[] = {
 	KUNIT_CASE(test_fprobe_syms),
 	KUNIT_CASE(test_fprobe_data),
 	KUNIT_CASE(test_fprobe_skip),
+	KUNIT_CASE(test_fprobe_multi),
 	{}
 };
 
diff --git a/lib/tests/test_list_sort.c b/lib/tests/test_list_sort.c
index 30879abc8a42..28158557b164 100644
--- a/lib/tests/test_list_sort.c
+++ b/lib/tests/test_list_sort.c
@@ -26,7 +26,7 @@ struct debug_el {
 	unsigned int serial;
 };
 
-static void check(struct kunit *test, struct debug_el *ela, struct debug_el *elb)
+static void check(struct kunit *test, const struct debug_el *ela, const struct debug_el *elb)
 {
 	struct debug_el **elts = test->priv;
 
@@ -46,7 +46,7 @@ static void check(struct kunit *test, struct debug_el *ela, struct debug_el *elb
 /* `priv` is the test pointer so check() can fail the test if the list is invalid. */
 static int cmp(void *priv, const struct list_head *a, const struct list_head *b)
 {
-	struct debug_el *ela, *elb;
+	const struct debug_el *ela, *elb;
 
 	ela = container_of(a, struct debug_el, list);
 	elb = container_of(b, struct debug_el, list);
diff --git a/lib/tests/test_ratelimit.c b/lib/tests/test_ratelimit.c
index bfaeca49304a..33cea5f3d28b 100644
--- a/lib/tests/test_ratelimit.c
+++ b/lib/tests/test_ratelimit.c
@@ -104,7 +104,7 @@ static void test_ratelimit_stress(struct kunit *test)
 	int i;
 	const int n_stress_kthread = cpumask_weight(cpu_online_mask);
 	struct stress_kthread skt = { 0 };
-	struct stress_kthread *sktp = kcalloc(n_stress_kthread, sizeof(*sktp), GFP_KERNEL);
+	struct stress_kthread *sktp = kzalloc_objs(*sktp, n_stress_kthread);
 
 	KUNIT_EXPECT_NOT_NULL_MSG(test, sktp, "Memory allocation failure");
 	for (i = 0; i < n_stress_kthread; i++) {
diff --git a/lib/tests/uuid_kunit.c b/lib/tests/uuid_kunit.c
new file mode 100644
index 000000000000..de71b2649dac
--- /dev/null
+++ b/lib/tests/uuid_kunit.c
@@ -0,0 +1,106 @@
+// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
+/*
+ * Test cases for lib/uuid.c module.
+ */
+
+#include <kunit/test.h>
+#include <linux/uuid.h>
+
+struct test_uuid_data {
+	const char *uuid;
+	guid_t le;
+	uuid_t be;
+};
+
+static const struct test_uuid_data test_uuid_test_data[] = {
+	{
+		.uuid = "c33f4995-3701-450e-9fbf-206a2e98e576",
+		.le = GUID_INIT(0xc33f4995, 0x3701, 0x450e, 0x9f, 0xbf, 0x20, 0x6a, 0x2e, 0x98, 0xe5, 0x76),
+		.be = UUID_INIT(0xc33f4995, 0x3701, 0x450e, 0x9f, 0xbf, 0x20, 0x6a, 0x2e, 0x98, 0xe5, 0x76),
+	},
+	{
+		.uuid = "64b4371c-77c1-48f9-8221-29f054fc023b",
+		.le = GUID_INIT(0x64b4371c, 0x77c1, 0x48f9, 0x82, 0x21, 0x29, 0xf0, 0x54, 0xfc, 0x02, 0x3b),
+		.be = UUID_INIT(0x64b4371c, 0x77c1, 0x48f9, 0x82, 0x21, 0x29, 0xf0, 0x54, 0xfc, 0x02, 0x3b),
+	},
+	{
+		.uuid = "0cb4ddff-a545-4401-9d06-688af53e7f84",
+		.le = GUID_INIT(0x0cb4ddff, 0xa545, 0x4401, 0x9d, 0x06, 0x68, 0x8a, 0xf5, 0x3e, 0x7f, 0x84),
+		.be = UUID_INIT(0x0cb4ddff, 0xa545, 0x4401, 0x9d, 0x06, 0x68, 0x8a, 0xf5, 0x3e, 0x7f, 0x84),
+	},
+};
+
+static const char * const test_uuid_wrong_data[] = {
+	"c33f4995-3701-450e-9fbf206a2e98e576 ",	/* no hyphen(s) */
+	"64b4371c-77c1-48f9-8221-29f054XX023b",	/* invalid character(s) */
+	"0cb4ddff-a545-4401-9d06-688af53e",	/* not enough data */
+};
+
+static void uuid_test_guid_valid(struct kunit *test)
+{
+	unsigned int i;
+	const struct test_uuid_data *data;
+	guid_t le;
+
+	for (i = 0; i < ARRAY_SIZE(test_uuid_test_data); i++) {
+		data = &test_uuid_test_data[i];
+		KUNIT_EXPECT_EQ(test, guid_parse(data->uuid, &le), 0);
+		KUNIT_EXPECT_TRUE(test, guid_equal(&data->le, &le));
+	}
+}
+
+static void uuid_test_uuid_valid(struct kunit *test)
+{
+	unsigned int i;
+	const struct test_uuid_data *data;
+	uuid_t be;
+
+	for (i = 0; i < ARRAY_SIZE(test_uuid_test_data); i++) {
+		data = &test_uuid_test_data[i];
+		KUNIT_EXPECT_EQ(test, uuid_parse(data->uuid, &be), 0);
+		KUNIT_EXPECT_TRUE(test, uuid_equal(&data->be, &be));
+	}
+}
+
+static void uuid_test_guid_invalid(struct kunit *test)
+{
+	unsigned int i;
+	const char *uuid;
+	guid_t le;
+
+	for (i = 0; i < ARRAY_SIZE(test_uuid_wrong_data); i++) {
+		uuid = test_uuid_wrong_data[i];
+		KUNIT_EXPECT_EQ(test, guid_parse(uuid, &le), -EINVAL);
+	}
+}
+
+static void uuid_test_uuid_invalid(struct kunit *test)
+{
+	unsigned int i;
+	const char *uuid;
+	uuid_t be;
+
+	for (i = 0; i < ARRAY_SIZE(test_uuid_wrong_data); i++) {
+		uuid = test_uuid_wrong_data[i];
+		KUNIT_EXPECT_EQ(test, uuid_parse(uuid, &be), -EINVAL);
+	}
+}
+
+static struct kunit_case uuid_test_cases[] = {
+	KUNIT_CASE(uuid_test_guid_valid),
+	KUNIT_CASE(uuid_test_uuid_valid),
+	KUNIT_CASE(uuid_test_guid_invalid),
+	KUNIT_CASE(uuid_test_uuid_invalid),
+	{},
+};
+
+static struct kunit_suite uuid_test_suite = {
+	.name = "uuid",
+	.test_cases = uuid_test_cases,
+};
+
+kunit_test_suite(uuid_test_suite);
+
+MODULE_AUTHOR("Andy Shevchenko <andriy.shevchenko@linux.intel.com>");
+MODULE_DESCRIPTION("Test cases for lib/uuid.c module");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/lib/usercopy.c b/lib/usercopy.c
index 7b17b83c8042..b00a3a957de6 100644
--- a/lib/usercopy.c
+++ b/lib/usercopy.c
@@ -12,7 +12,7 @@
 
 /* out-of-line parts */
 
-#if !defined(INLINE_COPY_FROM_USER) || defined(CONFIG_RUST)
+#if !defined(INLINE_COPY_FROM_USER)
 unsigned long _copy_from_user(void *to, const void __user *from, unsigned long n)
 {
 	return _inline_copy_from_user(to, from, n);
@@ -20,7 +20,7 @@ unsigned long _copy_from_user(void *to, const void __user *from, unsigned long n
 EXPORT_SYMBOL(_copy_from_user);
 #endif
 
-#if !defined(INLINE_COPY_TO_USER) || defined(CONFIG_RUST)
+#if !defined(INLINE_COPY_TO_USER)
 unsigned long _copy_to_user(void __user *to, const void *from, unsigned long n)
 {
 	return _inline_copy_to_user(to, from, n);
diff --git a/lib/uuid.c b/lib/uuid.c
index e309b4c5be3d..e8543c668dc7 100644
--- a/lib/uuid.c
+++ b/lib/uuid.c
@@ -10,6 +10,7 @@
 #include <linux/ctype.h>
 #include <linux/errno.h>
 #include <linux/export.h>
+#include <linux/hex.h>
 #include <linux/uuid.h>
 #include <linux/random.h>
 
diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c
index 95df0153f05a..4399e143d43a 100644
--- a/lib/vdso/gettimeofday.c
+++ b/lib/vdso/gettimeofday.c
@@ -421,7 +421,7 @@ static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time
 #endif /* VDSO_HAS_TIME */
 
 #ifdef VDSO_HAS_CLOCK_GETRES
-static __maybe_unused
+static __always_inline
 bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t clock,
 				 struct __kernel_timespec *res)
 {
diff --git a/lib/vsprintf.c b/lib/vsprintf.c
index eb0cb11d0d12..800b8ac49f53 100644
--- a/lib/vsprintf.c
+++ b/lib/vsprintf.c
@@ -26,6 +26,7 @@
 #include <linux/types.h>
 #include <linux/string.h>
 #include <linux/ctype.h>
+#include <linux/hex.h>
 #include <linux/kernel.h>
 #include <linux/kallsyms.h>
 #include <linux/math64.h>
@@ -582,17 +583,18 @@ char *number(char *buf, char *end, unsigned long long num,
 	return buf;
 }
 
+#define special_hex_spec(size)					\
+(struct printf_spec) {						\
+	.field_width = 2 + 2 * (size),		/* 0x + hex */	\
+	.flags = SPECIAL | SMALL | ZEROPAD,			\
+	.base = 16,						\
+	.precision = -1,					\
+}
+
 static noinline_for_stack
 char *special_hex_number(char *buf, char *end, unsigned long long num, int size)
 {
-	struct printf_spec spec;
-
-	spec.field_width = 2 + 2 * size;	/* 0x + hex */
-	spec.flags = SPECIAL | SMALL | ZEROPAD;
-	spec.base = 16;
-	spec.precision = -1;
-
-	return number(buf, end, num, spec);
+	return number(buf, end, num, special_hex_spec(size));
 }
 
 static void move_right(char *buf, char *end, unsigned len, unsigned spaces)
@@ -1164,18 +1166,11 @@ char *range_string(char *buf, char *end, const struct range *range,
 	char sym[sizeof("[range 0x0123456789abcdef-0x0123456789abcdef]")];
 	char *p = sym, *pend = sym + sizeof(sym);
 
-	struct printf_spec range_spec = {
-		.field_width = 2 + 2 * sizeof(range->start), /* 0x + 2 * 8 */
-		.flags = SPECIAL | SMALL | ZEROPAD,
-		.base = 16,
-		.precision = -1,
-	};
-
 	if (check_pointer(&buf, end, range, spec))
 		return buf;
 
 	p = string_nocheck(p, pend, "[range ", default_str_spec);
-	p = hex_range(p, pend, range->start, range->end, range_spec);
+	p = hex_range(p, pend, range->start, range->end, special_hex_spec(sizeof(range->start)));
 	*p++ = ']';
 	*p = '\0';
 
@@ -1928,9 +1923,6 @@ char *rtc_str(char *buf, char *end, const struct rtc_time *tm,
 	bool found = true;
 	int count = 2;
 
-	if (check_pointer(&buf, end, tm, spec))
-		return buf;
-
 	switch (fmt[count]) {
 	case 'd':
 		have_t = false;
@@ -1993,12 +1985,39 @@ char *time64_str(char *buf, char *end, const time64_t time,
 }
 
 static noinline_for_stack
+char *timespec64_str(char *buf, char *end, const struct timespec64 *ts,
+		     struct printf_spec spec, const char *fmt)
+{
+	static const struct printf_spec default_dec09_spec = {
+		.base = 10,
+		.field_width = 9,
+		.precision = -1,
+		.flags = ZEROPAD,
+	};
+
+	if (fmt[2] == 'p')
+		buf = number(buf, end, ts->tv_sec, default_dec_spec);
+	else
+		buf = time64_str(buf, end, ts->tv_sec, spec, fmt);
+	if (buf < end)
+		*buf = '.';
+	buf++;
+
+	return number(buf, end, ts->tv_nsec, default_dec09_spec);
+}
+
+static noinline_for_stack
 char *time_and_date(char *buf, char *end, void *ptr, struct printf_spec spec,
 		    const char *fmt)
 {
+	if (check_pointer(&buf, end, ptr, spec))
+		return buf;
+
 	switch (fmt[1]) {
 	case 'R':
 		return rtc_str(buf, end, (const struct rtc_time *)ptr, spec, fmt);
+	case 'S':
+		return timespec64_str(buf, end, (const struct timespec64 *)ptr, spec, fmt);
 	case 'T':
 		return time64_str(buf, end, *(const time64_t *)ptr, spec, fmt);
 	default:
@@ -2462,9 +2481,11 @@ early_param("no_hash_pointers", no_hash_pointers_enable);
  * - 'd[234]' For a dentry name (optionally 2-4 last components)
  * - 'D[234]' Same as 'd' but for a struct file
  * - 'g' For block_device name (gendisk + partition number)
- * - 't[RT][dt][r][s]' For time and date as represented by:
+ * - 't[RST][dt][r][s]' For time and date as represented by:
  *      R    struct rtc_time
+ *      S    struct timespec64
  *      T    time64_t
+ * - 'tSp' For time represented by struct timespec64 printed as <seconds>.<nanoseconds>
  * - 'C' For a clock, it prints the name (Common Clock Framework) or address
  *       (legacy clock framework) of the clock
  * - 'G' For flags to be printed as a collection of symbolic strings that would
@@ -2883,10 +2904,11 @@ int vsnprintf(char *buf, size_t size, const char *fmt_str, va_list args)
 
 		case FORMAT_STATE_NUM: {
 			unsigned long long num;
-			if (fmt.size <= sizeof(int))
-				num = convert_num_spec(va_arg(args, int), fmt.size, spec);
-			else
+
+			if (fmt.size > sizeof(int))
 				num = va_arg(args, long long);
+			else
+				num = convert_num_spec(va_arg(args, int), fmt.size, spec);
 			str = number(str, end, num, spec);
 			continue;
 		}
@@ -3054,8 +3076,8 @@ EXPORT_SYMBOL(scnprintf);
  * @fmt: The format string to use
  * @args: Arguments for the format string
  *
- * The function returns the number of characters written
- * into @buf. Use vsnprintf() or vscnprintf() in order to avoid
+ * The return value is the number of characters written into @buf not including
+ * the trailing '\0'. Use vsnprintf() or vscnprintf() in order to avoid
  * buffer overflows.
  *
  * If you're not already dealing with a va_list consider using sprintf().
@@ -3074,8 +3096,8 @@ EXPORT_SYMBOL(vsprintf);
  * @fmt: The format string to use
  * @...: Arguments for the format string
  *
- * The function returns the number of characters written
- * into @buf. Use snprintf() or scnprintf() in order to avoid
+ * The return value is the number of characters written into @buf not including
+ * the trailing '\0'. Use snprintf() or scnprintf() in order to avoid
  * buffer overflows.
  *
  * See the vsnprintf() documentation for format string extensions over C99.
@@ -3394,11 +3416,10 @@ int bstr_printf(char *buf, size_t size, const char *fmt_str, const u32 *bin_buf)
 			goto out;
 
 		case FORMAT_STATE_NUM:
-			if (fmt.size > sizeof(int)) {
+			if (fmt.size > sizeof(int))
 				num = get_arg(long long);
-			} else {
+			else
 				num = convert_num_spec(get_arg(int), fmt.size, spec);
-			}
 			str = number(str, end, num, spec);
 			continue;
 		}
diff --git a/lib/xxhash.c b/lib/xxhash.c
index cf629766f376..4125b3e3cf7f 100644
--- a/lib/xxhash.c
+++ b/lib/xxhash.c
@@ -73,21 +73,6 @@ static const uint64_t PRIME64_3 =  1609587929392839161ULL;
 static const uint64_t PRIME64_4 =  9650029242287828579ULL;
 static const uint64_t PRIME64_5 =  2870177450012600261ULL;
 
-/*-**************************
- *  Utils
- ***************************/
-void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src)
-{
-	memcpy(dst, src, sizeof(*dst));
-}
-EXPORT_SYMBOL(xxh32_copy_state);
-
-void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src)
-{
-	memcpy(dst, src, sizeof(*dst));
-}
-EXPORT_SYMBOL(xxh64_copy_state);
-
 /*-***************************
  * Simple Hash Functions
  ****************************/
@@ -239,20 +224,6 @@ EXPORT_SYMBOL(xxh64);
 /*-**************************************************
  * Advanced Hash Functions
  ***************************************************/
-void xxh32_reset(struct xxh32_state *statePtr, const uint32_t seed)
-{
-	/* use a local state for memcpy() to avoid strict-aliasing warnings */
-	struct xxh32_state state;
-
-	memset(&state, 0, sizeof(state));
-	state.v1 = seed + PRIME32_1 + PRIME32_2;
-	state.v2 = seed + PRIME32_2;
-	state.v3 = seed + 0;
-	state.v4 = seed - PRIME32_1;
-	memcpy(statePtr, &state, sizeof(state));
-}
-EXPORT_SYMBOL(xxh32_reset);
-
 void xxh64_reset(struct xxh64_state *statePtr, const uint64_t seed)
 {
 	/* use a local state for memcpy() to avoid strict-aliasing warnings */
diff --git a/lib/xz/xz_dec_bcj.c b/lib/xz/xz_dec_bcj.c
index 8237db17eee3..cc49a300a5b2 100644
--- a/lib/xz/xz_dec_bcj.c
+++ b/lib/xz/xz_dec_bcj.c
@@ -20,7 +20,6 @@ struct xz_dec_bcj {
 	enum {
 		BCJ_X86 = 4,        /* x86 or x86-64 */
 		BCJ_POWERPC = 5,    /* Big endian only */
-		BCJ_IA64 = 6,       /* Big or little endian */
 		BCJ_ARM = 7,        /* Little endian only */
 		BCJ_ARMTHUMB = 8,   /* Little endian only */
 		BCJ_SPARC = 9,      /* Big or little endian */
@@ -180,92 +179,6 @@ static size_t bcj_powerpc(struct xz_dec_bcj *s, uint8_t *buf, size_t size)
 }
 #endif
 
-#ifdef XZ_DEC_IA64
-static size_t bcj_ia64(struct xz_dec_bcj *s, uint8_t *buf, size_t size)
-{
-	static const uint8_t branch_table[32] = {
-		0, 0, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0,
-		4, 4, 6, 6, 0, 0, 7, 7,
-		4, 4, 0, 0, 4, 4, 0, 0
-	};
-
-	/*
-	 * The local variables take a little bit stack space, but it's less
-	 * than what LZMA2 decoder takes, so it doesn't make sense to reduce
-	 * stack usage here without doing that for the LZMA2 decoder too.
-	 */
-
-	/* Loop counters */
-	size_t i;
-	size_t j;
-
-	/* Instruction slot (0, 1, or 2) in the 128-bit instruction word */
-	uint32_t slot;
-
-	/* Bitwise offset of the instruction indicated by slot */
-	uint32_t bit_pos;
-
-	/* bit_pos split into byte and bit parts */
-	uint32_t byte_pos;
-	uint32_t bit_res;
-
-	/* Address part of an instruction */
-	uint32_t addr;
-
-	/* Mask used to detect which instructions to convert */
-	uint32_t mask;
-
-	/* 41-bit instruction stored somewhere in the lowest 48 bits */
-	uint64_t instr;
-
-	/* Instruction normalized with bit_res for easier manipulation */
-	uint64_t norm;
-
-	size &= ~(size_t)15;
-
-	for (i = 0; i < size; i += 16) {
-		mask = branch_table[buf[i] & 0x1F];
-		for (slot = 0, bit_pos = 5; slot < 3; ++slot, bit_pos += 41) {
-			if (((mask >> slot) & 1) == 0)
-				continue;
-
-			byte_pos = bit_pos >> 3;
-			bit_res = bit_pos & 7;
-			instr = 0;
-			for (j = 0; j < 6; ++j)
-				instr |= (uint64_t)(buf[i + j + byte_pos])
-						<< (8 * j);
-
-			norm = instr >> bit_res;
-
-			if (((norm >> 37) & 0x0F) == 0x05
-					&& ((norm >> 9) & 0x07) == 0) {
-				addr = (norm >> 13) & 0x0FFFFF;
-				addr |= ((uint32_t)(norm >> 36) & 1) << 20;
-				addr <<= 4;
-				addr -= s->pos + (uint32_t)i;
-				addr >>= 4;
-
-				norm &= ~((uint64_t)0x8FFFFF << 13);
-				norm |= (uint64_t)(addr & 0x0FFFFF) << 13;
-				norm |= (uint64_t)(addr & 0x100000)
-						<< (36 - 20);
-
-				instr &= (1 << bit_res) - 1;
-				instr |= norm << bit_res;
-
-				for (j = 0; j < 6; j++)
-					buf[i + j + byte_pos]
-						= (uint8_t)(instr >> (8 * j));
-			}
-		}
-	}
-
-	return i;
-}
-#endif
-
 #ifdef XZ_DEC_ARM
 static size_t bcj_arm(struct xz_dec_bcj *s, uint8_t *buf, size_t size)
 {
@@ -509,11 +422,6 @@ static void bcj_apply(struct xz_dec_bcj *s,
 		filtered = bcj_powerpc(s, buf, size);
 		break;
 #endif
-#ifdef XZ_DEC_IA64
-	case BCJ_IA64:
-		filtered = bcj_ia64(s, buf, size);
-		break;
-#endif
 #ifdef XZ_DEC_ARM
 	case BCJ_ARM:
 		filtered = bcj_arm(s, buf, size);
@@ -683,7 +591,7 @@ enum xz_ret xz_dec_bcj_run(struct xz_dec_bcj *s, struct xz_dec_lzma2 *lzma2,
 
 struct xz_dec_bcj *xz_dec_bcj_create(bool single_call)
 {
-	struct xz_dec_bcj *s = kmalloc(sizeof(*s), GFP_KERNEL);
+	struct xz_dec_bcj *s = kmalloc_obj(*s);
 	if (s != NULL)
 		s->single_call = single_call;
 
@@ -699,9 +607,6 @@ enum xz_ret xz_dec_bcj_reset(struct xz_dec_bcj *s, uint8_t id)
 #ifdef XZ_DEC_POWERPC
 	case BCJ_POWERPC:
 #endif
-#ifdef XZ_DEC_IA64
-	case BCJ_IA64:
-#endif
 #ifdef XZ_DEC_ARM
 	case BCJ_ARM:
 #endif
diff --git a/lib/xz/xz_dec_lzma2.c b/lib/xz/xz_dec_lzma2.c
index 83bb66b6016d..4b783ac94e71 100644
--- a/lib/xz/xz_dec_lzma2.c
+++ b/lib/xz/xz_dec_lzma2.c
@@ -1138,7 +1138,7 @@ enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, struct xz_buf *b)
 
 struct xz_dec_lzma2 *xz_dec_lzma2_create(enum xz_mode mode, uint32_t dict_max)
 {
-	struct xz_dec_lzma2 *s = kmalloc(sizeof(*s), GFP_KERNEL);
+	struct xz_dec_lzma2 *s = kmalloc_obj(*s);
 	if (s == NULL)
 		return NULL;
 
@@ -1296,7 +1296,7 @@ struct xz_dec_microlzma *xz_dec_microlzma_alloc(enum xz_mode mode,
 	if (dict_size < 4096 || dict_size > (3U << 30))
 		return NULL;
 
-	s = kmalloc(sizeof(*s), GFP_KERNEL);
+	s = kmalloc_obj(*s);
 	if (s == NULL)
 		return NULL;
 
diff --git a/lib/xz/xz_dec_stream.c b/lib/xz/xz_dec_stream.c
index f9d003684d56..59bfd54ffee7 100644
--- a/lib/xz/xz_dec_stream.c
+++ b/lib/xz/xz_dec_stream.c
@@ -784,7 +784,7 @@ enum xz_ret xz_dec_run(struct xz_dec *s, struct xz_buf *b)
 
 struct xz_dec *xz_dec_init(enum xz_mode mode, uint32_t dict_max)
 {
-	struct xz_dec *s = kmalloc(sizeof(*s), GFP_KERNEL);
+	struct xz_dec *s = kmalloc_obj(*s);
 	if (s == NULL)
 		return NULL;
 
diff --git a/lib/xz/xz_private.h b/lib/xz/xz_private.h
index 8409784b1639..6775078f3cce 100644
--- a/lib/xz/xz_private.h
+++ b/lib/xz/xz_private.h
@@ -24,9 +24,6 @@
 #		ifdef CONFIG_XZ_DEC_POWERPC
 #			define XZ_DEC_POWERPC
 #		endif
-#		ifdef CONFIG_XZ_DEC_IA64
-#			define XZ_DEC_IA64
-#		endif
 #		ifdef CONFIG_XZ_DEC_ARM
 #			define XZ_DEC_ARM
 #		endif
@@ -103,7 +100,6 @@
  */
 #ifndef XZ_DEC_BCJ
 #	if defined(XZ_DEC_X86) || defined(XZ_DEC_POWERPC) \
-			|| defined(XZ_DEC_IA64) \
 			|| defined(XZ_DEC_ARM) || defined(XZ_DEC_ARMTHUMB) \
 			|| defined(XZ_DEC_SPARC) || defined(XZ_DEC_ARM64) \
 			|| defined(XZ_DEC_RISCV)
diff --git a/lib/zlib_inflate/infutil.c b/lib/zlib_inflate/infutil.c
index 4824c2cc7a09..12169aacd3f1 100644
--- a/lib/zlib_inflate/infutil.c
+++ b/lib/zlib_inflate/infutil.c
@@ -14,7 +14,7 @@ int zlib_inflate_blob(void *gunzip_buf, unsigned int sz,
 	int rc;
 
 	rc = -ENOMEM;
-	strm = kmalloc(sizeof(*strm), GFP_KERNEL);
+	strm = kmalloc_obj(*strm);
 	if (strm == NULL)
 		goto gunzip_nomem1;
 	strm->workspace = kmalloc(zlib_inflate_workspacesize(), GFP_KERNEL);
author	Dmitry Torokhov <dmitry.torokhov@gmail.com>	2026-04-20 04:28:57 +0300
committer	Dmitry Torokhov <dmitry.torokhov@gmail.com>	2026-04-20 04:28:57 +0300
commit	f4b369c6fe0ceaba2da2daff8c9eb415f85926dd (patch)
tree	30465d0a429b2c224685b5d8e804bf053c4d129a /lib
parent	ff14dafde15c11403fac61367a34fea08926e9ee (diff)
parent	2ca45e57ea027fffe3350ae5e21ad9cecb0dce74 (diff)
download	linux-f4b369c6fe0ceaba2da2daff8c9eb415f85926dd.tar.xz