Merge tag 'libcrypto-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux

Pull crypto library updates from Eric Biggers:

 - Migrate more hash algorithms from the traditional crypto subsystem to
   lib/crypto/

   Like the algorithms migrated earlier (e.g. SHA-*), this simplifies
   the implementations, improves performance, enables further
   simplifications in calling code, and solves various other issues:

     - AES CBC-based MACs (AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC)

         - Support these algorithms in lib/crypto/ using the AES library
           and the existing arm64 assembly code

         - Reimplement the traditional crypto API's "cmac(aes)",
           "xcbc(aes)", and "cbcmac(aes)" on top of the library

         - Convert mac80211 to use the AES-CMAC library. Note: several
           other subsystems can use it too and will be converted later

         - Drop the broken, nonstandard, and likely unused support for
           "xcbc(aes)" with key lengths other than 128 bits

         - Enable optimizations by default

     - GHASH

         - Migrate the standalone GHASH code into lib/crypto/

         - Integrate the GHASH code more closely with the very similar
           POLYVAL code, and improve the generic GHASH implementation to
           resist cache-timing attacks and use much less memory

         - Reimplement the AES-GCM library and the "gcm" crypto_aead
           template on top of the GHASH library. Remove "ghash" from the
           crypto_shash API, as it's no longer needed

         - Enable optimizations by default

     - SM3

         - Migrate the kernel's existing SM3 code into lib/crypto/, and
           reimplement the traditional crypto API's "sm3" on top of it

         - I don't recommend using SM3, but this cleanup is worthwhile
           to organize the code the same way as other algorithms

 - Testing improvements:

     - Add a KUnit test suite for each of the new library APIs

     - Migrate the existing ChaCha20Poly1305 test to KUnit

     - Make the KUnit all_tests.config enable all crypto library tests

     - Move the test kconfig options to the Runtime Testing menu

 - Other updates to arch-optimized crypto code:

     - Optimize SHA-256 for Zhaoxin CPUs using the Padlock Hash Engine

     - Remove some MD5 implementations that are no longer worth keeping

     - Drop big endian and voluntary preemption support from the arm64
       code, as those configurations are no longer supported on arm64

 - Make jitterentropy and samples/tsm-mr use the crypto library APIs

* tag 'libcrypto-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux: (66 commits)
  lib/crypto: arm64: Assume a little-endian kernel
  arm64: fpsimd: Remove obsolete cond_yield macro
  lib/crypto: arm64/sha3: Remove obsolete chunking logic
  lib/crypto: arm64/sha512: Remove obsolete chunking logic
  lib/crypto: arm64/sha256: Remove obsolete chunking logic
  lib/crypto: arm64/sha1: Remove obsolete chunking logic
  lib/crypto: arm64/poly1305: Remove obsolete chunking logic
  lib/crypto: arm64/gf128hash: Remove obsolete chunking logic
  lib/crypto: arm64/chacha: Remove obsolete chunking logic
  lib/crypto: arm64/aes: Remove obsolete chunking logic
  lib/crypto: Include <crypto/utils.h> instead of <crypto/algapi.h>
  lib/crypto: aesgcm: Don't disable IRQs during AES block encryption
  lib/crypto: aescfb: Don't disable IRQs during AES block encryption
  lib/crypto: tests: Migrate ChaCha20Poly1305 self-test to KUnit
  lib/crypto: sparc: Drop optimized MD5 code
  lib/crypto: mips: Drop optimized MD5 code
  lib: Move crypto library tests to Runtime Testing menu
  crypto: sm3 - Remove 'struct sm3_state'
  crypto: sm3 - Remove the original "sm3_block_generic()"
  crypto: sm3 - Remove sm3_base.h
  ...

10 files changed, 385 insertions, 215 deletions

diff --git a/include/crypto/aes-cbc-macs.h b/include/crypto/aes-cbc-macs.h
new file mode 100644
index 000000000000..e61df108b926
--- /dev/null
+++ b/include/crypto/aes-cbc-macs.h
@@ -0,0 +1,154 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Support for AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC
+ *
+ * Copyright 2026 Google LLC
+ */
+#ifndef _CRYPTO_AES_CBC_MACS_H
+#define _CRYPTO_AES_CBC_MACS_H
+
+#include <crypto/aes.h>
+
+/**
+ * struct aes_cmac_key - Prepared key for AES-CMAC or AES-XCBC-MAC
+ * @aes: The AES key for cipher block chaining
+ * @k_final: Finalization subkeys for the final block.
+ *	     k_final[0] (CMAC K1, XCBC-MAC K2) is used if it's a full block.
+ *	     k_final[1] (CMAC K2, XCBC-MAC K3) is used if it's a partial block.
+ */
+struct aes_cmac_key {
+	struct aes_enckey aes;
+	union {
+		u8 b[AES_BLOCK_SIZE];
+		__be64 w[2];
+	} k_final[2];
+};
+
+/**
+ * struct aes_cmac_ctx - Context for computing an AES-CMAC or AES-XCBC-MAC value
+ * @key: Pointer to the key struct.  A pointer is used rather than a copy of the
+ *	 struct, since the key struct size may be large.  It is assumed that the
+ *	 key lives at least as long as the context.
+ * @partial_len: Number of bytes that have been XOR'ed into @h since the last
+ *		 AES encryption.  This is 0 if no data has been processed yet,
+ *		 or between 1 and AES_BLOCK_SIZE inclusive otherwise.
+ * @h: The current chaining value
+ */
+struct aes_cmac_ctx {
+	const struct aes_cmac_key *key;
+	size_t partial_len;
+	u8 h[AES_BLOCK_SIZE];
+};
+
+/**
+ * aes_cmac_preparekey() - Prepare a key for AES-CMAC
+ * @key: (output) The key struct to initialize
+ * @in_key: The raw AES key
+ * @key_len: Length of the raw key in bytes.  The supported values are
+ *	     AES_KEYSIZE_128, AES_KEYSIZE_192, and AES_KEYSIZE_256.
+ *
+ * Context: Any context.
+ * Return: 0 on success or -EINVAL if the given key length is invalid.  No other
+ *	   errors are possible, so callers that always pass a valid key length
+ *	   don't need to check for errors.
+ */
+int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key,
+			size_t key_len);
+
+/**
+ * aes_xcbcmac_preparekey() - Prepare a key for AES-XCBC-MAC
+ * @key: (output) The key struct to initialize
+ * @in_key: The raw key.  As per the AES-XCBC-MAC specification (RFC 3566), this
+ *	    is 128 bits, matching the internal use of AES-128.
+ *
+ * AES-XCBC-MAC and AES-CMAC are the same except for the key preparation.  After
+ * that step, AES-XCBC-MAC is supported via the aes_cmac_* functions.
+ *
+ * New users should use AES-CMAC instead of AES-XCBC-MAC.
+ *
+ * Context: Any context.
+ */
+void aes_xcbcmac_preparekey(struct aes_cmac_key *key,
+			    const u8 in_key[at_least AES_KEYSIZE_128]);
+
+/**
+ * aes_cmac_init() - Start computing an AES-CMAC or AES-XCBC-MAC value
+ * @ctx: (output) The context to initialize
+ * @key: The key to use.  Note that a pointer to the key is saved in the
+ *	 context, so the key must live at least as long as the context.
+ *
+ * This supports both AES-CMAC and AES-XCBC-MAC.  Which one is done depends on
+ * whether aes_cmac_preparekey() or aes_xcbcmac_preparekey() was called.
+ */
+static inline void aes_cmac_init(struct aes_cmac_ctx *ctx,
+				 const struct aes_cmac_key *key)
+{
+	*ctx = (struct aes_cmac_ctx){ .key = key };
+}
+
+/**
+ * aes_cmac_update() - Update an AES-CMAC or AES-XCBC-MAC context with more data
+ * @ctx: The context to update; must have been initialized
+ * @data: The message data
+ * @data_len: The data length in bytes.  Doesn't need to be block-aligned.
+ *
+ * This can be called any number of times.
+ *
+ * Context: Any context.
+ */
+void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len);
+
+/**
+ * aes_cmac_final() - Finish computing an AES-CMAC or AES-XCBC-MAC value
+ * @ctx: The context to finalize; must have been initialized
+ * @out: (output) The resulting MAC
+ *
+ * After finishing, this zeroizes @ctx.  So the caller does not need to do it.
+ *
+ * Context: Any context.
+ */
+void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[at_least AES_BLOCK_SIZE]);
+
+/**
+ * aes_cmac() - Compute AES-CMAC or AES-XCBC-MAC in one shot
+ * @key: The key to use
+ * @data: The message data
+ * @data_len: The data length in bytes
+ * @out: (output) The resulting AES-CMAC or AES-XCBC-MAC value
+ *
+ * This supports both AES-CMAC and AES-XCBC-MAC.  Which one is done depends on
+ * whether aes_cmac_preparekey() or aes_xcbcmac_preparekey() was called.
+ *
+ * Context: Any context.
+ */
+static inline void aes_cmac(const struct aes_cmac_key *key, const u8 *data,
+			    size_t data_len, u8 out[at_least AES_BLOCK_SIZE])
+{
+	struct aes_cmac_ctx ctx;
+
+	aes_cmac_init(&ctx, key);
+	aes_cmac_update(&ctx, data, data_len);
+	aes_cmac_final(&ctx, out);
+}
+
+/*
+ * AES-CBC-MAC support.  This is provided only for use by the implementation of
+ * AES-CCM.  It should have no other users.  Warning: unlike AES-CMAC and
+ * AES-XCBC-MAC, AES-CBC-MAC isn't a secure MAC for variable-length messages.
+ */
+struct aes_cbcmac_ctx {
+	const struct aes_enckey *key;
+	size_t partial_len;
+	u8 h[AES_BLOCK_SIZE];
+};
+static inline void aes_cbcmac_init(struct aes_cbcmac_ctx *ctx,
+				   const struct aes_enckey *key)
+{
+	*ctx = (struct aes_cbcmac_ctx){ .key = key };
+}
+void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data,
+		       size_t data_len);
+void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx,
+		      u8 out[at_least AES_BLOCK_SIZE]);
+
+#endif /* _CRYPTO_AES_CBC_MACS_H */
diff --git a/include/crypto/aes.h b/include/crypto/aes.h
index cbf1cc96db52..16fbfd93e2bd 100644
--- a/include/crypto/aes.h
+++ b/include/crypto/aes.h
@@ -167,6 +167,72 @@ int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
 #ifdef CONFIG_ARM64
 int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
 		     unsigned int key_len);
+asmlinkage void neon_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
+				     int rounds, int blocks);
+asmlinkage void neon_aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[],
+				     int rounds, int blocks);
+asmlinkage void neon_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
+				     int rounds, int blocks, u8 iv[]);
+asmlinkage void neon_aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[],
+				     int rounds, int blocks, u8 iv[]);
+asmlinkage void neon_aes_cbc_cts_encrypt(u8 out[], u8 const in[],
+					 u32 const rk[], int rounds, int bytes,
+					 u8 const iv[]);
+asmlinkage void neon_aes_cbc_cts_decrypt(u8 out[], u8 const in[],
+					 u32 const rk[], int rounds, int bytes,
+					 u8 const iv[]);
+asmlinkage void neon_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
+				     int rounds, int bytes, u8 ctr[]);
+asmlinkage void neon_aes_xctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
+				      int rounds, int bytes, u8 ctr[],
+				      int byte_ctr);
+asmlinkage void neon_aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[],
+				     int rounds, int bytes, u32 const rk2[],
+				     u8 iv[], int first);
+asmlinkage void neon_aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[],
+				     int rounds, int bytes, u32 const rk2[],
+				     u8 iv[], int first);
+asmlinkage void neon_aes_essiv_cbc_encrypt(u8 out[], u8 const in[],
+					   u32 const rk1[], int rounds,
+					   int blocks, u8 iv[],
+					   u32 const rk2[]);
+asmlinkage void neon_aes_essiv_cbc_decrypt(u8 out[], u8 const in[],
+					   u32 const rk1[], int rounds,
+					   int blocks, u8 iv[],
+					   u32 const rk2[]);
+
+asmlinkage void ce_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
+				   int rounds, int blocks);
+asmlinkage void ce_aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[],
+				   int rounds, int blocks);
+asmlinkage void ce_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
+				   int rounds, int blocks, u8 iv[]);
+asmlinkage void ce_aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[],
+				   int rounds, int blocks, u8 iv[]);
+asmlinkage void ce_aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
+				       int rounds, int bytes, u8 const iv[]);
+asmlinkage void ce_aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
+				       int rounds, int bytes, u8 const iv[]);
+asmlinkage void ce_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
+				   int rounds, int bytes, u8 ctr[]);
+asmlinkage void ce_aes_xctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
+				    int rounds, int bytes, u8 ctr[],
+				    int byte_ctr);
+asmlinkage void ce_aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[],
+				   int rounds, int bytes, u32 const rk2[],
+				   u8 iv[], int first);
+asmlinkage void ce_aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[],
+				   int rounds, int bytes, u32 const rk2[],
+				   u8 iv[], int first);
+asmlinkage void ce_aes_essiv_cbc_encrypt(u8 out[], u8 const in[],
+					 u32 const rk1[], int rounds,
+					 int blocks, u8 iv[], u32 const rk2[]);
+asmlinkage void ce_aes_essiv_cbc_decrypt(u8 out[], u8 const in[],
+					 u32 const rk1[], int rounds,
+					 int blocks, u8 iv[], u32 const rk2[]);
+asmlinkage void ce_aes_mac_update(u8 const in[], u32 const rk[], int rounds,
+				  size_t blocks, u8 dg[], int enc_before,
+				  int enc_after);
 #elif defined(CONFIG_PPC)
 void ppc_expand_key_128(u32 *key_enc, const u8 *key);
 void ppc_expand_key_192(u32 *key_enc, const u8 *key);
diff --git a/include/crypto/chacha20poly1305.h b/include/crypto/chacha20poly1305.h
index 0f71b037702d..0f6d99170aaf 100644
--- a/include/crypto/chacha20poly1305.h
+++ b/include/crypto/chacha20poly1305.h
@@ -46,6 +46,4 @@ bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len
 					 const u64 nonce,
 					 const u8 key[at_least CHACHA20POLY1305_KEY_SIZE]);
 
-bool chacha20poly1305_selftest(void);
-
 #endif /* __CHACHA20POLY1305_H */
diff --git a/include/crypto/gcm.h b/include/crypto/gcm.h
index b524e47bd4d0..1d5f39ff1dc4 100644
--- a/include/crypto/gcm.h
+++ b/include/crypto/gcm.h
@@ -4,7 +4,7 @@
 #include <linux/errno.h>
 
 #include <crypto/aes.h>
-#include <crypto/gf128mul.h>
+#include <crypto/gf128hash.h>
 
 #define GCM_AES_IV_SIZE 12
 #define GCM_RFC4106_IV_SIZE 8
@@ -65,7 +65,7 @@ static inline int crypto_ipsec_check_assoclen(unsigned int assoclen)
 }
 
 struct aesgcm_ctx {
-	be128			ghash_key;
+	struct ghash_key	ghash_key;
 	struct aes_enckey	aes_key;
 	unsigned int		authsize;
 };
diff --git a/include/crypto/polyval.h b/include/crypto/gf128hash.h
index b28b8ef11353..41c557d55965 100644
--- a/include/crypto/polyval.h
+++ b/include/crypto/gf128hash.h
@@ -1,13 +1,14 @@
 /* SPDX-License-Identifier: GPL-2.0-or-later */
 /*
- * POLYVAL library API
+ * GF(2^128) polynomial hashing: GHASH and POLYVAL
  *
  * Copyright 2025 Google LLC
  */
 
-#ifndef _CRYPTO_POLYVAL_H
-#define _CRYPTO_POLYVAL_H
+#ifndef _CRYPTO_GF128HASH_H
+#define _CRYPTO_GF128HASH_H
 
+#include <crypto/ghash.h>
 #include <linux/string.h>
 #include <linux/types.h>
 
@@ -34,6 +35,24 @@ struct polyval_elem {
 };
 
 /**
+ * struct ghash_key - Prepared key for GHASH
+ *
+ * Use ghash_preparekey() to initialize this.
+ */
+struct ghash_key {
+#if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_PPC64)
+	/** @htable: GHASH key format used by the POWER8 assembly code */
+	u64 htable[4][2];
+#elif defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && \
+	(defined(CONFIG_RISCV) || defined(CONFIG_S390))
+	/** @h_raw: The hash key H, in GHASH format */
+	u8 h_raw[GHASH_BLOCK_SIZE];
+#endif
+	/** @h: The hash key H, in POLYVAL format */
+	struct polyval_elem h;
+};
+
+/**
  * struct polyval_key - Prepared key for POLYVAL
  *
  * This may contain just the raw key H, or it may contain precomputed key
@@ -44,20 +63,28 @@ struct polyval_elem {
  * exponentiation repeats the POLYVAL dot operation, with its "extra" x^-128.
  */
 struct polyval_key {
-#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH
-#ifdef CONFIG_ARM64
-	/** @h_powers: Powers of the hash key H^8 through H^1 */
-	struct polyval_elem h_powers[8];
-#elif defined(CONFIG_X86)
+#if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && \
+	(defined(CONFIG_ARM64) || defined(CONFIG_X86))
 	/** @h_powers: Powers of the hash key H^8 through H^1 */
 	struct polyval_elem h_powers[8];
 #else
-#error "Unhandled arch"
-#endif
-#else /* CONFIG_CRYPTO_LIB_POLYVAL_ARCH */
 	/** @h: The hash key H */
 	struct polyval_elem h;
-#endif /* !CONFIG_CRYPTO_LIB_POLYVAL_ARCH */
+#endif
+};
+
+/**
+ * struct ghash_ctx - Context for computing a GHASH value
+ * @key: Pointer to the prepared GHASH key.  The user of the API is
+ *	 responsible for ensuring that the key lives as long as the context.
+ * @acc: The accumulator.  It is stored in POLYVAL format rather than GHASH
+ *	 format, since most implementations want it in POLYVAL format.
+ * @partial: Number of data bytes processed so far modulo GHASH_BLOCK_SIZE
+ */
+struct ghash_ctx {
+	const struct ghash_key *key;
+	struct polyval_elem acc;
+	size_t partial;
 };
 
 /**
@@ -74,6 +101,18 @@ struct polyval_ctx {
 };
 
 /**
+ * ghash_preparekey() - Prepare a GHASH key
+ * @key: (output) The key structure to initialize
+ * @raw_key: The raw hash key
+ *
+ * Initialize a GHASH key structure from a raw key.
+ *
+ * Context: Any context.
+ */
+void ghash_preparekey(struct ghash_key *key,
+		      const u8 raw_key[GHASH_BLOCK_SIZE]);
+
+/**
  * polyval_preparekey() - Prepare a POLYVAL key
  * @key: (output) The key structure to initialize
  * @raw_key: The raw hash key
@@ -84,18 +123,20 @@ struct polyval_ctx {
  *
  * Context: Any context.
  */
-#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH
 void polyval_preparekey(struct polyval_key *key,
 			const u8 raw_key[POLYVAL_BLOCK_SIZE]);
 
-#else
-static inline void polyval_preparekey(struct polyval_key *key,
-				      const u8 raw_key[POLYVAL_BLOCK_SIZE])
+/**
+ * ghash_init() - Initialize a GHASH context for a new message
+ * @ctx: The context to initialize
+ * @key: The key to use.  Note that a pointer to the key is saved in the
+ *	 context, so the key must live at least as long as the context.
+ */
+static inline void ghash_init(struct ghash_ctx *ctx,
+			      const struct ghash_key *key)
 {
-	/* Just a simple copy, so inline it. */
-	memcpy(key->h.bytes, raw_key, POLYVAL_BLOCK_SIZE);
+	*ctx = (struct ghash_ctx){ .key = key };
 }
-#endif
 
 /**
  * polyval_init() - Initialize a POLYVAL context for a new message
@@ -142,6 +183,18 @@ static inline void polyval_export_blkaligned(const struct polyval_ctx *ctx,
 }
 
 /**
+ * ghash_update() - Update a GHASH context with message data
+ * @ctx: The context to update; must have been initialized
+ * @data: The message data
+ * @len: The data length in bytes.  Doesn't need to be block-aligned.
+ *
+ * This can be called any number of times.
+ *
+ * Context: Any context.
+ */
+void ghash_update(struct ghash_ctx *ctx, const u8 *data, size_t len);
+
+/**
  * polyval_update() - Update a POLYVAL context with message data
  * @ctx: The context to update; must have been initialized
  * @data: The message data
@@ -154,6 +207,20 @@ static inline void polyval_export_blkaligned(const struct polyval_ctx *ctx,
 void polyval_update(struct polyval_ctx *ctx, const u8 *data, size_t len);
 
 /**
+ * ghash_final() - Finish computing a GHASH value
+ * @ctx: The context to finalize
+ * @out: The output value
+ *
+ * If the total data length isn't a multiple of GHASH_BLOCK_SIZE, then the
+ * final block is automatically zero-padded.
+ *
+ * After finishing, this zeroizes @ctx.  So the caller does not need to do it.
+ *
+ * Context: Any context.
+ */
+void ghash_final(struct ghash_ctx *ctx, u8 out[GHASH_BLOCK_SIZE]);
+
+/**
  * polyval_final() - Finish computing a POLYVAL value
  * @ctx: The context to finalize
  * @out: The output value
@@ -168,6 +235,25 @@ void polyval_update(struct polyval_ctx *ctx, const u8 *data, size_t len);
 void polyval_final(struct polyval_ctx *ctx, u8 out[POLYVAL_BLOCK_SIZE]);
 
 /**
+ * ghash() - Compute a GHASH value
+ * @key: The prepared key
+ * @data: The message data
+ * @len: The data length in bytes.  Doesn't need to be block-aligned.
+ * @out: The output value
+ *
+ * Context: Any context.
+ */
+static inline void ghash(const struct ghash_key *key, const u8 *data,
+			 size_t len, u8 out[GHASH_BLOCK_SIZE])
+{
+	struct ghash_ctx ctx;
+
+	ghash_init(&ctx, key);
+	ghash_update(&ctx, data, len);
+	ghash_final(&ctx, out);
+}
+
+/**
  * polyval() - Compute a POLYVAL value
  * @key: The prepared key
  * @data: The message data
@@ -187,4 +273,4 @@ static inline void polyval(const struct polyval_key *key,
 	polyval_final(&ctx, out);
 }
 
-#endif /* _CRYPTO_POLYVAL_H */
+#endif /* _CRYPTO_GF128HASH_H */
diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h
index b0853f7cada0..6ed2a8351902 100644
--- a/include/crypto/gf128mul.h
+++ b/include/crypto/gf128mul.h
@@ -215,25 +215,14 @@ static inline void gf128mul_x_ble(le128 *r, const le128 *x)
 	r->b = cpu_to_le64((b << 1) ^ _tt);
 }
 
-/* 4k table optimization */
-
-struct gf128mul_4k {
-	be128 t[256];
-};
-
-struct gf128mul_4k *gf128mul_init_4k_lle(const be128 *g);
-void gf128mul_4k_lle(be128 *a, const struct gf128mul_4k *t);
 void gf128mul_x8_ble(le128 *r, const le128 *x);
-static inline void gf128mul_free_4k(struct gf128mul_4k *t)
-{
-	kfree_sensitive(t);
-}
-
 
 /* 64k table optimization, implemented for bbe */
 
 struct gf128mul_64k {
-	struct gf128mul_4k *t[16];
+	struct {
+		be128 t[256];
+	} *t[16];
 };
 
 /* First initialize with the constant factor with which you
diff --git a/include/crypto/ghash.h b/include/crypto/ghash.h
index 043d938e9a2c..d187e5af9925 100644
--- a/include/crypto/ghash.h
+++ b/include/crypto/ghash.h
@@ -6,19 +6,7 @@
 #ifndef __CRYPTO_GHASH_H__
 #define __CRYPTO_GHASH_H__
 
-#include <linux/types.h>
-
 #define GHASH_BLOCK_SIZE	16
 #define GHASH_DIGEST_SIZE	16
 
-struct gf128mul_4k;
-
-struct ghash_ctx {
-	struct gf128mul_4k *gf128;
-};
-
-struct ghash_desc_ctx {
-	u8 buffer[GHASH_BLOCK_SIZE];
-};
-
 #endif
diff --git a/include/crypto/internal/blockhash.h b/include/crypto/internal/blockhash.h
deleted file mode 100644
index 52d9d4c82493..000000000000
--- a/include/crypto/internal/blockhash.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Handle partial blocks for block hash.
- *
- * Copyright (c) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
- * Copyright (c) 2025 Herbert Xu <herbert@gondor.apana.org.au>
- */
-
-#ifndef _CRYPTO_INTERNAL_BLOCKHASH_H
-#define _CRYPTO_INTERNAL_BLOCKHASH_H
-
-#include <linux/string.h>
-#include <linux/types.h>
-
-#define BLOCK_HASH_UPDATE_BASE(block_fn, state, src, nbytes, bs, dv,	\
-			       buf, buflen)				\
-	({								\
-		typeof(block_fn) *_block_fn = &(block_fn);		\
-		typeof(state + 0) _state = (state);			\
-		unsigned int _buflen = (buflen);			\
-		size_t _nbytes = (nbytes);				\
-		unsigned int _bs = (bs);				\
-		const u8 *_src = (src);					\
-		u8 *_buf = (buf);					\
-		while ((_buflen + _nbytes) >= _bs) {			\
-			const u8 *data = _src;				\
-			size_t len = _nbytes;				\
-			size_t blocks;					\
-			int remain;					\
-			if (_buflen) {					\
-				remain = _bs - _buflen;			\
-				memcpy(_buf + _buflen, _src, remain);	\
-				data = _buf;				\
-				len = _bs;				\
-			}						\
-			remain = len % bs;				\
-			blocks = (len - remain) / (dv);			\
-			(*_block_fn)(_state, data, blocks);		\
-			_src += len - remain - _buflen;			\
-			_nbytes -= len - remain - _buflen;		\
-			_buflen = 0;					\
-		}							\
-		memcpy(_buf + _buflen, _src, _nbytes);			\
-		_buflen += _nbytes;					\
-	})
-
-#define BLOCK_HASH_UPDATE(block, state, src, nbytes, bs, buf, buflen) \
-	BLOCK_HASH_UPDATE_BASE(block, state, src, nbytes, bs, 1, buf, buflen)
-#define BLOCK_HASH_UPDATE_BLOCKS(block, state, src, nbytes, bs, buf, buflen) \
-	BLOCK_HASH_UPDATE_BASE(block, state, src, nbytes, bs, bs, buf, buflen)
-
-#endif	/* _CRYPTO_INTERNAL_BLOCKHASH_H */
diff --git a/include/crypto/sm3.h b/include/crypto/sm3.h
index c8d02c86c298..371e8a661705 100644
--- a/include/crypto/sm3.h
+++ b/include/crypto/sm3.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * Common values for SM3 algorithm
+ * SM3 hash algorithm
  *
  * Copyright (C) 2017 ARM Limited or its affiliates.
  * Copyright (C) 2017 Gilad Ben-Yossef <gilad@benyossef.com>
@@ -14,10 +14,6 @@
 
 #define SM3_DIGEST_SIZE	32
 #define SM3_BLOCK_SIZE	64
-#define SM3_STATE_SIZE	40
-
-#define SM3_T1		0x79CC4519
-#define SM3_T2		0x7A879D8A
 
 #define SM3_IVA		0x7380166f
 #define SM3_IVB		0x4914b2b9
@@ -28,37 +24,64 @@
 #define SM3_IVG		0xe38dee4d
 #define SM3_IVH		0xb0fb0e4e
 
-extern const u8 sm3_zero_message_hash[SM3_DIGEST_SIZE];
+/* State for the SM3 compression function */
+struct sm3_block_state {
+	u32 h[SM3_DIGEST_SIZE / 4];
+};
 
-struct sm3_state {
-	u32 state[SM3_DIGEST_SIZE / 4];
-	u64 count;
-	u8 buffer[SM3_BLOCK_SIZE];
+/**
+ * struct sm3_ctx - Context for hashing a message with SM3
+ * @state: the compression function state
+ * @bytecount: number of bytes processed so far
+ * @buf: partial block buffer; bytecount % SM3_BLOCK_SIZE bytes are valid
+ */
+struct sm3_ctx {
+	struct sm3_block_state state;
+	u64 bytecount;
+	u8 buf[SM3_BLOCK_SIZE] __aligned(__alignof__(__be64));
 };
 
-/*
- * Stand-alone implementation of the SM3 algorithm. It is designed to
- * have as little dependencies as possible so it can be used in the
- * kexec_file purgatory. In other cases you should generally use the
- * hash APIs from include/crypto/hash.h. Especially when hashing large
- * amounts of data as those APIs may be hw-accelerated.
+/**
+ * sm3_init() - Initialize an SM3 context for a new message
+ * @ctx: the context to initialize
+ *
+ * If you don't need incremental computation, consider sm3() instead.
+ *
+ * Context: Any context.
+ */
+void sm3_init(struct sm3_ctx *ctx);
+
+/**
+ * sm3_update() - Update an SM3 context with message data
+ * @ctx: the context to update; must have been initialized
+ * @data: the message data
+ * @len: the data length in bytes
+ *
+ * This can be called any number of times.
  *
- * For details see lib/crypto/sm3.c
+ * Context: Any context.
  */
+void sm3_update(struct sm3_ctx *ctx, const u8 *data, size_t len);
 
-static inline void sm3_init(struct sm3_state *sctx)
-{
-	sctx->state[0] = SM3_IVA;
-	sctx->state[1] = SM3_IVB;
-	sctx->state[2] = SM3_IVC;
-	sctx->state[3] = SM3_IVD;
-	sctx->state[4] = SM3_IVE;
-	sctx->state[5] = SM3_IVF;
-	sctx->state[6] = SM3_IVG;
-	sctx->state[7] = SM3_IVH;
-	sctx->count = 0;
-}
+/**
+ * sm3_final() - Finish computing an SM3 message digest
+ * @ctx: the context to finalize; must have been initialized
+ * @out: (output) the resulting SM3 message digest
+ *
+ * After finishing, this zeroizes @ctx.  So the caller does not need to do it.
+ *
+ * Context: Any context.
+ */
+void sm3_final(struct sm3_ctx *ctx, u8 out[at_least SM3_DIGEST_SIZE]);
 
-void sm3_block_generic(struct sm3_state *sctx, u8 const *data, int blocks);
+/**
+ * sm3() - Compute SM3 message digest in one shot
+ * @data: the message data
+ * @len: the data length in bytes
+ * @out: (output) the resulting SM3 message digest
+ *
+ * Context: Any context.
+ */
+void sm3(const u8 *data, size_t len, u8 out[at_least SM3_DIGEST_SIZE]);
 
-#endif
+#endif /* _CRYPTO_SM3_H */
diff --git a/include/crypto/sm3_base.h b/include/crypto/sm3_base.h
deleted file mode 100644
index 7c53570bc05e..000000000000
--- a/include/crypto/sm3_base.h
+++ /dev/null
@@ -1,82 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * sm3_base.h - core logic for SM3 implementations
- *
- * Copyright (C) 2017 ARM Limited or its affiliates.
- * Written by Gilad Ben-Yossef <gilad@benyossef.com>
- */
-
-#ifndef _CRYPTO_SM3_BASE_H
-#define _CRYPTO_SM3_BASE_H
-
-#include <crypto/internal/hash.h>
-#include <crypto/sm3.h>
-#include <linux/math.h>
-#include <linux/module.h>
-#include <linux/string.h>
-#include <linux/types.h>
-#include <linux/unaligned.h>
-
-typedef void (sm3_block_fn)(struct sm3_state *sst, u8 const *src, int blocks);
-
-static inline int sm3_base_init(struct shash_desc *desc)
-{
-	sm3_init(shash_desc_ctx(desc));
-	return 0;
-}
-
-static inline int sm3_base_do_update_blocks(struct shash_desc *desc,
-					    const u8 *data, unsigned int len,
-					    sm3_block_fn *block_fn)
-{
-	unsigned int remain = len - round_down(len, SM3_BLOCK_SIZE);
-	struct sm3_state *sctx = shash_desc_ctx(desc);
-
-	sctx->count += len - remain;
-	block_fn(sctx, data, len / SM3_BLOCK_SIZE);
-	return remain;
-}
-
-static inline int sm3_base_do_finup(struct shash_desc *desc,
-				    const u8 *src, unsigned int len,
-				    sm3_block_fn *block_fn)
-{
-	unsigned int bit_offset = SM3_BLOCK_SIZE / 8 - 1;
-	struct sm3_state *sctx = shash_desc_ctx(desc);
-	union {
-		__be64 b64[SM3_BLOCK_SIZE / 4];
-		u8 u8[SM3_BLOCK_SIZE * 2];
-	} block = {};
-
-	if (len >= SM3_BLOCK_SIZE) {
-		int remain;
-
-		remain = sm3_base_do_update_blocks(desc, src, len, block_fn);
-		src += len - remain;
-		len = remain;
-	}
-
-	if (len >= bit_offset * 8)
-		bit_offset += SM3_BLOCK_SIZE / 8;
-	memcpy(&block, src, len);
-	block.u8[len] = 0x80;
-	sctx->count += len;
-	block.b64[bit_offset] = cpu_to_be64(sctx->count << 3);
-	block_fn(sctx, block.u8, (bit_offset + 1) * 8 / SM3_BLOCK_SIZE);
-	memzero_explicit(&block, sizeof(block));
-
-	return 0;
-}
-
-static inline int sm3_base_finish(struct shash_desc *desc, u8 *out)
-{
-	struct sm3_state *sctx = shash_desc_ctx(desc);
-	__be32 *digest = (__be32 *)out;
-	int i;
-
-	for (i = 0; i < SM3_DIGEST_SIZE / sizeof(__be32); i++)
-		put_unaligned_be32(sctx->state[i], digest++);
-	return 0;
-}
-
-#endif /* _CRYPTO_SM3_BASE_H */