diff options
154 files changed, 4879 insertions, 4875 deletions
diff --git a/MAINTAINERS b/MAINTAINERS index b4be104bb2a8..3adc870d523b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12279,10 +12279,10 @@ F: arch/powerpc/crypto/aes_cbc.c F: arch/powerpc/crypto/aes_ctr.c F: arch/powerpc/crypto/aes_xts.c F: arch/powerpc/crypto/aesp8-ppc.* -F: arch/powerpc/crypto/ghash.c -F: arch/powerpc/crypto/ghashp8-ppc.pl F: arch/powerpc/crypto/ppc-xlate.pl F: arch/powerpc/crypto/vmx.c +F: lib/crypto/powerpc/gf128hash.h +F: lib/crypto/powerpc/ghashp8-ppc.pl IBM ServeRAID RAID DRIVER S: Orphan diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index b9c28c818b7c..f884b8b2fd93 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -3,26 +3,17 @@ menu "Accelerated Cryptographic Algorithms for CPU (arm)" config CRYPTO_GHASH_ARM_CE - tristate "Hash functions: GHASH (PMULL/NEON/ARMv8 Crypto Extensions)" + tristate "AEAD cipher: AES in GCM mode (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_AEAD - select CRYPTO_HASH - select CRYPTO_CRYPTD select CRYPTO_LIB_AES select CRYPTO_LIB_GF128MUL help - GCM GHASH function (NIST SP800-38D) + AEAD cipher: AES-GCM Architecture: arm using - - PMULL (Polynomial Multiply Long) instructions - - NEON (Advanced SIMD) extensions - ARMv8 Crypto Extensions - Use an implementation of GHASH (used by the GCM AEAD chaining mode) - that uses the 64x64 to 128 bit polynomial multiplication (vmull.p64) - that is part of the ARMv8 Crypto Extensions, or a slower variant that - uses the vmull.p8 instruction that is part of the basic NEON ISA. - config CRYPTO_AES_ARM_BS tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (bit-sliced NEON)" depends on KERNEL_MODE_NEON diff --git a/arch/arm/crypto/ghash-ce-core.S b/arch/arm/crypto/ghash-ce-core.S index 858c0d66798b..a449525d61f8 100644 --- a/arch/arm/crypto/ghash-ce-core.S +++ b/arch/arm/crypto/ghash-ce-core.S @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Accelerated GHASH implementation with NEON/ARMv8 vmull.p8/64 instructions. + * Accelerated AES-GCM implementation with ARMv8 Crypto Extensions. * * Copyright (C) 2015 - 2017 Linaro Ltd. * Copyright (C) 2023 Google LLC. <ardb@google.com> @@ -29,39 +29,10 @@ XM_H .req d7 XH_L .req d8 - t0l .req d10 - t0h .req d11 - t1l .req d12 - t1h .req d13 - t2l .req d14 - t2h .req d15 - t3l .req d16 - t3h .req d17 - t4l .req d18 - t4h .req d19 - - t0q .req q5 - t1q .req q6 - t2q .req q7 - t3q .req q8 - t4q .req q9 XH2 .req q9 - s1l .req d20 - s1h .req d21 - s2l .req d22 - s2h .req d23 - s3l .req d24 - s3h .req d25 - s4l .req d26 - s4h .req d27 - MASK .req d28 - SHASH2_p8 .req d28 - k16 .req d29 - k32 .req d30 - k48 .req d31 SHASH2_p64 .req d31 HH .req q10 @@ -93,72 +64,6 @@ .text - .macro __pmull_p64, rd, rn, rm, b1, b2, b3, b4 - vmull.p64 \rd, \rn, \rm - .endm - - /* - * This implementation of 64x64 -> 128 bit polynomial multiplication - * using vmull.p8 instructions (8x8 -> 16) is taken from the paper - * "Fast Software Polynomial Multiplication on ARM Processors Using - * the NEON Engine" by Danilo Camara, Conrado Gouvea, Julio Lopez and - * Ricardo Dahab (https://hal.inria.fr/hal-01506572) - * - * It has been slightly tweaked for in-order performance, and to allow - * 'rq' to overlap with 'ad' or 'bd'. - */ - .macro __pmull_p8, rq, ad, bd, b1=t4l, b2=t3l, b3=t4l, b4=t3l - vext.8 t0l, \ad, \ad, #1 @ A1 - .ifc \b1, t4l - vext.8 t4l, \bd, \bd, #1 @ B1 - .endif - vmull.p8 t0q, t0l, \bd @ F = A1*B - vext.8 t1l, \ad, \ad, #2 @ A2 - vmull.p8 t4q, \ad, \b1 @ E = A*B1 - .ifc \b2, t3l - vext.8 t3l, \bd, \bd, #2 @ B2 - .endif - vmull.p8 t1q, t1l, \bd @ H = A2*B - vext.8 t2l, \ad, \ad, #3 @ A3 - vmull.p8 t3q, \ad, \b2 @ G = A*B2 - veor t0q, t0q, t4q @ L = E + F - .ifc \b3, t4l - vext.8 t4l, \bd, \bd, #3 @ B3 - .endif - vmull.p8 t2q, t2l, \bd @ J = A3*B - veor t0l, t0l, t0h @ t0 = (L) (P0 + P1) << 8 - veor t1q, t1q, t3q @ M = G + H - .ifc \b4, t3l - vext.8 t3l, \bd, \bd, #4 @ B4 - .endif - vmull.p8 t4q, \ad, \b3 @ I = A*B3 - veor t1l, t1l, t1h @ t1 = (M) (P2 + P3) << 16 - vmull.p8 t3q, \ad, \b4 @ K = A*B4 - vand t0h, t0h, k48 - vand t1h, t1h, k32 - veor t2q, t2q, t4q @ N = I + J - veor t0l, t0l, t0h - veor t1l, t1l, t1h - veor t2l, t2l, t2h @ t2 = (N) (P4 + P5) << 24 - vand t2h, t2h, k16 - veor t3l, t3l, t3h @ t3 = (K) (P6 + P7) << 32 - vmov.i64 t3h, #0 - vext.8 t0q, t0q, t0q, #15 - veor t2l, t2l, t2h - vext.8 t1q, t1q, t1q, #14 - vmull.p8 \rq, \ad, \bd @ D = A*B - vext.8 t2q, t2q, t2q, #13 - vext.8 t3q, t3q, t3q, #12 - veor t0q, t0q, t1q - veor t2q, t2q, t3q - veor \rq, \rq, t0q - veor \rq, \rq, t2q - .endm - - // - // PMULL (64x64->128) based reduction for CPUs that can do - // it in a single instruction. - // .macro __pmull_reduce_p64 vmull.p64 T1, XL_L, MASK @@ -170,30 +75,7 @@ vmull.p64 XL, T1_H, MASK .endm - // - // Alternative reduction for CPUs that lack support for the - // 64x64->128 PMULL instruction - // - .macro __pmull_reduce_p8 - veor XL_H, XL_H, XM_L - veor XH_L, XH_L, XM_H - - vshl.i64 T1, XL, #57 - vshl.i64 T2, XL, #62 - veor T1, T1, T2 - vshl.i64 T2, XL, #63 - veor T1, T1, T2 - veor XL_H, XL_H, T1_L - veor XH_L, XH_L, T1_H - - vshr.u64 T1, XL, #1 - veor XH, XH, XL - veor XL, XL, T1 - vshr.u64 T1, T1, #6 - vshr.u64 XL, XL, #1 - .endm - - .macro ghash_update, pn, enc, aggregate=1, head=1 + .macro ghash_update, enc, aggregate=1, head=1 vld1.64 {XL}, [r1] .if \head @@ -206,8 +88,7 @@ b 3f .endif -0: .ifc \pn, p64 - .if \aggregate +0: .if \aggregate tst r0, #3 // skip until #blocks is a bne 2f // round multiple of 4 @@ -288,7 +169,6 @@ b 1b .endif - .endif 2: vld1.8 {T1}, [r2]! @@ -308,15 +188,15 @@ veor T1_L, T1_L, XL_H veor XL, XL, IN1 - __pmull_\pn XH, XL_H, SHASH_H, s1h, s2h, s3h, s4h @ a1 * b1 + vmull.p64 XH, XL_H, SHASH_H @ a1 * b1 veor T1, T1, XL - __pmull_\pn XL, XL_L, SHASH_L, s1l, s2l, s3l, s4l @ a0 * b0 - __pmull_\pn XM, T1_L, SHASH2_\pn @ (a1+a0)(b1+b0) + vmull.p64 XL, XL_L, SHASH_L @ a0 * b0 + vmull.p64 XM, T1_L, SHASH2_p64 @ (a1+a0)(b1+b0) 4: veor T1, XL, XH veor XM, XM, T1 - __pmull_reduce_\pn + __pmull_reduce_p64 veor T1, T1, XH veor XL, XL, T1 @@ -325,8 +205,8 @@ .endm /* - * void pmull_ghash_update(int blocks, u64 dg[], const char *src, - * struct ghash_key const *k, const char *head) + * void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, + * u64 const h[4][2], const char *head) */ ENTRY(pmull_ghash_update_p64) vld1.64 {SHASH}, [r3]! @@ -341,35 +221,12 @@ ENTRY(pmull_ghash_update_p64) vmov.i8 MASK, #0xe1 vshl.u64 MASK, MASK, #57 - ghash_update p64 + ghash_update vst1.64 {XL}, [r1] bx lr ENDPROC(pmull_ghash_update_p64) -ENTRY(pmull_ghash_update_p8) - vld1.64 {SHASH}, [r3] - veor SHASH2_p8, SHASH_L, SHASH_H - - vext.8 s1l, SHASH_L, SHASH_L, #1 - vext.8 s2l, SHASH_L, SHASH_L, #2 - vext.8 s3l, SHASH_L, SHASH_L, #3 - vext.8 s4l, SHASH_L, SHASH_L, #4 - vext.8 s1h, SHASH_H, SHASH_H, #1 - vext.8 s2h, SHASH_H, SHASH_H, #2 - vext.8 s3h, SHASH_H, SHASH_H, #3 - vext.8 s4h, SHASH_H, SHASH_H, #4 - - vmov.i64 k16, #0xffff - vmov.i64 k32, #0xffffffff - vmov.i64 k48, #0xffffffffffff - - ghash_update p8 - vst1.64 {XL}, [r1] - - bx lr -ENDPROC(pmull_ghash_update_p8) - e0 .req q9 e1 .req q10 e2 .req q11 @@ -536,7 +393,7 @@ ENTRY(pmull_gcm_encrypt) vld1.64 {SHASH}, [r3] - ghash_update p64, enc, head=0 + ghash_update enc, head=0 vst1.64 {XL}, [r1] pop {r4-r8, pc} @@ -554,7 +411,7 @@ ENTRY(pmull_gcm_decrypt) vld1.64 {SHASH}, [r3] - ghash_update p64, dec, head=0 + ghash_update dec, head=0 vst1.64 {XL}, [r1] pop {r4-r8, pc} @@ -603,7 +460,7 @@ ENTRY(pmull_gcm_enc_final) vshl.u64 MASK, MASK, #57 mov r0, #1 bne 3f // process head block first - ghash_update p64, aggregate=0, head=0 + ghash_update aggregate=0, head=0 vrev64.8 XL, XL vext.8 XL, XL, XL, #8 @@ -660,7 +517,7 @@ ENTRY(pmull_gcm_dec_final) vshl.u64 MASK, MASK, #57 mov r0, #1 bne 3f // process head block first - ghash_update p64, aggregate=0, head=0 + ghash_update aggregate=0, head=0 vrev64.8 XL, XL vext.8 XL, XL, XL, #8 diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c index 454adcc62cc6..9aa0ada5b627 100644 --- a/arch/arm/crypto/ghash-ce-glue.c +++ b/arch/arm/crypto/ghash-ce-glue.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions. + * AES-GCM using ARMv8 Crypto Extensions * * Copyright (C) 2015 - 2018 Linaro Ltd. * Copyright (C) 2023 Google LLC. @@ -14,7 +14,6 @@ #include <crypto/gf128mul.h> #include <crypto/ghash.h> #include <crypto/internal/aead.h> -#include <crypto/internal/hash.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> #include <linux/cpufeature.h> @@ -25,20 +24,14 @@ #include <linux/string.h> #include <linux/unaligned.h> -MODULE_DESCRIPTION("GHASH hash function using ARMv8 Crypto Extensions"); +MODULE_DESCRIPTION("AES-GCM using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel <ardb@kernel.org>"); MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("ghash"); MODULE_ALIAS_CRYPTO("gcm(aes)"); MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))"); #define RFC4106_NONCE_SIZE 4 -struct ghash_key { - be128 k; - u64 h[][2]; -}; - struct gcm_key { u64 h[4][2]; u32 rk[AES_MAX_KEYLENGTH_U32]; @@ -46,84 +39,8 @@ struct gcm_key { u8 nonce[]; // for RFC4106 nonce }; -struct arm_ghash_desc_ctx { - u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)]; -}; - asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, - u64 const h[][2], const char *head); - -asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, - u64 const h[][2], const char *head); - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_p64); - -static int ghash_init(struct shash_desc *desc) -{ - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - - *ctx = (struct arm_ghash_desc_ctx){}; - return 0; -} - -static void ghash_do_update(int blocks, u64 dg[], const char *src, - struct ghash_key *key, const char *head) -{ - kernel_neon_begin(); - if (static_branch_likely(&use_p64)) - pmull_ghash_update_p64(blocks, dg, src, key->h, head); - else - pmull_ghash_update_p8(blocks, dg, src, key->h, head); - kernel_neon_end(); -} - -static int ghash_update(struct shash_desc *desc, const u8 *src, - unsigned int len) -{ - struct ghash_key *key = crypto_shash_ctx(desc->tfm); - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - int blocks; - - blocks = len / GHASH_BLOCK_SIZE; - ghash_do_update(blocks, ctx->digest, src, key, NULL); - return len - blocks * GHASH_BLOCK_SIZE; -} - -static int ghash_export(struct shash_desc *desc, void *out) -{ - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - u8 *dst = out; - - put_unaligned_be64(ctx->digest[1], dst); - put_unaligned_be64(ctx->digest[0], dst + 8); - return 0; -} - -static int ghash_import(struct shash_desc *desc, const void *in) -{ - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - const u8 *src = in; - - ctx->digest[1] = get_unaligned_be64(src); - ctx->digest[0] = get_unaligned_be64(src + 8); - return 0; -} - -static int ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *dst) -{ - struct ghash_key *key = crypto_shash_ctx(desc->tfm); - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - - if (len) { - u8 buf[GHASH_BLOCK_SIZE] = {}; - - memcpy(buf, src, len); - ghash_do_update(1, ctx->digest, buf, key, NULL); - memzero_explicit(buf, sizeof(buf)); - } - return ghash_export(desc, dst); -} + u64 const h[4][2], const char *head); static void ghash_reflect(u64 h[], const be128 *k) { @@ -136,53 +53,6 @@ static void ghash_reflect(u64 h[], const be128 *k) h[1] ^= 0xc200000000000000UL; } -static int ghash_setkey(struct crypto_shash *tfm, - const u8 *inkey, unsigned int keylen) -{ - struct ghash_key *key = crypto_shash_ctx(tfm); - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - /* needed for the fallback */ - memcpy(&key->k, inkey, GHASH_BLOCK_SIZE); - ghash_reflect(key->h[0], &key->k); - - if (static_branch_likely(&use_p64)) { - be128 h = key->k; - - gf128mul_lle(&h, &key->k); - ghash_reflect(key->h[1], &h); - - gf128mul_lle(&h, &key->k); - ghash_reflect(key->h[2], &h); - - gf128mul_lle(&h, &key->k); - ghash_reflect(key->h[3], &h); - } - return 0; -} - -static struct shash_alg ghash_alg = { - .digestsize = GHASH_DIGEST_SIZE, - .init = ghash_init, - .update = ghash_update, - .finup = ghash_finup, - .setkey = ghash_setkey, - .export = ghash_export, - .import = ghash_import, - .descsize = sizeof(struct arm_ghash_desc_ctx), - .statesize = sizeof(struct ghash_desc_ctx), - - .base.cra_name = "ghash", - .base.cra_driver_name = "ghash-ce", - .base.cra_priority = 300, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .base.cra_blocksize = GHASH_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ghash_key) + sizeof(u64[2]), - .base.cra_module = THIS_MODULE, -}; - void pmull_gcm_encrypt(int blocks, u64 dg[], const char *src, struct gcm_key const *k, char *dst, const char *iv, int rounds, u32 counter); @@ -561,39 +431,15 @@ static struct aead_alg gcm_aes_algs[] = {{ static int __init ghash_ce_mod_init(void) { - int err; - - if (!(elf_hwcap & HWCAP_NEON)) + if (!(elf_hwcap & HWCAP_NEON) || !(elf_hwcap2 & HWCAP2_PMULL)) return -ENODEV; - if (elf_hwcap2 & HWCAP2_PMULL) { - err = crypto_register_aeads(gcm_aes_algs, - ARRAY_SIZE(gcm_aes_algs)); - if (err) - return err; - ghash_alg.base.cra_ctxsize += 3 * sizeof(u64[2]); - static_branch_enable(&use_p64); - } - - err = crypto_register_shash(&ghash_alg); - if (err) - goto err_aead; - - return 0; - -err_aead: - if (elf_hwcap2 & HWCAP2_PMULL) - crypto_unregister_aeads(gcm_aes_algs, - ARRAY_SIZE(gcm_aes_algs)); - return err; + return crypto_register_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs)); } static void __exit ghash_ce_mod_exit(void) { - crypto_unregister_shash(&ghash_alg); - if (elf_hwcap2 & HWCAP2_PMULL) - crypto_unregister_aeads(gcm_aes_algs, - ARRAY_SIZE(gcm_aes_algs)); + crypto_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs)); } module_init(ghash_ce_mod_init); diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index b67d5b1fc45b..b4458bee767a 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1916,9 +1916,9 @@ CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_ECHAINIV=y CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_SHA3=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_GHASH_ARM64_CE=y -CONFIG_CRYPTO_SM3_ARM64_CE=m CONFIG_CRYPTO_AES_ARM64_CE_BLK=y CONFIG_CRYPTO_AES_ARM64_BS=m CONFIG_CRYPTO_AES_ARM64_CE_CCM=y diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 81ed892b3b72..0ac0fbfea10c 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -3,40 +3,17 @@ menu "Accelerated Cryptographic Algorithms for CPU (arm64)" config CRYPTO_GHASH_ARM64_CE - tristate "Hash functions: GHASH (ARMv8 Crypto Extensions)" + tristate "AEAD cipher: AES in GCM mode (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON - select CRYPTO_HASH select CRYPTO_LIB_AES select CRYPTO_LIB_GF128MUL select CRYPTO_AEAD help - GCM GHASH function (NIST SP800-38D) + AEAD cipher: AES-GCM Architecture: arm64 using: - ARMv8 Crypto Extensions -config CRYPTO_SM3_NEON - tristate "Hash functions: SM3 (NEON)" - depends on KERNEL_MODE_NEON - select CRYPTO_HASH - select CRYPTO_LIB_SM3 - help - SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012) - - Architecture: arm64 using: - - NEON (Advanced SIMD) extensions - -config CRYPTO_SM3_ARM64_CE - tristate "Hash functions: SM3 (ARMv8.2 Crypto Extensions)" - depends on KERNEL_MODE_NEON - select CRYPTO_HASH - select CRYPTO_LIB_SM3 - help - SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012) - - Architecture: arm64 using: - - ARMv8.2 Crypto Extensions - config CRYPTO_AES_ARM64_CE_BLK tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON @@ -144,7 +121,7 @@ config CRYPTO_AES_ARM64_CE_CCM select CRYPTO_ALGAPI select CRYPTO_AES_ARM64_CE_BLK select CRYPTO_AEAD - select CRYPTO_LIB_AES + select CRYPTO_LIB_AES_CBC_MACS help AEAD cipher: AES cipher algorithms (FIPS-197) with CCM (Counter with Cipher Block Chaining-Message Authentication Code) diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile index 3574e917bc37..a169f9033401 100644 --- a/arch/arm64/crypto/Makefile +++ b/arch/arm64/crypto/Makefile @@ -5,12 +5,6 @@ # Copyright (C) 2014 Linaro Ltd <ard.biesheuvel@linaro.org> # -obj-$(CONFIG_CRYPTO_SM3_NEON) += sm3-neon.o -sm3-neon-y := sm3-neon-glue.o sm3-neon-core.o - -obj-$(CONFIG_CRYPTO_SM3_ARM64_CE) += sm3-ce.o -sm3-ce-y := sm3-ce-glue.o sm3-ce-core.o - obj-$(CONFIG_CRYPTO_SM4_ARM64_CE) += sm4-ce-cipher.o sm4-ce-cipher-y := sm4-ce-cipher-glue.o sm4-ce-cipher-core.o @@ -33,10 +27,10 @@ obj-$(CONFIG_CRYPTO_AES_ARM64_CE_CCM) += aes-ce-ccm.o aes-ce-ccm-y := aes-ce-ccm-glue.o aes-ce-ccm-core.o obj-$(CONFIG_CRYPTO_AES_ARM64_CE_BLK) += aes-ce-blk.o -aes-ce-blk-y := aes-glue-ce.o aes-ce.o +aes-ce-blk-y := aes-glue-ce.o obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o -aes-neon-blk-y := aes-glue-neon.o aes-neon.o +aes-neon-blk-y := aes-glue-neon.o obj-$(CONFIG_CRYPTO_AES_ARM64_BS) += aes-neon-bs.o aes-neon-bs-y := aes-neonbs-core.o aes-neonbs-glue.o diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c index db371ac051fc..a304375ce724 100644 --- a/arch/arm64/crypto/aes-ce-ccm-glue.c +++ b/arch/arm64/crypto/aes-ce-ccm-glue.c @@ -31,10 +31,6 @@ static int num_rounds(struct crypto_aes_ctx *ctx) return 6 + ctx->key_length / 4; } -asmlinkage u32 ce_aes_mac_update(u8 const in[], u32 const rk[], int rounds, - int blocks, u8 dg[], int enc_before, - int enc_after); - asmlinkage void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes, u32 const rk[], u32 rounds, u8 mac[], u8 ctr[], u8 const final_iv[]); @@ -105,16 +101,11 @@ static u32 ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes, u32 blocks = abytes / AES_BLOCK_SIZE; if (macp == AES_BLOCK_SIZE || (!macp && blocks > 0)) { - u32 rem = ce_aes_mac_update(in, rk, rounds, blocks, mac, - macp, enc_after); - u32 adv = (blocks - rem) * AES_BLOCK_SIZE; - + ce_aes_mac_update(in, rk, rounds, blocks, mac, macp, + enc_after); macp = enc_after ? 0 : AES_BLOCK_SIZE; - in += adv; - abytes -= adv; - - if (unlikely(rem)) - macp = 0; + in += blocks * AES_BLOCK_SIZE; + abytes -= blocks * AES_BLOCK_SIZE; } else { u32 l = min(AES_BLOCK_SIZE - macp, abytes); diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index 92f43e1cd097..85497cfe76d8 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -7,7 +7,6 @@ #include <crypto/aes.h> #include <crypto/ctr.h> -#include <crypto/internal/hash.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> #include <crypto/sha2.h> @@ -37,7 +36,6 @@ #define aes_xctr_encrypt ce_aes_xctr_encrypt #define aes_xts_encrypt ce_aes_xts_encrypt #define aes_xts_decrypt ce_aes_xts_decrypt -#define aes_mac_update ce_aes_mac_update MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS/XCTR using ARMv8 Crypto Extensions"); #else #define MODE "neon" @@ -54,7 +52,6 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS/XCTR using ARMv8 Crypto Extensions"); #define aes_xctr_encrypt neon_aes_xctr_encrypt #define aes_xts_encrypt neon_aes_xts_encrypt #define aes_xts_decrypt neon_aes_xts_decrypt -#define aes_mac_update neon_aes_mac_update MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS/XCTR using ARMv8 NEON"); #endif #if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS) @@ -66,53 +63,11 @@ MODULE_ALIAS_CRYPTO("xctr(aes)"); #endif MODULE_ALIAS_CRYPTO("cts(cbc(aes))"); MODULE_ALIAS_CRYPTO("essiv(cbc(aes),sha256)"); -MODULE_ALIAS_CRYPTO("cmac(aes)"); -MODULE_ALIAS_CRYPTO("xcbc(aes)"); -MODULE_ALIAS_CRYPTO("cbcmac(aes)"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); MODULE_LICENSE("GPL v2"); -/* defined in aes-modes.S */ -asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int blocks); -asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int blocks); - -asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int blocks, u8 iv[]); -asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int blocks, u8 iv[]); - -asmlinkage void aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int bytes, u8 const iv[]); -asmlinkage void aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int bytes, u8 const iv[]); - -asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int bytes, u8 ctr[]); - -asmlinkage void aes_xctr_encrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int bytes, u8 ctr[], int byte_ctr); - -asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[], - int rounds, int bytes, u32 const rk2[], u8 iv[], - int first); -asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[], - int rounds, int bytes, u32 const rk2[], u8 iv[], - int first); - -asmlinkage void aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[], - int rounds, int blocks, u8 iv[], - u32 const rk2[]); -asmlinkage void aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[], - int rounds, int blocks, u8 iv[], - u32 const rk2[]); - -asmlinkage int aes_mac_update(u8 const in[], u32 const rk[], int rounds, - int blocks, u8 dg[], int enc_before, - int enc_after); - struct crypto_aes_xts_ctx { struct crypto_aes_ctx key1; struct crypto_aes_ctx __aligned(8) key2; @@ -123,15 +78,6 @@ struct crypto_aes_essiv_cbc_ctx { struct crypto_aes_ctx __aligned(8) key2; }; -struct mac_tfm_ctx { - struct crypto_aes_ctx key; - u8 __aligned(8) consts[]; -}; - -struct mac_desc_ctx { - u8 dg[AES_BLOCK_SIZE]; -}; - static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key, unsigned int key_len) { @@ -762,222 +708,19 @@ static struct skcipher_alg aes_algs[] = { { .decrypt = essiv_cbc_decrypt, } }; -static int cbcmac_setkey(struct crypto_shash *tfm, const u8 *in_key, - unsigned int key_len) -{ - struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); - - return aes_expandkey(&ctx->key, in_key, key_len); -} - -static void cmac_gf128_mul_by_x(be128 *y, const be128 *x) -{ - u64 a = be64_to_cpu(x->a); - u64 b = be64_to_cpu(x->b); - - y->a = cpu_to_be64((a << 1) | (b >> 63)); - y->b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0)); -} - -static int cmac_setkey(struct crypto_shash *tfm, const u8 *in_key, - unsigned int key_len) -{ - struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); - be128 *consts = (be128 *)ctx->consts; - int rounds = 6 + key_len / 4; - int err; - - err = cbcmac_setkey(tfm, in_key, key_len); - if (err) - return err; - - /* encrypt the zero vector */ - scoped_ksimd() - aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, - ctx->key.key_enc, rounds, 1); - - cmac_gf128_mul_by_x(consts, consts); - cmac_gf128_mul_by_x(consts + 1, consts); - - return 0; -} - -static int xcbc_setkey(struct crypto_shash *tfm, const u8 *in_key, - unsigned int key_len) -{ - static u8 const ks[3][AES_BLOCK_SIZE] = { - { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, - { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, - { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, - }; - - struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); - int rounds = 6 + key_len / 4; - u8 key[AES_BLOCK_SIZE]; - int err; - - err = cbcmac_setkey(tfm, in_key, key_len); - if (err) - return err; - - scoped_ksimd() { - aes_ecb_encrypt(key, ks[0], ctx->key.key_enc, rounds, 1); - aes_ecb_encrypt(ctx->consts, ks[1], ctx->key.key_enc, rounds, 2); - } - - return cbcmac_setkey(tfm, key, sizeof(key)); -} - -static int mac_init(struct shash_desc *desc) -{ - struct mac_desc_ctx *ctx = shash_desc_ctx(desc); - - memset(ctx->dg, 0, AES_BLOCK_SIZE); - return 0; -} - -static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks, - u8 dg[], int enc_before) -{ - int rounds = 6 + ctx->key_length / 4; - int rem; - - do { - scoped_ksimd() - rem = aes_mac_update(in, ctx->key_enc, rounds, blocks, - dg, enc_before, !enc_before); - in += (blocks - rem) * AES_BLOCK_SIZE; - blocks = rem; - } while (blocks); -} - -static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len) -{ - struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct mac_desc_ctx *ctx = shash_desc_ctx(desc); - int blocks = len / AES_BLOCK_SIZE; - - len %= AES_BLOCK_SIZE; - mac_do_update(&tctx->key, p, blocks, ctx->dg, 0); - return len; -} - -static int cbcmac_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct mac_desc_ctx *ctx = shash_desc_ctx(desc); - - if (len) { - crypto_xor(ctx->dg, src, len); - mac_do_update(&tctx->key, NULL, 0, ctx->dg, 1); - } - memcpy(out, ctx->dg, AES_BLOCK_SIZE); - return 0; -} - -static int cmac_finup(struct shash_desc *desc, const u8 *src, unsigned int len, - u8 *out) -{ - struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct mac_desc_ctx *ctx = shash_desc_ctx(desc); - u8 *consts = tctx->consts; - - crypto_xor(ctx->dg, src, len); - if (len != AES_BLOCK_SIZE) { - ctx->dg[len] ^= 0x80; - consts += AES_BLOCK_SIZE; - } - mac_do_update(&tctx->key, consts, 1, ctx->dg, 0); - memcpy(out, ctx->dg, AES_BLOCK_SIZE); - return 0; -} - -static struct shash_alg mac_algs[] = { { - .base.cra_name = "cmac(aes)", - .base.cra_driver_name = "cmac-aes-" MODE, - .base.cra_priority = PRIO, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINAL_NONZERO, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + - 2 * AES_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - - .digestsize = AES_BLOCK_SIZE, - .init = mac_init, - .update = mac_update, - .finup = cmac_finup, - .setkey = cmac_setkey, - .descsize = sizeof(struct mac_desc_ctx), -}, { - .base.cra_name = "xcbc(aes)", - .base.cra_driver_name = "xcbc-aes-" MODE, - .base.cra_priority = PRIO, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINAL_NONZERO, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + - 2 * AES_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - - .digestsize = AES_BLOCK_SIZE, - .init = mac_init, - .update = mac_update, - .finup = cmac_finup, - .setkey = xcbc_setkey, - .descsize = sizeof(struct mac_desc_ctx), -}, { - .base.cra_name = "cbcmac(aes)", - .base.cra_driver_name = "cbcmac-aes-" MODE, - .base.cra_priority = PRIO, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct mac_tfm_ctx), - .base.cra_module = THIS_MODULE, - - .digestsize = AES_BLOCK_SIZE, - .init = mac_init, - .update = mac_update, - .finup = cbcmac_finup, - .setkey = cbcmac_setkey, - .descsize = sizeof(struct mac_desc_ctx), -} }; - static void aes_exit(void) { - crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs)); crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } static int __init aes_init(void) { - int err; - - err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); - if (err) - return err; - - err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs)); - if (err) - goto unregister_ciphers; - - return 0; - -unregister_ciphers: - crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); - return err; + return crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } #ifdef USE_V8_CRYPTO_EXTENSIONS module_cpu_feature_match(AES, aes_init); -EXPORT_SYMBOL_NS(ce_aes_mac_update, "CRYPTO_INTERNAL"); #else module_init(aes_init); -EXPORT_SYMBOL(neon_aes_ecb_encrypt); -EXPORT_SYMBOL(neon_aes_cbc_encrypt); -EXPORT_SYMBOL(neon_aes_ctr_encrypt); -EXPORT_SYMBOL(neon_aes_xts_encrypt); -EXPORT_SYMBOL(neon_aes_xts_decrypt); #endif module_exit(aes_exit); diff --git a/arch/arm64/crypto/aes-neonbs-glue.c b/arch/arm64/crypto/aes-neonbs-glue.c index 00530b291010..5bcbac979893 100644 --- a/arch/arm64/crypto/aes-neonbs-glue.c +++ b/arch/arm64/crypto/aes-neonbs-glue.c @@ -17,6 +17,7 @@ MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); MODULE_DESCRIPTION("Bit sliced AES using NEON instructions"); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); MODULE_LICENSE("GPL v2"); MODULE_ALIAS_CRYPTO("ecb(aes)"); @@ -42,20 +43,6 @@ asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[], asmlinkage void aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, int blocks, u8 iv[]); -/* borrowed from aes-neon-blk.ko */ -asmlinkage void neon_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int blocks); -asmlinkage void neon_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int blocks, u8 iv[]); -asmlinkage void neon_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int bytes, u8 ctr[]); -asmlinkage void neon_aes_xts_encrypt(u8 out[], u8 const in[], - u32 const rk1[], int rounds, int bytes, - u32 const rk2[], u8 iv[], int first); -asmlinkage void neon_aes_xts_decrypt(u8 out[], u8 const in[], - u32 const rk1[], int rounds, int bytes, - u32 const rk2[], u8 iv[], int first); - struct aesbs_ctx { u8 rk[13 * (8 * AES_BLOCK_SIZE) + 32]; int rounds; diff --git a/arch/arm64/crypto/ghash-ce-core.S b/arch/arm64/crypto/ghash-ce-core.S index 23ee9a5eaf27..33772d8fe6b5 100644 --- a/arch/arm64/crypto/ghash-ce-core.S +++ b/arch/arm64/crypto/ghash-ce-core.S @@ -1,12 +1,11 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Accelerated GHASH implementation with ARMv8 PMULL instructions. + * Accelerated AES-GCM implementation with ARMv8 Crypto Extensions. * * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org> */ #include <linux/linkage.h> -#include <linux/cfi_types.h> #include <asm/assembler.h> SHASH .req v0 @@ -19,31 +18,6 @@ XH .req v7 IN1 .req v7 - k00_16 .req v8 - k32_48 .req v9 - - t3 .req v10 - t4 .req v11 - t5 .req v12 - t6 .req v13 - t7 .req v14 - t8 .req v15 - t9 .req v16 - - perm1 .req v17 - perm2 .req v18 - perm3 .req v19 - - sh1 .req v20 - sh2 .req v21 - sh3 .req v22 - sh4 .req v23 - - ss1 .req v24 - ss2 .req v25 - ss3 .req v26 - ss4 .req v27 - XL2 .req v8 XM2 .req v9 XH2 .req v10 @@ -60,90 +34,6 @@ .text .arch armv8-a+crypto - .macro __pmull_p64, rd, rn, rm - pmull \rd\().1q, \rn\().1d, \rm\().1d - .endm - - .macro __pmull2_p64, rd, rn, rm - pmull2 \rd\().1q, \rn\().2d, \rm\().2d - .endm - - .macro __pmull_p8, rq, ad, bd - ext t3.8b, \ad\().8b, \ad\().8b, #1 // A1 - ext t5.8b, \ad\().8b, \ad\().8b, #2 // A2 - ext t7.8b, \ad\().8b, \ad\().8b, #3 // A3 - - __pmull_p8_\bd \rq, \ad - .endm - - .macro __pmull2_p8, rq, ad, bd - tbl t3.16b, {\ad\().16b}, perm1.16b // A1 - tbl t5.16b, {\ad\().16b}, perm2.16b // A2 - tbl t7.16b, {\ad\().16b}, perm3.16b // A3 - - __pmull2_p8_\bd \rq, \ad - .endm - - .macro __pmull_p8_SHASH, rq, ad - __pmull_p8_tail \rq, \ad\().8b, SHASH.8b, 8b,, sh1, sh2, sh3, sh4 - .endm - - .macro __pmull_p8_SHASH2, rq, ad - __pmull_p8_tail \rq, \ad\().8b, SHASH2.8b, 8b,, ss1, ss2, ss3, ss4 - .endm - - .macro __pmull2_p8_SHASH, rq, ad - __pmull_p8_tail \rq, \ad\().16b, SHASH.16b, 16b, 2, sh1, sh2, sh3, sh4 - .endm - - .macro __pmull_p8_tail, rq, ad, bd, nb, t, b1, b2, b3, b4 - pmull\t t3.8h, t3.\nb, \bd // F = A1*B - pmull\t t4.8h, \ad, \b1\().\nb // E = A*B1 - pmull\t t5.8h, t5.\nb, \bd // H = A2*B - pmull\t t6.8h, \ad, \b2\().\nb // G = A*B2 - pmull\t t7.8h, t7.\nb, \bd // J = A3*B - pmull\t t8.8h, \ad, \b3\().\nb // I = A*B3 - pmull\t t9.8h, \ad, \b4\().\nb // K = A*B4 - pmull\t \rq\().8h, \ad, \bd // D = A*B - - eor t3.16b, t3.16b, t4.16b // L = E + F - eor t5.16b, t5.16b, t6.16b // M = G + H - eor t7.16b, t7.16b, t8.16b // N = I + J - - uzp1 t4.2d, t3.2d, t5.2d - uzp2 t3.2d, t3.2d, t5.2d - uzp1 t6.2d, t7.2d, t9.2d - uzp2 t7.2d, t7.2d, t9.2d - - // t3 = (L) (P0 + P1) << 8 - // t5 = (M) (P2 + P3) << 16 - eor t4.16b, t4.16b, t3.16b - and t3.16b, t3.16b, k32_48.16b - - // t7 = (N) (P4 + P5) << 24 - // t9 = (K) (P6 + P7) << 32 - eor t6.16b, t6.16b, t7.16b - and t7.16b, t7.16b, k00_16.16b - - eor t4.16b, t4.16b, t3.16b - eor t6.16b, t6.16b, t7.16b - - zip2 t5.2d, t4.2d, t3.2d - zip1 t3.2d, t4.2d, t3.2d - zip2 t9.2d, t6.2d, t7.2d - zip1 t7.2d, t6.2d, t7.2d - - ext t3.16b, t3.16b, t3.16b, #15 - ext t5.16b, t5.16b, t5.16b, #14 - ext t7.16b, t7.16b, t7.16b, #13 - ext t9.16b, t9.16b, t9.16b, #12 - - eor t3.16b, t3.16b, t5.16b - eor t7.16b, t7.16b, t9.16b - eor \rq\().16b, \rq\().16b, t3.16b - eor \rq\().16b, \rq\().16b, t7.16b - .endm - .macro __pmull_pre_p64 add x8, x3, #16 ld1 {HH.2d-HH4.2d}, [x8] @@ -160,43 +50,6 @@ shl MASK.2d, MASK.2d, #57 .endm - .macro __pmull_pre_p8 - ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 - eor SHASH2.16b, SHASH2.16b, SHASH.16b - - // k00_16 := 0x0000000000000000_000000000000ffff - // k32_48 := 0x00000000ffffffff_0000ffffffffffff - movi k32_48.2d, #0xffffffff - mov k32_48.h[2], k32_48.h[0] - ushr k00_16.2d, k32_48.2d, #32 - - // prepare the permutation vectors - mov_q x5, 0x080f0e0d0c0b0a09 - movi T1.8b, #8 - dup perm1.2d, x5 - eor perm1.16b, perm1.16b, T1.16b - ushr perm2.2d, perm1.2d, #8 - ushr perm3.2d, perm1.2d, #16 - ushr T1.2d, perm1.2d, #24 - sli perm2.2d, perm1.2d, #56 - sli perm3.2d, perm1.2d, #48 - sli T1.2d, perm1.2d, #40 - - // precompute loop invariants - tbl sh1.16b, {SHASH.16b}, perm1.16b - tbl sh2.16b, {SHASH.16b}, perm2.16b - tbl sh3.16b, {SHASH.16b}, perm3.16b - tbl sh4.16b, {SHASH.16b}, T1.16b - ext ss1.8b, SHASH2.8b, SHASH2.8b, #1 - ext ss2.8b, SHASH2.8b, SHASH2.8b, #2 - ext ss3.8b, SHASH2.8b, SHASH2.8b, #3 - ext ss4.8b, SHASH2.8b, SHASH2.8b, #4 - .endm - - // - // PMULL (64x64->128) based reduction for CPUs that can do - // it in a single instruction. - // .macro __pmull_reduce_p64 pmull T2.1q, XL.1d, MASK.1d eor XM.16b, XM.16b, T1.16b @@ -209,39 +62,15 @@ pmull XL.1q, XL.1d, MASK.1d .endm - // - // Alternative reduction for CPUs that lack support for the - // 64x64->128 PMULL instruction - // - .macro __pmull_reduce_p8 - eor XM.16b, XM.16b, T1.16b - - mov XL.d[1], XM.d[0] - mov XH.d[0], XM.d[1] - - shl T1.2d, XL.2d, #57 - shl T2.2d, XL.2d, #62 - eor T2.16b, T2.16b, T1.16b - shl T1.2d, XL.2d, #63 - eor T2.16b, T2.16b, T1.16b - ext T1.16b, XL.16b, XH.16b, #8 - eor T2.16b, T2.16b, T1.16b - - mov XL.d[1], T2.d[0] - mov XH.d[0], T2.d[1] - - ushr T2.2d, XL.2d, #1 - eor XH.16b, XH.16b, XL.16b - eor XL.16b, XL.16b, T2.16b - ushr T2.2d, T2.2d, #6 - ushr XL.2d, XL.2d, #1 - .endm - - .macro __pmull_ghash, pn + /* + * void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, + * u64 const h[4][2], const char *head) + */ +SYM_FUNC_START(pmull_ghash_update_p64) ld1 {SHASH.2d}, [x3] ld1 {XL.2d}, [x1] - __pmull_pre_\pn + __pmull_pre_p64 /* do the head block first, if supplied */ cbz x4, 0f @@ -249,7 +78,7 @@ mov x4, xzr b 3f -0: .ifc \pn, p64 +0: tbnz w0, #0, 2f // skip until #blocks is a tbnz w0, #1, 2f // round multiple of 4 @@ -314,7 +143,6 @@ cbz w0, 5f b 1b - .endif 2: ld1 {T1.2d}, [x2], #16 sub w0, w0, #1 @@ -327,16 +155,16 @@ CPU_LE( rev64 T1.16b, T1.16b ) eor T1.16b, T1.16b, T2.16b eor XL.16b, XL.16b, IN1.16b - __pmull2_\pn XH, XL, SHASH // a1 * b1 + pmull2 XH.1q, XL.2d, SHASH.2d // a1 * b1 eor T1.16b, T1.16b, XL.16b - __pmull_\pn XL, XL, SHASH // a0 * b0 - __pmull_\pn XM, T1, SHASH2 // (a1 + a0)(b1 + b0) + pmull XL.1q, XL.1d, SHASH.1d // a0 * b0 + pmull XM.1q, T1.1d, SHASH2.1d // (a1 + a0)(b1 + b0) 4: eor T2.16b, XL.16b, XH.16b ext T1.16b, XL.16b, XH.16b, #8 eor XM.16b, XM.16b, T2.16b - __pmull_reduce_\pn + __pmull_reduce_p64 eor T2.16b, T2.16b, XH.16b eor XL.16b, XL.16b, T2.16b @@ -345,20 +173,8 @@ CPU_LE( rev64 T1.16b, T1.16b ) 5: st1 {XL.2d}, [x1] ret - .endm - - /* - * void pmull_ghash_update(int blocks, u64 dg[], const char *src, - * struct ghash_key const *k, const char *head) - */ -SYM_TYPED_FUNC_START(pmull_ghash_update_p64) - __pmull_ghash p64 SYM_FUNC_END(pmull_ghash_update_p64) -SYM_TYPED_FUNC_START(pmull_ghash_update_p8) - __pmull_ghash p8 -SYM_FUNC_END(pmull_ghash_update_p8) - KS0 .req v8 KS1 .req v9 KS2 .req v10 @@ -597,18 +413,19 @@ CPU_LE( rev w8, w8 ) .endm /* - * void pmull_gcm_encrypt(int blocks, u8 dst[], const u8 src[], - * struct ghash_key const *k, u64 dg[], u8 ctr[], - * int rounds, u8 tag) + * void pmull_gcm_encrypt(int bytes, u8 dst[], const u8 src[], + * u64 const h[4][2], u64 dg[], u8 ctr[], + * u32 const rk[], int rounds, u8 tag[]) */ SYM_FUNC_START(pmull_gcm_encrypt) pmull_gcm_do_crypt 1 SYM_FUNC_END(pmull_gcm_encrypt) /* - * void pmull_gcm_decrypt(int blocks, u8 dst[], const u8 src[], - * struct ghash_key const *k, u64 dg[], u8 ctr[], - * int rounds, u8 tag) + * int pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[], + * u64 const h[4][2], u64 dg[], u8 ctr[], + * u32 const rk[], int rounds, const u8 l[], + * const u8 tag[], u64 authsize) */ SYM_FUNC_START(pmull_gcm_decrypt) pmull_gcm_do_crypt 0 diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c index 63bb9e062251..eaf2932ceaf5 100644 --- a/arch/arm64/crypto/ghash-ce-glue.c +++ b/arch/arm64/crypto/ghash-ce-glue.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Accelerated GHASH implementation with ARMv8 PMULL instructions. + * AES-GCM using ARMv8 Crypto Extensions * * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org> */ @@ -11,7 +11,6 @@ #include <crypto/ghash.h> #include <crypto/gf128mul.h> #include <crypto/internal/aead.h> -#include <crypto/internal/hash.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> #include <linux/cpufeature.h> @@ -23,121 +22,41 @@ #include <asm/simd.h> -MODULE_DESCRIPTION("GHASH and AES-GCM using ARMv8 Crypto Extensions"); +MODULE_DESCRIPTION("AES-GCM using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("ghash"); +MODULE_ALIAS_CRYPTO("gcm(aes)"); +MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))"); #define RFC4106_NONCE_SIZE 4 -struct ghash_key { +struct arm_ghash_key { be128 k; - u64 h[][2]; -}; - -struct arm_ghash_desc_ctx { - u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)]; + u64 h[4][2]; }; struct gcm_aes_ctx { struct aes_enckey aes_key; u8 nonce[RFC4106_NONCE_SIZE]; - struct ghash_key ghash_key; + struct arm_ghash_key ghash_key; }; asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, - u64 const h[][2], const char *head); - -asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, - u64 const h[][2], const char *head); + u64 const h[4][2], const char *head); asmlinkage void pmull_gcm_encrypt(int bytes, u8 dst[], const u8 src[], - u64 const h[][2], u64 dg[], u8 ctr[], + u64 const h[4][2], u64 dg[], u8 ctr[], u32 const rk[], int rounds, u8 tag[]); asmlinkage int pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[], - u64 const h[][2], u64 dg[], u8 ctr[], + u64 const h[4][2], u64 dg[], u8 ctr[], u32 const rk[], int rounds, const u8 l[], const u8 tag[], u64 authsize); -static int ghash_init(struct shash_desc *desc) -{ - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - - *ctx = (struct arm_ghash_desc_ctx){}; - return 0; -} - -static __always_inline -void ghash_do_simd_update(int blocks, u64 dg[], const char *src, - struct ghash_key *key, const char *head, - void (*simd_update)(int blocks, u64 dg[], - const char *src, - u64 const h[][2], - const char *head)) +static void ghash_do_simd_update(int blocks, u64 dg[], const char *src, + struct arm_ghash_key *key, const char *head) { scoped_ksimd() - simd_update(blocks, dg, src, key->h, head); -} - -/* avoid hogging the CPU for too long */ -#define MAX_BLOCKS (SZ_64K / GHASH_BLOCK_SIZE) - -static int ghash_update(struct shash_desc *desc, const u8 *src, - unsigned int len) -{ - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - struct ghash_key *key = crypto_shash_ctx(desc->tfm); - int blocks; - - blocks = len / GHASH_BLOCK_SIZE; - len -= blocks * GHASH_BLOCK_SIZE; - - do { - int chunk = min(blocks, MAX_BLOCKS); - - ghash_do_simd_update(chunk, ctx->digest, src, key, NULL, - pmull_ghash_update_p8); - blocks -= chunk; - src += chunk * GHASH_BLOCK_SIZE; - } while (unlikely(blocks > 0)); - return len; -} - -static int ghash_export(struct shash_desc *desc, void *out) -{ - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - u8 *dst = out; - - put_unaligned_be64(ctx->digest[1], dst); - put_unaligned_be64(ctx->digest[0], dst + 8); - return 0; -} - -static int ghash_import(struct shash_desc *desc, const void *in) -{ - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - const u8 *src = in; - - ctx->digest[1] = get_unaligned_be64(src); - ctx->digest[0] = get_unaligned_be64(src + 8); - return 0; -} - -static int ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *dst) -{ - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - struct ghash_key *key = crypto_shash_ctx(desc->tfm); - - if (len) { - u8 buf[GHASH_BLOCK_SIZE] = {}; - - memcpy(buf, src, len); - ghash_do_simd_update(1, ctx->digest, buf, key, NULL, - pmull_ghash_update_p8); - memzero_explicit(buf, sizeof(buf)); - } - return ghash_export(desc, dst); + pmull_ghash_update_p64(blocks, dg, src, key->h, head); } static void ghash_reflect(u64 h[], const be128 *k) @@ -151,41 +70,6 @@ static void ghash_reflect(u64 h[], const be128 *k) h[1] ^= 0xc200000000000000UL; } -static int ghash_setkey(struct crypto_shash *tfm, - const u8 *inkey, unsigned int keylen) -{ - struct ghash_key *key = crypto_shash_ctx(tfm); - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - /* needed for the fallback */ - memcpy(&key->k, inkey, GHASH_BLOCK_SIZE); - - ghash_reflect(key->h[0], &key->k); - return 0; -} - -static struct shash_alg ghash_alg = { - .base.cra_name = "ghash", - .base.cra_driver_name = "ghash-neon", - .base.cra_priority = 150, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .base.cra_blocksize = GHASH_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ghash_key) + sizeof(u64[2]), - .base.cra_module = THIS_MODULE, - - .digestsize = GHASH_DIGEST_SIZE, - .init = ghash_init, - .update = ghash_update, - .finup = ghash_finup, - .setkey = ghash_setkey, - .export = ghash_export, - .import = ghash_import, - .descsize = sizeof(struct arm_ghash_desc_ctx), - .statesize = sizeof(struct ghash_desc_ctx), -}; - static int gcm_aes_setkey(struct crypto_aead *tfm, const u8 *inkey, unsigned int keylen) { @@ -240,9 +124,7 @@ static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[], int blocks = count / GHASH_BLOCK_SIZE; ghash_do_simd_update(blocks, dg, src, &ctx->ghash_key, - *buf_count ? buf : NULL, - pmull_ghash_update_p64); - + *buf_count ? buf : NULL); src += blocks * GHASH_BLOCK_SIZE; count %= GHASH_BLOCK_SIZE; *buf_count = 0; @@ -275,8 +157,7 @@ static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[], u32 len) if (buf_count) { memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count); - ghash_do_simd_update(1, dg, buf, &ctx->ghash_key, NULL, - pmull_ghash_update_p64); + ghash_do_simd_update(1, dg, buf, &ctx->ghash_key, NULL); } } @@ -482,8 +363,7 @@ static struct aead_alg gcm_aes_algs[] = {{ .base.cra_driver_name = "gcm-aes-ce", .base.cra_priority = 300, .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct gcm_aes_ctx) + - 4 * sizeof(u64[2]), + .base.cra_ctxsize = sizeof(struct gcm_aes_ctx), .base.cra_module = THIS_MODULE, }, { .ivsize = GCM_RFC4106_IV_SIZE, @@ -498,29 +378,21 @@ static struct aead_alg gcm_aes_algs[] = {{ .base.cra_driver_name = "rfc4106-gcm-aes-ce", .base.cra_priority = 300, .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct gcm_aes_ctx) + - 4 * sizeof(u64[2]), + .base.cra_ctxsize = sizeof(struct gcm_aes_ctx), .base.cra_module = THIS_MODULE, }}; static int __init ghash_ce_mod_init(void) { - if (!cpu_have_named_feature(ASIMD)) + if (!cpu_have_named_feature(ASIMD) || !cpu_have_named_feature(PMULL)) return -ENODEV; - if (cpu_have_named_feature(PMULL)) - return crypto_register_aeads(gcm_aes_algs, - ARRAY_SIZE(gcm_aes_algs)); - - return crypto_register_shash(&ghash_alg); + return crypto_register_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs)); } static void __exit ghash_ce_mod_exit(void) { - if (cpu_have_named_feature(PMULL)) - crypto_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs)); - else - crypto_unregister_shash(&ghash_alg); + crypto_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs)); } static const struct cpu_feature __maybe_unused ghash_cpu_feature[] = { diff --git a/arch/arm64/crypto/sm3-ce-glue.c b/arch/arm64/crypto/sm3-ce-glue.c deleted file mode 100644 index 24c1fcfae072..000000000000 --- a/arch/arm64/crypto/sm3-ce-glue.c +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * sm3-ce-glue.c - SM3 secure hash using ARMv8.2 Crypto Extensions - * - * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org> - */ - -#include <crypto/internal/hash.h> -#include <crypto/sm3.h> -#include <crypto/sm3_base.h> -#include <linux/cpufeature.h> -#include <linux/kernel.h> -#include <linux/module.h> - -#include <asm/simd.h> - -MODULE_DESCRIPTION("SM3 secure hash using ARMv8 Crypto Extensions"); -MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); -MODULE_LICENSE("GPL v2"); - -asmlinkage void sm3_ce_transform(struct sm3_state *sst, u8 const *src, - int blocks); - -static int sm3_ce_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - int remain; - - scoped_ksimd() { - remain = sm3_base_do_update_blocks(desc, data, len, sm3_ce_transform); - } - return remain; -} - -static int sm3_ce_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - scoped_ksimd() { - sm3_base_do_finup(desc, data, len, sm3_ce_transform); - } - return sm3_base_finish(desc, out); -} - -static struct shash_alg sm3_alg = { - .digestsize = SM3_DIGEST_SIZE, - .init = sm3_base_init, - .update = sm3_ce_update, - .finup = sm3_ce_finup, - .descsize = SM3_STATE_SIZE, - .base.cra_name = "sm3", - .base.cra_driver_name = "sm3-ce", - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize = SM3_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - .base.cra_priority = 400, -}; - -static int __init sm3_ce_mod_init(void) -{ - return crypto_register_shash(&sm3_alg); -} - -static void __exit sm3_ce_mod_fini(void) -{ - crypto_unregister_shash(&sm3_alg); -} - -module_cpu_feature_match(SM3, sm3_ce_mod_init); -module_exit(sm3_ce_mod_fini); diff --git a/arch/arm64/crypto/sm3-neon-glue.c b/arch/arm64/crypto/sm3-neon-glue.c deleted file mode 100644 index 15f30cc24f32..000000000000 --- a/arch/arm64/crypto/sm3-neon-glue.c +++ /dev/null @@ -1,67 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * sm3-neon-glue.c - SM3 secure hash using NEON instructions - * - * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> - */ - -#include <asm/simd.h> -#include <crypto/internal/hash.h> -#include <crypto/sm3.h> -#include <crypto/sm3_base.h> -#include <linux/cpufeature.h> -#include <linux/kernel.h> -#include <linux/module.h> - - -asmlinkage void sm3_neon_transform(struct sm3_state *sst, u8 const *src, - int blocks); - -static int sm3_neon_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - scoped_ksimd() - return sm3_base_do_update_blocks(desc, data, len, - sm3_neon_transform); -} - -static int sm3_neon_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - scoped_ksimd() - sm3_base_do_finup(desc, data, len, sm3_neon_transform); - return sm3_base_finish(desc, out); -} - -static struct shash_alg sm3_alg = { - .digestsize = SM3_DIGEST_SIZE, - .init = sm3_base_init, - .update = sm3_neon_update, - .finup = sm3_neon_finup, - .descsize = SM3_STATE_SIZE, - .base.cra_name = "sm3", - .base.cra_driver_name = "sm3-neon", - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize = SM3_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - .base.cra_priority = 200, -}; - -static int __init sm3_neon_init(void) -{ - return crypto_register_shash(&sm3_alg); -} - -static void __exit sm3_neon_fini(void) -{ - crypto_unregister_shash(&sm3_alg); -} - -module_init(sm3_neon_init); -module_exit(sm3_neon_fini); - -MODULE_DESCRIPTION("SM3 secure hash using NEON instructions"); -MODULE_AUTHOR("Jussi Kivilinna <jussi.kivilinna@iki.fi>"); -MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>"); -MODULE_LICENSE("GPL v2"); diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index d3d46e5f7188..9d7c9ae5ac96 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -748,28 +748,6 @@ alternative_else_nop_endif set_sctlr sctlr_el2, \reg .endm - /* - * Check whether asm code should yield as soon as it is able. This is - * the case if we are currently running in task context, and the - * TIF_NEED_RESCHED flag is set. (Note that the TIF_NEED_RESCHED flag - * is stored negated in the top word of the thread_info::preempt_count - * field) - */ - .macro cond_yield, lbl:req, tmp:req, tmp2 -#ifdef CONFIG_PREEMPT_VOLUNTARY - get_current_task \tmp - ldr \tmp, [\tmp, #TSK_TI_PREEMPT] - /* - * If we are serving a softirq, there is no point in yielding: the - * softirq will not be preempted no matter what we do, so we should - * run to completion as quickly as we can. The preempt_count field will - * have BIT(SOFTIRQ_SHIFT) set in this case, so the zero check will - * catch this case too. - */ - cbz \tmp, \lbl -#endif - .endm - /* * Branch Target Identifier (BTI) */ diff --git a/arch/loongarch/configs/loongson32_defconfig b/arch/loongarch/configs/loongson32_defconfig index 276b1577e0be..7abbb21f4f8f 100644 --- a/arch/loongarch/configs/loongson32_defconfig +++ b/arch/loongarch/configs/loongson32_defconfig @@ -1080,7 +1080,7 @@ CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m CONFIG_CRYPTO_TWOFISH=m CONFIG_CRYPTO_CHACHA20POLY1305=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_DEFLATE=m CONFIG_CRYPTO_LZO=m diff --git a/arch/loongarch/configs/loongson64_defconfig b/arch/loongarch/configs/loongson64_defconfig index a14db1a95e7e..51ccd18ecdae 100644 --- a/arch/loongarch/configs/loongson64_defconfig +++ b/arch/loongarch/configs/loongson64_defconfig @@ -1113,7 +1113,7 @@ CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m CONFIG_CRYPTO_TWOFISH=m CONFIG_CRYPTO_CHACHA20POLY1305=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_DEFLATE=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index 31d16cba9879..03a8c192a7a3 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -581,7 +581,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index c0c419ec9a9e..0aee1939ac7a 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -538,7 +538,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index 2b7547ecc4c4..756256770afc 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -558,7 +558,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index 0b63787cff0d..8cfb75bb0add 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -530,7 +530,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index 308836b60bba..b2f5c9749e9b 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -540,7 +540,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index 97e108c0d24f..c4fddaaa6a86 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -557,7 +557,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index 7e9f83af9af4..926f12bc3d1d 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -644,7 +644,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index 2fe33271d249..e507012dbbc1 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -530,7 +530,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index 4308daaa7f74..6195cedd914b 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -531,7 +531,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index 36eb29ec54ee..9087bd9e3c35 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -547,7 +547,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index 524a89fa6953..25115bda7c8a 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -528,7 +528,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index f4fbc65c52d9..15a086634ba5 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -528,7 +528,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig index 2d056f1fc90f..b247f7ed973e 100644 --- a/arch/powerpc/crypto/Kconfig +++ b/arch/powerpc/crypto/Kconfig @@ -54,12 +54,11 @@ config CRYPTO_DEV_VMX_ENCRYPT select CRYPTO_AES select CRYPTO_CBC select CRYPTO_CTR - select CRYPTO_GHASH select CRYPTO_XTS default m help Support for VMX cryptographic acceleration instructions on Power8 CPU. - This module supports acceleration for AES and GHASH in hardware. If you - choose 'M' here, this module will be called vmx-crypto. + This module supports acceleration for AES in hardware. If you choose + 'M' here, this module will be called vmx-crypto. endmenu diff --git a/arch/powerpc/crypto/Makefile b/arch/powerpc/crypto/Makefile index 3ac0886282a2..a1fe102a90ae 100644 --- a/arch/powerpc/crypto/Makefile +++ b/arch/powerpc/crypto/Makefile @@ -11,7 +11,7 @@ obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o aes-ppc-spe-y := aes-spe-glue.o aes-gcm-p10-crypto-y := aes-gcm-p10-glue.o aes-gcm-p10.o ghashp10-ppc.o aesp10-ppc.o -vmx-crypto-objs := vmx.o ghashp8-ppc.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o +vmx-crypto-objs := vmx.o aes_cbc.o aes_ctr.o aes_xts.o ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y) override flavour := linux-ppc64le @@ -26,14 +26,10 @@ endif quiet_cmd_perl = PERL $@ cmd_perl = $(PERL) $< $(flavour) > $@ -targets += aesp10-ppc.S ghashp10-ppc.S ghashp8-ppc.S +targets += aesp10-ppc.S ghashp10-ppc.S $(obj)/aesp10-ppc.S $(obj)/ghashp10-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE $(call if_changed,perl) -$(obj)/ghashp8-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE - $(call if_changed,perl) - OBJECT_FILES_NON_STANDARD_aesp10-ppc.o := y OBJECT_FILES_NON_STANDARD_ghashp10-ppc.o := y -OBJECT_FILES_NON_STANDARD_ghashp8-ppc.o := y diff --git a/arch/powerpc/crypto/aesp8-ppc.h b/arch/powerpc/crypto/aesp8-ppc.h index 6862c605cc33..c68f5b6965fa 100644 --- a/arch/powerpc/crypto/aesp8-ppc.h +++ b/arch/powerpc/crypto/aesp8-ppc.h @@ -2,7 +2,6 @@ #include <linux/types.h> #include <crypto/aes.h> -extern struct shash_alg p8_ghash_alg; extern struct skcipher_alg p8_aes_cbc_alg; extern struct skcipher_alg p8_aes_ctr_alg; extern struct skcipher_alg p8_aes_xts_alg; diff --git a/arch/powerpc/crypto/ghash.c b/arch/powerpc/crypto/ghash.c deleted file mode 100644 index 7308735bdb33..000000000000 --- a/arch/powerpc/crypto/ghash.c +++ /dev/null @@ -1,160 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * GHASH routines supporting VMX instructions on the Power 8 - * - * Copyright (C) 2015, 2019 International Business Machines Inc. - * - * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com> - * - * Extended by Daniel Axtens <dja@axtens.net> to replace the fallback - * mechanism. The new approach is based on arm64 code, which is: - * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org> - */ - -#include "aesp8-ppc.h" -#include <asm/switch_to.h> -#include <crypto/aes.h> -#include <crypto/gf128mul.h> -#include <crypto/ghash.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> -#include <linux/err.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/string.h> -#include <linux/uaccess.h> - -void gcm_init_p8(u128 htable[16], const u64 Xi[2]); -void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]); -void gcm_ghash_p8(u64 Xi[2], const u128 htable[16], - const u8 *in, size_t len); - -struct p8_ghash_ctx { - /* key used by vector asm */ - u128 htable[16]; - /* key used by software fallback */ - be128 key; -}; - -struct p8_ghash_desc_ctx { - u64 shash[2]; -}; - -static int p8_ghash_init(struct shash_desc *desc) -{ - struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - memset(dctx->shash, 0, GHASH_DIGEST_SIZE); - return 0; -} - -static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen) -{ - struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(tfm)); - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - preempt_disable(); - pagefault_disable(); - enable_kernel_vsx(); - gcm_init_p8(ctx->htable, (const u64 *) key); - disable_kernel_vsx(); - pagefault_enable(); - preempt_enable(); - - memcpy(&ctx->key, key, GHASH_BLOCK_SIZE); - - return 0; -} - -static inline void __ghash_block(struct p8_ghash_ctx *ctx, - struct p8_ghash_desc_ctx *dctx, - const u8 *src) -{ - if (crypto_simd_usable()) { - preempt_disable(); - pagefault_disable(); - enable_kernel_vsx(); - gcm_ghash_p8(dctx->shash, ctx->htable, src, GHASH_BLOCK_SIZE); - disable_kernel_vsx(); - pagefault_enable(); - preempt_enable(); - } else { - crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE); - gf128mul_lle((be128 *)dctx->shash, &ctx->key); - } -} - -static inline int __ghash_blocks(struct p8_ghash_ctx *ctx, - struct p8_ghash_desc_ctx *dctx, - const u8 *src, unsigned int srclen) -{ - int remain = srclen - round_down(srclen, GHASH_BLOCK_SIZE); - - srclen -= remain; - if (crypto_simd_usable()) { - preempt_disable(); - pagefault_disable(); - enable_kernel_vsx(); - gcm_ghash_p8(dctx->shash, ctx->htable, - src, srclen); - disable_kernel_vsx(); - pagefault_enable(); - preempt_enable(); - } else { - do { - crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE); - gf128mul_lle((be128 *)dctx->shash, &ctx->key); - srclen -= GHASH_BLOCK_SIZE; - src += GHASH_BLOCK_SIZE; - } while (srclen); - } - - return remain; -} - -static int p8_ghash_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); - struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - return __ghash_blocks(ctx, dctx, src, srclen); -} - -static int p8_ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); - struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - if (len) { - u8 buf[GHASH_BLOCK_SIZE] = {}; - - memcpy(buf, src, len); - __ghash_block(ctx, dctx, buf); - memzero_explicit(buf, sizeof(buf)); - } - memcpy(out, dctx->shash, GHASH_DIGEST_SIZE); - return 0; -} - -struct shash_alg p8_ghash_alg = { - .digestsize = GHASH_DIGEST_SIZE, - .init = p8_ghash_init, - .update = p8_ghash_update, - .finup = p8_ghash_finup, - .setkey = p8_ghash_setkey, - .descsize = sizeof(struct p8_ghash_desc_ctx), - .base = { - .cra_name = "ghash", - .cra_driver_name = "p8_ghash", - .cra_priority = 1000, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct p8_ghash_ctx), - .cra_module = THIS_MODULE, - }, -}; diff --git a/arch/powerpc/crypto/vmx.c b/arch/powerpc/crypto/vmx.c index 7d2beb774f99..08da5311dfdf 100644 --- a/arch/powerpc/crypto/vmx.c +++ b/arch/powerpc/crypto/vmx.c @@ -14,7 +14,6 @@ #include <linux/cpufeature.h> #include <linux/crypto.h> #include <asm/cputable.h> -#include <crypto/internal/hash.h> #include <crypto/internal/skcipher.h> #include "aesp8-ppc.h" @@ -23,13 +22,9 @@ static int __init p8_init(void) { int ret; - ret = crypto_register_shash(&p8_ghash_alg); - if (ret) - goto err; - ret = crypto_register_skcipher(&p8_aes_cbc_alg); if (ret) - goto err_unregister_ghash; + goto err; ret = crypto_register_skcipher(&p8_aes_ctr_alg); if (ret) @@ -45,8 +40,6 @@ err_unregister_aes_ctr: crypto_unregister_skcipher(&p8_aes_ctr_alg); err_unregister_aes_cbc: crypto_unregister_skcipher(&p8_aes_cbc_alg); -err_unregister_ghash: - crypto_unregister_shash(&p8_ghash_alg); err: return ret; } @@ -56,7 +49,6 @@ static void __exit p8_exit(void) crypto_unregister_skcipher(&p8_aes_xts_alg); crypto_unregister_skcipher(&p8_aes_ctr_alg); crypto_unregister_skcipher(&p8_aes_cbc_alg); - crypto_unregister_shash(&p8_ghash_alg); } module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, p8_init); diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index 22d4eaab15f3..6905232ddb03 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -17,30 +17,6 @@ config CRYPTO_AES_RISCV64 - Zvkb vector crypto extension (CTR) - Zvkg vector crypto extension (XTS) -config CRYPTO_GHASH_RISCV64 - tristate "Hash functions: GHASH" - depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ - RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS - select CRYPTO_GCM - help - GCM GHASH function (NIST SP 800-38D) - - Architecture: riscv64 using: - - Zvkg vector crypto extension - -config CRYPTO_SM3_RISCV64 - tristate "Hash functions: SM3 (ShangMi 3)" - depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ - RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS - select CRYPTO_HASH - select CRYPTO_LIB_SM3 - help - SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012) - - Architecture: riscv64 using: - - Zvksh vector crypto extension - - Zvkb vector crypto extension - config CRYPTO_SM4_RISCV64 tristate "Ciphers: SM4 (ShangMi 4)" depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile index 183495a95cc0..8cf31db57fc4 100644 --- a/arch/riscv/crypto/Makefile +++ b/arch/riscv/crypto/Makefile @@ -4,11 +4,5 @@ obj-$(CONFIG_CRYPTO_AES_RISCV64) += aes-riscv64.o aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \ aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o -obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o -ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o - -obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o -sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o - obj-$(CONFIG_CRYPTO_SM4_RISCV64) += sm4-riscv64.o sm4-riscv64-y := sm4-riscv64-glue.o sm4-riscv64-zvksed-zvkb.o diff --git a/arch/riscv/crypto/ghash-riscv64-glue.c b/arch/riscv/crypto/ghash-riscv64-glue.c deleted file mode 100644 index d86073d25387..000000000000 --- a/arch/riscv/crypto/ghash-riscv64-glue.c +++ /dev/null @@ -1,146 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * GHASH using the RISC-V vector crypto extensions - * - * Copyright (C) 2023 VRULL GmbH - * Author: Heiko Stuebner <heiko.stuebner@vrull.eu> - * - * Copyright (C) 2023 SiFive, Inc. - * Author: Jerry Shih <jerry.shih@sifive.com> - */ - -#include <asm/simd.h> -#include <asm/vector.h> -#include <crypto/b128ops.h> -#include <crypto/gf128mul.h> -#include <crypto/ghash.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> -#include <crypto/utils.h> -#include <linux/errno.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/string.h> - -asmlinkage void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data, - size_t len); - -struct riscv64_ghash_tfm_ctx { - be128 key; -}; - -struct riscv64_ghash_desc_ctx { - be128 accumulator; -}; - -static int riscv64_ghash_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen) -{ - struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(tfm); - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - memcpy(&tctx->key, key, GHASH_BLOCK_SIZE); - - return 0; -} - -static int riscv64_ghash_init(struct shash_desc *desc) -{ - struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - *dctx = (struct riscv64_ghash_desc_ctx){}; - - return 0; -} - -static inline void -riscv64_ghash_blocks(const struct riscv64_ghash_tfm_ctx *tctx, - struct riscv64_ghash_desc_ctx *dctx, - const u8 *src, size_t srclen) -{ - /* The srclen is nonzero and a multiple of 16. */ - if (crypto_simd_usable()) { - kernel_vector_begin(); - ghash_zvkg(&dctx->accumulator, &tctx->key, src, srclen); - kernel_vector_end(); - } else { - do { - crypto_xor((u8 *)&dctx->accumulator, src, - GHASH_BLOCK_SIZE); - gf128mul_lle(&dctx->accumulator, &tctx->key); - src += GHASH_BLOCK_SIZE; - srclen -= GHASH_BLOCK_SIZE; - } while (srclen); - } -} - -static int riscv64_ghash_update(struct shash_desc *desc, const u8 *src, - unsigned int srclen) -{ - const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - riscv64_ghash_blocks(tctx, dctx, src, - round_down(srclen, GHASH_BLOCK_SIZE)); - return srclen - round_down(srclen, GHASH_BLOCK_SIZE); -} - -static int riscv64_ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - if (len) { - u8 buf[GHASH_BLOCK_SIZE] = {}; - - memcpy(buf, src, len); - riscv64_ghash_blocks(tctx, dctx, buf, GHASH_BLOCK_SIZE); - memzero_explicit(buf, sizeof(buf)); - } - - memcpy(out, &dctx->accumulator, GHASH_DIGEST_SIZE); - return 0; -} - -static struct shash_alg riscv64_ghash_alg = { - .init = riscv64_ghash_init, - .update = riscv64_ghash_update, - .finup = riscv64_ghash_finup, - .setkey = riscv64_ghash_setkey, - .descsize = sizeof(struct riscv64_ghash_desc_ctx), - .digestsize = GHASH_DIGEST_SIZE, - .base = { - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct riscv64_ghash_tfm_ctx), - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_name = "ghash", - .cra_driver_name = "ghash-riscv64-zvkg", - .cra_module = THIS_MODULE, - }, -}; - -static int __init riscv64_ghash_mod_init(void) -{ - if (riscv_isa_extension_available(NULL, ZVKG) && - riscv_vector_vlen() >= 128) - return crypto_register_shash(&riscv64_ghash_alg); - - return -ENODEV; -} - -static void __exit riscv64_ghash_mod_exit(void) -{ - crypto_unregister_shash(&riscv64_ghash_alg); -} - -module_init(riscv64_ghash_mod_init); -module_exit(riscv64_ghash_mod_exit); - -MODULE_DESCRIPTION("GHASH (RISC-V accelerated)"); -MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>"); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("ghash"); diff --git a/arch/riscv/crypto/sm3-riscv64-glue.c b/arch/riscv/crypto/sm3-riscv64-glue.c deleted file mode 100644 index abdfe4a63a27..000000000000 --- a/arch/riscv/crypto/sm3-riscv64-glue.c +++ /dev/null @@ -1,97 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * SM3 using the RISC-V vector crypto extensions - * - * Copyright (C) 2023 VRULL GmbH - * Author: Heiko Stuebner <heiko.stuebner@vrull.eu> - * - * Copyright (C) 2023 SiFive, Inc. - * Author: Jerry Shih <jerry.shih@sifive.com> - */ - -#include <asm/simd.h> -#include <asm/vector.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> -#include <crypto/sm3.h> -#include <crypto/sm3_base.h> -#include <linux/kernel.h> -#include <linux/module.h> - -/* - * Note: the asm function only uses the 'state' field of struct sm3_state. - * It is assumed to be the first field. - */ -asmlinkage void sm3_transform_zvksh_zvkb( - struct sm3_state *state, const u8 *data, int num_blocks); - -static void sm3_block(struct sm3_state *state, const u8 *data, - int num_blocks) -{ - /* - * Ensure struct sm3_state begins directly with the SM3 - * 256-bit internal state, as this is what the asm function expects. - */ - BUILD_BUG_ON(offsetof(struct sm3_state, state) != 0); - - if (crypto_simd_usable()) { - kernel_vector_begin(); - sm3_transform_zvksh_zvkb(state, data, num_blocks); - kernel_vector_end(); - } else { - sm3_block_generic(state, data, num_blocks); - } -} - -static int riscv64_sm3_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sm3_base_do_update_blocks(desc, data, len, sm3_block); -} - -static int riscv64_sm3_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sm3_base_do_finup(desc, data, len, sm3_block); - return sm3_base_finish(desc, out); -} - -static struct shash_alg riscv64_sm3_alg = { - .init = sm3_base_init, - .update = riscv64_sm3_update, - .finup = riscv64_sm3_finup, - .descsize = SM3_STATE_SIZE, - .digestsize = SM3_DIGEST_SIZE, - .base = { - .cra_blocksize = SM3_BLOCK_SIZE, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_priority = 300, - .cra_name = "sm3", - .cra_driver_name = "sm3-riscv64-zvksh-zvkb", - .cra_module = THIS_MODULE, - }, -}; - -static int __init riscv64_sm3_mod_init(void) -{ - if (riscv_isa_extension_available(NULL, ZVKSH) && - riscv_isa_extension_available(NULL, ZVKB) && - riscv_vector_vlen() >= 128) - return crypto_register_shash(&riscv64_sm3_alg); - - return -ENODEV; -} - -static void __exit riscv64_sm3_mod_exit(void) -{ - crypto_unregister_shash(&riscv64_sm3_alg); -} - -module_init(riscv64_sm3_mod_init); -module_exit(riscv64_sm3_mod_exit); - -MODULE_DESCRIPTION("SM3 (RISC-V accelerated)"); -MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>"); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("sm3"); diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index 98fd0a2f51c6..f8fcc29adbd3 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -797,7 +797,7 @@ CONFIG_CRYPTO_MD5=y CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_CRC32=m @@ -809,7 +809,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_GHASH_S390=m CONFIG_CRYPTO_AES_S390=m CONFIG_CRYPTO_DES_S390=m CONFIG_CRYPTO_HMAC_S390=m diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index 0f4cedcab3ce..3c6ccc0de018 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -781,7 +781,7 @@ CONFIG_CRYPTO_MD5=y CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_CRC32=m @@ -794,7 +794,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_GHASH_S390=m CONFIG_CRYPTO_AES_S390=m CONFIG_CRYPTO_DES_S390=m CONFIG_CRYPTO_HMAC_S390=m diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig index 79a2d0034258..ee83052dbc15 100644 --- a/arch/s390/crypto/Kconfig +++ b/arch/s390/crypto/Kconfig @@ -2,16 +2,6 @@ menu "Accelerated Cryptographic Algorithms for CPU (s390)" -config CRYPTO_GHASH_S390 - tristate "Hash functions: GHASH" - select CRYPTO_HASH - help - GCM GHASH hash function (NIST SP800-38D) - - Architecture: s390 - - It is available as of z196. - config CRYPTO_AES_S390 tristate "Ciphers: AES, modes: ECB, CBC, CTR, XTS, GCM" select CRYPTO_SKCIPHER diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile index 387a229e1038..4449c1b19ef5 100644 --- a/arch/s390/crypto/Makefile +++ b/arch/s390/crypto/Makefile @@ -7,7 +7,6 @@ obj-$(CONFIG_CRYPTO_DES_S390) += des_s390.o obj-$(CONFIG_CRYPTO_AES_S390) += aes_s390.o obj-$(CONFIG_CRYPTO_PAES_S390) += paes_s390.o obj-$(CONFIG_S390_PRNG) += prng.o -obj-$(CONFIG_CRYPTO_GHASH_S390) += ghash_s390.o obj-$(CONFIG_CRYPTO_HMAC_S390) += hmac_s390.o obj-$(CONFIG_CRYPTO_PHMAC_S390) += phmac_s390.o obj-y += arch_random.o diff --git a/arch/s390/crypto/ghash_s390.c b/arch/s390/crypto/ghash_s390.c deleted file mode 100644 index dcbcee37cb63..000000000000 --- a/arch/s390/crypto/ghash_s390.c +++ /dev/null @@ -1,144 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Cryptographic API. - * - * s390 implementation of the GHASH algorithm for GCM (Galois/Counter Mode). - * - * Copyright IBM Corp. 2011 - * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com> - */ - -#include <asm/cpacf.h> -#include <crypto/ghash.h> -#include <crypto/internal/hash.h> -#include <linux/cpufeature.h> -#include <linux/err.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/string.h> - -struct s390_ghash_ctx { - u8 key[GHASH_BLOCK_SIZE]; -}; - -struct s390_ghash_desc_ctx { - u8 icv[GHASH_BLOCK_SIZE]; - u8 key[GHASH_BLOCK_SIZE]; -}; - -static int ghash_init(struct shash_desc *desc) -{ - struct s390_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); - struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - memset(dctx, 0, sizeof(*dctx)); - memcpy(dctx->key, ctx->key, GHASH_BLOCK_SIZE); - - return 0; -} - -static int ghash_setkey(struct crypto_shash *tfm, - const u8 *key, unsigned int keylen) -{ - struct s390_ghash_ctx *ctx = crypto_shash_ctx(tfm); - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - memcpy(ctx->key, key, GHASH_BLOCK_SIZE); - - return 0; -} - -static int ghash_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - unsigned int n; - - n = srclen & ~(GHASH_BLOCK_SIZE - 1); - cpacf_kimd(CPACF_KIMD_GHASH, dctx, src, n); - return srclen - n; -} - -static void ghash_flush(struct s390_ghash_desc_ctx *dctx, const u8 *src, - unsigned int len) -{ - if (len) { - u8 buf[GHASH_BLOCK_SIZE] = {}; - - memcpy(buf, src, len); - cpacf_kimd(CPACF_KIMD_GHASH, dctx, buf, GHASH_BLOCK_SIZE); - memzero_explicit(buf, sizeof(buf)); - } -} - -static int ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *dst) -{ - struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - ghash_flush(dctx, src, len); - memcpy(dst, dctx->icv, GHASH_BLOCK_SIZE); - return 0; -} - -static int ghash_export(struct shash_desc *desc, void *out) -{ - struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - memcpy(out, dctx->icv, GHASH_DIGEST_SIZE); - return 0; -} - -static int ghash_import(struct shash_desc *desc, const void *in) -{ - struct s390_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); - struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - memcpy(dctx->icv, in, GHASH_DIGEST_SIZE); - memcpy(dctx->key, ctx->key, GHASH_BLOCK_SIZE); - return 0; -} - -static struct shash_alg ghash_alg = { - .digestsize = GHASH_DIGEST_SIZE, - .init = ghash_init, - .update = ghash_update, - .finup = ghash_finup, - .setkey = ghash_setkey, - .export = ghash_export, - .import = ghash_import, - .statesize = sizeof(struct ghash_desc_ctx), - .descsize = sizeof(struct s390_ghash_desc_ctx), - .base = { - .cra_name = "ghash", - .cra_driver_name = "ghash-s390", - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct s390_ghash_ctx), - .cra_module = THIS_MODULE, - }, -}; - -static int __init ghash_mod_init(void) -{ - if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_GHASH)) - return -ENODEV; - - return crypto_register_shash(&ghash_alg); -} - -static void __exit ghash_mod_exit(void) -{ - crypto_unregister_shash(&ghash_alg); -} - -module_cpu_feature_match(S390_CPU_FEATURE_MSA, ghash_mod_init); -module_exit(ghash_mod_exit); - -MODULE_ALIAS_CRYPTO("ghash"); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("GHASH hash function, s390 implementation"); diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 7fb2319a0916..822cbf414269 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -331,27 +331,4 @@ config CRYPTO_AEGIS128_AESNI_SSE2 - AES-NI (AES New Instructions) - SSE4.1 (Streaming SIMD Extensions 4.1) -config CRYPTO_SM3_AVX_X86_64 - tristate "Hash functions: SM3 (AVX)" - depends on 64BIT - select CRYPTO_HASH - select CRYPTO_LIB_SM3 - help - SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3 - - Architecture: x86_64 using: - - AVX (Advanced Vector Extensions) - - If unsure, say N. - -config CRYPTO_GHASH_CLMUL_NI_INTEL - tristate "Hash functions: GHASH (CLMUL-NI)" - depends on 64BIT - select CRYPTO_CRYPTD - help - GCM GHASH hash function (NIST SP800-38D) - - Architecture: x86_64 using: - - CLMUL-NI (carry-less multiplication new instructions) - endmenu diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index b21ad0978c52..3d6d5087a65e 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -50,12 +50,6 @@ aesni-intel-$(CONFIG_64BIT) += aes-ctr-avx-x86_64.o \ aes-gcm-vaes-avx512.o \ aes-xts-avx-x86_64.o -obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o -ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o - -obj-$(CONFIG_CRYPTO_SM3_AVX_X86_64) += sm3-avx-x86_64.o -sm3-avx-x86_64-y := sm3-avx-asm_64.o sm3_avx_glue.o - obj-$(CONFIG_CRYPTO_SM4_AESNI_AVX_X86_64) += sm4-aesni-avx-x86_64.o sm4-aesni-avx-x86_64-y := sm4-aesni-avx-asm_64.o sm4_aesni_avx_glue.o diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index e6c38d1d8a92..f522fff9231e 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -25,6 +25,7 @@ #include <crypto/aes.h> #include <crypto/b128ops.h> #include <crypto/gcm.h> +#include <crypto/gf128mul.h> #include <crypto/xts.h> #include <asm/cpu_device_id.h> #include <asm/simd.h> diff --git a/arch/x86/crypto/ghash-clmulni-intel_glue.c b/arch/x86/crypto/ghash-clmulni-intel_glue.c deleted file mode 100644 index aea5d4d06be7..000000000000 --- a/arch/x86/crypto/ghash-clmulni-intel_glue.c +++ /dev/null @@ -1,163 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Accelerated GHASH implementation with Intel PCLMULQDQ-NI - * instructions. This file contains glue code. - * - * Copyright (c) 2009 Intel Corp. - * Author: Huang Ying <ying.huang@intel.com> - */ - -#include <asm/cpu_device_id.h> -#include <asm/simd.h> -#include <crypto/b128ops.h> -#include <crypto/ghash.h> -#include <crypto/internal/hash.h> -#include <crypto/utils.h> -#include <linux/errno.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/string.h> -#include <linux/unaligned.h> - -asmlinkage void clmul_ghash_mul(char *dst, const le128 *shash); - -asmlinkage int clmul_ghash_update(char *dst, const char *src, - unsigned int srclen, const le128 *shash); - -struct x86_ghash_ctx { - le128 shash; -}; - -static int ghash_init(struct shash_desc *desc) -{ - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - memset(dctx, 0, sizeof(*dctx)); - - return 0; -} - -static int ghash_setkey(struct crypto_shash *tfm, - const u8 *key, unsigned int keylen) -{ - struct x86_ghash_ctx *ctx = crypto_shash_ctx(tfm); - u64 a, b; - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - /* - * GHASH maps bits to polynomial coefficients backwards, which makes it - * hard to implement. But it can be shown that the GHASH multiplication - * - * D * K (mod x^128 + x^7 + x^2 + x + 1) - * - * (where D is a data block and K is the key) is equivalent to: - * - * bitreflect(D) * bitreflect(K) * x^(-127) - * (mod x^128 + x^127 + x^126 + x^121 + 1) - * - * So, the code below precomputes: - * - * bitreflect(K) * x^(-127) (mod x^128 + x^127 + x^126 + x^121 + 1) - * - * ... but in Montgomery form (so that Montgomery multiplication can be - * used), i.e. with an extra x^128 factor, which means actually: - * - * bitreflect(K) * x (mod x^128 + x^127 + x^126 + x^121 + 1) - * - * The within-a-byte part of bitreflect() cancels out GHASH's built-in - * reflection, and thus bitreflect() is actually a byteswap. - */ - a = get_unaligned_be64(key); - b = get_unaligned_be64(key + 8); - ctx->shash.a = cpu_to_le64((a << 1) | (b >> 63)); - ctx->shash.b = cpu_to_le64((b << 1) | (a >> 63)); - if (a >> 63) - ctx->shash.a ^= cpu_to_le64((u64)0xc2 << 56); - return 0; -} - -static int ghash_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct x86_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - u8 *dst = dctx->buffer; - int remain; - - kernel_fpu_begin(); - remain = clmul_ghash_update(dst, src, srclen, &ctx->shash); - kernel_fpu_end(); - return remain; -} - -static void ghash_flush(struct x86_ghash_ctx *ctx, struct ghash_desc_ctx *dctx, - const u8 *src, unsigned int len) -{ - u8 *dst = dctx->buffer; - - kernel_fpu_begin(); - if (len) { - crypto_xor(dst, src, len); - clmul_ghash_mul(dst, &ctx->shash); - } - kernel_fpu_end(); -} - -static int ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *dst) -{ - struct x86_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - u8 *buf = dctx->buffer; - - ghash_flush(ctx, dctx, src, len); - memcpy(dst, buf, GHASH_BLOCK_SIZE); - - return 0; -} - -static struct shash_alg ghash_alg = { - .digestsize = GHASH_DIGEST_SIZE, - .init = ghash_init, - .update = ghash_update, - .finup = ghash_finup, - .setkey = ghash_setkey, - .descsize = sizeof(struct ghash_desc_ctx), - .base = { - .cra_name = "ghash", - .cra_driver_name = "ghash-pclmulqdqni", - .cra_priority = 400, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct x86_ghash_ctx), - .cra_module = THIS_MODULE, - }, -}; - -static const struct x86_cpu_id pcmul_cpu_id[] = { - X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL), /* Pickle-Mickle-Duck */ - {} -}; -MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id); - -static int __init ghash_pclmulqdqni_mod_init(void) -{ - if (!x86_match_cpu(pcmul_cpu_id)) - return -ENODEV; - - return crypto_register_shash(&ghash_alg); -} - -static void __exit ghash_pclmulqdqni_mod_exit(void) -{ - crypto_unregister_shash(&ghash_alg); -} - -module_init(ghash_pclmulqdqni_mod_init); -module_exit(ghash_pclmulqdqni_mod_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("GHASH hash function, accelerated by PCLMULQDQ-NI"); -MODULE_ALIAS_CRYPTO("ghash"); diff --git a/arch/x86/crypto/sm3_avx_glue.c b/arch/x86/crypto/sm3_avx_glue.c deleted file mode 100644 index 6e8c42b9dc8e..000000000000 --- a/arch/x86/crypto/sm3_avx_glue.c +++ /dev/null @@ -1,100 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * SM3 Secure Hash Algorithm, AVX assembler accelerated. - * specified in: https://datatracker.ietf.org/doc/html/draft-sca-cfrg-sm3-02 - * - * Copyright (C) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> -#include <crypto/sm3.h> -#include <crypto/sm3_base.h> -#include <linux/cpufeature.h> -#include <linux/kernel.h> -#include <linux/module.h> - -asmlinkage void sm3_transform_avx(struct sm3_state *state, - const u8 *data, int nblocks); - -static int sm3_avx_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - int remain; - - /* - * Make sure struct sm3_state begins directly with the SM3 - * 256-bit internal state, as this is what the asm functions expect. - */ - BUILD_BUG_ON(offsetof(struct sm3_state, state) != 0); - - kernel_fpu_begin(); - remain = sm3_base_do_update_blocks(desc, data, len, sm3_transform_avx); - kernel_fpu_end(); - return remain; -} - -static int sm3_avx_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - kernel_fpu_begin(); - sm3_base_do_finup(desc, data, len, sm3_transform_avx); - kernel_fpu_end(); - return sm3_base_finish(desc, out); -} - -static struct shash_alg sm3_avx_alg = { - .digestsize = SM3_DIGEST_SIZE, - .init = sm3_base_init, - .update = sm3_avx_update, - .finup = sm3_avx_finup, - .descsize = SM3_STATE_SIZE, - .base = { - .cra_name = "sm3", - .cra_driver_name = "sm3-avx", - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SM3_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init sm3_avx_mod_init(void) -{ - const char *feature_name; - - if (!boot_cpu_has(X86_FEATURE_AVX)) { - pr_info("AVX instruction are not detected.\n"); - return -ENODEV; - } - - if (!boot_cpu_has(X86_FEATURE_BMI2)) { - pr_info("BMI2 instruction are not detected.\n"); - return -ENODEV; - } - - if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, - &feature_name)) { - pr_info("CPU feature '%s' is not supported.\n", feature_name); - return -ENODEV; - } - - return crypto_register_shash(&sm3_avx_alg); -} - -static void __exit sm3_avx_mod_exit(void) -{ - crypto_unregister_shash(&sm3_avx_alg); -} - -module_init(sm3_avx_mod_init); -module_exit(sm3_avx_mod_exit); - -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>"); -MODULE_DESCRIPTION("SM3 Secure Hash Algorithm, AVX assembler accelerated"); -MODULE_ALIAS_CRYPTO("sm3"); -MODULE_ALIAS_CRYPTO("sm3-avx"); diff --git a/crypto/Kconfig b/crypto/Kconfig index 7e5ea61168c3..5a2bee919d9f 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -355,6 +355,8 @@ config CRYPTO_AES tristate "AES (Advanced Encryption Standard)" select CRYPTO_ALGAPI select CRYPTO_LIB_AES + select CRYPTO_LIB_AES_CBC_MACS if CRYPTO_CMAC || CRYPTO_XCBC || CRYPTO_CCM + select CRYPTO_HASH if CRYPTO_CMAC || CRYPTO_XCBC || CRYPTO_CCM help AES cipher algorithms (Rijndael)(FIPS-197, ISO/IEC 18033-3) @@ -678,7 +680,7 @@ config CRYPTO_ECB config CRYPTO_HCTR2 tristate "HCTR2" select CRYPTO_XCTR - select CRYPTO_LIB_POLYVAL + select CRYPTO_LIB_GF128HASH select CRYPTO_MANAGER help HCTR2 length-preserving encryption mode @@ -786,7 +788,7 @@ config CRYPTO_GCM tristate "GCM (Galois/Counter Mode) and GMAC (GCM MAC)" select CRYPTO_CTR select CRYPTO_AEAD - select CRYPTO_GHASH + select CRYPTO_LIB_GF128HASH select CRYPTO_MANAGER help GCM (Galois/Counter Mode) authenticated encryption mode and GMAC @@ -880,13 +882,6 @@ config CRYPTO_CMAC CMAC (Cipher-based Message Authentication Code) authentication mode (NIST SP800-38B and IETF RFC4493) -config CRYPTO_GHASH - tristate "GHASH" - select CRYPTO_HASH - select CRYPTO_LIB_GF128MUL - help - GCM GHASH function (NIST SP800-38D) - config CRYPTO_HMAC tristate "HMAC (Keyed-Hash MAC)" select CRYPTO_HASH @@ -973,7 +968,7 @@ config CRYPTO_SHA3 help SHA-3 secure hash algorithms (FIPS 202, ISO/IEC 10118-3) -config CRYPTO_SM3_GENERIC +config CRYPTO_SM3 tristate "SM3 (ShangMi 3)" select CRYPTO_HASH select CRYPTO_LIB_SM3 @@ -1169,8 +1164,8 @@ endif # if CRYPTO_DRBG_MENU config CRYPTO_JITTERENTROPY tristate "CPU Jitter Non-Deterministic RNG (Random Number Generator)" + select CRYPTO_LIB_SHA3 select CRYPTO_RNG - select CRYPTO_SHA3 help CPU Jitter RNG (Random Number Generator) from the Jitterentropy library diff --git a/crypto/Makefile b/crypto/Makefile index 8eb3f9a629d8..c396e9e4a8bc 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -82,7 +82,7 @@ obj-$(CONFIG_CRYPTO_SHA1) += sha1.o obj-$(CONFIG_CRYPTO_SHA256) += sha256.o obj-$(CONFIG_CRYPTO_SHA512) += sha512.o obj-$(CONFIG_CRYPTO_SHA3) += sha3.o -obj-$(CONFIG_CRYPTO_SM3_GENERIC) += sm3_generic.o +obj-$(CONFIG_CRYPTO_SM3) += sm3.o obj-$(CONFIG_CRYPTO_STREEBOG) += streebog_generic.o obj-$(CONFIG_CRYPTO_WP512) += wp512.o CFLAGS_wp512.o := $(call cc-option,-fno-schedule-insns) # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=79149 @@ -170,7 +170,6 @@ UBSAN_SANITIZE_jitterentropy.o = n jitterentropy_rng-y := jitterentropy.o jitterentropy-kcapi.o obj-$(CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE) += jitterentropy-testing.o obj-$(CONFIG_CRYPTO_BENCHMARK) += tcrypt.o -obj-$(CONFIG_CRYPTO_GHASH) += ghash-generic.o obj-$(CONFIG_CRYPTO_USER_API) += af_alg.o obj-$(CONFIG_CRYPTO_USER_API_HASH) += algif_hash.o obj-$(CONFIG_CRYPTO_USER_API_SKCIPHER) += algif_skcipher.o diff --git a/crypto/aes.c b/crypto/aes.c index ae8385df0ce5..6bf23eb0503f 100644 --- a/crypto/aes.c +++ b/crypto/aes.c @@ -5,8 +5,10 @@ * Copyright 2026 Google LLC */ +#include <crypto/aes-cbc-macs.h> #include <crypto/aes.h> #include <crypto/algapi.h> +#include <crypto/internal/hash.h> #include <linux/module.h> static_assert(__alignof__(struct aes_key) <= CRYPTO_MINALIGN); @@ -33,6 +35,98 @@ static void crypto_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) aes_decrypt(key, out, in); } +static_assert(__alignof__(struct aes_cmac_key) <= CRYPTO_MINALIGN); +#define AES_CMAC_KEY(tfm) ((struct aes_cmac_key *)crypto_shash_ctx(tfm)) +#define AES_CMAC_CTX(desc) ((struct aes_cmac_ctx *)shash_desc_ctx(desc)) + +static int __maybe_unused crypto_aes_cmac_setkey(struct crypto_shash *tfm, + const u8 *in_key, + unsigned int key_len) +{ + return aes_cmac_preparekey(AES_CMAC_KEY(tfm), in_key, key_len); +} + +static int __maybe_unused crypto_aes_xcbc_setkey(struct crypto_shash *tfm, + const u8 *in_key, + unsigned int key_len) +{ + if (key_len != AES_KEYSIZE_128) + return -EINVAL; + aes_xcbcmac_preparekey(AES_CMAC_KEY(tfm), in_key); + return 0; +} + +static int __maybe_unused crypto_aes_cmac_init(struct shash_desc *desc) +{ + aes_cmac_init(AES_CMAC_CTX(desc), AES_CMAC_KEY(desc->tfm)); + return 0; +} + +static int __maybe_unused crypto_aes_cmac_update(struct shash_desc *desc, + const u8 *data, + unsigned int len) +{ + aes_cmac_update(AES_CMAC_CTX(desc), data, len); + return 0; +} + +static int __maybe_unused crypto_aes_cmac_final(struct shash_desc *desc, + u8 *out) +{ + aes_cmac_final(AES_CMAC_CTX(desc), out); + return 0; +} + +static int __maybe_unused crypto_aes_cmac_digest(struct shash_desc *desc, + const u8 *data, + unsigned int len, u8 *out) +{ + aes_cmac(AES_CMAC_KEY(desc->tfm), data, len, out); + return 0; +} + +static_assert(__alignof__(struct aes_enckey) <= CRYPTO_MINALIGN); +#define AES_CBCMAC_KEY(tfm) ((struct aes_enckey *)crypto_shash_ctx(tfm)) +#define AES_CBCMAC_CTX(desc) ((struct aes_cbcmac_ctx *)shash_desc_ctx(desc)) + +static int __maybe_unused crypto_aes_cbcmac_setkey(struct crypto_shash *tfm, + const u8 *in_key, + unsigned int key_len) +{ + return aes_prepareenckey(AES_CBCMAC_KEY(tfm), in_key, key_len); +} + +static int __maybe_unused crypto_aes_cbcmac_init(struct shash_desc *desc) +{ + aes_cbcmac_init(AES_CBCMAC_CTX(desc), AES_CBCMAC_KEY(desc->tfm)); + return 0; +} + +static int __maybe_unused crypto_aes_cbcmac_update(struct shash_desc *desc, + const u8 *data, + unsigned int len) +{ + aes_cbcmac_update(AES_CBCMAC_CTX(desc), data, len); + return 0; +} + +static int __maybe_unused crypto_aes_cbcmac_final(struct shash_desc *desc, + u8 *out) +{ + aes_cbcmac_final(AES_CBCMAC_CTX(desc), out); + return 0; +} + +static int __maybe_unused crypto_aes_cbcmac_digest(struct shash_desc *desc, + const u8 *data, + unsigned int len, u8 *out) +{ + aes_cbcmac_init(AES_CBCMAC_CTX(desc), AES_CBCMAC_KEY(desc->tfm)); + aes_cbcmac_update(AES_CBCMAC_CTX(desc), data, len); + aes_cbcmac_final(AES_CBCMAC_CTX(desc), out); + return 0; +} + static struct crypto_alg alg = { .cra_name = "aes", .cra_driver_name = "aes-lib", @@ -48,19 +142,106 @@ static struct crypto_alg alg = { .cia_decrypt = crypto_aes_decrypt } } }; +static struct shash_alg mac_algs[] = { +#if IS_ENABLED(CONFIG_CRYPTO_CMAC) + { + .base.cra_name = "cmac(aes)", + .base.cra_driver_name = "cmac-aes-lib", + .base.cra_priority = 300, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct aes_cmac_key), + .base.cra_module = THIS_MODULE, + .digestsize = AES_BLOCK_SIZE, + .setkey = crypto_aes_cmac_setkey, + .init = crypto_aes_cmac_init, + .update = crypto_aes_cmac_update, + .final = crypto_aes_cmac_final, + .digest = crypto_aes_cmac_digest, + .descsize = sizeof(struct aes_cmac_ctx), + }, +#endif +#if IS_ENABLED(CONFIG_CRYPTO_XCBC) + { + /* + * Note that the only difference between xcbc(aes) and cmac(aes) + * is the preparekey function. + */ + .base.cra_name = "xcbc(aes)", + .base.cra_driver_name = "xcbc-aes-lib", + .base.cra_priority = 300, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct aes_cmac_key), + .base.cra_module = THIS_MODULE, + .digestsize = AES_BLOCK_SIZE, + .setkey = crypto_aes_xcbc_setkey, + .init = crypto_aes_cmac_init, + .update = crypto_aes_cmac_update, + .final = crypto_aes_cmac_final, + .digest = crypto_aes_cmac_digest, + .descsize = sizeof(struct aes_cmac_ctx), + }, +#endif +#if IS_ENABLED(CONFIG_CRYPTO_CCM) + { + .base.cra_name = "cbcmac(aes)", + .base.cra_driver_name = "cbcmac-aes-lib", + .base.cra_priority = 300, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct aes_enckey), + .base.cra_module = THIS_MODULE, + .digestsize = AES_BLOCK_SIZE, + .setkey = crypto_aes_cbcmac_setkey, + .init = crypto_aes_cbcmac_init, + .update = crypto_aes_cbcmac_update, + .final = crypto_aes_cbcmac_final, + .digest = crypto_aes_cbcmac_digest, + .descsize = sizeof(struct aes_cbcmac_ctx), + }, +#endif +}; + static int __init crypto_aes_mod_init(void) { - return crypto_register_alg(&alg); + int err = crypto_register_alg(&alg); + + if (err) + return err; + + if (ARRAY_SIZE(mac_algs) > 0) { + err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs)); + if (err) + goto err_unregister_alg; + } /* Else, CONFIG_CRYPTO_HASH might not be enabled. */ + return 0; + +err_unregister_alg: + crypto_unregister_alg(&alg); + return err; } module_init(crypto_aes_mod_init); static void __exit crypto_aes_mod_exit(void) { + if (ARRAY_SIZE(mac_algs) > 0) + crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs)); crypto_unregister_alg(&alg); } module_exit(crypto_aes_mod_exit); MODULE_DESCRIPTION("Crypto API support for AES block cipher"); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); MODULE_LICENSE("GPL"); MODULE_ALIAS_CRYPTO("aes"); MODULE_ALIAS_CRYPTO("aes-lib"); +#if IS_ENABLED(CONFIG_CRYPTO_CMAC) +MODULE_ALIAS_CRYPTO("cmac(aes)"); +MODULE_ALIAS_CRYPTO("cmac-aes-lib"); +#endif +#if IS_ENABLED(CONFIG_CRYPTO_XCBC) +MODULE_ALIAS_CRYPTO("xcbc(aes)"); +MODULE_ALIAS_CRYPTO("xcbc-aes-lib"); +#endif +#if IS_ENABLED(CONFIG_CRYPTO_CCM) +MODULE_ALIAS_CRYPTO("cbcmac(aes)"); +MODULE_ALIAS_CRYPTO("cbcmac-aes-lib"); +#endif diff --git a/crypto/gcm.c b/crypto/gcm.c index e1e878d37410..5f16b237b3c5 100644 --- a/crypto/gcm.c +++ b/crypto/gcm.c @@ -5,13 +5,11 @@ * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi> */ -#include <crypto/gf128mul.h> #include <crypto/internal/aead.h> #include <crypto/internal/skcipher.h> -#include <crypto/internal/hash.h> #include <crypto/scatterwalk.h> #include <crypto/gcm.h> -#include <crypto/hash.h> +#include <crypto/gf128hash.h> #include <linux/err.h> #include <linux/init.h> #include <linux/kernel.h> @@ -20,12 +18,11 @@ struct gcm_instance_ctx { struct crypto_skcipher_spawn ctr; - struct crypto_ahash_spawn ghash; }; struct crypto_gcm_ctx { struct crypto_skcipher *ctr; - struct crypto_ahash *ghash; + struct ghash_key ghash; }; struct crypto_rfc4106_ctx { @@ -52,31 +49,15 @@ struct crypto_rfc4543_req_ctx { struct aead_request subreq; }; -struct crypto_gcm_ghash_ctx { - unsigned int cryptlen; - struct scatterlist *src; - int (*complete)(struct aead_request *req, u32 flags); -}; - struct crypto_gcm_req_priv_ctx { u8 iv[16]; u8 auth_tag[16]; u8 iauth_tag[16]; struct scatterlist src[3]; struct scatterlist dst[3]; - struct scatterlist sg; - struct crypto_gcm_ghash_ctx ghash_ctx; - union { - struct ahash_request ahreq; - struct skcipher_request skreq; - } u; + struct skcipher_request skreq; /* Must be last */ }; -static struct { - u8 buf[16]; - struct scatterlist sg; -} *gcm_zeroes; - static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( struct aead_request *req) { @@ -89,10 +70,9 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); - struct crypto_ahash *ghash = ctx->ghash; struct crypto_skcipher *ctr = ctx->ctr; struct { - be128 hash; + u8 h[GHASH_BLOCK_SIZE]; u8 iv[16]; struct crypto_wait wait; @@ -115,14 +95,14 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, return -ENOMEM; crypto_init_wait(&data->wait); - sg_init_one(data->sg, &data->hash, sizeof(data->hash)); + sg_init_one(data->sg, data->h, sizeof(data->h)); skcipher_request_set_tfm(&data->req, ctr); skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP | CRYPTO_TFM_REQ_MAY_BACKLOG, crypto_req_done, &data->wait); skcipher_request_set_crypt(&data->req, data->sg, data->sg, - sizeof(data->hash), data->iv); + sizeof(data->h), data->iv); err = crypto_wait_req(crypto_skcipher_encrypt(&data->req), &data->wait); @@ -130,10 +110,7 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, if (err) goto out; - crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK); - crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) & - CRYPTO_TFM_REQ_MASK); - err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128)); + ghash_preparekey(&ctx->ghash, data->h); out: kfree_sensitive(data); return err; @@ -176,7 +153,7 @@ static void crypto_gcm_init_crypt(struct aead_request *req, struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct skcipher_request *skreq = &pctx->u.skreq; + struct skcipher_request *skreq = &pctx->skreq; struct scatterlist *dst; dst = req->src == req->dst ? pctx->src : pctx->dst; @@ -187,244 +164,65 @@ static void crypto_gcm_init_crypt(struct aead_request *req, pctx->iv); } -static inline unsigned int gcm_remain(unsigned int len) -{ - len &= 0xfU; - return len ? 16 - len : 0; -} - -static void gcm_hash_len_done(void *data, int err); - -static int gcm_hash_update(struct aead_request *req, - crypto_completion_t compl, - struct scatterlist *src, - unsigned int len, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct ahash_request *ahreq = &pctx->u.ahreq; - - ahash_request_set_callback(ahreq, flags, compl, req); - ahash_request_set_crypt(ahreq, src, NULL, len); - - return crypto_ahash_update(ahreq); -} - -static int gcm_hash_remain(struct aead_request *req, - unsigned int remain, - crypto_completion_t compl, u32 flags) -{ - return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags); -} - -static int gcm_hash_len(struct aead_request *req, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct ahash_request *ahreq = &pctx->u.ahreq; - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - be128 lengths; - - lengths.a = cpu_to_be64(req->assoclen * 8); - lengths.b = cpu_to_be64(gctx->cryptlen * 8); - memcpy(pctx->iauth_tag, &lengths, 16); - sg_init_one(&pctx->sg, pctx->iauth_tag, 16); - ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req); - ahash_request_set_crypt(ahreq, &pctx->sg, - pctx->iauth_tag, sizeof(lengths)); - - return crypto_ahash_finup(ahreq); -} - -static int gcm_hash_len_continue(struct aead_request *req, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - - return gctx->complete(req, flags); -} - -static void gcm_hash_len_done(void *data, int err) -{ - struct aead_request *req = data; - - if (err) - goto out; - - err = gcm_hash_len_continue(req, 0); - if (err == -EINPROGRESS) - return; - -out: - aead_request_complete(req, err); -} - -static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags) -{ - return gcm_hash_len(req, flags) ?: - gcm_hash_len_continue(req, flags); -} - -static void gcm_hash_crypt_remain_done(void *data, int err) -{ - struct aead_request *req = data; - - if (err) - goto out; - - err = gcm_hash_crypt_remain_continue(req, 0); - if (err == -EINPROGRESS) - return; - -out: - aead_request_complete(req, err); -} - -static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - unsigned int remain; - - remain = gcm_remain(gctx->cryptlen); - if (remain) - return gcm_hash_remain(req, remain, - gcm_hash_crypt_remain_done, flags) ?: - gcm_hash_crypt_remain_continue(req, flags); - - return gcm_hash_crypt_remain_continue(req, flags); -} - -static void gcm_hash_crypt_done(void *data, int err) -{ - struct aead_request *req = data; - - if (err) - goto out; - - err = gcm_hash_crypt_continue(req, 0); - if (err == -EINPROGRESS) - return; - -out: - aead_request_complete(req, err); -} - -static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - - if (gctx->cryptlen) - return gcm_hash_update(req, gcm_hash_crypt_done, - gctx->src, gctx->cryptlen, flags) ?: - gcm_hash_crypt_continue(req, flags); - - return gcm_hash_crypt_remain_continue(req, flags); -} - -static void gcm_hash_assoc_remain_done(void *data, int err) -{ - struct aead_request *req = data; - - if (err) - goto out; - - err = gcm_hash_assoc_remain_continue(req, 0); - if (err == -EINPROGRESS) - return; - -out: - aead_request_complete(req, err); -} - -static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags) +static void ghash_update_sg_and_pad(struct ghash_ctx *ghash, + struct scatterlist *sg, unsigned int len) { - unsigned int remain; + static const u8 zeroes[GHASH_BLOCK_SIZE]; - remain = gcm_remain(req->assoclen); - if (remain) - return gcm_hash_remain(req, remain, - gcm_hash_assoc_remain_done, flags) ?: - gcm_hash_assoc_remain_continue(req, flags); + if (len) { + unsigned int pad_len = -len % GHASH_BLOCK_SIZE; + struct scatter_walk walk; - return gcm_hash_assoc_remain_continue(req, flags); -} + scatterwalk_start(&walk, sg); + do { + unsigned int n = scatterwalk_next(&walk, len); -static void gcm_hash_assoc_done(void *data, int err) -{ - struct aead_request *req = data; + ghash_update(ghash, walk.addr, n); + scatterwalk_done_src(&walk, n); + len -= n; + } while (len); - if (err) - goto out; - - err = gcm_hash_assoc_continue(req, 0); - if (err == -EINPROGRESS) - return; - -out: - aead_request_complete(req, err); -} - -static int gcm_hash_init_continue(struct aead_request *req, u32 flags) -{ - if (req->assoclen) - return gcm_hash_update(req, gcm_hash_assoc_done, - req->src, req->assoclen, flags) ?: - gcm_hash_assoc_continue(req, flags); - - return gcm_hash_assoc_remain_continue(req, flags); + if (pad_len) + ghash_update(ghash, zeroes, pad_len); + } } -static void gcm_hash_init_done(void *data, int err) +static void gcm_hash(struct aead_request *req, struct scatterlist *ctext, + unsigned int datalen, u8 out[GHASH_BLOCK_SIZE]) { - struct aead_request *req = data; - - if (err) - goto out; + const struct crypto_gcm_ctx *ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); + __be64 lengths[2] = { + cpu_to_be64(8 * (u64)req->assoclen), + cpu_to_be64(8 * (u64)datalen), + }; + struct ghash_ctx ghash; - err = gcm_hash_init_continue(req, 0); - if (err == -EINPROGRESS) - return; + ghash_init(&ghash, &ctx->ghash); -out: - aead_request_complete(req, err); -} + /* Associated data, then zero-padding to the next 16-byte boundary */ + ghash_update_sg_and_pad(&ghash, req->src, req->assoclen); -static int gcm_hash(struct aead_request *req, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct ahash_request *ahreq = &pctx->u.ahreq; - struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + /* Ciphertext, then zero-padding to the next 16-byte boundary */ + ghash_update_sg_and_pad(&ghash, ctext, datalen); - ahash_request_set_tfm(ahreq, ctx->ghash); + /* Lengths block */ + ghash_update(&ghash, (const u8 *)lengths, sizeof(lengths)); - ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req); - return crypto_ahash_init(ahreq) ?: - gcm_hash_init_continue(req, flags); + ghash_final(&ghash, out); } -static int gcm_enc_copy_hash(struct aead_request *req, u32 flags) +static int gcm_add_auth_tag(struct aead_request *req) { - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct crypto_aead *aead = crypto_aead_reqtfm(req); - u8 *auth_tag = pctx->auth_tag; - - crypto_xor(auth_tag, pctx->iauth_tag, 16); - scatterwalk_map_and_copy(auth_tag, req->dst, - req->assoclen + req->cryptlen, - crypto_aead_authsize(aead), 1); - return 0; -} - -static int gcm_encrypt_continue(struct aead_request *req, u32 flags) -{ struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst); - gctx->cryptlen = req->cryptlen; - gctx->complete = gcm_enc_copy_hash; - - return gcm_hash(req, flags); + gcm_hash(req, sg_next(req->src == req->dst ? pctx->src : pctx->dst), + req->cryptlen, pctx->iauth_tag); + crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16); + memcpy_to_sglist(req->dst, req->assoclen + req->cryptlen, + pctx->auth_tag, crypto_aead_authsize(aead)); + return 0; } static void gcm_encrypt_done(void *data, int err) @@ -434,9 +232,7 @@ static void gcm_encrypt_done(void *data, int err) if (err) goto out; - err = gcm_encrypt_continue(req, 0); - if (err == -EINPROGRESS) - return; + err = gcm_add_auth_tag(req); out: aead_request_complete(req, err); @@ -445,15 +241,14 @@ out: static int crypto_gcm_encrypt(struct aead_request *req) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct skcipher_request *skreq = &pctx->u.skreq; + struct skcipher_request *skreq = &pctx->skreq; u32 flags = aead_request_flags(req); crypto_gcm_init_common(req); crypto_gcm_init_crypt(req, req->cryptlen); skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req); - return crypto_skcipher_encrypt(skreq) ?: - gcm_encrypt_continue(req, flags); + return crypto_skcipher_encrypt(skreq) ?: gcm_add_auth_tag(req); } static int crypto_gcm_verify(struct aead_request *req) @@ -481,35 +276,21 @@ static void gcm_decrypt_done(void *data, int err) aead_request_complete(req, err); } -static int gcm_dec_hash_continue(struct aead_request *req, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct skcipher_request *skreq = &pctx->u.skreq; - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - - crypto_gcm_init_crypt(req, gctx->cryptlen); - skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req); - return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req); -} - static int crypto_gcm_decrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - unsigned int authsize = crypto_aead_authsize(aead); - unsigned int cryptlen = req->cryptlen; - u32 flags = aead_request_flags(req); - - cryptlen -= authsize; + struct skcipher_request *skreq = &pctx->skreq; + unsigned int datalen = req->cryptlen - crypto_aead_authsize(aead); crypto_gcm_init_common(req); - gctx->src = sg_next(pctx->src); - gctx->cryptlen = cryptlen; - gctx->complete = gcm_dec_hash_continue; + gcm_hash(req, sg_next(pctx->src), datalen, pctx->iauth_tag); - return gcm_hash(req, flags); + crypto_gcm_init_crypt(req, datalen); + skcipher_request_set_callback(skreq, aead_request_flags(req), + gcm_decrypt_done, req); + return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req); } static int crypto_gcm_init_tfm(struct crypto_aead *tfm) @@ -518,43 +299,26 @@ static int crypto_gcm_init_tfm(struct crypto_aead *tfm) struct gcm_instance_ctx *ictx = aead_instance_ctx(inst); struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_skcipher *ctr; - struct crypto_ahash *ghash; unsigned long align; - int err; - - ghash = crypto_spawn_ahash(&ictx->ghash); - if (IS_ERR(ghash)) - return PTR_ERR(ghash); ctr = crypto_spawn_skcipher(&ictx->ctr); - err = PTR_ERR(ctr); if (IS_ERR(ctr)) - goto err_free_hash; + return PTR_ERR(ctr); ctx->ctr = ctr; - ctx->ghash = ghash; align = crypto_aead_alignmask(tfm); align &= ~(crypto_tfm_ctx_alignment() - 1); crypto_aead_set_reqsize(tfm, - align + offsetof(struct crypto_gcm_req_priv_ctx, u) + - max(sizeof(struct skcipher_request) + - crypto_skcipher_reqsize(ctr), - sizeof(struct ahash_request) + - crypto_ahash_reqsize(ghash))); - + align + sizeof(struct crypto_gcm_req_priv_ctx) + + crypto_skcipher_reqsize(ctr)); return 0; - -err_free_hash: - crypto_free_ahash(ghash); - return err; } static void crypto_gcm_exit_tfm(struct crypto_aead *tfm) { struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); - crypto_free_ahash(ctx->ghash); crypto_free_skcipher(ctx->ctr); } @@ -563,20 +327,16 @@ static void crypto_gcm_free(struct aead_instance *inst) struct gcm_instance_ctx *ctx = aead_instance_ctx(inst); crypto_drop_skcipher(&ctx->ctr); - crypto_drop_ahash(&ctx->ghash); kfree(inst); } static int crypto_gcm_create_common(struct crypto_template *tmpl, - struct rtattr **tb, - const char *ctr_name, - const char *ghash_name) + struct rtattr **tb, const char *ctr_name) { struct skcipher_alg_common *ctr; u32 mask; struct aead_instance *inst; struct gcm_instance_ctx *ctx; - struct hash_alg_common *ghash; int err; err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask); @@ -588,17 +348,6 @@ static int crypto_gcm_create_common(struct crypto_template *tmpl, return -ENOMEM; ctx = aead_instance_ctx(inst); - err = crypto_grab_ahash(&ctx->ghash, aead_crypto_instance(inst), - ghash_name, 0, mask); - if (err) - goto err_free_inst; - ghash = crypto_spawn_ahash_alg(&ctx->ghash); - - err = -EINVAL; - if (strcmp(ghash->base.cra_name, "ghash") != 0 || - ghash->digestsize != 16) - goto err_free_inst; - err = crypto_grab_skcipher(&ctx->ctr, aead_crypto_instance(inst), ctr_name, 0, mask); if (err) @@ -617,13 +366,11 @@ static int crypto_gcm_create_common(struct crypto_template *tmpl, goto err_free_inst; if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, - "gcm_base(%s,%s)", ctr->base.cra_driver_name, - ghash->base.cra_driver_name) >= - CRYPTO_MAX_ALG_NAME) + "gcm_base(%s,ghash-lib)", + ctr->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto err_free_inst; - inst->alg.base.cra_priority = (ghash->base.cra_priority + - ctr->base.cra_priority) / 2; + inst->alg.base.cra_priority = ctr->base.cra_priority; inst->alg.base.cra_blocksize = 1; inst->alg.base.cra_alignmask = ctr->base.cra_alignmask; inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx); @@ -660,7 +407,7 @@ static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb) CRYPTO_MAX_ALG_NAME) return -ENAMETOOLONG; - return crypto_gcm_create_common(tmpl, tb, ctr_name, "ghash"); + return crypto_gcm_create_common(tmpl, tb, ctr_name); } static int crypto_gcm_base_create(struct crypto_template *tmpl, @@ -677,7 +424,16 @@ static int crypto_gcm_base_create(struct crypto_template *tmpl, if (IS_ERR(ghash_name)) return PTR_ERR(ghash_name); - return crypto_gcm_create_common(tmpl, tb, ctr_name, ghash_name); + /* + * Originally this parameter allowed requesting a specific + * implementation of GHASH. This is no longer supported. Now the best + * implementation of GHASH is just always used. + */ + if (strcmp(ghash_name, "ghash") != 0 && + strcmp(ghash_name, "ghash-lib") != 0) + return -EINVAL; + + return crypto_gcm_create_common(tmpl, tb, ctr_name); } static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key, @@ -1096,25 +852,12 @@ static struct crypto_template crypto_gcm_tmpls[] = { static int __init crypto_gcm_module_init(void) { - int err; - - gcm_zeroes = kzalloc_obj(*gcm_zeroes); - if (!gcm_zeroes) - return -ENOMEM; - - sg_init_one(&gcm_zeroes->sg, gcm_zeroes->buf, sizeof(gcm_zeroes->buf)); - - err = crypto_register_templates(crypto_gcm_tmpls, - ARRAY_SIZE(crypto_gcm_tmpls)); - if (err) - kfree(gcm_zeroes); - - return err; + return crypto_register_templates(crypto_gcm_tmpls, + ARRAY_SIZE(crypto_gcm_tmpls)); } static void __exit crypto_gcm_module_exit(void) { - kfree(gcm_zeroes); crypto_unregister_templates(crypto_gcm_tmpls, ARRAY_SIZE(crypto_gcm_tmpls)); } diff --git a/crypto/ghash-generic.c b/crypto/ghash-generic.c deleted file mode 100644 index e5803c249c12..000000000000 --- a/crypto/ghash-generic.c +++ /dev/null @@ -1,162 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * GHASH: hash function for GCM (Galois/Counter Mode). - * - * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi> - * Copyright (c) 2009 Intel Corp. - * Author: Huang Ying <ying.huang@intel.com> - */ - -/* - * GHASH is a keyed hash function used in GCM authentication tag generation. - * - * The original GCM paper [1] presents GHASH as a function GHASH(H, A, C) which - * takes a 16-byte hash key H, additional authenticated data A, and a ciphertext - * C. It formats A and C into a single byte string X, interprets X as a - * polynomial over GF(2^128), and evaluates this polynomial at the point H. - * - * However, the NIST standard for GCM [2] presents GHASH as GHASH(H, X) where X - * is the already-formatted byte string containing both A and C. - * - * "ghash" in the Linux crypto API uses the 'X' (pre-formatted) convention, - * since the API supports only a single data stream per hash. Thus, the - * formatting of 'A' and 'C' is done in the "gcm" template, not in "ghash". - * - * The reason "ghash" is separate from "gcm" is to allow "gcm" to use an - * accelerated "ghash" when a standalone accelerated "gcm(aes)" is unavailable. - * It is generally inappropriate to use "ghash" for other purposes, since it is - * an "ε-almost-XOR-universal hash function", not a cryptographic hash function. - * It can only be used securely in crypto modes specially designed to use it. - * - * [1] The Galois/Counter Mode of Operation (GCM) - * (http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.694.695&rep=rep1&type=pdf) - * [2] Recommendation for Block Cipher Modes of Operation: Galois/Counter Mode (GCM) and GMAC - * (https://csrc.nist.gov/publications/detail/sp/800-38d/final) - */ - -#include <crypto/gf128mul.h> -#include <crypto/ghash.h> -#include <crypto/internal/hash.h> -#include <crypto/utils.h> -#include <linux/err.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/string.h> - -static int ghash_init(struct shash_desc *desc) -{ - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - memset(dctx, 0, sizeof(*dctx)); - - return 0; -} - -static int ghash_setkey(struct crypto_shash *tfm, - const u8 *key, unsigned int keylen) -{ - struct ghash_ctx *ctx = crypto_shash_ctx(tfm); - be128 k; - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - if (ctx->gf128) - gf128mul_free_4k(ctx->gf128); - - BUILD_BUG_ON(sizeof(k) != GHASH_BLOCK_SIZE); - memcpy(&k, key, GHASH_BLOCK_SIZE); /* avoid violating alignment rules */ - ctx->gf128 = gf128mul_init_4k_lle(&k); - memzero_explicit(&k, GHASH_BLOCK_SIZE); - - if (!ctx->gf128) - return -ENOMEM; - - return 0; -} - -static int ghash_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); - u8 *dst = dctx->buffer; - - do { - crypto_xor(dst, src, GHASH_BLOCK_SIZE); - gf128mul_4k_lle((be128 *)dst, ctx->gf128); - src += GHASH_BLOCK_SIZE; - srclen -= GHASH_BLOCK_SIZE; - } while (srclen >= GHASH_BLOCK_SIZE); - - return srclen; -} - -static void ghash_flush(struct shash_desc *desc, const u8 *src, - unsigned int len) -{ - struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - u8 *dst = dctx->buffer; - - if (len) { - crypto_xor(dst, src, len); - gf128mul_4k_lle((be128 *)dst, ctx->gf128); - } -} - -static int ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *dst) -{ - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - u8 *buf = dctx->buffer; - - ghash_flush(desc, src, len); - memcpy(dst, buf, GHASH_BLOCK_SIZE); - - return 0; -} - -static void ghash_exit_tfm(struct crypto_tfm *tfm) -{ - struct ghash_ctx *ctx = crypto_tfm_ctx(tfm); - if (ctx->gf128) - gf128mul_free_4k(ctx->gf128); -} - -static struct shash_alg ghash_alg = { - .digestsize = GHASH_DIGEST_SIZE, - .init = ghash_init, - .update = ghash_update, - .finup = ghash_finup, - .setkey = ghash_setkey, - .descsize = sizeof(struct ghash_desc_ctx), - .base = { - .cra_name = "ghash", - .cra_driver_name = "ghash-generic", - .cra_priority = 100, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct ghash_ctx), - .cra_module = THIS_MODULE, - .cra_exit = ghash_exit_tfm, - }, -}; - -static int __init ghash_mod_init(void) -{ - return crypto_register_shash(&ghash_alg); -} - -static void __exit ghash_mod_exit(void) -{ - crypto_unregister_shash(&ghash_alg); -} - -module_init(ghash_mod_init); -module_exit(ghash_mod_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("GHASH hash function"); -MODULE_ALIAS_CRYPTO("ghash"); -MODULE_ALIAS_CRYPTO("ghash-generic"); diff --git a/crypto/hctr2.c b/crypto/hctr2.c index f4cd6c29b4d3..ad5edf9366ac 100644 --- a/crypto/hctr2.c +++ b/crypto/hctr2.c @@ -16,9 +16,9 @@ * (https://eprint.iacr.org/2021/1441.pdf) */ +#include <crypto/gf128hash.h> #include <crypto/internal/cipher.h> #include <crypto/internal/skcipher.h> -#include <crypto/polyval.h> #include <crypto/scatterwalk.h> #include <linux/module.h> diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c index 7c880cf34c52..4ad729357441 100644 --- a/crypto/jitterentropy-kcapi.c +++ b/crypto/jitterentropy-kcapi.c @@ -37,7 +37,6 @@ * DAMAGE. */ -#include <crypto/hash.h> #include <crypto/sha3.h> #include <linux/fips.h> #include <linux/kernel.h> @@ -48,8 +47,6 @@ #include "jitterentropy.h" -#define JENT_CONDITIONING_HASH "sha3-256" - /*************************************************************************** * Helper function ***************************************************************************/ @@ -101,22 +98,14 @@ void jent_get_nstime(__u64 *out) jent_raw_hires_entropy_store(tmp); } -int jent_hash_time(void *hash_state, __u64 time, u8 *addtl, - unsigned int addtl_len, __u64 hash_loop_cnt, - unsigned int stuck) +void jent_hash_time(struct sha3_ctx *hash_state, __u64 time, u8 *addtl, + unsigned int addtl_len, __u64 hash_loop_cnt, + unsigned int stuck) { - struct shash_desc *hash_state_desc = (struct shash_desc *)hash_state; - SHASH_DESC_ON_STACK(desc, hash_state_desc->tfm); + struct sha3_ctx tmp_state; /* zeroized by sha3_final() */ u8 intermediary[SHA3_256_DIGEST_SIZE]; __u64 j = 0; - int ret; - - desc->tfm = hash_state_desc->tfm; - if (sizeof(intermediary) != crypto_shash_digestsize(desc->tfm)) { - pr_warn_ratelimited("Unexpected digest size\n"); - return -EINVAL; - } kmsan_unpoison_memory(intermediary, sizeof(intermediary)); /* @@ -130,24 +119,20 @@ int jent_hash_time(void *hash_state, __u64 time, u8 *addtl, * * Note, it does not matter which or how much data you inject, we are * interested in one Keccack1600 compression operation performed with - * the crypto_shash_final. + * the sha3_final. */ for (j = 0; j < hash_loop_cnt; j++) { - ret = crypto_shash_init(desc) ?: - crypto_shash_update(desc, intermediary, - sizeof(intermediary)) ?: - crypto_shash_finup(desc, addtl, addtl_len, intermediary); - if (ret) - goto err; + sha3_256_init(&tmp_state); + sha3_update(&tmp_state, intermediary, sizeof(intermediary)); + sha3_update(&tmp_state, addtl, addtl_len); + sha3_final(&tmp_state, intermediary); } /* * Inject the data from the previous loop into the pool. This data is * not considered to contain any entropy, but it stirs the pool a bit. */ - ret = crypto_shash_update(hash_state_desc, intermediary, sizeof(intermediary)); - if (ret) - goto err; + sha3_update(hash_state, intermediary, sizeof(intermediary)); /* * Insert the time stamp into the hash context representing the pool. @@ -162,30 +147,24 @@ int jent_hash_time(void *hash_state, __u64 time, u8 *addtl, time = 0; } - ret = crypto_shash_update(hash_state_desc, (u8 *)&time, sizeof(__u64)); - -err: - shash_desc_zero(desc); + sha3_update(hash_state, (u8 *)&time, sizeof(__u64)); memzero_explicit(intermediary, sizeof(intermediary)); - - return ret; } -int jent_read_random_block(void *hash_state, char *dst, unsigned int dst_len) +void jent_read_random_block(struct sha3_ctx *hash_state, char *dst, + unsigned int dst_len) { - struct shash_desc *hash_state_desc = (struct shash_desc *)hash_state; u8 jent_block[SHA3_256_DIGEST_SIZE]; + /* Obtain data from entropy pool and re-initialize it */ - int ret = crypto_shash_final(hash_state_desc, jent_block) ?: - crypto_shash_init(hash_state_desc) ?: - crypto_shash_update(hash_state_desc, jent_block, - sizeof(jent_block)); + sha3_final(hash_state, jent_block); + sha3_256_init(hash_state); + sha3_update(hash_state, jent_block, sizeof(jent_block)); - if (!ret && dst_len) + if (dst_len) memcpy(dst, jent_block, dst_len); memzero_explicit(jent_block, sizeof(jent_block)); - return ret; } /*************************************************************************** @@ -195,8 +174,7 @@ int jent_read_random_block(void *hash_state, char *dst, unsigned int dst_len) struct jitterentropy { spinlock_t jent_lock; struct rand_data *entropy_collector; - struct crypto_shash *tfm; - struct shash_desc *sdesc; + struct sha3_ctx hash_state; }; static void jent_kcapi_cleanup(struct crypto_tfm *tfm) @@ -205,15 +183,7 @@ static void jent_kcapi_cleanup(struct crypto_tfm *tfm) spin_lock(&rng->jent_lock); - if (rng->sdesc) { - shash_desc_zero(rng->sdesc); - kfree(rng->sdesc); - } - rng->sdesc = NULL; - - if (rng->tfm) - crypto_free_shash(rng->tfm); - rng->tfm = NULL; + memzero_explicit(&rng->hash_state, sizeof(rng->hash_state)); if (rng->entropy_collector) jent_entropy_collector_free(rng->entropy_collector); @@ -224,34 +194,15 @@ static void jent_kcapi_cleanup(struct crypto_tfm *tfm) static int jent_kcapi_init(struct crypto_tfm *tfm) { struct jitterentropy *rng = crypto_tfm_ctx(tfm); - struct crypto_shash *hash; - struct shash_desc *sdesc; - int size, ret = 0; + int ret = 0; spin_lock_init(&rng->jent_lock); /* Use SHA3-256 as conditioner */ - hash = crypto_alloc_shash(JENT_CONDITIONING_HASH, 0, 0); - if (IS_ERR(hash)) { - pr_err("Cannot allocate conditioning digest\n"); - return PTR_ERR(hash); - } - rng->tfm = hash; - - size = sizeof(struct shash_desc) + crypto_shash_descsize(hash); - sdesc = kmalloc(size, GFP_KERNEL); - if (!sdesc) { - ret = -ENOMEM; - goto err; - } - - sdesc->tfm = hash; - crypto_shash_init(sdesc); - rng->sdesc = sdesc; + sha3_256_init(&rng->hash_state); - rng->entropy_collector = - jent_entropy_collector_alloc(CONFIG_CRYPTO_JITTERENTROPY_OSR, 0, - sdesc); + rng->entropy_collector = jent_entropy_collector_alloc( + CONFIG_CRYPTO_JITTERENTROPY_OSR, 0, &rng->hash_state); if (!rng->entropy_collector) { ret = -ENOMEM; goto err; @@ -326,23 +277,16 @@ static struct rng_alg jent_alg = { static int __init jent_mod_init(void) { - SHASH_DESC_ON_STACK(desc, tfm); - struct crypto_shash *tfm; + struct sha3_ctx hash_state; int ret = 0; jent_testing_init(); - tfm = crypto_alloc_shash(JENT_CONDITIONING_HASH, 0, 0); - if (IS_ERR(tfm)) { - jent_testing_exit(); - return PTR_ERR(tfm); - } + sha3_256_init(&hash_state); - desc->tfm = tfm; - crypto_shash_init(desc); - ret = jent_entropy_init(CONFIG_CRYPTO_JITTERENTROPY_OSR, 0, desc, NULL); - shash_desc_zero(desc); - crypto_free_shash(tfm); + ret = jent_entropy_init(CONFIG_CRYPTO_JITTERENTROPY_OSR, 0, &hash_state, + NULL); + memzero_explicit(&hash_state, sizeof(hash_state)); if (ret) { /* Handle permanent health test error */ if (fips_enabled) diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c index 3f93cdc9a7af..d5832caa8ab3 100644 --- a/crypto/jitterentropy.c +++ b/crypto/jitterentropy.c @@ -68,7 +68,7 @@ struct rand_data { * of the RNG are marked as SENSITIVE. A user must not * access that information while the RNG executes its loops to * calculate the next random value. */ - void *hash_state; /* SENSITIVE hash state entropy pool */ + struct sha3_ctx *hash_state; /* SENSITIVE hash state entropy pool */ __u64 prev_time; /* SENSITIVE Previous time stamp */ __u64 last_delta; /* SENSITIVE stuck test */ __s64 last_delta2; /* SENSITIVE stuck test */ @@ -417,10 +417,9 @@ static __u64 jent_loop_shuffle(unsigned int bits, unsigned int min) * time [in] time stamp to be injected * stuck [in] Is the time stamp identified as stuck? * - * Output: - * updated hash context in the entropy collector or error code + * Output: updated hash context in the entropy collector */ -static int jent_condition_data(struct rand_data *ec, __u64 time, int stuck) +static void jent_condition_data(struct rand_data *ec, __u64 time, int stuck) { #define SHA3_HASH_LOOP (1<<3) struct { @@ -435,8 +434,8 @@ static int jent_condition_data(struct rand_data *ec, __u64 time, int stuck) ec->apt_base }; - return jent_hash_time(ec->hash_state, time, (u8 *)&addtl, sizeof(addtl), - SHA3_HASH_LOOP, stuck); + jent_hash_time(ec->hash_state, time, (u8 *)&addtl, sizeof(addtl), + SHA3_HASH_LOOP, stuck); } /* @@ -538,8 +537,7 @@ static int jent_measure_jitter(struct rand_data *ec, __u64 *ret_current_delta) stuck = jent_stuck(ec, current_delta); /* Now call the next noise sources which also injects the data */ - if (jent_condition_data(ec, current_delta, stuck)) - stuck = 1; + jent_condition_data(ec, current_delta, stuck); /* return the raw entropy value */ if (ret_current_delta) @@ -597,7 +595,7 @@ static void jent_gen_entropy(struct rand_data *ec) * @return 0 when request is fulfilled or an error * * The following error codes can occur: - * -1 entropy_collector is NULL or the generation failed + * -1 entropy_collector is NULL * -2 Intermittent health failure * -3 Permanent health failure */ @@ -640,8 +638,7 @@ int jent_read_entropy(struct rand_data *ec, unsigned char *data, } tocopy = min(DATA_SIZE_BITS / 8, len); - if (jent_read_random_block(ec->hash_state, p, tocopy)) - return -1; + jent_read_random_block(ec->hash_state, p, tocopy); len -= tocopy; p += tocopy; @@ -656,7 +653,7 @@ int jent_read_entropy(struct rand_data *ec, unsigned char *data, struct rand_data *jent_entropy_collector_alloc(unsigned int osr, unsigned int flags, - void *hash_state) + struct sha3_ctx *hash_state) { struct rand_data *entropy_collector; @@ -704,8 +701,8 @@ void jent_entropy_collector_free(struct rand_data *entropy_collector) jent_zfree(entropy_collector); } -int jent_entropy_init(unsigned int osr, unsigned int flags, void *hash_state, - struct rand_data *p_ec) +int jent_entropy_init(unsigned int osr, unsigned int flags, + struct sha3_ctx *hash_state, struct rand_data *p_ec) { /* * If caller provides an allocated ec, reuse it which implies that the diff --git a/crypto/jitterentropy.h b/crypto/jitterentropy.h index 4c5dbf2a8d8f..5bb15cb33000 100644 --- a/crypto/jitterentropy.h +++ b/crypto/jitterentropy.h @@ -1,24 +1,27 @@ // SPDX-License-Identifier: GPL-2.0-or-later +struct sha3_ctx; extern void *jent_kvzalloc(unsigned int len); extern void jent_kvzfree(void *ptr, unsigned int len); extern void *jent_zalloc(unsigned int len); extern void jent_zfree(void *ptr); extern void jent_get_nstime(__u64 *out); -extern int jent_hash_time(void *hash_state, __u64 time, u8 *addtl, - unsigned int addtl_len, __u64 hash_loop_cnt, - unsigned int stuck); -int jent_read_random_block(void *hash_state, char *dst, unsigned int dst_len); +void jent_hash_time(struct sha3_ctx *hash_state, __u64 time, u8 *addtl, + unsigned int addtl_len, __u64 hash_loop_cnt, + unsigned int stuck); +void jent_read_random_block(struct sha3_ctx *hash_state, char *dst, + unsigned int dst_len); struct rand_data; extern int jent_entropy_init(unsigned int osr, unsigned int flags, - void *hash_state, struct rand_data *p_ec); + struct sha3_ctx *hash_state, + struct rand_data *p_ec); extern int jent_read_entropy(struct rand_data *ec, unsigned char *data, unsigned int len); -extern struct rand_data *jent_entropy_collector_alloc(unsigned int osr, - unsigned int flags, - void *hash_state); +extern struct rand_data * +jent_entropy_collector_alloc(unsigned int osr, unsigned int flags, + struct sha3_ctx *hash_state); extern void jent_entropy_collector_free(struct rand_data *entropy_collector); #ifdef CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE diff --git a/crypto/sm3.c b/crypto/sm3.c new file mode 100644 index 000000000000..05111a99b851 --- /dev/null +++ b/crypto/sm3.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * SM3 secure hash, as specified by OSCCA GM/T 0004-2012 SM3 and + * described at https://tools.ietf.org/html/draft-shen-sm3-hash-01 + * + * Copyright (C) 2017 ARM Limited or its affiliates. + * Written by Gilad Ben-Yossef <gilad@benyossef.com> + * Copyright (C) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> + * Copyright 2026 Google LLC + */ + +#include <crypto/internal/hash.h> +#include <crypto/sm3.h> +#include <linux/kernel.h> +#include <linux/module.h> + +#define SM3_CTX(desc) ((struct sm3_ctx *)shash_desc_ctx(desc)) + +static int crypto_sm3_init(struct shash_desc *desc) +{ + sm3_init(SM3_CTX(desc)); + return 0; +} + +static int crypto_sm3_update(struct shash_desc *desc, + const u8 *data, unsigned int len) +{ + sm3_update(SM3_CTX(desc), data, len); + return 0; +} + +static int crypto_sm3_final(struct shash_desc *desc, u8 *out) +{ + sm3_final(SM3_CTX(desc), out); + return 0; +} + +static int crypto_sm3_digest(struct shash_desc *desc, + const u8 *data, unsigned int len, u8 *out) +{ + sm3(data, len, out); + return 0; +} + +static int crypto_sm3_export_core(struct shash_desc *desc, void *out) +{ + memcpy(out, SM3_CTX(desc), sizeof(struct sm3_ctx)); + return 0; +} + +static int crypto_sm3_import_core(struct shash_desc *desc, const void *in) +{ + memcpy(SM3_CTX(desc), in, sizeof(struct sm3_ctx)); + return 0; +} + +static struct shash_alg sm3_alg = { + .base.cra_name = "sm3", + .base.cra_driver_name = "sm3-lib", + .base.cra_priority = 300, + .base.cra_blocksize = SM3_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .digestsize = SM3_DIGEST_SIZE, + .init = crypto_sm3_init, + .update = crypto_sm3_update, + .final = crypto_sm3_final, + .digest = crypto_sm3_digest, + .export_core = crypto_sm3_export_core, + .import_core = crypto_sm3_import_core, + .descsize = sizeof(struct sm3_ctx), +}; + +static int __init crypto_sm3_mod_init(void) +{ + return crypto_register_shash(&sm3_alg); +} +module_init(crypto_sm3_mod_init); + +static void __exit crypto_sm3_mod_exit(void) +{ + crypto_unregister_shash(&sm3_alg); +} +module_exit(crypto_sm3_mod_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Crypto API support for SM3"); + +MODULE_ALIAS_CRYPTO("sm3"); +MODULE_ALIAS_CRYPTO("sm3-lib"); diff --git a/crypto/sm3_generic.c b/crypto/sm3_generic.c deleted file mode 100644 index 7529139fcc96..000000000000 --- a/crypto/sm3_generic.c +++ /dev/null @@ -1,72 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * SM3 secure hash, as specified by OSCCA GM/T 0004-2012 SM3 and - * described at https://tools.ietf.org/html/draft-shen-sm3-hash-01 - * - * Copyright (C) 2017 ARM Limited or its affiliates. - * Written by Gilad Ben-Yossef <gilad@benyossef.com> - * Copyright (C) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> - */ - -#include <crypto/internal/hash.h> -#include <crypto/sm3.h> -#include <crypto/sm3_base.h> -#include <linux/kernel.h> -#include <linux/module.h> - -const u8 sm3_zero_message_hash[SM3_DIGEST_SIZE] = { - 0x1A, 0xB2, 0x1D, 0x83, 0x55, 0xCF, 0xA1, 0x7F, - 0x8e, 0x61, 0x19, 0x48, 0x31, 0xE8, 0x1A, 0x8F, - 0x22, 0xBE, 0xC8, 0xC7, 0x28, 0xFE, 0xFB, 0x74, - 0x7E, 0xD0, 0x35, 0xEB, 0x50, 0x82, 0xAA, 0x2B -}; -EXPORT_SYMBOL_GPL(sm3_zero_message_hash); - -static int crypto_sm3_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sm3_base_do_update_blocks(desc, data, len, sm3_block_generic); -} - -static int crypto_sm3_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash) -{ - sm3_base_do_finup(desc, data, len, sm3_block_generic); - return sm3_base_finish(desc, hash); -} - -static struct shash_alg sm3_alg = { - .digestsize = SM3_DIGEST_SIZE, - .init = sm3_base_init, - .update = crypto_sm3_update, - .finup = crypto_sm3_finup, - .descsize = SM3_STATE_SIZE, - .base = { - .cra_name = "sm3", - .cra_driver_name = "sm3-generic", - .cra_priority = 100, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SM3_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init sm3_generic_mod_init(void) -{ - return crypto_register_shash(&sm3_alg); -} - -static void __exit sm3_generic_mod_fini(void) -{ - crypto_unregister_shash(&sm3_alg); -} - -module_init(sm3_generic_mod_init); -module_exit(sm3_generic_mod_fini); - -MODULE_LICENSE("GPL v2"); -MODULE_DESCRIPTION("SM3 Secure Hash Algorithm"); - -MODULE_ALIAS_CRYPTO("sm3"); -MODULE_ALIAS_CRYPTO("sm3-generic"); diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index aded37546137..1773f5f71351 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -1650,10 +1650,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) ret = min(ret, tcrypt_test("rfc4309(ccm(aes))")); break; - case 46: - ret = min(ret, tcrypt_test("ghash")); - break; - case 48: ret = min(ret, tcrypt_test("sha3-224")); break; @@ -2251,11 +2247,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) test_hash_speed("blake2b-512", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; fallthrough; - case 318: - klen = 16; - test_hash_speed("ghash", sec, generic_hash_speed_template); - if (mode > 300 && mode < 400) break; - fallthrough; case 319: test_hash_speed("crc32c", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 4985411dedae..60b6e4379aa6 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4388,7 +4388,7 @@ static const struct alg_test_desc alg_test_descs[] = { }, { #endif .alg = "cbcmac(aes)", - .generic_driver = "cbcmac(aes-lib)", + .generic_driver = "cbcmac-aes-lib", .test = alg_test_hash, .suite = { .hash = __VECS(aes_cbcmac_tv_template) @@ -4401,7 +4401,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "ccm(aes)", - .generic_driver = "ccm_base(ctr(aes-lib),cbcmac(aes-lib))", + .generic_driver = "ccm_base(ctr(aes-lib),cbcmac-aes-lib)", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -4429,7 +4429,7 @@ static const struct alg_test_desc alg_test_descs[] = { }, }, { .alg = "cmac(aes)", - .generic_driver = "cmac(aes-lib)", + .generic_driver = "cmac-aes-lib", .fips_allowed = 1, .test = alg_test_hash, .suite = { @@ -4965,7 +4965,7 @@ static const struct alg_test_desc alg_test_descs[] = { }, { #endif /* CONFIG_CRYPTO_DH_RFC7919_GROUPS */ .alg = "gcm(aes)", - .generic_driver = "gcm_base(ctr(aes-lib),ghash-generic)", + .generic_driver = "gcm_base(ctr(aes-lib),ghash-lib)", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -4973,25 +4973,19 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "gcm(aria)", - .generic_driver = "gcm_base(ctr(aria-generic),ghash-generic)", + .generic_driver = "gcm_base(ctr(aria-generic),ghash-lib)", .test = alg_test_aead, .suite = { .aead = __VECS(aria_gcm_tv_template) } }, { .alg = "gcm(sm4)", - .generic_driver = "gcm_base(ctr(sm4-generic),ghash-generic)", + .generic_driver = "gcm_base(ctr(sm4-generic),ghash-lib)", .test = alg_test_aead, .suite = { .aead = __VECS(sm4_gcm_tv_template) } }, { - .alg = "ghash", - .test = alg_test_hash, - .suite = { - .hash = __VECS(ghash_tv_template) - } - }, { .alg = "hctr2(aes)", .generic_driver = "hctr2_base(xctr(aes-lib),polyval-lib)", .test = alg_test_skcipher, @@ -5085,6 +5079,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "hmac(sm3)", + .generic_driver = "hmac(sm3-lib)", .test = alg_test_hash, .suite = { .hash = __VECS(hmac_sm3_tv_template) @@ -5314,7 +5309,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "rfc4106(gcm(aes))", - .generic_driver = "rfc4106(gcm_base(ctr(aes-lib),ghash-generic))", + .generic_driver = "rfc4106(gcm_base(ctr(aes-lib),ghash-lib))", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -5326,7 +5321,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "rfc4309(ccm(aes))", - .generic_driver = "rfc4309(ccm_base(ctr(aes-lib),cbcmac(aes-lib)))", + .generic_driver = "rfc4309(ccm_base(ctr(aes-lib),cbcmac-aes-lib))", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -5338,7 +5333,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "rfc4543(gcm(aes))", - .generic_driver = "rfc4543(gcm_base(ctr(aes-lib),ghash-generic))", + .generic_driver = "rfc4543(gcm_base(ctr(aes-lib),ghash-lib))", .test = alg_test_aead, .suite = { .aead = { @@ -5452,6 +5447,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "sm3", + .generic_driver = "sm3-lib", .test = alg_test_hash, .suite = { .hash = __VECS(sm3_tv_template) @@ -5515,7 +5511,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "xcbc(aes)", - .generic_driver = "xcbc(aes-lib)", + .generic_driver = "xcbc-aes-lib", .test = alg_test_hash, .suite = { .hash = __VECS(aes_xcbc128_tv_template) diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 1c69c11c0cdb..a3274abacfde 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -6183,115 +6183,6 @@ static const struct hash_testvec wp256_tv_template[] = { }, }; -static const struct hash_testvec ghash_tv_template[] = -{ - { - .key = "\xdf\xa6\xbf\x4d\xed\x81\xdb\x03" - "\xff\xca\xff\x95\xf8\x30\xf0\x61", - .ksize = 16, - .plaintext = "\x95\x2b\x2a\x56\xa5\x60\x04a\xc0" - "\xb3\x2b\x66\x56\xa0\x5b\x40\xb6", - .psize = 16, - .digest = "\xda\x53\xeb\x0a\xd2\xc5\x5b\xb6" - "\x4f\xc4\x80\x2c\xc3\xfe\xda\x60", - }, { - .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" - "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", - .ksize = 16, - .plaintext = "what do ya want for nothing?", - .psize = 28, - .digest = "\x3e\x1f\x5c\x4d\x65\xf0\xef\xce" - "\x0d\x61\x06\x27\x66\x51\xd5\xe2", - }, { - .key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", - .ksize = 16, - .plaintext = "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" - "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" - "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" - "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd", - .psize = 50, - .digest = "\xfb\x49\x8a\x36\xe1\x96\xe1\x96" - "\xe1\x96\xe1\x96\xe1\x96\xe1\x96", - }, { - .key = "\xda\x53\xeb\x0a\xd2\xc5\x5b\xb6" - "\x4f\xc4\x80\x2c\xc3\xfe\xda\x60", - .ksize = 16, - .plaintext = "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" - "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" - "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" - "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd", - .psize = 50, - .digest = "\x2b\x5c\x0c\x7f\x52\xd1\x60\xc2" - "\x49\xed\x6e\x32\x7a\xa9\xbe\x08", - }, { - .key = "\x95\x2b\x2a\x56\xa5\x60\x04a\xc0" - "\xb3\x2b\x66\x56\xa0\x5b\x40\xb6", - .ksize = 16, - .plaintext = "Test With Truncation", - .psize = 20, - .digest = "\xf8\x94\x87\x2a\x4b\x63\x99\x28" - "\x23\xf7\x93\xf7\x19\xf5\x96\xd9", - }, { - .key = "\x0a\x1b\x2c\x3d\x4e\x5f\x64\x71" - "\x82\x93\xa4\xb5\xc6\xd7\xe8\xf9", - .ksize = 16, - .plaintext = "\x56\x6f\x72\x20\x6c\x61\x75\x74" - "\x65\x72\x20\x4c\x61\x75\x73\x63" - "\x68\x65\x6e\x20\x75\x6e\x64\x20" - "\x53\x74\x61\x75\x6e\x65\x6e\x20" - "\x73\x65\x69\x20\x73\x74\x69\x6c" - "\x6c\x2c\x0a\x64\x75\x20\x6d\x65" - "\x69\x6e\x20\x74\x69\x65\x66\x74" - "\x69\x65\x66\x65\x73\x20\x4c\x65" - "\x62\x65\x6e\x3b\x0a\x64\x61\x73" - "\x73\x20\x64\x75\x20\x77\x65\x69" - "\xc3\x9f\x74\x20\x77\x61\x73\x20" - "\x64\x65\x72\x20\x57\x69\x6e\x64" - "\x20\x64\x69\x72\x20\x77\x69\x6c" - "\x6c\x2c\x0a\x65\x68\x20\x6e\x6f" - "\x63\x68\x20\x64\x69\x65\x20\x42" - "\x69\x72\x6b\x65\x6e\x20\x62\x65" - "\x62\x65\x6e\x2e\x0a\x0a\x55\x6e" - "\x64\x20\x77\x65\x6e\x6e\x20\x64" - "\x69\x72\x20\x65\x69\x6e\x6d\x61" - "\x6c\x20\x64\x61\x73\x20\x53\x63" - "\x68\x77\x65\x69\x67\x65\x6e\x20" - "\x73\x70\x72\x61\x63\x68\x2c\x0a" - "\x6c\x61\x73\x73\x20\x64\x65\x69" - "\x6e\x65\x20\x53\x69\x6e\x6e\x65" - "\x20\x62\x65\x73\x69\x65\x67\x65" - "\x6e\x2e\x0a\x4a\x65\x64\x65\x6d" - "\x20\x48\x61\x75\x63\x68\x65\x20" - "\x67\x69\x62\x74\x20\x64\x69\x63" - "\x68\x2c\x20\x67\x69\x62\x20\x6e" - "\x61\x63\x68\x2c\x0a\x65\x72\x20" - "\x77\x69\x72\x64\x20\x64\x69\x63" - "\x68\x20\x6c\x69\x65\x62\x65\x6e" - "\x20\x75\x6e\x64\x20\x77\x69\x65" - "\x67\x65\x6e\x2e\x0a\x0a\x55\x6e" - "\x64\x20\x64\x61\x6e\x6e\x20\x6d" - "\x65\x69\x6e\x65\x20\x53\x65\x65" - "\x6c\x65\x20\x73\x65\x69\x74\x20" - "\x77\x65\x69\x74\x2c\x20\x73\x65" - "\x69\x20\x77\x65\x69\x74\x2c\x0a" - "\x64\x61\x73\x73\x20\x64\x69\x72" - "\x20\x64\x61\x73\x20\x4c\x65\x62" - "\x65\x6e\x20\x67\x65\x6c\x69\x6e" - "\x67\x65\x2c\x0a\x62\x72\x65\x69" - "\x74\x65\x20\x64\x69\x63\x68\x20" - "\x77\x69\x65\x20\x65\x69\x6e\x20" - "\x46\x65\x69\x65\x72\x6b\x6c\x65" - "\x69\x64\x0a\xc3\xbc\x62\x65\x72" - "\x20\x64\x69\x65\x20\x73\x69\x6e" - "\x6e\x65\x6e\x64\x65\x6e\x20\x44" - "\x69\x6e\x67\x65\x2e\x2e\x2e\x0a", - .psize = 400, - .digest = "\xad\xb1\xc1\xe9\x56\x70\x31\x1d" - "\xbb\x5b\xdf\x5e\x70\x72\x1a\x57", - }, -}; - /* * HMAC-MD5 test vectors from RFC2202 * (These need to be fixed to not use strlen). diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 8d3b5d2890f8..9960100e6066 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -843,7 +843,7 @@ config CRYPTO_DEV_CCREE select CRYPTO_CTR select CRYPTO_XTS select CRYPTO_SM4_GENERIC - select CRYPTO_SM3_GENERIC + select CRYPTO_SM3 help Say 'Y' to enable a driver for the REE interface of the Arm TrustZone CryptoCell family of processors. Currently the diff --git a/drivers/crypto/starfive/Kconfig b/drivers/crypto/starfive/Kconfig index 0fe389e9f932..11518ca3eea1 100644 --- a/drivers/crypto/starfive/Kconfig +++ b/drivers/crypto/starfive/Kconfig @@ -10,7 +10,7 @@ config CRYPTO_DEV_JH7110 select CRYPTO_HMAC select CRYPTO_SHA256 select CRYPTO_SHA512 - select CRYPTO_SM3_GENERIC + select CRYPTO_SM3 select CRYPTO_RSA select CRYPTO_AES select CRYPTO_CCM diff --git a/drivers/crypto/starfive/jh7110-aes.c b/drivers/crypto/starfive/jh7110-aes.c index c1dc1e43e117..a0713aa21250 100644 --- a/drivers/crypto/starfive/jh7110-aes.c +++ b/drivers/crypto/starfive/jh7110-aes.c @@ -1003,12 +1003,12 @@ static int starfive_aes_ctr_init_tfm(struct crypto_skcipher *tfm) static int starfive_aes_ccm_init_tfm(struct crypto_aead *tfm) { - return starfive_aes_aead_init_tfm(tfm, "ccm_base(ctr(aes-lib),cbcmac(aes-lib))"); + return starfive_aes_aead_init_tfm(tfm, "ccm_base(ctr(aes-lib),cbcmac-aes-lib)"); } static int starfive_aes_gcm_init_tfm(struct crypto_aead *tfm) { - return starfive_aes_aead_init_tfm(tfm, "gcm_base(ctr(aes-lib),ghash-generic)"); + return starfive_aes_aead_init_tfm(tfm, "gcm_base(ctr(aes-lib),ghash-lib)"); } static struct skcipher_engine_alg skcipher_algs[] = { diff --git a/drivers/crypto/starfive/jh7110-hash.c b/drivers/crypto/starfive/jh7110-hash.c index 54b7af4a7aee..008a47baa165 100644 --- a/drivers/crypto/starfive/jh7110-hash.c +++ b/drivers/crypto/starfive/jh7110-hash.c @@ -520,7 +520,7 @@ static int starfive_sha512_init_tfm(struct crypto_ahash *hash) static int starfive_sm3_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "sm3-generic", + return starfive_hash_init_tfm(hash, "sm3-lib", STARFIVE_HASH_SM3, 0); } @@ -550,7 +550,7 @@ static int starfive_hmac_sha512_init_tfm(struct crypto_ahash *hash) static int starfive_hmac_sm3_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "hmac(sm3-generic)", + return starfive_hash_init_tfm(hash, "hmac(sm3-lib)", STARFIVE_HASH_SM3, 1); } @@ -795,7 +795,7 @@ static struct ahash_engine_alg algs_sha2_sm3[] = { .base.exit_tfm = starfive_hash_exit_tfm, .base.halg = { .digestsize = SM3_DIGEST_SIZE, - .statesize = sizeof(struct sm3_state), + .statesize = sizeof(struct sm3_ctx), .base = { .cra_name = "sm3", .cra_driver_name = "sm3-starfive", @@ -824,7 +824,7 @@ static struct ahash_engine_alg algs_sha2_sm3[] = { .base.setkey = starfive_hash_setkey, .base.halg = { .digestsize = SM3_DIGEST_SIZE, - .statesize = sizeof(struct sm3_state), + .statesize = sizeof(struct sm3_ctx), .base = { .cra_name = "hmac(sm3)", .cra_driver_name = "sm3-hmac-starfive", diff --git a/drivers/virt/coco/guest/Kconfig b/drivers/virt/coco/guest/Kconfig index 3d5e1d05bf34..da570dc4bd48 100644 --- a/drivers/virt/coco/guest/Kconfig +++ b/drivers/virt/coco/guest/Kconfig @@ -13,5 +13,4 @@ config TSM_REPORTS config TSM_MEASUREMENTS select TSM_GUEST select CRYPTO_HASH_INFO - select CRYPTO bool diff --git a/include/crypto/aes-cbc-macs.h b/include/crypto/aes-cbc-macs.h new file mode 100644 index 000000000000..e61df108b926 --- /dev/null +++ b/include/crypto/aes-cbc-macs.h @@ -0,0 +1,154 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Support for AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_CBC_MACS_H +#define _CRYPTO_AES_CBC_MACS_H + +#include <crypto/aes.h> + +/** + * struct aes_cmac_key - Prepared key for AES-CMAC or AES-XCBC-MAC + * @aes: The AES key for cipher block chaining + * @k_final: Finalization subkeys for the final block. + * k_final[0] (CMAC K1, XCBC-MAC K2) is used if it's a full block. + * k_final[1] (CMAC K2, XCBC-MAC K3) is used if it's a partial block. + */ +struct aes_cmac_key { + struct aes_enckey aes; + union { + u8 b[AES_BLOCK_SIZE]; + __be64 w[2]; + } k_final[2]; +}; + +/** + * struct aes_cmac_ctx - Context for computing an AES-CMAC or AES-XCBC-MAC value + * @key: Pointer to the key struct. A pointer is used rather than a copy of the + * struct, since the key struct size may be large. It is assumed that the + * key lives at least as long as the context. + * @partial_len: Number of bytes that have been XOR'ed into @h since the last + * AES encryption. This is 0 if no data has been processed yet, + * or between 1 and AES_BLOCK_SIZE inclusive otherwise. + * @h: The current chaining value + */ +struct aes_cmac_ctx { + const struct aes_cmac_key *key; + size_t partial_len; + u8 h[AES_BLOCK_SIZE]; +}; + +/** + * aes_cmac_preparekey() - Prepare a key for AES-CMAC + * @key: (output) The key struct to initialize + * @in_key: The raw AES key + * @key_len: Length of the raw key in bytes. The supported values are + * AES_KEYSIZE_128, AES_KEYSIZE_192, and AES_KEYSIZE_256. + * + * Context: Any context. + * Return: 0 on success or -EINVAL if the given key length is invalid. No other + * errors are possible, so callers that always pass a valid key length + * don't need to check for errors. + */ +int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key, + size_t key_len); + +/** + * aes_xcbcmac_preparekey() - Prepare a key for AES-XCBC-MAC + * @key: (output) The key struct to initialize + * @in_key: The raw key. As per the AES-XCBC-MAC specification (RFC 3566), this + * is 128 bits, matching the internal use of AES-128. + * + * AES-XCBC-MAC and AES-CMAC are the same except for the key preparation. After + * that step, AES-XCBC-MAC is supported via the aes_cmac_* functions. + * + * New users should use AES-CMAC instead of AES-XCBC-MAC. + * + * Context: Any context. + */ +void aes_xcbcmac_preparekey(struct aes_cmac_key *key, + const u8 in_key[at_least AES_KEYSIZE_128]); + +/** + * aes_cmac_init() - Start computing an AES-CMAC or AES-XCBC-MAC value + * @ctx: (output) The context to initialize + * @key: The key to use. Note that a pointer to the key is saved in the + * context, so the key must live at least as long as the context. + * + * This supports both AES-CMAC and AES-XCBC-MAC. Which one is done depends on + * whether aes_cmac_preparekey() or aes_xcbcmac_preparekey() was called. + */ +static inline void aes_cmac_init(struct aes_cmac_ctx *ctx, + const struct aes_cmac_key *key) +{ + *ctx = (struct aes_cmac_ctx){ .key = key }; +} + +/** + * aes_cmac_update() - Update an AES-CMAC or AES-XCBC-MAC context with more data + * @ctx: The context to update; must have been initialized + * @data: The message data + * @data_len: The data length in bytes. Doesn't need to be block-aligned. + * + * This can be called any number of times. + * + * Context: Any context. + */ +void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len); + +/** + * aes_cmac_final() - Finish computing an AES-CMAC or AES-XCBC-MAC value + * @ctx: The context to finalize; must have been initialized + * @out: (output) The resulting MAC + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[at_least AES_BLOCK_SIZE]); + +/** + * aes_cmac() - Compute AES-CMAC or AES-XCBC-MAC in one shot + * @key: The key to use + * @data: The message data + * @data_len: The data length in bytes + * @out: (output) The resulting AES-CMAC or AES-XCBC-MAC value + * + * This supports both AES-CMAC and AES-XCBC-MAC. Which one is done depends on + * whether aes_cmac_preparekey() or aes_xcbcmac_preparekey() was called. + * + * Context: Any context. + */ +static inline void aes_cmac(const struct aes_cmac_key *key, const u8 *data, + size_t data_len, u8 out[at_least AES_BLOCK_SIZE]) +{ + struct aes_cmac_ctx ctx; + + aes_cmac_init(&ctx, key); + aes_cmac_update(&ctx, data, data_len); + aes_cmac_final(&ctx, out); +} + +/* + * AES-CBC-MAC support. This is provided only for use by the implementation of + * AES-CCM. It should have no other users. Warning: unlike AES-CMAC and + * AES-XCBC-MAC, AES-CBC-MAC isn't a secure MAC for variable-length messages. + */ +struct aes_cbcmac_ctx { + const struct aes_enckey *key; + size_t partial_len; + u8 h[AES_BLOCK_SIZE]; +}; +static inline void aes_cbcmac_init(struct aes_cbcmac_ctx *ctx, + const struct aes_enckey *key) +{ + *ctx = (struct aes_cbcmac_ctx){ .key = key }; +} +void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data, + size_t data_len); +void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, + u8 out[at_least AES_BLOCK_SIZE]); + +#endif /* _CRYPTO_AES_CBC_MACS_H */ diff --git a/include/crypto/aes.h b/include/crypto/aes.h index cbf1cc96db52..16fbfd93e2bd 100644 --- a/include/crypto/aes.h +++ b/include/crypto/aes.h @@ -167,6 +167,72 @@ int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, #ifdef CONFIG_ARM64 int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, unsigned int key_len); +asmlinkage void neon_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks); +asmlinkage void neon_aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks); +asmlinkage void neon_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks, u8 iv[]); +asmlinkage void neon_aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks, u8 iv[]); +asmlinkage void neon_aes_cbc_cts_encrypt(u8 out[], u8 const in[], + u32 const rk[], int rounds, int bytes, + u8 const iv[]); +asmlinkage void neon_aes_cbc_cts_decrypt(u8 out[], u8 const in[], + u32 const rk[], int rounds, int bytes, + u8 const iv[]); +asmlinkage void neon_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 ctr[]); +asmlinkage void neon_aes_xctr_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 ctr[], + int byte_ctr); +asmlinkage void neon_aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[], + int rounds, int bytes, u32 const rk2[], + u8 iv[], int first); +asmlinkage void neon_aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[], + int rounds, int bytes, u32 const rk2[], + u8 iv[], int first); +asmlinkage void neon_aes_essiv_cbc_encrypt(u8 out[], u8 const in[], + u32 const rk1[], int rounds, + int blocks, u8 iv[], + u32 const rk2[]); +asmlinkage void neon_aes_essiv_cbc_decrypt(u8 out[], u8 const in[], + u32 const rk1[], int rounds, + int blocks, u8 iv[], + u32 const rk2[]); + +asmlinkage void ce_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks); +asmlinkage void ce_aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks); +asmlinkage void ce_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks, u8 iv[]); +asmlinkage void ce_aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks, u8 iv[]); +asmlinkage void ce_aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 const iv[]); +asmlinkage void ce_aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 const iv[]); +asmlinkage void ce_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 ctr[]); +asmlinkage void ce_aes_xctr_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 ctr[], + int byte_ctr); +asmlinkage void ce_aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[], + int rounds, int bytes, u32 const rk2[], + u8 iv[], int first); +asmlinkage void ce_aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[], + int rounds, int bytes, u32 const rk2[], + u8 iv[], int first); +asmlinkage void ce_aes_essiv_cbc_encrypt(u8 out[], u8 const in[], + u32 const rk1[], int rounds, + int blocks, u8 iv[], u32 const rk2[]); +asmlinkage void ce_aes_essiv_cbc_decrypt(u8 out[], u8 const in[], + u32 const rk1[], int rounds, + int blocks, u8 iv[], u32 const rk2[]); +asmlinkage void ce_aes_mac_update(u8 const in[], u32 const rk[], int rounds, + size_t blocks, u8 dg[], int enc_before, + int enc_after); #elif defined(CONFIG_PPC) void ppc_expand_key_128(u32 *key_enc, const u8 *key); void ppc_expand_key_192(u32 *key_enc, const u8 *key); diff --git a/include/crypto/chacha20poly1305.h b/include/crypto/chacha20poly1305.h index 0f71b037702d..0f6d99170aaf 100644 --- a/include/crypto/chacha20poly1305.h +++ b/include/crypto/chacha20poly1305.h @@ -46,6 +46,4 @@ bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len const u64 nonce, const u8 key[at_least CHACHA20POLY1305_KEY_SIZE]); -bool chacha20poly1305_selftest(void); - #endif /* __CHACHA20POLY1305_H */ diff --git a/include/crypto/gcm.h b/include/crypto/gcm.h index b524e47bd4d0..1d5f39ff1dc4 100644 --- a/include/crypto/gcm.h +++ b/include/crypto/gcm.h @@ -4,7 +4,7 @@ #include <linux/errno.h> #include <crypto/aes.h> -#include <crypto/gf128mul.h> +#include <crypto/gf128hash.h> #define GCM_AES_IV_SIZE 12 #define GCM_RFC4106_IV_SIZE 8 @@ -65,7 +65,7 @@ static inline int crypto_ipsec_check_assoclen(unsigned int assoclen) } struct aesgcm_ctx { - be128 ghash_key; + struct ghash_key ghash_key; struct aes_enckey aes_key; unsigned int authsize; }; diff --git a/include/crypto/polyval.h b/include/crypto/gf128hash.h index b28b8ef11353..41c557d55965 100644 --- a/include/crypto/polyval.h +++ b/include/crypto/gf128hash.h @@ -1,13 +1,14 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * POLYVAL library API + * GF(2^128) polynomial hashing: GHASH and POLYVAL * * Copyright 2025 Google LLC */ -#ifndef _CRYPTO_POLYVAL_H -#define _CRYPTO_POLYVAL_H +#ifndef _CRYPTO_GF128HASH_H +#define _CRYPTO_GF128HASH_H +#include <crypto/ghash.h> #include <linux/string.h> #include <linux/types.h> @@ -34,6 +35,24 @@ struct polyval_elem { }; /** + * struct ghash_key - Prepared key for GHASH + * + * Use ghash_preparekey() to initialize this. + */ +struct ghash_key { +#if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_PPC64) + /** @htable: GHASH key format used by the POWER8 assembly code */ + u64 htable[4][2]; +#elif defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && \ + (defined(CONFIG_RISCV) || defined(CONFIG_S390)) + /** @h_raw: The hash key H, in GHASH format */ + u8 h_raw[GHASH_BLOCK_SIZE]; +#endif + /** @h: The hash key H, in POLYVAL format */ + struct polyval_elem h; +}; + +/** * struct polyval_key - Prepared key for POLYVAL * * This may contain just the raw key H, or it may contain precomputed key @@ -44,20 +63,28 @@ struct polyval_elem { * exponentiation repeats the POLYVAL dot operation, with its "extra" x^-128. */ struct polyval_key { -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH -#ifdef CONFIG_ARM64 - /** @h_powers: Powers of the hash key H^8 through H^1 */ - struct polyval_elem h_powers[8]; -#elif defined(CONFIG_X86) +#if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && \ + (defined(CONFIG_ARM64) || defined(CONFIG_X86)) /** @h_powers: Powers of the hash key H^8 through H^1 */ struct polyval_elem h_powers[8]; #else -#error "Unhandled arch" -#endif -#else /* CONFIG_CRYPTO_LIB_POLYVAL_ARCH */ /** @h: The hash key H */ struct polyval_elem h; -#endif /* !CONFIG_CRYPTO_LIB_POLYVAL_ARCH */ +#endif +}; + +/** + * struct ghash_ctx - Context for computing a GHASH value + * @key: Pointer to the prepared GHASH key. The user of the API is + * responsible for ensuring that the key lives as long as the context. + * @acc: The accumulator. It is stored in POLYVAL format rather than GHASH + * format, since most implementations want it in POLYVAL format. + * @partial: Number of data bytes processed so far modulo GHASH_BLOCK_SIZE + */ +struct ghash_ctx { + const struct ghash_key *key; + struct polyval_elem acc; + size_t partial; }; /** @@ -74,6 +101,18 @@ struct polyval_ctx { }; /** + * ghash_preparekey() - Prepare a GHASH key + * @key: (output) The key structure to initialize + * @raw_key: The raw hash key + * + * Initialize a GHASH key structure from a raw key. + * + * Context: Any context. + */ +void ghash_preparekey(struct ghash_key *key, + const u8 raw_key[GHASH_BLOCK_SIZE]); + +/** * polyval_preparekey() - Prepare a POLYVAL key * @key: (output) The key structure to initialize * @raw_key: The raw hash key @@ -84,18 +123,20 @@ struct polyval_ctx { * * Context: Any context. */ -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH void polyval_preparekey(struct polyval_key *key, const u8 raw_key[POLYVAL_BLOCK_SIZE]); -#else -static inline void polyval_preparekey(struct polyval_key *key, - const u8 raw_key[POLYVAL_BLOCK_SIZE]) +/** + * ghash_init() - Initialize a GHASH context for a new message + * @ctx: The context to initialize + * @key: The key to use. Note that a pointer to the key is saved in the + * context, so the key must live at least as long as the context. + */ +static inline void ghash_init(struct ghash_ctx *ctx, + const struct ghash_key *key) { - /* Just a simple copy, so inline it. */ - memcpy(key->h.bytes, raw_key, POLYVAL_BLOCK_SIZE); + *ctx = (struct ghash_ctx){ .key = key }; } -#endif /** * polyval_init() - Initialize a POLYVAL context for a new message @@ -142,6 +183,18 @@ static inline void polyval_export_blkaligned(const struct polyval_ctx *ctx, } /** + * ghash_update() - Update a GHASH context with message data + * @ctx: The context to update; must have been initialized + * @data: The message data + * @len: The data length in bytes. Doesn't need to be block-aligned. + * + * This can be called any number of times. + * + * Context: Any context. + */ +void ghash_update(struct ghash_ctx *ctx, const u8 *data, size_t len); + +/** * polyval_update() - Update a POLYVAL context with message data * @ctx: The context to update; must have been initialized * @data: The message data @@ -154,6 +207,20 @@ static inline void polyval_export_blkaligned(const struct polyval_ctx *ctx, void polyval_update(struct polyval_ctx *ctx, const u8 *data, size_t len); /** + * ghash_final() - Finish computing a GHASH value + * @ctx: The context to finalize + * @out: The output value + * + * If the total data length isn't a multiple of GHASH_BLOCK_SIZE, then the + * final block is automatically zero-padded. + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void ghash_final(struct ghash_ctx *ctx, u8 out[GHASH_BLOCK_SIZE]); + +/** * polyval_final() - Finish computing a POLYVAL value * @ctx: The context to finalize * @out: The output value @@ -168,6 +235,25 @@ void polyval_update(struct polyval_ctx *ctx, const u8 *data, size_t len); void polyval_final(struct polyval_ctx *ctx, u8 out[POLYVAL_BLOCK_SIZE]); /** + * ghash() - Compute a GHASH value + * @key: The prepared key + * @data: The message data + * @len: The data length in bytes. Doesn't need to be block-aligned. + * @out: The output value + * + * Context: Any context. + */ +static inline void ghash(const struct ghash_key *key, const u8 *data, + size_t len, u8 out[GHASH_BLOCK_SIZE]) +{ + struct ghash_ctx ctx; + + ghash_init(&ctx, key); + ghash_update(&ctx, data, len); + ghash_final(&ctx, out); +} + +/** * polyval() - Compute a POLYVAL value * @key: The prepared key * @data: The message data @@ -187,4 +273,4 @@ static inline void polyval(const struct polyval_key *key, polyval_final(&ctx, out); } -#endif /* _CRYPTO_POLYVAL_H */ +#endif /* _CRYPTO_GF128HASH_H */ diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h index b0853f7cada0..6ed2a8351902 100644 --- a/include/crypto/gf128mul.h +++ b/include/crypto/gf128mul.h @@ -215,25 +215,14 @@ static inline void gf128mul_x_ble(le128 *r, const le128 *x) r->b = cpu_to_le64((b << 1) ^ _tt); } -/* 4k table optimization */ - -struct gf128mul_4k { - be128 t[256]; -}; - -struct gf128mul_4k *gf128mul_init_4k_lle(const be128 *g); -void gf128mul_4k_lle(be128 *a, const struct gf128mul_4k *t); void gf128mul_x8_ble(le128 *r, const le128 *x); -static inline void gf128mul_free_4k(struct gf128mul_4k *t) -{ - kfree_sensitive(t); -} - /* 64k table optimization, implemented for bbe */ struct gf128mul_64k { - struct gf128mul_4k *t[16]; + struct { + be128 t[256]; + } *t[16]; }; /* First initialize with the constant factor with which you diff --git a/include/crypto/ghash.h b/include/crypto/ghash.h index 043d938e9a2c..d187e5af9925 100644 --- a/include/crypto/ghash.h +++ b/include/crypto/ghash.h @@ -6,19 +6,7 @@ #ifndef __CRYPTO_GHASH_H__ #define __CRYPTO_GHASH_H__ -#include <linux/types.h> - #define GHASH_BLOCK_SIZE 16 #define GHASH_DIGEST_SIZE 16 -struct gf128mul_4k; - -struct ghash_ctx { - struct gf128mul_4k *gf128; -}; - -struct ghash_desc_ctx { - u8 buffer[GHASH_BLOCK_SIZE]; -}; - #endif diff --git a/include/crypto/internal/blockhash.h b/include/crypto/internal/blockhash.h deleted file mode 100644 index 52d9d4c82493..000000000000 --- a/include/crypto/internal/blockhash.h +++ /dev/null @@ -1,52 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Handle partial blocks for block hash. - * - * Copyright (c) 2015 Linaro Ltd <ard.biesheuvel@linaro.org> - * Copyright (c) 2025 Herbert Xu <herbert@gondor.apana.org.au> - */ - -#ifndef _CRYPTO_INTERNAL_BLOCKHASH_H -#define _CRYPTO_INTERNAL_BLOCKHASH_H - -#include <linux/string.h> -#include <linux/types.h> - -#define BLOCK_HASH_UPDATE_BASE(block_fn, state, src, nbytes, bs, dv, \ - buf, buflen) \ - ({ \ - typeof(block_fn) *_block_fn = &(block_fn); \ - typeof(state + 0) _state = (state); \ - unsigned int _buflen = (buflen); \ - size_t _nbytes = (nbytes); \ - unsigned int _bs = (bs); \ - const u8 *_src = (src); \ - u8 *_buf = (buf); \ - while ((_buflen + _nbytes) >= _bs) { \ - const u8 *data = _src; \ - size_t len = _nbytes; \ - size_t blocks; \ - int remain; \ - if (_buflen) { \ - remain = _bs - _buflen; \ - memcpy(_buf + _buflen, _src, remain); \ - data = _buf; \ - len = _bs; \ - } \ - remain = len % bs; \ - blocks = (len - remain) / (dv); \ - (*_block_fn)(_state, data, blocks); \ - _src += len - remain - _buflen; \ - _nbytes -= len - remain - _buflen; \ - _buflen = 0; \ - } \ - memcpy(_buf + _buflen, _src, _nbytes); \ - _buflen += _nbytes; \ - }) - -#define BLOCK_HASH_UPDATE(block, state, src, nbytes, bs, buf, buflen) \ - BLOCK_HASH_UPDATE_BASE(block, state, src, nbytes, bs, 1, buf, buflen) -#define BLOCK_HASH_UPDATE_BLOCKS(block, state, src, nbytes, bs, buf, buflen) \ - BLOCK_HASH_UPDATE_BASE(block, state, src, nbytes, bs, bs, buf, buflen) - -#endif /* _CRYPTO_INTERNAL_BLOCKHASH_H */ diff --git a/include/crypto/sm3.h b/include/crypto/sm3.h index c8d02c86c298..371e8a661705 100644 --- a/include/crypto/sm3.h +++ b/include/crypto/sm3.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Common values for SM3 algorithm + * SM3 hash algorithm * * Copyright (C) 2017 ARM Limited or its affiliates. * Copyright (C) 2017 Gilad Ben-Yossef <gilad@benyossef.com> @@ -14,10 +14,6 @@ #define SM3_DIGEST_SIZE 32 #define SM3_BLOCK_SIZE 64 -#define SM3_STATE_SIZE 40 - -#define SM3_T1 0x79CC4519 -#define SM3_T2 0x7A879D8A #define SM3_IVA 0x7380166f #define SM3_IVB 0x4914b2b9 @@ -28,37 +24,64 @@ #define SM3_IVG 0xe38dee4d #define SM3_IVH 0xb0fb0e4e -extern const u8 sm3_zero_message_hash[SM3_DIGEST_SIZE]; +/* State for the SM3 compression function */ +struct sm3_block_state { + u32 h[SM3_DIGEST_SIZE / 4]; +}; -struct sm3_state { - u32 state[SM3_DIGEST_SIZE / 4]; - u64 count; - u8 buffer[SM3_BLOCK_SIZE]; +/** + * struct sm3_ctx - Context for hashing a message with SM3 + * @state: the compression function state + * @bytecount: number of bytes processed so far + * @buf: partial block buffer; bytecount % SM3_BLOCK_SIZE bytes are valid + */ +struct sm3_ctx { + struct sm3_block_state state; + u64 bytecount; + u8 buf[SM3_BLOCK_SIZE] __aligned(__alignof__(__be64)); }; -/* - * Stand-alone implementation of the SM3 algorithm. It is designed to - * have as little dependencies as possible so it can be used in the - * kexec_file purgatory. In other cases you should generally use the - * hash APIs from include/crypto/hash.h. Especially when hashing large - * amounts of data as those APIs may be hw-accelerated. +/** + * sm3_init() - Initialize an SM3 context for a new message + * @ctx: the context to initialize + * + * If you don't need incremental computation, consider sm3() instead. + * + * Context: Any context. + */ +void sm3_init(struct sm3_ctx *ctx); + +/** + * sm3_update() - Update an SM3 context with message data + * @ctx: the context to update; must have been initialized + * @data: the message data + * @len: the data length in bytes + * + * This can be called any number of times. * - * For details see lib/crypto/sm3.c + * Context: Any context. */ +void sm3_update(struct sm3_ctx *ctx, const u8 *data, size_t len); -static inline void sm3_init(struct sm3_state *sctx) -{ - sctx->state[0] = SM3_IVA; - sctx->state[1] = SM3_IVB; - sctx->state[2] = SM3_IVC; - sctx->state[3] = SM3_IVD; - sctx->state[4] = SM3_IVE; - sctx->state[5] = SM3_IVF; - sctx->state[6] = SM3_IVG; - sctx->state[7] = SM3_IVH; - sctx->count = 0; -} +/** + * sm3_final() - Finish computing an SM3 message digest + * @ctx: the context to finalize; must have been initialized + * @out: (output) the resulting SM3 message digest + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void sm3_final(struct sm3_ctx *ctx, u8 out[at_least SM3_DIGEST_SIZE]); -void sm3_block_generic(struct sm3_state *sctx, u8 const *data, int blocks); +/** + * sm3() - Compute SM3 message digest in one shot + * @data: the message data + * @len: the data length in bytes + * @out: (output) the resulting SM3 message digest + * + * Context: Any context. + */ +void sm3(const u8 *data, size_t len, u8 out[at_least SM3_DIGEST_SIZE]); -#endif +#endif /* _CRYPTO_SM3_H */ diff --git a/include/crypto/sm3_base.h b/include/crypto/sm3_base.h deleted file mode 100644 index 7c53570bc05e..000000000000 --- a/include/crypto/sm3_base.h +++ /dev/null @@ -1,82 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * sm3_base.h - core logic for SM3 implementations - * - * Copyright (C) 2017 ARM Limited or its affiliates. - * Written by Gilad Ben-Yossef <gilad@benyossef.com> - */ - -#ifndef _CRYPTO_SM3_BASE_H -#define _CRYPTO_SM3_BASE_H - -#include <crypto/internal/hash.h> -#include <crypto/sm3.h> -#include <linux/math.h> -#include <linux/module.h> -#include <linux/string.h> -#include <linux/types.h> -#include <linux/unaligned.h> - -typedef void (sm3_block_fn)(struct sm3_state *sst, u8 const *src, int blocks); - -static inline int sm3_base_init(struct shash_desc *desc) -{ - sm3_init(shash_desc_ctx(desc)); - return 0; -} - -static inline int sm3_base_do_update_blocks(struct shash_desc *desc, - const u8 *data, unsigned int len, - sm3_block_fn *block_fn) -{ - unsigned int remain = len - round_down(len, SM3_BLOCK_SIZE); - struct sm3_state *sctx = shash_desc_ctx(desc); - - sctx->count += len - remain; - block_fn(sctx, data, len / SM3_BLOCK_SIZE); - return remain; -} - -static inline int sm3_base_do_finup(struct shash_desc *desc, - const u8 *src, unsigned int len, - sm3_block_fn *block_fn) -{ - unsigned int bit_offset = SM3_BLOCK_SIZE / 8 - 1; - struct sm3_state *sctx = shash_desc_ctx(desc); - union { - __be64 b64[SM3_BLOCK_SIZE / 4]; - u8 u8[SM3_BLOCK_SIZE * 2]; - } block = {}; - - if (len >= SM3_BLOCK_SIZE) { - int remain; - - remain = sm3_base_do_update_blocks(desc, src, len, block_fn); - src += len - remain; - len = remain; - } - - if (len >= bit_offset * 8) - bit_offset += SM3_BLOCK_SIZE / 8; - memcpy(&block, src, len); - block.u8[len] = 0x80; - sctx->count += len; - block.b64[bit_offset] = cpu_to_be64(sctx->count << 3); - block_fn(sctx, block.u8, (bit_offset + 1) * 8 / SM3_BLOCK_SIZE); - memzero_explicit(&block, sizeof(block)); - - return 0; -} - -static inline int sm3_base_finish(struct shash_desc *desc, u8 *out) -{ - struct sm3_state *sctx = shash_desc_ctx(desc); - __be32 *digest = (__be32 *)out; - int i; - - for (i = 0; i < SM3_DIGEST_SIZE / sizeof(__be32); i++) - put_unaligned_be32(sctx->state[i], digest++); - return 0; -} - -#endif /* _CRYPTO_SM3_BASE_H */ diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 30d069676309..94eb2667b7e1 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -3058,6 +3058,8 @@ config HW_BREAKPOINT_KUNIT_TEST If unsure, say N. +source "lib/crypto/tests/Kconfig" + config SIPHASH_KUNIT_TEST tristate "Perform selftest on siphash functions" if !KUNIT_ALL_TESTS depends on KUNIT diff --git a/lib/crypto/.kunitconfig b/lib/crypto/.kunitconfig index 6b2ce28ae509..3efc854a2c08 100644 --- a/lib/crypto/.kunitconfig +++ b/lib/crypto/.kunitconfig @@ -1,28 +1,13 @@ CONFIG_KUNIT=y -# These kconfig options select all the CONFIG_CRYPTO_LIB_* symbols that have a -# corresponding KUnit test. Those symbols cannot be directly enabled here, -# since they are hidden symbols. -CONFIG_CRYPTO=y -CONFIG_CRYPTO_ADIANTUM=y -CONFIG_CRYPTO_BLAKE2B=y -CONFIG_CRYPTO_CHACHA20POLY1305=y -CONFIG_CRYPTO_HCTR2=y -CONFIG_CRYPTO_MD5=y -CONFIG_CRYPTO_MLDSA=y -CONFIG_CRYPTO_SHA1=y -CONFIG_CRYPTO_SHA256=y -CONFIG_CRYPTO_SHA512=y -CONFIG_CRYPTO_SHA3=y -CONFIG_INET=y -CONFIG_IPV6=y -CONFIG_NET=y -CONFIG_NETDEVICES=y -CONFIG_WIREGUARD=y +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=y +CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST=y CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST=y CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST=y CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_GHASH_KUNIT_TEST=y CONFIG_CRYPTO_LIB_MD5_KUNIT_TEST=y CONFIG_CRYPTO_LIB_MLDSA_KUNIT_TEST=y CONFIG_CRYPTO_LIB_NH_KUNIT_TEST=y @@ -32,3 +17,4 @@ CONFIG_CRYPTO_LIB_SHA1_KUNIT_TEST=y CONFIG_CRYPTO_LIB_SHA256_KUNIT_TEST=y CONFIG_CRYPTO_LIB_SHA512_KUNIT_TEST=y CONFIG_CRYPTO_LIB_SHA3_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_SM3_KUNIT_TEST=y diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 032f9755f999..d3904b72dae7 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -1,7 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -menu "Crypto library routines" - config CRYPTO_HASH_INFO bool @@ -10,6 +8,8 @@ config CRYPTO_LIB_UTILS config CRYPTO_LIB_AES tristate + # Select dependencies of modes that are part of libaes. + select CRYPTO_LIB_UTILS if CRYPTO_LIB_AES_CBC_MACS config CRYPTO_LIB_AES_ARCH bool @@ -28,10 +28,18 @@ config CRYPTO_LIB_AESCFB select CRYPTO_LIB_AES select CRYPTO_LIB_UTILS +config CRYPTO_LIB_AES_CBC_MACS + tristate + select CRYPTO_LIB_AES + help + The AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC library functions. Select + this if your module uses any of the functions from + <crypto/aes-cbc-macs.h>. + config CRYPTO_LIB_AESGCM tristate select CRYPTO_LIB_AES - select CRYPTO_LIB_GF128MUL + select CRYPTO_LIB_GF128HASH select CRYPTO_LIB_UTILS config CRYPTO_LIB_ARC4 @@ -70,7 +78,7 @@ config CRYPTO_LIB_CHACHA_ARCH bool depends on CRYPTO_LIB_CHACHA && !UML && !KMSAN default y if ARM - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if MIPS && CPU_MIPS32_R2 default y if PPC64 && CPU_LITTLE_ENDIAN && VSX default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ @@ -100,6 +108,23 @@ config CRYPTO_LIB_CURVE25519_GENERIC config CRYPTO_LIB_DES tristate +config CRYPTO_LIB_GF128HASH + tristate + help + The GHASH and POLYVAL library functions. Select this if your module + uses any of the functions from <crypto/gf128hash.h>. + +config CRYPTO_LIB_GF128HASH_ARCH + bool + depends on CRYPTO_LIB_GF128HASH && !UML + default y if ARM && KERNEL_MODE_NEON + default y if ARM64 + default y if PPC64 && VSX + default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS + default y if S390 + default y if X86_64 + config CRYPTO_LIB_MD5 tristate help @@ -109,9 +134,7 @@ config CRYPTO_LIB_MD5 config CRYPTO_LIB_MD5_ARCH bool depends on CRYPTO_LIB_MD5 && !UML - default y if MIPS && CPU_CAVIUM_OCTEON default y if PPC - default y if SPARC64 config CRYPTO_LIB_MLDSA tristate @@ -130,7 +153,7 @@ config CRYPTO_LIB_NH_ARCH bool depends on CRYPTO_LIB_NH && !UML && !KMSAN default y if ARM && KERNEL_MODE_NEON - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if X86_64 config CRYPTO_LIB_POLY1305 @@ -143,7 +166,7 @@ config CRYPTO_LIB_POLY1305_ARCH bool depends on CRYPTO_LIB_POLY1305 && !UML && !KMSAN default y if ARM - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if MIPS # The PPC64 code needs to be fixed to work in softirq context. default y if PPC64 && CPU_LITTLE_ENDIAN && VSX && BROKEN @@ -168,18 +191,6 @@ config CRYPTO_LIB_POLY1305_RSIZE default 9 if ARM || ARM64 default 1 -config CRYPTO_LIB_POLYVAL - tristate - help - The POLYVAL library functions. Select this if your module uses any of - the functions from <crypto/polyval.h>. - -config CRYPTO_LIB_POLYVAL_ARCH - bool - depends on CRYPTO_LIB_POLYVAL && !UML - default y if ARM64 && KERNEL_MODE_NEON - default y if X86_64 - config CRYPTO_LIB_CHACHA20POLY1305 tristate select CRYPTO_LIB_CHACHA @@ -196,7 +207,7 @@ config CRYPTO_LIB_SHA1_ARCH bool depends on CRYPTO_LIB_SHA1 && !UML default y if ARM - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if MIPS && CPU_CAVIUM_OCTEON default y if PPC default y if S390 @@ -252,12 +263,19 @@ config CRYPTO_LIB_SHA3 config CRYPTO_LIB_SHA3_ARCH bool depends on CRYPTO_LIB_SHA3 && !UML - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if S390 config CRYPTO_LIB_SM3 tristate + help + The SM3 library functions. Select this if your module uses any of the + functions from <crypto/sm3.h>. -source "lib/crypto/tests/Kconfig" - -endmenu +config CRYPTO_LIB_SM3_ARCH + bool + depends on CRYPTO_LIB_SM3 && !UML + default y if ARM64 + default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS + default y if X86_64 diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index dc7a56f7287d..4ad91f390038 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -8,6 +8,10 @@ quiet_cmd_perlasm = PERLASM $@ quiet_cmd_perlasm_with_args = PERLASM $@ cmd_perlasm_with_args = $(PERL) $(<) void $(@) +ppc64-perlasm-flavour-y := linux-ppc64 +ppc64-perlasm-flavour-$(CONFIG_PPC64_ELF_ABI_V2) := linux-ppc64-elfv2 +ppc64-perlasm-flavour-$(CONFIG_CPU_LITTLE_ENDIAN) := linux-ppc64le + obj-$(CONFIG_KUNIT) += tests/ obj-$(CONFIG_CRYPTO_HASH_INFO) += hash_info.o @@ -23,11 +27,10 @@ ifeq ($(CONFIG_CRYPTO_LIB_AES_ARCH),y) CFLAGS_aes.o += -I$(src)/$(SRCARCH) libaes-$(CONFIG_ARM) += arm/aes-cipher-core.o - -ifeq ($(CONFIG_ARM64),y) -libaes-y += arm64/aes-cipher-core.o -libaes-$(CONFIG_KERNEL_MODE_NEON) += arm64/aes-ce-core.o -endif +libaes-$(CONFIG_ARM64) += arm64/aes-cipher-core.o \ + arm64/aes-ce-core.o \ + arm64/aes-ce.o \ + arm64/aes-neon.o ifeq ($(CONFIG_PPC),y) ifeq ($(CONFIG_SPE),y) @@ -37,11 +40,8 @@ libaes-y += powerpc/aes-spe-core.o \ powerpc/aes-tab-4k.o else libaes-y += powerpc/aesp8-ppc.o -aes-perlasm-flavour-y := linux-ppc64 -aes-perlasm-flavour-$(CONFIG_PPC64_ELF_ABI_V2) := linux-ppc64-elfv2 -aes-perlasm-flavour-$(CONFIG_CPU_LITTLE_ENDIAN) := linux-ppc64le quiet_cmd_perlasm_aes = PERLASM $@ - cmd_perlasm_aes = $(PERL) $< $(aes-perlasm-flavour-y) $@ + cmd_perlasm_aes = $(PERL) $< $(ppc64-perlasm-flavour-y) $@ # Use if_changed instead of cmd, in case the flavour changed. $(obj)/powerpc/aesp8-ppc.S: $(src)/powerpc/aesp8-ppc.pl FORCE $(call if_changed,perlasm_aes) @@ -125,7 +125,6 @@ endif # CONFIG_CRYPTO_LIB_CHACHA_ARCH obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305) += libchacha20poly1305.o libchacha20poly1305-y += chacha20poly1305.o -libchacha20poly1305-$(CONFIG_CRYPTO_SELFTESTS) += chacha20poly1305-selftest.o ################################################################################ @@ -158,12 +157,39 @@ libdes-y := des.o ################################################################################ +obj-$(CONFIG_CRYPTO_LIB_GF128HASH) += libgf128hash.o +libgf128hash-y := gf128hash.o +ifeq ($(CONFIG_CRYPTO_LIB_GF128HASH_ARCH),y) +CFLAGS_gf128hash.o += -I$(src)/$(SRCARCH) +libgf128hash-$(CONFIG_ARM) += arm/ghash-neon-core.o +libgf128hash-$(CONFIG_ARM64) += arm64/ghash-neon-core.o \ + arm64/polyval-ce-core.o + +ifeq ($(CONFIG_PPC),y) +libgf128hash-y += powerpc/ghashp8-ppc.o +quiet_cmd_perlasm_ghash = PERLASM $@ + cmd_perlasm_ghash = $(PERL) $< $(ppc64-perlasm-flavour-y) $@ +$(obj)/powerpc/ghashp8-ppc.S: $(src)/powerpc/ghashp8-ppc.pl FORCE + $(call if_changed,perlasm_ghash) +targets += powerpc/ghashp8-ppc.S +OBJECT_FILES_NON_STANDARD_powerpc/ghashp8-ppc.o := y +endif + +libgf128hash-$(CONFIG_RISCV) += riscv/ghash-riscv64-zvkg.o +libgf128hash-$(CONFIG_X86) += x86/ghash-pclmul.o \ + x86/polyval-pclmul-avx.o +endif # CONFIG_CRYPTO_LIB_GF128HASH_ARCH + +# clean-files must be defined unconditionally +clean-files += powerpc/ghashp8-ppc.S + +################################################################################ + obj-$(CONFIG_CRYPTO_LIB_MD5) += libmd5.o libmd5-y := md5.o ifeq ($(CONFIG_CRYPTO_LIB_MD5_ARCH),y) CFLAGS_md5.o += -I$(src)/$(SRCARCH) libmd5-$(CONFIG_PPC) += powerpc/md5-asm.o -libmd5-$(CONFIG_SPARC) += sparc/md5_asm.o endif # CONFIG_CRYPTO_LIB_MD5_ARCH ################################################################################ @@ -255,16 +281,6 @@ clean-files += arm/poly1305-core.S \ ################################################################################ -obj-$(CONFIG_CRYPTO_LIB_POLYVAL) += libpolyval.o -libpolyval-y := polyval.o -ifeq ($(CONFIG_CRYPTO_LIB_POLYVAL_ARCH),y) -CFLAGS_polyval.o += -I$(src)/$(SRCARCH) -libpolyval-$(CONFIG_ARM64) += arm64/polyval-ce-core.o -libpolyval-$(CONFIG_X86) += x86/polyval-pclmul-avx.o -endif - -################################################################################ - obj-$(CONFIG_CRYPTO_LIB_SHA1) += libsha1.o libsha1-y := sha1.o ifeq ($(CONFIG_CRYPTO_LIB_SHA1_ARCH),y) @@ -300,10 +316,9 @@ AFLAGS_arm/sha256-core.o += $(aflags-thumb2-y) endif ifeq ($(CONFIG_ARM64),y) -libsha256-y += arm64/sha256-core.o +libsha256-y += arm64/sha256-ce.o arm64/sha256-core.o $(obj)/arm64/sha256-core.S: $(src)/arm64/sha2-armv8.pl $(call cmd,perlasm_with_args) -libsha256-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha256-ce.o endif libsha256-$(CONFIG_PPC) += powerpc/sha256-spe-asm.o @@ -330,10 +345,9 @@ AFLAGS_arm/sha512-core.o += $(aflags-thumb2-y) endif ifeq ($(CONFIG_ARM64),y) -libsha512-y += arm64/sha512-core.o +libsha512-y += arm64/sha512-ce-core.o arm64/sha512-core.o $(obj)/arm64/sha512-core.S: $(src)/arm64/sha2-armv8.pl $(call cmd,perlasm_with_args) -libsha512-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha512-ce-core.o endif libsha512-$(CONFIG_RISCV) += riscv/sha512-riscv64-zvknhb-zvkb.o @@ -355,13 +369,22 @@ endif # CONFIG_CRYPTO_LIB_SHA3_ARCH ################################################################################ +obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o +libsm3-y := sm3.o +ifeq ($(CONFIG_CRYPTO_LIB_SM3_ARCH),y) +CFLAGS_sm3.o += -I$(src)/$(SRCARCH) +libsm3-$(CONFIG_ARM64) += arm64/sm3-ce-core.o \ + arm64/sm3-neon-core.o +libsm3-$(CONFIG_RISCV) += riscv/sm3-riscv64-zvksh-zvkb.o +libsm3-$(CONFIG_X86) += x86/sm3-avx-asm_64.o +endif # CONFIG_CRYPTO_LIB_SM3_ARCH + +################################################################################ + obj-$(CONFIG_MPILIB) += mpi/ obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) += simd.o -obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o -libsm3-y := sm3.o - # clean-files must be defined unconditionally clean-files += arm/sha256-core.S arm/sha512-core.S clean-files += arm64/sha256-core.S arm64/sha512-core.S diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c index b73e19f1bb95..ca733f15b2a8 100644 --- a/lib/crypto/aes.c +++ b/lib/crypto/aes.c @@ -4,12 +4,15 @@ * Copyright 2026 Google LLC */ +#include <crypto/aes-cbc-macs.h> #include <crypto/aes.h> +#include <crypto/utils.h> #include <linux/cache.h> #include <linux/crypto.h> #include <linux/export.h> #include <linux/module.h> #include <linux/unaligned.h> +#include "fips.h" static const u8 ____cacheline_aligned aes_sbox[] = { 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, @@ -512,10 +515,235 @@ void aes_decrypt(const struct aes_key *key, u8 out[AES_BLOCK_SIZE], } EXPORT_SYMBOL(aes_decrypt); -#ifdef aes_mod_init_arch +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) + +#ifndef aes_cbcmac_blocks_arch +static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], + const struct aes_enckey *key, const u8 *data, + size_t nblocks, bool enc_before, + bool enc_after) +{ + return false; +} +#endif + +/* This assumes nblocks >= 1. */ +static void aes_cbcmac_blocks(u8 h[AES_BLOCK_SIZE], + const struct aes_enckey *key, const u8 *data, + size_t nblocks, bool enc_before, bool enc_after) +{ + if (aes_cbcmac_blocks_arch(h, key, data, nblocks, enc_before, + enc_after)) + return; + + if (enc_before) + aes_encrypt(key, h, h); + for (; nblocks > 1; nblocks--) { + crypto_xor(h, data, AES_BLOCK_SIZE); + data += AES_BLOCK_SIZE; + aes_encrypt(key, h, h); + } + crypto_xor(h, data, AES_BLOCK_SIZE); + if (enc_after) + aes_encrypt(key, h, h); +} + +int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key, + size_t key_len) +{ + u64 hi, lo, mask; + int err; + + /* Prepare the AES key. */ + err = aes_prepareenckey(&key->aes, in_key, key_len); + if (err) + return err; + + /* + * Prepare the subkeys K1 and K2 by encrypting the all-zeroes block, + * then multiplying by 'x' and 'x^2' (respectively) in GF(2^128). + * Reference: NIST SP 800-38B, Section 6.1 "Subkey Generation". + */ + memset(key->k_final[0].b, 0, AES_BLOCK_SIZE); + aes_encrypt(&key->aes, key->k_final[0].b, key->k_final[0].b); + hi = be64_to_cpu(key->k_final[0].w[0]); + lo = be64_to_cpu(key->k_final[0].w[1]); + for (int i = 0; i < 2; i++) { + mask = ((s64)hi >> 63) & 0x87; + hi = (hi << 1) ^ (lo >> 63); + lo = (lo << 1) ^ mask; + key->k_final[i].w[0] = cpu_to_be64(hi); + key->k_final[i].w[1] = cpu_to_be64(lo); + } + return 0; +} +EXPORT_SYMBOL_GPL(aes_cmac_preparekey); + +void aes_xcbcmac_preparekey(struct aes_cmac_key *key, + const u8 in_key[AES_KEYSIZE_128]) +{ + static const u8 constants[3][AES_BLOCK_SIZE] = { + { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, + { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, + { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, + }; + u8 new_aes_key[AES_BLOCK_SIZE]; + + static_assert(AES_BLOCK_SIZE == AES_KEYSIZE_128); + aes_prepareenckey(&key->aes, in_key, AES_BLOCK_SIZE); + aes_encrypt(&key->aes, new_aes_key, constants[0]); + aes_encrypt(&key->aes, key->k_final[0].b, constants[1]); + aes_encrypt(&key->aes, key->k_final[1].b, constants[2]); + aes_prepareenckey(&key->aes, new_aes_key, AES_BLOCK_SIZE); + memzero_explicit(new_aes_key, AES_BLOCK_SIZE); +} +EXPORT_SYMBOL_GPL(aes_xcbcmac_preparekey); + +void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len) +{ + bool enc_before = false; + size_t nblocks; + + if (ctx->partial_len) { + /* XOR data into a pending block. */ + size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); + + crypto_xor(&ctx->h[ctx->partial_len], data, l); + data += l; + data_len -= l; + ctx->partial_len += l; + if (data_len == 0) { + /* + * Either the pending block hasn't been filled yet, or + * no more data was given so it's not yet known whether + * the block is the final block. + */ + return; + } + /* Pending block has been filled and isn't the final block. */ + enc_before = true; + } + + nblocks = data_len / AES_BLOCK_SIZE; + data_len %= AES_BLOCK_SIZE; + if (nblocks == 0) { + /* 0 additional full blocks, then optionally a partial block */ + if (enc_before) + aes_encrypt(&ctx->key->aes, ctx->h, ctx->h); + crypto_xor(ctx->h, data, data_len); + ctx->partial_len = data_len; + } else if (data_len != 0) { + /* 1 or more additional full blocks, then a partial block */ + aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, + enc_before, /* enc_after= */ true); + data += nblocks * AES_BLOCK_SIZE; + crypto_xor(ctx->h, data, data_len); + ctx->partial_len = data_len; + } else { + /* + * 1 or more additional full blocks only. Encryption of the + * last block is delayed until it's known whether it's the final + * block in the message or not. + */ + aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, + enc_before, /* enc_after= */ false); + ctx->partial_len = AES_BLOCK_SIZE; + } +} +EXPORT_SYMBOL_GPL(aes_cmac_update); + +void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) +{ + if (ctx->partial_len == AES_BLOCK_SIZE) { + /* Final block is a full block. Use k_final[0]. */ + crypto_xor(ctx->h, ctx->key->k_final[0].b, AES_BLOCK_SIZE); + } else { + /* Final block is a partial block. Pad, and use k_final[1]. */ + ctx->h[ctx->partial_len] ^= 0x80; + crypto_xor(ctx->h, ctx->key->k_final[1].b, AES_BLOCK_SIZE); + } + aes_encrypt(&ctx->key->aes, out, ctx->h); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(aes_cmac_final); + +void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data, + size_t data_len) +{ + bool enc_before = false; + size_t nblocks; + + if (ctx->partial_len) { + size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); + + crypto_xor(&ctx->h[ctx->partial_len], data, l); + data += l; + data_len -= l; + ctx->partial_len += l; + if (ctx->partial_len < AES_BLOCK_SIZE) + return; + enc_before = true; + } + + nblocks = data_len / AES_BLOCK_SIZE; + data_len %= AES_BLOCK_SIZE; + if (nblocks == 0) { + if (enc_before) + aes_encrypt(ctx->key, ctx->h, ctx->h); + } else { + aes_cbcmac_blocks(ctx->h, ctx->key, data, nblocks, enc_before, + /* enc_after= */ true); + data += nblocks * AES_BLOCK_SIZE; + } + crypto_xor(ctx->h, data, data_len); + ctx->partial_len = data_len; +} +EXPORT_SYMBOL_NS_GPL(aes_cbcmac_update, "CRYPTO_INTERNAL"); + +void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) +{ + if (ctx->partial_len) + aes_encrypt(ctx->key, out, ctx->h); + else + memcpy(out, ctx->h, AES_BLOCK_SIZE); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_NS_GPL(aes_cbcmac_final, "CRYPTO_INTERNAL"); + +/* + * FIPS cryptographic algorithm self-test for AES-CMAC. As per the FIPS 140-3 + * Implementation Guidance, a cryptographic algorithm self-test for at least one + * of AES-GCM, AES-CCM, AES-CMAC, or AES-GMAC is required if any of those modes + * is implemented. This fulfills that requirement via AES-CMAC. + * + * This is just for FIPS. The full tests are in the KUnit test suite. + */ +static void __init aes_cmac_fips_test(void) +{ + struct aes_cmac_key key; + u8 mac[AES_BLOCK_SIZE]; + + if (aes_cmac_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != + 0) + panic("aes: CMAC FIPS self-test failed (preparekey)\n"); + aes_cmac(&key, fips_test_data, sizeof(fips_test_data), mac); + if (memcmp(fips_test_aes_cmac_value, mac, sizeof(mac)) != 0) + panic("aes: CMAC FIPS self-test failed (wrong MAC)\n"); + memzero_explicit(&key, sizeof(key)); +} +#else /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ +static inline void aes_cmac_fips_test(void) +{ +} +#endif /* !CONFIG_CRYPTO_LIB_AES_CBC_MACS */ + static int __init aes_mod_init(void) { +#ifdef aes_mod_init_arch aes_mod_init_arch(); +#endif + if (fips_enabled) + aes_cmac_fips_test(); return 0; } subsys_initcall(aes_mod_init); @@ -524,7 +752,6 @@ static void __exit aes_mod_exit(void) { } module_exit(aes_mod_exit); -#endif MODULE_DESCRIPTION("AES block cipher"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); diff --git a/lib/crypto/aescfb.c b/lib/crypto/aescfb.c index 147e5211728f..82cd55436055 100644 --- a/lib/crypto/aescfb.c +++ b/lib/crypto/aescfb.c @@ -6,28 +6,9 @@ */ #include <crypto/aes.h> -#include <crypto/algapi.h> +#include <crypto/utils.h> #include <linux/export.h> #include <linux/module.h> -#include <asm/irqflags.h> - -static void aescfb_encrypt_block(const struct aes_enckey *key, void *dst, - const void *src) -{ - unsigned long flags; - - /* - * In AES-CFB, the AES encryption operates on known 'plaintext' (the IV - * and ciphertext), making it susceptible to timing attacks on the - * encryption key. The AES library already mitigates this risk to some - * extent by pulling the entire S-box into the caches before doing any - * substitutions, but this strategy is more effective when running with - * interrupts disabled. - */ - local_irq_save(flags); - aes_encrypt(key, dst, src); - local_irq_restore(flags); -} /** * aescfb_encrypt - Perform AES-CFB encryption on a block of data @@ -45,7 +26,7 @@ void aescfb_encrypt(const struct aes_enckey *key, u8 *dst, const u8 *src, const u8 *v = iv; while (len > 0) { - aescfb_encrypt_block(key, ks, v); + aes_encrypt(key, ks, v); crypto_xor_cpy(dst, src, ks, min(len, AES_BLOCK_SIZE)); v = dst; @@ -72,7 +53,7 @@ void aescfb_decrypt(const struct aes_enckey *key, u8 *dst, const u8 *src, { u8 ks[2][AES_BLOCK_SIZE]; - aescfb_encrypt_block(key, ks[0], iv); + aes_encrypt(key, ks[0], iv); for (int i = 0; len > 0; i ^= 1) { if (len > AES_BLOCK_SIZE) @@ -81,7 +62,7 @@ void aescfb_decrypt(const struct aes_enckey *key, u8 *dst, const u8 *src, * performing the XOR, as that may update in place and * overwrite the ciphertext. */ - aescfb_encrypt_block(key, ks[!i], src); + aes_encrypt(key, ks[!i], src); crypto_xor_cpy(dst, src, ks[i], min(len, AES_BLOCK_SIZE)); diff --git a/lib/crypto/aesgcm.c b/lib/crypto/aesgcm.c index 02f5b5f32c76..1da31e1f747d 100644 --- a/lib/crypto/aesgcm.c +++ b/lib/crypto/aesgcm.c @@ -5,30 +5,10 @@ * Copyright 2022 Google LLC */ -#include <crypto/algapi.h> #include <crypto/gcm.h> -#include <crypto/ghash.h> +#include <crypto/utils.h> #include <linux/export.h> #include <linux/module.h> -#include <asm/irqflags.h> - -static void aesgcm_encrypt_block(const struct aes_enckey *key, void *dst, - const void *src) -{ - unsigned long flags; - - /* - * In AES-GCM, both the GHASH key derivation and the CTR mode - * encryption operate on known plaintext, making them susceptible to - * timing attacks on the encryption key. The AES library already - * mitigates this risk to some extent by pulling the entire S-box into - * the caches before doing any substitutions, but this strategy is more - * effective when running with interrupts disabled. - */ - local_irq_save(flags); - aes_encrypt(key, dst, src); - local_irq_restore(flags); -} /** * aesgcm_expandkey - Expands the AES and GHASH keys for the AES-GCM key @@ -45,7 +25,7 @@ static void aesgcm_encrypt_block(const struct aes_enckey *key, void *dst, int aesgcm_expandkey(struct aesgcm_ctx *ctx, const u8 *key, unsigned int keysize, unsigned int authsize) { - u8 kin[AES_BLOCK_SIZE] = {}; + u8 h[AES_BLOCK_SIZE] = {}; int ret; ret = crypto_gcm_check_authsize(authsize) ?: @@ -54,24 +34,13 @@ int aesgcm_expandkey(struct aesgcm_ctx *ctx, const u8 *key, return ret; ctx->authsize = authsize; - aesgcm_encrypt_block(&ctx->aes_key, &ctx->ghash_key, kin); - + aes_encrypt(&ctx->aes_key, h, h); + ghash_preparekey(&ctx->ghash_key, h); + memzero_explicit(h, sizeof(h)); return 0; } EXPORT_SYMBOL(aesgcm_expandkey); -static void aesgcm_ghash(be128 *ghash, const be128 *key, const void *src, - int len) -{ - while (len > 0) { - crypto_xor((u8 *)ghash, src, min(len, GHASH_BLOCK_SIZE)); - gf128mul_lle(ghash, key); - - src += GHASH_BLOCK_SIZE; - len -= GHASH_BLOCK_SIZE; - } -} - /** * aesgcm_mac - Generates the authentication tag using AES-GCM algorithm. * @ctx: The data structure that will hold the AES-GCM key schedule @@ -88,20 +57,33 @@ static void aesgcm_ghash(be128 *ghash, const be128 *key, const void *src, static void aesgcm_mac(const struct aesgcm_ctx *ctx, const u8 *src, int src_len, const u8 *assoc, int assoc_len, __be32 *ctr, u8 *authtag) { - be128 tail = { cpu_to_be64(assoc_len * 8), cpu_to_be64(src_len * 8) }; - u8 buf[AES_BLOCK_SIZE]; - be128 ghash = {}; + static const u8 zeroes[GHASH_BLOCK_SIZE]; + __be64 tail[2] = { + cpu_to_be64((u64)assoc_len * 8), + cpu_to_be64((u64)src_len * 8), + }; + struct ghash_ctx ghash; + u8 ghash_out[AES_BLOCK_SIZE]; + u8 enc_ctr[AES_BLOCK_SIZE]; + + ghash_init(&ghash, &ctx->ghash_key); - aesgcm_ghash(&ghash, &ctx->ghash_key, assoc, assoc_len); - aesgcm_ghash(&ghash, &ctx->ghash_key, src, src_len); - aesgcm_ghash(&ghash, &ctx->ghash_key, &tail, sizeof(tail)); + ghash_update(&ghash, assoc, assoc_len); + ghash_update(&ghash, zeroes, -assoc_len & (GHASH_BLOCK_SIZE - 1)); + + ghash_update(&ghash, src, src_len); + ghash_update(&ghash, zeroes, -src_len & (GHASH_BLOCK_SIZE - 1)); + + ghash_update(&ghash, (const u8 *)&tail, sizeof(tail)); + + ghash_final(&ghash, ghash_out); ctr[3] = cpu_to_be32(1); - aesgcm_encrypt_block(&ctx->aes_key, buf, ctr); - crypto_xor_cpy(authtag, buf, (u8 *)&ghash, ctx->authsize); + aes_encrypt(&ctx->aes_key, enc_ctr, (const u8 *)ctr); + crypto_xor_cpy(authtag, ghash_out, enc_ctr, ctx->authsize); - memzero_explicit(&ghash, sizeof(ghash)); - memzero_explicit(buf, sizeof(buf)); + memzero_explicit(ghash_out, sizeof(ghash_out)); + memzero_explicit(enc_ctr, sizeof(enc_ctr)); } static void aesgcm_crypt(const struct aesgcm_ctx *ctx, u8 *dst, const u8 *src, @@ -119,7 +101,7 @@ static void aesgcm_crypt(const struct aesgcm_ctx *ctx, u8 *dst, const u8 *src, * len', this cannot happen, so no explicit test is necessary. */ ctr[3] = cpu_to_be32(n++); - aesgcm_encrypt_block(&ctx->aes_key, buf, ctr); + aes_encrypt(&ctx->aes_key, buf, (const u8 *)ctr); crypto_xor_cpy(dst, src, buf, min(len, AES_BLOCK_SIZE)); dst += AES_BLOCK_SIZE; diff --git a/lib/crypto/arm/gf128hash.h b/lib/crypto/arm/gf128hash.h new file mode 100644 index 000000000000..c33c8cbe51fe --- /dev/null +++ b/lib/crypto/arm/gf128hash.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * GHASH, arm optimized + * + * Copyright 2026 Google LLC + */ + +#include <asm/hwcap.h> +#include <asm/neon.h> +#include <asm/simd.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); + +void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg, + const u8 *src, const struct polyval_elem *h); + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_neon) && may_use_simd()) { + do { + /* Allow rescheduling every 4 KiB. */ + size_t n = + min_t(size_t, nblocks, 4096 / GHASH_BLOCK_SIZE); + + scoped_ksimd() + pmull_ghash_update_p8(n, acc, data, &key->h); + data += n * GHASH_BLOCK_SIZE; + nblocks -= n; + } while (nblocks); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (elf_hwcap & HWCAP_NEON) + static_branch_enable(&have_neon); +} diff --git a/lib/crypto/arm/ghash-neon-core.S b/lib/crypto/arm/ghash-neon-core.S new file mode 100644 index 000000000000..eeffd12504a9 --- /dev/null +++ b/lib/crypto/arm/ghash-neon-core.S @@ -0,0 +1,209 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Accelerated GHASH implementation with NEON vmull.p8 instructions. + * + * Copyright (C) 2015 - 2017 Linaro Ltd. + * Copyright (C) 2023 Google LLC. <ardb@google.com> + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> + + .fpu neon + + SHASH .req q0 + T1 .req q1 + XL .req q2 + XM .req q3 + XH .req q4 + IN1 .req q4 + + SHASH_L .req d0 + SHASH_H .req d1 + T1_L .req d2 + T1_H .req d3 + XL_L .req d4 + XL_H .req d5 + XM_L .req d6 + XM_H .req d7 + XH_L .req d8 + + t0l .req d10 + t0h .req d11 + t1l .req d12 + t1h .req d13 + t2l .req d14 + t2h .req d15 + t3l .req d16 + t3h .req d17 + t4l .req d18 + t4h .req d19 + + t0q .req q5 + t1q .req q6 + t2q .req q7 + t3q .req q8 + t4q .req q9 + + s1l .req d20 + s1h .req d21 + s2l .req d22 + s2h .req d23 + s3l .req d24 + s3h .req d25 + s4l .req d26 + s4h .req d27 + + SHASH2_p8 .req d28 + + k16 .req d29 + k32 .req d30 + k48 .req d31 + + T2 .req q7 + + .text + + /* + * This implementation of 64x64 -> 128 bit polynomial multiplication + * using vmull.p8 instructions (8x8 -> 16) is taken from the paper + * "Fast Software Polynomial Multiplication on ARM Processors Using + * the NEON Engine" by Danilo Camara, Conrado Gouvea, Julio Lopez and + * Ricardo Dahab (https://hal.inria.fr/hal-01506572) + * + * It has been slightly tweaked for in-order performance, and to allow + * 'rq' to overlap with 'ad' or 'bd'. + */ + .macro __pmull_p8, rq, ad, bd, b1=t4l, b2=t3l, b3=t4l, b4=t3l + vext.8 t0l, \ad, \ad, #1 @ A1 + .ifc \b1, t4l + vext.8 t4l, \bd, \bd, #1 @ B1 + .endif + vmull.p8 t0q, t0l, \bd @ F = A1*B + vext.8 t1l, \ad, \ad, #2 @ A2 + vmull.p8 t4q, \ad, \b1 @ E = A*B1 + .ifc \b2, t3l + vext.8 t3l, \bd, \bd, #2 @ B2 + .endif + vmull.p8 t1q, t1l, \bd @ H = A2*B + vext.8 t2l, \ad, \ad, #3 @ A3 + vmull.p8 t3q, \ad, \b2 @ G = A*B2 + veor t0q, t0q, t4q @ L = E + F + .ifc \b3, t4l + vext.8 t4l, \bd, \bd, #3 @ B3 + .endif + vmull.p8 t2q, t2l, \bd @ J = A3*B + veor t0l, t0l, t0h @ t0 = (L) (P0 + P1) << 8 + veor t1q, t1q, t3q @ M = G + H + .ifc \b4, t3l + vext.8 t3l, \bd, \bd, #4 @ B4 + .endif + vmull.p8 t4q, \ad, \b3 @ I = A*B3 + veor t1l, t1l, t1h @ t1 = (M) (P2 + P3) << 16 + vmull.p8 t3q, \ad, \b4 @ K = A*B4 + vand t0h, t0h, k48 + vand t1h, t1h, k32 + veor t2q, t2q, t4q @ N = I + J + veor t0l, t0l, t0h + veor t1l, t1l, t1h + veor t2l, t2l, t2h @ t2 = (N) (P4 + P5) << 24 + vand t2h, t2h, k16 + veor t3l, t3l, t3h @ t3 = (K) (P6 + P7) << 32 + vmov.i64 t3h, #0 + vext.8 t0q, t0q, t0q, #15 + veor t2l, t2l, t2h + vext.8 t1q, t1q, t1q, #14 + vmull.p8 \rq, \ad, \bd @ D = A*B + vext.8 t2q, t2q, t2q, #13 + vext.8 t3q, t3q, t3q, #12 + veor t0q, t0q, t1q + veor t2q, t2q, t3q + veor \rq, \rq, t0q + veor \rq, \rq, t2q + .endm + + .macro __pmull_reduce_p8 + veor XL_H, XL_H, XM_L + veor XH_L, XH_L, XM_H + + vshl.i64 T1, XL, #57 + vshl.i64 T2, XL, #62 + veor T1, T1, T2 + vshl.i64 T2, XL, #63 + veor T1, T1, T2 + veor XL_H, XL_H, T1_L + veor XH_L, XH_L, T1_H + + vshr.u64 T1, XL, #1 + veor XH, XH, XL + veor XL, XL, T1 + vshr.u64 T1, T1, #6 + vshr.u64 XL, XL, #1 + .endm + + .macro vrev64_if_be a +#ifdef CONFIG_CPU_BIG_ENDIAN + vrev64.8 \a, \a +#endif + .endm + + .macro ghash_update + vld1.64 {XL}, [r1] + vrev64_if_be XL + +0: + vld1.8 {T1}, [r2]! + subs r0, r0, #1 + + /* multiply XL by SHASH in GF(2^128) */ + vrev64.8 T1, T1 + + vext.8 IN1, T1, T1, #8 + veor T1_L, T1_L, XL_H + veor XL, XL, IN1 + + __pmull_p8 XH, XL_H, SHASH_H, s1h, s2h, s3h, s4h @ a1 * b1 + veor T1, T1, XL + __pmull_p8 XL, XL_L, SHASH_L, s1l, s2l, s3l, s4l @ a0 * b0 + __pmull_p8 XM, T1_L, SHASH2_p8 @ (a1+a0)(b1+b0) + + veor T1, XL, XH + veor XM, XM, T1 + + __pmull_reduce_p8 + + veor T1, T1, XH + veor XL, XL, T1 + + bne 0b + .endm + + /* + * void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg, + * const u8 *src, + * const struct polyval_elem *h) + */ +ENTRY(pmull_ghash_update_p8) + vld1.64 {SHASH}, [r3] + vrev64_if_be SHASH + veor SHASH2_p8, SHASH_L, SHASH_H + + vext.8 s1l, SHASH_L, SHASH_L, #1 + vext.8 s2l, SHASH_L, SHASH_L, #2 + vext.8 s3l, SHASH_L, SHASH_L, #3 + vext.8 s4l, SHASH_L, SHASH_L, #4 + vext.8 s1h, SHASH_H, SHASH_H, #1 + vext.8 s2h, SHASH_H, SHASH_H, #2 + vext.8 s3h, SHASH_H, SHASH_H, #3 + vext.8 s4h, SHASH_H, SHASH_H, #4 + + vmov.i64 k16, #0xffff + vmov.i64 k32, #0xffffffff + vmov.i64 k48, #0xffffffffffff + + ghash_update + vrev64_if_be XL + vst1.64 {XL}, [r1] + + bx lr +ENDPROC(pmull_ghash_update_p8) diff --git a/arch/arm64/crypto/aes-ce.S b/lib/crypto/arm64/aes-ce.S index b262eaa9170c..b853e02f7b1e 100644 --- a/arch/arm64/crypto/aes-ce.S +++ b/lib/crypto/arm64/aes-ce.S @@ -1,7 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * linux/arch/arm64/crypto/aes-ce.S - AES cipher for ARMv8 with - * Crypto Extensions + * AES cipher for ARMv8 with Crypto Extensions * * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> */ diff --git a/lib/crypto/arm64/aes-cipher-core.S b/lib/crypto/arm64/aes-cipher-core.S index 651f701c56a8..0b05ec4be65f 100644 --- a/lib/crypto/arm64/aes-cipher-core.S +++ b/lib/crypto/arm64/aes-cipher-core.S @@ -87,11 +87,6 @@ ldp w8, w9, [rk], #16 ldp w10, w11, [rk, #-8] -CPU_BE( rev w4, w4 ) -CPU_BE( rev w5, w5 ) -CPU_BE( rev w6, w6 ) -CPU_BE( rev w7, w7 ) - eor w4, w4, w8 eor w5, w5, w9 eor w6, w6, w10 @@ -112,11 +107,6 @@ CPU_BE( rev w7, w7 ) 3: adr_l tt, \ltab \round w4, w5, w6, w7, w8, w9, w10, w11, \bsz, b -CPU_BE( rev w4, w4 ) -CPU_BE( rev w5, w5 ) -CPU_BE( rev w6, w6 ) -CPU_BE( rev w7, w7 ) - stp w4, w5, [out] stp w6, w7, [out, #8] ret diff --git a/arch/arm64/crypto/aes-modes.S b/lib/crypto/arm64/aes-modes.S index 0e834a2c062c..10e537317eaf 100644 --- a/arch/arm64/crypto/aes-modes.S +++ b/lib/crypto/arm64/aes-modes.S @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * linux/arch/arm64/crypto/aes-modes.S - chaining mode wrappers for AES + * Chaining mode wrappers for AES * * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> */ @@ -815,9 +815,11 @@ AES_FUNC_START(aes_xts_decrypt) b .Lxtsdecctsout AES_FUNC_END(aes_xts_decrypt) +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) /* - * aes_mac_update(u8 const in[], u32 const rk[], int rounds, - * int blocks, u8 dg[], int enc_before, int enc_after) + * void aes_mac_update(u8 const in[], u32 const rk[], int rounds, + * size_t blocks, u8 dg[], int enc_before, + * int enc_after); */ AES_FUNC_START(aes_mac_update) ld1 {v0.16b}, [x4] /* get dg */ @@ -827,7 +829,7 @@ AES_FUNC_START(aes_mac_update) encrypt_block v0, w2, x1, x7, w8 .Lmacloop4x: - subs w3, w3, #4 + subs x3, x3, #4 bmi .Lmac1x ld1 {v1.16b-v4.16b}, [x0], #64 /* get next pt block */ eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */ @@ -837,22 +839,21 @@ AES_FUNC_START(aes_mac_update) eor v0.16b, v0.16b, v3.16b encrypt_block v0, w2, x1, x7, w8 eor v0.16b, v0.16b, v4.16b - cmp w3, wzr - csinv x5, x6, xzr, eq + cmp x3, xzr + csinv w5, w6, wzr, eq cbz w5, .Lmacout encrypt_block v0, w2, x1, x7, w8 st1 {v0.16b}, [x4] /* return dg */ - cond_yield .Lmacout, x7, x8 b .Lmacloop4x .Lmac1x: - add w3, w3, #4 + add x3, x3, #4 .Lmacloop: - cbz w3, .Lmacout + cbz x3, .Lmacout ld1 {v1.16b}, [x0], #16 /* get next pt block */ eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */ - subs w3, w3, #1 - csinv x5, x6, xzr, eq + subs x3, x3, #1 + csinv w5, w6, wzr, eq cbz w5, .Lmacout .Lmacenc: @@ -861,6 +862,6 @@ AES_FUNC_START(aes_mac_update) .Lmacout: st1 {v0.16b}, [x4] /* return dg */ - mov w0, w3 ret AES_FUNC_END(aes_mac_update) +#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ diff --git a/arch/arm64/crypto/aes-neon.S b/lib/crypto/arm64/aes-neon.S index 3a8961b6ea51..f37b1dbd887f 100644 --- a/arch/arm64/crypto/aes-neon.S +++ b/lib/crypto/arm64/aes-neon.S @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * linux/arch/arm64/crypto/aes-neon.S - AES cipher for ARMv8 NEON + * AES cipher for ARMv8 NEON * * Copyright (C) 2013 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> */ diff --git a/lib/crypto/arm64/aes.h b/lib/crypto/arm64/aes.h index 63eea6271ef9..9e9e45a6f787 100644 --- a/lib/crypto/arm64/aes.h +++ b/lib/crypto/arm64/aes.h @@ -11,6 +11,7 @@ #include <linux/unaligned.h> #include <linux/cpufeature.h> +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes); struct aes_block { @@ -28,6 +29,9 @@ asmlinkage void __aes_ce_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE], asmlinkage u32 __aes_ce_sub(u32 l); asmlinkage void __aes_ce_invert(struct aes_block *out, const struct aes_block *in); +asmlinkage void neon_aes_mac_update(u8 const in[], u32 const rk[], int rounds, + size_t blocks, u8 dg[], int enc_before, + int enc_after); /* * Expand an AES key using the crypto extensions if supported and usable or @@ -48,8 +52,7 @@ static void aes_expandkey_arm64(u32 rndkeys[], u32 *inv_rndkeys, struct aes_block *key_enc, *key_dec; int i, j; - if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || - !static_branch_likely(&have_aes) || unlikely(!may_use_simd())) { + if (!static_branch_likely(&have_aes) || unlikely(!may_use_simd())) { aes_expandkey_generic(rndkeys, inv_rndkeys, in_key, key_len); return; } @@ -126,12 +129,40 @@ int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, } EXPORT_SYMBOL(ce_aes_expandkey); +EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_cts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_cts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_ctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_xctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_xts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_xts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL"); + +EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_cts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_cts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_ctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_xctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_xts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_xts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL"); +#if IS_MODULE(CONFIG_CRYPTO_AES_ARM64_CE_CCM) +EXPORT_SYMBOL_NS_GPL(ce_aes_mac_update, "CRYPTO_INTERNAL"); +#endif + static void aes_encrypt_arch(const struct aes_enckey *key, u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_aes) && likely(may_use_simd())) { + if (static_branch_likely(&have_aes) && likely(may_use_simd())) { scoped_ksimd() __aes_ce_encrypt(key->k.rndkeys, out, in, key->nrounds); } else { @@ -143,8 +174,7 @@ static void aes_decrypt_arch(const struct aes_key *key, u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_aes) && likely(may_use_simd())) { + if (static_branch_likely(&have_aes) && likely(may_use_simd())) { scoped_ksimd() __aes_ce_decrypt(key->inv_k.inv_rndkeys, out, in, key->nrounds); @@ -154,11 +184,36 @@ static void aes_decrypt_arch(const struct aes_key *key, } } -#ifdef CONFIG_KERNEL_MODE_NEON +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) +#define aes_cbcmac_blocks_arch aes_cbcmac_blocks_arch +static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], + const struct aes_enckey *key, const u8 *data, + size_t nblocks, bool enc_before, + bool enc_after) +{ + if (static_branch_likely(&have_neon) && likely(may_use_simd())) { + scoped_ksimd() { + if (static_branch_likely(&have_aes)) + ce_aes_mac_update(data, key->k.rndkeys, + key->nrounds, nblocks, h, + enc_before, enc_after); + else + neon_aes_mac_update(data, key->k.rndkeys, + key->nrounds, nblocks, h, + enc_before, enc_after); + } + return true; + } + return false; +} +#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ + #define aes_mod_init_arch aes_mod_init_arch static void aes_mod_init_arch(void) { - if (cpu_have_named_feature(AES)) - static_branch_enable(&have_aes); + if (cpu_have_named_feature(ASIMD)) { + static_branch_enable(&have_neon); + if (cpu_have_named_feature(AES)) + static_branch_enable(&have_aes); + } } -#endif /* CONFIG_KERNEL_MODE_NEON */ diff --git a/lib/crypto/arm64/chacha-neon-core.S b/lib/crypto/arm64/chacha-neon-core.S index 80079586ecc7..cb18eec968bd 100644 --- a/lib/crypto/arm64/chacha-neon-core.S +++ b/lib/crypto/arm64/chacha-neon-core.S @@ -531,10 +531,6 @@ SYM_FUNC_START(chacha_4block_xor_neon) add v3.4s, v3.4s, v19.4s add a2, a2, w8 add a3, a3, w9 -CPU_BE( rev a0, a0 ) -CPU_BE( rev a1, a1 ) -CPU_BE( rev a2, a2 ) -CPU_BE( rev a3, a3 ) ld4r {v24.4s-v27.4s}, [x0], #16 ld4r {v28.4s-v31.4s}, [x0] @@ -555,10 +551,6 @@ CPU_BE( rev a3, a3 ) add v7.4s, v7.4s, v23.4s add a6, a6, w8 add a7, a7, w9 -CPU_BE( rev a4, a4 ) -CPU_BE( rev a5, a5 ) -CPU_BE( rev a6, a6 ) -CPU_BE( rev a7, a7 ) // x8[0-3] += s2[0] // x9[0-3] += s2[1] @@ -576,10 +568,6 @@ CPU_BE( rev a7, a7 ) add v11.4s, v11.4s, v27.4s add a10, a10, w8 add a11, a11, w9 -CPU_BE( rev a8, a8 ) -CPU_BE( rev a9, a9 ) -CPU_BE( rev a10, a10 ) -CPU_BE( rev a11, a11 ) // x12[0-3] += s3[0] // x13[0-3] += s3[1] @@ -597,10 +585,6 @@ CPU_BE( rev a11, a11 ) add v15.4s, v15.4s, v31.4s add a14, a14, w8 add a15, a15, w9 -CPU_BE( rev a12, a12 ) -CPU_BE( rev a13, a13 ) -CPU_BE( rev a14, a14 ) -CPU_BE( rev a15, a15 ) // interleave 32-bit words in state n, n+1 ldp w6, w7, [x2], #64 diff --git a/lib/crypto/arm64/chacha.h b/lib/crypto/arm64/chacha.h index ca8c6a8b0578..c6f8ddf98e2d 100644 --- a/lib/crypto/arm64/chacha.h +++ b/lib/crypto/arm64/chacha.h @@ -36,9 +36,9 @@ asmlinkage void hchacha_block_neon(const struct chacha_state *state, static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static void chacha_doneon(struct chacha_state *state, u8 *dst, const u8 *src, - int bytes, int nrounds) + unsigned int bytes, int nrounds) { - while (bytes > 0) { + while (bytes) { int l = min(bytes, CHACHA_BLOCK_SIZE * 5); if (l <= CHACHA_BLOCK_SIZE) { @@ -76,16 +76,8 @@ static void chacha_crypt_arch(struct chacha_state *state, u8 *dst, !crypto_simd_usable()) return chacha_crypt_generic(state, dst, src, bytes, nrounds); - do { - unsigned int todo = min_t(unsigned int, bytes, SZ_4K); - - scoped_ksimd() - chacha_doneon(state, dst, src, todo, nrounds); - - bytes -= todo; - src += todo; - dst += todo; - } while (bytes); + scoped_ksimd() + chacha_doneon(state, dst, src, bytes, nrounds); } #define chacha_mod_init_arch chacha_mod_init_arch diff --git a/lib/crypto/arm64/gf128hash.h b/lib/crypto/arm64/gf128hash.h new file mode 100644 index 000000000000..1d1179f87f8d --- /dev/null +++ b/lib/crypto/arm64/gf128hash.h @@ -0,0 +1,121 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * GHASH and POLYVAL, arm64 optimized + * + * Copyright 2025 Google LLC + */ +#include <asm/simd.h> +#include <linux/cpufeature.h> + +#define NUM_H_POWERS 8 + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_asimd); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); + +asmlinkage void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg, + const u8 *src, + const struct polyval_elem *h); +asmlinkage void polyval_mul_pmull(struct polyval_elem *a, + const struct polyval_elem *b); +asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc, + const struct polyval_key *key, + const u8 *data, size_t nblocks); + +#define polyval_preparekey_arch polyval_preparekey_arch +static void polyval_preparekey_arch(struct polyval_key *key, + const u8 raw_key[POLYVAL_BLOCK_SIZE]) +{ + static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS); + memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE); + if (static_branch_likely(&have_pmull) && may_use_simd()) { + scoped_ksimd() { + for (int i = NUM_H_POWERS - 2; i >= 0; i--) { + key->h_powers[i] = key->h_powers[i + 1]; + polyval_mul_pmull( + &key->h_powers[i], + &key->h_powers[NUM_H_POWERS - 1]); + } + } + } else { + for (int i = NUM_H_POWERS - 2; i >= 0; i--) { + key->h_powers[i] = key->h_powers[i + 1]; + polyval_mul_generic(&key->h_powers[i], + &key->h_powers[NUM_H_POWERS - 1]); + } + } +} + +static void polyval_mul_arm64(struct polyval_elem *a, + const struct polyval_elem *b) +{ + if (static_branch_likely(&have_asimd) && may_use_simd()) { + static const u8 zeroes[GHASH_BLOCK_SIZE]; + + scoped_ksimd() { + if (static_branch_likely(&have_pmull)) { + polyval_mul_pmull(a, b); + } else { + /* + * Note that this is indeed equivalent to a + * POLYVAL multiplication, since it takes the + * accumulator and key in POLYVAL format, and + * byte-swapping a block of zeroes is a no-op. + */ + pmull_ghash_update_p8(1, a, zeroes, b); + } + } + } else { + polyval_mul_generic(a, b); + } +} + +#define ghash_mul_arch ghash_mul_arch +static void ghash_mul_arch(struct polyval_elem *acc, + const struct ghash_key *key) +{ + polyval_mul_arm64(acc, &key->h); +} + +#define polyval_mul_arch polyval_mul_arch +static void polyval_mul_arch(struct polyval_elem *acc, + const struct polyval_key *key) +{ + polyval_mul_arm64(acc, &key->h_powers[NUM_H_POWERS - 1]); +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_asimd) && may_use_simd()) { + scoped_ksimd() + pmull_ghash_update_p8(nblocks, acc, data, &key->h); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define polyval_blocks_arch polyval_blocks_arch +static void polyval_blocks_arch(struct polyval_elem *acc, + const struct polyval_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_pmull) && may_use_simd()) { + scoped_ksimd() + polyval_blocks_pmull(acc, key, data, nblocks); + } else { + polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1], + data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (cpu_have_named_feature(ASIMD)) { + static_branch_enable(&have_asimd); + if (cpu_have_named_feature(PMULL)) + static_branch_enable(&have_pmull); + } +} diff --git a/lib/crypto/arm64/ghash-neon-core.S b/lib/crypto/arm64/ghash-neon-core.S new file mode 100644 index 000000000000..4c5799172b49 --- /dev/null +++ b/lib/crypto/arm64/ghash-neon-core.S @@ -0,0 +1,220 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Accelerated GHASH implementation with ARMv8 ASIMD instructions. + * + * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org> + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> + + SHASH .req v0 + SHASH2 .req v1 + T1 .req v2 + T2 .req v3 + XM .req v5 + XL .req v6 + XH .req v7 + IN1 .req v7 + + k00_16 .req v8 + k32_48 .req v9 + + t3 .req v10 + t4 .req v11 + t5 .req v12 + t6 .req v13 + t7 .req v14 + t8 .req v15 + t9 .req v16 + + perm1 .req v17 + perm2 .req v18 + perm3 .req v19 + + sh1 .req v20 + sh2 .req v21 + sh3 .req v22 + sh4 .req v23 + + ss1 .req v24 + ss2 .req v25 + ss3 .req v26 + ss4 .req v27 + + .text + + .macro __pmull_p8, rq, ad, bd + ext t3.8b, \ad\().8b, \ad\().8b, #1 // A1 + ext t5.8b, \ad\().8b, \ad\().8b, #2 // A2 + ext t7.8b, \ad\().8b, \ad\().8b, #3 // A3 + + __pmull_p8_\bd \rq, \ad + .endm + + .macro __pmull2_p8, rq, ad, bd + tbl t3.16b, {\ad\().16b}, perm1.16b // A1 + tbl t5.16b, {\ad\().16b}, perm2.16b // A2 + tbl t7.16b, {\ad\().16b}, perm3.16b // A3 + + __pmull2_p8_\bd \rq, \ad + .endm + + .macro __pmull_p8_SHASH, rq, ad + __pmull_p8_tail \rq, \ad\().8b, SHASH.8b, 8b,, sh1, sh2, sh3, sh4 + .endm + + .macro __pmull_p8_SHASH2, rq, ad + __pmull_p8_tail \rq, \ad\().8b, SHASH2.8b, 8b,, ss1, ss2, ss3, ss4 + .endm + + .macro __pmull2_p8_SHASH, rq, ad + __pmull_p8_tail \rq, \ad\().16b, SHASH.16b, 16b, 2, sh1, sh2, sh3, sh4 + .endm + + .macro __pmull_p8_tail, rq, ad, bd, nb, t, b1, b2, b3, b4 + pmull\t t3.8h, t3.\nb, \bd // F = A1*B + pmull\t t4.8h, \ad, \b1\().\nb // E = A*B1 + pmull\t t5.8h, t5.\nb, \bd // H = A2*B + pmull\t t6.8h, \ad, \b2\().\nb // G = A*B2 + pmull\t t7.8h, t7.\nb, \bd // J = A3*B + pmull\t t8.8h, \ad, \b3\().\nb // I = A*B3 + pmull\t t9.8h, \ad, \b4\().\nb // K = A*B4 + pmull\t \rq\().8h, \ad, \bd // D = A*B + + eor t3.16b, t3.16b, t4.16b // L = E + F + eor t5.16b, t5.16b, t6.16b // M = G + H + eor t7.16b, t7.16b, t8.16b // N = I + J + + uzp1 t4.2d, t3.2d, t5.2d + uzp2 t3.2d, t3.2d, t5.2d + uzp1 t6.2d, t7.2d, t9.2d + uzp2 t7.2d, t7.2d, t9.2d + + // t3 = (L) (P0 + P1) << 8 + // t5 = (M) (P2 + P3) << 16 + eor t4.16b, t4.16b, t3.16b + and t3.16b, t3.16b, k32_48.16b + + // t7 = (N) (P4 + P5) << 24 + // t9 = (K) (P6 + P7) << 32 + eor t6.16b, t6.16b, t7.16b + and t7.16b, t7.16b, k00_16.16b + + eor t4.16b, t4.16b, t3.16b + eor t6.16b, t6.16b, t7.16b + + zip2 t5.2d, t4.2d, t3.2d + zip1 t3.2d, t4.2d, t3.2d + zip2 t9.2d, t6.2d, t7.2d + zip1 t7.2d, t6.2d, t7.2d + + ext t3.16b, t3.16b, t3.16b, #15 + ext t5.16b, t5.16b, t5.16b, #14 + ext t7.16b, t7.16b, t7.16b, #13 + ext t9.16b, t9.16b, t9.16b, #12 + + eor t3.16b, t3.16b, t5.16b + eor t7.16b, t7.16b, t9.16b + eor \rq\().16b, \rq\().16b, t3.16b + eor \rq\().16b, \rq\().16b, t7.16b + .endm + + .macro __pmull_pre_p8 + ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 + eor SHASH2.16b, SHASH2.16b, SHASH.16b + + // k00_16 := 0x0000000000000000_000000000000ffff + // k32_48 := 0x00000000ffffffff_0000ffffffffffff + movi k32_48.2d, #0xffffffff + mov k32_48.h[2], k32_48.h[0] + ushr k00_16.2d, k32_48.2d, #32 + + // prepare the permutation vectors + mov_q x5, 0x080f0e0d0c0b0a09 + movi T1.8b, #8 + dup perm1.2d, x5 + eor perm1.16b, perm1.16b, T1.16b + ushr perm2.2d, perm1.2d, #8 + ushr perm3.2d, perm1.2d, #16 + ushr T1.2d, perm1.2d, #24 + sli perm2.2d, perm1.2d, #56 + sli perm3.2d, perm1.2d, #48 + sli T1.2d, perm1.2d, #40 + + // precompute loop invariants + tbl sh1.16b, {SHASH.16b}, perm1.16b + tbl sh2.16b, {SHASH.16b}, perm2.16b + tbl sh3.16b, {SHASH.16b}, perm3.16b + tbl sh4.16b, {SHASH.16b}, T1.16b + ext ss1.8b, SHASH2.8b, SHASH2.8b, #1 + ext ss2.8b, SHASH2.8b, SHASH2.8b, #2 + ext ss3.8b, SHASH2.8b, SHASH2.8b, #3 + ext ss4.8b, SHASH2.8b, SHASH2.8b, #4 + .endm + + .macro __pmull_reduce_p8 + eor XM.16b, XM.16b, T1.16b + + mov XL.d[1], XM.d[0] + mov XH.d[0], XM.d[1] + + shl T1.2d, XL.2d, #57 + shl T2.2d, XL.2d, #62 + eor T2.16b, T2.16b, T1.16b + shl T1.2d, XL.2d, #63 + eor T2.16b, T2.16b, T1.16b + ext T1.16b, XL.16b, XH.16b, #8 + eor T2.16b, T2.16b, T1.16b + + mov XL.d[1], T2.d[0] + mov XH.d[0], T2.d[1] + + ushr T2.2d, XL.2d, #1 + eor XH.16b, XH.16b, XL.16b + eor XL.16b, XL.16b, T2.16b + ushr T2.2d, T2.2d, #6 + ushr XL.2d, XL.2d, #1 + .endm + + /* + * void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg, + * const u8 *src, + * const struct polyval_elem *h) + */ +SYM_FUNC_START(pmull_ghash_update_p8) + ld1 {SHASH.2d}, [x3] + ld1 {XL.2d}, [x1] + + __pmull_pre_p8 + +0: ld1 {T1.2d}, [x2], #16 + sub x0, x0, #1 + + /* multiply XL by SHASH in GF(2^128) */ + rev64 T1.16b, T1.16b + + ext T2.16b, XL.16b, XL.16b, #8 + ext IN1.16b, T1.16b, T1.16b, #8 + eor T1.16b, T1.16b, T2.16b + eor XL.16b, XL.16b, IN1.16b + + __pmull2_p8 XH, XL, SHASH // a1 * b1 + eor T1.16b, T1.16b, XL.16b + __pmull_p8 XL, XL, SHASH // a0 * b0 + __pmull_p8 XM, T1, SHASH2 // (a1 + a0)(b1 + b0) + + eor T2.16b, XL.16b, XH.16b + ext T1.16b, XL.16b, XH.16b, #8 + eor XM.16b, XM.16b, T2.16b + + __pmull_reduce_p8 + + eor T2.16b, T2.16b, XH.16b + eor XL.16b, XL.16b, T2.16b + + cbnz x0, 0b + + st1 {XL.2d}, [x1] + ret +SYM_FUNC_END(pmull_ghash_update_p8) diff --git a/lib/crypto/arm64/poly1305.h b/lib/crypto/arm64/poly1305.h index b77669767cd6..3d4bde857699 100644 --- a/lib/crypto/arm64/poly1305.h +++ b/lib/crypto/arm64/poly1305.h @@ -27,17 +27,11 @@ static void poly1305_blocks(struct poly1305_block_state *state, const u8 *src, unsigned int len, u32 padbit) { if (static_branch_likely(&have_neon) && likely(may_use_simd())) { - do { - unsigned int todo = min_t(unsigned int, len, SZ_4K); - - scoped_ksimd() - poly1305_blocks_neon(state, src, todo, padbit); - - len -= todo; - src += todo; - } while (len); - } else + scoped_ksimd() + poly1305_blocks_neon(state, src, len, padbit); + } else { poly1305_blocks_arm64(state, src, len, padbit); + } } #define poly1305_mod_init_arch poly1305_mod_init_arch diff --git a/lib/crypto/arm64/polyval.h b/lib/crypto/arm64/polyval.h deleted file mode 100644 index a39763395e9b..000000000000 --- a/lib/crypto/arm64/polyval.h +++ /dev/null @@ -1,80 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * POLYVAL library functions, arm64 optimized - * - * Copyright 2025 Google LLC - */ -#include <asm/simd.h> -#include <linux/cpufeature.h> - -#define NUM_H_POWERS 8 - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); - -asmlinkage void polyval_mul_pmull(struct polyval_elem *a, - const struct polyval_elem *b); -asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc, - const struct polyval_key *key, - const u8 *data, size_t nblocks); - -static void polyval_preparekey_arch(struct polyval_key *key, - const u8 raw_key[POLYVAL_BLOCK_SIZE]) -{ - static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS); - memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE); - if (static_branch_likely(&have_pmull) && may_use_simd()) { - scoped_ksimd() { - for (int i = NUM_H_POWERS - 2; i >= 0; i--) { - key->h_powers[i] = key->h_powers[i + 1]; - polyval_mul_pmull( - &key->h_powers[i], - &key->h_powers[NUM_H_POWERS - 1]); - } - } - } else { - for (int i = NUM_H_POWERS - 2; i >= 0; i--) { - key->h_powers[i] = key->h_powers[i + 1]; - polyval_mul_generic(&key->h_powers[i], - &key->h_powers[NUM_H_POWERS - 1]); - } - } -} - -static void polyval_mul_arch(struct polyval_elem *acc, - const struct polyval_key *key) -{ - if (static_branch_likely(&have_pmull) && may_use_simd()) { - scoped_ksimd() - polyval_mul_pmull(acc, &key->h_powers[NUM_H_POWERS - 1]); - } else { - polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]); - } -} - -static void polyval_blocks_arch(struct polyval_elem *acc, - const struct polyval_key *key, - const u8 *data, size_t nblocks) -{ - if (static_branch_likely(&have_pmull) && may_use_simd()) { - do { - /* Allow rescheduling every 4 KiB. */ - size_t n = min_t(size_t, nblocks, - 4096 / POLYVAL_BLOCK_SIZE); - - scoped_ksimd() - polyval_blocks_pmull(acc, key, data, n); - data += n * POLYVAL_BLOCK_SIZE; - nblocks -= n; - } while (nblocks); - } else { - polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1], - data, nblocks); - } -} - -#define polyval_mod_init_arch polyval_mod_init_arch -static void polyval_mod_init_arch(void) -{ - if (cpu_have_named_feature(PMULL)) - static_branch_enable(&have_pmull); -} diff --git a/lib/crypto/arm64/sha1-ce-core.S b/lib/crypto/arm64/sha1-ce-core.S index 8fbd4767f0f0..40f2a6c8d0c5 100644 --- a/lib/crypto/arm64/sha1-ce-core.S +++ b/lib/crypto/arm64/sha1-ce-core.S @@ -62,10 +62,10 @@ .endm /* - * size_t __sha1_ce_transform(struct sha1_block_state *state, - * const u8 *data, size_t nblocks); + * void sha1_ce_transform(struct sha1_block_state *state, + * const u8 *data, size_t nblocks); */ -SYM_FUNC_START(__sha1_ce_transform) +SYM_FUNC_START(sha1_ce_transform) /* load round constants */ loadrc k0.4s, 0x5a827999, w6 loadrc k1.4s, 0x6ed9eba1, w6 @@ -80,10 +80,10 @@ SYM_FUNC_START(__sha1_ce_transform) 0: ld1 {v8.4s-v11.4s}, [x1], #64 sub x2, x2, #1 -CPU_LE( rev32 v8.16b, v8.16b ) -CPU_LE( rev32 v9.16b, v9.16b ) -CPU_LE( rev32 v10.16b, v10.16b ) -CPU_LE( rev32 v11.16b, v11.16b ) + rev32 v8.16b, v8.16b + rev32 v9.16b, v9.16b + rev32 v10.16b, v10.16b + rev32 v11.16b, v11.16b add t0.4s, v8.4s, k0.4s mov dg0v.16b, dgav.16b @@ -116,15 +116,11 @@ CPU_LE( rev32 v11.16b, v11.16b ) add dgbv.2s, dgbv.2s, dg1v.2s add dgav.4s, dgav.4s, dg0v.4s - /* return early if voluntary preemption is needed */ - cond_yield 1f, x5, x6 - /* handled all input blocks? */ cbnz x2, 0b /* store new state */ -1: st1 {dgav.4s}, [x0] + st1 {dgav.4s}, [x0] str dgb, [x0, #16] - mov x0, x2 ret -SYM_FUNC_END(__sha1_ce_transform) +SYM_FUNC_END(sha1_ce_transform) diff --git a/lib/crypto/arm64/sha1.h b/lib/crypto/arm64/sha1.h index bc7071f1be09..112c5d443c56 100644 --- a/lib/crypto/arm64/sha1.h +++ b/lib/crypto/arm64/sha1.h @@ -9,22 +9,15 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); -asmlinkage size_t __sha1_ce_transform(struct sha1_block_state *state, - const u8 *data, size_t nblocks); +asmlinkage void sha1_ce_transform(struct sha1_block_state *state, + const u8 *data, size_t nblocks); static void sha1_blocks(struct sha1_block_state *state, const u8 *data, size_t nblocks) { if (static_branch_likely(&have_ce) && likely(may_use_simd())) { - do { - size_t rem; - - scoped_ksimd() - rem = __sha1_ce_transform(state, data, nblocks); - - data += (nblocks - rem) * SHA1_BLOCK_SIZE; - nblocks = rem; - } while (nblocks); + scoped_ksimd() + sha1_ce_transform(state, data, nblocks); } else { sha1_blocks_generic(state, data, nblocks); } diff --git a/lib/crypto/arm64/sha256-ce.S b/lib/crypto/arm64/sha256-ce.S index e4bfe42a61a9..8fdbf0a9ff89 100644 --- a/lib/crypto/arm64/sha256-ce.S +++ b/lib/crypto/arm64/sha256-ce.S @@ -79,11 +79,11 @@ .endm /* - * size_t __sha256_ce_transform(struct sha256_block_state *state, - * const u8 *data, size_t nblocks); + * void sha256_ce_transform(struct sha256_block_state *state, + * const u8 *data, size_t nblocks); */ .text -SYM_FUNC_START(__sha256_ce_transform) +SYM_FUNC_START(sha256_ce_transform) load_round_constants x8 @@ -94,10 +94,10 @@ SYM_FUNC_START(__sha256_ce_transform) 0: ld1 {v16.4s-v19.4s}, [x1], #64 sub x2, x2, #1 -CPU_LE( rev32 v16.16b, v16.16b ) -CPU_LE( rev32 v17.16b, v17.16b ) -CPU_LE( rev32 v18.16b, v18.16b ) -CPU_LE( rev32 v19.16b, v19.16b ) + rev32 v16.16b, v16.16b + rev32 v17.16b, v17.16b + rev32 v18.16b, v18.16b + rev32 v19.16b, v19.16b add t0.4s, v16.4s, v0.4s mov dg0v.16b, dgav.16b @@ -127,17 +127,13 @@ CPU_LE( rev32 v19.16b, v19.16b ) add dgav.4s, dgav.4s, dg0v.4s add dgbv.4s, dgbv.4s, dg1v.4s - /* return early if voluntary preemption is needed */ - cond_yield 1f, x5, x6 - /* handled all input blocks? */ cbnz x2, 0b /* store new state */ -1: st1 {dgav.4s, dgbv.4s}, [x0] - mov x0, x2 + st1 {dgav.4s, dgbv.4s}, [x0] ret -SYM_FUNC_END(__sha256_ce_transform) +SYM_FUNC_END(sha256_ce_transform) .unreq dga .unreq dgav @@ -293,14 +289,14 @@ SYM_FUNC_START(sha256_ce_finup2x) ld1 {v20.4s-v23.4s}, [data2], #64 .Lfinup2x_loop_have_data: // Convert the words of the data blocks from big endian. -CPU_LE( rev32 v16.16b, v16.16b ) -CPU_LE( rev32 v17.16b, v17.16b ) -CPU_LE( rev32 v18.16b, v18.16b ) -CPU_LE( rev32 v19.16b, v19.16b ) -CPU_LE( rev32 v20.16b, v20.16b ) -CPU_LE( rev32 v21.16b, v21.16b ) -CPU_LE( rev32 v22.16b, v22.16b ) -CPU_LE( rev32 v23.16b, v23.16b ) + rev32 v16.16b, v16.16b + rev32 v17.16b, v17.16b + rev32 v18.16b, v18.16b + rev32 v19.16b, v19.16b + rev32 v20.16b, v20.16b + rev32 v21.16b, v21.16b + rev32 v22.16b, v22.16b + rev32 v23.16b, v23.16b .Lfinup2x_loop_have_bswapped_data: // Save the original state for each block. @@ -340,11 +336,8 @@ CPU_LE( rev32 v23.16b, v23.16b ) sub w8, len, #64 // w8 = len - 64 add data1, data1, w8, sxtw // data1 += len - 64 add data2, data2, w8, sxtw // data2 += len - 64 -CPU_LE( mov x9, #0x80 ) -CPU_LE( fmov d16, x9 ) -CPU_BE( movi v16.16b, #0 ) -CPU_BE( mov x9, #0x8000000000000000 ) -CPU_BE( mov v16.d[1], x9 ) + mov x9, #0x80 + fmov d16, x9 movi v17.16b, #0 stp q16, q17, [sp, #64] stp q17, q17, [sp, #96] @@ -352,7 +345,7 @@ CPU_BE( mov v16.d[1], x9 ) cmp len, #56 b.ge 1f // will count spill into its own block? lsl count, count, #3 -CPU_LE( rev count, count ) + rev count, count str count, [x9, #56] mov final_step, #2 // won't need count-only block b 2f @@ -397,10 +390,10 @@ CPU_LE( rev count, count ) .Lfinup2x_done: // Write the two digests with all bytes in the correct order. -CPU_LE( rev32 state0_a.16b, state0_a.16b ) -CPU_LE( rev32 state1_a.16b, state1_a.16b ) -CPU_LE( rev32 state0_b.16b, state0_b.16b ) -CPU_LE( rev32 state1_b.16b, state1_b.16b ) + rev32 state0_a.16b, state0_a.16b + rev32 state1_a.16b, state1_a.16b + rev32 state0_b.16b, state0_b.16b + rev32 state1_b.16b, state1_b.16b st1 {state0_a.4s-state1_a.4s}, [out1] st1 {state0_b.4s-state1_b.4s}, [out2] add sp, sp, #128 diff --git a/lib/crypto/arm64/sha256.h b/lib/crypto/arm64/sha256.h index 568dff0f276a..b4353d3c4dd0 100644 --- a/lib/crypto/arm64/sha256.h +++ b/lib/crypto/arm64/sha256.h @@ -14,27 +14,17 @@ asmlinkage void sha256_block_data_order(struct sha256_block_state *state, const u8 *data, size_t nblocks); asmlinkage void sha256_block_neon(struct sha256_block_state *state, const u8 *data, size_t nblocks); -asmlinkage size_t __sha256_ce_transform(struct sha256_block_state *state, - const u8 *data, size_t nblocks); +asmlinkage void sha256_ce_transform(struct sha256_block_state *state, + const u8 *data, size_t nblocks); static void sha256_blocks(struct sha256_block_state *state, const u8 *data, size_t nblocks) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_neon) && likely(may_use_simd())) { - if (static_branch_likely(&have_ce)) { - do { - size_t rem; - - scoped_ksimd() - rem = __sha256_ce_transform(state, data, - nblocks); - - data += (nblocks - rem) * SHA256_BLOCK_SIZE; - nblocks = rem; - } while (nblocks); - } else { - scoped_ksimd() + if (static_branch_likely(&have_neon) && likely(may_use_simd())) { + scoped_ksimd() { + if (static_branch_likely(&have_ce)) + sha256_ce_transform(state, data, nblocks); + else sha256_block_neon(state, data, nblocks); } } else { @@ -56,14 +46,9 @@ static bool sha256_finup_2x_arch(const struct __sha256_ctx *ctx, u8 out1[SHA256_DIGEST_SIZE], u8 out2[SHA256_DIGEST_SIZE]) { - /* - * The assembly requires len >= SHA256_BLOCK_SIZE && len <= INT_MAX. - * Further limit len to 65536 to avoid spending too long with preemption - * disabled. (Of course, in practice len is nearly always 4096 anyway.) - */ - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_ce) && len >= SHA256_BLOCK_SIZE && - len <= 65536 && likely(may_use_simd())) { + /* The assembly requires len >= SHA256_BLOCK_SIZE && len <= INT_MAX. */ + if (static_branch_likely(&have_ce) && len >= SHA256_BLOCK_SIZE && + len <= INT_MAX && likely(may_use_simd())) { scoped_ksimd() sha256_ce_finup2x(ctx, data1, data2, len, out1, out2); kmsan_unpoison_memory(out1, SHA256_DIGEST_SIZE); @@ -78,7 +63,6 @@ static bool sha256_finup_2x_is_optimized_arch(void) return static_key_enabled(&have_ce); } -#ifdef CONFIG_KERNEL_MODE_NEON #define sha256_mod_init_arch sha256_mod_init_arch static void sha256_mod_init_arch(void) { @@ -88,4 +72,3 @@ static void sha256_mod_init_arch(void) static_branch_enable(&have_ce); } } -#endif /* CONFIG_KERNEL_MODE_NEON */ diff --git a/lib/crypto/arm64/sha3-ce-core.S b/lib/crypto/arm64/sha3-ce-core.S index ace90b506490..b8ab01987ae0 100644 --- a/lib/crypto/arm64/sha3-ce-core.S +++ b/lib/crypto/arm64/sha3-ce-core.S @@ -37,8 +37,8 @@ .endm /* - * size_t sha3_ce_transform(struct sha3_state *state, const u8 *data, - * size_t nblocks, size_t block_size) + * void sha3_ce_transform(struct sha3_state *state, const u8 *data, + * size_t nblocks, size_t block_size) * * block_size is assumed to be one of 72 (SHA3-512), 104 (SHA3-384), 136 * (SHA3-256 and SHAKE256), 144 (SHA3-224), or 168 (SHAKE128). @@ -185,18 +185,16 @@ SYM_FUNC_START(sha3_ce_transform) eor v0.16b, v0.16b, v31.16b cbnz w8, 3b - cond_yield 4f, x8, x9 cbnz x2, 0b /* save state */ -4: st1 { v0.1d- v3.1d}, [x0], #32 + st1 { v0.1d- v3.1d}, [x0], #32 st1 { v4.1d- v7.1d}, [x0], #32 st1 { v8.1d-v11.1d}, [x0], #32 st1 {v12.1d-v15.1d}, [x0], #32 st1 {v16.1d-v19.1d}, [x0], #32 st1 {v20.1d-v23.1d}, [x0], #32 st1 {v24.1d}, [x0] - mov x0, x2 ret SYM_FUNC_END(sha3_ce_transform) diff --git a/lib/crypto/arm64/sha3.h b/lib/crypto/arm64/sha3.h index b602f1b3b282..eaaba3224acc 100644 --- a/lib/crypto/arm64/sha3.h +++ b/lib/crypto/arm64/sha3.h @@ -12,22 +12,15 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3); -asmlinkage size_t sha3_ce_transform(struct sha3_state *state, const u8 *data, - size_t nblocks, size_t block_size); +asmlinkage void sha3_ce_transform(struct sha3_state *state, const u8 *data, + size_t nblocks, size_t block_size); static void sha3_absorb_blocks(struct sha3_state *state, const u8 *data, size_t nblocks, size_t block_size) { if (static_branch_likely(&have_sha3) && likely(may_use_simd())) { - do { - size_t rem; - - scoped_ksimd() - rem = sha3_ce_transform(state, data, nblocks, - block_size); - data += (nblocks - rem) * block_size; - nblocks = rem; - } while (nblocks); + scoped_ksimd() + sha3_ce_transform(state, data, nblocks, block_size); } else { sha3_absorb_blocks_generic(state, data, nblocks, block_size); } diff --git a/lib/crypto/arm64/sha512-ce-core.S b/lib/crypto/arm64/sha512-ce-core.S index ffd51acfd1ee..daa2c63a20c0 100644 --- a/lib/crypto/arm64/sha512-ce-core.S +++ b/lib/crypto/arm64/sha512-ce-core.S @@ -93,11 +93,11 @@ .endm /* - * size_t __sha512_ce_transform(struct sha512_block_state *state, - * const u8 *data, size_t nblocks); + * void sha512_ce_transform(struct sha512_block_state *state, + * const u8 *data, size_t nblocks); */ .text -SYM_FUNC_START(__sha512_ce_transform) +SYM_FUNC_START(sha512_ce_transform) /* load state */ ld1 {v8.2d-v11.2d}, [x0] @@ -110,14 +110,14 @@ SYM_FUNC_START(__sha512_ce_transform) ld1 {v16.2d-v19.2d}, [x1], #64 sub x2, x2, #1 -CPU_LE( rev64 v12.16b, v12.16b ) -CPU_LE( rev64 v13.16b, v13.16b ) -CPU_LE( rev64 v14.16b, v14.16b ) -CPU_LE( rev64 v15.16b, v15.16b ) -CPU_LE( rev64 v16.16b, v16.16b ) -CPU_LE( rev64 v17.16b, v17.16b ) -CPU_LE( rev64 v18.16b, v18.16b ) -CPU_LE( rev64 v19.16b, v19.16b ) + rev64 v12.16b, v12.16b + rev64 v13.16b, v13.16b + rev64 v14.16b, v14.16b + rev64 v15.16b, v15.16b + rev64 v16.16b, v16.16b + rev64 v17.16b, v17.16b + rev64 v18.16b, v18.16b + rev64 v19.16b, v19.16b mov x4, x3 // rc pointer @@ -186,12 +186,10 @@ CPU_LE( rev64 v19.16b, v19.16b ) add v10.2d, v10.2d, v2.2d add v11.2d, v11.2d, v3.2d - cond_yield 3f, x4, x5 /* handled all input blocks? */ cbnz x2, 0b /* store new state */ -3: st1 {v8.2d-v11.2d}, [x0] - mov x0, x2 + st1 {v8.2d-v11.2d}, [x0] ret -SYM_FUNC_END(__sha512_ce_transform) +SYM_FUNC_END(sha512_ce_transform) diff --git a/lib/crypto/arm64/sha512.h b/lib/crypto/arm64/sha512.h index 7eb7ef04d268..5da27e6e23ea 100644 --- a/lib/crypto/arm64/sha512.h +++ b/lib/crypto/arm64/sha512.h @@ -12,34 +12,24 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha512_insns); asmlinkage void sha512_block_data_order(struct sha512_block_state *state, const u8 *data, size_t nblocks); -asmlinkage size_t __sha512_ce_transform(struct sha512_block_state *state, - const u8 *data, size_t nblocks); +asmlinkage void sha512_ce_transform(struct sha512_block_state *state, + const u8 *data, size_t nblocks); static void sha512_blocks(struct sha512_block_state *state, const u8 *data, size_t nblocks) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_sha512_insns) && + if (static_branch_likely(&have_sha512_insns) && likely(may_use_simd())) { - do { - size_t rem; - - scoped_ksimd() - rem = __sha512_ce_transform(state, data, nblocks); - - data += (nblocks - rem) * SHA512_BLOCK_SIZE; - nblocks = rem; - } while (nblocks); + scoped_ksimd() + sha512_ce_transform(state, data, nblocks); } else { sha512_block_data_order(state, data, nblocks); } } -#ifdef CONFIG_KERNEL_MODE_NEON #define sha512_mod_init_arch sha512_mod_init_arch static void sha512_mod_init_arch(void) { if (cpu_have_named_feature(SHA512)) static_branch_enable(&have_sha512_insns); } -#endif /* CONFIG_KERNEL_MODE_NEON */ diff --git a/arch/arm64/crypto/sm3-ce-core.S b/lib/crypto/arm64/sm3-ce-core.S index ca70cfacd0d0..ee7f900d7cff 100644 --- a/arch/arm64/crypto/sm3-ce-core.S +++ b/lib/crypto/arm64/sm3-ce-core.S @@ -6,7 +6,6 @@ */ #include <linux/linkage.h> -#include <linux/cfi_types.h> #include <asm/assembler.h> .irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 @@ -70,11 +69,11 @@ .endm /* - * void sm3_ce_transform(struct sm3_state *sst, u8 const *src, - * int blocks) + * void sm3_ce_transform(struct sm3_block_state *state, + * const u8 *data, size_t nblocks) */ .text -SYM_TYPED_FUNC_START(sm3_ce_transform) +SYM_FUNC_START(sm3_ce_transform) /* load state */ ld1 {v8.4s-v9.4s}, [x0] rev64 v8.4s, v8.4s @@ -87,15 +86,15 @@ SYM_TYPED_FUNC_START(sm3_ce_transform) /* load input */ 0: ld1 {v0.16b-v3.16b}, [x1], #64 - sub w2, w2, #1 + sub x2, x2, #1 mov v15.16b, v8.16b mov v16.16b, v9.16b -CPU_LE( rev32 v0.16b, v0.16b ) -CPU_LE( rev32 v1.16b, v1.16b ) -CPU_LE( rev32 v2.16b, v2.16b ) -CPU_LE( rev32 v3.16b, v3.16b ) + rev32 v0.16b, v0.16b + rev32 v1.16b, v1.16b + rev32 v2.16b, v2.16b + rev32 v3.16b, v3.16b ext v11.16b, v13.16b, v13.16b, #4 @@ -123,7 +122,7 @@ CPU_LE( rev32 v3.16b, v3.16b ) eor v9.16b, v9.16b, v16.16b /* handled all input blocks? */ - cbnz w2, 0b + cbnz x2, 0b /* save state */ rev64 v8.4s, v8.4s diff --git a/arch/arm64/crypto/sm3-neon-core.S b/lib/crypto/arm64/sm3-neon-core.S index 4357e0e51be3..ad874af13802 100644 --- a/arch/arm64/crypto/sm3-neon-core.S +++ b/lib/crypto/arm64/sm3-neon-core.S @@ -9,7 +9,6 @@ */ #include <linux/linkage.h> -#include <linux/cfi_types.h> #include <asm/assembler.h> /* Context structure */ @@ -345,14 +344,14 @@ /* - * Transform blocks*64 bytes (blocks*16 32-bit words) at 'src'. + * Transform nblocks*64 bytes (nblocks*16 32-bit words) at 'data'. * - * void sm3_neon_transform(struct sm3_state *sst, u8 const *src, - * int blocks) + * void sm3_neon_transform(struct sm3_block_state *state, + * const u8 *data, size_t nblocks) */ .text .align 3 -SYM_TYPED_FUNC_START(sm3_neon_transform) +SYM_FUNC_START(sm3_neon_transform) ldp ra, rb, [RSTATE, #0] ldp rc, rd, [RSTATE, #8] ldp re, rf, [RSTATE, #16] diff --git a/lib/crypto/arm64/sm3.h b/lib/crypto/arm64/sm3.h new file mode 100644 index 000000000000..beb9cd82bb7d --- /dev/null +++ b/lib/crypto/arm64/sm3.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM3 optimized for ARM64 + * + * Copyright 2026 Google LLC + */ +#include <asm/simd.h> +#include <linux/cpufeature.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); + +asmlinkage void sm3_neon_transform(struct sm3_block_state *state, + const u8 *data, size_t nblocks); +asmlinkage void sm3_ce_transform(struct sm3_block_state *state, + const u8 *data, size_t nblocks); + +static void sm3_blocks(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_neon) && likely(may_use_simd())) { + scoped_ksimd() { + if (static_branch_likely(&have_ce)) + sm3_ce_transform(state, data, nblocks); + else + sm3_neon_transform(state, data, nblocks); + } + } else { + sm3_blocks_generic(state, data, nblocks); + } +} + +#define sm3_mod_init_arch sm3_mod_init_arch +static void sm3_mod_init_arch(void) +{ + if (cpu_have_named_feature(ASIMD)) { + static_branch_enable(&have_neon); + if (cpu_have_named_feature(SM3)) + static_branch_enable(&have_ce); + } +} diff --git a/lib/crypto/chacha.c b/lib/crypto/chacha.c index e0c7cb4af318..86e5d382a4e0 100644 --- a/lib/crypto/chacha.c +++ b/lib/crypto/chacha.c @@ -5,8 +5,8 @@ * Copyright (C) 2015 Martin Willi */ -#include <crypto/algapi.h> // for crypto_xor_cpy #include <crypto/chacha.h> +#include <crypto/utils.h> #include <linux/export.h> #include <linux/kernel.h> #include <linux/module.h> diff --git a/lib/crypto/chacha20poly1305.c b/lib/crypto/chacha20poly1305.c index 212ce33562af..ea42a28f4ff7 100644 --- a/lib/crypto/chacha20poly1305.c +++ b/lib/crypto/chacha20poly1305.c @@ -356,20 +356,6 @@ bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len } EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); -static int __init chacha20poly1305_init(void) -{ - if (IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) && - WARN_ON(!chacha20poly1305_selftest())) - return -ENODEV; - return 0; -} - -static void __exit chacha20poly1305_exit(void) -{ -} - -module_init(chacha20poly1305_init); -module_exit(chacha20poly1305_exit); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction"); MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); diff --git a/lib/crypto/fips.h b/lib/crypto/fips.h index 023410c2e0db..9fc49747db64 100644 --- a/lib/crypto/fips.h +++ b/lib/crypto/fips.h @@ -43,3 +43,8 @@ static const u8 fips_test_sha3_256_value[] __initconst __maybe_unused = { 0xba, 0x9b, 0xb6, 0xaa, 0x32, 0xa7, 0x97, 0x00, 0x98, 0xdb, 0xff, 0xe7, 0xc6, 0xde, 0xb5, 0x82, }; + +static const u8 fips_test_aes_cmac_value[] __initconst __maybe_unused = { + 0xc5, 0x88, 0x28, 0x55, 0xd7, 0x2c, 0x00, 0xb6, + 0x6a, 0xa7, 0xfc, 0x82, 0x90, 0x81, 0xcf, 0x18, +}; diff --git a/lib/crypto/polyval.c b/lib/crypto/gf128hash.c index 5796275f574a..2650603d8ba8 100644 --- a/lib/crypto/polyval.c +++ b/lib/crypto/gf128hash.c @@ -1,34 +1,41 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * POLYVAL library functions + * GF(2^128) polynomial hashing: GHASH and POLYVAL * * Copyright 2025 Google LLC */ -#include <crypto/polyval.h> +#include <crypto/gf128hash.h> #include <linux/export.h> #include <linux/module.h> #include <linux/string.h> #include <linux/unaligned.h> /* - * POLYVAL is an almost-XOR-universal hash function. Similar to GHASH, POLYVAL - * interprets the message as the coefficients of a polynomial in GF(2^128) and - * evaluates that polynomial at a secret point. POLYVAL has a simple - * mathematical relationship with GHASH, but it uses a better field convention - * which makes it easier and faster to implement. + * GHASH and POLYVAL are almost-XOR-universal hash functions. They interpret + * the message as the coefficients of a polynomial in the finite field GF(2^128) + * and evaluate that polynomial at a secret point. * - * POLYVAL is not a cryptographic hash function, and it should be used only by - * algorithms that are specifically designed to use it. + * Neither GHASH nor POLYVAL is a cryptographic hash function. They should be + * used only by algorithms that are specifically designed to use them. * - * POLYVAL is specified by "AES-GCM-SIV: Nonce Misuse-Resistant Authenticated - * Encryption" (https://datatracker.ietf.org/doc/html/rfc8452) + * GHASH is the older variant, defined as part of GCM in NIST SP 800-38D + * (https://nvlpubs.nist.gov/nistpubs/legacy/sp/nistspecialpublication800-38d.pdf). + * GHASH is hard to implement directly, due to its backwards mapping between + * bits and polynomial coefficients. GHASH implementations typically pre and + * post-process the inputs and outputs (mainly by byte-swapping) to convert the + * GHASH computation into an equivalent computation over a different, + * easier-to-use representation of GF(2^128). * - * POLYVAL is also used by HCTR2. See "Length-preserving encryption with HCTR2" - * (https://eprint.iacr.org/2021/1441.pdf). + * POLYVAL is a newer GF(2^128) polynomial hash, originally defined as part of + * AES-GCM-SIV (https://datatracker.ietf.org/doc/html/rfc8452) and also used by + * HCTR2 (https://eprint.iacr.org/2021/1441.pdf). It uses that easier-to-use + * field representation directly, eliminating the data conversion steps. * - * This file provides a library API for POLYVAL. This API can delegate to - * either a generic implementation or an architecture-optimized implementation. + * This file provides library APIs for GHASH and POLYVAL. These APIs can + * delegate to either a generic implementation or an architecture-optimized + * implementation. Due to the mathematical relationship between GHASH and + * POLYVAL, in some cases code for one is reused with the other. * * For the generic implementation, we don't use the traditional table approach * to GF(2^128) multiplication. That approach is not constant-time and requires @@ -205,6 +212,19 @@ polyval_mul_generic(struct polyval_elem *a, const struct polyval_elem *b) a->hi = cpu_to_le64(c3); } +static void __maybe_unused ghash_blocks_generic(struct polyval_elem *acc, + const struct polyval_elem *key, + const u8 *data, size_t nblocks) +{ + do { + acc->lo ^= + cpu_to_le64(get_unaligned_be64((__be64 *)(data + 8))); + acc->hi ^= cpu_to_le64(get_unaligned_be64((__be64 *)data)); + polyval_mul_generic(acc, key); + data += GHASH_BLOCK_SIZE; + } while (--nblocks); +} + static void __maybe_unused polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key, const u8 *data, size_t nblocks) @@ -217,16 +237,118 @@ polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key, } while (--nblocks); } -/* Include the arch-optimized implementation of POLYVAL, if one is available. */ -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH -#include "polyval.h" /* $(SRCARCH)/polyval.h */ +/* Convert the key from GHASH format to POLYVAL format. */ +static void __maybe_unused ghash_key_to_polyval(const u8 in[GHASH_BLOCK_SIZE], + struct polyval_elem *out) +{ + u64 hi = get_unaligned_be64(&in[0]); + u64 lo = get_unaligned_be64(&in[8]); + u64 mask = (s64)hi >> 63; + + hi = (hi << 1) ^ (lo >> 63) ^ (mask & ((u64)0xc2 << 56)); + lo = (lo << 1) ^ (mask & 1); + out->lo = cpu_to_le64(lo); + out->hi = cpu_to_le64(hi); +} + +/* Convert the accumulator from POLYVAL format to GHASH format. */ +static void polyval_acc_to_ghash(const struct polyval_elem *in, + u8 out[GHASH_BLOCK_SIZE]) +{ + put_unaligned_be64(le64_to_cpu(in->hi), &out[0]); + put_unaligned_be64(le64_to_cpu(in->lo), &out[8]); +} + +/* Convert the accumulator from GHASH format to POLYVAL format. */ +static void __maybe_unused ghash_acc_to_polyval(const u8 in[GHASH_BLOCK_SIZE], + struct polyval_elem *out) +{ + out->lo = cpu_to_le64(get_unaligned_be64(&in[8])); + out->hi = cpu_to_le64(get_unaligned_be64(&in[0])); +} + +#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH +#include "gf128hash.h" /* $(SRCARCH)/gf128hash.h */ +#endif + +void ghash_preparekey(struct ghash_key *key, const u8 raw_key[GHASH_BLOCK_SIZE]) +{ +#ifdef ghash_preparekey_arch + ghash_preparekey_arch(key, raw_key); +#else + ghash_key_to_polyval(raw_key, &key->h); +#endif +} +EXPORT_SYMBOL_GPL(ghash_preparekey); + +static void ghash_mul(struct ghash_ctx *ctx) +{ +#ifdef ghash_mul_arch + ghash_mul_arch(&ctx->acc, ctx->key); +#elif defined(ghash_blocks_arch) + static const u8 zeroes[GHASH_BLOCK_SIZE]; + + ghash_blocks_arch(&ctx->acc, ctx->key, zeroes, 1); +#else + polyval_mul_generic(&ctx->acc, &ctx->key->h); +#endif +} + +/* nblocks is always >= 1. */ +static void ghash_blocks(struct ghash_ctx *ctx, const u8 *data, size_t nblocks) +{ +#ifdef ghash_blocks_arch + ghash_blocks_arch(&ctx->acc, ctx->key, data, nblocks); +#else + ghash_blocks_generic(&ctx->acc, &ctx->key->h, data, nblocks); +#endif +} + +void ghash_update(struct ghash_ctx *ctx, const u8 *data, size_t len) +{ + if (unlikely(ctx->partial)) { + size_t n = min(len, GHASH_BLOCK_SIZE - ctx->partial); + + len -= n; + while (n--) + ctx->acc.bytes[GHASH_BLOCK_SIZE - 1 - ctx->partial++] ^= + *data++; + if (ctx->partial < GHASH_BLOCK_SIZE) + return; + ghash_mul(ctx); + } + if (len >= GHASH_BLOCK_SIZE) { + size_t nblocks = len / GHASH_BLOCK_SIZE; + + ghash_blocks(ctx, data, nblocks); + data += len & ~(GHASH_BLOCK_SIZE - 1); + len &= GHASH_BLOCK_SIZE - 1; + } + for (size_t i = 0; i < len; i++) + ctx->acc.bytes[GHASH_BLOCK_SIZE - 1 - i] ^= data[i]; + ctx->partial = len; +} +EXPORT_SYMBOL_GPL(ghash_update); + +void ghash_final(struct ghash_ctx *ctx, u8 out[GHASH_BLOCK_SIZE]) +{ + if (unlikely(ctx->partial)) + ghash_mul(ctx); + polyval_acc_to_ghash(&ctx->acc, out); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(ghash_final); + void polyval_preparekey(struct polyval_key *key, const u8 raw_key[POLYVAL_BLOCK_SIZE]) { +#ifdef polyval_preparekey_arch polyval_preparekey_arch(key, raw_key); +#else + memcpy(key->h.bytes, raw_key, POLYVAL_BLOCK_SIZE); +#endif } EXPORT_SYMBOL_GPL(polyval_preparekey); -#endif /* Else, polyval_preparekey() is an inline function. */ /* * polyval_mul_generic() and polyval_blocks_generic() take the key as a @@ -238,17 +360,22 @@ EXPORT_SYMBOL_GPL(polyval_preparekey); static void polyval_mul(struct polyval_ctx *ctx) { -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH +#ifdef polyval_mul_arch polyval_mul_arch(&ctx->acc, ctx->key); +#elif defined(polyval_blocks_arch) + static const u8 zeroes[POLYVAL_BLOCK_SIZE]; + + polyval_blocks_arch(&ctx->acc, ctx->key, zeroes, 1); #else polyval_mul_generic(&ctx->acc, &ctx->key->h); #endif } +/* nblocks is always >= 1. */ static void polyval_blocks(struct polyval_ctx *ctx, const u8 *data, size_t nblocks) { -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH +#ifdef polyval_blocks_arch polyval_blocks_arch(&ctx->acc, ctx->key, data, nblocks); #else polyval_blocks_generic(&ctx->acc, &ctx->key->h, data, nblocks); @@ -289,19 +416,19 @@ void polyval_final(struct polyval_ctx *ctx, u8 out[POLYVAL_BLOCK_SIZE]) } EXPORT_SYMBOL_GPL(polyval_final); -#ifdef polyval_mod_init_arch -static int __init polyval_mod_init(void) +#ifdef gf128hash_mod_init_arch +static int __init gf128hash_mod_init(void) { - polyval_mod_init_arch(); + gf128hash_mod_init_arch(); return 0; } -subsys_initcall(polyval_mod_init); +subsys_initcall(gf128hash_mod_init); -static void __exit polyval_mod_exit(void) +static void __exit gf128hash_mod_exit(void) { } -module_exit(polyval_mod_exit); +module_exit(gf128hash_mod_exit); #endif -MODULE_DESCRIPTION("POLYVAL almost-XOR-universal hash function"); +MODULE_DESCRIPTION("GF(2^128) polynomial hashing: GHASH and POLYVAL"); MODULE_LICENSE("GPL"); diff --git a/lib/crypto/gf128mul.c b/lib/crypto/gf128mul.c index e5a727b15f07..7ebf07ce1168 100644 --- a/lib/crypto/gf128mul.c +++ b/lib/crypto/gf128mul.c @@ -127,27 +127,9 @@ (i & 0x02 ? 0x0384 : 0) ^ (i & 0x01 ? 0x01c2 : 0) \ ) -static const u16 gf128mul_table_le[256] = gf128mul_dat(xda_le); static const u16 gf128mul_table_be[256] = gf128mul_dat(xda_be); -/* - * The following functions multiply a field element by x^8 in - * the polynomial field representation. They use 64-bit word operations - * to gain speed but compensate for machine endianness and hence work - * correctly on both styles of machine. - */ - -static void gf128mul_x8_lle(be128 *x) -{ - u64 a = be64_to_cpu(x->a); - u64 b = be64_to_cpu(x->b); - u64 _tt = gf128mul_table_le[b & 0xff]; - - x->b = cpu_to_be64((b >> 8) | (a << 56)); - x->a = cpu_to_be64((a >> 8) ^ (_tt << 48)); -} - -/* time invariant version of gf128mul_x8_lle */ +/* A table-less implementation of multiplying by x^8 */ static void gf128mul_x8_lle_ti(be128 *x) { u64 a = be64_to_cpu(x->a); @@ -305,58 +287,5 @@ void gf128mul_64k_bbe(be128 *a, const struct gf128mul_64k *t) } EXPORT_SYMBOL(gf128mul_64k_bbe); -/* This version uses 4k bytes of table space. - A 16 byte buffer has to be multiplied by a 16 byte key - value in GF(2^128). If we consider a GF(2^128) value in a - single byte, we can construct a table of the 256 16 byte - values that result from the 256 values of this byte. - This requires 4096 bytes. If we take the highest byte in - the buffer and use this table to get the result, we then - have to multiply by x^120 to get the final value. For the - next highest byte the result has to be multiplied by x^112 - and so on. But we can do this by accumulating the result - in an accumulator starting with the result for the top - byte. We repeatedly multiply the accumulator value by - x^8 and then add in (i.e. xor) the 16 bytes of the next - lower byte in the buffer, stopping when we reach the - lowest byte. This requires a 4096 byte table. -*/ -struct gf128mul_4k *gf128mul_init_4k_lle(const be128 *g) -{ - struct gf128mul_4k *t; - int j, k; - - t = kzalloc_obj(*t); - if (!t) - goto out; - - t->t[128] = *g; - for (j = 64; j > 0; j >>= 1) - gf128mul_x_lle(&t->t[j], &t->t[j+j]); - - for (j = 2; j < 256; j += j) - for (k = 1; k < j; ++k) - be128_xor(&t->t[j + k], &t->t[j], &t->t[k]); - -out: - return t; -} -EXPORT_SYMBOL(gf128mul_init_4k_lle); - -void gf128mul_4k_lle(be128 *a, const struct gf128mul_4k *t) -{ - u8 *ap = (u8 *)a; - be128 r[1]; - int i = 15; - - *r = t->t[ap[15]]; - while (i--) { - gf128mul_x8_lle(r); - be128_xor(r, r, &t->t[ap[i]]); - } - *a = *r; -} -EXPORT_SYMBOL(gf128mul_4k_lle); - MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Functions for multiplying elements of GF(2^128)"); diff --git a/lib/crypto/memneq.c b/lib/crypto/memneq.c index 44daacb8cb51..08924acd44bc 100644 --- a/lib/crypto/memneq.c +++ b/lib/crypto/memneq.c @@ -59,7 +59,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include <crypto/algapi.h> +#include <crypto/utils.h> #include <linux/export.h> #include <linux/module.h> #include <linux/unaligned.h> @@ -159,7 +159,7 @@ static inline unsigned long __crypto_memneq_16(const void *a, const void *b) /* Compare two areas of memory without leaking timing information, * and with special optimizations for common sizes. Users should * not call this function directly, but should instead use - * crypto_memneq defined in crypto/algapi.h. + * crypto_memneq defined in crypto/utils.h. */ noinline unsigned long __crypto_memneq(const void *a, const void *b, size_t size) diff --git a/lib/crypto/mips/md5.h b/lib/crypto/mips/md5.h deleted file mode 100644 index e08e28aeffa4..000000000000 --- a/lib/crypto/mips/md5.h +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Cryptographic API. - * - * MD5 Message Digest Algorithm (RFC1321). - * - * Adapted for OCTEON by Aaro Koskinen <aaro.koskinen@iki.fi>. - * - * Based on crypto/md5.c, which is: - * - * Derived from cryptoapi implementation, originally based on the - * public domain implementation written by Colin Plumb in 1993. - * - * Copyright (c) Cryptoapi developers. - * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. - */ - -#include <asm/octeon/crypto.h> -#include <asm/octeon/octeon.h> - -/* - * We pass everything as 64-bit. OCTEON can handle misaligned data. - */ - -static void md5_blocks(struct md5_block_state *state, - const u8 *data, size_t nblocks) -{ - struct octeon_cop2_state cop2_state; - u64 *state64 = (u64 *)state; - unsigned long flags; - - if (!octeon_has_crypto()) - return md5_blocks_generic(state, data, nblocks); - - cpu_to_le32_array(state->h, ARRAY_SIZE(state->h)); - - flags = octeon_crypto_enable(&cop2_state); - write_octeon_64bit_hash_dword(state64[0], 0); - write_octeon_64bit_hash_dword(state64[1], 1); - - do { - const u64 *block = (const u64 *)data; - - write_octeon_64bit_block_dword(block[0], 0); - write_octeon_64bit_block_dword(block[1], 1); - write_octeon_64bit_block_dword(block[2], 2); - write_octeon_64bit_block_dword(block[3], 3); - write_octeon_64bit_block_dword(block[4], 4); - write_octeon_64bit_block_dword(block[5], 5); - write_octeon_64bit_block_dword(block[6], 6); - octeon_md5_start(block[7]); - - data += MD5_BLOCK_SIZE; - } while (--nblocks); - - state64[0] = read_octeon_64bit_hash_dword(0); - state64[1] = read_octeon_64bit_hash_dword(1); - octeon_crypto_disable(&cop2_state, flags); - - le32_to_cpu_array(state->h, ARRAY_SIZE(state->h)); -} diff --git a/lib/crypto/powerpc/.gitignore b/lib/crypto/powerpc/.gitignore index 598ca7aff6b1..7aa71d83f739 100644 --- a/lib/crypto/powerpc/.gitignore +++ b/lib/crypto/powerpc/.gitignore @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only aesp8-ppc.S +ghashp8-ppc.S diff --git a/lib/crypto/powerpc/gf128hash.h b/lib/crypto/powerpc/gf128hash.h new file mode 100644 index 000000000000..629cd325d0c7 --- /dev/null +++ b/lib/crypto/powerpc/gf128hash.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * GHASH routines supporting VMX instructions on the Power 8 + * + * Copyright (C) 2015, 2019 International Business Machines Inc. + * Copyright (C) 2014 - 2018 Linaro Ltd. + * Copyright 2026 Google LLC + */ + +#include <asm/simd.h> +#include <asm/switch_to.h> +#include <linux/cpufeature.h> +#include <linux/jump_label.h> +#include <linux/preempt.h> +#include <linux/uaccess.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto); + +void gcm_init_p8(u64 htable[4][2], const u8 h[16]); +void gcm_gmult_p8(u8 Xi[16], const u64 htable[4][2]); +void gcm_ghash_p8(u8 Xi[16], const u64 htable[4][2], const u8 *in, size_t len); + +#define ghash_preparekey_arch ghash_preparekey_arch +static void ghash_preparekey_arch(struct ghash_key *key, + const u8 raw_key[GHASH_BLOCK_SIZE]) +{ + ghash_key_to_polyval(raw_key, &key->h); + + if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) { + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + gcm_init_p8(key->htable, raw_key); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + } else { + /* This reproduces gcm_init_p8() on both LE and BE systems. */ + key->htable[0][0] = 0; + key->htable[0][1] = 0xc200000000000000; + + key->htable[1][0] = 0; + key->htable[1][1] = le64_to_cpu(key->h.lo); + + key->htable[2][0] = le64_to_cpu(key->h.lo); + key->htable[2][1] = le64_to_cpu(key->h.hi); + + key->htable[3][0] = le64_to_cpu(key->h.hi); + key->htable[3][1] = 0; + } +} + +#define ghash_mul_arch ghash_mul_arch +static void ghash_mul_arch(struct polyval_elem *acc, + const struct ghash_key *key) +{ + if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) { + u8 ghash_acc[GHASH_BLOCK_SIZE]; + + polyval_acc_to_ghash(acc, ghash_acc); + + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + gcm_gmult_p8(ghash_acc, key->htable); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + + ghash_acc_to_polyval(ghash_acc, acc); + memzero_explicit(ghash_acc, sizeof(ghash_acc)); + } else { + polyval_mul_generic(acc, &key->h); + } +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) { + u8 ghash_acc[GHASH_BLOCK_SIZE]; + + polyval_acc_to_ghash(acc, ghash_acc); + + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + gcm_ghash_p8(ghash_acc, key->htable, data, + nblocks * GHASH_BLOCK_SIZE); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + + ghash_acc_to_polyval(ghash_acc, acc); + memzero_explicit(ghash_acc, sizeof(ghash_acc)); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (cpu_has_feature(CPU_FTR_ARCH_207S) && + (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO)) + static_branch_enable(&have_vec_crypto); +} diff --git a/arch/powerpc/crypto/ghashp8-ppc.pl b/lib/crypto/powerpc/ghashp8-ppc.pl index 041e633c214f..7c38eedc02cc 100644 --- a/arch/powerpc/crypto/ghashp8-ppc.pl +++ b/lib/crypto/powerpc/ghashp8-ppc.pl @@ -47,6 +47,7 @@ if ($flavour =~ /64/) { $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or +( $xlate="${dir}../../../arch/powerpc/crypto/ppc-xlate.pl" and -f $xlate) or die "can't locate ppc-xlate.pl"; open STDOUT,"| $^X $xlate $flavour $output" || die "can't call $xlate: $!"; diff --git a/lib/crypto/riscv/gf128hash.h b/lib/crypto/riscv/gf128hash.h new file mode 100644 index 000000000000..4301a0384f60 --- /dev/null +++ b/lib/crypto/riscv/gf128hash.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * GHASH, RISC-V optimized + * + * Copyright (C) 2023 VRULL GmbH + * Copyright (C) 2023 SiFive, Inc. + * Copyright 2026 Google LLC + */ + +#include <asm/simd.h> +#include <asm/vector.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkg); + +asmlinkage void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE], + const u8 key[GHASH_BLOCK_SIZE], + const u8 *data, size_t nblocks); + +#define ghash_preparekey_arch ghash_preparekey_arch +static void ghash_preparekey_arch(struct ghash_key *key, + const u8 raw_key[GHASH_BLOCK_SIZE]) +{ + /* Save key in POLYVAL format for fallback */ + ghash_key_to_polyval(raw_key, &key->h); + + /* Save key in GHASH format for zvkg */ + memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE); +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_zvkg) && likely(may_use_simd())) { + u8 ghash_acc[GHASH_BLOCK_SIZE]; + + polyval_acc_to_ghash(acc, ghash_acc); + + kernel_vector_begin(); + ghash_zvkg(ghash_acc, key->h_raw, data, nblocks); + kernel_vector_end(); + + ghash_acc_to_polyval(ghash_acc, acc); + memzero_explicit(ghash_acc, sizeof(ghash_acc)); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (riscv_isa_extension_available(NULL, ZVKG) && + riscv_vector_vlen() >= 128) + static_branch_enable(&have_zvkg); +} diff --git a/arch/riscv/crypto/ghash-riscv64-zvkg.S b/lib/crypto/riscv/ghash-riscv64-zvkg.S index f2b43fb4d434..6a2a2f2bc7c8 100644 --- a/arch/riscv/crypto/ghash-riscv64-zvkg.S +++ b/lib/crypto/riscv/ghash-riscv64-zvkg.S @@ -50,12 +50,13 @@ #define ACCUMULATOR a0 #define KEY a1 #define DATA a2 -#define LEN a3 +#define NBLOCKS a3 -// void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data, -// size_t len); +// void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE], +// const u8 key[GHASH_BLOCK_SIZE], +// const u8 *data, size_t nblocks); // -// |len| must be nonzero and a multiple of 16 (GHASH_BLOCK_SIZE). +// |nblocks| must be nonzero. SYM_FUNC_START(ghash_zvkg) vsetivli zero, 4, e32, m1, ta, ma vle32.v v1, (ACCUMULATOR) @@ -64,8 +65,8 @@ SYM_FUNC_START(ghash_zvkg) vle32.v v3, (DATA) vghsh.vv v1, v2, v3 addi DATA, DATA, 16 - addi LEN, LEN, -16 - bnez LEN, .Lnext_block + addi NBLOCKS, NBLOCKS, -1 + bnez NBLOCKS, .Lnext_block vse32.v v1, (ACCUMULATOR) ret diff --git a/arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S b/lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S index 4fe754846f65..a1d4468b0485 100644 --- a/arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S +++ b/lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S @@ -80,7 +80,8 @@ // For the next 8 rounds, w0 and w1 are swapped. .endm -// void sm3_transform_zvksh_zvkb(u32 state[8], const u8 *data, int num_blocks); +// void sm3_transform_zvksh_zvkb(struct sm3_block_state *state, +// const u8 *data, size_t nblocks); SYM_FUNC_START(sm3_transform_zvksh_zvkb) // Load the state and endian-swap each 32-bit word. diff --git a/lib/crypto/riscv/sm3.h b/lib/crypto/riscv/sm3.h new file mode 100644 index 000000000000..c1fbee7094e6 --- /dev/null +++ b/lib/crypto/riscv/sm3.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM3 using the RISC-V vector crypto extensions + * + * Copyright (C) 2023 VRULL GmbH + * Author: Heiko Stuebner <heiko.stuebner@vrull.eu> + * + * Copyright (C) 2023 SiFive, Inc. + * Author: Jerry Shih <jerry.shih@sifive.com> + */ + +#include <asm/simd.h> +#include <asm/vector.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions); + +asmlinkage void sm3_transform_zvksh_zvkb(struct sm3_block_state *state, + const u8 *data, size_t nblocks); + +static void sm3_blocks(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_extensions) && likely(may_use_simd())) { + kernel_vector_begin(); + sm3_transform_zvksh_zvkb(state, data, nblocks); + kernel_vector_end(); + } else { + sm3_blocks_generic(state, data, nblocks); + } +} + +#define sm3_mod_init_arch sm3_mod_init_arch +static void sm3_mod_init_arch(void) +{ + if (riscv_isa_extension_available(NULL, ZVKSH) && + riscv_isa_extension_available(NULL, ZVKB) && + riscv_vector_vlen() >= 128) + static_branch_enable(&have_extensions); +} diff --git a/lib/crypto/s390/gf128hash.h b/lib/crypto/s390/gf128hash.h new file mode 100644 index 000000000000..1e46ce4bca40 --- /dev/null +++ b/lib/crypto/s390/gf128hash.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * GHASH optimized using the CP Assist for Cryptographic Functions (CPACF) + * + * Copyright 2026 Google LLC + */ +#include <asm/cpacf.h> +#include <linux/cpufeature.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_ghash); + +#define ghash_preparekey_arch ghash_preparekey_arch +static void ghash_preparekey_arch(struct ghash_key *key, + const u8 raw_key[GHASH_BLOCK_SIZE]) +{ + /* Save key in POLYVAL format for fallback */ + ghash_key_to_polyval(raw_key, &key->h); + + /* Save key in GHASH format for CPACF_KIMD_GHASH */ + memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE); +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_cpacf_ghash)) { + /* + * CPACF_KIMD_GHASH requires the accumulator and key in a single + * buffer, each using the GHASH convention. + */ + u8 ctx[2][GHASH_BLOCK_SIZE] __aligned(8); + + polyval_acc_to_ghash(acc, ctx[0]); + memcpy(ctx[1], key->h_raw, GHASH_BLOCK_SIZE); + + cpacf_kimd(CPACF_KIMD_GHASH, ctx, data, + nblocks * GHASH_BLOCK_SIZE); + + ghash_acc_to_polyval(ctx[0], acc); + memzero_explicit(ctx, sizeof(ctx)); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (cpu_have_feature(S390_CPU_FEATURE_MSA) && + cpacf_query_func(CPACF_KIMD, CPACF_KIMD_GHASH)) + static_branch_enable(&have_cpacf_ghash); +} diff --git a/lib/crypto/sm3.c b/lib/crypto/sm3.c index c6b9ad8a3ac6..b02b8a247adf 100644 --- a/lib/crypto/sm3.c +++ b/lib/crypto/sm3.c @@ -15,6 +15,13 @@ #include <linux/string.h> #include <linux/unaligned.h> +static const struct sm3_block_state sm3_iv = { + .h = { + SM3_IVA, SM3_IVB, SM3_IVC, SM3_IVD, + SM3_IVE, SM3_IVF, SM3_IVG, SM3_IVH, + }, +}; + static const u32 ____cacheline_aligned K[64] = { 0x79cc4519, 0xf3988a32, 0xe7311465, 0xce6228cb, 0x9cc45197, 0x3988a32f, 0x7311465e, 0xe6228cbc, @@ -72,18 +79,19 @@ static const u32 ____cacheline_aligned K[64] = { ^ rol32(W[(i-13) & 0x0f], 7) \ ^ W[(i-6) & 0x0f]) -static void sm3_transform(struct sm3_state *sctx, u8 const *data, u32 W[16]) +static void sm3_block_generic(struct sm3_block_state *state, + const u8 data[SM3_BLOCK_SIZE], u32 W[16]) { u32 a, b, c, d, e, f, g, h, ss1, ss2; - a = sctx->state[0]; - b = sctx->state[1]; - c = sctx->state[2]; - d = sctx->state[3]; - e = sctx->state[4]; - f = sctx->state[5]; - g = sctx->state[6]; - h = sctx->state[7]; + a = state->h[0]; + b = state->h[1]; + c = state->h[2]; + d = state->h[3]; + e = state->h[4]; + f = state->h[5]; + g = state->h[6]; + h = state->h[7]; R1(a, b, c, d, e, f, g, h, K[0], I(0), I(4)); R1(d, a, b, c, h, e, f, g, K[1], I(1), I(5)); @@ -153,14 +161,14 @@ static void sm3_transform(struct sm3_state *sctx, u8 const *data, u32 W[16]) R2(c, d, a, b, g, h, e, f, K[62], W1(62), W2(66)); R2(b, c, d, a, f, g, h, e, K[63], W1(63), W2(67)); - sctx->state[0] ^= a; - sctx->state[1] ^= b; - sctx->state[2] ^= c; - sctx->state[3] ^= d; - sctx->state[4] ^= e; - sctx->state[5] ^= f; - sctx->state[6] ^= g; - sctx->state[7] ^= h; + state->h[0] ^= a; + state->h[1] ^= b; + state->h[2] ^= c; + state->h[3] ^= d; + state->h[4] ^= e; + state->h[5] ^= f; + state->h[6] ^= g; + state->h[7] ^= h; } #undef R #undef R1 @@ -169,18 +177,114 @@ static void sm3_transform(struct sm3_state *sctx, u8 const *data, u32 W[16]) #undef W1 #undef W2 -void sm3_block_generic(struct sm3_state *sctx, u8 const *data, int blocks) +static void __maybe_unused sm3_blocks_generic(struct sm3_block_state *state, + const u8 *data, size_t nblocks) { u32 W[16]; do { - sm3_transform(sctx, data, W); + sm3_block_generic(state, data, W); data += SM3_BLOCK_SIZE; - } while (--blocks); + } while (--nblocks); memzero_explicit(W, sizeof(W)); } -EXPORT_SYMBOL_GPL(sm3_block_generic); -MODULE_DESCRIPTION("Generic SM3 library"); +#ifdef CONFIG_CRYPTO_LIB_SM3_ARCH +#include "sm3.h" /* $(SRCARCH)/sm3.h */ +#else +#define sm3_blocks sm3_blocks_generic +#endif + +void sm3_init(struct sm3_ctx *ctx) +{ + ctx->state = sm3_iv; + ctx->bytecount = 0; +} +EXPORT_SYMBOL_GPL(sm3_init); + +void sm3_update(struct sm3_ctx *ctx, const u8 *data, size_t len) +{ + size_t partial = ctx->bytecount % SM3_BLOCK_SIZE; + + ctx->bytecount += len; + + if (partial + len >= SM3_BLOCK_SIZE) { + size_t nblocks; + + if (partial) { + size_t l = SM3_BLOCK_SIZE - partial; + + memcpy(&ctx->buf[partial], data, l); + data += l; + len -= l; + + sm3_blocks(&ctx->state, ctx->buf, 1); + } + + nblocks = len / SM3_BLOCK_SIZE; + len %= SM3_BLOCK_SIZE; + + if (nblocks) { + sm3_blocks(&ctx->state, data, nblocks); + data += nblocks * SM3_BLOCK_SIZE; + } + partial = 0; + } + if (len) + memcpy(&ctx->buf[partial], data, len); +} +EXPORT_SYMBOL_GPL(sm3_update); + +static void __sm3_final(struct sm3_ctx *ctx, u8 out[SM3_DIGEST_SIZE]) +{ + u64 bitcount = ctx->bytecount << 3; + size_t partial = ctx->bytecount % SM3_BLOCK_SIZE; + + ctx->buf[partial++] = 0x80; + if (partial > SM3_BLOCK_SIZE - 8) { + memset(&ctx->buf[partial], 0, SM3_BLOCK_SIZE - partial); + sm3_blocks(&ctx->state, ctx->buf, 1); + partial = 0; + } + memset(&ctx->buf[partial], 0, SM3_BLOCK_SIZE - 8 - partial); + *(__be64 *)&ctx->buf[SM3_BLOCK_SIZE - 8] = cpu_to_be64(bitcount); + sm3_blocks(&ctx->state, ctx->buf, 1); + + for (size_t i = 0; i < SM3_DIGEST_SIZE; i += 4) + put_unaligned_be32(ctx->state.h[i / 4], out + i); +} + +void sm3_final(struct sm3_ctx *ctx, u8 out[SM3_DIGEST_SIZE]) +{ + __sm3_final(ctx, out); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(sm3_final); + +void sm3(const u8 *data, size_t len, u8 out[SM3_DIGEST_SIZE]) +{ + struct sm3_ctx ctx; + + sm3_init(&ctx); + sm3_update(&ctx, data, len); + sm3_final(&ctx, out); +} +EXPORT_SYMBOL_GPL(sm3); + +#ifdef sm3_mod_init_arch +static int __init sm3_mod_init(void) +{ + sm3_mod_init_arch(); + return 0; +} +subsys_initcall(sm3_mod_init); + +static void __exit sm3_mod_exit(void) +{ +} +module_exit(sm3_mod_exit); +#endif + +MODULE_DESCRIPTION("SM3 library functions"); MODULE_LICENSE("GPL v2"); diff --git a/lib/crypto/sparc/md5.h b/lib/crypto/sparc/md5.h deleted file mode 100644 index 3995f3e075eb..000000000000 --- a/lib/crypto/sparc/md5.h +++ /dev/null @@ -1,48 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * MD5 accelerated using the sparc64 crypto opcodes - * - * Copyright (c) Alan Smithee. - * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> - * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> - * Copyright (c) Mathias Krause <minipli@googlemail.com> - * Copyright (c) Cryptoapi developers. - * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> - */ - -#include <asm/elf.h> -#include <asm/opcodes.h> -#include <asm/pstate.h> - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_md5_opcodes); - -asmlinkage void md5_sparc64_transform(struct md5_block_state *state, - const u8 *data, size_t nblocks); - -static void md5_blocks(struct md5_block_state *state, - const u8 *data, size_t nblocks) -{ - if (static_branch_likely(&have_md5_opcodes)) { - cpu_to_le32_array(state->h, ARRAY_SIZE(state->h)); - md5_sparc64_transform(state, data, nblocks); - le32_to_cpu_array(state->h, ARRAY_SIZE(state->h)); - } else { - md5_blocks_generic(state, data, nblocks); - } -} - -#define md5_mod_init_arch md5_mod_init_arch -static void md5_mod_init_arch(void) -{ - unsigned long cfr; - - if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) - return; - - __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); - if (!(cfr & CFR_MD5)) - return; - - static_branch_enable(&have_md5_opcodes); - pr_info("Using sparc64 md5 opcode optimized MD5 implementation\n"); -} diff --git a/lib/crypto/sparc/md5_asm.S b/lib/crypto/sparc/md5_asm.S deleted file mode 100644 index 60b544e4d205..000000000000 --- a/lib/crypto/sparc/md5_asm.S +++ /dev/null @@ -1,70 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include <linux/linkage.h> -#include <asm/opcodes.h> -#include <asm/visasm.h> - -ENTRY(md5_sparc64_transform) - /* %o0 = digest, %o1 = data, %o2 = rounds */ - VISEntryHalf - ld [%o0 + 0x00], %f0 - ld [%o0 + 0x04], %f1 - andcc %o1, 0x7, %g0 - ld [%o0 + 0x08], %f2 - bne,pn %xcc, 10f - ld [%o0 + 0x0c], %f3 - -1: - ldd [%o1 + 0x00], %f8 - ldd [%o1 + 0x08], %f10 - ldd [%o1 + 0x10], %f12 - ldd [%o1 + 0x18], %f14 - ldd [%o1 + 0x20], %f16 - ldd [%o1 + 0x28], %f18 - ldd [%o1 + 0x30], %f20 - ldd [%o1 + 0x38], %f22 - - MD5 - - subcc %o2, 1, %o2 - bne,pt %xcc, 1b - add %o1, 0x40, %o1 - -5: - st %f0, [%o0 + 0x00] - st %f1, [%o0 + 0x04] - st %f2, [%o0 + 0x08] - st %f3, [%o0 + 0x0c] - retl - VISExitHalf -10: - alignaddr %o1, %g0, %o1 - - ldd [%o1 + 0x00], %f10 -1: - ldd [%o1 + 0x08], %f12 - ldd [%o1 + 0x10], %f14 - ldd [%o1 + 0x18], %f16 - ldd [%o1 + 0x20], %f18 - ldd [%o1 + 0x28], %f20 - ldd [%o1 + 0x30], %f22 - ldd [%o1 + 0x38], %f24 - ldd [%o1 + 0x40], %f26 - - faligndata %f10, %f12, %f8 - faligndata %f12, %f14, %f10 - faligndata %f14, %f16, %f12 - faligndata %f16, %f18, %f14 - faligndata %f18, %f20, %f16 - faligndata %f20, %f22, %f18 - faligndata %f22, %f24, %f20 - faligndata %f24, %f26, %f22 - - MD5 - - subcc %o2, 1, %o2 - fsrc2 %f26, %f10 - bne,pt %xcc, 1b - add %o1, 0x40, %o1 - - ba,a,pt %xcc, 5b -ENDPROC(md5_sparc64_transform) diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index 0de289b429a9..9409c1a935c3 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -1,9 +1,18 @@ # SPDX-License-Identifier: GPL-2.0-or-later +config CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST + tristate "KUnit tests for AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_AES_CBC_MACS + default KUNIT_ALL_TESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + help + KUnit tests for the AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC message + authentication codes. + config CRYPTO_LIB_BLAKE2B_KUNIT_TEST tristate "KUnit tests for BLAKE2b" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_BLAKE2B - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the BLAKE2b cryptographic hash function. @@ -11,25 +20,42 @@ config CRYPTO_LIB_BLAKE2B_KUNIT_TEST config CRYPTO_LIB_BLAKE2S_KUNIT_TEST tristate "KUnit tests for BLAKE2s" if !KUNIT_ALL_TESTS depends on KUNIT - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE # No need to depend on CRYPTO_LIB_BLAKE2S here, as that option doesn't # exist; the BLAKE2s code is always built-in for the /dev/random driver. help KUnit tests for the BLAKE2s cryptographic hash function. +config CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST + tristate "KUnit tests for ChaCha20Poly1305" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_CHACHA20POLY1305 + default KUNIT_ALL_TESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + help + KUnit tests for the ChaCha20Poly1305 authenticated encryption + algorithm. + config CRYPTO_LIB_CURVE25519_KUNIT_TEST tristate "KUnit tests for Curve25519" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_CURVE25519 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the Curve25519 Diffie-Hellman function. +config CRYPTO_LIB_GHASH_KUNIT_TEST + tristate "KUnit tests for GHASH" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_GF128HASH + default KUNIT_ALL_TESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + help + KUnit tests for the GHASH library functions. + config CRYPTO_LIB_MD5_KUNIT_TEST tristate "KUnit tests for MD5" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_MD5 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the MD5 cryptographic hash function and its @@ -38,7 +64,7 @@ config CRYPTO_LIB_MD5_KUNIT_TEST config CRYPTO_LIB_MLDSA_KUNIT_TEST tristate "KUnit tests for ML-DSA" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_MLDSA - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the ML-DSA digital signature algorithm. @@ -46,22 +72,22 @@ config CRYPTO_LIB_MLDSA_KUNIT_TEST config CRYPTO_LIB_NH_KUNIT_TEST tristate "KUnit tests for NH" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_NH - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS help KUnit tests for the NH almost-universal hash function. config CRYPTO_LIB_POLY1305_KUNIT_TEST tristate "KUnit tests for Poly1305" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_POLY1305 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the Poly1305 library functions. config CRYPTO_LIB_POLYVAL_KUNIT_TEST tristate "KUnit tests for POLYVAL" if !KUNIT_ALL_TESTS - depends on KUNIT && CRYPTO_LIB_POLYVAL - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + depends on KUNIT && CRYPTO_LIB_GF128HASH + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the POLYVAL library functions. @@ -69,7 +95,7 @@ config CRYPTO_LIB_POLYVAL_KUNIT_TEST config CRYPTO_LIB_SHA1_KUNIT_TEST tristate "KUnit tests for SHA-1" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_SHA1 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the SHA-1 cryptographic hash function and its @@ -80,7 +106,7 @@ config CRYPTO_LIB_SHA1_KUNIT_TEST config CRYPTO_LIB_SHA256_KUNIT_TEST tristate "KUnit tests for SHA-224 and SHA-256" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_SHA256 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the SHA-224 and SHA-256 cryptographic hash functions @@ -91,7 +117,7 @@ config CRYPTO_LIB_SHA256_KUNIT_TEST config CRYPTO_LIB_SHA512_KUNIT_TEST tristate "KUnit tests for SHA-384 and SHA-512" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_SHA512 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the SHA-384 and SHA-512 cryptographic hash functions @@ -100,13 +126,47 @@ config CRYPTO_LIB_SHA512_KUNIT_TEST config CRYPTO_LIB_SHA3_KUNIT_TEST tristate "KUnit tests for SHA-3" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_SHA3 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the SHA3 cryptographic hash and XOF functions, including SHA3-224, SHA3-256, SHA3-384, SHA3-512, SHAKE128 and SHAKE256. +config CRYPTO_LIB_SM3_KUNIT_TEST + tristate "KUnit tests for SM3" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_SM3 + default KUNIT_ALL_TESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + help + KUnit tests for the SM3 cryptographic hash function. + +config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT + tristate "Enable all crypto library code for KUnit tests" + depends on KUNIT + select CRYPTO_LIB_AES_CBC_MACS + select CRYPTO_LIB_BLAKE2B + select CRYPTO_LIB_CHACHA20POLY1305 + select CRYPTO_LIB_CURVE25519 + select CRYPTO_LIB_GF128HASH + select CRYPTO_LIB_MD5 + select CRYPTO_LIB_MLDSA + select CRYPTO_LIB_NH + select CRYPTO_LIB_POLY1305 + select CRYPTO_LIB_SHA1 + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 + select CRYPTO_LIB_SHA3 + select CRYPTO_LIB_SM3 + help + Enable all the crypto library code that has KUnit tests. + + Enable this only if you'd like to test all the crypto library code, + even code that wouldn't otherwise need to be built. + + You'll still need to enable the tests themselves, either individually + or using KUNIT_ALL_TESTS. + config CRYPTO_LIB_BENCHMARK_VISIBLE bool diff --git a/lib/crypto/tests/Makefile b/lib/crypto/tests/Makefile index f4262379f56c..a739413500b6 100644 --- a/lib/crypto/tests/Makefile +++ b/lib/crypto/tests/Makefile @@ -1,8 +1,11 @@ # SPDX-License-Identifier: GPL-2.0-or-later +obj-$(CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST) += aes_cbc_macs_kunit.o obj-$(CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST) += blake2b_kunit.o obj-$(CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST) += blake2s_kunit.o +obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST) += chacha20poly1305_kunit.o obj-$(CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST) += curve25519_kunit.o +obj-$(CONFIG_CRYPTO_LIB_GHASH_KUNIT_TEST) += ghash_kunit.o obj-$(CONFIG_CRYPTO_LIB_MD5_KUNIT_TEST) += md5_kunit.o obj-$(CONFIG_CRYPTO_LIB_MLDSA_KUNIT_TEST) += mldsa_kunit.o obj-$(CONFIG_CRYPTO_LIB_NH_KUNIT_TEST) += nh_kunit.o @@ -12,3 +15,4 @@ obj-$(CONFIG_CRYPTO_LIB_SHA1_KUNIT_TEST) += sha1_kunit.o obj-$(CONFIG_CRYPTO_LIB_SHA256_KUNIT_TEST) += sha224_kunit.o sha256_kunit.o obj-$(CONFIG_CRYPTO_LIB_SHA512_KUNIT_TEST) += sha384_kunit.o sha512_kunit.o obj-$(CONFIG_CRYPTO_LIB_SHA3_KUNIT_TEST) += sha3_kunit.o +obj-$(CONFIG_CRYPTO_LIB_SM3_KUNIT_TEST) += sm3_kunit.o diff --git a/lib/crypto/tests/aes-cmac-testvecs.h b/lib/crypto/tests/aes-cmac-testvecs.h new file mode 100644 index 000000000000..8177862ba5a3 --- /dev/null +++ b/lib/crypto/tests/aes-cmac-testvecs.h @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py aes-cmac */ + +static const struct { + size_t data_len; + u8 digest[AES_BLOCK_SIZE]; +} hash_testvecs[] = { + { + .data_len = 0, + .digest = { + 0x9a, 0xeb, 0x94, 0xc1, 0xe9, 0xc1, 0x57, 0x49, + 0x7e, 0xba, 0x66, 0x47, 0x9f, 0x03, 0x2c, 0x5b, + }, + }, + { + .data_len = 1, + .digest = { + 0x52, 0xef, 0x65, 0xda, 0x7b, 0x92, 0x0c, 0x0f, + 0xdd, 0xd6, 0xb9, 0x68, 0x3f, 0xcc, 0x5f, 0xea, + }, + }, + { + .data_len = 2, + .digest = { + 0xc3, 0x95, 0x15, 0xea, 0x16, 0x33, 0xbe, 0x5a, + 0xd4, 0x2c, 0x25, 0x06, 0x15, 0xc6, 0x10, 0x19, + }, + }, + { + .data_len = 3, + .digest = { + 0x82, 0x41, 0x41, 0xd5, 0x33, 0x26, 0x0b, 0xb6, + 0xc8, 0xf7, 0x8d, 0x76, 0x8a, 0xfc, 0x0e, 0xe4, + }, + }, + { + .data_len = 16, + .digest = { + 0x94, 0x09, 0x80, 0x07, 0xba, 0x7c, 0xed, 0xd2, + 0x74, 0x72, 0x30, 0x26, 0xb5, 0x11, 0x64, 0xc1, + }, + }, + { + .data_len = 32, + .digest = { + 0xeb, 0xcf, 0x1e, 0x67, 0x21, 0x64, 0x93, 0xa0, + 0xea, 0xc4, 0xb9, 0x2d, 0x55, 0xc8, 0xac, 0x99, + }, + }, + { + .data_len = 48, + .digest = { + 0xd0, 0xd6, 0xdb, 0xe2, 0x45, 0x98, 0x6a, 0x7a, + 0x5f, 0xd6, 0xcd, 0x9d, 0x12, 0x26, 0x20, 0x87, + }, + }, + { + .data_len = 49, + .digest = { + 0x63, 0x25, 0x3c, 0xe2, 0x2a, 0xfa, 0xe3, 0x1e, + 0x54, 0x10, 0x18, 0x28, 0xc6, 0xb8, 0xcb, 0x58, + }, + }, + { + .data_len = 63, + .digest = { + 0x4d, 0xab, 0xae, 0x99, 0x90, 0x13, 0x3f, 0x4f, + 0x42, 0x0f, 0x19, 0x94, 0xa2, 0x52, 0xfd, 0xaf, + }, + }, + { + .data_len = 64, + .digest = { + 0xf7, 0x49, 0xb9, 0xa7, 0xf9, 0x3e, 0xa0, 0xca, + 0xb2, 0x6c, 0xd7, 0x87, 0x7d, 0x1e, 0xd2, 0xcb, + }, + }, + { + .data_len = 65, + .digest = { + 0x27, 0x2c, 0xb7, 0xc8, 0xdd, 0x26, 0xa9, 0xfe, + 0x37, 0x64, 0x84, 0x38, 0xa5, 0x7e, 0xbc, 0x04, + }, + }, + { + .data_len = 127, + .digest = { + 0xfd, 0x1f, 0x01, 0xa4, 0xea, 0x9b, 0xbd, 0xef, + 0x09, 0x97, 0x57, 0x60, 0x95, 0x23, 0xcc, 0x71, + }, + }, + { + .data_len = 128, + .digest = { + 0x28, 0x21, 0xee, 0x56, 0x9f, 0x38, 0xd6, 0x0e, + 0xe3, 0x22, 0x06, 0x20, 0xad, 0xd8, 0x33, 0x74, + }, + }, + { + .data_len = 129, + .digest = { + 0x07, 0x28, 0x4a, 0x2a, 0xd3, 0x85, 0xa6, 0x87, + 0x5c, 0x01, 0x8c, 0xb9, 0xd3, 0x4b, 0xce, 0x20, + }, + }, + { + .data_len = 256, + .digest = { + 0xe6, 0x12, 0x25, 0x6b, 0xf9, 0x69, 0x4d, 0x5a, + 0x1a, 0xb0, 0xe6, 0x11, 0x46, 0x24, 0x08, 0xdf, + }, + }, + { + .data_len = 511, + .digest = { + 0xce, 0x28, 0x1f, 0x14, 0xb9, 0xcc, 0x7e, 0x1f, + 0xb5, 0x13, 0x2b, 0x45, 0x04, 0x54, 0xe9, 0x5f, + }, + }, + { + .data_len = 513, + .digest = { + 0x63, 0x12, 0xbd, 0x85, 0x60, 0x1b, 0x99, 0x7e, + 0x0a, 0xf7, 0x0f, 0xc1, 0xb5, 0x66, 0xf8, 0x9a, + }, + }, + { + .data_len = 1000, + .digest = { + 0xbd, 0x49, 0x5e, 0x21, 0xc6, 0x58, 0x74, 0x6b, + 0x21, 0xc2, 0x62, 0x6a, 0x15, 0xca, 0x1d, 0x8a, + }, + }, + { + .data_len = 3333, + .digest = { + 0xfe, 0x6b, 0xfa, 0xfc, 0x4c, 0x0b, 0x63, 0x0d, + 0x41, 0x7f, 0xa9, 0xd8, 0xba, 0xe3, 0xce, 0xce, + }, + }, + { + .data_len = 4096, + .digest = { + 0x41, 0x7c, 0xbc, 0x2e, 0x2f, 0xff, 0xdf, 0x09, + 0x31, 0xc5, 0x79, 0x0a, 0x1d, 0x6e, 0x46, 0xec, + }, + }, + { + .data_len = 4128, + .digest = { + 0x6a, 0x9d, 0x86, 0xa8, 0xab, 0xa5, 0xc1, 0xc5, + 0x0d, 0x54, 0xf3, 0x51, 0x10, 0x46, 0x25, 0x5a, + }, + }, + { + .data_len = 4160, + .digest = { + 0x4c, 0x50, 0xff, 0x2a, 0xe9, 0xca, 0x9e, 0x07, + 0x8a, 0x86, 0x67, 0x5e, 0xe5, 0x0a, 0xfd, 0x69, + }, + }, + { + .data_len = 4224, + .digest = { + 0x3a, 0xfa, 0x80, 0x9d, 0x80, 0xe3, 0x1e, 0x95, + 0x53, 0x93, 0x39, 0x17, 0xd3, 0xda, 0x49, 0x15, + }, + }, + { + .data_len = 16384, + .digest = { + 0x48, 0xf4, 0x4f, 0x2d, 0x5d, 0xf2, 0x02, 0xcf, + 0x51, 0x3c, 0x1b, 0x12, 0x80, 0x8f, 0xb0, 0xd6, + }, + }, +}; + +static const u8 hash_testvec_consolidated[AES_BLOCK_SIZE] = { + 0x41, 0xad, 0x25, 0xa1, 0xeb, 0xce, 0x6b, 0x9c, + 0x06, 0xdf, 0x47, 0xc4, 0x3a, 0x59, 0x50, 0x07, +}; diff --git a/lib/crypto/tests/aes_cbc_macs_kunit.c b/lib/crypto/tests/aes_cbc_macs_kunit.c new file mode 100644 index 000000000000..ae3745212f03 --- /dev/null +++ b/lib/crypto/tests/aes_cbc_macs_kunit.c @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2026 Google LLC + */ +#include <crypto/aes-cbc-macs.h> +#include "aes-cmac-testvecs.h" + +/* + * A fixed key used when presenting AES-CMAC as an unkeyed hash function in + * order to reuse hash-test-template.h. At the beginning of the test suite, + * this is initialized to a key prepared from bytes generated from a fixed seed. + */ +static struct aes_cmac_key test_key; + +static void aes_cmac_init_withtestkey(struct aes_cmac_ctx *ctx) +{ + aes_cmac_init(ctx, &test_key); +} + +static void aes_cmac_withtestkey(const u8 *data, size_t data_len, + u8 out[AES_BLOCK_SIZE]) +{ + aes_cmac(&test_key, data, data_len, out); +} + +#define HASH aes_cmac_withtestkey +#define HASH_CTX aes_cmac_ctx +#define HASH_SIZE AES_BLOCK_SIZE +#define HASH_INIT aes_cmac_init_withtestkey +#define HASH_UPDATE aes_cmac_update +#define HASH_FINAL aes_cmac_final +#include "hash-test-template.h" + +static int aes_cbc_macs_suite_init(struct kunit_suite *suite) +{ + u8 raw_key[AES_KEYSIZE_256]; + int err; + + rand_bytes_seeded_from_len(raw_key, sizeof(raw_key)); + err = aes_cmac_preparekey(&test_key, raw_key, sizeof(raw_key)); + if (err) + return err; + return hash_suite_init(suite); +} + +static void aes_cbc_macs_suite_exit(struct kunit_suite *suite) +{ + hash_suite_exit(suite); +} + +/* Verify compatibility of the AES-CMAC implementation with RFC 4493. */ +static void test_aes_cmac_rfc4493(struct kunit *test) +{ + static const u8 raw_key[AES_KEYSIZE_128] = { + 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, + 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c, + }; + static const struct { + size_t data_len; + const u8 data[40]; + const u8 mac[AES_BLOCK_SIZE]; + } testvecs[] = { + { + /* Example 1 from RFC 4493 */ + .data_len = 0, + .mac = { + 0xbb, 0x1d, 0x69, 0x29, 0xe9, 0x59, 0x37, 0x28, + 0x7f, 0xa3, 0x7d, 0x12, 0x9b, 0x75, 0x67, 0x46, + }, + + }, + { + /* Example 2 from RFC 4493 */ + .data = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + }, + .data_len = 16, + .mac = { + 0x07, 0x0a, 0x16, 0xb4, 0x6b, 0x4d, 0x41, 0x44, + 0xf7, 0x9b, 0xdd, 0x9d, 0xd0, 0x4a, 0x28, 0x7c, + }, + }, + { + /* Example 3 from RFC 4493 */ + .data = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, + 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, + }, + .data_len = 40, + .mac = { + 0xdf, 0xa6, 0x67, 0x47, 0xde, 0x9a, 0xe6, 0x30, + 0x30, 0xca, 0x32, 0x61, 0x14, 0x97, 0xc8, 0x27, + }, + }, + }; + struct aes_cmac_key key; + int err; + + err = aes_cmac_preparekey(&key, raw_key, sizeof(raw_key)); + KUNIT_ASSERT_EQ(test, err, 0); + + for (size_t i = 0; i < ARRAY_SIZE(testvecs); i++) { + u8 mac[AES_BLOCK_SIZE]; + + aes_cmac(&key, testvecs[i].data, testvecs[i].data_len, mac); + KUNIT_ASSERT_MEMEQ(test, mac, testvecs[i].mac, AES_BLOCK_SIZE); + } +} + +/* + * Verify compatibility of the AES-XCBC-MAC implementation with RFC 3566. + * + * Additional AES-XCBC-MAC tests are not necessary, since the AES-XCBC-MAC + * implementation is well covered by the AES-CMAC tests already. Only the key + * preparation function differs; the rest of the code is shared. + */ +static void test_aes_xcbcmac_rfc3566(struct kunit *test) +{ + struct aes_cmac_key key; + /* AES-XCBC-MAC Test Case #4 from RFC 3566 */ + static const u8 raw_key[AES_KEYSIZE_128] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + }; + static const u8 message[20] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, + 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, + }; + static const u8 expected_mac[AES_BLOCK_SIZE] = { + 0x47, 0xf5, 0x1b, 0x45, 0x64, 0x96, 0x62, 0x15, + 0xb8, 0x98, 0x5c, 0x63, 0x05, 0x5e, 0xd3, 0x08, + }; + u8 actual_mac[AES_BLOCK_SIZE]; + + aes_xcbcmac_preparekey(&key, raw_key); + aes_cmac(&key, message, sizeof(message), actual_mac); + KUNIT_ASSERT_MEMEQ(test, actual_mac, expected_mac, AES_BLOCK_SIZE); +} + +static void test_aes_cbcmac_rfc3610(struct kunit *test) +{ + /* + * The following AES-CBC-MAC test vector is extracted from RFC 3610 + * Packet Vector #11. It required some rearrangement to get the actual + * input to AES-CBC-MAC from the values given. + */ + static const u8 raw_key[AES_KEYSIZE_128] = { + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + }; + const size_t unpadded_data_len = 52; + static const u8 data[64] = { + /* clang-format off */ + /* CCM header */ + 0x61, 0x00, 0x00, 0x00, 0x0d, 0x0c, 0x0b, 0x0a, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0x00, 0x14, + /* CCM additional authentication blocks */ + 0x00, 0x0c, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, + 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x00, 0x00, + /* CCM message blocks */ + 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, + 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, + 0x1c, 0x1d, 0x1e, 0x1f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* clang-format on */ + }; + static const u8 expected_mac[AES_BLOCK_SIZE] = { + 0x6b, 0x5e, 0x24, 0x34, 0x12, 0xcc, 0xc2, 0xad, + 0x6f, 0x1b, 0x11, 0xc3, 0xa1, 0xa9, 0xd8, 0xbc, + }; + struct aes_enckey key; + struct aes_cbcmac_ctx ctx; + u8 actual_mac[AES_BLOCK_SIZE]; + int err; + + err = aes_prepareenckey(&key, raw_key, sizeof(raw_key)); + KUNIT_ASSERT_EQ(test, err, 0); + + /* + * Trailing zeroes should not affect the CBC-MAC value, up to the next + * AES block boundary. + */ + for (size_t data_len = unpadded_data_len; data_len <= sizeof(data); + data_len++) { + aes_cbcmac_init(&ctx, &key); + aes_cbcmac_update(&ctx, data, data_len); + aes_cbcmac_final(&ctx, actual_mac); + KUNIT_ASSERT_MEMEQ(test, actual_mac, expected_mac, + AES_BLOCK_SIZE); + + /* Incremental computations should produce the same result. */ + for (size_t part1_len = 0; part1_len <= data_len; part1_len++) { + aes_cbcmac_init(&ctx, &key); + aes_cbcmac_update(&ctx, data, part1_len); + aes_cbcmac_update(&ctx, &data[part1_len], + data_len - part1_len); + aes_cbcmac_final(&ctx, actual_mac); + KUNIT_ASSERT_MEMEQ(test, actual_mac, expected_mac, + AES_BLOCK_SIZE); + } + } +} + +static struct kunit_case aes_cbc_macs_test_cases[] = { + HASH_KUNIT_CASES, + KUNIT_CASE(test_aes_cmac_rfc4493), + KUNIT_CASE(test_aes_xcbcmac_rfc3566), + KUNIT_CASE(test_aes_cbcmac_rfc3610), + KUNIT_CASE(benchmark_hash), + {}, +}; + +static struct kunit_suite aes_cbc_macs_test_suite = { + .name = "aes_cbc_macs", + .test_cases = aes_cbc_macs_test_cases, + .suite_init = aes_cbc_macs_suite_init, + .suite_exit = aes_cbc_macs_suite_exit, +}; +kunit_test_suite(aes_cbc_macs_test_suite); + +MODULE_DESCRIPTION( + "KUnit tests and benchmark for AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC"); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/chacha20poly1305-selftest.c b/lib/crypto/tests/chacha20poly1305_kunit.c index e4c85bc5a6d7..97a68fab88a7 100644 --- a/lib/crypto/chacha20poly1305-selftest.c +++ b/lib/crypto/tests/chacha20poly1305_kunit.c @@ -6,9 +6,8 @@ #include <crypto/chacha20poly1305.h> #include <crypto/chacha.h> #include <crypto/poly1305.h> - +#include <kunit/test.h> #include <linux/unaligned.h> -#include <linux/bug.h> #include <linux/init.h> #include <linux/mm.h> #include <linux/kernel.h> @@ -27,7 +26,7 @@ struct chacha20poly1305_testvec { * chapoly construction. */ -static const u8 enc_input001[] __initconst = { +static const u8 enc_input001[] = { 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, @@ -63,7 +62,7 @@ static const u8 enc_input001[] __initconst = { 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d }; -static const u8 enc_output001[] __initconst = { +static const u8 enc_output001[] = { 0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4, 0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd, 0x5e, 0x80, 0x5c, 0xfd, 0x34, 0x5c, 0xf3, 0x89, @@ -101,95 +100,95 @@ static const u8 enc_output001[] __initconst = { 0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f, 0x38 }; -static const u8 enc_assoc001[] __initconst = { +static const u8 enc_assoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; -static const u8 enc_nonce001[] __initconst = { +static const u8 enc_nonce001[] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 }; -static const u8 enc_key001[] __initconst = { +static const u8 enc_key001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 }; -static const u8 enc_input002[] __initconst = { }; -static const u8 enc_output002[] __initconst = { +static const u8 enc_input002[] = { }; +static const u8 enc_output002[] = { 0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1, 0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92 }; -static const u8 enc_assoc002[] __initconst = { }; -static const u8 enc_nonce002[] __initconst = { +static const u8 enc_assoc002[] = { }; +static const u8 enc_nonce002[] = { 0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e }; -static const u8 enc_key002[] __initconst = { +static const u8 enc_key002[] = { 0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f, 0x2d, 0x29, 0x25, 0x76, 0xd5, 0x75, 0x27, 0x86, 0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, 0xc5, 0xef, 0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68 }; -static const u8 enc_input003[] __initconst = { }; -static const u8 enc_output003[] __initconst = { +static const u8 enc_input003[] = { }; +static const u8 enc_output003[] = { 0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6, 0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77 }; -static const u8 enc_assoc003[] __initconst = { +static const u8 enc_assoc003[] = { 0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b }; -static const u8 enc_nonce003[] __initconst = { +static const u8 enc_nonce003[] = { 0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d }; -static const u8 enc_key003[] __initconst = { +static const u8 enc_key003[] = { 0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88, 0x34, 0xd1, 0x13, 0xaf, 0x57, 0xa1, 0xeb, 0x3a, 0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, 0xbc, 0x08, 0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d }; -static const u8 enc_input004[] __initconst = { +static const u8 enc_input004[] = { 0xa4 }; -static const u8 enc_output004[] __initconst = { +static const u8 enc_output004[] = { 0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2, 0x6d, 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac, 0x89 }; -static const u8 enc_assoc004[] __initconst = { +static const u8 enc_assoc004[] = { 0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40 }; -static const u8 enc_nonce004[] __initconst = { +static const u8 enc_nonce004[] = { 0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4 }; -static const u8 enc_key004[] __initconst = { +static const u8 enc_key004[] = { 0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8, 0x31, 0x80, 0x82, 0xd7, 0xd8, 0xe8, 0xb5, 0xa1, 0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, 0xa3, 0x3d, 0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e }; -static const u8 enc_input005[] __initconst = { +static const u8 enc_input005[] = { 0x2d }; -static const u8 enc_output005[] __initconst = { +static const u8 enc_output005[] = { 0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e, 0x6c, 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c, 0xac }; -static const u8 enc_assoc005[] __initconst = { }; -static const u8 enc_nonce005[] __initconst = { +static const u8 enc_assoc005[] = { }; +static const u8 enc_nonce005[] = { 0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30 }; -static const u8 enc_key005[] __initconst = { +static const u8 enc_key005[] = { 0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31, 0x0e, 0x92, 0x89, 0x8b, 0xf4, 0x93, 0xc7, 0x87, 0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, 0xa7, 0x01, 0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87 }; -static const u8 enc_input006[] __initconst = { +static const u8 enc_input006[] = { 0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a, 0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92, 0x3c, 0xd9, 0x24, 0x11, 0xa9, 0x71, 0xf9, 0x37, @@ -208,7 +207,7 @@ static const u8 enc_input006[] __initconst = { 0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde, 0x8f }; -static const u8 enc_output006[] __initconst = { +static const u8 enc_output006[] = { 0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1, 0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15, 0x81, 0x2c, 0xb5, 0xf0, 0xc6, 0x2b, 0xc7, 0x8c, @@ -229,20 +228,20 @@ static const u8 enc_output006[] __initconst = { 0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a, 0xeb }; -static const u8 enc_assoc006[] __initconst = { +static const u8 enc_assoc006[] = { 0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b }; -static const u8 enc_nonce006[] __initconst = { +static const u8 enc_nonce006[] = { 0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c }; -static const u8 enc_key006[] __initconst = { +static const u8 enc_key006[] = { 0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae, 0xdf, 0x72, 0x7f, 0x53, 0x72, 0x25, 0x1e, 0x78, 0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, 0x93, 0xf9, 0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01 }; -static const u8 enc_input007[] __initconst = { +static const u8 enc_input007[] = { 0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5, 0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a, 0x67, 0x30, 0x12, 0xe2, 0x34, 0x77, 0x4b, 0xc1, @@ -276,7 +275,7 @@ static const u8 enc_input007[] __initconst = { 0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, 0x2b, 0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3 }; -static const u8 enc_output007[] __initconst = { +static const u8 enc_output007[] = { 0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c, 0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8, 0xc9, 0x50, 0xc3, 0xc6, 0xa5, 0xe3, 0xa4, 0x7c, @@ -312,18 +311,18 @@ static const u8 enc_output007[] __initconst = { 0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42, 0x3a, 0x00, 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53 }; -static const u8 enc_assoc007[] __initconst = { }; -static const u8 enc_nonce007[] __initconst = { +static const u8 enc_assoc007[] = { }; +static const u8 enc_nonce007[] = { 0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0 }; -static const u8 enc_key007[] __initconst = { +static const u8 enc_key007[] = { 0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd, 0xf9, 0x3f, 0xcd, 0xd9, 0xa0, 0x1e, 0x42, 0x4c, 0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, 0x05, 0x80, 0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01 }; -static const u8 enc_input008[] __initconst = { +static const u8 enc_input008[] = { 0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10, 0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2, 0x0f, 0xc2, 0x8b, 0x28, 0xdc, 0xba, 0xb4, 0x3c, @@ -389,7 +388,7 @@ static const u8 enc_input008[] __initconst = { 0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b, 0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3 }; -static const u8 enc_output008[] __initconst = { +static const u8 enc_output008[] = { 0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd, 0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1, 0x1e, 0x6b, 0xd2, 0xbc, 0x11, 0xf4, 0x28, 0x93, @@ -457,18 +456,18 @@ static const u8 enc_output008[] __initconst = { 0x54, 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f, 0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54 }; -static const u8 enc_assoc008[] __initconst = { }; -static const u8 enc_nonce008[] __initconst = { +static const u8 enc_assoc008[] = { }; +static const u8 enc_nonce008[] = { 0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02 }; -static const u8 enc_key008[] __initconst = { +static const u8 enc_key008[] = { 0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53, 0xc1, 0x44, 0xe9, 0x81, 0x18, 0xdc, 0xf5, 0xf0, 0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, 0x44, 0x86, 0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba }; -static const u8 enc_input009[] __initconst = { +static const u8 enc_input009[] = { 0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b, 0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8, 0x3a, 0x6b, 0xd7, 0x81, 0x96, 0x35, 0x97, 0xca, @@ -535,7 +534,7 @@ static const u8 enc_input009[] __initconst = { 0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52, 0x65 }; -static const u8 enc_output009[] __initconst = { +static const u8 enc_output009[] = { 0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf, 0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66, 0x4b, 0x2e, 0x0c, 0x27, 0x9c, 0x96, 0x4c, 0x72, @@ -604,21 +603,21 @@ static const u8 enc_output009[] __initconst = { 0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8, 0xae }; -static const u8 enc_assoc009[] __initconst = { +static const u8 enc_assoc009[] = { 0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e, 0xef }; -static const u8 enc_nonce009[] __initconst = { +static const u8 enc_nonce009[] = { 0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78 }; -static const u8 enc_key009[] __initconst = { +static const u8 enc_key009[] = { 0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5, 0xcc, 0x1a, 0xd7, 0xc1, 0x57, 0x72, 0xea, 0x86, 0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, 0x9b, 0xb2, 0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b }; -static const u8 enc_input010[] __initconst = { +static const u8 enc_input010[] = { 0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf, 0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c, 0xef, 0x0a, 0xa9, 0x48, 0x5f, 0x5f, 0x37, 0x22, @@ -748,7 +747,7 @@ static const u8 enc_input010[] __initconst = { 0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c, 0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f }; -static const u8 enc_output010[] __initconst = { +static const u8 enc_output010[] = { 0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b, 0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74, 0xa6, 0xdd, 0xbd, 0x95, 0xeb, 0xf9, 0xa4, 0xf1, @@ -880,21 +879,21 @@ static const u8 enc_output010[] __initconst = { 0x80, 0x6b, 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9, 0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0 }; -static const u8 enc_assoc010[] __initconst = { +static const u8 enc_assoc010[] = { 0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27, 0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2 }; -static const u8 enc_nonce010[] __initconst = { +static const u8 enc_nonce010[] = { 0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30 }; -static const u8 enc_key010[] __initconst = { +static const u8 enc_key010[] = { 0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44, 0x34, 0xda, 0x7f, 0x57, 0x03, 0x39, 0x0c, 0xaf, 0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, 0x8e, 0x74, 0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7 }; -static const u8 enc_input011[] __initconst = { +static const u8 enc_input011[] = { 0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b, 0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b, 0x45, 0x60, 0xbe, 0x9a, 0x31, 0x9f, 0xff, 0x5d, @@ -1138,7 +1137,7 @@ static const u8 enc_input011[] __initconst = { 0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61, 0x10, 0x1e, 0xbf, 0xec, 0xa8 }; -static const u8 enc_output011[] __initconst = { +static const u8 enc_output011[] = { 0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8, 0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc, 0x3c, 0xba, 0x6a, 0x77, 0x47, 0xdb, 0xe3, 0x74, @@ -1384,20 +1383,20 @@ static const u8 enc_output011[] __initconst = { 0x86, 0xc4, 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e, 0x2b, 0xdf, 0xcd, 0xf9, 0x3c }; -static const u8 enc_assoc011[] __initconst = { +static const u8 enc_assoc011[] = { 0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7 }; -static const u8 enc_nonce011[] __initconst = { +static const u8 enc_nonce011[] = { 0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa }; -static const u8 enc_key011[] __initconst = { +static const u8 enc_key011[] = { 0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85, 0xf2, 0xfb, 0xed, 0x7b, 0xd0, 0x9e, 0x97, 0xca, 0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, 0xcc, 0x52, 0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38 }; -static const u8 enc_input012[] __initconst = { +static const u8 enc_input012[] = { 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, @@ -1651,7 +1650,7 @@ static const u8 enc_input012[] __initconst = { 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, 0x78, 0xec, 0x00 }; -static const u8 enc_output012[] __initconst = { +static const u8 enc_output012[] = { 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, @@ -1907,7 +1906,7 @@ static const u8 enc_output012[] __initconst = { 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, 0x70, 0xcf, 0xd6 }; -static const u8 enc_assoc012[] __initconst = { +static const u8 enc_assoc012[] = { 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, @@ -1917,10 +1916,10 @@ static const u8 enc_assoc012[] __initconst = { 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 }; -static const u8 enc_nonce012[] __initconst = { +static const u8 enc_nonce012[] = { 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 }; -static const u8 enc_key012[] __initconst = { +static const u8 enc_key012[] = { 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, @@ -1928,7 +1927,7 @@ static const u8 enc_key012[] __initconst = { }; /* wycheproof - rfc7539 */ -static const u8 enc_input013[] __initconst = { +static const u8 enc_input013[] = { 0x4c, 0x61, 0x64, 0x69, 0x65, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x47, 0x65, 0x6e, 0x74, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x20, 0x6f, 0x66, 0x20, @@ -1945,7 +1944,7 @@ static const u8 enc_input013[] __initconst = { 0x75, 0x6c, 0x64, 0x20, 0x62, 0x65, 0x20, 0x69, 0x74, 0x2e }; -static const u8 enc_output013[] __initconst = { +static const u8 enc_output013[] = { 0xd3, 0x1a, 0x8d, 0x34, 0x64, 0x8e, 0x60, 0xdb, 0x7b, 0x86, 0xaf, 0xbc, 0x53, 0xef, 0x7e, 0xc2, 0xa4, 0xad, 0xed, 0x51, 0x29, 0x6e, 0x08, 0xfe, @@ -1964,15 +1963,15 @@ static const u8 enc_output013[] __initconst = { 0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60, 0x06, 0x91 }; -static const u8 enc_assoc013[] __initconst = { +static const u8 enc_assoc013[] = { 0x50, 0x51, 0x52, 0x53, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7 }; -static const u8 enc_nonce013[] __initconst = { +static const u8 enc_nonce013[] = { 0x07, 0x00, 0x00, 0x00, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47 }; -static const u8 enc_key013[] __initconst = { +static const u8 enc_key013[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -1980,17 +1979,17 @@ static const u8 enc_key013[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input014[] __initconst = { }; -static const u8 enc_output014[] __initconst = { +static const u8 enc_input014[] = { }; +static const u8 enc_output014[] = { 0x76, 0xac, 0xb3, 0x42, 0xcf, 0x31, 0x66, 0xa5, 0xb6, 0x3c, 0x0c, 0x0e, 0xa1, 0x38, 0x3c, 0x8d }; -static const u8 enc_assoc014[] __initconst = { }; -static const u8 enc_nonce014[] __initconst = { +static const u8 enc_assoc014[] = { }; +static const u8 enc_nonce014[] = { 0x4d, 0xa5, 0xbf, 0x8d, 0xfd, 0x58, 0x52, 0xc1, 0xea, 0x12, 0x37, 0x9d }; -static const u8 enc_key014[] __initconst = { +static const u8 enc_key014[] = { 0x80, 0xba, 0x31, 0x92, 0xc8, 0x03, 0xce, 0x96, 0x5e, 0xa3, 0x71, 0xd5, 0xff, 0x07, 0x3c, 0xf0, 0xf4, 0x3b, 0x6a, 0x2a, 0xb5, 0x76, 0xb2, 0x08, @@ -1998,19 +1997,19 @@ static const u8 enc_key014[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input015[] __initconst = { }; -static const u8 enc_output015[] __initconst = { +static const u8 enc_input015[] = { }; +static const u8 enc_output015[] = { 0x90, 0x6f, 0xa6, 0x28, 0x4b, 0x52, 0xf8, 0x7b, 0x73, 0x59, 0xcb, 0xaa, 0x75, 0x63, 0xc7, 0x09 }; -static const u8 enc_assoc015[] __initconst = { +static const u8 enc_assoc015[] = { 0xbd, 0x50, 0x67, 0x64, 0xf2, 0xd2, 0xc4, 0x10 }; -static const u8 enc_nonce015[] __initconst = { +static const u8 enc_nonce015[] = { 0xa9, 0x2e, 0xf0, 0xac, 0x99, 0x1d, 0xd5, 0x16, 0xa3, 0xc6, 0xf6, 0x89 }; -static const u8 enc_key015[] __initconst = { +static const u8 enc_key015[] = { 0x7a, 0x4c, 0xd7, 0x59, 0x17, 0x2e, 0x02, 0xeb, 0x20, 0x4d, 0xb2, 0xc3, 0xf5, 0xc7, 0x46, 0x22, 0x7d, 0xf5, 0x84, 0xfc, 0x13, 0x45, 0x19, 0x63, @@ -2018,20 +2017,20 @@ static const u8 enc_key015[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input016[] __initconst = { +static const u8 enc_input016[] = { 0x2a }; -static const u8 enc_output016[] __initconst = { +static const u8 enc_output016[] = { 0x3a, 0xca, 0xc2, 0x7d, 0xec, 0x09, 0x68, 0x80, 0x1e, 0x9f, 0x6e, 0xde, 0xd6, 0x9d, 0x80, 0x75, 0x22 }; -static const u8 enc_assoc016[] __initconst = { }; -static const u8 enc_nonce016[] __initconst = { +static const u8 enc_assoc016[] = { }; +static const u8 enc_nonce016[] = { 0x99, 0xe2, 0x3e, 0xc4, 0x89, 0x85, 0xbc, 0xcd, 0xee, 0xab, 0x60, 0xf1 }; -static const u8 enc_key016[] __initconst = { +static const u8 enc_key016[] = { 0xcc, 0x56, 0xb6, 0x80, 0x55, 0x2e, 0xb7, 0x50, 0x08, 0xf5, 0x48, 0x4b, 0x4c, 0xb8, 0x03, 0xfa, 0x50, 0x63, 0xeb, 0xd6, 0xea, 0xb9, 0x1f, 0x6a, @@ -2039,22 +2038,22 @@ static const u8 enc_key016[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input017[] __initconst = { +static const u8 enc_input017[] = { 0x51 }; -static const u8 enc_output017[] __initconst = { +static const u8 enc_output017[] = { 0xc4, 0x16, 0x83, 0x10, 0xca, 0x45, 0xb1, 0xf7, 0xc6, 0x6c, 0xad, 0x4e, 0x99, 0xe4, 0x3f, 0x72, 0xb9 }; -static const u8 enc_assoc017[] __initconst = { +static const u8 enc_assoc017[] = { 0x91, 0xca, 0x6c, 0x59, 0x2c, 0xbc, 0xca, 0x53 }; -static const u8 enc_nonce017[] __initconst = { +static const u8 enc_nonce017[] = { 0xab, 0x0d, 0xca, 0x71, 0x6e, 0xe0, 0x51, 0xd2, 0x78, 0x2f, 0x44, 0x03 }; -static const u8 enc_key017[] __initconst = { +static const u8 enc_key017[] = { 0x46, 0xf0, 0x25, 0x49, 0x65, 0xf7, 0x69, 0xd5, 0x2b, 0xdb, 0x4a, 0x70, 0xb4, 0x43, 0x19, 0x9f, 0x8e, 0xf2, 0x07, 0x52, 0x0d, 0x12, 0x20, 0xc5, @@ -2062,20 +2061,20 @@ static const u8 enc_key017[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input018[] __initconst = { +static const u8 enc_input018[] = { 0x5c, 0x60 }; -static const u8 enc_output018[] __initconst = { +static const u8 enc_output018[] = { 0x4d, 0x13, 0x91, 0xe8, 0xb6, 0x1e, 0xfb, 0x39, 0xc1, 0x22, 0x19, 0x54, 0x53, 0x07, 0x7b, 0x22, 0xe5, 0xe2 }; -static const u8 enc_assoc018[] __initconst = { }; -static const u8 enc_nonce018[] __initconst = { +static const u8 enc_assoc018[] = { }; +static const u8 enc_nonce018[] = { 0x46, 0x1a, 0xf1, 0x22, 0xe9, 0xf2, 0xe0, 0x34, 0x7e, 0x03, 0xf2, 0xdb }; -static const u8 enc_key018[] __initconst = { +static const u8 enc_key018[] = { 0x2f, 0x7f, 0x7e, 0x4f, 0x59, 0x2b, 0xb3, 0x89, 0x19, 0x49, 0x89, 0x74, 0x35, 0x07, 0xbf, 0x3e, 0xe9, 0xcb, 0xde, 0x17, 0x86, 0xb6, 0x69, 0x5f, @@ -2083,22 +2082,22 @@ static const u8 enc_key018[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input019[] __initconst = { +static const u8 enc_input019[] = { 0xdd, 0xf2 }; -static const u8 enc_output019[] __initconst = { +static const u8 enc_output019[] = { 0xb6, 0x0d, 0xea, 0xd0, 0xfd, 0x46, 0x97, 0xec, 0x2e, 0x55, 0x58, 0x23, 0x77, 0x19, 0xd0, 0x24, 0x37, 0xa2 }; -static const u8 enc_assoc019[] __initconst = { +static const u8 enc_assoc019[] = { 0x88, 0x36, 0x4f, 0xc8, 0x06, 0x05, 0x18, 0xbf }; -static const u8 enc_nonce019[] __initconst = { +static const u8 enc_nonce019[] = { 0x61, 0x54, 0x6b, 0xa5, 0xf1, 0x72, 0x05, 0x90, 0xb6, 0x04, 0x0a, 0xc6 }; -static const u8 enc_key019[] __initconst = { +static const u8 enc_key019[] = { 0xc8, 0x83, 0x3d, 0xce, 0x5e, 0xa9, 0xf2, 0x48, 0xaa, 0x20, 0x30, 0xea, 0xcf, 0xe7, 0x2b, 0xff, 0xe6, 0x9a, 0x62, 0x0c, 0xaf, 0x79, 0x33, 0x44, @@ -2106,20 +2105,20 @@ static const u8 enc_key019[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input020[] __initconst = { +static const u8 enc_input020[] = { 0xab, 0x85, 0xe9, 0xc1, 0x57, 0x17, 0x31 }; -static const u8 enc_output020[] __initconst = { +static const u8 enc_output020[] = { 0x5d, 0xfe, 0x34, 0x40, 0xdb, 0xb3, 0xc3, 0xed, 0x7a, 0x43, 0x4e, 0x26, 0x02, 0xd3, 0x94, 0x28, 0x1e, 0x0a, 0xfa, 0x9f, 0xb7, 0xaa, 0x42 }; -static const u8 enc_assoc020[] __initconst = { }; -static const u8 enc_nonce020[] __initconst = { +static const u8 enc_assoc020[] = { }; +static const u8 enc_nonce020[] = { 0x3c, 0x4e, 0x65, 0x4d, 0x66, 0x3f, 0xa4, 0x59, 0x6d, 0xc5, 0x5b, 0xb7 }; -static const u8 enc_key020[] __initconst = { +static const u8 enc_key020[] = { 0x55, 0x56, 0x81, 0x58, 0xd3, 0xa6, 0x48, 0x3f, 0x1f, 0x70, 0x21, 0xea, 0xb6, 0x9b, 0x70, 0x3f, 0x61, 0x42, 0x51, 0xca, 0xdc, 0x1a, 0xf5, 0xd3, @@ -2127,22 +2126,22 @@ static const u8 enc_key020[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input021[] __initconst = { +static const u8 enc_input021[] = { 0x4e, 0xe5, 0xcd, 0xa2, 0x0d, 0x42, 0x90 }; -static const u8 enc_output021[] __initconst = { +static const u8 enc_output021[] = { 0x4b, 0xd4, 0x72, 0x12, 0x94, 0x1c, 0xe3, 0x18, 0x5f, 0x14, 0x08, 0xee, 0x7f, 0xbf, 0x18, 0xf5, 0xab, 0xad, 0x6e, 0x22, 0x53, 0xa1, 0xba }; -static const u8 enc_assoc021[] __initconst = { +static const u8 enc_assoc021[] = { 0x84, 0xe4, 0x6b, 0xe8, 0xc0, 0x91, 0x90, 0x53 }; -static const u8 enc_nonce021[] __initconst = { +static const u8 enc_nonce021[] = { 0x58, 0x38, 0x93, 0x75, 0xc6, 0x9e, 0xe3, 0x98, 0xde, 0x94, 0x83, 0x96 }; -static const u8 enc_key021[] __initconst = { +static const u8 enc_key021[] = { 0xe3, 0xc0, 0x9e, 0x7f, 0xab, 0x1a, 0xef, 0xb5, 0x16, 0xda, 0x6a, 0x33, 0x02, 0x2a, 0x1d, 0xd4, 0xeb, 0x27, 0x2c, 0x80, 0xd5, 0x40, 0xc5, 0xda, @@ -2150,20 +2149,20 @@ static const u8 enc_key021[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input022[] __initconst = { +static const u8 enc_input022[] = { 0xbe, 0x33, 0x08, 0xf7, 0x2a, 0x2c, 0x6a, 0xed }; -static const u8 enc_output022[] __initconst = { +static const u8 enc_output022[] = { 0x8e, 0x94, 0x39, 0xa5, 0x6e, 0xee, 0xc8, 0x17, 0xfb, 0xe8, 0xa6, 0xed, 0x8f, 0xab, 0xb1, 0x93, 0x75, 0x39, 0xdd, 0x6c, 0x00, 0xe9, 0x00, 0x21 }; -static const u8 enc_assoc022[] __initconst = { }; -static const u8 enc_nonce022[] __initconst = { +static const u8 enc_assoc022[] = { }; +static const u8 enc_nonce022[] = { 0x4f, 0x07, 0xaf, 0xed, 0xfd, 0xc3, 0xb6, 0xc2, 0x36, 0x18, 0x23, 0xd3 }; -static const u8 enc_key022[] __initconst = { +static const u8 enc_key022[] = { 0x51, 0xe4, 0xbf, 0x2b, 0xad, 0x92, 0xb7, 0xaf, 0xf1, 0xa4, 0xbc, 0x05, 0x55, 0x0b, 0xa8, 0x1d, 0xf4, 0xb9, 0x6f, 0xab, 0xf4, 0x1c, 0x12, 0xc7, @@ -2171,22 +2170,22 @@ static const u8 enc_key022[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input023[] __initconst = { +static const u8 enc_input023[] = { 0xa4, 0xc9, 0xc2, 0x80, 0x1b, 0x71, 0xf7, 0xdf }; -static const u8 enc_output023[] __initconst = { +static const u8 enc_output023[] = { 0xb9, 0xb9, 0x10, 0x43, 0x3a, 0xf0, 0x52, 0xb0, 0x45, 0x30, 0xf5, 0x1a, 0xee, 0xe0, 0x24, 0xe0, 0xa4, 0x45, 0xa6, 0x32, 0x8f, 0xa6, 0x7a, 0x18 }; -static const u8 enc_assoc023[] __initconst = { +static const u8 enc_assoc023[] = { 0x66, 0xc0, 0xae, 0x70, 0x07, 0x6c, 0xb1, 0x4d }; -static const u8 enc_nonce023[] __initconst = { +static const u8 enc_nonce023[] = { 0xb4, 0xea, 0x66, 0x6e, 0xe1, 0x19, 0x56, 0x33, 0x66, 0x48, 0x4a, 0x78 }; -static const u8 enc_key023[] __initconst = { +static const u8 enc_key023[] = { 0x11, 0x31, 0xc1, 0x41, 0x85, 0x77, 0xa0, 0x54, 0xde, 0x7a, 0x4a, 0xc5, 0x51, 0x95, 0x0f, 0x1a, 0x05, 0x3f, 0x9a, 0xe4, 0x6e, 0x5b, 0x75, 0xfe, @@ -2194,22 +2193,22 @@ static const u8 enc_key023[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input024[] __initconst = { +static const u8 enc_input024[] = { 0x42, 0xba, 0xae, 0x59, 0x78, 0xfe, 0xaf, 0x5c, 0x36, 0x8d, 0x14, 0xe0 }; -static const u8 enc_output024[] __initconst = { +static const u8 enc_output024[] = { 0xff, 0x7d, 0xc2, 0x03, 0xb2, 0x6c, 0x46, 0x7a, 0x6b, 0x50, 0xdb, 0x33, 0x57, 0x8c, 0x0f, 0x27, 0x58, 0xc2, 0xe1, 0x4e, 0x36, 0xd4, 0xfc, 0x10, 0x6d, 0xcb, 0x29, 0xb4 }; -static const u8 enc_assoc024[] __initconst = { }; -static const u8 enc_nonce024[] __initconst = { +static const u8 enc_assoc024[] = { }; +static const u8 enc_nonce024[] = { 0x9a, 0x59, 0xfc, 0xe2, 0x6d, 0xf0, 0x00, 0x5e, 0x07, 0x53, 0x86, 0x56 }; -static const u8 enc_key024[] __initconst = { +static const u8 enc_key024[] = { 0x99, 0xb6, 0x2b, 0xd5, 0xaf, 0xbe, 0x3f, 0xb0, 0x15, 0xbd, 0xe9, 0x3f, 0x0a, 0xbf, 0x48, 0x39, 0x57, 0xa1, 0xc3, 0xeb, 0x3c, 0xa5, 0x9c, 0xb5, @@ -2217,24 +2216,24 @@ static const u8 enc_key024[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input025[] __initconst = { +static const u8 enc_input025[] = { 0xfd, 0xc8, 0x5b, 0x94, 0xa4, 0xb2, 0xa6, 0xb7, 0x59, 0xb1, 0xa0, 0xda }; -static const u8 enc_output025[] __initconst = { +static const u8 enc_output025[] = { 0x9f, 0x88, 0x16, 0xde, 0x09, 0x94, 0xe9, 0x38, 0xd9, 0xe5, 0x3f, 0x95, 0xd0, 0x86, 0xfc, 0x6c, 0x9d, 0x8f, 0xa9, 0x15, 0xfd, 0x84, 0x23, 0xa7, 0xcf, 0x05, 0x07, 0x2f }; -static const u8 enc_assoc025[] __initconst = { +static const u8 enc_assoc025[] = { 0xa5, 0x06, 0xe1, 0xa5, 0xc6, 0x90, 0x93, 0xf9 }; -static const u8 enc_nonce025[] __initconst = { +static const u8 enc_nonce025[] = { 0x58, 0xdb, 0xd4, 0xad, 0x2c, 0x4a, 0xd3, 0x5d, 0xd9, 0x06, 0xe9, 0xce }; -static const u8 enc_key025[] __initconst = { +static const u8 enc_key025[] = { 0x85, 0xf3, 0x5b, 0x62, 0x82, 0xcf, 0xf4, 0x40, 0xbc, 0x10, 0x20, 0xc8, 0x13, 0x6f, 0xf2, 0x70, 0x31, 0x11, 0x0f, 0xa6, 0x3e, 0xc1, 0x6f, 0x1e, @@ -2242,22 +2241,22 @@ static const u8 enc_key025[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input026[] __initconst = { +static const u8 enc_input026[] = { 0x51, 0xf8, 0xc1, 0xf7, 0x31, 0xea, 0x14, 0xac, 0xdb, 0x21, 0x0a, 0x6d, 0x97, 0x3e, 0x07 }; -static const u8 enc_output026[] __initconst = { +static const u8 enc_output026[] = { 0x0b, 0x29, 0x63, 0x8e, 0x1f, 0xbd, 0xd6, 0xdf, 0x53, 0x97, 0x0b, 0xe2, 0x21, 0x00, 0x42, 0x2a, 0x91, 0x34, 0x08, 0x7d, 0x67, 0xa4, 0x6e, 0x79, 0x17, 0x8d, 0x0a, 0x93, 0xf5, 0xe1, 0xd2 }; -static const u8 enc_assoc026[] __initconst = { }; -static const u8 enc_nonce026[] __initconst = { +static const u8 enc_assoc026[] = { }; +static const u8 enc_nonce026[] = { 0x68, 0xab, 0x7f, 0xdb, 0xf6, 0x19, 0x01, 0xda, 0xd4, 0x61, 0xd2, 0x3c }; -static const u8 enc_key026[] __initconst = { +static const u8 enc_key026[] = { 0x67, 0x11, 0x96, 0x27, 0xbd, 0x98, 0x8e, 0xda, 0x90, 0x62, 0x19, 0xe0, 0x8c, 0x0d, 0x0d, 0x77, 0x9a, 0x07, 0xd2, 0x08, 0xce, 0x8a, 0x4f, 0xe0, @@ -2265,24 +2264,24 @@ static const u8 enc_key026[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input027[] __initconst = { +static const u8 enc_input027[] = { 0x97, 0x46, 0x9d, 0xa6, 0x67, 0xd6, 0x11, 0x0f, 0x9c, 0xbd, 0xa1, 0xd1, 0xa2, 0x06, 0x73 }; -static const u8 enc_output027[] __initconst = { +static const u8 enc_output027[] = { 0x32, 0xdb, 0x66, 0xc4, 0xa3, 0x81, 0x9d, 0x81, 0x55, 0x74, 0x55, 0xe5, 0x98, 0x0f, 0xed, 0xfe, 0xae, 0x30, 0xde, 0xc9, 0x4e, 0x6a, 0xd3, 0xa9, 0xee, 0xa0, 0x6a, 0x0d, 0x70, 0x39, 0x17 }; -static const u8 enc_assoc027[] __initconst = { +static const u8 enc_assoc027[] = { 0x64, 0x53, 0xa5, 0x33, 0x84, 0x63, 0x22, 0x12 }; -static const u8 enc_nonce027[] __initconst = { +static const u8 enc_nonce027[] = { 0xd9, 0x5b, 0x32, 0x43, 0xaf, 0xae, 0xf7, 0x14, 0xc5, 0x03, 0x5b, 0x6a }; -static const u8 enc_key027[] __initconst = { +static const u8 enc_key027[] = { 0xe6, 0xf1, 0x11, 0x8d, 0x41, 0xe4, 0xb4, 0x3f, 0xb5, 0x82, 0x21, 0xb7, 0xed, 0x79, 0x67, 0x38, 0x34, 0xe0, 0xd8, 0xac, 0x5c, 0x4f, 0xa6, 0x0b, @@ -2290,22 +2289,22 @@ static const u8 enc_key027[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input028[] __initconst = { +static const u8 enc_input028[] = { 0x54, 0x9b, 0x36, 0x5a, 0xf9, 0x13, 0xf3, 0xb0, 0x81, 0x13, 0x1c, 0xcb, 0x6b, 0x82, 0x55, 0x88 }; -static const u8 enc_output028[] __initconst = { +static const u8 enc_output028[] = { 0xe9, 0x11, 0x0e, 0x9f, 0x56, 0xab, 0x3c, 0xa4, 0x83, 0x50, 0x0c, 0xea, 0xba, 0xb6, 0x7a, 0x13, 0x83, 0x6c, 0xca, 0xbf, 0x15, 0xa6, 0xa2, 0x2a, 0x51, 0xc1, 0x07, 0x1c, 0xfa, 0x68, 0xfa, 0x0c }; -static const u8 enc_assoc028[] __initconst = { }; -static const u8 enc_nonce028[] __initconst = { +static const u8 enc_assoc028[] = { }; +static const u8 enc_nonce028[] = { 0x2f, 0xcb, 0x1b, 0x38, 0xa9, 0x9e, 0x71, 0xb8, 0x47, 0x40, 0xad, 0x9b }; -static const u8 enc_key028[] __initconst = { +static const u8 enc_key028[] = { 0x59, 0xd4, 0xea, 0xfb, 0x4d, 0xe0, 0xcf, 0xc7, 0xd3, 0xdb, 0x99, 0xa8, 0xf5, 0x4b, 0x15, 0xd7, 0xb3, 0x9f, 0x0a, 0xcc, 0x8d, 0xa6, 0x97, 0x63, @@ -2313,24 +2312,24 @@ static const u8 enc_key028[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input029[] __initconst = { +static const u8 enc_input029[] = { 0x55, 0xa4, 0x65, 0x64, 0x4f, 0x5b, 0x65, 0x09, 0x28, 0xcb, 0xee, 0x7c, 0x06, 0x32, 0x14, 0xd6 }; -static const u8 enc_output029[] __initconst = { +static const u8 enc_output029[] = { 0xe4, 0xb1, 0x13, 0xcb, 0x77, 0x59, 0x45, 0xf3, 0xd3, 0xa8, 0xae, 0x9e, 0xc1, 0x41, 0xc0, 0x0c, 0x7c, 0x43, 0xf1, 0x6c, 0xe0, 0x96, 0xd0, 0xdc, 0x27, 0xc9, 0x58, 0x49, 0xdc, 0x38, 0x3b, 0x7d }; -static const u8 enc_assoc029[] __initconst = { +static const u8 enc_assoc029[] = { 0x03, 0x45, 0x85, 0x62, 0x1a, 0xf8, 0xd7, 0xff }; -static const u8 enc_nonce029[] __initconst = { +static const u8 enc_nonce029[] = { 0x11, 0x8a, 0x69, 0x64, 0xc2, 0xd3, 0xe3, 0x80, 0x07, 0x1f, 0x52, 0x66 }; -static const u8 enc_key029[] __initconst = { +static const u8 enc_key029[] = { 0xb9, 0x07, 0xa4, 0x50, 0x75, 0x51, 0x3f, 0xe8, 0xa8, 0x01, 0x9e, 0xde, 0xe3, 0xf2, 0x59, 0x14, 0x87, 0xb2, 0xa0, 0x30, 0xb0, 0x3c, 0x6e, 0x1d, @@ -2338,24 +2337,24 @@ static const u8 enc_key029[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input030[] __initconst = { +static const u8 enc_input030[] = { 0x3f, 0xf1, 0x51, 0x4b, 0x1c, 0x50, 0x39, 0x15, 0x91, 0x8f, 0x0c, 0x0c, 0x31, 0x09, 0x4a, 0x6e, 0x1f }; -static const u8 enc_output030[] __initconst = { +static const u8 enc_output030[] = { 0x02, 0xcc, 0x3a, 0xcb, 0x5e, 0xe1, 0xfc, 0xdd, 0x12, 0xa0, 0x3b, 0xb8, 0x57, 0x97, 0x64, 0x74, 0xd3, 0xd8, 0x3b, 0x74, 0x63, 0xa2, 0xc3, 0x80, 0x0f, 0xe9, 0x58, 0xc2, 0x8e, 0xaa, 0x29, 0x08, 0x13 }; -static const u8 enc_assoc030[] __initconst = { }; -static const u8 enc_nonce030[] __initconst = { +static const u8 enc_assoc030[] = { }; +static const u8 enc_nonce030[] = { 0x45, 0xaa, 0xa3, 0xe5, 0xd1, 0x6d, 0x2d, 0x42, 0xdc, 0x03, 0x44, 0x5d }; -static const u8 enc_key030[] __initconst = { +static const u8 enc_key030[] = { 0x3b, 0x24, 0x58, 0xd8, 0x17, 0x6e, 0x16, 0x21, 0xc0, 0xcc, 0x24, 0xc0, 0xc0, 0xe2, 0x4c, 0x1e, 0x80, 0xd7, 0x2f, 0x7e, 0xe9, 0x14, 0x9a, 0x4b, @@ -2363,26 +2362,26 @@ static const u8 enc_key030[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input031[] __initconst = { +static const u8 enc_input031[] = { 0x63, 0x85, 0x8c, 0xa3, 0xe2, 0xce, 0x69, 0x88, 0x7b, 0x57, 0x8a, 0x3c, 0x16, 0x7b, 0x42, 0x1c, 0x9c }; -static const u8 enc_output031[] __initconst = { +static const u8 enc_output031[] = { 0x35, 0x76, 0x64, 0x88, 0xd2, 0xbc, 0x7c, 0x2b, 0x8d, 0x17, 0xcb, 0xbb, 0x9a, 0xbf, 0xad, 0x9e, 0x6d, 0x1f, 0x39, 0x1e, 0x65, 0x7b, 0x27, 0x38, 0xdd, 0xa0, 0x84, 0x48, 0xcb, 0xa2, 0x81, 0x1c, 0xeb }; -static const u8 enc_assoc031[] __initconst = { +static const u8 enc_assoc031[] = { 0x9a, 0xaf, 0x29, 0x9e, 0xee, 0xa7, 0x8f, 0x79 }; -static const u8 enc_nonce031[] __initconst = { +static const u8 enc_nonce031[] = { 0xf0, 0x38, 0x4f, 0xb8, 0x76, 0x12, 0x14, 0x10, 0x63, 0x3d, 0x99, 0x3d }; -static const u8 enc_key031[] __initconst = { +static const u8 enc_key031[] = { 0xf6, 0x0c, 0x6a, 0x1b, 0x62, 0x57, 0x25, 0xf7, 0x6c, 0x70, 0x37, 0xb4, 0x8f, 0xe3, 0x57, 0x7f, 0xa7, 0xf7, 0xb8, 0x7b, 0x1b, 0xd5, 0xa9, 0x82, @@ -2390,24 +2389,24 @@ static const u8 enc_key031[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input032[] __initconst = { +static const u8 enc_input032[] = { 0x10, 0xf1, 0xec, 0xf9, 0xc6, 0x05, 0x84, 0x66, 0x5d, 0x9a, 0xe5, 0xef, 0xe2, 0x79, 0xe7, 0xf7, 0x37, 0x7e, 0xea, 0x69, 0x16, 0xd2, 0xb1, 0x11 }; -static const u8 enc_output032[] __initconst = { +static const u8 enc_output032[] = { 0x42, 0xf2, 0x6c, 0x56, 0xcb, 0x4b, 0xe2, 0x1d, 0x9d, 0x8d, 0x0c, 0x80, 0xfc, 0x99, 0xdd, 0xe0, 0x0d, 0x75, 0xf3, 0x80, 0x74, 0xbf, 0xe7, 0x64, 0x54, 0xaa, 0x7e, 0x13, 0xd4, 0x8f, 0xff, 0x7d, 0x75, 0x57, 0x03, 0x94, 0x57, 0x04, 0x0a, 0x3a }; -static const u8 enc_assoc032[] __initconst = { }; -static const u8 enc_nonce032[] __initconst = { +static const u8 enc_assoc032[] = { }; +static const u8 enc_nonce032[] = { 0xe6, 0xb1, 0xad, 0xf2, 0xfd, 0x58, 0xa8, 0x76, 0x2c, 0x65, 0xf3, 0x1b }; -static const u8 enc_key032[] __initconst = { +static const u8 enc_key032[] = { 0x02, 0x12, 0xa8, 0xde, 0x50, 0x07, 0xed, 0x87, 0xb3, 0x3f, 0x1a, 0x70, 0x90, 0xb6, 0x11, 0x4f, 0x9e, 0x08, 0xce, 0xfd, 0x96, 0x07, 0xf2, 0xc2, @@ -2415,26 +2414,26 @@ static const u8 enc_key032[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input033[] __initconst = { +static const u8 enc_input033[] = { 0x92, 0x22, 0xf9, 0x01, 0x8e, 0x54, 0xfd, 0x6d, 0xe1, 0x20, 0x08, 0x06, 0xa9, 0xee, 0x8e, 0x4c, 0xc9, 0x04, 0xd2, 0x9f, 0x25, 0xcb, 0xa1, 0x93 }; -static const u8 enc_output033[] __initconst = { +static const u8 enc_output033[] = { 0x12, 0x30, 0x32, 0x43, 0x7b, 0x4b, 0xfd, 0x69, 0x20, 0xe8, 0xf7, 0xe7, 0xe0, 0x08, 0x7a, 0xe4, 0x88, 0x9e, 0xbe, 0x7a, 0x0a, 0xd0, 0xe9, 0x00, 0x3c, 0xf6, 0x8f, 0x17, 0x95, 0x50, 0xda, 0x63, 0xd3, 0xb9, 0x6c, 0x2d, 0x55, 0x41, 0x18, 0x65 }; -static const u8 enc_assoc033[] __initconst = { +static const u8 enc_assoc033[] = { 0x3e, 0x8b, 0xc5, 0xad, 0xe1, 0x82, 0xff, 0x08 }; -static const u8 enc_nonce033[] __initconst = { +static const u8 enc_nonce033[] = { 0x6b, 0x28, 0x2e, 0xbe, 0xcc, 0x54, 0x1b, 0xcd, 0x78, 0x34, 0xed, 0x55 }; -static const u8 enc_key033[] __initconst = { +static const u8 enc_key033[] = { 0xc5, 0xbc, 0x09, 0x56, 0x56, 0x46, 0xe7, 0xed, 0xda, 0x95, 0x4f, 0x1f, 0x73, 0x92, 0x23, 0xda, 0xda, 0x20, 0xb9, 0x5c, 0x44, 0xab, 0x03, 0x3d, @@ -2442,13 +2441,13 @@ static const u8 enc_key033[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input034[] __initconst = { +static const u8 enc_input034[] = { 0xb0, 0x53, 0x99, 0x92, 0x86, 0xa2, 0x82, 0x4f, 0x42, 0xcc, 0x8c, 0x20, 0x3a, 0xb2, 0x4e, 0x2c, 0x97, 0xa6, 0x85, 0xad, 0xcc, 0x2a, 0xd3, 0x26, 0x62, 0x55, 0x8e, 0x55, 0xa5, 0xc7, 0x29 }; -static const u8 enc_output034[] __initconst = { +static const u8 enc_output034[] = { 0x45, 0xc7, 0xd6, 0xb5, 0x3a, 0xca, 0xd4, 0xab, 0xb6, 0x88, 0x76, 0xa6, 0xe9, 0x6a, 0x48, 0xfb, 0x59, 0x52, 0x4d, 0x2c, 0x92, 0xc9, 0xd8, 0xa1, @@ -2456,12 +2455,12 @@ static const u8 enc_output034[] __initconst = { 0x6d, 0x3c, 0xa1, 0x0e, 0x31, 0x1b, 0x69, 0x5f, 0x3e, 0xae, 0x15, 0x51, 0x65, 0x24, 0x93 }; -static const u8 enc_assoc034[] __initconst = { }; -static const u8 enc_nonce034[] __initconst = { +static const u8 enc_assoc034[] = { }; +static const u8 enc_nonce034[] = { 0x04, 0xa9, 0xbe, 0x03, 0x50, 0x8a, 0x5f, 0x31, 0x37, 0x1a, 0x6f, 0xd2 }; -static const u8 enc_key034[] __initconst = { +static const u8 enc_key034[] = { 0x2e, 0xb5, 0x1c, 0x46, 0x9a, 0xa8, 0xeb, 0x9e, 0x6c, 0x54, 0xa8, 0x34, 0x9b, 0xae, 0x50, 0xa2, 0x0f, 0x0e, 0x38, 0x27, 0x11, 0xbb, 0xa1, 0x15, @@ -2469,13 +2468,13 @@ static const u8 enc_key034[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input035[] __initconst = { +static const u8 enc_input035[] = { 0xf4, 0x52, 0x06, 0xab, 0xc2, 0x55, 0x52, 0xb2, 0xab, 0xc9, 0xab, 0x7f, 0xa2, 0x43, 0x03, 0x5f, 0xed, 0xaa, 0xdd, 0xc3, 0xb2, 0x29, 0x39, 0x56, 0xf1, 0xea, 0x6e, 0x71, 0x56, 0xe7, 0xeb }; -static const u8 enc_output035[] __initconst = { +static const u8 enc_output035[] = { 0x46, 0xa8, 0x0c, 0x41, 0x87, 0x02, 0x47, 0x20, 0x08, 0x46, 0x27, 0x58, 0x00, 0x80, 0xdd, 0xe5, 0xa3, 0xf4, 0xa1, 0x10, 0x93, 0xa7, 0x07, 0x6e, @@ -2483,14 +2482,14 @@ static const u8 enc_output035[] __initconst = { 0x4d, 0x4a, 0xa2, 0x83, 0x5a, 0x52, 0xe7, 0x2d, 0x14, 0xdf, 0x0e, 0x4f, 0x47, 0xf2, 0x5f }; -static const u8 enc_assoc035[] __initconst = { +static const u8 enc_assoc035[] = { 0x37, 0x46, 0x18, 0xa0, 0x6e, 0xa9, 0x8a, 0x48 }; -static const u8 enc_nonce035[] __initconst = { +static const u8 enc_nonce035[] = { 0x47, 0x0a, 0x33, 0x9e, 0xcb, 0x32, 0x19, 0xb8, 0xb8, 0x1a, 0x1f, 0x8b }; -static const u8 enc_key035[] __initconst = { +static const u8 enc_key035[] = { 0x7f, 0x5b, 0x74, 0xc0, 0x7e, 0xd1, 0xb4, 0x0f, 0xd1, 0x43, 0x58, 0xfe, 0x2f, 0xf2, 0xa7, 0x40, 0xc1, 0x16, 0xc7, 0x70, 0x65, 0x10, 0xe6, 0xa4, @@ -2498,13 +2497,13 @@ static const u8 enc_key035[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input036[] __initconst = { +static const u8 enc_input036[] = { 0xb9, 0xc5, 0x54, 0xcb, 0xc3, 0x6a, 0xc1, 0x8a, 0xe8, 0x97, 0xdf, 0x7b, 0xee, 0xca, 0xc1, 0xdb, 0xeb, 0x4e, 0xaf, 0xa1, 0x56, 0xbb, 0x60, 0xce, 0x2e, 0x5d, 0x48, 0xf0, 0x57, 0x15, 0xe6, 0x78 }; -static const u8 enc_output036[] __initconst = { +static const u8 enc_output036[] = { 0xea, 0x29, 0xaf, 0xa4, 0x9d, 0x36, 0xe8, 0x76, 0x0f, 0x5f, 0xe1, 0x97, 0x23, 0xb9, 0x81, 0x1e, 0xd5, 0xd5, 0x19, 0x93, 0x4a, 0x44, 0x0f, 0x50, @@ -2512,12 +2511,12 @@ static const u8 enc_output036[] __initconst = { 0x22, 0x25, 0x41, 0xaf, 0x46, 0xb8, 0x65, 0x33, 0xc6, 0xb6, 0x8d, 0x2f, 0xf1, 0x08, 0xa7, 0xea }; -static const u8 enc_assoc036[] __initconst = { }; -static const u8 enc_nonce036[] __initconst = { +static const u8 enc_assoc036[] = { }; +static const u8 enc_nonce036[] = { 0x72, 0xcf, 0xd9, 0x0e, 0xf3, 0x02, 0x6c, 0xa2, 0x2b, 0x7e, 0x6e, 0x6a }; -static const u8 enc_key036[] __initconst = { +static const u8 enc_key036[] = { 0xe1, 0x73, 0x1d, 0x58, 0x54, 0xe1, 0xb7, 0x0c, 0xb3, 0xff, 0xe8, 0xb7, 0x86, 0xa2, 0xb3, 0xeb, 0xf0, 0x99, 0x43, 0x70, 0x95, 0x47, 0x57, 0xb9, @@ -2525,13 +2524,13 @@ static const u8 enc_key036[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input037[] __initconst = { +static const u8 enc_input037[] = { 0x6b, 0x26, 0x04, 0x99, 0x6c, 0xd3, 0x0c, 0x14, 0xa1, 0x3a, 0x52, 0x57, 0xed, 0x6c, 0xff, 0xd3, 0xbc, 0x5e, 0x29, 0xd6, 0xb9, 0x7e, 0xb1, 0x79, 0x9e, 0xb3, 0x35, 0xe2, 0x81, 0xea, 0x45, 0x1e }; -static const u8 enc_output037[] __initconst = { +static const u8 enc_output037[] = { 0x6d, 0xad, 0x63, 0x78, 0x97, 0x54, 0x4d, 0x8b, 0xf6, 0xbe, 0x95, 0x07, 0xed, 0x4d, 0x1b, 0xb2, 0xe9, 0x54, 0xbc, 0x42, 0x7e, 0x5d, 0xe7, 0x29, @@ -2539,14 +2538,14 @@ static const u8 enc_output037[] __initconst = { 0x7b, 0x99, 0x7d, 0x93, 0xc9, 0x82, 0x18, 0x9d, 0x70, 0x95, 0xdc, 0x79, 0x4c, 0x74, 0x62, 0x32 }; -static const u8 enc_assoc037[] __initconst = { +static const u8 enc_assoc037[] = { 0x23, 0x33, 0xe5, 0xce, 0x0f, 0x93, 0xb0, 0x59 }; -static const u8 enc_nonce037[] __initconst = { +static const u8 enc_nonce037[] = { 0x26, 0x28, 0x80, 0xd4, 0x75, 0xf3, 0xda, 0xc5, 0x34, 0x0d, 0xd1, 0xb8 }; -static const u8 enc_key037[] __initconst = { +static const u8 enc_key037[] = { 0x27, 0xd8, 0x60, 0x63, 0x1b, 0x04, 0x85, 0xa4, 0x10, 0x70, 0x2f, 0xea, 0x61, 0xbc, 0x87, 0x3f, 0x34, 0x42, 0x26, 0x0c, 0xad, 0xed, 0x4a, 0xbd, @@ -2554,7 +2553,7 @@ static const u8 enc_key037[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input038[] __initconst = { +static const u8 enc_input038[] = { 0x97, 0x3d, 0x0c, 0x75, 0x38, 0x26, 0xba, 0xe4, 0x66, 0xcf, 0x9a, 0xbb, 0x34, 0x93, 0x15, 0x2e, 0x9d, 0xe7, 0x81, 0x9e, 0x2b, 0xd0, 0xc7, 0x11, @@ -2562,7 +2561,7 @@ static const u8 enc_input038[] __initconst = { 0x1a, 0xa3, 0xce, 0xdc, 0x0d, 0xfd, 0x7b, 0x46, 0x7e, 0x26, 0x22, 0x8b, 0xc8, 0x6c, 0x9a }; -static const u8 enc_output038[] __initconst = { +static const u8 enc_output038[] = { 0xfb, 0xa7, 0x8a, 0xe4, 0xf9, 0xd8, 0x08, 0xa6, 0x2e, 0x3d, 0xa4, 0x0b, 0xe2, 0xcb, 0x77, 0x00, 0xc3, 0x61, 0x3d, 0x9e, 0xb2, 0xc5, 0x29, 0xc6, @@ -2572,12 +2571,12 @@ static const u8 enc_output038[] __initconst = { 0x04, 0x69, 0x56, 0xdb, 0x3a, 0x51, 0x29, 0x08, 0xbd, 0x7a, 0xfc, 0x8f, 0x2a, 0xb0, 0xa9 }; -static const u8 enc_assoc038[] __initconst = { }; -static const u8 enc_nonce038[] __initconst = { +static const u8 enc_assoc038[] = { }; +static const u8 enc_nonce038[] = { 0xe7, 0x4a, 0x51, 0x5e, 0x7e, 0x21, 0x02, 0xb9, 0x0b, 0xef, 0x55, 0xd2 }; -static const u8 enc_key038[] __initconst = { +static const u8 enc_key038[] = { 0xcf, 0x0d, 0x40, 0xa4, 0x64, 0x4e, 0x5f, 0x51, 0x81, 0x51, 0x65, 0xd5, 0x30, 0x1b, 0x22, 0x63, 0x1f, 0x45, 0x44, 0xc4, 0x9a, 0x18, 0x78, 0xe3, @@ -2585,7 +2584,7 @@ static const u8 enc_key038[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input039[] __initconst = { +static const u8 enc_input039[] = { 0xa9, 0x89, 0x95, 0x50, 0x4d, 0xf1, 0x6f, 0x74, 0x8b, 0xfb, 0x77, 0x85, 0xff, 0x91, 0xee, 0xb3, 0xb6, 0x60, 0xea, 0x9e, 0xd3, 0x45, 0x0c, 0x3d, @@ -2593,7 +2592,7 @@ static const u8 enc_input039[] __initconst = { 0xa9, 0x97, 0x8d, 0x75, 0x54, 0x2e, 0xf9, 0x1c, 0x45, 0x67, 0x62, 0x21, 0x56, 0x40, 0xb9 }; -static const u8 enc_output039[] __initconst = { +static const u8 enc_output039[] = { 0xa1, 0xff, 0xed, 0x80, 0x76, 0x18, 0x29, 0xec, 0xce, 0x24, 0x2e, 0x0e, 0x88, 0xb1, 0x38, 0x04, 0x90, 0x16, 0xbc, 0xa0, 0x18, 0xda, 0x2b, 0x6e, @@ -2603,14 +2602,14 @@ static const u8 enc_output039[] __initconst = { 0xcb, 0xf0, 0xbe, 0xfd, 0xa0, 0xb7, 0x02, 0x42, 0xc6, 0x40, 0xd7, 0xcd, 0x02, 0xd7, 0xa3 }; -static const u8 enc_assoc039[] __initconst = { +static const u8 enc_assoc039[] = { 0xb3, 0xe4, 0x06, 0x46, 0x83, 0xb0, 0x2d, 0x84 }; -static const u8 enc_nonce039[] __initconst = { +static const u8 enc_nonce039[] = { 0xd4, 0xd8, 0x07, 0x34, 0x16, 0x83, 0x82, 0x5b, 0x31, 0xcd, 0x4d, 0x95 }; -static const u8 enc_key039[] __initconst = { +static const u8 enc_key039[] = { 0x6c, 0xbf, 0xd7, 0x1c, 0x64, 0x5d, 0x18, 0x4c, 0xf5, 0xd2, 0x3c, 0x40, 0x2b, 0xdb, 0x0d, 0x25, 0xec, 0x54, 0x89, 0x8c, 0x8a, 0x02, 0x73, 0xd4, @@ -2618,7 +2617,7 @@ static const u8 enc_key039[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input040[] __initconst = { +static const u8 enc_input040[] = { 0xd0, 0x96, 0x80, 0x31, 0x81, 0xbe, 0xef, 0x9e, 0x00, 0x8f, 0xf8, 0x5d, 0x5d, 0xdc, 0x38, 0xdd, 0xac, 0xf0, 0xf0, 0x9e, 0xe5, 0xf7, 0xe0, 0x7f, @@ -2628,7 +2627,7 @@ static const u8 enc_input040[] __initconst = { 0x18, 0xf1, 0x18, 0x55, 0x86, 0xbf, 0xea, 0x9d, 0x4c, 0x68, 0x5d, 0x50, 0xe4, 0xbb, 0x9a, 0x82 }; -static const u8 enc_output040[] __initconst = { +static const u8 enc_output040[] = { 0x9a, 0x4e, 0xf2, 0x2b, 0x18, 0x16, 0x77, 0xb5, 0x75, 0x5c, 0x08, 0xf7, 0x47, 0xc0, 0xf8, 0xd8, 0xe8, 0xd4, 0xc1, 0x8a, 0x9c, 0xc2, 0x40, 0x5c, @@ -2640,12 +2639,12 @@ static const u8 enc_output040[] __initconst = { 0x9f, 0xf3, 0x42, 0x7a, 0x0f, 0x32, 0xfa, 0x56, 0x6d, 0x9c, 0xa0, 0xa7, 0x8a, 0xef, 0xc0, 0x13 }; -static const u8 enc_assoc040[] __initconst = { }; -static const u8 enc_nonce040[] __initconst = { +static const u8 enc_assoc040[] = { }; +static const u8 enc_nonce040[] = { 0xd6, 0x10, 0x40, 0xa3, 0x13, 0xed, 0x49, 0x28, 0x23, 0xcc, 0x06, 0x5b }; -static const u8 enc_key040[] __initconst = { +static const u8 enc_key040[] = { 0x5b, 0x1d, 0x10, 0x35, 0xc0, 0xb1, 0x7e, 0xe0, 0xb0, 0x44, 0x47, 0x67, 0xf8, 0x0a, 0x25, 0xb8, 0xc1, 0xb7, 0x41, 0xf4, 0xb5, 0x0a, 0x4d, 0x30, @@ -2653,7 +2652,7 @@ static const u8 enc_key040[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input041[] __initconst = { +static const u8 enc_input041[] = { 0x94, 0xee, 0x16, 0x6d, 0x6d, 0x6e, 0xcf, 0x88, 0x32, 0x43, 0x71, 0x36, 0xb4, 0xae, 0x80, 0x5d, 0x42, 0x88, 0x64, 0x35, 0x95, 0x86, 0xd9, 0x19, @@ -2663,7 +2662,7 @@ static const u8 enc_input041[] __initconst = { 0x4a, 0x10, 0x8c, 0x7d, 0x7c, 0xe3, 0x4e, 0x6c, 0x6f, 0x8e, 0xa1, 0xbe, 0xc0, 0x56, 0x73, 0x17 }; -static const u8 enc_output041[] __initconst = { +static const u8 enc_output041[] = { 0x5f, 0xbb, 0xde, 0xcc, 0x34, 0xbe, 0x20, 0x16, 0x14, 0xf6, 0x36, 0x03, 0x1e, 0xeb, 0x42, 0xf1, 0xca, 0xce, 0x3c, 0x79, 0xa1, 0x2c, 0xff, 0xd8, @@ -2675,14 +2674,14 @@ static const u8 enc_output041[] __initconst = { 0x09, 0x26, 0x3d, 0xa7, 0xb4, 0xcb, 0x92, 0x14, 0x52, 0xf9, 0x7d, 0xca, 0x40, 0xf5, 0x80, 0xec }; -static const u8 enc_assoc041[] __initconst = { +static const u8 enc_assoc041[] = { 0x71, 0x93, 0xf6, 0x23, 0x66, 0x33, 0x21, 0xa2 }; -static const u8 enc_nonce041[] __initconst = { +static const u8 enc_nonce041[] = { 0xd3, 0x1c, 0x21, 0xab, 0xa1, 0x75, 0xb7, 0x0d, 0xe4, 0xeb, 0xb1, 0x9c }; -static const u8 enc_key041[] __initconst = { +static const u8 enc_key041[] = { 0x97, 0xd6, 0x35, 0xc4, 0xf4, 0x75, 0x74, 0xd9, 0x99, 0x8a, 0x90, 0x87, 0x5d, 0xa1, 0xd3, 0xa2, 0x84, 0xb7, 0x55, 0xb2, 0xd3, 0x92, 0x97, 0xa5, @@ -2690,7 +2689,7 @@ static const u8 enc_key041[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input042[] __initconst = { +static const u8 enc_input042[] = { 0xb4, 0x29, 0xeb, 0x80, 0xfb, 0x8f, 0xe8, 0xba, 0xed, 0xa0, 0xc8, 0x5b, 0x9c, 0x33, 0x34, 0x58, 0xe7, 0xc2, 0x99, 0x2e, 0x55, 0x84, 0x75, 0x06, @@ -2705,7 +2704,7 @@ static const u8 enc_input042[] __initconst = { 0xc3, 0xea, 0x46, 0xc8, 0x76, 0x1d, 0x50, 0xf5, 0x92 }; -static const u8 enc_output042[] __initconst = { +static const u8 enc_output042[] = { 0xd0, 0x10, 0x2f, 0x6c, 0x25, 0x8b, 0xf4, 0x97, 0x42, 0xce, 0xc3, 0x4c, 0xf2, 0xd0, 0xfe, 0xdf, 0x23, 0xd1, 0x05, 0xfb, 0x4c, 0x84, 0xcf, 0x98, @@ -2722,12 +2721,12 @@ static const u8 enc_output042[] __initconst = { 0x19, 0x61, 0x21, 0x27, 0xce, 0x49, 0x99, 0x3b, 0xb0 }; -static const u8 enc_assoc042[] __initconst = { }; -static const u8 enc_nonce042[] __initconst = { +static const u8 enc_assoc042[] = { }; +static const u8 enc_nonce042[] = { 0x17, 0xc8, 0x6a, 0x8a, 0xbb, 0xb7, 0xe0, 0x03, 0xac, 0xde, 0x27, 0x99 }; -static const u8 enc_key042[] __initconst = { +static const u8 enc_key042[] = { 0xfe, 0x6e, 0x55, 0xbd, 0xae, 0xd1, 0xf7, 0x28, 0x4c, 0xa5, 0xfc, 0x0f, 0x8c, 0x5f, 0x2b, 0x8d, 0xf5, 0x6d, 0xc0, 0xf4, 0x9e, 0x8c, 0xa6, 0x6a, @@ -2735,7 +2734,7 @@ static const u8 enc_key042[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input043[] __initconst = { +static const u8 enc_input043[] = { 0xce, 0xb5, 0x34, 0xce, 0x50, 0xdc, 0x23, 0xff, 0x63, 0x8a, 0xce, 0x3e, 0xf6, 0x3a, 0xb2, 0xcc, 0x29, 0x73, 0xee, 0xad, 0xa8, 0x07, 0x85, 0xfc, @@ -2750,7 +2749,7 @@ static const u8 enc_input043[] __initconst = { 0xdc, 0x47, 0x97, 0x0d, 0x84, 0xa6, 0xad, 0x31, 0x76 }; -static const u8 enc_output043[] __initconst = { +static const u8 enc_output043[] = { 0x75, 0x45, 0x39, 0x1b, 0x51, 0xde, 0x01, 0xd5, 0xc5, 0x3d, 0xfa, 0xca, 0x77, 0x79, 0x09, 0x06, 0x3e, 0x58, 0xed, 0xee, 0x4b, 0xb1, 0x22, 0x7e, @@ -2767,14 +2766,14 @@ static const u8 enc_output043[] __initconst = { 0xc7, 0xbb, 0xf1, 0x0d, 0xca, 0xdd, 0x7f, 0xac, 0xf6 }; -static const u8 enc_assoc043[] __initconst = { +static const u8 enc_assoc043[] = { 0xa1, 0x1c, 0x40, 0xb6, 0x03, 0x76, 0x73, 0x30 }; -static const u8 enc_nonce043[] __initconst = { +static const u8 enc_nonce043[] = { 0x46, 0x36, 0x2f, 0x45, 0xd6, 0x37, 0x9e, 0x63, 0xe5, 0x22, 0x94, 0x60 }; -static const u8 enc_key043[] __initconst = { +static const u8 enc_key043[] = { 0xaa, 0xbc, 0x06, 0x34, 0x74, 0xe6, 0x5c, 0x4c, 0x3e, 0x9b, 0xdc, 0x48, 0x0d, 0xea, 0x97, 0xb4, 0x51, 0x10, 0xc8, 0x61, 0x88, 0x46, 0xff, 0x6b, @@ -2782,24 +2781,24 @@ static const u8 enc_key043[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input044[] __initconst = { +static const u8 enc_input044[] = { 0xe5, 0xcc, 0xaa, 0x44, 0x1b, 0xc8, 0x14, 0x68, 0x8f, 0x8f, 0x6e, 0x8f, 0x28, 0xb5, 0x00, 0xb2 }; -static const u8 enc_output044[] __initconst = { +static const u8 enc_output044[] = { 0x7e, 0x72, 0xf5, 0xa1, 0x85, 0xaf, 0x16, 0xa6, 0x11, 0x92, 0x1b, 0x43, 0x8f, 0x74, 0x9f, 0x0b, 0x12, 0x42, 0xc6, 0x70, 0x73, 0x23, 0x34, 0x02, 0x9a, 0xdf, 0xe1, 0xc5, 0x00, 0x16, 0x51, 0xe4 }; -static const u8 enc_assoc044[] __initconst = { +static const u8 enc_assoc044[] = { 0x02 }; -static const u8 enc_nonce044[] __initconst = { +static const u8 enc_nonce044[] = { 0x87, 0x34, 0x5f, 0x10, 0x55, 0xfd, 0x9e, 0x21, 0x02, 0xd5, 0x06, 0x56 }; -static const u8 enc_key044[] __initconst = { +static const u8 enc_key044[] = { 0x7d, 0x00, 0xb4, 0x80, 0x95, 0xad, 0xfa, 0x32, 0x72, 0x05, 0x06, 0x07, 0xb2, 0x64, 0x18, 0x50, 0x02, 0xba, 0x99, 0x95, 0x7c, 0x49, 0x8b, 0xe0, @@ -2807,24 +2806,24 @@ static const u8 enc_key044[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input045[] __initconst = { +static const u8 enc_input045[] = { 0x02, 0xcd, 0xe1, 0x68, 0xfb, 0xa3, 0xf5, 0x44, 0xbb, 0xd0, 0x33, 0x2f, 0x7a, 0xde, 0xad, 0xa8 }; -static const u8 enc_output045[] __initconst = { +static const u8 enc_output045[] = { 0x85, 0xf2, 0x9a, 0x71, 0x95, 0x57, 0xcd, 0xd1, 0x4d, 0x1f, 0x8f, 0xff, 0xab, 0x6d, 0x9e, 0x60, 0x73, 0x2c, 0xa3, 0x2b, 0xec, 0xd5, 0x15, 0xa1, 0xed, 0x35, 0x3f, 0x54, 0x2e, 0x99, 0x98, 0x58 }; -static const u8 enc_assoc045[] __initconst = { +static const u8 enc_assoc045[] = { 0xb6, 0x48 }; -static const u8 enc_nonce045[] __initconst = { +static const u8 enc_nonce045[] = { 0x87, 0xa3, 0x16, 0x3e, 0xc0, 0x59, 0x8a, 0xd9, 0x5b, 0x3a, 0xa7, 0x13 }; -static const u8 enc_key045[] __initconst = { +static const u8 enc_key045[] = { 0x64, 0x32, 0x71, 0x7f, 0x1d, 0xb8, 0x5e, 0x41, 0xac, 0x78, 0x36, 0xbc, 0xe2, 0x51, 0x85, 0xa0, 0x80, 0xd5, 0x76, 0x2b, 0x9e, 0x2b, 0x18, 0x44, @@ -2832,25 +2831,25 @@ static const u8 enc_key045[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input046[] __initconst = { +static const u8 enc_input046[] = { 0x16, 0xdd, 0xd2, 0x3f, 0xf5, 0x3f, 0x3d, 0x23, 0xc0, 0x63, 0x34, 0x48, 0x70, 0x40, 0xeb, 0x47 }; -static const u8 enc_output046[] __initconst = { +static const u8 enc_output046[] = { 0xc1, 0xb2, 0x95, 0x93, 0x6d, 0x56, 0xfa, 0xda, 0xc0, 0x3e, 0x5f, 0x74, 0x2b, 0xff, 0x73, 0xa1, 0x39, 0xc4, 0x57, 0xdb, 0xab, 0x66, 0x38, 0x2b, 0xab, 0xb3, 0xb5, 0x58, 0x00, 0xcd, 0xa5, 0xb8 }; -static const u8 enc_assoc046[] __initconst = { +static const u8 enc_assoc046[] = { 0xbd, 0x4c, 0xd0, 0x2f, 0xc7, 0x50, 0x2b, 0xbd, 0xbd, 0xf6, 0xc9, 0xa3, 0xcb, 0xe8, 0xf0 }; -static const u8 enc_nonce046[] __initconst = { +static const u8 enc_nonce046[] = { 0x6f, 0x57, 0x3a, 0xa8, 0x6b, 0xaa, 0x49, 0x2b, 0xa4, 0x65, 0x96, 0xdf }; -static const u8 enc_key046[] __initconst = { +static const u8 enc_key046[] = { 0x8e, 0x34, 0xcf, 0x73, 0xd2, 0x45, 0xa1, 0x08, 0x2a, 0x92, 0x0b, 0x86, 0x36, 0x4e, 0xb8, 0x96, 0xc4, 0x94, 0x64, 0x67, 0xbc, 0xb3, 0xd5, 0x89, @@ -2858,25 +2857,25 @@ static const u8 enc_key046[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input047[] __initconst = { +static const u8 enc_input047[] = { 0x62, 0x3b, 0x78, 0x50, 0xc3, 0x21, 0xe2, 0xcf, 0x0c, 0x6f, 0xbc, 0xc8, 0xdf, 0xd1, 0xaf, 0xf2 }; -static const u8 enc_output047[] __initconst = { +static const u8 enc_output047[] = { 0xc8, 0x4c, 0x9b, 0xb7, 0xc6, 0x1c, 0x1b, 0xcb, 0x17, 0x77, 0x2a, 0x1c, 0x50, 0x0c, 0x50, 0x95, 0xdb, 0xad, 0xf7, 0xa5, 0x13, 0x8c, 0xa0, 0x34, 0x59, 0xa2, 0xcd, 0x65, 0x83, 0x1e, 0x09, 0x2f }; -static const u8 enc_assoc047[] __initconst = { +static const u8 enc_assoc047[] = { 0x89, 0xcc, 0xe9, 0xfb, 0x47, 0x44, 0x1d, 0x07, 0xe0, 0x24, 0x5a, 0x66, 0xfe, 0x8b, 0x77, 0x8b }; -static const u8 enc_nonce047[] __initconst = { +static const u8 enc_nonce047[] = { 0x1a, 0x65, 0x18, 0xf0, 0x2e, 0xde, 0x1d, 0xa6, 0x80, 0x92, 0x66, 0xd9 }; -static const u8 enc_key047[] __initconst = { +static const u8 enc_key047[] = { 0xcb, 0x55, 0x75, 0xf5, 0xc7, 0xc4, 0x5c, 0x91, 0xcf, 0x32, 0x0b, 0x13, 0x9f, 0xb5, 0x94, 0x23, 0x75, 0x60, 0xd0, 0xa3, 0xe6, 0xf8, 0x65, 0xa6, @@ -2884,26 +2883,26 @@ static const u8 enc_key047[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input048[] __initconst = { +static const u8 enc_input048[] = { 0x87, 0xb3, 0xa4, 0xd7, 0xb2, 0x6d, 0x8d, 0x32, 0x03, 0xa0, 0xde, 0x1d, 0x64, 0xef, 0x82, 0xe3 }; -static const u8 enc_output048[] __initconst = { +static const u8 enc_output048[] = { 0x94, 0xbc, 0x80, 0x62, 0x1e, 0xd1, 0xe7, 0x1b, 0x1f, 0xd2, 0xb5, 0xc3, 0xa1, 0x5e, 0x35, 0x68, 0x33, 0x35, 0x11, 0x86, 0x17, 0x96, 0x97, 0x84, 0x01, 0x59, 0x8b, 0x96, 0x37, 0x22, 0xf5, 0xb3 }; -static const u8 enc_assoc048[] __initconst = { +static const u8 enc_assoc048[] = { 0xd1, 0x9f, 0x2d, 0x98, 0x90, 0x95, 0xf7, 0xab, 0x03, 0xa5, 0xfd, 0xe8, 0x44, 0x16, 0xe0, 0x0c, 0x0e }; -static const u8 enc_nonce048[] __initconst = { +static const u8 enc_nonce048[] = { 0x56, 0x4d, 0xee, 0x49, 0xab, 0x00, 0xd2, 0x40, 0xfc, 0x10, 0x68, 0xc3 }; -static const u8 enc_key048[] __initconst = { +static const u8 enc_key048[] = { 0xa5, 0x56, 0x9e, 0x72, 0x9a, 0x69, 0xb2, 0x4b, 0xa6, 0xe0, 0xff, 0x15, 0xc4, 0x62, 0x78, 0x97, 0x43, 0x68, 0x24, 0xc9, 0x41, 0xe9, 0xd0, 0x0b, @@ -2911,27 +2910,27 @@ static const u8 enc_key048[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input049[] __initconst = { +static const u8 enc_input049[] = { 0xe6, 0x01, 0xb3, 0x85, 0x57, 0x79, 0x7d, 0xa2, 0xf8, 0xa4, 0x10, 0x6a, 0x08, 0x9d, 0x1d, 0xa6 }; -static const u8 enc_output049[] __initconst = { +static const u8 enc_output049[] = { 0x29, 0x9b, 0x5d, 0x3f, 0x3d, 0x03, 0xc0, 0x87, 0x20, 0x9a, 0x16, 0xe2, 0x85, 0x14, 0x31, 0x11, 0x4b, 0x45, 0x4e, 0xd1, 0x98, 0xde, 0x11, 0x7e, 0x83, 0xec, 0x49, 0xfa, 0x8d, 0x85, 0x08, 0xd6 }; -static const u8 enc_assoc049[] __initconst = { +static const u8 enc_assoc049[] = { 0x5e, 0x64, 0x70, 0xfa, 0xcd, 0x99, 0xc1, 0xd8, 0x1e, 0x37, 0xcd, 0x44, 0x01, 0x5f, 0xe1, 0x94, 0x80, 0xa2, 0xa4, 0xd3, 0x35, 0x2a, 0x4f, 0xf5, 0x60, 0xc0, 0x64, 0x0f, 0xdb, 0xda }; -static const u8 enc_nonce049[] __initconst = { +static const u8 enc_nonce049[] = { 0xdf, 0x87, 0x13, 0xe8, 0x7e, 0xc3, 0xdb, 0xcf, 0xad, 0x14, 0xd5, 0x3e }; -static const u8 enc_key049[] __initconst = { +static const u8 enc_key049[] = { 0x56, 0x20, 0x74, 0x65, 0xb4, 0xe4, 0x8e, 0x6d, 0x04, 0x63, 0x0f, 0x4a, 0x42, 0xf3, 0x5c, 0xfc, 0x16, 0x3a, 0xb2, 0x89, 0xc2, 0x2a, 0x2b, 0x47, @@ -2939,27 +2938,27 @@ static const u8 enc_key049[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input050[] __initconst = { +static const u8 enc_input050[] = { 0xdc, 0x9e, 0x9e, 0xaf, 0x11, 0xe3, 0x14, 0x18, 0x2d, 0xf6, 0xa4, 0xeb, 0xa1, 0x7a, 0xec, 0x9c }; -static const u8 enc_output050[] __initconst = { +static const u8 enc_output050[] = { 0x60, 0x5b, 0xbf, 0x90, 0xae, 0xb9, 0x74, 0xf6, 0x60, 0x2b, 0xc7, 0x78, 0x05, 0x6f, 0x0d, 0xca, 0x38, 0xea, 0x23, 0xd9, 0x90, 0x54, 0xb4, 0x6b, 0x42, 0xff, 0xe0, 0x04, 0x12, 0x9d, 0x22, 0x04 }; -static const u8 enc_assoc050[] __initconst = { +static const u8 enc_assoc050[] = { 0xba, 0x44, 0x6f, 0x6f, 0x9a, 0x0c, 0xed, 0x22, 0x45, 0x0f, 0xeb, 0x10, 0x73, 0x7d, 0x90, 0x07, 0xfd, 0x69, 0xab, 0xc1, 0x9b, 0x1d, 0x4d, 0x90, 0x49, 0xa5, 0x55, 0x1e, 0x86, 0xec, 0x2b, 0x37 }; -static const u8 enc_nonce050[] __initconst = { +static const u8 enc_nonce050[] = { 0x8d, 0xf4, 0xb1, 0x5a, 0x88, 0x8c, 0x33, 0x28, 0x6a, 0x7b, 0x76, 0x51 }; -static const u8 enc_key050[] __initconst = { +static const u8 enc_key050[] = { 0x39, 0x37, 0x98, 0x6a, 0xf8, 0x6d, 0xaf, 0xc1, 0xba, 0x0c, 0x46, 0x72, 0xd8, 0xab, 0xc4, 0x6c, 0x20, 0x70, 0x62, 0x68, 0x2d, 0x9c, 0x26, 0x4a, @@ -2967,28 +2966,28 @@ static const u8 enc_key050[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input051[] __initconst = { +static const u8 enc_input051[] = { 0x81, 0xce, 0x84, 0xed, 0xe9, 0xb3, 0x58, 0x59, 0xcc, 0x8c, 0x49, 0xa8, 0xf6, 0xbe, 0x7d, 0xc6 }; -static const u8 enc_output051[] __initconst = { +static const u8 enc_output051[] = { 0x7b, 0x7c, 0xe0, 0xd8, 0x24, 0x80, 0x9a, 0x70, 0xde, 0x32, 0x56, 0x2c, 0xcf, 0x2c, 0x2b, 0xbd, 0x15, 0xd4, 0x4a, 0x00, 0xce, 0x0d, 0x19, 0xb4, 0x23, 0x1f, 0x92, 0x1e, 0x22, 0xbc, 0x0a, 0x43 }; -static const u8 enc_assoc051[] __initconst = { +static const u8 enc_assoc051[] = { 0xd4, 0x1a, 0x82, 0x8d, 0x5e, 0x71, 0x82, 0x92, 0x47, 0x02, 0x19, 0x05, 0x40, 0x2e, 0xa2, 0x57, 0xdc, 0xcb, 0xc3, 0xb8, 0x0f, 0xcd, 0x56, 0x75, 0x05, 0x6b, 0x68, 0xbb, 0x59, 0xe6, 0x2e, 0x88, 0x73 }; -static const u8 enc_nonce051[] __initconst = { +static const u8 enc_nonce051[] = { 0xbe, 0x40, 0xe5, 0xf1, 0xa1, 0x18, 0x17, 0xa0, 0xa8, 0xfa, 0x89, 0x49 }; -static const u8 enc_key051[] __initconst = { +static const u8 enc_key051[] = { 0x36, 0x37, 0x2a, 0xbc, 0xdb, 0x78, 0xe0, 0x27, 0x96, 0x46, 0xac, 0x3d, 0x17, 0x6b, 0x96, 0x74, 0xe9, 0x15, 0x4e, 0xec, 0xf0, 0xd5, 0x46, 0x9c, @@ -2996,17 +2995,17 @@ static const u8 enc_key051[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input052[] __initconst = { +static const u8 enc_input052[] = { 0xa6, 0x67, 0x47, 0xc8, 0x9e, 0x85, 0x7a, 0xf3, 0xa1, 0x8e, 0x2c, 0x79, 0x50, 0x00, 0x87, 0xed }; -static const u8 enc_output052[] __initconst = { +static const u8 enc_output052[] = { 0xca, 0x82, 0xbf, 0xf3, 0xe2, 0xf3, 0x10, 0xcc, 0xc9, 0x76, 0x67, 0x2c, 0x44, 0x15, 0xe6, 0x9b, 0x57, 0x63, 0x8c, 0x62, 0xa5, 0xd8, 0x5d, 0xed, 0x77, 0x4f, 0x91, 0x3c, 0x81, 0x3e, 0xa0, 0x32 }; -static const u8 enc_assoc052[] __initconst = { +static const u8 enc_assoc052[] = { 0x3f, 0x2d, 0xd4, 0x9b, 0xbf, 0x09, 0xd6, 0x9a, 0x78, 0xa3, 0xd8, 0x0e, 0xa2, 0x56, 0x66, 0x14, 0xfc, 0x37, 0x94, 0x74, 0x19, 0x6c, 0x1a, 0xae, @@ -3014,11 +3013,11 @@ static const u8 enc_assoc052[] __initconst = { 0x6f, 0x42, 0xca, 0x42, 0x05, 0x6a, 0x97, 0x92, 0xcc, 0x1b, 0x9f, 0xb3, 0xc7, 0xd2, 0x61 }; -static const u8 enc_nonce052[] __initconst = { +static const u8 enc_nonce052[] = { 0x84, 0xc8, 0x7d, 0xae, 0x4e, 0xee, 0x27, 0x73, 0x0e, 0xc3, 0x5d, 0x12 }; -static const u8 enc_key052[] __initconst = { +static const u8 enc_key052[] = { 0x9f, 0x14, 0x79, 0xed, 0x09, 0x7d, 0x7f, 0xe5, 0x29, 0xc1, 0x1f, 0x2f, 0x5a, 0xdd, 0x9a, 0xaf, 0xf4, 0xa1, 0xca, 0x0b, 0x68, 0x99, 0x7a, 0x2c, @@ -3026,13 +3025,13 @@ static const u8 enc_key052[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input053[] __initconst = { +static const u8 enc_input053[] = { 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, 0x80, 0xe0, 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe }; -static const u8 enc_output053[] __initconst = { +static const u8 enc_output053[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -3040,14 +3039,14 @@ static const u8 enc_output053[] __initconst = { 0xe6, 0xd3, 0xd7, 0x32, 0x4a, 0x1c, 0xbb, 0xa7, 0x77, 0xbb, 0xb0, 0xec, 0xdd, 0xa3, 0x78, 0x07 }; -static const u8 enc_assoc053[] __initconst = { +static const u8 enc_assoc053[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce053[] __initconst = { +static const u8 enc_nonce053[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key053[] __initconst = { +static const u8 enc_key053[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3055,7 +3054,7 @@ static const u8 enc_key053[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input054[] __initconst = { +static const u8 enc_input054[] = { 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, @@ -3065,7 +3064,7 @@ static const u8 enc_input054[] __initconst = { 0x08, 0x69, 0xff, 0xd2, 0xec, 0x5e, 0x26, 0xe5, 0x53, 0xb7, 0xb2, 0x27, 0xfe, 0x87, 0xfd, 0xbd }; -static const u8 enc_output054[] __initconst = { +static const u8 enc_output054[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -3077,14 +3076,14 @@ static const u8 enc_output054[] __initconst = { 0x06, 0x2d, 0xe6, 0x79, 0x5f, 0x27, 0x4f, 0xd2, 0xa3, 0x05, 0xd7, 0x69, 0x80, 0xbc, 0x9c, 0xce }; -static const u8 enc_assoc054[] __initconst = { +static const u8 enc_assoc054[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce054[] __initconst = { +static const u8 enc_nonce054[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key054[] __initconst = { +static const u8 enc_key054[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3092,7 +3091,7 @@ static const u8 enc_key054[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input055[] __initconst = { +static const u8 enc_input055[] = { 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, @@ -3110,7 +3109,7 @@ static const u8 enc_input055[] __initconst = { 0x77, 0xd3, 0x0b, 0xc5, 0x76, 0x92, 0xed, 0x38, 0xfb, 0xac, 0x01, 0x88, 0x38, 0x04, 0x88, 0xc7 }; -static const u8 enc_output055[] __initconst = { +static const u8 enc_output055[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -3130,14 +3129,14 @@ static const u8 enc_output055[] __initconst = { 0xd8, 0xb4, 0x79, 0x02, 0xba, 0xae, 0xaf, 0xb3, 0x42, 0x03, 0x05, 0x15, 0x29, 0xaf, 0x28, 0x2e }; -static const u8 enc_assoc055[] __initconst = { +static const u8 enc_assoc055[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce055[] __initconst = { +static const u8 enc_nonce055[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key055[] __initconst = { +static const u8 enc_key055[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3145,13 +3144,13 @@ static const u8 enc_key055[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input056[] __initconst = { +static const u8 enc_input056[] = { 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, 0x7f, 0x1f, 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41 }; -static const u8 enc_output056[] __initconst = { +static const u8 enc_output056[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3159,14 +3158,14 @@ static const u8 enc_output056[] __initconst = { 0xb3, 0x89, 0x1c, 0x84, 0x9c, 0xb5, 0x2c, 0x27, 0x74, 0x7e, 0xdf, 0xcf, 0x31, 0x21, 0x3b, 0xb6 }; -static const u8 enc_assoc056[] __initconst = { +static const u8 enc_assoc056[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce056[] __initconst = { +static const u8 enc_nonce056[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key056[] __initconst = { +static const u8 enc_key056[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3174,7 +3173,7 @@ static const u8 enc_key056[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input057[] __initconst = { +static const u8 enc_input057[] = { 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, @@ -3184,7 +3183,7 @@ static const u8 enc_input057[] __initconst = { 0xf7, 0x96, 0x00, 0x2d, 0x13, 0xa1, 0xd9, 0x1a, 0xac, 0x48, 0x4d, 0xd8, 0x01, 0x78, 0x02, 0x42 }; -static const u8 enc_output057[] __initconst = { +static const u8 enc_output057[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3196,14 +3195,14 @@ static const u8 enc_output057[] __initconst = { 0xf0, 0xc1, 0x2d, 0x26, 0xef, 0x03, 0x02, 0x9b, 0x62, 0xc0, 0x08, 0xda, 0x27, 0xc5, 0xdc, 0x68 }; -static const u8 enc_assoc057[] __initconst = { +static const u8 enc_assoc057[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce057[] __initconst = { +static const u8 enc_nonce057[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key057[] __initconst = { +static const u8 enc_key057[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3211,7 +3210,7 @@ static const u8 enc_key057[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input058[] __initconst = { +static const u8 enc_input058[] = { 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, @@ -3229,7 +3228,7 @@ static const u8 enc_input058[] __initconst = { 0x88, 0x2c, 0xf4, 0x3a, 0x89, 0x6d, 0x12, 0xc7, 0x04, 0x53, 0xfe, 0x77, 0xc7, 0xfb, 0x77, 0x38 }; -static const u8 enc_output058[] __initconst = { +static const u8 enc_output058[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3249,14 +3248,14 @@ static const u8 enc_output058[] __initconst = { 0xee, 0x65, 0x78, 0x30, 0x01, 0xc2, 0x56, 0x91, 0xfa, 0x28, 0xd0, 0xf5, 0xf1, 0xc1, 0xd7, 0x62 }; -static const u8 enc_assoc058[] __initconst = { +static const u8 enc_assoc058[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce058[] __initconst = { +static const u8 enc_nonce058[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key058[] __initconst = { +static const u8 enc_key058[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3264,13 +3263,13 @@ static const u8 enc_key058[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input059[] __initconst = { +static const u8 enc_input059[] = { 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, 0x80, 0xe0, 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e }; -static const u8 enc_output059[] __initconst = { +static const u8 enc_output059[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, @@ -3278,14 +3277,14 @@ static const u8 enc_output059[] __initconst = { 0x79, 0xba, 0x7a, 0x29, 0xf5, 0xa7, 0xbb, 0x75, 0x79, 0x7a, 0xf8, 0x7a, 0x61, 0x01, 0x29, 0xa4 }; -static const u8 enc_assoc059[] __initconst = { +static const u8 enc_assoc059[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; -static const u8 enc_nonce059[] __initconst = { +static const u8 enc_nonce059[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key059[] __initconst = { +static const u8 enc_key059[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3293,7 +3292,7 @@ static const u8 enc_key059[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input060[] __initconst = { +static const u8 enc_input060[] = { 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, @@ -3303,7 +3302,7 @@ static const u8 enc_input060[] __initconst = { 0x08, 0x69, 0xff, 0x52, 0xec, 0x5e, 0x26, 0x65, 0x53, 0xb7, 0xb2, 0xa7, 0xfe, 0x87, 0xfd, 0x3d }; -static const u8 enc_output060[] __initconst = { +static const u8 enc_output060[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, @@ -3315,14 +3314,14 @@ static const u8 enc_output060[] __initconst = { 0x36, 0xb1, 0x74, 0x38, 0x19, 0xe1, 0xb9, 0xba, 0x15, 0x51, 0xe8, 0xed, 0x92, 0x2a, 0x95, 0x9a }; -static const u8 enc_assoc060[] __initconst = { +static const u8 enc_assoc060[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; -static const u8 enc_nonce060[] __initconst = { +static const u8 enc_nonce060[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key060[] __initconst = { +static const u8 enc_key060[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3330,7 +3329,7 @@ static const u8 enc_key060[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input061[] __initconst = { +static const u8 enc_input061[] = { 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, @@ -3348,7 +3347,7 @@ static const u8 enc_input061[] __initconst = { 0x77, 0xd3, 0x0b, 0x45, 0x76, 0x92, 0xed, 0xb8, 0xfb, 0xac, 0x01, 0x08, 0x38, 0x04, 0x88, 0x47 }; -static const u8 enc_output061[] __initconst = { +static const u8 enc_output061[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, @@ -3368,14 +3367,14 @@ static const u8 enc_output061[] __initconst = { 0xfe, 0xac, 0x49, 0x55, 0x55, 0x4e, 0x80, 0x6f, 0x3a, 0x19, 0x02, 0xe2, 0x44, 0x32, 0xc0, 0x8a }; -static const u8 enc_assoc061[] __initconst = { +static const u8 enc_assoc061[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; -static const u8 enc_nonce061[] __initconst = { +static const u8 enc_nonce061[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key061[] __initconst = { +static const u8 enc_key061[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3383,13 +3382,13 @@ static const u8 enc_key061[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input062[] __initconst = { +static const u8 enc_input062[] = { 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, 0x7f, 0x1f, 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1 }; -static const u8 enc_output062[] __initconst = { +static const u8 enc_output062[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, @@ -3397,14 +3396,14 @@ static const u8 enc_output062[] __initconst = { 0x20, 0xa3, 0x79, 0x8d, 0xf1, 0x29, 0x2c, 0x59, 0x72, 0xbf, 0x97, 0x41, 0xae, 0xc3, 0x8a, 0x19 }; -static const u8 enc_assoc062[] __initconst = { +static const u8 enc_assoc062[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; -static const u8 enc_nonce062[] __initconst = { +static const u8 enc_nonce062[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key062[] __initconst = { +static const u8 enc_key062[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3412,7 +3411,7 @@ static const u8 enc_key062[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input063[] __initconst = { +static const u8 enc_input063[] = { 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, @@ -3422,7 +3421,7 @@ static const u8 enc_input063[] __initconst = { 0xf7, 0x96, 0x00, 0xad, 0x13, 0xa1, 0xd9, 0x9a, 0xac, 0x48, 0x4d, 0x58, 0x01, 0x78, 0x02, 0xc2 }; -static const u8 enc_output063[] __initconst = { +static const u8 enc_output063[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, @@ -3434,14 +3433,14 @@ static const u8 enc_output063[] __initconst = { 0xc0, 0x3d, 0x9f, 0x67, 0x35, 0x4a, 0x97, 0xb2, 0xf0, 0x74, 0xf7, 0x55, 0x15, 0x57, 0xe4, 0x9c }; -static const u8 enc_assoc063[] __initconst = { +static const u8 enc_assoc063[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; -static const u8 enc_nonce063[] __initconst = { +static const u8 enc_nonce063[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key063[] __initconst = { +static const u8 enc_key063[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3449,7 +3448,7 @@ static const u8 enc_key063[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input064[] __initconst = { +static const u8 enc_input064[] = { 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, @@ -3467,7 +3466,7 @@ static const u8 enc_input064[] __initconst = { 0x88, 0x2c, 0xf4, 0xba, 0x89, 0x6d, 0x12, 0x47, 0x04, 0x53, 0xfe, 0xf7, 0xc7, 0xfb, 0x77, 0xb8 }; -static const u8 enc_output064[] __initconst = { +static const u8 enc_output064[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, @@ -3487,14 +3486,14 @@ static const u8 enc_output064[] __initconst = { 0xc8, 0x6d, 0xa8, 0xdd, 0x65, 0x22, 0x86, 0xd5, 0x02, 0x13, 0xd3, 0x28, 0xd6, 0x3e, 0x40, 0x06 }; -static const u8 enc_assoc064[] __initconst = { +static const u8 enc_assoc064[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; -static const u8 enc_nonce064[] __initconst = { +static const u8 enc_nonce064[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key064[] __initconst = { +static const u8 enc_key064[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3502,13 +3501,13 @@ static const u8 enc_key064[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input065[] __initconst = { +static const u8 enc_input065[] = { 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, 0xff, 0x1f, 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41 }; -static const u8 enc_output065[] __initconst = { +static const u8 enc_output065[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, @@ -3516,14 +3515,14 @@ static const u8 enc_output065[] __initconst = { 0xbe, 0xde, 0x90, 0x83, 0xce, 0xb3, 0x6d, 0xdf, 0xe5, 0xfa, 0x81, 0x1f, 0x95, 0x47, 0x1c, 0x67 }; -static const u8 enc_assoc065[] __initconst = { +static const u8 enc_assoc065[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff }; -static const u8 enc_nonce065[] __initconst = { +static const u8 enc_nonce065[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key065[] __initconst = { +static const u8 enc_key065[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3531,7 +3530,7 @@ static const u8 enc_key065[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input066[] __initconst = { +static const u8 enc_input066[] = { 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, @@ -3541,7 +3540,7 @@ static const u8 enc_input066[] __initconst = { 0x77, 0x96, 0x00, 0x2d, 0x93, 0xa1, 0xd9, 0x1a, 0x2c, 0x48, 0x4d, 0xd8, 0x81, 0x78, 0x02, 0x42 }; -static const u8 enc_output066[] __initconst = { +static const u8 enc_output066[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, @@ -3553,14 +3552,14 @@ static const u8 enc_output066[] __initconst = { 0x30, 0x08, 0x74, 0xbb, 0x06, 0x92, 0xb6, 0x89, 0xde, 0xad, 0x9a, 0xe1, 0x5b, 0x06, 0x73, 0x90 }; -static const u8 enc_assoc066[] __initconst = { +static const u8 enc_assoc066[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff }; -static const u8 enc_nonce066[] __initconst = { +static const u8 enc_nonce066[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key066[] __initconst = { +static const u8 enc_key066[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3568,7 +3567,7 @@ static const u8 enc_key066[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input067[] __initconst = { +static const u8 enc_input067[] = { 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, @@ -3586,7 +3585,7 @@ static const u8 enc_input067[] __initconst = { 0x08, 0x2c, 0xf4, 0x3a, 0x09, 0x6d, 0x12, 0xc7, 0x84, 0x53, 0xfe, 0x77, 0x47, 0xfb, 0x77, 0x38 }; -static const u8 enc_output067[] __initconst = { +static const u8 enc_output067[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, @@ -3606,14 +3605,14 @@ static const u8 enc_output067[] __initconst = { 0x99, 0xca, 0xd8, 0x5f, 0x45, 0xca, 0x40, 0x94, 0x2d, 0x0d, 0x4d, 0x5e, 0x95, 0x0a, 0xde, 0x22 }; -static const u8 enc_assoc067[] __initconst = { +static const u8 enc_assoc067[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff }; -static const u8 enc_nonce067[] __initconst = { +static const u8 enc_nonce067[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key067[] __initconst = { +static const u8 enc_key067[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3621,13 +3620,13 @@ static const u8 enc_key067[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input068[] __initconst = { +static const u8 enc_input068[] = { 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, 0x80, 0xe0, 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41 }; -static const u8 enc_output068[] __initconst = { +static const u8 enc_output068[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, @@ -3635,14 +3634,14 @@ static const u8 enc_output068[] __initconst = { 0x8b, 0xbe, 0x14, 0x52, 0x72, 0xe7, 0xc2, 0xd9, 0xa1, 0x89, 0x1a, 0x3a, 0xb0, 0x98, 0x3d, 0x9d }; -static const u8 enc_assoc068[] __initconst = { +static const u8 enc_assoc068[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce068[] __initconst = { +static const u8 enc_nonce068[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key068[] __initconst = { +static const u8 enc_key068[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3650,7 +3649,7 @@ static const u8 enc_key068[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input069[] __initconst = { +static const u8 enc_input069[] = { 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, @@ -3660,7 +3659,7 @@ static const u8 enc_input069[] __initconst = { 0x08, 0x69, 0xff, 0xd2, 0x13, 0xa1, 0xd9, 0x1a, 0x53, 0xb7, 0xb2, 0x27, 0x01, 0x78, 0x02, 0x42 }; -static const u8 enc_output069[] __initconst = { +static const u8 enc_output069[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, @@ -3672,14 +3671,14 @@ static const u8 enc_output069[] __initconst = { 0x3b, 0x41, 0x86, 0x19, 0x13, 0xa8, 0xf6, 0xde, 0x7f, 0x61, 0xe2, 0x25, 0x63, 0x1b, 0xc3, 0x82 }; -static const u8 enc_assoc069[] __initconst = { +static const u8 enc_assoc069[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce069[] __initconst = { +static const u8 enc_nonce069[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key069[] __initconst = { +static const u8 enc_key069[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3687,7 +3686,7 @@ static const u8 enc_key069[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input070[] __initconst = { +static const u8 enc_input070[] = { 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, @@ -3705,7 +3704,7 @@ static const u8 enc_input070[] __initconst = { 0x77, 0xd3, 0x0b, 0xc5, 0x89, 0x6d, 0x12, 0xc7, 0xfb, 0xac, 0x01, 0x88, 0xc7, 0xfb, 0x77, 0x38 }; -static const u8 enc_output070[] __initconst = { +static const u8 enc_output070[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, @@ -3725,14 +3724,14 @@ static const u8 enc_output070[] __initconst = { 0x84, 0x28, 0xbc, 0xf0, 0x23, 0xec, 0x6b, 0xf3, 0x1f, 0xd9, 0xef, 0xb2, 0x03, 0xff, 0x08, 0x71 }; -static const u8 enc_assoc070[] __initconst = { +static const u8 enc_assoc070[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce070[] __initconst = { +static const u8 enc_nonce070[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key070[] __initconst = { +static const u8 enc_key070[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3740,13 +3739,13 @@ static const u8 enc_key070[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input071[] __initconst = { +static const u8 enc_input071[] = { 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, 0x7f, 0x1f, 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe }; -static const u8 enc_output071[] __initconst = { +static const u8 enc_output071[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, @@ -3754,14 +3753,14 @@ static const u8 enc_output071[] __initconst = { 0x13, 0x9f, 0xdf, 0x64, 0x74, 0xea, 0x24, 0xf5, 0x49, 0xb0, 0x75, 0x82, 0x5f, 0x2c, 0x76, 0x20 }; -static const u8 enc_assoc071[] __initconst = { +static const u8 enc_assoc071[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce071[] __initconst = { +static const u8 enc_nonce071[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key071[] __initconst = { +static const u8 enc_key071[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3769,7 +3768,7 @@ static const u8 enc_key071[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input072[] __initconst = { +static const u8 enc_input072[] = { 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, @@ -3779,7 +3778,7 @@ static const u8 enc_input072[] __initconst = { 0xf7, 0x96, 0x00, 0x2d, 0xec, 0x5e, 0x26, 0xe5, 0xac, 0x48, 0x4d, 0xd8, 0xfe, 0x87, 0xfd, 0xbd }; -static const u8 enc_output072[] __initconst = { +static const u8 enc_output072[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, @@ -3791,14 +3790,14 @@ static const u8 enc_output072[] __initconst = { 0xbb, 0xad, 0x8d, 0x86, 0x3b, 0x83, 0x5a, 0x8e, 0x86, 0x64, 0xfd, 0x1d, 0x45, 0x66, 0xb6, 0xb4 }; -static const u8 enc_assoc072[] __initconst = { +static const u8 enc_assoc072[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce072[] __initconst = { +static const u8 enc_nonce072[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key072[] __initconst = { +static const u8 enc_key072[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3806,7 +3805,7 @@ static const u8 enc_key072[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input073[] __initconst = { +static const u8 enc_input073[] = { 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, @@ -3824,7 +3823,7 @@ static const u8 enc_input073[] __initconst = { 0x88, 0x2c, 0xf4, 0x3a, 0x76, 0x92, 0xed, 0x38, 0x04, 0x53, 0xfe, 0x77, 0x38, 0x04, 0x88, 0xc7 }; -static const u8 enc_output073[] __initconst = { +static const u8 enc_output073[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, @@ -3844,14 +3843,14 @@ static const u8 enc_output073[] __initconst = { 0x42, 0xf2, 0x35, 0x42, 0x97, 0x84, 0x9a, 0x51, 0x1d, 0x53, 0xe5, 0x57, 0x17, 0x72, 0xf7, 0x1f }; -static const u8 enc_assoc073[] __initconst = { +static const u8 enc_assoc073[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce073[] __initconst = { +static const u8 enc_nonce073[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key073[] __initconst = { +static const u8 enc_key073[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3859,7 +3858,7 @@ static const u8 enc_key073[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input074[] __initconst = { +static const u8 enc_input074[] = { 0xd4, 0x50, 0x0b, 0xf0, 0x09, 0x49, 0x35, 0x51, 0xc3, 0x80, 0xad, 0xf5, 0x2c, 0x57, 0x3a, 0x69, 0xdf, 0x7e, 0x8b, 0x76, 0x24, 0x63, 0x33, 0x0f, @@ -3877,7 +3876,7 @@ static const u8 enc_input074[] __initconst = { 0x93, 0x76, 0x71, 0xa0, 0x62, 0x9b, 0xd9, 0x5c, 0x99, 0x15, 0xc7, 0x85, 0x55, 0x77, 0x1e, 0x7a }; -static const u8 enc_output074[] __initconst = { +static const u8 enc_output074[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3897,7 +3896,7 @@ static const u8 enc_output074[] __initconst = { 0x0b, 0x30, 0x0d, 0x8d, 0xa5, 0x6c, 0x21, 0x85, 0x75, 0x52, 0x79, 0x55, 0x3c, 0x4c, 0x82, 0xca }; -static const u8 enc_assoc074[] __initconst = { +static const u8 enc_assoc074[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3907,11 +3906,11 @@ static const u8 enc_assoc074[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce074[] __initconst = { +static const u8 enc_nonce074[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x00, 0x02, 0x50, 0x6e }; -static const u8 enc_key074[] __initconst = { +static const u8 enc_key074[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -3919,7 +3918,7 @@ static const u8 enc_key074[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input075[] __initconst = { +static const u8 enc_input075[] = { 0x7d, 0xe8, 0x7f, 0x67, 0x29, 0x94, 0x52, 0x75, 0xd0, 0x65, 0x5d, 0xa4, 0xc7, 0xfd, 0xe4, 0x56, 0x9e, 0x16, 0xf1, 0x11, 0xb5, 0xeb, 0x26, 0xc2, @@ -3937,7 +3936,7 @@ static const u8 enc_input075[] __initconst = { 0x3a, 0xc1, 0xee, 0x54, 0xc2, 0x9e, 0xe4, 0xc1, 0x70, 0xde, 0x40, 0x8f, 0x66, 0x69, 0x21, 0x94 }; -static const u8 enc_output075[] __initconst = { +static const u8 enc_output075[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3957,7 +3956,7 @@ static const u8 enc_output075[] __initconst = { 0xc5, 0x78, 0xe2, 0xaa, 0x44, 0xd3, 0x09, 0xb7, 0xb6, 0xa5, 0x19, 0x3b, 0xdc, 0x61, 0x18, 0xf5 }; -static const u8 enc_assoc075[] __initconst = { +static const u8 enc_assoc075[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3967,11 +3966,11 @@ static const u8 enc_assoc075[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce075[] __initconst = { +static const u8 enc_nonce075[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x00, 0x03, 0x18, 0xa5 }; -static const u8 enc_key075[] __initconst = { +static const u8 enc_key075[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -3979,7 +3978,7 @@ static const u8 enc_key075[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input076[] __initconst = { +static const u8 enc_input076[] = { 0x1b, 0x99, 0x6f, 0x9a, 0x3c, 0xcc, 0x67, 0x85, 0xde, 0x22, 0xff, 0x5b, 0x8a, 0xdd, 0x95, 0x02, 0xce, 0x03, 0xa0, 0xfa, 0xf5, 0x99, 0x2a, 0x09, @@ -3997,7 +3996,7 @@ static const u8 enc_input076[] __initconst = { 0xc3, 0x53, 0xd0, 0xcf, 0x93, 0x8d, 0xcc, 0xb9, 0xef, 0xad, 0x8f, 0xed, 0xbe, 0x46, 0xda, 0xa5 }; -static const u8 enc_output076[] __initconst = { +static const u8 enc_output076[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4017,7 +4016,7 @@ static const u8 enc_output076[] __initconst = { 0x4b, 0x0b, 0xda, 0x8a, 0xd0, 0x43, 0x83, 0x0d, 0x83, 0x19, 0xab, 0x82, 0xc5, 0x0c, 0x76, 0x63 }; -static const u8 enc_assoc076[] __initconst = { +static const u8 enc_assoc076[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4027,10 +4026,10 @@ static const u8 enc_assoc076[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce076[] __initconst = { +static const u8 enc_nonce076[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xb4, 0xf0 }; -static const u8 enc_key076[] __initconst = { +static const u8 enc_key076[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4038,7 +4037,7 @@ static const u8 enc_key076[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input077[] __initconst = { +static const u8 enc_input077[] = { 0x86, 0xcb, 0xac, 0xae, 0x4d, 0x3f, 0x74, 0xae, 0x01, 0x21, 0x3e, 0x05, 0x51, 0xcc, 0x15, 0x16, 0x0e, 0xa1, 0xbe, 0x84, 0x08, 0xe3, 0xd5, 0xd7, @@ -4056,7 +4055,7 @@ static const u8 enc_input077[] __initconst = { 0x0e, 0x76, 0x2b, 0x43, 0x0c, 0x4d, 0x51, 0x7c, 0x97, 0x10, 0x70, 0x68, 0xf4, 0x98, 0xef, 0x7f }; -static const u8 enc_output077[] __initconst = { +static const u8 enc_output077[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4076,7 +4075,7 @@ static const u8 enc_output077[] __initconst = { 0x4b, 0xc9, 0x8f, 0x72, 0xc4, 0x94, 0xc2, 0xa4, 0x3c, 0x2b, 0x15, 0xa1, 0x04, 0x3f, 0x1c, 0xfa }; -static const u8 enc_assoc077[] __initconst = { +static const u8 enc_assoc077[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4086,10 +4085,10 @@ static const u8 enc_assoc077[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce077[] __initconst = { +static const u8 enc_nonce077[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0xfb, 0x66 }; -static const u8 enc_key077[] __initconst = { +static const u8 enc_key077[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4097,7 +4096,7 @@ static const u8 enc_key077[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input078[] __initconst = { +static const u8 enc_input078[] = { 0xfa, 0xb1, 0xcd, 0xdf, 0x4f, 0xe1, 0x98, 0xef, 0x63, 0xad, 0xd8, 0x81, 0xd6, 0xea, 0xd6, 0xc5, 0x76, 0x37, 0xbb, 0xe9, 0x20, 0x18, 0xca, 0x7c, @@ -4115,7 +4114,7 @@ static const u8 enc_input078[] __initconst = { 0xc3, 0x24, 0x60, 0x41, 0x43, 0x21, 0x43, 0xe9, 0xab, 0x3a, 0x6d, 0x2c, 0xcc, 0x2f, 0x4d, 0x62 }; -static const u8 enc_output078[] __initconst = { +static const u8 enc_output078[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4135,7 +4134,7 @@ static const u8 enc_output078[] __initconst = { 0xf7, 0xe9, 0xe1, 0x51, 0xb0, 0x25, 0x33, 0xc7, 0x46, 0x58, 0xbf, 0xc7, 0x73, 0x7c, 0x68, 0x0d }; -static const u8 enc_assoc078[] __initconst = { +static const u8 enc_assoc078[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4145,10 +4144,10 @@ static const u8 enc_assoc078[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce078[] __initconst = { +static const u8 enc_nonce078[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0xbb, 0x90 }; -static const u8 enc_key078[] __initconst = { +static const u8 enc_key078[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4156,7 +4155,7 @@ static const u8 enc_key078[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input079[] __initconst = { +static const u8 enc_input079[] = { 0x22, 0x72, 0x02, 0xbe, 0x7f, 0x35, 0x15, 0xe9, 0xd1, 0xc0, 0x2e, 0xea, 0x2f, 0x19, 0x50, 0xb6, 0x48, 0x1b, 0x04, 0x8a, 0x4c, 0x91, 0x50, 0x6c, @@ -4174,7 +4173,7 @@ static const u8 enc_input079[] __initconst = { 0xfb, 0xe1, 0xee, 0x45, 0xb1, 0xb2, 0x1f, 0x71, 0x62, 0xe2, 0xfc, 0xaa, 0x74, 0x2a, 0xbe, 0xfd }; -static const u8 enc_output079[] __initconst = { +static const u8 enc_output079[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4194,7 +4193,7 @@ static const u8 enc_output079[] __initconst = { 0x79, 0x5b, 0xcf, 0xf6, 0x47, 0xc5, 0x53, 0xc2, 0xe4, 0xeb, 0x6e, 0x0e, 0xaf, 0xd9, 0xe0, 0x4e }; -static const u8 enc_assoc079[] __initconst = { +static const u8 enc_assoc079[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4204,10 +4203,10 @@ static const u8 enc_assoc079[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce079[] __initconst = { +static const u8 enc_nonce079[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x48, 0x4a }; -static const u8 enc_key079[] __initconst = { +static const u8 enc_key079[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4215,7 +4214,7 @@ static const u8 enc_key079[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input080[] __initconst = { +static const u8 enc_input080[] = { 0xfa, 0xe5, 0x83, 0x45, 0xc1, 0x6c, 0xb0, 0xf5, 0xcc, 0x53, 0x7f, 0x2b, 0x1b, 0x34, 0x69, 0xc9, 0x69, 0x46, 0x3b, 0x3e, 0xa7, 0x1b, 0xcf, 0x6b, @@ -4233,7 +4232,7 @@ static const u8 enc_input080[] __initconst = { 0x18, 0x01, 0xce, 0x33, 0xc4, 0xe4, 0xa7, 0x7d, 0x83, 0x1d, 0x3c, 0xe3, 0x4e, 0x84, 0x10, 0xe1 }; -static const u8 enc_output080[] __initconst = { +static const u8 enc_output080[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4253,7 +4252,7 @@ static const u8 enc_output080[] __initconst = { 0x19, 0x46, 0xd6, 0x53, 0x96, 0x0f, 0x94, 0x7a, 0x74, 0xd3, 0xe8, 0x09, 0x3c, 0xf4, 0x85, 0x02 }; -static const u8 enc_assoc080[] __initconst = { +static const u8 enc_assoc080[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4263,10 +4262,10 @@ static const u8 enc_assoc080[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce080[] __initconst = { +static const u8 enc_nonce080[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x2f, 0x40 }; -static const u8 enc_key080[] __initconst = { +static const u8 enc_key080[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4274,7 +4273,7 @@ static const u8 enc_key080[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input081[] __initconst = { +static const u8 enc_input081[] = { 0xeb, 0xb2, 0x16, 0xdd, 0xd7, 0xca, 0x70, 0x92, 0x15, 0xf5, 0x03, 0xdf, 0x9c, 0xe6, 0x3c, 0x5c, 0xd2, 0x19, 0x4e, 0x7d, 0x90, 0x99, 0xe8, 0xa9, @@ -4292,7 +4291,7 @@ static const u8 enc_input081[] __initconst = { 0x83, 0xfd, 0xca, 0x39, 0xd2, 0xe1, 0x4f, 0x23, 0xd0, 0x0a, 0x58, 0x26, 0x64, 0xf4, 0xec, 0xb1 }; -static const u8 enc_output081[] __initconst = { +static const u8 enc_output081[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4312,7 +4311,7 @@ static const u8 enc_output081[] __initconst = { 0x36, 0xc3, 0x00, 0x29, 0x85, 0xdd, 0x21, 0xba, 0xf8, 0x95, 0xd6, 0x33, 0x57, 0x3f, 0x12, 0xc0 }; -static const u8 enc_assoc081[] __initconst = { +static const u8 enc_assoc081[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4322,10 +4321,10 @@ static const u8 enc_assoc081[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce081[] __initconst = { +static const u8 enc_nonce081[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x93, 0x35 }; -static const u8 enc_key081[] __initconst = { +static const u8 enc_key081[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4333,7 +4332,7 @@ static const u8 enc_key081[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input082[] __initconst = { +static const u8 enc_input082[] = { 0x40, 0x8a, 0xe6, 0xef, 0x1c, 0x7e, 0xf0, 0xfb, 0x2c, 0x2d, 0x61, 0x08, 0x16, 0xfc, 0x78, 0x49, 0xef, 0xa5, 0x8f, 0x78, 0x27, 0x3f, 0x5f, 0x16, @@ -4351,7 +4350,7 @@ static const u8 enc_input082[] __initconst = { 0x2b, 0x68, 0x9f, 0x93, 0x55, 0xd9, 0xc1, 0x83, 0x80, 0x1f, 0x6a, 0xcc, 0x31, 0x3f, 0x89, 0x07 }; -static const u8 enc_output082[] __initconst = { +static const u8 enc_output082[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4371,7 +4370,7 @@ static const u8 enc_output082[] __initconst = { 0x65, 0x14, 0x51, 0x8e, 0x0a, 0x26, 0x41, 0x42, 0xe0, 0xb7, 0x35, 0x1f, 0x96, 0x7f, 0xc2, 0xae }; -static const u8 enc_assoc082[] __initconst = { +static const u8 enc_assoc082[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4381,10 +4380,10 @@ static const u8 enc_assoc082[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce082[] __initconst = { +static const u8 enc_nonce082[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0xf7, 0xd5 }; -static const u8 enc_key082[] __initconst = { +static const u8 enc_key082[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4392,7 +4391,7 @@ static const u8 enc_key082[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input083[] __initconst = { +static const u8 enc_input083[] = { 0x0a, 0x0a, 0x24, 0x49, 0x9b, 0xca, 0xde, 0x58, 0xcf, 0x15, 0x76, 0xc3, 0x12, 0xac, 0xa9, 0x84, 0x71, 0x8c, 0xb4, 0xcc, 0x7e, 0x01, 0x53, 0xf5, @@ -4410,7 +4409,7 @@ static const u8 enc_input083[] __initconst = { 0xb3, 0x06, 0xa2, 0x1b, 0x42, 0xd4, 0xc3, 0xba, 0x6e, 0x6f, 0x0c, 0xbc, 0xc8, 0x1e, 0x87, 0x7a }; -static const u8 enc_output083[] __initconst = { +static const u8 enc_output083[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4430,7 +4429,7 @@ static const u8 enc_output083[] __initconst = { 0x4c, 0x19, 0x4d, 0xa6, 0xa9, 0x9f, 0xd6, 0x5b, 0x40, 0xe9, 0xca, 0xd7, 0x98, 0xf4, 0x4b, 0x19 }; -static const u8 enc_assoc083[] __initconst = { +static const u8 enc_assoc083[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4440,10 +4439,10 @@ static const u8 enc_assoc083[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce083[] __initconst = { +static const u8 enc_nonce083[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0xfc, 0xe4 }; -static const u8 enc_key083[] __initconst = { +static const u8 enc_key083[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4451,7 +4450,7 @@ static const u8 enc_key083[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input084[] __initconst = { +static const u8 enc_input084[] = { 0x4a, 0x0a, 0xaf, 0xf8, 0x49, 0x47, 0x29, 0x18, 0x86, 0x91, 0x70, 0x13, 0x40, 0xf3, 0xce, 0x2b, 0x8a, 0x78, 0xee, 0xd3, 0xa0, 0xf0, 0x65, 0x99, @@ -4469,7 +4468,7 @@ static const u8 enc_input084[] __initconst = { 0x5d, 0x7e, 0x51, 0x3b, 0xe5, 0xb8, 0xea, 0x97, 0x13, 0x10, 0xd5, 0xbf, 0x16, 0xba, 0x7a, 0xee }; -static const u8 enc_output084[] __initconst = { +static const u8 enc_output084[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4489,7 +4488,7 @@ static const u8 enc_output084[] __initconst = { 0xc8, 0xae, 0x77, 0x88, 0xcd, 0x28, 0x74, 0xab, 0xc1, 0x38, 0x54, 0x1e, 0x11, 0xfd, 0x05, 0x87 }; -static const u8 enc_assoc084[] __initconst = { +static const u8 enc_assoc084[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4499,10 +4498,10 @@ static const u8 enc_assoc084[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce084[] __initconst = { +static const u8 enc_nonce084[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0x84, 0x86, 0xa8 }; -static const u8 enc_key084[] __initconst = { +static const u8 enc_key084[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4510,7 +4509,7 @@ static const u8 enc_key084[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input085[] __initconst = { +static const u8 enc_input085[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x78, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -4528,7 +4527,7 @@ static const u8 enc_input085[] __initconst = { 0xf7, 0x4d, 0x5b, 0xf8, 0x67, 0x1c, 0x5a, 0x8a, 0x50, 0x92, 0xf6, 0x1d, 0x54, 0xc9, 0xaa, 0x5b }; -static const u8 enc_output085[] __initconst = { +static const u8 enc_output085[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4548,7 +4547,7 @@ static const u8 enc_output085[] __initconst = { 0x93, 0x3a, 0x51, 0x63, 0xc7, 0xf6, 0x23, 0x68, 0x32, 0x7b, 0x3f, 0xbc, 0x10, 0x36, 0xc9, 0x43 }; -static const u8 enc_assoc085[] __initconst = { +static const u8 enc_assoc085[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4558,10 +4557,10 @@ static const u8 enc_assoc085[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce085[] __initconst = { +static const u8 enc_nonce085[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key085[] __initconst = { +static const u8 enc_key085[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4569,7 +4568,7 @@ static const u8 enc_key085[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input086[] __initconst = { +static const u8 enc_input086[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4579,7 +4578,7 @@ static const u8 enc_input086[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output086[] __initconst = { +static const u8 enc_output086[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4591,17 +4590,17 @@ static const u8 enc_output086[] __initconst = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }; -static const u8 enc_assoc086[] __initconst = { +static const u8 enc_assoc086[] = { 0x85, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa6, 0x90, 0x2f, 0xcb, 0xc8, 0x83, 0xbb, 0xc1, 0x80, 0xb2, 0x56, 0xae, 0x34, 0xad, 0x7f, 0x00 }; -static const u8 enc_nonce086[] __initconst = { +static const u8 enc_nonce086[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key086[] __initconst = { +static const u8 enc_key086[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4609,7 +4608,7 @@ static const u8 enc_key086[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input087[] __initconst = { +static const u8 enc_input087[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4619,7 +4618,7 @@ static const u8 enc_input087[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output087[] __initconst = { +static const u8 enc_output087[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4631,17 +4630,17 @@ static const u8 enc_output087[] __initconst = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_assoc087[] __initconst = { +static const u8 enc_assoc087[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x24, 0x7e, 0x50, 0x64, 0x2a, 0x1c, 0x0a, 0x2f, 0x8f, 0x77, 0x21, 0x96, 0x09, 0xdb, 0xa9, 0x58 }; -static const u8 enc_nonce087[] __initconst = { +static const u8 enc_nonce087[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key087[] __initconst = { +static const u8 enc_key087[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4649,7 +4648,7 @@ static const u8 enc_key087[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input088[] __initconst = { +static const u8 enc_input088[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4659,7 +4658,7 @@ static const u8 enc_input088[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output088[] __initconst = { +static const u8 enc_output088[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4671,17 +4670,17 @@ static const u8 enc_output088[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_assoc088[] __initconst = { +static const u8 enc_assoc088[] = { 0x7c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xd9, 0xe7, 0x2c, 0x06, 0x4a, 0xc8, 0x96, 0x1f, 0x3f, 0xa5, 0x85, 0xe0, 0xe2, 0xab, 0xd6, 0x00 }; -static const u8 enc_nonce088[] __initconst = { +static const u8 enc_nonce088[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key088[] __initconst = { +static const u8 enc_key088[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4689,7 +4688,7 @@ static const u8 enc_key088[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input089[] __initconst = { +static const u8 enc_input089[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4699,7 +4698,7 @@ static const u8 enc_input089[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output089[] __initconst = { +static const u8 enc_output089[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4711,17 +4710,17 @@ static const u8 enc_output089[] __initconst = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; -static const u8 enc_assoc089[] __initconst = { +static const u8 enc_assoc089[] = { 0x65, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x95, 0xaf, 0x0f, 0x4d, 0x0b, 0x68, 0x6e, 0xae, 0xcc, 0xca, 0x43, 0x07, 0xd5, 0x96, 0xf5, 0x02 }; -static const u8 enc_nonce089[] __initconst = { +static const u8 enc_nonce089[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key089[] __initconst = { +static const u8 enc_key089[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4729,7 +4728,7 @@ static const u8 enc_key089[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input090[] __initconst = { +static const u8 enc_input090[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4739,7 +4738,7 @@ static const u8 enc_input090[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output090[] __initconst = { +static const u8 enc_output090[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4751,17 +4750,17 @@ static const u8 enc_output090[] __initconst = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; -static const u8 enc_assoc090[] __initconst = { +static const u8 enc_assoc090[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x85, 0x40, 0xb4, 0x64, 0x35, 0x77, 0x07, 0xbe, 0x3a, 0x39, 0xd5, 0x5c, 0x34, 0xf8, 0xbc, 0xb3 }; -static const u8 enc_nonce090[] __initconst = { +static const u8 enc_nonce090[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key090[] __initconst = { +static const u8 enc_key090[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4769,7 +4768,7 @@ static const u8 enc_key090[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input091[] __initconst = { +static const u8 enc_input091[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4779,7 +4778,7 @@ static const u8 enc_input091[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output091[] __initconst = { +static const u8 enc_output091[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4791,17 +4790,17 @@ static const u8 enc_output091[] __initconst = { 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00 }; -static const u8 enc_assoc091[] __initconst = { +static const u8 enc_assoc091[] = { 0x4f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x66, 0x23, 0xd9, 0x90, 0xb8, 0x98, 0xd8, 0x30, 0xd2, 0x12, 0xaf, 0x23, 0x83, 0x33, 0x07, 0x01 }; -static const u8 enc_nonce091[] __initconst = { +static const u8 enc_nonce091[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key091[] __initconst = { +static const u8 enc_key091[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4809,7 +4808,7 @@ static const u8 enc_key091[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input092[] __initconst = { +static const u8 enc_input092[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4819,7 +4818,7 @@ static const u8 enc_input092[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output092[] __initconst = { +static const u8 enc_output092[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4831,17 +4830,17 @@ static const u8 enc_output092[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_assoc092[] __initconst = { +static const u8 enc_assoc092[] = { 0x83, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x5f, 0x16, 0xd0, 0x9f, 0x17, 0x78, 0x72, 0x11, 0xb7, 0xd4, 0x84, 0xe0, 0x24, 0xf8, 0x97, 0x01 }; -static const u8 enc_nonce092[] __initconst = { +static const u8 enc_nonce092[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key092[] __initconst = { +static const u8 enc_key092[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4849,7 +4848,7 @@ static const u8 enc_key092[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input093[] __initconst = { +static const u8 enc_input093[] = { 0x00, 0x52, 0x35, 0xd2, 0xa9, 0x19, 0xf2, 0x8d, 0x3d, 0xb7, 0x66, 0x4a, 0x34, 0xae, 0x6b, 0x44, 0x4d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -4861,7 +4860,7 @@ static const u8 enc_input093[] __initconst = { 0x83, 0xdc, 0xe9, 0xf3, 0x07, 0x3e, 0xfa, 0xdb, 0x7d, 0x23, 0xb8, 0x7a, 0xce, 0x35, 0x16, 0x8c }; -static const u8 enc_output093[] __initconst = { +static const u8 enc_output093[] = { 0x00, 0x39, 0xe2, 0xfd, 0x2f, 0xd3, 0x12, 0x14, 0x9e, 0x98, 0x98, 0x80, 0x88, 0x48, 0x13, 0xe7, 0xca, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4875,13 +4874,13 @@ static const u8 enc_output093[] __initconst = { 0xa5, 0x19, 0xac, 0x1a, 0x35, 0xb4, 0xa5, 0x77, 0x87, 0x51, 0x0a, 0xf7, 0x8d, 0x8d, 0x20, 0x0a }; -static const u8 enc_assoc093[] __initconst = { +static const u8 enc_assoc093[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce093[] __initconst = { +static const u8 enc_nonce093[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key093[] __initconst = { +static const u8 enc_key093[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4889,7 +4888,7 @@ static const u8 enc_key093[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input094[] __initconst = { +static const u8 enc_input094[] = { 0xd3, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xe5, 0xda, 0x78, 0x76, 0x6f, 0xa1, 0x92, 0x90, @@ -4903,7 +4902,7 @@ static const u8 enc_input094[] __initconst = { 0x01, 0x49, 0xef, 0x50, 0x4b, 0x71, 0xb1, 0x20, 0xca, 0x4f, 0xf3, 0x95, 0x19, 0xc2, 0xc2, 0x10 }; -static const u8 enc_output094[] __initconst = { +static const u8 enc_output094[] = { 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x62, 0x18, 0xb2, 0x7f, 0x83, 0xb8, 0xb4, 0x66, @@ -4919,13 +4918,13 @@ static const u8 enc_output094[] __initconst = { 0x30, 0x2f, 0xe8, 0x2a, 0xb0, 0xa0, 0x9a, 0xf6, 0x44, 0x00, 0xd0, 0x15, 0xae, 0x83, 0xd9, 0xcc }; -static const u8 enc_assoc094[] __initconst = { +static const u8 enc_assoc094[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce094[] __initconst = { +static const u8 enc_nonce094[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key094[] __initconst = { +static const u8 enc_key094[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4933,7 +4932,7 @@ static const u8 enc_key094[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input095[] __initconst = { +static const u8 enc_input095[] = { 0xe9, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x6d, 0xf1, 0x39, 0x4e, 0xdc, 0x53, 0x9b, 0x5b, @@ -4947,7 +4946,7 @@ static const u8 enc_input095[] __initconst = { 0x99, 0x52, 0xae, 0x08, 0x18, 0xc3, 0x89, 0x79, 0xc0, 0x74, 0x13, 0x71, 0x1a, 0x9a, 0xf7, 0x13 }; -static const u8 enc_output095[] __initconst = { +static const u8 enc_output095[] = { 0xe9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xea, 0x33, 0xf3, 0x47, 0x30, 0x4a, 0xbd, 0xad, @@ -4963,13 +4962,13 @@ static const u8 enc_output095[] __initconst = { 0x98, 0xa7, 0xe8, 0x36, 0xe0, 0xee, 0x4d, 0x02, 0x35, 0x00, 0xd0, 0x55, 0x7e, 0xc2, 0xcb, 0xe0 }; -static const u8 enc_assoc095[] __initconst = { +static const u8 enc_assoc095[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce095[] __initconst = { +static const u8 enc_nonce095[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key095[] __initconst = { +static const u8 enc_key095[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4977,7 +4976,7 @@ static const u8 enc_key095[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input096[] __initconst = { +static const u8 enc_input096[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x64, 0xf9, 0x0f, 0x5b, 0x26, 0x92, 0xb8, 0x60, @@ -4987,7 +4986,7 @@ static const u8 enc_input096[] __initconst = { 0x03, 0x35, 0x61, 0xe7, 0xca, 0xca, 0x6d, 0x94, 0x1d, 0xc3, 0xcd, 0x69, 0x14, 0xad, 0x69, 0x04 }; -static const u8 enc_output096[] __initconst = { +static const u8 enc_output096[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe3, 0x3b, 0xc5, 0x52, 0xca, 0x8b, 0x9e, 0x96, @@ -4999,13 +4998,13 @@ static const u8 enc_output096[] __initconst = { 0x6a, 0xb8, 0xdc, 0xe2, 0xc5, 0x9d, 0xa4, 0x73, 0x71, 0x30, 0xb0, 0x25, 0x2f, 0x68, 0xa8, 0xd8 }; -static const u8 enc_assoc096[] __initconst = { +static const u8 enc_assoc096[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce096[] __initconst = { +static const u8 enc_nonce096[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key096[] __initconst = { +static const u8 enc_key096[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5013,7 +5012,7 @@ static const u8 enc_key096[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input097[] __initconst = { +static const u8 enc_input097[] = { 0x68, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xb0, 0x8f, 0x25, 0x67, 0x5b, 0x9b, 0xcb, 0xf6, @@ -5027,7 +5026,7 @@ static const u8 enc_input097[] __initconst = { 0x65, 0x0e, 0xc6, 0x2d, 0x75, 0x70, 0x72, 0xce, 0xe6, 0xff, 0x23, 0x31, 0x86, 0xdd, 0x1c, 0x8f }; -static const u8 enc_output097[] __initconst = { +static const u8 enc_output097[] = { 0x68, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x37, 0x4d, 0xef, 0x6e, 0xb7, 0x82, 0xed, 0x00, @@ -5043,13 +5042,13 @@ static const u8 enc_output097[] __initconst = { 0x04, 0x4d, 0xea, 0x60, 0x88, 0x80, 0x41, 0x2b, 0xfd, 0xff, 0xcf, 0x35, 0x57, 0x9e, 0x9b, 0x26 }; -static const u8 enc_assoc097[] __initconst = { +static const u8 enc_assoc097[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce097[] __initconst = { +static const u8 enc_nonce097[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key097[] __initconst = { +static const u8 enc_key097[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5057,7 +5056,7 @@ static const u8 enc_key097[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input098[] __initconst = { +static const u8 enc_input098[] = { 0x6d, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xa1, 0x61, 0xb5, 0xab, 0x04, 0x09, 0x00, 0x62, @@ -5071,7 +5070,7 @@ static const u8 enc_input098[] __initconst = { 0x8e, 0xdc, 0x36, 0x6c, 0xd6, 0x97, 0x65, 0x6f, 0xca, 0x81, 0xfb, 0x13, 0x3c, 0xed, 0x79, 0xa1 }; -static const u8 enc_output098[] __initconst = { +static const u8 enc_output098[] = { 0x6d, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x26, 0xa3, 0x7f, 0xa2, 0xe8, 0x10, 0x26, 0x94, @@ -5087,13 +5086,13 @@ static const u8 enc_output098[] __initconst = { 0x1e, 0x6b, 0xea, 0x63, 0x14, 0x54, 0x2e, 0x2e, 0xf9, 0xff, 0xcf, 0x45, 0x0b, 0x2e, 0x98, 0x2b }; -static const u8 enc_assoc098[] __initconst = { +static const u8 enc_assoc098[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce098[] __initconst = { +static const u8 enc_nonce098[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key098[] __initconst = { +static const u8 enc_key098[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5101,7 +5100,7 @@ static const u8 enc_key098[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input099[] __initconst = { +static const u8 enc_input099[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xfc, 0x01, 0xb8, 0x91, 0xe5, 0xf0, 0xf9, 0x12, @@ -5111,7 +5110,7 @@ static const u8 enc_input099[] __initconst = { 0x60, 0xcd, 0x9e, 0xa1, 0x0c, 0x29, 0xa3, 0x66, 0x54, 0xe7, 0xa2, 0x8e, 0x76, 0x1b, 0xec, 0xd8 }; -static const u8 enc_output099[] __initconst = { +static const u8 enc_output099[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7b, 0xc3, 0x72, 0x98, 0x09, 0xe9, 0xdf, 0xe4, @@ -5123,13 +5122,13 @@ static const u8 enc_output099[] __initconst = { 0xed, 0x20, 0x17, 0xc8, 0xdb, 0xa4, 0x77, 0x56, 0x29, 0x04, 0x9d, 0x78, 0x6e, 0x3b, 0xce, 0xb1 }; -static const u8 enc_assoc099[] __initconst = { +static const u8 enc_assoc099[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce099[] __initconst = { +static const u8 enc_nonce099[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key099[] __initconst = { +static const u8 enc_key099[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5137,7 +5136,7 @@ static const u8 enc_key099[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input100[] __initconst = { +static const u8 enc_input100[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x6b, 0x6d, 0xc9, 0xd2, 0x1a, 0x81, 0x9e, 0x70, @@ -5147,7 +5146,7 @@ static const u8 enc_input100[] __initconst = { 0x10, 0xb9, 0x2f, 0x5f, 0xfe, 0xf9, 0x8b, 0x84, 0x7c, 0xf1, 0x7a, 0x9c, 0x98, 0xd8, 0x83, 0xe5 }; -static const u8 enc_output100[] __initconst = { +static const u8 enc_output100[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xec, 0xaf, 0x03, 0xdb, 0xf6, 0x98, 0xb8, 0x86, @@ -5159,13 +5158,13 @@ static const u8 enc_output100[] __initconst = { 0x07, 0x3f, 0x17, 0xcb, 0x67, 0x78, 0x64, 0x59, 0x25, 0x04, 0x9d, 0x88, 0x22, 0xcb, 0xca, 0xb6 }; -static const u8 enc_assoc100[] __initconst = { +static const u8 enc_assoc100[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce100[] __initconst = { +static const u8 enc_nonce100[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key100[] __initconst = { +static const u8 enc_key100[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5173,7 +5172,7 @@ static const u8 enc_key100[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input101[] __initconst = { +static const u8 enc_input101[] = { 0xff, 0xcb, 0x2b, 0x11, 0x06, 0xf8, 0x23, 0x4c, 0x5e, 0x99, 0xd4, 0xdb, 0x4c, 0x70, 0x48, 0xde, 0x32, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5185,7 +5184,7 @@ static const u8 enc_input101[] __initconst = { 0xce, 0xbe, 0xf5, 0xe9, 0x88, 0x5a, 0x80, 0xea, 0x76, 0xd9, 0x75, 0xc1, 0x44, 0xa4, 0x18, 0x88 }; -static const u8 enc_output101[] __initconst = { +static const u8 enc_output101[] = { 0xff, 0xa0, 0xfc, 0x3e, 0x80, 0x32, 0xc3, 0xd5, 0xfd, 0xb6, 0x2a, 0x11, 0xf0, 0x96, 0x30, 0x7d, 0xb5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5199,13 +5198,13 @@ static const u8 enc_output101[] __initconst = { 0x8b, 0x9b, 0xb4, 0xb4, 0x86, 0x12, 0x89, 0x65, 0x8c, 0x69, 0x6a, 0x83, 0x40, 0x15, 0x04, 0x05 }; -static const u8 enc_assoc101[] __initconst = { +static const u8 enc_assoc101[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce101[] __initconst = { +static const u8 enc_nonce101[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key101[] __initconst = { +static const u8 enc_key101[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5213,7 +5212,7 @@ static const u8 enc_key101[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input102[] __initconst = { +static const u8 enc_input102[] = { 0x6f, 0x9e, 0x70, 0xed, 0x3b, 0x8b, 0xac, 0xa0, 0x26, 0xe4, 0x6a, 0x5a, 0x09, 0x43, 0x15, 0x8d, 0x21, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5225,7 +5224,7 @@ static const u8 enc_input102[] __initconst = { 0xd4, 0x36, 0x51, 0xfd, 0x14, 0x9c, 0x26, 0x0b, 0xcb, 0xdd, 0x7b, 0x12, 0x68, 0x01, 0x31, 0x8c }; -static const u8 enc_output102[] __initconst = { +static const u8 enc_output102[] = { 0x6f, 0xf5, 0xa7, 0xc2, 0xbd, 0x41, 0x4c, 0x39, 0x85, 0xcb, 0x94, 0x90, 0xb5, 0xa5, 0x6d, 0x2e, 0xa6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5239,13 +5238,13 @@ static const u8 enc_output102[] __initconst = { 0x8b, 0x3b, 0xbd, 0x51, 0x64, 0x44, 0x59, 0x56, 0x8d, 0x81, 0xca, 0x1f, 0xa7, 0x2c, 0xe4, 0x04 }; -static const u8 enc_assoc102[] __initconst = { +static const u8 enc_assoc102[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce102[] __initconst = { +static const u8 enc_nonce102[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key102[] __initconst = { +static const u8 enc_key102[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5253,7 +5252,7 @@ static const u8 enc_key102[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input103[] __initconst = { +static const u8 enc_input103[] = { 0x41, 0x2b, 0x08, 0x0a, 0x3e, 0x19, 0xc1, 0x0d, 0x44, 0xa1, 0xaf, 0x1e, 0xab, 0xde, 0xb4, 0xce, 0x35, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5265,7 +5264,7 @@ static const u8 enc_input103[] __initconst = { 0xb3, 0xd4, 0xe9, 0x90, 0x90, 0x34, 0xc6, 0x14, 0xb1, 0x0a, 0xff, 0x55, 0x25, 0xd0, 0x9d, 0x8d }; -static const u8 enc_output103[] __initconst = { +static const u8 enc_output103[] = { 0x41, 0x40, 0xdf, 0x25, 0xb8, 0xd3, 0x21, 0x94, 0xe7, 0x8e, 0x51, 0xd4, 0x17, 0x38, 0xcc, 0x6d, 0xb2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5279,13 +5278,13 @@ static const u8 enc_output103[] __initconst = { 0x86, 0xfb, 0xab, 0x2b, 0x4a, 0x94, 0xf4, 0x7a, 0xa5, 0x6f, 0x0a, 0xea, 0x65, 0xd1, 0x10, 0x08 }; -static const u8 enc_assoc103[] __initconst = { +static const u8 enc_assoc103[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce103[] __initconst = { +static const u8 enc_nonce103[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key103[] __initconst = { +static const u8 enc_key103[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5293,7 +5292,7 @@ static const u8 enc_key103[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input104[] __initconst = { +static const u8 enc_input104[] = { 0xb2, 0x47, 0xa7, 0x47, 0x23, 0x49, 0x1a, 0xac, 0xac, 0xaa, 0xd7, 0x09, 0xc9, 0x1e, 0x93, 0x2b, 0x31, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5305,7 +5304,7 @@ static const u8 enc_input104[] __initconst = { 0x42, 0x89, 0x79, 0x44, 0xc2, 0xa2, 0x8f, 0xa1, 0x76, 0x11, 0xd7, 0xfa, 0x5c, 0x22, 0xad, 0x8f }; -static const u8 enc_output104[] __initconst = { +static const u8 enc_output104[] = { 0xb2, 0x2c, 0x70, 0x68, 0xa5, 0x83, 0xfa, 0x35, 0x0f, 0x85, 0x29, 0xc3, 0x75, 0xf8, 0xeb, 0x88, 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5319,13 +5318,13 @@ static const u8 enc_output104[] __initconst = { 0xa0, 0x19, 0xac, 0x2e, 0xd6, 0x67, 0xe1, 0x7d, 0xa1, 0x6f, 0x0a, 0xfa, 0x19, 0x61, 0x0d, 0x0d }; -static const u8 enc_assoc104[] __initconst = { +static const u8 enc_assoc104[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce104[] __initconst = { +static const u8 enc_nonce104[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key104[] __initconst = { +static const u8 enc_key104[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5333,7 +5332,7 @@ static const u8 enc_key104[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input105[] __initconst = { +static const u8 enc_input105[] = { 0x74, 0x0f, 0x9e, 0x49, 0xf6, 0x10, 0xef, 0xa5, 0x85, 0xb6, 0x59, 0xca, 0x6e, 0xd8, 0xb4, 0x99, 0x2d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5345,7 +5344,7 @@ static const u8 enc_input105[] __initconst = { 0x99, 0x7a, 0xeb, 0x0c, 0x27, 0x95, 0x62, 0x46, 0x69, 0xc3, 0x87, 0xf9, 0x11, 0x6a, 0xc1, 0x8d }; -static const u8 enc_output105[] __initconst = { +static const u8 enc_output105[] = { 0x74, 0x64, 0x49, 0x66, 0x70, 0xda, 0x0f, 0x3c, 0x26, 0x99, 0xa7, 0x00, 0xd2, 0x3e, 0xcc, 0x3a, 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5359,13 +5358,13 @@ static const u8 enc_output105[] __initconst = { 0x73, 0x6e, 0x18, 0x18, 0x16, 0x96, 0xa5, 0x88, 0x9c, 0x31, 0x59, 0xfa, 0xab, 0xab, 0x20, 0xfd }; -static const u8 enc_assoc105[] __initconst = { +static const u8 enc_assoc105[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce105[] __initconst = { +static const u8 enc_nonce105[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key105[] __initconst = { +static const u8 enc_key105[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5373,7 +5372,7 @@ static const u8 enc_key105[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input106[] __initconst = { +static const u8 enc_input106[] = { 0xad, 0xba, 0x5d, 0x10, 0x5b, 0xc8, 0xaa, 0x06, 0x2c, 0x23, 0x36, 0xcb, 0x88, 0x9d, 0xdb, 0xd5, 0x37, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5385,7 +5384,7 @@ static const u8 enc_input106[] __initconst = { 0xcf, 0x2b, 0x22, 0x5d, 0xb1, 0x60, 0x7a, 0x10, 0xe6, 0xd5, 0x40, 0x1e, 0x53, 0xb4, 0x2a, 0x8d }; -static const u8 enc_output106[] __initconst = { +static const u8 enc_output106[] = { 0xad, 0xd1, 0x8a, 0x3f, 0xdd, 0x02, 0x4a, 0x9f, 0x8f, 0x0c, 0xc8, 0x01, 0x34, 0x7b, 0xa3, 0x76, 0xb0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5399,13 +5398,13 @@ static const u8 enc_output106[] __initconst = { 0xba, 0xd5, 0x8f, 0x10, 0xa9, 0x1e, 0x6a, 0x88, 0x9a, 0xba, 0x32, 0xfd, 0x17, 0xd8, 0x33, 0x1a }; -static const u8 enc_assoc106[] __initconst = { +static const u8 enc_assoc106[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce106[] __initconst = { +static const u8 enc_nonce106[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key106[] __initconst = { +static const u8 enc_key106[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5413,7 +5412,7 @@ static const u8 enc_key106[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input107[] __initconst = { +static const u8 enc_input107[] = { 0xfe, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xc0, 0x01, 0xed, 0xc5, 0xda, 0x44, 0x2e, 0x71, @@ -5427,7 +5426,7 @@ static const u8 enc_input107[] __initconst = { 0x00, 0x26, 0x6e, 0xa1, 0xe4, 0x36, 0x44, 0xa3, 0x4d, 0x8d, 0xd1, 0xdc, 0x93, 0xf2, 0xfa, 0x13 }; -static const u8 enc_output107[] __initconst = { +static const u8 enc_output107[] = { 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x47, 0xc3, 0x27, 0xcc, 0x36, 0x5d, 0x08, 0x87, @@ -5443,13 +5442,13 @@ static const u8 enc_output107[] __initconst = { 0xd6, 0x8c, 0xe1, 0x74, 0x07, 0x9a, 0xdd, 0x02, 0x8d, 0xd0, 0x5c, 0xf8, 0x14, 0x63, 0x04, 0x88 }; -static const u8 enc_assoc107[] __initconst = { +static const u8 enc_assoc107[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce107[] __initconst = { +static const u8 enc_nonce107[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key107[] __initconst = { +static const u8 enc_key107[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5457,7 +5456,7 @@ static const u8 enc_key107[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input108[] __initconst = { +static const u8 enc_input108[] = { 0xb5, 0x13, 0xb0, 0x6a, 0xb9, 0xac, 0x14, 0x43, 0x5a, 0xcb, 0x8a, 0xa3, 0xa3, 0x7a, 0xfd, 0xb6, 0x54, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5469,7 +5468,7 @@ static const u8 enc_input108[] __initconst = { 0xb9, 0xc2, 0x7c, 0x30, 0x28, 0xaa, 0x8d, 0x69, 0xef, 0x06, 0xaf, 0xc0, 0xb5, 0x9e, 0xda, 0x8e }; -static const u8 enc_output108[] __initconst = { +static const u8 enc_output108[] = { 0xb5, 0x78, 0x67, 0x45, 0x3f, 0x66, 0xf4, 0xda, 0xf9, 0xe4, 0x74, 0x69, 0x1f, 0x9c, 0x85, 0x15, 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5483,13 +5482,13 @@ static const u8 enc_output108[] __initconst = { 0xaa, 0x48, 0xa3, 0x88, 0x7d, 0x4b, 0x05, 0x96, 0x99, 0xc2, 0xfd, 0xf9, 0xc6, 0x78, 0x7e, 0x0a }; -static const u8 enc_assoc108[] __initconst = { +static const u8 enc_assoc108[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce108[] __initconst = { +static const u8 enc_nonce108[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key108[] __initconst = { +static const u8 enc_key108[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5497,7 +5496,7 @@ static const u8 enc_key108[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input109[] __initconst = { +static const u8 enc_input109[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xd4, 0xf1, 0x09, 0xe8, 0x14, 0xce, 0xa8, 0x5a, @@ -5511,7 +5510,7 @@ static const u8 enc_input109[] __initconst = { 0x1b, 0x64, 0x89, 0xba, 0x84, 0xd8, 0xf5, 0x59, 0x82, 0x9e, 0xd9, 0xbd, 0xa2, 0x29, 0x0f, 0x16 }; -static const u8 enc_output109[] __initconst = { +static const u8 enc_output109[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x53, 0x33, 0xc3, 0xe1, 0xf8, 0xd7, 0x8e, 0xac, @@ -5527,13 +5526,13 @@ static const u8 enc_output109[] __initconst = { 0xb9, 0x36, 0xa8, 0x17, 0xf2, 0x21, 0x1a, 0xf1, 0x29, 0xe2, 0xcf, 0x16, 0x0f, 0xd4, 0x2b, 0xcb }; -static const u8 enc_assoc109[] __initconst = { +static const u8 enc_assoc109[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce109[] __initconst = { +static const u8 enc_nonce109[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key109[] __initconst = { +static const u8 enc_key109[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5541,7 +5540,7 @@ static const u8 enc_key109[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input110[] __initconst = { +static const u8 enc_input110[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xdf, 0x4c, 0x62, 0x03, 0x2d, 0x41, 0x19, 0xb5, @@ -5555,7 +5554,7 @@ static const u8 enc_input110[] __initconst = { 0xb2, 0xa0, 0xc1, 0x84, 0x4b, 0x4e, 0x35, 0xd4, 0x1e, 0x5d, 0xa2, 0x10, 0xf6, 0x2f, 0x84, 0x12 }; -static const u8 enc_output110[] __initconst = { +static const u8 enc_output110[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x58, 0x8e, 0xa8, 0x0a, 0xc1, 0x58, 0x3f, 0x43, @@ -5571,13 +5570,13 @@ static const u8 enc_output110[] __initconst = { 0x9f, 0x7a, 0xc4, 0x35, 0x1f, 0x6b, 0x91, 0xe6, 0x30, 0x97, 0xa7, 0x13, 0x11, 0x5d, 0x05, 0xbe }; -static const u8 enc_assoc110[] __initconst = { +static const u8 enc_assoc110[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce110[] __initconst = { +static const u8 enc_nonce110[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key110[] __initconst = { +static const u8 enc_key110[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5585,7 +5584,7 @@ static const u8 enc_key110[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input111[] __initconst = { +static const u8 enc_input111[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x13, 0xf8, 0x0a, 0x00, 0x6d, 0xc1, 0xbb, 0xda, @@ -5599,7 +5598,7 @@ static const u8 enc_input111[] __initconst = { 0x2b, 0x6c, 0x89, 0x1d, 0x37, 0xc7, 0xe1, 0x1a, 0x56, 0x41, 0x91, 0x9c, 0x49, 0x4d, 0x95, 0x16 }; -static const u8 enc_output111[] __initconst = { +static const u8 enc_output111[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x94, 0x3a, 0xc0, 0x09, 0x81, 0xd8, 0x9d, 0x2c, @@ -5615,13 +5614,13 @@ static const u8 enc_output111[] __initconst = { 0x9a, 0x18, 0xa8, 0x28, 0x07, 0x02, 0x69, 0xf4, 0x47, 0x00, 0xd0, 0x09, 0xe7, 0x17, 0x1c, 0xc9 }; -static const u8 enc_assoc111[] __initconst = { +static const u8 enc_assoc111[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce111[] __initconst = { +static const u8 enc_nonce111[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key111[] __initconst = { +static const u8 enc_key111[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5629,7 +5628,7 @@ static const u8 enc_key111[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input112[] __initconst = { +static const u8 enc_input112[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x82, 0xe5, 0x9b, 0x45, 0x82, 0x91, 0x50, 0x38, @@ -5643,7 +5642,7 @@ static const u8 enc_input112[] __initconst = { 0xfe, 0x55, 0xf9, 0x2a, 0xdc, 0x08, 0xb5, 0xaa, 0x95, 0x48, 0xa9, 0x2d, 0x63, 0xaf, 0xe1, 0x13 }; -static const u8 enc_output112[] __initconst = { +static const u8 enc_output112[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x05, 0x27, 0x51, 0x4c, 0x6e, 0x88, 0x76, 0xce, @@ -5659,13 +5658,13 @@ static const u8 enc_output112[] __initconst = { 0xb4, 0x36, 0xa8, 0x2b, 0x93, 0xd5, 0x55, 0xf7, 0x43, 0x00, 0xd0, 0x19, 0x9b, 0xa7, 0x18, 0xce }; -static const u8 enc_assoc112[] __initconst = { +static const u8 enc_assoc112[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce112[] __initconst = { +static const u8 enc_nonce112[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key112[] __initconst = { +static const u8 enc_key112[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5673,7 +5672,7 @@ static const u8 enc_key112[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input113[] __initconst = { +static const u8 enc_input113[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xf1, 0xd1, 0x28, 0x87, 0xb7, 0x21, 0x69, 0x86, @@ -5687,7 +5686,7 @@ static const u8 enc_input113[] __initconst = { 0xef, 0xe3, 0x69, 0x79, 0xed, 0x9e, 0x7d, 0x3e, 0xc9, 0x52, 0x41, 0x4e, 0x49, 0xb1, 0x30, 0x16 }; -static const u8 enc_output113[] __initconst = { +static const u8 enc_output113[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x76, 0x13, 0xe2, 0x8e, 0x5b, 0x38, 0x4f, 0x70, @@ -5703,13 +5702,13 @@ static const u8 enc_output113[] __initconst = { 0xce, 0x54, 0xa8, 0x2e, 0x1f, 0xa9, 0x42, 0xfa, 0x3f, 0x00, 0xd0, 0x29, 0x4f, 0x37, 0x15, 0xd3 }; -static const u8 enc_assoc113[] __initconst = { +static const u8 enc_assoc113[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce113[] __initconst = { +static const u8 enc_nonce113[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key113[] __initconst = { +static const u8 enc_key113[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5717,7 +5716,7 @@ static const u8 enc_key113[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input114[] __initconst = { +static const u8 enc_input114[] = { 0xcb, 0xf1, 0xda, 0x9e, 0x0b, 0xa9, 0x37, 0x73, 0x74, 0xe6, 0x9e, 0x1c, 0x0e, 0x60, 0x0c, 0xfc, 0x34, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5729,7 +5728,7 @@ static const u8 enc_input114[] __initconst = { 0x66, 0x68, 0xdb, 0xc8, 0xf5, 0xf2, 0x0e, 0xf2, 0xb3, 0xf3, 0x8f, 0x00, 0xe2, 0x03, 0x17, 0x88 }; -static const u8 enc_output114[] __initconst = { +static const u8 enc_output114[] = { 0xcb, 0x9a, 0x0d, 0xb1, 0x8d, 0x63, 0xd7, 0xea, 0xd7, 0xc9, 0x60, 0xd6, 0xb2, 0x86, 0x74, 0x5f, 0xb3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5743,13 +5742,13 @@ static const u8 enc_output114[] __initconst = { 0x23, 0x83, 0xab, 0x0b, 0x79, 0x92, 0x05, 0x69, 0x9b, 0x51, 0x0a, 0xa7, 0x09, 0xbf, 0x31, 0xf1 }; -static const u8 enc_assoc114[] __initconst = { +static const u8 enc_assoc114[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce114[] __initconst = { +static const u8 enc_nonce114[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key114[] __initconst = { +static const u8 enc_key114[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5757,7 +5756,7 @@ static const u8 enc_key114[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input115[] __initconst = { +static const u8 enc_input115[] = { 0x8f, 0x27, 0x86, 0x94, 0xc4, 0xe9, 0xda, 0xeb, 0xd5, 0x8d, 0x3e, 0x5b, 0x96, 0x6e, 0x8b, 0x68, 0x42, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5769,7 +5768,7 @@ static const u8 enc_input115[] __initconst = { 0xde, 0x04, 0x9a, 0x00, 0xa8, 0x64, 0x06, 0x4b, 0xbc, 0xd4, 0x6f, 0xe4, 0xe4, 0x5b, 0x42, 0x8f }; -static const u8 enc_output115[] __initconst = { +static const u8 enc_output115[] = { 0x8f, 0x4c, 0x51, 0xbb, 0x42, 0x23, 0x3a, 0x72, 0x76, 0xa2, 0xc0, 0x91, 0x2a, 0x88, 0xf3, 0xcb, 0xc5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5783,13 +5782,13 @@ static const u8 enc_output115[] __initconst = { 0x8b, 0xfb, 0xab, 0x17, 0xa9, 0xe0, 0xb8, 0x74, 0x8b, 0x51, 0x0a, 0xe7, 0xd9, 0xfd, 0x23, 0x05 }; -static const u8 enc_assoc115[] __initconst = { +static const u8 enc_assoc115[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce115[] __initconst = { +static const u8 enc_nonce115[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key115[] __initconst = { +static const u8 enc_key115[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5797,7 +5796,7 @@ static const u8 enc_key115[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input116[] __initconst = { +static const u8 enc_input116[] = { 0xd5, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x9a, 0x22, 0xd7, 0x0a, 0x48, 0xe2, 0x4f, 0xdd, @@ -5811,7 +5810,7 @@ static const u8 enc_input116[] __initconst = { 0x3f, 0x91, 0xf8, 0xe7, 0xc7, 0xb1, 0x96, 0x25, 0x64, 0x61, 0x9c, 0x5e, 0x7e, 0x9b, 0xf6, 0x13 }; -static const u8 enc_output116[] __initconst = { +static const u8 enc_output116[] = { 0xd5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x1d, 0xe0, 0x1d, 0x03, 0xa4, 0xfb, 0x69, 0x2b, @@ -5827,13 +5826,13 @@ static const u8 enc_output116[] __initconst = { 0x49, 0xbc, 0x6e, 0x9f, 0xc5, 0x1c, 0x4d, 0x50, 0x30, 0x36, 0x64, 0x4d, 0x84, 0x27, 0x73, 0xd2 }; -static const u8 enc_assoc116[] __initconst = { +static const u8 enc_assoc116[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce116[] __initconst = { +static const u8 enc_nonce116[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key116[] __initconst = { +static const u8 enc_key116[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5841,7 +5840,7 @@ static const u8 enc_key116[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input117[] __initconst = { +static const u8 enc_input117[] = { 0xdb, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x75, 0xd5, 0x64, 0x3a, 0xa5, 0xaf, 0x93, 0x4d, @@ -5855,7 +5854,7 @@ static const u8 enc_input117[] __initconst = { 0x28, 0x3f, 0x6b, 0x32, 0x18, 0x07, 0x5f, 0xc9, 0x5f, 0x6b, 0xb4, 0xff, 0x45, 0x6d, 0xc1, 0x11 }; -static const u8 enc_output117[] __initconst = { +static const u8 enc_output117[] = { 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf2, 0x17, 0xae, 0x33, 0x49, 0xb6, 0xb5, 0xbb, @@ -5871,13 +5870,13 @@ static const u8 enc_output117[] __initconst = { 0x63, 0xda, 0x6e, 0xa2, 0x51, 0xf0, 0x39, 0x53, 0x2c, 0x36, 0x64, 0x5d, 0x38, 0xb7, 0x6f, 0xd7 }; -static const u8 enc_assoc117[] __initconst = { +static const u8 enc_assoc117[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce117[] __initconst = { +static const u8 enc_nonce117[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key117[] __initconst = { +static const u8 enc_key117[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5885,7 +5884,7 @@ static const u8 enc_key117[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input118[] __initconst = { +static const u8 enc_input118[] = { 0x93, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x62, 0x48, 0x39, 0x60, 0x42, 0x16, 0xe4, 0x03, @@ -5899,7 +5898,7 @@ static const u8 enc_input118[] __initconst = { 0x90, 0xec, 0xf2, 0x1a, 0x04, 0xe6, 0x30, 0x85, 0x8b, 0xb6, 0x56, 0x52, 0xb5, 0xb1, 0x80, 0x16 }; -static const u8 enc_output118[] __initconst = { +static const u8 enc_output118[] = { 0x93, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe5, 0x8a, 0xf3, 0x69, 0xae, 0x0f, 0xc2, 0xf5, @@ -5915,13 +5914,13 @@ static const u8 enc_output118[] __initconst = { 0x73, 0xeb, 0x27, 0x24, 0xb5, 0xc4, 0x05, 0xf0, 0x4d, 0x00, 0xd0, 0xf1, 0x58, 0x40, 0xa1, 0xc1 }; -static const u8 enc_assoc118[] __initconst = { +static const u8 enc_assoc118[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce118[] __initconst = { +static const u8 enc_nonce118[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key118[] __initconst = { +static const u8 enc_key118[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5929,7 +5928,7 @@ static const u8 enc_key118[] __initconst = { }; static const struct chacha20poly1305_testvec -chacha20poly1305_enc_vectors[] __initconst = { +chacha20poly1305_enc_vectors[] = { { enc_input001, enc_output001, enc_assoc001, enc_nonce001, enc_key001, sizeof(enc_input001), sizeof(enc_assoc001), sizeof(enc_nonce001) }, { enc_input002, enc_output002, enc_assoc002, enc_nonce002, enc_key002, @@ -6168,7 +6167,7 @@ chacha20poly1305_enc_vectors[] __initconst = { sizeof(enc_input118), sizeof(enc_assoc118), sizeof(enc_nonce118) } }; -static const u8 dec_input001[] __initconst = { +static const u8 dec_input001[] = { 0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4, 0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd, 0x5e, 0x80, 0x5c, 0xfd, 0x34, 0x5c, 0xf3, 0x89, @@ -6206,7 +6205,7 @@ static const u8 dec_input001[] __initconst = { 0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f, 0x38 }; -static const u8 dec_output001[] __initconst = { +static const u8 dec_output001[] = { 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, @@ -6242,95 +6241,95 @@ static const u8 dec_output001[] __initconst = { 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d }; -static const u8 dec_assoc001[] __initconst = { +static const u8 dec_assoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; -static const u8 dec_nonce001[] __initconst = { +static const u8 dec_nonce001[] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 }; -static const u8 dec_key001[] __initconst = { +static const u8 dec_key001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 }; -static const u8 dec_input002[] __initconst = { +static const u8 dec_input002[] = { 0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1, 0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92 }; -static const u8 dec_output002[] __initconst = { }; -static const u8 dec_assoc002[] __initconst = { }; -static const u8 dec_nonce002[] __initconst = { +static const u8 dec_output002[] = { }; +static const u8 dec_assoc002[] = { }; +static const u8 dec_nonce002[] = { 0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e }; -static const u8 dec_key002[] __initconst = { +static const u8 dec_key002[] = { 0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f, 0x2d, 0x29, 0x25, 0x76, 0xd5, 0x75, 0x27, 0x86, 0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, 0xc5, 0xef, 0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68 }; -static const u8 dec_input003[] __initconst = { +static const u8 dec_input003[] = { 0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6, 0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77 }; -static const u8 dec_output003[] __initconst = { }; -static const u8 dec_assoc003[] __initconst = { +static const u8 dec_output003[] = { }; +static const u8 dec_assoc003[] = { 0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b }; -static const u8 dec_nonce003[] __initconst = { +static const u8 dec_nonce003[] = { 0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d }; -static const u8 dec_key003[] __initconst = { +static const u8 dec_key003[] = { 0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88, 0x34, 0xd1, 0x13, 0xaf, 0x57, 0xa1, 0xeb, 0x3a, 0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, 0xbc, 0x08, 0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d }; -static const u8 dec_input004[] __initconst = { +static const u8 dec_input004[] = { 0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2, 0x6d, 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac, 0x89 }; -static const u8 dec_output004[] __initconst = { +static const u8 dec_output004[] = { 0xa4 }; -static const u8 dec_assoc004[] __initconst = { +static const u8 dec_assoc004[] = { 0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40 }; -static const u8 dec_nonce004[] __initconst = { +static const u8 dec_nonce004[] = { 0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4 }; -static const u8 dec_key004[] __initconst = { +static const u8 dec_key004[] = { 0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8, 0x31, 0x80, 0x82, 0xd7, 0xd8, 0xe8, 0xb5, 0xa1, 0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, 0xa3, 0x3d, 0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e }; -static const u8 dec_input005[] __initconst = { +static const u8 dec_input005[] = { 0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e, 0x6c, 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c, 0xac }; -static const u8 dec_output005[] __initconst = { +static const u8 dec_output005[] = { 0x2d }; -static const u8 dec_assoc005[] __initconst = { }; -static const u8 dec_nonce005[] __initconst = { +static const u8 dec_assoc005[] = { }; +static const u8 dec_nonce005[] = { 0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30 }; -static const u8 dec_key005[] __initconst = { +static const u8 dec_key005[] = { 0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31, 0x0e, 0x92, 0x89, 0x8b, 0xf4, 0x93, 0xc7, 0x87, 0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, 0xa7, 0x01, 0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87 }; -static const u8 dec_input006[] __initconst = { +static const u8 dec_input006[] = { 0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1, 0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15, 0x81, 0x2c, 0xb5, 0xf0, 0xc6, 0x2b, 0xc7, 0x8c, @@ -6351,7 +6350,7 @@ static const u8 dec_input006[] __initconst = { 0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a, 0xeb }; -static const u8 dec_output006[] __initconst = { +static const u8 dec_output006[] = { 0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a, 0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92, 0x3c, 0xd9, 0x24, 0x11, 0xa9, 0x71, 0xf9, 0x37, @@ -6370,20 +6369,20 @@ static const u8 dec_output006[] __initconst = { 0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde, 0x8f }; -static const u8 dec_assoc006[] __initconst = { +static const u8 dec_assoc006[] = { 0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b }; -static const u8 dec_nonce006[] __initconst = { +static const u8 dec_nonce006[] = { 0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c }; -static const u8 dec_key006[] __initconst = { +static const u8 dec_key006[] = { 0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae, 0xdf, 0x72, 0x7f, 0x53, 0x72, 0x25, 0x1e, 0x78, 0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, 0x93, 0xf9, 0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01 }; -static const u8 dec_input007[] __initconst = { +static const u8 dec_input007[] = { 0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c, 0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8, 0xc9, 0x50, 0xc3, 0xc6, 0xa5, 0xe3, 0xa4, 0x7c, @@ -6419,7 +6418,7 @@ static const u8 dec_input007[] __initconst = { 0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42, 0x3a, 0x00, 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53 }; -static const u8 dec_output007[] __initconst = { +static const u8 dec_output007[] = { 0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5, 0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a, 0x67, 0x30, 0x12, 0xe2, 0x34, 0x77, 0x4b, 0xc1, @@ -6453,18 +6452,18 @@ static const u8 dec_output007[] __initconst = { 0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, 0x2b, 0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3 }; -static const u8 dec_assoc007[] __initconst = { }; -static const u8 dec_nonce007[] __initconst = { +static const u8 dec_assoc007[] = { }; +static const u8 dec_nonce007[] = { 0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0 }; -static const u8 dec_key007[] __initconst = { +static const u8 dec_key007[] = { 0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd, 0xf9, 0x3f, 0xcd, 0xd9, 0xa0, 0x1e, 0x42, 0x4c, 0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, 0x05, 0x80, 0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01 }; -static const u8 dec_input008[] __initconst = { +static const u8 dec_input008[] = { 0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd, 0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1, 0x1e, 0x6b, 0xd2, 0xbc, 0x11, 0xf4, 0x28, 0x93, @@ -6532,7 +6531,7 @@ static const u8 dec_input008[] __initconst = { 0x54, 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f, 0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54 }; -static const u8 dec_output008[] __initconst = { +static const u8 dec_output008[] = { 0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10, 0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2, 0x0f, 0xc2, 0x8b, 0x28, 0xdc, 0xba, 0xb4, 0x3c, @@ -6598,18 +6597,18 @@ static const u8 dec_output008[] __initconst = { 0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b, 0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3 }; -static const u8 dec_assoc008[] __initconst = { }; -static const u8 dec_nonce008[] __initconst = { +static const u8 dec_assoc008[] = { }; +static const u8 dec_nonce008[] = { 0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02 }; -static const u8 dec_key008[] __initconst = { +static const u8 dec_key008[] = { 0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53, 0xc1, 0x44, 0xe9, 0x81, 0x18, 0xdc, 0xf5, 0xf0, 0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, 0x44, 0x86, 0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba }; -static const u8 dec_input009[] __initconst = { +static const u8 dec_input009[] = { 0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf, 0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66, 0x4b, 0x2e, 0x0c, 0x27, 0x9c, 0x96, 0x4c, 0x72, @@ -6678,7 +6677,7 @@ static const u8 dec_input009[] __initconst = { 0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8, 0xae }; -static const u8 dec_output009[] __initconst = { +static const u8 dec_output009[] = { 0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b, 0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8, 0x3a, 0x6b, 0xd7, 0x81, 0x96, 0x35, 0x97, 0xca, @@ -6745,21 +6744,21 @@ static const u8 dec_output009[] __initconst = { 0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52, 0x65 }; -static const u8 dec_assoc009[] __initconst = { +static const u8 dec_assoc009[] = { 0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e, 0xef }; -static const u8 dec_nonce009[] __initconst = { +static const u8 dec_nonce009[] = { 0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78 }; -static const u8 dec_key009[] __initconst = { +static const u8 dec_key009[] = { 0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5, 0xcc, 0x1a, 0xd7, 0xc1, 0x57, 0x72, 0xea, 0x86, 0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, 0x9b, 0xb2, 0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b }; -static const u8 dec_input010[] __initconst = { +static const u8 dec_input010[] = { 0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b, 0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74, 0xa6, 0xdd, 0xbd, 0x95, 0xeb, 0xf9, 0xa4, 0xf1, @@ -6891,7 +6890,7 @@ static const u8 dec_input010[] __initconst = { 0x80, 0x6b, 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9, 0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0 }; -static const u8 dec_output010[] __initconst = { +static const u8 dec_output010[] = { 0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf, 0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c, 0xef, 0x0a, 0xa9, 0x48, 0x5f, 0x5f, 0x37, 0x22, @@ -7021,21 +7020,21 @@ static const u8 dec_output010[] __initconst = { 0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c, 0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f }; -static const u8 dec_assoc010[] __initconst = { +static const u8 dec_assoc010[] = { 0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27, 0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2 }; -static const u8 dec_nonce010[] __initconst = { +static const u8 dec_nonce010[] = { 0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30 }; -static const u8 dec_key010[] __initconst = { +static const u8 dec_key010[] = { 0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44, 0x34, 0xda, 0x7f, 0x57, 0x03, 0x39, 0x0c, 0xaf, 0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, 0x8e, 0x74, 0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7 }; -static const u8 dec_input011[] __initconst = { +static const u8 dec_input011[] = { 0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8, 0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc, 0x3c, 0xba, 0x6a, 0x77, 0x47, 0xdb, 0xe3, 0x74, @@ -7281,7 +7280,7 @@ static const u8 dec_input011[] __initconst = { 0x86, 0xc4, 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e, 0x2b, 0xdf, 0xcd, 0xf9, 0x3c }; -static const u8 dec_output011[] __initconst = { +static const u8 dec_output011[] = { 0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b, 0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b, 0x45, 0x60, 0xbe, 0x9a, 0x31, 0x9f, 0xff, 0x5d, @@ -7525,20 +7524,20 @@ static const u8 dec_output011[] __initconst = { 0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61, 0x10, 0x1e, 0xbf, 0xec, 0xa8 }; -static const u8 dec_assoc011[] __initconst = { +static const u8 dec_assoc011[] = { 0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7 }; -static const u8 dec_nonce011[] __initconst = { +static const u8 dec_nonce011[] = { 0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa }; -static const u8 dec_key011[] __initconst = { +static const u8 dec_key011[] = { 0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85, 0xf2, 0xfb, 0xed, 0x7b, 0xd0, 0x9e, 0x97, 0xca, 0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, 0xcc, 0x52, 0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38 }; -static const u8 dec_input012[] __initconst = { +static const u8 dec_input012[] = { 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, @@ -7794,7 +7793,7 @@ static const u8 dec_input012[] __initconst = { 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, 0x70, 0xcf, 0xd6 }; -static const u8 dec_output012[] __initconst = { +static const u8 dec_output012[] = { 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, @@ -8048,7 +8047,7 @@ static const u8 dec_output012[] __initconst = { 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, 0x78, 0xec, 0x00 }; -static const u8 dec_assoc012[] __initconst = { +static const u8 dec_assoc012[] = { 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, @@ -8058,17 +8057,17 @@ static const u8 dec_assoc012[] __initconst = { 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 }; -static const u8 dec_nonce012[] __initconst = { +static const u8 dec_nonce012[] = { 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 }; -static const u8 dec_key012[] __initconst = { +static const u8 dec_key012[] = { 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 }; -static const u8 dec_input013[] __initconst = { +static const u8 dec_input013[] = { 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, @@ -8324,7 +8323,7 @@ static const u8 dec_input013[] __initconst = { 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, 0x70, 0xcf, 0xd7 }; -static const u8 dec_output013[] __initconst = { +static const u8 dec_output013[] = { 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, @@ -8578,7 +8577,7 @@ static const u8 dec_output013[] __initconst = { 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, 0x78, 0xec, 0x00 }; -static const u8 dec_assoc013[] __initconst = { +static const u8 dec_assoc013[] = { 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, @@ -8588,10 +8587,10 @@ static const u8 dec_assoc013[] __initconst = { 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 }; -static const u8 dec_nonce013[] __initconst = { +static const u8 dec_nonce013[] = { 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 }; -static const u8 dec_key013[] __initconst = { +static const u8 dec_key013[] = { 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, @@ -8599,7 +8598,7 @@ static const u8 dec_key013[] __initconst = { }; static const struct chacha20poly1305_testvec -chacha20poly1305_dec_vectors[] __initconst = { +chacha20poly1305_dec_vectors[] = { { dec_input001, dec_output001, dec_assoc001, dec_nonce001, dec_key001, sizeof(dec_input001), sizeof(dec_assoc001), sizeof(dec_nonce001) }, { dec_input002, dec_output002, dec_assoc002, dec_nonce002, dec_key002, @@ -8629,7 +8628,7 @@ chacha20poly1305_dec_vectors[] __initconst = { true } }; -static const u8 xenc_input001[] __initconst = { +static const u8 xenc_input001[] = { 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, @@ -8665,7 +8664,7 @@ static const u8 xenc_input001[] __initconst = { 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d }; -static const u8 xenc_output001[] __initconst = { +static const u8 xenc_output001[] = { 0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77, 0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92, 0x2e, 0xed, 0x93, 0xcf, 0x0f, 0x71, 0x88, 0x18, @@ -8703,16 +8702,16 @@ static const u8 xenc_output001[] __initconst = { 0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5, 0x9c }; -static const u8 xenc_assoc001[] __initconst = { +static const u8 xenc_assoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; -static const u8 xenc_nonce001[] __initconst = { +static const u8 xenc_nonce001[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 }; -static const u8 xenc_key001[] __initconst = { +static const u8 xenc_key001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, @@ -8720,12 +8719,12 @@ static const u8 xenc_key001[] __initconst = { }; static const struct chacha20poly1305_testvec -xchacha20poly1305_enc_vectors[] __initconst = { +xchacha20poly1305_enc_vectors[] = { { xenc_input001, xenc_output001, xenc_assoc001, xenc_nonce001, xenc_key001, sizeof(xenc_input001), sizeof(xenc_assoc001), sizeof(xenc_nonce001) } }; -static const u8 xdec_input001[] __initconst = { +static const u8 xdec_input001[] = { 0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77, 0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92, 0x2e, 0xed, 0x93, 0xcf, 0x0f, 0x71, 0x88, 0x18, @@ -8763,7 +8762,7 @@ static const u8 xdec_input001[] __initconst = { 0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5, 0x9c }; -static const u8 xdec_output001[] __initconst = { +static const u8 xdec_output001[] = { 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, @@ -8799,16 +8798,16 @@ static const u8 xdec_output001[] __initconst = { 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d }; -static const u8 xdec_assoc001[] __initconst = { +static const u8 xdec_assoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; -static const u8 xdec_nonce001[] __initconst = { +static const u8 xdec_nonce001[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 }; -static const u8 xdec_key001[] __initconst = { +static const u8 xdec_key001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, @@ -8816,15 +8815,15 @@ static const u8 xdec_key001[] __initconst = { }; static const struct chacha20poly1305_testvec -xchacha20poly1305_dec_vectors[] __initconst = { +xchacha20poly1305_dec_vectors[] = { { xdec_input001, xdec_output001, xdec_assoc001, xdec_nonce001, xdec_key001, sizeof(xdec_input001), sizeof(xdec_assoc001), sizeof(xdec_nonce001) } }; -/* This is for the selftests-only, since it is only useful for the purpose of +/* This is for the tests only, since it is only useful for the purpose of * testing the underlying primitives and interactions. */ -static void __init +static void chacha20poly1305_encrypt_bignonce(u8 *dst, const u8 *src, const size_t src_len, const u8 *ad, const size_t ad_len, const u8 nonce[12], @@ -8858,11 +8857,12 @@ chacha20poly1305_encrypt_bignonce(u8 *dst, const u8 *src, const size_t src_len, poly1305_final(&poly1305_state, dst + src_len); } -static void __init -chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, - const u8 *ad, const size_t ad_len, - const u8 *nonce, const size_t nonce_len, - const u8 key[CHACHA20POLY1305_KEY_SIZE]) +static void +chacha20poly1305_test_encrypt(struct kunit *test, u8 *dst, + const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 *nonce, const size_t nonce_len, + const u8 key[CHACHA20POLY1305_KEY_SIZE]) { if (nonce_len == 8) chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, @@ -8871,10 +8871,10 @@ chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, chacha20poly1305_encrypt_bignonce(dst, src, src_len, ad, ad_len, nonce, key); else - BUG(); + KUNIT_FAIL(test, "bad nonce_len: %zu", nonce_len); } -static bool __init +static bool decryption_success(bool func_ret, bool expect_failure, int memcmp_result) { if (expect_failure) @@ -8882,25 +8882,23 @@ decryption_success(bool func_ret, bool expect_failure, int memcmp_result) return func_ret && !memcmp_result; } -bool __init chacha20poly1305_selftest(void) +static void test_chacha20poly1305(struct kunit *test) { enum { MAXIMUM_TEST_BUFFER_LEN = 1UL << 12 }; size_t i, j, k, total_len; u8 *computed_output = NULL, *input = NULL; - bool success = true, ret; + bool ret; struct scatterlist sg_src[3]; - computed_output = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); - input = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); - if (!computed_output || !input) { - pr_err("chacha20poly1305 self-test malloc: FAIL\n"); - success = false; - goto out; - } + computed_output = kunit_kmalloc(test, MAXIMUM_TEST_BUFFER_LEN, + GFP_KERNEL); + input = kunit_kmalloc(test, MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, computed_output); + KUNIT_ASSERT_NOT_NULL(test, input); for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); - chacha20poly1305_selftest_encrypt(computed_output, + chacha20poly1305_test_encrypt(test, computed_output, chacha20poly1305_enc_vectors[i].input, chacha20poly1305_enc_vectors[i].ilen, chacha20poly1305_enc_vectors[i].assoc, @@ -8908,14 +8906,13 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_enc_vectors[i].nonce, chacha20poly1305_enc_vectors[i].nlen, chacha20poly1305_enc_vectors[i].key); - if (memcmp(computed_output, - chacha20poly1305_enc_vectors[i].output, - chacha20poly1305_enc_vectors[i].ilen + - POLY1305_DIGEST_SIZE)) { - pr_err("chacha20poly1305 encryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + KUNIT_EXPECT_TRUE_MSG( + test, + memcmp(computed_output, + chacha20poly1305_enc_vectors[i].output, + chacha20poly1305_enc_vectors[i].ilen + + POLY1305_DIGEST_SIZE) == 0, + "chacha20poly1305 encryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { @@ -8931,14 +8928,13 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_enc_vectors[i].alen, get_unaligned_le64(chacha20poly1305_enc_vectors[i].nonce), chacha20poly1305_enc_vectors[i].key); - if (!ret || memcmp(computed_output, - chacha20poly1305_enc_vectors[i].output, - chacha20poly1305_enc_vectors[i].ilen + - POLY1305_DIGEST_SIZE)) { - pr_err("chacha20poly1305 sg encryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + KUNIT_EXPECT_TRUE_MSG( + test, + ret && memcmp(computed_output, + chacha20poly1305_enc_vectors[i].output, + chacha20poly1305_enc_vectors[i].ilen + + POLY1305_DIGEST_SIZE) == 0, + "chacha20poly1305 sg encryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) { @@ -8950,16 +8946,15 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_dec_vectors[i].alen, get_unaligned_le64(chacha20poly1305_dec_vectors[i].nonce), chacha20poly1305_dec_vectors[i].key); - if (!decryption_success(ret, - chacha20poly1305_dec_vectors[i].failure, + KUNIT_EXPECT_TRUE_MSG( + test, + decryption_success( + ret, chacha20poly1305_dec_vectors[i].failure, memcmp(computed_output, chacha20poly1305_dec_vectors[i].output, chacha20poly1305_dec_vectors[i].ilen - - POLY1305_DIGEST_SIZE))) { - pr_err("chacha20poly1305 decryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + POLY1305_DIGEST_SIZE)), + "chacha20poly1305 decryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) { @@ -8973,15 +8968,15 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_dec_vectors[i].alen, get_unaligned_le64(chacha20poly1305_dec_vectors[i].nonce), chacha20poly1305_dec_vectors[i].key); - if (!decryption_success(ret, - chacha20poly1305_dec_vectors[i].failure, - memcmp(computed_output, chacha20poly1305_dec_vectors[i].output, - chacha20poly1305_dec_vectors[i].ilen - - POLY1305_DIGEST_SIZE))) { - pr_err("chacha20poly1305 sg decryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + KUNIT_EXPECT_TRUE_MSG( + test, + decryption_success( + ret, chacha20poly1305_dec_vectors[i].failure, + memcmp(computed_output, + chacha20poly1305_dec_vectors[i].output, + chacha20poly1305_dec_vectors[i].ilen - + POLY1305_DIGEST_SIZE)), + "chacha20poly1305 sg decryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_enc_vectors); ++i) { @@ -8993,14 +8988,13 @@ bool __init chacha20poly1305_selftest(void) xchacha20poly1305_enc_vectors[i].alen, xchacha20poly1305_enc_vectors[i].nonce, xchacha20poly1305_enc_vectors[i].key); - if (memcmp(computed_output, - xchacha20poly1305_enc_vectors[i].output, - xchacha20poly1305_enc_vectors[i].ilen + - POLY1305_DIGEST_SIZE)) { - pr_err("xchacha20poly1305 encryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + KUNIT_EXPECT_TRUE_MSG( + test, + memcmp(computed_output, + xchacha20poly1305_enc_vectors[i].output, + xchacha20poly1305_enc_vectors[i].ilen + + POLY1305_DIGEST_SIZE) == 0, + "xchacha20poly1305 encryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_dec_vectors); ++i) { @@ -9012,16 +9006,15 @@ bool __init chacha20poly1305_selftest(void) xchacha20poly1305_dec_vectors[i].alen, xchacha20poly1305_dec_vectors[i].nonce, xchacha20poly1305_dec_vectors[i].key); - if (!decryption_success(ret, - xchacha20poly1305_dec_vectors[i].failure, + KUNIT_EXPECT_TRUE_MSG( + test, + decryption_success( + ret, xchacha20poly1305_dec_vectors[i].failure, memcmp(computed_output, xchacha20poly1305_dec_vectors[i].output, xchacha20poly1305_dec_vectors[i].ilen - - POLY1305_DIGEST_SIZE))) { - pr_err("xchacha20poly1305 decryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + POLY1305_DIGEST_SIZE)), + "xchacha20poly1305 decryption test %zu: FAIL", i + 1); } for (total_len = POLY1305_DIGEST_SIZE; IS_ENABLED(DEBUG_CHACHA20POLY1305_SLOW_CHUNK_TEST) @@ -9067,16 +9060,26 @@ bool __init chacha20poly1305_selftest(void) continue; chunkfail: - pr_err("chacha20poly1305 chunked self-test %zu/%zu/%zu: FAIL\n", - total_len, i, j); - success = false; + KUNIT_FAIL( + test, + "chacha20poly1305 chunked test %zu/%zu/%zu: FAIL\n", + total_len, i, j); } } } - -out: - kfree(computed_output); - kfree(input); - return success; } + +static struct kunit_case chacha20poly1305_test_cases[] = { + KUNIT_CASE(test_chacha20poly1305), + {}, +}; + +static struct kunit_suite chacha20poly1305_test_suite = { + .name = "chacha20poly1305", + .test_cases = chacha20poly1305_test_cases, +}; +kunit_test_suite(chacha20poly1305_test_suite); + +MODULE_DESCRIPTION("KUnit tests for ChaCha20Poly1305"); +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/tests/ghash-testvecs.h b/lib/crypto/tests/ghash-testvecs.h new file mode 100644 index 000000000000..759eb4072336 --- /dev/null +++ b/lib/crypto/tests/ghash-testvecs.h @@ -0,0 +1,186 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py ghash */ + +static const struct { + size_t data_len; + u8 digest[GHASH_DIGEST_SIZE]; +} hash_testvecs[] = { + { + .data_len = 0, + .digest = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, + }, + { + .data_len = 1, + .digest = { + 0x13, 0x91, 0xa1, 0x11, 0x08, 0xc3, 0x7e, 0xeb, + 0x21, 0x42, 0x4a, 0xd6, 0x45, 0x0f, 0x41, 0xa7, + }, + }, + { + .data_len = 2, + .digest = { + 0xde, 0x00, 0x63, 0x3f, 0x71, 0x0f, 0xc6, 0x29, + 0x53, 0x2e, 0x49, 0xd9, 0xc2, 0xb7, 0x73, 0xce, + }, + }, + { + .data_len = 3, + .digest = { + 0xcf, 0xc7, 0xa8, 0x20, 0x24, 0xe9, 0x7a, 0x6c, + 0x2c, 0x2a, 0x34, 0x70, 0x26, 0xba, 0xd5, 0x9a, + }, + }, + { + .data_len = 16, + .digest = { + 0xaa, 0xe0, 0xdc, 0x7f, 0xcf, 0x8b, 0xe6, 0x0c, + 0x2e, 0x93, 0x89, 0x7d, 0x68, 0x4e, 0xc2, 0x63, + }, + }, + { + .data_len = 32, + .digest = { + 0x4b, 0x8b, 0x93, 0x5c, 0x79, 0xad, 0x85, 0x08, + 0xd3, 0x8a, 0xcd, 0xdd, 0x4c, 0x6e, 0x0e, 0x6f, + }, + }, + { + .data_len = 48, + .digest = { + 0xfa, 0xa0, 0x25, 0xdd, 0x61, 0x9a, 0x52, 0x9a, + 0xea, 0xee, 0xc6, 0x62, 0xb2, 0xba, 0x11, 0x49, + }, + }, + { + .data_len = 49, + .digest = { + 0x23, 0xf1, 0x05, 0xeb, 0x30, 0x40, 0xb9, 0x1d, + 0xe6, 0x35, 0x51, 0x4e, 0x0f, 0xc0, 0x1b, 0x9e, + }, + }, + { + .data_len = 63, + .digest = { + 0x8d, 0xcf, 0xa0, 0xc8, 0x83, 0x21, 0x06, 0x81, + 0xc6, 0x36, 0xd5, 0x62, 0xbf, 0xa0, 0xcd, 0x9c, + }, + }, + { + .data_len = 64, + .digest = { + 0xe7, 0xca, 0xbe, 0xe7, 0x66, 0xc8, 0x85, 0xad, + 0xbc, 0xaf, 0x58, 0x21, 0xd7, 0x67, 0x82, 0x15, + }, + }, + { + .data_len = 65, + .digest = { + 0x9f, 0x48, 0x10, 0xd9, 0xa2, 0x6b, 0x9d, 0xe0, + 0xb1, 0x87, 0xe1, 0x39, 0xc3, 0xd7, 0xee, 0x09, + }, + }, + { + .data_len = 127, + .digest = { + 0xa4, 0x36, 0xb7, 0x82, 0xd2, 0x67, 0x7e, 0xaf, + 0x5d, 0xfd, 0x67, 0x9c, 0x1d, 0x9f, 0xe4, 0xf7, + }, + }, + { + .data_len = 128, + .digest = { + 0x57, 0xe7, 0x1d, 0x78, 0xf0, 0x8e, 0xc7, 0x0c, + 0x15, 0xee, 0x18, 0xc4, 0xd1, 0x75, 0x90, 0xaa, + }, + }, + { + .data_len = 129, + .digest = { + 0x9b, 0xad, 0x81, 0xa9, 0x22, 0xb2, 0x19, 0x53, + 0x60, 0x30, 0xe7, 0xa0, 0x4f, 0xd6, 0x72, 0x42, + }, + }, + { + .data_len = 256, + .digest = { + 0xf7, 0x33, 0x42, 0xbf, 0x58, 0xde, 0x88, 0x0f, + 0x8d, 0x3d, 0xa6, 0x11, 0x14, 0xc3, 0xf1, 0xdc, + }, + }, + { + .data_len = 511, + .digest = { + 0x59, 0xdc, 0xa9, 0xc0, 0x4e, 0xd6, 0x97, 0xb3, + 0x60, 0xaf, 0xa8, 0xa0, 0xea, 0x54, 0x8e, 0xc3, + }, + }, + { + .data_len = 513, + .digest = { + 0xa2, 0x23, 0x37, 0xcc, 0x97, 0xec, 0xea, 0xbe, + 0xd6, 0xc7, 0x13, 0xf7, 0x93, 0x73, 0xc0, 0x64, + }, + }, + { + .data_len = 1000, + .digest = { + 0x46, 0x8b, 0x43, 0x77, 0x9b, 0xc2, 0xfc, 0xa4, + 0x68, 0x6a, 0x6c, 0x07, 0xa4, 0x6f, 0x47, 0x65, + }, + }, + { + .data_len = 3333, + .digest = { + 0x69, 0x7f, 0x19, 0xc3, 0xb9, 0xa4, 0xff, 0x40, + 0xe3, 0x03, 0x71, 0xa3, 0x88, 0x8a, 0xf1, 0xbd, + }, + }, + { + .data_len = 4096, + .digest = { + 0x4d, 0x65, 0xe6, 0x9c, 0xeb, 0x6a, 0x46, 0x8d, + 0xe9, 0x32, 0x96, 0x72, 0xb3, 0x0d, 0x08, 0xa9, + }, + }, + { + .data_len = 4128, + .digest = { + 0xfc, 0xa1, 0x74, 0x46, 0x21, 0x64, 0xa7, 0x64, + 0xbe, 0x47, 0x03, 0x1e, 0x05, 0xf7, 0xd8, 0x37, + }, + }, + { + .data_len = 4160, + .digest = { + 0x70, 0x5b, 0xe9, 0x17, 0xab, 0xd5, 0xa2, 0xee, + 0xcb, 0x39, 0xa4, 0x81, 0x2f, 0x41, 0x70, 0xae, + }, + }, + { + .data_len = 4224, + .digest = { + 0x07, 0xbd, 0xb6, 0x52, 0xe2, 0x75, 0x2c, 0x33, + 0x6d, 0x1b, 0x63, 0x56, 0x58, 0xda, 0x98, 0x55, + }, + }, + { + .data_len = 16384, + .digest = { + 0x9c, 0xb5, 0xf4, 0x14, 0xe8, 0xa8, 0x4a, 0xde, + 0xee, 0x7b, 0xbb, 0xd6, 0x21, 0x6d, 0x6a, 0x69, + }, + }, +}; + +static const u8 hash_testvec_consolidated[GHASH_DIGEST_SIZE] = { + 0x08, 0xef, 0xf5, 0x27, 0xb1, 0xca, 0xd4, 0x1d, + 0xad, 0x38, 0x69, 0x88, 0x6b, 0x16, 0xdf, 0xa8, +}; + +static const u8 ghash_allones_hashofhashes[GHASH_DIGEST_SIZE] = { + 0xef, 0x85, 0x58, 0xf8, 0x54, 0x9c, 0x5e, 0x54, + 0xd9, 0xbe, 0x04, 0x1f, 0xff, 0xff, 0xff, 0xff, +}; diff --git a/lib/crypto/tests/ghash_kunit.c b/lib/crypto/tests/ghash_kunit.c new file mode 100644 index 000000000000..68b3837a3607 --- /dev/null +++ b/lib/crypto/tests/ghash_kunit.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2026 Google LLC + */ +#include <crypto/gf128hash.h> +#include "ghash-testvecs.h" + +/* + * A fixed key used when presenting GHASH as an unkeyed hash function in order + * to reuse hash-test-template.h. At the beginning of the test suite, this is + * initialized to a key prepared from bytes generated from a fixed seed. + */ +static struct ghash_key test_key; + +static void ghash_init_withtestkey(struct ghash_ctx *ctx) +{ + ghash_init(ctx, &test_key); +} + +static void ghash_withtestkey(const u8 *data, size_t len, + u8 out[GHASH_BLOCK_SIZE]) +{ + ghash(&test_key, data, len, out); +} + +/* Generate the HASH_KUNIT_CASES using hash-test-template.h. */ +#define HASH ghash_withtestkey +#define HASH_CTX ghash_ctx +#define HASH_SIZE GHASH_BLOCK_SIZE +#define HASH_INIT ghash_init_withtestkey +#define HASH_UPDATE ghash_update +#define HASH_FINAL ghash_final +#include "hash-test-template.h" + +/* + * Test a key and messages containing all one bits. This is useful to detect + * overflow bugs in implementations that emulate carryless multiplication using + * a series of standard multiplications with the bits spread out. + */ +static void test_ghash_allones_key_and_message(struct kunit *test) +{ + struct ghash_key key; + struct ghash_ctx hashofhashes_ctx; + u8 hash[GHASH_BLOCK_SIZE]; + + static_assert(TEST_BUF_LEN >= 4096); + memset(test_buf, 0xff, 4096); + + ghash_preparekey(&key, test_buf); + ghash_init(&hashofhashes_ctx, &key); + for (size_t len = 0; len <= 4096; len += 16) { + ghash(&key, test_buf, len, hash); + ghash_update(&hashofhashes_ctx, hash, sizeof(hash)); + } + ghash_final(&hashofhashes_ctx, hash); + KUNIT_ASSERT_MEMEQ(test, hash, ghash_allones_hashofhashes, + sizeof(hash)); +} + +#define MAX_LEN_FOR_KEY_CHECK 1024 + +/* + * Given two prepared keys which should be identical (but may differ in + * alignment and/or whether they are followed by a guard page or not), verify + * that they produce consistent results on various data lengths. + */ +static void check_key_consistency(struct kunit *test, + const struct ghash_key *key1, + const struct ghash_key *key2) +{ + u8 *data = test_buf; + u8 hash1[GHASH_BLOCK_SIZE]; + u8 hash2[GHASH_BLOCK_SIZE]; + + rand_bytes(data, MAX_LEN_FOR_KEY_CHECK); + KUNIT_ASSERT_MEMEQ(test, key1, key2, sizeof(*key1)); + + for (int i = 0; i < 100; i++) { + size_t len = rand_length(MAX_LEN_FOR_KEY_CHECK); + + ghash(key1, data, len, hash1); + ghash(key2, data, len, hash2); + KUNIT_ASSERT_MEMEQ(test, hash1, hash2, sizeof(hash1)); + } +} + +/* Test that no buffer overreads occur on either raw_key or ghash_key. */ +static void test_ghash_with_guarded_key(struct kunit *test) +{ + u8 raw_key[GHASH_BLOCK_SIZE]; + u8 *guarded_raw_key = &test_buf[TEST_BUF_LEN - sizeof(raw_key)]; + struct ghash_key key1, key2; + struct ghash_key *guarded_key = + (struct ghash_key *)&test_buf[TEST_BUF_LEN - sizeof(key1)]; + + /* Prepare with regular buffers. */ + rand_bytes(raw_key, sizeof(raw_key)); + ghash_preparekey(&key1, raw_key); + + /* Prepare with guarded raw_key, then check that it works. */ + memcpy(guarded_raw_key, raw_key, sizeof(raw_key)); + ghash_preparekey(&key2, guarded_raw_key); + check_key_consistency(test, &key1, &key2); + + /* Prepare guarded ghash_key, then check that it works. */ + ghash_preparekey(guarded_key, raw_key); + check_key_consistency(test, &key1, guarded_key); +} + +/* + * Test that ghash_key only needs to be aligned to + * __alignof__(struct ghash_key), i.e. 8 bytes. The assembly code may prefer + * 16-byte or higher alignment, but it mustn't require it. + */ +static void test_ghash_with_minimally_aligned_key(struct kunit *test) +{ + u8 raw_key[GHASH_BLOCK_SIZE]; + struct ghash_key key; + struct ghash_key *minaligned_key = + (struct ghash_key *)&test_buf[MAX_LEN_FOR_KEY_CHECK + + __alignof__(struct ghash_key)]; + + KUNIT_ASSERT_TRUE(test, IS_ALIGNED((uintptr_t)minaligned_key, + __alignof__(struct ghash_key))); + KUNIT_ASSERT_TRUE(test, !IS_ALIGNED((uintptr_t)minaligned_key, + 2 * __alignof__(struct ghash_key))); + + rand_bytes(raw_key, sizeof(raw_key)); + ghash_preparekey(&key, raw_key); + ghash_preparekey(minaligned_key, raw_key); + check_key_consistency(test, &key, minaligned_key); +} + +struct ghash_irq_test_state { + struct ghash_key expected_key; + u8 raw_key[GHASH_BLOCK_SIZE]; +}; + +static bool ghash_irq_test_func(void *state_) +{ + struct ghash_irq_test_state *state = state_; + struct ghash_key key; + + ghash_preparekey(&key, state->raw_key); + return memcmp(&key, &state->expected_key, sizeof(key)) == 0; +} + +/* + * Test that ghash_preparekey() produces the same output regardless of whether + * FPU or vector registers are usable when it is called. + */ +static void test_ghash_preparekey_in_irqs(struct kunit *test) +{ + struct ghash_irq_test_state state; + + rand_bytes(state.raw_key, sizeof(state.raw_key)); + ghash_preparekey(&state.expected_key, state.raw_key); + kunit_run_irq_test(test, ghash_irq_test_func, 200000, &state); +} + +static int ghash_suite_init(struct kunit_suite *suite) +{ + u8 raw_key[GHASH_BLOCK_SIZE]; + + rand_bytes_seeded_from_len(raw_key, sizeof(raw_key)); + ghash_preparekey(&test_key, raw_key); + return hash_suite_init(suite); +} + +static void ghash_suite_exit(struct kunit_suite *suite) +{ + hash_suite_exit(suite); +} + +static struct kunit_case ghash_test_cases[] = { + HASH_KUNIT_CASES, + KUNIT_CASE(test_ghash_allones_key_and_message), + KUNIT_CASE(test_ghash_with_guarded_key), + KUNIT_CASE(test_ghash_with_minimally_aligned_key), + KUNIT_CASE(test_ghash_preparekey_in_irqs), + KUNIT_CASE(benchmark_hash), + {}, +}; + +static struct kunit_suite ghash_test_suite = { + .name = "ghash", + .test_cases = ghash_test_cases, + .suite_init = ghash_suite_init, + .suite_exit = ghash_suite_exit, +}; +kunit_test_suite(ghash_test_suite); + +MODULE_DESCRIPTION("KUnit tests and benchmark for GHASH"); +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/tests/polyval_kunit.c b/lib/crypto/tests/polyval_kunit.c index f47f41a39a41..d1f53a690ab8 100644 --- a/lib/crypto/tests/polyval_kunit.c +++ b/lib/crypto/tests/polyval_kunit.c @@ -2,7 +2,7 @@ /* * Copyright 2025 Google LLC */ -#include <crypto/polyval.h> +#include <crypto/gf128hash.h> #include "polyval-testvecs.h" /* diff --git a/lib/crypto/tests/sm3-testvecs.h b/lib/crypto/tests/sm3-testvecs.h new file mode 100644 index 000000000000..5e788c29f487 --- /dev/null +++ b/lib/crypto/tests/sm3-testvecs.h @@ -0,0 +1,231 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py sm3 */ + +static const struct { + size_t data_len; + u8 digest[SM3_DIGEST_SIZE]; +} hash_testvecs[] = { + { + .data_len = 0, + .digest = { + 0x1a, 0xb2, 0x1d, 0x83, 0x55, 0xcf, 0xa1, 0x7f, + 0x8e, 0x61, 0x19, 0x48, 0x31, 0xe8, 0x1a, 0x8f, + 0x22, 0xbe, 0xc8, 0xc7, 0x28, 0xfe, 0xfb, 0x74, + 0x7e, 0xd0, 0x35, 0xeb, 0x50, 0x82, 0xaa, 0x2b, + }, + }, + { + .data_len = 1, + .digest = { + 0xb6, 0x22, 0x2c, 0x39, 0xdc, 0x14, 0x9a, 0xee, + 0x01, 0x9a, 0xcb, 0x0d, 0xe6, 0xc6, 0x75, 0x6e, + 0x8f, 0x18, 0x7b, 0x0e, 0xe8, 0x98, 0x61, 0x71, + 0x2b, 0xd8, 0x38, 0xa9, 0xee, 0x2c, 0x1e, 0x93, + }, + }, + { + .data_len = 2, + .digest = { + 0x62, 0x0c, 0x66, 0x77, 0x67, 0x28, 0x74, 0x8a, + 0xe3, 0x64, 0xea, 0x44, 0x6a, 0x3f, 0x34, 0x61, + 0x55, 0xc5, 0xaa, 0xb2, 0x6c, 0x67, 0x97, 0x68, + 0x68, 0xae, 0x4d, 0x64, 0xa8, 0xb6, 0x72, 0x3e, + }, + }, + { + .data_len = 3, + .digest = { + 0x71, 0xd4, 0x63, 0xb1, 0xfa, 0x27, 0xc7, 0xae, + 0x65, 0xed, 0x5c, 0x93, 0x70, 0xe0, 0x09, 0x34, + 0x2f, 0x42, 0xe6, 0x71, 0x16, 0x8e, 0x90, 0x90, + 0x9a, 0xdd, 0xa6, 0x44, 0x66, 0x71, 0x18, 0xf9, + }, + }, + { + .data_len = 16, + .digest = { + 0x79, 0x0b, 0x68, 0xb5, 0x41, 0xc1, 0x97, 0xa0, + 0x50, 0xe6, 0x93, 0x70, 0xf6, 0x98, 0x49, 0xea, + 0x92, 0xc9, 0xd0, 0xb1, 0x46, 0xbd, 0x4a, 0x0c, + 0x8e, 0xe8, 0xf3, 0xe4, 0x8f, 0x90, 0x33, 0x3c, + }, + }, + { + .data_len = 32, + .digest = { + 0x32, 0x9f, 0xa3, 0x18, 0x18, 0x45, 0xe0, 0x28, + 0xd3, 0xa4, 0x41, 0x3a, 0x25, 0x62, 0x9c, 0x95, + 0xab, 0xfe, 0x02, 0xe0, 0x37, 0x7d, 0x3c, 0xc4, + 0xce, 0x69, 0x57, 0x5a, 0x07, 0x0e, 0xb9, 0xf5, + }, + }, + { + .data_len = 48, + .digest = { + 0x0c, 0xcf, 0x7c, 0x48, 0x44, 0xa0, 0xb0, 0x8d, + 0xdf, 0xbe, 0x22, 0x14, 0x7e, 0xd4, 0xc3, 0x8d, + 0x6a, 0x23, 0xfc, 0x44, 0x0e, 0x0f, 0xde, 0xa5, + 0x7c, 0x8b, 0xc4, 0x8b, 0xab, 0x8c, 0x87, 0x41, + }, + }, + { + .data_len = 49, + .digest = { + 0xb3, 0x76, 0x8b, 0x19, 0xf9, 0x10, 0xa9, 0x56, + 0x4f, 0xce, 0x27, 0xaa, 0x65, 0x96, 0xe5, 0xdb, + 0x90, 0x9b, 0x92, 0xcd, 0x32, 0x0d, 0x16, 0xac, + 0xf8, 0xd0, 0x66, 0x62, 0x10, 0xf0, 0x44, 0xdf, + }, + }, + { + .data_len = 63, + .digest = { + 0x07, 0xc9, 0x45, 0x65, 0x9f, 0x68, 0x75, 0xc3, + 0x74, 0xb2, 0x3b, 0x0c, 0x97, 0x05, 0xd3, 0x13, + 0xc0, 0xb6, 0x21, 0xed, 0xf6, 0x10, 0x7a, 0xed, + 0xec, 0xd8, 0x10, 0x29, 0xbf, 0x7a, 0x78, 0x37, + }, + }, + { + .data_len = 64, + .digest = { + 0x3e, 0x69, 0x18, 0x45, 0xd8, 0x25, 0x6f, 0x44, + 0xc0, 0x02, 0xe5, 0xcf, 0xcd, 0x94, 0x42, 0xa9, + 0xd5, 0x12, 0x62, 0x10, 0x15, 0xa0, 0xf9, 0x16, + 0x19, 0x2d, 0x8d, 0x63, 0x31, 0xf2, 0x2f, 0x36, + }, + }, + { + .data_len = 65, + .digest = { + 0x6b, 0x3e, 0xc0, 0x20, 0xb7, 0x74, 0x30, 0xa0, + 0xc6, 0x5c, 0xee, 0xbe, 0xdc, 0xe6, 0xe5, 0x4f, + 0x3c, 0x61, 0x8d, 0x91, 0xac, 0x31, 0x4b, 0x2a, + 0xdf, 0x1c, 0xef, 0x24, 0xdc, 0x0a, 0x10, 0xe8, + }, + }, + { + .data_len = 127, + .digest = { + 0xab, 0xd6, 0xa1, 0xbf, 0x39, 0x43, 0x75, 0xda, + 0xbf, 0xc7, 0x22, 0xcc, 0x4e, 0xfc, 0xe4, 0x42, + 0x6d, 0x1b, 0x87, 0x25, 0x64, 0x7f, 0x88, 0xf7, + 0xc3, 0x0a, 0x0a, 0x4c, 0xd6, 0xa7, 0x68, 0xae, + }, + }, + { + .data_len = 128, + .digest = { + 0x1b, 0x70, 0xd4, 0x5f, 0x6c, 0xe4, 0x2d, 0x58, + 0x2d, 0x0f, 0x9a, 0x12, 0x34, 0xbb, 0x5e, 0x38, + 0xd8, 0x1f, 0x6a, 0x46, 0x8a, 0xef, 0xdb, 0x68, + 0x18, 0x62, 0xbb, 0x85, 0xfc, 0xc4, 0x6e, 0x2e, + }, + }, + { + .data_len = 129, + .digest = { + 0x33, 0x62, 0xba, 0xa7, 0x4a, 0xbc, 0xd7, 0x7b, + 0xd4, 0x67, 0x6d, 0x3e, 0xea, 0xe8, 0xb0, 0x64, + 0x0d, 0xf3, 0xae, 0x1d, 0x52, 0x24, 0x11, 0x9f, + 0xda, 0xa9, 0x7f, 0xd5, 0x22, 0x1a, 0xde, 0x8a, + }, + }, + { + .data_len = 256, + .digest = { + 0x70, 0xa8, 0xa6, 0x2b, 0xfb, 0x1f, 0x3b, 0x5a, + 0xcc, 0x71, 0x76, 0x9e, 0x25, 0x4c, 0xfa, 0x8f, + 0x39, 0x4a, 0x21, 0x8a, 0x9d, 0x74, 0x8d, 0x2c, + 0x31, 0xa5, 0xb5, 0xff, 0x30, 0xc1, 0x14, 0xc4, + }, + }, + { + .data_len = 511, + .digest = { + 0x39, 0xd0, 0x8c, 0x5f, 0xfc, 0x36, 0xc2, 0x7c, + 0xdb, 0x8b, 0x2e, 0xdc, 0x9d, 0x1b, 0xd1, 0xba, + 0x9b, 0x52, 0x6b, 0x35, 0x46, 0x46, 0x75, 0x73, + 0xe5, 0x62, 0x96, 0x6e, 0xf3, 0xba, 0xd9, 0x19, + }, + }, + { + .data_len = 513, + .digest = { + 0x76, 0xa0, 0x3d, 0xa2, 0x5f, 0xd4, 0xa6, 0xbe, + 0x6b, 0xdb, 0xed, 0x14, 0x9e, 0xa8, 0x15, 0x77, + 0xa9, 0x38, 0x30, 0x6b, 0x68, 0xfa, 0xb6, 0xe2, + 0x93, 0x19, 0x24, 0x72, 0x67, 0x20, 0x72, 0xc3, + }, + }, + { + .data_len = 1000, + .digest = { + 0x16, 0xbc, 0x33, 0x77, 0x0b, 0xcf, 0x93, 0x5e, + 0xec, 0x7d, 0x8d, 0x3c, 0xae, 0xd9, 0x75, 0xdf, + 0x46, 0x24, 0x17, 0x7e, 0x03, 0x88, 0xf2, 0x75, + 0xa9, 0x18, 0xa6, 0x1c, 0x7a, 0x74, 0x0d, 0xf3, + }, + }, + { + .data_len = 3333, + .digest = { + 0xdb, 0x54, 0x89, 0xe7, 0x1c, 0x50, 0xf2, 0xbf, + 0xde, 0x3a, 0xbf, 0x5b, 0xee, 0x5a, 0x46, 0x62, + 0x20, 0xb1, 0x80, 0x48, 0xac, 0x56, 0x33, 0xb3, + 0xbb, 0x3f, 0xfa, 0x02, 0xc6, 0x43, 0xb5, 0x8c, + }, + }, + { + .data_len = 4096, + .digest = { + 0xdf, 0x0d, 0xed, 0x3b, 0x8f, 0xea, 0x81, 0xfd, + 0xd6, 0x34, 0xae, 0x74, 0x24, 0x3a, 0x15, 0x38, + 0xe7, 0xcf, 0x45, 0x7e, 0x8f, 0xf5, 0x50, 0x6c, + 0xaa, 0x27, 0x23, 0x92, 0x6d, 0xab, 0x3b, 0xde, + }, + }, + { + .data_len = 4128, + .digest = { + 0x6a, 0xbd, 0x56, 0x5a, 0xf1, 0xc6, 0x40, 0x4d, + 0xf3, 0x50, 0x77, 0x87, 0x86, 0x63, 0x1b, 0x4d, + 0x21, 0x99, 0x96, 0xad, 0x24, 0x62, 0xce, 0xc0, + 0x3e, 0xb7, 0x35, 0x52, 0x56, 0x0e, 0x55, 0x85, + }, + }, + { + .data_len = 4160, + .digest = { + 0x5b, 0xc1, 0x1f, 0x25, 0x43, 0xa3, 0x1c, 0xa0, + 0x8c, 0xfc, 0x41, 0xf1, 0xcc, 0xb3, 0x95, 0x95, + 0xe0, 0xb9, 0xd3, 0x29, 0xf4, 0x08, 0x31, 0x47, + 0x6d, 0x09, 0xa8, 0x2e, 0xa5, 0xf4, 0xf1, 0x8d, + }, + }, + { + .data_len = 4224, + .digest = { + 0xec, 0x56, 0x1e, 0xa6, 0x1f, 0xb2, 0x87, 0xb2, + 0x7e, 0x15, 0xd6, 0x30, 0x08, 0x74, 0xb0, 0x48, + 0x90, 0x2a, 0xbe, 0x2f, 0x80, 0x3a, 0x88, 0xcc, + 0xd7, 0xc5, 0x87, 0x8c, 0x04, 0xef, 0x78, 0x71, + }, + }, + { + .data_len = 16384, + .digest = { + 0xe7, 0xb8, 0x84, 0x20, 0xff, 0xd5, 0x53, 0xe6, + 0x14, 0x31, 0x12, 0x75, 0xb7, 0x9a, 0x4f, 0x63, + 0x63, 0x00, 0xfe, 0x2c, 0x54, 0xee, 0x06, 0xfc, + 0x12, 0x16, 0xe5, 0xdc, 0xa4, 0x40, 0x62, 0x12, + }, + }, +}; + +static const u8 hash_testvec_consolidated[SM3_DIGEST_SIZE] = { + 0xe6, 0x58, 0xd4, 0x8e, 0x74, 0x92, 0xdf, 0xfe, + 0x58, 0x05, 0xe5, 0x29, 0x83, 0xfb, 0xb7, 0x51, + 0x7e, 0x66, 0x0c, 0x49, 0x3e, 0x11, 0x7e, 0x9b, + 0xb1, 0x83, 0x3a, 0xa6, 0xb0, 0x3c, 0xf5, 0xe0, +}; diff --git a/lib/crypto/tests/sm3_kunit.c b/lib/crypto/tests/sm3_kunit.c new file mode 100644 index 000000000000..dc8136acdff6 --- /dev/null +++ b/lib/crypto/tests/sm3_kunit.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2026 Google LLC + */ +#include <crypto/sm3.h> +#include "sm3-testvecs.h" + +#define HASH sm3 +#define HASH_CTX sm3_ctx +#define HASH_SIZE SM3_DIGEST_SIZE +#define HASH_INIT sm3_init +#define HASH_UPDATE sm3_update +#define HASH_FINAL sm3_final +#include "hash-test-template.h" + +static struct kunit_case sm3_test_cases[] = { + HASH_KUNIT_CASES, + KUNIT_CASE(benchmark_hash), + {}, +}; + +static struct kunit_suite sm3_test_suite = { + .name = "sm3", + .test_cases = sm3_test_cases, + .suite_init = hash_suite_init, + .suite_exit = hash_suite_exit, +}; +kunit_test_suite(sm3_test_suite); + +MODULE_DESCRIPTION("KUnit tests and benchmark for SM3"); +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/x86/polyval.h b/lib/crypto/x86/gf128hash.h index ef8797521420..6b79b06caab0 100644 --- a/lib/crypto/x86/polyval.h +++ b/lib/crypto/x86/gf128hash.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * POLYVAL library functions, x86_64 optimized + * GHASH and POLYVAL, x86_64 optimized * * Copyright 2025 Google LLC */ @@ -9,14 +9,22 @@ #define NUM_H_POWERS 8 +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul_avx); +asmlinkage void polyval_mul_pclmul(struct polyval_elem *a, + const struct polyval_elem *b); asmlinkage void polyval_mul_pclmul_avx(struct polyval_elem *a, const struct polyval_elem *b); + +asmlinkage void ghash_blocks_pclmul(struct polyval_elem *acc, + const struct polyval_elem *key, + const u8 *data, size_t nblocks); asmlinkage void polyval_blocks_pclmul_avx(struct polyval_elem *acc, const struct polyval_key *key, const u8 *data, size_t nblocks); +#define polyval_preparekey_arch polyval_preparekey_arch static void polyval_preparekey_arch(struct polyval_key *key, const u8 raw_key[POLYVAL_BLOCK_SIZE]) { @@ -40,18 +48,58 @@ static void polyval_preparekey_arch(struct polyval_key *key, } } -static void polyval_mul_arch(struct polyval_elem *acc, - const struct polyval_key *key) +static void polyval_mul_x86(struct polyval_elem *a, + const struct polyval_elem *b) { - if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) { + if (static_branch_likely(&have_pclmul) && irq_fpu_usable()) { kernel_fpu_begin(); - polyval_mul_pclmul_avx(acc, &key->h_powers[NUM_H_POWERS - 1]); + if (static_branch_likely(&have_pclmul_avx)) + polyval_mul_pclmul_avx(a, b); + else + polyval_mul_pclmul(a, b); kernel_fpu_end(); } else { - polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]); + polyval_mul_generic(a, b); + } +} + +#define ghash_mul_arch ghash_mul_arch +static void ghash_mul_arch(struct polyval_elem *acc, + const struct ghash_key *key) +{ + polyval_mul_x86(acc, &key->h); +} + +#define polyval_mul_arch polyval_mul_arch +static void polyval_mul_arch(struct polyval_elem *acc, + const struct polyval_key *key) +{ + polyval_mul_x86(acc, &key->h_powers[NUM_H_POWERS - 1]); +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_pclmul) && irq_fpu_usable()) { + do { + /* Allow rescheduling every 4 KiB. */ + size_t n = min_t(size_t, nblocks, + 4096 / GHASH_BLOCK_SIZE); + + kernel_fpu_begin(); + ghash_blocks_pclmul(acc, &key->h, data, n); + kernel_fpu_end(); + data += n * GHASH_BLOCK_SIZE; + nblocks -= n; + } while (nblocks); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); } } +#define polyval_blocks_arch polyval_blocks_arch static void polyval_blocks_arch(struct polyval_elem *acc, const struct polyval_key *key, const u8 *data, size_t nblocks) @@ -74,10 +122,12 @@ static void polyval_blocks_arch(struct polyval_elem *acc, } } -#define polyval_mod_init_arch polyval_mod_init_arch -static void polyval_mod_init_arch(void) +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) { - if (boot_cpu_has(X86_FEATURE_PCLMULQDQ) && - boot_cpu_has(X86_FEATURE_AVX)) - static_branch_enable(&have_pclmul_avx); + if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) { + static_branch_enable(&have_pclmul); + if (boot_cpu_has(X86_FEATURE_AVX)) + static_branch_enable(&have_pclmul_avx); + } } diff --git a/arch/x86/crypto/ghash-clmulni-intel_asm.S b/lib/crypto/x86/ghash-pclmul.S index c4fbaa82ed7a..6ffb5aea6063 100644 --- a/arch/x86/crypto/ghash-clmulni-intel_asm.S +++ b/lib/crypto/x86/ghash-pclmul.S @@ -21,8 +21,8 @@ .Lbswap_mask: .octa 0x000102030405060708090a0b0c0d0e0f -#define DATA %xmm0 -#define SHASH %xmm1 +#define ACC %xmm0 +#define KEY %xmm1 #define T1 %xmm2 #define T2 %xmm3 #define T3 %xmm4 @@ -34,100 +34,94 @@ /* * __clmul_gf128mul_ble: internal ABI * input: - * DATA: operand1 - * SHASH: operand2, hash_key << 1 mod poly + * ACC: operand1 + * KEY: operand2, hash_key << 1 mod poly * output: - * DATA: operand1 * operand2 mod poly + * ACC: operand1 * operand2 mod poly * changed: * T1 * T2 * T3 */ SYM_FUNC_START_LOCAL(__clmul_gf128mul_ble) - movaps DATA, T1 - pshufd $0b01001110, DATA, T2 - pshufd $0b01001110, SHASH, T3 - pxor DATA, T2 - pxor SHASH, T3 + movaps ACC, T1 + pshufd $0b01001110, ACC, T2 + pshufd $0b01001110, KEY, T3 + pxor ACC, T2 + pxor KEY, T3 - pclmulqdq $0x00, SHASH, DATA # DATA = a0 * b0 - pclmulqdq $0x11, SHASH, T1 # T1 = a1 * b1 + pclmulqdq $0x00, KEY, ACC # ACC = a0 * b0 + pclmulqdq $0x11, KEY, T1 # T1 = a1 * b1 pclmulqdq $0x00, T3, T2 # T2 = (a1 + a0) * (b1 + b0) - pxor DATA, T2 + pxor ACC, T2 pxor T1, T2 # T2 = a0 * b1 + a1 * b0 movaps T2, T3 pslldq $8, T3 psrldq $8, T2 - pxor T3, DATA - pxor T2, T1 # <T1:DATA> is result of + pxor T3, ACC + pxor T2, T1 # <T1:ACC> is result of # carry-less multiplication # first phase of the reduction - movaps DATA, T3 + movaps ACC, T3 psllq $1, T3 - pxor DATA, T3 + pxor ACC, T3 psllq $5, T3 - pxor DATA, T3 + pxor ACC, T3 psllq $57, T3 movaps T3, T2 pslldq $8, T2 psrldq $8, T3 - pxor T2, DATA + pxor T2, ACC pxor T3, T1 # second phase of the reduction - movaps DATA, T2 + movaps ACC, T2 psrlq $5, T2 - pxor DATA, T2 + pxor ACC, T2 psrlq $1, T2 - pxor DATA, T2 + pxor ACC, T2 psrlq $1, T2 pxor T2, T1 - pxor T1, DATA + pxor T1, ACC RET SYM_FUNC_END(__clmul_gf128mul_ble) -/* void clmul_ghash_mul(char *dst, const le128 *shash) */ -SYM_FUNC_START(clmul_ghash_mul) +/* + * void polyval_mul_pclmul(struct polyval_elem *a, + * const struct polyval_elem *b) + */ +SYM_FUNC_START(polyval_mul_pclmul) FRAME_BEGIN - movups (%rdi), DATA - movups (%rsi), SHASH - movaps .Lbswap_mask(%rip), BSWAP - pshufb BSWAP, DATA + movups (%rdi), ACC + movups (%rsi), KEY call __clmul_gf128mul_ble - pshufb BSWAP, DATA - movups DATA, (%rdi) + movups ACC, (%rdi) FRAME_END RET -SYM_FUNC_END(clmul_ghash_mul) +SYM_FUNC_END(polyval_mul_pclmul) /* - * int clmul_ghash_update(char *dst, const char *src, unsigned int srclen, - * const le128 *shash); + * void ghash_blocks_pclmul(struct polyval_elem *acc, + * const struct polyval_elem *key, + * const u8 *data, size_t nblocks) */ -SYM_FUNC_START(clmul_ghash_update) +SYM_FUNC_START(ghash_blocks_pclmul) FRAME_BEGIN - cmp $16, %rdx - jb .Lupdate_just_ret # check length movaps .Lbswap_mask(%rip), BSWAP - movups (%rdi), DATA - movups (%rcx), SHASH - pshufb BSWAP, DATA + movups (%rdi), ACC + movups (%rsi), KEY .align 4 -.Lupdate_loop: - movups (%rsi), IN1 +.Lnext_block: + movups (%rdx), IN1 pshufb BSWAP, IN1 - pxor IN1, DATA + pxor IN1, ACC call __clmul_gf128mul_ble - sub $16, %rdx - add $16, %rsi - cmp $16, %rdx - jge .Lupdate_loop - pshufb BSWAP, DATA - movups DATA, (%rdi) -.Lupdate_just_ret: - mov %rdx, %rax + add $16, %rdx + dec %rcx + jnz .Lnext_block + movups ACC, (%rdi) FRAME_END RET -SYM_FUNC_END(clmul_ghash_update) +SYM_FUNC_END(ghash_blocks_pclmul) diff --git a/lib/crypto/x86/sha256.h b/lib/crypto/x86/sha256.h index 38e33b22a092..0ee69d8e39fe 100644 --- a/lib/crypto/x86/sha256.h +++ b/lib/crypto/x86/sha256.h @@ -31,6 +31,27 @@ DEFINE_X86_SHA256_FN(sha256_blocks_avx, sha256_transform_avx); DEFINE_X86_SHA256_FN(sha256_blocks_avx2, sha256_transform_rorx); DEFINE_X86_SHA256_FN(sha256_blocks_ni, sha256_ni_transform); +#define PHE_ALIGNMENT 16 +static void sha256_blocks_phe(struct sha256_block_state *state, + const u8 *data, size_t nblocks) +{ + /* + * On Zhaoxin processors, XSHA256 requires the %rdi register + * in 64-bit mode (or %edi in 32-bit mode) to point to + * a 32-byte, 16-byte-aligned buffer. + */ + u8 buf[32 + PHE_ALIGNMENT - 1]; + u8 *dst = PTR_ALIGN(&buf[0], PHE_ALIGNMENT); + size_t padding = -1; + + memcpy(dst, state, SHA256_DIGEST_SIZE); + asm volatile(".byte 0xf3,0x0f,0xa6,0xd0" /* REP XSHA256 */ + : "+a"(padding), "+c"(nblocks), "+S"(data) + : "D"(dst) + : "memory"); + memcpy(state, dst, SHA256_DIGEST_SIZE); +} + static void sha256_blocks(struct sha256_block_state *state, const u8 *data, size_t nblocks) { @@ -79,6 +100,10 @@ static void sha256_mod_init_arch(void) if (boot_cpu_has(X86_FEATURE_SHA_NI)) { static_call_update(sha256_blocks_x86, sha256_blocks_ni); static_branch_enable(&have_sha_ni); + } else if (IS_ENABLED(CONFIG_CPU_SUP_ZHAOXIN) && + boot_cpu_has(X86_FEATURE_PHE_EN) && + boot_cpu_data.x86 >= 0x07) { + static_call_update(sha256_blocks_x86, sha256_blocks_phe); } else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL) && boot_cpu_has(X86_FEATURE_AVX)) { diff --git a/arch/x86/crypto/sm3-avx-asm_64.S b/lib/crypto/x86/sm3-avx-asm_64.S index 503bab450a91..a1925b136010 100644 --- a/arch/x86/crypto/sm3-avx-asm_64.S +++ b/lib/crypto/x86/sm3-avx-asm_64.S @@ -12,10 +12,9 @@ */ #include <linux/linkage.h> -#include <linux/cfi_types.h> #include <asm/frame.h> -/* Context structure */ +/* State structure */ #define state_h0 0 #define state_h1 4 @@ -325,13 +324,13 @@ /* * Transform nblocks*64 bytes (nblocks*16 32-bit words) at DATA. * - * void sm3_transform_avx(struct sm3_state *state, - * const u8 *data, int nblocks); + * void sm3_transform_avx(struct sm3_block_state *state, + * const u8 *data, size_t nblocks); */ -SYM_TYPED_FUNC_START(sm3_transform_avx) +SYM_FUNC_START(sm3_transform_avx) /* input: - * %rdi: ctx, CTX - * %rsi: data (64*nblks bytes) + * %rdi: state + * %rsi: data * %rdx: nblocks */ vzeroupper; diff --git a/lib/crypto/x86/sm3.h b/lib/crypto/x86/sm3.h new file mode 100644 index 000000000000..3834780f2f6a --- /dev/null +++ b/lib/crypto/x86/sm3.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM3 optimized for x86_64 + * + * Copyright 2026 Google LLC + */ +#include <asm/fpu/api.h> +#include <linux/static_call.h> + +asmlinkage void sm3_transform_avx(struct sm3_block_state *state, + const u8 *data, size_t nblocks); + +static void sm3_blocks_avx(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + if (likely(irq_fpu_usable())) { + kernel_fpu_begin(); + sm3_transform_avx(state, data, nblocks); + kernel_fpu_end(); + } else { + sm3_blocks_generic(state, data, nblocks); + } +} + +DEFINE_STATIC_CALL(sm3_blocks_x86, sm3_blocks_generic); + +static void sm3_blocks(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + static_call(sm3_blocks_x86)(state, data, nblocks); +} + +#define sm3_mod_init_arch sm3_mod_init_arch +static void sm3_mod_init_arch(void) +{ + if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_BMI2) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) + static_call_update(sm3_blocks_x86, sm3_blocks_avx); +} diff --git a/net/mac80211/Kconfig b/net/mac80211/Kconfig index cf0f7780fb10..d6bc295e23a1 100644 --- a/net/mac80211/Kconfig +++ b/net/mac80211/Kconfig @@ -3,11 +3,11 @@ config MAC80211 tristate "Generic IEEE 802.11 Networking Stack (mac80211)" depends on CFG80211 select CRYPTO + select CRYPTO_LIB_AES_CBC_MACS select CRYPTO_LIB_ARC4 select CRYPTO_AES select CRYPTO_CCM select CRYPTO_GCM - select CRYPTO_CMAC select CRC32 help This option enables the hardware independent IEEE 802.11 diff --git a/net/mac80211/aes_cmac.c b/net/mac80211/aes_cmac.c index 0827965455dc..55b674ad7d7a 100644 --- a/net/mac80211/aes_cmac.c +++ b/net/mac80211/aes_cmac.c @@ -7,10 +7,9 @@ #include <linux/kernel.h> #include <linux/types.h> -#include <linux/crypto.h> #include <linux/export.h> #include <linux/err.h> -#include <crypto/aes.h> +#include <crypto/aes-cbc-macs.h> #include <net/mac80211.h> #include "key.h" @@ -20,65 +19,25 @@ static const u8 zero[IEEE80211_CMAC_256_MIC_LEN]; -int ieee80211_aes_cmac(struct crypto_shash *tfm, const u8 *aad, - const u8 *data, size_t data_len, u8 *mic, - unsigned int mic_len) +void ieee80211_aes_cmac(const struct aes_cmac_key *key, const u8 *aad, + const u8 *data, size_t data_len, u8 *mic, + unsigned int mic_len) { - int err; - SHASH_DESC_ON_STACK(desc, tfm); + struct aes_cmac_ctx ctx; u8 out[AES_BLOCK_SIZE]; const __le16 *fc; - desc->tfm = tfm; - - err = crypto_shash_init(desc); - if (err) - return err; - err = crypto_shash_update(desc, aad, AAD_LEN); - if (err) - return err; + aes_cmac_init(&ctx, key); + aes_cmac_update(&ctx, aad, AAD_LEN); fc = (const __le16 *)aad; if (ieee80211_is_beacon(*fc)) { /* mask Timestamp field to zero */ - err = crypto_shash_update(desc, zero, 8); - if (err) - return err; - err = crypto_shash_update(desc, data + 8, - data_len - 8 - mic_len); - if (err) - return err; + aes_cmac_update(&ctx, zero, 8); + aes_cmac_update(&ctx, data + 8, data_len - 8 - mic_len); } else { - err = crypto_shash_update(desc, data, data_len - mic_len); - if (err) - return err; + aes_cmac_update(&ctx, data, data_len - mic_len); } - err = crypto_shash_finup(desc, zero, mic_len, out); - if (err) - return err; + aes_cmac_update(&ctx, zero, mic_len); + aes_cmac_final(&ctx, out); memcpy(mic, out, mic_len); - - return 0; -} - -struct crypto_shash *ieee80211_aes_cmac_key_setup(const u8 key[], - size_t key_len) -{ - struct crypto_shash *tfm; - - tfm = crypto_alloc_shash("cmac(aes)", 0, 0); - if (!IS_ERR(tfm)) { - int err = crypto_shash_setkey(tfm, key, key_len); - - if (err) { - crypto_free_shash(tfm); - return ERR_PTR(err); - } - } - - return tfm; -} - -void ieee80211_aes_cmac_key_free(struct crypto_shash *tfm) -{ - crypto_free_shash(tfm); } diff --git a/net/mac80211/aes_cmac.h b/net/mac80211/aes_cmac.h index 5f971a8298cb..c7a6df47b327 100644 --- a/net/mac80211/aes_cmac.h +++ b/net/mac80211/aes_cmac.h @@ -6,14 +6,10 @@ #ifndef AES_CMAC_H #define AES_CMAC_H -#include <linux/crypto.h> -#include <crypto/hash.h> +#include <crypto/aes-cbc-macs.h> -struct crypto_shash *ieee80211_aes_cmac_key_setup(const u8 key[], - size_t key_len); -int ieee80211_aes_cmac(struct crypto_shash *tfm, const u8 *aad, - const u8 *data, size_t data_len, u8 *mic, - unsigned int mic_len); -void ieee80211_aes_cmac_key_free(struct crypto_shash *tfm); +void ieee80211_aes_cmac(const struct aes_cmac_key *key, const u8 *aad, + const u8 *data, size_t data_len, u8 *mic, + unsigned int mic_len); #endif /* AES_CMAC_H */ diff --git a/net/mac80211/fils_aead.c b/net/mac80211/fils_aead.c index 912c46f74d24..d2f4a17eab99 100644 --- a/net/mac80211/fils_aead.c +++ b/net/mac80211/fils_aead.c @@ -4,13 +4,11 @@ * Copyright 2016, Qualcomm Atheros, Inc. */ -#include <crypto/aes.h> -#include <crypto/hash.h> +#include <crypto/aes-cbc-macs.h> #include <crypto/skcipher.h> #include <crypto/utils.h> #include "ieee80211_i.h" -#include "aes_cmac.h" #include "fils_aead.h" static void gf_mulx(u8 *pad) @@ -22,31 +20,35 @@ static void gf_mulx(u8 *pad) put_unaligned_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0), pad + 8); } -static int aes_s2v(struct crypto_shash *tfm, +static int aes_s2v(const u8 *in_key, size_t key_len, size_t num_elem, const u8 *addr[], size_t len[], u8 *v) { u8 d[AES_BLOCK_SIZE], tmp[AES_BLOCK_SIZE] = {}; - SHASH_DESC_ON_STACK(desc, tfm); + struct aes_cmac_key key; + struct aes_cmac_ctx ctx; size_t i; + int res; - desc->tfm = tfm; + res = aes_cmac_preparekey(&key, in_key, key_len); + if (res) + return res; /* D = AES-CMAC(K, <zero>) */ - crypto_shash_digest(desc, tmp, AES_BLOCK_SIZE, d); + aes_cmac(&key, tmp, AES_BLOCK_SIZE, d); for (i = 0; i < num_elem - 1; i++) { /* D = dbl(D) xor AES_CMAC(K, Si) */ gf_mulx(d); /* dbl */ - crypto_shash_digest(desc, addr[i], len[i], tmp); + aes_cmac(&key, addr[i], len[i], tmp); crypto_xor(d, tmp, AES_BLOCK_SIZE); } - crypto_shash_init(desc); + aes_cmac_init(&ctx, &key); if (len[i] >= AES_BLOCK_SIZE) { /* len(Sn) >= 128 */ /* T = Sn xorend D */ - crypto_shash_update(desc, addr[i], len[i] - AES_BLOCK_SIZE); + aes_cmac_update(&ctx, addr[i], len[i] - AES_BLOCK_SIZE); crypto_xor(d, addr[i] + len[i] - AES_BLOCK_SIZE, AES_BLOCK_SIZE); } else { @@ -57,8 +59,10 @@ static int aes_s2v(struct crypto_shash *tfm, d[len[i]] ^= 0x80; } /* V = AES-CMAC(K, T) */ - crypto_shash_finup(desc, d, AES_BLOCK_SIZE, v); + aes_cmac_update(&ctx, d, AES_BLOCK_SIZE); + aes_cmac_final(&ctx, v); + memzero_explicit(&key, sizeof(key)); return 0; } @@ -69,7 +73,6 @@ static int aes_siv_encrypt(const u8 *key, size_t key_len, size_t len[], u8 *out) { u8 v[AES_BLOCK_SIZE]; - struct crypto_shash *tfm; struct crypto_skcipher *tfm2; struct skcipher_request *req; int res; @@ -83,15 +86,7 @@ static int aes_siv_encrypt(const u8 *key, size_t key_len, num_elem++; /* S2V */ - - tfm = crypto_alloc_shash("cmac(aes)", 0, 0); - if (IS_ERR(tfm)) - return PTR_ERR(tfm); - /* K1 for S2V */ - res = crypto_shash_setkey(tfm, key, key_len); - if (!res) - res = aes_s2v(tfm, num_elem, addr, len, v); - crypto_free_shash(tfm); + res = aes_s2v(key /* K1 */, key_len, num_elem, addr, len, v); if (res) return res; @@ -146,7 +141,6 @@ static int aes_siv_decrypt(const u8 *key, size_t key_len, size_t num_elem, const u8 *addr[], size_t len[], u8 *out) { - struct crypto_shash *tfm; struct crypto_skcipher *tfm2; struct skcipher_request *req; struct scatterlist src[1], dst[1]; @@ -198,15 +192,7 @@ static int aes_siv_decrypt(const u8 *key, size_t key_len, return res; /* S2V */ - - tfm = crypto_alloc_shash("cmac(aes)", 0, 0); - if (IS_ERR(tfm)) - return PTR_ERR(tfm); - /* K1 for S2V */ - res = crypto_shash_setkey(tfm, key, key_len); - if (!res) - res = aes_s2v(tfm, num_elem, addr, len, check); - crypto_free_shash(tfm); + res = aes_s2v(key /* K1 */, key_len, num_elem, addr, len, check); if (res) return res; if (memcmp(check, frame_iv, AES_BLOCK_SIZE) != 0) diff --git a/net/mac80211/key.c b/net/mac80211/key.c index 04c8809173d7..4b8965633df3 100644 --- a/net/mac80211/key.c +++ b/net/mac80211/key.c @@ -690,10 +690,9 @@ ieee80211_key_alloc(u32 cipher, int idx, size_t key_len, * Initialize AES key state here as an optimization so that * it does not need to be initialized for every packet. */ - key->u.aes_cmac.tfm = - ieee80211_aes_cmac_key_setup(key_data, key_len); - if (IS_ERR(key->u.aes_cmac.tfm)) { - err = PTR_ERR(key->u.aes_cmac.tfm); + err = aes_cmac_preparekey(&key->u.aes_cmac.key, key_data, + key_len); + if (err) { kfree(key); return ERR_PTR(err); } @@ -750,10 +749,6 @@ static void ieee80211_key_free_common(struct ieee80211_key *key) case WLAN_CIPHER_SUITE_CCMP_256: ieee80211_aes_key_free(key->u.ccmp.tfm); break; - case WLAN_CIPHER_SUITE_AES_CMAC: - case WLAN_CIPHER_SUITE_BIP_CMAC_256: - ieee80211_aes_cmac_key_free(key->u.aes_cmac.tfm); - break; case WLAN_CIPHER_SUITE_BIP_GMAC_128: case WLAN_CIPHER_SUITE_BIP_GMAC_256: ieee80211_aes_gmac_key_free(key->u.aes_gmac.tfm); diff --git a/net/mac80211/key.h b/net/mac80211/key.h index 1fa0f4f78962..826e4e9387c5 100644 --- a/net/mac80211/key.h +++ b/net/mac80211/key.h @@ -12,6 +12,7 @@ #include <linux/list.h> #include <linux/crypto.h> #include <linux/rcupdate.h> +#include <crypto/aes-cbc-macs.h> #include <crypto/arc4.h> #include <net/mac80211.h> @@ -93,7 +94,7 @@ struct ieee80211_key { } ccmp; struct { u8 rx_pn[IEEE80211_CMAC_PN_LEN]; - struct crypto_shash *tfm; + struct aes_cmac_key key; u32 replays; /* dot11RSNAStatsCMACReplays */ u32 icverrors; /* dot11RSNAStatsCMACICVErrors */ } aes_cmac; diff --git a/net/mac80211/wpa.c b/net/mac80211/wpa.c index fdf98c21d32c..59324b367bdd 100644 --- a/net/mac80211/wpa.c +++ b/net/mac80211/wpa.c @@ -872,11 +872,8 @@ ieee80211_crypto_aes_cmac_encrypt(struct ieee80211_tx_data *tx, bip_aad(skb, aad); - if (ieee80211_aes_cmac(key->u.aes_cmac.tfm, aad, - skb->data + 24, skb->len - 24, - mmie->mic, mic_len)) - return TX_DROP; - + ieee80211_aes_cmac(&key->u.aes_cmac.key, aad, skb->data + 24, + skb->len - 24, mmie->mic, mic_len); return TX_CONTINUE; } @@ -918,10 +915,8 @@ ieee80211_crypto_aes_cmac_decrypt(struct ieee80211_rx_data *rx, if (!(status->flag & RX_FLAG_DECRYPTED)) { /* hardware didn't decrypt/verify MIC */ bip_aad(skb, aad); - if (ieee80211_aes_cmac(key->u.aes_cmac.tfm, aad, - skb->data + 24, skb->len - 24, - mic, mic_len)) - return RX_DROP_U_DECRYPT_FAIL; + ieee80211_aes_cmac(&key->u.aes_cmac.key, aad, skb->data + 24, + skb->len - 24, mic, mic_len); if (crypto_memneq(mic, mmie->mic, mic_len)) { key->u.aes_cmac.icverrors++; return RX_DROP_U_MIC_FAIL; diff --git a/samples/Kconfig b/samples/Kconfig index 5bc7c9e5a59e..a75e8e78330d 100644 --- a/samples/Kconfig +++ b/samples/Kconfig @@ -186,6 +186,8 @@ config SAMPLE_TIMER config SAMPLE_TSM_MR tristate "TSM measurement sample" + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 select TSM_MEASUREMENTS select VIRT_DRIVERS help diff --git a/samples/tsm-mr/tsm_mr_sample.c b/samples/tsm-mr/tsm_mr_sample.c index a2c652148639..c79dbc1e0456 100644 --- a/samples/tsm-mr/tsm_mr_sample.c +++ b/samples/tsm-mr/tsm_mr_sample.c @@ -6,7 +6,7 @@ #include <linux/module.h> #include <linux/tsm-mr.h> #include <linux/miscdevice.h> -#include <crypto/hash.h> +#include <crypto/sha2.h> static struct { u8 static_mr[SHA384_DIGEST_SIZE]; @@ -23,47 +23,45 @@ static struct { static int sample_report_refresh(const struct tsm_measurements *tm) { - struct crypto_shash *tfm; - int rc; - - tfm = crypto_alloc_shash(hash_algo_name[HASH_ALGO_SHA512], 0, 0); - if (IS_ERR(tfm)) { - pr_err("crypto_alloc_shash failed: %ld\n", PTR_ERR(tfm)); - return PTR_ERR(tfm); - } - - rc = crypto_shash_tfm_digest(tfm, (u8 *)&sample_report, - offsetof(typeof(sample_report), - report_digest), - sample_report.report_digest); - crypto_free_shash(tfm); - if (rc) - pr_err("crypto_shash_tfm_digest failed: %d\n", rc); - return rc; + sha512((const u8 *)&sample_report, + offsetof(typeof(sample_report), report_digest), + sample_report.report_digest); + return 0; } static int sample_report_extend_mr(const struct tsm_measurements *tm, const struct tsm_measurement_register *mr, const u8 *data) { - SHASH_DESC_ON_STACK(desc, 0); - int rc; - - desc->tfm = crypto_alloc_shash(hash_algo_name[mr->mr_hash], 0, 0); - if (IS_ERR(desc->tfm)) { - pr_err("crypto_alloc_shash failed: %ld\n", PTR_ERR(desc->tfm)); - return PTR_ERR(desc->tfm); + union { + struct sha256_ctx sha256; + struct sha384_ctx sha384; + struct sha512_ctx sha512; + } ctx; + + switch (mr->mr_hash) { + case HASH_ALGO_SHA256: + sha256_init(&ctx.sha256); + sha256_update(&ctx.sha256, mr->mr_value, mr->mr_size); + sha256_update(&ctx.sha256, data, mr->mr_size); + sha256_final(&ctx.sha256, mr->mr_value); + return 0; + case HASH_ALGO_SHA384: + sha384_init(&ctx.sha384); + sha384_update(&ctx.sha384, mr->mr_value, mr->mr_size); + sha384_update(&ctx.sha384, data, mr->mr_size); + sha384_final(&ctx.sha384, mr->mr_value); + return 0; + case HASH_ALGO_SHA512: + sha512_init(&ctx.sha512); + sha512_update(&ctx.sha512, mr->mr_value, mr->mr_size); + sha512_update(&ctx.sha512, data, mr->mr_size); + sha512_final(&ctx.sha512, mr->mr_value); + return 0; + default: + pr_err("Unsupported hash algorithm: %d\n", mr->mr_hash); + return -EOPNOTSUPP; } - - rc = crypto_shash_init(desc); - if (!rc) - rc = crypto_shash_update(desc, mr->mr_value, mr->mr_size); - if (!rc) - rc = crypto_shash_finup(desc, data, mr->mr_size, mr->mr_value); - crypto_free_shash(desc->tfm); - if (rc) - pr_err("SHA calculation failed: %d\n", rc); - return rc; } #define MR_(mr, hash) .mr_value = &sample_report.mr, TSM_MR_(mr, hash) diff --git a/scripts/crypto/gen-fips-testvecs.py b/scripts/crypto/gen-fips-testvecs.py index db873f88619a..9f18bcb97412 100755 --- a/scripts/crypto/gen-fips-testvecs.py +++ b/scripts/crypto/gen-fips-testvecs.py @@ -3,8 +3,12 @@ # # Script that generates lib/crypto/fips.h # +# Requires that python-cryptography be installed. +# # Copyright 2025 Google LLC +import cryptography.hazmat.primitives.ciphers +import cryptography.hazmat.primitives.cmac import hashlib import hmac @@ -34,3 +38,9 @@ for alg in 'sha1', 'sha256', 'sha512': print_static_u8_array_definition(f'fips_test_sha3_256_value', hashlib.sha3_256(fips_test_data).digest()) + +aes = cryptography.hazmat.primitives.ciphers.algorithms.AES(fips_test_key) +aes_cmac = cryptography.hazmat.primitives.cmac.CMAC(aes) +aes_cmac.update(fips_test_data) +print_static_u8_array_definition('fips_test_aes_cmac_value', + aes_cmac.finalize()) diff --git a/scripts/crypto/gen-hash-testvecs.py b/scripts/crypto/gen-hash-testvecs.py index 8eeb650fcada..f356f87e1c77 100755 --- a/scripts/crypto/gen-hash-testvecs.py +++ b/scripts/crypto/gen-hash-testvecs.py @@ -3,8 +3,12 @@ # # Script that generates test vectors for the given hash function. # +# Requires that python-cryptography be installed. +# # Copyright 2025 Google LLC +import cryptography.hazmat.primitives.ciphers +import cryptography.hazmat.primitives.cmac import hashlib import hmac import sys @@ -24,6 +28,20 @@ def rand_bytes(length): out.append((seed >> 16) % 256) return bytes(out) +AES_256_KEY_SIZE = 32 + +# AES-CMAC. Just wraps the implementation from python-cryptography. +class AesCmac: + def __init__(self, key): + aes = cryptography.hazmat.primitives.ciphers.algorithms.AES(key) + self.cmac = cryptography.hazmat.primitives.cmac.CMAC(aes) + + def update(self, data): + self.cmac.update(data) + + def digest(self): + return self.cmac.finalize() + POLY1305_KEY_SIZE = 32 # A straightforward, unoptimized implementation of Poly1305. @@ -50,6 +68,52 @@ class Poly1305: m = (self.h + self.s) % 2**128 return m.to_bytes(16, byteorder='little') +GHASH_POLY = sum((1 << i) for i in [128, 7, 2, 1, 0]) +GHASH_BLOCK_SIZE = 16 + +# A straightforward, unoptimized implementation of GHASH. +class Ghash: + + @staticmethod + def reflect_bits_in_bytes(v): + res = 0 + for offs in range(0, 128, 8): + for bit in range(8): + if (v & (1 << (offs + bit))) != 0: + res ^= 1 << (offs + 7 - bit) + return res + + @staticmethod + def bytes_to_poly(data): + return Ghash.reflect_bits_in_bytes(int.from_bytes(data, byteorder='little')) + + @staticmethod + def poly_to_bytes(poly): + return Ghash.reflect_bits_in_bytes(poly).to_bytes(16, byteorder='little') + + def __init__(self, key): + assert len(key) == 16 + self.h = Ghash.bytes_to_poly(key) + self.acc = 0 + + # Note: this supports partial blocks only at the end. + def update(self, data): + for i in range(0, len(data), 16): + # acc += block + self.acc ^= Ghash.bytes_to_poly(data[i:i+16]) + # acc = (acc * h) mod GHASH_POLY + product = 0 + for j in range(127, -1, -1): + if (self.h & (1 << j)) != 0: + product ^= self.acc << j + if (product & (1 << (128 + j))) != 0: + product ^= GHASH_POLY << j + self.acc = product + return self + + def digest(self): + return Ghash.poly_to_bytes(self.acc) + POLYVAL_POLY = sum((1 << i) for i in [128, 127, 126, 121, 0]) POLYVAL_BLOCK_SIZE = 16 @@ -80,9 +144,14 @@ class Polyval: return self.acc.to_bytes(16, byteorder='little') def hash_init(alg): + # The keyed hash functions are assigned a fixed random key here, to present + # them as unkeyed hash functions. This allows all the test cases for + # unkeyed hash functions to work on them. + if alg == 'aes-cmac': + return AesCmac(rand_bytes(AES_256_KEY_SIZE)) + if alg == 'ghash': + return Ghash(rand_bytes(GHASH_BLOCK_SIZE)) if alg == 'poly1305': - # Use a fixed random key here, to present Poly1305 as an unkeyed hash. - # This allows all the test cases for unkeyed hashes to work on Poly1305. return Poly1305(rand_bytes(POLY1305_KEY_SIZE)) if alg == 'polyval': return Polyval(rand_bytes(POLYVAL_BLOCK_SIZE)) @@ -116,6 +185,8 @@ def print_c_struct_u8_array_field(name, value): print('\t\t},') def alg_digest_size_const(alg): + if alg == 'aes-cmac': + return 'AES_BLOCK_SIZE' if alg.startswith('blake2'): return f'{alg.upper()}_HASH_SIZE' return f"{alg.upper().replace('-', '_')}_DIGEST_SIZE" @@ -234,6 +305,15 @@ def gen_additional_poly1305_testvecs(): 'poly1305_allones_macofmacs[POLY1305_DIGEST_SIZE]', Poly1305(key).update(data).digest()) +def gen_additional_ghash_testvecs(): + key = b'\xff' * GHASH_BLOCK_SIZE + hashes = b'' + for data_len in range(0, 4097, 16): + hashes += Ghash(key).update(b'\xff' * data_len).digest() + print_static_u8_array_definition( + 'ghash_allones_hashofhashes[GHASH_DIGEST_SIZE]', + Ghash(key).update(hashes).digest()) + def gen_additional_polyval_testvecs(): key = b'\xff' * POLYVAL_BLOCK_SIZE hashes = b'' @@ -245,16 +325,22 @@ def gen_additional_polyval_testvecs(): if len(sys.argv) != 2: sys.stderr.write('Usage: gen-hash-testvecs.py ALGORITHM\n') - sys.stderr.write('ALGORITHM may be any supported by Python hashlib; or poly1305, polyval, or sha3.\n') + sys.stderr.write('ALGORITHM may be any supported by Python hashlib;\n') + sys.stderr.write(' or aes-cmac, ghash, nh, poly1305, polyval, or sha3.\n') sys.stderr.write('Example: gen-hash-testvecs.py sha512\n') sys.exit(1) alg = sys.argv[1] print('/* SPDX-License-Identifier: GPL-2.0-or-later */') print(f'/* This file was generated by: {sys.argv[0]} {" ".join(sys.argv[1:])} */') -if alg.startswith('blake2'): +if alg == 'aes-cmac': + gen_unkeyed_testvecs(alg) +elif alg.startswith('blake2'): gen_unkeyed_testvecs(alg) gen_additional_blake2_testvecs(alg) +elif alg == 'ghash': + gen_unkeyed_testvecs(alg) + gen_additional_ghash_testvecs() elif alg == 'nh': gen_nh_testvecs() elif alg == 'poly1305': @@ -270,6 +356,9 @@ elif alg == 'sha3': print() print('/* SHAKE test vectors */') gen_additional_sha3_testvecs() +elif alg == 'sm3': + gen_unkeyed_testvecs(alg) + # Kernel doesn't implement HMAC-SM3 library functions yet. else: gen_unkeyed_testvecs(alg) gen_hmac_testvecs(alg) diff --git a/security/integrity/ima/Kconfig b/security/integrity/ima/Kconfig index 976e75f9b9ba..862fbee2b174 100644 --- a/security/integrity/ima/Kconfig +++ b/security/integrity/ima/Kconfig @@ -111,7 +111,7 @@ choice config IMA_DEFAULT_HASH_SM3 bool "SM3" - depends on CRYPTO_SM3_GENERIC=y + depends on CRYPTO_SM3=y endchoice config IMA_DEFAULT_HASH diff --git a/tools/testing/kunit/configs/all_tests.config b/tools/testing/kunit/configs/all_tests.config index 422e186cf3cf..6910b07082da 100644 --- a/tools/testing/kunit/configs/all_tests.config +++ b/tools/testing/kunit/configs/all_tests.config @@ -44,6 +44,8 @@ CONFIG_REGMAP_BUILD=y CONFIG_AUDIT=y +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=y + CONFIG_PRIME_NUMBERS=y CONFIG_SECURITY=y |