diff options
| -rw-r--r-- | include/crypto/gf128hash.h | 95 | ||||
| -rw-r--r-- | lib/crypto/gf128hash.c | 145 |
2 files changed, 227 insertions, 13 deletions
diff --git a/include/crypto/gf128hash.h b/include/crypto/gf128hash.h index 1052041e3499..5090fbaa87f8 100644 --- a/include/crypto/gf128hash.h +++ b/include/crypto/gf128hash.h @@ -11,6 +11,8 @@ #include <linux/string.h> #include <linux/types.h> +#define GHASH_BLOCK_SIZE 16 +#define GHASH_DIGEST_SIZE 16 #define POLYVAL_BLOCK_SIZE 16 #define POLYVAL_DIGEST_SIZE 16 @@ -34,6 +36,16 @@ struct polyval_elem { }; /** + * struct ghash_key - Prepared key for GHASH + * + * Use ghash_preparekey() to initialize this. + */ +struct ghash_key { + /** @h: The hash key H, in POLYVAL format */ + struct polyval_elem h; +}; + +/** * struct polyval_key - Prepared key for POLYVAL * * This may contain just the raw key H, or it may contain precomputed key @@ -55,6 +67,20 @@ struct polyval_key { }; /** + * struct ghash_ctx - Context for computing a GHASH value + * @key: Pointer to the prepared GHASH key. The user of the API is + * responsible for ensuring that the key lives as long as the context. + * @acc: The accumulator. It is stored in POLYVAL format rather than GHASH + * format, since most implementations want it in POLYVAL format. + * @partial: Number of data bytes processed so far modulo GHASH_BLOCK_SIZE + */ +struct ghash_ctx { + const struct ghash_key *key; + struct polyval_elem acc; + size_t partial; +}; + +/** * struct polyval_ctx - Context for computing a POLYVAL value * @key: Pointer to the prepared POLYVAL key. The user of the API is * responsible for ensuring that the key lives as long as the context. @@ -68,6 +94,18 @@ struct polyval_ctx { }; /** + * ghash_preparekey() - Prepare a GHASH key + * @key: (output) The key structure to initialize + * @raw_key: The raw hash key + * + * Initialize a GHASH key structure from a raw key. + * + * Context: Any context. + */ +void ghash_preparekey(struct ghash_key *key, + const u8 raw_key[GHASH_BLOCK_SIZE]); + +/** * polyval_preparekey() - Prepare a POLYVAL key * @key: (output) The key structure to initialize * @raw_key: The raw hash key @@ -82,6 +120,18 @@ void polyval_preparekey(struct polyval_key *key, const u8 raw_key[POLYVAL_BLOCK_SIZE]); /** + * ghash_init() - Initialize a GHASH context for a new message + * @ctx: The context to initialize + * @key: The key to use. Note that a pointer to the key is saved in the + * context, so the key must live at least as long as the context. + */ +static inline void ghash_init(struct ghash_ctx *ctx, + const struct ghash_key *key) +{ + *ctx = (struct ghash_ctx){ .key = key }; +} + +/** * polyval_init() - Initialize a POLYVAL context for a new message * @ctx: The context to initialize * @key: The key to use. Note that a pointer to the key is saved in the @@ -126,6 +176,18 @@ static inline void polyval_export_blkaligned(const struct polyval_ctx *ctx, } /** + * ghash_update() - Update a GHASH context with message data + * @ctx: The context to update; must have been initialized + * @data: The message data + * @len: The data length in bytes. Doesn't need to be block-aligned. + * + * This can be called any number of times. + * + * Context: Any context. + */ +void ghash_update(struct ghash_ctx *ctx, const u8 *data, size_t len); + +/** * polyval_update() - Update a POLYVAL context with message data * @ctx: The context to update; must have been initialized * @data: The message data @@ -138,6 +200,20 @@ static inline void polyval_export_blkaligned(const struct polyval_ctx *ctx, void polyval_update(struct polyval_ctx *ctx, const u8 *data, size_t len); /** + * ghash_final() - Finish computing a GHASH value + * @ctx: The context to finalize + * @out: The output value + * + * If the total data length isn't a multiple of GHASH_BLOCK_SIZE, then the + * final block is automatically zero-padded. + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void ghash_final(struct ghash_ctx *ctx, u8 out[GHASH_BLOCK_SIZE]); + +/** * polyval_final() - Finish computing a POLYVAL value * @ctx: The context to finalize * @out: The output value @@ -152,6 +228,25 @@ void polyval_update(struct polyval_ctx *ctx, const u8 *data, size_t len); void polyval_final(struct polyval_ctx *ctx, u8 out[POLYVAL_BLOCK_SIZE]); /** + * ghash() - Compute a GHASH value + * @key: The prepared key + * @data: The message data + * @len: The data length in bytes. Doesn't need to be block-aligned. + * @out: The output value + * + * Context: Any context. + */ +static inline void ghash(const struct ghash_key *key, const u8 *data, + size_t len, u8 out[GHASH_BLOCK_SIZE]) +{ + struct ghash_ctx ctx; + + ghash_init(&ctx, key); + ghash_update(&ctx, data, len); + ghash_final(&ctx, out); +} + +/** * polyval() - Compute a POLYVAL value * @key: The prepared key * @data: The message data diff --git a/lib/crypto/gf128hash.c b/lib/crypto/gf128hash.c index 05f44a9193f7..2650603d8ba8 100644 --- a/lib/crypto/gf128hash.c +++ b/lib/crypto/gf128hash.c @@ -12,23 +12,30 @@ #include <linux/unaligned.h> /* - * POLYVAL is an almost-XOR-universal hash function. Similar to GHASH, POLYVAL - * interprets the message as the coefficients of a polynomial in GF(2^128) and - * evaluates that polynomial at a secret point. POLYVAL has a simple - * mathematical relationship with GHASH, but it uses a better field convention - * which makes it easier and faster to implement. + * GHASH and POLYVAL are almost-XOR-universal hash functions. They interpret + * the message as the coefficients of a polynomial in the finite field GF(2^128) + * and evaluate that polynomial at a secret point. * - * POLYVAL is not a cryptographic hash function, and it should be used only by - * algorithms that are specifically designed to use it. + * Neither GHASH nor POLYVAL is a cryptographic hash function. They should be + * used only by algorithms that are specifically designed to use them. * - * POLYVAL is specified by "AES-GCM-SIV: Nonce Misuse-Resistant Authenticated - * Encryption" (https://datatracker.ietf.org/doc/html/rfc8452) + * GHASH is the older variant, defined as part of GCM in NIST SP 800-38D + * (https://nvlpubs.nist.gov/nistpubs/legacy/sp/nistspecialpublication800-38d.pdf). + * GHASH is hard to implement directly, due to its backwards mapping between + * bits and polynomial coefficients. GHASH implementations typically pre and + * post-process the inputs and outputs (mainly by byte-swapping) to convert the + * GHASH computation into an equivalent computation over a different, + * easier-to-use representation of GF(2^128). * - * POLYVAL is also used by HCTR2. See "Length-preserving encryption with HCTR2" - * (https://eprint.iacr.org/2021/1441.pdf). + * POLYVAL is a newer GF(2^128) polynomial hash, originally defined as part of + * AES-GCM-SIV (https://datatracker.ietf.org/doc/html/rfc8452) and also used by + * HCTR2 (https://eprint.iacr.org/2021/1441.pdf). It uses that easier-to-use + * field representation directly, eliminating the data conversion steps. * - * This file provides a library API for POLYVAL. This API can delegate to - * either a generic implementation or an architecture-optimized implementation. + * This file provides library APIs for GHASH and POLYVAL. These APIs can + * delegate to either a generic implementation or an architecture-optimized + * implementation. Due to the mathematical relationship between GHASH and + * POLYVAL, in some cases code for one is reused with the other. * * For the generic implementation, we don't use the traditional table approach * to GF(2^128) multiplication. That approach is not constant-time and requires @@ -205,6 +212,19 @@ polyval_mul_generic(struct polyval_elem *a, const struct polyval_elem *b) a->hi = cpu_to_le64(c3); } +static void __maybe_unused ghash_blocks_generic(struct polyval_elem *acc, + const struct polyval_elem *key, + const u8 *data, size_t nblocks) +{ + do { + acc->lo ^= + cpu_to_le64(get_unaligned_be64((__be64 *)(data + 8))); + acc->hi ^= cpu_to_le64(get_unaligned_be64((__be64 *)data)); + polyval_mul_generic(acc, key); + data += GHASH_BLOCK_SIZE; + } while (--nblocks); +} + static void __maybe_unused polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key, const u8 *data, size_t nblocks) @@ -217,10 +237,108 @@ polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key, } while (--nblocks); } +/* Convert the key from GHASH format to POLYVAL format. */ +static void __maybe_unused ghash_key_to_polyval(const u8 in[GHASH_BLOCK_SIZE], + struct polyval_elem *out) +{ + u64 hi = get_unaligned_be64(&in[0]); + u64 lo = get_unaligned_be64(&in[8]); + u64 mask = (s64)hi >> 63; + + hi = (hi << 1) ^ (lo >> 63) ^ (mask & ((u64)0xc2 << 56)); + lo = (lo << 1) ^ (mask & 1); + out->lo = cpu_to_le64(lo); + out->hi = cpu_to_le64(hi); +} + +/* Convert the accumulator from POLYVAL format to GHASH format. */ +static void polyval_acc_to_ghash(const struct polyval_elem *in, + u8 out[GHASH_BLOCK_SIZE]) +{ + put_unaligned_be64(le64_to_cpu(in->hi), &out[0]); + put_unaligned_be64(le64_to_cpu(in->lo), &out[8]); +} + +/* Convert the accumulator from GHASH format to POLYVAL format. */ +static void __maybe_unused ghash_acc_to_polyval(const u8 in[GHASH_BLOCK_SIZE], + struct polyval_elem *out) +{ + out->lo = cpu_to_le64(get_unaligned_be64(&in[8])); + out->hi = cpu_to_le64(get_unaligned_be64(&in[0])); +} + #ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH #include "gf128hash.h" /* $(SRCARCH)/gf128hash.h */ #endif +void ghash_preparekey(struct ghash_key *key, const u8 raw_key[GHASH_BLOCK_SIZE]) +{ +#ifdef ghash_preparekey_arch + ghash_preparekey_arch(key, raw_key); +#else + ghash_key_to_polyval(raw_key, &key->h); +#endif +} +EXPORT_SYMBOL_GPL(ghash_preparekey); + +static void ghash_mul(struct ghash_ctx *ctx) +{ +#ifdef ghash_mul_arch + ghash_mul_arch(&ctx->acc, ctx->key); +#elif defined(ghash_blocks_arch) + static const u8 zeroes[GHASH_BLOCK_SIZE]; + + ghash_blocks_arch(&ctx->acc, ctx->key, zeroes, 1); +#else + polyval_mul_generic(&ctx->acc, &ctx->key->h); +#endif +} + +/* nblocks is always >= 1. */ +static void ghash_blocks(struct ghash_ctx *ctx, const u8 *data, size_t nblocks) +{ +#ifdef ghash_blocks_arch + ghash_blocks_arch(&ctx->acc, ctx->key, data, nblocks); +#else + ghash_blocks_generic(&ctx->acc, &ctx->key->h, data, nblocks); +#endif +} + +void ghash_update(struct ghash_ctx *ctx, const u8 *data, size_t len) +{ + if (unlikely(ctx->partial)) { + size_t n = min(len, GHASH_BLOCK_SIZE - ctx->partial); + + len -= n; + while (n--) + ctx->acc.bytes[GHASH_BLOCK_SIZE - 1 - ctx->partial++] ^= + *data++; + if (ctx->partial < GHASH_BLOCK_SIZE) + return; + ghash_mul(ctx); + } + if (len >= GHASH_BLOCK_SIZE) { + size_t nblocks = len / GHASH_BLOCK_SIZE; + + ghash_blocks(ctx, data, nblocks); + data += len & ~(GHASH_BLOCK_SIZE - 1); + len &= GHASH_BLOCK_SIZE - 1; + } + for (size_t i = 0; i < len; i++) + ctx->acc.bytes[GHASH_BLOCK_SIZE - 1 - i] ^= data[i]; + ctx->partial = len; +} +EXPORT_SYMBOL_GPL(ghash_update); + +void ghash_final(struct ghash_ctx *ctx, u8 out[GHASH_BLOCK_SIZE]) +{ + if (unlikely(ctx->partial)) + ghash_mul(ctx); + polyval_acc_to_ghash(&ctx->acc, out); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(ghash_final); + void polyval_preparekey(struct polyval_key *key, const u8 raw_key[POLYVAL_BLOCK_SIZE]) { @@ -253,6 +371,7 @@ static void polyval_mul(struct polyval_ctx *ctx) #endif } +/* nblocks is always >= 1. */ static void polyval_blocks(struct polyval_ctx *ctx, const u8 *data, size_t nblocks) { |