Merge tag 'libcrypto-updates-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux

Pull crypto library updates from Eric Biggers:
 "This is the main crypto library pull request for 6.17. The main focus
  this cycle is on reorganizing the SHA-1 and SHA-2 code, providing
  high-quality library APIs for SHA-1 and SHA-2 including HMAC support,
  and establishing conventions for lib/crypto/ going forward:

   - Migrate the SHA-1 and SHA-512 code (and also SHA-384 which shares
     most of the SHA-512 code) into lib/crypto/. This includes both the
     generic and architecture-optimized code. Greatly simplify how the
     architecture-optimized code is integrated. Add an easy-to-use
     library API for each SHA variant, including HMAC support. Finally,
     reimplement the crypto_shash support on top of the library API.

   - Apply the same reorganization to the SHA-256 code (and also SHA-224
     which shares most of the SHA-256 code). This is a somewhat smaller
     change, due to my earlier work on SHA-256. But this brings in all
     the same additional improvements that I made for SHA-1 and SHA-512.

  There are also some smaller changes:

   - Move the architecture-optimized ChaCha, Poly1305, and BLAKE2s code
     from arch/$(SRCARCH)/lib/crypto/ to lib/crypto/$(SRCARCH)/. For
     these algorithms it's just a move, not a full reorganization yet.

   - Fix the MIPS chacha-core.S to build with the clang assembler.

   - Fix the Poly1305 functions to work in all contexts.

   - Fix a performance regression in the x86_64 Poly1305 code.

   - Clean up the x86_64 SHA-NI optimized SHA-1 assembly code.

  Note that since the new organization of the SHA code is much simpler,
  the diffstat of this pull request is negative, despite the addition of
  new fully-documented library APIs for multiple SHA and HMAC-SHA
  variants.

  These APIs will allow further simplifications across the kernel as
  users start using them instead of the old-school crypto API. (I've
  already written a lot of such conversion patches, removing over 1000
  more lines of code. But most of those will target 6.18 or later)"

* tag 'libcrypto-updates-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux: (67 commits)
  lib/crypto: arm64/sha512-ce: Drop compatibility macros for older binutils
  lib/crypto: x86/sha1-ni: Convert to use rounds macros
  lib/crypto: x86/sha1-ni: Minor optimizations and cleanup
  crypto: sha1 - Remove sha1_base.h
  lib/crypto: x86/sha1: Migrate optimized code into library
  lib/crypto: sparc/sha1: Migrate optimized code into library
  lib/crypto: s390/sha1: Migrate optimized code into library
  lib/crypto: powerpc/sha1: Migrate optimized code into library
  lib/crypto: mips/sha1: Migrate optimized code into library
  lib/crypto: arm64/sha1: Migrate optimized code into library
  lib/crypto: arm/sha1: Migrate optimized code into library
  crypto: sha1 - Use same state format as legacy drivers
  crypto: sha1 - Wrap library and add HMAC support
  lib/crypto: sha1: Add HMAC support
  lib/crypto: sha1: Add SHA-1 library functions
  lib/crypto: sha1: Rename sha1_init() to sha1_init_raw()
  crypto: x86/sha1 - Rename conflicting symbol
  lib/crypto: sha2: Add hmac_sha*_init_usingrawkey()
  lib/crypto: arm/poly1305: Remove unneeded empty weak function
  lib/crypto: x86/poly1305: Fix performance regression on short messages
  ...

9 files changed, 810 insertions, 531 deletions

diff --git a/crypto/Kconfig b/crypto/Kconfig
index e1cfd0d4cc8f..23bd98981ae8 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -986,15 +986,16 @@ config CRYPTO_SHA1
 	select CRYPTO_HASH
 	select CRYPTO_LIB_SHA1
 	help
-	  SHA-1 secure hash algorithm (FIPS 180, ISO/IEC 10118-3)
+	  SHA-1 secure hash algorithm (FIPS 180, ISO/IEC 10118-3), including
+	  HMAC support.
 
 config CRYPTO_SHA256
 	tristate "SHA-224 and SHA-256"
 	select CRYPTO_HASH
 	select CRYPTO_LIB_SHA256
-	select CRYPTO_LIB_SHA256_GENERIC
 	help
-	  SHA-224 and SHA-256 secure hash algorithms (FIPS 180, ISO/IEC 10118-3)
+	  SHA-224 and SHA-256 secure hash algorithms (FIPS 180, ISO/IEC
+	  10118-3), including HMAC support.
 
 	  This is required for IPsec AH (XFRM_AH) and IPsec ESP (XFRM_ESP).
 	  Used by the btrfs filesystem, Ceph, NFS, and SMB.
@@ -1002,8 +1003,10 @@ config CRYPTO_SHA256
 config CRYPTO_SHA512
 	tristate "SHA-384 and SHA-512"
 	select CRYPTO_HASH
+	select CRYPTO_LIB_SHA512
 	help
-	  SHA-384 and SHA-512 secure hash algorithms (FIPS 180, ISO/IEC 10118-3)
+	  SHA-384 and SHA-512 secure hash algorithms (FIPS 180, ISO/IEC
+	  10118-3), including HMAC support.
 
 config CRYPTO_SHA3
 	tristate "SHA-3"
@@ -1420,9 +1423,6 @@ config CRYPTO_USER_API_ENABLE_OBSOLETE
 
 endmenu
 
-config CRYPTO_HASH_INFO
-	bool
-
 if !KMSAN # avoid false positives from assembly
 if ARM
 source "arch/arm/crypto/Kconfig"
diff --git a/crypto/Makefile b/crypto/Makefile
index 55dd56332dc8..6c5d59369dac 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -75,10 +75,9 @@ obj-$(CONFIG_CRYPTO_NULL) += crypto_null.o
 obj-$(CONFIG_CRYPTO_MD4) += md4.o
 obj-$(CONFIG_CRYPTO_MD5) += md5.o
 obj-$(CONFIG_CRYPTO_RMD160) += rmd160.o
-obj-$(CONFIG_CRYPTO_SHA1) += sha1_generic.o
+obj-$(CONFIG_CRYPTO_SHA1) += sha1.o
 obj-$(CONFIG_CRYPTO_SHA256) += sha256.o
-CFLAGS_sha256.o += -DARCH=$(ARCH)
-obj-$(CONFIG_CRYPTO_SHA512) += sha512_generic.o
+obj-$(CONFIG_CRYPTO_SHA512) += sha512.o
 obj-$(CONFIG_CRYPTO_SHA3) += sha3_generic.o
 obj-$(CONFIG_CRYPTO_SM3_GENERIC) += sm3_generic.o
 obj-$(CONFIG_CRYPTO_STREEBOG) += streebog_generic.o
@@ -203,7 +202,6 @@ obj-$(CONFIG_CRYPTO_ECRDSA) += ecrdsa_generic.o
 obj-$(CONFIG_XOR_BLOCKS) += xor.o
 obj-$(CONFIG_ASYNC_CORE) += async_tx/
 obj-$(CONFIG_ASYMMETRIC_KEY_TYPE) += asymmetric_keys/
-obj-$(CONFIG_CRYPTO_HASH_INFO) += hash_info.o
 crypto_simd-y := simd.o
 obj-$(CONFIG_CRYPTO_SIMD) += crypto_simd.o
 
diff --git a/crypto/hash_info.c b/crypto/hash_info.c
deleted file mode 100644
index 9a467638c971..000000000000
--- a/crypto/hash_info.c
+++ /dev/null
@@ -1,63 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Hash Info: Hash algorithms information
- *
- * Copyright (c) 2013 Dmitry Kasatkin <d.kasatkin@samsung.com>
- */
-
-#include <linux/export.h>
-#include <crypto/hash_info.h>
-
-const char *const hash_algo_name[HASH_ALGO__LAST] = {
-	[HASH_ALGO_MD4]		= "md4",
-	[HASH_ALGO_MD5]		= "md5",
-	[HASH_ALGO_SHA1]	= "sha1",
-	[HASH_ALGO_RIPE_MD_160]	= "rmd160",
-	[HASH_ALGO_SHA256]	= "sha256",
-	[HASH_ALGO_SHA384]	= "sha384",
-	[HASH_ALGO_SHA512]	= "sha512",
-	[HASH_ALGO_SHA224]	= "sha224",
-	[HASH_ALGO_RIPE_MD_128]	= "rmd128",
-	[HASH_ALGO_RIPE_MD_256]	= "rmd256",
-	[HASH_ALGO_RIPE_MD_320]	= "rmd320",
-	[HASH_ALGO_WP_256]	= "wp256",
-	[HASH_ALGO_WP_384]	= "wp384",
-	[HASH_ALGO_WP_512]	= "wp512",
-	[HASH_ALGO_TGR_128]	= "tgr128",
-	[HASH_ALGO_TGR_160]	= "tgr160",
-	[HASH_ALGO_TGR_192]	= "tgr192",
-	[HASH_ALGO_SM3_256]	= "sm3",
-	[HASH_ALGO_STREEBOG_256] = "streebog256",
-	[HASH_ALGO_STREEBOG_512] = "streebog512",
-	[HASH_ALGO_SHA3_256]    = "sha3-256",
-	[HASH_ALGO_SHA3_384]    = "sha3-384",
-	[HASH_ALGO_SHA3_512]    = "sha3-512",
-};
-EXPORT_SYMBOL_GPL(hash_algo_name);
-
-const int hash_digest_size[HASH_ALGO__LAST] = {
-	[HASH_ALGO_MD4]		= MD5_DIGEST_SIZE,
-	[HASH_ALGO_MD5]		= MD5_DIGEST_SIZE,
-	[HASH_ALGO_SHA1]	= SHA1_DIGEST_SIZE,
-	[HASH_ALGO_RIPE_MD_160]	= RMD160_DIGEST_SIZE,
-	[HASH_ALGO_SHA256]	= SHA256_DIGEST_SIZE,
-	[HASH_ALGO_SHA384]	= SHA384_DIGEST_SIZE,
-	[HASH_ALGO_SHA512]	= SHA512_DIGEST_SIZE,
-	[HASH_ALGO_SHA224]	= SHA224_DIGEST_SIZE,
-	[HASH_ALGO_RIPE_MD_128]	= RMD128_DIGEST_SIZE,
-	[HASH_ALGO_RIPE_MD_256]	= RMD256_DIGEST_SIZE,
-	[HASH_ALGO_RIPE_MD_320]	= RMD320_DIGEST_SIZE,
-	[HASH_ALGO_WP_256]	= WP256_DIGEST_SIZE,
-	[HASH_ALGO_WP_384]	= WP384_DIGEST_SIZE,
-	[HASH_ALGO_WP_512]	= WP512_DIGEST_SIZE,
-	[HASH_ALGO_TGR_128]	= TGR128_DIGEST_SIZE,
-	[HASH_ALGO_TGR_160]	= TGR160_DIGEST_SIZE,
-	[HASH_ALGO_TGR_192]	= TGR192_DIGEST_SIZE,
-	[HASH_ALGO_SM3_256]	= SM3256_DIGEST_SIZE,
-	[HASH_ALGO_STREEBOG_256] = STREEBOG256_DIGEST_SIZE,
-	[HASH_ALGO_STREEBOG_512] = STREEBOG512_DIGEST_SIZE,
-	[HASH_ALGO_SHA3_256]    = SHA3_256_DIGEST_SIZE,
-	[HASH_ALGO_SHA3_384]    = SHA3_384_DIGEST_SIZE,
-	[HASH_ALGO_SHA3_512]    = SHA3_512_DIGEST_SIZE,
-};
-EXPORT_SYMBOL_GPL(hash_digest_size);
diff --git a/crypto/sha1.c b/crypto/sha1.c
new file mode 100644
index 000000000000..ecef4bf2d9c0
--- /dev/null
+++ b/crypto/sha1.c
@@ -0,0 +1,201 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Crypto API support for SHA-1 and HMAC-SHA1
+ *
+ * Copyright (c) Alan Smithee.
+ * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
+ * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
+ * Copyright 2025 Google LLC
+ */
+#include <crypto/internal/hash.h>
+#include <crypto/sha1.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+
+/*
+ * Export and import functions.  crypto_shash wants a particular format that
+ * matches that used by some legacy drivers.  It currently is the same as the
+ * library SHA context, except the value in bytecount must be block-aligned and
+ * the remainder must be stored in an extra u8 appended to the struct.
+ */
+
+#define SHA1_SHASH_STATE_SIZE (sizeof(struct sha1_ctx) + 1)
+static_assert(sizeof(struct sha1_ctx) == sizeof(struct sha1_state));
+static_assert(offsetof(struct sha1_ctx, state) == offsetof(struct sha1_state, state));
+static_assert(offsetof(struct sha1_ctx, bytecount) == offsetof(struct sha1_state, count));
+static_assert(offsetof(struct sha1_ctx, buf) == offsetof(struct sha1_state, buffer));
+
+static int __crypto_sha1_export(const struct sha1_ctx *ctx0, void *out)
+{
+	struct sha1_ctx ctx = *ctx0;
+	unsigned int partial;
+	u8 *p = out;
+
+	partial = ctx.bytecount % SHA1_BLOCK_SIZE;
+	ctx.bytecount -= partial;
+	memcpy(p, &ctx, sizeof(ctx));
+	p += sizeof(ctx);
+	*p = partial;
+	return 0;
+}
+
+static int __crypto_sha1_import(struct sha1_ctx *ctx, const void *in)
+{
+	const u8 *p = in;
+
+	memcpy(ctx, p, sizeof(*ctx));
+	p += sizeof(*ctx);
+	ctx->bytecount += *p;
+	return 0;
+}
+
+const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE] = {
+	0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d,
+	0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90,
+	0xaf, 0xd8, 0x07, 0x09
+};
+EXPORT_SYMBOL_GPL(sha1_zero_message_hash);
+
+#define SHA1_CTX(desc) ((struct sha1_ctx *)shash_desc_ctx(desc))
+
+static int crypto_sha1_init(struct shash_desc *desc)
+{
+	sha1_init(SHA1_CTX(desc));
+	return 0;
+}
+
+static int crypto_sha1_update(struct shash_desc *desc,
+			      const u8 *data, unsigned int len)
+{
+	sha1_update(SHA1_CTX(desc), data, len);
+	return 0;
+}
+
+static int crypto_sha1_final(struct shash_desc *desc, u8 *out)
+{
+	sha1_final(SHA1_CTX(desc), out);
+	return 0;
+}
+
+static int crypto_sha1_digest(struct shash_desc *desc,
+			      const u8 *data, unsigned int len, u8 *out)
+{
+	sha1(data, len, out);
+	return 0;
+}
+
+static int crypto_sha1_export(struct shash_desc *desc, void *out)
+{
+	return __crypto_sha1_export(SHA1_CTX(desc), out);
+}
+
+static int crypto_sha1_import(struct shash_desc *desc, const void *in)
+{
+	return __crypto_sha1_import(SHA1_CTX(desc), in);
+}
+
+#define HMAC_SHA1_KEY(tfm) ((struct hmac_sha1_key *)crypto_shash_ctx(tfm))
+#define HMAC_SHA1_CTX(desc) ((struct hmac_sha1_ctx *)shash_desc_ctx(desc))
+
+static int crypto_hmac_sha1_setkey(struct crypto_shash *tfm,
+				   const u8 *raw_key, unsigned int keylen)
+{
+	hmac_sha1_preparekey(HMAC_SHA1_KEY(tfm), raw_key, keylen);
+	return 0;
+}
+
+static int crypto_hmac_sha1_init(struct shash_desc *desc)
+{
+	hmac_sha1_init(HMAC_SHA1_CTX(desc), HMAC_SHA1_KEY(desc->tfm));
+	return 0;
+}
+
+static int crypto_hmac_sha1_update(struct shash_desc *desc,
+				   const u8 *data, unsigned int len)
+{
+	hmac_sha1_update(HMAC_SHA1_CTX(desc), data, len);
+	return 0;
+}
+
+static int crypto_hmac_sha1_final(struct shash_desc *desc, u8 *out)
+{
+	hmac_sha1_final(HMAC_SHA1_CTX(desc), out);
+	return 0;
+}
+
+static int crypto_hmac_sha1_digest(struct shash_desc *desc,
+				   const u8 *data, unsigned int len, u8 *out)
+{
+	hmac_sha1(HMAC_SHA1_KEY(desc->tfm), data, len, out);
+	return 0;
+}
+
+static int crypto_hmac_sha1_export(struct shash_desc *desc, void *out)
+{
+	return __crypto_sha1_export(&HMAC_SHA1_CTX(desc)->sha_ctx, out);
+}
+
+static int crypto_hmac_sha1_import(struct shash_desc *desc, const void *in)
+{
+	struct hmac_sha1_ctx *ctx = HMAC_SHA1_CTX(desc);
+
+	ctx->ostate = HMAC_SHA1_KEY(desc->tfm)->ostate;
+	return __crypto_sha1_import(&ctx->sha_ctx, in);
+}
+
+static struct shash_alg algs[] = {
+	{
+		.base.cra_name		= "sha1",
+		.base.cra_driver_name	= "sha1-lib",
+		.base.cra_priority	= 300,
+		.base.cra_blocksize	= SHA1_BLOCK_SIZE,
+		.base.cra_module	= THIS_MODULE,
+		.digestsize		= SHA1_DIGEST_SIZE,
+		.init			= crypto_sha1_init,
+		.update			= crypto_sha1_update,
+		.final			= crypto_sha1_final,
+		.digest			= crypto_sha1_digest,
+		.export			= crypto_sha1_export,
+		.import			= crypto_sha1_import,
+		.descsize		= sizeof(struct sha1_ctx),
+		.statesize		= SHA1_SHASH_STATE_SIZE,
+	},
+	{
+		.base.cra_name		= "hmac(sha1)",
+		.base.cra_driver_name	= "hmac-sha1-lib",
+		.base.cra_priority	= 300,
+		.base.cra_blocksize	= SHA1_BLOCK_SIZE,
+		.base.cra_ctxsize	= sizeof(struct hmac_sha1_key),
+		.base.cra_module	= THIS_MODULE,
+		.digestsize		= SHA1_DIGEST_SIZE,
+		.setkey			= crypto_hmac_sha1_setkey,
+		.init			= crypto_hmac_sha1_init,
+		.update			= crypto_hmac_sha1_update,
+		.final			= crypto_hmac_sha1_final,
+		.digest			= crypto_hmac_sha1_digest,
+		.export			= crypto_hmac_sha1_export,
+		.import			= crypto_hmac_sha1_import,
+		.descsize		= sizeof(struct hmac_sha1_ctx),
+		.statesize		= SHA1_SHASH_STATE_SIZE,
+	},
+};
+
+static int __init crypto_sha1_mod_init(void)
+{
+	return crypto_register_shashes(algs, ARRAY_SIZE(algs));
+}
+module_init(crypto_sha1_mod_init);
+
+static void __exit crypto_sha1_mod_exit(void)
+{
+	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
+}
+module_exit(crypto_sha1_mod_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Crypto API support for SHA-1 and HMAC-SHA1");
+
+MODULE_ALIAS_CRYPTO("sha1");
+MODULE_ALIAS_CRYPTO("sha1-lib");
+MODULE_ALIAS_CRYPTO("hmac(sha1)");
+MODULE_ALIAS_CRYPTO("hmac-sha1-lib");
diff --git a/crypto/sha1_generic.c b/crypto/sha1_generic.c
deleted file mode 100644
index 024e8043bab0..000000000000
--- a/crypto/sha1_generic.c
+++ /dev/null
@@ -1,87 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Cryptographic API.
- *
- * SHA1 Secure Hash Algorithm.
- *
- * Derived from cryptoapi implementation, adapted for in-place
- * scatterlist interface.
- *
- * Copyright (c) Alan Smithee.
- * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
- * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
- */
-#include <crypto/internal/hash.h>
-#include <crypto/sha1.h>
-#include <crypto/sha1_base.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/string.h>
-
-const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE] = {
-	0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d,
-	0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90,
-	0xaf, 0xd8, 0x07, 0x09
-};
-EXPORT_SYMBOL_GPL(sha1_zero_message_hash);
-
-static void sha1_generic_block_fn(struct sha1_state *sst, u8 const *src,
-				  int blocks)
-{
-	u32 temp[SHA1_WORKSPACE_WORDS];
-
-	while (blocks--) {
-		sha1_transform(sst->state, src, temp);
-		src += SHA1_BLOCK_SIZE;
-	}
-	memzero_explicit(temp, sizeof(temp));
-}
-
-static int crypto_sha1_update(struct shash_desc *desc, const u8 *data,
-			      unsigned int len)
-{
-	return sha1_base_do_update_blocks(desc, data, len,
-					  sha1_generic_block_fn);
-}
-
-static int crypto_sha1_finup(struct shash_desc *desc, const u8 *data,
-			     unsigned int len, u8 *out)
-{
-	sha1_base_do_finup(desc, data, len, sha1_generic_block_fn);
-	return sha1_base_finish(desc, out);
-}
-
-static struct shash_alg alg = {
-	.digestsize	=	SHA1_DIGEST_SIZE,
-	.init		=	sha1_base_init,
-	.update		=	crypto_sha1_update,
-	.finup		=	crypto_sha1_finup,
-	.descsize	=	SHA1_STATE_SIZE,
-	.base		=	{
-		.cra_name	=	"sha1",
-		.cra_driver_name=	"sha1-generic",
-		.cra_priority	=	100,
-		.cra_flags	=	CRYPTO_AHASH_ALG_BLOCK_ONLY,
-		.cra_blocksize	=	SHA1_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-};
-
-static int __init sha1_generic_mod_init(void)
-{
-	return crypto_register_shash(&alg);
-}
-
-static void __exit sha1_generic_mod_fini(void)
-{
-	crypto_unregister_shash(&alg);
-}
-
-module_init(sha1_generic_mod_init);
-module_exit(sha1_generic_mod_fini);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm");
-
-MODULE_ALIAS_CRYPTO("sha1");
-MODULE_ALIAS_CRYPTO("sha1-generic");
diff --git a/crypto/sha256.c b/crypto/sha256.c
index 4aeb213bab11..052806559f06 100644
--- a/crypto/sha256.c
+++ b/crypto/sha256.c
@@ -1,17 +1,57 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
- * Crypto API wrapper for the SHA-256 and SHA-224 library functions
+ * Crypto API support for SHA-224, SHA-256, HMAC-SHA224, and HMAC-SHA256
  *
  * Copyright (c) Jean-Luc Cooke <jlcooke@certainkey.com>
  * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
  * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
  * SHA224 Support Copyright 2007 Intel Corporation <jonathan.lynch@intel.com>
+ * Copyright 2025 Google LLC
  */
 #include <crypto/internal/hash.h>
-#include <crypto/internal/sha2.h>
+#include <crypto/sha2.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 
+/*
+ * Export and import functions.  crypto_shash wants a particular format that
+ * matches that used by some legacy drivers.  It currently is the same as the
+ * library SHA context, except the value in bytecount must be block-aligned and
+ * the remainder must be stored in an extra u8 appended to the struct.
+ */
+
+#define SHA256_SHASH_STATE_SIZE 105
+static_assert(offsetof(struct __sha256_ctx, state) == 0);
+static_assert(offsetof(struct __sha256_ctx, bytecount) == 32);
+static_assert(offsetof(struct __sha256_ctx, buf) == 40);
+static_assert(sizeof(struct __sha256_ctx) + 1 == SHA256_SHASH_STATE_SIZE);
+
+static int __crypto_sha256_export(const struct __sha256_ctx *ctx0, void *out)
+{
+	struct __sha256_ctx ctx = *ctx0;
+	unsigned int partial;
+	u8 *p = out;
+
+	partial = ctx.bytecount % SHA256_BLOCK_SIZE;
+	ctx.bytecount -= partial;
+	memcpy(p, &ctx, sizeof(ctx));
+	p += sizeof(ctx);
+	*p = partial;
+	return 0;
+}
+
+static int __crypto_sha256_import(struct __sha256_ctx *ctx, const void *in)
+{
+	const u8 *p = in;
+
+	memcpy(ctx, p, sizeof(*ctx));
+	p += sizeof(*ctx);
+	ctx->bytecount += *p;
+	return 0;
+}
+
+/* SHA-224 */
+
 const u8 sha224_zero_message_hash[SHA224_DIGEST_SIZE] = {
 	0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47,
 	0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2,
@@ -20,6 +60,46 @@ const u8 sha224_zero_message_hash[SHA224_DIGEST_SIZE] = {
 };
 EXPORT_SYMBOL_GPL(sha224_zero_message_hash);
 
+#define SHA224_CTX(desc) ((struct sha224_ctx *)shash_desc_ctx(desc))
+
+static int crypto_sha224_init(struct shash_desc *desc)
+{
+	sha224_init(SHA224_CTX(desc));
+	return 0;
+}
+
+static int crypto_sha224_update(struct shash_desc *desc,
+				const u8 *data, unsigned int len)
+{
+	sha224_update(SHA224_CTX(desc), data, len);
+	return 0;
+}
+
+static int crypto_sha224_final(struct shash_desc *desc, u8 *out)
+{
+	sha224_final(SHA224_CTX(desc), out);
+	return 0;
+}
+
+static int crypto_sha224_digest(struct shash_desc *desc,
+				const u8 *data, unsigned int len, u8 *out)
+{
+	sha224(data, len, out);
+	return 0;
+}
+
+static int crypto_sha224_export(struct shash_desc *desc, void *out)
+{
+	return __crypto_sha256_export(&SHA224_CTX(desc)->ctx, out);
+}
+
+static int crypto_sha224_import(struct shash_desc *desc, const void *in)
+{
+	return __crypto_sha256_import(&SHA224_CTX(desc)->ctx, in);
+}
+
+/* SHA-256 */
+
 const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE] = {
 	0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14,
 	0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24,
@@ -28,256 +108,241 @@ const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE] = {
 };
 EXPORT_SYMBOL_GPL(sha256_zero_message_hash);
 
+#define SHA256_CTX(desc) ((struct sha256_ctx *)shash_desc_ctx(desc))
+
 static int crypto_sha256_init(struct shash_desc *desc)
 {
-	sha256_block_init(shash_desc_ctx(desc));
+	sha256_init(SHA256_CTX(desc));
+	return 0;
+}
+
+static int crypto_sha256_update(struct shash_desc *desc,
+				const u8 *data, unsigned int len)
+{
+	sha256_update(SHA256_CTX(desc), data, len);
+	return 0;
+}
+
+static int crypto_sha256_final(struct shash_desc *desc, u8 *out)
+{
+	sha256_final(SHA256_CTX(desc), out);
 	return 0;
 }
 
-static inline int crypto_sha256_update(struct shash_desc *desc, const u8 *data,
-				       unsigned int len, bool force_generic)
+static int crypto_sha256_digest(struct shash_desc *desc,
+				const u8 *data, unsigned int len, u8 *out)
 {
-	struct crypto_sha256_state *sctx = shash_desc_ctx(desc);
-	int remain = len % SHA256_BLOCK_SIZE;
+	sha256(data, len, out);
+	return 0;
+}
 
-	sctx->count += len - remain;
-	sha256_choose_blocks(sctx->state, data, len / SHA256_BLOCK_SIZE,
-			     force_generic, !force_generic);
-	return remain;
+static int crypto_sha256_export(struct shash_desc *desc, void *out)
+{
+	return __crypto_sha256_export(&SHA256_CTX(desc)->ctx, out);
 }
 
-static int crypto_sha256_update_generic(struct shash_desc *desc, const u8 *data,
-					unsigned int len)
+static int crypto_sha256_import(struct shash_desc *desc, const void *in)
 {
-	return crypto_sha256_update(desc, data, len, true);
+	return __crypto_sha256_import(&SHA256_CTX(desc)->ctx, in);
 }
 
-static int crypto_sha256_update_lib(struct shash_desc *desc, const u8 *data,
-				    unsigned int len)
+/* HMAC-SHA224 */
+
+#define HMAC_SHA224_KEY(tfm) ((struct hmac_sha224_key *)crypto_shash_ctx(tfm))
+#define HMAC_SHA224_CTX(desc) ((struct hmac_sha224_ctx *)shash_desc_ctx(desc))
+
+static int crypto_hmac_sha224_setkey(struct crypto_shash *tfm,
+				     const u8 *raw_key, unsigned int keylen)
 {
-	sha256_update(shash_desc_ctx(desc), data, len);
+	hmac_sha224_preparekey(HMAC_SHA224_KEY(tfm), raw_key, keylen);
 	return 0;
 }
 
-static int crypto_sha256_update_arch(struct shash_desc *desc, const u8 *data,
-				     unsigned int len)
+static int crypto_hmac_sha224_init(struct shash_desc *desc)
 {
-	return crypto_sha256_update(desc, data, len, false);
+	hmac_sha224_init(HMAC_SHA224_CTX(desc), HMAC_SHA224_KEY(desc->tfm));
+	return 0;
 }
 
-static int crypto_sha256_final_lib(struct shash_desc *desc, u8 *out)
+static int crypto_hmac_sha224_update(struct shash_desc *desc,
+				     const u8 *data, unsigned int len)
 {
-	sha256_final(shash_desc_ctx(desc), out);
+	hmac_sha224_update(HMAC_SHA224_CTX(desc), data, len);
 	return 0;
 }
 
-static __always_inline int crypto_sha256_finup(struct shash_desc *desc,
-					       const u8 *data,
-					       unsigned int len, u8 *out,
-					       bool force_generic)
+static int crypto_hmac_sha224_final(struct shash_desc *desc, u8 *out)
 {
-	struct crypto_sha256_state *sctx = shash_desc_ctx(desc);
-	unsigned int remain = len;
-	u8 *buf;
-
-	if (len >= SHA256_BLOCK_SIZE)
-		remain = crypto_sha256_update(desc, data, len, force_generic);
-	sctx->count += remain;
-	buf = memcpy(sctx + 1, data + len - remain, remain);
-	sha256_finup(sctx, buf, remain, out,
-		     crypto_shash_digestsize(desc->tfm), force_generic,
-		     !force_generic);
+	hmac_sha224_final(HMAC_SHA224_CTX(desc), out);
 	return 0;
 }
 
-static int crypto_sha256_finup_generic(struct shash_desc *desc, const u8 *data,
-				       unsigned int len, u8 *out)
+static int crypto_hmac_sha224_digest(struct shash_desc *desc,
+				     const u8 *data, unsigned int len,
+				     u8 *out)
 {
-	return crypto_sha256_finup(desc, data, len, out, true);
+	hmac_sha224(HMAC_SHA224_KEY(desc->tfm), data, len, out);
+	return 0;
 }
 
-static int crypto_sha256_finup_arch(struct shash_desc *desc, const u8 *data,
-				    unsigned int len, u8 *out)
+static int crypto_hmac_sha224_export(struct shash_desc *desc, void *out)
 {
-	return crypto_sha256_finup(desc, data, len, out, false);
+	return __crypto_sha256_export(&HMAC_SHA224_CTX(desc)->ctx.sha_ctx, out);
 }
 
-static int crypto_sha256_digest_generic(struct shash_desc *desc, const u8 *data,
-					unsigned int len, u8 *out)
+static int crypto_hmac_sha224_import(struct shash_desc *desc, const void *in)
 {
-	crypto_sha256_init(desc);
-	return crypto_sha256_finup_generic(desc, data, len, out);
+	struct hmac_sha224_ctx *ctx = HMAC_SHA224_CTX(desc);
+
+	ctx->ctx.ostate = HMAC_SHA224_KEY(desc->tfm)->key.ostate;
+	return __crypto_sha256_import(&ctx->ctx.sha_ctx, in);
 }
 
-static int crypto_sha256_digest_lib(struct shash_desc *desc, const u8 *data,
-				    unsigned int len, u8 *out)
+/* HMAC-SHA256 */
+
+#define HMAC_SHA256_KEY(tfm) ((struct hmac_sha256_key *)crypto_shash_ctx(tfm))
+#define HMAC_SHA256_CTX(desc) ((struct hmac_sha256_ctx *)shash_desc_ctx(desc))
+
+static int crypto_hmac_sha256_setkey(struct crypto_shash *tfm,
+				     const u8 *raw_key, unsigned int keylen)
 {
-	sha256(data, len, out);
+	hmac_sha256_preparekey(HMAC_SHA256_KEY(tfm), raw_key, keylen);
 	return 0;
 }
 
-static int crypto_sha256_digest_arch(struct shash_desc *desc, const u8 *data,
-				     unsigned int len, u8 *out)
+static int crypto_hmac_sha256_init(struct shash_desc *desc)
 {
-	crypto_sha256_init(desc);
-	return crypto_sha256_finup_arch(desc, data, len, out);
+	hmac_sha256_init(HMAC_SHA256_CTX(desc), HMAC_SHA256_KEY(desc->tfm));
+	return 0;
 }
 
-static int crypto_sha224_init(struct shash_desc *desc)
+static int crypto_hmac_sha256_update(struct shash_desc *desc,
+				     const u8 *data, unsigned int len)
 {
-	sha224_block_init(shash_desc_ctx(desc));
+	hmac_sha256_update(HMAC_SHA256_CTX(desc), data, len);
 	return 0;
 }
 
-static int crypto_sha224_final_lib(struct shash_desc *desc, u8 *out)
+static int crypto_hmac_sha256_final(struct shash_desc *desc, u8 *out)
 {
-	sha224_final(shash_desc_ctx(desc), out);
+	hmac_sha256_final(HMAC_SHA256_CTX(desc), out);
 	return 0;
 }
 
-static int crypto_sha256_import_lib(struct shash_desc *desc, const void *in)
+static int crypto_hmac_sha256_digest(struct shash_desc *desc,
+				     const u8 *data, unsigned int len,
+				     u8 *out)
 {
-	struct sha256_state *sctx = shash_desc_ctx(desc);
-	const u8 *p = in;
-
-	memcpy(sctx, p, sizeof(*sctx));
-	p += sizeof(*sctx);
-	sctx->count += *p;
+	hmac_sha256(HMAC_SHA256_KEY(desc->tfm), data, len, out);
 	return 0;
 }
 
-static int crypto_sha256_export_lib(struct shash_desc *desc, void *out)
+static int crypto_hmac_sha256_export(struct shash_desc *desc, void *out)
 {
-	struct sha256_state *sctx0 = shash_desc_ctx(desc);
-	struct sha256_state sctx = *sctx0;
-	unsigned int partial;
-	u8 *p = out;
+	return __crypto_sha256_export(&HMAC_SHA256_CTX(desc)->ctx.sha_ctx, out);
+}
 
-	partial = sctx.count % SHA256_BLOCK_SIZE;
-	sctx.count -= partial;
-	memcpy(p, &sctx, sizeof(sctx));
-	p += sizeof(sctx);
-	*p = partial;
-	return 0;
+static int crypto_hmac_sha256_import(struct shash_desc *desc, const void *in)
+{
+	struct hmac_sha256_ctx *ctx = HMAC_SHA256_CTX(desc);
+
+	ctx->ctx.ostate = HMAC_SHA256_KEY(desc->tfm)->key.ostate;
+	return __crypto_sha256_import(&ctx->ctx.sha_ctx, in);
 }
 
+/* Algorithm definitions */
+
 static struct shash_alg algs[] = {
 	{
-		.base.cra_name		= "sha256",
-		.base.cra_driver_name	= "sha256-generic",
-		.base.cra_priority	= 100,
-		.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |
-					  CRYPTO_AHASH_ALG_FINUP_MAX,
-		.base.cra_blocksize	= SHA256_BLOCK_SIZE,
-		.base.cra_module	= THIS_MODULE,
-		.digestsize		= SHA256_DIGEST_SIZE,
-		.init			= crypto_sha256_init,
-		.update			= crypto_sha256_update_generic,
-		.finup			= crypto_sha256_finup_generic,
-		.digest			= crypto_sha256_digest_generic,
-		.descsize		= sizeof(struct crypto_sha256_state),
-	},
-	{
 		.base.cra_name		= "sha224",
-		.base.cra_driver_name	= "sha224-generic",
-		.base.cra_priority	= 100,
-		.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |
-					  CRYPTO_AHASH_ALG_FINUP_MAX,
+		.base.cra_driver_name	= "sha224-lib",
+		.base.cra_priority	= 300,
 		.base.cra_blocksize	= SHA224_BLOCK_SIZE,
 		.base.cra_module	= THIS_MODULE,
 		.digestsize		= SHA224_DIGEST_SIZE,
 		.init			= crypto_sha224_init,
-		.update			= crypto_sha256_update_generic,
-		.finup			= crypto_sha256_finup_generic,
-		.descsize		= sizeof(struct crypto_sha256_state),
+		.update			= crypto_sha224_update,
+		.final			= crypto_sha224_final,
+		.digest			= crypto_sha224_digest,
+		.export			= crypto_sha224_export,
+		.import			= crypto_sha224_import,
+		.descsize		= sizeof(struct sha224_ctx),
+		.statesize		= SHA256_SHASH_STATE_SIZE,
 	},
 	{
 		.base.cra_name		= "sha256",
 		.base.cra_driver_name	= "sha256-lib",
+		.base.cra_priority	= 300,
 		.base.cra_blocksize	= SHA256_BLOCK_SIZE,
 		.base.cra_module	= THIS_MODULE,
 		.digestsize		= SHA256_DIGEST_SIZE,
 		.init			= crypto_sha256_init,
-		.update			= crypto_sha256_update_lib,
-		.final			= crypto_sha256_final_lib,
-		.digest			= crypto_sha256_digest_lib,
-		.descsize		= sizeof(struct sha256_state),
-		.statesize		= sizeof(struct crypto_sha256_state) +
-					  SHA256_BLOCK_SIZE + 1,
-		.import			= crypto_sha256_import_lib,
-		.export			= crypto_sha256_export_lib,
+		.update			= crypto_sha256_update,
+		.final			= crypto_sha256_final,
+		.digest			= crypto_sha256_digest,
+		.export			= crypto_sha256_export,
+		.import			= crypto_sha256_import,
+		.descsize		= sizeof(struct sha256_ctx),
+		.statesize		= SHA256_SHASH_STATE_SIZE,
 	},
 	{
-		.base.cra_name		= "sha224",
-		.base.cra_driver_name	= "sha224-lib",
+		.base.cra_name		= "hmac(sha224)",
+		.base.cra_driver_name	= "hmac-sha224-lib",
+		.base.cra_priority	= 300,
 		.base.cra_blocksize	= SHA224_BLOCK_SIZE,
+		.base.cra_ctxsize	= sizeof(struct hmac_sha224_key),
 		.base.cra_module	= THIS_MODULE,
 		.digestsize		= SHA224_DIGEST_SIZE,
-		.init			= crypto_sha224_init,
-		.update			= crypto_sha256_update_lib,
-		.final			= crypto_sha224_final_lib,
-		.descsize		= sizeof(struct sha256_state),
-		.statesize		= sizeof(struct crypto_sha256_state) +
-					  SHA256_BLOCK_SIZE + 1,
-		.import			= crypto_sha256_import_lib,
-		.export			= crypto_sha256_export_lib,
+		.setkey			= crypto_hmac_sha224_setkey,
+		.init			= crypto_hmac_sha224_init,
+		.update			= crypto_hmac_sha224_update,
+		.final			= crypto_hmac_sha224_final,
+		.digest			= crypto_hmac_sha224_digest,
+		.export			= crypto_hmac_sha224_export,
+		.import			= crypto_hmac_sha224_import,
+		.descsize		= sizeof(struct hmac_sha224_ctx),
+		.statesize		= SHA256_SHASH_STATE_SIZE,
 	},
 	{
-		.base.cra_name		= "sha256",
-		.base.cra_driver_name	= "sha256-" __stringify(ARCH),
+		.base.cra_name		= "hmac(sha256)",
+		.base.cra_driver_name	= "hmac-sha256-lib",
 		.base.cra_priority	= 300,
-		.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |
-					  CRYPTO_AHASH_ALG_FINUP_MAX,
 		.base.cra_blocksize	= SHA256_BLOCK_SIZE,
+		.base.cra_ctxsize	= sizeof(struct hmac_sha256_key),
 		.base.cra_module	= THIS_MODULE,
 		.digestsize		= SHA256_DIGEST_SIZE,
-		.init			= crypto_sha256_init,
-		.update			= crypto_sha256_update_arch,
-		.finup			= crypto_sha256_finup_arch,
-		.digest			= crypto_sha256_digest_arch,
-		.descsize		= sizeof(struct crypto_sha256_state),
-	},
-	{
-		.base.cra_name		= "sha224",
-		.base.cra_driver_name	= "sha224-" __stringify(ARCH),
-		.base.cra_priority	= 300,
-		.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |
-					  CRYPTO_AHASH_ALG_FINUP_MAX,
-		.base.cra_blocksize	= SHA224_BLOCK_SIZE,
-		.base.cra_module	= THIS_MODULE,
-		.digestsize		= SHA224_DIGEST_SIZE,
-		.init			= crypto_sha224_init,
-		.update			= crypto_sha256_update_arch,
-		.finup			= crypto_sha256_finup_arch,
-		.descsize		= sizeof(struct crypto_sha256_state),
+		.setkey			= crypto_hmac_sha256_setkey,
+		.init			= crypto_hmac_sha256_init,
+		.update			= crypto_hmac_sha256_update,
+		.final			= crypto_hmac_sha256_final,
+		.digest			= crypto_hmac_sha256_digest,
+		.export			= crypto_hmac_sha256_export,
+		.import			= crypto_hmac_sha256_import,
+		.descsize		= sizeof(struct hmac_sha256_ctx),
+		.statesize		= SHA256_SHASH_STATE_SIZE,
 	},
 };
 
-static unsigned int num_algs;
-
 static int __init crypto_sha256_mod_init(void)
 {
-	/* register the arch flavours only if they differ from generic */
-	num_algs = ARRAY_SIZE(algs);
-	BUILD_BUG_ON(ARRAY_SIZE(algs)