crypto: tegra - Add Tegra Security Engine driver

Add support for Tegra Security Engine which can accelerate various
crypto algorithms. The Engine has two separate instances within for
AES and HASH algorithms respectively.

The driver registers two crypto engines - one for AES and another for
HASH algorithms and these operate independently and both uses the host1x
bus. Additionally, it provides  hardware-assisted key protection for up
to 15 symmetric keys which it can use for the cipher operations.

Signed-off-by: Akhil R <akhilrajeev@nvidia.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

8 files changed, 4166 insertions, 0 deletions

diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 3d02702456a5..bb27690f8f7c 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -660,6 +660,14 @@ config CRYPTO_DEV_ROCKCHIP_DEBUG
 	  This will create /sys/kernel/debug/rk3288_crypto/stats for displaying
 	  the number of requests per algorithm and other internal stats.
 
+config CRYPTO_DEV_TEGRA
+	tristate "Enable Tegra Security Engine"
+	depends on TEGRA_HOST1X
+	select CRYPTO_ENGINE
+
+	help
+	  Select this to enable Tegra Security Engine which accelerates various
+	  AES encryption/decryption and HASH algorithms.
 
 config CRYPTO_DEV_ZYNQMP_AES
 	tristate "Support for Xilinx ZynqMP AES hw accelerator"
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 95331bc6456b..ad4ccef67d12 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -41,6 +41,7 @@ obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o
 obj-$(CONFIG_CRYPTO_DEV_SL3516) += gemini/
 obj-y += stm32/
 obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o
+obj-$(CONFIG_CRYPTO_DEV_TEGRA) += tegra/
 obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio/
 #obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/
 obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) += bcm/
diff --git a/drivers/crypto/tegra/Makefile b/drivers/crypto/tegra/Makefile
new file mode 100644
index 000000000000..a32001e58eb2
--- /dev/null
+++ b/drivers/crypto/tegra/Makefile
@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0-only
+// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+tegra-se-objs := tegra-se-key.o tegra-se-main.o
+
+tegra-se-y += tegra-se-aes.o
+tegra-se-y += tegra-se-hash.o
+
+obj-$(CONFIG_CRYPTO_DEV_TEGRA) += tegra-se.o
diff --git a/drivers/crypto/tegra/tegra-se-aes.c b/drivers/crypto/tegra/tegra-se-aes.c
new file mode 100644
index 000000000000..adc6cdab389e
--- /dev/null
+++ b/drivers/crypto/tegra/tegra-se-aes.c
@@ -0,0 +1,1933 @@
+// SPDX-License-Identifier: GPL-2.0-only
+// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+/*
+ * Crypto driver to handle block cipher algorithms using NVIDIA Security Engine.
+ */
+
+#include <linux/clk.h>
+#include <linux/dma-mapping.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+
+#include <crypto/aead.h>
+#include <crypto/aes.h>
+#include <crypto/engine.h>
+#include <crypto/gcm.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/xts.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/hash.h>
+#include <crypto/internal/skcipher.h>
+
+#include "tegra-se.h"
+
+struct tegra_aes_ctx {
+	struct tegra_se *se;
+	u32 alg;
+	u32 ivsize;
+	u32 key1_id;
+	u32 key2_id;
+};
+
+struct tegra_aes_reqctx {
+	struct tegra_se_datbuf datbuf;
+	bool encrypt;
+	u32 config;
+	u32 crypto_config;
+	u32 len;
+	u32 *iv;
+};
+
+struct tegra_aead_ctx {
+	struct tegra_se *se;
+	unsigned int authsize;
+	u32 alg;
+	u32 keylen;
+	u32 key_id;
+};
+
+struct tegra_aead_reqctx {
+	struct tegra_se_datbuf inbuf;
+	struct tegra_se_datbuf outbuf;
+	struct scatterlist *src_sg;
+	struct scatterlist *dst_sg;
+	unsigned int assoclen;
+	unsigned int cryptlen;
+	unsigned int authsize;
+	bool encrypt;
+	u32 config;
+	u32 crypto_config;
+	u32 key_id;
+	u32 iv[4];
+	u8 authdata[16];
+};
+
+struct tegra_cmac_ctx {
+	struct tegra_se *se;
+	unsigned int alg;
+	u32 key_id;
+	struct crypto_shash *fallback_tfm;
+};
+
+struct tegra_cmac_reqctx {
+	struct scatterlist *src_sg;
+	struct tegra_se_datbuf datbuf;
+	struct tegra_se_datbuf residue;
+	unsigned int total_len;
+	unsigned int blk_size;
+	unsigned int task;
+	u32 crypto_config;
+	u32 config;
+	u32 key_id;
+	u32 *iv;
+	u32 result[CMAC_RESULT_REG_COUNT];
+};
+
+/* increment counter (128-bit int) */
+static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums)
+{
+	do {
+		--bits;
+		nums += counter[bits];
+		counter[bits] = nums & 0xff;
+		nums >>= 8;
+	} while (bits && nums);
+}
+
+static void tegra_cbc_iv_copyback(struct skcipher_request *req, struct tegra_aes_ctx *ctx)
+{
+	struct tegra_aes_reqctx *rctx = skcipher_request_ctx(req);
+	unsigned int offset;
+
+	offset = req->cryptlen - ctx->ivsize;
+
+	if (rctx->encrypt)
+		memcpy(req->iv, rctx->datbuf.buf + offset, ctx->ivsize);
+	else
+		scatterwalk_map_and_copy(req->iv, req->src, offset, ctx->ivsize, 0);
+}
+
+static void tegra_aes_update_iv(struct skcipher_request *req, struct tegra_aes_ctx *ctx)
+{
+	int num;
+
+	if (ctx->alg == SE_ALG_CBC) {
+		tegra_cbc_iv_copyback(req, ctx);
+	} else if (ctx->alg == SE_ALG_CTR) {
+		num = req->cryptlen / ctx->ivsize;
+		if (req->cryptlen % ctx->ivsize)
+			num++;
+
+		ctr_iv_inc(req->iv, ctx->ivsize, num);
+	}
+}
+
+static int tegra234_aes_crypto_cfg(u32 alg, bool encrypt)
+{
+	switch (alg) {
+	case SE_ALG_CMAC:
+	case SE_ALG_GMAC:
+	case SE_ALG_GCM:
+	case SE_ALG_GCM_FINAL:
+		return 0;
+	case SE_ALG_CBC:
+		if (encrypt)
+			return SE_CRYPTO_CFG_CBC_ENCRYPT;
+		else
+			return SE_CRYPTO_CFG_CBC_DECRYPT;
+	case SE_ALG_ECB:
+		if (encrypt)
+			return SE_CRYPTO_CFG_ECB_ENCRYPT;
+		else
+			return SE_CRYPTO_CFG_ECB_DECRYPT;
+	case SE_ALG_XTS:
+		if (encrypt)
+			return SE_CRYPTO_CFG_XTS_ENCRYPT;
+		else
+			return SE_CRYPTO_CFG_XTS_DECRYPT;
+
+	case SE_ALG_CTR:
+		return SE_CRYPTO_CFG_CTR;
+	case SE_ALG_CBC_MAC:
+		return SE_CRYPTO_CFG_CBC_MAC;
+
+	default:
+		break;
+	}
+
+	return -EINVAL;
+}
+
+static int tegra234_aes_cfg(u32 alg, bool encrypt)
+{
+	switch (alg) {
+	case SE_ALG_CBC:
+	case SE_ALG_ECB:
+	case SE_ALG_XTS:
+	case SE_ALG_CTR:
+		if (encrypt)
+			return SE_CFG_AES_ENCRYPT;
+		else
+			return SE_CFG_AES_DECRYPT;
+
+	case SE_ALG_GMAC:
+		if (encrypt)
+			return SE_CFG_GMAC_ENCRYPT;
+		else
+			return SE_CFG_GMAC_DECRYPT;
+
+	case SE_ALG_GCM:
+		if (encrypt)
+			return SE_CFG_GCM_ENCRYPT;
+		else
+			return SE_CFG_GCM_DECRYPT;
+
+	case SE_ALG_GCM_FINAL:
+		if (encrypt)
+			return SE_CFG_GCM_FINAL_ENCRYPT;
+		else
+			return SE_CFG_GCM_FINAL_DECRYPT;
+
+	case SE_ALG_CMAC:
+		return SE_CFG_CMAC;
+
+	case SE_ALG_CBC_MAC:
+		return SE_AES_ENC_ALG_AES_ENC |
+		       SE_AES_DST_HASH_REG;
+	}
+	return -EINVAL;
+}
+
+static unsigned int tegra_aes_prep_cmd(struct tegra_aes_ctx *ctx,
+				       struct tegra_aes_reqctx *rctx)
+{
+	unsigned int data_count, res_bits, i = 0, j;
+	struct tegra_se *se = ctx->se;
+	u32 *cpuvaddr = se->cmdbuf->addr;
+	dma_addr_t addr = rctx->datbuf.addr;
+
+	data_count = rctx->len / AES_BLOCK_SIZE;
+	res_bits = (rctx->len % AES_BLOCK_SIZE) * 8;
+
+	/*
+	 * Hardware processes data_count + 1 blocks.
+	 * Reduce 1 block if there is no residue
+	 */
+	if (!res_bits)
+		data_count--;
+
+	if (rctx->iv) {
+		cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT);
+		cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr);
+		for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++)
+			cpuvaddr[i++] = rctx->iv[j];
+	}
+
+	cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1);
+	cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count) |
+			SE_LAST_BLOCK_RES_BITS(res_bits);
+
+	cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6);
+	cpuvaddr[i++] = rctx->config;
+	cpuvaddr[i++] = rctx->crypto_config;
+
+	/* Source address setting */
+	cpuvaddr[i++] = lower_32_bits(addr);
+	cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(addr)) | SE_ADDR_HI_SZ(rctx->len);
+
+	/* Destination address setting */
+	cpuvaddr[i++] = lower_32_bits(addr);
+	cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(addr)) |
+			SE_ADDR_HI_SZ(rctx->len);
+
+	cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1);
+	cpuvaddr[i++] = SE_AES_OP_WRSTALL | SE_AES_OP_LASTBUF |
+			SE_AES_OP_START;
+
+	cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1);
+	cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) |
+			host1x_uclass_incr_syncpt_indx_f(se->syncpt_id);
+
+	dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n", rctx->config, rctx->crypto_config);
+
+	return i;
+}
+
+static int tegra_aes_do_one_req(struct crypto_engine *engine, void *areq)
+{
+	struct skcipher_request *req = container_of(areq, struct skcipher_request, base);
+	struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
+	struct tegra_aes_reqctx *rctx = skcipher_request_ctx(req);
+	struct tegra_se *se = ctx->se;
+	unsigned int cmdlen;
+	int ret;
+
+	rctx->datbuf.buf = dma_alloc_coherent(se->dev, SE_AES_BUFLEN,
+					      &rctx->datbuf.addr, GFP_KERNEL);
+	if (!rctx->datbuf.buf)
+		return -ENOMEM;
+
+	rctx->datbuf.size = SE_AES_BUFLEN;
+	rctx->iv = (u32 *)req->iv;
+	rctx->len = req->cryptlen;
+
+	/* Pad input to AES Block size */
+	if (ctx->alg != SE_ALG_XTS) {
+		if (rctx->len % AES_BLOCK_SIZE)
+			rctx->len += AES_BLOCK_SIZE - (rctx->len % AES_BLOCK_SIZE);
+	}
+
+	scatterwalk_map_and_copy(rctx->datbuf.buf, req->src, 0, req->cryptlen, 0);
+
+	/* Prepare the command and submit for execution */
+	cmdlen = tegra_aes_prep_cmd(ctx, rctx);
+	ret = tegra_se_host1x_submit(se, cmdlen);
+
+	/* Copy the result */
+	tegra_aes_update_iv(req, ctx);
+	scatterwalk_map_and_copy(rctx->datbuf.buf, req->dst, 0, req->cryptlen, 1);
+
+	/* Free the buffer */
+	dma_free_coherent(ctx->se->dev, SE_AES_BUFLEN,
+			  rctx->datbuf.buf, rctx->datbuf.addr);
+
+	crypto_finalize_skcipher_request(se->engine, req, ret);
+
+	return 0;
+}
+
+static int tegra_aes_cra_init(struct crypto_skcipher *tfm)
+{
+	struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
+	struct tegra_se_alg *se_alg;
+	const char *algname;
+	int ret;
+
+	se_alg = container_of(alg, struct tegra_se_alg, alg.skcipher.base);
+
+	crypto_skcipher_set_reqsize(tfm, sizeof(struct tegra_aes_reqctx));
+
+	ctx->ivsize = crypto_skcipher_ivsize(tfm);
+	ctx->se = se_alg->se_dev;
+	ctx->key1_id = 0;
+	ctx->key2_id = 0;
+
+	algname = crypto_tfm_alg_name(&tfm->base);
+	ret = se_algname_to_algid(algname);
+	if (ret < 0) {
+		dev_err(ctx->se->dev, "invalid algorithm\n");
+		return ret;
+	}
+
+	ctx->alg = ret;
+
+	return 0;
+}
+
+static void tegra_aes_cra_exit(struct crypto_skcipher *tfm)
+{
+	struct tegra_aes_ctx *ctx = crypto_tfm_ctx(&tfm->base);
+
+	if (ctx->key1_id)
+		tegra_key_invalidate(ctx->se, ctx->key1_id, ctx->alg);
+
+	if (ctx->key2_id)
+		tegra_key_invalidate(ctx->se, ctx->key2_id, ctx->alg);
+}
+
+static int tegra_aes_setkey(struct crypto_skcipher *tfm,
+			    const u8 *key, u32 keylen)
+{
+	struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+	if (aes_check_keylen(keylen)) {
+		dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen);
+		return -EINVAL;
+	}
+
+	return tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key1_id);
+}
+
+static int tegra_xts_setkey(struct crypto_skcipher *tfm,
+			    const u8 *key, u32 keylen)
+{
+	struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
+	u32 len = keylen / 2;
+	int ret;
+
+	ret = xts_verify_key(tfm, key, keylen);
+	if (ret || aes_check_keylen(len)) {
+		dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen);
+		return -EINVAL;
+	}
+
+	ret = tegra_key_submit(ctx->se, key, len,
+			       ctx->alg, &ctx->key1_id);
+	if (ret)
+		return ret;
+
+	return tegra_key_submit(ctx->se, key + len, len,
+			       ctx->alg, &ctx->key2_id);
+
+	return 0;
+}
+
+static int tegra_aes_kac_manifest(u32 user, u32 alg, u32 keylen)
+{
+	int manifest;
+
+	manifest = SE_KAC_USER_NS;
+
+	switch (alg) {
+	case SE_ALG_CBC:
+	case SE_ALG_ECB:
+	case SE_ALG_CTR:
+		manifest |= SE_KAC_ENC;
+		break;
+	case SE_ALG_XTS:
+		manifest |= SE_KAC_XTS;
+		break;
+	case SE_ALG_GCM:
+		manifest |= SE_KAC_GCM;
+		break;
+	case SE_ALG_CMAC:
+		manifest |= SE_KAC_CMAC;
+		break;
+	case SE_ALG_CBC_MAC:
+		manifest |= SE_KAC_ENC;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (keylen) {
+	case AES_KEYSIZE_128:
+		manifest |= SE_KAC_SIZE_128;
+		break;
+	case AES_KEYSIZE_192:
+		manifest |= SE_KAC_SIZE_192;
+		break;
+	case AES_KEYSIZE_256:
+		manifest |= SE_KAC_SIZE_256;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return manifest;
+}
+
+static int tegra_aes_crypt(struct skcipher_request *req, bool encrypt)
+
+{
+	struct crypto_skcipher *tfm;
+	struct tegra_aes_ctx *ctx;
+	struct tegra_aes_reqctx *rctx;
+
+	tfm = crypto_skcipher_reqtfm(req);
+	ctx  = crypto_skcipher_ctx(tfm);
+	rctx = skcipher_request_ctx(req);
+
+	if (ctx->alg != SE_ALG_XTS) {
+		if (!IS_ALIGNED(req->cryptlen, crypto_skcipher_blocksize(tfm))) {
+			dev_dbg(ctx->se->dev, "invalid length (%d)", req->cryptlen);
+			return -EINVAL;
+		}
+	} else if (req->cryptlen < XTS_BLOCK_SIZE) {
+		dev_dbg(ctx->se->dev, "invalid length (%d)", req->cryptlen);
+		return -EINVAL;
+	}
+
+	if (!req->cryptlen)
+		return 0;
+
+	rctx->encrypt = encrypt;
+	rctx->config = tegra234_aes_cfg(ctx->alg, encrypt);
+	rctx->crypto_config = tegra234_aes_crypto_cfg(ctx->alg, encrypt);
+	rctx->crypto_config |= SE_AES_KEY_INDEX(ctx->key1_id);
+
+	if (ctx->key2_id)
+		rctx->crypto_config |= SE_AES_KEY2_INDEX(ctx->key2_id);
+
+	return crypto_transfer_skcipher_request_to_engine(ctx->se->engine, req);
+}
+
+static int tegra_aes_encrypt(struct skcipher_request *req)
+{
+	return tegra_aes_crypt(req, true);
+}
+
+static int tegra_aes_decrypt(struct skcipher_request *req)
+{
+	return tegra_aes_crypt(req, false);
+}
+
+static struct tegra_se_alg tegra_aes_algs[] = {
+	{
+		.alg.skcipher.op.do_one_request	= tegra_aes_do_one_req,
+		.alg.skcipher.base = {
+			.init = tegra_aes_cra_init,
+			.exit = tegra_aes_cra_exit,
+			.setkey	= tegra_aes_setkey,
+			.encrypt = tegra_aes_encrypt,
+			.decrypt = tegra_aes_decrypt,
+			.min_keysize = AES_MIN_KEY_SIZE,
+			.max_keysize = AES_MAX_KEY_SIZE,
+			.ivsize	= AES_BLOCK_SIZE,
+			.base = {
+				.cra_name = "cbc(aes)",
+				.cra_driver_name = "cbc-aes-tegra",
+				.cra_priority = 500,
+				.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC,
+				.cra_blocksize = AES_BLOCK_SIZE,
+				.cra_ctxsize = sizeof(struct tegra_aes_ctx),
+				.cra_alignmask = 0xf,
+				.cra_module = THIS_MODULE,
+			},
+		}
+	}, {
+		.alg.skcipher.op.do_one_request	= tegra_aes_do_one_req,
+		.alg.skcipher.base = {
+			.init = tegra_aes_cra_init,
+			.exit = tegra_aes_cra_exit,
+			.setkey	= tegra_aes_setkey,
+			.encrypt = tegra_aes_encrypt,
+			.decrypt = tegra_aes_decrypt,
+			.min_keysize = AES_MIN_KEY_SIZE,
+			.max_keysize = AES_MAX_KEY_SIZE,
+			.base = {
+				.cra_name = "ecb(aes)",
+				.cra_driver_name = "ecb-aes-tegra",
+				.cra_priority = 500,
+				.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC,
+				.cra_blocksize = AES_BLOCK_SIZE,
+				.cra_ctxsize = sizeof(struct tegra_aes_ctx),
+				.cra_alignmask = 0xf,
+				.cra_module = THIS_MODULE,
+			},
+		}
+	}, {
+		.alg.skcipher.op.do_one_request	= tegra_aes_do_one_req,
+		.alg.skcipher.base = {
+			.init = tegra_aes_cra_init,
+			.exit = tegra_aes_cra_exit,
+			.setkey = tegra_aes_setkey,
+			.encrypt = tegra_aes_encrypt,
+			.decrypt = tegra_aes_decrypt,
+			.min_keysize = AES_MIN_KEY_SIZE,
+			.max_keysize = AES_MAX_KEY_SIZE,
+			.ivsize	= AES_BLOCK_SIZE,
+			.base = {
+				.cra_name = "ctr(aes)",
+				.cra_driver_name = "ctr-aes-tegra",
+				.cra_priority = 500,
+				.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC,
+				.cra_blocksize = 1,
+				.cra_ctxsize = sizeof(struct tegra_aes_ctx),
+				.cra_alignmask = 0xf,
+				.cra_module = THIS_MODULE,
+			},
+		}
+	}, {
+		.alg.skcipher.op.do_one_request	= tegra_aes_do_one_req,
+		.alg.skcipher.base = {
+			.init = tegra_aes_cra_init,
+			.exit = tegra_aes_cra_exit,
+			.setkey	= tegra_xts_setkey,
+			.encrypt = tegra_aes_encrypt,
+			.decrypt = tegra_aes_decrypt,
+			.min_keysize = 2 * AES_MIN_KEY_SIZE,
+			.max_keysize = 2 * AES_MAX_KEY_SIZE,
+			.ivsize	= AES_BLOCK_SIZE,
+			.base = {
+				.cra_name = "xts(aes)",
+				.cra_driver_name = "xts-aes-tegra",
+				.cra_priority = 500,
+				.cra_blocksize = AES_BLOCK_SIZE,
+				.cra_ctxsize	   = sizeof(struct tegra_aes_ctx),
+				.cra_alignmask	   = (__alignof__(u64) - 1),
+				.cra_module	   = THIS_MODULE,
+			},
+		}
+	},
+};
+
+static unsigned int tegra_gmac_prep_cmd(struct tegra_aead_ctx *ctx,
+					struct tegra_aead_reqctx *rctx)
+{
+	unsigned int data_count, res_bits, i = 0;
+	struct tegra_se *se = ctx->se;
+	u32 *cpuvaddr = se->cmdbuf->addr;
+
+	data_count = (rctx->assoclen / AES_BLOCK_SIZE);
+	res_bits = (rctx->assoclen % AES_BLOCK_SIZE) * 8;
+
+	/*
+	 * Hardware processes data_count + 1 blocks.
+	 * Reduce 1 block if there is no residue
+	 */
+	if (!res_bits)
+		data_count--;
+
+	cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1);
+	cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count) |
+			SE_LAST_BLOCK_RES_BITS(res_bits);
+
+	cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 4);
+	cpuvaddr[i++] = rctx->config;
+	cpuvaddr[i++] = rctx->crypto_config;
+	cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr);
+	cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) |
+			SE_ADDR_HI_SZ(rctx->assoclen);
+
+	cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1);
+	cpuvaddr[i++] = SE_AES_OP_WRSTALL | SE_AES_OP_FINAL |
+			SE_AES_OP_INIT | SE_AES_OP_LASTBUF |
+			SE_AES_OP_START;
+
+	cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1);
+	cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) |
+			host1x_uclass_incr_syncpt_indx_f(se->syncpt_id);
+
+	return i;
+}
+
+static unsigned int tegra_gcm_crypt_prep_cmd(struct tegra_aead_ctx *ctx,
+					     struct tegra_aead_reqctx *rctx)
+{
+	unsigned int data_count, res_bits, i = 0, j;
+	struct tegra_se *se = ctx->se;
+	u32 *cpuvaddr = se->cmdbuf->addr, op;
+
+	data_count = (rctx->cryptlen / AES_BLOCK_SIZE);
+	res_bits = (rctx->cryptlen % AES_BLOCK_SIZE) * 8;
+	op = SE_AES_OP_WRSTALL | SE_AES_OP_FINAL |
+	     SE_AES_OP_LASTBUF | SE_AES_OP_START;
+
+	/*
+	 * If there is no assoc data,
+	 * this will be the init command
+	 */
+	if (!rctx->assoclen)
+		op |= SE_AES_OP_INIT;
+
+	/*
+	 * Hardware processes data_count + 1 blocks.
+	 * Reduce 1 block if there is no residue
+	 */
+	if (!res_bits)
+		data_count--;
+
+	cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT);
+	cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr);
+	for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++)
+		cpuvaddr[i++] = rctx->iv[j];
+
+	cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1);
+	cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count) |
+			SE_LAST_BLOCK_RES_BITS(res_bits);
+
+	cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6);
+	cpuvaddr[i++] = rctx->config;
+	cpuvaddr[i++] = rctx->crypto_config;
+
+	/* Source Address */
+	cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr);
+	cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) |
+			SE_ADDR_HI_SZ(rctx->cryptlen);
+
+	/* Destination Address */
+	cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr);
+	cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) |
+			SE_ADDR_HI_SZ(rctx->cryptlen);
+
+	cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1);
+	cpuvaddr[i++] = op;
+
+	cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1);
+	cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) |
+			host1x_uclass_incr_syncpt_indx_f(se->syncpt_id);
+
+	dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n", rctx->config, rctx->crypto_config);
+	return i;
+}
+
+static int tegra_gcm_prep_final_cmd(struct tegra_se *se, u32 *cpuvaddr,
+				    struct tegra_aead_reqctx *rctx)
+{
+	unsigned int i = 0, j;
+	u32 op;
+
+	op = SE_AES_OP_WRSTALL | SE_AES_OP_FINAL |
+	     SE_AES_OP_LASTBUF | SE_AES_OP_START;
+
+	/*
+	 * Set init for zero sized vector
+	 */
+	if (!rctx->assoclen && !rctx->cryptlen)
+		op |= SE_AES_OP_INIT;
+
+	cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->aad_len, 2);
+	cpuvaddr[i++] = rctx->assoclen * 8;
+	cpuvaddr[i++] = 0;
+
+	cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->cryp_msg_len, 2);
+	cpuvaddr[i++] = rctx->cryptlen * 8;
+	cpuvaddr[i++] = 0;
+
+	cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT);
+	cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr);
+	for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++)
+		cpuvaddr[i++] = rctx->iv[j];
+
+	cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6);
+	cpuvaddr[i++] = rctx->config;
+	cpuvaddr[i++] = rctx->crypto_config;
+	cpuvaddr[i++] = 0;
+	cpuvaddr[i++] = 0;
+
+	/* Destination Address */
+	cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr);
+	cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) |
+			SE_ADDR_HI_SZ(0x10); /* HW always generates 128-bit tag */
+
+	cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1);
+	cpuvaddr[i++] = op;
+
+	cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1);
+	cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) |
+			host1x_uclass_incr_syncpt_indx_f(se->syncpt_id);
+
+	dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n", rctx->config, rctx->crypto_config);
+
+	return i;
+}
+
+static int tegra_gcm_do_gmac(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx)
+{
+	struct tegra_se *se = ctx->se;
+	unsigned int cmdlen;
+
+	scatterwalk_map_and_copy(rctx->inbuf.buf,
+				 rctx->src_sg, 0, rctx->assoclen, 0);
+
+	rctx->config = tegra234_aes_cfg(SE_ALG_GMAC, rctx->encrypt);
+	rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GMAC, rctx->encrypt) |
+			      SE_AES_KEY_INDEX(ctx->key_id);
+
+	cmdlen = tegra_gmac_prep_cmd(ctx, rctx);
+
+	return tegra_se_host1x_submit(se, cmdlen);
+}
+
+static int tegra_gcm_do_crypt(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx)
+{
+	struct tegra_se *se = ctx->se;
+	int cmdlen, ret;
+
+	scatterwalk_map_and_copy(rctx->inbuf.buf, rctx->src_sg,
+				 rctx->assoclen, rctx->cryptlen, 0);
+
+	rctx->config = tegra234_aes_cfg(SE_ALG_GCM, rctx->encrypt);
+	rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GCM, rctx->encrypt) |
+			      SE_AES_KEY_INDEX(ctx->key_id);
+
+	/* Prepare command and submit */
+	cmdlen = tegra_gcm_crypt_prep_cmd(ctx, rctx);
+	ret = tegra_se_host1x_submit(se, cmdlen);
+	if (ret)
+		return ret;
+
+	/* Copy the result */
+	scatterwalk_map_and_copy(rctx->outbuf.buf, rctx->dst_sg,
+				 rctx->assoclen, rctx->cryptlen, 1);
+
+	return 0;
+}
+
+static int tegra_gcm_do_final(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx)
+{
+	struct tegra_se *se = ctx->se;
+	u32 *cpuvaddr = se->cmdbuf->addr;
+	int cmdlen, ret, offset;
+
+	rctx->config = tegra234_aes_cfg(SE_ALG_GCM_FINAL, rctx->encrypt);
+	rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GCM_FINAL, rctx->encrypt) |
+			      SE_AES_KEY_INDEX(ctx->key_id);
+
+	/* Prepare command and submit */
+	cmdlen = tegra_gcm_prep_final_cmd(se, cpuvaddr, rctx);
+	ret = tegra_se_host1x_submit(se, cmdlen);
+	if (ret)
+		return ret;
+
+	if (rctx->encrypt) {
+		/* Copy the result */
+		offset = rctx->assoclen + rctx->cryptlen;
+		scatterwalk_map_and_copy(rctx->outbuf.buf, rctx->dst_sg,
+					 offset, rctx->authsize, 1);
+	}
+
+	return 0;
+}
+
+static int tegra_gcm_do_verify(struct tegra_se *se, struct tegra_aead_reqctx *rctx)
+{
+	unsigned int offset;
+	u8 mac[16];
+
+	offset = rctx->assoclen + rctx->cryptlen;
+	scatterwalk_map_and_copy(mac, rctx->src_sg, offset, rctx->authsize, 0);
+
+	if (crypto_memneq(rctx->outbuf.buf, mac, rctx->authsize))
+		return -EBADMSG;
+
+	return 0;
+}
+
+static inline int tegra_ccm_check_iv(const u8 *iv)
+{
+	/* iv[0] gives value of q-1
+	 * 2 <= q <= 8 as per NIST 800-38C notation
+	 * 2 <= L <= 8, so 1 <= L' <= 7. as per rfc 3610 notation
+	 */
+	if (iv[0] < 1 || iv[0] > 7) {
+		pr_debug("ccm_check_iv failed %d\n", iv[0]);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static unsigned int tegra_cbcmac_prep_cmd(struct tegra_aead_ctx *ctx,
+					  struct tegra_aead_reqctx *rctx)
+{
+	unsigned int data_count, i = 0;
+	struct tegra_se *se = ctx->se;
+	u32 *cpuvaddr = se->cmdbuf->addr;
+
+	data_count = (rctx->inbuf.size / AES_BLOCK_SIZE) - 1;
+
+	cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1);
+	cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count);
+
+	cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6);
+	cpuvaddr[i++] = rctx->config;
+	cpuvaddr[i++] = rctx->crypto_config;
+
+	cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr);
+	cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) |
+			SE_ADDR_HI_SZ(rctx->inbuf.size);
+
+	cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr);
+	cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) |
+			SE_ADDR_HI_SZ(0x10); /* HW always generates 128 bit tag */
+
+	cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1);
+	cpuvaddr[i++] = SE_AES_OP_WRSTALL |
+			SE_AES_OP_LASTBUF | SE_AES_OP_START;
+
+	cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1);
+	cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) |
+			host1x_uclass_incr_syncpt_indx_f(se->syncpt_id);
+
+	return i;
+}
+
+static unsigned int tegra_ctr_prep_cmd(struct tegra_aead_ctx *ctx,
+				       struct tegra_aead_reqctx *rctx)
+{
+	unsigned int i = 0, j;
+	struct tegra_se *se = ctx->se;
+	u32 *cpuvaddr = se->cmdbuf->addr;
+
+	cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT);
+	cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr);
+	for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++)
+		cpuvaddr[i++] = rctx->iv[j];
+
+	cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1);
+	cpuvaddr[i++] = (rctx->inbuf.size / AES_BLOCK_SIZE) - 1;
+	cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6);
+	cpuvaddr[i++] = rctx->config;
+	cpuvaddr[i++] = rctx->crypto_config;
+
+	/* Source address setting */
+	cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr);
+	cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) |
+			SE_ADDR_HI_SZ(rctx->inbuf.size);
+
+	/* Destination address setting */
+	cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr);
+	cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) |
+			SE_ADDR_HI_SZ(rctx->inbuf.size);
+
+	cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1);
+	cpuvaddr[i++] = SE_AES_OP_WRSTALL | SE_AES_OP_LASTBUF |
+			SE_AES_OP_START;
+
+	cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1);
+	cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) |
+			host1x_uclass_incr_syncpt_indx_f(se->syncpt_id);
+
+	dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n",
+		rctx->config, rctx->crypto_config);
+
+	return i;
+}
+
+static int tegra_ccm_do_cbcmac(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx)
+{
+	struct tegra_se *se = ctx->se;
+	int cmdlen;
+
+	rctx->config = tegra234_aes_cfg(SE_ALG_CBC_MAC, rctx->encrypt);
+	rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_CBC_MAC,
+						      rctx->encrypt) |
+						      SE_AES_KEY_INDEX(ctx->key_id);
+
+	/* Prepare command and submit */
+	cmdlen = tegra_cbcmac_prep_cmd(ctx, rctx);
+
+	return tegra_se_host1x_submit(se, cmdlen);
+}
+
+static int tegra_ccm_set_msg_len(u8 *block, unsigned int msglen, int csize)
+{
+	__be32 data;
+
+	memset(block, 0, csize);
+	block += csize;
+
+	if (csize >= 4)
+		csize = 4;
+	else if (msglen > (1 << (8 * csize)))
+		return -EOVERFLOW;
+
+	data = cpu_to_be32(msglen);
+	memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
+
+	return 0;
+}
+
+static int tegra_ccm_format_nonce(struct tegra_aead_reqctx *rctx, u8 *nonce)
+{
+	unsigned int q, t;
+	u8 *q_ptr, *iv = (u8 *)rctx->iv;
+
+	memcpy(nonce, rctx->iv, 16);
+
+	/*** 1. Prepare Flags Octet ***/
+
+	/* Encode t (mac length) */
+	t = rctx->authsize;
+	nonce[0] |= (((t - 2) / 2) << 3);
+
+	/* Adata */
+	if (rctx->assoclen)
+		nonce[0] |= (1 << 6);
+
+	/*** Encode Q - message length ***/
+	q = iv[0] + 1;
+	q_ptr = nonce + 16 - q;
+
+	return tegra_ccm_set_msg_len(q_ptr, rctx->cryptlen, q);
+}
+
+static int tegra_ccm_format_adata(u8 *adata, unsigned int a)
+{
+	int len = 0;
+
+	/* add control info for associated data
+	 * RFC 3610 and NIST Special Publication 800-38C
+	 */
+	if (a < 65280) {
+		*(__be16 *)adata = cpu_to_be16(a);
+		len = 2;
+	} else	{
+		*(__be16 *)adata = cpu_to_be16(0xfffe);
+		*(__be32 *)&adata[2] = cpu_to_be32(a);
+		len = 6;
+	}
+
+	return len;
+}
+
+static int tegra_ccm_add_padding(u8 *buf, unsigned int len)
+{
+	unsigned int padlen = 16 - (len % 16);
+	u8 padding[16] = {0};
+
+	if (padlen == 16)
+		return 0;
+
+	memcpy(buf, padding, padlen);
+
+	return padlen;
+}
+
+static int tegra_ccm_format_blocks(struct tegra_aead_reqctx *rctx)
+{
+	unsigned int alen = 0, offset = 0;
+	u8 nonce[16], adata[16];
+	int ret;
+
+	ret = tegra_ccm_format_nonce(rctx, nonce);
+	if (ret)
+		return ret;
+
+	memcpy(rctx->inbuf.buf, nonce, 16);
+	offset = 16;
+
+	if (rctx->assoclen) {
+		alen = tegra_ccm_format_adata(adata, rctx->assoclen);
+		memcpy(rctx->inbuf.buf + offset, adata, alen);
+		offset += alen;
+
+		scatterwalk_map_and_copy(rctx->inbuf.buf + offset,
+					 rctx->src_sg, 0, rctx->assoclen, 0);
+
+		offset += rctx->assoclen;
+		offset += tegra_ccm_add_padding(rctx->inbuf.buf + offset,
+					 rctx->assoclen + alen);
+	}
+
+	return offset;
+}
+
+static int tegra_ccm_mac_result(struct tegra_se *se, struct tegra_aead_reqctx *rctx)