summaryrefslogtreecommitdiff
path: root/drivers/crypto/inside-secure
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/crypto/inside-secure')
-rw-r--r--drivers/crypto/inside-secure/safexcel.c114
-rw-r--r--drivers/crypto/inside-secure/safexcel.h22
-rw-r--r--drivers/crypto/inside-secure/safexcel_cipher.c5
-rw-r--r--drivers/crypto/inside-secure/safexcel_hash.c258
4 files changed, 266 insertions, 133 deletions
diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c
index 225e74a7f724..d4a81be0d7d2 100644
--- a/drivers/crypto/inside-secure/safexcel.c
+++ b/drivers/crypto/inside-secure/safexcel.c
@@ -235,7 +235,7 @@ static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)
/* Configure DMA tx control */
val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
- val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUG;
+ val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUF;
writel(val,
EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);
@@ -332,7 +332,7 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
val = EIP197_HIA_DSE_CFG_DIS_DEBUG;
val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8);
val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS);
- val |= EIP197_HIA_DSE_CFG_ALLWAYS_BUFFERABLE;
+ val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE;
/* FIXME: instability issues can occur for EIP97 but disabling it impact
* performances.
*/
@@ -354,7 +354,7 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
val |= EIP197_PROTOCOL_ENCRYPT_ONLY | EIP197_PROTOCOL_HASH_ONLY;
val |= EIP197_ALG_AES_ECB | EIP197_ALG_AES_CBC;
val |= EIP197_ALG_SHA1 | EIP197_ALG_HMAC_SHA1;
- val |= EIP197_ALG_SHA2;
+ val |= EIP197_ALG_SHA2 | EIP197_ALG_HMAC_SHA2;
writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN);
/* Command Descriptor Rings prepare */
@@ -432,20 +432,18 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
}
/* Called with ring's lock taken */
-static int safexcel_try_push_requests(struct safexcel_crypto_priv *priv,
- int ring, int reqs)
+static void safexcel_try_push_requests(struct safexcel_crypto_priv *priv,
+ int ring)
{
- int coal = min_t(int, reqs, EIP197_MAX_BATCH_SZ);
+ int coal = min_t(int, priv->ring[ring].requests, EIP197_MAX_BATCH_SZ);
if (!coal)
- return 0;
+ return;
/* Configure when we want an interrupt */
writel(EIP197_HIA_RDR_THRESH_PKT_MODE |
EIP197_HIA_RDR_THRESH_PROC_PKT(coal),
EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_THRESH);
-
- return coal;
}
void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
@@ -490,6 +488,15 @@ handle_req:
if (backlog)
backlog->complete(backlog, -EINPROGRESS);
+ /* In case the send() helper did not issue any command to push
+ * to the engine because the input data was cached, continue to
+ * dequeue other requests as this is valid and not an error.
+ */
+ if (!commands && !results) {
+ kfree(request);
+ continue;
+ }
+
spin_lock_bh(&priv->ring[ring].egress_lock);
list_add_tail(&request->list, &priv->ring[ring].list);
spin_unlock_bh(&priv->ring[ring].egress_lock);
@@ -512,14 +519,13 @@ finalize:
spin_lock_bh(&priv->ring[ring].egress_lock);
+ priv->ring[ring].requests += nreq;
+
if (!priv->ring[ring].busy) {
- nreq -= safexcel_try_push_requests(priv, ring, nreq);
- if (nreq)
- priv->ring[ring].busy = true;
+ safexcel_try_push_requests(priv, ring);
+ priv->ring[ring].busy = true;
}
- priv->ring[ring].requests_left += nreq;
-
spin_unlock_bh(&priv->ring[ring].egress_lock);
/* let the RDR know we have pending descriptors */
@@ -531,25 +537,6 @@ finalize:
EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
}
-void safexcel_free_context(struct safexcel_crypto_priv *priv,
- struct crypto_async_request *req,
- int result_sz)
-{
- struct safexcel_context *ctx = crypto_tfm_ctx(req->tfm);
-
- if (ctx->result_dma)
- dma_unmap_single(priv->dev, ctx->result_dma, result_sz,
- DMA_FROM_DEVICE);
-
- if (ctx->cache) {
- dma_unmap_single(priv->dev, ctx->cache_dma, ctx->cache_sz,
- DMA_TO_DEVICE);
- kfree(ctx->cache);
- ctx->cache = NULL;
- ctx->cache_sz = 0;
- }
-}
-
void safexcel_complete(struct safexcel_crypto_priv *priv, int ring)
{
struct safexcel_command_desc *cdesc;
@@ -623,7 +610,7 @@ static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv
{
struct safexcel_request *sreq;
struct safexcel_context *ctx;
- int ret, i, nreq, ndesc, tot_descs, done;
+ int ret, i, nreq, ndesc, tot_descs, handled = 0;
bool should_complete;
handle_results:
@@ -659,6 +646,7 @@ handle_results:
kfree(sreq);
tot_descs += ndesc;
+ handled++;
}
acknowledge:
@@ -677,11 +665,10 @@ acknowledge:
requests_left:
spin_lock_bh(&priv->ring[ring].egress_lock);
- done = safexcel_try_push_requests(priv, ring,
- priv->ring[ring].requests_left);
+ priv->ring[ring].requests -= handled;
+ safexcel_try_push_requests(priv, ring);
- priv->ring[ring].requests_left -= done;
- if (!done && !priv->ring[ring].requests_left)
+ if (!priv->ring[ring].requests)
priv->ring[ring].busy = false;
spin_unlock_bh(&priv->ring[ring].egress_lock);
@@ -781,6 +768,8 @@ static struct safexcel_alg_template *safexcel_algs[] = {
&safexcel_alg_sha224,
&safexcel_alg_sha256,
&safexcel_alg_hmac_sha1,
+ &safexcel_alg_hmac_sha224,
+ &safexcel_alg_hmac_sha256,
};
static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
@@ -894,29 +883,44 @@ static int safexcel_probe(struct platform_device *pdev)
return PTR_ERR(priv->base);
}
- priv->clk = of_clk_get(dev->of_node, 0);
- if (!IS_ERR(priv->clk)) {
+ priv->clk = devm_clk_get(&pdev->dev, NULL);
+ ret = PTR_ERR_OR_ZERO(priv->clk);
+ /* The clock isn't mandatory */
+ if (ret != -ENOENT) {
+ if (ret)
+ return ret;
+
ret = clk_prepare_enable(priv->clk);
if (ret) {
dev_err(dev, "unable to enable clk (%d)\n", ret);
return ret;
}
- } else {
- /* The clock isn't mandatory */
- if (PTR_ERR(priv->clk) == -EPROBE_DEFER)
- return -EPROBE_DEFER;
+ }
+
+ priv->reg_clk = devm_clk_get(&pdev->dev, "reg");
+ ret = PTR_ERR_OR_ZERO(priv->reg_clk);
+ /* The clock isn't mandatory */
+ if (ret != -ENOENT) {
+ if (ret)
+ goto err_core_clk;
+
+ ret = clk_prepare_enable(priv->reg_clk);
+ if (ret) {
+ dev_err(dev, "unable to enable reg clk (%d)\n", ret);
+ goto err_core_clk;
+ }
}
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
if (ret)
- goto err_clk;
+ goto err_reg_clk;
priv->context_pool = dmam_pool_create("safexcel-context", dev,
sizeof(struct safexcel_context_record),
1, 0);
if (!priv->context_pool) {
ret = -ENOMEM;
- goto err_clk;
+ goto err_reg_clk;
}
safexcel_configure(priv);
@@ -931,12 +935,12 @@ static int safexcel_probe(struct platform_device *pdev)
&priv->ring[i].cdr,
&priv->ring[i].rdr);
if (ret)
- goto err_clk;
+ goto err_reg_clk;
ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL);
if (!ring_irq) {
ret = -ENOMEM;
- goto err_clk;
+ goto err_reg_clk;
}
ring_irq->priv = priv;
@@ -948,7 +952,7 @@ static int safexcel_probe(struct platform_device *pdev)
ring_irq);
if (irq < 0) {
ret = irq;
- goto err_clk;
+ goto err_reg_clk;
}
priv->ring[i].work_data.priv = priv;
@@ -959,10 +963,10 @@ static int safexcel_probe(struct platform_device *pdev)
priv->ring[i].workqueue = create_singlethread_workqueue(wq_name);
if (!priv->ring[i].workqueue) {
ret = -ENOMEM;
- goto err_clk;
+ goto err_reg_clk;
}
- priv->ring[i].requests_left = 0;
+ priv->ring[i].requests = 0;
priv->ring[i].busy = false;
crypto_init_queue(&priv->ring[i].queue,
@@ -980,18 +984,20 @@ static int safexcel_probe(struct platform_device *pdev)
ret = safexcel_hw_init(priv);
if (ret) {
dev_err(dev, "EIP h/w init failed (%d)\n", ret);
- goto err_clk;
+ goto err_reg_clk;
}
ret = safexcel_register_algorithms(priv);
if (ret) {
dev_err(dev, "Failed to register algorithms (%d)\n", ret);
- goto err_clk;
+ goto err_reg_clk;
}
return 0;
-err_clk:
+err_reg_clk:
+ clk_disable_unprepare(priv->reg_clk);
+err_core_clk:
clk_disable_unprepare(priv->clk);
return ret;
}
diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h
index 4e219c21608b..b470a849721f 100644
--- a/drivers/crypto/inside-secure/safexcel.h
+++ b/drivers/crypto/inside-secure/safexcel.h
@@ -135,7 +135,7 @@
/* EIP197_HIA_xDR_DMA_CFG */
#define EIP197_HIA_xDR_WR_RES_BUF BIT(22)
-#define EIP197_HIA_xDR_WR_CTRL_BUG BIT(23)
+#define EIP197_HIA_xDR_WR_CTRL_BUF BIT(23)
#define EIP197_HIA_xDR_WR_OWN_BUF BIT(24)
#define EIP197_HIA_xDR_CFG_WR_CACHE(n) (((n) & 0x7) << 25)
#define EIP197_HIA_xDR_CFG_RD_CACHE(n) (((n) & 0x7) << 29)
@@ -179,7 +179,7 @@
#define EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(n) ((n) << 0)
#define EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(n) (((n) & 0x7) << 4)
#define EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(n) ((n) << 8)
-#define EIP197_HIA_DSE_CFG_ALLWAYS_BUFFERABLE GENMASK(15, 14)
+#define EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE GENMASK(15, 14)
#define EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(n) ((n) << 16)
#define EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(n) (((n) & 0x7) << 20)
#define EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(n) ((n) << 24)
@@ -525,6 +525,7 @@ struct safexcel_crypto_priv {
void __iomem *base;
struct device *dev;
struct clk *clk;
+ struct clk *reg_clk;
struct safexcel_config config;
enum safexcel_eip_version version;
@@ -551,10 +552,8 @@ struct safexcel_crypto_priv {
struct crypto_queue queue;
spinlock_t queue_lock;
- /* Number of requests in the engine that needs the threshold
- * interrupt to be set up.
- */
- int requests_left;
+ /* Number of requests in the engine. */
+ int requests;
/* The ring is currently handling at least one request */
bool busy;
@@ -580,12 +579,6 @@ struct safexcel_context {
int ring;
bool needs_inv;
bool exit_inv;
-
- /* Used for ahash requests */
- dma_addr_t result_dma;
- void *cache;
- dma_addr_t cache_dma;
- unsigned int cache_sz;
};
/*
@@ -609,9 +602,6 @@ struct safexcel_inv_result {
void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring);
void safexcel_complete(struct safexcel_crypto_priv *priv, int ring);
-void safexcel_free_context(struct safexcel_crypto_priv *priv,
- struct crypto_async_request *req,
- int result_sz);
int safexcel_invalidate_cache(struct crypto_async_request *async,
struct safexcel_crypto_priv *priv,
dma_addr_t ctxr_dma, int ring,
@@ -643,5 +633,7 @@ extern struct safexcel_alg_template safexcel_alg_sha1;
extern struct safexcel_alg_template safexcel_alg_sha224;
extern struct safexcel_alg_template safexcel_alg_sha256;
extern struct safexcel_alg_template safexcel_alg_hmac_sha1;
+extern struct safexcel_alg_template safexcel_alg_hmac_sha224;
+extern struct safexcel_alg_template safexcel_alg_hmac_sha256;
#endif
diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c
index 63a8768ed2ae..bafb60505fab 100644
--- a/drivers/crypto/inside-secure/safexcel_cipher.c
+++ b/drivers/crypto/inside-secure/safexcel_cipher.c
@@ -58,7 +58,8 @@ static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx,
token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
token[0].packet_length = length;
- token[0].stat = EIP197_TOKEN_STAT_LAST_PACKET;
+ token[0].stat = EIP197_TOKEN_STAT_LAST_PACKET |
+ EIP197_TOKEN_STAT_LAST_HASH;
token[0].instructions = EIP197_TOKEN_INS_LAST |
EIP197_TOKEN_INS_TYPE_CRYTO |
EIP197_TOKEN_INS_TYPE_OUTPUT;
@@ -456,7 +457,7 @@ static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm)
queue_work(priv->ring[ring].workqueue,
&priv->ring[ring].work_data.work);
- wait_for_completion_interruptible(&result.completion);
+ wait_for_completion(&result.completion);
if (result.error) {
dev_warn(priv->dev,
diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c
index 122a2a58e98f..317b9e480312 100644
--- a/drivers/crypto/inside-secure/safexcel_hash.c
+++ b/drivers/crypto/inside-secure/safexcel_hash.c
@@ -21,10 +21,9 @@ struct safexcel_ahash_ctx {
struct safexcel_crypto_priv *priv;
u32 alg;
- u32 digest;
- u32 ipad[SHA1_DIGEST_SIZE / sizeof(u32)];
- u32 opad[SHA1_DIGEST_SIZE / sizeof(u32)];
+ u32 ipad[SHA256_DIGEST_SIZE / sizeof(u32)];
+ u32 opad[SHA256_DIGEST_SIZE / sizeof(u32)];
};
struct safexcel_ahash_req {
@@ -34,6 +33,9 @@ struct safexcel_ahash_req {
bool needs_inv;
int nents;
+ dma_addr_t result_dma;
+
+ u32 digest;
u8 state_sz; /* expected sate size, only set once */
u32 state[SHA256_DIGEST_SIZE / sizeof(u32)] __aligned(sizeof(u32));
@@ -42,6 +44,9 @@ struct safexcel_ahash_req {
u64 processed;
u8 cache[SHA256_BLOCK_SIZE] __aligned(sizeof(u32));
+ dma_addr_t cache_dma;
+ unsigned int cache_sz;
+
u8 cache_next[SHA256_BLOCK_SIZE] __aligned(sizeof(u32));
};
@@ -49,6 +54,8 @@ struct safexcel_ahash_export_state {
u64 len;
u64 processed;
+ u32 digest;
+
u32 state[SHA256_DIGEST_SIZE / sizeof(u32)];
u8 cache[SHA256_BLOCK_SIZE];
};
@@ -82,9 +89,9 @@ static void safexcel_context_control(struct safexcel_ahash_ctx *ctx,
cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_HASH_OUT;
cdesc->control_data.control0 |= ctx->alg;
- cdesc->control_data.control0 |= ctx->digest;
+ cdesc->control_data.control0 |= req->digest;
- if (ctx->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) {
+ if (req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) {
if (req->processed) {
if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(6);
@@ -112,12 +119,12 @@ static void safexcel_context_control(struct safexcel_ahash_ctx *ctx,
if (req->finish)
ctx->base.ctxr->data[i] = cpu_to_le32(req->processed / blocksize);
}
- } else if (ctx->digest == CONTEXT_CONTROL_DIGEST_HMAC) {
- cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(10);
+ } else if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC) {
+ cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(2 * req->state_sz / sizeof(u32));
- memcpy(ctx->base.ctxr->data, ctx->ipad, digestsize);
- memcpy(ctx->base.ctxr->data + digestsize / sizeof(u32),
- ctx->opad, digestsize);
+ memcpy(ctx->base.ctxr->data, ctx->ipad, req->state_sz);
+ memcpy(ctx->base.ctxr->data + req->state_sz / sizeof(u32),
+ ctx->opad, req->state_sz);
}
}
@@ -149,16 +156,26 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
safexcel_complete(priv, ring);
spin_unlock_bh(&priv->ring[ring].egress_lock);
- if (sreq->finish)
- memcpy(areq->result, sreq->state,
- crypto_ahash_digestsize(ahash));
-
if (sreq->nents) {
dma_unmap_sg(priv->dev, areq->src, sreq->nents, DMA_TO_DEVICE);
sreq->nents = 0;
}
- safexcel_free_context(priv, async, sreq->state_sz);
+ if (sreq->result_dma) {
+ dma_unmap_single(priv->dev, sreq->result_dma, sreq->state_sz,
+ DMA_FROM_DEVICE);
+ sreq->result_dma = 0;
+ }
+
+ if (sreq->cache_dma) {
+ dma_unmap_single(priv->dev, sreq->cache_dma, sreq->cache_sz,
+ DMA_TO_DEVICE);
+ sreq->cache_dma = 0;
+ }
+
+ if (sreq->finish)
+ memcpy(areq->result, sreq->state,
+ crypto_ahash_digestsize(ahash));
cache_len = sreq->len - sreq->processed;
if (cache_len)
@@ -184,7 +201,7 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
int i, queued, len, cache_len, extra, n_cdesc = 0, ret = 0;
queued = len = req->len - req->processed;
- if (queued < crypto_ahash_blocksize(ahash))
+ if (queued <= crypto_ahash_blocksize(ahash))
cache_len = queued;
else
cache_len = queued - areq->nbytes;
@@ -198,7 +215,7 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
/* If this is not the last request and the queued data
* is a multiple of a block, cache the last one for now.
*/
- extra = queued - crypto_ahash_blocksize(ahash);
+ extra = crypto_ahash_blocksize(ahash);
if (extra) {
sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
@@ -220,24 +237,17 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
/* Add a command descriptor for the cached data, if any */
if (cache_len) {
- ctx->base.cache = kzalloc(cache_len, EIP197_GFP_FLAGS(*async));
- if (!ctx->base.cache) {
- ret = -ENOMEM;
- goto unlock;
- }
- memcpy(ctx->base.cache, req->cache, cache_len);
- ctx->base.cache_dma = dma_map_single(priv->dev, ctx->base.cache,
- cache_len, DMA_TO_DEVICE);
- if (dma_mapping_error(priv->dev, ctx->base.cache_dma)) {
- ret = -EINVAL;
- goto free_cache;
+ req->cache_dma = dma_map_single(priv->dev, req->cache,
+ cache_len, DMA_TO_DEVICE);
+ if (dma_mapping_error(priv->dev, req->cache_dma)) {
+ spin_unlock_bh(&priv->ring[ring].egress_lock);
+ return -EINVAL;
}
- ctx->base.cache_sz = cache_len;
+ req->cache_sz = cache_len;
first_cdesc = safexcel_add_cdesc(priv, ring, 1,
(cache_len == len),
- ctx->base.cache_dma,
- cache_len, len,
+ req->cache_dma, cache_len, len,
ctx->base.ctxr_dma);
if (IS_ERR(first_cdesc)) {
ret = PTR_ERR(first_cdesc);
@@ -271,7 +281,7 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
sglen, len, ctx->base.ctxr_dma);
if (IS_ERR(cdesc)) {
ret = PTR_ERR(cdesc);
- goto cdesc_rollback;
+ goto unmap_sg;
}
n_cdesc++;
@@ -291,19 +301,19 @@ send_command:
/* Add the token */
safexcel_hash_token(first_cdesc, len, req->state_sz);
- ctx->base.result_dma = dma_map_single(priv->dev, req->state,
- req->state_sz, DMA_FROM_DEVICE);
- if (dma_mapping_error(priv->dev, ctx->base.result_dma)) {
+ req->result_dma = dma_map_single(priv->dev, req->state, req->state_sz,
+ DMA_FROM_DEVICE);
+ if (dma_mapping_error(priv->dev, req->result_dma)) {
ret = -EINVAL;
- goto cdesc_rollback;
+ goto unmap_sg;
}
/* Add a result descriptor */
- rdesc = safexcel_add_rdesc(priv, ring, 1, 1, ctx->base.result_dma,
+ rdesc = safexcel_add_rdesc(priv, ring, 1, 1, req->result_dma,
req->state_sz);
if (IS_ERR(rdesc)) {
ret = PTR_ERR(rdesc);
- goto cdesc_rollback;
+ goto unmap_result;
}
spin_unlock_bh(&priv->ring[ring].egress_lock);
@@ -315,20 +325,21 @@ send_command:
*results = 1;
return 0;
+unmap_result:
+ dma_unmap_single(priv->dev, req->result_dma, req->state_sz,
+ DMA_FROM_DEVICE);
+unmap_sg:
+ dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE);
cdesc_rollback:
for (i = 0; i < n_cdesc; i++)
safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
unmap_cache:
- if (ctx->base.cache_dma) {
- dma_unmap_single(priv->dev, ctx->base.cache_dma,
- ctx->base.cache_sz, DMA_TO_DEVICE);
- ctx->base.cache_sz = 0;
+ if (req->cache_dma) {
+ dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz,
+ DMA_TO_DEVICE);
+ req->cache_sz = 0;
}
-free_cache:
- kfree(ctx->base.cache);
- ctx->base.cache = NULL;
-unlock:
spin_unlock_bh(&priv->ring[ring].egress_lock);
return ret;
}
@@ -493,7 +504,7 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
queue_work(priv->ring[ring].workqueue,
&priv->ring[ring].work_data.work);
- wait_for_completion_interruptible(&result.completion);
+ wait_for_completion(&result.completion);
if (result.error) {
dev_warn(priv->dev, "hash: completion error (%d)\n",
@@ -550,7 +561,7 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq)
if (ctx->base.ctxr) {
if (priv->version == EIP197 &&
!ctx->base.needs_inv && req->processed &&
- ctx->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED)
+ req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED)
/* We're still setting needs_inv here, even though it is
* cleared right away, because the needs_inv flag can be
* set in other functions and we want to keep the same
@@ -585,7 +596,6 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq)
static int safexcel_ahash_update(struct ahash_request *areq)
{
- struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
@@ -601,7 +611,7 @@ static int safexcel_ahash_update(struct ahash_request *areq)
* We're not doing partial updates when performing an hmac request.
* Everything will be handled by the final() call.
*/
- if (ctx->digest == CONTEXT_CONTROL_DIGEST_HMAC)
+ if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC)
return 0;
if (req->hmac)
@@ -660,6 +670,8 @@ static int safexcel_ahash_export(struct ahash_request *areq, void *out)
export->len = req->len;
export->processed = req->processed;
+ export->digest = req->digest;
+
memcpy(export->state, req->state, req->state_sz);
memcpy(export->cache, req->cache, crypto_ahash_blocksize(ahash));
@@ -680,6 +692,8 @@ static int safexcel_ahash_import(struct ahash_request *areq, const void *in)
req->len = export->len;
req->processed = export->processed;
+ req->digest = export->digest;
+
memcpy(req->cache, export->cache, crypto_ahash_blocksize(ahash));
memcpy(req->state, export->state, req->state_sz);
@@ -716,7 +730,7 @@ static int safexcel_sha1_init(struct ahash_request *areq)
req->state[4] = SHA1_H4;
ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
- ctx->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
+ req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
req->state_sz = SHA1_DIGEST_SIZE;
return 0;
@@ -783,10 +797,10 @@ struct safexcel_alg_template safexcel_alg_sha1 = {
static int safexcel_hmac_sha1_init(struct ahash_request *areq)
{
- struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
+ struct safexcel_ahash_req *req = ahash_request_ctx(areq);
safexcel_sha1_init(areq);
- ctx->digest = CONTEXT_CONTROL_DIGEST_HMAC;
+ req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
return 0;
}
@@ -839,7 +853,7 @@ static int safexcel_hmac_init_pad(struct ahash_request *areq,
init_completion(&result.completion);
ret = crypto_ahash_digest(areq);
- if (ret == -EINPROGRESS) {
+ if (ret == -EINPROGRESS || ret == -EBUSY) {
wait_for_completion_interruptible(&result.completion);
ret = result.error;
}
@@ -949,20 +963,21 @@ free_ahash:
return ret;
}
-static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
- unsigned int keylen)
+static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen, const char *alg,
+ unsigned int state_sz)
{
struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
struct safexcel_crypto_priv *priv = ctx->priv;
struct safexcel_ahash_export_state istate, ostate;
int ret, i;
- ret = safexcel_hmac_setkey("safexcel-sha1", key, keylen, &istate, &ostate);
+ ret = safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate);
if (ret)
return ret;
if (priv->version == EIP197 && ctx->base.ctxr) {
- for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) {
+ for (i = 0; i < state_sz / sizeof(u32); i++) {
if (ctx->ipad[i] != le32_to_cpu(istate.state[i]) ||
ctx->opad[i] != le32_to_cpu(ostate.state[i])) {
ctx->base.needs_inv = true;
@@ -971,12 +986,19 @@ static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
}
}
- memcpy(ctx->ipad, &istate.state, SHA1_DIGEST_SIZE);
- memcpy(ctx->opad, &ostate.state, SHA1_DIGEST_SIZE);
+ memcpy(ctx->ipad, &istate.state, state_sz);
+ memcpy(ctx->opad, &ostate.state, state_sz);
return 0;
}
+static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha1",
+ SHA1_DIGEST_SIZE);
+}
+
struct safexcel_alg_template safexcel_alg_hmac_sha1 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
.alg.ahash = {
@@ -1024,7 +1046,7 @@ static int safexcel_sha256_init(struct ahash_request *areq)
req->state[7] = SHA256_H7;
ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
- ctx->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
+ req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
req->state_sz = SHA256_DIGEST_SIZE;
return 0;
@@ -1086,7 +1108,7 @@ static int safexcel_sha224_init(struct ahash_request *areq)
req->state[7] = SHA224_H7;
ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
- ctx->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
+ req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
req->state_sz = SHA256_DIGEST_SIZE;
return 0;
@@ -1130,3 +1152,115 @@ struct safexcel_alg_template safexcel_alg_sha224 = {
},
},
};
+
+static int safexcel_hmac_sha224_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha224",
+ SHA256_DIGEST_SIZE);
+}
+
+static int safexcel_hmac_sha224_init(struct ahash_request *areq)
+{
+ struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+
+ safexcel_sha224_init(areq);
+ req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
+ return 0;
+}
+
+static int safexcel_hmac_sha224_digest(struct ahash_request *areq)
+{
+ int ret = safexcel_hmac_sha224_init(areq);
+
+ if (ret)
+ return ret;
+
+ return safexcel_ahash_finup(areq);
+}
+
+struct safexcel_alg_template safexcel_alg_hmac_sha224 = {
+ .type = SAFEXCEL_ALG_TYPE_AHASH,
+ .alg.ahash = {
+ .init = safexcel_hmac_sha224_init,
+ .update = safexcel_ahash_update,
+ .final = safexcel_ahash_final,
+ .finup = safexcel_ahash_finup,
+ .digest = safexcel_hmac_sha224_digest,
+ .setkey = safexcel_hmac_sha224_setkey,
+ .export = safexcel_ahash_export,
+ .import = safexcel_ahash_import,
+ .halg = {
+ .digestsize = SHA224_DIGEST_SIZE,
+ .statesize = sizeof(struct safexcel_ahash_export_state),
+ .base = {
+ .cra_name = "hmac(sha224)",
+ .cra_driver_name = "safexcel-hmac-sha224",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = SHA224_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
+ .cra_init = safexcel_ahash_cra_init,
+ .cra_exit = safexcel_ahash_cra_exit,
+ .cra_module = THIS_MODULE,
+ },
+ },
+ },
+};
+
+static int safexcel_hmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha256",
+ SHA256_DIGEST_SIZE);
+}
+
+static int safexcel_hmac_sha256_init(struct ahash_request *areq)
+{
+ struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+
+ safexcel_sha256_init(areq);
+ req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
+ return 0;
+}
+
+static int safexcel_hmac_sha256_digest(struct ahash_request *areq)
+{
+ int ret = safexcel_hmac_sha256_init(areq);
+
+ if (ret)
+ return ret;
+
+ return safexcel_ahash_finup(areq);
+}
+
+struct safexcel_alg_template safexcel_alg_hmac_sha256 = {
+ .type = SAFEXCEL_ALG_TYPE_AHASH,
+ .alg.ahash = {
+ .init = safexcel_hmac_sha256_init,
+ .update = safexcel_ahash_update,
+ .final = safexcel_ahash_final,
+ .finup = safexcel_ahash_finup,
+ .digest = safexcel_hmac_sha256_digest,
+ .setkey = safexcel_hmac_sha256_setkey,
+ .export = safexcel_ahash_export,
+ .import = safexcel_ahash_import,
+ .halg = {
+ .digestsize = SHA256_DIGEST_SIZE,
+ .statesize = sizeof(struct safexcel_ahash_export_state),
+ .base = {
+ .cra_name = "hmac(sha256)",
+ .cra_driver_name = "safexcel-hmac-sha256",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
+ .cra_init = safexcel_ahash_cra_init,
+ .cra_exit = safexcel_ahash_cra_exit,
+ .cra_module = THIS_MODULE,
+ },
+ },
+ },
+};
generated by cgit v1.2.3 (git 2.39.0) at 2025-01-16 23:02:46 +0300