From daf88f3757ecb7aed4dd05a1ca8c8b4ac378d6c3 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 7 Oct 2020 13:43:45 +0100 Subject: crypto: qat - remove unused function Remove unused function qat_dh_get_params(). This is to fix the following warning when compiling the driver with CC=clang W=1 drivers/crypto/qat/qat_common/qat_asym_algs.c:207:34: warning: unused function 'qat_dh_get_params' [-Wunused-function] Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_asym_algs.c | 5 ----- 1 file changed, 5 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index 846569ec9066..f112078be868 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -204,11 +204,6 @@ static unsigned long qat_dh_fn_id(unsigned int len, bool g2) }; } -static inline struct qat_dh_ctx *qat_dh_get_params(struct crypto_kpp *tfm) -{ - return kpp_tfm_ctx(tfm); -} - static int qat_dh_compute_value(struct kpp_request *req) { struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); -- cgit v1.2.3 From 195404db27f9533c71fdcb78d32a77075c2cb4a2 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Thu, 8 Oct 2020 09:34:55 +0000 Subject: crypto: talitos - Endianess in current_desc_hdr() current_desc_hdr() compares the value of the current descriptor with the next_desc member of the talitos_desc struct. While the current descriptor is obtained from in_be32() which return CPU ordered bytes, next_desc member is in big endian order. Convert the current descriptor into big endian before comparing it with next_desc. This fixes a sparse warning. Fixes: 37b5e8897eb5 ("crypto: talitos - chain in buffered data for ahash on SEC1") Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 66773892f665..1de6b0138126 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -478,7 +478,7 @@ static u32 current_desc_hdr(struct device *dev, int ch) iter = tail; while (priv->chan[ch].fifo[iter].dma_desc != cur_desc && - priv->chan[ch].fifo[iter].desc->next_desc != cur_desc) { + priv->chan[ch].fifo[iter].desc->next_desc != cpu_to_be32(cur_desc)) { iter = (iter + 1) & (priv->fifo_len - 1); if (iter == tail) { dev_err(dev, "couldn't locate current descriptor\n"); @@ -486,7 +486,7 @@ static u32 current_desc_hdr(struct device *dev, int ch) } } - if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc) { + if (priv->chan[ch].fifo[iter].desc->next_desc == cpu_to_be32(cur_desc)) { struct talitos_edesc *edesc; edesc = container_of(priv->chan[ch].fifo[iter].desc, -- cgit v1.2.3 From 0237616173fd363a54bd272aa3bd376faa1d7caa Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Thu, 8 Oct 2020 09:34:56 +0000 Subject: crypto: talitos - Fix return type of current_desc_hdr() current_desc_hdr() returns a u32 but in fact this is a __be32, leading to a lot of sparse warnings. Change the return type to __be32 and ensure it is handled as sure by the caller. Fixes: 3e721aeb3df3 ("crypto: talitos - handle descriptor not found in error path") Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 1de6b0138126..a713a35dc502 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -460,7 +460,7 @@ DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE) /* * locate current (offending) descriptor */ -static u32 current_desc_hdr(struct device *dev, int ch) +static __be32 current_desc_hdr(struct device *dev, int ch) { struct talitos_private *priv = dev_get_drvdata(dev); int tail, iter; @@ -501,13 +501,13 @@ static u32 current_desc_hdr(struct device *dev, int ch) /* * user diagnostics; report root cause of error based on execution unit status */ -static void report_eu_error(struct device *dev, int ch, u32 desc_hdr) +static void report_eu_error(struct device *dev, int ch, __be32 desc_hdr) { struct talitos_private *priv = dev_get_drvdata(dev); int i; if (!desc_hdr) - desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF); + desc_hdr = cpu_to_be32(in_be32(priv->chan[ch].reg + TALITOS_DESCBUF)); switch (desc_hdr & DESC_HDR_SEL0_MASK) { case DESC_HDR_SEL0_AFEU: -- cgit v1.2.3 From b2161cc0c9c461feca04cc832a5defb0ac24a513 Mon Sep 17 00:00:00 2001 From: Shiju Jose Date: Fri, 9 Oct 2020 09:19:38 +0100 Subject: crypto: hisilicon - Fix doc warnings in sgl.c and qm.c Fix following warnings caused by mismatch between function parameters and function comments. drivers/crypto/hisilicon/sgl.c:256: warning: Excess function parameter 'hw_sgl_dma' description in 'hisi_acc_sg_buf_unmap' drivers/crypto/hisilicon/sgl.c:256: warning: Excess function parameter 'pool' description in 'hisi_acc_sg_buf_unmap' drivers/crypto/hisilicon/qm.c:1849: warning: Function parameter or member 'qp' not described in 'qm_drain_qp' drivers/crypto/hisilicon/qm.c:2420: warning: Function parameter or member 'qm' not described in 'hisi_qm_set_vft' drivers/crypto/hisilicon/qm.c:2420: warning: Function parameter or member 'fun_num' not described in 'hisi_qm_set_vft' drivers/crypto/hisilicon/qm.c:2420: warning: Function parameter or member 'base' not described in 'hisi_qm_set_vft' drivers/crypto/hisilicon/qm.c:2420: warning: Function parameter or member 'number' not described in 'hisi_qm_set_vft' drivers/crypto/hisilicon/qm.c:2620: warning: Function parameter or member 'qm' not described in 'qm_clear_queues' Signed-off-by: Shiju Jose Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 13 +++++++++++++ drivers/crypto/hisilicon/sgl.c | 2 -- 2 files changed, 13 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 530f23116d7c..050fe4e74523 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -1843,6 +1843,9 @@ int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg) EXPORT_SYMBOL_GPL(hisi_qm_start_qp); /** + * qm_drain_qp() - Drain a qp. + * @qp: The qp we want to drain. + * * Determine whether the queue is cleared by judging the tail pointers of * sq and cq. */ @@ -2486,6 +2489,12 @@ int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number) EXPORT_SYMBOL_GPL(hisi_qm_get_vft); /** + * hisi_qm_set_vft() - Set vft to a qm. + * @qm: The qm we want to set its vft. + * @fun_num: The function number. + * @base: The base number of queue in vft. + * @number: The number of queues in vft. + * * This function is alway called in PF driver, it is used to assign queues * among PF and VFs. * @@ -2690,7 +2699,11 @@ static int qm_stop_started_qp(struct hisi_qm *qm) return 0; } + /** + * qm_clear_queues() - Clear all queues memory in a qm. + * @qm: The qm in which the queues will be cleared. + * * This function clears all queues memory in a qm. Reset of accelerator can * use this to clear queues. */ diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c index 725a739800b0..3bff6394acaf 100644 --- a/drivers/crypto/hisilicon/sgl.c +++ b/drivers/crypto/hisilicon/sgl.c @@ -246,8 +246,6 @@ EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_map_to_hw_sgl); * @dev: The device which hw sgl belongs to. * @sgl: Related scatterlist. * @hw_sgl: Virtual address of hw sgl. - * @hw_sgl_dma: DMA address of hw sgl. - * @pool: Pool which hw sgl is allocated in. * * This function unmaps allocated hw sgl. */ -- cgit v1.2.3 From c98e233062cd9d0e2f10e445a671f0799daaef67 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Sat, 10 Oct 2020 17:47:36 +0100 Subject: crypto: inside-secure - Fix sizeof() mismatch An incorrect sizeof() is being used, sizeof(priv->ring[i].rdr_req) is not correct, it should be sizeof(*priv->ring[i].rdr_req). Note that since the size of ** is the same size as * this is not causing any issues. Addresses-Coverity: ("Sizeof not portable (SIZEOF_MISMATCH)") Fixes: 9744fec95f06 ("crypto: inside-secure - remove request list to improve performance") Signed-off-by: Colin Ian King Acked-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index eb2418450f12..2e1562108a85 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1639,7 +1639,7 @@ static int safexcel_probe_generic(void *pdev, priv->ring[i].rdr_req = devm_kcalloc(dev, EIP197_DEFAULT_RING_SIZE, - sizeof(priv->ring[i].rdr_req), + sizeof(*priv->ring[i].rdr_req), GFP_KERNEL); if (!priv->ring[i].rdr_req) return -ENOMEM; -- cgit v1.2.3 From e4e37acc3bb0ce6152077e24cf9faad71f3c10b6 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Mon, 12 Oct 2020 21:38:17 +0100 Subject: crypto: qat - update IV in software Do IV update calculations in software for AES-CBC and AES-CTR. This allows to embed the IV on the request descriptor and removes the allocation of the IV buffer in the data path. In addition, this change allows the support of QAT devices that are not capable of updating the IV buffer when performing an AES-CBC or AES-CTR operation. Signed-off-by: Marco Chiappero Co-developed-by: Mateusz Polrola Signed-off-by: Mateusz Polrola Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Tested-by: Indrasena Reddy Gali Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_algs.c | 136 +++++++++++++++++------------ drivers/crypto/qat/qat_common/qat_crypto.h | 11 ++- 2 files changed, 89 insertions(+), 58 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index d552dbcfe0a0..a38afc61f6d2 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include "adf_accel_devices.h" @@ -90,6 +91,7 @@ struct qat_alg_skcipher_ctx { struct qat_crypto_instance *inst; struct crypto_skcipher *ftfm; bool fallback; + int mode; }; static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg) @@ -214,24 +216,7 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, return 0; } -static void qat_alg_init_hdr_iv_updt(struct icp_qat_fw_comn_req_hdr *header) -{ - ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, - ICP_QAT_FW_CIPH_IV_64BIT_PTR); - ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_UPDATE_STATE); -} - -static void qat_alg_init_hdr_no_iv_updt(struct icp_qat_fw_comn_req_hdr *header) -{ - ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, - ICP_QAT_FW_CIPH_IV_16BYTE_DATA); - ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_UPDATE_STATE); -} - -static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header, - int aead) +static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) { header->hdr_flags = ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); @@ -241,12 +226,12 @@ static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header, QAT_COMN_PTR_TYPE_SGL); ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags, ICP_QAT_FW_LA_PARTIAL_NONE); - if (aead) - qat_alg_init_hdr_no_iv_updt(header); - else - qat_alg_init_hdr_iv_updt(header); + ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, + ICP_QAT_FW_CIPH_IV_16BYTE_DATA); ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, ICP_QAT_FW_LA_NO_PROTO); + ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_UPDATE_STATE); } static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm, @@ -281,7 +266,7 @@ static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm, return -EFAULT; /* Request setup */ - qat_alg_init_common_hdr(header, 1); + qat_alg_init_common_hdr(header); header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH; ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, ICP_QAT_FW_LA_DIGEST_IN_BUFFER); @@ -368,7 +353,7 @@ static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm, return -EFAULT; /* Request setup */ - qat_alg_init_common_hdr(header, 1); + qat_alg_init_common_hdr(header); header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER; ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, ICP_QAT_FW_LA_DIGEST_IN_BUFFER); @@ -432,7 +417,7 @@ static void qat_alg_skcipher_init_com(struct qat_alg_skcipher_ctx *ctx, struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl; memcpy(cd->aes.key, key, keylen); - qat_alg_init_common_hdr(header, 0); + qat_alg_init_common_hdr(header); header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER; cd_pars->u.s.content_desc_params_sz = sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3; @@ -787,6 +772,61 @@ static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp, areq->base.complete(&areq->base, res); } +static void qat_alg_update_iv_ctr_mode(struct qat_crypto_request *qat_req) +{ + struct skcipher_request *sreq = qat_req->skcipher_req; + u64 iv_lo_prev; + u64 iv_lo; + u64 iv_hi; + + memcpy(qat_req->iv, sreq->iv, AES_BLOCK_SIZE); + + iv_lo = be64_to_cpu(qat_req->iv_lo); + iv_hi = be64_to_cpu(qat_req->iv_hi); + + iv_lo_prev = iv_lo; + iv_lo += DIV_ROUND_UP(sreq->cryptlen, AES_BLOCK_SIZE); + if (iv_lo < iv_lo_prev) + iv_hi++; + + qat_req->iv_lo = cpu_to_be64(iv_lo); + qat_req->iv_hi = cpu_to_be64(iv_hi); +} + +static void qat_alg_update_iv_cbc_mode(struct qat_crypto_request *qat_req) +{ + struct skcipher_request *sreq = qat_req->skcipher_req; + int offset = sreq->cryptlen - AES_BLOCK_SIZE; + struct scatterlist *sgl; + + if (qat_req->encryption) + sgl = sreq->dst; + else + sgl = sreq->src; + + scatterwalk_map_and_copy(qat_req->iv, sgl, offset, AES_BLOCK_SIZE, 0); +} + +static void qat_alg_update_iv(struct qat_crypto_request *qat_req) +{ + struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx; + struct device *dev = &GET_DEV(ctx->inst->accel_dev); + + switch (ctx->mode) { + case ICP_QAT_HW_CIPHER_CTR_MODE: + qat_alg_update_iv_ctr_mode(qat_req); + break; + case ICP_QAT_HW_CIPHER_CBC_MODE: + qat_alg_update_iv_cbc_mode(qat_req); + break; + case ICP_QAT_HW_CIPHER_XTS_MODE: + break; + default: + dev_warn(dev, "Unsupported IV update for cipher mode %d\n", + ctx->mode); + } +} + static void qat_skcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp, struct qat_crypto_request *qat_req) { @@ -794,16 +834,16 @@ static void qat_skcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp, struct qat_crypto_instance *inst = ctx->inst; struct skcipher_request *sreq = qat_req->skcipher_req; u8 stat_filed = qat_resp->comn_resp.comn_status; - struct device *dev = &GET_DEV(ctx->inst->accel_dev); int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed); qat_alg_free_bufl(inst, qat_req); if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK)) res = -EINVAL; + if (qat_req->encryption) + qat_alg_update_iv(qat_req); + memcpy(sreq->iv, qat_req->iv, AES_BLOCK_SIZE); - dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, - qat_req->iv_paddr); sreq->base.complete(&sreq->base, res); } @@ -981,6 +1021,8 @@ static int qat_alg_skcipher_setkey(struct crypto_skcipher *tfm, { struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); + ctx->mode = mode; + if (ctx->enc_cd) return qat_alg_skcipher_rekey(ctx, key, keylen, mode); else @@ -1035,23 +1077,14 @@ static int qat_alg_skcipher_encrypt(struct skcipher_request *req) struct qat_crypto_request *qat_req = skcipher_request_ctx(req); struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_bulk_req *msg; - struct device *dev = &GET_DEV(ctx->inst->accel_dev); int ret, ctr = 0; if (req->cryptlen == 0) return 0; - qat_req->iv = dma_alloc_coherent(dev, AES_BLOCK_SIZE, - &qat_req->iv_paddr, GFP_ATOMIC); - if (!qat_req->iv) - return -ENOMEM; - ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req); - if (unlikely(ret)) { - dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, - qat_req->iv_paddr); + if (unlikely(ret)) return ret; - } msg = &qat_req->req; *msg = ctx->enc_fw_req; @@ -1061,19 +1094,18 @@ static int qat_alg_skcipher_encrypt(struct skcipher_request *req) qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req; qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; + qat_req->encryption = true; cipher_param = (void *)&qat_req->req.serv_specif_rqpars; cipher_param->cipher_length = req->cryptlen; cipher_param->cipher_offset = 0; - cipher_param->u.s.cipher_IV_ptr = qat_req->iv_paddr; - memcpy(qat_req->iv, req->iv, AES_BLOCK_SIZE); + memcpy(cipher_param->u.cipher_IV_array, req->iv, AES_BLOCK_SIZE); + do { ret = adf_send_message(ctx->inst->sym_tx, (u32 *)msg); } while (ret == -EAGAIN && ctr++ < 10); if (ret == -EAGAIN) { qat_alg_free_bufl(ctx->inst, qat_req); - dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, - qat_req->iv_paddr); return -EBUSY; } return -EINPROGRESS; @@ -1113,23 +1145,14 @@ static int qat_alg_skcipher_decrypt(struct skcipher_request *req) struct qat_crypto_request *qat_req = skcipher_request_ctx(req); struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_bulk_req *msg; - struct device *dev = &GET_DEV(ctx->inst->accel_dev); int ret, ctr = 0; if (req->cryptlen == 0) return 0; - qat_req->iv = dma_alloc_coherent(dev, AES_BLOCK_SIZE, - &qat_req->iv_paddr, GFP_ATOMIC); - if (!qat_req->iv) - return -ENOMEM; - ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req); - if (unlikely(ret)) { - dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, - qat_req->iv_paddr); + if (unlikely(ret)) return ret; - } msg = &qat_req->req; *msg = ctx->dec_fw_req; @@ -1139,19 +1162,20 @@ static int qat_alg_skcipher_decrypt(struct skcipher_request *req) qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req; qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; + qat_req->encryption = false; cipher_param = (void *)&qat_req->req.serv_specif_rqpars; cipher_param->cipher_length = req->cryptlen; cipher_param->cipher_offset = 0; - cipher_param->u.s.cipher_IV_ptr = qat_req->iv_paddr; - memcpy(qat_req->iv, req->iv, AES_BLOCK_SIZE); + memcpy(cipher_param->u.cipher_IV_array, req->iv, AES_BLOCK_SIZE); + + qat_alg_update_iv(qat_req); + do { ret = adf_send_message(ctx->inst->sym_tx, (u32 *)msg); } while (ret == -EAGAIN && ctr++ < 10); if (ret == -EAGAIN) { qat_alg_free_bufl(ctx->inst, qat_req); - dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, - qat_req->iv_paddr); return -EBUSY; } return -EINPROGRESS; diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h index 12682d1e9f5f..8d11e94cbf08 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.h +++ b/drivers/crypto/qat/qat_common/qat_crypto.h @@ -3,6 +3,7 @@ #ifndef _QAT_CRYPTO_INSTANCE_H_ #define _QAT_CRYPTO_INSTANCE_H_ +#include #include #include #include "adf_accel_devices.h" @@ -44,8 +45,14 @@ struct qat_crypto_request { struct qat_crypto_request_buffs buf; void (*cb)(struct icp_qat_fw_la_resp *resp, struct qat_crypto_request *req); - void *iv; - dma_addr_t iv_paddr; + union { + struct { + __be64 iv_hi; + __be64 iv_lo; + }; + u8 iv[AES_BLOCK_SIZE]; + }; + bool encryption; }; #endif -- cgit v1.2.3 From 7b07ed5042c5d21467af5aa055f2b49b2e661a83 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:18 +0100 Subject: crypto: qat - mask device capabilities with soft straps Enable acceleration engines (AEs) and accelerators based on soft straps and fuses. When looping with a number of AEs or accelerators, ignore the ones that are disabled. This patch is based on earlier work done by Conor McLoughlin. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 34 +++++++++++++++++----- drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h | 1 + drivers/crypto/qat/qat_c3xxx/adf_drv.c | 6 ++-- .../crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c | 4 +-- drivers/crypto/qat/qat_c3xxxvf/adf_drv.c | 4 +-- drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c | 34 +++++++++++++++++----- drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h | 1 + drivers/crypto/qat/qat_c62x/adf_drv.c | 6 ++-- drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c | 4 +-- drivers/crypto/qat/qat_c62xvf/adf_drv.c | 4 +-- drivers/crypto/qat/qat_common/adf_accel_devices.h | 5 ++-- drivers/crypto/qat/qat_common/qat_hal.c | 27 +++++++++-------- .../crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 20 ++++++++----- drivers/crypto/qat/qat_dh895xcc/adf_drv.c | 4 +-- .../qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c | 4 +-- drivers/crypto/qat/qat_dh895xccvf/adf_drv.c | 4 +-- 16 files changed, 109 insertions(+), 53 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index aee494d3da52..4b2f5aa83391 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -17,15 +17,33 @@ static struct adf_hw_device_class c3xxx_class = { .instances = 0 }; -static u32 get_accel_mask(u32 fuse) +static u32 get_accel_mask(struct adf_hw_device_data *self) { - return (~fuse) >> ADF_C3XXX_ACCELERATORS_REG_OFFSET & - ADF_C3XXX_ACCELERATORS_MASK; + u32 straps = self->straps; + u32 fuses = self->fuses; + u32 accel; + + accel = ~(fuses | straps) >> ADF_C3XXX_ACCELERATORS_REG_OFFSET; + accel &= ADF_C3XXX_ACCELERATORS_MASK; + + return accel; } -static u32 get_ae_mask(u32 fuse) +static u32 get_ae_mask(struct adf_hw_device_data *self) { - return (~fuse) & ADF_C3XXX_ACCELENGINES_MASK; + u32 straps = self->straps; + u32 fuses = self->fuses; + unsigned long disabled; + u32 ae_disable; + int accel; + + /* If an accel is disabled, then disable the corresponding two AEs */ + disabled = ~get_accel_mask(self) & ADF_C3XXX_ACCELERATORS_MASK; + ae_disable = BIT(1) | BIT(0); + for_each_set_bit(accel, &disabled, ADF_C3XXX_MAX_ACCELERATORS) + straps |= ae_disable << (accel << 1); + + return ~(fuses | straps) & ADF_C3XXX_ACCELENGINES_MASK; } static u32 get_num_accels(struct adf_hw_device_data *self) @@ -109,11 +127,13 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_device = accel_dev->hw_device; struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_C3XXX_PMISC_BAR]; + unsigned long accel_mask = hw_device->accel_mask; + unsigned long ae_mask = hw_device->ae_mask; void __iomem *csr = misc_bar->virt_addr; unsigned int val, i; /* Enable Accel Engine error detection & correction */ - for (i = 0; i < hw_device->get_num_aes(hw_device); i++) { + for_each_set_bit(i, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { val = ADF_CSR_RD(csr, ADF_C3XXX_AE_CTX_ENABLES(i)); val |= ADF_C3XXX_ENABLE_AE_ECC_ERR; ADF_CSR_WR(csr, ADF_C3XXX_AE_CTX_ENABLES(i), val); @@ -123,7 +143,7 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) } /* Enable shared memory error detection & correction */ - for (i = 0; i < hw_device->get_num_accels(hw_device); i++) { + for_each_set_bit(i, &accel_mask, ADF_C3XXX_MAX_ACCELERATORS) { val = ADF_CSR_RD(csr, ADF_C3XXX_UERRSSMSH(i)); val |= ADF_C3XXX_ERRSSMSH_EN; ADF_CSR_WR(csr, ADF_C3XXX_UERRSSMSH(i), val); diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h index 8b5dd2c94ebf..94097816f68a 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h @@ -18,6 +18,7 @@ #define ADF_C3XXX_SMIAPF1_MASK_OFFSET (0x3A000 + 0x30) #define ADF_C3XXX_SMIA0_MASK 0xFFFF #define ADF_C3XXX_SMIA1_MASK 0x1 +#define ADF_C3XXX_SOFTSTRAP_CSR_OFFSET 0x2EC /* Error detection and correction */ #define ADF_C3XXX_AE_CTX_ENABLES(i) (i * 0x1000 + 0x20818) #define ADF_C3XXX_AE_MISC_CONTROL(i) (i * 0x1000 + 0x20960) diff --git a/drivers/crypto/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/qat/qat_c3xxx/adf_drv.c index ed0e8e33fe4b..da6e88026988 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_drv.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_drv.c @@ -126,10 +126,12 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, &hw_data->fuses); + pci_read_config_dword(pdev, ADF_C3XXX_SOFTSTRAP_CSR_OFFSET, + &hw_data->straps); /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); accel_pci_dev->sku = hw_data->get_sku(hw_data); /* If the device has no acceleration engines then ignore it. */ if (!hw_data->accel_mask || !hw_data->ae_mask || diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c index d2fedbd7113c..cdf8c500ef2a 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -11,12 +11,12 @@ static struct adf_hw_device_class c3xxxiov_class = { .instances = 0 }; -static u32 get_accel_mask(u32 fuse) +static u32 get_accel_mask(struct adf_hw_device_data *self) { return ADF_C3XXXIOV_ACCELERATORS_MASK; } -static u32 get_ae_mask(u32 fuse) +static u32 get_ae_mask(struct adf_hw_device_data *self) { return ADF_C3XXXIOV_ACCELENGINES_MASK; } diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c index 456979b136a2..1d1532e8fb6d 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c @@ -119,8 +119,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_c3xxxiov(accel_dev->hw_device); /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); accel_pci_dev->sku = hw_data->get_sku(hw_data); /* Create dev top level debugfs entry */ diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index 844ad5ed33fc..c0b5751e9682 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -22,15 +22,33 @@ static struct adf_hw_device_class c62x_class = { .instances = 0 }; -static u32 get_accel_mask(u32 fuse) +static u32 get_accel_mask(struct adf_hw_device_data *self) { - return (~fuse) >> ADF_C62X_ACCELERATORS_REG_OFFSET & - ADF_C62X_ACCELERATORS_MASK; + u32 straps = self->straps; + u32 fuses = self->fuses; + u32 accel; + + accel = ~(fuses | straps) >> ADF_C62X_ACCELERATORS_REG_OFFSET; + accel &= ADF_C62X_ACCELERATORS_MASK; + + return accel; } -static u32 get_ae_mask(u32 fuse) +static u32 get_ae_mask(struct adf_hw_device_data *self) { - return (~fuse) & ADF_C62X_ACCELENGINES_MASK; + u32 straps = self->straps; + u32 fuses = self->fuses; + unsigned long disabled; + u32 ae_disable; + int accel; + + /* If an accel is disabled, then disable the corresponding two AEs */ + disabled = ~get_accel_mask(self) & ADF_C62X_ACCELERATORS_MASK; + ae_disable = BIT(1) | BIT(0); + for_each_set_bit(accel, &disabled, ADF_C62X_MAX_ACCELERATORS) + straps |= ae_disable << (accel << 1); + + return ~(fuses | straps) & ADF_C62X_ACCELENGINES_MASK; } static u32 get_num_accels(struct adf_hw_device_data *self) @@ -119,11 +137,13 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_device = accel_dev->hw_device; struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_C62X_PMISC_BAR]; + unsigned long accel_mask = hw_device->accel_mask; + unsigned long ae_mask = hw_device->ae_mask; void __iomem *csr = misc_bar->virt_addr; unsigned int val, i; /* Enable Accel Engine error detection & correction */ - for (i = 0; i < hw_device->get_num_aes(hw_device); i++) { + for_each_set_bit(i, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { val = ADF_CSR_RD(csr, ADF_C62X_AE_CTX_ENABLES(i)); val |= ADF_C62X_ENABLE_AE_ECC_ERR; ADF_CSR_WR(csr, ADF_C62X_AE_CTX_ENABLES(i), val); @@ -133,7 +153,7 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) } /* Enable shared memory error detection & correction */ - for (i = 0; i < hw_device->get_num_accels(hw_device); i++) { + for_each_set_bit(i, &accel_mask, ADF_C62X_MAX_ACCELERATORS) { val = ADF_CSR_RD(csr, ADF_C62X_UERRSSMSH(i)); val |= ADF_C62X_ERRSSMSH_EN; ADF_CSR_WR(csr, ADF_C62X_UERRSSMSH(i), val); diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h index 88504d2bf30d..a2e2961a2102 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h @@ -19,6 +19,7 @@ #define ADF_C62X_SMIAPF1_MASK_OFFSET (0x3A000 + 0x30) #define ADF_C62X_SMIA0_MASK 0xFFFF #define ADF_C62X_SMIA1_MASK 0x1 +#define ADF_C62X_SOFTSTRAP_CSR_OFFSET 0x2EC /* Error detection and correction */ #define ADF_C62X_AE_CTX_ENABLES(i) (i * 0x1000 + 0x20818) #define ADF_C62X_AE_MISC_CONTROL(i) (i * 0x1000 + 0x20960) diff --git a/drivers/crypto/qat/qat_c62x/adf_drv.c b/drivers/crypto/qat/qat_c62x/adf_drv.c index d8e7c9c25590..3da697a566ad 100644 --- a/drivers/crypto/qat/qat_c62x/adf_drv.c +++ b/drivers/crypto/qat/qat_c62x/adf_drv.c @@ -126,10 +126,12 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, &hw_data->fuses); + pci_read_config_dword(pdev, ADF_C62X_SOFTSTRAP_CSR_OFFSET, + &hw_data->straps); /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); accel_pci_dev->sku = hw_data->get_sku(hw_data); /* If the device has no acceleration engines then ignore it. */ if (!hw_data->accel_mask || !hw_data->ae_mask || diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c index 29fd3f1091ab..a2543f75e81f 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -11,12 +11,12 @@ static struct adf_hw_device_class c62xiov_class = { .instances = 0 }; -static u32 get_accel_mask(u32 fuse) +static u32 get_accel_mask(struct adf_hw_device_data *self) { return ADF_C62XIOV_ACCELERATORS_MASK; } -static u32 get_ae_mask(u32 fuse) +static u32 get_ae_mask(struct adf_hw_device_data *self) { return ADF_C62XIOV_ACCELENGINES_MASK; } diff --git a/drivers/crypto/qat/qat_c62xvf/adf_drv.c b/drivers/crypto/qat/qat_c62xvf/adf_drv.c index b9810f79eb84..04742a6d91ca 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_drv.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_drv.c @@ -119,8 +119,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_c62xiov(accel_dev->hw_device); /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); accel_pci_dev->sku = hw_data->get_sku(hw_data); /* Create dev top level debugfs entry */ diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 06952ece53d9..411a505e1f59 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -104,8 +104,8 @@ struct adf_etr_ring_data; struct adf_hw_device_data { struct adf_hw_device_class *dev_class; - u32 (*get_accel_mask)(u32 fuse); - u32 (*get_ae_mask)(u32 fuse); + u32 (*get_accel_mask)(struct adf_hw_device_data *self); + u32 (*get_ae_mask)(struct adf_hw_device_data *self); u32 (*get_sram_bar_id)(struct adf_hw_device_data *self); u32 (*get_misc_bar_id)(struct adf_hw_device_data *self); u32 (*get_etr_bar_id)(struct adf_hw_device_data *self); @@ -131,6 +131,7 @@ struct adf_hw_device_data { const char *fw_name; const char *fw_mmp_name; u32 fuses; + u32 straps; u32 accel_capabilities_mask; u32 instance_id; u16 accel_mask; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 6b9d47682d04..bc07199459e7 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -346,11 +346,12 @@ static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle, static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle) { + unsigned long ae_mask = handle->hal_handle->ae_mask; unsigned int base_cnt, cur_cnt; unsigned char ae; int times = MAX_RETRY_TIMES; - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); base_cnt &= 0xffff; @@ -384,6 +385,7 @@ int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle, static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) { + unsigned long ae_mask = handle->hal_handle->ae_mask; unsigned int misc_ctl; unsigned char ae; @@ -393,7 +395,7 @@ static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl & (~MC_TIMESTAMP_ENABLE)); - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0); qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0); } @@ -438,6 +440,7 @@ static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) #define SHRAM_INIT_CYCLES 2060 int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) { + unsigned long ae_mask = handle->hal_handle->ae_mask; unsigned int ae_reset_csr; unsigned char ae; unsigned int clk_csr; @@ -464,7 +467,7 @@ int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) goto out_err; /* Set undefined power-up/reset states to reasonable default values */ - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, INIT_CTX_ENABLE_VALUE); qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX, @@ -570,10 +573,11 @@ static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle, static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle) { + unsigned long ae_mask = handle->hal_handle->ae_mask; unsigned char ae; unsigned short reg; - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) { qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS, reg, 0); @@ -585,6 +589,7 @@ static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle) static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) { + unsigned long ae_mask = handle->hal_handle->ae_mask; unsigned char ae; unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX; int times = MAX_RETRY_TIMES; @@ -592,7 +597,7 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) unsigned int savctx = 0; int ret = 0; - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL); csr_val &= ~(1 << MMC_SHARE_CS_BITPOS); qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val); @@ -613,7 +618,7 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0); qat_hal_enable_ctx(handle, ae, ctx_mask); } - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { /* wait for AE to finish */ do { ret = qat_hal_wait_cycles(handle, ae, 20, 1); @@ -654,6 +659,8 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct adf_bar *misc_bar = &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)]; + unsigned long ae_mask = hw_data->ae_mask; + unsigned int csr_val = 0; struct adf_bar *sram_bar; handle = kzalloc(sizeof(*handle), GFP_KERNEL); @@ -689,9 +696,7 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) /* create AE objects */ handle->hal_handle->upc_mask = 0x1ffff; handle->hal_handle->max_ustore = 0x4000; - for (ae = 0; ae < ICP_QAT_UCLO_MAX_AE; ae++) { - if (!(hw_data->ae_mask & (1 << ae))) - continue; + for_each_set_bit(ae, &ae_mask, ICP_QAT_UCLO_MAX_AE) { handle->hal_handle->aes[ae].free_addr = 0; handle->hal_handle->aes[ae].free_size = handle->hal_handle->max_ustore; @@ -714,9 +719,7 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) } /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */ - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - unsigned int csr_val = 0; - + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE); csr_val |= 0x1; qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val); diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index b975c263446d..6a0d01103136 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -24,15 +24,19 @@ static struct adf_hw_device_class dh895xcc_class = { .instances = 0 }; -static u32 get_accel_mask(u32 fuse) +static u32 get_accel_mask(struct adf_hw_device_data *self) { - return (~fuse) >> ADF_DH895XCC_ACCELERATORS_REG_OFFSET & - ADF_DH895XCC_ACCELERATORS_MASK; + u32 fuses = self->fuses; + + return ~fuses >> ADF_DH895XCC_ACCELERATORS_REG_OFFSET & + ADF_DH895XCC_ACCELERATORS_MASK; } -static u32 get_ae_mask(u32 fuse) +static u32 get_ae_mask(struct adf_hw_device_data *self) { - return (~fuse) & ADF_DH895XCC_ACCELENGINES_MASK; + u32 fuses = self->fuses; + + return ~fuses & ADF_DH895XCC_ACCELENGINES_MASK; } static u32 get_num_accels(struct adf_hw_device_data *self) @@ -131,11 +135,13 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_device = accel_dev->hw_device; struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_DH895XCC_PMISC_BAR]; + unsigned long accel_mask = hw_device->accel_mask; + unsigned long ae_mask = hw_device->ae_mask; void __iomem *csr = misc_bar->virt_addr; unsigned int val, i; /* Enable Accel Engine error detection & correction */ - for (i = 0; i < hw_device->get_num_aes(hw_device); i++) { + for_each_set_bit(i, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { val = ADF_CSR_RD(csr, ADF_DH895XCC_AE_CTX_ENABLES(i)); val |= ADF_DH895XCC_ENABLE_AE_ECC_ERR; ADF_CSR_WR(csr, ADF_DH895XCC_AE_CTX_ENABLES(i), val); @@ -145,7 +151,7 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) } /* Enable shared memory error detection & correction */ - for (i = 0; i < hw_device->get_num_accels(hw_device); i++) { + for_each_set_bit(i, &accel_mask, ADF_DH895XCC_MAX_ACCELERATORS) { val = ADF_CSR_RD(csr, ADF_DH895XCC_UERRSSMSH(i)); val |= ADF_DH895XCC_ERRSSMSH_EN; ADF_CSR_WR(csr, ADF_DH895XCC_UERRSSMSH(i), val); diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c index ecb4f6f20e22..d7941bc2bafd 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c @@ -128,8 +128,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) &hw_data->fuses); /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); accel_pci_dev->sku = hw_data->get_sku(hw_data); /* If the device has no acceleration engines then ignore it. */ if (!hw_data->accel_mask || !hw_data->ae_mask || diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c index 5246f0524ca3..737f9132f71a 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -11,12 +11,12 @@ static struct adf_hw_device_class dh895xcciov_class = { .instances = 0 }; -static u32 get_accel_mask(u32 fuse) +static u32 get_accel_mask(struct adf_hw_device_data *self) { return ADF_DH895XCCIOV_ACCELERATORS_MASK; } -static u32 get_ae_mask(u32 fuse) +static u32 get_ae_mask(struct adf_hw_device_data *self) { return ADF_DH895XCCIOV_ACCELENGINES_MASK; } diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c index 404cf9df6922..c972554a755e 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c @@ -119,8 +119,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_dh895xcciov(accel_dev->hw_device); /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); accel_pci_dev->sku = hw_data->get_sku(hw_data); /* Create dev top level debugfs entry */ -- cgit v1.2.3 From 70b9bd3929da903d54dbd7d1ca2465333077cfd9 Mon Sep 17 00:00:00 2001 From: Ahsan Atta Date: Mon, 12 Oct 2020 21:38:19 +0100 Subject: crypto: qat - num_rings_per_bank is device dependent This change is to allow support for QAT devices that may not have 16 rings per bank. The rings structure in bank is allocated dynamically based on the number of banks supported by a device. Note that in the error path in adf_init_bank(), ring->inflights is set to NULL after the free to silence a false positive double free reported by clang scan-build. Signed-off-by: Ahsan Atta Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 1 + .../crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c | 1 + drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c | 1 + drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c | 1 + drivers/crypto/qat/qat_common/adf_accel_devices.h | 3 ++ drivers/crypto/qat/qat_common/adf_transport.c | 42 +++++++++++++++------- .../crypto/qat/qat_common/adf_transport_debug.c | 10 ++++-- .../crypto/qat/qat_common/adf_transport_internal.h | 2 +- .../crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 1 + .../qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c | 1 + 10 files changed, 47 insertions(+), 16 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index 4b2f5aa83391..62b0b290ff85 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -176,6 +176,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->dev_class = &c3xxx_class; hw_data->instance_id = c3xxx_class.instances++; hw_data->num_banks = ADF_C3XXX_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; hw_data->num_accel = ADF_C3XXX_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_C3XXX_MAX_ACCELENGINES; diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c index cdf8c500ef2a..80a355e85a72 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -69,6 +69,7 @@ void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &c3xxxiov_class; hw_data->num_banks = ADF_C3XXXIOV_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; hw_data->num_accel = ADF_C3XXXIOV_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_C3XXXIOV_MAX_ACCELENGINES; diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index c0b5751e9682..1334b43e46e4 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -186,6 +186,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->dev_class = &c62x_class; hw_data->instance_id = c62x_class.instances++; hw_data->num_banks = ADF_C62X_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; hw_data->num_accel = ADF_C62X_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_C62X_MAX_ACCELENGINES; diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c index a2543f75e81f..7725387e58f8 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -69,6 +69,7 @@ void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &c62xiov_class; hw_data->num_banks = ADF_C62XIOV_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; hw_data->num_accel = ADF_C62XIOV_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_C62XIOV_MAX_ACCELENGINES; diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 411a505e1f59..85b423d28f77 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -139,6 +139,7 @@ struct adf_hw_device_data { u16 tx_rings_mask; u8 tx_rx_gap; u8 num_banks; + u8 num_rings_per_bank; u8 num_accel; u8 num_logical_accel; u8 num_engines; @@ -156,6 +157,8 @@ struct adf_hw_device_data { #define GET_BARS(accel_dev) ((accel_dev)->accel_pci_dev.pci_bars) #define GET_HW_DATA(accel_dev) (accel_dev->hw_device) #define GET_MAX_BANKS(accel_dev) (GET_HW_DATA(accel_dev)->num_banks) +#define GET_NUM_RINGS_PER_BANK(accel_dev) \ + GET_HW_DATA(accel_dev)->num_rings_per_bank #define GET_MAX_ACCELENGINES(accel_dev) (GET_HW_DATA(accel_dev)->num_engines) #define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c index 2ad774017200..24ddaaaa55b1 100644 --- a/drivers/crypto/qat/qat_common/adf_transport.c +++ b/drivers/crypto/qat/qat_common/adf_transport.c @@ -190,6 +190,7 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, struct adf_etr_ring_data **ring_ptr) { struct adf_etr_data *transport_data = accel_dev->transport; + u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(accel_dev); struct adf_etr_bank_data *bank; struct adf_etr_ring_data *ring; char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; @@ -219,7 +220,7 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, dev_err(&GET_DEV(accel_dev), "Can't get ring number\n"); return -EFAULT; } - if (ring_num >= ADF_ETR_MAX_RINGS_PER_BANK) { + if (ring_num >= num_rings_per_bank) { dev_err(&GET_DEV(accel_dev), "Invalid ring number\n"); return -EFAULT; } @@ -286,15 +287,15 @@ void adf_remove_ring(struct adf_etr_ring_data *ring) static void adf_ring_response_handler(struct adf_etr_bank_data *bank) { - u32 empty_rings, i; + u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(bank->accel_dev); + unsigned long empty_rings; + int i; empty_rings = READ_CSR_E_STAT(bank->csr_addr, bank->bank_number); empty_rings = ~empty_rings & bank->irq_mask; - for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; ++i) { - if (empty_rings & (1 << i)) - adf_handle_response(&bank->rings[i]); - } + for_each_set_bit(i, &empty_rings, num_rings_per_bank) + adf_handle_response(&bank->rings[i]); } void adf_response_handler(uintptr_t bank_addr) @@ -343,9 +344,12 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, u32 bank_num, void __iomem *csr_addr) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u8 num_rings_per_bank = hw_data->num_rings_per_bank; struct adf_etr_ring_data *ring; struct adf_etr_ring_data *tx_ring; u32 i, coalesc_enabled = 0; + unsigned long ring_mask; + int size; memset(bank, 0, sizeof(*bank)); bank->bank_number = bank_num; @@ -353,6 +357,13 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, bank->accel_dev = accel_dev; spin_lock_init(&bank->lock); + /* Allocate the rings in the bank */ + size = num_rings_per_bank * sizeof(struct adf_etr_ring_data); + bank->rings = kzalloc_node(size, GFP_KERNEL, + dev_to_node(&GET_DEV(accel_dev))); + if (!bank->rings) + return -ENOMEM; + /* Enable IRQ coalescing always. This will allow to use * the optimised flag and coalesc register. * If it is disabled in the config file just use min time value */ @@ -363,7 +374,7 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, else bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME; - for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; i++) { + for (i = 0; i < num_rings_per_bank; i++) { WRITE_CSR_RING_CONFIG(csr_addr, bank_num, i, 0); WRITE_CSR_RING_BASE(csr_addr, bank_num, i, 0); ring = &bank->rings[i]; @@ -394,11 +405,13 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, WRITE_CSR_INT_SRCSEL(csr_addr, bank_num); return 0; err: - for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; i++) { + ring_mask = hw_data->tx_rings_mask; + for_each_set_bit(i, &ring_mask, num_rings_per_bank) { ring = &bank->rings[i]; - if (hw_data->tx_rings_mask & (1 << i)) - kfree(ring->inflights); + kfree(ring->inflights); + ring->inflights = NULL; } + kfree(bank->rings); return -ENOMEM; } @@ -464,11 +477,12 @@ EXPORT_SYMBOL_GPL(adf_init_etr_data); static void cleanup_bank(struct adf_etr_bank_data *bank) { + struct adf_accel_dev *accel_dev = bank->accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u8 num_rings_per_bank = hw_data->num_rings_per_bank; u32 i; - for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; i++) { - struct adf_accel_dev *accel_dev = bank->accel_dev; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; + for (i = 0; i < num_rings_per_bank; i++) { struct adf_etr_ring_data *ring = &bank->rings[i]; if (bank->ring_mask & (1 << i)) @@ -477,6 +491,7 @@ static void cleanup_bank(struct adf_etr_bank_data *bank) if (hw_data->tx_rings_mask & (1 << i)) kfree(ring->inflights); } + kfree(bank->rings); adf_bank_debugfs_rm(bank); memset(bank, 0, sizeof(*bank)); } @@ -507,6 +522,7 @@ void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev) if (etr_data) { adf_cleanup_etr_handles(accel_dev); debugfs_remove(etr_data->debug); + kfree(etr_data->banks->rings); kfree(etr_data->banks); kfree(etr_data); accel_dev->transport = NULL; diff --git a/drivers/crypto/qat/qat_common/adf_transport_debug.c b/drivers/crypto/qat/qat_common/adf_transport_debug.c index dac25ba47260..da79d734c035 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_debug.c +++ b/drivers/crypto/qat/qat_common/adf_transport_debug.c @@ -117,11 +117,14 @@ void adf_ring_debugfs_rm(struct adf_etr_ring_data *ring) static void *adf_bank_start(struct seq_file *sfile, loff_t *pos) { + struct adf_etr_bank_data *bank = sfile->private; + u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(bank->accel_dev); + mutex_lock(&bank_read_lock); if (*pos == 0) return SEQ_START_TOKEN; - if (*pos >= ADF_ETR_MAX_RINGS_PER_BANK) + if (*pos >= num_rings_per_bank) return NULL; return pos; @@ -129,7 +132,10 @@ static void *adf_bank_start(struct seq_file *sfile, loff_t *pos) static void *adf_bank_next(struct seq_file *sfile, void *v, loff_t *pos) { - if (++(*pos) >= ADF_ETR_MAX_RINGS_PER_BANK) + struct adf_etr_bank_data *bank = sfile->private; + u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(bank->accel_dev); + + if (++(*pos) >= num_rings_per_bank) return NULL; return pos; diff --git a/drivers/crypto/qat/qat_common/adf_transport_internal.h b/drivers/crypto/qat/qat_common/adf_transport_internal.h index c7faf4e2d302..501bcf0f1809 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_internal.h +++ b/drivers/crypto/qat/qat_common/adf_transport_internal.h @@ -28,7 +28,7 @@ struct adf_etr_ring_data { }; struct adf_etr_bank_data { - struct adf_etr_ring_data rings[ADF_ETR_MAX_RINGS_PER_BANK]; + struct adf_etr_ring_data *rings; struct tasklet_struct resp_handler; void __iomem *csr_addr; u32 irq_coalesc_timer; diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 6a0d01103136..1f3ea3ba1cee 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -185,6 +185,7 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->dev_class = &dh895xcc_class; hw_data->instance_id = dh895xcc_class.instances++; hw_data->num_banks = ADF_DH895XCC_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; hw_data->num_accel = ADF_DH895XCC_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_DH895XCC_MAX_ACCELENGINES; diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c index 737f9132f71a..eca144bc1d67 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -69,6 +69,7 @@ void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &dh895xcciov_class; hw_data->num_banks = ADF_DH895XCCIOV_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; hw_data->num_accel = ADF_DH895XCCIOV_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_DH895XCCIOV_MAX_ACCELENGINES; -- cgit v1.2.3 From c4e8428673425c5b0c7280387c5cb35d60959b97 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:20 +0100 Subject: crypto: qat - fix configuration of iov threads The number of AE2FUNC_MAP registers is different in every QAT device (c62x, c3xxx and dh895xcc) although the logic and the register offsets are the same across devices. This patch separates the logic that configures the iov threads in a common function that takes as input the number of AE2FUNC_MAP registers supported by a device. The function is then added to the adf_hw_device_data structure of each device, and called with the appropriate parameters. The configure iov thread logic is added to a new file, adf_gen2_hw_data.c, that is going to contain code that is shared across QAT GEN2 devices. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 9 ++++ drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h | 4 ++ drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c | 9 ++++ drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h | 4 ++ drivers/crypto/qat/qat_common/Makefile | 1 + drivers/crypto/qat/qat_common/adf_accel_devices.h | 2 + drivers/crypto/qat/qat_common/adf_gen2_hw_data.c | 38 +++++++++++++ drivers/crypto/qat/qat_common/adf_gen2_hw_data.h | 30 +++++++++++ drivers/crypto/qat/qat_common/adf_sriov.c | 63 ++-------------------- .../crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 9 ++++ .../crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h | 5 ++ 11 files changed, 115 insertions(+), 59 deletions(-) create mode 100644 drivers/crypto/qat/qat_common/adf_gen2_hw_data.c create mode 100644 drivers/crypto/qat/qat_common/adf_gen2_hw_data.h (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index 62b0b290ff85..f449b2a0e82d 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "adf_c3xxx_hw_data.h" /* Worker thread to service arbiter mappings based on dev SKUs */ @@ -171,6 +172,13 @@ static int adf_pf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) return 0; } +static void configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) +{ + adf_gen2_cfg_iov_thds(accel_dev, enable, + ADF_C3XXX_AE2FUNC_MAP_GRP_A_NUM_REGS, + ADF_C3XXX_AE2FUNC_MAP_GRP_B_NUM_REGS); +} + void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &c3xxx_class; @@ -199,6 +207,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->fw_mmp_name = ADF_C3XXX_MMP; hw_data->init_admin_comms = adf_init_admin_comms; hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->configure_iov_threads = configure_iov_threads; hw_data->disable_iov = adf_disable_sriov; hw_data->send_admin_init = adf_send_admin_init; hw_data->init_arb = adf_init_arb; diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h index 94097816f68a..fece8e38025a 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h @@ -31,6 +31,10 @@ #define ADF_C3XXX_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) #define ADF_C3XXX_VINTMSK_OFFSET(i) (0x3A000 + 0x200 + ((i) * 0x04)) +/* AE to function mapping */ +#define ADF_C3XXX_AE2FUNC_MAP_GRP_A_NUM_REGS 48 +#define ADF_C3XXX_AE2FUNC_MAP_GRP_B_NUM_REGS 6 + /* Firmware Binary */ #define ADF_C3XXX_FW "qat_c3xxx.bin" #define ADF_C3XXX_MMP "qat_c3xxx_mmp.bin" diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index 1334b43e46e4..d7bed610ae86 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "adf_c62x_hw_data.h" /* Worker thread to service arbiter mappings based on dev SKUs */ @@ -181,6 +182,13 @@ static int adf_pf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) return 0; } +static void configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) +{ + adf_gen2_cfg_iov_thds(accel_dev, enable, + ADF_C62X_AE2FUNC_MAP_GRP_A_NUM_REGS, + ADF_C62X_AE2FUNC_MAP_GRP_B_NUM_REGS); +} + void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &c62x_class; @@ -209,6 +217,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->fw_mmp_name = ADF_C62X_MMP; hw_data->init_admin_comms = adf_init_admin_comms; hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->configure_iov_threads = configure_iov_threads; hw_data->disable_iov = adf_disable_sriov; hw_data->send_admin_init = adf_send_admin_init; hw_data->init_arb = adf_init_arb; diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h index a2e2961a2102..53d3cb577f5b 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h @@ -32,6 +32,10 @@ #define ADF_C62X_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) #define ADF_C62X_VINTMSK_OFFSET(i) (0x3A000 + 0x200 + ((i) * 0x04)) +/* AE to function mapping */ +#define ADF_C62X_AE2FUNC_MAP_GRP_A_NUM_REGS 80 +#define ADF_C62X_AE2FUNC_MAP_GRP_B_NUM_REGS 10 + /* Firmware Binary */ #define ADF_C62X_FW "qat_c62x.bin" #define ADF_C62X_MMP "qat_c62x_mmp.bin" diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/qat/qat_common/Makefile index 47a8e3d8b81a..25d28516dcdd 100644 --- a/drivers/crypto/qat/qat_common/Makefile +++ b/drivers/crypto/qat/qat_common/Makefile @@ -10,6 +10,7 @@ intel_qat-objs := adf_cfg.o \ adf_transport.o \ adf_admin.o \ adf_hw_arbiter.o \ + adf_gen2_hw_data.o \ qat_crypto.o \ qat_algs.o \ qat_asym_algs.o \ diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 85b423d28f77..d7a27d15e137 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -125,6 +125,8 @@ struct adf_hw_device_data { void (*get_arb_mapping)(struct adf_accel_dev *accel_dev, const u32 **cfg); void (*disable_iov)(struct adf_accel_dev *accel_dev); + void (*configure_iov_threads)(struct adf_accel_dev *accel_dev, + bool enable); void (*enable_ints)(struct adf_accel_dev *accel_dev); int (*enable_vf2pf_comms)(struct adf_accel_dev *accel_dev); void (*reset_device)(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c new file mode 100644 index 000000000000..26e345e3d7c3 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2020 Intel Corporation */ +#include "adf_gen2_hw_data.h" + +void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, + int num_a_regs, int num_b_regs) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *pmisc_addr; + struct adf_bar *pmisc; + int pmisc_id, i; + u32 reg; + + pmisc_id = hw_data->get_misc_bar_id(hw_data); + pmisc = &GET_BARS(accel_dev)[pmisc_id]; + pmisc_addr = pmisc->virt_addr; + + /* Set/Unset Valid bit in AE Thread to PCIe Function Mapping Group A */ + for (i = 0; i < num_a_regs; i++) { + reg = READ_CSR_AE2FUNCTION_MAP_A(pmisc_addr, i); + if (enable) + reg |= AE2FUNCTION_MAP_VALID; + else + reg &= ~AE2FUNCTION_MAP_VALID; + WRITE_CSR_AE2FUNCTION_MAP_A(pmisc_addr, i, reg); + } + + /* Set/Unset Valid bit in AE Thread to PCIe Function Mapping Group B */ + for (i = 0; i < num_b_regs; i++) { + reg = READ_CSR_AE2FUNCTION_MAP_B(pmisc_addr, i); + if (enable) + reg |= AE2FUNCTION_MAP_VALID; + else + reg &= ~AE2FUNCTION_MAP_VALID; + WRITE_CSR_AE2FUNCTION_MAP_B(pmisc_addr, i, reg); + } +} +EXPORT_SYMBOL_GPL(adf_gen2_cfg_iov_thds); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h new file mode 100644 index 000000000000..1d348425d5f4 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2020 Intel Corporation */ +#ifndef ADF_GEN2_HW_DATA_H_ +#define ADF_GEN2_HW_DATA_H_ + +#include "adf_accel_devices.h" + +/* AE to function map */ +#define AE2FUNCTION_MAP_A_OFFSET (0x3A400 + 0x190) +#define AE2FUNCTION_MAP_B_OFFSET (0x3A400 + 0x310) +#define AE2FUNCTION_MAP_REG_SIZE 4 +#define AE2FUNCTION_MAP_VALID BIT(7) + +#define READ_CSR_AE2FUNCTION_MAP_A(pmisc_bar_addr, index) \ + ADF_CSR_RD(pmisc_bar_addr, AE2FUNCTION_MAP_A_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index)) +#define WRITE_CSR_AE2FUNCTION_MAP_A(pmisc_bar_addr, index, value) \ + ADF_CSR_WR(pmisc_bar_addr, AE2FUNCTION_MAP_A_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index), value) +#define READ_CSR_AE2FUNCTION_MAP_B(pmisc_bar_addr, index) \ + ADF_CSR_RD(pmisc_bar_addr, AE2FUNCTION_MAP_B_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index)) +#define WRITE_CSR_AE2FUNCTION_MAP_B(pmisc_bar_addr, index, value) \ + ADF_CSR_WR(pmisc_bar_addr, AE2FUNCTION_MAP_B_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index), value) + +void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, + int num_a_regs, int num_b_regs); + +#endif diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index 963b2bea78f2..dde6c57ef15a 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -10,31 +10,6 @@ static struct workqueue_struct *pf2vf_resp_wq; -#define ME2FUNCTION_MAP_A_OFFSET (0x3A400 + 0x190) -#define ME2FUNCTION_MAP_A_NUM_REGS 96 - -#define ME2FUNCTION_MAP_B_OFFSET (0x3A400 + 0x310) -#define ME2FUNCTION_MAP_B_NUM_REGS 12 - -#define ME2FUNCTION_MAP_REG_SIZE 4 -#define ME2FUNCTION_MAP_VALID BIT(7) - -#define READ_CSR_ME2FUNCTION_MAP_A(pmisc_bar_addr, index) \ - ADF_CSR_RD(pmisc_bar_addr, ME2FUNCTION_MAP_A_OFFSET + \ - ME2FUNCTION_MAP_REG_SIZE * index) - -#define WRITE_CSR_ME2FUNCTION_MAP_A(pmisc_bar_addr, index, value) \ - ADF_CSR_WR(pmisc_bar_addr, ME2FUNCTION_MAP_A_OFFSET + \ - ME2FUNCTION_MAP_REG_SIZE * index, value) - -#define READ_CSR_ME2FUNCTION_MAP_B(pmisc_bar_addr, index) \ - ADF_CSR_RD(pmisc_bar_addr, ME2FUNCTION_MAP_B_OFFSET + \ - ME2FUNCTION_MAP_REG_SIZE * index) - -#define WRITE_CSR_ME2FUNCTION_MAP_B(pmisc_bar_addr, index, value) \ - ADF_CSR_WR(pmisc_bar_addr, ME2FUNCTION_MAP_B_OFFSET + \ - ME2FUNCTION_MAP_REG_SIZE * index, value) - struct adf_pf2vf_resp { struct work_struct pf2vf_resp_work; struct adf_accel_vf_info *vf_info; @@ -68,12 +43,8 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) struct pci_dev *pdev = accel_to_pci_dev(accel_dev); int totalvfs = pci_sriov_get_totalvfs(pdev); struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_bar *pmisc = - &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; - void __iomem *pmisc_addr = pmisc->virt_addr; struct adf_accel_vf_info *vf_info; int i; - u32 reg; for (i = 0, vf_info = accel_dev->pf.vf_info; i < totalvfs; i++, vf_info++) { @@ -90,19 +61,8 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) DEFAULT_RATELIMIT_BURST); } - /* Set Valid bits in ME Thread to PCIe Function Mapping Group A */ - for (i = 0; i < ME2FUNCTION_MAP_A_NUM_REGS; i++) { - reg = READ_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i); - reg |= ME2FUNCTION_MAP_VALID; - WRITE_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i, reg); - } - - /* Set Valid bits in ME Thread to PCIe Function Mapping Group B */ - for (i = 0; i < ME2FUNCTION_MAP_B_NUM_REGS; i++) { - reg = READ_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i); - reg |= ME2FUNCTION_MAP_VALID; - WRITE_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i, reg); - } + /* Set Valid bits in AE Thread to PCIe Function Mapping */ + hw_data->configure_iov_threads(accel_dev, true); /* Enable VF to PF interrupts for all VFs */ adf_enable_vf2pf_interrupts(accel_dev, GENMASK_ULL(totalvfs - 1, 0)); @@ -127,12 +87,8 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) void adf_disable_sriov(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_bar *pmisc = - &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; - void __iomem *pmisc_addr = pmisc->virt_addr; int totalvfs = pci_sriov_get_totalvfs(accel_to_pci_dev(accel_dev)); struct adf_accel_vf_info *vf; - u32 reg; int i; if (!accel_dev->pf.vf_info) @@ -145,19 +101,8 @@ void adf_disable_sriov(struct adf_accel_dev *accel_dev) /* Disable VF to PF interrupts */ adf_disable_vf2pf_interrupts(accel_dev, 0xFFFFFFFF); - /* Clear Valid bits in ME Thread to PCIe Function Mapping Group A */ - for (i = 0; i < ME2FUNCTION_MAP_A_NUM_REGS; i++) { - reg = READ_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i); - reg &= ~ME2FUNCTION_MAP_VALID; - WRITE_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i, reg); - } - - /* Clear Valid bits in ME Thread to PCIe Function Mapping Group B */ - for (i = 0; i < ME2FUNCTION_MAP_B_NUM_REGS; i++) { - reg = READ_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i); - reg &= ~ME2FUNCTION_MAP_VALID; - WRITE_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i, reg); - } + /* Clear Valid bits in AE Thread to PCIe Function Mapping */ + hw_data->configure_iov_threads(accel_dev, false); for (i = 0, vf = accel_dev->pf.vf_info; i < totalvfs; i++, vf++) { tasklet_disable(&vf->vf2pf_bh_tasklet); diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 1f3ea3ba1cee..7b2f13ff49fd 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "adf_dh895xcc_hw_data.h" /* Worker thread to service arbiter mappings based on dev SKUs */ @@ -180,6 +181,13 @@ static int adf_pf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) return 0; } +static void configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) +{ + adf_gen2_cfg_iov_thds(accel_dev, enable, + ADF_DH895XCC_AE2FUNC_MAP_GRP_A_NUM_REGS, + ADF_DH895XCC_AE2FUNC_MAP_GRP_B_NUM_REGS); +} + void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &dh895xcc_class; @@ -208,6 +216,7 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->fw_mmp_name = ADF_DH895XCC_MMP; hw_data->init_admin_comms = adf_init_admin_comms; hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->configure_iov_threads = configure_iov_threads; hw_data->disable_iov = adf_disable_sriov; hw_data->send_admin_init = adf_send_admin_init; hw_data->init_arb = adf_init_arb; diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h index 082a04466dca..4d613923d155 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h @@ -36,6 +36,11 @@ #define ADF_DH895XCC_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) #define ADF_DH895XCC_VINTMSK_OFFSET(i) (0x3A000 + 0x200 + ((i) * 0x04)) + +/* AE to function mapping */ +#define ADF_DH895XCC_AE2FUNC_MAP_GRP_A_NUM_REGS 96 +#define ADF_DH895XCC_AE2FUNC_MAP_GRP_B_NUM_REGS 12 + /* FW names */ #define ADF_DH895XCC_FW "qat_895xcc.bin" #define ADF_DH895XCC_MMP "qat_895xcc_mmp.bin" -- cgit v1.2.3 From f9c697c4bfc6002d92e8015b65fe2a79acadb416 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:21 +0100 Subject: crypto: qat - split transport CSR access logic Abstract access to transport CSRs and move generation specific code into adf_gen2_hw_data.c in preparation for the introduction of the qat_4xxx driver. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 1 + .../crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c | 2 + drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c | 1 + drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c | 2 + drivers/crypto/qat/qat_common/adf_accel_devices.h | 27 +++++++ drivers/crypto/qat/qat_common/adf_gen2_hw_data.c | 85 +++++++++++++++++++++ drivers/crypto/qat/qat_common/adf_gen2_hw_data.h | 1 + drivers/crypto/qat/qat_common/adf_isr.c | 4 +- drivers/crypto/qat/qat_common/adf_transport.c | 86 +++++++++++++++------- .../crypto/qat/qat_common/adf_transport_debug.c | 22 +++--- drivers/crypto/qat/qat_common/adf_vf_isr.c | 5 +- .../crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 1 + .../qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c | 2 + 13 files changed, 198 insertions(+), 41 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index f449b2a0e82d..7af38b947cfe 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -217,6 +217,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; hw_data->reset_device = adf_reset_flr; hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } void adf_clean_hw_data_c3xxx(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c index 80a355e85a72..15f6b9bdfb22 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "adf_c3xxxvf_hw_data.h" static struct adf_hw_device_class c3xxxiov_class = { @@ -98,6 +99,7 @@ void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data) hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; hw_data->dev_class->instances++; adf_devmgr_update_class_index(hw_data); + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } void adf_clean_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index d7bed610ae86..c18fb77dd8ec 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -227,6 +227,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; hw_data->reset_device = adf_reset_flr; hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } void adf_clean_hw_data_c62x(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c index 7725387e58f8..d231583428c9 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "adf_c62xvf_hw_data.h" static struct adf_hw_device_class c62xiov_class = { @@ -98,6 +99,7 @@ void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data) hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; hw_data->dev_class->instances++; adf_devmgr_update_class_index(hw_data); + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } void adf_clean_hw_data_c62xiov(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index d7a27d15e137..459e22076813 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -97,6 +97,31 @@ struct adf_hw_device_class { u32 instances; } __packed; +struct adf_hw_csr_ops { + u32 (*read_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, + u32 ring); + void (*write_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, + u32 ring, u32 value); + u32 (*read_csr_ring_tail)(void __iomem *csr_base_addr, u32 bank, + u32 ring); + void (*write_csr_ring_tail)(void __iomem *csr_base_addr, u32 bank, + u32 ring, u32 value); + u32 (*read_csr_e_stat)(void __iomem *csr_base_addr, u32 bank); + void (*write_csr_ring_config)(void __iomem *csr_base_addr, u32 bank, + u32 ring, u32 value); + void (*write_csr_ring_base)(void __iomem *csr_base_addr, u32 bank, + u32 ring, dma_addr_t addr); + void (*write_csr_int_flag)(void __iomem *csr_base_addr, u32 bank, + u32 value); + void (*write_csr_int_srcsel)(void __iomem *csr_base_addr, u32 bank); + void (*write_csr_int_col_en)(void __iomem *csr_base_addr, u32 bank, + u32 value); + void (*write_csr_int_col_ctl)(void __iomem *csr_base_addr, u32 bank, + u32 value); + void (*write_csr_int_flag_and_col)(void __iomem *csr_base_addr, + u32 bank, u32 value); +}; + struct adf_cfg_device_data; struct adf_accel_dev; struct adf_etr_data; @@ -130,6 +155,7 @@ struct adf_hw_device_data { void (*enable_ints)(struct adf_accel_dev *accel_dev); int (*enable_vf2pf_comms)(struct adf_accel_dev *accel_dev); void (*reset_device)(struct adf_accel_dev *accel_dev); + struct adf_hw_csr_ops csr_ops; const char *fw_name; const char *fw_mmp_name; u32 fuses; @@ -162,6 +188,7 @@ struct adf_hw_device_data { #define GET_NUM_RINGS_PER_BANK(accel_dev) \ GET_HW_DATA(accel_dev)->num_rings_per_bank #define GET_MAX_ACCELENGINES(accel_dev) (GET_HW_DATA(accel_dev)->num_engines) +#define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_ops) #define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev struct adf_admin_comms; diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index 26e345e3d7c3..07a9211bf7f9 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2020 Intel Corporation */ #include "adf_gen2_hw_data.h" +#include "adf_transport_access_macros.h" void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs) @@ -36,3 +37,87 @@ void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, } } EXPORT_SYMBOL_GPL(adf_gen2_cfg_iov_thds); + +static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); +} + +static void write_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_TAIL(csr_base_addr, bank, ring); +} + +static void write_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_e_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_E_STAT(csr_base_addr, bank); +} + +static void write_csr_ring_config(void __iomem *csr_base_addr, u32 bank, + u32 ring, u32 value) +{ + WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value); +} + +static void write_csr_ring_base(void __iomem *csr_base_addr, u32 bank, u32 ring, + dma_addr_t addr) +{ + WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr); +} + +static void write_csr_int_flag(void __iomem *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_FLAG(csr_base_addr, bank, value); +} + +static void write_csr_int_srcsel(void __iomem *csr_base_addr, u32 bank) +{ + WRITE_CSR_INT_SRCSEL(csr_base_addr, bank); +} + +static void write_csr_int_col_en(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value); +} + +static void write_csr_int_col_ctl(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value); +} + +static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value); +} + +void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) +{ + csr_ops->read_csr_ring_head = read_csr_ring_head; + csr_ops->write_csr_ring_head = write_csr_ring_head; + csr_ops->read_csr_ring_tail = read_csr_ring_tail; + csr_ops->write_csr_ring_tail = write_csr_ring_tail; + csr_ops->read_csr_e_stat = read_csr_e_stat; + csr_ops->write_csr_ring_config = write_csr_ring_config; + csr_ops->write_csr_ring_base = write_csr_ring_base; + csr_ops->write_csr_int_flag = write_csr_int_flag; + csr_ops->write_csr_int_srcsel = write_csr_int_srcsel; + csr_ops->write_csr_int_col_en = write_csr_int_col_en; + csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl; + csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; +} +EXPORT_SYMBOL_GPL(adf_gen2_init_hw_csr_ops); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index 1d348425d5f4..e6d3919a56a1 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -26,5 +26,6 @@ void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs); +void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); #endif diff --git a/drivers/crypto/qat/qat_common/adf_isr.c b/drivers/crypto/qat/qat_common/adf_isr.c index 36136f7db509..5444f0ea0a1d 100644 --- a/drivers/crypto/qat/qat_common/adf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_isr.c @@ -50,8 +50,10 @@ static void adf_disable_msix(struct adf_accel_pci *pci_dev_info) static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr) { struct adf_etr_bank_data *bank = bank_ptr; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); - WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, 0); + csr_ops->write_csr_int_flag_and_col(bank->csr_addr, bank->bank_number, + 0); tasklet_hi_schedule(&bank->resp_handler); return IRQ_HANDLED; } diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c index 24ddaaaa55b1..03fb7812818b 100644 --- a/drivers/crypto/qat/qat_common/adf_transport.c +++ b/drivers/crypto/qat/qat_common/adf_transport.c @@ -54,24 +54,32 @@ static void adf_unreserve_ring(struct adf_etr_bank_data *bank, u32 ring) static void adf_enable_ring_irq(struct adf_etr_bank_data *bank, u32 ring) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); + spin_lock_bh(&bank->lock); bank->irq_mask |= (1 << ring); spin_unlock_bh(&bank->lock); - WRITE_CSR_INT_COL_EN(bank->csr_addr, bank->bank_number, bank->irq_mask); - WRITE_CSR_INT_COL_CTL(bank->csr_addr, bank->bank_number, - bank->irq_coalesc_timer); + csr_ops->write_csr_int_col_en(bank->csr_addr, bank->bank_number, + bank->irq_mask); + csr_ops->write_csr_int_col_ctl(bank->csr_addr, bank->bank_number, + bank->irq_coalesc_timer); } static void adf_disable_ring_irq(struct adf_etr_bank_data *bank, u32 ring) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); + spin_lock_bh(&bank->lock); bank->irq_mask &= ~(1 << ring); spin_unlock_bh(&bank->lock); - WRITE_CSR_INT_COL_EN(bank->csr_addr, bank->bank_number, bank->irq_mask); + csr_ops->write_csr_int_col_en(bank->csr_addr, bank->bank_number, + bank->irq_mask); } int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); + if (atomic_add_return(1, ring->inflights) > ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) { atomic_dec(ring->inflights); @@ -84,14 +92,17 @@ int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg) ring->tail = adf_modulo(ring->tail + ADF_MSG_SIZE_TO_BYTES(ring->msg_size), ADF_RING_SIZE_MODULO(ring->ring_size)); - WRITE_CSR_RING_TAIL(ring->bank->csr_addr, ring->bank->bank_number, - ring->ring_number, ring->tail); + csr_ops->write_csr_ring_tail(ring->bank->csr_addr, + ring->bank->bank_number, ring->ring_number, + ring->tail); spin_unlock_bh(&ring->lock); + return 0; } static int adf_handle_response(struct adf_etr_ring_data *ring) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); u32 msg_counter = 0; u32 *msg = (u32 *)((uintptr_t)ring->base_addr + ring->head); @@ -105,30 +116,36 @@ static int adf_handle_response(struct adf_etr_ring_data *ring) msg_counter++; msg = (u32 *)((uintptr_t)ring->base_addr + ring->head); } - if (msg_counter > 0) - WRITE_CSR_RING_HEAD(ring->bank->csr_addr, - ring->bank->bank_number, - ring->ring_number, ring->head); + if (msg_counter > 0) { + csr_ops->write_csr_ring_head(ring->bank->csr_addr, + ring->bank->bank_number, + ring->ring_number, ring->head); + } return 0; } static void adf_configure_tx_ring(struct adf_etr_ring_data *ring) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); u32 ring_config = BUILD_RING_CONFIG(ring->ring_size); - WRITE_CSR_RING_CONFIG(ring->bank->csr_addr, ring->bank->bank_number, - ring->ring_number, ring_config); + csr_ops->write_csr_ring_config(ring->bank->csr_addr, + ring->bank->bank_number, + ring->ring_number, ring_config); + } static void adf_configure_rx_ring(struct adf_etr_ring_data *ring) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); u32 ring_config = BUILD_RESP_RING_CONFIG(ring->ring_size, ADF_RING_NEAR_WATERMARK_512, ADF_RING_NEAR_WATERMARK_0); - WRITE_CSR_RING_CONFIG(ring->bank->csr_addr, ring->bank->bank_number, - ring->ring_number, ring_config); + csr_ops->write_csr_ring_config(ring->bank->csr_addr, + ring->bank->bank_number, + ring->ring_number, ring_config); } static int adf_init_ring(struct adf_etr_ring_data *ring) @@ -136,6 +153,7 @@ static int adf_init_ring(struct adf_etr_ring_data *ring) struct adf_etr_bank_data *bank = ring->bank; struct adf_accel_dev *accel_dev = bank->accel_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); u64 ring_base; u32 ring_size_bytes = ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size); @@ -163,8 +181,9 @@ static int adf_init_ring(struct adf_etr_ring_data *ring) adf_configure_rx_ring(ring); ring_base = BUILD_RING_BASE_ADDR(ring->dma_addr, ring->ring_size); - WRITE_CSR_RING_BASE(ring->bank->csr_addr, ring->bank->bank_number, - ring->ring_number, ring_base); + csr_ops->write_csr_ring_base(ring->bank->csr_addr, + ring->bank->bank_number, ring->ring_number, + ring_base); spin_lock_init(&ring->lock); return 0; } @@ -269,15 +288,17 @@ err: void adf_remove_ring(struct adf_etr_ring_data *ring) { struct adf_etr_bank_data *bank = ring->bank; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); /* Disable interrupts for the given ring */ adf_disable_ring_irq(bank, ring->ring_number); /* Clear PCI config space */ - WRITE_CSR_RING_CONFIG(bank->csr_addr, bank->bank_number, - ring->ring_number, 0); - WRITE_CSR_RING_BASE(bank->csr_addr, bank->bank_number, - ring->ring_number, 0); + + csr_ops->write_csr_ring_config(bank->csr_addr, bank->bank_number, + ring->ring_number, 0); + csr_ops->write_csr_ring_base(bank->csr_addr, bank->bank_number, + ring->ring_number, 0); adf_ring_debugfs_rm(ring); adf_unreserve_ring(bank, ring->ring_number); /* Disable HW arbitration for the given ring */ @@ -287,11 +308,14 @@ void adf_remove_ring(struct adf_etr_ring_data *ring) static void adf_ring_response_handler(struct adf_etr_bank_data *bank) { - u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(bank->accel_dev); + struct adf_accel_dev *accel_dev = bank->accel_dev; + u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(accel_dev); + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); unsigned long empty_rings; int i; - empty_rings = READ_CSR_E_STAT(bank->csr_addr, bank->bank_number); + empty_rings = csr_ops->read_csr_e_stat(bank->csr_addr, + bank->bank_number); empty_rings = ~empty_rings & bank->irq_mask; for_each_set_bit(i, &empty_rings, num_rings_per_bank) @@ -301,11 +325,13 @@ static void adf_ring_response_handler(struct adf_etr_bank_data *bank) void adf_response_handler(uintptr_t bank_addr) { struct adf_etr_bank_data *bank = (void *)bank_addr; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); /* Handle all the responses and reenable IRQs */ adf_ring_response_handler(bank); - WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, - bank->irq_mask); + + csr_ops->write_csr_int_flag_and_col(bank->csr_addr, bank->bank_number, + bank->irq_mask); } static inline int adf_get_cfg_int(struct adf_accel_dev *accel_dev, @@ -345,6 +371,7 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, { struct adf_hw_device_data *hw_data = accel_dev->hw_device; u8 num_rings_per_bank = hw_data->num_rings_per_bank; + struct adf_hw_csr_ops *csr_ops = &hw_data->csr_ops; struct adf_etr_ring_data *ring; struct adf_etr_ring_data *tx_ring; u32 i, coalesc_enabled = 0; @@ -375,8 +402,9 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME; for (i = 0; i < num_rings_per_bank; i++) { - WRITE_CSR_RING_CONFIG(csr_addr, bank_num, i, 0); - WRITE_CSR_RING_BASE(csr_addr, bank_num, i, 0); + csr_ops->write_csr_ring_config(csr_addr, bank_num, i, 0); + csr_ops->write_csr_ring_base(csr_addr, bank_num, i, 0); + ring = &bank->rings[i]; if (hw_data->tx_rings_mask & (1 << i)) { ring->inflights = @@ -401,8 +429,10 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, goto err; } - WRITE_CSR_INT_FLAG(csr_addr, bank_num, ADF_BANK_INT_FLAG_CLEAR_MASK); - WRITE_CSR_INT_SRCSEL(csr_addr, bank_num); + csr_ops->write_csr_int_flag(csr_addr, bank_num, + ADF_BANK_INT_FLAG_CLEAR_MASK); + csr_ops->write_csr_int_srcsel(csr_addr, bank_num); + return 0; err: ring_mask = hw_data->tx_rings_mask; diff --git a/drivers/crypto/qat/qat_common/adf_transport_debug.c b/drivers/crypto/qat/qat_common/adf_transport_debug.c index da79d734c035..1205186ad51e 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_debug.c +++ b/drivers/crypto/qat/qat_common/adf_transport_debug.c @@ -42,16 +42,17 @@ static int adf_ring_show(struct seq_file *sfile, void *v) { struct adf_etr_ring_data *ring = sfile->private; struct adf_etr_bank_data *bank = ring->bank; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); void __iomem *csr = ring->bank->csr_addr; if (v == SEQ_START_TOKEN) { int head, tail, empty; - head = READ_CSR_RING_HEAD(csr, bank->bank_number, - ring->ring_number); - tail = READ_CSR_RING_TAIL(csr, bank->bank_number, - ring->ring_number); - empty = READ_CSR_E_STAT(csr, bank->bank_number); + head = csr_ops->read_csr_ring_head(csr, bank->bank_number, + ring->ring_number); + tail = csr_ops->read_csr_ring_tail(csr, bank->bank_number, + ring->ring_number); + empty = csr_ops->read_csr_e_stat(csr, bank->bank_number); seq_puts(sfile, "------- Ring configuration -------\n"); seq_printf(sfile, "ring name: %s\n", @@ -144,6 +145,7 @@ static void *adf_bank_next(struct seq_file *sfile, void *v, loff_t *pos) static int adf_bank_show(struct seq_file *sfile, void *v) { struct adf_etr_bank_data *bank = sfile->private; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); if (v == SEQ_START_TOKEN) { seq_printf(sfile, "------- Bank %d configuration -------\n", @@ -157,11 +159,11 @@ static int adf_bank_show(struct seq_file *sfile, void *v) if (!(bank->ring_mask & 1 << ring_id)) return 0; - head = READ_CSR_RING_HEAD(csr, bank->bank_number, - ring->ring_number); - tail = READ_CSR_RING_TAIL(csr, bank->bank_number, - ring->ring_number); - empty = READ_CSR_E_STAT(csr, bank->bank_number); + head = csr_ops->read_csr_ring_head(csr, bank->bank_number, + ring->ring_number); + tail = csr_ops->read_csr_ring_tail(csr, bank->bank_number, + ring->ring_number); + empty = csr_ops->read_csr_e_stat(csr, bank->bank_number); seq_printf(sfile, "ring num %02d, head %04x, tail %04x, empty: %d\n", diff --git a/drivers/crypto/qat/qat_common/adf_vf_isr.c b/drivers/crypto/qat/qat_common/adf_vf_isr.c index c4a44dc6af3e..38d316a42ba6 100644 --- a/drivers/crypto/qat/qat_common/adf_vf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_vf_isr.c @@ -156,6 +156,7 @@ static irqreturn_t adf_isr(int irq, void *privdata) { struct adf_accel_dev *accel_dev = privdata; struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_hw_csr_ops *csr_ops = &hw_data->csr_ops; struct adf_bar *pmisc = &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; void __iomem *pmisc_bar_addr = pmisc->virt_addr; @@ -180,8 +181,8 @@ static irqreturn_t adf_isr(int irq, void *privdata) struct adf_etr_bank_data *bank = &etr_data->banks[0]; /* Disable Flag and Coalesce Ring Interrupts */ - WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, - 0); + csr_ops->write_csr_int_flag_and_col(bank->csr_addr, + bank->bank_number, 0); tasklet_hi_schedule(&bank->resp_handler); return IRQ_HANDLED; } diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 7b2f13ff49fd..39423316664b 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -226,6 +226,7 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; hw_data->reset_device = adf_reset_sbr; hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c index eca144bc1d67..f14fb82ed6df 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "adf_dh895xccvf_hw_data.h" static struct adf_hw_device_class dh895xcciov_class = { @@ -98,6 +99,7 @@ void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; hw_data->dev_class->instances++; adf_devmgr_update_class_index(hw_data); + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } void adf_clean_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) -- cgit v1.2.3 From d1e86b4c2cdc2d4a15767705696beaabbdc3a57f Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:22 +0100 Subject: crypto: qat - relocate GEN2 CSR access code Move gen2 specific transport macros to adf_gen2_hw_data.c. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_gen2_hw_data.c | 1 - drivers/crypto/qat/qat_common/adf_gen2_hw_data.h | 68 ++++++++++++++++++++++ .../qat/qat_common/adf_transport_access_macros.h | 64 -------------------- 3 files changed, 68 insertions(+), 65 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index 07a9211bf7f9..9011c94156a9 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2020 Intel Corporation */ #include "adf_gen2_hw_data.h" -#include "adf_transport_access_macros.h" void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs) diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index e6d3919a56a1..592aee627762 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -5,6 +5,74 @@ #include "adf_accel_devices.h" +/* Transport access */ +#define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL +#define ADF_BANK_INT_SRC_SEL_MASK_X 0x44444444UL +#define ADF_RING_CSR_RING_CONFIG 0x000 +#define ADF_RING_CSR_RING_LBASE 0x040 +#define ADF_RING_CSR_RING_UBASE 0x080 +#define ADF_RING_CSR_RING_HEAD 0x0C0 +#define ADF_RING_CSR_RING_TAIL 0x100 +#define ADF_RING_CSR_E_STAT 0x14C +#define ADF_RING_CSR_INT_FLAG 0x170 +#define ADF_RING_CSR_INT_SRCSEL 0x174 +#define ADF_RING_CSR_INT_SRCSEL_2 0x178 +#define ADF_RING_CSR_INT_COL_EN 0x17C +#define ADF_RING_CSR_INT_COL_CTL 0x180 +#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 +#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 +#define ADF_RING_BUNDLE_SIZE 0x1000 + +#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ + ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2)) +#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ + ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2)) +#define READ_CSR_E_STAT(csr_base_addr, bank) \ + ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_E_STAT) +#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_CONFIG + ((ring) << 2), value) +#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ +do { \ + u32 l_base = 0, u_base = 0; \ + l_base = (u32)((value) & 0xFFFFFFFF); \ + u_base = (u32)(((value) & 0xFFFFFFFF00000000ULL) >> 32); \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_LBASE + ((ring) << 2), l_base); \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_UBASE + ((ring) << 2), u_base); \ +} while (0) + +#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2), value) +#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2), value) +#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_FLAG, value) +#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ +do { \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_SRCSEL, ADF_BANK_INT_SRC_SEL_MASK_0); \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_SRCSEL_2, ADF_BANK_INT_SRC_SEL_MASK_X); \ +} while (0) +#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_COL_EN, value) +#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_COL_CTL, \ + ADF_RING_CSR_INT_COL_CTL_ENABLE | (value)) +#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_FLAG_AND_COL, value) + /* AE to function map */ #define AE2FUNCTION_MAP_A_OFFSET (0x3A400 + 0x190) #define AE2FUNCTION_MAP_B_OFFSET (0x3A400 + 0x310) diff --git a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h index 950d1988556c..4642b0b5cfb0 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h +++ b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h @@ -4,23 +4,7 @@ #define ADF_TRANSPORT_ACCESS_MACROS_H #include "adf_accel_devices.h" -#define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL -#define ADF_BANK_INT_SRC_SEL_MASK_X 0x44444444UL #define ADF_BANK_INT_FLAG_CLEAR_MASK 0xFFFF -#define ADF_RING_CSR_RING_CONFIG 0x000 -#define ADF_RING_CSR_RING_LBASE 0x040 -#define ADF_RING_CSR_RING_UBASE 0x080 -#define ADF_RING_CSR_RING_HEAD 0x0C0 -#define ADF_RING_CSR_RING_TAIL 0x100 -#define ADF_RING_CSR_E_STAT 0x14C -#define ADF_RING_CSR_INT_FLAG 0x170 -#define ADF_RING_CSR_INT_SRCSEL 0x174 -#define ADF_RING_CSR_INT_SRCSEL_2 0x178 -#define ADF_RING_CSR_INT_COL_EN 0x17C -#define ADF_RING_CSR_INT_COL_CTL 0x180 -#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 -#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 -#define ADF_RING_BUNDLE_SIZE 0x1000 #define ADF_RING_CONFIG_NEAR_FULL_WM 0x0A #define ADF_RING_CONFIG_NEAR_EMPTY_WM 0x05 #define ADF_COALESCING_MIN_TIME 0x1FF @@ -74,52 +58,4 @@ | size) #define BUILD_RING_BASE_ADDR(addr, size) \ ((addr >> 6) & (0xFFFFFFFFFFFFFFFFULL << size)) -#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ - ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_HEAD + (ring << 2)) -#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ - ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_TAIL + (ring << 2)) -#define READ_CSR_E_STAT(csr_base_addr, bank) \ - ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_E_STAT) -#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_CONFIG + (ring << 2), value) -#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ -do { \ - u32 l_base = 0, u_base = 0; \ - l_base = (u32)(value & 0xFFFFFFFF); \ - u_base = (u32)((value & 0xFFFFFFFF00000000ULL) >> 32); \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_LBASE + (ring << 2), l_base); \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_UBASE + (ring << 2), u_base); \ -} while (0) -#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_HEAD + (ring << 2), value) -#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_TAIL + (ring << 2), value) -#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_INT_FLAG, value) -#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ -do { \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_INT_SRCSEL, ADF_BANK_INT_SRC_SEL_MASK_0); \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_INT_SRCSEL_2, ADF_BANK_INT_SRC_SEL_MASK_X); \ -} while (0) -#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_INT_COL_EN, value) -#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_INT_COL_CTL, \ - ADF_RING_CSR_INT_COL_CTL_ENABLE | value) -#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_INT_FLAG_AND_COL, value) #endif -- cgit v1.2.3 From fc920eccd33b6b0ceb9ebe568ef783b5a816606f Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:23 +0100 Subject: crypto: qat - abstract admin interface Abstract access to admin interface and move generation specific code into adf_gen2_hw_data.c in preparation for the introduction of the qat_4xxx driver. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 1 + drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c | 1 + drivers/crypto/qat/qat_common/adf_accel_devices.h | 7 ++++++ drivers/crypto/qat/qat_common/adf_admin.c | 25 +++++++++++++--------- drivers/crypto/qat/qat_common/adf_gen2_hw_data.c | 8 +++++++ drivers/crypto/qat/qat_common/adf_gen2_hw_data.h | 6 ++++++ .../crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 1 + 7 files changed, 39 insertions(+), 10 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index 7af38b947cfe..f72ed415800e 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -202,6 +202,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->get_misc_bar_id = get_misc_bar_id; hw_data->get_pf2vf_offset = get_pf2vf_offset; hw_data->get_vintmsk_offset = get_vintmsk_offset; + hw_data->get_admin_info = adf_gen2_get_admin_info; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_C3XXX_FW; hw_data->fw_mmp_name = ADF_C3XXX_MMP; diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index c18fb77dd8ec..d4443523dc9d 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -212,6 +212,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->get_misc_bar_id = get_misc_bar_id; hw_data->get_pf2vf_offset = get_pf2vf_offset; hw_data->get_vintmsk_offset = get_vintmsk_offset; + hw_data->get_admin_info = adf_gen2_get_admin_info; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_C62X_FW; hw_data->fw_mmp_name = ADF_C62X_MMP; diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 459e22076813..5f57850c2e8d 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -97,6 +97,12 @@ struct adf_hw_device_class { u32 instances; } __packed; +struct admin_info { + u32 admin_msg_ur; + u32 admin_msg_lr; + u32 mailbox_offset; +}; + struct adf_hw_csr_ops { u32 (*read_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, u32 ring); @@ -138,6 +144,7 @@ struct adf_hw_device_data { u32 (*get_num_accels)(struct adf_hw_device_data *self); u32 (*get_pf2vf_offset)(u32 i); u32 (*get_vintmsk_offset)(u32 i); + void (*get_admin_info)(struct admin_info *admin_csrs_info); enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self); int (*alloc_irq)(struct adf_accel_dev *accel_dev); void (*free_irq)(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c index ec9b390276d6..3ae7c89ce82a 100644 --- a/drivers/crypto/qat/qat_common/adf_admin.c +++ b/drivers/crypto/qat/qat_common/adf_admin.c @@ -10,11 +10,7 @@ #include "adf_common_drv.h" #include "icp_qat_fw_init_admin.h" -/* Admin Messages Registers */ -#define ADF_DH895XCC_ADMINMSGUR_OFFSET (0x3A000 + 0x574) -#define ADF_DH895XCC_ADMINMSGLR_OFFSET (0x3A000 + 0x578) -#define ADF_DH895XCC_MAILBOX_BASE_OFFSET 0x20970 -#define ADF_DH895XCC_MAILBOX_STRIDE 0x1000 +#define ADF_ADMIN_MAILBOX_STRIDE 0x1000 #define ADF_ADMINMSG_LEN 32 #define ADF_CONST_TABLE_SIZE 1024 #define ADF_ADMIN_POLL_DELAY_US 20 @@ -118,7 +114,7 @@ static int adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev, u32 ae, struct adf_admin_comms *admin = accel_dev->admin; int offset = ae * ADF_ADMINMSG_LEN * 2; void __iomem *mailbox = admin->mailbox_addr; - int mb_offset = ae * ADF_DH895XCC_MAILBOX_STRIDE; + int mb_offset = ae * ADF_ADMIN_MAILBOX_STRIDE; struct icp_qat_fw_init_admin_req *request = in; mutex_lock(&admin->lock); @@ -225,8 +221,9 @@ int adf_init_admin_comms(struct adf_accel_dev *accel_dev) struct adf_bar *pmisc = &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; void __iomem *csr = pmisc->virt_addr; - void __iomem *mailbox = (void __iomem *)((uintptr_t)csr + - ADF_DH895XCC_MAILBOX_BASE_OFFSET); + struct admin_info admin_csrs_info; + u32 mailbox_offset, adminmsg_u, adminmsg_l; + void __iomem *mailbox; u64 reg_val; admin = kzalloc_node(sizeof(*accel_dev->admin), GFP_KERNEL, @@ -254,9 +251,17 @@ int adf_init_admin_comms(struct adf_accel_dev *accel_dev) } memcpy(admin->virt_tbl_addr, const_tab, sizeof(const_tab)); + hw_data->get_admin_info(&admin_csrs_info); + + mailbox_offset = admin_csrs_info.mailbox_offset; + mailbox = (void __iomem *)((uintptr_t)csr + mailbox_offset); + adminmsg_u = admin_csrs_info.admin_msg_ur; + adminmsg_l = admin_csrs_info.admin_msg_lr; + reg_val = (u64)admin->phy_addr; - ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGUR_OFFSET, reg_val >> 32); - ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGLR_OFFSET, reg_val); + ADF_CSR_WR(csr, adminmsg_u, upper_32_bits(reg_val)); + ADF_CSR_WR(csr, adminmsg_l, lower_32_bits(reg_val)); + mutex_init(&admin->lock); admin->mailbox_addr = mailbox; accel_dev->admin = admin; diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index 9011c94156a9..15a0bc921d7e 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -37,6 +37,14 @@ void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, } EXPORT_SYMBOL_GPL(adf_gen2_cfg_iov_thds); +void adf_gen2_get_admin_info(struct admin_info *admin_csrs_info) +{ + admin_csrs_info->mailbox_offset = ADF_MAILBOX_BASE_OFFSET; + admin_csrs_info->admin_msg_ur = ADF_ADMINMSGUR_OFFSET; + admin_csrs_info->admin_msg_lr = ADF_ADMINMSGLR_OFFSET; +} +EXPORT_SYMBOL_GPL(adf_gen2_get_admin_info); + static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) { return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index 592aee627762..e9d2591b2be8 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -92,8 +92,14 @@ do { \ ADF_CSR_WR(pmisc_bar_addr, AE2FUNCTION_MAP_B_OFFSET + \ AE2FUNCTION_MAP_REG_SIZE * (index), value) +/* Admin Interface Offsets */ +#define ADF_ADMINMSGUR_OFFSET (0x3A000 + 0x574) +#define ADF_ADMINMSGLR_OFFSET (0x3A000 + 0x578) +#define ADF_MAILBOX_BASE_OFFSET 0x20970 + void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs); void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); +void adf_gen2_get_admin_info(struct admin_info *admin_csrs_info); #endif diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 39423316664b..c568e9808cec 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -210,6 +210,7 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->get_misc_bar_id = get_misc_bar_id; hw_data->get_pf2vf_offset = get_pf2vf_offset; hw_data->get_vintmsk_offset = get_vintmsk_offset; + hw_data->get_admin_info = adf_gen2_get_admin_info; hw_data->get_sram_bar_id = get_sram_bar_id; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_DH895XCC_FW; -- cgit v1.2.3 From f9ff4d3820d5966bd7b86cc97f475bd8c3c04313 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:24 +0100 Subject: crypto: qat - add packed to init admin structures Add packed attribute to the structures icp_qat_fw_init_admin_req and icp_qat_fw_init_admin_resp as they are accessed by firmware. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h b/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h index d4d188cd7ed0..3868bcbed252 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h @@ -39,7 +39,7 @@ struct icp_qat_fw_init_admin_req { }; __u32 resrvd4; -}; +} __packed; struct icp_qat_fw_init_admin_resp { __u8 flags; @@ -92,7 +92,7 @@ struct icp_qat_fw_init_admin_resp { __u64 resrvd8; }; }; -}; +} __packed; #define ICP_QAT_FW_COMN_HEARTBEAT_OK 0 #define ICP_QAT_FW_COMN_HEARTBEAT_BLOCKED 1 -- cgit v1.2.3 From fa7e41e649e33b54d0780af83caca6cbe573b0bb Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:25 +0100 Subject: crypto: qat - rename ME in AE Rename occurrences of ME in the admin module with the acronym AE (Acceleration Engine) as the two are equivalent. This is to keep a single acronym for engined in the codebase and follow the documentation in https://01.org/intel-quickassist-technology. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_admin.c | 6 +++--- drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c index 3ae7c89ce82a..13a5e8659682 100644 --- a/drivers/crypto/qat/qat_common/adf_admin.c +++ b/drivers/crypto/qat/qat_common/adf_admin.c @@ -163,7 +163,7 @@ static int adf_send_admin(struct adf_accel_dev *accel_dev, return 0; } -static int adf_init_me(struct adf_accel_dev *accel_dev) +static int adf_init_ae(struct adf_accel_dev *accel_dev) { struct icp_qat_fw_init_admin_req req; struct icp_qat_fw_init_admin_resp resp; @@ -172,7 +172,7 @@ static int adf_init_me(struct adf_accel_dev *accel_dev) memset(&req, 0, sizeof(req)); memset(&resp, 0, sizeof(resp)); - req.cmd_id = ICP_QAT_FW_INIT_ME; + req.cmd_id = ICP_QAT_FW_INIT_AE; return adf_send_admin(accel_dev, &req, &resp, ae_mask); } @@ -206,7 +206,7 @@ int adf_send_admin_init(struct adf_accel_dev *accel_dev) { int ret; - ret = adf_init_me(accel_dev); + ret = adf_init_ae(accel_dev); if (ret) return ret; diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h b/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h index 3868bcbed252..f05ad17fbdd6 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h @@ -6,7 +6,7 @@ #include "icp_qat_fw.h" enum icp_qat_fw_init_admin_cmd_id { - ICP_QAT_FW_INIT_ME = 0, + ICP_QAT_FW_INIT_AE = 0, ICP_QAT_FW_TRNG_ENABLE = 1, ICP_QAT_FW_TRNG_DISABLE = 2, ICP_QAT_FW_CONSTANTS_CFG = 3, -- cgit v1.2.3 From 09b08885169998134ef40a6cad6f66e833582083 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:26 +0100 Subject: crypto: qat - change admin sequence Call adf_set_fw_constants() before adf_init_ae(). This is required by QAT GEN4 devices, which expect that the FW_CONSTANTS_CFG command is sent to the admin AEs before the FW_INIT_AE command. Swapping the order of the two commands (FW_INIT_AE and FW_CONSTANTS_CFG) is allowed in QAT GEN2 devices as the firmware can handle those in any order. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_admin.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c index 13a5e8659682..6d94746d266f 100644 --- a/drivers/crypto/qat/qat_common/adf_admin.c +++ b/drivers/crypto/qat/qat_common/adf_admin.c @@ -206,11 +206,11 @@ int adf_send_admin_init(struct adf_accel_dev *accel_dev) { int ret; - ret = adf_init_ae(accel_dev); + ret = adf_set_fw_constants(accel_dev); if (ret) return ret; - return adf_set_fw_constants(accel_dev); + return adf_init_ae(accel_dev); } EXPORT_SYMBOL_GPL(adf_send_admin_init); -- cgit v1.2.3 From 2d499c049ecdde25106bc391de644722f48793b5 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:27 +0100 Subject: crypto: qat - use admin mask to send fw constants Introduce admin AE mask. If this mask set, the fw constant message is sent only to engines that belong to that set, otherwise it is sent to all engines. This is in preparation for the qat_4xxx driver where the constant message should be sent only to admin engines. In GEN2 devices (c62x, c3xxx and dh895xcc), the admin AE mask is 0 and the fw constants message is sent to all AEs. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_devices.h | 1 + drivers/crypto/qat/qat_common/adf_admin.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 5f57850c2e8d..779f62fde3bd 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -171,6 +171,7 @@ struct adf_hw_device_data { u32 instance_id; u16 accel_mask; u16 ae_mask; + u32 admin_ae_mask; u16 tx_rings_mask; u8 tx_rx_gap; u8 num_banks; diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c index 6d94746d266f..dcd580d2afe2 100644 --- a/drivers/crypto/qat/qat_common/adf_admin.c +++ b/drivers/crypto/qat/qat_common/adf_admin.c @@ -182,7 +182,7 @@ static int adf_set_fw_constants(struct adf_accel_dev *accel_dev) struct icp_qat_fw_init_admin_req req; struct icp_qat_fw_init_admin_resp resp; struct adf_hw_device_data *hw_device = accel_dev->hw_device; - u32 ae_mask = hw_device->ae_mask; + u32 ae_mask = hw_device->admin_ae_mask ?: hw_device->ae_mask; memset(&req, 0, sizeof(req)); memset(&resp, 0, sizeof(resp)); -- cgit v1.2.3 From 59ab4d9aa5c9f5361d95cf8dc306797dc48e68b4 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:28 +0100 Subject: crypto: qat - update constants table Extend admin contansts table to support QAT GEN4 devices. This change does not affect QAT GEN2 devices (c62x, c3xxx and dh895xcc) as the table was extended in an unused area which is not referenced by any of those drivers and devices. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_admin.c | 44 +++++++++++++++---------------- 1 file changed, 22 insertions(+), 22 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c index dcd580d2afe2..7c2ca54229aa 100644 --- a/drivers/crypto/qat/qat_common/adf_admin.c +++ b/drivers/crypto/qat/qat_common/adf_admin.c @@ -66,28 +66,28 @@ static const u8 const_tab[1024] __aligned(1024) = { 0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, 0x0e, 0x52, 0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f, 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, -0x7e, 0x21, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x7e, 0x21, 0x79, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x18, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x01, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x15, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x02, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x14, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x02, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x24, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, +0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x01, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x01, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x2B, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -- cgit v1.2.3 From 8b5b80db8b85a78929fb0ad50acea32eeefa7ebd Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:29 +0100 Subject: crypto: qat - remove writes into WQCFG WQCFG registers contain the correct values after reset in all generations of QAT. No need to write into them. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_hw_arbiter.c | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c index d4162783f970..cbb9f0b8ff74 100644 --- a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c +++ b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c @@ -11,7 +11,6 @@ #define ADF_ARB_REG_SLOT 0x1000 #define ADF_ARB_WTR_OFFSET 0x010 #define ADF_ARB_RO_EN_OFFSET 0x090 -#define ADF_ARB_WQCFG_OFFSET 0x100 #define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 #define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C @@ -28,10 +27,6 @@ ADF_ARB_WRK_2_SER_MAP_OFFSET) + \ (ADF_ARB_REG_SIZE * index), value) -#define WRITE_CSR_ARB_WQCFG(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ - ADF_ARB_WQCFG_OFFSET) + (ADF_ARB_REG_SIZE * index), value) - int adf_init_arb(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; @@ -45,10 +40,6 @@ int adf_init_arb(struct adf_accel_dev *accel_dev) for (arb = 0; arb < ADF_ARB_NUM; arb++) WRITE_CSR_ARB_SARCONFIG(csr, arb, arb_cfg); - /* Setup worker queue registers */ - for (i = 0; i < hw_data->num_engines; i++) - WRITE_CSR_ARB_WQCFG(csr, i, i); - /* Map worker threads to service arbiters */ hw_data->get_arb_mapping(accel_dev, &thd_2_arb_cfg); @@ -84,10 +75,6 @@ void adf_exit_arb(struct adf_accel_dev *accel_dev) for (i = 0; i < ADF_ARB_NUM; i++) WRITE_CSR_ARB_SARCONFIG(csr, i, 0); - /* Shutdown work queue */ - for (i = 0; i < hw_data->num_engines; i++) - WRITE_CSR_ARB_WQCFG(csr, i, 0); - /* Unmap worker threads to service arbiters */ for (i = 0; i < hw_data->num_engines; i++) WRITE_CSR_ARB_WRK_2_SER_MAP(csr, i, 0); -- cgit v1.2.3 From c685d7a7ceb2961dca61914bf473304e08138e11 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:30 +0100 Subject: crypto: qat - remove unused macros in arbiter module Remove the unused macros ADF_ARB_WTR_SIZE, ADF_ARB_WTR_OFFSET and ADF_ARB_RO_EN_OFFSET. These macros were left in commit 34074205bb9f ("crypto: qat - remove redundant arbiter configuration") that removed the logic that used those defines. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_hw_arbiter.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c index cbb9f0b8ff74..be2fd264a223 100644 --- a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c +++ b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c @@ -6,11 +6,8 @@ #define ADF_ARB_NUM 4 #define ADF_ARB_REG_SIZE 0x4 -#define ADF_ARB_WTR_SIZE 0x20 #define ADF_ARB_OFFSET 0x30000 #define ADF_ARB_REG_SLOT 0x1000 -#define ADF_ARB_WTR_OFFSET 0x010 -#define ADF_ARB_RO_EN_OFFSET 0x090 #define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 #define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C -- cgit v1.2.3 From 369eb4aaae46b1e40142fbe0ef27b2646c21e1e9 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:31 +0100 Subject: crypto: qat - abstract arbiter access The arbiter configuration, the offset to the arbiter config CSR and the offset to the worker thread to service arbiter CSR are going to be different in QAT GEN4 devices although the logic that uses them is the same across all QAT generations. This patch reworks the gen-specific parts of the arbiter access code by introducing the arb_info structure, that contains the values that are generation specific, and a function in the structure adf_hw_device_data, get_arb_info(), that allows to get them. Since the arbiter values for QAT GEN2 devices (c62x, c3xxx and dh895xcc) are the same, a single function, adf_gen2_get_arb_info() is provided in adf_gen2_hw_data.c and referenced by each QAT GEN2 driver. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 1 + drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c | 1 + drivers/crypto/qat/qat_common/adf_accel_devices.h | 7 ++++ drivers/crypto/qat/qat_common/adf_gen2_hw_data.c | 8 +++++ drivers/crypto/qat/qat_common/adf_gen2_hw_data.h | 6 ++++ drivers/crypto/qat/qat_common/adf_hw_arbiter.c | 41 ++++++++++++++-------- .../crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 1 + 7 files changed, 50 insertions(+), 15 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index f72ed415800e..f3f33dc0c316 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -203,6 +203,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->get_pf2vf_offset = get_pf2vf_offset; hw_data->get_vintmsk_offset = get_vintmsk_offset; hw_data->get_admin_info = adf_gen2_get_admin_info; + hw_data->get_arb_info = adf_gen2_get_arb_info; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_C3XXX_FW; hw_data->fw_mmp_name = ADF_C3XXX_MMP; diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index d4443523dc9d..53c03b2f763f 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -213,6 +213,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->get_pf2vf_offset = get_pf2vf_offset; hw_data->get_vintmsk_offset = get_vintmsk_offset; hw_data->get_admin_info = adf_gen2_get_admin_info; + hw_data->get_arb_info = adf_gen2_get_arb_info; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_C62X_FW; hw_data->fw_mmp_name = ADF_C62X_MMP; diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 779f62fde3bd..951072feb176 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -97,6 +97,12 @@ struct adf_hw_device_class { u32 instances; } __packed; +struct arb_info { + u32 arb_cfg; + u32 arb_offset; + u32 wt2sam_offset; +}; + struct admin_info { u32 admin_msg_ur; u32 admin_msg_lr; @@ -144,6 +150,7 @@ struct adf_hw_device_data { u32 (*get_num_accels)(struct adf_hw_device_data *self); u32 (*get_pf2vf_offset)(u32 i); u32 (*get_vintmsk_offset)(u32 i); + void (*get_arb_info)(struct arb_info *arb_csrs_info); void (*get_admin_info)(struct admin_info *admin_csrs_info); enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self); int (*alloc_irq)(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index 15a0bc921d7e..b2f770cc29d8 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -45,6 +45,14 @@ void adf_gen2_get_admin_info(struct admin_info *admin_csrs_info) } EXPORT_SYMBOL_GPL(adf_gen2_get_admin_info); +void adf_gen2_get_arb_info(struct arb_info *arb_info) +{ + arb_info->arb_cfg = ADF_ARB_CONFIG; + arb_info->arb_offset = ADF_ARB_OFFSET; + arb_info->wt2sam_offset = ADF_ARB_WRK_2_SER_MAP_OFFSET; +} +EXPORT_SYMBOL_GPL(adf_gen2_get_arb_info); + static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) { return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index e9d2591b2be8..fe4ea3220bca 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -97,9 +97,15 @@ do { \ #define ADF_ADMINMSGLR_OFFSET (0x3A000 + 0x578) #define ADF_MAILBOX_BASE_OFFSET 0x20970 +/* Arbiter configuration */ +#define ADF_ARB_OFFSET 0x30000 +#define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 +#define ADF_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) + void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs); void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); void adf_gen2_get_admin_info(struct admin_info *admin_csrs_info); +void adf_gen2_get_arb_info(struct arb_info *arb_info); #endif diff --git a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c index be2fd264a223..9dc9d58f6093 100644 --- a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c +++ b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c @@ -6,36 +6,39 @@ #define ADF_ARB_NUM 4 #define ADF_ARB_REG_SIZE 0x4 -#define ADF_ARB_OFFSET 0x30000 #define ADF_ARB_REG_SLOT 0x1000 -#define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 #define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C #define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \ ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \ (ADF_ARB_REG_SLOT * index), value) -#define WRITE_CSR_ARB_SARCONFIG(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, ADF_ARB_OFFSET + \ - (ADF_ARB_REG_SIZE * index), value) +#define WRITE_CSR_ARB_SARCONFIG(csr_addr, arb_offset, index, value) \ + ADF_CSR_WR(csr_addr, (arb_offset) + \ + (ADF_ARB_REG_SIZE * (index)), value) -#define WRITE_CSR_ARB_WRK_2_SER_MAP(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ - ADF_ARB_WRK_2_SER_MAP_OFFSET) + \ - (ADF_ARB_REG_SIZE * index), value) +#define WRITE_CSR_ARB_WT2SAM(csr_addr, arb_offset, wt_offset, index, value) \ + ADF_CSR_WR(csr_addr, ((arb_offset) + (wt_offset)) + \ + (ADF_ARB_REG_SIZE * (index)), value) int adf_init_arb(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; void __iomem *csr = accel_dev->transport->banks[0].csr_addr; - u32 arb_cfg = 0x1 << 31 | 0x4 << 4 | 0x1; - u32 arb, i; + u32 arb_off, wt_off, arb_cfg; const u32 *thd_2_arb_cfg; + struct arb_info info; + int arb, i; + + hw_data->get_arb_info(&info); + arb_cfg = info.arb_cfg; + arb_off = info.arb_offset; + wt_off = info.wt2sam_offset; /* Service arb configured for 32 bytes responses and * ring flow control check enabled. */ for (arb = 0; arb < ADF_ARB_NUM; arb++) - WRITE_CSR_ARB_SARCONFIG(csr, arb, arb_cfg); + WRITE_CSR_ARB_SARCONFIG(csr, arb_off, arb, arb_cfg); /* Map worker threads to service arbiters */ hw_data->get_arb_mapping(accel_dev, &thd_2_arb_cfg); @@ -44,7 +47,7 @@ int adf_init_arb(struct adf_accel_dev *accel_dev) return -EFAULT; for (i = 0; i < hw_data->num_engines; i++) - WRITE_CSR_ARB_WRK_2_SER_MAP(csr, i, *(thd_2_arb_cfg + i)); + WRITE_CSR_ARB_WT2SAM(csr, arb_off, wt_off, i, thd_2_arb_cfg[i]); return 0; } @@ -60,21 +63,29 @@ void adf_update_ring_arb(struct adf_etr_ring_data *ring) void adf_exit_arb(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 arb_off, wt_off; + struct arb_info info; void __iomem *csr; unsigned int i; + hw_data->get_arb_info(&info); + arb_off = info.arb_offset; + wt_off = info.wt2sam_offset; + if (!accel_dev->transport) return; csr = accel_dev->transport->banks[0].csr_addr; + hw_data->get_arb_info(&info); + /* Reset arbiter configuration */ for (i = 0; i < ADF_ARB_NUM; i++) - WRITE_CSR_ARB_SARCONFIG(csr, i, 0); + WRITE_CSR_ARB_SARCONFIG(csr, arb_off, i, 0); /* Unmap worker threads to service arbiters */ for (i = 0; i < hw_data->num_engines; i++) - WRITE_CSR_ARB_WRK_2_SER_MAP(csr, i, 0); + WRITE_CSR_ARB_WT2SAM(csr, arb_off, wt_off, i, 0); /* Disable arbitration on all rings */ for (i = 0; i < GET_MAX_BANKS(accel_dev); i++) diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index c568e9808cec..2e7017a3ad46 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -211,6 +211,7 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->get_pf2vf_offset = get_pf2vf_offset; hw_data->get_vintmsk_offset = get_vintmsk_offset; hw_data->get_admin_info = adf_gen2_get_admin_info; + hw_data->get_arb_info = adf_gen2_get_arb_info; hw_data->get_sram_bar_id = get_sram_bar_id; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_DH895XCC_FW; -- cgit v1.2.3 From ad1332aa67eca6223b130cb593621ee16439b902 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Mon, 12 Oct 2020 21:38:32 +0100 Subject: crypto: qat - add support for capability detection Add logic to detect device capabilities for c62x, c3xxx and dh895xcc. Read fuses, straps and legfuses CSRs and build the device capabilities mask. This will be used to understand if a certain service is supported by a device. This patch is based on earlier work done by Conor McLoughlin. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 2 ++ drivers/crypto/qat/qat_c3xxx/adf_drv.c | 5 ++-- drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c | 2 ++ drivers/crypto/qat/qat_c62x/adf_drv.c | 5 ++-- drivers/crypto/qat/qat_common/adf_accel_devices.h | 1 + drivers/crypto/qat/qat_common/adf_gen2_hw_data.c | 30 ++++++++++++++++++++++ drivers/crypto/qat/qat_common/adf_gen2_hw_data.h | 4 +++ drivers/crypto/qat/qat_common/icp_qat_hw.h | 23 +++++++++++++++++ .../crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 25 ++++++++++++++++++ drivers/crypto/qat/qat_dh895xcc/adf_drv.c | 5 ++-- 10 files changed, 93 insertions(+), 9 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index f3f33dc0c316..eb45f1b1ae3e 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -5,6 +5,7 @@ #include #include #include "adf_c3xxx_hw_data.h" +#include "icp_qat_hw.h" /* Worker thread to service arbiter mappings based on dev SKUs */ static const u32 thrd_to_arb_map_6_me_sku[] = { @@ -195,6 +196,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->enable_error_correction = adf_enable_error_correction; hw_data->get_accel_mask = get_accel_mask; hw_data->get_ae_mask = get_ae_mask; + hw_data->get_accel_cap = adf_gen2_get_accel_cap; hw_data->get_num_accels = get_num_accels; hw_data->get_num_aes = get_num_aes; hw_data->get_sram_bar_id = get_sram_bar_id; diff --git a/drivers/crypto/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/qat/qat_c3xxx/adf_drv.c index da6e88026988..7fb3343ae8b0 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_drv.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_drv.c @@ -177,9 +177,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err_disable; } - /* Read accelerator capabilities mask */ - pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, - &hw_data->accel_capabilities_mask); + /* Get accelerator capabilities mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); /* Find and map all the device's BARS */ i = 0; diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index 53c03b2f763f..babdffbcb846 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -5,6 +5,7 @@ #include #include #include "adf_c62x_hw_data.h" +#include "icp_qat_hw.h" /* Worker thread to service arbiter mappings based on dev SKUs */ static const u32 thrd_to_arb_map_8_me_sku[] = { @@ -205,6 +206,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->enable_error_correction = adf_enable_error_correction; hw_data->get_accel_mask = get_accel_mask; hw_data->get_ae_mask = get_ae_mask; + hw_data->get_accel_cap = adf_gen2_get_accel_cap; hw_data->get_num_accels = get_num_accels; hw_data->get_num_aes = get_num_aes; hw_data->get_sram_bar_id = get_sram_bar_id; diff --git a/drivers/crypto/qat/qat_c62x/adf_drv.c b/drivers/crypto/qat/qat_c62x/adf_drv.c index 3da697a566ad..1f5de442e1e6 100644 --- a/drivers/crypto/qat/qat_c62x/adf_drv.c +++ b/drivers/crypto/qat/qat_c62x/adf_drv.c @@ -177,9 +177,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err_disable; } - /* Read accelerator capabilities mask */ - pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, - &hw_data->accel_capabilities_mask); + /* Get accelerator capabilities mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); /* Find and map all the device's BARS */ i = (hw_data->fuses & ADF_DEVICE_FUSECTL_MASK) ? 1 : 0; diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 951072feb176..692e39e5e878 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -143,6 +143,7 @@ struct adf_hw_device_data { struct adf_hw_device_class *dev_class; u32 (*get_accel_mask)(struct adf_hw_device_data *self); u32 (*get_ae_mask)(struct adf_hw_device_data *self); + u32 (*get_accel_cap)(struct adf_accel_dev *accel_dev); u32 (*get_sram_bar_id)(struct adf_hw_device_data *self); u32 (*get_misc_bar_id)(struct adf_hw_device_data *self); u32 (*get_etr_bar_id)(struct adf_hw_device_data *self); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index b2f770cc29d8..d5560e714167 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2020 Intel Corporation */ #include "adf_gen2_hw_data.h" +#include "icp_qat_hw.h" +#include void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs) @@ -136,3 +138,31 @@ void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; } EXPORT_SYMBOL_GPL(adf_gen2_init_hw_csr_ops); + +u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; + u32 straps = hw_data->straps; + u32 fuses = hw_data->fuses; + u32 legfuses; + u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + + /* Read accelerator capabilities mask */ + pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, &legfuses); + + if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + if (legfuses & ICP_ACCEL_MASK_PKE_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + + if ((straps | fuses) & ADF_POWERGATE_PKE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + + return capabilities; +} +EXPORT_SYMBOL_GPL(adf_gen2_get_accel_cap); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index fe4ea3220bca..6c860aedb301 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -102,10 +102,14 @@ do { \ #define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 #define ADF_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) +/* Power gating */ +#define ADF_POWERGATE_PKE BIT(24) + void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs); void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); void adf_gen2_get_admin_info(struct admin_info *admin_csrs_info); void adf_gen2_get_arb_info(struct arb_info *arb_info); +u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev); #endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_hw.h b/drivers/crypto/qat/qat_common/icp_qat_hw.h index c4b6ef1506ab..4aa5d724e11b 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hw.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hw.h @@ -65,6 +65,29 @@ struct icp_qat_hw_auth_config { __u32 reserved; }; +enum icp_qat_slice_mask { + ICP_ACCEL_MASK_CIPHER_SLICE = BIT(0), + ICP_ACCEL_MASK_AUTH_SLICE = BIT(1), + ICP_ACCEL_MASK_PKE_SLICE = BIT(2), + ICP_ACCEL_MASK_COMPRESS_SLICE = BIT(3), + ICP_ACCEL_MASK_LZS_SLICE = BIT(4), + ICP_ACCEL_MASK_EIA3_SLICE = BIT(5), + ICP_ACCEL_MASK_SHA3_SLICE = BIT(6), +}; + +enum icp_qat_capabilities_mask { + ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC = BIT(0), + ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC = BIT(1), + ICP_ACCEL_CAPABILITIES_CIPHER = BIT(2), + ICP_ACCEL_CAPABILITIES_AUTHENTICATION = BIT(3), + ICP_ACCEL_CAPABILITIES_RESERVED_1 = BIT(4), + ICP_ACCEL_CAPABILITIES_COMPRESSION = BIT(5), + ICP_ACCEL_CAPABILITIES_LZS_COMPRESSION = BIT(6), + ICP_ACCEL_CAPABILITIES_RAND = BIT(7), + ICP_ACCEL_CAPABILITIES_ZUC = BIT(8), + ICP_ACCEL_CAPABILITIES_SHA3 = BIT(9), +}; + #define QAT_AUTH_MODE_BITPOS 4 #define QAT_AUTH_MODE_MASK 0xF #define QAT_AUTH_ALGO_BITPOS 0 diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 2e7017a3ad46..7970ebb67f28 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -5,6 +5,7 @@ #include #include #include "adf_dh895xcc_hw_data.h" +#include "icp_qat_hw.h" /* Worker thread to service arbiter mappings based on dev SKUs */ static const u32 thrd_to_arb_map_sku4[] = { @@ -83,6 +84,29 @@ static u32 get_sram_bar_id(struct adf_hw_device_data *self) return ADF_DH895XCC_SRAM_BAR; } +static u32 get_accel_cap(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; + u32 capabilities; + u32 legfuses; + + capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + + /* Read accelerator capabilities mask */ + pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, &legfuses); + + if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + if (legfuses & ICP_ACCEL_MASK_PKE_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + + return capabilities; +} + static enum dev_sku_info get_sku(struct adf_hw_device_data *self) { int sku = (self->fuses & ADF_DH895XCC_FUSECTL_SKU_MASK) @@ -204,6 +228,7 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->enable_error_correction = adf_enable_error_correction; hw_data->get_accel_mask = get_accel_mask; hw_data->get_ae_mask = get_ae_mask; + hw_data->get_accel_cap = get_accel_cap; hw_data->get_num_accels = get_num_accels; hw_data->get_num_aes = get_num_aes; hw_data->get_etr_bar_id = get_etr_bar_id; diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c index d7941bc2bafd..a9ec4357144c 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c @@ -177,9 +177,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err_disable; } - /* Read accelerator capabilities mask */ - pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, - &hw_data->accel_capabilities_mask); + /* Get accelerator capabilities mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); /* Find and map all the device's BARS */ i = 0; -- cgit v1.2.3 From dbf568755e7781a4ce8bd3c587c2c388600ae661 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:33 +0100 Subject: crypto: qat - register crypto instances based on capability Introduce the function adf_hw_dev_has_crypto() that returns true if a device supports symmetric crypto, asymmetric crypto and authentication services. If a device has crypto capabilities, add crypto instances to the configuration. This is done since the function that allows to retrieve crypto instances, qat_crypto_get_instance_node(), return instances that support all crypto services. Signed-off-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_crypto.c | 7 ++++++- drivers/crypto/qat/qat_common/qat_crypto.h | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c index ab621b7dbd20..089d5d7b738e 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.c +++ b/drivers/crypto/qat/qat_common/qat_crypto.c @@ -117,10 +117,15 @@ int qat_crypto_dev_config(struct adf_accel_dev *accel_dev) { int cpus = num_online_cpus(); int banks = GET_MAX_BANKS(accel_dev); - int instances = min(cpus, banks); char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; int i; unsigned long val; + int instances; + + if (adf_hw_dev_has_crypto(accel_dev)) + instances = min(cpus, banks); + else + instances = 0; if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC)) goto err; diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h index 8d11e94cbf08..b6a4c95ae003 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.h +++ b/drivers/crypto/qat/qat_common/qat_crypto.h @@ -55,4 +55,19 @@ struct qat_crypto_request { bool encryption; }; +static inline bool adf_hw_dev_has_crypto(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + u32 mask = ~hw_device->accel_capabilities_mask; + + if (mask & ADF_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC) + return false; + if (mask & ADF_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC) + return false; + if (mask & ADF_ACCEL_CAPABILITIES_AUTHENTICATION) + return false; + + return true; +} + #endif -- cgit v1.2.3 From 070a34c9dfa3eeb0b18d63c6fec5dfe7861a43ac Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:34 +0100 Subject: crypto: qat - enable ring after pair is programmed Enable arbitration on the TX ring only after the RX ring is programmed. Before this change, arbitration was enabled on the TX ring before the RX ring was programmed allowing the HW to process a request before having the ring pair configured. With this change, the arbitration logic is programmed only if the TX half of the ring mask matches the RX half. This change does not affect QAT GEN2 devices (c62x, c3xxx and dh895xcc), but it is a must for QAT GEN4 devices since the CSRs of the ring pair are locked after arbitration is enabled on the TX ring. Signed-off-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Reviewed-by: Maksim Lukoshkov Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_hw_arbiter.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c index 9dc9d58f6093..bd03c8f54eb4 100644 --- a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c +++ b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c @@ -55,9 +55,27 @@ EXPORT_SYMBOL_GPL(adf_init_arb); void adf_update_ring_arb(struct adf_etr_ring_data *ring) { + struct adf_accel_dev *accel_dev = ring->bank->accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 tx_ring_mask = hw_data->tx_rings_mask; + u32 shift = hw_data->tx_rx_gap; + u32 arben, arben_tx, arben_rx; + u32 rx_ring_mask; + + /* + * Enable arbitration on a ring only if the TX half of the ring mask + * matches the RX part. This results in writes to CSR on both TX and + * RX update - only one is necessary, but both are done for + * simplicity. + */ + rx_ring_mask = tx_ring_mask << shift; + arben_tx = (ring->bank->ring_mask & tx_ring_mask) >> 0; + arben_rx = (ring->bank->ring_mask & rx_ring_mask) >> shift; + arben = arben_tx & arben_rx; + WRITE_CSR_ARB_RINGSRVARBEN(ring->bank->csr_addr, ring->bank->bank_number, - ring->bank->ring_mask & 0xFF); + arben); } void adf_exit_arb(struct adf_accel_dev *accel_dev) -- cgit v1.2.3 From 95a212bb7f1c9c75a5ae72277c734f8bf8fc04e4 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:35 +0100 Subject: crypto: qat - abstract build ring base Abstract the implementation of BUILD_RING_BASE_ADDR. This is in preparation for the introduction of the qat_4xxx driver since the value of the ring base differs between QAT GEN2 and QAT GEN4 devices. Signed-off-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Reviewed-by: Maksim Lukoshkov Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_devices.h | 1 + drivers/crypto/qat/qat_common/adf_gen2_hw_data.c | 6 ++++++ drivers/crypto/qat/qat_common/adf_gen2_hw_data.h | 2 ++ drivers/crypto/qat/qat_common/adf_transport.c | 4 +++- drivers/crypto/qat/qat_common/adf_transport_access_macros.h | 2 -- 5 files changed, 12 insertions(+), 3 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 692e39e5e878..1fd32c56b119 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -110,6 +110,7 @@ struct admin_info { }; struct adf_hw_csr_ops { + u64 (*build_csr_ring_base_addr)(dma_addr_t addr, u32 size); u32 (*read_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, u32 ring); void (*write_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index d5560e714167..5de359165ab4 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -55,6 +55,11 @@ void adf_gen2_get_arb_info(struct arb_info *arb_info) } EXPORT_SYMBOL_GPL(adf_gen2_get_arb_info); +static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size) +{ + return BUILD_RING_BASE_ADDR(addr, size); +} + static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) { return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); @@ -124,6 +129,7 @@ static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank, void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) { + csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr; csr_ops->read_csr_ring_head = read_csr_ring_head; csr_ops->write_csr_ring_head = write_csr_ring_head; csr_ops->read_csr_ring_tail = read_csr_ring_tail; diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index 6c860aedb301..212ff395201f 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -23,6 +23,8 @@ #define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 #define ADF_RING_BUNDLE_SIZE 0x1000 +#define BUILD_RING_BASE_ADDR(addr, size) \ + (((addr) >> 6) & (0xFFFFFFFFFFFFFFFFULL << (size))) #define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ ADF_RING_CSR_RING_HEAD + ((ring) << 2)) diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c index 03fb7812818b..dd8f94fcb9a8 100644 --- a/drivers/crypto/qat/qat_common/adf_transport.c +++ b/drivers/crypto/qat/qat_common/adf_transport.c @@ -180,7 +180,9 @@ static int adf_init_ring(struct adf_etr_ring_data *ring) else adf_configure_rx_ring(ring); - ring_base = BUILD_RING_BASE_ADDR(ring->dma_addr, ring->ring_size); + ring_base = csr_ops->build_csr_ring_base_addr(ring->dma_addr, + ring->ring_size); + csr_ops->write_csr_ring_base(ring->bank->csr_addr, ring->bank->bank_number, ring->ring_number, ring_base); diff --git a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h index 4642b0b5cfb0..12b1605a740e 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h +++ b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h @@ -56,6 +56,4 @@ ((watermark_nf << ADF_RING_CONFIG_NEAR_FULL_WM) \ | (watermark_ne << ADF_RING_CONFIG_NEAR_EMPTY_WM) \ | size) -#define BUILD_RING_BASE_ADDR(addr, size) \ - ((addr >> 6) & (0xFFFFFFFFFFFFFFFFULL << size)) #endif -- cgit v1.2.3 From fe779a46e85fa4433c650c8698065ea6f715cdba Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:36 +0100 Subject: crypto: qat - replace constant masks with GENMASK Replace constant 0xFFFFFFFFFFFFFFFFULL with GENMASK_ULL(63, 0) and 0xFFFFFFFF with GENMASK(31, 0) as they are masks. This makes code less error prone. Suggested-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_gen2_hw_data.h | 2 +- drivers/crypto/qat/qat_common/adf_sriov.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index 212ff395201f..04236a442f3c 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -24,7 +24,7 @@ #define ADF_RING_BUNDLE_SIZE 0x1000 #define BUILD_RING_BASE_ADDR(addr, size) \ - (((addr) >> 6) & (0xFFFFFFFFFFFFFFFFULL << (size))) + (((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) #define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ ADF_RING_CSR_RING_HEAD + ((ring) << 2)) diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index dde6c57ef15a..0e8eab057d2d 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -99,7 +99,7 @@ void adf_disable_sriov(struct adf_accel_dev *accel_dev) pci_disable_sriov(accel_to_pci_dev(accel_dev)); /* Disable VF to PF interrupts */ - adf_disable_vf2pf_interrupts(accel_dev, 0xFFFFFFFF); + adf_disable_vf2pf_interrupts(accel_dev, GENMASK(31, 0)); /* Clear Valid bits in AE Thread to PCIe Function Mapping */ hw_data->configure_iov_threads(accel_dev, false); -- cgit v1.2.3 From 097430ff7809c65697451a56b48fbe520b2ea27c Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:37 +0100 Subject: crypto: qat - use BIT_ULL() - 1 pattern for masks Replace occurrences of the pattern GENMASK_ULL(var - 1, 0)) with BIT_ULL(var) - 1 since it produces better code and it is easier to read. Suggested-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_sriov.c | 2 +- drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index 0e8eab057d2d..9a0f6db83106 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -65,7 +65,7 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) hw_data->configure_iov_threads(accel_dev, true); /* Enable VF to PF interrupts for all VFs */ - adf_enable_vf2pf_interrupts(accel_dev, GENMASK_ULL(totalvfs - 1, 0)); + adf_enable_vf2pf_interrupts(accel_dev, BIT_ULL(totalvfs) - 1); /* * Due to the hardware design, when SR-IOV and the ring arbiter diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 7970ebb67f28..1e83d9397b11 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -195,7 +195,7 @@ static void adf_enable_ints(struct adf_accel_dev *accel_dev) /* Enable bundle and misc interrupts */ ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF0_MASK_OFFSET, accel_dev->pf.vf_info ? 0 : - GENMASK_ULL(GET_MAX_BANKS(accel_dev) - 1, 0)); + BIT_ULL(GET_MAX_BANKS(accel_dev)) - 1); ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF1_MASK_OFFSET, ADF_DH895XCC_SMIA1_MASK); } -- cgit v1.2.3 From 02690ddcb8f17e33246e66826a55ad02750f8e76 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:38 +0100 Subject: crypto: qat - abstract writes to arbiter enable Abstract writes to the service arbiter enable register. This is in preparation for the introduction of the qat_4xxx driver since the arbitration enable register differes between QAT GEN2 and QAT GEN4 devices. Signed-off-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Reviewed-by: Maksim Lukoshkov Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_devices.h | 2 ++ drivers/crypto/qat/qat_common/adf_gen2_hw_data.c | 7 +++++++ drivers/crypto/qat/qat_common/adf_gen2_hw_data.h | 6 ++++++ drivers/crypto/qat/qat_common/adf_hw_arbiter.c | 15 +++++---------- 4 files changed, 20 insertions(+), 10 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 1fd32c56b119..a3b63dfe4d7b 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -133,6 +133,8 @@ struct adf_hw_csr_ops { u32 value); void (*write_csr_int_flag_and_col)(void __iomem *csr_base_addr, u32 bank, u32 value); + void (*write_csr_ring_srv_arb_en)(void __iomem *csr_base_addr, u32 bank, + u32 value); }; struct adf_cfg_device_data; diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index 5de359165ab4..1aa17303838d 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -127,6 +127,12 @@ static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank, WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value); } +static void write_csr_ring_srv_arb_en(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value); +} + void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) { csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr; @@ -142,6 +148,7 @@ void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) csr_ops->write_csr_int_col_en = write_csr_int_col_en; csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl; csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; + csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en; } EXPORT_SYMBOL_GPL(adf_gen2_init_hw_csr_ops); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index 04236a442f3c..3816e6500352 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -103,6 +103,12 @@ do { \ #define ADF_ARB_OFFSET 0x30000 #define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 #define ADF_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) +#define ADF_ARB_REG_SLOT 0x1000 +#define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C + +#define WRITE_CSR_RING_SRV_ARB_EN(csr_addr, index, value) \ + ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \ + (ADF_ARB_REG_SLOT * (index)), value) /* Power gating */ #define ADF_POWERGATE_PKE BIT(24) diff --git a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c index bd03c8f54eb4..9f5240d9488b 100644 --- a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c +++ b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c @@ -6,12 +6,6 @@ #define ADF_ARB_NUM 4 #define ADF_ARB_REG_SIZE 0x4 -#define ADF_ARB_REG_SLOT 0x1000 -#define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C - -#define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \ - (ADF_ARB_REG_SLOT * index), value) #define WRITE_CSR_ARB_SARCONFIG(csr_addr, arb_offset, index, value) \ ADF_CSR_WR(csr_addr, (arb_offset) + \ @@ -57,6 +51,7 @@ void adf_update_ring_arb(struct adf_etr_ring_data *ring) { struct adf_accel_dev *accel_dev = ring->bank->accel_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); u32 tx_ring_mask = hw_data->tx_rings_mask; u32 shift = hw_data->tx_rx_gap; u32 arben, arben_tx, arben_rx; @@ -73,14 +68,14 @@ void adf_update_ring_arb(struct adf_etr_ring_data *ring) arben_rx = (ring->bank->ring_mask & rx_ring_mask) >> shift; arben = arben_tx & arben_rx; - WRITE_CSR_ARB_RINGSRVARBEN(ring->bank->csr_addr, - ring->bank->bank_number, - arben); + csr_ops->write_csr_ring_srv_arb_en(ring->bank->csr_addr, + ring->bank->bank_number, arben); } void adf_exit_arb(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); u32 arb_off, wt_off; struct arb_info info; void __iomem *csr; @@ -107,6 +102,6 @@ void adf_exit_arb(struct adf_accel_dev *accel_dev) /* Disable arbitration on all rings */ for (i = 0; i < GET_MAX_BANKS(accel_dev); i++) - WRITE_CSR_ARB_RINGSRVARBEN(csr, i, 0); + csr_ops->write_csr_ring_srv_arb_en(csr, i, 0); } EXPORT_SYMBOL_GPL(adf_exit_arb); -- cgit v1.2.3 From 264f590f2beb80d29dc33d5b60f93d2787f429b8 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:39 +0100 Subject: crypto: qat - remove hardcoded bank irq clear flag mask Replace hardcoded value of the bank interrupt clear flag mask with a value calculated on the fly which is based on the number of rings present in a bank. This is to support devices that have a number of rings per bank different than 16. Signed-off-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_transport.c | 4 ++-- drivers/crypto/qat/qat_common/adf_transport_access_macros.h | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c index dd8f94fcb9a8..5a7030acdc33 100644 --- a/drivers/crypto/qat/qat_common/adf_transport.c +++ b/drivers/crypto/qat/qat_common/adf_transport.c @@ -374,6 +374,7 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, struct adf_hw_device_data *hw_data = accel_dev->hw_device; u8 num_rings_per_bank = hw_data->num_rings_per_bank; struct adf_hw_csr_ops *csr_ops = &hw_data->csr_ops; + u32 irq_mask = BIT(num_rings_per_bank) - 1; struct adf_etr_ring_data *ring; struct adf_etr_ring_data *tx_ring; u32 i, coalesc_enabled = 0; @@ -431,8 +432,7 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, goto err; } - csr_ops->write_csr_int_flag(csr_addr, bank_num, - ADF_BANK_INT_FLAG_CLEAR_MASK); + csr_ops->write_csr_int_flag(csr_addr, bank_num, irq_mask); csr_ops->write_csr_int_srcsel(csr_addr, bank_num); return 0; diff --git a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h index 12b1605a740e..3b6b0267bbec 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h +++ b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h @@ -4,7 +4,6 @@ #define ADF_TRANSPORT_ACCESS_MACROS_H #include "adf_accel_devices.h" -#define ADF_BANK_INT_FLAG_CLEAR_MASK 0xFFFF #define ADF_RING_CONFIG_NEAR_FULL_WM 0x0A #define ADF_RING_CONFIG_NEAR_EMPTY_WM 0x05 #define ADF_COALESCING_MIN_TIME 0x1FF -- cgit v1.2.3 From 185aa9c3543e5b7303e3e9c97a9b8f06fec04afe Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:40 +0100 Subject: crypto: qat - call functions in adf_sriov if available Call the function configure_iov_threads(), adf_enable_vf2pf_interrupts() and adf_pf2vf_notify_restarting() only if present in the struct adf_hw_device_data of the device. This is to allow for QAT drivers that do not implement those functions. Signed-off-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Reviewed-by: Maksim Lukoshkov Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_sriov.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index 9a0f6db83106..d887640355d4 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -62,10 +62,12 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) } /* Set Valid bits in AE Thread to PCIe Function Mapping */ - hw_data->configure_iov_threads(accel_dev, true); + if (hw_data->configure_iov_threads) + hw_data->configure_iov_threads(accel_dev, true); /* Enable VF to PF interrupts for all VFs */ - adf_enable_vf2pf_interrupts(accel_dev, BIT_ULL(totalvfs) - 1); + if (hw_data->get_pf2vf_offset) + adf_enable_vf2pf_interrupts(accel_dev, BIT_ULL(totalvfs) - 1); /* * Due to the hardware design, when SR-IOV and the ring arbiter @@ -94,15 +96,18 @@ void adf_disable_sriov(struct adf_accel_dev *accel_dev) if (!accel_dev->pf.vf_info) return; - adf_pf2vf_notify_restarting(accel_dev); + if (hw_data->get_pf2vf_offset) + adf_pf2vf_notify_restarting(accel_dev); pci_disable_sriov(accel_to_pci_dev(accel_dev)); /* Disable VF to PF interrupts */ - adf_disable_vf2pf_interrupts(accel_dev, GENMASK(31, 0)); + if (hw_data->get_pf2vf_offset) + adf_disable_vf2pf_interrupts(accel_dev, GENMASK(31, 0)); /* Clear Valid bits in AE Thread to PCIe Function Mapping */ - hw_data->configure_iov_threads(accel_dev, false); + if (hw_data->configure_iov_threads) + hw_data->configure_iov_threads(accel_dev, false); for (i = 0, vf = accel_dev->pf.vf_info; i < totalvfs; i++, vf++) { tasklet_disable(&vf->vf2pf_bh_tasklet); -- cgit v1.2.3 From 83ab8c4b2d893cf8718236e539287661c34df9ff Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:41 +0100 Subject: crypto: qat - remove unnecessary void* casts Remove superfluous casts to void* in function qat_crypto_dev_config(). Suggested-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_crypto.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c index 089d5d7b738e..ab1716f7044d 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.c +++ b/drivers/crypto/qat/qat_common/qat_crypto.c @@ -135,61 +135,61 @@ int qat_crypto_dev_config(struct adf_accel_dev *accel_dev) val = i; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i); if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + key, &val, ADF_DEC)) goto err; snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, i); if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + key, &val, ADF_DEC)) goto err; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); val = 128; if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + key, &val, ADF_DEC)) goto err; val = 512; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + key, &val, ADF_DEC)) goto err; val = 0; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + key, &val, ADF_DEC)) goto err; val = 2; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + key, &val, ADF_DEC)) goto err; val = 8; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + key, &val, ADF_DEC)) goto err; val = 10; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + key, &val, ADF_DEC)) goto err; val = ADF_COALESCING_DEF_TIME; snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0", - key, (void *)&val, ADF_DEC)) + key, &val, ADF_DEC)) goto err; } val = i; if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - ADF_NUM_CY, (void *)&val, ADF_DEC)) + ADF_NUM_CY, &val, ADF_DEC)) goto err; set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); -- cgit v1.2.3 From 69d0fc22cf91c9488fcc6e6095b70e3f228ad580 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:42 +0100 Subject: crypto: qat - change return value in adf_cfg_add_key_value_param() If the parameter type provided to adf_cfg_add_key_value_param() is invalid, return -EINVAL instead of -1 that is treated as -EPERM and may confuse. Suggested-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_cfg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/qat/qat_common/adf_cfg.c index 22ae32838113..f2a29c70d61a 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg.c +++ b/drivers/crypto/qat/qat_common/adf_cfg.c @@ -243,7 +243,7 @@ int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, } else { dev_err(&GET_DEV(accel_dev), "Unknown type given.\n"); kfree(key_val); - return -1; + return -EINVAL; } key_val->type = type; down_write(&cfg->lock); -- cgit v1.2.3 From 48710b1f31aef65918cb4396da530ad6ba00cd32 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:43 +0100 Subject: crypto: qat - change return value in adf_cfg_key_val_get() If a key is not found in the internal key value storage, return -ENODATA instead of -1 that is treated as -EPERM and may confuse. Suggested-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_cfg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/qat/qat_common/adf_cfg.c index f2a29c70d61a..575b6f002303 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg.c +++ b/drivers/crypto/qat/qat_common/adf_cfg.c @@ -196,7 +196,7 @@ static int adf_cfg_key_val_get(struct adf_accel_dev *accel_dev, memcpy(val, keyval->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES); return 0; } - return -1; + return -ENODATA; } /** -- cgit v1.2.3 From 533179ce377f0dc4aa56857feeae012731dbef9a Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:44 +0100 Subject: crypto: qat - refactor qat_crypto_create_instances() Refactor function qat_crypto_create_instances() to propagate errors to the caller. Suggested-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_crypto.c | 68 ++++++++++++++++++------------ 1 file changed, 41 insertions(+), 27 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c index ab1716f7044d..735463042987 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.c +++ b/drivers/crypto/qat/qat_common/qat_crypto.c @@ -202,83 +202,97 @@ EXPORT_SYMBOL_GPL(qat_crypto_dev_config); static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) { - int i; - unsigned long bank; - unsigned long num_inst, num_msg_sym, num_msg_asym; - int msg_size; - struct qat_crypto_instance *inst; + unsigned long bank, num_inst, num_msg_sym, num_msg_asym; char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + struct qat_crypto_instance *inst; + int msg_size; + int ret; + int i; INIT_LIST_HEAD(&accel_dev->crypto_list); - if (adf_cfg_get_param_value(accel_dev, SEC, ADF_NUM_CY, val)) - return -EFAULT; + ret = adf_cfg_get_param_value(accel_dev, SEC, ADF_NUM_CY, val); + if (ret) + return ret; - if (kstrtoul(val, 0, &num_inst)) - return -EFAULT; + ret = kstrtoul(val, 0, &num_inst); + if (ret) + return ret; for (i = 0; i < num_inst; i++) { inst = kzalloc_node(sizeof(*inst), GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev))); - if (!inst) + if (!inst) { + ret = -ENOMEM; goto err; + } list_add_tail(&inst->list, &accel_dev->crypto_list); inst->id = i; atomic_set(&inst->refctr, 0); inst->accel_dev = accel_dev; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i); - if (adf_cfg_get_param_value(accel_dev, SEC, key, val)) + ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); + if (ret) goto err; - if (kstrtoul(val, 10, &bank)) + ret = kstrtoul(val, 10, &bank); + if (ret) goto err; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); - if (adf_cfg_get_param_value(accel_dev, SEC, key, val)) + ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); + if (ret) goto err; - if (kstrtoul(val, 10, &num_msg_sym)) + ret = kstrtoul(val, 10, &num_msg_sym); + if (ret) goto err; num_msg_sym = num_msg_sym >> 1; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); - if (adf_cfg_get_param_value(accel_dev, SEC, key, val)) + ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); + if (ret) goto err; - if (kstrtoul(val, 10, &num_msg_asym)) + ret = kstrtoul(val, 10, &num_msg_asym); + if (ret) goto err; num_msg_asym = num_msg_asym >> 1; msg_size = ICP_QAT_FW_REQ_DEFAULT_SZ; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); - if (adf_create_ring(accel_dev, SEC, bank, num_msg_sym, - msg_size, key, NULL, 0, &inst->sym_tx)) + ret = adf_create_ring(accel_dev, SEC, bank, num_msg_sym, + msg_size, key, NULL, 0, &inst->sym_tx); + if (ret) goto err; msg_size = msg_size >> 1; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); - if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym, - msg_size, key, NULL, 0, &inst->pke_tx)) + ret = adf_create_ring(accel_dev, SEC, bank, num_msg_asym, + msg_size, key, NULL, 0, &inst->pke_tx); + if (ret) goto err; msg_size = ICP_QAT_FW_RESP_DEFAULT_SZ; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); - if (adf_create_ring(accel_dev, SEC, bank, num_msg_sym, - msg_size, key, qat_alg_callback, 0, - &inst->sym_rx)) + ret = adf_create_ring(accel_dev, SEC, bank, num_msg_sym, + msg_size, key, qat_alg_callback, 0, + &inst->sym_rx); + if (ret) goto err; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); - if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym, - msg_size, key, qat_alg_asym_callback, 0, - &inst->pke_rx)) + ret = adf_create_ring(accel_dev, SEC, bank, num_msg_asym, + msg_size, key, qat_alg_asym_callback, 0, + &inst->pke_rx); + if (ret) goto err; } return 0; err: qat_crypto_free_instances(accel_dev); - return -ENOMEM; + return ret; } static int qat_crypto_init(struct adf_accel_dev *accel_dev) -- cgit v1.2.3 From 54e3d7538fa53e35977f1bfadafba9c4bb2bd149 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:45 +0100 Subject: crypto: qat - refactor qat_crypto_dev_config() Refactor function qat_crypto_dev_config() to propagate errors to the caller. Suggested-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_crypto.c | 67 ++++++++++++++++++------------ 1 file changed, 41 insertions(+), 26 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c index 735463042987..8d8ac5e48f47 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.c +++ b/drivers/crypto/qat/qat_common/qat_crypto.c @@ -115,88 +115,103 @@ struct qat_crypto_instance *qat_crypto_get_instance_node(int node) */ int qat_crypto_dev_config(struct adf_accel_dev *accel_dev) { - int cpus = num_online_cpus(); - int banks = GET_MAX_BANKS(accel_dev); char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; - int i; + int banks = GET_MAX_BANKS(accel_dev); + int cpus = num_online_cpus(); unsigned long val; int instances; + int ret; + int i; if (adf_hw_dev_has_crypto(accel_dev)) instances = min(cpus, banks); else instances = 0; - if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC)) + ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC); + if (ret) goto err; - if (adf_cfg_section_add(accel_dev, "Accelerator0")) + + ret = adf_cfg_section_add(accel_dev, "Accelerator0"); + if (ret) goto err; + for (i = 0; i < instances; i++) { val = i; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); val = 128; - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; val = 512; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; val = 0; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; val = 2; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; val = 8; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; val = 10; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; val = ADF_COALESCING_DEF_TIME; snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); - if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0", - key, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0", + key, &val, ADF_DEC); + if (ret) goto err; } val = i; - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - ADF_NUM_CY, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, + &val, ADF_DEC); + if (ret) goto err; set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); return 0; err: dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n"); - return -EINVAL; + return ret; } EXPORT_SYMBOL_GPL(qat_crypto_dev_config); -- cgit v1.2.3 From 657ad678e0c49a6e2f237f875b7a6c2d5065bbc8 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:46 +0100 Subject: crypto: qat - allow for instances in different banks Allow for crypto instances to be configured with symmetric crypto rings that belong to a bank that is different from the one where asymmetric crypto rings are located. This is to allow for devices with banks made of a single ring pair. In these, crypto instances will be composed of two separate banks. Changed string literals are not exposed to the user space. Signed-off-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_cfg_strings.h | 3 ++- drivers/crypto/qat/qat_common/qat_crypto.c | 34 +++++++++++++++++++------ 2 files changed, 28 insertions(+), 9 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_cfg_strings.h b/drivers/crypto/qat/qat_common/adf_cfg_strings.h index 314790f5b0af..09651e1f937a 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg_strings.h +++ b/drivers/crypto/qat/qat_common/adf_cfg_strings.h @@ -18,7 +18,8 @@ #define ADF_RING_DC_TX "RingTx" #define ADF_RING_DC_RX "RingRx" #define ADF_ETRMGR_BANK "Bank" -#define ADF_RING_BANK_NUM "BankNumber" +#define ADF_RING_SYM_BANK_NUM "BankSymNumber" +#define ADF_RING_ASYM_BANK_NUM "BankAsymNumber" #define ADF_CY "Cy" #define ADF_DC "Dc" #define ADF_ETRMGR_COALESCING_ENABLED "InterruptCoalescingEnabled" diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c index 8d8ac5e48f47..ece6776fbd53 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.c +++ b/drivers/crypto/qat/qat_common/qat_crypto.c @@ -138,7 +138,13 @@ int qat_crypto_dev_config(struct adf_accel_dev *accel_dev) for (i = 0; i < instances; i++) { val = i; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i); + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i); ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, key, &val, ADF_DEC); if (ret) @@ -217,9 +223,10 @@ EXPORT_SYMBOL_GPL(qat_crypto_dev_config); static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) { - unsigned long bank, num_inst, num_msg_sym, num_msg_asym; + unsigned long num_inst, num_msg_sym, num_msg_asym; char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + unsigned long sym_bank, asym_bank; struct qat_crypto_instance *inst; int msg_size; int ret; @@ -246,14 +253,25 @@ static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) inst->id = i; atomic_set(&inst->refctr, 0); inst->accel_dev = accel_dev; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i); + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i); + ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); + if (ret) + goto err; + + ret = kstrtoul(val, 10, &sym_bank); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i); ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); if (ret) goto err; - ret = kstrtoul(val, 10, &bank); + ret = kstrtoul(val, 10, &asym_bank); if (ret) goto err; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); if (ret) @@ -277,28 +295,28 @@ static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) msg_size = ICP_QAT_FW_REQ_DEFAULT_SZ; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); - ret = adf_create_ring(accel_dev, SEC, bank, num_msg_sym, + ret = adf_create_ring(accel_dev, SEC, sym_bank, num_msg_sym, msg_size, key, NULL, 0, &inst->sym_tx); if (ret) goto err; msg_size = msg_size >> 1; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); - ret = adf_create_ring(accel_dev, SEC, bank, num_msg_asym, + ret = adf_create_ring(accel_dev, SEC, asym_bank, num_msg_asym, msg_size, key, NULL, 0, &inst->pke_tx); if (ret) goto err; msg_size = ICP_QAT_FW_RESP_DEFAULT_SZ; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); - ret = adf_create_ring(accel_dev, SEC, bank, num_msg_sym, + ret = adf_create_ring(accel_dev, SEC, sym_bank, num_msg_sym, msg_size, key, qat_alg_callback, 0, &inst->sym_rx); if (ret) goto err; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); - ret = adf_create_ring(accel_dev, SEC, bank, num_msg_asym, + ret = adf_create_ring(accel_dev, SEC, asym_bank, num_msg_asym, msg_size, key, qat_alg_asym_callback, 0, &inst->pke_rx); if (ret) -- cgit v1.2.3 From 3f9ae998355baa35284ed1a4c715b588c21dac08 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:47 +0100 Subject: crypto: qat - extend ae_mask Change type of ae_mask in adf_hw_device_data to allow for devices with more than 16 Acceleration Engines (AEs). Signed-off-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_devices.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index a3b63dfe4d7b..996d25565b11 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -181,7 +181,7 @@ struct adf_hw_device_data { u32 accel_capabilities_mask; u32 instance_id; u16 accel_mask; - u16 ae_mask; + u32 ae_mask; u32 admin_ae_mask; u16 tx_rings_mask; u8 tx_rx_gap; -- cgit v1.2.3 From e68f9cf6601b53c7ee8ece552fbafcd9cfa47faa Mon Sep 17 00:00:00 2001 From: Longfang Liu Date: Thu, 15 Oct 2020 10:23:03 +0800 Subject: crypto: hisilicon - delete unused structure member variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Remove unused member‘pending_reqs' in‘sec_qp_ctx' structure. 2. Remove unused member‘status' in‘sec_dev' structure. Signed-off-by: Longfang Liu Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec2/sec.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 037762b531e2..08491912afd5 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -109,7 +109,6 @@ struct sec_qp_ctx { struct list_head backlog; struct hisi_acc_sgl_pool *c_in_pool; struct hisi_acc_sgl_pool *c_out_pool; - atomic_t pending_reqs; }; enum sec_alg_type { @@ -180,7 +179,6 @@ struct sec_dev { struct sec_debug debug; u32 ctx_q_num; bool iommu_used; - unsigned long status; }; void sec_destroy_qps(struct hisi_qp **qps, int qp_num); -- cgit v1.2.3 From 633e507fba02d62ed573a8dfb430fc3b881b23d6 Mon Sep 17 00:00:00 2001 From: Longfang Liu Date: Thu, 15 Oct 2020 10:23:04 +0800 Subject: crypto: hisilicon - fixes some coding style Clean up extra blank lines Signed-off-by: Longfang Liu Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec2/sec_crypto.c | 17 ++++++----------- drivers/crypto/hisilicon/sec2/sec_main.c | 30 ++++++++++++------------------ 2 files changed, 18 insertions(+), 29 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index bb493423668c..87bc08afe567 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -101,6 +101,7 @@ static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx) req->qp_ctx = qp_ctx; qp_ctx->req_list[req_id] = req; + return req_id; } @@ -317,6 +318,7 @@ static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res) j * SEC_PBUF_PKG + pbuf_page_offset; } } + return 0; } @@ -345,12 +347,12 @@ static int sec_alg_resource_alloc(struct sec_ctx *ctx, } return 0; + alloc_pbuf_fail: if (ctx->alg_type == SEC_AEAD) sec_free_mac_resource(dev, qp_ctx->res); alloc_fail: sec_free_civ_resource(dev, res); - return ret; } @@ -419,7 +421,6 @@ err_free_c_in_pool: hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); err_destroy_idr: idr_destroy(&qp_ctx->req_idr); - return ret; } @@ -557,9 +558,9 @@ static int sec_skcipher_init(struct crypto_skcipher *tfm) goto err_cipher_init; return 0; + err_cipher_init: sec_ctx_base_uninit(ctx); - return ret; } @@ -740,7 +741,6 @@ static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, if (unlikely(pbuf_length != copy_size)) dev_err(dev, "copy pbuf data to dst error!\n"); - } static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, @@ -913,9 +913,9 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, } return 0; + bad_key: memzero_explicit(&keys, sizeof(struct crypto_authenc_keys)); - return -EINVAL; } @@ -966,7 +966,6 @@ static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req) unmap_req_buf: ctx->req_op->buf_unmap(ctx, req); - return ret; } @@ -1107,7 +1106,6 @@ static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, atomic64_inc(&ctx->sec->debug.dfx.recv_busy_cnt); } - sk_req->base.complete(&sk_req->base, err); } @@ -1279,7 +1277,6 @@ err_send_req: sec_request_untransfer(ctx, req); err_uninit_req: sec_request_uninit(ctx, req); - return ret; } @@ -1349,7 +1346,6 @@ err_cipher_init: sec_auth_uninit(ctx); err_auth_init: sec_ctx_base_uninit(ctx); - return ret; } @@ -1437,8 +1433,8 @@ static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) } return 0; } - dev_err(dev, "skcipher algorithm error!\n"); + return -EINVAL; } @@ -1554,7 +1550,6 @@ static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) if (unlikely(c_alg != SEC_CALG_AES)) { dev_err(SEC_CTX_DEV(ctx), "aead crypto alg error!\n"); return -EINVAL; - } if (sreq->c_req.encrypt) sreq->c_req.c_len = req->cryptlen; diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 548896394c4b..2f52581b6d3c 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -660,12 +660,10 @@ static int sec_debugfs_init(struct hisi_qm *qm) if (ret) goto failed_to_create; - return 0; failed_to_create: debugfs_remove_recursive(sec_debugfs_root); - return ret; } @@ -683,13 +681,13 @@ static void sec_log_hw_error(struct hisi_qm *qm, u32 err_sts) while (errs->msg) { if (errs->int_msk & err_sts) { dev_err(dev, "%s [error status=0x%x] found\n", - errs->msg, errs->int_msk); + errs->msg, errs->int_msk); if (SEC_CORE_INT_STATUS_M_ECC & errs->int_msk) { err_val = readl(qm->io_base + SEC_CORE_SRAM_ECC_ERR_INFO); dev_err(dev, "multi ecc sram num=0x%x\n", - SEC_ECC_NUM(err_val)); + SEC_ECC_NUM(err_val)); } } errs++; @@ -724,13 +722,13 @@ static const struct hisi_qm_err_ini sec_err_ini = { .log_dev_hw_err = sec_log_hw_error, .open_axi_master_ooo = sec_open_axi_master_ooo, .err_info = { - .ce = QM_BASE_CE, - .nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT | - QM_ACC_WB_NOT_READY_TIMEOUT, - .fe = 0, - .ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC, - .msi_wr_port = BIT(0), - .acpi_rst = "SRST", + .ce = QM_BASE_CE, + .nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT | + QM_ACC_WB_NOT_READY_TIMEOUT, + .fe = 0, + .ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC, + .msi_wr_port = BIT(0), + .acpi_rst = "SRST", } }; @@ -899,17 +897,13 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) err_alg_unregister: hisi_qm_alg_unregister(qm, &sec_devices); - err_qm_stop: sec_debugfs_exit(qm); hisi_qm_stop(qm, QM_NORMAL); - err_probe_uninit: sec_probe_uninit(qm); - err_qm_uninit: sec_qm_uninit(qm); - return ret; } @@ -936,9 +930,9 @@ static void sec_remove(struct pci_dev *pdev) static const struct pci_error_handlers sec_err_handler = { .error_detected = hisi_qm_dev_err_detected, - .slot_reset = hisi_qm_dev_slot_reset, - .reset_prepare = hisi_qm_reset_prepare, - .reset_done = hisi_qm_reset_done, + .slot_reset = hisi_qm_dev_slot_reset, + .reset_prepare = hisi_qm_reset_prepare, + .reset_done = hisi_qm_reset_done, }; static struct pci_driver sec_pci_driver = { -- cgit v1.2.3 From ad0bb4e4d226e76950531cb4260402bbbb8997b5 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 15 Oct 2020 14:31:53 +1100 Subject: crypto: sa2ul - Reduce stack usage This patch reduces the stack usage in sa2ul: 1. Move the exported sha state into sa_prepare_iopads so that it can occupy the same space as the k_pad buffer. 2. Use one buffer for ipad/opad in sa_prepare_iopads. 3. Remove ipad/opad buffer from sa_set_sc_auth. 4. Use async skcipher fallback and remove on-stack request from sa_cipher_run. Reported-by: kernel test robot Fixes: d2c8ac187fc9 ("crypto: sa2ul - Add AEAD algorithm support") Signed-off-by: Herbert Xu --- drivers/crypto/sa2ul.c | 118 +++++++++++++++++++++++++------------------------ drivers/crypto/sa2ul.h | 6 +-- 2 files changed, 63 insertions(+), 61 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index eda93fab95fe..c357010a159e 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -9,8 +9,10 @@ * Tero Kristo */ #include +#include #include #include +#include #include #include #include @@ -361,42 +363,45 @@ static void sa_swiz_128(u8 *in, u16 len) } /* Prepare the ipad and opad from key as per SHA algorithm step 1*/ -static void prepare_kiopad(u8 *k_ipad, u8 *k_opad, const u8 *key, u16 key_sz) +static void prepare_kipad(u8 *k_ipad, const u8 *key, u16 key_sz) { int i; - for (i = 0; i < key_sz; i++) { + for (i = 0; i < key_sz; i++) k_ipad[i] = key[i] ^ 0x36; - k_opad[i] = key[i] ^ 0x5c; - } /* Instead of XOR with 0 */ - for (; i < SHA1_BLOCK_SIZE; i++) { + for (; i < SHA1_BLOCK_SIZE; i++) k_ipad[i] = 0x36; +} + +static void prepare_kopad(u8 *k_opad, const u8 *key, u16 key_sz) +{ + int i; + + for (i = 0; i < key_sz; i++) + k_opad[i] = key[i] ^ 0x5c; + + /* Instead of XOR with 0 */ + for (; i < SHA1_BLOCK_SIZE; i++) k_opad[i] = 0x5c; - } } -static void sa_export_shash(struct shash_desc *hash, int block_size, +static void sa_export_shash(void *state, struct shash_desc *hash, int digest_size, __be32 *out) { - union { - struct sha1_state sha1; - struct sha256_state sha256; - struct sha512_state sha512; - } sha; - void *state; + struct sha1_state *sha1; + struct sha256_state *sha256; u32 *result; - int i; switch (digest_size) { case SHA1_DIGEST_SIZE: - state = &sha.sha1; - result = sha.sha1.state; + sha1 = state; + result = sha1->state; break; case SHA256_DIGEST_SIZE: - state = &sha.sha256; - result = sha.sha256.state; + sha256 = state; + result = sha256->state; break; default: dev_err(sa_k3_dev, "%s: bad digest_size=%d\n", __func__, @@ -406,8 +411,7 @@ static void sa_export_shash(struct shash_desc *hash, int block_size, crypto_shash_export(hash, state); - for (i = 0; i < digest_size >> 2; i++) - out[i] = cpu_to_be32(result[i]); + cpu_to_be32_array(out, result, digest_size / 4); } static void sa_prepare_iopads(struct algo_data *data, const u8 *key, @@ -416,24 +420,28 @@ static void sa_prepare_iopads(struct algo_data *data, const u8 *key, SHASH_DESC_ON_STACK(shash, data->ctx->shash); int block_size = crypto_shash_blocksize(data->ctx->shash); int digest_size = crypto_shash_digestsize(data->ctx->shash); - u8 k_ipad[SHA1_BLOCK_SIZE]; - u8 k_opad[SHA1_BLOCK_SIZE]; + union { + struct sha1_state sha1; + struct sha256_state sha256; + u8 k_pad[SHA1_BLOCK_SIZE]; + } sha; shash->tfm = data->ctx->shash; - prepare_kiopad(k_ipad, k_opad, key, key_sz); - - memzero_explicit(ipad, block_size); - memzero_explicit(opad, block_size); + prepare_kipad(sha.k_pad, key, key_sz); crypto_shash_init(shash); - crypto_shash_update(shash, k_ipad, block_size); - sa_export_shash(shash, block_size, digest_size, ipad); + crypto_shash_update(shash, sha.k_pad, block_size); + sa_export_shash(&sha, shash, digest_size, ipad); + + prepare_kopad(sha.k_pad, key, key_sz); crypto_shash_init(shash); - crypto_shash_update(shash, k_opad, block_size); + crypto_shash_update(shash, sha.k_pad, block_size); - sa_export_shash(shash, block_size, digest_size, opad); + sa_export_shash(&sha, shash, digest_size, opad); + + memzero_explicit(&sha, sizeof(sha)); } /* Derive the inverse key used in AES-CBC decryption operation */ @@ -506,7 +514,8 @@ static int sa_set_sc_enc(struct algo_data *ad, const u8 *key, u16 key_sz, static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz, u8 *sc_buf) { - __be32 ipad[64], opad[64]; + __be32 *ipad = (void *)(sc_buf + 32); + __be32 *opad = (void *)(sc_buf + 64); /* Set Authentication mode selector to hash processing */ sc_buf[0] = SA_HASH_PROCESSING; @@ -515,14 +524,9 @@ static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz, sc_buf[1] |= ad->auth_ctrl; /* Copy the keys or ipad/opad */ - if (ad->keyed_mac) { + if (ad->keyed_mac) ad->prep_iopad(ad, key, key_sz, ipad, opad); - - /* Copy ipad to AuthKey */ - memcpy(&sc_buf[32], ipad, ad->hash_size); - /* Copy opad to Aux-1 */ - memcpy(&sc_buf[64], opad, ad->hash_size); - } else { + else { /* basic hash */ sc_buf[1] |= SA_BASIC_HASH; } @@ -819,7 +823,7 @@ static void sa_cipher_cra_exit(struct crypto_skcipher *tfm) sa_free_ctx_info(&ctx->enc, data); sa_free_ctx_info(&ctx->dec, data); - crypto_free_sync_skcipher(ctx->fallback.skcipher); + crypto_free_skcipher(ctx->fallback.skcipher); } static int sa_cipher_cra_init(struct crypto_skcipher *tfm) @@ -827,6 +831,7 @@ static int sa_cipher_cra_init(struct crypto_skcipher *tfm) struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); const char *name = crypto_tfm_alg_name(&tfm->base); + struct crypto_skcipher *child; int ret; memzero_explicit(ctx, sizeof(*ctx)); @@ -841,14 +846,17 @@ static int sa_cipher_cra_init(struct crypto_skcipher *tfm) return ret; } - ctx->fallback.skcipher = - crypto_alloc_sync_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); + child = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(ctx->fallback.skcipher)) { + if (IS_ERR(child)) { dev_err(sa_k3_dev, "Error allocating fallback algo %s\n", name); - return PTR_ERR(ctx->fallback.skcipher); + return PTR_ERR(child); } + ctx->fallback.skcipher = child; + crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(child) + + sizeof(struct skcipher_request)); + dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, ctx->dec.sc_id, &ctx->dec.sc_phys); @@ -859,6 +867,7 @@ static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen, struct algo_data *ad) { struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct crypto_skcipher *child = ctx->fallback.skcipher; int cmdl_len; struct sa_cmdl_cfg cfg; int ret; @@ -874,12 +883,10 @@ static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key, cfg.enc_eng_id = ad->enc_eng.eng_id; cfg.iv_size = crypto_skcipher_ivsize(tfm); - crypto_sync_skcipher_clear_flags(ctx->fallback.skcipher, + crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(child, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); - crypto_sync_skcipher_set_flags(ctx->fallback.skcipher, - tfm->base.crt_flags & - CRYPTO_TFM_REQ_MASK); - ret = crypto_sync_skcipher_setkey(ctx->fallback.skcipher, key, keylen); + ret = crypto_skcipher_setkey(child, key, keylen); if (ret) return ret; @@ -1270,7 +1277,6 @@ static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc) crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); struct crypto_alg *alg = req->base.tfm->__crt_alg; struct sa_req sa_req = { 0 }; - int ret; if (!req->cryptlen) return 0; @@ -1282,20 +1288,18 @@ static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc) if (req->cryptlen > SA_MAX_DATA_SZ || (req->cryptlen >= SA_UNSAFE_DATA_SZ_MIN && req->cryptlen <= SA_UNSAFE_DATA_SZ_MAX)) { - SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback.skcipher); + struct skcipher_request *subreq = skcipher_request_ctx(req); - skcipher_request_set_sync_tfm(subreq, ctx->fallback.skcipher); + skcipher_request_set_tfm(subreq, ctx->fallback.skcipher); skcipher_request_set_callback(subreq, req->base.flags, - NULL, NULL); + req->base.complete, + req->base.data); skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, req->iv); if (enc) - ret = crypto_skcipher_encrypt(subreq); + return crypto_skcipher_encrypt(subreq); else - ret = crypto_skcipher_decrypt(subreq); - - skcipher_request_zero(subreq); - return ret; + return crypto_skcipher_decrypt(subreq); } sa_req.size = req->cryptlen; diff --git a/drivers/crypto/sa2ul.h b/drivers/crypto/sa2ul.h index 7f7e3fe60d11..bb40df3876e5 100644 --- a/drivers/crypto/sa2ul.h +++ b/drivers/crypto/sa2ul.h @@ -12,10 +12,8 @@ #ifndef _K3_SA2UL_ #define _K3_SA2UL_ -#include -#include -#include #include +#include #define SA_ENGINE_ENABLE_CONTROL 0x1000 @@ -311,7 +309,7 @@ struct sa_tfm_ctx { struct crypto_shash *shash; /* for fallback */ union { - struct crypto_sync_skcipher *skcipher; + struct crypto_skcipher *skcipher; struct crypto_ahash *ahash; struct crypto_aead *aead; } fallback; -- cgit v1.2.3 From 383e8a823014532ffd81c787ef9009f1c2bd3b79 Mon Sep 17 00:00:00 2001 From: Zhang Qilong Date: Fri, 16 Oct 2020 17:05:36 +0800 Subject: crypto: omap-aes - fix the reference count leak of omap device pm_runtime_get_sync() will increment pm usage counter even when it returns an error code. We should call put operation in error handling paths of omap_aes_hw_init. Signed-off-by: Zhang Qilong Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 4fd14d90cc40..70ea5784a024 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -105,6 +105,7 @@ static int omap_aes_hw_init(struct omap_aes_dev *dd) err = pm_runtime_get_sync(dd->dev); if (err < 0) { + pm_runtime_put_noidle(dd->dev); dev_err(dd->dev, "failed to get sync: %d\n", err); return err; } -- cgit v1.2.3 From f84754dbc55e3abd8241e3038b615af65c745f47 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Mon, 2 Nov 2020 00:22:55 +0100 Subject: soc/fsl/qbman: Add an argument to signal if NAPI processing is required. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dpaa_eth_napi_schedule() and caam_qi_napi_schedule() schedule NAPI if invoked from: - Hard interrupt context - Any context which is not serving soft interrupts Any context which is not serving soft interrupts includes hard interrupts so the in_irq() check is redundant. caam_qi_napi_schedule() has a comment about this: /* * In case of threaded ISR, for RT kernels in_irq() does not return * appropriate value, so use in_serving_softirq to distinguish between * softirq and irq contexts. */ if (in_irq() || !in_serving_softirq()) This has nothing to do with RT. Even on a non RT kernel force threaded interrupts run obviously in thread context and therefore in_irq() returns false when invoked from the handler. The extension of the in_irq() check with !in_serving_softirq() was there when the drivers were added, but in the out of tree FSL BSP the original condition was in_irq() which got extended due to failures on RT. The usage of in_xxx() in drivers is phased out and Linus clearly requested that code which changes behaviour depending on context should either be separated or the context be conveyed in an argument passed by the caller, which usually knows the context. Right he is, the above construct is clearly showing why. The following callchains have been analyzed to end up in dpaa_eth_napi_schedule(): qman_p_poll_dqrr() __poll_portal_fast() fq->cb.dqrr() dpaa_eth_napi_schedule() portal_isr() __poll_portal_fast() fq->cb.dqrr() dpaa_eth_napi_schedule() Both need to schedule NAPI. The crypto part has another code path leading up to this: kill_fq() empty_retired_fq() qman_p_poll_dqrr() __poll_portal_fast() fq->cb.dqrr() dpaa_eth_napi_schedule() kill_fq() is called from task context and ends up scheduling NAPI, but that's pointless and an unintended side effect of the !in_serving_softirq() check. The code path: caam_qi_poll() -> qman_p_poll_dqrr() is invoked from NAPI and I *assume* from crypto's NAPI device and not from qbman's NAPI device. I *guess* it is okay to skip scheduling NAPI (because this is what happens now) but could be changed if it is wrong due to `budget' handling. Add an argument to __poll_portal_fast() which is true if NAPI needs to be scheduled. This requires propagating the value to the caller including `qman_cb_dqrr' typedef which is used by the dpaa and the crypto driver. Signed-off-by: Sebastian Andrzej Siewior Cc: Aymen Sghaier Cc: Herbert XS Cc: Li Yang Reviewed-by: Horia Geantă Signed-off-by: Jakub Kicinski Reviewed-by: Madalin Bucur Tested-by: Camelia Groza --- drivers/crypto/caam/qi.c | 3 ++- drivers/net/ethernet/freescale/dpaa/dpaa_eth.c | 12 ++++++++---- drivers/soc/fsl/qbman/qman.c | 12 ++++++------ drivers/soc/fsl/qbman/qman_test_api.c | 6 ++++-- drivers/soc/fsl/qbman/qman_test_stash.c | 6 ++++-- include/soc/fsl/qman.h | 3 ++- 6 files changed, 26 insertions(+), 16 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index ec53528d8205..57f6ab6bfb56 100644 --- a/drivers/crypto/caam/qi.c +++ b/drivers/crypto/caam/qi.c @@ -564,7 +564,8 @@ static int caam_qi_napi_schedule(struct qman_portal *p, struct caam_napi *np) static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p, struct qman_fq *rsp_fq, - const struct qm_dqrr_entry *dqrr) + const struct qm_dqrr_entry *dqrr, + bool sched_napi) { struct caam_napi *caam_napi = raw_cpu_ptr(&pcpu_qipriv.caam_napi); struct caam_drv_req *drv_req; diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c index 06cc863f4dd6..98ead77c673e 100644 --- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c @@ -2316,7 +2316,8 @@ static inline int dpaa_eth_napi_schedule(struct dpaa_percpu_priv *percpu_priv, static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal, struct qman_fq *fq, - const struct qm_dqrr_entry *dq) + const struct qm_dqrr_entry *dq, + bool sched_napi) { struct dpaa_fq *dpaa_fq = container_of(fq, struct dpaa_fq, fq_base); struct dpaa_percpu_priv *percpu_priv; @@ -2343,7 +2344,8 @@ static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal, static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal, struct qman_fq *fq, - const struct qm_dqrr_entry *dq) + const struct qm_dqrr_entry *dq, + bool sched_napi) { struct skb_shared_hwtstamps *shhwtstamps; struct rtnl_link_stats64 *percpu_stats; @@ -2460,7 +2462,8 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal, static enum qman_cb_dqrr_result conf_error_dqrr(struct qman_portal *portal, struct qman_fq *fq, - const struct qm_dqrr_entry *dq) + const struct qm_dqrr_entry *dq, + bool sched_napi) { struct dpaa_percpu_priv *percpu_priv; struct net_device *net_dev; @@ -2481,7 +2484,8 @@ static enum qman_cb_dqrr_result conf_error_dqrr(struct qman_portal *portal, static enum qman_cb_dqrr_result conf_dflt_dqrr(struct qman_portal *portal, struct qman_fq *fq, - const struct qm_dqrr_entry *dq) + const struct qm_dqrr_entry *dq, + bool sched_napi) { struct dpaa_percpu_priv *percpu_priv; struct net_device *net_dev; diff --git a/drivers/soc/fsl/qbman/qman.c b/drivers/soc/fsl/qbman/qman.c index 9888a7061873..101def7dc73d 100644 --- a/drivers/soc/fsl/qbman/qman.c +++ b/drivers/soc/fsl/qbman/qman.c @@ -1159,7 +1159,7 @@ static u32 fq_to_tag(struct qman_fq *fq) static u32 __poll_portal_slow(struct qman_portal *p, u32 is); static inline unsigned int __poll_portal_fast(struct qman_portal *p, - unsigned int poll_limit); + unsigned int poll_limit, bool sched_napi); static void qm_congestion_task(struct work_struct *work); static void qm_mr_process_task(struct work_struct *work); @@ -1174,7 +1174,7 @@ static irqreturn_t portal_isr(int irq, void *ptr) /* DQRR-handling if it's interrupt-driven */ if (is & QM_PIRQ_DQRI) { - __poll_portal_fast(p, QMAN_POLL_LIMIT); + __poll_portal_fast(p, QMAN_POLL_LIMIT, true); clear = QM_DQAVAIL_MASK | QM_PIRQ_DQRI; } /* Handling of anything else that's interrupt-driven */ @@ -1602,7 +1602,7 @@ static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq) * user callbacks to call into any QMan API. */ static inline unsigned int __poll_portal_fast(struct qman_portal *p, - unsigned int poll_limit) + unsigned int poll_limit, bool sched_napi) { const struct qm_dqrr_entry *dq; struct qman_fq *fq; @@ -1636,7 +1636,7 @@ static inline unsigned int __poll_portal_fast(struct qman_portal *p, * and we don't want multiple if()s in the critical * path (SDQCR). */ - res = fq->cb.dqrr(p, fq, dq); + res = fq->cb.dqrr(p, fq, dq, sched_napi); if (res == qman_cb_dqrr_stop) break; /* Check for VDQCR completion */ @@ -1646,7 +1646,7 @@ static inline unsigned int __poll_portal_fast(struct qman_portal *p, /* SDQCR: context_b points to the FQ */ fq = tag_to_fq(be32_to_cpu(dq->context_b)); /* Now let the callback do its stuff */ - res = fq->cb.dqrr(p, fq, dq); + res = fq->cb.dqrr(p, fq, dq, sched_napi); /* * The callback can request that we exit without * consuming this entry nor advancing; @@ -1753,7 +1753,7 @@ EXPORT_SYMBOL(qman_start_using_portal); int qman_p_poll_dqrr(struct qman_portal *p, unsigned int limit) { - return __poll_portal_fast(p, limit); + return __poll_portal_fast(p, limit, false); } EXPORT_SYMBOL(qman_p_poll_dqrr); diff --git a/drivers/soc/fsl/qbman/qman_test_api.c b/drivers/soc/fsl/qbman/qman_test_api.c index 7066b2f1467c..28fbddc3c204 100644 --- a/drivers/soc/fsl/qbman/qman_test_api.c +++ b/drivers/soc/fsl/qbman/qman_test_api.c @@ -45,7 +45,8 @@ static enum qman_cb_dqrr_result cb_dqrr(struct qman_portal *, struct qman_fq *, - const struct qm_dqrr_entry *); + const struct qm_dqrr_entry *, + bool sched_napi); static void cb_ern(struct qman_portal *, struct qman_fq *, const union qm_mr_entry *); static void cb_fqs(struct qman_portal *, struct qman_fq *, @@ -208,7 +209,8 @@ failed: static enum qman_cb_dqrr_result cb_dqrr(struct qman_portal *p, struct qman_fq *fq, - const struct qm_dqrr_entry *dq) + const struct qm_dqrr_entry *dq, + bool sched_napi) { if (WARN_ON(fd_neq(&fd_dq, &dq->fd))) { pr_err("BADNESS: dequeued frame doesn't match;\n"); diff --git a/drivers/soc/fsl/qbman/qman_test_stash.c b/drivers/soc/fsl/qbman/qman_test_stash.c index e87b65403b67..b7e8e5ec884c 100644 --- a/drivers/soc/fsl/qbman/qman_test_stash.c +++ b/drivers/soc/fsl/qbman/qman_test_stash.c @@ -275,7 +275,8 @@ static inline int process_frame_data(struct hp_handler *handler, static enum qman_cb_dqrr_result normal_dqrr(struct qman_portal *portal, struct qman_fq *fq, - const struct qm_dqrr_entry *dqrr) + const struct qm_dqrr_entry *dqrr, + bool sched_napi) { struct hp_handler *handler = (struct hp_handler *)fq; @@ -293,7 +294,8 @@ skip: static enum qman_cb_dqrr_result special_dqrr(struct qman_portal *portal, struct qman_fq *fq, - const struct qm_dqrr_entry *dqrr) + const struct qm_dqrr_entry *dqrr, + bool sched_napi) { struct hp_handler *handler = (struct hp_handler *)fq; diff --git a/include/soc/fsl/qman.h b/include/soc/fsl/qman.h index 9f484113cfda..59eeba31c192 100644 --- a/include/soc/fsl/qman.h +++ b/include/soc/fsl/qman.h @@ -689,7 +689,8 @@ enum qman_cb_dqrr_result { }; typedef enum qman_cb_dqrr_result (*qman_cb_dqrr)(struct qman_portal *qm, struct qman_fq *fq, - const struct qm_dqrr_entry *dqrr); + const struct qm_dqrr_entry *dqrr, + bool sched_napi); /* * This callback type is used when handling ERNs, FQRNs and FQRLs via MR. They -- cgit v1.2.3 From de3f5d9e271f7d19c9bdb3ba443fa6a435075f1a Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Mon, 2 Nov 2020 00:22:57 +0100 Subject: crypto: caam: Replace in_irq() usage. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver uses in_irq() + in_serving_softirq() magic to decide if NAPI scheduling is required or packet processing. The usage of in_*() in drivers is phased out and Linus clearly requested that code which changes behaviour depending on context should either be separated or the context be conveyed in an argument passed by the caller, which usually knows the context. Use the `sched_napi' argument passed by the callback. It is set true if called from the interrupt handler and NAPI should be scheduled. Signed-off-by: Sebastian Andrzej Siewior Cc: Aymen Sghaier Cc: Herbert Xu Cc: Li Yang Reviewed-by: Horia Geantă Signed-off-by: Jakub Kicinski Reviewed-by: Madalin Bucur Tested-by: Camelia Groza --- drivers/crypto/caam/qi.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index 57f6ab6bfb56..8163f5df8ebf 100644 --- a/drivers/crypto/caam/qi.c +++ b/drivers/crypto/caam/qi.c @@ -545,14 +545,10 @@ static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested) } } -static int caam_qi_napi_schedule(struct qman_portal *p, struct caam_napi *np) +static int caam_qi_napi_schedule(struct qman_portal *p, struct caam_napi *np, + bool sched_napi) { - /* - * In case of threaded ISR, for RT kernels in_irq() does not return - * appropriate value, so use in_serving_softirq to distinguish between - * softirq and irq contexts. - */ - if (unlikely(in_irq() || !in_serving_softirq())) { + if (sched_napi) { /* Disable QMan IRQ source and invoke NAPI */ qman_p_irqsource_remove(p, QM_PIRQ_DQRI); np->p = p; @@ -574,7 +570,7 @@ static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p, struct caam_drv_private *priv = dev_get_drvdata(qidev); u32 status; - if (caam_qi_napi_schedule(p, caam_napi)) + if (caam_qi_napi_schedule(p, caam_napi, sched_napi)) return qman_cb_dqrr_stop; fd = &dqrr->fd; -- cgit v1.2.3 From 77fdd94f091f88b045e4f45ea94cac463cbd3221 Mon Sep 17 00:00:00 2001 From: Tom Rix Date: Mon, 19 Oct 2020 12:36:53 -0700 Subject: crypto: atmel-sha - remove unneeded break A break is not needed if it is preceded by a return Signed-off-by: Tom Rix Reviewed-by: Alexandre Belloni Signed-off-by: Herbert Xu --- drivers/crypto/atmel-sha.c | 1 - 1 file changed, 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index 75ccf41a7cb9..0eb6f54e3b66 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -459,7 +459,6 @@ static int atmel_sha_init(struct ahash_request *req) break; default: return -EINVAL; - break; } ctx->bufcnt = 0; -- cgit v1.2.3 From 2d653936eb2cf613526290caa512b46e4c0d71ae Mon Sep 17 00:00:00 2001 From: Iuliana Prodan Date: Mon, 26 Oct 2020 21:06:26 +0200 Subject: crypto: caam - enable crypto-engine retry mechanism MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the new crypto_engine_alloc_init_and_set() function to initialize crypto-engine and enable retry mechanism. Set the maximum size for crypto-engine software queue based on Job Ring size (JOBR_DEPTH) and a threshold (reserved for the non-crypto-API requests that are not passed through crypto-engine). The callback for do_batch_requests is NULL, since CAAM doesn't support linked requests. Signed-off-by: Iuliana Prodan Reviewed-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/intern.h | 8 ++++++++ drivers/crypto/caam/jr.c | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index 9112279a4de0..7d45b21bd55a 100644 --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -16,6 +16,14 @@ /* Currently comes from Kconfig param as a ^2 (driver-required) */ #define JOBR_DEPTH (1 << CONFIG_CRYPTO_DEV_FSL_CAAM_RINGSIZE) +/* + * Maximum size for crypto-engine software queue based on Job Ring + * size (JOBR_DEPTH) and a THRESHOLD (reserved for the non-crypto-API + * requests that are not passed through crypto-engine) + */ +#define THRESHOLD 15 +#define CRYPTO_ENGINE_MAX_QLEN (JOBR_DEPTH - THRESHOLD) + /* Kconfig params for interrupt coalescing if selected (else zero) */ #ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_INTC #define JOBR_INTC JRCFG_ICEN diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index 6f669966ba2c..7f2b1101f567 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -550,7 +550,9 @@ static int caam_jr_probe(struct platform_device *pdev) } /* Initialize crypto engine */ - jrpriv->engine = crypto_engine_alloc_init(jrdev, false); + jrpriv->engine = crypto_engine_alloc_init_and_set(jrdev, true, NULL, + false, + CRYPTO_ENGINE_MAX_QLEN); if (!jrpriv->engine) { dev_err(jrdev, "Could not init crypto-engine\n"); return -ENOMEM; -- cgit v1.2.3 From 76bb6a7b5b427fcbb1996f8d002888be81371320 Mon Sep 17 00:00:00 2001 From: Tom Rix Date: Tue, 27 Oct 2020 12:01:25 -0700 Subject: crypto: cavium/nitrox - remove unneeded semicolon A semicolon is not needed after a switch statement. Signed-off-by: Tom Rix Signed-off-by: Herbert Xu --- drivers/crypto/cavium/nitrox/nitrox_mbx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/cavium/nitrox/nitrox_mbx.c b/drivers/crypto/cavium/nitrox/nitrox_mbx.c index b51b0449b478..73993f9e2311 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_mbx.c +++ b/drivers/crypto/cavium/nitrox/nitrox_mbx.c @@ -112,7 +112,7 @@ static void pf2vf_resp_handler(struct work_struct *work) case MBX_MSG_TYPE_ACK: case MBX_MSG_TYPE_NACK: break; - }; + } kfree(pf2vf_resp); } -- cgit v1.2.3 From cc9a99fb89a91b841556ff5aa97f7a9d0e01cbcc Mon Sep 17 00:00:00 2001 From: Tom Rix Date: Tue, 27 Oct 2020 12:04:07 -0700 Subject: crypto: qat - remove unneeded semicolon A semicolon is not needed after a switch statement. Signed-off-by: Tom Rix Signed-off-by: Tom Rix Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_algs.c | 2 +- drivers/crypto/qat/qat_common/qat_asym_algs.c | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index a38afc61f6d2..0fab8bb8ca59 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -105,7 +105,7 @@ static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg) return ICP_QAT_HW_SHA512_STATE1_SZ; default: return -EFAULT; - }; + } return -EFAULT; } diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index f112078be868..2c863d25327a 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -201,7 +201,7 @@ static unsigned long qat_dh_fn_id(unsigned int len, bool g2) return g2 ? PKE_DH_G2_4096 : PKE_DH_4096; default: return 0; - }; + } } static int qat_dh_compute_value(struct kpp_request *req) @@ -572,7 +572,7 @@ static unsigned long qat_rsa_enc_fn_id(unsigned int len) return PKE_RSA_EP_4096; default: return 0; - }; + } } #define PKE_RSA_DP1_512 0x1c161b3c @@ -601,7 +601,7 @@ static unsigned long qat_rsa_dec_fn_id(unsigned int len) return PKE_RSA_DP1_4096; default: return 0; - }; + } } #define PKE_RSA_DP2_512 0x1c131b57 @@ -630,7 +630,7 @@ static unsigned long qat_rsa_dec_fn_id_crt(unsigned int len) return PKE_RSA_DP2_4096; default: return 0; - }; + } } static int qat_rsa_enc(struct akcipher_request *req) -- cgit v1.2.3 From 1b5644f29983b2847162de4d3b5bce23faa136ab Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 31 Oct 2020 17:07:01 +0800 Subject: crypto: hisilicon/qm - numbers are replaced by macros Some numbers are replaced by macros to avoid incomprehension. Signed-off-by: Weili Qian Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 050fe4e74523..1de3aac1dd56 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -473,7 +473,7 @@ static int qm_wait_mb_ready(struct hisi_qm *qm) return readl_relaxed_poll_timeout(qm->io_base + QM_MB_CMD_SEND_BASE, val, !((val >> QM_MB_BUSY_SHIFT) & - 0x1), 10, 1000); + 0x1), POLL_PERIOD, POLL_TIMEOUT); } /* 128 bit should be written to hardware at one time to trigger a mailbox */ @@ -583,7 +583,8 @@ static int qm_dev_mem_reset(struct hisi_qm *qm) writel(0x1, qm->io_base + QM_MEM_START_INIT); return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val, - val & BIT(0), 10, 1000); + val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT); } static u32 qm_get_irq_num_v1(struct hisi_qm *qm) @@ -804,7 +805,8 @@ static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, int ret; ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, - val & BIT(0), 10, 1000); + val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT); if (ret) return ret; @@ -818,7 +820,8 @@ static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE); return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, - val & BIT(0), 10, 1000); + val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT); } /* The config should be conducted after qm_dev_mem_reset() */ @@ -1785,10 +1788,11 @@ static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) INIT_QC_COMMON(cqc, qp->cqe_dma, pasid); if (ver == QM_HW_V1) { - cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0, 4)); + cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0, + QM_QC_CQE_SIZE)); cqc->w8 = cpu_to_le16(QM_Q_DEPTH - 1); } else { - cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(4)); + cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE)); cqc->w8 = 0; } cqc->dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT); @@ -2011,7 +2015,8 @@ static void hisi_qm_cache_wb(struct hisi_qm *qm) writel(0x1, qm->io_base + QM_CACHE_WB_START); if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE, - val, val & BIT(0), 10, 1000)) + val, val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT)) dev_err(&qm->pdev->dev, "QM writeback sqc cache fail!\n"); } -- cgit v1.2.3 From e926d753a6128035a72a99490daa56a4f9a49f83 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 31 Oct 2020 17:07:02 +0800 Subject: crypto: hisilicon/qm - modify the return type of function The returns of 'qm_get_hw_error_status' and 'qm_get_dev_err_status' are values from the hardware registers, which should not be defined as 'int', so update as 'u32'. Signed-off-by: Weili Qian Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 1de3aac1dd56..d5d06ae106f2 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -3291,7 +3291,7 @@ pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev, } EXPORT_SYMBOL_GPL(hisi_qm_dev_err_detected); -static int qm_get_hw_error_status(struct hisi_qm *qm) +static u32 qm_get_hw_error_status(struct hisi_qm *qm) { return readl(qm->io_base + QM_ABNORMAL_INT_STATUS); } @@ -3590,7 +3590,7 @@ restart_fail: return ret; } -static int qm_get_dev_err_status(struct hisi_qm *qm) +static u32 qm_get_dev_err_status(struct hisi_qm *qm) { return qm->err_ini->get_dev_hw_err_status(qm); } -- cgit v1.2.3 From a8ff38bd829420d4813c22d018f412d5f1bb65df Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 31 Oct 2020 17:07:03 +0800 Subject: crypto: hisilicon/qm - modify the return type of debugfs interface Since 'qm_create_debugfs_file' always returns 0, change it as 'void'. Signed-off-by: Weili Qian Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_main.c | 4 +--- drivers/crypto/hisilicon/qm.c | 19 ++++--------------- drivers/crypto/hisilicon/qm.h | 2 +- drivers/crypto/hisilicon/sec2/sec_main.c | 4 +--- drivers/crypto/hisilicon/zip/zip_main.c | 4 +--- 5 files changed, 8 insertions(+), 25 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index a33394d91bbf..e5c991913f09 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -705,9 +705,7 @@ static int hpre_debugfs_init(struct hisi_qm *qm) qm->debug.sqe_mask_offset = HPRE_SQE_MASK_OFFSET; qm->debug.sqe_mask_len = HPRE_SQE_MASK_LEN; - ret = hisi_qm_debug_init(qm); - if (ret) - goto failed_to_create; + hisi_qm_debug_init(qm); if (qm->pdev->device == HPRE_PCI_DEVICE_ID) { ret = hpre_ctrl_debug_init(qm); diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index d5d06ae106f2..627479fb48e6 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -1520,7 +1520,7 @@ static const struct file_operations qm_cmd_fops = { .write = qm_cmd_write, }; -static int qm_create_debugfs_file(struct hisi_qm *qm, enum qm_debug_file index) +static void qm_create_debugfs_file(struct hisi_qm *qm, enum qm_debug_file index) { struct dentry *qm_d = qm->debug.qm_d; struct debugfs_file *file = qm->debug.files + index; @@ -1531,8 +1531,6 @@ static int qm_create_debugfs_file(struct hisi_qm *qm, enum qm_debug_file index) file->index = index; mutex_init(&file->lock); file->debug = &qm->debug; - - return 0; } static void qm_hw_error_init_v1(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) @@ -2824,12 +2822,12 @@ DEFINE_DEBUGFS_ATTRIBUTE(qm_atomic64_ops, qm_debugfs_atomic64_get, * * Create qm related debugfs files. */ -int hisi_qm_debug_init(struct hisi_qm *qm) +void hisi_qm_debug_init(struct hisi_qm *qm) { struct qm_dfx *dfx = &qm->debug.dfx; struct dentry *qm_d; void *data; - int i, ret; + int i; qm_d = debugfs_create_dir("qm", qm->debug.debug_root); qm->debug.qm_d = qm_d; @@ -2837,10 +2835,7 @@ int hisi_qm_debug_init(struct hisi_qm *qm) /* only show this in PF */ if (qm->fun_type == QM_HW_PF) for (i = CURRENT_Q; i < DEBUG_FILE_NUM; i++) - if (qm_create_debugfs_file(qm, i)) { - ret = -ENOENT; - goto failed_to_create; - } + qm_create_debugfs_file(qm, i); debugfs_create_file("regs", 0444, qm->debug.qm_d, qm, &qm_regs_fops); @@ -2856,12 +2851,6 @@ int hisi_qm_debug_init(struct hisi_qm *qm) data, &qm_atomic64_ops); } - - return 0; - -failed_to_create: - debugfs_remove_recursive(qm_d); - return ret; } EXPORT_SYMBOL_GPL(hisi_qm_debug_init); diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h index 0420f4ce7197..8624d1288afe 100644 --- a/drivers/crypto/hisilicon/qm.h +++ b/drivers/crypto/hisilicon/qm.h @@ -350,7 +350,7 @@ void hisi_qm_release_qp(struct hisi_qp *qp); int hisi_qp_send(struct hisi_qp *qp, const void *msg); int hisi_qm_get_free_qp_num(struct hisi_qm *qm); int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number); -int hisi_qm_debug_init(struct hisi_qm *qm); +void hisi_qm_debug_init(struct hisi_qm *qm); enum qm_hw_ver hisi_qm_get_hw_version(struct pci_dev *pdev); void hisi_qm_debug_regs_clear(struct hisi_qm *qm); int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs); diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 2f52581b6d3c..b35c1c2271a3 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -652,9 +652,7 @@ static int sec_debugfs_init(struct hisi_qm *qm) sec_debugfs_root); qm->debug.sqe_mask_offset = SEC_SQE_MASK_OFFSET; qm->debug.sqe_mask_len = SEC_SQE_MASK_LEN; - ret = hisi_qm_debug_init(qm); - if (ret) - goto failed_to_create; + hisi_qm_debug_init(qm); ret = sec_debug_init(qm); if (ret) diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 4bd2c811abba..3d1524b63ea8 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -590,9 +590,7 @@ static int hisi_zip_debugfs_init(struct hisi_qm *qm) qm->debug.sqe_mask_offset = HZIP_SQE_MASK_OFFSET; qm->debug.sqe_mask_len = HZIP_SQE_MASK_LEN; qm->debug.debug_root = dev_d; - ret = hisi_qm_debug_init(qm); - if (ret) - goto failed_to_create; + hisi_qm_debug_init(qm); if (qm->fun_type == QM_HW_PF) { ret = hisi_zip_ctrl_debug_init(qm); -- cgit v1.2.3 From cd1aff982af43d47557a44d1758d9d5c06247d59 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 31 Oct 2020 17:07:04 +0800 Subject: crypto: hisilicon/qm - modify return type of 'qm_set_sqctype' Since 'qm_set_sqctype' always returns 0, change it as 'void'. Signed-off-by: Weili Qian Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 627479fb48e6..17f84dbc8bbe 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -2118,7 +2118,7 @@ static void hisi_qm_uacce_stop_queue(struct uacce_queue *q) hisi_qm_stop_qp(q->priv); } -static int qm_set_sqctype(struct uacce_queue *q, u16 type) +static void qm_set_sqctype(struct uacce_queue *q, u16 type) { struct hisi_qm *qm = q->uacce->priv; struct hisi_qp *qp = q->priv; @@ -2126,8 +2126,6 @@ static int qm_set_sqctype(struct uacce_queue *q, u16 type) down_write(&qm->qps_lock); qp->alg_type = type; up_write(&qm->qps_lock); - - return 0; } static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd, -- cgit v1.2.3 From 09493afbc62781bd9fba6224af89fd78fe33b8ba Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 31 Oct 2020 17:07:05 +0800 Subject: crypto: hisilicon/qm - replace 'sprintf' with 'scnprintf' Replace 'sprintf' with 'scnprintf' to avoid overrun. Signed-off-by: Weili Qian Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 17f84dbc8bbe..25c54146cca1 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -932,7 +932,8 @@ static ssize_t qm_debug_read(struct file *filp, char __user *buf, return -EINVAL; } mutex_unlock(&file->lock); - ret = sprintf(tbuf, "%u\n", val); + + ret = scnprintf(tbuf, QM_DBG_TMP_BUF_LEN, "%u\n", val); return simple_read_from_buffer(buf, count, pos, tbuf, ret); } -- cgit v1.2.3 From 3bf1ef9d513ea405672bbc245964acc1990f4e80 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 31 Oct 2020 17:07:06 +0800 Subject: crypto: hisilicon/qm - split 'qm_qp_ctx_cfg' into smaller pieces 'qm_qp_ctx_cfg' initializes configuration of SQ and CQ, split it into two pieces to improve code readability. Signed-off-by: Weili Qian Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 25c54146cca1..4c5cc60e9f7d 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -1735,19 +1735,15 @@ void hisi_qm_release_qp(struct hisi_qp *qp) } EXPORT_SYMBOL_GPL(hisi_qm_release_qp); -static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) +static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) { struct hisi_qm *qm = qp->qm; struct device *dev = &qm->pdev->dev; enum qm_hw_ver ver = qm->ver; struct qm_sqc *sqc; - struct qm_cqc *cqc; dma_addr_t sqc_dma; - dma_addr_t cqc_dma; int ret; - qm_init_qp_status(qp); - sqc = kzalloc(sizeof(struct qm_sqc), GFP_KERNEL); if (!sqc) return -ENOMEM; @@ -1772,12 +1768,23 @@ static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) ret = qm_mb(qm, QM_MB_CMD_SQC, sqc_dma, qp_id, 0); dma_unmap_single(dev, sqc_dma, sizeof(struct qm_sqc), DMA_TO_DEVICE); kfree(sqc); - if (ret) - return ret; + + return ret; +} + +static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) +{ + struct hisi_qm *qm = qp->qm; + struct device *dev = &qm->pdev->dev; + enum qm_hw_ver ver = qm->ver; + struct qm_cqc *cqc; + dma_addr_t cqc_dma; + int ret; cqc = kzalloc(sizeof(struct qm_cqc), GFP_KERNEL); if (!cqc) return -ENOMEM; + cqc_dma = dma_map_single(dev, cqc, sizeof(struct qm_cqc), DMA_TO_DEVICE); if (dma_mapping_error(dev, cqc_dma)) { @@ -1792,7 +1799,7 @@ static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) cqc->w8 = cpu_to_le16(QM_Q_DEPTH - 1); } else { cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE)); - cqc->w8 = 0; + cqc->w8 = 0; /* rand_qc */ } cqc->dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT); @@ -1803,6 +1810,19 @@ static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) return ret; } +static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) +{ + int ret; + + qm_init_qp_status(qp); + + ret = qm_sq_ctx_cfg(qp, qp_id, pasid); + if (ret) + return ret; + + return qm_cq_ctx_cfg(qp, qp_id, pasid); +} + static int qm_start_qp_nolock(struct hisi_qp *qp, unsigned long arg) { struct hisi_qm *qm = qp->qm; -- cgit v1.2.3 From 53737881da6e505a435f8a10d10a19dd0c888823 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 31 Oct 2020 17:07:07 +0800 Subject: crypto: hisilicon/qm - split 'qm_eq_ctx_cfg' into smaller pieces 'qm_eq_ctx_cfg' initializes configuration of EQ and AEQ, split it into two pieces to improve code readability. Signed-off-by: Weili Qian Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 44 +++++++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 12 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 4c5cc60e9f7d..6e8d20d73d54 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -1735,7 +1735,7 @@ void hisi_qm_release_qp(struct hisi_qp *qp) } EXPORT_SYMBOL_GPL(hisi_qm_release_qp); -static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) +static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) { struct hisi_qm *qm = qp->qm; struct device *dev = &qm->pdev->dev; @@ -1772,7 +1772,7 @@ static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) return ret; } -static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) +static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) { struct hisi_qm *qm = qp->qm; struct device *dev = &qm->pdev->dev; @@ -1784,7 +1784,6 @@ static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) cqc = kzalloc(sizeof(struct qm_cqc), GFP_KERNEL); if (!cqc) return -ENOMEM; - cqc_dma = dma_map_single(dev, cqc, sizeof(struct qm_cqc), DMA_TO_DEVICE); if (dma_mapping_error(dev, cqc_dma)) { @@ -1810,7 +1809,7 @@ static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) return ret; } -static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) +static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) { int ret; @@ -2550,14 +2549,10 @@ static int qm_eq_ctx_cfg(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; struct qm_eqc *eqc; - struct qm_aeqc *aeqc; dma_addr_t eqc_dma; - dma_addr_t aeqc_dma; int ret; - qm_init_eq_aeq_status(qm); - - eqc = kzalloc(sizeof(struct qm_eqc), GFP_KERNEL); + eqc = kzalloc(sizeof(struct qm_eqc), GFP_KERNEL); //todo if (!eqc) return -ENOMEM; eqc_dma = dma_map_single(dev, eqc, sizeof(struct qm_eqc), @@ -2572,11 +2567,20 @@ static int qm_eq_ctx_cfg(struct hisi_qm *qm) if (qm->ver == QM_HW_V1) eqc->dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE); eqc->dw6 = cpu_to_le32((QM_EQ_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); + ret = qm_mb(qm, QM_MB_CMD_EQC, eqc_dma, 0, 0); dma_unmap_single(dev, eqc_dma, sizeof(struct qm_eqc), DMA_TO_DEVICE); kfree(eqc); - if (ret) - return ret; + + return ret; +} + +static int qm_aeq_ctx_cfg(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + struct qm_aeqc *aeqc; + dma_addr_t aeqc_dma; + int ret; aeqc = kzalloc(sizeof(struct qm_aeqc), GFP_KERNEL); if (!aeqc) @@ -2599,6 +2603,22 @@ static int qm_eq_ctx_cfg(struct hisi_qm *qm) return ret; } +static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + int ret; + + qm_init_eq_aeq_status(qm); + + ret = qm_eq_ctx_cfg(qm); + if (ret) { + dev_err(dev, "Set eqc failed!\n"); + return ret; + } + + return qm_aeq_ctx_cfg(qm); +} + static int __hisi_qm_start(struct hisi_qm *qm) { int ret; @@ -2615,7 +2635,7 @@ static int __hisi_qm_start(struct hisi_qm *qm) return ret; } - ret = qm_eq_ctx_cfg(qm); + ret = qm_eq_aeq_ctx_cfg(qm); if (ret) return ret; -- cgit v1.2.3 From fefc046f2b0e0bdbd4d5944f6b489039678e001e Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 31 Oct 2020 17:07:08 +0800 Subject: crypto: hisilicon/qm - split 'hisi_qm_init' into smaller pieces 'hisi_qm_init' initializes configuration of QM. To improve code readability, split it into two pieces. Signed-off-by: Weili Qian Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 86 ++++++++++++++++++++++++++----------------- 1 file changed, 53 insertions(+), 33 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 6e8d20d73d54..f21ccae0e8ea 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -2442,6 +2442,16 @@ static void hisi_qm_pre_init(struct hisi_qm *qm) qm->is_frozen = false; } +static void hisi_qm_pci_uninit(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + + pci_free_irq_vectors(pdev); + iounmap(qm->io_base); + pci_release_mem_regions(pdev); + pci_disable_device(pdev); +} + /** * hisi_qm_uninit() - Uninitialize qm. * @qm: The qm needed uninit. @@ -2460,9 +2470,6 @@ void hisi_qm_uninit(struct hisi_qm *qm) return; } - uacce_remove(qm->uacce); - qm->uacce = NULL; - hisi_qp_memory_uninit(qm, qm->qp_num); idr_destroy(&qm->qp_idr); @@ -2474,10 +2481,9 @@ void hisi_qm_uninit(struct hisi_qm *qm) } qm_irq_unregister(qm); - pci_free_irq_vectors(pdev); - iounmap(qm->io_base); - pci_release_mem_regions(pdev); - pci_disable_device(pdev); + hisi_qm_pci_uninit(qm); + uacce_remove(qm->uacce); + qm->uacce = NULL; up_write(&qm->qps_lock); } @@ -4038,34 +4044,22 @@ void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list) } EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister); -/** - * hisi_qm_init() - Initialize configures about qm. - * @qm: The qm needing init. - * - * This function init qm, then we can call hisi_qm_start to put qm into work. - */ -int hisi_qm_init(struct hisi_qm *qm) +static int hisi_qm_pci_init(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; struct device *dev = &pdev->dev; unsigned int num_vec; int ret; - hisi_qm_pre_init(qm); - - ret = qm_alloc_uacce(qm); - if (ret < 0) - dev_warn(&pdev->dev, "fail to alloc uacce (%d)\n", ret); - ret = pci_enable_device_mem(pdev); if (ret < 0) { - dev_err(&pdev->dev, "Failed to enable device mem!\n"); - goto err_remove_uacce; + dev_err(dev, "Failed to enable device mem!\n"); + return ret; } ret = pci_request_mem_regions(pdev, qm->dev_name); if (ret < 0) { - dev_err(&pdev->dev, "Failed to request mem regions!\n"); + dev_err(dev, "Failed to request mem regions!\n"); goto err_disable_pcidev; } @@ -4093,9 +4087,42 @@ int hisi_qm_init(struct hisi_qm *qm) goto err_iounmap; } + return 0; + +err_iounmap: + iounmap(qm->io_base); +err_release_mem_regions: + pci_release_mem_regions(pdev); +err_disable_pcidev: + pci_disable_device(pdev); + return ret; +} + +/** + * hisi_qm_init() - Initialize configures about qm. + * @qm: The qm needing init. + * + * This function init qm, then we can call hisi_qm_start to put qm into work. + */ +int hisi_qm_init(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct device *dev = &pdev->dev; + int ret; + + hisi_qm_pre_init(qm); + + ret = qm_alloc_uacce(qm); + if (ret < 0) + dev_warn(dev, "fail to alloc uacce (%d)\n", ret); + + ret = hisi_qm_pci_init(qm); + if (ret) + goto err_remove_uacce; + ret = qm_irq_register(qm); if (ret) - goto err_free_irq_vectors; + goto err_pci_uninit; if (qm->fun_type == QM_HW_VF && qm->ver != QM_HW_V1) { /* v2 starts to support get vft by mailbox */ @@ -4118,14 +4145,8 @@ int hisi_qm_init(struct hisi_qm *qm) err_irq_unregister: qm_irq_unregister(qm); -err_free_irq_vectors: - pci_free_irq_vectors(pdev); -err_iounmap: - iounmap(qm->io_base); -err_release_mem_regions: - pci_release_mem_regions(pdev); -err_disable_pcidev: - pci_disable_device(pdev); +err_pci_uninit: + hisi_qm_pci_uninit(qm); err_remove_uacce: uacce_remove(qm->uacce); qm->uacce = NULL; @@ -4133,7 +4154,6 @@ err_remove_uacce: } EXPORT_SYMBOL_GPL(hisi_qm_init); - MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Zhou Wang "); MODULE_DESCRIPTION("HiSilicon Accelerator queue manager driver"); -- cgit v1.2.3 From ab95bd2aa904e4f53b7358efeea1d57693fb7889 Mon Sep 17 00:00:00 2001 From: Horia Geantă Date: Sun, 1 Nov 2020 22:05:53 +0200 Subject: crypto: caam - fix printing on xts fallback allocation error path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the time xts fallback tfm allocation fails the device struct hasn't been enabled yet in the caam xts tfm's private context. Fix this by using the device struct from xts algorithm's private context or, when not available, by replacing dev_err with pr_err. Fixes: 9d9b14dbe077 ("crypto: caam/jr - add fallback for XTS with more than 8B IV") Fixes: 83e8aa912138 ("crypto: caam/qi - add fallback for XTS with more than 8B IV") Fixes: 36e2d7cfdcf1 ("crypto: caam/qi2 - add fallback for XTS with more than 8B IV") Signed-off-by: Horia Geantă Reviewed-by: Iuliana Prodan Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 4 ++-- drivers/crypto/caam/caamalg_qi.c | 4 ++-- drivers/crypto/caam/caamalg_qi2.c | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index cf5bd7666dfc..8697ae53b063 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -3404,8 +3404,8 @@ static int caam_cra_init(struct crypto_skcipher *tfm) fallback = crypto_alloc_skcipher(tfm_name, 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(fallback)) { - dev_err(ctx->jrdev, "Failed to allocate %s fallback: %ld\n", - tfm_name, PTR_ERR(fallback)); + pr_err("Failed to allocate %s fallback: %ld\n", + tfm_name, PTR_ERR(fallback)); return PTR_ERR(fallback); } diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index 66f60d78bdc8..a24ae966df4a 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -2502,8 +2502,8 @@ static int caam_cra_init(struct crypto_skcipher *tfm) fallback = crypto_alloc_skcipher(tfm_name, 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(fallback)) { - dev_err(ctx->jrdev, "Failed to allocate %s fallback: %ld\n", - tfm_name, PTR_ERR(fallback)); + pr_err("Failed to allocate %s fallback: %ld\n", + tfm_name, PTR_ERR(fallback)); return PTR_ERR(fallback); } diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index 98c1ff1744bb..a780e627838a 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -1611,7 +1611,8 @@ static int caam_cra_init_skcipher(struct crypto_skcipher *tfm) fallback = crypto_alloc_skcipher(tfm_name, 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(fallback)) { - dev_err(ctx->dev, "Failed to allocate %s fallback: %ld\n", + dev_err(caam_alg->caam.dev, + "Failed to allocate %s fallback: %ld\n", tfm_name, PTR_ERR(fallback)); return PTR_ERR(fallback); } -- cgit v1.2.3 From ec3c5b32fccc537e7184dcf39387b2e57d12f517 Mon Sep 17 00:00:00 2001 From: Adam Guerin Date: Mon, 2 Nov 2020 17:04:54 +0000 Subject: crypto: qat - remove cast for mailbox CSR Remove cast for mailbox CSR in adf_admin.c as it is not needed. Suggested-by: Andy Shevchenko Signed-off-by: Adam Guerin Reviewed-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_admin.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c index 7c2ca54229aa..43680e178242 100644 --- a/drivers/crypto/qat/qat_common/adf_admin.c +++ b/drivers/crypto/qat/qat_common/adf_admin.c @@ -254,7 +254,7 @@ int adf_init_admin_comms(struct adf_accel_dev *accel_dev) hw_data->get_admin_info(&admin_csrs_info); mailbox_offset = admin_csrs_info.mailbox_offset; - mailbox = (void __iomem *)((uintptr_t)csr + mailbox_offset); + mailbox = csr + mailbox_offset; adminmsg_u = admin_csrs_info.admin_msg_ur; adminmsg_l = admin_csrs_info.admin_msg_lr; -- cgit v1.2.3 From 802c6c3ef3bc3eeca95d3ad55642fb7a973d185a Mon Sep 17 00:00:00 2001 From: Adam Guerin Date: Tue, 3 Nov 2020 17:29:36 +0000 Subject: crypto: qat - replace pci with PCI in comments Change all lower case pci in comments to be upper case PCI. Suggested-by: Andy Shevchenko Signed-off-by: Adam Guerin Reviewed-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_dev_mgr.c | 4 ++-- drivers/crypto/qat/qat_common/adf_sriov.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/qat/qat_common/adf_dev_mgr.c index 92ec035576df..29dc2e3d38ff 100644 --- a/drivers/crypto/qat/qat_common/adf_dev_mgr.c +++ b/drivers/crypto/qat/qat_common/adf_dev_mgr.c @@ -285,9 +285,9 @@ struct adf_accel_dev *adf_devmgr_get_first(void) /** * adf_devmgr_pci_to_accel_dev() - Get accel_dev associated with the pci_dev. - * @pci_dev: Pointer to pci device. + * @pci_dev: Pointer to PCI device. * - * Function returns acceleration device associated with the given pci device. + * Function returns acceleration device associated with the given PCI device. * To be used by QAT device specific drivers. * * Return: pointer to accel_dev or NULL if not found. diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index d887640355d4..8c822c2861c2 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -122,13 +122,13 @@ EXPORT_SYMBOL_GPL(adf_disable_sriov); /** * adf_sriov_configure() - Enable SRIOV for the device - * @pdev: Pointer to pci device. + * @pdev: Pointer to PCI device. * @numvfs: Number of virtual functions (VFs) to enable. * * Note that the @numvfs parameter is ignored and all VFs supported by the * device are enabled due to the design of the hardware. * - * Function enables SRIOV for the pci device. + * Function enables SRIOV for the PCI device. * * Return: number of VFs enabled on success, error code otherwise. */ -- cgit v1.2.3 From cd078cb6a0f2e360b073b34f78a53b9410fa3e7a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 6 Nov 2020 17:53:52 +1100 Subject: crypto: cavium/nitrox - Fix sparse warnings This patch fixes all the sparse warnings in cavium/nitrox: - Fix endianness warnings by adding the correct markers to unions. - Add missing header inclusions for prototypes. - Move nitrox_sriov_configure prototype into the isr header file. Signed-off-by: Herbert Xu --- drivers/crypto/cavium/nitrox/nitrox_aead.c | 10 +++++----- drivers/crypto/cavium/nitrox/nitrox_debugfs.c | 1 + drivers/crypto/cavium/nitrox/nitrox_hal.c | 1 + drivers/crypto/cavium/nitrox/nitrox_isr.c | 1 + drivers/crypto/cavium/nitrox/nitrox_isr.h | 9 +++++++++ drivers/crypto/cavium/nitrox/nitrox_main.c | 13 ------------- drivers/crypto/cavium/nitrox/nitrox_mbx.c | 1 + drivers/crypto/cavium/nitrox/nitrox_req.h | 4 ++++ drivers/crypto/cavium/nitrox/nitrox_reqmgr.c | 7 ++++--- 9 files changed, 26 insertions(+), 21 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/cavium/nitrox/nitrox_aead.c b/drivers/crypto/cavium/nitrox/nitrox_aead.c index 1be2571363fe..e5d8607ecb1d 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_aead.c +++ b/drivers/crypto/cavium/nitrox/nitrox_aead.c @@ -45,9 +45,9 @@ static int nitrox_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, /* fill crypto context */ fctx = nctx->u.fctx; - flags.f = be64_to_cpu(fctx->flags.f); + flags.fu = be64_to_cpu(fctx->flags.f); flags.w0.aes_keylen = aes_keylen; - fctx->flags.f = cpu_to_be64(flags.f); + fctx->flags.f = cpu_to_be64(flags.fu); /* copy enc key to context */ memset(&fctx->crypto, 0, sizeof(fctx->crypto)); @@ -63,9 +63,9 @@ static int nitrox_aead_setauthsize(struct crypto_aead *aead, struct flexi_crypto_context *fctx = nctx->u.fctx; union fc_ctx_flags flags; - flags.f = be64_to_cpu(fctx->flags.f); + flags.fu = be64_to_cpu(fctx->flags.f); flags.w0.mac_len = authsize; - fctx->flags.f = cpu_to_be64(flags.f); + fctx->flags.f = cpu_to_be64(flags.fu); aead->authsize = authsize; @@ -319,7 +319,7 @@ static int nitrox_gcm_common_init(struct crypto_aead *aead) flags->w0.iv_source = IV_FROM_DPTR; /* ask microcode to calculate ipad/opad */ flags->w0.auth_input_type = 1; - flags->f = be64_to_cpu(flags->f); + flags->f = cpu_to_be64(flags->fu); return 0; } diff --git a/drivers/crypto/cavium/nitrox/nitrox_debugfs.c b/drivers/crypto/cavium/nitrox/nitrox_debugfs.c index 16f7d0bd1303..741572a01995 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_debugfs.c +++ b/drivers/crypto/cavium/nitrox/nitrox_debugfs.c @@ -3,6 +3,7 @@ #include #include "nitrox_csr.h" +#include "nitrox_debugfs.h" #include "nitrox_dev.h" static int firmware_show(struct seq_file *s, void *v) diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.c b/drivers/crypto/cavium/nitrox/nitrox_hal.c index 34a2f4f30a7e..13b137410b75 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_hal.c +++ b/drivers/crypto/cavium/nitrox/nitrox_hal.c @@ -3,6 +3,7 @@ #include "nitrox_dev.h" #include "nitrox_csr.h" +#include "nitrox_hal.h" #define PLL_REF_CLK 50 #define MAX_CSR_RETRIES 10 diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.c b/drivers/crypto/cavium/nitrox/nitrox_isr.c index 3dec570a190a..99b053094f5a 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_isr.c +++ b/drivers/crypto/cavium/nitrox/nitrox_isr.c @@ -7,6 +7,7 @@ #include "nitrox_csr.h" #include "nitrox_common.h" #include "nitrox_hal.h" +#include "nitrox_isr.h" #include "nitrox_mbx.h" /** diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.h b/drivers/crypto/cavium/nitrox/nitrox_isr.h index 1062c9336c1f..2bb123cd2f1c 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_isr.h +++ b/drivers/crypto/cavium/nitrox/nitrox_isr.h @@ -9,4 +9,13 @@ void nitrox_unregister_interrupts(struct nitrox_device *ndev); int nitrox_sriov_register_interupts(struct nitrox_device *ndev); void nitrox_sriov_unregister_interrupts(struct nitrox_device *ndev); +#ifdef CONFIG_PCI_IOV +int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs); +#else +static inline int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs) +{ + return 0; +} +#endif + #endif /* __NITROX_ISR_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c index 9d14be97e381..facc8e6bc580 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_main.c +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -49,15 +49,6 @@ static unsigned int qlen = DEFAULT_CMD_QLEN; module_param(qlen, uint, 0644); MODULE_PARM_DESC(qlen, "Command queue length - default 2048"); -#ifdef CONFIG_PCI_IOV -int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs); -#else -int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs) -{ - return 0; -} -#endif - /** * struct ucode - Firmware Header * @id: microcode ID @@ -555,10 +546,8 @@ static void nitrox_remove(struct pci_dev *pdev) nitrox_remove_from_devlist(ndev); -#ifdef CONFIG_PCI_IOV /* disable SR-IOV */ nitrox_sriov_configure(pdev, 0); -#endif nitrox_crypto_unregister(); nitrox_debugfs_exit(ndev); nitrox_pf_sw_cleanup(ndev); @@ -584,9 +573,7 @@ static struct pci_driver nitrox_driver = { .probe = nitrox_probe, .remove = nitrox_remove, .shutdown = nitrox_shutdown, -#ifdef CONFIG_PCI_IOV .sriov_configure = nitrox_sriov_configure, -#endif }; module_pci_driver(nitrox_driver); diff --git a/drivers/crypto/cavium/nitrox/nitrox_mbx.c b/drivers/crypto/cavium/nitrox/nitrox_mbx.c index 73993f9e2311..c1af9d4fca6e 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_mbx.c +++ b/drivers/crypto/cavium/nitrox/nitrox_mbx.c @@ -4,6 +4,7 @@ #include "nitrox_csr.h" #include "nitrox_hal.h" #include "nitrox_dev.h" +#include "nitrox_mbx.h" #define RING_TO_VFNO(_x, _y) ((_x) / (_y)) diff --git a/drivers/crypto/cavium/nitrox/nitrox_req.h b/drivers/crypto/cavium/nitrox/nitrox_req.h index 12282c1b14f5..ed174883c8e3 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_req.h +++ b/drivers/crypto/cavium/nitrox/nitrox_req.h @@ -149,6 +149,7 @@ struct auth_keys { union fc_ctx_flags { __be64 f; + u64 fu; struct { #if defined(__BIG_ENDIAN_BITFIELD) u64 cipher_type : 4; @@ -280,6 +281,7 @@ struct nitrox_rfc4106_rctx { * - packet payload bytes */ union pkt_instr_hdr { + __be64 bev; u64 value; struct { #if defined(__BIG_ENDIAN_BITFIELD) @@ -324,6 +326,7 @@ union pkt_instr_hdr { * @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases. */ union pkt_hdr { + __be64 bev[2]; u64 value[2]; struct { #if defined(__BIG_ENDIAN_BITFIELD) @@ -370,6 +373,7 @@ union pkt_hdr { * sglist components at [RPTR] on the remote host. */ union slc_store_info { + __be64 bev[2]; u64 value[2]; struct { #if defined(__BIG_ENDIAN_BITFIELD) diff --git a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c index 5826c2c98a50..53ef06792133 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c +++ b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c @@ -3,6 +3,7 @@ #include #include +#include "nitrox_common.h" #include "nitrox_dev.h" #include "nitrox_req.h" #include "nitrox_csr.h" @@ -448,7 +449,7 @@ int nitrox_process_se_request(struct nitrox_device *ndev, sr->instr.ih.s.ssz = sr->out.sgmap_cnt; sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr); sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes; - sr->instr.ih.value = cpu_to_be64(sr->instr.ih.value); + sr->instr.ih.bev = cpu_to_be64(sr->instr.ih.value); /* word 2 */ sr->instr.irh.value[0] = 0; @@ -460,7 +461,7 @@ int nitrox_process_se_request(struct nitrox_device *ndev, sr->instr.irh.s.ctxc = req->ctrl.s.ctxc; sr->instr.irh.s.arg = req->ctrl.s.arg; sr->instr.irh.s.opcode = req->opcode; - sr->instr.irh.value[0] = cpu_to_be64(sr->instr.irh.value[0]); + sr->instr.irh.bev[0] = cpu_to_be64(sr->instr.irh.value[0]); /* word 3 */ sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle); @@ -468,7 +469,7 @@ int nitrox_process_se_request(struct nitrox_device *ndev, /* word 4 */ sr->instr.slc.value[0] = 0; sr->instr.slc.s.ssz = sr->out.sgmap_cnt; - sr->instr.slc.value[0] = cpu_to_be64(sr->instr.slc.value[0]); + sr->instr.slc.bev[0] = cpu_to_be64(sr->instr.slc.value[0]); /* word 5 */ sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma); -- cgit v1.2.3 From f21406b0511d53f9370d9de8713dea7c09f9f248 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Fri, 6 Nov 2020 19:27:39 +0800 Subject: crypto: qat - support for mof format in fw loader Implement infrastructure for the Multiple Object File (MOF) format in the firmware loader. This will allow to load a specific firmware image contained inside an MOF file. This patch is based on earlier work done by Pingchao Yang. Signed-off-by: Giovanni Cabiddu Reviewed-by: Jack Xu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_engine.c | 2 +- drivers/crypto/qat/qat_common/adf_common_drv.h | 2 +- .../qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/icp_qat_uclo.h | 69 ++++++ drivers/crypto/qat/qat_common/qat_uclo.c | 262 ++++++++++++++++++++- 5 files changed, 331 insertions(+), 5 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_accel_engine.c b/drivers/crypto/qat/qat_common/adf_accel_engine.c index c8ad85b882be..1da4176356ab 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_engine.c +++ b/drivers/crypto/qat/qat_common/adf_accel_engine.c @@ -38,7 +38,7 @@ int adf_ae_fw_load(struct adf_accel_dev *accel_dev) dev_err(&GET_DEV(accel_dev), "Failed to load MMP\n"); goto out_err; } - if (qat_uclo_map_obj(loader_data->fw_loader, uof_addr, uof_size)) { + if (qat_uclo_map_obj(loader_data->fw_loader, uof_addr, uof_size, NULL)) { dev_err(&GET_DEV(accel_dev), "Failed to map FW\n"); goto out_err; } diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index f22342f612c1..8e6e346fd841 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -184,7 +184,7 @@ void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle); int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, void *addr_ptr, int mem_size); int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, - void *addr_ptr, int mem_size); + void *addr_ptr, u32 mem_size, char *obj_name); #if defined(CONFIG_PCI_IOV) int adf_sriov_configure(struct pci_dev *pdev, int numvfs); void adf_disable_sriov(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 3e8e291cd122..7d44786a223a 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -27,6 +27,7 @@ struct icp_qat_fw_loader_handle { struct pci_dev *pci_dev; void *obj_handle; void *sobj_handle; + void *mobj_handle; bool fw_auth; void __iomem *hal_sram_addr_v; void __iomem *hal_cap_g_ctl_csr_addr_v; diff --git a/drivers/crypto/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/qat/qat_common/icp_qat_uclo.h index 8fe1ec344fa2..101de1430896 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_uclo.h +++ b/drivers/crypto/qat/qat_common/icp_qat_uclo.h @@ -31,6 +31,15 @@ #define ICP_QAT_SUOF_FID 0x53554f46 #define ICP_QAT_SUOF_MAJVER 0x0 #define ICP_QAT_SUOF_MINVER 0x1 +#define ICP_QAT_SUOF_OBJ_NAME_LEN 128 +#define ICP_QAT_MOF_OBJ_ID_LEN 8 +#define ICP_QAT_MOF_OBJ_CHUNKID_LEN 8 +#define ICP_QAT_MOF_FID 0x00666f6d +#define ICP_QAT_MOF_MAJVER 0x0 +#define ICP_QAT_MOF_MINVER 0x1 +#define ICP_QAT_MOF_SYM_OBJS "SYM_OBJS" +#define ICP_QAT_SUOF_OBJS "SUF_OBJS" +#define ICP_QAT_SUOF_IMAG "SUF_IMAG" #define ICP_QAT_SIMG_AE_INIT_SEQ_LEN (50 * sizeof(unsigned long long)) #define ICP_QAT_SIMG_AE_INSTS_LEN (0x4000 * sizeof(unsigned long long)) #define ICP_QAT_CSS_FWSK_MODULUS_LEN 256 @@ -481,4 +490,64 @@ struct icp_qat_suof_objhdr { unsigned int img_length; unsigned int reserved; }; + +struct icp_qat_mof_file_hdr { + unsigned int file_id; + unsigned int checksum; + char min_ver; + char maj_ver; + unsigned short reserved; + unsigned short max_chunks; + unsigned short num_chunks; +}; + +struct icp_qat_mof_chunkhdr { + char chunk_id[ICP_QAT_MOF_OBJ_ID_LEN]; + u64 offset; + u64 size; +}; + +struct icp_qat_mof_str_table { + unsigned int tab_len; + unsigned int strings; +}; + +struct icp_qat_mof_obj_hdr { + unsigned short max_chunks; + unsigned short num_chunks; + unsigned int reserved; +}; + +struct icp_qat_mof_obj_chunkhdr { + char chunk_id[ICP_QAT_MOF_OBJ_CHUNKID_LEN]; + u64 offset; + u64 size; + unsigned int name; + unsigned int reserved; +}; + +struct icp_qat_mof_objhdr { + char *obj_name; + char *obj_buf; + unsigned int obj_size; +}; + +struct icp_qat_mof_table { + unsigned int num_objs; + struct icp_qat_mof_objhdr *obj_hdr; +}; + +struct icp_qat_mof_handle { + unsigned int file_id; + unsigned int checksum; + char min_ver; + char maj_ver; + char *mof_buf; + u32 mof_size; + char *sym_str; + unsigned int sym_size; + char *uobjs_hdr; + char *sobjs_hdr; + struct icp_qat_mof_table obj_table; +}; #endif diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 5d1f28cd6680..b475f6bfb90b 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1437,18 +1437,272 @@ out_objbuf_err: return -ENOMEM; } +static int qat_uclo_map_mof_file_hdr(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_mof_file_hdr *mof_ptr, + u32 mof_size) +{ + struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; + unsigned int min_ver_offset; + unsigned int checksum; + + mobj_handle->file_id = ICP_QAT_MOF_FID; + mobj_handle->mof_buf = (char *)mof_ptr; + mobj_handle->mof_size = mof_size; + + min_ver_offset = mof_size - offsetof(struct icp_qat_mof_file_hdr, + min_ver); + checksum = qat_uclo_calc_str_checksum(&mof_ptr->min_ver, + min_ver_offset); + if (checksum != mof_ptr->checksum) { + pr_err("QAT: incorrect MOF checksum\n"); + return -EINVAL; + } + + mobj_handle->checksum = mof_ptr->checksum; + mobj_handle->min_ver = mof_ptr->min_ver; + mobj_handle->maj_ver = mof_ptr->maj_ver; + return 0; +} + +static void qat_uclo_del_mof(struct icp_qat_fw_loader_handle *handle) +{ + struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; + + kfree(mobj_handle->obj_table.obj_hdr); + mobj_handle->obj_table.obj_hdr = NULL; + kfree(handle->mobj_handle); + handle->mobj_handle = NULL; +} + +static int qat_uclo_seek_obj_inside_mof(struct icp_qat_mof_handle *mobj_handle, + char *obj_name, char **obj_ptr, + unsigned int *obj_size) +{ + struct icp_qat_mof_objhdr *obj_hdr = mobj_handle->obj_table.obj_hdr; + unsigned int i; + + for (i = 0; i < mobj_handle->obj_table.num_objs; i++) { + if (!strncmp(obj_hdr[i].obj_name, obj_name, + ICP_QAT_SUOF_OBJ_NAME_LEN)) { + *obj_ptr = obj_hdr[i].obj_buf; + *obj_size = obj_hdr[i].obj_size; + return 0; + } + } + + pr_err("QAT: object %s is not found inside MOF\n", obj_name); + return -EINVAL; +} + +static int qat_uclo_map_obj_from_mof(struct icp_qat_mof_handle *mobj_handle, + struct icp_qat_mof_objhdr *mobj_hdr, + struct icp_qat_mof_obj_chunkhdr *obj_chunkhdr) +{ + u8 *obj; + + if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_UOF_IMAG, + ICP_QAT_MOF_OBJ_CHUNKID_LEN)) { + obj = mobj_handle->uobjs_hdr + obj_chunkhdr->offset; + } else if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_SUOF_IMAG, + ICP_QAT_MOF_OBJ_CHUNKID_LEN)) { + obj = mobj_handle->sobjs_hdr + obj_chunkhdr->offset; + } else { + pr_err("QAT: unsupported chunk id\n"); + return -EINVAL; + } + mobj_hdr->obj_buf = obj; + mobj_hdr->obj_size = (unsigned int)obj_chunkhdr->size; + mobj_hdr->obj_name = obj_chunkhdr->name + mobj_handle->sym_str; + return 0; +} + +static int qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle) +{ + struct icp_qat_mof_obj_chunkhdr *uobj_chunkhdr; + struct icp_qat_mof_obj_chunkhdr *sobj_chunkhdr; + struct icp_qat_mof_obj_hdr *uobj_hdr; + struct icp_qat_mof_obj_hdr *sobj_hdr; + struct icp_qat_mof_objhdr *mobj_hdr; + unsigned int uobj_chunk_num = 0; + unsigned int sobj_chunk_num = 0; + unsigned int *valid_chunk; + int ret, i; + + uobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->uobjs_hdr; + sobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->sobjs_hdr; + if (uobj_hdr) + uobj_chunk_num = uobj_hdr->num_chunks; + if (sobj_hdr) + sobj_chunk_num = sobj_hdr->num_chunks; + + mobj_hdr = kzalloc((uobj_chunk_num + sobj_chunk_num) * + sizeof(*mobj_hdr), GFP_KERNEL); + if (!mobj_hdr) + return -ENOMEM; + + mobj_handle->obj_table.obj_hdr = mobj_hdr; + valid_chunk = &mobj_handle->obj_table.num_objs; + uobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *) + ((uintptr_t)uobj_hdr + sizeof(*uobj_hdr)); + sobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *) + ((uintptr_t)sobj_hdr + sizeof(*sobj_hdr)); + + /* map uof objects */ + for (i = 0; i < uobj_chunk_num; i++) { + ret = qat_uclo_map_obj_from_mof(mobj_handle, + &mobj_hdr[*valid_chunk], + &uobj_chunkhdr[i]); + if (ret) + return ret; + (*valid_chunk)++; + } + + /* map suof objects */ + for (i = 0; i < sobj_chunk_num; i++) { + ret = qat_uclo_map_obj_from_mof(mobj_handle, + &mobj_hdr[*valid_chunk], + &sobj_chunkhdr[i]); + if (ret) + return ret; + (*valid_chunk)++; + } + + if ((uobj_chunk_num + sobj_chunk_num) != *valid_chunk) { + pr_err("QAT: inconsistent UOF/SUOF chunk amount\n"); + return -EINVAL; + } + return 0; +} + +static void qat_uclo_map_mof_symobjs(struct icp_qat_mof_handle *mobj_handle, + struct icp_qat_mof_chunkhdr *mof_chunkhdr) +{ + char **sym_str = (char **)&mobj_handle->sym_str; + unsigned int *sym_size = &mobj_handle->sym_size; + struct icp_qat_mof_str_table *str_table_obj; + + *sym_size = *(unsigned int *)(uintptr_t) + (mof_chunkhdr->offset + mobj_handle->mof_buf); + *sym_str = (char *)(uintptr_t) + (mobj_handle->mof_buf + mof_chunkhdr->offset + + sizeof(str_table_obj->tab_len)); +} + +static void qat_uclo_map_mof_chunk(struct icp_qat_mof_handle *mobj_handle, + struct icp_qat_mof_chunkhdr *mof_chunkhdr) +{ + char *chunk_id = mof_chunkhdr->chunk_id; + + if (!strncmp(chunk_id, ICP_QAT_MOF_SYM_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) + qat_uclo_map_mof_symobjs(mobj_handle, mof_chunkhdr); + else if (!strncmp(chunk_id, ICP_QAT_UOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) + mobj_handle->uobjs_hdr = mobj_handle->mof_buf + + mof_chunkhdr->offset; + else if (!strncmp(chunk_id, ICP_QAT_SUOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) + mobj_handle->sobjs_hdr = mobj_handle->mof_buf + + mof_chunkhdr->offset; +} + +static int qat_uclo_check_mof_format(struct icp_qat_mof_file_hdr *mof_hdr) +{ + int maj = mof_hdr->maj_ver & 0xff; + int min = mof_hdr->min_ver & 0xff; + + if (mof_hdr->file_id != ICP_QAT_MOF_FID) { + pr_err("QAT: invalid header 0x%x\n", mof_hdr->file_id); + return -EINVAL; + } + + if (mof_hdr->num_chunks <= 0x1) { + pr_err("QAT: MOF chunk amount is incorrect\n"); + return -EINVAL; + } + if (maj != ICP_QAT_MOF_MAJVER || min != ICP_QAT_MOF_MINVER) { + pr_err("QAT: bad MOF version, major 0x%x, minor 0x%x\n", + maj, min); + return -EINVAL; + } + return 0; +} + +static int qat_uclo_map_mof_obj(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_mof_file_hdr *mof_ptr, + u32 mof_size, char *obj_name, char **obj_ptr, + unsigned int *obj_size) +{ + struct icp_qat_mof_chunkhdr *mof_chunkhdr; + unsigned int file_id = mof_ptr->file_id; + struct icp_qat_mof_handle *mobj_handle; + unsigned short chunks_num; + unsigned int i; + int ret; + + if (file_id == ICP_QAT_UOF_FID || file_id == ICP_QAT_SUOF_FID) { + if (obj_ptr) + *obj_ptr = (char *)mof_ptr; + if (obj_size) + *obj_size = mof_size; + return 0; + } + if (qat_uclo_check_mof_format(mof_ptr)) + return -EINVAL; + + mobj_handle = kzalloc(sizeof(*mobj_handle), GFP_KERNEL); + if (!mobj_handle) + return -ENOMEM; + + handle->mobj_handle = mobj_handle; + ret = qat_uclo_map_mof_file_hdr(handle, mof_ptr, mof_size); + if (ret) + return ret; + + mof_chunkhdr = (void *)mof_ptr + sizeof(*mof_ptr); + chunks_num = mof_ptr->num_chunks; + + /* Parse MOF file chunks */ + for (i = 0; i < chunks_num; i++) + qat_uclo_map_mof_chunk(mobj_handle, &mof_chunkhdr[i]); + + /* All sym_objs uobjs and sobjs should be available */ + if (!mobj_handle->sym_str || + (!mobj_handle->uobjs_hdr && !mobj_handle->sobjs_hdr)) + return -EINVAL; + + ret = qat_uclo_map_objs_from_mof(mobj_handle); + if (ret) + return ret; + + /* Seek specified uof object in MOF */ + return qat_uclo_seek_obj_inside_mof(mobj_handle, obj_name, + obj_ptr, obj_size); +} + int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, - void *addr_ptr, int mem_size) + void *addr_ptr, u32 mem_size, char *obj_name) { + char *obj_addr; + u32 obj_size; + int ret; + BUILD_BUG_ON(ICP_QAT_UCLO_MAX_AE >= (sizeof(handle->hal_handle->ae_mask) * 8)); if (!handle || !addr_ptr || mem_size < 24) return -EINVAL; + if (obj_name) { + ret = qat_uclo_map_mof_obj(handle, addr_ptr, mem_size, obj_name, + &obj_addr, &obj_size); + if (ret) + return ret; + } else { + obj_addr = addr_ptr; + obj_size = mem_size; + } + return (handle->fw_auth) ? - qat_uclo_map_suof_obj(handle, addr_ptr, mem_size) : - qat_uclo_map_uof_obj(handle, addr_ptr, mem_size); + qat_uclo_map_suof_obj(handle, obj_addr, obj_size) : + qat_uclo_map_uof_obj(handle, obj_addr, obj_size); } void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle) @@ -1456,6 +1710,8 @@ void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle) struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned int a; + if (handle->mobj_handle) + qat_uclo_del_mof(handle); if (handle->sobj_handle) qat_uclo_del_suof(handle); if (!obj_handle) -- cgit v1.2.3 From 3b5c130fb2e4c045369791c33c83b59f6e84f7d6 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:40 +0800 Subject: crypto: qat - fix status check in qat_hal_put_rel_rd_xfer() The return value of qat_hal_rd_ae_csr() is always a CSR value and never a status and should not be stored in the status variable of qat_hal_put_rel_rd_xfer(). This removes the assignment as qat_hal_rd_ae_csr() is not expected to fail. A more comprehensive handling of the theoretical corner case which could result in a fail will be submitted in a separate patch. Fixes: 8c9478a400b7 ("crypto: qat - reduce stack size with KASAN") Signed-off-by: Jack Xu Reviewed-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_hal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index bc07199459e7..5da8475ed876 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -1149,7 +1149,7 @@ static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle, unsigned short mask; unsigned short dr_offset = 0x10; - status = ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); if (CE_INUSE_CONTEXTS & ctx_enables) { if (ctx & 0x1) { pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx); -- cgit v1.2.3 From 49a6cccec4f89777a0ffd2a6d4c3b1af33636b3a Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:41 +0800 Subject: crypto: qat - fix CSR access Do not mask the AE number with the AE mask when accessing the AE local CSRs. Bit 12 of the local CSR address is the start of AE number so just take out the AE mask here. Signed-off-by: Jack Xu Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_hal.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index c0e9fc0c93dd..b48b313623fe 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -94,15 +94,13 @@ enum fcu_sts { #define SET_GLB_CSR(handle, csr, val) SET_CAP_CSR(handle, csr + GLOBAL_CSR, val) #define GET_GLB_CSR(handle, csr) GET_CAP_CSR(handle, GLOBAL_CSR + csr) #define AE_CSR(handle, ae) \ - ((char __iomem *)handle->hal_cap_ae_local_csr_addr_v + \ - ((ae & handle->hal_handle->ae_mask) << 12)) + ((char __iomem *)handle->hal_cap_ae_local_csr_addr_v + (ae << 12)) #define AE_CSR_ADDR(handle, ae, csr) (AE_CSR(handle, ae) + (0x3ff & csr)) #define SET_AE_CSR(handle, ae, csr, val) \ ADF_CSR_WR(AE_CSR_ADDR(handle, ae, csr), 0, val) #define GET_AE_CSR(handle, ae, csr) ADF_CSR_RD(AE_CSR_ADDR(handle, ae, csr), 0) #define AE_XFER(handle, ae) \ - ((char __iomem *)handle->hal_cap_ae_xfer_csr_addr_v + \ - ((ae & handle->hal_handle->ae_mask) << 12)) + ((char __iomem *)handle->hal_cap_ae_xfer_csr_addr_v + (ae << 12)) #define AE_XFER_ADDR(handle, ae, reg) (AE_XFER(handle, ae) + \ ((reg & 0xff) << 2)) #define SET_AE_XFER(handle, ae, reg, val) \ -- cgit v1.2.3 From ed8fc0c1b9c72f016130244b6a46483576fe8e5a Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:42 +0800 Subject: crypto: qat - fix error message Change message in error path of qat_uclo_check_image_compat() to report an incompatible firmware image that contains a neighbor register table. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_uclo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index b475f6bfb90b..063af33c6ca6 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -536,7 +536,7 @@ qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj, (encap_uof_obj->beg_uof + code_page->neigh_reg_tab_offset); if (neigh_reg_tab->entry_num) { - pr_err("QAT: UOF can't contain shared control store feature\n"); + pr_err("QAT: UOF can't contain neighbor register table\n"); return -EINVAL; } if (image->numpages > 1) { -- cgit v1.2.3 From 8ec0bee5d43e8582e6320dd9907d01c58844d3d3 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:43 +0800 Subject: crypto: qat - remove unnecessary parenthesis Remove unnecessary parenthesis across the firmware loader. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_hal.c | 6 +++--- drivers/crypto/qat/qat_common/qat_uclo.c | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 5da8475ed876..bbfb2b1b6fee 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -158,7 +158,7 @@ int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle, { unsigned int csr, new_csr; - if ((mode != 4) && (mode != 8)) { + if (mode != 4 && mode != 8) { pr_err("QAT: bad ctx mode=%d\n", mode); return -EINVAL; } @@ -430,7 +430,7 @@ static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0); csr_val = ADF_CSR_RD(csr_addr, 0); } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--); - if ((times < 0)) { + if (times < 0) { pr_err("QAT: Fail to init eSram!\n"); return -EFAULT; } @@ -1128,7 +1128,7 @@ int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle, plm_init = plm_init->next; } /* exec micro codes */ - if (micro_inst_arry && (micro_inst_num > 0)) { + if (micro_inst_arry && micro_inst_num > 0) { micro_inst_arry[micro_inst_num++] = 0x0E000010000ull; stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec, micro_inst_arry, diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 063af33c6ca6..4b2079353aa3 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -74,7 +74,7 @@ static int qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data) static char *qat_uclo_get_string(struct icp_qat_uof_strtable *str_table, unsigned int str_offset) { - if ((!str_table->table_len) || (str_offset > str_table->table_len)) + if (!str_table->table_len || str_offset > str_table->table_len) return NULL; return (char *)(((uintptr_t)(str_table->strings)) + str_offset); } @@ -736,8 +736,8 @@ static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle) return -EINVAL; } maj_ver = obj_handle->prod_rev & 0xff; - if ((obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver) || - (obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver)) { + if (obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver || + obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver) { pr_err("QAT: UOF majVer 0x%x out of range\n", maj_ver); return -EINVAL; } @@ -1064,8 +1064,8 @@ static int qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle, return -EINVAL; } maj_ver = prod_rev & 0xff; - if ((maj_ver > img_ae_mode->devmax_ver) || - (maj_ver < img_ae_mode->devmin_ver)) { + if (maj_ver > img_ae_mode->devmax_ver || + maj_ver < img_ae_mode->devmin_ver) { pr_err("QAT: incompatible device majver 0x%x\n", maj_ver); return -EINVAL; } @@ -1108,7 +1108,7 @@ static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, unsigned int i = 0; struct icp_qat_suof_img_hdr img_header; - if (!suof_ptr || (suof_size == 0)) { + if (!suof_ptr || suof_size == 0) { pr_err("QAT: input parameter SUOF pointer/size is NULL\n"); return -EINVAL; } @@ -1199,7 +1199,7 @@ static void qat_uclo_simg_free(struct icp_qat_fw_loader_handle *handle, { dma_free_coherent(&handle->pci_dev->dev, (size_t)(dram_desc->dram_size), - (dram_desc->dram_base_addr_v), + dram_desc->dram_base_addr_v, dram_desc->dram_bus_addr); memset(dram_desc, 0, sizeof(*dram_desc)); } @@ -1851,7 +1851,7 @@ static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle) if (qat_uclo_map_auth_fw(handle, (char *)simg_hdr[i].simg_buf, (unsigned int) - (simg_hdr[i].simg_len), + simg_hdr[i].simg_len, &desc)) goto wr_err; if (qat_uclo_auth_fw(handle, desc)) -- cgit v1.2.3 From 54fa5d4bf71246906efc3cd85ebddbc961af3498 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:44 +0800 Subject: crypto: qat - introduce additional parenthesis Introduce additional parenthesis to resolve a warninga reported by checkpatch. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_hal.h | 14 +++++++------- drivers/crypto/qat/qat_common/qat_hal.c | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index b48b313623fe..5640bb278bb1 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -88,23 +88,23 @@ enum fcu_sts { #define FW_AUTH_MAX_RETRY 300 #define SET_CAP_CSR(handle, csr, val) \ - ADF_CSR_WR(handle->hal_cap_g_ctl_csr_addr_v, csr, val) + ADF_CSR_WR((handle)->hal_cap_g_ctl_csr_addr_v, csr, val) #define GET_CAP_CSR(handle, csr) \ - ADF_CSR_RD(handle->hal_cap_g_ctl_csr_addr_v, csr) + ADF_CSR_RD((handle)->hal_cap_g_ctl_csr_addr_v, csr) #define SET_GLB_CSR(handle, csr, val) SET_CAP_CSR(handle, csr + GLOBAL_CSR, val) #define GET_GLB_CSR(handle, csr) GET_CAP_CSR(handle, GLOBAL_CSR + csr) #define AE_CSR(handle, ae) \ - ((char __iomem *)handle->hal_cap_ae_local_csr_addr_v + (ae << 12)) -#define AE_CSR_ADDR(handle, ae, csr) (AE_CSR(handle, ae) + (0x3ff & csr)) + ((char __iomem *)(handle)->hal_cap_ae_local_csr_addr_v + ((ae) << 12)) +#define AE_CSR_ADDR(handle, ae, csr) (AE_CSR(handle, ae) + (0x3ff & (csr))) #define SET_AE_CSR(handle, ae, csr, val) \ ADF_CSR_WR(AE_CSR_ADDR(handle, ae, csr), 0, val) #define GET_AE_CSR(handle, ae, csr) ADF_CSR_RD(AE_CSR_ADDR(handle, ae, csr), 0) #define AE_XFER(handle, ae) \ - ((char __iomem *)handle->hal_cap_ae_xfer_csr_addr_v + (ae << 12)) + ((char __iomem *)(handle)->hal_cap_ae_xfer_csr_addr_v + ((ae) << 12)) #define AE_XFER_ADDR(handle, ae, reg) (AE_XFER(handle, ae) + \ - ((reg & 0xff) << 2)) + (((reg) & 0xff) << 2)) #define SET_AE_XFER(handle, ae, reg, val) \ ADF_CSR_WR(AE_XFER_ADDR(handle, ae, reg), 0, val) #define SRAM_WRITE(handle, addr, val) \ - ADF_CSR_WR(handle->hal_sram_addr_v, addr, val) + ADF_CSR_WR((handle)->hal_sram_addr_v, addr, val) #endif diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index bbfb2b1b6fee..c628ea30e3c2 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -33,7 +33,7 @@ ((((const_val) << 12) & 0x0FF00000ull) | \ (((const_val) << 0) & 0x000000FFull)))) -#define AE(handle, ae) handle->hal_handle->aes[ae] +#define AE(handle, ae) ((handle)->hal_handle->aes[ae]) static const u64 inst_4b[] = { 0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull, @@ -150,8 +150,8 @@ static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle, return 0; } -#define CLR_BIT(wrd, bit) (wrd & ~(1 << bit)) -#define SET_BIT(wrd, bit) (wrd | 1 << bit) +#define CLR_BIT(wrd, bit) ((wrd) & ~(1 << (bit))) +#define SET_BIT(wrd, bit) ((wrd) | 1 << (bit)) int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned char mode) -- cgit v1.2.3 From ee103cf1f5af03ae300dcfdbe0f51ae12ce35dec Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:45 +0800 Subject: crypto: qat - rename qat_uclo_del_uof_obj() Rename the function qat_uclo_del_uof_obj() in qat_uclo_del_obj() since it frees the memory allocated for all firmware objects. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_engine.c | 2 +- drivers/crypto/qat/qat_common/adf_common_drv.h | 2 +- drivers/crypto/qat/qat_common/qat_uclo.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_accel_engine.c b/drivers/crypto/qat/qat_common/adf_accel_engine.c index 1da4176356ab..2c4a8c7c736e 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_engine.c +++ b/drivers/crypto/qat/qat_common/adf_accel_engine.c @@ -61,7 +61,7 @@ void adf_ae_fw_release(struct adf_accel_dev *accel_dev) if (!hw_device->fw_name) return; - qat_uclo_del_uof_obj(loader_data->fw_loader); + qat_uclo_del_obj(loader_data->fw_loader); qat_hal_deinit(loader_data->fw_loader); release_firmware(loader_data->uof_fw); release_firmware(loader_data->mmp_fw); diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 8e6e346fd841..22ac0517d15d 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -180,7 +180,7 @@ int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, int qat_hal_wr_lm(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned short lm_addr, unsigned int value); int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle); -void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle); +void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle); int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, void *addr_ptr, int mem_size); int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 4b2079353aa3..dc2f2dcf21b8 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1705,7 +1705,7 @@ int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, qat_uclo_map_uof_obj(handle, obj_addr, obj_size); } -void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle) +void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned int a; -- cgit v1.2.3 From 72b67d9507b954b5b5f4cecb74bcefff61a28d8c Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:46 +0800 Subject: crypto: qat - add support for relative FW ucode loading Improve the way micro instructions (FW code) are uploaded to Accelerator Engines (AEs). If code starts at PC zero (absolute addressing), read uwords with no relative address. Otherwise, use relative addressing to the page region. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_uclo.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index dc2f2dcf21b8..1c03205c7166 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1735,21 +1735,22 @@ static void qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle, u64 *uword, unsigned int addr_p, unsigned int raddr, u64 fill) { + unsigned int i, addr; u64 uwrd = 0; - unsigned int i; if (!encap_page) { *uword = fill; return; } + addr = (encap_page->page_region) ? raddr : addr_p; for (i = 0; i < encap_page->uwblock_num; i++) { - if (raddr >= encap_page->uwblock[i].start_addr && - raddr <= encap_page->uwblock[i].start_addr + + if (addr >= encap_page->uwblock[i].start_addr && + addr <= encap_page->uwblock[i].start_addr + encap_page->uwblock[i].words_num - 1) { - raddr -= encap_page->uwblock[i].start_addr; - raddr *= obj_handle->uword_in_bytes; + addr -= encap_page->uwblock[i].start_addr; + addr *= obj_handle->uword_in_bytes; memcpy(&uwrd, (void *)(((uintptr_t) - encap_page->uwblock[i].micro_words) + raddr), + encap_page->uwblock[i].micro_words) + addr), obj_handle->uword_in_bytes); uwrd = uwrd & 0xbffffffffffull; } -- cgit v1.2.3 From fe278bf35c59bbc9bc695dac58bfa7120ba125ba Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:47 +0800 Subject: crypto: qat - change type for ctx_mask Change type for ctx_mask from unsigned char to unsigned long to avoid type casting. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_common_drv.h | 8 ++++---- drivers/crypto/qat/qat_common/qat_hal.c | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 22ac0517d15d..8109e2ab4257 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -163,19 +163,19 @@ int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle, unsigned char ae, struct icp_qat_uof_batch_init *lm_init_header); int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, enum icp_qat_uof_regtype reg_type, unsigned short reg_num, unsigned int regdata); int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, enum icp_qat_uof_regtype reg_type, unsigned short reg_num, unsigned int regdata); int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, enum icp_qat_uof_regtype reg_type, unsigned short reg_num, unsigned int regdata); int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, unsigned short reg_num, unsigned int regdata); int qat_hal_wr_lm(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned short lm_addr, unsigned int value); diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index c628ea30e3c2..a9243758a959 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -1274,7 +1274,7 @@ static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle } int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, enum icp_qat_uof_regtype reg_type, unsigned short reg_num, unsigned int regdata) { @@ -1294,7 +1294,7 @@ int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, } else { reg = reg_num; type = reg_type; - if (!test_bit(ctx, (unsigned long *)&ctx_mask)) + if (!test_bit(ctx, &ctx_mask)) continue; } stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata); @@ -1308,7 +1308,7 @@ int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, } int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, enum icp_qat_uof_regtype reg_type, unsigned short reg_num, unsigned int regdata) { @@ -1328,7 +1328,7 @@ int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, } else { reg = reg_num; type = reg_type; - if (!test_bit(ctx, (unsigned long *)&ctx_mask)) + if (!test_bit(ctx, &ctx_mask)) continue; } stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg, @@ -1343,7 +1343,7 @@ int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, } int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, enum icp_qat_uof_regtype reg_type, unsigned short reg_num, unsigned int regdata) { @@ -1363,7 +1363,7 @@ int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, } else { reg = reg_num; type = reg_type; - if (!test_bit(ctx, (unsigned long *)&ctx_mask)) + if (!test_bit(ctx, &ctx_mask)) continue; } stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg, @@ -1378,7 +1378,7 @@ int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, } int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, unsigned short reg_num, unsigned int regdata) { int stat = 0; @@ -1388,7 +1388,7 @@ int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, return -EINVAL; for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { - if (!test_bit(ctx, (unsigned long *)&ctx_mask)) + if (!test_bit(ctx, &ctx_mask)) continue; stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata); if (stat) { -- cgit v1.2.3 From 82b3230658a90e60c36b426b4ca8d176c4d7ebc8 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:48 +0800 Subject: crypto: qat - change micro word data mask Change micro word data mask since the Acceleration Engine (AE) instruction codes have been changed in the new generation QAT devices. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_uclo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 1c03205c7166..6423b1ea7021 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1752,7 +1752,7 @@ static void qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle, memcpy(&uwrd, (void *)(((uintptr_t) encap_page->uwblock[i].micro_words) + addr), obj_handle->uword_in_bytes); - uwrd = uwrd & 0xbffffffffffull; + uwrd = uwrd & GENMASK_ULL(43, 0); } } *uword = uwrd; -- cgit v1.2.3 From 10fb050caef99d75895bf0978188090d3ed676c2 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:49 +0800 Subject: crypto: qat - refactor AE start Change the API and the behaviour of the qat_hal_start() function. With this change, the function starts under the hood all acceleration engines (AEs) and there is no longer need to call it for each engine. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_engine.c | 9 ++------- drivers/crypto/qat/qat_common/adf_common_drv.h | 3 +-- drivers/crypto/qat/qat_common/qat_hal.c | 24 +++++++++++++++--------- 3 files changed, 18 insertions(+), 18 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_accel_engine.c b/drivers/crypto/qat/qat_common/adf_accel_engine.c index 2c4a8c7c736e..08aaaf2b4659 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_engine.c +++ b/drivers/crypto/qat/qat_common/adf_accel_engine.c @@ -74,17 +74,12 @@ int adf_ae_start(struct adf_accel_dev *accel_dev) { struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; struct adf_hw_device_data *hw_data = accel_dev->hw_device; - u32 ae_ctr, ae, max_aes = GET_MAX_ACCELENGINES(accel_dev); + u32 ae_ctr; if (!hw_data->fw_name) return 0; - for (ae = 0, ae_ctr = 0; ae < max_aes; ae++) { - if (hw_data->ae_mask & (1 << ae)) { - qat_hal_start(loader_data->fw_loader, ae, 0xFF); - ae_ctr++; - } - } + ae_ctr = qat_hal_start(loader_data->fw_loader); dev_info(&GET_DEV(accel_dev), "qat_dev%d started %d acceleration engines\n", accel_dev->accel_id, ae_ctr); diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 8109e2ab4257..945608b71937 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -133,8 +133,7 @@ void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev); int qat_hal_init(struct adf_accel_dev *accel_dev); void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle); -void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae, - unsigned int ctx_mask); +int qat_hal_start(struct icp_qat_fw_loader_handle *handle); void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned int ctx_mask); void qat_hal_reset(struct icp_qat_fw_loader_handle *handle); diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index a9243758a959..f127233eec17 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -742,26 +742,32 @@ void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle) kfree(handle); } -void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae, - unsigned int ctx_mask) +int qat_hal_start(struct icp_qat_fw_loader_handle *handle) { + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned int fcu_sts; + unsigned char ae; + u32 ae_ctr = 0; int retry = 0; - unsigned int fcu_sts = 0; if (handle->fw_auth) { + ae_ctr = hweight32(ae_mask); SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_START); do { msleep(FW_AUTH_WAIT_PERIOD); fcu_sts = GET_CAP_CSR(handle, FCU_STATUS); if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1)) - return; + return ae_ctr; } while (retry++ < FW_AUTH_MAX_RETRY); - pr_err("QAT: start error (AE 0x%x FCU_STS = 0x%x)\n", ae, - fcu_sts); + pr_err("QAT: start error (FCU_STS = 0x%x)\n", fcu_sts); + return 0; } else { - qat_hal_put_wakeup_event(handle, ae, (~ctx_mask) & - ICP_QAT_UCLO_AE_ALL_CTX, 0x10000); - qat_hal_enable_ctx(handle, ae, ctx_mask); + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + qat_hal_put_wakeup_event(handle, ae, 0, 0x10000); + qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX); + ae_ctr++; + } + return ae_ctr; } } -- cgit v1.2.3 From 8f87b6271ec85c09af6406665324c74bb79e3216 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:50 +0800 Subject: crypto: qat - remove global CSRs helpers Include the offset of GLOBAL_CSR directly into the enum hal_global_csr and remove the macros SET_GLB_CSR/GET_GLB_CSR to simplify the global CSR access. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_hal.h | 9 +++------ drivers/crypto/qat/qat_common/qat_hal.c | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 16 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index 5640bb278bb1..c2166dacdf5b 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -5,9 +5,9 @@ #include "icp_qat_fw_loader_handle.h" enum hal_global_csr { - MISC_CONTROL = 0x04, - ICP_RESET = 0x0c, - ICP_GLOBAL_CLK_ENABLE = 0x50 + MISC_CONTROL = 0xA04, + ICP_RESET = 0xA0c, + ICP_GLOBAL_CLK_ENABLE = 0xA50 }; enum hal_ae_csr { @@ -78,7 +78,6 @@ enum fcu_sts { #define XCWE_VOLUNTARY (0x1) #define LCS_STATUS (0x1) #define MMC_SHARE_CS_BITPOS 2 -#define GLOBAL_CSR 0xA00 #define FCU_CTRL_AE_POS 0x8 #define FCU_AUTH_STS_MASK 0x7 #define FCU_STS_DONE_POS 0x9 @@ -91,8 +90,6 @@ enum fcu_sts { ADF_CSR_WR((handle)->hal_cap_g_ctl_csr_addr_v, csr, val) #define GET_CAP_CSR(handle, csr) \ ADF_CSR_RD((handle)->hal_cap_g_ctl_csr_addr_v, csr) -#define SET_GLB_CSR(handle, csr, val) SET_CAP_CSR(handle, csr + GLOBAL_CSR, val) -#define GET_GLB_CSR(handle, csr) GET_CAP_CSR(handle, GLOBAL_CSR + csr) #define AE_CSR(handle, ae) \ ((char __iomem *)(handle)->hal_cap_ae_local_csr_addr_v + ((ae) << 12)) #define AE_CSR_ADDR(handle, ae, csr) (AE_CSR(handle, ae) + (0x3ff & (csr))) diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index f127233eec17..15ebb57ea14a 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -273,10 +273,10 @@ void qat_hal_reset(struct icp_qat_fw_loader_handle *handle) { unsigned int ae_reset_csr; - ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET); + ae_reset_csr = GET_CAP_CSR(handle, ICP_RESET); ae_reset_csr |= handle->hal_handle->ae_mask << RST_CSR_AE_LSB; ae_reset_csr |= handle->hal_handle->slice_mask << RST_CSR_QAT_LSB; - SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr); + SET_CAP_CSR(handle, ICP_RESET, ae_reset_csr); } static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle, @@ -390,9 +390,9 @@ static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) unsigned char ae; /* stop the timestamp timers */ - misc_ctl = GET_GLB_CSR(handle, MISC_CONTROL); + misc_ctl = GET_CAP_CSR(handle, MISC_CONTROL); if (misc_ctl & MC_TIMESTAMP_ENABLE) - SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl & + SET_CAP_CSR(handle, MISC_CONTROL, misc_ctl & (~MC_TIMESTAMP_ENABLE)); for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { @@ -400,7 +400,7 @@ static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0); } /* start timestamp timers */ - SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE); + SET_CAP_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE); } #define ESRAM_AUTO_TINIT BIT(2) @@ -448,21 +448,21 @@ int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) unsigned int csr; /* write to the reset csr */ - ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET); + ae_reset_csr = GET_CAP_CSR(handle, ICP_RESET); ae_reset_csr &= ~(handle->hal_handle->ae_mask << RST_CSR_AE_LSB); ae_reset_csr &= ~(handle->hal_handle->slice_mask << RST_CSR_QAT_LSB); do { - SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr); + SET_CAP_CSR(handle, ICP_RESET, ae_reset_csr); if (!(times--)) goto out_err; - csr = GET_GLB_CSR(handle, ICP_RESET); + csr = GET_CAP_CSR(handle, ICP_RESET); } while ((handle->hal_handle->ae_mask | (handle->hal_handle->slice_mask << RST_CSR_QAT_LSB)) & csr); /* enable clock */ - clk_csr = GET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE); + clk_csr = GET_CAP_CSR(handle, ICP_GLOBAL_CLK_ENABLE); clk_csr |= handle->hal_handle->ae_mask << 0; clk_csr |= handle->hal_handle->slice_mask << 20; - SET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr); + SET_CAP_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr); if (qat_hal_check_ae_alive(handle)) goto out_err; -- cgit v1.2.3 From 97b9840195307a9af1c47bed69a63880a13bbb0d Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:51 +0800 Subject: crypto: qat - move defines to header files Move the definition of ICP_QAT_AE_OFFSET, ICP_QAT_CAP_OFFSET, LOCAL_TO_XFER_REG_OFFSET and ICP_QAT_EP_OFFSET from qat_hal.c to icp_qat_hal.h to avoid the definition of generation specific constants in qat_hal.c. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_hal.h | 4 ++++ drivers/crypto/qat/qat_common/qat_hal.c | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index c2166dacdf5b..eff9a3811435 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -85,6 +85,10 @@ enum fcu_sts { #define FCU_LOADED_AE_POS 0x16 #define FW_AUTH_WAIT_PERIOD 10 #define FW_AUTH_MAX_RETRY 300 +#define ICP_QAT_AE_OFFSET 0x20000 +#define ICP_QAT_CAP_OFFSET (ICP_QAT_AE_OFFSET + 0x10000) +#define LOCAL_TO_XFER_REG_OFFSET 0x800 +#define ICP_QAT_EP_OFFSET 0x3a000 #define SET_CAP_CSR(handle, csr, val) \ ADF_CSR_WR((handle)->hal_cap_g_ctl_csr_addr_v, csr, val) diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 15ebb57ea14a..5bf42f01a3de 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -646,10 +646,6 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) return 0; } -#define ICP_QAT_AE_OFFSET 0x20000 -#define ICP_QAT_CAP_OFFSET (ICP_QAT_AE_OFFSET + 0x10000) -#define LOCAL_TO_XFER_REG_OFFSET 0x800 -#define ICP_QAT_EP_OFFSET 0x3a000 int qat_hal_init(struct adf_accel_dev *accel_dev) { unsigned char ae; -- cgit v1.2.3 From ecb917ad0fc2bcc6c9f9e3db005c08f6e2d22f88 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:52 +0800 Subject: crypto: qat - refactor qat_uclo_set_ae_mode() Refactor qat_uclo_set_ae_mode() by moving the logic that sets the AE modes to a separate function, qat_hal_set_modes(). Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_uclo.c | 69 ++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 26 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 6423b1ea7021..095e1b469412 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -869,16 +869,52 @@ static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) return 0; } +static int qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uclo_objhandle *obj_handle, + unsigned char ae, + struct icp_qat_uof_image *uof_image) +{ + unsigned char mode; + int ret; + + mode = ICP_QAT_CTX_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_ctx_mode(handle, ae, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_ctx_mode error\n"); + return ret; + } + mode = ICP_QAT_NN_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_nn_mode(handle, ae, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); + return ret; + } + mode = ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n"); + return ret; + } + mode = ICP_QAT_LOC_MEM1_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n"); + return ret; + } + return 0; +} + static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle) { - unsigned char ae, nn_mode, s; struct icp_qat_uof_image *uof_image; struct icp_qat_uclo_aedata *ae_data; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned char ae, s; + int error; for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - if (!test_bit(ae, - (unsigned long *)&handle->hal_handle->ae_mask)) + if (!test_bit(ae, &ae_mask)) continue; ae_data = &obj_handle->ae_data[ae]; for (s = 0; s < min_t(unsigned int, ae_data->slice_num, @@ -886,29 +922,10 @@ static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle) if (!obj_handle->ae_data[ae].ae_slices[s].encap_image) continue; uof_image = ae_data->ae_slices[s].encap_image->img_ptr; - if (qat_hal_set_ae_ctx_mode(handle, ae, - (char)ICP_QAT_CTX_MODE - (uof_image->ae_mode))) { - pr_err("QAT: qat_hal_set_ae_ctx_mode error\n"); - return -EFAULT; - } - nn_mode = ICP_QAT_NN_MODE(uof_image->ae_mode); - if (qat_hal_set_ae_nn_mode(handle, ae, nn_mode)) { - pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); - return -EFAULT; - } - if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, - (char)ICP_QAT_LOC_MEM0_MODE - (uof_image->ae_mode))) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n"); - return -EFAULT; - } - if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, - (char)ICP_QAT_LOC_MEM1_MODE - (uof_image->ae_mode))) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n"); - return -EFAULT; - } + error = qat_hal_set_modes(handle, obj_handle, ae, + uof_image); + if (error) + return error; } } return 0; -- cgit v1.2.3 From 58c173b9cb84aa40111c971d957980e4418a1c5e Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:53 +0800 Subject: crypto: qat - refactor long expressions Replace long expressions with local variables in the functions qat_uclo_wr_uimage_page(), qat_uclo_init_globals() and qat_uclo_init_umem_seg() to improve readability. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_uclo.c | 41 ++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 18 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 095e1b469412..8d08dac94ea9 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -324,6 +324,7 @@ static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle, { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned int ae, ustore_size, uaddr, i; + struct icp_qat_uclo_aedata *aed; ustore_size = obj_handle->ustore_phy_size; if (qat_uclo_fetch_initmem_ae(handle, init_mem, ustore_size, &ae)) @@ -333,11 +334,10 @@ static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle, return -EINVAL; /* set the highest ustore address referenced */ uaddr = (init_mem->addr + init_mem->num_in_bytes) >> 0x2; - for (i = 0; i < obj_handle->ae_data[ae].slice_num; i++) { - if (obj_handle->ae_data[ae].ae_slices[i]. - encap_image->uwords_num < uaddr) - obj_handle->ae_data[ae].ae_slices[i]. - encap_image->uwords_num = uaddr; + aed = &obj_handle->ae_data[ae]; + for (i = 0; i < aed->slice_num; i++) { + if (aed->ae_slices[i].encap_image->uwords_num < uaddr) + aed->ae_slices[i].encap_image->uwords_num = uaddr; } return 0; } @@ -845,6 +845,7 @@ static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle, static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + struct icp_qat_uclo_aedata *aed; unsigned int s, ae; if (obj_handle->global_inited) @@ -855,13 +856,13 @@ static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) return -EINVAL; } } + for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - for (s = 0; s < obj_handle->ae_data[ae].slice_num; s++) { - if (!obj_handle->ae_data[ae].ae_slices[s].encap_image) + aed = &obj_handle->ae_data[ae]; + for (s = 0; s < aed->slice_num; s++) { + if (!aed->ae_slices[s].encap_image) continue; - if (qat_uclo_init_reg_sym(handle, ae, - obj_handle->ae_data[ae]. - ae_slices[s].encap_image)) + if (qat_uclo_init_reg_sym(handle, ae, aed->ae_slices[s].encap_image)) return -EINVAL; } } @@ -1820,6 +1821,8 @@ static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, struct icp_qat_uof_image *image) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_assigned = image->ae_assigned; + struct icp_qat_uclo_aedata *aed; unsigned int ctx_mask, s; struct icp_qat_uclo_page *page; unsigned char ae; @@ -1832,24 +1835,26 @@ static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, /* load the default page and set assigned CTX PC * to the entrypoint address */ for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - if (!test_bit(ae, (unsigned long *)&image->ae_assigned)) + if (!test_bit(ae, &ae_assigned)) continue; + + aed = &obj_handle->ae_data[ae]; /* find the slice to which this image is assigned */ - for (s = 0; s < obj_handle->ae_data[ae].slice_num; s++) { - if (image->ctx_assigned & obj_handle->ae_data[ae]. - ae_slices[s].ctx_mask_assigned) + for (s = 0; s < aed->slice_num; s++) { + if (image->ctx_assigned & + aed->ae_slices[s].ctx_mask_assigned) break; } - if (s >= obj_handle->ae_data[ae].slice_num) + if (s >= aed->slice_num) continue; - page = obj_handle->ae_data[ae].ae_slices[s].page; + page = aed->ae_slices[s].page; if (!page->encap_page->def_page) continue; qat_uclo_wr_uimage_raw_page(handle, page->encap_page, ae); - page = obj_handle->ae_data[ae].ae_slices[s].page; + page = aed->ae_slices[s].page; for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) - obj_handle->ae_data[ae].ae_slices[s].cur_page[ctx] = + aed->ae_slices[s].cur_page[ctx] = (ctx_mask & (1 << ctx)) ? page : NULL; qat_hal_set_live_ctx(handle, (unsigned char)ae, image->ctx_assigned); -- cgit v1.2.3 From fc5f3f86e5afb4008c6dea054fe4df302edd84df Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:54 +0800 Subject: crypto: qat - introduce chip info structure Introduce the chip info structure which contains device specific information. The initialization path has been split between common and hardware specific in order to facilitate the introduction of the next generation hardware. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../qat/qat_common/icp_qat_fw_loader_handle.h | 7 +- drivers/crypto/qat/qat_common/qat_hal.c | 110 +++++++++++++++------ drivers/crypto/qat/qat_common/qat_uclo.c | 6 +- 3 files changed, 88 insertions(+), 35 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 7d44786a223a..6b1ad629357b 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -22,13 +22,18 @@ struct icp_qat_fw_loader_hal_handle { unsigned int max_ustore; }; +struct icp_qat_fw_loader_chip_info { + bool sram_visible; + bool fw_auth; +}; + struct icp_qat_fw_loader_handle { struct icp_qat_fw_loader_hal_handle *hal_handle; + struct icp_qat_fw_loader_chip_info *chip_info; struct pci_dev *pci_dev; void *obj_handle; void *sobj_handle; void *mobj_handle; - bool fw_auth; void __iomem *hal_sram_addr_v; void __iomem *hal_cap_g_ctl_csr_addr_v; void __iomem *hal_cap_ae_xfer_csr_addr_v; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 5bf42f01a3de..2faf8638526c 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -646,23 +646,41 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) return 0; } -int qat_hal_init(struct adf_accel_dev *accel_dev) +static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, + struct adf_accel_dev *accel_dev) { - unsigned char ae; - unsigned int max_en_ae_id = 0; - struct icp_qat_fw_loader_handle *handle; struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct adf_bar *misc_bar = &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)]; - unsigned long ae_mask = hw_data->ae_mask; - unsigned int csr_val = 0; + unsigned int max_en_ae_id = 0; struct adf_bar *sram_bar; + unsigned int csr_val = 0; + unsigned long ae_mask; + unsigned char ae = 0; + int ret = 0; - handle = kzalloc(sizeof(*handle), GFP_KERNEL); - if (!handle) - return -ENOMEM; + handle->pci_dev = pci_info->pci_dev; + switch (handle->pci_dev->device) { + case PCI_DEVICE_ID_INTEL_QAT_C62X: + case PCI_DEVICE_ID_INTEL_QAT_C3XXX: + handle->chip_info->sram_visible = false; + handle->chip_info->fw_auth = true; + break; + case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: + handle->chip_info->sram_visible = true; + handle->chip_info->fw_auth = false; + break; + default: + ret = -EINVAL; + goto out_err; + } + if (handle->chip_info->sram_visible) { + sram_bar = + &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; + handle->hal_sram_addr_v = sram_bar->virt_addr; + } handle->hal_cap_g_ctl_csr_addr_v = (void __iomem *)((uintptr_t)misc_bar->virt_addr + ICP_QAT_CAP_OFFSET); @@ -676,22 +694,14 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + LOCAL_TO_XFER_REG_OFFSET); handle->pci_dev = pci_info->pci_dev; - if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_DH895XCC) { - sram_bar = - &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; - handle->hal_sram_addr_v = sram_bar->virt_addr; - } - handle->fw_auth = (handle->pci_dev->device == - PCI_DEVICE_ID_INTEL_QAT_DH895XCC) ? false : true; - handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL); - if (!handle->hal_handle) - goto out_hal_handle; handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid; handle->hal_handle->ae_mask = hw_data->ae_mask; handle->hal_handle->slice_mask = hw_data->accel_mask; /* create AE objects */ handle->hal_handle->upc_mask = 0x1ffff; handle->hal_handle->max_ustore = 0x4000; + + ae_mask = handle->hal_handle->ae_mask; for_each_set_bit(ae, &ae_mask, ICP_QAT_UCLO_MAX_AE) { handle->hal_handle->aes[ae].free_addr = 0; handle->hal_handle->aes[ae].free_size = @@ -703,37 +713,75 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) max_en_ae_id = ae; } handle->hal_handle->ae_max_num = max_en_ae_id + 1; + + /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */ + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE); + csr_val |= 0x1; + qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val); + } +out_err: + return ret; +} + +int qat_hal_init(struct adf_accel_dev *accel_dev) +{ + struct icp_qat_fw_loader_handle *handle; + int ret = 0; + + handle = kzalloc(sizeof(*handle), GFP_KERNEL); + if (!handle) + return -ENOMEM; + + handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL); + if (!handle->hal_handle) { + ret = -ENOMEM; + goto out_hal_handle; + } + + handle->chip_info = kzalloc(sizeof(*handle->chip_info), GFP_KERNEL); + if (!handle->chip_info) { + ret = -ENOMEM; + goto out_chip_info; + } + + ret = qat_hal_chip_init(handle, accel_dev); + if (ret) { + dev_err(&GET_DEV(accel_dev), "qat_hal_chip_init error\n"); + goto out_err; + } + /* take all AEs out of reset */ - if (qat_hal_clr_reset(handle)) { + ret = qat_hal_clr_reset(handle); + if (ret) { dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n"); goto out_err; } + qat_hal_clear_xfer(handle); - if (!handle->fw_auth) { - if (qat_hal_clear_gpr(handle)) + if (!handle->chip_info->fw_auth) { + ret = qat_hal_clear_gpr(handle); + if (ret) goto out_err; } - /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */ - for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { - csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE); - csr_val |= 0x1; - qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val); - } accel_dev->fw_loader->fw_loader = handle; return 0; out_err: + kfree(handle->chip_info); +out_chip_info: kfree(handle->hal_handle); out_hal_handle: kfree(handle); - return -EFAULT; + return ret; } void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle) { if (!handle) return; + kfree(handle->chip_info); kfree(handle->hal_handle); kfree(handle); } @@ -746,7 +794,7 @@ int qat_hal_start(struct icp_qat_fw_loader_handle *handle) u32 ae_ctr = 0; int retry = 0; - if (handle->fw_auth) { + if (handle->chip_info->fw_auth) { ae_ctr = hweight32(ae_mask); SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_START); do { @@ -770,7 +818,7 @@ int qat_hal_start(struct icp_qat_fw_loader_handle *handle) void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned int ctx_mask) { - if (!handle->fw_auth) + if (!handle->chip_info->fw_auth) qat_hal_disable_ctx(handle, ae, ctx_mask); } diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 8d08dac94ea9..1533981baf3a 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1405,7 +1405,7 @@ int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc = NULL; int status = 0; - if (handle->fw_auth) { + if (handle->chip_info->fw_auth) { if (!qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc)) status = qat_uclo_auth_fw(handle, desc); qat_uclo_ummap_auth_fw(handle, &desc); @@ -1718,7 +1718,7 @@ int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, obj_size = mem_size; } - return (handle->fw_auth) ? + return (handle->chip_info->fw_auth) ? qat_uclo_map_suof_obj(handle, obj_addr, obj_size) : qat_uclo_map_uof_obj(handle, obj_addr, obj_size); } @@ -1909,6 +1909,6 @@ static int qat_uclo_wr_uof_img(struct icp_qat_fw_loader_handle *handle) int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle) { - return (handle->fw_auth) ? qat_uclo_wr_suof_img(handle) : + return (handle->chip_info->fw_auth) ? qat_uclo_wr_suof_img(handle) : qat_uclo_wr_uof_img(handle); } -- cgit v1.2.3 From 49c1327328ad2245bf06bffc2591d3eb99b51c60 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:55 +0800 Subject: crypto: qat - replace check based on DID Modify condition in qat_uclo_wr_mimage() to use a capability of the device (sram_visible), rather than the device ID, so the check is not specific to devices of the same type. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_uclo.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 1533981baf3a..5774916497bd 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1410,9 +1410,11 @@ int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, status = qat_uclo_auth_fw(handle, desc); qat_uclo_ummap_auth_fw(handle, &desc); } else { - if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_C3XXX) { - pr_err("QAT: C3XXX doesn't support unsigned MMP\n"); - return -EINVAL; + if (!handle->chip_info->sram_visible) { + dev_dbg(&handle->pci_dev->dev, + "QAT MMP fw not loaded for device 0x%x", + handle->pci_dev->device); + return status; } qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, mem_size); } -- cgit v1.2.3 From d25cf2c7a057d6c01e44d4696fde1d05bfec11bb Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:56 +0800 Subject: crypto: qat - add next neighbor to chip_info Introduce the next neighbor (NN) capability in chip_info as NN registers are not supported in certain SKUs of QAT. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/qat_hal.c | 11 ++++++++++- drivers/crypto/qat/qat_common/qat_uclo.c | 12 +++++++----- 3 files changed, 18 insertions(+), 6 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 6b1ad629357b..8025be597d18 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -24,6 +24,7 @@ struct icp_qat_fw_loader_hal_handle { struct icp_qat_fw_loader_chip_info { bool sram_visible; + bool nn; bool fw_auth; }; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 2faf8638526c..e0d0ab43fd12 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -603,7 +603,9 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val); csr_val = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); csr_val &= IGNORE_W1C_MASK; - csr_val |= CE_NN_MODE; + if (handle->chip_info->nn) + csr_val |= CE_NN_MODE; + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val); qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst), (u64 *)inst); @@ -665,10 +667,12 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, case PCI_DEVICE_ID_INTEL_QAT_C62X: case PCI_DEVICE_ID_INTEL_QAT_C3XXX: handle->chip_info->sram_visible = false; + handle->chip_info->nn = true; handle->chip_info->fw_auth = true; break; case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: handle->chip_info->sram_visible = true; + handle->chip_info->nn = true; handle->chip_info->fw_auth = false; break; default: @@ -1433,6 +1437,11 @@ int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, { int stat = 0; unsigned char ctx; + if (!handle->chip_info->nn) { + dev_err(&handle->pci_dev->dev, "QAT: No next neigh in 0x%x\n", + handle->pci_dev->device); + return -EINVAL; + } if (ctx_mask == 0) return -EINVAL; diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 5774916497bd..fce075874962 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -884,11 +884,13 @@ static int qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle, pr_err("QAT: qat_hal_set_ae_ctx_mode error\n"); return ret; } - mode = ICP_QAT_NN_MODE(uof_image->ae_mode); - ret = qat_hal_set_ae_nn_mode(handle, ae, mode); - if (ret) { - pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); - return ret; + if (handle->chip_info->nn) { + mode = ICP_QAT_NN_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_nn_mode(handle, ae, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); + return ret; + } } mode = ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, mode); -- cgit v1.2.3 From 8b487ae26ad58abd602ab2d0cda6988306a6da20 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:57 +0800 Subject: crypto: qat - add support for lm2 and lm3 Add support for local memory lm2 and lm3 which is introduced in the next generation of QAT devices. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_common_drv.h | 2 + .../qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/icp_qat_hal.h | 9 +++ drivers/crypto/qat/qat_common/icp_qat_uclo.h | 5 ++ drivers/crypto/qat/qat_common/qat_hal.c | 70 +++++++++++++++++++++- drivers/crypto/qat/qat_common/qat_uclo.c | 16 +++++ 6 files changed, 101 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 945608b71937..f4c90c701670 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -178,6 +178,8 @@ int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, unsigned short reg_num, unsigned int regdata); int qat_hal_wr_lm(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned short lm_addr, unsigned int value); +void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char mode); int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle); void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle); int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, void *addr_ptr, diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 8025be597d18..3c587105d09d 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -25,6 +25,7 @@ struct icp_qat_fw_loader_hal_handle { struct icp_qat_fw_loader_chip_info { bool sram_visible; bool nn; + bool lm2lm3; bool fw_auth; }; diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index eff9a3811435..82ac33a4500f 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -26,8 +26,14 @@ enum hal_ae_csr { CTX_WAKEUP_EVENTS_INDIRECT = 0x050, LM_ADDR_0_INDIRECT = 0x060, LM_ADDR_1_INDIRECT = 0x068, + LM_ADDR_2_INDIRECT = 0x0cc, + LM_ADDR_3_INDIRECT = 0x0d4, INDIRECT_LM_ADDR_0_BYTE_INDEX = 0x0e0, INDIRECT_LM_ADDR_1_BYTE_INDEX = 0x0e8, + INDIRECT_LM_ADDR_2_BYTE_INDEX = 0x10c, + INDIRECT_LM_ADDR_3_BYTE_INDEX = 0x114, + INDIRECT_T_INDEX = 0x0f8, + INDIRECT_T_INDEX_BYTE_INDEX = 0x0fc, FUTURE_COUNT_SIGNAL_INDIRECT = 0x078, TIMESTAMP_LOW = 0x0c0, TIMESTAMP_HIGH = 0x0c4, @@ -68,6 +74,9 @@ enum fcu_sts { #define CE_ENABLE_BITPOS 0x8 #define CE_LMADDR_0_GLOBAL_BITPOS 16 #define CE_LMADDR_1_GLOBAL_BITPOS 17 +#define CE_LMADDR_2_GLOBAL_BITPOS 22 +#define CE_LMADDR_3_GLOBAL_BITPOS 23 +#define CE_T_INDEX_GLOBAL_BITPOS 21 #define CE_NN_MODE_BITPOS 20 #define CE_REG_PAR_ERR_BITPOS 25 #define CE_BREAKPOINT_BITPOS 27 diff --git a/drivers/crypto/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/qat/qat_common/icp_qat_uclo.h index 101de1430896..5728a81d9dea 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_uclo.h +++ b/drivers/crypto/qat/qat_common/icp_qat_uclo.h @@ -69,6 +69,9 @@ #define ICP_QAT_LOC_MEM0_MODE(ae_mode) (((ae_mode) >> 0x8) & 0x1) #define ICP_QAT_LOC_MEM1_MODE(ae_mode) (((ae_mode) >> 0x9) & 0x1) +#define ICP_QAT_LOC_MEM2_MODE(ae_mode) (((ae_mode) >> 0x6) & 0x1) +#define ICP_QAT_LOC_MEM3_MODE(ae_mode) (((ae_mode) >> 0x7) & 0x1) +#define ICP_QAT_LOC_TINDEX_MODE(ae_mode) (((ae_mode) >> 0xe) & 0x1) enum icp_qat_uof_mem_region { ICP_QAT_UOF_SRAM_REGION = 0x0, @@ -98,6 +101,8 @@ enum icp_qat_uof_regtype { ICP_LMEM0 = 27, ICP_LMEM1 = 28, ICP_NEIGH_REL = 31, + ICP_LMEM2 = 61, + ICP_LMEM3 = 62, }; enum icp_qat_css_fwtype { diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index e0d0ab43fd12..70fc93f31e79 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -210,6 +210,16 @@ int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle, SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) : CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS); break; + case ICP_LMEM2: + new_csr = (mode) ? + SET_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS) : + CLR_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS); + break; + case ICP_LMEM3: + new_csr = (mode) ? + SET_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS) : + CLR_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS); + break; default: pr_err("QAT: lmType = 0x%x\n", lm_type); return -EINVAL; @@ -220,6 +230,20 @@ int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle, return 0; } +void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char mode) +{ + unsigned int csr, new_csr; + + csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + csr &= IGNORE_W1C_MASK; + new_csr = (mode) ? + SET_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS) : + CLR_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS); + if (new_csr != csr) + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); +} + static unsigned short qat_hal_get_reg_addr(unsigned int type, unsigned short reg_num) { @@ -259,6 +283,12 @@ static unsigned short qat_hal_get_reg_addr(unsigned int type, case ICP_LMEM1: reg_addr = 0x220; break; + case ICP_LMEM2: + reg_addr = 0x2c0; + break; + case ICP_LMEM3: + reg_addr = 0x2e0; + break; case ICP_NO_DEST: reg_addr = 0x300 | (reg_num & 0xff); break; @@ -668,11 +698,13 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, case PCI_DEVICE_ID_INTEL_QAT_C3XXX: handle->chip_info->sram_visible = false; handle->chip_info->nn = true; + handle->chip_info->lm2lm3 = false; handle->chip_info->fw_auth = true; break; case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: handle->chip_info->sram_visible = true; handle->chip_info->nn = true; + handle->chip_info->lm2lm3 = false; handle->chip_info->fw_auth = false; break; default: @@ -889,9 +921,12 @@ static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, int code_off, unsigned int max_cycle, unsigned int *endpc) { + unsigned int ind_lm_addr_byte0 = 0, ind_lm_addr_byte1 = 0; + unsigned int ind_lm_addr_byte2 = 0, ind_lm_addr_byte3 = 0; + unsigned int ind_t_index = 0, ind_t_index_byte = 0; + unsigned int ind_lm_addr0 = 0, ind_lm_addr1 = 0; + unsigned int ind_lm_addr2 = 0, ind_lm_addr3 = 0; u64 savuwords[MAX_EXEC_INST]; - unsigned int ind_lm_addr0, ind_lm_addr1; - unsigned int ind_lm_addr_byte0, ind_lm_addr_byte1; unsigned int ind_cnt_sig; unsigned int ind_sig, act_sig; unsigned int csr_val = 0, newcsr_val; @@ -910,6 +945,20 @@ static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, INDIRECT_LM_ADDR_0_BYTE_INDEX); ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX); + if (handle->chip_info->lm2lm3) { + ind_lm_addr2 = qat_hal_rd_indr_csr(handle, ae, ctx, + LM_ADDR_2_INDIRECT); + ind_lm_addr3 = qat_hal_rd_indr_csr(handle, ae, ctx, + LM_ADDR_3_INDIRECT); + ind_lm_addr_byte2 = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_LM_ADDR_2_BYTE_INDEX); + ind_lm_addr_byte3 = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_LM_ADDR_3_BYTE_INDEX); + ind_t_index = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_T_INDEX); + ind_t_index_byte = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_T_INDEX_BYTE_INDEX); + } if (inst_num <= MAX_EXEC_INST) qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords); qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events); @@ -967,6 +1016,23 @@ static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0); qat_hal_wr_indr_csr(handle, ae, (1 << ctx), INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1); + if (handle->chip_info->lm2lm3) { + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_2_INDIRECT, + ind_lm_addr2); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_3_INDIRECT, + ind_lm_addr3); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), + INDIRECT_LM_ADDR_2_BYTE_INDEX, + ind_lm_addr_byte2); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), + INDIRECT_LM_ADDR_3_BYTE_INDEX, + ind_lm_addr_byte3); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), + INDIRECT_T_INDEX, ind_t_index); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), + INDIRECT_T_INDEX_BYTE_INDEX, + ind_t_index_byte); + } qat_hal_wr_indr_csr(handle, ae, (1 << ctx), FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig); qat_hal_wr_indr_csr(handle, ae, (1 << ctx), diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index fce075874962..4a90b150199c 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -904,6 +904,22 @@ static int qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle, pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n"); return ret; } + if (handle->chip_info->lm2lm3) { + mode = ICP_QAT_LOC_MEM2_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM2, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_lm_mode LMEM2 error\n"); + return ret; + } + mode = ICP_QAT_LOC_MEM3_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM3, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_lm_mode LMEM3 error\n"); + return ret; + } + mode = ICP_QAT_LOC_TINDEX_MODE(uof_image->ae_mode); + qat_hal_set_ae_tindex_mode(handle, ae, mode); + } return 0; } -- cgit v1.2.3 From 4f07195d638e82a1a1f17f55d7aa8a74dacabfe9 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:58 +0800 Subject: crypto: qat - add local memory size to chip info Add the local memory size to the chip info since the size of this memory will be different in the next generation of QAT devices. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/qat_hal.c | 2 ++ drivers/crypto/qat/qat_common/qat_uclo.c | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 3c587105d09d..0fa5c22fd9c0 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -26,6 +26,7 @@ struct icp_qat_fw_loader_chip_info { bool sram_visible; bool nn; bool lm2lm3; + u32 lm_size; bool fw_auth; }; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 70fc93f31e79..44cf797ace71 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -699,12 +699,14 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->sram_visible = false; handle->chip_info->nn = true; handle->chip_info->lm2lm3 = false; + handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->fw_auth = true; break; case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: handle->chip_info->sram_visible = true; handle->chip_info->nn = true; handle->chip_info->lm2lm3 = false; + handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->fw_auth = false; break; default: diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 4a90b150199c..32c64a48926f 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -311,7 +311,7 @@ static int qat_uclo_init_lmem_seg(struct icp_qat_fw_loader_handle *handle, unsigned int ae; if (qat_uclo_fetch_initmem_ae(handle, init_mem, - ICP_QAT_UCLO_MAX_LMEM_REG, &ae)) + handle->chip_info->lm_size, &ae)) return -EINVAL; if (qat_uclo_create_batch_init_list(handle, init_mem, ae, &obj_handle->lm_init_tab[ae])) -- cgit v1.2.3 From cb439361a39bb8dcdd0856e9182e29678c8327af Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:59 +0800 Subject: crypto: qat - add reset CSR and mask to chip info Add reset CSR offset and mask to chip info since they are different in new QAT devices. This also simplifies the reset/clrReset functions by using the reset mask. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../qat/qat_common/icp_qat_fw_loader_handle.h | 2 ++ drivers/crypto/qat/qat_common/qat_hal.c | 39 ++++++++++++---------- 2 files changed, 24 insertions(+), 17 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 0fa5c22fd9c0..5e2c0ef6d26c 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -27,6 +27,8 @@ struct icp_qat_fw_loader_chip_info { bool nn; bool lm2lm3; u32 lm_size; + u32 icp_rst_csr; + u32 icp_rst_mask; bool fw_auth; }; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 44cf797ace71..0d64e074fb44 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -301,12 +301,13 @@ static unsigned short qat_hal_get_reg_addr(unsigned int type, void qat_hal_reset(struct icp_qat_fw_loader_handle *handle) { - unsigned int ae_reset_csr; + unsigned int reset_mask = handle->chip_info->icp_rst_mask; + unsigned int reset_csr = handle->chip_info->icp_rst_csr; + unsigned int csr_val; - ae_reset_csr = GET_CAP_CSR(handle, ICP_RESET); - ae_reset_csr |= handle->hal_handle->ae_mask << RST_CSR_AE_LSB; - ae_reset_csr |= handle->hal_handle->slice_mask << RST_CSR_QAT_LSB; - SET_CAP_CSR(handle, ICP_RESET, ae_reset_csr); + csr_val = GET_CAP_CSR(handle, reset_csr); + csr_val |= reset_mask; + SET_CAP_CSR(handle, reset_csr, csr_val); } static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle, @@ -470,28 +471,27 @@ static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) #define SHRAM_INIT_CYCLES 2060 int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) { + unsigned int reset_mask = handle->chip_info->icp_rst_mask; + unsigned int reset_csr = handle->chip_info->icp_rst_csr; unsigned long ae_mask = handle->hal_handle->ae_mask; - unsigned int ae_reset_csr; - unsigned char ae; + unsigned char ae = 0; unsigned int clk_csr; unsigned int times = 100; - unsigned int csr; + unsigned int csr_val; /* write to the reset csr */ - ae_reset_csr = GET_CAP_CSR(handle, ICP_RESET); - ae_reset_csr &= ~(handle->hal_handle->ae_mask << RST_CSR_AE_LSB); - ae_reset_csr &= ~(handle->hal_handle->slice_mask << RST_CSR_QAT_LSB); + csr_val = GET_CAP_CSR(handle, reset_csr); + csr_val &= ~reset_mask; do { - SET_CAP_CSR(handle, ICP_RESET, ae_reset_csr); + SET_CAP_CSR(handle, reset_csr, csr_val); if (!(times--)) goto out_err; - csr = GET_CAP_CSR(handle, ICP_RESET); - } while ((handle->hal_handle->ae_mask | - (handle->hal_handle->slice_mask << RST_CSR_QAT_LSB)) & csr); + csr_val = GET_CAP_CSR(handle, reset_csr); + csr_val &= reset_mask; + } while (csr_val); /* enable clock */ clk_csr = GET_CAP_CSR(handle, ICP_GLOBAL_CLK_ENABLE); - clk_csr |= handle->hal_handle->ae_mask << 0; - clk_csr |= handle->hal_handle->slice_mask << 20; + clk_csr |= reset_mask; SET_CAP_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr); if (qat_hal_check_ae_alive(handle)) goto out_err; @@ -700,6 +700,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->nn = true; handle->chip_info->lm2lm3 = false; handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; + handle->chip_info->icp_rst_csr = ICP_RESET; handle->chip_info->fw_auth = true; break; case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: @@ -707,6 +708,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->nn = true; handle->chip_info->lm2lm3 = false; handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; + handle->chip_info->icp_rst_csr = ICP_RESET; handle->chip_info->fw_auth = false; break; default: @@ -719,6 +721,9 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; handle->hal_sram_addr_v = sram_bar->virt_addr; } + + handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) | + (hw_data->accel_mask << RST_CSR_QAT_LSB); handle->hal_cap_g_ctl_csr_addr_v = (void __iomem *)((uintptr_t)misc_bar->virt_addr + ICP_QAT_CAP_OFFSET); -- cgit v1.2.3 From 767358119fca655056eb7340eb8a0ec4652bc888 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:00 +0800 Subject: crypto: qat - add clock enable CSR to chip info Add global clock enable CSR to the chip info since the CSR offset will be different in the next generation of QAT devices. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/qat_hal.c | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 5e2c0ef6d26c..1d6ab3407dc9 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -29,6 +29,7 @@ struct icp_qat_fw_loader_chip_info { u32 lm_size; u32 icp_rst_csr; u32 icp_rst_mask; + u32 glb_clk_enable_csr; bool fw_auth; }; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 0d64e074fb44..6e6bca281ab7 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -471,11 +471,11 @@ static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) #define SHRAM_INIT_CYCLES 2060 int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) { + unsigned int clk_csr = handle->chip_info->glb_clk_enable_csr; unsigned int reset_mask = handle->chip_info->icp_rst_mask; unsigned int reset_csr = handle->chip_info->icp_rst_csr; unsigned long ae_mask = handle->hal_handle->ae_mask; unsigned char ae = 0; - unsigned int clk_csr; unsigned int times = 100; unsigned int csr_val; @@ -490,9 +490,9 @@ int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) csr_val &= reset_mask; } while (csr_val); /* enable clock */ - clk_csr = GET_CAP_CSR(handle, ICP_GLOBAL_CLK_ENABLE); - clk_csr |= reset_mask; - SET_CAP_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr); + csr_val = GET_CAP_CSR(handle, clk_csr); + csr_val |= reset_mask; + SET_CAP_CSR(handle, clk_csr, csr_val); if (qat_hal_check_ae_alive(handle)) goto out_err; @@ -701,6 +701,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->lm2lm3 = false; handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->icp_rst_csr = ICP_RESET; + handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; handle->chip_info->fw_auth = true; break; case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: @@ -709,6 +710,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->lm2lm3 = false; handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->icp_rst_csr = ICP_RESET; + handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; handle->chip_info->fw_auth = false; break; default: -- cgit v1.2.3 From c4909d327cc3b001583de29fde988a22856a5b38 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:01 +0800 Subject: crypto: qat - add wake up event to chip info Add the wake up event to chip info since this value will be different in the next generation of QAT devices. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/icp_qat_hal.h | 1 + drivers/crypto/qat/qat_common/qat_hal.c | 5 ++++- 3 files changed, 6 insertions(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 1d6ab3407dc9..090c3e73938c 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -30,6 +30,7 @@ struct icp_qat_fw_loader_chip_info { u32 icp_rst_csr; u32 icp_rst_mask; u32 glb_clk_enable_csr; + u32 wakeup_event_val; bool fw_auth; }; diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index 82ac33a4500f..b3aa4c8a3ba8 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -87,6 +87,7 @@ enum fcu_sts { #define XCWE_VOLUNTARY (0x1) #define LCS_STATUS (0x1) #define MMC_SHARE_CS_BITPOS 2 +#define WAKEUP_EVENT 0x10000 #define FCU_CTRL_AE_POS 0x8 #define FCU_AUTH_STS_MASK 0x7 #define FCU_STS_DONE_POS 0x9 diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 6e6bca281ab7..c073e4e3e3ae 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -702,6 +702,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->icp_rst_csr = ICP_RESET; handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; + handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = true; break; case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: @@ -711,6 +712,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->icp_rst_csr = ICP_RESET; handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; + handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = false; break; default: @@ -834,6 +836,7 @@ void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle) int qat_hal_start(struct icp_qat_fw_loader_handle *handle) { unsigned long ae_mask = handle->hal_handle->ae_mask; + u32 wakeup_val = handle->chip_info->wakeup_event_val; unsigned int fcu_sts; unsigned char ae; u32 ae_ctr = 0; @@ -852,7 +855,7 @@ int qat_hal_start(struct icp_qat_fw_loader_handle *handle) return 0; } else { for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { - qat_hal_put_wakeup_event(handle, ae, 0, 0x10000); + qat_hal_put_wakeup_event(handle, ae, 0, wakeup_val); qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX); ae_ctr++; } -- cgit v1.2.3 From 4f1e941560e1c29299c3b1b9f90f4aea790d12dc Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:02 +0800 Subject: crypto: qat - add misc control CSR to chip info Add misc control CSR to chip info since the CSR offset will be different in the next generation of QAT devices. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/qat_hal.c | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 090c3e73938c..81dba42248bf 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -30,6 +30,7 @@ struct icp_qat_fw_loader_chip_info { u32 icp_rst_csr; u32 icp_rst_mask; u32 glb_clk_enable_csr; + u32 misc_ctl_csr; u32 wakeup_event_val; bool fw_auth; }; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index c073e4e3e3ae..eae1a5e0efeb 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -417,13 +417,14 @@ int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle, static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) { unsigned long ae_mask = handle->hal_handle->ae_mask; - unsigned int misc_ctl; + unsigned int misc_ctl_csr, misc_ctl; unsigned char ae; + misc_ctl_csr = handle->chip_info->misc_ctl_csr; /* stop the timestamp timers */ - misc_ctl = GET_CAP_CSR(handle, MISC_CONTROL); + misc_ctl = GET_CAP_CSR(handle, misc_ctl_csr); if (misc_ctl & MC_TIMESTAMP_ENABLE) - SET_CAP_CSR(handle, MISC_CONTROL, misc_ctl & + SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl & (~MC_TIMESTAMP_ENABLE)); for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { @@ -431,7 +432,7 @@ static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0); } /* start timestamp timers */ - SET_CAP_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE); + SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl | MC_TIMESTAMP_ENABLE); } #define ESRAM_AUTO_TINIT BIT(2) @@ -702,6 +703,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->icp_rst_csr = ICP_RESET; handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; + handle->chip_info->misc_ctl_csr = MISC_CONTROL; handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = true; break; @@ -712,6 +714,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->icp_rst_csr = ICP_RESET; handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; + handle->chip_info->misc_ctl_csr = MISC_CONTROL; handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = false; break; -- cgit v1.2.3 From 912eebeb72b499de3b25cda9e2b9e3f5e91bed12 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:03 +0800 Subject: crypto: qat - add check for null pointer Add null pointer check when freeing the memory for firmware. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_uclo.c | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 32c64a48926f..7b02c4e165c6 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1233,11 +1233,15 @@ static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle, static void qat_uclo_simg_free(struct icp_qat_fw_loader_handle *handle, struct icp_firml_dram_desc *dram_desc) { - dma_free_coherent(&handle->pci_dev->dev, - (size_t)(dram_desc->dram_size), - dram_desc->dram_base_addr_v, - dram_desc->dram_bus_addr); - memset(dram_desc, 0, sizeof(*dram_desc)); + if (handle && dram_desc && dram_desc->dram_base_addr_v) { + dma_free_coherent(&handle->pci_dev->dev, + (size_t)(dram_desc->dram_size), + dram_desc->dram_base_addr_v, + dram_desc->dram_bus_addr); + } + + if (dram_desc) + memset(dram_desc, 0, sizeof(*dram_desc)); } static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle, @@ -1245,12 +1249,14 @@ static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle, { struct icp_firml_dram_desc dram_desc; - dram_desc.dram_base_addr_v = *desc; - dram_desc.dram_bus_addr = ((struct icp_qat_auth_chunk *) - (*desc))->chunk_bus_addr; - dram_desc.dram_size = ((struct icp_qat_auth_chunk *) - (*desc))->chunk_size; - qat_uclo_simg_free(handle, &dram_desc); + if (*desc) { + dram_desc.dram_base_addr_v = *desc; + dram_desc.dram_bus_addr = ((struct icp_qat_auth_chunk *) + (*desc))->chunk_bus_addr; + dram_desc.dram_size = ((struct icp_qat_auth_chunk *) + (*desc))->chunk_size; + qat_uclo_simg_free(handle, &dram_desc); + } } static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, -- cgit v1.2.3 From bf8313c71c887c3c8676c23a20b6f8eb1d56bd4f Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:04 +0800 Subject: crypto: qat - use ae_mask Use ae_mask to decide which Accelerator Engine (AE) to target in AE related operations, instead of a sequential loop, to skip AEs that are fused out. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_uclo.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 7b02c4e165c6..0b1cf0708e2e 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -373,6 +373,7 @@ static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, unsigned int ustore_size; unsigned int patt_pos; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; u64 *fill_data; uof_image = image->img_ptr; @@ -385,7 +386,7 @@ static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, sizeof(u64)); page = image->page; - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { if (!test_bit(ae, (unsigned long *)&uof_image->ae_assigned)) continue; ustore_size = obj_handle->ae_data[ae].eff_ustore_size; @@ -406,6 +407,7 @@ static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle) int i, ae; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; struct icp_qat_uof_initmem *initmem = obj_handle->init_mem_tab.init_mem; + unsigned long ae_mask = handle->hal_handle->ae_mask; for (i = 0; i < obj_handle->init_mem_tab.entry_num; i++) { if (initmem->num_in_bytes) { @@ -418,7 +420,8 @@ static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle) (sizeof(struct icp_qat_uof_memvar_attr) * initmem->val_attr_num)); } - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { if (qat_hal_batch_wr_lm(handle, ae, obj_handle->lm_init_tab[ae])) { pr_err("QAT: fail to batch init lmem for AE %d\n", ae); @@ -649,11 +652,9 @@ static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) int i, ae; int mflag = 0; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; - for (ae = 0; ae < max_ae; ae++) { - if (!test_bit(ae, - (unsigned long *)&handle->hal_handle->ae_mask)) - continue; + for_each_set_bit(ae, &ae_mask, max_ae) { for (i = 0; i < obj_handle->uimage_num; i++) { if (!test_bit(ae, (unsigned long *) &obj_handle->ae_uimage[i].img_ptr->ae_assigned)) @@ -845,6 +846,7 @@ static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle, static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; struct icp_qat_uclo_aedata *aed; unsigned int s, ae; @@ -857,7 +859,7 @@ static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) } } - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { aed = &obj_handle->ae_data[ae]; for (s = 0; s < aed->slice_num; s++) { if (!aed->ae_slices[s].encap_image) @@ -932,9 +934,7 @@ static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle) unsigned char ae, s; int error; - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - if (!test_bit(ae, &ae_mask)) - continue; + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { ae_data = &obj_handle->ae_data[ae]; for (s = 0; s < min_t(unsigned int, ae_data->slice_num, ICP_QAT_UCLO_MAX_CTX); s++) { @@ -1372,13 +1372,14 @@ static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, unsigned int fcu_sts; struct icp_qat_simg_ae_mode *virt_addr; unsigned int fcu_loaded_ae_pos = FCU_LOADED_AE_POS; + unsigned long ae_mask = handle->hal_handle->ae_mask; virt_addr = (void *)((uintptr_t)desc + sizeof(struct icp_qat_auth_chunk) + sizeof(struct icp_qat_css_hdr) + ICP_QAT_CSS_FWSK_PUB_LEN + ICP_QAT_CSS_SIGNATURE_LEN); - for (i = 0; i < handle->hal_handle->ae_max_num; i++) { + for_each_set_bit(i, &ae_mask, handle->hal_handle->ae_max_num) { int retry = 0; if (!((virt_addr->ae_mask >> i) & 0x1)) @@ -1847,6 +1848,7 @@ static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, struct icp_qat_uof_image *image) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; unsigned long ae_assigned = image->ae_assigned; struct icp_qat_uclo_aedata *aed; unsigned int ctx_mask, s; @@ -1860,7 +1862,7 @@ static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, ctx_mask = 0x55; /* load the default page and set assigned CTX PC * to the entrypoint address */ - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { if (!test_bit(ae, &ae_assigned)) continue; -- cgit v1.2.3 From 9e0f74b717e41811921e647e7a6121cc55987f41 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:05 +0800 Subject: crypto: qat - add CSS3K support Add support for CSS3K, which uses RSA3K as image signature algorithm, to support the next generation of QAT devices. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/icp_qat_uclo.h | 54 ++++++++++++++++------ drivers/crypto/qat/qat_common/qat_hal.c | 2 + drivers/crypto/qat/qat_common/qat_uclo.c | 51 ++++++++++---------- 4 files changed, 68 insertions(+), 40 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 81dba42248bf..29710e88e8b8 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -33,6 +33,7 @@ struct icp_qat_fw_loader_chip_info { u32 misc_ctl_csr; u32 wakeup_event_val; bool fw_auth; + bool css_3k; }; struct icp_qat_fw_loader_handle { diff --git a/drivers/crypto/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/qat/qat_common/icp_qat_uclo.h index 5728a81d9dea..4315b4504c26 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_uclo.h +++ b/drivers/crypto/qat/qat_common/icp_qat_uclo.h @@ -42,24 +42,48 @@ #define ICP_QAT_SUOF_IMAG "SUF_IMAG" #define ICP_QAT_SIMG_AE_INIT_SEQ_LEN (50 * sizeof(unsigned long long)) #define ICP_QAT_SIMG_AE_INSTS_LEN (0x4000 * sizeof(unsigned long long)) -#define ICP_QAT_CSS_FWSK_MODULUS_LEN 256 -#define ICP_QAT_CSS_FWSK_EXPONENT_LEN 4 -#define ICP_QAT_CSS_FWSK_PAD_LEN 252 -#define ICP_QAT_CSS_FWSK_PUB_LEN (ICP_QAT_CSS_FWSK_MODULUS_LEN + \ - ICP_QAT_CSS_FWSK_EXPONENT_LEN + \ - ICP_QAT_CSS_FWSK_PAD_LEN) -#define ICP_QAT_CSS_SIGNATURE_LEN 256 + +#define DSS_FWSK_MODULUS_LEN 384 /* RSA3K */ +#define DSS_FWSK_EXPONENT_LEN 4 +#define DSS_FWSK_PADDING_LEN 380 +#define DSS_SIGNATURE_LEN 384 /* RSA3K */ + +#define CSS_FWSK_MODULUS_LEN 256 /* RSA2K */ +#define CSS_FWSK_EXPONENT_LEN 4 +#define CSS_FWSK_PADDING_LEN 252 +#define CSS_SIGNATURE_LEN 256 /* RSA2K */ + +#define ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) ((handle)->chip_info->css_3k ? \ + DSS_FWSK_MODULUS_LEN : \ + CSS_FWSK_MODULUS_LEN) + +#define ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle) ((handle)->chip_info->css_3k ? \ + DSS_FWSK_EXPONENT_LEN : \ + CSS_FWSK_EXPONENT_LEN) + +#define ICP_QAT_CSS_FWSK_PAD_LEN(handle) ((handle)->chip_info->css_3k ? \ + DSS_FWSK_PADDING_LEN : \ + CSS_FWSK_PADDING_LEN) + +#define ICP_QAT_CSS_FWSK_PUB_LEN(handle) (ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + \ + ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle) + \ + ICP_QAT_CSS_FWSK_PAD_LEN(handle)) + +#define ICP_QAT_CSS_SIGNATURE_LEN(handle) ((handle)->chip_info->css_3k ? \ + DSS_SIGNATURE_LEN : \ + CSS_SIGNATURE_LEN) + #define ICP_QAT_CSS_AE_IMG_LEN (sizeof(struct icp_qat_simg_ae_mode) + \ ICP_QAT_SIMG_AE_INIT_SEQ_LEN + \ ICP_QAT_SIMG_AE_INSTS_LEN) -#define ICP_QAT_CSS_AE_SIMG_LEN (sizeof(struct icp_qat_css_hdr) + \ - ICP_QAT_CSS_FWSK_PUB_LEN + \ - ICP_QAT_CSS_SIGNATURE_LEN + \ - ICP_QAT_CSS_AE_IMG_LEN) -#define ICP_QAT_AE_IMG_OFFSET (sizeof(struct icp_qat_css_hdr) + \ - ICP_QAT_CSS_FWSK_MODULUS_LEN + \ - ICP_QAT_CSS_FWSK_EXPONENT_LEN + \ - ICP_QAT_CSS_SIGNATURE_LEN) +#define ICP_QAT_CSS_AE_SIMG_LEN(handle) (sizeof(struct icp_qat_css_hdr) + \ + ICP_QAT_CSS_FWSK_PUB_LEN(handle) + \ + ICP_QAT_CSS_SIGNATURE_LEN(handle) + \ + ICP_QAT_CSS_AE_IMG_LEN) +#define ICP_QAT_AE_IMG_OFFSET(handle) (sizeof(struct icp_qat_css_hdr) + \ + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + \ + ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle) + \ + ICP_QAT_CSS_SIGNATURE_LEN(handle)) #define ICP_QAT_CSS_MAX_IMAGE_LEN 0x40000 #define ICP_QAT_CTX_MODE(ae_mode) ((ae_mode) & 0xf) diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index eae1a5e0efeb..8470139bcfe8 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -706,6 +706,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->misc_ctl_csr = MISC_CONTROL; handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = true; + handle->chip_info->css_3k = false; break; case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: handle->chip_info->sram_visible = true; @@ -717,6 +718,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->misc_ctl_csr = MISC_CONTROL; handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = false; + handle->chip_info->css_3k = false; break; default: ret = -EINVAL; diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 0b1cf0708e2e..933b6357971f 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1039,10 +1039,11 @@ static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle, return 0; } -static void qat_uclo_map_simg(struct icp_qat_suof_handle *suof_handle, +static void qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle, struct icp_qat_suof_img_hdr *suof_img_hdr, struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) { + struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; struct icp_qat_simg_ae_mode *ae_mode; struct icp_qat_suof_objhdr *suof_objhdr; @@ -1057,10 +1058,10 @@ static void qat_uclo_map_simg(struct icp_qat_suof_handle *suof_handle, suof_img_hdr->css_key = (suof_img_hdr->css_header + sizeof(struct icp_qat_css_hdr)); suof_img_hdr->css_signature = suof_img_hdr->css_key + - ICP_QAT_CSS_FWSK_MODULUS_LEN + - ICP_QAT_CSS_FWSK_EXPONENT_LEN; + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + + ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle); suof_img_hdr->css_simg = suof_img_hdr->css_signature + - ICP_QAT_CSS_SIGNATURE_LEN; + ICP_QAT_CSS_SIGNATURE_LEN(handle); ae_mode = (struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg); suof_img_hdr->ae_mask = ae_mode->ae_mask; @@ -1169,7 +1170,7 @@ static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, } for (i = 0; i < suof_handle->img_table.num_simgs; i++) { - qat_uclo_map_simg(handle->sobj_handle, &suof_img_hdr[i], + qat_uclo_map_simg(handle, &suof_img_hdr[i], &suof_chunk_hdr[1 + i]); ret = qat_uclo_check_simg_compat(handle, &suof_img_hdr[i]); @@ -1270,13 +1271,13 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, unsigned int length, simg_offset = sizeof(*auth_chunk); struct icp_firml_dram_desc img_desc; - if (size > (ICP_QAT_AE_IMG_OFFSET + ICP_QAT_CSS_MAX_IMAGE_LEN)) { + if (size > (ICP_QAT_AE_IMG_OFFSET(handle) + ICP_QAT_CSS_MAX_IMAGE_LEN)) { pr_err("QAT: error, input image size overflow %d\n", size); return -EINVAL; } length = (css_hdr->fw_type == CSS_AE_FIRMWARE) ? - ICP_QAT_CSS_AE_SIMG_LEN + simg_offset : - size + ICP_QAT_CSS_FWSK_PAD_LEN + simg_offset; + ICP_QAT_CSS_AE_SIMG_LEN(handle) + simg_offset : + size + ICP_QAT_CSS_FWSK_PAD_LEN(handle) + simg_offset; if (qat_uclo_simg_alloc(handle, &img_desc, length)) { pr_err("QAT: error, allocate continuous dram fail\n"); return -ENOMEM; @@ -1303,42 +1304,42 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, memcpy((void *)(uintptr_t)virt_addr, (void *)(image + sizeof(*css_hdr)), - ICP_QAT_CSS_FWSK_MODULUS_LEN); + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)); /* padding */ - memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN), - 0, ICP_QAT_CSS_FWSK_PAD_LEN); + memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), + 0, ICP_QAT_CSS_FWSK_PAD_LEN(handle)); /* exponent */ - memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN + - ICP_QAT_CSS_FWSK_PAD_LEN), + memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + + ICP_QAT_CSS_FWSK_PAD_LEN(handle)), (void *)(image + sizeof(*css_hdr) + - ICP_QAT_CSS_FWSK_MODULUS_LEN), + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), sizeof(unsigned int)); /* signature */ bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high, auth_desc->fwsk_pub_low) + - ICP_QAT_CSS_FWSK_PUB_LEN; - virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN; + ICP_QAT_CSS_FWSK_PUB_LEN(handle); + virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(handle); auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); auth_desc->signature_low = (unsigned int)bus_addr; memcpy((void *)(uintptr_t)virt_addr, (void *)(image + sizeof(*css_hdr) + - ICP_QAT_CSS_FWSK_MODULUS_LEN + - ICP_QAT_CSS_FWSK_EXPONENT_LEN), - ICP_QAT_CSS_SIGNATURE_LEN); + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + + ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle)), + ICP_QAT_CSS_SIGNATURE_LEN(handle)); bus_addr = ADD_ADDR(auth_desc->signature_high, auth_desc->signature_low) + - ICP_QAT_CSS_SIGNATURE_LEN; - virt_addr += ICP_QAT_CSS_SIGNATURE_LEN; + ICP_QAT_CSS_SIGNATURE_LEN(handle); + virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); auth_desc->img_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); auth_desc->img_low = (unsigned int)bus_addr; - auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET; + auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(handle); memcpy((void *)(uintptr_t)virt_addr, - (void *)(image + ICP_QAT_AE_IMG_OFFSET), + (void *)(image + ICP_QAT_AE_IMG_OFFSET(handle)), auth_desc->img_len); virt_addr = virt_base; /* AE firmware */ @@ -1377,8 +1378,8 @@ static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, virt_addr = (void *)((uintptr_t)desc + sizeof(struct icp_qat_auth_chunk) + sizeof(struct icp_qat_css_hdr) + - ICP_QAT_CSS_FWSK_PUB_LEN + - ICP_QAT_CSS_SIGNATURE_LEN); + ICP_QAT_CSS_FWSK_PUB_LEN(handle) + + ICP_QAT_CSS_SIGNATURE_LEN(handle)); for_each_set_bit(i, &ae_mask, handle->hal_handle->ae_max_num) { int retry = 0; -- cgit v1.2.3 From d707d3f23e69181da71877b2a687560fdad81ad0 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:06 +0800 Subject: crypto: qat - add FCU CSRs to chip info Add firmware control unit (FCU) CSRs to chip info so the firmware authentication code is common between all devices. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../qat/qat_common/icp_qat_fw_loader_handle.h | 6 +++ drivers/crypto/qat/qat_common/qat_hal.c | 19 +++++++++- drivers/crypto/qat/qat_common/qat_uclo.c | 44 +++++++++++++++------- 3 files changed, 53 insertions(+), 16 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 29710e88e8b8..e280a077303f 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -34,6 +34,12 @@ struct icp_qat_fw_loader_chip_info { u32 wakeup_event_val; bool fw_auth; bool css_3k; + u32 fcu_ctl_csr; + u32 fcu_sts_csr; + u32 fcu_dram_addr_hi; + u32 fcu_dram_addr_lo; + u32 fcu_loaded_ae_csr; + u8 fcu_loaded_ae_pos; }; struct icp_qat_fw_loader_handle { diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 8470139bcfe8..da138fb11a63 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -707,6 +707,12 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = true; handle->chip_info->css_3k = false; + handle->chip_info->fcu_ctl_csr = FCU_CONTROL; + handle->chip_info->fcu_sts_csr = FCU_STATUS; + handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI; + handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO; + handle->chip_info->fcu_loaded_ae_csr = FCU_STATUS; + handle->chip_info->fcu_loaded_ae_pos = FCU_LOADED_AE_POS; break; case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: handle->chip_info->sram_visible = true; @@ -719,6 +725,12 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = false; handle->chip_info->css_3k = false; + handle->chip_info->fcu_ctl_csr = 0; + handle->chip_info->fcu_sts_csr = 0; + handle->chip_info->fcu_dram_addr_hi = 0; + handle->chip_info->fcu_dram_addr_lo = 0; + handle->chip_info->fcu_loaded_ae_csr = 0; + handle->chip_info->fcu_loaded_ae_pos = 0; break; default: ret = -EINVAL; @@ -842,17 +854,20 @@ int qat_hal_start(struct icp_qat_fw_loader_handle *handle) { unsigned long ae_mask = handle->hal_handle->ae_mask; u32 wakeup_val = handle->chip_info->wakeup_event_val; + u32 fcu_ctl_csr, fcu_sts_csr; unsigned int fcu_sts; unsigned char ae; u32 ae_ctr = 0; int retry = 0; if (handle->chip_info->fw_auth) { + fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; + fcu_sts_csr = handle->chip_info->fcu_sts_csr; ae_ctr = hweight32(ae_mask); - SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_START); + SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_START); do { msleep(FW_AUTH_WAIT_PERIOD); - fcu_sts = GET_CAP_CSR(handle, FCU_STATUS); + fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1)) return ae_ctr; } while (retry++ < FW_AUTH_MAX_RETRY); diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 933b6357971f..3c5746d52756 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1190,18 +1190,26 @@ static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc) { - unsigned int fcu_sts, retry = 0; + u32 fcu_sts, retry = 0; + u32 fcu_ctl_csr, fcu_sts_csr; + u32 fcu_dram_hi_csr, fcu_dram_lo_csr; u64 bus_addr; bus_addr = ADD_ADDR(desc->css_hdr_high, desc->css_hdr_low) - sizeof(struct icp_qat_auth_chunk); - SET_CAP_CSR(handle, FCU_DRAM_ADDR_HI, (bus_addr >> BITS_IN_DWORD)); - SET_CAP_CSR(handle, FCU_DRAM_ADDR_LO, bus_addr); - SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_AUTH); + + fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; + fcu_sts_csr = handle->chip_info->fcu_sts_csr; + fcu_dram_hi_csr = handle->chip_info->fcu_dram_addr_hi; + fcu_dram_lo_csr = handle->chip_info->fcu_dram_addr_lo; + + SET_CAP_CSR(handle, fcu_dram_hi_csr, (bus_addr >> BITS_IN_DWORD)); + SET_CAP_CSR(handle, fcu_dram_lo_csr, bus_addr); + SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_AUTH); do { msleep(FW_AUTH_WAIT_PERIOD); - fcu_sts = GET_CAP_CSR(handle, FCU_STATUS); + fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_FAIL) goto auth_fail; if (((fcu_sts >> FCU_STS_AUTHFWLD_POS) & 0x1)) @@ -1369,11 +1377,16 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc) { - unsigned int i; - unsigned int fcu_sts; struct icp_qat_simg_ae_mode *virt_addr; - unsigned int fcu_loaded_ae_pos = FCU_LOADED_AE_POS; unsigned long ae_mask = handle->hal_handle->ae_mask; + u32 fcu_sts_csr, fcu_ctl_csr; + u32 loaded_aes, loaded_csr; + unsigned int i; + u32 fcu_sts; + + fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; + fcu_sts_csr = handle->chip_info->fcu_sts_csr; + loaded_csr = handle->chip_info->fcu_loaded_ae_csr; virt_addr = (void *)((uintptr_t)desc + sizeof(struct icp_qat_auth_chunk) + @@ -1389,16 +1402,19 @@ static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, pr_err("QAT: AE %d is active\n", i); return -EINVAL; } - SET_CAP_CSR(handle, FCU_CONTROL, + SET_CAP_CSR(handle, fcu_ctl_csr, (FCU_CTRL_CMD_LOAD | (i << FCU_CTRL_AE_POS))); do { msleep(FW_AUTH_WAIT_PERIOD); - fcu_sts = GET_CAP_CSR(handle, FCU_STATUS); - if (((fcu_sts & FCU_AUTH_STS_MASK) == - FCU_STS_LOAD_DONE) && - ((fcu_sts >> fcu_loaded_ae_pos) & (1 << i))) - break; + fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); + if ((fcu_sts & FCU_AUTH_STS_MASK) == + FCU_STS_LOAD_DONE) { + loaded_aes = GET_CAP_CSR(handle, loaded_csr); + loaded_aes >>= handle->chip_info->fcu_loaded_ae_pos; + if (loaded_aes & (1 << i)) + break; + } } while (retry++ < FW_AUTH_MAX_RETRY); if (retry > FW_AUTH_MAX_RETRY) { pr_err("QAT: firmware load failed timeout %x\n", retry); -- cgit v1.2.3 From 244f572cebec71f7a79a94706eded4d2213a4f1b Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:07 +0800 Subject: crypto: qat - allow to target specific AEs Introduce new API, qat_uclo_set_cfg_ae_mask(), to allow the load of the firmware image to a subset of Acceleration Engines (AEs). This is required by the next generation of QAT devices to be able to load different firmware images to the device. Signed-off-by: Jack Xu Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_common_drv.h | 2 ++ .../qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/icp_qat_hal.h | 2 ++ drivers/crypto/qat/qat_common/icp_qat_uclo.h | 2 +- drivers/crypto/qat/qat_common/qat_hal.c | 1 + drivers/crypto/qat/qat_common/qat_uclo.c | 42 ++++++++++++++++++---- 6 files changed, 42 insertions(+), 8 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index f4c90c701670..c61476553728 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -186,6 +186,8 @@ int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, void *addr_ptr, int mem_size); int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, void *addr_ptr, u32 mem_size, char *obj_name); +int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle, + unsigned int cfg_ae_mask); #if defined(CONFIG_PCI_IOV) int adf_sriov_configure(struct pci_dev *pdev, int numvfs); void adf_disable_sriov(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index e280a077303f..cc9b83d965af 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -49,6 +49,7 @@ struct icp_qat_fw_loader_handle { void *obj_handle; void *sobj_handle; void *mobj_handle; + unsigned int cfg_ae_mask; void __iomem *hal_sram_addr_v; void __iomem *hal_cap_g_ctl_csr_addr_v; void __iomem *hal_cap_ae_xfer_csr_addr_v; diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index b3aa4c8a3ba8..02476b2ceee1 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -68,6 +68,8 @@ enum fcu_sts { FCU_STS_LOAD_FAIL = 4, FCU_STS_BUSY = 5 }; + +#define ALL_AE_MASK 0xFFFFFFFF #define UA_ECS (0x1 << 31) #define ACS_ABO_BITPOS 31 #define ACS_ACNO 0x7 diff --git a/drivers/crypto/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/qat/qat_common/icp_qat_uclo.h index 4315b4504c26..0ec8a5ab51b5 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_uclo.h +++ b/drivers/crypto/qat/qat_common/icp_qat_uclo.h @@ -432,7 +432,7 @@ struct icp_qat_suof_handle { struct icp_qat_fw_auth_desc { unsigned int img_len; - unsigned int reserved; + unsigned int ae_mask; unsigned int css_hdr_high; unsigned int css_hdr_low; unsigned int img_high; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index da138fb11a63..94c0b04088b5 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -761,6 +761,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid; handle->hal_handle->ae_mask = hw_data->ae_mask; handle->hal_handle->slice_mask = hw_data->accel_mask; + handle->cfg_ae_mask = ALL_AE_MASK; /* create AE objects */ handle->hal_handle->upc_mask = 0x1ffff; handle->hal_handle->max_ustore = 0x4000; diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 3c5746d52756..c6b309d107f3 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -374,6 +374,7 @@ static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, unsigned int patt_pos; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long cfg_ae_mask = handle->cfg_ae_mask; u64 *fill_data; uof_image = image->img_ptr; @@ -389,6 +390,10 @@ static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { if (!test_bit(ae, (unsigned long *)&uof_image->ae_assigned)) continue; + + if (!test_bit(ae, &cfg_ae_mask)) + continue; + ustore_size = obj_handle->ae_data[ae].eff_ustore_size; patt_pos = page->beg_addr_p + page->micro_words_num; @@ -653,8 +658,12 @@ static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) int mflag = 0; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long cfg_ae_mask = handle->cfg_ae_mask; for_each_set_bit(ae, &ae_mask, max_ae) { + if (!test_bit(ae, &cfg_ae_mask)) + continue; + for (i = 0; i < obj_handle->uimage_num; i++) { if (!test_bit(ae, (unsigned long *) &obj_handle->ae_uimage[i].img_ptr->ae_assigned)) @@ -931,10 +940,14 @@ static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle) struct icp_qat_uclo_aedata *ae_data; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long cfg_ae_mask = handle->cfg_ae_mask; unsigned char ae, s; int error; for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + if (!test_bit(ae, &cfg_ae_mask)) + continue; + ae_data = &obj_handle->ae_data[ae]; for (s = 0; s < min_t(unsigned int, ae_data->slice_num, ICP_QAT_UCLO_MAX_CTX); s++) { @@ -1176,6 +1189,7 @@ static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, &suof_img_hdr[i]); if (ret) return ret; + suof_img_hdr[i].ae_mask &= handle->cfg_ae_mask; if ((suof_img_hdr[i].ae_mask & 0x1) != 0) ae0_img = i; } @@ -1277,6 +1291,7 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_auth_chunk *auth_chunk; u64 virt_addr, bus_addr, virt_base; unsigned int length, simg_offset = sizeof(*auth_chunk); + struct icp_qat_simg_ae_mode *simg_ae_mode; struct icp_firml_dram_desc img_desc; if (size > (ICP_QAT_AE_IMG_OFFSET(handle) + ICP_QAT_CSS_MAX_IMAGE_LEN)) { @@ -1366,6 +1381,11 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, auth_desc->img_ae_insts_high = (unsigned int) (bus_addr >> BITS_IN_DWORD); auth_desc->img_ae_insts_low = (unsigned int)bus_addr; + virt_addr += sizeof(struct icp_qat_css_hdr); + virt_addr += ICP_QAT_CSS_FWSK_PUB_LEN(handle); + virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); + simg_ae_mode = (struct icp_qat_simg_ae_mode *)(uintptr_t)virt_addr; + auth_desc->ae_mask = simg_ae_mode->ae_mask & handle->cfg_ae_mask; } else { auth_desc->img_ae_insts_high = auth_desc->img_high; auth_desc->img_ae_insts_low = auth_desc->img_low; @@ -1377,7 +1397,6 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc) { - struct icp_qat_simg_ae_mode *virt_addr; unsigned long ae_mask = handle->hal_handle->ae_mask; u32 fcu_sts_csr, fcu_ctl_csr; u32 loaded_aes, loaded_csr; @@ -1388,15 +1407,10 @@ static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, fcu_sts_csr = handle->chip_info->fcu_sts_csr; loaded_csr = handle->chip_info->fcu_loaded_ae_csr; - virt_addr = (void *)((uintptr_t)desc + - sizeof(struct icp_qat_auth_chunk) + - sizeof(struct icp_qat_css_hdr) + - ICP_QAT_CSS_FWSK_PUB_LEN(handle) + - ICP_QAT_CSS_SIGNATURE_LEN(handle)); for_each_set_bit(i, &ae_mask, handle->hal_handle->ae_max_num) { int retry = 0; - if (!((virt_addr->ae_mask >> i) & 0x1)) + if (!((desc->ae_mask >> i) & 0x1)) continue; if (qat_hal_check_ae_active(handle, i)) { pr_err("QAT: AE %d is active\n", i); @@ -1866,6 +1880,7 @@ static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long cfg_ae_mask = handle->cfg_ae_mask; unsigned long ae_assigned = image->ae_assigned; struct icp_qat_uclo_aedata *aed; unsigned int ctx_mask, s; @@ -1880,6 +1895,9 @@ static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, /* load the default page and set assigned CTX PC * to the entrypoint address */ for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + if (!test_bit(ae, &cfg_ae_mask)) + continue; + if (!test_bit(ae, &ae_assigned)) continue; @@ -1957,3 +1975,13 @@ int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle) return (handle->chip_info->fw_auth) ? qat_uclo_wr_suof_img(handle) : qat_uclo_wr_uof_img(handle); } + +int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle, + unsigned int cfg_ae_mask) +{ + if (!cfg_ae_mask) + return -EINVAL; + + handle->cfg_ae_mask = cfg_ae_mask; + return 0; +} -- cgit v1.2.3 From bd684d83c789dd4882a539075e45d46011a95bdf Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:08 +0800 Subject: crypto: qat - add support for shared ustore Add support for shared ustore mode support. This is required by the next generation of QAT devices to share the same fw image across engines. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/qat_hal.c | 2 ++ drivers/crypto/qat/qat_common/qat_uclo.c | 29 ++++++++++++---------- 3 files changed, 19 insertions(+), 13 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index cc9b83d965af..5b9f2e8c9451 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -34,6 +34,7 @@ struct icp_qat_fw_loader_chip_info { u32 wakeup_event_val; bool fw_auth; bool css_3k; + bool tgroup_share_ustore; u32 fcu_ctl_csr; u32 fcu_sts_csr; u32 fcu_dram_addr_hi; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 94c0b04088b5..6ccfb8cf3a07 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -707,6 +707,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = true; handle->chip_info->css_3k = false; + handle->chip_info->tgroup_share_ustore = false; handle->chip_info->fcu_ctl_csr = FCU_CONTROL; handle->chip_info->fcu_sts_csr = FCU_STATUS; handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI; @@ -725,6 +726,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = false; handle->chip_info->css_3k = false; + handle->chip_info->tgroup_share_ustore = false; handle->chip_info->fcu_ctl_csr = 0; handle->chip_info->fcu_sts_csr = 0; handle->chip_info->fcu_dram_addr_hi = 0; diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index c6b309d107f3..b280fb0722c5 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1180,21 +1180,24 @@ static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, if (!suof_img_hdr) return -ENOMEM; suof_handle->img_table.simg_hdr = suof_img_hdr; - } - for (i = 0; i < suof_handle->img_table.num_simgs; i++) { - qat_uclo_map_simg(handle, &suof_img_hdr[i], - &suof_chunk_hdr[1 + i]); - ret = qat_uclo_check_simg_compat(handle, - &suof_img_hdr[i]); - if (ret) - return ret; - suof_img_hdr[i].ae_mask &= handle->cfg_ae_mask; - if ((suof_img_hdr[i].ae_mask & 0x1) != 0) - ae0_img = i; + for (i = 0; i < suof_handle->img_table.num_simgs; i++) { + qat_uclo_map_simg(handle, &suof_img_hdr[i], + &suof_chunk_hdr[1 + i]); + ret = qat_uclo_check_simg_compat(handle, + &suof_img_hdr[i]); + if (ret) + return ret; + suof_img_hdr[i].ae_mask &= handle->cfg_ae_mask; + if ((suof_img_hdr[i].ae_mask & 0x1) != 0) + ae0_img = i; + } + + if (!handle->chip_info->tgroup_share_ustore) { + qat_uclo_tail_img(suof_img_hdr, ae0_img, + suof_handle->img_table.num_simgs); + } } - qat_uclo_tail_img(suof_img_hdr, ae0_img, - suof_handle->img_table.num_simgs); return 0; } -- cgit v1.2.3 From 2778d64cf3f5517642555781df5628488e7d8186 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:09 +0800 Subject: crypto: qat - add support for broadcasting mode Add support for broadcasting mode in firmware loader to enable the next generation of QAT devices. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/icp_qat_hal.h | 10 +++ drivers/crypto/qat/qat_common/qat_hal.c | 1 + drivers/crypto/qat/qat_common/qat_uclo.c | 90 +++++++++++++++++++++- 4 files changed, 99 insertions(+), 3 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 5b9f2e8c9451..b8f3463be6ef 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -15,6 +15,7 @@ struct icp_qat_fw_loader_ae_data { struct icp_qat_fw_loader_hal_handle { struct icp_qat_fw_loader_ae_data aes[ICP_QAT_UCLO_MAX_AE]; unsigned int ae_mask; + unsigned int admin_ae_mask; unsigned int slice_mask; unsigned int revision_id; unsigned int ae_max_num; diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index 02476b2ceee1..8372f18ebc80 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -53,6 +53,15 @@ enum fcu_csr { FCU_RAMBASE_ADDR_LO = 0x8d8 }; +enum fcu_csr_4xxx { + FCU_CONTROL_4XXX = 0x1000, + FCU_STATUS_4XXX = 0x1004, + FCU_ME_BROADCAST_MASK_TYPE = 0x1008, + FCU_AE_LOADED_4XXX = 0x1010, + FCU_DRAM_ADDR_LO_4XXX = 0x1014, + FCU_DRAM_ADDR_HI_4XXX = 0x1018, +}; + enum fcu_cmd { FCU_CTRL_CMD_NOOP = 0, FCU_CTRL_CMD_AUTH = 1, @@ -90,6 +99,7 @@ enum fcu_sts { #define LCS_STATUS (0x1) #define MMC_SHARE_CS_BITPOS 2 #define WAKEUP_EVENT 0x10000 +#define FCU_CTRL_BROADCAST_POS 0x4 #define FCU_CTRL_AE_POS 0x8 #define FCU_AUTH_STS_MASK 0x7 #define FCU_STS_DONE_POS 0x9 diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 6ccfb8cf3a07..a3c1f2163910 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -762,6 +762,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->pci_dev = pci_info->pci_dev; handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid; handle->hal_handle->ae_mask = hw_data->ae_mask; + handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask; handle->hal_handle->slice_mask = hw_data->accel_mask; handle->cfg_ae_mask = ALL_AE_MASK; /* create AE objects */ diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index b280fb0722c5..c089c2709376 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1239,6 +1239,83 @@ auth_fail: return -EINVAL; } +static bool qat_uclo_is_broadcast(struct icp_qat_fw_loader_handle *handle, + int imgid) +{ + struct icp_qat_suof_handle *sobj_handle; + + if (!handle->chip_info->tgroup_share_ustore) + return false; + + sobj_handle = (struct icp_qat_suof_handle *)handle->sobj_handle; + if (handle->hal_handle->admin_ae_mask & + sobj_handle->img_table.simg_hdr[imgid].ae_mask) + return false; + + return true; +} + +static int qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_fw_auth_desc *desc) +{ + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long desc_ae_mask = desc->ae_mask; + u32 fcu_sts, ae_broadcast_mask = 0; + u32 fcu_loaded_csr, ae_loaded; + u32 fcu_sts_csr, fcu_ctl_csr; + unsigned int ae, retry = 0; + + if (handle->chip_info->tgroup_share_ustore) { + fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; + fcu_sts_csr = handle->chip_info->fcu_sts_csr; + fcu_loaded_csr = handle->chip_info->fcu_loaded_ae_csr; + } else { + pr_err("Chip 0x%x doesn't support broadcast load\n", + handle->pci_dev->device); + return -EINVAL; + } + + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + if (qat_hal_check_ae_active(handle, (unsigned char)ae)) { + pr_err("QAT: Broadcast load failed. AE is not enabled or active.\n"); + return -EINVAL; + } + + if (test_bit(ae, &desc_ae_mask)) + ae_broadcast_mask |= 1 << ae; + } + + if (ae_broadcast_mask) { + SET_CAP_CSR(handle, FCU_ME_BROADCAST_MASK_TYPE, + ae_broadcast_mask); + + SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_LOAD); + + do { + msleep(FW_AUTH_WAIT_PERIOD); + fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); + fcu_sts &= FCU_AUTH_STS_MASK; + + if (fcu_sts == FCU_STS_LOAD_FAIL) { + pr_err("Broadcast load failed: 0x%x)\n", fcu_sts); + return -EINVAL; + } else if (fcu_sts == FCU_STS_LOAD_DONE) { + ae_loaded = GET_CAP_CSR(handle, fcu_loaded_csr); + ae_loaded >>= handle->chip_info->fcu_loaded_ae_pos; + + if ((ae_loaded & ae_broadcast_mask) == ae_broadcast_mask) + break; + } + } while (retry++ < FW_AUTH_MAX_RETRY); + + if (retry > FW_AUTH_MAX_RETRY) { + pr_err("QAT: broadcast load failed timeout %d\n", retry); + return -EINVAL; + } + } + return 0; +} + static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle, struct icp_firml_dram_desc *dram_desc, unsigned int size) @@ -1420,7 +1497,9 @@ static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, return -EINVAL; } SET_CAP_CSR(handle, fcu_ctl_csr, - (FCU_CTRL_CMD_LOAD | (i << FCU_CTRL_AE_POS))); + (FCU_CTRL_CMD_LOAD | + (1 << FCU_CTRL_BROADCAST_POS) | + (i << FCU_CTRL_AE_POS))); do { msleep(FW_AUTH_WAIT_PERIOD); @@ -1945,8 +2024,13 @@ static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle) goto wr_err; if (qat_uclo_auth_fw(handle, desc)) goto wr_err; - if (qat_uclo_load_fw(handle, desc)) - goto wr_err; + if (qat_uclo_is_broadcast(handle, i)) { + if (qat_uclo_broadcast_load_fw(handle, desc)) + goto wr_err; + } else { + if (qat_uclo_load_fw(handle, desc)) + goto wr_err; + } qat_uclo_ummap_auth_fw(handle, &desc); } return 0; -- cgit v1.2.3 From 9c0cef2364750c00ab380cc8902dbbc91e230183 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:10 +0800 Subject: crypto: qat - add gen4 firmware loader Add support for the QAT gen4 devices in the firmware loader. Signed-off-by: Jack Xu Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_devices.h | 2 + drivers/crypto/qat/qat_common/icp_qat_hal.h | 12 +++- drivers/crypto/qat/qat_common/icp_qat_uclo.h | 2 + drivers/crypto/qat/qat_common/qat_hal.c | 77 ++++++++++++++++++----- drivers/crypto/qat/qat_common/qat_uclo.c | 2 + 5 files changed, 78 insertions(+), 17 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 996d25565b11..5694422ec66c 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -15,6 +15,8 @@ #define ADF_C62XVF_DEVICE_NAME "c6xxvf" #define ADF_C3XXX_DEVICE_NAME "c3xxx" #define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" +#define ADF_4XXX_PCI_DEVICE_ID 0x4940 +#define ADF_4XXXIOV_PCI_DEVICE_ID 0x4941 #define ADF_ERRSOU3 (0x3A000 + 0x0C) #define ADF_ERRSOU5 (0x3A000 + 0xD8) #define ADF_DEVICE_FUSECTL_OFFSET 0x40 diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index 8372f18ebc80..20b2ee1fc65a 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -10,6 +10,14 @@ enum hal_global_csr { ICP_GLOBAL_CLK_ENABLE = 0xA50 }; +enum { + MISC_CONTROL_C4XXX = 0xAA0, + ICP_RESET_CPP0 = 0x938, + ICP_RESET_CPP1 = 0x93c, + ICP_GLOBAL_CLK_ENABLE_CPP0 = 0x964, + ICP_GLOBAL_CLK_ENABLE_CPP1 = 0x968 +}; + enum hal_ae_csr { USTORE_ADDRESS = 0x000, USTORE_DATA_LOWER = 0x004, @@ -111,7 +119,9 @@ enum fcu_sts { #define ICP_QAT_CAP_OFFSET (ICP_QAT_AE_OFFSET + 0x10000) #define LOCAL_TO_XFER_REG_OFFSET 0x800 #define ICP_QAT_EP_OFFSET 0x3a000 - +#define ICP_QAT_EP_OFFSET_4XXX 0x200000 /* HI MMIO CSRs */ +#define ICP_QAT_AE_OFFSET_4XXX 0x600000 +#define ICP_QAT_CAP_OFFSET_4XXX 0x640000 #define SET_CAP_CSR(handle, csr, val) \ ADF_CSR_WR((handle)->hal_cap_g_ctl_csr_addr_v, csr, val) #define GET_CAP_CSR(handle, csr) \ diff --git a/drivers/crypto/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/qat/qat_common/icp_qat_uclo.h index 0ec8a5ab51b5..4b36869bf460 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_uclo.h +++ b/drivers/crypto/qat/qat_common/icp_qat_uclo.h @@ -6,6 +6,7 @@ #define ICP_QAT_AC_895XCC_DEV_TYPE 0x00400000 #define ICP_QAT_AC_C62X_DEV_TYPE 0x01000000 #define ICP_QAT_AC_C3XXX_DEV_TYPE 0x02000000 +#define ICP_QAT_AC_4XXX_A_DEV_TYPE 0x08000000 #define ICP_QAT_UCLO_MAX_AE 12 #define ICP_QAT_UCLO_MAX_CTX 8 #define ICP_QAT_UCLO_MAX_UIMAGE (ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX) @@ -13,6 +14,7 @@ #define ICP_QAT_UCLO_MAX_XFER_REG 128 #define ICP_QAT_UCLO_MAX_GPR_REG 128 #define ICP_QAT_UCLO_MAX_LMEM_REG 1024 +#define ICP_QAT_UCLO_MAX_LMEM_REG_2X 1280 #define ICP_QAT_UCLO_AE_ALL_CTX 0xff #define ICP_QAT_UOF_OBJID_LEN 8 #define ICP_QAT_UOF_FID 0xc6c2 diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index a3c1f2163910..bd3028126cbe 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -695,6 +695,39 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->pci_dev = pci_info->pci_dev; switch (handle->pci_dev->device) { + case ADF_4XXX_PCI_DEVICE_ID: + handle->chip_info->sram_visible = false; + handle->chip_info->nn = false; + handle->chip_info->lm2lm3 = true; + handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG_2X; + handle->chip_info->icp_rst_csr = ICP_RESET_CPP0; + handle->chip_info->icp_rst_mask = 0x100015; + handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE_CPP0; + handle->chip_info->misc_ctl_csr = MISC_CONTROL_C4XXX; + handle->chip_info->wakeup_event_val = 0x80000000; + handle->chip_info->fw_auth = true; + handle->chip_info->css_3k = true; + handle->chip_info->tgroup_share_ustore = true; + handle->chip_info->fcu_ctl_csr = FCU_CONTROL_4XXX; + handle->chip_info->fcu_sts_csr = FCU_STATUS_4XXX; + handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI_4XXX; + handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO_4XXX; + handle->chip_info->fcu_loaded_ae_csr = FCU_AE_LOADED_4XXX; + handle->chip_info->fcu_loaded_ae_pos = 0; + + handle->hal_cap_g_ctl_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_CAP_OFFSET_4XXX); + handle->hal_cap_ae_xfer_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_AE_OFFSET_4XXX); + handle->hal_ep_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_EP_OFFSET_4XXX); + handle->hal_cap_ae_local_csr_addr_v = + (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + + LOCAL_TO_XFER_REG_OFFSET); + break; case PCI_DEVICE_ID_INTEL_QAT_C62X: case PCI_DEVICE_ID_INTEL_QAT_C3XXX: handle->chip_info->sram_visible = false; @@ -702,6 +735,8 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->lm2lm3 = false; handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->icp_rst_csr = ICP_RESET; + handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) | + (hw_data->accel_mask << RST_CSR_QAT_LSB); handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; handle->chip_info->misc_ctl_csr = MISC_CONTROL; handle->chip_info->wakeup_event_val = WAKEUP_EVENT; @@ -714,6 +749,18 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO; handle->chip_info->fcu_loaded_ae_csr = FCU_STATUS; handle->chip_info->fcu_loaded_ae_pos = FCU_LOADED_AE_POS; + handle->hal_cap_g_ctl_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_CAP_OFFSET); + handle->hal_cap_ae_xfer_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_AE_OFFSET); + handle->hal_ep_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_EP_OFFSET); + handle->hal_cap_ae_local_csr_addr_v = + (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + + LOCAL_TO_XFER_REG_OFFSET); break; case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: handle->chip_info->sram_visible = true; @@ -721,6 +768,8 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->lm2lm3 = false; handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->icp_rst_csr = ICP_RESET; + handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) | + (hw_data->accel_mask << RST_CSR_QAT_LSB); handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; handle->chip_info->misc_ctl_csr = MISC_CONTROL; handle->chip_info->wakeup_event_val = WAKEUP_EVENT; @@ -733,6 +782,18 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->fcu_dram_addr_lo = 0; handle->chip_info->fcu_loaded_ae_csr = 0; handle->chip_info->fcu_loaded_ae_pos = 0; + handle->hal_cap_g_ctl_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_CAP_OFFSET); + handle->hal_cap_ae_xfer_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_AE_OFFSET); + handle->hal_ep_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_EP_OFFSET); + handle->hal_cap_ae_local_csr_addr_v = + (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + + LOCAL_TO_XFER_REG_OFFSET); break; default: ret = -EINVAL; @@ -744,22 +805,6 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; handle->hal_sram_addr_v = sram_bar->virt_addr; } - - handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) | - (hw_data->accel_mask << RST_CSR_QAT_LSB); - handle->hal_cap_g_ctl_csr_addr_v = - (void __iomem *)((uintptr_t)misc_bar->virt_addr + - ICP_QAT_CAP_OFFSET); - handle->hal_cap_ae_xfer_csr_addr_v = - (void __iomem *)((uintptr_t)misc_bar->virt_addr + - ICP_QAT_AE_OFFSET); - handle->hal_ep_csr_addr_v = - (void __iomem *)((uintptr_t)misc_bar->virt_addr + - ICP_QAT_EP_OFFSET); - handle->hal_cap_ae_local_csr_addr_v = - (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + - LOCAL_TO_XFER_REG_OFFSET); - handle->pci_dev = pci_info->pci_dev; handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid; handle->hal_handle->ae_mask = hw_data->ae_mask; handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask; diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index c089c2709376..1fb5fc852f6b 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -728,6 +728,8 @@ qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) return ICP_QAT_AC_C62X_DEV_TYPE; case PCI_DEVICE_ID_INTEL_QAT_C3XXX: return ICP_QAT_AC_C3XXX_DEV_TYPE; + case ADF_4XXX_PCI_DEVICE_ID: + return ICP_QAT_AC_4XXX_A_DEV_TYPE; default: pr_err("QAT: unsupported device 0x%x\n", handle->pci_dev->device); -- cgit v1.2.3 From 5761498c4d074560b38a203000afc6e971ffaa7f Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Mon, 9 Nov 2020 17:00:27 +0800 Subject: crypto: hisilicon/sec2 - Fix aead authentication setting key error Fix aead auth setting key process error. if use soft shash function, driver need to use digest size replace of the user input key length. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec2/sec_crypto.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 87bc08afe567..891e04914615 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -857,7 +857,7 @@ static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, struct crypto_authenc_keys *keys) { struct crypto_shash *hash_tfm = ctx->hash_tfm; - int blocksize, ret; + int blocksize, digestsize, ret; if (!keys->authkeylen) { pr_err("hisi_sec2: aead auth key error!\n"); @@ -865,6 +865,7 @@ static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, } blocksize = crypto_shash_blocksize(hash_tfm); + digestsize = crypto_shash_digestsize(hash_tfm); if (keys->authkeylen > blocksize) { ret = crypto_shash_tfm_digest(hash_tfm, keys->authkey, keys->authkeylen, ctx->a_key); @@ -872,7 +873,7 @@ static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, pr_err("hisi_sec2: aead auth digest error!\n"); return -EINVAL; } - ctx->a_key_len = blocksize; + ctx->a_key_len = digestsize; } else { memcpy(ctx->a_key, keys->authkey, keys->authkeylen); ctx->a_key_len = keys->authkeylen; -- cgit v1.2.3 From 0049a1329bb9569196a551bb658500394d98ed50 Mon Sep 17 00:00:00 2001 From: Horia Geantă Date: Thu, 12 Nov 2020 11:21:46 +0200 Subject: crypto: caam/qi - simplify error path for context allocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wang Qing reports that IS_ERR_OR_NULL() should be matched with PTR_ERR_OR_ZERO(), not PTR_ERR(). As it turns out, the error path always returns an error code, i.e. NULL is never returned. Update the code accordingly - s/IS_ERR_OR_NULL/IS_ERR. Reported-by: Wang Qing Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg_qi.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index a24ae966df4a..189a7438b29c 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -852,7 +852,7 @@ static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx, cpu = smp_processor_id(); drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc); - if (!IS_ERR_OR_NULL(drv_ctx)) + if (!IS_ERR(drv_ctx)) drv_ctx->op_type = type; ctx->drv_ctx[type] = drv_ctx; @@ -955,7 +955,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, struct caam_drv_ctx *drv_ctx; drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT); - if (IS_ERR_OR_NULL(drv_ctx)) + if (IS_ERR(drv_ctx)) return (struct aead_edesc *)drv_ctx; /* allocate space for base edesc and hw desc commands, link tables */ @@ -1165,7 +1165,7 @@ static inline int aead_crypt(struct aead_request *req, bool encrypt) /* allocate extended descriptor */ edesc = aead_edesc_alloc(req, encrypt); - if (IS_ERR_OR_NULL(edesc)) + if (IS_ERR(edesc)) return PTR_ERR(edesc); /* Create and submit job descriptor */ @@ -1259,7 +1259,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, struct caam_drv_ctx *drv_ctx; drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT); - if (IS_ERR_OR_NULL(drv_ctx)) + if (IS_ERR(drv_ctx)) return (struct skcipher_edesc *)drv_ctx; src_nents = sg_nents_for_len(req->src, req->cryptlen); -- cgit v1.2.3 From 5bdad829c31a09069fd508534f03c2ea1576ac75 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Thu, 12 Nov 2020 13:07:02 -0700 Subject: crypto: crypto4xx - Replace bitwise OR with logical OR in crypto4xx_build_pd Clang warns: drivers/crypto/amcc/crypto4xx_core.c:921:60: warning: operator '?:' has lower precedence than '|'; '|' will be evaluated first [-Wbitwise-conditional-parentheses] (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ^ drivers/crypto/amcc/crypto4xx_core.c:921:60: note: place parentheses around the '|' expression to silence this warning (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ? ^ ) drivers/crypto/amcc/crypto4xx_core.c:921:60: note: place parentheses around the '?:' expression to evaluate it first (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ? ^ ( 1 warning generated. It looks like this should have been a logical OR so that PD_CTL_HASH_FINAL gets added to the w bitmask if crypto_tfm_alg_type is either CRYPTO_ALG_TYPE_AHASH or CRYPTO_ALG_TYPE_AEAD. Change the operator so that everything works properly. Fixes: 4b5b79998af6 ("crypto: crypto4xx - fix stalls under heavy load") Link: https://github.com/ClangBuiltLinux/linux/issues/1198 Signed-off-by: Nathan Chancellor Reviewed-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 981de43ea5e2..2e3690f65786 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -917,7 +917,7 @@ int crypto4xx_build_pd(struct crypto_async_request *req, } pd->pd_ctl.w = PD_CTL_HOST_READY | - ((crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AHASH) | + ((crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AHASH) || (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ? PD_CTL_HASH_FINAL : 0); pd->pd_ctl_len.w = 0x00400000 | (assoclen + datalen); -- cgit v1.2.3 From a24d22b225ce158651378869a6b88105c4bdb887 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 12 Nov 2020 21:20:21 -0800 Subject: crypto: sha - split sha.h into sha1.h and sha2.h Currently contains declarations for both SHA-1 and SHA-2, and contains declarations for SHA-3. This organization is inconsistent, but more importantly SHA-1 is no longer considered to be cryptographically secure. So to the extent possible, SHA-1 shouldn't be grouped together with any of the other SHA versions, and usage of it should be phased out. Therefore, split into two headers and , and make everyone explicitly specify whether they want the declarations for SHA-1, SHA-2, or both. This avoids making the SHA-1 declarations visible to files that don't want anything to do with SHA-1. It also prepares for potentially moving sha1.h into a new insecure/ or dangerous/ directory. Signed-off-by: Eric Biggers Acked-by: Ard Biesheuvel Acked-by: Jason A. Donenfeld Signed-off-by: Herbert Xu --- arch/arm/crypto/sha1-ce-glue.c | 2 +- arch/arm/crypto/sha1.h | 2 +- arch/arm/crypto/sha1_glue.c | 2 +- arch/arm/crypto/sha1_neon_glue.c | 2 +- arch/arm/crypto/sha2-ce-glue.c | 2 +- arch/arm/crypto/sha256_glue.c | 2 +- arch/arm/crypto/sha256_neon_glue.c | 2 +- arch/arm/crypto/sha512-glue.c | 2 +- arch/arm/crypto/sha512-neon-glue.c | 2 +- arch/arm64/crypto/aes-glue.c | 2 +- arch/arm64/crypto/sha1-ce-glue.c | 2 +- arch/arm64/crypto/sha2-ce-glue.c | 2 +- arch/arm64/crypto/sha256-glue.c | 2 +- arch/arm64/crypto/sha512-ce-glue.c | 2 +- arch/arm64/crypto/sha512-glue.c | 2 +- arch/mips/cavium-octeon/crypto/octeon-sha1.c | 2 +- arch/mips/cavium-octeon/crypto/octeon-sha256.c | 2 +- arch/mips/cavium-octeon/crypto/octeon-sha512.c | 2 +- arch/powerpc/crypto/sha1-spe-glue.c | 2 +- arch/powerpc/crypto/sha1.c | 2 +- arch/powerpc/crypto/sha256-spe-glue.c | 2 +- arch/s390/crypto/sha.h | 3 +- arch/s390/crypto/sha1_s390.c | 2 +- arch/s390/crypto/sha256_s390.c | 2 +- arch/s390/crypto/sha3_256_s390.c | 1 - arch/s390/crypto/sha3_512_s390.c | 1 - arch/s390/crypto/sha512_s390.c | 2 +- arch/s390/purgatory/purgatory.c | 2 +- arch/sparc/crypto/sha1_glue.c | 2 +- arch/sparc/crypto/sha256_glue.c | 2 +- arch/sparc/crypto/sha512_glue.c | 2 +- arch/x86/crypto/sha1_ssse3_glue.c | 2 +- arch/x86/crypto/sha256_ssse3_glue.c | 2 +- arch/x86/crypto/sha512_ssse3_glue.c | 2 +- arch/x86/purgatory/purgatory.c | 2 +- crypto/asymmetric_keys/asym_tpm.c | 2 +- crypto/sha1_generic.c | 2 +- crypto/sha256_generic.c | 2 +- crypto/sha512_generic.c | 2 +- drivers/char/random.c | 2 +- drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h | 2 +- drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c | 3 +- drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h | 3 +- drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c | 3 +- drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h | 3 +- drivers/crypto/amcc/crypto4xx_alg.c | 2 +- drivers/crypto/amcc/crypto4xx_core.c | 2 +- drivers/crypto/atmel-authenc.h | 3 +- drivers/crypto/atmel-sha.c | 3 +- drivers/crypto/axis/artpec6_crypto.c | 3 +- drivers/crypto/bcm/cipher.c | 3 +- drivers/crypto/bcm/cipher.h | 3 +- drivers/crypto/bcm/spu.h | 3 +- drivers/crypto/caam/compat.h | 3 +- drivers/crypto/cavium/nitrox/nitrox_aead.c | 1 - drivers/crypto/ccp/ccp-crypto-sha.c | 3 +- drivers/crypto/ccp/ccp-crypto.h | 3 +- drivers/crypto/ccree/cc_driver.h | 3 +- drivers/crypto/chelsio/chcr_algo.c | 3 +- drivers/crypto/hisilicon/sec2/sec_crypto.c | 3 +- drivers/crypto/img-hash.c | 3 +- drivers/crypto/inside-secure/safexcel.h | 3 +- drivers/crypto/inside-secure/safexcel_cipher.c | 3 +- drivers/crypto/inside-secure/safexcel_hash.c | 3 +- drivers/crypto/ixp4xx_crypto.c | 2 +- drivers/crypto/marvell/cesa/hash.c | 3 +- drivers/crypto/marvell/octeontx/otx_cptvf_algs.c | 3 +- drivers/crypto/mediatek/mtk-sha.c | 3 +- drivers/crypto/mxs-dcp.c | 3 +- drivers/crypto/n2_core.c | 3 +- drivers/crypto/nx/nx-sha256.c | 2 +- drivers/crypto/nx/nx-sha512.c | 2 +- drivers/crypto/nx/nx.c | 2 +- drivers/crypto/omap-sham.c | 3 +- drivers/crypto/padlock-sha.c | 3 +- drivers/crypto/picoxcell_crypto.c | 3 +- drivers/crypto/qat/qat_common/qat_algs.c | 3 +- drivers/crypto/qce/common.c | 3 +- drivers/crypto/qce/core.c | 1 - drivers/crypto/qce/sha.h | 3 +- drivers/crypto/rockchip/rk3288_crypto.h | 3 +- drivers/crypto/s5p-sss.c | 3 +- drivers/crypto/sa2ul.c | 3 +- drivers/crypto/sa2ul.h | 3 +- drivers/crypto/sahara.c | 3 +- drivers/crypto/stm32/stm32-hash.c | 3 +- drivers/crypto/talitos.c | 3 +- drivers/crypto/ux500/hash/hash_core.c | 3 +- drivers/firmware/efi/embedded-firmware.c | 2 +- .../chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c | 3 +- .../ethernet/chelsio/inline_crypto/chtls/chtls.h | 3 +- drivers/nfc/s3fwrn5/firmware.c | 2 +- drivers/tee/tee_core.c | 2 +- fs/crypto/fname.c | 2 +- fs/crypto/hkdf.c | 2 +- fs/ubifs/auth.c | 1 - fs/verity/fsverity_private.h | 2 +- include/crypto/hash_info.h | 3 +- include/crypto/sha.h | 167 --------------------- include/crypto/sha1.h | 46 ++++++ include/crypto/sha1_base.h | 2 +- include/crypto/sha2.h | 134 +++++++++++++++++ include/crypto/sha256_base.h | 2 +- include/crypto/sha512_base.h | 2 +- include/linux/ccp.h | 3 +- include/linux/filter.h | 2 +- include/linux/purgatory.h | 2 +- kernel/crash_core.c | 2 +- kernel/kexec_core.c | 1 - kernel/kexec_file.c | 2 +- lib/crypto/sha256.c | 2 +- lib/digsig.c | 2 +- lib/sha1.c | 2 +- net/ipv6/seg6_hmac.c | 1 - net/mptcp/crypto.c | 2 +- net/mptcp/options.c | 2 +- net/mptcp/subflow.c | 2 +- security/integrity/integrity.h | 2 +- security/keys/encrypted-keys/encrypted.c | 2 +- security/keys/trusted-keys/trusted_tpm1.c | 2 +- sound/soc/codecs/cros_ec_codec.c | 2 +- 121 files changed, 335 insertions(+), 285 deletions(-) delete mode 100644 include/crypto/sha.h create mode 100644 include/crypto/sha1.h create mode 100644 include/crypto/sha2.h (limited to 'drivers/crypto') diff --git a/arch/arm/crypto/sha1-ce-glue.c b/arch/arm/crypto/sha1-ce-glue.c index e79b1fb4b4dc..de9100c67b37 100644 --- a/arch/arm/crypto/sha1-ce-glue.c +++ b/arch/arm/crypto/sha1-ce-glue.c @@ -7,7 +7,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha1.h b/arch/arm/crypto/sha1.h index 758db3e9ff0a..b1b7e21da2c3 100644 --- a/arch/arm/crypto/sha1.h +++ b/arch/arm/crypto/sha1.h @@ -3,7 +3,7 @@ #define ASM_ARM_CRYPTO_SHA1_H #include -#include +#include extern int sha1_update_arm(struct shash_desc *desc, const u8 *data, unsigned int len); diff --git a/arch/arm/crypto/sha1_glue.c b/arch/arm/crypto/sha1_glue.c index 4e954b3f7ecd..6c2b849e459d 100644 --- a/arch/arm/crypto/sha1_glue.c +++ b/arch/arm/crypto/sha1_glue.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/arm/crypto/sha1_neon_glue.c b/arch/arm/crypto/sha1_neon_glue.c index 0071e5e4411a..cfe36ae0f3f5 100644 --- a/arch/arm/crypto/sha1_neon_glue.c +++ b/arch/arm/crypto/sha1_neon_glue.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha2-ce-glue.c b/arch/arm/crypto/sha2-ce-glue.c index 87f0b62386c6..c62ce89dd3e0 100644 --- a/arch/arm/crypto/sha2-ce-glue.c +++ b/arch/arm/crypto/sha2-ce-glue.c @@ -7,7 +7,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha256_glue.c b/arch/arm/crypto/sha256_glue.c index b8a4f79020cf..433ee4ddce6c 100644 --- a/arch/arm/crypto/sha256_glue.c +++ b/arch/arm/crypto/sha256_glue.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha256_neon_glue.c b/arch/arm/crypto/sha256_neon_glue.c index 79820b9e2541..701706262ef3 100644 --- a/arch/arm/crypto/sha256_neon_glue.c +++ b/arch/arm/crypto/sha256_neon_glue.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha512-glue.c b/arch/arm/crypto/sha512-glue.c index 8775aa42bbbe..0635a65aa488 100644 --- a/arch/arm/crypto/sha512-glue.c +++ b/arch/arm/crypto/sha512-glue.c @@ -6,7 +6,7 @@ */ #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha512-neon-glue.c b/arch/arm/crypto/sha512-neon-glue.c index 96cb94403540..c879ad32db51 100644 --- a/arch/arm/crypto/sha512-neon-glue.c +++ b/arch/arm/crypto/sha512-neon-glue.c @@ -7,7 +7,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index 395bbf64b2ab..34b8a89197be 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c index c63b99211db3..c93121bcfdeb 100644 --- a/arch/arm64/crypto/sha1-ce-glue.c +++ b/arch/arm64/crypto/sha1-ce-glue.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c index 5e956d7582a5..31ba3da5e61b 100644 --- a/arch/arm64/crypto/sha2-ce-glue.c +++ b/arch/arm64/crypto/sha2-ce-glue.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/sha256-glue.c b/arch/arm64/crypto/sha256-glue.c index 77bc6e72abae..9462f6088b3f 100644 --- a/arch/arm64/crypto/sha256-glue.c +++ b/arch/arm64/crypto/sha256-glue.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/sha512-ce-glue.c b/arch/arm64/crypto/sha512-ce-glue.c index dc890a719f54..faa83f6cf376 100644 --- a/arch/arm64/crypto/sha512-ce-glue.c +++ b/arch/arm64/crypto/sha512-ce-glue.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/sha512-glue.c b/arch/arm64/crypto/sha512-glue.c index 370ccb29602f..2acff1c7df5d 100644 --- a/arch/arm64/crypto/sha512-glue.c +++ b/arch/arm64/crypto/sha512-glue.c @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha1.c b/arch/mips/cavium-octeon/crypto/octeon-sha1.c index 75e79b47abfe..30f1d75208a5 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha1.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha1.c @@ -14,7 +14,7 @@ */ #include -#include +#include #include #include #include diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha256.c b/arch/mips/cavium-octeon/crypto/octeon-sha256.c index a682ce76716a..36cb92895d72 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha256.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha256.c @@ -15,7 +15,7 @@ */ #include -#include +#include #include #include #include diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha512.c b/arch/mips/cavium-octeon/crypto/octeon-sha512.c index 50722a0cfb53..359f039820d8 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha512.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha512.c @@ -14,7 +14,7 @@ */ #include -#include +#include #include #include #include diff --git a/arch/powerpc/crypto/sha1-spe-glue.c b/arch/powerpc/crypto/sha1-spe-glue.c index cb57be4ada61..b1e577cbf00c 100644 --- a/arch/powerpc/crypto/sha1-spe-glue.c +++ b/arch/powerpc/crypto/sha1-spe-glue.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/crypto/sha1.c b/arch/powerpc/crypto/sha1.c index b40dc50a6908..7a55d790cdb1 100644 --- a/arch/powerpc/crypto/sha1.c +++ b/arch/powerpc/crypto/sha1.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include void powerpc_sha_transform(u32 *state, const u8 *src); diff --git a/arch/powerpc/crypto/sha256-spe-glue.c b/arch/powerpc/crypto/sha256-spe-glue.c index ceb0b6c980b3..88530ae0791f 100644 --- a/arch/powerpc/crypto/sha256-spe-glue.c +++ b/arch/powerpc/crypto/sha256-spe-glue.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/s390/crypto/sha.h b/arch/s390/crypto/sha.h index ada2f98c27b7..65ea12fc87a1 100644 --- a/arch/s390/crypto/sha.h +++ b/arch/s390/crypto/sha.h @@ -11,7 +11,8 @@ #define _CRYPTO_ARCH_S390_SHA_H #include -#include +#include +#include #include /* must be big enough for the largest SHA variant */ diff --git a/arch/s390/crypto/sha1_s390.c b/arch/s390/crypto/sha1_s390.c index 698b1e6d3c14..a3fabf310a38 100644 --- a/arch/s390/crypto/sha1_s390.c +++ b/arch/s390/crypto/sha1_s390.c @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include "sha.h" diff --git a/arch/s390/crypto/sha256_s390.c b/arch/s390/crypto/sha256_s390.c index b52c87e44939..24983f175676 100644 --- a/arch/s390/crypto/sha256_s390.c +++ b/arch/s390/crypto/sha256_s390.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include "sha.h" diff --git a/arch/s390/crypto/sha3_256_s390.c b/arch/s390/crypto/sha3_256_s390.c index 460cbbbaa44a..30ac49b635bf 100644 --- a/arch/s390/crypto/sha3_256_s390.c +++ b/arch/s390/crypto/sha3_256_s390.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include diff --git a/arch/s390/crypto/sha3_512_s390.c b/arch/s390/crypto/sha3_512_s390.c index 72cf460a53e5..e70d50f7620f 100644 --- a/arch/s390/crypto/sha3_512_s390.c +++ b/arch/s390/crypto/sha3_512_s390.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c index ad29db085a18..29a6bd404c59 100644 --- a/arch/s390/crypto/sha512_s390.c +++ b/arch/s390/crypto/sha512_s390.c @@ -8,7 +8,7 @@ * Author(s): Jan Glauber (jang@de.ibm.com) */ #include -#include +#include #include #include #include diff --git a/arch/s390/purgatory/purgatory.c b/arch/s390/purgatory/purgatory.c index 0a423bcf6746..030efda05dbe 100644 --- a/arch/s390/purgatory/purgatory.c +++ b/arch/s390/purgatory/purgatory.c @@ -9,7 +9,7 @@ #include #include -#include +#include #include int verify_sha256_digest(void) diff --git a/arch/sparc/crypto/sha1_glue.c b/arch/sparc/crypto/sha1_glue.c index dc017782be52..86a654cce5ab 100644 --- a/arch/sparc/crypto/sha1_glue.c +++ b/arch/sparc/crypto/sha1_glue.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/sparc/crypto/sha256_glue.c b/arch/sparc/crypto/sha256_glue.c index ca2547df9652..60ec524cf9ca 100644 --- a/arch/sparc/crypto/sha256_glue.c +++ b/arch/sparc/crypto/sha256_glue.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/sparc/crypto/sha512_glue.c b/arch/sparc/crypto/sha512_glue.c index 3b2ca732ff7a..273ce21918c1 100644 --- a/arch/sparc/crypto/sha512_glue.c +++ b/arch/sparc/crypto/sha512_glue.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c index 18200135603f..44340a1139e0 100644 --- a/arch/x86/crypto/sha1_ssse3_glue.c +++ b/arch/x86/crypto/sha1_ssse3_glue.c @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/x86/crypto/sha256_ssse3_glue.c b/arch/x86/crypto/sha256_ssse3_glue.c index dd06249229e1..3a5f6be7dbba 100644 --- a/arch/x86/crypto/sha256_ssse3_glue.c +++ b/arch/x86/crypto/sha256_ssse3_glue.c @@ -35,7 +35,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/crypto/sha512_ssse3_glue.c b/arch/x86/crypto/sha512_ssse3_glue.c index b0b05c93409e..30e70f4fe2f7 100644 --- a/arch/x86/crypto/sha512_ssse3_glue.c +++ b/arch/x86/crypto/sha512_ssse3_glue.c @@ -34,7 +34,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/x86/purgatory/purgatory.c b/arch/x86/purgatory/purgatory.c index 7b37a412f829..f03b64d9cb51 100644 --- a/arch/x86/purgatory/purgatory.c +++ b/arch/x86/purgatory/purgatory.c @@ -9,7 +9,7 @@ */ #include -#include +#include #include #include "../boot/string.h" diff --git a/crypto/asymmetric_keys/asym_tpm.c b/crypto/asymmetric_keys/asym_tpm.c index 378b18b9bc34..511932aa94a6 100644 --- a/crypto/asymmetric_keys/asym_tpm.c +++ b/crypto/asymmetric_keys/asym_tpm.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/crypto/sha1_generic.c b/crypto/sha1_generic.c index 1d43472fecbd..325b57fe28dc 100644 --- a/crypto/sha1_generic.c +++ b/crypto/sha1_generic.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include diff --git a/crypto/sha256_generic.c b/crypto/sha256_generic.c index 88156e3e2a33..3b377197236e 100644 --- a/crypto/sha256_generic.c +++ b/crypto/sha256_generic.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c index e34d09dd9971..c72d72ad828e 100644 --- a/crypto/sha512_generic.c +++ b/crypto/sha512_generic.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/char/random.c b/drivers/char/random.c index 2a41b21623ae..5f3b8ac9d97b 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -336,7 +336,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h index 163962f9e284..5c291e4a6857 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c index fa2f1b4fad7b..4927a6c82d32 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -13,7 +13,8 @@ #include #include #include -#include +#include +#include #include #include "sun8i-ce.h" diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h index 558027516aed..cec781d5063c 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h @@ -16,7 +16,8 @@ #include #include #include -#include +#include +#include /* CE Registers */ #define CE_TDQ 0x00 diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c index b6ab2054f217..11cbcbc83a7b 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c @@ -13,7 +13,8 @@ #include #include #include -#include +#include +#include #include #include "sun8i-ss.h" diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h index 1a66457f4a20..28188685b910 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h @@ -15,7 +15,8 @@ #include #include #include -#include +#include +#include #define SS_START 1 diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index 7729a637fb02..a3fa849b139a 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include "crypto4xx_reg_def.h" diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 2e3690f65786..8d1b918a0533 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/crypto/atmel-authenc.h b/drivers/crypto/atmel-authenc.h index c6530a1c8c20..45171e89a7d2 100644 --- a/drivers/crypto/atmel-authenc.h +++ b/drivers/crypto/atmel-authenc.h @@ -16,7 +16,8 @@ #include #include -#include +#include +#include #include "atmel-sha-regs.h" struct atmel_aes_dev; diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index 0eb6f54e3b66..352d80cb5ae9 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -33,7 +33,8 @@ #include #include #include -#include +#include +#include #include #include #include "atmel-sha-regs.h" diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c index 809c3033ca74..9ad188cffd0d 100644 --- a/drivers/crypto/axis/artpec6_crypto.c +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -28,7 +28,8 @@ #include #include #include -#include +#include +#include #include /* Max length of a line in all cache levels for Artpec SoCs. */ diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 50d169e61b41..30390a7324b2 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -26,11 +26,12 @@ #include #include #include -#include #include #include #include #include +#include +#include #include #include "util.h" diff --git a/drivers/crypto/bcm/cipher.h b/drivers/crypto/bcm/cipher.h index 035c8389cb3d..0ad5892b445d 100644 --- a/drivers/crypto/bcm/cipher.h +++ b/drivers/crypto/bcm/cipher.h @@ -16,7 +16,8 @@ #include #include #include -#include +#include +#include #include #include "spu.h" diff --git a/drivers/crypto/bcm/spu.h b/drivers/crypto/bcm/spu.h index dd132389bcaa..1c386a2d5506 100644 --- a/drivers/crypto/bcm/spu.h +++ b/drivers/crypto/bcm/spu.h @@ -17,7 +17,8 @@ #include #include -#include +#include +#include enum spu_cipher_alg { CIPHER_ALG_NONE = 0x0, diff --git a/drivers/crypto/caam/compat.h b/drivers/crypto/caam/compat.h index c3c22a8de4c0..c4f79764172b 100644 --- a/drivers/crypto/caam/compat.h +++ b/drivers/crypto/caam/compat.h @@ -34,7 +34,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/cavium/nitrox/nitrox_aead.c b/drivers/crypto/cavium/nitrox/nitrox_aead.c index e5d8607ecb1d..c93c4e41d267 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_aead.c +++ b/drivers/crypto/cavium/nitrox/nitrox_aead.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c index 8fbfdb9e8cd3..74fa5360e722 100644 --- a/drivers/crypto/ccp/ccp-crypto-sha.c +++ b/drivers/crypto/ccp/ccp-crypto-sha.c @@ -17,7 +17,8 @@ #include #include #include -#include +#include +#include #include #include diff --git a/drivers/crypto/ccp/ccp-crypto.h b/drivers/crypto/ccp/ccp-crypto.h index aed3d2192d01..e42450d07168 100644 --- a/drivers/crypto/ccp/ccp-crypto.h +++ b/drivers/crypto/ccp/ccp-crypto.h @@ -19,7 +19,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h index af77b2020350..ed2b2f13a256 100644 --- a/drivers/crypto/ccree/cc_driver.h +++ b/drivers/crypto/ccree/cc_driver.h @@ -17,7 +17,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 13b908ea4873..f5a336634daa 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -53,7 +53,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 891e04914615..2eaa516b3231 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -7,7 +7,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c index 91f555ccbb31..e813115d5432 100644 --- a/drivers/crypto/img-hash.c +++ b/drivers/crypto/img-hash.c @@ -19,7 +19,8 @@ #include #include -#include +#include +#include #define CR_RESET 0 #define CR_RESET_SET 1 diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 9045f2d7f4c6..ce1e611a163e 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -11,7 +11,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 9bcfb79a030f..d68ef16650d4 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -18,7 +18,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 56d5ccb5cc00..50fb6d90a2e0 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -8,7 +8,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index 276012e7c482..8b0f17fc09fb 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/crypto/marvell/cesa/hash.c b/drivers/crypto/marvell/cesa/hash.c index add7ea011c98..8cf9fd518d86 100644 --- a/drivers/crypto/marvell/cesa/hash.c +++ b/drivers/crypto/marvell/cesa/hash.c @@ -11,7 +11,8 @@ #include #include -#include +#include +#include #include #include diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c index 90bb31329d4b..ccbef01888d4 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c @@ -13,7 +13,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c index 3d5d7d68b03b..f55aacdafbef 100644 --- a/drivers/crypto/mediatek/mtk-sha.c +++ b/drivers/crypto/mediatek/mtk-sha.c @@ -10,7 +10,8 @@ */ #include -#include +#include +#include #include "mtk-platform.h" #define SHA_ALIGN_MSK (sizeof(u32) - 1) diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c index 909a7eb748e3..d6a7784d2988 100644 --- a/drivers/crypto/mxs-dcp.c +++ b/drivers/crypto/mxs-dcp.c @@ -17,7 +17,8 @@ #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index 3642bf83d809..3b0bf6fea491 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -15,7 +15,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/nx/nx-sha256.c b/drivers/crypto/nx/nx-sha256.c index 02fb53453195..90d9a37a57f6 100644 --- a/drivers/crypto/nx/nx-sha256.c +++ b/drivers/crypto/nx/nx-sha256.c @@ -8,7 +8,7 @@ */ #include -#include +#include #include #include #include diff --git a/drivers/crypto/nx/nx-sha512.c b/drivers/crypto/nx/nx-sha512.c index 4c7a3e3eeebf..eb8627a0f317 100644 --- a/drivers/crypto/nx/nx-sha512.c +++ b/drivers/crypto/nx/nx-sha512.c @@ -8,7 +8,7 @@ */ #include -#include +#include #include #include diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index 40882d6d52c1..0d2dc5be7f19 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index a3b38d2c92e7..ae0d320d3c60 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -35,7 +35,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c index a697a4a3f2d0..6865c7f1fc1a 100644 --- a/drivers/crypto/padlock-sha.c +++ b/drivers/crypto/padlock-sha.c @@ -9,7 +9,8 @@ #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c index fb34bf92861d..84f9c16d984c 100644 --- a/drivers/crypto/picoxcell_crypto.c +++ b/drivers/crypto/picoxcell_crypto.c @@ -8,7 +8,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 0fab8bb8ca59..b3a68d986417 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -6,7 +6,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/qce/common.c b/drivers/crypto/qce/common.c index 5006e74c40cd..a73db2a5637f 100644 --- a/drivers/crypto/qce/common.c +++ b/drivers/crypto/qce/common.c @@ -7,7 +7,8 @@ #include #include #include -#include +#include +#include #include "cipher.h" #include "common.h" diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c index ea616b7259ae..5e6717f9bbda 100644 --- a/drivers/crypto/qce/core.c +++ b/drivers/crypto/qce/core.c @@ -13,7 +13,6 @@ #include #include #include -#include #include "core.h" #include "cipher.h" diff --git a/drivers/crypto/qce/sha.h b/drivers/crypto/qce/sha.h index d63526e3804d..a22695361f16 100644 --- a/drivers/crypto/qce/sha.h +++ b/drivers/crypto/qce/sha.h @@ -7,7 +7,8 @@ #define _SHA_H_ #include -#include +#include +#include #include "common.h" #include "core.h" diff --git a/drivers/crypto/rockchip/rk3288_crypto.h b/drivers/crypto/rockchip/rk3288_crypto.h index 3db595570c9c..97278c2574ff 100644 --- a/drivers/crypto/rockchip/rk3288_crypto.h +++ b/drivers/crypto/rockchip/rk3288_crypto.h @@ -12,7 +12,8 @@ #include #include -#include +#include +#include #define _SBF(v, f) ((v) << (f)) diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 88a6c853ffd7..682c8a450a57 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -30,7 +30,8 @@ #include #include -#include +#include +#include #include #define _SBF(s, v) ((v) << (s)) diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index c357010a159e..f300b0a5958a 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -25,7 +25,8 @@ #include #include #include -#include +#include +#include #include "sa2ul.h" diff --git a/drivers/crypto/sa2ul.h b/drivers/crypto/sa2ul.h index bb40df3876e5..f597ddecde34 100644 --- a/drivers/crypto/sa2ul.h +++ b/drivers/crypto/sa2ul.h @@ -13,7 +13,8 @@ #define _K3_SA2UL_ #include -#include +#include +#include #define SA_ENGINE_ENABLE_CONTROL 0x1000 diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index d60679c79822..8b5be29cb4dc 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -15,7 +15,8 @@ #include #include #include -#include +#include +#include #include #include diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c index e3e25278a970..7ac0573ef663 100644 --- a/drivers/crypto/stm32/stm32-hash.c +++ b/drivers/crypto/stm32/stm32-hash.c @@ -25,7 +25,8 @@ #include #include #include -#include +#include +#include #include #define HASH_CR 0x00 diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index a713a35dc502..4fd85f31630a 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -31,7 +31,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/ux500/hash/hash_core.c b/drivers/crypto/ux500/hash/hash_core.c index 3d407eebb2ba..da284b0ea1b2 100644 --- a/drivers/crypto/ux500/hash/hash_core.c +++ b/drivers/crypto/ux500/hash/hash_core.c @@ -31,7 +31,8 @@ #include #include -#include +#include +#include #include #include diff --git a/drivers/firmware/efi/embedded-firmware.c b/drivers/firmware/efi/embedded-firmware.c index 21ae0c48232a..f5be8e22305b 100644 --- a/drivers/firmware/efi/embedded-firmware.c +++ b/drivers/firmware/efi/embedded-firmware.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include /* Exported for use by lib/test_firmware.c only */ LIST_HEAD(efi_embedded_fw_list); diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c index 072299b14b8d..47d9268a7e3c 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c @@ -51,7 +51,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h index 2d3dfdd2a716..65617752c630 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h @@ -9,7 +9,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/nfc/s3fwrn5/firmware.c b/drivers/nfc/s3fwrn5/firmware.c index ec930ee2c847..5d5ad8307211 100644 --- a/drivers/nfc/s3fwrn5/firmware.c +++ b/drivers/nfc/s3fwrn5/firmware.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include "s3fwrn5.h" #include "firmware.h" diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c index f53bf336c0a2..d70d4be91096 100644 --- a/drivers/tee/tee_core.c +++ b/drivers/tee/tee_core.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include "tee_private.h" #define TEE_NUM_DEVICES 32 diff --git a/fs/crypto/fname.c b/fs/crypto/fname.c index 1fbe6c24d705..cf06ea3870eb 100644 --- a/fs/crypto/fname.c +++ b/fs/crypto/fname.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include "fscrypt_private.h" diff --git a/fs/crypto/hkdf.c b/fs/crypto/hkdf.c index 0cba7928446d..e0ec21055505 100644 --- a/fs/crypto/hkdf.c +++ b/fs/crypto/hkdf.c @@ -10,7 +10,7 @@ */ #include -#include +#include #include "fscrypt_private.h" diff --git a/fs/ubifs/auth.c b/fs/ubifs/auth.c index b93b3cd10bfd..0886d835f597 100644 --- a/fs/ubifs/auth.c +++ b/fs/ubifs/auth.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h index e96d99d5145e..6a8f2e3cce6c 100644 --- a/fs/verity/fsverity_private.h +++ b/fs/verity/fsverity_private.h @@ -14,7 +14,7 @@ #define pr_fmt(fmt) "fs-verity: " fmt -#include +#include #include #include diff --git a/include/crypto/hash_info.h b/include/crypto/hash_info.h index eb9d2e368969..dd4f06785049 100644 --- a/include/crypto/hash_info.h +++ b/include/crypto/hash_info.h @@ -8,7 +8,8 @@ #ifndef _CRYPTO_HASH_INFO_H #define _CRYPTO_HASH_INFO_H -#include +#include +#include #include #include diff --git a/include/crypto/sha.h b/include/crypto/sha.h deleted file mode 100644 index 4ff3da816630..000000000000 --- a/include/crypto/sha.h +++ /dev/null @@ -1,167 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Common values for SHA algorithms - */ - -#ifndef _CRYPTO_SHA_H -#define _CRYPTO_SHA_H - -#include - -#define SHA1_DIGEST_SIZE 20 -#define SHA1_BLOCK_SIZE 64 - -#define SHA224_DIGEST_SIZE 28 -#define SHA224_BLOCK_SIZE 64 - -#define SHA256_DIGEST_SIZE 32 -#define SHA256_BLOCK_SIZE 64 - -#define SHA384_DIGEST_SIZE 48 -#define SHA384_BLOCK_SIZE 128 - -#define SHA512_DIGEST_SIZE 64 -#define SHA512_BLOCK_SIZE 128 - -#define SHA1_H0 0x67452301UL -#define SHA1_H1 0xefcdab89UL -#define SHA1_H2 0x98badcfeUL -#define SHA1_H3 0x10325476UL -#define SHA1_H4 0xc3d2e1f0UL - -#define SHA224_H0 0xc1059ed8UL -#define SHA224_H1 0x367cd507UL -#define SHA224_H2 0x3070dd17UL -#define SHA224_H3 0xf70e5939UL -#define SHA224_H4 0xffc00b31UL -#define SHA224_H5 0x68581511UL -#define SHA224_H6 0x64f98fa7UL -#define SHA224_H7 0xbefa4fa4UL - -#define SHA256_H0 0x6a09e667UL -#define SHA256_H1 0xbb67ae85UL -#define SHA256_H2 0x3c6ef372UL -#define SHA256_H3 0xa54ff53aUL -#define SHA256_H4 0x510e527fUL -#define SHA256_H5 0x9b05688cUL -#define SHA256_H6 0x1f83d9abUL -#define SHA256_H7 0x5be0cd19UL - -#define SHA384_H0 0xcbbb9d5dc1059ed8ULL -#define SHA384_H1 0x629a292a367cd507ULL -#define SHA384_H2 0x9159015a3070dd17ULL -#define SHA384_H3 0x152fecd8f70e5939ULL -#define SHA384_H4 0x67332667ffc00b31ULL -#define SHA384_H5 0x8eb44a8768581511ULL -#define SHA384_H6 0xdb0c2e0d64f98fa7ULL -#define SHA384_H7 0x47b5481dbefa4fa4ULL - -#define SHA512_H0 0x6a09e667f3bcc908ULL -#define SHA512_H1 0xbb67ae8584caa73bULL -#define SHA512_H2 0x3c6ef372fe94f82bULL -#define SHA512_H3 0xa54ff53a5f1d36f1ULL -#define SHA512_H4 0x510e527fade682d1ULL -#define SHA512_H5 0x9b05688c2b3e6c1fULL -#define SHA512_H6 0x1f83d9abfb41bd6bULL -#define SHA512_H7 0x5be0cd19137e2179ULL - -extern const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE]; - -extern const u8 sha224_zero_message_hash[SHA224_DIGEST_SIZE]; - -extern const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE]; - -extern const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE]; - -extern const u8 sha512_zero_message_hash[SHA512_DIGEST_SIZE]; - -struct sha1_state { - u32 state[SHA1_DIGEST_SIZE / 4]; - u64 count; - u8 buffer[SHA1_BLOCK_SIZE]; -}; - -struct sha256_state { - u32 state[SHA256_DIGEST_SIZE / 4]; - u64 count; - u8 buf[SHA256_BLOCK_SIZE]; -}; - -struct sha512_state { - u64 state[SHA512_DIGEST_SIZE / 8]; - u64 count[2]; - u8 buf[SHA512_BLOCK_SIZE]; -}; - -struct shash_desc; - -extern int crypto_sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len); - -extern int crypto_sha1_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash); - -extern int crypto_sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len); - -extern int crypto_sha256_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash); - -extern int crypto_sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len); - -extern int crypto_sha512_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash); - -/* - * An implementation of SHA-1's compression function. Don't use in new code! - * You shouldn't be using SHA-1, and even if you *have* to use SHA-1, this isn't - * the correct way to hash something with SHA-1 (use crypto_shash instead). - */ -#define SHA1_DIGEST_WORDS (SHA1_DIGEST_SIZE / 4) -#define SHA1_WORKSPACE_WORDS 16 -void sha1_init(__u32 *buf); -void sha1_transform(__u32 *digest, const char *data, __u32 *W); - -/* - * Stand-alone implementation of the SHA256 algorithm. It is designed to - * have as little dependencies as possible so it can be used in the - * kexec_file purgatory. In other cases you should generally use the - * hash APIs from include/crypto/hash.h. Especially when hashing large - * amounts of data as those APIs may be hw-accelerated. - * - * For details see lib/crypto/sha256.c - */ - -static inline void sha256_init(struct sha256_state *sctx) -{ - sctx->state[0] = SHA256_H0; - sctx->state[1] = SHA256_H1; - sctx->state[2] = SHA256_H2; - sctx->state[3] = SHA256_H3; - sctx->state[4] = SHA256_H4; - sctx->state[5] = SHA256_H5; - sctx->state[6] = SHA256_H6; - sctx->state[7] = SHA256_H7; - sctx->count = 0; -} -void sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len); -void sha256_final(struct sha256_state *sctx, u8 *out); -void sha256(const u8 *data, unsigned int len, u8 *out); - -static inline void sha224_init(struct sha256_state *sctx) -{ - sctx->state[0] = SHA224_H0; - sctx->state[1] = SHA224_H1; - sctx->state[2] = SHA224_H2; - sctx->state[3] = SHA224_H3; - sctx->state[4] = SHA224_H4; - sctx->state[5] = SHA224_H5; - sctx->state[6] = SHA224_H6; - sctx->state[7] = SHA224_H7; - sctx->count = 0; -} -void sha224_update(struct sha256_state *sctx, const u8 *data, unsigned int len); -void sha224_final(struct sha256_state *sctx, u8 *out); - -#endif diff --git a/include/crypto/sha1.h b/include/crypto/sha1.h new file mode 100644 index 000000000000..044ecea60ac8 --- /dev/null +++ b/include/crypto/sha1.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Common values for SHA-1 algorithms + */ + +#ifndef _CRYPTO_SHA1_H +#define _CRYPTO_SHA1_H + +#include + +#define SHA1_DIGEST_SIZE 20 +#define SHA1_BLOCK_SIZE 64 + +#define SHA1_H0 0x67452301UL +#define SHA1_H1 0xefcdab89UL +#define SHA1_H2 0x98badcfeUL +#define SHA1_H3 0x10325476UL +#define SHA1_H4 0xc3d2e1f0UL + +extern const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE]; + +struct sha1_state { + u32 state[SHA1_DIGEST_SIZE / 4]; + u64 count; + u8 buffer[SHA1_BLOCK_SIZE]; +}; + +struct shash_desc; + +extern int crypto_sha1_update(struct shash_desc *desc, const u8 *data, + unsigned int len); + +extern int crypto_sha1_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *hash); + +/* + * An implementation of SHA-1's compression function. Don't use in new code! + * You shouldn't be using SHA-1, and even if you *have* to use SHA-1, this isn't + * the correct way to hash something with SHA-1 (use crypto_shash instead). + */ +#define SHA1_DIGEST_WORDS (SHA1_DIGEST_SIZE / 4) +#define SHA1_WORKSPACE_WORDS 16 +void sha1_init(__u32 *buf); +void sha1_transform(__u32 *digest, const char *data, __u32 *W); + +#endif /* _CRYPTO_SHA1_H */ diff --git a/include/crypto/sha1_base.h b/include/crypto/sha1_base.h index a5d6033efef7..2e0e7c3827d1 100644 --- a/include/crypto/sha1_base.h +++ b/include/crypto/sha1_base.h @@ -9,7 +9,7 @@ #define _CRYPTO_SHA1_BASE_H #include -#include +#include #include #include #include diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h new file mode 100644 index 000000000000..2838f529f31e --- /dev/null +++ b/include/crypto/sha2.h @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Common values for SHA-2 algorithms + */ + +#ifndef _CRYPTO_SHA2_H +#define _CRYPTO_SHA2_H + +#include + +#define SHA224_DIGEST_SIZE 28 +#define SHA224_BLOCK_SIZE 64 + +#define SHA256_DIGEST_SIZE 32 +#define SHA256_BLOCK_SIZE 64 + +#define SHA384_DIGEST_SIZE 48 +#define SHA384_BLOCK_SIZE 128 + +#define SHA512_DIGEST_SIZE 64 +#define SHA512_BLOCK_SIZE 128 + +#define SHA224_H0 0xc1059ed8UL +#define SHA224_H1 0x367cd507UL +#define SHA224_H2 0x3070dd17UL +#define SHA224_H3 0xf70e5939UL +#define SHA224_H4 0xffc00b31UL +#define SHA224_H5 0x68581511UL +#define SHA224_H6 0x64f98fa7UL +#define SHA224_H7 0xbefa4fa4UL + +#define SHA256_H0 0x6a09e667UL +#define SHA256_H1 0xbb67ae85UL +#define SHA256_H2 0x3c6ef372UL +#define SHA256_H3 0xa54ff53aUL +#define SHA256_H4 0x510e527fUL +#define SHA256_H5 0x9b05688cUL +#define SHA256_H6 0x1f83d9abUL +#define SHA256_H7 0x5be0cd19UL + +#define SHA384_H0 0xcbbb9d5dc1059ed8ULL +#define SHA384_H1 0x629a292a367cd507ULL +#define SHA384_H2 0x9159015a3070dd17ULL +#define SHA384_H3 0x152fecd8f70e5939ULL +#define SHA384_H4 0x67332667ffc00b31ULL +#define SHA384_H5 0x8eb44a8768581511ULL +#define SHA384_H6 0xdb0c2e0d64f98fa7ULL +#define SHA384_H7 0x47b5481dbefa4fa4ULL + +#define SHA512_H0 0x6a09e667f3bcc908ULL +#define SHA512_H1 0xbb67ae8584caa73bULL +#define SHA512_H2 0x3c6ef372fe94f82bULL +#define SHA512_H3 0xa54ff53a5f1d36f1ULL +#define SHA512_H4 0x510e527fade682d1ULL +#define SHA512_H5 0x9b05688c2b3e6c1fULL +#define SHA512_H6 0x1f83d9abfb41bd6bULL +#define SHA512_H7 0x5be0cd19137e2179ULL + +extern const u8 sha224_zero_message_hash[SHA224_DIGEST_SIZE]; + +extern const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE]; + +extern const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE]; + +extern const u8 sha512_zero_message_hash[SHA512_DIGEST_SIZE]; + +struct sha256_state { + u32 state[SHA256_DIGEST_SIZE / 4]; + u64 count; + u8 buf[SHA256_BLOCK_SIZE]; +}; + +struct sha512_state { + u64 state[SHA512_DIGEST_SIZE / 8]; + u64 count[2]; + u8 buf[SHA512_BLOCK_SIZE]; +}; + +struct shash_desc; + +extern int crypto_sha256_update(struct shash_desc *desc, const u8 *data, + unsigned int len); + +extern int crypto_sha256_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *hash); + +extern int crypto_sha512_update(struct shash_desc *desc, const u8 *data, + unsigned int len); + +extern int crypto_sha512_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *hash); + +/* + * Stand-alone implementation of the SHA256 algorithm. It is designed to + * have as little dependencies as possible so it can be used in the + * kexec_file purgatory. In other cases you should generally use the + * hash APIs from include/crypto/hash.h. Especially when hashing large + * amounts of data as those APIs may be hw-accelerated. + * + * For details see lib/crypto/sha256.c + */ + +static inline void sha256_init(struct sha256_state *sctx) +{ + sctx->state[0] = SHA256_H0; + sctx->state[1] = SHA256_H1; + sctx->state[2] = SHA256_H2; + sctx->state[3] = SHA256_H3; + sctx->state[4] = SHA256_H4; + sctx->state[5] = SHA256_H5; + sctx->state[6] = SHA256_H6; + sctx->state[7] = SHA256_H7; + sctx->count = 0; +} +void sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len); +void sha256_final(struct sha256_state *sctx, u8 *out); +void sha256(const u8 *data, unsigned int len, u8 *out); + +static inline void sha224_init(struct sha256_state *sctx) +{ + sctx->state[0] = SHA224_H0; + sctx->state[1] = SHA224_H1; + sctx->state[2] = SHA224_H2; + sctx->state[3] = SHA224_H3; + sctx->state[4] = SHA224_H4; + sctx->state[5] = SHA224_H5; + sctx->state[6] = SHA224_H6; + sctx->state[7] = SHA224_H7; + sctx->count = 0; +} +void sha224_update(struct sha256_state *sctx, const u8 *data, unsigned int len); +void sha224_final(struct sha256_state *sctx, u8 *out); + +#endif /* _CRYPTO_SHA2_H */ diff --git a/include/crypto/sha256_base.h b/include/crypto/sha256_base.h index 93f9fd21cc06..76173c613058 100644 --- a/include/crypto/sha256_base.h +++ b/include/crypto/sha256_base.h @@ -9,7 +9,7 @@ #define _CRYPTO_SHA256_BASE_H #include -#include +#include #include #include #include diff --git a/include/crypto/sha512_base.h b/include/crypto/sha512_base.h index 93ab73baa38e..b370b3340b16 100644 --- a/include/crypto/sha512_base.h +++ b/include/crypto/sha512_base.h @@ -9,7 +9,7 @@ #define _CRYPTO_SHA512_BASE_H #include -#include +#include #include #include #include diff --git a/include/linux/ccp.h b/include/linux/ccp.h index a5dfbaf2470d..868924dec5a1 100644 --- a/include/linux/ccp.h +++ b/include/linux/ccp.h @@ -15,7 +15,8 @@ #include #include #include -#include +#include +#include struct ccp_device; struct ccp_cmd; diff --git a/include/linux/filter.h b/include/linux/filter.h index 72d62cbc1578..6c00140538b9 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include diff --git a/include/linux/purgatory.h b/include/linux/purgatory.h index b950e961cfa8..d7dc1559427f 100644 --- a/include/linux/purgatory.h +++ b/include/linux/purgatory.h @@ -3,7 +3,7 @@ #define _LINUX_PURGATORY_H #include -#include +#include #include struct kexec_sha_region { diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 106e4500fd53..4fcfe0b70c4e 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -11,7 +11,7 @@ #include #include -#include +#include /* vmcoreinfo stuff */ unsigned char *vmcoreinfo_data; diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 8798a8183974..4f8efc278aa7 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -42,7 +42,6 @@ #include #include -#include #include "kexec_internal.h" DEFINE_MUTEX(kexec_mutex); diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index e21f6b9234f7..b02086d70492 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index cdef37c05972..72a4b0b1df28 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include static const u32 SHA256_K[] = { diff --git a/lib/digsig.c b/lib/digsig.c index e0627c3e53b2..04b5e55ed95f 100644 --- a/lib/digsig.c +++ b/lib/digsig.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/lib/sha1.c b/lib/sha1.c index 49257a915bb6..9bd1935a1472 100644 --- a/lib/sha1.c +++ b/lib/sha1.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include /* diff --git a/net/ipv6/seg6_hmac.c b/net/ipv6/seg6_hmac.c index 85dddfe3a2c6..687d95dce085 100644 --- a/net/ipv6/seg6_hmac.c +++ b/net/ipv6/seg6_hmac.c @@ -35,7 +35,6 @@ #include #include -#include #include #include #include diff --git a/net/mptcp/crypto.c b/net/mptcp/crypto.c index 05d398d3fde4..b472dc149856 100644 --- a/net/mptcp/crypto.c +++ b/net/mptcp/crypto.c @@ -21,7 +21,7 @@ */ #include -#include +#include #include #include "protocol.h" diff --git a/net/mptcp/options.c b/net/mptcp/options.c index a044dd43411d..90cd52df99a6 100644 --- a/net/mptcp/options.c +++ b/net/mptcp/options.c @@ -7,7 +7,7 @@ #define pr_fmt(fmt) "MPTCP: " fmt #include -#include +#include #include #include #include "protocol.h" diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index ac4a1fe3550b..b229ae914d76 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/security/integrity/integrity.h b/security/integrity/integrity.h index 413c803c5208..547425c20e11 100644 --- a/security/integrity/integrity.h +++ b/security/integrity/integrity.h @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include diff --git a/security/keys/encrypted-keys/encrypted.c b/security/keys/encrypted-keys/encrypted.c index 192e531c146f..87432b35d771 100644 --- a/security/keys/encrypted-keys/encrypted.c +++ b/security/keys/encrypted-keys/encrypted.c @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include #include "encrypted.h" diff --git a/security/keys/trusted-keys/trusted_tpm1.c b/security/keys/trusted-keys/trusted_tpm1.c index b9fe02e5f84f..74d82093cbaa 100644 --- a/security/keys/trusted-keys/trusted_tpm1.c +++ b/security/keys/trusted-keys/trusted_tpm1.c @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/sound/soc/codecs/cros_ec_codec.c b/sound/soc/codecs/cros_ec_codec.c index 28f039adfa13..58894bf47514 100644 --- a/sound/soc/codecs/cros_ec_codec.c +++ b/sound/soc/codecs/cros_ec_codec.c @@ -8,7 +8,7 @@ * EC for audio function. */ -#include +#include #include #include #include -- cgit v1.2.3 From 1dc440355e472a60a98cb4ec9aa5ec56267a96fc Mon Sep 17 00:00:00 2001 From: Yang Shen Date: Fri, 13 Nov 2020 17:32:35 +0800 Subject: crypto: hisilicon/zip - add a work_queue for zip irq The patch 'irqchip/gic-v3-its: Balance initial LPI affinity across CPUs' set the IRQ to an uncentain CPU. If an IRQ is bound to the CPU used by the thread which is sending request, the throughput will be just half. So allocate a 'work_queue' and set as 'WQ_UNBOUND' to do the back half work on some different CPUS. Signed-off-by: Yang Shen Reviewed-by: Zaibo Xu Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/zip/zip_main.c | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 3d1524b63ea8..4fb5a32bf830 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -747,6 +747,8 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { + int ret; + qm->pdev = pdev; qm->ver = pdev->revision; qm->algs = "zlib\ngzip"; @@ -772,7 +774,25 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->qp_num = HZIP_QUEUE_NUM_V1 - HZIP_PF_DEF_Q_NUM; } - return hisi_qm_init(qm); + qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM | + WQ_UNBOUND, num_online_cpus(), + pci_name(qm->pdev)); + if (!qm->wq) { + pci_err(qm->pdev, "fail to alloc workqueue\n"); + return -ENOMEM; + } + + ret = hisi_qm_init(qm); + if (ret) + destroy_workqueue(qm->wq); + + return ret; +} + +static void hisi_zip_qm_uninit(struct hisi_qm *qm) +{ + hisi_qm_uninit(qm); + destroy_workqueue(qm->wq); } static int hisi_zip_probe_init(struct hisi_zip *hisi_zip) @@ -854,7 +874,7 @@ err_dev_err_uninit: hisi_qm_dev_err_uninit(qm); err_qm_uninit: - hisi_qm_uninit(qm); + hisi_zip_qm_uninit(qm); return ret; } @@ -872,7 +892,7 @@ static void hisi_zip_remove(struct pci_dev *pdev) hisi_zip_debugfs_exit(qm); hisi_qm_stop(qm, QM_NORMAL); hisi_qm_dev_err_uninit(qm); - hisi_qm_uninit(qm); + hisi_zip_qm_uninit(qm); } static const struct pci_error_handlers hisi_zip_err_handler = { -- cgit v1.2.3 From ff8107200367f4abe0e5bce66a245e8d0f2d229e Mon Sep 17 00:00:00 2001 From: Zhang Qilong Date: Fri, 13 Nov 2020 21:17:28 +0800 Subject: crypto: omap-aes - Fix PM disable depth imbalance in omap_aes_probe The pm_runtime_enable will increase power disable depth. Thus a pairing decrement is needed on the error handling path to keep it balanced according to context. Fixes: f7b2b5dd6a62a ("crypto: omap-aes - add error check for pm_runtime_get_sync") Signed-off-by: Zhang Qilong Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 70ea5784a024..a45bdcf3026d 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -1138,7 +1138,7 @@ static int omap_aes_probe(struct platform_device *pdev) if (err < 0) { dev_err(dev, "%s: failed to get_sync(%d)\n", __func__, err); - goto err_res; + goto err_pm_disable; } omap_aes_dma_stop(dd); @@ -1247,6 +1247,7 @@ err_engine: omap_aes_dma_cleanup(dd); err_irq: tasklet_kill(&dd->done_task); +err_pm_disable: pm_runtime_disable(dev); err_res: dd = NULL; -- cgit v1.2.3 From e73916f20829be09932326ab0c9627b29df553e8 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Fri, 13 Nov 2020 16:46:41 +0000 Subject: crypto: qat - target fw images to specific AEs Introduce support for devices that require multiple firmware images. If a device requires more than a firmware image to operate, load the image to the appropriate Acceleration Engine (AE). Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_devices.h | 3 ++ drivers/crypto/qat/qat_common/adf_accel_engine.c | 58 +++++++++++++++++++++-- 2 files changed, 56 insertions(+), 5 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 5694422ec66c..1484f10dbfdf 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -175,6 +175,9 @@ struct adf_hw_device_data { void (*enable_ints)(struct adf_accel_dev *accel_dev); int (*enable_vf2pf_comms)(struct adf_accel_dev *accel_dev); void (*reset_device)(struct adf_accel_dev *accel_dev); + char *(*uof_get_name)(u32 obj_num); + u32 (*uof_get_num_objs)(void); + u32 (*uof_get_ae_mask)(u32 obj_num); struct adf_hw_csr_ops csr_ops; const char *fw_name; const char *fw_mmp_name; diff --git a/drivers/crypto/qat/qat_common/adf_accel_engine.c b/drivers/crypto/qat/qat_common/adf_accel_engine.c index 08aaaf2b4659..ca4eae8cdd0b 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_engine.c +++ b/drivers/crypto/qat/qat_common/adf_accel_engine.c @@ -7,12 +7,55 @@ #include "adf_common_drv.h" #include "icp_qat_uclo.h" +static int adf_ae_fw_load_images(struct adf_accel_dev *accel_dev, void *fw_addr, + u32 fw_size) +{ + struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + struct icp_qat_fw_loader_handle *loader; + char *obj_name; + u32 num_objs; + u32 ae_mask; + int i; + + loader = loader_data->fw_loader; + num_objs = hw_device->uof_get_num_objs(); + + for (i = 0; i < num_objs; i++) { + obj_name = hw_device->uof_get_name(i); + ae_mask = hw_device->uof_get_ae_mask(i); + + if (qat_uclo_set_cfg_ae_mask(loader, ae_mask)) { + dev_err(&GET_DEV(accel_dev), + "Invalid mask for UOF image\n"); + goto out_err; + } + if (qat_uclo_map_obj(loader, fw_addr, fw_size, obj_name)) { + dev_err(&GET_DEV(accel_dev), + "Failed to map UOF firmware\n"); + goto out_err; + } + if (qat_uclo_wr_all_uimage(loader)) { + dev_err(&GET_DEV(accel_dev), + "Failed to load UOF firmware\n"); + goto out_err; + } + qat_uclo_del_obj(loader); + } + + return 0; + +out_err: + adf_ae_fw_release(accel_dev); + return -EFAULT; +} + int adf_ae_fw_load(struct adf_accel_dev *accel_dev) { struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; struct adf_hw_device_data *hw_device = accel_dev->hw_device; - void *uof_addr, *mmp_addr; - u32 uof_size, mmp_size; + void *fw_addr, *mmp_addr; + u32 fw_size, mmp_size; if (!hw_device->fw_name) return 0; @@ -30,15 +73,20 @@ int adf_ae_fw_load(struct adf_accel_dev *accel_dev) goto out_err; } - uof_size = loader_data->uof_fw->size; - uof_addr = (void *)loader_data->uof_fw->data; + fw_size = loader_data->uof_fw->size; + fw_addr = (void *)loader_data->uof_fw->data; mmp_size = loader_data->mmp_fw->size; mmp_addr = (void *)loader_data->mmp_fw->data; + if (qat_uclo_wr_mimage(loader_data->fw_loader, mmp_addr, mmp_size)) { dev_err(&GET_DEV(accel_dev), "Failed to load MMP\n"); goto out_err; } - if (qat_uclo_map_obj(loader_data->fw_loader, uof_addr, uof_size, NULL)) { + + if (hw_device->uof_get_num_objs) + return adf_ae_fw_load_images(accel_dev, fw_addr, fw_size); + + if (qat_uclo_map_obj(loader_data->fw_loader, fw_addr, fw_size, NULL)) { dev_err(&GET_DEV(accel_dev), "Failed to map FW\n"); goto out_err; } -- cgit v1.2.3 From 6644f49e7b71e13bcc5498d9fea8862c5ce96388 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Fri, 13 Nov 2020 16:46:42 +0000 Subject: crypto: qat - add hook to initialize vector routing table Add an hook to initialize the vector routing table with the default values before MSIx is enabled. The new function set_msix_rttable() is called only if present in the struct adf_hw_device_data of the device. This is to allow for QAT devices that do not support that functionality. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_devices.h | 1 + drivers/crypto/qat/qat_common/adf_isr.c | 3 +++ 2 files changed, 4 insertions(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 1484f10dbfdf..26164d71f1d6 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -175,6 +175,7 @@ struct adf_hw_device_data { void (*enable_ints)(struct adf_accel_dev *accel_dev); int (*enable_vf2pf_comms)(struct adf_accel_dev *accel_dev); void (*reset_device)(struct adf_accel_dev *accel_dev); + void (*set_msix_rttable)(struct adf_accel_dev *accel_dev); char *(*uof_get_name)(u32 obj_num); u32 (*uof_get_num_objs)(void); u32 (*uof_get_ae_mask)(u32 obj_num); diff --git a/drivers/crypto/qat/qat_common/adf_isr.c b/drivers/crypto/qat/qat_common/adf_isr.c index 5444f0ea0a1d..c45853463530 100644 --- a/drivers/crypto/qat/qat_common/adf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_isr.c @@ -21,6 +21,9 @@ static int adf_enable_msix(struct adf_accel_dev *accel_dev) struct adf_hw_device_data *hw_data = accel_dev->hw_device; u32 msix_num_entries = 1; + if (hw_data->set_msix_rttable) + hw_data->set_msix_rttable(accel_dev); + /* If SR-IOV is disabled, add entries for each bank */ if (!accel_dev->pf.vf_info) { int i; -- cgit v1.2.3 From 8c8268166e83425243789c5781c92e7fa33b703b Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Fri, 13 Nov 2020 16:46:43 +0000 Subject: crypto: qat - add qat_4xxx driver Add support for QAT 4xxx devices. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/Kconfig | 11 + drivers/crypto/qat/Makefile | 1 + drivers/crypto/qat/qat_4xxx/Makefile | 4 + drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 218 +++++++++++++++ drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h | 75 +++++ drivers/crypto/qat/qat_4xxx/adf_drv.c | 320 ++++++++++++++++++++++ drivers/crypto/qat/qat_common/Makefile | 1 + drivers/crypto/qat/qat_common/adf_accel_devices.h | 1 + drivers/crypto/qat/qat_common/adf_cfg_common.h | 3 +- drivers/crypto/qat/qat_common/adf_gen4_hw_data.c | 101 +++++++ drivers/crypto/qat/qat_common/adf_gen4_hw_data.h | 99 +++++++ 11 files changed, 833 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/qat/qat_4xxx/Makefile create mode 100644 drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c create mode 100644 drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h create mode 100644 drivers/crypto/qat/qat_4xxx/adf_drv.c create mode 100644 drivers/crypto/qat/qat_common/adf_gen4_hw_data.c create mode 100644 drivers/crypto/qat/qat_common/adf_gen4_hw_data.h (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/Kconfig b/drivers/crypto/qat/Kconfig index 2006322345de..beb379b23dc3 100644 --- a/drivers/crypto/qat/Kconfig +++ b/drivers/crypto/qat/Kconfig @@ -46,6 +46,17 @@ config CRYPTO_DEV_QAT_C62X To compile this as a module, choose M here: the module will be called qat_c62x. +config CRYPTO_DEV_QAT_4XXX + tristate "Support for Intel(R) QAT_4XXX" + depends on X86 && PCI + select CRYPTO_DEV_QAT + help + Support for Intel(R) QuickAssist Technology QAT_4xxx + for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_4xxx. + config CRYPTO_DEV_QAT_DH895xCCVF tristate "Support for Intel(R) DH895xCC Virtual Function" depends on X86 && PCI diff --git a/drivers/crypto/qat/Makefile b/drivers/crypto/qat/Makefile index 7dd15e751d02..258c8a626ce0 100644 --- a/drivers/crypto/qat/Makefile +++ b/drivers/crypto/qat/Makefile @@ -3,6 +3,7 @@ obj-$(CONFIG_CRYPTO_DEV_QAT) += qat_common/ obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc/ obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx/ obj-$(CONFIG_CRYPTO_DEV_QAT_C62X) += qat_c62x/ +obj-$(CONFIG_CRYPTO_DEV_QAT_4XXX) += qat_4xxx/ obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf/ obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXXVF) += qat_c3xxxvf/ obj-$(CONFIG_CRYPTO_DEV_QAT_C62XVF) += qat_c62xvf/ diff --git a/drivers/crypto/qat/qat_4xxx/Makefile b/drivers/crypto/qat/qat_4xxx/Makefile new file mode 100644 index 000000000000..ff9c8b5897ea --- /dev/null +++ b/drivers/crypto/qat/qat_4xxx/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +ccflags-y := -I $(srctree)/$(src)/../qat_common +obj-$(CONFIG_CRYPTO_DEV_QAT_4XXX) += qat_4xxx.o +qat_4xxx-objs := adf_drv.o adf_4xxx_hw_data.o diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c new file mode 100644 index 000000000000..e7a7c1e3da28 --- /dev/null +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2020 Intel Corporation */ +#include +#include +#include +#include +#include "adf_4xxx_hw_data.h" + +struct adf_fw_config { + u32 ae_mask; + char *obj_name; +}; + +static struct adf_fw_config adf_4xxx_fw_config[] = { + {0xF0, ADF_4XXX_SYM_OBJ}, + {0xF, ADF_4XXX_ASYM_OBJ}, + {0x100, ADF_4XXX_ADMIN_OBJ}, +}; + +/* Worker thread to service arbiter mappings */ +static u32 thrd_to_arb_map[] = { + 0x5555555, 0x5555555, 0x5555555, 0x5555555, + 0xAAAAAAA, 0xAAAAAAA, 0xAAAAAAA, 0xAAAAAAA, + 0x0 +}; + +static struct adf_hw_device_class adf_4xxx_class = { + .name = ADF_4XXX_DEVICE_NAME, + .type = DEV_4XXX, + .instances = 0, +}; + +static u32 get_accel_mask(struct adf_hw_device_data *self) +{ + return ADF_4XXX_ACCELERATORS_MASK; +} + +static u32 get_ae_mask(struct adf_hw_device_data *self) +{ + u32 me_disable = self->fuses; + + return ~me_disable & ADF_4XXX_ACCELENGINES_MASK; +} + +static u32 get_num_accels(struct adf_hw_device_data *self) +{ + return ADF_4XXX_MAX_ACCELERATORS; +} + +static u32 get_num_aes(struct adf_hw_device_data *self) +{ + if (!self || !self->ae_mask) + return 0; + + return hweight32(self->ae_mask); +} + +static u32 get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_4XXX_PMISC_BAR; +} + +static u32 get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_4XXX_ETR_BAR; +} + +static u32 get_sram_bar_id(struct adf_hw_device_data *self) +{ + return ADF_4XXX_SRAM_BAR; +} + +/* + * The vector routing table is used to select the MSI-X entry to use for each + * interrupt source. + * The first ADF_4XXX_ETR_MAX_BANKS entries correspond to ring interrupts. + * The final entry corresponds to VF2PF or error interrupts. + * This vector table could be used to configure one MSI-X entry to be shared + * between multiple interrupt sources. + * + * The default routing is set to have a one to one correspondence between the + * interrupt source and the MSI-X entry used. + */ +static void set_msix_default_rttable(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr; + int i; + + csr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr; + for (i = 0; i <= ADF_4XXX_ETR_MAX_BANKS; i++) + ADF_CSR_WR(csr, ADF_4XXX_MSIX_RTTABLE_OFFSET(i), i); +} + +static enum dev_sku_info get_sku(struct adf_hw_device_data *self) +{ + return DEV_SKU_1; +} + +static void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, + u32 const **arb_map_config) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + unsigned long ae_mask = hw_device->ae_mask; + int i; + + for_each_clear_bit(i, &ae_mask, ADF_4XXX_MAX_ACCELENGINES) + thrd_to_arb_map[i] = 0; + + *arb_map_config = thrd_to_arb_map; +} + +static void get_arb_info(struct arb_info *arb_info) +{ + arb_info->arb_cfg = ADF_4XXX_ARB_CONFIG; + arb_info->arb_offset = ADF_4XXX_ARB_OFFSET; + arb_info->wt2sam_offset = ADF_4XXX_ARB_WRK_2_SER_MAP_OFFSET; +} + +static void get_admin_info(struct admin_info *admin_csrs_info) +{ + admin_csrs_info->mailbox_offset = ADF_4XXX_MAILBOX_BASE_OFFSET; + admin_csrs_info->admin_msg_ur = ADF_4XXX_ADMINMSGUR_OFFSET; + admin_csrs_info->admin_msg_lr = ADF_4XXX_ADMINMSGLR_OFFSET; +} + +static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) +{ + struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR]; + void __iomem *csr = misc_bar->virt_addr; + + /* Enable all in errsou3 except VFLR notification on host */ + ADF_CSR_WR(csr, ADF_4XXX_ERRMSK3, ADF_4XXX_VFLNOTIFY); +} + +static void adf_enable_ints(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr; + + addr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr; + + /* Enable bundle interrupts */ + ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_RP_X0_MASK_OFFSET, 0); + ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_RP_X1_MASK_OFFSET, 0); + + /* Enable misc interrupts */ + ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_MASK_OFFSET, 0); +} + +static int adf_pf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static u32 uof_get_num_objs(void) +{ + return ARRAY_SIZE(adf_4xxx_fw_config); +} + +static char *uof_get_name(u32 obj_num) +{ + return adf_4xxx_fw_config[obj_num].obj_name; +} + +static u32 uof_get_ae_mask(u32 obj_num) +{ + return adf_4xxx_fw_config[obj_num].ae_mask; +} + +void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &adf_4xxx_class; + hw_data->instance_id = adf_4xxx_class.instances++; + hw_data->num_banks = ADF_4XXX_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_4XXX_NUM_RINGS_PER_BANK; + hw_data->num_accel = ADF_4XXX_MAX_ACCELERATORS; + hw_data->num_engines = ADF_4XXX_MAX_ACCELENGINES; + hw_data->num_logical_accel = 1; + hw_data->tx_rx_gap = ADF_4XXX_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_4XXX_TX_RINGS_MASK; + hw_data->alloc_irq = adf_isr_resource_alloc; + hw_data->free_irq = adf_isr_resource_free; + hw_data->enable_error_correction = adf_enable_error_correction; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = get_num_accels; + hw_data->get_num_aes = get_num_aes; + hw_data->get_sram_bar_id = get_sram_bar_id; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_arb_info = get_arb_info; + hw_data->get_admin_info = get_admin_info; + hw_data->get_sku = get_sku; + hw_data->fw_name = ADF_4XXX_FW; + hw_data->fw_mmp_name = ADF_4XXX_MMP; + hw_data->init_admin_comms = adf_init_admin_comms; + hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->disable_iov = adf_disable_sriov; + hw_data->send_admin_init = adf_send_admin_init; + hw_data->init_arb = adf_init_arb; + hw_data->exit_arb = adf_exit_arb; + hw_data->get_arb_mapping = adf_get_arbiter_mapping; + hw_data->enable_ints = adf_enable_ints; + hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; + hw_data->reset_device = adf_reset_flr; + hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; + hw_data->admin_ae_mask = ADF_4XXX_ADMIN_AE_MASK; + hw_data->uof_get_num_objs = uof_get_num_objs; + hw_data->uof_get_name = uof_get_name; + hw_data->uof_get_ae_mask = uof_get_ae_mask; + hw_data->set_msix_rttable = set_msix_default_rttable; + + adf_gen4_init_hw_csr_ops(&hw_data->csr_ops); +} + +void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; +} diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h new file mode 100644 index 000000000000..cdde0be886bf --- /dev/null +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef ADF_4XXX_HW_DATA_H_ +#define ADF_4XXX_HW_DATA_H_ + +#include + +/* PCIe configuration space */ +#define ADF_4XXX_SRAM_BAR 0 +#define ADF_4XXX_PMISC_BAR 1 +#define ADF_4XXX_ETR_BAR 2 +#define ADF_4XXX_RX_RINGS_OFFSET 1 +#define ADF_4XXX_TX_RINGS_MASK 0x1 +#define ADF_4XXX_MAX_ACCELERATORS 1 +#define ADF_4XXX_MAX_ACCELENGINES 9 +#define ADF_4XXX_BAR_MASK (BIT(0) | BIT(2) | BIT(4)) + +/* Physical function fuses */ +#define ADF_4XXX_FUSECTL0_OFFSET (0x2C8) +#define ADF_4XXX_FUSECTL1_OFFSET (0x2CC) +#define ADF_4XXX_FUSECTL2_OFFSET (0x2D0) +#define ADF_4XXX_FUSECTL3_OFFSET (0x2D4) +#define ADF_4XXX_FUSECTL4_OFFSET (0x2D8) +#define ADF_4XXX_FUSECTL5_OFFSET (0x2DC) + +#define ADF_4XXX_ACCELERATORS_MASK (0x1) +#define ADF_4XXX_ACCELENGINES_MASK (0x1FF) +#define ADF_4XXX_ADMIN_AE_MASK (0x100) + +#define ADF_4XXX_ETR_MAX_BANKS 64 + +/* MSIX interrupt */ +#define ADF_4XXX_SMIAPF_RP_X0_MASK_OFFSET (0x41A040) +#define ADF_4XXX_SMIAPF_RP_X1_MASK_OFFSET (0x41A044) +#define ADF_4XXX_SMIAPF_MASK_OFFSET (0x41A084) +#define ADF_4XXX_MSIX_RTTABLE_OFFSET(i) (0x409000 + ((i) * 0x04)) + +/* Bank and ring configuration */ +#define ADF_4XXX_NUM_RINGS_PER_BANK 2 + +/* Error source registers */ +#define ADF_4XXX_ERRSOU0 (0x41A200) +#define ADF_4XXX_ERRSOU1 (0x41A204) +#define ADF_4XXX_ERRSOU2 (0x41A208) +#define ADF_4XXX_ERRSOU3 (0x41A20C) + +/* Error source mask registers */ +#define ADF_4XXX_ERRMSK0 (0x41A210) +#define ADF_4XXX_ERRMSK1 (0x41A214) +#define ADF_4XXX_ERRMSK2 (0x41A218) +#define ADF_4XXX_ERRMSK3 (0x41A21C) + +#define ADF_4XXX_VFLNOTIFY BIT(7) + +/* Arbiter configuration */ +#define ADF_4XXX_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) +#define ADF_4XXX_ARB_OFFSET (0x0) +#define ADF_4XXX_ARB_WRK_2_SER_MAP_OFFSET (0x400) + +/* Admin Interface Reg Offset */ +#define ADF_4XXX_ADMINMSGUR_OFFSET (0x500574) +#define ADF_4XXX_ADMINMSGLR_OFFSET (0x500578) +#define ADF_4XXX_MAILBOX_BASE_OFFSET (0x600970) + +/* Firmware Binaries */ +#define ADF_4XXX_FW "qat_4xxx.bin" +#define ADF_4XXX_MMP "qat_4xxx_mmp.bin" +#define ADF_4XXX_SYM_OBJ "qat_4xxx_sym.bin" +#define ADF_4XXX_ASYM_OBJ "qat_4xxx_asym.bin" +#define ADF_4XXX_ADMIN_OBJ "qat_4xxx_admin.bin" + +void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data); + +#endif diff --git a/drivers/crypto/qat/qat_4xxx/adf_drv.c b/drivers/crypto/qat/qat_4xxx/adf_drv.c new file mode 100644 index 000000000000..de5a955f406a --- /dev/null +++ b/drivers/crypto/qat/qat_4xxx/adf_drv.c @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2020 Intel Corporation */ +#include +#include +#include + +#include +#include +#include + +#include "adf_4xxx_hw_data.h" +#include "qat_crypto.h" +#include "adf_transport_access_macros.h" + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, ADF_4XXX_PCI_DEVICE_ID), }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + if (accel_dev->hw_device) { + adf_clean_hw_data_4xxx(accel_dev->hw_device); + accel_dev->hw_device = NULL; + } + adf_cfg_dev_remove(accel_dev); + debugfs_remove(accel_dev->debugfs_dir); + adf_devmgr_rm_dev(accel_dev, NULL); +} + +static int adf_crypto_dev_config(struct adf_accel_dev *accel_dev) +{ + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + int banks = GET_MAX_BANKS(accel_dev); + int cpus = num_online_cpus(); + unsigned long bank, val; + int instances; + int ret; + int i; + + if (adf_hw_dev_has_crypto(accel_dev)) + instances = min(cpus, banks / 2); + else + instances = 0; + + ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC); + if (ret) + goto err; + + ret = adf_cfg_section_add(accel_dev, "Accelerator0"); + if (ret) + goto err; + + for (i = 0; i < instances; i++) { + val = i; + bank = i * 2; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &bank, ADF_DEC); + if (ret) + goto err; + + bank += 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &bank, ADF_DEC); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, + i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); + val = 128; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 512; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 0; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 0; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = ADF_COALESCING_DEF_TIME; + snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); + ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0", + key, &val, ADF_DEC); + if (ret) + goto err; + } + + val = i; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, + &val, ADF_DEC); + if (ret) + goto err; + + set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); + return 0; +err: + dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n"); + return ret; +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + char name[ADF_DEVICE_NAME_LENGTH]; + unsigned int i, bar_nr; + unsigned long bar_mask; + struct adf_bar *bar; + int ret; + + if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) { + /* + * If the accelerator is connected to a node with no memory + * there is no point in using the accelerator since the remote + * memory transaction will be very slow. + */ + dev_err(&pdev->dev, "Invalid NUMA configuration.\n"); + return -EINVAL; + } + + accel_dev = devm_kzalloc(&pdev->dev, sizeof(*accel_dev), GFP_KERNEL); + if (!accel_dev) + return -ENOMEM; + + INIT_LIST_HEAD(&accel_dev->crypto_list); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + + /* + * Add accel device to accel table + * This should be called before adf_cleanup_accel is called + */ + if (adf_devmgr_add_dev(accel_dev, NULL)) { + dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); + return -EFAULT; + } + + accel_dev->owner = THIS_MODULE; + /* Allocate and initialise device hardware meta-data structure */ + hw_data = devm_kzalloc(&pdev->dev, sizeof(*hw_data), GFP_KERNEL); + if (!hw_data) { + ret = -ENOMEM; + goto out_err; + } + + accel_dev->hw_device = hw_data; + adf_init_hw_data_4xxx(accel_dev->hw_device); + + pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); + pci_read_config_dword(pdev, ADF_4XXX_FUSECTL4_OFFSET, &hw_data->fuses); + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + /* If the device has no acceleration engines then ignore it */ + if (!hw_data->accel_mask || !hw_data->ae_mask || + (~hw_data->ae_mask & 0x01)) { + dev_err(&pdev->dev, "No acceleration units found.\n"); + ret = -EFAULT; + goto out_err; + } + + /* Create dev top level debugfs entry */ + snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, + hw_data->dev_class->name, pci_name(pdev)); + + accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + + /* Enable PCI device */ + ret = pcim_enable_device(pdev); + if (ret) { + dev_err(&pdev->dev, "Can't enable PCI device.\n"); + goto out_err; + } + + /* Set DMA identifier */ + if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { + if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) { + dev_err(&pdev->dev, "No usable DMA configuration.\n"); + ret = -EFAULT; + goto out_err; + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + } + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); + } + + /* Find and map all the device's BARS */ + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_4XXX_BAR_MASK; + + ret = pcim_iomap_regions_request_all(pdev, bar_mask, pci_name(pdev)); + if (ret) { + dev_err(&pdev->dev, "Failed to map pci regions.\n"); + goto out_err; + } + + i = 0; + for_each_set_bit(bar_nr, &bar_mask, PCI_STD_NUM_BARS) { + bar = &accel_pci_dev->pci_bars[i++]; + bar->virt_addr = pcim_iomap_table(pdev)[bar_nr]; + } + + pci_set_master(pdev); + + if (adf_enable_aer(accel_dev)) { + dev_err(&pdev->dev, "Failed to enable aer.\n"); + ret = -EFAULT; + goto out_err; + } + + if (pci_save_state(pdev)) { + dev_err(&pdev->dev, "Failed to save pci state.\n"); + ret = -ENOMEM; + goto out_err_disable_aer; + } + + ret = adf_crypto_dev_config(accel_dev); + if (ret) + goto out_err_disable_aer; + + ret = adf_dev_init(accel_dev); + if (ret) + goto out_err_dev_shutdown; + + ret = adf_dev_start(accel_dev); + if (ret) + goto out_err_dev_stop; + + return ret; + +out_err_dev_stop: + adf_dev_stop(accel_dev); +out_err_dev_shutdown: + adf_dev_shutdown(accel_dev); +out_err_disable_aer: + adf_disable_aer(accel_dev); +out_err: + adf_cleanup_accel(accel_dev); + return ret; +} + +static void adf_remove(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Driver removal failed\n"); + return; + } + adf_dev_stop(accel_dev); + adf_dev_shutdown(accel_dev); + adf_disable_aer(accel_dev); + adf_cleanup_accel(accel_dev); +} + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_4XXX_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .sriov_configure = adf_sriov_configure, +}; + +module_pci_driver(adf_driver); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_FIRMWARE(ADF_4XXX_FW); +MODULE_FIRMWARE(ADF_4XXX_MMP); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_VERSION(ADF_DRV_VERSION); +MODULE_SOFTDEP("pre: crypto-intel_qat"); diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/qat/qat_common/Makefile index 25d28516dcdd..9c57abdf56b7 100644 --- a/drivers/crypto/qat/qat_common/Makefile +++ b/drivers/crypto/qat/qat_common/Makefile @@ -11,6 +11,7 @@ intel_qat-objs := adf_cfg.o \ adf_admin.o \ adf_hw_arbiter.o \ adf_gen2_hw_data.o \ + adf_gen4_hw_data.o \ qat_crypto.o \ qat_algs.o \ qat_asym_algs.o \ diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 26164d71f1d6..c46a5805b294 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -15,6 +15,7 @@ #define ADF_C62XVF_DEVICE_NAME "c6xxvf" #define ADF_C3XXX_DEVICE_NAME "c3xxx" #define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" +#define ADF_4XXX_DEVICE_NAME "4xxx" #define ADF_4XXX_PCI_DEVICE_ID 0x4940 #define ADF_4XXXIOV_PCI_DEVICE_ID 0x4941 #define ADF_ERRSOU3 (0x3A000 + 0x0C) diff --git a/drivers/crypto/qat/qat_common/adf_cfg_common.h b/drivers/crypto/qat/qat_common/adf_cfg_common.h index 1ef46ccfba47..4fabb70b1f18 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg_common.h +++ b/drivers/crypto/qat/qat_common/adf_cfg_common.h @@ -32,7 +32,8 @@ enum adf_device_type { DEV_C62X, DEV_C62XVF, DEV_C3XXX, - DEV_C3XXXVF + DEV_C3XXXVF, + DEV_4XXX, }; struct adf_dev_status_info { diff --git a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c new file mode 100644 index 000000000000..b72ff58e0bc7 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2020 Intel Corporation */ +#include "adf_accel_devices.h" +#include "adf_gen4_hw_data.h" + +static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size) +{ + return BUILD_RING_BASE_ADDR(addr, size); +} + +static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); +} + +static void write_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_TAIL(csr_base_addr, bank, ring); +} + +static void write_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_e_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_E_STAT(csr_base_addr, bank); +} + +static void write_csr_ring_config(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value); +} + +static void write_csr_ring_base(void __iomem *csr_base_addr, u32 bank, u32 ring, + dma_addr_t addr) +{ + WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr); +} + +static void write_csr_int_flag(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_FLAG(csr_base_addr, bank, value); +} + +static void write_csr_int_srcsel(void __iomem *csr_base_addr, u32 bank) +{ + WRITE_CSR_INT_SRCSEL(csr_base_addr, bank); +} + +static void write_csr_int_col_en(void __iomem *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value); +} + +static void write_csr_int_col_ctl(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value); +} + +static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value); +} + +static void write_csr_ring_srv_arb_en(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value); +} + +void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) +{ + csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr; + csr_ops->read_csr_ring_head = read_csr_ring_head; + csr_ops->write_csr_ring_head = write_csr_ring_head; + csr_ops->read_csr_ring_tail = read_csr_ring_tail; + csr_ops->write_csr_ring_tail = write_csr_ring_tail; + csr_ops->read_csr_e_stat = read_csr_e_stat; + csr_ops->write_csr_ring_config = write_csr_ring_config; + csr_ops->write_csr_ring_base = write_csr_ring_base; + csr_ops->write_csr_int_flag = write_csr_int_flag; + csr_ops->write_csr_int_srcsel = write_csr_int_srcsel; + csr_ops->write_csr_int_col_en = write_csr_int_col_en; + csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl; + csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; + csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_hw_csr_ops); diff --git a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h new file mode 100644 index 000000000000..8ab62b2ac311 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2020 Intel Corporation */ +#ifndef ADF_GEN4_HW_CSR_DATA_H_ +#define ADF_GEN4_HW_CSR_DATA_H_ + +#include "adf_accel_devices.h" + +/* Transport access */ +#define ADF_BANK_INT_SRC_SEL_MASK 0x44UL +#define ADF_RING_CSR_RING_CONFIG 0x1000 +#define ADF_RING_CSR_RING_LBASE 0x1040 +#define ADF_RING_CSR_RING_UBASE 0x1080 +#define ADF_RING_CSR_RING_HEAD 0x0C0 +#define ADF_RING_CSR_RING_TAIL 0x100 +#define ADF_RING_CSR_E_STAT 0x14C +#define ADF_RING_CSR_INT_FLAG 0x170 +#define ADF_RING_CSR_INT_SRCSEL 0x174 +#define ADF_RING_CSR_INT_COL_CTL 0x180 +#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 +#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 +#define ADF_RING_CSR_INT_COL_EN 0x17C +#define ADF_RING_CSR_ADDR_OFFSET 0x100000 +#define ADF_RING_BUNDLE_SIZE 0x2000 + +#define BUILD_RING_BASE_ADDR(addr, size) \ + ((((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) << 6) +#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2)) +#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2)) +#define READ_CSR_E_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_E_STAT) +#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_CONFIG + ((ring) << 2), value) +#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ +do { \ + void __iomem *_csr_base_addr = csr_base_addr; \ + u32 _bank = bank; \ + u32 _ring = ring; \ + dma_addr_t _value = value; \ + u32 l_base = 0, u_base = 0; \ + l_base = lower_32_bits(_value); \ + u_base = upper_32_bits(_value); \ + ADF_CSR_WR((_csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (_bank) + \ + ADF_RING_CSR_RING_LBASE + ((_ring) << 2), l_base); \ + ADF_CSR_WR((_csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (_bank) + \ + ADF_RING_CSR_RING_UBASE + ((_ring) << 2), u_base); \ +} while (0) + +#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2), value) +#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2), value) +#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_FLAG, (value)) +#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_SRCSEL, ADF_BANK_INT_SRC_SEL_MASK) +#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_COL_EN, (value)) +#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_COL_CTL, \ + ADF_RING_CSR_INT_COL_CTL_ENABLE | (value)) +#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_FLAG_AND_COL, (value)) + +/* Arbiter configuration */ +#define ADF_RING_CSR_RING_SRV_ARB_EN 0x19C + +#define WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_SRV_ARB_EN, (value)) + +void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); + +#endif -- cgit v1.2.3 From 2e0e386af88429ec28fb9ab4531096a370578e97 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Sat, 14 Nov 2020 12:12:27 +0000 Subject: crypto: kconfig - fix a couple of spelling mistakes There are a couple of spelling mistakes in two crypto Kconfig files. Fix these. Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 2 +- drivers/crypto/allwinner/Kconfig | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 37da0c070a88..129bbdc4993b 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -648,7 +648,7 @@ choice default CRYPTO_DEV_QCE_ENABLE_ALL depends on CRYPTO_DEV_QCE help - This option allows to choose whether to build support for all algorihtms + This option allows to choose whether to build support for all algorithms (default), hashes-only, or skciphers-only. The QCE engine does not appear to scale as well as the CPU to handle diff --git a/drivers/crypto/allwinner/Kconfig b/drivers/crypto/allwinner/Kconfig index 0cdfe0e8cc66..180c8a9db819 100644 --- a/drivers/crypto/allwinner/Kconfig +++ b/drivers/crypto/allwinner/Kconfig @@ -43,7 +43,7 @@ config CRYPTO_DEV_SUN8I_CE depends on CRYPTO_DEV_ALLWINNER depends on PM help - Select y here to have support for the crypto Engine availlable on + Select y here to have support for the crypto Engine available on Allwinner SoC H2+, H3, H5, H6, R40 and A64. The Crypto Engine handle AES/3DES ciphers in ECB/CBC mode. -- cgit v1.2.3 From 732b764099f651a088fd931d7b8121b6aa84e62e Mon Sep 17 00:00:00 2001 From: Corentin Labbe Date: Sun, 15 Nov 2020 19:08:07 +0000 Subject: crypto: sun8i-ce - fix two error path's memory leak This patch fixes the following smatch warnings: drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c:412 sun8i_ce_hash_run() warn: possible memory leak of 'result' Note: "buf" is leaked as well. Furthermore, in case of ENOMEM, crypto_finalize_hash_request() was not called which was an error. Fixes: 56f6d5aee88d ("crypto: sun8i-ce - support hash algorithms") Reported-by: kernel test robot Reported-by: Dan Carpenter Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c index 4927a6c82d32..7ca4979190d0 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -263,13 +263,13 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) u32 common; u64 byte_count; __le32 *bf; - void *buf; + void *buf = NULL; int j, i, todo; int nbw = 0; u64 fill, min_fill; __be64 *bebits; __le64 *lebits; - void *result; + void *result = NULL; u64 bs; int digestsize; dma_addr_t addr_res, addr_pad; @@ -286,13 +286,17 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) /* the padding could be up to two block. */ buf = kzalloc(bs * 2, GFP_KERNEL | GFP_DMA); - if (!buf) - return -ENOMEM; + if (!buf) { + err = -ENOMEM; + goto theend; + } bf = (__le32 *)buf; result = kzalloc(digestsize, GFP_KERNEL | GFP_DMA); - if (!result) - return -ENOMEM; + if (!result) { + err = -ENOMEM; + goto theend; + } flow = rctx->flow; chan = &ce->chanlist[flow]; @@ -404,11 +408,11 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_TO_DEVICE); dma_unmap_single(ce->dev, addr_res, digestsize, DMA_FROM_DEVICE); - kfree(buf); memcpy(areq->result, result, algt->alg.hash.halg.digestsize); - kfree(result); theend: + kfree(buf); + kfree(result); crypto_finalize_hash_request(engine, breq, err); return 0; } -- cgit v1.2.3 From 8cbc3448214a9d4534f8381ec23ef5add0ae8d91 Mon Sep 17 00:00:00 2001 From: Thara Gopinath Date: Thu, 19 Nov 2020 10:52:30 -0500 Subject: crypto: qce - Enable support for crypto engine on sdm845 Add support Qualcomm Crypto Engine accelerated encryption and authentication algorithms on sdm845. Reviewed-by: Bjorn Andersson Signed-off-by: Thara Gopinath Signed-off-by: Herbert Xu --- drivers/crypto/qce/core.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c index 5e6717f9bbda..80b75085c265 100644 --- a/drivers/crypto/qce/core.c +++ b/drivers/crypto/qce/core.c @@ -159,7 +159,21 @@ static int qce_check_version(struct qce_device *qce) return -ENODEV; qce->burst_size = QCE_BAM_BURST_SIZE; - qce->pipe_pair_id = 1; + + /* + * Rx and tx pipes are treated as a pair inside CE. + * Pipe pair number depends on the actual BAM dma pipe + * that is used for transfers. The BAM dma pipes are passed + * from the device tree and used to derive the pipe pair + * id in the CE driver as follows. + * BAM dma pipes(rx, tx) CE pipe pair id + * 0,1 0 + * 2,3 1 + * 4,5 2 + * 6,7 3 + * ... + */ + qce->pipe_pair_id = qce->dma.rxchan->chan_id >> 1; dev_dbg(qce->dev, "Crypto device found, version %d.%d.%d\n", major, minor, step); @@ -260,6 +274,7 @@ static int qce_crypto_remove(struct platform_device *pdev) static const struct of_device_id qce_crypto_of_match[] = { { .compatible = "qcom,crypto-v5.1", }, + { .compatible = "qcom,crypto-v5.4", }, {} }; MODULE_DEVICE_TABLE(of, qce_crypto_of_match); -- cgit v1.2.3 From 1148a9654b5a69611d33e14719251c6ec20f5f2c Mon Sep 17 00:00:00 2001 From: Thara Gopinath Date: Thu, 19 Nov 2020 10:52:31 -0500 Subject: crypto: qce - Fix SHA result buffer corruption issues Partial hash was being copied into the final result buffer without the entire message block processed. Depending on how the end user processes this result buffer, errors vary from result buffer corruption to result buffer poisoing. Fix this issue by ensuring that only the final hash value is copied into the result buffer. Reviewed-by: Bjorn Andersson Signed-off-by: Thara Gopinath Signed-off-by: Herbert Xu --- drivers/crypto/qce/sha.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c index 87be96a0b0bb..61c418c12345 100644 --- a/drivers/crypto/qce/sha.c +++ b/drivers/crypto/qce/sha.c @@ -48,7 +48,7 @@ static void qce_ahash_done(void *data) dma_unmap_sg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE); memcpy(rctx->digest, result->auth_iv, digestsize); - if (req->result) + if (req->result && rctx->last_blk) memcpy(req->result, result->auth_iv, digestsize); rctx->byte_count[0] = cpu_to_be32(result->auth_byte_count[0]); -- cgit v1.2.3 From da6d57948fa23f6d3aa7405b9f1c1de96f7e450e Mon Sep 17 00:00:00 2001 From: kernel test robot Date: Fri, 13 Nov 2020 18:14:00 +0100 Subject: crypto: qat - fix excluded_middle.cocci warnings Condition !A || A && B is equivalent to !A || B. Generated by: scripts/coccinelle/misc/excluded_middle.cocci Fixes: b76f0ea01312 ("coccinelle: misc: add excluded_middle.cocci script") CC: Denis Efremov Reported-by: kernel test robot Signed-off-by: kernel test robot Signed-off-by: Julia Lawall Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_dev_mgr.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/qat/qat_common/adf_dev_mgr.c index 29dc2e3d38ff..4c752eed10fe 100644 --- a/drivers/crypto/qat/qat_common/adf_dev_mgr.c +++ b/drivers/crypto/qat/qat_common/adf_dev_mgr.c @@ -151,8 +151,8 @@ int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev, mutex_lock(&table_lock); atomic_set(&accel_dev->ref_count, 0); - /* PF on host or VF on guest */ - if (!accel_dev->is_vf || (accel_dev->is_vf && !pf)) { + /* PF on host or VF on guest - optimized to remove redundant is_vf */ + if (!accel_dev->is_vf || !pf) { struct vf_id_map *map; list_for_each(itr, &accel_table) { @@ -248,7 +248,8 @@ void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev, struct adf_accel_dev *pf) { mutex_lock(&table_lock); - if (!accel_dev->is_vf || (accel_dev->is_vf && !pf)) { + /* PF on host or VF on guest - optimized to remove redundant is_vf */ + if (!accel_dev->is_vf || !pf) { id_map[accel_dev->accel_id] = 0; num_devices--; } else if (accel_dev->is_vf && pf) { -- cgit v1.2.3 From 56c6da16c3631f953fb20d8b7ddccdf493377ad4 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Fri, 20 Nov 2020 17:02:32 +0800 Subject: crypto: hisilicon/trng - add HiSilicon TRNG driver support Move existing char/hw_random/hisi-trng-v2.c to crypto/hisilicon/trng.c. Signed-off-by: Weili Qian Reviewed-by: Zaibo Xu Signed-off-by: Herbert Xu --- arch/arm64/configs/defconfig | 1 + drivers/crypto/hisilicon/Kconfig | 7 +++ drivers/crypto/hisilicon/Makefile | 1 + drivers/crypto/hisilicon/trng/Makefile | 2 + drivers/crypto/hisilicon/trng/trng.c | 99 ++++++++++++++++++++++++++++++++++ 5 files changed, 110 insertions(+) create mode 100644 drivers/crypto/hisilicon/trng/Makefile create mode 100644 drivers/crypto/hisilicon/trng/trng.c (limited to 'drivers/crypto') diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 17a2df6a263e..646c3b41c60a 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1081,6 +1081,7 @@ CONFIG_CRYPTO_DEV_CCREE=m CONFIG_CRYPTO_DEV_HISI_SEC2=m CONFIG_CRYPTO_DEV_HISI_ZIP=m CONFIG_CRYPTO_DEV_HISI_HPRE=m +CONFIG_CRYPTO_DEV_HISI_TRNG=m CONFIG_CMA_SIZE_MBYTES=32 CONFIG_PRINTK_TIME=y CONFIG_DEBUG_INFO=y diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig index 9c3b3ca815e6..2cd3298920fa 100644 --- a/drivers/crypto/hisilicon/Kconfig +++ b/drivers/crypto/hisilicon/Kconfig @@ -71,3 +71,10 @@ config CRYPTO_DEV_HISI_HPRE help Support for HiSilicon HPRE(High Performance RSA Engine) accelerator, which can accelerate RSA and DH algorithms. + +config CRYPTO_DEV_HISI_TRNG + tristate "Support for HISI TRNG Driver" + depends on ARM64 && ACPI + select HW_RANDOM + help + Support for HiSilicon TRNG Driver. diff --git a/drivers/crypto/hisilicon/Makefile b/drivers/crypto/hisilicon/Makefile index 7f5f74c72baa..1e89269a2e4b 100644 --- a/drivers/crypto/hisilicon/Makefile +++ b/drivers/crypto/hisilicon/Makefile @@ -5,3 +5,4 @@ obj-$(CONFIG_CRYPTO_DEV_HISI_SEC2) += sec2/ obj-$(CONFIG_CRYPTO_DEV_HISI_QM) += hisi_qm.o hisi_qm-objs = qm.o sgl.o obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += zip/ +obj-$(CONFIG_CRYPTO_DEV_HISI_TRNG) += trng/ diff --git a/drivers/crypto/hisilicon/trng/Makefile b/drivers/crypto/hisilicon/trng/Makefile new file mode 100644 index 000000000000..d909079f351c --- /dev/null +++ b/drivers/crypto/hisilicon/trng/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CRYPTO_DEV_HISI_TRNG) += hisi-trng-v2.o +hisi-trng-v2-objs = trng.o diff --git a/drivers/crypto/hisilicon/trng/trng.c b/drivers/crypto/hisilicon/trng/trng.c new file mode 100644 index 000000000000..6a65b8232ce0 --- /dev/null +++ b/drivers/crypto/hisilicon/trng/trng.c @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 HiSilicon Limited. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HISI_TRNG_REG 0x00F0 +#define HISI_TRNG_BYTES 4 +#define HISI_TRNG_QUALITY 512 +#define SLEEP_US 10 +#define TIMEOUT_US 10000 + +struct hisi_trng { + void __iomem *base; + struct hwrng rng; +}; + +static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait) +{ + struct hisi_trng *trng; + int currsize = 0; + u32 val = 0; + u32 ret; + + trng = container_of(rng, struct hisi_trng, rng); + + do { + ret = readl_poll_timeout(trng->base + HISI_TRNG_REG, val, + val, SLEEP_US, TIMEOUT_US); + if (ret) + return currsize; + + if (max - currsize >= HISI_TRNG_BYTES) { + memcpy(buf + currsize, &val, HISI_TRNG_BYTES); + currsize += HISI_TRNG_BYTES; + if (currsize == max) + return currsize; + continue; + } + + /* copy remaining bytes */ + memcpy(buf + currsize, &val, max - currsize); + currsize = max; + } while (currsize < max); + + return currsize; +} + +static int hisi_trng_probe(struct platform_device *pdev) +{ + struct hisi_trng *trng; + int ret; + + trng = devm_kzalloc(&pdev->dev, sizeof(*trng), GFP_KERNEL); + if (!trng) + return -ENOMEM; + + trng->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(trng->base)) + return PTR_ERR(trng->base); + + trng->rng.name = pdev->name; + trng->rng.read = hisi_trng_read; + trng->rng.quality = HISI_TRNG_QUALITY; + + ret = devm_hwrng_register(&pdev->dev, &trng->rng); + if (ret) + dev_err(&pdev->dev, "failed to register hwrng!\n"); + + return ret; +} + +static const struct acpi_device_id hisi_trng_acpi_match[] = { + { "HISI02B3", 0 }, + { } +}; +MODULE_DEVICE_TABLE(acpi, hisi_trng_acpi_match); + +static struct platform_driver hisi_trng_driver = { + .probe = hisi_trng_probe, + .driver = { + .name = "hisi-trng-v2", + .acpi_match_table = ACPI_PTR(hisi_trng_acpi_match), + }, +}; + +module_platform_driver(hisi_trng_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Weili Qian "); +MODULE_AUTHOR("Zaibo Xu "); +MODULE_DESCRIPTION("HiSilicon true random number generator V2 driver"); -- cgit v1.2.3 From e4d9d10ef4be0366316b3114593e4becf5b98a49 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Fri, 20 Nov 2020 17:02:33 +0800 Subject: crypto: hisilicon/trng - add support for PRNG This patch adds support for pseudo random number generator(PRNG) in Crypto subsystem. Signed-off-by: Weili Qian Reviewed-by: Zaibo Xu Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/Kconfig | 1 + drivers/crypto/hisilicon/trng/trng.c | 241 ++++++++++++++++++++++++++++++++++- 2 files changed, 239 insertions(+), 3 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig index 2cd3298920fa..843192666dc3 100644 --- a/drivers/crypto/hisilicon/Kconfig +++ b/drivers/crypto/hisilicon/Kconfig @@ -76,5 +76,6 @@ config CRYPTO_DEV_HISI_TRNG tristate "Support for HISI TRNG Driver" depends on ARM64 && ACPI select HW_RANDOM + select CRYPTO_RNG help Support for HiSilicon TRNG Driver. diff --git a/drivers/crypto/hisilicon/trng/trng.c b/drivers/crypto/hisilicon/trng/trng.c index 6a65b8232ce0..a5033cf09e2b 100644 --- a/drivers/crypto/hisilicon/trng/trng.c +++ b/drivers/crypto/hisilicon/trng/trng.c @@ -2,26 +2,181 @@ /* Copyright (c) 2019 HiSilicon Limited. */ #include +#include #include #include #include #include #include +#include #include +#include #include #include +#include #define HISI_TRNG_REG 0x00F0 #define HISI_TRNG_BYTES 4 #define HISI_TRNG_QUALITY 512 #define SLEEP_US 10 #define TIMEOUT_US 10000 +#define SW_DRBG_NUM_SHIFT 2 +#define SW_DRBG_KEY_BASE 0x082C +#define SW_DRBG_SEED(n) (SW_DRBG_KEY_BASE - ((n) << SW_DRBG_NUM_SHIFT)) +#define SW_DRBG_SEED_REGS_NUM 12 +#define SW_DRBG_SEED_SIZE 48 +#define SW_DRBG_BLOCKS 0x0830 +#define SW_DRBG_INIT 0x0834 +#define SW_DRBG_GEN 0x083c +#define SW_DRBG_STATUS 0x0840 +#define SW_DRBG_BLOCKS_NUM 4095 +#define SW_DRBG_DATA_BASE 0x0850 +#define SW_DRBG_DATA_NUM 4 +#define SW_DRBG_DATA(n) (SW_DRBG_DATA_BASE - ((n) << SW_DRBG_NUM_SHIFT)) +#define SW_DRBG_BYTES 16 +#define SW_DRBG_ENABLE_SHIFT 12 +#define SEED_SHIFT_24 24 +#define SEED_SHIFT_16 16 +#define SEED_SHIFT_8 8 + +struct hisi_trng_list { + struct mutex lock; + struct list_head list; + bool is_init; +}; struct hisi_trng { void __iomem *base; + struct hisi_trng_list *trng_list; + struct list_head list; struct hwrng rng; + bool is_used; + struct mutex mutex; +}; + +struct hisi_trng_ctx { + struct hisi_trng *trng; }; +static atomic_t trng_active_devs; +static struct hisi_trng_list trng_devices; + +static void hisi_trng_set_seed(struct hisi_trng *trng, const u8 *seed) +{ + u32 val, seed_reg, i; + + for (i = 0; i < SW_DRBG_SEED_SIZE; + i += SW_DRBG_SEED_SIZE / SW_DRBG_SEED_REGS_NUM) { + val = seed[i] << SEED_SHIFT_24; + val |= seed[i + 1UL] << SEED_SHIFT_16; + val |= seed[i + 2UL] << SEED_SHIFT_8; + val |= seed[i + 3UL]; + + seed_reg = (i >> SW_DRBG_NUM_SHIFT) % SW_DRBG_SEED_REGS_NUM; + writel(val, trng->base + SW_DRBG_SEED(seed_reg)); + } +} + +static int hisi_trng_seed(struct crypto_rng *tfm, const u8 *seed, + unsigned int slen) +{ + struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); + struct hisi_trng *trng = ctx->trng; + u32 val = 0; + int ret = 0; + + if (slen < SW_DRBG_SEED_SIZE) { + pr_err("slen(%u) is not matched with trng(%d)\n", slen, + SW_DRBG_SEED_SIZE); + return -EINVAL; + } + + writel(0x0, trng->base + SW_DRBG_BLOCKS); + hisi_trng_set_seed(trng, seed); + + writel(SW_DRBG_BLOCKS_NUM | (0x1 << SW_DRBG_ENABLE_SHIFT), + trng->base + SW_DRBG_BLOCKS); + writel(0x1, trng->base + SW_DRBG_INIT); + + ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, + val, val & BIT(0), SLEEP_US, TIMEOUT_US); + if (ret) + pr_err("fail to init trng(%d)\n", ret); + + return ret; +} + +static int hisi_trng_generate(struct crypto_rng *tfm, const u8 *src, + unsigned int slen, u8 *dstn, unsigned int dlen) +{ + struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); + struct hisi_trng *trng = ctx->trng; + u32 data[SW_DRBG_DATA_NUM]; + u32 currsize = 0; + u32 val = 0; + int ret; + u32 i; + + if (dlen > SW_DRBG_BLOCKS_NUM * SW_DRBG_BYTES || dlen == 0) { + pr_err("dlen(%d) exceeds limit(%d)!\n", dlen, + SW_DRBG_BLOCKS_NUM * SW_DRBG_BYTES); + return -EINVAL; + } + + do { + ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, + val, val & BIT(1), SLEEP_US, TIMEOUT_US); + if (ret) { + pr_err("fail to generate random number(%d)!\n", ret); + break; + } + + for (i = 0; i < SW_DRBG_DATA_NUM; i++) + data[i] = readl(trng->base + SW_DRBG_DATA(i)); + + if (dlen - currsize >= SW_DRBG_BYTES) { + memcpy(dstn + currsize, data, SW_DRBG_BYTES); + currsize += SW_DRBG_BYTES; + } else { + memcpy(dstn + currsize, data, dlen - currsize); + currsize = dlen; + } + + writel(0x1, trng->base + SW_DRBG_GEN); + } while (currsize < dlen); + + return ret; +} + +static int hisi_trng_init(struct crypto_tfm *tfm) +{ + struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); + struct hisi_trng *trng; + int ret = -EBUSY; + + mutex_lock(&trng_devices.lock); + list_for_each_entry(trng, &trng_devices.list, list) { + if (!trng->is_used) { + trng->is_used = true; + ctx->trng = trng; + ret = 0; + break; + } + } + mutex_unlock(&trng_devices.lock); + + return ret; +} + +static void hisi_trng_exit(struct crypto_tfm *tfm) +{ + struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); + + mutex_lock(&trng_devices.lock); + ctx->trng->is_used = false; + mutex_unlock(&trng_devices.lock); +} + static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait) { struct hisi_trng *trng; @@ -53,6 +208,42 @@ static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait) return currsize; } +static struct rng_alg hisi_trng_alg = { + .generate = hisi_trng_generate, + .seed = hisi_trng_seed, + .seedsize = SW_DRBG_SEED_SIZE, + .base = { + .cra_name = "stdrng", + .cra_driver_name = "hisi_stdrng", + .cra_priority = 300, + .cra_ctxsize = sizeof(struct hisi_trng_ctx), + .cra_module = THIS_MODULE, + .cra_init = hisi_trng_init, + .cra_exit = hisi_trng_exit, + }, +}; + +static void hisi_trng_add_to_list(struct hisi_trng *trng) +{ + mutex_lock(&trng_devices.lock); + list_add_tail(&trng->list, &trng_devices.list); + mutex_unlock(&trng_devices.lock); +} + +static int hisi_trng_del_from_list(struct hisi_trng *trng) +{ + int ret = -EBUSY; + + mutex_lock(&trng_devices.lock); + if (!trng->is_used) { + list_del(&trng->list); + ret = 0; + } + mutex_unlock(&trng_devices.lock); + + return ret; +} + static int hisi_trng_probe(struct platform_device *pdev) { struct hisi_trng *trng; @@ -62,19 +253,62 @@ static int hisi_trng_probe(struct platform_device *pdev) if (!trng) return -ENOMEM; + platform_set_drvdata(pdev, trng); + trng->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(trng->base)) return PTR_ERR(trng->base); + trng->is_used = false; + if (!trng_devices.is_init) { + INIT_LIST_HEAD(&trng_devices.list); + mutex_init(&trng_devices.lock); + trng_devices.is_init = true; + } + + hisi_trng_add_to_list(trng); + if (atomic_add_return(1, &trng_active_devs) == 1) { + ret = crypto_register_rng(&hisi_trng_alg); + if (ret) { + dev_err(&pdev->dev, + "failed to register crypto(%d)\n", ret); + atomic_sub_return(1, &trng_active_devs); + goto err_remove_from_list; + } + } + trng->rng.name = pdev->name; trng->rng.read = hisi_trng_read; trng->rng.quality = HISI_TRNG_QUALITY; - ret = devm_hwrng_register(&pdev->dev, &trng->rng); - if (ret) - dev_err(&pdev->dev, "failed to register hwrng!\n"); + if (ret) { + dev_err(&pdev->dev, "failed to register hwrng: %d!\n", ret); + goto err_crypto_unregister; + } return ret; + +err_crypto_unregister: + if (atomic_sub_return(1, &trng_active_devs) == 0) + crypto_unregister_rng(&hisi_trng_alg); + +err_remove_from_list: + hisi_trng_del_from_list(trng); + return ret; +} + +static int hisi_trng_remove(struct platform_device *pdev) +{ + struct hisi_trng *trng = platform_get_drvdata(pdev); + + /* Wait until the task is finished */ + while (hisi_trng_del_from_list(trng)) + ; + + if (atomic_sub_return(1, &trng_active_devs) == 0) + crypto_unregister_rng(&hisi_trng_alg); + + return 0; } static const struct acpi_device_id hisi_trng_acpi_match[] = { @@ -85,6 +319,7 @@ MODULE_DEVICE_TABLE(acpi, hisi_trng_acpi_match); static struct platform_driver hisi_trng_driver = { .probe = hisi_trng_probe, + .remove = hisi_trng_remove, .driver = { .name = "hisi-trng-v2", .acpi_match_table = ACPI_PTR(hisi_trng_acpi_match), -- cgit v1.2.3 From abc6146aba40ea3b8996773e6ad0e60fd70f7a3f Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Fri, 20 Nov 2020 12:34:56 -0600 Subject: crypto: ccree - Fix fall-through warnings for Clang In preparation to enable -Wimplicit-fallthrough for Clang, fix multiple warnings by explicitly adding multiple break statements instead of letting the code fall through to the next case. Link: https://github.com/KSPP/linux/issues/115 Signed-off-by: Gustavo A. R. Silva Acked-by: Gilad Ben-Yossef Signed-off-by: Herbert Xu --- drivers/crypto/ccree/cc_cipher.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c index dafa6577a845..cdfee501fbd9 100644 --- a/drivers/crypto/ccree/cc_cipher.c +++ b/drivers/crypto/ccree/cc_cipher.c @@ -97,6 +97,7 @@ static int validate_keys_sizes(struct cc_cipher_ctx *ctx_p, u32 size) case S_DIN_to_SM4: if (size == SM4_KEY_SIZE) return 0; + break; default: break; } @@ -139,9 +140,11 @@ static int validate_data_size(struct cc_cipher_ctx *ctx_p, case DRV_CIPHER_CBC: if (IS_ALIGNED(size, SM4_BLOCK_SIZE)) return 0; + break; default: break; } + break; default: break; } -- cgit v1.2.3 From d4f9afb23378f50e40cd3bc8aee35679bfd1d27b Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 21 Nov 2020 08:31:31 +0100 Subject: crypto: cavium/zip - Use dma_set_mask_and_coherent to simplify code 'pci_set_dma_mask()' + 'pci_set_consistent_dma_mask()' can be replaced by an equivalent 'dma_set_mask_and_coherent()' which is much less verbose. Signed-off-by: Christophe JAILLET Signed-off-by: Herbert Xu --- drivers/crypto/cavium/zip/zip_main.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/cavium/zip/zip_main.c b/drivers/crypto/cavium/zip/zip_main.c index d35216e2f6cd..812b4ac9afd6 100644 --- a/drivers/crypto/cavium/zip/zip_main.c +++ b/drivers/crypto/cavium/zip/zip_main.c @@ -263,15 +263,9 @@ static int zip_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto err_disable_device; } - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); if (err) { - dev_err(dev, "Unable to get usable DMA configuration\n"); - goto err_release_regions; - } - - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); - if (err) { - dev_err(dev, "Unable to get 48-bit DMA for allocations\n"); + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); goto err_release_regions; } -- cgit v1.2.3 From 7f6c383b9505f7f9ec75831b154be97e7df49cac Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 21 Nov 2020 08:49:16 +0100 Subject: crypto: marvell/octeontx - Use dma_set_mask_and_coherent to simplify code 'pci_set_dma_mask()' + 'pci_set_consistent_dma_mask()' can be replaced by an equivalent 'dma_set_mask_and_coherent()' which is much less verbose. Signed-off-by: Christophe JAILLET Signed-off-by: Herbert Xu --- drivers/crypto/marvell/octeontx/otx_cptpf_main.c | 10 ++-------- drivers/crypto/marvell/octeontx/otx_cptvf_main.c | 10 ++-------- 2 files changed, 4 insertions(+), 16 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_main.c b/drivers/crypto/marvell/octeontx/otx_cptpf_main.c index 34bb3063eb70..14a42559f81d 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptpf_main.c +++ b/drivers/crypto/marvell/octeontx/otx_cptpf_main.c @@ -212,15 +212,9 @@ static int otx_cpt_probe(struct pci_dev *pdev, goto err_disable_device; } - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); if (err) { - dev_err(dev, "Unable to get usable DMA configuration\n"); - goto err_release_regions; - } - - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); - if (err) { - dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n"); + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); goto err_release_regions; } diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c index 228fe8e47e0e..c076d0b3ad5f 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c @@ -804,15 +804,9 @@ static int otx_cptvf_probe(struct pci_dev *pdev, dev_err(dev, "PCI request regions failed 0x%x\n", err); goto disable_device; } - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); if (err) { - dev_err(dev, "Unable to get usable DMA configuration\n"); - goto release_regions; - } - - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); - if (err) { - dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n"); + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); goto release_regions; } -- cgit v1.2.3 From aeb4d8c0f855304c158195ebae3917e244e63e57 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 21 Nov 2020 08:56:47 +0100 Subject: crypto: cavium - Use dma_set_mask_and_coherent to simplify code 'pci_set_dma_mask()' + 'pci_set_consistent_dma_mask()' can be replaced by an equivalent 'dma_set_mask_and_coherent()' which is much less verbose. Signed-off-by: Christophe JAILLET Signed-off-by: Herbert Xu --- drivers/crypto/cavium/cpt/cptpf_main.c | 10 ++-------- drivers/crypto/cavium/cpt/cptvf_main.c | 10 ++-------- 2 files changed, 4 insertions(+), 16 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/cavium/cpt/cptpf_main.c b/drivers/crypto/cavium/cpt/cptpf_main.c index 781949027451..24d63bdc5dd2 100644 --- a/drivers/crypto/cavium/cpt/cptpf_main.c +++ b/drivers/crypto/cavium/cpt/cptpf_main.c @@ -569,15 +569,9 @@ static int cpt_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto cpt_err_disable_device; } - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); if (err) { - dev_err(dev, "Unable to get usable DMA configuration\n"); - goto cpt_err_release_regions; - } - - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); - if (err) { - dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n"); + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); goto cpt_err_release_regions; } diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c index a15245992cf9..f016448e43bb 100644 --- a/drivers/crypto/cavium/cpt/cptvf_main.c +++ b/drivers/crypto/cavium/cpt/cptvf_main.c @@ -687,15 +687,9 @@ static int cptvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } /* Mark as VF driver */ cptvf->flags |= CPT_FLAG_VF_DRIVER; - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); if (err) { - dev_err(dev, "Unable to get usable DMA configuration\n"); - goto cptvf_err_release_regions; - } - - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); - if (err) { - dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n"); + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); goto cptvf_err_release_regions; } -- cgit v1.2.3 From 05c2a705917b77e0915cca3551583583f4eafcf8 Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Sun, 22 Nov 2020 09:51:53 +0200 Subject: crypto: ccree - rework cache parameters handling Rework the setting of DMA cache parameters, program more appropriate values and explicitly set sharability domain. Signed-off-by: Gilad Ben-Yossef Signed-off-by: Herbert Xu --- drivers/crypto/ccree/cc_driver.c | 75 +++++++++++++++++++++++++++++++--------- drivers/crypto/ccree/cc_driver.h | 6 ++-- drivers/crypto/ccree/cc_pm.c | 2 +- 3 files changed, 63 insertions(+), 20 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c index 6f519d3e896c..d0e59e942568 100644 --- a/drivers/crypto/ccree/cc_driver.c +++ b/drivers/crypto/ccree/cc_driver.c @@ -100,6 +100,57 @@ static const struct of_device_id arm_ccree_dev_of_match[] = { }; MODULE_DEVICE_TABLE(of, arm_ccree_dev_of_match); +static void init_cc_cache_params(struct cc_drvdata *drvdata) +{ + struct device *dev = drvdata_to_dev(drvdata); + u32 cache_params, ace_const, val, mask; + + /* compute CC_AXIM_CACHE_PARAMS */ + cache_params = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS)); + dev_dbg(dev, "Cache params previous: 0x%08X\n", cache_params); + + /* non cached or write-back, write allocate */ + val = drvdata->coherent ? 0xb : 0x2; + + mask = CC_GENMASK(CC_AXIM_CACHE_PARAMS_AWCACHE); + cache_params &= ~mask; + cache_params |= FIELD_PREP(mask, val); + + mask = CC_GENMASK(CC_AXIM_CACHE_PARAMS_AWCACHE_LAST); + cache_params &= ~mask; + cache_params |= FIELD_PREP(mask, val); + + mask = CC_GENMASK(CC_AXIM_CACHE_PARAMS_ARCACHE); + cache_params &= ~mask; + cache_params |= FIELD_PREP(mask, val); + + drvdata->cache_params = cache_params; + + dev_dbg(dev, "Cache params current: 0x%08X\n", cache_params); + + if (drvdata->hw_rev <= CC_HW_REV_710) + return; + + /* compute CC_AXIM_ACE_CONST */ + ace_const = cc_ioread(drvdata, CC_REG(AXIM_ACE_CONST)); + dev_dbg(dev, "ACE-const previous: 0x%08X\n", ace_const); + + /* system or outer-sharable */ + val = drvdata->coherent ? 0x2 : 0x3; + + mask = CC_GENMASK(CC_AXIM_ACE_CONST_ARDOMAIN); + ace_const &= ~mask; + ace_const |= FIELD_PREP(mask, val); + + mask = CC_GENMASK(CC_AXIM_ACE_CONST_AWDOMAIN); + ace_const &= ~mask; + ace_const |= FIELD_PREP(mask, val); + + dev_dbg(dev, "ACE-const current: 0x%08X\n", ace_const); + + drvdata->ace_const = ace_const; +} + static u32 cc_read_idr(struct cc_drvdata *drvdata, const u32 *idr_offsets) { int i; @@ -218,9 +269,9 @@ bool cc_wait_for_reset_completion(struct cc_drvdata *drvdata) return false; } -int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe) +int init_cc_regs(struct cc_drvdata *drvdata) { - unsigned int val, cache_params; + unsigned int val; struct device *dev = drvdata_to_dev(drvdata); /* Unmask all AXI interrupt sources AXI_CFG1 register */ @@ -245,19 +296,9 @@ int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe) cc_iowrite(drvdata, CC_REG(HOST_IMR), ~val); - cache_params = (drvdata->coherent ? CC_COHERENT_CACHE_PARAMS : 0x0); - - val = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS)); - - if (is_probe) - dev_dbg(dev, "Cache params previous: 0x%08X\n", val); - - cc_iowrite(drvdata, CC_REG(AXIM_CACHE_PARAMS), cache_params); - val = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS)); - - if (is_probe) - dev_dbg(dev, "Cache params current: 0x%08X (expect: 0x%08X)\n", - val, cache_params); + cc_iowrite(drvdata, CC_REG(AXIM_CACHE_PARAMS), drvdata->cache_params); + if (drvdata->hw_rev >= CC_HW_REV_712) + cc_iowrite(drvdata, CC_REG(AXIM_ACE_CONST), drvdata->ace_const); return 0; } @@ -445,7 +486,9 @@ static int init_cc_resources(struct platform_device *plat_dev) } dev_dbg(dev, "Registered to IRQ: %d\n", irq); - rc = init_cc_regs(new_drvdata, true); + init_cc_cache_params(new_drvdata); + + rc = init_cc_regs(new_drvdata); if (rc) { dev_err(dev, "init_cc_regs failed\n"); goto post_pm_err; diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h index ed2b2f13a256..5f1d4602eb8f 100644 --- a/drivers/crypto/ccree/cc_driver.h +++ b/drivers/crypto/ccree/cc_driver.h @@ -50,8 +50,6 @@ enum cc_std_body { CC_STD_ALL = 0x3 }; -#define CC_COHERENT_CACHE_PARAMS 0xEEE - #define CC_PINS_FULL 0x0 #define CC_PINS_SLIM 0x9F @@ -156,6 +154,8 @@ struct cc_drvdata { int std_bodies; bool sec_disabled; u32 comp_mask; + u32 cache_params; + u32 ace_const; }; struct cc_crypto_alg { @@ -206,7 +206,7 @@ static inline void dump_byte_array(const char *name, const u8 *the_array, } bool cc_wait_for_reset_completion(struct cc_drvdata *drvdata); -int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe); +int init_cc_regs(struct cc_drvdata *drvdata); void fini_cc_regs(struct cc_drvdata *drvdata); unsigned int cc_get_default_hash_len(struct cc_drvdata *drvdata); diff --git a/drivers/crypto/ccree/cc_pm.c b/drivers/crypto/ccree/cc_pm.c index 3c65bf070c90..d5421b0c6831 100644 --- a/drivers/crypto/ccree/cc_pm.c +++ b/drivers/crypto/ccree/cc_pm.c @@ -45,7 +45,7 @@ static int cc_pm_resume(struct device *dev) } cc_iowrite(drvdata, CC_REG(HOST_POWER_DOWN_EN), POWER_DOWN_DISABLE); - rc = init_cc_regs(drvdata, false); + rc = init_cc_regs(drvdata); if (rc) { dev_err(dev, "init_cc_regs (%x)\n", rc); return rc; -- cgit v1.2.3 From f2d4576a9d38f0a35bc1a5436f5e6e15b6c46aa4 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 27 Nov 2020 17:23:29 +1100 Subject: crypto: cpt - Fix sparse warnings in cptpf This patch fixes a few sparse warnings that were missed in the last round. Signed-off-by: Herbert Xu --- drivers/crypto/cavium/cpt/cptpf_main.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/cavium/cpt/cptpf_main.c b/drivers/crypto/cavium/cpt/cptpf_main.c index 24d63bdc5dd2..711b1acdd4e0 100644 --- a/drivers/crypto/cavium/cpt/cptpf_main.c +++ b/drivers/crypto/cavium/cpt/cptpf_main.c @@ -244,7 +244,7 @@ cpt_init_fail: struct ucode_header { u8 version[CPT_UCODE_VERSION_SZ]; - u32 code_length; + __be32 code_length; u32 data_length; u64 sram_address; }; @@ -288,10 +288,10 @@ static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae) /* Byte swap 64-bit */ for (j = 0; j < (mcode->code_size / 8); j++) - ((u64 *)mcode->code)[j] = cpu_to_be64(((u64 *)mcode->code)[j]); + ((__be64 *)mcode->code)[j] = cpu_to_be64(((u64 *)mcode->code)[j]); /* MC needs 16-bit swap */ for (j = 0; j < (mcode->code_size / 2); j++) - ((u16 *)mcode->code)[j] = cpu_to_be16(((u16 *)mcode->code)[j]); + ((__be16 *)mcode->code)[j] = cpu_to_be16(((u16 *)mcode->code)[j]); dev_dbg(dev, "mcode->code_size = %u\n", mcode->code_size); dev_dbg(dev, "mcode->is_ae = %u\n", mcode->is_ae); -- cgit v1.2.3 From 88574332451380f4b51f6ca88ab9810e714bfb9b Mon Sep 17 00:00:00 2001 From: Mike Healy Date: Thu, 26 Nov 2020 11:51:48 +0000 Subject: crypto: keembay - Add support for Keem Bay OCS AES/SM4 Add support for the AES/SM4 crypto engine included in the Offload and Crypto Subsystem (OCS) of the Intel Keem Bay SoC, thus enabling hardware-acceleration for the following transformations: - ecb(aes), cbc(aes), ctr(aes), cts(cbc(aes)), gcm(aes) and cbc(aes); supported for 128-bit and 256-bit keys. - ecb(sm4), cbc(sm4), ctr(sm4), cts(cbc(sm4)), gcm(sm4) and cbc(sm4); supported for 128-bit keys. The driver passes crypto manager self-tests, including the extra tests (CRYPTO_MANAGER_EXTRA_TESTS=y). Signed-off-by: Mike Healy Co-developed-by: Daniele Alessandrelli Signed-off-by: Daniele Alessandrelli Acked-by: Mark Gross Signed-off-by: Herbert Xu --- MAINTAINERS | 10 + drivers/crypto/Kconfig | 2 + drivers/crypto/Makefile | 1 + drivers/crypto/keembay/Kconfig | 39 + drivers/crypto/keembay/Makefile | 5 + drivers/crypto/keembay/keembay-ocs-aes-core.c | 1713 +++++++++++++++++++++++++ drivers/crypto/keembay/ocs-aes.c | 1489 +++++++++++++++++++++ drivers/crypto/keembay/ocs-aes.h | 129 ++ 8 files changed, 3388 insertions(+) create mode 100644 drivers/crypto/keembay/Kconfig create mode 100644 drivers/crypto/keembay/Makefile create mode 100644 drivers/crypto/keembay/keembay-ocs-aes-core.c create mode 100644 drivers/crypto/keembay/ocs-aes.c create mode 100644 drivers/crypto/keembay/ocs-aes.h (limited to 'drivers/crypto') diff --git a/MAINTAINERS b/MAINTAINERS index a835a74d4b29..3f81dd8e4004 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8958,6 +8958,16 @@ M: Deepak Saxena S: Maintained F: drivers/char/hw_random/ixp4xx-rng.c +INTEL KEEM BAY OCS AES/SM4 CRYPTO DRIVER +M: Daniele Alessandrelli +S: Maintained +F: Documentation/devicetree/bindings/crypto/intel,keembay-ocs-aes.yaml +F: drivers/crypto/keembay/Kconfig +F: drivers/crypto/keembay/Makefile +F: drivers/crypto/keembay/keembay-ocs-aes-core.c +F: drivers/crypto/keembay/ocs-aes.c +F: drivers/crypto/keembay/ocs-aes.h + INTEL MANAGEMENT ENGINE (mei) M: Tomas Winkler L: linux-kernel@vger.kernel.org diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 129bbdc4993b..808eecef0f32 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -900,4 +900,6 @@ config CRYPTO_DEV_SA2UL used for crypto offload. Select this if you want to use hardware acceleration for cryptographic algorithms on these devices. +source "drivers/crypto/keembay/Kconfig" + endif # CRYPTO_HW diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 53fc115cf459..fff9a70348e1 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -51,3 +51,4 @@ obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) += axis/ obj-$(CONFIG_CRYPTO_DEV_ZYNQMP_AES) += xilinx/ obj-y += hisilicon/ obj-$(CONFIG_CRYPTO_DEV_AMLOGIC_GXL) += amlogic/ +obj-y += keembay/ diff --git a/drivers/crypto/keembay/Kconfig b/drivers/crypto/keembay/Kconfig new file mode 100644 index 000000000000..3c16797b25b9 --- /dev/null +++ b/drivers/crypto/keembay/Kconfig @@ -0,0 +1,39 @@ +config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 + tristate "Support for Intel Keem Bay OCS AES/SM4 HW acceleration" + depends on OF || COMPILE_TEST + select CRYPTO_SKCIPHER + select CRYPTO_AEAD + select CRYPTO_ENGINE + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) AES and + SM4 cihper hardware acceleration for use with Crypto API. + + Provides HW acceleration for the following transformations: + cbc(aes), ctr(aes), ccm(aes), gcm(aes), cbc(sm4), ctr(sm4), ccm(sm4) + and gcm(sm4). + + Optionally, support for the following transformations can also be + enabled: ecb(aes), cts(cbc(aes)), ecb(sm4) and cts(cbc(sm4)). + +config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB + bool "Support for Intel Keem Bay OCS AES/SM4 ECB HW acceleration" + depends on CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) + AES/SM4 ECB mode hardware acceleration for use with Crypto API. + + Provides OCS version of ecb(aes) and ecb(sm4) + + Intel does not recommend use of ECB mode with AES/SM4. + +config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS + bool "Support for Intel Keem Bay OCS AES/SM4 CTS HW acceleration" + depends on CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) + AES/SM4 CBC with CTS mode hardware acceleration for use with + Crypto API. + + Provides OCS version of cts(cbc(aes)) and cts(cbc(sm4)). + + Intel does not recommend use of CTS mode with AES/SM4. diff --git a/drivers/crypto/keembay/Makefile b/drivers/crypto/keembay/Makefile new file mode 100644 index 000000000000..f21e2c4ab3b3 --- /dev/null +++ b/drivers/crypto/keembay/Makefile @@ -0,0 +1,5 @@ +# +# Makefile for Intel Keem Bay OCS Crypto API Linux drivers +# +obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4) += keembay-ocs-aes.o +keembay-ocs-aes-objs := keembay-ocs-aes-core.o ocs-aes.o diff --git a/drivers/crypto/keembay/keembay-ocs-aes-core.c b/drivers/crypto/keembay/keembay-ocs-aes-core.c new file mode 100644 index 000000000000..b6b25d994af3 --- /dev/null +++ b/drivers/crypto/keembay/keembay-ocs-aes-core.c @@ -0,0 +1,1713 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS AES Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include "ocs-aes.h" + +#define KMB_OCS_PRIORITY 350 +#define DRV_NAME "keembay-ocs-aes" + +#define OCS_AES_MIN_KEY_SIZE 16 +#define OCS_AES_MAX_KEY_SIZE 32 +#define OCS_AES_KEYSIZE_128 16 +#define OCS_AES_KEYSIZE_192 24 +#define OCS_AES_KEYSIZE_256 32 +#define OCS_SM4_KEY_SIZE 16 + +/** + * struct ocs_aes_tctx - OCS AES Transform context + * @engine_ctx: Engine context. + * @aes_dev: The OCS AES device. + * @key: AES/SM4 key. + * @key_len: The length (in bytes) of @key. + * @cipher: OCS cipher to use (either AES or SM4). + * @sw_cipher: The cipher to use as fallback. + * @use_fallback: Whether or not fallback cipher should be used. + */ +struct ocs_aes_tctx { + struct crypto_engine_ctx engine_ctx; + struct ocs_aes_dev *aes_dev; + u8 key[OCS_AES_KEYSIZE_256]; + unsigned int key_len; + enum ocs_cipher cipher; + union { + struct crypto_sync_skcipher *sk; + struct crypto_aead *aead; + } sw_cipher; + bool use_fallback; +}; + +/** + * struct ocs_aes_rctx - OCS AES Request context. + * @instruction: Instruction to be executed (encrypt / decrypt). + * @mode: Mode to use (ECB, CBC, CTR, CCm, GCM, CTS) + * @src_nents: Number of source SG entries. + * @dst_nents: Number of destination SG entries. + * @src_dma_count: The number of DMA-mapped entries of the source SG. + * @dst_dma_count: The number of DMA-mapped entries of the destination SG. + * @in_place: Whether or not this is an in place request, i.e., + * src_sg == dst_sg. + * @src_dll: OCS DMA linked list for input data. + * @dst_dll: OCS DMA linked list for output data. + * @last_ct_blk: Buffer to hold last cipher text block (only used in CBC + * mode). + * @cts_swap: Whether or not CTS swap must be performed. + * @aad_src_dll: OCS DMA linked list for input AAD data. + * @aad_dst_dll: OCS DMA linked list for output AAD data. + * @in_tag: Buffer to hold input encrypted tag (only used for + * CCM/GCM decrypt). + * @out_tag: Buffer to hold output encrypted / decrypted tag (only + * used for GCM encrypt / decrypt). + */ +struct ocs_aes_rctx { + /* Fields common across all modes. */ + enum ocs_instruction instruction; + enum ocs_mode mode; + int src_nents; + int dst_nents; + int src_dma_count; + int dst_dma_count; + bool in_place; + struct ocs_dll_desc src_dll; + struct ocs_dll_desc dst_dll; + + /* CBC specific */ + u8 last_ct_blk[AES_BLOCK_SIZE]; + + /* CTS specific */ + int cts_swap; + + /* CCM/GCM specific */ + struct ocs_dll_desc aad_src_dll; + struct ocs_dll_desc aad_dst_dll; + u8 in_tag[AES_BLOCK_SIZE]; + + /* GCM specific */ + u8 out_tag[AES_BLOCK_SIZE]; +}; + +/* Driver data. */ +struct ocs_aes_drv { + struct list_head dev_list; + spinlock_t lock; /* Protects dev_list. */ +}; + +static struct ocs_aes_drv ocs_aes = { + .dev_list = LIST_HEAD_INIT(ocs_aes.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(ocs_aes.lock), +}; + +static struct ocs_aes_dev *kmb_ocs_aes_find_dev(struct ocs_aes_tctx *tctx) +{ + struct ocs_aes_dev *aes_dev; + + spin_lock(&ocs_aes.lock); + + if (tctx->aes_dev) { + aes_dev = tctx->aes_dev; + goto exit; + } + + /* Only a single OCS device available */ + aes_dev = list_first_entry(&ocs_aes.dev_list, struct ocs_aes_dev, list); + tctx->aes_dev = aes_dev; + +exit: + spin_unlock(&ocs_aes.lock); + + return aes_dev; +} + +/* + * Ensure key is 128-bit or 256-bit for AES or 128-bit for SM4 and an actual + * key is being passed in. + * + * Return: 0 if key is valid, -EINVAL otherwise. + */ +static int check_key(const u8 *in_key, size_t key_len, enum ocs_cipher cipher) +{ + if (!in_key) + return -EINVAL; + + /* For AES, only 128-byte or 256-byte keys are supported. */ + if (cipher == OCS_AES && (key_len == OCS_AES_KEYSIZE_128 || + key_len == OCS_AES_KEYSIZE_256)) + return 0; + + /* For SM4, only 128-byte keys are supported. */ + if (cipher == OCS_SM4 && key_len == OCS_AES_KEYSIZE_128) + return 0; + + /* Everything else is unsupported. */ + return -EINVAL; +} + +/* Save key into transformation context. */ +static int save_key(struct ocs_aes_tctx *tctx, const u8 *in_key, size_t key_len, + enum ocs_cipher cipher) +{ + int ret; + + ret = check_key(in_key, key_len, cipher); + if (ret) + return ret; + + memcpy(tctx->key, in_key, key_len); + tctx->key_len = key_len; + tctx->cipher = cipher; + + return 0; +} + +/* Set key for symmetric cypher. */ +static int kmb_ocs_sk_set_key(struct crypto_skcipher *tfm, const u8 *in_key, + size_t key_len, enum ocs_cipher cipher) +{ + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + + /* Fallback is used for AES with 192-bit key. */ + tctx->use_fallback = (cipher == OCS_AES && + key_len == OCS_AES_KEYSIZE_192); + + if (!tctx->use_fallback) + return save_key(tctx, in_key, key_len, cipher); + + crypto_sync_skcipher_clear_flags(tctx->sw_cipher.sk, + CRYPTO_TFM_REQ_MASK); + crypto_sync_skcipher_set_flags(tctx->sw_cipher.sk, + tfm->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + + return crypto_sync_skcipher_setkey(tctx->sw_cipher.sk, in_key, key_len); +} + +/* Set key for AEAD cipher. */ +static int kmb_ocs_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, + size_t key_len, enum ocs_cipher cipher) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); + + /* Fallback is used for AES with 192-bit key. */ + tctx->use_fallback = (cipher == OCS_AES && + key_len == OCS_AES_KEYSIZE_192); + + if (!tctx->use_fallback) + return save_key(tctx, in_key, key_len, cipher); + + crypto_aead_clear_flags(tctx->sw_cipher.aead, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(tctx->sw_cipher.aead, + crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK); + + return crypto_aead_setkey(tctx->sw_cipher.aead, in_key, key_len); +} + +/* Swap two AES blocks in SG lists. */ +static void sg_swap_blocks(struct scatterlist *sgl, unsigned int nents, + off_t blk1_offset, off_t blk2_offset) +{ + u8 tmp_buf1[AES_BLOCK_SIZE], tmp_buf2[AES_BLOCK_SIZE]; + + /* + * No easy way to copy within sg list, so copy both blocks to temporary + * buffers first. + */ + sg_pcopy_to_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk1_offset); + sg_pcopy_to_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk2_offset); + sg_pcopy_from_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk2_offset); + sg_pcopy_from_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk1_offset); +} + +/* Initialize request context to default values. */ +static void ocs_aes_init_rctx(struct ocs_aes_rctx *rctx) +{ + /* Zero everything. */ + memset(rctx, 0, sizeof(*rctx)); + + /* Set initial value for DMA addresses. */ + rctx->src_dll.dma_addr = DMA_MAPPING_ERROR; + rctx->dst_dll.dma_addr = DMA_MAPPING_ERROR; + rctx->aad_src_dll.dma_addr = DMA_MAPPING_ERROR; + rctx->aad_dst_dll.dma_addr = DMA_MAPPING_ERROR; +} + +static int kmb_ocs_sk_validate_input(struct skcipher_request *req, + enum ocs_mode mode) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + int iv_size = crypto_skcipher_ivsize(tfm); + + switch (mode) { + case OCS_MODE_ECB: + /* Ensure input length is multiple of block size */ + if (req->cryptlen % AES_BLOCK_SIZE != 0) + return -EINVAL; + + return 0; + + case OCS_MODE_CBC: + /* Ensure input length is multiple of block size */ + if (req->cryptlen % AES_BLOCK_SIZE != 0) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!req->iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + /* + * NOTE: Since req->cryptlen == 0 case was already handled in + * kmb_ocs_sk_common(), the above two conditions also guarantee + * that: cryptlen >= iv_size + */ + return 0; + + case OCS_MODE_CTR: + /* Ensure IV is present and block size in length */ + if (!req->iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + return 0; + + case OCS_MODE_CTS: + /* Ensure input length >= block size */ + if (req->cryptlen < AES_BLOCK_SIZE) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!req->iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + default: + return -EINVAL; + } +} + +/* + * Called by encrypt() / decrypt() skcipher functions. + * + * Use fallback if needed, otherwise initialize context and enqueue request + * into engine. + */ +static int kmb_ocs_sk_common(struct skcipher_request *req, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + enum ocs_mode mode) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct ocs_aes_dev *aes_dev; + int rc; + + if (tctx->use_fallback) { + SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, tctx->sw_cipher.sk); + + skcipher_request_set_sync_tfm(subreq, tctx->sw_cipher.sk); + skcipher_request_set_callback(subreq, req->base.flags, NULL, + NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + + if (instruction == OCS_ENCRYPT) + rc = crypto_skcipher_encrypt(subreq); + else + rc = crypto_skcipher_decrypt(subreq); + + skcipher_request_zero(subreq); + + return rc; + } + + /* + * If cryptlen == 0, no processing needed for ECB, CBC and CTR. + * + * For CTS continue: kmb_ocs_sk_validate_input() will return -EINVAL. + */ + if (!req->cryptlen && mode != OCS_MODE_CTS) + return 0; + + rc = kmb_ocs_sk_validate_input(req, mode); + if (rc) + return rc; + + aes_dev = kmb_ocs_aes_find_dev(tctx); + if (!aes_dev) + return -ENODEV; + + if (cipher != tctx->cipher) + return -EINVAL; + + ocs_aes_init_rctx(rctx); + rctx->instruction = instruction; + rctx->mode = mode; + + return crypto_transfer_skcipher_request_to_engine(aes_dev->engine, req); +} + +static void cleanup_ocs_dma_linked_list(struct device *dev, + struct ocs_dll_desc *dll) +{ + if (dll->vaddr) + dma_free_coherent(dev, dll->size, dll->vaddr, dll->dma_addr); + dll->vaddr = NULL; + dll->size = 0; + dll->dma_addr = DMA_MAPPING_ERROR; +} + +static void kmb_ocs_sk_dma_cleanup(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct device *dev = tctx->aes_dev->dev; + + if (rctx->src_dma_count) { + dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + rctx->src_dma_count = 0; + } + + if (rctx->dst_dma_count) { + dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ? + DMA_BIDIRECTIONAL : + DMA_FROM_DEVICE); + rctx->dst_dma_count = 0; + } + + /* Clean up OCS DMA linked lists */ + cleanup_ocs_dma_linked_list(dev, &rctx->src_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll); +} + +static int kmb_ocs_sk_prepare_inplace(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + int iv_size = crypto_skcipher_ivsize(tfm); + int rc; + + /* + * For CBC decrypt, save last block (iv) to last_ct_blk buffer. + * + * Note: if we are here, we already checked that cryptlen >= iv_size + * and iv_size == AES_BLOCK_SIZE (i.e., the size of last_ct_blk); see + * kmb_ocs_sk_validate_input(). + */ + if (rctx->mode == OCS_MODE_CBC && rctx->instruction == OCS_DECRYPT) + scatterwalk_map_and_copy(rctx->last_ct_blk, req->src, + req->cryptlen - iv_size, iv_size, 0); + + /* For CTS decrypt, swap last two blocks, if needed. */ + if (rctx->cts_swap && rctx->instruction == OCS_DECRYPT) + sg_swap_blocks(req->dst, rctx->dst_nents, + req->cryptlen - AES_BLOCK_SIZE, + req->cryptlen - (2 * AES_BLOCK_SIZE)); + + /* src and dst buffers are the same, use bidirectional DMA mapping. */ + rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, + rctx->dst_nents, DMA_BIDIRECTIONAL); + if (rctx->dst_dma_count == 0) { + dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); + return -ENOMEM; + } + + /* Create DST linked list */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, &rctx->dst_dll, + req->cryptlen, 0); + if (rc) + return rc; + /* + * If descriptor creation was successful, set the src_dll.dma_addr to + * the value of dst_dll.dma_addr, as we do in-place AES operation on + * the src. + */ + rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr; + + return 0; +} + +static int kmb_ocs_sk_prepare_notinplace(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + int rc; + + rctx->src_nents = sg_nents_for_len(req->src, req->cryptlen); + if (rctx->src_nents < 0) + return -EBADMSG; + + /* Map SRC SG. */ + rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src, + rctx->src_nents, DMA_TO_DEVICE); + if (rctx->src_dma_count == 0) { + dev_err(tctx->aes_dev->dev, "Failed to map source sg\n"); + return -ENOMEM; + } + + /* Create SRC linked list */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, + rctx->src_dma_count, &rctx->src_dll, + req->cryptlen, 0); + if (rc) + return rc; + + /* Map DST SG. */ + rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, + rctx->dst_nents, DMA_FROM_DEVICE); + if (rctx->dst_dma_count == 0) { + dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); + return -ENOMEM; + } + + /* Create DST linked list */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, &rctx->dst_dll, + req->cryptlen, 0); + if (rc) + return rc; + + /* If this is not a CTS decrypt operation with swapping, we are done. */ + if (!(rctx->cts_swap && rctx->instruction == OCS_DECRYPT)) + return 0; + + /* + * Otherwise, we have to copy src to dst (as we cannot modify src). + * Use OCS AES bypass mode to copy src to dst via DMA. + * + * NOTE: for anything other than small data sizes this is rather + * inefficient. + */ + rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->dst_dll.dma_addr, + rctx->src_dll.dma_addr, req->cryptlen); + if (rc) + return rc; + + /* + * Now dst == src, so clean up what we did so far and use in_place + * logic. + */ + kmb_ocs_sk_dma_cleanup(req); + rctx->in_place = true; + + return kmb_ocs_sk_prepare_inplace(req); +} + +static int kmb_ocs_sk_run(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct ocs_aes_dev *aes_dev = tctx->aes_dev; + int iv_size = crypto_skcipher_ivsize(tfm); + int rc; + + rctx->dst_nents = sg_nents_for_len(req->dst, req->cryptlen); + if (rctx->dst_nents < 0) + return -EBADMSG; + + /* + * If 2 blocks or greater, and multiple of block size swap last two + * blocks to be compatible with other crypto API CTS implementations: + * OCS mode uses CBC-CS2, whereas other crypto API implementations use + * CBC-CS3. + * CBC-CS2 and CBC-CS3 defined by: + * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38a-add.pdf + */ + rctx->cts_swap = (rctx->mode == OCS_MODE_CTS && + req->cryptlen > AES_BLOCK_SIZE && + req->cryptlen % AES_BLOCK_SIZE == 0); + + rctx->in_place = (req->src == req->dst); + + if (rctx->in_place) + rc = kmb_ocs_sk_prepare_inplace(req); + else + rc = kmb_ocs_sk_prepare_notinplace(req); + + if (rc) + goto error; + + rc = ocs_aes_op(aes_dev, rctx->mode, tctx->cipher, rctx->instruction, + rctx->dst_dll.dma_addr, rctx->src_dll.dma_addr, + req->cryptlen, req->iv, iv_size); + if (rc) + goto error; + + /* Clean-up DMA before further processing output. */ + kmb_ocs_sk_dma_cleanup(req); + + /* For CTS Encrypt, swap last 2 blocks, if needed. */ + if (rctx->cts_swap && rctx->instruction == OCS_ENCRYPT) { + sg_swap_blocks(req->dst, rctx->dst_nents, + req->cryptlen - AES_BLOCK_SIZE, + req->cryptlen - (2 * AES_BLOCK_SIZE)); + return 0; + } + + /* For CBC copy IV to req->IV. */ + if (rctx->mode == OCS_MODE_CBC) { + /* CBC encrypt case. */ + if (rctx->instruction == OCS_ENCRYPT) { + scatterwalk_map_and_copy(req->iv, req->dst, + req->cryptlen - iv_size, + iv_size, 0); + return 0; + } + /* CBC decrypt case. */ + if (rctx->in_place) + memcpy(req->iv, rctx->last_ct_blk, iv_size); + else + scatterwalk_map_and_copy(req->iv, req->src, + req->cryptlen - iv_size, + iv_size, 0); + return 0; + } + /* For all other modes there's nothing to do. */ + + return 0; + +error: + kmb_ocs_sk_dma_cleanup(req); + + return rc; +} + +static int kmb_ocs_aead_validate_input(struct aead_request *req, + enum ocs_instruction instruction, + enum ocs_mode mode) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + int tag_size = crypto_aead_authsize(tfm); + int iv_size = crypto_aead_ivsize(tfm); + + /* For decrypt crytplen == len(PT) + len(tag). */ + if (instruction == OCS_DECRYPT && req->cryptlen < tag_size) + return -EINVAL; + + /* IV is mandatory. */ + if (!req->iv) + return -EINVAL; + + switch (mode) { + case OCS_MODE_GCM: + if (iv_size != GCM_AES_IV_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_CCM: + /* Ensure IV is present and block size in length */ + if (iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + default: + return -EINVAL; + } +} + +/* + * Called by encrypt() / decrypt() aead functions. + * + * Use fallback if needed, otherwise initialize context and enqueue request + * into engine. + */ +static int kmb_ocs_aead_common(struct aead_request *req, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + enum ocs_mode mode) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + struct ocs_aes_dev *dd; + int rc; + + if (tctx->use_fallback) { + struct aead_request *subreq = aead_request_ctx(req); + + aead_request_set_tfm(subreq, tctx->sw_cipher.aead); + aead_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + aead_request_set_ad(subreq, req->assoclen); + rc = crypto_aead_setauthsize(tctx->sw_cipher.aead, + crypto_aead_authsize(crypto_aead_reqtfm(req))); + if (rc) + return rc; + + return (instruction == OCS_ENCRYPT) ? + crypto_aead_encrypt(subreq) : + crypto_aead_decrypt(subreq); + } + + rc = kmb_ocs_aead_validate_input(req, instruction, mode); + if (rc) + return rc; + + dd = kmb_ocs_aes_find_dev(tctx); + if (!dd) + return -ENODEV; + + if (cipher != tctx->cipher) + return -EINVAL; + + ocs_aes_init_rctx(rctx); + rctx->instruction = instruction; + rctx->mode = mode; + + return crypto_transfer_aead_request_to_engine(dd->engine, req); +} + +static void kmb_ocs_aead_dma_cleanup(struct aead_request *req) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + struct device *dev = tctx->aes_dev->dev; + + if (rctx->src_dma_count) { + dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + rctx->src_dma_count = 0; + } + + if (rctx->dst_dma_count) { + dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ? + DMA_BIDIRECTIONAL : + DMA_FROM_DEVICE); + rctx->dst_dma_count = 0; + } + /* Clean up OCS DMA linked lists */ + cleanup_ocs_dma_linked_list(dev, &rctx->src_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->aad_src_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->aad_dst_dll); +} + +/** + * kmb_ocs_aead_dma_prepare() - Do DMA mapping for AEAD processing. + * @req: The AEAD request being processed. + * @src_dll_size: Where to store the length of the data mapped into the + * src_dll OCS DMA list. + * + * Do the following: + * - DMA map req->src and req->dst + * - Initialize the following OCS DMA linked lists: rctx->src_dll, + * rctx->dst_dll, rctx->aad_src_dll and rxtc->aad_dst_dll. + * + * Return: 0 on success, negative error code otherwise. + */ +static int kmb_ocs_aead_dma_prepare(struct aead_request *req, u32 *src_dll_size) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + u32 in_size; /* The length of the data to be mapped by src_dll. */ + u32 out_size; /* The length of the data to be mapped by dst_dll. */ + u32 dst_size; /* The length of the data in dst_sg. */ + int rc; + + /* Get number of entries in input data SG list. */ + rctx->src_nents = sg_nents_for_len(req->src, + req->assoclen + req->cryptlen); + if (rctx->src_nents < 0) + return -EBADMSG; + + if (rctx->instruction == OCS_DECRYPT) { + /* + * For decrypt: + * - src sg list is: AAD|CT|tag + * - dst sg list expects: AAD|PT + * + * in_size == len(CT); out_size == len(PT) + */ + + /* req->cryptlen includes both CT and tag. */ + in_size = req->cryptlen - tag_size; + + /* out_size = PT size == CT size */ + out_size = in_size; + + /* len(dst_sg) == len(AAD) + len(PT) */ + dst_size = req->assoclen + out_size; + + /* + * Copy tag from source SG list to 'in_tag' buffer. + * + * Note: this needs to be done here, before DMA mapping src_sg. + */ + sg_pcopy_to_buffer(req->src, rctx->src_nents, rctx->in_tag, + tag_size, req->assoclen + in_size); + + } else { /* OCS_ENCRYPT */ + /* + * For encrypt: + * src sg list is: AAD|PT + * dst sg list expects: AAD|CT|tag + */ + /* in_size == len(PT) */ + in_size = req->cryptlen; + + /* + * In CCM mode the OCS engine appends the tag to the ciphertext, + * but in GCM mode the tag must be read from the tag registers + * and appended manually below + */ + out_size = (rctx->mode == OCS_MODE_CCM) ? in_size + tag_size : + in_size; + /* len(dst_sg) == len(AAD) + len(CT) + len(tag) */ + dst_size = req->assoclen + in_size + tag_size; + } + *src_dll_size = in_size; + + /* Get number of entries in output data SG list. */ + rctx->dst_nents = sg_nents_for_len(req->dst, dst_size); + if (rctx->dst_nents < 0) + return -EBADMSG; + + rctx->in_place = (req->src == req->dst) ? 1 : 0; + + /* Map destination; use bidirectional mapping for in-place case. */ + rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, + rctx->dst_nents, + rctx->in_place ? DMA_BIDIRECTIONAL : + DMA_FROM_DEVICE); + if (rctx->dst_dma_count == 0 && rctx->dst_nents != 0) { + dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); + return -ENOMEM; + } + + /* Create AAD DST list: maps dst[0:AAD_SIZE-1]. */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, + &rctx->aad_dst_dll, req->assoclen, + 0); + if (rc) + return rc; + + /* Create DST list: maps dst[AAD_SIZE:out_size] */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, &rctx->dst_dll, + out_size, req->assoclen); + if (rc) + return rc; + + if (rctx->in_place) { + /* If this is not CCM encrypt, we are done. */ + if (!(rctx->mode == OCS_MODE_CCM && + rctx->instruction == OCS_ENCRYPT)) { + /* + * SRC and DST are the same, so re-use the same DMA + * addresses (to avoid allocating new DMA lists + * identical to the dst ones). + */ + rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr; + rctx->aad_src_dll.dma_addr = rctx->aad_dst_dll.dma_addr; + + return 0; + } + /* + * For CCM encrypt the input and output linked lists contain + * different amounts of data, so, we need to create different + * SRC and AAD SRC lists, even for the in-place case. + */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, + &rctx->aad_src_dll, + req->assoclen, 0); + if (rc) + return rc; + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, + &rctx->src_dll, in_size, + req->assoclen); + if (rc) + return rc; + + return 0; + } + /* Not in-place case. */ + + /* Map source SG. */ + rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src, + rctx->src_nents, DMA_TO_DEVICE); + if (rctx->src_dma_count == 0 && rctx->src_nents != 0) { + dev_err(tctx->aes_dev->dev, "Failed to map source sg\n"); + return -ENOMEM; + } + + /* Create AAD SRC list. */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, + rctx->src_dma_count, + &rctx->aad_src_dll, + req->assoclen, 0); + if (rc) + return rc; + + /* Create SRC list. */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, + rctx->src_dma_count, + &rctx->src_dll, in_size, + req->assoclen); + if (rc) + return rc; + + if (req->assoclen == 0) + return 0; + + /* Copy AAD from src sg to dst sg using OCS DMA. */ + rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->aad_dst_dll.dma_addr, + rctx->aad_src_dll.dma_addr, req->cryptlen); + if (rc) + dev_err(tctx->aes_dev->dev, + "Failed to copy source AAD to destination AAD\n"); + + return rc; +} + +static int kmb_ocs_aead_run(struct aead_request *req) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + u32 in_size; /* The length of the data mapped by src_dll. */ + int rc; + + rc = kmb_ocs_aead_dma_prepare(req, &in_size); + if (rc) + goto exit; + + /* For CCM, we just call the OCS processing and we are done. */ + if (rctx->mode == OCS_MODE_CCM) { + rc = ocs_aes_ccm_op(tctx->aes_dev, tctx->cipher, + rctx->instruction, rctx->dst_dll.dma_addr, + rctx->src_dll.dma_addr, in_size, + req->iv, + rctx->aad_src_dll.dma_addr, req->assoclen, + rctx->in_tag, tag_size); + goto exit; + } + /* GCM case; invoke OCS processing. */ + rc = ocs_aes_gcm_op(tctx->aes_dev, tctx->cipher, + rctx->instruction, + rctx->dst_dll.dma_addr, + rctx->src_dll.dma_addr, in_size, + req->iv, + rctx->aad_src_dll.dma_addr, req->assoclen, + rctx->out_tag, tag_size); + if (rc) + goto exit; + + /* For GCM decrypt, we have to compare in_tag with out_tag. */ + if (rctx->instruction == OCS_DECRYPT) { + rc = memcmp(rctx->in_tag, rctx->out_tag, tag_size) ? + -EBADMSG : 0; + goto exit; + } + + /* For GCM encrypt, we must manually copy out_tag to DST sg. */ + + /* Clean-up must be called before the sg_pcopy_from_buffer() below. */ + kmb_ocs_aead_dma_cleanup(req); + + /* Copy tag to destination sg after AAD and CT. */ + sg_pcopy_from_buffer(req->dst, rctx->dst_nents, rctx->out_tag, + tag_size, req->assoclen + req->cryptlen); + + /* Return directly as DMA cleanup already done. */ + return 0; + +exit: + kmb_ocs_aead_dma_cleanup(req); + + return rc; +} + +static int kmb_ocs_aes_sk_do_one_request(struct crypto_engine *engine, + void *areq) +{ + struct skcipher_request *req = + container_of(areq, struct skcipher_request, base); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + int err; + + if (!tctx->aes_dev) { + err = -ENODEV; + goto exit; + } + + err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key, + tctx->cipher); + if (err) + goto exit; + + err = kmb_ocs_sk_run(req); + +exit: + crypto_finalize_skcipher_request(engine, req, err); + + return 0; +} + +static int kmb_ocs_aes_aead_do_one_request(struct crypto_engine *engine, + void *areq) +{ + struct aead_request *req = container_of(areq, + struct aead_request, base); + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + int err; + + if (!tctx->aes_dev) + return -ENODEV; + + err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key, + tctx->cipher); + if (err) + goto exit; + + err = kmb_ocs_aead_run(req); + +exit: + crypto_finalize_aead_request(tctx->aes_dev->engine, req, err); + + return 0; +} + +static int kmb_ocs_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_AES); +} + +static int kmb_ocs_aes_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_AES); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB +static int kmb_ocs_aes_ecb_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_ECB); +} + +static int kmb_ocs_aes_ecb_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_ECB); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + +static int kmb_ocs_aes_cbc_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_aes_cbc_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_aes_ctr_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTR); +} + +static int kmb_ocs_aes_ctr_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTR); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS +static int kmb_ocs_aes_cts_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTS); +} + +static int kmb_ocs_aes_cts_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTS); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ + +static int kmb_ocs_aes_gcm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_aes_gcm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_aes_ccm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CCM); +} + +static int kmb_ocs_aes_ccm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CCM); +} + +static int kmb_ocs_sm4_set_key(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_SM4); +} + +static int kmb_ocs_sm4_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_SM4); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB +static int kmb_ocs_sm4_ecb_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_ECB); +} + +static int kmb_ocs_sm4_ecb_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_ECB); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + +static int kmb_ocs_sm4_cbc_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_sm4_cbc_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_sm4_ctr_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTR); +} + +static int kmb_ocs_sm4_ctr_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTR); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS +static int kmb_ocs_sm4_cts_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTS); +} + +static int kmb_ocs_sm4_cts_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTS); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ + +static int kmb_ocs_sm4_gcm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_sm4_gcm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_sm4_ccm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CCM); +} + +static int kmb_ocs_sm4_ccm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CCM); +} + +static inline int ocs_common_init(struct ocs_aes_tctx *tctx) +{ + tctx->engine_ctx.op.prepare_request = NULL; + tctx->engine_ctx.op.do_one_request = kmb_ocs_aes_sk_do_one_request; + tctx->engine_ctx.op.unprepare_request = NULL; + + return 0; +} + +static int ocs_aes_init_tfm(struct crypto_skcipher *tfm) +{ + const char *alg_name = crypto_tfm_alg_name(&tfm->base); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct crypto_sync_skcipher *blk; + + /* set fallback cipher in case it will be needed */ + blk = crypto_alloc_sync_skcipher(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(blk)) + return PTR_ERR(blk); + + tctx->sw_cipher.sk = blk; + + crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); + + return ocs_common_init(tctx); +} + +static int ocs_sm4_init_tfm(struct crypto_skcipher *tfm) +{ + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + + crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); + + return ocs_common_init(tctx); +} + +static inline void clear_key(struct ocs_aes_tctx *tctx) +{ + memzero_explicit(tctx->key, OCS_AES_KEYSIZE_256); + + /* Zero key registers if set */ + if (tctx->aes_dev) + ocs_aes_set_key(tctx->aes_dev, OCS_AES_KEYSIZE_256, + tctx->key, OCS_AES); +} + +static void ocs_exit_tfm(struct crypto_skcipher *tfm) +{ + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + + clear_key(tctx); + + if (tctx->sw_cipher.sk) { + crypto_free_sync_skcipher(tctx->sw_cipher.sk); + tctx->sw_cipher.sk = NULL; + } +} + +static inline int ocs_common_aead_init(struct ocs_aes_tctx *tctx) +{ + tctx->engine_ctx.op.prepare_request = NULL; + tctx->engine_ctx.op.do_one_request = kmb_ocs_aes_aead_do_one_request; + tctx->engine_ctx.op.unprepare_request = NULL; + + return 0; +} + +static int ocs_aes_aead_cra_init(struct crypto_aead *tfm) +{ + const char *alg_name = crypto_tfm_alg_name(&tfm->base); + struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); + struct crypto_aead *blk; + + /* Set fallback cipher in case it will be needed */ + blk = crypto_alloc_aead(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(blk)) + return PTR_ERR(blk); + + tctx->sw_cipher.aead = blk; + + crypto_aead_set_reqsize(tfm, + max(sizeof(struct ocs_aes_rctx), + (sizeof(struct aead_request) + + crypto_aead_reqsize(tctx->sw_cipher.aead)))); + + return ocs_common_aead_init(tctx); +} + +static int kmb_ocs_aead_ccm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + switch (authsize) { + case 4: + case 6: + case 8: + case 10: + case 12: + case 14: + case 16: + return 0; + default: + return -EINVAL; + } +} + +static int kmb_ocs_aead_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + return crypto_gcm_check_authsize(authsize); +} + +static int ocs_sm4_aead_cra_init(struct crypto_aead *tfm) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); + + crypto_aead_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); + + return ocs_common_aead_init(tctx); +} + +static void ocs_aead_cra_exit(struct crypto_aead *tfm) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); + + clear_key(tctx); + + if (tctx->sw_cipher.aead) { + crypto_free_aead(tctx->sw_cipher.aead); + tctx->sw_cipher.aead = NULL; + } +} + +static struct skcipher_alg algs[] = { +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB + { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_AES_MIN_KEY_SIZE, + .max_keysize = OCS_AES_MAX_KEY_SIZE, + .setkey = kmb_ocs_aes_set_key, + .encrypt = kmb_ocs_aes_ecb_encrypt, + .decrypt = kmb_ocs_aes_ecb_decrypt, + .init = ocs_aes_init_tfm, + .exit = ocs_exit_tfm, + }, +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_AES_MIN_KEY_SIZE, + .max_keysize = OCS_AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_aes_set_key, + .encrypt = kmb_ocs_aes_cbc_encrypt, + .decrypt = kmb_ocs_aes_cbc_decrypt, + .init = ocs_aes_init_tfm, + .exit = ocs_exit_tfm, + }, + { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_AES_MIN_KEY_SIZE, + .max_keysize = OCS_AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_aes_set_key, + .encrypt = kmb_ocs_aes_ctr_encrypt, + .decrypt = kmb_ocs_aes_ctr_decrypt, + .init = ocs_aes_init_tfm, + .exit = ocs_exit_tfm, + }, +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS + { + .base.cra_name = "cts(cbc(aes))", + .base.cra_driver_name = "cts-aes-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_AES_MIN_KEY_SIZE, + .max_keysize = OCS_AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_aes_set_key, + .encrypt = kmb_ocs_aes_cts_encrypt, + .decrypt = kmb_ocs_aes_cts_decrypt, + .init = ocs_aes_init_tfm, + .exit = ocs_exit_tfm, + }, +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB + { + .base.cra_name = "ecb(sm4)", + .base.cra_driver_name = "ecb-sm4-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_SM4_KEY_SIZE, + .max_keysize = OCS_SM4_KEY_SIZE, + .setkey = kmb_ocs_sm4_set_key, + .encrypt = kmb_ocs_sm4_ecb_encrypt, + .decrypt = kmb_ocs_sm4_ecb_decrypt, + .init = ocs_sm4_init_tfm, + .exit = ocs_exit_tfm, + }, +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + { + .base.cra_name = "cbc(sm4)", + .base.cra_driver_name = "cbc-sm4-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_SM4_KEY_SIZE, + .max_keysize = OCS_SM4_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_sm4_set_key, + .encrypt = kmb_ocs_sm4_cbc_encrypt, + .decrypt = kmb_ocs_sm4_cbc_decrypt, + .init = ocs_sm4_init_tfm, + .exit = ocs_exit_tfm, + }, + { + .base.cra_name = "ctr(sm4)", + .base.cra_driver_name = "ctr-sm4-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_SM4_KEY_SIZE, + .max_keysize = OCS_SM4_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_sm4_set_key, + .encrypt = kmb_ocs_sm4_ctr_encrypt, + .decrypt = kmb_ocs_sm4_ctr_decrypt, + .init = ocs_sm4_init_tfm, + .exit = ocs_exit_tfm, + }, +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS + { + .base.cra_name = "cts(cbc(sm4))", + .base.cra_driver_name = "cts-sm4-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_SM4_KEY_SIZE, + .max_keysize = OCS_SM4_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_sm4_set_key, + .encrypt = kmb_ocs_sm4_cts_encrypt, + .decrypt = kmb_ocs_sm4_cts_decrypt, + .init = ocs_sm4_init_tfm, + .exit = ocs_exit_tfm, + } +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ +}; + +static struct aead_alg algs_aead[] = { + { + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = ocs_aes_aead_cra_init, + .exit = ocs_aead_cra_exit, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setauthsize = kmb_ocs_aead_gcm_setauthsize, + .setkey = kmb_ocs_aes_aead_set_key, + .encrypt = kmb_ocs_aes_gcm_encrypt, + .decrypt = kmb_ocs_aes_gcm_decrypt, + }, + { + .base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "ccm-aes-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = ocs_aes_aead_cra_init, + .exit = ocs_aead_cra_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setauthsize = kmb_ocs_aead_ccm_setauthsize, + .setkey = kmb_ocs_aes_aead_set_key, + .encrypt = kmb_ocs_aes_ccm_encrypt, + .decrypt = kmb_ocs_aes_ccm_decrypt, + }, + { + .base = { + .cra_name = "gcm(sm4)", + .cra_driver_name = "gcm-sm4-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = ocs_sm4_aead_cra_init, + .exit = ocs_aead_cra_exit, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setauthsize = kmb_ocs_aead_gcm_setauthsize, + .setkey = kmb_ocs_sm4_aead_set_key, + .encrypt = kmb_ocs_sm4_gcm_encrypt, + .decrypt = kmb_ocs_sm4_gcm_decrypt, + }, + { + .base = { + .cra_name = "ccm(sm4)", + .cra_driver_name = "ccm-sm4-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = ocs_sm4_aead_cra_init, + .exit = ocs_aead_cra_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setauthsize = kmb_ocs_aead_ccm_setauthsize, + .setkey = kmb_ocs_sm4_aead_set_key, + .encrypt = kmb_ocs_sm4_ccm_encrypt, + .decrypt = kmb_ocs_sm4_ccm_decrypt, + } +}; + +static void unregister_aes_algs(struct ocs_aes_dev *aes_dev) +{ + crypto_unregister_aeads(algs_aead, ARRAY_SIZE(algs_aead)); + crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); +} + +static int register_aes_algs(struct ocs_aes_dev *aes_dev) +{ + int ret; + + /* + * If any algorithm fails to register, all preceding algorithms that + * were successfully registered will be automatically unregistered. + */ + ret = crypto_register_aeads(algs_aead, ARRAY_SIZE(algs_aead)); + if (ret) + return ret; + + ret = crypto_register_skciphers(algs, ARRAY_SIZE(algs)); + if (ret) + crypto_unregister_aeads(algs_aead, ARRAY_SIZE(algs)); + + return ret; +} + +/* Device tree driver match. */ +static const struct of_device_id kmb_ocs_aes_of_match[] = { + { + .compatible = "intel,keembay-ocs-aes", + }, + {} +}; + +static int kmb_ocs_aes_remove(struct platform_device *pdev) +{ + struct ocs_aes_dev *aes_dev; + + aes_dev = platform_get_drvdata(pdev); + if (!aes_dev) + return -ENODEV; + + unregister_aes_algs(aes_dev); + + spin_lock(&ocs_aes.lock); + list_del(&aes_dev->list); + spin_unlock(&ocs_aes.lock); + + crypto_engine_exit(aes_dev->engine); + + return 0; +} + +static int kmb_ocs_aes_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ocs_aes_dev *aes_dev; + struct resource *aes_mem; + int rc; + + aes_dev = devm_kzalloc(dev, sizeof(*aes_dev), GFP_KERNEL); + if (!aes_dev) + return -ENOMEM; + + aes_dev->dev = dev; + + platform_set_drvdata(pdev, aes_dev); + + rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (rc) { + dev_err(dev, "Failed to set 32 bit dma mask %d\n", rc); + return rc; + } + + /* Get base register address. */ + aes_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!aes_mem) { + dev_err(dev, "Could not retrieve io mem resource\n"); + return -ENODEV; + } + + aes_dev->base_reg = devm_ioremap_resource(&pdev->dev, aes_mem); + if (IS_ERR(aes_dev->base_reg)) { + dev_err(dev, "Failed to get base address\n"); + return PTR_ERR(aes_dev->base_reg); + } + + /* Get and request IRQ */ + aes_dev->irq = platform_get_irq(pdev, 0); + if (aes_dev->irq < 0) + return aes_dev->irq; + + rc = devm_request_threaded_irq(dev, aes_dev->irq, ocs_aes_irq_handler, + NULL, 0, "keembay-ocs-aes", aes_dev); + if (rc < 0) { + dev_err(dev, "Could not request IRQ\n"); + return rc; + } + + INIT_LIST_HEAD(&aes_dev->list); + spin_lock(&ocs_aes.lock); + list_add_tail(&aes_dev->list, &ocs_aes.dev_list); + spin_unlock(&ocs_aes.lock); + + init_completion(&aes_dev->irq_completion); + + /* Initialize crypto engine */ + aes_dev->engine = crypto_engine_alloc_init(dev, true); + if (!aes_dev->engine) + goto list_del; + + rc = crypto_engine_start(aes_dev->engine); + if (rc) { + dev_err(dev, "Could not start crypto engine\n"); + goto cleanup; + } + + rc = register_aes_algs(aes_dev); + if (rc) { + dev_err(dev, + "Could not register OCS algorithms with Crypto API\n"); + goto cleanup; + } + + return 0; + +cleanup: + crypto_engine_exit(aes_dev->engine); +list_del: + spin_lock(&ocs_aes.lock); + list_del(&aes_dev->list); + spin_unlock(&ocs_aes.lock); + + return rc; +} + +/* The OCS driver is a platform device. */ +static struct platform_driver kmb_ocs_aes_driver = { + .probe = kmb_ocs_aes_probe, + .remove = kmb_ocs_aes_remove, + .driver = { + .name = DRV_NAME, + .of_match_table = kmb_ocs_aes_of_match, + }, +}; + +module_platform_driver(kmb_ocs_aes_driver); + +MODULE_DESCRIPTION("Intel Keem Bay Offload and Crypto Subsystem (OCS) AES/SM4 Driver"); +MODULE_LICENSE("GPL"); + +MODULE_ALIAS_CRYPTO("cbc-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ctr-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("gcm-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ccm-aes-keembay-ocs"); + +MODULE_ALIAS_CRYPTO("cbc-sm4-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ctr-sm4-keembay-ocs"); +MODULE_ALIAS_CRYPTO("gcm-sm4-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ccm-sm4-keembay-ocs"); + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB +MODULE_ALIAS_CRYPTO("ecb-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ecb-sm4-keembay-ocs"); +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS +MODULE_ALIAS_CRYPTO("cts-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("cts-sm4-keembay-ocs"); +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ diff --git a/drivers/crypto/keembay/ocs-aes.c b/drivers/crypto/keembay/ocs-aes.c new file mode 100644 index 000000000000..cc286adb1c4a --- /dev/null +++ b/drivers/crypto/keembay/ocs-aes.c @@ -0,0 +1,1489 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS AES Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include "ocs-aes.h" + +#define AES_COMMAND_OFFSET 0x0000 +#define AES_KEY_0_OFFSET 0x0004 +#define AES_KEY_1_OFFSET 0x0008 +#define AES_KEY_2_OFFSET 0x000C +#define AES_KEY_3_OFFSET 0x0010 +#define AES_KEY_4_OFFSET 0x0014 +#define AES_KEY_5_OFFSET 0x0018 +#define AES_KEY_6_OFFSET 0x001C +#define AES_KEY_7_OFFSET 0x0020 +#define AES_IV_0_OFFSET 0x0024 +#define AES_IV_1_OFFSET 0x0028 +#define AES_IV_2_OFFSET 0x002C +#define AES_IV_3_OFFSET 0x0030 +#define AES_ACTIVE_OFFSET 0x0034 +#define AES_STATUS_OFFSET 0x0038 +#define AES_KEY_SIZE_OFFSET 0x0044 +#define AES_IER_OFFSET 0x0048 +#define AES_ISR_OFFSET 0x005C +#define AES_MULTIPURPOSE1_0_OFFSET 0x0200 +#define AES_MULTIPURPOSE1_1_OFFSET 0x0204 +#define AES_MULTIPURPOSE1_2_OFFSET 0x0208 +#define AES_MULTIPURPOSE1_3_OFFSET 0x020C +#define AES_MULTIPURPOSE2_0_OFFSET 0x0220 +#define AES_MULTIPURPOSE2_1_OFFSET 0x0224 +#define AES_MULTIPURPOSE2_2_OFFSET 0x0228 +#define AES_MULTIPURPOSE2_3_OFFSET 0x022C +#define AES_BYTE_ORDER_CFG_OFFSET 0x02C0 +#define AES_TLEN_OFFSET 0x0300 +#define AES_T_MAC_0_OFFSET 0x0304 +#define AES_T_MAC_1_OFFSET 0x0308 +#define AES_T_MAC_2_OFFSET 0x030C +#define AES_T_MAC_3_OFFSET 0x0310 +#define AES_PLEN_OFFSET 0x0314 +#define AES_A_DMA_SRC_ADDR_OFFSET 0x0400 +#define AES_A_DMA_DST_ADDR_OFFSET 0x0404 +#define AES_A_DMA_SRC_SIZE_OFFSET 0x0408 +#define AES_A_DMA_DST_SIZE_OFFSET 0x040C +#define AES_A_DMA_DMA_MODE_OFFSET 0x0410 +#define AES_A_DMA_NEXT_SRC_DESCR_OFFSET 0x0418 +#define AES_A_DMA_NEXT_DST_DESCR_OFFSET 0x041C +#define AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET 0x0420 +#define AES_A_DMA_LOG_OFFSET 0x0424 +#define AES_A_DMA_STATUS_OFFSET 0x0428 +#define AES_A_DMA_PERF_CNTR_OFFSET 0x042C +#define AES_A_DMA_MSI_ISR_OFFSET 0x0480 +#define AES_A_DMA_MSI_IER_OFFSET 0x0484 +#define AES_A_DMA_MSI_MASK_OFFSET 0x0488 +#define AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET 0x0600 +#define AES_A_DMA_OUTBUFFER_READ_FIFO_OFFSET 0x0700 + +/* + * AES_A_DMA_DMA_MODE register. + * Default: 0x00000000. + * bit[31] ACTIVE + * This bit activates the DMA. When the DMA finishes, it resets + * this bit to zero. + * bit[30:26] Unused by this driver. + * bit[25] SRC_LINK_LIST_EN + * Source link list enable bit. When the linked list is terminated + * this bit is reset by the DMA. + * bit[24] DST_LINK_LIST_EN + * Destination link list enable bit. When the linked list is + * terminated this bit is reset by the DMA. + * bit[23:0] Unused by this driver. + */ +#define AES_A_DMA_DMA_MODE_ACTIVE BIT(31) +#define AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN BIT(25) +#define AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN BIT(24) + +/* + * AES_ACTIVE register + * default 0x00000000 + * bit[31:10] Reserved + * bit[9] LAST_ADATA + * bit[8] LAST_GCX + * bit[7:2] Reserved + * bit[1] TERMINATION + * bit[0] TRIGGER + */ +#define AES_ACTIVE_LAST_ADATA BIT(9) +#define AES_ACTIVE_LAST_CCM_GCM BIT(8) +#define AES_ACTIVE_TERMINATION BIT(1) +#define AES_ACTIVE_TRIGGER BIT(0) + +#define AES_DISABLE_INT 0x00000000 +#define AES_DMA_CPD_ERR_INT BIT(8) +#define AES_DMA_OUTBUF_RD_ERR_INT BIT(7) +#define AES_DMA_OUTBUF_WR_ERR_INT BIT(6) +#define AES_DMA_INBUF_RD_ERR_INT BIT(5) +#define AES_DMA_INBUF_WR_ERR_INT BIT(4) +#define AES_DMA_BAD_COMP_INT BIT(3) +#define AES_DMA_SAI_INT BIT(2) +#define AES_DMA_SRC_DONE_INT BIT(0) +#define AES_COMPLETE_INT BIT(1) + +#define AES_DMA_MSI_MASK_CLEAR BIT(0) + +#define AES_128_BIT_KEY 0x00000000 +#define AES_256_BIT_KEY BIT(0) + +#define AES_DEACTIVATE_PERF_CNTR 0x00000000 +#define AES_ACTIVATE_PERF_CNTR BIT(0) + +#define AES_MAX_TAG_SIZE_U32 4 + +#define OCS_LL_DMA_FLAG_TERMINATE BIT(31) + +/* + * There is an inconsistency in the documentation. This is documented as a + * 11-bit value, but it is actually 10-bits. + */ +#define AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK 0x3FF + +/* + * During CCM decrypt, the OCS block needs to finish processing the ciphertext + * before the tag is written. For 128-bit mode this required delay is 28 OCS + * clock cycles. For 256-bit mode it is 36 OCS clock cycles. + */ +#define CCM_DECRYPT_DELAY_TAG_CLK_COUNT 36UL + +/* + * During CCM decrypt there must be a delay of at least 42 OCS clock cycles + * between setting the TRIGGER bit in AES_ACTIVE and setting the LAST_CCM_GCM + * bit in the same register (as stated in the OCS databook) + */ +#define CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT 42UL + +/* See RFC3610 section 2.2 */ +#define L_PRIME_MIN (1) +#define L_PRIME_MAX (7) +/* + * CCM IV format from RFC 3610 section 2.3 + * + * Octet Number Contents + * ------------ --------- + * 0 Flags + * 1 ... 15-L Nonce N + * 16-L ... 15 Counter i + * + * Flags = L' = L - 1 + */ +#define L_PRIME_IDX 0 +#define COUNTER_START(lprime) (16 - ((lprime) + 1)) +#define COUNTER_LEN(lprime) ((lprime) + 1) + +enum aes_counter_mode { + AES_CTR_M_NO_INC = 0, + AES_CTR_M_32_INC = 1, + AES_CTR_M_64_INC = 2, + AES_CTR_M_128_INC = 3, +}; + +/** + * struct ocs_dma_linked_list - OCS DMA linked list entry. + * @src_addr: Source address of the data. + * @src_len: Length of data to be fetched. + * @next: Next dma_list to fetch. + * @ll_flags: Flags (Freeze @ terminate) for the DMA engine. + */ +struct ocs_dma_linked_list { + u32 src_addr; + u32 src_len; + u32 next; + u32 ll_flags; +} __packed; + +/* + * Set endianness of inputs and outputs + * AES_BYTE_ORDER_CFG + * default 0x00000000 + * bit [10] - KEY_HI_LO_SWAP + * bit [9] - KEY_HI_SWAP_DWORDS_IN_OCTWORD + * bit [8] - KEY_HI_SWAP_BYTES_IN_DWORD + * bit [7] - KEY_LO_SWAP_DWORDS_IN_OCTWORD + * bit [6] - KEY_LO_SWAP_BYTES_IN_DWORD + * bit [5] - IV_SWAP_DWORDS_IN_OCTWORD + * bit [4] - IV_SWAP_BYTES_IN_DWORD + * bit [3] - DOUT_SWAP_DWORDS_IN_OCTWORD + * bit [2] - DOUT_SWAP_BYTES_IN_DWORD + * bit [1] - DOUT_SWAP_DWORDS_IN_OCTWORD + * bit [0] - DOUT_SWAP_BYTES_IN_DWORD + */ +static inline void aes_a_set_endianness(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(0x7FF, aes_dev->base_reg + AES_BYTE_ORDER_CFG_OFFSET); +} + +/* Trigger AES process start. */ +static inline void aes_a_op_trigger(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_TRIGGER, aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* Indicate last bulk of data. */ +static inline void aes_a_op_termination(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_TERMINATION, + aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* + * Set LAST_CCM_GCM in AES_ACTIVE register and clear all other bits. + * + * Called when DMA is programmed to fetch the last batch of data. + * - For AES-CCM it is called for the last batch of Payload data and Ciphertext + * data. + * - For AES-GCM, it is called for the last batch of Plaintext data and + * Ciphertext data. + */ +static inline void aes_a_set_last_gcx(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_LAST_CCM_GCM, + aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* Wait for LAST_CCM_GCM bit to be unset. */ +static inline void aes_a_wait_last_gcx(const struct ocs_aes_dev *aes_dev) +{ + u32 aes_active_reg; + + do { + aes_active_reg = ioread32(aes_dev->base_reg + + AES_ACTIVE_OFFSET); + } while (aes_active_reg & AES_ACTIVE_LAST_CCM_GCM); +} + +/* Wait for 10 bits of input occupancy. */ +static void aes_a_dma_wait_input_buffer_occupancy(const struct ocs_aes_dev *aes_dev) +{ + u32 reg; + + do { + reg = ioread32(aes_dev->base_reg + AES_A_DMA_STATUS_OFFSET); + } while (reg & AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK); +} + + /* + * Set LAST_CCM_GCM and LAST_ADATA bits in AES_ACTIVE register (and clear all + * other bits). + * + * Called when DMA is programmed to fetch the last batch of Associated Data + * (CCM case) or Additional Authenticated Data (GCM case). + */ +static inline void aes_a_set_last_gcx_and_adata(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_LAST_ADATA | AES_ACTIVE_LAST_CCM_GCM, + aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* Set DMA src and dst transfer size to 0 */ +static inline void aes_a_dma_set_xfer_size_zero(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); + iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); +} + +/* Activate DMA for zero-byte transfer case. */ +static inline void aes_a_dma_active(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Activate DMA and enable src linked list */ +static inline void aes_a_dma_active_src_ll_en(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | + AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Activate DMA and enable dst linked list */ +static inline void aes_a_dma_active_dst_ll_en(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | + AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Activate DMA and enable src and dst linked lists */ +static inline void aes_a_dma_active_src_dst_ll_en(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | + AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN | + AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Reset PERF_CNTR to 0 and activate it */ +static inline void aes_a_dma_reset_and_activate_perf_cntr(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(0x00000000, aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET); + iowrite32(AES_ACTIVATE_PERF_CNTR, + aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); +} + +/* Wait until PERF_CNTR is > delay, then deactivate it */ +static inline void aes_a_dma_wait_and_deactivate_perf_cntr(const struct ocs_aes_dev *aes_dev, + int delay) +{ + while (ioread32(aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET) < delay) + ; + iowrite32(AES_DEACTIVATE_PERF_CNTR, + aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); +} + +/* Disable AES and DMA IRQ. */ +static void aes_irq_disable(struct ocs_aes_dev *aes_dev) +{ + u32 isr_val = 0; + + /* Disable interrupts */ + iowrite32(AES_DISABLE_INT, + aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); + iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); + + /* Clear any pending interrupt */ + isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); + if (isr_val) + iowrite32(isr_val, + aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); + + isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); + if (isr_val) + iowrite32(isr_val, + aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); + + isr_val = ioread32(aes_dev->base_reg + AES_ISR_OFFSET); + if (isr_val) + iowrite32(isr_val, aes_dev->base_reg + AES_ISR_OFFSET); +} + +/* Enable AES or DMA IRQ. IRQ is disabled once fired. */ +static void aes_irq_enable(struct ocs_aes_dev *aes_dev, u8 irq) +{ + if (irq == AES_COMPLETE_INT) { + /* Ensure DMA error interrupts are enabled */ + iowrite32(AES_DMA_CPD_ERR_INT | + AES_DMA_OUTBUF_RD_ERR_INT | + AES_DMA_OUTBUF_WR_ERR_INT | + AES_DMA_INBUF_RD_ERR_INT | + AES_DMA_INBUF_WR_ERR_INT | + AES_DMA_BAD_COMP_INT | + AES_DMA_SAI_INT, + aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); + /* + * AES_IER + * default 0x00000000 + * bits [31:3] - reserved + * bit [2] - EN_SKS_ERR + * bit [1] - EN_AES_COMPLETE + * bit [0] - reserved + */ + iowrite32(AES_COMPLETE_INT, aes_dev->base_reg + AES_IER_OFFSET); + return; + } + if (irq == AES_DMA_SRC_DONE_INT) { + /* Ensure AES interrupts are disabled */ + iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); + /* + * DMA_MSI_IER + * default 0x00000000 + * bits [31:9] - reserved + * bit [8] - CPD_ERR_INT_EN + * bit [7] - OUTBUF_RD_ERR_INT_EN + * bit [6] - OUTBUF_WR_ERR_INT_EN + * bit [5] - INBUF_RD_ERR_INT_EN + * bit [4] - INBUF_WR_ERR_INT_EN + * bit [3] - BAD_COMP_INT_EN + * bit [2] - SAI_INT_EN + * bit [1] - DST_DONE_INT_EN + * bit [0] - SRC_DONE_INT_EN + */ + iowrite32(AES_DMA_CPD_ERR_INT | + AES_DMA_OUTBUF_RD_ERR_INT | + AES_DMA_OUTBUF_WR_ERR_INT | + AES_DMA_INBUF_RD_ERR_INT | + AES_DMA_INBUF_WR_ERR_INT | + AES_DMA_BAD_COMP_INT | + AES_DMA_SAI_INT | + AES_DMA_SRC_DONE_INT, + aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); + } +} + +/* Enable and wait for IRQ (either from OCS AES engine or DMA) */ +static int ocs_aes_irq_enable_and_wait(struct ocs_aes_dev *aes_dev, u8 irq) +{ + int rc; + + reinit_completion(&aes_dev->irq_completion); + aes_irq_enable(aes_dev, irq); + rc = wait_for_completion_interruptible(&aes_dev->irq_completion); + if (rc) + return rc; + + return aes_dev->dma_err_mask ? -EIO : 0; +} + +/* Configure DMA to OCS, linked list mode */ +static inline void dma_to_ocs_aes_ll(struct ocs_aes_dev *aes_dev, + dma_addr_t dma_list) +{ + iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); + iowrite32(dma_list, + aes_dev->base_reg + AES_A_DMA_NEXT_SRC_DESCR_OFFSET); +} + +/* Configure DMA from OCS, linked list mode */ +static inline void dma_from_ocs_aes_ll(struct ocs_aes_dev *aes_dev, + dma_addr_t dma_list) +{ + iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); + iowrite32(dma_list, + aes_dev->base_reg + AES_A_DMA_NEXT_DST_DESCR_OFFSET); +} + +irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id) +{ + struct ocs_aes_dev *aes_dev = dev_id; + u32 aes_dma_isr; + + /* Read DMA ISR status. */ + aes_dma_isr = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); + + /* Disable and clear interrupts. */ + aes_irq_disable(aes_dev); + + /* Save DMA error status. */ + aes_dev->dma_err_mask = aes_dma_isr & + (AES_DMA_CPD_ERR_INT | + AES_DMA_OUTBUF_RD_ERR_INT | + AES_DMA_OUTBUF_WR_ERR_INT | + AES_DMA_INBUF_RD_ERR_INT | + AES_DMA_INBUF_WR_ERR_INT | + AES_DMA_BAD_COMP_INT | + AES_DMA_SAI_INT); + + /* Signal IRQ completion. */ + complete(&aes_dev->irq_completion); + + return IRQ_HANDLED; +} + +/** + * ocs_aes_set_key() - Write key into OCS AES hardware. + * @aes_dev: The OCS AES device to write the key to. + * @key_size: The size of the key (in bytes). + * @key: The key to write. + * @cipher: The cipher the key is for. + * + * For AES @key_size must be either 16 or 32. For SM4 @key_size must be 16. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, u32 key_size, const u8 *key, + enum ocs_cipher cipher) +{ + const u32 *key_u32; + u32 val; + int i; + + /* OCS AES supports 128-bit and 256-bit keys only. */ + if (cipher == OCS_AES && !(key_size == 32 || key_size == 16)) { + dev_err(aes_dev->dev, + "%d-bit keys not supported by AES cipher\n", + key_size * 8); + return -EINVAL; + } + /* OCS SM4 supports 128-bit keys only. */ + if (cipher == OCS_SM4 && key_size != 16) { + dev_err(aes_dev->dev, + "%d-bit keys not supported for SM4 cipher\n", + key_size * 8); + return -EINVAL; + } + + if (!key) + return -EINVAL; + + key_u32 = (const u32 *)key; + + /* Write key to AES_KEY[0-7] registers */ + for (i = 0; i < (key_size / sizeof(u32)); i++) { + iowrite32(key_u32[i], + aes_dev->base_reg + AES_KEY_0_OFFSET + + (i * sizeof(u32))); + } + /* + * Write key size + * bits [31:1] - reserved + * bit [0] - AES_KEY_SIZE + * 0 - 128 bit key + * 1 - 256 bit key + */ + val = (key_size == 16) ? AES_128_BIT_KEY : AES_256_BIT_KEY; + iowrite32(val, aes_dev->base_reg + AES_KEY_SIZE_OFFSET); + + return 0; +} + +/* Write AES_COMMAND */ +static inline void set_ocs_aes_command(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_mode mode, + enum ocs_instruction instruction) +{ + u32 val; + + /* AES_COMMAND + * default 0x000000CC + * bit [14] - CIPHER_SELECT + * 0 - AES + * 1 - SM4 + * bits [11:8] - OCS_AES_MODE + * 0000 - ECB + * 0001 - CBC + * 0010 - CTR + * 0110 - CCM + * 0111 - GCM + * 1001 - CTS + * bits [7:6] - AES_INSTRUCTION + * 00 - ENCRYPT + * 01 - DECRYPT + * 10 - EXPAND + * 11 - BYPASS + * bits [3:2] - CTR_M_BITS + * 00 - No increment + * 01 - Least significant 32 bits are incremented + * 10 - Least significant 64 bits are incremented + * 11 - Full 128 bits are incremented + */ + val = (cipher << 14) | (mode << 8) | (instruction << 6) | + (AES_CTR_M_128_INC << 2); + iowrite32(val, aes_dev->base_reg + AES_COMMAND_OFFSET); +} + +static void ocs_aes_init(struct ocs_aes_dev *aes_dev, + enum ocs_mode mode, + enum ocs_cipher cipher, + enum ocs_instruction instruction) +{ + /* Ensure interrupts are disabled and pending interrupts cleared. */ + aes_irq_disable(aes_dev); + + /* Set endianness recommended by data-sheet. */ + aes_a_set_endianness(aes_dev); + + /* Set AES_COMMAND register. */ + set_ocs_aes_command(aes_dev, cipher, mode, instruction); +} + +/* + * Write the byte length of the last AES/SM4 block of Payload data (without + * zero padding and without the length of the MAC) in register AES_PLEN. + */ +static inline void ocs_aes_write_last_data_blk_len(struct ocs_aes_dev *aes_dev, + u32 size) +{ + u32 val; + + if (size == 0) { + val = 0; + goto exit; + } + + val = size % AES_BLOCK_SIZE; + if (val == 0) + val = AES_BLOCK_SIZE; + +exit: + iowrite32(val, aes_dev->base_reg + AES_PLEN_OFFSET); +} + +/* + * Validate inputs according to mode. + * If OK return 0; else return -EINVAL. + */ +static int ocs_aes_validate_inputs(dma_addr_t src_dma_list, u32 src_size, + const u8 *iv, u32 iv_size, + dma_addr_t aad_dma_list, u32 aad_size, + const u8 *tag, u32 tag_size, + enum ocs_cipher cipher, enum ocs_mode mode, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list) +{ + /* Ensure cipher, mode and instruction are valid. */ + if (!(cipher == OCS_AES || cipher == OCS_SM4)) + return -EINVAL; + + if (mode != OCS_MODE_ECB && mode != OCS_MODE_CBC && + mode != OCS_MODE_CTR && mode != OCS_MODE_CCM && + mode != OCS_MODE_GCM && mode != OCS_MODE_CTS) + return -EINVAL; + + if (instruction != OCS_ENCRYPT && instruction != OCS_DECRYPT && + instruction != OCS_EXPAND && instruction != OCS_BYPASS) + return -EINVAL; + + /* + * When instruction is OCS_BYPASS, OCS simply copies data from source + * to destination using DMA. + * + * AES mode is irrelevant, but both source and destination DMA + * linked-list must be defined. + */ + if (instruction == OCS_BYPASS) { + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + return 0; + } + + /* + * For performance reasons switch based on mode to limit unnecessary + * conditionals for each mode + */ + switch (mode) { + case OCS_MODE_ECB: + /* Ensure input length is multiple of block size */ + if (src_size % AES_BLOCK_SIZE != 0) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + return 0; + + case OCS_MODE_CBC: + /* Ensure input length is multiple of block size */ + if (src_size % AES_BLOCK_SIZE != 0) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_CTR: + /* Ensure input length of 1 byte or greater */ + if (src_size == 0) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_CTS: + /* Ensure input length >= block size */ + if (src_size < AES_BLOCK_SIZE) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_GCM: + /* Ensure IV is present and GCM_AES_IV_SIZE in length */ + if (!iv || iv_size != GCM_AES_IV_SIZE) + return -EINVAL; + + /* + * If input data present ensure source and destination linked + * lists are created + */ + if (src_size && (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR)) + return -EINVAL; + + /* If aad present ensure aad linked list is created */ + if (aad_size && aad_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure tag destination is set */ + if (!tag) + return -EINVAL; + + /* Just ensure that tag_size doesn't cause overflows. */ + if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32))) + return -EINVAL; + + return 0; + + case OCS_MODE_CCM: + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + /* 2 <= L <= 8, so 1 <= L' <= 7 */ + if (iv[L_PRIME_IDX] < L_PRIME_MIN || + iv[L_PRIME_IDX] > L_PRIME_MAX) + return -EINVAL; + + /* If aad present ensure aad linked list is created */ + if (aad_size && aad_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Just ensure that tag_size doesn't cause overflows. */ + if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32))) + return -EINVAL; + + if (instruction == OCS_DECRYPT) { + /* + * If input data present ensure source and destination + * linked lists are created + */ + if (src_size && (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR)) + return -EINVAL; + + /* Ensure input tag is present */ + if (!tag) + return -EINVAL; + + return 0; + } + + /* Instruction == OCS_ENCRYPT */ + + /* + * Destination linked list always required (for tag even if no + * input data) + */ + if (dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* If input data present ensure src linked list is created */ + if (src_size && src_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + return 0; + + default: + return -EINVAL; + } +} + +/** + * ocs_aes_op() - Perform AES/SM4 operation. + * @aes_dev: The OCS AES device to use. + * @mode: The mode to use (ECB, CBC, CTR, or CTS). + * @cipher: The cipher to use (AES or SM4). + * @instruction: The instruction to perform (encrypt or decrypt). + * @dst_dma_list: The OCS DMA list mapping output memory. + * @src_dma_list: The OCS DMA list mapping input payload data. + * @src_size: The amount of data mapped by @src_dma_list. + * @iv: The IV vector. + * @iv_size: The size (in bytes) of @iv. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_op(struct ocs_aes_dev *aes_dev, + enum ocs_mode mode, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + u32 iv_size) +{ + u32 *iv32; + int rc; + + rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, iv_size, 0, 0, + NULL, 0, cipher, mode, instruction, + dst_dma_list); + if (rc) + return rc; + /* + * ocs_aes_validate_inputs() is a generic check, now ensure mode is not + * GCM or CCM. + */ + if (mode == OCS_MODE_GCM || mode == OCS_MODE_CCM) + return -EINVAL; + + /* Cast IV to u32 array. */ + iv32 = (u32 *)iv; + + ocs_aes_init(aes_dev, mode, cipher, instruction); + + if (mode == OCS_MODE_CTS) { + /* Write the byte length of the last data block to engine. */ + ocs_aes_write_last_data_blk_len(aes_dev, src_size); + } + + /* ECB is the only mode that doesn't use IV. */ + if (mode != OCS_MODE_ECB) { + iowrite32(iv32[0], aes_dev->base_reg + AES_IV_0_OFFSET); + iowrite32(iv32[1], aes_dev->base_reg + AES_IV_1_OFFSET); + iowrite32(iv32[2], aes_dev->base_reg + AES_IV_2_OFFSET); + iowrite32(iv32[3], aes_dev->base_reg + AES_IV_3_OFFSET); + } + + /* Set AES_ACTIVE.TRIGGER to start the operation. */ + aes_a_op_trigger(aes_dev); + + /* Configure and activate input / output DMA. */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + + if (mode == OCS_MODE_CTS) { + /* + * For CTS mode, instruct engine to activate ciphertext + * stealing if last block of data is incomplete. + */ + aes_a_set_last_gcx(aes_dev); + } else { + /* For all other modes, just write the 'termination' bit. */ + aes_a_op_termination(aes_dev); + } + + /* Wait for engine to complete processing. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); + if (rc) + return rc; + + if (mode == OCS_MODE_CTR) { + /* Read back IV for streaming mode */ + iv32[0] = ioread32(aes_dev->base_reg + AES_IV_0_OFFSET); + iv32[1] = ioread32(aes_dev->base_reg + AES_IV_1_OFFSET); + iv32[2] = ioread32(aes_dev->base_reg + AES_IV_2_OFFSET); + iv32[3] = ioread32(aes_dev->base_reg + AES_IV_3_OFFSET); + } + + return 0; +} + +/* Compute and write J0 to engine registers. */ +static void ocs_aes_gcm_write_j0(const struct ocs_aes_dev *aes_dev, + const u8 *iv) +{ + const u32 *j0 = (u32 *)iv; + + /* + * IV must be 12 bytes; Other sizes not supported as Linux crypto API + * does only expects/allows 12 byte IV for GCM + */ + iowrite32(0x00000001, aes_dev->base_reg + AES_IV_0_OFFSET); + iowrite32(__swab32(j0[2]), aes_dev->base_reg + AES_IV_1_OFFSET); + iowrite32(__swab32(j0[1]), aes_dev->base_reg + AES_IV_2_OFFSET); + iowrite32(__swab32(j0[0]), aes_dev->base_reg + AES_IV_3_OFFSET); +} + +/* Read GCM tag from engine registers. */ +static inline void ocs_aes_gcm_read_tag(struct ocs_aes_dev *aes_dev, + u8 *tag, u32 tag_size) +{ + u32 tag_u32[AES_MAX_TAG_SIZE_U32]; + + /* + * The Authentication Tag T is stored in Little Endian order in the + * registers with the most significant bytes stored from AES_T_MAC[3] + * downward. + */ + tag_u32[0] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_3_OFFSET)); + tag_u32[1] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_2_OFFSET)); + tag_u32[2] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_1_OFFSET)); + tag_u32[3] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_0_OFFSET)); + + memcpy(tag, tag_u32, tag_size); +} + +/** + * ocs_aes_gcm_op() - Perform GCM operation. + * @aes_dev: The OCS AES device to use. + * @cipher: The Cipher to use (AES or SM4). + * @instruction: The instruction to perform (encrypt or decrypt). + * @dst_dma_list: The OCS DMA list mapping output memory. + * @src_dma_list: The OCS DMA list mapping input payload data. + * @src_size: The amount of data mapped by @src_dma_list. + * @iv: The input IV vector. + * @aad_dma_list: The OCS DMA list mapping input AAD data. + * @aad_size: The amount of data mapped by @aad_dma_list. + * @out_tag: Where to store computed tag. + * @tag_size: The size (in bytes) of @out_tag. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + const u8 *iv, + dma_addr_t aad_dma_list, + u32 aad_size, + u8 *out_tag, + u32 tag_size) +{ + u64 bit_len; + u32 val; + int rc; + + rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, + GCM_AES_IV_SIZE, aad_dma_list, + aad_size, out_tag, tag_size, cipher, + OCS_MODE_GCM, instruction, + dst_dma_list); + if (rc) + return rc; + + ocs_aes_init(aes_dev, OCS_MODE_GCM, cipher, instruction); + + /* Compute and write J0 to OCS HW. */ + ocs_aes_gcm_write_j0(aes_dev, iv); + + /* Write out_tag byte length */ + iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); + + /* Write the byte length of the last plaintext / ciphertext block. */ + ocs_aes_write_last_data_blk_len(aes_dev, src_size); + + /* Write ciphertext bit length */ + bit_len = src_size * 8; + val = bit_len & 0xFFFFFFFF; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_0_OFFSET); + val = bit_len >> 32; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_1_OFFSET); + + /* Write aad bit length */ + bit_len = aad_size * 8; + val = bit_len & 0xFFFFFFFF; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_2_OFFSET); + val = bit_len >> 32; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_3_OFFSET); + + /* Set AES_ACTIVE.TRIGGER to start the operation. */ + aes_a_op_trigger(aes_dev); + + /* Process AAD. */ + if (aad_size) { + /* If aad present, configure DMA to feed it to the engine. */ + dma_to_ocs_aes_ll(aes_dev, aad_dma_list); + aes_a_dma_active_src_ll_en(aes_dev); + + /* Instructs engine to pad last block of aad, if needed. */ + aes_a_set_last_gcx_and_adata(aes_dev); + + /* Wait for DMA transfer to complete. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); + if (rc) + return rc; + } else { + aes_a_set_last_gcx_and_adata(aes_dev); + } + + /* Wait until adata (if present) has been processed. */ + aes_a_wait_last_gcx(aes_dev); + aes_a_dma_wait_input_buffer_occupancy(aes_dev); + + /* Now process payload. */ + if (src_size) { + /* Configure and activate DMA for both input and output data. */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + } else { + aes_a_dma_set_xfer_size_zero(aes_dev); + aes_a_dma_active(aes_dev); + } + + /* Instruct AES/SMA4 engine payload processing is over. */ + aes_a_set_last_gcx(aes_dev); + + /* Wait for OCS AES engine to complete processing. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); + if (rc) + return rc; + + ocs_aes_gcm_read_tag(aes_dev, out_tag, tag_size); + + return 0; +} + +/* Write encrypted tag to AES/SM4 engine. */ +static void ocs_aes_ccm_write_encrypted_tag(struct ocs_aes_dev *aes_dev, + const u8 *in_tag, u32 tag_size) +{ + int i; + + /* Ensure DMA input buffer is empty */ + aes_a_dma_wait_input_buffer_occupancy(aes_dev); + + /* + * During CCM decrypt, the OCS block needs to finish processing the + * ciphertext before the tag is written. So delay needed after DMA has + * completed writing the ciphertext + */ + aes_a_dma_reset_and_activate_perf_cntr(aes_dev); + aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev, + CCM_DECRYPT_DELAY_TAG_CLK_COUNT); + + /* Write encrypted tag to AES/SM4 engine. */ + for (i = 0; i < tag_size; i++) { + iowrite8(in_tag[i], aes_dev->base_reg + + AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); + } +} + +/* + * Write B0 CCM block to OCS AES HW. + * + * Note: B0 format is documented in NIST Special Publication 800-38C + * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf + * (see Section A.2.1) + */ +static int ocs_aes_ccm_write_b0(const struct ocs_aes_dev *aes_dev, + const u8 *iv, u32 adata_size, u32 tag_size, + u32 cryptlen) +{ + u8 b0[16]; /* CCM B0 block is 16 bytes long. */ + int i, q; + + /* Initialize B0 to 0. */ + memset(b0, 0, sizeof(b0)); + + /* + * B0[0] is the 'Flags Octet' and has the following structure: + * bit 7: Reserved + * bit 6: Adata flag + * bit 5-3: t value encoded as (t-2)/2 + * bit 2-0: q value encoded as q - 1 + */ + /* If there is AAD data, set the Adata flag. */ + if (adata_size) + b0[0] |= BIT(6); + /* + * t denotes the octet length of T. + * t can only be an element of { 4, 6, 8, 10, 12, 14, 16} and is + * encoded as (t - 2) / 2 + */ + b0[0] |= (((tag_size - 2) / 2) & 0x7) << 3; + /* + * q is the octet length of Q. + * q can only be an element of {2, 3, 4, 5, 6, 7, 8} and is encoded as + * q - 1 == iv[0] + */ + b0[0] |= iv[0] & 0x7; + /* + * Copy the Nonce N from IV to B0; N is located in iv[1]..iv[15 - q] + * and must be copied to b0[1]..b0[15-q]. + * q == iv[0] + 1 + */ + q = iv[0] + 1; + for (i = 1; i <= 15 - q; i++) + b0[i] = iv[i]; + /* + * The rest of B0 must contain Q, i.e., the message length. + * Q is encoded in q octets, in big-endian order, so to write it, we + * start from the end of B0 and we move backward. + */ + i = sizeof(b0) - 1; + while (q) { + b0[i] = cryptlen & 0xff; + cryptlen >>= 8; + i--; + q--; + } + /* + * If cryptlen is not zero at this point, it means that its original + * value was too big. + */ + if (cryptlen) + return -EOVERFLOW; + /* Now write B0 to OCS AES input buffer. */ + for (i = 0; i < sizeof(b0); i++) + iowrite8(b0[i], aes_dev->base_reg + + AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); + return 0; +} + +/* + * Write adata length to OCS AES HW. + * + * Note: adata len encoding is documented in NIST Special Publication 800-38C + * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf + * (see Section A.2.2) + */ +static void ocs_aes_ccm_write_adata_len(const struct ocs_aes_dev *aes_dev, + u64 adata_len) +{ + u8 enc_a[10]; /* Maximum encoded size: 10 octets. */ + int i, len; + + /* + * adata_len ('a') is encoded as follows: + * If 0 < a < 2^16 - 2^8 ==> 'a' encoded as [a]16, i.e., two octets + * (big endian). + * If 2^16 - 2^8 ≤ a < 2^32 ==> 'a' encoded as 0xff || 0xfe || [a]32, + * i.e., six octets (big endian). + * If 2^32 ≤ a < 2^64 ==> 'a' encoded as 0xff || 0xff || [a]64, + * i.e., ten octets (big endian). + */ + if (adata_len < 65280) { + len = 2; + *(__be16 *)enc_a = cpu_to_be16(adata_len); + } else if (adata_len <= 0xFFFFFFFF) { + len = 6; + *(__be16 *)enc_a = cpu_to_be16(0xfffe); + *(__be32 *)&enc_a[2] = cpu_to_be32(adata_len); + } else { /* adata_len >= 2^32 */ + len = 10; + *(__be16 *)enc_a = cpu_to_be16(0xffff); + *(__be64 *)&enc_a[2] = cpu_to_be64(adata_len); + } + for (i = 0; i < len; i++) + iowrite8(enc_a[i], + aes_dev->base_reg + + AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); +} + +static int ocs_aes_ccm_do_adata(struct ocs_aes_dev *aes_dev, + dma_addr_t adata_dma_list, u32 adata_size) +{ + int rc; + + if (!adata_size) { + /* Since no aad the LAST_GCX bit can be set now */ + aes_a_set_last_gcx_and_adata(aes_dev); + goto exit; + } + + /* Adata case. */ + + /* + * Form the encoding of the Associated data length and write it + * to the AES/SM4 input buffer. + */ + ocs_aes_ccm_write_adata_len(aes_dev, adata_size); + + /* Configure the AES/SM4 DMA to fetch the Associated Data */ + dma_to_ocs_aes_ll(aes_dev, adata_dma_list); + + /* Activate DMA to fetch Associated data. */ + aes_a_dma_active_src_ll_en(aes_dev); + + /* Set LAST_GCX and LAST_ADATA in AES ACTIVE register. */ + aes_a_set_last_gcx_and_adata(aes_dev); + + /* Wait for DMA transfer to complete. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); + if (rc) + return rc; + +exit: + /* Wait until adata (if present) has been processed. */ + aes_a_wait_last_gcx(aes_dev); + aes_a_dma_wait_input_buffer_occupancy(aes_dev); + + return 0; +} + +static int ocs_aes_ccm_encrypt_do_payload(struct ocs_aes_dev *aes_dev, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size) +{ + if (src_size) { + /* + * Configure and activate DMA for both input and output + * data. + */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + } else { + /* Configure and activate DMA for output data only. */ + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_dst_ll_en(aes_dev); + } + + /* + * Set the LAST GCX bit in AES_ACTIVE Register to instruct + * AES/SM4 engine to pad the last block of data. + */ + aes_a_set_last_gcx(aes_dev); + + /* We are done, wait for IRQ and return. */ + return ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); +} + +static int ocs_aes_ccm_decrypt_do_payload(struct ocs_aes_dev *aes_dev, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size) +{ + if (!src_size) { + /* Let engine process 0-length input. */ + aes_a_dma_set_xfer_size_zero(aes_dev); + aes_a_dma_active(aes_dev); + aes_a_set_last_gcx(aes_dev); + + return 0; + } + + /* + * Configure and activate DMA for both input and output + * data. + */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + /* + * Set the LAST GCX bit in AES_ACTIVE Register; this allows the + * AES/SM4 engine to differentiate between encrypted data and + * encrypted MAC. + */ + aes_a_set_last_gcx(aes_dev); + /* + * Enable DMA DONE interrupt; once DMA transfer is over, + * interrupt handler will process the MAC/tag. + */ + return ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); +} + +/* + * Compare Tag to Yr. + * + * Only used at the end of CCM decrypt. If tag == yr, message authentication + * has succeeded. + */ +static inline int ccm_compare_tag_to_yr(struct ocs_aes_dev *aes_dev, + u8 tag_size_bytes) +{ + u32 tag[AES_MAX_TAG_SIZE_U32]; + u32 yr[AES_MAX_TAG_SIZE_U32]; + u8 i; + + /* Read Tag and Yr from AES registers. */ + for (i = 0; i < AES_MAX_TAG_SIZE_U32; i++) { + tag[i] = ioread32(aes_dev->base_reg + + AES_T_MAC_0_OFFSET + (i * sizeof(u32))); + yr[i] = ioread32(aes_dev->base_reg + + AES_MULTIPURPOSE2_0_OFFSET + + (i * sizeof(u32))); + } + + return memcmp(tag, yr, tag_size_bytes) ? -EBADMSG : 0; +} + +/** + * ocs_aes_ccm_op() - Perform CCM operation. + * @aes_dev: The OCS AES device to use. + * @cipher: The Cipher to use (AES or SM4). + * @instruction: The instruction to perform (encrypt or decrypt). + * @dst_dma_list: The OCS DMA list mapping output memory. + * @src_dma_list: The OCS DMA list mapping input payload data. + * @src_size: The amount of data mapped by @src_dma_list. + * @iv: The input IV vector. + * @adata_dma_list: The OCS DMA list mapping input A-data. + * @adata_size: The amount of data mapped by @adata_dma_list. + * @in_tag: Input tag. + * @tag_size: The size (in bytes) of @in_tag. + * + * Note: for encrypt the tag is appended to the ciphertext (in the memory + * mapped by @dst_dma_list). + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + dma_addr_t adata_dma_list, + u32 adata_size, + u8 *in_tag, + u32 tag_size) +{ + u32 *iv_32; + u8 lprime; + int rc; + + rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, + AES_BLOCK_SIZE, adata_dma_list, adata_size, + in_tag, tag_size, cipher, OCS_MODE_CCM, + instruction, dst_dma_list); + if (rc) + return rc; + + ocs_aes_init(aes_dev, OCS_MODE_CCM, cipher, instruction); + + /* + * Note: rfc 3610 and NIST 800-38C require counter of zero to encrypt + * auth tag so ensure this is the case + */ + lprime = iv[L_PRIME_IDX]; + memset(&iv[COUNTER_START(lprime)], 0, COUNTER_LEN(lprime)); + + /* + * Nonce is already converted to ctr0 before being passed into this + * function as iv. + */ + iv_32 = (u32 *)iv; + iowrite32(__swab32(iv_32[0]), + aes_dev->base_reg + AES_MULTIPURPOSE1_3_OFFSET); + iowrite32(__swab32(iv_32[1]), + aes_dev->base_reg + AES_MULTIPURPOSE1_2_OFFSET); + iowrite32(__swab32(iv_32[2]), + aes_dev->base_reg + AES_MULTIPURPOSE1_1_OFFSET); + iowrite32(__swab32(iv_32[3]), + aes_dev->base_reg + AES_MULTIPURPOSE1_0_OFFSET); + + /* Write MAC/tag length in register AES_TLEN */ + iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); + /* + * Write the byte length of the last AES/SM4 block of Payload data + * (without zero padding and without the length of the MAC) in register + * AES_PLEN. + */ + ocs_aes_write_last_data_blk_len(aes_dev, src_size); + + /* Set AES_ACTIVE.TRIGGER to start the operation. */ + aes_a_op_trigger(aes_dev); + + aes_a_dma_reset_and_activate_perf_cntr(aes_dev); + + /* Form block B0 and write it to the AES/SM4 input buffer. */ + rc = ocs_aes_ccm_write_b0(aes_dev, iv, adata_size, tag_size, src_size); + if (rc) + return rc; + /* + * Ensure there has been at least CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT + * clock cycles since TRIGGER bit was set + */ + aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev, + CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT); + + /* Process Adata. */ + ocs_aes_ccm_do_adata(aes_dev, adata_dma_list, adata_size); + + /* For Encrypt case we just process the payload and return. */ + if (instruction == OCS_ENCRYPT) { + return ocs_aes_ccm_encrypt_do_payload(aes_dev, dst_dma_list, + src_dma_list, src_size); + } + /* For Decypt we need to process the payload and then the tag. */ + rc = ocs_aes_ccm_decrypt_do_payload(aes_dev, dst_dma_list, + src_dma_list, src_size); + if (rc) + return rc; + + /* Process MAC/tag directly: feed tag to engine and wait for IRQ. */ + ocs_aes_ccm_write_encrypted_tag(aes_dev, in_tag, tag_size); + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); + if (rc) + return rc; + + return ccm_compare_tag_to_yr(aes_dev, tag_size); +} + +/** + * ocs_create_linked_list_from_sg() - Create OCS DMA linked list from SG list. + * @aes_dev: The OCS AES device the list will be created for. + * @sg: The SG list OCS DMA linked list will be created from. When + * passed to this function, @sg must have been already mapped + * with dma_map_sg(). + * @sg_dma_count: The number of DMA-mapped entries in @sg. This must be the + * value returned by dma_map_sg() when @sg was mapped. + * @dll_desc: The OCS DMA dma_list to use to store information about the + * created linked list. + * @data_size: The size of the data (from the SG list) to be mapped into the + * OCS DMA linked list. + * @data_offset: The offset (within the SG list) of the data to be mapped. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev, + struct scatterlist *sg, + int sg_dma_count, + struct ocs_dll_desc *dll_desc, + size_t data_size, size_t data_offset) +{ + struct ocs_dma_linked_list *ll = NULL; + struct scatterlist *sg_tmp; + unsigned int tmp; + int dma_nents; + int i; + + if (!dll_desc || !sg || !aes_dev) + return -EINVAL; + + /* Default values for when no ddl_desc is created. */ + dll_desc->vaddr = NULL; + dll_desc->dma_addr = DMA_MAPPING_ERROR; + dll_desc->size = 0; + + if (data_size == 0) + return 0; + + /* Loop over sg_list until we reach entry at specified offset. */ + while (data_offset >= sg_dma_len(sg)) { + data_offset -= sg_dma_len(sg); + sg_dma_count--; + sg = sg_next(sg); + /* If we reach the end of the list, offset was invalid. */ + if (!sg || sg_dma_count == 0) + return -EINVAL; + } + + /* Compute number of DMA-mapped SG entries to add into OCS DMA list. */ + dma_nents = 0; + tmp = 0; + sg_tmp = sg; + while (tmp < data_offset + data_size) { + /* If we reach the end of the list, data_size was invalid. */ + if (!sg_tmp) + return -EINVAL; + tmp += sg_dma_len(sg_tmp); + dma_nents++; + sg_tmp = sg_next(sg_tmp); + } + if (dma_nents > sg_dma_count) + return -EINVAL; + + /* Allocate the DMA list, one entry for each SG entry. */ + dll_desc->size = sizeof(struct ocs_dma_linked_list) * dma_nents; + dll_desc->vaddr = dma_alloc_coherent(aes_dev->dev, dll_desc->size, + &dll_desc->dma_addr, GFP_KERNEL); + if (!dll_desc->vaddr) + return -ENOMEM; + + /* Populate DMA linked list entries. */ + ll = dll_desc->vaddr; + for (i = 0; i < dma_nents; i++, sg = sg_next(sg)) { + ll[i].src_addr = sg_dma_address(sg) + data_offset; + ll[i].src_len = (sg_dma_len(sg) - data_offset) < data_size ? + (sg_dma_len(sg) - data_offset) : data_size; + data_offset = 0; + data_size -= ll[i].src_len; + /* Current element points to the DMA address of the next one. */ + ll[i].next = dll_desc->dma_addr + (sizeof(*ll) * (i + 1)); + ll[i].ll_flags = 0; + } + /* Terminate last element. */ + ll[i - 1].next = 0; + ll[i - 1].ll_flags = OCS_LL_DMA_FLAG_TERMINATE; + + return 0; +} diff --git a/drivers/crypto/keembay/ocs-aes.h b/drivers/crypto/keembay/ocs-aes.h new file mode 100644 index 000000000000..c035fc48b7ed --- /dev/null +++ b/drivers/crypto/keembay/ocs-aes.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Intel Keem Bay OCS AES Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#ifndef _CRYPTO_OCS_AES_H +#define _CRYPTO_OCS_AES_H + +#include + +enum ocs_cipher { + OCS_AES = 0, + OCS_SM4 = 1, +}; + +enum ocs_mode { + OCS_MODE_ECB = 0, + OCS_MODE_CBC = 1, + OCS_MODE_CTR = 2, + OCS_MODE_CCM = 6, + OCS_MODE_GCM = 7, + OCS_MODE_CTS = 9, +}; + +enum ocs_instruction { + OCS_ENCRYPT = 0, + OCS_DECRYPT = 1, + OCS_EXPAND = 2, + OCS_BYPASS = 3, +}; + +/** + * struct ocs_aes_dev - AES device context. + * @list: List head for insertion into device list hold + * by driver. + * @dev: OCS AES device. + * @irq: IRQ number. + * @base_reg: IO base address of OCS AES. + * @irq_copy_completion: Completion to indicate IRQ has been triggered. + * @dma_err_mask: Error reported by OCS DMA interrupts. + * @engine: Crypto engine for the device. + */ +struct ocs_aes_dev { + struct list_head list; + struct device *dev; + int irq; + void __iomem *base_reg; + struct completion irq_completion; + u32 dma_err_mask; + struct crypto_engine *engine; +}; + +/** + * struct ocs_dll_desc - Descriptor of an OCS DMA Linked List. + * @vaddr: Virtual address of the linked list head. + * @dma_addr: DMA address of the linked list head. + * @size: Size (in bytes) of the linked list. + */ +struct ocs_dll_desc { + void *vaddr; + dma_addr_t dma_addr; + size_t size; +}; + +int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, const u32 key_size, + const u8 *key, const enum ocs_cipher cipher); + +int ocs_aes_op(struct ocs_aes_dev *aes_dev, + enum ocs_mode mode, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + u32 iv_size); + +/** + * ocs_aes_bypass_op() - Use OCS DMA to copy data. + * @aes_dev: The OCS AES device to use. + * @dst_dma_list: The OCS DMA list mapping the memory where input data + * will be copied to. + * @src_dma_list: The OCS DMA list mapping input data. + * @src_size: The amount of data to copy. + */ +static inline int ocs_aes_bypass_op(struct ocs_aes_dev *aes_dev, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, u32 src_size) +{ + return ocs_aes_op(aes_dev, OCS_MODE_ECB, OCS_AES, OCS_BYPASS, + dst_dma_list, src_dma_list, src_size, NULL, 0); +} + +int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + const u8 *iv, + dma_addr_t aad_dma_list, + u32 aad_size, + u8 *out_tag, + u32 tag_size); + +int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + dma_addr_t adata_dma_list, + u32 adata_size, + u8 *in_tag, + u32 tag_size); + +int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev, + struct scatterlist *sg, + int sg_dma_count, + struct ocs_dll_desc *dll_desc, + size_t data_size, + size_t data_offset); + +irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id); + +#endif -- cgit v1.2.3 From a320dc2ff80b8f93b2b8acf2e3ead8ff5ad0bcff Mon Sep 17 00:00:00 2001 From: Yejune Deng Date: Tue, 1 Dec 2020 14:50:18 +0800 Subject: crypto: hisilicon/trng - replace atomic_add_return() a set of atomic_inc_return() looks more neater Signed-off-by: Yejune Deng Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/trng/trng.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/hisilicon/trng/trng.c b/drivers/crypto/hisilicon/trng/trng.c index a5033cf09e2b..29712685498a 100644 --- a/drivers/crypto/hisilicon/trng/trng.c +++ b/drivers/crypto/hisilicon/trng/trng.c @@ -267,12 +267,12 @@ static int hisi_trng_probe(struct platform_device *pdev) } hisi_trng_add_to_list(trng); - if (atomic_add_return(1, &trng_active_devs) == 1) { + if (atomic_inc_return(&trng_active_devs) == 1) { ret = crypto_register_rng(&hisi_trng_alg); if (ret) { dev_err(&pdev->dev, "failed to register crypto(%d)\n", ret); - atomic_sub_return(1, &trng_active_devs); + atomic_dec_return(&trng_active_devs); goto err_remove_from_list; } } @@ -289,7 +289,7 @@ static int hisi_trng_probe(struct platform_device *pdev) return ret; err_crypto_unregister: - if (atomic_sub_return(1, &trng_active_devs) == 0) + if (atomic_dec_return(&trng_active_devs) == 0) crypto_unregister_rng(&hisi_trng_alg); err_remove_from_list: @@ -305,7 +305,7 @@ static int hisi_trng_remove(struct platform_device *pdev) while (hisi_trng_del_from_list(trng)) ; - if (atomic_sub_return(1, &trng_active_devs) == 0) + if (atomic_dec_return(&trng_active_devs) == 0) crypto_unregister_rng(&hisi_trng_alg); return 0; -- cgit v1.2.3 From d33a23b0532d5d1b5b700e8641661261e7dbef61 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Fri, 4 Dec 2020 00:20:04 +0100 Subject: crypto: atmel-i2c - select CONFIG_BITREVERSE The bitreverse helper is almost always built into the kernel, but in a rare randconfig build it is possible to hit a case in which it is a loadable module while the atmel-i2c driver is built-in: arm-linux-gnueabi-ld: drivers/crypto/atmel-i2c.o: in function `atmel_i2c_checksum': atmel-i2c.c:(.text+0xa0): undefined reference to `byte_rev_table' Add one more 'select' statement to prevent this. Fixes: 11105693fa05 ("crypto: atmel-ecc - introduce Microchip / Atmel ECC driver") Signed-off-by: Arnd Bergmann Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 808eecef0f32..bbd51703e738 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -548,6 +548,7 @@ config CRYPTO_DEV_ATMEL_SHA config CRYPTO_DEV_ATMEL_I2C tristate + select BITREVERSE config CRYPTO_DEV_ATMEL_ECC tristate "Support for Microchip / Atmel ECC hw accelerator" -- cgit v1.2.3 From 67916c9516893528ecce060ada1f58af0ce33d93 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Tue, 1 Dec 2020 14:24:49 +0000 Subject: crypto: qat - add AES-CTR support for QAT GEN4 devices Add support for AES-CTR for QAT GEN4 devices. Also, introduce the capability ICP_ACCEL_CAPABILITIES_AES_V2 and the helper macro HW_CAP_AES_V2, which allow to distinguish between different HW generations. Co-developed-by: Tomasz Kowalik Signed-off-by: Tomasz Kowalik Co-developed-by: Mateusz Polrola Signed-off-by: Mateusz Polrola Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_fw_la.h | 7 +++++++ drivers/crypto/qat/qat_common/icp_qat_hw.h | 17 ++++++++++++++++- drivers/crypto/qat/qat_common/qat_algs.c | 17 ++++++++++++++++- 3 files changed, 39 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_la.h b/drivers/crypto/qat/qat_common/icp_qat_fw_la.h index 6757ec09d81f..28fa17f14be4 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_la.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_la.h @@ -33,6 +33,9 @@ struct icp_qat_fw_la_bulk_req { struct icp_qat_fw_comn_req_cd_ctrl cd_ctrl; }; +#define ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE 1 +#define QAT_LA_SLICE_TYPE_BITPOS 14 +#define QAT_LA_SLICE_TYPE_MASK 0x3 #define ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS 1 #define ICP_QAT_FW_LA_GCM_IV_LEN_NOT_12_OCTETS 0 #define QAT_FW_LA_ZUC_3G_PROTO_FLAG_BITPOS 12 @@ -179,6 +182,10 @@ struct icp_qat_fw_la_bulk_req { QAT_FIELD_SET(flags, val, QAT_LA_PARTIAL_BITPOS, \ QAT_LA_PARTIAL_MASK) +#define ICP_QAT_FW_LA_SLICE_TYPE_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_SLICE_TYPE_BITPOS, \ + QAT_LA_SLICE_TYPE_MASK) + struct icp_qat_fw_cipher_req_hdr_cd_pars { union { struct { diff --git a/drivers/crypto/qat/qat_common/icp_qat_hw.h b/drivers/crypto/qat/qat_common/icp_qat_hw.h index 4aa5d724e11b..e39e8a2d51a7 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hw.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hw.h @@ -65,6 +65,11 @@ struct icp_qat_hw_auth_config { __u32 reserved; }; +struct icp_qat_hw_ucs_cipher_config { + __u32 val; + __u32 reserved[3]; +}; + enum icp_qat_slice_mask { ICP_ACCEL_MASK_CIPHER_SLICE = BIT(0), ICP_ACCEL_MASK_AUTH_SLICE = BIT(1), @@ -86,6 +91,8 @@ enum icp_qat_capabilities_mask { ICP_ACCEL_CAPABILITIES_RAND = BIT(7), ICP_ACCEL_CAPABILITIES_ZUC = BIT(8), ICP_ACCEL_CAPABILITIES_SHA3 = BIT(9), + /* Bits 10-25 are currently reserved */ + ICP_ACCEL_CAPABILITIES_AES_V2 = BIT(26) }; #define QAT_AUTH_MODE_BITPOS 4 @@ -278,7 +285,15 @@ struct icp_qat_hw_cipher_aes256_f8 { __u8 key[ICP_QAT_HW_AES_256_F8_KEY_SZ]; }; +struct icp_qat_hw_ucs_cipher_aes256_f8 { + struct icp_qat_hw_ucs_cipher_config cipher_config; + __u8 key[ICP_QAT_HW_AES_256_F8_KEY_SZ]; +}; + struct icp_qat_hw_cipher_algo_blk { - struct icp_qat_hw_cipher_aes256_f8 aes; + union { + struct icp_qat_hw_cipher_aes256_f8 aes; + struct icp_qat_hw_ucs_cipher_aes256_f8 ucs_aes; + }; } __aligned(64); #endif diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index b3a68d986417..84d1a3545c3a 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -33,6 +33,10 @@ ICP_QAT_HW_CIPHER_KEY_CONVERT, \ ICP_QAT_HW_CIPHER_DECRYPT) +#define HW_CAP_AES_V2(accel_dev) \ + (GET_HW_DATA(accel_dev)->accel_capabilities_mask & \ + ICP_ACCEL_CAPABILITIES_AES_V2) + static DEFINE_MUTEX(algs_lock); static unsigned int active_devs; @@ -416,12 +420,23 @@ static void qat_alg_skcipher_init_com(struct qat_alg_skcipher_ctx *ctx, struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr; struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl; + bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev); + int mode = ctx->mode; - memcpy(cd->aes.key, key, keylen); qat_alg_init_common_hdr(header); header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER; cd_pars->u.s.content_desc_params_sz = sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3; + + if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_CTR_MODE) { + ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE); + keylen = round_up(keylen, 16); + memcpy(cd->ucs_aes.key, key, keylen); + } else { + memcpy(cd->aes.key, key, keylen); + } + /* Cipher CD config setup */ cd_ctrl->cipher_key_sz = keylen >> 3; cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3; -- cgit v1.2.3 From 5106dfeaeabea73d5132daab1d89d57b57fa98b7 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Tue, 1 Dec 2020 14:24:50 +0000 Subject: crypto: qat - add AES-XTS support for QAT GEN4 devices Add handling of AES-XTS specific to QAT GEN4 devices. Co-developed-by: Tomaszx Kowalik Signed-off-by: Tomaszx Kowalik Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_algs.c | 96 +++++++++++++++++++++++++++++--- 1 file changed, 89 insertions(+), 7 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 84d1a3545c3a..31c7a206a629 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -33,6 +33,11 @@ ICP_QAT_HW_CIPHER_KEY_CONVERT, \ ICP_QAT_HW_CIPHER_DECRYPT) +#define QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode) \ + ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \ + ICP_QAT_HW_CIPHER_NO_CONVERT, \ + ICP_QAT_HW_CIPHER_DECRYPT) + #define HW_CAP_AES_V2(accel_dev) \ (GET_HW_DATA(accel_dev)->accel_capabilities_mask & \ ICP_ACCEL_CAPABILITIES_AES_V2) @@ -95,6 +100,7 @@ struct qat_alg_skcipher_ctx { struct icp_qat_fw_la_bulk_req dec_fw_req; struct qat_crypto_instance *inst; struct crypto_skcipher *ftfm; + struct crypto_cipher *tweak; bool fallback; int mode; }; @@ -428,7 +434,16 @@ static void qat_alg_skcipher_init_com(struct qat_alg_skcipher_ctx *ctx, cd_pars->u.s.content_desc_params_sz = sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3; - if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_CTR_MODE) { + if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) { + ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE); + + /* Store both XTS keys in CD, only the first key is sent + * to the HW, the second key is used for tweak calculation + */ + memcpy(cd->ucs_aes.key, key, keylen); + keylen = keylen / 2; + } else if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_CTR_MODE) { ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags, ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE); keylen = round_up(keylen, 16); @@ -458,6 +473,28 @@ static void qat_alg_skcipher_init_enc(struct qat_alg_skcipher_ctx *ctx, enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode); } +static void qat_alg_xts_reverse_key(const u8 *key_forward, unsigned int keylen, + u8 *key_reverse) +{ + struct crypto_aes_ctx aes_expanded; + int nrounds; + u8 *key; + + aes_expandkey(&aes_expanded, key_forward, keylen); + if (keylen == AES_KEYSIZE_128) { + nrounds = 10; + key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds); + memcpy(key_reverse, key, AES_BLOCK_SIZE); + } else { + /* AES_KEYSIZE_256 */ + nrounds = 14; + key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds); + memcpy(key_reverse, key, AES_BLOCK_SIZE); + memcpy(key_reverse + AES_BLOCK_SIZE, key - AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + } +} + static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx, int alg, const u8 *key, unsigned int keylen, int mode) @@ -465,16 +502,26 @@ static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx, struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd; struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req; struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; + bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev); qat_alg_skcipher_init_com(ctx, req, dec_cd, key, keylen); cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr; - if (mode != ICP_QAT_HW_CIPHER_CTR_MODE) + if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) { + /* Key reversing not supported, set no convert */ + dec_cd->aes.cipher_config.val = + QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode); + + /* In-place key reversal */ + qat_alg_xts_reverse_key(dec_cd->ucs_aes.key, keylen / 2, + dec_cd->ucs_aes.key); + } else if (mode != ICP_QAT_HW_CIPHER_CTR_MODE) { dec_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg, mode); - else + } else { dec_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode); + } } static int qat_alg_validate_key(int key_len, int *alg, int mode) @@ -1081,8 +1128,33 @@ static int qat_alg_skcipher_xts_setkey(struct crypto_skcipher *tfm, ctx->fallback = false; - return qat_alg_skcipher_setkey(tfm, key, keylen, - ICP_QAT_HW_CIPHER_XTS_MODE); + ret = qat_alg_skcipher_setkey(tfm, key, keylen, + ICP_QAT_HW_CIPHER_XTS_MODE); + if (ret) + return ret; + + if (HW_CAP_AES_V2(ctx->inst->accel_dev)) + ret = crypto_cipher_setkey(ctx->tweak, key + (keylen / 2), + keylen / 2); + + return ret; +} + +static void qat_alg_set_req_iv(struct qat_crypto_request *qat_req) +{ + struct icp_qat_fw_la_cipher_req_params *cipher_param; + struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx; + bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev); + u8 *iv = qat_req->skcipher_req->iv; + + cipher_param = (void *)&qat_req->req.serv_specif_rqpars; + + if (aes_v2_capable && ctx->mode == ICP_QAT_HW_CIPHER_XTS_MODE) + crypto_cipher_encrypt_one(ctx->tweak, + (u8 *)cipher_param->u.cipher_IV_array, + iv); + else + memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE); } static int qat_alg_skcipher_encrypt(struct skcipher_request *req) @@ -1114,7 +1186,8 @@ static int qat_alg_skcipher_encrypt(struct skcipher_request *req) cipher_param = (void *)&qat_req->req.serv_specif_rqpars; cipher_param->cipher_length = req->cryptlen; cipher_param->cipher_offset = 0; - memcpy(cipher_param->u.cipher_IV_array, req->iv, AES_BLOCK_SIZE); + + qat_alg_set_req_iv(qat_req); do { ret = adf_send_message(ctx->inst->sym_tx, (u32 *)msg); @@ -1182,8 +1255,8 @@ static int qat_alg_skcipher_decrypt(struct skcipher_request *req) cipher_param = (void *)&qat_req->req.serv_specif_rqpars; cipher_param->cipher_length = req->cryptlen; cipher_param->cipher_offset = 0; - memcpy(cipher_param->u.cipher_IV_array, req->iv, AES_BLOCK_SIZE); + qat_alg_set_req_iv(qat_req); qat_alg_update_iv(qat_req); do { @@ -1293,6 +1366,12 @@ static int qat_alg_skcipher_init_xts_tfm(struct crypto_skcipher *tfm) if (IS_ERR(ctx->ftfm)) return PTR_ERR(ctx->ftfm); + ctx->tweak = crypto_alloc_cipher("aes", 0, 0); + if (IS_ERR(ctx->tweak)) { + crypto_free_skcipher(ctx->ftfm); + return PTR_ERR(ctx->tweak); + } + reqsize = max(sizeof(struct qat_crypto_request), sizeof(struct skcipher_request) + crypto_skcipher_reqsize(ctx->ftfm)); @@ -1335,6 +1414,9 @@ static void qat_alg_skcipher_exit_xts_tfm(struct crypto_skcipher *tfm) if (ctx->ftfm) crypto_free_skcipher(ctx->ftfm); + if (ctx->tweak) + crypto_free_cipher(ctx->tweak); + qat_alg_skcipher_exit_tfm(tfm); } -- cgit v1.2.3 From 93cebeb1c21a65b92636aaa278a32fbc0415ec67 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Tue, 1 Dec 2020 14:24:51 +0000 Subject: crypto: qat - add capability detection logic in qat_4xxx Add logic to detect device capabilities in qat_4xxx driver. Read fuses and build the device capabilities mask. This will enable services and handling specific to QAT 4xxx devices. Co-developed-by: Tomaszx Kowalik Signed-off-by: Tomaszx Kowalik Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 24 ++++++++++++++++++++++++ drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h | 11 +++++++++++ drivers/crypto/qat/qat_4xxx/adf_drv.c | 3 +++ 3 files changed, 38 insertions(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c index e7a7c1e3da28..344bfae45bff 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -5,6 +5,7 @@ #include #include #include "adf_4xxx_hw_data.h" +#include "icp_qat_hw.h" struct adf_fw_config { u32 ae_mask; @@ -91,6 +92,28 @@ static void set_msix_default_rttable(struct adf_accel_dev *accel_dev) ADF_CSR_WR(csr, ADF_4XXX_MSIX_RTTABLE_OFFSET(i), i); } +static u32 get_accel_cap(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; + u32 fusectl1; + u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION | + ICP_ACCEL_CAPABILITIES_AES_V2; + + /* Read accelerator capabilities mask */ + pci_read_config_dword(pdev, ADF_4XXX_FUSECTL1_OFFSET, &fusectl1); + + if (fusectl1 & ICP_ACCEL_4XXX_MASK_CIPHER_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + if (fusectl1 & ICP_ACCEL_4XXX_MASK_AUTH_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + if (fusectl1 & ICP_ACCEL_4XXX_MASK_PKE_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + + return capabilities; +} + static enum dev_sku_info get_sku(struct adf_hw_device_data *self) { return DEV_SKU_1; @@ -189,6 +212,7 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) hw_data->get_misc_bar_id = get_misc_bar_id; hw_data->get_arb_info = get_arb_info; hw_data->get_admin_info = get_admin_info; + hw_data->get_accel_cap = get_accel_cap; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_4XXX_FW; hw_data->fw_mmp_name = ADF_4XXX_MMP; diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h index cdde0be886bf..4fe2a776293c 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h @@ -69,6 +69,17 @@ #define ADF_4XXX_ASYM_OBJ "qat_4xxx_asym.bin" #define ADF_4XXX_ADMIN_OBJ "qat_4xxx_admin.bin" +/* qat_4xxx fuse bits are different from old GENs, redefine them */ +enum icp_qat_4xxx_slice_mask { + ICP_ACCEL_4XXX_MASK_CIPHER_SLICE = BIT(0), + ICP_ACCEL_4XXX_MASK_AUTH_SLICE = BIT(1), + ICP_ACCEL_4XXX_MASK_PKE_SLICE = BIT(2), + ICP_ACCEL_4XXX_MASK_COMPRESS_SLICE = BIT(3), + ICP_ACCEL_4XXX_MASK_UCS_SLICE = BIT(4), + ICP_ACCEL_4XXX_MASK_EIA3_SLICE = BIT(5), + ICP_ACCEL_4XXX_MASK_SMX_SLICE = BIT(6), +}; + void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data); void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data); diff --git a/drivers/crypto/qat/qat_4xxx/adf_drv.c b/drivers/crypto/qat/qat_4xxx/adf_drv.c index de5a955f406a..a8805c815d16 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_drv.c +++ b/drivers/crypto/qat/qat_4xxx/adf_drv.c @@ -233,6 +233,9 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); } + /* Get accelerator capabilities mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); + /* Find and map all the device's BARS */ bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_4XXX_BAR_MASK; -- cgit v1.2.3 From c5b840a28b5257173f65b756cc6f3d06870c7881 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Wed, 16 Dec 2020 14:14:59 +0100 Subject: crypto: keembay - CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 should depend on ARCH_KEEMBAY The Intel Keem Bay Offload and Crypto Subsystem (OCS) is only present on Intel Keem Bay SoCs. Hence add a dependency on ARCH_KEEMBAY, to prevent asking the user about this driver when configuring a kernel without Intel Keem Bay platform support. While at it, fix a misspelling of "cipher". Fixes: 88574332451380f4 ("crypto: keembay - Add support for Keem Bay OCS AES/SM4") Signed-off-by: Geert Uytterhoeven Acked-by: Daniele Alessandrelli Signed-off-by: Herbert Xu --- drivers/crypto/keembay/Kconfig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/keembay/Kconfig b/drivers/crypto/keembay/Kconfig index 3c16797b25b9..6f62c838a3fa 100644 --- a/drivers/crypto/keembay/Kconfig +++ b/drivers/crypto/keembay/Kconfig @@ -1,12 +1,12 @@ config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 tristate "Support for Intel Keem Bay OCS AES/SM4 HW acceleration" - depends on OF || COMPILE_TEST + depends on ARCH_KEEMBAY || COMPILE_TEST select CRYPTO_SKCIPHER select CRYPTO_AEAD select CRYPTO_ENGINE help Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) AES and - SM4 cihper hardware acceleration for use with Crypto API. + SM4 cipher hardware acceleration for use with Crypto API. Provides HW acceleration for the following transformations: cbc(aes), ctr(aes), ccm(aes), gcm(aes), cbc(sm4), ctr(sm4), ccm(sm4) -- cgit v1.2.3 From 167316a5a08370064d7f8b96835dae271a8ce3ee Mon Sep 17 00:00:00 2001 From: Daniele Alessandrelli Date: Thu, 17 Dec 2020 16:35:10 +0000 Subject: crypto: keembay - Add dependency on HAS_IOMEM Add dependency for CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 on HAS_IOMEM to prevent build failures. Fixes: 88574332451380f4 ("crypto: keembay - Add support for Keem Bay OCS AES/SM4") Reported-by: kernel test robot Signed-off-by: Daniele Alessandrelli Signed-off-by: Herbert Xu --- drivers/crypto/keembay/Kconfig | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/keembay/Kconfig b/drivers/crypto/keembay/Kconfig index 6f62c838a3fa..f2e17b0c4fa0 100644 --- a/drivers/crypto/keembay/Kconfig +++ b/drivers/crypto/keembay/Kconfig @@ -1,5 +1,6 @@ config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 tristate "Support for Intel Keem Bay OCS AES/SM4 HW acceleration" + depends on HAS_IOMEM depends on ARCH_KEEMBAY || COMPILE_TEST select CRYPTO_SKCIPHER select CRYPTO_AEAD -- cgit v1.2.3 From c0e583ab2016de8dedfb73934d4c4e8ff5bd896c Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Tue, 22 Dec 2020 13:00:24 +0000 Subject: crypto: qat - add CRYPTO_AES to Kconfig dependencies This patch includes a missing dependency (CRYPTO_AES) which may lead to an "undefined reference to `aes_expandkey'" linking error. Fixes: 5106dfeaeabe ("crypto: qat - add AES-XTS support for QAT GEN4 devices") Reported-by: kernel test robot Signed-off-by: Marco Chiappero Signed-off-by: Herbert Xu --- drivers/crypto/qat/Kconfig | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/Kconfig b/drivers/crypto/qat/Kconfig index beb379b23dc3..846a3d90b41a 100644 --- a/drivers/crypto/qat/Kconfig +++ b/drivers/crypto/qat/Kconfig @@ -11,6 +11,7 @@ config CRYPTO_DEV_QAT select CRYPTO_SHA1 select CRYPTO_SHA256 select CRYPTO_SHA512 + select CRYPTO_AES select FW_LOADER config CRYPTO_DEV_QAT_DH895xCC -- cgit v1.2.3