diff options
Diffstat (limited to 'drivers/crypto/hisilicon')
| -rw-r--r-- | drivers/crypto/hisilicon/hpre/hpre_crypto.c | 15 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/hpre/hpre_main.c | 13 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/qm.c | 295 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/sec2/sec.h | 1 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/sec2/sec_crypto.c | 125 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/sec2/sec_main.c | 13 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/zip/Makefile | 2 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/zip/dae_main.c | 262 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/zip/zip.h | 8 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/zip/zip_main.c | 52 |
10 files changed, 617 insertions, 169 deletions
diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 2a2910261210..1550c3818383 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -39,6 +39,8 @@ struct hpre_ctx; #define HPRE_DFX_SEC_TO_US 1000000 #define HPRE_DFX_US_TO_NS 1000 +#define HPRE_ENABLE_HPCORE_SHIFT 7 + /* due to nist p521 */ #define HPRE_ECC_MAX_KSZ 66 @@ -131,6 +133,8 @@ struct hpre_ctx { }; /* for ecc algorithms */ unsigned int curve_id; + /* for high performance core */ + u8 enable_hpcore; }; struct hpre_asym_request { @@ -1487,11 +1491,13 @@ static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp) if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); + /* Do unmap before data processing */ + hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src); + p = sg_virt(areq->dst); memmove(p, p + ctx->key_sz - curve_sz, curve_sz); memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz); - hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src); kpp_request_complete(areq, ret); atomic64_inc(&dfx[HPRE_RECV_CNT].value); @@ -1619,6 +1625,8 @@ static int hpre_ecdh_compute_value(struct kpp_request *req) } msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_ECC_MUL); + msg->resv1 = ctx->enable_hpcore << HPRE_ENABLE_HPCORE_SHIFT; + ret = hpre_send(ctx, msg); if (likely(!ret)) return -EINPROGRESS; @@ -1653,6 +1661,7 @@ static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); ctx->curve_id = ECC_CURVE_NIST_P256; + ctx->enable_hpcore = 1; kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); @@ -1801,9 +1810,11 @@ static void hpre_curve25519_cb(struct hpre_ctx *ctx, void *resp) if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); + /* Do unmap before data processing */ + hpre_curve25519_hw_data_clr_all(ctx, req, areq->dst, areq->src); + hpre_key_to_big_end(sg_virt(areq->dst), CURVE25519_KEY_SIZE); - hpre_curve25519_hw_data_clr_all(ctx, req, areq->dst, areq->src); kpp_request_complete(areq, ret); atomic64_inc(&dfx[HPRE_RECV_CNT].value); diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 96fde9437b4b..f5b47e5ff48a 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -1209,7 +1209,6 @@ static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->mode = uacce_mode; qm->pdev = pdev; - qm->ver = pdev->revision; qm->sqe_size = HPRE_SQE_SIZE; qm->dev_name = hpre_name; @@ -1396,6 +1395,17 @@ static enum acc_err_result hpre_get_err_result(struct hisi_qm *qm) return ACC_ERR_RECOVERED; } +static bool hpre_dev_is_abnormal(struct hisi_qm *qm) +{ + u32 err_status; + + err_status = hpre_get_hw_err_status(qm); + if (err_status & qm->err_info.dev_shutdown_mask) + return true; + + return false; +} + static void hpre_err_info_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; @@ -1428,6 +1438,7 @@ static const struct hisi_qm_err_ini hpre_err_ini = { .show_last_dfx_regs = hpre_show_last_dfx_regs, .err_info_init = hpre_err_info_init, .get_err_result = hpre_get_err_result, + .dev_is_abnormal = hpre_dev_is_abnormal, }; static int hpre_pf_probe_init(struct hpre *hpre) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 19c1b5d3c954..7c41f9593d03 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -30,8 +30,6 @@ /* mailbox */ #define QM_MB_PING_ALL_VFS 0xffff -#define QM_MB_CMD_DATA_SHIFT 32 -#define QM_MB_CMD_DATA_MASK GENMASK(31, 0) #define QM_MB_STATUS_MASK GENMASK(12, 9) /* sqc shift */ @@ -102,6 +100,8 @@ #define QM_PM_CTRL 0x100148 #define QM_IDLE_DISABLE BIT(9) +#define QM_SUB_VERSION_ID 0x210 + #define QM_VFT_CFG_DATA_L 0x100064 #define QM_VFT_CFG_DATA_H 0x100068 #define QM_SQC_VFT_BUF_SIZE (7ULL << 8) @@ -119,6 +119,7 @@ #define QM_SQC_VFT_BASE_MASK_V2 GENMASK(15, 0) #define QM_SQC_VFT_NUM_SHIFT_V2 45 #define QM_SQC_VFT_NUM_MASK_V2 GENMASK(9, 0) +#define QM_MAX_QC_TYPE 2 #define QM_ABNORMAL_INT_SOURCE 0x100000 #define QM_ABNORMAL_INT_MASK 0x100004 @@ -176,6 +177,10 @@ #define QM_IFC_INT_MASK 0x0024 #define QM_IFC_INT_STATUS 0x0028 #define QM_IFC_INT_SET_V 0x002C +#define QM_PF2VF_PF_W 0x104700 +#define QM_VF2PF_PF_R 0x104800 +#define QM_VF2PF_VF_W 0x320 +#define QM_PF2VF_VF_R 0x380 #define QM_IFC_SEND_ALL_VFS GENMASK(6, 0) #define QM_IFC_INT_SOURCE_CLR GENMASK(63, 0) #define QM_IFC_INT_SOURCE_MASK BIT(0) @@ -185,8 +190,11 @@ #define QM_WAIT_DST_ACK 10 #define QM_MAX_PF_WAIT_COUNT 10 #define QM_MAX_VF_WAIT_COUNT 40 -#define QM_VF_RESET_WAIT_US 20000 -#define QM_VF_RESET_WAIT_CNT 3000 +#define QM_VF_RESET_WAIT_US 20000 +#define QM_VF_RESET_WAIT_CNT 3000 +#define QM_VF2PF_REG_SIZE 4 +#define QM_IFC_CMD_MASK GENMASK(31, 0) +#define QM_IFC_DATA_SHIFT 32 #define QM_VF_RESET_WAIT_TIMEOUT_US \ (QM_VF_RESET_WAIT_US * QM_VF_RESET_WAIT_CNT) @@ -234,8 +242,6 @@ #define QM_QOS_MAX_CIR_U 6 #define QM_AUTOSUSPEND_DELAY 3000 -#define QM_DEV_ALG_MAX_LEN 256 - /* abnormal status value for stopping queue */ #define QM_STOP_QUEUE_FAIL 1 #define QM_DUMP_SQC_FAIL 3 @@ -276,7 +282,7 @@ enum qm_alg_type { ALG_TYPE_1, }; -enum qm_mb_cmd { +enum qm_ifc_cmd { QM_PF_FLR_PREPARE = 0x01, QM_PF_SRST_PREPARE, QM_PF_RESET_DONE, @@ -333,6 +339,7 @@ static const struct hisi_qm_cap_info qm_cap_info_comm[] = { {QM_SUPPORT_STOP_FUNC, 0x3100, 0, BIT(10), 0x0, 0x0, 0x1}, {QM_SUPPORT_MB_COMMAND, 0x3100, 0, BIT(11), 0x0, 0x0, 0x1}, {QM_SUPPORT_SVA_PREFETCH, 0x3100, 0, BIT(14), 0x0, 0x0, 0x1}, + {QM_SUPPORT_DAE, 0x3100, 0, BIT(15), 0x0, 0x0, 0x0}, }; static const struct hisi_qm_cap_info qm_cap_info_pf[] = { @@ -396,6 +403,11 @@ struct hisi_qm_hw_ops { void (*hw_error_uninit)(struct hisi_qm *qm); enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm); int (*set_msi)(struct hisi_qm *qm, bool set); + + /* (u64)msg = (u32)data << 32 | (enum qm_ifc_cmd)cmd */ + int (*set_ifc_begin)(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num); + void (*set_ifc_end)(struct hisi_qm *qm); + int (*get_ifc)(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num); }; struct hisi_qm_hw_error { @@ -501,15 +513,20 @@ static u32 qm_get_dev_err_status(struct hisi_qm *qm) /* Check if the error causes the master ooo block */ static bool qm_check_dev_error(struct hisi_qm *qm) { - u32 val, dev_val; + struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); + u32 err_status; - if (qm->fun_type == QM_HW_VF) + if (pf_qm->fun_type == QM_HW_VF) return false; - val = qm_get_hw_error_status(qm) & qm->err_info.qm_shutdown_mask; - dev_val = qm_get_dev_err_status(qm) & qm->err_info.dev_shutdown_mask; + err_status = qm_get_hw_error_status(pf_qm); + if (err_status & pf_qm->err_info.qm_shutdown_mask) + return true; + + if (pf_qm->err_ini->dev_is_abnormal) + return pf_qm->err_ini->dev_is_abnormal(pf_qm); - return val || dev_val; + return false; } static int qm_wait_reset_finish(struct hisi_qm *qm) @@ -654,7 +671,6 @@ EXPORT_SYMBOL_GPL(hisi_qm_mb); /* op 0: set xqc information to hardware, 1: get xqc information from hardware. */ int qm_set_and_get_xqc(struct hisi_qm *qm, u8 cmd, void *xqc, u32 qp_id, bool op) { - struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); struct qm_mailbox mailbox; dma_addr_t xqc_dma; void *tmp_xqc; @@ -688,7 +704,7 @@ int qm_set_and_get_xqc(struct hisi_qm *qm, u8 cmd, void *xqc, u32 qp_id, bool op } /* Setting xqc will fail if master OOO is blocked. */ - if (qm_check_dev_error(pf_qm)) { + if (qm_check_dev_error(qm)) { dev_err(&qm->pdev->dev, "failed to send mailbox since qm is stop!\n"); return -EIO; } @@ -846,7 +862,7 @@ int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *d return -EINVAL; } - algs = devm_kzalloc(dev, QM_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL); + algs = devm_kzalloc(dev, QM_DEV_ALG_MAX_LEN, GFP_KERNEL); if (!algs) return -ENOMEM; @@ -855,10 +871,10 @@ int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *d strcat(algs, dev_algs[i].alg); ptr = strrchr(algs, '\n'); - if (ptr) { + if (ptr) *ptr = '\0'; - qm->uacce->algs = algs; - } + + qm->uacce->algs = algs; return 0; } @@ -1052,11 +1068,10 @@ static void qm_disable_qp(struct hisi_qm *qm, u32 qp_id) static void qm_reset_function(struct hisi_qm *qm) { - struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); struct device *dev = &qm->pdev->dev; int ret; - if (qm_check_dev_error(pf_qm)) + if (qm_check_dev_error(qm)) return; ret = qm_reset_prepare_ready(qm); @@ -1540,17 +1555,15 @@ static void qm_clear_cmd_interrupt(struct hisi_qm *qm, u64 vf_mask) static void qm_handle_vf_msg(struct hisi_qm *qm, u32 vf_id) { struct device *dev = &qm->pdev->dev; - u32 cmd; - u64 msg; + enum qm_ifc_cmd cmd; int ret; - ret = qm_get_mb_cmd(qm, &msg, vf_id); + ret = qm->ops->get_ifc(qm, &cmd, NULL, vf_id); if (ret) { - dev_err(dev, "failed to get msg from VF(%u)!\n", vf_id); + dev_err(dev, "failed to get command from VF(%u)!\n", vf_id); return; } - cmd = msg & QM_MB_CMD_DATA_MASK; switch (cmd) { case QM_VF_PREPARE_FAIL: dev_err(dev, "failed to stop VF(%u)!\n", vf_id); @@ -1562,7 +1575,7 @@ static void qm_handle_vf_msg(struct hisi_qm *qm, u32 vf_id) case QM_VF_START_DONE: break; default: - dev_err(dev, "unsupported cmd %u sent by VF(%u)!\n", cmd, vf_id); + dev_err(dev, "unsupported command(0x%x) sent by VF(%u)!\n", cmd, vf_id); break; } } @@ -1630,17 +1643,14 @@ static void qm_trigger_pf_interrupt(struct hisi_qm *qm) writel(val, qm->io_base + QM_IFC_INT_SET_V); } -static int qm_ping_single_vf(struct hisi_qm *qm, u64 cmd, u32 fun_num) +static int qm_ping_single_vf(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) { struct device *dev = &qm->pdev->dev; - struct qm_mailbox mailbox; int cnt = 0; u64 val; int ret; - qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, fun_num, 0); - mutex_lock(&qm->mailbox_lock); - ret = qm_mb_nolock(qm, &mailbox); + ret = qm->ops->set_ifc_begin(qm, cmd, data, fun_num); if (ret) { dev_err(dev, "failed to send command to vf(%u)!\n", fun_num); goto err_unlock; @@ -1662,27 +1672,23 @@ static int qm_ping_single_vf(struct hisi_qm *qm, u64 cmd, u32 fun_num) } err_unlock: - mutex_unlock(&qm->mailbox_lock); + qm->ops->set_ifc_end(qm); return ret; } -static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd) +static int qm_ping_all_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd) { struct device *dev = &qm->pdev->dev; u32 vfs_num = qm->vfs_num; - struct qm_mailbox mailbox; u64 val = 0; int cnt = 0; int ret; u32 i; - qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, QM_MB_PING_ALL_VFS, 0); - mutex_lock(&qm->mailbox_lock); - /* PF sends command to all VFs by mailbox */ - ret = qm_mb_nolock(qm, &mailbox); + ret = qm->ops->set_ifc_begin(qm, cmd, 0, QM_MB_PING_ALL_VFS); if (ret) { - dev_err(dev, "failed to send command to VFs!\n"); - mutex_unlock(&qm->mailbox_lock); + dev_err(dev, "failed to send command(0x%x) to all vfs!\n", cmd); + qm->ops->set_ifc_end(qm); return ret; } @@ -1692,7 +1698,7 @@ static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd) val = readq(qm->io_base + QM_IFC_READY_STATUS); /* If all VFs acked, PF notifies VFs successfully. */ if (!(val & GENMASK(vfs_num, 1))) { - mutex_unlock(&qm->mailbox_lock); + qm->ops->set_ifc_end(qm); return 0; } @@ -1700,7 +1706,7 @@ static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd) break; } - mutex_unlock(&qm->mailbox_lock); + qm->ops->set_ifc_end(qm); /* Check which vf respond timeout. */ for (i = 1; i <= vfs_num; i++) { @@ -1711,18 +1717,15 @@ static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd) return -ETIMEDOUT; } -static int qm_ping_pf(struct hisi_qm *qm, u64 cmd) +static int qm_ping_pf(struct hisi_qm *qm, enum qm_ifc_cmd cmd) { - struct qm_mailbox mailbox; int cnt = 0; u32 val; int ret; - qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, 0, 0); - mutex_lock(&qm->mailbox_lock); - ret = qm_mb_nolock(qm, &mailbox); + ret = qm->ops->set_ifc_begin(qm, cmd, 0, 0); if (ret) { - dev_err(&qm->pdev->dev, "failed to send command to PF!\n"); + dev_err(&qm->pdev->dev, "failed to send command(0x%x) to PF!\n", cmd); goto unlock; } @@ -1741,7 +1744,8 @@ static int qm_ping_pf(struct hisi_qm *qm, u64 cmd) } unlock: - mutex_unlock(&qm->mailbox_lock); + qm->ops->set_ifc_end(qm); + return ret; } @@ -1842,6 +1846,94 @@ static int qm_set_msi_v3(struct hisi_qm *qm, bool set) return ret; } +static int qm_set_ifc_begin_v3(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) +{ + struct qm_mailbox mailbox; + u64 msg; + + msg = cmd | (u64)data << QM_IFC_DATA_SHIFT; + + qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, msg, fun_num, 0); + mutex_lock(&qm->mailbox_lock); + return qm_mb_nolock(qm, &mailbox); +} + +static void qm_set_ifc_end_v3(struct hisi_qm *qm) +{ + mutex_unlock(&qm->mailbox_lock); +} + +static int qm_get_ifc_v3(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num) +{ + u64 msg; + int ret; + + ret = qm_get_mb_cmd(qm, &msg, fun_num); + if (ret) + return ret; + + *cmd = msg & QM_IFC_CMD_MASK; + + if (data) + *data = msg >> QM_IFC_DATA_SHIFT; + + return 0; +} + +static int qm_set_ifc_begin_v4(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) +{ + uintptr_t offset; + u64 msg; + + if (qm->fun_type == QM_HW_PF) + offset = QM_PF2VF_PF_W; + else + offset = QM_VF2PF_VF_W; + + msg = cmd | (u64)data << QM_IFC_DATA_SHIFT; + + mutex_lock(&qm->ifc_lock); + writeq(msg, qm->io_base + offset); + + return 0; +} + +static void qm_set_ifc_end_v4(struct hisi_qm *qm) +{ + mutex_unlock(&qm->ifc_lock); +} + +static u64 qm_get_ifc_pf(struct hisi_qm *qm, u32 fun_num) +{ + uintptr_t offset; + + offset = QM_VF2PF_PF_R + QM_VF2PF_REG_SIZE * fun_num; + + return (u64)readl(qm->io_base + offset); +} + +static u64 qm_get_ifc_vf(struct hisi_qm *qm) +{ + return readq(qm->io_base + QM_PF2VF_VF_R); +} + +static int qm_get_ifc_v4(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num) +{ + u64 msg; + + if (qm->fun_type == QM_HW_PF) + msg = qm_get_ifc_pf(qm, fun_num); + else + msg = qm_get_ifc_vf(qm); + + *cmd = msg & QM_IFC_CMD_MASK; + + if (data) + *data = msg >> QM_IFC_DATA_SHIFT; + + return 0; +} + static const struct hisi_qm_hw_ops qm_hw_ops_v1 = { .qm_db = qm_db_v1, .hw_error_init = qm_hw_error_init_v1, @@ -1864,6 +1956,21 @@ static const struct hisi_qm_hw_ops qm_hw_ops_v3 = { .hw_error_uninit = qm_hw_error_uninit_v3, .hw_error_handle = qm_hw_error_handle_v2, .set_msi = qm_set_msi_v3, + .set_ifc_begin = qm_set_ifc_begin_v3, + .set_ifc_end = qm_set_ifc_end_v3, + .get_ifc = qm_get_ifc_v3, +}; + +static const struct hisi_qm_hw_ops qm_hw_ops_v4 = { + .get_vft = qm_get_vft_v2, + .qm_db = qm_db_v2, + .hw_error_init = qm_hw_error_init_v3, + .hw_error_uninit = qm_hw_error_uninit_v3, + .hw_error_handle = qm_hw_error_handle_v2, + .set_msi = qm_set_msi_v3, + .set_ifc_begin = qm_set_ifc_begin_v4, + .set_ifc_end = qm_set_ifc_end_v4, + .get_ifc = qm_get_ifc_v4, }; static void *qm_get_avail_sqe(struct hisi_qp *qp) @@ -2156,12 +2263,11 @@ static int qm_wait_qp_empty(struct hisi_qm *qm, u32 *state, u32 qp_id) static int qm_drain_qp(struct hisi_qp *qp) { struct hisi_qm *qm = qp->qm; - struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); u32 state = 0; int ret; /* No need to judge if master OOO is blocked. */ - if (qm_check_dev_error(pf_qm)) + if (qm_check_dev_error(qm)) return 0; /* HW V3 supports drain qp by device */ @@ -2475,7 +2581,7 @@ static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd, sizeof(struct hisi_qp_ctx))) return -EFAULT; - if (qp_ctx.qc_type != 0 && qp_ctx.qc_type != 1) + if (qp_ctx.qc_type > QM_MAX_QC_TYPE) return -EINVAL; qm_set_sqctype(q, qp_ctx.qc_type); @@ -2843,11 +2949,14 @@ static void hisi_qm_pre_init(struct hisi_qm *qm) qm->ops = &qm_hw_ops_v1; else if (qm->ver == QM_HW_V2) qm->ops = &qm_hw_ops_v2; - else + else if (qm->ver == QM_HW_V3) qm->ops = &qm_hw_ops_v3; + else + qm->ops = &qm_hw_ops_v4; pci_set_drvdata(pdev, qm); mutex_init(&qm->mailbox_lock); + mutex_init(&qm->ifc_lock); init_rwsem(&qm->qps_lock); qm->qp_in_used = 0; if (test_bit(QM_SUPPORT_RPM, &qm->caps)) { @@ -3607,7 +3716,6 @@ static u32 qm_get_shaper_vft_qos(struct hisi_qm *qm, u32 fun_index) static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num) { struct device *dev = &qm->pdev->dev; - u64 mb_cmd; u32 qos; int ret; @@ -3617,10 +3725,9 @@ static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num) return; } - mb_cmd = QM_PF_SET_QOS | (u64)qos << QM_MB_CMD_DATA_SHIFT; - ret = qm_ping_single_vf(qm, mb_cmd, fun_num); + ret = qm_ping_single_vf(qm, QM_PF_SET_QOS, qos, fun_num); if (ret) - dev_err(dev, "failed to send cmd to VF(%u)!\n", fun_num); + dev_err(dev, "failed to send command(0x%x) to VF(%u)!\n", QM_PF_SET_QOS, fun_num); } static int qm_vf_read_qos(struct hisi_qm *qm) @@ -4109,7 +4216,7 @@ stop_fail: return ret; } -static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd, +static int qm_try_stop_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd, enum qm_stop_reason stop_reason) { struct pci_dev *pdev = qm->pdev; @@ -4122,7 +4229,7 @@ static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd, if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { ret = qm_ping_all_vfs(qm, cmd); if (ret) - pci_err(pdev, "failed to send cmd to all VFs before PF reset!\n"); + pci_err(pdev, "failed to send command to all VFs before PF reset!\n"); } else { ret = qm_vf_reset_prepare(qm, stop_reason); if (ret) @@ -4137,6 +4244,12 @@ static int qm_controller_reset_prepare(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; int ret; + if (qm->err_ini->set_priv_status) { + ret = qm->err_ini->set_priv_status(qm); + if (ret) + return ret; + } + ret = qm_reset_prepare_ready(qm); if (ret) { pci_err(pdev, "Controller reset not ready!\n"); @@ -4298,7 +4411,7 @@ restart_fail: return ret; } -static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_mb_cmd cmd) +static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd) { struct pci_dev *pdev = qm->pdev; int ret; @@ -4527,7 +4640,7 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) * Check whether there is an ECC mbit error, If it occurs, need to * wait for soft reset to fix it. */ - while (qm_check_dev_error(pf_qm)) { + while (qm_check_dev_error(qm)) { msleep(++delay); if (delay > QM_RESET_WAIT_TIMEOUT) return; @@ -4675,7 +4788,7 @@ static void hisi_qm_controller_reset(struct work_struct *rst_work) static void qm_pf_reset_vf_prepare(struct hisi_qm *qm, enum qm_stop_reason stop_reason) { - enum qm_mb_cmd cmd = QM_VF_PREPARE_DONE; + enum qm_ifc_cmd cmd = QM_VF_PREPARE_DONE; struct pci_dev *pdev = qm->pdev; int ret; @@ -4709,7 +4822,7 @@ out: static void qm_pf_reset_vf_done(struct hisi_qm *qm) { - enum qm_mb_cmd cmd = QM_VF_START_DONE; + enum qm_ifc_cmd cmd = QM_VF_START_DONE; struct pci_dev *pdev = qm->pdev; int ret; @@ -4732,7 +4845,6 @@ static int qm_wait_pf_reset_finish(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; u32 val, cmd; - u64 msg; int ret; /* Wait for reset to finish */ @@ -4749,16 +4861,15 @@ static int qm_wait_pf_reset_finish(struct hisi_qm *qm) * Whether message is got successfully, * VF needs to ack PF by clearing the interrupt. */ - ret = qm_get_mb_cmd(qm, &msg, 0); + ret = qm->ops->get_ifc(qm, &cmd, NULL, 0); qm_clear_cmd_interrupt(qm, 0); if (ret) { - dev_err(dev, "failed to get msg from PF in reset done!\n"); + dev_err(dev, "failed to get command from PF in reset done!\n"); return ret; } - cmd = msg & QM_MB_CMD_DATA_MASK; if (cmd != QM_PF_RESET_DONE) { - dev_err(dev, "the cmd(%u) is not reset done!\n", cmd); + dev_err(dev, "the command(0x%x) is not reset done!\n", cmd); ret = -EINVAL; } @@ -4795,22 +4906,21 @@ err_get_status: static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num) { struct device *dev = &qm->pdev->dev; - u64 msg; - u32 cmd; + enum qm_ifc_cmd cmd; + u32 data; int ret; /* * Get the msg from source by sending mailbox. Whether message is got * successfully, destination needs to ack source by clearing the interrupt. */ - ret = qm_get_mb_cmd(qm, &msg, fun_num); + ret = qm->ops->get_ifc(qm, &cmd, &data, fun_num); qm_clear_cmd_interrupt(qm, BIT(fun_num)); if (ret) { - dev_err(dev, "failed to get msg from source!\n"); + dev_err(dev, "failed to get command from source!\n"); return; } - cmd = msg & QM_MB_CMD_DATA_MASK; switch (cmd) { case QM_PF_FLR_PREPARE: qm_pf_reset_vf_process(qm, QM_DOWN); @@ -4822,10 +4932,10 @@ static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num) qm_vf_get_qos(qm, fun_num); break; case QM_PF_SET_QOS: - qm->mb_qos = msg >> QM_MB_CMD_DATA_SHIFT; + qm->mb_qos = data; break; default: - dev_err(dev, "unsupported cmd %u sent by function(%u)!\n", cmd, fun_num); + dev_err(dev, "unsupported command(0x%x) sent by function(%u)!\n", cmd, fun_num); break; } } @@ -5114,7 +5224,7 @@ static int qm_pre_store_caps(struct hisi_qm *qm) size_t i, size; size = ARRAY_SIZE(qm_cap_query_info); - qm_cap = devm_kzalloc(&pdev->dev, sizeof(*qm_cap) * size, GFP_KERNEL); + qm_cap = devm_kcalloc(&pdev->dev, sizeof(*qm_cap), size, GFP_KERNEL); if (!qm_cap) return -ENOMEM; @@ -5167,6 +5277,20 @@ static int qm_get_hw_caps(struct hisi_qm *qm) return qm_pre_store_caps(qm); } +static void qm_get_version(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 sub_version_id; + + qm->ver = pdev->revision; + + if (pdev->revision == QM_HW_V3) { + sub_version_id = readl(qm->io_base + QM_SUB_VERSION_ID); + if (sub_version_id) + qm->ver = sub_version_id; + } +} + static int qm_get_pci_res(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -5186,6 +5310,8 @@ static int qm_get_pci_res(struct hisi_qm *qm) goto err_request_mem_regions; } + qm_get_version(qm); + ret = qm_get_hw_caps(qm); if (ret) goto err_ioremap; @@ -5205,6 +5331,7 @@ static int qm_get_pci_res(struct hisi_qm *qm) qm->db_interval = 0; } + hisi_qm_pre_init(qm); ret = qm_get_qp_num(qm); if (ret) goto err_db_ioremap; @@ -5247,6 +5374,14 @@ static int qm_clear_device(struct hisi_qm *qm) return ret; } + if (qm->err_ini->set_priv_status) { + ret = qm->err_ini->set_priv_status(qm); + if (ret) { + writel(0x0, qm->io_base + ACC_MASTER_GLOBAL_CTRL); + return ret; + } + } + return qm_reset_device(qm); } @@ -5461,8 +5596,6 @@ int hisi_qm_init(struct hisi_qm *qm) struct device *dev = &pdev->dev; int ret; - hisi_qm_pre_init(qm); - ret = hisi_qm_pci_init(qm); if (ret) return ret; @@ -5598,6 +5731,12 @@ static int qm_prepare_for_suspend(struct hisi_qm *qm) if (ret) return ret; + if (qm->err_ini->set_priv_status) { + ret = qm->err_ini->set_priv_status(qm); + if (ret) + return ret; + } + ret = qm_set_pf_mse(qm, false); if (ret) pci_err(pdev, "failed to disable MSE before suspending!\n"); diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 4b9970230822..703920b49c7c 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -37,7 +37,6 @@ struct sec_aead_req { u8 *a_ivin; dma_addr_t a_ivin_dma; struct aead_request *aead_req; - bool fallback; }; /* SEC request of Crypto */ diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 66bc07da9eb6..8ea5305bc320 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -57,7 +57,6 @@ #define SEC_TYPE_MASK 0x0F #define SEC_DONE_MASK 0x0001 #define SEC_ICV_MASK 0x000E -#define SEC_SQE_LEN_RATE_MASK 0x3 #define SEC_TOTAL_IV_SZ(depth) (SEC_IV_SIZE * (depth)) #define SEC_SGL_SGE_NR 128 @@ -80,16 +79,16 @@ #define SEC_TOTAL_PBUF_SZ(depth) (PAGE_SIZE * SEC_PBUF_PAGE_NUM(depth) + \ SEC_PBUF_LEFT_SZ(depth)) -#define SEC_SQE_LEN_RATE 4 #define SEC_SQE_CFLAG 2 #define SEC_SQE_AEAD_FLAG 3 #define SEC_SQE_DONE 0x1 #define SEC_ICV_ERR 0x2 -#define MIN_MAC_LEN 4 #define MAC_LEN_MASK 0x1U #define MAX_INPUT_DATA_LEN 0xFFFE00 #define BITS_MASK 0xFF +#define WORD_MASK 0x3 #define BYTE_BITS 0x8 +#define BYTES_TO_WORDS(bcount) ((bcount) >> 2) #define SEC_XTS_NAME_SZ 0x3 #define IV_CM_CAL_NUM 2 #define IV_CL_MASK 0x7 @@ -691,14 +690,10 @@ static int sec_skcipher_fbtfm_init(struct crypto_skcipher *tfm) c_ctx->fallback = false; - /* Currently, only XTS mode need fallback tfm when using 192bit key */ - if (likely(strncmp(alg, "xts", SEC_XTS_NAME_SZ))) - return 0; - c_ctx->fbtfm = crypto_alloc_sync_skcipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(c_ctx->fbtfm)) { - pr_err("failed to alloc xts mode fallback tfm!\n"); + pr_err("failed to alloc fallback tfm for %s!\n", alg); return PTR_ERR(c_ctx->fbtfm); } @@ -858,7 +853,7 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, } memcpy(c_ctx->c_key, key, keylen); - if (c_ctx->fallback && c_ctx->fbtfm) { + if (c_ctx->fbtfm) { ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); if (ret) { dev_err(dev, "failed to set fallback skcipher key!\n"); @@ -1090,11 +1085,6 @@ static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, struct crypto_shash *hash_tfm = ctx->hash_tfm; int blocksize, digestsize, ret; - if (!keys->authkeylen) { - pr_err("hisi_sec2: aead auth key error!\n"); - return -EINVAL; - } - blocksize = crypto_shash_blocksize(hash_tfm); digestsize = crypto_shash_digestsize(hash_tfm); if (keys->authkeylen > blocksize) { @@ -1106,7 +1096,8 @@ static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, } ctx->a_key_len = digestsize; } else { - memcpy(ctx->a_key, keys->authkey, keys->authkeylen); + if (keys->authkeylen) + memcpy(ctx->a_key, keys->authkey, keys->authkeylen); ctx->a_key_len = keys->authkeylen; } @@ -1160,8 +1151,10 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, } ret = crypto_authenc_extractkeys(&keys, key, keylen); - if (ret) + if (ret) { + dev_err(dev, "sec extract aead keys err!\n"); goto bad_key; + } ret = sec_aead_aes_set_key(c_ctx, &keys); if (ret) { @@ -1175,12 +1168,6 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, goto bad_key; } - if (ctx->a_ctx.a_key_len & SEC_SQE_LEN_RATE_MASK) { - ret = -EINVAL; - dev_err(dev, "AUTH key length error!\n"); - goto bad_key; - } - ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); if (ret) { dev_err(dev, "set sec fallback key err!\n"); @@ -1583,11 +1570,10 @@ static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma); - sec_sqe->type2.mac_key_alg = cpu_to_le32(authsize / SEC_SQE_LEN_RATE); + sec_sqe->type2.mac_key_alg = cpu_to_le32(BYTES_TO_WORDS(authsize)); sec_sqe->type2.mac_key_alg |= - cpu_to_le32((u32)((ctx->a_key_len) / - SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET); + cpu_to_le32((u32)BYTES_TO_WORDS(ctx->a_key_len) << SEC_AKEY_OFFSET); sec_sqe->type2.mac_key_alg |= cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET); @@ -1639,12 +1625,10 @@ static void sec_auth_bd_fill_ex_v3(struct sec_auth_ctx *ctx, int dir, sqe3->a_key_addr = cpu_to_le64(ctx->a_key_dma); sqe3->auth_mac_key |= - cpu_to_le32((u32)(authsize / - SEC_SQE_LEN_RATE) << SEC_MAC_OFFSET_V3); + cpu_to_le32(BYTES_TO_WORDS(authsize) << SEC_MAC_OFFSET_V3); sqe3->auth_mac_key |= - cpu_to_le32((u32)(ctx->a_key_len / - SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET_V3); + cpu_to_le32((u32)BYTES_TO_WORDS(ctx->a_key_len) << SEC_AKEY_OFFSET_V3); sqe3->auth_mac_key |= cpu_to_le32((u32)(ctx->a_alg) << SEC_AUTH_ALG_OFFSET_V3); @@ -2003,8 +1987,7 @@ static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm) return sec_aead_ctx_init(tfm, "sha512"); } -static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx, - struct sec_req *sreq) +static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx, struct sec_req *sreq) { u32 cryptlen = sreq->c_req.sk_req->cryptlen; struct device *dev = ctx->dev; @@ -2026,10 +2009,6 @@ static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx, } break; case SEC_CMODE_CTR: - if (unlikely(ctx->sec->qm.ver < QM_HW_V3)) { - dev_err(dev, "skcipher HW version error!\n"); - ret = -EINVAL; - } break; default: ret = -EINVAL; @@ -2038,17 +2017,21 @@ static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx, return ret; } -static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) +static int sec_skcipher_param_check(struct sec_ctx *ctx, + struct sec_req *sreq, bool *need_fallback) { struct skcipher_request *sk_req = sreq->c_req.sk_req; struct device *dev = ctx->dev; u8 c_alg = ctx->c_ctx.c_alg; - if (unlikely(!sk_req->src || !sk_req->dst || - sk_req->cryptlen > MAX_INPUT_DATA_LEN)) { + if (unlikely(!sk_req->src || !sk_req->dst)) { dev_err(dev, "skcipher input param error!\n"); return -EINVAL; } + + if (sk_req->cryptlen > MAX_INPUT_DATA_LEN) + *need_fallback = true; + sreq->c_req.c_len = sk_req->cryptlen; if (ctx->pbuf_supported && sk_req->cryptlen <= SEC_PBUF_SZ) @@ -2106,6 +2089,7 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req); struct sec_req *req = skcipher_request_ctx(sk_req); struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + bool need_fallback = false; int ret; if (!sk_req->cryptlen) { @@ -2119,11 +2103,11 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) req->c_req.encrypt = encrypt; req->ctx = ctx; - ret = sec_skcipher_param_check(ctx, req); + ret = sec_skcipher_param_check(ctx, req, &need_fallback); if (unlikely(ret)) return -EINVAL; - if (unlikely(ctx->c_ctx.fallback)) + if (unlikely(ctx->c_ctx.fallback || need_fallback)) return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); return ctx->req_op->process(ctx, req); @@ -2231,52 +2215,35 @@ static int sec_aead_spec_check(struct sec_ctx *ctx, struct sec_req *sreq) struct crypto_aead *tfm = crypto_aead_reqtfm(req); size_t sz = crypto_aead_authsize(tfm); u8 c_mode = ctx->c_ctx.c_mode; - struct device *dev = ctx->dev; int ret; - /* Hardware does not handle cases where authsize is less than 4 bytes */ - if (unlikely(sz < MIN_MAC_LEN)) { - sreq->aead_req.fallback = true; + if (unlikely(ctx->sec->qm.ver == QM_HW_V2 && !sreq->c_req.c_len)) return -EINVAL; - } if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN || - req->assoclen > SEC_MAX_AAD_LEN)) { - dev_err(dev, "aead input spec error!\n"); + req->assoclen > SEC_MAX_AAD_LEN)) return -EINVAL; - } if (c_mode == SEC_CMODE_CCM) { - if (unlikely(req->assoclen > SEC_MAX_CCM_AAD_LEN)) { - dev_err_ratelimited(dev, "CCM input aad parameter is too long!\n"); + if (unlikely(req->assoclen > SEC_MAX_CCM_AAD_LEN)) return -EINVAL; - } - ret = aead_iv_demension_check(req); - if (ret) { - dev_err(dev, "aead input iv param error!\n"); - return ret; - } - } - if (sreq->c_req.encrypt) - sreq->c_req.c_len = req->cryptlen; - else - sreq->c_req.c_len = req->cryptlen - sz; - if (c_mode == SEC_CMODE_CBC) { - if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { - dev_err(dev, "aead crypto length error!\n"); + ret = aead_iv_demension_check(req); + if (unlikely(ret)) + return -EINVAL; + } else if (c_mode == SEC_CMODE_CBC) { + if (unlikely(sz & WORD_MASK)) + return -EINVAL; + if (unlikely(ctx->a_ctx.a_key_len & WORD_MASK)) return -EINVAL; - } } return 0; } -static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) +static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq, bool *need_fallback) { struct aead_request *req = sreq->aead_req.aead_req; - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - size_t authsize = crypto_aead_authsize(tfm); struct device *dev = ctx->dev; u8 c_alg = ctx->c_ctx.c_alg; @@ -2285,12 +2252,10 @@ static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) return -EINVAL; } - if (ctx->sec->qm.ver == QM_HW_V2) { - if (unlikely(!req->cryptlen || (!sreq->c_req.encrypt && - req->cryptlen <= authsize))) { - sreq->aead_req.fallback = true; - return -EINVAL; - } + if (unlikely(ctx->c_ctx.c_mode == SEC_CMODE_CBC && + sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { + dev_err(dev, "aead cbc mode input data length error!\n"); + return -EINVAL; } /* Support AES or SM4 */ @@ -2299,8 +2264,10 @@ static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) return -EINVAL; } - if (unlikely(sec_aead_spec_check(ctx, sreq))) + if (unlikely(sec_aead_spec_check(ctx, sreq))) { + *need_fallback = true; return -EINVAL; + } if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <= SEC_PBUF_SZ) @@ -2344,17 +2311,19 @@ static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); struct sec_req *req = aead_request_ctx(a_req); struct sec_ctx *ctx = crypto_aead_ctx(tfm); + size_t sz = crypto_aead_authsize(tfm); + bool need_fallback = false; int ret; req->flag = a_req->base.flags; req->aead_req.aead_req = a_req; req->c_req.encrypt = encrypt; req->ctx = ctx; - req->aead_req.fallback = false; + req->c_req.c_len = a_req->cryptlen - (req->c_req.encrypt ? 0 : sz); - ret = sec_aead_param_check(ctx, req); + ret = sec_aead_param_check(ctx, req, &need_fallback); if (unlikely(ret)) { - if (req->aead_req.fallback) + if (need_fallback) return sec_aead_soft_crypto(ctx, a_req, encrypt); return -EINVAL; } diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 8ec5333bb5aa..72cf48d1f3ab 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -1097,6 +1097,17 @@ static enum acc_err_result sec_get_err_result(struct hisi_qm *qm) return ACC_ERR_RECOVERED; } +static bool sec_dev_is_abnormal(struct hisi_qm *qm) +{ + u32 err_status; + + err_status = sec_get_hw_err_status(qm); + if (err_status & qm->err_info.dev_shutdown_mask) + return true; + + return false; +} + static void sec_err_info_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; @@ -1129,6 +1140,7 @@ static const struct hisi_qm_err_ini sec_err_ini = { .show_last_dfx_regs = sec_show_last_dfx_regs, .err_info_init = sec_err_info_init, .get_err_result = sec_get_err_result, + .dev_is_abnormal = sec_dev_is_abnormal, }; static int sec_pf_probe_init(struct sec_dev *sec) @@ -1180,7 +1192,6 @@ static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) int ret; qm->pdev = pdev; - qm->ver = pdev->revision; qm->mode = uacce_mode; qm->sqe_size = SEC_SQE_SIZE; qm->dev_name = sec_name; diff --git a/drivers/crypto/hisilicon/zip/Makefile b/drivers/crypto/hisilicon/zip/Makefile index a936f099ee22..13de020b77d6 100644 --- a/drivers/crypto/hisilicon/zip/Makefile +++ b/drivers/crypto/hisilicon/zip/Makefile @@ -1,2 +1,2 @@ obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += hisi_zip.o -hisi_zip-objs = zip_main.o zip_crypto.o +hisi_zip-objs = zip_main.o zip_crypto.o dae_main.o diff --git a/drivers/crypto/hisilicon/zip/dae_main.c b/drivers/crypto/hisilicon/zip/dae_main.c new file mode 100644 index 000000000000..6f22e4c36e49 --- /dev/null +++ b/drivers/crypto/hisilicon/zip/dae_main.c @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2024 HiSilicon Limited. */ + +#include <linux/bitops.h> +#include <linux/io.h> +#include <linux/uacce.h> +#include "zip.h" + +/* memory */ +#define DAE_MEM_START_OFFSET 0x331040 +#define DAE_MEM_DONE_OFFSET 0x331044 +#define DAE_MEM_START_MASK 0x1 +#define DAE_MEM_DONE_MASK 0x1 +#define DAE_REG_RD_INTVRL_US 10 +#define DAE_REG_RD_TMOUT_US USEC_PER_SEC + +#define DAE_ALG_NAME "hashagg" + +/* error */ +#define DAE_AXI_CFG_OFFSET 0x331000 +#define DAE_AXI_SHUTDOWN_MASK (BIT(0) | BIT(5)) +#define DAE_ERR_SOURCE_OFFSET 0x331C84 +#define DAE_ERR_STATUS_OFFSET 0x331C88 +#define DAE_ERR_CE_OFFSET 0x331CA0 +#define DAE_ERR_CE_MASK BIT(3) +#define DAE_ERR_NFE_OFFSET 0x331CA4 +#define DAE_ERR_NFE_MASK 0x17 +#define DAE_ERR_FE_OFFSET 0x331CA8 +#define DAE_ERR_FE_MASK 0 +#define DAE_ECC_MBIT_MASK BIT(2) +#define DAE_ECC_INFO_OFFSET 0x33400C +#define DAE_ERR_SHUTDOWN_OFFSET 0x331CAC +#define DAE_ERR_SHUTDOWN_MASK 0x17 +#define DAE_ERR_ENABLE_OFFSET 0x331C80 +#define DAE_ERR_ENABLE_MASK (DAE_ERR_FE_MASK | DAE_ERR_NFE_MASK | DAE_ERR_CE_MASK) +#define DAE_AM_CTRL_GLOBAL_OFFSET 0x330000 +#define DAE_AM_RETURN_OFFSET 0x330150 +#define DAE_AM_RETURN_MASK 0x3 +#define DAE_AXI_CFG_OFFSET 0x331000 +#define DAE_AXI_SHUTDOWN_EN_MASK (BIT(0) | BIT(5)) + +struct hisi_dae_hw_error { + u32 int_msk; + const char *msg; +}; + +static const struct hisi_dae_hw_error dae_hw_error[] = { + { .int_msk = BIT(0), .msg = "dae_axi_bus_err" }, + { .int_msk = BIT(1), .msg = "dae_axi_poison_err" }, + { .int_msk = BIT(2), .msg = "dae_ecc_2bit_err" }, + { .int_msk = BIT(3), .msg = "dae_ecc_1bit_err" }, + { .int_msk = BIT(4), .msg = "dae_fsm_hbeat_err" }, +}; + +static inline bool dae_is_support(struct hisi_qm *qm) +{ + if (test_bit(QM_SUPPORT_DAE, &qm->caps)) + return true; + + return false; +} + +int hisi_dae_set_user_domain(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!dae_is_support(qm)) + return 0; + + val = readl(qm->io_base + DAE_MEM_START_OFFSET); + val |= DAE_MEM_START_MASK; + writel(val, qm->io_base + DAE_MEM_START_OFFSET); + ret = readl_relaxed_poll_timeout(qm->io_base + DAE_MEM_DONE_OFFSET, val, + val & DAE_MEM_DONE_MASK, + DAE_REG_RD_INTVRL_US, DAE_REG_RD_TMOUT_US); + if (ret) + pci_err(qm->pdev, "failed to init dae memory!\n"); + + return ret; +} + +int hisi_dae_set_alg(struct hisi_qm *qm) +{ + size_t len; + + if (!dae_is_support(qm)) + return 0; + + if (!qm->uacce) + return 0; + + len = strlen(qm->uacce->algs); + /* A line break may be required */ + if (len + strlen(DAE_ALG_NAME) + 1 >= QM_DEV_ALG_MAX_LEN) { + pci_err(qm->pdev, "algorithm name is too long!\n"); + return -EINVAL; + } + + if (len) + strcat((char *)qm->uacce->algs, "\n"); + + strcat((char *)qm->uacce->algs, DAE_ALG_NAME); + + return 0; +} + +static void hisi_dae_master_ooo_ctrl(struct hisi_qm *qm, bool enable) +{ + u32 axi_val, err_val; + + axi_val = readl(qm->io_base + DAE_AXI_CFG_OFFSET); + if (enable) { + axi_val |= DAE_AXI_SHUTDOWN_MASK; + err_val = DAE_ERR_SHUTDOWN_MASK; + } else { + axi_val &= ~DAE_AXI_SHUTDOWN_MASK; + err_val = 0; + } + + writel(axi_val, qm->io_base + DAE_AXI_CFG_OFFSET); + writel(err_val, qm->io_base + DAE_ERR_SHUTDOWN_OFFSET); +} + +void hisi_dae_hw_error_enable(struct hisi_qm *qm) +{ + if (!dae_is_support(qm)) + return; + + /* clear dae hw error source if having */ + writel(DAE_ERR_ENABLE_MASK, qm->io_base + DAE_ERR_SOURCE_OFFSET); + + /* configure error type */ + writel(DAE_ERR_CE_MASK, qm->io_base + DAE_ERR_CE_OFFSET); + writel(DAE_ERR_NFE_MASK, qm->io_base + DAE_ERR_NFE_OFFSET); + writel(DAE_ERR_FE_MASK, qm->io_base + DAE_ERR_FE_OFFSET); + + hisi_dae_master_ooo_ctrl(qm, true); + + /* enable dae hw error interrupts */ + writel(DAE_ERR_ENABLE_MASK, qm->io_base + DAE_ERR_ENABLE_OFFSET); +} + +void hisi_dae_hw_error_disable(struct hisi_qm *qm) +{ + if (!dae_is_support(qm)) + return; + + writel(0, qm->io_base + DAE_ERR_ENABLE_OFFSET); + hisi_dae_master_ooo_ctrl(qm, false); +} + +static u32 hisi_dae_get_hw_err_status(struct hisi_qm *qm) +{ + return readl(qm->io_base + DAE_ERR_STATUS_OFFSET); +} + +static void hisi_dae_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) +{ + if (!dae_is_support(qm)) + return; + + writel(err_sts, qm->io_base + DAE_ERR_SOURCE_OFFSET); +} + +static void hisi_dae_disable_error_report(struct hisi_qm *qm, u32 err_type) +{ + writel(DAE_ERR_NFE_MASK & (~err_type), qm->io_base + DAE_ERR_NFE_OFFSET); +} + +static void hisi_dae_log_hw_error(struct hisi_qm *qm, u32 err_type) +{ + const struct hisi_dae_hw_error *err = dae_hw_error; + struct device *dev = &qm->pdev->dev; + u32 ecc_info; + size_t i; + + for (i = 0; i < ARRAY_SIZE(dae_hw_error); i++) { + err = &dae_hw_error[i]; + if (!(err->int_msk & err_type)) + continue; + + dev_err(dev, "%s [error status=0x%x] found\n", + err->msg, err->int_msk); + + if (err->int_msk & DAE_ECC_MBIT_MASK) { + ecc_info = readl(qm->io_base + DAE_ECC_INFO_OFFSET); + dev_err(dev, "dae multi ecc sram info 0x%x\n", ecc_info); + } + } +} + +enum acc_err_result hisi_dae_get_err_result(struct hisi_qm *qm) +{ + u32 err_status; + + if (!dae_is_support(qm)) + return ACC_ERR_NONE; + + err_status = hisi_dae_get_hw_err_status(qm); + if (!err_status) + return ACC_ERR_NONE; + + hisi_dae_log_hw_error(qm, err_status); + + if (err_status & DAE_ERR_NFE_MASK) { + /* Disable the same error reporting until device is recovered. */ + hisi_dae_disable_error_report(qm, err_status); + return ACC_ERR_NEED_RESET; + } + hisi_dae_clear_hw_err_status(qm, err_status); + + return ACC_ERR_RECOVERED; +} + +bool hisi_dae_dev_is_abnormal(struct hisi_qm *qm) +{ + u32 err_status; + + if (!dae_is_support(qm)) + return false; + + err_status = hisi_dae_get_hw_err_status(qm); + if (err_status & DAE_ERR_NFE_MASK) + return true; + + return false; +} + +int hisi_dae_close_axi_master_ooo(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!dae_is_support(qm)) + return 0; + + val = readl(qm->io_base + DAE_AM_CTRL_GLOBAL_OFFSET); + val |= BIT(0); + writel(val, qm->io_base + DAE_AM_CTRL_GLOBAL_OFFSET); + + ret = readl_relaxed_poll_timeout(qm->io_base + DAE_AM_RETURN_OFFSET, + val, (val == DAE_AM_RETURN_MASK), + DAE_REG_RD_INTVRL_US, DAE_REG_RD_TMOUT_US); + if (ret) + dev_err(&qm->pdev->dev, "failed to close dae axi ooo!\n"); + + return ret; +} + +void hisi_dae_open_axi_master_ooo(struct hisi_qm *qm) +{ + u32 val; + + if (!dae_is_support(qm)) + return; + + val = readl(qm->io_base + DAE_AXI_CFG_OFFSET); + + writel(val & ~DAE_AXI_SHUTDOWN_EN_MASK, qm->io_base + DAE_AXI_CFG_OFFSET); + writel(val | DAE_AXI_SHUTDOWN_EN_MASK, qm->io_base + DAE_AXI_CFG_OFFSET); +} diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h index 2fecf346c3c9..9fb2a9c01132 100644 --- a/drivers/crypto/hisilicon/zip/zip.h +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -103,4 +103,12 @@ int zip_create_qps(struct hisi_qp **qps, int qp_num, int node); int hisi_zip_register_to_crypto(struct hisi_qm *qm); void hisi_zip_unregister_from_crypto(struct hisi_qm *qm); bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg); +int hisi_dae_set_user_domain(struct hisi_qm *qm); +int hisi_dae_set_alg(struct hisi_qm *qm); +void hisi_dae_hw_error_disable(struct hisi_qm *qm); +void hisi_dae_hw_error_enable(struct hisi_qm *qm); +void hisi_dae_open_axi_master_ooo(struct hisi_qm *qm); +int hisi_dae_close_axi_master_ooo(struct hisi_qm *qm); +bool hisi_dae_dev_is_abnormal(struct hisi_qm *qm); +enum acc_err_result hisi_dae_get_err_result(struct hisi_qm *qm); #endif diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 9239b251c2d7..d8ba23b7cc7d 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -582,7 +582,7 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) hisi_zip_enable_clock_gate(qm); - return 0; + return hisi_dae_set_user_domain(qm); } static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable) @@ -631,6 +631,8 @@ static void hisi_zip_hw_error_enable(struct hisi_qm *qm) /* enable ZIP hw error interrupts */ writel(0, qm->io_base + HZIP_CORE_INT_MASK_REG); + + hisi_dae_hw_error_enable(qm); } static void hisi_zip_hw_error_disable(struct hisi_qm *qm) @@ -643,6 +645,8 @@ static void hisi_zip_hw_error_disable(struct hisi_qm *qm) writel(ce | nfe | HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_MASK_REG); hisi_zip_master_ooo_ctrl(qm, false); + + hisi_dae_hw_error_disable(qm); } static inline struct hisi_qm *file_to_qm(struct ctrl_debug_file *file) @@ -1129,6 +1133,8 @@ static void hisi_zip_open_axi_master_ooo(struct hisi_qm *qm) writel(val | HZIP_AXI_SHUTDOWN_ENABLE, qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); + + hisi_dae_open_axi_master_ooo(qm); } static void hisi_zip_close_axi_master_ooo(struct hisi_qm *qm) @@ -1147,8 +1153,11 @@ static void hisi_zip_close_axi_master_ooo(struct hisi_qm *qm) static enum acc_err_result hisi_zip_get_err_result(struct hisi_qm *qm) { + enum acc_err_result zip_result = ACC_ERR_NONE; + enum acc_err_result dae_result; u32 err_status; + /* Get device hardware new error status */ err_status = hisi_zip_get_hw_err_status(qm); if (err_status) { if (err_status & qm->err_info.ecc_2bits_mask) @@ -1159,11 +1168,32 @@ static enum acc_err_result hisi_zip_get_err_result(struct hisi_qm *qm) /* Disable the same error reporting until device is recovered. */ hisi_zip_disable_error_report(qm, err_status); return ACC_ERR_NEED_RESET; + } else { + hisi_zip_clear_hw_err_status(qm, err_status); } - hisi_zip_clear_hw_err_status(qm, err_status); } - return ACC_ERR_RECOVERED; + dae_result = hisi_dae_get_err_result(qm); + + return (zip_result == ACC_ERR_NEED_RESET || + dae_result == ACC_ERR_NEED_RESET) ? + ACC_ERR_NEED_RESET : ACC_ERR_RECOVERED; +} + +static bool hisi_zip_dev_is_abnormal(struct hisi_qm *qm) +{ + u32 err_status; + + err_status = hisi_zip_get_hw_err_status(qm); + if (err_status & qm->err_info.dev_shutdown_mask) + return true; + + return hisi_dae_dev_is_abnormal(qm); +} + +static int hisi_zip_set_priv_status(struct hisi_qm *qm) +{ + return hisi_dae_close_axi_master_ooo(qm); } static void hisi_zip_err_info_init(struct hisi_qm *qm) @@ -1200,6 +1230,8 @@ static const struct hisi_qm_err_ini hisi_zip_err_ini = { .show_last_dfx_regs = hisi_zip_show_last_dfx_regs, .err_info_init = hisi_zip_err_info_init, .get_err_result = hisi_zip_get_err_result, + .set_priv_status = hisi_zip_set_priv_status, + .dev_is_abnormal = hisi_zip_dev_is_abnormal, }; static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) @@ -1264,7 +1296,6 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) int ret; qm->pdev = pdev; - qm->ver = pdev->revision; qm->mode = uacce_mode; qm->sqe_size = HZIP_SQE_SIZE; qm->dev_name = hisi_zip_name; @@ -1301,17 +1332,24 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) ret = zip_pre_store_cap_reg(qm); if (ret) { pci_err(qm->pdev, "Failed to pre-store capability registers!\n"); - hisi_qm_uninit(qm); - return ret; + goto err_qm_uninit; } alg_msk = qm->cap_tables.dev_cap_table[ZIP_ALG_BITMAP].cap_val; ret = hisi_qm_set_algs(qm, alg_msk, zip_dev_algs, ARRAY_SIZE(zip_dev_algs)); if (ret) { pci_err(qm->pdev, "Failed to set zip algs!\n"); - hisi_qm_uninit(qm); + goto err_qm_uninit; } + ret = hisi_dae_set_alg(qm); + if (ret) + goto err_qm_uninit; + + return 0; + +err_qm_uninit: + hisi_qm_uninit(qm); return ret; } |