diff options
Diffstat (limited to 'drivers/crypto/hisilicon')
| -rw-r--r-- | drivers/crypto/hisilicon/Kconfig | 4 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/hpre/hpre.h | 3 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/hpre/hpre_crypto.c | 20 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/hpre/hpre_main.c | 160 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/qm.c | 619 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/qm.h | 72 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/sec2/sec.h | 12 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/sec2/sec_crypto.c | 260 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/sec2/sec_main.c | 294 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/zip/zip.h | 2 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/zip/zip_crypto.c | 54 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/zip/zip_main.c | 324 |
12 files changed, 1149 insertions, 675 deletions
diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig index 8851161f722f..f09c6cf7823e 100644 --- a/drivers/crypto/hisilicon/Kconfig +++ b/drivers/crypto/hisilicon/Kconfig @@ -27,6 +27,7 @@ config CRYPTO_DEV_HISI_SEC2 select CRYPTO_SHA256 select CRYPTO_SHA512 depends on PCI && PCI_MSI + depends on UACCE || UACCE=n depends on ARM64 || (COMPILE_TEST && 64BIT) help Support for HiSilicon SEC Engine of version 2 in crypto subsystem. @@ -40,6 +41,7 @@ config CRYPTO_DEV_HISI_QM tristate depends on ARM64 || COMPILE_TEST depends on PCI && PCI_MSI + depends on UACCE || UACCE=n help HiSilicon accelerator engines use a common queue management interface. Specific engine driver may use this module. @@ -49,6 +51,7 @@ config CRYPTO_DEV_HISI_ZIP depends on PCI && PCI_MSI depends on ARM64 || (COMPILE_TEST && 64BIT) depends on !CPU_BIG_ENDIAN || COMPILE_TEST + depends on UACCE || UACCE=n select CRYPTO_DEV_HISI_QM help Support for HiSilicon ZIP Driver @@ -56,6 +59,7 @@ config CRYPTO_DEV_HISI_ZIP config CRYPTO_DEV_HISI_HPRE tristate "Support for HISI HPRE accelerator" depends on PCI && PCI_MSI + depends on UACCE || UACCE=n depends on ARM64 || (COMPILE_TEST && 64BIT) select CRYPTO_DEV_HISI_QM select CRYPTO_DH diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h index ddf13ea9862a..03d512ec6336 100644 --- a/drivers/crypto/hisilicon/hpre/hpre.h +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -46,7 +46,6 @@ struct hpre_debug { struct hpre { struct hisi_qm qm; - struct list_head list; struct hpre_debug debug; u32 num_vfs; unsigned long status; @@ -76,7 +75,7 @@ struct hpre_sqe { __le32 rsvd1[_HPRE_SQE_ALIGN_EXT]; }; -struct hpre *hpre_find_device(int node); +struct hisi_qp *hpre_create_qp(void); int hpre_algs_register(void); void hpre_algs_unregister(void); diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 5d400d69e8e4..65425250b2e9 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -147,26 +147,18 @@ static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req) static struct hisi_qp *hpre_get_qp_and_start(void) { struct hisi_qp *qp; - struct hpre *hpre; int ret; - /* find the proper hpre device, which is near the current CPU core */ - hpre = hpre_find_device(cpu_to_node(smp_processor_id())); - if (!hpre) { - pr_err("Can not find proper hpre device!\n"); - return ERR_PTR(-ENODEV); - } - - qp = hisi_qm_create_qp(&hpre->qm, 0); - if (IS_ERR(qp)) { - pci_err(hpre->qm.pdev, "Can not create qp!\n"); + qp = hpre_create_qp(); + if (!qp) { + pr_err("Can not create hpre qp!\n"); return ERR_PTR(-ENODEV); } ret = hisi_qm_start_qp(qp, 0); if (ret < 0) { - hisi_qm_release_qp(qp); - pci_err(hpre->qm.pdev, "Can not start qp!\n"); + hisi_qm_free_qps(&qp, 1); + pci_err(qp->qm->pdev, "Can not start qp!\n"); return ERR_PTR(-EINVAL); } @@ -338,7 +330,7 @@ static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all) if (is_clear_all) { idr_destroy(&ctx->req_idr); kfree(ctx->req_list); - hisi_qm_release_qp(ctx->qp); + hisi_qm_free_qps(&ctx->qp, 1); } ctx->crt_g2_mode = false; diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 401747de67a8..88be53bf4a38 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -82,8 +82,7 @@ #define HPRE_VIA_MSI_DSM 1 -static LIST_HEAD(hpre_list); -static DEFINE_MUTEX(hpre_list_lock); +static struct hisi_qm_list hpre_devices; static const char hpre_name[] = "hisi_hpre"; static struct dentry *hpre_debugfs_root; static const struct pci_device_id hpre_dev_ids[] = { @@ -196,43 +195,17 @@ static u32 hpre_pf_q_num = HPRE_PF_DEF_Q_NUM; module_param_cb(hpre_pf_q_num, &hpre_pf_q_num_ops, &hpre_pf_q_num, 0444); MODULE_PARM_DESC(hpre_pf_q_num, "Number of queues in PF of CS(1-1024)"); -static inline void hpre_add_to_list(struct hpre *hpre) +struct hisi_qp *hpre_create_qp(void) { - mutex_lock(&hpre_list_lock); - list_add_tail(&hpre->list, &hpre_list); - mutex_unlock(&hpre_list_lock); -} - -static inline void hpre_remove_from_list(struct hpre *hpre) -{ - mutex_lock(&hpre_list_lock); - list_del(&hpre->list); - mutex_unlock(&hpre_list_lock); -} + int node = cpu_to_node(smp_processor_id()); + struct hisi_qp *qp = NULL; + int ret; -struct hpre *hpre_find_device(int node) -{ - struct hpre *hpre, *ret = NULL; - int min_distance = INT_MAX; - struct device *dev; - int dev_node = 0; - - mutex_lock(&hpre_list_lock); - list_for_each_entry(hpre, &hpre_list, list) { - dev = &hpre->qm.pdev->dev; -#ifdef CONFIG_NUMA - dev_node = dev->numa_node; - if (dev_node < 0) - dev_node = 0; -#endif - if (node_distance(dev_node, node) < min_distance) { - ret = hpre; - min_distance = node_distance(dev_node, node); - } - } - mutex_unlock(&hpre_list_lock); + ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, 0, node, &qp); + if (!ret) + return qp; - return ret; + return NULL; } static int hpre_cfg_by_dsm(struct hisi_qm *qm) @@ -349,18 +322,14 @@ static void hpre_cnt_regs_clear(struct hisi_qm *qm) hisi_qm_debug_regs_clear(qm); } -static void hpre_hw_error_disable(struct hpre *hpre) +static void hpre_hw_error_disable(struct hisi_qm *qm) { - struct hisi_qm *qm = &hpre->qm; - /* disable hpre hw error interrupts */ writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_INT_MASK); } -static void hpre_hw_error_enable(struct hpre *hpre) +static void hpre_hw_error_enable(struct hisi_qm *qm) { - struct hisi_qm *qm = &hpre->qm; - /* enable hpre hw error interrupts */ writel(HPRE_CORE_INT_ENABLE, qm->io_base + HPRE_INT_MASK); writel(HPRE_HAC_RAS_CE_ENABLE, qm->io_base + HPRE_RAS_CE_ENB); @@ -713,13 +682,39 @@ static int hpre_qm_pre_init(struct hisi_qm *qm, struct pci_dev *pdev) return 0; } -static void hpre_hw_err_init(struct hpre *hpre) +static void hpre_log_hw_error(struct hisi_qm *qm, u32 err_sts) { - hisi_qm_hw_error_init(&hpre->qm, QM_BASE_CE, QM_BASE_NFE, - 0, QM_DB_RANDOM_INVALID); - hpre_hw_error_enable(hpre); + const struct hpre_hw_error *err = hpre_hw_errors; + struct device *dev = &qm->pdev->dev; + + while (err->msg) { + if (err->int_msk & err_sts) + dev_warn(dev, "%s [error status=0x%x] found\n", + err->msg, err->int_msk); + err++; + } + + writel(err_sts, qm->io_base + HPRE_HAC_SOURCE_INT); +} + +static u32 hpre_get_hw_err_status(struct hisi_qm *qm) +{ + return readl(qm->io_base + HPRE_HAC_INT_STATUS); } +static const struct hisi_qm_err_ini hpre_err_ini = { + .hw_err_enable = hpre_hw_error_enable, + .hw_err_disable = hpre_hw_error_disable, + .get_dev_hw_err_status = hpre_get_hw_err_status, + .log_dev_hw_err = hpre_log_hw_error, + .err_info = { + .ce = QM_BASE_CE, + .nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT, + .fe = 0, + .msi = QM_DB_RANDOM_INVALID, + } +}; + static int hpre_pf_probe_init(struct hpre *hpre) { struct hisi_qm *qm = &hpre->qm; @@ -731,7 +726,8 @@ static int hpre_pf_probe_init(struct hpre *hpre) if (ret) return ret; - hpre_hw_err_init(hpre); + qm->err_ini = &hpre_err_ini; + hisi_qm_dev_err_init(qm); return 0; } @@ -776,22 +772,21 @@ static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) dev_warn(&pdev->dev, "init debugfs fail!\n"); - hpre_add_to_list(hpre); + hisi_qm_add_to_list(qm, &hpre_devices); ret = hpre_algs_register(); if (ret < 0) { - hpre_remove_from_list(hpre); pci_err(pdev, "fail to register algs to crypto!\n"); goto err_with_qm_start; } return 0; err_with_qm_start: + hisi_qm_del_from_list(qm, &hpre_devices); hisi_qm_stop(qm); err_with_err_init: - if (pdev->is_physfn) - hpre_hw_error_disable(hpre); + hisi_qm_dev_err_uninit(qm); err_with_qm_init: hisi_qm_uninit(qm); @@ -907,7 +902,7 @@ static void hpre_remove(struct pci_dev *pdev) int ret; hpre_algs_unregister(); - hpre_remove_from_list(hpre); + hisi_qm_del_from_list(qm, &hpre_devices); if (qm->fun_type == QM_HW_PF && hpre->num_vfs != 0) { ret = hpre_sriov_disable(pdev); if (ret) { @@ -922,69 +917,13 @@ static void hpre_remove(struct pci_dev *pdev) hpre_debugfs_exit(hpre); hisi_qm_stop(qm); - if (qm->fun_type == QM_HW_PF) - hpre_hw_error_disable(hpre); + hisi_qm_dev_err_uninit(qm); hisi_qm_uninit(qm); } -static void hpre_log_hw_error(struct hpre *hpre, u32 err_sts) -{ - const struct hpre_hw_error *err = hpre_hw_errors; - struct device *dev = &hpre->qm.pdev->dev; - - while (err->msg) { - if (err->int_msk & err_sts) - dev_warn(dev, "%s [error status=0x%x] found\n", - err->msg, err->int_msk); - err++; - } -} - -static pci_ers_result_t hpre_hw_error_handle(struct hpre *hpre) -{ - u32 err_sts; - - /* read err sts */ - err_sts = readl(hpre->qm.io_base + HPRE_HAC_INT_STATUS); - if (err_sts) { - hpre_log_hw_error(hpre, err_sts); - - /* clear error interrupts */ - writel(err_sts, hpre->qm.io_base + HPRE_HAC_SOURCE_INT); - return PCI_ERS_RESULT_NEED_RESET; - } - - return PCI_ERS_RESULT_RECOVERED; -} - -static pci_ers_result_t hpre_process_hw_error(struct pci_dev *pdev) -{ - struct hpre *hpre = pci_get_drvdata(pdev); - pci_ers_result_t qm_ret, hpre_ret; - - /* log qm error */ - qm_ret = hisi_qm_hw_error_handle(&hpre->qm); - - /* log hpre error */ - hpre_ret = hpre_hw_error_handle(hpre); - - return (qm_ret == PCI_ERS_RESULT_NEED_RESET || - hpre_ret == PCI_ERS_RESULT_NEED_RESET) ? - PCI_ERS_RESULT_NEED_RESET : PCI_ERS_RESULT_RECOVERED; -} - -static pci_ers_result_t hpre_error_detected(struct pci_dev *pdev, - pci_channel_state_t state) -{ - pci_info(pdev, "PCI error detected, state(=%d)!!\n", state); - if (state == pci_channel_io_perm_failure) - return PCI_ERS_RESULT_DISCONNECT; - - return hpre_process_hw_error(pdev); -} static const struct pci_error_handlers hpre_err_handler = { - .error_detected = hpre_error_detected, + .error_detected = hisi_qm_dev_err_detected, }; static struct pci_driver hpre_pci_driver = { @@ -1013,6 +952,7 @@ static int __init hpre_init(void) { int ret; + hisi_qm_init_list(&hpre_devices); hpre_register_debugfs(); ret = pci_register_driver(&hpre_pci_driver); diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index b57da5ef8b5b..f795fb557630 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -9,6 +9,9 @@ #include <linux/log2.h> #include <linux/seq_file.h> #include <linux/slab.h> +#include <linux/uacce.h> +#include <linux/uaccess.h> +#include <uapi/misc/uacce/hisi_qm.h> #include "qm.h" /* eq/aeq irq enable */ @@ -269,6 +272,12 @@ struct qm_doorbell { __le16 priority; }; +struct hisi_qm_resource { + struct hisi_qm *qm; + int distance; + struct list_head list; +}; + struct hisi_qm_hw_ops { int (*get_vft)(struct hisi_qm *qm, u32 *base, u32 *number); void (*qm_db)(struct hisi_qm *qm, u16 qn, @@ -277,6 +286,7 @@ struct hisi_qm_hw_ops { int (*debug_init)(struct hisi_qm *qm); void (*hw_error_init)(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe, u32 msi); + void (*hw_error_uninit)(struct hisi_qm *qm); pci_ers_result_t (*hw_error_handle)(struct hisi_qm *qm); }; @@ -465,9 +475,14 @@ static void qm_cq_head_update(struct hisi_qp *qp) static void qm_poll_qp(struct hisi_qp *qp, struct hisi_qm *qm) { - struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head; + if (qp->event_cb) { + qp->event_cb(qp); + return; + } if (qp->req_cb) { + struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head; + while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) { dma_rmb(); qp->req_cb(qp, qp->sqe + qm->sqe_size * @@ -485,17 +500,9 @@ static void qm_poll_qp(struct hisi_qp *qp, struct hisi_qm *qm) } } -static void qm_qp_work_func(struct work_struct *work) +static void qm_work_process(struct work_struct *work) { - struct hisi_qp *qp; - - qp = container_of(work, struct hisi_qp, work); - qm_poll_qp(qp, qp->qm); -} - -static irqreturn_t qm_irq_handler(int irq, void *data) -{ - struct hisi_qm *qm = data; + struct hisi_qm *qm = container_of(work, struct hisi_qm, work); struct qm_eqe *eqe = qm->eqe + qm->status.eq_head; struct hisi_qp *qp; int eqe_num = 0; @@ -504,7 +511,7 @@ static irqreturn_t qm_irq_handler(int irq, void *data) eqe_num++; qp = qm_to_hisi_qp(qm, eqe); if (qp) - queue_work(qp->wq, &qp->work); + qm_poll_qp(qp, qm); if (qm->status.eq_head == QM_Q_DEPTH - 1) { qm->status.eqc_phase = !qm->status.eqc_phase; @@ -522,6 +529,17 @@ static irqreturn_t qm_irq_handler(int irq, void *data) } qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); +} + +static irqreturn_t do_qm_irq(int irq, void *data) +{ + struct hisi_qm *qm = (struct hisi_qm *)data; + + /* the workqueue created by device driver of QM */ + if (qm->wq) + queue_work(qm->wq, &qm->work); + else + schedule_work(&qm->work); return IRQ_HANDLED; } @@ -531,7 +549,7 @@ static irqreturn_t qm_irq(int irq, void *data) struct hisi_qm *qm = data; if (readl(qm->io_base + QM_VF_EQ_INT_SOURCE)) - return qm_irq_handler(irq, data); + return do_qm_irq(irq, data); dev_err(&qm->pdev->dev, "invalid int source\n"); qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); @@ -1011,43 +1029,45 @@ static void qm_hw_error_init_v2(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe, writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK); } +static void qm_hw_error_uninit_v2(struct hisi_qm *qm) +{ + writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK); +} + static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status) { - const struct hisi_qm_hw_error *err = qm_hw_error; + const struct hisi_qm_hw_error *err; struct device *dev = &qm->pdev->dev; u32 reg_val, type, vf_num; + int i; - while (err->msg) { - if (err->int_msk & error_status) { - dev_err(dev, "%s [error status=0x%x] found\n", - err->msg, err->int_msk); - - if (error_status & QM_DB_TIMEOUT) { - reg_val = readl(qm->io_base + - QM_ABNORMAL_INF01); - type = (reg_val & QM_DB_TIMEOUT_TYPE) >> - QM_DB_TIMEOUT_TYPE_SHIFT; - vf_num = reg_val & QM_DB_TIMEOUT_VF; - dev_err(dev, "qm %s doorbell timeout in function %u\n", - qm_db_timeout[type], vf_num); - } - - if (error_status & QM_OF_FIFO_OF) { - reg_val = readl(qm->io_base + - QM_ABNORMAL_INF00); - type = (reg_val & QM_FIFO_OVERFLOW_TYPE) >> - QM_FIFO_OVERFLOW_TYPE_SHIFT; - vf_num = reg_val & QM_FIFO_OVERFLOW_VF; - - if (type < ARRAY_SIZE(qm_fifo_overflow)) - dev_err(dev, "qm %s fifo overflow in function %u\n", - qm_fifo_overflow[type], - vf_num); - else - dev_err(dev, "unknown error type\n"); - } + for (i = 0; i < ARRAY_SIZE(qm_hw_error); i++) { + err = &qm_hw_error[i]; + if (!(err->int_msk & error_status)) + continue; + + dev_err(dev, "%s [error status=0x%x] found\n", + err->msg, err->int_msk); + + if (err->int_msk & QM_DB_TIMEOUT) { + reg_val = readl(qm->io_base + QM_ABNORMAL_INF01); + type = (reg_val & QM_DB_TIMEOUT_TYPE) >> + QM_DB_TIMEOUT_TYPE_SHIFT; + vf_num = reg_val & QM_DB_TIMEOUT_VF; + dev_err(dev, "qm %s doorbell timeout in function %u\n", + qm_db_timeout[type], vf_num); + } else if (err->int_msk & QM_OF_FIFO_OF) { + reg_val = readl(qm->io_base + QM_ABNORMAL_INF00); + type = (reg_val & QM_FIFO_OVERFLOW_TYPE) >> + QM_FIFO_OVERFLOW_TYPE_SHIFT; + vf_num = reg_val & QM_FIFO_OVERFLOW_VF; + + if (type < ARRAY_SIZE(qm_fifo_overflow)) + dev_err(dev, "qm %s fifo overflow in function %u\n", + qm_fifo_overflow[type], vf_num); + else + dev_err(dev, "unknown error type\n"); } - err++; } } @@ -1082,6 +1102,7 @@ static const struct hisi_qm_hw_ops qm_hw_ops_v2 = { .qm_db = qm_db_v2, .get_irq_num = qm_get_irq_num_v2, .hw_error_init = qm_hw_error_init_v2, + .hw_error_uninit = qm_hw_error_uninit_v2, .hw_error_handle = qm_hw_error_handle_v2, }; @@ -1147,20 +1168,9 @@ struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type) qp->qp_id = qp_id; qp->alg_type = alg_type; - INIT_WORK(&qp->work, qm_qp_work_func); - qp->wq = alloc_workqueue("hisi_qm", WQ_UNBOUND | WQ_HIGHPRI | - WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0); - if (!qp->wq) { - ret = -EFAULT; - goto err_free_qp_mem; - } return qp; -err_free_qp_mem: - if (qm->use_dma_api) - dma_free_coherent(dev, qp->qdma.size, qp->qdma.va, - qp->qdma.dma); err_clear_bit: write_lock(&qm->qps_lock); qm->qp_array[qp_id] = NULL; @@ -1269,7 +1279,7 @@ static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) * @qp: The qp we want to start to run. * @arg: Accelerator specific argument. * - * After this function, qp can receive request from user. Return qp_id if + * After this function, qp can receive request from user. Return 0 if * successful, Return -EBUSY if failed. */ int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg) @@ -1314,7 +1324,7 @@ int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg) dev_dbg(dev, "queue %d started\n", qp_id); - return qp_id; + return 0; } EXPORT_SYMBOL_GPL(hisi_qm_start_qp); @@ -1395,6 +1405,214 @@ static void hisi_qm_cache_wb(struct hisi_qm *qm) } } +static void qm_qp_event_notifier(struct hisi_qp *qp) +{ + wake_up_interruptible(&qp->uacce_q->wait); +} + +static int hisi_qm_get_available_instances(struct uacce_device *uacce) +{ + int i, ret; + struct hisi_qm *qm = uacce->priv; + + read_lock(&qm->qps_lock); + for (i = 0, ret = 0; i < qm->qp_num; i++) + if (!qm->qp_array[i]) + ret++; + read_unlock(&qm->qps_lock); + + return ret; +} + +static int hisi_qm_uacce_get_queue(struct uacce_device *uacce, + unsigned long arg, + struct uacce_queue *q) +{ + struct hisi_qm *qm = uacce->priv; + struct hisi_qp *qp; + u8 alg_type = 0; + + qp = hisi_qm_create_qp(qm, alg_type); + if (IS_ERR(qp)) + return PTR_ERR(qp); + + q->priv = qp; + q->uacce = uacce; + qp->uacce_q = q; + qp->event_cb = qm_qp_event_notifier; + qp->pasid = arg; + + return 0; +} + +static void hisi_qm_uacce_put_queue(struct uacce_queue *q) +{ + struct hisi_qp *qp = q->priv; + + hisi_qm_cache_wb(qp->qm); + hisi_qm_release_qp(qp); +} + +/* map sq/cq/doorbell to user space */ +static int hisi_qm_uacce_mmap(struct uacce_queue *q, + struct vm_area_struct *vma, + struct uacce_qfile_region *qfr) +{ + struct hisi_qp *qp = q->priv; + struct hisi_qm *qm = qp->qm; + size_t sz = vma->vm_end - vma->vm_start; + struct pci_dev *pdev = qm->pdev; + struct device *dev = &pdev->dev; + unsigned long vm_pgoff; + int ret; + + switch (qfr->type) { + case UACCE_QFRT_MMIO: + if (qm->ver == QM_HW_V2) { + if (sz > PAGE_SIZE * (QM_DOORBELL_PAGE_NR + + QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE)) + return -EINVAL; + } else { + if (sz > PAGE_SIZE * QM_DOORBELL_PAGE_NR) + return -EINVAL; + } + + vma->vm_flags |= VM_IO; + + return remap_pfn_range(vma, vma->vm_start, + qm->phys_base >> PAGE_SHIFT, + sz, pgprot_noncached(vma->vm_page_prot)); + case UACCE_QFRT_DUS: + if (sz != qp->qdma.size) + return -EINVAL; + + /* + * dma_mmap_coherent() requires vm_pgoff as 0 + * restore vm_pfoff to initial value for mmap() + */ + vm_pgoff = vma->vm_pgoff; + vma->vm_pgoff = 0; + ret = dma_mmap_coherent(dev, vma, qp->qdma.va, + qp->qdma.dma, sz); + vma->vm_pgoff = vm_pgoff; + return ret; + + default: + return -EINVAL; + } +} + +static int hisi_qm_uacce_start_queue(struct uacce_queue *q) +{ + struct hisi_qp *qp = q->priv; + + return hisi_qm_start_qp(qp, qp->pasid); +} + +static void hisi_qm_uacce_stop_queue(struct uacce_queue *q) +{ + hisi_qm_stop_qp(q->priv); +} + +static int qm_set_sqctype(struct uacce_queue *q, u16 type) +{ + struct hisi_qm *qm = q->uacce->priv; + struct hisi_qp *qp = q->priv; + + write_lock(&qm->qps_lock); + qp->alg_type = type; + write_unlock(&qm->qps_lock); + + return 0; +} + +static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd, + unsigned long arg) +{ + struct hisi_qp *qp = q->priv; + struct hisi_qp_ctx qp_ctx; + + if (cmd == UACCE_CMD_QM_SET_QP_CTX) { + if (copy_from_user(&qp_ctx, (void __user *)arg, + sizeof(struct hisi_qp_ctx))) + return -EFAULT; + + if (qp_ctx.qc_type != 0 && qp_ctx.qc_type != 1) + return -EINVAL; + + qm_set_sqctype(q, qp_ctx.qc_type); + qp_ctx.id = qp->qp_id; + + if (copy_to_user((void __user *)arg, &qp_ctx, + sizeof(struct hisi_qp_ctx))) + return -EFAULT; + } else { + return -EINVAL; + } + + return 0; +} + +static const struct uacce_ops uacce_qm_ops = { + .get_available_instances = hisi_qm_get_available_instances, + .get_queue = hisi_qm_uacce_get_queue, + .put_queue = hisi_qm_uacce_put_queue, + .start_queue = hisi_qm_uacce_start_queue, + .stop_queue = hisi_qm_uacce_stop_queue, + .mmap = hisi_qm_uacce_mmap, + .ioctl = hisi_qm_uacce_ioctl, +}; + +static int qm_alloc_uacce(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct uacce_device *uacce; + unsigned long mmio_page_nr; + unsigned long dus_page_nr; + struct uacce_interface interface = { + .flags = UACCE_DEV_SVA, + .ops = &uacce_qm_ops, + }; + + strncpy(interface.name, pdev->driver->name, sizeof(interface.name)); + + uacce = uacce_alloc(&pdev->dev, &interface); + if (IS_ERR(uacce)) + return PTR_ERR(uacce); + + if (uacce->flags & UACCE_DEV_SVA) { + qm->use_sva = true; + } else { + /* only consider sva case */ + uacce_remove(uacce); + qm->uacce = NULL; + return -EINVAL; + } + + uacce->is_vf = pdev->is_virtfn; + uacce->priv = qm; + uacce->algs = qm->algs; + + if (qm->ver == QM_HW_V1) { + mmio_page_nr = QM_DOORBELL_PAGE_NR; + uacce->api_ver = HISI_QM_API_VER_BASE; + } else { + mmio_page_nr = QM_DOORBELL_PAGE_NR + + QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE; + uacce->api_ver = HISI_QM_API_VER2_BASE; + } + + dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * QM_Q_DEPTH + + sizeof(struct qm_cqe) * QM_Q_DEPTH) >> PAGE_SHIFT; + + uacce->qf_pg_num[UACCE_QFRT_MMIO] = mmio_page_nr; + uacce->qf_pg_num[UACCE_QFRT_DUS] = dus_page_nr; + + qm->uacce = uacce; + + return 0; +} + /** * hisi_qm_get_free_qp_num() - Get free number of qp in qm. * @qm: The qm which want to get free qp. @@ -1437,10 +1655,14 @@ int hisi_qm_init(struct hisi_qm *qm) return -EINVAL; } + ret = qm_alloc_uacce(qm); + if (ret < 0) + dev_warn(&pdev->dev, "fail to alloc uacce (%d)\n", ret); + ret = pci_enable_device_mem(pdev); if (ret < 0) { dev_err(&pdev->dev, "Failed to enable device mem!\n"); - return ret; + goto err_remove_uacce; } ret = pci_request_mem_regions(pdev, qm->dev_name); @@ -1449,8 +1671,9 @@ int hisi_qm_init(struct hisi_qm *qm) goto err_disable_pcidev; } - qm->io_base = ioremap(pci_resource_start(pdev, PCI_BAR_2), - pci_resource_len(qm->pdev, PCI_BAR_2)); + qm->phys_base = pci_resource_start(pdev, PCI_BAR_2); + qm->phys_size = pci_resource_len(qm->pdev, PCI_BAR_2); + qm->io_base = ioremap(qm->phys_base, qm->phys_size); if (!qm->io_base) { ret = -EIO; goto err_release_mem_regions; @@ -1479,6 +1702,7 @@ int hisi_qm_init(struct hisi_qm *qm) qm->qp_in_used = 0; mutex_init(&qm->mailbox_lock); rwlock_init(&qm->qps_lock); + INIT_WORK(&qm->work, qm_work_process); dev_dbg(dev, "init qm %s with %s\n", pdev->is_physfn ? "pf" : "vf", qm->use_dma_api ? "dma api" : "iommu api"); @@ -1493,6 +1717,9 @@ err_release_mem_regions: pci_release_mem_regions(pdev); err_disable_pcidev: pci_disable_device(pdev); +err_remove_uacce: + uacce_remove(qm->uacce); + qm->uacce = NULL; return ret; } @@ -1509,6 +1736,9 @@ void hisi_qm_uninit(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; struct device *dev = &pdev->dev; + uacce_remove(qm->uacce); + qm->uacce = NULL; + if (qm->use_dma_api && qm->qdma.va) { hisi_qm_cache_wb(qm); dma_free_coherent(dev, qm->qdma.size, @@ -1856,43 +2086,30 @@ void hisi_qm_debug_regs_clear(struct hisi_qm *qm) } EXPORT_SYMBOL_GPL(hisi_qm_debug_regs_clear); -/** - * hisi_qm_hw_error_init() - Configure qm hardware error report method. - * @qm: The qm which we want to configure. - * @ce: Bit mask of correctable error configure. - * @nfe: Bit mask of non-fatal error configure. - * @fe: Bit mask of fatal error configure. - * @msi: Bit mask of error reported by message signal interrupt. - * - * Hardware errors of qm can be reported either by RAS interrupts which will - * be handled by UEFI and then PCIe AER or by device MSI. User can configure - * each error to use either of above two methods. For RAS interrupts, we can - * configure an error as one of correctable error, non-fatal error or - * fatal error. - * - * Bits indicating errors can be configured to ce, nfe, fe and msi to enable - * related report methods. Error report will be masked if related error bit - * does not configure. - */ -void hisi_qm_hw_error_init(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe, - u32 msi) +static void qm_hw_error_init(struct hisi_qm *qm) { + const struct hisi_qm_err_info *err_info = &qm->err_ini->err_info; + if (!qm->ops->hw_error_init) { dev_err(&qm->pdev->dev, "QM doesn't support hw error handling!\n"); return; } - qm->ops->hw_error_init(qm, ce, nfe, fe, msi); + qm->ops->hw_error_init(qm, err_info->ce, err_info->nfe, + err_info->fe, err_info->msi); } -EXPORT_SYMBOL_GPL(hisi_qm_hw_error_init); -/** - * hisi_qm_hw_error_handle() - Handle qm non-fatal hardware errors. - * @qm: The qm which has non-fatal hardware errors. - * - * Accelerators use this function to handle qm non-fatal hardware errors. - */ -pci_ers_result_t hisi_qm_hw_error_handle(struct hisi_qm *qm) +static void qm_hw_error_uninit(struct hisi_qm *qm) +{ + if (!qm->ops->hw_error_uninit) { + dev_err(&qm->pdev->dev, "Unexpected QM hw error uninit!\n"); + return; + } + + qm->ops->hw_error_uninit(qm); +} + +static pci_ers_result_t qm_hw_error_handle(struct hisi_qm *qm) { if (!qm->ops->hw_error_handle) { dev_err(&qm->pdev->dev, "QM doesn't support hw error report!\n"); @@ -1901,7 +2118,6 @@ pci_ers_result_t hisi_qm_hw_error_handle(struct hisi_qm *qm) return qm->ops->hw_error_handle(qm); } -EXPORT_SYMBOL_GPL(hisi_qm_hw_error_handle); /** * hisi_qm_get_hw_version() - Get hardware version of a qm. @@ -1922,6 +2138,229 @@ enum qm_hw_ver hisi_qm_get_hw_version(struct pci_dev *pdev) } EXPORT_SYMBOL_GPL(hisi_qm_get_hw_version); +/** + * hisi_qm_dev_err_init() - Initialize device error configuration. + * @qm: The qm for which we want to do error initialization. + * + * Initialize QM and device error related configuration. + */ +void hisi_qm_dev_err_init(struct hisi_qm *qm) +{ + if (qm->fun_type == QM_HW_VF) + return; + + qm_hw_error_init(qm); + + if (!qm->err_ini->hw_err_enable) { + dev_err(&qm->pdev->dev, "Device doesn't support hw error init!\n"); + return; + } + qm->err_ini->hw_err_enable(qm); +} +EXPORT_SYMBOL_GPL(hisi_qm_dev_err_init); + +/** + * hisi_qm_dev_err_uninit() - Uninitialize device error configuration. + * @qm: The qm for which we want to do error uninitialization. + * + * Uninitialize QM and device error related configuration. + */ +void hisi_qm_dev_err_uninit(struct hisi_qm *qm) +{ + if (qm->fun_type == QM_HW_VF) + return; + + qm_hw_error_uninit(qm); + + if (!qm->err_ini->hw_err_disable) { + dev_err(&qm->pdev->dev, "Unexpected device hw error uninit!\n"); + return; + } + qm->err_ini->hw_err_disable(qm); +} +EXPORT_SYMBOL_GPL(hisi_qm_dev_err_uninit); + +/** + * hisi_qm_free_qps() - free multiple queue pairs. + * @qps: The queue pairs need to be freed. + * @qp_num: The num of queue pairs. + */ +void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num) +{ + int i; + + if (!qps || qp_num <= 0) + return; + + for (i = qp_num - 1; i >= 0; i--) + hisi_qm_release_qp(qps[i]); +} +EXPORT_SYMBOL_GPL(hisi_qm_free_qps); + +static void free_list(struct list_head *head) +{ + struct hisi_qm_resource *res, *tmp; + + list_for_each_entry_safe(res, tmp, head, list) { + list_del(&res->list); + kfree(res); + } +} + +static int hisi_qm_sort_devices(int node, struct list_head *head, + struct hisi_qm_list *qm_list) +{ + struct hisi_qm_resource *res, *tmp; + struct hisi_qm *qm; + struct list_head *n; + struct device *dev; + int dev_node = 0; + + list_for_each_entry(qm, &qm_list->list, list) { + dev = &qm->pdev->dev; + + if (IS_ENABLED(CONFIG_NUMA)) { + dev_node = dev_to_node(dev); + if (dev_node < 0) + dev_node = 0; + } + + res = kzalloc(sizeof(*res), GFP_KERNEL); + if (!res) + return -ENOMEM; + + res->qm = qm; + res->distance = node_distance(dev_node, node); + n = head; + list_for_each_entry(tmp, head, list) { + if (res->distance < tmp->distance) { + n = &tmp->list; + break; + } + } + list_add_tail(&res->list, n); + } + + return 0; +} + +/** + * hisi_qm_alloc_qps_node() - Create multiple queue pairs. + * @qm_list: The list of all available devices. + * @qp_num: The number of queue pairs need created. + * @alg_type: The algorithm type. + * @node: The numa node. + * @qps: The queue pairs need created. + * + * This function will sort all available device according to numa distance. + * Then try to create all queue pairs from one device, if all devices do + * not meet the requirements will return error. + */ +int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, + u8 alg_type, int node, struct hisi_qp **qps) +{ + struct hisi_qm_resource *tmp; + int ret = -ENODEV; + LIST_HEAD(head); + int i; + + if (!qps || !qm_list || qp_num <= 0) + return -EINVAL; + + mutex_lock(&qm_list->lock); + if (hisi_qm_sort_devices(node, &head, qm_list)) { + mutex_unlock(&qm_list->lock); + goto err; + } + + list_for_each_entry(tmp, &head, list) { + for (i = 0; i < qp_num; i++) { + qps[i] = hisi_qm_create_qp(tmp->qm, alg_type); + if (IS_ERR(qps[i])) { + hisi_qm_free_qps(qps, i); + break; + } + } + + if (i == qp_num) { + ret = 0; + break; + } + } + + mutex_unlock(&qm_list->lock); + if (ret) + pr_info("Failed to create qps, node[%d], alg[%d], qp[%d]!\n", + node, alg_type, qp_num); + +err: + free_list(&head); + return ret; +} +EXPORT_SYMBOL_GPL(hisi_qm_alloc_qps_node); + +static pci_ers_result_t qm_dev_err_handle(struct hisi_qm *qm) +{ + u32 err_sts; + + if (!qm->err_ini->get_dev_hw_err_status) { + dev_err(&qm->pdev->dev, "Device doesn't support get hw error status!\n"); + return PCI_ERS_RESULT_NONE; + } + + /* get device hardware error status */ + err_sts = qm->err_ini->get_dev_hw_err_status(qm); + if (err_sts) { + if (!qm->err_ini->log_dev_hw_err) { + dev_err(&qm->pdev->dev, "Device doesn't support log hw error!\n"); + return PCI_ERS_RESULT_NEED_RESET; + } + + qm->err_ini->log_dev_hw_err(qm, err_sts); + return PCI_ERS_RESULT_NEED_RESET; + } + + return PCI_ERS_RESULT_RECOVERED; +} + +static pci_ers_result_t qm_process_dev_error(struct pci_dev *pdev) +{ + struct hisi_qm *qm = pci_get_drvdata(pdev); + pci_ers_result_t qm_ret, dev_ret; + + /* log qm error */ + qm_ret = qm_hw_error_handle(qm); + + /* log device error */ + dev_ret = qm_dev_err_handle(qm); + + return (qm_ret == PCI_ERS_RESULT_NEED_RESET || + dev_ret == PCI_ERS_RESULT_NEED_RESET) ? + PCI_ERS_RESULT_NEED_RESET : PCI_ERS_RESULT_RECOVERED; +} + +/** + * hisi_qm_dev_err_detected() - Get device and qm error status then log it. + * @pdev: The PCI device which need report error. + * @state: The connectivity between CPU and device. + * + * We register this function into PCIe AER handlers, It will report device or + * qm hardware error status when error occur. + */ +pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + if (pdev->is_virtfn) + return PCI_ERS_RESULT_NONE; + + pci_info(pdev, "PCI error detected, state(=%d)!!\n", state); + if (state == pci_channel_io_perm_failure) + return PCI_ERS_RESULT_DISCONNECT; + + return qm_process_dev_error(pdev); +} +EXPORT_SYMBOL_GPL(hisi_qm_dev_err_detected); + MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>"); MODULE_DESCRIPTION("HiSilicon Accelerator queue manager driver"); diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h index 078b8f1f1b77..ec5b6f48db6c 100644 --- a/drivers/crypto/hisilicon/qm.h +++ b/drivers/crypto/hisilicon/qm.h @@ -77,6 +77,9 @@ #define HISI_ACC_SGL_SGE_NR_MAX 255 +/* page number for queue file region */ +#define QM_DOORBELL_PAGE_NR 1 + enum qp_state { QP_STOP, }; @@ -125,6 +128,28 @@ struct hisi_qm_status { unsigned long flags; }; +struct hisi_qm; + +struct hisi_qm_err_info { + u32 ce; + u32 nfe; + u32 fe; + u32 msi; +}; + +struct hisi_qm_err_ini { + void (*hw_err_enable)(struct hisi_qm *qm); + void (*hw_err_disable)(struct hisi_qm *qm); + u32 (*get_dev_hw_err_status)(struct hisi_qm *qm); + void (*log_dev_hw_err)(struct hisi_qm *qm, u32 err_sts); + struct hisi_qm_err_info err_info; +}; + +struct hisi_qm_list { + struct mutex lock; + struct list_head list; +}; + struct hisi_qm { enum qm_hw_ver ver; enum qm_fun_type fun_type; @@ -136,6 +161,7 @@ struct hisi_qm { u32 qp_num; u32 qp_in_used; u32 ctrl_qp_num; + struct list_head list; struct qm_dma qdma; struct qm_sqc *sqc; @@ -148,6 +174,7 @@ struct hisi_qm { dma_addr_t aeqe_dma; struct hisi_qm_status status; + const struct hisi_qm_err_ini *err_ini; rwlock_t qps_lock; unsigned long *qp_bitmap; @@ -162,7 +189,15 @@ struct hisi_qm { u32 error_mask; u32 msi_mask; + struct workqueue_struct *wq; + struct work_struct work; + + const char *algs; bool use_dma_api; + bool use_sva; + resource_size_t phys_base; + resource_size_t phys_size; + struct uacce_device *uacce; }; struct hisi_qp_status { @@ -192,12 +227,35 @@ struct hisi_qp { struct hisi_qp_ops *hw_ops; void *qp_ctx; void (*req_cb)(struct hisi_qp *qp, void *data); - struct work_struct work; - struct workqueue_struct *wq; + void (*event_cb)(struct hisi_qp *qp); struct hisi_qm *qm; + u16 pasid; + struct uacce_queue *uacce_q; }; +static inline void hisi_qm_init_list(struct hisi_qm_list *qm_list) +{ + INIT_LIST_HEAD(&qm_list->list); + mutex_init(&qm_list->lock); +} + +static inline void hisi_qm_add_to_list(struct hisi_qm *qm, + struct hisi_qm_list *qm_list) +{ + mutex_lock(&qm_list->lock); + list_add_tail(&qm->list, &qm_list->list); + mutex_unlock(&qm_list->lock); +} + +static inline void hisi_qm_del_from_list(struct hisi_qm *qm, + struct hisi_qm_list *qm_list) +{ + mutex_lock(&qm_list->lock); + list_del(&qm->list); + mutex_unlock(&qm_list->lock); +} + int hisi_qm_init(struct hisi_qm *qm); void hisi_qm_uninit(struct hisi_qm *qm); int hisi_qm_start(struct hisi_qm *qm); @@ -211,11 +269,12 @@ int hisi_qm_get_free_qp_num(struct hisi_qm *qm); int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number); int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base, u32 number); int hisi_qm_debug_init(struct hisi_qm *qm); -void hisi_qm_hw_error_init(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe, - u32 msi); -pci_ers_result_t hisi_qm_hw_error_handle(struct hisi_qm *qm); enum qm_hw_ver hisi_qm_get_hw_version(struct pci_dev *pdev); void hisi_qm_debug_regs_clear(struct hisi_qm *qm); +void hisi_qm_dev_err_init(struct hisi_qm *qm); +void hisi_qm_dev_err_uninit(struct hisi_qm *qm); +pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev, + pci_channel_state_t state); struct hisi_acc_sgl_pool; struct hisi_acc_hw_sgl *hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, @@ -227,4 +286,7 @@ struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev, u32 count, u32 sge_nr); void hisi_acc_free_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool); +int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, + u8 alg_type, int node, struct hisi_qp **qps); +void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num); #endif diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 13e2d8d7be94..3598fa17beb2 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -11,6 +11,8 @@ /* Algorithm resource per hardware SEC queue */ struct sec_alg_res { + u8 *pbuf; + dma_addr_t pbuf_dma; u8 *c_ivin; dma_addr_t c_ivin_dma; u8 *out_mac; @@ -23,6 +25,8 @@ struct sec_cipher_req { dma_addr_t c_in_dma; struct hisi_acc_hw_sgl *c_out; dma_addr_t c_out_dma; + u8 *c_ivin; + dma_addr_t c_ivin_dma; struct skcipher_request *sk_req; u32 c_len; bool encrypt; @@ -48,6 +52,7 @@ struct sec_req { /* Status of the SEC request */ bool fake_busy; + bool use_pbuf; }; /** @@ -114,6 +119,7 @@ struct sec_ctx { struct sec_qp_ctx *qp_ctx; struct sec_dev *sec; const struct sec_req_op *req_op; + struct hisi_qp **qps; /* Half queues for encipher, and half for decipher */ u32 hlf_q_num; @@ -128,6 +134,7 @@ struct sec_ctx { atomic_t dec_qcyclic; enum sec_alg_type alg_type; + bool pbuf_supported; struct sec_cipher_ctx c_ctx; struct sec_auth_ctx a_ctx; }; @@ -162,14 +169,15 @@ struct sec_debug { struct sec_dev { struct hisi_qm qm; - struct list_head list; struct sec_debug debug; u32 ctx_q_num; + bool iommu_used; u32 num_vfs; unsigned long status; }; -struct sec_dev *sec_find_device(int node); +void sec_destroy_qps(struct hisi_qp **qps, int qp_num); +struct hisi_qp **sec_create_qps(void); int sec_register_to_crypto(void); void sec_unregister_from_crypto(void); #endif diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index a2cfcc9ccd94..7f1c6a31b82f 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -46,7 +46,21 @@ #define SEC_CIPHER_AUTH 0xfe #define SEC_AUTH_CIPHER 0x1 #define SEC_MAX_MAC_LEN 64 +#define SEC_MAX_AAD_LEN 65535 #define SEC_TOTAL_MAC_SZ (SEC_MAX_MAC_LEN * QM_Q_DEPTH) + +#define SEC_PBUF_SZ 512 +#define SEC_PBUF_IV_OFFSET SEC_PBUF_SZ +#define SEC_PBUF_MAC_OFFSET (SEC_PBUF_SZ + SEC_IV_SIZE) +#define SEC_PBUF_PKG (SEC_PBUF_SZ + SEC_IV_SIZE + \ + SEC_MAX_MAC_LEN * 2) +#define SEC_PBUF_NUM (PAGE_SIZE / SEC_PBUF_PKG) +#define SEC_PBUF_PAGE_NUM (QM_Q_DEPTH / SEC_PBUF_NUM) +#define SEC_PBUF_LEFT_SZ (SEC_PBUF_PKG * (QM_Q_DEPTH - \ + SEC_PBUF_PAGE_NUM * SEC_PBUF_NUM)) +#define SEC_TOTAL_PBUF_SZ (PAGE_SIZE * SEC_PBUF_PAGE_NUM + \ + SEC_PBUF_LEFT_SZ) + #define SEC_SQE_LEN_RATE 4 #define SEC_SQE_CFLAG 2 #define SEC_SQE_AEAD_FLAG 3 @@ -110,12 +124,12 @@ static void sec_free_req_id(struct sec_req *req) mutex_unlock(&qp_ctx->req_lock); } -static int sec_aead_verify(struct sec_req *req, struct sec_qp_ctx *qp_ctx) +static int sec_aead_verify(struct sec_req *req) { struct aead_request *aead_req = req->aead_req.aead_req; struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); - u8 *mac_out = qp_ctx->res[req->req_id].out_mac; size_t authsize = crypto_aead_authsize(tfm); + u8 *mac_out = req->aead_req.out_mac; u8 *mac = mac_out + SEC_MAX_MAC_LEN; struct scatterlist *sgl = aead_req->src; size_t sz; @@ -163,7 +177,7 @@ static void sec_req_cb(struct hisi_qp *qp, void *resp) } if (ctx->alg_type == SEC_AEAD && !req->c_req.encrypt) - err = sec_aead_verify(req, qp_ctx); + err = sec_aead_verify(req); atomic64_inc(&ctx->sec->debug.dfx.recv_cnt); @@ -245,6 +259,50 @@ static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res) res->out_mac, res->out_mac_dma); } +static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res) +{ + if (res->pbuf) + dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ, + res->pbuf, res->pbuf_dma); +} + +/* + * To improve performance, pbuffer is used for + * small packets (< 512Bytes) as IOMMU translation using. + */ +static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res) +{ + int pbuf_page_offset; + int i, j, k; + + res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ, + &res->pbuf_dma, GFP_KERNEL); + if (!res->pbuf) + return -ENOMEM; + + /* + * SEC_PBUF_PKG contains data pbuf, iv and + * out_mac : <SEC_PBUF|SEC_IV|SEC_MAC> + * Every PAGE contains six SEC_PBUF_PKG + * The sec_qp_ctx contains QM_Q_DEPTH numbers of SEC_PBUF_PKG + * So we need SEC_PBUF_PAGE_NUM numbers of PAGE + * for the SEC_TOTAL_PBUF_SZ + */ + for (i = 0; i <= SEC_PBUF_PAGE_NUM; i++) { + pbuf_page_offset = PAGE_SIZE * i; + for (j = 0; j < SEC_PBUF_NUM; j++) { + k = i * SEC_PBUF_NUM + j; + if (k == QM_Q_DEPTH) + break; + res[k].pbuf = res->pbuf + + j * SEC_PBUF_PKG + pbuf_page_offset; + res[k].pbuf_dma = res->pbuf_dma + + j * SEC_PBUF_PKG + pbuf_page_offset; + } + } + return 0; +} + static int sec_alg_resource_alloc(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { @@ -259,11 +317,18 @@ static int sec_alg_resource_alloc(struct sec_ctx *ctx, if (ctx->alg_type == SEC_AEAD) { ret = sec_alloc_mac_resource(dev, res); if (ret) - goto get_fail; + goto alloc_fail; + } + if (ctx->pbuf_supported) { + ret = sec_alloc_pbuf_resource(dev, res); + if (ret) { + dev_err(dev, "fail to alloc pbuf dma resource!\n"); + goto alloc_fail; + } } return 0; -get_fail: +alloc_fail: sec_free_civ_resource(dev, res); return ret; @@ -276,6 +341,8 @@ static void sec_alg_resource_free(struct sec_ctx *ctx, sec_free_civ_resource(dev, qp_ctx->res); + if (ctx->pbuf_supported) + sec_free_pbuf_resource(dev, qp_ctx->res); if (ctx->alg_type == SEC_AEAD) sec_free_mac_resource(dev, qp_ctx->res); } @@ -288,11 +355,8 @@ static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx, struct hisi_qp *qp; int ret = -ENOMEM; - qp = hisi_qm_create_qp(qm, alg_type); - if (IS_ERR(qp)) - return PTR_ERR(qp); - qp_ctx = &ctx->qp_ctx[qp_ctx_id]; + qp = ctx->qps[qp_ctx_id]; qp->req_type = 0; qp->qp_ctx = qp_ctx; qp->req_cb = sec_req_cb; @@ -335,7 +399,6 @@ err_free_c_in_pool: hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); err_destroy_idr: idr_destroy(&qp_ctx->req_idr); - hisi_qm_release_qp(qp); return ret; } @@ -352,7 +415,6 @@ static void sec_release_qp_ctx(struct sec_ctx *ctx, hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); idr_destroy(&qp_ctx->req_idr); - hisi_qm_release_qp(qp_ctx->qp); } static int sec_ctx_base_init(struct sec_ctx *ctx) @@ -360,14 +422,18 @@ static int sec_ctx_base_init(struct sec_ctx *ctx) struct sec_dev *sec; int i, ret; - sec = sec_find_device(cpu_to_node(smp_processor_id())); - if (!sec) { - pr_err("Can not find proper Hisilicon SEC device!\n"); + ctx->qps = sec_create_qps(); + if (!ctx->qps) { + pr_err("Can not create sec qps!\n"); return -ENODEV; } + + sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm); ctx->sec = sec; ctx->hlf_q_num = sec->ctx_q_num >> 1; + ctx->pbuf_supported = ctx->sec->iommu_used; + /* Half of queue depth is taken as fake requests limit in the queue. */ ctx->fake_req_limit = QM_Q_DEPTH >> 1; ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx), @@ -386,6 +452,7 @@ err_sec_release_qp_ctx: for (i = i - 1; i >= 0; i--) sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); + sec_destroy_qps(ctx->qps, sec->ctx_q_num); kfree(ctx->qp_ctx); return ret; } @@ -397,6 +464,7 @@ static void sec_ctx_base_uninit(struct sec_ctx *ctx) for (i = 0; i < ctx->sec->ctx_q_num; i++) sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); + sec_destroy_qps(ctx->qps, ctx->sec->ctx_q_num); kfree(ctx->qp_ctx); } @@ -447,7 +515,6 @@ static int sec_skcipher_init(struct crypto_skcipher *tfm) struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); int ret; - ctx = crypto_skcipher_ctx(tfm); ctx->alg_type = SEC_SKCIPHER; crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req)); ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm); @@ -591,11 +658,94 @@ GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC) GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS) GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC) -static int sec_cipher_map(struct device *dev, struct sec_req *req, +static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, + struct scatterlist *src) +{ + struct aead_request *aead_req = req->aead_req.aead_req; + struct sec_cipher_req *c_req = &req->c_req; + struct sec_qp_ctx *qp_ctx = req->qp_ctx; + struct device *dev = SEC_CTX_DEV(ctx); + int copy_size, pbuf_length; + int req_id = req->req_id; + + if (ctx->alg_type == SEC_AEAD) + copy_size = aead_req->cryptlen + aead_req->assoclen; + else + copy_size = c_req->c_len; + + pbuf_length = sg_copy_to_buffer(src, sg_nents(src), + qp_ctx->res[req_id].pbuf, + copy_size); + + if (unlikely(pbuf_length != copy_size)) { + dev_err(dev, "copy src data to pbuf error!\n"); + return -EINVAL; + } + + c_req->c_in_dma = qp_ctx->res[req_id].pbuf_dma; + + if (!c_req->c_in_dma) { + dev_err(dev, "fail to set pbuffer address!\n"); + return -ENOMEM; + } + + c_req->c_out_dma = c_req->c_in_dma; + + return 0; +} + +static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, + struct scatterlist *dst) +{ + struct aead_request *aead_req = req->aead_req.aead_req; + struct sec_cipher_req *c_req = &req->c_req; + struct sec_qp_ctx *qp_ctx = req->qp_ctx; + struct device *dev = SEC_CTX_DEV(ctx); + int copy_size, pbuf_length; + int req_id = req->req_id; + + if (ctx->alg_type == SEC_AEAD) + copy_size = c_req->c_len + aead_req->assoclen; + else + copy_size = c_req->c_len; + + pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), + qp_ctx->res[req_id].pbuf, + copy_size); + + if (unlikely(pbuf_length != copy_size)) + dev_err(dev, "copy pbuf data to dst error!\n"); + +} + +static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, struct scatterlist *src, struct scatterlist *dst) { struct sec_cipher_req *c_req = &req->c_req; + struct sec_aead_req *a_req = &req->aead_req; struct sec_qp_ctx *qp_ctx = req->qp_ctx; + struct sec_alg_res *res = &qp_ctx->res[req->req_id]; + struct device *dev = SEC_CTX_DEV(ctx); + int ret; + + if (req->use_pbuf) { + ret = sec_cipher_pbuf_map(ctx, req, src); + c_req->c_ivin = res->pbuf + SEC_PBUF_IV_OFFSET; + c_req->c_ivin_dma = res->pbuf_dma + SEC_PBUF_IV_OFFSET; + if (ctx->alg_type == SEC_AEAD) { + a_req->out_mac = res->pbuf + SEC_PBUF_MAC_OFFSET; + a_req->out_mac_dma = res->pbuf_dma + + SEC_PBUF_MAC_OFFSET; + } + + return ret; + } + c_req->c_ivin = res->c_ivin; + c_req->c_ivin_dma = res->c_ivin_dma; + if (ctx->alg_type == SEC_AEAD) { + a_req->out_mac = res->out_mac; + a_req->out_mac_dma = res->out_mac_dma; + } c_req->c_in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src, qp_ctx->c_in_pool, @@ -626,29 +776,34 @@ static int sec_cipher_map(struct device *dev, struct sec_req *req, return 0; } -static void sec_cipher_unmap(struct device *dev, struct sec_cipher_req *req, +static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req, struct scatterlist *src, struct scatterlist *dst) { - if (dst != src) - hisi_acc_sg_buf_unmap(dev, src, req->c_in); + struct sec_cipher_req *c_req = &req->c_req; + struct device *dev = SEC_CTX_DEV(ctx); + + if (req->use_pbuf) { + sec_cipher_pbuf_unmap(ctx, req, dst); + } else { + if (dst != src) + hisi_acc_sg_buf_unmap(dev, src, c_req->c_in); - hisi_acc_sg_buf_unmap(dev, dst, req->c_out); + hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out); + } } static int sec_skcipher_sgl_map(struct sec_ctx *ctx, struct sec_req *req) { struct skcipher_request *sq = req->c_req.sk_req; - return sec_cipher_map(SEC_CTX_DEV(ctx), req, sq->src, sq->dst); + return sec_cipher_map(ctx, req, sq->src, sq->dst); } static void sec_skcipher_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) { - struct device *dev = SEC_CTX_DEV(ctx); - struct sec_cipher_req *c_req = &req->c_req; - struct skcipher_request *sk_req = c_req->sk_req; + struct skcipher_request *sq = req->c_req.sk_req; - sec_cipher_unmap(dev, c_req, sk_req->src, sk_req->dst); + sec_cipher_unmap(ctx, req, sq->src, sq->dst); } static int sec_aead_aes_set_key(struct sec_cipher_ctx *c_ctx, @@ -759,16 +914,14 @@ static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req) { struct aead_request *aq = req->aead_req.aead_req; - return sec_cipher_map(SEC_CTX_DEV(ctx), req, aq->src, aq->dst); + return sec_cipher_map(ctx, req, aq->src, aq->dst); } static void sec_aead_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) { - struct device *dev = SEC_CTX_DEV(ctx); - struct sec_cipher_req *cq = &req->c_req; struct aead_request *aq = req->aead_req.aead_req; - sec_cipher_unmap(dev, cq, aq->src, aq->dst); + sec_cipher_unmap(ctx, req, aq->src, aq->dst); } static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req) @@ -801,9 +954,9 @@ static void sec_request_untransfer(struct sec_ctx *ctx, struct sec_req *req) static void sec_skcipher_copy_iv(struct sec_ctx *ctx, struct sec_req *req) { struct skcipher_request *sk_req = req->c_req.sk_req; - u8 *c_ivin = req->qp_ctx->res[req->req_id].c_ivin; + struct sec_cipher_req *c_req = &req->c_req; - memcpy(c_ivin, sk_req->iv, ctx->c_ctx.ivsize); + memcpy(c_req->c_ivin, sk_req->iv, ctx->c_ctx.ivsize); } static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) @@ -818,8 +971,7 @@ static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) memset(sec_sqe, 0, sizeof(struct sec_sqe)); sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma); - sec_sqe->type2.c_ivin_addr = - cpu_to_le64(req->qp_ctx->res[req->req_id].c_ivin_dma); + sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); sec_sqe->type2.data_src_addr = cpu_to_le64(c_req->c_in_dma); sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma); @@ -836,7 +988,10 @@ static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET; sec_sqe->type_cipher_auth = bd_type | cipher; - sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET; + if (req->use_pbuf) + sa_type = SEC_PBUF << SEC_SRC_SGL_OFFSET; + else + sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET; scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET; if (c_req->c_in_dma != c_req->c_out_dma) de = 0x1 << SEC_DE_OFFSET; @@ -844,7 +999,10 @@ static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) sec_sqe->sds_sa_type = (de | scene | sa_type); /* Just set DST address type */ - da_type = SEC_SGL << SEC_DST_SGL_OFFSET; + if (req->use_pbuf) + da_type = SEC_PBUF << SEC_DST_SGL_OFFSET; + else + da_type = SEC_SGL << SEC_DST_SGL_OFFSET; sec_sqe->sdm_addr_type |= da_type; sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len); @@ -904,9 +1062,9 @@ static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, static void sec_aead_copy_iv(struct sec_ctx *ctx, struct sec_req *req) { struct aead_request *aead_req = req->aead_req.aead_req; - u8 *c_ivin = req->qp_ctx->res[req->req_id].c_ivin; + struct sec_cipher_req *c_req = &req->c_req; - memcpy(c_ivin, aead_req->iv, ctx->c_ctx.ivsize); + memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize); } static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, @@ -939,8 +1097,7 @@ static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); - sec_sqe->type2.mac_addr = - cpu_to_le64(req->qp_ctx->res[req->req_id].out_mac_dma); + sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma); } static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req) @@ -964,6 +1121,7 @@ static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err) { struct aead_request *a_req = req->aead_req.aead_req; struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); + struct sec_aead_req *aead_req = &req->aead_req; struct sec_cipher_req *c_req = &req->c_req; size_t authsize = crypto_aead_authsize(tfm); struct sec_qp_ctx *qp_ctx = req->qp_ctx; @@ -979,7 +1137,7 @@ static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err) struct scatterlist *sgl = a_req->dst; sz = sg_pcopy_from_buffer(sgl, sg_nents(sgl), - qp_ctx->res[req->req_id].out_mac, + aead_req->out_mac, authsize, a_req->cryptlen + a_req->assoclen); @@ -1031,6 +1189,7 @@ static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req) static int sec_process(struct sec_ctx *ctx, struct sec_req *req) { + struct sec_cipher_req *c_req = &req->c_req; int ret; ret = sec_request_init(ctx, req); @@ -1057,12 +1216,10 @@ err_send_req: /* As failing, restore the IV from user */ if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) { if (ctx->alg_type == SEC_SKCIPHER) - memcpy(req->c_req.sk_req->iv, - req->qp_ctx->res[req->req_id].c_ivin, + memcpy(req->c_req.sk_req->iv, c_req->c_ivin, ctx->c_ctx.ivsize); else - memcpy(req->aead_req.aead_req->iv, - req->qp_ctx->res[req->req_id].c_ivin, + memcpy(req->aead_req.aead_req->iv, c_req->c_ivin, ctx->c_ctx.ivsize); } @@ -1208,6 +1365,12 @@ static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) return -EINVAL; } sreq->c_req.c_len = sk_req->cryptlen; + + if (ctx->pbuf_supported && sk_req->cryptlen <= SEC_PBUF_SZ) + sreq->use_pbuf = true; + else + sreq->use_pbuf = false; + if (c_alg == SEC_CALG_3DES) { if (unlikely(sk_req->cryptlen & (DES3_EDE_BLOCK_SIZE - 1))) { dev_err(dev, "skcipher 3des input length error!\n"); @@ -1321,11 +1484,18 @@ static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) struct crypto_aead *tfm = crypto_aead_reqtfm(req); size_t authsize = crypto_aead_authsize(tfm); - if (unlikely(!req->src || !req->dst || !req->cryptlen)) { + if (unlikely(!req->src || !req->dst || !req->cryptlen || + req->assoclen > SEC_MAX_AAD_LEN)) { dev_err(SEC_CTX_DEV(ctx), "aead input param error!\n"); return -EINVAL; } + if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <= + SEC_PBUF_SZ) + sreq->use_pbuf = true; + else + sreq->use_pbuf = false; + /* Support AES only */ if (unlikely(c_alg != SEC_CALG_AES)) { dev_err(SEC_CTX_DEV(ctx), "aead crypto alg error!\n"); diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 2bbaf1e2dae7..1f54ebe164b6 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -7,6 +7,7 @@ #include <linux/debugfs.h> #include <linux/init.h> #include <linux/io.h> +#include <linux/iommu.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/pci.h> @@ -89,8 +90,7 @@ struct sec_hw_error { static const char sec_name[] = "hisi_sec2"; static struct dentry *sec_debugfs_root; -static LIST_HEAD(sec_list); -static DEFINE_MUTEX(sec_list_lock); +static struct hisi_qm_list sec_devices; static const struct sec_hw_error sec_hw_errors[] = { {.int_msk = BIT(0), .msg = "sec_axi_rresp_err_rint"}, @@ -105,37 +105,6 @@ static const struct sec_hw_error sec_hw_errors[] = { { /* sentinel */ } }; -struct sec_dev *sec_find_device(int node) -{ -#define SEC_NUMA_MAX_DISTANCE 100 - int min_distance = SEC_NUMA_MAX_DISTANCE; - int dev_node = 0, free_qp_num = 0; - struct sec_dev *sec, *ret = NULL; - struct hisi_qm *qm; - struct device *dev; - - mutex_lock(&sec_list_lock); - list_for_each_entry(sec, &sec_list, list) { - qm = &sec->qm; - dev = &qm->pdev->dev; -#ifdef CONFIG_NUMA - dev_node = dev->numa_node; - if (dev_node < 0) - dev_node = 0; -#endif - if (node_distance(dev_node, node) < min_distance) { - free_qp_num = hisi_qm_get_free_qp_num(qm); - if (free_qp_num >= sec->ctx_q_num) { - ret = sec; - min_distance = node_distance(dev_node, node); - } - } - } - mutex_unlock(&sec_list_lock); - - return ret; -} - static const char * const sec_dbg_file_name[] = { [SEC_CURRENT_QM] = "current_qm", [SEC_CLEAR_ENABLE] = "clear_enable", @@ -238,6 +207,32 @@ static u32 ctx_q_num = SEC_CTX_Q_NUM_DEF; module_param_cb(ctx_q_num, &sec_ctx_q_num_ops, &ctx_q_num, 0444); MODULE_PARM_DESC(ctx_q_num, "Queue num in ctx (24 default, 2, 4, ..., 32)"); +void sec_destroy_qps(struct hisi_qp **qps, int qp_num) +{ + hisi_qm_free_qps(qps, qp_num); + kfree(qps); +} + +struct hisi_qp **sec_create_qps(void) +{ + int node = cpu_to_node(smp_processor_id()); + u32 ctx_num = ctx_q_num; + struct hisi_qp **qps; + int ret; + + qps = kcalloc(ctx_num, sizeof(struct hisi_qp *), GFP_KERNEL); + if (!qps) + return NULL; + + ret = hisi_qm_alloc_qps_node(&sec_devices, ctx_num, 0, node, qps); + if (!ret) + return qps; + + kfree(qps); + return NULL; +} + + static const struct pci_device_id sec_dev_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, SEC_PF_PCI_DEVICE_ID) }, { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, SEC_VF_PCI_DEVICE_ID) }, @@ -245,20 +240,6 @@ static const struct pci_device_id sec_dev_ids[] = { }; MODULE_DEVICE_TABLE(pci, sec_dev_ids); -static inline void sec_add_to_list(struct sec_dev *sec) -{ - mutex_lock(&sec_list_lock); - list_add_tail(&sec->list, &sec_list); - mutex_unlock(&sec_list_lock); -} - -static inline void sec_remove_from_list(struct sec_dev *sec) -{ - mutex_lock(&sec_list_lock); - list_del(&sec->list); - mutex_unlock(&sec_list_lock); -} - static u8 sec_get_endian(struct sec_dev *sec) { struct hisi_qm *qm = &sec->qm; @@ -384,9 +365,8 @@ static void sec_debug_regs_clear(struct hisi_qm *qm) hisi_qm_debug_regs_clear(qm); } -static void sec_hw_error_enable(struct sec_dev *sec) +static void sec_hw_error_enable(struct hisi_qm *qm) { - struct hisi_qm *qm = &sec->qm; u32 val; if (qm->ver == QM_HW_V1) { @@ -414,9 +394,8 @@ static void sec_hw_error_enable(struct sec_dev *sec) writel(val, qm->io_base + SEC_CONTROL_REG); } -static void sec_hw_error_disable(struct sec_dev *sec) +static void sec_hw_error_disable(struct hisi_qm *qm) { - struct hisi_qm *qm = &sec->qm; u32 val; val = readl(qm->io_base + SEC_CONTROL_REG); @@ -435,27 +414,6 @@ static void sec_hw_error_disable(struct sec_dev *sec) writel(val, qm->io_base + SEC_CONTROL_REG); } -static void sec_hw_error_init(struct sec_dev *sec) -{ - if (sec->qm.fun_type == QM_HW_VF) - return; - - hisi_qm_hw_error_init(&sec->qm, QM_BASE_CE, - QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT - | QM_ACC_WB_NOT_READY_TIMEOUT, 0, - QM_DB_RANDOM_INVALID); - sec_hw_error_enable(sec); -} - -static void sec_hw_error_uninit(struct sec_dev *sec) -{ - if (sec->qm.fun_type == QM_HW_VF) - return; - - sec_hw_error_disable(sec); - writel(GENMASK(12, 0), sec->qm.io_base + SEC_QM_ABNORMAL_INT_MASK); -} - static u32 sec_current_qm_read(struct sec_debug_file *file) { struct hisi_qm *qm = file->qm; @@ -695,6 +653,51 @@ static void sec_debugfs_exit(struct sec_dev *sec) debugfs_remove_recursive(sec->qm.debug.debug_root); } +static void sec_log_hw_error(struct hisi_qm *qm, u32 err_sts) +{ + const struct sec_hw_error *errs = sec_hw_errors; + struct device *dev = &qm->pdev->dev; + u32 err_val; + + while (errs->msg) { + if (errs->int_msk & err_sts) { + dev_err(dev, "%s [error status=0x%x] found\n", + errs->msg, errs->int_msk); + + if (SEC_CORE_INT_STATUS_M_ECC & errs->int_msk) { + err_val = readl(qm->io_base + + SEC_CORE_SRAM_ECC_ERR_INFO); + dev_err(dev, "multi ecc sram num=0x%x\n", + SEC_ECC_NUM(err_val)); + dev_err(dev, "multi ecc sram addr=0x%x\n", + SEC_ECC_ADDR(err_val)); + } + } + errs++; + } + + writel(err_sts, qm->io_base + SEC_CORE_INT_SOURCE); +} + +static u32 sec_get_hw_err_status(struct hisi_qm *qm) +{ + return readl(qm->io_base + SEC_CORE_INT_STATUS); +} + +static const struct hisi_qm_err_ini sec_err_ini = { + .hw_err_enable = sec_hw_error_enable, + .hw_err_disable = sec_hw_error_disable, + .get_dev_hw_err_status = sec_get_hw_err_status, + .log_dev_hw_err = sec_log_hw_error, + .err_info = { + .ce = QM_BASE_CE, + .nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT | + QM_ACC_WB_NOT_READY_TIMEOUT, + .fe = 0, + .msi = QM_DB_RANDOM_INVALID, + } +}; + static int sec_pf_probe_init(struct sec_dev *sec) { struct hisi_qm *qm = &sec->qm; @@ -713,11 +716,13 @@ static int sec_pf_probe_init(struct sec_dev *sec) return -EINVAL; } + qm->err_ini = &sec_err_ini; + ret = sec_set_user_domain_and_cache(sec); if (ret) return ret; - sec_hw_error_init(sec); + hisi_qm_dev_err_init(qm); sec_debug_regs_clear(qm); return 0; @@ -750,12 +755,30 @@ static void sec_qm_uninit(struct hisi_qm *qm) static int sec_probe_init(struct hisi_qm *qm, struct sec_dev *sec) { + int ret; + + /* + * WQ_HIGHPRI: SEC request must be low delayed, + * so need a high priority workqueue. + * WQ_UNBOUND: SEC task is likely with long + * running CPU intensive workloads. + */ + qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | + WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus(), + pci_name(qm->pdev)); + if (!qm->wq) { + pci_err(qm->pdev, "fail to alloc workqueue\n"); + return -ENOMEM; + } + if (qm->fun_type == QM_HW_PF) { qm->qp_base = SEC_PF_DEF_Q_BASE; qm->qp_num = pf_q_num; qm->debug.curr_qm_qp_num = pf_q_num; - return sec_pf_probe_init(sec); + ret = sec_pf_probe_init(sec); + if (ret) + goto err_probe_uninit; } else if (qm->fun_type == QM_HW_VF) { /* * have no way to get qm configure in VM in v1 hardware, @@ -768,18 +791,43 @@ static int sec_probe_init(struct hisi_qm *qm, struct sec_dev *sec) qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM; } else if (qm->ver == QM_HW_V2) { /* v2 starts to support get vft by mailbox */ - return hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num); + ret = hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num); + if (ret) + goto err_probe_uninit; } } else { - return -ENODEV; + ret = -ENODEV; + goto err_probe_uninit; } return 0; +err_probe_uninit: + destroy_workqueue(qm->wq); + return ret; } -static void sec_probe_uninit(struct sec_dev *sec) +static void sec_probe_uninit(struct hisi_qm *qm) { - sec_hw_error_uninit(sec); + hisi_qm_dev_err_uninit(qm); + + destroy_workqueue(qm->wq); +} + +static void sec_iommu_used_check(struct sec_dev *sec) +{ + struct iommu_domain *domain; + struct device *dev = &sec->qm.pdev->dev; + + domain = iommu_get_domain_for_dev(dev); + + /* Check if iommu is used */ + sec->iommu_used = false; + if (domain) { + if (domain->type & __IOMMU_DOMAIN_PAGING) + sec->iommu_used = true; + dev_info(dev, "SMMU Opened, the iommu type = %u\n", + domain->type); + } } static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) @@ -795,6 +843,7 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) pci_set_drvdata(pdev, sec); sec->ctx_q_num = ctx_q_num; + sec_iommu_used_check(sec); qm = &sec->qm; @@ -820,7 +869,7 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) pci_warn(pdev, "Failed to init debugfs!\n"); - sec_add_to_list(sec); + hisi_qm_add_to_list(qm, &sec_devices); ret = sec_register_to_crypto(); if (ret < 0) { @@ -831,12 +880,12 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) return 0; err_remove_from_list: - sec_remove_from_list(sec); + hisi_qm_del_from_list(qm, &sec_devices); sec_debugfs_exit(sec); hisi_qm_stop(qm); err_probe_uninit: - sec_probe_uninit(sec); + sec_probe_uninit(qm); err_qm_uninit: sec_qm_uninit(qm); @@ -955,7 +1004,7 @@ static void sec_remove(struct pci_dev *pdev) sec_unregister_from_crypto(); - sec_remove_from_list(sec); + hisi_qm_del_from_list(qm, &sec_devices); if (qm->fun_type == QM_HW_PF && sec->num_vfs) (void)sec_sriov_disable(pdev); @@ -967,89 +1016,13 @@ static void sec_remove(struct pci_dev *pdev) if (qm->fun_type == QM_HW_PF) sec_debug_regs_clear(qm); - sec_probe_uninit(sec); + sec_probe_uninit(qm); sec_qm_uninit(qm); } -static void sec_log_hw_error(struct sec_dev *sec, u32 err_sts) -{ - const struct sec_hw_error *errs = sec_hw_errors; - struct device *dev = &sec->qm.pdev->dev; - u32 err_val; - - while (errs->msg) { - if (errs->int_msk & err_sts) { - dev_err(dev, "%s [error status=0x%x] found\n", - errs->msg, errs->int_msk); - - if (SEC_CORE_INT_STATUS_M_ECC & err_sts) { - err_val = readl(sec->qm.io_base + - SEC_CORE_SRAM_ECC_ERR_INFO); - dev_err(dev, "multi ecc sram num=0x%x\n", - SEC_ECC_NUM(err_val)); - dev_err(dev, "multi ecc sram addr=0x%x\n", - SEC_ECC_ADDR(err_val)); - } - } - errs++; - } -} - -static pci_ers_result_t sec_hw_error_handle(struct sec_dev *sec) -{ - u32 err_sts; - - /* read err sts */ - err_sts = readl(sec->qm.io_base + SEC_CORE_INT_STATUS); - if (err_sts) { - sec_log_hw_error(sec, err_sts); - - /* clear error interrupts */ - writel(err_sts, sec->qm.io_base + SEC_CORE_INT_SOURCE); - - return PCI_ERS_RESULT_NEED_RESET; - } - - return PCI_ERS_RESULT_RECOVERED; -} - -static pci_ers_result_t sec_process_hw_error(struct pci_dev *pdev) -{ - struct sec_dev *sec = pci_get_drvdata(pdev); - pci_ers_result_t qm_ret, sec_ret; - - if (!sec) { - pci_err(pdev, "Can't recover error during device init\n"); - return PCI_ERS_RESULT_NONE; - } - - /* log qm error */ - qm_ret = hisi_qm_hw_error_handle(&sec->qm); - - /* log sec error */ - sec_ret = sec_hw_error_handle(sec); - - return (qm_ret == PCI_ERS_RESULT_NEED_RESET || - sec_ret == PCI_ERS_RESULT_NEED_RESET) ? - PCI_ERS_RESULT_NEED_RESET : PCI_ERS_RESULT_RECOVERED; -} - -static pci_ers_result_t sec_error_detected(struct pci_dev *pdev, - pci_channel_state_t state) -{ - if (pdev->is_virtfn) - return PCI_ERS_RESULT_NONE; - - pci_info(pdev, "PCI error detected, state(=%d)!!\n", state); - if (state == pci_channel_io_perm_failure) - return PCI_ERS_RESULT_DISCONNECT; - - return sec_process_hw_error(pdev); -} - static const struct pci_error_handlers sec_err_handler = { - .error_detected = sec_error_detected, + .error_detected = hisi_qm_dev_err_detected, }; static struct pci_driver sec_pci_driver = { @@ -1078,6 +1051,7 @@ static int __init sec_init(void) { int ret; + hisi_qm_init_list(&sec_devices); sec_register_debugfs(); ret = pci_register_driver(&sec_pci_driver); diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h index bc1db26598bb..82dc6f867171 100644 --- a/drivers/crypto/hisilicon/zip/zip.h +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -68,7 +68,7 @@ struct hisi_zip_sqe { u32 rsvd1[4]; }; -struct hisi_zip *find_zip_device(int node); +int zip_create_qps(struct hisi_qp **qps, int ctx_num); int hisi_zip_register_to_crypto(void); void hisi_zip_unregister_from_crypto(void); #endif diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index 9815d5e3ccd0..369ec3220574 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -132,29 +132,25 @@ static void hisi_zip_fill_sqe(struct hisi_zip_sqe *sqe, u8 req_type, sqe->dest_addr_h = upper_32_bits(d_addr); } -static int hisi_zip_create_qp(struct hisi_qm *qm, struct hisi_zip_qp_ctx *ctx, - int alg_type, int req_type) +static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *ctx, + int alg_type, int req_type) { - struct hisi_qp *qp; + struct device *dev = &qp->qm->pdev->dev; int ret; - qp = hisi_qm_create_qp(qm, alg_type); - if (IS_ERR(qp)) - return PTR_ERR(qp); - qp->req_type = req_type; + qp->alg_type = alg_type; qp->qp_ctx = ctx; - ctx->qp = qp; ret = hisi_qm_start_qp(qp, 0); - if (ret < 0) - goto err_release_qp; + if (ret < 0) { + dev_err(dev, "start qp failed!\n"); + return ret; + } - return 0; + ctx->qp = qp; -err_release_qp: - hisi_qm_release_qp(qp); - return ret; + return 0; } static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *ctx) @@ -165,34 +161,34 @@ static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *ctx) static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type) { + struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; struct hisi_zip *hisi_zip; - struct hisi_qm *qm; int ret, i, j; - /* find the proper zip device */ - hisi_zip = find_zip_device(cpu_to_node(smp_processor_id())); - if (!hisi_zip) { - pr_err("Failed to find a proper ZIP device!\n"); + ret = zip_create_qps(qps, HZIP_CTX_Q_NUM); + if (ret) { + pr_err("Can not create zip qps!\n"); return -ENODEV; } - qm = &hisi_zip->qm; + + hisi_zip = container_of(qps[0]->qm, struct hisi_zip, qm); for (i = 0; i < HZIP_CTX_Q_NUM; i++) { /* alg_type = 0 for compress, 1 for decompress in hw sqe */ - ret = hisi_zip_create_qp(qm, &hisi_zip_ctx->qp_ctx[i], i, - req_type); - if (ret) - goto err; + ret = hisi_zip_start_qp(qps[i], &hisi_zip_ctx->qp_ctx[i], i, + req_type); + if (ret) { + for (j = i - 1; j >= 0; j--) + hisi_qm_stop_qp(hisi_zip_ctx->qp_ctx[j].qp); + + hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM); + return ret; + } hisi_zip_ctx->qp_ctx[i].zip_dev = hisi_zip; } return 0; -err: - for (j = i - 1; j >= 0; j--) - hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[j]); - - return ret; } static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx) diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index e1bab1a91333..fcc85d2dbd07 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -11,6 +11,7 @@ #include <linux/pci.h> #include <linux/seq_file.h> #include <linux/topology.h> +#include <linux/uacce.h> #include "zip.h" #define PCI_DEVICE_ID_ZIP_PF 0xa250 @@ -60,13 +61,17 @@ #define HZIP_CORE_DEBUG_DECOMP_5 0x309000 #define HZIP_CORE_INT_SOURCE 0x3010A0 -#define HZIP_CORE_INT_MASK 0x3010A4 +#define HZIP_CORE_INT_MASK_REG 0x3010A4 #define HZIP_CORE_INT_STATUS 0x3010AC #define HZIP_CORE_INT_STATUS_M_ECC BIT(1) #define HZIP_CORE_SRAM_ECC_ERR_INFO 0x301148 -#define SRAM_ECC_ERR_NUM_SHIFT 16 -#define SRAM_ECC_ERR_ADDR_SHIFT 24 -#define HZIP_CORE_INT_DISABLE 0x000007FF +#define HZIP_CORE_INT_RAS_CE_ENB 0x301160 +#define HZIP_CORE_INT_RAS_NFE_ENB 0x301164 +#define HZIP_CORE_INT_RAS_FE_ENB 0x301168 +#define HZIP_CORE_INT_RAS_NFE_ENABLE 0x7FE +#define HZIP_SRAM_ECC_ERR_NUM_SHIFT 16 +#define HZIP_SRAM_ECC_ERR_ADDR_SHIFT 24 +#define HZIP_CORE_INT_MASK_ALL GENMASK(10, 0) #define HZIP_COMP_CORE_NUM 2 #define HZIP_DECOMP_CORE_NUM 6 #define HZIP_CORE_NUM (HZIP_COMP_CORE_NUM + \ @@ -83,77 +88,7 @@ static const char hisi_zip_name[] = "hisi_zip"; static struct dentry *hzip_debugfs_root; -static LIST_HEAD(hisi_zip_list); -static DEFINE_MUTEX(hisi_zip_list_lock); - -struct hisi_zip_resource { - struct hisi_zip *hzip; - int distance; - struct list_head list; -}; - -static void free_list(struct list_head *head) -{ - struct hisi_zip_resource *res, *tmp; - - list_for_each_entry_safe(res, tmp, head, list) { - list_del(&res->list); - kfree(res); - } -} - -struct hisi_zip *find_zip_device(int node) -{ - struct hisi_zip_resource *res, *tmp; - struct hisi_zip *ret = NULL; - struct hisi_zip *hisi_zip; - struct list_head *n; - struct device *dev; - LIST_HEAD(head); - - mutex_lock(&hisi_zip_list_lock); - - if (IS_ENABLED(CONFIG_NUMA)) { - list_for_each_entry(hisi_zip, &hisi_zip_list, list) { - res = kzalloc(sizeof(*res), GFP_KERNEL); - if (!res) - goto err; - - dev = &hisi_zip->qm.pdev->dev; - res->hzip = hisi_zip; - res->distance = node_distance(dev_to_node(dev), node); - - n = &head; - list_for_each_entry(tmp, &head, list) { - if (res->distance < tmp->distance) { - n = &tmp->list; - break; - } - } - list_add_tail(&res->list, n); - } - - list_for_each_entry(tmp, &head, list) { - if (hisi_qm_get_free_qp_num(&tmp->hzip->qm)) { - ret = tmp->hzip; - break; - } - } - - free_list(&head); - } else { - ret = list_first_entry(&hisi_zip_list, struct hisi_zip, list); - } - - mutex_unlock(&hisi_zip_list_lock); - - return ret; - -err: - free_list(&head); - mutex_unlock(&hisi_zip_list_lock); - return NULL; -} +static struct hisi_qm_list zip_devices; struct hisi_zip_hw_error { u32 int_msk; @@ -297,9 +232,6 @@ static u32 pf_q_num = HZIP_PF_DEF_Q_NUM; module_param_cb(pf_q_num, &pf_q_num_ops, &pf_q_num, 0444); MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 1-4096, v2 1-1024)"); -static int uacce_mode; -module_param(uacce_mode, int, 0); - static u32 vfs_num; module_param(vfs_num, uint, 0444); MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63)"); @@ -311,18 +243,11 @@ static const struct pci_device_id hisi_zip_dev_ids[] = { }; MODULE_DEVICE_TABLE(pci, hisi_zip_dev_ids); -static inline void hisi_zip_add_to_list(struct hisi_zip *hisi_zip) +int zip_create_qps(struct hisi_qp **qps, int qp_num) { - mutex_lock(&hisi_zip_list_lock); - list_add_tail(&hisi_zip->list, &hisi_zip_list); - mutex_unlock(&hisi_zip_list_lock); -} + int node = cpu_to_node(smp_processor_id()); -static inline void hisi_zip_remove_from_list(struct hisi_zip *hisi_zip) -{ - mutex_lock(&hisi_zip_list_lock); - list_del(&hisi_zip->list); - mutex_unlock(&hisi_zip_list_lock); + return hisi_qm_alloc_qps_node(&zip_devices, qp_num, 0, node, qps); } static void hisi_zip_set_user_domain_and_cache(struct hisi_zip *hisi_zip) @@ -353,8 +278,14 @@ static void hisi_zip_set_user_domain_and_cache(struct hisi_zip *hisi_zip) writel(AXUSER_BASE, base + HZIP_BD_RUSER_32_63); writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63); writel(AXUSER_BASE, base + HZIP_BD_WUSER_32_63); - writel(AXUSER_BASE, base + HZIP_DATA_RUSER_32_63); - writel(AXUSER_BASE, base + HZIP_DATA_WUSER_32_63); + + if (hisi_zip->qm.use_sva) { + writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_RUSER_32_63); + writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_WUSER_32_63); + } else { + writel(AXUSER_BASE, base + HZIP_DATA_RUSER_32_63); + writel(AXUSER_BASE, base + HZIP_DATA_WUSER_32_63); + } /* let's open all compression/decompression cores */ writel(DECOMP_CHECK_ENABLE | ALL_COMP_DECOMP_EN, @@ -366,27 +297,32 @@ static void hisi_zip_set_user_domain_and_cache(struct hisi_zip *hisi_zip) FIELD_PREP(CQC_CACHE_WB_THRD, 1), base + QM_CACHE_CTL); } -static void hisi_zip_hw_error_set_state(struct hisi_zip *hisi_zip, bool state) +static void hisi_zip_hw_error_enable(struct hisi_qm *qm) { - struct hisi_qm *qm = &hisi_zip->qm; - if (qm->ver == QM_HW_V1) { - writel(HZIP_CORE_INT_DISABLE, qm->io_base + HZIP_CORE_INT_MASK); + writel(HZIP_CORE_INT_MASK_ALL, + qm->io_base + HZIP_CORE_INT_MASK_REG); dev_info(&qm->pdev->dev, "Does not support hw error handle\n"); return; } - if (state) { - /* clear ZIP hw error source if having */ - writel(HZIP_CORE_INT_DISABLE, hisi_zip->qm.io_base + - HZIP_CORE_INT_SOURCE); - /* enable ZIP hw error interrupts */ - writel(0, hisi_zip->qm.io_base + HZIP_CORE_INT_MASK); - } else { - /* disable ZIP hw error interrupts */ - writel(HZIP_CORE_INT_DISABLE, - hisi_zip->qm.io_base + HZIP_CORE_INT_MASK); - } + /* clear ZIP hw error source if having */ + writel(HZIP_CORE_INT_MASK_ALL, qm->io_base + HZIP_CORE_INT_SOURCE); + + /* configure error type */ + writel(0x1, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); + writel(0x0, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB); + writel(HZIP_CORE_INT_RAS_NFE_ENABLE, + qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); + + /* enable ZIP hw error interrupts */ + writel(0, qm->io_base + HZIP_CORE_INT_MASK_REG); +} + +static void hisi_zip_hw_error_disable(struct hisi_qm *qm) +{ + /* disable ZIP hw error interrupts */ + writel(HZIP_CORE_INT_MASK_ALL, qm->io_base + HZIP_CORE_INT_MASK_REG); } static inline struct hisi_qm *file_to_qm(struct ctrl_debug_file *file) @@ -638,14 +574,53 @@ static void hisi_zip_debugfs_exit(struct hisi_zip *hisi_zip) hisi_zip_debug_regs_clear(hisi_zip); } -static void hisi_zip_hw_error_init(struct hisi_zip *hisi_zip) +static void hisi_zip_log_hw_error(struct hisi_qm *qm, u32 err_sts) +{ + const struct hisi_zip_hw_error *err = zip_hw_error; + struct device *dev = &qm->pdev->dev; + u32 err_val; + + while (err->msg) { + if (err->int_msk & err_sts) { + dev_err(dev, "%s [error status=0x%x] found\n", + err->msg, err->int_msk); + + if (err->int_msk & HZIP_CORE_INT_STATUS_M_ECC) { + err_val = readl(qm->io_base + + HZIP_CORE_SRAM_ECC_ERR_INFO); + dev_err(dev, "hisi-zip multi ecc sram num=0x%x\n", + ((err_val >> + HZIP_SRAM_ECC_ERR_NUM_SHIFT) & 0xFF)); + dev_err(dev, "hisi-zip multi ecc sram addr=0x%x\n", + (err_val >> + HZIP_SRAM_ECC_ERR_ADDR_SHIFT)); + } + } + err++; + } + + writel(err_sts, qm->io_base + HZIP_CORE_INT_SOURCE); +} + +static u32 hisi_zip_get_hw_err_status(struct hisi_qm *qm) { - hisi_qm_hw_error_init(&hisi_zip->qm, QM_BASE_CE, - QM_BASE_NFE | QM_ACC_WB_NOT_READY_TIMEOUT, 0, - QM_DB_RANDOM_INVALID); - hisi_zip_hw_error_set_state(hisi_zip, true); + return readl(qm->io_base + HZIP_CORE_INT_STATUS); } +static const struct hisi_qm_err_ini hisi_zip_err_ini = { + .hw_err_enable = hisi_zip_hw_error_enable, + .hw_err_disable = hisi_zip_hw_error_disable, + .get_dev_hw_err_status = hisi_zip_get_hw_err_status, + .log_dev_hw_err = hisi_zip_log_hw_error, + .err_info = { + .ce = QM_BASE_CE, + .nfe = QM_BASE_NFE | + QM_ACC_WB_NOT_READY_TIMEOUT, + .fe = 0, + .msi = QM_DB_RANDOM_INVALID, + } +}; + static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) { struct hisi_qm *qm = &hisi_zip->qm; @@ -671,8 +646,10 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) return -EINVAL; } + qm->err_ini = &hisi_zip_err_ini; + hisi_zip_set_user_domain_and_cache(hisi_zip); - hisi_zip_hw_error_init(hisi_zip); + hisi_qm_dev_err_init(qm); hisi_zip_debug_regs_clear(hisi_zip); return 0; @@ -791,27 +768,15 @@ static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id) pci_set_drvdata(pdev, hisi_zip); qm = &hisi_zip->qm; + qm->use_dma_api = true; qm->pdev = pdev; qm->ver = rev_id; + qm->algs = "zlib\ngzip"; qm->sqe_size = HZIP_SQE_SIZE; qm->dev_name = hisi_zip_name; qm->fun_type = (pdev->device == PCI_DEVICE_ID_ZIP_PF) ? QM_HW_PF : QM_HW_VF; - switch (uacce_mode) { - case 0: - qm->use_dma_api = true; - break; - case 1: - qm->use_dma_api = false; - break; - case 2: - qm->use_dma_api = true; - break; - default: - return -EINVAL; - } - ret = hisi_qm_init(qm); if (ret) { dev_err(&pdev->dev, "Failed to init qm!\n"); @@ -849,7 +814,13 @@ static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) dev_err(&pdev->dev, "Failed to init debugfs (%d)!\n", ret); - hisi_zip_add_to_list(hisi_zip); + hisi_qm_add_to_list(qm, &zip_devices); + + if (qm->uacce) { + ret = uacce_register(qm->uacce); + if (ret) + goto err_qm_uninit; + } if (qm->fun_type == QM_HW_PF && vfs_num > 0) { ret = hisi_zip_sriov_enable(pdev, vfs_num); @@ -860,7 +831,7 @@ static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id) return 0; err_remove_from_list: - hisi_zip_remove_from_list(hisi_zip); + hisi_qm_del_from_list(qm, &zip_devices); hisi_zip_debugfs_exit(hisi_zip); hisi_qm_stop(qm); err_qm_uninit: @@ -887,92 +858,13 @@ static void hisi_zip_remove(struct pci_dev *pdev) hisi_zip_debugfs_exit(hisi_zip); hisi_qm_stop(qm); - if (qm->fun_type == QM_HW_PF) - hisi_zip_hw_error_set_state(hisi_zip, false); - + hisi_qm_dev_err_uninit(qm); hisi_qm_uninit(qm); - hisi_zip_remove_from_list(hisi_zip); -} - -static void hisi_zip_log_hw_error(struct hisi_zip *hisi_zip, u32 err_sts) -{ - const struct hisi_zip_hw_error *err = zip_hw_error; - struct device *dev = &hisi_zip->qm.pdev->dev; - u32 err_val; - - while (err->msg) { - if (err->int_msk & err_sts) { - dev_warn(dev, "%s [error status=0x%x] found\n", - err->msg, err->int_msk); - - if (HZIP_CORE_INT_STATUS_M_ECC & err->int_msk) { - err_val = readl(hisi_zip->qm.io_base + - HZIP_CORE_SRAM_ECC_ERR_INFO); - dev_warn(dev, "hisi-zip multi ecc sram num=0x%x\n", - ((err_val >> SRAM_ECC_ERR_NUM_SHIFT) & - 0xFF)); - dev_warn(dev, "hisi-zip multi ecc sram addr=0x%x\n", - (err_val >> SRAM_ECC_ERR_ADDR_SHIFT)); - } - } - err++; - } -} - -static pci_ers_result_t hisi_zip_hw_error_handle(struct hisi_zip *hisi_zip) -{ - u32 err_sts; - - /* read err sts */ - err_sts = readl(hisi_zip->qm.io_base + HZIP_CORE_INT_STATUS); - - if (err_sts) { - hisi_zip_log_hw_error(hisi_zip, err_sts); - /* clear error interrupts */ - writel(err_sts, hisi_zip->qm.io_base + HZIP_CORE_INT_SOURCE); - - return PCI_ERS_RESULT_NEED_RESET; - } - - return PCI_ERS_RESULT_RECOVERED; -} - -static pci_ers_result_t hisi_zip_process_hw_error(struct pci_dev *pdev) -{ - struct hisi_zip *hisi_zip = pci_get_drvdata(pdev); - struct device *dev = &pdev->dev; - pci_ers_result_t qm_ret, zip_ret; - - if (!hisi_zip) { - dev_err(dev, - "Can't recover ZIP-error occurred during device init\n"); - return PCI_ERS_RESULT_NONE; - } - - qm_ret = hisi_qm_hw_error_handle(&hisi_zip->qm); - - zip_ret = hisi_zip_hw_error_handle(hisi_zip); - - return (qm_ret == PCI_ERS_RESULT_NEED_RESET || - zip_ret == PCI_ERS_RESULT_NEED_RESET) ? - PCI_ERS_RESULT_NEED_RESET : PCI_ERS_RESULT_RECOVERED; -} - -static pci_ers_result_t hisi_zip_error_detected(struct pci_dev *pdev, - pci_channel_state_t state) -{ - if (pdev->is_virtfn) - return PCI_ERS_RESULT_NONE; - - dev_info(&pdev->dev, "PCI error detected, state(=%d)!!\n", state); - if (state == pci_channel_io_perm_failure) - return PCI_ERS_RESULT_DISCONNECT; - - return hisi_zip_process_hw_error(pdev); + hisi_qm_del_from_list(qm, &zip_devices); } static const struct pci_error_handlers hisi_zip_err_handler = { - .error_detected = hisi_zip_error_detected, + .error_detected = hisi_qm_dev_err_detected, }; static struct pci_driver hisi_zip_pci_driver = { @@ -1002,6 +894,7 @@ static int __init hisi_zip_init(void) { int ret; + hisi_qm_init_list(&zip_devices); hisi_zip_register_debugfs(); ret = pci_register_driver(&hisi_zip_pci_driver); @@ -1010,12 +903,10 @@ static int __init hisi_zip_init(void) goto err_pci; } - if (uacce_mode == 0 || uacce_mode == 2) { - ret = hisi_zip_register_to_crypto(); - if (ret < 0) { - pr_err("Failed to register driver to crypto.\n"); - goto err_crypto; - } + ret = hisi_zip_register_to_crypto(); + if (ret < 0) { + pr_err("Failed to register driver to crypto.\n"); + goto err_crypto; } return 0; @@ -1030,8 +921,7 @@ err_pci: static void __exit hisi_zip_exit(void) { - if (uacce_mode == 0 || uacce_mode == 2) - hisi_zip_unregister_from_crypto(); + hisi_zip_unregister_from_crypto(); pci_unregister_driver(&hisi_zip_pci_driver); hisi_zip_unregister_debugfs(); } |