diff options
Diffstat (limited to 'drivers/crypto/hisilicon/sec2/sec_main.c')
-rw-r--r-- | drivers/crypto/hisilicon/sec2/sec_main.c | 379 |
1 files changed, 145 insertions, 234 deletions
diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 1f54ebe164b6..a4cb58b54b25 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -80,6 +80,9 @@ #define SEC_VF_CNT_MASK 0xffffffc0 #define SEC_DBGFS_VAL_MAX_LEN 20 +#define SEC_SQE_MASK_OFFSET 64 +#define SEC_SQE_MASK_LEN 48 + #define SEC_ADDR(qm, offset) ((qm)->io_base + (offset) + \ SEC_ENGINE_PF_CFG_OFF + SEC_ACC_COMMON_REG_OFF) @@ -88,6 +91,11 @@ struct sec_hw_error { const char *msg; }; +struct sec_dfx_item { + const char *name; + u32 offset; +}; + static const char sec_name[] = "hisi_sec2"; static struct dentry *sec_debugfs_root; static struct hisi_qm_list sec_devices; @@ -110,7 +118,16 @@ static const char * const sec_dbg_file_name[] = { [SEC_CLEAR_ENABLE] = "clear_enable", }; -static struct debugfs_reg32 sec_dfx_regs[] = { +static struct sec_dfx_item sec_dfx_labels[] = { + {"send_cnt", offsetof(struct sec_dfx, send_cnt)}, + {"recv_cnt", offsetof(struct sec_dfx, recv_cnt)}, + {"send_busy_cnt", offsetof(struct sec_dfx, send_busy_cnt)}, + {"err_bd_cnt", offsetof(struct sec_dfx, err_bd_cnt)}, + {"invalid_req_cnt", offsetof(struct sec_dfx, invalid_req_cnt)}, + {"done_flag_cnt", offsetof(struct sec_dfx, done_flag_cnt)}, +}; + +static const struct debugfs_reg32 sec_dfx_regs[] = { {"SEC_PF_ABNORMAL_INT_SOURCE ", 0x301010}, {"SEC_SAA_EN ", 0x301270}, {"SEC_BD_LATENCY_MIN ", 0x301600}, @@ -136,45 +153,14 @@ static struct debugfs_reg32 sec_dfx_regs[] = { static int sec_pf_q_num_set(const char *val, const struct kernel_param *kp) { - struct pci_dev *pdev; - u32 n, q_num; - u8 rev_id; - int ret; - - if (!val) - return -EINVAL; - - pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI, - SEC_PF_PCI_DEVICE_ID, NULL); - if (!pdev) { - q_num = min_t(u32, SEC_QUEUE_NUM_V1, SEC_QUEUE_NUM_V2); - pr_info("No device, suppose queue number is %d!\n", q_num); - } else { - rev_id = pdev->revision; - - switch (rev_id) { - case QM_HW_V1: - q_num = SEC_QUEUE_NUM_V1; - break; - case QM_HW_V2: - q_num = SEC_QUEUE_NUM_V2; - break; - default: - return -EINVAL; - } - } - - ret = kstrtou32(val, 10, &n); - if (ret || !n || n > q_num) - return -EINVAL; - - return param_set_int(val, kp); + return q_num_set(val, kp, SEC_PF_PCI_DEVICE_ID); } static const struct kernel_param_ops sec_pf_q_num_ops = { .set = sec_pf_q_num_set, .get = param_get_int, }; + static u32 pf_q_num = SEC_PF_DEF_Q_NUM; module_param_cb(pf_q_num, &sec_pf_q_num_ops, &pf_q_num, 0444); MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 0-4096, v2 0-1024)"); @@ -207,6 +193,15 @@ static u32 ctx_q_num = SEC_CTX_Q_NUM_DEF; module_param_cb(ctx_q_num, &sec_ctx_q_num_ops, &ctx_q_num, 0444); MODULE_PARM_DESC(ctx_q_num, "Queue num in ctx (24 default, 2, 4, ..., 32)"); +static const struct kernel_param_ops vfs_num_ops = { + .set = vfs_num_set, + .get = param_get_int, +}; + +static u32 vfs_num; +module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444); +MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)"); + void sec_destroy_qps(struct hisi_qp **qps, int qp_num) { hisi_qm_free_qps(qps, qp_num); @@ -240,9 +235,8 @@ static const struct pci_device_id sec_dev_ids[] = { }; MODULE_DEVICE_TABLE(pci, sec_dev_ids); -static u8 sec_get_endian(struct sec_dev *sec) +static u8 sec_get_endian(struct hisi_qm *qm) { - struct hisi_qm *qm = &sec->qm; u32 reg; /* @@ -270,9 +264,8 @@ static u8 sec_get_endian(struct sec_dev *sec) return SEC_64BE; } -static int sec_engine_init(struct sec_dev *sec) +static int sec_engine_init(struct hisi_qm *qm) { - struct hisi_qm *qm = &sec->qm; int ret; u32 reg; @@ -315,7 +308,7 @@ static int sec_engine_init(struct sec_dev *sec) /* config endian */ reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG)); - reg |= sec_get_endian(sec); + reg |= sec_get_endian(qm); writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG)); /* Enable sm4 xts mode multiple iv */ @@ -325,10 +318,8 @@ static int sec_engine_init(struct sec_dev *sec) return 0; } -static int sec_set_user_domain_and_cache(struct sec_dev *sec) +static int sec_set_user_domain_and_cache(struct hisi_qm *qm) { - struct hisi_qm *qm = &sec->qm; - /* qm user domain */ writel(AXUSER_BASE, qm->io_base + QM_ARUSER_M_CFG_1); writel(ARUSER_M_CFG_ENABLE, qm->io_base + QM_ARUSER_M_CFG_ENABLE); @@ -349,7 +340,7 @@ static int sec_set_user_domain_and_cache(struct sec_dev *sec) CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) | FIELD_PREP(CQC_CACHE_WB_THRD, 1), qm->io_base + QM_CACHE_CTL); - return sec_engine_init(sec); + return sec_engine_init(qm); } /* sec_debug_regs_clear() - clear the sec debug regs */ @@ -424,23 +415,22 @@ static u32 sec_current_qm_read(struct sec_debug_file *file) static int sec_current_qm_write(struct sec_debug_file *file, u32 val) { struct hisi_qm *qm = file->qm; - struct sec_dev *sec = container_of(qm, struct sec_dev, qm); u32 vfq_num; u32 tmp; - if (val > sec->num_vfs) + if (val > qm->vfs_num) return -EINVAL; /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */ if (!val) { qm->debug.curr_qm_qp_num = qm->qp_num; } else { - vfq_num = (qm->ctrl_qp_num - qm->qp_num) / sec->num_vfs; + vfq_num = (qm->ctrl_qp_num - qm->qp_num) / qm->vfs_num; - if (val == sec->num_vfs) + if (val == qm->vfs_num) qm->debug.curr_qm_qp_num = qm->ctrl_qp_num - qm->qp_num - - (sec->num_vfs - 1) * vfq_num; + (qm->vfs_num - 1) * vfq_num; else qm->debug.curr_qm_qp_num = vfq_num; } @@ -570,10 +560,22 @@ static const struct file_operations sec_dbg_fops = { static int sec_debugfs_atomic64_get(void *data, u64 *val) { *val = atomic64_read((atomic64_t *)data); + + return 0; +} + +static int sec_debugfs_atomic64_set(void *data, u64 val) +{ + if (val) + return -EINVAL; + + atomic64_set((atomic64_t *)data, 0); + return 0; } + DEFINE_DEBUGFS_ATTRIBUTE(sec_atomic64_ops, sec_debugfs_atomic64_get, - NULL, "%lld\n"); + sec_debugfs_atomic64_set, "%lld\n"); static int sec_core_debug_init(struct sec_dev *sec) { @@ -582,6 +584,7 @@ static int sec_core_debug_init(struct sec_dev *sec) struct sec_dfx *dfx = &sec->debug.dfx; struct debugfs_regset32 *regset; struct dentry *tmp_d; + int i; tmp_d = debugfs_create_dir("sec_dfx", sec->qm.debug.debug_root); @@ -593,13 +596,15 @@ static int sec_core_debug_init(struct sec_dev *sec) regset->nregs = ARRAY_SIZE(sec_dfx_regs); regset->base = qm->io_base; - debugfs_create_regset32("regs", 0444, tmp_d, regset); - - debugfs_create_file("send_cnt", 0444, tmp_d, - &dfx->send_cnt, &sec_atomic64_ops); + if (qm->pdev->device == SEC_PF_PCI_DEVICE_ID) + debugfs_create_regset32("regs", 0444, tmp_d, regset); - debugfs_create_file("recv_cnt", 0444, tmp_d, - &dfx->recv_cnt, &sec_atomic64_ops); + for (i = 0; i < ARRAY_SIZE(sec_dfx_labels); i++) { + atomic64_t *data = (atomic64_t *)((uintptr_t)dfx + + sec_dfx_labels[i].offset); + debugfs_create_file(sec_dfx_labels[i].name, 0644, + tmp_d, data, &sec_atomic64_ops); + } return 0; } @@ -630,6 +635,9 @@ static int sec_debugfs_init(struct sec_dev *sec) qm->debug.debug_root = debugfs_create_dir(dev_name(dev), sec_debugfs_root); + + qm->debug.sqe_mask_offset = SEC_SQE_MASK_OFFSET; + qm->debug.sqe_mask_len = SEC_SQE_MASK_LEN; ret = hisi_qm_debug_init(qm); if (ret) goto failed_to_create; @@ -675,8 +683,6 @@ static void sec_log_hw_error(struct hisi_qm *qm, u32 err_sts) } errs++; } - - writel(err_sts, qm->io_base + SEC_CORE_INT_SOURCE); } static u32 sec_get_hw_err_status(struct hisi_qm *qm) @@ -684,17 +690,36 @@ static u32 sec_get_hw_err_status(struct hisi_qm *qm) return readl(qm->io_base + SEC_CORE_INT_STATUS); } +static void sec_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) +{ + writel(err_sts, qm->io_base + SEC_CORE_INT_SOURCE); +} + +static void sec_open_axi_master_ooo(struct hisi_qm *qm) +{ + u32 val; + + val = readl(SEC_ADDR(qm, SEC_CONTROL_REG)); + writel(val & SEC_AXI_SHUTDOWN_DISABLE, SEC_ADDR(qm, SEC_CONTROL_REG)); + writel(val | SEC_AXI_SHUTDOWN_ENABLE, SEC_ADDR(qm, SEC_CONTROL_REG)); +} + static const struct hisi_qm_err_ini sec_err_ini = { + .hw_init = sec_set_user_domain_and_cache, .hw_err_enable = sec_hw_error_enable, .hw_err_disable = sec_hw_error_disable, .get_dev_hw_err_status = sec_get_hw_err_status, + .clear_dev_hw_err_status = sec_clear_hw_err_status, .log_dev_hw_err = sec_log_hw_error, + .open_axi_master_ooo = sec_open_axi_master_ooo, .err_info = { .ce = QM_BASE_CE, .nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT | QM_ACC_WB_NOT_READY_TIMEOUT, .fe = 0, - .msi = QM_DB_RANDOM_INVALID, + .ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC, + .msi_wr_port = BIT(0), + .acpi_rst = "SRST", } }; @@ -703,22 +728,14 @@ static int sec_pf_probe_init(struct sec_dev *sec) struct hisi_qm *qm = &sec->qm; int ret; - switch (qm->ver) { - case QM_HW_V1: + if (qm->ver == QM_HW_V1) qm->ctrl_qp_num = SEC_QUEUE_NUM_V1; - break; - - case QM_HW_V2: + else qm->ctrl_qp_num = SEC_QUEUE_NUM_V2; - break; - - default: - return -EINVAL; - } qm->err_ini = &sec_err_ini; - ret = sec_set_user_domain_and_cache(sec); + ret = sec_set_user_domain_and_cache(qm); if (ret) return ret; @@ -730,32 +747,30 @@ static int sec_pf_probe_init(struct sec_dev *sec) static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { - enum qm_hw_ver rev_id; - - rev_id = hisi_qm_get_hw_version(pdev); - if (rev_id == QM_HW_UNKNOWN) - return -ENODEV; + int ret; qm->pdev = pdev; - qm->ver = rev_id; - + qm->ver = pdev->revision; qm->sqe_size = SEC_SQE_SIZE; qm->dev_name = sec_name; + qm->fun_type = (pdev->device == SEC_PF_PCI_DEVICE_ID) ? QM_HW_PF : QM_HW_VF; - qm->use_dma_api = true; - - return hisi_qm_init(qm); -} - -static void sec_qm_uninit(struct hisi_qm *qm) -{ - hisi_qm_uninit(qm); -} - -static int sec_probe_init(struct hisi_qm *qm, struct sec_dev *sec) -{ - int ret; + if (qm->fun_type == QM_HW_PF) { + qm->qp_base = SEC_PF_DEF_Q_BASE; + qm->qp_num = pf_q_num; + qm->debug.curr_qm_qp_num = pf_q_num; + qm->qm_list = &sec_devices; + } else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) { + /* + * have no way to get qm configure in VM in v1 hardware, + * so currently force PF to uses SEC_PF_DEF_Q_NUM, and force + * to trigger only one VF in v1 hardware. + * v2 hardware has no such problem. + */ + qm->qp_base = SEC_PF_DEF_Q_NUM; + qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM; + } /* * WQ_HIGHPRI: SEC request must be low delayed, @@ -763,47 +778,38 @@ static int sec_probe_init(struct hisi_qm *qm, struct sec_dev *sec) * WQ_UNBOUND: SEC task is likely with long * running CPU intensive workloads. */ - qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | - WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus(), - pci_name(qm->pdev)); + qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM | + WQ_UNBOUND, num_online_cpus(), + pci_name(qm->pdev)); if (!qm->wq) { pci_err(qm->pdev, "fail to alloc workqueue\n"); return -ENOMEM; } - if (qm->fun_type == QM_HW_PF) { - qm->qp_base = SEC_PF_DEF_Q_BASE; - qm->qp_num = pf_q_num; - qm->debug.curr_qm_qp_num = pf_q_num; + ret = hisi_qm_init(qm); + if (ret) + destroy_workqueue(qm->wq); + + return ret; +} +static void sec_qm_uninit(struct hisi_qm *qm) +{ + hisi_qm_uninit(qm); +} + +static int sec_probe_init(struct sec_dev *sec) +{ + struct hisi_qm *qm = &sec->qm; + int ret; + + if (qm->fun_type == QM_HW_PF) { ret = sec_pf_probe_init(sec); if (ret) - goto err_probe_uninit; - } else if (qm->fun_type == QM_HW_VF) { - /* - * have no way to get qm configure in VM in v1 hardware, - * so currently force PF to uses SEC_PF_DEF_Q_NUM, and force - * to trigger only one VF in v1 hardware. - * v2 hardware has no such problem. - */ - if (qm->ver == QM_HW_V1) { - qm->qp_base = SEC_PF_DEF_Q_NUM; - qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM; - } else if (qm->ver == QM_HW_V2) { - /* v2 starts to support get vft by mailbox */ - ret = hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num); - if (ret) - goto err_probe_uninit; - } - } else { - ret = -ENODEV; - goto err_probe_uninit; + return ret; } return 0; -err_probe_uninit: - destroy_workqueue(qm->wq); - return ret; } static void sec_probe_uninit(struct hisi_qm *qm) @@ -840,20 +846,17 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (!sec) return -ENOMEM; - pci_set_drvdata(pdev, sec); - - sec->ctx_q_num = ctx_q_num; - sec_iommu_used_check(sec); - qm = &sec->qm; - ret = sec_qm_init(qm, pdev); if (ret) { - pci_err(pdev, "Failed to pre init qm!\n"); + pci_err(pdev, "Failed to init SEC QM (%d)!\n", ret); return ret; } - ret = sec_probe_init(qm, sec); + sec->ctx_q_num = ctx_q_num; + sec_iommu_used_check(sec); + + ret = sec_probe_init(sec); if (ret) { pci_err(pdev, "Failed to probe!\n"); goto err_qm_uninit; @@ -877,8 +880,17 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_remove_from_list; } + if (qm->fun_type == QM_HW_PF && vfs_num) { + ret = hisi_qm_sriov_enable(pdev, vfs_num); + if (ret < 0) + goto err_crypto_unregister; + } + return 0; +err_crypto_unregister: + sec_unregister_from_crypto(); + err_remove_from_list: hisi_qm_del_from_list(qm, &sec_devices); sec_debugfs_exit(sec); @@ -893,110 +905,6 @@ err_qm_uninit: return ret; } -/* now we only support equal assignment */ -static int sec_vf_q_assign(struct sec_dev *sec, u32 num_vfs) -{ - struct hisi_qm *qm = &sec->qm; - u32 qp_num = qm->qp_num; - u32 q_base = qp_num; - u32 q_num, remain_q_num; - int i, j, ret; - - if (!num_vfs) - return -EINVAL; - - remain_q_num = qm->ctrl_qp_num - qp_num; - q_num = remain_q_num / num_vfs; - - for (i = 1; i <= num_vfs; i++) { - if (i == num_vfs) - q_num += remain_q_num % num_vfs; - ret = hisi_qm_set_vft(qm, i, q_base, q_num); - if (ret) { - for (j = i; j > 0; j--) - hisi_qm_set_vft(qm, j, 0, 0); - return ret; - } - q_base += q_num; - } - - return 0; -} - -static int sec_clear_vft_config(struct sec_dev *sec) -{ - struct hisi_qm *qm = &sec->qm; - u32 num_vfs = sec->num_vfs; - int ret; - u32 i; - - for (i = 1; i <= num_vfs; i++) { - ret = hisi_qm_set_vft(qm, i, 0, 0); - if (ret) - return ret; - } - - sec->num_vfs = 0; - - return 0; -} - -static int sec_sriov_enable(struct pci_dev *pdev, int max_vfs) -{ - struct sec_dev *sec = pci_get_drvdata(pdev); - int pre_existing_vfs, ret; - u32 num_vfs; - - pre_existing_vfs = pci_num_vf(pdev); - - if (pre_existing_vfs) { - pci_err(pdev, "Can't enable VF. Please disable at first!\n"); - return 0; - } - - num_vfs = min_t(u32, max_vfs, SEC_VF_NUM); - - ret = sec_vf_q_assign(sec, num_vfs); - if (ret) { - pci_err(pdev, "Can't assign queues for VF!\n"); - return ret; - } - - sec->num_vfs = num_vfs; - - ret = pci_enable_sriov(pdev, num_vfs); - if (ret) { - pci_err(pdev, "Can't enable VF!\n"); - sec_clear_vft_config(sec); - return ret; - } - - return num_vfs; -} - -static int sec_sriov_disable(struct pci_dev *pdev) -{ - struct sec_dev *sec = pci_get_drvdata(pdev); - - if (pci_vfs_assigned(pdev)) { - pci_err(pdev, "Can't disable VFs while VFs are assigned!\n"); - return -EPERM; - } - - /* remove in sec_pci_driver will be called to free VF resources */ - pci_disable_sriov(pdev); - - return sec_clear_vft_config(sec); -} - -static int sec_sriov_configure(struct pci_dev *pdev, int num_vfs) -{ - if (num_vfs) - return sec_sriov_enable(pdev, num_vfs); - else - return sec_sriov_disable(pdev); -} - static void sec_remove(struct pci_dev *pdev) { struct sec_dev *sec = pci_get_drvdata(pdev); @@ -1006,8 +914,8 @@ static void sec_remove(struct pci_dev *pdev) hisi_qm_del_from_list(qm, &sec_devices); - if (qm->fun_type == QM_HW_PF && sec->num_vfs) - (void)sec_sriov_disable(pdev); + if (qm->fun_type == QM_HW_PF && qm->vfs_num) + hisi_qm_sriov_disable(pdev); sec_debugfs_exit(sec); @@ -1023,6 +931,9 @@ static void sec_remove(struct pci_dev *pdev) static const struct pci_error_handlers sec_err_handler = { .error_detected = hisi_qm_dev_err_detected, + .slot_reset = hisi_qm_dev_slot_reset, + .reset_prepare = hisi_qm_reset_prepare, + .reset_done = hisi_qm_reset_done, }; static struct pci_driver sec_pci_driver = { @@ -1031,7 +942,7 @@ static struct pci_driver sec_pci_driver = { .probe = sec_probe, .remove = sec_remove, .err_handler = &sec_err_handler, - .sriov_configure = sec_sriov_configure, + .sriov_configure = hisi_qm_sriov_configure, }; static void sec_register_debugfs(void) |