diff options
Diffstat (limited to 'drivers/crypto')
37 files changed, 1292 insertions, 3229 deletions
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 0a9cdd31cbd9..19ab145f912e 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -200,23 +200,6 @@ config S390_PRNG It is available as of z9. -config CRYPTO_DEV_NIAGARA2 - tristate "Niagara2 Stream Processing Unit driver" - select CRYPTO_LIB_DES - select CRYPTO_SKCIPHER - select CRYPTO_HASH - select CRYPTO_MD5 - select CRYPTO_SHA1 - select CRYPTO_SHA256 - depends on SPARC64 - help - Each core of a Niagara2 processor contains a Stream - Processing Unit, which itself contains several cryptographic - sub-units. One set provides the Modular Arithmetic Unit, - used for SSL offload. The other set provides the Cipher - Group, which can perform encryption, decryption, hashing, - checksumming, and raw copies. - config CRYPTO_DEV_SL3516 tristate "Storlink SL3516 crypto offloader" depends on ARCH_GEMINI || COMPILE_TEST diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index ad4ccef67d12..fef18ffdb128 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -21,8 +21,6 @@ obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o obj-$(CONFIG_CRYPTO_DEV_IMGTEC_HASH) += img-hash.o obj-$(CONFIG_CRYPTO_DEV_MARVELL) += marvell/ obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o -obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o -n2_crypto-y := n2_core.o n2_asm.o obj-$(CONFIG_CRYPTO_DEV_NX) += nx/ obj-$(CONFIG_CRYPTO_DEV_OMAP) += omap-crypto.o obj-$(CONFIG_CRYPTO_DEV_OMAP_AES) += omap-aes-driver.o diff --git a/drivers/crypto/bcm/spu.c b/drivers/crypto/bcm/spu.c index 6283e8c6d51d..86c227caa722 100644 --- a/drivers/crypto/bcm/spu.c +++ b/drivers/crypto/bcm/spu.c @@ -836,7 +836,6 @@ u16 spum_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms) u32 cipher_bits = 0; u32 ecf_bits = 0; u8 sctx_words = 0; - u8 *ptr = spu_hdr; flow_log("%s()\n", __func__); flow_log(" cipher alg:%u mode:%u type %u\n", cipher_parms->alg, @@ -847,7 +846,6 @@ u16 spum_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms) /* starting out: zero the header (plus some) */ memset(spu_hdr, 0, sizeof(struct SPUHEADER)); - ptr += sizeof(struct SPUHEADER); /* format master header word */ /* Do not set the next bit even though the datasheet says to */ @@ -861,10 +859,8 @@ u16 spum_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms) /* copy the encryption keys in the SAD entry */ if (cipher_parms->alg) { - if (cipher_parms->key_len) { - ptr += cipher_parms->key_len; + if (cipher_parms->key_len) sctx_words += cipher_parms->key_len / 4; - } /* * if encrypting then set IV size, use SCTX IV unless no IV @@ -873,7 +869,6 @@ u16 spum_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms) if (cipher_parms->iv_len) { /* Use SCTX IV */ ecf_bits |= SCTX_IV; - ptr += cipher_parms->iv_len; sctx_words += cipher_parms->iv_len / 4; } } diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index 7701d00bcb3a..b6e7c0b29d4e 100644 --- a/drivers/crypto/caam/qi.c +++ b/drivers/crypto/caam/qi.c @@ -122,12 +122,12 @@ int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req) qm_fd_addr_set64(&fd, addr); do { + refcount_inc(&req->drv_ctx->refcnt); ret = qman_enqueue(req->drv_ctx->req_fq, &fd); - if (likely(!ret)) { - refcount_inc(&req->drv_ctx->refcnt); + if (likely(!ret)) return 0; - } + refcount_dec(&req->drv_ctx->refcnt); if (ret != -EBUSY) break; num_retries++; diff --git a/drivers/crypto/ccp/dbc.c b/drivers/crypto/ccp/dbc.c index 5b105a23f699..410084a9039c 100644 --- a/drivers/crypto/ccp/dbc.c +++ b/drivers/crypto/ccp/dbc.c @@ -7,6 +7,8 @@ * Author: Mario Limonciello <mario.limonciello@amd.com> */ +#include <linux/mutex.h> + #include "dbc.h" #define DBC_DEFAULT_TIMEOUT (10 * MSEC_PER_SEC) @@ -137,64 +139,49 @@ static long dbc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) return -ENODEV; dbc_dev = psp_master->dbc_data; - mutex_lock(&dbc_dev->ioctl_mutex); + guard(mutex)(&dbc_dev->ioctl_mutex); switch (cmd) { case DBCIOCNONCE: - if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_nonce))) { - ret = -EFAULT; - goto unlock; - } + if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_nonce))) + return -EFAULT; ret = send_dbc_nonce(dbc_dev); if (ret) - goto unlock; + return ret; - if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_nonce))) { - ret = -EFAULT; - goto unlock; - } + if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_nonce))) + return -EFAULT; break; case DBCIOCUID: - if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_setuid))) { - ret = -EFAULT; - goto unlock; - } + if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_setuid))) + return -EFAULT; *dbc_dev->payload_size = dbc_dev->header_size + sizeof(struct dbc_user_setuid); ret = send_dbc_cmd(dbc_dev, PSP_DYNAMIC_BOOST_SET_UID); if (ret) - goto unlock; + return ret; - if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_setuid))) { - ret = -EFAULT; - goto unlock; - } + if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_setuid))) + return -EFAULT; break; case DBCIOCPARAM: - if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_param))) { - ret = -EFAULT; - goto unlock; - } + if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_param))) + return -EFAULT; *dbc_dev->payload_size = dbc_dev->header_size + sizeof(struct dbc_user_param); ret = send_dbc_parameter(dbc_dev); if (ret) - goto unlock; + return ret; - if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_param))) { - ret = -EFAULT; - goto unlock; - } + if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_param))) + return -EFAULT; break; default: - ret = -EINVAL; - + return -EINVAL; } -unlock: - mutex_unlock(&dbc_dev->ioctl_mutex); - return ret; + return 0; } static const struct file_operations dbc_fops = { diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index af018afd9cd7..2e87ca0e292a 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -249,7 +249,7 @@ static struct file *open_file_as_root(const char *filename, int flags, umode_t m fp = file_open_root(&root, filename, flags, mode); path_put(&root); - revert_creds(old_cred); + put_cred(revert_creds(old_cred)); return fp; } diff --git a/drivers/crypto/ccp/sp-dev.c b/drivers/crypto/ccp/sp-dev.c index 7eb3e4668286..3467f6db4f50 100644 --- a/drivers/crypto/ccp/sp-dev.c +++ b/drivers/crypto/ccp/sp-dev.c @@ -19,6 +19,7 @@ #include <linux/types.h> #include <linux/ccp.h> +#include "sev-dev.h" #include "ccp-dev.h" #include "sp-dev.h" @@ -253,8 +254,12 @@ unlock: static int __init sp_mod_init(void) { #ifdef CONFIG_X86 + static bool initialized; int ret; + if (initialized) + return 0; + ret = sp_pci_init(); if (ret) return ret; @@ -263,6 +268,8 @@ static int __init sp_mod_init(void) psp_pci_init(); #endif + initialized = true; + return 0; #endif @@ -279,6 +286,13 @@ static int __init sp_mod_init(void) return -ENODEV; } +#if IS_BUILTIN(CONFIG_KVM_AMD) && IS_ENABLED(CONFIG_KVM_AMD_SEV) +int __init sev_module_init(void) +{ + return sp_mod_init(); +} +#endif + static void __exit sp_mod_exit(void) { #ifdef CONFIG_X86 diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c index 248d98fd8c48..157f9a9ed636 100644 --- a/drivers/crypto/ccp/sp-pci.c +++ b/drivers/crypto/ccp/sp-pci.c @@ -189,14 +189,17 @@ static bool sp_pci_is_master(struct sp_device *sp) pdev_new = to_pci_dev(dev_new); pdev_cur = to_pci_dev(dev_cur); - if (pdev_new->bus->number < pdev_cur->bus->number) - return true; + if (pci_domain_nr(pdev_new->bus) != pci_domain_nr(pdev_cur->bus)) + return pci_domain_nr(pdev_new->bus) < pci_domain_nr(pdev_cur->bus); - if (PCI_SLOT(pdev_new->devfn) < PCI_SLOT(pdev_cur->devfn)) - return true; + if (pdev_new->bus->number != pdev_cur->bus->number) + return pdev_new->bus->number < pdev_cur->bus->number; - if (PCI_FUNC(pdev_new->devfn) < PCI_FUNC(pdev_cur->devfn)) - return true; + if (PCI_SLOT(pdev_new->devfn) != PCI_SLOT(pdev_cur->devfn)) + return PCI_SLOT(pdev_new->devfn) < PCI_SLOT(pdev_cur->devfn); + + if (PCI_FUNC(pdev_new->devfn) != PCI_FUNC(pdev_cur->devfn)) + return PCI_FUNC(pdev_new->devfn) < PCI_FUNC(pdev_cur->devfn); return false; } diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 96fde9437b4b..f5b47e5ff48a 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -1209,7 +1209,6 @@ static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->mode = uacce_mode; qm->pdev = pdev; - qm->ver = pdev->revision; qm->sqe_size = HPRE_SQE_SIZE; qm->dev_name = hpre_name; @@ -1396,6 +1395,17 @@ static enum acc_err_result hpre_get_err_result(struct hisi_qm *qm) return ACC_ERR_RECOVERED; } +static bool hpre_dev_is_abnormal(struct hisi_qm *qm) +{ + u32 err_status; + + err_status = hpre_get_hw_err_status(qm); + if (err_status & qm->err_info.dev_shutdown_mask) + return true; + + return false; +} + static void hpre_err_info_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; @@ -1428,6 +1438,7 @@ static const struct hisi_qm_err_ini hpre_err_ini = { .show_last_dfx_regs = hpre_show_last_dfx_regs, .err_info_init = hpre_err_info_init, .get_err_result = hpre_get_err_result, + .dev_is_abnormal = hpre_dev_is_abnormal, }; static int hpre_pf_probe_init(struct hpre *hpre) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 19c1b5d3c954..d3f5d108b898 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -30,8 +30,6 @@ /* mailbox */ #define QM_MB_PING_ALL_VFS 0xffff -#define QM_MB_CMD_DATA_SHIFT 32 -#define QM_MB_CMD_DATA_MASK GENMASK(31, 0) #define QM_MB_STATUS_MASK GENMASK(12, 9) /* sqc shift */ @@ -102,6 +100,8 @@ #define QM_PM_CTRL 0x100148 #define QM_IDLE_DISABLE BIT(9) +#define QM_SUB_VERSION_ID 0x210 + #define QM_VFT_CFG_DATA_L 0x100064 #define QM_VFT_CFG_DATA_H 0x100068 #define QM_SQC_VFT_BUF_SIZE (7ULL << 8) @@ -119,6 +119,7 @@ #define QM_SQC_VFT_BASE_MASK_V2 GENMASK(15, 0) #define QM_SQC_VFT_NUM_SHIFT_V2 45 #define QM_SQC_VFT_NUM_MASK_V2 GENMASK(9, 0) +#define QM_MAX_QC_TYPE 2 #define QM_ABNORMAL_INT_SOURCE 0x100000 #define QM_ABNORMAL_INT_MASK 0x100004 @@ -176,6 +177,10 @@ #define QM_IFC_INT_MASK 0x0024 #define QM_IFC_INT_STATUS 0x0028 #define QM_IFC_INT_SET_V 0x002C +#define QM_PF2VF_PF_W 0x104700 +#define QM_VF2PF_PF_R 0x104800 +#define QM_VF2PF_VF_W 0x320 +#define QM_PF2VF_VF_R 0x380 #define QM_IFC_SEND_ALL_VFS GENMASK(6, 0) #define QM_IFC_INT_SOURCE_CLR GENMASK(63, 0) #define QM_IFC_INT_SOURCE_MASK BIT(0) @@ -185,8 +190,11 @@ #define QM_WAIT_DST_ACK 10 #define QM_MAX_PF_WAIT_COUNT 10 #define QM_MAX_VF_WAIT_COUNT 40 -#define QM_VF_RESET_WAIT_US 20000 -#define QM_VF_RESET_WAIT_CNT 3000 +#define QM_VF_RESET_WAIT_US 20000 +#define QM_VF_RESET_WAIT_CNT 3000 +#define QM_VF2PF_REG_SIZE 4 +#define QM_IFC_CMD_MASK GENMASK(31, 0) +#define QM_IFC_DATA_SHIFT 32 #define QM_VF_RESET_WAIT_TIMEOUT_US \ (QM_VF_RESET_WAIT_US * QM_VF_RESET_WAIT_CNT) @@ -234,8 +242,6 @@ #define QM_QOS_MAX_CIR_U 6 #define QM_AUTOSUSPEND_DELAY 3000 -#define QM_DEV_ALG_MAX_LEN 256 - /* abnormal status value for stopping queue */ #define QM_STOP_QUEUE_FAIL 1 #define QM_DUMP_SQC_FAIL 3 @@ -276,7 +282,7 @@ enum qm_alg_type { ALG_TYPE_1, }; -enum qm_mb_cmd { +enum qm_ifc_cmd { QM_PF_FLR_PREPARE = 0x01, QM_PF_SRST_PREPARE, QM_PF_RESET_DONE, @@ -333,6 +339,7 @@ static const struct hisi_qm_cap_info qm_cap_info_comm[] = { {QM_SUPPORT_STOP_FUNC, 0x3100, 0, BIT(10), 0x0, 0x0, 0x1}, {QM_SUPPORT_MB_COMMAND, 0x3100, 0, BIT(11), 0x0, 0x0, 0x1}, {QM_SUPPORT_SVA_PREFETCH, 0x3100, 0, BIT(14), 0x0, 0x0, 0x1}, + {QM_SUPPORT_DAE, 0x3100, 0, BIT(15), 0x0, 0x0, 0x0}, }; static const struct hisi_qm_cap_info qm_cap_info_pf[] = { @@ -396,6 +403,11 @@ struct hisi_qm_hw_ops { void (*hw_error_uninit)(struct hisi_qm *qm); enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm); int (*set_msi)(struct hisi_qm *qm, bool set); + + /* (u64)msg = (u32)data << 32 | (enum qm_ifc_cmd)cmd */ + int (*set_ifc_begin)(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num); + void (*set_ifc_end)(struct hisi_qm *qm); + int (*get_ifc)(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num); }; struct hisi_qm_hw_error { @@ -501,15 +513,20 @@ static u32 qm_get_dev_err_status(struct hisi_qm *qm) /* Check if the error causes the master ooo block */ static bool qm_check_dev_error(struct hisi_qm *qm) { - u32 val, dev_val; + struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); + u32 err_status; - if (qm->fun_type == QM_HW_VF) + if (pf_qm->fun_type == QM_HW_VF) return false; - val = qm_get_hw_error_status(qm) & qm->err_info.qm_shutdown_mask; - dev_val = qm_get_dev_err_status(qm) & qm->err_info.dev_shutdown_mask; + err_status = qm_get_hw_error_status(pf_qm); + if (err_status & pf_qm->err_info.qm_shutdown_mask) + return true; + + if (pf_qm->err_ini->dev_is_abnormal) + return pf_qm->err_ini->dev_is_abnormal(pf_qm); - return val || dev_val; + return false; } static int qm_wait_reset_finish(struct hisi_qm *qm) @@ -654,7 +671,6 @@ EXPORT_SYMBOL_GPL(hisi_qm_mb); /* op 0: set xqc information to hardware, 1: get xqc information from hardware. */ int qm_set_and_get_xqc(struct hisi_qm *qm, u8 cmd, void *xqc, u32 qp_id, bool op) { - struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); struct qm_mailbox mailbox; dma_addr_t xqc_dma; void *tmp_xqc; @@ -688,7 +704,7 @@ int qm_set_and_get_xqc(struct hisi_qm *qm, u8 cmd, void *xqc, u32 qp_id, bool op } /* Setting xqc will fail if master OOO is blocked. */ - if (qm_check_dev_error(pf_qm)) { + if (qm_check_dev_error(qm)) { dev_err(&qm->pdev->dev, "failed to send mailbox since qm is stop!\n"); return -EIO; } @@ -855,10 +871,10 @@ int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *d strcat(algs, dev_algs[i].alg); ptr = strrchr(algs, '\n'); - if (ptr) { + if (ptr) *ptr = '\0'; - qm->uacce->algs = algs; - } + + qm->uacce->algs = algs; return 0; } @@ -1052,11 +1068,10 @@ static void qm_disable_qp(struct hisi_qm *qm, u32 qp_id) static void qm_reset_function(struct hisi_qm *qm) { - struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); struct device *dev = &qm->pdev->dev; int ret; - if (qm_check_dev_error(pf_qm)) + if (qm_check_dev_error(qm)) return; ret = qm_reset_prepare_ready(qm); @@ -1540,17 +1555,15 @@ static void qm_clear_cmd_interrupt(struct hisi_qm *qm, u64 vf_mask) static void qm_handle_vf_msg(struct hisi_qm *qm, u32 vf_id) { struct device *dev = &qm->pdev->dev; - u32 cmd; - u64 msg; + enum qm_ifc_cmd cmd; int ret; - ret = qm_get_mb_cmd(qm, &msg, vf_id); + ret = qm->ops->get_ifc(qm, &cmd, NULL, vf_id); if (ret) { - dev_err(dev, "failed to get msg from VF(%u)!\n", vf_id); + dev_err(dev, "failed to get command from VF(%u)!\n", vf_id); return; } - cmd = msg & QM_MB_CMD_DATA_MASK; switch (cmd) { case QM_VF_PREPARE_FAIL: dev_err(dev, "failed to stop VF(%u)!\n", vf_id); @@ -1562,7 +1575,7 @@ static void qm_handle_vf_msg(struct hisi_qm *qm, u32 vf_id) case QM_VF_START_DONE: break; default: - dev_err(dev, "unsupported cmd %u sent by VF(%u)!\n", cmd, vf_id); + dev_err(dev, "unsupported command(0x%x) sent by VF(%u)!\n", cmd, vf_id); break; } } @@ -1630,17 +1643,14 @@ static void qm_trigger_pf_interrupt(struct hisi_qm *qm) writel(val, qm->io_base + QM_IFC_INT_SET_V); } -static int qm_ping_single_vf(struct hisi_qm *qm, u64 cmd, u32 fun_num) +static int qm_ping_single_vf(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) { struct device *dev = &qm->pdev->dev; - struct qm_mailbox mailbox; int cnt = 0; u64 val; int ret; - qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, fun_num, 0); - mutex_lock(&qm->mailbox_lock); - ret = qm_mb_nolock(qm, &mailbox); + ret = qm->ops->set_ifc_begin(qm, cmd, data, fun_num); if (ret) { dev_err(dev, "failed to send command to vf(%u)!\n", fun_num); goto err_unlock; @@ -1662,27 +1672,23 @@ static int qm_ping_single_vf(struct hisi_qm *qm, u64 cmd, u32 fun_num) } err_unlock: - mutex_unlock(&qm->mailbox_lock); + qm->ops->set_ifc_end(qm); return ret; } -static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd) +static int qm_ping_all_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd) { struct device *dev = &qm->pdev->dev; u32 vfs_num = qm->vfs_num; - struct qm_mailbox mailbox; u64 val = 0; int cnt = 0; int ret; u32 i; - qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, QM_MB_PING_ALL_VFS, 0); - mutex_lock(&qm->mailbox_lock); - /* PF sends command to all VFs by mailbox */ - ret = qm_mb_nolock(qm, &mailbox); + ret = qm->ops->set_ifc_begin(qm, cmd, 0, QM_MB_PING_ALL_VFS); if (ret) { - dev_err(dev, "failed to send command to VFs!\n"); - mutex_unlock(&qm->mailbox_lock); + dev_err(dev, "failed to send command(0x%x) to all vfs!\n", cmd); + qm->ops->set_ifc_end(qm); return ret; } @@ -1692,7 +1698,7 @@ static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd) val = readq(qm->io_base + QM_IFC_READY_STATUS); /* If all VFs acked, PF notifies VFs successfully. */ if (!(val & GENMASK(vfs_num, 1))) { - mutex_unlock(&qm->mailbox_lock); + qm->ops->set_ifc_end(qm); return 0; } @@ -1700,7 +1706,7 @@ static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd) break; } - mutex_unlock(&qm->mailbox_lock); + qm->ops->set_ifc_end(qm); /* Check which vf respond timeout. */ for (i = 1; i <= vfs_num; i++) { @@ -1711,18 +1717,15 @@ static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd) return -ETIMEDOUT; } -static int qm_ping_pf(struct hisi_qm *qm, u64 cmd) +static int qm_ping_pf(struct hisi_qm *qm, enum qm_ifc_cmd cmd) { - struct qm_mailbox mailbox; int cnt = 0; u32 val; int ret; - qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, 0, 0); - mutex_lock(&qm->mailbox_lock); - ret = qm_mb_nolock(qm, &mailbox); + ret = qm->ops->set_ifc_begin(qm, cmd, 0, 0); if (ret) { - dev_err(&qm->pdev->dev, "failed to send command to PF!\n"); + dev_err(&qm->pdev->dev, "failed to send command(0x%x) to PF!\n", cmd); goto unlock; } @@ -1741,7 +1744,8 @@ static int qm_ping_pf(struct hisi_qm *qm, u64 cmd) } unlock: - mutex_unlock(&qm->mailbox_lock); + qm->ops->set_ifc_end(qm); + return ret; } @@ -1842,6 +1846,94 @@ static int qm_set_msi_v3(struct hisi_qm *qm, bool set) return ret; } +static int qm_set_ifc_begin_v3(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) +{ + struct qm_mailbox mailbox; + u64 msg; + + msg = cmd | (u64)data << QM_IFC_DATA_SHIFT; + + qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, msg, fun_num, 0); + mutex_lock(&qm->mailbox_lock); + return qm_mb_nolock(qm, &mailbox); +} + +static void qm_set_ifc_end_v3(struct hisi_qm *qm) +{ + mutex_unlock(&qm->mailbox_lock); +} + +static int qm_get_ifc_v3(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num) +{ + u64 msg; + int ret; + + ret = qm_get_mb_cmd(qm, &msg, fun_num); + if (ret) + return ret; + + *cmd = msg & QM_IFC_CMD_MASK; + + if (data) + *data = msg >> QM_IFC_DATA_SHIFT; + + return 0; +} + +static int qm_set_ifc_begin_v4(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) +{ + uintptr_t offset; + u64 msg; + + if (qm->fun_type == QM_HW_PF) + offset = QM_PF2VF_PF_W; + else + offset = QM_VF2PF_VF_W; + + msg = cmd | (u64)data << QM_IFC_DATA_SHIFT; + + mutex_lock(&qm->ifc_lock); + writeq(msg, qm->io_base + offset); + + return 0; +} + +static void qm_set_ifc_end_v4(struct hisi_qm *qm) +{ + mutex_unlock(&qm->ifc_lock); +} + +static u64 qm_get_ifc_pf(struct hisi_qm *qm, u32 fun_num) +{ + uintptr_t offset; + + offset = QM_VF2PF_PF_R + QM_VF2PF_REG_SIZE * fun_num; + + return (u64)readl(qm->io_base + offset); +} + +static u64 qm_get_ifc_vf(struct hisi_qm *qm) +{ + return readq(qm->io_base + QM_PF2VF_VF_R); +} + +static int qm_get_ifc_v4(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num) +{ + u64 msg; + + if (qm->fun_type == QM_HW_PF) + msg = qm_get_ifc_pf(qm, fun_num); + else + msg = qm_get_ifc_vf(qm); + + *cmd = msg & QM_IFC_CMD_MASK; + + if (data) + *data = msg >> QM_IFC_DATA_SHIFT; + + return 0; +} + static const struct hisi_qm_hw_ops qm_hw_ops_v1 = { .qm_db = qm_db_v1, .hw_error_init = qm_hw_error_init_v1, @@ -1864,6 +1956,21 @@ static const struct hisi_qm_hw_ops qm_hw_ops_v3 = { .hw_error_uninit = qm_hw_error_uninit_v3, .hw_error_handle = qm_hw_error_handle_v2, .set_msi = qm_set_msi_v3, + .set_ifc_begin = qm_set_ifc_begin_v3, + .set_ifc_end = qm_set_ifc_end_v3, + .get_ifc = qm_get_ifc_v3, +}; + +static const struct hisi_qm_hw_ops qm_hw_ops_v4 = { + .get_vft = qm_get_vft_v2, + .qm_db = qm_db_v2, + .hw_error_init = qm_hw_error_init_v3, + .hw_error_uninit = qm_hw_error_uninit_v3, + .hw_error_handle = qm_hw_error_handle_v2, + .set_msi = qm_set_msi_v3, + .set_ifc_begin = qm_set_ifc_begin_v4, + .set_ifc_end = qm_set_ifc_end_v4, + .get_ifc = qm_get_ifc_v4, }; static void *qm_get_avail_sqe(struct hisi_qp *qp) @@ -2156,12 +2263,11 @@ static int qm_wait_qp_empty(struct hisi_qm *qm, u32 *state, u32 qp_id) static int qm_drain_qp(struct hisi_qp *qp) { struct hisi_qm *qm = qp->qm; - struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); u32 state = 0; int ret; /* No need to judge if master OOO is blocked. */ - if (qm_check_dev_error(pf_qm)) + if (qm_check_dev_error(qm)) return 0; /* HW V3 supports drain qp by device */ @@ -2475,7 +2581,7 @@ static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd, sizeof(struct hisi_qp_ctx))) return -EFAULT; - if (qp_ctx.qc_type != 0 && qp_ctx.qc_type != 1) + if (qp_ctx.qc_type > QM_MAX_QC_TYPE) return -EINVAL; qm_set_sqctype(q, qp_ctx.qc_type); @@ -2843,11 +2949,14 @@ static void hisi_qm_pre_init(struct hisi_qm *qm) qm->ops = &qm_hw_ops_v1; else if (qm->ver == QM_HW_V2) qm->ops = &qm_hw_ops_v2; - else + else if (qm->ver == QM_HW_V3) qm->ops = &qm_hw_ops_v3; + else + qm->ops = &qm_hw_ops_v4; pci_set_drvdata(pdev, qm); mutex_init(&qm->mailbox_lock); + mutex_init(&qm->ifc_lock); init_rwsem(&qm->qps_lock); qm->qp_in_used = 0; if (test_bit(QM_SUPPORT_RPM, &qm->caps)) { @@ -3607,7 +3716,6 @@ static u32 qm_get_shaper_vft_qos(struct hisi_qm *qm, u32 fun_index) static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num) { struct device *dev = &qm->pdev->dev; - u64 mb_cmd; u32 qos; int ret; @@ -3617,10 +3725,9 @@ static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num) return; } - mb_cmd = QM_PF_SET_QOS | (u64)qos << QM_MB_CMD_DATA_SHIFT; - ret = qm_ping_single_vf(qm, mb_cmd, fun_num); + ret = qm_ping_single_vf(qm, QM_PF_SET_QOS, qos, fun_num); if (ret) - dev_err(dev, "failed to send cmd to VF(%u)!\n", fun_num); + dev_err(dev, "failed to send command(0x%x) to VF(%u)!\n", QM_PF_SET_QOS, fun_num); } static int qm_vf_read_qos(struct hisi_qm *qm) @@ -4109,7 +4216,7 @@ stop_fail: return ret; } -static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd, +static int qm_try_stop_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd, enum qm_stop_reason stop_reason) { struct pci_dev *pdev = qm->pdev; @@ -4122,7 +4229,7 @@ static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd, if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { ret = qm_ping_all_vfs(qm, cmd); if (ret) - pci_err(pdev, "failed to send cmd to all VFs before PF reset!\n"); + pci_err(pdev, "failed to send command to all VFs before PF reset!\n"); } else { ret = qm_vf_reset_prepare(qm, stop_reason); if (ret) @@ -4137,6 +4244,12 @@ static int qm_controller_reset_prepare(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; int ret; + if (qm->err_ini->set_priv_status) { + ret = qm->err_ini->set_priv_status(qm); + if (ret) + return ret; + } + ret = qm_reset_prepare_ready(qm); if (ret) { pci_err(pdev, "Controller reset not ready!\n"); @@ -4298,7 +4411,7 @@ restart_fail: return ret; } -static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_mb_cmd cmd) +static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd) { struct pci_dev *pdev = qm->pdev; int ret; @@ -4527,7 +4640,7 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) * Check whether there is an ECC mbit error, If it occurs, need to * wait for soft reset to fix it. */ - while (qm_check_dev_error(pf_qm)) { + while (qm_check_dev_error(qm)) { msleep(++delay); if (delay > QM_RESET_WAIT_TIMEOUT) return; @@ -4675,7 +4788,7 @@ static void hisi_qm_controller_reset(struct work_struct *rst_work) static void qm_pf_reset_vf_prepare(struct hisi_qm *qm, enum qm_stop_reason stop_reason) { - enum qm_mb_cmd cmd = QM_VF_PREPARE_DONE; + enum qm_ifc_cmd cmd = QM_VF_PREPARE_DONE; struct pci_dev *pdev = qm->pdev; int ret; @@ -4709,7 +4822,7 @@ out: static void qm_pf_reset_vf_done(struct hisi_qm *qm) { - enum qm_mb_cmd cmd = QM_VF_START_DONE; + enum qm_ifc_cmd cmd = QM_VF_START_DONE; struct pci_dev *pdev = qm->pdev; int ret; @@ -4732,7 +4845,6 @@ static int qm_wait_pf_reset_finish(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; u32 val, cmd; - u64 msg; int ret; /* Wait for reset to finish */ @@ -4749,16 +4861,15 @@ static int qm_wait_pf_reset_finish(struct hisi_qm *qm) * Whether message is got successfully, * VF needs to ack PF by clearing the interrupt. */ - ret = qm_get_mb_cmd(qm, &msg, 0); + ret = qm->ops->get_ifc(qm, &cmd, NULL, 0); qm_clear_cmd_interrupt(qm, 0); if (ret) { - dev_err(dev, "failed to get msg from PF in reset done!\n"); + dev_err(dev, "failed to get command from PF in reset done!\n"); return ret; } - cmd = msg & QM_MB_CMD_DATA_MASK; if (cmd != QM_PF_RESET_DONE) { - dev_err(dev, "the cmd(%u) is not reset done!\n", cmd); + dev_err(dev, "the command(0x%x) is not reset done!\n", cmd); ret = -EINVAL; } @@ -4795,22 +4906,21 @@ err_get_status: static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num) { struct device *dev = &qm->pdev->dev; - u64 msg; - u32 cmd; + enum qm_ifc_cmd cmd; + u32 data; int ret; /* * Get the msg from source by sending mailbox. Whether message is got * successfully, destination needs to ack source by clearing the interrupt. */ - ret = qm_get_mb_cmd(qm, &msg, fun_num); + ret = qm->ops->get_ifc(qm, &cmd, &data, fun_num); qm_clear_cmd_interrupt(qm, BIT(fun_num)); if (ret) { - dev_err(dev, "failed to get msg from source!\n"); + dev_err(dev, "failed to get command from source!\n"); return; } - cmd = msg & QM_MB_CMD_DATA_MASK; switch (cmd) { case QM_PF_FLR_PREPARE: qm_pf_reset_vf_process(qm, QM_DOWN); @@ -4822,10 +4932,10 @@ static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num) qm_vf_get_qos(qm, fun_num); break; case QM_PF_SET_QOS: - qm->mb_qos = msg >> QM_MB_CMD_DATA_SHIFT; + qm->mb_qos = data; break; default: - dev_err(dev, "unsupported cmd %u sent by function(%u)!\n", cmd, fun_num); + dev_err(dev, "unsupported command(0x%x) sent by function(%u)!\n", cmd, fun_num); break; } } @@ -5167,6 +5277,20 @@ static int qm_get_hw_caps(struct hisi_qm *qm) return qm_pre_store_caps(qm); } +static void qm_get_version(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 sub_version_id; + + qm->ver = pdev->revision; + + if (pdev->revision == QM_HW_V3) { + sub_version_id = readl(qm->io_base + QM_SUB_VERSION_ID); + if (sub_version_id) + qm->ver = sub_version_id; + } +} + static int qm_get_pci_res(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -5186,6 +5310,8 @@ static int qm_get_pci_res(struct hisi_qm *qm) goto err_request_mem_regions; } + qm_get_version(qm); + ret = qm_get_hw_caps(qm); if (ret) goto err_ioremap; @@ -5205,6 +5331,7 @@ static int qm_get_pci_res(struct hisi_qm *qm) qm->db_interval = 0; } + hisi_qm_pre_init(qm); ret = qm_get_qp_num(qm); if (ret) goto err_db_ioremap; @@ -5247,6 +5374,14 @@ static int qm_clear_device(struct hisi_qm *qm) return ret; } + if (qm->err_ini->set_priv_status) { + ret = qm->err_ini->set_priv_status(qm); + if (ret) { + writel(0x0, qm->io_base + ACC_MASTER_GLOBAL_CTRL); + return ret; + } + } + return qm_reset_device(qm); } @@ -5461,8 +5596,6 @@ int hisi_qm_init(struct hisi_qm *qm) struct device *dev = &pdev->dev; int ret; - hisi_qm_pre_init(qm); - ret = hisi_qm_pci_init(qm); if (ret) return ret; @@ -5598,6 +5731,12 @@ static int qm_prepare_for_suspend(struct hisi_qm *qm) if (ret) return ret; + if (qm->err_ini->set_priv_status) { + ret = qm->err_ini->set_priv_status(qm); + if (ret) + return ret; + } + ret = qm_set_pf_mse(qm, false); if (ret) pci_err(pdev, "failed to disable MSE before suspending!\n"); diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 4b9970230822..703920b49c7c 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -37,7 +37,6 @@ struct sec_aead_req { u8 *a_ivin; dma_addr_t a_ivin_dma; struct aead_request *aead_req; - bool fallback; }; /* SEC request of Crypto */ diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 66bc07da9eb6..8ea5305bc320 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -57,7 +57,6 @@ #define SEC_TYPE_MASK 0x0F #define SEC_DONE_MASK 0x0001 #define SEC_ICV_MASK 0x000E -#define SEC_SQE_LEN_RATE_MASK 0x3 #define SEC_TOTAL_IV_SZ(depth) (SEC_IV_SIZE * (depth)) #define SEC_SGL_SGE_NR 128 @@ -80,16 +79,16 @@ #define SEC_TOTAL_PBUF_SZ(depth) (PAGE_SIZE * SEC_PBUF_PAGE_NUM(depth) + \ SEC_PBUF_LEFT_SZ(depth)) -#define SEC_SQE_LEN_RATE 4 #define SEC_SQE_CFLAG 2 #define SEC_SQE_AEAD_FLAG 3 #define SEC_SQE_DONE 0x1 #define SEC_ICV_ERR 0x2 -#define MIN_MAC_LEN 4 #define MAC_LEN_MASK 0x1U #define MAX_INPUT_DATA_LEN 0xFFFE00 #define BITS_MASK 0xFF +#define WORD_MASK 0x3 #define BYTE_BITS 0x8 +#define BYTES_TO_WORDS(bcount) ((bcount) >> 2) #define SEC_XTS_NAME_SZ 0x3 #define IV_CM_CAL_NUM 2 #define IV_CL_MASK 0x7 @@ -691,14 +690,10 @@ static int sec_skcipher_fbtfm_init(struct crypto_skcipher *tfm) c_ctx->fallback = false; - /* Currently, only XTS mode need fallback tfm when using 192bit key */ - if (likely(strncmp(alg, "xts", SEC_XTS_NAME_SZ))) - return 0; - c_ctx->fbtfm = crypto_alloc_sync_skcipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(c_ctx->fbtfm)) { - pr_err("failed to alloc xts mode fallback tfm!\n"); + pr_err("failed to alloc fallback tfm for %s!\n", alg); return PTR_ERR(c_ctx->fbtfm); } @@ -858,7 +853,7 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, } memcpy(c_ctx->c_key, key, keylen); - if (c_ctx->fallback && c_ctx->fbtfm) { + if (c_ctx->fbtfm) { ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); if (ret) { dev_err(dev, "failed to set fallback skcipher key!\n"); @@ -1090,11 +1085,6 @@ static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, struct crypto_shash *hash_tfm = ctx->hash_tfm; int blocksize, digestsize, ret; - if (!keys->authkeylen) { - pr_err("hisi_sec2: aead auth key error!\n"); - return -EINVAL; - } - blocksize = crypto_shash_blocksize(hash_tfm); digestsize = crypto_shash_digestsize(hash_tfm); if (keys->authkeylen > blocksize) { @@ -1106,7 +1096,8 @@ static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, } ctx->a_key_len = digestsize; } else { - memcpy(ctx->a_key, keys->authkey, keys->authkeylen); + if (keys->authkeylen) + memcpy(ctx->a_key, keys->authkey, keys->authkeylen); ctx->a_key_len = keys->authkeylen; } @@ -1160,8 +1151,10 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, } ret = crypto_authenc_extractkeys(&keys, key, keylen); - if (ret) + if (ret) { + dev_err(dev, "sec extract aead keys err!\n"); goto bad_key; + } ret = sec_aead_aes_set_key(c_ctx, &keys); if (ret) { @@ -1175,12 +1168,6 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, goto bad_key; } - if (ctx->a_ctx.a_key_len & SEC_SQE_LEN_RATE_MASK) { - ret = -EINVAL; - dev_err(dev, "AUTH key length error!\n"); - goto bad_key; - } - ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); if (ret) { dev_err(dev, "set sec fallback key err!\n"); @@ -1583,11 +1570,10 @@ static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma); - sec_sqe->type2.mac_key_alg = cpu_to_le32(authsize / SEC_SQE_LEN_RATE); + sec_sqe->type2.mac_key_alg = cpu_to_le32(BYTES_TO_WORDS(authsize)); sec_sqe->type2.mac_key_alg |= - cpu_to_le32((u32)((ctx->a_key_len) / - SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET); + cpu_to_le32((u32)BYTES_TO_WORDS(ctx->a_key_len) << SEC_AKEY_OFFSET); sec_sqe->type2.mac_key_alg |= cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET); @@ -1639,12 +1625,10 @@ static void sec_auth_bd_fill_ex_v3(struct sec_auth_ctx *ctx, int dir, sqe3->a_key_addr = cpu_to_le64(ctx->a_key_dma); sqe3->auth_mac_key |= - cpu_to_le32((u32)(authsize / - SEC_SQE_LEN_RATE) << SEC_MAC_OFFSET_V3); + cpu_to_le32(BYTES_TO_WORDS(authsize) << SEC_MAC_OFFSET_V3); sqe3->auth_mac_key |= - cpu_to_le32((u32)(ctx->a_key_len / - SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET_V3); + cpu_to_le32((u32)BYTES_TO_WORDS(ctx->a_key_len) << SEC_AKEY_OFFSET_V3); sqe3->auth_mac_key |= cpu_to_le32((u32)(ctx->a_alg) << SEC_AUTH_ALG_OFFSET_V3); @@ -2003,8 +1987,7 @@ static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm) return sec_aead_ctx_init(tfm, "sha512"); } -static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx, - struct sec_req *sreq) +static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx, struct sec_req *sreq) { u32 cryptlen = sreq->c_req.sk_req->cryptlen; struct device *dev = ctx->dev; @@ -2026,10 +2009,6 @@ static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx, } break; case SEC_CMODE_CTR: - if (unlikely(ctx->sec->qm.ver < QM_HW_V3)) { - dev_err(dev, "skcipher HW version error!\n"); - ret = -EINVAL; - } break; default: ret = -EINVAL; @@ -2038,17 +2017,21 @@ static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx, return ret; } -static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) +static int sec_skcipher_param_check(struct sec_ctx *ctx, + struct sec_req *sreq, bool *need_fallback) { struct skcipher_request *sk_req = sreq->c_req.sk_req; struct device *dev = ctx->dev; u8 c_alg = ctx->c_ctx.c_alg; - if (unlikely(!sk_req->src || !sk_req->dst || - sk_req->cryptlen > MAX_INPUT_DATA_LEN)) { + if (unlikely(!sk_req->src || !sk_req->dst)) { dev_err(dev, "skcipher input param error!\n"); return -EINVAL; } + + if (sk_req->cryptlen > MAX_INPUT_DATA_LEN) + *need_fallback = true; + sreq->c_req.c_len = sk_req->cryptlen; if (ctx->pbuf_supported && sk_req->cryptlen <= SEC_PBUF_SZ) @@ -2106,6 +2089,7 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req); struct sec_req *req = skcipher_request_ctx(sk_req); struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + bool need_fallback = false; int ret; if (!sk_req->cryptlen) { @@ -2119,11 +2103,11 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) req->c_req.encrypt = encrypt; req->ctx = ctx; - ret = sec_skcipher_param_check(ctx, req); + ret = sec_skcipher_param_check(ctx, req, &need_fallback); if (unlikely(ret)) return -EINVAL; - if (unlikely(ctx->c_ctx.fallback)) + if (unlikely(ctx->c_ctx.fallback || need_fallback)) return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); return ctx->req_op->process(ctx, req); @@ -2231,52 +2215,35 @@ static int sec_aead_spec_check(struct sec_ctx *ctx, struct sec_req *sreq) struct crypto_aead *tfm = crypto_aead_reqtfm(req); size_t sz = crypto_aead_authsize(tfm); u8 c_mode = ctx->c_ctx.c_mode; - struct device *dev = ctx->dev; int ret; - /* Hardware does not handle cases where authsize is less than 4 bytes */ - if (unlikely(sz < MIN_MAC_LEN)) { - sreq->aead_req.fallback = true; + if (unlikely(ctx->sec->qm.ver == QM_HW_V2 && !sreq->c_req.c_len)) return -EINVAL; - } if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN || - req->assoclen > SEC_MAX_AAD_LEN)) { - dev_err(dev, "aead input spec error!\n"); + req->assoclen > SEC_MAX_AAD_LEN)) return -EINVAL; - } if (c_mode == SEC_CMODE_CCM) { - if (unlikely(req->assoclen > SEC_MAX_CCM_AAD_LEN)) { - dev_err_ratelimited(dev, "CCM input aad parameter is too long!\n"); + if (unlikely(req->assoclen > SEC_MAX_CCM_AAD_LEN)) return -EINVAL; - } - ret = aead_iv_demension_check(req); - if (ret) { - dev_err(dev, "aead input iv param error!\n"); - return ret; - } - } - if (sreq->c_req.encrypt) - sreq->c_req.c_len = req->cryptlen; - else - sreq->c_req.c_len = req->cryptlen - sz; - if (c_mode == SEC_CMODE_CBC) { - if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { - dev_err(dev, "aead crypto length error!\n"); + ret = aead_iv_demension_check(req); + if (unlikely(ret)) + return -EINVAL; + } else if (c_mode == SEC_CMODE_CBC) { + if (unlikely(sz & WORD_MASK)) + return -EINVAL; + if (unlikely(ctx->a_ctx.a_key_len & WORD_MASK)) return -EINVAL; - } } return 0; } -static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) +static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq, bool *need_fallback) { struct aead_request *req = sreq->aead_req.aead_req; - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - size_t authsize = crypto_aead_authsize(tfm); struct device *dev = ctx->dev; u8 c_alg = ctx->c_ctx.c_alg; @@ -2285,12 +2252,10 @@ static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) return -EINVAL; } - if (ctx->sec->qm.ver == QM_HW_V2) { - if (unlikely(!req->cryptlen || (!sreq->c_req.encrypt && - req->cryptlen <= authsize))) { - sreq->aead_req.fallback = true; - return -EINVAL; - } + if (unlikely(ctx->c_ctx.c_mode == SEC_CMODE_CBC && + sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { + dev_err(dev, "aead cbc mode input data length error!\n"); + return -EINVAL; } /* Support AES or SM4 */ @@ -2299,8 +2264,10 @@ static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) return -EINVAL; } - if (unlikely(sec_aead_spec_check(ctx, sreq))) + if (unlikely(sec_aead_spec_check(ctx, sreq))) { + *need_fallback = true; return -EINVAL; + } if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <= SEC_PBUF_SZ) @@ -2344,17 +2311,19 @@ static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); struct sec_req *req = aead_request_ctx(a_req); struct sec_ctx *ctx = crypto_aead_ctx(tfm); + size_t sz = crypto_aead_authsize(tfm); + bool need_fallback = false; int ret; req->flag = a_req->base.flags; req->aead_req.aead_req = a_req; req->c_req.encrypt = encrypt; req->ctx = ctx; - req->aead_req.fallback = false; + req->c_req.c_len = a_req->cryptlen - (req->c_req.encrypt ? 0 : sz); - ret = sec_aead_param_check(ctx, req); + ret = sec_aead_param_check(ctx, req, &need_fallback); if (unlikely(ret)) { - if (req->aead_req.fallback) + if (need_fallback) return sec_aead_soft_crypto(ctx, a_req, encrypt); return -EINVAL; } diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 8ec5333bb5aa..72cf48d1f3ab 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -1097,6 +1097,17 @@ static enum acc_err_result sec_get_err_result(struct hisi_qm *qm) return ACC_ERR_RECOVERED; } +static bool sec_dev_is_abnormal(struct hisi_qm *qm) +{ + u32 err_status; + + err_status = sec_get_hw_err_status(qm); + if (err_status & qm->err_info.dev_shutdown_mask) + return true; + + return false; +} + static void sec_err_info_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; @@ -1129,6 +1140,7 @@ static const struct hisi_qm_err_ini sec_err_ini = { .show_last_dfx_regs = sec_show_last_dfx_regs, .err_info_init = sec_err_info_init, .get_err_result = sec_get_err_result, + .dev_is_abnormal = sec_dev_is_abnormal, }; static int sec_pf_probe_init(struct sec_dev *sec) @@ -1180,7 +1192,6 @@ static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) int ret; qm->pdev = pdev; - qm->ver = pdev->revision; qm->mode = uacce_mode; qm->sqe_size = SEC_SQE_SIZE; qm->dev_name = sec_name; diff --git a/drivers/crypto/hisilicon/zip/Makefile b/drivers/crypto/hisilicon/zip/Makefile index a936f099ee22..13de020b77d6 100644 --- a/drivers/crypto/hisilicon/zip/Makefile +++ b/drivers/crypto/hisilicon/zip/Makefile @@ -1,2 +1,2 @@ obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += hisi_zip.o -hisi_zip-objs = zip_main.o zip_crypto.o +hisi_zip-objs = zip_main.o zip_crypto.o dae_main.o diff --git a/drivers/crypto/hisilicon/zip/dae_main.c b/drivers/crypto/hisilicon/zip/dae_main.c new file mode 100644 index 000000000000..6f22e4c36e49 --- /dev/null +++ b/drivers/crypto/hisilicon/zip/dae_main.c @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2024 HiSilicon Limited. */ + +#include <linux/bitops.h> +#include <linux/io.h> +#include <linux/uacce.h> +#include "zip.h" + +/* memory */ +#define DAE_MEM_START_OFFSET 0x331040 +#define DAE_MEM_DONE_OFFSET 0x331044 +#define DAE_MEM_START_MASK 0x1 +#define DAE_MEM_DONE_MASK 0x1 +#define DAE_REG_RD_INTVRL_US 10 +#define DAE_REG_RD_TMOUT_US USEC_PER_SEC + +#define DAE_ALG_NAME "hashagg" + +/* error */ +#define DAE_AXI_CFG_OFFSET 0x331000 +#define DAE_AXI_SHUTDOWN_MASK (BIT(0) | BIT(5)) +#define DAE_ERR_SOURCE_OFFSET 0x331C84 +#define DAE_ERR_STATUS_OFFSET 0x331C88 +#define DAE_ERR_CE_OFFSET 0x331CA0 +#define DAE_ERR_CE_MASK BIT(3) +#define DAE_ERR_NFE_OFFSET 0x331CA4 +#define DAE_ERR_NFE_MASK 0x17 +#define DAE_ERR_FE_OFFSET 0x331CA8 +#define DAE_ERR_FE_MASK 0 +#define DAE_ECC_MBIT_MASK BIT(2) +#define DAE_ECC_INFO_OFFSET 0x33400C +#define DAE_ERR_SHUTDOWN_OFFSET 0x331CAC +#define DAE_ERR_SHUTDOWN_MASK 0x17 +#define DAE_ERR_ENABLE_OFFSET 0x331C80 +#define DAE_ERR_ENABLE_MASK (DAE_ERR_FE_MASK | DAE_ERR_NFE_MASK | DAE_ERR_CE_MASK) +#define DAE_AM_CTRL_GLOBAL_OFFSET 0x330000 +#define DAE_AM_RETURN_OFFSET 0x330150 +#define DAE_AM_RETURN_MASK 0x3 +#define DAE_AXI_CFG_OFFSET 0x331000 +#define DAE_AXI_SHUTDOWN_EN_MASK (BIT(0) | BIT(5)) + +struct hisi_dae_hw_error { + u32 int_msk; + const char *msg; +}; + +static const struct hisi_dae_hw_error dae_hw_error[] = { + { .int_msk = BIT(0), .msg = "dae_axi_bus_err" }, + { .int_msk = BIT(1), .msg = "dae_axi_poison_err" }, + { .int_msk = BIT(2), .msg = "dae_ecc_2bit_err" }, + { .int_msk = BIT(3), .msg = "dae_ecc_1bit_err" }, + { .int_msk = BIT(4), .msg = "dae_fsm_hbeat_err" }, +}; + +static inline bool dae_is_support(struct hisi_qm *qm) +{ + if (test_bit(QM_SUPPORT_DAE, &qm->caps)) + return true; + + return false; +} + +int hisi_dae_set_user_domain(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!dae_is_support(qm)) + return 0; + + val = readl(qm->io_base + DAE_MEM_START_OFFSET); + val |= DAE_MEM_START_MASK; + writel(val, qm->io_base + DAE_MEM_START_OFFSET); + ret = readl_relaxed_poll_timeout(qm->io_base + DAE_MEM_DONE_OFFSET, val, + val & DAE_MEM_DONE_MASK, + DAE_REG_RD_INTVRL_US, DAE_REG_RD_TMOUT_US); + if (ret) + pci_err(qm->pdev, "failed to init dae memory!\n"); + + return ret; +} + +int hisi_dae_set_alg(struct hisi_qm *qm) +{ + size_t len; + + if (!dae_is_support(qm)) + return 0; + + if (!qm->uacce) + return 0; + + len = strlen(qm->uacce->algs); + /* A line break may be required */ + if (len + strlen(DAE_ALG_NAME) + 1 >= QM_DEV_ALG_MAX_LEN) { + pci_err(qm->pdev, "algorithm name is too long!\n"); + return -EINVAL; + } + + if (len) + strcat((char *)qm->uacce->algs, "\n"); + + strcat((char *)qm->uacce->algs, DAE_ALG_NAME); + + return 0; +} + +static void hisi_dae_master_ooo_ctrl(struct hisi_qm *qm, bool enable) +{ + u32 axi_val, err_val; + + axi_val = readl(qm->io_base + DAE_AXI_CFG_OFFSET); + if (enable) { + axi_val |= DAE_AXI_SHUTDOWN_MASK; + err_val = DAE_ERR_SHUTDOWN_MASK; + } else { + axi_val &= ~DAE_AXI_SHUTDOWN_MASK; + err_val = 0; + } + + writel(axi_val, qm->io_base + DAE_AXI_CFG_OFFSET); + writel(err_val, qm->io_base + DAE_ERR_SHUTDOWN_OFFSET); +} + +void hisi_dae_hw_error_enable(struct hisi_qm *qm) +{ + if (!dae_is_support(qm)) + return; + + /* clear dae hw error source if having */ + writel(DAE_ERR_ENABLE_MASK, qm->io_base + DAE_ERR_SOURCE_OFFSET); + + /* configure error type */ + writel(DAE_ERR_CE_MASK, qm->io_base + DAE_ERR_CE_OFFSET); + writel(DAE_ERR_NFE_MASK, qm->io_base + DAE_ERR_NFE_OFFSET); + writel(DAE_ERR_FE_MASK, qm->io_base + DAE_ERR_FE_OFFSET); + + hisi_dae_master_ooo_ctrl(qm, true); + + /* enable dae hw error interrupts */ + writel(DAE_ERR_ENABLE_MASK, qm->io_base + DAE_ERR_ENABLE_OFFSET); +} + +void hisi_dae_hw_error_disable(struct hisi_qm *qm) +{ + if (!dae_is_support(qm)) + return; + + writel(0, qm->io_base + DAE_ERR_ENABLE_OFFSET); + hisi_dae_master_ooo_ctrl(qm, false); +} + +static u32 hisi_dae_get_hw_err_status(struct hisi_qm *qm) +{ + return readl(qm->io_base + DAE_ERR_STATUS_OFFSET); +} + +static void hisi_dae_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) +{ + if (!dae_is_support(qm)) + return; + + writel(err_sts, qm->io_base + DAE_ERR_SOURCE_OFFSET); +} + +static void hisi_dae_disable_error_report(struct hisi_qm *qm, u32 err_type) +{ + writel(DAE_ERR_NFE_MASK & (~err_type), qm->io_base + DAE_ERR_NFE_OFFSET); +} + +static void hisi_dae_log_hw_error(struct hisi_qm *qm, u32 err_type) +{ + const struct hisi_dae_hw_error *err = dae_hw_error; + struct device *dev = &qm->pdev->dev; + u32 ecc_info; + size_t i; + + for (i = 0; i < ARRAY_SIZE(dae_hw_error); i++) { + err = &dae_hw_error[i]; + if (!(err->int_msk & err_type)) + continue; + + dev_err(dev, "%s [error status=0x%x] found\n", + err->msg, err->int_msk); + + if (err->int_msk & DAE_ECC_MBIT_MASK) { + ecc_info = readl(qm->io_base + DAE_ECC_INFO_OFFSET); + dev_err(dev, "dae multi ecc sram info 0x%x\n", ecc_info); + } + } +} + +enum acc_err_result hisi_dae_get_err_result(struct hisi_qm *qm) +{ + u32 err_status; + + if (!dae_is_support(qm)) + return ACC_ERR_NONE; + + err_status = hisi_dae_get_hw_err_status(qm); + if (!err_status) + return ACC_ERR_NONE; + + hisi_dae_log_hw_error(qm, err_status); + + if (err_status & DAE_ERR_NFE_MASK) { + /* Disable the same error reporting until device is recovered. */ + hisi_dae_disable_error_report(qm, err_status); + return ACC_ERR_NEED_RESET; + } + hisi_dae_clear_hw_err_status(qm, err_status); + + return ACC_ERR_RECOVERED; +} + +bool hisi_dae_dev_is_abnormal(struct hisi_qm *qm) +{ + u32 err_status; + + if (!dae_is_support(qm)) + return false; + + err_status = hisi_dae_get_hw_err_status(qm); + if (err_status & DAE_ERR_NFE_MASK) + return true; + + return false; +} + +int hisi_dae_close_axi_master_ooo(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (!dae_is_support(qm)) + return 0; + + val = readl(qm->io_base + DAE_AM_CTRL_GLOBAL_OFFSET); + val |= BIT(0); + writel(val, qm->io_base + DAE_AM_CTRL_GLOBAL_OFFSET); + + ret = readl_relaxed_poll_timeout(qm->io_base + DAE_AM_RETURN_OFFSET, + val, (val == DAE_AM_RETURN_MASK), + DAE_REG_RD_INTVRL_US, DAE_REG_RD_TMOUT_US); + if (ret) + dev_err(&qm->pdev->dev, "failed to close dae axi ooo!\n"); + + return ret; +} + +void hisi_dae_open_axi_master_ooo(struct hisi_qm *qm) +{ + u32 val; + + if (!dae_is_support(qm)) + return; + + val = readl(qm->io_base + DAE_AXI_CFG_OFFSET); + + writel(val & ~DAE_AXI_SHUTDOWN_EN_MASK, qm->io_base + DAE_AXI_CFG_OFFSET); + writel(val | DAE_AXI_SHUTDOWN_EN_MASK, qm->io_base + DAE_AXI_CFG_OFFSET); +} diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h index 2fecf346c3c9..9fb2a9c01132 100644 --- a/drivers/crypto/hisilicon/zip/zip.h +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -103,4 +103,12 @@ int zip_create_qps(struct hisi_qp **qps, int qp_num, int node); int hisi_zip_register_to_crypto(struct hisi_qm *qm); void hisi_zip_unregister_from_crypto(struct hisi_qm *qm); bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg); +int hisi_dae_set_user_domain(struct hisi_qm *qm); +int hisi_dae_set_alg(struct hisi_qm *qm); +void hisi_dae_hw_error_disable(struct hisi_qm *qm); +void hisi_dae_hw_error_enable(struct hisi_qm *qm); +void hisi_dae_open_axi_master_ooo(struct hisi_qm *qm); +int hisi_dae_close_axi_master_ooo(struct hisi_qm *qm); +bool hisi_dae_dev_is_abnormal(struct hisi_qm *qm); +enum acc_err_result hisi_dae_get_err_result(struct hisi_qm *qm); #endif diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 9239b251c2d7..d8ba23b7cc7d 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -582,7 +582,7 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) hisi_zip_enable_clock_gate(qm); - return 0; + return hisi_dae_set_user_domain(qm); } static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable) @@ -631,6 +631,8 @@ static void hisi_zip_hw_error_enable(struct hisi_qm *qm) /* enable ZIP hw error interrupts */ writel(0, qm->io_base + HZIP_CORE_INT_MASK_REG); + + hisi_dae_hw_error_enable(qm); } static void hisi_zip_hw_error_disable(struct hisi_qm *qm) @@ -643,6 +645,8 @@ static void hisi_zip_hw_error_disable(struct hisi_qm *qm) writel(ce | nfe | HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_MASK_REG); hisi_zip_master_ooo_ctrl(qm, false); + + hisi_dae_hw_error_disable(qm); } static inline struct hisi_qm *file_to_qm(struct ctrl_debug_file *file) @@ -1129,6 +1133,8 @@ static void hisi_zip_open_axi_master_ooo(struct hisi_qm *qm) writel(val | HZIP_AXI_SHUTDOWN_ENABLE, qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); + + hisi_dae_open_axi_master_ooo(qm); } static void hisi_zip_close_axi_master_ooo(struct hisi_qm *qm) @@ -1147,8 +1153,11 @@ static void hisi_zip_close_axi_master_ooo(struct hisi_qm *qm) static enum acc_err_result hisi_zip_get_err_result(struct hisi_qm *qm) { + enum acc_err_result zip_result = ACC_ERR_NONE; + enum acc_err_result dae_result; u32 err_status; + /* Get device hardware new error status */ err_status = hisi_zip_get_hw_err_status(qm); if (err_status) { if (err_status & qm->err_info.ecc_2bits_mask) @@ -1159,11 +1168,32 @@ static enum acc_err_result hisi_zip_get_err_result(struct hisi_qm *qm) /* Disable the same error reporting until device is recovered. */ hisi_zip_disable_error_report(qm, err_status); return ACC_ERR_NEED_RESET; + } else { + hisi_zip_clear_hw_err_status(qm, err_status); } - hisi_zip_clear_hw_err_status(qm, err_status); } - return ACC_ERR_RECOVERED; + dae_result = hisi_dae_get_err_result(qm); + + return (zip_result == ACC_ERR_NEED_RESET || + dae_result == ACC_ERR_NEED_RESET) ? + ACC_ERR_NEED_RESET : ACC_ERR_RECOVERED; +} + +static bool hisi_zip_dev_is_abnormal(struct hisi_qm *qm) +{ + u32 err_status; + + err_status = hisi_zip_get_hw_err_status(qm); + if (err_status & qm->err_info.dev_shutdown_mask) + return true; + + return hisi_dae_dev_is_abnormal(qm); +} + +static int hisi_zip_set_priv_status(struct hisi_qm *qm) +{ + return hisi_dae_close_axi_master_ooo(qm); } static void hisi_zip_err_info_init(struct hisi_qm *qm) @@ -1200,6 +1230,8 @@ static const struct hisi_qm_err_ini hisi_zip_err_ini = { .show_last_dfx_regs = hisi_zip_show_last_dfx_regs, .err_info_init = hisi_zip_err_info_init, .get_err_result = hisi_zip_get_err_result, + .set_priv_status = hisi_zip_set_priv_status, + .dev_is_abnormal = hisi_zip_dev_is_abnormal, }; static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) @@ -1264,7 +1296,6 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) int ret; qm->pdev = pdev; - qm->ver = pdev->revision; qm->mode = uacce_mode; qm->sqe_size = HZIP_SQE_SIZE; qm->dev_name = hisi_zip_name; @@ -1301,17 +1332,24 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) ret = zip_pre_store_cap_reg(qm); if (ret) { pci_err(qm->pdev, "Failed to pre-store capability registers!\n"); - hisi_qm_uninit(qm); - return ret; + goto err_qm_uninit; } alg_msk = qm->cap_tables.dev_cap_table[ZIP_ALG_BITMAP].cap_val; ret = hisi_qm_set_algs(qm, alg_msk, zip_dev_algs, ARRAY_SIZE(zip_dev_algs)); if (ret) { pci_err(qm->pdev, "Failed to set zip algs!\n"); - hisi_qm_uninit(qm); + goto err_qm_uninit; } + ret = hisi_dae_set_alg(qm); + if (ret) + goto err_qm_uninit; + + return 0; + +err_qm_uninit: + hisi_qm_uninit(qm); return ret; } diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index c3776b0de51d..990ea46955bb 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -1537,7 +1537,7 @@ static int iaa_comp_acompress(struct acomp_req *req) iaa_wq = idxd_wq_get_private(wq); if (!req->dst) { - gfp_t flags = req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; + gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; /* incompressible data will always be < 2 * slen */ req->dlen = 2 * req->slen; @@ -1619,7 +1619,7 @@ out: static int iaa_comp_adecompress_alloc_dest(struct acomp_req *req) { - gfp_t flags = req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; struct crypto_tfm *tfm = req->base.tfm; dma_addr_t src_addr, dst_addr; diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c index 9faef33e54bd..a17adc4beda2 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c @@ -420,6 +420,7 @@ static void adf_gen4_set_err_mask(struct adf_dev_err_mask *dev_err_mask) dev_err_mask->parerr_cpr_xlt_mask = ADF_420XX_PARITYERRORMASK_CPR_XLT_MASK; dev_err_mask->parerr_dcpr_ucs_mask = ADF_420XX_PARITYERRORMASK_DCPR_UCS_MASK; dev_err_mask->parerr_pke_mask = ADF_420XX_PARITYERRORMASK_PKE_MASK; + dev_err_mask->parerr_wat_wcp_mask = ADF_420XX_PARITYERRORMASK_WAT_WCP_MASK; dev_err_mask->ssmfeatren_mask = ADF_420XX_SSMFEATREN_MASK; } diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_ras.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_ras.c index 2dd3772bf58a..0f7f00a19e7d 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_ras.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_ras.c @@ -695,7 +695,7 @@ static bool adf_handle_slice_hang_error(struct adf_accel_dev *accel_dev, if (err_mask->parerr_wat_wcp_mask) adf_poll_slicehang_csr(accel_dev, csr, ADF_GEN4_SLICEHANGSTATUS_WAT_WCP, - "ath_cph"); + "wat_wcp"); return false; } @@ -1043,63 +1043,16 @@ static bool adf_handle_ssmcpppar_err(struct adf_accel_dev *accel_dev, return reset_required; } -static bool adf_handle_rf_parr_err(struct adf_accel_dev *accel_dev, +static void adf_handle_rf_parr_err(struct adf_accel_dev *accel_dev, void __iomem *csr, u32 iastatssm) { - struct adf_dev_err_mask *err_mask = GET_ERR_MASK(accel_dev); - u32 reg; - if (!(iastatssm & ADF_GEN4_IAINTSTATSSM_SSMSOFTERRORPARITY_BIT)) - return false; - - reg = ADF_CSR_RD(csr, ADF_GEN4_SSMSOFTERRORPARITY_SRC); - reg &= ADF_GEN4_SSMSOFTERRORPARITY_SRC_BIT; - if (reg) { - ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); - ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITY_SRC, reg); - } - - reg = ADF_CSR_RD(csr, ADF_GEN4_SSMSOFTERRORPARITY_ATH_CPH); - reg &= err_mask->parerr_ath_cph_mask; - if (reg) { - ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); - ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITY_ATH_CPH, reg); - } - - reg = ADF_CSR_RD(csr, ADF_GEN4_SSMSOFTERRORPARITY_CPR_XLT); - reg &= err_mask->parerr_cpr_xlt_mask; - if (reg) { - ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); - ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITY_CPR_XLT, reg); - } - - reg = ADF_CSR_RD(csr, ADF_GEN4_SSMSOFTERRORPARITY_DCPR_UCS); - reg &= err_mask->parerr_dcpr_ucs_mask; - if (reg) { - ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); - ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITY_DCPR_UCS, reg); - } - - reg = ADF_CSR_RD(csr, ADF_GEN4_SSMSOFTERRORPARITY_PKE); - reg &= err_mask->parerr_pke_mask; - if (reg) { - ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); - ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITY_PKE, reg); - } - - if (err_mask->parerr_wat_wcp_mask) { - reg = ADF_CSR_RD(csr, ADF_GEN4_SSMSOFTERRORPARITY_WAT_WCP); - reg &= err_mask->parerr_wat_wcp_mask; - if (reg) { - ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); - ADF_CSR_WR(csr, ADF_GEN4_SSMSOFTERRORPARITY_WAT_WCP, - reg); - } - } + return; + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); dev_err(&GET_DEV(accel_dev), "Slice ssm soft parity error reported"); - return false; + return; } static bool adf_handle_ser_err_ssmsh(struct adf_accel_dev *accel_dev, @@ -1171,8 +1124,8 @@ static bool adf_handle_iaintstatssm(struct adf_accel_dev *accel_dev, reset_required |= adf_handle_slice_hang_error(accel_dev, csr, iastatssm); reset_required |= adf_handle_spppar_err(accel_dev, csr, iastatssm); reset_required |= adf_handle_ssmcpppar_err(accel_dev, csr, iastatssm); - reset_required |= adf_handle_rf_parr_err(accel_dev, csr, iastatssm); reset_required |= adf_handle_ser_err_ssmsh(accel_dev, csr, iastatssm); + adf_handle_rf_parr_err(accel_dev, csr, iastatssm); ADF_CSR_WR(csr, ADF_GEN4_IAINTSTATSSM, iastatssm); diff --git a/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c index c8241f5a0a26..f20ae7e35a0d 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c +++ b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c @@ -473,22 +473,6 @@ unlock_and_exit: } DEFINE_SHOW_STORE_ATTRIBUTE(tl_control); -static int get_rp_index_from_file(const struct file *f, u8 *rp_id, u8 rp_num) -{ - char alpha; - u8 index; - int ret; - - ret = sscanf(f->f_path.dentry->d_name.name, ADF_TL_RP_REGS_FNAME, &alpha); - if (ret != 1) - return -EINVAL; - - index = ADF_TL_DBG_RP_INDEX_ALPHA(alpha); - *rp_id = index; - - return 0; -} - static int adf_tl_dbg_change_rp_index(struct adf_accel_dev *accel_dev, unsigned int new_rp_num, unsigned int rp_regs_index) @@ -611,18 +595,11 @@ static int tl_rp_data_show(struct seq_file *s, void *unused) { struct adf_accel_dev *accel_dev = s->private; u8 rp_regs_index; - u8 max_rp; - int ret; if (!accel_dev) return -EINVAL; - max_rp = GET_TL_DATA(accel_dev).max_rp; - ret = get_rp_index_from_file(s->file, &rp_regs_index, max_rp); - if (ret) { - dev_dbg(&GET_DEV(accel_dev), "invalid RP data file name\n"); - return ret; - } + rp_regs_index = debugfs_get_aux_num(s->file); return tl_print_rp_data(accel_dev, s, rp_regs_index); } @@ -635,7 +612,6 @@ static ssize_t tl_rp_data_write(struct file *file, const char __user *userbuf, struct adf_telemetry *telemetry; unsigned int new_rp_num; u8 rp_regs_index; - u8 max_rp; int ret; accel_dev = seq_f->private; @@ -643,15 +619,10 @@ static ssize_t tl_rp_data_write(struct file *file, const char __user *userbuf, return -EINVAL; telemetry = accel_dev->telemetry; - max_rp = GET_TL_DATA(accel_dev).max_rp; mutex_lock(&telemetry->wr_lock); - ret = get_rp_index_from_file(file, &rp_regs_index, max_rp); - if (ret) { - dev_dbg(&GET_DEV(accel_dev), "invalid RP data file name\n"); - goto unlock_and_exit; - } + rp_regs_index = debugfs_get_aux_num(file); ret = kstrtou32_from_user(userbuf, count, 10, &new_rp_num); if (ret) @@ -689,7 +660,8 @@ void adf_tl_dbgfs_add(struct adf_accel_dev *accel_dev) for (i = 0; i < max_rp; i++) { snprintf(name, sizeof(name), ADF_TL_RP_REGS_FNAME, ADF_TL_DBG_RP_ALPHA_INDEX(i)); - debugfs_create_file(name, 0644, dir, accel_dev, &tl_rp_data_fops); + debugfs_create_file_aux_num(name, 0644, dir, accel_dev, i, + &tl_rp_data_fops); } } diff --git a/drivers/crypto/n2_asm.S b/drivers/crypto/n2_asm.S deleted file mode 100644 index 9a67dbf340f4..000000000000 --- a/drivers/crypto/n2_asm.S +++ /dev/null @@ -1,96 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* n2_asm.S: Hypervisor calls for NCS support. - * - * Copyright (C) 2009 David S. Miller <davem@davemloft.net> - */ - -#include <linux/linkage.h> -#include <asm/hypervisor.h> -#include "n2_core.h" - - /* o0: queue type - * o1: RA of queue - * o2: num entries in queue - * o3: address of queue handle return - */ -ENTRY(sun4v_ncs_qconf) - mov HV_FAST_NCS_QCONF, %o5 - ta HV_FAST_TRAP - stx %o1, [%o3] - retl - nop -ENDPROC(sun4v_ncs_qconf) - - /* %o0: queue handle - * %o1: address of queue type return - * %o2: address of queue base address return - * %o3: address of queue num entries return - */ -ENTRY(sun4v_ncs_qinfo) - mov %o1, %g1 - mov %o2, %g2 - mov %o3, %g3 - mov HV_FAST_NCS_QINFO, %o5 - ta HV_FAST_TRAP - stx %o1, [%g1] - stx %o2, [%g2] - stx %o3, [%g3] - retl - nop -ENDPROC(sun4v_ncs_qinfo) - - /* %o0: queue handle - * %o1: address of head offset return - */ -ENTRY(sun4v_ncs_gethead) - mov %o1, %o2 - mov HV_FAST_NCS_GETHEAD, %o5 - ta HV_FAST_TRAP - stx %o1, [%o2] - retl - nop -ENDPROC(sun4v_ncs_gethead) - - /* %o0: queue handle - * %o1: address of tail offset return - */ -ENTRY(sun4v_ncs_gettail) - mov %o1, %o2 - mov HV_FAST_NCS_GETTAIL, %o5 - ta HV_FAST_TRAP - stx %o1, [%o2] - retl - nop -ENDPROC(sun4v_ncs_gettail) - - /* %o0: queue handle - * %o1: new tail offset - */ -ENTRY(sun4v_ncs_settail) - mov HV_FAST_NCS_SETTAIL, %o5 - ta HV_FAST_TRAP - retl - nop -ENDPROC(sun4v_ncs_settail) - - /* %o0: queue handle - * %o1: address of devino return - */ -ENTRY(sun4v_ncs_qhandle_to_devino) - mov %o1, %o2 - mov HV_FAST_NCS_QHANDLE_TO_DEVINO, %o5 - ta HV_FAST_TRAP - stx %o1, [%o2] - retl - nop -ENDPROC(sun4v_ncs_qhandle_to_devino) - - /* %o0: queue handle - * %o1: new head offset - */ -ENTRY(sun4v_ncs_sethead_marker) - mov HV_FAST_NCS_SETHEAD_MARKER, %o5 - ta HV_FAST_TRAP - retl - nop -ENDPROC(sun4v_ncs_sethead_marker) diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c deleted file mode 100644 index 14c302d2db79..000000000000 --- a/drivers/crypto/n2_core.c +++ /dev/null @@ -1,2168 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support. - * - * Copyright (C) 2010, 2011 David S. Miller <davem@davemloft.net> - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/of.h> -#include <linux/of_address.h> -#include <linux/platform_device.h> -#include <linux/cpumask.h> -#include <linux/slab.h> -#include <linux/interrupt.h> -#include <linux/crypto.h> -#include <crypto/md5.h> -#include <crypto/sha1.h> -#include <crypto/sha2.h> -#include <crypto/aes.h> -#include <crypto/internal/des.h> -#include <linux/mutex.h> -#include <linux/delay.h> -#include <linux/sched.h> - -#include <crypto/internal/hash.h> -#include <crypto/internal/skcipher.h> -#include <crypto/scatterwalk.h> -#include <crypto/algapi.h> - -#include <asm/hypervisor.h> -#include <asm/mdesc.h> - -#include "n2_core.h" - -#define DRV_MODULE_NAME "n2_crypto" -#define DRV_MODULE_VERSION "0.2" -#define DRV_MODULE_RELDATE "July 28, 2011" - -static const char version[] = - DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; - -MODULE_AUTHOR("David S. Miller <davem@davemloft.net>"); -MODULE_DESCRIPTION("Niagara2 Crypto driver"); -MODULE_LICENSE("GPL"); -MODULE_VERSION(DRV_MODULE_VERSION); - -#define N2_CRA_PRIORITY 200 - -static DEFINE_MUTEX(spu_lock); - -struct spu_queue { - cpumask_t sharing; - unsigned long qhandle; - - spinlock_t lock; - u8 q_type; - void *q; - unsigned long head; - unsigned long tail; - struct list_head jobs; - - unsigned long devino; - - char irq_name[32]; - unsigned int irq; - - struct list_head list; -}; - -struct spu_qreg { - struct spu_queue *queue; - unsigned long type; -}; - -static struct spu_queue **cpu_to_cwq; -static struct spu_queue **cpu_to_mau; - -static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off) -{ - if (q->q_type == HV_NCS_QTYPE_MAU) { - off += MAU_ENTRY_SIZE; - if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES)) - off = 0; - } else { - off += CWQ_ENTRY_SIZE; - if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES)) - off = 0; - } - return off; -} - -struct n2_request_common { - struct list_head entry; - unsigned int offset; -}; -#define OFFSET_NOT_RUNNING (~(unsigned int)0) - -/* An async job request records the final tail value it used in - * n2_request_common->offset, test to see if that offset is in - * the range old_head, new_head, inclusive. - */ -static inline bool job_finished(struct spu_queue *q, unsigned int offset, - unsigned long old_head, unsigned long new_head) -{ - if (old_head <= new_head) { - if (offset > old_head && offset <= new_head) - return true; - } else { - if (offset > old_head || offset <= new_head) - return true; - } - return false; -} - -/* When the HEAD marker is unequal to the actual HEAD, we get - * a virtual device INO interrupt. We should process the - * completed CWQ entries and adjust the HEAD marker to clear - * the IRQ. - */ -static irqreturn_t cwq_intr(int irq, void *dev_id) -{ - unsigned long off, new_head, hv_ret; - struct spu_queue *q = dev_id; - - pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n", - smp_processor_id(), q->qhandle); - - spin_lock(&q->lock); - - hv_ret = sun4v_ncs_gethead(q->qhandle, &new_head); - - pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n", - smp_processor_id(), new_head, hv_ret); - - for (off = q->head; off != new_head; off = spu_next_offset(q, off)) { - /* XXX ... XXX */ - } - - hv_ret = sun4v_ncs_sethead_marker(q->qhandle, new_head); - if (hv_ret == HV_EOK) - q->head = new_head; - - spin_unlock(&q->lock); - - return IRQ_HANDLED; -} - -static irqreturn_t mau_intr(int irq, void *dev_id) -{ - struct spu_queue *q = dev_id; - unsigned long head, hv_ret; - - spin_lock(&q->lock); - - pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n", - smp_processor_id(), q->qhandle); - - hv_ret = sun4v_ncs_gethead(q->qhandle, &head); - - pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n", - smp_processor_id(), head, hv_ret); - - sun4v_ncs_sethead_marker(q->qhandle, head); - - spin_unlock(&q->lock); - - return IRQ_HANDLED; -} - -static void *spu_queue_next(struct spu_queue *q, void *cur) -{ - return q->q + spu_next_offset(q, cur - q->q); -} - -static int spu_queue_num_free(struct spu_queue *q) -{ - unsigned long head = q->head; - unsigned long tail = q->tail; - unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES); - unsigned long diff; - - if (head > tail) - diff = head - tail; - else - diff = (end - tail) + head; - - return (diff / CWQ_ENTRY_SIZE) - 1; -} - -static void *spu_queue_alloc(struct spu_queue *q, int num_entries) -{ - int avail = spu_queue_num_free(q); - - if (avail >= num_entries) - return q->q + q->tail; - - return NULL; -} - -static unsigned long spu_queue_submit(struct spu_queue *q, void *last) -{ - unsigned long hv_ret, new_tail; - - new_tail = spu_next_offset(q, last - q->q); - - hv_ret = sun4v_ncs_settail(q->qhandle, new_tail); - if (hv_ret == HV_EOK) - q->tail = new_tail; - return hv_ret; -} - -static u64 control_word_base(unsigned int len, unsigned int hmac_key_len, - int enc_type, int auth_type, - unsigned int hash_len, - bool sfas, bool sob, bool eob, bool encrypt, - int opcode) -{ - u64 word = (len - 1) & CONTROL_LEN; - - word |= ((u64) opcode << CONTROL_OPCODE_SHIFT); - word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT); - word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT); - if (sfas) - word |= CONTROL_STORE_FINAL_AUTH_STATE; - if (sob) - word |= CONTROL_START_OF_BLOCK; - if (eob) - word |= CONTROL_END_OF_BLOCK; - if (encrypt) - word |= CONTROL_ENCRYPT; - if (hmac_key_len) - word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT; - if (hash_len) - word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT; - - return word; -} - -#if 0 -static inline bool n2_should_run_async(struct spu_queue *qp, int this_len) -{ - if (this_len >= 64 || - qp->head != qp->tail) - return true; - return false; -} -#endif - -struct n2_ahash_alg { - struct list_head entry; - const u8 *hash_zero; - const u8 *hash_init; - u8 hw_op_hashsz; - u8 digest_size; - u8 auth_type; - u8 hmac_type; - struct ahash_alg alg; -}; - -static inline struct n2_ahash_alg *n2_ahash_alg(struct crypto_tfm *tfm) -{ - struct crypto_alg *alg = tfm->__crt_alg; - struct ahash_alg *ahash_alg; - - ahash_alg = container_of(alg, struct ahash_alg, halg.base); - - return container_of(ahash_alg, struct n2_ahash_alg, alg); -} - -struct n2_hmac_alg { - const char *child_alg; - struct n2_ahash_alg derived; -}; - -static inline struct n2_hmac_alg *n2_hmac_alg(struct crypto_tfm *tfm) -{ - struct crypto_alg *alg = tfm->__crt_alg; - struct ahash_alg *ahash_alg; - - ahash_alg = container_of(alg, struct ahash_alg, halg.base); - - return container_of(ahash_alg, struct n2_hmac_alg, derived.alg); -} - -struct n2_hash_ctx { - struct crypto_ahash *fallback_tfm; -}; - -#define N2_HASH_KEY_MAX 32 /* HW limit for all HMAC requests */ - -struct n2_hmac_ctx { - struct n2_hash_ctx base; - - struct crypto_shash *child_shash; - - int hash_key_len; - unsigned char hash_key[N2_HASH_KEY_MAX]; -}; - -struct n2_hash_req_ctx { - union { - struct md5_state md5; - struct sha1_state sha1; - struct sha256_state sha256; - } u; - - struct ahash_request fallback_req; -}; - -static int n2_hash_async_init(struct ahash_request *req) -{ - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); - - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - - return crypto_ahash_init(&rctx->fallback_req); -} - -static int n2_hash_async_update(struct ahash_request *req) -{ - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); - - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - - return crypto_ahash_update(&rctx->fallback_req); -} - -static int n2_hash_async_final(struct ahash_request *req) -{ - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); - - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = req->result; - - return crypto_ahash_final(&rctx->fallback_req); -} - -static int n2_hash_async_finup(struct ahash_request *req) -{ - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); - - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; - - return crypto_ahash_finup(&rctx->fallback_req); -} - -static int n2_hash_async_noimport(struct ahash_request *req, const void *in) -{ - return -ENOSYS; -} - -static int n2_hash_async_noexport(struct ahash_request *req, void *out) -{ - return -ENOSYS; -} - -static int n2_hash_cra_init(struct crypto_tfm *tfm) -{ - const char *fallback_driver_name = crypto_tfm_alg_name(tfm); - struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); - struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash); - struct crypto_ahash *fallback_tfm; - int err; - - fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(fallback_tfm)) { - pr_warn("Fallback driver '%s' could not be loaded!\n", - fallback_driver_name); - err = PTR_ERR(fallback_tfm); - goto out; - } - - crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) + - crypto_ahash_reqsize(fallback_tfm))); - - ctx->fallback_tfm = fallback_tfm; - return 0; - -out: - return err; -} - -static void n2_hash_cra_exit(struct crypto_tfm *tfm) -{ - struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); - struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash); - - crypto_free_ahash(ctx->fallback_tfm); -} - -static int n2_hmac_cra_init(struct crypto_tfm *tfm) -{ - const char *fallback_driver_name = crypto_tfm_alg_name(tfm); - struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); - struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash); - struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm); - struct crypto_ahash *fallback_tfm; - struct crypto_shash *child_shash; - int err; - - fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(fallback_tfm)) { - pr_warn("Fallback driver '%s' could not be loaded!\n", - fallback_driver_name); - err = PTR_ERR(fallback_tfm); - goto out; - } - - child_shash = crypto_alloc_shash(n2alg->child_alg, 0, 0); - if (IS_ERR(child_shash)) { - pr_warn("Child shash '%s' could not be loaded!\n", - n2alg->child_alg); - err = PTR_ERR(child_shash); - goto out_free_fallback; - } - - crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) + - crypto_ahash_reqsize(fallback_tfm))); - - ctx->child_shash = child_shash; - ctx->base.fallback_tfm = fallback_tfm; - return 0; - -out_free_fallback: - crypto_free_ahash(fallback_tfm); - -out: - return err; -} - -static void n2_hmac_cra_exit(struct crypto_tfm *tfm) -{ - struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); - struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash); - - crypto_free_ahash(ctx->base.fallback_tfm); - crypto_free_shash(ctx->child_shash); -} - -static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key, - unsigned int keylen) -{ - struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm); - struct crypto_shash *child_shash = ctx->child_shash; - struct crypto_ahash *fallback_tfm; - int err, bs, ds; - - fallback_tfm = ctx->base.fallback_tfm; - err = crypto_ahash_setkey(fallback_tfm, key, keylen); - if (err) - return err; - - bs = crypto_shash_blocksize(child_shash); - ds = crypto_shash_digestsize(child_shash); - BUG_ON(ds > N2_HASH_KEY_MAX); - if (keylen > bs) { - err = crypto_shash_tfm_digest(child_shash, key, keylen, - ctx->hash_key); - if (err) - return err; - keylen = ds; - } else if (keylen <= N2_HASH_KEY_MAX) - memcpy(ctx->hash_key, key, keylen); - - ctx->hash_key_len = keylen; - - return err; -} - -static unsigned long wait_for_tail(struct spu_queue *qp) -{ - unsigned long head, hv_ret; - - do { - hv_ret = sun4v_ncs_gethead(qp->qhandle, &head); - if (hv_ret != HV_EOK) { - pr_err("Hypervisor error on gethead\n"); - break; - } - if (head == qp->tail) { - qp->head = head; - break; - } - } while (1); - return hv_ret; -} - -static unsigned long submit_and_wait_for_tail(struct spu_queue *qp, - struct cwq_initial_entry *ent) -{ - unsigned long hv_ret = spu_queue_submit(qp, ent); - - if (hv_ret == HV_EOK) - hv_ret = wait_for_tail(qp); - - return hv_ret; -} - -static int n2_do_async_digest(struct ahash_request *req, - unsigned int auth_type, unsigned int digest_size, - unsigned int result_size, void *hash_loc, - unsigned long auth_key, unsigned int auth_key_len) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct cwq_initial_entry *ent; - struct crypto_hash_walk walk; - struct spu_queue *qp; - unsigned long flags; - int err = -ENODEV; - int nbytes, cpu; - - /* The total effective length of the operation may not - * exceed 2^16. - */ - if (unlikely(req->nbytes > (1 << 16))) { - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); - - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = - req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; - - return crypto_ahash_digest(&rctx->fallback_req); - } - - nbytes = crypto_hash_walk_first(req, &walk); - - cpu = get_cpu(); - qp = cpu_to_cwq[cpu]; - if (!qp) - goto out; - - spin_lock_irqsave(&qp->lock, flags); - - /* XXX can do better, improve this later by doing a by-hand scatterlist - * XXX walk, etc. - */ - ent = qp->q + qp->tail; - - ent->control = control_word_base(nbytes, auth_key_len, 0, - auth_type, digest_size, - false, true, false, false, - OPCODE_INPLACE_BIT | - OPCODE_AUTH_MAC); - ent->src_addr = __pa(walk.data); - ent->auth_key_addr = auth_key; - ent->auth_iv_addr = __pa(hash_loc); - ent->final_auth_state_addr = 0UL; - ent->enc_key_addr = 0UL; - ent->enc_iv_addr = 0UL; - ent->dest_addr = __pa(hash_loc); - - nbytes = crypto_hash_walk_done(&walk, 0); - while (nbytes > 0) { - ent = spu_queue_next(qp, ent); - - ent->control = (nbytes - 1); - ent->src_addr = __pa(walk.data); - ent->auth_key_addr = 0UL; - ent->auth_iv_addr = 0UL; - ent->final_auth_state_addr = 0UL; - ent->enc_key_addr = 0UL; - ent->enc_iv_addr = 0UL; - ent->dest_addr = 0UL; - - nbytes = crypto_hash_walk_done(&walk, 0); - } - ent->control |= CONTROL_END_OF_BLOCK; - - if (submit_and_wait_for_tail(qp, ent) != HV_EOK) - err = -EINVAL; - else - err = 0; - - spin_unlock_irqrestore(&qp->lock, flags); - - if (!err) - memcpy(req->result, hash_loc, result_size); -out: - put_cpu(); - - return err; -} - -static int n2_hash_async_digest(struct ahash_request *req) -{ - struct n2_ahash_alg *n2alg = n2_ahash_alg(req->base.tfm); - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - int ds; - - ds = n2alg->digest_size; - if (unlikely(req->nbytes == 0)) { - memcpy(req->result, n2alg->hash_zero, ds); - return 0; - } - memcpy(&rctx->u, n2alg->hash_init, n2alg->hw_op_hashsz); - - return n2_do_async_digest(req, n2alg->auth_type, - n2alg->hw_op_hashsz, ds, - &rctx->u, 0UL, 0); -} - -static int n2_hmac_async_digest(struct ahash_request *req) -{ - struct n2_hmac_alg *n2alg = n2_hmac_alg(req->base.tfm); - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm); - int ds; - - ds = n2alg->derived.digest_size; - if (unlikely(req->nbytes == 0) || - unlikely(ctx->hash_key_len > N2_HASH_KEY_MAX)) { - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); - - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = - req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; - - return crypto_ahash_digest(&rctx->fallback_req); - } - memcpy(&rctx->u, n2alg->derived.hash_init, - n2alg->derived.hw_op_hashsz); - - return n2_do_async_digest(req, n2alg->derived.hmac_type, - n2alg->derived.hw_op_hashsz, ds, - &rctx->u, - __pa(&ctx->hash_key), - ctx->hash_key_len); -} - -struct n2_skcipher_context { - int key_len; - int enc_type; - union { - u8 aes[AES_MAX_KEY_SIZE]; - u8 des[DES_KEY_SIZE]; - u8 des3[3 * DES_KEY_SIZE]; - } key; -}; - -#define N2_CHUNK_ARR_LEN 16 - -struct n2_crypto_chunk { - struct list_head entry; - unsigned long iv_paddr : 44; - unsigned long arr_len : 20; - unsigned long dest_paddr; - unsigned long dest_final; - struct { - unsigned long src_paddr : 44; - unsigned long src_len : 20; - } arr[N2_CHUNK_ARR_LEN]; -}; - -struct n2_request_context { - struct skcipher_walk walk; - struct list_head chunk_list; - struct n2_crypto_chunk chunk; - u8 temp_iv[16]; -}; - -/* The SPU allows some level of flexibility for partial cipher blocks - * being specified in a descriptor. - * - * It merely requires that every descriptor's length field is at least - * as large as the cipher block size. This means that a cipher block - * can span at most 2 descriptors. However, this does not allow a - * partial block to span into the final descriptor as that would - * violate the rule (since every descriptor's length must be at lest - * the block size). So, for example, assuming an 8 byte block size: - * - * 0xe --> 0xa --> 0x8 - * - * is a valid length sequence, whereas: - * - * 0xe --> 0xb --> 0x7 - * - * is not a valid sequence. - */ - -struct n2_skcipher_alg { - struct list_head entry; - u8 enc_type; - struct skcipher_alg skcipher; -}; - -static inline struct n2_skcipher_alg *n2_skcipher_alg(struct crypto_skcipher *tfm) -{ - struct skcipher_alg *alg = crypto_skcipher_alg(tfm); - - return container_of(alg, struct n2_skcipher_alg, skcipher); -} - -static int n2_aes_setkey(struct crypto_skcipher *skcipher, const u8 *key, - unsigned int keylen) -{ - struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher); - struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm); - struct n2_skcipher_alg *n2alg = n2_skcipher_alg(skcipher); - - ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK); - - switch (keylen) { - case AES_KEYSIZE_128: - ctx->enc_type |= ENC_TYPE_ALG_AES128; - break; - case AES_KEYSIZE_192: - ctx->enc_type |= ENC_TYPE_ALG_AES192; - break; - case AES_KEYSIZE_256: - ctx->enc_type |= ENC_TYPE_ALG_AES256; - break; - default: - return -EINVAL; - } - - ctx->key_len = keylen; - memcpy(ctx->key.aes, key, keylen); - return 0; -} - -static int n2_des_setkey(struct crypto_skcipher *skcipher, const u8 *key, - unsigned int keylen) -{ - struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher); - struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm); - struct n2_skcipher_alg *n2alg = n2_skcipher_alg(skcipher); - int err; - - err = verify_skcipher_des_key(skcipher, key); - if (err) - return err; - - ctx->enc_type = n2alg->enc_type; - - ctx->key_len = keylen; - memcpy(ctx->key.des, key, keylen); - return 0; -} - -static int n2_3des_setkey(struct crypto_skcipher *skcipher, const u8 *key, - unsigned int keylen) -{ - struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher); - struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm); - struct n2_skcipher_alg *n2alg = n2_skcipher_alg(skcipher); - int err; - - err = verify_skcipher_des3_key(skcipher, key); - if (err) - return err; - - ctx->enc_type = n2alg->enc_type; - - ctx->key_len = keylen; - memcpy(ctx->key.des3, key, keylen); - return 0; -} - -static inline int skcipher_descriptor_len(int nbytes, unsigned int block_size) -{ - int this_len = nbytes; - - this_len -= (nbytes & (block_size - 1)); - return this_len > (1 << 16) ? (1 << 16) : this_len; -} - -static int __n2_crypt_chunk(struct crypto_skcipher *skcipher, - struct n2_crypto_chunk *cp, - struct spu_queue *qp, bool encrypt) -{ - struct n2_skcipher_context *ctx = crypto_skcipher_ctx(skcipher); - struct cwq_initial_entry *ent; - bool in_place; - int i; - - ent = spu_queue_alloc(qp, cp->arr_len); - if (!ent) { - pr_info("queue_alloc() of %d fails\n", - cp->arr_len); - return -EBUSY; - } - - in_place = (cp->dest_paddr == cp->arr[0].src_paddr); - - ent->control = control_word_base(cp->arr[0].src_len, - 0, ctx->enc_type, 0, 0, - false, true, false, encrypt, - OPCODE_ENCRYPT | - (in_place ? OPCODE_INPLACE_BIT : 0)); - ent->src_addr = cp->arr[0].src_paddr; - ent->auth_key_addr = 0UL; - ent->auth_iv_addr = 0UL; - ent->final_auth_state_addr = 0UL; - ent->enc_key_addr = __pa(&ctx->key); - ent->enc_iv_addr = cp->iv_paddr; - ent->dest_addr = (in_place ? 0UL : cp->dest_paddr); - - for (i = 1; i < cp->arr_len; i++) { - ent = spu_queue_next(qp, ent); - - ent->control = cp->arr[i].src_len - 1; - ent->src_addr = cp->arr[i].src_paddr; - ent->auth_key_addr = 0UL; - ent->auth_iv_addr = 0UL; - ent->final_auth_state_addr = 0UL; - ent->enc_key_addr = 0UL; - ent->enc_iv_addr = 0UL; - ent->dest_addr = 0UL; - } - ent->control |= CONTROL_END_OF_BLOCK; - - return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0; -} - -static int n2_compute_chunks(struct skcipher_request *req) -{ - struct n2_request_context *rctx = skcipher_request_ctx(req); - struct skcipher_walk *walk = &rctx->walk; - struct n2_crypto_chunk *chunk; - unsigned long dest_prev; - unsigned int tot_len; - bool prev_in_place; - int err, nbytes; - - err = skcipher_walk_async(walk, req); - if (err) - return err; - - INIT_LIST_HEAD(&rctx->chunk_list); - - chunk = &rctx->chunk; - INIT_LIST_HEAD(&chunk->entry); - - chunk->iv_paddr = 0UL; - chunk->arr_len = 0; - chunk->dest_paddr = 0UL; - - prev_in_place = false; - dest_prev = ~0UL; - tot_len = 0; - - while ((nbytes = walk->nbytes) != 0) { - unsigned long dest_paddr, src_paddr; - bool in_place; - int this_len; - - src_paddr = (page_to_phys(walk->src.phys.page) + - walk->src.phys.offset); - dest_paddr = (page_to_phys(walk->dst.phys.page) + - walk->dst.phys.offset); - in_place = (src_paddr == dest_paddr); - this_len = skcipher_descriptor_len(nbytes, walk->blocksize); - - if (chunk->arr_len != 0) { - if (in_place != prev_in_place || - (!prev_in_place && - dest_paddr != dest_prev) || - chunk->arr_len == N2_CHUNK_ARR_LEN || - tot_len + this_len > (1 << 16)) { - chunk->dest_final = dest_prev; - list_add_tail(&chunk->entry, - &rctx->chunk_list); - chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC); - if (!chunk) { - err = -ENOMEM; - break; - } - INIT_LIST_HEAD(&chunk->entry); - } - } - if (chunk->arr_len == 0) { - chunk->dest_paddr = dest_paddr; - tot_len = 0; - } - chunk->arr[chunk->arr_len].src_paddr = src_paddr; - chunk->arr[chunk->arr_len].src_len = this_len; - chunk->arr_len++; - - dest_prev = dest_paddr + this_len; - prev_in_place = in_place; - tot_len += this_len; - - err = skcipher_walk_done(walk, nbytes - this_len); - if (err) - break; - } - if (!err && chunk->arr_len != 0) { - chunk->dest_final = dest_prev; - list_add_tail(&chunk->entry, &rctx->chunk_list); - } - - return err; -} - -static void n2_chunk_complete(struct skcipher_request *req, void *final_iv) -{ - struct n2_request_context *rctx = skcipher_request_ctx(req); - struct n2_crypto_chunk *c, *tmp; - - if (final_iv) - memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize); - - list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) { - list_del(&c->entry); - if (unlikely(c != &rctx->chunk)) - kfree(c); - } - -} - -static int n2_do_ecb(struct skcipher_request *req, bool encrypt) -{ - struct n2_request_context *rctx = skcipher_request_ctx(req); - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - int err = n2_compute_chunks(req); - struct n2_crypto_chunk *c, *tmp; - unsigned long flags, hv_ret; - struct spu_queue *qp; - - if (err) - return err; - - qp = cpu_to_cwq[get_cpu()]; - err = -ENODEV; - if (!qp) - goto out; - - spin_lock_irqsave(&qp->lock, flags); - - list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) { - err = __n2_crypt_chunk(tfm, c, qp, encrypt); - if (err) - break; - list_del(&c->entry); - if (unlikely(c != &rctx->chunk)) - kfree(c); - } - if (!err) { - hv_ret = wait_for_tail(qp); - if (hv_ret != HV_EOK) - err = -EINVAL; - } - - spin_unlock_irqrestore(&qp->lock, flags); - -out: - put_cpu(); - - n2_chunk_complete(req, NULL); - return err; -} - -static int n2_encrypt_ecb(struct skcipher_request *req) -{ - return n2_do_ecb(req, true); -} - -static int n2_decrypt_ecb(struct skcipher_request *req) -{ - return n2_do_ecb(req, false); -} - -static int n2_do_chaining(struct skcipher_request *req, bool encrypt) -{ - struct n2_request_context *rctx = skcipher_request_ctx(req); - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - unsigned long flags, hv_ret, iv_paddr; - int err = n2_compute_chunks(req); - struct n2_crypto_chunk *c, *tmp; - struct spu_queue *qp; - void *final_iv_addr; - - final_iv_addr = NULL; - - if (err) - return err; - - qp = cpu_to_cwq[get_cpu()]; - err = -ENODEV; - if (!qp) - goto out; - - spin_lock_irqsave(&qp->lock, flags); - - if (encrypt) { - iv_paddr = __pa(rctx->walk.iv); - list_for_each_entry_safe(c, tmp, &rctx->chunk_list, - entry) { - c->iv_paddr = iv_paddr; - err = __n2_crypt_chunk(tfm, c, qp, true); - if (err) - break; - iv_paddr = c->dest_final - rctx->walk.blocksize; - list_del(&c->entry); - if (unlikely(c != &rctx->chunk)) - kfree(c); - } - final_iv_addr = __va(iv_paddr); - } else { - list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list, - entry) { - if (c == &rctx->chunk) { - iv_paddr = __pa(rctx->walk.iv); - } else { - iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr + - tmp->arr[tmp->arr_len-1].src_len - - rctx->walk.blocksize); - } - if (!final_iv_addr) { - unsigned long pa; - - pa = (c->arr[c->arr_len-1].src_paddr + - c->arr[c->arr_len-1].src_len - - rctx->walk.blocksize); - final_iv_addr = rctx->temp_iv; - memcpy(rctx->temp_iv, __va(pa), - rctx->walk.blocksize); - } - c->iv_paddr = iv_paddr; - err = __n2_crypt_chunk(tfm, c, qp, false); - if (err) - break; - list_del(&c->entry); - if (unlikely(c != &rctx->chunk)) - kfree(c); - } - } - if (!err) { - hv_ret = wait_for_tail(qp); - if (hv_ret != HV_EOK) - err = -EINVAL; - } - - spin_unlock_irqrestore(&qp->lock, flags); - -out: - put_cpu(); - - n2_chunk_complete(req, err ? NULL : final_iv_addr); - return err; -} - -static int n2_encrypt_chaining(struct skcipher_request *req) -{ - return n2_do_chaining(req, true); -} - -static int n2_decrypt_chaining(struct skcipher_request *req) -{ - return n2_do_chaining(req, false); -} - -struct n2_skcipher_tmpl { - const char *name; - const char *drv_name; - u8 block_size; - u8 enc_type; - struct skcipher_alg skcipher; -}; - -static const struct n2_skcipher_tmpl skcipher_tmpls[] = { - /* DES: ECB CBC and CFB are supported */ - { .name = "ecb(des)", - .drv_name = "ecb-des", - .block_size = DES_BLOCK_SIZE, - .enc_type = (ENC_TYPE_ALG_DES | - ENC_TYPE_CHAINING_ECB), - .skcipher = { - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .setkey = n2_des_setkey, - .encrypt = n2_encrypt_ecb, - .decrypt = n2_decrypt_ecb, - }, - }, - { .name = "cbc(des)", - .drv_name = "cbc-des", - .block_size = DES_BLOCK_SIZE, - .enc_type = (ENC_TYPE_ALG_DES | - ENC_TYPE_CHAINING_CBC), - .skcipher = { - .ivsize = DES_BLOCK_SIZE, - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .setkey = n2_des_setkey, - .encrypt = n2_encrypt_chaining, - .decrypt = n2_decrypt_chaining, - }, - }, - - /* 3DES: ECB CBC and CFB are supported */ - { .name = "ecb(des3_ede)", - .drv_name = "ecb-3des", - .block_size = DES_BLOCK_SIZE, - .enc_type = (ENC_TYPE_ALG_3DES | - ENC_TYPE_CHAINING_ECB), - .skcipher = { - .min_keysize = 3 * DES_KEY_SIZE, - .max_keysize = 3 * DES_KEY_SIZE, - .setkey = n2_3des_setkey, - .encrypt = n2_encrypt_ecb, - .decrypt = n2_decrypt_ecb, - }, - }, - { .name = "cbc(des3_ede)", - .drv_name = "cbc-3des", - .block_size = DES_BLOCK_SIZE, - .enc_type = (ENC_TYPE_ALG_3DES | - ENC_TYPE_CHAINING_CBC), - .skcipher = { - .ivsize = DES_BLOCK_SIZE, - .min_keysize = 3 * DES_KEY_SIZE, - .max_keysize = 3 * DES_KEY_SIZE, - .setkey = n2_3des_setkey, - .encrypt = n2_encrypt_chaining, - .decrypt = n2_decrypt_chaining, - }, - }, - - /* AES: ECB CBC and CTR are supported */ - { .name = "ecb(aes)", - .drv_name = "ecb-aes", - .block_size = AES_BLOCK_SIZE, - .enc_type = (ENC_TYPE_ALG_AES128 | - ENC_TYPE_CHAINING_ECB), - .skcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = n2_aes_setkey, - .encrypt = n2_encrypt_ecb, - .decrypt = n2_decrypt_ecb, - }, - }, - { .name = "cbc(aes)", - .drv_name = "cbc-aes", - .block_size = AES_BLOCK_SIZE, - .enc_type = (ENC_TYPE_ALG_AES128 | - ENC_TYPE_CHAINING_CBC), - .skcipher = { - .ivsize = AES_BLOCK_SIZE, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = n2_aes_setkey, - .encrypt = n2_encrypt_chaining, - .decrypt = n2_decrypt_chaining, - }, - }, - { .name = "ctr(aes)", - .drv_name = "ctr-aes", - .block_size = AES_BLOCK_SIZE, - .enc_type = (ENC_TYPE_ALG_AES128 | - ENC_TYPE_CHAINING_COUNTER), - .skcipher = { - .ivsize = AES_BLOCK_SIZE, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = n2_aes_setkey, - .encrypt = n2_encrypt_chaining, - .decrypt = n2_encrypt_chaining, - }, - }, - -}; -#define NUM_CIPHER_TMPLS ARRAY_SIZE(skcipher_tmpls) - -static LIST_HEAD(skcipher_algs); - -struct n2_hash_tmpl { - const char *name; - const u8 *hash_zero; - const u8 *hash_init; - u8 hw_op_hashsz; - u8 digest_size; - u8 statesize; - u8 block_size; - u8 auth_type; - u8 hmac_type; -}; - -static const __le32 n2_md5_init[MD5_HASH_WORDS] = { - cpu_to_le32(MD5_H0), - cpu_to_le32(MD5_H1), - cpu_to_le32(MD5_H2), - cpu_to_le32(MD5_H3), -}; -static const u32 n2_sha1_init[SHA1_DIGEST_SIZE / 4] = { - SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, -}; -static const u32 n2_sha256_init[SHA256_DIGEST_SIZE / 4] = { - SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, - SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7, -}; -static const u32 n2_sha224_init[SHA256_DIGEST_SIZE / 4] = { - SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3, - SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7, -}; - -static const struct n2_hash_tmpl hash_tmpls[] = { - { .name = "md5", - .hash_zero = md5_zero_message_hash, - .hash_init = (u8 *)n2_md5_init, - .auth_type = AUTH_TYPE_MD5, - .hmac_type = AUTH_TYPE_HMAC_MD5, - .hw_op_hashsz = MD5_DIGEST_SIZE, - .digest_size = MD5_DIGEST_SIZE, - .statesize = sizeof(struct md5_state), - .block_size = MD5_HMAC_BLOCK_SIZE }, - { .name = "sha1", - .hash_zero = sha1_zero_message_hash, - .hash_init = (u8 *)n2_sha1_init, - .auth_type = AUTH_TYPE_SHA1, - .hmac_type = AUTH_TYPE_HMAC_SHA1, - .hw_op_hashsz = SHA1_DIGEST_SIZE, - .digest_size = SHA1_DIGEST_SIZE, - .statesize = sizeof(struct sha1_state), - .block_size = SHA1_BLOCK_SIZE }, - { .name = "sha256", - .hash_zero = sha256_zero_message_hash, - .hash_init = (u8 *)n2_sha256_init, - .auth_type = AUTH_TYPE_SHA256, - .hmac_type = AUTH_TYPE_HMAC_SHA256, - .hw_op_hashsz = SHA256_DIGEST_SIZE, - .digest_size = SHA256_DIGEST_SIZE, - .statesize = sizeof(struct sha256_state), - .block_size = SHA256_BLOCK_SIZE }, - { .name = "sha224", - .hash_zero = sha224_zero_message_hash, - .hash_init = (u8 *)n2_sha224_init, - .auth_type = AUTH_TYPE_SHA256, - .hmac_type = AUTH_TYPE_RESERVED, - .hw_op_hashsz = SHA256_DIGEST_SIZE, - .digest_size = SHA224_DIGEST_SIZE, - .statesize = sizeof(struct sha256_state), - .block_size = SHA224_BLOCK_SIZE }, -}; -#define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls) - -static LIST_HEAD(ahash_algs); -static LIST_HEAD(hmac_algs); - -static int algs_registered; - -static void __n2_unregister_algs(void) -{ - struct n2_skcipher_alg *skcipher, *skcipher_tmp; - struct n2_ahash_alg *alg, *alg_tmp; - struct n2_hmac_alg *hmac, *hmac_tmp; - - list_for_each_entry_safe(skcipher, skcipher_tmp, &skcipher_algs, entry) { - crypto_unregister_skcipher(&skcipher->skcipher); - list_del(&skcipher->entry); - kfree(skcipher); - } - list_for_each_entry_safe(hmac, hmac_tmp, &hmac_algs, derived.entry) { - crypto_unregister_ahash(&hmac->derived.alg); - list_del(&hmac->derived.entry); - kfree(hmac); - } - list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) { - crypto_unregister_ahash(&alg->alg); - list_del(&alg->entry); - kfree(alg); - } -} - -static int n2_skcipher_init_tfm(struct crypto_skcipher *tfm) -{ - crypto_skcipher_set_reqsize(tfm, sizeof(struct n2_request_context)); - return 0; -} - -static int __n2_register_one_skcipher(const struct n2_skcipher_tmpl *tmpl) -{ - struct n2_skcipher_alg *p = kzalloc(sizeof(*p), GFP_KERNEL); - struct skcipher_alg *alg; - int err; - - if (!p) - return -ENOMEM; - - alg = &p->skcipher; - *alg = tmpl->skcipher; - - snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); - snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->drv_name); - alg->base.cra_priority = N2_CRA_PRIORITY; - alg->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY; - alg->base.cra_blocksize = tmpl->block_size; - p->enc_type = tmpl->enc_type; - alg->base.cra_ctxsize = sizeof(struct n2_skcipher_context); - alg->base.cra_module = THIS_MODULE; - alg->init = n2_skcipher_init_tfm; - - list_add(&p->entry, &skcipher_algs); - err = crypto_register_skcipher(alg); - if (err) { - pr_err("%s alg registration failed\n", alg->base.cra_name); - list_del(&p->entry); - kfree(p); - } else { - pr_info("%s alg registered\n", alg->base.cra_name); - } - return err; -} - -static int __n2_register_one_hmac(struct n2_ahash_alg *n2ahash) -{ - struct n2_hmac_alg *p = kzalloc(sizeof(*p), GFP_KERNEL); - struct ahash_alg *ahash; - struct crypto_alg *base; - int err; - - if (!p) - return -ENOMEM; - - p->child_alg = n2ahash->alg.halg.base.cra_name; - memcpy(&p->derived, n2ahash, sizeof(struct n2_ahash_alg)); - INIT_LIST_HEAD(&p->derived.entry); - - ahash = &p->derived.alg; - ahash->digest = n2_hmac_async_digest; - ahash->setkey = n2_hmac_async_setkey; - - base = &ahash->halg.base; - err = -EINVAL; - if (snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", - p->child_alg) >= CRYPTO_MAX_ALG_NAME) - goto out_free_p; - if (snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2", - p->child_alg) >= CRYPTO_MAX_ALG_NAME) - goto out_free_p; - - base->cra_ctxsize = sizeof(struct n2_hmac_ctx); - base->cra_init = n2_hmac_cra_init; - base->cra_exit = n2_hmac_cra_exit; - - list_add(&p->derived.entry, &hmac_algs); - err = crypto_register_ahash(ahash); - if (err) { - pr_err("%s alg registration failed\n", base->cra_name); - list_del(&p->derived.entry); -out_free_p: - kfree(p); - } else { - pr_info("%s alg registered\n", base->cra_name); - } - return err; -} - -static int __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl) -{ - struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL); - struct hash_alg_common *halg; - struct crypto_alg *base; - struct ahash_alg *ahash; - int err; - - if (!p) - return -ENOMEM; - - p->hash_zero = tmpl->hash_zero; - p->hash_init = tmpl->hash_init; - p->auth_type = tmpl->auth_type; - p->hmac_type = tmpl->hmac_type; - p->hw_op_hashsz = tmpl->hw_op_hashsz; - p->digest_size = tmpl->digest_size; - - ahash = &p->alg; - ahash->init = n2_hash_async_init; - ahash->update = n2_hash_async_update; - ahash->final = n2_hash_async_final; - ahash->finup = n2_hash_async_finup; - ahash->digest = n2_hash_async_digest; - ahash->export = n2_hash_async_noexport; - ahash->import = n2_hash_async_noimport; - - halg = &ahash->halg; - halg->digestsize = tmpl->digest_size; - halg->statesize = tmpl->statesize; - - base = &halg->base; - snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); - snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name); - base->cra_priority = N2_CRA_PRIORITY; - base->cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_NEED_FALLBACK; - base->cra_blocksize = tmpl->block_size; - base->cra_ctxsize = sizeof(struct n2_hash_ctx); - base->cra_module = THIS_MODULE; - base->cra_init = n2_hash_cra_init; - base->cra_exit = n2_hash_cra_exit; - - list_add(&p->entry, &ahash_algs); - err = crypto_register_ahash(ahash); - if (err) { - pr_err("%s alg registration failed\n", base->cra_name); - list_del(&p->entry); - kfree(p); - } else { - pr_info("%s alg registered\n", base->cra_name); - } - if (!err && p->hmac_type != AUTH_TYPE_RESERVED) - err = __n2_register_one_hmac(p); - return err; -} - -static int n2_register_algs(void) -{ - int i, err = 0; - - mutex_lock(&spu_lock); - if (algs_registered++) - goto out; - - for (i = 0; i < NUM_HASH_TMPLS; i++) { - err = __n2_register_one_ahash(&hash_tmpls[i]); - if (err) { - __n2_unregister_algs(); - goto out; - } - } - for (i = 0; i < NUM_CIPHER_TMPLS; i++) { - err = __n2_register_one_skcipher(&skcipher_tmpls[i]); - if (err) { - __n2_unregister_algs(); - goto out; - } - } - -out: - mutex_unlock(&spu_lock); - return err; -} - -static void n2_unregister_algs(void) -{ - mutex_lock(&spu_lock); - if (!--algs_registered) - __n2_unregister_algs(); - mutex_unlock(&spu_lock); -} - -/* To map CWQ queues to interrupt sources, the hypervisor API provides - * a devino. This isn't very useful to us because all of the - * interrupts listed in the device_node have been translated to - * Linux virtual IRQ cookie numbers. - * - * So we have to back-translate, going through the 'intr' and 'ino' - * property tables of the n2cp MDESC node, matching it with the OF - * 'interrupts' property entries, in order to figure out which - * devino goes to which already-translated IRQ. - */ -static int find_devino_index(struct platform_device *dev, struct spu_mdesc_info *ip, - unsigned long dev_ino) -{ - const unsigned int *dev_intrs; - unsigned int intr; - int i; - - for (i = 0; i < ip->num_intrs; i++) { - if (ip->ino_table[i].ino == dev_ino) - break; - } - if (i == ip->num_intrs) - return -ENODEV; - - intr = ip->ino_table[i].intr; - - dev_intrs = of_get_property(dev->dev.of_node, "interrupts", NULL); - if (!dev_intrs) - return -ENODEV; - - for (i = 0; i < dev->archdata.num_irqs; i++) { - if (dev_intrs[i] == intr) - return i; - } - - return -ENODEV; -} - -static int spu_map_ino(struct platform_device *dev, struct spu_mdesc_info *ip, - const char *irq_name, struct spu_queue *p, - irq_handler_t handler) -{ - unsigned long herr; - int index; - - herr = sun4v_ncs_qhandle_to_devino(p->qhandle, &p->devino); - if (herr) - return -EINVAL; - - index = find_devino_index(dev, ip, p->devino); - if (index < 0) - return index; - - p->irq = dev->archdata.irqs[index]; - - sprintf(p->irq_name, "%s-%d", irq_name, index); - - return request_irq(p->irq, handler, 0, p->irq_name, p); -} - -static struct kmem_cache *queue_cache[2]; - -static void *new_queue(unsigned long q_type) -{ - return kmem_cache_zalloc(queue_cache[q_type - 1], GFP_KERNEL); -} - -static void free_queue(void *p, unsigned long q_type) -{ - kmem_cache_free(queue_cache[q_type - 1], p); -} - -static int queue_cache_init(void) -{ - if (!queue_cache[HV_NCS_QTYPE_MAU - 1]) - queue_cache[HV_NCS_QTYPE_MAU - 1] = - kmem_cache_create("mau_queue", - (MAU_NUM_ENTRIES * - MAU_ENTRY_SIZE), - MAU_ENTRY_SIZE, 0, NULL); - if (!queue_cache[HV_NCS_QTYPE_MAU - 1]) - return -ENOMEM; - - if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) - queue_cache[HV_NCS_QTYPE_CWQ - 1] = - kmem_cache_create("cwq_queue", - (CWQ_NUM_ENTRIES * - CWQ_ENTRY_SIZE), - CWQ_ENTRY_SIZE, 0, NULL); - if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) { - kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]); - queue_cache[HV_NCS_QTYPE_MAU - 1] = NULL; - return -ENOMEM; - } - return 0; -} - -static void queue_cache_destroy(void) -{ - kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]); - kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_CWQ - 1]); - queue_cache[HV_NCS_QTYPE_MAU - 1] = NULL; - queue_cache[HV_NCS_QTYPE_CWQ - 1] = NULL; -} - -static long spu_queue_register_workfn(void *arg) -{ - struct spu_qreg *qr = arg; - struct spu_queue *p = qr->queue; - unsigned long q_type = qr->type; - unsigned long hv_ret; - - hv_ret = sun4v_ncs_qconf(q_type, __pa(p->q), - CWQ_NUM_ENTRIES, &p->qhandle); - if (!hv_ret) - sun4v_ncs_sethead_marker(p->qhandle, 0); - - return hv_ret ? -EINVAL : 0; -} - -static int spu_queue_register(struct spu_queue *p, unsigned long q_type) -{ - int cpu = cpumask_any_and(&p->sharing, cpu_online_mask); - struct spu_qreg qr = { .queue = p, .type = q_type }; - - return work_on_cpu_safe(cpu, spu_queue_register_workfn, &qr); -} - -static int spu_queue_setup(struct spu_queue *p) -{ - int err; - - p->q = new_queue(p->q_type); - if (!p->q) - return -ENOMEM; - - err = spu_queue_register(p, p->q_type); - if (err) { - free_queue(p->q, p->q_type); - p->q = NULL; - } - - return err; -} - -static void spu_queue_destroy(struct spu_queue *p) -{ - unsigned long hv_ret; - - if (!p->q) - return; - - hv_ret = sun4v_ncs_qconf(p->q_type, p->qhandle, 0, &p->qhandle); - - if (!hv_ret) - free_queue(p->q, p->q_type); -} - -static void spu_list_destroy(struct list_head *list) -{ - struct spu_queue *p, *n; - - list_for_each_entry_safe(p, n, list, list) { - int i; - - for (i = 0; i < NR_CPUS; i++) { - if (cpu_to_cwq[i] == p) - cpu_to_cwq[i] = NULL; - } - - if (p->irq) { - free_irq(p->irq, p); - p->irq = 0; - } - spu_queue_destroy(p); - list_del(&p->list); - kfree(p); - } -} - -/* Walk the backward arcs of a CWQ 'exec-unit' node, - * gathering cpu membership information. - */ -static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc, - struct platform_device *dev, - u64 node, struct spu_queue *p, - struct spu_queue **table) -{ - u64 arc; - - mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) { - u64 tgt = mdesc_arc_target(mdesc, arc); - const char *name = mdesc_node_name(mdesc, tgt); - const u64 *id; - - if (strcmp(name, "cpu")) - continue; - id = mdesc_get_property(mdesc, tgt, "id", NULL); - if (table[*id] != NULL) { - dev_err(&dev->dev, "%pOF: SPU cpu slot already set.\n", - dev->dev.of_node); - return -EINVAL; - } - cpumask_set_cpu(*id, &p->sharing); - table[*id] = p; - } - return 0; -} - -/* Process an 'exec-unit' MDESC node of type 'cwq'. */ -static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list, - struct platform_device *dev, struct mdesc_handle *mdesc, - u64 node, const char *iname, unsigned long q_type, - irq_handler_t handler, struct spu_queue **table) -{ - struct spu_queue *p; - int err; - - p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL); - if (!p) { - dev_err(&dev->dev, "%pOF: Could not allocate SPU queue.\n", - dev->dev.of_node); - return -ENOMEM; - } - - cpumask_clear(&p->sharing); - spin_lock_init(&p->lock); - p->q_type = q_type; - INIT_LIST_HEAD(&p->jobs); - list_add(&p->list, list); - - err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table); - if (err) - return err; - - err = spu_queue_setup(p); - if (err) - return err; - - return spu_map_ino(dev, ip, iname, p, handler); -} - -static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct platform_device *dev, - struct spu_mdesc_info *ip, struct list_head *list, - const char *exec_name, unsigned long q_type, - irq_handler_t handler, struct spu_queue **table) -{ - int err = 0; - u64 node; - - mdesc_for_each_node_by_name(mdesc, node, "exec-unit") { - const char *type; - - type = mdesc_get_property(mdesc, node, "type", NULL); - if (!type || strcmp(type, exec_name)) - continue; - - err = handle_exec_unit(ip, list, dev, mdesc, node, - exec_name, q_type, handler, table); - if (err) { - spu_list_destroy(list); - break; - } - } - - return err; -} - -static int get_irq_props(struct mdesc_handle *mdesc, u64 node, - struct spu_mdesc_info *ip) -{ - const u64 *ino; - int ino_len; - int i; - - ino = mdesc_get_property(mdesc, node, "ino", &ino_len); - if (!ino) { - printk("NO 'ino'\n"); - return -ENODEV; - } - - ip->num_intrs = ino_len / sizeof(u64); - ip->ino_table = kzalloc((sizeof(struct ino_blob) * - ip->num_intrs), - GFP_KERNEL); - if (!ip->ino_table) - return -ENOMEM; - - for (i = 0; i < ip->num_intrs; i++) { - struct ino_blob *b = &ip->ino_table[i]; - b->intr = i + 1; - b->ino = ino[i]; - } - - return 0; -} - -static int grab_mdesc_irq_props(struct mdesc_handle *mdesc, - struct platform_device *dev, - struct spu_mdesc_info *ip, - const char *node_name) -{ - u64 node, reg; - - if (of_property_read_reg(dev->dev.of_node, 0, ®, NULL) < 0) - return -ENODEV; - - mdesc_for_each_node_by_name(mdesc, node, "virtual-device") { - const char *name; - const u64 *chdl; - - name = mdesc_get_property(mdesc, node, "name", NULL); - if (!name || strcmp(name, node_name)) - continue; - chdl = mdesc_get_property(mdesc, node, "cfg-handle", NULL); - if (!chdl || (*chdl != reg)) - continue; - ip->cfg_handle = *chdl; - return get_irq_props(mdesc, node, ip); - } - - return -ENODEV; -} - -static unsigned long n2_spu_hvapi_major; -static unsigned long n2_spu_hvapi_minor; - -static int n2_spu_hvapi_register(void) -{ - int err; - - n2_spu_hvapi_major = 2; - n2_spu_hvapi_minor = 0; - - err = sun4v_hvapi_register(HV_GRP_NCS, - n2_spu_hvapi_major, - &n2_spu_hvapi_minor); - - if (!err) - pr_info("Registered NCS HVAPI version %lu.%lu\n", - n2_spu_hvapi_major, - n2_spu_hvapi_minor); - - return err; -} - -static void n2_spu_hvapi_unregister(void) -{ - sun4v_hvapi_unregister(HV_GRP_NCS); -} - -static int global_ref; - -static int grab_global_resources(void) -{ - int err = 0; - - mutex_lock(&spu_lock); - - if (global_ref++) - goto out; - - err = n2_spu_hvapi_register(); - if (err) - goto out; - - err = queue_cache_init(); - if (err) - goto out_hvapi_release; - - err = -ENOMEM; - cpu_to_cwq = kcalloc(NR_CPUS, sizeof(struct spu_queue *), - GFP_KERNEL); - if (!cpu_to_cwq) - goto out_queue_cache_destroy; - - cpu_to_mau = kcalloc(NR_CPUS, sizeof(struct spu_queue *), - GFP_KERNEL); - if (!cpu_to_mau) - goto out_free_cwq_table; - - err = 0; - -out: - if (err) - global_ref--; - mutex_unlock(&spu_lock); - return err; - -out_free_cwq_table: - kfree(cpu_to_cwq); - cpu_to_cwq = NULL; - -out_queue_cache_destroy: - queue_cache_destroy(); - -out_hvapi_release: - n2_spu_hvapi_unregister(); - goto out; -} - -static void release_global_resources(void) -{ - mutex_lock(&spu_lock); - if (!--global_ref) { - kfree(cpu_to_cwq); - cpu_to_cwq = NULL; - - kfree(cpu_to_mau); - cpu_to_mau = NULL; - - queue_cache_destroy(); - n2_spu_hvapi_unregister(); - } - mutex_unlock(&spu_lock); -} - -static struct n2_crypto *alloc_n2cp(void) -{ - struct n2_crypto *np = kzalloc(sizeof(struct n2_crypto), GFP_KERNEL); - - if (np) - INIT_LIST_HEAD(&np->cwq_list); - - return np; -} - -static void free_n2cp(struct n2_crypto *np) -{ - kfree(np->cwq_info.ino_table); - np->cwq_info.ino_table = NULL; - - kfree(np); -} - -static void n2_spu_driver_version(void) -{ - static int n2_spu_version_printed; - - if (n2_spu_version_printed++ == 0) - pr_info("%s", version); -} - -static int n2_crypto_probe(struct platform_device *dev) -{ - struct mdesc_handle *mdesc; - struct n2_crypto *np; - int err; - - n2_spu_driver_version(); - - pr_info("Found N2CP at %pOF\n", dev->dev.of_node); - - np = alloc_n2cp(); - if (!np) { - dev_err(&dev->dev, "%pOF: Unable to allocate n2cp.\n", - dev->dev.of_node); - return -ENOMEM; - } - - err = grab_global_resources(); - if (err) { - dev_err(&dev->dev, "%pOF: Unable to grab global resources.\n", - dev->dev.of_node); - goto out_free_n2cp; - } - - mdesc = mdesc_grab(); - - if (!mdesc) { - dev_err(&dev->dev, "%pOF: Unable to grab MDESC.\n", - dev->dev.of_node); - err = -ENODEV; - goto out_free_global; - } - err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp"); - if (err) { - dev_err(&dev->dev, "%pOF: Unable to grab IRQ props.\n", - dev->dev.of_node); - mdesc_release(mdesc); - goto out_free_global; - } - - err = spu_mdesc_scan(mdesc, dev, &np->cwq_info, &np->cwq_list, - "cwq", HV_NCS_QTYPE_CWQ, cwq_intr, - cpu_to_cwq); - mdesc_release(mdesc); - - if (err) { - dev_err(&dev->dev, "%pOF: CWQ MDESC scan failed.\n", - dev->dev.of_node); - goto out_free_global; - } - - err = n2_register_algs(); - if (err) { - dev_err(&dev->dev, "%pOF: Unable to register algorithms.\n", - dev->dev.of_node); - goto out_free_spu_list; - } - - dev_set_drvdata(&dev->dev, np); - - return 0; - -out_free_spu_list: - spu_list_destroy(&np->cwq_list); - -out_free_global: - release_global_resources(); - -out_free_n2cp: - free_n2cp(np); - - return err; -} - -static void n2_crypto_remove(struct platform_device *dev) -{ - struct n2_crypto *np = dev_get_drvdata(&dev->dev); - - n2_unregister_algs(); - - spu_list_destroy(&np->cwq_list); - - release_global_resources(); - - free_n2cp(np); -} - -static struct n2_mau *alloc_ncp(void) -{ - struct n2_mau *mp = kzalloc(sizeof(struct n2_mau), GFP_KERNEL); - - if (mp) - INIT_LIST_HEAD(&mp->mau_list); - - return mp; -} - -static void free_ncp(struct n2_mau *mp) -{ - kfree(mp->mau_info.ino_table); - mp->mau_info.ino_table = NULL; - - kfree(mp); -} - -static int n2_mau_probe(struct platform_device *dev) -{ - struct mdesc_handle *mdesc; - struct n2_mau *mp; - int err; - - n2_spu_driver_version(); - - pr_info("Found NCP at %pOF\n", dev->dev.of_node); - - mp = alloc_ncp(); - if (!mp) { - dev_err(&dev->dev, "%pOF: Unable to allocate ncp.\n", - dev->dev.of_node); - return -ENOMEM; - } - - err = grab_global_resources(); - if (err) { - dev_err(&dev->dev, "%pOF: Unable to grab global resources.\n", - dev->dev.of_node); - goto out_free_ncp; - } - - mdesc = mdesc_grab(); - - if (!mdesc) { - dev_err(&dev->dev, "%pOF: Unable to grab MDESC.\n", - dev->dev.of_node); - err = -ENODEV; - goto out_free_global; - } - - err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp"); - if (err) { - dev_err(&dev->dev, "%pOF: Unable to grab IRQ props.\n", - dev->dev.of_node); - mdesc_release(mdesc); - goto out_free_global; - } - - err = spu_mdesc_scan(mdesc, dev, &mp->mau_info, &mp->mau_list, - "mau", HV_NCS_QTYPE_MAU, mau_intr, - cpu_to_mau); - mdesc_release(mdesc); - - if (err) { - dev_err(&dev->dev, "%pOF: MAU MDESC scan failed.\n", - dev->dev.of_node); - goto out_free_global; - } - - dev_set_drvdata(&dev->dev, mp); - - return 0; - -out_free_global: - release_global_resources(); - -out_free_ncp: - free_ncp(mp); - - return err; -} - -static void n2_mau_remove(struct platform_device *dev) -{ - struct n2_mau *mp = dev_get_drvdata(&dev->dev); - - spu_list_destroy(&mp->mau_list); - - release_global_resources(); - - free_ncp(mp); -} - -static const struct of_device_id n2_crypto_match[] = { - { - .name = "n2cp", - .compatible = "SUNW,n2-cwq", - }, - { - .name = "n2cp", - .compatible = "SUNW,vf-cwq", - }, - { - .name = "n2cp", - .compatible = "SUNW,kt-cwq", - }, - {}, -}; - -MODULE_DEVICE_TABLE(of, n2_crypto_match); - -static struct platform_driver n2_crypto_driver = { - .driver = { - .name = "n2cp", - .of_match_table = n2_crypto_match, - }, - .probe = n2_crypto_probe, - .remove = n2_crypto_remove, -}; - -static const struct of_device_id n2_mau_match[] = { - { - .name = "ncp", - .compatible = "SUNW,n2-mau", - }, - { - .name = "ncp", - .compatible = "SUNW,vf-mau", - }, - { - .name = "ncp", - .compatible = "SUNW,kt-mau", - }, - {}, -}; - -MODULE_DEVICE_TABLE(of, n2_mau_match); - -static struct platform_driver n2_mau_driver = { - .driver = { - .name = "ncp", - .of_match_table = n2_mau_match, - }, - .probe = n2_mau_probe, - .remove = n2_mau_remove, -}; - -static struct platform_driver * const drivers[] = { - &n2_crypto_driver, - &n2_mau_driver, -}; - -static int __init n2_init(void) -{ - return platform_register_drivers(drivers, ARRAY_SIZE(drivers)); -} - -static void __exit n2_exit(void) -{ - platform_unregister_drivers(drivers, ARRAY_SIZE(drivers)); -} - -module_init(n2_init); -module_exit(n2_exit); diff --git a/drivers/crypto/n2_core.h b/drivers/crypto/n2_core.h deleted file mode 100644 index 2406763b0306..000000000000 --- a/drivers/crypto/n2_core.h +++ /dev/null @@ -1,232 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _N2_CORE_H -#define _N2_CORE_H - -#ifndef __ASSEMBLY__ - -struct ino_blob { - u64 intr; - u64 ino; -}; - -struct spu_mdesc_info { - u64 cfg_handle; - struct ino_blob *ino_table; - int num_intrs; -}; - -struct n2_crypto { - struct spu_mdesc_info cwq_info; - struct list_head cwq_list; -}; - -struct n2_mau { - struct spu_mdesc_info mau_info; - struct list_head mau_list; -}; - -#define CWQ_ENTRY_SIZE 64 -#define CWQ_NUM_ENTRIES 64 - -#define MAU_ENTRY_SIZE 64 -#define MAU_NUM_ENTRIES 64 - -struct cwq_initial_entry { - u64 control; - u64 src_addr; - u64 auth_key_addr; - u64 auth_iv_addr; - u64 final_auth_state_addr; - u64 enc_key_addr; - u64 enc_iv_addr; - u64 dest_addr; -}; - -struct cwq_ext_entry { - u64 len; - u64 src_addr; - u64 resv1; - u64 resv2; - u64 resv3; - u64 resv4; - u64 resv5; - u64 resv6; -}; - -struct cwq_final_entry { - u64 control; - u64 src_addr; - u64 resv1; - u64 resv2; - u64 resv3; - u64 resv4; - u64 resv5; - u64 resv6; -}; - -#define CONTROL_LEN 0x000000000000ffffULL -#define CONTROL_LEN_SHIFT 0 -#define CONTROL_HMAC_KEY_LEN 0x0000000000ff0000ULL -#define CONTROL_HMAC_KEY_LEN_SHIFT 16 -#define CONTROL_ENC_TYPE 0x00000000ff000000ULL -#define CONTROL_ENC_TYPE_SHIFT 24 -#define ENC_TYPE_ALG_RC4_STREAM 0x00ULL -#define ENC_TYPE_ALG_RC4_NOSTREAM 0x04ULL -#define ENC_TYPE_ALG_DES 0x08ULL -#define ENC_TYPE_ALG_3DES 0x0cULL -#define ENC_TYPE_ALG_AES128 0x10ULL -#define ENC_TYPE_ALG_AES192 0x14ULL -#define ENC_TYPE_ALG_AES256 0x18ULL -#define ENC_TYPE_ALG_RESERVED 0x1cULL -#define ENC_TYPE_ALG_MASK 0x1cULL -#define ENC_TYPE_CHAINING_ECB 0x00ULL -#define ENC_TYPE_CHAINING_CBC 0x01ULL -#define ENC_TYPE_CHAINING_CFB 0x02ULL -#define ENC_TYPE_CHAINING_COUNTER 0x03ULL -#define ENC_TYPE_CHAINING_MASK 0x03ULL -#define CONTROL_AUTH_TYPE 0x0000001f00000000ULL -#define CONTROL_AUTH_TYPE_SHIFT 32 -#define AUTH_TYPE_RESERVED 0x00ULL -#define AUTH_TYPE_MD5 0x01ULL -#define AUTH_TYPE_SHA1 0x02ULL -#define AUTH_TYPE_SHA256 0x03ULL -#define AUTH_TYPE_CRC32 0x04ULL -#define AUTH_TYPE_HMAC_MD5 0x05ULL -#define AUTH_TYPE_HMAC_SHA1 0x06ULL -#define AUTH_TYPE_HMAC_SHA256 0x07ULL -#define AUTH_TYPE_TCP_CHECKSUM 0x08ULL -#define AUTH_TYPE_SSL_HMAC_MD5 0x09ULL -#define AUTH_TYPE_SSL_HMAC_SHA1 0x0aULL -#define AUTH_TYPE_SSL_HMAC_SHA256 0x0bULL -#define CONTROL_STRAND 0x000000e000000000ULL -#define CONTROL_STRAND_SHIFT 37 -#define CONTROL_HASH_LEN 0x0000ff0000000000ULL -#define CONTROL_HASH_LEN_SHIFT 40 -#define CONTROL_INTERRUPT 0x0001000000000000ULL -#define CONTROL_STORE_FINAL_AUTH_STATE 0x0002000000000000ULL -#define CONTROL_RESERVED 0x001c000000000000ULL -#define CONTROL_HV_DONE 0x0004000000000000ULL -#define CONTROL_HV_PROTOCOL_ERROR 0x0008000000000000ULL -#define CONTROL_HV_HARDWARE_ERROR 0x0010000000000000ULL -#define CONTROL_END_OF_BLOCK 0x0020000000000000ULL -#define CONTROL_START_OF_BLOCK 0x0040000000000000ULL -#define CONTROL_ENCRYPT 0x0080000000000000ULL -#define CONTROL_OPCODE 0xff00000000000000ULL -#define CONTROL_OPCODE_SHIFT 56 -#define OPCODE_INPLACE_BIT 0x80ULL -#define OPCODE_SSL_KEYBLOCK 0x10ULL -#define OPCODE_COPY 0x20ULL -#define OPCODE_ENCRYPT 0x40ULL -#define OPCODE_AUTH_MAC 0x41ULL - -#endif /* !(__ASSEMBLY__) */ - -/* NCS v2.0 hypervisor interfaces */ -#define HV_NCS_QTYPE_MAU 0x01 -#define HV_NCS_QTYPE_CWQ 0x02 - -/* ncs_qconf() - * TRAP: HV_FAST_TRAP - * FUNCTION: HV_FAST_NCS_QCONF - * ARG0: Queue type (HV_NCS_QTYPE_{MAU,CWQ}) - * ARG1: Real address of queue, or handle for unconfigure - * ARG2: Number of entries in queue, zero for unconfigure - * RET0: status - * RET1: queue handle - * - * Configure a queue in the stream processing unit. - * - * The real address given as the base must be 64-byte - * aligned. - * - * The queue size can range from a minimum of 2 to a maximum - * of 64. The queue size must be a power of two. - * - * To unconfigure a queue, specify a length of zero and place - * the queue handle into ARG1. - * - * On configure success the hypervisor will set the FIRST, HEAD, - * and TAIL registers to the address of the first entry in the - * queue. The LAST register will be set to point to the last - * entry in the queue. - */ -#define HV_FAST_NCS_QCONF 0x111 - -/* ncs_qinfo() - * TRAP: HV_FAST_TRAP - * FUNCTION: HV_FAST_NCS_QINFO - * ARG0: Queue handle - * RET0: status - * RET1: Queue type (HV_NCS_QTYPE_{MAU,CWQ}) - * RET2: Queue base address - * RET3: Number of entries - */ -#define HV_FAST_NCS_QINFO 0x112 - -/* ncs_gethead() - * TRAP: HV_FAST_TRAP - * FUNCTION: HV_FAST_NCS_GETHEAD - * ARG0: Queue handle - * RET0: status - * RET1: queue head offset - */ -#define HV_FAST_NCS_GETHEAD 0x113 - -/* ncs_gettail() - * TRAP: HV_FAST_TRAP - * FUNCTION: HV_FAST_NCS_GETTAIL - * ARG0: Queue handle - * RET0: status - * RET1: queue tail offset - */ -#define HV_FAST_NCS_GETTAIL 0x114 - -/* ncs_settail() - * TRAP: HV_FAST_TRAP - * FUNCTION: HV_FAST_NCS_SETTAIL - * ARG0: Queue handle - * ARG1: New tail offset - * RET0: status - */ -#define HV_FAST_NCS_SETTAIL 0x115 - -/* ncs_qhandle_to_devino() - * TRAP: HV_FAST_TRAP - * FUNCTION: HV_FAST_NCS_QHANDLE_TO_DEVINO - * ARG0: Queue handle - * RET0: status - * RET1: devino - */ -#define HV_FAST_NCS_QHANDLE_TO_DEVINO 0x116 - -/* ncs_sethead_marker() - * TRAP: HV_FAST_TRAP - * FUNCTION: HV_FAST_NCS_SETHEAD_MARKER - * ARG0: Queue handle - * ARG1: New head offset - * RET0: status - */ -#define HV_FAST_NCS_SETHEAD_MARKER 0x117 - -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_ncs_qconf(unsigned long queue_type, - unsigned long queue_ra, - unsigned long num_entries, - unsigned long *qhandle); -extern unsigned long sun4v_ncs_qinfo(unsigned long qhandle, - unsigned long *queue_type, - unsigned long *queue_ra, - unsigned long *num_entries); -extern unsigned long sun4v_ncs_gethead(unsigned long qhandle, - unsigned long *head); -extern unsigned long sun4v_ncs_gettail(unsigned long qhandle, - unsigned long *tail); -extern unsigned long sun4v_ncs_settail(unsigned long qhandle, - unsigned long tail); -extern unsigned long sun4v_ncs_qhandle_to_devino(unsigned long qhandle, - unsigned long *devino); -extern unsigned long sun4v_ncs_sethead_marker(unsigned long qhandle, - unsigned long head); -#endif /* !(__ASSEMBLY__) */ - -#endif /* _N2_CORE_H */ diff --git a/drivers/crypto/nx/nx-common-pseries.c b/drivers/crypto/nx/nx-common-pseries.c index 1660c5cf3641..56129bdf53ab 100644 --- a/drivers/crypto/nx/nx-common-pseries.c +++ b/drivers/crypto/nx/nx-common-pseries.c @@ -1145,6 +1145,7 @@ static void __init nxcop_get_capabilities(void) { struct hv_vas_all_caps *hv_caps; struct hv_nx_cop_caps *hv_nxc; + u64 feat; int rc; hv_caps = kmalloc(sizeof(*hv_caps), GFP_KERNEL); @@ -1155,27 +1156,26 @@ static void __init nxcop_get_capabilities(void) */ rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES, 0, (u64)virt_to_phys(hv_caps)); + if (!rc) + feat = be64_to_cpu(hv_caps->feat_type); + kfree(hv_caps); if (rc) - goto out; + return; + if (!(feat & VAS_NX_GZIP_FEAT_BIT)) + return; - caps_feat = be64_to_cpu(hv_caps->feat_type); /* * NX-GZIP feature available */ - if (caps_feat & VAS_NX_GZIP_FEAT_BIT) { - hv_nxc = kmalloc(sizeof(*hv_nxc), GFP_KERNEL); - if (!hv_nxc) - goto out; - /* - * Get capabilities for NX-GZIP feature - */ - rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES, - VAS_NX_GZIP_FEAT, - (u64)virt_to_phys(hv_nxc)); - } else { - pr_err("NX-GZIP feature is not available\n"); - rc = -EINVAL; - } + hv_nxc = kmalloc(sizeof(*hv_nxc), GFP_KERNEL); + if (!hv_nxc) + return; + /* + * Get capabilities for NX-GZIP feature + */ + rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES, + VAS_NX_GZIP_FEAT, + (u64)virt_to_phys(hv_nxc)); if (!rc) { nx_cop_caps.descriptor = be64_to_cpu(hv_nxc->descriptor); @@ -1185,13 +1185,10 @@ static void __init nxcop_get_capabilities(void) be64_to_cpu(hv_nxc->min_compress_len); nx_cop_caps.min_decompress_len = be64_to_cpu(hv_nxc->min_decompress_len); - } else { - caps_feat = 0; + caps_feat = feat; } kfree(hv_nxc); -out: - kfree(hv_caps); } static const struct vio_device_id nx842_vio_driver_ids[] = { diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index e27b84616743..551dd32a8db0 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -18,7 +18,6 @@ #include <crypto/internal/aead.h> #include <crypto/internal/engine.h> #include <crypto/internal/skcipher.h> -#include <crypto/scatterwalk.h> #include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/err.h> @@ -272,9 +271,9 @@ static int omap_aes_crypt_dma(struct omap_aes_dev *dd, int ret; if (dd->pio_only) { - scatterwalk_start(&dd->in_walk, dd->in_sg); + dd->in_sg_offset = 0; if (out_sg_len) - scatterwalk_start(&dd->out_walk, dd->out_sg); + dd->out_sg_offset = 0; /* Enable DATAIN interrupt and let it take care of the rest */ @@ -871,21 +870,18 @@ static irqreturn_t omap_aes_irq(int irq, void *dev_id) BUG_ON(!dd->in_sg); - BUG_ON(_calc_walked(in) > dd->in_sg->length); + BUG_ON(dd->in_sg_offset > dd->in_sg->length); - src = sg_virt(dd->in_sg) + _calc_walked(in); + src = sg_virt(dd->in_sg) + dd->in_sg_offset; for (i = 0; i < AES_BLOCK_WORDS; i++) { omap_aes_write(dd, AES_REG_DATA_N(dd, i), *src); - - scatterwalk_advance(&dd->in_walk, 4); - if (dd->in_sg->length == _calc_walked(in)) { + dd->in_sg_offset += 4; + if (dd->in_sg_offset == dd->in_sg->length) { dd->in_sg = sg_next(dd->in_sg); if (dd->in_sg) { - scatterwalk_start(&dd->in_walk, - dd->in_sg); - src = sg_virt(dd->in_sg) + - _calc_walked(in); + dd->in_sg_offset = 0; + src = sg_virt(dd->in_sg); } } else { src++; @@ -904,20 +900,18 @@ static irqreturn_t omap_aes_irq(int irq, void *dev_id) BUG_ON(!dd->out_sg); - BUG_ON(_calc_walked(out) > dd->out_sg->length); + BUG_ON(dd->out_sg_offset > dd->out_sg->length); - dst = sg_virt(dd->out_sg) + _calc_walked(out); + dst = sg_virt(dd->out_sg) + dd->out_sg_offset; for (i = 0; i < AES_BLOCK_WORDS; i++) { *dst = omap_aes_read(dd, AES_REG_DATA_N(dd, i)); - scatterwalk_advance(&dd->out_walk, 4); - if (dd->out_sg->length == _calc_walked(out)) { + dd->out_sg_offset += 4; + if (dd->out_sg_offset == dd->out_sg->length) { dd->out_sg = sg_next(dd->out_sg); if (dd->out_sg) { - scatterwalk_start(&dd->out_walk, - dd->out_sg); - dst = sg_virt(dd->out_sg) + - _calc_walked(out); + dd->out_sg_offset = 0; + dst = sg_virt(dd->out_sg); } } else { dst++; diff --git a/drivers/crypto/omap-aes.h b/drivers/crypto/omap-aes.h index 0f35c9164764..41d67780fd45 100644 --- a/drivers/crypto/omap-aes.h +++ b/drivers/crypto/omap-aes.h @@ -14,8 +14,6 @@ #define DST_MAXBURST 4 #define DMA_MIN (DST_MAXBURST * sizeof(u32)) -#define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset) - /* * OMAP TRM gives bitfields as start:end, where start is the higher bit * number. For example 7:0 @@ -186,8 +184,8 @@ struct omap_aes_dev { struct scatterlist out_sgl; struct scatterlist *orig_out; - struct scatter_walk in_walk; - struct scatter_walk out_walk; + unsigned int in_sg_offset; + unsigned int out_sg_offset; struct dma_chan *dma_lch_in; struct dma_chan *dma_lch_out; int in_sg_len; diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index 498cbd585ed1..a099460d5f21 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -19,7 +19,6 @@ #include <crypto/engine.h> #include <crypto/internal/des.h> #include <crypto/internal/skcipher.h> -#include <crypto/scatterwalk.h> #include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/err.h> @@ -40,8 +39,6 @@ #define DES_BLOCK_WORDS (DES_BLOCK_SIZE >> 2) -#define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset) - #define DES_REG_KEY(dd, x) ((dd)->pdata->key_ofs - \ ((x ^ 0x01) * 0x04)) @@ -152,8 +149,8 @@ struct omap_des_dev { struct scatterlist out_sgl; struct scatterlist *orig_out; - struct scatter_walk in_walk; - struct scatter_walk out_walk; + unsigned int in_sg_offset; + unsigned int out_sg_offset; struct dma_chan *dma_lch_in; struct dma_chan *dma_lch_out; int in_sg_len; @@ -379,8 +376,8 @@ static int omap_des_crypt_dma(struct crypto_tfm *tfm, int ret; if (dd->pio_only) { - scatterwalk_start(&dd->in_walk, dd->in_sg); - scatterwalk_start(&dd->out_walk, dd->out_sg); + dd->in_sg_offset = 0; + dd->out_sg_offset = 0; /* Enable DATAIN interrupt and let it take care of the rest */ @@ -836,21 +833,18 @@ static irqreturn_t omap_des_irq(int irq, void *dev_id) BUG_ON(!dd->in_sg); - BUG_ON(_calc_walked(in) > dd->in_sg->length); + BUG_ON(dd->in_sg_offset > dd->in_sg->length); - src = sg_virt(dd->in_sg) + _calc_walked(in); + src = sg_virt(dd->in_sg) + dd->in_sg_offset; for (i = 0; i < DES_BLOCK_WORDS; i++) { omap_des_write(dd, DES_REG_DATA_N(dd, i), *src); - - scatterwalk_advance(&dd->in_walk, 4); - if (dd->in_sg->length == _calc_walked(in)) { + dd->in_sg_offset += 4; + if (dd->in_sg_offset == dd->in_sg->length) { dd->in_sg = sg_next(dd->in_sg); if (dd->in_sg) { - scatterwalk_start(&dd->in_walk, - dd->in_sg); - src = sg_virt(dd->in_sg) + - _calc_walked(in); + dd->in_sg_offset = 0; + src = sg_virt(dd->in_sg); } } else { src++; @@ -869,20 +863,18 @@ static irqreturn_t omap_des_irq(int irq, void *dev_id) BUG_ON(!dd->out_sg); - BUG_ON(_calc_walked(out) > dd->out_sg->length); + BUG_ON(dd->out_sg_offset > dd->out_sg->length); - dst = sg_virt(dd->out_sg) + _calc_walked(out); + dst = sg_virt(dd->out_sg) + dd->out_sg_offset; for (i = 0; i < DES_BLOCK_WORDS; i++) { *dst = omap_des_read(dd, DES_REG_DATA_N(dd, i)); - scatterwalk_advance(&dd->out_walk, 4); - if (dd->out_sg->length == _calc_walked(out)) { + dd->out_sg_offset += 4; + if (dd->out_sg_offset == dd->out_sg->length) { dd->out_sg = sg_next(dd->out_sg); if (dd->out_sg) { - scatterwalk_start(&dd->out_walk, - dd->out_sg); - dst = sg_virt(dd->out_sg) + - _calc_walked(out); + dd->out_sg_offset = 0; + dst = sg_virt(dd->out_sg); } } else { dst++; diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c index 848e5e802b92..e95e84486d9a 100644 --- a/drivers/crypto/qce/core.c +++ b/drivers/crypto/qce/core.c @@ -3,14 +3,15 @@ * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved. */ +#include <linux/cleanup.h> #include <linux/clk.h> +#include <linux/device.h> #include <linux/dma-mapping.h> #include <linux/interconnect.h> #include <linux/interrupt.h> #include <linux/module.h> #include <linux/mod_devicetable.h> #include <linux/platform_device.h> -#include <linux/spinlock.h> #include <linux/types.h> #include <crypto/algapi.h> #include <crypto/internal/hash.h> @@ -37,9 +38,10 @@ static const struct qce_algo_ops *qce_ops[] = { #endif }; -static void qce_unregister_algs(struct qce_device *qce) +static void qce_unregister_algs(void *data) { const struct qce_algo_ops *ops; + struct qce_device *qce = data; int i; for (i = 0; i < ARRAY_SIZE(qce_ops); i++) { @@ -48,7 +50,7 @@ static void qce_unregister_algs(struct qce_device *qce) } } -static int qce_register_algs(struct qce_device *qce) +static int devm_qce_register_algs(struct qce_device *qce) { const struct qce_algo_ops *ops; int i, j, ret = -ENODEV; @@ -63,7 +65,7 @@ static int qce_register_algs(struct qce_device *qce) } } - return 0; + return devm_add_action_or_reset(qce->dev, qce_unregister_algs, qce); } static int qce_handle_request(struct crypto_async_request *async_req) @@ -87,55 +89,49 @@ static int qce_handle_queue(struct qce_device *qce, struct crypto_async_request *req) { struct crypto_async_request *async_req, *backlog; - unsigned long flags; int ret = 0, err; - spin_lock_irqsave(&qce->lock, flags); + scoped_guard(mutex, &qce->lock) { + if (req) + ret = crypto_enqueue_request(&qce->queue, req); - if (req) - ret = crypto_enqueue_request(&qce->queue, req); + /* busy, do not dequeue request */ + if (qce->req) + return ret; - /* busy, do not dequeue request */ - if (qce->req) { - spin_unlock_irqrestore(&qce->lock, flags); - return ret; + backlog = crypto_get_backlog(&qce->queue); + async_req = crypto_dequeue_request(&qce->queue); + if (async_req) + qce->req = async_req; } - backlog = crypto_get_backlog(&qce->queue); - async_req = crypto_dequeue_request(&qce->queue); - if (async_req) - qce->req = async_req; - - spin_unlock_irqrestore(&qce->lock, flags); - if (!async_req) return ret; if (backlog) { - spin_lock_bh(&qce->lock); - crypto_request_complete(backlog, -EINPROGRESS); - spin_unlock_bh(&qce->lock); + scoped_guard(mutex, &qce->lock) + crypto_request_complete(backlog, -EINPROGRESS); } err = qce_handle_request(async_req); if (err) { qce->result = err; - tasklet_schedule(&qce->done_tasklet); + schedule_work(&qce->done_work); } return ret; } -static void qce_tasklet_req_done(unsigned long data) +static void qce_req_done_work(struct work_struct *work) { - struct qce_device *qce = (struct qce_device *)data; + struct qce_device *qce = container_of(work, struct qce_device, + done_work); struct crypto_async_request *req; - unsigned long flags; - spin_lock_irqsave(&qce->lock, flags); - req = qce->req; - qce->req = NULL; - spin_unlock_irqrestore(&qce->lock, flags); + scoped_guard(mutex, &qce->lock) { + req = qce->req; + qce->req = NULL; + } if (req) crypto_request_complete(req, qce->result); @@ -152,7 +148,7 @@ static int qce_async_request_enqueue(struct qce_device *qce, static void qce_async_request_done(struct qce_device *qce, int ret) { qce->result = ret; - tasklet_schedule(&qce->done_tasklet); + schedule_work(&qce->done_work); } static int qce_check_version(struct qce_device *qce) @@ -212,15 +208,15 @@ static int qce_crypto_probe(struct platform_device *pdev) if (ret < 0) return ret; - qce->core = devm_clk_get_optional(qce->dev, "core"); + qce->core = devm_clk_get_optional_enabled(qce->dev, "core"); if (IS_ERR(qce->core)) return PTR_ERR(qce->core); - qce->iface = devm_clk_get_optional(qce->dev, "iface"); + qce->iface = devm_clk_get_optional_enabled(qce->dev, "iface"); if (IS_ERR(qce->iface)) return PTR_ERR(qce->iface); - qce->bus = devm_clk_get_optional(qce->dev, "bus"); + qce->bus = devm_clk_get_optional_enabled(qce->dev, "bus"); if (IS_ERR(qce->bus)) return PTR_ERR(qce->bus); @@ -232,64 +228,25 @@ static int qce_crypto_probe(struct platform_device *pdev) if (ret) return ret; - ret = clk_prepare_enable(qce->core); + ret = devm_qce_dma_request(qce->dev, &qce->dma); if (ret) - goto err_mem_path_disable; - - ret = clk_prepare_enable(qce->iface); - if (ret) - goto err_clks_core; - - ret = clk_prepare_enable(qce->bus); - if (ret) - goto err_clks_iface; + return ret; - ret = qce_dma_request(qce->dev, &qce->dma); + ret = qce_check_version(qce); if (ret) - goto err_clks; + return ret; - ret = qce_check_version(qce); + ret = devm_mutex_init(qce->dev, &qce->lock); if (ret) - goto err_dma; + return ret; - spin_lock_init(&qce->lock); - tasklet_init(&qce->done_tasklet, qce_tasklet_req_done, - (unsigned long)qce); + INIT_WORK(&qce->done_work, qce_req_done_work); crypto_init_queue(&qce->queue, QCE_QUEUE_LENGTH); qce->async_req_enqueue = qce_async_request_enqueue; qce->async_req_done = qce_async_request_done; - ret = qce_register_algs(qce); - if (ret) - goto err_dma; - - return 0; - -err_dma: - qce_dma_release(&qce->dma); -err_clks: - clk_disable_unprepare(qce->bus); -err_clks_iface: - clk_disable_unprepare(qce->iface); -err_clks_core: - clk_disable_unprepare(qce->core); -err_mem_path_disable: - icc_set_bw(qce->mem_path, 0, 0); - - return ret; -} - -static void qce_crypto_remove(struct platform_device *pdev) -{ - struct qce_device *qce = platform_get_drvdata(pdev); - - tasklet_kill(&qce->done_tasklet); - qce_unregister_algs(qce); - qce_dma_release(&qce->dma); - clk_disable_unprepare(qce->bus); - clk_disable_unprepare(qce->iface); - clk_disable_unprepare(qce->core); + return devm_qce_register_algs(qce); } static const struct of_device_id qce_crypto_of_match[] = { @@ -302,7 +259,6 @@ MODULE_DEVICE_TABLE(of, qce_crypto_of_match); static struct platform_driver qce_crypto_driver = { .probe = qce_crypto_probe, - .remove = qce_crypto_remove, .driver = { .name = KBUILD_MODNAME, .of_match_table = qce_crypto_of_match, diff --git a/drivers/crypto/qce/core.h b/drivers/crypto/qce/core.h index 228fcd69ec51..eb6fa7a8b64a 100644 --- a/drivers/crypto/qce/core.h +++ b/drivers/crypto/qce/core.h @@ -6,13 +6,16 @@ #ifndef _CORE_H_ #define _CORE_H_ +#include <linux/mutex.h> +#include <linux/workqueue.h> + #include "dma.h" /** * struct qce_device - crypto engine device structure * @queue: crypto request queue * @lock: the lock protects queue and req - * @done_tasklet: done tasklet object + * @done_work: workqueue context * @req: current active request * @result: result of current transform * @base: virtual IO base @@ -28,8 +31,8 @@ */ struct qce_device { struct crypto_queue queue; - spinlock_t lock; - struct tasklet_struct done_tasklet; + struct mutex lock; + struct work_struct done_work; struct crypto_async_request *req; int result; void __iomem *base; diff --git a/drivers/crypto/qce/dma.c b/drivers/crypto/qce/dma.c index 46db5bf366b4..1dec7aea852d 100644 --- a/drivers/crypto/qce/dma.c +++ b/drivers/crypto/qce/dma.c @@ -3,12 +3,22 @@ * Copyright (c) 2012-2014, The Linux Foundation. All rights reserved. */ +#include <linux/device.h> #include <linux/dmaengine.h> #include <crypto/scatterwalk.h> #include "dma.h" -int qce_dma_request(struct device *dev, struct qce_dma_data *dma) +static void qce_dma_release(void *data) +{ + struct qce_dma_data *dma = data; + + dma_release_channel(dma->txchan); + dma_release_channel(dma->rxchan); + kfree(dma->result_buf); +} + +int devm_qce_dma_request(struct device *dev, struct qce_dma_data *dma) { int ret; @@ -31,7 +41,8 @@ int qce_dma_request(struct device *dev, struct qce_dma_data *dma) dma->ignore_buf = dma->result_buf + QCE_RESULT_BUF_SZ; - return 0; + return devm_add_action_or_reset(dev, qce_dma_release, dma); + error_nomem: dma_release_channel(dma->rxchan); error_rx: @@ -39,13 +50,6 @@ error_rx: return ret; } -void qce_dma_release(struct qce_dma_data *dma) -{ - dma_release_channel(dma->txchan); - dma_release_channel(dma->rxchan); - kfree(dma->result_buf); -} - struct scatterlist * qce_sgtable_add(struct sg_table *sgt, struct scatterlist *new_sgl, unsigned int max_len) diff --git a/drivers/crypto/qce/dma.h b/drivers/crypto/qce/dma.h index 786402169360..31629185000e 100644 --- a/drivers/crypto/qce/dma.h +++ b/drivers/crypto/qce/dma.h @@ -34,8 +34,7 @@ struct qce_dma_data { void *ignore_buf; }; -int qce_dma_request(struct device *dev, struct qce_dma_data *dma); -void qce_dma_release(struct qce_dma_data *dma); +int devm_qce_dma_request(struct device *dev, struct qce_dma_data *dma); int qce_dma_prep_sgs(struct qce_dma_data *dma, struct scatterlist *sg_in, int in_ents, struct scatterlist *sg_out, int out_ents, dma_async_tx_callback cb, void *cb_param); diff --git a/drivers/crypto/tegra/tegra-se-aes.c b/drivers/crypto/tegra/tegra-se-aes.c index d734c9a56786..0e07d0523291 100644 --- a/drivers/crypto/tegra/tegra-se-aes.c +++ b/drivers/crypto/tegra/tegra-se-aes.c @@ -28,6 +28,9 @@ struct tegra_aes_ctx { u32 ivsize; u32 key1_id; u32 key2_id; + u32 keylen; + u8 key1[AES_MAX_KEY_SIZE]; + u8 key2[AES_MAX_KEY_SIZE]; }; struct tegra_aes_reqctx { @@ -43,8 +46,9 @@ struct tegra_aead_ctx { struct tegra_se *se; unsigned int authsize; u32 alg; - u32 keylen; u32 key_id; + u32 keylen; + u8 key[AES_MAX_KEY_SIZE]; }; struct tegra_aead_reqctx { @@ -56,8 +60,8 @@ struct tegra_aead_reqctx { unsigned int cryptlen; unsigned int authsize; bool encrypt; - u32 config; u32 crypto_config; + u32 config; u32 key_id; u32 iv[4]; u8 authdata[16]; @@ -67,6 +71,8 @@ struct tegra_cmac_ctx { struct tegra_se *se; unsigned int alg; u32 key_id; + u32 keylen; + u8 key[AES_MAX_KEY_SIZE]; struct crypto_shash *fallback_tfm; }; @@ -260,17 +266,13 @@ static int tegra_aes_do_one_req(struct crypto_engine *engine, void *areq) struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); struct tegra_aes_reqctx *rctx = skcipher_request_ctx(req); struct tegra_se *se = ctx->se; - unsigned int cmdlen; + unsigned int cmdlen, key1_id, key2_id; int ret; - rctx->datbuf.buf = dma_alloc_coherent(se->dev, SE_AES_BUFLEN, - &rctx->datbuf.addr, GFP_KERNEL); - if (!rctx->datbuf.buf) - return -ENOMEM; - - rctx->datbuf.size = SE_AES_BUFLEN; - rctx->iv = (u32 *)req->iv; + rctx->iv = (ctx->alg == SE_ALG_ECB) ? NULL : (u32 *)req->iv; rctx->len = req->cryptlen; + key1_id = ctx->key1_id; + key2_id = ctx->key2_id; /* Pad input to AES Block size */ if (ctx->alg != SE_ALG_XTS) { @@ -278,20 +280,59 @@ static int tegra_aes_do_one_req(struct crypto_engine *engine, void *areq) rctx->len += AES_BLOCK_SIZE - (rctx->len % AES_BLOCK_SIZE); } + rctx->datbuf.size = rctx->len; + rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->datbuf.size, + &rctx->datbuf.addr, GFP_KERNEL); + if (!rctx->datbuf.buf) { + ret = -ENOMEM; + goto out_finalize; + } + scatterwalk_map_and_copy(rctx->datbuf.buf, req->src, 0, req->cryptlen, 0); + rctx->config = tegra234_aes_cfg(ctx->alg, rctx->encrypt); + rctx->crypto_config = tegra234_aes_crypto_cfg(ctx->alg, rctx->encrypt); + + if (!key1_id) { + ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key1, + ctx->keylen, ctx->alg, &key1_id); + if (ret) + goto out; + } + + rctx->crypto_config |= SE_AES_KEY_INDEX(key1_id); + + if (ctx->alg == SE_ALG_XTS) { + if (!key2_id) { + ret = tegra_key_submit_reserved_xts(ctx->se, ctx->key2, + ctx->keylen, ctx->alg, &key2_id); + if (ret) + goto out; + } + + rctx->crypto_config |= SE_AES_KEY2_INDEX(key2_id); + } + /* Prepare the command and submit for execution */ cmdlen = tegra_aes_prep_cmd(ctx, rctx); - ret = tegra_se_host1x_submit(se, cmdlen); + ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); /* Copy the result */ tegra_aes_update_iv(req, ctx); scatterwalk_map_and_copy(rctx->datbuf.buf, req->dst, 0, req->cryptlen, 1); +out: /* Free the buffer */ - dma_free_coherent(ctx->se->dev, SE_AES_BUFLEN, + dma_free_coherent(ctx->se->dev, rctx->datbuf.size, rctx->datbuf.buf, rctx->datbuf.addr); + if (tegra_key_is_reserved(key1_id)) + tegra_key_invalidate_reserved(ctx->se, key1_id, ctx->alg); + + if (tegra_key_is_reserved(key2_id)) + tegra_key_invalidate_reserved(ctx->se, key2_id, ctx->alg); + +out_finalize: crypto_finalize_skcipher_request(se->engine, req, ret); return 0; @@ -313,6 +354,7 @@ static int tegra_aes_cra_init(struct crypto_skcipher *tfm) ctx->se = se_alg->se_dev; ctx->key1_id = 0; ctx->key2_id = 0; + ctx->keylen = 0; algname = crypto_tfm_alg_name(&tfm->base); ret = se_algname_to_algid(algname); @@ -341,13 +383,20 @@ static int tegra_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, u32 keylen) { struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + int ret; if (aes_check_keylen(keylen)) { dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); return -EINVAL; } - return tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key1_id); + ret = tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key1_id); + if (ret) { + ctx->keylen = keylen; + memcpy(ctx->key1, key, keylen); + } + + return 0; } static int tegra_xts_setkey(struct crypto_skcipher *tfm, @@ -365,11 +414,17 @@ static int tegra_xts_setkey(struct crypto_skcipher *tfm, ret = tegra_key_submit(ctx->se, key, len, ctx->alg, &ctx->key1_id); - if (ret) - return ret; + if (ret) { + ctx->keylen = len; + memcpy(ctx->key1, key, len); + } - return tegra_key_submit(ctx->se, key + len, len, + ret = tegra_key_submit(ctx->se, key + len, len, ctx->alg, &ctx->key2_id); + if (ret) { + ctx->keylen = len; + memcpy(ctx->key2, key + len, len); + } return 0; } @@ -444,12 +499,6 @@ static int tegra_aes_crypt(struct skcipher_request *req, bool encrypt) return 0; rctx->encrypt = encrypt; - rctx->config = tegra234_aes_cfg(ctx->alg, encrypt); - rctx->crypto_config = tegra234_aes_crypto_cfg(ctx->alg, encrypt); - rctx->crypto_config |= SE_AES_KEY_INDEX(ctx->key1_id); - - if (ctx->key2_id) - rctx->crypto_config |= SE_AES_KEY2_INDEX(ctx->key2_id); return crypto_transfer_skcipher_request_to_engine(ctx->se->engine, req); } @@ -715,11 +764,11 @@ static int tegra_gcm_do_gmac(struct tegra_aead_ctx *ctx, struct tegra_aead_reqct rctx->config = tegra234_aes_cfg(SE_ALG_GMAC, rctx->encrypt); rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GMAC, rctx->encrypt) | - SE_AES_KEY_INDEX(ctx->key_id); + SE_AES_KEY_INDEX(rctx->key_id); cmdlen = tegra_gmac_prep_cmd(ctx, rctx); - return tegra_se_host1x_submit(se, cmdlen); + return tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); } static int tegra_gcm_do_crypt(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) @@ -732,11 +781,11 @@ static int tegra_gcm_do_crypt(struct tegra_aead_ctx *ctx, struct tegra_aead_reqc rctx->config = tegra234_aes_cfg(SE_ALG_GCM, rctx->encrypt); rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GCM, rctx->encrypt) | - SE_AES_KEY_INDEX(ctx->key_id); + SE_AES_KEY_INDEX(rctx->key_id); /* Prepare command and submit */ cmdlen = tegra_gcm_crypt_prep_cmd(ctx, rctx); - ret = tegra_se_host1x_submit(se, cmdlen); + ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); if (ret) return ret; @@ -755,11 +804,11 @@ static int tegra_gcm_do_final(struct tegra_aead_ctx *ctx, struct tegra_aead_reqc rctx->config = tegra234_aes_cfg(SE_ALG_GCM_FINAL, rctx->encrypt); rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GCM_FINAL, rctx->encrypt) | - SE_AES_KEY_INDEX(ctx->key_id); + SE_AES_KEY_INDEX(rctx->key_id); /* Prepare command and submit */ cmdlen = tegra_gcm_prep_final_cmd(se, cpuvaddr, rctx); - ret = tegra_se_host1x_submit(se, cmdlen); + ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); if (ret) return ret; @@ -886,12 +935,12 @@ static int tegra_ccm_do_cbcmac(struct tegra_aead_ctx *ctx, struct tegra_aead_req rctx->config = tegra234_aes_cfg(SE_ALG_CBC_MAC, rctx->encrypt); rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_CBC_MAC, rctx->encrypt) | - SE_AES_KEY_INDEX(ctx->key_id); + SE_AES_KEY_INDEX(rctx->key_id); /* Prepare command and submit */ cmdlen = tegra_cbcmac_prep_cmd(ctx, rctx); - return tegra_se_host1x_submit(se, cmdlen); + return tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); } static int tegra_ccm_set_msg_len(u8 *block, unsigned int msglen, int csize) @@ -1073,7 +1122,7 @@ static int tegra_ccm_do_ctr(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx rctx->config = tegra234_aes_cfg(SE_ALG_CTR, rctx->encrypt); rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_CTR, rctx->encrypt) | - SE_AES_KEY_INDEX(ctx->key_id); + SE_AES_KEY_INDEX(rctx->key_id); /* Copy authdata in the top of buffer for encryption/decryption */ if (rctx->encrypt) @@ -1098,7 +1147,7 @@ static int tegra_ccm_do_ctr(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx /* Prepare command and submit */ cmdlen = tegra_ctr_prep_cmd(ctx, rctx); - ret = tegra_se_host1x_submit(se, cmdlen); + ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); if (ret) return ret; @@ -1117,6 +1166,11 @@ static int tegra_ccm_crypt_init(struct aead_request *req, struct tegra_se *se, rctx->assoclen = req->assoclen; rctx->authsize = crypto_aead_authsize(tfm); + if (rctx->encrypt) + rctx->cryptlen = req->cryptlen; + else + rctx->cryptlen = req->cryptlen - rctx->authsize; + memcpy(iv, req->iv, 16); ret = tegra_ccm_check_iv(iv); @@ -1145,30 +1199,35 @@ static int tegra_ccm_do_one_req(struct crypto_engine *engine, void *areq) struct tegra_se *se = ctx->se; int ret; + ret = tegra_ccm_crypt_init(req, se, rctx); + if (ret) + goto out_finalize; + + rctx->key_id = ctx->key_id; + /* Allocate buffers required */ - rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, SE_AES_BUFLEN, + rctx->inbuf.size = rctx->assoclen + rctx->authsize + rctx->cryptlen + 100; + rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, rctx->inbuf.size, &rctx->inbuf.addr, GFP_KERNEL); if (!rctx->inbuf.buf) - return -ENOMEM; - - rctx->inbuf.size = SE_AES_BUFLEN; + goto out_finalize; - rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, SE_AES_BUFLEN, + rctx->outbuf.size = rctx->assoclen + rctx->authsize + rctx->cryptlen + 100; + rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, rctx->outbuf.size, &rctx->outbuf.addr, GFP_KERNEL); if (!rctx->outbuf.buf) { ret = -ENOMEM; - goto outbuf_err; + goto out_free_inbuf; } - rctx->outbuf.size = SE_AES_BUFLEN; - - ret = tegra_ccm_crypt_init(req, se, rctx); - if (ret) - goto out; + if (!ctx->key_id) { + ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key, + ctx->keylen, ctx->alg, &rctx->key_id); + if (ret) + goto out; + } if (rctx->encrypt) { - rctx->cryptlen = req->cryptlen; - /* CBC MAC Operation */ ret = tegra_ccm_compute_auth(ctx, rctx); if (ret) @@ -1179,8 +1238,6 @@ static int tegra_ccm_do_one_req(struct crypto_engine *engine, void *areq) if (ret) goto out; } else { - rctx->cryptlen = req->cryptlen - ctx->authsize; - /* CTR operation */ ret = tegra_ccm_do_ctr(ctx, rctx); if (ret) @@ -1193,13 +1250,17 @@ static int tegra_ccm_do_one_req(struct crypto_engine *engine, void *areq) } out: - dma_free_coherent(ctx->se->dev, SE_AES_BUFLEN, + dma_free_coherent(ctx->se->dev, rctx->inbuf.size, rctx->outbuf.buf, rctx->outbuf.addr); -outbuf_err: - dma_free_coherent(ctx->se->dev, SE_AES_BUFLEN, +out_free_inbuf: + dma_free_coherent(ctx->se->dev, rctx->outbuf.size, rctx->inbuf.buf, rctx->inbuf.addr); + if (tegra_key_is_reserved(rctx->key_id)) + tegra_key_invalidate_reserved(ctx->se, rctx->key_id, ctx->alg); + +out_finalize: crypto_finalize_aead_request(ctx->se->engine, req, ret); return 0; @@ -1213,23 +1274,6 @@ static int tegra_gcm_do_one_req(struct crypto_engine *engine, void *areq) struct tegra_aead_reqctx *rctx = aead_request_ctx(req); int ret; - /* Allocate buffers required */ - rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, SE_AES_BUFLEN, - &rctx->inbuf.addr, GFP_KERNEL); - if (!rctx->inbuf.buf) - return -ENOMEM; - - rctx->inbuf.size = SE_AES_BUFLEN; - - rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, SE_AES_BUFLEN, - &rctx->outbuf.addr, GFP_KERNEL); - if (!rctx->outbuf.buf) { - ret = -ENOMEM; - goto outbuf_err; - } - - rctx->outbuf.size = SE_AES_BUFLEN; - rctx->src_sg = req->src; rctx->dst_sg = req->dst; rctx->assoclen = req->assoclen; @@ -1243,6 +1287,32 @@ static int tegra_gcm_do_one_req(struct crypto_engine *engine, void *areq) memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); rctx->iv[3] = (1 << 24); + rctx->key_id = ctx->key_id; + + /* Allocate buffers required */ + rctx->inbuf.size = rctx->assoclen + rctx->authsize + rctx->cryptlen; + rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, rctx->inbuf.size, + &rctx->inbuf.addr, GFP_KERNEL); + if (!rctx->inbuf.buf) { + ret = -ENOMEM; + goto out_finalize; + } + + rctx->outbuf.size = rctx->assoclen + rctx->authsize + rctx->cryptlen; + rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, rctx->outbuf.size, + &rctx->outbuf.addr, GFP_KERNEL); + if (!rctx->outbuf.buf) { + ret = -ENOMEM; + goto out_free_inbuf; + } + + if (!ctx->key_id) { + ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key, + ctx->keylen, ctx->alg, &rctx->key_id); + if (ret) + goto out; + } + /* If there is associated data perform GMAC operation */ if (rctx->assoclen) { ret = tegra_gcm_do_gmac(ctx, rctx); @@ -1266,14 +1336,17 @@ static int tegra_gcm_do_one_req(struct crypto_engine *engine, void *areq) ret = tegra_gcm_do_verify(ctx->se, rctx); out: - dma_free_coherent(ctx->se->dev, SE_AES_BUFLEN, + dma_free_coherent(ctx->se->dev, rctx->outbuf.size, rctx->outbuf.buf, rctx->outbuf.addr); -outbuf_err: - dma_free_coherent(ctx->se->dev, SE_AES_BUFLEN, +out_free_inbuf: + dma_free_coherent(ctx->se->dev, rctx->inbuf.size, rctx->inbuf.buf, rctx->inbuf.addr); - /* Finalize the request if there are no errors */ + if (tegra_key_is_reserved(rctx->key_id)) + tegra_key_invalidate_reserved(ctx->se, rctx->key_id, ctx->alg); + +out_finalize: crypto_finalize_aead_request(ctx->se->engine, req, ret); return 0; @@ -1295,6 +1368,7 @@ static int tegra_aead_cra_init(struct crypto_aead *tfm) ctx->se = se_alg->se_dev; ctx->key_id = 0; + ctx->keylen = 0; ret = se_algname_to_algid(algname); if (ret < 0) { @@ -1376,13 +1450,20 @@ static int tegra_aead_setkey(struct crypto_aead *tfm, const u8 *key, u32 keylen) { struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); + int ret; if (aes_check_keylen(keylen)) { dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); return -EINVAL; } - return tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); + ret = tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); + if (ret) { + ctx->keylen = keylen; + memcpy(ctx->key, key, keylen); + } + + return 0; } static unsigned int tegra_cmac_prep_cmd(struct tegra_cmac_ctx *ctx, @@ -1456,6 +1537,35 @@ static void tegra_cmac_paste_result(struct tegra_se *se, struct tegra_cmac_reqct se->base + se->hw->regs->result + (i * 4)); } +static int tegra_cmac_do_init(struct ahash_request *req) +{ + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_se *se = ctx->se; + int i; + + rctx->total_len = 0; + rctx->datbuf.size = 0; + rctx->residue.size = 0; + rctx->key_id = ctx->key_id; + rctx->task |= SHA_FIRST; + rctx->blk_size = crypto_ahash_blocksize(tfm); + + rctx->residue.buf = dma_alloc_coherent(se->dev, rctx->blk_size * 2, + &rctx->residue.addr, GFP_KERNEL); + if (!rctx->residue.buf) + return -ENOMEM; + + rctx->residue.size = 0; + + /* Clear any previous result */ + for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) + writel(0, se->base + se->hw->regs->result + (i * 4)); + + return 0; +} + static int tegra_cmac_do_update(struct ahash_request *req) { struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); @@ -1483,7 +1593,7 @@ static int tegra_cmac_do_update(struct ahash_request *req) rctx->datbuf.size = (req->nbytes + rctx->residue.size) - nresidue; rctx->total_len += rctx->datbuf.size; rctx->config = tegra234_aes_cfg(SE_ALG_CMAC, 0); - rctx->crypto_config = SE_AES_KEY_INDEX(ctx->key_id); + rctx->crypto_config = SE_AES_KEY_INDEX(rctx->key_id); /* * Keep one block and residue bytes in residue and @@ -1497,6 +1607,11 @@ static int tegra_cmac_do_update(struct ahash_request *req) return 0; } + rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->datbuf.size, + &rctx->datbuf.addr, GFP_KERNEL); + if (!rctx->datbuf.buf) + return -ENOMEM; + /* Copy the previous residue first */ if (rctx->residue.size) memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); @@ -1511,23 +1626,19 @@ static int tegra_cmac_do_update(struct ahash_request *req) rctx->residue.size = nresidue; /* - * If this is not the first 'update' call, paste the previous copied + * If this is not the first task, paste the previous copied * intermediate results to the registers so that it gets picked up. - * This is to support the import/export functionality. */ if (!(rctx->task & SHA_FIRST)) tegra_cmac_paste_result(ctx->se, rctx); cmdlen = tegra_cmac_prep_cmd(ctx, rctx); + ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); - ret = tegra_se_host1x_submit(se, cmdlen); - /* - * If this is not the final update, copy the intermediate results - * from the registers so that it can be used in the next 'update' - * call. This is to support the import/export functionality. - */ - if (!(rctx->task & SHA_FINAL)) - tegra_cmac_copy_result(ctx->se, rctx); + tegra_cmac_copy_result(ctx->se, rctx); + + dma_free_coherent(ctx->se->dev, rctx->datbuf.size, + rctx->datbuf.buf, rctx->datbuf.addr); return ret; } @@ -1543,17 +1654,34 @@ static int tegra_cmac_do_final(struct ahash_request *req) if (!req->nbytes && !rctx->total_len && ctx->fallback_tfm) { return crypto_shash_tfm_digest(ctx->fallback_tfm, - rctx->datbuf.buf, 0, req->result); + NULL, 0, req->result); + } + + if (rctx->residue.size) { + rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->residue.size, + &rctx->datbuf.addr, GFP_KERNEL); + if (!rctx->datbuf.buf) { + ret = -ENOMEM; + goto out_free; + } + + memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); } - memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); rctx->datbuf.size = rctx->residue.size; rctx->total_len += rctx->residue.size; rctx->config = tegra234_aes_cfg(SE_ALG_CMAC, 0); + /* + * If this is not the first task, paste the previous copied + * intermediate results to the registers so that it gets picked up. + */ + if (!(rctx->task & SHA_FIRST)) + tegra_cmac_paste_result(ctx->se, rctx); + /* Prepare command and submit */ cmdlen = tegra_cmac_prep_cmd(ctx, rctx); - ret = tegra_se_host1x_submit(se, cmdlen); + ret = tegra_se_host1x_submit(se, se->cmdbuf, cmdlen); if (ret) goto out; @@ -1565,8 +1693,10 @@ static int tegra_cmac_do_final(struct ahash_request *req) writel(0, se->base + se->hw->regs->result + (i * 4)); out: - dma_free_coherent(se->dev, SE_SHA_BUFLEN, - rctx->datbuf.buf, rctx->datbuf.addr); + if (rctx->residue.size) + dma_free_coherent(se->dev, rctx->datbuf.size, + rctx->datbuf.buf, rctx->datbuf.addr); +out_free: dma_free_coherent(se->dev, crypto_ahash_blocksize(tfm) * 2, rctx->residue.buf, rctx->residue.addr); return ret; @@ -1579,17 +1709,41 @@ static int tegra_cmac_do_one_req(struct crypto_engine *engine, void *areq) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); struct tegra_se *se = ctx->se; - int ret; + int ret = 0; + + if (rctx->task & SHA_INIT) { + ret = tegra_cmac_do_init(req); + if (ret) + goto out; + + rctx->task &= ~SHA_INIT; + } + + if (!ctx->key_id) { + ret = tegra_key_submit_reserved_aes(ctx->se, ctx->key, + ctx->keylen, ctx->alg, &rctx->key_id); + if (ret) + goto out; + } if (rctx->task & SHA_UPDATE) { ret = tegra_cmac_do_update(req); + if (ret) + goto out; + rctx->task &= ~SHA_UPDATE; } if (rctx->task & SHA_FINAL) { ret = tegra_cmac_do_final(req); + if (ret) + goto out; + rctx->task &= ~SHA_FINAL; } +out: + if (tegra_key_is_reserved(rctx->key_id)) + tegra_key_invalidate_reserved(ctx->se, rctx->key_id, ctx->alg); crypto_finalize_hash_request(se->engine, req, ret); @@ -1631,6 +1785,7 @@ static int tegra_cmac_cra_init(struct crypto_tfm *tfm) ctx->se = se_alg->se_dev; ctx->key_id = 0; + ctx->keylen = 0; ret = se_algname_to_algid(algname); if (ret < 0) { @@ -1655,51 +1810,11 @@ static void tegra_cmac_cra_exit(struct crypto_tfm *tfm) tegra_key_invalidate(ctx->se, ctx->key_id, ctx->alg); } -static int tegra_cmac_init(struct ahash_request *req) -{ - struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); - struct tegra_se *se = ctx->se; - int i; - - rctx->total_len = 0; - rctx->datbuf.size = 0; - rctx->residue.size = 0; - rctx->task = SHA_FIRST; - rctx->blk_size = crypto_ahash_blocksize(tfm); - - rctx->residue.buf = dma_alloc_coherent(se->dev, rctx->blk_size * 2, - &rctx->residue.addr, GFP_KERNEL); - if (!rctx->residue.buf) - goto resbuf_fail; - - rctx->residue.size = 0; - - rctx->datbuf.buf = dma_alloc_coherent(se->dev, SE_SHA_BUFLEN, - &rctx->datbuf.addr, GFP_KERNEL); - if (!rctx->datbuf.buf) - goto datbuf_fail; - - rctx->datbuf.size = 0; - - /* Clear any previous result */ - for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) - writel(0, se->base + se->hw->regs->result + (i * 4)); - - return 0; - -datbuf_fail: - dma_free_coherent(se->dev, rctx->blk_size, rctx->residue.buf, - rctx->residue.addr); -resbuf_fail: - return -ENOMEM; -} - static int tegra_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + int ret; if (aes_check_keylen(keylen)) { dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); @@ -1709,7 +1824,24 @@ static int tegra_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, if (ctx->fallback_tfm) crypto_shash_setkey(ctx->fallback_tfm, key, keylen); - return tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); + ret = tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); + if (ret) { + ctx->keylen = keylen; + memcpy(ctx->key, key, keylen); + } + + return 0; +} + +static int tegra_cmac_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + + rctx->task = SHA_INIT; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); } static int tegra_cmac_update(struct ahash_request *req) @@ -1750,13 +1882,9 @@ static int tegra_cmac_digest(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); - int ret; - ret = tegra_cmac_init(req); - if (ret) - return ret; + rctx->task |= SHA_INIT | SHA_UPDATE | SHA_FINAL; - rctx->task |= SHA_UPDATE | SHA_FINAL; return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); } diff --git a/drivers/crypto/tegra/tegra-se-hash.c b/drivers/crypto/tegra/tegra-se-hash.c index 0b5cdd5676b1..42d007b7af45 100644 --- a/drivers/crypto/tegra/tegra-se-hash.c +++ b/drivers/crypto/tegra/tegra-se-hash.c @@ -34,6 +34,7 @@ struct tegra_sha_reqctx { struct tegra_se_datbuf datbuf; struct tegra_se_datbuf residue; struct tegra_se_datbuf digest; + struct tegra_se_datbuf intr_res; unsigned int alg; unsigned int config; unsigned int total_len; @@ -211,9 +212,62 @@ static int tegra_sha_fallback_export(struct ahash_request *req, void *out) return crypto_ahash_export(&rctx->fallback_req, out); } -static int tegra_sha_prep_cmd(struct tegra_se *se, u32 *cpuvaddr, +static int tegra_se_insert_hash_result(struct tegra_sha_ctx *ctx, u32 *cpuvaddr, + struct tegra_sha_reqctx *rctx) +{ + __be32 *res_be = (__be32 *)rctx->intr_res.buf; + u32 *res = (u32 *)rctx->intr_res.buf; + int i = 0, j; + + cpuvaddr[i++] = 0; + cpuvaddr[i++] = host1x_opcode_setpayload(HASH_RESULT_REG_COUNT); + cpuvaddr[i++] = se_host1x_opcode_incr_w(SE_SHA_HASH_RESULT); + + for (j = 0; j < HASH_RESULT_REG_COUNT; j++) { + int idx = j; + + /* + * The initial, intermediate and final hash value of SHA-384, SHA-512 + * in SHA_HASH_RESULT registers follow the below layout of bytes. + * + * +---------------+------------+ + * | HASH_RESULT_0 | B4...B7 | + * +---------------+------------+ + * | HASH_RESULT_1 | B0...B3 | + * +---------------+------------+ + * | HASH_RESULT_2 | B12...B15 | + * +---------------+------------+ + * | HASH_RESULT_3 | B8...B11 | + * +---------------+------------+ + * | ...... | + * +---------------+------------+ + * | HASH_RESULT_14| B60...B63 | + * +---------------+------------+ + * | HASH_RESULT_15| B56...B59 | + * +---------------+------------+ + * + */ + if (ctx->alg == SE_ALG_SHA384 || ctx->alg == SE_ALG_SHA512) + idx = (j % 2) ? j - 1 : j + 1; + + /* For SHA-1, SHA-224, SHA-256, SHA-384, SHA-512 the initial + * intermediate and final hash value when stored in + * SHA_HASH_RESULT registers, the byte order is NOT in + * little-endian. + */ + if (ctx->alg <= SE_ALG_SHA512) + cpuvaddr[i++] = be32_to_cpu(res_be[idx]); + else + cpuvaddr[i++] = res[idx]; + } + + return i; +} + +static int tegra_sha_prep_cmd(struct tegra_sha_ctx *ctx, u32 *cpuvaddr, struct tegra_sha_reqctx *rctx) { + struct tegra_se *se = ctx->se; u64 msg_len, msg_left; int i = 0; @@ -241,7 +295,7 @@ static int tegra_sha_prep_cmd(struct tegra_se *se, u32 *cpuvaddr, cpuvaddr[i++] = upper_32_bits(msg_left); cpuvaddr[i++] = 0; cpuvaddr[i++] = 0; - cpuvaddr[i++] = host1x_opcode_setpayload(6); + cpuvaddr[i++] = host1x_opcode_setpayload(2); cpuvaddr[i++] = se_host1x_opcode_incr_w(SE_SHA_CFG); cpuvaddr[i++] = rctx->config; @@ -249,15 +303,29 @@ static int tegra_sha_prep_cmd(struct tegra_se *se, u32 *cpuvaddr, cpuvaddr[i++] = SE_SHA_TASK_HASH_INIT; rctx->task &= ~SHA_FIRST; } else { - cpuvaddr[i++] = 0; + /* + * If it isn't the first task, program the HASH_RESULT register + * with the intermediate result from the previous task + */ + i += tegra_se_insert_hash_result(ctx, cpuvaddr + i, rctx); } + cpuvaddr[i++] = host1x_opcode_setpayload(4); + cpuvaddr[i++] = se_host1x_opcode_incr_w(SE_SHA_IN_ADDR); cpuvaddr[i++] = rctx->datbuf.addr; cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(upper_32_bits(rctx->datbuf.addr)) | SE_ADDR_HI_SZ(rctx->datbuf.size)); - cpuvaddr[i++] = rctx->digest.addr; - cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(upper_32_bits(rctx->digest.addr)) | - SE_ADDR_HI_SZ(rctx->digest.size)); + + if (rctx->task & SHA_UPDATE) { + cpuvaddr[i++] = rctx->intr_res.addr; + cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(upper_32_bits(rctx->intr_res.addr)) | + SE_ADDR_HI_SZ(rctx->intr_res.size)); + } else { + cpuvaddr[i++] = rctx->digest.addr; + cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(upper_32_bits(rctx->digest.addr)) | + SE_ADDR_HI_SZ(rctx->digest.size)); + } + if (rctx->key_id) { cpuvaddr[i++] = host1x_opcode_setpayload(1); cpuvaddr[i++] = se_host1x_opcode_nonincr_w(SE_SHA_CRYPTO_CFG); @@ -266,42 +334,72 @@ static int tegra_sha_prep_cmd(struct tegra_se *se, u32 *cpuvaddr, cpuvaddr[i++] = host1x_opcode_setpayload(1); cpuvaddr[i++] = se_host1x_opcode_nonincr_w(SE_SHA_OPERATION); - cpuvaddr[i++] = SE_SHA_OP_WRSTALL | - SE_SHA_OP_START | + cpuvaddr[i++] = SE_SHA_OP_WRSTALL | SE_SHA_OP_START | SE_SHA_OP_LASTBUF; cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); - dev_dbg(se->dev, "msg len %llu msg left %llu cfg %#x", - msg_len, msg_left, rctx->config); + dev_dbg(se->dev, "msg len %llu msg left %llu sz %zd cfg %#x", + msg_len, msg_left, rctx->datbuf.size, rctx->config); return i; } -static void tegra_sha_copy_hash_result(struct tegra_se *se, struct tegra_sha_reqctx *rctx) +static int tegra_sha_do_init(struct ahash_request *req) { - int i; + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_se *se = ctx->se; - for (i = 0; i < HASH_RESULT_REG_COUNT; i++) - rctx->result[i] = readl(se->base + se->hw->regs->result + (i * 4)); -} + if (ctx->fallback) + return tegra_sha_fallback_init(req); -static void tegra_sha_paste_hash_result(struct tegra_se *se, struct tegra_sha_reqctx *rctx) -{ - int i; + rctx->total_len = 0; + rctx->datbuf.size = 0; + rctx->residue.size = 0; + rctx->key_id = ctx->key_id; + rctx->task |= SHA_FIRST; + rctx->alg = ctx->alg; + rctx->blk_size = crypto_ahash_blocksize(tfm); + rctx->digest.size = crypto_ahash_digestsize(tfm); + + rctx->digest.buf = dma_alloc_coherent(se->dev, rctx->digest.size, + &rctx->digest.addr, GFP_KERNEL); + if (!rctx->digest.buf) + goto digbuf_fail; + + rctx->residue.buf = dma_alloc_coherent(se->dev, rctx->blk_size, + &rctx->residue.addr, GFP_KERNEL); + if (!rctx->residue.buf) + goto resbuf_fail; + + rctx->intr_res.size = HASH_RESULT_REG_COUNT * 4; + rctx->intr_res.buf = dma_alloc_coherent(se->dev, rctx->intr_res.size, + &rctx->intr_res.addr, GFP_KERNEL); + if (!rctx->intr_res.buf) + goto intr_res_fail; + + return 0; - for (i = 0; i < HASH_RESULT_REG_COUNT; i++) - writel(rctx->result[i], - se->base + se->hw->regs->result + (i * 4)); +intr_res_fail: + dma_free_coherent(se->dev, rctx->residue.size, rctx->residue.buf, + rctx->residue.addr); +resbuf_fail: + dma_free_coherent(se->dev, rctx->digest.size, rctx->digest.buf, + rctx->digest.addr); +digbuf_fail: + return -ENOMEM; } static int tegra_sha_do_update(struct ahash_request *req) { struct tegra_sha_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct tegra_se *se = ctx->se; unsigned int nblks, nresidue, size, ret; - u32 *cpuvaddr = ctx->se->cmdbuf->addr; + u32 *cpuvaddr = se->cmdbuf->addr; nresidue = (req->nbytes + rctx->residue.size) % rctx->blk_size; nblks = (req->nbytes + rctx->residue.size) / rctx->blk_size; @@ -317,7 +415,6 @@ static int tegra_sha_do_update(struct ahash_request *req) rctx->src_sg = req->src; rctx->datbuf.size = (req->nbytes + rctx->residue.size) - nresidue; - rctx->total_len += rctx->datbuf.size; /* * If nbytes are less than a block size, copy it residue and @@ -326,11 +423,16 @@ static int tegra_sha_do_update(struct ahash_request *req) if (nblks < 1) { scatterwalk_map_and_copy(rctx->residue.buf + rctx->residue.size, rctx->src_sg, 0, req->nbytes, 0); - rctx->residue.size += req->nbytes; + return 0; } + rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->datbuf.size, + &rctx->datbuf.addr, GFP_KERNEL); + if (!rctx->datbuf.buf) + return -ENOMEM; + /* Copy the previous residue first */ if (rctx->residue.size) memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); @@ -343,29 +445,16 @@ static int tegra_sha_do_update(struct ahash_request *req) /* Update residue value with the residue after current block */ rctx->residue.size = nresidue; + rctx->total_len += rctx->datbuf.size; rctx->config = tegra_sha_get_config(rctx->alg) | - SE_SHA_DST_HASH_REG; - - /* - * If this is not the first 'update' call, paste the previous copied - * intermediate results to the registers so that it gets picked up. - * This is to support the import/export functionality. - */ - if (!(rctx->task & SHA_FIRST)) - tegra_sha_paste_hash_result(ctx->se, rctx); + SE_SHA_DST_MEMORY; - size = tegra_sha_prep_cmd(ctx->se, cpuvaddr, rctx); + size = tegra_sha_prep_cmd(ctx, cpuvaddr, rctx); + ret = tegra_se_host1x_submit(se, se->cmdbuf, size); - ret = tegra_se_host1x_submit(ctx->se, size); - - /* - * If this is not the final update, copy the intermediate results - * from the registers so that it can be used in the next 'update' - * call. This is to support the import/export functionality. - */ - if (!(rctx->task & SHA_FINAL)) - tegra_sha_copy_hash_result(ctx->se, rctx); + dma_free_coherent(se->dev, rctx->datbuf.size, + rctx->datbuf.buf, rctx->datbuf.addr); return ret; } @@ -379,16 +468,25 @@ static int tegra_sha_do_final(struct ahash_request *req) u32 *cpuvaddr = se->cmdbuf->addr; int size, ret = 0; - memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); + if (rctx->residue.size) { + rctx->datbuf.buf = dma_alloc_coherent(se->dev, rctx->residue.size, + &rctx->datbuf.addr, GFP_KERNEL); + if (!rctx->datbuf.buf) { + ret = -ENOMEM; + goto out_free; + } + + memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); + } + rctx->datbuf.size = rctx->residue.size; rctx->total_len += rctx->residue.size; rctx->config = tegra_sha_get_config(rctx->alg) | SE_SHA_DST_MEMORY; - size = tegra_sha_prep_cmd(se, cpuvaddr, rctx); - - ret = tegra_se_host1x_submit(se, size); + size = tegra_sha_prep_cmd(ctx, cpuvaddr, rctx); + ret = tegra_se_host1x_submit(se, se->cmdbuf, size); if (ret) goto out; @@ -396,12 +494,18 @@ static int tegra_sha_do_final(struct ahash_request *req) memcpy(req->result, rctx->digest.buf, rctx->digest.size); out: - dma_free_coherent(se->dev, SE_SHA_BUFLEN, - rctx->datbuf.buf, rctx->datbuf.addr); + if (rctx->residue.size) + dma_free_coherent(se->dev, rctx->datbuf.size, + rctx->datbuf.buf, rctx->datbuf.addr); +out_free: dma_free_coherent(se->dev, crypto_ahash_blocksize(tfm), rctx->residue.buf, rctx->residue.addr); dma_free_coherent(se->dev, rctx->digest.size, rctx->digest.buf, rctx->digest.addr); + + dma_free_coherent(se->dev, rctx->intr_res.size, rctx->intr_res.buf, + rctx->intr_res.addr); + return ret; } @@ -414,16 +518,31 @@ static int tegra_sha_do_one_req(struct crypto_engine *engine, void *areq) struct tegra_se *se = ctx->se; int ret = 0; + if (rctx->task & SHA_INIT) { + ret = tegra_sha_do_init(req); + if (ret) + goto out; + + rctx->task &= ~SHA_INIT; + } + if (rctx->task & SHA_UPDATE) { ret = tegra_sha_do_update(req); + if (ret) + goto out; + rctx->task &= ~SHA_UPDATE; } if (rctx->task & SHA_FINAL) { ret = tegra_sha_do_final(req); + if (ret) + goto out; + rctx->task &= ~SHA_FINAL; } +out: crypto_finalize_hash_request(se->engine, req, ret); return 0; @@ -497,52 +616,6 @@ static void tegra_sha_cra_exit(struct crypto_tfm *tfm) tegra_key_invalidate(ctx->se, ctx->key_id, ctx->alg); } -static int tegra_sha_init(struct ahash_request *req) -{ - struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); - struct tegra_se *se = ctx->se; - - if (ctx->fallback) - return tegra_sha_fallback_init(req); - - rctx->total_len = 0; - rctx->datbuf.size = 0; - rctx->residue.size = 0; - rctx->key_id = ctx->key_id; - rctx->task = SHA_FIRST; - rctx->alg = ctx->alg; - rctx->blk_size = crypto_ahash_blocksize(tfm); - rctx->digest.size = crypto_ahash_digestsize(tfm); - - rctx->digest.buf = dma_alloc_coherent(se->dev, rctx->digest.size, - &rctx->digest.addr, GFP_KERNEL); - if (!rctx->digest.buf) - goto digbuf_fail; - - rctx->residue.buf = dma_alloc_coherent(se->dev, rctx->blk_size, - &rctx->residue.addr, GFP_KERNEL); - if (!rctx->residue.buf) - goto resbuf_fail; - - rctx->datbuf.buf = dma_alloc_coherent(se->dev, SE_SHA_BUFLEN, - &rctx->datbuf.addr, GFP_KERNEL); - if (!rctx->datbuf.buf) - goto datbuf_fail; - - return 0; - -datbuf_fail: - dma_free_coherent(se->dev, rctx->blk_size, rctx->residue.buf, - rctx->residue.addr); -resbuf_fail: - dma_free_coherent(se->dev, SE_SHA_BUFLEN, rctx->datbuf.buf, - rctx->datbuf.addr); -digbuf_fail: - return -ENOMEM; -} - static int tegra_hmac_fallback_setkey(struct tegra_sha_ctx *ctx, const u8 *key, unsigned int keylen) { @@ -559,13 +632,29 @@ static int tegra_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + int ret; if (aes_check_keylen(keylen)) return tegra_hmac_fallback_setkey(ctx, key, keylen); + ret = tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); + if (ret) + return tegra_hmac_fallback_setkey(ctx, key, keylen); + ctx->fallback = false; - return tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); + return 0; +} + +static int tegra_sha_init(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + rctx->task = SHA_INIT; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); } static int tegra_sha_update(struct ahash_request *req) @@ -615,16 +704,12 @@ static int tegra_sha_digest(struct ahash_request *req) struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); - int ret; if (ctx->fallback) return tegra_sha_fallback_digest(req); - ret = tegra_sha_init(req); - if (ret) - return ret; + rctx->task |= SHA_INIT | SHA_UPDATE | SHA_FINAL; - rctx->task |= SHA_UPDATE | SHA_FINAL; return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); } diff --git a/drivers/crypto/tegra/tegra-se-key.c b/drivers/crypto/tegra/tegra-se-key.c index ac14678dbd30..956fa9b4e9b1 100644 --- a/drivers/crypto/tegra/tegra-se-key.c +++ b/drivers/crypto/tegra/tegra-se-key.c @@ -115,11 +115,17 @@ static int tegra_key_insert(struct tegra_se *se, const u8 *key, u32 keylen, u16 slot, u32 alg) { const u32 *keyval = (u32 *)key; - u32 *addr = se->cmdbuf->addr, size; + u32 *addr = se->keybuf->addr, size; + int ret; + + mutex_lock(&kslt_lock); size = tegra_key_prep_ins_cmd(se, addr, keyval, keylen, slot, alg); + ret = tegra_se_host1x_submit(se, se->keybuf, size); + + mutex_unlock(&kslt_lock); - return tegra_se_host1x_submit(se, size); + return ret; } void tegra_key_invalidate(struct tegra_se *se, u32 keyid, u32 alg) @@ -135,6 +141,23 @@ void tegra_key_invalidate(struct tegra_se *se, u32 keyid, u32 alg) tegra_keyslot_free(keyid); } +void tegra_key_invalidate_reserved(struct tegra_se *se, u32 keyid, u32 alg) +{ + u8 zkey[AES_MAX_KEY_SIZE] = {0}; + + if (!keyid) + return; + + /* Overwrite the key with 0s */ + tegra_key_insert(se, zkey, AES_MAX_KEY_SIZE, keyid, alg); +} + +inline int tegra_key_submit_reserved(struct tegra_se *se, const u8 *key, + u32 keylen, u32 alg, u32 *keyid) +{ + return tegra_key_insert(se, key, keylen, *keyid, alg); +} + int tegra_key_submit(struct tegra_se *se, const u8 *key, u32 keylen, u32 alg, u32 *keyid) { int ret; @@ -143,7 +166,7 @@ int tegra_key_submit(struct tegra_se *se, const u8 *key, u32 keylen, u32 alg, u3 if (!tegra_key_in_kslt(*keyid)) { *keyid = tegra_keyslot_alloc(); if (!(*keyid)) { - dev_err(se->dev, "failed to allocate key slot\n"); + dev_dbg(se->dev, "failed to allocate key slot\n"); return -ENOMEM; } } diff --git a/drivers/crypto/tegra/tegra-se-main.c b/drivers/crypto/tegra/tegra-se-main.c index 918c0b10614d..1c94f1de0546 100644 --- a/drivers/crypto/tegra/tegra-se-main.c +++ b/drivers/crypto/tegra/tegra-se-main.c @@ -141,7 +141,7 @@ static struct tegra_se_cmdbuf *tegra_se_host1x_bo_alloc(struct tegra_se *se, ssi return cmdbuf; } -int tegra_se_host1x_submit(struct tegra_se *se, u32 size) +int tegra_se_host1x_submit(struct tegra_se *se, struct tegra_se_cmdbuf *cmdbuf, u32 size) { struct host1x_job *job; int ret; @@ -160,9 +160,9 @@ int tegra_se_host1x_submit(struct tegra_se *se, u32 size) job->engine_fallback_streamid = se->stream_id; job->engine_streamid_offset = SE_STREAM_ID; - se->cmdbuf->words = size; + cmdbuf->words = size; - host1x_job_add_gather(job, &se->cmdbuf->bo, size, 0); + host1x_job_add_gather(job, &cmdbuf->bo, size, 0); ret = host1x_job_pin(job, se->dev); if (ret) { @@ -220,14 +220,22 @@ static int tegra_se_client_init(struct host1x_client *client) goto syncpt_put; } + se->keybuf = tegra_se_host1x_bo_alloc(se, SZ_4K); + if (!se->keybuf) { + ret = -ENOMEM; + goto cmdbuf_put; + } + ret = se->hw->init_alg(se); if (ret) { dev_err(se->dev, "failed to register algorithms\n"); - goto cmdbuf_put; + goto keybuf_put; } return 0; +keybuf_put: + tegra_se_cmdbuf_put(&se->keybuf->bo); cmdbuf_put: tegra_se_cmdbuf_put(&se->cmdbuf->bo); syncpt_put: diff --git a/drivers/crypto/tegra/tegra-se.h b/drivers/crypto/tegra/tegra-se.h index b9dd7ceb8783..b6cac9384f66 100644 --- a/drivers/crypto/tegra/tegra-se.h +++ b/drivers/crypto/tegra/tegra-se.h @@ -24,6 +24,7 @@ #define SE_STREAM_ID 0x90 #define SE_SHA_CFG 0x4004 +#define SE_SHA_IN_ADDR 0x400c #define SE_SHA_KEY_ADDR 0x4094 #define SE_SHA_KEY_DATA 0x4098 #define SE_SHA_KEYMANIFEST 0x409c @@ -340,12 +341,14 @@ #define SE_CRYPTO_CTR_REG_COUNT 4 #define SE_MAX_KEYSLOT 15 #define SE_MAX_MEM_ALLOC SZ_4M -#define SE_AES_BUFLEN 0x8000 -#define SE_SHA_BUFLEN 0x2000 + +#define TEGRA_AES_RESERVED_KSLT 14 +#define TEGRA_XTS_RESERVED_KSLT 15 #define SHA_FIRST BIT(0) -#define SHA_UPDATE BIT(1) -#define SHA_FINAL BIT(2) +#define SHA_INIT BIT(1) +#define SHA_UPDATE BIT(2) +#define SHA_FINAL BIT(3) /* Security Engine operation modes */ enum se_aes_alg { @@ -420,6 +423,7 @@ struct tegra_se { struct host1x_client client; struct host1x_channel *channel; struct tegra_se_cmdbuf *cmdbuf; + struct tegra_se_cmdbuf *keybuf; struct crypto_engine *engine; struct host1x_syncpt *syncpt; struct device *dev; @@ -501,8 +505,33 @@ void tegra_deinit_aes(struct tegra_se *se); void tegra_deinit_hash(struct tegra_se *se); int tegra_key_submit(struct tegra_se *se, const u8 *key, u32 keylen, u32 alg, u32 *keyid); + +int tegra_key_submit_reserved(struct tegra_se *se, const u8 *key, + u32 keylen, u32 alg, u32 *keyid); + void tegra_key_invalidate(struct tegra_se *se, u32 keyid, u32 alg); -int tegra_se_host1x_submit(struct tegra_se *se, u32 size); +void tegra_key_invalidate_reserved(struct tegra_se *se, u32 keyid, u32 alg); +int tegra_se_host1x_submit(struct tegra_se *se, struct tegra_se_cmdbuf *cmdbuf, u32 size); + +static inline int tegra_key_submit_reserved_aes(struct tegra_se *se, const u8 *key, + u32 keylen, u32 alg, u32 *keyid) +{ + *keyid = TEGRA_AES_RESERVED_KSLT; + return tegra_key_submit_reserved(se, key, keylen, alg, keyid); +} + +static inline int tegra_key_submit_reserved_xts(struct tegra_se *se, const u8 *key, + u32 keylen, u32 alg, u32 *keyid) +{ + *keyid = TEGRA_XTS_RESERVED_KSLT; + return tegra_key_submit_reserved(se, key, keylen, alg, keyid); +} + +static inline bool tegra_key_is_reserved(u32 keyid) +{ + return ((keyid == TEGRA_AES_RESERVED_KSLT) || + (keyid == TEGRA_XTS_RESERVED_KSLT)); +} /* HOST1x OPCODES */ static inline u32 host1x_opcode_setpayload(unsigned int payload) |