From 0a8a9c8cf624f40b6f9fdd79a34c9c904bed0721 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 8 May 2025 13:22:16 +0800 Subject: crypto: marvell/cesa - Do not chain submitted requests commit 0413bcf0fc460a68a2a7a8354aee833293d7d693 upstream. This driver tries to chain requests together before submitting them to hardware in order to reduce completion interrupts. However, it even extends chains that have already been submitted to hardware. This is dangerous because there is no way of knowing whether the hardware has already read the DMA memory in question or not. Fix this by splitting the chain list into two. One for submitted requests and one for requests that have not yet been submitted. Only extend the latter. Reported-by: Klaus Kudielka Fixes: 85030c5168f1 ("crypto: marvell - Add support for chaining crypto requests in TDMA mode") Cc: Signed-off-by: Herbert Xu Signed-off-by: Greg Kroah-Hartman --- drivers/crypto/marvell/cesa/cesa.c | 2 +- drivers/crypto/marvell/cesa/cesa.h | 9 ++++--- drivers/crypto/marvell/cesa/tdma.c | 53 +++++++++++++++++++++++--------------- 3 files changed, 39 insertions(+), 25 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/marvell/cesa/cesa.c b/drivers/crypto/marvell/cesa/cesa.c index fa08f10e6f3f..9c21f5d835d2 100644 --- a/drivers/crypto/marvell/cesa/cesa.c +++ b/drivers/crypto/marvell/cesa/cesa.c @@ -94,7 +94,7 @@ static int mv_cesa_std_process(struct mv_cesa_engine *engine, u32 status) static int mv_cesa_int_process(struct mv_cesa_engine *engine, u32 status) { - if (engine->chain.first && engine->chain.last) + if (engine->chain_hw.first && engine->chain_hw.last) return mv_cesa_tdma_process(engine, status); return mv_cesa_std_process(engine, status); diff --git a/drivers/crypto/marvell/cesa/cesa.h b/drivers/crypto/marvell/cesa/cesa.h index d215a6bed6bc..50ca1039fdaa 100644 --- a/drivers/crypto/marvell/cesa/cesa.h +++ b/drivers/crypto/marvell/cesa/cesa.h @@ -440,8 +440,10 @@ struct mv_cesa_dev { * SRAM * @queue: fifo of the pending crypto requests * @load: engine load counter, useful for load balancing - * @chain: list of the current tdma descriptors being processed - * by this engine. + * @chain_hw: list of the current tdma descriptors being processed + * by the hardware. + * @chain_sw: list of the current tdma descriptors that will be + * submitted to the hardware. * @complete_queue: fifo of the processed requests by the engine * * Structure storing CESA engine information. @@ -463,7 +465,8 @@ struct mv_cesa_engine { struct gen_pool *pool; struct crypto_queue queue; atomic_t load; - struct mv_cesa_tdma_chain chain; + struct mv_cesa_tdma_chain chain_hw; + struct mv_cesa_tdma_chain chain_sw; struct list_head complete_queue; int irq; }; diff --git a/drivers/crypto/marvell/cesa/tdma.c b/drivers/crypto/marvell/cesa/tdma.c index 388a06e180d6..243305354420 100644 --- a/drivers/crypto/marvell/cesa/tdma.c +++ b/drivers/crypto/marvell/cesa/tdma.c @@ -38,6 +38,15 @@ void mv_cesa_dma_step(struct mv_cesa_req *dreq) { struct mv_cesa_engine *engine = dreq->engine; + spin_lock_bh(&engine->lock); + if (engine->chain_sw.first == dreq->chain.first) { + engine->chain_sw.first = NULL; + engine->chain_sw.last = NULL; + } + engine->chain_hw.first = dreq->chain.first; + engine->chain_hw.last = dreq->chain.last; + spin_unlock_bh(&engine->lock); + writel_relaxed(0, engine->regs + CESA_SA_CFG); mv_cesa_set_int_mask(engine, CESA_SA_INT_ACC0_IDMA_DONE); @@ -96,25 +105,27 @@ void mv_cesa_dma_prepare(struct mv_cesa_req *dreq, void mv_cesa_tdma_chain(struct mv_cesa_engine *engine, struct mv_cesa_req *dreq) { - if (engine->chain.first == NULL && engine->chain.last == NULL) { - engine->chain.first = dreq->chain.first; - engine->chain.last = dreq->chain.last; - } else { - struct mv_cesa_tdma_desc *last; + struct mv_cesa_tdma_desc *last = engine->chain_sw.last; - last = engine->chain.last; + /* + * Break the DMA chain if the request being queued needs the IV + * regs to be set before lauching the request. + */ + if (!last || dreq->chain.first->flags & CESA_TDMA_SET_STATE) + engine->chain_sw.first = dreq->chain.first; + else { last->next = dreq->chain.first; - engine->chain.last = dreq->chain.last; - - /* - * Break the DMA chain if the CESA_TDMA_BREAK_CHAIN is set on - * the last element of the current chain, or if the request - * being queued needs the IV regs to be set before lauching - * the request. - */ - if (!(last->flags & CESA_TDMA_BREAK_CHAIN) && - !(dreq->chain.first->flags & CESA_TDMA_SET_STATE)) - last->next_dma = cpu_to_le32(dreq->chain.first->cur_dma); + last->next_dma = cpu_to_le32(dreq->chain.first->cur_dma); + } + last = dreq->chain.last; + engine->chain_sw.last = last; + /* + * Break the DMA chain if the CESA_TDMA_BREAK_CHAIN is set on + * the last element of the current chain. + */ + if (last->flags & CESA_TDMA_BREAK_CHAIN) { + engine->chain_sw.first = NULL; + engine->chain_sw.last = NULL; } } @@ -127,7 +138,7 @@ int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status) tdma_cur = readl(engine->regs + CESA_TDMA_CUR); - for (tdma = engine->chain.first; tdma; tdma = next) { + for (tdma = engine->chain_hw.first; tdma; tdma = next) { spin_lock_bh(&engine->lock); next = tdma->next; spin_unlock_bh(&engine->lock); @@ -149,12 +160,12 @@ int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status) &backlog); /* Re-chaining to the next request */ - engine->chain.first = tdma->next; + engine->chain_hw.first = tdma->next; tdma->next = NULL; /* If this is the last request, clear the chain */ - if (engine->chain.first == NULL) - engine->chain.last = NULL; + if (engine->chain_hw.first == NULL) + engine->chain_hw.last = NULL; spin_unlock_bh(&engine->lock); ctx = crypto_tfm_ctx(req->tfm); -- cgit v1.2.3 From 9a0371cd9a0e631b80492a04741fc414d394946c Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 26 Mar 2025 15:59:53 +0000 Subject: crypto: qat - add shutdown handler to qat_c3xxx commit 71e0cc1eab584d6f95526a5e8c69ec666ca33e1b upstream. During a warm reset via kexec, the system bypasses the driver removal sequence, meaning that the remove() callback is not invoked. If a QAT device is not shutdown properly, the device driver will fail to load in a newly rebooted kernel. This might result in output like the following after the kexec reboot: QAT: AE0 is inactive!! QAT: failed to get device out of reset c3xxx 0000:3f:00.0: qat_hal_clr_reset error c3xxx 0000:3f:00.0: Failed to init the AEs c3xxx 0000:3f:00.0: Failed to initialise Acceleration Engine c3xxx 0000:3f:00.0: Resetting device qat_dev0 c3xxx 0000:3f:00.0: probe with driver c3xxx failed with error -14 Implement the shutdown() handler that hooks into the reboot notifier list. This brings down the QAT device and ensures it is shut down properly. Cc: Fixes: 890c55f4dc0e ("crypto: qat - add support for c3xxx accel type") Reviewed-by: Ahsan Atta Reviewed-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu Signed-off-by: Greg Kroah-Hartman --- drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c index b825b35ab4bf..566258e80371 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c @@ -19,6 +19,13 @@ #include #include "adf_c3xxx_hw_data.h" +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + static const struct pci_device_id adf_pci_tbl[] = { { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C3XXX), }, { } @@ -33,6 +40,7 @@ static struct pci_driver adf_driver = { .name = ADF_C3XXX_DEVICE_NAME, .probe = adf_probe, .remove = adf_remove, + .shutdown = adf_shutdown, .sriov_configure = adf_sriov_configure, .err_handler = &adf_err_handler, }; -- cgit v1.2.3 From b507b769811369a860852a97f657d6183fe11629 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 26 Mar 2025 15:59:47 +0000 Subject: crypto: qat - add shutdown handler to qat_420xx commit 097143f23a1164bfd1b6f70279d229be44da2e30 upstream. During a warm reset via kexec, the system bypasses the driver removal sequence, meaning that the remove() callback is not invoked. If a QAT device is not shutdown properly, the device driver will fail to load in a newly rebooted kernel. This might result in output like the following after the kexec reboot: 420xx 0000:01:00.0: Failed to power up the device 420xx 0000:01:00.0: Failed to initialize device 420xx 0000:01:00.0: Resetting device qat_dev0 420xx 0000:01:00.0: probe with driver 420xx failed with error -14 Implement the shutdown() handler that hooks into the reboot notifier list. This brings down the QAT device and ensures it is shut down properly. Cc: Fixes: fcf60f4bcf54 ("crypto: qat - add support for 420xx devices") Reviewed-by: Ahsan Atta Reviewed-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu Signed-off-by: Greg Kroah-Hartman --- drivers/crypto/intel/qat/qat_420xx/adf_drv.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c index 8084aa0f7f41..b4731f02deb8 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c @@ -186,11 +186,19 @@ static void adf_remove(struct pci_dev *pdev) adf_cleanup_accel(accel_dev); } +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + static struct pci_driver adf_driver = { .id_table = adf_pci_tbl, .name = ADF_420XX_DEVICE_NAME, .probe = adf_probe, .remove = adf_remove, + .shutdown = adf_shutdown, .sriov_configure = adf_sriov_configure, .err_handler = &adf_err_handler, }; -- cgit v1.2.3 From 62c679b2e12378b6ceec5a3cd2a97ad775b1ef2d Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 26 Mar 2025 15:59:46 +0000 Subject: crypto: qat - add shutdown handler to qat_4xxx commit 845bc952024dbf482c7434daeac66f764642d52d upstream. During a warm reset via kexec, the system bypasses the driver removal sequence, meaning that the remove() callback is not invoked. If a QAT device is not shutdown properly, the device driver will fail to load in a newly rebooted kernel. This might result in output like the following after the kexec reboot: 4xxx 0000:01:00.0: Failed to power up the device 4xxx 0000:01:00.0: Failed to initialize device 4xxx 0000:01:00.0: Resetting device qat_dev0 4xxx 0000:01:00.0: probe with driver 4xxx failed with error -14 Implement the shutdown() handler that hooks into the reboot notifier list. This brings down the QAT device and ensures it is shut down properly. Cc: Fixes: 8c8268166e83 ("crypto: qat - add qat_4xxx driver") Link: https://lore.kernel.org/all/Z-DGQrhRj9niR9iZ@gondor.apana.org.au/ Reported-by: Randy Wright Closes: https://issues.redhat.com/browse/RHEL-84366 Reviewed-by: Ahsan Atta Reviewed-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu Signed-off-by: Greg Kroah-Hartman --- drivers/crypto/intel/qat/qat_4xxx/adf_drv.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c index 5537a9991e4e..1ac415ef3c31 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c @@ -188,11 +188,19 @@ static void adf_remove(struct pci_dev *pdev) adf_cleanup_accel(accel_dev); } +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + static struct pci_driver adf_driver = { .id_table = adf_pci_tbl, .name = ADF_4XXX_DEVICE_NAME, .probe = adf_probe, .remove = adf_remove, + .shutdown = adf_shutdown, .sriov_configure = adf_sriov_configure, .err_handler = &adf_err_handler, }; -- cgit v1.2.3 From 366b56c9d3f5bef6181cd66263f45bf0917beca5 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 26 Mar 2025 15:59:51 +0000 Subject: crypto: qat - add shutdown handler to qat_c62x commit a9a6e9279b2998e2610c70b0dfc80a234f97c76c upstream. During a warm reset via kexec, the system bypasses the driver removal sequence, meaning that the remove() callback is not invoked. If a QAT device is not shutdown properly, the device driver will fail to load in a newly rebooted kernel. This might result in output like the following after the kexec reboot: QAT: AE0 is inactive!! QAT: failed to get device out of reset c6xx 0000:3f:00.0: qat_hal_clr_reset error c6xx 0000:3f:00.0: Failed to init the AEs c6xx 0000:3f:00.0: Failed to initialise Acceleration Engine c6xx 0000:3f:00.0: Resetting device qat_dev0 c6xx 0000:3f:00.0: probe with driver c6xx failed with error -14 Implement the shutdown() handler that hooks into the reboot notifier list. This brings down the QAT device and ensures it is shut down properly. Cc: Fixes: a6dabee6c8ba ("crypto: qat - add support for c62x accel type") Reviewed-by: Ahsan Atta Reviewed-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu Signed-off-by: Greg Kroah-Hartman --- drivers/crypto/intel/qat/qat_c62x/adf_drv.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c index 8a7bdec358d6..ce541a721952 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c @@ -19,6 +19,13 @@ #include #include "adf_c62x_hw_data.h" +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + static const struct pci_device_id adf_pci_tbl[] = { { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C62X), }, { } @@ -33,6 +40,7 @@ static struct pci_driver adf_driver = { .name = ADF_C62X_DEVICE_NAME, .probe = adf_probe, .remove = adf_remove, + .shutdown = adf_shutdown, .sriov_configure = adf_sriov_configure, .err_handler = &adf_err_handler, }; -- cgit v1.2.3 From e7ae9012bee8377107aa27be9d0223b960c50e2a Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 26 Mar 2025 15:59:49 +0000 Subject: crypto: qat - add shutdown handler to qat_dh895xcc commit 2c4e8b228733bfbcaf49408fdf94d220f6eb78fc upstream. During a warm reset via kexec, the system bypasses the driver removal sequence, meaning that the remove() callback is not invoked. If a QAT device is not shutdown properly, the device driver will fail to load in a newly rebooted kernel. This might result in output like the following after the kexec reboot: QAT: AE0 is inactive!! QAT: failed to get device out of reset dh895xcc 0000:3f:00.0: qat_hal_clr_reset error dh895xcc 0000:3f:00.0: Failed to init the AEs dh895xcc 0000:3f:00.0: Failed to initialise Acceleration Engine dh895xcc 0000:3f:00.0: Resetting device qat_dev0 dh895xcc 0000:3f:00.0: probe with driver dh895xcc failed with error -14 Implement the shutdown() handler that hooks into the reboot notifier list. This brings down the QAT device and ensures it is shut down properly. Cc: Fixes: 7afa232e76ce ("crypto: qat - Intel(R) QAT DH895xcc accelerator") Reviewed-by: Ahsan Atta Reviewed-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu Signed-off-by: Greg Kroah-Hartman --- drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c index 07e9d7e52861..f36e625266ed 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c @@ -19,6 +19,13 @@ #include #include "adf_dh895xcc_hw_data.h" +static void adf_shutdown(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + adf_dev_down(accel_dev); +} + static const struct pci_device_id adf_pci_tbl[] = { { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_DH895XCC), }, { } @@ -33,6 +40,7 @@ static struct pci_driver adf_driver = { .name = ADF_DH895XCC_DEVICE_NAME, .probe = adf_probe, .remove = adf_remove, + .shutdown = adf_shutdown, .sriov_configure = adf_sriov_configure, .err_handler = &adf_err_handler, }; -- cgit v1.2.3