summaryrefslogtreecommitdiff
path: root/drivers/ufs
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2023-06-30 21:57:07 +0300
committerLinus Torvalds <torvalds@linux-foundation.org>2023-06-30 21:57:07 +0300
commitca7ce08d6a063e0ccb91dc57f9bc213120d0d1a7 (patch)
tree99ce2811fdf52befc76d2cad95b16040423ecd10 /drivers/ufs
parent1546cd4bfda49fd6faad47eb30f4e744e2d79a8f (diff)
parentaf92c02fb2090692f4920ea4b74870940260cf49 (diff)
downloadlinux-ca7ce08d6a063e0ccb91dc57f9bc213120d0d1a7.tar.xz
Merge tag 'scsi-misc' of git://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi
Pull SCSI updates from James Bottomley: "Updates to the usual drivers (ufs, pm80xx, libata-scsi, smartpqi, lpfc, qla2xxx). We have a couple of major core changes impacting other systems: - Command Duration Limits, which spills into block and ATA - block level Persistent Reservation Operations, which touches block, nvme, target and dm Both of these are added with merge commits containing a cover letter explaining what's going on" * tag 'scsi-misc' of git://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi: (187 commits) scsi: core: Improve warning message in scsi_device_block() scsi: core: Replace scsi_target_block() with scsi_block_targets() scsi: core: Don't wait for quiesce in scsi_device_block() scsi: core: Don't wait for quiesce in scsi_stop_queue() scsi: core: Merge scsi_internal_device_block() and device_block() scsi: sg: Increase number of devices scsi: bsg: Increase number of devices scsi: qla2xxx: Remove unused nvme_ls_waitq wait queue scsi: ufs: ufs-pci: Add support for Intel Arrow Lake scsi: sd: sd_zbc: Use PAGE_SECTORS_SHIFT scsi: ufs: wb: Add explicit flush_threshold sysfs attribute scsi: ufs: ufs-qcom: Switch to the new ICE API scsi: ufs: dt-bindings: qcom: Add ICE phandle scsi: ufs: ufs-mediatek: Set UFSHCD_QUIRK_MCQ_BROKEN_RTC quirk scsi: ufs: ufs-mediatek: Set UFSHCD_QUIRK_MCQ_BROKEN_INTR quirk scsi: ufs: core: Add host quirk UFSHCD_QUIRK_MCQ_BROKEN_RTC scsi: ufs: core: Add host quirk UFSHCD_QUIRK_MCQ_BROKEN_INTR scsi: ufs: core: Remove dedicated hwq for dev command scsi: ufs: core: mcq: Fix the incorrect OCS value for the device command scsi: ufs: dt-bindings: samsung,exynos: Drop unneeded quotes ...
Diffstat (limited to 'drivers/ufs')
-rw-r--r--drivers/ufs/core/ufs-fault-injection.c2
-rw-r--r--drivers/ufs/core/ufs-hwmon.c2
-rw-r--r--drivers/ufs/core/ufs-mcq.c296
-rw-r--r--drivers/ufs/core/ufs-sysfs.c35
-rw-r--r--drivers/ufs/core/ufshcd-crypto.c2
-rw-r--r--drivers/ufs/core/ufshcd-priv.h27
-rw-r--r--drivers/ufs/core/ufshcd.c471
-rw-r--r--drivers/ufs/core/ufshpb.c6
-rw-r--r--drivers/ufs/core/ufshpb.h2
-rw-r--r--drivers/ufs/host/Kconfig2
-rw-r--r--drivers/ufs/host/Makefile4
-rw-r--r--drivers/ufs/host/cdns-pltfrm.c1
-rw-r--r--drivers/ufs/host/tc-dwc-g210-pci.c10
-rw-r--r--drivers/ufs/host/tc-dwc-g210-pltfrm.c1
-rw-r--r--drivers/ufs/host/ufs-exynos.c3
-rw-r--r--drivers/ufs/host/ufs-hisi.c25
-rw-r--r--drivers/ufs/host/ufs-mediatek.c6
-rw-r--r--drivers/ufs/host/ufs-qcom-ice.c244
-rw-r--r--drivers/ufs/host/ufs-qcom.c102
-rw-r--r--drivers/ufs/host/ufs-qcom.h32
-rw-r--r--drivers/ufs/host/ufs-sprd.c1
-rw-r--r--drivers/ufs/host/ufshcd-pci.c11
-rw-r--r--drivers/ufs/host/ufshcd-pltfrm.c6
-rw-r--r--drivers/ufs/host/ufshcd-pltfrm.h1
24 files changed, 739 insertions, 553 deletions
diff --git a/drivers/ufs/core/ufs-fault-injection.c b/drivers/ufs/core/ufs-fault-injection.c
index 7ac7c4e7ff83..5b1184aac585 100644
--- a/drivers/ufs/core/ufs-fault-injection.c
+++ b/drivers/ufs/core/ufs-fault-injection.c
@@ -54,7 +54,7 @@ static int ufs_fault_set(const char *val, const struct kernel_param *kp)
if (!setup_fault_attr(attr, (char *)val))
return -EINVAL;
- strlcpy(kp->arg, val, FAULT_INJ_STR_SIZE);
+ strscpy(kp->arg, val, FAULT_INJ_STR_SIZE);
return 0;
}
diff --git a/drivers/ufs/core/ufs-hwmon.c b/drivers/ufs/core/ufs-hwmon.c
index 4c6a872b7a7c..101d7082446f 100644
--- a/drivers/ufs/core/ufs-hwmon.c
+++ b/drivers/ufs/core/ufs-hwmon.c
@@ -146,7 +146,7 @@ static umode_t ufs_hwmon_is_visible(const void *_data, enum hwmon_sensor_types t
return 0;
}
-static const struct hwmon_channel_info *ufs_hwmon_info[] = {
+static const struct hwmon_channel_info *const ufs_hwmon_info[] = {
HWMON_CHANNEL_INFO(temp, HWMON_T_ENABLE | HWMON_T_INPUT | HWMON_T_CRIT | HWMON_T_LCRIT),
NULL
};
diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c
index 51b3c6ae781d..6fb0e007af63 100644
--- a/drivers/ufs/core/ufs-mcq.c
+++ b/drivers/ufs/core/ufs-mcq.c
@@ -12,21 +12,26 @@
#include <linux/module.h>
#include <linux/platform_device.h>
#include "ufshcd-priv.h"
+#include <linux/delay.h>
+#include <scsi/scsi_cmnd.h>
+#include <linux/bitfield.h>
+#include <linux/iopoll.h>
#define MAX_QUEUE_SUP GENMASK(7, 0)
#define UFS_MCQ_MIN_RW_QUEUES 2
#define UFS_MCQ_MIN_READ_QUEUES 0
-#define UFS_MCQ_NUM_DEV_CMD_QUEUES 1
#define UFS_MCQ_MIN_POLL_QUEUES 0
#define QUEUE_EN_OFFSET 31
#define QUEUE_ID_OFFSET 16
-#define MAX_DEV_CMD_ENTRIES 2
#define MCQ_CFG_MAC_MASK GENMASK(16, 8)
#define MCQ_QCFG_SIZE 0x40
#define MCQ_ENTRY_SIZE_IN_DWORD 8
#define CQE_UCD_BA GENMASK_ULL(63, 7)
+/* Max mcq register polling time in microseconds */
+#define MCQ_POLL_US 500000
+
static int rw_queue_count_set(const char *val, const struct kernel_param *kp)
{
return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_RW_QUEUES,
@@ -108,8 +113,7 @@ struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba,
u32 utag = blk_mq_unique_tag(req);
u32 hwq = blk_mq_unique_tag_to_hwq(utag);
- /* uhq[0] is used to serve device commands */
- return &hba->uhq[hwq + UFSHCD_MCQ_IO_QUEUE_OFFSET];
+ return &hba->uhq[hwq];
}
/**
@@ -153,8 +157,7 @@ static int ufshcd_mcq_config_nr_queues(struct ufs_hba *hba)
/* maxq is 0 based value */
hba_maxq = FIELD_GET(MAX_QUEUE_SUP, hba->mcq_capabilities) + 1;
- tot_queues = UFS_MCQ_NUM_DEV_CMD_QUEUES + read_queues + poll_queues +
- rw_queues;
+ tot_queues = read_queues + poll_queues + rw_queues;
if (hba_maxq < tot_queues) {
dev_err(hba->dev, "Total queues (%d) exceeds HC capacity (%d)\n",
@@ -162,7 +165,7 @@ static int ufshcd_mcq_config_nr_queues(struct ufs_hba *hba)
return -EOPNOTSUPP;
}
- rem = hba_maxq - UFS_MCQ_NUM_DEV_CMD_QUEUES;
+ rem = hba_maxq;
if (rw_queues) {
hba->nr_queues[HCTX_TYPE_DEFAULT] = rw_queues;
@@ -188,7 +191,7 @@ static int ufshcd_mcq_config_nr_queues(struct ufs_hba *hba)
for (i = 0; i < HCTX_MAX_TYPES; i++)
host->nr_hw_queues += hba->nr_queues[i];
- hba->nr_hw_queues = host->nr_hw_queues + UFS_MCQ_NUM_DEV_CMD_QUEUES;
+ hba->nr_hw_queues = host->nr_hw_queues;
return 0;
}
@@ -270,44 +273,57 @@ static int ufshcd_mcq_get_tag(struct ufs_hba *hba,
}
static void ufshcd_mcq_process_cqe(struct ufs_hba *hba,
- struct ufs_hw_queue *hwq)
+ struct ufs_hw_queue *hwq)
{
struct cq_entry *cqe = ufshcd_mcq_cur_cqe(hwq);
int tag = ufshcd_mcq_get_tag(hba, hwq, cqe);
- ufshcd_compl_one_cqe(hba, tag, cqe);
+ if (cqe->command_desc_base_addr) {
+ ufshcd_compl_one_cqe(hba, tag, cqe);
+ /* After processed the cqe, mark it empty (invalid) entry */
+ cqe->command_desc_base_addr = 0;
+ }
}
-unsigned long ufshcd_mcq_poll_cqe_nolock(struct ufs_hba *hba,
- struct ufs_hw_queue *hwq)
+void ufshcd_mcq_compl_all_cqes_lock(struct ufs_hba *hba,
+ struct ufs_hw_queue *hwq)
{
- unsigned long completed_reqs = 0;
+ unsigned long flags;
+ u32 entries = hwq->max_entries;
- ufshcd_mcq_update_cq_tail_slot(hwq);
- while (!ufshcd_mcq_is_cq_empty(hwq)) {
+ spin_lock_irqsave(&hwq->cq_lock, flags);
+ while (entries > 0) {
ufshcd_mcq_process_cqe(hba, hwq);
ufshcd_mcq_inc_cq_head_slot(hwq);
- completed_reqs++;
+ entries--;
}
- if (completed_reqs)
- ufshcd_mcq_update_cq_head(hwq);
-
- return completed_reqs;
+ ufshcd_mcq_update_cq_tail_slot(hwq);
+ hwq->cq_head_slot = hwq->cq_tail_slot;
+ spin_unlock_irqrestore(&hwq->cq_lock, flags);
}
-EXPORT_SYMBOL_GPL(ufshcd_mcq_poll_cqe_nolock);
unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba,
struct ufs_hw_queue *hwq)
{
- unsigned long completed_reqs, flags;
+ unsigned long completed_reqs = 0;
+ unsigned long flags;
spin_lock_irqsave(&hwq->cq_lock, flags);
- completed_reqs = ufshcd_mcq_poll_cqe_nolock(hba, hwq);
+ ufshcd_mcq_update_cq_tail_slot(hwq);
+ while (!ufshcd_mcq_is_cq_empty(hwq)) {
+ ufshcd_mcq_process_cqe(hba, hwq);
+ ufshcd_mcq_inc_cq_head_slot(hwq);
+ completed_reqs++;
+ }
+
+ if (completed_reqs)
+ ufshcd_mcq_update_cq_head(hwq);
spin_unlock_irqrestore(&hwq->cq_lock, flags);
return completed_reqs;
}
+EXPORT_SYMBOL_GPL(ufshcd_mcq_poll_cqe_lock);
void ufshcd_mcq_make_queues_operational(struct ufs_hba *hba)
{
@@ -420,13 +436,243 @@ int ufshcd_mcq_init(struct ufs_hba *hba)
hwq->max_entries = hba->nutrs;
spin_lock_init(&hwq->sq_lock);
spin_lock_init(&hwq->cq_lock);
+ mutex_init(&hwq->sq_mutex);
}
/* The very first HW queue serves device commands */
hba->dev_cmd_queue = &hba->uhq[0];
- /* Give dev_cmd_queue the minimal number of entries */
- hba->dev_cmd_queue->max_entries = MAX_DEV_CMD_ENTRIES;
host->host_tagset = 1;
return 0;
}
+
+static int ufshcd_mcq_sq_stop(struct ufs_hba *hba, struct ufs_hw_queue *hwq)
+{
+ void __iomem *reg;
+ u32 id = hwq->id, val;
+ int err;
+
+ if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)
+ return -ETIMEDOUT;
+
+ writel(SQ_STOP, mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTC);
+ reg = mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTS;
+ err = read_poll_timeout(readl, val, val & SQ_STS, 20,
+ MCQ_POLL_US, false, reg);
+ if (err)
+ dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n",
+ __func__, id, err);
+ return err;
+}
+
+static int ufshcd_mcq_sq_start(struct ufs_hba *hba, struct ufs_hw_queue *hwq)
+{
+ void __iomem *reg;
+ u32 id = hwq->id, val;
+ int err;
+
+ if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)
+ return -ETIMEDOUT;
+
+ writel(SQ_START, mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTC);
+ reg = mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTS;
+ err = read_poll_timeout(readl, val, !(val & SQ_STS), 20,
+ MCQ_POLL_US, false, reg);
+ if (err)
+ dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n",
+ __func__, id, err);
+ return err;
+}
+
+/**
+ * ufshcd_mcq_sq_cleanup - Clean up submission queue resources
+ * associated with the pending command.
+ * @hba - per adapter instance.
+ * @task_tag - The command's task tag.
+ *
+ * Returns 0 for success; error code otherwise.
+ */
+int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag)
+{
+ struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
+ struct scsi_cmnd *cmd = lrbp->cmd;
+ struct ufs_hw_queue *hwq;
+ void __iomem *reg, *opr_sqd_base;
+ u32 nexus, id, val;
+ int err;
+
+ if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)
+ return -ETIMEDOUT;
+
+ if (task_tag != hba->nutrs - UFSHCD_NUM_RESERVED) {
+ if (!cmd)
+ return -EINVAL;
+ hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
+ } else {
+ hwq = hba->dev_cmd_queue;
+ }
+
+ id = hwq->id;
+
+ mutex_lock(&hwq->sq_mutex);
+
+ /* stop the SQ fetching before working on it */
+ err = ufshcd_mcq_sq_stop(hba, hwq);
+ if (err)
+ goto unlock;
+
+ /* SQCTI = EXT_IID, IID, LUN, Task Tag */
+ nexus = lrbp->lun << 8 | task_tag;
+ opr_sqd_base = mcq_opr_base(hba, OPR_SQD, id);
+ writel(nexus, opr_sqd_base + REG_SQCTI);
+
+ /* SQRTCy.ICU = 1 */
+ writel(SQ_ICU, opr_sqd_base + REG_SQRTC);
+
+ /* Poll SQRTSy.CUS = 1. Return result from SQRTSy.RTC */
+ reg = opr_sqd_base + REG_SQRTS;
+ err = read_poll_timeout(readl, val, val & SQ_CUS, 20,
+ MCQ_POLL_US, false, reg);
+ if (err)
+ dev_err(hba->dev, "%s: failed. hwq=%d, tag=%d err=%ld\n",
+ __func__, id, task_tag,
+ FIELD_GET(SQ_ICU_ERR_CODE_MASK, readl(reg)));
+
+ if (ufshcd_mcq_sq_start(hba, hwq))
+ err = -ETIMEDOUT;
+
+unlock:
+ mutex_unlock(&hwq->sq_mutex);
+ return err;
+}
+
+/**
+ * ufshcd_mcq_nullify_sqe - Nullify the submission queue entry.
+ * Write the sqe's Command Type to 0xF. The host controller will not
+ * fetch any sqe with Command Type = 0xF.
+ *
+ * @utrd - UTP Transfer Request Descriptor to be nullified.
+ */
+static void ufshcd_mcq_nullify_sqe(struct utp_transfer_req_desc *utrd)
+{
+ u32 dword_0;
+
+ dword_0 = le32_to_cpu(utrd->header.dword_0);
+ dword_0 &= ~UPIU_COMMAND_TYPE_MASK;
+ dword_0 |= FIELD_PREP(UPIU_COMMAND_TYPE_MASK, 0xF);
+ utrd->header.dword_0 = cpu_to_le32(dword_0);
+}
+
+/**
+ * ufshcd_mcq_sqe_search - Search for the command in the submission queue
+ * If the command is in the submission queue and not issued to the device yet,
+ * nullify the sqe so the host controller will skip fetching the sqe.
+ *
+ * @hba - per adapter instance.
+ * @hwq - Hardware Queue to be searched.
+ * @task_tag - The command's task tag.
+ *
+ * Returns true if the SQE containing the command is present in the SQ
+ * (not fetched by the controller); returns false if the SQE is not in the SQ.
+ */
+static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba,
+ struct ufs_hw_queue *hwq, int task_tag)
+{
+ struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
+ struct utp_transfer_req_desc *utrd;
+ u32 mask = hwq->max_entries - 1;
+ __le64 cmd_desc_base_addr;
+ bool ret = false;
+ u64 addr, match;
+ u32 sq_head_slot;
+
+ if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)
+ return true;
+
+ mutex_lock(&hwq->sq_mutex);
+
+ ufshcd_mcq_sq_stop(hba, hwq);
+ sq_head_slot = ufshcd_mcq_get_sq_head_slot(hwq);
+ if (sq_head_slot == hwq->sq_tail_slot)
+ goto out;
+
+ cmd_desc_base_addr = lrbp->utr_descriptor_ptr->command_desc_base_addr;
+ addr = le64_to_cpu(cmd_desc_base_addr) & CQE_UCD_BA;
+
+ while (sq_head_slot != hwq->sq_tail_slot) {
+ utrd = hwq->sqe_base_addr +
+ sq_head_slot * sizeof(struct utp_transfer_req_desc);
+ match = le64_to_cpu(utrd->command_desc_base_addr) & CQE_UCD_BA;
+ if (addr == match) {
+ ufshcd_mcq_nullify_sqe(utrd);
+ ret = true;
+ goto out;
+ }
+ sq_head_slot = (sq_head_slot + 1) & mask;
+ }
+
+out:
+ ufshcd_mcq_sq_start(hba, hwq);
+ mutex_unlock(&hwq->sq_mutex);
+ return ret;
+}
+
+/**
+ * ufshcd_mcq_abort - Abort the command in MCQ.
+ * @cmd - The command to be aborted.
+ *
+ * Returns SUCCESS or FAILED error codes
+ */
+int ufshcd_mcq_abort(struct scsi_cmnd *cmd)
+{
+ struct Scsi_Host *host = cmd->device->host;
+ struct ufs_hba *hba = shost_priv(host);
+ int tag = scsi_cmd_to_rq(cmd)->tag;
+ struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+ struct ufs_hw_queue *hwq;
+ int err = FAILED;
+
+ if (!ufshcd_cmd_inflight(lrbp->cmd)) {
+ dev_err(hba->dev,
+ "%s: skip abort. cmd at tag %d already completed.\n",
+ __func__, tag);
+ goto out;
+ }
+
+ /* Skip task abort in case previous aborts failed and report failure */
+ if (lrbp->req_abort_skip) {
+ dev_err(hba->dev, "%s: skip abort. tag %d failed earlier\n",
+ __func__, tag);
+ goto out;
+ }
+
+ hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
+
+ if (ufshcd_mcq_sqe_search(hba, hwq, tag)) {
+ /*
+ * Failure. The command should not be "stuck" in SQ for
+ * a long time which resulted in command being aborted.
+ */
+ dev_err(hba->dev, "%s: cmd found in sq. hwq=%d, tag=%d\n",
+ __func__, hwq->id, tag);
+ goto out;
+ }
+
+ /*
+ * The command is not in the submission queue, and it is not
+ * in the completion queue either. Query the device to see if
+ * the command is being processed in the device.
+ */
+ if (ufshcd_try_to_abort_task(hba, tag)) {
+ dev_err(hba->dev, "%s: device abort failed %d\n", __func__, err);
+ lrbp->req_abort_skip = true;
+ goto out;
+ }
+
+ err = SUCCESS;
+ if (ufshcd_cmd_inflight(lrbp->cmd))
+ ufshcd_release_scsi_cmd(hba, lrbp);
+
+out:
+ return err;
+}
diff --git a/drivers/ufs/core/ufs-sysfs.c b/drivers/ufs/core/ufs-sysfs.c
index 883f0e44b54e..6c72075750dd 100644
--- a/drivers/ufs/core/ufs-sysfs.c
+++ b/drivers/ufs/core/ufs-sysfs.c
@@ -168,7 +168,7 @@ static ssize_t auto_hibern8_show(struct device *dev,
}
pm_runtime_get_sync(hba->dev);
- ufshcd_hold(hba, false);
+ ufshcd_hold(hba);
ahit = ufshcd_readl(hba, REG_AUTO_HIBERNATE_IDLE_TIMER);
ufshcd_release(hba);
pm_runtime_put_sync(hba->dev);
@@ -298,6 +298,37 @@ out:
return res < 0 ? res : count;
}
+static ssize_t wb_flush_threshold_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct ufs_hba *hba = dev_get_drvdata(dev);
+
+ return sysfs_emit(buf, "%u\n", hba->vps->wb_flush_threshold);
+}
+
+static ssize_t wb_flush_threshold_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct ufs_hba *hba = dev_get_drvdata(dev);
+ unsigned int wb_flush_threshold;
+
+ if (kstrtouint(buf, 0, &wb_flush_threshold))
+ return -EINVAL;
+
+ /* The range of values for wb_flush_threshold is (0,10] */
+ if (wb_flush_threshold > UFS_WB_BUF_REMAIN_PERCENT(100) ||
+ wb_flush_threshold == 0) {
+ dev_err(dev, "The value of wb_flush_threshold is invalid!\n");
+ return -EINVAL;
+ }
+
+ hba->vps->wb_flush_threshold = wb_flush_threshold;
+
+ return count;
+}
+
static DEVICE_ATTR_RW(rpm_lvl);
static DEVICE_ATTR_RO(rpm_target_dev_state);
static DEVICE_ATTR_RO(rpm_target_link_state);
@@ -307,6 +338,7 @@ static DEVICE_ATTR_RO(spm_target_link_state);
static DEVICE_ATTR_RW(auto_hibern8);
static DEVICE_ATTR_RW(wb_on);
static DEVICE_ATTR_RW(enable_wb_buf_flush);
+static DEVICE_ATTR_RW(wb_flush_threshold);
static struct attribute *ufs_sysfs_ufshcd_attrs[] = {
&dev_attr_rpm_lvl.attr,
@@ -318,6 +350,7 @@ static struct attribute *ufs_sysfs_ufshcd_attrs[] = {
&dev_attr_auto_hibern8.attr,
&dev_attr_wb_on.attr,
&dev_attr_enable_wb_buf_flush.attr,
+ &dev_attr_wb_flush_threshold.attr,
NULL
};
diff --git a/drivers/ufs/core/ufshcd-crypto.c b/drivers/ufs/core/ufshcd-crypto.c
index 198360fe5e8e..f2c4422cab86 100644
--- a/drivers/ufs/core/ufshcd-crypto.c
+++ b/drivers/ufs/core/ufshcd-crypto.c
@@ -24,7 +24,7 @@ static int ufshcd_program_key(struct ufs_hba *hba,
u32 slot_offset = hba->crypto_cfg_register + slot * sizeof(*cfg);
int err = 0;
- ufshcd_hold(hba, false);
+ ufshcd_hold(hba);
if (hba->vops && hba->vops->program_key) {
err = hba->vops->program_key(hba, cfg, slot);
diff --git a/drivers/ufs/core/ufshcd-priv.h b/drivers/ufs/core/ufshcd-priv.h
index d53b93c21a0c..9566a95aeed9 100644
--- a/drivers/ufs/core/ufshcd-priv.h
+++ b/drivers/ufs/core/ufshcd-priv.h
@@ -71,22 +71,24 @@ void ufshcd_mcq_config_mac(struct ufs_hba *hba, u32 max_active_cmds);
void ufshcd_mcq_select_mcq_mode(struct ufs_hba *hba);
u32 ufshcd_mcq_read_cqis(struct ufs_hba *hba, int i);
void ufshcd_mcq_write_cqis(struct ufs_hba *hba, u32 val, int i);
-unsigned long ufshcd_mcq_poll_cqe_nolock(struct ufs_hba *hba,
- struct ufs_hw_queue *hwq);
struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba,
struct request *req);
unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba,
struct ufs_hw_queue *hwq);
+void ufshcd_mcq_compl_all_cqes_lock(struct ufs_hba *hba,
+ struct ufs_hw_queue *hwq);
+bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd);
+int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag);
+int ufshcd_mcq_abort(struct scsi_cmnd *cmd);
+int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
+void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
+ struct ufshcd_lrb *lrbp);
-#define UFSHCD_MCQ_IO_QUEUE_OFFSET 1
#define SD_ASCII_STD true
#define SD_RAW false
int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
u8 **buf, bool ascii);
-int ufshcd_hold(struct ufs_hba *hba, bool async);
-void ufshcd_release(struct ufs_hba *hba);
-
int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd);
int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
@@ -366,10 +368,11 @@ static inline bool ufs_is_valid_unit_desc_lun(struct ufs_dev_info *dev_info, u8
static inline void ufshcd_inc_sq_tail(struct ufs_hw_queue *q)
__must_hold(&q->sq_lock)
{
- u32 mask = q->max_entries - 1;
u32 val;
- q->sq_tail_slot = (q->sq_tail_slot + 1) & mask;
+ q->sq_tail_slot++;
+ if (q->sq_tail_slot == q->max_entries)
+ q->sq_tail_slot = 0;
val = q->sq_tail_slot * sizeof(struct utp_transfer_req_desc);
writel(val, q->mcq_sq_tail);
}
@@ -404,4 +407,12 @@ static inline struct cq_entry *ufshcd_mcq_cur_cqe(struct ufs_hw_queue *q)
return cqe + q->cq_head_slot;
}
+
+static inline u32 ufshcd_mcq_get_sq_head_slot(struct ufs_hw_queue *q)
+{
+ u32 val = readl(q->mcq_sq_head);
+
+ return val / sizeof(struct utp_transfer_req_desc);
+}
+
#endif /* _UFSHCD_PRIV_H_ */
diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c
index e7e79f515e14..983fae84d9e8 100644
--- a/drivers/ufs/core/ufshcd.c
+++ b/drivers/ufs/core/ufshcd.c
@@ -98,7 +98,7 @@
/* Polling time to wait for fDeviceInit */
#define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */
-/* UFSHC 4.0 compliant HC support this mode, refer param_set_mcq_mode() */
+/* UFSHC 4.0 compliant HC support this mode. */
static bool use_mcq_mode = true;
static bool is_mcq_supported(struct ufs_hba *hba)
@@ -106,23 +106,7 @@ static bool is_mcq_supported(struct ufs_hba *hba)
return hba->mcq_sup && use_mcq_mode;
}
-static int param_set_mcq_mode(const char *val, const struct kernel_param *kp)
-{
- int ret;
-
- ret = param_set_bool(val, kp);
- if (ret)
- return ret;
-
- return 0;
-}
-
-static const struct kernel_param_ops mcq_mode_ops = {
- .set = param_set_mcq_mode,
- .get = param_get_bool,
-};
-
-module_param_cb(use_mcq_mode, &mcq_mode_ops, &use_mcq_mode, 0644);
+module_param(use_mcq_mode, bool, 0644);
MODULE_PARM_DESC(use_mcq_mode, "Control MCQ mode for controllers starting from UFSHCI 4.0. 1 - enable MCQ, 0 - disable MCQ. MCQ is enabled by default");
#define ufshcd_toggle_vreg(_dev, _vreg, _on) \
@@ -173,7 +157,6 @@ EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
enum {
UFSHCD_MAX_CHANNEL = 0,
UFSHCD_MAX_ID = 1,
- UFSHCD_NUM_RESERVED = 1,
UFSHCD_CMD_PER_LUN = 32 - UFSHCD_NUM_RESERVED,
UFSHCD_CAN_QUEUE = 32 - UFSHCD_NUM_RESERVED,
};
@@ -301,7 +284,6 @@ static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on);
static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on);
static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
struct ufs_vreg *vreg);
-static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba,
bool enable);
static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba);
@@ -1205,7 +1187,7 @@ static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
bool timeout = false, do_last_check = false;
ktime_t start;
- ufshcd_hold(hba, false);
+ ufshcd_hold(hba);
spin_lock_irqsave(hba->host->host_lock, flags);
/*
* Wait for all the outstanding tasks/transfer requests.
@@ -1326,7 +1308,7 @@ static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba, u64 timeout_us)
}
/* let's not get into low power until clock scaling is completed */
- ufshcd_hold(hba, false);
+ ufshcd_hold(hba);
out:
return ret;
@@ -1656,7 +1638,7 @@ static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
goto out;
ufshcd_rpm_get_sync(hba);
- ufshcd_hold(hba, false);
+ ufshcd_hold(hba);
hba->clk_scaling.is_enabled = value;
@@ -1739,7 +1721,7 @@ static void ufshcd_ungate_work(struct work_struct *work)
spin_lock_irqsave(hba->host->host_lock, flags);
if (hba->clk_gating.state == CLKS_ON) {
spin_unlock_irqrestore(hba->host->host_lock, flags);
- goto unblock_reqs;
+ return;
}
spin_unlock_irqrestore(hba->host->host_lock, flags);
@@ -1762,25 +1744,21 @@ static void ufshcd_ungate_work(struct work_struct *work)
}
hba->clk_gating.is_suspended = false;
}
-unblock_reqs:
- ufshcd_scsi_unblock_requests(hba);
}
/**
* ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
* Also, exit from hibern8 mode and set the link as active.
* @hba: per adapter instance
- * @async: This indicates whether caller should ungate clocks asynchronously.
*/
-int ufshcd_hold(struct ufs_hba *hba, bool async)
+void ufshcd_hold(struct ufs_hba *hba)
{
- int rc = 0;
bool flush_result;
unsigned long flags;
if (!ufshcd_is_clkgating_allowed(hba) ||
!hba->clk_gating.is_initialized)
- goto out;
+ return;
spin_lock_irqsave(hba->host->host_lock, flags);
hba->clk_gating.active_reqs++;
@@ -1797,15 +1775,10 @@ start:
*/
if (ufshcd_can_hibern8_during_gating(hba) &&
ufshcd_is_link_hibern8(hba)) {
- if (async) {
- rc = -EAGAIN;
- hba->clk_gating.active_reqs--;
- break;
- }
spin_unlock_irqrestore(hba->host->host_lock, flags);
flush_result = flush_work(&hba->clk_gating.ungate_work);
if (hba->clk_gating.is_suspended && !flush_result)
- goto out;
+ return;
spin_lock_irqsave(hba->host->host_lock, flags);
goto start;
}
@@ -1827,21 +1800,14 @@ start:
hba->clk_gating.state = REQ_CLKS_ON;
trace_ufshcd_clk_gating(dev_name(hba->dev),
hba->clk_gating.state);
- if (queue_work(hba->clk_gating.clk_gating_workq,
- &hba->clk_gating.ungate_work))
- ufshcd_scsi_block_requests(hba);
+ queue_work(hba->clk_gating.clk_gating_workq,
+ &hba->clk_gating.ungate_work);
/*
* fall through to check if we should wait for this
* work to be done or not.
*/
fallthrough;
case REQ_CLKS_ON:
- if (async) {
- rc = -EAGAIN;
- hba->clk_gating.active_reqs--;
- break;
- }
-
spin_unlock_irqrestore(hba->host->host_lock, flags);
flush_work(&hba->clk_gating.ungate_work);
/* Make sure state is CLKS_ON before returning */
@@ -1853,8 +1819,6 @@ start:
break;
}
spin_unlock_irqrestore(hba->host->host_lock, flags);
-out:
- return rc;
}
EXPORT_SYMBOL_GPL(ufshcd_hold);
@@ -2086,7 +2050,7 @@ static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
ufshcd_remove_clk_gating_sysfs(hba);
/* Ungate the clock if necessary. */
- ufshcd_hold(hba, false);
+ ufshcd_hold(hba);
hba->clk_gating.is_initialized = false;
ufshcd_release(hba);
@@ -2336,18 +2300,20 @@ static inline int ufshcd_hba_capabilities(struct ufs_hba *hba)
/* Read crypto capabilities */
err = ufshcd_hba_init_crypto_capabilities(hba);
- if (err)
+ if (err) {
dev_err(hba->dev, "crypto setup failed\n");
+ return err;
+ }
hba->mcq_sup = FIELD_GET(MASK_MCQ_SUPPORT, hba->capabilities);
if (!hba->mcq_sup)
- return err;
+ return 0;
hba->mcq_capabilities = ufshcd_readl(hba, REG_MCQCAP);
hba->ext_iid_sup = FIELD_GET(MASK_EXT_IID_SUPPORT,
hba->mcq_capabilities);
- return err;
+ return 0;
}
/**
@@ -2482,7 +2448,7 @@ int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD)
return 0;
- ufshcd_hold(hba, false);
+ ufshcd_hold(hba);
mutex_lock(&hba->uic_cmd_mutex);
ufshcd_add_delay_before_dme_cmd(hba);
@@ -2533,7 +2499,7 @@ static void ufshcd_sgl_to_prdt(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int
* 11b to indicate Dword granularity. A value of '3'
* indicates 4 bytes, '7' indicates 8 bytes, etc."
*/
- WARN_ONCE(len > 256 * 1024, "len = %#x\n", len);
+ WARN_ONCE(len > SZ_256K, "len = %#x\n", len);
prd->size = cpu_to_le32(len - 1);
prd->addr = cpu_to_le64(sg->dma_address);
prd->reserved = 0;
@@ -2885,12 +2851,6 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
WARN_ONCE(tag < 0 || tag >= hba->nutrs, "Invalid tag %d\n", tag);
- /*
- * Allows the UFS error handler to wait for prior ufshcd_queuecommand()
- * calls.
- */
- rcu_read_lock();
-
switch (hba->ufshcd_state) {
case UFSHCD_STATE_OPERATIONAL:
break;
@@ -2936,16 +2896,9 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
hba->req_abort_count = 0;
- err = ufshcd_hold(hba, true);
- if (err) {
- err = SCSI_MLQUEUE_HOST_BUSY;
- goto out;
- }
- WARN_ON(ufshcd_is_clkgating_allowed(hba) &&
- (hba->clk_gating.state != CLKS_ON));
+ ufshcd_hold(hba);
lrbp = &hba->lrb[tag];
- WARN_ON(lrbp->cmd);
lrbp->cmd = cmd;
lrbp->task_tag = tag;
lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
@@ -2961,7 +2914,6 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
err = ufshcd_map_sg(hba, lrbp);
if (err) {
- lrbp->cmd = NULL;
ufshcd_release(hba);
goto out;
}
@@ -2972,8 +2924,6 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
ufshcd_send_command(hba, tag, hwq);
out:
- rcu_read_unlock();
-
if (ufs_trigger_eh()) {
unsigned long flags;
@@ -2999,13 +2949,50 @@ static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
}
/*
- * Clear all the requests from the controller for which a bit has been set in
- * @mask and wait until the controller confirms that these requests have been
- * cleared.
+ * Check with the block layer if the command is inflight
+ * @cmd: command to check.
+ *
+ * Returns true if command is inflight; false if not.
+ */
+bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd)
+{
+ struct request *rq;
+
+ if (!cmd)
+ return false;
+
+ rq = scsi_cmd_to_rq(cmd);
+ if (!blk_mq_request_started(rq))
+ return false;
+
+ return true;
+}
+
+/*
+ * Clear the pending command in the controller and wait until
+ * the controller confirms that the command has been cleared.
+ * @hba: per adapter instance
+ * @task_tag: The tag number of the command to be cleared.
*/
-static int ufshcd_clear_cmds(struct ufs_hba *hba, u32 mask)
+static int ufshcd_clear_cmd(struct ufs_hba *hba, u32 task_tag)
{
+ u32 mask = 1U << task_tag;
unsigned long flags;
+ int err;
+
+ if (is_mcq_enabled(hba)) {
+ /*
+ * MCQ mode. Clean up the MCQ resources similar to
+ * what the ufshcd_utrl_clear() does for SDB mode.
+ */
+ err = ufshcd_mcq_sq_cleanup(hba, task_tag);
+ if (err) {
+ dev_err(hba->dev, "%s: failed tag=%d. err=%d\n",
+ __func__, task_tag, err);
+ return err;
+ }
+ return 0;
+ }
/* clear outstanding transaction before retry */
spin_lock_irqsave(hba->host->host_lock, flags);
@@ -3099,14 +3086,23 @@ retry:
* not trigger any race conditions.
*/
hba->dev_cmd.complete = NULL;
- err = ufshcd_get_tr_ocs(lrbp, hba->dev_cmd.cqe);
+ err = ufshcd_get_tr_ocs(lrbp, NULL);
if (!err)
err = ufshcd_dev_cmd_completion(hba, lrbp);
} else {
err = -ETIMEDOUT;
dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
__func__, lrbp->task_tag);
- if (ufshcd_clear_cmds(hba, 1U << lrbp->task_tag) == 0) {
+
+ /* MCQ mode */
+ if (is_mcq_enabled(hba)) {
+ err = ufshcd_clear_cmd(hba, lrbp->task_tag);
+ hba->dev_cmd.complete = NULL;
+ return err;
+ }
+
+ /* SDB mode */
+ if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0) {
/* successfully cleared the command, retry if needed */
err = -EAGAIN;
/*
@@ -3180,13 +3176,12 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
down_read(&hba->clk_scaling_lock);
lrbp = &hba->lrb[tag];
- WARN_ON(lrbp->cmd);
+ lrbp->cmd = NULL;
err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
if (unlikely(err))
goto out;
hba->dev_cmd.complete = &wait;
- hba->dev_cmd.cqe = NULL;
ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
@@ -3267,7 +3262,7 @@ int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
BUG_ON(!hba);
- ufshcd_hold(hba, false);
+ ufshcd_hold(hba);
mutex_lock(&hba->dev_cmd.lock);
ufshcd_init_query(hba, &request, &response, opcode, idn, index,
selector);
@@ -3341,7 +3336,7 @@ int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
return -EINVAL;
}
- ufshcd_hold(hba, false);
+ ufshcd_hold(hba);
mutex_lock(&hba->dev_cmd.lock);
ufshcd_init_query(hba, &request, &response, opcode, idn, index,
@@ -3437,7 +3432,7 @@ static int __ufshcd_query_descriptor(struct ufs_hba *hba,
return -EINVAL;
}
- ufshcd_hold(hba, false);
+ ufshcd_hold(hba);
mutex_lock(&hba->dev_cmd.lock);
ufshcd_init_query(hba, &request, &response, opcode, idn, index,
@@ -3779,7 +3774,7 @@ static int ufshcd_memory_alloc(struct ufs_hba *hba)
/*
* Allocate memory for UTP Transfer descriptors
- * UFSHCI requires 1024 byte alignment of UTRD
+ * UFSHCI requires 1KB alignment of UTRD
*/
utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
@@ -3787,7 +3782,7 @@ static int ufshcd_memory_alloc(struct ufs_hba *hba)
&hba->utrdl_dma_addr,
GFP_KERNEL);
if (!hba->utrdl_base_addr ||
- WARN_ON(hba->utrdl_dma_addr & (1024 - 1))) {
+ WARN_ON(hba->utrdl_dma_addr & (SZ_1K - 1))) {
dev_err(hba->dev,
"Transfer Descriptor Memory allocation failed\n");
goto out;
@@ -3803,7 +3798,7 @@ static int ufshcd_memory_alloc(struct ufs_hba *hba)
goto skip_utmrdl;
/*
* Allocate memory for UTP Task Management descriptors
- * UFSHCI requires 1024 byte alignment of UTMRD
+ * UFSHCI requires 1KB alignment of UTMRD
*/
utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
@@ -3811,7 +3806,7 @@ static int ufshcd_memory_alloc(struct ufs_hba *hba)
&hba->utmrdl_dma_addr,
GFP_KERNEL);
if (!hba->utmrdl_base_addr ||
- WARN_ON(hba->utmrdl_dma_addr & (1024 - 1))) {
+ WARN_ON(hba->utmrdl_dma_addr & (SZ_1K - 1))) {
dev_err(hba->dev,
"Task Management Descriptor Memory allocation failed\n");
goto out;
@@ -3868,10 +3863,8 @@ static void ufshcd_host_memory_configure(struct ufs_hba *hba)
/* Configure UTRD with command descriptor base address */
cmd_desc_element_addr =
(cmd_desc_dma_addr + (cmd_desc_size * i));
- utrdlp[i].command_desc_base_addr_lo =
- cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
- utrdlp[i].command_desc_base_addr_hi =
- cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
+ utrdlp[i].command_desc_base_addr =
+ cpu_to_le64(cmd_desc_element_addr);
/* Response upiu and prdt offset should be in double words */
if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
@@ -4255,7 +4248,7 @@ int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
uic_cmd.command = UIC_CMD_DME_SET;
uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
uic_cmd.argument3 = mode;
- ufshcd_hold(hba, false);
+ ufshcd_hold(hba);
ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
ufshcd_release(hba);
@@ -4362,7 +4355,7 @@ void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit)
if (update &&
!pm_runtime_suspended(&hba->ufs_device_wlun->sdev_gendev)) {
ufshcd_rpm_get_sync(hba);
- ufshcd_hold(hba, false);
+ ufshcd_hold(hba);
ufshcd_auto_hibern8_enable(hba);
ufshcd_release(hba);
ufshcd_rpm_put_sync(hba);
@@ -4955,7 +4948,7 @@ static int ufshcd_verify_dev_init(struct ufs_hba *hba)
int err = 0;
int retries;
- ufshcd_hold(hba, false);
+ ufshcd_hold(hba);
mutex_lock(&hba->dev_cmd.lock);
for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
@@ -5148,7 +5141,7 @@ static int ufshcd_slave_configure(struct scsi_device *sdev)
blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
if (hba->quirks & UFSHCD_QUIRK_4KB_DMA_ALIGNMENT)
- blk_queue_update_dma_alignment(q, 4096 - 1);
+ blk_queue_update_dma_alignment(q, SZ_4K - 1);
/*
* Block runtime-pm until all consumers are added.
* Refer ufshcd_setup_links().
@@ -5416,13 +5409,12 @@ static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
}
/* Release the resources allocated for processing a SCSI command. */
-static void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp)
+void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
+ struct ufshcd_lrb *lrbp)
{
struct scsi_cmnd *cmd = lrbp->cmd;
scsi_dma_unmap(cmd);
- lrbp->cmd = NULL; /* Mark the command as completed. */
ufshcd_release(hba);
ufshcd_clk_scaling_update_busy(hba);
}
@@ -5438,6 +5430,7 @@ void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag,
{
struct ufshcd_lrb *lrbp;
struct scsi_cmnd *cmd;
+ enum utp_ocs ocs;
lrbp = &hba->lrb[task_tag];
lrbp->compl_time_stamp = ktime_get();
@@ -5453,8 +5446,11 @@ void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag,
} else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
if (hba->dev_cmd.complete) {
- hba->dev_cmd.cqe = cqe;
- ufshcd_add_command_trace(hba, task_tag, UFS_DEV_COMP);
+ if (cqe) {
+ ocs = le32_to_cpu(cqe->status) & MASK_OCS;
+ lrbp->utr_descriptor_ptr->header.dword_2 =
+ cpu_to_le32(ocs);
+ }
complete(hba->dev_cmd.complete);
ufshcd_clk_scaling_update_busy(hba);
}
@@ -5507,7 +5503,7 @@ static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num)
struct ufs_hw_queue *hwq;
if (is_mcq_enabled(hba)) {
- hwq = &hba->uhq[queue_num + UFSHCD_MCQ_IO_QUEUE_OFFSET];
+ hwq = &hba->uhq[queue_num];
return ufshcd_mcq_poll_cqe_lock(hba, hwq);
}
@@ -5532,6 +5528,57 @@ static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num)
}
/**
+ * ufshcd_mcq_compl_pending_transfer - MCQ mode function. It is
+ * invoked from the error handler context or ufshcd_host_reset_and_restore()
+ * to complete the pending transfers and free the resources associated with
+ * the scsi command.
+ *
+ * @hba: per adapter instance
+ * @force_compl: This flag is set to true when invoked
+ * from ufshcd_host_reset_and_restore() in which case it requires special
+ * handling because the host controller has been reset by ufshcd_hba_stop().
+ */
+static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba,
+ bool force_compl)
+{
+ struct ufs_hw_queue *hwq;
+ struct ufshcd_lrb *lrbp;
+ struct scsi_cmnd *cmd;
+ unsigned long flags;
+ u32 hwq_num, utag;
+ int tag;
+
+ for (tag = 0; tag < hba->nutrs; tag++) {
+ lrbp = &hba->lrb[tag];
+ cmd = lrbp->cmd;
+ if (!ufshcd_cmd_inflight(cmd) ||
+ test_bit(SCMD_STATE_COMPLETE, &cmd->state))
+ continue;
+
+ utag = blk_mq_unique_tag(scsi_cmd_to_rq(cmd));
+ hwq_num = blk_mq_unique_tag_to_hwq(utag);
+ hwq = &hba->uhq[hwq_num];
+
+ if (force_compl) {
+ ufshcd_mcq_compl_all_cqes_lock(hba, hwq);
+ /*
+ * For those cmds of which the cqes are not present
+ * in the cq, complete them explicitly.
+ */
+ if (cmd && !test_bit(SCMD_STATE_COMPLETE, &cmd->state)) {
+ spin_lock_irqsave(&hwq->cq_lock, flags);
+ set_host_byte(cmd, DID_REQUEUE);
+ ufshcd_release_scsi_cmd(hba, lrbp);
+ scsi_done(cmd);
+ spin_unlock_irqrestore(&hwq->cq_lock, flags);
+ }
+ } else {
+ ufshcd_mcq_poll_cqe_lock(hba, hwq);
+ }
+ }
+}
+
+/**
* ufshcd_transfer_req_compl - handle SCSI and query command completion
* @hba: per adapter instance
*
@@ -6095,9 +6142,13 @@ out:
}
/* Complete requests that have door-bell cleared */
-static void ufshcd_complete_requests(struct ufs_hba *hba)
+static void ufshcd_complete_requests(struct ufs_hba *hba, bool force_compl)
{
- ufshcd_transfer_req_compl(hba);
+ if (is_mcq_enabled(hba))
+ ufshcd_mcq_compl_pending_transfer(hba, force_compl);
+ else
+ ufshcd_transfer_req_compl(hba);
+
ufshcd_tmc_handler(hba);
}
@@ -6241,22 +6292,22 @@ static void ufshcd_err_handling_prepare(struct ufs_hba *hba)
ufshcd_setup_vreg(hba, true);
ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
- ufshcd_hold(hba, false);
+ ufshcd_hold(hba);
if (!ufshcd_is_clkgating_allowed(hba))
ufshcd_setup_clocks(hba, true);
ufshcd_release(hba);
pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM;
ufshcd_vops_resume(hba, pm_op);
} else {
- ufshcd_hold(hba, false);
+ ufshcd_hold(hba);
if (ufshcd_is_clkscaling_supported(hba) &&
hba->clk_scaling.is_enabled)
ufshcd_suspend_clkscaling(hba);
ufshcd_clk_scaling_allow(hba, false);
}
ufshcd_scsi_block_requests(hba);
- /* Drain ufshcd_queuecommand() */
- synchronize_rcu();
+ /* Wait for ongoing ufshcd_queuecommand() calls to finish. */
+ blk_mq_wait_quiesce_done(&hba->host->tag_set);
cancel_work_sync(&hba->eeh_work);
}
@@ -6338,18 +6389,36 @@ static bool ufshcd_abort_all(struct ufs_hba *hba)
bool needs_reset = false;
int tag, ret;
- /* Clear pending transfer requests */
- for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
- ret = ufshcd_try_to_abort_task(hba, tag);
- dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
- hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
- ret ? "failed" : "succeeded");
- if (ret) {
- needs_reset = true;
- goto out;
+ if (is_mcq_enabled(hba)) {
+ struct ufshcd_lrb *lrbp;
+ int tag;
+
+ for (tag = 0; tag < hba->nutrs; tag++) {
+ lrbp = &hba->lrb[tag];
+ if (!ufshcd_cmd_inflight(lrbp->cmd))
+ continue;
+ ret = ufshcd_try_to_abort_task(hba, tag);
+ dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
+ hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
+ ret ? "failed" : "succeeded");
+ if (ret) {
+ needs_reset = true;
+ goto out;
+ }
+ }
+ } else {
+ /* Clear pending transfer requests */
+ for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
+ ret = ufshcd_try_to_abort_task(hba, tag);
+ dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
+ hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
+ ret ? "failed" : "succeeded");
+ if (ret) {
+ needs_reset = true;
+ goto out;
+ }
}
}
-
/* Clear pending task management requests */
for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
if (ufshcd_clear_tm_cmd(hba, tag)) {
@@ -6360,7 +6429,7 @@ static bool ufshcd_abort_all(struct ufs_hba *hba)
out:
/* Complete the requests that are cleared by s/w */
- ufshcd_complete_requests(hba);
+ ufshcd_complete_requests(hba, false);
return needs_reset;
}
@@ -6400,7 +6469,7 @@ static void ufshcd_err_handler(struct work_struct *work)
spin_unlock_irqrestore(hba->host->host_lock, flags);
ufshcd_err_handling_prepare(hba);
/* Complete requests that have door-bell cleared by h/w */
- ufshcd_complete_requests(hba);
+ ufshcd_complete_requests(hba, false);
spin_lock_irqsave(hba->host->host_lock, flags);
again:
needs_restore = false;
@@ -6771,7 +6840,7 @@ static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba)
ufshcd_mcq_write_cqis(hba, events, i);
if (events & UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS)
- ufshcd_mcq_poll_cqe_nolock(hba, hwq);
+ ufshcd_mcq_poll_cqe_lock(hba, hwq);
}
return IRQ_HANDLED;
@@ -6901,7 +6970,7 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
return PTR_ERR(req);
req->end_io_data = &wait;
- ufshcd_hold(hba, false);
+ ufshcd_hold(hba);
spin_lock_irqsave(host->host_lock, flags);
@@ -7037,7 +7106,6 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
down_read(&hba->clk_scaling_lock);
lrbp = &hba->lrb[tag];
- WARN_ON(lrbp->cmd);
lrbp->cmd = NULL;
lrbp->task_tag = tag;
lrbp->lun = 0;
@@ -7138,7 +7206,7 @@ int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
cmd_type = DEV_CMD_TYPE_NOP;
fallthrough;
case UPIU_TRANSACTION_QUERY_REQ:
- ufshcd_hold(hba, false);
+ ufshcd_hold(hba);
mutex_lock(&hba->dev_cmd.lock);
err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
desc_buff, buff_len,
@@ -7204,12 +7272,11 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r
u16 ehs_len;
/* Protects use of hba->reserved_slot. */
- ufshcd_hold(hba, false);
+ ufshcd_hold(hba);
mutex_lock(&hba->dev_cmd.lock);
down_read(&hba->clk_scaling_lock);
lrbp = &hba->lrb[tag];
- WARN_ON(lrbp->cmd);
lrbp->cmd = NULL;
lrbp->task_tag = tag;
lrbp->lun = UFS_UPIU_RPMB_WLUN;
@@ -7281,7 +7348,9 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
unsigned long flags, pending_reqs = 0, not_cleared = 0;
struct Scsi_Host *host;
struct ufs_hba *hba;
- u32 pos;
+ struct ufs_hw_queue *hwq;
+ struct ufshcd_lrb *lrbp;
+ u32 pos, not_cleared_mask = 0;
int err;
u8 resp = 0xF, lun;
@@ -7296,6 +7365,20 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
goto out;
}
+ if (is_mcq_enabled(hba)) {
+ for (pos = 0; pos < hba->nutrs; pos++) {
+ lrbp = &hba->lrb[pos];
+ if (ufshcd_cmd_inflight(lrbp->cmd) &&
+ lrbp->lun == lun) {
+ ufshcd_clear_cmd(hba, pos);
+ hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(lrbp->cmd));
+ ufshcd_mcq_poll_cqe_lock(hba, hwq);
+ }
+ }
+ err = 0;
+ goto out;
+ }
+
/* clear the commands that were pending for corresponding LUN */
spin_lock_irqsave(&hba->outstanding_lock, flags);
for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs)
@@ -7304,17 +7387,20 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
hba->outstanding_reqs &= ~pending_reqs;
spin_unlock_irqrestore(&hba->outstanding_lock, flags);
- if (ufshcd_clear_cmds(hba, pending_reqs) < 0) {
- spin_lock_irqsave(&hba->outstanding_lock, flags);
- not_cleared = pending_reqs &
- ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
- hba->outstanding_reqs |= not_cleared;
- spin_unlock_irqrestore(&hba->outstanding_lock, flags);
+ for_each_set_bit(pos, &pending_reqs, hba->nutrs) {
+ if (ufshcd_clear_cmd(hba, pos) < 0) {
+ spin_lock_irqsave(&hba->outstanding_lock, flags);
+ not_cleared = 1U << pos &
+ ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
+ hba->outstanding_reqs |= not_cleared;
+ not_cleared_mask |= not_cleared;
+ spin_unlock_irqrestore(&hba->outstanding_lock, flags);
- dev_err(hba->dev, "%s: failed to clear requests %#lx\n",
- __func__, not_cleared);
+ dev_err(hba->dev, "%s: failed to clear request %d\n",
+ __func__, pos);
+ }
}
- __ufshcd_transfer_req_compl(hba, pending_reqs & ~not_cleared);
+ __ufshcd_transfer_req_compl(hba, pending_reqs & ~not_cleared_mask);
out:
hba->req_abort_count = 0;
@@ -7352,7 +7438,7 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
*
* Returns zero on success, non-zero on failure
*/
-static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
+int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
{
struct ufshcd_lrb *lrbp = &hba->lrb[tag];
int err = 0;
@@ -7375,6 +7461,20 @@ static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
*/
dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
__func__, tag);
+ if (is_mcq_enabled(hba)) {
+ /* MCQ mode */
+ if (ufshcd_cmd_inflight(lrbp->cmd)) {
+ /* sleep for max. 200us same delay as in SDB mode */
+ usleep_range(100, 200);
+ continue;
+ }
+ /* command completed already */
+ dev_err(hba->dev, "%s: cmd at tag=%d is cleared.\n",
+ __func__, tag);
+ goto out;
+ }
+
+ /* Single Doorbell Mode */
reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
if (reg & (1 << tag)) {
/* sleep for max. 200us to stabilize */
@@ -7411,7 +7511,7 @@ static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
goto out;
}
- err = ufshcd_clear_cmds(hba, 1U << tag);
+ err = ufshcd_clear_cmd(hba, tag);
if (err)
dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
__func__, tag, err);
@@ -7439,14 +7539,17 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
WARN_ONCE(tag < 0, "Invalid tag %d\n", tag);
- ufshcd_hold(hba, false);
- reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
- /* If command is already aborted/completed, return FAILED. */
- if (!(test_bit(tag, &hba->outstanding_reqs))) {
- dev_err(hba->dev,
- "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
- __func__, tag, hba->outstanding_reqs, reg);
- goto release;
+ ufshcd_hold(hba);
+
+ if (!is_mcq_enabled(hba)) {
+ reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
+ if (!test_bit(tag, &hba->outstanding_reqs)) {
+ /* If command is already aborted/completed, return FAILED. */
+ dev_err(hba->dev,
+ "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
+ __func__, tag, hba->outstanding_reqs, reg);
+ goto release;
+ }
}
/* Print Transfer Request of aborted task */
@@ -7471,7 +7574,8 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
}
hba->req_abort_count++;
- if (!(reg & (1 << tag))) {
+ if (!is_mcq_enabled(hba) && !(reg & (1 << tag))) {
+ /* only execute this code in single doorbell mode */
dev_err(hba->dev,
"%s: cmd was completed, but without a notifying intr, tag = %d",
__func__, tag);
@@ -7497,6 +7601,12 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
goto release;
}
+ if (is_mcq_enabled(hba)) {
+ /* MCQ mode. Branch off to handle abort for mcq mode */
+ err = ufshcd_mcq_abort(cmd);
+ goto release;
+ }
+
/* Skip task abort in case previous aborts failed and report failure */
if (lrbp->req_abort_skip) {
dev_err(hba->dev, "%s: skipping abort\n", __func__);
@@ -7552,7 +7662,7 @@ static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
ufshpb_toggle_state(hba, HPB_PRESENT, HPB_RESET);
ufshcd_hba_stop(hba);
hba->silence_err_logs = true;
- ufshcd_complete_requests(hba);
+ ufshcd_complete_requests(hba, true);
hba->silence_err_logs = false;
/* scale up clocks to max frequency before full reinitialization */
@@ -8502,11 +8612,15 @@ err:
static void ufshcd_config_mcq(struct ufs_hba *hba)
{
int ret;
+ u32 intrs;
ret = ufshcd_mcq_vops_config_esi(hba);
dev_info(hba->dev, "ESI %sconfigured\n", ret ? "is not " : "");
- ufshcd_enable_intr(hba, UFSHCD_ENABLE_MCQ_INTRS);
+ intrs = UFSHCD_ENABLE_MCQ_INTRS;
+ if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_INTR)
+ intrs &= ~MCQ_CQ_EVENT_STATUS;
+ ufshcd_enable_intr(hba, intrs);
ufshcd_mcq_make_queues_operational(hba);
ufshcd_mcq_config_mac(hba, hba->nutrs);
@@ -8774,7 +8888,7 @@ static const struct scsi_host_template ufshcd_driver_template = {
.cmd_per_lun = UFSHCD_CMD_PER_LUN,
.can_queue = UFSHCD_CAN_QUEUE,
.max_segment_size = PRDT_DATA_BYTE_COUNT_MAX,
- .max_sectors = (1 << 20) / SECTOR_SIZE, /* 1 MiB */
+ .max_sectors = SZ_1M / SECTOR_SIZE,
.max_host_blocked = 1,
.track_queue_depth = 1,
.skip_settle_delay = 1,
@@ -9184,7 +9298,8 @@ static int ufshcd_execute_start_stop(struct scsi_device *sdev,
};
return scsi_execute_cmd(sdev, cdb, REQ_OP_DRV_IN, /*buffer=*/NULL,
- /*bufflen=*/0, /*timeout=*/HZ, /*retries=*/0, &args);
+ /*bufflen=*/0, /*timeout=*/10 * HZ, /*retries=*/0,
+ &args);
}
/**
@@ -9430,7 +9545,7 @@ static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
* If we can't transition into any of the low power modes
* just gate the clocks.
*/
- ufshcd_hold(hba, false);
+ ufshcd_hold(hba);
hba->clk_gating.is_suspended = true;
if (ufshcd_is_clkscaling_supported(hba))
@@ -9775,28 +9890,6 @@ out:
}
#endif
-static void ufshcd_wl_shutdown(struct device *dev)
-{
- struct scsi_device *sdev = to_scsi_device(dev);
- struct ufs_hba *hba;
-
- hba = shost_priv(sdev->host);
-
- down(&hba->host_sem);
- hba->shutting_down = true;
- up(&hba->host_sem);
-
- /* Turn on everything while shutting down */
- ufshcd_rpm_get_sync(hba);
- scsi_device_quiesce(sdev);
- shost_for_each_device(sdev, hba->host) {
- if (sdev == hba->ufs_device_wlun)
- continue;
- scsi_device_quiesce(sdev);
- }
- __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM);
-}
-
/**
* ufshcd_suspend - helper function for suspend operations
* @hba: per adapter instance
@@ -9981,25 +10074,34 @@ int ufshcd_runtime_resume(struct device *dev)
EXPORT_SYMBOL(ufshcd_runtime_resume);
#endif /* CONFIG_PM */
-/**
- * ufshcd_shutdown - shutdown routine
- * @hba: per adapter instance
- *
- * This function would turn off both UFS device and UFS hba
- * regulators. It would also disable clocks.
- *
- * Returns 0 always to allow force shutdown even in case of errors.
- */
-int ufshcd_shutdown(struct ufs_hba *hba)
+static void ufshcd_wl_shutdown(struct device *dev)
{
+ struct scsi_device *sdev = to_scsi_device(dev);
+ struct ufs_hba *hba = shost_priv(sdev->host);
+
+ down(&hba->host_sem);
+ hba->shutting_down = true;
+ up(&hba->host_sem);
+
+ /* Turn on everything while shutting down */
+ ufshcd_rpm_get_sync(hba);
+ scsi_device_quiesce(sdev);
+ shost_for_each_device(sdev, hba->host) {
+ if (sdev == hba->ufs_device_wlun)
+ continue;
+ scsi_device_quiesce(sdev);
+ }
+ __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM);
+
+ /*
+ * Next, turn off the UFS controller and the UFS regulators. Disable
+ * clocks.
+ */
if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
ufshcd_suspend(hba);
hba->is_powered = false;
- /* allow force shutdown even in case of errors */
- return 0;
}
-EXPORT_SYMBOL(ufshcd_shutdown);
/**
* ufshcd_remove - de-allocate SCSI host and host memory space
@@ -10226,6 +10328,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
host->max_channel = UFSHCD_MAX_CHANNEL;
host->unique_id = host->host_no;
host->max_cmd_len = UFS_CDB_SIZE;
+ host->queuecommand_may_block = !!(hba->caps & UFSHCD_CAP_CLK_GATING);
hba->max_pwr_info.is_valid = false;
diff --git a/drivers/ufs/core/ufshpb.c b/drivers/ufs/core/ufshpb.c
index a46a7666c891..255f8b38d0c2 100644
--- a/drivers/ufs/core/ufshpb.c
+++ b/drivers/ufs/core/ufshpb.c
@@ -30,7 +30,7 @@ static struct kmem_cache *ufshpb_mctx_cache;
static mempool_t *ufshpb_mctx_pool;
static mempool_t *ufshpb_page_pool;
/* A cache size of 2MB can cache ppn in the 1GB range. */
-static unsigned int ufshpb_host_map_kbytes = 2048;
+static unsigned int ufshpb_host_map_kbytes = SZ_2K;
static int tot_active_srgn_pages;
static struct workqueue_struct *ufshpb_wq;
@@ -2461,7 +2461,7 @@ static void ufshpb_hpb_lu_prepared(struct ufs_hba *hba)
init_success = !ufshpb_check_hpb_reset_query(hba);
- pool_size = PAGE_ALIGN(ufshpb_host_map_kbytes * 1024) / PAGE_SIZE;
+ pool_size = PAGE_ALIGN(ufshpb_host_map_kbytes * SZ_1K) / PAGE_SIZE;
if (pool_size > tot_active_srgn_pages) {
mempool_resize(ufshpb_mctx_pool, tot_active_srgn_pages);
mempool_resize(ufshpb_page_pool, tot_active_srgn_pages);
@@ -2527,7 +2527,7 @@ static int ufshpb_init_mem_wq(struct ufs_hba *hba)
return -ENOMEM;
}
- pool_size = PAGE_ALIGN(ufshpb_host_map_kbytes * 1024) / PAGE_SIZE;
+ pool_size = PAGE_ALIGN(ufshpb_host_map_kbytes * SZ_1K) / PAGE_SIZE;
dev_info(hba->dev, "%s:%d ufshpb_host_map_kbytes %u pool_size %u\n",
__func__, __LINE__, ufshpb_host_map_kbytes, pool_size);
diff --git a/drivers/ufs/core/ufshpb.h b/drivers/ufs/core/ufshpb.h
index 0d6e6004d783..b428bbdd2799 100644
--- a/drivers/ufs/core/ufshpb.h
+++ b/drivers/ufs/core/ufshpb.h
@@ -25,7 +25,7 @@
/* hpb map & entries macro */
#define HPB_RGN_SIZE_UNIT 512
-#define HPB_ENTRY_BLOCK_SIZE 4096
+#define HPB_ENTRY_BLOCK_SIZE SZ_4K
#define HPB_ENTRY_SIZE 0x8
#define PINNED_NOT_SET U32_MAX
diff --git a/drivers/ufs/host/Kconfig b/drivers/ufs/host/Kconfig
index 8793e3433580..16624ba08050 100644
--- a/drivers/ufs/host/Kconfig
+++ b/drivers/ufs/host/Kconfig
@@ -59,7 +59,7 @@ config SCSI_UFS_QCOM
depends on SCSI_UFSHCD_PLATFORM && ARCH_QCOM
depends on GENERIC_MSI_IRQ
depends on RESET_CONTROLLER
- select QCOM_SCM if SCSI_UFS_CRYPTO
+ select QCOM_INLINE_CRYPTO_ENGINE if SCSI_UFS_CRYPTO
help
This selects the QCOM specific additions to UFSHCD platform driver.
UFS host on QCOM needs some vendor specific configuration before
diff --git a/drivers/ufs/host/Makefile b/drivers/ufs/host/Makefile
index d7c5bf7fa512..4573aead02eb 100644
--- a/drivers/ufs/host/Makefile
+++ b/drivers/ufs/host/Makefile
@@ -3,9 +3,7 @@
obj-$(CONFIG_SCSI_UFS_DWC_TC_PCI) += tc-dwc-g210-pci.o ufshcd-dwc.o tc-dwc-g210.o
obj-$(CONFIG_SCSI_UFS_DWC_TC_PLATFORM) += tc-dwc-g210-pltfrm.o ufshcd-dwc.o tc-dwc-g210.o
obj-$(CONFIG_SCSI_UFS_CDNS_PLATFORM) += cdns-pltfrm.o
-obj-$(CONFIG_SCSI_UFS_QCOM) += ufs_qcom.o
-ufs_qcom-y += ufs-qcom.o
-ufs_qcom-$(CONFIG_SCSI_UFS_CRYPTO) += ufs-qcom-ice.o
+obj-$(CONFIG_SCSI_UFS_QCOM) += ufs-qcom.o
obj-$(CONFIG_SCSI_UFS_EXYNOS) += ufs-exynos.o
obj-$(CONFIG_SCSI_UFSHCD_PCI) += ufshcd-pci.o
obj-$(CONFIG_SCSI_UFSHCD_PLATFORM) += ufshcd-pltfrm.o
diff --git a/drivers/ufs/host/cdns-pltfrm.c b/drivers/ufs/host/cdns-pltfrm.c
index e05c0ae64eea..26761425a76c 100644
--- a/drivers/ufs/host/cdns-pltfrm.c
+++ b/drivers/ufs/host/cdns-pltfrm.c
@@ -328,7 +328,6 @@ static const struct dev_pm_ops cdns_ufs_dev_pm_ops = {
static struct platform_driver cdns_ufs_pltfrm_driver = {
.probe = cdns_ufs_pltfrm_probe,
.remove = cdns_ufs_pltfrm_remove,
- .shutdown = ufshcd_pltfrm_shutdown,
.driver = {
.name = "cdns-ufshcd",
.pm = &cdns_ufs_dev_pm_ops,
diff --git a/drivers/ufs/host/tc-dwc-g210-pci.c b/drivers/ufs/host/tc-dwc-g210-pci.c
index 92b8ad4b58fe..f96fe5855841 100644
--- a/drivers/ufs/host/tc-dwc-g210-pci.c
+++ b/drivers/ufs/host/tc-dwc-g210-pci.c
@@ -33,15 +33,6 @@ static struct ufs_hba_variant_ops tc_dwc_g210_pci_hba_vops = {
};
/**
- * tc_dwc_g210_pci_shutdown - main function to put the controller in reset state
- * @pdev: pointer to PCI device handle
- */
-static void tc_dwc_g210_pci_shutdown(struct pci_dev *pdev)
-{
- ufshcd_shutdown((struct ufs_hba *)pci_get_drvdata(pdev));
-}
-
-/**
* tc_dwc_g210_pci_remove - de-allocate PCI/SCSI host and host memory space
* data structure memory
* @pdev: pointer to PCI handle
@@ -137,7 +128,6 @@ static struct pci_driver tc_dwc_g210_pci_driver = {
.id_table = tc_dwc_g210_pci_tbl,
.probe = tc_dwc_g210_pci_probe,
.remove = tc_dwc_g210_pci_remove,
- .shutdown = tc_dwc_g210_pci_shutdown,
.driver = {
.pm = &tc_dwc_g210_pci_pm_ops
},
diff --git a/drivers/ufs/host/tc-dwc-g210-pltfrm.c b/drivers/ufs/host/tc-dwc-g210-pltfrm.c
index f15a84d0c176..4d5389dd9585 100644
--- a/drivers/ufs/host/tc-dwc-g210-pltfrm.c
+++ b/drivers/ufs/host/tc-dwc-g210-pltfrm.c
@@ -92,7 +92,6 @@ static const struct dev_pm_ops tc_dwc_g210_pltfm_pm_ops = {
static struct platform_driver tc_dwc_g210_pltfm_driver = {
.probe = tc_dwc_g210_pltfm_probe,
.remove = tc_dwc_g210_pltfm_remove,
- .shutdown = ufshcd_pltfrm_shutdown,
.driver = {
.name = "tc-dwc-g210-pltfm",
.pm = &tc_dwc_g210_pltfm_pm_ops,
diff --git a/drivers/ufs/host/ufs-exynos.c b/drivers/ufs/host/ufs-exynos.c
index 0bf5390739e1..3396e0388512 100644
--- a/drivers/ufs/host/ufs-exynos.c
+++ b/drivers/ufs/host/ufs-exynos.c
@@ -1306,7 +1306,7 @@ static int exynos_ufs_hce_enable_notify(struct ufs_hba *hba,
* (ufshcd_async_scan()). Note: this callback may also be called
* from other functions than ufshcd_init().
*/
- hba->host->max_segment_size = 4096;
+ hba->host->max_segment_size = SZ_4K;
if (ufs->drv_data->pre_hce_enable) {
ret = ufs->drv_data->pre_hce_enable(ufs);
@@ -1757,7 +1757,6 @@ static const struct dev_pm_ops exynos_ufs_pm_ops = {
static struct platform_driver exynos_ufs_pltform = {
.probe = exynos_ufs_probe,
.remove = exynos_ufs_remove,
- .shutdown = ufshcd_pltfrm_shutdown,
.driver = {
.name = "exynos-ufshc",
.pm = &exynos_ufs_pm_ops,
diff --git a/drivers/ufs/host/ufs-hisi.c b/drivers/ufs/host/ufs-hisi.c
index 4c423eba8aa9..5b3060cd0ab8 100644
--- a/drivers/ufs/host/ufs-hisi.c
+++ b/drivers/ufs/host/ufs-hisi.c
@@ -335,29 +335,29 @@ static void ufs_hisi_pwr_change_pre_change(struct ufs_hba *hba)
/* PA_TxSkip */
ufshcd_dme_set(hba, UIC_ARG_MIB(0x155c), 0x0);
/*PA_PWRModeUserData0 = 8191, default is 0*/
- ufshcd_dme_set(hba, UIC_ARG_MIB(0x15b0), 8191);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(0x15b0), SZ_8K - 1);
/*PA_PWRModeUserData1 = 65535, default is 0*/
- ufshcd_dme_set(hba, UIC_ARG_MIB(0x15b1), 65535);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(0x15b1), SZ_64K - 1);
/*PA_PWRModeUserData2 = 32767, default is 0*/
- ufshcd_dme_set(hba, UIC_ARG_MIB(0x15b2), 32767);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(0x15b2), SZ_32K - 1);
/*DME_FC0ProtectionTimeOutVal = 8191, default is 0*/
- ufshcd_dme_set(hba, UIC_ARG_MIB(0xd041), 8191);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(0xd041), SZ_8K - 1);
/*DME_TC0ReplayTimeOutVal = 65535, default is 0*/
- ufshcd_dme_set(hba, UIC_ARG_MIB(0xd042), 65535);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(0xd042), SZ_64K - 1);
/*DME_AFC0ReqTimeOutVal = 32767, default is 0*/
- ufshcd_dme_set(hba, UIC_ARG_MIB(0xd043), 32767);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(0xd043), SZ_32K - 1);
/*PA_PWRModeUserData3 = 8191, default is 0*/
- ufshcd_dme_set(hba, UIC_ARG_MIB(0x15b3), 8191);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(0x15b3), SZ_8K - 1);
/*PA_PWRModeUserData4 = 65535, default is 0*/
- ufshcd_dme_set(hba, UIC_ARG_MIB(0x15b4), 65535);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(0x15b4), SZ_64K - 1);
/*PA_PWRModeUserData5 = 32767, default is 0*/
- ufshcd_dme_set(hba, UIC_ARG_MIB(0x15b5), 32767);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(0x15b5), SZ_32K - 1);
/*DME_FC1ProtectionTimeOutVal = 8191, default is 0*/
- ufshcd_dme_set(hba, UIC_ARG_MIB(0xd044), 8191);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(0xd044), SZ_8K - 1);
/*DME_TC1ReplayTimeOutVal = 65535, default is 0*/
- ufshcd_dme_set(hba, UIC_ARG_MIB(0xd045), 65535);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(0xd045), SZ_64K - 1);
/*DME_AFC1ReqTimeOutVal = 32767, default is 0*/
- ufshcd_dme_set(hba, UIC_ARG_MIB(0xd046), 32767);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(0xd046), SZ_32K - 1);
}
static int ufs_hisi_pwr_change_notify(struct ufs_hba *hba,
@@ -593,7 +593,6 @@ static const struct dev_pm_ops ufs_hisi_pm_ops = {
static struct platform_driver ufs_hisi_pltform = {
.probe = ufs_hisi_probe,
.remove = ufs_hisi_remove,
- .shutdown = ufshcd_pltfrm_shutdown,
.driver = {
.name = "ufshcd-hisi",
.pm = &ufs_hisi_pm_ops,
diff --git a/drivers/ufs/host/ufs-mediatek.c b/drivers/ufs/host/ufs-mediatek.c
index 73e217260390..e68b05976f9e 100644
--- a/drivers/ufs/host/ufs-mediatek.c
+++ b/drivers/ufs/host/ufs-mediatek.c
@@ -410,9 +410,6 @@ static int ufs_mtk_wait_link_state(struct ufs_hba *hba, u32 state,
usleep_range(100, 200);
} while (ktime_before(time_checked, timeout));
- if (val == state)
- return 0;
-
return -ETIMEDOUT;
}
@@ -901,6 +898,8 @@ static int ufs_mtk_init(struct ufs_hba *hba)
hba->caps |= UFSHCD_CAP_CLK_SCALING;
hba->quirks |= UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL;
+ hba->quirks |= UFSHCD_QUIRK_MCQ_BROKEN_INTR;
+ hba->quirks |= UFSHCD_QUIRK_MCQ_BROKEN_RTC;
hba->vps->wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(80);
if (host->caps & UFS_MTK_CAP_DISABLE_AH8)
@@ -1650,7 +1649,6 @@ static const struct dev_pm_ops ufs_mtk_pm_ops = {
static struct platform_driver ufs_mtk_pltform = {
.probe = ufs_mtk_probe,
.remove = ufs_mtk_remove,
- .shutdown = ufshcd_pltfrm_shutdown,
.driver = {
.name = "ufshcd-mtk",
.pm = &ufs_mtk_pm_ops,
diff --git a/drivers/ufs/host/ufs-qcom-ice.c b/drivers/ufs/host/ufs-qcom-ice.c
deleted file mode 100644
index 453978877ae9..000000000000
--- a/drivers/ufs/host/ufs-qcom-ice.c
+++ /dev/null
@@ -1,244 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Qualcomm ICE (Inline Crypto Engine) support.
- *
- * Copyright (c) 2014-2019, The Linux Foundation. All rights reserved.
- * Copyright 2019 Google LLC
- */
-
-#include <linux/delay.h>
-#include <linux/platform_device.h>
-#include <linux/firmware/qcom/qcom_scm.h>
-
-#include "ufs-qcom.h"
-
-#define AES_256_XTS_KEY_SIZE 64
-
-/* QCOM ICE registers */
-
-#define QCOM_ICE_REG_CONTROL 0x0000
-#define QCOM_ICE_REG_RESET 0x0004
-#define QCOM_ICE_REG_VERSION 0x0008
-#define QCOM_ICE_REG_FUSE_SETTING 0x0010
-#define QCOM_ICE_REG_PARAMETERS_1 0x0014
-#define QCOM_ICE_REG_PARAMETERS_2 0x0018
-#define QCOM_ICE_REG_PARAMETERS_3 0x001C
-#define QCOM_ICE_REG_PARAMETERS_4 0x0020
-#define QCOM_ICE_REG_PARAMETERS_5 0x0024
-
-/* QCOM ICE v3.X only */
-#define QCOM_ICE_GENERAL_ERR_STTS 0x0040
-#define QCOM_ICE_INVALID_CCFG_ERR_STTS 0x0030
-#define QCOM_ICE_GENERAL_ERR_MASK 0x0044
-
-/* QCOM ICE v2.X only */
-#define QCOM_ICE_REG_NON_SEC_IRQ_STTS 0x0040
-#define QCOM_ICE_REG_NON_SEC_IRQ_MASK 0x0044
-
-#define QCOM_ICE_REG_NON_SEC_IRQ_CLR 0x0048
-#define QCOM_ICE_REG_STREAM1_ERROR_SYNDROME1 0x0050
-#define QCOM_ICE_REG_STREAM1_ERROR_SYNDROME2 0x0054
-#define QCOM_ICE_REG_STREAM2_ERROR_SYNDROME1 0x0058
-#define QCOM_ICE_REG_STREAM2_ERROR_SYNDROME2 0x005C
-#define QCOM_ICE_REG_STREAM1_BIST_ERROR_VEC 0x0060
-#define QCOM_ICE_REG_STREAM2_BIST_ERROR_VEC 0x0064
-#define QCOM_ICE_REG_STREAM1_BIST_FINISH_VEC 0x0068
-#define QCOM_ICE_REG_STREAM2_BIST_FINISH_VEC 0x006C
-#define QCOM_ICE_REG_BIST_STATUS 0x0070
-#define QCOM_ICE_REG_BYPASS_STATUS 0x0074
-#define QCOM_ICE_REG_ADVANCED_CONTROL 0x1000
-#define QCOM_ICE_REG_ENDIAN_SWAP 0x1004
-#define QCOM_ICE_REG_TEST_BUS_CONTROL 0x1010
-#define QCOM_ICE_REG_TEST_BUS_REG 0x1014
-
-/* BIST ("built-in self-test"?) status flags */
-#define QCOM_ICE_BIST_STATUS_MASK 0xF0000000
-
-#define QCOM_ICE_FUSE_SETTING_MASK 0x1
-#define QCOM_ICE_FORCE_HW_KEY0_SETTING_MASK 0x2
-#define QCOM_ICE_FORCE_HW_KEY1_SETTING_MASK 0x4
-
-#define qcom_ice_writel(host, val, reg) \
- writel((val), (host)->ice_mmio + (reg))
-#define qcom_ice_readl(host, reg) \
- readl((host)->ice_mmio + (reg))
-
-static bool qcom_ice_supported(struct ufs_qcom_host *host)
-{
- struct device *dev = host->hba->dev;
- u32 regval = qcom_ice_readl(host, QCOM_ICE_REG_VERSION);
- int major = regval >> 24;
- int minor = (regval >> 16) & 0xFF;
- int step = regval & 0xFFFF;
-
- /* For now this driver only supports ICE version 3. */
- if (major != 3) {
- dev_warn(dev, "Unsupported ICE version: v%d.%d.%d\n",
- major, minor, step);
- return false;
- }
-
- dev_info(dev, "Found QC Inline Crypto Engine (ICE) v%d.%d.%d\n",
- major, minor, step);
-
- /* If fuses are blown, ICE might not work in the standard way. */
- regval = qcom_ice_readl(host, QCOM_ICE_REG_FUSE_SETTING);
- if (regval & (QCOM_ICE_FUSE_SETTING_MASK |
- QCOM_ICE_FORCE_HW_KEY0_SETTING_MASK |
- QCOM_ICE_FORCE_HW_KEY1_SETTING_MASK)) {
- dev_warn(dev, "Fuses are blown; ICE is unusable!\n");
- return false;
- }
- return true;
-}
-
-int ufs_qcom_ice_init(struct ufs_qcom_host *host)
-{
- struct ufs_hba *hba = host->hba;
- struct device *dev = hba->dev;
- struct platform_device *pdev = to_platform_device(dev);
- struct resource *res;
- int err;
-
- if (!(ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES) &
- MASK_CRYPTO_SUPPORT))
- return 0;
-
- res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ice");
- if (!res) {
- dev_warn(dev, "ICE registers not found\n");
- goto disable;
- }
-
- if (!qcom_scm_ice_available()) {
- dev_warn(dev, "ICE SCM interface not found\n");
- goto disable;
- }
-
- host->ice_mmio = devm_ioremap_resource(dev, res);
- if (IS_ERR(host->ice_mmio)) {
- err = PTR_ERR(host->ice_mmio);
- return err;
- }
-
- if (!qcom_ice_supported(host))
- goto disable;
-
- return 0;
-
-disable:
- dev_warn(dev, "Disabling inline encryption support\n");
- hba->caps &= ~UFSHCD_CAP_CRYPTO;
- return 0;
-}
-
-static void qcom_ice_low_power_mode_enable(struct ufs_qcom_host *host)
-{
- u32 regval;
-
- regval = qcom_ice_readl(host, QCOM_ICE_REG_ADVANCED_CONTROL);
- /*
- * Enable low power mode sequence
- * [0]-0, [1]-0, [2]-0, [3]-E, [4]-0, [5]-0, [6]-0, [7]-0
- */
- regval |= 0x7000;
- qcom_ice_writel(host, regval, QCOM_ICE_REG_ADVANCED_CONTROL);
-}
-
-static void qcom_ice_optimization_enable(struct ufs_qcom_host *host)
-{
- u32 regval;
-
- /* ICE Optimizations Enable Sequence */
- regval = qcom_ice_readl(host, QCOM_ICE_REG_ADVANCED_CONTROL);
- regval |= 0xD807100;
- /* ICE HPG requires delay before writing */
- udelay(5);
- qcom_ice_writel(host, regval, QCOM_ICE_REG_ADVANCED_CONTROL);
- udelay(5);
-}
-
-int ufs_qcom_ice_enable(struct ufs_qcom_host *host)
-{
- if (!(host->hba->caps & UFSHCD_CAP_CRYPTO))
- return 0;
- qcom_ice_low_power_mode_enable(host);
- qcom_ice_optimization_enable(host);
- return ufs_qcom_ice_resume(host);
-}
-
-/* Poll until all BIST bits are reset */
-static int qcom_ice_wait_bist_status(struct ufs_qcom_host *host)
-{
- int count;
- u32 reg;
-
- for (count = 0; count < 100; count++) {
- reg = qcom_ice_readl(host, QCOM_ICE_REG_BIST_STATUS);
- if (!(reg & QCOM_ICE_BIST_STATUS_MASK))
- break;
- udelay(50);
- }
- if (reg)
- return -ETIMEDOUT;
- return 0;
-}
-
-int ufs_qcom_ice_resume(struct ufs_qcom_host *host)
-{
- int err;
-
- if (!(host->hba->caps & UFSHCD_CAP_CRYPTO))
- return 0;
-
- err = qcom_ice_wait_bist_status(host);
- if (err) {
- dev_err(host->hba->dev, "BIST status error (%d)\n", err);
- return err;
- }
- return 0;
-}
-
-/*
- * Program a key into a QC ICE keyslot, or evict a keyslot. QC ICE requires
- * vendor-specific SCM calls for this; it doesn't support the standard way.
- */
-int ufs_qcom_ice_program_key(struct ufs_hba *hba,
- const union ufs_crypto_cfg_entry *cfg, int slot)
-{
- union ufs_crypto_cap_entry cap;
- union {
- u8 bytes[AES_256_XTS_KEY_SIZE];
- u32 words[AES_256_XTS_KEY_SIZE / sizeof(u32)];
- } key;
- int i;
- int err;
-
- if (!(cfg->config_enable & UFS_CRYPTO_CONFIGURATION_ENABLE))
- return qcom_scm_ice_invalidate_key(slot);
-
- /* Only AES-256-XTS has been tested so far. */
- cap = hba->crypto_cap_array[cfg->crypto_cap_idx];
- if (cap.algorithm_id != UFS_CRYPTO_ALG_AES_XTS ||
- cap.key_size != UFS_CRYPTO_KEY_SIZE_256) {
- dev_err_ratelimited(hba->dev,
- "Unhandled crypto capability; algorithm_id=%d, key_size=%d\n",
- cap.algorithm_id, cap.key_size);
- return -EINVAL;
- }
-
- memcpy(key.bytes, cfg->crypto_key, AES_256_XTS_KEY_SIZE);
-
- /*
- * The SCM call byte-swaps the 32-bit words of the key. So we have to
- * do the same, in order for the final key be correct.
- */
- for (i = 0; i < ARRAY_SIZE(key.words); i++)
- __cpu_to_be32s(&key.words[i]);
-
- err = qcom_scm_ice_set_key(slot, key.bytes, AES_256_XTS_KEY_SIZE,
- QCOM_SCM_ICE_CIPHER_AES_256_XTS,
- cfg->data_unit_size);
- memzero_explicit(&key, sizeof(key));
- return err;
-}
diff --git a/drivers/ufs/host/ufs-qcom.c b/drivers/ufs/host/ufs-qcom.c
index 82d02e7f3b4f..8d6fd4c3324f 100644
--- a/drivers/ufs/host/ufs-qcom.c
+++ b/drivers/ufs/host/ufs-qcom.c
@@ -15,6 +15,8 @@
#include <linux/reset-controller.h>
#include <linux/devfreq.h>
+#include <soc/qcom/ice.h>
+
#include <ufs/ufshcd.h>
#include "ufshcd-pltfrm.h"
#include <ufs/unipro.h>
@@ -55,6 +57,100 @@ static struct ufs_qcom_host *rcdev_to_ufs_host(struct reset_controller_dev *rcd)
return container_of(rcd, struct ufs_qcom_host, rcdev);
}
+#ifdef CONFIG_SCSI_UFS_CRYPTO
+
+static inline void ufs_qcom_ice_enable(struct ufs_qcom_host *host)
+{
+ if (host->hba->caps & UFSHCD_CAP_CRYPTO)
+ qcom_ice_enable(host->ice);
+}
+
+static int ufs_qcom_ice_init(struct ufs_qcom_host *host)
+{
+ struct ufs_hba *hba = host->hba;
+ struct device *dev = hba->dev;
+ struct qcom_ice *ice;
+
+ ice = of_qcom_ice_get(dev);
+ if (ice == ERR_PTR(-EOPNOTSUPP)) {
+ dev_warn(dev, "Disabling inline encryption support\n");
+ ice = NULL;
+ }
+
+ if (IS_ERR_OR_NULL(ice))
+ return PTR_ERR_OR_ZERO(ice);
+
+ host->ice = ice;
+ hba->caps |= UFSHCD_CAP_CRYPTO;
+
+ return 0;
+}
+
+static inline int ufs_qcom_ice_resume(struct ufs_qcom_host *host)
+{
+ if (host->hba->caps & UFSHCD_CAP_CRYPTO)
+ return qcom_ice_resume(host->ice);
+
+ return 0;
+}
+
+static inline int ufs_qcom_ice_suspend(struct ufs_qcom_host *host)
+{
+ if (host->hba->caps & UFSHCD_CAP_CRYPTO)
+ return qcom_ice_suspend(host->ice);
+
+ return 0;
+}
+
+static int ufs_qcom_ice_program_key(struct ufs_hba *hba,
+ const union ufs_crypto_cfg_entry *cfg,
+ int slot)
+{
+ struct ufs_qcom_host *host = ufshcd_get_variant(hba);
+ union ufs_crypto_cap_entry cap;
+ bool config_enable =
+ cfg->config_enable & UFS_CRYPTO_CONFIGURATION_ENABLE;
+
+ /* Only AES-256-XTS has been tested so far. */
+ cap = hba->crypto_cap_array[cfg->crypto_cap_idx];
+ if (cap.algorithm_id != UFS_CRYPTO_ALG_AES_XTS ||
+ cap.key_size != UFS_CRYPTO_KEY_SIZE_256)
+ return -EINVAL;
+
+ if (config_enable)
+ return qcom_ice_program_key(host->ice,
+ QCOM_ICE_CRYPTO_ALG_AES_XTS,
+ QCOM_ICE_CRYPTO_KEY_SIZE_256,
+ cfg->crypto_key,
+ cfg->data_unit_size, slot);
+ else
+ return qcom_ice_evict_key(host->ice, slot);
+}
+
+#else
+
+#define ufs_qcom_ice_program_key NULL
+
+static inline void ufs_qcom_ice_enable(struct ufs_qcom_host *host)
+{
+}
+
+static int ufs_qcom_ice_init(struct ufs_qcom_host *host)
+{
+ return 0;
+}
+
+static inline int ufs_qcom_ice_resume(struct ufs_qcom_host *host)
+{
+ return 0;
+}
+
+static inline int ufs_qcom_ice_suspend(struct ufs_qcom_host *host)
+{
+ return 0;
+}
+#endif
+
static int ufs_qcom_host_clk_get(struct device *dev,
const char *name, struct clk **clk_out, bool optional)
{
@@ -607,7 +703,7 @@ static int ufs_qcom_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op,
ufs_qcom_disable_lane_clks(host);
}
- return 0;
+ return ufs_qcom_ice_suspend(host);
}
static int ufs_qcom_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
@@ -853,7 +949,6 @@ static void ufs_qcom_set_caps(struct ufs_hba *hba)
hba->caps |= UFSHCD_CAP_CLK_SCALING | UFSHCD_CAP_WB_WITH_CLK_SCALING;
hba->caps |= UFSHCD_CAP_AUTO_BKOPS_SUSPEND;
hba->caps |= UFSHCD_CAP_WB_EN;
- hba->caps |= UFSHCD_CAP_CRYPTO;
hba->caps |= UFSHCD_CAP_AGGR_POWER_COLLAPSE;
hba->caps |= UFSHCD_CAP_RPM_AUTOSUSPEND;
@@ -1556,7 +1651,7 @@ static irqreturn_t ufs_qcom_mcq_esi_handler(int irq, void *__hba)
struct ufs_hw_queue *hwq = &hba->uhq[id];
ufshcd_mcq_write_cqis(hba, 0x1, id);
- ufshcd_mcq_poll_cqe_nolock(hba, hwq);
+ ufshcd_mcq_poll_cqe_lock(hba, hwq);
return IRQ_HANDLED;
}
@@ -1723,7 +1818,6 @@ static const struct dev_pm_ops ufs_qcom_pm_ops = {
static struct platform_driver ufs_qcom_pltform = {
.probe = ufs_qcom_probe,
.remove = ufs_qcom_remove,
- .shutdown = ufshcd_pltfrm_shutdown,
.driver = {
.name = "ufshcd-qcom",
.pm = &ufs_qcom_pm_ops,
diff --git a/drivers/ufs/host/ufs-qcom.h b/drivers/ufs/host/ufs-qcom.h
index 39e774254fb2..6289ad5a42d0 100644
--- a/drivers/ufs/host/ufs-qcom.h
+++ b/drivers/ufs/host/ufs-qcom.h
@@ -7,6 +7,7 @@
#include <linux/reset-controller.h>
#include <linux/reset.h>
+#include <soc/qcom/ice.h>
#include <ufs/ufshcd.h>
#define MAX_UFS_QCOM_HOSTS 1
@@ -205,12 +206,13 @@ struct ufs_qcom_host {
struct clk *tx_l1_sync_clk;
bool is_lane_clks_enabled;
+#ifdef CONFIG_SCSI_UFS_CRYPTO
+ struct qcom_ice *ice;
+#endif
+
void __iomem *dev_ref_clk_ctrl_mmio;
bool is_dev_ref_clk_enabled;
struct ufs_hw_version hw_ver;
-#ifdef CONFIG_SCSI_UFS_CRYPTO
- void __iomem *ice_mmio;
-#endif
u32 dev_ref_clk_en_mask;
@@ -248,28 +250,4 @@ static inline bool ufs_qcom_cap_qunipro(struct ufs_qcom_host *host)
return host->caps & UFS_QCOM_CAP_QUNIPRO;
}
-/* ufs-qcom-ice.c */
-
-#ifdef CONFIG_SCSI_UFS_CRYPTO
-int ufs_qcom_ice_init(struct ufs_qcom_host *host);
-int ufs_qcom_ice_enable(struct ufs_qcom_host *host);
-int ufs_qcom_ice_resume(struct ufs_qcom_host *host);
-int ufs_qcom_ice_program_key(struct ufs_hba *hba,
- const union ufs_crypto_cfg_entry *cfg, int slot);
-#else
-static inline int ufs_qcom_ice_init(struct ufs_qcom_host *host)
-{
- return 0;
-}
-static inline int ufs_qcom_ice_enable(struct ufs_qcom_host *host)
-{
- return 0;
-}
-static inline int ufs_qcom_ice_resume(struct ufs_qcom_host *host)
-{
- return 0;
-}
-#define ufs_qcom_ice_program_key NULL
-#endif /* !CONFIG_SCSI_UFS_CRYPTO */
-
#endif /* UFS_QCOM_H_ */
diff --git a/drivers/ufs/host/ufs-sprd.c b/drivers/ufs/host/ufs-sprd.c
index 051f3f40d92c..2bad75dd6d58 100644
--- a/drivers/ufs/host/ufs-sprd.c
+++ b/drivers/ufs/host/ufs-sprd.c
@@ -444,7 +444,6 @@ static const struct dev_pm_ops ufs_sprd_pm_ops = {
static struct platform_driver ufs_sprd_pltform = {
.probe = ufs_sprd_probe,
.remove = ufs_sprd_remove,
- .shutdown = ufshcd_pltfrm_shutdown,
.driver = {
.name = "ufshcd-sprd",
.pm = &ufs_sprd_pm_ops,
diff --git a/drivers/ufs/host/ufshcd-pci.c b/drivers/ufs/host/ufshcd-pci.c
index 9c911787f84c..cf3987773051 100644
--- a/drivers/ufs/host/ufshcd-pci.c
+++ b/drivers/ufs/host/ufshcd-pci.c
@@ -505,15 +505,6 @@ static int ufshcd_pci_restore(struct device *dev)
#endif
/**
- * ufshcd_pci_shutdown - main function to put the controller in reset state
- * @pdev: pointer to PCI device handle
- */
-static void ufshcd_pci_shutdown(struct pci_dev *pdev)
-{
- ufshcd_shutdown((struct ufs_hba *)pci_get_drvdata(pdev));
-}
-
-/**
* ufshcd_pci_remove - de-allocate PCI/SCSI host and host memory space
* data structure memory
* @pdev: pointer to PCI handle
@@ -608,6 +599,7 @@ static const struct pci_device_id ufshcd_pci_tbl[] = {
{ PCI_VDEVICE(INTEL, 0x54FF), (kernel_ulong_t)&ufs_intel_adl_hba_vops },
{ PCI_VDEVICE(INTEL, 0x7E47), (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
{ PCI_VDEVICE(INTEL, 0xA847), (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
+ { PCI_VDEVICE(INTEL, 0x7747), (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
{ } /* terminate list */
};
@@ -618,7 +610,6 @@ static struct pci_driver ufshcd_pci_driver = {
.id_table = ufshcd_pci_tbl,
.probe = ufshcd_pci_probe,
.remove = ufshcd_pci_remove,
- .shutdown = ufshcd_pci_shutdown,
.driver = {
.pm = &ufshcd_pci_pm_ops
},
diff --git a/drivers/ufs/host/ufshcd-pltfrm.c b/drivers/ufs/host/ufshcd-pltfrm.c
index 5739ff007828..0b7430033047 100644
--- a/drivers/ufs/host/ufshcd-pltfrm.c
+++ b/drivers/ufs/host/ufshcd-pltfrm.c
@@ -190,12 +190,6 @@ out:
return err;
}
-void ufshcd_pltfrm_shutdown(struct platform_device *pdev)
-{
- ufshcd_shutdown((struct ufs_hba *)platform_get_drvdata(pdev));
-}
-EXPORT_SYMBOL_GPL(ufshcd_pltfrm_shutdown);
-
static void ufshcd_init_lanes_per_dir(struct ufs_hba *hba)
{
struct device *dev = hba->dev;
diff --git a/drivers/ufs/host/ufshcd-pltfrm.h b/drivers/ufs/host/ufshcd-pltfrm.h
index 2e4ba2bfbcad..2df108f4ac13 100644
--- a/drivers/ufs/host/ufshcd-pltfrm.h
+++ b/drivers/ufs/host/ufshcd-pltfrm.h
@@ -31,7 +31,6 @@ int ufshcd_get_pwr_dev_param(const struct ufs_dev_params *dev_param,
void ufshcd_init_pwr_dev_param(struct ufs_dev_params *dev_param);
int ufshcd_pltfrm_init(struct platform_device *pdev,
const struct ufs_hba_variant_ops *vops);
-void ufshcd_pltfrm_shutdown(struct platform_device *pdev);
int ufshcd_populate_vreg(struct device *dev, const char *name,
struct ufs_vreg **out_vreg);