diff options
Diffstat (limited to 'drivers/scsi/ufs/ufshcd.c')
| -rw-r--r-- | drivers/scsi/ufs/ufshcd.c | 541 |
1 files changed, 290 insertions, 251 deletions
diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c index ad4fc829cbb2..307622284239 100644 --- a/drivers/scsi/ufs/ufshcd.c +++ b/drivers/scsi/ufs/ufshcd.c @@ -1,40 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Universal Flash Storage Host controller driver Core - * - * This code is based on drivers/scsi/ufs/ufshcd.c * Copyright (C) 2011-2013 Samsung India Software Operations * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved. * * Authors: * Santosh Yaraganavi <santosh.sy@samsung.com> * Vinayak Holikatti <h.vinayak@samsung.com> - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * See the COPYING file in the top-level directory or visit - * <http://www.gnu.org/licenses/gpl-2.0.html> - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * This program is provided "AS IS" and "WITH ALL FAULTS" and - * without warranty of any kind. You are solely responsible for - * determining the appropriateness of using and distributing - * the program and assume all risks associated with your exercise - * of rights with respect to the program, including but not limited - * to infringement of third party rights, the risks and costs of - * program errors, damage to or loss of data, programs or equipment, - * and unavailability or interruption of operations. Under no - * circumstances will the contributor of this Program be liable for - * any damages of any kind arising from your use or distribution of - * this program. - * - * The Linux Foundation chooses to take subject only to the GPLv2 - * license terms, and distributes only under these terms. */ #include <linux/async.h> @@ -48,6 +20,7 @@ #include "unipro.h" #include "ufs-sysfs.h" #include "ufs_bsg.h" +#include "ufshcd-crypto.h" #include <asm/unaligned.h> #include <linux/blkdev.h> @@ -216,23 +189,22 @@ ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state, static struct ufs_dev_fix ufs_fixups[] = { /* UFS cards deviations table */ - UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL, + UFS_FIX(UFS_VENDOR_MICRON, UFS_ANY_MODEL, UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM), UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL, + UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM | + UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE | UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS), - UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL, - UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE), + UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL, + UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME), + UFS_FIX(UFS_VENDOR_SKHYNIX, "hB8aL1" /*H28U62301AMR*/, + UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME), UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL, UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM), UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG", UFS_DEVICE_QUIRK_PA_TACTIVATE), UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG", UFS_DEVICE_QUIRK_PA_TACTIVATE), - UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL, - UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME), - UFS_FIX(UFS_VENDOR_SKHYNIX, "hB8aL1" /*H28U62301AMR*/, - UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME), - END_FIX }; @@ -340,6 +312,26 @@ static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag, &descp->input_param1); } +static void ufshcd_add_uic_command_trace(struct ufs_hba *hba, + struct uic_command *ucmd, + const char *str) +{ + u32 cmd; + + if (!trace_ufshcd_uic_command_enabled()) + return; + + if (!strcmp(str, "send")) + cmd = ucmd->command; + else + cmd = ufshcd_readl(hba, REG_UIC_COMMAND); + + trace_ufshcd_uic_command(dev_name(hba->dev), str, cmd, + ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1), + ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2), + ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3)); +} + static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag, const char *str) { @@ -672,7 +664,11 @@ static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp) */ static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos) { - ufshcd_writel(hba, ~(1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR); + if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR) + ufshcd_writel(hba, (1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR); + else + ufshcd_writel(hba, ~(1 << pos), + REG_UTP_TRANSFER_REQ_LIST_CLEAR); } /** @@ -682,7 +678,10 @@ static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos) */ static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos) { - ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR); + if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR) + ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR); + else + ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR); } /** @@ -839,7 +838,12 @@ static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba) */ static inline void ufshcd_hba_start(struct ufs_hba *hba) { - ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE); + u32 val = CONTROLLER_ENABLE; + + if (ufshcd_crypto_enable(hba)) + val |= CRYPTO_GENERAL_ENABLE; + + ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE); } /** @@ -1314,6 +1318,7 @@ static int ufshcd_devfreq_get_dev_status(struct device *dev, unsigned long flags; struct list_head *clk_list = &hba->clk_list_head; struct ufs_clk_info *clki; + ktime_t curr_t; if (!ufshcd_is_clkscaling_supported(hba)) return -EINVAL; @@ -1321,6 +1326,7 @@ static int ufshcd_devfreq_get_dev_status(struct device *dev, memset(stat, 0, sizeof(*stat)); spin_lock_irqsave(hba->host->host_lock, flags); + curr_t = ktime_get(); if (!scaling->window_start_t) goto start_window; @@ -1332,18 +1338,17 @@ static int ufshcd_devfreq_get_dev_status(struct device *dev, */ stat->current_frequency = clki->curr_freq; if (scaling->is_busy_started) - scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(), - scaling->busy_start_t)); + scaling->tot_busy_t += ktime_us_delta(curr_t, + scaling->busy_start_t); - stat->total_time = jiffies_to_usecs((long)jiffies - - (long)scaling->window_start_t); + stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t); stat->busy_time = scaling->tot_busy_t; start_window: - scaling->window_start_t = jiffies; + scaling->window_start_t = curr_t; scaling->tot_busy_t = 0; if (hba->outstanding_reqs) { - scaling->busy_start_t = ktime_get(); + scaling->busy_start_t = curr_t; scaling->is_busy_started = true; } else { scaling->busy_start_t = 0; @@ -1877,6 +1882,7 @@ static void ufshcd_exit_clk_gating(struct ufs_hba *hba) static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba) { bool queue_resume_work = false; + ktime_t curr_t = ktime_get(); if (!ufshcd_is_clkscaling_supported(hba)) return; @@ -1892,13 +1898,13 @@ static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba) &hba->clk_scaling.resume_work); if (!hba->clk_scaling.window_start_t) { - hba->clk_scaling.window_start_t = jiffies; + hba->clk_scaling.window_start_t = curr_t; hba->clk_scaling.tot_busy_t = 0; hba->clk_scaling.is_busy_started = false; } if (!hba->clk_scaling.is_busy_started) { - hba->clk_scaling.busy_start_t = ktime_get(); + hba->clk_scaling.busy_start_t = curr_t; hba->clk_scaling.is_busy_started = true; } } @@ -1925,8 +1931,11 @@ static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba) static inline void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag) { - hba->lrb[task_tag].issue_time_stamp = ktime_get(); - hba->lrb[task_tag].compl_time_stamp = ktime_set(0, 0); + struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; + + lrbp->issue_time_stamp = ktime_get(); + lrbp->compl_time_stamp = ktime_set(0, 0); + ufshcd_vops_setup_xfer_req(hba, task_tag, (lrbp->cmd ? true : false)); ufshcd_add_command_trace(hba, task_tag, "send"); ufshcd_clk_scaling_start_busy(hba); __set_bit(task_tag, &hba->outstanding_reqs); @@ -1996,15 +2005,26 @@ int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) /** * ufshcd_hba_capabilities - Read controller capabilities * @hba: per adapter instance + * + * Return: 0 on success, negative on error. */ -static inline void ufshcd_hba_capabilities(struct ufs_hba *hba) +static inline int ufshcd_hba_capabilities(struct ufs_hba *hba) { + int err; + hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES); /* nutrs and nutmrs are 0 based values */ hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1; hba->nutmrs = ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1; + + /* Read crypto capabilities */ + err = ufshcd_hba_init_crypto_capabilities(hba); + if (err) + dev_err(hba->dev, "crypto setup failed\n"); + + return err; } /** @@ -2052,6 +2072,8 @@ ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2); ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3); + ufshcd_add_uic_command_trace(hba, uic_cmd, "send"); + /* Write UIC Cmd */ ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK, REG_UIC_COMMAND); @@ -2161,8 +2183,14 @@ static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) return sg_segments; if (sg_segments) { - lrbp->utr_descriptor_ptr->prd_table_length = - cpu_to_le16((u16)sg_segments); + + if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) + lrbp->utr_descriptor_ptr->prd_table_length = + cpu_to_le16((sg_segments * + sizeof(struct ufshcd_sg_entry))); + else + lrbp->utr_descriptor_ptr->prd_table_length = + cpu_to_le16((u16) (sg_segments)); prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr; @@ -2232,11 +2260,13 @@ static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs) * @cmd_dir: requests data direction */ static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, - u32 *upiu_flags, enum dma_data_direction cmd_dir) + u8 *upiu_flags, enum dma_data_direction cmd_dir) { struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr; u32 data_direction; u32 dword_0; + u32 dword_1 = 0; + u32 dword_3 = 0; if (cmd_dir == DMA_FROM_DEVICE) { data_direction = UTP_DEVICE_TO_HOST; @@ -2254,10 +2284,12 @@ static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, if (lrbp->intr_cmd) dword_0 |= UTP_REQ_DESC_INT_CMD; + /* Prepare crypto related dwords */ + ufshcd_prepare_req_desc_hdr_crypto(lrbp, &dword_0, &dword_1, &dword_3); + /* Transfer request descriptor header fields */ req_desc->header.dword_0 = cpu_to_le32(dword_0); - /* dword_1 is reserved, hence it is set to 0 */ - req_desc->header.dword_1 = 0; + req_desc->header.dword_1 = cpu_to_le32(dword_1); /* * assigning invalid value for command status. Controller * updates OCS on command completion, with the command @@ -2265,8 +2297,7 @@ static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, */ req_desc->header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS); - /* dword_3 is reserved, hence it is set to 0 */ - req_desc->header.dword_3 = 0; + req_desc->header.dword_3 = cpu_to_le32(dword_3); req_desc->prd_table_length = 0; } @@ -2278,7 +2309,7 @@ static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, * @upiu_flags: flags */ static -void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags) +void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags) { struct scsi_cmnd *cmd = lrbp->cmd; struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; @@ -2311,7 +2342,7 @@ void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags) * @upiu_flags: flags */ static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp, u32 upiu_flags) + struct ufshcd_lrb *lrbp, u8 upiu_flags) { struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; struct ufs_query *query = &hba->dev_cmd.query; @@ -2367,7 +2398,7 @@ static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp) */ static int ufshcd_comp_devman_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { - u32 upiu_flags; + u8 upiu_flags; int ret = 0; if ((hba->ufs_version == UFSHCI_VERSION_10) || @@ -2395,7 +2426,7 @@ static int ufshcd_comp_devman_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) */ static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { - u32 upiu_flags; + u8 upiu_flags; int ret = 0; if ((hba->ufs_version == UFSHCI_VERSION_10) || @@ -2521,6 +2552,9 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) lrbp->task_tag = tag; lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun); lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false; + + ufshcd_prepare_lrbp_crypto(cmd->request, lrbp); + lrbp->req_abort_skip = false; ufshcd_comp_scsi_upiu(hba, lrbp); @@ -2536,7 +2570,6 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) /* issue command to the controller */ spin_lock_irqsave(hba->host->host_lock, flags); - ufshcd_vops_setup_xfer_req(hba, tag, true); ufshcd_send_command(hba, tag); out_unlock: spin_unlock_irqrestore(hba->host->host_lock, flags); @@ -2554,6 +2587,7 @@ static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, lrbp->task_tag = tag; lrbp->lun = 0; /* device management cmd is not specific to any LUN */ lrbp->intr_cmd = true; /* No interrupt aggregation */ + ufshcd_prepare_lrbp_crypto(NULL, lrbp); hba->dev_cmd.type = cmd_type; return ufshcd_comp_devman_upiu(hba, lrbp); @@ -2723,7 +2757,6 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, /* Make sure descriptors are ready before ringing the doorbell */ wmb(); spin_lock_irqsave(hba->host->host_lock, flags); - ufshcd_vops_setup_xfer_req(hba, tag, false); ufshcd_send_command(hba, tag); spin_unlock_irqrestore(hba->host->host_lock, flags); @@ -3053,94 +3086,36 @@ int ufshcd_query_descriptor_retry(struct ufs_hba *hba, } /** - * ufshcd_read_desc_length - read the specified descriptor length from header - * @hba: Pointer to adapter instance - * @desc_id: descriptor idn value - * @desc_index: descriptor index - * @desc_length: pointer to variable to read the length of descriptor - * - * Return 0 in case of success, non-zero otherwise - */ -static int ufshcd_read_desc_length(struct ufs_hba *hba, - enum desc_idn desc_id, - int desc_index, - int *desc_length) -{ - int ret; - u8 header[QUERY_DESC_HDR_SIZE]; - int header_len = QUERY_DESC_HDR_SIZE; - - if (desc_id >= QUERY_DESC_IDN_MAX) - return -EINVAL; - - ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC, - desc_id, desc_index, 0, header, - &header_len); - - if (ret) { - dev_err(hba->dev, "%s: Failed to get descriptor header id %d", - __func__, desc_id); - return ret; - } else if (desc_id != header[QUERY_DESC_DESC_TYPE_OFFSET]) { - dev_warn(hba->dev, "%s: descriptor header id %d and desc_id %d mismatch", - __func__, header[QUERY_DESC_DESC_TYPE_OFFSET], - desc_id); - ret = -EINVAL; - } - - *desc_length = header[QUERY_DESC_LENGTH_OFFSET]; - return ret; - -} - -/** * ufshcd_map_desc_id_to_length - map descriptor IDN to its length * @hba: Pointer to adapter instance * @desc_id: descriptor idn value * @desc_len: mapped desc length (out) - * - * Return 0 in case of success, non-zero otherwise */ -int ufshcd_map_desc_id_to_length(struct ufs_hba *hba, - enum desc_idn desc_id, int *desc_len) +void ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id, + int *desc_len) { - switch (desc_id) { - case QUERY_DESC_IDN_DEVICE: - *desc_len = hba->desc_size.dev_desc; - break; - case QUERY_DESC_IDN_POWER: - *desc_len = hba->desc_size.pwr_desc; - break; - case QUERY_DESC_IDN_GEOMETRY: - *desc_len = hba->desc_size.geom_desc; - break; - case QUERY_DESC_IDN_CONFIGURATION: - *desc_len = hba->desc_size.conf_desc; - break; - case QUERY_DESC_IDN_UNIT: - *desc_len = hba->desc_size.unit_desc; - break; - case QUERY_DESC_IDN_INTERCONNECT: - *desc_len = hba->desc_size.interc_desc; - break; - case QUERY_DESC_IDN_STRING: - *desc_len = QUERY_DESC_MAX_SIZE; - break; - case QUERY_DESC_IDN_HEALTH: - *desc_len = hba->desc_size.hlth_desc; - break; - case QUERY_DESC_IDN_RFU_0: - case QUERY_DESC_IDN_RFU_1: - *desc_len = 0; - break; - default: + if (desc_id >= QUERY_DESC_IDN_MAX || desc_id == QUERY_DESC_IDN_RFU_0 || + desc_id == QUERY_DESC_IDN_RFU_1) *desc_len = 0; - return -EINVAL; - } - return 0; + else + *desc_len = hba->desc_size[desc_id]; } EXPORT_SYMBOL(ufshcd_map_desc_id_to_length); +static void ufshcd_update_desc_length(struct ufs_hba *hba, + enum desc_idn desc_id, int desc_index, + unsigned char desc_len) +{ + if (hba->desc_size[desc_id] == QUERY_DESC_MAX_SIZE && + desc_id != QUERY_DESC_IDN_STRING && desc_index != UFS_RPMB_UNIT) + /* For UFS 3.1, the normal unit descriptor is 10 bytes larger + * than the RPMB unit, however, both descriptors share the same + * desc_idn, to cover both unit descriptors with one length, we + * choose the normal unit descriptor length by desc_index. + */ + hba->desc_size[desc_id] = desc_len; +} + /** * ufshcd_read_desc_param - read the specified descriptor parameter * @hba: Pointer to adapter instance @@ -3168,16 +3143,11 @@ int ufshcd_read_desc_param(struct ufs_hba *hba, if (desc_id >= QUERY_DESC_IDN_MAX || !param_size) return -EINVAL; - /* Get the max length of descriptor from structure filled up at probe - * time. - */ - ret = ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len); - - /* Sanity checks */ - if (ret || !buff_len) { - dev_err(hba->dev, "%s: Failed to get full descriptor length", - __func__); - return ret; + /* Get the length of descriptor */ + ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len); + if (!buff_len) { + dev_err(hba->dev, "%s: Failed to get desc length", __func__); + return -EINVAL; } /* Check whether we need temp memory */ @@ -3209,9 +3179,13 @@ int ufshcd_read_desc_param(struct ufs_hba *hba, goto out; } + /* Update descriptor length */ + buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET]; + ufshcd_update_desc_length(hba, desc_id, desc_index, buff_len); + /* Check wherher we will not copy more data, than available */ - if (is_kmalloc && param_size > buff_len) - param_size = buff_len; + if (is_kmalloc && (param_offset + param_size) > buff_len) + param_size = buff_len - param_offset; if (is_kmalloc) memcpy(param_read_buf, &desc_buf[param_offset], param_size); @@ -3221,16 +3195,6 @@ out: return ret; } -static inline int ufshcd_read_desc(struct ufs_hba *hba, - enum desc_idn desc_id, - int desc_index, - void *buf, - u32 size) -{ - return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size); -} - - /** * struct uc_string_id - unicode string * @@ -3278,9 +3242,8 @@ int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, if (!uc_str) return -ENOMEM; - ret = ufshcd_read_desc(hba, QUERY_DESC_IDN_STRING, - desc_index, uc_str, - QUERY_DESC_MAX_SIZE); + ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0, + (u8 *)uc_str, QUERY_DESC_MAX_SIZE); if (ret < 0) { dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n", QUERY_REQ_RETRIES, ret); @@ -3511,11 +3474,21 @@ static void ufshcd_host_memory_configure(struct ufs_hba *hba) cpu_to_le32(upper_32_bits(cmd_desc_element_addr)); /* Response upiu and prdt offset should be in double words */ - utrdlp[i].response_upiu_offset = - cpu_to_le16(response_offset >> 2); - utrdlp[i].prd_table_offset = cpu_to_le16(prdt_offset >> 2); - utrdlp[i].response_upiu_length = - cpu_to_le16(ALIGNED_UPIU_SIZE >> 2); + if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) { + utrdlp[i].response_upiu_offset = + cpu_to_le16(response_offset); + utrdlp[i].prd_table_offset = + cpu_to_le16(prdt_offset); + utrdlp[i].response_upiu_length = + cpu_to_le16(ALIGNED_UPIU_SIZE); + } else { + utrdlp[i].response_upiu_offset = + cpu_to_le16(response_offset >> 2); + utrdlp[i].prd_table_offset = + cpu_to_le16(prdt_offset >> 2); + utrdlp[i].response_upiu_length = + cpu_to_le16(ALIGNED_UPIU_SIZE >> 2); + } ufshcd_init_lrb(hba, &hba->lrb[i], i); } @@ -3545,6 +3518,52 @@ static int ufshcd_dme_link_startup(struct ufs_hba *hba) "dme-link-startup: error code %d\n", ret); return ret; } +/** + * ufshcd_dme_reset - UIC command for DME_RESET + * @hba: per adapter instance + * + * DME_RESET command is issued in order to reset UniPro stack. + * This function now deals with cold reset. + * + * Returns 0 on success, non-zero value on failure + */ +static int ufshcd_dme_reset(struct ufs_hba *hba) +{ + struct uic_command uic_cmd = {0}; + int ret; + + uic_cmd.command = UIC_CMD_DME_RESET; + + ret = ufshcd_send_uic_cmd(hba, &uic_cmd); + if (ret) + dev_err(hba->dev, + "dme-reset: error code %d\n", ret); + + return ret; +} + +/** + * ufshcd_dme_enable - UIC command for DME_ENABLE + * @hba: per adapter instance + * + * DME_ENABLE command is issued in order to enable UniPro stack. + * + * Returns 0 on success, non-zero value on failure + */ +static int ufshcd_dme_enable(struct ufs_hba *hba) +{ + struct uic_command uic_cmd = {0}; + int ret; + + uic_cmd.command = UIC_CMD_DME_ENABLE; + + ret = ufshcd_send_uic_cmd(hba, &uic_cmd); + if (ret) + dev_err(hba->dev, + "dme-reset: error code %d\n", ret); + + return ret; +} static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba) { @@ -4269,7 +4288,7 @@ static inline void ufshcd_hba_stop(struct ufs_hba *hba) } /** - * ufshcd_hba_enable - initialize the controller + * ufshcd_hba_execute_hce - initialize the controller * @hba: per adapter instance * * The controller resets itself and controller firmware initialization @@ -4278,7 +4297,7 @@ static inline void ufshcd_hba_stop(struct ufs_hba *hba) * * Returns 0 on success, non-zero value on failure */ -int ufshcd_hba_enable(struct ufs_hba *hba) +static int ufshcd_hba_execute_hce(struct ufs_hba *hba) { int retry; @@ -4326,6 +4345,32 @@ int ufshcd_hba_enable(struct ufs_hba *hba) return 0; } + +int ufshcd_hba_enable(struct ufs_hba *hba) +{ + int ret; + + if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) { + ufshcd_set_link_off(hba); + ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE); + + /* enable UIC related interrupts */ + ufshcd_enable_intr(hba, UFSHCD_UIC_MASK); + ret = ufshcd_dme_reset(hba); + if (!ret) { + ret = ufshcd_dme_enable(hba); + if (!ret) + ufshcd_vops_hce_enable_notify(hba, POST_CHANGE); + if (ret) + dev_err(hba->dev, + "Host controller enable failed with non-hce\n"); + } + } else { + ret = ufshcd_hba_execute_hce(hba); + } + + return ret; +} EXPORT_SYMBOL_GPL(ufshcd_hba_enable); static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer) @@ -4650,6 +4695,8 @@ static int ufshcd_slave_configure(struct scsi_device *sdev) if (ufshcd_is_rpm_autosuspend_allowed(hba)) sdev->rpm_autosuspend = 1; + ufshcd_crypto_setup_rq_keyslot_manager(hba, q); + return 0; } @@ -4724,6 +4771,12 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) /* overall command status of utrd */ ocs = ufshcd_get_tr_ocs(lrbp); + if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) { + if (be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_1) & + MASK_RSP_UPIU_RESULT) + ocs = OCS_SUCCESS; + } + switch (ocs) { case OCS_SUCCESS: result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); @@ -4792,6 +4845,9 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) case OCS_MISMATCH_RESP_UPIU_SIZE: case OCS_PEER_COMM_FAILURE: case OCS_FATAL_ERROR: + case OCS_DEVICE_FATAL_ERROR: + case OCS_INVALID_CRYPTO_CONFIG: + case OCS_GENERAL_CRYPTO_ERROR: default: result |= DID_ERROR << 16; dev_err(hba->dev, @@ -4833,6 +4889,10 @@ static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status) complete(hba->uic_async_done); retval = IRQ_HANDLED; } + + if (retval == IRQ_HANDLED) + ufshcd_add_uic_command_trace(hba, hba->active_uic_cmd, + "complete"); return retval; } @@ -4851,6 +4911,7 @@ static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, for_each_set_bit(index, &completed_reqs, hba->nutrs) { lrbp = &hba->lrb[index]; + lrbp->compl_time_stamp = ktime_get(); cmd = lrbp->cmd; if (cmd) { ufshcd_add_command_trace(hba, index, "complete"); @@ -4859,13 +4920,11 @@ static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, cmd->result = result; /* Mark completed command as NULL in LRB */ lrbp->cmd = NULL; - lrbp->compl_time_stamp = ktime_get(); /* Do not touch lrbp after scsi done */ cmd->scsi_done(cmd); __ufshcd_release(hba); } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE || lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) { - lrbp->compl_time_stamp = ktime_get(); if (hba->dev_cmd.complete) { ufshcd_add_command_trace(hba, index, "dev_complete"); @@ -4902,7 +4961,8 @@ static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba) * false interrupt if device completes another request after resetting * aggregation and before reading the DB. */ - if (ufshcd_is_intr_aggr_allowed(hba)) + if (ufshcd_is_intr_aggr_allowed(hba) && + !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR)) ufshcd_reset_intr_aggr(hba); tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); @@ -6090,7 +6150,7 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, int tag; struct completion wait; unsigned long flags; - u32 upiu_flags; + u8 upiu_flags; down_read(&hba->clk_scaling_lock); @@ -6112,6 +6172,7 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, lrbp->task_tag = tag; lrbp->lun = 0; lrbp->intr_cmd = true; + ufshcd_prepare_lrbp_crypto(NULL, lrbp); hba->dev_cmd.type = cmd_type; switch (hba->ufs_version) { @@ -6703,7 +6764,7 @@ out: static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba) { int ret; - int buff_len = hba->desc_size.pwr_desc; + int buff_len = hba->desc_size[QUERY_DESC_IDN_POWER]; u8 *desc_buf; u32 icc_level; @@ -6711,8 +6772,8 @@ static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba) if (!desc_buf) return; - ret = ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0, - desc_buf, buff_len); + ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0, + desc_buf, buff_len); if (ret) { dev_err(hba->dev, "%s: Failed reading power descriptor.len = %d ret = %d", @@ -6815,20 +6876,31 @@ out: static void ufshcd_wb_probe(struct ufs_hba *hba, u8 *desc_buf) { + struct ufs_dev_info *dev_info = &hba->dev_info; u8 lun; u32 d_lu_wb_buf_alloc; if (!ufshcd_is_wb_allowed(hba)) return; + /* + * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or + * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES + * enabled + */ + if (!(dev_info->wspecversion >= 0x310 || + dev_info->wspecversion == 0x220 || + (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES))) + goto wb_disabled; - if (hba->desc_size.dev_desc < DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP + 4) + if (hba->desc_size[QUERY_DESC_IDN_DEVICE] < + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP + 4) goto wb_disabled; - hba->dev_info.d_ext_ufs_feature_sup = + dev_info->d_ext_ufs_feature_sup = get_unaligned_be32(desc_buf + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); - if (!(hba->dev_info.d_ext_ufs_feature_sup & UFS_DEV_WRITE_BOOSTER_SUP)) + if (!(dev_info->d_ext_ufs_feature_sup & UFS_DEV_WRITE_BOOSTER_SUP)) goto wb_disabled; /* @@ -6837,17 +6909,17 @@ static void ufshcd_wb_probe(struct ufs_hba *hba, u8 *desc_buf) * a max of 1 lun would have wb buffer configured. * Now only shared buffer mode is supported. */ - hba->dev_info.b_wb_buffer_type = + dev_info->b_wb_buffer_type = desc_buf[DEVICE_DESC_PARAM_WB_TYPE]; - hba->dev_info.b_presrv_uspc_en = + dev_info->b_presrv_uspc_en = desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN]; - if (hba->dev_info.b_wb_buffer_type == WB_BUF_MODE_SHARED) { - hba->dev_info.d_wb_alloc_units = + if (dev_info->b_wb_buffer_type == WB_BUF_MODE_SHARED) { + dev_info->d_wb_alloc_units = get_unaligned_be32(desc_buf + DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS); - if (!hba->dev_info.d_wb_alloc_units) + if (!dev_info->d_wb_alloc_units) goto wb_disabled; } else { for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) { @@ -6858,7 +6930,7 @@ static void ufshcd_wb_probe(struct ufs_hba *hba, u8 *desc_buf) (u8 *)&d_lu_wb_buf_alloc, sizeof(d_lu_wb_buf_alloc)); if (d_lu_wb_buf_alloc) { - hba->dev_info.wb_dedicated_lu = lun; + dev_info->wb_dedicated_lu = lun; break; } } @@ -6903,21 +6975,18 @@ static void ufs_fixup_device_setup(struct ufs_hba *hba) static int ufs_get_device_desc(struct ufs_hba *hba) { int err; - size_t buff_len; u8 model_index; u8 *desc_buf; struct ufs_dev_info *dev_info = &hba->dev_info; - buff_len = max_t(size_t, hba->desc_size.dev_desc, - QUERY_DESC_MAX_SIZE + 1); - desc_buf = kmalloc(buff_len, GFP_KERNEL); + desc_buf = kmalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); if (!desc_buf) { err = -ENOMEM; goto out; } - err = ufshcd_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, desc_buf, - hba->desc_size.dev_desc); + err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf, + hba->desc_size[QUERY_DESC_IDN_DEVICE]); if (err) { dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n", __func__, err); @@ -6947,14 +7016,7 @@ static int ufs_get_device_desc(struct ufs_hba *hba) ufs_fixup_device_setup(hba); - /* - * Probe WB only for UFS-3.1 devices or UFS devices with quirk - * UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES enabled - */ - if (dev_info->wspecversion >= 0x310 || - dev_info->wspecversion == 0x220 || - (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES)) - ufshcd_wb_probe(hba, desc_buf); + ufshcd_wb_probe(hba, desc_buf); /* * ufshcd_read_string_desc returns size of the string @@ -7146,61 +7208,21 @@ static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba) hba->req_abort_count = 0; } -static void ufshcd_init_desc_sizes(struct ufs_hba *hba) -{ - int err; - - err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_DEVICE, 0, - &hba->desc_size.dev_desc); - if (err) - hba->desc_size.dev_desc = QUERY_DESC_DEVICE_DEF_SIZE; - - err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_POWER, 0, - &hba->desc_size.pwr_desc); - if (err) - hba->desc_size.pwr_desc = QUERY_DESC_POWER_DEF_SIZE; - - err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_INTERCONNECT, 0, - &hba->desc_size.interc_desc); - if (err) - hba->desc_size.interc_desc = QUERY_DESC_INTERCONNECT_DEF_SIZE; - - err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_CONFIGURATION, 0, - &hba->desc_size.conf_desc); - if (err) - hba->desc_size.conf_desc = QUERY_DESC_CONFIGURATION_DEF_SIZE; - - err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_UNIT, 0, - &hba->desc_size.unit_desc); - if (err) - hba->desc_size.unit_desc = QUERY_DESC_UNIT_DEF_SIZE; - - err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_GEOMETRY, 0, - &hba->desc_size.geom_desc); - if (err) - hba->desc_size.geom_desc = QUERY_DESC_GEOMETRY_DEF_SIZE; - - err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_HEALTH, 0, - &hba->desc_size.hlth_desc); - if (err) - hba->desc_size.hlth_desc = QUERY_DESC_HEALTH_DEF_SIZE; -} - static int ufshcd_device_geo_params_init(struct ufs_hba *hba) { int err; size_t buff_len; u8 *desc_buf; - buff_len = hba->desc_size.geom_desc; + buff_len = hba->desc_size[QUERY_DESC_IDN_GEOMETRY]; desc_buf = kmalloc(buff_len, GFP_KERNEL); if (!desc_buf) { err = -ENOMEM; goto out; } - err = ufshcd_read_desc(hba, QUERY_DESC_IDN_GEOMETRY, 0, - desc_buf, buff_len); + err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0, + desc_buf, buff_len); if (err) { dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n", __func__, err); @@ -7288,10 +7310,11 @@ out: static int ufshcd_device_params_init(struct ufs_hba *hba) { bool flag; - int ret; + int ret, i; - /* Init check for device descriptor sizes */ - ufshcd_init_desc_sizes(hba); + /* Init device descriptor sizes */ + for (i = 0; i < QUERY_DESC_IDN_MAX; i++) + hba->desc_size[i] = QUERY_DESC_MAX_SIZE; /* Init UFS geometry descriptor related parameters */ ret = ufshcd_device_geo_params_init(hba); @@ -8084,6 +8107,8 @@ out: static void ufshcd_vreg_set_lpm(struct ufs_hba *hba) { + bool vcc_off = false; + /* * It seems some UFS devices may keep drawing more than sleep current * (atleast for 500us) from UFS rails (especially from VCCQ rail). @@ -8112,13 +8137,22 @@ static void ufshcd_vreg_set_lpm(struct ufs_hba *hba) if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) && !hba->dev_info.is_lu_power_on_wp) { ufshcd_setup_vreg(hba, false); + vcc_off = true; } else if (!ufshcd_is_ufs_dev_active(hba)) { ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false); + vcc_off = true; if (!ufshcd_is_link_active(hba)) { ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq); ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2); } } + + /* + * Some UFS devices require delay after VCC power rail is turned-off. + */ + if (vcc_off && hba->vreg_info.vcc && + hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM) + usleep_range(5000, 5100); } static int ufshcd_vreg_set_hpm(struct ufs_hba *hba) @@ -8659,6 +8693,7 @@ EXPORT_SYMBOL_GPL(ufshcd_remove); */ void ufshcd_dealloc_host(struct ufs_hba *hba) { + ufshcd_crypto_destroy_keyslot_manager(hba); scsi_host_put(hba->host); } EXPORT_SYMBOL_GPL(ufshcd_dealloc_host); @@ -8759,7 +8794,9 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) goto out_error; /* Read capabilities registers */ - ufshcd_hba_capabilities(hba); + err = ufshcd_hba_capabilities(hba); + if (err) + goto out_disable; /* Get UFS version supported by the controller */ hba->ufs_version = ufshcd_get_ufs_version(hba); @@ -8869,6 +8906,8 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) /* Reset the attached device */ ufshcd_vops_device_reset(hba); + ufshcd_init_crypto(hba); + /* Host controller enable */ err = ufshcd_hba_enable(hba); if (err) { |