diff options
Diffstat (limited to 'drivers/scsi/ufs')
| -rw-r--r-- | drivers/scsi/ufs/Kconfig | 2 | ||||
| -rw-r--r-- | drivers/scsi/ufs/cdns-pltfrm.c | 3 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufs-exynos.c | 15 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufs-exynos.h | 13 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufs-hisi.c | 13 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufs-hisi.h | 13 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufs-mediatek-trace.h | 36 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufs-mediatek.c | 286 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufs-mediatek.h | 35 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufs-qcom.c | 40 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufs-qcom.h | 11 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufs-sysfs.c | 7 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufs.h | 3 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufshcd-dwc.c | 11 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufshcd-pci.c | 73 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufshcd-pltfrm.c | 38 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufshcd-pltfrm.h | 1 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufshcd.c | 541 | ||||
| -rw-r--r-- | drivers/scsi/ufs/ufshcd.h | 153 | ||||
| -rw-r--r-- | drivers/scsi/ufs/unipro.h | 6 |
20 files changed, 846 insertions, 454 deletions
diff --git a/drivers/scsi/ufs/Kconfig b/drivers/scsi/ufs/Kconfig index dcdb4eb1f90b..3f6dfed4fe84 100644 --- a/drivers/scsi/ufs/Kconfig +++ b/drivers/scsi/ufs/Kconfig @@ -99,7 +99,7 @@ config SCSI_UFS_DWC_TC_PLATFORM config SCSI_UFS_QCOM tristate "QCOM specific hooks to UFS controller platform driver" depends on SCSI_UFSHCD_PLATFORM && ARCH_QCOM - select QCOM_SCM + select QCOM_SCM if SCSI_UFS_CRYPTO select RESET_CONTROLLER help This selects the QCOM specific additions to UFSHCD platform driver. diff --git a/drivers/scsi/ufs/cdns-pltfrm.c b/drivers/scsi/ufs/cdns-pltfrm.c index da065a259f6e..149391faa19c 100644 --- a/drivers/scsi/ufs/cdns-pltfrm.c +++ b/drivers/scsi/ufs/cdns-pltfrm.c @@ -221,8 +221,7 @@ static int cdns_ufs_init(struct ufs_hba *hba) return -ENOMEM; ufshcd_set_variant(hba, host); - if (hba->vops && hba->vops->phy_initialization) - status = hba->vops->phy_initialization(hba); + status = ufshcd_vops_phy_initialization(hba); return status; } diff --git a/drivers/scsi/ufs/ufs-exynos.c b/drivers/scsi/ufs/ufs-exynos.c index 5e6b95dbb578..a8770ff14588 100644 --- a/drivers/scsi/ufs/ufs-exynos.c +++ b/drivers/scsi/ufs/ufs-exynos.c @@ -617,20 +617,7 @@ static int exynos_ufs_pre_pwr_mode(struct ufs_hba *hba, goto out; } - - ufs_exynos_cap.tx_lanes = UFS_EXYNOS_LIMIT_NUM_LANES_TX; - ufs_exynos_cap.rx_lanes = UFS_EXYNOS_LIMIT_NUM_LANES_RX; - ufs_exynos_cap.hs_rx_gear = UFS_EXYNOS_LIMIT_HSGEAR_RX; - ufs_exynos_cap.hs_tx_gear = UFS_EXYNOS_LIMIT_HSGEAR_TX; - ufs_exynos_cap.pwm_rx_gear = UFS_EXYNOS_LIMIT_PWMGEAR_RX; - ufs_exynos_cap.pwm_tx_gear = UFS_EXYNOS_LIMIT_PWMGEAR_TX; - ufs_exynos_cap.rx_pwr_pwm = UFS_EXYNOS_LIMIT_RX_PWR_PWM; - ufs_exynos_cap.tx_pwr_pwm = UFS_EXYNOS_LIMIT_TX_PWR_PWM; - ufs_exynos_cap.rx_pwr_hs = UFS_EXYNOS_LIMIT_RX_PWR_HS; - ufs_exynos_cap.tx_pwr_hs = UFS_EXYNOS_LIMIT_TX_PWR_HS; - ufs_exynos_cap.hs_rate = UFS_EXYNOS_LIMIT_HS_RATE; - ufs_exynos_cap.desired_working_mode = - UFS_EXYNOS_LIMIT_DESIRED_MODE; + ufshcd_init_pwr_dev_param(&ufs_exynos_cap); ret = ufshcd_get_pwr_dev_param(&ufs_exynos_cap, dev_max_params, dev_req_params); diff --git a/drivers/scsi/ufs/ufs-exynos.h b/drivers/scsi/ufs/ufs-exynos.h index 76d6e39efb2f..06ee565f7eb0 100644 --- a/drivers/scsi/ufs/ufs-exynos.h +++ b/drivers/scsi/ufs/ufs-exynos.h @@ -90,19 +90,6 @@ struct exynos_ufs; #define SLOW 1 #define FAST 2 -#define UFS_EXYNOS_LIMIT_NUM_LANES_RX 2 -#define UFS_EXYNOS_LIMIT_NUM_LANES_TX 2 -#define UFS_EXYNOS_LIMIT_HSGEAR_RX UFS_HS_G3 -#define UFS_EXYNOS_LIMIT_HSGEAR_TX UFS_HS_G3 -#define UFS_EXYNOS_LIMIT_PWMGEAR_RX UFS_PWM_G4 -#define UFS_EXYNOS_LIMIT_PWMGEAR_TX UFS_PWM_G4 -#define UFS_EXYNOS_LIMIT_RX_PWR_PWM SLOW_MODE -#define UFS_EXYNOS_LIMIT_TX_PWR_PWM SLOW_MODE -#define UFS_EXYNOS_LIMIT_RX_PWR_HS FAST_MODE -#define UFS_EXYNOS_LIMIT_TX_PWR_HS FAST_MODE -#define UFS_EXYNOS_LIMIT_HS_RATE PA_HS_MODE_B -#define UFS_EXYNOS_LIMIT_DESIRED_MODE FAST - #define RX_ADV_FINE_GRAN_SUP_EN 0x1 #define RX_ADV_FINE_GRAN_STEP_VAL 0x3 #define RX_ADV_MIN_ACTV_TIME_CAP 0x9 diff --git a/drivers/scsi/ufs/ufs-hisi.c b/drivers/scsi/ufs/ufs-hisi.c index 074a6a055a4c..0aa58131e791 100644 --- a/drivers/scsi/ufs/ufs-hisi.c +++ b/drivers/scsi/ufs/ufs-hisi.c @@ -293,18 +293,7 @@ static int ufs_hisi_link_startup_notify(struct ufs_hba *hba, static void ufs_hisi_set_dev_cap(struct ufs_dev_params *hisi_param) { - hisi_param->rx_lanes = UFS_HISI_LIMIT_NUM_LANES_RX; - hisi_param->tx_lanes = UFS_HISI_LIMIT_NUM_LANES_TX; - hisi_param->hs_rx_gear = UFS_HISI_LIMIT_HSGEAR_RX; - hisi_param->hs_tx_gear = UFS_HISI_LIMIT_HSGEAR_TX; - hisi_param->pwm_rx_gear = UFS_HISI_LIMIT_PWMGEAR_RX; - hisi_param->pwm_tx_gear = UFS_HISI_LIMIT_PWMGEAR_TX; - hisi_param->rx_pwr_pwm = UFS_HISI_LIMIT_RX_PWR_PWM; - hisi_param->tx_pwr_pwm = UFS_HISI_LIMIT_TX_PWR_PWM; - hisi_param->rx_pwr_hs = UFS_HISI_LIMIT_RX_PWR_HS; - hisi_param->tx_pwr_hs = UFS_HISI_LIMIT_TX_PWR_HS; - hisi_param->hs_rate = UFS_HISI_LIMIT_HS_RATE; - hisi_param->desired_working_mode = UFS_HISI_LIMIT_DESIRED_MODE; + ufshcd_init_pwr_dev_param(hisi_param); } static void ufs_hisi_pwr_change_pre_change(struct ufs_hba *hba) diff --git a/drivers/scsi/ufs/ufs-hisi.h b/drivers/scsi/ufs/ufs-hisi.h index 3231d3d81c98..5a90c0f4e90c 100644 --- a/drivers/scsi/ufs/ufs-hisi.h +++ b/drivers/scsi/ufs/ufs-hisi.h @@ -76,19 +76,6 @@ enum { #define SLOW 1 #define FAST 2 -#define UFS_HISI_LIMIT_NUM_LANES_RX 2 -#define UFS_HISI_LIMIT_NUM_LANES_TX 2 -#define UFS_HISI_LIMIT_HSGEAR_RX UFS_HS_G3 -#define UFS_HISI_LIMIT_HSGEAR_TX UFS_HS_G3 -#define UFS_HISI_LIMIT_PWMGEAR_RX UFS_PWM_G4 -#define UFS_HISI_LIMIT_PWMGEAR_TX UFS_PWM_G4 -#define UFS_HISI_LIMIT_RX_PWR_PWM SLOW_MODE -#define UFS_HISI_LIMIT_TX_PWR_PWM SLOW_MODE -#define UFS_HISI_LIMIT_RX_PWR_HS FAST_MODE -#define UFS_HISI_LIMIT_TX_PWR_HS FAST_MODE -#define UFS_HISI_LIMIT_HS_RATE PA_HS_MODE_B -#define UFS_HISI_LIMIT_DESIRED_MODE FAST - #define UFS_HISI_CAP_RESERVED BIT(0) #define UFS_HISI_CAP_PHY10nm BIT(1) diff --git a/drivers/scsi/ufs/ufs-mediatek-trace.h b/drivers/scsi/ufs/ufs-mediatek-trace.h new file mode 100644 index 000000000000..895e82ea6ece --- /dev/null +++ b/drivers/scsi/ufs/ufs-mediatek-trace.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 MediaTek Inc. + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM ufs_mtk + +#if !defined(_TRACE_EVENT_UFS_MEDIATEK_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_EVENT_UFS_MEDIATEK_H + +#include <linux/tracepoint.h> + +TRACE_EVENT(ufs_mtk_event, + TP_PROTO(unsigned int type, unsigned int data), + TP_ARGS(type, data), + + TP_STRUCT__entry( + __field(unsigned int, type) + __field(unsigned int, data) + ), + + TP_fast_assign( + __entry->type = type; + __entry->data = data; + ), + + TP_printk("ufs:event=%u data=%u", + __entry->type, __entry->data) + ); +#endif + +#undef TRACE_INCLUDE_PATH +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_PATH ../../drivers/scsi/ufs/ +#define TRACE_INCLUDE_FILE ufs-mediatek-trace +#include <trace/define_trace.h> diff --git a/drivers/scsi/ufs/ufs-mediatek.c b/drivers/scsi/ufs/ufs-mediatek.c index 8df73bc2f8cb..80618af7c872 100644 --- a/drivers/scsi/ufs/ufs-mediatek.c +++ b/drivers/scsi/ufs/ufs-mediatek.c @@ -24,10 +24,16 @@ #include "unipro.h" #include "ufs-mediatek.h" +#define CREATE_TRACE_POINTS +#include "ufs-mediatek-trace.h" + #define ufs_mtk_smc(cmd, val, res) \ arm_smccc_smc(MTK_SIP_UFS_CONTROL, \ cmd, val, 0, 0, 0, 0, 0, &(res)) +#define ufs_mtk_va09_pwr_ctrl(res, on) \ + ufs_mtk_smc(UFS_MTK_SIP_VA09_PWR_CTRL, on, res) + #define ufs_mtk_crypto_ctrl(res, enable) \ ufs_mtk_smc(UFS_MTK_SIP_CRYPTO_CTRL, enable, res) @@ -45,18 +51,8 @@ static struct ufs_dev_fix ufs_mtk_dev_fixups[] = { END_FIX }; -static const struct ufs_mtk_host_cfg ufs_mtk_mt8192_cfg = { - .caps = UFS_MTK_CAP_BOOST_CRYPT_ENGINE, -}; - static const struct of_device_id ufs_mtk_of_match[] = { - { - .compatible = "mediatek,mt8183-ufshci", - }, - { - .compatible = "mediatek,mt8192-ufshci", - .data = &ufs_mtk_mt8192_cfg - }, + { .compatible = "mediatek,mt8183-ufshci" }, {}, }; @@ -64,7 +60,21 @@ static bool ufs_mtk_is_boost_crypt_enabled(struct ufs_hba *hba) { struct ufs_mtk_host *host = ufshcd_get_variant(hba); - return (host->caps & UFS_MTK_CAP_BOOST_CRYPT_ENGINE); + return !!(host->caps & UFS_MTK_CAP_BOOST_CRYPT_ENGINE); +} + +static bool ufs_mtk_is_va09_supported(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + return !!(host->caps & UFS_MTK_CAP_VA09_PWR_CTRL); +} + +static bool ufs_mtk_is_broken_vcc(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + return !!(host->caps & UFS_MTK_CAP_BROKEN_VCC); } static void ufs_mtk_cfg_unipro_cg(struct ufs_hba *hba, bool enable) @@ -158,6 +168,7 @@ static int ufs_mtk_hce_enable_notify(struct ufs_hba *hba, enum ufs_notify_change_status status) { struct ufs_mtk_host *host = ufshcd_get_variant(hba); + unsigned long flags; if (status == PRE_CHANGE) { if (host->unipro_lpm) { @@ -169,6 +180,17 @@ static int ufs_mtk_hce_enable_notify(struct ufs_hba *hba, if (hba->caps & UFSHCD_CAP_CRYPTO) ufs_mtk_crypto_enable(hba); + + if (host->caps & UFS_MTK_CAP_DISABLE_AH8) { + spin_lock_irqsave(hba->host->host_lock, flags); + ufshcd_writel(hba, 0, + REG_AUTO_HIBERNATE_IDLE_TIMER); + spin_unlock_irqrestore(hba->host->host_lock, + flags); + + hba->capabilities &= ~MASK_AUTO_HIBERN8_SUPPORT; + hba->ahit = 0; + } } return 0; @@ -300,21 +322,46 @@ static int ufs_mtk_wait_link_state(struct ufs_hba *hba, u32 state, return -ETIMEDOUT; } -static void ufs_mtk_mphy_power_on(struct ufs_hba *hba, bool on) +static int ufs_mtk_mphy_power_on(struct ufs_hba *hba, bool on) { struct ufs_mtk_host *host = ufshcd_get_variant(hba); struct phy *mphy = host->mphy; + struct arm_smccc_res res; + int ret = 0; - if (!mphy) - return; + if (!mphy || !(on ^ host->mphy_powered_on)) + return 0; - if (on && !host->mphy_powered_on) + if (on) { + if (ufs_mtk_is_va09_supported(hba)) { + ret = regulator_enable(host->reg_va09); + if (ret < 0) + goto out; + /* wait 200 us to stablize VA09 */ + usleep_range(200, 210); + ufs_mtk_va09_pwr_ctrl(res, 1); + } phy_power_on(mphy); - else if (!on && host->mphy_powered_on) + } else { phy_power_off(mphy); - else - return; - host->mphy_powered_on = on; + if (ufs_mtk_is_va09_supported(hba)) { + ufs_mtk_va09_pwr_ctrl(res, 0); + ret = regulator_disable(host->reg_va09); + if (ret < 0) + goto out; + } + } +out: + if (ret) { + dev_info(hba->dev, + "failed to %s va09: %d\n", + on ? "enable" : "disable", + ret); + } else { + host->mphy_powered_on = on; + } + + return ret; } static int ufs_mtk_get_host_clk(struct device *dev, const char *name, @@ -402,7 +449,7 @@ static int ufs_mtk_init_host_clk(struct ufs_hba *hba, const char *name, return ret; } -static void ufs_mtk_init_host_caps(struct ufs_hba *hba) +static void ufs_mtk_init_boost_crypt(struct ufs_hba *hba) { struct ufs_mtk_host *host = ufshcd_get_variant(hba); struct ufs_mtk_crypt_cfg *cfg; @@ -410,11 +457,6 @@ static void ufs_mtk_init_host_caps(struct ufs_hba *hba) struct regulator *reg; u32 volt; - host->caps = host->cfg->caps; - - if (!ufs_mtk_is_boost_crypt_enabled(hba)) - return; - host->crypt = devm_kzalloc(dev, sizeof(*(host->crypt)), GFP_KERNEL); if (!host->crypt) @@ -448,11 +490,54 @@ static void ufs_mtk_init_host_caps(struct ufs_hba *hba) cfg->reg_vcore = reg; cfg->vcore_volt = volt; - dev_info(dev, "caps: boost-crypt"); - return; + host->caps |= UFS_MTK_CAP_BOOST_CRYPT_ENGINE; disable_caps: - host->caps &= ~UFS_MTK_CAP_BOOST_CRYPT_ENGINE; + return; +} + +static void ufs_mtk_init_va09_pwr_ctrl(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + host->reg_va09 = regulator_get(hba->dev, "va09"); + if (!host->reg_va09) + dev_info(hba->dev, "failed to get va09"); + else + host->caps |= UFS_MTK_CAP_VA09_PWR_CTRL; +} + +static void ufs_mtk_init_host_caps(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + struct device_node *np = hba->dev->of_node; + + if (of_property_read_bool(np, "mediatek,ufs-boost-crypt")) + ufs_mtk_init_boost_crypt(hba); + + if (of_property_read_bool(np, "mediatek,ufs-support-va09")) + ufs_mtk_init_va09_pwr_ctrl(hba); + + if (of_property_read_bool(np, "mediatek,ufs-disable-ah8")) + host->caps |= UFS_MTK_CAP_DISABLE_AH8; + + if (of_property_read_bool(np, "mediatek,ufs-broken-vcc")) + host->caps |= UFS_MTK_CAP_BROKEN_VCC; + + dev_info(hba->dev, "caps: 0x%x", host->caps); +} + +static void ufs_mtk_scale_perf(struct ufs_hba *hba, bool up) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + ufs_mtk_boost_crypt(hba, up); + ufs_mtk_setup_ref_clk(hba, up); + + if (up) + phy_power_on(host->mphy); + else + phy_power_off(host->mphy); } /** @@ -467,8 +552,8 @@ static int ufs_mtk_setup_clocks(struct ufs_hba *hba, bool on, enum ufs_notify_change_status status) { struct ufs_mtk_host *host = ufshcd_get_variant(hba); - int ret = 0; bool clk_pwr_off = false; + int ret = 0; /* * In case ufs_mtk_init() is not yet done, simply ignore. @@ -496,20 +581,33 @@ static int ufs_mtk_setup_clocks(struct ufs_hba *hba, bool on, clk_pwr_off = true; } - if (clk_pwr_off) { - ufs_mtk_boost_crypt(hba, on); - ufs_mtk_setup_ref_clk(hba, on); - ufs_mtk_mphy_power_on(hba, on); - } + if (clk_pwr_off) + ufs_mtk_scale_perf(hba, false); } else if (on && status == POST_CHANGE) { - ufs_mtk_mphy_power_on(hba, on); - ufs_mtk_setup_ref_clk(hba, on); - ufs_mtk_boost_crypt(hba, on); + ufs_mtk_scale_perf(hba, true); } return ret; } +static void ufs_mtk_get_controller_version(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + int ret, ver = 0; + + if (host->hw_ver.major) + return; + + /* Set default (minimum) version anyway */ + host->hw_ver.major = 2; + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_LOCALVERINFO), &ver); + if (!ret) { + if (ver >= UFS_UNIPRO_VER_1_8) + host->hw_ver.major = 3; + } +} + /** * ufs_mtk_init - find other essential mmio bases * @hba: host controller instance @@ -537,17 +635,14 @@ static int ufs_mtk_init(struct ufs_hba *hba) host->hba = hba; ufshcd_set_variant(hba, host); - /* Get host capability and platform data */ id = of_match_device(ufs_mtk_of_match, dev); if (!id) { err = -EINVAL; goto out; } - if (id->data) { - host->cfg = (struct ufs_mtk_host_cfg *)id->data; - ufs_mtk_init_host_caps(hba); - } + /* Initialize host capability */ + ufs_mtk_init_host_caps(hba); err = ufs_mtk_bind_mphy(hba); if (err) @@ -568,6 +663,9 @@ static int ufs_mtk_init(struct ufs_hba *hba) hba->caps |= UFSHCD_CAP_WB_EN; hba->vps->wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(80); + if (host->caps & UFS_MTK_CAP_DISABLE_AH8) + hba->caps |= UFSHCD_CAP_HIBERN8_WITH_CLK_GATING; + /* * ufshcd_vops_init() is invoked after * ufshcd_setup_clock(true) in ufshcd_hba_init() thus @@ -575,6 +673,7 @@ static int ufs_mtk_init(struct ufs_hba *hba) * * Enable phy clocks specifically here. */ + ufs_mtk_mphy_power_on(hba, true); ufs_mtk_setup_clocks(hba, true, POST_CHANGE); goto out; @@ -589,22 +688,13 @@ static int ufs_mtk_pre_pwr_change(struct ufs_hba *hba, struct ufs_pa_layer_attr *dev_max_params, struct ufs_pa_layer_attr *dev_req_params) { + struct ufs_mtk_host *host = ufshcd_get_variant(hba); struct ufs_dev_params host_cap; int ret; - host_cap.tx_lanes = UFS_MTK_LIMIT_NUM_LANES_TX; - host_cap.rx_lanes = UFS_MTK_LIMIT_NUM_LANES_RX; - host_cap.hs_rx_gear = UFS_MTK_LIMIT_HSGEAR_RX; - host_cap.hs_tx_gear = UFS_MTK_LIMIT_HSGEAR_TX; - host_cap.pwm_rx_gear = UFS_MTK_LIMIT_PWMGEAR_RX; - host_cap.pwm_tx_gear = UFS_MTK_LIMIT_PWMGEAR_TX; - host_cap.rx_pwr_pwm = UFS_MTK_LIMIT_RX_PWR_PWM; - host_cap.tx_pwr_pwm = UFS_MTK_LIMIT_TX_PWR_PWM; - host_cap.rx_pwr_hs = UFS_MTK_LIMIT_RX_PWR_HS; - host_cap.tx_pwr_hs = UFS_MTK_LIMIT_TX_PWR_HS; - host_cap.hs_rate = UFS_MTK_LIMIT_HS_RATE; - host_cap.desired_working_mode = - UFS_MTK_LIMIT_DESIRED_MODE; + ufshcd_init_pwr_dev_param(&host_cap); + host_cap.hs_rx_gear = UFS_HS_G4; + host_cap.hs_tx_gear = UFS_HS_G4; ret = ufshcd_get_pwr_dev_param(&host_cap, dev_max_params, @@ -614,6 +704,12 @@ static int ufs_mtk_pre_pwr_change(struct ufs_hba *hba, __func__); } + if (host->hw_ver.major >= 3) { + ret = ufshcd_dme_configure_adapt(hba, + dev_req_params->gear_tx, + PA_INITIAL_ADAPT); + } + return ret; } @@ -639,14 +735,14 @@ static int ufs_mtk_pwr_change_notify(struct ufs_hba *hba, return ret; } -static int ufs_mtk_unipro_set_pm(struct ufs_hba *hba, bool lpm) +static int ufs_mtk_unipro_set_lpm(struct ufs_hba *hba, bool lpm) { int ret; struct ufs_mtk_host *host = ufshcd_get_variant(hba); ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VS_UNIPROPOWERDOWNCONTROL, 0), - lpm); + lpm ? 1 : 0); if (!ret || !lpm) { /* * Forcibly set as non-LPM mode if UIC commands is failed @@ -664,7 +760,9 @@ static int ufs_mtk_pre_link(struct ufs_hba *hba) int ret; u32 tmp; - ret = ufs_mtk_unipro_set_pm(hba, false); + ufs_mtk_get_controller_version(hba); + + ret = ufs_mtk_unipro_set_lpm(hba, false); if (ret) return ret; @@ -743,7 +841,7 @@ static int ufs_mtk_link_startup_notify(struct ufs_hba *hba, return ret; } -static void ufs_mtk_device_reset(struct ufs_hba *hba) +static int ufs_mtk_device_reset(struct ufs_hba *hba) { struct arm_smccc_res res; @@ -764,6 +862,8 @@ static void ufs_mtk_device_reset(struct ufs_hba *hba) usleep_range(10000, 15000); dev_info(hba->dev, "device reset done\n"); + + return 0; } static int ufs_mtk_link_set_hpm(struct ufs_hba *hba) @@ -774,7 +874,7 @@ static int ufs_mtk_link_set_hpm(struct ufs_hba *hba) if (err) return err; - err = ufs_mtk_unipro_set_pm(hba, false); + err = ufs_mtk_unipro_set_lpm(hba, false); if (err) return err; @@ -795,10 +895,10 @@ static int ufs_mtk_link_set_lpm(struct ufs_hba *hba) { int err; - err = ufs_mtk_unipro_set_pm(hba, true); + err = ufs_mtk_unipro_set_lpm(hba, true); if (err) { /* Resume UniPro state for following error recovery */ - ufs_mtk_unipro_set_pm(hba, false); + ufs_mtk_unipro_set_lpm(hba, false); return err; } @@ -824,40 +924,52 @@ static int ufs_mtk_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) if (ufshcd_is_link_hibern8(hba)) { err = ufs_mtk_link_set_lpm(hba); - if (err) { - /* - * Set link as off state enforcedly to trigger - * ufshcd_host_reset_and_restore() in ufshcd_suspend() - * for completed host reset. - */ - ufshcd_set_link_off(hba); - return -EAGAIN; - } + if (err) + goto fail; + } + + if (!ufshcd_is_link_active(hba)) { /* * Make sure no error will be returned to prevent * ufshcd_suspend() re-enabling regulators while vreg is still * in low-power mode. */ ufs_mtk_vreg_set_lpm(hba, true); + err = ufs_mtk_mphy_power_on(hba, false); + if (err) + goto fail; } return 0; +fail: + /* + * Set link as off state enforcedly to trigger + * ufshcd_host_reset_and_restore() in ufshcd_suspend() + * for completed host reset. + */ + ufshcd_set_link_off(hba); + return -EAGAIN; } static int ufs_mtk_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) { int err; + err = ufs_mtk_mphy_power_on(hba, true); + if (err) + goto fail; + + ufs_mtk_vreg_set_lpm(hba, false); + if (ufshcd_is_link_hibern8(hba)) { - ufs_mtk_vreg_set_lpm(hba, false); err = ufs_mtk_link_set_hpm(hba); - if (err) { - err = ufshcd_link_recovery(hba); - return err; - } + if (err) + goto fail; } return 0; +fail: + return ufshcd_link_recovery(hba); } static void ufs_mtk_dbg_register_dump(struct ufs_hba *hba) @@ -901,6 +1013,25 @@ static int ufs_mtk_apply_dev_quirks(struct ufs_hba *hba) static void ufs_mtk_fixup_dev_quirks(struct ufs_hba *hba) { ufshcd_fixup_dev_quirks(hba, ufs_mtk_dev_fixups); + + if (ufs_mtk_is_broken_vcc(hba) && hba->vreg_info.vcc && + (hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)) { + hba->vreg_info.vcc->always_on = true; + /* + * VCC will be kept always-on thus we don't + * need any delay during regulator operations + */ + hba->dev_quirks &= ~(UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM | + UFS_DEVICE_QUIRK_DELAY_AFTER_LPM); + } +} + +static void ufs_mtk_event_notify(struct ufs_hba *hba, + enum ufs_event_type evt, void *data) +{ + unsigned int val = *(u32 *)data; + + trace_ufs_mtk_event(evt, val); } /* @@ -922,6 +1053,7 @@ static const struct ufs_hba_variant_ops ufs_hba_mtk_vops = { .resume = ufs_mtk_resume, .dbg_register_dump = ufs_mtk_dbg_register_dump, .device_reset = ufs_mtk_device_reset, + .event_notify = ufs_mtk_event_notify, }; /** diff --git a/drivers/scsi/ufs/ufs-mediatek.h b/drivers/scsi/ufs/ufs-mediatek.h index 2b6a1312c9bc..3f0d3bb769e8 100644 --- a/drivers/scsi/ufs/ufs-mediatek.h +++ b/drivers/scsi/ufs/ufs-mediatek.h @@ -31,22 +31,6 @@ #define REFCLK_REQ_TIMEOUT_US 3000 /* - * Vendor specific pre-defined parameters - */ -#define UFS_MTK_LIMIT_NUM_LANES_RX 2 -#define UFS_MTK_LIMIT_NUM_LANES_TX 2 -#define UFS_MTK_LIMIT_HSGEAR_RX UFS_HS_G3 -#define UFS_MTK_LIMIT_HSGEAR_TX UFS_HS_G3 -#define UFS_MTK_LIMIT_PWMGEAR_RX UFS_PWM_G4 -#define UFS_MTK_LIMIT_PWMGEAR_TX UFS_PWM_G4 -#define UFS_MTK_LIMIT_RX_PWR_PWM SLOW_MODE -#define UFS_MTK_LIMIT_TX_PWR_PWM SLOW_MODE -#define UFS_MTK_LIMIT_RX_PWR_HS FAST_MODE -#define UFS_MTK_LIMIT_TX_PWR_HS FAST_MODE -#define UFS_MTK_LIMIT_HS_RATE PA_HS_MODE_B -#define UFS_MTK_LIMIT_DESIRED_MODE UFS_HS_MODE - -/* * Other attributes */ #define VS_DEBUGCLOCKENABLE 0xD0A1 @@ -69,6 +53,7 @@ enum { * SiP commands */ #define MTK_SIP_UFS_CONTROL MTK_SIP_SMC_CMD(0x276) +#define UFS_MTK_SIP_VA09_PWR_CTRL BIT(0) #define UFS_MTK_SIP_DEVICE_RESET BIT(1) #define UFS_MTK_SIP_CRYPTO_CTRL BIT(2) #define UFS_MTK_SIP_REF_CLK_NOTIFICATION BIT(3) @@ -94,6 +79,9 @@ enum { */ enum ufs_mtk_host_caps { UFS_MTK_CAP_BOOST_CRYPT_ENGINE = 1 << 0, + UFS_MTK_CAP_VA09_PWR_CTRL = 1 << 1, + UFS_MTK_CAP_DISABLE_AH8 = 1 << 2, + UFS_MTK_CAP_BROKEN_VCC = 1 << 3, }; struct ufs_mtk_crypt_cfg { @@ -104,19 +92,22 @@ struct ufs_mtk_crypt_cfg { int vcore_volt; }; -struct ufs_mtk_host_cfg { - enum ufs_mtk_host_caps caps; +struct ufs_mtk_hw_ver { + u8 step; + u8 minor; + u8 major; }; struct ufs_mtk_host { - struct ufs_hba *hba; struct phy *mphy; - struct ufs_mtk_host_cfg *cfg; - struct ufs_mtk_crypt_cfg *crypt; - enum ufs_mtk_host_caps caps; + struct regulator *reg_va09; struct reset_control *hci_reset; struct reset_control *unipro_reset; struct reset_control *crypto_reset; + struct ufs_hba *hba; + struct ufs_mtk_crypt_cfg *crypt; + struct ufs_mtk_hw_ver hw_ver; + enum ufs_mtk_host_caps caps; bool mphy_powered_on; bool unipro_lpm; bool ref_clk_enabled; diff --git a/drivers/scsi/ufs/ufs-qcom.c b/drivers/scsi/ufs/ufs-qcom.c index f9d6ef356540..2206b1e4b774 100644 --- a/drivers/scsi/ufs/ufs-qcom.c +++ b/drivers/scsi/ufs/ufs-qcom.c @@ -691,19 +691,8 @@ static int ufs_qcom_pwr_change_notify(struct ufs_hba *hba, switch (status) { case PRE_CHANGE: - ufs_qcom_cap.tx_lanes = UFS_QCOM_LIMIT_NUM_LANES_TX; - ufs_qcom_cap.rx_lanes = UFS_QCOM_LIMIT_NUM_LANES_RX; - ufs_qcom_cap.hs_rx_gear = UFS_QCOM_LIMIT_HSGEAR_RX; - ufs_qcom_cap.hs_tx_gear = UFS_QCOM_LIMIT_HSGEAR_TX; - ufs_qcom_cap.pwm_rx_gear = UFS_QCOM_LIMIT_PWMGEAR_RX; - ufs_qcom_cap.pwm_tx_gear = UFS_QCOM_LIMIT_PWMGEAR_TX; - ufs_qcom_cap.rx_pwr_pwm = UFS_QCOM_LIMIT_RX_PWR_PWM; - ufs_qcom_cap.tx_pwr_pwm = UFS_QCOM_LIMIT_TX_PWR_PWM; - ufs_qcom_cap.rx_pwr_hs = UFS_QCOM_LIMIT_RX_PWR_HS; - ufs_qcom_cap.tx_pwr_hs = UFS_QCOM_LIMIT_TX_PWR_HS; + ufshcd_init_pwr_dev_param(&ufs_qcom_cap); ufs_qcom_cap.hs_rate = UFS_QCOM_LIMIT_HS_RATE; - ufs_qcom_cap.desired_working_mode = - UFS_QCOM_LIMIT_DESIRED_MODE; if (host->hw_ver.major == 0x1) { /* @@ -734,17 +723,9 @@ static int ufs_qcom_pwr_change_notify(struct ufs_hba *hba, ufs_qcom_dev_ref_clk_ctrl(host, true); if (host->hw_ver.major >= 0x4) { - if (dev_req_params->gear_tx == UFS_HS_G4) { - /* INITIAL ADAPT */ - ufshcd_dme_set(hba, - UIC_ARG_MIB(PA_TXHSADAPTTYPE), - PA_INITIAL_ADAPT); - } else { - /* NO ADAPT */ - ufshcd_dme_set(hba, - UIC_ARG_MIB(PA_TXHSADAPTTYPE), - PA_NO_ADAPT); - } + ufshcd_dme_configure_adapt(hba, + dev_req_params->gear_tx, + PA_INITIAL_ADAPT); } break; case POST_CHANGE: @@ -863,6 +844,7 @@ static void ufs_qcom_set_caps(struct ufs_hba *hba) hba->caps |= UFSHCD_CAP_AUTO_BKOPS_SUSPEND; hba->caps |= UFSHCD_CAP_WB_EN; hba->caps |= UFSHCD_CAP_CRYPTO; + hba->caps |= UFSHCD_CAP_AGGR_POWER_COLLAPSE; if (host->hw_ver.major >= 0x2) { host->caps = UFS_QCOM_CAP_QUNIPRO | @@ -977,6 +959,7 @@ static int ufs_qcom_init(struct ufs_hba *hba) struct platform_device *pdev = to_platform_device(dev); struct ufs_qcom_host *host; struct resource *res; + struct ufs_clk_info *clki; if (strlen(android_boot_dev) && strcmp(android_boot_dev, dev_name(dev))) return -ENODEV; @@ -1075,6 +1058,11 @@ static int ufs_qcom_init(struct ufs_hba *hba) } } + list_for_each_entry(clki, &hba->clk_list_head, list) { + if (!strcmp(clki->name, "core_clk_unipro")) + clki->keep_link_active = true; + } + err = ufs_qcom_init_lane_clks(host); if (err) goto out_variant_clear; @@ -1421,13 +1409,13 @@ static void ufs_qcom_dump_dbg_regs(struct ufs_hba *hba) * * Toggles the (optional) reset line to reset the attached device. */ -static void ufs_qcom_device_reset(struct ufs_hba *hba) +static int ufs_qcom_device_reset(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); /* reset gpio is optional */ if (!host->device_reset) - return; + return -EOPNOTSUPP; /* * The UFS device shall detect reset pulses of 1us, sleep for 10us to @@ -1438,6 +1426,8 @@ static void ufs_qcom_device_reset(struct ufs_hba *hba) gpiod_set_value_cansleep(host->device_reset, 0); usleep_range(10, 15); + + return 0; } #if IS_ENABLED(CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND) diff --git a/drivers/scsi/ufs/ufs-qcom.h b/drivers/scsi/ufs/ufs-qcom.h index 3f4922743b3e..8208e3a3ef59 100644 --- a/drivers/scsi/ufs/ufs-qcom.h +++ b/drivers/scsi/ufs/ufs-qcom.h @@ -27,18 +27,7 @@ #define SLOW 1 #define FAST 2 -#define UFS_QCOM_LIMIT_NUM_LANES_RX 2 -#define UFS_QCOM_LIMIT_NUM_LANES_TX 2 -#define UFS_QCOM_LIMIT_HSGEAR_RX UFS_HS_G3 -#define UFS_QCOM_LIMIT_HSGEAR_TX UFS_HS_G3 -#define UFS_QCOM_LIMIT_PWMGEAR_RX UFS_PWM_G4 -#define UFS_QCOM_LIMIT_PWMGEAR_TX UFS_PWM_G4 -#define UFS_QCOM_LIMIT_RX_PWR_PWM SLOW_MODE -#define UFS_QCOM_LIMIT_TX_PWR_PWM SLOW_MODE -#define UFS_QCOM_LIMIT_RX_PWR_HS FAST_MODE -#define UFS_QCOM_LIMIT_TX_PWR_HS FAST_MODE #define UFS_QCOM_LIMIT_HS_RATE PA_HS_MODE_B -#define UFS_QCOM_LIMIT_DESIRED_MODE FAST /* QCOM UFS host controller vendor specific registers */ enum { diff --git a/drivers/scsi/ufs/ufs-sysfs.c b/drivers/scsi/ufs/ufs-sysfs.c index bdcd27faa054..08e72b7eef6a 100644 --- a/drivers/scsi/ufs/ufs-sysfs.c +++ b/drivers/scsi/ufs/ufs-sysfs.c @@ -28,6 +28,7 @@ static const char *ufschd_ufs_dev_pwr_mode_to_string( case UFS_ACTIVE_PWR_MODE: return "ACTIVE"; case UFS_SLEEP_PWR_MODE: return "SLEEP"; case UFS_POWERDOWN_PWR_MODE: return "POWERDOWN"; + case UFS_DEEPSLEEP_PWR_MODE: return "DEEPSLEEP"; default: return "UNKNOWN"; } } @@ -38,6 +39,7 @@ static inline ssize_t ufs_sysfs_pm_lvl_store(struct device *dev, bool rpm) { struct ufs_hba *hba = dev_get_drvdata(dev); + struct ufs_dev_info *dev_info = &hba->dev_info; unsigned long flags, value; if (kstrtoul(buf, 0, &value)) @@ -46,6 +48,11 @@ static inline ssize_t ufs_sysfs_pm_lvl_store(struct device *dev, if (value >= UFS_PM_LVL_MAX) return -EINVAL; + if (ufs_pm_lvl_states[value].dev_state == UFS_DEEPSLEEP_PWR_MODE && + (!(hba->caps & UFSHCD_CAP_DEEPSLEEP) || + !(dev_info->wspecversion >= 0x310))) + return -EINVAL; + spin_lock_irqsave(hba->host->host_lock, flags); if (rpm) hba->rpm_lvl = value; diff --git a/drivers/scsi/ufs/ufs.h b/drivers/scsi/ufs/ufs.h index f8ab16f30fdc..14dfda735adf 100644 --- a/drivers/scsi/ufs/ufs.h +++ b/drivers/scsi/ufs/ufs.h @@ -330,7 +330,6 @@ enum { UFS_DEV_WRITE_BOOSTER_SUP = BIT(8), }; -#define POWER_DESC_MAX_SIZE 0x62 #define POWER_DESC_MAX_ACTV_ICC_LVLS 16 /* Attribute bActiveICCLevel parameter bit masks definitions */ @@ -442,6 +441,7 @@ enum ufs_dev_pwr_mode { UFS_ACTIVE_PWR_MODE = 1, UFS_SLEEP_PWR_MODE = 2, UFS_POWERDOWN_PWR_MODE = 3, + UFS_DEEPSLEEP_PWR_MODE = 4, }; #define UFS_WB_BUF_REMAIN_PERCENT(val) ((val) / 10) @@ -512,6 +512,7 @@ struct ufs_query_res { struct ufs_vreg { struct regulator *reg; const char *name; + bool always_on; bool enabled; int min_uV; int max_uV; diff --git a/drivers/scsi/ufs/ufshcd-dwc.c b/drivers/scsi/ufs/ufshcd-dwc.c index 6a901da2d15a..5bb9d3a88795 100644 --- a/drivers/scsi/ufs/ufshcd-dwc.c +++ b/drivers/scsi/ufs/ufshcd-dwc.c @@ -120,13 +120,10 @@ int ufshcd_dwc_link_startup_notify(struct ufs_hba *hba, if (status == PRE_CHANGE) { ufshcd_dwc_program_clk_div(hba, DWC_UFS_REG_HCLKDIV_DIV_125); - if (hba->vops->phy_initialization) { - err = hba->vops->phy_initialization(hba); - if (err) { - dev_err(hba->dev, "Phy setup failed (%d)\n", - err); - goto out; - } + err = ufshcd_vops_phy_initialization(hba); + if (err) { + dev_err(hba->dev, "Phy setup failed (%d)\n", err); + goto out; } } else { /* POST_CHANGE */ err = ufshcd_dwc_link_is_up(hba); diff --git a/drivers/scsi/ufs/ufshcd-pci.c b/drivers/scsi/ufs/ufshcd-pci.c index df3a564c3e33..fadd566025b8 100644 --- a/drivers/scsi/ufs/ufshcd-pci.c +++ b/drivers/scsi/ufs/ufshcd-pci.c @@ -148,6 +148,8 @@ static int ufs_intel_common_init(struct ufs_hba *hba) { struct intel_host *host; + hba->caps |= UFSHCD_CAP_RPM_AUTOSUSPEND; + host = devm_kzalloc(hba->dev, sizeof(*host), GFP_KERNEL); if (!host) return -ENOMEM; @@ -163,6 +165,41 @@ static void ufs_intel_common_exit(struct ufs_hba *hba) intel_ltr_hide(hba->dev); } +static int ufs_intel_resume(struct ufs_hba *hba, enum ufs_pm_op op) +{ + /* + * To support S4 (suspend-to-disk) with spm_lvl other than 5, the base + * address registers must be restored because the restore kernel can + * have used different addresses. + */ + ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr), + REG_UTP_TRANSFER_REQ_LIST_BASE_L); + ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr), + REG_UTP_TRANSFER_REQ_LIST_BASE_H); + ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr), + REG_UTP_TASK_REQ_LIST_BASE_L); + ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr), + REG_UTP_TASK_REQ_LIST_BASE_H); + + if (ufshcd_is_link_hibern8(hba)) { + int ret = ufshcd_uic_hibern8_exit(hba); + + if (!ret) { + ufshcd_set_link_active(hba); + } else { + dev_err(hba->dev, "%s: hibern8 exit failed %d\n", + __func__, ret); + /* + * Force reset and restore. Any other actions can lead + * to an unrecoverable state. + */ + ufshcd_set_link_off(hba); + } + } + + return 0; +} + static int ufs_intel_ehl_init(struct ufs_hba *hba) { hba->quirks |= UFSHCD_QUIRK_BROKEN_AUTO_HIBERN8; @@ -174,6 +211,7 @@ static struct ufs_hba_variant_ops ufs_intel_cnl_hba_vops = { .init = ufs_intel_common_init, .exit = ufs_intel_common_exit, .link_startup_notify = ufs_intel_link_startup_notify, + .resume = ufs_intel_resume, }; static struct ufs_hba_variant_ops ufs_intel_ehl_hba_vops = { @@ -181,6 +219,7 @@ static struct ufs_hba_variant_ops ufs_intel_ehl_hba_vops = { .init = ufs_intel_ehl_init, .exit = ufs_intel_common_exit, .link_startup_notify = ufs_intel_link_startup_notify, + .resume = ufs_intel_resume, }; #ifdef CONFIG_PM_SLEEP @@ -207,6 +246,30 @@ static int ufshcd_pci_resume(struct device *dev) { return ufshcd_system_resume(dev_get_drvdata(dev)); } + +/** + * ufshcd_pci_poweroff - suspend-to-disk poweroff function + * @dev: pointer to PCI device handle + * + * Returns 0 if successful + * Returns non-zero otherwise + */ +static int ufshcd_pci_poweroff(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + int spm_lvl = hba->spm_lvl; + int ret; + + /* + * For poweroff we need to set the UFS device to PowerDown mode. + * Force spm_lvl to ensure that. + */ + hba->spm_lvl = 5; + ret = ufshcd_system_suspend(hba); + hba->spm_lvl = spm_lvl; + return ret; +} + #endif /* !CONFIG_PM_SLEEP */ #ifdef CONFIG_PM @@ -302,8 +365,14 @@ ufshcd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) } static const struct dev_pm_ops ufshcd_pci_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(ufshcd_pci_suspend, - ufshcd_pci_resume) +#ifdef CONFIG_PM_SLEEP + .suspend = ufshcd_pci_suspend, + .resume = ufshcd_pci_resume, + .freeze = ufshcd_pci_suspend, + .thaw = ufshcd_pci_resume, + .poweroff = ufshcd_pci_poweroff, + .restore = ufshcd_pci_resume, +#endif SET_RUNTIME_PM_OPS(ufshcd_pci_runtime_suspend, ufshcd_pci_runtime_resume, ufshcd_pci_runtime_idle) diff --git a/drivers/scsi/ufs/ufshcd-pltfrm.c b/drivers/scsi/ufs/ufshcd-pltfrm.c index 3db0af66c71c..1a69949a4ea1 100644 --- a/drivers/scsi/ufs/ufshcd-pltfrm.c +++ b/drivers/scsi/ufs/ufshcd-pltfrm.c @@ -92,6 +92,8 @@ static int ufshcd_parse_clock_info(struct ufs_hba *hba) clki->min_freq = clkfreq[i]; clki->max_freq = clkfreq[i+1]; clki->name = kstrdup(name, GFP_KERNEL); + if (!strcmp(name, "ref_clk")) + clki->keep_link_active = true; dev_dbg(dev, "%s: min %u max %u name %s\n", "freq-table-hz", clki->min_freq, clki->max_freq, clki->name); list_add_tail(&clki->list, &hba->clk_list_head); @@ -132,25 +134,6 @@ static int ufshcd_populate_vreg(struct device *dev, const char *name, dev_info(dev, "%s: unable to find %s\n", __func__, prop_name); vreg->max_uA = 0; } - - if (!strcmp(name, "vcc")) { - if (of_property_read_bool(np, "vcc-supply-1p8")) { - vreg->min_uV = UFS_VREG_VCC_1P8_MIN_UV; - vreg->max_uV = UFS_VREG_VCC_1P8_MAX_UV; - } else { - vreg->min_uV = UFS_VREG_VCC_MIN_UV; - vreg->max_uV = UFS_VREG_VCC_MAX_UV; - } - } else if (!strcmp(name, "vccq")) { - vreg->min_uV = UFS_VREG_VCCQ_MIN_UV; - vreg->max_uV = UFS_VREG_VCCQ_MAX_UV; - } else if (!strcmp(name, "vccq2")) { - vreg->min_uV = UFS_VREG_VCCQ2_MIN_UV; - vreg->max_uV = UFS_VREG_VCCQ2_MAX_UV; - } - - goto out; - out: if (!ret) *out_vreg = vreg; @@ -354,6 +337,23 @@ int ufshcd_get_pwr_dev_param(struct ufs_dev_params *pltfrm_param, } EXPORT_SYMBOL_GPL(ufshcd_get_pwr_dev_param); +void ufshcd_init_pwr_dev_param(struct ufs_dev_params *dev_param) +{ + dev_param->tx_lanes = 2; + dev_param->rx_lanes = 2; + dev_param->hs_rx_gear = UFS_HS_G3; + dev_param->hs_tx_gear = UFS_HS_G3; + dev_param->pwm_rx_gear = UFS_PWM_G4; + dev_param->pwm_tx_gear = UFS_PWM_G4; + dev_param->rx_pwr_pwm = SLOW_MODE; + dev_param->tx_pwr_pwm = SLOW_MODE; + dev_param->rx_pwr_hs = FAST_MODE; + dev_param->tx_pwr_hs = FAST_MODE; + dev_param->hs_rate = PA_HS_MODE_B; + dev_param->desired_working_mode = UFS_HS_MODE; +} +EXPORT_SYMBOL_GPL(ufshcd_init_pwr_dev_param); + /** * ufshcd_pltfrm_init - probe routine of the driver * @pdev: pointer to Platform device handle diff --git a/drivers/scsi/ufs/ufshcd-pltfrm.h b/drivers/scsi/ufs/ufshcd-pltfrm.h index b79cdf9129a0..772a8e848098 100644 --- a/drivers/scsi/ufs/ufshcd-pltfrm.h +++ b/drivers/scsi/ufs/ufshcd-pltfrm.h @@ -28,6 +28,7 @@ struct ufs_dev_params { int ufshcd_get_pwr_dev_param(struct ufs_dev_params *dev_param, struct ufs_pa_layer_attr *dev_max, struct ufs_pa_layer_attr *agreed_pwr); +void ufshcd_init_pwr_dev_param(struct ufs_dev_params *dev_param); int ufshcd_pltfrm_init(struct platform_device *pdev, const struct ufs_hba_variant_ops *vops); void ufshcd_pltfrm_shutdown(struct platform_device *pdev); diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c index 0c148fcd24de..82ad31781bc9 100644 --- a/drivers/scsi/ufs/ufshcd.c +++ b/drivers/scsi/ufs/ufshcd.c @@ -163,6 +163,11 @@ struct ufs_pm_lvl_states ufs_pm_lvl_states[] = { {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE}, {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE}, {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE}, + /* + * For DeepSleep, the link is first put in hibern8 and then off. + * Leaving the link in hibern8 is not supported. + */ + {UFS_DEEPSLEEP_PWR_MODE, UIC_LINK_OFF_STATE}, }; static inline enum ufs_dev_pwr_mode @@ -220,9 +225,8 @@ static int ufshcd_reset_and_restore(struct ufs_hba *hba); static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd); static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag); static void ufshcd_hba_exit(struct ufs_hba *hba); +static int ufshcd_clear_ua_wluns(struct ufs_hba *hba); static int ufshcd_probe_hba(struct ufs_hba *hba, bool async); -static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on, - bool skip_ref_clk); static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on); static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba); static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba); @@ -245,6 +249,8 @@ static int ufshcd_wb_buf_flush_disable(struct ufs_hba *hba); static int ufshcd_wb_ctrl(struct ufs_hba *hba, bool enable); static int ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set); static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable); +static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba); +static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba); static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag) { @@ -348,7 +354,7 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag, const char *str) { sector_t lba = -1; - u8 opcode = 0; + u8 opcode = 0, group_id = 0; u32 intr, doorbell; struct ufshcd_lrb *lrbp = &hba->lrb[tag]; struct scsi_cmnd *cmd = lrbp->cmd; @@ -374,13 +380,20 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, lba = cmd->request->bio->bi_iter.bi_sector; transfer_len = be32_to_cpu( lrbp->ucd_req_ptr->sc.exp_data_transfer_len); + if (opcode == WRITE_10) + group_id = lrbp->cmd->cmnd[6]; + } else if (opcode == UNMAP) { + if (cmd->request) { + lba = scsi_get_lba(cmd); + transfer_len = blk_rq_bytes(cmd->request); + } } } intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS); doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); trace_ufshcd_command(dev_name(hba->dev), str, tag, - doorbell, transfer_len, intr, lba, opcode); + doorbell, transfer_len, intr, lba, opcode, group_id); } static void ufshcd_print_clk_freqs(struct ufs_hba *hba) @@ -399,20 +412,25 @@ static void ufshcd_print_clk_freqs(struct ufs_hba *hba) } } -static void ufshcd_print_err_hist(struct ufs_hba *hba, - struct ufs_err_reg_hist *err_hist, - char *err_name) +static void ufshcd_print_evt(struct ufs_hba *hba, u32 id, + char *err_name) { int i; bool found = false; + struct ufs_event_hist *e; - for (i = 0; i < UFS_ERR_REG_HIST_LENGTH; i++) { - int p = (i + err_hist->pos) % UFS_ERR_REG_HIST_LENGTH; + if (id >= UFS_EVT_CNT) + return; - if (err_hist->tstamp[p] == 0) + e = &hba->ufs_stats.event[id]; + + for (i = 0; i < UFS_EVENT_HIST_LENGTH; i++) { + int p = (i + e->pos) % UFS_EVENT_HIST_LENGTH; + + if (e->tstamp[p] == 0) continue; dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p, - err_hist->reg[p], ktime_to_us(err_hist->tstamp[p])); + e->val[p], ktime_to_us(e->tstamp[p])); found = true; } @@ -420,26 +438,26 @@ static void ufshcd_print_err_hist(struct ufs_hba *hba, dev_err(hba->dev, "No record of %s\n", err_name); } -static void ufshcd_print_host_regs(struct ufs_hba *hba) +static void ufshcd_print_evt_hist(struct ufs_hba *hba) { ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: "); - ufshcd_print_err_hist(hba, &hba->ufs_stats.pa_err, "pa_err"); - ufshcd_print_err_hist(hba, &hba->ufs_stats.dl_err, "dl_err"); - ufshcd_print_err_hist(hba, &hba->ufs_stats.nl_err, "nl_err"); - ufshcd_print_err_hist(hba, &hba->ufs_stats.tl_err, "tl_err"); - ufshcd_print_err_hist(hba, &hba->ufs_stats.dme_err, "dme_err"); - ufshcd_print_err_hist(hba, &hba->ufs_stats.auto_hibern8_err, - "auto_hibern8_err"); - ufshcd_print_err_hist(hba, &hba->ufs_stats.fatal_err, "fatal_err"); - ufshcd_print_err_hist(hba, &hba->ufs_stats.link_startup_err, - "link_startup_fail"); - ufshcd_print_err_hist(hba, &hba->ufs_stats.resume_err, "resume_fail"); - ufshcd_print_err_hist(hba, &hba->ufs_stats.suspend_err, - "suspend_fail"); - ufshcd_print_err_hist(hba, &hba->ufs_stats.dev_reset, "dev_reset"); - ufshcd_print_err_hist(hba, &hba->ufs_stats.host_reset, "host_reset"); - ufshcd_print_err_hist(hba, &hba->ufs_stats.task_abort, "task_abort"); + ufshcd_print_evt(hba, UFS_EVT_PA_ERR, "pa_err"); + ufshcd_print_evt(hba, UFS_EVT_DL_ERR, "dl_err"); + ufshcd_print_evt(hba, UFS_EVT_NL_ERR, "nl_err"); + ufshcd_print_evt(hba, UFS_EVT_TL_ERR, "tl_err"); + ufshcd_print_evt(hba, UFS_EVT_DME_ERR, "dme_err"); + ufshcd_print_evt(hba, UFS_EVT_AUTO_HIBERN8_ERR, + "auto_hibern8_err"); + ufshcd_print_evt(hba, UFS_EVT_FATAL_ERR, "fatal_err"); + ufshcd_print_evt(hba, UFS_EVT_LINK_STARTUP_FAIL, + "link_startup_fail"); + ufshcd_print_evt(hba, UFS_EVT_RESUME_ERR, "resume_fail"); + ufshcd_print_evt(hba, UFS_EVT_SUSPEND_ERR, + "suspend_fail"); + ufshcd_print_evt(hba, UFS_EVT_DEV_RESET, "dev_reset"); + ufshcd_print_evt(hba, UFS_EVT_HOST_RESET, "host_reset"); + ufshcd_print_evt(hba, UFS_EVT_ABORT, "task_abort"); ufshcd_vops_dbg_register_dump(hba); } @@ -563,6 +581,23 @@ static void ufshcd_print_pwr_info(struct ufs_hba *hba) hba->pwr_info.hs_rate); } +static void ufshcd_device_reset(struct ufs_hba *hba) +{ + int err; + + err = ufshcd_vops_device_reset(hba); + + if (!err) { + ufshcd_set_ufs_dev_active(hba); + if (ufshcd_is_wb_allowed(hba)) { + hba->wb_enabled = false; + hba->wb_buf_flush_enabled = false; + } + } + if (err != -EOPNOTSUPP) + ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, err); +} + void ufshcd_delay_us(unsigned long us, unsigned long tolerance) { if (!us) @@ -1102,7 +1137,6 @@ out: */ static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up) { - #define UFS_MIN_GEAR_TO_SCALE_DOWN UFS_HS_G1 int ret = 0; struct ufs_pa_layer_attr new_pwr_info; @@ -1113,16 +1147,16 @@ static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up) memcpy(&new_pwr_info, &hba->pwr_info, sizeof(struct ufs_pa_layer_attr)); - if (hba->pwr_info.gear_tx > UFS_MIN_GEAR_TO_SCALE_DOWN - || hba->pwr_info.gear_rx > UFS_MIN_GEAR_TO_SCALE_DOWN) { + if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear || + hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) { /* save the current power mode */ memcpy(&hba->clk_scaling.saved_pwr_info.info, &hba->pwr_info, sizeof(struct ufs_pa_layer_attr)); /* scale down gear */ - new_pwr_info.gear_tx = UFS_MIN_GEAR_TO_SCALE_DOWN; - new_pwr_info.gear_rx = UFS_MIN_GEAR_TO_SCALE_DOWN; + new_pwr_info.gear_tx = hba->clk_scaling.min_gear; + new_pwr_info.gear_rx = hba->clk_scaling.min_gear; } } @@ -1555,6 +1589,7 @@ static void ufshcd_ungate_work(struct work_struct *work) } spin_unlock_irqrestore(hba->host->host_lock, flags); + ufshcd_hba_vreg_set_hpm(hba); ufshcd_setup_clocks(hba, true); ufshcd_enable_irq(hba); @@ -1714,12 +1749,10 @@ static void ufshcd_gate_work(struct work_struct *work) ufshcd_disable_irq(hba); - if (!ufshcd_is_link_active(hba)) - ufshcd_setup_clocks(hba, false); - else - /* If link is active, device ref_clk can't be switched off */ - __ufshcd_setup_clocks(hba, false, true); + ufshcd_setup_clocks(hba, false); + /* Put the host controller in low power mode if possible */ + ufshcd_hba_vreg_set_lpm(hba); /* * In case you are here to cancel this work the gating state * would be marked as REQ_CLKS_ON. In this case keep the state @@ -1751,8 +1784,9 @@ static void __ufshcd_release(struct ufs_hba *hba) if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL || - ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks || - hba->active_uic_cmd || hba->uic_async_done) + hba->outstanding_tasks || + hba->active_uic_cmd || hba->uic_async_done || + hba->clk_gating.state == CLKS_OFF) return; hba->clk_gating.state = REQ_CLKS_OFF; @@ -1814,19 +1848,19 @@ static ssize_t ufshcd_clkgate_enable_store(struct device *dev, return -EINVAL; value = !!value; + + spin_lock_irqsave(hba->host->host_lock, flags); if (value == hba->clk_gating.is_enabled) goto out; - if (value) { - ufshcd_release(hba); - } else { - spin_lock_irqsave(hba->host->host_lock, flags); + if (value) + __ufshcd_release(hba); + else hba->clk_gating.active_reqs++; - spin_unlock_irqrestore(hba->host->host_lock, flags); - } hba->clk_gating.is_enabled = value; out: + spin_unlock_irqrestore(hba->host->host_lock, flags); return count; } @@ -1837,6 +1871,9 @@ static void ufshcd_init_clk_scaling(struct ufs_hba *hba) if (!ufshcd_is_clkscaling_supported(hba)) return; + if (!hba->clk_scaling.min_gear) + hba->clk_scaling.min_gear = UFS_HS_G1; + INIT_WORK(&hba->clk_scaling.suspend_work, ufshcd_clk_scaling_suspend_work); INIT_WORK(&hba->clk_scaling.resume_work, @@ -1874,7 +1911,7 @@ static void ufshcd_init_clk_gating(struct ufs_hba *hba) snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d", hba->host->host_no); hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name, - WQ_MEM_RECLAIM); + WQ_MEM_RECLAIM | WQ_HIGHPRI); hba->clk_gating.is_enabled = true; @@ -2557,6 +2594,14 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) (hba->clk_gating.state != CLKS_ON)); lrbp = &hba->lrb[tag]; + if (unlikely(lrbp->in_use)) { + if (hba->pm_op_in_progress) + set_host_byte(cmd, DID_BAD_TARGET); + else + err = SCSI_MLQUEUE_HOST_BUSY; + ufshcd_release(hba); + goto out; + } WARN_ON(lrbp->cmd); lrbp->cmd = cmd; @@ -2799,6 +2844,11 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, init_completion(&wait); lrbp = &hba->lrb[tag]; + if (unlikely(lrbp->in_use)) { + err = -EBUSY; + goto out; + } + WARN_ON(lrbp->cmd); err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag); if (unlikely(err)) @@ -2815,6 +2865,7 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout); +out: ufshcd_add_query_upiu_trace(hba, tag, err ? "query_complete_err" : "query_complete"); @@ -2960,14 +3011,14 @@ int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, BUG_ON(!hba); - ufshcd_hold(hba, false); if (!attr_val) { dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n", __func__, opcode); - err = -EINVAL; - goto out; + return -EINVAL; } + ufshcd_hold(hba, false); + mutex_lock(&hba->dev_cmd.lock); ufshcd_init_query(hba, &request, &response, opcode, idn, index, selector); @@ -2999,7 +3050,6 @@ int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, out_unlock: mutex_unlock(&hba->dev_cmd.lock); -out: ufshcd_release(hba); return err; } @@ -3051,21 +3101,20 @@ static int __ufshcd_query_descriptor(struct ufs_hba *hba, BUG_ON(!hba); - ufshcd_hold(hba, false); if (!desc_buf) { dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n", __func__, opcode); - err = -EINVAL; - goto out; + return -EINVAL; } if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) { dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n", __func__, *buf_len); - err = -EINVAL; - goto out; + return -EINVAL; } + ufshcd_hold(hba, false); + mutex_lock(&hba->dev_cmd.lock); ufshcd_init_query(hba, &request, &response, opcode, idn, index, selector); @@ -3100,7 +3149,6 @@ static int __ufshcd_query_descriptor(struct ufs_hba *hba, out_unlock: hba->dev_cmd.query.descriptor = NULL; mutex_unlock(&hba->dev_cmd.lock); -out: ufshcd_release(hba); return err; } @@ -3601,6 +3649,22 @@ static int ufshcd_dme_reset(struct ufs_hba *hba) return ret; } +int ufshcd_dme_configure_adapt(struct ufs_hba *hba, + int agreed_gear, + int adapt_val) +{ + int ret; + + if (agreed_gear != UFS_HS_G4) + adapt_val = PA_NO_ADAPT; + + ret = ufshcd_dme_set(hba, + UIC_ARG_MIB(PA_TXHSADAPTTYPE), + adapt_val); + return ret; +} +EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt); + /** * ufshcd_dme_enable - UIC command for DME_ENABLE * @hba: per adapter instance @@ -3619,7 +3683,7 @@ static int ufshcd_dme_enable(struct ufs_hba *hba) ret = ufshcd_send_uic_cmd(hba, &uic_cmd); if (ret) dev_err(hba->dev, - "dme-reset: error code %d\n", ret); + "dme-enable: error code %d\n", ret); return ret; } @@ -3854,7 +3918,7 @@ out: if (ret) { ufshcd_print_host_state(hba); ufshcd_print_pwr_info(hba); - ufshcd_print_host_regs(hba); + ufshcd_print_evt_hist(hba); } spin_lock_irqsave(hba->host->host_lock, flags); @@ -3918,7 +3982,7 @@ int ufshcd_link_recovery(struct ufs_hba *hba) spin_unlock_irqrestore(hba->host->host_lock, flags); /* Reset the attached device */ - ufshcd_vops_device_reset(hba); + ufshcd_device_reset(hba); ret = ufshcd_host_reset_and_restore(hba); @@ -4350,8 +4414,10 @@ static inline void ufshcd_hba_stop(struct ufs_hba *hba) */ static int ufshcd_hba_execute_hce(struct ufs_hba *hba) { - int retry; + int retry_outer = 3; + int retry_inner; +start: if (!ufshcd_is_hba_active(hba)) /* change controller state to "reset state" */ ufshcd_hba_stop(hba); @@ -4377,13 +4443,17 @@ static int ufshcd_hba_execute_hce(struct ufs_hba *hba) ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100); /* wait for the host controller to complete initialization */ - retry = 50; + retry_inner = 50; while (ufshcd_is_hba_active(hba)) { - if (retry) { - retry--; + if (retry_inner) { + retry_inner--; } else { dev_err(hba->dev, "Controller enable failed\n"); + if (retry_outer) { + retry_outer--; + goto start; + } return -EIO; } usleep_range(1000, 1100); @@ -4460,14 +4530,21 @@ static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba) return ufshcd_disable_tx_lcc(hba, true); } -void ufshcd_update_reg_hist(struct ufs_err_reg_hist *reg_hist, - u32 reg) +void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val) { - reg_hist->reg[reg_hist->pos] = reg; - reg_hist->tstamp[reg_hist->pos] = ktime_get(); - reg_hist->pos = (reg_hist->pos + 1) % UFS_ERR_REG_HIST_LENGTH; + struct ufs_event_hist *e; + + if (id >= UFS_EVT_CNT) + return; + + e = &hba->ufs_stats.event[id]; + e->val[e->pos] = val; + e->tstamp[e->pos] = ktime_get(); + e->pos = (e->pos + 1) % UFS_EVENT_HIST_LENGTH; + + ufshcd_vops_event_notify(hba, id, &val); } -EXPORT_SYMBOL_GPL(ufshcd_update_reg_hist); +EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist); /** * ufshcd_link_startup - Initialize unipro link startup @@ -4496,7 +4573,8 @@ link_startup: /* check if device is detected by inter-connect layer */ if (!ret && !ufshcd_is_device_present(hba)) { - ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err, + ufshcd_update_evt_hist(hba, + UFS_EVT_LINK_STARTUP_FAIL, 0); dev_err(hba->dev, "%s: Device not present\n", __func__); ret = -ENXIO; @@ -4509,7 +4587,8 @@ link_startup: * succeeds. So reset the local Uni-Pro and try again. */ if (ret && ufshcd_hba_enable(hba)) { - ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err, + ufshcd_update_evt_hist(hba, + UFS_EVT_LINK_STARTUP_FAIL, (u32)ret); goto out; } @@ -4517,7 +4596,8 @@ link_startup: if (ret) { /* failed to get the link up... retire */ - ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err, + ufshcd_update_evt_hist(hba, + UFS_EVT_LINK_STARTUP_FAIL, (u32)ret); goto out; } @@ -4551,7 +4631,7 @@ out: dev_err(hba->dev, "link startup failed %d\n", ret); ufshcd_print_host_state(hba); ufshcd_print_pwr_info(hba); - ufshcd_print_host_regs(hba); + ufshcd_print_evt_hist(hba); } return ret; } @@ -4906,7 +4986,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) dev_err(hba->dev, "OCS error from controller = %x for tag %d\n", ocs, lrbp->task_tag); - ufshcd_print_host_regs(hba); + ufshcd_print_evt_hist(hba); ufshcd_print_host_state(hba); break; } /* end of switch */ @@ -4964,9 +5044,11 @@ static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, struct scsi_cmnd *cmd; int result; int index; + bool update_scaling = false; for_each_set_bit(index, &completed_reqs, hba->nutrs) { lrbp = &hba->lrb[index]; + lrbp->in_use = false; lrbp->compl_time_stamp = ktime_get(); cmd = lrbp->cmd; if (cmd) { @@ -4979,15 +5061,17 @@ static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, /* Do not touch lrbp after scsi done */ cmd->scsi_done(cmd); __ufshcd_release(hba); + update_scaling = true; } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE || lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) { if (hba->dev_cmd.complete) { ufshcd_add_command_trace(hba, index, "dev_complete"); complete(hba->dev_cmd.complete); + update_scaling = true; } } - if (ufshcd_is_clkscaling_supported(hba)) + if (ufshcd_is_clkscaling_supported(hba) && update_scaling) hba->clk_scaling.active_reqs--; } @@ -5632,7 +5716,9 @@ static inline void ufshcd_schedule_eh_work(struct ufs_hba *hba) static void ufshcd_err_handling_prepare(struct ufs_hba *hba) { pm_runtime_get_sync(hba->dev); - if (pm_runtime_suspended(hba->dev)) { + if (pm_runtime_status_suspended(hba->dev) || hba->is_sys_suspended) { + enum ufs_pm_op pm_op; + /* * Don't assume anything of pm_runtime_get_sync(), if * resume fails, irq and clocks can be OFF, and powers @@ -5647,7 +5733,8 @@ static void ufshcd_err_handling_prepare(struct ufs_hba *hba) if (!ufshcd_is_clkgating_allowed(hba)) ufshcd_setup_clocks(hba, true); ufshcd_release(hba); - ufshcd_vops_resume(hba, UFS_RUNTIME_PM); + pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM; + ufshcd_vops_resume(hba, pm_op); } else { ufshcd_hold(hba, false); if (hba->clk_scaling.is_allowed) { @@ -5668,7 +5755,7 @@ static void ufshcd_err_handling_unprepare(struct ufs_hba *hba) static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba) { - return (hba->ufshcd_state == UFSHCD_STATE_ERROR || + return (!hba->is_powered || hba->ufshcd_state == UFSHCD_STATE_ERROR || (!(hba->saved_err || hba->saved_uic_err || hba->force_reset || ufshcd_is_link_broken(hba)))); } @@ -5681,6 +5768,7 @@ static void ufshcd_recover_pm_error(struct ufs_hba *hba) struct request_queue *q; int ret; + hba->is_sys_suspended = false; /* * Set RPM status of hba device to RPM_ACTIVE, * this also clears its runtime error. @@ -5739,11 +5827,13 @@ static void ufshcd_err_handler(struct work_struct *work) hba = container_of(work, struct ufs_hba, eh_work); + down(&hba->eh_sem); spin_lock_irqsave(hba->host->host_lock, flags); if (ufshcd_err_handling_should_stop(hba)) { if (hba->ufshcd_state != UFSHCD_STATE_ERROR) hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; spin_unlock_irqrestore(hba->host->host_lock, flags); + up(&hba->eh_sem); return; } ufshcd_set_eh_in_progress(hba); @@ -5751,20 +5841,18 @@ static void ufshcd_err_handler(struct work_struct *work) ufshcd_err_handling_prepare(hba); spin_lock_irqsave(hba->host->host_lock, flags); ufshcd_scsi_block_requests(hba); - /* - * A full reset and restore might have happened after preparation - * is finished, double check whether we should stop. - */ - if (ufshcd_err_handling_should_stop(hba)) { - if (hba->ufshcd_state != UFSHCD_STATE_ERROR) - hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; - goto out; - } hba->ufshcd_state = UFSHCD_STATE_RESET; /* Complete requests that have door-bell cleared by h/w */ ufshcd_complete_requests(hba); + /* + * A full reset and restore might have happened after preparation + * is finished, double check whether we should stop. + */ + if (ufshcd_err_handling_should_stop(hba)) + goto skip_err_handling; + if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) { bool ret; @@ -5772,17 +5860,10 @@ static void ufshcd_err_handler(struct work_struct *work) /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */ ret = ufshcd_quirk_dl_nac_errors(hba); spin_lock_irqsave(hba->host->host_lock, flags); - if (!ret && !hba->force_reset && ufshcd_is_link_active(hba)) + if (!ret && ufshcd_err_handling_should_stop(hba)) goto skip_err_handling; } - if (hba->force_reset || ufshcd_is_link_broken(hba) || - ufshcd_is_saved_err_fatal(hba) || - ((hba->saved_err & UIC_ERROR) && - (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR | - UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))) - needs_reset = true; - if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) || (hba->saved_uic_err && (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) { @@ -5791,7 +5872,7 @@ static void ufshcd_err_handler(struct work_struct *work) spin_unlock_irqrestore(hba->host->host_lock, flags); ufshcd_print_host_state(hba); ufshcd_print_pwr_info(hba); - ufshcd_print_host_regs(hba); + ufshcd_print_evt_hist(hba); ufshcd_print_tmrs(hba, hba->outstanding_tasks); ufshcd_print_trs(hba, hba->outstanding_reqs, pr_prdt); spin_lock_irqsave(hba->host->host_lock, flags); @@ -5802,8 +5883,14 @@ static void ufshcd_err_handler(struct work_struct *work) * transfers forcefully because they will get cleared during * host reset and restore */ - if (needs_reset) + if (hba->force_reset || ufshcd_is_link_broken(hba) || + ufshcd_is_saved_err_fatal(hba) || + ((hba->saved_err & UIC_ERROR) && + (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR | + UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))) { + needs_reset = true; goto do_reset; + } /* * If LINERESET was caught, UFS might have been put to PWM mode, @@ -5911,12 +5998,11 @@ skip_err_handling: dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x", __func__, hba->saved_err, hba->saved_uic_err); } - -out: ufshcd_clear_eh_in_progress(hba); spin_unlock_irqrestore(hba->host->host_lock, flags); ufshcd_scsi_unblock_requests(hba); ufshcd_err_handling_unprepare(hba); + up(&hba->eh_sem); } /** @@ -5936,7 +6022,7 @@ static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba) reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER); if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) && (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) { - ufshcd_update_reg_hist(&hba->ufs_stats.pa_err, reg); + ufshcd_update_evt_hist(hba, UFS_EVT_PA_ERR, reg); /* * To know whether this error is fatal or not, DB timeout * must be checked but this error is handled separately. @@ -5966,7 +6052,7 @@ static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba) reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER); if ((reg & UIC_DATA_LINK_LAYER_ERROR) && (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) { - ufshcd_update_reg_hist(&hba->ufs_stats.dl_err, reg); + ufshcd_update_evt_hist(hba, UFS_EVT_DL_ERR, reg); if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT) hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR; @@ -5985,7 +6071,7 @@ static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba) reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER); if ((reg & UIC_NETWORK_LAYER_ERROR) && (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) { - ufshcd_update_reg_hist(&hba->ufs_stats.nl_err, reg); + ufshcd_update_evt_hist(hba, UFS_EVT_NL_ERR, reg); hba->uic_error |= UFSHCD_UIC_NL_ERROR; retval |= IRQ_HANDLED; } @@ -5993,7 +6079,7 @@ static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba) reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER); if ((reg & UIC_TRANSPORT_LAYER_ERROR) && (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) { - ufshcd_update_reg_hist(&hba->ufs_stats.tl_err, reg); + ufshcd_update_evt_hist(hba, UFS_EVT_TL_ERR, reg); hba->uic_error |= UFSHCD_UIC_TL_ERROR; retval |= IRQ_HANDLED; } @@ -6001,7 +6087,7 @@ static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba) reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME); if ((reg & UIC_DME_ERROR) && (reg & UIC_DME_ERROR_CODE_MASK)) { - ufshcd_update_reg_hist(&hba->ufs_stats.dme_err, reg); + ufshcd_update_evt_hist(hba, UFS_EVT_DME_ERR, reg); hba->uic_error |= UFSHCD_UIC_DME_ERROR; retval |= IRQ_HANDLED; } @@ -6043,7 +6129,8 @@ static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba) irqreturn_t retval = IRQ_NONE; if (hba->errors & INT_FATAL_ERRORS) { - ufshcd_update_reg_hist(&hba->ufs_stats.fatal_err, hba->errors); + ufshcd_update_evt_hist(hba, UFS_EVT_FATAL_ERR, + hba->errors); queue_eh_work = true; } @@ -6060,7 +6147,7 @@ static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba) __func__, (hba->errors & UIC_HIBERNATE_ENTER) ? "Enter" : "Exit", hba->errors, ufshcd_get_upmcrs(hba)); - ufshcd_update_reg_hist(&hba->ufs_stats.auto_hibern8_err, + ufshcd_update_evt_hist(hba, UFS_EVT_AUTO_HIBERN8_ERR, hba->errors); ufshcd_set_link_broken(hba); queue_eh_work = true; @@ -6075,7 +6162,8 @@ static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba) hba->saved_uic_err |= hba->uic_error; /* dump controller state before resetting */ - if ((hba->saved_err & (INT_FATAL_ERRORS)) || + if ((hba->saved_err & + (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) || (hba->saved_uic_err && (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) { dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n", @@ -6407,8 +6495,12 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, init_completion(&wait); lrbp = &hba->lrb[tag]; - WARN_ON(lrbp->cmd); + if (unlikely(lrbp->in_use)) { + err = -EBUSY; + goto out; + } + WARN_ON(lrbp->cmd); lrbp->cmd = NULL; lrbp->sense_bufflen = 0; lrbp->sense_buffer = NULL; @@ -6480,6 +6572,7 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, } } +out: blk_put_request(req); out_unlock: up_read(&hba->clk_scaling_lock); @@ -6601,7 +6694,7 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd) out: hba->req_abort_count = 0; - ufshcd_update_reg_hist(&hba->ufs_stats.dev_reset, (u32)err); + ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err); if (!err) { err = SUCCESS; } else { @@ -6624,7 +6717,8 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) /** * ufshcd_try_to_abort_task - abort a specific task - * @cmd: SCSI command pointer + * @hba: Pointer to adapter instance + * @tag: Task tag/index to be aborted * * Abort the pending command in device by sending UFS_ABORT_TASK task management * command, and in host controller by clearing the door-bell register. There can @@ -6729,16 +6823,6 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) BUG(); } - /* - * Task abort to the device W-LUN is illegal. When this command - * will fail, due to spec violation, scsi err handling next step - * will be to send LU reset which, again, is a spec violation. - * To avoid these unnecessary/illegal step we skip to the last error - * handling stage: reset and restore. - */ - if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) - return ufshcd_eh_host_reset_handler(cmd); - ufshcd_hold(hba, false); reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); /* If command is already aborted/completed, return SUCCESS */ @@ -6759,10 +6843,10 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) * to reduce repeated printouts. For other aborted requests only print * basic details. */ - scsi_print_command(hba->lrb[tag].cmd); + scsi_print_command(cmd); if (!hba->req_abort_count) { - ufshcd_update_reg_hist(&hba->ufs_stats.task_abort, 0); - ufshcd_print_host_regs(hba); + ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag); + ufshcd_print_evt_hist(hba); ufshcd_print_host_state(hba); ufshcd_print_pwr_info(hba); ufshcd_print_trs(hba, 1 << tag, true); @@ -6778,6 +6862,29 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) goto cleanup; } + /* + * Task abort to the device W-LUN is illegal. When this command + * will fail, due to spec violation, scsi err handling next step + * will be to send LU reset which, again, is a spec violation. + * To avoid these unnecessary/illegal steps, first we clean up + * the lrb taken by this cmd and mark the lrb as in_use, then + * queue the eh_work and bail. + */ + if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) { + ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, lrbp->lun); + spin_lock_irqsave(host->host_lock, flags); + if (lrbp->cmd) { + __ufshcd_transfer_req_compl(hba, (1UL << tag)); + __set_bit(tag, &hba->outstanding_reqs); + lrbp->in_use = true; + hba->force_reset = true; + ufshcd_schedule_eh_work(hba); + } + + spin_unlock_irqrestore(host->host_lock, flags); + goto out; + } + /* Skip task abort in case previous aborts failed and report failure */ if (lrbp->req_abort_skip) err = -EIO; @@ -6841,11 +6948,12 @@ static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) /* Establish the link again and restore the device */ err = ufshcd_probe_hba(hba, false); - + if (!err) + ufshcd_clear_ua_wluns(hba); out: if (err) dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err); - ufshcd_update_reg_hist(&hba->ufs_stats.host_reset, (u32)err); + ufshcd_update_evt_hist(hba, UFS_EVT_HOST_RESET, (u32)err); return err; } @@ -6879,7 +6987,7 @@ static int ufshcd_reset_and_restore(struct ufs_hba *hba) do { /* Reset the attached device */ - ufshcd_vops_device_reset(hba); + ufshcd_device_reset(hba); err = ufshcd_host_reset_and_restore(hba); } while (err && --retries); @@ -6891,6 +6999,7 @@ static int ufshcd_reset_and_restore(struct ufs_hba *hba) */ scsi_report_bus_reset(hba->host, 0); if (err) { + hba->ufshcd_state = UFSHCD_STATE_ERROR; hba->saved_err |= saved_err; hba->saved_uic_err |= saved_uic_err; } @@ -7092,7 +7201,6 @@ static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev) static int ufshcd_scsi_add_wlus(struct ufs_hba *hba) { int ret = 0; - struct scsi_device *sdev_rpmb; struct scsi_device *sdev_boot; hba->sdev_ufs_device = __scsi_add_device(hba->host, 0, 0, @@ -7105,14 +7213,14 @@ static int ufshcd_scsi_add_wlus(struct ufs_hba *hba) ufshcd_blk_pm_runtime_init(hba->sdev_ufs_device); scsi_device_put(hba->sdev_ufs_device); - sdev_rpmb = __scsi_add_device(hba->host, 0, 0, + hba->sdev_rpmb = __scsi_add_device(hba->host, 0, 0, ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL); - if (IS_ERR(sdev_rpmb)) { - ret = PTR_ERR(sdev_rpmb); + if (IS_ERR(hba->sdev_rpmb)) { + ret = PTR_ERR(hba->sdev_rpmb); goto remove_sdev_ufs_device; } - ufshcd_blk_pm_runtime_init(sdev_rpmb); - scsi_device_put(sdev_rpmb); + ufshcd_blk_pm_runtime_init(hba->sdev_rpmb); + scsi_device_put(hba->sdev_rpmb); sdev_boot = __scsi_add_device(hba->host, 0, 0, ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL); @@ -7636,6 +7744,63 @@ out: return ret; } +static int +ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp); + +static int ufshcd_clear_ua_wlun(struct ufs_hba *hba, u8 wlun) +{ + struct scsi_device *sdp; + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(hba->host->host_lock, flags); + if (wlun == UFS_UPIU_UFS_DEVICE_WLUN) + sdp = hba->sdev_ufs_device; + else if (wlun == UFS_UPIU_RPMB_WLUN) + sdp = hba->sdev_rpmb; + else + BUG(); + if (sdp) { + ret = scsi_device_get(sdp); + if (!ret && !scsi_device_online(sdp)) { + ret = -ENODEV; + scsi_device_put(sdp); + } + } else { + ret = -ENODEV; + } + spin_unlock_irqrestore(hba->host->host_lock, flags); + if (ret) + goto out_err; + + ret = ufshcd_send_request_sense(hba, sdp); + scsi_device_put(sdp); +out_err: + if (ret) + dev_err(hba->dev, "%s: UAC clear LU=%x ret = %d\n", + __func__, wlun, ret); + return ret; +} + +static int ufshcd_clear_ua_wluns(struct ufs_hba *hba) +{ + int ret = 0; + + if (!hba->wlun_dev_clr_ua) + goto out; + + ret = ufshcd_clear_ua_wlun(hba, UFS_UPIU_UFS_DEVICE_WLUN); + if (!ret) + ret = ufshcd_clear_ua_wlun(hba, UFS_UPIU_RPMB_WLUN); + if (!ret) + hba->wlun_dev_clr_ua = false; +out: + if (ret) + dev_err(hba->dev, "%s: Failed to clear UAC WLUNS ret = %d\n", + __func__, ret); + return ret; +} + /** * ufshcd_probe_hba - probe hba to detect device and initialize * @hba: per-adapter instance @@ -7739,8 +7904,10 @@ static void ufshcd_async_scan(void *data, async_cookie_t cookie) struct ufs_hba *hba = (struct ufs_hba *)data; int ret; + down(&hba->eh_sem); /* Initialize hba, detect and initialize UFS device */ ret = ufshcd_probe_hba(hba, true); + up(&hba->eh_sem); if (ret) goto out; @@ -7755,6 +7922,8 @@ out: pm_runtime_put_sync(hba->dev); ufshcd_exit_clk_scaling(hba); ufshcd_hba_exit(hba); + } else { + ufshcd_clear_ua_wluns(hba); } } @@ -7895,7 +8064,7 @@ static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg) { int ret = 0; - if (!vreg || !vreg->enabled) + if (!vreg || !vreg->enabled || vreg->always_on) goto out; ret = regulator_disable(vreg->reg); @@ -7988,8 +8157,7 @@ static int ufshcd_init_hba_vreg(struct ufs_hba *hba) return 0; } -static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on, - bool skip_ref_clk) +static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on) { int ret = 0; struct ufs_clk_info *clki; @@ -8007,7 +8175,12 @@ static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on, list_for_each_entry(clki, head, list) { if (!IS_ERR_OR_NULL(clki->clk)) { - if (skip_ref_clk && !strcmp(clki->name, "ref_clk")) + /* + * Don't disable clocks which are needed + * to keep the link active. + */ + if (ufshcd_is_link_active(hba) && + clki->keep_link_active) continue; clk_state_changed = on ^ clki->enabled; @@ -8052,11 +8225,6 @@ out: return ret; } -static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on) -{ - return __ufshcd_setup_clocks(hba, on, false); -} - static int ufshcd_init_clocks(struct ufs_hba *hba) { int ret = 0; @@ -8113,15 +8281,9 @@ static int ufshcd_variant_hba_init(struct ufs_hba *hba) err = ufshcd_vops_init(hba); if (err) - goto out; - - err = ufshcd_vops_setup_regulators(hba, true); - if (err) - ufshcd_vops_exit(hba); -out: - if (err) dev_err(hba->dev, "%s: variant %s init failed err %d\n", __func__, ufshcd_get_var_name(hba), err); +out: return err; } @@ -8130,8 +8292,6 @@ static void ufshcd_variant_hba_exit(struct ufs_hba *hba) if (!hba->vops) return; - ufshcd_vops_setup_regulators(hba, false); - ufshcd_vops_exit(hba); } @@ -8273,13 +8433,7 @@ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, * handling context. */ hba->host->eh_noresume = 1; - if (hba->wlun_dev_clr_ua) { - ret = ufshcd_send_request_sense(hba, sdp); - if (ret) - goto out; - /* Unit attention condition is cleared now */ - hba->wlun_dev_clr_ua = false; - } + ufshcd_clear_ua_wluns(hba); cmd[4] = pwr_mode << 4; @@ -8300,7 +8454,7 @@ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, if (!ret) hba->curr_dev_pwr_mode = pwr_mode; -out: + scsi_device_put(sdp); hba->host->eh_noresume = 0; return ret; @@ -8327,7 +8481,8 @@ static int ufshcd_link_state_transition(struct ufs_hba *hba, } /* * If autobkops is enabled, link can't be turned off because - * turning off the link would also turn off the device. + * turning off the link would also turn off the device, except in the + * case of DeepSleep where the device is expected to remain powered. */ else if ((req_link_state == UIC_LINK_OFF_STATE) && (!check_for_bkops || !hba->auto_bkops_enabled)) { @@ -8337,6 +8492,9 @@ static int ufshcd_link_state_transition(struct ufs_hba *hba, * put the link in low power mode is to send the DME end point * to device and then send the DME reset command to local * unipro. But putting the link in hibern8 is much faster. + * + * Note also that putting the link in Hibern8 is a requirement + * for entering DeepSleep. */ ret = ufshcd_uic_hibern8_enter(hba); if (ret) { @@ -8440,13 +8598,13 @@ out: static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba) { - if (ufshcd_is_link_off(hba)) + if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba)) ufshcd_setup_hba_vreg(hba, false); } static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba) { - if (ufshcd_is_link_off(hba)) + if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba)) ufshcd_setup_hba_vreg(hba, true); } @@ -8469,6 +8627,7 @@ static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba) static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) { int ret = 0; + int check_for_bkops; enum ufs_pm_level pm_lvl; enum ufs_dev_pwr_mode req_dev_pwr_mode; enum uic_link_state req_link_state; @@ -8540,11 +8699,9 @@ static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) } if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) { - if ((ufshcd_is_runtime_pm(pm_op) && !hba->auto_bkops_enabled) || - !ufshcd_is_runtime_pm(pm_op)) { + if (!ufshcd_is_runtime_pm(pm_op)) /* ensure that bkops is disabled */ ufshcd_disable_auto_bkops(hba); - } if (!hba->dev_info.b_rpm_dev_flush_capable) { ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode); @@ -8554,7 +8711,13 @@ static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) } flush_work(&hba->eeh_work); - ret = ufshcd_link_state_transition(hba, req_link_state, 1); + + /* + * In the case of DeepSleep, the device is expected to remain powered + * with the link off, so do not check for bkops. + */ + check_for_bkops = !ufshcd_is_ufs_dev_deepsleep(hba); + ret = ufshcd_link_state_transition(hba, req_link_state, check_for_bkops); if (ret) goto set_dev_active; @@ -8575,11 +8738,7 @@ disable_clks: */ ufshcd_disable_irq(hba); - if (!ufshcd_is_link_active(hba)) - ufshcd_setup_clocks(hba, false); - else - /* If link is active, device ref_clk can't be switched off */ - __ufshcd_setup_clocks(hba, false, true); + ufshcd_setup_clocks(hba, false); if (ufshcd_is_clkgating_allowed(hba)) { hba->clk_gating.state = CLKS_OFF; @@ -8595,11 +8754,25 @@ set_link_active: if (hba->clk_scaling.is_allowed) ufshcd_resume_clkscaling(hba); ufshcd_vreg_set_hpm(hba); + /* + * Device hardware reset is required to exit DeepSleep. Also, for + * DeepSleep, the link is off so host reset and restore will be done + * further below. + */ + if (ufshcd_is_ufs_dev_deepsleep(hba)) { + ufshcd_device_reset(hba); + WARN_ON(!ufshcd_is_link_off(hba)); + } if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba)) ufshcd_set_link_active(hba); else if (ufshcd_is_link_off(hba)) ufshcd_host_reset_and_restore(hba); set_dev_active: + /* Can also get here needing to exit DeepSleep */ + if (ufshcd_is_ufs_dev_deepsleep(hba)) { + ufshcd_device_reset(hba); + ufshcd_host_reset_and_restore(hba); + } if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE)) ufshcd_disable_auto_bkops(hba); enable_gating: @@ -8617,7 +8790,7 @@ out: hba->pm_op_in_progress = 0; if (ret) - ufshcd_update_reg_hist(&hba->ufs_stats.suspend_err, (u32)ret); + ufshcd_update_evt_hist(hba, UFS_EVT_SUSPEND_ERR, (u32)ret); return ret; } @@ -8661,6 +8834,9 @@ static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) if (ret) goto disable_vreg; + /* For DeepSleep, the only supported option is to have the link off */ + WARN_ON(ufshcd_is_ufs_dev_deepsleep(hba) && !ufshcd_is_link_off(hba)); + if (ufshcd_is_link_hibern8(hba)) { ret = ufshcd_uic_hibern8_exit(hba); if (!ret) { @@ -8674,6 +8850,8 @@ static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) /* * A full initialization of the host and the device is * required since the link was put to off during suspend. + * Note, in the case of DeepSleep, the device will exit + * DeepSleep due to device reset. */ ret = ufshcd_reset_and_restore(hba); /* @@ -8736,7 +8914,7 @@ disable_irq_and_vops_clks: out: hba->pm_op_in_progress = 0; if (ret) - ufshcd_update_reg_hist(&hba->ufs_stats.resume_err, (u32)ret); + ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)ret); return ret; } @@ -8753,6 +8931,7 @@ int ufshcd_system_suspend(struct ufs_hba *hba) int ret = 0; ktime_t start = ktime_get(); + down(&hba->eh_sem); if (!hba || !hba->is_powered) return 0; @@ -8783,6 +8962,8 @@ out: hba->curr_dev_pwr_mode, hba->uic_link_state); if (!ret) hba->is_sys_suspended = true; + else + up(&hba->eh_sem); return ret; } EXPORT_SYMBOL(ufshcd_system_suspend); @@ -8799,8 +8980,10 @@ int ufshcd_system_resume(struct ufs_hba *hba) int ret = 0; ktime_t start = ktime_get(); - if (!hba) + if (!hba) { + up(&hba->eh_sem); return -EINVAL; + } if (!hba->is_powered || pm_runtime_suspended(hba->dev)) /* @@ -8816,6 +8999,7 @@ out: hba->curr_dev_pwr_mode, hba->uic_link_state); if (!ret) hba->is_sys_suspended = false; + up(&hba->eh_sem); return ret; } EXPORT_SYMBOL(ufshcd_system_resume); @@ -8907,6 +9091,7 @@ int ufshcd_shutdown(struct ufs_hba *hba) { int ret = 0; + down(&hba->eh_sem); if (!hba->is_powered) goto out; @@ -8919,6 +9104,8 @@ int ufshcd_shutdown(struct ufs_hba *hba) out: if (ret) dev_err(hba->dev, "%s failed, err %d\n", __func__, ret); + hba->is_powered = false; + up(&hba->eh_sem); /* allow force shutdown even in case of errors */ return 0; } @@ -9114,6 +9301,8 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) INIT_WORK(&hba->eh_work, ufshcd_err_handler); INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler); + sema_init(&hba->eh_sem, 1); + /* Initialize UIC command mutex */ mutex_init(&hba->uic_cmd_mutex); @@ -9177,7 +9366,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) } /* Reset the attached device */ - ufshcd_vops_device_reset(hba); + ufshcd_device_reset(hba); ufshcd_init_crypto(hba); @@ -9185,7 +9374,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) err = ufshcd_hba_enable(hba); if (err) { dev_err(hba->dev, "Host controller enable failed\n"); - ufshcd_print_host_regs(hba); + ufshcd_print_evt_hist(hba); ufshcd_print_host_state(hba); goto free_tmf_queue; } diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h index e0f00a42371c..aa9ea3552323 100644 --- a/drivers/scsi/ufs/ufshcd.h +++ b/drivers/scsi/ufs/ufshcd.h @@ -58,6 +58,29 @@ enum dev_cmd_type { DEV_CMD_TYPE_QUERY = 0x1, }; +enum ufs_event_type { + /* uic specific errors */ + UFS_EVT_PA_ERR = 0, + UFS_EVT_DL_ERR, + UFS_EVT_NL_ERR, + UFS_EVT_TL_ERR, + UFS_EVT_DME_ERR, + + /* fatal errors */ + UFS_EVT_AUTO_HIBERN8_ERR, + UFS_EVT_FATAL_ERR, + UFS_EVT_LINK_STARTUP_FAIL, + UFS_EVT_RESUME_ERR, + UFS_EVT_SUSPEND_ERR, + + /* abnormal events */ + UFS_EVT_DEV_RESET, + UFS_EVT_HOST_RESET, + UFS_EVT_ABORT, + + UFS_EVT_CNT, +}; + /** * struct uic_command - UIC command structure * @command: UIC command @@ -116,16 +139,22 @@ enum uic_link_state { ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE) #define ufshcd_set_ufs_dev_poweroff(h) \ ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE) +#define ufshcd_set_ufs_dev_deepsleep(h) \ + ((h)->curr_dev_pwr_mode = UFS_DEEPSLEEP_PWR_MODE) #define ufshcd_is_ufs_dev_active(h) \ ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE) #define ufshcd_is_ufs_dev_sleep(h) \ ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE) #define ufshcd_is_ufs_dev_poweroff(h) \ ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE) +#define ufshcd_is_ufs_dev_deepsleep(h) \ + ((h)->curr_dev_pwr_mode == UFS_DEEPSLEEP_PWR_MODE) /* * UFS Power management levels. - * Each level is in increasing order of power savings. + * Each level is in increasing order of power savings, except DeepSleep + * which is lower than PowerDown with power on but not PowerDown with + * power off. */ enum ufs_pm_level { UFS_PM_LVL_0, /* UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE */ @@ -134,6 +163,7 @@ enum ufs_pm_level { UFS_PM_LVL_3, /* UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE */ UFS_PM_LVL_4, /* UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE */ UFS_PM_LVL_5, /* UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE */ + UFS_PM_LVL_6, /* UFS_DEEPSLEEP_PWR_MODE, UIC_LINK_OFF_STATE */ UFS_PM_LVL_MAX }; @@ -165,6 +195,7 @@ struct ufs_pm_lvl_states { * @crypto_key_slot: the key slot to use for inline crypto (-1 if none) * @data_unit_num: the data unit number for the first block for inline crypto * @req_abort_skip: skip request abort task flag + * @in_use: indicates that this lrb is still in use */ struct ufshcd_lrb { struct utp_transfer_req_desc *utr_descriptor_ptr; @@ -194,6 +225,7 @@ struct ufshcd_lrb { #endif bool req_abort_skip; + bool in_use; }; /** @@ -229,6 +261,8 @@ struct ufs_dev_cmd { * @max_freq: maximum frequency supported by the clock * @min_freq: min frequency that can be used for clock scaling * @curr_freq: indicates the current frequency that it is set to + * @keep_link_active: indicates that the clk should not be disabled if + link is active * @enabled: variable to check against multiple enable/disable */ struct ufs_clk_info { @@ -238,6 +272,7 @@ struct ufs_clk_info { u32 max_freq; u32 min_freq; u32 curr_freq; + bool keep_link_active; bool enabled; }; @@ -269,7 +304,6 @@ struct ufs_pwr_mode_info { * @get_ufs_hci_version: called to get UFS HCI version * @clk_scale_notify: notifies that clks are scaled up/down * @setup_clocks: called before touching any of the controller registers - * @setup_regulators: called before accessing the host controller * @hce_enable_notify: called before and after HCE enable bit is set to allow * variant specific Uni-Pro initialization. * @link_startup_notify: called before and after Link startup is carried out @@ -289,6 +323,7 @@ struct ufs_pwr_mode_info { * @phy_initialization: used to initialize phys * @device_reset: called to issue a reset pulse on the UFS device * @program_key: program or evict an inline encryption key + * @event_notify: called to notify important events */ struct ufs_hba_variant_ops { const char *name; @@ -299,7 +334,6 @@ struct ufs_hba_variant_ops { enum ufs_notify_change_status); int (*setup_clocks)(struct ufs_hba *, bool, enum ufs_notify_change_status); - int (*setup_regulators)(struct ufs_hba *, bool); int (*hce_enable_notify)(struct ufs_hba *, enum ufs_notify_change_status); int (*link_startup_notify)(struct ufs_hba *, @@ -318,12 +352,14 @@ struct ufs_hba_variant_ops { int (*resume)(struct ufs_hba *, enum ufs_pm_op); void (*dbg_register_dump)(struct ufs_hba *hba); int (*phy_initialization)(struct ufs_hba *); - void (*device_reset)(struct ufs_hba *hba); + int (*device_reset)(struct ufs_hba *hba); void (*config_scaling_param)(struct ufs_hba *hba, struct devfreq_dev_profile *profile, void *data); int (*program_key)(struct ufs_hba *hba, const union ufs_crypto_cfg_entry *cfg, int slot); + void (*event_notify)(struct ufs_hba *hba, + enum ufs_event_type evt, void *data); }; /* clock gating state */ @@ -382,6 +418,7 @@ struct ufs_saved_pwr_info { * @workq: workqueue to schedule devfreq suspend/resume work * @suspend_work: worker to suspend devfreq * @resume_work: worker to resume devfreq + * @min_gear: lowest HS gear to scale down to * @is_allowed: tracks if scaling is currently allowed or not * @is_busy_started: tracks if busy period has started or not * @is_suspended: tracks if devfreq is suspended or not @@ -396,22 +433,23 @@ struct ufs_clk_scaling { struct workqueue_struct *workq; struct work_struct suspend_work; struct work_struct resume_work; + u32 min_gear; bool is_allowed; bool is_busy_started; bool is_suspended; }; -#define UFS_ERR_REG_HIST_LENGTH 8 +#define UFS_EVENT_HIST_LENGTH 8 /** - * struct ufs_err_reg_hist - keeps history of errors + * struct ufs_event_hist - keeps history of errors * @pos: index to indicate cyclic buffer position * @reg: cyclic buffer for registers value * @tstamp: cyclic buffer for time stamp */ -struct ufs_err_reg_hist { +struct ufs_event_hist { int pos; - u32 reg[UFS_ERR_REG_HIST_LENGTH]; - ktime_t tstamp[UFS_ERR_REG_HIST_LENGTH]; + u32 val[UFS_EVENT_HIST_LENGTH]; + ktime_t tstamp[UFS_EVENT_HIST_LENGTH]; }; /** @@ -422,19 +460,6 @@ struct ufs_err_reg_hist { * reset this after link-startup. * @last_hibern8_exit_tstamp: Set time after the hibern8 exit. * Clear after the first successful command completion. - * @pa_err: tracks pa-uic errors - * @dl_err: tracks dl-uic errors - * @nl_err: tracks nl-uic errors - * @tl_err: tracks tl-uic errors - * @dme_err: tracks dme errors - * @auto_hibern8_err: tracks auto-hibernate errors - * @fatal_err: tracks fatal errors - * @linkup_err: tracks link-startup errors - * @resume_err: tracks resume errors - * @suspend_err: tracks suspend errors - * @dev_reset: tracks device reset events - * @host_reset: tracks host reset events - * @tsk_abort: tracks task abort events */ struct ufs_stats { u32 last_intr_status; @@ -442,25 +467,7 @@ struct ufs_stats { u32 hibern8_exit_cnt; ktime_t last_hibern8_exit_tstamp; - - /* uic specific errors */ - struct ufs_err_reg_hist pa_err; - struct ufs_err_reg_hist dl_err; - struct ufs_err_reg_hist nl_err; - struct ufs_err_reg_hist tl_err; - struct ufs_err_reg_hist dme_err; - - /* fatal errors */ - struct ufs_err_reg_hist auto_hibern8_err; - struct ufs_err_reg_hist fatal_err; - struct ufs_err_reg_hist link_startup_err; - struct ufs_err_reg_hist resume_err; - struct ufs_err_reg_hist suspend_err; - - /* abnormal events */ - struct ufs_err_reg_hist dev_reset; - struct ufs_err_reg_hist host_reset; - struct ufs_err_reg_hist task_abort; + struct ufs_event_hist event[UFS_EVT_CNT]; }; enum ufshcd_quirks { @@ -594,6 +601,21 @@ enum ufshcd_caps { * inline crypto engine, if it is present */ UFSHCD_CAP_CRYPTO = 1 << 8, + + /* + * This capability allows the controller regulators to be put into + * lpm mode aggressively during clock gating. + * This would increase power savings. + */ + UFSHCD_CAP_AGGR_POWER_COLLAPSE = 1 << 9, + + /* + * This capability allows the host controller driver to use DeepSleep, + * if it is supported by the UFS device. The host controller driver must + * support device hardware reset via the hba->device_reset() callback, + * in order to exit DeepSleep state. + */ + UFSHCD_CAP_DEEPSLEEP = 1 << 10, }; struct ufs_hba_variant_params { @@ -683,6 +705,7 @@ struct ufs_hba { * "UFS device" W-LU. */ struct scsi_device *sdev_ufs_device; + struct scsi_device *sdev_rpmb; enum ufs_dev_pwr_mode curr_dev_pwr_mode; enum uic_link_state uic_link_state; @@ -730,6 +753,7 @@ struct ufs_hba { u32 intr_mask; u16 ee_ctrl_mask; bool is_powered; + struct semaphore eh_sem; /* Work Queues */ struct workqueue_struct *eh_wq; @@ -831,6 +855,12 @@ return true; #endif } +static inline bool ufshcd_can_aggressive_pc(struct ufs_hba *hba) +{ + return !!(ufshcd_is_link_hibern8(hba) && + (hba->caps & UFSHCD_CAP_AGGR_POWER_COLLAPSE)); +} + static inline bool ufshcd_is_auto_hibern8_supported(struct ufs_hba *hba) { return (hba->capabilities & MASK_AUTO_HIBERN8_SUPPORT) && @@ -882,8 +912,7 @@ int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask, u32 val, unsigned long interval_us, unsigned long timeout_ms); void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk); -void ufshcd_update_reg_hist(struct ufs_err_reg_hist *reg_hist, - u32 reg); +void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val); static inline void check_upiu_size(void) { @@ -930,6 +959,9 @@ extern int ufshcd_runtime_idle(struct ufs_hba *hba); extern int ufshcd_system_suspend(struct ufs_hba *hba); extern int ufshcd_system_resume(struct ufs_hba *hba); extern int ufshcd_shutdown(struct ufs_hba *hba); +extern int ufshcd_dme_configure_adapt(struct ufs_hba *hba, + int agreed_gear, + int adapt_val); extern int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel, u8 attr_set, u32 mib_val, u8 peer); extern int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel, @@ -1076,6 +1108,14 @@ static inline int ufshcd_vops_clk_scale_notify(struct ufs_hba *hba, return 0; } +static inline void ufshcd_vops_event_notify(struct ufs_hba *hba, + enum ufs_event_type evt, + void *data) +{ + if (hba->vops && hba->vops->event_notify) + hba->vops->event_notify(hba, evt, data); +} + static inline int ufshcd_vops_setup_clocks(struct ufs_hba *hba, bool on, enum ufs_notify_change_status status) { @@ -1084,14 +1124,6 @@ static inline int ufshcd_vops_setup_clocks(struct ufs_hba *hba, bool on, return 0; } -static inline int ufshcd_vops_setup_regulators(struct ufs_hba *hba, bool status) -{ - if (hba->vops && hba->vops->setup_regulators) - return hba->vops->setup_regulators(hba, status); - - return 0; -} - static inline int ufshcd_vops_hce_enable_notify(struct ufs_hba *hba, bool status) { @@ -1109,6 +1141,14 @@ static inline int ufshcd_vops_link_startup_notify(struct ufs_hba *hba, return 0; } +static inline int ufshcd_vops_phy_initialization(struct ufs_hba *hba) +{ + if (hba->vops && hba->vops->phy_initialization) + return hba->vops->phy_initialization(hba); + + return 0; +} + static inline int ufshcd_vops_pwr_change_notify(struct ufs_hba *hba, bool status, struct ufs_pa_layer_attr *dev_max_params, @@ -1178,13 +1218,12 @@ static inline void ufshcd_vops_dbg_register_dump(struct ufs_hba *hba) hba->vops->dbg_register_dump(hba); } -static inline void ufshcd_vops_device_reset(struct ufs_hba *hba) +static inline int ufshcd_vops_device_reset(struct ufs_hba *hba) { - if (hba->vops && hba->vops->device_reset) { - hba->vops->device_reset(hba); - ufshcd_set_ufs_dev_active(hba); - ufshcd_update_reg_hist(&hba->ufs_stats.dev_reset, 0); - } + if (hba->vops && hba->vops->device_reset) + return hba->vops->device_reset(hba); + + return -EOPNOTSUPP; } static inline void ufshcd_vops_config_scaling_param(struct ufs_hba *hba, diff --git a/drivers/scsi/ufs/unipro.h b/drivers/scsi/ufs/unipro.h index f6b52ce36de6..8e9e486a4f7b 100644 --- a/drivers/scsi/ufs/unipro.h +++ b/drivers/scsi/ufs/unipro.h @@ -237,8 +237,10 @@ enum ufs_unipro_ver { UFS_UNIPRO_VER_RESERVED = 0, UFS_UNIPRO_VER_1_40 = 1, /* UniPro version 1.40 */ UFS_UNIPRO_VER_1_41 = 2, /* UniPro version 1.41 */ - UFS_UNIPRO_VER_1_6 = 3, /* UniPro version 1.6 */ - UFS_UNIPRO_VER_MAX = 4, /* UniPro unsupported version */ + UFS_UNIPRO_VER_1_6 = 3, /* UniPro version 1.6 */ + UFS_UNIPRO_VER_1_61 = 4, /* UniPro version 1.61 */ + UFS_UNIPRO_VER_1_8 = 5, /* UniPro version 1.8 */ + UFS_UNIPRO_VER_MAX = 6, /* UniPro unsupported version */ /* UniPro version field mask in PA_LOCALVERINFO */ UFS_UNIPRO_VER_MASK = 0xF, }; |