scsi: ufs: core: Add support for TX Equalization

MIPI Unipro3.0 introduced PA_TxEQGnSetting and PA_PreCodeEn attributes for
TX Equalization and Pre-Coding. It is Host Software's responsibility to
configure these attributes for both host and device before initiating Power
Mode Change to High-Speed Gears.

MIPI Unipro3.0 also introduced TX Equalization Training (EQTR) to identify
optimal TX Equalization settings for use by both Host's and Device's
UniPro. TX EQTR shall be initiated from the most reliable High-Speed Gear
(HS-G1) targeting High-Speed Gears (HS-G4 to HS-G6).

Implement TX Equalization configuration and TX EQTR procedure as defined
in UFSHCI v5.0 specification. The TX EQTR procedure determines the optimal
TX Equalization settings by iterating through all possible PreShoot and
DeEmphasis combinations and selecting the best combinations for both Host
and Device based on Figure of Merit (FOM) evaluation.

Reviewed-by: Bean Huo <beanhuo@micron.com>
Reviewed-by: Bart Van Assche <bvanassche@acm.org>
Signed-off-by: Can Guo <can.guo@oss.qualcomm.com>
Reviewed-by: Peter Wang <peter.wang@mediatek.com>
Link: https://patch.msgid.link/20260325152154.1604082-5-can.guo@oss.qualcomm.com
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>

2 files changed, 247 insertions, 2 deletions

diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h
index 16facaee3e77..35b1288327d0 100644
--- a/include/ufs/ufshcd.h
+++ b/include/ufs/ufshcd.h
@@ -287,6 +287,84 @@ struct ufs_pwr_mode_info {
 	struct ufs_pa_layer_attr info;
 };
 
+#define UFS_MAX_LANES	2
+
+/**
+ * struct tx_eqtr_iter - TX Equalization Training iterator
+ * @preshoot_bitmap: PreShoot bitmap
+ * @deemphasis_bitmap: DeEmphasis bitmap
+ * @preshoot: PreShoot value
+ * @deemphasis: DeEmphasis value
+ * @fom: Figure-of-Merit read out from RX_FOM
+ * @is_updated: Flag to indicate if updated since previous iteration
+ */
+struct tx_eqtr_iter {
+	unsigned long preshoot_bitmap;
+	unsigned long deemphasis_bitmap;
+	u8 preshoot;
+	u8 deemphasis;
+	u8 fom[UFS_MAX_LANES];
+	bool is_updated;
+};
+
+/**
+ * struct ufshcd_tx_eq_settings - TX Equalization settings
+ * @preshoot: PreShoot value
+ * @deemphasis: DeEmphasis value
+ * @fom_val: Figure-of-Merit value read out from RX_FOM (Bit[6:0])
+ * @precode_en: Flag to indicate whether need to enable pre-coding
+ */
+struct ufshcd_tx_eq_settings {
+	u8 preshoot;
+	u8 deemphasis;
+	u8 fom_val;
+	bool precode_en;
+};
+
+/**
+ * struct ufshcd_tx_eqtr_data - Data used during TX Equalization Training procedure
+ * @host: Optimal TX EQ settings identified for host TX Lanes during TX EQTR
+ * @device: Optimal TX EQ settings identified for device TX Lanes during TX EQTR
+ * @host_fom: Host TX EQTR FOM record
+ * @device_fom: Device TX EQTR FOM record
+ */
+struct ufshcd_tx_eqtr_data {
+	struct ufshcd_tx_eq_settings host[UFS_MAX_LANES];
+	struct ufshcd_tx_eq_settings device[UFS_MAX_LANES];
+	u8 host_fom[UFS_MAX_LANES][TX_HS_NUM_PRESHOOT][TX_HS_NUM_DEEMPHASIS];
+	u8 device_fom[UFS_MAX_LANES][TX_HS_NUM_PRESHOOT][TX_HS_NUM_DEEMPHASIS];
+};
+
+/**
+ * struct ufshcd_tx_eqtr_record - TX Equalization Training record
+ * @host_fom: Host TX EQTR FOM record
+ * @device_fom: Device TX EQTR FOM record
+ * @last_record_ts: Timestamp of the most recent TX EQTR record
+ * @last_record_index: Index of the most recent TX EQTR record
+ */
+struct ufshcd_tx_eqtr_record {
+	u8 host_fom[UFS_MAX_LANES][TX_HS_NUM_PRESHOOT][TX_HS_NUM_DEEMPHASIS];
+	u8 device_fom[UFS_MAX_LANES][TX_HS_NUM_PRESHOOT][TX_HS_NUM_DEEMPHASIS];
+	ktime_t last_record_ts;
+	u16 last_record_index;
+};
+
+/**
+ * struct ufshcd_tx_eq_params - TX Equalization parameters structure
+ * @host: TX EQ settings for host TX Lanes
+ * @device: TX EQ settings for device TX Lanes
+ * @eqtr_record: Pointer to TX EQTR record
+ * @is_valid: True if parameter contains valid TX Equalization settings
+ * @is_applied: True if settings have been applied to UniPro of both sides
+ */
+struct ufshcd_tx_eq_params {
+	struct ufshcd_tx_eq_settings host[UFS_MAX_LANES];
+	struct ufshcd_tx_eq_settings device[UFS_MAX_LANES];
+	struct ufshcd_tx_eqtr_record *eqtr_record;
+	bool is_valid;
+	bool is_applied;
+};
+
 /**
  * struct ufs_hba_variant_ops - variant specific callbacks
  * @name: variant name
@@ -330,6 +408,11 @@ struct ufs_pwr_mode_info {
  * @config_esi: called to config Event Specific Interrupt
  * @config_scsi_dev: called to configure SCSI device parameters
  * @freq_to_gear_speed: called to map clock frequency to the max supported gear speed
+ * @apply_tx_eqtr_settings: called to apply settings for TX Equalization
+ *	Training settings.
+ * @get_rx_fom: called to get Figure of Merit (FOM) value.
+ * @tx_eqtr_notify: called before and after TX Equalization Training procedure
+ *	to allow platform vendor specific configs to take place.
  */
 struct ufs_hba_variant_ops {
 	const char *name;
@@ -381,6 +464,17 @@ struct ufs_hba_variant_ops {
 	int	(*config_esi)(struct ufs_hba *hba);
 	void	(*config_scsi_dev)(struct scsi_device *sdev);
 	u32	(*freq_to_gear_speed)(struct ufs_hba *hba, unsigned long freq);
+	int	(*get_rx_fom)(struct ufs_hba *hba,
+			      struct ufs_pa_layer_attr *pwr_mode,
+			      struct tx_eqtr_iter *h_iter,
+			      struct tx_eqtr_iter *d_iter);
+	int	(*apply_tx_eqtr_settings)(struct ufs_hba *hba,
+					  struct ufs_pa_layer_attr *pwr_mode,
+					  struct tx_eqtr_iter *h_iter,
+					  struct tx_eqtr_iter *d_iter);
+	int	(*tx_eqtr_notify)(struct ufs_hba *hba,
+				  enum ufs_notify_change_status status,
+				  struct ufs_pa_layer_attr *pwr_mode);
 };
 
 /* clock gating state  */
@@ -779,6 +873,13 @@ enum ufshcd_caps {
 	 * WriteBooster when scaling the clock down.
 	 */
 	UFSHCD_CAP_WB_WITH_CLK_SCALING			= 1 << 12,
+
+	/*
+	 * This capability allows the host controller driver to apply TX
+	 * Equalization settings discovered from UFS attributes, variant
+	 * specific operations and TX Equaliztion Training procedure.
+	 */
+	UFSHCD_CAP_TX_EQUALIZATION			= 1 << 13,
 };
 
 struct ufs_hba_variant_params {
@@ -955,6 +1056,15 @@ enum ufshcd_mcq_opr {
  * @dev_lvl_exception_count: count of device level exceptions since last reset
  * @dev_lvl_exception_id: vendor specific information about the device level exception event.
  * @rpmbs: list of OP-TEE RPMB devices (one per RPMB region)
+ * @host_preshoot_cap: a bitfield to indicate supported PreShoot dBs of host's TX lanes, cache of
+ *	host M-PHY TX_HS_PreShoot_Setting_Capability Attribute (ID 0x15)
+ * @host_deemphasis_cap: a bitfield to indicate supported DeEmphasis dBs of host's TX lanes, cache
+ *	of host M-PHY TX_HS_DeEmphasis_Setting_Capability Attribute (ID 0x12)
+ * @device_preshoot_cap: a bitfield to indicate supported PreShoot dBs of device's TX lanes, cache
+ *	of device M-PHY TX_HS_PreShoot_Setting_Capability Attribute (ID 0x15)
+ * @device_deemphasis_cap: a bitfield to indicate supported DeEmphasis dBs of device's TX lanes,
+ *	cache of device M-PHY TX_HS_DeEmphasis_Setting_Capability Attribute (ID 0x12)
+ * @tx_eq_params: TX Equalization settings
  */
 struct ufs_hba {
 	void __iomem *mmio_base;
@@ -1128,6 +1238,12 @@ struct ufs_hba {
 	u64 dev_lvl_exception_id;
 	u32 vcc_off_delay_us;
 	struct list_head rpmbs;
+
+	u8 host_preshoot_cap;
+	u8 host_deemphasis_cap;
+	u8 device_preshoot_cap;
+	u8 device_deemphasis_cap;
+	struct ufshcd_tx_eq_params tx_eq_params[UFS_HS_GEAR_MAX];
 };
 
 /**
@@ -1272,6 +1388,13 @@ static inline bool ufshcd_enable_wb_if_scaling_up(struct ufs_hba *hba)
 	return hba->caps & UFSHCD_CAP_WB_WITH_CLK_SCALING;
 }
 
+static inline bool ufshcd_is_tx_eq_supported(struct ufs_hba *hba)
+{
+	return hba->caps & UFSHCD_CAP_TX_EQUALIZATION &&
+	       hba->ufs_version >= ufshci_version(5, 0) &&
+	       hba->dev_info.wspecversion >= 0x500;
+}
+
 #define ufsmcq_writel(hba, val, reg)	\
 	writel((val), (hba)->mcq_base + (reg))
 #define ufsmcq_readl(hba, reg)	\
@@ -1287,6 +1410,18 @@ static inline bool ufshcd_enable_wb_if_scaling_up(struct ufs_hba *hba)
 #define ufshcd_readl(hba, reg)	\
 	readl((hba)->mmio_base + (reg))
 
+static inline const char *ufs_hs_rate_to_str(enum ufs_hs_gear_rate rate)
+{
+	switch (rate) {
+	case PA_HS_MODE_A:
+		return "A";
+	case PA_HS_MODE_B:
+		return "B";
+	default:
+		return "Unknown";
+	}
+}
+
 /**
  * ufshcd_rmwl - perform read/modify/write for a controller register
  * @hba: per adapter instance
diff --git a/include/ufs/unipro.h b/include/ufs/unipro.h
index 71a5f643400c..4aa592130b4e 100644
--- a/include/ufs/unipro.h
+++ b/include/ufs/unipro.h
@@ -10,6 +10,8 @@
  * M-TX Configuration Attributes
  */
 #define TX_HIBERN8TIME_CAPABILITY		0x000F
+#define TX_HS_DEEMPHASIS_SETTING_CAP		0x0012
+#define TX_HS_PRESHOOT_SETTING_CAP		0x0015
 #define TX_MODE					0x0021
 #define TX_HSRATE_SERIES			0x0022
 #define TX_HSGEAR				0x0023
@@ -38,6 +40,9 @@
 /*
  * M-RX Configuration Attributes
  */
+#define RX_HS_G5_ADAPT_INITIAL_CAP		0x0074
+#define RX_HS_G6_ADAPT_INITIAL_CAP		0x007B
+#define RX_HS_G6_ADAPT_INITIAL_L0L1L2L3_CAP	0x007D
 #define RX_HS_G1_SYNC_LENGTH_CAP		0x008B
 #define RX_HS_G1_PREP_LENGTH_CAP		0x008C
 #define RX_MIN_ACTIVATETIME_CAPABILITY		0x008F
@@ -50,6 +55,7 @@
 #define RX_HIBERN8TIME_CAP			0x0092
 #define RX_ADV_HIBERN8TIME_CAP			0x0099
 #define RX_ADV_MIN_ACTIVATETIME_CAP		0x009A
+#define RX_HS_G4_ADAPT_INITIAL_CAP		0x009F
 #define RX_MODE					0x00A1
 #define RX_HSRATE_SERIES			0x00A2
 #define RX_HSGEAR				0x00A3
@@ -64,6 +70,7 @@
 #define CFGRXCDR8				0x00BA
 #define CFGRXOVR8				0x00BD
 #define CFGRXOVR6				0x00BF
+#define RX_FOM					0x00C2
 #define RXDIRECTCTRL2				0x00C7
 #define CFGRXOVR4				0x00E9
 #define RX_REFCLKFREQ				0x00EB
@@ -73,7 +80,6 @@
 #define ENARXDIRECTCFG3				0x00F3
 #define ENARXDIRECTCFG2				0x00F4
 
-
 #define is_mphy_tx_attr(attr)			(attr < RX_MODE)
 #define RX_ADV_FINE_GRAN_STEP(x)		((((x) & 0x3) << 1) | 0x1)
 #define SYNC_LEN_FINE(x)			((x) & 0x3F)
@@ -99,6 +105,18 @@
 
 #define UNIPRO_CB_OFFSET(x)			(0x8000 | x)
 
+#define ADAPT_LENGTH_MASK			0x7F
+#define ADAPT_RANGE_BIT				BIT(7)
+#define IS_ADAPT_RANGE_COARSE(x)		((x) & ADAPT_RANGE_BIT)
+
+/* Adapt definitions */
+#define ADAPT_LENGTH_MAX			0x91
+#define ADAPT_L0L3_LENGTH_MAX			0x90
+#define ADAPT_L0L1L2L3_LENGTH_MAX		0x8C
+#define TADAPT_FACTOR				650
+#define TADAPT_L0L3_FACTOR			(1 << 9)
+#define TADAPT_L0L1L2L3_FACTOR			(1 << 15)
+
 /*
  * PHY Adapter attributes
  */
@@ -164,10 +182,26 @@
 #define PA_PACPERRORCOUNT	0x15C1
 #define PA_PHYTESTCONTROL	0x15C2
 #define PA_TXHSG4SYNCLENGTH	0x15D0
+#define PA_PEERRXHSG4ADAPTINITIAL		0x15D3
 #define PA_TXHSADAPTTYPE	0x15D4
 #define PA_TXHSG5SYNCLENGTH	0x15D6
+#define PA_PEERRXHSG5ADAPTINITIAL		0x15D9
+#define PA_PEERRXHSG6ADAPTREFRESHL0L1L2L3	0x15DE
+#define PA_PEERRXHSG6ADAPTINITIALL0L3		0x15DF
+#define PA_PEERRXHSG6ADAPTINITIALL0L1L2L3	0x15E0
+#define PA_TXEQG1SETTING			0x15E1
+#define PA_TXEQG2SETTING			0x15E2
+#define PA_TXEQG3SETTING			0x15E3
+#define PA_TXEQG4SETTING			0x15E4
+#define PA_TXEQG5SETTING			0x15E5
+#define PA_TXEQG6SETTING			0x15E6
+#define PA_TXEQTRSETTING			0x15E7
+#define PA_PEERTXEQTRSETTING			0x15E8
+#define PA_PRECODEEN				0x15E9
+#define PA_EQTR_GEAR				0x15EA
+#define PA_TXADAPTLENGTH_EQTR			0x15EB
 
-/* Adpat type for PA_TXHSADAPTTYPE attribute */
+/* Adapt type for PA_TXHSADAPTTYPE attribute */
 #define PA_REFRESH_ADAPT       0x00
 #define PA_INITIAL_ADAPT       0x01
 #define PA_NO_ADAPT            0x03
@@ -187,6 +221,82 @@
 /* PHY Adapter Protocol Constants */
 #define PA_MAXDATALANES	4
 
+/*
+ * TX EQTR's minimum TAdapt should not be less than 10us.
+ * This value is rounded up into the nearest Unit Intervals (UI)
+ */
+#define TX_EQTR_HS_G4_MIN_T_ADAPT		166400
+#define TX_EQTR_HS_G5_MIN_T_ADAPT		332800
+#define TX_EQTR_HS_G6_MIN_T_ADAPT		262144
+
+#define TX_EQTR_HS_G4_ADAPT_DEFAULT		0x88
+#define TX_EQTR_HS_G5_ADAPT_DEFAULT		0x89
+#define TX_EQTR_HS_G6_ADAPT_DEFAULT		0x89
+
+#define TX_EQTR_CAP_MASK			0x7F
+
+#define TX_EQTR_ADAPT_LENGTH_L0L1L2L3_SHIFT	8
+#define TX_EQTR_ADAPT_RESERVED			0xFF
+
+#define TX_HS_NUM_PRESHOOT			8
+#define TX_HS_NUM_DEEMPHASIS			8
+#define TX_HS_PRESHOOT_SHIFT			4
+#define TX_HS_DEEMPHASIS_SHIFT			4
+#define TX_HS_PRESHOOT_OFFSET			0
+#define TX_HS_DEEMPHASIS_OFFSET			16
+
+#define TX_HS_PRESHOOT_LANE_SHIFT(lane) \
+	(TX_HS_PRESHOOT_OFFSET + (lane) * TX_HS_PRESHOOT_SHIFT)
+#define TX_HS_DEEMPHASIS_LANE_SHIFT(lane) \
+	(TX_HS_DEEMPHASIS_OFFSET + (lane) * TX_HS_DEEMPHASIS_SHIFT)
+
+#define TX_HS_PRESHOOT_BITS(lane, val) \
+	((val) << TX_HS_PRESHOOT_LANE_SHIFT(lane))
+#define TX_HS_DEEMPHASIS_BITS(lane, val) \
+	((val) << TX_HS_DEEMPHASIS_LANE_SHIFT(lane))
+
+#define RX_FOM_VALUE_MASK			0x7F
+#define RX_FOM_PRECODING_EN_BIT			BIT(7)
+
+#define PRECODEEN_TX_OFFSET			0
+#define PRECODEEN_RX_OFFSET			4
+#define PRECODEEN_TX_BIT(lane)		(1 << (PRECODEEN_TX_OFFSET + (lane)))
+#define PRECODEEN_RX_BIT(lane)		(1 << (PRECODEEN_RX_OFFSET + (lane)))
+
+enum ufs_tx_eq_preset {
+	UFS_TX_EQ_PRESET_P0,
+	UFS_TX_EQ_PRESET_P1,
+	UFS_TX_EQ_PRESET_P2,
+	UFS_TX_EQ_PRESET_P3,
+	UFS_TX_EQ_PRESET_P4,
+	UFS_TX_EQ_PRESET_P5,
+	UFS_TX_EQ_PRESET_P6,
+	UFS_TX_EQ_PRESET_P7,
+	UFS_TX_EQ_PRESET_MAX,
+};
+
+enum ufs_tx_hs_preshoot {
+	UFS_TX_HS_PRESHOOT_DB_0P0,
+	UFS_TX_HS_PRESHOOT_DB_0P4,
+	UFS_TX_HS_PRESHOOT_DB_0P8,
+	UFS_TX_HS_PRESHOOT_DB_1P2,
+	UFS_TX_HS_PRESHOOT_DB_1P6,
+	UFS_TX_HS_PRESHOOT_DB_2P5,
+	UFS_TX_HS_PRESHOOT_DB_3P5,
+	UFS_TX_HS_PRESHOOT_DB_4P7,
+};
+
+enum ufs_tx_hs_deemphasis {
+	UFS_TX_HS_DEEMPHASIS_DB_0P0,
+	UFS_TX_HS_DEEMPHASIS_DB_0P8,
+	UFS_TX_HS_DEEMPHASIS_DB_1P6,
+	UFS_TX_HS_DEEMPHASIS_DB_2P5,
+	UFS_TX_HS_DEEMPHASIS_DB_3P5,
+	UFS_TX_HS_DEEMPHASIS_DB_4P7,
+	UFS_TX_HS_DEEMPHASIS_DB_6P0,
+	UFS_TX_HS_DEEMPHASIS_DB_7P6,
+};
+
 #define DL_FC0ProtectionTimeOutVal_Default	8191
 #define DL_TC0ReplayTimeOutVal_Default		65535
 #define DL_AFC0ReqTimeOutVal_Default		32767