1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2026 Qualcomm Technologies, Inc.
4 *
5 * Author:
6 * Can Guo <can.guo@oss.qualcomm.com>
7 */
8
9 #include <linux/bitops.h>
10 #include <linux/delay.h>
11 #include <linux/errno.h>
12 #include <linux/kernel.h>
13 #include <linux/sched/mm.h>
14 #include <ufs/ufshcd.h>
15 #include <ufs/unipro.h>
16 #include "ufshcd-priv.h"
17
18 #define TX_EQ_SETTING_MASK 0x7
19 #define TX_EQ_SETTINGS_VALID_BIT BIT(15)
20
21 static bool use_adaptive_txeq;
22 module_param(use_adaptive_txeq, bool, 0644);
23 MODULE_PARM_DESC(use_adaptive_txeq, "Find and apply optimal TX Equalization settings before changing Power Mode (default: false)");
24
txeq_gear_set(const char * val,const struct kernel_param * kp)25 static int txeq_gear_set(const char *val, const struct kernel_param *kp)
26 {
27 return param_set_uint_minmax(val, kp, UFS_HS_G1, UFS_HS_GEAR_MAX);
28 }
29
30 static const struct kernel_param_ops txeq_gear_ops = {
31 .set = txeq_gear_set,
32 .get = param_get_uint,
33 };
34
35 static unsigned int adaptive_txeq_gear = UFS_HS_G6;
36 module_param_cb(adaptive_txeq_gear, &txeq_gear_ops, &adaptive_txeq_gear, 0644);
37 MODULE_PARM_DESC(adaptive_txeq_gear, "For HS-Gear[n] and above, adaptive txeq shall be used");
38
39 static bool use_txeq_presets;
40 module_param(use_txeq_presets, bool, 0644);
41 MODULE_PARM_DESC(use_txeq_presets, "Use only the 8 TX Equalization Presets (pre-defined Pre-Shoot & De-Emphasis combinations) for TX EQTR (default: false)");
42
43 static bool txeq_presets_selected[UFS_TX_EQ_PRESET_MAX] = {[0 ... (UFS_TX_EQ_PRESET_MAX - 1)] = 1};
44 module_param_array(txeq_presets_selected, bool, NULL, 0644);
45 MODULE_PARM_DESC(txeq_presets_selected, "Use only the selected Presets out of the 8 TX Equalization Presets for TX EQTR");
46
txeq_setting_sel_set(const char * val,const struct kernel_param * kp)47 static int txeq_setting_sel_set(const char *val, const struct kernel_param *kp)
48 {
49 return param_set_uint_minmax(val, kp, 0, 1);
50 }
51
52 static const struct kernel_param_ops txeq_setting_sel_ops = {
53 .set = txeq_setting_sel_set,
54 .get = param_get_uint,
55 };
56
57 static unsigned int txeq_setting_sel;
58 module_param_cb(txeq_setting_sel, &txeq_setting_sel_ops, &txeq_setting_sel, 0644);
59 MODULE_PARM_DESC(txeq_setting_sel, "The qTxEQGnSettings and wTxEQGnSettingsExt Attributes selector used to retrieve and store TX Equalization settings");
60
61 static bool retrieve_txeq_setting = true;
62 module_param(retrieve_txeq_setting, bool, 0644);
63 MODULE_PARM_DESC(retrieve_txeq_setting, "Retrieve TX Equalization settings from qTxEQGnSettings and wTxEQGnSettingsExt Attributes (default: true)");
64
65 static bool store_txeq_setting = true;
66 module_param(store_txeq_setting, bool, 0644);
67 MODULE_PARM_DESC(store_txeq_setting, "Store the optimal TX Equalization settings to qTxEQGnSettings and wTxEQGnSettingsExt Attributes (default: true)");
68
69 /*
70 * ufs_tx_eq_preset - Table of minimum required list of presets.
71 *
72 * A HS-G6 capable M-TX shall support the presets defined in M-PHY v6.0 spec.
73 * Preset Pre-Shoot(dB) De-Emphasis(dB)
74 * P0 0.0 0.0
75 * P1 0.0 0.8
76 * P2 0.0 1.6
77 * P3 0.8 0.0
78 * P4 1.6 0.0
79 * P5 0.8 0.8
80 * P6 0.8 1.6
81 * P7 1.6 0.8
82 */
83 static const struct __ufs_tx_eq_preset {
84 u8 preshoot;
85 u8 deemphasis;
86 } ufs_tx_eq_preset[UFS_TX_EQ_PRESET_MAX] = {
87 [UFS_TX_EQ_PRESET_P0] = {UFS_TX_HS_PRESHOOT_DB_0P0, UFS_TX_HS_DEEMPHASIS_DB_0P0},
88 [UFS_TX_EQ_PRESET_P1] = {UFS_TX_HS_PRESHOOT_DB_0P0, UFS_TX_HS_DEEMPHASIS_DB_0P8},
89 [UFS_TX_EQ_PRESET_P2] = {UFS_TX_HS_PRESHOOT_DB_0P0, UFS_TX_HS_DEEMPHASIS_DB_1P6},
90 [UFS_TX_EQ_PRESET_P3] = {UFS_TX_HS_PRESHOOT_DB_0P8, UFS_TX_HS_DEEMPHASIS_DB_0P0},
91 [UFS_TX_EQ_PRESET_P4] = {UFS_TX_HS_PRESHOOT_DB_1P6, UFS_TX_HS_DEEMPHASIS_DB_0P0},
92 [UFS_TX_EQ_PRESET_P5] = {UFS_TX_HS_PRESHOOT_DB_0P8, UFS_TX_HS_DEEMPHASIS_DB_0P8},
93 [UFS_TX_EQ_PRESET_P6] = {UFS_TX_HS_PRESHOOT_DB_0P8, UFS_TX_HS_DEEMPHASIS_DB_1P6},
94 [UFS_TX_EQ_PRESET_P7] = {UFS_TX_HS_PRESHOOT_DB_1P6, UFS_TX_HS_DEEMPHASIS_DB_0P8},
95 };
96
97 /*
98 * pa_peer_rx_adapt_initial - Table of UniPro PA_PeerRxHSGnAdaptInitial
99 * attribute IDs for High Speed (HS) Gears.
100 *
101 * This table maps HS Gears to their respective UniPro PA_PeerRxHSGnAdaptInitial
102 * attribute IDs. Entries for Gears 1-3 are 0 (unsupported).
103 */
104 static const u32 pa_peer_rx_adapt_initial[UFS_HS_GEAR_MAX] = {
105 0,
106 0,
107 0,
108 PA_PEERRXHSG4ADAPTINITIAL,
109 PA_PEERRXHSG5ADAPTINITIAL,
110 PA_PEERRXHSG6ADAPTINITIALL0L3
111 };
112
113 /*
114 * rx_adapt_initial_cap - Table of M-PHY RX_HS_Gn_ADAPT_INITIAL_Capability
115 * attribute IDs for High Speed (HS) Gears.
116 *
117 * This table maps HS Gears to their respective M-PHY
118 * RX_HS_Gn_ADAPT_INITIAL_Capability attribute IDs. Entries for Gears 1-3 are 0
119 * (unsupported).
120 */
121 static const u32 rx_adapt_initial_cap[UFS_HS_GEAR_MAX] = {
122 0,
123 0,
124 0,
125 RX_HS_G4_ADAPT_INITIAL_CAP,
126 RX_HS_G5_ADAPT_INITIAL_CAP,
127 RX_HS_G6_ADAPT_INITIAL_CAP
128 };
129
130 /*
131 * pa_tx_eq_setting - Table of UniPro PA_TxEQGnSetting attribute IDs for High
132 * Speed (HS) Gears.
133 *
134 * This table maps HS Gears to their respective UniPro PA_TxEQGnSetting
135 * attribute IDs.
136 */
137 static const u32 pa_tx_eq_setting[UFS_HS_GEAR_MAX] = {
138 PA_TXEQG1SETTING,
139 PA_TXEQG2SETTING,
140 PA_TXEQG3SETTING,
141 PA_TXEQG4SETTING,
142 PA_TXEQG5SETTING,
143 PA_TXEQG6SETTING
144 };
145
146 /*
147 * Decode Device TX Equalization PreShoot value based on qTxEQGnSettings bit assignment:
148 * bit[3:0]: Device TX Logical LANE 0 PreShoot
149 * bit[7:4]: Device TX Logical LANE 1 PreShoot
150 */
tx_eq_device_preshoot_decode(u64 eq,u8 lane)151 static inline u8 tx_eq_device_preshoot_decode(u64 eq, u8 lane)
152 {
153 return (u8)((eq >> (lane * TX_HS_PRESHOOT_SHIFT)) & TX_EQ_SETTING_MASK);
154 }
155
156 /*
157 * Decode Device TX Equalization DeEmphasis value based on qTxEQGnSettings bit assignment:
158 * bit[19:16]: Device TX Logical LANE 0 DeEmphasis
159 * bit[23:20]: Device TX Logical LANE 1 DeEmphasis
160 */
tx_eq_device_deemphasis_decode(u64 eq,u8 lane)161 static inline u8 tx_eq_device_deemphasis_decode(u64 eq, u8 lane)
162 {
163 return (u8)((eq >> (lane * TX_HS_DEEMPHASIS_SHIFT + 16)) & TX_EQ_SETTING_MASK);
164 }
165
166 /*
167 * Decode Host TX Equalization PreShoot value based on qTxEQGnSettings bit assignment:
168 * bit[35:32]: Host TX Logical LANE 0 PreShoot
169 * bit[39:36]: Host TX Logical LANE 1 PreShoot
170 */
tx_eq_host_preshoot_decode(u64 eq,u8 lane)171 static inline u8 tx_eq_host_preshoot_decode(u64 eq, u8 lane)
172 {
173 return (u8)((eq >> (lane * TX_HS_PRESHOOT_SHIFT + 32)) & TX_EQ_SETTING_MASK);
174 }
175
176 /*
177 * Decode Host TX Equalization DeEmphasis value based on qTxEQGnSettings bit assignment:
178 * bit[51:48]: Host TX Logical LANE 0 DeEmphasis
179 * bit[55:52]: Host TX Logical LANE 1 DeEmphasis
180 */
tx_eq_host_deemphasis_decode(u64 eq,u8 lane)181 static inline u8 tx_eq_host_deemphasis_decode(u64 eq, u8 lane)
182 {
183 return (u8)((eq >> (lane * TX_HS_DEEMPHASIS_SHIFT + 48)) & TX_EQ_SETTING_MASK);
184 }
185
186 /*
187 * Decode Device TX precode_en indication based on wTxEQGnSettingsExt bit assignment:
188 * bit[0]: PreCodeEn for Device TX Logical LANE 0
189 * bit[1]: PreCodeEn for Device TX Logical LANE 1
190 */
tx_eq_device_precode_en_decode(u16 eq_ext,u8 lane)191 static inline bool tx_eq_device_precode_en_decode(u16 eq_ext, u8 lane)
192 {
193 return eq_ext & BIT(lane);
194 }
195
196 /*
197 * Decode Host TX precode_en indication based on wTxEQGnSettingsExt bit assignment:
198 * bit[4]: PreCodeEn for Device RX Logical LANE 0
199 * bit[5]: PreCodeEn for Device RX Logical LANE 1
200 */
tx_eq_host_precode_en_decode(u16 eq_ext,u8 lane)201 static inline bool tx_eq_host_precode_en_decode(u16 eq_ext, u8 lane)
202 {
203 return eq_ext & BIT(lane + 4);
204 }
205
206 /*
207 * Encode Device TX Equalization PreShoot value based on qTxEQGnSettings bit assignment:
208 * bit[3:0]: Device TX Logical LANE 0 PreShoot
209 * bit[7:4]: Device TX Logical LANE 1 PreShoot
210 */
tx_eq_device_preshoot_encode(u64 val,u8 lane)211 static inline u64 tx_eq_device_preshoot_encode(u64 val, u8 lane)
212 {
213 return (val & TX_EQ_SETTING_MASK) << (lane * TX_HS_PRESHOOT_SHIFT);
214 }
215
216 /*
217 * Encode Device TX Equalization DeEmphasis value based on qTxEQGnSettings bit assignment:
218 * bit[19:16]: Device TX Logical LANE 0 DeEmphasis
219 * bit[23:20]: Device TX Logical LANE 1 DeEmphasis
220 */
tx_eq_device_deemphasis_encode(u64 val,u8 lane)221 static inline u64 tx_eq_device_deemphasis_encode(u64 val, u8 lane)
222 {
223 return (val & TX_EQ_SETTING_MASK) << (lane * TX_HS_DEEMPHASIS_SHIFT + 16);
224 }
225
226 /*
227 * Encode Host TX Equalization PreShoot value based on qTxEQGnSettings bit assignment:
228 * bit[35:32]: Host TX Logical LANE 0 PreShoot
229 * bit[39:36]: Host TX Logical LANE 1 PreShoot
230 */
tx_eq_host_preshoot_encode(u64 val,u8 lane)231 static inline u64 tx_eq_host_preshoot_encode(u64 val, u8 lane)
232 {
233 return (val & TX_EQ_SETTING_MASK) << (lane * TX_HS_PRESHOOT_SHIFT + 32);
234 }
235
236 /*
237 * Encode Host TX Equalization DeEmphasis value based on qTxEQGnSettings bit assignment:
238 * bit[51:48]: Host TX Logical LANE 0 DeEmphasis
239 * bit[55:52]: Host TX Logical LANE 1 DeEmphasis
240 */
tx_eq_host_deemphasis_encode(u64 val,u8 lane)241 static inline u64 tx_eq_host_deemphasis_encode(u64 val, u8 lane)
242 {
243 return (val & TX_EQ_SETTING_MASK) << (lane * TX_HS_DEEMPHASIS_SHIFT + 48);
244 }
245
246 /*
247 * Encode Device precode_en based on wTxEQGnSettingsExt bit assignment:
248 * bit[0]: PreCodeEn for Device TX Logical LANE 0
249 * bit[1]: PreCodeEn for Device TX Logical LANE 1
250 */
tx_eq_device_precode_en_encode(bool en,u8 lane)251 static inline u16 tx_eq_device_precode_en_encode(bool en, u8 lane)
252 {
253 return (u16)en << lane;
254 }
255
256 /*
257 * Encode Host precode_en based on wTxEQGnSettingsExt bit assignment:
258 * bit[4]: PreCodeEn for Device RX Logical LANE 0
259 * bit[5]: PreCodeEn for Device RX Logical LANE 1
260 */
tx_eq_host_precode_en_encode(bool en,u8 lane)261 static inline u16 tx_eq_host_precode_en_encode(bool en, u8 lane)
262 {
263 return (u16)en << (lane + 4);
264 }
265
266 /**
267 * ufshcd_configure_precoding - Configure Pre-Coding for all active lanes
268 * @hba: per adapter instance
269 * @params: TX EQ parameters data structure
270 *
271 * Bit[7] in RX_FOM indicates that the receiver needs to enable Pre-Coding when
272 * set. Pre-Coding must be enabled on both the transmitter and receiver to
273 * ensure proper operation.
274 *
275 * Returns 0 on success, non-zero error code otherwise
276 */
ufshcd_configure_precoding(struct ufs_hba * hba,struct ufshcd_tx_eq_params * params)277 static int ufshcd_configure_precoding(struct ufs_hba *hba,
278 struct ufshcd_tx_eq_params *params)
279 {
280 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
281 u32 local_precode_en = 0;
282 u32 peer_precode_en = 0;
283 int lane, ret;
284
285 /* Enable Pre-Coding for Host's TX & Device's RX pair */
286 for (lane = 0; lane < pwr_info->lane_tx; lane++) {
287 if (params->host[lane].precode_en) {
288 local_precode_en |= PRECODEEN_TX_BIT(lane);
289 peer_precode_en |= PRECODEEN_RX_BIT(lane);
290 }
291 }
292
293 /* Enable Pre-Coding for Device's TX & Host's RX pair */
294 for (lane = 0; lane < pwr_info->lane_rx; lane++) {
295 if (params->device[lane].precode_en) {
296 peer_precode_en |= PRECODEEN_TX_BIT(lane);
297 local_precode_en |= PRECODEEN_RX_BIT(lane);
298 }
299 }
300
301 if (!local_precode_en && !peer_precode_en) {
302 dev_dbg(hba->dev, "Pre-Coding is not required for Host and Device\n");
303 return 0;
304 }
305
306 /* Set local PA_PreCodeEn */
307 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PRECODEEN), local_precode_en);
308 if (ret) {
309 dev_err(hba->dev, "Failed to set local PA_PreCodeEn: %d\n", ret);
310 return ret;
311 }
312
313 /* Set peer PA_PreCodeEn */
314 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_PRECODEEN), peer_precode_en);
315 if (ret) {
316 dev_err(hba->dev, "Failed to set peer PA_PreCodeEn: %d\n", ret);
317 return ret;
318 }
319
320 dev_dbg(hba->dev, "Local PA_PreCodeEn: 0x%02x, Peer PA_PreCodeEn: 0x%02x\n",
321 local_precode_en, peer_precode_en);
322
323 return 0;
324 }
325
ufshcd_print_tx_eq_params(struct ufs_hba * hba)326 void ufshcd_print_tx_eq_params(struct ufs_hba *hba)
327 {
328 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
329 struct ufshcd_tx_eq_params *params;
330 u32 gear = hba->pwr_info.gear_tx;
331 int lane;
332
333 if (!ufshcd_is_tx_eq_supported(hba))
334 return;
335
336 if (gear < UFS_HS_G1 || gear > UFS_HS_GEAR_MAX)
337 return;
338
339 params = &hba->tx_eq_params[gear - 1];
340 if (!params->is_valid || !params->is_applied)
341 return;
342
343 for (lane = 0; lane < pwr_info->lane_tx; lane++)
344 dev_dbg(hba->dev, "Host TX Lane %d: PreShoot %u, DeEmphasis %u, FOM %u, PreCodeEn %d\n",
345 lane, params->host[lane].preshoot,
346 params->host[lane].deemphasis,
347 params->host[lane].fom_val,
348 params->host[lane].precode_en);
349
350 for (lane = 0; lane < pwr_info->lane_rx; lane++)
351 dev_dbg(hba->dev, "Device TX Lane %d: PreShoot %u, DeEmphasis %u, FOM %u, PreCodeEn %d\n",
352 lane, params->device[lane].preshoot,
353 params->device[lane].deemphasis,
354 params->device[lane].fom_val,
355 params->device[lane].precode_en);
356 }
357
358 static inline u32
ufshcd_compose_tx_eq_setting(struct ufshcd_tx_eq_settings * settings,int num_lanes)359 ufshcd_compose_tx_eq_setting(struct ufshcd_tx_eq_settings *settings,
360 int num_lanes)
361 {
362 u32 setting = 0;
363 int lane;
364
365 for (lane = 0; lane < num_lanes; lane++, settings++) {
366 setting |= TX_HS_PRESHOOT_BITS(lane, settings->preshoot);
367 setting |= TX_HS_DEEMPHASIS_BITS(lane, settings->deemphasis);
368 }
369
370 return setting;
371 }
372
373 /**
374 * ufshcd_apply_tx_eq_settings - Apply TX Equalization settings for target gear
375 * @hba: per adapter instance
376 * @params: TX EQ parameters data structure
377 * @gear: target gear
378 *
379 * Returns 0 on success, negative error code otherwise
380 */
ufshcd_apply_tx_eq_settings(struct ufs_hba * hba,struct ufshcd_tx_eq_params * params,u32 gear)381 int ufshcd_apply_tx_eq_settings(struct ufs_hba *hba,
382 struct ufshcd_tx_eq_params *params, u32 gear)
383 {
384 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
385 u32 setting;
386 int ret;
387
388 /* Compose settings for Host's TX Lanes */
389 setting = ufshcd_compose_tx_eq_setting(params->host, pwr_info->lane_tx);
390 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(pa_tx_eq_setting[gear - 1]), setting);
391 if (ret)
392 return ret;
393
394 /* Compose settings for Device's TX Lanes */
395 setting = ufshcd_compose_tx_eq_setting(params->device, pwr_info->lane_rx);
396 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(pa_tx_eq_setting[gear - 1]), setting);
397 if (ret)
398 return ret;
399
400 /* Configure Pre-Coding */
401 if (gear >= UFS_HS_G6) {
402 ret = ufshcd_configure_precoding(hba, params);
403 if (ret) {
404 dev_err(hba->dev, "Failed to configure pre-coding: %d\n", ret);
405 return ret;
406 }
407 }
408
409 return 0;
410 }
411 EXPORT_SYMBOL_GPL(ufshcd_apply_tx_eq_settings);
412
413 /**
414 * ufshcd_evaluate_tx_eqtr_fom - Evaluate TX EQTR FOM results
415 * @hba: per adapter instance
416 * @pwr_mode: target power mode containing gear and rate information
417 * @eqtr_data: TX EQTR data structure
418 * @h_iter: host TX EQTR iterator data structure
419 * @d_iter: device TX EQTR iterator data structure
420 *
421 * Evaluate TX EQTR FOM results, update host and device TX EQTR data accordingy
422 * if FOM have been improved compared to previous iteration, and record TX EQTR
423 * FOM results.
424 */
ufshcd_evaluate_tx_eqtr_fom(struct ufs_hba * hba,struct ufs_pa_layer_attr * pwr_mode,struct ufshcd_tx_eqtr_data * eqtr_data,struct tx_eqtr_iter * h_iter,struct tx_eqtr_iter * d_iter)425 static void ufshcd_evaluate_tx_eqtr_fom(struct ufs_hba *hba,
426 struct ufs_pa_layer_attr *pwr_mode,
427 struct ufshcd_tx_eqtr_data *eqtr_data,
428 struct tx_eqtr_iter *h_iter,
429 struct tx_eqtr_iter *d_iter)
430 {
431 u8 preshoot, deemphasis, fom_value;
432 bool precode_en;
433 int lane;
434
435 for (lane = 0; h_iter->is_updated && lane < pwr_mode->lane_tx; lane++) {
436 preshoot = h_iter->preshoot;
437 deemphasis = h_iter->deemphasis;
438 fom_value = h_iter->fom[lane] & RX_FOM_VALUE_MASK;
439 precode_en = h_iter->fom[lane] & RX_FOM_PRECODING_EN_BIT;
440
441 /* Record host TX EQTR FOM */
442 eqtr_data->host_fom[lane][preshoot][deemphasis] = h_iter->fom[lane];
443
444 /* Check if FOM has been improved for host's TX Lanes */
445 if (fom_value > eqtr_data->host[lane].fom_val) {
446 eqtr_data->host[lane].preshoot = preshoot;
447 eqtr_data->host[lane].deemphasis = deemphasis;
448 eqtr_data->host[lane].fom_val = fom_value;
449 eqtr_data->host[lane].precode_en = precode_en;
450 }
451
452 dev_dbg(hba->dev, "TX EQTR: Host TX Lane %d: PreShoot %u, DeEmphasis %u, FOM value %u, PreCodeEn %d\n",
453 lane, preshoot, deemphasis, fom_value, precode_en);
454 }
455
456 for (lane = 0; d_iter->is_updated && lane < pwr_mode->lane_rx; lane++) {
457 preshoot = d_iter->preshoot;
458 deemphasis = d_iter->deemphasis;
459 fom_value = d_iter->fom[lane] & RX_FOM_VALUE_MASK;
460 precode_en = d_iter->fom[lane] & RX_FOM_PRECODING_EN_BIT;
461
462 /* Record device TX EQTR FOM */
463 eqtr_data->device_fom[lane][preshoot][deemphasis] = d_iter->fom[lane];
464
465 /* Check if FOM has been improved for Device's TX Lanes */
466 if (fom_value > eqtr_data->device[lane].fom_val) {
467 eqtr_data->device[lane].preshoot = preshoot;
468 eqtr_data->device[lane].deemphasis = deemphasis;
469 eqtr_data->device[lane].fom_val = fom_value;
470 eqtr_data->device[lane].precode_en = precode_en;
471 }
472
473 dev_dbg(hba->dev, "TX EQTR: Device TX Lane %d: PreShoot %u, DeEmphasis %u, FOM value %u, PreCodeEn %d\n",
474 lane, preshoot, deemphasis, fom_value, precode_en);
475 }
476 }
477
478 /**
479 * ufshcd_get_rx_fom - Get Figure of Merit (FOM) for both sides
480 * @hba: per adapter instance
481 * @pwr_mode: target power mode containing gear and rate information
482 * @h_iter: host TX EQTR iterator data structure
483 * @d_iter: device TX EQTR iterator data structure
484 *
485 * Returns 0 on success, negative error code if get_rx_fom vops fails.
486 * RX_FOM DME get failures are logged and treated as 0 FOM for that lane.
487 */
ufshcd_get_rx_fom(struct ufs_hba * hba,struct ufs_pa_layer_attr * pwr_mode,struct tx_eqtr_iter * h_iter,struct tx_eqtr_iter * d_iter)488 static int ufshcd_get_rx_fom(struct ufs_hba *hba,
489 struct ufs_pa_layer_attr *pwr_mode,
490 struct tx_eqtr_iter *h_iter,
491 struct tx_eqtr_iter *d_iter)
492 {
493 int lane, ret;
494 u32 fom;
495
496 /* Get FOM of host's TX lanes from device's RX_FOM. */
497 for (lane = 0; lane < pwr_mode->lane_tx; lane++) {
498 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB_SEL(RX_FOM,
499 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
500 &fom);
501 if (ret) {
502 h_iter->fom[lane] = 0;
503 dev_dbg(hba->dev, "Failed to get FOM for Host TX Lane %d: %d\n",
504 lane, ret);
505 continue;
506 }
507
508 h_iter->fom[lane] = (u8)fom;
509 }
510
511 /* Get FOM of device's TX lanes from host's RX_FOM. */
512 for (lane = 0; lane < pwr_mode->lane_rx; lane++) {
513 ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(RX_FOM,
514 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
515 &fom);
516 if (ret) {
517 d_iter->fom[lane] = 0;
518 dev_dbg(hba->dev, "Failed to get FOM for Device TX Lane %d: %d\n",
519 lane, ret);
520 continue;
521 }
522
523 d_iter->fom[lane] = (u8)fom;
524 }
525
526 ret = ufshcd_vops_get_rx_fom(hba, pwr_mode, h_iter, d_iter);
527 if (ret)
528 dev_err(hba->dev, "Failed to get FOM via vops: %d\n", ret);
529
530 return ret;
531 }
532
ufshcd_is_txeq_presets_used(struct ufs_hba * hba)533 bool ufshcd_is_txeq_presets_used(struct ufs_hba *hba)
534 {
535 return use_txeq_presets;
536 }
537
ufshcd_is_txeq_preset_selected(u8 preshoot,u8 deemphasis)538 bool ufshcd_is_txeq_preset_selected(u8 preshoot, u8 deemphasis)
539 {
540 int i;
541
542 for (i = 0; i < UFS_TX_EQ_PRESET_MAX; i++) {
543 if (!txeq_presets_selected[i])
544 continue;
545
546 if (preshoot == ufs_tx_eq_preset[i].preshoot &&
547 deemphasis == ufs_tx_eq_preset[i].deemphasis)
548 return true;
549 }
550
551 return false;
552 }
553
554 /**
555 * tx_eqtr_iter_try_update - Try to update a TX EQTR iterator
556 * @iter: TX EQTR iterator data structure
557 * @preshoot: PreShoot value
558 * @deemphasis: DeEmphasis value
559 *
560 * This function validates whether the provided PreShoot and DeEmphasis
561 * combination can be used or not. If yes, it updates the TX EQTR iterator with
562 * the provided PreShoot and DeEmphasis, it also sets the is_updated flag
563 * to indicate the iterator has been updated.
564 */
tx_eqtr_iter_try_update(struct tx_eqtr_iter * iter,u8 preshoot,u8 deemphasis)565 static void tx_eqtr_iter_try_update(struct tx_eqtr_iter *iter,
566 u8 preshoot, u8 deemphasis)
567 {
568 if (!test_bit(preshoot, &iter->preshoot_bitmap) ||
569 !test_bit(deemphasis, &iter->deemphasis_bitmap) ||
570 (use_txeq_presets && !ufshcd_is_txeq_preset_selected(preshoot, deemphasis))) {
571 iter->is_updated = false;
572 return;
573 }
574
575 iter->preshoot = preshoot;
576 iter->deemphasis = deemphasis;
577 iter->is_updated = true;
578 }
579
580 /**
581 * tx_eqtr_iter_update() - Update host and deviceTX EQTR iterators
582 * @preshoot: PreShoot value
583 * @deemphasis: DeEmphasis value
584 * @h_iter: Host TX EQTR iterator data structure
585 * @d_iter: Device TX EQTR iterator data structure
586 *
587 * Updates host and device TX Equalization training iterators with the
588 * provided PreShoot and DeEmphasis.
589 *
590 * Return: true if host and/or device TX Equalization training iterator has
591 * been updated to the provided PreShoot and DeEmphasis, false otherwise.
592 */
tx_eqtr_iter_update(u8 preshoot,u8 deemphasis,struct tx_eqtr_iter * h_iter,struct tx_eqtr_iter * d_iter)593 static bool tx_eqtr_iter_update(u8 preshoot, u8 deemphasis,
594 struct tx_eqtr_iter *h_iter,
595 struct tx_eqtr_iter *d_iter)
596 {
597 tx_eqtr_iter_try_update(h_iter, preshoot, deemphasis);
598 tx_eqtr_iter_try_update(d_iter, preshoot, deemphasis);
599
600 return h_iter->is_updated || d_iter->is_updated;
601 }
602
603 /**
604 * ufshcd_tx_eqtr_iter_init - Initialize host and device TX EQTR iterators
605 * @hba: per adapter instance
606 * @h_iter: host TX EQTR iterator data structure
607 * @d_iter: device TX EQTR iterator data structure
608 *
609 * This function initializes the TX EQTR iterator structures for both host and
610 * device by reading their TX equalization capabilities. The capabilities are
611 * cached in the hba structure to avoid redundant DME operations in subsequent
612 * calls. In the TX EQTR procedure, the iterator structures are updated by
613 * tx_eqtr_iter_update() to systematically iterate through supported TX
614 * Equalization setting combinations.
615 *
616 * Returns 0 on success, negative error code otherwise
617 */
ufshcd_tx_eqtr_iter_init(struct ufs_hba * hba,struct tx_eqtr_iter * h_iter,struct tx_eqtr_iter * d_iter)618 static int ufshcd_tx_eqtr_iter_init(struct ufs_hba *hba,
619 struct tx_eqtr_iter *h_iter,
620 struct tx_eqtr_iter *d_iter)
621 {
622 u32 cap;
623 int ret;
624
625 if (!hba->host_preshoot_cap) {
626 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(TX_HS_PRESHOOT_SETTING_CAP), &cap);
627 if (ret)
628 return ret;
629
630 hba->host_preshoot_cap = cap & TX_EQTR_CAP_MASK;
631 }
632
633 if (!hba->host_deemphasis_cap) {
634 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(TX_HS_DEEMPHASIS_SETTING_CAP), &cap);
635 if (ret)
636 return ret;
637
638 hba->host_deemphasis_cap = cap & TX_EQTR_CAP_MASK;
639 }
640
641 if (!hba->device_preshoot_cap) {
642 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(TX_HS_PRESHOOT_SETTING_CAP), &cap);
643 if (ret)
644 return ret;
645
646 hba->device_preshoot_cap = cap & TX_EQTR_CAP_MASK;
647 }
648
649 if (!hba->device_deemphasis_cap) {
650 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(TX_HS_DEEMPHASIS_SETTING_CAP), &cap);
651 if (ret)
652 return ret;
653
654 hba->device_deemphasis_cap = cap & TX_EQTR_CAP_MASK;
655 }
656
657 /*
658 * Support PreShoot & DeEmphasis of value 0 is mandatory, hence they are
659 * not reflected in PreShoot/DeEmphasis capabilities. Left shift the
660 * capability bitmap by 1 and set bit[0] to reflect value 0 is
661 * supported, such that test_bit() can be used later for convenience.
662 */
663 h_iter->preshoot_bitmap = (hba->host_preshoot_cap << 0x1) | 0x1;
664 h_iter->deemphasis_bitmap = (hba->host_deemphasis_cap << 0x1) | 0x1;
665 d_iter->preshoot_bitmap = (hba->device_preshoot_cap << 0x1) | 0x1;
666 d_iter->deemphasis_bitmap = (hba->device_deemphasis_cap << 0x1) | 0x1;
667
668 return 0;
669 }
670
671 /**
672 * adapt_cap_to_t_adapt - Calculate TAdapt from adapt capability
673 * @adapt_cap: Adapt capability
674 *
675 * For NRZ:
676 * IF (ADAPT_range = FINE)
677 * TADAPT = 650 x (ADAPT_length + 1)
678 * ELSE (IF ADAPT_range = COARSE)
679 * TADAPT = 650 x 2^ADAPT_length
680 *
681 * Returns calculated TAdapt value in term of Unit Intervals (UI)
682 */
adapt_cap_to_t_adapt(u32 adapt_cap)683 static inline u64 adapt_cap_to_t_adapt(u32 adapt_cap)
684 {
685 u64 tadapt;
686 u8 adapt_length = adapt_cap & ADAPT_LENGTH_MASK;
687
688 if (!IS_ADAPT_RANGE_COARSE(adapt_cap))
689 tadapt = TADAPT_FACTOR * (adapt_length + 1);
690 else
691 tadapt = TADAPT_FACTOR * (1 << adapt_length);
692
693 return tadapt;
694 }
695
696 /**
697 * adapt_cap_to_t_adapt_l0l3 - Calculate TAdapt_L0_L3 from adapt capability
698 * @adapt_cap: Adapt capability
699 *
700 * For PAM-4:
701 * IF (ADAPT_range = FINE)
702 * TADAPT_L0_L3 = 2^9 x ADAPT_length
703 * ELSE IF (ADAPT_range = COARSE)
704 * TADAPT_L0_L3 = 2^9 x (2^ADAPT_length)
705 *
706 * Returns calculated TAdapt value in term of Unit Intervals (UI)
707 */
adapt_cap_to_t_adapt_l0l3(u32 adapt_cap)708 static inline u64 adapt_cap_to_t_adapt_l0l3(u32 adapt_cap)
709 {
710 u64 tadapt;
711 u8 adapt_length = adapt_cap & ADAPT_LENGTH_MASK;
712
713 if (!IS_ADAPT_RANGE_COARSE(adapt_cap))
714 tadapt = TADAPT_L0L3_FACTOR * adapt_length;
715 else
716 tadapt = TADAPT_L0L3_FACTOR * (1 << adapt_length);
717
718 return tadapt;
719 }
720
721 /**
722 * adapt_cap_to_t_adapt_l0l1l2l3 - Calculate TAdapt_L0_L1_L2_L3 from adapt capability
723 * @adapt_cap: Adapt capability
724 *
725 * For PAM-4:
726 * IF (ADAPT_range_L0_L1_L2_L3 = FINE)
727 * TADAPT_L0_L1_L2_L3 = 2^15 x (ADAPT_length_L0_L1_L2_L3 + 1)
728 * ELSE IF (ADAPT_range_L0_L1_L2_L3 = COARSE)
729 * TADAPT_L0_L1_L2_L3 = 2^15 x 2^ADAPT_length_L0_L1_L2_L3
730 *
731 * Returns calculated TAdapt value in term of Unit Intervals (UI)
732 */
adapt_cap_to_t_adapt_l0l1l2l3(u32 adapt_cap)733 static inline u64 adapt_cap_to_t_adapt_l0l1l2l3(u32 adapt_cap)
734 {
735 u64 tadapt;
736 u8 adapt_length = adapt_cap & ADAPT_LENGTH_MASK;
737
738 if (!IS_ADAPT_RANGE_COARSE(adapt_cap))
739 tadapt = TADAPT_L0L1L2L3_FACTOR * (adapt_length + 1);
740 else
741 tadapt = TADAPT_L0L1L2L3_FACTOR * (1 << adapt_length);
742
743 return tadapt;
744 }
745
746 /**
747 * ufshcd_setup_tx_eqtr_adapt_length - Setup TX adapt length for EQTR
748 * @hba: per adapter instance
749 * @params: TX EQ parameters data structure
750 * @gear: target gear for EQTR
751 *
752 * This function determines and configures the proper TX adapt length (TAdapt)
753 * for the TX EQTR procedure based on the target gear and RX adapt capabilities
754 * of both host and device.
755 *
756 * Guidelines from MIPI UniPro v3.0 spec - select the minimum Adapt Length for
757 * the Equalization Training procedure based on the following conditions:
758 *
759 * If the target High-Speed Gear n is HS-G4 or HS-G5:
760 * PA_TxAdaptLength_EQTR[7:0] >= Max (10us, RX_HS_Gn_ADAPT_INITIAL_Capability,
761 * PA_PeerRxHsGnAdaptInitial)
762 * PA_TxAdaptLength_EQTR[7:0] shall be shorter than PACP_REQUEST_TIMER (10ms)
763 * PA_TxAdaptLength_EQTR[15:8] is not relevant for HS-G4 and HS-G5. This field
764 * is set to 255 (reserved value).
765 *
766 * If the target High-Speed Gear n is HS-G6:
767 * PA_TxAdapthLength_EQTR >= 10us
768 * PA_TxAdapthLength_EQTR[7:0] >= Max (RX_HS_G6_ADAPT_INITIAL_Capability,
769 * PA_PeerRxHsG6AdaptInitialL0L3)
770 * PA_TxAdapthLength_EQTR[15:8] >= Max (RX_HS_G6_ADAPT_INITIAL_L0_L1_L2_L3_Capability,
771 * PA_PeerRxHsG6AdaptInitialL0L1L2L3)
772 * PA_TxAdaptLength_EQTR shall be shorter than PACP_REQUEST_TIMER value of 10ms.
773 *
774 * Since adapt capabilities encode both range (fine/coarse) and length values,
775 * direct comparison is not possible. This function converts adapt capabilities
776 * to actual time durations in Unit Intervals (UI) using the Adapt time
777 * calculation formular in M-PHY v6.0 spec (Table 8), then selects the maximum
778 * to ensure both host and device use adequate TX adapt length.
779 *
780 * Returns 0 on success, negative error code otherwise
781 */
ufshcd_setup_tx_eqtr_adapt_length(struct ufs_hba * hba,struct ufshcd_tx_eq_params * params,u32 gear)782 static int ufshcd_setup_tx_eqtr_adapt_length(struct ufs_hba *hba,
783 struct ufshcd_tx_eq_params *params,
784 u32 gear)
785 {
786 struct ufshcd_tx_eqtr_record *rec = params->eqtr_record;
787 u32 adapt_eqtr;
788 int ret;
789
790 if (rec && rec->saved_adapt_eqtr) {
791 adapt_eqtr = rec->saved_adapt_eqtr;
792 goto set_adapt_eqtr;
793 }
794
795 if (gear == UFS_HS_G4 || gear == UFS_HS_G5) {
796 u64 t_adapt, t_adapt_local, t_adapt_peer;
797 u32 adapt_cap_local, adapt_cap_peer, adapt_length;
798
799 ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(rx_adapt_initial_cap[gear - 1],
800 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
801 &adapt_cap_local);
802 if (ret)
803 return ret;
804
805 if (adapt_cap_local > ADAPT_LENGTH_MAX) {
806 dev_err(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP (0x%x) exceeds MAX\n",
807 gear, adapt_cap_local);
808 return -EINVAL;
809 }
810
811 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(pa_peer_rx_adapt_initial[gear - 1]),
812 &adapt_cap_peer);
813 if (ret)
814 return ret;
815
816 if (adapt_cap_peer > ADAPT_LENGTH_MAX) {
817 dev_err(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP (0x%x) exceeds MAX\n",
818 gear, adapt_cap_peer);
819 return -EINVAL;
820 }
821
822 t_adapt_local = adapt_cap_to_t_adapt(adapt_cap_local);
823 t_adapt_peer = adapt_cap_to_t_adapt(adapt_cap_peer);
824 t_adapt = max(t_adapt_local, t_adapt_peer);
825
826 dev_dbg(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP = 0x%x\n",
827 gear, adapt_cap_local);
828 dev_dbg(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_CAP = 0x%x\n",
829 gear, adapt_cap_peer);
830 dev_dbg(hba->dev, "t_adapt_local = %llu UI, t_adapt_peer = %llu UI\n",
831 t_adapt_local, t_adapt_peer);
832 dev_dbg(hba->dev, "TAdapt %llu UI selected for TX EQTR\n",
833 t_adapt);
834
835 adapt_length = (t_adapt_local >= t_adapt_peer) ?
836 adapt_cap_local : adapt_cap_peer;
837
838 if (gear == UFS_HS_G4 && t_adapt < TX_EQTR_HS_G4_MIN_T_ADAPT) {
839 dev_dbg(hba->dev, "TAdapt %llu UI is too short for TX EQTR for HS-G%u, use default Adapt 0x%x\n",
840 t_adapt, gear, TX_EQTR_HS_G4_ADAPT_DEFAULT);
841 adapt_length = TX_EQTR_HS_G4_ADAPT_DEFAULT;
842 } else if (gear == UFS_HS_G5 && t_adapt < TX_EQTR_HS_G5_MIN_T_ADAPT) {
843 dev_dbg(hba->dev, "TAdapt %llu UI is too short for TX EQTR for HS-G%u, use default Adapt 0x%x\n",
844 t_adapt, gear, TX_EQTR_HS_G5_ADAPT_DEFAULT);
845 adapt_length = TX_EQTR_HS_G5_ADAPT_DEFAULT;
846 }
847
848 adapt_eqtr = adapt_length |
849 (TX_EQTR_ADAPT_RESERVED << TX_EQTR_ADAPT_LENGTH_L0L1L2L3_SHIFT);
850 } else if (gear == UFS_HS_G6) {
851 u64 t_adapt, t_adapt_l0l3, t_adapt_l0l3_local, t_adapt_l0l3_peer;
852 u64 t_adapt_l0l1l2l3, t_adapt_l0l1l2l3_local, t_adapt_l0l1l2l3_peer;
853 u32 adapt_l0l3_cap_local, adapt_l0l3_cap_peer, adapt_length_l0l3;
854 u32 adapt_l0l1l2l3_cap_local, adapt_l0l1l2l3_cap_peer, adapt_length_l0l1l2l3;
855
856 ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(rx_adapt_initial_cap[gear - 1],
857 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
858 &adapt_l0l3_cap_local);
859 if (ret)
860 return ret;
861
862 if (adapt_l0l3_cap_local > ADAPT_L0L3_LENGTH_MAX) {
863 dev_err(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP (0x%x) exceeds MAX\n",
864 gear, adapt_l0l3_cap_local);
865 return -EINVAL;
866 }
867
868 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(pa_peer_rx_adapt_initial[gear - 1]),
869 &adapt_l0l3_cap_peer);
870 if (ret)
871 return ret;
872
873 if (adapt_l0l3_cap_peer > ADAPT_L0L3_LENGTH_MAX) {
874 dev_err(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_CAP (0x%x) exceeds MAX\n",
875 gear, adapt_l0l3_cap_peer);
876 return -EINVAL;
877 }
878
879 t_adapt_l0l3_local = adapt_cap_to_t_adapt_l0l3(adapt_l0l3_cap_local);
880 t_adapt_l0l3_peer = adapt_cap_to_t_adapt_l0l3(adapt_l0l3_cap_peer);
881
882 dev_dbg(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP = 0x%x\n",
883 gear, adapt_l0l3_cap_local);
884 dev_dbg(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_CAP = 0x%x\n",
885 gear, adapt_l0l3_cap_peer);
886 dev_dbg(hba->dev, "t_adapt_l0l3_local = %llu UI, t_adapt_l0l3_peer = %llu UI\n",
887 t_adapt_l0l3_local, t_adapt_l0l3_peer);
888
889 ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(RX_HS_G6_ADAPT_INITIAL_L0L1L2L3_CAP,
890 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
891 &adapt_l0l1l2l3_cap_local);
892 if (ret)
893 return ret;
894
895 if (adapt_l0l1l2l3_cap_local > ADAPT_L0L1L2L3_LENGTH_MAX) {
896 dev_err(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_L0L1L2L3_CAP (0x%x) exceeds MAX\n",
897 gear, adapt_l0l1l2l3_cap_local);
898
899 if (!(hba->quirks & UFSHCD_QUIRK_EXTENDED_TX_EQTR_ADAPT_LENGTH_L0L1L2L3))
900 return -EINVAL;
901 }
902
903 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTINITIALL0L1L2L3),
904 &adapt_l0l1l2l3_cap_peer);
905 if (ret)
906 return ret;
907
908 if (adapt_l0l1l2l3_cap_peer > ADAPT_L0L1L2L3_LENGTH_MAX) {
909 dev_err(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_L0L1L2L3_CAP (0x%x) exceeds MAX\n",
910 gear, adapt_l0l1l2l3_cap_peer);
911
912 if (!(hba->quirks & UFSHCD_QUIRK_EXTENDED_TX_EQTR_ADAPT_LENGTH_L0L1L2L3))
913 return -EINVAL;
914 }
915
916 t_adapt_l0l1l2l3_local = adapt_cap_to_t_adapt_l0l1l2l3(adapt_l0l1l2l3_cap_local);
917 t_adapt_l0l1l2l3_peer = adapt_cap_to_t_adapt_l0l1l2l3(adapt_l0l1l2l3_cap_peer);
918
919 dev_dbg(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_L0L1L2L3_CAP = 0x%x\n",
920 gear, adapt_l0l1l2l3_cap_local);
921 dev_dbg(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_L0L1L2L3_CAP = 0x%x\n",
922 gear, adapt_l0l1l2l3_cap_peer);
923 dev_dbg(hba->dev, "t_adapt_l0l1l2l3_local = %llu UI, t_adapt_l0l1l2l3_peer = %llu UI\n",
924 t_adapt_l0l1l2l3_local, t_adapt_l0l1l2l3_peer);
925
926 t_adapt_l0l1l2l3 = max(t_adapt_l0l1l2l3_local, t_adapt_l0l1l2l3_peer);
927 t_adapt_l0l3 = max(t_adapt_l0l3_local, t_adapt_l0l3_peer);
928 t_adapt = t_adapt_l0l3 + t_adapt_l0l1l2l3;
929
930 dev_dbg(hba->dev, "TAdapt %llu PAM-4 UI selected for TX EQTR\n",
931 t_adapt);
932
933 adapt_length_l0l3 = (t_adapt_l0l3_local >= t_adapt_l0l3_peer) ?
934 adapt_l0l3_cap_local : adapt_l0l3_cap_peer;
935 adapt_length_l0l1l2l3 = (t_adapt_l0l1l2l3_local >= t_adapt_l0l1l2l3_peer) ?
936 adapt_l0l1l2l3_cap_local : adapt_l0l1l2l3_cap_peer;
937
938 if (t_adapt < TX_EQTR_HS_G6_MIN_T_ADAPT) {
939 dev_dbg(hba->dev, "TAdapt %llu UI is too short for TX EQTR for HS-G%u, use default Adapt 0x%x\n",
940 t_adapt, gear, TX_EQTR_HS_G6_ADAPT_DEFAULT);
941 adapt_length_l0l3 = TX_EQTR_HS_G6_ADAPT_DEFAULT;
942 }
943
944 adapt_eqtr = adapt_length_l0l3 |
945 (adapt_length_l0l1l2l3 << TX_EQTR_ADAPT_LENGTH_L0L1L2L3_SHIFT);
946 } else {
947 return -EINVAL;
948 }
949
950 if (rec)
951 rec->saved_adapt_eqtr = (u16)adapt_eqtr;
952
953 set_adapt_eqtr:
954 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXADAPTLENGTH_EQTR), adapt_eqtr);
955 if (ret)
956 dev_err(hba->dev, "Failed to set adapt length for TX EQTR: %d\n", ret);
957 else
958 dev_dbg(hba->dev, "PA_TXADAPTLENGTH_EQTR configured to 0x%08x\n", adapt_eqtr);
959
960 return ret;
961 }
962
963 /**
964 * ufshcd_compose_tx_eqtr_setting - Compose TX EQTR setting
965 * @iter: TX EQTR iterator data structure
966 * @num_lanes: number of active lanes
967 *
968 * Returns composed TX EQTR setting, same setting is used for all active lanes
969 */
ufshcd_compose_tx_eqtr_setting(struct tx_eqtr_iter * iter,int num_lanes)970 static inline u32 ufshcd_compose_tx_eqtr_setting(struct tx_eqtr_iter *iter,
971 int num_lanes)
972 {
973 u32 setting = 0;
974 int lane;
975
976 for (lane = 0; lane < num_lanes; lane++) {
977 setting |= TX_HS_PRESHOOT_BITS(lane, iter->preshoot);
978 setting |= TX_HS_DEEMPHASIS_BITS(lane, iter->deemphasis);
979 }
980
981 return setting;
982 }
983
984 /**
985 * ufshcd_apply_tx_eqtr_settings - Apply TX EQTR setting
986 * @hba: per adapter instance
987 * @pwr_mode: target power mode containing gear and rate information
988 * @h_iter: host TX EQTR iterator data structure
989 * @d_iter: device TX EQTR iterator data structure
990 *
991 * Returns 0 on success, negative error code otherwise
992 */
ufshcd_apply_tx_eqtr_settings(struct ufs_hba * hba,struct ufs_pa_layer_attr * pwr_mode,struct tx_eqtr_iter * h_iter,struct tx_eqtr_iter * d_iter)993 static int ufshcd_apply_tx_eqtr_settings(struct ufs_hba *hba,
994 struct ufs_pa_layer_attr *pwr_mode,
995 struct tx_eqtr_iter *h_iter,
996 struct tx_eqtr_iter *d_iter)
997 {
998 u32 setting;
999 int ret;
1000
1001 setting = ufshcd_compose_tx_eqtr_setting(h_iter, pwr_mode->lane_tx);
1002 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXEQTRSETTING), setting);
1003 if (ret)
1004 return ret;
1005
1006 setting = ufshcd_compose_tx_eqtr_setting(d_iter, pwr_mode->lane_rx);
1007 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PEERTXEQTRSETTING), setting);
1008 if (ret)
1009 return ret;
1010
1011 ret = ufshcd_vops_apply_tx_eqtr_settings(hba, pwr_mode, h_iter, d_iter);
1012
1013 return ret;
1014 }
1015
1016 /**
1017 * ufshcd_update_tx_eq_params - Update TX Equalization params
1018 * @params: TX EQ parameters data structure
1019 * @pwr_mode: target power mode containing gear and rate
1020 * @eqtr_data: TX EQTR data structure
1021 *
1022 * Update TX Equalization params using results from TX EQTR data. Check also
1023 * the TX EQTR FOM value for each TX lane in the TX EQTR data. If a TX lane got
1024 * a FOM value of 0, restore the TX Equalization settings from the last known
1025 * valid TX Equalization params for that specific TX lane.
1026 */
1027 static inline void
ufshcd_update_tx_eq_params(struct ufshcd_tx_eq_params * params,struct ufs_pa_layer_attr * pwr_mode,struct ufshcd_tx_eqtr_data * eqtr_data)1028 ufshcd_update_tx_eq_params(struct ufshcd_tx_eq_params *params,
1029 struct ufs_pa_layer_attr *pwr_mode,
1030 struct ufshcd_tx_eqtr_data *eqtr_data)
1031 {
1032 struct ufshcd_tx_eqtr_record *rec = params->eqtr_record;
1033
1034 if (params->is_valid) {
1035 int lane;
1036
1037 for (lane = 0; lane < pwr_mode->lane_tx; lane++)
1038 if (eqtr_data->host[lane].fom_val == 0)
1039 eqtr_data->host[lane] = params->host[lane];
1040
1041 for (lane = 0; lane < pwr_mode->lane_rx; lane++)
1042 if (eqtr_data->device[lane].fom_val == 0)
1043 eqtr_data->device[lane] = params->device[lane];
1044 }
1045
1046 memcpy(params->host, eqtr_data->host, sizeof(params->host));
1047 memcpy(params->device, eqtr_data->device, sizeof(params->device));
1048
1049 if (!rec)
1050 return;
1051
1052 memcpy(rec->host_fom, eqtr_data->host_fom, sizeof(rec->host_fom));
1053 memcpy(rec->device_fom, eqtr_data->device_fom, sizeof(rec->device_fom));
1054 rec->last_record_ts = ktime_get();
1055 rec->last_record_index++;
1056 }
1057
1058 /**
1059 * __ufshcd_tx_eqtr - TX Equalization Training (EQTR) procedure
1060 * @hba: per adapter instance
1061 * @params: TX EQ parameters data structure
1062 * @pwr_mode: target power mode containing gear and rate information
1063 *
1064 * This function implements the complete TX EQTR procedure as defined in UFSHCI
1065 * v5.0 specification. It iterates through all possible combinations of PreShoot
1066 * and DeEmphasis settings to find the optimal TX Equalization settings for all
1067 * active lanes.
1068 *
1069 * Returns 0 on success, negative error code otherwise
1070 */
__ufshcd_tx_eqtr(struct ufs_hba * hba,struct ufshcd_tx_eq_params * params,struct ufs_pa_layer_attr * pwr_mode)1071 static int __ufshcd_tx_eqtr(struct ufs_hba *hba,
1072 struct ufshcd_tx_eq_params *params,
1073 struct ufs_pa_layer_attr *pwr_mode)
1074 {
1075 struct ufshcd_tx_eqtr_data *eqtr_data __free(kfree) =
1076 kzalloc_obj(*eqtr_data);
1077 struct tx_eqtr_iter h_iter = {};
1078 struct tx_eqtr_iter d_iter = {};
1079 u32 gear = pwr_mode->gear_tx;
1080 u8 preshoot, deemphasis;
1081 ktime_t start;
1082 int ret;
1083
1084 if (!eqtr_data)
1085 return -ENOMEM;
1086
1087 dev_info(hba->dev, "Start TX EQTR procedure for HS-G%u, Rate-%s, RX Lanes: %u, TX Lanes: %u\n",
1088 gear, ufs_hs_rate_to_str(pwr_mode->hs_rate),
1089 pwr_mode->lane_rx, pwr_mode->lane_tx);
1090
1091 start = ktime_get();
1092
1093 /* Step 1 - Determine the TX Adapt Length for EQTR */
1094 ret = ufshcd_setup_tx_eqtr_adapt_length(hba, params, gear);
1095 if (ret) {
1096 dev_err(hba->dev, "Failed to setup TX EQTR Adaptation length: %d\n", ret);
1097 return ret;
1098 }
1099
1100 /* Step 2 - Determine TX Equalization setting capabilities */
1101 ret = ufshcd_tx_eqtr_iter_init(hba, &h_iter, &d_iter);
1102 if (ret) {
1103 dev_err(hba->dev, "Failed to init TX EQTR data: %d\n", ret);
1104 return ret;
1105 }
1106
1107 /* TX EQTR main loop */
1108 for (preshoot = 0; preshoot < TX_HS_NUM_PRESHOOT; preshoot++) {
1109 for (deemphasis = 0; deemphasis < TX_HS_NUM_DEEMPHASIS; deemphasis++) {
1110 if (!tx_eqtr_iter_update(preshoot, deemphasis, &h_iter, &d_iter))
1111 continue;
1112
1113 /* Step 3 - Apply TX EQTR settings */
1114 ret = ufshcd_apply_tx_eqtr_settings(hba, pwr_mode, &h_iter, &d_iter);
1115 if (ret) {
1116 dev_err(hba->dev, "Failed to apply TX EQTR settings (PreShoot %u, DeEmphasis %u): %d\n",
1117 preshoot, deemphasis, ret);
1118 return ret;
1119 }
1120
1121 /* Step 4 - Trigger UIC TX EQTR */
1122 ret = ufshcd_uic_tx_eqtr(hba, gear);
1123 if (ret) {
1124 dev_err(hba->dev, "Failed to trigger UIC TX EQTR for target gear %u: %d\n",
1125 gear, ret);
1126 return ret;
1127 }
1128
1129 /* Step 5 - Get FOM */
1130 ret = ufshcd_get_rx_fom(hba, pwr_mode, &h_iter, &d_iter);
1131 if (ret) {
1132 dev_err(hba->dev, "Failed to get RX_FOM: %d\n",
1133 ret);
1134 return ret;
1135 }
1136
1137 ufshcd_evaluate_tx_eqtr_fom(hba, pwr_mode, eqtr_data, &h_iter, &d_iter);
1138 }
1139 }
1140
1141 dev_info(hba->dev, "TX EQTR procedure completed! Time elapsed: %llu ms\n",
1142 ktime_to_ms(ktime_sub(ktime_get(), start)));
1143
1144 ufshcd_update_tx_eq_params(params, pwr_mode, eqtr_data);
1145
1146 return ret;
1147 }
1148
1149 /**
1150 * ufshcd_tx_eqtr_prepare - Prepare UFS link for TX EQTR procedure
1151 * @hba: per adapter instance
1152 * @pwr_mode: target power mode containing gear and rate
1153 *
1154 * This function prepares the UFS link for TX Equalization Training (EQTR) by
1155 * establishing the proper initial conditions required by the EQTR procedure.
1156 * It ensures that EQTR starts from the most reliable Power Mode (HS-G1) with
1157 * all connected lanes activated and sets host TX HS Adapt Type to INITIAL.
1158 *
1159 * Returns 0 on successful preparation, negative error code on failure
1160 */
ufshcd_tx_eqtr_prepare(struct ufs_hba * hba,struct ufs_pa_layer_attr * pwr_mode)1161 static int ufshcd_tx_eqtr_prepare(struct ufs_hba *hba,
1162 struct ufs_pa_layer_attr *pwr_mode)
1163 {
1164 struct ufs_pa_layer_attr pwr_mode_hs_g1 = {
1165 /* TX EQTR shall be initiated from the most reliable HS-G1 */
1166 .gear_rx = UFS_HS_G1,
1167 .gear_tx = UFS_HS_G1,
1168 .lane_rx = pwr_mode->lane_rx,
1169 .lane_tx = pwr_mode->lane_tx,
1170 .pwr_rx = FAST_MODE,
1171 .pwr_tx = FAST_MODE,
1172 /* Use the target power mode's HS rate */
1173 .hs_rate = pwr_mode->hs_rate,
1174 };
1175 u32 rate = pwr_mode->hs_rate;
1176 int ret;
1177
1178 /* Change power mode to HS-G1, activate all connected lanes. */
1179 ret = ufshcd_change_power_mode(hba, &pwr_mode_hs_g1,
1180 UFSHCD_PMC_POLICY_DONT_FORCE);
1181 if (ret) {
1182 dev_err(hba->dev, "TX EQTR: Failed to change power mode to HS-G1, Rate-%s: %d\n",
1183 ufs_hs_rate_to_str(rate), ret);
1184 return ret;
1185 }
1186
1187 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXHSADAPTTYPE),
1188 PA_INITIAL_ADAPT);
1189 if (ret)
1190 dev_err(hba->dev, "TX EQTR: Failed to set Host Adapt type to INITIAL: %d\n",
1191 ret);
1192
1193 return ret;
1194 }
1195
ufshcd_tx_eqtr_unprepare(struct ufs_hba * hba,struct ufs_pa_layer_attr * pwr_mode)1196 static void ufshcd_tx_eqtr_unprepare(struct ufs_hba *hba,
1197 struct ufs_pa_layer_attr *pwr_mode)
1198 {
1199 int err;
1200
1201 if (pwr_mode->pwr_rx == SLOWAUTO_MODE || pwr_mode->hs_rate == 0)
1202 return;
1203
1204 err = ufshcd_change_power_mode(hba, pwr_mode,
1205 UFSHCD_PMC_POLICY_DONT_FORCE);
1206 if (err)
1207 dev_err(hba->dev, "%s: Failed to restore Power Mode: %d\n",
1208 __func__, err);
1209 }
1210
1211 /**
1212 * ufshcd_tx_eqtr - Perform TX EQTR procedures with vops callbacks
1213 * @hba: per adapter instance
1214 * @params: TX EQ parameters data structure to populate
1215 * @pwr_mode: target power mode containing gear and rate information
1216 *
1217 * This is the main entry point for performing TX Equalization Training (EQTR)
1218 * procedure as defined in UFSCHI v5.0 specification. It serves as a wrapper
1219 * around __ufshcd_tx_eqtr() to provide vops support through the variant
1220 * operations framework.
1221 *
1222 * Returns 0 on success, negative error code on failure
1223 */
ufshcd_tx_eqtr(struct ufs_hba * hba,struct ufshcd_tx_eq_params * params,struct ufs_pa_layer_attr * pwr_mode)1224 static int ufshcd_tx_eqtr(struct ufs_hba *hba,
1225 struct ufshcd_tx_eq_params *params,
1226 struct ufs_pa_layer_attr *pwr_mode)
1227 {
1228 struct ufs_pa_layer_attr old_pwr_info;
1229 unsigned int noio_flag;
1230 int notify_ret;
1231 int ret;
1232
1233 /*
1234 * ufshcd_tx_eqtr() is called from a power-mode-change context where
1235 * I/O is suspended. Use memalloc_noio_save() to propagate GFP_NOIO
1236 * to all allocations in the call tree instead of tagging each call
1237 * site individually.
1238 */
1239 noio_flag = memalloc_noio_save();
1240
1241 if (!params->eqtr_record) {
1242 params->eqtr_record = devm_kzalloc(hba->dev,
1243 sizeof(*params->eqtr_record),
1244 GFP_KERNEL);
1245 if (!params->eqtr_record) {
1246 ret = -ENOMEM;
1247 goto out_noio_restore;
1248 }
1249 }
1250
1251 memcpy(&old_pwr_info, &hba->pwr_info, sizeof(struct ufs_pa_layer_attr));
1252
1253 ret = ufshcd_tx_eqtr_prepare(hba, pwr_mode);
1254 if (ret) {
1255 dev_err(hba->dev, "Failed to prepare TX EQTR: %d\n", ret);
1256 goto out_unprepare;
1257 }
1258
1259 ret = ufshcd_vops_tx_eqtr_notify(hba, PRE_CHANGE, pwr_mode);
1260 if (ret) {
1261 dev_err(hba->dev, "TX EQTR PRE_CHANGE notify failed: %d\n", ret);
1262 goto out_unprepare;
1263 }
1264
1265 ret = __ufshcd_tx_eqtr(hba, params, pwr_mode);
1266
1267 notify_ret = ufshcd_vops_tx_eqtr_notify(hba, POST_CHANGE, pwr_mode);
1268 if (notify_ret)
1269 dev_err(hba->dev, "TX EQTR POST_CHANGE notify failed: %d\n", notify_ret);
1270
1271 if (!ret)
1272 ret = notify_ret;
1273
1274 out_unprepare:
1275 if (ret)
1276 ufshcd_tx_eqtr_unprepare(hba, &old_pwr_info);
1277
1278 out_noio_restore:
1279 memalloc_noio_restore(noio_flag);
1280
1281 return ret;
1282 }
1283
1284 /**
1285 * ufshcd_config_tx_eq_settings - Configure TX Equalization settings
1286 * @hba: per adapter instance
1287 * @pwr_mode: target power mode containing gear and rate information
1288 * @force_tx_eqtr: execute the TX EQTR procedure
1289 *
1290 * This function finds and sets the TX Equalization settings for the given
1291 * target power mode.
1292 *
1293 * Returns 0 on success, error code otherwise
1294 */
ufshcd_config_tx_eq_settings(struct ufs_hba * hba,struct ufs_pa_layer_attr * pwr_mode,bool force_tx_eqtr)1295 int ufshcd_config_tx_eq_settings(struct ufs_hba *hba,
1296 struct ufs_pa_layer_attr *pwr_mode,
1297 bool force_tx_eqtr)
1298 {
1299 struct ufshcd_tx_eq_params *params;
1300 u32 gear, rate;
1301
1302 if (!ufshcd_is_tx_eq_supported(hba) || !use_adaptive_txeq)
1303 return 0;
1304
1305 if (!hba->max_pwr_info.is_valid) {
1306 dev_err(hba->dev, "Max power info is invalid\n");
1307 return -EINVAL;
1308 }
1309
1310 if (!pwr_mode) {
1311 dev_err(hba->dev, "Target power mode is NULL\n");
1312 return -EINVAL;
1313 }
1314
1315 gear = pwr_mode->gear_tx;
1316 rate = pwr_mode->hs_rate;
1317
1318 if (gear < UFS_HS_G1 || gear > UFS_HS_GEAR_MAX) {
1319 dev_err(hba->dev, "Invalid HS-Gear (%u) for TX Equalization\n",
1320 gear);
1321 return -EINVAL;
1322 } else if (gear < max_t(u32, adaptive_txeq_gear, UFS_HS_G4)) {
1323 /* TX EQTR is supported for HS-G4 and higher Gears */
1324 return 0;
1325 }
1326
1327 if (rate != PA_HS_MODE_A && rate != PA_HS_MODE_B) {
1328 dev_err(hba->dev, "Invalid HS-Rate (%u) for TX Equalization\n",
1329 rate);
1330 return -EINVAL;
1331 }
1332
1333 params = &hba->tx_eq_params[gear - 1];
1334 /*
1335 * TX EQTR must run for the following cases:
1336 * 1. TX EQ settings are invalid.
1337 * 2. TX EQ settings are from Device Tree.
1338 * 3. TX EQTR procedure is forced.
1339 */
1340 if (!params->is_valid || params->from_dt || force_tx_eqtr) {
1341 int ret;
1342
1343 ret = ufshcd_tx_eqtr(hba, params, pwr_mode);
1344 if (ret) {
1345 dev_err(hba->dev, "Failed to train TX Equalization for HS-G%u, Rate-%s: %d\n",
1346 gear, ufs_hs_rate_to_str(rate), ret);
1347 return ret;
1348 }
1349
1350 /* Mark TX Equalization settings as valid */
1351 params->is_valid = true;
1352 params->is_trained = true;
1353 /* TX EQTR succeeds, clear from_dt flag */
1354 params->from_dt = false;
1355 params->is_applied = false;
1356 }
1357
1358 if (params->is_valid && !params->is_applied) {
1359 int ret;
1360
1361 ret = ufshcd_apply_tx_eq_settings(hba, params, gear);
1362 if (ret) {
1363 dev_err(hba->dev, "Failed to apply TX Equalization settings for HS-G%u, Rate-%s: %d\n",
1364 gear, ufs_hs_rate_to_str(rate), ret);
1365 return ret;
1366 }
1367
1368 params->is_applied = true;
1369 }
1370
1371 return 0;
1372 }
1373
1374 /**
1375 * ufshcd_apply_valid_tx_eq_settings - Apply valid TX Equalization settings
1376 * @hba: per-adapter instance
1377 *
1378 * This function iterates through all supported High-Speed (HS) gears and
1379 * applies valid TX Equalization settings to both Host and Device.
1380 */
ufshcd_apply_valid_tx_eq_settings(struct ufs_hba * hba)1381 void ufshcd_apply_valid_tx_eq_settings(struct ufs_hba *hba)
1382 {
1383 struct ufshcd_tx_eq_params *params;
1384 int gear, err;
1385
1386 if (!ufshcd_is_tx_eq_supported(hba))
1387 return;
1388
1389 if (!hba->max_pwr_info.is_valid) {
1390 dev_err(hba->dev, "Max power info is invalid, cannot apply TX Equalization settings\n");
1391 return;
1392 }
1393
1394 for (gear = UFS_HS_G1; gear <= UFS_HS_GEAR_MAX; gear++) {
1395 params = &hba->tx_eq_params[gear - 1];
1396
1397 if (params->is_valid) {
1398 err = ufshcd_apply_tx_eq_settings(hba, params, gear);
1399 if (err) {
1400 params->is_applied = false;
1401 dev_err(hba->dev, "Failed to apply TX Equalization settings for HS-G%u: %d\n",
1402 gear, err);
1403 } else {
1404 params->is_applied = true;
1405 }
1406 }
1407 }
1408 }
1409
1410 /**
1411 * ufshcd_retrain_tx_eq - Retrain TX Equalization and apply new settings
1412 * @hba: per-adapter instance
1413 * @gear: target High-Speed (HS) gear for retraining
1414 *
1415 * This function initiates a refresh of the TX Equalization settings for a
1416 * specific HS gear. It scales the clocks to maximum frequency, negotiates the
1417 * power mode with the device, retrains TX EQ and applies new TX EQ settings
1418 * by conducting a Power Mode change.
1419 *
1420 * Returns 0 on success, non-zero error code otherwise
1421 */
ufshcd_retrain_tx_eq(struct ufs_hba * hba,u32 gear)1422 int ufshcd_retrain_tx_eq(struct ufs_hba *hba, u32 gear)
1423 {
1424 struct ufs_pa_layer_attr new_pwr_info, final_params = {};
1425 int ret;
1426
1427 if (!ufshcd_is_tx_eq_supported(hba) || !use_adaptive_txeq)
1428 return -EOPNOTSUPP;
1429
1430 if (gear < adaptive_txeq_gear)
1431 return -ERANGE;
1432
1433 ufshcd_hold(hba);
1434
1435 ret = ufshcd_pause_command_processing(hba, 1 * USEC_PER_SEC);
1436 if (ret) {
1437 ufshcd_release(hba);
1438 return ret;
1439 }
1440
1441 /* scale up clocks to max frequency before TX EQTR */
1442 if (ufshcd_is_clkscaling_supported(hba))
1443 ufshcd_scale_clks(hba, ULONG_MAX, true);
1444
1445 new_pwr_info = hba->pwr_info;
1446 new_pwr_info.gear_tx = gear;
1447 new_pwr_info.gear_rx = gear;
1448
1449 ret = ufshcd_vops_negotiate_pwr_mode(hba, &new_pwr_info, &final_params);
1450 if (ret)
1451 memcpy(&final_params, &new_pwr_info, sizeof(final_params));
1452
1453 if (final_params.gear_tx != gear) {
1454 dev_err(hba->dev, "Negotiated Gear (%u) does not match target Gear (%u)\n",
1455 final_params.gear_tx, gear);
1456 ret = -EINVAL;
1457 goto out;
1458 }
1459
1460 ret = ufshcd_config_tx_eq_settings(hba, &final_params, true);
1461 if (ret) {
1462 dev_err(hba->dev, "Failed to config TX Equalization for HS-G%u, Rate-%s: %d\n",
1463 final_params.gear_tx,
1464 ufs_hs_rate_to_str(final_params.hs_rate), ret);
1465 goto out;
1466 }
1467
1468 /* Change Power Mode to apply the new TX EQ settings */
1469 ret = ufshcd_change_power_mode(hba, &final_params,
1470 UFSHCD_PMC_POLICY_FORCE);
1471 if (ret)
1472 dev_err(hba->dev, "%s: Failed to change Power Mode to HS-G%u, Rate-%s: %d\n",
1473 __func__, final_params.gear_tx,
1474 ufs_hs_rate_to_str(final_params.hs_rate), ret);
1475
1476 out:
1477 ufshcd_resume_command_processing(hba);
1478 ufshcd_release(hba);
1479
1480 return ret;
1481 }
1482
1483 /**
1484 * ufshcd_extract_tx_eq_settings_attrs - Extract TX Equalization settings from UFS attributes
1485 * @hba: per adapter instance
1486 * @gear: target gear
1487 *
1488 * This function extracts previously stored TX Equalization settings from UFS
1489 * attributes qTxEQGnSettings and wTxEQGnSettingsExt. These attributes contain
1490 * the optimal TX Equalization parameters (PreShoot, DeEmphasis, and PreCoding
1491 * enable) that were determined during a previous EQTR procedure.
1492 *
1493 * The function reads:
1494 * 1. qTxEQGnSettings (64-bit): Main attribute containing PreShoot and
1495 * DeEmphasis values for both host and device TX lanes
1496 * 2. wTxEQGnSettingsExt (16-bit): Extended attribute containing PreCoding
1497 * enable flags and validity indicator
1498 */
ufshcd_extract_tx_eq_settings_attrs(struct ufs_hba * hba,u8 gear)1499 static void ufshcd_extract_tx_eq_settings_attrs(struct ufs_hba *hba, u8 gear)
1500 {
1501 struct ufshcd_tx_eq_params *params;
1502 u32 lane, eq_ext;
1503 int ret;
1504 u64 eq;
1505
1506 ret = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_READ_ATTR,
1507 QUERY_ATTR_IDN_TX_EQ_GN_SETTINGS_EXT, gear - 1,
1508 (u8)txeq_setting_sel, &eq_ext);
1509 if (ret)
1510 return;
1511
1512 dev_dbg(hba->dev, "%s: HS-G%u wTxEQGnSettingsExt (Selector %u) = 0x%08x\n",
1513 __func__, gear, txeq_setting_sel, eq_ext);
1514
1515 if (!(eq_ext & TX_EQ_SETTINGS_VALID_BIT))
1516 return;
1517
1518 ret = ufshcd_query_attr_qword(hba, UPIU_QUERY_OPCODE_READ_ATTR,
1519 QUERY_ATTR_IDN_TX_EQ_GN_SETTINGS,
1520 gear - 1, (u8)txeq_setting_sel, &eq);
1521 if (ret)
1522 return;
1523
1524 dev_dbg(hba->dev, "%s: HS-G%u qTxEQGnSettings (Selector %u) = 0x%016llx\n",
1525 __func__, gear, txeq_setting_sel, eq);
1526
1527 params = &hba->tx_eq_params[gear - 1];
1528
1529 for (lane = 0; lane < UFS_MAX_LANES; lane++) {
1530 params->host[lane].preshoot = tx_eq_host_preshoot_decode(eq, lane);
1531 params->host[lane].deemphasis = tx_eq_host_deemphasis_decode(eq, lane);
1532 params->host[lane].precode_en = tx_eq_host_precode_en_decode(eq_ext, lane);
1533
1534 params->device[lane].preshoot = tx_eq_device_preshoot_decode(eq, lane);
1535 params->device[lane].deemphasis = tx_eq_device_deemphasis_decode(eq, lane);
1536 params->device[lane].precode_en = tx_eq_device_precode_en_decode(eq_ext, lane);
1537 }
1538
1539 params->is_valid = true;
1540 /*
1541 * Optimal TX EQ settings are retrieved from UFS device attributes,
1542 * clear from_dt flag to avoid TX EQTR procedure.
1543 */
1544 params->from_dt = false;
1545 }
1546
ufshcd_retrieve_tx_eq_settings(struct ufs_hba * hba)1547 void ufshcd_retrieve_tx_eq_settings(struct ufs_hba *hba)
1548 {
1549 u8 gear = (u8)adaptive_txeq_gear;
1550
1551 if (!hba->max_pwr_info.is_valid || !ufshcd_is_tx_eq_supported(hba) ||
1552 !use_adaptive_txeq || !retrieve_txeq_setting)
1553 return;
1554
1555 for (; gear <= UFS_HS_GEAR_MAX; gear++)
1556 ufshcd_extract_tx_eq_settings_attrs(hba, gear);
1557 }
1558
1559 /**
1560 * ufshcd_update_tx_eq_settings_attrs - Update TX EQ settings in UFS attributes
1561 * @hba: per adapter instance
1562 * @gear: target gear
1563 *
1564 * This function stores the optimal TX Equalization settings obtained from
1565 * TX EQTR procedure into UFS device attributes for future fast-path retrieval.
1566 * The settings are stored in two complementary attributes:
1567 *
1568 * 1. qTxEQGnSettings (64-bit): Main attribute containing PreShoot and
1569 * DeEmphasis values for both host and device TX lanes
1570 * 2. wTxEQGnSettingsExt (16-bit): Extended attribute containing PreCoding
1571 * enable flags and validity indicator
1572 */
ufshcd_update_tx_eq_settings_attrs(struct ufs_hba * hba,u8 gear)1573 static void ufshcd_update_tx_eq_settings_attrs(struct ufs_hba *hba, u8 gear)
1574 {
1575 struct ufshcd_tx_eq_params *params;
1576 u32 lane, eq_ext = 0;
1577 u64 eq = 0;
1578 int ret;
1579
1580 params = &hba->tx_eq_params[gear - 1];
1581 if (!params->is_valid || !params->is_trained)
1582 return;
1583
1584 for (lane = 0; lane < UFS_MAX_LANES; lane++) {
1585 eq |= tx_eq_host_preshoot_encode((u64)params->host[lane].preshoot, lane);
1586 eq |= tx_eq_host_deemphasis_encode((u64)params->host[lane].deemphasis, lane);
1587 eq_ext |= tx_eq_host_precode_en_encode(params->host[lane].precode_en, lane);
1588
1589 eq |= tx_eq_device_preshoot_encode((u64)params->device[lane].preshoot, lane);
1590 eq |= tx_eq_device_deemphasis_encode((u64)params->device[lane].deemphasis, lane);
1591 eq_ext |= tx_eq_device_precode_en_encode(params->device[lane].precode_en, lane);
1592 }
1593
1594 /* Set validity flag to indicate valid settings are stored */
1595 eq_ext |= TX_EQ_SETTINGS_VALID_BIT;
1596
1597 /* Write qTxEQGnSettings */
1598 ret = ufshcd_query_attr_qword(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
1599 QUERY_ATTR_IDN_TX_EQ_GN_SETTINGS,
1600 gear - 1, (u8)txeq_setting_sel, &eq);
1601 if (ret)
1602 return;
1603
1604 /* Write wTxEQGnSettingsExt */
1605 ret = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
1606 QUERY_ATTR_IDN_TX_EQ_GN_SETTINGS_EXT, gear - 1,
1607 (u8)txeq_setting_sel, &eq_ext);
1608 if (ret)
1609 return;
1610
1611 dev_dbg(hba->dev, "%s: Saved HS-G%u qTxEQGnSettings (Selector %u) = 0x%016llx\n",
1612 __func__, gear, txeq_setting_sel, eq);
1613 dev_dbg(hba->dev, "%s: Saved HS-G%u wTxEQGnSettingsExt (Selector %u) = 0x%08x\n",
1614 __func__, gear, txeq_setting_sel, eq_ext);
1615 }
1616
ufshcd_store_tx_eq_settings(struct ufs_hba * hba)1617 void ufshcd_store_tx_eq_settings(struct ufs_hba *hba)
1618 {
1619 u8 gear = (u8)adaptive_txeq_gear;
1620
1621 if (!hba->max_pwr_info.is_valid || !ufshcd_is_tx_eq_supported(hba) ||
1622 !use_adaptive_txeq || !store_txeq_setting)
1623 return;
1624
1625 for (; gear <= UFS_HS_GEAR_MAX; gear++)
1626 ufshcd_update_tx_eq_settings_attrs(hba, gear);
1627 }
1628