1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2026 Qualcomm Technologies, Inc. 4 * 5 * Author: 6 * Can Guo <can.guo@oss.qualcomm.com> 7 */ 8 9 #include <linux/bitops.h> 10 #include <linux/delay.h> 11 #include <linux/errno.h> 12 #include <linux/kernel.h> 13 #include <linux/sched/mm.h> 14 #include <ufs/ufshcd.h> 15 #include <ufs/unipro.h> 16 #include "ufshcd-priv.h" 17 18 #define TX_EQ_SETTING_MASK 0x7 19 #define TX_EQ_SETTINGS_VALID_BIT BIT(15) 20 21 static bool use_adaptive_txeq; 22 module_param(use_adaptive_txeq, bool, 0644); 23 MODULE_PARM_DESC(use_adaptive_txeq, "Find and apply optimal TX Equalization settings before changing Power Mode (default: false)"); 24 25 static int txeq_gear_set(const char *val, const struct kernel_param *kp) 26 { 27 return param_set_uint_minmax(val, kp, UFS_HS_G1, UFS_HS_GEAR_MAX); 28 } 29 30 static const struct kernel_param_ops txeq_gear_ops = { 31 .set = txeq_gear_set, 32 .get = param_get_uint, 33 }; 34 35 static unsigned int adaptive_txeq_gear = UFS_HS_G6; 36 module_param_cb(adaptive_txeq_gear, &txeq_gear_ops, &adaptive_txeq_gear, 0644); 37 MODULE_PARM_DESC(adaptive_txeq_gear, "For HS-Gear[n] and above, adaptive txeq shall be used"); 38 39 static bool use_txeq_presets; 40 module_param(use_txeq_presets, bool, 0644); 41 MODULE_PARM_DESC(use_txeq_presets, "Use only the 8 TX Equalization Presets (pre-defined Pre-Shoot & De-Emphasis combinations) for TX EQTR (default: false)"); 42 43 static bool txeq_presets_selected[UFS_TX_EQ_PRESET_MAX] = {[0 ... (UFS_TX_EQ_PRESET_MAX - 1)] = 1}; 44 module_param_array(txeq_presets_selected, bool, NULL, 0644); 45 MODULE_PARM_DESC(txeq_presets_selected, "Use only the selected Presets out of the 8 TX Equalization Presets for TX EQTR"); 46 47 static int txeq_setting_sel_set(const char *val, const struct kernel_param *kp) 48 { 49 return param_set_uint_minmax(val, kp, 0, 1); 50 } 51 52 static const struct kernel_param_ops txeq_setting_sel_ops = { 53 .set = txeq_setting_sel_set, 54 .get = param_get_uint, 55 }; 56 57 static unsigned int txeq_setting_sel; 58 module_param_cb(txeq_setting_sel, &txeq_setting_sel_ops, &txeq_setting_sel, 0644); 59 MODULE_PARM_DESC(txeq_setting_sel, "The qTxEQGnSettings and wTxEQGnSettingsExt Attributes selector used to retrieve and store TX Equalization settings"); 60 61 static bool retrieve_txeq_setting = true; 62 module_param(retrieve_txeq_setting, bool, 0644); 63 MODULE_PARM_DESC(retrieve_txeq_setting, "Retrieve TX Equalization settings from qTxEQGnSettings and wTxEQGnSettingsExt Attributes (default: true)"); 64 65 static bool store_txeq_setting = true; 66 module_param(store_txeq_setting, bool, 0644); 67 MODULE_PARM_DESC(store_txeq_setting, "Store the optimal TX Equalization settings to qTxEQGnSettings and wTxEQGnSettingsExt Attributes (default: true)"); 68 69 /* 70 * ufs_tx_eq_preset - Table of minimum required list of presets. 71 * 72 * A HS-G6 capable M-TX shall support the presets defined in M-PHY v6.0 spec. 73 * Preset Pre-Shoot(dB) De-Emphasis(dB) 74 * P0 0.0 0.0 75 * P1 0.0 0.8 76 * P2 0.0 1.6 77 * P3 0.8 0.0 78 * P4 1.6 0.0 79 * P5 0.8 0.8 80 * P6 0.8 1.6 81 * P7 1.6 0.8 82 */ 83 static const struct __ufs_tx_eq_preset { 84 u8 preshoot; 85 u8 deemphasis; 86 } ufs_tx_eq_preset[UFS_TX_EQ_PRESET_MAX] = { 87 [UFS_TX_EQ_PRESET_P0] = {UFS_TX_HS_PRESHOOT_DB_0P0, UFS_TX_HS_DEEMPHASIS_DB_0P0}, 88 [UFS_TX_EQ_PRESET_P1] = {UFS_TX_HS_PRESHOOT_DB_0P0, UFS_TX_HS_DEEMPHASIS_DB_0P8}, 89 [UFS_TX_EQ_PRESET_P2] = {UFS_TX_HS_PRESHOOT_DB_0P0, UFS_TX_HS_DEEMPHASIS_DB_1P6}, 90 [UFS_TX_EQ_PRESET_P3] = {UFS_TX_HS_PRESHOOT_DB_0P8, UFS_TX_HS_DEEMPHASIS_DB_0P0}, 91 [UFS_TX_EQ_PRESET_P4] = {UFS_TX_HS_PRESHOOT_DB_1P6, UFS_TX_HS_DEEMPHASIS_DB_0P0}, 92 [UFS_TX_EQ_PRESET_P5] = {UFS_TX_HS_PRESHOOT_DB_0P8, UFS_TX_HS_DEEMPHASIS_DB_0P8}, 93 [UFS_TX_EQ_PRESET_P6] = {UFS_TX_HS_PRESHOOT_DB_0P8, UFS_TX_HS_DEEMPHASIS_DB_1P6}, 94 [UFS_TX_EQ_PRESET_P7] = {UFS_TX_HS_PRESHOOT_DB_1P6, UFS_TX_HS_DEEMPHASIS_DB_0P8}, 95 }; 96 97 /* 98 * pa_peer_rx_adapt_initial - Table of UniPro PA_PeerRxHSGnAdaptInitial 99 * attribute IDs for High Speed (HS) Gears. 100 * 101 * This table maps HS Gears to their respective UniPro PA_PeerRxHSGnAdaptInitial 102 * attribute IDs. Entries for Gears 1-3 are 0 (unsupported). 103 */ 104 static const u32 pa_peer_rx_adapt_initial[UFS_HS_GEAR_MAX] = { 105 0, 106 0, 107 0, 108 PA_PEERRXHSG4ADAPTINITIAL, 109 PA_PEERRXHSG5ADAPTINITIAL, 110 PA_PEERRXHSG6ADAPTINITIALL0L3 111 }; 112 113 /* 114 * rx_adapt_initial_cap - Table of M-PHY RX_HS_Gn_ADAPT_INITIAL_Capability 115 * attribute IDs for High Speed (HS) Gears. 116 * 117 * This table maps HS Gears to their respective M-PHY 118 * RX_HS_Gn_ADAPT_INITIAL_Capability attribute IDs. Entries for Gears 1-3 are 0 119 * (unsupported). 120 */ 121 static const u32 rx_adapt_initial_cap[UFS_HS_GEAR_MAX] = { 122 0, 123 0, 124 0, 125 RX_HS_G4_ADAPT_INITIAL_CAP, 126 RX_HS_G5_ADAPT_INITIAL_CAP, 127 RX_HS_G6_ADAPT_INITIAL_CAP 128 }; 129 130 /* 131 * pa_tx_eq_setting - Table of UniPro PA_TxEQGnSetting attribute IDs for High 132 * Speed (HS) Gears. 133 * 134 * This table maps HS Gears to their respective UniPro PA_TxEQGnSetting 135 * attribute IDs. 136 */ 137 static const u32 pa_tx_eq_setting[UFS_HS_GEAR_MAX] = { 138 PA_TXEQG1SETTING, 139 PA_TXEQG2SETTING, 140 PA_TXEQG3SETTING, 141 PA_TXEQG4SETTING, 142 PA_TXEQG5SETTING, 143 PA_TXEQG6SETTING 144 }; 145 146 /* 147 * Decode Device TX Equalization PreShoot value based on qTxEQGnSettings bit assignment: 148 * bit[3:0]: Device TX Logical LANE 0 PreShoot 149 * bit[7:4]: Device TX Logical LANE 1 PreShoot 150 */ 151 static inline u8 tx_eq_device_preshoot_decode(u64 eq, u8 lane) 152 { 153 return (u8)((eq >> (lane * TX_HS_PRESHOOT_SHIFT)) & TX_EQ_SETTING_MASK); 154 } 155 156 /* 157 * Decode Device TX Equalization DeEmphasis value based on qTxEQGnSettings bit assignment: 158 * bit[19:16]: Device TX Logical LANE 0 DeEmphasis 159 * bit[23:20]: Device TX Logical LANE 1 DeEmphasis 160 */ 161 static inline u8 tx_eq_device_deemphasis_decode(u64 eq, u8 lane) 162 { 163 return (u8)((eq >> (lane * TX_HS_DEEMPHASIS_SHIFT + 16)) & TX_EQ_SETTING_MASK); 164 } 165 166 /* 167 * Decode Host TX Equalization PreShoot value based on qTxEQGnSettings bit assignment: 168 * bit[35:32]: Host TX Logical LANE 0 PreShoot 169 * bit[39:36]: Host TX Logical LANE 1 PreShoot 170 */ 171 static inline u8 tx_eq_host_preshoot_decode(u64 eq, u8 lane) 172 { 173 return (u8)((eq >> (lane * TX_HS_PRESHOOT_SHIFT + 32)) & TX_EQ_SETTING_MASK); 174 } 175 176 /* 177 * Decode Host TX Equalization DeEmphasis value based on qTxEQGnSettings bit assignment: 178 * bit[51:48]: Host TX Logical LANE 0 DeEmphasis 179 * bit[55:52]: Host TX Logical LANE 1 DeEmphasis 180 */ 181 static inline u8 tx_eq_host_deemphasis_decode(u64 eq, u8 lane) 182 { 183 return (u8)((eq >> (lane * TX_HS_DEEMPHASIS_SHIFT + 48)) & TX_EQ_SETTING_MASK); 184 } 185 186 /* 187 * Decode Device TX precode_en indication based on wTxEQGnSettingsExt bit assignment: 188 * bit[0]: PreCodeEn for Device TX Logical LANE 0 189 * bit[1]: PreCodeEn for Device TX Logical LANE 1 190 */ 191 static inline bool tx_eq_device_precode_en_decode(u16 eq_ext, u8 lane) 192 { 193 return eq_ext & BIT(lane); 194 } 195 196 /* 197 * Decode Host TX precode_en indication based on wTxEQGnSettingsExt bit assignment: 198 * bit[4]: PreCodeEn for Device RX Logical LANE 0 199 * bit[5]: PreCodeEn for Device RX Logical LANE 1 200 */ 201 static inline bool tx_eq_host_precode_en_decode(u16 eq_ext, u8 lane) 202 { 203 return eq_ext & BIT(lane + 4); 204 } 205 206 /* 207 * Encode Device TX Equalization PreShoot value based on qTxEQGnSettings bit assignment: 208 * bit[3:0]: Device TX Logical LANE 0 PreShoot 209 * bit[7:4]: Device TX Logical LANE 1 PreShoot 210 */ 211 static inline u64 tx_eq_device_preshoot_encode(u64 val, u8 lane) 212 { 213 return (val & TX_EQ_SETTING_MASK) << (lane * TX_HS_PRESHOOT_SHIFT); 214 } 215 216 /* 217 * Encode Device TX Equalization DeEmphasis value based on qTxEQGnSettings bit assignment: 218 * bit[19:16]: Device TX Logical LANE 0 DeEmphasis 219 * bit[23:20]: Device TX Logical LANE 1 DeEmphasis 220 */ 221 static inline u64 tx_eq_device_deemphasis_encode(u64 val, u8 lane) 222 { 223 return (val & TX_EQ_SETTING_MASK) << (lane * TX_HS_DEEMPHASIS_SHIFT + 16); 224 } 225 226 /* 227 * Encode Host TX Equalization PreShoot value based on qTxEQGnSettings bit assignment: 228 * bit[35:32]: Host TX Logical LANE 0 PreShoot 229 * bit[39:36]: Host TX Logical LANE 1 PreShoot 230 */ 231 static inline u64 tx_eq_host_preshoot_encode(u64 val, u8 lane) 232 { 233 return (val & TX_EQ_SETTING_MASK) << (lane * TX_HS_PRESHOOT_SHIFT + 32); 234 } 235 236 /* 237 * Encode Host TX Equalization DeEmphasis value based on qTxEQGnSettings bit assignment: 238 * bit[51:48]: Host TX Logical LANE 0 DeEmphasis 239 * bit[55:52]: Host TX Logical LANE 1 DeEmphasis 240 */ 241 static inline u64 tx_eq_host_deemphasis_encode(u64 val, u8 lane) 242 { 243 return (val & TX_EQ_SETTING_MASK) << (lane * TX_HS_DEEMPHASIS_SHIFT + 48); 244 } 245 246 /* 247 * Encode Device precode_en based on wTxEQGnSettingsExt bit assignment: 248 * bit[0]: PreCodeEn for Device TX Logical LANE 0 249 * bit[1]: PreCodeEn for Device TX Logical LANE 1 250 */ 251 static inline u16 tx_eq_device_precode_en_encode(bool en, u8 lane) 252 { 253 return (u16)en << lane; 254 } 255 256 /* 257 * Encode Host precode_en based on wTxEQGnSettingsExt bit assignment: 258 * bit[4]: PreCodeEn for Device RX Logical LANE 0 259 * bit[5]: PreCodeEn for Device RX Logical LANE 1 260 */ 261 static inline u16 tx_eq_host_precode_en_encode(bool en, u8 lane) 262 { 263 return (u16)en << (lane + 4); 264 } 265 266 /** 267 * ufshcd_configure_precoding - Configure Pre-Coding for all active lanes 268 * @hba: per adapter instance 269 * @params: TX EQ parameters data structure 270 * 271 * Bit[7] in RX_FOM indicates that the receiver needs to enable Pre-Coding when 272 * set. Pre-Coding must be enabled on both the transmitter and receiver to 273 * ensure proper operation. 274 * 275 * Returns 0 on success, non-zero error code otherwise 276 */ 277 static int ufshcd_configure_precoding(struct ufs_hba *hba, 278 struct ufshcd_tx_eq_params *params) 279 { 280 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; 281 u32 local_precode_en = 0; 282 u32 peer_precode_en = 0; 283 int lane, ret; 284 285 /* Enable Pre-Coding for Host's TX & Device's RX pair */ 286 for (lane = 0; lane < pwr_info->lane_tx; lane++) { 287 if (params->host[lane].precode_en) { 288 local_precode_en |= PRECODEEN_TX_BIT(lane); 289 peer_precode_en |= PRECODEEN_RX_BIT(lane); 290 } 291 } 292 293 /* Enable Pre-Coding for Device's TX & Host's RX pair */ 294 for (lane = 0; lane < pwr_info->lane_rx; lane++) { 295 if (params->device[lane].precode_en) { 296 peer_precode_en |= PRECODEEN_TX_BIT(lane); 297 local_precode_en |= PRECODEEN_RX_BIT(lane); 298 } 299 } 300 301 if (!local_precode_en && !peer_precode_en) { 302 dev_dbg(hba->dev, "Pre-Coding is not required for Host and Device\n"); 303 return 0; 304 } 305 306 /* Set local PA_PreCodeEn */ 307 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PRECODEEN), local_precode_en); 308 if (ret) { 309 dev_err(hba->dev, "Failed to set local PA_PreCodeEn: %d\n", ret); 310 return ret; 311 } 312 313 /* Set peer PA_PreCodeEn */ 314 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_PRECODEEN), peer_precode_en); 315 if (ret) { 316 dev_err(hba->dev, "Failed to set peer PA_PreCodeEn: %d\n", ret); 317 return ret; 318 } 319 320 dev_dbg(hba->dev, "Local PA_PreCodeEn: 0x%02x, Peer PA_PreCodeEn: 0x%02x\n", 321 local_precode_en, peer_precode_en); 322 323 return 0; 324 } 325 326 void ufshcd_print_tx_eq_params(struct ufs_hba *hba) 327 { 328 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; 329 struct ufshcd_tx_eq_params *params; 330 u32 gear = hba->pwr_info.gear_tx; 331 int lane; 332 333 if (!ufshcd_is_tx_eq_supported(hba)) 334 return; 335 336 if (gear < UFS_HS_G1 || gear > UFS_HS_GEAR_MAX) 337 return; 338 339 params = &hba->tx_eq_params[gear - 1]; 340 if (!params->is_valid || !params->is_applied) 341 return; 342 343 for (lane = 0; lane < pwr_info->lane_tx; lane++) 344 dev_dbg(hba->dev, "Host TX Lane %d: PreShoot %u, DeEmphasis %u, FOM %u, PreCodeEn %d\n", 345 lane, params->host[lane].preshoot, 346 params->host[lane].deemphasis, 347 params->host[lane].fom_val, 348 params->host[lane].precode_en); 349 350 for (lane = 0; lane < pwr_info->lane_rx; lane++) 351 dev_dbg(hba->dev, "Device TX Lane %d: PreShoot %u, DeEmphasis %u, FOM %u, PreCodeEn %d\n", 352 lane, params->device[lane].preshoot, 353 params->device[lane].deemphasis, 354 params->device[lane].fom_val, 355 params->device[lane].precode_en); 356 } 357 358 static inline u32 359 ufshcd_compose_tx_eq_setting(struct ufshcd_tx_eq_settings *settings, 360 int num_lanes) 361 { 362 u32 setting = 0; 363 int lane; 364 365 for (lane = 0; lane < num_lanes; lane++, settings++) { 366 setting |= TX_HS_PRESHOOT_BITS(lane, settings->preshoot); 367 setting |= TX_HS_DEEMPHASIS_BITS(lane, settings->deemphasis); 368 } 369 370 return setting; 371 } 372 373 /** 374 * ufshcd_apply_tx_eq_settings - Apply TX Equalization settings for target gear 375 * @hba: per adapter instance 376 * @params: TX EQ parameters data structure 377 * @gear: target gear 378 * 379 * Returns 0 on success, negative error code otherwise 380 */ 381 int ufshcd_apply_tx_eq_settings(struct ufs_hba *hba, 382 struct ufshcd_tx_eq_params *params, u32 gear) 383 { 384 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; 385 u32 setting; 386 int ret; 387 388 /* Compose settings for Host's TX Lanes */ 389 setting = ufshcd_compose_tx_eq_setting(params->host, pwr_info->lane_tx); 390 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(pa_tx_eq_setting[gear - 1]), setting); 391 if (ret) 392 return ret; 393 394 /* Compose settings for Device's TX Lanes */ 395 setting = ufshcd_compose_tx_eq_setting(params->device, pwr_info->lane_rx); 396 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(pa_tx_eq_setting[gear - 1]), setting); 397 if (ret) 398 return ret; 399 400 /* Configure Pre-Coding */ 401 if (gear >= UFS_HS_G6) { 402 ret = ufshcd_configure_precoding(hba, params); 403 if (ret) { 404 dev_err(hba->dev, "Failed to configure pre-coding: %d\n", ret); 405 return ret; 406 } 407 } 408 409 return 0; 410 } 411 EXPORT_SYMBOL_GPL(ufshcd_apply_tx_eq_settings); 412 413 /** 414 * ufshcd_evaluate_tx_eqtr_fom - Evaluate TX EQTR FOM results 415 * @hba: per adapter instance 416 * @pwr_mode: target power mode containing gear and rate information 417 * @eqtr_data: TX EQTR data structure 418 * @h_iter: host TX EQTR iterator data structure 419 * @d_iter: device TX EQTR iterator data structure 420 * 421 * Evaluate TX EQTR FOM results, update host and device TX EQTR data accordingy 422 * if FOM have been improved compared to previous iteration, and record TX EQTR 423 * FOM results. 424 */ 425 static void ufshcd_evaluate_tx_eqtr_fom(struct ufs_hba *hba, 426 struct ufs_pa_layer_attr *pwr_mode, 427 struct ufshcd_tx_eqtr_data *eqtr_data, 428 struct tx_eqtr_iter *h_iter, 429 struct tx_eqtr_iter *d_iter) 430 { 431 u8 preshoot, deemphasis, fom_value; 432 bool precode_en; 433 int lane; 434 435 for (lane = 0; h_iter->is_updated && lane < pwr_mode->lane_tx; lane++) { 436 preshoot = h_iter->preshoot; 437 deemphasis = h_iter->deemphasis; 438 fom_value = h_iter->fom[lane] & RX_FOM_VALUE_MASK; 439 precode_en = h_iter->fom[lane] & RX_FOM_PRECODING_EN_BIT; 440 441 /* Record host TX EQTR FOM */ 442 eqtr_data->host_fom[lane][preshoot][deemphasis] = h_iter->fom[lane]; 443 444 /* Check if FOM has been improved for host's TX Lanes */ 445 if (fom_value > eqtr_data->host[lane].fom_val) { 446 eqtr_data->host[lane].preshoot = preshoot; 447 eqtr_data->host[lane].deemphasis = deemphasis; 448 eqtr_data->host[lane].fom_val = fom_value; 449 eqtr_data->host[lane].precode_en = precode_en; 450 } 451 452 dev_dbg(hba->dev, "TX EQTR: Host TX Lane %d: PreShoot %u, DeEmphasis %u, FOM value %u, PreCodeEn %d\n", 453 lane, preshoot, deemphasis, fom_value, precode_en); 454 } 455 456 for (lane = 0; d_iter->is_updated && lane < pwr_mode->lane_rx; lane++) { 457 preshoot = d_iter->preshoot; 458 deemphasis = d_iter->deemphasis; 459 fom_value = d_iter->fom[lane] & RX_FOM_VALUE_MASK; 460 precode_en = d_iter->fom[lane] & RX_FOM_PRECODING_EN_BIT; 461 462 /* Record device TX EQTR FOM */ 463 eqtr_data->device_fom[lane][preshoot][deemphasis] = d_iter->fom[lane]; 464 465 /* Check if FOM has been improved for Device's TX Lanes */ 466 if (fom_value > eqtr_data->device[lane].fom_val) { 467 eqtr_data->device[lane].preshoot = preshoot; 468 eqtr_data->device[lane].deemphasis = deemphasis; 469 eqtr_data->device[lane].fom_val = fom_value; 470 eqtr_data->device[lane].precode_en = precode_en; 471 } 472 473 dev_dbg(hba->dev, "TX EQTR: Device TX Lane %d: PreShoot %u, DeEmphasis %u, FOM value %u, PreCodeEn %d\n", 474 lane, preshoot, deemphasis, fom_value, precode_en); 475 } 476 } 477 478 /** 479 * ufshcd_get_rx_fom - Get Figure of Merit (FOM) for both sides 480 * @hba: per adapter instance 481 * @pwr_mode: target power mode containing gear and rate information 482 * @h_iter: host TX EQTR iterator data structure 483 * @d_iter: device TX EQTR iterator data structure 484 * 485 * Returns 0 on success, negative error code if get_rx_fom vops fails. 486 * RX_FOM DME get failures are logged and treated as 0 FOM for that lane. 487 */ 488 static int ufshcd_get_rx_fom(struct ufs_hba *hba, 489 struct ufs_pa_layer_attr *pwr_mode, 490 struct tx_eqtr_iter *h_iter, 491 struct tx_eqtr_iter *d_iter) 492 { 493 int lane, ret; 494 u32 fom; 495 496 /* Get FOM of host's TX lanes from device's RX_FOM. */ 497 for (lane = 0; lane < pwr_mode->lane_tx; lane++) { 498 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB_SEL(RX_FOM, 499 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)), 500 &fom); 501 if (ret) { 502 h_iter->fom[lane] = 0; 503 dev_dbg(hba->dev, "Failed to get FOM for Host TX Lane %d: %d\n", 504 lane, ret); 505 continue; 506 } 507 508 h_iter->fom[lane] = (u8)fom; 509 } 510 511 /* Get FOM of device's TX lanes from host's RX_FOM. */ 512 for (lane = 0; lane < pwr_mode->lane_rx; lane++) { 513 ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(RX_FOM, 514 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)), 515 &fom); 516 if (ret) { 517 d_iter->fom[lane] = 0; 518 dev_dbg(hba->dev, "Failed to get FOM for Device TX Lane %d: %d\n", 519 lane, ret); 520 continue; 521 } 522 523 d_iter->fom[lane] = (u8)fom; 524 } 525 526 ret = ufshcd_vops_get_rx_fom(hba, pwr_mode, h_iter, d_iter); 527 if (ret) 528 dev_err(hba->dev, "Failed to get FOM via vops: %d\n", ret); 529 530 return ret; 531 } 532 533 bool ufshcd_is_txeq_presets_used(struct ufs_hba *hba) 534 { 535 return use_txeq_presets; 536 } 537 538 bool ufshcd_is_txeq_preset_selected(u8 preshoot, u8 deemphasis) 539 { 540 int i; 541 542 for (i = 0; i < UFS_TX_EQ_PRESET_MAX; i++) { 543 if (!txeq_presets_selected[i]) 544 continue; 545 546 if (preshoot == ufs_tx_eq_preset[i].preshoot && 547 deemphasis == ufs_tx_eq_preset[i].deemphasis) 548 return true; 549 } 550 551 return false; 552 } 553 554 /** 555 * tx_eqtr_iter_try_update - Try to update a TX EQTR iterator 556 * @iter: TX EQTR iterator data structure 557 * @preshoot: PreShoot value 558 * @deemphasis: DeEmphasis value 559 * 560 * This function validates whether the provided PreShoot and DeEmphasis 561 * combination can be used or not. If yes, it updates the TX EQTR iterator with 562 * the provided PreShoot and DeEmphasis, it also sets the is_updated flag 563 * to indicate the iterator has been updated. 564 */ 565 static void tx_eqtr_iter_try_update(struct tx_eqtr_iter *iter, 566 u8 preshoot, u8 deemphasis) 567 { 568 if (!test_bit(preshoot, &iter->preshoot_bitmap) || 569 !test_bit(deemphasis, &iter->deemphasis_bitmap) || 570 (use_txeq_presets && !ufshcd_is_txeq_preset_selected(preshoot, deemphasis))) { 571 iter->is_updated = false; 572 return; 573 } 574 575 iter->preshoot = preshoot; 576 iter->deemphasis = deemphasis; 577 iter->is_updated = true; 578 } 579 580 /** 581 * tx_eqtr_iter_update() - Update host and deviceTX EQTR iterators 582 * @preshoot: PreShoot value 583 * @deemphasis: DeEmphasis value 584 * @h_iter: Host TX EQTR iterator data structure 585 * @d_iter: Device TX EQTR iterator data structure 586 * 587 * Updates host and device TX Equalization training iterators with the 588 * provided PreShoot and DeEmphasis. 589 * 590 * Return: true if host and/or device TX Equalization training iterator has 591 * been updated to the provided PreShoot and DeEmphasis, false otherwise. 592 */ 593 static bool tx_eqtr_iter_update(u8 preshoot, u8 deemphasis, 594 struct tx_eqtr_iter *h_iter, 595 struct tx_eqtr_iter *d_iter) 596 { 597 tx_eqtr_iter_try_update(h_iter, preshoot, deemphasis); 598 tx_eqtr_iter_try_update(d_iter, preshoot, deemphasis); 599 600 return h_iter->is_updated || d_iter->is_updated; 601 } 602 603 /** 604 * ufshcd_tx_eqtr_iter_init - Initialize host and device TX EQTR iterators 605 * @hba: per adapter instance 606 * @h_iter: host TX EQTR iterator data structure 607 * @d_iter: device TX EQTR iterator data structure 608 * 609 * This function initializes the TX EQTR iterator structures for both host and 610 * device by reading their TX equalization capabilities. The capabilities are 611 * cached in the hba structure to avoid redundant DME operations in subsequent 612 * calls. In the TX EQTR procedure, the iterator structures are updated by 613 * tx_eqtr_iter_update() to systematically iterate through supported TX 614 * Equalization setting combinations. 615 * 616 * Returns 0 on success, negative error code otherwise 617 */ 618 static int ufshcd_tx_eqtr_iter_init(struct ufs_hba *hba, 619 struct tx_eqtr_iter *h_iter, 620 struct tx_eqtr_iter *d_iter) 621 { 622 u32 cap; 623 int ret; 624 625 if (!hba->host_preshoot_cap) { 626 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(TX_HS_PRESHOOT_SETTING_CAP), &cap); 627 if (ret) 628 return ret; 629 630 hba->host_preshoot_cap = cap & TX_EQTR_CAP_MASK; 631 } 632 633 if (!hba->host_deemphasis_cap) { 634 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(TX_HS_DEEMPHASIS_SETTING_CAP), &cap); 635 if (ret) 636 return ret; 637 638 hba->host_deemphasis_cap = cap & TX_EQTR_CAP_MASK; 639 } 640 641 if (!hba->device_preshoot_cap) { 642 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(TX_HS_PRESHOOT_SETTING_CAP), &cap); 643 if (ret) 644 return ret; 645 646 hba->device_preshoot_cap = cap & TX_EQTR_CAP_MASK; 647 } 648 649 if (!hba->device_deemphasis_cap) { 650 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(TX_HS_DEEMPHASIS_SETTING_CAP), &cap); 651 if (ret) 652 return ret; 653 654 hba->device_deemphasis_cap = cap & TX_EQTR_CAP_MASK; 655 } 656 657 /* 658 * Support PreShoot & DeEmphasis of value 0 is mandatory, hence they are 659 * not reflected in PreShoot/DeEmphasis capabilities. Left shift the 660 * capability bitmap by 1 and set bit[0] to reflect value 0 is 661 * supported, such that test_bit() can be used later for convenience. 662 */ 663 h_iter->preshoot_bitmap = (hba->host_preshoot_cap << 0x1) | 0x1; 664 h_iter->deemphasis_bitmap = (hba->host_deemphasis_cap << 0x1) | 0x1; 665 d_iter->preshoot_bitmap = (hba->device_preshoot_cap << 0x1) | 0x1; 666 d_iter->deemphasis_bitmap = (hba->device_deemphasis_cap << 0x1) | 0x1; 667 668 return 0; 669 } 670 671 /** 672 * adapt_cap_to_t_adapt - Calculate TAdapt from adapt capability 673 * @adapt_cap: Adapt capability 674 * 675 * For NRZ: 676 * IF (ADAPT_range = FINE) 677 * TADAPT = 650 x (ADAPT_length + 1) 678 * ELSE (IF ADAPT_range = COARSE) 679 * TADAPT = 650 x 2^ADAPT_length 680 * 681 * Returns calculated TAdapt value in term of Unit Intervals (UI) 682 */ 683 static inline u64 adapt_cap_to_t_adapt(u32 adapt_cap) 684 { 685 u64 tadapt; 686 u8 adapt_length = adapt_cap & ADAPT_LENGTH_MASK; 687 688 if (!IS_ADAPT_RANGE_COARSE(adapt_cap)) 689 tadapt = TADAPT_FACTOR * (adapt_length + 1); 690 else 691 tadapt = TADAPT_FACTOR * (1 << adapt_length); 692 693 return tadapt; 694 } 695 696 /** 697 * adapt_cap_to_t_adapt_l0l3 - Calculate TAdapt_L0_L3 from adapt capability 698 * @adapt_cap: Adapt capability 699 * 700 * For PAM-4: 701 * IF (ADAPT_range = FINE) 702 * TADAPT_L0_L3 = 2^9 x ADAPT_length 703 * ELSE IF (ADAPT_range = COARSE) 704 * TADAPT_L0_L3 = 2^9 x (2^ADAPT_length) 705 * 706 * Returns calculated TAdapt value in term of Unit Intervals (UI) 707 */ 708 static inline u64 adapt_cap_to_t_adapt_l0l3(u32 adapt_cap) 709 { 710 u64 tadapt; 711 u8 adapt_length = adapt_cap & ADAPT_LENGTH_MASK; 712 713 if (!IS_ADAPT_RANGE_COARSE(adapt_cap)) 714 tadapt = TADAPT_L0L3_FACTOR * adapt_length; 715 else 716 tadapt = TADAPT_L0L3_FACTOR * (1 << adapt_length); 717 718 return tadapt; 719 } 720 721 /** 722 * adapt_cap_to_t_adapt_l0l1l2l3 - Calculate TAdapt_L0_L1_L2_L3 from adapt capability 723 * @adapt_cap: Adapt capability 724 * 725 * For PAM-4: 726 * IF (ADAPT_range_L0_L1_L2_L3 = FINE) 727 * TADAPT_L0_L1_L2_L3 = 2^15 x (ADAPT_length_L0_L1_L2_L3 + 1) 728 * ELSE IF (ADAPT_range_L0_L1_L2_L3 = COARSE) 729 * TADAPT_L0_L1_L2_L3 = 2^15 x 2^ADAPT_length_L0_L1_L2_L3 730 * 731 * Returns calculated TAdapt value in term of Unit Intervals (UI) 732 */ 733 static inline u64 adapt_cap_to_t_adapt_l0l1l2l3(u32 adapt_cap) 734 { 735 u64 tadapt; 736 u8 adapt_length = adapt_cap & ADAPT_LENGTH_MASK; 737 738 if (!IS_ADAPT_RANGE_COARSE(adapt_cap)) 739 tadapt = TADAPT_L0L1L2L3_FACTOR * (adapt_length + 1); 740 else 741 tadapt = TADAPT_L0L1L2L3_FACTOR * (1 << adapt_length); 742 743 return tadapt; 744 } 745 746 /** 747 * ufshcd_setup_tx_eqtr_adapt_length - Setup TX adapt length for EQTR 748 * @hba: per adapter instance 749 * @params: TX EQ parameters data structure 750 * @gear: target gear for EQTR 751 * 752 * This function determines and configures the proper TX adapt length (TAdapt) 753 * for the TX EQTR procedure based on the target gear and RX adapt capabilities 754 * of both host and device. 755 * 756 * Guidelines from MIPI UniPro v3.0 spec - select the minimum Adapt Length for 757 * the Equalization Training procedure based on the following conditions: 758 * 759 * If the target High-Speed Gear n is HS-G4 or HS-G5: 760 * PA_TxAdaptLength_EQTR[7:0] >= Max (10us, RX_HS_Gn_ADAPT_INITIAL_Capability, 761 * PA_PeerRxHsGnAdaptInitial) 762 * PA_TxAdaptLength_EQTR[7:0] shall be shorter than PACP_REQUEST_TIMER (10ms) 763 * PA_TxAdaptLength_EQTR[15:8] is not relevant for HS-G4 and HS-G5. This field 764 * is set to 255 (reserved value). 765 * 766 * If the target High-Speed Gear n is HS-G6: 767 * PA_TxAdapthLength_EQTR >= 10us 768 * PA_TxAdapthLength_EQTR[7:0] >= Max (RX_HS_G6_ADAPT_INITIAL_Capability, 769 * PA_PeerRxHsG6AdaptInitialL0L3) 770 * PA_TxAdapthLength_EQTR[15:8] >= Max (RX_HS_G6_ADAPT_INITIAL_L0_L1_L2_L3_Capability, 771 * PA_PeerRxHsG6AdaptInitialL0L1L2L3) 772 * PA_TxAdaptLength_EQTR shall be shorter than PACP_REQUEST_TIMER value of 10ms. 773 * 774 * Since adapt capabilities encode both range (fine/coarse) and length values, 775 * direct comparison is not possible. This function converts adapt capabilities 776 * to actual time durations in Unit Intervals (UI) using the Adapt time 777 * calculation formular in M-PHY v6.0 spec (Table 8), then selects the maximum 778 * to ensure both host and device use adequate TX adapt length. 779 * 780 * Returns 0 on success, negative error code otherwise 781 */ 782 static int ufshcd_setup_tx_eqtr_adapt_length(struct ufs_hba *hba, 783 struct ufshcd_tx_eq_params *params, 784 u32 gear) 785 { 786 struct ufshcd_tx_eqtr_record *rec = params->eqtr_record; 787 u32 adapt_eqtr; 788 int ret; 789 790 if (rec && rec->saved_adapt_eqtr) { 791 adapt_eqtr = rec->saved_adapt_eqtr; 792 goto set_adapt_eqtr; 793 } 794 795 if (gear == UFS_HS_G4 || gear == UFS_HS_G5) { 796 u64 t_adapt, t_adapt_local, t_adapt_peer; 797 u32 adapt_cap_local, adapt_cap_peer, adapt_length; 798 799 ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(rx_adapt_initial_cap[gear - 1], 800 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)), 801 &adapt_cap_local); 802 if (ret) 803 return ret; 804 805 if (adapt_cap_local > ADAPT_LENGTH_MAX) { 806 dev_err(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP (0x%x) exceeds MAX\n", 807 gear, adapt_cap_local); 808 return -EINVAL; 809 } 810 811 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(pa_peer_rx_adapt_initial[gear - 1]), 812 &adapt_cap_peer); 813 if (ret) 814 return ret; 815 816 if (adapt_cap_peer > ADAPT_LENGTH_MAX) { 817 dev_err(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP (0x%x) exceeds MAX\n", 818 gear, adapt_cap_peer); 819 return -EINVAL; 820 } 821 822 t_adapt_local = adapt_cap_to_t_adapt(adapt_cap_local); 823 t_adapt_peer = adapt_cap_to_t_adapt(adapt_cap_peer); 824 t_adapt = max(t_adapt_local, t_adapt_peer); 825 826 dev_dbg(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP = 0x%x\n", 827 gear, adapt_cap_local); 828 dev_dbg(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_CAP = 0x%x\n", 829 gear, adapt_cap_peer); 830 dev_dbg(hba->dev, "t_adapt_local = %llu UI, t_adapt_peer = %llu UI\n", 831 t_adapt_local, t_adapt_peer); 832 dev_dbg(hba->dev, "TAdapt %llu UI selected for TX EQTR\n", 833 t_adapt); 834 835 adapt_length = (t_adapt_local >= t_adapt_peer) ? 836 adapt_cap_local : adapt_cap_peer; 837 838 if (gear == UFS_HS_G4 && t_adapt < TX_EQTR_HS_G4_MIN_T_ADAPT) { 839 dev_dbg(hba->dev, "TAdapt %llu UI is too short for TX EQTR for HS-G%u, use default Adapt 0x%x\n", 840 t_adapt, gear, TX_EQTR_HS_G4_ADAPT_DEFAULT); 841 adapt_length = TX_EQTR_HS_G4_ADAPT_DEFAULT; 842 } else if (gear == UFS_HS_G5 && t_adapt < TX_EQTR_HS_G5_MIN_T_ADAPT) { 843 dev_dbg(hba->dev, "TAdapt %llu UI is too short for TX EQTR for HS-G%u, use default Adapt 0x%x\n", 844 t_adapt, gear, TX_EQTR_HS_G5_ADAPT_DEFAULT); 845 adapt_length = TX_EQTR_HS_G5_ADAPT_DEFAULT; 846 } 847 848 adapt_eqtr = adapt_length | 849 (TX_EQTR_ADAPT_RESERVED << TX_EQTR_ADAPT_LENGTH_L0L1L2L3_SHIFT); 850 } else if (gear == UFS_HS_G6) { 851 u64 t_adapt, t_adapt_l0l3, t_adapt_l0l3_local, t_adapt_l0l3_peer; 852 u64 t_adapt_l0l1l2l3, t_adapt_l0l1l2l3_local, t_adapt_l0l1l2l3_peer; 853 u32 adapt_l0l3_cap_local, adapt_l0l3_cap_peer, adapt_length_l0l3; 854 u32 adapt_l0l1l2l3_cap_local, adapt_l0l1l2l3_cap_peer, adapt_length_l0l1l2l3; 855 856 ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(rx_adapt_initial_cap[gear - 1], 857 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)), 858 &adapt_l0l3_cap_local); 859 if (ret) 860 return ret; 861 862 if (adapt_l0l3_cap_local > ADAPT_L0L3_LENGTH_MAX) { 863 dev_err(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP (0x%x) exceeds MAX\n", 864 gear, adapt_l0l3_cap_local); 865 return -EINVAL; 866 } 867 868 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(pa_peer_rx_adapt_initial[gear - 1]), 869 &adapt_l0l3_cap_peer); 870 if (ret) 871 return ret; 872 873 if (adapt_l0l3_cap_peer > ADAPT_L0L3_LENGTH_MAX) { 874 dev_err(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_CAP (0x%x) exceeds MAX\n", 875 gear, adapt_l0l3_cap_peer); 876 return -EINVAL; 877 } 878 879 t_adapt_l0l3_local = adapt_cap_to_t_adapt_l0l3(adapt_l0l3_cap_local); 880 t_adapt_l0l3_peer = adapt_cap_to_t_adapt_l0l3(adapt_l0l3_cap_peer); 881 882 dev_dbg(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP = 0x%x\n", 883 gear, adapt_l0l3_cap_local); 884 dev_dbg(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_CAP = 0x%x\n", 885 gear, adapt_l0l3_cap_peer); 886 dev_dbg(hba->dev, "t_adapt_l0l3_local = %llu UI, t_adapt_l0l3_peer = %llu UI\n", 887 t_adapt_l0l3_local, t_adapt_l0l3_peer); 888 889 ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(RX_HS_G6_ADAPT_INITIAL_L0L1L2L3_CAP, 890 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)), 891 &adapt_l0l1l2l3_cap_local); 892 if (ret) 893 return ret; 894 895 if (adapt_l0l1l2l3_cap_local > ADAPT_L0L1L2L3_LENGTH_MAX) { 896 dev_err(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_L0L1L2L3_CAP (0x%x) exceeds MAX\n", 897 gear, adapt_l0l1l2l3_cap_local); 898 899 if (!(hba->quirks & UFSHCD_QUIRK_EXTENDED_TX_EQTR_ADAPT_LENGTH_L0L1L2L3)) 900 return -EINVAL; 901 } 902 903 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTINITIALL0L1L2L3), 904 &adapt_l0l1l2l3_cap_peer); 905 if (ret) 906 return ret; 907 908 if (adapt_l0l1l2l3_cap_peer > ADAPT_L0L1L2L3_LENGTH_MAX) { 909 dev_err(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_L0L1L2L3_CAP (0x%x) exceeds MAX\n", 910 gear, adapt_l0l1l2l3_cap_peer); 911 912 if (!(hba->quirks & UFSHCD_QUIRK_EXTENDED_TX_EQTR_ADAPT_LENGTH_L0L1L2L3)) 913 return -EINVAL; 914 } 915 916 t_adapt_l0l1l2l3_local = adapt_cap_to_t_adapt_l0l1l2l3(adapt_l0l1l2l3_cap_local); 917 t_adapt_l0l1l2l3_peer = adapt_cap_to_t_adapt_l0l1l2l3(adapt_l0l1l2l3_cap_peer); 918 919 dev_dbg(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_L0L1L2L3_CAP = 0x%x\n", 920 gear, adapt_l0l1l2l3_cap_local); 921 dev_dbg(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_L0L1L2L3_CAP = 0x%x\n", 922 gear, adapt_l0l1l2l3_cap_peer); 923 dev_dbg(hba->dev, "t_adapt_l0l1l2l3_local = %llu UI, t_adapt_l0l1l2l3_peer = %llu UI\n", 924 t_adapt_l0l1l2l3_local, t_adapt_l0l1l2l3_peer); 925 926 t_adapt_l0l1l2l3 = max(t_adapt_l0l1l2l3_local, t_adapt_l0l1l2l3_peer); 927 t_adapt_l0l3 = max(t_adapt_l0l3_local, t_adapt_l0l3_peer); 928 t_adapt = t_adapt_l0l3 + t_adapt_l0l1l2l3; 929 930 dev_dbg(hba->dev, "TAdapt %llu PAM-4 UI selected for TX EQTR\n", 931 t_adapt); 932 933 adapt_length_l0l3 = (t_adapt_l0l3_local >= t_adapt_l0l3_peer) ? 934 adapt_l0l3_cap_local : adapt_l0l3_cap_peer; 935 adapt_length_l0l1l2l3 = (t_adapt_l0l1l2l3_local >= t_adapt_l0l1l2l3_peer) ? 936 adapt_l0l1l2l3_cap_local : adapt_l0l1l2l3_cap_peer; 937 938 if (t_adapt < TX_EQTR_HS_G6_MIN_T_ADAPT) { 939 dev_dbg(hba->dev, "TAdapt %llu UI is too short for TX EQTR for HS-G%u, use default Adapt 0x%x\n", 940 t_adapt, gear, TX_EQTR_HS_G6_ADAPT_DEFAULT); 941 adapt_length_l0l3 = TX_EQTR_HS_G6_ADAPT_DEFAULT; 942 } 943 944 adapt_eqtr = adapt_length_l0l3 | 945 (adapt_length_l0l1l2l3 << TX_EQTR_ADAPT_LENGTH_L0L1L2L3_SHIFT); 946 } else { 947 return -EINVAL; 948 } 949 950 if (rec) 951 rec->saved_adapt_eqtr = (u16)adapt_eqtr; 952 953 set_adapt_eqtr: 954 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXADAPTLENGTH_EQTR), adapt_eqtr); 955 if (ret) 956 dev_err(hba->dev, "Failed to set adapt length for TX EQTR: %d\n", ret); 957 else 958 dev_dbg(hba->dev, "PA_TXADAPTLENGTH_EQTR configured to 0x%08x\n", adapt_eqtr); 959 960 return ret; 961 } 962 963 /** 964 * ufshcd_compose_tx_eqtr_setting - Compose TX EQTR setting 965 * @iter: TX EQTR iterator data structure 966 * @num_lanes: number of active lanes 967 * 968 * Returns composed TX EQTR setting, same setting is used for all active lanes 969 */ 970 static inline u32 ufshcd_compose_tx_eqtr_setting(struct tx_eqtr_iter *iter, 971 int num_lanes) 972 { 973 u32 setting = 0; 974 int lane; 975 976 for (lane = 0; lane < num_lanes; lane++) { 977 setting |= TX_HS_PRESHOOT_BITS(lane, iter->preshoot); 978 setting |= TX_HS_DEEMPHASIS_BITS(lane, iter->deemphasis); 979 } 980 981 return setting; 982 } 983 984 /** 985 * ufshcd_apply_tx_eqtr_settings - Apply TX EQTR setting 986 * @hba: per adapter instance 987 * @pwr_mode: target power mode containing gear and rate information 988 * @h_iter: host TX EQTR iterator data structure 989 * @d_iter: device TX EQTR iterator data structure 990 * 991 * Returns 0 on success, negative error code otherwise 992 */ 993 static int ufshcd_apply_tx_eqtr_settings(struct ufs_hba *hba, 994 struct ufs_pa_layer_attr *pwr_mode, 995 struct tx_eqtr_iter *h_iter, 996 struct tx_eqtr_iter *d_iter) 997 { 998 u32 setting; 999 int ret; 1000 1001 setting = ufshcd_compose_tx_eqtr_setting(h_iter, pwr_mode->lane_tx); 1002 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXEQTRSETTING), setting); 1003 if (ret) 1004 return ret; 1005 1006 setting = ufshcd_compose_tx_eqtr_setting(d_iter, pwr_mode->lane_rx); 1007 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PEERTXEQTRSETTING), setting); 1008 if (ret) 1009 return ret; 1010 1011 ret = ufshcd_vops_apply_tx_eqtr_settings(hba, pwr_mode, h_iter, d_iter); 1012 1013 return ret; 1014 } 1015 1016 /** 1017 * ufshcd_update_tx_eq_params - Update TX Equalization params 1018 * @params: TX EQ parameters data structure 1019 * @pwr_mode: target power mode containing gear and rate 1020 * @eqtr_data: TX EQTR data structure 1021 * 1022 * Update TX Equalization params using results from TX EQTR data. Check also 1023 * the TX EQTR FOM value for each TX lane in the TX EQTR data. If a TX lane got 1024 * a FOM value of 0, restore the TX Equalization settings from the last known 1025 * valid TX Equalization params for that specific TX lane. 1026 */ 1027 static inline void 1028 ufshcd_update_tx_eq_params(struct ufshcd_tx_eq_params *params, 1029 struct ufs_pa_layer_attr *pwr_mode, 1030 struct ufshcd_tx_eqtr_data *eqtr_data) 1031 { 1032 struct ufshcd_tx_eqtr_record *rec = params->eqtr_record; 1033 1034 if (params->is_valid) { 1035 int lane; 1036 1037 for (lane = 0; lane < pwr_mode->lane_tx; lane++) 1038 if (eqtr_data->host[lane].fom_val == 0) 1039 eqtr_data->host[lane] = params->host[lane]; 1040 1041 for (lane = 0; lane < pwr_mode->lane_rx; lane++) 1042 if (eqtr_data->device[lane].fom_val == 0) 1043 eqtr_data->device[lane] = params->device[lane]; 1044 } 1045 1046 memcpy(params->host, eqtr_data->host, sizeof(params->host)); 1047 memcpy(params->device, eqtr_data->device, sizeof(params->device)); 1048 1049 if (!rec) 1050 return; 1051 1052 memcpy(rec->host_fom, eqtr_data->host_fom, sizeof(rec->host_fom)); 1053 memcpy(rec->device_fom, eqtr_data->device_fom, sizeof(rec->device_fom)); 1054 rec->last_record_ts = ktime_get(); 1055 rec->last_record_index++; 1056 } 1057 1058 /** 1059 * __ufshcd_tx_eqtr - TX Equalization Training (EQTR) procedure 1060 * @hba: per adapter instance 1061 * @params: TX EQ parameters data structure 1062 * @pwr_mode: target power mode containing gear and rate information 1063 * 1064 * This function implements the complete TX EQTR procedure as defined in UFSHCI 1065 * v5.0 specification. It iterates through all possible combinations of PreShoot 1066 * and DeEmphasis settings to find the optimal TX Equalization settings for all 1067 * active lanes. 1068 * 1069 * Returns 0 on success, negative error code otherwise 1070 */ 1071 static int __ufshcd_tx_eqtr(struct ufs_hba *hba, 1072 struct ufshcd_tx_eq_params *params, 1073 struct ufs_pa_layer_attr *pwr_mode) 1074 { 1075 struct ufshcd_tx_eqtr_data *eqtr_data __free(kfree) = 1076 kzalloc_obj(*eqtr_data); 1077 struct tx_eqtr_iter h_iter = {}; 1078 struct tx_eqtr_iter d_iter = {}; 1079 u32 gear = pwr_mode->gear_tx; 1080 u8 preshoot, deemphasis; 1081 ktime_t start; 1082 int ret; 1083 1084 if (!eqtr_data) 1085 return -ENOMEM; 1086 1087 dev_info(hba->dev, "Start TX EQTR procedure for HS-G%u, Rate-%s, RX Lanes: %u, TX Lanes: %u\n", 1088 gear, ufs_hs_rate_to_str(pwr_mode->hs_rate), 1089 pwr_mode->lane_rx, pwr_mode->lane_tx); 1090 1091 start = ktime_get(); 1092 1093 /* Step 1 - Determine the TX Adapt Length for EQTR */ 1094 ret = ufshcd_setup_tx_eqtr_adapt_length(hba, params, gear); 1095 if (ret) { 1096 dev_err(hba->dev, "Failed to setup TX EQTR Adaptation length: %d\n", ret); 1097 return ret; 1098 } 1099 1100 /* Step 2 - Determine TX Equalization setting capabilities */ 1101 ret = ufshcd_tx_eqtr_iter_init(hba, &h_iter, &d_iter); 1102 if (ret) { 1103 dev_err(hba->dev, "Failed to init TX EQTR data: %d\n", ret); 1104 return ret; 1105 } 1106 1107 /* TX EQTR main loop */ 1108 for (preshoot = 0; preshoot < TX_HS_NUM_PRESHOOT; preshoot++) { 1109 for (deemphasis = 0; deemphasis < TX_HS_NUM_DEEMPHASIS; deemphasis++) { 1110 if (!tx_eqtr_iter_update(preshoot, deemphasis, &h_iter, &d_iter)) 1111 continue; 1112 1113 /* Step 3 - Apply TX EQTR settings */ 1114 ret = ufshcd_apply_tx_eqtr_settings(hba, pwr_mode, &h_iter, &d_iter); 1115 if (ret) { 1116 dev_err(hba->dev, "Failed to apply TX EQTR settings (PreShoot %u, DeEmphasis %u): %d\n", 1117 preshoot, deemphasis, ret); 1118 return ret; 1119 } 1120 1121 /* Step 4 - Trigger UIC TX EQTR */ 1122 ret = ufshcd_uic_tx_eqtr(hba, gear); 1123 if (ret) { 1124 dev_err(hba->dev, "Failed to trigger UIC TX EQTR for target gear %u: %d\n", 1125 gear, ret); 1126 return ret; 1127 } 1128 1129 /* Step 5 - Get FOM */ 1130 ret = ufshcd_get_rx_fom(hba, pwr_mode, &h_iter, &d_iter); 1131 if (ret) { 1132 dev_err(hba->dev, "Failed to get RX_FOM: %d\n", 1133 ret); 1134 return ret; 1135 } 1136 1137 ufshcd_evaluate_tx_eqtr_fom(hba, pwr_mode, eqtr_data, &h_iter, &d_iter); 1138 } 1139 } 1140 1141 dev_info(hba->dev, "TX EQTR procedure completed! Time elapsed: %llu ms\n", 1142 ktime_to_ms(ktime_sub(ktime_get(), start))); 1143 1144 ufshcd_update_tx_eq_params(params, pwr_mode, eqtr_data); 1145 1146 return ret; 1147 } 1148 1149 /** 1150 * ufshcd_tx_eqtr_prepare - Prepare UFS link for TX EQTR procedure 1151 * @hba: per adapter instance 1152 * @pwr_mode: target power mode containing gear and rate 1153 * 1154 * This function prepares the UFS link for TX Equalization Training (EQTR) by 1155 * establishing the proper initial conditions required by the EQTR procedure. 1156 * It ensures that EQTR starts from the most reliable Power Mode (HS-G1) with 1157 * all connected lanes activated and sets host TX HS Adapt Type to INITIAL. 1158 * 1159 * Returns 0 on successful preparation, negative error code on failure 1160 */ 1161 static int ufshcd_tx_eqtr_prepare(struct ufs_hba *hba, 1162 struct ufs_pa_layer_attr *pwr_mode) 1163 { 1164 struct ufs_pa_layer_attr pwr_mode_hs_g1 = { 1165 /* TX EQTR shall be initiated from the most reliable HS-G1 */ 1166 .gear_rx = UFS_HS_G1, 1167 .gear_tx = UFS_HS_G1, 1168 .lane_rx = pwr_mode->lane_rx, 1169 .lane_tx = pwr_mode->lane_tx, 1170 .pwr_rx = FAST_MODE, 1171 .pwr_tx = FAST_MODE, 1172 /* Use the target power mode's HS rate */ 1173 .hs_rate = pwr_mode->hs_rate, 1174 }; 1175 u32 rate = pwr_mode->hs_rate; 1176 int ret; 1177 1178 /* Change power mode to HS-G1, activate all connected lanes. */ 1179 ret = ufshcd_change_power_mode(hba, &pwr_mode_hs_g1, 1180 UFSHCD_PMC_POLICY_DONT_FORCE); 1181 if (ret) { 1182 dev_err(hba->dev, "TX EQTR: Failed to change power mode to HS-G1, Rate-%s: %d\n", 1183 ufs_hs_rate_to_str(rate), ret); 1184 return ret; 1185 } 1186 1187 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXHSADAPTTYPE), 1188 PA_INITIAL_ADAPT); 1189 if (ret) 1190 dev_err(hba->dev, "TX EQTR: Failed to set Host Adapt type to INITIAL: %d\n", 1191 ret); 1192 1193 return ret; 1194 } 1195 1196 static void ufshcd_tx_eqtr_unprepare(struct ufs_hba *hba, 1197 struct ufs_pa_layer_attr *pwr_mode) 1198 { 1199 int err; 1200 1201 if (pwr_mode->pwr_rx == SLOWAUTO_MODE || pwr_mode->hs_rate == 0) 1202 return; 1203 1204 err = ufshcd_change_power_mode(hba, pwr_mode, 1205 UFSHCD_PMC_POLICY_DONT_FORCE); 1206 if (err) 1207 dev_err(hba->dev, "%s: Failed to restore Power Mode: %d\n", 1208 __func__, err); 1209 } 1210 1211 /** 1212 * ufshcd_tx_eqtr - Perform TX EQTR procedures with vops callbacks 1213 * @hba: per adapter instance 1214 * @params: TX EQ parameters data structure to populate 1215 * @pwr_mode: target power mode containing gear and rate information 1216 * 1217 * This is the main entry point for performing TX Equalization Training (EQTR) 1218 * procedure as defined in UFSCHI v5.0 specification. It serves as a wrapper 1219 * around __ufshcd_tx_eqtr() to provide vops support through the variant 1220 * operations framework. 1221 * 1222 * Returns 0 on success, negative error code on failure 1223 */ 1224 static int ufshcd_tx_eqtr(struct ufs_hba *hba, 1225 struct ufshcd_tx_eq_params *params, 1226 struct ufs_pa_layer_attr *pwr_mode) 1227 { 1228 struct ufs_pa_layer_attr old_pwr_info; 1229 unsigned int noio_flag; 1230 int notify_ret; 1231 int ret; 1232 1233 /* 1234 * ufshcd_tx_eqtr() is called from a power-mode-change context where 1235 * I/O is suspended. Use memalloc_noio_save() to propagate GFP_NOIO 1236 * to all allocations in the call tree instead of tagging each call 1237 * site individually. 1238 */ 1239 noio_flag = memalloc_noio_save(); 1240 1241 if (!params->eqtr_record) { 1242 params->eqtr_record = devm_kzalloc(hba->dev, 1243 sizeof(*params->eqtr_record), 1244 GFP_KERNEL); 1245 if (!params->eqtr_record) { 1246 ret = -ENOMEM; 1247 goto out_noio_restore; 1248 } 1249 } 1250 1251 memcpy(&old_pwr_info, &hba->pwr_info, sizeof(struct ufs_pa_layer_attr)); 1252 1253 ret = ufshcd_tx_eqtr_prepare(hba, pwr_mode); 1254 if (ret) { 1255 dev_err(hba->dev, "Failed to prepare TX EQTR: %d\n", ret); 1256 goto out_unprepare; 1257 } 1258 1259 ret = ufshcd_vops_tx_eqtr_notify(hba, PRE_CHANGE, pwr_mode); 1260 if (ret) { 1261 dev_err(hba->dev, "TX EQTR PRE_CHANGE notify failed: %d\n", ret); 1262 goto out_unprepare; 1263 } 1264 1265 ret = __ufshcd_tx_eqtr(hba, params, pwr_mode); 1266 1267 notify_ret = ufshcd_vops_tx_eqtr_notify(hba, POST_CHANGE, pwr_mode); 1268 if (notify_ret) 1269 dev_err(hba->dev, "TX EQTR POST_CHANGE notify failed: %d\n", notify_ret); 1270 1271 if (!ret) 1272 ret = notify_ret; 1273 1274 out_unprepare: 1275 if (ret) 1276 ufshcd_tx_eqtr_unprepare(hba, &old_pwr_info); 1277 1278 out_noio_restore: 1279 memalloc_noio_restore(noio_flag); 1280 1281 return ret; 1282 } 1283 1284 /** 1285 * ufshcd_config_tx_eq_settings - Configure TX Equalization settings 1286 * @hba: per adapter instance 1287 * @pwr_mode: target power mode containing gear and rate information 1288 * @force_tx_eqtr: execute the TX EQTR procedure 1289 * 1290 * This function finds and sets the TX Equalization settings for the given 1291 * target power mode. 1292 * 1293 * Returns 0 on success, error code otherwise 1294 */ 1295 int ufshcd_config_tx_eq_settings(struct ufs_hba *hba, 1296 struct ufs_pa_layer_attr *pwr_mode, 1297 bool force_tx_eqtr) 1298 { 1299 struct ufshcd_tx_eq_params *params; 1300 u32 gear, rate; 1301 1302 if (!ufshcd_is_tx_eq_supported(hba) || !use_adaptive_txeq) 1303 return 0; 1304 1305 if (!hba->max_pwr_info.is_valid) { 1306 dev_err(hba->dev, "Max power info is invalid\n"); 1307 return -EINVAL; 1308 } 1309 1310 if (!pwr_mode) { 1311 dev_err(hba->dev, "Target power mode is NULL\n"); 1312 return -EINVAL; 1313 } 1314 1315 gear = pwr_mode->gear_tx; 1316 rate = pwr_mode->hs_rate; 1317 1318 if (gear < UFS_HS_G1 || gear > UFS_HS_GEAR_MAX) { 1319 dev_err(hba->dev, "Invalid HS-Gear (%u) for TX Equalization\n", 1320 gear); 1321 return -EINVAL; 1322 } else if (gear < max_t(u32, adaptive_txeq_gear, UFS_HS_G4)) { 1323 /* TX EQTR is supported for HS-G4 and higher Gears */ 1324 return 0; 1325 } 1326 1327 if (rate != PA_HS_MODE_A && rate != PA_HS_MODE_B) { 1328 dev_err(hba->dev, "Invalid HS-Rate (%u) for TX Equalization\n", 1329 rate); 1330 return -EINVAL; 1331 } 1332 1333 params = &hba->tx_eq_params[gear - 1]; 1334 /* 1335 * TX EQTR must run for the following cases: 1336 * 1. TX EQ settings are invalid. 1337 * 2. TX EQ settings are from Device Tree. 1338 * 3. TX EQTR procedure is forced. 1339 */ 1340 if (!params->is_valid || params->from_dt || force_tx_eqtr) { 1341 int ret; 1342 1343 ret = ufshcd_tx_eqtr(hba, params, pwr_mode); 1344 if (ret) { 1345 dev_err(hba->dev, "Failed to train TX Equalization for HS-G%u, Rate-%s: %d\n", 1346 gear, ufs_hs_rate_to_str(rate), ret); 1347 return ret; 1348 } 1349 1350 /* Mark TX Equalization settings as valid */ 1351 params->is_valid = true; 1352 params->is_trained = true; 1353 /* TX EQTR succeeds, clear from_dt flag */ 1354 params->from_dt = false; 1355 params->is_applied = false; 1356 } 1357 1358 if (params->is_valid && !params->is_applied) { 1359 int ret; 1360 1361 ret = ufshcd_apply_tx_eq_settings(hba, params, gear); 1362 if (ret) { 1363 dev_err(hba->dev, "Failed to apply TX Equalization settings for HS-G%u, Rate-%s: %d\n", 1364 gear, ufs_hs_rate_to_str(rate), ret); 1365 return ret; 1366 } 1367 1368 params->is_applied = true; 1369 } 1370 1371 return 0; 1372 } 1373 1374 /** 1375 * ufshcd_apply_valid_tx_eq_settings - Apply valid TX Equalization settings 1376 * @hba: per-adapter instance 1377 * 1378 * This function iterates through all supported High-Speed (HS) gears and 1379 * applies valid TX Equalization settings to both Host and Device. 1380 */ 1381 void ufshcd_apply_valid_tx_eq_settings(struct ufs_hba *hba) 1382 { 1383 struct ufshcd_tx_eq_params *params; 1384 int gear, err; 1385 1386 if (!ufshcd_is_tx_eq_supported(hba)) 1387 return; 1388 1389 if (!hba->max_pwr_info.is_valid) { 1390 dev_err(hba->dev, "Max power info is invalid, cannot apply TX Equalization settings\n"); 1391 return; 1392 } 1393 1394 for (gear = UFS_HS_G1; gear <= UFS_HS_GEAR_MAX; gear++) { 1395 params = &hba->tx_eq_params[gear - 1]; 1396 1397 if (params->is_valid) { 1398 err = ufshcd_apply_tx_eq_settings(hba, params, gear); 1399 if (err) { 1400 params->is_applied = false; 1401 dev_err(hba->dev, "Failed to apply TX Equalization settings for HS-G%u: %d\n", 1402 gear, err); 1403 } else { 1404 params->is_applied = true; 1405 } 1406 } 1407 } 1408 } 1409 1410 /** 1411 * ufshcd_retrain_tx_eq - Retrain TX Equalization and apply new settings 1412 * @hba: per-adapter instance 1413 * @gear: target High-Speed (HS) gear for retraining 1414 * 1415 * This function initiates a refresh of the TX Equalization settings for a 1416 * specific HS gear. It scales the clocks to maximum frequency, negotiates the 1417 * power mode with the device, retrains TX EQ and applies new TX EQ settings 1418 * by conducting a Power Mode change. 1419 * 1420 * Returns 0 on success, non-zero error code otherwise 1421 */ 1422 int ufshcd_retrain_tx_eq(struct ufs_hba *hba, u32 gear) 1423 { 1424 struct ufs_pa_layer_attr new_pwr_info, final_params = {}; 1425 int ret; 1426 1427 if (!ufshcd_is_tx_eq_supported(hba) || !use_adaptive_txeq) 1428 return -EOPNOTSUPP; 1429 1430 if (gear < adaptive_txeq_gear) 1431 return -ERANGE; 1432 1433 ufshcd_hold(hba); 1434 1435 ret = ufshcd_pause_command_processing(hba, 1 * USEC_PER_SEC); 1436 if (ret) { 1437 ufshcd_release(hba); 1438 return ret; 1439 } 1440 1441 /* scale up clocks to max frequency before TX EQTR */ 1442 if (ufshcd_is_clkscaling_supported(hba)) 1443 ufshcd_scale_clks(hba, ULONG_MAX, true); 1444 1445 new_pwr_info = hba->pwr_info; 1446 new_pwr_info.gear_tx = gear; 1447 new_pwr_info.gear_rx = gear; 1448 1449 ret = ufshcd_vops_negotiate_pwr_mode(hba, &new_pwr_info, &final_params); 1450 if (ret) 1451 memcpy(&final_params, &new_pwr_info, sizeof(final_params)); 1452 1453 if (final_params.gear_tx != gear) { 1454 dev_err(hba->dev, "Negotiated Gear (%u) does not match target Gear (%u)\n", 1455 final_params.gear_tx, gear); 1456 ret = -EINVAL; 1457 goto out; 1458 } 1459 1460 ret = ufshcd_config_tx_eq_settings(hba, &final_params, true); 1461 if (ret) { 1462 dev_err(hba->dev, "Failed to config TX Equalization for HS-G%u, Rate-%s: %d\n", 1463 final_params.gear_tx, 1464 ufs_hs_rate_to_str(final_params.hs_rate), ret); 1465 goto out; 1466 } 1467 1468 /* Change Power Mode to apply the new TX EQ settings */ 1469 ret = ufshcd_change_power_mode(hba, &final_params, 1470 UFSHCD_PMC_POLICY_FORCE); 1471 if (ret) 1472 dev_err(hba->dev, "%s: Failed to change Power Mode to HS-G%u, Rate-%s: %d\n", 1473 __func__, final_params.gear_tx, 1474 ufs_hs_rate_to_str(final_params.hs_rate), ret); 1475 1476 out: 1477 ufshcd_resume_command_processing(hba); 1478 ufshcd_release(hba); 1479 1480 return ret; 1481 } 1482 1483 /** 1484 * ufshcd_extract_tx_eq_settings_attrs - Extract TX Equalization settings from UFS attributes 1485 * @hba: per adapter instance 1486 * @gear: target gear 1487 * 1488 * This function extracts previously stored TX Equalization settings from UFS 1489 * attributes qTxEQGnSettings and wTxEQGnSettingsExt. These attributes contain 1490 * the optimal TX Equalization parameters (PreShoot, DeEmphasis, and PreCoding 1491 * enable) that were determined during a previous EQTR procedure. 1492 * 1493 * The function reads: 1494 * 1. qTxEQGnSettings (64-bit): Main attribute containing PreShoot and 1495 * DeEmphasis values for both host and device TX lanes 1496 * 2. wTxEQGnSettingsExt (16-bit): Extended attribute containing PreCoding 1497 * enable flags and validity indicator 1498 */ 1499 static void ufshcd_extract_tx_eq_settings_attrs(struct ufs_hba *hba, u8 gear) 1500 { 1501 struct ufshcd_tx_eq_params *params; 1502 u32 lane, eq_ext; 1503 int ret; 1504 u64 eq; 1505 1506 ret = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_READ_ATTR, 1507 QUERY_ATTR_IDN_TX_EQ_GN_SETTINGS_EXT, gear - 1, 1508 (u8)txeq_setting_sel, &eq_ext); 1509 if (ret) 1510 return; 1511 1512 dev_dbg(hba->dev, "%s: HS-G%u wTxEQGnSettingsExt (Selector %u) = 0x%08x\n", 1513 __func__, gear, txeq_setting_sel, eq_ext); 1514 1515 if (!(eq_ext & TX_EQ_SETTINGS_VALID_BIT)) 1516 return; 1517 1518 ret = ufshcd_query_attr_qword(hba, UPIU_QUERY_OPCODE_READ_ATTR, 1519 QUERY_ATTR_IDN_TX_EQ_GN_SETTINGS, 1520 gear - 1, (u8)txeq_setting_sel, &eq); 1521 if (ret) 1522 return; 1523 1524 dev_dbg(hba->dev, "%s: HS-G%u qTxEQGnSettings (Selector %u) = 0x%016llx\n", 1525 __func__, gear, txeq_setting_sel, eq); 1526 1527 params = &hba->tx_eq_params[gear - 1]; 1528 1529 for (lane = 0; lane < UFS_MAX_LANES; lane++) { 1530 params->host[lane].preshoot = tx_eq_host_preshoot_decode(eq, lane); 1531 params->host[lane].deemphasis = tx_eq_host_deemphasis_decode(eq, lane); 1532 params->host[lane].precode_en = tx_eq_host_precode_en_decode(eq_ext, lane); 1533 1534 params->device[lane].preshoot = tx_eq_device_preshoot_decode(eq, lane); 1535 params->device[lane].deemphasis = tx_eq_device_deemphasis_decode(eq, lane); 1536 params->device[lane].precode_en = tx_eq_device_precode_en_decode(eq_ext, lane); 1537 } 1538 1539 params->is_valid = true; 1540 /* 1541 * Optimal TX EQ settings are retrieved from UFS device attributes, 1542 * clear from_dt flag to avoid TX EQTR procedure. 1543 */ 1544 params->from_dt = false; 1545 } 1546 1547 void ufshcd_retrieve_tx_eq_settings(struct ufs_hba *hba) 1548 { 1549 u8 gear = (u8)adaptive_txeq_gear; 1550 1551 if (!hba->max_pwr_info.is_valid || !ufshcd_is_tx_eq_supported(hba) || 1552 !use_adaptive_txeq || !retrieve_txeq_setting) 1553 return; 1554 1555 for (; gear <= UFS_HS_GEAR_MAX; gear++) 1556 ufshcd_extract_tx_eq_settings_attrs(hba, gear); 1557 } 1558 1559 /** 1560 * ufshcd_update_tx_eq_settings_attrs - Update TX EQ settings in UFS attributes 1561 * @hba: per adapter instance 1562 * @gear: target gear 1563 * 1564 * This function stores the optimal TX Equalization settings obtained from 1565 * TX EQTR procedure into UFS device attributes for future fast-path retrieval. 1566 * The settings are stored in two complementary attributes: 1567 * 1568 * 1. qTxEQGnSettings (64-bit): Main attribute containing PreShoot and 1569 * DeEmphasis values for both host and device TX lanes 1570 * 2. wTxEQGnSettingsExt (16-bit): Extended attribute containing PreCoding 1571 * enable flags and validity indicator 1572 */ 1573 static void ufshcd_update_tx_eq_settings_attrs(struct ufs_hba *hba, u8 gear) 1574 { 1575 struct ufshcd_tx_eq_params *params; 1576 u32 lane, eq_ext = 0; 1577 u64 eq = 0; 1578 int ret; 1579 1580 params = &hba->tx_eq_params[gear - 1]; 1581 if (!params->is_valid || !params->is_trained) 1582 return; 1583 1584 for (lane = 0; lane < UFS_MAX_LANES; lane++) { 1585 eq |= tx_eq_host_preshoot_encode((u64)params->host[lane].preshoot, lane); 1586 eq |= tx_eq_host_deemphasis_encode((u64)params->host[lane].deemphasis, lane); 1587 eq_ext |= tx_eq_host_precode_en_encode(params->host[lane].precode_en, lane); 1588 1589 eq |= tx_eq_device_preshoot_encode((u64)params->device[lane].preshoot, lane); 1590 eq |= tx_eq_device_deemphasis_encode((u64)params->device[lane].deemphasis, lane); 1591 eq_ext |= tx_eq_device_precode_en_encode(params->device[lane].precode_en, lane); 1592 } 1593 1594 /* Set validity flag to indicate valid settings are stored */ 1595 eq_ext |= TX_EQ_SETTINGS_VALID_BIT; 1596 1597 /* Write qTxEQGnSettings */ 1598 ret = ufshcd_query_attr_qword(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 1599 QUERY_ATTR_IDN_TX_EQ_GN_SETTINGS, 1600 gear - 1, (u8)txeq_setting_sel, &eq); 1601 if (ret) 1602 return; 1603 1604 /* Write wTxEQGnSettingsExt */ 1605 ret = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 1606 QUERY_ATTR_IDN_TX_EQ_GN_SETTINGS_EXT, gear - 1, 1607 (u8)txeq_setting_sel, &eq_ext); 1608 if (ret) 1609 return; 1610 1611 dev_dbg(hba->dev, "%s: Saved HS-G%u qTxEQGnSettings (Selector %u) = 0x%016llx\n", 1612 __func__, gear, txeq_setting_sel, eq); 1613 dev_dbg(hba->dev, "%s: Saved HS-G%u wTxEQGnSettingsExt (Selector %u) = 0x%08x\n", 1614 __func__, gear, txeq_setting_sel, eq_ext); 1615 } 1616 1617 void ufshcd_store_tx_eq_settings(struct ufs_hba *hba) 1618 { 1619 u8 gear = (u8)adaptive_txeq_gear; 1620 1621 if (!hba->max_pwr_info.is_valid || !ufshcd_is_tx_eq_supported(hba) || 1622 !use_adaptive_txeq || !store_txeq_setting) 1623 return; 1624 1625 for (; gear <= UFS_HS_GEAR_MAX; gear++) 1626 ufshcd_update_tx_eq_settings_attrs(hba, gear); 1627 } 1628