1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2026 Qualcomm Technologies, Inc. 4 * 5 * Author: 6 * Can Guo <can.guo@oss.qualcomm.com> 7 */ 8 9 #include <linux/bitops.h> 10 #include <linux/delay.h> 11 #include <linux/errno.h> 12 #include <linux/kernel.h> 13 #include <ufs/ufshcd.h> 14 #include <ufs/unipro.h> 15 #include "ufshcd-priv.h" 16 17 static bool use_adaptive_txeq; 18 module_param(use_adaptive_txeq, bool, 0644); 19 MODULE_PARM_DESC(use_adaptive_txeq, "Find and apply optimal TX Equalization settings before changing Power Mode (default: false)"); 20 21 static int txeq_gear_set(const char *val, const struct kernel_param *kp) 22 { 23 return param_set_uint_minmax(val, kp, UFS_HS_G1, UFS_HS_GEAR_MAX); 24 } 25 26 static const struct kernel_param_ops txeq_gear_ops = { 27 .set = txeq_gear_set, 28 .get = param_get_uint, 29 }; 30 31 static unsigned int adaptive_txeq_gear = UFS_HS_G6; 32 module_param_cb(adaptive_txeq_gear, &txeq_gear_ops, &adaptive_txeq_gear, 0644); 33 MODULE_PARM_DESC(adaptive_txeq_gear, "For HS-Gear[n] and above, adaptive txeq shall be used"); 34 35 static bool use_txeq_presets; 36 module_param(use_txeq_presets, bool, 0644); 37 MODULE_PARM_DESC(use_txeq_presets, "Use only the 8 TX Equalization Presets (pre-defined Pre-Shoot & De-Emphasis combinations) for TX EQTR (default: false)"); 38 39 static bool txeq_presets_selected[UFS_TX_EQ_PRESET_MAX] = {[0 ... (UFS_TX_EQ_PRESET_MAX - 1)] = 1}; 40 module_param_array(txeq_presets_selected, bool, NULL, 0644); 41 MODULE_PARM_DESC(txeq_presets_selected, "Use only the selected Presets out of the 8 TX Equalization Presets for TX EQTR"); 42 43 /* 44 * ufs_tx_eq_preset - Table of minimum required list of presets. 45 * 46 * A HS-G6 capable M-TX shall support the presets defined in M-PHY v6.0 spec. 47 * Preset Pre-Shoot(dB) De-Emphasis(dB) 48 * P0 0.0 0.0 49 * P1 0.0 0.8 50 * P2 0.0 1.6 51 * P3 0.8 0.0 52 * P4 1.6 0.0 53 * P5 0.8 0.8 54 * P6 0.8 1.6 55 * P7 1.6 0.8 56 */ 57 static const struct __ufs_tx_eq_preset { 58 u8 preshoot; 59 u8 deemphasis; 60 } ufs_tx_eq_preset[UFS_TX_EQ_PRESET_MAX] = { 61 [UFS_TX_EQ_PRESET_P0] = {UFS_TX_HS_PRESHOOT_DB_0P0, UFS_TX_HS_DEEMPHASIS_DB_0P0}, 62 [UFS_TX_EQ_PRESET_P1] = {UFS_TX_HS_PRESHOOT_DB_0P0, UFS_TX_HS_DEEMPHASIS_DB_0P8}, 63 [UFS_TX_EQ_PRESET_P2] = {UFS_TX_HS_PRESHOOT_DB_0P0, UFS_TX_HS_DEEMPHASIS_DB_1P6}, 64 [UFS_TX_EQ_PRESET_P3] = {UFS_TX_HS_PRESHOOT_DB_0P8, UFS_TX_HS_DEEMPHASIS_DB_0P0}, 65 [UFS_TX_EQ_PRESET_P4] = {UFS_TX_HS_PRESHOOT_DB_1P6, UFS_TX_HS_DEEMPHASIS_DB_0P0}, 66 [UFS_TX_EQ_PRESET_P5] = {UFS_TX_HS_PRESHOOT_DB_0P8, UFS_TX_HS_DEEMPHASIS_DB_0P8}, 67 [UFS_TX_EQ_PRESET_P6] = {UFS_TX_HS_PRESHOOT_DB_0P8, UFS_TX_HS_DEEMPHASIS_DB_1P6}, 68 [UFS_TX_EQ_PRESET_P7] = {UFS_TX_HS_PRESHOOT_DB_1P6, UFS_TX_HS_DEEMPHASIS_DB_0P8}, 69 }; 70 71 /* 72 * pa_peer_rx_adapt_initial - Table of UniPro PA_PeerRxHSGnAdaptInitial 73 * attribute IDs for High Speed (HS) Gears. 74 * 75 * This table maps HS Gears to their respective UniPro PA_PeerRxHSGnAdaptInitial 76 * attribute IDs. Entries for Gears 1-3 are 0 (unsupported). 77 */ 78 static const u32 pa_peer_rx_adapt_initial[UFS_HS_GEAR_MAX] = { 79 0, 80 0, 81 0, 82 PA_PEERRXHSG4ADAPTINITIAL, 83 PA_PEERRXHSG5ADAPTINITIAL, 84 PA_PEERRXHSG6ADAPTINITIALL0L3 85 }; 86 87 /* 88 * rx_adapt_initial_cap - Table of M-PHY RX_HS_Gn_ADAPT_INITIAL_Capability 89 * attribute IDs for High Speed (HS) Gears. 90 * 91 * This table maps HS Gears to their respective M-PHY 92 * RX_HS_Gn_ADAPT_INITIAL_Capability attribute IDs. Entries for Gears 1-3 are 0 93 * (unsupported). 94 */ 95 static const u32 rx_adapt_initial_cap[UFS_HS_GEAR_MAX] = { 96 0, 97 0, 98 0, 99 RX_HS_G4_ADAPT_INITIAL_CAP, 100 RX_HS_G5_ADAPT_INITIAL_CAP, 101 RX_HS_G6_ADAPT_INITIAL_CAP 102 }; 103 104 /* 105 * pa_tx_eq_setting - Table of UniPro PA_TxEQGnSetting attribute IDs for High 106 * Speed (HS) Gears. 107 * 108 * This table maps HS Gears to their respective UniPro PA_TxEQGnSetting 109 * attribute IDs. 110 */ 111 static const u32 pa_tx_eq_setting[UFS_HS_GEAR_MAX] = { 112 PA_TXEQG1SETTING, 113 PA_TXEQG2SETTING, 114 PA_TXEQG3SETTING, 115 PA_TXEQG4SETTING, 116 PA_TXEQG5SETTING, 117 PA_TXEQG6SETTING 118 }; 119 120 /** 121 * ufshcd_configure_precoding - Configure Pre-Coding for all active lanes 122 * @hba: per adapter instance 123 * @params: TX EQ parameters data structure 124 * 125 * Bit[7] in RX_FOM indicates that the receiver needs to enable Pre-Coding when 126 * set. Pre-Coding must be enabled on both the transmitter and receiver to 127 * ensure proper operation. 128 * 129 * Returns 0 on success, non-zero error code otherwise 130 */ 131 static int ufshcd_configure_precoding(struct ufs_hba *hba, 132 struct ufshcd_tx_eq_params *params) 133 { 134 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; 135 u32 local_precode_en = 0; 136 u32 peer_precode_en = 0; 137 int lane, ret; 138 139 /* Enable Pre-Coding for Host's TX & Device's RX pair */ 140 for (lane = 0; lane < pwr_info->lane_tx; lane++) { 141 if (params->host[lane].precode_en) { 142 local_precode_en |= PRECODEEN_TX_BIT(lane); 143 peer_precode_en |= PRECODEEN_RX_BIT(lane); 144 } 145 } 146 147 /* Enable Pre-Coding for Device's TX & Host's RX pair */ 148 for (lane = 0; lane < pwr_info->lane_rx; lane++) { 149 if (params->device[lane].precode_en) { 150 peer_precode_en |= PRECODEEN_TX_BIT(lane); 151 local_precode_en |= PRECODEEN_RX_BIT(lane); 152 } 153 } 154 155 if (!local_precode_en && !peer_precode_en) { 156 dev_dbg(hba->dev, "Pre-Coding is not required for Host and Device\n"); 157 return 0; 158 } 159 160 /* Set local PA_PreCodeEn */ 161 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PRECODEEN), local_precode_en); 162 if (ret) { 163 dev_err(hba->dev, "Failed to set local PA_PreCodeEn: %d\n", ret); 164 return ret; 165 } 166 167 /* Set peer PA_PreCodeEn */ 168 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_PRECODEEN), peer_precode_en); 169 if (ret) { 170 dev_err(hba->dev, "Failed to set peer PA_PreCodeEn: %d\n", ret); 171 return ret; 172 } 173 174 dev_dbg(hba->dev, "Local PA_PreCodeEn: 0x%02x, Peer PA_PreCodeEn: 0x%02x\n", 175 local_precode_en, peer_precode_en); 176 177 return 0; 178 } 179 180 void ufshcd_print_tx_eq_params(struct ufs_hba *hba) 181 { 182 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; 183 struct ufshcd_tx_eq_params *params; 184 u32 gear = hba->pwr_info.gear_tx; 185 int lane; 186 187 if (!ufshcd_is_tx_eq_supported(hba)) 188 return; 189 190 if (gear < UFS_HS_G1 || gear > UFS_HS_GEAR_MAX) 191 return; 192 193 params = &hba->tx_eq_params[gear - 1]; 194 if (!params->is_valid || !params->is_applied) 195 return; 196 197 for (lane = 0; lane < pwr_info->lane_tx; lane++) 198 dev_dbg(hba->dev, "Host TX Lane %d: PreShoot %u, DeEmphasis %u, FOM %u, PreCodeEn %d\n", 199 lane, params->host[lane].preshoot, 200 params->host[lane].deemphasis, 201 params->host[lane].fom_val, 202 params->host[lane].precode_en); 203 204 for (lane = 0; lane < pwr_info->lane_rx; lane++) 205 dev_dbg(hba->dev, "Device TX Lane %d: PreShoot %u, DeEmphasis %u, FOM %u, PreCodeEn %d\n", 206 lane, params->device[lane].preshoot, 207 params->device[lane].deemphasis, 208 params->device[lane].fom_val, 209 params->device[lane].precode_en); 210 } 211 212 static inline u32 213 ufshcd_compose_tx_eq_setting(struct ufshcd_tx_eq_settings *settings, 214 int num_lanes) 215 { 216 u32 setting = 0; 217 int lane; 218 219 for (lane = 0; lane < num_lanes; lane++, settings++) { 220 setting |= TX_HS_PRESHOOT_BITS(lane, settings->preshoot); 221 setting |= TX_HS_DEEMPHASIS_BITS(lane, settings->deemphasis); 222 } 223 224 return setting; 225 } 226 227 /** 228 * ufshcd_apply_tx_eq_settings - Apply TX Equalization settings for target gear 229 * @hba: per adapter instance 230 * @params: TX EQ parameters data structure 231 * @gear: target gear 232 * 233 * Returns 0 on success, negative error code otherwise 234 */ 235 int ufshcd_apply_tx_eq_settings(struct ufs_hba *hba, 236 struct ufshcd_tx_eq_params *params, u32 gear) 237 { 238 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; 239 u32 setting; 240 int ret; 241 242 /* Compose settings for Host's TX Lanes */ 243 setting = ufshcd_compose_tx_eq_setting(params->host, pwr_info->lane_tx); 244 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(pa_tx_eq_setting[gear - 1]), setting); 245 if (ret) 246 return ret; 247 248 /* Compose settings for Device's TX Lanes */ 249 setting = ufshcd_compose_tx_eq_setting(params->device, pwr_info->lane_rx); 250 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(pa_tx_eq_setting[gear - 1]), setting); 251 if (ret) 252 return ret; 253 254 /* Configure Pre-Coding */ 255 if (gear >= UFS_HS_G6) { 256 ret = ufshcd_configure_precoding(hba, params); 257 if (ret) { 258 dev_err(hba->dev, "Failed to configure pre-coding: %d\n", ret); 259 return ret; 260 } 261 } 262 263 return 0; 264 } 265 EXPORT_SYMBOL_GPL(ufshcd_apply_tx_eq_settings); 266 267 /** 268 * ufshcd_evaluate_tx_eqtr_fom - Evaluate TX EQTR FOM results 269 * @hba: per adapter instance 270 * @pwr_mode: target power mode containing gear and rate information 271 * @eqtr_data: TX EQTR data structure 272 * @h_iter: host TX EQTR iterator data structure 273 * @d_iter: device TX EQTR iterator data structure 274 * 275 * Evaluate TX EQTR FOM results, update host and device TX EQTR data accordingy 276 * if FOM have been improved compared to previous iteration, and record TX EQTR 277 * FOM results. 278 */ 279 static void ufshcd_evaluate_tx_eqtr_fom(struct ufs_hba *hba, 280 struct ufs_pa_layer_attr *pwr_mode, 281 struct ufshcd_tx_eqtr_data *eqtr_data, 282 struct tx_eqtr_iter *h_iter, 283 struct tx_eqtr_iter *d_iter) 284 { 285 u8 preshoot, deemphasis, fom_value; 286 bool precode_en; 287 int lane; 288 289 for (lane = 0; h_iter->is_updated && lane < pwr_mode->lane_tx; lane++) { 290 preshoot = h_iter->preshoot; 291 deemphasis = h_iter->deemphasis; 292 fom_value = h_iter->fom[lane] & RX_FOM_VALUE_MASK; 293 precode_en = h_iter->fom[lane] & RX_FOM_PRECODING_EN_BIT; 294 295 /* Record host TX EQTR FOM */ 296 eqtr_data->host_fom[lane][preshoot][deemphasis] = h_iter->fom[lane]; 297 298 /* Check if FOM has been improved for host's TX Lanes */ 299 if (fom_value > eqtr_data->host[lane].fom_val) { 300 eqtr_data->host[lane].preshoot = preshoot; 301 eqtr_data->host[lane].deemphasis = deemphasis; 302 eqtr_data->host[lane].fom_val = fom_value; 303 eqtr_data->host[lane].precode_en = precode_en; 304 } 305 306 dev_dbg(hba->dev, "TX EQTR: Host TX Lane %d: PreShoot %u, DeEmphasis %u, FOM value %u, PreCodeEn %d\n", 307 lane, preshoot, deemphasis, fom_value, precode_en); 308 } 309 310 for (lane = 0; d_iter->is_updated && lane < pwr_mode->lane_rx; lane++) { 311 preshoot = d_iter->preshoot; 312 deemphasis = d_iter->deemphasis; 313 fom_value = d_iter->fom[lane] & RX_FOM_VALUE_MASK; 314 precode_en = d_iter->fom[lane] & RX_FOM_PRECODING_EN_BIT; 315 316 /* Record device TX EQTR FOM */ 317 eqtr_data->device_fom[lane][preshoot][deemphasis] = d_iter->fom[lane]; 318 319 /* Check if FOM has been improved for Device's TX Lanes */ 320 if (fom_value > eqtr_data->device[lane].fom_val) { 321 eqtr_data->device[lane].preshoot = preshoot; 322 eqtr_data->device[lane].deemphasis = deemphasis; 323 eqtr_data->device[lane].fom_val = fom_value; 324 eqtr_data->device[lane].precode_en = precode_en; 325 } 326 327 dev_dbg(hba->dev, "TX EQTR: Device TX Lane %d: PreShoot %u, DeEmphasis %u, FOM value %u, PreCodeEn %d\n", 328 lane, preshoot, deemphasis, fom_value, precode_en); 329 } 330 } 331 332 /** 333 * ufshcd_get_rx_fom - Get Figure of Merit (FOM) for both sides 334 * @hba: per adapter instance 335 * @pwr_mode: target power mode containing gear and rate information 336 * @h_iter: host TX EQTR iterator data structure 337 * @d_iter: device TX EQTR iterator data structure 338 * 339 * Returns 0 on success, negative error code otherwise 340 */ 341 static int ufshcd_get_rx_fom(struct ufs_hba *hba, 342 struct ufs_pa_layer_attr *pwr_mode, 343 struct tx_eqtr_iter *h_iter, 344 struct tx_eqtr_iter *d_iter) 345 { 346 int lane, ret; 347 u32 fom; 348 349 /* Get FOM of host's TX lanes from device's RX_FOM. */ 350 for (lane = 0; lane < pwr_mode->lane_tx; lane++) { 351 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB_SEL(RX_FOM, 352 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)), 353 &fom); 354 if (ret) 355 return ret; 356 357 h_iter->fom[lane] = (u8)fom; 358 } 359 360 /* Get FOM of device's TX lanes from host's RX_FOM. */ 361 for (lane = 0; lane < pwr_mode->lane_rx; lane++) { 362 ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(RX_FOM, 363 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)), 364 &fom); 365 if (ret) 366 return ret; 367 368 d_iter->fom[lane] = (u8)fom; 369 } 370 371 ret = ufshcd_vops_get_rx_fom(hba, pwr_mode, h_iter, d_iter); 372 if (ret) 373 dev_err(hba->dev, "Failed to get FOM via vops: %d\n", ret); 374 375 return ret; 376 } 377 378 bool ufshcd_is_txeq_presets_used(struct ufs_hba *hba) 379 { 380 return use_txeq_presets; 381 } 382 383 bool ufshcd_is_txeq_preset_selected(u8 preshoot, u8 deemphasis) 384 { 385 int i; 386 387 for (i = 0; i < UFS_TX_EQ_PRESET_MAX; i++) { 388 if (!txeq_presets_selected[i]) 389 continue; 390 391 if (preshoot == ufs_tx_eq_preset[i].preshoot && 392 deemphasis == ufs_tx_eq_preset[i].deemphasis) 393 return true; 394 } 395 396 return false; 397 } 398 399 /** 400 * tx_eqtr_iter_try_update - Try to update a TX EQTR iterator 401 * @iter: TX EQTR iterator data structure 402 * @preshoot: PreShoot value 403 * @deemphasis: DeEmphasis value 404 * 405 * This function validates whether the provided PreShoot and DeEmphasis 406 * combination can be used or not. If yes, it updates the TX EQTR iterator with 407 * the provided PreShoot and DeEmphasis, it also sets the is_updated flag 408 * to indicate the iterator has been updated. 409 */ 410 static void tx_eqtr_iter_try_update(struct tx_eqtr_iter *iter, 411 u8 preshoot, u8 deemphasis) 412 { 413 if (!test_bit(preshoot, &iter->preshoot_bitmap) || 414 !test_bit(deemphasis, &iter->deemphasis_bitmap) || 415 (use_txeq_presets && !ufshcd_is_txeq_preset_selected(preshoot, deemphasis))) { 416 iter->is_updated = false; 417 return; 418 } 419 420 iter->preshoot = preshoot; 421 iter->deemphasis = deemphasis; 422 iter->is_updated = true; 423 } 424 425 /** 426 * tx_eqtr_iter_update() - Update host and deviceTX EQTR iterators 427 * @preshoot: PreShoot value 428 * @deemphasis: DeEmphasis value 429 * @h_iter: Host TX EQTR iterator data structure 430 * @d_iter: Device TX EQTR iterator data structure 431 * 432 * Updates host and device TX Equalization training iterators with the 433 * provided PreShoot and DeEmphasis. 434 * 435 * Return: true if host and/or device TX Equalization training iterator has 436 * been updated to the provided PreShoot and DeEmphasis, false otherwise. 437 */ 438 static bool tx_eqtr_iter_update(u8 preshoot, u8 deemphasis, 439 struct tx_eqtr_iter *h_iter, 440 struct tx_eqtr_iter *d_iter) 441 { 442 tx_eqtr_iter_try_update(h_iter, preshoot, deemphasis); 443 tx_eqtr_iter_try_update(d_iter, preshoot, deemphasis); 444 445 return h_iter->is_updated || d_iter->is_updated; 446 } 447 448 /** 449 * ufshcd_tx_eqtr_iter_init - Initialize host and device TX EQTR iterators 450 * @hba: per adapter instance 451 * @h_iter: host TX EQTR iterator data structure 452 * @d_iter: device TX EQTR iterator data structure 453 * 454 * This function initializes the TX EQTR iterator structures for both host and 455 * device by reading their TX equalization capabilities. The capabilities are 456 * cached in the hba structure to avoid redundant DME operations in subsequent 457 * calls. In the TX EQTR procedure, the iterator structures are updated by 458 * tx_eqtr_iter_update() to systematically iterate through supported TX 459 * Equalization setting combinations. 460 * 461 * Returns 0 on success, negative error code otherwise 462 */ 463 static int ufshcd_tx_eqtr_iter_init(struct ufs_hba *hba, 464 struct tx_eqtr_iter *h_iter, 465 struct tx_eqtr_iter *d_iter) 466 { 467 u32 cap; 468 int ret; 469 470 if (!hba->host_preshoot_cap) { 471 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(TX_HS_PRESHOOT_SETTING_CAP), &cap); 472 if (ret) 473 return ret; 474 475 hba->host_preshoot_cap = cap & TX_EQTR_CAP_MASK; 476 } 477 478 if (!hba->host_deemphasis_cap) { 479 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(TX_HS_DEEMPHASIS_SETTING_CAP), &cap); 480 if (ret) 481 return ret; 482 483 hba->host_deemphasis_cap = cap & TX_EQTR_CAP_MASK; 484 } 485 486 if (!hba->device_preshoot_cap) { 487 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(TX_HS_PRESHOOT_SETTING_CAP), &cap); 488 if (ret) 489 return ret; 490 491 hba->device_preshoot_cap = cap & TX_EQTR_CAP_MASK; 492 } 493 494 if (!hba->device_deemphasis_cap) { 495 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(TX_HS_DEEMPHASIS_SETTING_CAP), &cap); 496 if (ret) 497 return ret; 498 499 hba->device_deemphasis_cap = cap & TX_EQTR_CAP_MASK; 500 } 501 502 /* 503 * Support PreShoot & DeEmphasis of value 0 is mandatory, hence they are 504 * not reflected in PreShoot/DeEmphasis capabilities. Left shift the 505 * capability bitmap by 1 and set bit[0] to reflect value 0 is 506 * supported, such that test_bit() can be used later for convenience. 507 */ 508 h_iter->preshoot_bitmap = (hba->host_preshoot_cap << 0x1) | 0x1; 509 h_iter->deemphasis_bitmap = (hba->host_deemphasis_cap << 0x1) | 0x1; 510 d_iter->preshoot_bitmap = (hba->device_preshoot_cap << 0x1) | 0x1; 511 d_iter->deemphasis_bitmap = (hba->device_deemphasis_cap << 0x1) | 0x1; 512 513 return 0; 514 } 515 516 /** 517 * adapt_cap_to_t_adapt - Calculate TAdapt from adapt capability 518 * @adapt_cap: Adapt capability 519 * 520 * For NRZ: 521 * IF (ADAPT_range = FINE) 522 * TADAPT = 650 x (ADAPT_length + 1) 523 * ELSE (IF ADAPT_range = COARSE) 524 * TADAPT = 650 x 2^ADAPT_length 525 * 526 * Returns calculated TAdapt value in term of Unit Intervals (UI) 527 */ 528 static inline u64 adapt_cap_to_t_adapt(u32 adapt_cap) 529 { 530 u64 tadapt; 531 u8 adapt_length = adapt_cap & ADAPT_LENGTH_MASK; 532 533 if (!IS_ADAPT_RANGE_COARSE(adapt_cap)) 534 tadapt = TADAPT_FACTOR * (adapt_length + 1); 535 else 536 tadapt = TADAPT_FACTOR * (1 << adapt_length); 537 538 return tadapt; 539 } 540 541 /** 542 * adapt_cap_to_t_adapt_l0l3 - Calculate TAdapt_L0_L3 from adapt capability 543 * @adapt_cap: Adapt capability 544 * 545 * For PAM-4: 546 * IF (ADAPT_range = FINE) 547 * TADAPT_L0_L3 = 2^9 x ADAPT_length 548 * ELSE IF (ADAPT_range = COARSE) 549 * TADAPT_L0_L3 = 2^9 x (2^ADAPT_length) 550 * 551 * Returns calculated TAdapt value in term of Unit Intervals (UI) 552 */ 553 static inline u64 adapt_cap_to_t_adapt_l0l3(u32 adapt_cap) 554 { 555 u64 tadapt; 556 u8 adapt_length = adapt_cap & ADAPT_LENGTH_MASK; 557 558 if (!IS_ADAPT_RANGE_COARSE(adapt_cap)) 559 tadapt = TADAPT_L0L3_FACTOR * adapt_length; 560 else 561 tadapt = TADAPT_L0L3_FACTOR * (1 << adapt_length); 562 563 return tadapt; 564 } 565 566 /** 567 * adapt_cap_to_t_adapt_l0l1l2l3 - Calculate TAdapt_L0_L1_L2_L3 from adapt capability 568 * @adapt_cap: Adapt capability 569 * 570 * For PAM-4: 571 * IF (ADAPT_range_L0_L1_L2_L3 = FINE) 572 * TADAPT_L0_L1_L2_L3 = 2^15 x (ADAPT_length_L0_L1_L2_L3 + 1) 573 * ELSE IF (ADAPT_range_L0_L1_L2_L3 = COARSE) 574 * TADAPT_L0_L1_L2_L3 = 2^15 x 2^ADAPT_length_L0_L1_L2_L3 575 * 576 * Returns calculated TAdapt value in term of Unit Intervals (UI) 577 */ 578 static inline u64 adapt_cap_to_t_adapt_l0l1l2l3(u32 adapt_cap) 579 { 580 u64 tadapt; 581 u8 adapt_length = adapt_cap & ADAPT_LENGTH_MASK; 582 583 if (!IS_ADAPT_RANGE_COARSE(adapt_cap)) 584 tadapt = TADAPT_L0L1L2L3_FACTOR * (adapt_length + 1); 585 else 586 tadapt = TADAPT_L0L1L2L3_FACTOR * (1 << adapt_length); 587 588 return tadapt; 589 } 590 591 /** 592 * ufshcd_setup_tx_eqtr_adapt_length - Setup TX adapt length for EQTR 593 * @hba: per adapter instance 594 * @params: TX EQ parameters data structure 595 * @gear: target gear for EQTR 596 * 597 * This function determines and configures the proper TX adapt length (TAdapt) 598 * for the TX EQTR procedure based on the target gear and RX adapt capabilities 599 * of both host and device. 600 * 601 * Guidelines from MIPI UniPro v3.0 spec - select the minimum Adapt Length for 602 * the Equalization Training procedure based on the following conditions: 603 * 604 * If the target High-Speed Gear n is HS-G4 or HS-G5: 605 * PA_TxAdaptLength_EQTR[7:0] >= Max (10us, RX_HS_Gn_ADAPT_INITIAL_Capability, 606 * PA_PeerRxHsGnAdaptInitial) 607 * PA_TxAdaptLength_EQTR[7:0] shall be shorter than PACP_REQUEST_TIMER (10ms) 608 * PA_TxAdaptLength_EQTR[15:8] is not relevant for HS-G4 and HS-G5. This field 609 * is set to 255 (reserved value). 610 * 611 * If the target High-Speed Gear n is HS-G6: 612 * PA_TxAdapthLength_EQTR >= 10us 613 * PA_TxAdapthLength_EQTR[7:0] >= Max (RX_HS_G6_ADAPT_INITIAL_Capability, 614 * PA_PeerRxHsG6AdaptInitialL0L3) 615 * PA_TxAdapthLength_EQTR[15:8] >= Max (RX_HS_G6_ADAPT_INITIAL_L0_L1_L2_L3_Capability, 616 * PA_PeerRxHsG6AdaptInitialL0L1L2L3) 617 * PA_TxAdaptLength_EQTR shall be shorter than PACP_REQUEST_TIMER value of 10ms. 618 * 619 * Since adapt capabilities encode both range (fine/coarse) and length values, 620 * direct comparison is not possible. This function converts adapt capabilities 621 * to actual time durations in Unit Intervals (UI) using the Adapt time 622 * calculation formular in M-PHY v6.0 spec (Table 8), then selects the maximum 623 * to ensure both host and device use adequate TX adapt length. 624 * 625 * Returns 0 on success, negative error code otherwise 626 */ 627 static int ufshcd_setup_tx_eqtr_adapt_length(struct ufs_hba *hba, 628 struct ufshcd_tx_eq_params *params, 629 u32 gear) 630 { 631 struct ufshcd_tx_eqtr_record *rec = params->eqtr_record; 632 u32 adapt_eqtr; 633 int ret; 634 635 if (rec && rec->saved_adapt_eqtr) { 636 adapt_eqtr = rec->saved_adapt_eqtr; 637 goto set_adapt_eqtr; 638 } 639 640 if (gear == UFS_HS_G4 || gear == UFS_HS_G5) { 641 u64 t_adapt, t_adapt_local, t_adapt_peer; 642 u32 adapt_cap_local, adapt_cap_peer, adapt_length; 643 644 ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(rx_adapt_initial_cap[gear - 1], 645 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)), 646 &adapt_cap_local); 647 if (ret) 648 return ret; 649 650 if (adapt_cap_local > ADAPT_LENGTH_MAX) { 651 dev_err(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP (0x%x) exceeds MAX\n", 652 gear, adapt_cap_local); 653 return -EINVAL; 654 } 655 656 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(pa_peer_rx_adapt_initial[gear - 1]), 657 &adapt_cap_peer); 658 if (ret) 659 return ret; 660 661 if (adapt_cap_peer > ADAPT_LENGTH_MAX) { 662 dev_err(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP (0x%x) exceeds MAX\n", 663 gear, adapt_cap_peer); 664 return -EINVAL; 665 } 666 667 t_adapt_local = adapt_cap_to_t_adapt(adapt_cap_local); 668 t_adapt_peer = adapt_cap_to_t_adapt(adapt_cap_peer); 669 t_adapt = max(t_adapt_local, t_adapt_peer); 670 671 dev_dbg(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP = 0x%x\n", 672 gear, adapt_cap_local); 673 dev_dbg(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_CAP = 0x%x\n", 674 gear, adapt_cap_peer); 675 dev_dbg(hba->dev, "t_adapt_local = %llu UI, t_adapt_peer = %llu UI\n", 676 t_adapt_local, t_adapt_peer); 677 dev_dbg(hba->dev, "TAdapt %llu UI selected for TX EQTR\n", 678 t_adapt); 679 680 adapt_length = (t_adapt_local >= t_adapt_peer) ? 681 adapt_cap_local : adapt_cap_peer; 682 683 if (gear == UFS_HS_G4 && t_adapt < TX_EQTR_HS_G4_MIN_T_ADAPT) { 684 dev_dbg(hba->dev, "TAdapt %llu UI is too short for TX EQTR for HS-G%u, use default Adapt 0x%x\n", 685 t_adapt, gear, TX_EQTR_HS_G4_ADAPT_DEFAULT); 686 adapt_length = TX_EQTR_HS_G4_ADAPT_DEFAULT; 687 } else if (gear == UFS_HS_G5 && t_adapt < TX_EQTR_HS_G5_MIN_T_ADAPT) { 688 dev_dbg(hba->dev, "TAdapt %llu UI is too short for TX EQTR for HS-G%u, use default Adapt 0x%x\n", 689 t_adapt, gear, TX_EQTR_HS_G5_ADAPT_DEFAULT); 690 adapt_length = TX_EQTR_HS_G5_ADAPT_DEFAULT; 691 } 692 693 adapt_eqtr = adapt_length | 694 (TX_EQTR_ADAPT_RESERVED << TX_EQTR_ADAPT_LENGTH_L0L1L2L3_SHIFT); 695 } else if (gear == UFS_HS_G6) { 696 u64 t_adapt, t_adapt_l0l3, t_adapt_l0l3_local, t_adapt_l0l3_peer; 697 u64 t_adapt_l0l1l2l3, t_adapt_l0l1l2l3_local, t_adapt_l0l1l2l3_peer; 698 u32 adapt_l0l3_cap_local, adapt_l0l3_cap_peer, adapt_length_l0l3; 699 u32 adapt_l0l1l2l3_cap_local, adapt_l0l1l2l3_cap_peer, adapt_length_l0l1l2l3; 700 701 ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(rx_adapt_initial_cap[gear - 1], 702 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)), 703 &adapt_l0l3_cap_local); 704 if (ret) 705 return ret; 706 707 if (adapt_l0l3_cap_local > ADAPT_L0L3_LENGTH_MAX) { 708 dev_err(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP (0x%x) exceeds MAX\n", 709 gear, adapt_l0l3_cap_local); 710 return -EINVAL; 711 } 712 713 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(pa_peer_rx_adapt_initial[gear - 1]), 714 &adapt_l0l3_cap_peer); 715 if (ret) 716 return ret; 717 718 if (adapt_l0l3_cap_peer > ADAPT_L0L3_LENGTH_MAX) { 719 dev_err(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_CAP (0x%x) exceeds MAX\n", 720 gear, adapt_l0l3_cap_peer); 721 return -EINVAL; 722 } 723 724 t_adapt_l0l3_local = adapt_cap_to_t_adapt_l0l3(adapt_l0l3_cap_local); 725 t_adapt_l0l3_peer = adapt_cap_to_t_adapt_l0l3(adapt_l0l3_cap_peer); 726 727 dev_dbg(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP = 0x%x\n", 728 gear, adapt_l0l3_cap_local); 729 dev_dbg(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_CAP = 0x%x\n", 730 gear, adapt_l0l3_cap_peer); 731 dev_dbg(hba->dev, "t_adapt_l0l3_local = %llu UI, t_adapt_l0l3_peer = %llu UI\n", 732 t_adapt_l0l3_local, t_adapt_l0l3_peer); 733 734 ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(RX_HS_G6_ADAPT_INITIAL_L0L1L2L3_CAP, 735 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)), 736 &adapt_l0l1l2l3_cap_local); 737 if (ret) 738 return ret; 739 740 if (adapt_l0l1l2l3_cap_local > ADAPT_L0L1L2L3_LENGTH_MAX) { 741 dev_err(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_L0L1L2L3_CAP (0x%x) exceeds MAX\n", 742 gear, adapt_l0l1l2l3_cap_local); 743 return -EINVAL; 744 } 745 746 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTINITIALL0L1L2L3), 747 &adapt_l0l1l2l3_cap_peer); 748 if (ret) 749 return ret; 750 751 if (adapt_l0l1l2l3_cap_peer > ADAPT_L0L1L2L3_LENGTH_MAX) { 752 dev_err(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_L0L1L2L3_CAP (0x%x) exceeds MAX\n", 753 gear, adapt_l0l1l2l3_cap_peer); 754 return -EINVAL; 755 } 756 757 t_adapt_l0l1l2l3_local = adapt_cap_to_t_adapt_l0l1l2l3(adapt_l0l1l2l3_cap_local); 758 t_adapt_l0l1l2l3_peer = adapt_cap_to_t_adapt_l0l1l2l3(adapt_l0l1l2l3_cap_peer); 759 760 dev_dbg(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_L0L1L2L3_CAP = 0x%x\n", 761 gear, adapt_l0l1l2l3_cap_local); 762 dev_dbg(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_L0L1L2L3_CAP = 0x%x\n", 763 gear, adapt_l0l1l2l3_cap_peer); 764 dev_dbg(hba->dev, "t_adapt_l0l1l2l3_local = %llu UI, t_adapt_l0l1l2l3_peer = %llu UI\n", 765 t_adapt_l0l1l2l3_local, t_adapt_l0l1l2l3_peer); 766 767 t_adapt_l0l1l2l3 = max(t_adapt_l0l1l2l3_local, t_adapt_l0l1l2l3_peer); 768 t_adapt_l0l3 = max(t_adapt_l0l3_local, t_adapt_l0l3_peer); 769 t_adapt = t_adapt_l0l3 + t_adapt_l0l1l2l3; 770 771 dev_dbg(hba->dev, "TAdapt %llu PAM-4 UI selected for TX EQTR\n", 772 t_adapt); 773 774 adapt_length_l0l3 = (t_adapt_l0l3_local >= t_adapt_l0l3_peer) ? 775 adapt_l0l3_cap_local : adapt_l0l3_cap_peer; 776 adapt_length_l0l1l2l3 = (t_adapt_l0l1l2l3_local >= t_adapt_l0l1l2l3_peer) ? 777 adapt_l0l1l2l3_cap_local : adapt_l0l1l2l3_cap_peer; 778 779 if (t_adapt < TX_EQTR_HS_G6_MIN_T_ADAPT) { 780 dev_dbg(hba->dev, "TAdapt %llu UI is too short for TX EQTR for HS-G%u, use default Adapt 0x%x\n", 781 t_adapt, gear, TX_EQTR_HS_G6_ADAPT_DEFAULT); 782 adapt_length_l0l3 = TX_EQTR_HS_G6_ADAPT_DEFAULT; 783 } 784 785 adapt_eqtr = adapt_length_l0l3 | 786 (adapt_length_l0l1l2l3 << TX_EQTR_ADAPT_LENGTH_L0L1L2L3_SHIFT); 787 } else { 788 return -EINVAL; 789 } 790 791 if (rec) 792 rec->saved_adapt_eqtr = (u16)adapt_eqtr; 793 794 set_adapt_eqtr: 795 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXADAPTLENGTH_EQTR), adapt_eqtr); 796 if (ret) 797 dev_err(hba->dev, "Failed to set adapt length for TX EQTR: %d\n", ret); 798 else 799 dev_dbg(hba->dev, "PA_TXADAPTLENGTH_EQTR configured to 0x%08x\n", adapt_eqtr); 800 801 return ret; 802 } 803 804 /** 805 * ufshcd_compose_tx_eqtr_setting - Compose TX EQTR setting 806 * @iter: TX EQTR iterator data structure 807 * @num_lanes: number of active lanes 808 * 809 * Returns composed TX EQTR setting, same setting is used for all active lanes 810 */ 811 static inline u32 ufshcd_compose_tx_eqtr_setting(struct tx_eqtr_iter *iter, 812 int num_lanes) 813 { 814 u32 setting = 0; 815 int lane; 816 817 for (lane = 0; lane < num_lanes; lane++) { 818 setting |= TX_HS_PRESHOOT_BITS(lane, iter->preshoot); 819 setting |= TX_HS_DEEMPHASIS_BITS(lane, iter->deemphasis); 820 } 821 822 return setting; 823 } 824 825 /** 826 * ufshcd_apply_tx_eqtr_settings - Apply TX EQTR setting 827 * @hba: per adapter instance 828 * @pwr_mode: target power mode containing gear and rate information 829 * @h_iter: host TX EQTR iterator data structure 830 * @d_iter: device TX EQTR iterator data structure 831 * 832 * Returns 0 on success, negative error code otherwise 833 */ 834 static int ufshcd_apply_tx_eqtr_settings(struct ufs_hba *hba, 835 struct ufs_pa_layer_attr *pwr_mode, 836 struct tx_eqtr_iter *h_iter, 837 struct tx_eqtr_iter *d_iter) 838 { 839 u32 setting; 840 int ret; 841 842 setting = ufshcd_compose_tx_eqtr_setting(h_iter, pwr_mode->lane_tx); 843 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXEQTRSETTING), setting); 844 if (ret) 845 return ret; 846 847 setting = ufshcd_compose_tx_eqtr_setting(d_iter, pwr_mode->lane_rx); 848 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PEERTXEQTRSETTING), setting); 849 if (ret) 850 return ret; 851 852 ret = ufshcd_vops_apply_tx_eqtr_settings(hba, pwr_mode, h_iter, d_iter); 853 854 return ret; 855 } 856 857 /** 858 * ufshcd_update_tx_eq_params - Update TX Equalization params 859 * @params: TX EQ parameters data structure 860 * @pwr_mode: target power mode containing gear and rate 861 * @eqtr_data: TX EQTR data structure 862 * 863 * Update TX Equalization params using results from TX EQTR data. Check also 864 * the TX EQTR FOM value for each TX lane in the TX EQTR data. If a TX lane got 865 * a FOM value of 0, restore the TX Equalization settings from the last known 866 * valid TX Equalization params for that specific TX lane. 867 */ 868 static inline void 869 ufshcd_update_tx_eq_params(struct ufshcd_tx_eq_params *params, 870 struct ufs_pa_layer_attr *pwr_mode, 871 struct ufshcd_tx_eqtr_data *eqtr_data) 872 { 873 struct ufshcd_tx_eqtr_record *rec = params->eqtr_record; 874 875 if (params->is_valid) { 876 int lane; 877 878 for (lane = 0; lane < pwr_mode->lane_tx; lane++) 879 if (eqtr_data->host[lane].fom_val == 0) 880 eqtr_data->host[lane] = params->host[lane]; 881 882 for (lane = 0; lane < pwr_mode->lane_rx; lane++) 883 if (eqtr_data->device[lane].fom_val == 0) 884 eqtr_data->device[lane] = params->device[lane]; 885 } 886 887 memcpy(params->host, eqtr_data->host, sizeof(params->host)); 888 memcpy(params->device, eqtr_data->device, sizeof(params->device)); 889 890 if (!rec) 891 return; 892 893 memcpy(rec->host_fom, eqtr_data->host_fom, sizeof(rec->host_fom)); 894 memcpy(rec->device_fom, eqtr_data->device_fom, sizeof(rec->device_fom)); 895 rec->last_record_ts = ktime_get(); 896 rec->last_record_index++; 897 } 898 899 /** 900 * __ufshcd_tx_eqtr - TX Equalization Training (EQTR) procedure 901 * @hba: per adapter instance 902 * @params: TX EQ parameters data structure 903 * @pwr_mode: target power mode containing gear and rate information 904 * 905 * This function implements the complete TX EQTR procedure as defined in UFSHCI 906 * v5.0 specification. It iterates through all possible combinations of PreShoot 907 * and DeEmphasis settings to find the optimal TX Equalization settings for all 908 * active lanes. 909 * 910 * Returns 0 on success, negative error code otherwise 911 */ 912 static int __ufshcd_tx_eqtr(struct ufs_hba *hba, 913 struct ufshcd_tx_eq_params *params, 914 struct ufs_pa_layer_attr *pwr_mode) 915 { 916 struct ufshcd_tx_eqtr_data *eqtr_data __free(kfree) = 917 kzalloc(sizeof(*eqtr_data), GFP_KERNEL); 918 struct tx_eqtr_iter h_iter = {}; 919 struct tx_eqtr_iter d_iter = {}; 920 u32 gear = pwr_mode->gear_tx; 921 u8 preshoot, deemphasis; 922 ktime_t start; 923 int ret; 924 925 if (!eqtr_data) 926 return -ENOMEM; 927 928 dev_info(hba->dev, "Start TX EQTR procedure for HS-G%u, Rate-%s, RX Lanes: %u, TX Lanes: %u\n", 929 gear, ufs_hs_rate_to_str(pwr_mode->hs_rate), 930 pwr_mode->lane_rx, pwr_mode->lane_tx); 931 932 start = ktime_get(); 933 934 /* Step 1 - Determine the TX Adapt Length for EQTR */ 935 ret = ufshcd_setup_tx_eqtr_adapt_length(hba, params, gear); 936 if (ret) { 937 dev_err(hba->dev, "Failed to setup TX EQTR Adaptation length: %d\n", ret); 938 return ret; 939 } 940 941 /* Step 2 - Determine TX Equalization setting capabilities */ 942 ret = ufshcd_tx_eqtr_iter_init(hba, &h_iter, &d_iter); 943 if (ret) { 944 dev_err(hba->dev, "Failed to init TX EQTR data: %d\n", ret); 945 return ret; 946 } 947 948 /* TX EQTR main loop */ 949 for (preshoot = 0; preshoot < TX_HS_NUM_PRESHOOT; preshoot++) { 950 for (deemphasis = 0; deemphasis < TX_HS_NUM_DEEMPHASIS; deemphasis++) { 951 if (!tx_eqtr_iter_update(preshoot, deemphasis, &h_iter, &d_iter)) 952 continue; 953 954 /* Step 3 - Apply TX EQTR settings */ 955 ret = ufshcd_apply_tx_eqtr_settings(hba, pwr_mode, &h_iter, &d_iter); 956 if (ret) { 957 dev_err(hba->dev, "Failed to apply TX EQTR settings (PreShoot %u, DeEmphasis %u): %d\n", 958 preshoot, deemphasis, ret); 959 return ret; 960 } 961 962 /* Step 4 - Trigger UIC TX EQTR */ 963 ret = ufshcd_uic_tx_eqtr(hba, gear); 964 if (ret) { 965 dev_err(hba->dev, "Failed to trigger UIC TX EQTR for target gear %u: %d\n", 966 gear, ret); 967 return ret; 968 } 969 970 /* Step 5 - Get FOM */ 971 ret = ufshcd_get_rx_fom(hba, pwr_mode, &h_iter, &d_iter); 972 if (ret) { 973 dev_err(hba->dev, "Failed to get RX_FOM: %d\n", 974 ret); 975 return ret; 976 } 977 978 ufshcd_evaluate_tx_eqtr_fom(hba, pwr_mode, eqtr_data, &h_iter, &d_iter); 979 } 980 } 981 982 dev_info(hba->dev, "TX EQTR procedure completed! Time elapsed: %llu ms\n", 983 ktime_to_ms(ktime_sub(ktime_get(), start))); 984 985 ufshcd_update_tx_eq_params(params, pwr_mode, eqtr_data); 986 987 return ret; 988 } 989 990 /** 991 * ufshcd_tx_eqtr_prepare - Prepare UFS link for TX EQTR procedure 992 * @hba: per adapter instance 993 * @pwr_mode: target power mode containing gear and rate 994 * 995 * This function prepares the UFS link for TX Equalization Training (EQTR) by 996 * establishing the proper initial conditions required by the EQTR procedure. 997 * It ensures that EQTR starts from the most reliable Power Mode (HS-G1) with 998 * all connected lanes activated and sets host TX HS Adapt Type to INITIAL. 999 * 1000 * Returns 0 on successful preparation, negative error code on failure 1001 */ 1002 static int ufshcd_tx_eqtr_prepare(struct ufs_hba *hba, 1003 struct ufs_pa_layer_attr *pwr_mode) 1004 { 1005 struct ufs_pa_layer_attr pwr_mode_hs_g1 = { 1006 /* TX EQTR shall be initiated from the most reliable HS-G1 */ 1007 .gear_rx = UFS_HS_G1, 1008 .gear_tx = UFS_HS_G1, 1009 .lane_rx = pwr_mode->lane_rx, 1010 .lane_tx = pwr_mode->lane_tx, 1011 .pwr_rx = FAST_MODE, 1012 .pwr_tx = FAST_MODE, 1013 /* Use the target power mode's HS rate */ 1014 .hs_rate = pwr_mode->hs_rate, 1015 }; 1016 u32 rate = pwr_mode->hs_rate; 1017 int ret; 1018 1019 /* Change power mode to HS-G1, activate all connected lanes. */ 1020 ret = ufshcd_change_power_mode(hba, &pwr_mode_hs_g1, 1021 UFSHCD_PMC_POLICY_DONT_FORCE); 1022 if (ret) { 1023 dev_err(hba->dev, "TX EQTR: Failed to change power mode to HS-G1, Rate-%s: %d\n", 1024 ufs_hs_rate_to_str(rate), ret); 1025 return ret; 1026 } 1027 1028 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXHSADAPTTYPE), 1029 PA_INITIAL_ADAPT); 1030 if (ret) 1031 dev_err(hba->dev, "TX EQTR: Failed to set Host Adapt type to INITIAL: %d\n", 1032 ret); 1033 1034 return ret; 1035 } 1036 1037 static void ufshcd_tx_eqtr_unprepare(struct ufs_hba *hba, 1038 struct ufs_pa_layer_attr *pwr_mode) 1039 { 1040 int err; 1041 1042 if (pwr_mode->pwr_rx == SLOWAUTO_MODE || pwr_mode->hs_rate == 0) 1043 return; 1044 1045 err = ufshcd_change_power_mode(hba, pwr_mode, 1046 UFSHCD_PMC_POLICY_DONT_FORCE); 1047 if (err) 1048 dev_err(hba->dev, "%s: Failed to restore Power Mode: %d\n", 1049 __func__, err); 1050 } 1051 1052 /** 1053 * ufshcd_tx_eqtr - Perform TX EQTR procedures with vops callbacks 1054 * @hba: per adapter instance 1055 * @params: TX EQ parameters data structure to populate 1056 * @pwr_mode: target power mode containing gear and rate information 1057 * 1058 * This is the main entry point for performing TX Equalization Training (EQTR) 1059 * procedure as defined in UFSCHI v5.0 specification. It serves as a wrapper 1060 * around __ufshcd_tx_eqtr() to provide vops support through the variant 1061 * operations framework. 1062 * 1063 * Returns 0 on success, negative error code on failure 1064 */ 1065 static int ufshcd_tx_eqtr(struct ufs_hba *hba, 1066 struct ufshcd_tx_eq_params *params, 1067 struct ufs_pa_layer_attr *pwr_mode) 1068 { 1069 struct ufs_pa_layer_attr old_pwr_info; 1070 int ret; 1071 1072 if (!params->eqtr_record) { 1073 params->eqtr_record = devm_kzalloc(hba->dev, 1074 sizeof(*params->eqtr_record), 1075 GFP_KERNEL); 1076 if (!params->eqtr_record) 1077 return -ENOMEM; 1078 } 1079 1080 memcpy(&old_pwr_info, &hba->pwr_info, sizeof(struct ufs_pa_layer_attr)); 1081 1082 ret = ufshcd_tx_eqtr_prepare(hba, pwr_mode); 1083 if (ret) { 1084 dev_err(hba->dev, "Failed to prepare TX EQTR: %d\n", ret); 1085 goto out; 1086 } 1087 1088 ret = ufshcd_vops_tx_eqtr_notify(hba, PRE_CHANGE, pwr_mode); 1089 if (ret) 1090 goto out; 1091 1092 ret = __ufshcd_tx_eqtr(hba, params, pwr_mode); 1093 if (ret) 1094 goto out; 1095 1096 ret = ufshcd_vops_tx_eqtr_notify(hba, POST_CHANGE, pwr_mode); 1097 1098 out: 1099 if (ret) 1100 ufshcd_tx_eqtr_unprepare(hba, &old_pwr_info); 1101 1102 return ret; 1103 } 1104 1105 /** 1106 * ufshcd_config_tx_eq_settings - Configure TX Equalization settings 1107 * @hba: per adapter instance 1108 * @pwr_mode: target power mode containing gear and rate information 1109 * @force_tx_eqtr: execute the TX EQTR procedure 1110 * 1111 * This function finds and sets the TX Equalization settings for the given 1112 * target power mode. 1113 * 1114 * Returns 0 on success, error code otherwise 1115 */ 1116 int ufshcd_config_tx_eq_settings(struct ufs_hba *hba, 1117 struct ufs_pa_layer_attr *pwr_mode, 1118 bool force_tx_eqtr) 1119 { 1120 struct ufshcd_tx_eq_params *params; 1121 u32 gear, rate; 1122 1123 if (!ufshcd_is_tx_eq_supported(hba) || !use_adaptive_txeq) 1124 return 0; 1125 1126 if (!hba->max_pwr_info.is_valid) { 1127 dev_err(hba->dev, "Max power info is invalid\n"); 1128 return -EINVAL; 1129 } 1130 1131 if (!pwr_mode) { 1132 dev_err(hba->dev, "Target power mode is NULL\n"); 1133 return -EINVAL; 1134 } 1135 1136 gear = pwr_mode->gear_tx; 1137 rate = pwr_mode->hs_rate; 1138 1139 if (gear < UFS_HS_G1 || gear > UFS_HS_GEAR_MAX) { 1140 dev_err(hba->dev, "Invalid HS-Gear (%u) for TX Equalization\n", 1141 gear); 1142 return -EINVAL; 1143 } else if (gear < max_t(u32, adaptive_txeq_gear, UFS_HS_G4)) { 1144 /* TX EQTR is supported for HS-G4 and higher Gears */ 1145 return 0; 1146 } 1147 1148 if (rate != PA_HS_MODE_A && rate != PA_HS_MODE_B) { 1149 dev_err(hba->dev, "Invalid HS-Rate (%u) for TX Equalization\n", 1150 rate); 1151 return -EINVAL; 1152 } 1153 1154 params = &hba->tx_eq_params[gear - 1]; 1155 if (!params->is_valid || force_tx_eqtr) { 1156 int ret; 1157 1158 ret = ufshcd_tx_eqtr(hba, params, pwr_mode); 1159 if (ret) { 1160 dev_err(hba->dev, "Failed to train TX Equalization for HS-G%u, Rate-%s: %d\n", 1161 gear, ufs_hs_rate_to_str(rate), ret); 1162 return ret; 1163 } 1164 1165 /* Mark TX Equalization settings as valid */ 1166 params->is_valid = true; 1167 params->is_applied = false; 1168 } 1169 1170 if (params->is_valid && !params->is_applied) { 1171 int ret; 1172 1173 ret = ufshcd_apply_tx_eq_settings(hba, params, gear); 1174 if (ret) { 1175 dev_err(hba->dev, "Failed to apply TX Equalization settings for HS-G%u, Rate-%s: %d\n", 1176 gear, ufs_hs_rate_to_str(rate), ret); 1177 return ret; 1178 } 1179 1180 params->is_applied = true; 1181 } 1182 1183 return 0; 1184 } 1185 1186 /** 1187 * ufshcd_apply_valid_tx_eq_settings - Apply valid TX Equalization settings 1188 * @hba: per-adapter instance 1189 * 1190 * This function iterates through all supported High-Speed (HS) gears and 1191 * applies valid TX Equalization settings to both Host and Device. 1192 */ 1193 void ufshcd_apply_valid_tx_eq_settings(struct ufs_hba *hba) 1194 { 1195 struct ufshcd_tx_eq_params *params; 1196 int gear, err; 1197 1198 if (!ufshcd_is_tx_eq_supported(hba)) 1199 return; 1200 1201 if (!hba->max_pwr_info.is_valid) { 1202 dev_err(hba->dev, "Max power info is invalid, cannot apply TX Equalization settings\n"); 1203 return; 1204 } 1205 1206 for (gear = UFS_HS_G1; gear <= UFS_HS_GEAR_MAX; gear++) { 1207 params = &hba->tx_eq_params[gear - 1]; 1208 1209 if (params->is_valid) { 1210 err = ufshcd_apply_tx_eq_settings(hba, params, gear); 1211 if (err) { 1212 params->is_applied = false; 1213 dev_err(hba->dev, "Failed to apply TX Equalization settings for HS-G%u: %d\n", 1214 gear, err); 1215 } else { 1216 params->is_applied = true; 1217 } 1218 } 1219 } 1220 } 1221 1222 /** 1223 * ufshcd_retrain_tx_eq - Retrain TX Equalization and apply new settings 1224 * @hba: per-adapter instance 1225 * @gear: target High-Speed (HS) gear for retraining 1226 * 1227 * This function initiates a refresh of the TX Equalization settings for a 1228 * specific HS gear. It scales the clocks to maximum frequency, negotiates the 1229 * power mode with the device, retrains TX EQ and applies new TX EQ settings 1230 * by conducting a Power Mode change. 1231 * 1232 * Returns 0 on success, non-zero error code otherwise 1233 */ 1234 int ufshcd_retrain_tx_eq(struct ufs_hba *hba, u32 gear) 1235 { 1236 struct ufs_pa_layer_attr new_pwr_info, final_params = {}; 1237 int ret; 1238 1239 if (!ufshcd_is_tx_eq_supported(hba) || !use_adaptive_txeq) 1240 return -EOPNOTSUPP; 1241 1242 if (gear < adaptive_txeq_gear) 1243 return -ERANGE; 1244 1245 ufshcd_hold(hba); 1246 1247 ret = ufshcd_pause_command_processing(hba, 1 * USEC_PER_SEC); 1248 if (ret) { 1249 ufshcd_release(hba); 1250 return ret; 1251 } 1252 1253 /* scale up clocks to max frequency before TX EQTR */ 1254 if (ufshcd_is_clkscaling_supported(hba)) 1255 ufshcd_scale_clks(hba, ULONG_MAX, true); 1256 1257 new_pwr_info = hba->pwr_info; 1258 new_pwr_info.gear_tx = gear; 1259 new_pwr_info.gear_rx = gear; 1260 1261 ret = ufshcd_vops_negotiate_pwr_mode(hba, &new_pwr_info, &final_params); 1262 if (ret) 1263 memcpy(&final_params, &new_pwr_info, sizeof(final_params)); 1264 1265 if (final_params.gear_tx != gear) { 1266 dev_err(hba->dev, "Negotiated Gear (%u) does not match target Gear (%u)\n", 1267 final_params.gear_tx, gear); 1268 ret = -EINVAL; 1269 goto out; 1270 } 1271 1272 ret = ufshcd_config_tx_eq_settings(hba, &final_params, true); 1273 if (ret) { 1274 dev_err(hba->dev, "Failed to config TX Equalization for HS-G%u, Rate-%s: %d\n", 1275 final_params.gear_tx, 1276 ufs_hs_rate_to_str(final_params.hs_rate), ret); 1277 goto out; 1278 } 1279 1280 /* Change Power Mode to apply the new TX EQ settings */ 1281 ret = ufshcd_change_power_mode(hba, &final_params, 1282 UFSHCD_PMC_POLICY_FORCE); 1283 if (ret) 1284 dev_err(hba->dev, "%s: Failed to change Power Mode to HS-G%u, Rate-%s: %d\n", 1285 __func__, final_params.gear_tx, 1286 ufs_hs_rate_to_str(final_params.hs_rate), ret); 1287 1288 out: 1289 ufshcd_resume_command_processing(hba); 1290 ufshcd_release(hba); 1291 1292 return ret; 1293 } 1294