1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2013-2016, Linux Foundation. All rights reserved.
4 */
5
6 #include <linux/acpi.h>
7 #include <linux/clk.h>
8 #include <linux/cleanup.h>
9 #include <linux/delay.h>
10 #include <linux/devfreq.h>
11 #include <linux/gpio/consumer.h>
12 #include <linux/interconnect.h>
13 #include <linux/module.h>
14 #include <linux/of.h>
15 #include <linux/phy/phy.h>
16 #include <linux/platform_device.h>
17 #include <linux/pm_domain.h>
18 #include <linux/reset-controller.h>
19 #include <linux/time.h>
20 #include <linux/unaligned.h>
21 #include <linux/units.h>
22
23 #include <soc/qcom/ice.h>
24
25 #include <ufs/ufshcd.h>
26 #include <ufs/ufshci.h>
27 #include <ufs/ufs_quirks.h>
28 #include <ufs/unipro.h>
29 #include "ufshcd-pltfrm.h"
30 #include "ufs-qcom.h"
31
32 #define MCQ_QCFGPTR_MASK GENMASK(7, 0)
33 #define MCQ_QCFGPTR_UNIT 0x200
34 #define MCQ_SQATTR_OFFSET(c) \
35 ((((c) >> 16) & MCQ_QCFGPTR_MASK) * MCQ_QCFGPTR_UNIT)
36 #define MCQ_QCFG_SIZE 0x40
37
38 /* De-emphasis for gear-5 */
39 #define DEEMPHASIS_3_5_dB 0x04
40 #define NO_DEEMPHASIS 0x0
41
42 #define UFS_ICE_SYNC_RST_SEL BIT(3)
43 #define UFS_ICE_SYNC_RST_SW BIT(4)
44
45 enum {
46 TSTBUS_UAWM,
47 TSTBUS_UARM,
48 TSTBUS_TXUC,
49 TSTBUS_RXUC,
50 TSTBUS_DFC,
51 TSTBUS_TRLUT,
52 TSTBUS_TMRLUT,
53 TSTBUS_OCSC,
54 TSTBUS_UTP_HCI,
55 TSTBUS_COMBINED,
56 TSTBUS_WRAPPER,
57 TSTBUS_UNIPRO,
58 TSTBUS_MAX,
59 };
60
61 #define QCOM_UFS_MAX_GEAR 5
62 #define QCOM_UFS_MAX_LANE 2
63
64 enum {
65 MODE_MIN,
66 MODE_PWM,
67 MODE_HS_RA,
68 MODE_HS_RB,
69 MODE_MAX,
70 };
71
72 static const struct __ufs_qcom_bw_table {
73 u32 mem_bw;
74 u32 cfg_bw;
75 } ufs_qcom_bw_table[MODE_MAX + 1][QCOM_UFS_MAX_GEAR + 1][QCOM_UFS_MAX_LANE + 1] = {
76 [MODE_MIN][0][0] = { 0, 0 }, /* Bandwidth values in KB/s */
77 [MODE_PWM][UFS_PWM_G1][UFS_LANE_1] = { 922, 1000 },
78 [MODE_PWM][UFS_PWM_G2][UFS_LANE_1] = { 1844, 1000 },
79 [MODE_PWM][UFS_PWM_G3][UFS_LANE_1] = { 3688, 1000 },
80 [MODE_PWM][UFS_PWM_G4][UFS_LANE_1] = { 7376, 1000 },
81 [MODE_PWM][UFS_PWM_G5][UFS_LANE_1] = { 14752, 1000 },
82 [MODE_PWM][UFS_PWM_G1][UFS_LANE_2] = { 1844, 1000 },
83 [MODE_PWM][UFS_PWM_G2][UFS_LANE_2] = { 3688, 1000 },
84 [MODE_PWM][UFS_PWM_G3][UFS_LANE_2] = { 7376, 1000 },
85 [MODE_PWM][UFS_PWM_G4][UFS_LANE_2] = { 14752, 1000 },
86 [MODE_PWM][UFS_PWM_G5][UFS_LANE_2] = { 29504, 1000 },
87 [MODE_HS_RA][UFS_HS_G1][UFS_LANE_1] = { 127796, 1000 },
88 [MODE_HS_RA][UFS_HS_G2][UFS_LANE_1] = { 255591, 1000 },
89 [MODE_HS_RA][UFS_HS_G3][UFS_LANE_1] = { 1492582, 102400 },
90 [MODE_HS_RA][UFS_HS_G4][UFS_LANE_1] = { 2915200, 204800 },
91 [MODE_HS_RA][UFS_HS_G5][UFS_LANE_1] = { 5836800, 409600 },
92 [MODE_HS_RA][UFS_HS_G1][UFS_LANE_2] = { 255591, 1000 },
93 [MODE_HS_RA][UFS_HS_G2][UFS_LANE_2] = { 511181, 1000 },
94 [MODE_HS_RA][UFS_HS_G3][UFS_LANE_2] = { 1492582, 204800 },
95 [MODE_HS_RA][UFS_HS_G4][UFS_LANE_2] = { 2915200, 409600 },
96 [MODE_HS_RA][UFS_HS_G5][UFS_LANE_2] = { 5836800, 819200 },
97 [MODE_HS_RB][UFS_HS_G1][UFS_LANE_1] = { 149422, 1000 },
98 [MODE_HS_RB][UFS_HS_G2][UFS_LANE_1] = { 298189, 1000 },
99 [MODE_HS_RB][UFS_HS_G3][UFS_LANE_1] = { 1492582, 102400 },
100 [MODE_HS_RB][UFS_HS_G4][UFS_LANE_1] = { 2915200, 204800 },
101 [MODE_HS_RB][UFS_HS_G5][UFS_LANE_1] = { 5836800, 409600 },
102 [MODE_HS_RB][UFS_HS_G1][UFS_LANE_2] = { 298189, 1000 },
103 [MODE_HS_RB][UFS_HS_G2][UFS_LANE_2] = { 596378, 1000 },
104 [MODE_HS_RB][UFS_HS_G3][UFS_LANE_2] = { 1492582, 204800 },
105 [MODE_HS_RB][UFS_HS_G4][UFS_LANE_2] = { 2915200, 409600 },
106 [MODE_HS_RB][UFS_HS_G5][UFS_LANE_2] = { 5836800, 819200 },
107 [MODE_MAX][0][0] = { 7643136, 819200 },
108 };
109
110 static const struct {
111 int nminor;
112 char *prefix;
113 } testbus_info[TSTBUS_MAX] = {
114 [TSTBUS_UAWM] = {32, "TSTBUS_UAWM"},
115 [TSTBUS_UARM] = {32, "TSTBUS_UARM"},
116 [TSTBUS_TXUC] = {32, "TSTBUS_TXUC"},
117 [TSTBUS_RXUC] = {32, "TSTBUS_RXUC"},
118 [TSTBUS_DFC] = {32, "TSTBUS_DFC"},
119 [TSTBUS_TRLUT] = {32, "TSTBUS_TRLUT"},
120 [TSTBUS_TMRLUT] = {32, "TSTBUS_TMRLUT"},
121 [TSTBUS_OCSC] = {32, "TSTBUS_OCSC"},
122 [TSTBUS_UTP_HCI] = {32, "TSTBUS_UTP_HCI"},
123 [TSTBUS_COMBINED] = {32, "TSTBUS_COMBINED"},
124 [TSTBUS_WRAPPER] = {32, "TSTBUS_WRAPPER"},
125 [TSTBUS_UNIPRO] = {256, "TSTBUS_UNIPRO"},
126 };
127
128 static void ufs_qcom_get_default_testbus_cfg(struct ufs_qcom_host *host);
129 static unsigned long ufs_qcom_opp_freq_to_clk_freq(struct ufs_hba *hba,
130 unsigned long freq, char *name);
131 static int ufs_qcom_set_core_clk_ctrl(struct ufs_hba *hba, bool is_scale_up, unsigned long freq);
132
rcdev_to_ufs_host(struct reset_controller_dev * rcd)133 static struct ufs_qcom_host *rcdev_to_ufs_host(struct reset_controller_dev *rcd)
134 {
135 return container_of(rcd, struct ufs_qcom_host, rcdev);
136 }
137
138 #ifdef CONFIG_SCSI_UFS_CRYPTO
139 /**
140 * ufs_qcom_config_ice_allocator() - ICE core allocator configuration
141 *
142 * @host: pointer to qcom specific variant structure.
143 */
ufs_qcom_config_ice_allocator(struct ufs_qcom_host * host)144 static void ufs_qcom_config_ice_allocator(struct ufs_qcom_host *host)
145 {
146 struct ufs_hba *hba = host->hba;
147 static const uint8_t val[4] = { NUM_RX_R1W0, NUM_TX_R0W1, NUM_RX_R1W1, NUM_TX_R1W1 };
148 u32 config;
149
150 if (!(host->caps & UFS_QCOM_CAP_ICE_CONFIG) ||
151 !(host->hba->caps & UFSHCD_CAP_CRYPTO))
152 return;
153
154 config = get_unaligned_le32(val);
155
156 ufshcd_writel(hba, ICE_ALLOCATOR_TYPE, REG_UFS_MEM_ICE_CONFIG);
157 ufshcd_writel(hba, config, REG_UFS_MEM_ICE_NUM_CORE);
158 }
159
ufs_qcom_ice_enable(struct ufs_qcom_host * host)160 static inline void ufs_qcom_ice_enable(struct ufs_qcom_host *host)
161 {
162 if (host->hba->caps & UFSHCD_CAP_CRYPTO)
163 qcom_ice_enable(host->ice);
164 }
165
166 static const struct blk_crypto_ll_ops ufs_qcom_crypto_ops; /* forward decl */
167
ufs_qcom_ice_init(struct ufs_qcom_host * host)168 static int ufs_qcom_ice_init(struct ufs_qcom_host *host)
169 {
170 struct ufs_hba *hba = host->hba;
171 struct blk_crypto_profile *profile = &hba->crypto_profile;
172 struct device *dev = hba->dev;
173 struct qcom_ice *ice;
174 union ufs_crypto_capabilities caps;
175 union ufs_crypto_cap_entry cap;
176 int err;
177 int i;
178
179 ice = devm_of_qcom_ice_get(dev);
180 if (IS_ERR(ice)) {
181 if (ice != ERR_PTR(-EOPNOTSUPP))
182 return PTR_ERR(ice);
183
184 dev_warn(dev, "Disabling inline encryption support\n");
185 return 0;
186 }
187
188 host->ice = ice;
189
190 /* Initialize the blk_crypto_profile */
191
192 caps.reg_val = cpu_to_le32(ufshcd_readl(hba, REG_UFS_CCAP));
193
194 /* The number of keyslots supported is (CFGC+1) */
195 err = devm_blk_crypto_profile_init(dev, profile, caps.config_count + 1);
196 if (err)
197 return err;
198
199 profile->ll_ops = ufs_qcom_crypto_ops;
200 profile->max_dun_bytes_supported = 8;
201 profile->key_types_supported = qcom_ice_get_supported_key_type(ice);
202 profile->dev = dev;
203
204 /*
205 * Currently this driver only supports AES-256-XTS. All known versions
206 * of ICE support it, but to be safe make sure it is really declared in
207 * the crypto capability registers. The crypto capability registers
208 * also give the supported data unit size(s).
209 */
210 for (i = 0; i < caps.num_crypto_cap; i++) {
211 cap.reg_val = cpu_to_le32(ufshcd_readl(hba,
212 REG_UFS_CRYPTOCAP +
213 i * sizeof(__le32)));
214 if (cap.algorithm_id == UFS_CRYPTO_ALG_AES_XTS &&
215 cap.key_size == UFS_CRYPTO_KEY_SIZE_256)
216 profile->modes_supported[BLK_ENCRYPTION_MODE_AES_256_XTS] |=
217 cap.sdus_mask * 512;
218 }
219
220 hba->caps |= UFSHCD_CAP_CRYPTO;
221 hba->quirks |= UFSHCD_QUIRK_CUSTOM_CRYPTO_PROFILE;
222 return 0;
223 }
224
ufs_qcom_ice_resume(struct ufs_qcom_host * host)225 static inline int ufs_qcom_ice_resume(struct ufs_qcom_host *host)
226 {
227 if (host->hba->caps & UFSHCD_CAP_CRYPTO)
228 return qcom_ice_resume(host->ice);
229
230 return 0;
231 }
232
ufs_qcom_ice_suspend(struct ufs_qcom_host * host)233 static inline int ufs_qcom_ice_suspend(struct ufs_qcom_host *host)
234 {
235 if (host->hba->caps & UFSHCD_CAP_CRYPTO)
236 return qcom_ice_suspend(host->ice);
237
238 return 0;
239 }
240
ufs_qcom_ice_keyslot_program(struct blk_crypto_profile * profile,const struct blk_crypto_key * key,unsigned int slot)241 static int ufs_qcom_ice_keyslot_program(struct blk_crypto_profile *profile,
242 const struct blk_crypto_key *key,
243 unsigned int slot)
244 {
245 struct ufs_hba *hba = ufs_hba_from_crypto_profile(profile);
246 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
247 int err;
248
249 ufshcd_hold(hba);
250 err = qcom_ice_program_key(host->ice, slot, key);
251 ufshcd_release(hba);
252 return err;
253 }
254
ufs_qcom_ice_keyslot_evict(struct blk_crypto_profile * profile,const struct blk_crypto_key * key,unsigned int slot)255 static int ufs_qcom_ice_keyslot_evict(struct blk_crypto_profile *profile,
256 const struct blk_crypto_key *key,
257 unsigned int slot)
258 {
259 struct ufs_hba *hba = ufs_hba_from_crypto_profile(profile);
260 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
261 int err;
262
263 ufshcd_hold(hba);
264 err = qcom_ice_evict_key(host->ice, slot);
265 ufshcd_release(hba);
266 return err;
267 }
268
ufs_qcom_ice_derive_sw_secret(struct blk_crypto_profile * profile,const u8 * eph_key,size_t eph_key_size,u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE])269 static int ufs_qcom_ice_derive_sw_secret(struct blk_crypto_profile *profile,
270 const u8 *eph_key, size_t eph_key_size,
271 u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE])
272 {
273 struct ufs_hba *hba = ufs_hba_from_crypto_profile(profile);
274 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
275
276 return qcom_ice_derive_sw_secret(host->ice, eph_key, eph_key_size,
277 sw_secret);
278 }
279
ufs_qcom_ice_import_key(struct blk_crypto_profile * profile,const u8 * raw_key,size_t raw_key_size,u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE])280 static int ufs_qcom_ice_import_key(struct blk_crypto_profile *profile,
281 const u8 *raw_key, size_t raw_key_size,
282 u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE])
283 {
284 struct ufs_hba *hba = ufs_hba_from_crypto_profile(profile);
285 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
286
287 return qcom_ice_import_key(host->ice, raw_key, raw_key_size, lt_key);
288 }
289
ufs_qcom_ice_generate_key(struct blk_crypto_profile * profile,u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE])290 static int ufs_qcom_ice_generate_key(struct blk_crypto_profile *profile,
291 u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE])
292 {
293 struct ufs_hba *hba = ufs_hba_from_crypto_profile(profile);
294 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
295
296 return qcom_ice_generate_key(host->ice, lt_key);
297 }
298
ufs_qcom_ice_prepare_key(struct blk_crypto_profile * profile,const u8 * lt_key,size_t lt_key_size,u8 eph_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE])299 static int ufs_qcom_ice_prepare_key(struct blk_crypto_profile *profile,
300 const u8 *lt_key, size_t lt_key_size,
301 u8 eph_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE])
302 {
303 struct ufs_hba *hba = ufs_hba_from_crypto_profile(profile);
304 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
305
306 return qcom_ice_prepare_key(host->ice, lt_key, lt_key_size, eph_key);
307 }
308
309 static const struct blk_crypto_ll_ops ufs_qcom_crypto_ops = {
310 .keyslot_program = ufs_qcom_ice_keyslot_program,
311 .keyslot_evict = ufs_qcom_ice_keyslot_evict,
312 .derive_sw_secret = ufs_qcom_ice_derive_sw_secret,
313 .import_key = ufs_qcom_ice_import_key,
314 .generate_key = ufs_qcom_ice_generate_key,
315 .prepare_key = ufs_qcom_ice_prepare_key,
316 };
317
318 #else
319
ufs_qcom_ice_enable(struct ufs_qcom_host * host)320 static inline void ufs_qcom_ice_enable(struct ufs_qcom_host *host)
321 {
322 }
323
ufs_qcom_ice_init(struct ufs_qcom_host * host)324 static int ufs_qcom_ice_init(struct ufs_qcom_host *host)
325 {
326 return 0;
327 }
328
ufs_qcom_ice_resume(struct ufs_qcom_host * host)329 static inline int ufs_qcom_ice_resume(struct ufs_qcom_host *host)
330 {
331 return 0;
332 }
333
ufs_qcom_ice_suspend(struct ufs_qcom_host * host)334 static inline int ufs_qcom_ice_suspend(struct ufs_qcom_host *host)
335 {
336 return 0;
337 }
338
ufs_qcom_config_ice_allocator(struct ufs_qcom_host * host)339 static void ufs_qcom_config_ice_allocator(struct ufs_qcom_host *host)
340 {
341 }
342
343 #endif
344
ufs_qcom_disable_lane_clks(struct ufs_qcom_host * host)345 static void ufs_qcom_disable_lane_clks(struct ufs_qcom_host *host)
346 {
347 if (!host->is_lane_clks_enabled)
348 return;
349
350 clk_bulk_disable_unprepare(host->num_clks, host->clks);
351
352 host->is_lane_clks_enabled = false;
353 }
354
ufs_qcom_enable_lane_clks(struct ufs_qcom_host * host)355 static int ufs_qcom_enable_lane_clks(struct ufs_qcom_host *host)
356 {
357 int err;
358
359 err = clk_bulk_prepare_enable(host->num_clks, host->clks);
360 if (err)
361 return err;
362
363 host->is_lane_clks_enabled = true;
364
365 return 0;
366 }
367
ufs_qcom_init_lane_clks(struct ufs_qcom_host * host)368 static int ufs_qcom_init_lane_clks(struct ufs_qcom_host *host)
369 {
370 int err;
371 struct device *dev = host->hba->dev;
372
373 if (has_acpi_companion(dev))
374 return 0;
375
376 err = devm_clk_bulk_get_all(dev, &host->clks);
377 if (err <= 0)
378 return err;
379
380 host->num_clks = err;
381
382 return 0;
383 }
384
ufs_qcom_check_hibern8(struct ufs_hba * hba)385 static int ufs_qcom_check_hibern8(struct ufs_hba *hba)
386 {
387 int err;
388 u32 tx_fsm_val;
389 unsigned long timeout = jiffies + msecs_to_jiffies(HBRN8_POLL_TOUT_MS);
390
391 do {
392 err = ufshcd_dme_get(hba,
393 UIC_ARG_MIB_SEL(MPHY_TX_FSM_STATE,
394 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
395 &tx_fsm_val);
396 if (err || tx_fsm_val == TX_FSM_HIBERN8)
397 break;
398
399 /* sleep for max. 200us */
400 usleep_range(100, 200);
401 } while (time_before(jiffies, timeout));
402
403 /*
404 * we might have scheduled out for long during polling so
405 * check the state again.
406 */
407 if (time_after(jiffies, timeout))
408 err = ufshcd_dme_get(hba,
409 UIC_ARG_MIB_SEL(MPHY_TX_FSM_STATE,
410 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
411 &tx_fsm_val);
412
413 if (err) {
414 dev_err(hba->dev, "%s: unable to get TX_FSM_STATE, err %d\n",
415 __func__, err);
416 } else if (tx_fsm_val != TX_FSM_HIBERN8) {
417 err = tx_fsm_val;
418 dev_err(hba->dev, "%s: invalid TX_FSM_STATE = %d\n",
419 __func__, err);
420 }
421
422 return err;
423 }
424
ufs_qcom_select_unipro_mode(struct ufs_qcom_host * host)425 static void ufs_qcom_select_unipro_mode(struct ufs_qcom_host *host)
426 {
427 ufshcd_rmwl(host->hba, QUNIPRO_SEL, QUNIPRO_SEL, REG_UFS_CFG1);
428
429 if (host->hw_ver.major >= 0x05)
430 ufshcd_rmwl(host->hba, QUNIPRO_G4_SEL, 0, REG_UFS_CFG0);
431 }
432
433 /*
434 * ufs_qcom_host_reset - reset host controller and PHY
435 */
ufs_qcom_host_reset(struct ufs_hba * hba)436 static int ufs_qcom_host_reset(struct ufs_hba *hba)
437 {
438 int ret;
439 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
440 bool reenable_intr;
441
442 if (!host->core_reset)
443 return 0;
444
445 reenable_intr = hba->is_irq_enabled;
446 ufshcd_disable_irq(hba);
447
448 ret = reset_control_assert(host->core_reset);
449 if (ret) {
450 dev_err(hba->dev, "%s: core_reset assert failed, err = %d\n",
451 __func__, ret);
452 return ret;
453 }
454
455 /*
456 * The hardware requirement for delay between assert/deassert
457 * is at least 3-4 sleep clock (32.7KHz) cycles, which comes to
458 * ~125us (4/32768). To be on the safe side add 200us delay.
459 */
460 usleep_range(200, 210);
461
462 ret = reset_control_deassert(host->core_reset);
463 if (ret) {
464 dev_err(hba->dev, "%s: core_reset deassert failed, err = %d\n",
465 __func__, ret);
466 return ret;
467 }
468
469 usleep_range(1000, 1100);
470
471 if (reenable_intr)
472 ufshcd_enable_irq(hba);
473
474 return 0;
475 }
476
ufs_qcom_get_hs_gear(struct ufs_hba * hba)477 static u32 ufs_qcom_get_hs_gear(struct ufs_hba *hba)
478 {
479 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
480
481 if (host->hw_ver.major >= 0x4)
482 return UFS_QCOM_MAX_GEAR(ufshcd_readl(hba, REG_UFS_PARAM0));
483
484 /* Default is HS-G3 */
485 return UFS_HS_G3;
486 }
487
ufs_qcom_power_up_sequence(struct ufs_hba * hba)488 static int ufs_qcom_power_up_sequence(struct ufs_hba *hba)
489 {
490 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
491 struct ufs_host_params *host_params = &host->host_params;
492 struct phy *phy = host->generic_phy;
493 enum phy_mode mode;
494 int ret;
495
496 /*
497 * HW ver 5 can only support up to HS-G5 Rate-A due to HW limitations.
498 * If the HS-G5 PHY gear is used, update host_params->hs_rate to Rate-A,
499 * so that the subsequent power mode change shall stick to Rate-A.
500 */
501 if (host->hw_ver.major == 0x5 && host->phy_gear == UFS_HS_G5)
502 host_params->hs_rate = PA_HS_MODE_A;
503
504 mode = host_params->hs_rate == PA_HS_MODE_B ? PHY_MODE_UFS_HS_B : PHY_MODE_UFS_HS_A;
505
506 /* Reset UFS Host Controller and PHY */
507 ret = ufs_qcom_host_reset(hba);
508 if (ret)
509 return ret;
510
511 if (phy->power_count)
512 phy_power_off(phy);
513
514
515 /* phy initialization - calibrate the phy */
516 ret = phy_init(phy);
517 if (ret) {
518 dev_err(hba->dev, "%s: phy init failed, ret = %d\n",
519 __func__, ret);
520 return ret;
521 }
522
523 ret = phy_set_mode_ext(phy, mode, host->phy_gear);
524 if (ret)
525 goto out_disable_phy;
526
527 /* power on phy - start serdes and phy's power and clocks */
528 ret = phy_power_on(phy);
529 if (ret) {
530 dev_err(hba->dev, "%s: phy power on failed, ret = %d\n",
531 __func__, ret);
532 goto out_disable_phy;
533 }
534
535 ret = phy_calibrate(phy);
536 if (ret) {
537 dev_err(hba->dev, "Failed to calibrate PHY: %d\n", ret);
538 goto out_disable_phy;
539 }
540
541 ufs_qcom_select_unipro_mode(host);
542
543 return 0;
544
545 out_disable_phy:
546 phy_exit(phy);
547
548 return ret;
549 }
550
551 /*
552 * The UTP controller has a number of internal clock gating cells (CGCs).
553 * Internal hardware sub-modules within the UTP controller control the CGCs.
554 * Hardware CGCs disable the clock to inactivate UTP sub-modules not involved
555 * in a specific operation, UTP controller CGCs are by default disabled and
556 * this function enables them (after every UFS link startup) to save some power
557 * leakage.
558 */
ufs_qcom_enable_hw_clk_gating(struct ufs_hba * hba)559 static void ufs_qcom_enable_hw_clk_gating(struct ufs_hba *hba)
560 {
561 int err;
562
563 /* Enable UTP internal clock gating */
564 ufshcd_rmwl(hba, REG_UFS_CFG2_CGC_EN_ALL, REG_UFS_CFG2_CGC_EN_ALL,
565 REG_UFS_CFG2);
566
567 /* Ensure that HW clock gating is enabled before next operations */
568 ufshcd_readl(hba, REG_UFS_CFG2);
569
570 /* Enable Unipro internal clock gating */
571 err = ufshcd_dme_rmw(hba, DL_VS_CLK_CFG_MASK,
572 DL_VS_CLK_CFG_MASK, DL_VS_CLK_CFG);
573 if (err)
574 goto out;
575
576 err = ufshcd_dme_rmw(hba, PA_VS_CLK_CFG_REG_MASK,
577 PA_VS_CLK_CFG_REG_MASK, PA_VS_CLK_CFG_REG);
578 if (err)
579 goto out;
580
581 err = ufshcd_dme_rmw(hba, DME_VS_CORE_CLK_CTRL_DME_HW_CGC_EN,
582 DME_VS_CORE_CLK_CTRL_DME_HW_CGC_EN,
583 DME_VS_CORE_CLK_CTRL);
584 out:
585 if (err)
586 dev_err(hba->dev, "hw clk gating enabled failed\n");
587 }
588
ufs_qcom_hce_enable_notify(struct ufs_hba * hba,enum ufs_notify_change_status status)589 static int ufs_qcom_hce_enable_notify(struct ufs_hba *hba,
590 enum ufs_notify_change_status status)
591 {
592 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
593 int err;
594
595 switch (status) {
596 case PRE_CHANGE:
597 err = ufs_qcom_power_up_sequence(hba);
598 if (err)
599 return err;
600
601 /*
602 * The PHY PLL output is the source of tx/rx lane symbol
603 * clocks, hence, enable the lane clocks only after PHY
604 * is initialized.
605 */
606 err = ufs_qcom_enable_lane_clks(host);
607 break;
608 case POST_CHANGE:
609 /* check if UFS PHY moved from DISABLED to HIBERN8 */
610 err = ufs_qcom_check_hibern8(hba);
611 ufs_qcom_enable_hw_clk_gating(hba);
612 ufs_qcom_ice_enable(host);
613 ufs_qcom_config_ice_allocator(host);
614 break;
615 default:
616 dev_err(hba->dev, "%s: invalid status %d\n", __func__, status);
617 err = -EINVAL;
618 break;
619 }
620 return err;
621 }
622
ufs_qcom_fw_managed_hce_enable_notify(struct ufs_hba * hba,enum ufs_notify_change_status status)623 static int ufs_qcom_fw_managed_hce_enable_notify(struct ufs_hba *hba,
624 enum ufs_notify_change_status status)
625 {
626 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
627
628 switch (status) {
629 case PRE_CHANGE:
630 ufs_qcom_select_unipro_mode(host);
631 break;
632 case POST_CHANGE:
633 ufs_qcom_enable_hw_clk_gating(hba);
634 ufs_qcom_ice_enable(host);
635 break;
636 default:
637 dev_err(hba->dev, "Invalid status %d\n", status);
638 return -EINVAL;
639 }
640
641 return 0;
642 }
643
644 /**
645 * ufs_qcom_cfg_timers - Configure ufs qcom cfg timers
646 *
647 * @hba: host controller instance
648 * @is_pre_scale_up: flag to check if pre scale up condition.
649 * @freq: target opp freq
650 * Return: zero for success and non-zero in case of a failure.
651 */
ufs_qcom_cfg_timers(struct ufs_hba * hba,bool is_pre_scale_up,unsigned long freq)652 static int ufs_qcom_cfg_timers(struct ufs_hba *hba, bool is_pre_scale_up, unsigned long freq)
653 {
654 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
655 struct ufs_clk_info *clki;
656 unsigned long clk_freq = 0;
657 u32 core_clk_cycles_per_us;
658
659 /*
660 * UTP controller uses SYS1CLK_1US_REG register for Interrupt
661 * Aggregation logic.
662 * It is mandatory to write SYS1CLK_1US_REG register on UFS host
663 * controller V4.0.0 onwards.
664 */
665 if (host->hw_ver.major < 4 && !ufshcd_is_intr_aggr_allowed(hba))
666 return 0;
667
668 if (hba->use_pm_opp && freq != ULONG_MAX) {
669 clk_freq = ufs_qcom_opp_freq_to_clk_freq(hba, freq, "core_clk");
670 if (clk_freq)
671 goto cfg_timers;
672 }
673
674 list_for_each_entry(clki, &hba->clk_list_head, list) {
675 if (!strcmp(clki->name, "core_clk")) {
676 if (freq == ULONG_MAX) {
677 clk_freq = clki->max_freq;
678 break;
679 }
680
681 if (is_pre_scale_up)
682 clk_freq = clki->max_freq;
683 else
684 clk_freq = clk_get_rate(clki->clk);
685 break;
686 }
687
688 }
689
690 cfg_timers:
691 /* If frequency is smaller than 1MHz, set to 1MHz */
692 if (clk_freq < DEFAULT_CLK_RATE_HZ)
693 clk_freq = DEFAULT_CLK_RATE_HZ;
694
695 core_clk_cycles_per_us = clk_freq / USEC_PER_SEC;
696 if (ufshcd_readl(hba, REG_UFS_SYS1CLK_1US) != core_clk_cycles_per_us) {
697 ufshcd_writel(hba, core_clk_cycles_per_us, REG_UFS_SYS1CLK_1US);
698 /*
699 * make sure above write gets applied before we return from
700 * this function.
701 */
702 ufshcd_readl(hba, REG_UFS_SYS1CLK_1US);
703 }
704
705 return 0;
706 }
707
ufs_qcom_link_startup_post_change(struct ufs_hba * hba)708 static void ufs_qcom_link_startup_post_change(struct ufs_hba *hba)
709 {
710 if (ufshcd_is_auto_hibern8_supported(hba))
711 ufshcd_rmwl(hba, UFS_HW_CLK_CTRL_EN, UFS_HW_CLK_CTRL_EN,
712 UFS_AH8_CFG);
713 }
714
ufs_qcom_link_startup_notify(struct ufs_hba * hba,enum ufs_notify_change_status status)715 static int ufs_qcom_link_startup_notify(struct ufs_hba *hba,
716 enum ufs_notify_change_status status)
717 {
718 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
719 int err = 0;
720
721 switch (status) {
722 case PRE_CHANGE:
723 if (ufs_qcom_cfg_timers(hba, false, ULONG_MAX)) {
724 dev_err(hba->dev, "%s: ufs_qcom_cfg_timers() failed\n",
725 __func__);
726 return -EINVAL;
727 }
728
729 err = ufs_qcom_set_core_clk_ctrl(hba, true, ULONG_MAX);
730 if (err)
731 dev_err(hba->dev, "cfg core clk ctrl failed\n");
732 /*
733 * Some UFS devices (and may be host) have issues if LCC is
734 * enabled. So we are setting PA_Local_TX_LCC_Enable to 0
735 * before link startup which will make sure that both host
736 * and device TX LCC are disabled once link startup is
737 * completed.
738 */
739 err = ufshcd_disable_host_tx_lcc(hba);
740
741 /*
742 * Restore HS/LS link startup mode set by bootloader
743 * after UFS reset clears REG_UFS_DEBUG_SPARE_CFG.
744 */
745 if (host->hw_ver.major > 0x6 ||
746 (host->hw_ver.major == 0x6 && host->hw_ver.minor >= 0x2))
747 ufshcd_writel(hba, host->boot_spare_cfg,
748 REG_UFS_DEBUG_SPARE_CFG);
749 break;
750 case POST_CHANGE:
751 ufs_qcom_link_startup_post_change(hba);
752 break;
753 default:
754 break;
755 }
756
757 return err;
758 }
759
ufs_qcom_device_reset_ctrl(struct ufs_hba * hba,bool asserted)760 static void ufs_qcom_device_reset_ctrl(struct ufs_hba *hba, bool asserted)
761 {
762 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
763
764 /* reset gpio is optional */
765 if (!host->device_reset)
766 return;
767
768 gpiod_set_value_cansleep(host->device_reset, asserted);
769 }
770
ufs_qcom_suspend(struct ufs_hba * hba,enum ufs_pm_op pm_op,enum ufs_notify_change_status status)771 static int ufs_qcom_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op,
772 enum ufs_notify_change_status status)
773 {
774 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
775
776 if (status == PRE_CHANGE)
777 return 0;
778
779 if (!ufs_qcom_is_link_active(hba))
780 ufs_qcom_disable_lane_clks(host);
781
782
783 /* reset the connected UFS device during power down */
784 if (ufs_qcom_is_link_off(hba) && host->device_reset) {
785 ufs_qcom_device_reset_ctrl(hba, true);
786 /*
787 * After sending the SSU command, asserting the rst_n
788 * line causes the device firmware to wake up and
789 * execute its reset routine.
790 *
791 * During this process, the device may draw current
792 * beyond the permissible limit for low-power mode (LPM).
793 * A 10ms delay, based on experimental observations,
794 * allows the UFS device to complete its hardware reset
795 * before transitioning the power rail to LPM.
796 */
797 usleep_range(10000, 11000);
798 }
799
800 return ufs_qcom_ice_suspend(host);
801 }
802
ufs_qcom_resume(struct ufs_hba * hba,enum ufs_pm_op pm_op)803 static int ufs_qcom_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
804 {
805 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
806 int err;
807 u32 reg_val;
808
809 err = ufs_qcom_enable_lane_clks(host);
810 if (err)
811 return err;
812
813 if ((!ufs_qcom_is_link_active(hba)) &&
814 host->hw_ver.major == 5 &&
815 host->hw_ver.minor == 0 &&
816 host->hw_ver.step == 0) {
817 ufshcd_writel(hba, UFS_ICE_SYNC_RST_SEL | UFS_ICE_SYNC_RST_SW, UFS_MEM_ICE_CFG);
818 reg_val = ufshcd_readl(hba, UFS_MEM_ICE_CFG);
819 reg_val &= ~(UFS_ICE_SYNC_RST_SEL | UFS_ICE_SYNC_RST_SW);
820 /*
821 * HW documentation doesn't recommend any delay between the
822 * reset set and clear. But we are enforcing an arbitrary delay
823 * to give flops enough time to settle in.
824 */
825 usleep_range(50, 100);
826 ufshcd_writel(hba, reg_val, UFS_MEM_ICE_CFG);
827 ufshcd_readl(hba, UFS_MEM_ICE_CFG);
828 }
829
830 return ufs_qcom_ice_resume(host);
831 }
832
ufs_qcom_fw_managed_suspend(struct ufs_hba * hba,enum ufs_pm_op pm_op,enum ufs_notify_change_status status)833 static int ufs_qcom_fw_managed_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op,
834 enum ufs_notify_change_status status)
835 {
836 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
837
838 if (status == PRE_CHANGE)
839 return 0;
840
841 pm_runtime_put_sync(hba->dev);
842
843 return ufs_qcom_ice_suspend(host);
844 }
845
ufs_qcom_fw_managed_resume(struct ufs_hba * hba,enum ufs_pm_op pm_op)846 static int ufs_qcom_fw_managed_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
847 {
848 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
849 int err;
850
851 err = pm_runtime_resume_and_get(hba->dev);
852 if (err) {
853 dev_err(hba->dev, "PM runtime resume failed: %d\n", err);
854 return err;
855 }
856
857 return ufs_qcom_ice_resume(host);
858 }
859
ufs_qcom_dev_ref_clk_ctrl(struct ufs_qcom_host * host,bool enable)860 static void ufs_qcom_dev_ref_clk_ctrl(struct ufs_qcom_host *host, bool enable)
861 {
862 if (host->dev_ref_clk_ctrl_mmio &&
863 (enable ^ host->is_dev_ref_clk_enabled)) {
864 u32 temp = readl_relaxed(host->dev_ref_clk_ctrl_mmio);
865
866 if (enable)
867 temp |= host->dev_ref_clk_en_mask;
868 else
869 temp &= ~host->dev_ref_clk_en_mask;
870
871 /*
872 * If we are here to disable this clock it might be immediately
873 * after entering into hibern8 in which case we need to make
874 * sure that device ref_clk is active for specific time after
875 * hibern8 enter.
876 */
877 if (!enable) {
878 unsigned long gating_wait;
879
880 gating_wait = host->hba->dev_info.clk_gating_wait_us;
881 if (!gating_wait) {
882 udelay(1);
883 } else {
884 /*
885 * bRefClkGatingWaitTime defines the minimum
886 * time for which the reference clock is
887 * required by device during transition from
888 * HS-MODE to LS-MODE or HIBERN8 state. Give it
889 * more delay to be on the safe side.
890 */
891 gating_wait += 10;
892 usleep_range(gating_wait, gating_wait + 10);
893 }
894 }
895
896 writel_relaxed(temp, host->dev_ref_clk_ctrl_mmio);
897
898 /*
899 * Make sure the write to ref_clk reaches the destination and
900 * not stored in a Write Buffer (WB).
901 */
902 readl(host->dev_ref_clk_ctrl_mmio);
903
904 /*
905 * If we call hibern8 exit after this, we need to make sure that
906 * device ref_clk is stable for at least 1us before the hibern8
907 * exit command.
908 */
909 if (enable)
910 udelay(1);
911
912 host->is_dev_ref_clk_enabled = enable;
913 }
914 }
915
ufs_qcom_icc_set_bw(struct ufs_qcom_host * host,u32 mem_bw,u32 cfg_bw)916 static int ufs_qcom_icc_set_bw(struct ufs_qcom_host *host, u32 mem_bw, u32 cfg_bw)
917 {
918 struct device *dev = host->hba->dev;
919 int ret;
920
921 ret = icc_set_bw(host->icc_ddr, 0, mem_bw);
922 if (ret < 0) {
923 dev_err(dev, "failed to set bandwidth request: %d\n", ret);
924 return ret;
925 }
926
927 ret = icc_set_bw(host->icc_cpu, 0, cfg_bw);
928 if (ret < 0) {
929 dev_err(dev, "failed to set bandwidth request: %d\n", ret);
930 return ret;
931 }
932
933 return 0;
934 }
935
ufs_qcom_get_bw_table(struct ufs_qcom_host * host)936 static struct __ufs_qcom_bw_table ufs_qcom_get_bw_table(struct ufs_qcom_host *host)
937 {
938 struct ufs_pa_layer_attr *p = &host->dev_req_params;
939 int gear = max_t(u32, p->gear_rx, p->gear_tx);
940 int lane = max_t(u32, p->lane_rx, p->lane_tx);
941
942 if (WARN_ONCE(gear > QCOM_UFS_MAX_GEAR,
943 "ICC scaling for UFS Gear (%d) not supported. Using Gear (%d) bandwidth\n",
944 gear, QCOM_UFS_MAX_GEAR))
945 gear = QCOM_UFS_MAX_GEAR;
946
947 if (WARN_ONCE(lane > QCOM_UFS_MAX_LANE,
948 "ICC scaling for UFS Lane (%d) not supported. Using Lane (%d) bandwidth\n",
949 lane, QCOM_UFS_MAX_LANE))
950 lane = QCOM_UFS_MAX_LANE;
951
952 if (ufshcd_is_hs_mode(p)) {
953 if (p->hs_rate == PA_HS_MODE_B)
954 return ufs_qcom_bw_table[MODE_HS_RB][gear][lane];
955 else
956 return ufs_qcom_bw_table[MODE_HS_RA][gear][lane];
957 } else {
958 return ufs_qcom_bw_table[MODE_PWM][gear][lane];
959 }
960 }
961
ufs_qcom_icc_update_bw(struct ufs_qcom_host * host)962 static int ufs_qcom_icc_update_bw(struct ufs_qcom_host *host)
963 {
964 struct __ufs_qcom_bw_table bw_table;
965
966 bw_table = ufs_qcom_get_bw_table(host);
967
968 return ufs_qcom_icc_set_bw(host, bw_table.mem_bw, bw_table.cfg_bw);
969 }
970
ufs_qcom_set_tx_hs_equalizer(struct ufs_hba * hba,u32 gear,u32 tx_lanes)971 static void ufs_qcom_set_tx_hs_equalizer(struct ufs_hba *hba, u32 gear, u32 tx_lanes)
972 {
973 u32 equalizer_val;
974 int ret, i;
975
976 /* Determine the equalizer value based on the gear */
977 equalizer_val = (gear == 5) ? DEEMPHASIS_3_5_dB : NO_DEEMPHASIS;
978
979 for (i = 0; i < tx_lanes; i++) {
980 ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_HS_EQUALIZER, i),
981 equalizer_val);
982 if (ret)
983 dev_err(hba->dev, "%s: failed equalizer lane %d\n",
984 __func__, i);
985 }
986 }
987
ufs_qcom_negotiate_pwr_mode(struct ufs_hba * hba,const struct ufs_pa_layer_attr * dev_max_params,struct ufs_pa_layer_attr * dev_req_params)988 static int ufs_qcom_negotiate_pwr_mode(struct ufs_hba *hba,
989 const struct ufs_pa_layer_attr *dev_max_params,
990 struct ufs_pa_layer_attr *dev_req_params)
991 {
992 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
993 struct ufs_host_params *host_params = &host->host_params;
994
995 return ufshcd_negotiate_pwr_params(host_params, dev_max_params, dev_req_params);
996 }
997
ufs_qcom_pwr_change_notify(struct ufs_hba * hba,enum ufs_notify_change_status status,struct ufs_pa_layer_attr * dev_req_params)998 static int ufs_qcom_pwr_change_notify(struct ufs_hba *hba,
999 enum ufs_notify_change_status status,
1000 struct ufs_pa_layer_attr *dev_req_params)
1001 {
1002 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1003 int ret = 0;
1004
1005 if (!dev_req_params) {
1006 pr_err("%s: incoming dev_req_params is NULL\n", __func__);
1007 return -EINVAL;
1008 }
1009
1010 switch (status) {
1011 case PRE_CHANGE:
1012 /*
1013 * During UFS driver probe, always update the PHY gear to match the negotiated
1014 * gear, so that, if quirk UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH is enabled,
1015 * the second init can program the optimal PHY settings. This allows one to start
1016 * the first init with either the minimum or the maximum support gear.
1017 */
1018 if (hba->ufshcd_state == UFSHCD_STATE_RESET) {
1019 /*
1020 * Skip REINIT if the negotiated gear matches with the
1021 * initial phy_gear. Otherwise, update the phy_gear to
1022 * program the optimal gear setting during REINIT.
1023 */
1024 if (host->phy_gear == dev_req_params->gear_tx)
1025 hba->quirks &= ~UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH;
1026 else
1027 host->phy_gear = dev_req_params->gear_tx;
1028 }
1029
1030 /* enable the device ref clock before changing to HS mode */
1031 if (!ufshcd_is_hs_mode(&hba->pwr_info) &&
1032 ufshcd_is_hs_mode(dev_req_params))
1033 ufs_qcom_dev_ref_clk_ctrl(host, true);
1034
1035 if (host->hw_ver.major >= 0x4) {
1036 ufshcd_dme_configure_adapt(hba,
1037 dev_req_params->gear_tx,
1038 PA_INITIAL_ADAPT);
1039 }
1040
1041 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TX_DEEMPHASIS_TUNING)
1042 ufs_qcom_set_tx_hs_equalizer(hba,
1043 dev_req_params->gear_tx, dev_req_params->lane_tx);
1044
1045 break;
1046 case POST_CHANGE:
1047 /* cache the power mode parameters to use internally */
1048 memcpy(&host->dev_req_params,
1049 dev_req_params, sizeof(*dev_req_params));
1050
1051 ufs_qcom_icc_update_bw(host);
1052
1053 /* disable the device ref clock if entered PWM mode */
1054 if (ufshcd_is_hs_mode(&hba->pwr_info) &&
1055 !ufshcd_is_hs_mode(dev_req_params))
1056 ufs_qcom_dev_ref_clk_ctrl(host, false);
1057 break;
1058 default:
1059 ret = -EINVAL;
1060 break;
1061 }
1062
1063 return ret;
1064 }
1065
ufs_qcom_quirk_host_pa_saveconfigtime(struct ufs_hba * hba)1066 static int ufs_qcom_quirk_host_pa_saveconfigtime(struct ufs_hba *hba)
1067 {
1068 int err;
1069 u32 pa_vs_config_reg1;
1070
1071 err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_VS_CONFIG_REG1),
1072 &pa_vs_config_reg1);
1073 if (err)
1074 return err;
1075
1076 /* Allow extension of MSB bits of PA_SaveConfigTime attribute */
1077 return ufshcd_dme_set(hba, UIC_ARG_MIB(PA_VS_CONFIG_REG1),
1078 (pa_vs_config_reg1 | (1 << 12)));
1079 }
1080
ufs_qcom_override_pa_tx_hsg1_sync_len(struct ufs_hba * hba)1081 static void ufs_qcom_override_pa_tx_hsg1_sync_len(struct ufs_hba *hba)
1082 {
1083 int err;
1084
1085 err = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TX_HSG1_SYNC_LENGTH),
1086 PA_TX_HSG1_SYNC_LENGTH_VAL);
1087 if (err)
1088 dev_err(hba->dev, "Failed (%d) set PA_TX_HSG1_SYNC_LENGTH\n", err);
1089 }
1090
1091 /**
1092 * ufs_qcom_double_t_adapt_l0l1l2l3 - Create a new adapt that doubles the
1093 * adaptation duration TADAPT_L0_L1_L2_L3 derived from the old adapt.
1094 *
1095 * @old_adapt: Original ADAPT_L0_L1_L2_L3 capability
1096 *
1097 * ADAPT_length_L0_L1_L2_L3 formula from M-PHY spec:
1098 * if (ADAPT_range_L0_L1_L2_L3 == COARSE) {
1099 * ADAPT_length_L0_L1_L2_L3 = [0, 12]
1100 * ADAPT_L0_L1_L2_L3 = 215 x 2^ADAPT_length_L0_L1_L2_L3
1101 * } else if (ADAPT_range_L0_L1_L2_L3 == FINE) {
1102 * ADAPT_length_L0_L1_L2_L3 = [0, 127]
1103 * TADAPT_L0_L1_L2_L3 = 215 x (ADAPT_length_L0_L1_L2_L3 + 1)
1104 * }
1105 *
1106 * To double the adaptation duration TADAPT_L0_L1_L2_L3:
1107 * 1. If adapt range is COARSE (1'b1), new adapt = old adapt + 1.
1108 * 2. If adapt range is FINE (1'b0):
1109 * a) If old adapt length is < 64, (new adapt + 1) = 2 * (old adapt + 1).
1110 * b) If old adapt length is >= 64, set new adapt to 0x88 using COARSE
1111 * range, because new adapt get from equation in a) shall exceed 127.
1112 *
1113 * Examples:
1114 * ADAPT_range_L0_L1_L2_L3 | ADAPT_length_L0_L1_L2_L3 | TADAPT_L0_L1_L2_L3 (PAM-4 UI)
1115 * 0 3 131072
1116 * 0 7 262144
1117 * 0 63 2097152
1118 * 0 64 2129920
1119 * 0 127 4194304
1120 * 1 8 8388608
1121 * 1 9 16777216
1122 * 1 10 33554432
1123 * 1 11 67108864
1124 * 1 12 134217728
1125 *
1126 * Return: new adapt.
1127 */
ufs_qcom_double_t_adapt_l0l1l2l3(u32 old_adapt)1128 static u32 ufs_qcom_double_t_adapt_l0l1l2l3(u32 old_adapt)
1129 {
1130 u32 adapt_length = old_adapt & ADAPT_LENGTH_MASK;
1131 u32 new_adapt;
1132
1133 if (IS_ADAPT_RANGE_COARSE(old_adapt)) {
1134 new_adapt = (adapt_length + 1) | ADAPT_RANGE_BIT;
1135 } else {
1136 if (adapt_length < 64)
1137 new_adapt = (adapt_length << 1) + 1;
1138 else
1139 /*
1140 * 0x88 is the very coarse Adapt value which is two
1141 * times of the largest fine Adapt value (0x7F)
1142 */
1143 new_adapt = 0x88;
1144 }
1145
1146 return new_adapt;
1147 }
1148
ufs_qcom_limit_max_gear(struct ufs_hba * hba,enum ufs_hs_gear_tag gear)1149 static void ufs_qcom_limit_max_gear(struct ufs_hba *hba,
1150 enum ufs_hs_gear_tag gear)
1151 {
1152 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1153 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
1154 struct ufs_host_params *host_params = &host->host_params;
1155
1156 host_params->hs_tx_gear = gear;
1157 host_params->hs_rx_gear = gear;
1158 pwr_info->gear_tx = gear;
1159 pwr_info->gear_rx = gear;
1160
1161 dev_warn(hba->dev, "Limited max gear of host and device to HS-G%d\n", gear);
1162 }
1163
ufs_qcom_fixup_tx_adapt_l0l1l2l3(struct ufs_hba * hba)1164 static void ufs_qcom_fixup_tx_adapt_l0l1l2l3(struct ufs_hba *hba)
1165 {
1166 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1167 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
1168 struct ufs_host_params *host_params = &host->host_params;
1169 u32 old_adapt, new_adapt, actual_adapt;
1170 bool limit_speed = false;
1171 int err;
1172
1173 if (host->hw_ver.major != 0x7 || host->hw_ver.minor > 0x1 ||
1174 host_params->hs_tx_gear <= UFS_HS_G5 ||
1175 pwr_info->gear_tx <= UFS_HS_G5)
1176 return;
1177
1178 err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTINITIALL0L1L2L3), &old_adapt);
1179 if (err)
1180 goto out;
1181
1182 if (old_adapt > ADAPT_L0L1L2L3_LENGTH_MAX) {
1183 dev_err(hba->dev, "PA_PeerRxHsG6AdaptInitialL0L1L2L3 value (0x%x) exceeds MAX\n",
1184 old_adapt);
1185 err = -ERANGE;
1186 goto out;
1187 }
1188
1189 new_adapt = ufs_qcom_double_t_adapt_l0l1l2l3(old_adapt);
1190 dev_dbg(hba->dev, "Original PA_PeerRxHsG6AdaptInitialL0L1L2L3 = 0x%x, new value = 0x%x\n",
1191 old_adapt, new_adapt);
1192
1193 /*
1194 * 0x8C is the max possible value allowed by UniPro v3.0 spec, some HWs
1195 * can accept 0x8D but some cannot.
1196 */
1197 if (new_adapt <= ADAPT_L0L1L2L3_LENGTH_MAX ||
1198 (new_adapt == ADAPT_L0L1L2L3_LENGTH_MAX + 1 && host->hw_ver.minor == 0x1)) {
1199 err = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTINITIALL0L1L2L3),
1200 new_adapt);
1201 if (err)
1202 goto out;
1203
1204 err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTINITIALL0L1L2L3),
1205 &actual_adapt);
1206 if (err)
1207 goto out;
1208
1209 if (actual_adapt != new_adapt) {
1210 limit_speed = true;
1211 dev_warn(hba->dev, "PA_PeerRxHsG6AdaptInitialL0L1L2L3 0x%x, expect 0x%x\n",
1212 actual_adapt, new_adapt);
1213 }
1214 } else {
1215 limit_speed = true;
1216 dev_warn(hba->dev, "New PA_PeerRxHsG6AdaptInitialL0L1L2L3 (0x%x) is too large!\n",
1217 new_adapt);
1218 }
1219
1220 err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTREFRESHL0L1L2L3), &old_adapt);
1221 if (err)
1222 goto out;
1223
1224 if (old_adapt > ADAPT_L0L1L2L3_LENGTH_MAX) {
1225 dev_err(hba->dev, "PA_PeerRxHsG6AdaptRefreshL0L1L2L3 value (0x%x) exceeds MAX\n",
1226 old_adapt);
1227 err = -ERANGE;
1228 goto out;
1229 }
1230
1231 new_adapt = ufs_qcom_double_t_adapt_l0l1l2l3(old_adapt);
1232 dev_dbg(hba->dev, "Original PA_PeerRxHsG6AdaptRefreshL0L1L2L3 = 0x%x, new value = 0x%x\n",
1233 old_adapt, new_adapt);
1234
1235 /*
1236 * 0x8C is the max possible value allowed by UniPro v3.0 spec, some HWs
1237 * can accept 0x8D but some cannot.
1238 */
1239 if (new_adapt <= ADAPT_L0L1L2L3_LENGTH_MAX ||
1240 (new_adapt == ADAPT_L0L1L2L3_LENGTH_MAX + 1 && host->hw_ver.minor == 0x1)) {
1241 err = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTREFRESHL0L1L2L3),
1242 new_adapt);
1243 if (err)
1244 goto out;
1245
1246 err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTREFRESHL0L1L2L3),
1247 &actual_adapt);
1248 if (err)
1249 goto out;
1250
1251 if (actual_adapt != new_adapt) {
1252 limit_speed = true;
1253 dev_warn(hba->dev, "PA_PeerRxHsG6AdaptRefreshL0L1L2L3 0x%x, expect 0x%x\n",
1254 new_adapt, actual_adapt);
1255 }
1256 } else {
1257 limit_speed = true;
1258 dev_warn(hba->dev, "New PA_PeerRxHsG6AdaptRefreshL0L1L2L3 (0x%x) is too large!\n",
1259 new_adapt);
1260 }
1261
1262 out:
1263 if (limit_speed || err)
1264 ufs_qcom_limit_max_gear(hba, UFS_HS_G5);
1265 }
1266
ufs_qcom_apply_dev_quirks(struct ufs_hba * hba)1267 static int ufs_qcom_apply_dev_quirks(struct ufs_hba *hba)
1268 {
1269 int err = 0;
1270
1271 ufs_qcom_fixup_tx_adapt_l0l1l2l3(hba);
1272
1273 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME)
1274 err = ufs_qcom_quirk_host_pa_saveconfigtime(hba);
1275
1276 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TX_HSG1_SYNC_LENGTH)
1277 ufs_qcom_override_pa_tx_hsg1_sync_len(hba);
1278
1279 return err;
1280 }
1281
1282 /* UFS device-specific quirks */
1283 static struct ufs_dev_quirk ufs_qcom_dev_fixups[] = {
1284 { .wmanufacturerid = UFS_VENDOR_SKHYNIX,
1285 .model = UFS_ANY_MODEL,
1286 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM },
1287 { .wmanufacturerid = UFS_VENDOR_WDC,
1288 .model = UFS_ANY_MODEL,
1289 .quirk = UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE },
1290 { .wmanufacturerid = UFS_VENDOR_SAMSUNG,
1291 .model = UFS_ANY_MODEL,
1292 .quirk = UFS_DEVICE_QUIRK_PA_TX_HSG1_SYNC_LENGTH |
1293 UFS_DEVICE_QUIRK_PA_TX_DEEMPHASIS_TUNING },
1294 {}
1295 };
1296
ufs_qcom_fixup_dev_quirks(struct ufs_hba * hba)1297 static void ufs_qcom_fixup_dev_quirks(struct ufs_hba *hba)
1298 {
1299 ufshcd_fixup_dev_quirks(hba, ufs_qcom_dev_fixups);
1300 }
1301
ufs_qcom_get_ufs_hci_version(struct ufs_hba * hba)1302 static u32 ufs_qcom_get_ufs_hci_version(struct ufs_hba *hba)
1303 {
1304 return ufshci_version(2, 0);
1305 }
1306
1307 /**
1308 * ufs_qcom_advertise_quirks - advertise the known QCOM UFS controller quirks
1309 * @hba: host controller instance
1310 *
1311 * QCOM UFS host controller might have some non standard behaviours (quirks)
1312 * than what is specified by UFSHCI specification. Advertise all such
1313 * quirks to standard UFS host controller driver so standard takes them into
1314 * account.
1315 */
ufs_qcom_advertise_quirks(struct ufs_hba * hba)1316 static void ufs_qcom_advertise_quirks(struct ufs_hba *hba)
1317 {
1318 const struct ufs_qcom_drvdata *drvdata = of_device_get_match_data(hba->dev);
1319 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1320
1321 if (host->hw_ver.major == 0x2)
1322 hba->quirks |= UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION;
1323
1324 if (host->hw_ver.major > 0x3)
1325 hba->quirks |= UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH;
1326
1327 if (host->hw_ver.major == 0x7 && host->hw_ver.minor == 0x1)
1328 hba->quirks |= UFSHCD_QUIRK_EXTENDED_TX_EQTR_ADAPT_LENGTH_L0L1L2L3;
1329
1330 if (drvdata && drvdata->quirks)
1331 hba->quirks |= drvdata->quirks;
1332 }
1333
ufs_qcom_set_phy_gear(struct ufs_qcom_host * host)1334 static void ufs_qcom_set_phy_gear(struct ufs_qcom_host *host)
1335 {
1336 struct ufs_host_params *host_params = &host->host_params;
1337 u32 dev_major;
1338
1339 /*
1340 * Default to powering up the PHY to the max gear possible, which is
1341 * backwards compatible with lower gears but not optimal from
1342 * a power usage point of view. After device negotiation, if the
1343 * gear is lower a reinit will be performed to program the PHY
1344 * to the ideal gear for this combo of controller and device.
1345 */
1346 host->phy_gear = host_params->hs_tx_gear;
1347
1348 if (host->hw_ver.major < 0x4) {
1349 /*
1350 * These controllers only have one PHY init sequence,
1351 * let's power up the PHY using that (the minimum supported
1352 * gear, UFS_HS_G2).
1353 */
1354 host->phy_gear = UFS_HS_G2;
1355 } else if (host->hw_ver.major >= 0x5) {
1356 host->boot_spare_cfg = ufshcd_readl(host->hba, REG_UFS_DEBUG_SPARE_CFG);
1357 dev_major = FIELD_GET(UFS_DEV_VER_MAJOR_MASK, host->boot_spare_cfg);
1358
1359 /*
1360 * Since the UFS device version is populated, let's remove the
1361 * REINIT quirk as the negotiated gear won't change during boot.
1362 * So there is no need to do reinit.
1363 */
1364 if (dev_major != 0x0)
1365 host->hba->quirks &= ~UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH;
1366
1367 /*
1368 * For UFS 3.1 device and older, power up the PHY using HS-G4
1369 * PHY gear to save power.
1370 */
1371 if (dev_major > 0x0 && dev_major < 0x4)
1372 host->phy_gear = UFS_HS_G4;
1373 }
1374 }
1375
ufs_qcom_parse_gear_limits(struct ufs_hba * hba)1376 static void ufs_qcom_parse_gear_limits(struct ufs_hba *hba)
1377 {
1378 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1379 struct ufs_host_params *host_params = &host->host_params;
1380 u32 hs_gear_old = host_params->hs_tx_gear;
1381
1382 ufshcd_parse_gear_limits(hba, host_params);
1383 if (host_params->hs_tx_gear != hs_gear_old) {
1384 host->phy_gear = host_params->hs_tx_gear;
1385 }
1386 }
1387
ufs_qcom_set_host_params(struct ufs_hba * hba)1388 static void ufs_qcom_set_host_params(struct ufs_hba *hba)
1389 {
1390 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1391 struct ufs_host_params *host_params = &host->host_params;
1392
1393 ufshcd_init_host_params(host_params);
1394
1395 /* This driver only supports symmetic gear setting i.e., hs_tx_gear == hs_rx_gear */
1396 host_params->hs_tx_gear = host_params->hs_rx_gear = ufs_qcom_get_hs_gear(hba);
1397 }
1398
ufs_qcom_set_host_caps(struct ufs_hba * hba)1399 static void ufs_qcom_set_host_caps(struct ufs_hba *hba)
1400 {
1401 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1402
1403 if (host->hw_ver.major >= 0x5)
1404 host->caps |= UFS_QCOM_CAP_ICE_CONFIG;
1405 }
1406
ufs_qcom_set_caps(struct ufs_hba * hba)1407 static void ufs_qcom_set_caps(struct ufs_hba *hba)
1408 {
1409 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1410
1411 hba->caps |= UFSHCD_CAP_CLK_GATING | UFSHCD_CAP_HIBERN8_WITH_CLK_GATING;
1412 hba->caps |= UFSHCD_CAP_CLK_SCALING | UFSHCD_CAP_WB_WITH_CLK_SCALING;
1413 hba->caps |= UFSHCD_CAP_AUTO_BKOPS_SUSPEND;
1414 hba->caps |= UFSHCD_CAP_WB_EN;
1415 hba->caps |= UFSHCD_CAP_AGGR_POWER_COLLAPSE;
1416 hba->caps |= UFSHCD_CAP_RPM_AUTOSUSPEND;
1417
1418 if (host->hw_ver.major >= 0x7)
1419 hba->caps |= UFSHCD_CAP_TX_EQUALIZATION;
1420
1421 ufs_qcom_set_host_caps(hba);
1422 }
1423
1424 /**
1425 * ufs_qcom_setup_clocks - enables/disable clocks
1426 * @hba: host controller instance
1427 * @on: If true, enable clocks else disable them.
1428 * @status: PRE_CHANGE or POST_CHANGE notify
1429 *
1430 * There are certain clocks which comes from the PHY so it needs
1431 * to be managed together along with controller clocks which also
1432 * provides a better power saving. Hence keep phy_power_off/on calls
1433 * in ufs_qcom_setup_clocks, so that PHY's regulators & clks can be
1434 * turned on/off along with UFS's clocks.
1435 *
1436 * Return: 0 on success, non-zero on failure.
1437 */
ufs_qcom_setup_clocks(struct ufs_hba * hba,bool on,enum ufs_notify_change_status status)1438 static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on,
1439 enum ufs_notify_change_status status)
1440 {
1441 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1442 struct phy *phy;
1443 int err;
1444
1445 /*
1446 * In case ufs_qcom_init() is not yet done, simply ignore.
1447 * This ufs_qcom_setup_clocks() shall be called from
1448 * ufs_qcom_init() after init is done.
1449 */
1450 if (!host)
1451 return 0;
1452
1453 phy = host->generic_phy;
1454
1455 switch (status) {
1456 case PRE_CHANGE:
1457 if (on) {
1458 ufs_qcom_icc_update_bw(host);
1459 if (ufs_qcom_is_link_hibern8(hba)) {
1460 err = ufs_qcom_enable_lane_clks(host);
1461 if (err) {
1462 dev_err(hba->dev, "enable lane clks failed, ret=%d\n", err);
1463 return err;
1464 }
1465 }
1466 } else {
1467 if (!ufs_qcom_is_link_active(hba)) {
1468 /* disable device ref_clk */
1469 ufs_qcom_dev_ref_clk_ctrl(host, false);
1470 }
1471
1472 err = phy_power_off(phy);
1473 if (err) {
1474 dev_err(hba->dev, "phy power off failed, ret=%d\n", err);
1475 return err;
1476 }
1477 }
1478 break;
1479 case POST_CHANGE:
1480 if (on) {
1481 err = phy_power_on(phy);
1482 if (err) {
1483 dev_err(hba->dev, "phy power on failed, ret = %d\n", err);
1484 return err;
1485 }
1486
1487 /* enable the device ref clock for HS mode*/
1488 if (ufshcd_is_hs_mode(&hba->pwr_info))
1489 ufs_qcom_dev_ref_clk_ctrl(host, true);
1490 } else {
1491 if (ufs_qcom_is_link_hibern8(hba))
1492 ufs_qcom_disable_lane_clks(host);
1493
1494 ufs_qcom_icc_set_bw(host, ufs_qcom_bw_table[MODE_MIN][0][0].mem_bw,
1495 ufs_qcom_bw_table[MODE_MIN][0][0].cfg_bw);
1496 }
1497 break;
1498 }
1499
1500 return 0;
1501 }
1502
1503 static int
ufs_qcom_reset_assert(struct reset_controller_dev * rcdev,unsigned long id)1504 ufs_qcom_reset_assert(struct reset_controller_dev *rcdev, unsigned long id)
1505 {
1506 struct ufs_qcom_host *host = rcdev_to_ufs_host(rcdev);
1507
1508 ufs_qcom_assert_reset(host->hba);
1509 /* provide 1ms delay to let the reset pulse propagate. */
1510 usleep_range(1000, 1100);
1511 return 0;
1512 }
1513
1514 static int
ufs_qcom_reset_deassert(struct reset_controller_dev * rcdev,unsigned long id)1515 ufs_qcom_reset_deassert(struct reset_controller_dev *rcdev, unsigned long id)
1516 {
1517 struct ufs_qcom_host *host = rcdev_to_ufs_host(rcdev);
1518
1519 ufs_qcom_deassert_reset(host->hba);
1520
1521 /*
1522 * after reset deassertion, phy will need all ref clocks,
1523 * voltage, current to settle down before starting serdes.
1524 */
1525 usleep_range(1000, 1100);
1526 return 0;
1527 }
1528
1529 static const struct reset_control_ops ufs_qcom_reset_ops = {
1530 .assert = ufs_qcom_reset_assert,
1531 .deassert = ufs_qcom_reset_deassert,
1532 };
1533
ufs_qcom_icc_init(struct ufs_qcom_host * host)1534 static int ufs_qcom_icc_init(struct ufs_qcom_host *host)
1535 {
1536 struct device *dev = host->hba->dev;
1537 int ret;
1538
1539 host->icc_ddr = devm_of_icc_get(dev, "ufs-ddr");
1540 if (IS_ERR(host->icc_ddr))
1541 return dev_err_probe(dev, PTR_ERR(host->icc_ddr),
1542 "failed to acquire interconnect path\n");
1543
1544 host->icc_cpu = devm_of_icc_get(dev, "cpu-ufs");
1545 if (IS_ERR(host->icc_cpu))
1546 return dev_err_probe(dev, PTR_ERR(host->icc_cpu),
1547 "failed to acquire interconnect path\n");
1548
1549 /*
1550 * Set Maximum bandwidth vote before initializing the UFS controller and
1551 * device. Ideally, a minimal interconnect vote would suffice for the
1552 * initialization, but a max vote would allow faster initialization.
1553 */
1554 ret = ufs_qcom_icc_set_bw(host, ufs_qcom_bw_table[MODE_MAX][0][0].mem_bw,
1555 ufs_qcom_bw_table[MODE_MAX][0][0].cfg_bw);
1556 if (ret < 0)
1557 return dev_err_probe(dev, ret, "failed to set bandwidth request\n");
1558
1559 return 0;
1560 }
1561
1562 /**
1563 * ufs_qcom_init - bind phy with controller
1564 * @hba: host controller instance
1565 *
1566 * Binds PHY with controller and powers up PHY enabling clocks
1567 * and regulators.
1568 *
1569 * Return: -EPROBE_DEFER if binding fails, returns negative error
1570 * on phy power up failure and returns zero on success.
1571 */
ufs_qcom_init(struct ufs_hba * hba)1572 static int ufs_qcom_init(struct ufs_hba *hba)
1573 {
1574 int err;
1575 struct device *dev = hba->dev;
1576 struct ufs_qcom_host *host;
1577 struct ufs_clk_info *clki;
1578 const struct ufs_qcom_drvdata *drvdata = of_device_get_match_data(hba->dev);
1579
1580 host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
1581 if (!host)
1582 return -ENOMEM;
1583
1584 /* Make a two way bind between the qcom host and the hba */
1585 host->hba = hba;
1586 ufshcd_set_variant(hba, host);
1587
1588 /* Setup the optional reset control of HCI */
1589 host->core_reset = devm_reset_control_get_optional(hba->dev, "rst");
1590 if (IS_ERR(host->core_reset)) {
1591 err = dev_err_probe(dev, PTR_ERR(host->core_reset),
1592 "Failed to get reset control\n");
1593 goto out_variant_clear;
1594 }
1595
1596 /* Fire up the reset controller. Failure here is non-fatal. */
1597 host->rcdev.of_node = dev->of_node;
1598 host->rcdev.ops = &ufs_qcom_reset_ops;
1599 host->rcdev.owner = dev->driver->owner;
1600 host->rcdev.nr_resets = 1;
1601 err = devm_reset_controller_register(dev, &host->rcdev);
1602 if (err)
1603 dev_warn(dev, "Failed to register reset controller\n");
1604
1605 if (!has_acpi_companion(dev)) {
1606 host->generic_phy = devm_phy_get(dev, "ufsphy");
1607 if (IS_ERR(host->generic_phy)) {
1608 err = dev_err_probe(dev, PTR_ERR(host->generic_phy), "Failed to get PHY\n");
1609 goto out_variant_clear;
1610 }
1611 }
1612
1613 err = ufs_qcom_icc_init(host);
1614 if (err)
1615 goto out_variant_clear;
1616
1617 host->device_reset = devm_gpiod_get_optional(dev, "reset",
1618 GPIOD_OUT_HIGH);
1619 if (IS_ERR(host->device_reset)) {
1620 err = dev_err_probe(dev, PTR_ERR(host->device_reset),
1621 "Failed to acquire device reset gpio\n");
1622 goto out_variant_clear;
1623 }
1624
1625 ufs_qcom_get_controller_revision(hba, &host->hw_ver.major,
1626 &host->hw_ver.minor, &host->hw_ver.step);
1627
1628 host->dev_ref_clk_ctrl_mmio = hba->mmio_base + REG_UFS_CFG1;
1629 host->dev_ref_clk_en_mask = BIT(26);
1630
1631 list_for_each_entry(clki, &hba->clk_list_head, list) {
1632 if (!strcmp(clki->name, "core_clk_unipro"))
1633 clki->keep_link_active = true;
1634 }
1635
1636 err = ufs_qcom_init_lane_clks(host);
1637 if (err)
1638 goto out_variant_clear;
1639
1640 ufs_qcom_set_caps(hba);
1641 ufs_qcom_advertise_quirks(hba);
1642 ufs_qcom_set_host_params(hba);
1643 ufs_qcom_set_phy_gear(host);
1644 ufs_qcom_parse_gear_limits(hba);
1645
1646 err = ufs_qcom_ice_init(host);
1647 if (err)
1648 goto out_variant_clear;
1649
1650 ufs_qcom_setup_clocks(hba, true, POST_CHANGE);
1651
1652 ufs_qcom_get_default_testbus_cfg(host);
1653 err = ufs_qcom_testbus_config(host);
1654 if (err)
1655 /* Failure is non-fatal */
1656 dev_warn(dev, "%s: failed to configure the testbus %d\n",
1657 __func__, err);
1658
1659 if (drvdata && drvdata->no_phy_retention)
1660 hba->spm_lvl = UFS_PM_LVL_5;
1661
1662 return 0;
1663
1664 out_variant_clear:
1665 ufshcd_set_variant(hba, NULL);
1666
1667 return err;
1668 }
1669
ufs_qcom_exit(struct ufs_hba * hba)1670 static void ufs_qcom_exit(struct ufs_hba *hba)
1671 {
1672 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1673
1674 ufs_qcom_disable_lane_clks(host);
1675 phy_power_off(host->generic_phy);
1676 phy_exit(host->generic_phy);
1677 }
1678
ufs_qcom_fw_managed_init(struct ufs_hba * hba)1679 static int ufs_qcom_fw_managed_init(struct ufs_hba *hba)
1680 {
1681 struct device *dev = hba->dev;
1682 struct ufs_qcom_host *host;
1683 int err;
1684
1685 host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
1686 if (!host)
1687 return -ENOMEM;
1688
1689 host->hba = hba;
1690 ufshcd_set_variant(hba, host);
1691
1692 ufs_qcom_get_controller_revision(hba, &host->hw_ver.major,
1693 &host->hw_ver.minor, &host->hw_ver.step);
1694
1695 err = ufs_qcom_ice_init(host);
1696 if (err)
1697 goto out_variant_clear;
1698
1699 ufs_qcom_get_default_testbus_cfg(host);
1700 err = ufs_qcom_testbus_config(host);
1701 if (err)
1702 /* Failure is non-fatal */
1703 dev_warn(dev, "Failed to configure the testbus %d\n", err);
1704
1705 hba->caps |= UFSHCD_CAP_WB_EN;
1706
1707 ufs_qcom_advertise_quirks(hba);
1708 host->hba->quirks &= ~UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH;
1709
1710 hba->spm_lvl = hba->rpm_lvl = hba->pm_lvl_min = UFS_PM_LVL_5;
1711
1712 ufs_qcom_set_host_params(hba);
1713 ufs_qcom_parse_gear_limits(hba);
1714
1715 return 0;
1716
1717 out_variant_clear:
1718 ufshcd_set_variant(hba, NULL);
1719 return err;
1720 }
1721
ufs_qcom_fw_managed_exit(struct ufs_hba * hba)1722 static void ufs_qcom_fw_managed_exit(struct ufs_hba *hba)
1723 {
1724 pm_runtime_put_sync(hba->dev);
1725 }
1726
1727 /**
1728 * ufs_qcom_set_clk_40ns_cycles - Configure 40ns clk cycles
1729 *
1730 * @hba: host controller instance
1731 * @cycles_in_1us: No of cycles in 1us to be configured
1732 *
1733 * Returns error if dme get/set configuration for 40ns fails
1734 * and returns zero on success.
1735 */
ufs_qcom_set_clk_40ns_cycles(struct ufs_hba * hba,u32 cycles_in_1us)1736 static int ufs_qcom_set_clk_40ns_cycles(struct ufs_hba *hba,
1737 u32 cycles_in_1us)
1738 {
1739 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1740 u32 cycles_in_40ns;
1741 u32 reg;
1742 int err;
1743
1744 /*
1745 * UFS host controller V4.0.0 onwards needs to program
1746 * PA_VS_CORE_CLK_40NS_CYCLES attribute per programmed
1747 * frequency of unipro core clk of UFS host controller.
1748 */
1749 if (host->hw_ver.major < 4)
1750 return 0;
1751
1752 /*
1753 * Generic formulae for cycles_in_40ns = (freq_unipro/25) is not
1754 * applicable for all frequencies. For ex: ceil(37.5 MHz/25) will
1755 * be 2 and ceil(403 MHZ/25) will be 17 whereas Hardware
1756 * specification expect to be 16. Hence use exact hardware spec
1757 * mandated value for cycles_in_40ns instead of calculating using
1758 * generic formulae.
1759 */
1760 switch (cycles_in_1us) {
1761 case UNIPRO_CORE_CLK_FREQ_403_MHZ:
1762 cycles_in_40ns = 16;
1763 break;
1764 case UNIPRO_CORE_CLK_FREQ_300_MHZ:
1765 cycles_in_40ns = 12;
1766 break;
1767 case UNIPRO_CORE_CLK_FREQ_201_5_MHZ:
1768 cycles_in_40ns = 8;
1769 break;
1770 case UNIPRO_CORE_CLK_FREQ_150_MHZ:
1771 cycles_in_40ns = 6;
1772 break;
1773 case UNIPRO_CORE_CLK_FREQ_100_MHZ:
1774 cycles_in_40ns = 4;
1775 break;
1776 case UNIPRO_CORE_CLK_FREQ_75_MHZ:
1777 cycles_in_40ns = 3;
1778 break;
1779 case UNIPRO_CORE_CLK_FREQ_37_5_MHZ:
1780 cycles_in_40ns = 2;
1781 break;
1782 default:
1783 dev_err(hba->dev, "UNIPRO clk freq %u MHz not supported\n",
1784 cycles_in_1us);
1785 return -EINVAL;
1786 }
1787
1788 err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_VS_CORE_CLK_40NS_CYCLES), ®);
1789 if (err)
1790 return err;
1791
1792 reg &= ~PA_VS_CORE_CLK_40NS_CYCLES_MASK;
1793 reg |= cycles_in_40ns;
1794
1795 return ufshcd_dme_set(hba, UIC_ARG_MIB(PA_VS_CORE_CLK_40NS_CYCLES), reg);
1796 }
1797
ufs_qcom_set_core_clk_ctrl(struct ufs_hba * hba,bool is_scale_up,unsigned long freq)1798 static int ufs_qcom_set_core_clk_ctrl(struct ufs_hba *hba, bool is_scale_up, unsigned long freq)
1799 {
1800 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1801 struct list_head *head = &hba->clk_list_head;
1802 struct ufs_clk_info *clki;
1803 u32 cycles_in_1us = 0;
1804 u32 core_clk_ctrl_reg;
1805 unsigned long clk_freq;
1806 int err;
1807
1808 if (hba->use_pm_opp && freq != ULONG_MAX) {
1809 clk_freq = ufs_qcom_opp_freq_to_clk_freq(hba, freq, "core_clk_unipro");
1810 if (clk_freq) {
1811 cycles_in_1us = ceil(clk_freq, HZ_PER_MHZ);
1812 goto set_core_clk_ctrl;
1813 }
1814 }
1815
1816 list_for_each_entry(clki, head, list) {
1817 if (!IS_ERR_OR_NULL(clki->clk) &&
1818 !strcmp(clki->name, "core_clk_unipro")) {
1819 if (!clki->max_freq) {
1820 cycles_in_1us = 150; /* default for backwards compatibility */
1821 break;
1822 }
1823
1824 if (freq == ULONG_MAX) {
1825 cycles_in_1us = ceil(clki->max_freq, HZ_PER_MHZ);
1826 break;
1827 }
1828
1829 if (is_scale_up)
1830 cycles_in_1us = ceil(clki->max_freq, HZ_PER_MHZ);
1831 else
1832 cycles_in_1us = ceil(clk_get_rate(clki->clk), HZ_PER_MHZ);
1833 break;
1834 }
1835 }
1836
1837 set_core_clk_ctrl:
1838 err = ufshcd_dme_get(hba,
1839 UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL),
1840 &core_clk_ctrl_reg);
1841 if (err)
1842 return err;
1843
1844 /* Bit mask is different for UFS host controller V4.0.0 onwards */
1845 if (host->hw_ver.major >= 4) {
1846 if (!FIELD_FIT(CLK_1US_CYCLES_MASK_V4, cycles_in_1us))
1847 return -ERANGE;
1848 core_clk_ctrl_reg &= ~CLK_1US_CYCLES_MASK_V4;
1849 core_clk_ctrl_reg |= FIELD_PREP(CLK_1US_CYCLES_MASK_V4, cycles_in_1us);
1850 } else {
1851 if (!FIELD_FIT(CLK_1US_CYCLES_MASK, cycles_in_1us))
1852 return -ERANGE;
1853 core_clk_ctrl_reg &= ~CLK_1US_CYCLES_MASK;
1854 core_clk_ctrl_reg |= FIELD_PREP(CLK_1US_CYCLES_MASK, cycles_in_1us);
1855 }
1856
1857 /* Clear CORE_CLK_DIV_EN */
1858 core_clk_ctrl_reg &= ~DME_VS_CORE_CLK_CTRL_CORE_CLK_DIV_EN_BIT;
1859
1860 err = ufshcd_dme_set(hba,
1861 UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL),
1862 core_clk_ctrl_reg);
1863 if (err)
1864 return err;
1865
1866 /* Configure unipro core clk 40ns attribute */
1867 return ufs_qcom_set_clk_40ns_cycles(hba, cycles_in_1us);
1868 }
1869
ufs_qcom_clk_scale_up_pre_change(struct ufs_hba * hba,unsigned long freq)1870 static int ufs_qcom_clk_scale_up_pre_change(struct ufs_hba *hba, unsigned long freq)
1871 {
1872 int ret;
1873
1874 ret = ufs_qcom_cfg_timers(hba, true, freq);
1875 if (ret) {
1876 dev_err(hba->dev, "%s ufs cfg timer failed\n", __func__);
1877 return ret;
1878 }
1879 /* set unipro core clock attributes and clear clock divider */
1880 return ufs_qcom_set_core_clk_ctrl(hba, true, freq);
1881 }
1882
ufs_qcom_clk_scale_up_post_change(struct ufs_hba * hba)1883 static int ufs_qcom_clk_scale_up_post_change(struct ufs_hba *hba)
1884 {
1885 return 0;
1886 }
1887
ufs_qcom_clk_scale_down_pre_change(struct ufs_hba * hba)1888 static int ufs_qcom_clk_scale_down_pre_change(struct ufs_hba *hba)
1889 {
1890 int err;
1891 u32 core_clk_ctrl_reg;
1892
1893 err = ufshcd_dme_get(hba,
1894 UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL),
1895 &core_clk_ctrl_reg);
1896
1897 /* make sure CORE_CLK_DIV_EN is cleared */
1898 if (!err &&
1899 (core_clk_ctrl_reg & DME_VS_CORE_CLK_CTRL_CORE_CLK_DIV_EN_BIT)) {
1900 core_clk_ctrl_reg &= ~DME_VS_CORE_CLK_CTRL_CORE_CLK_DIV_EN_BIT;
1901 err = ufshcd_dme_set(hba,
1902 UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL),
1903 core_clk_ctrl_reg);
1904 }
1905
1906 return err;
1907 }
1908
ufs_qcom_clk_scale_down_post_change(struct ufs_hba * hba,unsigned long freq)1909 static int ufs_qcom_clk_scale_down_post_change(struct ufs_hba *hba, unsigned long freq)
1910 {
1911 int ret;
1912
1913 ret = ufs_qcom_cfg_timers(hba, false, freq);
1914 if (ret) {
1915 dev_err(hba->dev, "%s: ufs_qcom_cfg_timers() failed\n", __func__);
1916 return ret;
1917 }
1918 /* set unipro core clock attributes and clear clock divider */
1919 return ufs_qcom_set_core_clk_ctrl(hba, false, freq);
1920 }
1921
ufs_qcom_clk_scale_notify(struct ufs_hba * hba,bool scale_up,unsigned long target_freq,enum ufs_notify_change_status status)1922 static int ufs_qcom_clk_scale_notify(struct ufs_hba *hba, bool scale_up,
1923 unsigned long target_freq,
1924 enum ufs_notify_change_status status)
1925 {
1926 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1927 int err;
1928
1929 /* check the host controller state before sending hibern8 cmd */
1930 if (!ufshcd_is_hba_active(hba))
1931 return 0;
1932
1933 if (status == PRE_CHANGE) {
1934 err = ufshcd_uic_hibern8_enter(hba);
1935 if (err)
1936 return err;
1937 if (scale_up)
1938 err = ufs_qcom_clk_scale_up_pre_change(hba, target_freq);
1939 else
1940 err = ufs_qcom_clk_scale_down_pre_change(hba);
1941
1942 if (err) {
1943 ufshcd_uic_hibern8_exit(hba);
1944 return err;
1945 }
1946 } else {
1947 if (scale_up)
1948 err = ufs_qcom_clk_scale_up_post_change(hba);
1949 else
1950 err = ufs_qcom_clk_scale_down_post_change(hba, target_freq);
1951
1952
1953 if (err) {
1954 ufshcd_uic_hibern8_exit(hba);
1955 return err;
1956 }
1957
1958 ufs_qcom_icc_update_bw(host);
1959 ufshcd_uic_hibern8_exit(hba);
1960 }
1961
1962 return 0;
1963 }
1964
ufs_qcom_enable_test_bus(struct ufs_qcom_host * host)1965 static void ufs_qcom_enable_test_bus(struct ufs_qcom_host *host)
1966 {
1967 ufshcd_rmwl(host->hba, UFS_REG_TEST_BUS_EN,
1968 UFS_REG_TEST_BUS_EN, REG_UFS_CFG1);
1969 ufshcd_rmwl(host->hba, TEST_BUS_EN, TEST_BUS_EN, REG_UFS_CFG1);
1970 }
1971
ufs_qcom_get_default_testbus_cfg(struct ufs_qcom_host * host)1972 static void ufs_qcom_get_default_testbus_cfg(struct ufs_qcom_host *host)
1973 {
1974 /* provide a legal default configuration */
1975 host->testbus.select_major = TSTBUS_UNIPRO;
1976 host->testbus.select_minor = 37;
1977 }
1978
ufs_qcom_testbus_cfg_is_ok(struct ufs_qcom_host * host)1979 static bool ufs_qcom_testbus_cfg_is_ok(struct ufs_qcom_host *host)
1980 {
1981 if (host->testbus.select_major >= TSTBUS_MAX) {
1982 dev_err(host->hba->dev,
1983 "%s: UFS_CFG1[TEST_BUS_SEL} may not equal 0x%05X\n",
1984 __func__, host->testbus.select_major);
1985 return false;
1986 }
1987
1988 return true;
1989 }
1990
ufs_qcom_testbus_config(struct ufs_qcom_host * host)1991 int ufs_qcom_testbus_config(struct ufs_qcom_host *host)
1992 {
1993 int reg;
1994 int offset;
1995 u32 mask = TEST_BUS_SUB_SEL_MASK;
1996
1997 if (!host)
1998 return -EINVAL;
1999
2000 if (!ufs_qcom_testbus_cfg_is_ok(host))
2001 return -EPERM;
2002
2003 switch (host->testbus.select_major) {
2004 case TSTBUS_UAWM:
2005 reg = UFS_TEST_BUS_CTRL_0;
2006 offset = 24;
2007 break;
2008 case TSTBUS_UARM:
2009 reg = UFS_TEST_BUS_CTRL_0;
2010 offset = 16;
2011 break;
2012 case TSTBUS_TXUC:
2013 reg = UFS_TEST_BUS_CTRL_0;
2014 offset = 8;
2015 break;
2016 case TSTBUS_RXUC:
2017 reg = UFS_TEST_BUS_CTRL_0;
2018 offset = 0;
2019 break;
2020 case TSTBUS_DFC:
2021 reg = UFS_TEST_BUS_CTRL_1;
2022 offset = 24;
2023 break;
2024 case TSTBUS_TRLUT:
2025 reg = UFS_TEST_BUS_CTRL_1;
2026 offset = 16;
2027 break;
2028 case TSTBUS_TMRLUT:
2029 reg = UFS_TEST_BUS_CTRL_1;
2030 offset = 8;
2031 break;
2032 case TSTBUS_OCSC:
2033 reg = UFS_TEST_BUS_CTRL_1;
2034 offset = 0;
2035 break;
2036 case TSTBUS_WRAPPER:
2037 reg = UFS_TEST_BUS_CTRL_2;
2038 offset = 16;
2039 break;
2040 case TSTBUS_COMBINED:
2041 reg = UFS_TEST_BUS_CTRL_2;
2042 offset = 8;
2043 break;
2044 case TSTBUS_UTP_HCI:
2045 reg = UFS_TEST_BUS_CTRL_2;
2046 offset = 0;
2047 break;
2048 case TSTBUS_UNIPRO:
2049 reg = UFS_UNIPRO_CFG;
2050 offset = 20;
2051 mask = 0xFFF;
2052 break;
2053 /*
2054 * No need for a default case, since
2055 * ufs_qcom_testbus_cfg_is_ok() checks that the configuration
2056 * is legal
2057 */
2058 }
2059 mask <<= offset;
2060 ufshcd_rmwl(host->hba, TEST_BUS_SEL,
2061 (u32)host->testbus.select_major << 19,
2062 REG_UFS_CFG1);
2063 ufshcd_rmwl(host->hba, mask,
2064 (u32)host->testbus.select_minor << offset,
2065 reg);
2066 ufs_qcom_enable_test_bus(host);
2067
2068 return 0;
2069 }
2070
ufs_qcom_dump_testbus(struct ufs_hba * hba)2071 static void ufs_qcom_dump_testbus(struct ufs_hba *hba)
2072 {
2073 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
2074 int i, j, nminor = 0, testbus_len = 0;
2075 char *prefix;
2076
2077 u32 *testbus __free(kfree) = kmalloc_array(256, sizeof(u32), GFP_KERNEL);
2078 if (!testbus)
2079 return;
2080
2081 for (j = 0; j < TSTBUS_MAX; j++) {
2082 nminor = testbus_info[j].nminor;
2083 prefix = testbus_info[j].prefix;
2084 host->testbus.select_major = j;
2085 testbus_len = nminor * sizeof(u32);
2086 for (i = 0; i < nminor; i++) {
2087 host->testbus.select_minor = i;
2088 ufs_qcom_testbus_config(host);
2089 testbus[i] = ufshcd_readl(hba, UFS_TEST_BUS);
2090 }
2091 print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET,
2092 16, 4, testbus, testbus_len, false);
2093 }
2094 }
2095
ufs_qcom_dump_regs(struct ufs_hba * hba,size_t offset,size_t len,const char * prefix,void __iomem * base)2096 static int ufs_qcom_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
2097 const char *prefix, void __iomem *base)
2098 {
2099 size_t pos;
2100
2101 if (offset % 4 != 0 || len % 4 != 0)
2102 return -EINVAL;
2103
2104 u32 *regs __free(kfree) = kzalloc(len, GFP_ATOMIC);
2105 if (!regs)
2106 return -ENOMEM;
2107
2108 for (pos = 0; pos < len; pos += 4)
2109 regs[pos / 4] = readl(base + offset + pos);
2110
2111 print_hex_dump(KERN_ERR, prefix,
2112 len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,
2113 16, 4, regs, len, false);
2114
2115 return 0;
2116 }
2117
ufs_qcom_dump_mcq_hci_regs(struct ufs_hba * hba)2118 static void ufs_qcom_dump_mcq_hci_regs(struct ufs_hba *hba)
2119 {
2120 struct ufshcd_mcq_opr_info_t *opr = &hba->mcq_opr[0];
2121 void __iomem *mcq_vs_base = hba->mcq_base + UFS_MEM_VS_BASE;
2122
2123 struct dump_info {
2124 void __iomem *base;
2125 size_t offset;
2126 size_t len;
2127 const char *prefix;
2128 };
2129
2130 struct dump_info mcq_dumps[] = {
2131 {hba->mcq_base, 0x0, 256 * 4, "MCQ HCI-0 "},
2132 {hba->mcq_base, 0x400, 256 * 4, "MCQ HCI-1 "},
2133 {mcq_vs_base, 0x0, 5 * 4, "MCQ VS-0 "},
2134 {opr->base, 0x0, 256 * 4, "MCQ SQD-0 "},
2135 {opr->base, 0x400, 256 * 4, "MCQ SQD-1 "},
2136 {opr->base, 0x800, 256 * 4, "MCQ SQD-2 "},
2137 {opr->base, 0xc00, 256 * 4, "MCQ SQD-3 "},
2138 {opr->base, 0x1000, 256 * 4, "MCQ SQD-4 "},
2139 {opr->base, 0x1400, 256 * 4, "MCQ SQD-5 "},
2140 {opr->base, 0x1800, 256 * 4, "MCQ SQD-6 "},
2141 {opr->base, 0x1c00, 256 * 4, "MCQ SQD-7 "},
2142
2143 };
2144
2145 for (int i = 0; i < ARRAY_SIZE(mcq_dumps); i++) {
2146 ufs_qcom_dump_regs(hba, mcq_dumps[i].offset, mcq_dumps[i].len,
2147 mcq_dumps[i].prefix, mcq_dumps[i].base);
2148 cond_resched();
2149 }
2150 }
2151
ufs_qcom_dump_dbg_regs(struct ufs_hba * hba)2152 static void ufs_qcom_dump_dbg_regs(struct ufs_hba *hba)
2153 {
2154 u32 reg;
2155 struct ufs_qcom_host *host;
2156
2157 host = ufshcd_get_variant(hba);
2158
2159 dev_err(hba->dev, "HW_H8_ENTER_CNT=%d\n", ufshcd_readl(hba, REG_UFS_HW_H8_ENTER_CNT));
2160 dev_err(hba->dev, "HW_H8_EXIT_CNT=%d\n", ufshcd_readl(hba, REG_UFS_HW_H8_EXIT_CNT));
2161
2162 dev_err(hba->dev, "SW_H8_ENTER_CNT=%d\n", ufshcd_readl(hba, REG_UFS_SW_H8_ENTER_CNT));
2163 dev_err(hba->dev, "SW_H8_EXIT_CNT=%d\n", ufshcd_readl(hba, REG_UFS_SW_H8_EXIT_CNT));
2164
2165 dev_err(hba->dev, "SW_AFTER_HW_H8_ENTER_CNT=%d\n",
2166 ufshcd_readl(hba, REG_UFS_SW_AFTER_HW_H8_ENTER_CNT));
2167
2168 ufshcd_dump_regs(hba, REG_UFS_SYS1CLK_1US, 16 * 4,
2169 "HCI Vendor Specific Registers ");
2170
2171 reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_REG_OCSC);
2172 ufshcd_dump_regs(hba, reg, 44 * 4, "UFS_UFS_DBG_RD_REG_OCSC ");
2173
2174 reg = ufshcd_readl(hba, REG_UFS_CFG1);
2175 reg |= UTP_DBG_RAMS_EN;
2176 ufshcd_writel(hba, reg, REG_UFS_CFG1);
2177
2178 reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_EDTL_RAM);
2179 ufshcd_dump_regs(hba, reg, 32 * 4, "UFS_UFS_DBG_RD_EDTL_RAM ");
2180
2181 reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_DESC_RAM);
2182 ufshcd_dump_regs(hba, reg, 128 * 4, "UFS_UFS_DBG_RD_DESC_RAM ");
2183
2184 reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_PRDT_RAM);
2185 ufshcd_dump_regs(hba, reg, 64 * 4, "UFS_UFS_DBG_RD_PRDT_RAM ");
2186
2187 /* clear bit 17 - UTP_DBG_RAMS_EN */
2188 ufshcd_rmwl(hba, UTP_DBG_RAMS_EN, 0, REG_UFS_CFG1);
2189
2190 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_UAWM);
2191 ufshcd_dump_regs(hba, reg, 4 * 4, "UFS_DBG_RD_REG_UAWM ");
2192
2193 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_UARM);
2194 ufshcd_dump_regs(hba, reg, 4 * 4, "UFS_DBG_RD_REG_UARM ");
2195
2196 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_TXUC);
2197 ufshcd_dump_regs(hba, reg, 48 * 4, "UFS_DBG_RD_REG_TXUC ");
2198
2199 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_RXUC);
2200 ufshcd_dump_regs(hba, reg, 27 * 4, "UFS_DBG_RD_REG_RXUC ");
2201
2202 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_DFC);
2203 ufshcd_dump_regs(hba, reg, 19 * 4, "UFS_DBG_RD_REG_DFC ");
2204
2205 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_TRLUT);
2206 ufshcd_dump_regs(hba, reg, 34 * 4, "UFS_DBG_RD_REG_TRLUT ");
2207
2208 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_TMRLUT);
2209 ufshcd_dump_regs(hba, reg, 9 * 4, "UFS_DBG_RD_REG_TMRLUT ");
2210
2211 if (hba->mcq_enabled) {
2212 reg = ufs_qcom_get_debug_reg_offset(host, UFS_RD_REG_MCQ);
2213 ufshcd_dump_regs(hba, reg, 64 * 4, "HCI MCQ Debug Registers ");
2214 }
2215
2216 /* ensure below dumps occur only in task context due to blocking calls. */
2217 if (in_task()) {
2218 /* Dump MCQ Host Vendor Specific Registers */
2219 if (hba->mcq_enabled)
2220 ufs_qcom_dump_mcq_hci_regs(hba);
2221
2222 /* voluntarily yield the CPU as we are dumping too much data */
2223 ufshcd_dump_regs(hba, UFS_TEST_BUS, 4, "UFS_TEST_BUS ");
2224 cond_resched();
2225 ufs_qcom_dump_testbus(hba);
2226 }
2227 }
2228
2229 /**
2230 * ufs_qcom_device_reset() - toggle the (optional) device reset line
2231 * @hba: per-adapter instance
2232 *
2233 * Toggles the (optional) reset line to reset the attached device.
2234 */
ufs_qcom_device_reset(struct ufs_hba * hba)2235 static int ufs_qcom_device_reset(struct ufs_hba *hba)
2236 {
2237 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
2238
2239 /* reset gpio is optional */
2240 if (!host->device_reset)
2241 return -EOPNOTSUPP;
2242
2243 /*
2244 * The UFS device shall detect reset pulses of 1us, sleep for 10us to
2245 * be on the safe side.
2246 */
2247 ufs_qcom_device_reset_ctrl(hba, true);
2248 usleep_range(10, 15);
2249
2250 ufs_qcom_device_reset_ctrl(hba, false);
2251 usleep_range(10, 15);
2252
2253 return 0;
2254 }
2255
2256 /**
2257 * ufs_qcom_fw_managed_device_reset - Reset UFS device under FW-managed design
2258 * @hba: pointer to UFS host bus adapter
2259 *
2260 * In the firmware-managed reset model, the power domain is powered on by genpd
2261 * before the UFS controller driver probes. For subsequent resets (such as
2262 * suspend/resume or recovery), the UFS driver must explicitly invoke PM runtime
2263 *
2264 * Return: 0 on success or a negative error code on failure.
2265 */
ufs_qcom_fw_managed_device_reset(struct ufs_hba * hba)2266 static int ufs_qcom_fw_managed_device_reset(struct ufs_hba *hba)
2267 {
2268 static bool is_boot = true;
2269 int err;
2270
2271 /* Skip reset on cold boot; perform it on subsequent calls */
2272 if (is_boot) {
2273 is_boot = false;
2274 return 0;
2275 }
2276
2277 pm_runtime_put_sync(hba->dev);
2278 err = pm_runtime_resume_and_get(hba->dev);
2279 if (err < 0) {
2280 dev_err(hba->dev, "PM runtime resume failed: %d\n", err);
2281 return err;
2282 }
2283
2284 return 0;
2285 }
2286
ufs_qcom_config_scaling_param(struct ufs_hba * hba,struct devfreq_dev_profile * p,struct devfreq_simple_ondemand_data * d)2287 static void ufs_qcom_config_scaling_param(struct ufs_hba *hba,
2288 struct devfreq_dev_profile *p,
2289 struct devfreq_simple_ondemand_data *d)
2290 {
2291 p->polling_ms = 60;
2292 p->timer = DEVFREQ_TIMER_DELAYED;
2293 d->upthreshold = 70;
2294 d->downdifferential = 65;
2295
2296 hba->clk_scaling.suspend_on_no_request = true;
2297 }
2298
ufs_qcom_mcq_config_resource(struct ufs_hba * hba)2299 static int ufs_qcom_mcq_config_resource(struct ufs_hba *hba)
2300 {
2301 struct platform_device *pdev = to_platform_device(hba->dev);
2302 struct resource *res;
2303
2304 /* Map the MCQ configuration region */
2305 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mcq");
2306 if (!res) {
2307 dev_err(hba->dev, "MCQ resource not found in device tree\n");
2308 return -ENODEV;
2309 }
2310
2311 hba->mcq_base = devm_ioremap_resource(hba->dev, res);
2312 if (IS_ERR(hba->mcq_base)) {
2313 dev_err(hba->dev, "Failed to map MCQ region: %ld\n",
2314 PTR_ERR(hba->mcq_base));
2315 return PTR_ERR(hba->mcq_base);
2316 }
2317
2318 return 0;
2319 }
2320
ufs_qcom_op_runtime_config(struct ufs_hba * hba)2321 static int ufs_qcom_op_runtime_config(struct ufs_hba *hba)
2322 {
2323 struct ufshcd_mcq_opr_info_t *opr;
2324 int i;
2325 u32 doorbell_offsets[OPR_MAX];
2326
2327 /*
2328 * Configure doorbell address offsets in MCQ configuration registers.
2329 * These values are offsets relative to mmio_base (UFS_HCI_BASE).
2330 *
2331 * Memory Layout:
2332 * - mmio_base = UFS_HCI_BASE
2333 * - mcq_base = MCQ_CONFIG_BASE = mmio_base + (UFS_QCOM_MCQCAP_QCFGPTR * 0x200)
2334 * - Doorbell registers are at: mmio_base + (UFS_QCOM_MCQCAP_QCFGPTR * 0x200) +
2335 * - UFS_QCOM_MCQ_SQD_OFFSET
2336 * - Which is also: mcq_base + UFS_QCOM_MCQ_SQD_OFFSET
2337 */
2338
2339 doorbell_offsets[OPR_SQD] = UFS_QCOM_SQD_ADDR_OFFSET;
2340 doorbell_offsets[OPR_SQIS] = UFS_QCOM_SQIS_ADDR_OFFSET;
2341 doorbell_offsets[OPR_CQD] = UFS_QCOM_CQD_ADDR_OFFSET;
2342 doorbell_offsets[OPR_CQIS] = UFS_QCOM_CQIS_ADDR_OFFSET;
2343
2344 /*
2345 * Configure MCQ operation registers.
2346 *
2347 * The doorbell registers are physically located within the MCQ region:
2348 * - doorbell_physical_addr = mmio_base + doorbell_offset
2349 * - doorbell_physical_addr = mcq_base + (doorbell_offset - MCQ_CONFIG_OFFSET)
2350 */
2351 for (i = 0; i < OPR_MAX; i++) {
2352 opr = &hba->mcq_opr[i];
2353 opr->offset = doorbell_offsets[i]; /* Offset relative to mmio_base */
2354 opr->stride = UFS_QCOM_MCQ_STRIDE; /* 256 bytes between queues */
2355
2356 /*
2357 * Calculate the actual doorbell base address within MCQ region:
2358 * base = mcq_base + (doorbell_offset - MCQ_CONFIG_OFFSET)
2359 */
2360 opr->base = hba->mcq_base + (opr->offset - UFS_QCOM_MCQ_CONFIG_OFFSET);
2361 }
2362
2363 return 0;
2364 }
2365
ufs_qcom_get_hba_mac(struct ufs_hba * hba)2366 static int ufs_qcom_get_hba_mac(struct ufs_hba *hba)
2367 {
2368 /* Qualcomm HC supports up to 64 */
2369 return MAX_SUPP_MAC;
2370 }
2371
ufs_qcom_get_outstanding_cqs(struct ufs_hba * hba,unsigned long * ocqs)2372 static int ufs_qcom_get_outstanding_cqs(struct ufs_hba *hba,
2373 unsigned long *ocqs)
2374 {
2375 /* Read from MCQ vendor-specific register in MCQ region */
2376 *ocqs = readl(hba->mcq_base + UFS_MEM_CQIS_VS);
2377
2378 return 0;
2379 }
2380
ufs_qcom_write_msi_msg(struct msi_desc * desc,struct msi_msg * msg)2381 static void ufs_qcom_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
2382 {
2383 struct device *dev = msi_desc_to_dev(desc);
2384 struct ufs_hba *hba = dev_get_drvdata(dev);
2385
2386 ufshcd_mcq_config_esi(hba, msg);
2387 }
2388
2389 struct ufs_qcom_irq {
2390 unsigned int irq;
2391 unsigned int idx;
2392 struct ufs_hba *hba;
2393 };
2394
ufs_qcom_mcq_esi_handler(int irq,void * data)2395 static irqreturn_t ufs_qcom_mcq_esi_handler(int irq, void *data)
2396 {
2397 struct ufs_qcom_irq *qi = data;
2398 struct ufs_hba *hba = qi->hba;
2399 struct ufs_hw_queue *hwq = &hba->uhq[qi->idx];
2400
2401 ufshcd_mcq_write_cqis(hba, 0x1, qi->idx);
2402 ufshcd_mcq_poll_cqe_lock(hba, hwq);
2403
2404 return IRQ_HANDLED;
2405 }
2406
ufs_qcom_config_esi(struct ufs_hba * hba)2407 static int ufs_qcom_config_esi(struct ufs_hba *hba)
2408 {
2409 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
2410 int nr_irqs, ret;
2411
2412 if (host->esi_enabled)
2413 return 0;
2414
2415 /*
2416 * 1. We only handle CQs as of now.
2417 * 2. Poll queues do not need ESI.
2418 */
2419 nr_irqs = hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL];
2420
2421 ret = platform_device_msi_init_and_alloc_irqs(hba->dev, nr_irqs,
2422 ufs_qcom_write_msi_msg);
2423 if (ret) {
2424 dev_warn(hba->dev, "Platform MSI not supported or failed, continuing without ESI\n");
2425 return ret; /* Continue without ESI */
2426 }
2427
2428 struct ufs_qcom_irq *qi = devm_kcalloc(hba->dev, nr_irqs, sizeof(*qi), GFP_KERNEL);
2429
2430 if (!qi) {
2431 platform_device_msi_free_irqs_all(hba->dev);
2432 return -ENOMEM;
2433 }
2434
2435 for (int idx = 0; idx < nr_irqs; idx++) {
2436 qi[idx].irq = msi_get_virq(hba->dev, idx);
2437 qi[idx].idx = idx;
2438 qi[idx].hba = hba;
2439
2440 ret = devm_request_irq(hba->dev, qi[idx].irq, ufs_qcom_mcq_esi_handler,
2441 IRQF_SHARED, "qcom-mcq-esi", qi + idx);
2442 if (ret) {
2443 /* Free previously allocated IRQs */
2444 for (int j = 0; j < idx; j++)
2445 devm_free_irq(hba->dev, qi[j].irq, qi + j);
2446 platform_device_msi_free_irqs_all(hba->dev);
2447 devm_kfree(hba->dev, qi);
2448 return ret;
2449 }
2450 }
2451
2452 if (host->hw_ver.major >= 6) {
2453 ufshcd_rmwl(hba, ESI_VEC_MASK, FIELD_PREP(ESI_VEC_MASK, MAX_ESI_VEC - 1),
2454 REG_UFS_CFG3);
2455 }
2456 ufshcd_mcq_enable_esi(hba);
2457 host->esi_enabled = true;
2458 return 0;
2459 }
2460
ufs_qcom_opp_freq_to_clk_freq(struct ufs_hba * hba,unsigned long freq,char * name)2461 static unsigned long ufs_qcom_opp_freq_to_clk_freq(struct ufs_hba *hba,
2462 unsigned long freq, char *name)
2463 {
2464 struct ufs_clk_info *clki;
2465 struct dev_pm_opp *opp;
2466 unsigned long clk_freq;
2467 int idx = 0;
2468 bool found = false;
2469
2470 opp = dev_pm_opp_find_freq_exact_indexed(hba->dev, freq, 0, true);
2471 if (IS_ERR(opp)) {
2472 dev_err(hba->dev, "Failed to find OPP for exact frequency %lu\n", freq);
2473 return 0;
2474 }
2475
2476 list_for_each_entry(clki, &hba->clk_list_head, list) {
2477 if (!strcmp(clki->name, name)) {
2478 found = true;
2479 break;
2480 }
2481
2482 idx++;
2483 }
2484
2485 if (!found) {
2486 dev_err(hba->dev, "Failed to find clock '%s' in clk list\n", name);
2487 dev_pm_opp_put(opp);
2488 return 0;
2489 }
2490
2491 clk_freq = dev_pm_opp_get_freq_indexed(opp, idx);
2492
2493 dev_pm_opp_put(opp);
2494
2495 return clk_freq;
2496 }
2497
ufs_qcom_freq_to_gear_speed(struct ufs_hba * hba,unsigned long freq)2498 static u32 ufs_qcom_freq_to_gear_speed(struct ufs_hba *hba, unsigned long freq)
2499 {
2500 u32 gear = UFS_HS_DONT_CHANGE;
2501 unsigned long unipro_freq;
2502
2503 if (!hba->use_pm_opp)
2504 return gear;
2505
2506 unipro_freq = ufs_qcom_opp_freq_to_clk_freq(hba, freq, "core_clk_unipro");
2507 switch (unipro_freq) {
2508 case 403000000:
2509 gear = UFS_HS_G5;
2510 break;
2511 case 300000000:
2512 gear = UFS_HS_G4;
2513 break;
2514 case 201500000:
2515 gear = UFS_HS_G3;
2516 break;
2517 case 150000000:
2518 case 100000000:
2519 gear = UFS_HS_G2;
2520 break;
2521 case 75000000:
2522 case 37500000:
2523 gear = UFS_HS_G1;
2524 break;
2525 default:
2526 dev_err(hba->dev, "%s: Unsupported clock freq : %lu\n", __func__, freq);
2527 return UFS_HS_DONT_CHANGE;
2528 }
2529
2530 return min_t(u32, gear, hba->max_pwr_info.info.gear_rx);
2531 }
2532
ufs_qcom_host_eom_config(struct ufs_hba * hba,int lane,const struct ufs_eom_coord * eom_coord,u32 target_test_count)2533 static int ufs_qcom_host_eom_config(struct ufs_hba *hba, int lane,
2534 const struct ufs_eom_coord *eom_coord,
2535 u32 target_test_count)
2536 {
2537 enum ufs_eom_eye_mask eye_mask = eom_coord->eye_mask;
2538 int v_step = eom_coord->v_step;
2539 int t_step = eom_coord->t_step;
2540 u32 volt_step, timing_step;
2541 int ret;
2542
2543 if (abs(v_step) > UFS_QCOM_EOM_VOLTAGE_STEPS_MAX) {
2544 dev_err(hba->dev, "Invalid EOM Voltage Step: %d\n", v_step);
2545 return -ERANGE;
2546 }
2547
2548 if (abs(t_step) > UFS_QCOM_EOM_TIMING_STEPS_MAX) {
2549 dev_err(hba->dev, "Invalid EOM Timing Step: %d\n", t_step);
2550 return -ERANGE;
2551 }
2552
2553 if (v_step < 0)
2554 volt_step = RX_EYEMON_NEGATIVE_STEP_BIT | (u32)(-v_step);
2555 else
2556 volt_step = (u32)v_step;
2557
2558 if (t_step < 0)
2559 timing_step = RX_EYEMON_NEGATIVE_STEP_BIT | (u32)(-t_step);
2560 else
2561 timing_step = (u32)t_step;
2562
2563 ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_EYEMON_ENABLE,
2564 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
2565 BIT(eye_mask) | RX_EYEMON_EXTENDED_VRANGE_BIT);
2566 if (ret) {
2567 dev_err(hba->dev, "Failed to enable Host EOM on Lane %d: %d\n",
2568 lane, ret);
2569 return ret;
2570 }
2571
2572 ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_EYEMON_TIMING_STEPS,
2573 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
2574 timing_step);
2575 if (ret) {
2576 dev_err(hba->dev, "Failed to set Host EOM timing step on Lane %d: %d\n",
2577 lane, ret);
2578 return ret;
2579 }
2580
2581 ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_EYEMON_VOLTAGE_STEPS,
2582 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
2583 volt_step);
2584 if (ret) {
2585 dev_err(hba->dev, "Failed to set Host EOM voltage step on Lane %d: %d\n",
2586 lane, ret);
2587 return ret;
2588 }
2589
2590 ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_EYEMON_TARGET_TEST_COUNT,
2591 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
2592 target_test_count);
2593 if (ret)
2594 dev_err(hba->dev, "Failed to set Host EOM target test count on Lane %d: %d\n",
2595 lane, ret);
2596
2597 return ret;
2598 }
2599
ufs_qcom_host_eom_may_stop(struct ufs_hba * hba,int lane,u32 target_test_count,u32 * err_count)2600 static int ufs_qcom_host_eom_may_stop(struct ufs_hba *hba, int lane,
2601 u32 target_test_count, u32 *err_count)
2602 {
2603 u32 start, tested_count, error_count;
2604 int ret;
2605
2606 ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(RX_EYEMON_START,
2607 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
2608 &start);
2609 if (ret) {
2610 dev_err(hba->dev, "Failed to get Host EOM start status on Lane %d: %d\n",
2611 lane, ret);
2612 return ret;
2613 }
2614
2615 if (start & 0x1)
2616 return -EAGAIN;
2617
2618 ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(RX_EYEMON_TESTED_COUNT,
2619 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
2620 &tested_count);
2621 if (ret) {
2622 dev_err(hba->dev, "Failed to get Host EOM tested count on Lane %d: %d\n",
2623 lane, ret);
2624 return ret;
2625 }
2626
2627 ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(RX_EYEMON_ERROR_COUNT,
2628 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
2629 &error_count);
2630 if (ret) {
2631 dev_err(hba->dev, "Failed to get Host EOM error count on Lane %d: %d\n",
2632 lane, ret);
2633 return ret;
2634 }
2635
2636 /* EOM can stop */
2637 if ((tested_count >= target_test_count - 3) || error_count > 0) {
2638 *err_count = error_count;
2639
2640 /* Disable EOM */
2641 ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_EYEMON_ENABLE,
2642 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
2643 0x0);
2644 if (ret) {
2645 dev_err(hba->dev, "Failed to disable Host EOM on Lane %d: %d\n",
2646 lane, ret);
2647 return ret;
2648 }
2649 } else {
2650 return -EAGAIN;
2651 }
2652
2653 return 0;
2654 }
2655
ufs_qcom_host_eom_scan(struct ufs_hba * hba,int num_lanes,const struct ufs_eom_coord * eom_coord,u32 target_test_count,u32 * err_count)2656 static int ufs_qcom_host_eom_scan(struct ufs_hba *hba, int num_lanes,
2657 const struct ufs_eom_coord *eom_coord,
2658 u32 target_test_count, u32 *err_count)
2659 {
2660 bool eom_stopped[PA_MAXDATALANES] = { 0 };
2661 int lane, ret;
2662 u32 setting;
2663
2664 if (!err_count || !eom_coord)
2665 return -EINVAL;
2666
2667 if (target_test_count < UFS_QCOM_EOM_TARGET_TEST_COUNT_MIN) {
2668 dev_err(hba->dev, "Target test count (%u) too small for Host EOM\n",
2669 target_test_count);
2670 return -ERANGE;
2671 }
2672
2673 for (lane = 0; lane < num_lanes; lane++) {
2674 ret = ufs_qcom_host_eom_config(hba, lane, eom_coord,
2675 target_test_count);
2676 if (ret) {
2677 dev_err(hba->dev, "Failed to config Host RX EOM: %d\n", ret);
2678 return ret;
2679 }
2680 }
2681
2682 /*
2683 * Trigger a PACP_PWR_req to kick start EOM, but not to really change
2684 * the Power Mode.
2685 */
2686 ret = ufshcd_uic_change_pwr_mode(hba, FAST_MODE << 4 | FAST_MODE);
2687 if (ret) {
2688 dev_err(hba->dev, "Failed to change power mode to kick start Host EOM: %d\n",
2689 ret);
2690 return ret;
2691 }
2692
2693 more_burst:
2694 /* Create burst on Host RX Lane. */
2695 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_LOCALVERINFO), &setting);
2696
2697 for (lane = 0; lane < num_lanes; lane++) {
2698 if (eom_stopped[lane])
2699 continue;
2700
2701 ret = ufs_qcom_host_eom_may_stop(hba, lane, target_test_count,
2702 &err_count[lane]);
2703 if (!ret) {
2704 eom_stopped[lane] = true;
2705 } else if (ret == -EAGAIN) {
2706 /* Need more burst to excercise EOM */
2707 goto more_burst;
2708 } else {
2709 dev_err(hba->dev, "Failed to stop Host EOM: %d\n", ret);
2710 return ret;
2711 }
2712
2713 dev_dbg(hba->dev, "Host RX Lane %d EOM, v_step %d, t_step %d, error count %u\n",
2714 lane, eom_coord->v_step, eom_coord->t_step,
2715 err_count[lane]);
2716 }
2717
2718 return 0;
2719 }
2720
ufs_qcom_host_sw_rx_fom(struct ufs_hba * hba,int num_lanes,u32 * fom)2721 static int ufs_qcom_host_sw_rx_fom(struct ufs_hba *hba, int num_lanes, u32 *fom)
2722 {
2723 const struct ufs_eom_coord *eom_coord = sw_rx_fom_eom_coords_g6;
2724 u32 eom_err_count[PA_MAXDATALANES] = { 0 };
2725 u32 curr_ahit;
2726 int lane, i, ret;
2727
2728 if (!fom)
2729 return -EINVAL;
2730
2731 /* Stop the auto hibernate idle timer */
2732 curr_ahit = ufshcd_readl(hba, REG_AUTO_HIBERNATE_IDLE_TIMER);
2733 if (curr_ahit)
2734 ufshcd_writel(hba, 0, REG_AUTO_HIBERNATE_IDLE_TIMER);
2735
2736 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXHSADAPTTYPE), PA_NO_ADAPT);
2737 if (ret) {
2738 dev_err(hba->dev, "Failed to select NO_ADAPT before starting Host EOM: %d\n", ret);
2739 goto out;
2740 }
2741
2742 for (i = 0; i < SW_RX_FOM_EOM_COORDS; i++, eom_coord++) {
2743 ret = ufs_qcom_host_eom_scan(hba, num_lanes, eom_coord,
2744 UFS_QCOM_EOM_TARGET_TEST_COUNT_G6,
2745 eom_err_count);
2746 if (ret) {
2747 dev_err(hba->dev, "Failed to run Host EOM scan: %d\n", ret);
2748 break;
2749 }
2750
2751 for (lane = 0; lane < num_lanes; lane++) {
2752 /* Bad coordinates have no weights */
2753 if (eom_err_count[lane])
2754 continue;
2755 fom[lane] += SW_RX_FOM_EOM_COORDS_WEIGHT;
2756 }
2757 }
2758
2759 out:
2760 /* Restore the auto hibernate idle timer */
2761 if (curr_ahit)
2762 ufshcd_writel(hba, curr_ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
2763
2764 return ret;
2765 }
2766
ufs_qcom_get_rx_fom(struct ufs_hba * hba,struct ufs_pa_layer_attr * pwr_mode,struct tx_eqtr_iter * h_iter,struct tx_eqtr_iter * d_iter)2767 static int ufs_qcom_get_rx_fom(struct ufs_hba *hba,
2768 struct ufs_pa_layer_attr *pwr_mode,
2769 struct tx_eqtr_iter *h_iter,
2770 struct tx_eqtr_iter *d_iter)
2771 {
2772 struct ufshcd_tx_eq_params *params __free(kfree) =
2773 kzalloc_obj(*params);
2774 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
2775 struct ufs_pa_layer_attr old_pwr_info;
2776 u32 fom[PA_MAXDATALANES] = { 0 };
2777 u32 gear = pwr_mode->gear_tx;
2778 u32 rate = pwr_mode->hs_rate;
2779 int lane, ret;
2780
2781 if (host->hw_ver.major != 0x7 || host->hw_ver.minor > 0x1 ||
2782 gear <= UFS_HS_G5 || !d_iter || !d_iter->is_updated)
2783 return 0;
2784
2785 if (gear < UFS_HS_G1 || gear > UFS_HS_GEAR_MAX)
2786 return -ERANGE;
2787
2788 if (!params)
2789 return -ENOMEM;
2790
2791 memcpy(&old_pwr_info, &hba->pwr_info, sizeof(struct ufs_pa_layer_attr));
2792
2793 memcpy(params, &hba->tx_eq_params[gear - 1], sizeof(struct ufshcd_tx_eq_params));
2794 for (lane = 0; lane < pwr_mode->lane_rx; lane++) {
2795 params->device[lane].preshoot = d_iter->preshoot;
2796 params->device[lane].deemphasis = d_iter->deemphasis;
2797 }
2798
2799 /* Use TX EQTR settings as Device's TX Equalization settings. */
2800 ret = ufshcd_apply_tx_eq_settings(hba, params, gear);
2801 if (ret) {
2802 dev_err(hba->dev, "%s: Failed to apply TX EQ settings for HS-G%u: %d\n",
2803 __func__, gear, ret);
2804 goto link_recover_and_restore;
2805 }
2806
2807 /* Force PMC to target HS Gear to use new TX Equalization settings. */
2808 ret = ufshcd_change_power_mode(hba, pwr_mode, UFSHCD_PMC_POLICY_FORCE);
2809 if (ret) {
2810 dev_err(hba->dev, "%s: Failed to change power mode to HS-G%u, Rate-%s: %d\n",
2811 __func__, gear, ufs_hs_rate_to_str(rate), ret);
2812 goto link_recover_and_restore;
2813 }
2814
2815 ret = ufs_qcom_host_sw_rx_fom(hba, pwr_mode->lane_rx, fom);
2816 if (ret)
2817 dev_err(hba->dev, "Failed to get SW FOM of TX (PreShoot: %u, DeEmphasis: %u): %d\n",
2818 d_iter->preshoot, d_iter->deemphasis, ret);
2819
2820 link_recover_and_restore:
2821 /* Restore Device's TX Equalization settings. */
2822 ret = ufshcd_apply_tx_eq_settings(hba, &hba->tx_eq_params[gear - 1], gear);
2823 if (ret) {
2824 dev_err(hba->dev, "%s: Failed to apply TX EQ settings for HS-G%u: %d\n",
2825 __func__, gear, ret);
2826 return ret;
2827 }
2828
2829 /* Restore Power Mode. */
2830 ret = ufshcd_change_power_mode(hba, &old_pwr_info, UFSHCD_PMC_POLICY_FORCE);
2831 if (ret) {
2832 dev_err(hba->dev, "%s: Failed to restore power mode to HS-G%u: %d\n",
2833 __func__, old_pwr_info.gear_tx, ret);
2834 return ret;
2835 }
2836
2837 for (lane = 0; lane < pwr_mode->lane_rx; lane++)
2838 d_iter->fom[lane] = fom[lane];
2839
2840 return 0;
2841 }
2842
ufs_qcom_apply_tx_eqtr_settings(struct ufs_hba * hba,struct ufs_pa_layer_attr * pwr_mode,struct tx_eqtr_iter * h_iter,struct tx_eqtr_iter * d_iter)2843 static int ufs_qcom_apply_tx_eqtr_settings(struct ufs_hba *hba,
2844 struct ufs_pa_layer_attr *pwr_mode,
2845 struct tx_eqtr_iter *h_iter,
2846 struct tx_eqtr_iter *d_iter)
2847 {
2848 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
2849 u32 setting = 0;
2850 int lane;
2851
2852 if (host->hw_ver.major != 0x7 || host->hw_ver.minor > 0x1)
2853 return 0;
2854
2855 for (lane = 0; lane < pwr_mode->lane_tx; lane++) {
2856 setting |= TX_HS_PRESHOOT_BITS(lane, h_iter->preshoot);
2857 setting |= TX_HS_DEEMPHASIS_BITS(lane, h_iter->deemphasis);
2858 }
2859
2860 return ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXEQG1SETTING), setting);
2861 }
2862
ufs_qcom_tx_eqtr_notify(struct ufs_hba * hba,enum ufs_notify_change_status status,struct ufs_pa_layer_attr * pwr_mode)2863 static int ufs_qcom_tx_eqtr_notify(struct ufs_hba *hba,
2864 enum ufs_notify_change_status status,
2865 struct ufs_pa_layer_attr *pwr_mode)
2866 {
2867 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
2868 struct ufs_pa_layer_attr pwr_mode_hs_g1 = {
2869 .gear_rx = UFS_HS_G1,
2870 .gear_tx = UFS_HS_G1,
2871 .lane_rx = pwr_mode->lane_rx,
2872 .lane_tx = pwr_mode->lane_tx,
2873 .pwr_rx = FAST_MODE,
2874 .pwr_tx = FAST_MODE,
2875 .hs_rate = pwr_mode->hs_rate,
2876 };
2877 u32 gear = pwr_mode->gear_tx;
2878 u32 rate = pwr_mode->hs_rate;
2879 int ret;
2880
2881 if (host->hw_ver.major != 0x7 || host->hw_ver.minor > 0x1)
2882 return 0;
2883
2884 if (status == PRE_CHANGE) {
2885 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TXEQG1SETTING),
2886 &host->saved_tx_eq_g1_setting);
2887 if (ret)
2888 return ret;
2889
2890 /* PMC to target HS Gear. */
2891 ret = ufshcd_change_power_mode(hba, pwr_mode,
2892 UFSHCD_PMC_POLICY_DONT_FORCE);
2893 if (ret)
2894 dev_err(hba->dev, "%s: Failed to PMC to target HS-G%u, Rate-%s: %d\n",
2895 __func__, gear, ufs_hs_rate_to_str(rate), ret);
2896 } else {
2897 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXEQG1SETTING),
2898 host->saved_tx_eq_g1_setting);
2899 if (ret)
2900 return ret;
2901
2902 /* PMC back to HS-G1. */
2903 ret = ufshcd_change_power_mode(hba, &pwr_mode_hs_g1,
2904 UFSHCD_PMC_POLICY_DONT_FORCE);
2905 if (ret)
2906 dev_err(hba->dev, "%s: Failed to PMC to HS-G1, Rate-%s: %d\n",
2907 __func__, ufs_hs_rate_to_str(rate), ret);
2908 }
2909
2910 return ret;
2911 }
2912
2913 /*
2914 * struct ufs_hba_qcom_vops - UFS QCOM specific variant operations
2915 *
2916 * The variant operations configure the necessary controller and PHY
2917 * handshake during initialization.
2918 */
2919 static const struct ufs_hba_variant_ops ufs_hba_qcom_vops = {
2920 .name = "qcom",
2921 .init = ufs_qcom_init,
2922 .exit = ufs_qcom_exit,
2923 .get_ufs_hci_version = ufs_qcom_get_ufs_hci_version,
2924 .clk_scale_notify = ufs_qcom_clk_scale_notify,
2925 .setup_clocks = ufs_qcom_setup_clocks,
2926 .hce_enable_notify = ufs_qcom_hce_enable_notify,
2927 .link_startup_notify = ufs_qcom_link_startup_notify,
2928 .negotiate_pwr_mode = ufs_qcom_negotiate_pwr_mode,
2929 .pwr_change_notify = ufs_qcom_pwr_change_notify,
2930 .apply_dev_quirks = ufs_qcom_apply_dev_quirks,
2931 .fixup_dev_quirks = ufs_qcom_fixup_dev_quirks,
2932 .suspend = ufs_qcom_suspend,
2933 .resume = ufs_qcom_resume,
2934 .dbg_register_dump = ufs_qcom_dump_dbg_regs,
2935 .device_reset = ufs_qcom_device_reset,
2936 .config_scaling_param = ufs_qcom_config_scaling_param,
2937 .mcq_config_resource = ufs_qcom_mcq_config_resource,
2938 .get_hba_mac = ufs_qcom_get_hba_mac,
2939 .op_runtime_config = ufs_qcom_op_runtime_config,
2940 .get_outstanding_cqs = ufs_qcom_get_outstanding_cqs,
2941 .config_esi = ufs_qcom_config_esi,
2942 .freq_to_gear_speed = ufs_qcom_freq_to_gear_speed,
2943 .get_rx_fom = ufs_qcom_get_rx_fom,
2944 .apply_tx_eqtr_settings = ufs_qcom_apply_tx_eqtr_settings,
2945 .tx_eqtr_notify = ufs_qcom_tx_eqtr_notify,
2946 };
2947
2948 static const struct ufs_hba_variant_ops ufs_hba_qcom_sa8255p_vops = {
2949 .name = "qcom-sa8255p",
2950 .init = ufs_qcom_fw_managed_init,
2951 .exit = ufs_qcom_fw_managed_exit,
2952 .hce_enable_notify = ufs_qcom_fw_managed_hce_enable_notify,
2953 .pwr_change_notify = ufs_qcom_pwr_change_notify,
2954 .apply_dev_quirks = ufs_qcom_apply_dev_quirks,
2955 .fixup_dev_quirks = ufs_qcom_fixup_dev_quirks,
2956 .suspend = ufs_qcom_fw_managed_suspend,
2957 .resume = ufs_qcom_fw_managed_resume,
2958 .dbg_register_dump = ufs_qcom_dump_dbg_regs,
2959 .device_reset = ufs_qcom_fw_managed_device_reset,
2960 };
2961
2962 /**
2963 * ufs_qcom_probe - probe routine of the driver
2964 * @pdev: pointer to Platform device handle
2965 *
2966 * Return: zero for success and non-zero for failure.
2967 */
ufs_qcom_probe(struct platform_device * pdev)2968 static int ufs_qcom_probe(struct platform_device *pdev)
2969 {
2970 int err;
2971 struct device *dev = &pdev->dev;
2972 const struct ufs_hba_variant_ops *vops;
2973 const struct ufs_qcom_drvdata *drvdata = device_get_match_data(dev);
2974
2975 if (drvdata && drvdata->vops)
2976 vops = drvdata->vops;
2977 else
2978 vops = &ufs_hba_qcom_vops;
2979
2980 /* Perform generic probe */
2981 err = ufshcd_pltfrm_init(pdev, vops);
2982 if (err)
2983 return dev_err_probe(dev, err, "ufshcd_pltfrm_init() failed\n");
2984
2985 return 0;
2986 }
2987
2988 /**
2989 * ufs_qcom_remove - set driver_data of the device to NULL
2990 * @pdev: pointer to platform device handle
2991 *
2992 * Always returns 0
2993 */
ufs_qcom_remove(struct platform_device * pdev)2994 static void ufs_qcom_remove(struct platform_device *pdev)
2995 {
2996 struct ufs_hba *hba = platform_get_drvdata(pdev);
2997 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
2998
2999 ufshcd_pltfrm_remove(pdev);
3000 if (host->esi_enabled)
3001 platform_device_msi_free_irqs_all(hba->dev);
3002 }
3003
3004 static const struct ufs_qcom_drvdata ufs_qcom_sm8550_drvdata = {
3005 .quirks = UFSHCD_QUIRK_BROKEN_LSDBS_CAP,
3006 .no_phy_retention = true,
3007 };
3008
3009 static const struct ufs_qcom_drvdata ufs_qcom_sa8255p_drvdata = {
3010 .vops = &ufs_hba_qcom_sa8255p_vops
3011 };
3012
3013 static const struct of_device_id ufs_qcom_of_match[] __maybe_unused = {
3014 { .compatible = "qcom,ufshc" },
3015 { .compatible = "qcom,sm8550-ufshc", .data = &ufs_qcom_sm8550_drvdata },
3016 { .compatible = "qcom,sm8650-ufshc", .data = &ufs_qcom_sm8550_drvdata },
3017 { .compatible = "qcom,sa8255p-ufshc", .data = &ufs_qcom_sa8255p_drvdata },
3018 {},
3019 };
3020 MODULE_DEVICE_TABLE(of, ufs_qcom_of_match);
3021
3022 #ifdef CONFIG_ACPI
3023 static const struct acpi_device_id ufs_qcom_acpi_match[] = {
3024 { "QCOM24A5" },
3025 { },
3026 };
3027 MODULE_DEVICE_TABLE(acpi, ufs_qcom_acpi_match);
3028 #endif
3029
3030 static const struct dev_pm_ops ufs_qcom_pm_ops = {
3031 SET_RUNTIME_PM_OPS(ufshcd_runtime_suspend, ufshcd_runtime_resume, NULL)
3032 .prepare = ufshcd_suspend_prepare,
3033 .complete = ufshcd_resume_complete,
3034 #ifdef CONFIG_PM_SLEEP
3035 .suspend = ufshcd_system_suspend,
3036 .resume = ufshcd_system_resume,
3037 .freeze = ufshcd_system_freeze,
3038 .restore = ufshcd_system_restore,
3039 .thaw = ufshcd_system_thaw,
3040 #endif
3041 };
3042
3043 static struct platform_driver ufs_qcom_pltform = {
3044 .probe = ufs_qcom_probe,
3045 .remove = ufs_qcom_remove,
3046 .driver = {
3047 .name = "ufshcd-qcom",
3048 .pm = &ufs_qcom_pm_ops,
3049 .of_match_table = of_match_ptr(ufs_qcom_of_match),
3050 .acpi_match_table = ACPI_PTR(ufs_qcom_acpi_match),
3051 },
3052 };
3053 module_platform_driver(ufs_qcom_pltform);
3054
3055 MODULE_DESCRIPTION("Qualcomm UFS host controller driver");
3056 MODULE_LICENSE("GPL v2");
3057