1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2013-2016, Linux Foundation. All rights reserved.
4 */
5
6 #include <linux/acpi.h>
7 #include <linux/clk.h>
8 #include <linux/cleanup.h>
9 #include <linux/delay.h>
10 #include <linux/devfreq.h>
11 #include <linux/gpio/consumer.h>
12 #include <linux/interconnect.h>
13 #include <linux/module.h>
14 #include <linux/of.h>
15 #include <linux/phy/phy.h>
16 #include <linux/platform_device.h>
17 #include <linux/pm_domain.h>
18 #include <linux/reset-controller.h>
19 #include <linux/time.h>
20 #include <linux/unaligned.h>
21 #include <linux/units.h>
22
23 #include <soc/qcom/ice.h>
24
25 #include <ufs/ufshcd.h>
26 #include <ufs/ufshci.h>
27 #include <ufs/ufs_quirks.h>
28 #include <ufs/unipro.h>
29 #include "ufshcd-pltfrm.h"
30 #include "ufs-qcom.h"
31
32 #define MCQ_QCFGPTR_MASK GENMASK(7, 0)
33 #define MCQ_QCFGPTR_UNIT 0x200
34 #define MCQ_SQATTR_OFFSET(c) \
35 ((((c) >> 16) & MCQ_QCFGPTR_MASK) * MCQ_QCFGPTR_UNIT)
36 #define MCQ_QCFG_SIZE 0x40
37
38 /* De-emphasis for gear-5 */
39 #define DEEMPHASIS_3_5_dB 0x04
40 #define NO_DEEMPHASIS 0x0
41
42 #define UFS_ICE_SYNC_RST_SEL BIT(3)
43 #define UFS_ICE_SYNC_RST_SW BIT(4)
44
45 enum {
46 TSTBUS_UAWM,
47 TSTBUS_UARM,
48 TSTBUS_TXUC,
49 TSTBUS_RXUC,
50 TSTBUS_DFC,
51 TSTBUS_TRLUT,
52 TSTBUS_TMRLUT,
53 TSTBUS_OCSC,
54 TSTBUS_UTP_HCI,
55 TSTBUS_COMBINED,
56 TSTBUS_WRAPPER,
57 TSTBUS_UNIPRO,
58 TSTBUS_MAX,
59 };
60
61 #define QCOM_UFS_MAX_GEAR 5
62 #define QCOM_UFS_MAX_LANE 2
63
64 enum {
65 MODE_MIN,
66 MODE_PWM,
67 MODE_HS_RA,
68 MODE_HS_RB,
69 MODE_MAX,
70 };
71
72 static const struct __ufs_qcom_bw_table {
73 u32 mem_bw;
74 u32 cfg_bw;
75 } ufs_qcom_bw_table[MODE_MAX + 1][QCOM_UFS_MAX_GEAR + 1][QCOM_UFS_MAX_LANE + 1] = {
76 [MODE_MIN][0][0] = { 0, 0 }, /* Bandwidth values in KB/s */
77 [MODE_PWM][UFS_PWM_G1][UFS_LANE_1] = { 922, 1000 },
78 [MODE_PWM][UFS_PWM_G2][UFS_LANE_1] = { 1844, 1000 },
79 [MODE_PWM][UFS_PWM_G3][UFS_LANE_1] = { 3688, 1000 },
80 [MODE_PWM][UFS_PWM_G4][UFS_LANE_1] = { 7376, 1000 },
81 [MODE_PWM][UFS_PWM_G5][UFS_LANE_1] = { 14752, 1000 },
82 [MODE_PWM][UFS_PWM_G1][UFS_LANE_2] = { 1844, 1000 },
83 [MODE_PWM][UFS_PWM_G2][UFS_LANE_2] = { 3688, 1000 },
84 [MODE_PWM][UFS_PWM_G3][UFS_LANE_2] = { 7376, 1000 },
85 [MODE_PWM][UFS_PWM_G4][UFS_LANE_2] = { 14752, 1000 },
86 [MODE_PWM][UFS_PWM_G5][UFS_LANE_2] = { 29504, 1000 },
87 [MODE_HS_RA][UFS_HS_G1][UFS_LANE_1] = { 127796, 1000 },
88 [MODE_HS_RA][UFS_HS_G2][UFS_LANE_1] = { 255591, 1000 },
89 [MODE_HS_RA][UFS_HS_G3][UFS_LANE_1] = { 1492582, 102400 },
90 [MODE_HS_RA][UFS_HS_G4][UFS_LANE_1] = { 2915200, 204800 },
91 [MODE_HS_RA][UFS_HS_G5][UFS_LANE_1] = { 5836800, 409600 },
92 [MODE_HS_RA][UFS_HS_G1][UFS_LANE_2] = { 255591, 1000 },
93 [MODE_HS_RA][UFS_HS_G2][UFS_LANE_2] = { 511181, 1000 },
94 [MODE_HS_RA][UFS_HS_G3][UFS_LANE_2] = { 1492582, 204800 },
95 [MODE_HS_RA][UFS_HS_G4][UFS_LANE_2] = { 2915200, 409600 },
96 [MODE_HS_RA][UFS_HS_G5][UFS_LANE_2] = { 5836800, 819200 },
97 [MODE_HS_RB][UFS_HS_G1][UFS_LANE_1] = { 149422, 1000 },
98 [MODE_HS_RB][UFS_HS_G2][UFS_LANE_1] = { 298189, 1000 },
99 [MODE_HS_RB][UFS_HS_G3][UFS_LANE_1] = { 1492582, 102400 },
100 [MODE_HS_RB][UFS_HS_G4][UFS_LANE_1] = { 2915200, 204800 },
101 [MODE_HS_RB][UFS_HS_G5][UFS_LANE_1] = { 5836800, 409600 },
102 [MODE_HS_RB][UFS_HS_G1][UFS_LANE_2] = { 298189, 1000 },
103 [MODE_HS_RB][UFS_HS_G2][UFS_LANE_2] = { 596378, 1000 },
104 [MODE_HS_RB][UFS_HS_G3][UFS_LANE_2] = { 1492582, 204800 },
105 [MODE_HS_RB][UFS_HS_G4][UFS_LANE_2] = { 2915200, 409600 },
106 [MODE_HS_RB][UFS_HS_G5][UFS_LANE_2] = { 5836800, 819200 },
107 [MODE_MAX][0][0] = { 7643136, 819200 },
108 };
109
110 static const struct {
111 int nminor;
112 char *prefix;
113 } testbus_info[TSTBUS_MAX] = {
114 [TSTBUS_UAWM] = {32, "TSTBUS_UAWM"},
115 [TSTBUS_UARM] = {32, "TSTBUS_UARM"},
116 [TSTBUS_TXUC] = {32, "TSTBUS_TXUC"},
117 [TSTBUS_RXUC] = {32, "TSTBUS_RXUC"},
118 [TSTBUS_DFC] = {32, "TSTBUS_DFC"},
119 [TSTBUS_TRLUT] = {32, "TSTBUS_TRLUT"},
120 [TSTBUS_TMRLUT] = {32, "TSTBUS_TMRLUT"},
121 [TSTBUS_OCSC] = {32, "TSTBUS_OCSC"},
122 [TSTBUS_UTP_HCI] = {32, "TSTBUS_UTP_HCI"},
123 [TSTBUS_COMBINED] = {32, "TSTBUS_COMBINED"},
124 [TSTBUS_WRAPPER] = {32, "TSTBUS_WRAPPER"},
125 [TSTBUS_UNIPRO] = {256, "TSTBUS_UNIPRO"},
126 };
127
128 static void ufs_qcom_get_default_testbus_cfg(struct ufs_qcom_host *host);
129 static unsigned long ufs_qcom_opp_freq_to_clk_freq(struct ufs_hba *hba,
130 unsigned long freq, char *name);
131 static int ufs_qcom_set_core_clk_ctrl(struct ufs_hba *hba, bool is_scale_up, unsigned long freq);
132
rcdev_to_ufs_host(struct reset_controller_dev * rcd)133 static struct ufs_qcom_host *rcdev_to_ufs_host(struct reset_controller_dev *rcd)
134 {
135 return container_of(rcd, struct ufs_qcom_host, rcdev);
136 }
137
138 #ifdef CONFIG_SCSI_UFS_CRYPTO
139 /**
140 * ufs_qcom_config_ice_allocator() - ICE core allocator configuration
141 *
142 * @host: pointer to qcom specific variant structure.
143 */
ufs_qcom_config_ice_allocator(struct ufs_qcom_host * host)144 static void ufs_qcom_config_ice_allocator(struct ufs_qcom_host *host)
145 {
146 struct ufs_hba *hba = host->hba;
147 static const uint8_t val[4] = { NUM_RX_R1W0, NUM_TX_R0W1, NUM_RX_R1W1, NUM_TX_R1W1 };
148 u32 config;
149
150 if (!(host->caps & UFS_QCOM_CAP_ICE_CONFIG) ||
151 !(host->hba->caps & UFSHCD_CAP_CRYPTO))
152 return;
153
154 config = get_unaligned_le32(val);
155
156 ufshcd_writel(hba, ICE_ALLOCATOR_TYPE, REG_UFS_MEM_ICE_CONFIG);
157 ufshcd_writel(hba, config, REG_UFS_MEM_ICE_NUM_CORE);
158 }
159
ufs_qcom_ice_enable(struct ufs_qcom_host * host)160 static inline void ufs_qcom_ice_enable(struct ufs_qcom_host *host)
161 {
162 if (host->hba->caps & UFSHCD_CAP_CRYPTO)
163 qcom_ice_enable(host->ice);
164 }
165
166 static const struct blk_crypto_ll_ops ufs_qcom_crypto_ops; /* forward decl */
167
ufs_qcom_ice_init(struct ufs_qcom_host * host)168 static int ufs_qcom_ice_init(struct ufs_qcom_host *host)
169 {
170 struct ufs_hba *hba = host->hba;
171 struct blk_crypto_profile *profile = &hba->crypto_profile;
172 struct device *dev = hba->dev;
173 struct qcom_ice *ice;
174 union ufs_crypto_capabilities caps;
175 union ufs_crypto_cap_entry cap;
176 int err;
177 int i;
178
179 ice = devm_of_qcom_ice_get(dev);
180 if (IS_ERR(ice)) {
181 if (ice != ERR_PTR(-EOPNOTSUPP))
182 return PTR_ERR(ice);
183
184 dev_warn(dev, "Disabling inline encryption support\n");
185 return 0;
186 }
187
188 host->ice = ice;
189
190 /* Initialize the blk_crypto_profile */
191
192 caps.reg_val = cpu_to_le32(ufshcd_readl(hba, REG_UFS_CCAP));
193
194 /* The number of keyslots supported is (CFGC+1) */
195 err = devm_blk_crypto_profile_init(dev, profile, caps.config_count + 1);
196 if (err)
197 return err;
198
199 profile->ll_ops = ufs_qcom_crypto_ops;
200 profile->max_dun_bytes_supported = 8;
201 profile->key_types_supported = qcom_ice_get_supported_key_type(ice);
202 profile->dev = dev;
203
204 /*
205 * Currently this driver only supports AES-256-XTS. All known versions
206 * of ICE support it, but to be safe make sure it is really declared in
207 * the crypto capability registers. The crypto capability registers
208 * also give the supported data unit size(s).
209 */
210 for (i = 0; i < caps.num_crypto_cap; i++) {
211 cap.reg_val = cpu_to_le32(ufshcd_readl(hba,
212 REG_UFS_CRYPTOCAP +
213 i * sizeof(__le32)));
214 if (cap.algorithm_id == UFS_CRYPTO_ALG_AES_XTS &&
215 cap.key_size == UFS_CRYPTO_KEY_SIZE_256)
216 profile->modes_supported[BLK_ENCRYPTION_MODE_AES_256_XTS] |=
217 cap.sdus_mask * 512;
218 }
219
220 hba->caps |= UFSHCD_CAP_CRYPTO;
221 hba->quirks |= UFSHCD_QUIRK_CUSTOM_CRYPTO_PROFILE;
222 return 0;
223 }
224
ufs_qcom_ice_resume(struct ufs_qcom_host * host)225 static inline int ufs_qcom_ice_resume(struct ufs_qcom_host *host)
226 {
227 if (host->hba->caps & UFSHCD_CAP_CRYPTO)
228 return qcom_ice_resume(host->ice);
229
230 return 0;
231 }
232
ufs_qcom_ice_suspend(struct ufs_qcom_host * host)233 static inline int ufs_qcom_ice_suspend(struct ufs_qcom_host *host)
234 {
235 if (host->hba->caps & UFSHCD_CAP_CRYPTO)
236 return qcom_ice_suspend(host->ice);
237
238 return 0;
239 }
240
ufs_qcom_ice_keyslot_program(struct blk_crypto_profile * profile,const struct blk_crypto_key * key,unsigned int slot)241 static int ufs_qcom_ice_keyslot_program(struct blk_crypto_profile *profile,
242 const struct blk_crypto_key *key,
243 unsigned int slot)
244 {
245 struct ufs_hba *hba = ufs_hba_from_crypto_profile(profile);
246 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
247 int err;
248
249 ufshcd_hold(hba);
250 err = qcom_ice_program_key(host->ice, slot, key);
251 ufshcd_release(hba);
252 return err;
253 }
254
ufs_qcom_ice_keyslot_evict(struct blk_crypto_profile * profile,const struct blk_crypto_key * key,unsigned int slot)255 static int ufs_qcom_ice_keyslot_evict(struct blk_crypto_profile *profile,
256 const struct blk_crypto_key *key,
257 unsigned int slot)
258 {
259 struct ufs_hba *hba = ufs_hba_from_crypto_profile(profile);
260 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
261 int err;
262
263 ufshcd_hold(hba);
264 err = qcom_ice_evict_key(host->ice, slot);
265 ufshcd_release(hba);
266 return err;
267 }
268
ufs_qcom_ice_derive_sw_secret(struct blk_crypto_profile * profile,const u8 * eph_key,size_t eph_key_size,u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE])269 static int ufs_qcom_ice_derive_sw_secret(struct blk_crypto_profile *profile,
270 const u8 *eph_key, size_t eph_key_size,
271 u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE])
272 {
273 struct ufs_hba *hba = ufs_hba_from_crypto_profile(profile);
274 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
275
276 return qcom_ice_derive_sw_secret(host->ice, eph_key, eph_key_size,
277 sw_secret);
278 }
279
ufs_qcom_ice_import_key(struct blk_crypto_profile * profile,const u8 * raw_key,size_t raw_key_size,u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE])280 static int ufs_qcom_ice_import_key(struct blk_crypto_profile *profile,
281 const u8 *raw_key, size_t raw_key_size,
282 u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE])
283 {
284 struct ufs_hba *hba = ufs_hba_from_crypto_profile(profile);
285 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
286
287 return qcom_ice_import_key(host->ice, raw_key, raw_key_size, lt_key);
288 }
289
ufs_qcom_ice_generate_key(struct blk_crypto_profile * profile,u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE])290 static int ufs_qcom_ice_generate_key(struct blk_crypto_profile *profile,
291 u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE])
292 {
293 struct ufs_hba *hba = ufs_hba_from_crypto_profile(profile);
294 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
295
296 return qcom_ice_generate_key(host->ice, lt_key);
297 }
298
ufs_qcom_ice_prepare_key(struct blk_crypto_profile * profile,const u8 * lt_key,size_t lt_key_size,u8 eph_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE])299 static int ufs_qcom_ice_prepare_key(struct blk_crypto_profile *profile,
300 const u8 *lt_key, size_t lt_key_size,
301 u8 eph_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE])
302 {
303 struct ufs_hba *hba = ufs_hba_from_crypto_profile(profile);
304 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
305
306 return qcom_ice_prepare_key(host->ice, lt_key, lt_key_size, eph_key);
307 }
308
309 static const struct blk_crypto_ll_ops ufs_qcom_crypto_ops = {
310 .keyslot_program = ufs_qcom_ice_keyslot_program,
311 .keyslot_evict = ufs_qcom_ice_keyslot_evict,
312 .derive_sw_secret = ufs_qcom_ice_derive_sw_secret,
313 .import_key = ufs_qcom_ice_import_key,
314 .generate_key = ufs_qcom_ice_generate_key,
315 .prepare_key = ufs_qcom_ice_prepare_key,
316 };
317
318 #else
319
ufs_qcom_ice_enable(struct ufs_qcom_host * host)320 static inline void ufs_qcom_ice_enable(struct ufs_qcom_host *host)
321 {
322 }
323
ufs_qcom_ice_init(struct ufs_qcom_host * host)324 static int ufs_qcom_ice_init(struct ufs_qcom_host *host)
325 {
326 return 0;
327 }
328
ufs_qcom_ice_resume(struct ufs_qcom_host * host)329 static inline int ufs_qcom_ice_resume(struct ufs_qcom_host *host)
330 {
331 return 0;
332 }
333
ufs_qcom_ice_suspend(struct ufs_qcom_host * host)334 static inline int ufs_qcom_ice_suspend(struct ufs_qcom_host *host)
335 {
336 return 0;
337 }
338
ufs_qcom_config_ice_allocator(struct ufs_qcom_host * host)339 static void ufs_qcom_config_ice_allocator(struct ufs_qcom_host *host)
340 {
341 }
342
343 #endif
344
ufs_qcom_disable_lane_clks(struct ufs_qcom_host * host)345 static void ufs_qcom_disable_lane_clks(struct ufs_qcom_host *host)
346 {
347 if (!host->is_lane_clks_enabled)
348 return;
349
350 clk_bulk_disable_unprepare(host->num_clks, host->clks);
351
352 host->is_lane_clks_enabled = false;
353 }
354
ufs_qcom_enable_lane_clks(struct ufs_qcom_host * host)355 static int ufs_qcom_enable_lane_clks(struct ufs_qcom_host *host)
356 {
357 int err;
358
359 err = clk_bulk_prepare_enable(host->num_clks, host->clks);
360 if (err)
361 return err;
362
363 host->is_lane_clks_enabled = true;
364
365 return 0;
366 }
367
ufs_qcom_init_lane_clks(struct ufs_qcom_host * host)368 static int ufs_qcom_init_lane_clks(struct ufs_qcom_host *host)
369 {
370 int err;
371 struct device *dev = host->hba->dev;
372
373 if (has_acpi_companion(dev))
374 return 0;
375
376 err = devm_clk_bulk_get_all(dev, &host->clks);
377 if (err <= 0)
378 return err;
379
380 host->num_clks = err;
381
382 return 0;
383 }
384
ufs_qcom_check_hibern8(struct ufs_hba * hba)385 static int ufs_qcom_check_hibern8(struct ufs_hba *hba)
386 {
387 int err;
388 u32 tx_fsm_val;
389 unsigned long timeout = jiffies + msecs_to_jiffies(HBRN8_POLL_TOUT_MS);
390
391 do {
392 err = ufshcd_dme_get(hba,
393 UIC_ARG_MIB_SEL(MPHY_TX_FSM_STATE,
394 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
395 &tx_fsm_val);
396 if (err || tx_fsm_val == TX_FSM_HIBERN8)
397 break;
398
399 /* sleep for max. 200us */
400 usleep_range(100, 200);
401 } while (time_before(jiffies, timeout));
402
403 /*
404 * we might have scheduled out for long during polling so
405 * check the state again.
406 */
407 if (time_after(jiffies, timeout))
408 err = ufshcd_dme_get(hba,
409 UIC_ARG_MIB_SEL(MPHY_TX_FSM_STATE,
410 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
411 &tx_fsm_val);
412
413 if (err) {
414 dev_err(hba->dev, "%s: unable to get TX_FSM_STATE, err %d\n",
415 __func__, err);
416 } else if (tx_fsm_val != TX_FSM_HIBERN8) {
417 err = tx_fsm_val;
418 dev_err(hba->dev, "%s: invalid TX_FSM_STATE = %d\n",
419 __func__, err);
420 }
421
422 return err;
423 }
424
ufs_qcom_select_unipro_mode(struct ufs_qcom_host * host)425 static void ufs_qcom_select_unipro_mode(struct ufs_qcom_host *host)
426 {
427 ufshcd_rmwl(host->hba, QUNIPRO_SEL, QUNIPRO_SEL, REG_UFS_CFG1);
428
429 if (host->hw_ver.major >= 0x05)
430 ufshcd_rmwl(host->hba, QUNIPRO_G4_SEL, 0, REG_UFS_CFG0);
431 }
432
433 /*
434 * ufs_qcom_host_reset - reset host controller and PHY
435 */
ufs_qcom_host_reset(struct ufs_hba * hba)436 static int ufs_qcom_host_reset(struct ufs_hba *hba)
437 {
438 int ret;
439 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
440 bool reenable_intr;
441
442 if (!host->core_reset)
443 return 0;
444
445 reenable_intr = hba->is_irq_enabled;
446 ufshcd_disable_irq(hba);
447
448 ret = reset_control_assert(host->core_reset);
449 if (ret) {
450 dev_err(hba->dev, "%s: core_reset assert failed, err = %d\n",
451 __func__, ret);
452 return ret;
453 }
454
455 /*
456 * The hardware requirement for delay between assert/deassert
457 * is at least 3-4 sleep clock (32.7KHz) cycles, which comes to
458 * ~125us (4/32768). To be on the safe side add 200us delay.
459 */
460 usleep_range(200, 210);
461
462 ret = reset_control_deassert(host->core_reset);
463 if (ret) {
464 dev_err(hba->dev, "%s: core_reset deassert failed, err = %d\n",
465 __func__, ret);
466 return ret;
467 }
468
469 usleep_range(1000, 1100);
470
471 if (reenable_intr)
472 ufshcd_enable_irq(hba);
473
474 return 0;
475 }
476
ufs_qcom_get_hs_gear(struct ufs_hba * hba)477 static u32 ufs_qcom_get_hs_gear(struct ufs_hba *hba)
478 {
479 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
480
481 if (host->hw_ver.major >= 0x4)
482 return UFS_QCOM_MAX_GEAR(ufshcd_readl(hba, REG_UFS_PARAM0));
483
484 /* Default is HS-G3 */
485 return UFS_HS_G3;
486 }
487
ufs_qcom_power_up_sequence(struct ufs_hba * hba)488 static int ufs_qcom_power_up_sequence(struct ufs_hba *hba)
489 {
490 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
491 struct ufs_host_params *host_params = &host->host_params;
492 struct phy *phy = host->generic_phy;
493 enum phy_mode mode;
494 int ret;
495
496 /*
497 * HW ver 5 can only support up to HS-G5 Rate-A due to HW limitations.
498 * If the HS-G5 PHY gear is used, update host_params->hs_rate to Rate-A,
499 * so that the subsequent power mode change shall stick to Rate-A.
500 */
501 if (host->hw_ver.major == 0x5 && host->phy_gear == UFS_HS_G5)
502 host_params->hs_rate = PA_HS_MODE_A;
503
504 mode = host_params->hs_rate == PA_HS_MODE_B ? PHY_MODE_UFS_HS_B : PHY_MODE_UFS_HS_A;
505
506 /* Reset UFS Host Controller and PHY */
507 ret = ufs_qcom_host_reset(hba);
508 if (ret)
509 return ret;
510
511 if (phy->power_count)
512 phy_power_off(phy);
513
514
515 /* phy initialization - calibrate the phy */
516 ret = phy_init(phy);
517 if (ret) {
518 dev_err(hba->dev, "%s: phy init failed, ret = %d\n",
519 __func__, ret);
520 return ret;
521 }
522
523 ret = phy_set_mode_ext(phy, mode, host->phy_gear);
524 if (ret)
525 goto out_disable_phy;
526
527 /* power on phy - start serdes and phy's power and clocks */
528 ret = phy_power_on(phy);
529 if (ret) {
530 dev_err(hba->dev, "%s: phy power on failed, ret = %d\n",
531 __func__, ret);
532 goto out_disable_phy;
533 }
534
535 ret = phy_calibrate(phy);
536 if (ret) {
537 dev_err(hba->dev, "Failed to calibrate PHY: %d\n", ret);
538 goto out_disable_phy;
539 }
540
541 ufs_qcom_select_unipro_mode(host);
542
543 return 0;
544
545 out_disable_phy:
546 phy_exit(phy);
547
548 return ret;
549 }
550
551 /*
552 * The UTP controller has a number of internal clock gating cells (CGCs).
553 * Internal hardware sub-modules within the UTP controller control the CGCs.
554 * Hardware CGCs disable the clock to inactivate UTP sub-modules not involved
555 * in a specific operation, UTP controller CGCs are by default disabled and
556 * this function enables them (after every UFS link startup) to save some power
557 * leakage.
558 */
ufs_qcom_enable_hw_clk_gating(struct ufs_hba * hba)559 static void ufs_qcom_enable_hw_clk_gating(struct ufs_hba *hba)
560 {
561 int err;
562
563 /* Enable UTP internal clock gating */
564 ufshcd_rmwl(hba, REG_UFS_CFG2_CGC_EN_ALL, REG_UFS_CFG2_CGC_EN_ALL,
565 REG_UFS_CFG2);
566
567 /* Ensure that HW clock gating is enabled before next operations */
568 ufshcd_readl(hba, REG_UFS_CFG2);
569
570 /* Enable Unipro internal clock gating */
571 err = ufshcd_dme_rmw(hba, DL_VS_CLK_CFG_MASK,
572 DL_VS_CLK_CFG_MASK, DL_VS_CLK_CFG);
573 if (err)
574 goto out;
575
576 err = ufshcd_dme_rmw(hba, PA_VS_CLK_CFG_REG_MASK,
577 PA_VS_CLK_CFG_REG_MASK, PA_VS_CLK_CFG_REG);
578 if (err)
579 goto out;
580
581 err = ufshcd_dme_rmw(hba, DME_VS_CORE_CLK_CTRL_DME_HW_CGC_EN,
582 DME_VS_CORE_CLK_CTRL_DME_HW_CGC_EN,
583 DME_VS_CORE_CLK_CTRL);
584 out:
585 if (err)
586 dev_err(hba->dev, "hw clk gating enabled failed\n");
587 }
588
ufs_qcom_hce_enable_notify(struct ufs_hba * hba,enum ufs_notify_change_status status)589 static int ufs_qcom_hce_enable_notify(struct ufs_hba *hba,
590 enum ufs_notify_change_status status)
591 {
592 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
593 int err;
594
595 switch (status) {
596 case PRE_CHANGE:
597 err = ufs_qcom_power_up_sequence(hba);
598 if (err)
599 return err;
600
601 /*
602 * The PHY PLL output is the source of tx/rx lane symbol
603 * clocks, hence, enable the lane clocks only after PHY
604 * is initialized.
605 */
606 err = ufs_qcom_enable_lane_clks(host);
607 break;
608 case POST_CHANGE:
609 /* check if UFS PHY moved from DISABLED to HIBERN8 */
610 err = ufs_qcom_check_hibern8(hba);
611 ufs_qcom_enable_hw_clk_gating(hba);
612 ufs_qcom_ice_enable(host);
613 ufs_qcom_config_ice_allocator(host);
614 break;
615 default:
616 dev_err(hba->dev, "%s: invalid status %d\n", __func__, status);
617 err = -EINVAL;
618 break;
619 }
620 return err;
621 }
622
ufs_qcom_fw_managed_hce_enable_notify(struct ufs_hba * hba,enum ufs_notify_change_status status)623 static int ufs_qcom_fw_managed_hce_enable_notify(struct ufs_hba *hba,
624 enum ufs_notify_change_status status)
625 {
626 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
627
628 switch (status) {
629 case PRE_CHANGE:
630 ufs_qcom_select_unipro_mode(host);
631 break;
632 case POST_CHANGE:
633 ufs_qcom_enable_hw_clk_gating(hba);
634 ufs_qcom_ice_enable(host);
635 break;
636 default:
637 dev_err(hba->dev, "Invalid status %d\n", status);
638 return -EINVAL;
639 }
640
641 return 0;
642 }
643
644 /**
645 * ufs_qcom_cfg_timers - Configure ufs qcom cfg timers
646 *
647 * @hba: host controller instance
648 * @is_pre_scale_up: flag to check if pre scale up condition.
649 * @freq: target opp freq
650 * Return: zero for success and non-zero in case of a failure.
651 */
ufs_qcom_cfg_timers(struct ufs_hba * hba,bool is_pre_scale_up,unsigned long freq)652 static int ufs_qcom_cfg_timers(struct ufs_hba *hba, bool is_pre_scale_up, unsigned long freq)
653 {
654 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
655 struct ufs_clk_info *clki;
656 unsigned long clk_freq = 0;
657 u32 core_clk_cycles_per_us;
658
659 /*
660 * UTP controller uses SYS1CLK_1US_REG register for Interrupt
661 * Aggregation logic.
662 * It is mandatory to write SYS1CLK_1US_REG register on UFS host
663 * controller V4.0.0 onwards.
664 */
665 if (host->hw_ver.major < 4 && !ufshcd_is_intr_aggr_allowed(hba))
666 return 0;
667
668 if (hba->use_pm_opp && freq != ULONG_MAX) {
669 clk_freq = ufs_qcom_opp_freq_to_clk_freq(hba, freq, "core_clk");
670 if (clk_freq)
671 goto cfg_timers;
672 }
673
674 list_for_each_entry(clki, &hba->clk_list_head, list) {
675 if (!strcmp(clki->name, "core_clk")) {
676 if (freq == ULONG_MAX) {
677 clk_freq = clki->max_freq;
678 break;
679 }
680
681 if (is_pre_scale_up)
682 clk_freq = clki->max_freq;
683 else
684 clk_freq = clk_get_rate(clki->clk);
685 break;
686 }
687
688 }
689
690 cfg_timers:
691 /* If frequency is smaller than 1MHz, set to 1MHz */
692 if (clk_freq < DEFAULT_CLK_RATE_HZ)
693 clk_freq = DEFAULT_CLK_RATE_HZ;
694
695 core_clk_cycles_per_us = clk_freq / USEC_PER_SEC;
696 if (ufshcd_readl(hba, REG_UFS_SYS1CLK_1US) != core_clk_cycles_per_us) {
697 ufshcd_writel(hba, core_clk_cycles_per_us, REG_UFS_SYS1CLK_1US);
698 /*
699 * make sure above write gets applied before we return from
700 * this function.
701 */
702 ufshcd_readl(hba, REG_UFS_SYS1CLK_1US);
703 }
704
705 return 0;
706 }
707
ufs_qcom_link_startup_notify(struct ufs_hba * hba,enum ufs_notify_change_status status)708 static int ufs_qcom_link_startup_notify(struct ufs_hba *hba,
709 enum ufs_notify_change_status status)
710 {
711 int err = 0;
712
713 switch (status) {
714 case PRE_CHANGE:
715 if (ufs_qcom_cfg_timers(hba, false, ULONG_MAX)) {
716 dev_err(hba->dev, "%s: ufs_qcom_cfg_timers() failed\n",
717 __func__);
718 return -EINVAL;
719 }
720
721 err = ufs_qcom_set_core_clk_ctrl(hba, true, ULONG_MAX);
722 if (err)
723 dev_err(hba->dev, "cfg core clk ctrl failed\n");
724 /*
725 * Some UFS devices (and may be host) have issues if LCC is
726 * enabled. So we are setting PA_Local_TX_LCC_Enable to 0
727 * before link startup which will make sure that both host
728 * and device TX LCC are disabled once link startup is
729 * completed.
730 */
731 err = ufshcd_disable_host_tx_lcc(hba);
732
733 break;
734 default:
735 break;
736 }
737
738 return err;
739 }
740
ufs_qcom_device_reset_ctrl(struct ufs_hba * hba,bool asserted)741 static void ufs_qcom_device_reset_ctrl(struct ufs_hba *hba, bool asserted)
742 {
743 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
744
745 /* reset gpio is optional */
746 if (!host->device_reset)
747 return;
748
749 gpiod_set_value_cansleep(host->device_reset, asserted);
750 }
751
ufs_qcom_suspend(struct ufs_hba * hba,enum ufs_pm_op pm_op,enum ufs_notify_change_status status)752 static int ufs_qcom_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op,
753 enum ufs_notify_change_status status)
754 {
755 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
756
757 if (status == PRE_CHANGE)
758 return 0;
759
760 if (!ufs_qcom_is_link_active(hba))
761 ufs_qcom_disable_lane_clks(host);
762
763
764 /* reset the connected UFS device during power down */
765 if (ufs_qcom_is_link_off(hba) && host->device_reset) {
766 ufs_qcom_device_reset_ctrl(hba, true);
767 /*
768 * After sending the SSU command, asserting the rst_n
769 * line causes the device firmware to wake up and
770 * execute its reset routine.
771 *
772 * During this process, the device may draw current
773 * beyond the permissible limit for low-power mode (LPM).
774 * A 10ms delay, based on experimental observations,
775 * allows the UFS device to complete its hardware reset
776 * before transitioning the power rail to LPM.
777 */
778 usleep_range(10000, 11000);
779 }
780
781 return ufs_qcom_ice_suspend(host);
782 }
783
ufs_qcom_resume(struct ufs_hba * hba,enum ufs_pm_op pm_op)784 static int ufs_qcom_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
785 {
786 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
787 int err;
788 u32 reg_val;
789
790 err = ufs_qcom_enable_lane_clks(host);
791 if (err)
792 return err;
793
794 if ((!ufs_qcom_is_link_active(hba)) &&
795 host->hw_ver.major == 5 &&
796 host->hw_ver.minor == 0 &&
797 host->hw_ver.step == 0) {
798 ufshcd_writel(hba, UFS_ICE_SYNC_RST_SEL | UFS_ICE_SYNC_RST_SW, UFS_MEM_ICE_CFG);
799 reg_val = ufshcd_readl(hba, UFS_MEM_ICE_CFG);
800 reg_val &= ~(UFS_ICE_SYNC_RST_SEL | UFS_ICE_SYNC_RST_SW);
801 /*
802 * HW documentation doesn't recommend any delay between the
803 * reset set and clear. But we are enforcing an arbitrary delay
804 * to give flops enough time to settle in.
805 */
806 usleep_range(50, 100);
807 ufshcd_writel(hba, reg_val, UFS_MEM_ICE_CFG);
808 ufshcd_readl(hba, UFS_MEM_ICE_CFG);
809 }
810
811 return ufs_qcom_ice_resume(host);
812 }
813
ufs_qcom_fw_managed_suspend(struct ufs_hba * hba,enum ufs_pm_op pm_op,enum ufs_notify_change_status status)814 static int ufs_qcom_fw_managed_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op,
815 enum ufs_notify_change_status status)
816 {
817 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
818
819 if (status == PRE_CHANGE)
820 return 0;
821
822 pm_runtime_put_sync(hba->dev);
823
824 return ufs_qcom_ice_suspend(host);
825 }
826
ufs_qcom_fw_managed_resume(struct ufs_hba * hba,enum ufs_pm_op pm_op)827 static int ufs_qcom_fw_managed_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
828 {
829 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
830 int err;
831
832 err = pm_runtime_resume_and_get(hba->dev);
833 if (err) {
834 dev_err(hba->dev, "PM runtime resume failed: %d\n", err);
835 return err;
836 }
837
838 return ufs_qcom_ice_resume(host);
839 }
840
ufs_qcom_dev_ref_clk_ctrl(struct ufs_qcom_host * host,bool enable)841 static void ufs_qcom_dev_ref_clk_ctrl(struct ufs_qcom_host *host, bool enable)
842 {
843 if (host->dev_ref_clk_ctrl_mmio &&
844 (enable ^ host->is_dev_ref_clk_enabled)) {
845 u32 temp = readl_relaxed(host->dev_ref_clk_ctrl_mmio);
846
847 if (enable)
848 temp |= host->dev_ref_clk_en_mask;
849 else
850 temp &= ~host->dev_ref_clk_en_mask;
851
852 /*
853 * If we are here to disable this clock it might be immediately
854 * after entering into hibern8 in which case we need to make
855 * sure that device ref_clk is active for specific time after
856 * hibern8 enter.
857 */
858 if (!enable) {
859 unsigned long gating_wait;
860
861 gating_wait = host->hba->dev_info.clk_gating_wait_us;
862 if (!gating_wait) {
863 udelay(1);
864 } else {
865 /*
866 * bRefClkGatingWaitTime defines the minimum
867 * time for which the reference clock is
868 * required by device during transition from
869 * HS-MODE to LS-MODE or HIBERN8 state. Give it
870 * more delay to be on the safe side.
871 */
872 gating_wait += 10;
873 usleep_range(gating_wait, gating_wait + 10);
874 }
875 }
876
877 writel_relaxed(temp, host->dev_ref_clk_ctrl_mmio);
878
879 /*
880 * Make sure the write to ref_clk reaches the destination and
881 * not stored in a Write Buffer (WB).
882 */
883 readl(host->dev_ref_clk_ctrl_mmio);
884
885 /*
886 * If we call hibern8 exit after this, we need to make sure that
887 * device ref_clk is stable for at least 1us before the hibern8
888 * exit command.
889 */
890 if (enable)
891 udelay(1);
892
893 host->is_dev_ref_clk_enabled = enable;
894 }
895 }
896
ufs_qcom_icc_set_bw(struct ufs_qcom_host * host,u32 mem_bw,u32 cfg_bw)897 static int ufs_qcom_icc_set_bw(struct ufs_qcom_host *host, u32 mem_bw, u32 cfg_bw)
898 {
899 struct device *dev = host->hba->dev;
900 int ret;
901
902 ret = icc_set_bw(host->icc_ddr, 0, mem_bw);
903 if (ret < 0) {
904 dev_err(dev, "failed to set bandwidth request: %d\n", ret);
905 return ret;
906 }
907
908 ret = icc_set_bw(host->icc_cpu, 0, cfg_bw);
909 if (ret < 0) {
910 dev_err(dev, "failed to set bandwidth request: %d\n", ret);
911 return ret;
912 }
913
914 return 0;
915 }
916
ufs_qcom_get_bw_table(struct ufs_qcom_host * host)917 static struct __ufs_qcom_bw_table ufs_qcom_get_bw_table(struct ufs_qcom_host *host)
918 {
919 struct ufs_pa_layer_attr *p = &host->dev_req_params;
920 int gear = max_t(u32, p->gear_rx, p->gear_tx);
921 int lane = max_t(u32, p->lane_rx, p->lane_tx);
922
923 if (WARN_ONCE(gear > QCOM_UFS_MAX_GEAR,
924 "ICC scaling for UFS Gear (%d) not supported. Using Gear (%d) bandwidth\n",
925 gear, QCOM_UFS_MAX_GEAR))
926 gear = QCOM_UFS_MAX_GEAR;
927
928 if (WARN_ONCE(lane > QCOM_UFS_MAX_LANE,
929 "ICC scaling for UFS Lane (%d) not supported. Using Lane (%d) bandwidth\n",
930 lane, QCOM_UFS_MAX_LANE))
931 lane = QCOM_UFS_MAX_LANE;
932
933 if (ufshcd_is_hs_mode(p)) {
934 if (p->hs_rate == PA_HS_MODE_B)
935 return ufs_qcom_bw_table[MODE_HS_RB][gear][lane];
936 else
937 return ufs_qcom_bw_table[MODE_HS_RA][gear][lane];
938 } else {
939 return ufs_qcom_bw_table[MODE_PWM][gear][lane];
940 }
941 }
942
ufs_qcom_icc_update_bw(struct ufs_qcom_host * host)943 static int ufs_qcom_icc_update_bw(struct ufs_qcom_host *host)
944 {
945 struct __ufs_qcom_bw_table bw_table;
946
947 bw_table = ufs_qcom_get_bw_table(host);
948
949 return ufs_qcom_icc_set_bw(host, bw_table.mem_bw, bw_table.cfg_bw);
950 }
951
ufs_qcom_set_tx_hs_equalizer(struct ufs_hba * hba,u32 gear,u32 tx_lanes)952 static void ufs_qcom_set_tx_hs_equalizer(struct ufs_hba *hba, u32 gear, u32 tx_lanes)
953 {
954 u32 equalizer_val;
955 int ret, i;
956
957 /* Determine the equalizer value based on the gear */
958 equalizer_val = (gear == 5) ? DEEMPHASIS_3_5_dB : NO_DEEMPHASIS;
959
960 for (i = 0; i < tx_lanes; i++) {
961 ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_HS_EQUALIZER, i),
962 equalizer_val);
963 if (ret)
964 dev_err(hba->dev, "%s: failed equalizer lane %d\n",
965 __func__, i);
966 }
967 }
968
ufs_qcom_negotiate_pwr_mode(struct ufs_hba * hba,const struct ufs_pa_layer_attr * dev_max_params,struct ufs_pa_layer_attr * dev_req_params)969 static int ufs_qcom_negotiate_pwr_mode(struct ufs_hba *hba,
970 const struct ufs_pa_layer_attr *dev_max_params,
971 struct ufs_pa_layer_attr *dev_req_params)
972 {
973 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
974 struct ufs_host_params *host_params = &host->host_params;
975
976 return ufshcd_negotiate_pwr_params(host_params, dev_max_params, dev_req_params);
977 }
978
ufs_qcom_pwr_change_notify(struct ufs_hba * hba,enum ufs_notify_change_status status,struct ufs_pa_layer_attr * dev_req_params)979 static int ufs_qcom_pwr_change_notify(struct ufs_hba *hba,
980 enum ufs_notify_change_status status,
981 struct ufs_pa_layer_attr *dev_req_params)
982 {
983 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
984 int ret = 0;
985
986 if (!dev_req_params) {
987 pr_err("%s: incoming dev_req_params is NULL\n", __func__);
988 return -EINVAL;
989 }
990
991 switch (status) {
992 case PRE_CHANGE:
993 /*
994 * During UFS driver probe, always update the PHY gear to match the negotiated
995 * gear, so that, if quirk UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH is enabled,
996 * the second init can program the optimal PHY settings. This allows one to start
997 * the first init with either the minimum or the maximum support gear.
998 */
999 if (hba->ufshcd_state == UFSHCD_STATE_RESET) {
1000 /*
1001 * Skip REINIT if the negotiated gear matches with the
1002 * initial phy_gear. Otherwise, update the phy_gear to
1003 * program the optimal gear setting during REINIT.
1004 */
1005 if (host->phy_gear == dev_req_params->gear_tx)
1006 hba->quirks &= ~UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH;
1007 else
1008 host->phy_gear = dev_req_params->gear_tx;
1009 }
1010
1011 /* enable the device ref clock before changing to HS mode */
1012 if (!ufshcd_is_hs_mode(&hba->pwr_info) &&
1013 ufshcd_is_hs_mode(dev_req_params))
1014 ufs_qcom_dev_ref_clk_ctrl(host, true);
1015
1016 if (host->hw_ver.major >= 0x4) {
1017 ufshcd_dme_configure_adapt(hba,
1018 dev_req_params->gear_tx,
1019 PA_INITIAL_ADAPT);
1020 }
1021
1022 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TX_DEEMPHASIS_TUNING)
1023 ufs_qcom_set_tx_hs_equalizer(hba,
1024 dev_req_params->gear_tx, dev_req_params->lane_tx);
1025
1026 break;
1027 case POST_CHANGE:
1028 /* cache the power mode parameters to use internally */
1029 memcpy(&host->dev_req_params,
1030 dev_req_params, sizeof(*dev_req_params));
1031
1032 ufs_qcom_icc_update_bw(host);
1033
1034 /* disable the device ref clock if entered PWM mode */
1035 if (ufshcd_is_hs_mode(&hba->pwr_info) &&
1036 !ufshcd_is_hs_mode(dev_req_params))
1037 ufs_qcom_dev_ref_clk_ctrl(host, false);
1038 break;
1039 default:
1040 ret = -EINVAL;
1041 break;
1042 }
1043
1044 return ret;
1045 }
1046
ufs_qcom_quirk_host_pa_saveconfigtime(struct ufs_hba * hba)1047 static int ufs_qcom_quirk_host_pa_saveconfigtime(struct ufs_hba *hba)
1048 {
1049 int err;
1050 u32 pa_vs_config_reg1;
1051
1052 err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_VS_CONFIG_REG1),
1053 &pa_vs_config_reg1);
1054 if (err)
1055 return err;
1056
1057 /* Allow extension of MSB bits of PA_SaveConfigTime attribute */
1058 return ufshcd_dme_set(hba, UIC_ARG_MIB(PA_VS_CONFIG_REG1),
1059 (pa_vs_config_reg1 | (1 << 12)));
1060 }
1061
ufs_qcom_override_pa_tx_hsg1_sync_len(struct ufs_hba * hba)1062 static void ufs_qcom_override_pa_tx_hsg1_sync_len(struct ufs_hba *hba)
1063 {
1064 int err;
1065
1066 err = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TX_HSG1_SYNC_LENGTH),
1067 PA_TX_HSG1_SYNC_LENGTH_VAL);
1068 if (err)
1069 dev_err(hba->dev, "Failed (%d) set PA_TX_HSG1_SYNC_LENGTH\n", err);
1070 }
1071
1072 /**
1073 * ufs_qcom_double_t_adapt_l0l1l2l3 - Create a new adapt that doubles the
1074 * adaptation duration TADAPT_L0_L1_L2_L3 derived from the old adapt.
1075 *
1076 * @old_adapt: Original ADAPT_L0_L1_L2_L3 capability
1077 *
1078 * ADAPT_length_L0_L1_L2_L3 formula from M-PHY spec:
1079 * if (ADAPT_range_L0_L1_L2_L3 == COARSE) {
1080 * ADAPT_length_L0_L1_L2_L3 = [0, 12]
1081 * ADAPT_L0_L1_L2_L3 = 215 x 2^ADAPT_length_L0_L1_L2_L3
1082 * } else if (ADAPT_range_L0_L1_L2_L3 == FINE) {
1083 * ADAPT_length_L0_L1_L2_L3 = [0, 127]
1084 * TADAPT_L0_L1_L2_L3 = 215 x (ADAPT_length_L0_L1_L2_L3 + 1)
1085 * }
1086 *
1087 * To double the adaptation duration TADAPT_L0_L1_L2_L3:
1088 * 1. If adapt range is COARSE (1'b1), new adapt = old adapt + 1.
1089 * 2. If adapt range is FINE (1'b0):
1090 * a) If old adapt length is < 64, (new adapt + 1) = 2 * (old adapt + 1).
1091 * b) If old adapt length is >= 64, set new adapt to 0x88 using COARSE
1092 * range, because new adapt get from equation in a) shall exceed 127.
1093 *
1094 * Examples:
1095 * ADAPT_range_L0_L1_L2_L3 | ADAPT_length_L0_L1_L2_L3 | TADAPT_L0_L1_L2_L3 (PAM-4 UI)
1096 * 0 3 131072
1097 * 0 7 262144
1098 * 0 63 2097152
1099 * 0 64 2129920
1100 * 0 127 4194304
1101 * 1 8 8388608
1102 * 1 9 16777216
1103 * 1 10 33554432
1104 * 1 11 67108864
1105 * 1 12 134217728
1106 *
1107 * Return: new adapt.
1108 */
ufs_qcom_double_t_adapt_l0l1l2l3(u32 old_adapt)1109 static u32 ufs_qcom_double_t_adapt_l0l1l2l3(u32 old_adapt)
1110 {
1111 u32 adapt_length = old_adapt & ADAPT_LENGTH_MASK;
1112 u32 new_adapt;
1113
1114 if (IS_ADAPT_RANGE_COARSE(old_adapt)) {
1115 new_adapt = (adapt_length + 1) | ADAPT_RANGE_BIT;
1116 } else {
1117 if (adapt_length < 64)
1118 new_adapt = (adapt_length << 1) + 1;
1119 else
1120 /*
1121 * 0x88 is the very coarse Adapt value which is two
1122 * times of the largest fine Adapt value (0x7F)
1123 */
1124 new_adapt = 0x88;
1125 }
1126
1127 return new_adapt;
1128 }
1129
ufs_qcom_limit_max_gear(struct ufs_hba * hba,enum ufs_hs_gear_tag gear)1130 static void ufs_qcom_limit_max_gear(struct ufs_hba *hba,
1131 enum ufs_hs_gear_tag gear)
1132 {
1133 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1134 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
1135 struct ufs_host_params *host_params = &host->host_params;
1136
1137 host_params->hs_tx_gear = gear;
1138 host_params->hs_rx_gear = gear;
1139 pwr_info->gear_tx = gear;
1140 pwr_info->gear_rx = gear;
1141
1142 dev_warn(hba->dev, "Limited max gear of host and device to HS-G%d\n", gear);
1143 }
1144
ufs_qcom_fixup_tx_adapt_l0l1l2l3(struct ufs_hba * hba)1145 static void ufs_qcom_fixup_tx_adapt_l0l1l2l3(struct ufs_hba *hba)
1146 {
1147 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1148 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
1149 struct ufs_host_params *host_params = &host->host_params;
1150 u32 old_adapt, new_adapt, actual_adapt;
1151 bool limit_speed = false;
1152 int err;
1153
1154 if (host->hw_ver.major != 0x7 || host->hw_ver.minor > 0x1 ||
1155 host_params->hs_tx_gear <= UFS_HS_G5 ||
1156 pwr_info->gear_tx <= UFS_HS_G5)
1157 return;
1158
1159 err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTINITIALL0L1L2L3), &old_adapt);
1160 if (err)
1161 goto out;
1162
1163 if (old_adapt > ADAPT_L0L1L2L3_LENGTH_MAX) {
1164 dev_err(hba->dev, "PA_PeerRxHsG6AdaptInitialL0L1L2L3 value (0x%x) exceeds MAX\n",
1165 old_adapt);
1166 err = -ERANGE;
1167 goto out;
1168 }
1169
1170 new_adapt = ufs_qcom_double_t_adapt_l0l1l2l3(old_adapt);
1171 dev_dbg(hba->dev, "Original PA_PeerRxHsG6AdaptInitialL0L1L2L3 = 0x%x, new value = 0x%x\n",
1172 old_adapt, new_adapt);
1173
1174 /*
1175 * 0x8C is the max possible value allowed by UniPro v3.0 spec, some HWs
1176 * can accept 0x8D but some cannot.
1177 */
1178 if (new_adapt <= ADAPT_L0L1L2L3_LENGTH_MAX ||
1179 (new_adapt == ADAPT_L0L1L2L3_LENGTH_MAX + 1 && host->hw_ver.minor == 0x1)) {
1180 err = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTINITIALL0L1L2L3),
1181 new_adapt);
1182 if (err)
1183 goto out;
1184
1185 err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTINITIALL0L1L2L3),
1186 &actual_adapt);
1187 if (err)
1188 goto out;
1189
1190 if (actual_adapt != new_adapt) {
1191 limit_speed = true;
1192 dev_warn(hba->dev, "PA_PeerRxHsG6AdaptInitialL0L1L2L3 0x%x, expect 0x%x\n",
1193 actual_adapt, new_adapt);
1194 }
1195 } else {
1196 limit_speed = true;
1197 dev_warn(hba->dev, "New PA_PeerRxHsG6AdaptInitialL0L1L2L3 (0x%x) is too large!\n",
1198 new_adapt);
1199 }
1200
1201 err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTREFRESHL0L1L2L3), &old_adapt);
1202 if (err)
1203 goto out;
1204
1205 if (old_adapt > ADAPT_L0L1L2L3_LENGTH_MAX) {
1206 dev_err(hba->dev, "PA_PeerRxHsG6AdaptRefreshL0L1L2L3 value (0x%x) exceeds MAX\n",
1207 old_adapt);
1208 err = -ERANGE;
1209 goto out;
1210 }
1211
1212 new_adapt = ufs_qcom_double_t_adapt_l0l1l2l3(old_adapt);
1213 dev_dbg(hba->dev, "Original PA_PeerRxHsG6AdaptRefreshL0L1L2L3 = 0x%x, new value = 0x%x\n",
1214 old_adapt, new_adapt);
1215
1216 /*
1217 * 0x8C is the max possible value allowed by UniPro v3.0 spec, some HWs
1218 * can accept 0x8D but some cannot.
1219 */
1220 if (new_adapt <= ADAPT_L0L1L2L3_LENGTH_MAX ||
1221 (new_adapt == ADAPT_L0L1L2L3_LENGTH_MAX + 1 && host->hw_ver.minor == 0x1)) {
1222 err = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTREFRESHL0L1L2L3),
1223 new_adapt);
1224 if (err)
1225 goto out;
1226
1227 err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTREFRESHL0L1L2L3),
1228 &actual_adapt);
1229 if (err)
1230 goto out;
1231
1232 if (actual_adapt != new_adapt) {
1233 limit_speed = true;
1234 dev_warn(hba->dev, "PA_PeerRxHsG6AdaptRefreshL0L1L2L3 0x%x, expect 0x%x\n",
1235 new_adapt, actual_adapt);
1236 }
1237 } else {
1238 limit_speed = true;
1239 dev_warn(hba->dev, "New PA_PeerRxHsG6AdaptRefreshL0L1L2L3 (0x%x) is too large!\n",
1240 new_adapt);
1241 }
1242
1243 out:
1244 if (limit_speed || err)
1245 ufs_qcom_limit_max_gear(hba, UFS_HS_G5);
1246 }
1247
ufs_qcom_apply_dev_quirks(struct ufs_hba * hba)1248 static int ufs_qcom_apply_dev_quirks(struct ufs_hba *hba)
1249 {
1250 int err = 0;
1251
1252 ufs_qcom_fixup_tx_adapt_l0l1l2l3(hba);
1253
1254 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME)
1255 err = ufs_qcom_quirk_host_pa_saveconfigtime(hba);
1256
1257 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TX_HSG1_SYNC_LENGTH)
1258 ufs_qcom_override_pa_tx_hsg1_sync_len(hba);
1259
1260 return err;
1261 }
1262
1263 /* UFS device-specific quirks */
1264 static struct ufs_dev_quirk ufs_qcom_dev_fixups[] = {
1265 { .wmanufacturerid = UFS_VENDOR_SKHYNIX,
1266 .model = UFS_ANY_MODEL,
1267 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM },
1268 { .wmanufacturerid = UFS_VENDOR_WDC,
1269 .model = UFS_ANY_MODEL,
1270 .quirk = UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE },
1271 { .wmanufacturerid = UFS_VENDOR_SAMSUNG,
1272 .model = UFS_ANY_MODEL,
1273 .quirk = UFS_DEVICE_QUIRK_PA_TX_HSG1_SYNC_LENGTH |
1274 UFS_DEVICE_QUIRK_PA_TX_DEEMPHASIS_TUNING },
1275 {}
1276 };
1277
ufs_qcom_fixup_dev_quirks(struct ufs_hba * hba)1278 static void ufs_qcom_fixup_dev_quirks(struct ufs_hba *hba)
1279 {
1280 ufshcd_fixup_dev_quirks(hba, ufs_qcom_dev_fixups);
1281 }
1282
ufs_qcom_get_ufs_hci_version(struct ufs_hba * hba)1283 static u32 ufs_qcom_get_ufs_hci_version(struct ufs_hba *hba)
1284 {
1285 return ufshci_version(2, 0);
1286 }
1287
1288 /**
1289 * ufs_qcom_advertise_quirks - advertise the known QCOM UFS controller quirks
1290 * @hba: host controller instance
1291 *
1292 * QCOM UFS host controller might have some non standard behaviours (quirks)
1293 * than what is specified by UFSHCI specification. Advertise all such
1294 * quirks to standard UFS host controller driver so standard takes them into
1295 * account.
1296 */
ufs_qcom_advertise_quirks(struct ufs_hba * hba)1297 static void ufs_qcom_advertise_quirks(struct ufs_hba *hba)
1298 {
1299 const struct ufs_qcom_drvdata *drvdata = of_device_get_match_data(hba->dev);
1300 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1301
1302 if (host->hw_ver.major == 0x2)
1303 hba->quirks |= UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION;
1304
1305 if (host->hw_ver.major > 0x3)
1306 hba->quirks |= UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH;
1307
1308 if (drvdata && drvdata->quirks)
1309 hba->quirks |= drvdata->quirks;
1310 }
1311
ufs_qcom_set_phy_gear(struct ufs_qcom_host * host)1312 static void ufs_qcom_set_phy_gear(struct ufs_qcom_host *host)
1313 {
1314 struct ufs_host_params *host_params = &host->host_params;
1315 u32 val, dev_major;
1316
1317 /*
1318 * Default to powering up the PHY to the max gear possible, which is
1319 * backwards compatible with lower gears but not optimal from
1320 * a power usage point of view. After device negotiation, if the
1321 * gear is lower a reinit will be performed to program the PHY
1322 * to the ideal gear for this combo of controller and device.
1323 */
1324 host->phy_gear = host_params->hs_tx_gear;
1325
1326 if (host->hw_ver.major < 0x4) {
1327 /*
1328 * These controllers only have one PHY init sequence,
1329 * let's power up the PHY using that (the minimum supported
1330 * gear, UFS_HS_G2).
1331 */
1332 host->phy_gear = UFS_HS_G2;
1333 } else if (host->hw_ver.major >= 0x5) {
1334 val = ufshcd_readl(host->hba, REG_UFS_DEBUG_SPARE_CFG);
1335 dev_major = FIELD_GET(UFS_DEV_VER_MAJOR_MASK, val);
1336
1337 /*
1338 * Since the UFS device version is populated, let's remove the
1339 * REINIT quirk as the negotiated gear won't change during boot.
1340 * So there is no need to do reinit.
1341 */
1342 if (dev_major != 0x0)
1343 host->hba->quirks &= ~UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH;
1344
1345 /*
1346 * For UFS 3.1 device and older, power up the PHY using HS-G4
1347 * PHY gear to save power.
1348 */
1349 if (dev_major > 0x0 && dev_major < 0x4)
1350 host->phy_gear = UFS_HS_G4;
1351 }
1352 }
1353
ufs_qcom_parse_gear_limits(struct ufs_hba * hba)1354 static void ufs_qcom_parse_gear_limits(struct ufs_hba *hba)
1355 {
1356 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1357 struct ufs_host_params *host_params = &host->host_params;
1358 u32 hs_gear_old = host_params->hs_tx_gear;
1359
1360 ufshcd_parse_gear_limits(hba, host_params);
1361 if (host_params->hs_tx_gear != hs_gear_old) {
1362 host->phy_gear = host_params->hs_tx_gear;
1363 }
1364 }
1365
ufs_qcom_set_host_params(struct ufs_hba * hba)1366 static void ufs_qcom_set_host_params(struct ufs_hba *hba)
1367 {
1368 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1369 struct ufs_host_params *host_params = &host->host_params;
1370
1371 ufshcd_init_host_params(host_params);
1372
1373 /* This driver only supports symmetic gear setting i.e., hs_tx_gear == hs_rx_gear */
1374 host_params->hs_tx_gear = host_params->hs_rx_gear = ufs_qcom_get_hs_gear(hba);
1375 }
1376
ufs_qcom_set_host_caps(struct ufs_hba * hba)1377 static void ufs_qcom_set_host_caps(struct ufs_hba *hba)
1378 {
1379 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1380
1381 if (host->hw_ver.major >= 0x5)
1382 host->caps |= UFS_QCOM_CAP_ICE_CONFIG;
1383 }
1384
ufs_qcom_set_caps(struct ufs_hba * hba)1385 static void ufs_qcom_set_caps(struct ufs_hba *hba)
1386 {
1387 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1388
1389 hba->caps |= UFSHCD_CAP_CLK_GATING | UFSHCD_CAP_HIBERN8_WITH_CLK_GATING;
1390 hba->caps |= UFSHCD_CAP_CLK_SCALING | UFSHCD_CAP_WB_WITH_CLK_SCALING;
1391 hba->caps |= UFSHCD_CAP_AUTO_BKOPS_SUSPEND;
1392 hba->caps |= UFSHCD_CAP_WB_EN;
1393 hba->caps |= UFSHCD_CAP_AGGR_POWER_COLLAPSE;
1394 hba->caps |= UFSHCD_CAP_RPM_AUTOSUSPEND;
1395
1396 if (host->hw_ver.major >= 0x7)
1397 hba->caps |= UFSHCD_CAP_TX_EQUALIZATION;
1398
1399 ufs_qcom_set_host_caps(hba);
1400 }
1401
1402 /**
1403 * ufs_qcom_setup_clocks - enables/disable clocks
1404 * @hba: host controller instance
1405 * @on: If true, enable clocks else disable them.
1406 * @status: PRE_CHANGE or POST_CHANGE notify
1407 *
1408 * There are certain clocks which comes from the PHY so it needs
1409 * to be managed together along with controller clocks which also
1410 * provides a better power saving. Hence keep phy_power_off/on calls
1411 * in ufs_qcom_setup_clocks, so that PHY's regulators & clks can be
1412 * turned on/off along with UFS's clocks.
1413 *
1414 * Return: 0 on success, non-zero on failure.
1415 */
ufs_qcom_setup_clocks(struct ufs_hba * hba,bool on,enum ufs_notify_change_status status)1416 static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on,
1417 enum ufs_notify_change_status status)
1418 {
1419 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1420 struct phy *phy;
1421 int err;
1422
1423 /*
1424 * In case ufs_qcom_init() is not yet done, simply ignore.
1425 * This ufs_qcom_setup_clocks() shall be called from
1426 * ufs_qcom_init() after init is done.
1427 */
1428 if (!host)
1429 return 0;
1430
1431 phy = host->generic_phy;
1432
1433 switch (status) {
1434 case PRE_CHANGE:
1435 if (on) {
1436 ufs_qcom_icc_update_bw(host);
1437 if (ufs_qcom_is_link_hibern8(hba)) {
1438 err = ufs_qcom_enable_lane_clks(host);
1439 if (err) {
1440 dev_err(hba->dev, "enable lane clks failed, ret=%d\n", err);
1441 return err;
1442 }
1443 }
1444 } else {
1445 if (!ufs_qcom_is_link_active(hba)) {
1446 /* disable device ref_clk */
1447 ufs_qcom_dev_ref_clk_ctrl(host, false);
1448 }
1449
1450 err = phy_power_off(phy);
1451 if (err) {
1452 dev_err(hba->dev, "phy power off failed, ret=%d\n", err);
1453 return err;
1454 }
1455 }
1456 break;
1457 case POST_CHANGE:
1458 if (on) {
1459 err = phy_power_on(phy);
1460 if (err) {
1461 dev_err(hba->dev, "phy power on failed, ret = %d\n", err);
1462 return err;
1463 }
1464
1465 /* enable the device ref clock for HS mode*/
1466 if (ufshcd_is_hs_mode(&hba->pwr_info))
1467 ufs_qcom_dev_ref_clk_ctrl(host, true);
1468 } else {
1469 if (ufs_qcom_is_link_hibern8(hba))
1470 ufs_qcom_disable_lane_clks(host);
1471
1472 ufs_qcom_icc_set_bw(host, ufs_qcom_bw_table[MODE_MIN][0][0].mem_bw,
1473 ufs_qcom_bw_table[MODE_MIN][0][0].cfg_bw);
1474 }
1475 break;
1476 }
1477
1478 return 0;
1479 }
1480
1481 static int
ufs_qcom_reset_assert(struct reset_controller_dev * rcdev,unsigned long id)1482 ufs_qcom_reset_assert(struct reset_controller_dev *rcdev, unsigned long id)
1483 {
1484 struct ufs_qcom_host *host = rcdev_to_ufs_host(rcdev);
1485
1486 ufs_qcom_assert_reset(host->hba);
1487 /* provide 1ms delay to let the reset pulse propagate. */
1488 usleep_range(1000, 1100);
1489 return 0;
1490 }
1491
1492 static int
ufs_qcom_reset_deassert(struct reset_controller_dev * rcdev,unsigned long id)1493 ufs_qcom_reset_deassert(struct reset_controller_dev *rcdev, unsigned long id)
1494 {
1495 struct ufs_qcom_host *host = rcdev_to_ufs_host(rcdev);
1496
1497 ufs_qcom_deassert_reset(host->hba);
1498
1499 /*
1500 * after reset deassertion, phy will need all ref clocks,
1501 * voltage, current to settle down before starting serdes.
1502 */
1503 usleep_range(1000, 1100);
1504 return 0;
1505 }
1506
1507 static const struct reset_control_ops ufs_qcom_reset_ops = {
1508 .assert = ufs_qcom_reset_assert,
1509 .deassert = ufs_qcom_reset_deassert,
1510 };
1511
ufs_qcom_icc_init(struct ufs_qcom_host * host)1512 static int ufs_qcom_icc_init(struct ufs_qcom_host *host)
1513 {
1514 struct device *dev = host->hba->dev;
1515 int ret;
1516
1517 host->icc_ddr = devm_of_icc_get(dev, "ufs-ddr");
1518 if (IS_ERR(host->icc_ddr))
1519 return dev_err_probe(dev, PTR_ERR(host->icc_ddr),
1520 "failed to acquire interconnect path\n");
1521
1522 host->icc_cpu = devm_of_icc_get(dev, "cpu-ufs");
1523 if (IS_ERR(host->icc_cpu))
1524 return dev_err_probe(dev, PTR_ERR(host->icc_cpu),
1525 "failed to acquire interconnect path\n");
1526
1527 /*
1528 * Set Maximum bandwidth vote before initializing the UFS controller and
1529 * device. Ideally, a minimal interconnect vote would suffice for the
1530 * initialization, but a max vote would allow faster initialization.
1531 */
1532 ret = ufs_qcom_icc_set_bw(host, ufs_qcom_bw_table[MODE_MAX][0][0].mem_bw,
1533 ufs_qcom_bw_table[MODE_MAX][0][0].cfg_bw);
1534 if (ret < 0)
1535 return dev_err_probe(dev, ret, "failed to set bandwidth request\n");
1536
1537 return 0;
1538 }
1539
1540 /**
1541 * ufs_qcom_init - bind phy with controller
1542 * @hba: host controller instance
1543 *
1544 * Binds PHY with controller and powers up PHY enabling clocks
1545 * and regulators.
1546 *
1547 * Return: -EPROBE_DEFER if binding fails, returns negative error
1548 * on phy power up failure and returns zero on success.
1549 */
ufs_qcom_init(struct ufs_hba * hba)1550 static int ufs_qcom_init(struct ufs_hba *hba)
1551 {
1552 int err;
1553 struct device *dev = hba->dev;
1554 struct ufs_qcom_host *host;
1555 struct ufs_clk_info *clki;
1556 const struct ufs_qcom_drvdata *drvdata = of_device_get_match_data(hba->dev);
1557
1558 host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
1559 if (!host)
1560 return -ENOMEM;
1561
1562 /* Make a two way bind between the qcom host and the hba */
1563 host->hba = hba;
1564 ufshcd_set_variant(hba, host);
1565
1566 /* Setup the optional reset control of HCI */
1567 host->core_reset = devm_reset_control_get_optional(hba->dev, "rst");
1568 if (IS_ERR(host->core_reset)) {
1569 err = dev_err_probe(dev, PTR_ERR(host->core_reset),
1570 "Failed to get reset control\n");
1571 goto out_variant_clear;
1572 }
1573
1574 /* Fire up the reset controller. Failure here is non-fatal. */
1575 host->rcdev.of_node = dev->of_node;
1576 host->rcdev.ops = &ufs_qcom_reset_ops;
1577 host->rcdev.owner = dev->driver->owner;
1578 host->rcdev.nr_resets = 1;
1579 err = devm_reset_controller_register(dev, &host->rcdev);
1580 if (err)
1581 dev_warn(dev, "Failed to register reset controller\n");
1582
1583 if (!has_acpi_companion(dev)) {
1584 host->generic_phy = devm_phy_get(dev, "ufsphy");
1585 if (IS_ERR(host->generic_phy)) {
1586 err = dev_err_probe(dev, PTR_ERR(host->generic_phy), "Failed to get PHY\n");
1587 goto out_variant_clear;
1588 }
1589 }
1590
1591 err = ufs_qcom_icc_init(host);
1592 if (err)
1593 goto out_variant_clear;
1594
1595 host->device_reset = devm_gpiod_get_optional(dev, "reset",
1596 GPIOD_OUT_HIGH);
1597 if (IS_ERR(host->device_reset)) {
1598 err = dev_err_probe(dev, PTR_ERR(host->device_reset),
1599 "Failed to acquire device reset gpio\n");
1600 goto out_variant_clear;
1601 }
1602
1603 ufs_qcom_get_controller_revision(hba, &host->hw_ver.major,
1604 &host->hw_ver.minor, &host->hw_ver.step);
1605
1606 host->dev_ref_clk_ctrl_mmio = hba->mmio_base + REG_UFS_CFG1;
1607 host->dev_ref_clk_en_mask = BIT(26);
1608
1609 list_for_each_entry(clki, &hba->clk_list_head, list) {
1610 if (!strcmp(clki->name, "core_clk_unipro"))
1611 clki->keep_link_active = true;
1612 }
1613
1614 err = ufs_qcom_init_lane_clks(host);
1615 if (err)
1616 goto out_variant_clear;
1617
1618 ufs_qcom_set_caps(hba);
1619 ufs_qcom_advertise_quirks(hba);
1620 ufs_qcom_set_host_params(hba);
1621 ufs_qcom_set_phy_gear(host);
1622 ufs_qcom_parse_gear_limits(hba);
1623
1624 err = ufs_qcom_ice_init(host);
1625 if (err)
1626 goto out_variant_clear;
1627
1628 ufs_qcom_setup_clocks(hba, true, POST_CHANGE);
1629
1630 ufs_qcom_get_default_testbus_cfg(host);
1631 err = ufs_qcom_testbus_config(host);
1632 if (err)
1633 /* Failure is non-fatal */
1634 dev_warn(dev, "%s: failed to configure the testbus %d\n",
1635 __func__, err);
1636
1637 if (drvdata && drvdata->no_phy_retention)
1638 hba->spm_lvl = UFS_PM_LVL_5;
1639
1640 return 0;
1641
1642 out_variant_clear:
1643 ufshcd_set_variant(hba, NULL);
1644
1645 return err;
1646 }
1647
ufs_qcom_exit(struct ufs_hba * hba)1648 static void ufs_qcom_exit(struct ufs_hba *hba)
1649 {
1650 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1651
1652 ufs_qcom_disable_lane_clks(host);
1653 phy_power_off(host->generic_phy);
1654 phy_exit(host->generic_phy);
1655 }
1656
ufs_qcom_fw_managed_init(struct ufs_hba * hba)1657 static int ufs_qcom_fw_managed_init(struct ufs_hba *hba)
1658 {
1659 struct device *dev = hba->dev;
1660 struct ufs_qcom_host *host;
1661 int err;
1662
1663 host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
1664 if (!host)
1665 return -ENOMEM;
1666
1667 host->hba = hba;
1668 ufshcd_set_variant(hba, host);
1669
1670 ufs_qcom_get_controller_revision(hba, &host->hw_ver.major,
1671 &host->hw_ver.minor, &host->hw_ver.step);
1672
1673 err = ufs_qcom_ice_init(host);
1674 if (err)
1675 goto out_variant_clear;
1676
1677 ufs_qcom_get_default_testbus_cfg(host);
1678 err = ufs_qcom_testbus_config(host);
1679 if (err)
1680 /* Failure is non-fatal */
1681 dev_warn(dev, "Failed to configure the testbus %d\n", err);
1682
1683 hba->caps |= UFSHCD_CAP_WB_EN;
1684
1685 ufs_qcom_advertise_quirks(hba);
1686 host->hba->quirks &= ~UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH;
1687
1688 hba->spm_lvl = hba->rpm_lvl = hba->pm_lvl_min = UFS_PM_LVL_5;
1689
1690 ufs_qcom_set_host_params(hba);
1691 ufs_qcom_parse_gear_limits(hba);
1692
1693 return 0;
1694
1695 out_variant_clear:
1696 ufshcd_set_variant(hba, NULL);
1697 return err;
1698 }
1699
ufs_qcom_fw_managed_exit(struct ufs_hba * hba)1700 static void ufs_qcom_fw_managed_exit(struct ufs_hba *hba)
1701 {
1702 pm_runtime_put_sync(hba->dev);
1703 }
1704
1705 /**
1706 * ufs_qcom_set_clk_40ns_cycles - Configure 40ns clk cycles
1707 *
1708 * @hba: host controller instance
1709 * @cycles_in_1us: No of cycles in 1us to be configured
1710 *
1711 * Returns error if dme get/set configuration for 40ns fails
1712 * and returns zero on success.
1713 */
ufs_qcom_set_clk_40ns_cycles(struct ufs_hba * hba,u32 cycles_in_1us)1714 static int ufs_qcom_set_clk_40ns_cycles(struct ufs_hba *hba,
1715 u32 cycles_in_1us)
1716 {
1717 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1718 u32 cycles_in_40ns;
1719 u32 reg;
1720 int err;
1721
1722 /*
1723 * UFS host controller V4.0.0 onwards needs to program
1724 * PA_VS_CORE_CLK_40NS_CYCLES attribute per programmed
1725 * frequency of unipro core clk of UFS host controller.
1726 */
1727 if (host->hw_ver.major < 4)
1728 return 0;
1729
1730 /*
1731 * Generic formulae for cycles_in_40ns = (freq_unipro/25) is not
1732 * applicable for all frequencies. For ex: ceil(37.5 MHz/25) will
1733 * be 2 and ceil(403 MHZ/25) will be 17 whereas Hardware
1734 * specification expect to be 16. Hence use exact hardware spec
1735 * mandated value for cycles_in_40ns instead of calculating using
1736 * generic formulae.
1737 */
1738 switch (cycles_in_1us) {
1739 case UNIPRO_CORE_CLK_FREQ_403_MHZ:
1740 cycles_in_40ns = 16;
1741 break;
1742 case UNIPRO_CORE_CLK_FREQ_300_MHZ:
1743 cycles_in_40ns = 12;
1744 break;
1745 case UNIPRO_CORE_CLK_FREQ_201_5_MHZ:
1746 cycles_in_40ns = 8;
1747 break;
1748 case UNIPRO_CORE_CLK_FREQ_150_MHZ:
1749 cycles_in_40ns = 6;
1750 break;
1751 case UNIPRO_CORE_CLK_FREQ_100_MHZ:
1752 cycles_in_40ns = 4;
1753 break;
1754 case UNIPRO_CORE_CLK_FREQ_75_MHZ:
1755 cycles_in_40ns = 3;
1756 break;
1757 case UNIPRO_CORE_CLK_FREQ_37_5_MHZ:
1758 cycles_in_40ns = 2;
1759 break;
1760 default:
1761 dev_err(hba->dev, "UNIPRO clk freq %u MHz not supported\n",
1762 cycles_in_1us);
1763 return -EINVAL;
1764 }
1765
1766 err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_VS_CORE_CLK_40NS_CYCLES), ®);
1767 if (err)
1768 return err;
1769
1770 reg &= ~PA_VS_CORE_CLK_40NS_CYCLES_MASK;
1771 reg |= cycles_in_40ns;
1772
1773 return ufshcd_dme_set(hba, UIC_ARG_MIB(PA_VS_CORE_CLK_40NS_CYCLES), reg);
1774 }
1775
ufs_qcom_set_core_clk_ctrl(struct ufs_hba * hba,bool is_scale_up,unsigned long freq)1776 static int ufs_qcom_set_core_clk_ctrl(struct ufs_hba *hba, bool is_scale_up, unsigned long freq)
1777 {
1778 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1779 struct list_head *head = &hba->clk_list_head;
1780 struct ufs_clk_info *clki;
1781 u32 cycles_in_1us = 0;
1782 u32 core_clk_ctrl_reg;
1783 unsigned long clk_freq;
1784 int err;
1785
1786 if (hba->use_pm_opp && freq != ULONG_MAX) {
1787 clk_freq = ufs_qcom_opp_freq_to_clk_freq(hba, freq, "core_clk_unipro");
1788 if (clk_freq) {
1789 cycles_in_1us = ceil(clk_freq, HZ_PER_MHZ);
1790 goto set_core_clk_ctrl;
1791 }
1792 }
1793
1794 list_for_each_entry(clki, head, list) {
1795 if (!IS_ERR_OR_NULL(clki->clk) &&
1796 !strcmp(clki->name, "core_clk_unipro")) {
1797 if (!clki->max_freq) {
1798 cycles_in_1us = 150; /* default for backwards compatibility */
1799 break;
1800 }
1801
1802 if (freq == ULONG_MAX) {
1803 cycles_in_1us = ceil(clki->max_freq, HZ_PER_MHZ);
1804 break;
1805 }
1806
1807 if (is_scale_up)
1808 cycles_in_1us = ceil(clki->max_freq, HZ_PER_MHZ);
1809 else
1810 cycles_in_1us = ceil(clk_get_rate(clki->clk), HZ_PER_MHZ);
1811 break;
1812 }
1813 }
1814
1815 set_core_clk_ctrl:
1816 err = ufshcd_dme_get(hba,
1817 UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL),
1818 &core_clk_ctrl_reg);
1819 if (err)
1820 return err;
1821
1822 /* Bit mask is different for UFS host controller V4.0.0 onwards */
1823 if (host->hw_ver.major >= 4) {
1824 if (!FIELD_FIT(CLK_1US_CYCLES_MASK_V4, cycles_in_1us))
1825 return -ERANGE;
1826 core_clk_ctrl_reg &= ~CLK_1US_CYCLES_MASK_V4;
1827 core_clk_ctrl_reg |= FIELD_PREP(CLK_1US_CYCLES_MASK_V4, cycles_in_1us);
1828 } else {
1829 if (!FIELD_FIT(CLK_1US_CYCLES_MASK, cycles_in_1us))
1830 return -ERANGE;
1831 core_clk_ctrl_reg &= ~CLK_1US_CYCLES_MASK;
1832 core_clk_ctrl_reg |= FIELD_PREP(CLK_1US_CYCLES_MASK, cycles_in_1us);
1833 }
1834
1835 /* Clear CORE_CLK_DIV_EN */
1836 core_clk_ctrl_reg &= ~DME_VS_CORE_CLK_CTRL_CORE_CLK_DIV_EN_BIT;
1837
1838 err = ufshcd_dme_set(hba,
1839 UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL),
1840 core_clk_ctrl_reg);
1841 if (err)
1842 return err;
1843
1844 /* Configure unipro core clk 40ns attribute */
1845 return ufs_qcom_set_clk_40ns_cycles(hba, cycles_in_1us);
1846 }
1847
ufs_qcom_clk_scale_up_pre_change(struct ufs_hba * hba,unsigned long freq)1848 static int ufs_qcom_clk_scale_up_pre_change(struct ufs_hba *hba, unsigned long freq)
1849 {
1850 int ret;
1851
1852 ret = ufs_qcom_cfg_timers(hba, true, freq);
1853 if (ret) {
1854 dev_err(hba->dev, "%s ufs cfg timer failed\n", __func__);
1855 return ret;
1856 }
1857 /* set unipro core clock attributes and clear clock divider */
1858 return ufs_qcom_set_core_clk_ctrl(hba, true, freq);
1859 }
1860
ufs_qcom_clk_scale_up_post_change(struct ufs_hba * hba)1861 static int ufs_qcom_clk_scale_up_post_change(struct ufs_hba *hba)
1862 {
1863 return 0;
1864 }
1865
ufs_qcom_clk_scale_down_pre_change(struct ufs_hba * hba)1866 static int ufs_qcom_clk_scale_down_pre_change(struct ufs_hba *hba)
1867 {
1868 int err;
1869 u32 core_clk_ctrl_reg;
1870
1871 err = ufshcd_dme_get(hba,
1872 UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL),
1873 &core_clk_ctrl_reg);
1874
1875 /* make sure CORE_CLK_DIV_EN is cleared */
1876 if (!err &&
1877 (core_clk_ctrl_reg & DME_VS_CORE_CLK_CTRL_CORE_CLK_DIV_EN_BIT)) {
1878 core_clk_ctrl_reg &= ~DME_VS_CORE_CLK_CTRL_CORE_CLK_DIV_EN_BIT;
1879 err = ufshcd_dme_set(hba,
1880 UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL),
1881 core_clk_ctrl_reg);
1882 }
1883
1884 return err;
1885 }
1886
ufs_qcom_clk_scale_down_post_change(struct ufs_hba * hba,unsigned long freq)1887 static int ufs_qcom_clk_scale_down_post_change(struct ufs_hba *hba, unsigned long freq)
1888 {
1889 int ret;
1890
1891 ret = ufs_qcom_cfg_timers(hba, false, freq);
1892 if (ret) {
1893 dev_err(hba->dev, "%s: ufs_qcom_cfg_timers() failed\n", __func__);
1894 return ret;
1895 }
1896 /* set unipro core clock attributes and clear clock divider */
1897 return ufs_qcom_set_core_clk_ctrl(hba, false, freq);
1898 }
1899
ufs_qcom_clk_scale_notify(struct ufs_hba * hba,bool scale_up,unsigned long target_freq,enum ufs_notify_change_status status)1900 static int ufs_qcom_clk_scale_notify(struct ufs_hba *hba, bool scale_up,
1901 unsigned long target_freq,
1902 enum ufs_notify_change_status status)
1903 {
1904 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1905 int err;
1906
1907 /* check the host controller state before sending hibern8 cmd */
1908 if (!ufshcd_is_hba_active(hba))
1909 return 0;
1910
1911 if (status == PRE_CHANGE) {
1912 err = ufshcd_uic_hibern8_enter(hba);
1913 if (err)
1914 return err;
1915 if (scale_up)
1916 err = ufs_qcom_clk_scale_up_pre_change(hba, target_freq);
1917 else
1918 err = ufs_qcom_clk_scale_down_pre_change(hba);
1919
1920 if (err) {
1921 ufshcd_uic_hibern8_exit(hba);
1922 return err;
1923 }
1924 } else {
1925 if (scale_up)
1926 err = ufs_qcom_clk_scale_up_post_change(hba);
1927 else
1928 err = ufs_qcom_clk_scale_down_post_change(hba, target_freq);
1929
1930
1931 if (err) {
1932 ufshcd_uic_hibern8_exit(hba);
1933 return err;
1934 }
1935
1936 ufs_qcom_icc_update_bw(host);
1937 ufshcd_uic_hibern8_exit(hba);
1938 }
1939
1940 return 0;
1941 }
1942
ufs_qcom_enable_test_bus(struct ufs_qcom_host * host)1943 static void ufs_qcom_enable_test_bus(struct ufs_qcom_host *host)
1944 {
1945 ufshcd_rmwl(host->hba, UFS_REG_TEST_BUS_EN,
1946 UFS_REG_TEST_BUS_EN, REG_UFS_CFG1);
1947 ufshcd_rmwl(host->hba, TEST_BUS_EN, TEST_BUS_EN, REG_UFS_CFG1);
1948 }
1949
ufs_qcom_get_default_testbus_cfg(struct ufs_qcom_host * host)1950 static void ufs_qcom_get_default_testbus_cfg(struct ufs_qcom_host *host)
1951 {
1952 /* provide a legal default configuration */
1953 host->testbus.select_major = TSTBUS_UNIPRO;
1954 host->testbus.select_minor = 37;
1955 }
1956
ufs_qcom_testbus_cfg_is_ok(struct ufs_qcom_host * host)1957 static bool ufs_qcom_testbus_cfg_is_ok(struct ufs_qcom_host *host)
1958 {
1959 if (host->testbus.select_major >= TSTBUS_MAX) {
1960 dev_err(host->hba->dev,
1961 "%s: UFS_CFG1[TEST_BUS_SEL} may not equal 0x%05X\n",
1962 __func__, host->testbus.select_major);
1963 return false;
1964 }
1965
1966 return true;
1967 }
1968
ufs_qcom_testbus_config(struct ufs_qcom_host * host)1969 int ufs_qcom_testbus_config(struct ufs_qcom_host *host)
1970 {
1971 int reg;
1972 int offset;
1973 u32 mask = TEST_BUS_SUB_SEL_MASK;
1974
1975 if (!host)
1976 return -EINVAL;
1977
1978 if (!ufs_qcom_testbus_cfg_is_ok(host))
1979 return -EPERM;
1980
1981 switch (host->testbus.select_major) {
1982 case TSTBUS_UAWM:
1983 reg = UFS_TEST_BUS_CTRL_0;
1984 offset = 24;
1985 break;
1986 case TSTBUS_UARM:
1987 reg = UFS_TEST_BUS_CTRL_0;
1988 offset = 16;
1989 break;
1990 case TSTBUS_TXUC:
1991 reg = UFS_TEST_BUS_CTRL_0;
1992 offset = 8;
1993 break;
1994 case TSTBUS_RXUC:
1995 reg = UFS_TEST_BUS_CTRL_0;
1996 offset = 0;
1997 break;
1998 case TSTBUS_DFC:
1999 reg = UFS_TEST_BUS_CTRL_1;
2000 offset = 24;
2001 break;
2002 case TSTBUS_TRLUT:
2003 reg = UFS_TEST_BUS_CTRL_1;
2004 offset = 16;
2005 break;
2006 case TSTBUS_TMRLUT:
2007 reg = UFS_TEST_BUS_CTRL_1;
2008 offset = 8;
2009 break;
2010 case TSTBUS_OCSC:
2011 reg = UFS_TEST_BUS_CTRL_1;
2012 offset = 0;
2013 break;
2014 case TSTBUS_WRAPPER:
2015 reg = UFS_TEST_BUS_CTRL_2;
2016 offset = 16;
2017 break;
2018 case TSTBUS_COMBINED:
2019 reg = UFS_TEST_BUS_CTRL_2;
2020 offset = 8;
2021 break;
2022 case TSTBUS_UTP_HCI:
2023 reg = UFS_TEST_BUS_CTRL_2;
2024 offset = 0;
2025 break;
2026 case TSTBUS_UNIPRO:
2027 reg = UFS_UNIPRO_CFG;
2028 offset = 20;
2029 mask = 0xFFF;
2030 break;
2031 /*
2032 * No need for a default case, since
2033 * ufs_qcom_testbus_cfg_is_ok() checks that the configuration
2034 * is legal
2035 */
2036 }
2037 mask <<= offset;
2038 ufshcd_rmwl(host->hba, TEST_BUS_SEL,
2039 (u32)host->testbus.select_major << 19,
2040 REG_UFS_CFG1);
2041 ufshcd_rmwl(host->hba, mask,
2042 (u32)host->testbus.select_minor << offset,
2043 reg);
2044 ufs_qcom_enable_test_bus(host);
2045
2046 return 0;
2047 }
2048
ufs_qcom_dump_testbus(struct ufs_hba * hba)2049 static void ufs_qcom_dump_testbus(struct ufs_hba *hba)
2050 {
2051 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
2052 int i, j, nminor = 0, testbus_len = 0;
2053 char *prefix;
2054
2055 u32 *testbus __free(kfree) = kmalloc_array(256, sizeof(u32), GFP_KERNEL);
2056 if (!testbus)
2057 return;
2058
2059 for (j = 0; j < TSTBUS_MAX; j++) {
2060 nminor = testbus_info[j].nminor;
2061 prefix = testbus_info[j].prefix;
2062 host->testbus.select_major = j;
2063 testbus_len = nminor * sizeof(u32);
2064 for (i = 0; i < nminor; i++) {
2065 host->testbus.select_minor = i;
2066 ufs_qcom_testbus_config(host);
2067 testbus[i] = ufshcd_readl(hba, UFS_TEST_BUS);
2068 }
2069 print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET,
2070 16, 4, testbus, testbus_len, false);
2071 }
2072 }
2073
ufs_qcom_dump_regs(struct ufs_hba * hba,size_t offset,size_t len,const char * prefix,void __iomem * base)2074 static int ufs_qcom_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
2075 const char *prefix, void __iomem *base)
2076 {
2077 size_t pos;
2078
2079 if (offset % 4 != 0 || len % 4 != 0)
2080 return -EINVAL;
2081
2082 u32 *regs __free(kfree) = kzalloc(len, GFP_ATOMIC);
2083 if (!regs)
2084 return -ENOMEM;
2085
2086 for (pos = 0; pos < len; pos += 4)
2087 regs[pos / 4] = readl(base + offset + pos);
2088
2089 print_hex_dump(KERN_ERR, prefix,
2090 len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,
2091 16, 4, regs, len, false);
2092
2093 return 0;
2094 }
2095
ufs_qcom_dump_mcq_hci_regs(struct ufs_hba * hba)2096 static void ufs_qcom_dump_mcq_hci_regs(struct ufs_hba *hba)
2097 {
2098 struct ufshcd_mcq_opr_info_t *opr = &hba->mcq_opr[0];
2099 void __iomem *mcq_vs_base = hba->mcq_base + UFS_MEM_VS_BASE;
2100
2101 struct dump_info {
2102 void __iomem *base;
2103 size_t offset;
2104 size_t len;
2105 const char *prefix;
2106 };
2107
2108 struct dump_info mcq_dumps[] = {
2109 {hba->mcq_base, 0x0, 256 * 4, "MCQ HCI-0 "},
2110 {hba->mcq_base, 0x400, 256 * 4, "MCQ HCI-1 "},
2111 {mcq_vs_base, 0x0, 5 * 4, "MCQ VS-0 "},
2112 {opr->base, 0x0, 256 * 4, "MCQ SQD-0 "},
2113 {opr->base, 0x400, 256 * 4, "MCQ SQD-1 "},
2114 {opr->base, 0x800, 256 * 4, "MCQ SQD-2 "},
2115 {opr->base, 0xc00, 256 * 4, "MCQ SQD-3 "},
2116 {opr->base, 0x1000, 256 * 4, "MCQ SQD-4 "},
2117 {opr->base, 0x1400, 256 * 4, "MCQ SQD-5 "},
2118 {opr->base, 0x1800, 256 * 4, "MCQ SQD-6 "},
2119 {opr->base, 0x1c00, 256 * 4, "MCQ SQD-7 "},
2120
2121 };
2122
2123 for (int i = 0; i < ARRAY_SIZE(mcq_dumps); i++) {
2124 ufs_qcom_dump_regs(hba, mcq_dumps[i].offset, mcq_dumps[i].len,
2125 mcq_dumps[i].prefix, mcq_dumps[i].base);
2126 cond_resched();
2127 }
2128 }
2129
ufs_qcom_dump_dbg_regs(struct ufs_hba * hba)2130 static void ufs_qcom_dump_dbg_regs(struct ufs_hba *hba)
2131 {
2132 u32 reg;
2133 struct ufs_qcom_host *host;
2134
2135 host = ufshcd_get_variant(hba);
2136
2137 dev_err(hba->dev, "HW_H8_ENTER_CNT=%d\n", ufshcd_readl(hba, REG_UFS_HW_H8_ENTER_CNT));
2138 dev_err(hba->dev, "HW_H8_EXIT_CNT=%d\n", ufshcd_readl(hba, REG_UFS_HW_H8_EXIT_CNT));
2139
2140 dev_err(hba->dev, "SW_H8_ENTER_CNT=%d\n", ufshcd_readl(hba, REG_UFS_SW_H8_ENTER_CNT));
2141 dev_err(hba->dev, "SW_H8_EXIT_CNT=%d\n", ufshcd_readl(hba, REG_UFS_SW_H8_EXIT_CNT));
2142
2143 dev_err(hba->dev, "SW_AFTER_HW_H8_ENTER_CNT=%d\n",
2144 ufshcd_readl(hba, REG_UFS_SW_AFTER_HW_H8_ENTER_CNT));
2145
2146 ufshcd_dump_regs(hba, REG_UFS_SYS1CLK_1US, 16 * 4,
2147 "HCI Vendor Specific Registers ");
2148
2149 reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_REG_OCSC);
2150 ufshcd_dump_regs(hba, reg, 44 * 4, "UFS_UFS_DBG_RD_REG_OCSC ");
2151
2152 reg = ufshcd_readl(hba, REG_UFS_CFG1);
2153 reg |= UTP_DBG_RAMS_EN;
2154 ufshcd_writel(hba, reg, REG_UFS_CFG1);
2155
2156 reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_EDTL_RAM);
2157 ufshcd_dump_regs(hba, reg, 32 * 4, "UFS_UFS_DBG_RD_EDTL_RAM ");
2158
2159 reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_DESC_RAM);
2160 ufshcd_dump_regs(hba, reg, 128 * 4, "UFS_UFS_DBG_RD_DESC_RAM ");
2161
2162 reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_PRDT_RAM);
2163 ufshcd_dump_regs(hba, reg, 64 * 4, "UFS_UFS_DBG_RD_PRDT_RAM ");
2164
2165 /* clear bit 17 - UTP_DBG_RAMS_EN */
2166 ufshcd_rmwl(hba, UTP_DBG_RAMS_EN, 0, REG_UFS_CFG1);
2167
2168 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_UAWM);
2169 ufshcd_dump_regs(hba, reg, 4 * 4, "UFS_DBG_RD_REG_UAWM ");
2170
2171 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_UARM);
2172 ufshcd_dump_regs(hba, reg, 4 * 4, "UFS_DBG_RD_REG_UARM ");
2173
2174 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_TXUC);
2175 ufshcd_dump_regs(hba, reg, 48 * 4, "UFS_DBG_RD_REG_TXUC ");
2176
2177 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_RXUC);
2178 ufshcd_dump_regs(hba, reg, 27 * 4, "UFS_DBG_RD_REG_RXUC ");
2179
2180 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_DFC);
2181 ufshcd_dump_regs(hba, reg, 19 * 4, "UFS_DBG_RD_REG_DFC ");
2182
2183 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_TRLUT);
2184 ufshcd_dump_regs(hba, reg, 34 * 4, "UFS_DBG_RD_REG_TRLUT ");
2185
2186 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_TMRLUT);
2187 ufshcd_dump_regs(hba, reg, 9 * 4, "UFS_DBG_RD_REG_TMRLUT ");
2188
2189 if (hba->mcq_enabled) {
2190 reg = ufs_qcom_get_debug_reg_offset(host, UFS_RD_REG_MCQ);
2191 ufshcd_dump_regs(hba, reg, 64 * 4, "HCI MCQ Debug Registers ");
2192 }
2193
2194 /* ensure below dumps occur only in task context due to blocking calls. */
2195 if (in_task()) {
2196 /* Dump MCQ Host Vendor Specific Registers */
2197 if (hba->mcq_enabled)
2198 ufs_qcom_dump_mcq_hci_regs(hba);
2199
2200 /* voluntarily yield the CPU as we are dumping too much data */
2201 ufshcd_dump_regs(hba, UFS_TEST_BUS, 4, "UFS_TEST_BUS ");
2202 cond_resched();
2203 ufs_qcom_dump_testbus(hba);
2204 }
2205 }
2206
2207 /**
2208 * ufs_qcom_device_reset() - toggle the (optional) device reset line
2209 * @hba: per-adapter instance
2210 *
2211 * Toggles the (optional) reset line to reset the attached device.
2212 */
ufs_qcom_device_reset(struct ufs_hba * hba)2213 static int ufs_qcom_device_reset(struct ufs_hba *hba)
2214 {
2215 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
2216
2217 /* reset gpio is optional */
2218 if (!host->device_reset)
2219 return -EOPNOTSUPP;
2220
2221 /*
2222 * The UFS device shall detect reset pulses of 1us, sleep for 10us to
2223 * be on the safe side.
2224 */
2225 ufs_qcom_device_reset_ctrl(hba, true);
2226 usleep_range(10, 15);
2227
2228 ufs_qcom_device_reset_ctrl(hba, false);
2229 usleep_range(10, 15);
2230
2231 return 0;
2232 }
2233
2234 /**
2235 * ufs_qcom_fw_managed_device_reset - Reset UFS device under FW-managed design
2236 * @hba: pointer to UFS host bus adapter
2237 *
2238 * In the firmware-managed reset model, the power domain is powered on by genpd
2239 * before the UFS controller driver probes. For subsequent resets (such as
2240 * suspend/resume or recovery), the UFS driver must explicitly invoke PM runtime
2241 *
2242 * Return: 0 on success or a negative error code on failure.
2243 */
ufs_qcom_fw_managed_device_reset(struct ufs_hba * hba)2244 static int ufs_qcom_fw_managed_device_reset(struct ufs_hba *hba)
2245 {
2246 static bool is_boot = true;
2247 int err;
2248
2249 /* Skip reset on cold boot; perform it on subsequent calls */
2250 if (is_boot) {
2251 is_boot = false;
2252 return 0;
2253 }
2254
2255 pm_runtime_put_sync(hba->dev);
2256 err = pm_runtime_resume_and_get(hba->dev);
2257 if (err < 0) {
2258 dev_err(hba->dev, "PM runtime resume failed: %d\n", err);
2259 return err;
2260 }
2261
2262 return 0;
2263 }
2264
ufs_qcom_config_scaling_param(struct ufs_hba * hba,struct devfreq_dev_profile * p,struct devfreq_simple_ondemand_data * d)2265 static void ufs_qcom_config_scaling_param(struct ufs_hba *hba,
2266 struct devfreq_dev_profile *p,
2267 struct devfreq_simple_ondemand_data *d)
2268 {
2269 p->polling_ms = 60;
2270 p->timer = DEVFREQ_TIMER_DELAYED;
2271 d->upthreshold = 70;
2272 d->downdifferential = 5;
2273
2274 hba->clk_scaling.suspend_on_no_request = true;
2275 }
2276
ufs_qcom_mcq_config_resource(struct ufs_hba * hba)2277 static int ufs_qcom_mcq_config_resource(struct ufs_hba *hba)
2278 {
2279 struct platform_device *pdev = to_platform_device(hba->dev);
2280 struct resource *res;
2281
2282 /* Map the MCQ configuration region */
2283 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mcq");
2284 if (!res) {
2285 dev_err(hba->dev, "MCQ resource not found in device tree\n");
2286 return -ENODEV;
2287 }
2288
2289 hba->mcq_base = devm_ioremap_resource(hba->dev, res);
2290 if (IS_ERR(hba->mcq_base)) {
2291 dev_err(hba->dev, "Failed to map MCQ region: %ld\n",
2292 PTR_ERR(hba->mcq_base));
2293 return PTR_ERR(hba->mcq_base);
2294 }
2295
2296 return 0;
2297 }
2298
ufs_qcom_op_runtime_config(struct ufs_hba * hba)2299 static int ufs_qcom_op_runtime_config(struct ufs_hba *hba)
2300 {
2301 struct ufshcd_mcq_opr_info_t *opr;
2302 int i;
2303 u32 doorbell_offsets[OPR_MAX];
2304
2305 /*
2306 * Configure doorbell address offsets in MCQ configuration registers.
2307 * These values are offsets relative to mmio_base (UFS_HCI_BASE).
2308 *
2309 * Memory Layout:
2310 * - mmio_base = UFS_HCI_BASE
2311 * - mcq_base = MCQ_CONFIG_BASE = mmio_base + (UFS_QCOM_MCQCAP_QCFGPTR * 0x200)
2312 * - Doorbell registers are at: mmio_base + (UFS_QCOM_MCQCAP_QCFGPTR * 0x200) +
2313 * - UFS_QCOM_MCQ_SQD_OFFSET
2314 * - Which is also: mcq_base + UFS_QCOM_MCQ_SQD_OFFSET
2315 */
2316
2317 doorbell_offsets[OPR_SQD] = UFS_QCOM_SQD_ADDR_OFFSET;
2318 doorbell_offsets[OPR_SQIS] = UFS_QCOM_SQIS_ADDR_OFFSET;
2319 doorbell_offsets[OPR_CQD] = UFS_QCOM_CQD_ADDR_OFFSET;
2320 doorbell_offsets[OPR_CQIS] = UFS_QCOM_CQIS_ADDR_OFFSET;
2321
2322 /*
2323 * Configure MCQ operation registers.
2324 *
2325 * The doorbell registers are physically located within the MCQ region:
2326 * - doorbell_physical_addr = mmio_base + doorbell_offset
2327 * - doorbell_physical_addr = mcq_base + (doorbell_offset - MCQ_CONFIG_OFFSET)
2328 */
2329 for (i = 0; i < OPR_MAX; i++) {
2330 opr = &hba->mcq_opr[i];
2331 opr->offset = doorbell_offsets[i]; /* Offset relative to mmio_base */
2332 opr->stride = UFS_QCOM_MCQ_STRIDE; /* 256 bytes between queues */
2333
2334 /*
2335 * Calculate the actual doorbell base address within MCQ region:
2336 * base = mcq_base + (doorbell_offset - MCQ_CONFIG_OFFSET)
2337 */
2338 opr->base = hba->mcq_base + (opr->offset - UFS_QCOM_MCQ_CONFIG_OFFSET);
2339 }
2340
2341 return 0;
2342 }
2343
ufs_qcom_get_hba_mac(struct ufs_hba * hba)2344 static int ufs_qcom_get_hba_mac(struct ufs_hba *hba)
2345 {
2346 /* Qualcomm HC supports up to 64 */
2347 return MAX_SUPP_MAC;
2348 }
2349
ufs_qcom_get_outstanding_cqs(struct ufs_hba * hba,unsigned long * ocqs)2350 static int ufs_qcom_get_outstanding_cqs(struct ufs_hba *hba,
2351 unsigned long *ocqs)
2352 {
2353 /* Read from MCQ vendor-specific register in MCQ region */
2354 *ocqs = readl(hba->mcq_base + UFS_MEM_CQIS_VS);
2355
2356 return 0;
2357 }
2358
ufs_qcom_write_msi_msg(struct msi_desc * desc,struct msi_msg * msg)2359 static void ufs_qcom_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
2360 {
2361 struct device *dev = msi_desc_to_dev(desc);
2362 struct ufs_hba *hba = dev_get_drvdata(dev);
2363
2364 ufshcd_mcq_config_esi(hba, msg);
2365 }
2366
2367 struct ufs_qcom_irq {
2368 unsigned int irq;
2369 unsigned int idx;
2370 struct ufs_hba *hba;
2371 };
2372
ufs_qcom_mcq_esi_handler(int irq,void * data)2373 static irqreturn_t ufs_qcom_mcq_esi_handler(int irq, void *data)
2374 {
2375 struct ufs_qcom_irq *qi = data;
2376 struct ufs_hba *hba = qi->hba;
2377 struct ufs_hw_queue *hwq = &hba->uhq[qi->idx];
2378
2379 ufshcd_mcq_write_cqis(hba, 0x1, qi->idx);
2380 ufshcd_mcq_poll_cqe_lock(hba, hwq);
2381
2382 return IRQ_HANDLED;
2383 }
2384
ufs_qcom_config_esi(struct ufs_hba * hba)2385 static int ufs_qcom_config_esi(struct ufs_hba *hba)
2386 {
2387 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
2388 int nr_irqs, ret;
2389
2390 if (host->esi_enabled)
2391 return 0;
2392
2393 /*
2394 * 1. We only handle CQs as of now.
2395 * 2. Poll queues do not need ESI.
2396 */
2397 nr_irqs = hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL];
2398
2399 ret = platform_device_msi_init_and_alloc_irqs(hba->dev, nr_irqs,
2400 ufs_qcom_write_msi_msg);
2401 if (ret) {
2402 dev_warn(hba->dev, "Platform MSI not supported or failed, continuing without ESI\n");
2403 return ret; /* Continue without ESI */
2404 }
2405
2406 struct ufs_qcom_irq *qi = devm_kcalloc(hba->dev, nr_irqs, sizeof(*qi), GFP_KERNEL);
2407
2408 if (!qi) {
2409 platform_device_msi_free_irqs_all(hba->dev);
2410 return -ENOMEM;
2411 }
2412
2413 for (int idx = 0; idx < nr_irqs; idx++) {
2414 qi[idx].irq = msi_get_virq(hba->dev, idx);
2415 qi[idx].idx = idx;
2416 qi[idx].hba = hba;
2417
2418 ret = devm_request_irq(hba->dev, qi[idx].irq, ufs_qcom_mcq_esi_handler,
2419 IRQF_SHARED, "qcom-mcq-esi", qi + idx);
2420 if (ret) {
2421 dev_err(hba->dev, "%s: Failed to request IRQ for %d, err = %d\n",
2422 __func__, qi[idx].irq, ret);
2423 /* Free previously allocated IRQs */
2424 for (int j = 0; j < idx; j++)
2425 devm_free_irq(hba->dev, qi[j].irq, qi + j);
2426 platform_device_msi_free_irqs_all(hba->dev);
2427 devm_kfree(hba->dev, qi);
2428 return ret;
2429 }
2430 }
2431
2432 if (host->hw_ver.major >= 6) {
2433 ufshcd_rmwl(hba, ESI_VEC_MASK, FIELD_PREP(ESI_VEC_MASK, MAX_ESI_VEC - 1),
2434 REG_UFS_CFG3);
2435 }
2436 ufshcd_mcq_enable_esi(hba);
2437 host->esi_enabled = true;
2438 return 0;
2439 }
2440
ufs_qcom_opp_freq_to_clk_freq(struct ufs_hba * hba,unsigned long freq,char * name)2441 static unsigned long ufs_qcom_opp_freq_to_clk_freq(struct ufs_hba *hba,
2442 unsigned long freq, char *name)
2443 {
2444 struct ufs_clk_info *clki;
2445 struct dev_pm_opp *opp;
2446 unsigned long clk_freq;
2447 int idx = 0;
2448 bool found = false;
2449
2450 opp = dev_pm_opp_find_freq_exact_indexed(hba->dev, freq, 0, true);
2451 if (IS_ERR(opp)) {
2452 dev_err(hba->dev, "Failed to find OPP for exact frequency %lu\n", freq);
2453 return 0;
2454 }
2455
2456 list_for_each_entry(clki, &hba->clk_list_head, list) {
2457 if (!strcmp(clki->name, name)) {
2458 found = true;
2459 break;
2460 }
2461
2462 idx++;
2463 }
2464
2465 if (!found) {
2466 dev_err(hba->dev, "Failed to find clock '%s' in clk list\n", name);
2467 dev_pm_opp_put(opp);
2468 return 0;
2469 }
2470
2471 clk_freq = dev_pm_opp_get_freq_indexed(opp, idx);
2472
2473 dev_pm_opp_put(opp);
2474
2475 return clk_freq;
2476 }
2477
ufs_qcom_freq_to_gear_speed(struct ufs_hba * hba,unsigned long freq)2478 static u32 ufs_qcom_freq_to_gear_speed(struct ufs_hba *hba, unsigned long freq)
2479 {
2480 u32 gear = UFS_HS_DONT_CHANGE;
2481 unsigned long unipro_freq;
2482
2483 if (!hba->use_pm_opp)
2484 return gear;
2485
2486 unipro_freq = ufs_qcom_opp_freq_to_clk_freq(hba, freq, "core_clk_unipro");
2487 switch (unipro_freq) {
2488 case 403000000:
2489 gear = UFS_HS_G5;
2490 break;
2491 case 300000000:
2492 gear = UFS_HS_G4;
2493 break;
2494 case 201500000:
2495 gear = UFS_HS_G3;
2496 break;
2497 case 150000000:
2498 case 100000000:
2499 gear = UFS_HS_G2;
2500 break;
2501 case 75000000:
2502 case 37500000:
2503 gear = UFS_HS_G1;
2504 break;
2505 default:
2506 dev_err(hba->dev, "%s: Unsupported clock freq : %lu\n", __func__, freq);
2507 return UFS_HS_DONT_CHANGE;
2508 }
2509
2510 return min_t(u32, gear, hba->max_pwr_info.info.gear_rx);
2511 }
2512
ufs_qcom_host_eom_config(struct ufs_hba * hba,int lane,const struct ufs_eom_coord * eom_coord,u32 target_test_count)2513 static int ufs_qcom_host_eom_config(struct ufs_hba *hba, int lane,
2514 const struct ufs_eom_coord *eom_coord,
2515 u32 target_test_count)
2516 {
2517 enum ufs_eom_eye_mask eye_mask = eom_coord->eye_mask;
2518 int v_step = eom_coord->v_step;
2519 int t_step = eom_coord->t_step;
2520 u32 volt_step, timing_step;
2521 int ret;
2522
2523 if (abs(v_step) > UFS_QCOM_EOM_VOLTAGE_STEPS_MAX) {
2524 dev_err(hba->dev, "Invalid EOM Voltage Step: %d\n", v_step);
2525 return -ERANGE;
2526 }
2527
2528 if (abs(t_step) > UFS_QCOM_EOM_TIMING_STEPS_MAX) {
2529 dev_err(hba->dev, "Invalid EOM Timing Step: %d\n", t_step);
2530 return -ERANGE;
2531 }
2532
2533 if (v_step < 0)
2534 volt_step = RX_EYEMON_NEGATIVE_STEP_BIT | (u32)(-v_step);
2535 else
2536 volt_step = (u32)v_step;
2537
2538 if (t_step < 0)
2539 timing_step = RX_EYEMON_NEGATIVE_STEP_BIT | (u32)(-t_step);
2540 else
2541 timing_step = (u32)t_step;
2542
2543 ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_EYEMON_ENABLE,
2544 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
2545 BIT(eye_mask) | RX_EYEMON_EXTENDED_VRANGE_BIT);
2546 if (ret) {
2547 dev_err(hba->dev, "Failed to enable Host EOM on Lane %d: %d\n",
2548 lane, ret);
2549 return ret;
2550 }
2551
2552 ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_EYEMON_TIMING_STEPS,
2553 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
2554 timing_step);
2555 if (ret) {
2556 dev_err(hba->dev, "Failed to set Host EOM timing step on Lane %d: %d\n",
2557 lane, ret);
2558 return ret;
2559 }
2560
2561 ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_EYEMON_VOLTAGE_STEPS,
2562 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
2563 volt_step);
2564 if (ret) {
2565 dev_err(hba->dev, "Failed to set Host EOM voltage step on Lane %d: %d\n",
2566 lane, ret);
2567 return ret;
2568 }
2569
2570 ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_EYEMON_TARGET_TEST_COUNT,
2571 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
2572 target_test_count);
2573 if (ret)
2574 dev_err(hba->dev, "Failed to set Host EOM target test count on Lane %d: %d\n",
2575 lane, ret);
2576
2577 return ret;
2578 }
2579
ufs_qcom_host_eom_may_stop(struct ufs_hba * hba,int lane,u32 target_test_count,u32 * err_count)2580 static int ufs_qcom_host_eom_may_stop(struct ufs_hba *hba, int lane,
2581 u32 target_test_count, u32 *err_count)
2582 {
2583 u32 start, tested_count, error_count;
2584 int ret;
2585
2586 ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(RX_EYEMON_START,
2587 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
2588 &start);
2589 if (ret) {
2590 dev_err(hba->dev, "Failed to get Host EOM start status on Lane %d: %d\n",
2591 lane, ret);
2592 return ret;
2593 }
2594
2595 if (start & 0x1)
2596 return -EAGAIN;
2597
2598 ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(RX_EYEMON_TESTED_COUNT,
2599 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
2600 &tested_count);
2601 if (ret) {
2602 dev_err(hba->dev, "Failed to get Host EOM tested count on Lane %d: %d\n",
2603 lane, ret);
2604 return ret;
2605 }
2606
2607 ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(RX_EYEMON_ERROR_COUNT,
2608 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
2609 &error_count);
2610 if (ret) {
2611 dev_err(hba->dev, "Failed to get Host EOM error count on Lane %d: %d\n",
2612 lane, ret);
2613 return ret;
2614 }
2615
2616 /* EOM can stop */
2617 if ((tested_count >= target_test_count - 3) || error_count > 0) {
2618 *err_count = error_count;
2619
2620 /* Disable EOM */
2621 ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_EYEMON_ENABLE,
2622 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
2623 0x0);
2624 if (ret) {
2625 dev_err(hba->dev, "Failed to disable Host EOM on Lane %d: %d\n",
2626 lane, ret);
2627 return ret;
2628 }
2629 } else {
2630 return -EAGAIN;
2631 }
2632
2633 return 0;
2634 }
2635
ufs_qcom_host_eom_scan(struct ufs_hba * hba,int num_lanes,const struct ufs_eom_coord * eom_coord,u32 target_test_count,u32 * err_count)2636 static int ufs_qcom_host_eom_scan(struct ufs_hba *hba, int num_lanes,
2637 const struct ufs_eom_coord *eom_coord,
2638 u32 target_test_count, u32 *err_count)
2639 {
2640 bool eom_stopped[PA_MAXDATALANES] = { 0 };
2641 int lane, ret;
2642 u32 setting;
2643
2644 if (!err_count || !eom_coord)
2645 return -EINVAL;
2646
2647 if (target_test_count < UFS_QCOM_EOM_TARGET_TEST_COUNT_MIN) {
2648 dev_err(hba->dev, "Target test count (%u) too small for Host EOM\n",
2649 target_test_count);
2650 return -ERANGE;
2651 }
2652
2653 for (lane = 0; lane < num_lanes; lane++) {
2654 ret = ufs_qcom_host_eom_config(hba, lane, eom_coord,
2655 target_test_count);
2656 if (ret) {
2657 dev_err(hba->dev, "Failed to config Host RX EOM: %d\n", ret);
2658 return ret;
2659 }
2660 }
2661
2662 /*
2663 * Trigger a PACP_PWR_req to kick start EOM, but not to really change
2664 * the Power Mode.
2665 */
2666 ret = ufshcd_uic_change_pwr_mode(hba, FAST_MODE << 4 | FAST_MODE);
2667 if (ret) {
2668 dev_err(hba->dev, "Failed to change power mode to kick start Host EOM: %d\n",
2669 ret);
2670 return ret;
2671 }
2672
2673 more_burst:
2674 /* Create burst on Host RX Lane. */
2675 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_LOCALVERINFO), &setting);
2676
2677 for (lane = 0; lane < num_lanes; lane++) {
2678 if (eom_stopped[lane])
2679 continue;
2680
2681 ret = ufs_qcom_host_eom_may_stop(hba, lane, target_test_count,
2682 &err_count[lane]);
2683 if (!ret) {
2684 eom_stopped[lane] = true;
2685 } else if (ret == -EAGAIN) {
2686 /* Need more burst to excercise EOM */
2687 goto more_burst;
2688 } else {
2689 dev_err(hba->dev, "Failed to stop Host EOM: %d\n", ret);
2690 return ret;
2691 }
2692
2693 dev_dbg(hba->dev, "Host RX Lane %d EOM, v_step %d, t_step %d, error count %u\n",
2694 lane, eom_coord->v_step, eom_coord->t_step,
2695 err_count[lane]);
2696 }
2697
2698 return 0;
2699 }
2700
ufs_qcom_host_sw_rx_fom(struct ufs_hba * hba,int num_lanes,u32 * fom)2701 static int ufs_qcom_host_sw_rx_fom(struct ufs_hba *hba, int num_lanes, u32 *fom)
2702 {
2703 const struct ufs_eom_coord *eom_coord = sw_rx_fom_eom_coords_g6;
2704 u32 eom_err_count[PA_MAXDATALANES] = { 0 };
2705 u32 curr_ahit;
2706 int lane, i, ret;
2707
2708 if (!fom)
2709 return -EINVAL;
2710
2711 /* Stop the auto hibernate idle timer */
2712 curr_ahit = ufshcd_readl(hba, REG_AUTO_HIBERNATE_IDLE_TIMER);
2713 if (curr_ahit)
2714 ufshcd_writel(hba, 0, REG_AUTO_HIBERNATE_IDLE_TIMER);
2715
2716 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXHSADAPTTYPE), PA_NO_ADAPT);
2717 if (ret) {
2718 dev_err(hba->dev, "Failed to select NO_ADAPT before starting Host EOM: %d\n", ret);
2719 goto out;
2720 }
2721
2722 for (i = 0; i < SW_RX_FOM_EOM_COORDS; i++, eom_coord++) {
2723 ret = ufs_qcom_host_eom_scan(hba, num_lanes, eom_coord,
2724 UFS_QCOM_EOM_TARGET_TEST_COUNT_G6,
2725 eom_err_count);
2726 if (ret) {
2727 dev_err(hba->dev, "Failed to run Host EOM scan: %d\n", ret);
2728 break;
2729 }
2730
2731 for (lane = 0; lane < num_lanes; lane++) {
2732 /* Bad coordinates have no weights */
2733 if (eom_err_count[lane])
2734 continue;
2735 fom[lane] += SW_RX_FOM_EOM_COORDS_WEIGHT;
2736 }
2737 }
2738
2739 out:
2740 /* Restore the auto hibernate idle timer */
2741 if (curr_ahit)
2742 ufshcd_writel(hba, curr_ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
2743
2744 return ret;
2745 }
2746
ufs_qcom_get_rx_fom(struct ufs_hba * hba,struct ufs_pa_layer_attr * pwr_mode,struct tx_eqtr_iter * h_iter,struct tx_eqtr_iter * d_iter)2747 static int ufs_qcom_get_rx_fom(struct ufs_hba *hba,
2748 struct ufs_pa_layer_attr *pwr_mode,
2749 struct tx_eqtr_iter *h_iter,
2750 struct tx_eqtr_iter *d_iter)
2751 {
2752 struct ufshcd_tx_eq_params *params __free(kfree) =
2753 kzalloc(sizeof(*params), GFP_KERNEL);
2754 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
2755 struct ufs_pa_layer_attr old_pwr_info;
2756 u32 fom[PA_MAXDATALANES] = { 0 };
2757 u32 gear = pwr_mode->gear_tx;
2758 u32 rate = pwr_mode->hs_rate;
2759 int lane, ret;
2760
2761 if (host->hw_ver.major != 0x7 || host->hw_ver.minor > 0x1 ||
2762 gear <= UFS_HS_G5 || !d_iter || !d_iter->is_updated)
2763 return 0;
2764
2765 if (gear < UFS_HS_G1 || gear > UFS_HS_GEAR_MAX)
2766 return -ERANGE;
2767
2768 if (!params)
2769 return -ENOMEM;
2770
2771 memcpy(&old_pwr_info, &hba->pwr_info, sizeof(struct ufs_pa_layer_attr));
2772
2773 memcpy(params, &hba->tx_eq_params[gear - 1], sizeof(struct ufshcd_tx_eq_params));
2774 for (lane = 0; lane < pwr_mode->lane_rx; lane++) {
2775 params->device[lane].preshoot = d_iter->preshoot;
2776 params->device[lane].deemphasis = d_iter->deemphasis;
2777 }
2778
2779 /* Use TX EQTR settings as Device's TX Equalization settings. */
2780 ret = ufshcd_apply_tx_eq_settings(hba, params, gear);
2781 if (ret) {
2782 dev_err(hba->dev, "%s: Failed to apply TX EQ settings for HS-G%u: %d\n",
2783 __func__, gear, ret);
2784 return ret;
2785 }
2786
2787 /* Force PMC to target HS Gear to use new TX Equalization settings. */
2788 ret = ufshcd_change_power_mode(hba, pwr_mode, UFSHCD_PMC_POLICY_FORCE);
2789 if (ret) {
2790 dev_err(hba->dev, "%s: Failed to change power mode to HS-G%u, Rate-%s: %d\n",
2791 __func__, gear, ufs_hs_rate_to_str(rate), ret);
2792 return ret;
2793 }
2794
2795 ret = ufs_qcom_host_sw_rx_fom(hba, pwr_mode->lane_rx, fom);
2796 if (ret) {
2797 dev_err(hba->dev, "Failed to get SW FOM of TX (PreShoot: %u, DeEmphasis: %u): %d\n",
2798 d_iter->preshoot, d_iter->deemphasis, ret);
2799 return ret;
2800 }
2801
2802 /* Restore Device's TX Equalization settings. */
2803 ret = ufshcd_apply_tx_eq_settings(hba, &hba->tx_eq_params[gear - 1], gear);
2804 if (ret) {
2805 dev_err(hba->dev, "%s: Failed to apply TX EQ settings for HS-G%u: %d\n",
2806 __func__, gear, ret);
2807 return ret;
2808 }
2809
2810 /* Restore Power Mode. */
2811 ret = ufshcd_change_power_mode(hba, &old_pwr_info, UFSHCD_PMC_POLICY_FORCE);
2812 if (ret) {
2813 dev_err(hba->dev, "%s: Failed to restore power mode to HS-G%u: %d\n",
2814 __func__, old_pwr_info.gear_tx, ret);
2815 return ret;
2816 }
2817
2818 for (lane = 0; lane < pwr_mode->lane_rx; lane++)
2819 d_iter->fom[lane] = fom[lane];
2820
2821 return 0;
2822 }
2823
ufs_qcom_apply_tx_eqtr_settings(struct ufs_hba * hba,struct ufs_pa_layer_attr * pwr_mode,struct tx_eqtr_iter * h_iter,struct tx_eqtr_iter * d_iter)2824 static int ufs_qcom_apply_tx_eqtr_settings(struct ufs_hba *hba,
2825 struct ufs_pa_layer_attr *pwr_mode,
2826 struct tx_eqtr_iter *h_iter,
2827 struct tx_eqtr_iter *d_iter)
2828 {
2829 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
2830 u32 setting = 0;
2831 int lane;
2832
2833 if (host->hw_ver.major != 0x7 || host->hw_ver.minor > 0x1)
2834 return 0;
2835
2836 for (lane = 0; lane < pwr_mode->lane_tx; lane++) {
2837 setting |= TX_HS_PRESHOOT_BITS(lane, h_iter->preshoot);
2838 setting |= TX_HS_DEEMPHASIS_BITS(lane, h_iter->deemphasis);
2839 }
2840
2841 return ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXEQG1SETTING), setting);
2842 }
2843
ufs_qcom_tx_eqtr_notify(struct ufs_hba * hba,enum ufs_notify_change_status status,struct ufs_pa_layer_attr * pwr_mode)2844 static int ufs_qcom_tx_eqtr_notify(struct ufs_hba *hba,
2845 enum ufs_notify_change_status status,
2846 struct ufs_pa_layer_attr *pwr_mode)
2847 {
2848 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
2849 struct ufs_pa_layer_attr pwr_mode_hs_g1 = {
2850 .gear_rx = UFS_HS_G1,
2851 .gear_tx = UFS_HS_G1,
2852 .lane_rx = pwr_mode->lane_rx,
2853 .lane_tx = pwr_mode->lane_tx,
2854 .pwr_rx = FAST_MODE,
2855 .pwr_tx = FAST_MODE,
2856 .hs_rate = pwr_mode->hs_rate,
2857 };
2858 u32 gear = pwr_mode->gear_tx;
2859 u32 rate = pwr_mode->hs_rate;
2860 int ret;
2861
2862 if (host->hw_ver.major != 0x7 || host->hw_ver.minor > 0x1)
2863 return 0;
2864
2865 if (status == PRE_CHANGE) {
2866 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TXEQG1SETTING),
2867 &host->saved_tx_eq_g1_setting);
2868 if (ret)
2869 return ret;
2870
2871 /* PMC to target HS Gear. */
2872 ret = ufshcd_change_power_mode(hba, pwr_mode,
2873 UFSHCD_PMC_POLICY_DONT_FORCE);
2874 if (ret)
2875 dev_err(hba->dev, "%s: Failed to PMC to target HS-G%u, Rate-%s: %d\n",
2876 __func__, gear, ufs_hs_rate_to_str(rate), ret);
2877 } else {
2878 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXEQG1SETTING),
2879 host->saved_tx_eq_g1_setting);
2880 if (ret)
2881 return ret;
2882
2883 /* PMC back to HS-G1. */
2884 ret = ufshcd_change_power_mode(hba, &pwr_mode_hs_g1,
2885 UFSHCD_PMC_POLICY_DONT_FORCE);
2886 if (ret)
2887 dev_err(hba->dev, "%s: Failed to PMC to HS-G1, Rate-%s: %d\n",
2888 __func__, ufs_hs_rate_to_str(rate), ret);
2889 }
2890
2891 return ret;
2892 }
2893
2894 /*
2895 * struct ufs_hba_qcom_vops - UFS QCOM specific variant operations
2896 *
2897 * The variant operations configure the necessary controller and PHY
2898 * handshake during initialization.
2899 */
2900 static const struct ufs_hba_variant_ops ufs_hba_qcom_vops = {
2901 .name = "qcom",
2902 .init = ufs_qcom_init,
2903 .exit = ufs_qcom_exit,
2904 .get_ufs_hci_version = ufs_qcom_get_ufs_hci_version,
2905 .clk_scale_notify = ufs_qcom_clk_scale_notify,
2906 .setup_clocks = ufs_qcom_setup_clocks,
2907 .hce_enable_notify = ufs_qcom_hce_enable_notify,
2908 .link_startup_notify = ufs_qcom_link_startup_notify,
2909 .negotiate_pwr_mode = ufs_qcom_negotiate_pwr_mode,
2910 .pwr_change_notify = ufs_qcom_pwr_change_notify,
2911 .apply_dev_quirks = ufs_qcom_apply_dev_quirks,
2912 .fixup_dev_quirks = ufs_qcom_fixup_dev_quirks,
2913 .suspend = ufs_qcom_suspend,
2914 .resume = ufs_qcom_resume,
2915 .dbg_register_dump = ufs_qcom_dump_dbg_regs,
2916 .device_reset = ufs_qcom_device_reset,
2917 .config_scaling_param = ufs_qcom_config_scaling_param,
2918 .mcq_config_resource = ufs_qcom_mcq_config_resource,
2919 .get_hba_mac = ufs_qcom_get_hba_mac,
2920 .op_runtime_config = ufs_qcom_op_runtime_config,
2921 .get_outstanding_cqs = ufs_qcom_get_outstanding_cqs,
2922 .config_esi = ufs_qcom_config_esi,
2923 .freq_to_gear_speed = ufs_qcom_freq_to_gear_speed,
2924 .get_rx_fom = ufs_qcom_get_rx_fom,
2925 .apply_tx_eqtr_settings = ufs_qcom_apply_tx_eqtr_settings,
2926 .tx_eqtr_notify = ufs_qcom_tx_eqtr_notify,
2927 };
2928
2929 static const struct ufs_hba_variant_ops ufs_hba_qcom_sa8255p_vops = {
2930 .name = "qcom-sa8255p",
2931 .init = ufs_qcom_fw_managed_init,
2932 .exit = ufs_qcom_fw_managed_exit,
2933 .hce_enable_notify = ufs_qcom_fw_managed_hce_enable_notify,
2934 .pwr_change_notify = ufs_qcom_pwr_change_notify,
2935 .apply_dev_quirks = ufs_qcom_apply_dev_quirks,
2936 .fixup_dev_quirks = ufs_qcom_fixup_dev_quirks,
2937 .suspend = ufs_qcom_fw_managed_suspend,
2938 .resume = ufs_qcom_fw_managed_resume,
2939 .dbg_register_dump = ufs_qcom_dump_dbg_regs,
2940 .device_reset = ufs_qcom_fw_managed_device_reset,
2941 };
2942
2943 /**
2944 * ufs_qcom_probe - probe routine of the driver
2945 * @pdev: pointer to Platform device handle
2946 *
2947 * Return: zero for success and non-zero for failure.
2948 */
ufs_qcom_probe(struct platform_device * pdev)2949 static int ufs_qcom_probe(struct platform_device *pdev)
2950 {
2951 int err;
2952 struct device *dev = &pdev->dev;
2953 const struct ufs_hba_variant_ops *vops;
2954 const struct ufs_qcom_drvdata *drvdata = device_get_match_data(dev);
2955
2956 if (drvdata && drvdata->vops)
2957 vops = drvdata->vops;
2958 else
2959 vops = &ufs_hba_qcom_vops;
2960
2961 /* Perform generic probe */
2962 err = ufshcd_pltfrm_init(pdev, vops);
2963 if (err)
2964 return dev_err_probe(dev, err, "ufshcd_pltfrm_init() failed\n");
2965
2966 return 0;
2967 }
2968
2969 /**
2970 * ufs_qcom_remove - set driver_data of the device to NULL
2971 * @pdev: pointer to platform device handle
2972 *
2973 * Always returns 0
2974 */
ufs_qcom_remove(struct platform_device * pdev)2975 static void ufs_qcom_remove(struct platform_device *pdev)
2976 {
2977 struct ufs_hba *hba = platform_get_drvdata(pdev);
2978 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
2979
2980 ufshcd_pltfrm_remove(pdev);
2981 if (host->esi_enabled)
2982 platform_device_msi_free_irqs_all(hba->dev);
2983 }
2984
2985 static const struct ufs_qcom_drvdata ufs_qcom_sm8550_drvdata = {
2986 .quirks = UFSHCD_QUIRK_BROKEN_LSDBS_CAP,
2987 .no_phy_retention = true,
2988 };
2989
2990 static const struct ufs_qcom_drvdata ufs_qcom_sa8255p_drvdata = {
2991 .vops = &ufs_hba_qcom_sa8255p_vops
2992 };
2993
2994 static const struct of_device_id ufs_qcom_of_match[] __maybe_unused = {
2995 { .compatible = "qcom,ufshc" },
2996 { .compatible = "qcom,sm8550-ufshc", .data = &ufs_qcom_sm8550_drvdata },
2997 { .compatible = "qcom,sm8650-ufshc", .data = &ufs_qcom_sm8550_drvdata },
2998 { .compatible = "qcom,sa8255p-ufshc", .data = &ufs_qcom_sa8255p_drvdata },
2999 {},
3000 };
3001 MODULE_DEVICE_TABLE(of, ufs_qcom_of_match);
3002
3003 #ifdef CONFIG_ACPI
3004 static const struct acpi_device_id ufs_qcom_acpi_match[] = {
3005 { "QCOM24A5" },
3006 { },
3007 };
3008 MODULE_DEVICE_TABLE(acpi, ufs_qcom_acpi_match);
3009 #endif
3010
3011 static const struct dev_pm_ops ufs_qcom_pm_ops = {
3012 SET_RUNTIME_PM_OPS(ufshcd_runtime_suspend, ufshcd_runtime_resume, NULL)
3013 .prepare = ufshcd_suspend_prepare,
3014 .complete = ufshcd_resume_complete,
3015 #ifdef CONFIG_PM_SLEEP
3016 .suspend = ufshcd_system_suspend,
3017 .resume = ufshcd_system_resume,
3018 .freeze = ufshcd_system_freeze,
3019 .restore = ufshcd_system_restore,
3020 .thaw = ufshcd_system_thaw,
3021 #endif
3022 };
3023
3024 static struct platform_driver ufs_qcom_pltform = {
3025 .probe = ufs_qcom_probe,
3026 .remove = ufs_qcom_remove,
3027 .driver = {
3028 .name = "ufshcd-qcom",
3029 .pm = &ufs_qcom_pm_ops,
3030 .of_match_table = of_match_ptr(ufs_qcom_of_match),
3031 .acpi_match_table = ACPI_PTR(ufs_qcom_acpi_match),
3032 },
3033 };
3034 module_platform_driver(ufs_qcom_pltform);
3035
3036 MODULE_DESCRIPTION("Qualcomm UFS host controller driver");
3037 MODULE_LICENSE("GPL v2");
3038