1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Universal Flash Storage Host controller driver Core 4 * Copyright (C) 2011-2013 Samsung India Software Operations 5 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved. 6 * 7 * Authors: 8 * Santosh Yaraganavi <santosh.sy@samsung.com> 9 * Vinayak Holikatti <h.vinayak@samsung.com> 10 */ 11 12 #include <linux/async.h> 13 #include <linux/devfreq.h> 14 #include <linux/nls.h> 15 #include <linux/of.h> 16 #include <linux/bitfield.h> 17 #include <linux/blk-pm.h> 18 #include <linux/blkdev.h> 19 #include <linux/clk.h> 20 #include <linux/delay.h> 21 #include <linux/hex.h> 22 #include <linux/interrupt.h> 23 #include <linux/module.h> 24 #include <linux/pm_opp.h> 25 #include <linux/regulator/consumer.h> 26 #include <linux/sched/clock.h> 27 #include <linux/iopoll.h> 28 #include <scsi/scsi_cmnd.h> 29 #include <scsi/scsi_dbg.h> 30 #include <scsi/scsi_driver.h> 31 #include <scsi/scsi_eh.h> 32 #include <scsi/scsi_tcq.h> 33 #include "ufshcd-priv.h" 34 #include <ufs/ufs_quirks.h> 35 #include <ufs/unipro.h> 36 #include "ufs-sysfs.h" 37 #include "ufs-debugfs.h" 38 #include "ufs-fault-injection.h" 39 #include "ufs_bsg.h" 40 #include "ufshcd-crypto.h" 41 #include <linux/unaligned.h> 42 43 #define CREATE_TRACE_POINTS 44 #include "ufs_trace.h" 45 46 #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\ 47 UTP_TASK_REQ_COMPL |\ 48 UFSHCD_ERROR_MASK) 49 50 /* UIC command timeout, unit: ms */ 51 enum { 52 UIC_CMD_TIMEOUT_DEFAULT = 500, 53 UIC_CMD_TIMEOUT_MAX = 5000, 54 }; 55 /* NOP OUT retries waiting for NOP IN response */ 56 #define NOP_OUT_RETRIES 10 57 /* Timeout after 50 msecs if NOP OUT hangs without response */ 58 #define NOP_OUT_TIMEOUT 50 /* msecs */ 59 60 /* Query request retries */ 61 #define QUERY_REQ_RETRIES 3 62 /* Query request timeout */ 63 enum { 64 QUERY_REQ_TIMEOUT_MIN = 1, 65 QUERY_REQ_TIMEOUT_DEFAULT = 1500, 66 QUERY_REQ_TIMEOUT_MAX = 30000 67 }; 68 69 /* Advanced RPMB request timeout */ 70 #define ADVANCED_RPMB_REQ_TIMEOUT 3000 /* 3 seconds */ 71 72 /* Task management command timeout */ 73 #define TM_CMD_TIMEOUT 100 /* msecs */ 74 75 /* maximum number of retries for a general UIC command */ 76 #define UFS_UIC_COMMAND_RETRIES 3 77 78 /* maximum number of link-startup retries */ 79 #define DME_LINKSTARTUP_RETRIES 3 80 81 /* maximum number of reset retries before giving up */ 82 #define MAX_HOST_RESET_RETRIES 5 83 84 /* Maximum number of error handler retries before giving up */ 85 #define MAX_ERR_HANDLER_RETRIES 5 86 87 /* Expose the flag value from utp_upiu_query.value */ 88 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF 89 90 /* Interrupt aggregation default timeout, unit: 40us */ 91 #define INT_AGGR_DEF_TO 0x02 92 93 /* default delay of autosuspend: 2000 ms */ 94 #define RPM_AUTOSUSPEND_DELAY_MS 2000 95 96 /* Default delay of RPM device flush delayed work */ 97 #define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000 98 99 /* Default value of wait time before gating device ref clock */ 100 #define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */ 101 102 /* Polling time to wait for fDeviceInit */ 103 #define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */ 104 105 /* Default RTC update every 10 seconds */ 106 #define UFS_RTC_UPDATE_INTERVAL_MS (10 * MSEC_PER_SEC) 107 108 /* bMaxNumOfRTT is equal to two after device manufacturing */ 109 #define DEFAULT_MAX_NUM_RTT 2 110 111 /* UFSHC 4.0 compliant HC support this mode. */ 112 static bool use_mcq_mode = true; 113 114 static bool is_mcq_supported(struct ufs_hba *hba) 115 { 116 return hba->mcq_sup && use_mcq_mode; 117 } 118 119 module_param(use_mcq_mode, bool, 0644); 120 MODULE_PARM_DESC(use_mcq_mode, "Control MCQ mode for controllers starting from UFSHCI 4.0. 1 - enable MCQ, 0 - disable MCQ. MCQ is enabled by default"); 121 122 static unsigned int uic_cmd_timeout = UIC_CMD_TIMEOUT_DEFAULT; 123 124 static int uic_cmd_timeout_set(const char *val, const struct kernel_param *kp) 125 { 126 return param_set_uint_minmax(val, kp, UIC_CMD_TIMEOUT_DEFAULT, 127 UIC_CMD_TIMEOUT_MAX); 128 } 129 130 static const struct kernel_param_ops uic_cmd_timeout_ops = { 131 .set = uic_cmd_timeout_set, 132 .get = param_get_uint, 133 }; 134 135 module_param_cb(uic_cmd_timeout, &uic_cmd_timeout_ops, &uic_cmd_timeout, 0644); 136 MODULE_PARM_DESC(uic_cmd_timeout, 137 "UFS UIC command timeout in milliseconds. Defaults to 500ms. Supported values range from 500ms to 5 seconds inclusively"); 138 139 static unsigned int dev_cmd_timeout = QUERY_REQ_TIMEOUT_DEFAULT; 140 141 static int dev_cmd_timeout_set(const char *val, const struct kernel_param *kp) 142 { 143 return param_set_uint_minmax(val, kp, QUERY_REQ_TIMEOUT_MIN, 144 QUERY_REQ_TIMEOUT_MAX); 145 } 146 147 static const struct kernel_param_ops dev_cmd_timeout_ops = { 148 .set = dev_cmd_timeout_set, 149 .get = param_get_uint, 150 }; 151 152 module_param_cb(dev_cmd_timeout, &dev_cmd_timeout_ops, &dev_cmd_timeout, 0644); 153 MODULE_PARM_DESC(dev_cmd_timeout, 154 "UFS Device command timeout in milliseconds. Defaults to 1.5s. Supported values range from 1ms to 30 seconds inclusively"); 155 156 #define ufshcd_toggle_vreg(_dev, _vreg, _on) \ 157 ({ \ 158 int _ret; \ 159 if (_on) \ 160 _ret = ufshcd_enable_vreg(_dev, _vreg); \ 161 else \ 162 _ret = ufshcd_disable_vreg(_dev, _vreg); \ 163 _ret; \ 164 }) 165 166 #define ufshcd_hex_dump(prefix_str, buf, len) do { \ 167 size_t __len = (len); \ 168 print_hex_dump(KERN_ERR, prefix_str, \ 169 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\ 170 16, 4, buf, __len, false); \ 171 } while (0) 172 173 int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len, 174 const char *prefix) 175 { 176 u32 *regs; 177 size_t pos; 178 179 if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */ 180 return -EINVAL; 181 182 regs = kzalloc(len, GFP_ATOMIC); 183 if (!regs) 184 return -ENOMEM; 185 186 for (pos = 0; pos < len; pos += 4) { 187 if (offset == 0 && 188 pos >= REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER && 189 pos <= REG_UIC_ERROR_CODE_DME) 190 continue; 191 regs[pos / 4] = ufshcd_readl(hba, offset + pos); 192 } 193 194 ufshcd_hex_dump(prefix, regs, len); 195 kfree(regs); 196 197 return 0; 198 } 199 EXPORT_SYMBOL_GPL(ufshcd_dump_regs); 200 201 enum { 202 UFSHCD_MAX_CHANNEL = 0, 203 UFSHCD_MAX_ID = 1, 204 }; 205 206 static const char *const ufshcd_state_name[] = { 207 [UFSHCD_STATE_RESET] = "reset", 208 [UFSHCD_STATE_OPERATIONAL] = "operational", 209 [UFSHCD_STATE_ERROR] = "error", 210 [UFSHCD_STATE_EH_SCHEDULED_FATAL] = "eh_fatal", 211 [UFSHCD_STATE_EH_SCHEDULED_NON_FATAL] = "eh_non_fatal", 212 }; 213 214 /* UFSHCD error handling flags */ 215 enum { 216 UFSHCD_EH_IN_PROGRESS = (1 << 0), 217 }; 218 219 /* UFSHCD UIC layer error flags */ 220 enum { 221 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */ 222 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */ 223 UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */ 224 UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */ 225 UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */ 226 UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */ 227 UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6), /* Generic PA error */ 228 }; 229 230 #define ufshcd_set_eh_in_progress(h) \ 231 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS) 232 #define ufshcd_eh_in_progress(h) \ 233 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS) 234 #define ufshcd_clear_eh_in_progress(h) \ 235 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS) 236 237 const struct ufs_pm_lvl_states ufs_pm_lvl_states[] = { 238 [UFS_PM_LVL_0] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE}, 239 [UFS_PM_LVL_1] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 240 [UFS_PM_LVL_2] = {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE}, 241 [UFS_PM_LVL_3] = {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 242 [UFS_PM_LVL_4] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 243 [UFS_PM_LVL_5] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE}, 244 /* 245 * For DeepSleep, the link is first put in hibern8 and then off. 246 * Leaving the link in hibern8 is not supported. 247 */ 248 [UFS_PM_LVL_6] = {UFS_DEEPSLEEP_PWR_MODE, UIC_LINK_OFF_STATE}, 249 }; 250 251 static inline enum ufs_dev_pwr_mode 252 ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl) 253 { 254 return ufs_pm_lvl_states[lvl].dev_state; 255 } 256 257 static inline enum uic_link_state 258 ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl) 259 { 260 return ufs_pm_lvl_states[lvl].link_state; 261 } 262 263 static inline enum ufs_pm_level 264 ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state, 265 enum uic_link_state link_state) 266 { 267 enum ufs_pm_level lvl; 268 269 for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) { 270 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) && 271 (ufs_pm_lvl_states[lvl].link_state == link_state)) 272 return lvl; 273 } 274 275 /* if no match found, return the level 0 */ 276 return UFS_PM_LVL_0; 277 } 278 279 static bool ufshcd_has_pending_tasks(struct ufs_hba *hba) 280 { 281 return hba->outstanding_tasks || hba->active_uic_cmd || 282 hba->uic_async_done; 283 } 284 285 static bool ufshcd_is_ufs_dev_busy(struct ufs_hba *hba) 286 { 287 return scsi_host_busy(hba->host) || ufshcd_has_pending_tasks(hba); 288 } 289 290 static const struct ufs_dev_quirk ufs_fixups[] = { 291 /* UFS cards deviations table */ 292 { .wmanufacturerid = UFS_VENDOR_MICRON, 293 .model = UFS_ANY_MODEL, 294 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM }, 295 { .wmanufacturerid = UFS_VENDOR_SAMSUNG, 296 .model = UFS_ANY_MODEL, 297 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM | 298 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE | 299 UFS_DEVICE_QUIRK_PA_HIBER8TIME | 300 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS }, 301 { .wmanufacturerid = UFS_VENDOR_SKHYNIX, 302 .model = UFS_ANY_MODEL, 303 .quirk = UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME }, 304 { .wmanufacturerid = UFS_VENDOR_SKHYNIX, 305 .model = "hB8aL1" /*H28U62301AMR*/, 306 .quirk = UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME }, 307 { .wmanufacturerid = UFS_VENDOR_TOSHIBA, 308 .model = UFS_ANY_MODEL, 309 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM }, 310 { .wmanufacturerid = UFS_VENDOR_TOSHIBA, 311 .model = "THGLF2G9C8KBADG", 312 .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE }, 313 { .wmanufacturerid = UFS_VENDOR_TOSHIBA, 314 .model = "THGLF2G9D8KBADG", 315 .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE }, 316 { .wmanufacturerid = UFS_VENDOR_TOSHIBA, 317 .model = "THGJFJT1E45BATP", 318 .quirk = UFS_DEVICE_QUIRK_NO_TIMESTAMP_SUPPORT }, 319 {} 320 }; 321 322 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba); 323 static void ufshcd_async_scan(void *data, async_cookie_t cookie); 324 static int ufshcd_reset_and_restore(struct ufs_hba *hba); 325 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd); 326 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag); 327 static void ufshcd_hba_exit(struct ufs_hba *hba); 328 static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params); 329 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params); 330 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on); 331 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba); 332 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba); 333 static void ufshcd_resume_clkscaling(struct ufs_hba *hba); 334 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba); 335 static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq, 336 bool scale_up); 337 static irqreturn_t ufshcd_intr(int irq, void *__hba); 338 static int ufshcd_change_power_mode(struct ufs_hba *hba, 339 struct ufs_pa_layer_attr *pwr_mode); 340 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on); 341 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on); 342 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba, 343 struct ufs_vreg *vreg); 344 static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba, 345 bool enable); 346 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba); 347 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba); 348 349 void ufshcd_enable_irq(struct ufs_hba *hba) 350 { 351 if (!hba->is_irq_enabled) { 352 enable_irq(hba->irq); 353 hba->is_irq_enabled = true; 354 } 355 } 356 EXPORT_SYMBOL_GPL(ufshcd_enable_irq); 357 358 void ufshcd_disable_irq(struct ufs_hba *hba) 359 { 360 if (hba->is_irq_enabled) { 361 disable_irq(hba->irq); 362 hba->is_irq_enabled = false; 363 } 364 } 365 EXPORT_SYMBOL_GPL(ufshcd_disable_irq); 366 367 /** 368 * ufshcd_enable_intr - enable interrupts 369 * @hba: per adapter instance 370 * @intrs: interrupt bits 371 */ 372 void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs) 373 { 374 u32 old_val = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 375 u32 new_val = old_val | intrs; 376 377 if (new_val != old_val) 378 ufshcd_writel(hba, new_val, REG_INTERRUPT_ENABLE); 379 } 380 381 /** 382 * ufshcd_disable_intr - disable interrupts 383 * @hba: per adapter instance 384 * @intrs: interrupt bits 385 */ 386 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs) 387 { 388 u32 old_val = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 389 u32 new_val = old_val & ~intrs; 390 391 if (new_val != old_val) 392 ufshcd_writel(hba, new_val, REG_INTERRUPT_ENABLE); 393 } 394 395 static void ufshcd_configure_wb(struct ufs_hba *hba) 396 { 397 if (!ufshcd_is_wb_allowed(hba)) 398 return; 399 400 ufshcd_wb_toggle(hba, true); 401 402 ufshcd_wb_toggle_buf_flush_during_h8(hba, true); 403 404 if (ufshcd_is_wb_buf_flush_allowed(hba)) 405 ufshcd_wb_toggle_buf_flush(hba, true); 406 } 407 408 static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, 409 struct ufshcd_lrb *lrb, 410 enum ufs_trace_str_t str_t) 411 { 412 struct utp_upiu_req *rq = lrb->ucd_req_ptr; 413 struct utp_upiu_header *header; 414 415 if (!trace_ufshcd_upiu_enabled()) 416 return; 417 418 if (str_t == UFS_CMD_SEND) 419 header = &rq->header; 420 else 421 header = &lrb->ucd_rsp_ptr->header; 422 423 trace_ufshcd_upiu(hba, str_t, header, &rq->sc.cdb, 424 UFS_TSF_CDB); 425 } 426 427 static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, 428 enum ufs_trace_str_t str_t, 429 struct utp_upiu_req *rq_rsp) 430 { 431 if (!trace_ufshcd_upiu_enabled()) 432 return; 433 434 trace_ufshcd_upiu(hba, str_t, &rq_rsp->header, 435 &rq_rsp->qr, UFS_TSF_OSF); 436 } 437 438 static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag, 439 enum ufs_trace_str_t str_t) 440 { 441 struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[tag]; 442 443 if (!trace_ufshcd_upiu_enabled()) 444 return; 445 446 if (str_t == UFS_TM_SEND) 447 trace_ufshcd_upiu(hba, str_t, 448 &descp->upiu_req.req_header, 449 &descp->upiu_req.input_param1, 450 UFS_TSF_TM_INPUT); 451 else 452 trace_ufshcd_upiu(hba, str_t, 453 &descp->upiu_rsp.rsp_header, 454 &descp->upiu_rsp.output_param1, 455 UFS_TSF_TM_OUTPUT); 456 } 457 458 static void ufshcd_add_uic_command_trace(struct ufs_hba *hba, 459 const struct uic_command *ucmd, 460 enum ufs_trace_str_t str_t) 461 { 462 u32 cmd; 463 464 if (!trace_ufshcd_uic_command_enabled()) 465 return; 466 467 if (str_t == UFS_CMD_SEND) 468 cmd = ucmd->command; 469 else 470 cmd = ufshcd_readl(hba, REG_UIC_COMMAND); 471 472 trace_ufshcd_uic_command(hba, str_t, cmd, 473 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1), 474 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2), 475 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3)); 476 } 477 478 static void ufshcd_add_command_trace(struct ufs_hba *hba, struct scsi_cmnd *cmd, 479 enum ufs_trace_str_t str_t) 480 { 481 u64 lba = 0; 482 u8 opcode = 0, group_id = 0; 483 u32 doorbell = 0; 484 u32 intr; 485 u32 hwq_id = 0; 486 struct request *rq = scsi_cmd_to_rq(cmd); 487 unsigned int tag = rq->tag; 488 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 489 int transfer_len = -1; 490 491 /* trace UPIU also */ 492 ufshcd_add_cmd_upiu_trace(hba, lrbp, str_t); 493 if (!trace_ufshcd_command_enabled()) 494 return; 495 496 opcode = cmd->cmnd[0]; 497 498 if (opcode == READ_10 || opcode == WRITE_10) { 499 /* 500 * Currently we only fully trace read(10) and write(10) commands 501 */ 502 transfer_len = 503 be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len); 504 lba = scsi_get_lba(cmd); 505 if (opcode == WRITE_10) 506 group_id = cmd->cmnd[6]; 507 } else if (opcode == UNMAP) { 508 /* 509 * The number of Bytes to be unmapped beginning with the lba. 510 */ 511 transfer_len = blk_rq_bytes(rq); 512 lba = scsi_get_lba(cmd); 513 } 514 515 intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 516 517 if (hba->mcq_enabled) { 518 struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); 519 520 hwq_id = hwq->id; 521 } else { 522 doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 523 } 524 trace_ufshcd_command(cmd->device, hba, str_t, tag, doorbell, hwq_id, 525 transfer_len, intr, lba, opcode, group_id); 526 } 527 528 static void ufshcd_print_clk_freqs(struct ufs_hba *hba) 529 { 530 struct ufs_clk_info *clki; 531 struct list_head *head = &hba->clk_list_head; 532 533 if (list_empty(head)) 534 return; 535 536 list_for_each_entry(clki, head, list) { 537 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq && 538 clki->max_freq) 539 dev_err(hba->dev, "clk: %s, rate: %u\n", 540 clki->name, clki->curr_freq); 541 } 542 } 543 544 static void ufshcd_print_evt(struct ufs_hba *hba, u32 id, 545 const char *err_name) 546 { 547 int i; 548 bool found = false; 549 const struct ufs_event_hist *e; 550 551 if (id >= UFS_EVT_CNT) 552 return; 553 554 e = &hba->ufs_stats.event[id]; 555 556 for (i = 0; i < UFS_EVENT_HIST_LENGTH; i++) { 557 int p = (i + e->pos) % UFS_EVENT_HIST_LENGTH; 558 559 if (e->tstamp[p] == 0) 560 continue; 561 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p, 562 e->val[p], div_u64(e->tstamp[p], 1000)); 563 found = true; 564 } 565 566 if (!found) 567 dev_err(hba->dev, "No record of %s\n", err_name); 568 else 569 dev_err(hba->dev, "%s: total cnt=%llu\n", err_name, e->cnt); 570 } 571 572 static void ufshcd_print_evt_hist(struct ufs_hba *hba) 573 { 574 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: "); 575 576 ufshcd_print_evt(hba, UFS_EVT_PA_ERR, "pa_err"); 577 ufshcd_print_evt(hba, UFS_EVT_DL_ERR, "dl_err"); 578 ufshcd_print_evt(hba, UFS_EVT_NL_ERR, "nl_err"); 579 ufshcd_print_evt(hba, UFS_EVT_TL_ERR, "tl_err"); 580 ufshcd_print_evt(hba, UFS_EVT_DME_ERR, "dme_err"); 581 ufshcd_print_evt(hba, UFS_EVT_AUTO_HIBERN8_ERR, 582 "auto_hibern8_err"); 583 ufshcd_print_evt(hba, UFS_EVT_FATAL_ERR, "fatal_err"); 584 ufshcd_print_evt(hba, UFS_EVT_LINK_STARTUP_FAIL, 585 "link_startup_fail"); 586 ufshcd_print_evt(hba, UFS_EVT_RESUME_ERR, "resume_fail"); 587 ufshcd_print_evt(hba, UFS_EVT_SUSPEND_ERR, 588 "suspend_fail"); 589 ufshcd_print_evt(hba, UFS_EVT_WL_RES_ERR, "wlun resume_fail"); 590 ufshcd_print_evt(hba, UFS_EVT_WL_SUSP_ERR, 591 "wlun suspend_fail"); 592 ufshcd_print_evt(hba, UFS_EVT_DEV_RESET, "dev_reset"); 593 ufshcd_print_evt(hba, UFS_EVT_HOST_RESET, "host_reset"); 594 ufshcd_print_evt(hba, UFS_EVT_ABORT, "task_abort"); 595 596 ufshcd_vops_dbg_register_dump(hba); 597 } 598 599 static void ufshcd_print_tr(struct ufs_hba *hba, struct scsi_cmnd *cmd, 600 bool pr_prdt) 601 { 602 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 603 const int tag = scsi_cmd_to_rq(cmd)->tag; 604 int prdt_length; 605 606 if (hba->monitor.enabled) { 607 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", tag, 608 div_u64(lrbp->issue_time_stamp_local_clock, 1000)); 609 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n", tag, 610 div_u64(lrbp->compl_time_stamp_local_clock, 1000)); 611 } 612 dev_err(hba->dev, 613 "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n", 614 tag, (u64)lrbp->utrd_dma_addr); 615 616 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr, 617 sizeof(struct utp_transfer_req_desc)); 618 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag, 619 (u64)lrbp->ucd_req_dma_addr); 620 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr, 621 sizeof(struct utp_upiu_req)); 622 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag, 623 (u64)lrbp->ucd_rsp_dma_addr); 624 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr, 625 sizeof(struct utp_upiu_rsp)); 626 627 prdt_length = le16_to_cpu( 628 lrbp->utr_descriptor_ptr->prd_table_length); 629 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) 630 prdt_length /= ufshcd_sg_entry_size(hba); 631 632 dev_err(hba->dev, 633 "UPIU[%d] - PRDT - %d entries phys@0x%llx\n", 634 tag, prdt_length, 635 (u64)lrbp->ucd_prdt_dma_addr); 636 637 if (pr_prdt) 638 ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr, 639 ufshcd_sg_entry_size(hba) * prdt_length); 640 } 641 642 static bool ufshcd_print_tr_iter(struct request *req, void *priv) 643 { 644 struct scsi_device *sdev = req->q->queuedata; 645 struct Scsi_Host *shost = sdev->host; 646 struct ufs_hba *hba = shost_priv(shost); 647 648 if (!blk_mq_is_reserved_rq(req)) 649 ufshcd_print_tr(hba, blk_mq_rq_to_pdu(req), *(bool *)priv); 650 651 return true; 652 } 653 654 /** 655 * ufshcd_print_trs_all - print trs for all started requests. 656 * @hba: per-adapter instance. 657 * @pr_prdt: need to print prdt or not. 658 */ 659 static void ufshcd_print_trs_all(struct ufs_hba *hba, bool pr_prdt) 660 { 661 blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_print_tr_iter, &pr_prdt); 662 } 663 664 static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap) 665 { 666 int tag; 667 668 for_each_set_bit(tag, &bitmap, hba->nutmrs) { 669 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag]; 670 671 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag); 672 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp)); 673 } 674 } 675 676 static void ufshcd_print_host_state(struct ufs_hba *hba) 677 { 678 const struct scsi_device *sdev_ufs = hba->ufs_device_wlun; 679 680 dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state); 681 dev_err(hba->dev, "%d outstanding reqs, tasks=0x%lx\n", 682 scsi_host_busy(hba->host), hba->outstanding_tasks); 683 dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n", 684 hba->saved_err, hba->saved_uic_err); 685 dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n", 686 hba->curr_dev_pwr_mode, hba->uic_link_state); 687 dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n", 688 hba->pm_op_in_progress, hba->is_sys_suspended); 689 dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n", 690 hba->auto_bkops_enabled, hba->host->host_self_blocked); 691 dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state); 692 dev_err(hba->dev, 693 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n", 694 div_u64(hba->ufs_stats.last_hibern8_exit_tstamp, 1000), 695 hba->ufs_stats.hibern8_exit_cnt); 696 dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n", 697 hba->eh_flags, hba->req_abort_count); 698 dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n", 699 hba->ufs_version, hba->capabilities, hba->caps); 700 dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks, 701 hba->dev_quirks); 702 if (sdev_ufs) 703 dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n", 704 sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev); 705 706 ufshcd_print_clk_freqs(hba); 707 } 708 709 /** 710 * ufshcd_print_pwr_info - print power params as saved in hba 711 * power info 712 * @hba: per-adapter instance 713 */ 714 static void ufshcd_print_pwr_info(struct ufs_hba *hba) 715 { 716 static const char * const names[] = { 717 "INVALID MODE", 718 "FAST MODE", 719 "SLOW_MODE", 720 "INVALID MODE", 721 "FASTAUTO_MODE", 722 "SLOWAUTO_MODE", 723 "INVALID MODE", 724 }; 725 726 /* 727 * Using dev_dbg to avoid messages during runtime PM to avoid 728 * never-ending cycles of messages written back to storage by user space 729 * causing runtime resume, causing more messages and so on. 730 */ 731 dev_dbg(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n", 732 __func__, 733 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx, 734 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx, 735 names[hba->pwr_info.pwr_rx], 736 names[hba->pwr_info.pwr_tx], 737 hba->pwr_info.hs_rate); 738 } 739 740 static void ufshcd_device_reset(struct ufs_hba *hba) 741 { 742 int err; 743 744 err = ufshcd_vops_device_reset(hba); 745 746 if (!err) { 747 ufshcd_set_ufs_dev_active(hba); 748 if (ufshcd_is_wb_allowed(hba)) { 749 hba->dev_info.wb_enabled = false; 750 hba->dev_info.wb_buf_flush_enabled = false; 751 } 752 if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE) 753 hba->dev_info.rtc_time_baseline = 0; 754 } 755 if (err != -EOPNOTSUPP) 756 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, err); 757 } 758 759 void ufshcd_delay_us(unsigned long us, unsigned long tolerance) 760 { 761 if (!us) 762 return; 763 764 if (us < 10) 765 udelay(us); 766 else 767 usleep_range(us, us + tolerance); 768 } 769 EXPORT_SYMBOL_GPL(ufshcd_delay_us); 770 771 /** 772 * ufshcd_wait_for_register - wait for register value to change 773 * @hba: per-adapter interface 774 * @reg: mmio register offset 775 * @mask: mask to apply to the read register value 776 * @val: value to wait for 777 * @interval_us: polling interval in microseconds 778 * @timeout_ms: timeout in milliseconds 779 * 780 * Return: -ETIMEDOUT on error, zero on success. 781 */ 782 static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask, 783 u32 val, unsigned long interval_us, 784 unsigned long timeout_ms) 785 { 786 u32 v; 787 788 val &= mask; /* ignore bits that we don't intend to wait on */ 789 790 return read_poll_timeout(ufshcd_readl, v, (v & mask) == val, 791 interval_us, timeout_ms * 1000, false, hba, reg); 792 } 793 794 /** 795 * ufshcd_get_intr_mask - Get the interrupt bit mask 796 * @hba: Pointer to adapter instance 797 * 798 * Return: interrupt bit mask per version 799 */ 800 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba) 801 { 802 if (hba->ufs_version <= ufshci_version(2, 0)) 803 return INTERRUPT_MASK_ALL_VER_11; 804 805 return INTERRUPT_MASK_ALL_VER_21; 806 } 807 808 /** 809 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA 810 * @hba: Pointer to adapter instance 811 * 812 * Return: UFSHCI version supported by the controller 813 */ 814 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba) 815 { 816 u32 ufshci_ver; 817 818 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION) 819 ufshci_ver = ufshcd_vops_get_ufs_hci_version(hba); 820 else 821 ufshci_ver = ufshcd_readl(hba, REG_UFS_VERSION); 822 823 /* 824 * UFSHCI v1.x uses a different version scheme, in order 825 * to allow the use of comparisons with the ufshci_version 826 * function, we convert it to the same scheme as ufs 2.0+. 827 */ 828 if (ufshci_ver & 0x00010000) 829 return ufshci_version(1, ufshci_ver & 0x00000100); 830 831 return ufshci_ver; 832 } 833 834 /** 835 * ufshcd_is_device_present - Check if any device connected to 836 * the host controller 837 * @hba: pointer to adapter instance 838 * 839 * Return: true if device present, false if no device detected 840 */ 841 static inline bool ufshcd_is_device_present(struct ufs_hba *hba) 842 { 843 return ufshcd_readl(hba, REG_CONTROLLER_STATUS) & DEVICE_PRESENT; 844 } 845 846 /** 847 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status 848 * @lrbp: pointer to local command reference block 849 * @cqe: pointer to the completion queue entry 850 * 851 * This function is used to get the OCS field from UTRD 852 * 853 * Return: the OCS field in the UTRD. 854 */ 855 static enum utp_ocs ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp, 856 struct cq_entry *cqe) 857 { 858 if (cqe) 859 return cqe->overall_status & MASK_OCS; 860 861 return lrbp->utr_descriptor_ptr->header.ocs & MASK_OCS; 862 } 863 864 /** 865 * ufshcd_utrl_clear() - Clear requests from the controller request list. 866 * @hba: per adapter instance 867 * @mask: mask with one bit set for each request to be cleared 868 */ 869 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 mask) 870 { 871 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR) 872 mask = ~mask; 873 /* 874 * From the UFSHCI specification: "UTP Transfer Request List CLear 875 * Register (UTRLCLR): This field is bit significant. Each bit 876 * corresponds to a slot in the UTP Transfer Request List, where bit 0 877 * corresponds to request slot 0. A bit in this field is set to ‘0’ 878 * by host software to indicate to the host controller that a transfer 879 * request slot is cleared. The host controller 880 * shall free up any resources associated to the request slot 881 * immediately, and shall set the associated bit in UTRLDBR to ‘0’. The 882 * host software indicates no change to request slots by setting the 883 * associated bits in this field to ‘1’. Bits in this field shall only 884 * be set ‘1’ or ‘0’ by host software when UTRLRSR is set to ‘1’." 885 */ 886 ufshcd_writel(hba, ~mask, REG_UTP_TRANSFER_REQ_LIST_CLEAR); 887 } 888 889 /** 890 * ufshcd_utmrl_clear - Clear a bit in UTMRLCLR register 891 * @hba: per adapter instance 892 * @pos: position of the bit to be cleared 893 */ 894 static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos) 895 { 896 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR) 897 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR); 898 else 899 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR); 900 } 901 902 /** 903 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY 904 * @reg: Register value of host controller status 905 * 906 * Return: 0 on success; a positive value if failed. 907 */ 908 static inline int ufshcd_get_lists_status(u32 reg) 909 { 910 return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY); 911 } 912 913 /** 914 * ufshcd_get_uic_cmd_result - Get the UIC command result 915 * @hba: Pointer to adapter instance 916 * 917 * This function gets the result of UIC command completion 918 * 919 * Return: 0 on success; non-zero value on error. 920 */ 921 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba) 922 { 923 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) & 924 MASK_UIC_COMMAND_RESULT; 925 } 926 927 /** 928 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command 929 * @hba: Pointer to adapter instance 930 * 931 * This function gets UIC command argument3 932 * 933 * Return: 0 on success; non-zero value on error. 934 */ 935 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba) 936 { 937 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3); 938 } 939 940 /** 941 * ufshcd_get_req_rsp - returns the TR response transaction type 942 * @ucd_rsp_ptr: pointer to response UPIU 943 * 944 * Return: UPIU type. 945 */ 946 static inline enum upiu_response_transaction 947 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr) 948 { 949 return ucd_rsp_ptr->header.transaction_code; 950 } 951 952 /** 953 * ufshcd_is_exception_event - Check if the device raised an exception event 954 * @ucd_rsp_ptr: pointer to response UPIU 955 * 956 * The function checks if the device raised an exception event indicated in 957 * the Device Information field of response UPIU. 958 * 959 * Return: true if exception is raised, false otherwise. 960 */ 961 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr) 962 { 963 return ucd_rsp_ptr->header.device_information & 1; 964 } 965 966 /** 967 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values. 968 * @hba: per adapter instance 969 */ 970 static inline void 971 ufshcd_reset_intr_aggr(struct ufs_hba *hba) 972 { 973 ufshcd_writel(hba, INT_AGGR_ENABLE | 974 INT_AGGR_COUNTER_AND_TIMER_RESET, 975 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); 976 } 977 978 /** 979 * ufshcd_config_intr_aggr - Configure interrupt aggregation values. 980 * @hba: per adapter instance 981 * @cnt: Interrupt aggregation counter threshold 982 * @tmout: Interrupt aggregation timeout value 983 */ 984 static inline void 985 ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout) 986 { 987 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE | 988 INT_AGGR_COUNTER_THLD_VAL(cnt) | 989 INT_AGGR_TIMEOUT_VAL(tmout), 990 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); 991 } 992 993 /** 994 * ufshcd_disable_intr_aggr - Disables interrupt aggregation. 995 * @hba: per adapter instance 996 */ 997 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba) 998 { 999 ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); 1000 } 1001 1002 /** 1003 * ufshcd_enable_run_stop_reg - Enable run-stop registers, 1004 * When run-stop registers are set to 1, it indicates the 1005 * host controller that it can process the requests 1006 * @hba: per adapter instance 1007 */ 1008 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba) 1009 { 1010 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT, 1011 REG_UTP_TASK_REQ_LIST_RUN_STOP); 1012 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT, 1013 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP); 1014 } 1015 1016 /** 1017 * ufshcd_hba_start - Start controller initialization sequence 1018 * @hba: per adapter instance 1019 */ 1020 static inline void ufshcd_hba_start(struct ufs_hba *hba) 1021 { 1022 u32 val = CONTROLLER_ENABLE; 1023 1024 if (ufshcd_crypto_enable(hba)) 1025 val |= CRYPTO_GENERAL_ENABLE; 1026 1027 ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE); 1028 } 1029 1030 /** 1031 * ufshcd_is_hba_active - Get controller state 1032 * @hba: per adapter instance 1033 * 1034 * Return: true if and only if the controller is active. 1035 */ 1036 bool ufshcd_is_hba_active(struct ufs_hba *hba) 1037 { 1038 return ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE; 1039 } 1040 EXPORT_SYMBOL_GPL(ufshcd_is_hba_active); 1041 1042 /** 1043 * ufshcd_pm_qos_init - initialize PM QoS request 1044 * @hba: per adapter instance 1045 */ 1046 void ufshcd_pm_qos_init(struct ufs_hba *hba) 1047 { 1048 guard(mutex)(&hba->pm_qos_mutex); 1049 1050 if (hba->pm_qos_enabled) 1051 return; 1052 1053 cpu_latency_qos_add_request(&hba->pm_qos_req, PM_QOS_DEFAULT_VALUE); 1054 1055 if (cpu_latency_qos_request_active(&hba->pm_qos_req)) 1056 hba->pm_qos_enabled = true; 1057 } 1058 1059 /** 1060 * ufshcd_pm_qos_exit - remove request from PM QoS 1061 * @hba: per adapter instance 1062 */ 1063 void ufshcd_pm_qos_exit(struct ufs_hba *hba) 1064 { 1065 guard(mutex)(&hba->pm_qos_mutex); 1066 1067 if (!hba->pm_qos_enabled) 1068 return; 1069 1070 cpu_latency_qos_remove_request(&hba->pm_qos_req); 1071 hba->pm_qos_enabled = false; 1072 } 1073 1074 /** 1075 * ufshcd_pm_qos_update - update PM QoS request 1076 * @hba: per adapter instance 1077 * @on: If True, vote for perf PM QoS mode otherwise power save mode 1078 */ 1079 void ufshcd_pm_qos_update(struct ufs_hba *hba, bool on) 1080 { 1081 guard(mutex)(&hba->pm_qos_mutex); 1082 1083 if (!hba->pm_qos_enabled) 1084 return; 1085 1086 cpu_latency_qos_update_request(&hba->pm_qos_req, on ? 0 : PM_QOS_DEFAULT_VALUE); 1087 } 1088 EXPORT_SYMBOL_GPL(ufshcd_pm_qos_update); 1089 1090 /** 1091 * ufshcd_set_clk_freq - set UFS controller clock frequencies 1092 * @hba: per adapter instance 1093 * @scale_up: If True, set max possible frequency othewise set low frequency 1094 * 1095 * Return: 0 if successful; < 0 upon failure. 1096 */ 1097 static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up) 1098 { 1099 int ret = 0; 1100 struct ufs_clk_info *clki; 1101 struct list_head *head = &hba->clk_list_head; 1102 1103 if (list_empty(head)) 1104 goto out; 1105 1106 list_for_each_entry(clki, head, list) { 1107 if (!IS_ERR_OR_NULL(clki->clk)) { 1108 if (scale_up && clki->max_freq) { 1109 if (clki->curr_freq == clki->max_freq) 1110 continue; 1111 1112 ret = clk_set_rate(clki->clk, clki->max_freq); 1113 if (ret) { 1114 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 1115 __func__, clki->name, 1116 clki->max_freq, ret); 1117 break; 1118 } 1119 trace_ufshcd_clk_scaling(hba, 1120 "scaled up", clki->name, 1121 clki->curr_freq, 1122 clki->max_freq); 1123 1124 clki->curr_freq = clki->max_freq; 1125 1126 } else if (!scale_up && clki->min_freq) { 1127 if (clki->curr_freq == clki->min_freq) 1128 continue; 1129 1130 ret = clk_set_rate(clki->clk, clki->min_freq); 1131 if (ret) { 1132 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 1133 __func__, clki->name, 1134 clki->min_freq, ret); 1135 break; 1136 } 1137 trace_ufshcd_clk_scaling(hba, 1138 "scaled down", clki->name, 1139 clki->curr_freq, 1140 clki->min_freq); 1141 clki->curr_freq = clki->min_freq; 1142 } 1143 } 1144 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__, 1145 clki->name, clk_get_rate(clki->clk)); 1146 } 1147 1148 out: 1149 return ret; 1150 } 1151 1152 int ufshcd_opp_config_clks(struct device *dev, struct opp_table *opp_table, 1153 struct dev_pm_opp *opp, void *data, 1154 bool scaling_down) 1155 { 1156 struct ufs_hba *hba = dev_get_drvdata(dev); 1157 struct list_head *head = &hba->clk_list_head; 1158 struct ufs_clk_info *clki; 1159 unsigned long freq; 1160 u8 idx = 0; 1161 int ret; 1162 1163 list_for_each_entry(clki, head, list) { 1164 if (!IS_ERR_OR_NULL(clki->clk)) { 1165 freq = dev_pm_opp_get_freq_indexed(opp, idx++); 1166 1167 /* Do not set rate for clocks having frequency as 0 */ 1168 if (!freq) 1169 continue; 1170 1171 ret = clk_set_rate(clki->clk, freq); 1172 if (ret) { 1173 dev_err(dev, "%s: %s clk set rate(%ldHz) failed, %d\n", 1174 __func__, clki->name, freq, ret); 1175 return ret; 1176 } 1177 1178 trace_ufshcd_clk_scaling(hba, 1179 (scaling_down ? "scaled down" : "scaled up"), 1180 clki->name, hba->clk_scaling.target_freq, freq); 1181 } 1182 } 1183 1184 return 0; 1185 } 1186 EXPORT_SYMBOL_GPL(ufshcd_opp_config_clks); 1187 1188 static int ufshcd_opp_set_rate(struct ufs_hba *hba, unsigned long freq) 1189 { 1190 struct dev_pm_opp *opp; 1191 int ret; 1192 1193 opp = dev_pm_opp_find_freq_floor_indexed(hba->dev, 1194 &freq, 0); 1195 if (IS_ERR(opp)) 1196 return PTR_ERR(opp); 1197 1198 ret = dev_pm_opp_set_opp(hba->dev, opp); 1199 dev_pm_opp_put(opp); 1200 1201 return ret; 1202 } 1203 1204 /** 1205 * ufshcd_scale_clks - scale up or scale down UFS controller clocks 1206 * @hba: per adapter instance 1207 * @freq: frequency to scale 1208 * @scale_up: True if scaling up and false if scaling down 1209 * 1210 * Return: 0 if successful; < 0 upon failure. 1211 */ 1212 static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq, 1213 bool scale_up) 1214 { 1215 int ret = 0; 1216 ktime_t start = ktime_get(); 1217 1218 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, freq, PRE_CHANGE); 1219 if (ret) 1220 goto out; 1221 1222 if (hba->use_pm_opp) 1223 ret = ufshcd_opp_set_rate(hba, freq); 1224 else 1225 ret = ufshcd_set_clk_freq(hba, scale_up); 1226 if (ret) 1227 goto out; 1228 1229 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, freq, POST_CHANGE); 1230 if (ret) { 1231 if (hba->use_pm_opp) 1232 ufshcd_opp_set_rate(hba, 1233 hba->devfreq->previous_freq); 1234 else 1235 ufshcd_set_clk_freq(hba, !scale_up); 1236 goto out; 1237 } 1238 1239 ufshcd_pm_qos_update(hba, scale_up); 1240 1241 out: 1242 trace_ufshcd_profile_clk_scaling(hba, 1243 (scale_up ? "up" : "down"), 1244 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 1245 return ret; 1246 } 1247 1248 /** 1249 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not 1250 * @hba: per adapter instance 1251 * @freq: frequency to scale 1252 * @scale_up: True if scaling up and false if scaling down 1253 * 1254 * Return: true if scaling is required, false otherwise. 1255 */ 1256 static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba, 1257 unsigned long freq, bool scale_up) 1258 { 1259 struct ufs_clk_info *clki; 1260 struct list_head *head = &hba->clk_list_head; 1261 1262 if (list_empty(head)) 1263 return false; 1264 1265 if (hba->use_pm_opp) 1266 return freq != hba->clk_scaling.target_freq; 1267 1268 list_for_each_entry(clki, head, list) { 1269 if (!IS_ERR_OR_NULL(clki->clk)) { 1270 if (scale_up && clki->max_freq) { 1271 if (clki->curr_freq == clki->max_freq) 1272 continue; 1273 return true; 1274 } else if (!scale_up && clki->min_freq) { 1275 if (clki->curr_freq == clki->min_freq) 1276 continue; 1277 return true; 1278 } 1279 } 1280 } 1281 1282 return false; 1283 } 1284 1285 /* 1286 * Determine the number of pending commands by counting the bits in the SCSI 1287 * device budget maps. This approach has been selected because a bit is set in 1288 * the budget map before scsi_host_queue_ready() checks the host_self_blocked 1289 * flag. The host_self_blocked flag can be modified by calling 1290 * scsi_block_requests() or scsi_unblock_requests(). 1291 */ 1292 static u32 ufshcd_pending_cmds(struct ufs_hba *hba) 1293 { 1294 struct scsi_device *sdev; 1295 unsigned long flags; 1296 u32 pending = 0; 1297 1298 spin_lock_irqsave(hba->host->host_lock, flags); 1299 __shost_for_each_device(sdev, hba->host) 1300 pending += scsi_device_busy(sdev); 1301 spin_unlock_irqrestore(hba->host->host_lock, flags); 1302 1303 return pending; 1304 } 1305 1306 /* 1307 * Wait until all pending SCSI commands and TMFs have finished or the timeout 1308 * has expired. 1309 * 1310 * Return: 0 upon success; -EBUSY upon timeout. 1311 */ 1312 static int ufshcd_wait_for_pending_cmds(struct ufs_hba *hba, 1313 u64 wait_timeout_us) 1314 { 1315 int ret = 0; 1316 u32 tm_doorbell; 1317 u32 tr_pending; 1318 bool timeout = false, do_last_check = false; 1319 ktime_t start; 1320 1321 ufshcd_hold(hba); 1322 /* 1323 * Wait for all the outstanding tasks/transfer requests. 1324 * Verify by checking the doorbell registers are clear. 1325 */ 1326 start = ktime_get(); 1327 do { 1328 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) { 1329 ret = -EBUSY; 1330 goto out; 1331 } 1332 1333 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL); 1334 tr_pending = ufshcd_pending_cmds(hba); 1335 if (!tm_doorbell && !tr_pending) { 1336 timeout = false; 1337 break; 1338 } else if (do_last_check) { 1339 break; 1340 } 1341 1342 io_schedule_timeout(msecs_to_jiffies(20)); 1343 if (ktime_to_us(ktime_sub(ktime_get(), start)) > 1344 wait_timeout_us) { 1345 timeout = true; 1346 /* 1347 * We might have scheduled out for long time so make 1348 * sure to check if doorbells are cleared by this time 1349 * or not. 1350 */ 1351 do_last_check = true; 1352 } 1353 } while (tm_doorbell || tr_pending); 1354 1355 if (timeout) { 1356 dev_err(hba->dev, 1357 "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n", 1358 __func__, tm_doorbell, tr_pending); 1359 ret = -EBUSY; 1360 } 1361 out: 1362 ufshcd_release(hba); 1363 return ret; 1364 } 1365 1366 /** 1367 * ufshcd_scale_gear - scale up/down UFS gear 1368 * @hba: per adapter instance 1369 * @target_gear: target gear to scale to 1370 * @scale_up: True for scaling up gear and false for scaling down 1371 * 1372 * Return: 0 for success; -EBUSY if scaling can't happen at this time; 1373 * non-zero for any other errors. 1374 */ 1375 static int ufshcd_scale_gear(struct ufs_hba *hba, u32 target_gear, bool scale_up) 1376 { 1377 int ret = 0; 1378 struct ufs_pa_layer_attr new_pwr_info; 1379 1380 if (target_gear) { 1381 new_pwr_info = hba->pwr_info; 1382 new_pwr_info.gear_tx = target_gear; 1383 new_pwr_info.gear_rx = target_gear; 1384 1385 goto config_pwr_mode; 1386 } 1387 1388 /* Legacy gear scaling, in case vops_freq_to_gear_speed() is not implemented */ 1389 if (scale_up) { 1390 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info, 1391 sizeof(struct ufs_pa_layer_attr)); 1392 } else { 1393 memcpy(&new_pwr_info, &hba->pwr_info, 1394 sizeof(struct ufs_pa_layer_attr)); 1395 1396 if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear || 1397 hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) { 1398 /* save the current power mode */ 1399 memcpy(&hba->clk_scaling.saved_pwr_info, 1400 &hba->pwr_info, 1401 sizeof(struct ufs_pa_layer_attr)); 1402 1403 /* scale down gear */ 1404 new_pwr_info.gear_tx = hba->clk_scaling.min_gear; 1405 new_pwr_info.gear_rx = hba->clk_scaling.min_gear; 1406 } 1407 } 1408 1409 config_pwr_mode: 1410 /* check if the power mode needs to be changed or not? */ 1411 ret = ufshcd_config_pwr_mode(hba, &new_pwr_info); 1412 if (ret) 1413 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)", 1414 __func__, ret, 1415 hba->pwr_info.gear_tx, hba->pwr_info.gear_rx, 1416 new_pwr_info.gear_tx, new_pwr_info.gear_rx); 1417 1418 return ret; 1419 } 1420 1421 /* 1422 * Wait until all pending SCSI commands and TMFs have finished or the timeout 1423 * has expired. 1424 * 1425 * Return: 0 upon success; -EBUSY upon timeout. 1426 */ 1427 static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba, u64 timeout_us) 1428 { 1429 int ret = 0; 1430 /* 1431 * make sure that there are no outstanding requests when 1432 * clock scaling is in progress 1433 */ 1434 mutex_lock(&hba->host->scan_mutex); 1435 blk_mq_quiesce_tagset(&hba->host->tag_set); 1436 mutex_lock(&hba->wb_mutex); 1437 down_write(&hba->clk_scaling_lock); 1438 1439 if (!hba->clk_scaling.is_allowed || 1440 ufshcd_wait_for_pending_cmds(hba, timeout_us)) { 1441 ret = -EBUSY; 1442 up_write(&hba->clk_scaling_lock); 1443 mutex_unlock(&hba->wb_mutex); 1444 blk_mq_unquiesce_tagset(&hba->host->tag_set); 1445 mutex_unlock(&hba->host->scan_mutex); 1446 goto out; 1447 } 1448 1449 /* let's not get into low power until clock scaling is completed */ 1450 ufshcd_hold(hba); 1451 1452 out: 1453 return ret; 1454 } 1455 1456 static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err) 1457 { 1458 up_write(&hba->clk_scaling_lock); 1459 mutex_unlock(&hba->wb_mutex); 1460 blk_mq_unquiesce_tagset(&hba->host->tag_set); 1461 mutex_unlock(&hba->host->scan_mutex); 1462 1463 /* Enable Write Booster if current gear requires it else disable it */ 1464 if (ufshcd_enable_wb_if_scaling_up(hba) && !err) 1465 ufshcd_wb_toggle(hba, hba->pwr_info.gear_rx >= hba->clk_scaling.wb_gear); 1466 1467 ufshcd_release(hba); 1468 } 1469 1470 /** 1471 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear 1472 * @hba: per adapter instance 1473 * @freq: frequency to scale 1474 * @scale_up: True for scaling up and false for scalin down 1475 * 1476 * Return: 0 for success; -EBUSY if scaling can't happen at this time; non-zero 1477 * for any other errors. 1478 */ 1479 static int ufshcd_devfreq_scale(struct ufs_hba *hba, unsigned long freq, 1480 bool scale_up) 1481 { 1482 u32 old_gear = hba->pwr_info.gear_rx; 1483 u32 new_gear = 0; 1484 int ret = 0; 1485 1486 new_gear = ufshcd_vops_freq_to_gear_speed(hba, freq); 1487 1488 ret = ufshcd_clock_scaling_prepare(hba, 1 * USEC_PER_SEC); 1489 if (ret) 1490 return ret; 1491 1492 /* scale down the gear before scaling down clocks */ 1493 if (!scale_up) { 1494 ret = ufshcd_scale_gear(hba, new_gear, false); 1495 if (ret) 1496 goto out_unprepare; 1497 } 1498 1499 ret = ufshcd_scale_clks(hba, freq, scale_up); 1500 if (ret) { 1501 if (!scale_up) 1502 ufshcd_scale_gear(hba, old_gear, true); 1503 goto out_unprepare; 1504 } 1505 1506 /* scale up the gear after scaling up clocks */ 1507 if (scale_up) { 1508 ret = ufshcd_scale_gear(hba, new_gear, true); 1509 if (ret) { 1510 ufshcd_scale_clks(hba, hba->devfreq->previous_freq, 1511 false); 1512 goto out_unprepare; 1513 } 1514 } 1515 1516 out_unprepare: 1517 ufshcd_clock_scaling_unprepare(hba, ret); 1518 return ret; 1519 } 1520 1521 static void ufshcd_clk_scaling_suspend_work(struct work_struct *work) 1522 { 1523 struct ufs_hba *hba = container_of(work, struct ufs_hba, 1524 clk_scaling.suspend_work); 1525 1526 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1527 { 1528 if (hba->clk_scaling.active_reqs || 1529 hba->clk_scaling.is_suspended) 1530 return; 1531 1532 hba->clk_scaling.is_suspended = true; 1533 hba->clk_scaling.window_start_t = 0; 1534 } 1535 1536 devfreq_suspend_device(hba->devfreq); 1537 } 1538 1539 static void ufshcd_clk_scaling_resume_work(struct work_struct *work) 1540 { 1541 struct ufs_hba *hba = container_of(work, struct ufs_hba, 1542 clk_scaling.resume_work); 1543 1544 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1545 { 1546 if (!hba->clk_scaling.is_suspended) 1547 return; 1548 hba->clk_scaling.is_suspended = false; 1549 } 1550 1551 devfreq_resume_device(hba->devfreq); 1552 } 1553 1554 static int ufshcd_devfreq_target(struct device *dev, 1555 unsigned long *freq, u32 flags) 1556 { 1557 int ret = 0; 1558 struct ufs_hba *hba = dev_get_drvdata(dev); 1559 ktime_t start; 1560 bool scale_up = false, sched_clk_scaling_suspend_work = false; 1561 struct list_head *clk_list = &hba->clk_list_head; 1562 struct ufs_clk_info *clki; 1563 1564 if (!ufshcd_is_clkscaling_supported(hba)) 1565 return -EINVAL; 1566 1567 if (hba->use_pm_opp) { 1568 struct dev_pm_opp *opp; 1569 1570 /* Get the recommended frequency from OPP framework */ 1571 opp = devfreq_recommended_opp(dev, freq, flags); 1572 if (IS_ERR(opp)) 1573 return PTR_ERR(opp); 1574 1575 dev_pm_opp_put(opp); 1576 } else { 1577 /* Override with the closest supported frequency */ 1578 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, 1579 list); 1580 *freq = (unsigned long) clk_round_rate(clki->clk, *freq); 1581 } 1582 1583 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1584 { 1585 if (ufshcd_eh_in_progress(hba)) 1586 return 0; 1587 1588 /* Skip scaling clock when clock scaling is suspended */ 1589 if (hba->clk_scaling.is_suspended) { 1590 dev_warn(hba->dev, "clock scaling is suspended, skip"); 1591 return 0; 1592 } 1593 1594 if (!hba->clk_scaling.active_reqs) 1595 sched_clk_scaling_suspend_work = true; 1596 1597 if (list_empty(clk_list)) 1598 goto out; 1599 1600 /* Decide based on the target or rounded-off frequency and update */ 1601 if (hba->use_pm_opp) 1602 scale_up = *freq > hba->clk_scaling.target_freq; 1603 else 1604 scale_up = *freq == clki->max_freq; 1605 1606 if (!hba->use_pm_opp && !scale_up) 1607 *freq = clki->min_freq; 1608 1609 /* Update the frequency */ 1610 if (!ufshcd_is_devfreq_scaling_required(hba, *freq, scale_up)) { 1611 ret = 0; 1612 goto out; /* no state change required */ 1613 } 1614 } 1615 1616 start = ktime_get(); 1617 ret = ufshcd_devfreq_scale(hba, *freq, scale_up); 1618 if (!ret) 1619 hba->clk_scaling.target_freq = *freq; 1620 1621 trace_ufshcd_profile_clk_scaling(hba, 1622 (scale_up ? "up" : "down"), 1623 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 1624 1625 out: 1626 if (sched_clk_scaling_suspend_work && 1627 (!scale_up || hba->clk_scaling.suspend_on_no_request)) 1628 queue_work(hba->clk_scaling.workq, 1629 &hba->clk_scaling.suspend_work); 1630 1631 return ret; 1632 } 1633 1634 static int ufshcd_devfreq_get_dev_status(struct device *dev, 1635 struct devfreq_dev_status *stat) 1636 { 1637 struct ufs_hba *hba = dev_get_drvdata(dev); 1638 struct ufs_clk_scaling *scaling = &hba->clk_scaling; 1639 ktime_t curr_t; 1640 1641 if (!ufshcd_is_clkscaling_supported(hba)) 1642 return -EINVAL; 1643 1644 memset(stat, 0, sizeof(*stat)); 1645 1646 guard(spinlock_irqsave)(&hba->clk_scaling.lock); 1647 1648 curr_t = ktime_get(); 1649 if (!scaling->window_start_t) 1650 goto start_window; 1651 1652 /* 1653 * If current frequency is 0, then the ondemand governor considers 1654 * there's no initial frequency set. And it always requests to set 1655 * to max. frequency. 1656 */ 1657 if (hba->use_pm_opp) { 1658 stat->current_frequency = hba->clk_scaling.target_freq; 1659 } else { 1660 struct list_head *clk_list = &hba->clk_list_head; 1661 struct ufs_clk_info *clki; 1662 1663 clki = list_first_entry(clk_list, struct ufs_clk_info, list); 1664 stat->current_frequency = clki->curr_freq; 1665 } 1666 1667 if (scaling->is_busy_started) 1668 scaling->tot_busy_t += ktime_us_delta(curr_t, 1669 scaling->busy_start_t); 1670 stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t); 1671 stat->busy_time = scaling->tot_busy_t; 1672 start_window: 1673 scaling->window_start_t = curr_t; 1674 scaling->tot_busy_t = 0; 1675 1676 if (scaling->active_reqs) { 1677 scaling->busy_start_t = curr_t; 1678 scaling->is_busy_started = true; 1679 } else { 1680 scaling->busy_start_t = 0; 1681 scaling->is_busy_started = false; 1682 } 1683 1684 return 0; 1685 } 1686 1687 static int ufshcd_devfreq_init(struct ufs_hba *hba) 1688 { 1689 struct list_head *clk_list = &hba->clk_list_head; 1690 struct ufs_clk_info *clki; 1691 struct devfreq *devfreq; 1692 int ret; 1693 1694 /* Skip devfreq if we don't have any clocks in the list */ 1695 if (list_empty(clk_list)) 1696 return 0; 1697 1698 if (!hba->use_pm_opp) { 1699 clki = list_first_entry(clk_list, struct ufs_clk_info, list); 1700 dev_pm_opp_add(hba->dev, clki->min_freq, 0); 1701 dev_pm_opp_add(hba->dev, clki->max_freq, 0); 1702 } 1703 1704 ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile, 1705 &hba->vps->ondemand_data); 1706 devfreq = devfreq_add_device(hba->dev, 1707 &hba->vps->devfreq_profile, 1708 DEVFREQ_GOV_SIMPLE_ONDEMAND, 1709 &hba->vps->ondemand_data); 1710 if (IS_ERR(devfreq)) { 1711 ret = PTR_ERR(devfreq); 1712 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret); 1713 1714 if (!hba->use_pm_opp) { 1715 dev_pm_opp_remove(hba->dev, clki->min_freq); 1716 dev_pm_opp_remove(hba->dev, clki->max_freq); 1717 } 1718 return ret; 1719 } 1720 1721 hba->devfreq = devfreq; 1722 1723 return 0; 1724 } 1725 1726 static void ufshcd_devfreq_remove(struct ufs_hba *hba) 1727 { 1728 struct list_head *clk_list = &hba->clk_list_head; 1729 1730 if (!hba->devfreq) 1731 return; 1732 1733 devfreq_remove_device(hba->devfreq); 1734 hba->devfreq = NULL; 1735 1736 if (!hba->use_pm_opp) { 1737 struct ufs_clk_info *clki; 1738 1739 clki = list_first_entry(clk_list, struct ufs_clk_info, list); 1740 dev_pm_opp_remove(hba->dev, clki->min_freq); 1741 dev_pm_opp_remove(hba->dev, clki->max_freq); 1742 } 1743 } 1744 1745 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba) 1746 { 1747 bool suspend = false; 1748 1749 cancel_work_sync(&hba->clk_scaling.suspend_work); 1750 cancel_work_sync(&hba->clk_scaling.resume_work); 1751 1752 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1753 { 1754 if (!hba->clk_scaling.is_suspended) { 1755 suspend = true; 1756 hba->clk_scaling.is_suspended = true; 1757 hba->clk_scaling.window_start_t = 0; 1758 } 1759 } 1760 1761 if (suspend) 1762 devfreq_suspend_device(hba->devfreq); 1763 } 1764 1765 static void ufshcd_resume_clkscaling(struct ufs_hba *hba) 1766 { 1767 bool resume = false; 1768 1769 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1770 { 1771 if (hba->clk_scaling.is_suspended) { 1772 resume = true; 1773 hba->clk_scaling.is_suspended = false; 1774 } 1775 } 1776 1777 if (resume) 1778 devfreq_resume_device(hba->devfreq); 1779 } 1780 1781 static ssize_t ufshcd_clkscale_enable_show(struct device *dev, 1782 struct device_attribute *attr, char *buf) 1783 { 1784 struct ufs_hba *hba = dev_get_drvdata(dev); 1785 1786 return sysfs_emit(buf, "%d\n", hba->clk_scaling.is_enabled); 1787 } 1788 1789 static ssize_t ufshcd_clkscale_enable_store(struct device *dev, 1790 struct device_attribute *attr, const char *buf, size_t count) 1791 { 1792 struct ufs_hba *hba = dev_get_drvdata(dev); 1793 struct ufs_clk_info *clki; 1794 unsigned long freq; 1795 u32 value; 1796 int err = 0; 1797 1798 if (kstrtou32(buf, 0, &value)) 1799 return -EINVAL; 1800 1801 down(&hba->host_sem); 1802 if (!ufshcd_is_user_access_allowed(hba)) { 1803 err = -EBUSY; 1804 goto out; 1805 } 1806 1807 value = !!value; 1808 if (value == hba->clk_scaling.is_enabled) 1809 goto out; 1810 1811 ufshcd_rpm_get_sync(hba); 1812 ufshcd_hold(hba); 1813 1814 hba->clk_scaling.is_enabled = value; 1815 1816 if (value) { 1817 ufshcd_resume_clkscaling(hba); 1818 goto out_rel; 1819 } 1820 1821 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list); 1822 freq = clki->max_freq; 1823 1824 ufshcd_suspend_clkscaling(hba); 1825 1826 if (!ufshcd_is_devfreq_scaling_required(hba, freq, true)) 1827 goto out_rel; 1828 1829 err = ufshcd_devfreq_scale(hba, freq, true); 1830 if (err) 1831 dev_err(hba->dev, "%s: failed to scale clocks up %d\n", 1832 __func__, err); 1833 else 1834 hba->clk_scaling.target_freq = freq; 1835 1836 out_rel: 1837 ufshcd_release(hba); 1838 ufshcd_rpm_put_sync(hba); 1839 out: 1840 up(&hba->host_sem); 1841 return err ? err : count; 1842 } 1843 1844 static void ufshcd_init_clk_scaling_sysfs(struct ufs_hba *hba) 1845 { 1846 hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show; 1847 hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store; 1848 sysfs_attr_init(&hba->clk_scaling.enable_attr.attr); 1849 hba->clk_scaling.enable_attr.attr.name = "clkscale_enable"; 1850 hba->clk_scaling.enable_attr.attr.mode = 0644; 1851 if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr)) 1852 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n"); 1853 } 1854 1855 static void ufshcd_remove_clk_scaling_sysfs(struct ufs_hba *hba) 1856 { 1857 if (hba->clk_scaling.enable_attr.attr.name) 1858 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr); 1859 } 1860 1861 static void ufshcd_init_clk_scaling(struct ufs_hba *hba) 1862 { 1863 if (!ufshcd_is_clkscaling_supported(hba)) 1864 return; 1865 1866 if (!hba->clk_scaling.min_gear) 1867 hba->clk_scaling.min_gear = UFS_HS_G1; 1868 1869 if (!hba->clk_scaling.wb_gear) 1870 /* Use intermediate gear speed HS_G3 as the default wb_gear */ 1871 hba->clk_scaling.wb_gear = UFS_HS_G3; 1872 1873 INIT_WORK(&hba->clk_scaling.suspend_work, 1874 ufshcd_clk_scaling_suspend_work); 1875 INIT_WORK(&hba->clk_scaling.resume_work, 1876 ufshcd_clk_scaling_resume_work); 1877 1878 spin_lock_init(&hba->clk_scaling.lock); 1879 1880 hba->clk_scaling.workq = alloc_ordered_workqueue( 1881 "ufs_clkscaling_%d", WQ_MEM_RECLAIM, hba->host->host_no); 1882 1883 hba->clk_scaling.is_initialized = true; 1884 } 1885 1886 static void ufshcd_exit_clk_scaling(struct ufs_hba *hba) 1887 { 1888 if (!hba->clk_scaling.is_initialized) 1889 return; 1890 1891 ufshcd_remove_clk_scaling_sysfs(hba); 1892 destroy_workqueue(hba->clk_scaling.workq); 1893 ufshcd_devfreq_remove(hba); 1894 hba->clk_scaling.is_initialized = false; 1895 } 1896 1897 static void ufshcd_ungate_work(struct work_struct *work) 1898 { 1899 int ret; 1900 struct ufs_hba *hba = container_of(work, struct ufs_hba, 1901 clk_gating.ungate_work); 1902 1903 cancel_delayed_work_sync(&hba->clk_gating.gate_work); 1904 1905 scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) { 1906 if (hba->clk_gating.state == CLKS_ON) 1907 return; 1908 } 1909 1910 ufshcd_hba_vreg_set_hpm(hba); 1911 ufshcd_setup_clocks(hba, true); 1912 1913 ufshcd_enable_irq(hba); 1914 1915 /* Exit from hibern8 */ 1916 if (ufshcd_can_hibern8_during_gating(hba)) { 1917 /* Prevent gating in this path */ 1918 hba->clk_gating.is_suspended = true; 1919 if (ufshcd_is_link_hibern8(hba)) { 1920 ret = ufshcd_uic_hibern8_exit(hba); 1921 if (ret) 1922 dev_err(hba->dev, "%s: hibern8 exit failed %d\n", 1923 __func__, ret); 1924 else 1925 ufshcd_set_link_active(hba); 1926 } 1927 hba->clk_gating.is_suspended = false; 1928 } 1929 } 1930 1931 /** 1932 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release. 1933 * Also, exit from hibern8 mode and set the link as active. 1934 * @hba: per adapter instance 1935 */ 1936 void ufshcd_hold(struct ufs_hba *hba) 1937 { 1938 bool flush_result; 1939 unsigned long flags; 1940 1941 if (!ufshcd_is_clkgating_allowed(hba) || 1942 !hba->clk_gating.is_initialized) 1943 return; 1944 spin_lock_irqsave(&hba->clk_gating.lock, flags); 1945 hba->clk_gating.active_reqs++; 1946 1947 start: 1948 switch (hba->clk_gating.state) { 1949 case CLKS_ON: 1950 /* 1951 * Wait for the ungate work to complete if in progress. 1952 * Though the clocks may be in ON state, the link could 1953 * still be in hibner8 state if hibern8 is allowed 1954 * during clock gating. 1955 * Make sure we exit hibern8 state also in addition to 1956 * clocks being ON. 1957 */ 1958 if (ufshcd_can_hibern8_during_gating(hba) && 1959 ufshcd_is_link_hibern8(hba)) { 1960 spin_unlock_irqrestore(&hba->clk_gating.lock, flags); 1961 flush_result = flush_work(&hba->clk_gating.ungate_work); 1962 if (hba->clk_gating.is_suspended && !flush_result) 1963 return; 1964 spin_lock_irqsave(&hba->clk_gating.lock, flags); 1965 goto start; 1966 } 1967 break; 1968 case REQ_CLKS_OFF: 1969 if (cancel_delayed_work(&hba->clk_gating.gate_work)) { 1970 hba->clk_gating.state = CLKS_ON; 1971 trace_ufshcd_clk_gating(hba, 1972 hba->clk_gating.state); 1973 break; 1974 } 1975 /* 1976 * If we are here, it means gating work is either done or 1977 * currently running. Hence, fall through to cancel gating 1978 * work and to enable clocks. 1979 */ 1980 fallthrough; 1981 case CLKS_OFF: 1982 hba->clk_gating.state = REQ_CLKS_ON; 1983 trace_ufshcd_clk_gating(hba, 1984 hba->clk_gating.state); 1985 queue_work(hba->clk_gating.clk_gating_workq, 1986 &hba->clk_gating.ungate_work); 1987 /* 1988 * fall through to check if we should wait for this 1989 * work to be done or not. 1990 */ 1991 fallthrough; 1992 case REQ_CLKS_ON: 1993 spin_unlock_irqrestore(&hba->clk_gating.lock, flags); 1994 flush_work(&hba->clk_gating.ungate_work); 1995 /* Make sure state is CLKS_ON before returning */ 1996 spin_lock_irqsave(&hba->clk_gating.lock, flags); 1997 goto start; 1998 default: 1999 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n", 2000 __func__, hba->clk_gating.state); 2001 break; 2002 } 2003 spin_unlock_irqrestore(&hba->clk_gating.lock, flags); 2004 } 2005 EXPORT_SYMBOL_GPL(ufshcd_hold); 2006 2007 static void ufshcd_gate_work(struct work_struct *work) 2008 { 2009 struct ufs_hba *hba = container_of(work, struct ufs_hba, 2010 clk_gating.gate_work.work); 2011 int ret; 2012 2013 scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) { 2014 /* 2015 * In case you are here to cancel this work the gating state 2016 * would be marked as REQ_CLKS_ON. In this case save time by 2017 * skipping the gating work and exit after changing the clock 2018 * state to CLKS_ON. 2019 */ 2020 if (hba->clk_gating.is_suspended || 2021 hba->clk_gating.state != REQ_CLKS_OFF) { 2022 hba->clk_gating.state = CLKS_ON; 2023 trace_ufshcd_clk_gating(hba, 2024 hba->clk_gating.state); 2025 return; 2026 } 2027 2028 if (hba->clk_gating.active_reqs) 2029 return; 2030 } 2031 2032 scoped_guard(spinlock_irqsave, hba->host->host_lock) { 2033 if (ufshcd_is_ufs_dev_busy(hba) || 2034 hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) 2035 return; 2036 } 2037 2038 /* put the link into hibern8 mode before turning off clocks */ 2039 if (ufshcd_can_hibern8_during_gating(hba)) { 2040 ret = ufshcd_uic_hibern8_enter(hba); 2041 if (ret) { 2042 hba->clk_gating.state = CLKS_ON; 2043 dev_err(hba->dev, "%s: hibern8 enter failed %d\n", 2044 __func__, ret); 2045 trace_ufshcd_clk_gating(hba, 2046 hba->clk_gating.state); 2047 return; 2048 } 2049 ufshcd_set_link_hibern8(hba); 2050 } 2051 2052 ufshcd_disable_irq(hba); 2053 2054 ufshcd_setup_clocks(hba, false); 2055 2056 /* Put the host controller in low power mode if possible */ 2057 ufshcd_hba_vreg_set_lpm(hba); 2058 /* 2059 * In case you are here to cancel this work the gating state 2060 * would be marked as REQ_CLKS_ON. In this case keep the state 2061 * as REQ_CLKS_ON which would anyway imply that clocks are off 2062 * and a request to turn them on is pending. By doing this way, 2063 * we keep the state machine in tact and this would ultimately 2064 * prevent from doing cancel work multiple times when there are 2065 * new requests arriving before the current cancel work is done. 2066 */ 2067 guard(spinlock_irqsave)(&hba->clk_gating.lock); 2068 if (hba->clk_gating.state == REQ_CLKS_OFF) { 2069 hba->clk_gating.state = CLKS_OFF; 2070 trace_ufshcd_clk_gating(hba, 2071 hba->clk_gating.state); 2072 } 2073 } 2074 2075 static void __ufshcd_release(struct ufs_hba *hba) 2076 { 2077 lockdep_assert_held(&hba->clk_gating.lock); 2078 2079 if (!ufshcd_is_clkgating_allowed(hba)) 2080 return; 2081 2082 hba->clk_gating.active_reqs--; 2083 2084 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended || 2085 !hba->clk_gating.is_initialized || 2086 hba->clk_gating.state == CLKS_OFF) 2087 return; 2088 2089 scoped_guard(spinlock_irqsave, hba->host->host_lock) { 2090 if (ufshcd_has_pending_tasks(hba) || 2091 hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) 2092 return; 2093 } 2094 2095 hba->clk_gating.state = REQ_CLKS_OFF; 2096 trace_ufshcd_clk_gating(hba, hba->clk_gating.state); 2097 queue_delayed_work(hba->clk_gating.clk_gating_workq, 2098 &hba->clk_gating.gate_work, 2099 msecs_to_jiffies(hba->clk_gating.delay_ms)); 2100 } 2101 2102 void ufshcd_release(struct ufs_hba *hba) 2103 { 2104 guard(spinlock_irqsave)(&hba->clk_gating.lock); 2105 __ufshcd_release(hba); 2106 } 2107 EXPORT_SYMBOL_GPL(ufshcd_release); 2108 2109 static ssize_t ufshcd_clkgate_delay_show(struct device *dev, 2110 struct device_attribute *attr, char *buf) 2111 { 2112 struct ufs_hba *hba = dev_get_drvdata(dev); 2113 2114 return sysfs_emit(buf, "%lu\n", hba->clk_gating.delay_ms); 2115 } 2116 2117 void ufshcd_clkgate_delay_set(struct device *dev, unsigned long value) 2118 { 2119 struct ufs_hba *hba = dev_get_drvdata(dev); 2120 2121 guard(spinlock_irqsave)(&hba->clk_gating.lock); 2122 hba->clk_gating.delay_ms = value; 2123 } 2124 EXPORT_SYMBOL_GPL(ufshcd_clkgate_delay_set); 2125 2126 static ssize_t ufshcd_clkgate_delay_store(struct device *dev, 2127 struct device_attribute *attr, const char *buf, size_t count) 2128 { 2129 unsigned long value; 2130 2131 if (kstrtoul(buf, 0, &value)) 2132 return -EINVAL; 2133 2134 ufshcd_clkgate_delay_set(dev, value); 2135 return count; 2136 } 2137 2138 static ssize_t ufshcd_clkgate_enable_show(struct device *dev, 2139 struct device_attribute *attr, char *buf) 2140 { 2141 struct ufs_hba *hba = dev_get_drvdata(dev); 2142 2143 return sysfs_emit(buf, "%d\n", hba->clk_gating.is_enabled); 2144 } 2145 2146 static ssize_t ufshcd_clkgate_enable_store(struct device *dev, 2147 struct device_attribute *attr, const char *buf, size_t count) 2148 { 2149 struct ufs_hba *hba = dev_get_drvdata(dev); 2150 u32 value; 2151 2152 if (kstrtou32(buf, 0, &value)) 2153 return -EINVAL; 2154 2155 value = !!value; 2156 2157 guard(spinlock_irqsave)(&hba->clk_gating.lock); 2158 2159 if (value == hba->clk_gating.is_enabled) 2160 return count; 2161 2162 if (value) 2163 __ufshcd_release(hba); 2164 else 2165 hba->clk_gating.active_reqs++; 2166 2167 hba->clk_gating.is_enabled = value; 2168 2169 return count; 2170 } 2171 2172 static void ufshcd_init_clk_gating_sysfs(struct ufs_hba *hba) 2173 { 2174 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show; 2175 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store; 2176 sysfs_attr_init(&hba->clk_gating.delay_attr.attr); 2177 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms"; 2178 hba->clk_gating.delay_attr.attr.mode = 0644; 2179 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr)) 2180 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n"); 2181 2182 hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show; 2183 hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store; 2184 sysfs_attr_init(&hba->clk_gating.enable_attr.attr); 2185 hba->clk_gating.enable_attr.attr.name = "clkgate_enable"; 2186 hba->clk_gating.enable_attr.attr.mode = 0644; 2187 if (device_create_file(hba->dev, &hba->clk_gating.enable_attr)) 2188 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n"); 2189 } 2190 2191 static void ufshcd_remove_clk_gating_sysfs(struct ufs_hba *hba) 2192 { 2193 if (hba->clk_gating.delay_attr.attr.name) 2194 device_remove_file(hba->dev, &hba->clk_gating.delay_attr); 2195 if (hba->clk_gating.enable_attr.attr.name) 2196 device_remove_file(hba->dev, &hba->clk_gating.enable_attr); 2197 } 2198 2199 static void ufshcd_init_clk_gating(struct ufs_hba *hba) 2200 { 2201 if (!ufshcd_is_clkgating_allowed(hba)) 2202 return; 2203 2204 hba->clk_gating.state = CLKS_ON; 2205 2206 hba->clk_gating.delay_ms = 150; 2207 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work); 2208 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work); 2209 2210 hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue( 2211 "ufs_clk_gating_%d", WQ_MEM_RECLAIM | WQ_HIGHPRI, 2212 hba->host->host_no); 2213 2214 ufshcd_init_clk_gating_sysfs(hba); 2215 2216 hba->clk_gating.is_enabled = true; 2217 hba->clk_gating.is_initialized = true; 2218 } 2219 2220 static void ufshcd_exit_clk_gating(struct ufs_hba *hba) 2221 { 2222 if (!hba->clk_gating.is_initialized) 2223 return; 2224 2225 ufshcd_remove_clk_gating_sysfs(hba); 2226 2227 /* Ungate the clock if necessary. */ 2228 ufshcd_hold(hba); 2229 hba->clk_gating.is_initialized = false; 2230 ufshcd_release(hba); 2231 2232 destroy_workqueue(hba->clk_gating.clk_gating_workq); 2233 } 2234 2235 static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba) 2236 { 2237 bool queue_resume_work = false; 2238 ktime_t curr_t; 2239 2240 if (!ufshcd_is_clkscaling_supported(hba)) 2241 return; 2242 2243 curr_t = ktime_get(); 2244 2245 guard(spinlock_irqsave)(&hba->clk_scaling.lock); 2246 2247 if (!hba->clk_scaling.active_reqs++) 2248 queue_resume_work = true; 2249 2250 if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) 2251 return; 2252 2253 if (queue_resume_work) 2254 queue_work(hba->clk_scaling.workq, 2255 &hba->clk_scaling.resume_work); 2256 2257 if (!hba->clk_scaling.window_start_t) { 2258 hba->clk_scaling.window_start_t = curr_t; 2259 hba->clk_scaling.tot_busy_t = 0; 2260 hba->clk_scaling.is_busy_started = false; 2261 } 2262 2263 if (!hba->clk_scaling.is_busy_started) { 2264 hba->clk_scaling.busy_start_t = curr_t; 2265 hba->clk_scaling.is_busy_started = true; 2266 } 2267 } 2268 2269 static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba) 2270 { 2271 struct ufs_clk_scaling *scaling = &hba->clk_scaling; 2272 2273 if (!ufshcd_is_clkscaling_supported(hba)) 2274 return; 2275 2276 guard(spinlock_irqsave)(&hba->clk_scaling.lock); 2277 2278 hba->clk_scaling.active_reqs--; 2279 if (!scaling->active_reqs && scaling->is_busy_started) { 2280 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(), 2281 scaling->busy_start_t)); 2282 scaling->busy_start_t = 0; 2283 scaling->is_busy_started = false; 2284 } 2285 } 2286 2287 static inline int ufshcd_monitor_opcode2dir(u8 opcode) 2288 { 2289 if (opcode == READ_6 || opcode == READ_10 || opcode == READ_16) 2290 return READ; 2291 else if (opcode == WRITE_6 || opcode == WRITE_10 || opcode == WRITE_16) 2292 return WRITE; 2293 else 2294 return -EINVAL; 2295 } 2296 2297 /* Must only be called for SCSI commands. */ 2298 static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba, 2299 struct scsi_cmnd *cmd) 2300 { 2301 const struct ufs_hba_monitor *m = &hba->monitor; 2302 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2303 2304 return m->enabled && 2305 (!m->chunk_size || m->chunk_size == cmd->sdb.length) && 2306 ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp); 2307 } 2308 2309 static void ufshcd_start_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2310 { 2311 int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]); 2312 unsigned long flags; 2313 2314 spin_lock_irqsave(hba->host->host_lock, flags); 2315 if (dir >= 0 && hba->monitor.nr_queued[dir]++ == 0) 2316 hba->monitor.busy_start_ts[dir] = ktime_get(); 2317 spin_unlock_irqrestore(hba->host->host_lock, flags); 2318 } 2319 2320 static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2321 { 2322 struct request *req = scsi_cmd_to_rq(cmd); 2323 const struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2324 int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]); 2325 unsigned long flags; 2326 2327 spin_lock_irqsave(hba->host->host_lock, flags); 2328 if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) { 2329 struct ufs_hba_monitor *m = &hba->monitor; 2330 ktime_t now, inc, lat; 2331 2332 now = lrbp->compl_time_stamp; 2333 inc = ktime_sub(now, m->busy_start_ts[dir]); 2334 m->total_busy[dir] = ktime_add(m->total_busy[dir], inc); 2335 m->nr_sec_rw[dir] += blk_rq_sectors(req); 2336 2337 /* Update latencies */ 2338 m->nr_req[dir]++; 2339 lat = ktime_sub(now, lrbp->issue_time_stamp); 2340 m->lat_sum[dir] += lat; 2341 if (m->lat_max[dir] < lat || !m->lat_max[dir]) 2342 m->lat_max[dir] = lat; 2343 if (m->lat_min[dir] > lat || !m->lat_min[dir]) 2344 m->lat_min[dir] = lat; 2345 2346 m->nr_queued[dir]--; 2347 /* Push forward the busy start of monitor */ 2348 m->busy_start_ts[dir] = now; 2349 } 2350 spin_unlock_irqrestore(hba->host->host_lock, flags); 2351 } 2352 2353 /* Returns %true for SCSI commands and %false for device management commands. */ 2354 static bool ufshcd_is_scsi_cmd(struct scsi_cmnd *cmd) 2355 { 2356 return !blk_mq_is_reserved_rq(scsi_cmd_to_rq(cmd)); 2357 } 2358 2359 /** 2360 * ufshcd_send_command - Send SCSI or device management commands 2361 * @hba: per adapter instance 2362 * @cmd: SCSI command or device management command pointer 2363 * @hwq: pointer to hardware queue instance 2364 */ 2365 static inline void ufshcd_send_command(struct ufs_hba *hba, 2366 struct scsi_cmnd *cmd, 2367 struct ufs_hw_queue *hwq) 2368 { 2369 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2370 const int tag = scsi_cmd_to_rq(cmd)->tag; 2371 unsigned long flags; 2372 2373 if (hba->monitor.enabled) { 2374 lrbp->issue_time_stamp = ktime_get(); 2375 lrbp->issue_time_stamp_local_clock = local_clock(); 2376 lrbp->compl_time_stamp = ktime_set(0, 0); 2377 lrbp->compl_time_stamp_local_clock = 0; 2378 } 2379 if (ufshcd_is_scsi_cmd(cmd)) { 2380 ufshcd_add_command_trace(hba, cmd, UFS_CMD_SEND); 2381 ufshcd_clk_scaling_start_busy(hba); 2382 if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) 2383 ufshcd_start_monitor(hba, cmd); 2384 } 2385 2386 if (hba->mcq_enabled) { 2387 int utrd_size = sizeof(struct utp_transfer_req_desc); 2388 struct utp_transfer_req_desc *src = lrbp->utr_descriptor_ptr; 2389 struct utp_transfer_req_desc *dest; 2390 2391 spin_lock(&hwq->sq_lock); 2392 dest = hwq->sqe_base_addr + hwq->sq_tail_slot; 2393 memcpy(dest, src, utrd_size); 2394 ufshcd_inc_sq_tail(hwq); 2395 spin_unlock(&hwq->sq_lock); 2396 } else { 2397 spin_lock_irqsave(&hba->outstanding_lock, flags); 2398 if (hba->vops && hba->vops->setup_xfer_req) 2399 hba->vops->setup_xfer_req(hba, tag, 2400 ufshcd_is_scsi_cmd(cmd)); 2401 __set_bit(tag, &hba->outstanding_reqs); 2402 ufshcd_writel(hba, 1 << tag, REG_UTP_TRANSFER_REQ_DOOR_BELL); 2403 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 2404 } 2405 } 2406 2407 /** 2408 * ufshcd_copy_sense_data - Copy sense data in case of check condition 2409 * @cmd: SCSI command 2410 */ 2411 static inline void ufshcd_copy_sense_data(struct scsi_cmnd *cmd) 2412 { 2413 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2414 u8 *const sense_buffer = cmd->sense_buffer; 2415 u16 resp_len; 2416 int len; 2417 2418 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header.data_segment_length); 2419 if (sense_buffer && resp_len) { 2420 int len_to_copy; 2421 2422 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len); 2423 len_to_copy = min_t(int, UFS_SENSE_SIZE, len); 2424 2425 memcpy(sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data, 2426 len_to_copy); 2427 } 2428 } 2429 2430 /** 2431 * ufshcd_copy_query_response() - Copy the Query Response and the data 2432 * descriptor 2433 * @hba: per adapter instance 2434 * @lrbp: pointer to local reference block 2435 * 2436 * Return: 0 upon success; < 0 upon failure. 2437 */ 2438 static 2439 int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 2440 { 2441 struct ufs_query_res *query_res = &hba->dev_cmd.query.response; 2442 2443 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE); 2444 2445 /* Get the descriptor */ 2446 if (hba->dev_cmd.query.descriptor && 2447 lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) { 2448 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + 2449 GENERAL_UPIU_REQUEST_SIZE; 2450 u16 resp_len; 2451 u16 buf_len; 2452 2453 /* data segment length */ 2454 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header 2455 .data_segment_length); 2456 buf_len = be16_to_cpu( 2457 hba->dev_cmd.query.request.upiu_req.length); 2458 if (likely(buf_len >= resp_len)) { 2459 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len); 2460 } else { 2461 dev_warn(hba->dev, 2462 "%s: rsp size %d is bigger than buffer size %d", 2463 __func__, resp_len, buf_len); 2464 return -EINVAL; 2465 } 2466 } 2467 2468 return 0; 2469 } 2470 2471 /** 2472 * ufshcd_hba_capabilities - Read controller capabilities 2473 * @hba: per adapter instance 2474 * 2475 * Return: 0 on success, negative on error. 2476 */ 2477 static inline int ufshcd_hba_capabilities(struct ufs_hba *hba) 2478 { 2479 int err; 2480 2481 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES); 2482 2483 /* nutrs and nutmrs are 0 based values */ 2484 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS_SDB) + 1; 2485 hba->nutmrs = 2486 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1; 2487 2488 hba->nortt = FIELD_GET(MASK_NUMBER_OUTSTANDING_RTT, hba->capabilities) + 1; 2489 2490 /* Read crypto capabilities */ 2491 err = ufshcd_hba_init_crypto_capabilities(hba); 2492 if (err) { 2493 dev_err(hba->dev, "crypto setup failed\n"); 2494 return err; 2495 } 2496 2497 /* 2498 * The UFSHCI 3.0 specification does not define MCQ_SUPPORT and 2499 * LSDB_SUPPORT, but [31:29] as reserved bits with reset value 0s, which 2500 * means we can simply read values regardless of version. 2501 */ 2502 hba->mcq_sup = FIELD_GET(MASK_MCQ_SUPPORT, hba->capabilities); 2503 /* 2504 * 0h: legacy single doorbell support is available 2505 * 1h: indicate that legacy single doorbell support has been removed 2506 */ 2507 if (!(hba->quirks & UFSHCD_QUIRK_BROKEN_LSDBS_CAP)) 2508 hba->lsdb_sup = !FIELD_GET(MASK_LSDB_SUPPORT, hba->capabilities); 2509 else 2510 hba->lsdb_sup = true; 2511 2512 hba->mcq_capabilities = ufshcd_readl(hba, REG_MCQCAP); 2513 2514 return 0; 2515 } 2516 2517 /** 2518 * ufshcd_ready_for_uic_cmd - Check if controller is ready 2519 * to accept UIC commands 2520 * @hba: per adapter instance 2521 * 2522 * Return: true on success, else false. 2523 */ 2524 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba) 2525 { 2526 u32 val; 2527 int ret = read_poll_timeout(ufshcd_readl, val, val & UIC_COMMAND_READY, 2528 500, uic_cmd_timeout * 1000, false, hba, 2529 REG_CONTROLLER_STATUS); 2530 return ret == 0; 2531 } 2532 2533 /** 2534 * ufshcd_get_upmcrs - Get the power mode change request status 2535 * @hba: Pointer to adapter instance 2536 * 2537 * This function gets the UPMCRS field of HCS register 2538 * 2539 * Return: value of UPMCRS field. 2540 */ 2541 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba) 2542 { 2543 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7; 2544 } 2545 2546 /** 2547 * ufshcd_dispatch_uic_cmd - Dispatch an UIC command to the Unipro layer 2548 * @hba: per adapter instance 2549 * @uic_cmd: UIC command 2550 */ 2551 static inline void 2552 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 2553 { 2554 lockdep_assert_held(&hba->uic_cmd_mutex); 2555 2556 WARN_ON(hba->active_uic_cmd); 2557 2558 hba->active_uic_cmd = uic_cmd; 2559 2560 /* Write Args */ 2561 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1); 2562 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2); 2563 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3); 2564 2565 ufshcd_add_uic_command_trace(hba, uic_cmd, UFS_CMD_SEND); 2566 2567 /* Write UIC Cmd */ 2568 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK, 2569 REG_UIC_COMMAND); 2570 } 2571 2572 /** 2573 * ufshcd_wait_for_uic_cmd - Wait for completion of an UIC command 2574 * @hba: per adapter instance 2575 * @uic_cmd: UIC command 2576 * 2577 * Return: 0 only if success. 2578 */ 2579 static int 2580 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 2581 { 2582 int ret; 2583 unsigned long flags; 2584 2585 lockdep_assert_held(&hba->uic_cmd_mutex); 2586 2587 if (wait_for_completion_timeout(&uic_cmd->done, 2588 msecs_to_jiffies(uic_cmd_timeout))) { 2589 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT; 2590 } else { 2591 ret = -ETIMEDOUT; 2592 dev_err(hba->dev, 2593 "uic cmd 0x%x with arg3 0x%x completion timeout\n", 2594 uic_cmd->command, uic_cmd->argument3); 2595 2596 if (!uic_cmd->cmd_active) { 2597 dev_err(hba->dev, "%s: UIC cmd has been completed, return the result\n", 2598 __func__); 2599 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT; 2600 } 2601 } 2602 2603 spin_lock_irqsave(hba->host->host_lock, flags); 2604 hba->active_uic_cmd = NULL; 2605 spin_unlock_irqrestore(hba->host->host_lock, flags); 2606 2607 return ret; 2608 } 2609 2610 /** 2611 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result 2612 * @hba: per adapter instance 2613 * @uic_cmd: UIC command 2614 * 2615 * Return: 0 if successful; < 0 upon failure. 2616 */ 2617 static int 2618 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 2619 { 2620 lockdep_assert_held(&hba->uic_cmd_mutex); 2621 2622 if (!ufshcd_ready_for_uic_cmd(hba)) { 2623 dev_err(hba->dev, 2624 "Controller not ready to accept UIC commands\n"); 2625 return -EIO; 2626 } 2627 2628 init_completion(&uic_cmd->done); 2629 2630 uic_cmd->cmd_active = true; 2631 ufshcd_dispatch_uic_cmd(hba, uic_cmd); 2632 2633 return 0; 2634 } 2635 2636 /** 2637 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result 2638 * @hba: per adapter instance 2639 * @uic_cmd: UIC command 2640 * 2641 * Return: 0 only if success. 2642 */ 2643 int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 2644 { 2645 unsigned long flags; 2646 int ret; 2647 2648 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD) 2649 return 0; 2650 2651 ufshcd_hold(hba); 2652 mutex_lock(&hba->uic_cmd_mutex); 2653 ufshcd_add_delay_before_dme_cmd(hba); 2654 2655 spin_lock_irqsave(hba->host->host_lock, flags); 2656 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL); 2657 spin_unlock_irqrestore(hba->host->host_lock, flags); 2658 2659 ret = __ufshcd_send_uic_cmd(hba, uic_cmd); 2660 if (!ret) 2661 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd); 2662 2663 mutex_unlock(&hba->uic_cmd_mutex); 2664 2665 ufshcd_release(hba); 2666 return ret; 2667 } 2668 2669 /** 2670 * ufshcd_sgl_to_prdt - SG list to PRTD (Physical Region Description Table, 4DW format) 2671 * @hba: per-adapter instance 2672 * @lrbp: pointer to local reference block 2673 * @sg_entries: The number of sg lists actually used 2674 * @sg_list: Pointer to SG list 2675 */ 2676 static void ufshcd_sgl_to_prdt(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int sg_entries, 2677 struct scatterlist *sg_list) 2678 { 2679 struct ufshcd_sg_entry *prd; 2680 struct scatterlist *sg; 2681 int i; 2682 2683 if (sg_entries) { 2684 2685 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) 2686 lrbp->utr_descriptor_ptr->prd_table_length = 2687 cpu_to_le16(sg_entries * ufshcd_sg_entry_size(hba)); 2688 else 2689 lrbp->utr_descriptor_ptr->prd_table_length = cpu_to_le16(sg_entries); 2690 2691 prd = lrbp->ucd_prdt_ptr; 2692 2693 for_each_sg(sg_list, sg, sg_entries, i) { 2694 const unsigned int len = sg_dma_len(sg); 2695 2696 /* 2697 * From the UFSHCI spec: "Data Byte Count (DBC): A '0' 2698 * based value that indicates the length, in bytes, of 2699 * the data block. A maximum of length of 256KB may 2700 * exist for any entry. Bits 1:0 of this field shall be 2701 * 11b to indicate Dword granularity. A value of '3' 2702 * indicates 4 bytes, '7' indicates 8 bytes, etc." 2703 */ 2704 WARN_ONCE(len > SZ_256K, "len = %#x\n", len); 2705 prd->size = cpu_to_le32(len - 1); 2706 prd->addr = cpu_to_le64(sg->dma_address); 2707 prd->reserved = 0; 2708 prd = (void *)prd + ufshcd_sg_entry_size(hba); 2709 } 2710 } else { 2711 lrbp->utr_descriptor_ptr->prd_table_length = 0; 2712 } 2713 } 2714 2715 /** 2716 * ufshcd_map_sg - Map scatter-gather list to prdt 2717 * @hba: per adapter instance 2718 * @cmd: SCSI command 2719 * 2720 * Return: 0 in case of success, non-zero value in case of failure. 2721 */ 2722 static int ufshcd_map_sg(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2723 { 2724 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2725 int sg_segments = scsi_dma_map(cmd); 2726 2727 if (sg_segments < 0) 2728 return sg_segments; 2729 2730 ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd)); 2731 2732 return ufshcd_crypto_fill_prdt(hba, cmd); 2733 } 2734 2735 /** 2736 * ufshcd_prepare_req_desc_hdr - Fill UTP Transfer request descriptor header according to request 2737 * descriptor according to request 2738 * @hba: per adapter instance 2739 * @lrbp: pointer to local reference block 2740 * @upiu_flags: flags required in the header 2741 * @cmd_dir: requests data direction 2742 * @ehs_length: Total EHS Length (in 32‐bytes units of all Extra Header Segments) 2743 */ 2744 static void 2745 ufshcd_prepare_req_desc_hdr(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, 2746 u8 *upiu_flags, enum dma_data_direction cmd_dir, 2747 int ehs_length) 2748 { 2749 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr; 2750 struct request_desc_header *h = &req_desc->header; 2751 enum utp_data_direction data_direction; 2752 2753 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE; 2754 2755 *h = (typeof(*h)){ }; 2756 2757 if (cmd_dir == DMA_FROM_DEVICE) { 2758 data_direction = UTP_DEVICE_TO_HOST; 2759 *upiu_flags = UPIU_CMD_FLAGS_READ; 2760 } else if (cmd_dir == DMA_TO_DEVICE) { 2761 data_direction = UTP_HOST_TO_DEVICE; 2762 *upiu_flags = UPIU_CMD_FLAGS_WRITE; 2763 } else { 2764 data_direction = UTP_NO_DATA_TRANSFER; 2765 *upiu_flags = UPIU_CMD_FLAGS_NONE; 2766 } 2767 2768 h->command_type = lrbp->command_type; 2769 h->data_direction = data_direction; 2770 h->ehs_length = ehs_length; 2771 2772 if (lrbp->intr_cmd) 2773 h->interrupt = 1; 2774 2775 /* Prepare crypto related dwords */ 2776 ufshcd_prepare_req_desc_hdr_crypto(lrbp, h); 2777 2778 /* 2779 * assigning invalid value for command status. Controller 2780 * updates OCS on command completion, with the command 2781 * status 2782 */ 2783 h->ocs = OCS_INVALID_COMMAND_STATUS; 2784 2785 req_desc->prd_table_length = 0; 2786 } 2787 2788 /** 2789 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc, 2790 * for scsi commands 2791 * @cmd: SCSI command 2792 * @upiu_flags: flags 2793 */ 2794 static void ufshcd_prepare_utp_scsi_cmd_upiu(struct scsi_cmnd *cmd, 2795 u8 upiu_flags) 2796 { 2797 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2798 const int tag = scsi_cmd_to_rq(cmd)->tag; 2799 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 2800 unsigned short cdb_len; 2801 2802 ucd_req_ptr->header = (struct utp_upiu_header){ 2803 .transaction_code = UPIU_TRANSACTION_COMMAND, 2804 .flags = upiu_flags, 2805 .lun = lrbp->lun, 2806 .task_tag = tag, 2807 .command_set_type = UPIU_COMMAND_SET_TYPE_SCSI, 2808 }; 2809 2810 WARN_ON_ONCE(ucd_req_ptr->header.task_tag != tag); 2811 2812 ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length); 2813 2814 cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE); 2815 memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len); 2816 2817 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 2818 } 2819 2820 /** 2821 * ufshcd_prepare_utp_query_req_upiu() - fill the utp_transfer_req_desc for query request 2822 * @hba: UFS hba 2823 * @cmd: SCSI command pointer 2824 * @upiu_flags: flags 2825 */ 2826 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, 2827 struct scsi_cmnd *cmd, u8 upiu_flags) 2828 { 2829 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2830 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 2831 const int tag = scsi_cmd_to_rq(cmd)->tag; 2832 struct ufs_query *query = &hba->dev_cmd.query; 2833 u16 len = be16_to_cpu(query->request.upiu_req.length); 2834 2835 /* Query request header */ 2836 ucd_req_ptr->header = (struct utp_upiu_header){ 2837 .transaction_code = UPIU_TRANSACTION_QUERY_REQ, 2838 .flags = upiu_flags, 2839 .lun = lrbp->lun, 2840 .task_tag = tag, 2841 .query_function = query->request.query_func, 2842 /* Data segment length only need for WRITE_DESC */ 2843 .data_segment_length = 2844 query->request.upiu_req.opcode == 2845 UPIU_QUERY_OPCODE_WRITE_DESC ? 2846 cpu_to_be16(len) : 2847 0, 2848 }; 2849 2850 /* Copy the Query Request buffer as is */ 2851 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req, 2852 QUERY_OSF_SIZE); 2853 2854 /* Copy the Descriptor */ 2855 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC) 2856 memcpy(ucd_req_ptr + 1, query->descriptor, len); 2857 } 2858 2859 static inline void ufshcd_prepare_utp_nop_upiu(struct scsi_cmnd *cmd) 2860 { 2861 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2862 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 2863 const int tag = scsi_cmd_to_rq(cmd)->tag; 2864 2865 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req)); 2866 2867 ucd_req_ptr->header = (struct utp_upiu_header){ 2868 .transaction_code = UPIU_TRANSACTION_NOP_OUT, 2869 .task_tag = tag, 2870 }; 2871 } 2872 2873 /** 2874 * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU) 2875 * for Device Management Purposes 2876 * @hba: per adapter instance 2877 * @cmd: SCSI command pointer 2878 * 2879 * Return: 0 upon success; < 0 upon failure. 2880 */ 2881 static int ufshcd_compose_devman_upiu(struct ufs_hba *hba, 2882 struct scsi_cmnd *cmd) 2883 { 2884 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2885 u8 upiu_flags; 2886 int ret = 0; 2887 2888 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0); 2889 2890 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY) 2891 ufshcd_prepare_utp_query_req_upiu(hba, cmd, upiu_flags); 2892 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP) 2893 ufshcd_prepare_utp_nop_upiu(cmd); 2894 else 2895 ret = -EINVAL; 2896 2897 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 2898 2899 return ret; 2900 } 2901 2902 /** 2903 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU) 2904 * for SCSI Purposes 2905 * @hba: per adapter instance 2906 * @cmd: SCSI command 2907 */ 2908 static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2909 { 2910 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2911 struct request *rq = scsi_cmd_to_rq(cmd); 2912 unsigned int ioprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq)); 2913 u8 upiu_flags; 2914 2915 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, 2916 cmd->sc_data_direction, 0); 2917 if (ioprio_class == IOPRIO_CLASS_RT) 2918 upiu_flags |= UPIU_CMD_FLAGS_CP; 2919 ufshcd_prepare_utp_scsi_cmd_upiu(cmd, upiu_flags); 2920 } 2921 2922 static void ufshcd_init_lrb(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2923 { 2924 const int i = scsi_cmd_to_rq(cmd)->tag; 2925 struct utp_transfer_cmd_desc *cmd_descp = 2926 (void *)hba->ucdl_base_addr + i * ufshcd_get_ucd_size(hba); 2927 struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr; 2928 dma_addr_t cmd_desc_element_addr = 2929 hba->ucdl_dma_addr + i * ufshcd_get_ucd_size(hba); 2930 u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset); 2931 u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset); 2932 struct ufshcd_lrb *lrb = scsi_cmd_priv(cmd); 2933 2934 lrb->utr_descriptor_ptr = utrdlp + i; 2935 lrb->utrd_dma_addr = 2936 hba->utrdl_dma_addr + i * sizeof(struct utp_transfer_req_desc); 2937 lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp->command_upiu; 2938 lrb->ucd_req_dma_addr = cmd_desc_element_addr; 2939 lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu; 2940 lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset; 2941 lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table; 2942 lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset; 2943 } 2944 2945 static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 2946 u8 lun, int tag) 2947 { 2948 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2949 2950 ufshcd_init_lrb(hba, cmd); 2951 2952 memset(lrbp->ucd_req_ptr, 0, sizeof(*lrbp->ucd_req_ptr)); 2953 2954 lrbp->lun = lun; 2955 ufshcd_prepare_lrbp_crypto(ufshcd_is_scsi_cmd(cmd) ? 2956 scsi_cmd_to_rq(cmd) : NULL, lrbp); 2957 } 2958 2959 static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 2960 u8 lun, int tag) 2961 { 2962 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2963 2964 __ufshcd_setup_cmd(hba, cmd, lun, tag); 2965 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba); 2966 lrbp->req_abort_skip = false; 2967 2968 ufshcd_comp_scsi_upiu(hba, cmd); 2969 } 2970 2971 /** 2972 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID 2973 * @upiu_wlun_id: UPIU W-LUN id 2974 * 2975 * Return: SCSI W-LUN id. 2976 */ 2977 static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id) 2978 { 2979 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE; 2980 } 2981 2982 static inline bool is_device_wlun(struct scsi_device *sdev) 2983 { 2984 return sdev->lun == 2985 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN); 2986 } 2987 2988 /* 2989 * Associate the UFS controller queue with the default and poll HCTX types. 2990 * Initialize the mq_map[] arrays. 2991 */ 2992 static void ufshcd_map_queues(struct Scsi_Host *shost) 2993 { 2994 struct ufs_hba *hba = shost_priv(shost); 2995 int i, queue_offset = 0; 2996 2997 if (!is_mcq_supported(hba)) { 2998 hba->nr_queues[HCTX_TYPE_DEFAULT] = 1; 2999 hba->nr_queues[HCTX_TYPE_READ] = 0; 3000 hba->nr_queues[HCTX_TYPE_POLL] = 1; 3001 hba->nr_hw_queues = 1; 3002 } 3003 3004 for (i = 0; i < shost->nr_maps; i++) { 3005 struct blk_mq_queue_map *map = &shost->tag_set.map[i]; 3006 3007 map->nr_queues = hba->nr_queues[i]; 3008 if (!map->nr_queues) 3009 continue; 3010 map->queue_offset = queue_offset; 3011 if (i == HCTX_TYPE_POLL && !is_mcq_supported(hba)) 3012 map->queue_offset = 0; 3013 3014 blk_mq_map_queues(map); 3015 queue_offset += map->nr_queues; 3016 } 3017 } 3018 3019 /* 3020 * The only purpose of this function is to make the SCSI core skip the memset() 3021 * call for the private command data. 3022 */ 3023 static int ufshcd_init_cmd_priv(struct Scsi_Host *host, struct scsi_cmnd *cmd) 3024 { 3025 return 0; 3026 } 3027 3028 /** 3029 * ufshcd_queuecommand - main entry point for SCSI requests 3030 * @host: SCSI host pointer 3031 * @cmd: command from SCSI Midlayer 3032 * 3033 * Return: 0 for success, non-zero in case of failure. 3034 */ 3035 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) 3036 { 3037 struct ufs_hba *hba = shost_priv(host); 3038 int tag = scsi_cmd_to_rq(cmd)->tag; 3039 int err = 0; 3040 struct ufs_hw_queue *hwq = NULL; 3041 3042 switch (hba->ufshcd_state) { 3043 case UFSHCD_STATE_OPERATIONAL: 3044 break; 3045 case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL: 3046 /* 3047 * SCSI error handler can call ->queuecommand() while UFS error 3048 * handler is in progress. Error interrupts could change the 3049 * state from UFSHCD_STATE_RESET to 3050 * UFSHCD_STATE_EH_SCHEDULED_NON_FATAL. Prevent requests 3051 * being issued in that case. 3052 */ 3053 if (ufshcd_eh_in_progress(hba)) { 3054 err = SCSI_MLQUEUE_HOST_BUSY; 3055 goto out; 3056 } 3057 break; 3058 case UFSHCD_STATE_EH_SCHEDULED_FATAL: 3059 /* 3060 * pm_runtime_get_sync() is used at error handling preparation 3061 * stage. If a scsi cmd, e.g. the SSU cmd, is sent from hba's 3062 * PM ops, it can never be finished if we let SCSI layer keep 3063 * retrying it, which gets err handler stuck forever. Neither 3064 * can we let the scsi cmd pass through, because UFS is in bad 3065 * state, the scsi cmd may eventually time out, which will get 3066 * err handler blocked for too long. So, just fail the scsi cmd 3067 * sent from PM ops, err handler can recover PM error anyways. 3068 */ 3069 if (hba->pm_op_in_progress) { 3070 hba->force_reset = true; 3071 set_host_byte(cmd, DID_BAD_TARGET); 3072 scsi_done(cmd); 3073 goto out; 3074 } 3075 fallthrough; 3076 case UFSHCD_STATE_RESET: 3077 err = SCSI_MLQUEUE_HOST_BUSY; 3078 goto out; 3079 case UFSHCD_STATE_ERROR: 3080 set_host_byte(cmd, DID_ERROR); 3081 scsi_done(cmd); 3082 goto out; 3083 } 3084 3085 hba->req_abort_count = 0; 3086 3087 ufshcd_hold(hba); 3088 3089 ufshcd_setup_scsi_cmd(hba, cmd, 3090 ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag); 3091 3092 err = ufshcd_map_sg(hba, cmd); 3093 if (err) { 3094 ufshcd_release(hba); 3095 goto out; 3096 } 3097 3098 if (hba->mcq_enabled) 3099 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); 3100 3101 ufshcd_send_command(hba, cmd, hwq); 3102 3103 out: 3104 if (ufs_trigger_eh(hba)) { 3105 unsigned long flags; 3106 3107 spin_lock_irqsave(hba->host->host_lock, flags); 3108 ufshcd_schedule_eh_work(hba); 3109 spin_unlock_irqrestore(hba->host->host_lock, flags); 3110 } 3111 3112 return err; 3113 } 3114 3115 static int ufshcd_queue_reserved_command(struct Scsi_Host *host, 3116 struct scsi_cmnd *cmd) 3117 { 3118 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3119 struct request *rq = scsi_cmd_to_rq(cmd); 3120 struct ufs_hba *hba = shost_priv(host); 3121 struct ufs_hw_queue *hwq = 3122 hba->mcq_enabled ? ufshcd_mcq_req_to_hwq(hba, rq) : NULL; 3123 3124 ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); 3125 ufshcd_send_command(hba, cmd, hwq); 3126 return 0; 3127 } 3128 3129 static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 3130 enum dev_cmd_type cmd_type, u8 lun, int tag) 3131 { 3132 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3133 3134 __ufshcd_setup_cmd(hba, cmd, lun, tag); 3135 lrbp->intr_cmd = true; /* No interrupt aggregation */ 3136 hba->dev_cmd.type = cmd_type; 3137 } 3138 3139 /* 3140 * Return: 0 upon success; < 0 upon failure. 3141 */ 3142 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 3143 enum dev_cmd_type cmd_type, int tag) 3144 { 3145 ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag); 3146 3147 return ufshcd_compose_devman_upiu(hba, cmd); 3148 } 3149 3150 /* 3151 * Check with the block layer if the command is inflight 3152 * @cmd: command to check. 3153 * 3154 * Return: true if command is inflight; false if not. 3155 */ 3156 bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd) 3157 { 3158 return cmd && blk_mq_rq_state(scsi_cmd_to_rq(cmd)) == MQ_RQ_IN_FLIGHT; 3159 } 3160 3161 /* 3162 * Clear the pending command in the controller and wait until 3163 * the controller confirms that the command has been cleared. 3164 * @hba: per adapter instance 3165 * @task_tag: The tag number of the command to be cleared. 3166 */ 3167 static int ufshcd_clear_cmd(struct ufs_hba *hba, u32 task_tag) 3168 { 3169 u32 mask; 3170 int err; 3171 3172 if (hba->mcq_enabled) { 3173 /* 3174 * MCQ mode. Clean up the MCQ resources similar to 3175 * what the ufshcd_utrl_clear() does for SDB mode. 3176 */ 3177 err = ufshcd_mcq_sq_cleanup(hba, task_tag); 3178 if (err) { 3179 dev_err(hba->dev, "%s: failed tag=%d. err=%d\n", 3180 __func__, task_tag, err); 3181 return err; 3182 } 3183 return 0; 3184 } 3185 3186 mask = 1U << task_tag; 3187 3188 /* clear outstanding transaction before retry */ 3189 ufshcd_utrl_clear(hba, mask); 3190 3191 /* 3192 * wait for h/w to clear corresponding bit in door-bell. 3193 * max. wait is 1 sec. 3194 */ 3195 return ufshcd_wait_for_register(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL, 3196 mask, ~mask, 1000, 1000); 3197 } 3198 3199 /** 3200 * ufshcd_dev_cmd_completion() - handles device management command responses 3201 * @hba: per adapter instance 3202 * @lrbp: pointer to local reference block 3203 * 3204 * Return: 0 upon success; < 0 upon failure. 3205 */ 3206 static int 3207 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 3208 { 3209 enum upiu_response_transaction resp; 3210 int err = 0; 3211 3212 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); 3213 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); 3214 3215 switch (resp) { 3216 case UPIU_TRANSACTION_NOP_IN: 3217 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) { 3218 err = -EINVAL; 3219 dev_err(hba->dev, "%s: unexpected response %x\n", 3220 __func__, resp); 3221 } 3222 break; 3223 case UPIU_TRANSACTION_QUERY_RSP: { 3224 u8 response = lrbp->ucd_rsp_ptr->header.response; 3225 3226 if (response == 0) { 3227 err = ufshcd_copy_query_response(hba, lrbp); 3228 } else { 3229 err = -EINVAL; 3230 dev_err(hba->dev, "%s: unexpected response in Query RSP: %x\n", 3231 __func__, response); 3232 } 3233 break; 3234 } 3235 case UPIU_TRANSACTION_REJECT_UPIU: 3236 /* TODO: handle Reject UPIU Response */ 3237 err = -EPERM; 3238 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n", 3239 __func__); 3240 break; 3241 case UPIU_TRANSACTION_RESPONSE: 3242 if (hba->dev_cmd.type != DEV_CMD_TYPE_RPMB) { 3243 err = -EINVAL; 3244 dev_err(hba->dev, "%s: unexpected response %x\n", __func__, resp); 3245 } 3246 break; 3247 default: 3248 err = -EINVAL; 3249 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n", 3250 __func__, resp); 3251 break; 3252 } 3253 3254 WARN_ONCE(err > 0, "Incorrect return value %d > 0\n", err); 3255 return err; 3256 } 3257 3258 static void ufshcd_dev_man_lock(struct ufs_hba *hba) 3259 { 3260 ufshcd_hold(hba); 3261 mutex_lock(&hba->dev_cmd.lock); 3262 down_read(&hba->clk_scaling_lock); 3263 } 3264 3265 static void ufshcd_dev_man_unlock(struct ufs_hba *hba) 3266 { 3267 up_read(&hba->clk_scaling_lock); 3268 mutex_unlock(&hba->dev_cmd.lock); 3269 ufshcd_release(hba); 3270 } 3271 3272 static struct scsi_cmnd *ufshcd_get_dev_mgmt_cmd(struct ufs_hba *hba) 3273 { 3274 /* 3275 * The caller must hold this lock to guarantee that the NOWAIT 3276 * allocation will succeed. 3277 */ 3278 lockdep_assert_held(&hba->dev_cmd.lock); 3279 3280 return scsi_get_internal_cmd( 3281 hba->host->pseudo_sdev, DMA_TO_DEVICE, 3282 BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT); 3283 } 3284 3285 static void ufshcd_put_dev_mgmt_cmd(struct scsi_cmnd *cmd) 3286 { 3287 scsi_put_internal_cmd(cmd); 3288 } 3289 3290 /* 3291 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3292 * < 0 if another error occurred. 3293 */ 3294 static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 3295 const u32 tag, int timeout) 3296 { 3297 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3298 struct request *rq = scsi_cmd_to_rq(cmd); 3299 blk_status_t sts; 3300 3301 rq->timeout = timeout; 3302 sts = blk_execute_rq(rq, true); 3303 if (sts != BLK_STS_OK) 3304 return blk_status_to_errno(sts); 3305 return lrbp->utr_descriptor_ptr->header.ocs; 3306 } 3307 3308 /** 3309 * ufshcd_exec_dev_cmd - API for sending device management requests 3310 * @hba: UFS hba 3311 * @cmd_type: specifies the type (NOP, Query...) 3312 * @timeout: timeout in milliseconds 3313 * 3314 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3315 * < 0 if another error occurred. 3316 * 3317 * NOTE: Since there is only one available tag for device management commands, 3318 * it is expected you hold the hba->dev_cmd.lock mutex. 3319 */ 3320 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, 3321 enum dev_cmd_type cmd_type, int timeout) 3322 { 3323 struct scsi_cmnd *cmd = ufshcd_get_dev_mgmt_cmd(hba); 3324 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3325 u32 tag; 3326 int err; 3327 3328 /* Protects use of hba->dev_cmd. */ 3329 lockdep_assert_held(&hba->dev_cmd.lock); 3330 3331 if (WARN_ON_ONCE(!cmd)) 3332 return -ENOMEM; 3333 3334 tag = scsi_cmd_to_rq(cmd)->tag; 3335 3336 err = ufshcd_compose_dev_cmd(hba, cmd, cmd_type, tag); 3337 if (unlikely(err)) 3338 goto out; 3339 3340 err = ufshcd_issue_dev_cmd(hba, cmd, tag, timeout); 3341 if (err == 0) 3342 err = ufshcd_dev_cmd_completion(hba, lrbp); 3343 3344 out: 3345 ufshcd_put_dev_mgmt_cmd(cmd); 3346 3347 return err; 3348 } 3349 3350 /** 3351 * ufshcd_init_query() - init the query response and request parameters 3352 * @hba: per-adapter instance 3353 * @request: address of the request pointer to be initialized 3354 * @response: address of the response pointer to be initialized 3355 * @opcode: operation to perform 3356 * @idn: flag idn to access 3357 * @index: LU number to access 3358 * @selector: query/flag/descriptor further identification 3359 */ 3360 static inline void ufshcd_init_query(struct ufs_hba *hba, 3361 struct ufs_query_req **request, struct ufs_query_res **response, 3362 enum query_opcode opcode, u8 idn, u8 index, u8 selector) 3363 { 3364 *request = &hba->dev_cmd.query.request; 3365 *response = &hba->dev_cmd.query.response; 3366 memset(*request, 0, sizeof(struct ufs_query_req)); 3367 memset(*response, 0, sizeof(struct ufs_query_res)); 3368 (*request)->upiu_req.opcode = opcode; 3369 (*request)->upiu_req.idn = idn; 3370 (*request)->upiu_req.index = index; 3371 (*request)->upiu_req.selector = selector; 3372 } 3373 3374 /* 3375 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3376 * < 0 if another error occurred. 3377 */ 3378 static int ufshcd_query_flag_retry(struct ufs_hba *hba, 3379 enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res) 3380 { 3381 int ret; 3382 int retries; 3383 3384 for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) { 3385 ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res); 3386 if (ret) 3387 dev_dbg(hba->dev, 3388 "%s: failed with error %d, retries %d\n", 3389 __func__, ret, retries); 3390 else 3391 break; 3392 } 3393 3394 if (ret) 3395 dev_err(hba->dev, 3396 "%s: query flag, opcode %d, idn %d, failed with error %d after %d retries\n", 3397 __func__, opcode, idn, ret, retries); 3398 return ret; 3399 } 3400 3401 /** 3402 * ufshcd_query_flag() - API function for sending flag query requests 3403 * @hba: per-adapter instance 3404 * @opcode: flag query to perform 3405 * @idn: flag idn to access 3406 * @index: flag index to access 3407 * @flag_res: the flag value after the query request completes 3408 * 3409 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3410 * < 0 if another error occurred. 3411 */ 3412 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, 3413 enum flag_idn idn, u8 index, bool *flag_res) 3414 { 3415 struct ufs_query_req *request = NULL; 3416 struct ufs_query_res *response = NULL; 3417 int err, selector = 0; 3418 int timeout = dev_cmd_timeout; 3419 3420 BUG_ON(!hba); 3421 3422 ufshcd_dev_man_lock(hba); 3423 3424 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 3425 selector); 3426 3427 switch (opcode) { 3428 case UPIU_QUERY_OPCODE_SET_FLAG: 3429 case UPIU_QUERY_OPCODE_CLEAR_FLAG: 3430 case UPIU_QUERY_OPCODE_TOGGLE_FLAG: 3431 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 3432 break; 3433 case UPIU_QUERY_OPCODE_READ_FLAG: 3434 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 3435 if (!flag_res) { 3436 /* No dummy reads */ 3437 dev_err(hba->dev, "%s: Invalid argument for read request\n", 3438 __func__); 3439 err = -EINVAL; 3440 goto out_unlock; 3441 } 3442 break; 3443 default: 3444 dev_err(hba->dev, 3445 "%s: Expected query flag opcode but got = %d\n", 3446 __func__, opcode); 3447 err = -EINVAL; 3448 goto out_unlock; 3449 } 3450 3451 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout); 3452 3453 if (err) { 3454 dev_err(hba->dev, 3455 "%s: Sending flag query for idn %d failed, err = %d\n", 3456 __func__, idn, err); 3457 goto out_unlock; 3458 } 3459 3460 if (flag_res) 3461 *flag_res = (be32_to_cpu(response->upiu_res.value) & 3462 MASK_QUERY_UPIU_FLAG_LOC) & 0x1; 3463 3464 out_unlock: 3465 ufshcd_dev_man_unlock(hba); 3466 return err; 3467 } 3468 3469 /** 3470 * ufshcd_query_attr - API function for sending attribute requests 3471 * @hba: per-adapter instance 3472 * @opcode: attribute opcode 3473 * @idn: attribute idn to access 3474 * @index: index field 3475 * @selector: selector field 3476 * @attr_val: the attribute value after the query request completes 3477 * 3478 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3479 * < 0 if another error occurred. 3480 */ 3481 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, 3482 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val) 3483 { 3484 struct ufs_query_req *request = NULL; 3485 struct ufs_query_res *response = NULL; 3486 int err; 3487 3488 BUG_ON(!hba); 3489 3490 if (!attr_val) { 3491 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n", 3492 __func__, opcode); 3493 return -EINVAL; 3494 } 3495 3496 ufshcd_dev_man_lock(hba); 3497 3498 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 3499 selector); 3500 3501 switch (opcode) { 3502 case UPIU_QUERY_OPCODE_WRITE_ATTR: 3503 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 3504 request->upiu_req.value = cpu_to_be32(*attr_val); 3505 break; 3506 case UPIU_QUERY_OPCODE_READ_ATTR: 3507 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 3508 break; 3509 default: 3510 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n", 3511 __func__, opcode); 3512 err = -EINVAL; 3513 goto out_unlock; 3514 } 3515 3516 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); 3517 3518 if (err) { 3519 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n", 3520 __func__, opcode, idn, index, err); 3521 goto out_unlock; 3522 } 3523 3524 *attr_val = be32_to_cpu(response->upiu_res.value); 3525 3526 out_unlock: 3527 ufshcd_dev_man_unlock(hba); 3528 return err; 3529 } 3530 3531 /** 3532 * ufshcd_query_attr_retry() - API function for sending query 3533 * attribute with retries 3534 * @hba: per-adapter instance 3535 * @opcode: attribute opcode 3536 * @idn: attribute idn to access 3537 * @index: index field 3538 * @selector: selector field 3539 * @attr_val: the attribute value after the query request 3540 * completes 3541 * 3542 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3543 * < 0 if another error occurred. 3544 */ 3545 int ufshcd_query_attr_retry(struct ufs_hba *hba, 3546 enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector, 3547 u32 *attr_val) 3548 { 3549 int ret = 0; 3550 u32 retries; 3551 3552 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { 3553 ret = ufshcd_query_attr(hba, opcode, idn, index, 3554 selector, attr_val); 3555 if (ret) 3556 dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n", 3557 __func__, ret, retries); 3558 else 3559 break; 3560 } 3561 3562 if (ret) 3563 dev_err(hba->dev, 3564 "%s: query attribute, idn %d, failed with error %d after %d retries\n", 3565 __func__, idn, ret, QUERY_REQ_RETRIES); 3566 return ret; 3567 } 3568 3569 /* 3570 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3571 * < 0 if another error occurred. 3572 */ 3573 static int __ufshcd_query_descriptor(struct ufs_hba *hba, 3574 enum query_opcode opcode, enum desc_idn idn, u8 index, 3575 u8 selector, u8 *desc_buf, int *buf_len) 3576 { 3577 struct ufs_query_req *request = NULL; 3578 struct ufs_query_res *response = NULL; 3579 int err; 3580 3581 BUG_ON(!hba); 3582 3583 if (!desc_buf) { 3584 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n", 3585 __func__, opcode); 3586 return -EINVAL; 3587 } 3588 3589 if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) { 3590 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n", 3591 __func__, *buf_len); 3592 return -EINVAL; 3593 } 3594 3595 ufshcd_dev_man_lock(hba); 3596 3597 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 3598 selector); 3599 hba->dev_cmd.query.descriptor = desc_buf; 3600 request->upiu_req.length = cpu_to_be16(*buf_len); 3601 3602 switch (opcode) { 3603 case UPIU_QUERY_OPCODE_WRITE_DESC: 3604 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 3605 break; 3606 case UPIU_QUERY_OPCODE_READ_DESC: 3607 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 3608 break; 3609 default: 3610 dev_err(hba->dev, 3611 "%s: Expected query descriptor opcode but got = 0x%.2x\n", 3612 __func__, opcode); 3613 err = -EINVAL; 3614 goto out_unlock; 3615 } 3616 3617 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); 3618 3619 if (err) { 3620 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n", 3621 __func__, opcode, idn, index, err); 3622 goto out_unlock; 3623 } 3624 3625 *buf_len = be16_to_cpu(response->upiu_res.length); 3626 3627 out_unlock: 3628 hba->dev_cmd.query.descriptor = NULL; 3629 ufshcd_dev_man_unlock(hba); 3630 return err; 3631 } 3632 3633 /** 3634 * ufshcd_query_descriptor_retry - API function for sending descriptor requests 3635 * @hba: per-adapter instance 3636 * @opcode: attribute opcode 3637 * @idn: attribute idn to access 3638 * @index: index field 3639 * @selector: selector field 3640 * @desc_buf: the buffer that contains the descriptor 3641 * @buf_len: length parameter passed to the device 3642 * 3643 * The buf_len parameter will contain, on return, the length parameter 3644 * received on the response. 3645 * 3646 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3647 * < 0 if another error occurred. 3648 */ 3649 int ufshcd_query_descriptor_retry(struct ufs_hba *hba, 3650 enum query_opcode opcode, 3651 enum desc_idn idn, u8 index, 3652 u8 selector, 3653 u8 *desc_buf, int *buf_len) 3654 { 3655 int err; 3656 int retries; 3657 3658 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { 3659 err = __ufshcd_query_descriptor(hba, opcode, idn, index, 3660 selector, desc_buf, buf_len); 3661 if (!err || err == -EINVAL) 3662 break; 3663 } 3664 3665 return err; 3666 } 3667 3668 /** 3669 * ufshcd_read_desc_param - read the specified descriptor parameter 3670 * @hba: Pointer to adapter instance 3671 * @desc_id: descriptor idn value 3672 * @desc_index: descriptor index 3673 * @param_offset: offset of the parameter to read 3674 * @param_read_buf: pointer to buffer where parameter would be read 3675 * @param_size: sizeof(param_read_buf) 3676 * 3677 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3678 * < 0 if another error occurred. 3679 */ 3680 int ufshcd_read_desc_param(struct ufs_hba *hba, 3681 enum desc_idn desc_id, 3682 int desc_index, 3683 u8 param_offset, 3684 u8 *param_read_buf, 3685 u8 param_size) 3686 { 3687 int ret; 3688 u8 *desc_buf; 3689 int buff_len = QUERY_DESC_MAX_SIZE; 3690 bool is_kmalloc = true; 3691 3692 /* Safety check */ 3693 if (desc_id >= QUERY_DESC_IDN_MAX || !param_size) 3694 return -EINVAL; 3695 3696 /* Check whether we need temp memory */ 3697 if (param_offset != 0 || param_size < buff_len) { 3698 desc_buf = kzalloc(buff_len, GFP_KERNEL); 3699 if (!desc_buf) 3700 return -ENOMEM; 3701 } else { 3702 desc_buf = param_read_buf; 3703 is_kmalloc = false; 3704 } 3705 3706 /* Request for full descriptor */ 3707 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC, 3708 desc_id, desc_index, 0, 3709 desc_buf, &buff_len); 3710 if (ret) { 3711 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n", 3712 __func__, desc_id, desc_index, param_offset, ret); 3713 goto out; 3714 } 3715 3716 /* Update descriptor length */ 3717 buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET]; 3718 3719 if (param_offset >= buff_len) { 3720 dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n", 3721 __func__, param_offset, desc_id, buff_len); 3722 ret = -EINVAL; 3723 goto out; 3724 } 3725 3726 /* Sanity check */ 3727 if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) { 3728 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n", 3729 __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]); 3730 ret = -EINVAL; 3731 goto out; 3732 } 3733 3734 if (is_kmalloc) { 3735 /* Make sure we don't copy more data than available */ 3736 if (param_offset >= buff_len) 3737 ret = -EINVAL; 3738 else 3739 memcpy(param_read_buf, &desc_buf[param_offset], 3740 min_t(u32, param_size, buff_len - param_offset)); 3741 } 3742 out: 3743 if (is_kmalloc) 3744 kfree(desc_buf); 3745 return ret; 3746 } 3747 3748 /** 3749 * struct uc_string_id - unicode string 3750 * 3751 * @len: size of this descriptor inclusive 3752 * @type: descriptor type 3753 * @uc: unicode string character 3754 */ 3755 struct uc_string_id { 3756 u8 len; 3757 u8 type; 3758 wchar_t uc[]; 3759 } __packed; 3760 3761 /* replace non-printable or non-ASCII characters with spaces */ 3762 static inline char ufshcd_remove_non_printable(u8 ch) 3763 { 3764 return (ch >= 0x20 && ch <= 0x7e) ? ch : ' '; 3765 } 3766 3767 /** 3768 * ufshcd_read_string_desc - read string descriptor 3769 * @hba: pointer to adapter instance 3770 * @desc_index: descriptor index 3771 * @buf: pointer to buffer where descriptor would be read, 3772 * the caller should free the memory. 3773 * @fmt: if %SD_ASCII_STD, convert from UTF-16 to ASCII 3774 * 3775 * Return: 3776 * * string size on success. 3777 * * -ENOMEM: on allocation failure 3778 * * -EINVAL: on a wrong parameter 3779 */ 3780 int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, u8 **buf, enum ufs_descr_fmt fmt) 3781 { 3782 struct uc_string_id *uc_str; 3783 u8 *str; 3784 int ret; 3785 3786 if (!buf) 3787 return -EINVAL; 3788 3789 uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 3790 if (!uc_str) 3791 return -ENOMEM; 3792 3793 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0, 3794 (u8 *)uc_str, QUERY_DESC_MAX_SIZE); 3795 if (ret < 0) { 3796 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n", 3797 QUERY_REQ_RETRIES, ret); 3798 str = NULL; 3799 goto out; 3800 } 3801 3802 if (uc_str->len <= QUERY_DESC_HDR_SIZE) { 3803 dev_dbg(hba->dev, "String Desc is of zero length\n"); 3804 str = NULL; 3805 ret = 0; 3806 goto out; 3807 } 3808 3809 if (fmt == SD_ASCII_STD) { 3810 ssize_t ascii_len; 3811 int i; 3812 /* remove header and divide by 2 to move from UTF16 to UTF8 */ 3813 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1; 3814 str = kzalloc(ascii_len, GFP_KERNEL); 3815 if (!str) { 3816 ret = -ENOMEM; 3817 goto out; 3818 } 3819 3820 /* 3821 * the descriptor contains string in UTF16 format 3822 * we need to convert to utf-8 so it can be displayed 3823 */ 3824 ret = utf16s_to_utf8s(uc_str->uc, 3825 uc_str->len - QUERY_DESC_HDR_SIZE, 3826 UTF16_BIG_ENDIAN, str, ascii_len - 1); 3827 3828 /* replace non-printable or non-ASCII characters with spaces */ 3829 for (i = 0; i < ret; i++) 3830 str[i] = ufshcd_remove_non_printable(str[i]); 3831 3832 str[ret++] = '\0'; 3833 3834 } else { 3835 str = kmemdup(uc_str->uc, uc_str->len, GFP_KERNEL); 3836 if (!str) { 3837 ret = -ENOMEM; 3838 goto out; 3839 } 3840 ret = uc_str->len; 3841 } 3842 out: 3843 *buf = str; 3844 kfree(uc_str); 3845 return ret; 3846 } 3847 3848 /** 3849 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter 3850 * @hba: Pointer to adapter instance 3851 * @lun: lun id 3852 * @param_offset: offset of the parameter to read 3853 * @param_read_buf: pointer to buffer where parameter would be read 3854 * @param_size: sizeof(param_read_buf) 3855 * 3856 * Return: 0 in case of success; < 0 upon failure. 3857 */ 3858 static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba, 3859 int lun, 3860 enum unit_desc_param param_offset, 3861 u8 *param_read_buf, 3862 u32 param_size) 3863 { 3864 /* 3865 * Unit descriptors are only available for general purpose LUs (LUN id 3866 * from 0 to 7) and RPMB Well known LU. 3867 */ 3868 if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun)) 3869 return -EOPNOTSUPP; 3870 3871 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun, 3872 param_offset, param_read_buf, param_size); 3873 } 3874 3875 static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba) 3876 { 3877 int err = 0; 3878 u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US; 3879 3880 if (hba->dev_info.wspecversion >= 0x300) { 3881 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 3882 QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0, 3883 &gating_wait); 3884 if (err) 3885 dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n", 3886 err, gating_wait); 3887 3888 if (gating_wait == 0) { 3889 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US; 3890 dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n", 3891 gating_wait); 3892 } 3893 3894 hba->dev_info.clk_gating_wait_us = gating_wait; 3895 } 3896 3897 return err; 3898 } 3899 3900 /** 3901 * ufshcd_memory_alloc - allocate memory for host memory space data structures 3902 * @hba: per adapter instance 3903 * 3904 * 1. Allocate DMA memory for Command Descriptor array 3905 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT 3906 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL). 3907 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List 3908 * (UTMRDL) 3909 * 4. Allocate memory for local reference block(lrb). 3910 * 3911 * Return: 0 for success, non-zero in case of failure. 3912 */ 3913 static int ufshcd_memory_alloc(struct ufs_hba *hba) 3914 { 3915 size_t utmrdl_size, utrdl_size, ucdl_size; 3916 3917 /* Allocate memory for UTP command descriptors */ 3918 ucdl_size = ufshcd_get_ucd_size(hba) * hba->nutrs; 3919 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev, 3920 ucdl_size, 3921 &hba->ucdl_dma_addr, 3922 GFP_KERNEL); 3923 3924 /* 3925 * UFSHCI requires UTP command descriptor to be 128 byte aligned. 3926 */ 3927 if (!hba->ucdl_base_addr || 3928 WARN_ON(hba->ucdl_dma_addr & (128 - 1))) { 3929 dev_err(hba->dev, 3930 "Command Descriptor Memory allocation failed\n"); 3931 goto out; 3932 } 3933 3934 /* 3935 * Allocate memory for UTP Transfer descriptors 3936 * UFSHCI requires 1KB alignment of UTRD 3937 */ 3938 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs); 3939 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev, 3940 utrdl_size, 3941 &hba->utrdl_dma_addr, 3942 GFP_KERNEL); 3943 if (!hba->utrdl_base_addr || 3944 WARN_ON(hba->utrdl_dma_addr & (SZ_1K - 1))) { 3945 dev_err(hba->dev, 3946 "Transfer Descriptor Memory allocation failed\n"); 3947 goto out; 3948 } 3949 3950 /* 3951 * Skip utmrdl allocation; it may have been 3952 * allocated during first pass and not released during 3953 * MCQ memory allocation. 3954 * See ufshcd_release_sdb_queue() and ufshcd_config_mcq() 3955 */ 3956 if (hba->utmrdl_base_addr) 3957 goto skip_utmrdl; 3958 /* 3959 * Allocate memory for UTP Task Management descriptors 3960 * UFSHCI requires 1KB alignment of UTMRD 3961 */ 3962 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs; 3963 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev, 3964 utmrdl_size, 3965 &hba->utmrdl_dma_addr, 3966 GFP_KERNEL); 3967 if (!hba->utmrdl_base_addr || 3968 WARN_ON(hba->utmrdl_dma_addr & (SZ_1K - 1))) { 3969 dev_err(hba->dev, 3970 "Task Management Descriptor Memory allocation failed\n"); 3971 goto out; 3972 } 3973 3974 skip_utmrdl: 3975 return 0; 3976 out: 3977 return -ENOMEM; 3978 } 3979 3980 /** 3981 * ufshcd_host_memory_configure - configure local reference block with 3982 * memory offsets 3983 * @hba: per adapter instance 3984 * 3985 * Configure Host memory space 3986 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA 3987 * address. 3988 * 2. Update each UTRD with Response UPIU offset, Response UPIU length 3989 * and PRDT offset. 3990 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT 3991 * into local reference block. 3992 */ 3993 static void ufshcd_host_memory_configure(struct ufs_hba *hba) 3994 { 3995 struct utp_transfer_req_desc *utrdlp; 3996 dma_addr_t cmd_desc_dma_addr; 3997 dma_addr_t cmd_desc_element_addr; 3998 u16 response_offset; 3999 u16 prdt_offset; 4000 int cmd_desc_size; 4001 int i; 4002 4003 utrdlp = hba->utrdl_base_addr; 4004 4005 response_offset = 4006 offsetof(struct utp_transfer_cmd_desc, response_upiu); 4007 prdt_offset = 4008 offsetof(struct utp_transfer_cmd_desc, prd_table); 4009 4010 cmd_desc_size = ufshcd_get_ucd_size(hba); 4011 cmd_desc_dma_addr = hba->ucdl_dma_addr; 4012 4013 for (i = 0; i < hba->nutrs; i++) { 4014 /* Configure UTRD with command descriptor base address */ 4015 cmd_desc_element_addr = 4016 (cmd_desc_dma_addr + (cmd_desc_size * i)); 4017 utrdlp[i].command_desc_base_addr = 4018 cpu_to_le64(cmd_desc_element_addr); 4019 4020 /* Response upiu and prdt offset should be in double words */ 4021 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) { 4022 utrdlp[i].response_upiu_offset = 4023 cpu_to_le16(response_offset); 4024 utrdlp[i].prd_table_offset = 4025 cpu_to_le16(prdt_offset); 4026 utrdlp[i].response_upiu_length = 4027 cpu_to_le16(ALIGNED_UPIU_SIZE); 4028 } else { 4029 utrdlp[i].response_upiu_offset = 4030 cpu_to_le16(response_offset >> 2); 4031 utrdlp[i].prd_table_offset = 4032 cpu_to_le16(prdt_offset >> 2); 4033 utrdlp[i].response_upiu_length = 4034 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2); 4035 } 4036 } 4037 } 4038 4039 /** 4040 * ufshcd_dme_link_startup - Notify Unipro to perform link startup 4041 * @hba: per adapter instance 4042 * 4043 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer, 4044 * in order to initialize the Unipro link startup procedure. 4045 * Once the Unipro links are up, the device connected to the controller 4046 * is detected. 4047 * 4048 * Return: 0 on success, non-zero value on failure. 4049 */ 4050 static int ufshcd_dme_link_startup(struct ufs_hba *hba) 4051 { 4052 struct uic_command uic_cmd = { 4053 .command = UIC_CMD_DME_LINK_STARTUP, 4054 }; 4055 int ret; 4056 4057 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4058 if (ret) 4059 dev_dbg(hba->dev, 4060 "dme-link-startup: error code %d\n", ret); 4061 return ret; 4062 } 4063 /** 4064 * ufshcd_dme_reset - UIC command for DME_RESET 4065 * @hba: per adapter instance 4066 * 4067 * DME_RESET command is issued in order to reset UniPro stack. 4068 * This function now deals with cold reset. 4069 * 4070 * Return: 0 on success, non-zero value on failure. 4071 */ 4072 int ufshcd_dme_reset(struct ufs_hba *hba) 4073 { 4074 struct uic_command uic_cmd = { 4075 .command = UIC_CMD_DME_RESET, 4076 }; 4077 int ret; 4078 4079 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4080 if (ret) 4081 dev_err(hba->dev, 4082 "dme-reset: error code %d\n", ret); 4083 4084 return ret; 4085 } 4086 EXPORT_SYMBOL_GPL(ufshcd_dme_reset); 4087 4088 int ufshcd_dme_configure_adapt(struct ufs_hba *hba, 4089 int agreed_gear, 4090 int adapt_val) 4091 { 4092 int ret; 4093 4094 if (agreed_gear < UFS_HS_G4) 4095 adapt_val = PA_NO_ADAPT; 4096 4097 ret = ufshcd_dme_set(hba, 4098 UIC_ARG_MIB(PA_TXHSADAPTTYPE), 4099 adapt_val); 4100 return ret; 4101 } 4102 EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt); 4103 4104 /** 4105 * ufshcd_dme_enable - UIC command for DME_ENABLE 4106 * @hba: per adapter instance 4107 * 4108 * DME_ENABLE command is issued in order to enable UniPro stack. 4109 * 4110 * Return: 0 on success, non-zero value on failure. 4111 */ 4112 int ufshcd_dme_enable(struct ufs_hba *hba) 4113 { 4114 struct uic_command uic_cmd = { 4115 .command = UIC_CMD_DME_ENABLE, 4116 }; 4117 int ret; 4118 4119 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4120 if (ret) 4121 dev_err(hba->dev, 4122 "dme-enable: error code %d\n", ret); 4123 4124 return ret; 4125 } 4126 EXPORT_SYMBOL_GPL(ufshcd_dme_enable); 4127 4128 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba) 4129 { 4130 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000 4131 unsigned long min_sleep_time_us; 4132 4133 if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS)) 4134 return; 4135 4136 /* 4137 * last_dme_cmd_tstamp will be 0 only for 1st call to 4138 * this function 4139 */ 4140 if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) { 4141 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US; 4142 } else { 4143 unsigned long delta = 4144 (unsigned long) ktime_to_us( 4145 ktime_sub(ktime_get(), 4146 hba->last_dme_cmd_tstamp)); 4147 4148 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US) 4149 min_sleep_time_us = 4150 MIN_DELAY_BEFORE_DME_CMDS_US - delta; 4151 else 4152 min_sleep_time_us = 0; /* no more delay required */ 4153 } 4154 4155 if (min_sleep_time_us > 0) { 4156 /* allow sleep for extra 50us if needed */ 4157 usleep_range(min_sleep_time_us, min_sleep_time_us + 50); 4158 } 4159 4160 /* update the last_dme_cmd_tstamp */ 4161 hba->last_dme_cmd_tstamp = ktime_get(); 4162 } 4163 4164 /** 4165 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET 4166 * @hba: per adapter instance 4167 * @attr_sel: uic command argument1 4168 * @attr_set: attribute set type as uic command argument2 4169 * @mib_val: setting value as uic command argument3 4170 * @peer: indicate whether peer or local 4171 * 4172 * Return: 0 on success, non-zero value on failure. 4173 */ 4174 int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel, 4175 u8 attr_set, u32 mib_val, u8 peer) 4176 { 4177 struct uic_command uic_cmd = { 4178 .command = peer ? UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET, 4179 .argument1 = attr_sel, 4180 .argument2 = UIC_ARG_ATTR_TYPE(attr_set), 4181 .argument3 = mib_val, 4182 }; 4183 static const char *const action[] = { 4184 "dme-set", 4185 "dme-peer-set" 4186 }; 4187 const char *set = action[!!peer]; 4188 int ret; 4189 int retries = UFS_UIC_COMMAND_RETRIES; 4190 4191 do { 4192 /* for peer attributes we retry upon failure */ 4193 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4194 if (ret) 4195 dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n", 4196 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret); 4197 } while (ret && peer && --retries); 4198 4199 if (ret) 4200 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n", 4201 set, UIC_GET_ATTR_ID(attr_sel), mib_val, 4202 UFS_UIC_COMMAND_RETRIES - retries); 4203 4204 return ret; 4205 } 4206 EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr); 4207 4208 /** 4209 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET 4210 * @hba: per adapter instance 4211 * @attr_sel: uic command argument1 4212 * @mib_val: the value of the attribute as returned by the UIC command 4213 * @peer: indicate whether peer or local 4214 * 4215 * Return: 0 on success, non-zero value on failure. 4216 */ 4217 int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel, 4218 u32 *mib_val, u8 peer) 4219 { 4220 struct uic_command uic_cmd = { 4221 .command = peer ? UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET, 4222 .argument1 = attr_sel, 4223 }; 4224 static const char *const action[] = { 4225 "dme-get", 4226 "dme-peer-get" 4227 }; 4228 const char *get = action[!!peer]; 4229 int ret; 4230 int retries = UFS_UIC_COMMAND_RETRIES; 4231 struct ufs_pa_layer_attr orig_pwr_info; 4232 struct ufs_pa_layer_attr temp_pwr_info; 4233 bool pwr_mode_change = false; 4234 4235 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) { 4236 orig_pwr_info = hba->pwr_info; 4237 temp_pwr_info = orig_pwr_info; 4238 4239 if (orig_pwr_info.pwr_tx == FAST_MODE || 4240 orig_pwr_info.pwr_rx == FAST_MODE) { 4241 temp_pwr_info.pwr_tx = FASTAUTO_MODE; 4242 temp_pwr_info.pwr_rx = FASTAUTO_MODE; 4243 pwr_mode_change = true; 4244 } else if (orig_pwr_info.pwr_tx == SLOW_MODE || 4245 orig_pwr_info.pwr_rx == SLOW_MODE) { 4246 temp_pwr_info.pwr_tx = SLOWAUTO_MODE; 4247 temp_pwr_info.pwr_rx = SLOWAUTO_MODE; 4248 pwr_mode_change = true; 4249 } 4250 if (pwr_mode_change) { 4251 ret = ufshcd_change_power_mode(hba, &temp_pwr_info); 4252 if (ret) 4253 goto out; 4254 } 4255 } 4256 4257 do { 4258 /* for peer attributes we retry upon failure */ 4259 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4260 if (ret) 4261 dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n", 4262 get, UIC_GET_ATTR_ID(attr_sel), ret); 4263 } while (ret && peer && --retries); 4264 4265 if (ret) 4266 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n", 4267 get, UIC_GET_ATTR_ID(attr_sel), 4268 UFS_UIC_COMMAND_RETRIES - retries); 4269 4270 if (mib_val) 4271 *mib_val = ret == 0 ? uic_cmd.argument3 : 0; 4272 4273 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE) 4274 && pwr_mode_change) 4275 ufshcd_change_power_mode(hba, &orig_pwr_info); 4276 out: 4277 return ret; 4278 } 4279 EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr); 4280 4281 /** 4282 * ufshcd_dme_rmw - get modify set a DME attribute 4283 * @hba: per adapter instance 4284 * @mask: indicates which bits to clear from the value that has been read 4285 * @val: actual value to write 4286 * @attr: dme attribute 4287 */ 4288 int ufshcd_dme_rmw(struct ufs_hba *hba, u32 mask, 4289 u32 val, u32 attr) 4290 { 4291 u32 cfg = 0; 4292 int err; 4293 4294 err = ufshcd_dme_get(hba, UIC_ARG_MIB(attr), &cfg); 4295 if (err) 4296 return err; 4297 4298 cfg &= ~mask; 4299 cfg |= (val & mask); 4300 4301 return ufshcd_dme_set(hba, UIC_ARG_MIB(attr), cfg); 4302 } 4303 EXPORT_SYMBOL_GPL(ufshcd_dme_rmw); 4304 4305 /** 4306 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power 4307 * state) and waits for it to take effect. 4308 * 4309 * @hba: per adapter instance 4310 * @cmd: UIC command to execute 4311 * 4312 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER & 4313 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host 4314 * and device UniPro link and hence it's final completion would be indicated by 4315 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in 4316 * addition to normal UIC command completion Status (UCCS). This function only 4317 * returns after the relevant status bits indicate the completion. 4318 * 4319 * Return: 0 on success, non-zero value on failure. 4320 */ 4321 static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) 4322 { 4323 DECLARE_COMPLETION_ONSTACK(uic_async_done); 4324 unsigned long flags; 4325 u8 status; 4326 int ret; 4327 4328 mutex_lock(&hba->uic_cmd_mutex); 4329 ufshcd_add_delay_before_dme_cmd(hba); 4330 4331 spin_lock_irqsave(hba->host->host_lock, flags); 4332 if (ufshcd_is_link_broken(hba)) { 4333 ret = -ENOLINK; 4334 goto out_unlock; 4335 } 4336 hba->uic_async_done = &uic_async_done; 4337 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL); 4338 spin_unlock_irqrestore(hba->host->host_lock, flags); 4339 ret = __ufshcd_send_uic_cmd(hba, cmd); 4340 if (ret) { 4341 dev_err(hba->dev, 4342 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n", 4343 cmd->command, cmd->argument3, ret); 4344 goto out; 4345 } 4346 4347 if (!wait_for_completion_timeout(hba->uic_async_done, 4348 msecs_to_jiffies(uic_cmd_timeout))) { 4349 dev_err(hba->dev, 4350 "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n", 4351 cmd->command, cmd->argument3); 4352 4353 if (!cmd->cmd_active) { 4354 dev_err(hba->dev, "%s: Power Mode Change operation has been completed, go check UPMCRS\n", 4355 __func__); 4356 goto check_upmcrs; 4357 } 4358 4359 ret = -ETIMEDOUT; 4360 goto out; 4361 } 4362 4363 check_upmcrs: 4364 status = ufshcd_get_upmcrs(hba); 4365 if (status != PWR_LOCAL) { 4366 dev_err(hba->dev, 4367 "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n", 4368 cmd->command, status); 4369 ret = (status != PWR_OK) ? status : -1; 4370 } 4371 out: 4372 if (ret) { 4373 ufshcd_print_host_state(hba); 4374 ufshcd_print_pwr_info(hba); 4375 ufshcd_print_evt_hist(hba); 4376 } 4377 4378 spin_lock_irqsave(hba->host->host_lock, flags); 4379 hba->active_uic_cmd = NULL; 4380 hba->uic_async_done = NULL; 4381 if (ret && !hba->pm_op_in_progress) { 4382 ufshcd_set_link_broken(hba); 4383 ufshcd_schedule_eh_work(hba); 4384 } 4385 out_unlock: 4386 spin_unlock_irqrestore(hba->host->host_lock, flags); 4387 mutex_unlock(&hba->uic_cmd_mutex); 4388 4389 /* 4390 * If the h8 exit fails during the runtime resume process, it becomes 4391 * stuck and cannot be recovered through the error handler. To fix 4392 * this, use link recovery instead of the error handler. 4393 */ 4394 if (ret && hba->pm_op_in_progress) 4395 ret = ufshcd_link_recovery(hba); 4396 4397 return ret; 4398 } 4399 4400 /** 4401 * ufshcd_send_bsg_uic_cmd - Send UIC commands requested via BSG layer and retrieve the result 4402 * @hba: per adapter instance 4403 * @uic_cmd: UIC command 4404 * 4405 * Return: 0 only if success. 4406 */ 4407 int ufshcd_send_bsg_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 4408 { 4409 int ret; 4410 4411 if (uic_cmd->argument1 != UIC_ARG_MIB(PA_PWRMODE) || 4412 uic_cmd->command != UIC_CMD_DME_SET) 4413 return ufshcd_send_uic_cmd(hba, uic_cmd); 4414 4415 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD) 4416 return 0; 4417 4418 ufshcd_hold(hba); 4419 ret = ufshcd_uic_pwr_ctrl(hba, uic_cmd); 4420 ufshcd_release(hba); 4421 4422 return ret; 4423 } 4424 4425 /** 4426 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage 4427 * using DME_SET primitives. 4428 * @hba: per adapter instance 4429 * @mode: powr mode value 4430 * 4431 * Return: 0 on success, non-zero value on failure. 4432 */ 4433 int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode) 4434 { 4435 struct uic_command uic_cmd = { 4436 .command = UIC_CMD_DME_SET, 4437 .argument1 = UIC_ARG_MIB(PA_PWRMODE), 4438 .argument3 = mode, 4439 }; 4440 int ret; 4441 4442 if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) { 4443 ret = ufshcd_dme_set(hba, 4444 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1); 4445 if (ret) { 4446 dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n", 4447 __func__, ret); 4448 goto out; 4449 } 4450 } 4451 4452 ufshcd_hold(hba); 4453 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 4454 ufshcd_release(hba); 4455 4456 out: 4457 return ret; 4458 } 4459 EXPORT_SYMBOL_GPL(ufshcd_uic_change_pwr_mode); 4460 4461 int ufshcd_link_recovery(struct ufs_hba *hba) 4462 { 4463 int ret; 4464 unsigned long flags; 4465 4466 spin_lock_irqsave(hba->host->host_lock, flags); 4467 hba->ufshcd_state = UFSHCD_STATE_RESET; 4468 ufshcd_set_eh_in_progress(hba); 4469 spin_unlock_irqrestore(hba->host->host_lock, flags); 4470 4471 /* Reset the attached device */ 4472 ufshcd_device_reset(hba); 4473 4474 ret = ufshcd_host_reset_and_restore(hba); 4475 4476 spin_lock_irqsave(hba->host->host_lock, flags); 4477 if (ret) 4478 hba->ufshcd_state = UFSHCD_STATE_ERROR; 4479 ufshcd_clear_eh_in_progress(hba); 4480 spin_unlock_irqrestore(hba->host->host_lock, flags); 4481 4482 if (ret) 4483 dev_err(hba->dev, "%s: link recovery failed, err %d", 4484 __func__, ret); 4485 4486 return ret; 4487 } 4488 EXPORT_SYMBOL_GPL(ufshcd_link_recovery); 4489 4490 int ufshcd_uic_hibern8_enter(struct ufs_hba *hba) 4491 { 4492 struct uic_command uic_cmd = { 4493 .command = UIC_CMD_DME_HIBER_ENTER, 4494 }; 4495 ktime_t start = ktime_get(); 4496 int ret; 4497 4498 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE); 4499 4500 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 4501 trace_ufshcd_profile_hibern8(hba, "enter", 4502 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 4503 4504 if (ret) 4505 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n", 4506 __func__, ret); 4507 else 4508 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, 4509 POST_CHANGE); 4510 4511 return ret; 4512 } 4513 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_enter); 4514 4515 int ufshcd_uic_hibern8_exit(struct ufs_hba *hba) 4516 { 4517 struct uic_command uic_cmd = { 4518 .command = UIC_CMD_DME_HIBER_EXIT, 4519 }; 4520 int ret; 4521 ktime_t start = ktime_get(); 4522 4523 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE); 4524 4525 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 4526 trace_ufshcd_profile_hibern8(hba, "exit", 4527 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 4528 4529 if (ret) { 4530 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n", 4531 __func__, ret); 4532 } else { 4533 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, 4534 POST_CHANGE); 4535 hba->ufs_stats.last_hibern8_exit_tstamp = local_clock(); 4536 hba->ufs_stats.hibern8_exit_cnt++; 4537 } 4538 4539 return ret; 4540 } 4541 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit); 4542 4543 static void ufshcd_configure_auto_hibern8(struct ufs_hba *hba) 4544 { 4545 if (!ufshcd_is_auto_hibern8_supported(hba)) 4546 return; 4547 4548 ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER); 4549 } 4550 4551 void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit) 4552 { 4553 const u32 cur_ahit = READ_ONCE(hba->ahit); 4554 4555 if (!ufshcd_is_auto_hibern8_supported(hba) || cur_ahit == ahit) 4556 return; 4557 4558 WRITE_ONCE(hba->ahit, ahit); 4559 if (!pm_runtime_suspended(&hba->ufs_device_wlun->sdev_gendev)) { 4560 ufshcd_rpm_get_sync(hba); 4561 ufshcd_hold(hba); 4562 ufshcd_configure_auto_hibern8(hba); 4563 ufshcd_release(hba); 4564 ufshcd_rpm_put_sync(hba); 4565 } 4566 } 4567 EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update); 4568 4569 /** 4570 * ufshcd_init_pwr_info - setting the POR (power on reset) 4571 * values in hba power info 4572 * @hba: per-adapter instance 4573 */ 4574 static void ufshcd_init_pwr_info(struct ufs_hba *hba) 4575 { 4576 hba->pwr_info.gear_rx = UFS_PWM_G1; 4577 hba->pwr_info.gear_tx = UFS_PWM_G1; 4578 hba->pwr_info.lane_rx = UFS_LANE_1; 4579 hba->pwr_info.lane_tx = UFS_LANE_1; 4580 hba->pwr_info.pwr_rx = SLOWAUTO_MODE; 4581 hba->pwr_info.pwr_tx = SLOWAUTO_MODE; 4582 hba->pwr_info.hs_rate = 0; 4583 } 4584 4585 /** 4586 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device 4587 * @hba: per-adapter instance 4588 * 4589 * Return: 0 upon success; < 0 upon failure. 4590 */ 4591 static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba) 4592 { 4593 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; 4594 4595 if (hba->max_pwr_info.is_valid) 4596 return 0; 4597 4598 if (hba->quirks & UFSHCD_QUIRK_HIBERN_FASTAUTO) { 4599 pwr_info->pwr_tx = FASTAUTO_MODE; 4600 pwr_info->pwr_rx = FASTAUTO_MODE; 4601 } else { 4602 pwr_info->pwr_tx = FAST_MODE; 4603 pwr_info->pwr_rx = FAST_MODE; 4604 } 4605 pwr_info->hs_rate = PA_HS_MODE_B; 4606 4607 /* Get the connected lane count */ 4608 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES), 4609 &pwr_info->lane_rx); 4610 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 4611 &pwr_info->lane_tx); 4612 4613 if (!pwr_info->lane_rx || !pwr_info->lane_tx) { 4614 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n", 4615 __func__, 4616 pwr_info->lane_rx, 4617 pwr_info->lane_tx); 4618 return -EINVAL; 4619 } 4620 4621 if (pwr_info->lane_rx != pwr_info->lane_tx) { 4622 dev_err(hba->dev, "%s: asymmetric connected lanes. rx=%d, tx=%d\n", 4623 __func__, 4624 pwr_info->lane_rx, 4625 pwr_info->lane_tx); 4626 return -EINVAL; 4627 } 4628 4629 /* 4630 * First, get the maximum gears of HS speed. 4631 * If a zero value, it means there is no HSGEAR capability. 4632 * Then, get the maximum gears of PWM speed. 4633 */ 4634 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx); 4635 if (!pwr_info->gear_rx) { 4636 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR), 4637 &pwr_info->gear_rx); 4638 if (!pwr_info->gear_rx) { 4639 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n", 4640 __func__, pwr_info->gear_rx); 4641 return -EINVAL; 4642 } 4643 pwr_info->pwr_rx = SLOW_MODE; 4644 } 4645 4646 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), 4647 &pwr_info->gear_tx); 4648 if (!pwr_info->gear_tx) { 4649 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR), 4650 &pwr_info->gear_tx); 4651 if (!pwr_info->gear_tx) { 4652 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n", 4653 __func__, pwr_info->gear_tx); 4654 return -EINVAL; 4655 } 4656 pwr_info->pwr_tx = SLOW_MODE; 4657 } 4658 4659 hba->max_pwr_info.is_valid = true; 4660 return 0; 4661 } 4662 4663 static int ufshcd_change_power_mode(struct ufs_hba *hba, 4664 struct ufs_pa_layer_attr *pwr_mode) 4665 { 4666 int ret; 4667 4668 /* if already configured to the requested pwr_mode */ 4669 if (!hba->force_pmc && 4670 pwr_mode->gear_rx == hba->pwr_info.gear_rx && 4671 pwr_mode->gear_tx == hba->pwr_info.gear_tx && 4672 pwr_mode->lane_rx == hba->pwr_info.lane_rx && 4673 pwr_mode->lane_tx == hba->pwr_info.lane_tx && 4674 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx && 4675 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx && 4676 pwr_mode->hs_rate == hba->pwr_info.hs_rate) { 4677 dev_dbg(hba->dev, "%s: power already configured\n", __func__); 4678 return 0; 4679 } 4680 4681 /* 4682 * Configure attributes for power mode change with below. 4683 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION, 4684 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION, 4685 * - PA_HSSERIES 4686 */ 4687 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx); 4688 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES), 4689 pwr_mode->lane_rx); 4690 if (pwr_mode->pwr_rx == FASTAUTO_MODE || 4691 pwr_mode->pwr_rx == FAST_MODE) 4692 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), true); 4693 else 4694 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), false); 4695 4696 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx); 4697 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES), 4698 pwr_mode->lane_tx); 4699 if (pwr_mode->pwr_tx == FASTAUTO_MODE || 4700 pwr_mode->pwr_tx == FAST_MODE) 4701 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), true); 4702 else 4703 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), false); 4704 4705 if (pwr_mode->pwr_rx == FASTAUTO_MODE || 4706 pwr_mode->pwr_tx == FASTAUTO_MODE || 4707 pwr_mode->pwr_rx == FAST_MODE || 4708 pwr_mode->pwr_tx == FAST_MODE) 4709 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES), 4710 pwr_mode->hs_rate); 4711 4712 if (!(hba->quirks & UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING)) { 4713 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0), 4714 DL_FC0ProtectionTimeOutVal_Default); 4715 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1), 4716 DL_TC0ReplayTimeOutVal_Default); 4717 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2), 4718 DL_AFC0ReqTimeOutVal_Default); 4719 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3), 4720 DL_FC1ProtectionTimeOutVal_Default); 4721 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4), 4722 DL_TC1ReplayTimeOutVal_Default); 4723 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5), 4724 DL_AFC1ReqTimeOutVal_Default); 4725 4726 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal), 4727 DL_FC0ProtectionTimeOutVal_Default); 4728 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal), 4729 DL_TC0ReplayTimeOutVal_Default); 4730 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal), 4731 DL_AFC0ReqTimeOutVal_Default); 4732 } 4733 4734 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4 4735 | pwr_mode->pwr_tx); 4736 4737 if (ret) { 4738 dev_err(hba->dev, 4739 "%s: power mode change failed %d\n", __func__, ret); 4740 } else { 4741 memcpy(&hba->pwr_info, pwr_mode, 4742 sizeof(struct ufs_pa_layer_attr)); 4743 } 4744 4745 return ret; 4746 } 4747 4748 /** 4749 * ufshcd_config_pwr_mode - configure a new power mode 4750 * @hba: per-adapter instance 4751 * @desired_pwr_mode: desired power configuration 4752 * 4753 * Return: 0 upon success; < 0 upon failure. 4754 */ 4755 int ufshcd_config_pwr_mode(struct ufs_hba *hba, 4756 struct ufs_pa_layer_attr *desired_pwr_mode) 4757 { 4758 struct ufs_pa_layer_attr final_params = { 0 }; 4759 int ret; 4760 4761 ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE, 4762 desired_pwr_mode, &final_params); 4763 4764 if (ret) 4765 memcpy(&final_params, desired_pwr_mode, sizeof(final_params)); 4766 4767 ret = ufshcd_change_power_mode(hba, &final_params); 4768 4769 if (!ret) 4770 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL, 4771 &final_params); 4772 4773 return ret; 4774 } 4775 EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode); 4776 4777 /** 4778 * ufshcd_complete_dev_init() - checks device readiness 4779 * @hba: per-adapter instance 4780 * 4781 * Set fDeviceInit flag and poll until device toggles it. 4782 * 4783 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 4784 * < 0 if another error occurred. 4785 */ 4786 static int ufshcd_complete_dev_init(struct ufs_hba *hba) 4787 { 4788 int err; 4789 bool flag_res = true; 4790 ktime_t timeout; 4791 4792 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG, 4793 QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL); 4794 if (err) { 4795 dev_err(hba->dev, 4796 "%s: setting fDeviceInit flag failed with error %d\n", 4797 __func__, err); 4798 goto out; 4799 } 4800 4801 /* Poll fDeviceInit flag to be cleared */ 4802 timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT); 4803 do { 4804 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG, 4805 QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res); 4806 if (!flag_res) 4807 break; 4808 usleep_range(500, 1000); 4809 } while (ktime_before(ktime_get(), timeout)); 4810 4811 if (err) { 4812 dev_err(hba->dev, 4813 "%s: reading fDeviceInit flag failed with error %d\n", 4814 __func__, err); 4815 } else if (flag_res) { 4816 dev_err(hba->dev, 4817 "%s: fDeviceInit was not cleared by the device\n", 4818 __func__); 4819 err = -EBUSY; 4820 } 4821 out: 4822 return err; 4823 } 4824 4825 /** 4826 * ufshcd_make_hba_operational - Make UFS controller operational 4827 * @hba: per adapter instance 4828 * 4829 * To bring UFS host controller to operational state, 4830 * 1. Enable required interrupts 4831 * 2. Configure interrupt aggregation 4832 * 3. Program UTRL and UTMRL base address 4833 * 4. Configure run-stop-registers 4834 * 4835 * Return: 0 if successful; < 0 upon failure. 4836 */ 4837 int ufshcd_make_hba_operational(struct ufs_hba *hba) 4838 { 4839 int err = 0; 4840 u32 reg; 4841 4842 /* Enable required interrupts */ 4843 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS); 4844 4845 /* Configure interrupt aggregation */ 4846 if (ufshcd_is_intr_aggr_allowed(hba)) 4847 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO); 4848 else 4849 ufshcd_disable_intr_aggr(hba); 4850 4851 /* Configure UTRL and UTMRL base address registers */ 4852 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr), 4853 REG_UTP_TRANSFER_REQ_LIST_BASE_L); 4854 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr), 4855 REG_UTP_TRANSFER_REQ_LIST_BASE_H); 4856 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr), 4857 REG_UTP_TASK_REQ_LIST_BASE_L); 4858 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr), 4859 REG_UTP_TASK_REQ_LIST_BASE_H); 4860 4861 /* 4862 * UCRDY, UTMRLDY and UTRLRDY bits must be 1 4863 */ 4864 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS); 4865 if (!(ufshcd_get_lists_status(reg))) { 4866 ufshcd_enable_run_stop_reg(hba); 4867 } else { 4868 dev_err(hba->dev, 4869 "Host controller not ready to process requests"); 4870 err = -EIO; 4871 } 4872 4873 return err; 4874 } 4875 EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational); 4876 4877 /** 4878 * ufshcd_hba_stop - Send controller to reset state 4879 * @hba: per adapter instance 4880 */ 4881 void ufshcd_hba_stop(struct ufs_hba *hba) 4882 { 4883 int err; 4884 4885 ufshcd_disable_irq(hba); 4886 ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE); 4887 err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE, 4888 CONTROLLER_ENABLE, CONTROLLER_DISABLE, 4889 10, 1); 4890 ufshcd_enable_irq(hba); 4891 if (err) 4892 dev_err(hba->dev, "%s: Controller disable failed\n", __func__); 4893 } 4894 EXPORT_SYMBOL_GPL(ufshcd_hba_stop); 4895 4896 /** 4897 * ufshcd_hba_execute_hce - initialize the controller 4898 * @hba: per adapter instance 4899 * 4900 * The controller resets itself and controller firmware initialization 4901 * sequence kicks off. When controller is ready it will set 4902 * the Host Controller Enable bit to 1. 4903 * 4904 * Return: 0 on success, non-zero value on failure. 4905 */ 4906 static int ufshcd_hba_execute_hce(struct ufs_hba *hba) 4907 { 4908 int retry; 4909 4910 for (retry = 3; retry > 0; retry--) { 4911 if (ufshcd_is_hba_active(hba)) 4912 /* change controller state to "reset state" */ 4913 ufshcd_hba_stop(hba); 4914 4915 /* UniPro link is disabled at this point */ 4916 ufshcd_set_link_off(hba); 4917 4918 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE); 4919 4920 /* start controller initialization sequence */ 4921 ufshcd_hba_start(hba); 4922 4923 /* 4924 * To initialize a UFS host controller HCE bit must be set to 1. 4925 * During initialization the HCE bit value changes from 1->0->1. 4926 * When the host controller completes initialization sequence 4927 * it sets the value of HCE bit to 1. The same HCE bit is read back 4928 * to check if the controller has completed initialization sequence. 4929 * So without this delay the value HCE = 1, set in the previous 4930 * instruction might be read back. 4931 * This delay can be changed based on the controller. 4932 */ 4933 ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100); 4934 4935 /* wait for the host controller to complete initialization */ 4936 if (!ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE, CONTROLLER_ENABLE, 4937 CONTROLLER_ENABLE, 1000, 50)) 4938 break; 4939 4940 dev_err(hba->dev, "Enabling the controller failed\n"); 4941 } 4942 4943 if (!retry) 4944 return -EIO; 4945 4946 /* enable UIC related interrupts */ 4947 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK); 4948 4949 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE); 4950 4951 return 0; 4952 } 4953 4954 int ufshcd_hba_enable(struct ufs_hba *hba) 4955 { 4956 int ret; 4957 4958 if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) { 4959 ufshcd_set_link_off(hba); 4960 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE); 4961 4962 /* enable UIC related interrupts */ 4963 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK); 4964 ret = ufshcd_dme_reset(hba); 4965 if (ret) { 4966 dev_err(hba->dev, "DME_RESET failed\n"); 4967 return ret; 4968 } 4969 4970 ret = ufshcd_dme_enable(hba); 4971 if (ret) { 4972 dev_err(hba->dev, "Enabling DME failed\n"); 4973 return ret; 4974 } 4975 4976 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE); 4977 } else { 4978 ret = ufshcd_hba_execute_hce(hba); 4979 } 4980 4981 return ret; 4982 } 4983 EXPORT_SYMBOL_GPL(ufshcd_hba_enable); 4984 4985 static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer) 4986 { 4987 int tx_lanes, i, err = 0; 4988 4989 if (!peer) 4990 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 4991 &tx_lanes); 4992 else 4993 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 4994 &tx_lanes); 4995 for (i = 0; i < tx_lanes; i++) { 4996 if (!peer) 4997 err = ufshcd_dme_set(hba, 4998 UIC_ARG_MIB_SEL(TX_LCC_ENABLE, 4999 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)), 5000 0); 5001 else 5002 err = ufshcd_dme_peer_set(hba, 5003 UIC_ARG_MIB_SEL(TX_LCC_ENABLE, 5004 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)), 5005 0); 5006 if (err) { 5007 dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d", 5008 __func__, peer, i, err); 5009 break; 5010 } 5011 } 5012 5013 return err; 5014 } 5015 5016 static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba) 5017 { 5018 return ufshcd_disable_tx_lcc(hba, true); 5019 } 5020 5021 void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val) 5022 { 5023 struct ufs_event_hist *e; 5024 5025 if (id >= UFS_EVT_CNT) 5026 return; 5027 5028 e = &hba->ufs_stats.event[id]; 5029 e->val[e->pos] = val; 5030 e->tstamp[e->pos] = local_clock(); 5031 e->cnt += 1; 5032 e->pos = (e->pos + 1) % UFS_EVENT_HIST_LENGTH; 5033 5034 ufshcd_vops_event_notify(hba, id, &val); 5035 } 5036 EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist); 5037 5038 /** 5039 * ufshcd_link_startup - Initialize unipro link startup 5040 * @hba: per adapter instance 5041 * 5042 * Return: 0 for success, non-zero in case of failure. 5043 */ 5044 static int ufshcd_link_startup(struct ufs_hba *hba) 5045 { 5046 int ret; 5047 int retries = DME_LINKSTARTUP_RETRIES; 5048 bool link_startup_again = false; 5049 5050 /* 5051 * If UFS device isn't active then we will have to issue link startup 5052 * 2 times to make sure the device state move to active. 5053 */ 5054 if (!(hba->quirks & UFSHCD_QUIRK_PERFORM_LINK_STARTUP_ONCE) && 5055 !ufshcd_is_ufs_dev_active(hba)) 5056 link_startup_again = true; 5057 5058 link_startup: 5059 do { 5060 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE); 5061 5062 ret = ufshcd_dme_link_startup(hba); 5063 5064 /* check if device is detected by inter-connect layer */ 5065 if (!ret && !ufshcd_is_device_present(hba)) { 5066 ufshcd_update_evt_hist(hba, 5067 UFS_EVT_LINK_STARTUP_FAIL, 5068 0); 5069 dev_err(hba->dev, "%s: Device not present\n", __func__); 5070 ret = -ENXIO; 5071 goto out; 5072 } 5073 5074 /* 5075 * DME link lost indication is only received when link is up, 5076 * but we can't be sure if the link is up until link startup 5077 * succeeds. So reset the local Uni-Pro and try again. 5078 */ 5079 if (ret && retries && ufshcd_hba_enable(hba)) { 5080 ufshcd_update_evt_hist(hba, 5081 UFS_EVT_LINK_STARTUP_FAIL, 5082 (u32)ret); 5083 goto out; 5084 } 5085 } while (ret && retries--); 5086 5087 if (ret) { 5088 /* failed to get the link up... retire */ 5089 ufshcd_update_evt_hist(hba, 5090 UFS_EVT_LINK_STARTUP_FAIL, 5091 (u32)ret); 5092 goto out; 5093 } 5094 5095 if (link_startup_again) { 5096 link_startup_again = false; 5097 retries = DME_LINKSTARTUP_RETRIES; 5098 goto link_startup; 5099 } 5100 5101 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */ 5102 ufshcd_init_pwr_info(hba); 5103 ufshcd_print_pwr_info(hba); 5104 5105 if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) { 5106 ret = ufshcd_disable_device_tx_lcc(hba); 5107 if (ret) 5108 goto out; 5109 } 5110 5111 /* Include any host controller configuration via UIC commands */ 5112 ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE); 5113 if (ret) 5114 goto out; 5115 5116 /* Clear UECPA once due to LINERESET has happened during LINK_STARTUP */ 5117 ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER); 5118 ret = ufshcd_make_hba_operational(hba); 5119 out: 5120 if (ret) 5121 dev_err(hba->dev, "link startup failed %d\n", ret); 5122 return ret; 5123 } 5124 5125 /** 5126 * ufshcd_verify_dev_init() - Verify device initialization 5127 * @hba: per-adapter instance 5128 * 5129 * Send NOP OUT UPIU and wait for NOP IN response to check whether the 5130 * device Transport Protocol (UTP) layer is ready after a reset. 5131 * If the UTP layer at the device side is not initialized, it may 5132 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT 5133 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations. 5134 * 5135 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 5136 * < 0 if another error occurred. 5137 */ 5138 static int ufshcd_verify_dev_init(struct ufs_hba *hba) 5139 { 5140 int err = 0; 5141 int retries; 5142 5143 ufshcd_dev_man_lock(hba); 5144 5145 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) { 5146 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP, 5147 hba->nop_out_timeout); 5148 5149 if (!err || err == -ETIMEDOUT) 5150 break; 5151 5152 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err); 5153 } 5154 5155 ufshcd_dev_man_unlock(hba); 5156 5157 if (err) 5158 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err); 5159 return err; 5160 } 5161 5162 /** 5163 * ufshcd_setup_links - associate link b/w device wlun and other luns 5164 * @sdev: pointer to SCSI device 5165 * @hba: pointer to ufs hba 5166 */ 5167 static void ufshcd_setup_links(struct ufs_hba *hba, struct scsi_device *sdev) 5168 { 5169 struct device_link *link; 5170 5171 /* 5172 * Device wlun is the supplier & rest of the luns are consumers. 5173 * This ensures that device wlun suspends after all other luns. 5174 */ 5175 if (hba->ufs_device_wlun) { 5176 link = device_link_add(&sdev->sdev_gendev, 5177 &hba->ufs_device_wlun->sdev_gendev, 5178 DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE); 5179 if (!link) { 5180 dev_err(&sdev->sdev_gendev, "Failed establishing link - %s\n", 5181 dev_name(&hba->ufs_device_wlun->sdev_gendev)); 5182 return; 5183 } 5184 hba->luns_avail--; 5185 /* Ignore REPORT_LUN wlun probing */ 5186 if (hba->luns_avail == 1) { 5187 ufshcd_rpm_put(hba); 5188 return; 5189 } 5190 } else { 5191 /* 5192 * Device wlun is probed. The assumption is that WLUNs are 5193 * scanned before other LUNs. 5194 */ 5195 hba->luns_avail--; 5196 } 5197 } 5198 5199 /** 5200 * ufshcd_lu_init - Initialize the relevant parameters of the LU 5201 * @hba: per-adapter instance 5202 * @sdev: pointer to SCSI device 5203 */ 5204 static void ufshcd_lu_init(struct ufs_hba *hba, struct scsi_device *sdev) 5205 { 5206 int len = QUERY_DESC_MAX_SIZE; 5207 u8 lun = ufshcd_scsi_to_upiu_lun(sdev->lun); 5208 u8 lun_qdepth = hba->nutrs; 5209 u8 *desc_buf; 5210 int ret; 5211 5212 desc_buf = kzalloc(len, GFP_KERNEL); 5213 if (!desc_buf) 5214 goto set_qdepth; 5215 5216 ret = ufshcd_read_unit_desc_param(hba, lun, 0, desc_buf, len); 5217 if (ret < 0) { 5218 if (ret == -EOPNOTSUPP) 5219 /* If LU doesn't support unit descriptor, its queue depth is set to 1 */ 5220 lun_qdepth = 1; 5221 kfree(desc_buf); 5222 goto set_qdepth; 5223 } 5224 5225 if (desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH]) { 5226 /* 5227 * In per-LU queueing architecture, bLUQueueDepth will not be 0, then we will 5228 * use the smaller between UFSHCI CAP.NUTRS and UFS LU bLUQueueDepth 5229 */ 5230 lun_qdepth = min_t(int, desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH], hba->nutrs); 5231 } 5232 /* 5233 * According to UFS device specification, the write protection mode is only supported by 5234 * normal LU, not supported by WLUN. 5235 */ 5236 if (hba->dev_info.f_power_on_wp_en && lun < hba->dev_info.max_lu_supported && 5237 !hba->dev_info.is_lu_power_on_wp && 5238 desc_buf[UNIT_DESC_PARAM_LU_WR_PROTECT] == UFS_LU_POWER_ON_WP) 5239 hba->dev_info.is_lu_power_on_wp = true; 5240 5241 /* In case of RPMB LU, check if advanced RPMB mode is enabled, and get region size */ 5242 if (desc_buf[UNIT_DESC_PARAM_UNIT_INDEX] == UFS_UPIU_RPMB_WLUN) { 5243 if (desc_buf[RPMB_UNIT_DESC_PARAM_REGION_EN] & BIT(4)) 5244 hba->dev_info.b_advanced_rpmb_en = true; 5245 hba->dev_info.rpmb_region_size[0] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION0_SIZE]; 5246 hba->dev_info.rpmb_region_size[1] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION1_SIZE]; 5247 hba->dev_info.rpmb_region_size[2] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION2_SIZE]; 5248 hba->dev_info.rpmb_region_size[3] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION3_SIZE]; 5249 } 5250 5251 5252 kfree(desc_buf); 5253 set_qdepth: 5254 /* 5255 * For WLUNs that don't support unit descriptor, queue depth is set to 1. For LUs whose 5256 * bLUQueueDepth == 0, the queue depth is set to a maximum value that host can queue. 5257 */ 5258 dev_dbg(hba->dev, "Set LU %x queue depth %d\n", lun, lun_qdepth); 5259 scsi_change_queue_depth(sdev, lun_qdepth); 5260 } 5261 5262 /** 5263 * ufshcd_sdev_init - handle initial SCSI device configurations 5264 * @sdev: pointer to SCSI device 5265 * 5266 * Return: success. 5267 */ 5268 static int ufshcd_sdev_init(struct scsi_device *sdev) 5269 { 5270 struct ufs_hba *hba; 5271 5272 hba = shost_priv(sdev->host); 5273 5274 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */ 5275 sdev->use_10_for_ms = 1; 5276 5277 /* DBD field should be set to 1 in mode sense(10) */ 5278 sdev->set_dbd_for_ms = 1; 5279 5280 /* allow SCSI layer to restart the device in case of errors */ 5281 sdev->allow_restart = 1; 5282 5283 /* REPORT SUPPORTED OPERATION CODES is not supported */ 5284 sdev->no_report_opcodes = 1; 5285 5286 /* WRITE_SAME command is not supported */ 5287 sdev->no_write_same = 1; 5288 5289 ufshcd_lu_init(hba, sdev); 5290 5291 ufshcd_setup_links(hba, sdev); 5292 5293 return 0; 5294 } 5295 5296 /** 5297 * ufshcd_change_queue_depth - change queue depth 5298 * @sdev: pointer to SCSI device 5299 * @depth: required depth to set 5300 * 5301 * Change queue depth and make sure the max. limits are not crossed. 5302 * 5303 * Return: new queue depth. 5304 */ 5305 static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth) 5306 { 5307 return scsi_change_queue_depth(sdev, min(depth, sdev->host->can_queue)); 5308 } 5309 5310 /** 5311 * ufshcd_sdev_configure - adjust SCSI device configurations 5312 * @sdev: pointer to SCSI device 5313 * @lim: queue limits 5314 * 5315 * Return: 0 (success). 5316 */ 5317 static int ufshcd_sdev_configure(struct scsi_device *sdev, 5318 struct queue_limits *lim) 5319 { 5320 struct ufs_hba *hba = shost_priv(sdev->host); 5321 struct request_queue *q = sdev->request_queue; 5322 5323 lim->dma_pad_mask = PRDT_DATA_BYTE_COUNT_PAD - 1; 5324 5325 /* 5326 * Block runtime-pm until all consumers are added. 5327 * Refer ufshcd_setup_links(). 5328 */ 5329 if (is_device_wlun(sdev)) 5330 pm_runtime_get_noresume(&sdev->sdev_gendev); 5331 else if (ufshcd_is_rpm_autosuspend_allowed(hba)) 5332 sdev->rpm_autosuspend = 1; 5333 /* 5334 * Do not print messages during runtime PM to avoid never-ending cycles 5335 * of messages written back to storage by user space causing runtime 5336 * resume, causing more messages and so on. 5337 */ 5338 sdev->silence_suspend = 1; 5339 5340 if (hba->vops && hba->vops->config_scsi_dev) 5341 hba->vops->config_scsi_dev(sdev); 5342 5343 ufshcd_crypto_register(hba, q); 5344 5345 return 0; 5346 } 5347 5348 /** 5349 * ufshcd_sdev_destroy - remove SCSI device configurations 5350 * @sdev: pointer to SCSI device 5351 */ 5352 static void ufshcd_sdev_destroy(struct scsi_device *sdev) 5353 { 5354 struct ufs_hba *hba; 5355 unsigned long flags; 5356 5357 hba = shost_priv(sdev->host); 5358 5359 /* Drop the reference as it won't be needed anymore */ 5360 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) { 5361 spin_lock_irqsave(hba->host->host_lock, flags); 5362 hba->ufs_device_wlun = NULL; 5363 spin_unlock_irqrestore(hba->host->host_lock, flags); 5364 } else if (hba->ufs_device_wlun) { 5365 struct device *supplier = NULL; 5366 5367 /* Ensure UFS Device WLUN exists and does not disappear */ 5368 spin_lock_irqsave(hba->host->host_lock, flags); 5369 if (hba->ufs_device_wlun) { 5370 supplier = &hba->ufs_device_wlun->sdev_gendev; 5371 get_device(supplier); 5372 } 5373 spin_unlock_irqrestore(hba->host->host_lock, flags); 5374 5375 if (supplier) { 5376 /* 5377 * If a LUN fails to probe (e.g. absent BOOT WLUN), the 5378 * device will not have been registered but can still 5379 * have a device link holding a reference to the device. 5380 */ 5381 device_link_remove(&sdev->sdev_gendev, supplier); 5382 put_device(supplier); 5383 } 5384 } 5385 } 5386 5387 /** 5388 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status 5389 * @cmd: SCSI command 5390 * @scsi_status: SCSI command status 5391 * 5392 * Return: value base on SCSI command status. 5393 */ 5394 static inline int ufshcd_scsi_cmd_status(struct scsi_cmnd *cmd, int scsi_status) 5395 { 5396 int result = 0; 5397 5398 switch (scsi_status) { 5399 case SAM_STAT_CHECK_CONDITION: 5400 ufshcd_copy_sense_data(cmd); 5401 fallthrough; 5402 case SAM_STAT_GOOD: 5403 result |= DID_OK << 16 | scsi_status; 5404 break; 5405 case SAM_STAT_TASK_SET_FULL: 5406 case SAM_STAT_BUSY: 5407 case SAM_STAT_TASK_ABORTED: 5408 ufshcd_copy_sense_data(cmd); 5409 result |= scsi_status; 5410 break; 5411 default: 5412 result |= DID_ERROR << 16; 5413 break; 5414 } /* end of switch */ 5415 5416 return result; 5417 } 5418 5419 /** 5420 * ufshcd_transfer_rsp_status - Get overall status of the response 5421 * @hba: per adapter instance 5422 * @cmd: SCSI command 5423 * @cqe: pointer to the completion queue entry 5424 * 5425 * Return: result of the command to notify SCSI midlayer. 5426 */ 5427 static inline int ufshcd_transfer_rsp_status(struct ufs_hba *hba, 5428 struct scsi_cmnd *cmd, 5429 struct cq_entry *cqe) 5430 { 5431 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 5432 const int tag = scsi_cmd_to_rq(cmd)->tag; 5433 int result = 0; 5434 int scsi_status; 5435 enum utp_ocs ocs; 5436 u8 upiu_flags; 5437 u32 resid; 5438 5439 upiu_flags = lrbp->ucd_rsp_ptr->header.flags; 5440 resid = be32_to_cpu(lrbp->ucd_rsp_ptr->sr.residual_transfer_count); 5441 /* 5442 * Test !overflow instead of underflow to support UFS devices that do 5443 * not set either flag. 5444 */ 5445 if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW)) 5446 scsi_set_resid(cmd, resid); 5447 5448 /* overall command status of utrd */ 5449 ocs = ufshcd_get_tr_ocs(lrbp, cqe); 5450 5451 if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) { 5452 if (lrbp->ucd_rsp_ptr->header.response || 5453 lrbp->ucd_rsp_ptr->header.status) 5454 ocs = OCS_SUCCESS; 5455 } 5456 5457 switch (ocs) { 5458 case OCS_SUCCESS: 5459 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); 5460 switch (ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr)) { 5461 case UPIU_TRANSACTION_RESPONSE: 5462 /* 5463 * get the result based on SCSI status response 5464 * to notify the SCSI midlayer of the command status 5465 */ 5466 scsi_status = lrbp->ucd_rsp_ptr->header.status; 5467 result = ufshcd_scsi_cmd_status(cmd, scsi_status); 5468 5469 /* 5470 * Currently we are only supporting BKOPs exception 5471 * events hence we can ignore BKOPs exception event 5472 * during power management callbacks. BKOPs exception 5473 * event is not expected to be raised in runtime suspend 5474 * callback as it allows the urgent bkops. 5475 * During system suspend, we are anyway forcefully 5476 * disabling the bkops and if urgent bkops is needed 5477 * it will be enabled on system resume. Long term 5478 * solution could be to abort the system suspend if 5479 * UFS device needs urgent BKOPs. 5480 */ 5481 if (!hba->pm_op_in_progress && 5482 !ufshcd_eh_in_progress(hba) && 5483 ufshcd_is_exception_event(lrbp->ucd_rsp_ptr)) 5484 /* Flushed in suspend */ 5485 schedule_work(&hba->eeh_work); 5486 break; 5487 case UPIU_TRANSACTION_REJECT_UPIU: 5488 /* TODO: handle Reject UPIU Response */ 5489 result = DID_ERROR << 16; 5490 dev_err(hba->dev, 5491 "Reject UPIU not fully implemented\n"); 5492 break; 5493 default: 5494 dev_err(hba->dev, 5495 "Unexpected request response code = %x\n", 5496 result); 5497 result = DID_ERROR << 16; 5498 break; 5499 } 5500 break; 5501 case OCS_ABORTED: 5502 case OCS_INVALID_COMMAND_STATUS: 5503 result |= DID_REQUEUE << 16; 5504 dev_warn(hba->dev, "OCS %s from controller for tag %d\n", 5505 ocs == OCS_ABORTED ? "aborted" : "invalid", tag); 5506 break; 5507 case OCS_INVALID_CMD_TABLE_ATTR: 5508 case OCS_INVALID_PRDT_ATTR: 5509 case OCS_MISMATCH_DATA_BUF_SIZE: 5510 case OCS_MISMATCH_RESP_UPIU_SIZE: 5511 case OCS_PEER_COMM_FAILURE: 5512 case OCS_FATAL_ERROR: 5513 case OCS_DEVICE_FATAL_ERROR: 5514 case OCS_INVALID_CRYPTO_CONFIG: 5515 case OCS_GENERAL_CRYPTO_ERROR: 5516 default: 5517 result |= DID_ERROR << 16; 5518 dev_err(hba->dev, "OCS error from controller = %x for tag %d\n", 5519 ocs, tag); 5520 ufshcd_print_evt_hist(hba); 5521 ufshcd_print_host_state(hba); 5522 break; 5523 } /* end of switch */ 5524 5525 if ((host_byte(result) != DID_OK) && 5526 (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs) { 5527 if (cqe) 5528 ufshcd_hex_dump("UPIU CQE: ", cqe, sizeof(struct cq_entry)); 5529 ufshcd_print_tr(hba, cmd, true); 5530 } 5531 return result; 5532 } 5533 5534 static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba, 5535 u32 intr_mask) 5536 { 5537 if (!ufshcd_is_auto_hibern8_supported(hba) || 5538 !ufshcd_is_auto_hibern8_enabled(hba)) 5539 return false; 5540 5541 if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK)) 5542 return false; 5543 5544 if (hba->active_uic_cmd && 5545 (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER || 5546 hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT)) 5547 return false; 5548 5549 return true; 5550 } 5551 5552 /** 5553 * ufshcd_uic_cmd_compl - handle completion of uic command 5554 * @hba: per adapter instance 5555 * @intr_status: interrupt status generated by the controller 5556 * 5557 * Return: 5558 * IRQ_HANDLED - If interrupt is valid 5559 * IRQ_NONE - If invalid interrupt 5560 */ 5561 static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status) 5562 { 5563 irqreturn_t retval = IRQ_NONE; 5564 struct uic_command *cmd; 5565 5566 guard(spinlock_irqsave)(hba->host->host_lock); 5567 cmd = hba->active_uic_cmd; 5568 if (!cmd) 5569 return retval; 5570 5571 if (ufshcd_is_auto_hibern8_error(hba, intr_status)) 5572 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status); 5573 5574 if (intr_status & UIC_COMMAND_COMPL) { 5575 cmd->argument2 |= ufshcd_get_uic_cmd_result(hba); 5576 cmd->argument3 = ufshcd_get_dme_attr_val(hba); 5577 if (!hba->uic_async_done) 5578 cmd->cmd_active = false; 5579 complete(&cmd->done); 5580 retval = IRQ_HANDLED; 5581 } 5582 5583 if (intr_status & UFSHCD_UIC_PWR_MASK && hba->uic_async_done) { 5584 cmd->cmd_active = false; 5585 complete(hba->uic_async_done); 5586 retval = IRQ_HANDLED; 5587 } 5588 5589 if (retval == IRQ_HANDLED) 5590 ufshcd_add_uic_command_trace(hba, cmd, UFS_CMD_COMP); 5591 5592 return retval; 5593 } 5594 5595 /* Release the resources allocated for processing a SCSI command. */ 5596 void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd) 5597 { 5598 scsi_dma_unmap(cmd); 5599 ufshcd_crypto_clear_prdt(hba, cmd); 5600 ufshcd_release(hba); 5601 ufshcd_clk_scaling_update_busy(hba); 5602 } 5603 5604 /** 5605 * ufshcd_compl_one_cqe - handle a completion queue entry 5606 * @hba: per adapter instance 5607 * @task_tag: the task tag of the request to be completed 5608 * @cqe: pointer to the completion queue entry 5609 */ 5610 void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, 5611 struct cq_entry *cqe) 5612 { 5613 struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag); 5614 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 5615 enum utp_ocs ocs; 5616 5617 if (WARN_ONCE(!cmd, "cqe->command_desc_base_addr = %#llx\n", 5618 le64_to_cpu(cqe->command_desc_base_addr))) 5619 return; 5620 5621 if (hba->monitor.enabled) { 5622 lrbp->compl_time_stamp = ktime_get(); 5623 lrbp->compl_time_stamp_local_clock = local_clock(); 5624 } 5625 if (ufshcd_is_scsi_cmd(cmd)) { 5626 if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) 5627 ufshcd_update_monitor(hba, cmd); 5628 ufshcd_add_command_trace(hba, cmd, UFS_CMD_COMP); 5629 cmd->result = ufshcd_transfer_rsp_status(hba, cmd, cqe); 5630 ufshcd_release_scsi_cmd(hba, cmd); 5631 } else { 5632 if (cqe) { 5633 ocs = cqe->overall_status & MASK_OCS; 5634 lrbp->utr_descriptor_ptr->header.ocs = ocs; 5635 } else { 5636 ocs = lrbp->utr_descriptor_ptr->header.ocs; 5637 } 5638 ufshcd_add_query_upiu_trace( 5639 hba, 5640 ocs == OCS_SUCCESS ? UFS_QUERY_COMP : UFS_QUERY_ERR, 5641 (struct utp_upiu_req *)lrbp->ucd_rsp_ptr); 5642 cmd->result = 0; 5643 } 5644 /* Do not touch lrbp after scsi_done() has been called. */ 5645 scsi_done(cmd); 5646 } 5647 5648 /** 5649 * __ufshcd_transfer_req_compl - handle SCSI and query command completion 5650 * @hba: per adapter instance 5651 * @completed_reqs: bitmask that indicates which requests to complete 5652 */ 5653 static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, 5654 unsigned long completed_reqs) 5655 { 5656 int tag; 5657 5658 for_each_set_bit(tag, &completed_reqs, hba->nutrs) 5659 ufshcd_compl_one_cqe(hba, tag, NULL); 5660 } 5661 5662 /* Any value that is not an existing queue number is fine for this constant. */ 5663 enum { 5664 UFSHCD_POLL_FROM_INTERRUPT_CONTEXT = -1 5665 }; 5666 5667 static void ufshcd_clear_polled(struct ufs_hba *hba, 5668 unsigned long *completed_reqs) 5669 { 5670 int tag; 5671 5672 for_each_set_bit(tag, completed_reqs, hba->nutrs) { 5673 struct scsi_cmnd *cmd = scsi_host_find_tag(hba->host, tag); 5674 5675 if (!cmd) 5676 continue; 5677 if (scsi_cmd_to_rq(cmd)->cmd_flags & REQ_POLLED) 5678 __clear_bit(tag, completed_reqs); 5679 } 5680 } 5681 5682 /* 5683 * Return: > 0 if one or more commands have been completed or 0 if no 5684 * requests have been completed. 5685 */ 5686 static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num) 5687 { 5688 struct ufs_hba *hba = shost_priv(shost); 5689 unsigned long completed_reqs, flags; 5690 u32 tr_doorbell; 5691 struct ufs_hw_queue *hwq; 5692 5693 if (hba->mcq_enabled) { 5694 hwq = &hba->uhq[queue_num]; 5695 5696 return ufshcd_mcq_poll_cqe_lock(hba, hwq); 5697 } 5698 5699 spin_lock_irqsave(&hba->outstanding_lock, flags); 5700 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 5701 completed_reqs = ~tr_doorbell & hba->outstanding_reqs; 5702 WARN_ONCE(completed_reqs & ~hba->outstanding_reqs, 5703 "completed: %#lx; outstanding: %#lx\n", completed_reqs, 5704 hba->outstanding_reqs); 5705 if (queue_num == UFSHCD_POLL_FROM_INTERRUPT_CONTEXT) { 5706 /* Do not complete polled requests from interrupt context. */ 5707 ufshcd_clear_polled(hba, &completed_reqs); 5708 } 5709 hba->outstanding_reqs &= ~completed_reqs; 5710 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 5711 5712 if (completed_reqs) 5713 __ufshcd_transfer_req_compl(hba, completed_reqs); 5714 5715 return completed_reqs != 0; 5716 } 5717 5718 static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv) 5719 { 5720 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); 5721 struct scsi_device *sdev = rq->q->queuedata; 5722 struct Scsi_Host *shost = sdev->host; 5723 struct ufs_hba *hba = shost_priv(shost); 5724 struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); 5725 5726 if (blk_mq_is_reserved_rq(rq) || !hwq) 5727 return true; 5728 5729 ufshcd_mcq_compl_all_cqes_lock(hba, hwq); 5730 5731 /* 5732 * For those cmds of which the cqes are not present in the cq, complete 5733 * them explicitly. 5734 */ 5735 scoped_guard(spinlock_irqsave, &hwq->cq_lock) { 5736 if (!test_bit(SCMD_STATE_COMPLETE, &cmd->state)) { 5737 set_host_byte(cmd, DID_REQUEUE); 5738 ufshcd_release_scsi_cmd(hba, cmd); 5739 scsi_done(cmd); 5740 } 5741 } 5742 5743 return true; 5744 } 5745 5746 static bool ufshcd_mcq_compl_one(struct request *rq, void *priv) 5747 { 5748 struct scsi_device *sdev = rq->q->queuedata; 5749 struct Scsi_Host *shost = sdev->host; 5750 struct ufs_hba *hba = shost_priv(shost); 5751 struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); 5752 5753 if (!blk_mq_is_reserved_rq(rq) && hwq) 5754 ufshcd_mcq_poll_cqe_lock(hba, hwq); 5755 5756 return true; 5757 } 5758 5759 /** 5760 * ufshcd_mcq_compl_pending_transfer - MCQ mode function. It is 5761 * invoked from the error handler context or ufshcd_host_reset_and_restore() 5762 * to complete the pending transfers and free the resources associated with 5763 * the scsi command. 5764 * 5765 * @hba: per adapter instance 5766 * @force_compl: This flag is set to true when invoked 5767 * from ufshcd_host_reset_and_restore() in which case it requires special 5768 * handling because the host controller has been reset by ufshcd_hba_stop(). 5769 */ 5770 static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba, 5771 bool force_compl) 5772 { 5773 blk_mq_tagset_busy_iter(&hba->host->tag_set, 5774 force_compl ? ufshcd_mcq_force_compl_one : 5775 ufshcd_mcq_compl_one, 5776 NULL); 5777 } 5778 5779 /** 5780 * ufshcd_transfer_req_compl - handle SCSI and query command completion 5781 * @hba: per adapter instance 5782 * 5783 * Return: 5784 * IRQ_HANDLED - If interrupt is valid 5785 * IRQ_NONE - If invalid interrupt 5786 */ 5787 static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba) 5788 { 5789 /* Resetting interrupt aggregation counters first and reading the 5790 * DOOR_BELL afterward allows us to handle all the completed requests. 5791 * In order to prevent other interrupts starvation the DB is read once 5792 * after reset. The down side of this solution is the possibility of 5793 * false interrupt if device completes another request after resetting 5794 * aggregation and before reading the DB. 5795 */ 5796 if (ufshcd_is_intr_aggr_allowed(hba) && 5797 !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR)) 5798 ufshcd_reset_intr_aggr(hba); 5799 5800 if (ufs_fail_completion(hba)) 5801 return IRQ_HANDLED; 5802 5803 /* 5804 * Ignore the ufshcd_poll() return value and return IRQ_HANDLED since we 5805 * do not want polling to trigger spurious interrupt complaints. 5806 */ 5807 ufshcd_poll(hba->host, UFSHCD_POLL_FROM_INTERRUPT_CONTEXT); 5808 5809 return IRQ_HANDLED; 5810 } 5811 5812 int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask) 5813 { 5814 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 5815 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, 5816 &ee_ctrl_mask); 5817 } 5818 5819 int ufshcd_write_ee_control(struct ufs_hba *hba) 5820 { 5821 int err; 5822 5823 mutex_lock(&hba->ee_ctrl_mutex); 5824 err = __ufshcd_write_ee_control(hba, hba->ee_ctrl_mask); 5825 mutex_unlock(&hba->ee_ctrl_mutex); 5826 if (err) 5827 dev_err(hba->dev, "%s: failed to write ee control %d\n", 5828 __func__, err); 5829 return err; 5830 } 5831 5832 int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask, 5833 const u16 *other_mask, u16 set, u16 clr) 5834 { 5835 u16 new_mask, ee_ctrl_mask; 5836 int err = 0; 5837 5838 mutex_lock(&hba->ee_ctrl_mutex); 5839 new_mask = (*mask & ~clr) | set; 5840 ee_ctrl_mask = new_mask | *other_mask; 5841 if (ee_ctrl_mask != hba->ee_ctrl_mask) 5842 err = __ufshcd_write_ee_control(hba, ee_ctrl_mask); 5843 /* Still need to update 'mask' even if 'ee_ctrl_mask' was unchanged */ 5844 if (!err) { 5845 hba->ee_ctrl_mask = ee_ctrl_mask; 5846 *mask = new_mask; 5847 } 5848 mutex_unlock(&hba->ee_ctrl_mutex); 5849 return err; 5850 } 5851 5852 /** 5853 * ufshcd_disable_ee - disable exception event 5854 * @hba: per-adapter instance 5855 * @mask: exception event to disable 5856 * 5857 * Disables exception event in the device so that the EVENT_ALERT 5858 * bit is not set. 5859 * 5860 * Return: zero on success, non-zero error value on failure. 5861 */ 5862 static inline int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask) 5863 { 5864 return ufshcd_update_ee_drv_mask(hba, 0, mask); 5865 } 5866 5867 /** 5868 * ufshcd_enable_ee - enable exception event 5869 * @hba: per-adapter instance 5870 * @mask: exception event to enable 5871 * 5872 * Enable corresponding exception event in the device to allow 5873 * device to alert host in critical scenarios. 5874 * 5875 * Return: zero on success, non-zero error value on failure. 5876 */ 5877 static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask) 5878 { 5879 return ufshcd_update_ee_drv_mask(hba, mask, 0); 5880 } 5881 5882 /** 5883 * ufshcd_enable_auto_bkops - Allow device managed BKOPS 5884 * @hba: per-adapter instance 5885 * 5886 * Allow device to manage background operations on its own. Enabling 5887 * this might lead to inconsistent latencies during normal data transfers 5888 * as the device is allowed to manage its own way of handling background 5889 * operations. 5890 * 5891 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 5892 * < 0 if another error occurred. 5893 */ 5894 static int ufshcd_enable_auto_bkops(struct ufs_hba *hba) 5895 { 5896 int err = 0; 5897 5898 if (hba->auto_bkops_enabled) 5899 goto out; 5900 5901 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG, 5902 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL); 5903 if (err) { 5904 dev_err(hba->dev, "%s: failed to enable bkops %d\n", 5905 __func__, err); 5906 goto out; 5907 } 5908 5909 hba->auto_bkops_enabled = true; 5910 trace_ufshcd_auto_bkops_state(hba, "Enabled"); 5911 5912 /* No need of URGENT_BKOPS exception from the device */ 5913 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); 5914 if (err) 5915 dev_err(hba->dev, "%s: failed to disable exception event %d\n", 5916 __func__, err); 5917 out: 5918 return err; 5919 } 5920 5921 /** 5922 * ufshcd_disable_auto_bkops - block device in doing background operations 5923 * @hba: per-adapter instance 5924 * 5925 * Disabling background operations improves command response latency but 5926 * has drawback of device moving into critical state where the device is 5927 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the 5928 * host is idle so that BKOPS are managed effectively without any negative 5929 * impacts. 5930 * 5931 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 5932 * < 0 if another error occurred. 5933 */ 5934 static int ufshcd_disable_auto_bkops(struct ufs_hba *hba) 5935 { 5936 int err = 0; 5937 5938 if (!hba->auto_bkops_enabled) 5939 goto out; 5940 5941 /* 5942 * If host assisted BKOPs is to be enabled, make sure 5943 * urgent bkops exception is allowed. 5944 */ 5945 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS); 5946 if (err) { 5947 dev_err(hba->dev, "%s: failed to enable exception event %d\n", 5948 __func__, err); 5949 goto out; 5950 } 5951 5952 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG, 5953 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL); 5954 if (err) { 5955 dev_err(hba->dev, "%s: failed to disable bkops %d\n", 5956 __func__, err); 5957 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); 5958 goto out; 5959 } 5960 5961 hba->auto_bkops_enabled = false; 5962 trace_ufshcd_auto_bkops_state(hba, "Disabled"); 5963 hba->is_urgent_bkops_lvl_checked = false; 5964 out: 5965 return err; 5966 } 5967 5968 /** 5969 * ufshcd_force_reset_auto_bkops - force reset auto bkops state 5970 * @hba: per adapter instance 5971 * 5972 * After a device reset the device may toggle the BKOPS_EN flag 5973 * to default value. The s/w tracking variables should be updated 5974 * as well. This function would change the auto-bkops state based on 5975 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND. 5976 */ 5977 static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba) 5978 { 5979 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) { 5980 hba->auto_bkops_enabled = false; 5981 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS; 5982 ufshcd_enable_auto_bkops(hba); 5983 } else { 5984 hba->auto_bkops_enabled = true; 5985 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS; 5986 ufshcd_disable_auto_bkops(hba); 5987 } 5988 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT; 5989 hba->is_urgent_bkops_lvl_checked = false; 5990 } 5991 5992 static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status) 5993 { 5994 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 5995 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status); 5996 } 5997 5998 /** 5999 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status 6000 * @hba: per-adapter instance 6001 * 6002 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn 6003 * flag in the device to permit background operations if the device 6004 * bkops_status is greater than or equal to the "hba->urgent_bkops_lvl", 6005 * disable otherwise. 6006 * 6007 * Return: 0 for success, non-zero in case of failure. 6008 * 6009 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag 6010 * to know whether auto bkops is enabled or disabled after this function 6011 * returns control to it. 6012 */ 6013 static int ufshcd_bkops_ctrl(struct ufs_hba *hba) 6014 { 6015 enum bkops_status status = hba->urgent_bkops_lvl; 6016 u32 curr_status = 0; 6017 int err; 6018 6019 err = ufshcd_get_bkops_status(hba, &curr_status); 6020 if (err) { 6021 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n", 6022 __func__, err); 6023 goto out; 6024 } else if (curr_status > BKOPS_STATUS_MAX) { 6025 dev_err(hba->dev, "%s: invalid BKOPS status %d\n", 6026 __func__, curr_status); 6027 err = -EINVAL; 6028 goto out; 6029 } 6030 6031 if (curr_status >= status) 6032 err = ufshcd_enable_auto_bkops(hba); 6033 else 6034 err = ufshcd_disable_auto_bkops(hba); 6035 out: 6036 return err; 6037 } 6038 6039 static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status) 6040 { 6041 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 6042 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status); 6043 } 6044 6045 static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba) 6046 { 6047 int err; 6048 u32 curr_status = 0; 6049 6050 if (hba->is_urgent_bkops_lvl_checked) 6051 goto enable_auto_bkops; 6052 6053 err = ufshcd_get_bkops_status(hba, &curr_status); 6054 if (err) { 6055 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n", 6056 __func__, err); 6057 goto out; 6058 } 6059 6060 /* 6061 * We are seeing that some devices are raising the urgent bkops 6062 * exception events even when BKOPS status doesn't indicate performace 6063 * impacted or critical. Handle these device by determining their urgent 6064 * bkops status at runtime. 6065 */ 6066 if (curr_status < BKOPS_STATUS_PERF_IMPACT) { 6067 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n", 6068 __func__, curr_status); 6069 /* update the current status as the urgent bkops level */ 6070 hba->urgent_bkops_lvl = curr_status; 6071 hba->is_urgent_bkops_lvl_checked = true; 6072 } 6073 6074 enable_auto_bkops: 6075 err = ufshcd_enable_auto_bkops(hba); 6076 out: 6077 if (err < 0) 6078 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n", 6079 __func__, err); 6080 } 6081 6082 /* 6083 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 6084 * < 0 if another error occurred. 6085 */ 6086 int ufshcd_read_device_lvl_exception_id(struct ufs_hba *hba, u64 *exception_id) 6087 { 6088 struct utp_upiu_query_v4_0 *upiu_resp; 6089 struct ufs_query_req *request = NULL; 6090 struct ufs_query_res *response = NULL; 6091 int err; 6092 6093 if (hba->dev_info.wspecversion < 0x410) 6094 return -EOPNOTSUPP; 6095 6096 ufshcd_hold(hba); 6097 mutex_lock(&hba->dev_cmd.lock); 6098 6099 ufshcd_init_query(hba, &request, &response, 6100 UPIU_QUERY_OPCODE_READ_ATTR, 6101 QUERY_ATTR_IDN_DEV_LVL_EXCEPTION_ID, 0, 0); 6102 6103 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 6104 6105 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); 6106 6107 if (err) { 6108 dev_err(hba->dev, "%s: failed to read device level exception %d\n", 6109 __func__, err); 6110 goto out; 6111 } 6112 6113 upiu_resp = (struct utp_upiu_query_v4_0 *)response; 6114 *exception_id = get_unaligned_be64(&upiu_resp->osf3); 6115 out: 6116 mutex_unlock(&hba->dev_cmd.lock); 6117 ufshcd_release(hba); 6118 6119 return err; 6120 } 6121 6122 static int __ufshcd_wb_toggle(struct ufs_hba *hba, bool set, enum flag_idn idn) 6123 { 6124 u8 index; 6125 enum query_opcode opcode = set ? UPIU_QUERY_OPCODE_SET_FLAG : 6126 UPIU_QUERY_OPCODE_CLEAR_FLAG; 6127 6128 index = ufshcd_wb_get_query_index(hba); 6129 return ufshcd_query_flag_retry(hba, opcode, idn, index, NULL); 6130 } 6131 6132 int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable) 6133 { 6134 int ret; 6135 6136 if (!ufshcd_is_wb_allowed(hba) || 6137 hba->dev_info.wb_enabled == enable) 6138 return 0; 6139 6140 ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_EN); 6141 if (ret) { 6142 dev_err(hba->dev, "%s: Write Booster %s failed %d\n", 6143 __func__, enable ? "enabling" : "disabling", ret); 6144 return ret; 6145 } 6146 6147 hba->dev_info.wb_enabled = enable; 6148 dev_dbg(hba->dev, "%s: Write Booster %s\n", 6149 __func__, enable ? "enabled" : "disabled"); 6150 6151 return ret; 6152 } 6153 6154 static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba, 6155 bool enable) 6156 { 6157 int ret; 6158 6159 ret = __ufshcd_wb_toggle(hba, enable, 6160 QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8); 6161 if (ret) { 6162 dev_err(hba->dev, "%s: WB-Buf Flush during H8 %s failed %d\n", 6163 __func__, enable ? "enabling" : "disabling", ret); 6164 return; 6165 } 6166 dev_dbg(hba->dev, "%s: WB-Buf Flush during H8 %s\n", 6167 __func__, enable ? "enabled" : "disabled"); 6168 } 6169 6170 int ufshcd_wb_toggle_buf_flush(struct ufs_hba *hba, bool enable) 6171 { 6172 int ret; 6173 6174 if (!ufshcd_is_wb_allowed(hba) || 6175 hba->dev_info.wb_buf_flush_enabled == enable) 6176 return 0; 6177 6178 ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN); 6179 if (ret) { 6180 dev_err(hba->dev, "%s: WB-Buf Flush %s failed %d\n", 6181 __func__, enable ? "enabling" : "disabling", ret); 6182 return ret; 6183 } 6184 6185 hba->dev_info.wb_buf_flush_enabled = enable; 6186 dev_dbg(hba->dev, "%s: WB-Buf Flush %s\n", 6187 __func__, enable ? "enabled" : "disabled"); 6188 6189 return ret; 6190 } 6191 6192 int ufshcd_wb_set_resize_en(struct ufs_hba *hba, enum wb_resize_en en_mode) 6193 { 6194 int ret; 6195 u8 index; 6196 6197 index = ufshcd_wb_get_query_index(hba); 6198 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 6199 QUERY_ATTR_IDN_WB_BUF_RESIZE_EN, index, 0, &en_mode); 6200 if (ret) 6201 dev_err(hba->dev, "%s: Enable WB buf resize operation failed %d\n", 6202 __func__, ret); 6203 6204 return ret; 6205 } 6206 6207 static bool ufshcd_wb_curr_buff_threshold_check(struct ufs_hba *hba, 6208 u32 avail_buf) 6209 { 6210 u32 cur_buf; 6211 int ret; 6212 u8 index; 6213 6214 index = ufshcd_wb_get_query_index(hba); 6215 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 6216 QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE, 6217 index, 0, &cur_buf); 6218 if (ret) { 6219 dev_err(hba->dev, "%s: dCurWriteBoosterBufferSize read failed %d\n", 6220 __func__, ret); 6221 return false; 6222 } 6223 6224 if (!cur_buf) { 6225 dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n", 6226 cur_buf); 6227 return false; 6228 } 6229 /* Let it continue to flush when available buffer exceeds threshold */ 6230 return avail_buf < hba->vps->wb_flush_threshold; 6231 } 6232 6233 static void ufshcd_wb_force_disable(struct ufs_hba *hba) 6234 { 6235 if (ufshcd_is_wb_buf_flush_allowed(hba)) 6236 ufshcd_wb_toggle_buf_flush(hba, false); 6237 6238 ufshcd_wb_toggle_buf_flush_during_h8(hba, false); 6239 ufshcd_wb_toggle(hba, false); 6240 hba->caps &= ~UFSHCD_CAP_WB_EN; 6241 6242 dev_info(hba->dev, "%s: WB force disabled\n", __func__); 6243 } 6244 6245 static bool ufshcd_is_wb_buf_lifetime_available(struct ufs_hba *hba) 6246 { 6247 u32 lifetime; 6248 int ret; 6249 u8 index; 6250 6251 index = ufshcd_wb_get_query_index(hba); 6252 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 6253 QUERY_ATTR_IDN_WB_BUFF_LIFE_TIME_EST, 6254 index, 0, &lifetime); 6255 if (ret) { 6256 dev_err(hba->dev, 6257 "%s: bWriteBoosterBufferLifeTimeEst read failed %d\n", 6258 __func__, ret); 6259 return false; 6260 } 6261 6262 if (lifetime == UFS_WB_EXCEED_LIFETIME) { 6263 dev_err(hba->dev, "%s: WB buf lifetime is exhausted 0x%02X\n", 6264 __func__, lifetime); 6265 return false; 6266 } 6267 6268 dev_dbg(hba->dev, "%s: WB buf lifetime is 0x%02X\n", 6269 __func__, lifetime); 6270 6271 return true; 6272 } 6273 6274 static bool ufshcd_wb_need_flush(struct ufs_hba *hba) 6275 { 6276 int ret; 6277 u32 avail_buf; 6278 u8 index; 6279 6280 if (!ufshcd_is_wb_allowed(hba)) 6281 return false; 6282 6283 if (!ufshcd_is_wb_buf_lifetime_available(hba)) { 6284 ufshcd_wb_force_disable(hba); 6285 return false; 6286 } 6287 6288 /* 6289 * With user-space reduction enabled, it's enough to enable flush 6290 * by checking only the available buffer. The threshold 6291 * defined here is > 90% full. 6292 * With user-space preserved enabled, the current-buffer 6293 * should be checked too because the wb buffer size can reduce 6294 * when disk tends to be full. This info is provided by current 6295 * buffer (dCurrentWriteBoosterBufferSize). 6296 */ 6297 index = ufshcd_wb_get_query_index(hba); 6298 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 6299 QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE, 6300 index, 0, &avail_buf); 6301 if (ret) { 6302 dev_warn(hba->dev, "%s: dAvailableWriteBoosterBufferSize read failed %d\n", 6303 __func__, ret); 6304 return false; 6305 } 6306 6307 if (!hba->dev_info.b_presrv_uspc_en) 6308 return avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10); 6309 6310 return ufshcd_wb_curr_buff_threshold_check(hba, avail_buf); 6311 } 6312 6313 static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work) 6314 { 6315 struct ufs_hba *hba = container_of(to_delayed_work(work), 6316 struct ufs_hba, 6317 rpm_dev_flush_recheck_work); 6318 /* 6319 * To prevent unnecessary VCC power drain after device finishes 6320 * WriteBooster buffer flush or Auto BKOPs, force runtime resume 6321 * after a certain delay to recheck the threshold by next runtime 6322 * suspend. 6323 */ 6324 ufshcd_rpm_get_sync(hba); 6325 ufshcd_rpm_put_sync(hba); 6326 } 6327 6328 /** 6329 * ufshcd_exception_event_handler - handle exceptions raised by device 6330 * @work: pointer to work data 6331 * 6332 * Read bExceptionEventStatus attribute from the device and handle the 6333 * exception event accordingly. 6334 */ 6335 static void ufshcd_exception_event_handler(struct work_struct *work) 6336 { 6337 struct ufs_hba *hba; 6338 int err; 6339 u32 status = 0; 6340 hba = container_of(work, struct ufs_hba, eeh_work); 6341 6342 err = ufshcd_get_ee_status(hba, &status); 6343 if (err) { 6344 dev_err(hba->dev, "%s: failed to get exception status %d\n", 6345 __func__, err); 6346 return; 6347 } 6348 6349 trace_ufshcd_exception_event(hba, status); 6350 6351 if (status & hba->ee_drv_mask & MASK_EE_URGENT_BKOPS) 6352 ufshcd_bkops_exception_event_handler(hba); 6353 6354 if (status & hba->ee_drv_mask & MASK_EE_URGENT_TEMP) 6355 ufs_hwmon_notify_event(hba, status & MASK_EE_URGENT_TEMP); 6356 6357 if (status & hba->ee_drv_mask & MASK_EE_HEALTH_CRITICAL) { 6358 hba->critical_health_count++; 6359 sysfs_notify(&hba->dev->kobj, NULL, "critical_health"); 6360 } 6361 6362 if (status & hba->ee_drv_mask & MASK_EE_DEV_LVL_EXCEPTION) { 6363 atomic_inc(&hba->dev_lvl_exception_count); 6364 sysfs_notify(&hba->dev->kobj, NULL, "device_lvl_exception_count"); 6365 } 6366 6367 ufs_debugfs_exception_event(hba, status); 6368 } 6369 6370 /* Complete requests that have door-bell cleared */ 6371 static void ufshcd_complete_requests(struct ufs_hba *hba, bool force_compl) 6372 { 6373 if (hba->mcq_enabled) 6374 ufshcd_mcq_compl_pending_transfer(hba, force_compl); 6375 else 6376 ufshcd_transfer_req_compl(hba); 6377 6378 ufshcd_tmc_handler(hba); 6379 } 6380 6381 /** 6382 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is 6383 * to recover from the DL NAC errors or not. 6384 * @hba: per-adapter instance 6385 * 6386 * Return: true if error handling is required, false otherwise. 6387 */ 6388 static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba) 6389 { 6390 unsigned long flags; 6391 bool err_handling = true; 6392 6393 spin_lock_irqsave(hba->host->host_lock, flags); 6394 /* 6395 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the 6396 * device fatal error and/or DL NAC & REPLAY timeout errors. 6397 */ 6398 if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR)) 6399 goto out; 6400 6401 if ((hba->saved_err & DEVICE_FATAL_ERROR) || 6402 ((hba->saved_err & UIC_ERROR) && 6403 (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR))) 6404 goto out; 6405 6406 if ((hba->saved_err & UIC_ERROR) && 6407 (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) { 6408 int err; 6409 /* 6410 * wait for 50ms to see if we can get any other errors or not. 6411 */ 6412 spin_unlock_irqrestore(hba->host->host_lock, flags); 6413 msleep(50); 6414 spin_lock_irqsave(hba->host->host_lock, flags); 6415 6416 /* 6417 * now check if we have got any other severe errors other than 6418 * DL NAC error? 6419 */ 6420 if ((hba->saved_err & INT_FATAL_ERRORS) || 6421 ((hba->saved_err & UIC_ERROR) && 6422 (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR))) 6423 goto out; 6424 6425 /* 6426 * As DL NAC is the only error received so far, send out NOP 6427 * command to confirm if link is still active or not. 6428 * - If we don't get any response then do error recovery. 6429 * - If we get response then clear the DL NAC error bit. 6430 */ 6431 6432 spin_unlock_irqrestore(hba->host->host_lock, flags); 6433 err = ufshcd_verify_dev_init(hba); 6434 spin_lock_irqsave(hba->host->host_lock, flags); 6435 6436 if (err) 6437 goto out; 6438 6439 /* Link seems to be alive hence ignore the DL NAC errors */ 6440 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR) 6441 hba->saved_err &= ~UIC_ERROR; 6442 /* clear NAC error */ 6443 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR; 6444 if (!hba->saved_uic_err) 6445 err_handling = false; 6446 } 6447 out: 6448 spin_unlock_irqrestore(hba->host->host_lock, flags); 6449 return err_handling; 6450 } 6451 6452 /* host lock must be held before calling this func */ 6453 static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba) 6454 { 6455 return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) || 6456 (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)); 6457 } 6458 6459 void ufshcd_schedule_eh_work(struct ufs_hba *hba) 6460 { 6461 lockdep_assert_held(hba->host->host_lock); 6462 6463 /* handle fatal errors only when link is not in error state */ 6464 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) { 6465 if (hba->force_reset || ufshcd_is_link_broken(hba) || 6466 ufshcd_is_saved_err_fatal(hba)) 6467 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL; 6468 else 6469 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL; 6470 queue_work(hba->eh_wq, &hba->eh_work); 6471 } 6472 } 6473 6474 void ufshcd_force_error_recovery(struct ufs_hba *hba) 6475 { 6476 spin_lock_irq(hba->host->host_lock); 6477 hba->force_reset = true; 6478 ufshcd_schedule_eh_work(hba); 6479 spin_unlock_irq(hba->host->host_lock); 6480 } 6481 EXPORT_SYMBOL_GPL(ufshcd_force_error_recovery); 6482 6483 static void ufshcd_clk_scaling_allow(struct ufs_hba *hba, bool allow) 6484 { 6485 mutex_lock(&hba->wb_mutex); 6486 down_write(&hba->clk_scaling_lock); 6487 hba->clk_scaling.is_allowed = allow; 6488 up_write(&hba->clk_scaling_lock); 6489 mutex_unlock(&hba->wb_mutex); 6490 } 6491 6492 static void ufshcd_clk_scaling_suspend(struct ufs_hba *hba, bool suspend) 6493 { 6494 if (suspend) { 6495 if (hba->clk_scaling.is_enabled) 6496 ufshcd_suspend_clkscaling(hba); 6497 ufshcd_clk_scaling_allow(hba, false); 6498 } else { 6499 ufshcd_clk_scaling_allow(hba, true); 6500 if (hba->clk_scaling.is_enabled) 6501 ufshcd_resume_clkscaling(hba); 6502 } 6503 } 6504 6505 static void ufshcd_err_handling_prepare(struct ufs_hba *hba) 6506 { 6507 /* 6508 * A WLUN resume failure could potentially lead to the HBA being 6509 * runtime suspended, so take an extra reference on hba->dev. 6510 */ 6511 pm_runtime_get_sync(hba->dev); 6512 ufshcd_rpm_get_sync(hba); 6513 if (pm_runtime_status_suspended(&hba->ufs_device_wlun->sdev_gendev) || 6514 hba->is_sys_suspended) { 6515 enum ufs_pm_op pm_op; 6516 6517 /* 6518 * Don't assume anything of resume, if 6519 * resume fails, irq and clocks can be OFF, and powers 6520 * can be OFF or in LPM. 6521 */ 6522 ufshcd_setup_hba_vreg(hba, true); 6523 ufshcd_enable_irq(hba); 6524 ufshcd_setup_vreg(hba, true); 6525 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq); 6526 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2); 6527 ufshcd_hold(hba); 6528 if (!ufshcd_is_clkgating_allowed(hba)) 6529 ufshcd_setup_clocks(hba, true); 6530 pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM; 6531 ufshcd_vops_resume(hba, pm_op); 6532 } else { 6533 ufshcd_hold(hba); 6534 if (ufshcd_is_clkscaling_supported(hba) && 6535 hba->clk_scaling.is_enabled) 6536 ufshcd_suspend_clkscaling(hba); 6537 ufshcd_clk_scaling_allow(hba, false); 6538 } 6539 /* Wait for ongoing ufshcd_queuecommand() calls to finish. */ 6540 blk_mq_quiesce_tagset(&hba->host->tag_set); 6541 cancel_work_sync(&hba->eeh_work); 6542 } 6543 6544 static void ufshcd_err_handling_unprepare(struct ufs_hba *hba) 6545 { 6546 blk_mq_unquiesce_tagset(&hba->host->tag_set); 6547 ufshcd_release(hba); 6548 if (ufshcd_is_clkscaling_supported(hba)) 6549 ufshcd_clk_scaling_suspend(hba, false); 6550 ufshcd_rpm_put(hba); 6551 pm_runtime_put(hba->dev); 6552 } 6553 6554 static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba) 6555 { 6556 return (!hba->is_powered || hba->shutting_down || 6557 !hba->ufs_device_wlun || 6558 hba->ufshcd_state == UFSHCD_STATE_ERROR || 6559 (!(hba->saved_err || hba->saved_uic_err || hba->force_reset || 6560 ufshcd_is_link_broken(hba)))); 6561 } 6562 6563 #ifdef CONFIG_PM 6564 static void ufshcd_recover_pm_error(struct ufs_hba *hba) 6565 { 6566 struct scsi_target *starget = hba->ufs_device_wlun->sdev_target; 6567 struct Scsi_Host *shost = hba->host; 6568 struct scsi_device *sdev; 6569 struct request_queue *q; 6570 bool resume_sdev_queues = false; 6571 6572 hba->is_sys_suspended = false; 6573 6574 /* 6575 * Ensure the parent's error status is cleared before proceeding 6576 * to the child, as the parent must be active to activate the child. 6577 */ 6578 if (hba->dev->power.runtime_error) { 6579 /* hba->dev has no functional parent thus simplily set RPM_ACTIVE */ 6580 pm_runtime_set_active(hba->dev); 6581 resume_sdev_queues = true; 6582 } 6583 6584 if (hba->ufs_device_wlun->sdev_gendev.power.runtime_error) { 6585 /* 6586 * starget, parent of wlun, might be suspended if wlun resume failed. 6587 * Make sure parent is resumed before set child (wlun) active. 6588 */ 6589 pm_runtime_get_sync(&starget->dev); 6590 pm_runtime_set_active(&hba->ufs_device_wlun->sdev_gendev); 6591 pm_runtime_put_sync(&starget->dev); 6592 resume_sdev_queues = true; 6593 } 6594 6595 /* 6596 * If wlun device had runtime error, we also need to resume those 6597 * consumer scsi devices in case any of them has failed to be 6598 * resumed due to supplier runtime resume failure. This is to unblock 6599 * blk_queue_enter in case there are bios waiting inside it. 6600 */ 6601 if (resume_sdev_queues) { 6602 shost_for_each_device(sdev, shost) { 6603 q = sdev->request_queue; 6604 if (q->dev && (q->rpm_status == RPM_SUSPENDED || 6605 q->rpm_status == RPM_SUSPENDING)) 6606 pm_request_resume(q->dev); 6607 } 6608 } 6609 } 6610 #else 6611 static inline void ufshcd_recover_pm_error(struct ufs_hba *hba) 6612 { 6613 } 6614 #endif 6615 6616 static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba) 6617 { 6618 struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info; 6619 u32 mode; 6620 6621 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode); 6622 6623 if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK)) 6624 return true; 6625 6626 if (pwr_info->pwr_tx != (mode & PWRMODE_MASK)) 6627 return true; 6628 6629 return false; 6630 } 6631 6632 static bool ufshcd_abort_one(struct request *rq, void *priv) 6633 { 6634 int *ret = priv; 6635 u32 tag = rq->tag; 6636 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); 6637 struct scsi_device *sdev = cmd->device; 6638 struct Scsi_Host *shost = sdev->host; 6639 struct ufs_hba *hba = shost_priv(shost); 6640 6641 if (blk_mq_is_reserved_rq(rq)) 6642 return true; 6643 6644 *ret = ufshcd_try_to_abort_task(hba, tag); 6645 dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag, 6646 ufshcd_is_scsi_cmd(cmd) ? cmd->cmnd[0] : -1, 6647 *ret ? "failed" : "succeeded"); 6648 6649 return *ret == 0; 6650 } 6651 6652 /** 6653 * ufshcd_abort_all - Abort all pending commands. 6654 * @hba: Host bus adapter pointer. 6655 * 6656 * Return: true if and only if the host controller needs to be reset. 6657 */ 6658 static bool ufshcd_abort_all(struct ufs_hba *hba) 6659 { 6660 int tag, ret = 0; 6661 6662 blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_abort_one, &ret); 6663 if (ret) 6664 goto out; 6665 6666 /* Clear pending task management requests */ 6667 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) { 6668 ret = ufshcd_clear_tm_cmd(hba, tag); 6669 if (ret) 6670 goto out; 6671 } 6672 6673 out: 6674 /* Complete the requests that are cleared by s/w */ 6675 ufshcd_complete_requests(hba, false); 6676 6677 return ret != 0; 6678 } 6679 6680 /** 6681 * ufshcd_err_handler - handle UFS errors that require s/w attention 6682 * @work: pointer to work structure 6683 */ 6684 static void ufshcd_err_handler(struct work_struct *work) 6685 { 6686 int retries = MAX_ERR_HANDLER_RETRIES; 6687 struct ufs_hba *hba; 6688 unsigned long flags; 6689 bool needs_restore; 6690 bool needs_reset; 6691 int pmc_err; 6692 6693 hba = container_of(work, struct ufs_hba, eh_work); 6694 6695 dev_info(hba->dev, 6696 "%s started; HBA state %s; powered %d; shutting down %d; saved_err = 0x%x; saved_uic_err = 0x%x; force_reset = %d%s\n", 6697 __func__, ufshcd_state_name[hba->ufshcd_state], 6698 hba->is_powered, hba->shutting_down, hba->saved_err, 6699 hba->saved_uic_err, hba->force_reset, 6700 ufshcd_is_link_broken(hba) ? "; link is broken" : ""); 6701 6702 if (hba->ufs_device_wlun) { 6703 /* 6704 * Use ufshcd_rpm_get_noresume() here to safely perform link 6705 * recovery even if an error occurs during runtime suspend or 6706 * runtime resume. This avoids potential deadlocks that could 6707 * happen if we tried to resume the device while a PM operation 6708 * is already in progress. 6709 */ 6710 ufshcd_rpm_get_noresume(hba); 6711 if (hba->pm_op_in_progress) { 6712 ufshcd_link_recovery(hba); 6713 ufshcd_rpm_put(hba); 6714 return; 6715 } 6716 ufshcd_rpm_put(hba); 6717 } 6718 6719 down(&hba->host_sem); 6720 spin_lock_irqsave(hba->host->host_lock, flags); 6721 if (ufshcd_err_handling_should_stop(hba)) { 6722 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) 6723 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; 6724 spin_unlock_irqrestore(hba->host->host_lock, flags); 6725 up(&hba->host_sem); 6726 return; 6727 } 6728 spin_unlock_irqrestore(hba->host->host_lock, flags); 6729 6730 ufshcd_err_handling_prepare(hba); 6731 6732 spin_lock_irqsave(hba->host->host_lock, flags); 6733 ufshcd_set_eh_in_progress(hba); 6734 spin_unlock_irqrestore(hba->host->host_lock, flags); 6735 6736 /* Complete requests that have door-bell cleared by h/w */ 6737 ufshcd_complete_requests(hba, false); 6738 spin_lock_irqsave(hba->host->host_lock, flags); 6739 again: 6740 needs_restore = false; 6741 needs_reset = false; 6742 6743 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) 6744 hba->ufshcd_state = UFSHCD_STATE_RESET; 6745 /* 6746 * A full reset and restore might have happened after preparation 6747 * is finished, double check whether we should stop. 6748 */ 6749 if (ufshcd_err_handling_should_stop(hba)) 6750 goto skip_err_handling; 6751 6752 if ((hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) && 6753 !hba->force_reset) { 6754 bool ret; 6755 6756 spin_unlock_irqrestore(hba->host->host_lock, flags); 6757 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */ 6758 ret = ufshcd_quirk_dl_nac_errors(hba); 6759 spin_lock_irqsave(hba->host->host_lock, flags); 6760 if (!ret && ufshcd_err_handling_should_stop(hba)) 6761 goto skip_err_handling; 6762 } 6763 6764 if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) || 6765 (hba->saved_uic_err && 6766 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) { 6767 bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR); 6768 6769 spin_unlock_irqrestore(hba->host->host_lock, flags); 6770 ufshcd_print_host_state(hba); 6771 ufshcd_print_pwr_info(hba); 6772 ufshcd_print_evt_hist(hba); 6773 ufshcd_print_tmrs(hba, hba->outstanding_tasks); 6774 ufshcd_print_trs_all(hba, pr_prdt); 6775 spin_lock_irqsave(hba->host->host_lock, flags); 6776 } 6777 6778 /* 6779 * if host reset is required then skip clearing the pending 6780 * transfers forcefully because they will get cleared during 6781 * host reset and restore 6782 */ 6783 if (hba->force_reset || ufshcd_is_link_broken(hba) || 6784 ufshcd_is_saved_err_fatal(hba) || 6785 ((hba->saved_err & UIC_ERROR) && 6786 (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR | 6787 UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))) { 6788 needs_reset = true; 6789 goto do_reset; 6790 } 6791 6792 /* 6793 * If LINERESET was caught, UFS might have been put to PWM mode, 6794 * check if power mode restore is needed. 6795 */ 6796 if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) { 6797 hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR; 6798 if (!hba->saved_uic_err) 6799 hba->saved_err &= ~UIC_ERROR; 6800 spin_unlock_irqrestore(hba->host->host_lock, flags); 6801 if (ufshcd_is_pwr_mode_restore_needed(hba)) 6802 needs_restore = true; 6803 spin_lock_irqsave(hba->host->host_lock, flags); 6804 if (!hba->saved_err && !needs_restore) 6805 goto skip_err_handling; 6806 } 6807 6808 hba->silence_err_logs = true; 6809 /* release lock as clear command might sleep */ 6810 spin_unlock_irqrestore(hba->host->host_lock, flags); 6811 6812 needs_reset = ufshcd_abort_all(hba); 6813 6814 spin_lock_irqsave(hba->host->host_lock, flags); 6815 hba->silence_err_logs = false; 6816 if (needs_reset) 6817 goto do_reset; 6818 6819 /* 6820 * After all reqs and tasks are cleared from doorbell, 6821 * now it is safe to retore power mode. 6822 */ 6823 if (needs_restore) { 6824 spin_unlock_irqrestore(hba->host->host_lock, flags); 6825 /* 6826 * Hold the scaling lock just in case dev cmds 6827 * are sent via bsg and/or sysfs. 6828 */ 6829 down_write(&hba->clk_scaling_lock); 6830 hba->force_pmc = true; 6831 pmc_err = ufshcd_config_pwr_mode(hba, &(hba->pwr_info)); 6832 if (pmc_err) { 6833 needs_reset = true; 6834 dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n", 6835 __func__, pmc_err); 6836 } 6837 hba->force_pmc = false; 6838 ufshcd_print_pwr_info(hba); 6839 up_write(&hba->clk_scaling_lock); 6840 spin_lock_irqsave(hba->host->host_lock, flags); 6841 } 6842 6843 do_reset: 6844 /* Fatal errors need reset */ 6845 if (needs_reset) { 6846 int err; 6847 6848 hba->force_reset = false; 6849 spin_unlock_irqrestore(hba->host->host_lock, flags); 6850 err = ufshcd_reset_and_restore(hba); 6851 if (err) 6852 dev_err(hba->dev, "%s: reset and restore failed with err %d\n", 6853 __func__, err); 6854 else 6855 ufshcd_recover_pm_error(hba); 6856 spin_lock_irqsave(hba->host->host_lock, flags); 6857 } 6858 6859 skip_err_handling: 6860 if (!needs_reset) { 6861 if (hba->ufshcd_state == UFSHCD_STATE_RESET) 6862 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; 6863 if (hba->saved_err || hba->saved_uic_err) 6864 dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x", 6865 __func__, hba->saved_err, hba->saved_uic_err); 6866 } 6867 /* Exit in an operational state or dead */ 6868 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL && 6869 hba->ufshcd_state != UFSHCD_STATE_ERROR) { 6870 if (--retries) 6871 goto again; 6872 hba->ufshcd_state = UFSHCD_STATE_ERROR; 6873 } 6874 ufshcd_clear_eh_in_progress(hba); 6875 spin_unlock_irqrestore(hba->host->host_lock, flags); 6876 ufshcd_err_handling_unprepare(hba); 6877 up(&hba->host_sem); 6878 6879 dev_info(hba->dev, "%s finished; HBA state %s\n", __func__, 6880 ufshcd_state_name[hba->ufshcd_state]); 6881 } 6882 6883 /** 6884 * ufshcd_update_uic_error - check and set fatal UIC error flags. 6885 * @hba: per-adapter instance 6886 * 6887 * Return: 6888 * IRQ_HANDLED - If interrupt is valid 6889 * IRQ_NONE - If invalid interrupt 6890 */ 6891 static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba) 6892 { 6893 u32 reg; 6894 irqreturn_t retval = IRQ_NONE; 6895 6896 /* PHY layer error */ 6897 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER); 6898 if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) && 6899 (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) { 6900 ufshcd_update_evt_hist(hba, UFS_EVT_PA_ERR, reg); 6901 /* 6902 * To know whether this error is fatal or not, DB timeout 6903 * must be checked but this error is handled separately. 6904 */ 6905 if (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK) 6906 dev_dbg(hba->dev, "%s: UIC Lane error reported\n", 6907 __func__); 6908 6909 /* Got a LINERESET indication. */ 6910 if (reg & UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) { 6911 struct uic_command *cmd = NULL; 6912 6913 hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR; 6914 if (hba->uic_async_done && hba->active_uic_cmd) 6915 cmd = hba->active_uic_cmd; 6916 /* 6917 * Ignore the LINERESET during power mode change 6918 * operation via DME_SET command. 6919 */ 6920 if (cmd && (cmd->command == UIC_CMD_DME_SET)) 6921 hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR; 6922 } 6923 retval |= IRQ_HANDLED; 6924 } 6925 6926 /* PA_INIT_ERROR is fatal and needs UIC reset */ 6927 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER); 6928 if ((reg & UIC_DATA_LINK_LAYER_ERROR) && 6929 (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) { 6930 ufshcd_update_evt_hist(hba, UFS_EVT_DL_ERR, reg); 6931 6932 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT) 6933 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR; 6934 else if (hba->dev_quirks & 6935 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) { 6936 if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED) 6937 hba->uic_error |= 6938 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR; 6939 else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT) 6940 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR; 6941 } 6942 retval |= IRQ_HANDLED; 6943 } 6944 6945 /* UIC NL/TL/DME errors needs software retry */ 6946 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER); 6947 if ((reg & UIC_NETWORK_LAYER_ERROR) && 6948 (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) { 6949 ufshcd_update_evt_hist(hba, UFS_EVT_NL_ERR, reg); 6950 hba->uic_error |= UFSHCD_UIC_NL_ERROR; 6951 retval |= IRQ_HANDLED; 6952 } 6953 6954 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER); 6955 if ((reg & UIC_TRANSPORT_LAYER_ERROR) && 6956 (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) { 6957 ufshcd_update_evt_hist(hba, UFS_EVT_TL_ERR, reg); 6958 hba->uic_error |= UFSHCD_UIC_TL_ERROR; 6959 retval |= IRQ_HANDLED; 6960 } 6961 6962 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME); 6963 if ((reg & UIC_DME_ERROR) && 6964 (reg & UIC_DME_ERROR_CODE_MASK)) { 6965 ufshcd_update_evt_hist(hba, UFS_EVT_DME_ERR, reg); 6966 hba->uic_error |= UFSHCD_UIC_DME_ERROR; 6967 retval |= IRQ_HANDLED; 6968 } 6969 6970 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n", 6971 __func__, hba->uic_error); 6972 return retval; 6973 } 6974 6975 /** 6976 * ufshcd_check_errors - Check for errors that need s/w attention 6977 * @hba: per-adapter instance 6978 * @intr_status: interrupt status generated by the controller 6979 * 6980 * Return: 6981 * IRQ_HANDLED - If interrupt is valid 6982 * IRQ_NONE - If invalid interrupt 6983 */ 6984 static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status) 6985 { 6986 bool queue_eh_work = false; 6987 irqreturn_t retval = IRQ_NONE; 6988 6989 guard(spinlock_irqsave)(hba->host->host_lock); 6990 hba->errors |= UFSHCD_ERROR_MASK & intr_status; 6991 6992 if (hba->errors & INT_FATAL_ERRORS) { 6993 ufshcd_update_evt_hist(hba, UFS_EVT_FATAL_ERR, 6994 hba->errors); 6995 queue_eh_work = true; 6996 } 6997 6998 if (hba->errors & UIC_ERROR) { 6999 hba->uic_error = 0; 7000 retval = ufshcd_update_uic_error(hba); 7001 if (hba->uic_error) 7002 queue_eh_work = true; 7003 } 7004 7005 if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) { 7006 dev_err(hba->dev, 7007 "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n", 7008 __func__, (hba->errors & UIC_HIBERNATE_ENTER) ? 7009 "Enter" : "Exit", 7010 hba->errors, ufshcd_get_upmcrs(hba)); 7011 ufshcd_update_evt_hist(hba, UFS_EVT_AUTO_HIBERN8_ERR, 7012 hba->errors); 7013 ufshcd_set_link_broken(hba); 7014 queue_eh_work = true; 7015 } 7016 7017 if (queue_eh_work) { 7018 /* 7019 * update the transfer error masks to sticky bits, let's do this 7020 * irrespective of current ufshcd_state. 7021 */ 7022 hba->saved_err |= hba->errors; 7023 hba->saved_uic_err |= hba->uic_error; 7024 7025 /* dump controller state before resetting */ 7026 if ((hba->saved_err & 7027 (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) || 7028 (hba->saved_uic_err && 7029 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) { 7030 dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n", 7031 __func__, hba->saved_err, 7032 hba->saved_uic_err); 7033 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, 7034 "host_regs: "); 7035 ufshcd_print_pwr_info(hba); 7036 } 7037 ufshcd_schedule_eh_work(hba); 7038 retval |= IRQ_HANDLED; 7039 } 7040 /* 7041 * if (!queue_eh_work) - 7042 * Other errors are either non-fatal where host recovers 7043 * itself without s/w intervention or errors that will be 7044 * handled by the SCSI core layer. 7045 */ 7046 hba->errors = 0; 7047 hba->uic_error = 0; 7048 7049 return retval; 7050 } 7051 7052 /** 7053 * ufshcd_tmc_handler - handle task management function completion 7054 * @hba: per adapter instance 7055 * 7056 * Return: 7057 * IRQ_HANDLED - If interrupt is valid 7058 * IRQ_NONE - If invalid interrupt 7059 */ 7060 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba) 7061 { 7062 unsigned long flags, pending, issued; 7063 irqreturn_t ret = IRQ_NONE; 7064 int tag; 7065 7066 spin_lock_irqsave(hba->host->host_lock, flags); 7067 pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL); 7068 issued = hba->outstanding_tasks & ~pending; 7069 for_each_set_bit(tag, &issued, hba->nutmrs) { 7070 struct request *req = hba->tmf_rqs[tag]; 7071 struct completion *c = req->end_io_data; 7072 7073 complete(c); 7074 ret = IRQ_HANDLED; 7075 } 7076 spin_unlock_irqrestore(hba->host->host_lock, flags); 7077 7078 return ret; 7079 } 7080 7081 /** 7082 * ufshcd_handle_mcq_cq_events - handle MCQ completion queue events 7083 * @hba: per adapter instance 7084 * 7085 * Return: IRQ_HANDLED if interrupt is handled. 7086 */ 7087 static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba) 7088 { 7089 struct ufs_hw_queue *hwq; 7090 unsigned long outstanding_cqs; 7091 unsigned int nr_queues; 7092 int i, ret; 7093 u32 events; 7094 7095 ret = ufshcd_vops_get_outstanding_cqs(hba, &outstanding_cqs); 7096 if (ret) 7097 outstanding_cqs = (1U << hba->nr_hw_queues) - 1; 7098 7099 /* Exclude the poll queues */ 7100 nr_queues = hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL]; 7101 for_each_set_bit(i, &outstanding_cqs, nr_queues) { 7102 hwq = &hba->uhq[i]; 7103 7104 events = ufshcd_mcq_read_cqis(hba, i); 7105 if (events) 7106 ufshcd_mcq_write_cqis(hba, events, i); 7107 7108 if (events & UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS) 7109 ufshcd_mcq_poll_cqe_lock(hba, hwq); 7110 } 7111 7112 return IRQ_HANDLED; 7113 } 7114 7115 /** 7116 * ufshcd_sl_intr - Interrupt service routine 7117 * @hba: per adapter instance 7118 * @intr_status: contains interrupts generated by the controller 7119 * 7120 * Return: 7121 * IRQ_HANDLED - If interrupt is valid 7122 * IRQ_NONE - If invalid interrupt 7123 */ 7124 static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status) 7125 { 7126 irqreturn_t retval = IRQ_NONE; 7127 7128 if (intr_status & UFSHCD_UIC_MASK) 7129 retval |= ufshcd_uic_cmd_compl(hba, intr_status); 7130 7131 if (intr_status & UFSHCD_ERROR_MASK || hba->errors) 7132 retval |= ufshcd_check_errors(hba, intr_status); 7133 7134 if (intr_status & UTP_TASK_REQ_COMPL) 7135 retval |= ufshcd_tmc_handler(hba); 7136 7137 if (intr_status & UTP_TRANSFER_REQ_COMPL) 7138 retval |= ufshcd_transfer_req_compl(hba); 7139 7140 if (intr_status & MCQ_CQ_EVENT_STATUS) 7141 retval |= ufshcd_handle_mcq_cq_events(hba); 7142 7143 return retval; 7144 } 7145 7146 /** 7147 * ufshcd_threaded_intr - Threaded interrupt service routine 7148 * @irq: irq number 7149 * @__hba: pointer to adapter instance 7150 * 7151 * Return: 7152 * IRQ_HANDLED - If interrupt is valid 7153 * IRQ_NONE - If invalid interrupt 7154 */ 7155 static irqreturn_t ufshcd_threaded_intr(int irq, void *__hba) 7156 { 7157 u32 last_intr_status, intr_status, enabled_intr_status = 0; 7158 irqreturn_t retval = IRQ_NONE; 7159 struct ufs_hba *hba = __hba; 7160 int retries = hba->nutrs; 7161 7162 last_intr_status = intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 7163 7164 /* 7165 * There could be max of hba->nutrs reqs in flight and in worst case 7166 * if the reqs get finished 1 by 1 after the interrupt status is 7167 * read, make sure we handle them by checking the interrupt status 7168 * again in a loop until we process all of the reqs before returning. 7169 */ 7170 while (intr_status && retries--) { 7171 enabled_intr_status = 7172 intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 7173 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS); 7174 if (enabled_intr_status) 7175 retval |= ufshcd_sl_intr(hba, enabled_intr_status); 7176 7177 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 7178 } 7179 7180 if (enabled_intr_status && retval == IRQ_NONE && 7181 (!(enabled_intr_status & UTP_TRANSFER_REQ_COMPL) || 7182 hba->outstanding_reqs) && !ufshcd_eh_in_progress(hba)) { 7183 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 0x%08x)\n", 7184 __func__, 7185 intr_status, 7186 last_intr_status, 7187 enabled_intr_status); 7188 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: "); 7189 } 7190 7191 return retval; 7192 } 7193 7194 /** 7195 * ufshcd_intr - Main interrupt service routine 7196 * @irq: irq number 7197 * @__hba: pointer to adapter instance 7198 * 7199 * Return: 7200 * IRQ_HANDLED - If interrupt is valid 7201 * IRQ_WAKE_THREAD - If handling is moved to threaded handled 7202 * IRQ_NONE - If invalid interrupt 7203 */ 7204 static irqreturn_t ufshcd_intr(int irq, void *__hba) 7205 { 7206 struct ufs_hba *hba = __hba; 7207 u32 intr_status, enabled_intr_status; 7208 7209 /* Move interrupt handling to thread when MCQ & ESI are not enabled */ 7210 if (!hba->mcq_enabled || !hba->mcq_esi_enabled) 7211 return IRQ_WAKE_THREAD; 7212 7213 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 7214 enabled_intr_status = intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 7215 7216 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS); 7217 7218 /* Directly handle interrupts since MCQ ESI handlers does the hard job */ 7219 return ufshcd_sl_intr(hba, enabled_intr_status); 7220 } 7221 7222 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag) 7223 { 7224 int err = 0; 7225 u32 mask = 1 << tag; 7226 7227 if (!test_bit(tag, &hba->outstanding_tasks)) 7228 goto out; 7229 7230 ufshcd_utmrl_clear(hba, tag); 7231 7232 /* poll for max. 1 sec to clear door bell register by h/w */ 7233 err = ufshcd_wait_for_register(hba, 7234 REG_UTP_TASK_REQ_DOOR_BELL, 7235 mask, 0, 1000, 1000); 7236 7237 dev_err(hba->dev, "Clearing task management function with tag %d %s\n", 7238 tag, err < 0 ? "failed" : "succeeded"); 7239 7240 out: 7241 return err; 7242 } 7243 7244 static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba, 7245 struct utp_task_req_desc *treq, u8 tm_function) 7246 { 7247 struct request_queue *q = hba->tmf_queue; 7248 struct Scsi_Host *host = hba->host; 7249 DECLARE_COMPLETION_ONSTACK(wait); 7250 struct request *req; 7251 unsigned long flags; 7252 int task_tag, err; 7253 7254 /* 7255 * blk_mq_alloc_request() is used here only to get a free tag. 7256 */ 7257 req = blk_mq_alloc_request(q, REQ_OP_DRV_OUT, 0); 7258 if (IS_ERR(req)) 7259 return PTR_ERR(req); 7260 7261 req->end_io_data = &wait; 7262 ufshcd_hold(hba); 7263 7264 spin_lock_irqsave(host->host_lock, flags); 7265 7266 task_tag = req->tag; 7267 hba->tmf_rqs[req->tag] = req; 7268 treq->upiu_req.req_header.task_tag = task_tag; 7269 7270 memcpy(hba->utmrdl_base_addr + task_tag, treq, sizeof(*treq)); 7271 ufshcd_vops_setup_task_mgmt(hba, task_tag, tm_function); 7272 7273 __set_bit(task_tag, &hba->outstanding_tasks); 7274 7275 spin_unlock_irqrestore(host->host_lock, flags); 7276 7277 /* send command to the controller */ 7278 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TASK_REQ_DOOR_BELL); 7279 7280 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_SEND); 7281 7282 /* wait until the task management command is completed */ 7283 err = wait_for_completion_io_timeout(&wait, 7284 msecs_to_jiffies(TM_CMD_TIMEOUT)); 7285 if (!err) { 7286 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_ERR); 7287 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n", 7288 __func__, tm_function); 7289 if (ufshcd_clear_tm_cmd(hba, task_tag)) 7290 dev_WARN(hba->dev, "%s: unable to clear tm cmd (slot %d) after timeout\n", 7291 __func__, task_tag); 7292 err = -ETIMEDOUT; 7293 } else { 7294 err = 0; 7295 memcpy(treq, hba->utmrdl_base_addr + task_tag, sizeof(*treq)); 7296 7297 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_COMP); 7298 } 7299 7300 spin_lock_irqsave(hba->host->host_lock, flags); 7301 hba->tmf_rqs[req->tag] = NULL; 7302 __clear_bit(task_tag, &hba->outstanding_tasks); 7303 spin_unlock_irqrestore(hba->host->host_lock, flags); 7304 7305 ufshcd_release(hba); 7306 blk_mq_free_request(req); 7307 7308 return err; 7309 } 7310 7311 /** 7312 * ufshcd_issue_tm_cmd - issues task management commands to controller 7313 * @hba: per adapter instance 7314 * @lun_id: LUN ID to which TM command is sent 7315 * @task_id: task ID to which the TM command is applicable 7316 * @tm_function: task management function opcode 7317 * @tm_response: task management service response return value 7318 * 7319 * Return: non-zero value on error, zero on success. 7320 */ 7321 static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id, 7322 u8 tm_function, u8 *tm_response) 7323 { 7324 struct utp_task_req_desc treq = { }; 7325 enum utp_ocs ocs_value; 7326 int err; 7327 7328 /* Configure task request descriptor */ 7329 treq.header.interrupt = 1; 7330 treq.header.ocs = OCS_INVALID_COMMAND_STATUS; 7331 7332 /* Configure task request UPIU */ 7333 treq.upiu_req.req_header.transaction_code = UPIU_TRANSACTION_TASK_REQ; 7334 treq.upiu_req.req_header.lun = lun_id; 7335 treq.upiu_req.req_header.tm_function = tm_function; 7336 7337 /* 7338 * The host shall provide the same value for LUN field in the basic 7339 * header and for Input Parameter. 7340 */ 7341 treq.upiu_req.input_param1 = cpu_to_be32(lun_id); 7342 treq.upiu_req.input_param2 = cpu_to_be32(task_id); 7343 7344 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function); 7345 if (err == -ETIMEDOUT) 7346 return err; 7347 7348 ocs_value = treq.header.ocs & MASK_OCS; 7349 if (ocs_value != OCS_SUCCESS) 7350 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", 7351 __func__, ocs_value); 7352 else if (tm_response) 7353 *tm_response = be32_to_cpu(treq.upiu_rsp.output_param1) & 7354 MASK_TM_SERVICE_RESP; 7355 return err; 7356 } 7357 7358 /** 7359 * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests 7360 * @hba: per-adapter instance 7361 * @req_upiu: upiu request 7362 * @rsp_upiu: upiu reply 7363 * @desc_buff: pointer to descriptor buffer, NULL if NA 7364 * @buff_len: descriptor size, 0 if NA 7365 * @cmd_type: specifies the type (NOP, Query...) 7366 * @desc_op: descriptor operation 7367 * 7368 * Those type of requests uses UTP Transfer Request Descriptor - utrd. 7369 * Therefore, it "rides" the device management infrastructure: uses its tag and 7370 * tasks work queues. 7371 * 7372 * Since there is only one available tag for device management commands, 7373 * the caller is expected to hold the hba->dev_cmd.lock mutex. 7374 * 7375 * Return: 0 upon success; < 0 upon failure. 7376 */ 7377 static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, 7378 struct utp_upiu_req *req_upiu, 7379 struct utp_upiu_req *rsp_upiu, 7380 u8 *desc_buff, int *buff_len, 7381 enum dev_cmd_type cmd_type, 7382 enum query_opcode desc_op) 7383 { 7384 struct scsi_cmnd *cmd = ufshcd_get_dev_mgmt_cmd(hba); 7385 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7386 u32 tag; 7387 int err = 0; 7388 u8 upiu_flags; 7389 7390 /* Protects use of hba->dev_cmd. */ 7391 lockdep_assert_held(&hba->dev_cmd.lock); 7392 7393 if (WARN_ON_ONCE(!cmd)) 7394 return -ENOMEM; 7395 7396 tag = scsi_cmd_to_rq(cmd)->tag; 7397 7398 ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag); 7399 7400 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0); 7401 7402 /* update the task tag in the request upiu */ 7403 req_upiu->header.task_tag = tag; 7404 7405 /* just copy the upiu request as it is */ 7406 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr)); 7407 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) { 7408 /* The Data Segment Area is optional depending upon the query 7409 * function value. for WRITE DESCRIPTOR, the data segment 7410 * follows right after the tsf. 7411 */ 7412 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len); 7413 *buff_len = 0; 7414 } 7415 7416 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 7417 7418 err = ufshcd_issue_dev_cmd(hba, cmd, tag, dev_cmd_timeout); 7419 if (err) 7420 goto put_dev_mgmt_cmd; 7421 7422 /* just copy the upiu response as it is */ 7423 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); 7424 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) { 7425 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu); 7426 u16 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header 7427 .data_segment_length); 7428 7429 if (*buff_len >= resp_len) { 7430 memcpy(desc_buff, descp, resp_len); 7431 *buff_len = resp_len; 7432 } else { 7433 dev_warn(hba->dev, 7434 "%s: rsp size %d is bigger than buffer size %d", 7435 __func__, resp_len, *buff_len); 7436 *buff_len = 0; 7437 err = -EINVAL; 7438 } 7439 } 7440 7441 put_dev_mgmt_cmd: 7442 ufshcd_put_dev_mgmt_cmd(cmd); 7443 7444 return err; 7445 } 7446 7447 /** 7448 * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands 7449 * @hba: per-adapter instance 7450 * @req_upiu: upiu request 7451 * @rsp_upiu: upiu reply - only 8 DW as we do not support scsi commands 7452 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target 7453 * @desc_buff: pointer to descriptor buffer, NULL if NA 7454 * @buff_len: descriptor size, 0 if NA 7455 * @desc_op: descriptor operation 7456 * 7457 * Supports UTP Transfer requests (nop and query), and UTP Task 7458 * Management requests. 7459 * It is up to the caller to fill the upiu conent properly, as it will 7460 * be copied without any further input validations. 7461 * 7462 * Return: 0 upon success; < 0 upon failure. 7463 */ 7464 int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, 7465 struct utp_upiu_req *req_upiu, 7466 struct utp_upiu_req *rsp_upiu, 7467 enum upiu_request_transaction msgcode, 7468 u8 *desc_buff, int *buff_len, 7469 enum query_opcode desc_op) 7470 { 7471 int err; 7472 enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY; 7473 struct utp_task_req_desc treq = { }; 7474 enum utp_ocs ocs_value; 7475 u8 tm_f = req_upiu->header.tm_function; 7476 7477 switch (msgcode) { 7478 case UPIU_TRANSACTION_NOP_OUT: 7479 cmd_type = DEV_CMD_TYPE_NOP; 7480 fallthrough; 7481 case UPIU_TRANSACTION_QUERY_REQ: 7482 ufshcd_dev_man_lock(hba); 7483 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu, 7484 desc_buff, buff_len, 7485 cmd_type, desc_op); 7486 ufshcd_dev_man_unlock(hba); 7487 7488 break; 7489 case UPIU_TRANSACTION_TASK_REQ: 7490 treq.header.interrupt = 1; 7491 treq.header.ocs = OCS_INVALID_COMMAND_STATUS; 7492 7493 memcpy(&treq.upiu_req, req_upiu, sizeof(*req_upiu)); 7494 7495 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f); 7496 if (err == -ETIMEDOUT) 7497 break; 7498 7499 ocs_value = treq.header.ocs & MASK_OCS; 7500 if (ocs_value != OCS_SUCCESS) { 7501 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__, 7502 ocs_value); 7503 break; 7504 } 7505 7506 memcpy(rsp_upiu, &treq.upiu_rsp, sizeof(*rsp_upiu)); 7507 7508 break; 7509 default: 7510 err = -EINVAL; 7511 7512 break; 7513 } 7514 7515 return err; 7516 } 7517 7518 /** 7519 * ufshcd_advanced_rpmb_req_handler - handle advanced RPMB request 7520 * @hba: per adapter instance 7521 * @req_upiu: upiu request 7522 * @rsp_upiu: upiu reply 7523 * @req_ehs: EHS field which contains Advanced RPMB Request Message 7524 * @rsp_ehs: EHS field which returns Advanced RPMB Response Message 7525 * @sg_cnt: The number of sg lists actually used 7526 * @sg_list: Pointer to SG list when DATA IN/OUT UPIU is required in ARPMB operation 7527 * @dir: DMA direction 7528 * 7529 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 7530 * < 0 if another error occurred. 7531 */ 7532 int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *req_upiu, 7533 struct utp_upiu_req *rsp_upiu, struct ufs_ehs *req_ehs, 7534 struct ufs_ehs *rsp_ehs, int sg_cnt, struct scatterlist *sg_list, 7535 enum dma_data_direction dir) 7536 { 7537 struct scsi_cmnd *cmd; 7538 struct ufshcd_lrb *lrbp; 7539 u32 tag; 7540 int err = 0; 7541 int result; 7542 u8 upiu_flags; 7543 u8 *ehs_data; 7544 u16 ehs_len; 7545 int ehs = (hba->capabilities & MASK_EHSLUTRD_SUPPORTED) ? 2 : 0; 7546 7547 ufshcd_dev_man_lock(hba); 7548 7549 cmd = ufshcd_get_dev_mgmt_cmd(hba); 7550 7551 if (WARN_ON_ONCE(!cmd)) { 7552 err = -ENOMEM; 7553 goto unlock; 7554 } 7555 7556 lrbp = scsi_cmd_priv(cmd); 7557 tag = scsi_cmd_to_rq(cmd)->tag; 7558 7559 ufshcd_setup_dev_cmd(hba, cmd, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, 7560 tag); 7561 7562 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, ehs); 7563 7564 /* update the task tag */ 7565 req_upiu->header.task_tag = tag; 7566 7567 /* copy the UPIU(contains CDB) request as it is */ 7568 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr)); 7569 /* Copy EHS, starting with byte32, immediately after the CDB package */ 7570 memcpy(lrbp->ucd_req_ptr + 1, req_ehs, sizeof(*req_ehs)); 7571 7572 if (dir != DMA_NONE && sg_list) 7573 ufshcd_sgl_to_prdt(hba, lrbp, sg_cnt, sg_list); 7574 7575 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 7576 7577 err = ufshcd_issue_dev_cmd(hba, cmd, tag, ADVANCED_RPMB_REQ_TIMEOUT); 7578 if (err) 7579 goto put_dev_mgmt_cmd; 7580 7581 err = ufshcd_dev_cmd_completion(hba, lrbp); 7582 if (!err) { 7583 /* Just copy the upiu response as it is */ 7584 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); 7585 /* Get the response UPIU result */ 7586 result = (lrbp->ucd_rsp_ptr->header.response << 8) | 7587 lrbp->ucd_rsp_ptr->header.status; 7588 7589 ehs_len = lrbp->ucd_rsp_ptr->header.ehs_length; 7590 /* 7591 * Since the bLength in EHS indicates the total size of the EHS Header and EHS Data 7592 * in 32 Byte units, the value of the bLength Request/Response for Advanced RPMB 7593 * Message is 02h 7594 */ 7595 if (ehs_len == 2 && rsp_ehs) { 7596 /* 7597 * ucd_rsp_ptr points to a buffer with a length of 512 bytes 7598 * (ALIGNED_UPIU_SIZE = 512), and the EHS data just starts from byte32 7599 */ 7600 ehs_data = (u8 *)lrbp->ucd_rsp_ptr + EHS_OFFSET_IN_RESPONSE; 7601 memcpy(rsp_ehs, ehs_data, ehs_len * 32); 7602 } 7603 } 7604 7605 put_dev_mgmt_cmd: 7606 ufshcd_put_dev_mgmt_cmd(cmd); 7607 7608 unlock: 7609 ufshcd_dev_man_unlock(hba); 7610 7611 return err ? : result; 7612 } 7613 7614 static bool ufshcd_clear_lu_cmds(struct request *req, void *priv) 7615 { 7616 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req); 7617 struct scsi_device *sdev = cmd->device; 7618 struct Scsi_Host *shost = sdev->host; 7619 struct ufs_hba *hba = shost_priv(shost); 7620 const u64 lun = *(u64 *)priv; 7621 const u32 tag = req->tag; 7622 7623 if (blk_mq_is_reserved_rq(req) || sdev->lun != lun) 7624 return true; 7625 7626 if (ufshcd_clear_cmd(hba, tag) < 0) { 7627 dev_err(hba->dev, "%s: failed to clear request %d\n", __func__, 7628 tag); 7629 return true; 7630 } 7631 7632 if (hba->mcq_enabled) { 7633 struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, req); 7634 7635 if (hwq) 7636 ufshcd_mcq_poll_cqe_lock(hba, hwq); 7637 return true; 7638 } 7639 7640 ufshcd_compl_one_cqe(hba, tag, NULL); 7641 return true; 7642 } 7643 7644 /** 7645 * ufshcd_eh_device_reset_handler() - Reset a single logical unit. 7646 * @cmd: SCSI command pointer 7647 * 7648 * Return: SUCCESS or FAILED. 7649 */ 7650 static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd) 7651 { 7652 struct Scsi_Host *host; 7653 struct ufs_hba *hba; 7654 int err; 7655 u8 resp = 0xF, lun; 7656 7657 host = cmd->device->host; 7658 hba = shost_priv(host); 7659 7660 lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun); 7661 err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp); 7662 if (err) { 7663 } else if (resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 7664 err = resp; 7665 } else { 7666 /* clear the commands that were pending for corresponding LUN */ 7667 blk_mq_tagset_busy_iter(&hba->host->tag_set, 7668 ufshcd_clear_lu_cmds, 7669 &cmd->device->lun); 7670 } 7671 7672 hba->req_abort_count = 0; 7673 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err); 7674 if (!err) { 7675 err = SUCCESS; 7676 } else { 7677 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); 7678 err = FAILED; 7679 } 7680 return err; 7681 } 7682 7683 static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) 7684 { 7685 int tag; 7686 7687 for_each_set_bit(tag, &bitmap, hba->nutrs) { 7688 struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); 7689 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7690 7691 lrbp->req_abort_skip = true; 7692 } 7693 } 7694 7695 /** 7696 * ufshcd_try_to_abort_task - abort a specific task 7697 * @hba: Pointer to adapter instance 7698 * @tag: Tag of the task to be aborted 7699 * 7700 * Abort the pending command in device by sending UFS_ABORT_TASK task management 7701 * command, and in host controller by clearing the door-bell register. There can 7702 * be race between controller sending the command to the device while abort is 7703 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is 7704 * really issued and then try to abort it. 7705 * 7706 * Return: zero on success, non-zero on failure. 7707 */ 7708 int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag) 7709 { 7710 struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); 7711 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7712 int err; 7713 int poll_cnt; 7714 u8 resp = 0xF; 7715 7716 for (poll_cnt = 100; poll_cnt; poll_cnt--) { 7717 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, tag, UFS_QUERY_TASK, 7718 &resp); 7719 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) { 7720 /* cmd pending in the device */ 7721 dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n", 7722 __func__, tag); 7723 break; 7724 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 7725 /* 7726 * cmd not pending in the device, check if it is 7727 * in transition. 7728 */ 7729 dev_info( 7730 hba->dev, 7731 "%s: cmd with tag %d not pending in the device.\n", 7732 __func__, tag); 7733 if (!ufshcd_cmd_inflight(cmd)) { 7734 dev_info(hba->dev, 7735 "%s: cmd with tag=%d completed.\n", 7736 __func__, tag); 7737 return 0; 7738 } 7739 usleep_range(100, 200); 7740 } else { 7741 dev_err(hba->dev, 7742 "%s: no response from device. tag = %d, err %d\n", 7743 __func__, tag, err); 7744 return err ? : resp; 7745 } 7746 } 7747 7748 if (!poll_cnt) 7749 return -EBUSY; 7750 7751 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, tag, UFS_ABORT_TASK, &resp); 7752 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 7753 if (!err) { 7754 err = resp; /* service response error */ 7755 dev_err(hba->dev, "%s: issued. tag = %d, err %d\n", 7756 __func__, tag, err); 7757 } 7758 return err; 7759 } 7760 7761 err = ufshcd_clear_cmd(hba, tag); 7762 if (err) 7763 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n", 7764 __func__, tag, err); 7765 7766 return err; 7767 } 7768 7769 /** 7770 * ufshcd_abort - scsi host template eh_abort_handler callback 7771 * @cmd: SCSI command pointer 7772 * 7773 * Return: SUCCESS or FAILED. 7774 */ 7775 static int ufshcd_abort(struct scsi_cmnd *cmd) 7776 { 7777 struct Scsi_Host *host = cmd->device->host; 7778 struct ufs_hba *hba = shost_priv(host); 7779 struct request *rq = scsi_cmd_to_rq(cmd); 7780 int tag = rq->tag; 7781 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7782 unsigned long flags; 7783 int err = FAILED; 7784 bool outstanding; 7785 u32 reg; 7786 7787 ufshcd_hold(hba); 7788 7789 if (!hba->mcq_enabled) { 7790 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 7791 if (!test_bit(tag, &hba->outstanding_reqs)) { 7792 /* If command is already aborted/completed, return FAILED. */ 7793 dev_err(hba->dev, 7794 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n", 7795 __func__, tag, hba->outstanding_reqs, reg); 7796 goto release; 7797 } 7798 } 7799 7800 /* Print Transfer Request of aborted task */ 7801 dev_info(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag); 7802 7803 /* 7804 * Print detailed info about aborted request. 7805 * As more than one request might get aborted at the same time, 7806 * print full information only for the first aborted request in order 7807 * to reduce repeated printouts. For other aborted requests only print 7808 * basic details. 7809 */ 7810 if (ufshcd_is_scsi_cmd(cmd)) 7811 scsi_print_command(cmd); 7812 if (!hba->req_abort_count) { 7813 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag); 7814 ufshcd_print_evt_hist(hba); 7815 ufshcd_print_host_state(hba); 7816 ufshcd_print_pwr_info(hba); 7817 ufshcd_print_tr(hba, cmd, true); 7818 } else { 7819 ufshcd_print_tr(hba, cmd, false); 7820 } 7821 hba->req_abort_count++; 7822 7823 if (!hba->mcq_enabled && !(reg & (1 << tag))) { 7824 /* only execute this code in single doorbell mode */ 7825 dev_err(hba->dev, 7826 "%s: cmd was completed, but without a notifying intr, tag = %d", 7827 __func__, tag); 7828 __ufshcd_transfer_req_compl(hba, 1UL << tag); 7829 goto release; 7830 } 7831 7832 /* 7833 * Task abort to the device W-LUN is illegal. When this command 7834 * will fail, due to spec violation, scsi err handling next step 7835 * will be to send LU reset which, again, is a spec violation. 7836 * To avoid these unnecessary/illegal steps, first we clean up 7837 * the lrb taken by this cmd and re-set it in outstanding_reqs, 7838 * then queue the eh_work and bail. 7839 */ 7840 if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) { 7841 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, lrbp->lun); 7842 7843 spin_lock_irqsave(host->host_lock, flags); 7844 hba->force_reset = true; 7845 ufshcd_schedule_eh_work(hba); 7846 spin_unlock_irqrestore(host->host_lock, flags); 7847 goto release; 7848 } 7849 7850 if (hba->mcq_enabled) { 7851 /* MCQ mode. Branch off to handle abort for mcq mode */ 7852 err = ufshcd_mcq_abort(cmd); 7853 goto release; 7854 } 7855 7856 /* Skip task abort in case previous aborts failed and report failure */ 7857 if (lrbp->req_abort_skip) { 7858 dev_err(hba->dev, "%s: skipping abort\n", __func__); 7859 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs); 7860 goto release; 7861 } 7862 7863 if (blk_mq_is_reserved_rq(rq)) 7864 err = ufshcd_clear_cmd(hba, tag); 7865 else 7866 err = ufshcd_try_to_abort_task(hba, tag); 7867 if (err) { 7868 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); 7869 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs); 7870 err = FAILED; 7871 goto release; 7872 } 7873 7874 /* 7875 * Clear the corresponding bit from outstanding_reqs since the command 7876 * has been aborted successfully. 7877 */ 7878 spin_lock_irqsave(&hba->outstanding_lock, flags); 7879 outstanding = __test_and_clear_bit(tag, &hba->outstanding_reqs); 7880 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 7881 7882 if (outstanding) 7883 ufshcd_release_scsi_cmd(hba, cmd); 7884 7885 err = SUCCESS; 7886 7887 release: 7888 /* Matches the ufshcd_hold() call at the start of this function. */ 7889 ufshcd_release(hba); 7890 return err; 7891 } 7892 7893 /** 7894 * ufshcd_process_probe_result - Process the ufshcd_probe_hba() result. 7895 * @hba: UFS host controller instance. 7896 * @probe_start: time when the ufshcd_probe_hba() call started. 7897 * @ret: ufshcd_probe_hba() return value. 7898 */ 7899 static void ufshcd_process_probe_result(struct ufs_hba *hba, 7900 ktime_t probe_start, int ret) 7901 { 7902 unsigned long flags; 7903 7904 spin_lock_irqsave(hba->host->host_lock, flags); 7905 if (ret) 7906 hba->ufshcd_state = UFSHCD_STATE_ERROR; 7907 else if (hba->ufshcd_state == UFSHCD_STATE_RESET) 7908 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; 7909 spin_unlock_irqrestore(hba->host->host_lock, flags); 7910 7911 trace_ufshcd_init(hba, ret, 7912 ktime_to_us(ktime_sub(ktime_get(), probe_start)), 7913 hba->curr_dev_pwr_mode, hba->uic_link_state); 7914 } 7915 7916 /** 7917 * ufshcd_host_reset_and_restore - reset and restore host controller 7918 * @hba: per-adapter instance 7919 * 7920 * Note that host controller reset may issue DME_RESET to 7921 * local and remote (device) Uni-Pro stack and the attributes 7922 * are reset to default state. 7923 * 7924 * Return: zero on success, non-zero on failure. 7925 */ 7926 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) 7927 { 7928 int err; 7929 7930 /* 7931 * Stop the host controller and complete the requests 7932 * cleared by h/w 7933 */ 7934 ufshcd_hba_stop(hba); 7935 hba->silence_err_logs = true; 7936 ufshcd_complete_requests(hba, true); 7937 hba->silence_err_logs = false; 7938 7939 /* scale up clocks to max frequency before full reinitialization */ 7940 if (ufshcd_is_clkscaling_supported(hba)) 7941 ufshcd_scale_clks(hba, ULONG_MAX, true); 7942 7943 err = ufshcd_hba_enable(hba); 7944 7945 /* Establish the link again and restore the device */ 7946 if (!err) { 7947 ktime_t probe_start = ktime_get(); 7948 7949 err = ufshcd_device_init(hba, /*init_dev_params=*/false); 7950 if (!err) 7951 err = ufshcd_probe_hba(hba, false); 7952 ufshcd_process_probe_result(hba, probe_start, err); 7953 } 7954 7955 if (err) 7956 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err); 7957 ufshcd_update_evt_hist(hba, UFS_EVT_HOST_RESET, (u32)err); 7958 return err; 7959 } 7960 7961 /** 7962 * ufshcd_reset_and_restore - reset and re-initialize host/device 7963 * @hba: per-adapter instance 7964 * 7965 * Reset and recover device, host and re-establish link. This 7966 * is helpful to recover the communication in fatal error conditions. 7967 * 7968 * Return: zero on success, non-zero on failure. 7969 */ 7970 static int ufshcd_reset_and_restore(struct ufs_hba *hba) 7971 { 7972 u32 saved_err = 0; 7973 u32 saved_uic_err = 0; 7974 int err = 0; 7975 unsigned long flags; 7976 int retries = MAX_HOST_RESET_RETRIES; 7977 7978 spin_lock_irqsave(hba->host->host_lock, flags); 7979 do { 7980 /* 7981 * This is a fresh start, cache and clear saved error first, 7982 * in case new error generated during reset and restore. 7983 */ 7984 saved_err |= hba->saved_err; 7985 saved_uic_err |= hba->saved_uic_err; 7986 hba->saved_err = 0; 7987 hba->saved_uic_err = 0; 7988 hba->force_reset = false; 7989 hba->ufshcd_state = UFSHCD_STATE_RESET; 7990 spin_unlock_irqrestore(hba->host->host_lock, flags); 7991 7992 /* Reset the attached device */ 7993 ufshcd_device_reset(hba); 7994 7995 err = ufshcd_host_reset_and_restore(hba); 7996 7997 spin_lock_irqsave(hba->host->host_lock, flags); 7998 if (err) 7999 continue; 8000 /* Do not exit unless operational or dead */ 8001 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL && 8002 hba->ufshcd_state != UFSHCD_STATE_ERROR && 8003 hba->ufshcd_state != UFSHCD_STATE_EH_SCHEDULED_NON_FATAL) 8004 err = -EAGAIN; 8005 } while (err && --retries); 8006 8007 /* 8008 * Inform scsi mid-layer that we did reset and allow to handle 8009 * Unit Attention properly. 8010 */ 8011 scsi_report_bus_reset(hba->host, 0); 8012 if (err) { 8013 hba->ufshcd_state = UFSHCD_STATE_ERROR; 8014 hba->saved_err |= saved_err; 8015 hba->saved_uic_err |= saved_uic_err; 8016 } 8017 spin_unlock_irqrestore(hba->host->host_lock, flags); 8018 8019 return err; 8020 } 8021 8022 /** 8023 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer 8024 * @cmd: SCSI command pointer 8025 * 8026 * Return: SUCCESS or FAILED. 8027 */ 8028 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd) 8029 { 8030 int err = SUCCESS; 8031 unsigned long flags; 8032 struct ufs_hba *hba; 8033 8034 hba = shost_priv(cmd->device->host); 8035 8036 /* 8037 * If runtime PM sent SSU and got a timeout, scsi_error_handler is 8038 * stuck in this function waiting for flush_work(&hba->eh_work). And 8039 * ufshcd_err_handler(eh_work) is stuck waiting for runtime PM. Do 8040 * ufshcd_link_recovery instead of eh_work to prevent deadlock. 8041 */ 8042 if (hba->pm_op_in_progress) { 8043 if (ufshcd_link_recovery(hba)) 8044 err = FAILED; 8045 8046 return err; 8047 } 8048 8049 spin_lock_irqsave(hba->host->host_lock, flags); 8050 hba->force_reset = true; 8051 ufshcd_schedule_eh_work(hba); 8052 dev_err(hba->dev, "%s: reset in progress - 1\n", __func__); 8053 spin_unlock_irqrestore(hba->host->host_lock, flags); 8054 8055 flush_work(&hba->eh_work); 8056 8057 spin_lock_irqsave(hba->host->host_lock, flags); 8058 if (hba->ufshcd_state == UFSHCD_STATE_ERROR) 8059 err = FAILED; 8060 spin_unlock_irqrestore(hba->host->host_lock, flags); 8061 8062 return err; 8063 } 8064 8065 /** 8066 * ufshcd_get_max_icc_level - calculate the ICC level 8067 * @sup_curr_uA: max. current supported by the regulator 8068 * @start_scan: row at the desc table to start scan from 8069 * @buff: power descriptor buffer 8070 * 8071 * Return: calculated max ICC level for specific regulator. 8072 */ 8073 static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, 8074 const char *buff) 8075 { 8076 int i; 8077 int curr_uA; 8078 u16 data; 8079 u16 unit; 8080 8081 for (i = start_scan; i >= 0; i--) { 8082 data = get_unaligned_be16(&buff[2 * i]); 8083 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >> 8084 ATTR_ICC_LVL_UNIT_OFFSET; 8085 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK; 8086 switch (unit) { 8087 case UFSHCD_NANO_AMP: 8088 curr_uA = curr_uA / 1000; 8089 break; 8090 case UFSHCD_MILI_AMP: 8091 curr_uA = curr_uA * 1000; 8092 break; 8093 case UFSHCD_AMP: 8094 curr_uA = curr_uA * 1000 * 1000; 8095 break; 8096 case UFSHCD_MICRO_AMP: 8097 default: 8098 break; 8099 } 8100 if (sup_curr_uA >= curr_uA) 8101 break; 8102 } 8103 if (i < 0) { 8104 i = 0; 8105 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i); 8106 } 8107 8108 return (u32)i; 8109 } 8110 8111 /** 8112 * ufshcd_find_max_sup_active_icc_level - calculate the max ICC level 8113 * In case regulators are not initialized we'll return 0 8114 * @hba: per-adapter instance 8115 * @desc_buf: power descriptor buffer to extract ICC levels from. 8116 * 8117 * Return: calculated ICC level. 8118 */ 8119 static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba, 8120 const u8 *desc_buf) 8121 { 8122 u32 icc_level = 0; 8123 8124 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq || 8125 !hba->vreg_info.vccq2) { 8126 /* 8127 * Using dev_dbg to avoid messages during runtime PM to avoid 8128 * never-ending cycles of messages written back to storage by 8129 * user space causing runtime resume, causing more messages and 8130 * so on. 8131 */ 8132 dev_dbg(hba->dev, 8133 "%s: Regulator capability was not set, actvIccLevel=%d", 8134 __func__, icc_level); 8135 goto out; 8136 } 8137 8138 if (hba->vreg_info.vcc->max_uA) 8139 icc_level = ufshcd_get_max_icc_level( 8140 hba->vreg_info.vcc->max_uA, 8141 POWER_DESC_MAX_ACTV_ICC_LVLS - 1, 8142 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]); 8143 8144 if (hba->vreg_info.vccq->max_uA) 8145 icc_level = ufshcd_get_max_icc_level( 8146 hba->vreg_info.vccq->max_uA, 8147 icc_level, 8148 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]); 8149 8150 if (hba->vreg_info.vccq2->max_uA) 8151 icc_level = ufshcd_get_max_icc_level( 8152 hba->vreg_info.vccq2->max_uA, 8153 icc_level, 8154 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]); 8155 out: 8156 return icc_level; 8157 } 8158 8159 static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba) 8160 { 8161 int ret; 8162 u8 *desc_buf; 8163 u32 icc_level; 8164 8165 desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 8166 if (!desc_buf) 8167 return; 8168 8169 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0, 8170 desc_buf, QUERY_DESC_MAX_SIZE); 8171 if (ret) { 8172 dev_err(hba->dev, 8173 "%s: Failed reading power descriptor ret = %d", 8174 __func__, ret); 8175 goto out; 8176 } 8177 8178 icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf); 8179 dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level); 8180 8181 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 8182 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level); 8183 8184 if (ret) 8185 dev_err(hba->dev, 8186 "%s: Failed configuring bActiveICCLevel = %d ret = %d", 8187 __func__, icc_level, ret); 8188 8189 out: 8190 kfree(desc_buf); 8191 } 8192 8193 static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev) 8194 { 8195 struct Scsi_Host *shost = sdev->host; 8196 8197 scsi_autopm_get_device(sdev); 8198 blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev); 8199 if (sdev->rpm_autosuspend) 8200 pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev, 8201 shost->rpm_autosuspend_delay); 8202 scsi_autopm_put_device(sdev); 8203 } 8204 8205 /** 8206 * ufshcd_scsi_add_wlus - Adds required W-LUs 8207 * @hba: per-adapter instance 8208 * 8209 * UFS device specification requires the UFS devices to support 4 well known 8210 * logical units: 8211 * "REPORT_LUNS" (address: 01h) 8212 * "UFS Device" (address: 50h) 8213 * "RPMB" (address: 44h) 8214 * "BOOT" (address: 30h) 8215 * UFS device's power management needs to be controlled by "POWER CONDITION" 8216 * field of SSU (START STOP UNIT) command. But this "power condition" field 8217 * will take effect only when its sent to "UFS device" well known logical unit 8218 * hence we require the scsi_device instance to represent this logical unit in 8219 * order for the UFS host driver to send the SSU command for power management. 8220 * 8221 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory 8222 * Block) LU so user space process can control this LU. User space may also 8223 * want to have access to BOOT LU. 8224 * 8225 * This function adds scsi device instances for each of all well known LUs 8226 * (except "REPORT LUNS" LU). 8227 * 8228 * Return: zero on success (all required W-LUs are added successfully), 8229 * non-zero error value on failure (if failed to add any of the required W-LU). 8230 */ 8231 static int ufshcd_scsi_add_wlus(struct ufs_hba *hba) 8232 { 8233 int ret = 0; 8234 struct scsi_device *sdev_boot, *sdev_rpmb; 8235 8236 hba->ufs_device_wlun = __scsi_add_device(hba->host, 0, 0, 8237 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL); 8238 if (IS_ERR(hba->ufs_device_wlun)) { 8239 ret = PTR_ERR(hba->ufs_device_wlun); 8240 hba->ufs_device_wlun = NULL; 8241 goto out; 8242 } 8243 scsi_device_put(hba->ufs_device_wlun); 8244 8245 sdev_rpmb = __scsi_add_device(hba->host, 0, 0, 8246 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL); 8247 if (IS_ERR(sdev_rpmb)) { 8248 ret = PTR_ERR(sdev_rpmb); 8249 hba->ufs_rpmb_wlun = NULL; 8250 dev_err(hba->dev, "%s: RPMB WLUN not found\n", __func__); 8251 goto remove_ufs_device_wlun; 8252 } 8253 hba->ufs_rpmb_wlun = sdev_rpmb; 8254 ufshcd_blk_pm_runtime_init(sdev_rpmb); 8255 scsi_device_put(sdev_rpmb); 8256 8257 sdev_boot = __scsi_add_device(hba->host, 0, 0, 8258 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL); 8259 if (IS_ERR(sdev_boot)) { 8260 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__); 8261 } else { 8262 ufshcd_blk_pm_runtime_init(sdev_boot); 8263 scsi_device_put(sdev_boot); 8264 } 8265 goto out; 8266 8267 remove_ufs_device_wlun: 8268 scsi_remove_device(hba->ufs_device_wlun); 8269 out: 8270 return ret; 8271 } 8272 8273 static void ufshcd_wb_probe(struct ufs_hba *hba, const u8 *desc_buf) 8274 { 8275 struct ufs_dev_info *dev_info = &hba->dev_info; 8276 u8 lun; 8277 u32 d_lu_wb_buf_alloc; 8278 u32 ext_ufs_feature; 8279 8280 if (!ufshcd_is_wb_allowed(hba)) 8281 return; 8282 8283 /* 8284 * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or 8285 * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES 8286 * enabled 8287 */ 8288 if (!(dev_info->wspecversion >= 0x310 || 8289 dev_info->wspecversion == 0x220 || 8290 (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES))) 8291 goto wb_disabled; 8292 8293 ext_ufs_feature = get_unaligned_be32(desc_buf + 8294 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); 8295 8296 if (!(ext_ufs_feature & UFS_DEV_WRITE_BOOSTER_SUP)) 8297 goto wb_disabled; 8298 8299 /* 8300 * WB may be supported but not configured while provisioning. The spec 8301 * says, in dedicated wb buffer mode, a max of 1 lun would have wb 8302 * buffer configured. 8303 */ 8304 dev_info->wb_buffer_type = desc_buf[DEVICE_DESC_PARAM_WB_TYPE]; 8305 8306 dev_info->ext_wb_sup = get_unaligned_be16(desc_buf + 8307 DEVICE_DESC_PARAM_EXT_WB_SUP); 8308 8309 dev_info->b_presrv_uspc_en = 8310 desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN]; 8311 8312 if (dev_info->wb_buffer_type == WB_BUF_MODE_SHARED) { 8313 if (!get_unaligned_be32(desc_buf + 8314 DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS)) 8315 goto wb_disabled; 8316 } else { 8317 for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) { 8318 d_lu_wb_buf_alloc = 0; 8319 ufshcd_read_unit_desc_param(hba, 8320 lun, 8321 UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS, 8322 (u8 *)&d_lu_wb_buf_alloc, 8323 sizeof(d_lu_wb_buf_alloc)); 8324 if (d_lu_wb_buf_alloc) { 8325 dev_info->wb_dedicated_lu = lun; 8326 break; 8327 } 8328 } 8329 8330 if (!d_lu_wb_buf_alloc) 8331 goto wb_disabled; 8332 } 8333 8334 if (!ufshcd_is_wb_buf_lifetime_available(hba)) 8335 goto wb_disabled; 8336 8337 return; 8338 8339 wb_disabled: 8340 hba->caps &= ~UFSHCD_CAP_WB_EN; 8341 } 8342 8343 static void ufshcd_temp_notif_probe(struct ufs_hba *hba, const u8 *desc_buf) 8344 { 8345 struct ufs_dev_info *dev_info = &hba->dev_info; 8346 u32 ext_ufs_feature; 8347 u8 mask = 0; 8348 8349 if (!(hba->caps & UFSHCD_CAP_TEMP_NOTIF) || dev_info->wspecversion < 0x300) 8350 return; 8351 8352 ext_ufs_feature = get_unaligned_be32(desc_buf + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); 8353 8354 if (ext_ufs_feature & UFS_DEV_LOW_TEMP_NOTIF) 8355 mask |= MASK_EE_TOO_LOW_TEMP; 8356 8357 if (ext_ufs_feature & UFS_DEV_HIGH_TEMP_NOTIF) 8358 mask |= MASK_EE_TOO_HIGH_TEMP; 8359 8360 if (mask) { 8361 ufshcd_enable_ee(hba, mask); 8362 ufs_hwmon_probe(hba, mask); 8363 } 8364 } 8365 8366 static void ufshcd_device_lvl_exception_probe(struct ufs_hba *hba, u8 *desc_buf) 8367 { 8368 u32 ext_ufs_feature; 8369 8370 if (hba->dev_info.wspecversion < 0x410) 8371 return; 8372 8373 ext_ufs_feature = get_unaligned_be32(desc_buf + 8374 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); 8375 if (!(ext_ufs_feature & UFS_DEV_LVL_EXCEPTION_SUP)) 8376 return; 8377 8378 atomic_set(&hba->dev_lvl_exception_count, 0); 8379 ufshcd_enable_ee(hba, MASK_EE_DEV_LVL_EXCEPTION); 8380 } 8381 8382 static void ufshcd_set_rtt(struct ufs_hba *hba) 8383 { 8384 struct ufs_dev_info *dev_info = &hba->dev_info; 8385 u32 rtt = 0; 8386 u32 dev_rtt = 0; 8387 int host_rtt_cap = hba->vops && hba->vops->max_num_rtt ? 8388 hba->vops->max_num_rtt : hba->nortt; 8389 8390 /* RTT override makes sense only for UFS-4.0 and above */ 8391 if (dev_info->wspecversion < 0x400) 8392 return; 8393 8394 if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 8395 QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &dev_rtt)) { 8396 dev_err(hba->dev, "failed reading bMaxNumOfRTT\n"); 8397 return; 8398 } 8399 8400 /* do not override if it was already written */ 8401 if (dev_rtt != DEFAULT_MAX_NUM_RTT) 8402 return; 8403 8404 rtt = min_t(int, dev_info->rtt_cap, host_rtt_cap); 8405 8406 if (rtt == dev_rtt) 8407 return; 8408 8409 if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 8410 QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &rtt)) 8411 dev_err(hba->dev, "failed writing bMaxNumOfRTT\n"); 8412 } 8413 8414 void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, 8415 const struct ufs_dev_quirk *fixups) 8416 { 8417 const struct ufs_dev_quirk *f; 8418 struct ufs_dev_info *dev_info = &hba->dev_info; 8419 8420 if (!fixups) 8421 return; 8422 8423 for (f = fixups; f->quirk; f++) { 8424 if ((f->wmanufacturerid == dev_info->wmanufacturerid || 8425 f->wmanufacturerid == UFS_ANY_VENDOR) && 8426 ((dev_info->model && 8427 STR_PRFX_EQUAL(f->model, dev_info->model)) || 8428 !strcmp(f->model, UFS_ANY_MODEL))) 8429 hba->dev_quirks |= f->quirk; 8430 } 8431 } 8432 EXPORT_SYMBOL_GPL(ufshcd_fixup_dev_quirks); 8433 8434 static void ufs_fixup_device_setup(struct ufs_hba *hba) 8435 { 8436 /* fix by general quirk table */ 8437 ufshcd_fixup_dev_quirks(hba, ufs_fixups); 8438 8439 /* allow vendors to fix quirks */ 8440 ufshcd_vops_fixup_dev_quirks(hba); 8441 } 8442 8443 static void ufshcd_update_rtc(struct ufs_hba *hba) 8444 { 8445 struct timespec64 ts64; 8446 int err; 8447 u32 val; 8448 8449 ktime_get_real_ts64(&ts64); 8450 8451 if (ts64.tv_sec < hba->dev_info.rtc_time_baseline) { 8452 dev_warn_once(hba->dev, "%s: Current time precedes previous setting!\n", __func__); 8453 return; 8454 } 8455 8456 /* 8457 * The Absolute RTC mode has a 136-year limit, spanning from 2010 to 2146. If a time beyond 8458 * 2146 is required, it is recommended to choose the relative RTC mode. 8459 */ 8460 val = ts64.tv_sec - hba->dev_info.rtc_time_baseline; 8461 8462 /* Skip update RTC if RPM state is not RPM_ACTIVE */ 8463 if (ufshcd_rpm_get_if_active(hba) <= 0) 8464 return; 8465 8466 err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, QUERY_ATTR_IDN_SECONDS_PASSED, 8467 0, 0, &val); 8468 ufshcd_rpm_put(hba); 8469 8470 if (err) 8471 dev_err(hba->dev, "%s: Failed to update rtc %d\n", __func__, err); 8472 else if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE) 8473 hba->dev_info.rtc_time_baseline = ts64.tv_sec; 8474 } 8475 8476 static void ufshcd_rtc_work(struct work_struct *work) 8477 { 8478 struct ufs_hba *hba; 8479 8480 hba = container_of(to_delayed_work(work), struct ufs_hba, ufs_rtc_update_work); 8481 8482 /* Update RTC only when there are no requests in progress and UFSHCI is operational */ 8483 if (!ufshcd_is_ufs_dev_busy(hba) && 8484 hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL && 8485 !hba->clk_gating.active_reqs) 8486 ufshcd_update_rtc(hba); 8487 8488 if (ufshcd_is_ufs_dev_active(hba) && hba->dev_info.rtc_update_period) 8489 schedule_delayed_work(&hba->ufs_rtc_update_work, 8490 msecs_to_jiffies(hba->dev_info.rtc_update_period)); 8491 } 8492 8493 static void ufs_init_rtc(struct ufs_hba *hba, u8 *desc_buf) 8494 { 8495 u16 periodic_rtc_update = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_FRQ_RTC]); 8496 struct ufs_dev_info *dev_info = &hba->dev_info; 8497 8498 if (periodic_rtc_update & UFS_RTC_TIME_BASELINE) { 8499 dev_info->rtc_type = UFS_RTC_ABSOLUTE; 8500 8501 /* 8502 * The concept of measuring time in Linux as the number of seconds elapsed since 8503 * 00:00:00 UTC on January 1, 1970, and UFS ABS RTC is elapsed from January 1st 8504 * 2010 00:00, here we need to adjust ABS baseline. 8505 */ 8506 dev_info->rtc_time_baseline = mktime64(2010, 1, 1, 0, 0, 0) - 8507 mktime64(1970, 1, 1, 0, 0, 0); 8508 } else { 8509 dev_info->rtc_type = UFS_RTC_RELATIVE; 8510 dev_info->rtc_time_baseline = 0; 8511 } 8512 8513 /* 8514 * We ignore TIME_PERIOD defined in wPeriodicRTCUpdate because Spec does not clearly state 8515 * how to calculate the specific update period for each time unit. And we disable periodic 8516 * RTC update work, let user configure by sysfs node according to specific circumstance. 8517 */ 8518 dev_info->rtc_update_period = 0; 8519 } 8520 8521 /** 8522 * ufshcd_create_device_id - Generate unique device identifier string 8523 * @hba: per-adapter instance 8524 * @desc_buf: device descriptor buffer 8525 * 8526 * Creates a unique device ID string combining manufacturer ID, spec version, 8527 * model name, serial number (as hex), device version, and manufacture date. 8528 * 8529 * Returns: Allocated device ID string on success, NULL on failure 8530 */ 8531 static char *ufshcd_create_device_id(struct ufs_hba *hba, u8 *desc_buf) 8532 { 8533 struct ufs_dev_info *dev_info = &hba->dev_info; 8534 u16 manufacture_date; 8535 u16 device_version; 8536 u8 *serial_number; 8537 char *serial_hex; 8538 char *device_id; 8539 u8 serial_index; 8540 int serial_len; 8541 int ret; 8542 8543 serial_index = desc_buf[DEVICE_DESC_PARAM_SN]; 8544 8545 ret = ufshcd_read_string_desc(hba, serial_index, &serial_number, SD_RAW); 8546 if (ret < 0) { 8547 dev_err(hba->dev, "Failed reading Serial Number. err = %d\n", ret); 8548 return NULL; 8549 } 8550 8551 device_version = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_DEV_VER]); 8552 manufacture_date = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_MANF_DATE]); 8553 8554 serial_len = ret; 8555 /* Allocate buffer for hex string: 2 chars per byte + null terminator */ 8556 serial_hex = kzalloc(serial_len * 2 + 1, GFP_KERNEL); 8557 if (!serial_hex) { 8558 kfree(serial_number); 8559 return NULL; 8560 } 8561 8562 bin2hex(serial_hex, serial_number, serial_len); 8563 8564 /* 8565 * Device ID format is ABI with secure world - do not change without firmware 8566 * coordination. 8567 */ 8568 device_id = kasprintf(GFP_KERNEL, "%04X-%04X-%s-%s-%04X-%04X", 8569 dev_info->wmanufacturerid, dev_info->wspecversion, 8570 dev_info->model, serial_hex, device_version, 8571 manufacture_date); 8572 8573 kfree(serial_hex); 8574 kfree(serial_number); 8575 8576 if (!device_id) 8577 dev_warn(hba->dev, "Failed to allocate unique device ID\n"); 8578 8579 return device_id; 8580 } 8581 8582 static int ufs_get_device_desc(struct ufs_hba *hba) 8583 { 8584 struct ufs_dev_info *dev_info = &hba->dev_info; 8585 struct Scsi_Host *shost = hba->host; 8586 int err; 8587 u8 model_index; 8588 u8 *desc_buf; 8589 8590 desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 8591 if (!desc_buf) { 8592 err = -ENOMEM; 8593 goto out; 8594 } 8595 8596 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf, 8597 QUERY_DESC_MAX_SIZE); 8598 if (err) { 8599 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n", 8600 __func__, err); 8601 goto out; 8602 } 8603 8604 /* 8605 * getting vendor (manufacturerID) and Bank Index in big endian 8606 * format 8607 */ 8608 dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 | 8609 desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1]; 8610 8611 /* getting Specification Version in big endian format */ 8612 dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 | 8613 desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1]; 8614 dev_info->bqueuedepth = desc_buf[DEVICE_DESC_PARAM_Q_DPTH]; 8615 8616 /* 8617 * According to the UFS standard, the UFS device queue depth 8618 * (bQueueDepth) must be in the range 1..255 if the shared queueing 8619 * architecture is supported. bQueueDepth is zero if the shared queueing 8620 * architecture is not supported. 8621 */ 8622 if (dev_info->bqueuedepth) 8623 shost->cmd_per_lun = min(hba->nutrs, dev_info->bqueuedepth) - 8624 UFSHCD_NUM_RESERVED; 8625 else 8626 shost->cmd_per_lun = shost->can_queue; 8627 8628 dev_info->rtt_cap = desc_buf[DEVICE_DESC_PARAM_RTT_CAP]; 8629 8630 dev_info->hid_sup = get_unaligned_be32(desc_buf + 8631 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP) & 8632 UFS_DEV_HID_SUPPORT; 8633 8634 model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME]; 8635 8636 err = ufshcd_read_string_desc(hba, model_index, 8637 &dev_info->model, SD_ASCII_STD); 8638 if (err < 0) { 8639 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n", 8640 __func__, err); 8641 goto out; 8642 } 8643 8644 /* Generate unique device ID */ 8645 dev_info->device_id = ufshcd_create_device_id(hba, desc_buf); 8646 8647 hba->luns_avail = desc_buf[DEVICE_DESC_PARAM_NUM_LU] + 8648 desc_buf[DEVICE_DESC_PARAM_NUM_WLU]; 8649 8650 ufs_fixup_device_setup(hba); 8651 8652 ufshcd_wb_probe(hba, desc_buf); 8653 8654 ufshcd_temp_notif_probe(hba, desc_buf); 8655 8656 if (dev_info->wspecversion >= 0x410) { 8657 hba->critical_health_count = 0; 8658 ufshcd_enable_ee(hba, MASK_EE_HEALTH_CRITICAL); 8659 } 8660 8661 ufs_init_rtc(hba, desc_buf); 8662 8663 ufshcd_device_lvl_exception_probe(hba, desc_buf); 8664 8665 /* 8666 * ufshcd_read_string_desc returns size of the string 8667 * reset the error value 8668 */ 8669 err = 0; 8670 8671 out: 8672 kfree(desc_buf); 8673 return err; 8674 } 8675 8676 static void ufs_put_device_desc(struct ufs_hba *hba) 8677 { 8678 struct ufs_dev_info *dev_info = &hba->dev_info; 8679 8680 kfree(dev_info->model); 8681 dev_info->model = NULL; 8682 kfree(dev_info->device_id); 8683 dev_info->device_id = NULL; 8684 } 8685 8686 /** 8687 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is 8688 * less than device PA_TACTIVATE time. 8689 * @hba: per-adapter instance 8690 * 8691 * Some UFS devices require host PA_TACTIVATE to be lower than device 8692 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk 8693 * for such devices. 8694 * 8695 * Return: zero on success, non-zero error value on failure. 8696 */ 8697 static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba) 8698 { 8699 int ret = 0; 8700 u32 granularity, peer_granularity; 8701 u32 pa_tactivate, peer_pa_tactivate; 8702 u32 pa_tactivate_us, peer_pa_tactivate_us; 8703 static const u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100}; 8704 8705 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY), 8706 &granularity); 8707 if (ret) 8708 goto out; 8709 8710 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY), 8711 &peer_granularity); 8712 if (ret) 8713 goto out; 8714 8715 if ((granularity < PA_GRANULARITY_MIN_VAL) || 8716 (granularity > PA_GRANULARITY_MAX_VAL)) { 8717 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d", 8718 __func__, granularity); 8719 return -EINVAL; 8720 } 8721 8722 if ((peer_granularity < PA_GRANULARITY_MIN_VAL) || 8723 (peer_granularity > PA_GRANULARITY_MAX_VAL)) { 8724 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d", 8725 __func__, peer_granularity); 8726 return -EINVAL; 8727 } 8728 8729 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate); 8730 if (ret) 8731 goto out; 8732 8733 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE), 8734 &peer_pa_tactivate); 8735 if (ret) 8736 goto out; 8737 8738 pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1]; 8739 peer_pa_tactivate_us = peer_pa_tactivate * 8740 gran_to_us_table[peer_granularity - 1]; 8741 8742 if (pa_tactivate_us >= peer_pa_tactivate_us) { 8743 u32 new_peer_pa_tactivate; 8744 8745 new_peer_pa_tactivate = pa_tactivate_us / 8746 gran_to_us_table[peer_granularity - 1]; 8747 new_peer_pa_tactivate++; 8748 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 8749 new_peer_pa_tactivate); 8750 } 8751 8752 out: 8753 return ret; 8754 } 8755 8756 /** 8757 * ufshcd_quirk_override_pa_h8time - Ensures proper adjustment of PA_HIBERN8TIME. 8758 * @hba: per-adapter instance 8759 * 8760 * Some UFS devices require specific adjustments to the PA_HIBERN8TIME parameter 8761 * to ensure proper hibernation timing. This function retrieves the current 8762 * PA_HIBERN8TIME value and increments it by 100us. 8763 */ 8764 static void ufshcd_quirk_override_pa_h8time(struct ufs_hba *hba) 8765 { 8766 u32 pa_h8time; 8767 int ret; 8768 8769 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_HIBERN8TIME), &pa_h8time); 8770 if (ret) { 8771 dev_err(hba->dev, "Failed to get PA_HIBERN8TIME: %d\n", ret); 8772 return; 8773 } 8774 8775 /* Increment by 1 to increase hibernation time by 100 µs */ 8776 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), pa_h8time + 1); 8777 if (ret) 8778 dev_err(hba->dev, "Failed updating PA_HIBERN8TIME: %d\n", ret); 8779 } 8780 8781 static void ufshcd_tune_unipro_params(struct ufs_hba *hba) 8782 { 8783 ufshcd_vops_apply_dev_quirks(hba); 8784 8785 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE) 8786 /* set 1ms timeout for PA_TACTIVATE */ 8787 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10); 8788 8789 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE) 8790 ufshcd_quirk_tune_host_pa_tactivate(hba); 8791 8792 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_HIBER8TIME) 8793 ufshcd_quirk_override_pa_h8time(hba); 8794 } 8795 8796 static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba) 8797 { 8798 hba->ufs_stats.hibern8_exit_cnt = 0; 8799 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); 8800 hba->req_abort_count = 0; 8801 } 8802 8803 static int ufshcd_device_geo_params_init(struct ufs_hba *hba) 8804 { 8805 int err; 8806 u8 *desc_buf; 8807 8808 desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 8809 if (!desc_buf) { 8810 err = -ENOMEM; 8811 goto out; 8812 } 8813 8814 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0, 8815 desc_buf, QUERY_DESC_MAX_SIZE); 8816 if (err) { 8817 dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n", 8818 __func__, err); 8819 goto out; 8820 } 8821 8822 if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1) 8823 hba->dev_info.max_lu_supported = 32; 8824 else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0) 8825 hba->dev_info.max_lu_supported = 8; 8826 8827 hba->dev_info.rpmb_io_size = desc_buf[GEOMETRY_DESC_PARAM_RPMB_RW_SIZE]; 8828 8829 out: 8830 kfree(desc_buf); 8831 return err; 8832 } 8833 8834 struct ufs_ref_clk { 8835 unsigned long freq_hz; 8836 enum ufs_ref_clk_freq val; 8837 }; 8838 8839 static const struct ufs_ref_clk ufs_ref_clk_freqs[] = { 8840 {19200000, REF_CLK_FREQ_19_2_MHZ}, 8841 {26000000, REF_CLK_FREQ_26_MHZ}, 8842 {38400000, REF_CLK_FREQ_38_4_MHZ}, 8843 {52000000, REF_CLK_FREQ_52_MHZ}, 8844 {0, REF_CLK_FREQ_INVAL}, 8845 }; 8846 8847 static enum ufs_ref_clk_freq 8848 ufs_get_bref_clk_from_hz(unsigned long freq) 8849 { 8850 int i; 8851 8852 for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++) 8853 if (ufs_ref_clk_freqs[i].freq_hz == freq) 8854 return ufs_ref_clk_freqs[i].val; 8855 8856 return REF_CLK_FREQ_INVAL; 8857 } 8858 8859 void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk) 8860 { 8861 unsigned long freq; 8862 8863 freq = clk_get_rate(refclk); 8864 8865 hba->dev_ref_clk_freq = 8866 ufs_get_bref_clk_from_hz(freq); 8867 8868 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL) 8869 dev_err(hba->dev, 8870 "invalid ref_clk setting = %ld\n", freq); 8871 } 8872 8873 static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba) 8874 { 8875 int err; 8876 u32 ref_clk; 8877 u32 freq = hba->dev_ref_clk_freq; 8878 8879 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 8880 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk); 8881 8882 if (err) { 8883 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n", 8884 err); 8885 goto out; 8886 } 8887 8888 if (ref_clk == freq) 8889 goto out; /* nothing to update */ 8890 8891 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 8892 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq); 8893 8894 if (err) { 8895 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n", 8896 ufs_ref_clk_freqs[freq].freq_hz); 8897 goto out; 8898 } 8899 8900 dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n", 8901 ufs_ref_clk_freqs[freq].freq_hz); 8902 8903 out: 8904 return err; 8905 } 8906 8907 static int ufshcd_device_params_init(struct ufs_hba *hba) 8908 { 8909 bool flag; 8910 int ret; 8911 8912 /* Init UFS geometry descriptor related parameters */ 8913 ret = ufshcd_device_geo_params_init(hba); 8914 if (ret) 8915 goto out; 8916 8917 /* Check and apply UFS device quirks */ 8918 ret = ufs_get_device_desc(hba); 8919 if (ret) { 8920 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n", 8921 __func__, ret); 8922 goto out; 8923 } 8924 8925 ufshcd_set_rtt(hba); 8926 8927 ufshcd_get_ref_clk_gating_wait(hba); 8928 8929 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG, 8930 QUERY_FLAG_IDN_PWR_ON_WPE, 0, &flag)) 8931 hba->dev_info.f_power_on_wp_en = flag; 8932 8933 /* Probe maximum power mode co-supported by both UFS host and device */ 8934 if (ufshcd_get_max_pwr_mode(hba)) 8935 dev_err(hba->dev, 8936 "%s: Failed getting max supported power mode\n", 8937 __func__); 8938 out: 8939 return ret; 8940 } 8941 8942 static void ufshcd_set_timestamp_attr(struct ufs_hba *hba) 8943 { 8944 int err; 8945 struct ufs_query_req *request = NULL; 8946 struct ufs_query_res *response = NULL; 8947 struct ufs_dev_info *dev_info = &hba->dev_info; 8948 struct utp_upiu_query_v4_0 *upiu_data; 8949 8950 if (dev_info->wspecversion < 0x400 || 8951 hba->dev_quirks & UFS_DEVICE_QUIRK_NO_TIMESTAMP_SUPPORT) 8952 return; 8953 8954 ufshcd_dev_man_lock(hba); 8955 8956 ufshcd_init_query(hba, &request, &response, 8957 UPIU_QUERY_OPCODE_WRITE_ATTR, 8958 QUERY_ATTR_IDN_TIMESTAMP, 0, 0); 8959 8960 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 8961 8962 upiu_data = (struct utp_upiu_query_v4_0 *)&request->upiu_req; 8963 8964 put_unaligned_be64(ktime_get_real_ns(), &upiu_data->osf3); 8965 8966 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); 8967 8968 if (err) 8969 dev_err(hba->dev, "%s: failed to set timestamp %d\n", 8970 __func__, err); 8971 8972 ufshcd_dev_man_unlock(hba); 8973 } 8974 8975 /** 8976 * ufshcd_add_lus - probe and add UFS logical units 8977 * @hba: per-adapter instance 8978 * 8979 * Return: 0 upon success; < 0 upon failure. 8980 */ 8981 static int ufshcd_add_lus(struct ufs_hba *hba) 8982 { 8983 int ret; 8984 8985 /* Add required well known logical units to scsi mid layer */ 8986 ret = ufshcd_scsi_add_wlus(hba); 8987 if (ret) 8988 goto out; 8989 8990 /* Initialize devfreq after UFS device is detected */ 8991 if (ufshcd_is_clkscaling_supported(hba)) { 8992 memcpy(&hba->clk_scaling.saved_pwr_info, 8993 &hba->pwr_info, 8994 sizeof(struct ufs_pa_layer_attr)); 8995 hba->clk_scaling.is_allowed = true; 8996 8997 ret = ufshcd_devfreq_init(hba); 8998 if (ret) 8999 goto out; 9000 9001 hba->clk_scaling.is_enabled = true; 9002 ufshcd_init_clk_scaling_sysfs(hba); 9003 } 9004 9005 /* 9006 * The RTC update code accesses the hba->ufs_device_wlun->sdev_gendev 9007 * pointer and hence must only be started after the WLUN pointer has 9008 * been initialized by ufshcd_scsi_add_wlus(). 9009 */ 9010 schedule_delayed_work(&hba->ufs_rtc_update_work, 9011 msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS)); 9012 9013 ufs_bsg_probe(hba); 9014 scsi_scan_host(hba->host); 9015 ufs_rpmb_probe(hba); 9016 9017 out: 9018 return ret; 9019 } 9020 9021 /* SDB - Single Doorbell */ 9022 static void ufshcd_release_sdb_queue(struct ufs_hba *hba, int nutrs) 9023 { 9024 size_t ucdl_size, utrdl_size; 9025 9026 ucdl_size = ufshcd_get_ucd_size(hba) * nutrs; 9027 dmam_free_coherent(hba->dev, ucdl_size, hba->ucdl_base_addr, 9028 hba->ucdl_dma_addr); 9029 9030 utrdl_size = sizeof(struct utp_transfer_req_desc) * nutrs; 9031 dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr, 9032 hba->utrdl_dma_addr); 9033 } 9034 9035 static int ufshcd_alloc_mcq(struct ufs_hba *hba) 9036 { 9037 int ret; 9038 int old_nutrs = hba->nutrs; 9039 9040 ret = ufshcd_get_hba_mac(hba); 9041 if (ret < 0) 9042 return ret; 9043 9044 hba->nutrs = ret; 9045 ret = ufshcd_mcq_init(hba); 9046 if (ret) 9047 goto err; 9048 9049 /* 9050 * Previously allocated memory for nutrs may not be enough in MCQ mode. 9051 * Number of supported tags in MCQ mode may be larger than SDB mode. 9052 */ 9053 if (hba->nutrs != old_nutrs) { 9054 ufshcd_release_sdb_queue(hba, old_nutrs); 9055 ret = ufshcd_memory_alloc(hba); 9056 if (ret) 9057 goto err; 9058 ufshcd_host_memory_configure(hba); 9059 } 9060 9061 ret = ufshcd_mcq_memory_alloc(hba); 9062 if (ret) 9063 goto err; 9064 9065 hba->host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; 9066 9067 return 0; 9068 err: 9069 hba->nutrs = old_nutrs; 9070 return ret; 9071 } 9072 9073 static void ufshcd_config_mcq(struct ufs_hba *hba) 9074 { 9075 int ret; 9076 9077 ret = ufshcd_mcq_vops_config_esi(hba); 9078 hba->mcq_esi_enabled = !ret; 9079 dev_info(hba->dev, "ESI %sconfigured\n", ret ? "is not " : ""); 9080 9081 ufshcd_mcq_make_queues_operational(hba); 9082 ufshcd_mcq_config_mac(hba, hba->nutrs); 9083 9084 dev_info(hba->dev, "MCQ configured, nr_queues=%d, io_queues=%d, read_queue=%d, poll_queues=%d, queue_depth=%d\n", 9085 hba->nr_hw_queues, hba->nr_queues[HCTX_TYPE_DEFAULT], 9086 hba->nr_queues[HCTX_TYPE_READ], hba->nr_queues[HCTX_TYPE_POLL], 9087 hba->nutrs); 9088 } 9089 9090 static int ufshcd_post_device_init(struct ufs_hba *hba) 9091 { 9092 int ret; 9093 9094 ufshcd_tune_unipro_params(hba); 9095 9096 /* UFS device is also active now */ 9097 ufshcd_set_ufs_dev_active(hba); 9098 ufshcd_force_reset_auto_bkops(hba); 9099 9100 ufshcd_set_timestamp_attr(hba); 9101 9102 if (!hba->max_pwr_info.is_valid) 9103 return 0; 9104 9105 /* 9106 * Set the right value to bRefClkFreq before attempting to 9107 * switch to HS gears. 9108 */ 9109 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL) 9110 ufshcd_set_dev_ref_clk(hba); 9111 /* Gear up to HS gear. */ 9112 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info); 9113 if (ret) { 9114 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n", 9115 __func__, ret); 9116 return ret; 9117 } 9118 9119 return 0; 9120 } 9121 9122 static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params) 9123 { 9124 int ret; 9125 9126 WARN_ON_ONCE(!hba->scsi_host_added); 9127 9128 hba->ufshcd_state = UFSHCD_STATE_RESET; 9129 9130 ret = ufshcd_link_startup(hba); 9131 if (ret) 9132 return ret; 9133 9134 if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION) 9135 return ret; 9136 9137 /* Debug counters initialization */ 9138 ufshcd_clear_dbg_ufs_stats(hba); 9139 9140 /* UniPro link is active now */ 9141 ufshcd_set_link_active(hba); 9142 9143 /* Reconfigure MCQ upon reset */ 9144 if (hba->mcq_enabled && !init_dev_params) { 9145 ufshcd_config_mcq(hba); 9146 ufshcd_mcq_enable(hba); 9147 } 9148 9149 /* Verify device initialization by sending NOP OUT UPIU */ 9150 ret = ufshcd_verify_dev_init(hba); 9151 if (ret) 9152 return ret; 9153 9154 /* Initiate UFS initialization, and waiting until completion */ 9155 ret = ufshcd_complete_dev_init(hba); 9156 if (ret) 9157 return ret; 9158 9159 /* 9160 * Initialize UFS device parameters used by driver, these 9161 * parameters are associated with UFS descriptors. 9162 */ 9163 if (init_dev_params) { 9164 ret = ufshcd_device_params_init(hba); 9165 if (ret) 9166 return ret; 9167 if (is_mcq_supported(hba) && 9168 hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH) { 9169 ufshcd_config_mcq(hba); 9170 ufshcd_mcq_enable(hba); 9171 } 9172 } 9173 9174 return ufshcd_post_device_init(hba); 9175 } 9176 9177 /** 9178 * ufshcd_probe_hba - probe hba to detect device and initialize it 9179 * @hba: per-adapter instance 9180 * @init_dev_params: whether or not to call ufshcd_device_params_init(). 9181 * 9182 * Execute link-startup and verify device initialization 9183 * 9184 * Return: 0 upon success; < 0 upon failure. 9185 */ 9186 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params) 9187 { 9188 int ret; 9189 9190 if (!hba->pm_op_in_progress && 9191 (hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH)) { 9192 /* Reset the device and controller before doing reinit */ 9193 ufshcd_device_reset(hba); 9194 ufs_put_device_desc(hba); 9195 ufshcd_hba_stop(hba); 9196 ret = ufshcd_hba_enable(hba); 9197 if (ret) { 9198 dev_err(hba->dev, "Host controller enable failed\n"); 9199 ufshcd_print_evt_hist(hba); 9200 ufshcd_print_host_state(hba); 9201 return ret; 9202 } 9203 9204 /* Reinit the device */ 9205 ret = ufshcd_device_init(hba, init_dev_params); 9206 if (ret) 9207 return ret; 9208 } 9209 9210 ufshcd_print_pwr_info(hba); 9211 9212 /* 9213 * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec) 9214 * and for removable UFS card as well, hence always set the parameter. 9215 * Note: Error handler may issue the device reset hence resetting 9216 * bActiveICCLevel as well so it is always safe to set this here. 9217 */ 9218 ufshcd_set_active_icc_lvl(hba); 9219 9220 /* Enable UFS Write Booster if supported */ 9221 ufshcd_configure_wb(hba); 9222 9223 if (hba->ee_usr_mask) 9224 ufshcd_write_ee_control(hba); 9225 ufshcd_configure_auto_hibern8(hba); 9226 9227 return 0; 9228 } 9229 9230 /** 9231 * ufshcd_async_scan - asynchronous execution for probing hba 9232 * @data: data pointer to pass to this function 9233 * @cookie: cookie data 9234 */ 9235 static void ufshcd_async_scan(void *data, async_cookie_t cookie) 9236 { 9237 struct ufs_hba *hba = (struct ufs_hba *)data; 9238 ktime_t probe_start; 9239 int ret; 9240 9241 down(&hba->host_sem); 9242 /* Initialize hba, detect and initialize UFS device */ 9243 probe_start = ktime_get(); 9244 ret = ufshcd_probe_hba(hba, true); 9245 ufshcd_process_probe_result(hba, probe_start, ret); 9246 up(&hba->host_sem); 9247 if (ret) 9248 goto out; 9249 9250 /* Probe and add UFS logical units */ 9251 ret = ufshcd_add_lus(hba); 9252 9253 out: 9254 pm_runtime_put_sync(hba->dev); 9255 9256 if (ret) 9257 dev_err(hba->dev, "%s failed: %d\n", __func__, ret); 9258 } 9259 9260 static enum scsi_timeout_action ufshcd_eh_timed_out(struct scsi_cmnd *scmd) 9261 { 9262 struct ufs_hba *hba = shost_priv(scmd->device->host); 9263 9264 if (!hba->system_suspending) { 9265 /* Activate the error handler in the SCSI core. */ 9266 return SCSI_EH_NOT_HANDLED; 9267 } 9268 9269 /* 9270 * If we get here we know that no TMFs are outstanding and also that 9271 * the only pending command is a START STOP UNIT command. Handle the 9272 * timeout of that command directly to prevent a deadlock between 9273 * ufshcd_set_dev_pwr_mode() and ufshcd_err_handler(). 9274 */ 9275 ufshcd_link_recovery(hba); 9276 dev_info(hba->dev, "%s() finished; outstanding_tasks = %#lx.\n", 9277 __func__, hba->outstanding_tasks); 9278 9279 return scsi_host_busy(hba->host) ? SCSI_EH_RESET_TIMER : SCSI_EH_DONE; 9280 } 9281 9282 static const struct attribute_group *ufshcd_driver_groups[] = { 9283 &ufs_sysfs_unit_descriptor_group, 9284 &ufs_sysfs_lun_attributes_group, 9285 NULL, 9286 }; 9287 9288 static struct ufs_hba_variant_params ufs_hba_vps = { 9289 .hba_enable_delay_us = 1000, 9290 .wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(40), 9291 .devfreq_profile.polling_ms = 100, 9292 .devfreq_profile.target = ufshcd_devfreq_target, 9293 .devfreq_profile.get_dev_status = ufshcd_devfreq_get_dev_status, 9294 .ondemand_data.upthreshold = 70, 9295 .ondemand_data.downdifferential = 5, 9296 }; 9297 9298 static const struct scsi_host_template ufshcd_driver_template = { 9299 .module = THIS_MODULE, 9300 .name = UFSHCD, 9301 .proc_name = UFSHCD, 9302 .map_queues = ufshcd_map_queues, 9303 .cmd_size = sizeof(struct ufshcd_lrb), 9304 .init_cmd_priv = ufshcd_init_cmd_priv, 9305 .queuecommand = ufshcd_queuecommand, 9306 .queue_reserved_command = ufshcd_queue_reserved_command, 9307 .nr_reserved_cmds = UFSHCD_NUM_RESERVED, 9308 .mq_poll = ufshcd_poll, 9309 .sdev_init = ufshcd_sdev_init, 9310 .sdev_configure = ufshcd_sdev_configure, 9311 .sdev_destroy = ufshcd_sdev_destroy, 9312 .change_queue_depth = ufshcd_change_queue_depth, 9313 .eh_abort_handler = ufshcd_abort, 9314 .eh_device_reset_handler = ufshcd_eh_device_reset_handler, 9315 .eh_host_reset_handler = ufshcd_eh_host_reset_handler, 9316 .eh_timed_out = ufshcd_eh_timed_out, 9317 .this_id = -1, 9318 .sg_tablesize = SG_ALL, 9319 .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX, 9320 .max_sectors = SZ_1M / SECTOR_SIZE, 9321 .max_host_blocked = 1, 9322 .track_queue_depth = 1, 9323 .skip_settle_delay = 1, 9324 .sdev_groups = ufshcd_driver_groups, 9325 }; 9326 9327 static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg, 9328 int ua) 9329 { 9330 int ret; 9331 9332 if (!vreg) 9333 return 0; 9334 9335 /* 9336 * "set_load" operation shall be required on those regulators 9337 * which specifically configured current limitation. Otherwise 9338 * zero max_uA may cause unexpected behavior when regulator is 9339 * enabled or set as high power mode. 9340 */ 9341 if (!vreg->max_uA) 9342 return 0; 9343 9344 ret = regulator_set_load(vreg->reg, ua); 9345 if (ret < 0) { 9346 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n", 9347 __func__, vreg->name, ua, ret); 9348 } 9349 9350 return ret; 9351 } 9352 9353 static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba, 9354 struct ufs_vreg *vreg) 9355 { 9356 return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA); 9357 } 9358 9359 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba, 9360 struct ufs_vreg *vreg) 9361 { 9362 if (!vreg) 9363 return 0; 9364 9365 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA); 9366 } 9367 9368 static int ufshcd_config_vreg(struct device *dev, 9369 struct ufs_vreg *vreg, bool on) 9370 { 9371 if (regulator_count_voltages(vreg->reg) <= 0) 9372 return 0; 9373 9374 return ufshcd_config_vreg_load(dev, vreg, on ? vreg->max_uA : 0); 9375 } 9376 9377 static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg) 9378 { 9379 int ret = 0; 9380 9381 if (!vreg || vreg->enabled) 9382 goto out; 9383 9384 ret = ufshcd_config_vreg(dev, vreg, true); 9385 if (!ret) 9386 ret = regulator_enable(vreg->reg); 9387 9388 if (!ret) 9389 vreg->enabled = true; 9390 else 9391 dev_err(dev, "%s: %s enable failed, err=%d\n", 9392 __func__, vreg->name, ret); 9393 out: 9394 return ret; 9395 } 9396 9397 static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg) 9398 { 9399 int ret = 0; 9400 9401 if (!vreg || !vreg->enabled || vreg->always_on) 9402 goto out; 9403 9404 ret = regulator_disable(vreg->reg); 9405 9406 if (!ret) { 9407 /* ignore errors on applying disable config */ 9408 ufshcd_config_vreg(dev, vreg, false); 9409 vreg->enabled = false; 9410 } else { 9411 dev_err(dev, "%s: %s disable failed, err=%d\n", 9412 __func__, vreg->name, ret); 9413 } 9414 out: 9415 return ret; 9416 } 9417 9418 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on) 9419 { 9420 int ret = 0; 9421 struct device *dev = hba->dev; 9422 struct ufs_vreg_info *info = &hba->vreg_info; 9423 9424 ret = ufshcd_toggle_vreg(dev, info->vcc, on); 9425 if (ret) 9426 goto out; 9427 9428 ret = ufshcd_toggle_vreg(dev, info->vccq, on); 9429 if (ret) 9430 goto out; 9431 9432 ret = ufshcd_toggle_vreg(dev, info->vccq2, on); 9433 9434 out: 9435 if (ret) { 9436 ufshcd_toggle_vreg(dev, info->vccq2, false); 9437 ufshcd_toggle_vreg(dev, info->vccq, false); 9438 ufshcd_toggle_vreg(dev, info->vcc, false); 9439 } 9440 return ret; 9441 } 9442 9443 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on) 9444 { 9445 struct ufs_vreg_info *info = &hba->vreg_info; 9446 9447 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on); 9448 } 9449 9450 int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg) 9451 { 9452 int ret = 0; 9453 9454 if (!vreg) 9455 goto out; 9456 9457 vreg->reg = devm_regulator_get(dev, vreg->name); 9458 if (IS_ERR(vreg->reg)) { 9459 ret = PTR_ERR(vreg->reg); 9460 dev_err(dev, "%s: %s get failed, err=%d\n", 9461 __func__, vreg->name, ret); 9462 } 9463 out: 9464 return ret; 9465 } 9466 EXPORT_SYMBOL_GPL(ufshcd_get_vreg); 9467 9468 static int ufshcd_init_vreg(struct ufs_hba *hba) 9469 { 9470 int ret = 0; 9471 struct device *dev = hba->dev; 9472 struct ufs_vreg_info *info = &hba->vreg_info; 9473 9474 ret = ufshcd_get_vreg(dev, info->vcc); 9475 if (ret) 9476 goto out; 9477 9478 ret = ufshcd_get_vreg(dev, info->vccq); 9479 if (!ret) 9480 ret = ufshcd_get_vreg(dev, info->vccq2); 9481 out: 9482 return ret; 9483 } 9484 9485 static int ufshcd_init_hba_vreg(struct ufs_hba *hba) 9486 { 9487 struct ufs_vreg_info *info = &hba->vreg_info; 9488 9489 return ufshcd_get_vreg(hba->dev, info->vdd_hba); 9490 } 9491 9492 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on) 9493 { 9494 int ret = 0; 9495 struct ufs_clk_info *clki; 9496 struct list_head *head = &hba->clk_list_head; 9497 ktime_t start = ktime_get(); 9498 bool clk_state_changed = false; 9499 9500 if (list_empty(head)) 9501 goto out; 9502 9503 ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE); 9504 if (ret) 9505 return ret; 9506 9507 list_for_each_entry(clki, head, list) { 9508 if (!IS_ERR_OR_NULL(clki->clk)) { 9509 /* 9510 * Don't disable clocks which are needed 9511 * to keep the link active. 9512 */ 9513 if (ufshcd_is_link_active(hba) && 9514 clki->keep_link_active) 9515 continue; 9516 9517 clk_state_changed = on ^ clki->enabled; 9518 if (on && !clki->enabled) { 9519 ret = clk_prepare_enable(clki->clk); 9520 if (ret) { 9521 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n", 9522 __func__, clki->name, ret); 9523 goto out; 9524 } 9525 } else if (!on && clki->enabled) { 9526 clk_disable_unprepare(clki->clk); 9527 } 9528 clki->enabled = on; 9529 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__, 9530 clki->name, on ? "en" : "dis"); 9531 } 9532 } 9533 9534 ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE); 9535 if (ret) 9536 return ret; 9537 9538 if (!ufshcd_is_clkscaling_supported(hba)) 9539 ufshcd_pm_qos_update(hba, on); 9540 out: 9541 if (ret) { 9542 list_for_each_entry(clki, head, list) { 9543 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled) 9544 clk_disable_unprepare(clki->clk); 9545 } 9546 } else if (!ret && on && hba->clk_gating.is_initialized) { 9547 scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) 9548 hba->clk_gating.state = CLKS_ON; 9549 trace_ufshcd_clk_gating(hba, 9550 hba->clk_gating.state); 9551 } 9552 9553 if (clk_state_changed) 9554 trace_ufshcd_profile_clk_gating(hba, 9555 (on ? "on" : "off"), 9556 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 9557 return ret; 9558 } 9559 9560 static enum ufs_ref_clk_freq ufshcd_parse_ref_clk_property(struct ufs_hba *hba) 9561 { 9562 u32 freq; 9563 int ret = device_property_read_u32(hba->dev, "ref-clk-freq", &freq); 9564 9565 if (ret) { 9566 dev_dbg(hba->dev, "Cannot query 'ref-clk-freq' property = %d", ret); 9567 return REF_CLK_FREQ_INVAL; 9568 } 9569 9570 return ufs_get_bref_clk_from_hz(freq); 9571 } 9572 9573 static int ufshcd_init_clocks(struct ufs_hba *hba) 9574 { 9575 int ret = 0; 9576 struct ufs_clk_info *clki; 9577 struct device *dev = hba->dev; 9578 struct list_head *head = &hba->clk_list_head; 9579 9580 if (list_empty(head)) 9581 goto out; 9582 9583 list_for_each_entry(clki, head, list) { 9584 if (!clki->name) 9585 continue; 9586 9587 clki->clk = devm_clk_get(dev, clki->name); 9588 if (IS_ERR(clki->clk)) { 9589 ret = PTR_ERR(clki->clk); 9590 dev_err(dev, "%s: %s clk get failed, %d\n", 9591 __func__, clki->name, ret); 9592 goto out; 9593 } 9594 9595 /* 9596 * Parse device ref clk freq as per device tree "ref_clk". 9597 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL 9598 * in ufshcd_alloc_host(). 9599 */ 9600 if (!strcmp(clki->name, "ref_clk")) 9601 ufshcd_parse_dev_ref_clk_freq(hba, clki->clk); 9602 9603 if (clki->max_freq) { 9604 ret = clk_set_rate(clki->clk, clki->max_freq); 9605 if (ret) { 9606 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 9607 __func__, clki->name, 9608 clki->max_freq, ret); 9609 goto out; 9610 } 9611 clki->curr_freq = clki->max_freq; 9612 } 9613 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__, 9614 clki->name, clk_get_rate(clki->clk)); 9615 } 9616 9617 /* Set Max. frequency for all clocks */ 9618 if (hba->use_pm_opp) { 9619 ret = ufshcd_opp_set_rate(hba, ULONG_MAX); 9620 if (ret) { 9621 dev_err(hba->dev, "%s: failed to set OPP: %d", __func__, 9622 ret); 9623 goto out; 9624 } 9625 } 9626 9627 out: 9628 return ret; 9629 } 9630 9631 static int ufshcd_variant_hba_init(struct ufs_hba *hba) 9632 { 9633 int err = 0; 9634 9635 if (!hba->vops) 9636 goto out; 9637 9638 err = ufshcd_vops_init(hba); 9639 if (err) 9640 dev_err_probe(hba->dev, err, 9641 "%s: variant %s init failed with err %d\n", 9642 __func__, ufshcd_get_var_name(hba), err); 9643 out: 9644 return err; 9645 } 9646 9647 static void ufshcd_variant_hba_exit(struct ufs_hba *hba) 9648 { 9649 if (!hba->vops) 9650 return; 9651 9652 ufshcd_vops_exit(hba); 9653 } 9654 9655 static int ufshcd_hba_init(struct ufs_hba *hba) 9656 { 9657 int err; 9658 9659 /* 9660 * Handle host controller power separately from the UFS device power 9661 * rails as it will help controlling the UFS host controller power 9662 * collapse easily which is different than UFS device power collapse. 9663 * Also, enable the host controller power before we go ahead with rest 9664 * of the initialization here. 9665 */ 9666 err = ufshcd_init_hba_vreg(hba); 9667 if (err) 9668 goto out; 9669 9670 err = ufshcd_setup_hba_vreg(hba, true); 9671 if (err) 9672 goto out; 9673 9674 err = ufshcd_init_clocks(hba); 9675 if (err) 9676 goto out_disable_hba_vreg; 9677 9678 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL) 9679 hba->dev_ref_clk_freq = ufshcd_parse_ref_clk_property(hba); 9680 9681 err = ufshcd_setup_clocks(hba, true); 9682 if (err) 9683 goto out_disable_hba_vreg; 9684 9685 err = ufshcd_init_vreg(hba); 9686 if (err) 9687 goto out_disable_clks; 9688 9689 err = ufshcd_setup_vreg(hba, true); 9690 if (err) 9691 goto out_disable_clks; 9692 9693 err = ufshcd_variant_hba_init(hba); 9694 if (err) 9695 goto out_disable_vreg; 9696 9697 ufs_debugfs_hba_init(hba); 9698 ufs_fault_inject_hba_init(hba); 9699 9700 hba->is_powered = true; 9701 goto out; 9702 9703 out_disable_vreg: 9704 ufshcd_setup_vreg(hba, false); 9705 out_disable_clks: 9706 ufshcd_setup_clocks(hba, false); 9707 out_disable_hba_vreg: 9708 ufshcd_setup_hba_vreg(hba, false); 9709 out: 9710 return err; 9711 } 9712 9713 static void ufshcd_hba_exit(struct ufs_hba *hba) 9714 { 9715 if (hba->is_powered) { 9716 ufshcd_pm_qos_exit(hba); 9717 ufshcd_exit_clk_scaling(hba); 9718 ufshcd_exit_clk_gating(hba); 9719 if (hba->eh_wq) 9720 destroy_workqueue(hba->eh_wq); 9721 ufs_debugfs_hba_exit(hba); 9722 ufshcd_variant_hba_exit(hba); 9723 ufshcd_setup_vreg(hba, false); 9724 ufshcd_setup_clocks(hba, false); 9725 ufshcd_setup_hba_vreg(hba, false); 9726 hba->is_powered = false; 9727 ufs_put_device_desc(hba); 9728 } 9729 } 9730 9731 static int ufshcd_execute_start_stop(struct scsi_device *sdev, 9732 enum ufs_dev_pwr_mode pwr_mode, 9733 struct scsi_sense_hdr *sshdr) 9734 { 9735 const unsigned char cdb[6] = { START_STOP, 0, 0, 0, pwr_mode << 4, 0 }; 9736 struct scsi_failure failure_defs[] = { 9737 { 9738 .allowed = 2, 9739 .result = SCMD_FAILURE_RESULT_ANY, 9740 }, 9741 }; 9742 struct scsi_failures failures = { 9743 .failure_definitions = failure_defs, 9744 }; 9745 const struct scsi_exec_args args = { 9746 .failures = &failures, 9747 .sshdr = sshdr, 9748 .req_flags = BLK_MQ_REQ_PM, 9749 .scmd_flags = SCMD_FAIL_IF_RECOVERING, 9750 }; 9751 9752 return scsi_execute_cmd(sdev, cdb, REQ_OP_DRV_IN, /*buffer=*/NULL, 9753 /*bufflen=*/0, /*timeout=*/10 * HZ, /*retries=*/0, 9754 &args); 9755 } 9756 9757 /** 9758 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device 9759 * power mode 9760 * @hba: per adapter instance 9761 * @pwr_mode: device power mode to set 9762 * 9763 * Return: 0 if requested power mode is set successfully; 9764 * < 0 if failed to set the requested power mode. 9765 */ 9766 static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, 9767 enum ufs_dev_pwr_mode pwr_mode) 9768 { 9769 struct scsi_sense_hdr sshdr; 9770 struct scsi_device *sdp; 9771 unsigned long flags; 9772 int ret; 9773 9774 spin_lock_irqsave(hba->host->host_lock, flags); 9775 sdp = hba->ufs_device_wlun; 9776 if (sdp && scsi_device_online(sdp)) 9777 ret = scsi_device_get(sdp); 9778 else 9779 ret = -ENODEV; 9780 spin_unlock_irqrestore(hba->host->host_lock, flags); 9781 9782 if (ret) 9783 return ret; 9784 9785 /* 9786 * If scsi commands fail, the scsi mid-layer schedules scsi error- 9787 * handling, which would wait for host to be resumed. Since we know 9788 * we are functional while we are here, skip host resume in error 9789 * handling context. 9790 */ 9791 hba->host->eh_noresume = 1; 9792 9793 /* 9794 * Current function would be generally called from the power management 9795 * callbacks hence set the RQF_PM flag so that it doesn't resume the 9796 * already suspended childs. 9797 */ 9798 ret = ufshcd_execute_start_stop(sdp, pwr_mode, &sshdr); 9799 if (ret) { 9800 sdev_printk(KERN_WARNING, sdp, 9801 "START_STOP failed for power mode: %d, result %x\n", 9802 pwr_mode, ret); 9803 if (ret > 0) { 9804 if (scsi_sense_valid(&sshdr)) 9805 scsi_print_sense_hdr(sdp, NULL, &sshdr); 9806 ret = -EIO; 9807 } 9808 } else { 9809 hba->curr_dev_pwr_mode = pwr_mode; 9810 } 9811 9812 scsi_device_put(sdp); 9813 hba->host->eh_noresume = 0; 9814 return ret; 9815 } 9816 9817 static int ufshcd_link_state_transition(struct ufs_hba *hba, 9818 enum uic_link_state req_link_state, 9819 bool check_for_bkops) 9820 { 9821 int ret = 0; 9822 9823 if (req_link_state == hba->uic_link_state) 9824 return 0; 9825 9826 if (req_link_state == UIC_LINK_HIBERN8_STATE) { 9827 ret = ufshcd_uic_hibern8_enter(hba); 9828 if (!ret) { 9829 ufshcd_set_link_hibern8(hba); 9830 } else { 9831 dev_err(hba->dev, "%s: hibern8 enter failed %d\n", 9832 __func__, ret); 9833 goto out; 9834 } 9835 } 9836 /* 9837 * If autobkops is enabled, link can't be turned off because 9838 * turning off the link would also turn off the device, except in the 9839 * case of DeepSleep where the device is expected to remain powered. 9840 */ 9841 else if ((req_link_state == UIC_LINK_OFF_STATE) && 9842 (!check_for_bkops || !hba->auto_bkops_enabled)) { 9843 /* 9844 * Let's make sure that link is in low power mode, we are doing 9845 * this currently by putting the link in Hibern8. Otherway to 9846 * put the link in low power mode is to send the DME end point 9847 * to device and then send the DME reset command to local 9848 * unipro. But putting the link in hibern8 is much faster. 9849 * 9850 * Note also that putting the link in Hibern8 is a requirement 9851 * for entering DeepSleep. 9852 */ 9853 ret = ufshcd_uic_hibern8_enter(hba); 9854 if (ret) { 9855 dev_err(hba->dev, "%s: hibern8 enter failed %d\n", 9856 __func__, ret); 9857 goto out; 9858 } 9859 /* 9860 * Change controller state to "reset state" which 9861 * should also put the link in off/reset state 9862 */ 9863 ufshcd_hba_stop(hba); 9864 /* 9865 * TODO: Check if we need any delay to make sure that 9866 * controller is reset 9867 */ 9868 ufshcd_set_link_off(hba); 9869 } 9870 9871 out: 9872 return ret; 9873 } 9874 9875 static void ufshcd_vreg_set_lpm(struct ufs_hba *hba) 9876 { 9877 bool vcc_off = false; 9878 9879 /* 9880 * It seems some UFS devices may keep drawing more than sleep current 9881 * (atleast for 500us) from UFS rails (especially from VCCQ rail). 9882 * To avoid this situation, add 2ms delay before putting these UFS 9883 * rails in LPM mode. 9884 */ 9885 if (!ufshcd_is_link_active(hba) && 9886 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM) 9887 usleep_range(2000, 2100); 9888 9889 /* 9890 * If UFS device is either in UFS_Sleep turn off VCC rail to save some 9891 * power. 9892 * 9893 * If UFS device and link is in OFF state, all power supplies (VCC, 9894 * VCCQ, VCCQ2) can be turned off if power on write protect is not 9895 * required. If UFS link is inactive (Hibern8 or OFF state) and device 9896 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode. 9897 * 9898 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway 9899 * in low power state which would save some power. 9900 * 9901 * If Write Booster is enabled and the device needs to flush the WB 9902 * buffer OR if bkops status is urgent for WB, keep Vcc on. 9903 */ 9904 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) && 9905 !hba->dev_info.is_lu_power_on_wp) { 9906 ufshcd_setup_vreg(hba, false); 9907 vcc_off = true; 9908 } else if (!ufshcd_is_ufs_dev_active(hba)) { 9909 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false); 9910 vcc_off = true; 9911 if (ufshcd_is_link_hibern8(hba) || ufshcd_is_link_off(hba)) { 9912 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq); 9913 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2); 9914 } 9915 } 9916 9917 /* 9918 * All UFS devices require delay after VCC power rail is turned-off. 9919 */ 9920 if (vcc_off && hba->vreg_info.vcc && !hba->vreg_info.vcc->always_on) 9921 usleep_range(hba->vcc_off_delay_us, 9922 hba->vcc_off_delay_us + 100); 9923 } 9924 9925 #ifdef CONFIG_PM 9926 static int ufshcd_vreg_set_hpm(struct ufs_hba *hba) 9927 { 9928 int ret = 0; 9929 9930 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) && 9931 !hba->dev_info.is_lu_power_on_wp) { 9932 ret = ufshcd_setup_vreg(hba, true); 9933 } else if (!ufshcd_is_ufs_dev_active(hba)) { 9934 if (!ufshcd_is_link_active(hba)) { 9935 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq); 9936 if (ret) 9937 goto vcc_disable; 9938 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2); 9939 if (ret) 9940 goto vccq_lpm; 9941 } 9942 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true); 9943 } 9944 goto out; 9945 9946 vccq_lpm: 9947 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq); 9948 vcc_disable: 9949 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false); 9950 out: 9951 return ret; 9952 } 9953 #endif /* CONFIG_PM */ 9954 9955 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba) 9956 { 9957 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba)) 9958 ufshcd_setup_hba_vreg(hba, false); 9959 } 9960 9961 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba) 9962 { 9963 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba)) 9964 ufshcd_setup_hba_vreg(hba, true); 9965 } 9966 9967 static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) 9968 { 9969 int ret = 0; 9970 bool check_for_bkops; 9971 enum ufs_pm_level pm_lvl; 9972 enum ufs_dev_pwr_mode req_dev_pwr_mode; 9973 enum uic_link_state req_link_state; 9974 9975 hba->pm_op_in_progress = true; 9976 if (pm_op != UFS_SHUTDOWN_PM) { 9977 pm_lvl = pm_op == UFS_RUNTIME_PM ? 9978 hba->rpm_lvl : hba->spm_lvl; 9979 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl); 9980 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl); 9981 } else { 9982 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE; 9983 req_link_state = UIC_LINK_OFF_STATE; 9984 } 9985 9986 /* 9987 * If we can't transition into any of the low power modes 9988 * just gate the clocks. 9989 */ 9990 ufshcd_hold(hba); 9991 hba->clk_gating.is_suspended = true; 9992 9993 if (ufshcd_is_clkscaling_supported(hba)) 9994 ufshcd_clk_scaling_suspend(hba, true); 9995 9996 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE && 9997 req_link_state == UIC_LINK_ACTIVE_STATE) { 9998 goto vops_suspend; 9999 } 10000 10001 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) && 10002 (req_link_state == hba->uic_link_state)) 10003 goto enable_scaling; 10004 10005 /* UFS device & link must be active before we enter in this function */ 10006 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) { 10007 /* Wait err handler finish or trigger err recovery */ 10008 if (!ufshcd_eh_in_progress(hba)) 10009 ufshcd_force_error_recovery(hba); 10010 ret = -EBUSY; 10011 goto enable_scaling; 10012 } 10013 10014 if (pm_op == UFS_RUNTIME_PM) { 10015 if (ufshcd_can_autobkops_during_suspend(hba)) { 10016 /* 10017 * The device is idle with no requests in the queue, 10018 * allow background operations if bkops status shows 10019 * that performance might be impacted. 10020 */ 10021 ret = ufshcd_bkops_ctrl(hba); 10022 if (ret) { 10023 /* 10024 * If return err in suspend flow, IO will hang. 10025 * Trigger error handler and break suspend for 10026 * error recovery. 10027 */ 10028 ufshcd_force_error_recovery(hba); 10029 ret = -EBUSY; 10030 goto enable_scaling; 10031 } 10032 } else { 10033 /* make sure that auto bkops is disabled */ 10034 ufshcd_disable_auto_bkops(hba); 10035 } 10036 /* 10037 * If device needs to do BKOP or WB buffer flush during 10038 * Hibern8, keep device power mode as "active power mode" 10039 * and VCC supply. 10040 */ 10041 hba->dev_info.b_rpm_dev_flush_capable = 10042 hba->auto_bkops_enabled || 10043 (((req_link_state == UIC_LINK_HIBERN8_STATE) || 10044 ((req_link_state == UIC_LINK_ACTIVE_STATE) && 10045 ufshcd_is_auto_hibern8_enabled(hba))) && 10046 ufshcd_wb_need_flush(hba)); 10047 } 10048 10049 flush_work(&hba->eeh_work); 10050 10051 ret = ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE); 10052 if (ret) 10053 goto enable_scaling; 10054 10055 if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) { 10056 if (pm_op != UFS_RUNTIME_PM) 10057 /* ensure that bkops is disabled */ 10058 ufshcd_disable_auto_bkops(hba); 10059 10060 if (!hba->dev_info.b_rpm_dev_flush_capable) { 10061 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode); 10062 if (ret && pm_op != UFS_SHUTDOWN_PM) { 10063 /* 10064 * If return err in suspend flow, IO will hang. 10065 * Trigger error handler and break suspend for 10066 * error recovery. 10067 */ 10068 ufshcd_force_error_recovery(hba); 10069 ret = -EBUSY; 10070 } 10071 if (ret) 10072 goto enable_scaling; 10073 } 10074 } 10075 10076 /* 10077 * In the case of DeepSleep, the device is expected to remain powered 10078 * with the link off, so do not check for bkops. 10079 */ 10080 check_for_bkops = !ufshcd_is_ufs_dev_deepsleep(hba); 10081 ret = ufshcd_link_state_transition(hba, req_link_state, check_for_bkops); 10082 if (ret && pm_op != UFS_SHUTDOWN_PM) { 10083 /* 10084 * If return err in suspend flow, IO will hang. 10085 * Trigger error handler and break suspend for 10086 * error recovery. 10087 */ 10088 ufshcd_force_error_recovery(hba); 10089 ret = -EBUSY; 10090 } 10091 if (ret) 10092 goto set_dev_active; 10093 10094 vops_suspend: 10095 /* 10096 * Call vendor specific suspend callback. As these callbacks may access 10097 * vendor specific host controller register space call them before the 10098 * host clocks are ON. 10099 */ 10100 ret = ufshcd_vops_suspend(hba, pm_op, POST_CHANGE); 10101 if (ret) 10102 goto set_link_active; 10103 10104 cancel_delayed_work_sync(&hba->ufs_rtc_update_work); 10105 goto out; 10106 10107 set_link_active: 10108 /* 10109 * Device hardware reset is required to exit DeepSleep. Also, for 10110 * DeepSleep, the link is off so host reset and restore will be done 10111 * further below. 10112 */ 10113 if (ufshcd_is_ufs_dev_deepsleep(hba)) { 10114 ufshcd_device_reset(hba); 10115 WARN_ON(!ufshcd_is_link_off(hba)); 10116 } 10117 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba)) 10118 ufshcd_set_link_active(hba); 10119 else if (ufshcd_is_link_off(hba)) 10120 ufshcd_host_reset_and_restore(hba); 10121 set_dev_active: 10122 /* Can also get here needing to exit DeepSleep */ 10123 if (ufshcd_is_ufs_dev_deepsleep(hba)) { 10124 ufshcd_device_reset(hba); 10125 ufshcd_host_reset_and_restore(hba); 10126 } 10127 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE)) 10128 ufshcd_disable_auto_bkops(hba); 10129 enable_scaling: 10130 if (ufshcd_is_clkscaling_supported(hba)) 10131 ufshcd_clk_scaling_suspend(hba, false); 10132 10133 hba->dev_info.b_rpm_dev_flush_capable = false; 10134 out: 10135 if (hba->dev_info.b_rpm_dev_flush_capable) { 10136 schedule_delayed_work(&hba->rpm_dev_flush_recheck_work, 10137 msecs_to_jiffies(RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS)); 10138 } 10139 10140 if (ret) { 10141 ufshcd_update_evt_hist(hba, UFS_EVT_WL_SUSP_ERR, (u32)ret); 10142 hba->clk_gating.is_suspended = false; 10143 ufshcd_release(hba); 10144 } 10145 hba->pm_op_in_progress = false; 10146 return ret; 10147 } 10148 10149 #ifdef CONFIG_PM 10150 static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) 10151 { 10152 int ret; 10153 enum uic_link_state old_link_state = hba->uic_link_state; 10154 10155 hba->pm_op_in_progress = true; 10156 10157 /* 10158 * Call vendor specific resume callback. As these callbacks may access 10159 * vendor specific host controller register space call them when the 10160 * host clocks are ON. 10161 */ 10162 ret = ufshcd_vops_resume(hba, pm_op); 10163 if (ret) 10164 goto out; 10165 10166 /* For DeepSleep, the only supported option is to have the link off */ 10167 WARN_ON(ufshcd_is_ufs_dev_deepsleep(hba) && !ufshcd_is_link_off(hba)); 10168 10169 if (ufshcd_is_link_hibern8(hba)) { 10170 ret = ufshcd_uic_hibern8_exit(hba); 10171 if (!ret) { 10172 ufshcd_set_link_active(hba); 10173 } else { 10174 dev_err(hba->dev, "%s: hibern8 exit failed %d\n", 10175 __func__, ret); 10176 goto vendor_suspend; 10177 } 10178 } else if (ufshcd_is_link_off(hba)) { 10179 /* 10180 * A full initialization of the host and the device is 10181 * required since the link was put to off during suspend. 10182 * Note, in the case of DeepSleep, the device will exit 10183 * DeepSleep due to device reset. 10184 */ 10185 ret = ufshcd_reset_and_restore(hba); 10186 /* 10187 * ufshcd_reset_and_restore() should have already 10188 * set the link state as active 10189 */ 10190 if (ret || !ufshcd_is_link_active(hba)) 10191 goto vendor_suspend; 10192 } 10193 10194 if (!ufshcd_is_ufs_dev_active(hba)) { 10195 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE); 10196 if (ret) 10197 goto set_old_link_state; 10198 ufshcd_set_timestamp_attr(hba); 10199 schedule_delayed_work(&hba->ufs_rtc_update_work, 10200 msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS)); 10201 } 10202 10203 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) 10204 ufshcd_enable_auto_bkops(hba); 10205 else 10206 /* 10207 * If BKOPs operations are urgently needed at this moment then 10208 * keep auto-bkops enabled or else disable it. 10209 */ 10210 ufshcd_bkops_ctrl(hba); 10211 10212 if (hba->ee_usr_mask) 10213 ufshcd_write_ee_control(hba); 10214 10215 if (ufshcd_is_clkscaling_supported(hba)) 10216 ufshcd_clk_scaling_suspend(hba, false); 10217 10218 if (hba->dev_info.b_rpm_dev_flush_capable) { 10219 hba->dev_info.b_rpm_dev_flush_capable = false; 10220 cancel_delayed_work(&hba->rpm_dev_flush_recheck_work); 10221 } 10222 10223 ufshcd_configure_auto_hibern8(hba); 10224 10225 goto out; 10226 10227 set_old_link_state: 10228 ufshcd_link_state_transition(hba, old_link_state, 0); 10229 vendor_suspend: 10230 ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE); 10231 ufshcd_vops_suspend(hba, pm_op, POST_CHANGE); 10232 out: 10233 if (ret) 10234 ufshcd_update_evt_hist(hba, UFS_EVT_WL_RES_ERR, (u32)ret); 10235 hba->clk_gating.is_suspended = false; 10236 ufshcd_release(hba); 10237 hba->pm_op_in_progress = false; 10238 return ret; 10239 } 10240 10241 static int ufshcd_wl_runtime_suspend(struct device *dev) 10242 { 10243 struct scsi_device *sdev = to_scsi_device(dev); 10244 struct ufs_hba *hba; 10245 int ret; 10246 ktime_t start = ktime_get(); 10247 10248 hba = shost_priv(sdev->host); 10249 10250 ret = __ufshcd_wl_suspend(hba, UFS_RUNTIME_PM); 10251 if (ret) 10252 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 10253 10254 trace_ufshcd_wl_runtime_suspend(hba, ret, 10255 ktime_to_us(ktime_sub(ktime_get(), start)), 10256 hba->curr_dev_pwr_mode, hba->uic_link_state); 10257 10258 return ret; 10259 } 10260 10261 static int ufshcd_wl_runtime_resume(struct device *dev) 10262 { 10263 struct scsi_device *sdev = to_scsi_device(dev); 10264 struct ufs_hba *hba; 10265 int ret = 0; 10266 ktime_t start = ktime_get(); 10267 10268 hba = shost_priv(sdev->host); 10269 10270 ret = __ufshcd_wl_resume(hba, UFS_RUNTIME_PM); 10271 if (ret) 10272 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 10273 10274 trace_ufshcd_wl_runtime_resume(hba, ret, 10275 ktime_to_us(ktime_sub(ktime_get(), start)), 10276 hba->curr_dev_pwr_mode, hba->uic_link_state); 10277 10278 return ret; 10279 } 10280 #endif 10281 10282 #ifdef CONFIG_PM_SLEEP 10283 static int ufshcd_wl_suspend(struct device *dev) 10284 { 10285 struct scsi_device *sdev = to_scsi_device(dev); 10286 struct ufs_hba *hba; 10287 int ret = 0; 10288 ktime_t start = ktime_get(); 10289 10290 hba = shost_priv(sdev->host); 10291 down(&hba->host_sem); 10292 hba->system_suspending = true; 10293 10294 if (pm_runtime_suspended(dev)) 10295 goto out; 10296 10297 ret = __ufshcd_wl_suspend(hba, UFS_SYSTEM_PM); 10298 if (ret) { 10299 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 10300 up(&hba->host_sem); 10301 } 10302 10303 out: 10304 if (!ret) 10305 hba->is_sys_suspended = true; 10306 trace_ufshcd_wl_suspend(hba, ret, 10307 ktime_to_us(ktime_sub(ktime_get(), start)), 10308 hba->curr_dev_pwr_mode, hba->uic_link_state); 10309 10310 return ret; 10311 } 10312 10313 static int ufshcd_wl_resume(struct device *dev) 10314 { 10315 struct scsi_device *sdev = to_scsi_device(dev); 10316 struct ufs_hba *hba; 10317 int ret = 0; 10318 ktime_t start = ktime_get(); 10319 10320 hba = shost_priv(sdev->host); 10321 10322 if (pm_runtime_suspended(dev)) 10323 goto out; 10324 10325 ret = __ufshcd_wl_resume(hba, UFS_SYSTEM_PM); 10326 if (ret) 10327 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 10328 out: 10329 trace_ufshcd_wl_resume(hba, ret, 10330 ktime_to_us(ktime_sub(ktime_get(), start)), 10331 hba->curr_dev_pwr_mode, hba->uic_link_state); 10332 if (!ret) 10333 hba->is_sys_suspended = false; 10334 hba->system_suspending = false; 10335 up(&hba->host_sem); 10336 return ret; 10337 } 10338 #endif 10339 10340 /** 10341 * ufshcd_suspend - helper function for suspend operations 10342 * @hba: per adapter instance 10343 * 10344 * This function will put disable irqs, turn off clocks 10345 * and set vreg and hba-vreg in lpm mode. 10346 * 10347 * Return: 0 upon success; < 0 upon failure. 10348 */ 10349 static int ufshcd_suspend(struct ufs_hba *hba) 10350 { 10351 int ret; 10352 10353 if (!hba->is_powered) 10354 return 0; 10355 /* 10356 * Disable the host irq as host controller as there won't be any 10357 * host controller transaction expected till resume. 10358 */ 10359 ufshcd_disable_irq(hba); 10360 ret = ufshcd_setup_clocks(hba, false); 10361 if (ret) { 10362 ufshcd_enable_irq(hba); 10363 goto out; 10364 } 10365 if (ufshcd_is_clkgating_allowed(hba)) { 10366 hba->clk_gating.state = CLKS_OFF; 10367 trace_ufshcd_clk_gating(hba, 10368 hba->clk_gating.state); 10369 } 10370 10371 ufshcd_vreg_set_lpm(hba); 10372 /* Put the host controller in low power mode if possible */ 10373 ufshcd_hba_vreg_set_lpm(hba); 10374 ufshcd_pm_qos_update(hba, false); 10375 out: 10376 if (ret) 10377 ufshcd_update_evt_hist(hba, UFS_EVT_SUSPEND_ERR, (u32)ret); 10378 return ret; 10379 } 10380 10381 #ifdef CONFIG_PM 10382 /** 10383 * ufshcd_resume - helper function for resume operations 10384 * @hba: per adapter instance 10385 * 10386 * This function basically turns on the regulators, clocks and 10387 * irqs of the hba. 10388 * 10389 * Return: 0 for success and non-zero for failure. 10390 */ 10391 static int ufshcd_resume(struct ufs_hba *hba) 10392 { 10393 int ret; 10394 10395 if (!hba->is_powered) 10396 return 0; 10397 10398 ufshcd_hba_vreg_set_hpm(hba); 10399 ret = ufshcd_vreg_set_hpm(hba); 10400 if (ret) 10401 goto out; 10402 10403 /* Make sure clocks are enabled before accessing controller */ 10404 ret = ufshcd_setup_clocks(hba, true); 10405 if (ret) 10406 goto disable_vreg; 10407 10408 /* enable the host irq as host controller would be active soon */ 10409 ufshcd_enable_irq(hba); 10410 10411 goto out; 10412 10413 disable_vreg: 10414 ufshcd_vreg_set_lpm(hba); 10415 out: 10416 if (ret) 10417 ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)ret); 10418 return ret; 10419 } 10420 #endif /* CONFIG_PM */ 10421 10422 #ifdef CONFIG_PM_SLEEP 10423 /** 10424 * ufshcd_system_suspend - system suspend callback 10425 * @dev: Device associated with the UFS controller. 10426 * 10427 * Executed before putting the system into a sleep state in which the contents 10428 * of main memory are preserved. 10429 * 10430 * Return: 0 for success and non-zero for failure. 10431 */ 10432 int ufshcd_system_suspend(struct device *dev) 10433 { 10434 struct ufs_hba *hba = dev_get_drvdata(dev); 10435 int ret = 0; 10436 ktime_t start = ktime_get(); 10437 10438 if (pm_runtime_suspended(hba->dev)) 10439 goto out; 10440 10441 ret = ufshcd_suspend(hba); 10442 out: 10443 trace_ufshcd_system_suspend(hba, ret, 10444 ktime_to_us(ktime_sub(ktime_get(), start)), 10445 hba->curr_dev_pwr_mode, hba->uic_link_state); 10446 return ret; 10447 } 10448 EXPORT_SYMBOL(ufshcd_system_suspend); 10449 10450 /** 10451 * ufshcd_system_resume - system resume callback 10452 * @dev: Device associated with the UFS controller. 10453 * 10454 * Executed after waking the system up from a sleep state in which the contents 10455 * of main memory were preserved. 10456 * 10457 * Return: 0 for success and non-zero for failure. 10458 */ 10459 int ufshcd_system_resume(struct device *dev) 10460 { 10461 struct ufs_hba *hba = dev_get_drvdata(dev); 10462 ktime_t start = ktime_get(); 10463 int ret = 0; 10464 10465 if (pm_runtime_suspended(hba->dev)) 10466 goto out; 10467 10468 ret = ufshcd_resume(hba); 10469 10470 out: 10471 trace_ufshcd_system_resume(hba, ret, 10472 ktime_to_us(ktime_sub(ktime_get(), start)), 10473 hba->curr_dev_pwr_mode, hba->uic_link_state); 10474 10475 return ret; 10476 } 10477 EXPORT_SYMBOL(ufshcd_system_resume); 10478 #endif /* CONFIG_PM_SLEEP */ 10479 10480 #ifdef CONFIG_PM 10481 /** 10482 * ufshcd_runtime_suspend - runtime suspend callback 10483 * @dev: Device associated with the UFS controller. 10484 * 10485 * Check the description of ufshcd_suspend() function for more details. 10486 * 10487 * Return: 0 for success and non-zero for failure. 10488 */ 10489 int ufshcd_runtime_suspend(struct device *dev) 10490 { 10491 struct ufs_hba *hba = dev_get_drvdata(dev); 10492 int ret; 10493 ktime_t start = ktime_get(); 10494 10495 ret = ufshcd_suspend(hba); 10496 10497 trace_ufshcd_runtime_suspend(hba, ret, 10498 ktime_to_us(ktime_sub(ktime_get(), start)), 10499 hba->curr_dev_pwr_mode, hba->uic_link_state); 10500 return ret; 10501 } 10502 EXPORT_SYMBOL(ufshcd_runtime_suspend); 10503 10504 /** 10505 * ufshcd_runtime_resume - runtime resume routine 10506 * @dev: Device associated with the UFS controller. 10507 * 10508 * This function basically brings controller 10509 * to active state. Following operations are done in this function: 10510 * 10511 * 1. Turn on all the controller related clocks 10512 * 2. Turn ON VCC rail 10513 * 10514 * Return: 0 upon success; < 0 upon failure. 10515 */ 10516 int ufshcd_runtime_resume(struct device *dev) 10517 { 10518 struct ufs_hba *hba = dev_get_drvdata(dev); 10519 int ret; 10520 ktime_t start = ktime_get(); 10521 10522 ret = ufshcd_resume(hba); 10523 10524 trace_ufshcd_runtime_resume(hba, ret, 10525 ktime_to_us(ktime_sub(ktime_get(), start)), 10526 hba->curr_dev_pwr_mode, hba->uic_link_state); 10527 return ret; 10528 } 10529 EXPORT_SYMBOL(ufshcd_runtime_resume); 10530 #endif /* CONFIG_PM */ 10531 10532 static void ufshcd_wl_shutdown(struct device *dev) 10533 { 10534 struct scsi_device *sdev = to_scsi_device(dev); 10535 struct ufs_hba *hba = shost_priv(sdev->host); 10536 10537 down(&hba->host_sem); 10538 hba->shutting_down = true; 10539 up(&hba->host_sem); 10540 10541 /* Turn on everything while shutting down */ 10542 ufshcd_rpm_get_sync(hba); 10543 scsi_device_quiesce(sdev); 10544 shost_for_each_device(sdev, hba->host) { 10545 if (sdev == hba->ufs_device_wlun) 10546 continue; 10547 mutex_lock(&sdev->state_mutex); 10548 scsi_device_set_state(sdev, SDEV_OFFLINE); 10549 mutex_unlock(&sdev->state_mutex); 10550 } 10551 __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM); 10552 10553 /* 10554 * Next, turn off the UFS controller and the UFS regulators. Disable 10555 * clocks. 10556 */ 10557 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba)) 10558 ufshcd_suspend(hba); 10559 10560 hba->is_powered = false; 10561 } 10562 10563 /** 10564 * ufshcd_remove - de-allocate SCSI host and host memory space 10565 * data structure memory 10566 * @hba: per adapter instance 10567 */ 10568 void ufshcd_remove(struct ufs_hba *hba) 10569 { 10570 if (hba->ufs_device_wlun) 10571 ufshcd_rpm_get_sync(hba); 10572 ufs_hwmon_remove(hba); 10573 ufs_bsg_remove(hba); 10574 ufs_rpmb_remove(hba); 10575 ufs_sysfs_remove_nodes(hba->dev); 10576 cancel_delayed_work_sync(&hba->ufs_rtc_update_work); 10577 blk_mq_destroy_queue(hba->tmf_queue); 10578 blk_put_queue(hba->tmf_queue); 10579 blk_mq_free_tag_set(&hba->tmf_tag_set); 10580 if (hba->scsi_host_added) 10581 scsi_remove_host(hba->host); 10582 /* disable interrupts */ 10583 ufshcd_disable_intr(hba, hba->intr_mask); 10584 ufshcd_hba_stop(hba); 10585 ufshcd_hba_exit(hba); 10586 } 10587 EXPORT_SYMBOL_GPL(ufshcd_remove); 10588 10589 #ifdef CONFIG_PM_SLEEP 10590 int ufshcd_system_freeze(struct device *dev) 10591 { 10592 10593 return ufshcd_system_suspend(dev); 10594 10595 } 10596 EXPORT_SYMBOL_GPL(ufshcd_system_freeze); 10597 10598 int ufshcd_system_restore(struct device *dev) 10599 { 10600 10601 struct ufs_hba *hba = dev_get_drvdata(dev); 10602 int ret; 10603 10604 ret = ufshcd_system_resume(dev); 10605 if (ret) 10606 return ret; 10607 10608 /* Configure UTRL and UTMRL base address registers */ 10609 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr), 10610 REG_UTP_TRANSFER_REQ_LIST_BASE_L); 10611 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr), 10612 REG_UTP_TRANSFER_REQ_LIST_BASE_H); 10613 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr), 10614 REG_UTP_TASK_REQ_LIST_BASE_L); 10615 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr), 10616 REG_UTP_TASK_REQ_LIST_BASE_H); 10617 /* 10618 * Make sure that UTRL and UTMRL base address registers 10619 * are updated with the latest queue addresses. Only after 10620 * updating these addresses, we can queue the new commands. 10621 */ 10622 ufshcd_readl(hba, REG_UTP_TASK_REQ_LIST_BASE_H); 10623 10624 return 0; 10625 10626 } 10627 EXPORT_SYMBOL_GPL(ufshcd_system_restore); 10628 10629 int ufshcd_system_thaw(struct device *dev) 10630 { 10631 return ufshcd_system_resume(dev); 10632 } 10633 EXPORT_SYMBOL_GPL(ufshcd_system_thaw); 10634 #endif /* CONFIG_PM_SLEEP */ 10635 10636 /** 10637 * ufshcd_set_dma_mask - Set dma mask based on the controller 10638 * addressing capability 10639 * @hba: per adapter instance 10640 * 10641 * Return: 0 for success, non-zero for failure. 10642 */ 10643 static int ufshcd_set_dma_mask(struct ufs_hba *hba) 10644 { 10645 if (hba->vops && hba->vops->set_dma_mask) 10646 return hba->vops->set_dma_mask(hba); 10647 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) { 10648 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64))) 10649 return 0; 10650 } 10651 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32)); 10652 } 10653 10654 /** 10655 * ufshcd_devres_release - devres cleanup handler, invoked during release of 10656 * hba->dev 10657 * @host: pointer to SCSI host 10658 */ 10659 static void ufshcd_devres_release(void *host) 10660 { 10661 scsi_host_put(host); 10662 } 10663 10664 /** 10665 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA) 10666 * @dev: pointer to device handle 10667 * @hba_handle: driver private handle 10668 * 10669 * Return: 0 on success, non-zero value on failure. 10670 * 10671 * NOTE: There is no corresponding ufshcd_dealloc_host() because this function 10672 * keeps track of its allocations using devres and deallocates everything on 10673 * device removal automatically. 10674 */ 10675 int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle) 10676 { 10677 struct Scsi_Host *host; 10678 struct ufs_hba *hba; 10679 int err = 0; 10680 10681 if (!dev) { 10682 dev_err(dev, 10683 "Invalid memory reference for dev is NULL\n"); 10684 err = -ENODEV; 10685 goto out_error; 10686 } 10687 10688 host = scsi_host_alloc(&ufshcd_driver_template, 10689 sizeof(struct ufs_hba)); 10690 if (!host) { 10691 dev_err(dev, "scsi_host_alloc failed\n"); 10692 err = -ENOMEM; 10693 goto out_error; 10694 } 10695 10696 err = devm_add_action_or_reset(dev, ufshcd_devres_release, 10697 host); 10698 if (err) 10699 return err; 10700 10701 host->nr_maps = HCTX_TYPE_POLL + 1; 10702 hba = shost_priv(host); 10703 hba->host = host; 10704 hba->dev = dev; 10705 hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL; 10706 hba->nop_out_timeout = NOP_OUT_TIMEOUT; 10707 ufshcd_set_sg_entry_size(hba, sizeof(struct ufshcd_sg_entry)); 10708 INIT_LIST_HEAD(&hba->clk_list_head); 10709 spin_lock_init(&hba->outstanding_lock); 10710 10711 *hba_handle = hba; 10712 10713 out_error: 10714 return err; 10715 } 10716 EXPORT_SYMBOL(ufshcd_alloc_host); 10717 10718 /* This function exists because blk_mq_alloc_tag_set() requires this. */ 10719 static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx, 10720 const struct blk_mq_queue_data *qd) 10721 { 10722 WARN_ON_ONCE(true); 10723 return BLK_STS_NOTSUPP; 10724 } 10725 10726 static const struct blk_mq_ops ufshcd_tmf_ops = { 10727 .queue_rq = ufshcd_queue_tmf, 10728 }; 10729 10730 static int ufshcd_add_scsi_host(struct ufs_hba *hba) 10731 { 10732 int err; 10733 10734 WARN_ON_ONCE(!hba->host->can_queue); 10735 WARN_ON_ONCE(!hba->host->cmd_per_lun); 10736 10737 if (is_mcq_supported(hba)) { 10738 ufshcd_mcq_enable(hba); 10739 err = ufshcd_alloc_mcq(hba); 10740 if (err) { 10741 /* Continue with SDB mode */ 10742 ufshcd_mcq_disable(hba); 10743 use_mcq_mode = false; 10744 dev_err(hba->dev, "MCQ mode is disabled, err=%d\n", 10745 err); 10746 } 10747 } 10748 if (!is_mcq_supported(hba) && !hba->lsdb_sup) { 10749 dev_err(hba->dev, 10750 "%s: failed to initialize (legacy doorbell mode not supported)\n", 10751 __func__); 10752 return -EINVAL; 10753 } 10754 10755 err = scsi_add_host(hba->host, hba->dev); 10756 if (err) { 10757 dev_err(hba->dev, "scsi_add_host failed\n"); 10758 return err; 10759 } 10760 hba->scsi_host_added = true; 10761 10762 hba->tmf_tag_set = (struct blk_mq_tag_set) { 10763 .nr_hw_queues = 1, 10764 .queue_depth = hba->nutmrs, 10765 .ops = &ufshcd_tmf_ops, 10766 }; 10767 err = blk_mq_alloc_tag_set(&hba->tmf_tag_set); 10768 if (err < 0) 10769 goto remove_scsi_host; 10770 hba->tmf_queue = blk_mq_alloc_queue(&hba->tmf_tag_set, NULL, NULL); 10771 if (IS_ERR(hba->tmf_queue)) { 10772 err = PTR_ERR(hba->tmf_queue); 10773 goto free_tmf_tag_set; 10774 } 10775 hba->tmf_rqs = devm_kcalloc(hba->dev, hba->nutmrs, 10776 sizeof(*hba->tmf_rqs), GFP_KERNEL); 10777 if (!hba->tmf_rqs) { 10778 err = -ENOMEM; 10779 goto free_tmf_queue; 10780 } 10781 10782 return 0; 10783 10784 free_tmf_queue: 10785 blk_mq_destroy_queue(hba->tmf_queue); 10786 blk_put_queue(hba->tmf_queue); 10787 10788 free_tmf_tag_set: 10789 blk_mq_free_tag_set(&hba->tmf_tag_set); 10790 10791 remove_scsi_host: 10792 if (hba->scsi_host_added) 10793 scsi_remove_host(hba->host); 10794 10795 return err; 10796 } 10797 10798 /** 10799 * ufshcd_init - Driver initialization routine 10800 * @hba: per-adapter instance 10801 * @mmio_base: base register address 10802 * @irq: Interrupt line of device 10803 * 10804 * Return: 0 on success; < 0 on failure. 10805 */ 10806 int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) 10807 { 10808 int err; 10809 struct Scsi_Host *host = hba->host; 10810 struct device *dev = hba->dev; 10811 10812 /* 10813 * dev_set_drvdata() must be called before any callbacks are registered 10814 * that use dev_get_drvdata() (frequency scaling, clock scaling, hwmon, 10815 * sysfs). 10816 */ 10817 dev_set_drvdata(dev, hba); 10818 10819 if (!mmio_base) { 10820 dev_err(hba->dev, 10821 "Invalid memory reference for mmio_base is NULL\n"); 10822 err = -ENODEV; 10823 goto out_error; 10824 } 10825 10826 hba->mmio_base = mmio_base; 10827 hba->irq = irq; 10828 hba->vps = &ufs_hba_vps; 10829 10830 /* 10831 * Initialize clk_gating.lock early since it is being used in 10832 * ufshcd_setup_clocks() 10833 */ 10834 spin_lock_init(&hba->clk_gating.lock); 10835 10836 /* Initialize mutex for PM QoS request synchronization */ 10837 mutex_init(&hba->pm_qos_mutex); 10838 10839 /* 10840 * Set the default power management level for runtime and system PM. 10841 * Host controller drivers can override them in their 10842 * 'ufs_hba_variant_ops::init' callback. 10843 * 10844 * Default power saving mode is to keep UFS link in Hibern8 state 10845 * and UFS device in sleep state. 10846 */ 10847 hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( 10848 UFS_SLEEP_PWR_MODE, 10849 UIC_LINK_HIBERN8_STATE); 10850 hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( 10851 UFS_SLEEP_PWR_MODE, 10852 UIC_LINK_HIBERN8_STATE); 10853 10854 /* 10855 * Most ufs devices require 1ms delay after vcc is powered off before 10856 * it can be powered on again. Set the default to 2ms. The platform 10857 * drivers can override this setting as needed. 10858 */ 10859 hba->vcc_off_delay_us = 2000; 10860 10861 err = ufshcd_hba_init(hba); 10862 if (err) 10863 goto out_error; 10864 10865 /* Read capabilities registers */ 10866 err = ufshcd_hba_capabilities(hba); 10867 if (err) 10868 goto out_disable; 10869 10870 /* Get UFS version supported by the controller */ 10871 hba->ufs_version = ufshcd_get_ufs_version(hba); 10872 10873 /* Get Interrupt bit mask per version */ 10874 hba->intr_mask = ufshcd_get_intr_mask(hba); 10875 10876 err = ufshcd_set_dma_mask(hba); 10877 if (err) { 10878 dev_err(hba->dev, "set dma mask failed\n"); 10879 goto out_disable; 10880 } 10881 10882 /* Allocate memory for host memory space */ 10883 err = ufshcd_memory_alloc(hba); 10884 if (err) { 10885 dev_err(hba->dev, "Memory allocation failed\n"); 10886 goto out_disable; 10887 } 10888 10889 /* Configure LRB */ 10890 ufshcd_host_memory_configure(hba); 10891 10892 host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; 10893 /* 10894 * Set the queue depth for WLUNs. ufs_get_device_desc() will increase 10895 * host->cmd_per_lun to a larger value. 10896 */ 10897 host->cmd_per_lun = 1; 10898 host->max_id = UFSHCD_MAX_ID; 10899 host->max_lun = UFS_MAX_LUNS; 10900 host->max_channel = UFSHCD_MAX_CHANNEL; 10901 host->unique_id = host->host_no; 10902 host->max_cmd_len = UFS_CDB_SIZE; 10903 host->queuecommand_may_block = !!(hba->caps & UFSHCD_CAP_CLK_GATING); 10904 10905 /* Use default RPM delay if host not set */ 10906 if (host->rpm_autosuspend_delay == 0) 10907 host->rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS; 10908 10909 hba->max_pwr_info.is_valid = false; 10910 10911 /* Initialize work queues */ 10912 hba->eh_wq = alloc_ordered_workqueue("ufs_eh_wq_%d", WQ_MEM_RECLAIM, 10913 hba->host->host_no); 10914 if (!hba->eh_wq) { 10915 dev_err(hba->dev, "%s: failed to create eh workqueue\n", 10916 __func__); 10917 err = -ENOMEM; 10918 goto out_disable; 10919 } 10920 INIT_WORK(&hba->eh_work, ufshcd_err_handler); 10921 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler); 10922 10923 sema_init(&hba->host_sem, 1); 10924 10925 /* Initialize UIC command mutex */ 10926 mutex_init(&hba->uic_cmd_mutex); 10927 10928 /* Initialize mutex for device management commands */ 10929 mutex_init(&hba->dev_cmd.lock); 10930 10931 /* Initialize mutex for exception event control */ 10932 mutex_init(&hba->ee_ctrl_mutex); 10933 10934 mutex_init(&hba->wb_mutex); 10935 10936 init_rwsem(&hba->clk_scaling_lock); 10937 10938 ufshcd_init_clk_gating(hba); 10939 10940 ufshcd_init_clk_scaling(hba); 10941 10942 /* 10943 * In order to avoid any spurious interrupt immediately after 10944 * registering UFS controller interrupt handler, clear any pending UFS 10945 * interrupt status and disable all the UFS interrupts. 10946 */ 10947 ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS), 10948 REG_INTERRUPT_STATUS); 10949 ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE); 10950 /* 10951 * Make sure that UFS interrupts are disabled and any pending interrupt 10952 * status is cleared before registering UFS interrupt handler. 10953 */ 10954 ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 10955 10956 /* IRQ registration */ 10957 err = devm_request_threaded_irq(dev, irq, ufshcd_intr, ufshcd_threaded_intr, 10958 IRQF_ONESHOT | IRQF_SHARED, UFSHCD, hba); 10959 if (err) { 10960 dev_err(hba->dev, "request irq failed\n"); 10961 goto out_disable; 10962 } else { 10963 hba->is_irq_enabled = true; 10964 } 10965 10966 /* Reset the attached device */ 10967 ufshcd_device_reset(hba); 10968 10969 ufshcd_init_crypto(hba); 10970 10971 /* Host controller enable */ 10972 err = ufshcd_hba_enable(hba); 10973 if (err) { 10974 dev_err(hba->dev, "Host controller enable failed\n"); 10975 ufshcd_print_evt_hist(hba); 10976 ufshcd_print_host_state(hba); 10977 goto out_disable; 10978 } 10979 10980 INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work, ufshcd_rpm_dev_flush_recheck_work); 10981 INIT_DELAYED_WORK(&hba->ufs_rtc_update_work, ufshcd_rtc_work); 10982 10983 /* Set the default auto-hiberate idle timer value to 150 ms */ 10984 if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) { 10985 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) | 10986 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3); 10987 } 10988 10989 err = ufshcd_add_scsi_host(hba); 10990 if (err) 10991 goto out_disable; 10992 10993 /* Hold auto suspend until async scan completes */ 10994 pm_runtime_get_sync(dev); 10995 10996 /* 10997 * We are assuming that device wasn't put in sleep/power-down 10998 * state exclusively during the boot stage before kernel. 10999 * This assumption helps avoid doing link startup twice during 11000 * ufshcd_probe_hba(). 11001 */ 11002 ufshcd_set_ufs_dev_active(hba); 11003 11004 /* Initialize hba, detect and initialize UFS device */ 11005 ktime_t probe_start = ktime_get(); 11006 11007 hba->ufshcd_state = UFSHCD_STATE_RESET; 11008 11009 err = ufshcd_link_startup(hba); 11010 if (err) 11011 goto out_disable; 11012 11013 if (hba->mcq_enabled) 11014 ufshcd_config_mcq(hba); 11015 11016 if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION) 11017 goto initialized; 11018 11019 /* Debug counters initialization */ 11020 ufshcd_clear_dbg_ufs_stats(hba); 11021 11022 /* UniPro link is active now */ 11023 ufshcd_set_link_active(hba); 11024 11025 /* Verify device initialization by sending NOP OUT UPIU */ 11026 err = ufshcd_verify_dev_init(hba); 11027 if (err) 11028 goto out_disable; 11029 11030 /* Initiate UFS initialization, and waiting until completion */ 11031 err = ufshcd_complete_dev_init(hba); 11032 if (err) 11033 goto out_disable; 11034 11035 err = ufshcd_device_params_init(hba); 11036 if (err) 11037 goto out_disable; 11038 11039 err = ufshcd_post_device_init(hba); 11040 11041 initialized: 11042 ufshcd_process_probe_result(hba, probe_start, err); 11043 if (err) 11044 goto out_disable; 11045 11046 ufs_sysfs_add_nodes(hba->dev); 11047 async_schedule(ufshcd_async_scan, hba); 11048 11049 device_enable_async_suspend(dev); 11050 ufshcd_pm_qos_init(hba); 11051 return 0; 11052 11053 out_disable: 11054 hba->is_irq_enabled = false; 11055 ufshcd_hba_exit(hba); 11056 out_error: 11057 return err > 0 ? -EIO : err; 11058 } 11059 EXPORT_SYMBOL_GPL(ufshcd_init); 11060 11061 void ufshcd_resume_complete(struct device *dev) 11062 { 11063 struct ufs_hba *hba = dev_get_drvdata(dev); 11064 11065 if (hba->complete_put) { 11066 ufshcd_rpm_put(hba); 11067 hba->complete_put = false; 11068 } 11069 } 11070 EXPORT_SYMBOL_GPL(ufshcd_resume_complete); 11071 11072 static bool ufshcd_rpm_ok_for_spm(struct ufs_hba *hba) 11073 { 11074 struct device *dev = &hba->ufs_device_wlun->sdev_gendev; 11075 enum ufs_dev_pwr_mode dev_pwr_mode; 11076 enum uic_link_state link_state; 11077 unsigned long flags; 11078 bool res; 11079 11080 spin_lock_irqsave(&dev->power.lock, flags); 11081 dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl); 11082 link_state = ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl); 11083 res = pm_runtime_suspended(dev) && 11084 hba->curr_dev_pwr_mode == dev_pwr_mode && 11085 hba->uic_link_state == link_state && 11086 !hba->dev_info.b_rpm_dev_flush_capable; 11087 spin_unlock_irqrestore(&dev->power.lock, flags); 11088 11089 return res; 11090 } 11091 11092 int __ufshcd_suspend_prepare(struct device *dev, bool rpm_ok_for_spm) 11093 { 11094 struct ufs_hba *hba = dev_get_drvdata(dev); 11095 int ret; 11096 11097 /* 11098 * SCSI assumes that runtime-pm and system-pm for scsi drivers 11099 * are same. And it doesn't wake up the device for system-suspend 11100 * if it's runtime suspended. But ufs doesn't follow that. 11101 * Refer ufshcd_resume_complete() 11102 */ 11103 if (hba->ufs_device_wlun) { 11104 /* Prevent runtime suspend */ 11105 ufshcd_rpm_get_noresume(hba); 11106 /* 11107 * Check if already runtime suspended in same state as system 11108 * suspend would be. 11109 */ 11110 if (!rpm_ok_for_spm || !ufshcd_rpm_ok_for_spm(hba)) { 11111 /* RPM state is not ok for SPM, so runtime resume */ 11112 ret = ufshcd_rpm_resume(hba); 11113 if (ret < 0 && ret != -EACCES) { 11114 ufshcd_rpm_put(hba); 11115 return ret; 11116 } 11117 } 11118 hba->complete_put = true; 11119 } 11120 return 0; 11121 } 11122 EXPORT_SYMBOL_GPL(__ufshcd_suspend_prepare); 11123 11124 int ufshcd_suspend_prepare(struct device *dev) 11125 { 11126 return __ufshcd_suspend_prepare(dev, true); 11127 } 11128 EXPORT_SYMBOL_GPL(ufshcd_suspend_prepare); 11129 11130 #ifdef CONFIG_PM_SLEEP 11131 static int ufshcd_wl_poweroff(struct device *dev) 11132 { 11133 struct scsi_device *sdev = to_scsi_device(dev); 11134 struct ufs_hba *hba = shost_priv(sdev->host); 11135 11136 __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM); 11137 return 0; 11138 } 11139 #endif 11140 11141 static int ufshcd_wl_probe(struct device *dev) 11142 { 11143 struct scsi_device *sdev = to_scsi_device(dev); 11144 11145 if (!is_device_wlun(sdev)) 11146 return -ENODEV; 11147 11148 blk_pm_runtime_init(sdev->request_queue, dev); 11149 pm_runtime_set_autosuspend_delay(dev, 0); 11150 pm_runtime_allow(dev); 11151 11152 return 0; 11153 } 11154 11155 static int ufshcd_wl_remove(struct device *dev) 11156 { 11157 pm_runtime_forbid(dev); 11158 return 0; 11159 } 11160 11161 static const struct dev_pm_ops ufshcd_wl_pm_ops = { 11162 #ifdef CONFIG_PM_SLEEP 11163 .suspend = ufshcd_wl_suspend, 11164 .resume = ufshcd_wl_resume, 11165 .freeze = ufshcd_wl_suspend, 11166 .thaw = ufshcd_wl_resume, 11167 .poweroff = ufshcd_wl_poweroff, 11168 .restore = ufshcd_wl_resume, 11169 #endif 11170 SET_RUNTIME_PM_OPS(ufshcd_wl_runtime_suspend, ufshcd_wl_runtime_resume, NULL) 11171 }; 11172 11173 static void ufshcd_check_header_layout(void) 11174 { 11175 /* 11176 * gcc compilers before version 10 cannot do constant-folding for 11177 * sub-byte bitfields. Hence skip the layout checks for gcc 9 and 11178 * before. 11179 */ 11180 if (IS_ENABLED(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 100000) 11181 return; 11182 11183 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11184 .cci = 3})[0] != 3); 11185 11186 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11187 .ehs_length = 2})[1] != 2); 11188 11189 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11190 .enable_crypto = 1})[2] 11191 != 0x80); 11192 11193 BUILD_BUG_ON((((u8 *)&(struct request_desc_header){ 11194 .command_type = 5, 11195 .data_direction = 3, 11196 .interrupt = 1, 11197 })[3]) != ((5 << 4) | (3 << 1) | 1)); 11198 11199 BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){ 11200 .dunl = cpu_to_le32(0xdeadbeef)})[1] != 11201 cpu_to_le32(0xdeadbeef)); 11202 11203 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11204 .ocs = 4})[8] != 4); 11205 11206 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11207 .cds = 5})[9] != 5); 11208 11209 BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){ 11210 .dunu = cpu_to_le32(0xbadcafe)})[3] != 11211 cpu_to_le32(0xbadcafe)); 11212 11213 BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){ 11214 .iid = 0xf })[4] != 0xf0); 11215 11216 BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){ 11217 .command_set_type = 0xf })[4] != 0xf); 11218 } 11219 11220 /* 11221 * ufs_dev_wlun_template - describes ufs device wlun 11222 * ufs-device wlun - used to send pm commands 11223 * All luns are consumers of ufs-device wlun. 11224 * 11225 * Currently, no sd driver is present for wluns. 11226 * Hence the no specific pm operations are performed. 11227 * With ufs design, SSU should be sent to ufs-device wlun. 11228 * Hence register a scsi driver for ufs wluns only. 11229 */ 11230 static struct scsi_driver ufs_dev_wlun_template = { 11231 .gendrv = { 11232 .name = "ufs_device_wlun", 11233 .probe = ufshcd_wl_probe, 11234 .remove = ufshcd_wl_remove, 11235 .pm = &ufshcd_wl_pm_ops, 11236 .shutdown = ufshcd_wl_shutdown, 11237 }, 11238 }; 11239 11240 static int __init ufshcd_core_init(void) 11241 { 11242 int ret; 11243 11244 ufshcd_check_header_layout(); 11245 11246 ufs_debugfs_init(); 11247 11248 ret = scsi_register_driver(&ufs_dev_wlun_template.gendrv); 11249 if (ret) 11250 ufs_debugfs_exit(); 11251 return ret; 11252 } 11253 11254 static void __exit ufshcd_core_exit(void) 11255 { 11256 ufs_debugfs_exit(); 11257 scsi_unregister_driver(&ufs_dev_wlun_template.gendrv); 11258 } 11259 11260 module_init(ufshcd_core_init); 11261 module_exit(ufshcd_core_exit); 11262 11263 MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>"); 11264 MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>"); 11265 MODULE_DESCRIPTION("Generic UFS host controller driver Core"); 11266 MODULE_SOFTDEP("pre: governor_simpleondemand"); 11267 MODULE_LICENSE("GPL"); 11268